diff --git a/.codeclimate.yml b/.codeclimate.yml deleted file mode 100644 index 8db1384401c781e28342727ad4506f12658a9bb6..0000000000000000000000000000000000000000 --- a/.codeclimate.yml +++ /dev/null @@ -1,11 +0,0 @@ -version: "2" -exclude_pattern: - - client/ - - cmd/weaviate-server/main.go - - # exclude all but the configure_weaviate.go file in restapi/, - # given that the other files are automatically generated. - - restapi/ - - !restapi/configure_weaviate* - - - test/ diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index eacc4ea7dd0fd13c3c0bf6cbb0e9c7fba1443cb4..0000000000000000000000000000000000000000 --- a/.dockerignore +++ /dev/null @@ -1,39 +0,0 @@ -.git -Dockerfile.* -docker-compose.yml -data* -logs/ -snapshots/ -snapshots-fs/ -snapshots-node2/ -snapshots-s3/ -snapshots-gcs/ -backups/ -backups-fs/ -backups-node2/ -backups-s3/ -backups-gcs/ -test/ -tools/ -**/*.fvecs # sift data -**/*.png # profiling -**/*.out -**/*.csv -**/*.md -**/*_test.go -**/.DS_Store -**/testdata/ -coverage-integration.txt -coverage-unit.txt - -# wikipedia dataset when downloaded according to docs -weaviate-wikipedia-* -# the local path when extracting the wiki dataset -var/weaviate/ - -# IDE files -.idea -.vscode/ - -# ignore archives, they are most likely part of some debugging and never part of actual code -*.tar.gz diff --git a/.gitattributes b/.gitattributes deleted file mode 100644 index a6344aac8c09253b3b630fb776ae94478aa0275b..0000000000000000000000000000000000000000 --- a/.gitattributes +++ /dev/null @@ -1,35 +0,0 @@ -*.7z filter=lfs diff=lfs merge=lfs -text -*.arrow filter=lfs diff=lfs merge=lfs -text -*.bin filter=lfs diff=lfs merge=lfs -text -*.bz2 filter=lfs diff=lfs merge=lfs -text -*.ckpt filter=lfs diff=lfs merge=lfs -text -*.ftz filter=lfs diff=lfs merge=lfs -text -*.gz filter=lfs diff=lfs merge=lfs -text -*.h5 filter=lfs diff=lfs merge=lfs -text -*.joblib filter=lfs diff=lfs merge=lfs -text -*.lfs.* filter=lfs diff=lfs merge=lfs -text -*.mlmodel filter=lfs diff=lfs merge=lfs -text -*.model filter=lfs diff=lfs merge=lfs -text -*.msgpack filter=lfs diff=lfs merge=lfs -text -*.npy filter=lfs diff=lfs merge=lfs -text -*.npz filter=lfs diff=lfs merge=lfs -text -*.onnx filter=lfs diff=lfs merge=lfs -text -*.ot filter=lfs diff=lfs merge=lfs -text -*.parquet filter=lfs diff=lfs merge=lfs -text -*.pb filter=lfs diff=lfs merge=lfs -text -*.pickle filter=lfs diff=lfs merge=lfs -text -*.pkl filter=lfs diff=lfs merge=lfs -text -*.pt filter=lfs diff=lfs merge=lfs -text -*.pth filter=lfs diff=lfs merge=lfs -text -*.rar filter=lfs diff=lfs merge=lfs -text -*.safetensors filter=lfs diff=lfs merge=lfs -text -saved_model/**/* filter=lfs diff=lfs merge=lfs -text -*.tar.* filter=lfs diff=lfs merge=lfs -text -*.tar filter=lfs diff=lfs merge=lfs -text -*.tflite filter=lfs diff=lfs merge=lfs -text -*.tgz filter=lfs diff=lfs merge=lfs -text -*.wasm filter=lfs diff=lfs merge=lfs -text -*.xz filter=lfs diff=lfs merge=lfs -text -*.zip filter=lfs diff=lfs merge=lfs -text -*.zst filter=lfs diff=lfs merge=lfs -text -*tfevents* filter=lfs diff=lfs merge=lfs -text diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS deleted file mode 100644 index f361689e38e4ddad63366bbefb471b2056c5bd81..0000000000000000000000000000000000000000 --- a/.github/CODEOWNERS +++ /dev/null @@ -1,3 +0,0 @@ -# Ci related folders -/.github/ @weaviate/core -/ci/ @weaviate/core diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml deleted file mode 100644 index 63a34738969e3dc4937cef9cd59bf032d45ae16f..0000000000000000000000000000000000000000 --- a/.github/ISSUE_TEMPLATE/config.yml +++ /dev/null @@ -1,9 +0,0 @@ -blank_issues_enabled: true -contact_links: - - name: Need Help? Go to the community forum - url: https://forum.weaviate.io - about: "Community-powered knowledge repository: search, ask questions, give feedback" - - - name: Want to chat? Go to the community slack - url: https://weaviate.io/slack - about: "Chat with the community about issues, questions, networking and feedback" diff --git a/.github/ISSUE_TEMPLATE/create_issue.yml b/.github/ISSUE_TEMPLATE/create_issue.yml deleted file mode 100644 index c7b9fbdefd2ce55961dc4e0371679b9011057c65..0000000000000000000000000000000000000000 --- a/.github/ISSUE_TEMPLATE/create_issue.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: Found a bug? -description: Report your bug here. -labels: ["bug", "triage"] -body: - - - type: markdown - attributes: - value: | - #### Before you get started - * Check to make sure someone hasn't already opened a similar [issue](https://github.com/weaviate/weaviate-io/issues). - * Check this example of a [good bug report](https://github.com/weaviate/weaviate/issues/3762). - * Read the [Contributor Guide](https://weaviate.io/developers/contributor-guide) and [Code of Conduct](https://weaviate.io/service/code-of-conduct). - - - type: textarea - id: how_to_reproduce - attributes: - label: How to reproduce this bug? - description: Specify the steps here in order to reproduce this bug. - validations: - required: true - - - type: textarea - id: expected_behavior - attributes: - label: What is the expected behavior? - validations: - required: true - - - type: textarea - id: actual_behavior - attributes: - label: What is the actual behavior? - validations: - required: true - - - type: textarea - id: suporting_information - attributes: - label: Supporting information - description: Please, paste any logs, context information (client version? environment variables?) or other details in here. - validations: - required: false - - - type: input - id: server_version - attributes: - label: Server Version - description: What Weaviate Version are you running? - validations: - required: true - - - type: checkboxes - id: terms - attributes: - label: Code of Conduct - description: This project has a Code of Conduct. All participants are expected to understand and follow the CoC. - options: - - label: I have read and agree to the Weaviate's [Contributor Guide](https://weaviate.io/developers/contributor-guide) and [Code of Conduct](https://weaviate.io/service/code-of-conduct) - required: true diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml deleted file mode 100644 index 5dc4b972f1ffe53f1589803b573333f2bb1b1d2e..0000000000000000000000000000000000000000 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Missing a Feature? -description: Suggest it here. -labels: ["feature request", "triage"] -body: - - type: markdown - attributes: - value: | - #### Before you get started - * Is this feature already in our [roadmap](https://weaviate.io/developers/weaviate/roadmap)? - * Check to make sure someone hasn't already [requested this feature](https://github.com/weaviate/weaviate/issues?q=is:issue+label:"feature+request"). if that's the case, give it a thumbs up 👍 - * Read the [Contributor Guide](https://weaviate.io/developers/contributor-guide) and [Code of Conduct](https://weaviate.io/service/code-of-conduct). - - - type: textarea - id: feature_request_body - attributes: - label: Describe your feature request - description: | - - Describe here your requested feature. Provide all necessary context, use cases and the reasoning on why this feature is a good one. - validations: - required: true - - - type: checkboxes - id: terms - attributes: - label: Code of Conduct - description: This project has a Code of Conduct that all participants are expected to understand and follow. - options: - - label: I have read and agree to the Weaviate's [Contributor Guide](https://weaviate.io/developers/contributor-guide) and [Code of Conduct](https://weaviate.io/service/code-of-conduct) - required: true diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md deleted file mode 100644 index c2e68fbd380f63dbc047e948485a12ea204216a6..0000000000000000000000000000000000000000 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ /dev/null @@ -1,9 +0,0 @@ -### What's being changed: - - -### Review checklist - -- [ ] Documentation has been updated, if necessary. Link to changed documentation: -- [ ] Chaos pipeline run or not necessary. Link to pipeline: -- [ ] All new code is covered by tests where it is reasonable. -- [ ] Performance tests have been run or not necessary. diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index 226d72eae9753bc6be1775dc5eed614ad5e6a6b5..0000000000000000000000000000000000000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,7 +0,0 @@ -version: 2 -updates: - - package-ecosystem: "github-actions" - directory: "/" - schedule: - # Check for updates to GitHub Actions every week - interval: "weekly" \ No newline at end of file diff --git a/.github/stale.yml b/.github/stale.yml deleted file mode 100644 index bbefb96270d911f62d28e9608299514ba641e83f..0000000000000000000000000000000000000000 --- a/.github/stale.yml +++ /dev/null @@ -1,33 +0,0 @@ -# Number of days of inactivity before an issue becomes stale -daysUntilStale: 60 -# Number of days of inactivity before a stale issue is closed -daysUntilClose: 7 -# Issues with these labels will never be considered stale -exemptLabels: - - pinned - - security - - bug - - backlog - - needs-investigation -# Label to use when marking an issue as stale -staleLabel: autoclosed -# Comment to post when marking an issue as stale. Set to `false` to disable -markComment: > - Thank you for your contribution to Weaviate. This issue has not received any - activity in a while and has therefore been marked as stale. Stale issues will - eventually be autoclosed. This does not mean that we are ruling out to work - on this issue, but it most likely has not been prioritized high enough in the - last months. - - If you believe that this issue should remain open, please leave a short reply. - This lets us know that the issue is not abandoned and acts as a reminder for our - team to consider prioritizing this again. - - Please also consider if you can make a contribution to help with the solution - of this issue. If you are willing to contribute, but don't know where to start, - please leave a quick message and we'll try to help you. - - Thank you, - The Weaviate Team -# Comment to post when closing a stale issue. Set to `false` to disable -closeComment: false diff --git a/.github/workflows/branch.yaml b/.github/workflows/branch.yaml deleted file mode 100644 index 1eb9caeeb882ba2c820ae4ea8aa91eb906da0392..0000000000000000000000000000000000000000 --- a/.github/workflows/branch.yaml +++ /dev/null @@ -1,33 +0,0 @@ -name: Tests - -on: - push: - branches: - - '**' # run on ever branch except master (is covered by PR) - - '!master' -jobs: - Unit-Tests: - name: unit-tests - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version: '1.21' - cache: true - - name: Unit test - run: ./test/run.sh --unit-only - Integration-Tests: - name: integration-tests - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version: '1.21' - cache: true - - name: Integration test - run: ./test/run.sh --integration-only - diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml deleted file mode 100644 index 6276d7a125f08cf8cd9c94820abfa4ead2cc6a29..0000000000000000000000000000000000000000 --- a/.github/workflows/linter.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: golangci-lint -on: - push: - branches: - - master - tags: - - '**' - pull_request: -jobs: - golangci: - name: golangci - runs-on: ubuntu-latest - steps: - - uses: actions/setup-go@v5 - with: - go-version: '1.21' - - uses: actions/checkout@v4 - - name: golangci-lint - uses: golangci/golangci-lint-action@v3 - with: - # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. - version: v1.54 - args: --timeout=5m - protolint: - name: protolint - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: yoheimuta/action-protolint@v1 diff --git a/.github/workflows/pull_requests.yaml b/.github/workflows/pull_requests.yaml deleted file mode 100644 index 566e4b86aff7e0b61ea19b639b59b156125d6331..0000000000000000000000000000000000000000 --- a/.github/workflows/pull_requests.yaml +++ /dev/null @@ -1,225 +0,0 @@ -name: Tests - -on: - push: - branches: - - master - tags: - - '**' - pull_request: - -jobs: - Run-Swagger: - name: run-swagger - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version: '1.21' - cache: true - - name: Run Swagger - run: ./tools/gen-code-from-swagger.sh - - name: Error on change - run: | - # check if anything is different - CHANGED=$(git status -s | wc -l) - if [ "$CHANGED" -gt 0 ]; then - echo "Please run ./tools/gen-code-from-swagger.sh script and commit changes:" - git status -s - exit 1 - else - exit 0 - fi - Vulnerability-Scanning: - name: vulnerability-scanning - runs-on: ubuntu-latest - if: ${{ !github.event.pull_request.head.repo.fork }} # no PRs from fork - steps: - - uses: actions/checkout@v4 - - name: Scan for Vulnerabilities in Code - uses: Templum/govulncheck-action@v1.0.0 - with: - go-version: '1.21' - package: ./... - fail-on-vuln: true - Unit-Tests: - name: unit-tests - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version: '1.21' - cache: true - - name: Unit test - run: ./test/run.sh --unit-only - - name: Archive code coverage results - uses: actions/upload-artifact@v4 - with: - name: coverage-report-unit - path: coverage-unit.txt - Integration-Tests: - name: integration-tests - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version: '1.21' - cache: true - - name: Integration test - run: ./test/run.sh --integration-only - - name: Archive code coverage results - uses: actions/upload-artifact@v4 - with: - name: coverage-report-integration - path: coverage-integration.txt - Modules-Acceptance-Tests: - name: modules-acceptance-tests - runs-on: ubuntu-latest-4-cores - strategy: - fail-fast: false - matrix: - test: ["--modules-backup-only", "--modules-except-backup"] - steps: - - uses: actions/checkout@v4 - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version: '1.21' - cache: true - - name: Acceptance tests (modules) - run: ./test/run.sh ${{ matrix.test }} - Acceptance-Tests: - name: acceptance-tests - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version: '1.21' - cache: true - - name: Acceptance tests - env: - WCS_DUMMY_CI_PW: ${{ secrets.WCS_DUMMY_CI_PW }} - WCS_DUMMY_CI_PW_2: ${{ secrets.WCS_DUMMY_CI_PW_2 }} - run: ./test/run.sh --acceptance-only - Codecov: - needs: [Unit-Tests, Integration-Tests] - name: codecov - runs-on: ubuntu-latest - if: ${{ (github.ref_type == 'branch') && (github.ref_name != 'master') }} - steps: - - uses: actions/checkout@v4 - - name: Download coverage artifacts integration - uses: actions/download-artifact@v4 - with: - name: coverage-report-unit - - name: Download coverage unit - uses: actions/download-artifact@v4 - with: - name: coverage-report-integration - - name: Codecov - uses: codecov/codecov-action@v3 - with: - fail_ci_if_error: false - files: ./coverage-integration.txt, ./coverage-unit.txt - verbose: true - Compile-and-upload-binaries: - name: compile-and-upload-binaries - runs-on: ubuntu-latest-4-cores - steps: - - uses: actions/checkout@v4 - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version: '1.21' - cache: true - - name: goreleaser - run: | - echo 'deb [trusted=yes] https://repo.goreleaser.com/apt/ /' | sudo tee /etc/apt/sources.list.d/goreleaser.list - sudo apt update - sudo apt install goreleaser - GIT_HASH=$(git rev-parse --short HEAD) goreleaser build --clean --snapshot - - name: Upload macos - uses: actions/upload-artifact@v4 - with: - name: binaries-macos-unsigned - path: dist/weaviate_darwin_all - - name: Upload windows - uses: actions/upload-artifact@v4 - with: - name: binaries-windows-amd64 - path: dist/weaviate_windows_amd64_v1 - - name: Upload windows - uses: actions/upload-artifact@v4 - with: - name: binaries-windows-arm64 - path: dist/weaviate_windows_arm64 - - name: Upload linux amd64 - uses: actions/upload-artifact@v4 - with: - name: binaries-linux-amd64 - path: dist/weaviate_linux_amd64_v1 - - name: Upload linux arm64 - uses: actions/upload-artifact@v4 - with: - name: binaries-linux-arm64 - path: dist/weaviate_linux_arm64 - - - Acceptance-Tests-windows: - name: acceptance-tests-windows - needs: Compile-and-upload-binaries - runs-on: windows-latest - env: - AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: true - PERSISTENCE_DATA_PATH: /tmp - QUERY_DEFAULTS_LIMIT: 20 - steps: - - uses: actions/checkout@v4 - - name: Download binaries - uses: actions/download-artifact@v4 - with: - name: binaries-windows-amd64 - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version: '1.21' - cache: true - - name: start weaviate - shell: bash - # Weaviate is started without a Vectorizer as running text2vec-contextionary on GH actions is difficult: - # - docker on GHA only supports windows container - which we currently are not build - # - building those containers without a windows machine is difficult to figure out - run: ./weaviate.exe --scheme http --port 8080 & - - name: run acceptance tests - shell: bash - run: go test -count 1 -race test/acceptance/actions/*.go # tests that don't need a Vectorizer - - Push-Docker: - needs: [Acceptance-Tests, Modules-Acceptance-Tests, Unit-Tests, Integration-Tests, Vulnerability-Scanning, Run-Swagger] - name: push-docker - runs-on: ubuntu-latest-8-cores - if: ${{ !github.event.pull_request.head.repo.fork }} # no PRs from fork - steps: - - uses: actions/checkout@v4 - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{secrets.DOCKER_USERNAME}} - password: ${{secrets.DOCKER_PASSWORD}} - - name: Push container - id: push-container - run: ./ci/push_docker.sh - env: - PR_TITLE: "${{ github.event.pull_request.title }}" - - name: Generate Report - env: - PREVIEW_TAG: "${{ steps.push-container.outputs.PREVIEW_TAG }}" - run: ./ci/generate_docker_report.sh \ No newline at end of file diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml deleted file mode 100644 index 12025d7f6149e0007ec9b259d035a7c89d0b898a..0000000000000000000000000000000000000000 --- a/.github/workflows/release.yaml +++ /dev/null @@ -1,32 +0,0 @@ -name: Generate release assets - -on: - release: - types: [published] - -env: - CGO_ENABLED: 0 - -permissions: - contents: write - -jobs: - releases-matrix: - name: Release precompiled binaries - runs-on: ubuntu-latest - strategy: - matrix: - goos: [linux] - goarch: [amd64, arm64] - steps: - - uses: actions/checkout@v4 - - uses: wangyoucao577/go-release-action@v1.42 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - goos: ${{ matrix.goos }} - goarch: ${{ matrix.goarch }} - goversion: "1.21" - project_path: "./cmd/weaviate-server" - extra_files: LICENSE README.md - ldflags: -w -extldflags "-static" -X github.com/weaviate/weaviate/usecases/config.GitHash='"$GITHASH"' - sha256sum: true diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 8c2095ccbf976a34f3453fc02b4292a9074cf652..0000000000000000000000000000000000000000 --- a/.gitignore +++ /dev/null @@ -1,109 +0,0 @@ -# ignore release dirs -nightly/* -dist/* -dist -nightly - -# ignore confectionary files -contextionary/contextionary.vocab -contextionary/contextionary.idx -contextionary/contextionary.knn -contextionary/stopwords.json -wget-log* - -# ignore update files -semver -cc-test-reporter - -# Ignore OLD files -*.OLD -*.OLD.go - -# Do not include dist tmp -dist_tmp - -# No weaviate bins -./weaviate - -# Logs -logs -*.log -npm-debug.log* - -# Runtime data -pids -*.pid -*.seed - -# Contents of temp-directory -temp -.temp - -# Directory for instrumented libs generated by jscoverage/JSCover -lib-cov - -# Coverage directory used by tools like istanbul -coverage - -# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) -.grunt - -# node-waf configuration -.lock-wscript - -# Compiled binary addons (http://nodejs.org/api/addons.html) -build/Release - -# Dependency directory -node_modules - -# Optional npm cache directory -.npm - -# Optional REPL history -.node_repl_history - -# MAC stuff -.DS_Store -.! - -# IDE files -.idea/ -.vscode/ - -# Local files -_local - -# Remove weave files -weave-current.json -weave-current-discovery.json - -# local nodes folders -graphqlapi/prototype/node_modules/ - -/data*/ -/snapshots*/ -/backups*/ -docker-compose/runtime-stable/data -docker-compose/runtime-unstable/data -c.out -/cmd/weaviate-server/build_artifacts/ - -esbackups/ -**/testdata/ - -# coverage files -coverage*.txt - -# go mod -vendor/ - -test/benchmark/benchmark_results.json -# benchmark results files copied from a remote run -benchmark_results_*.json - -# local credentials -creds.json - -# VSCode -cmd/weaviate-server/__debug_bin* \ No newline at end of file diff --git a/.golangci.yml b/.golangci.yml deleted file mode 100644 index f98dd5b6724b419f988e014a24d1293d0b8f4f0c..0000000000000000000000000000000000000000 --- a/.golangci.yml +++ /dev/null @@ -1,62 +0,0 @@ -linters: - disable: - - errcheck - enable: - - misspell - - bodyclose - - gofumpt - - exhaustive - - govet - - unused - - nolintlint -linters-settings: - exhaustive: - # Presence of "default" case in switch statements satisfies exhaustiveness, - # even if all enum members are not listed. - default-signifies-exhaustive: true -issues: - exclude-rules: - # Exclude some staticcheck messages - - linters: - - staticcheck - text: "S1034" - - linters: - - staticcheck - text: "SA1029:" - - linters: - - staticcheck - text: "SA1015:" - - linters: - - staticcheck - text: "SA5011" - - linters: - - govet - text: "composites" - - linters: - - staticcheck - path: adapters/handlers/grpc/v1/filters.go - text: 'SA1019' # TODO: remove this once deprecated gRPC fields are removed from API - - linters: - - staticcheck - path: adapters/handlers/grpc/v1/prepare_reply.go - text: 'SA1019' # TODO: remove this once deprecated gRPC fields are removed from API - - linters: - - staticcheck - path: adapters/handlers/grpc/v1/parse_search_request.go - text: 'SA1019' # TODO: remove this once deprecated gRPC fields are removed from API - - linters: - - staticcheck - path: adapters/handlers/grpc/v1/batch_parse_request.go - text: 'SA1019' # TODO: remove this once deprecated gRPC fields are removed from API - - linters: - - staticcheck - path: adapters/handlers/grpc/v1/service.go - text: 'SA1019' # TODO: remove this once deprecated gRPC fields are removed from API - - linters: - - staticcheck - path: test/acceptance/grpc/grpc_test_deprecated.go - text: 'SA1019' # TODO: remove this once deprecated gRPC fields are removed from API -run: - build-tags: - - integrationTest - - integrationTestSlow diff --git a/.goreleaser.yaml b/.goreleaser.yaml deleted file mode 100644 index e5c28bb14489c0a76e4412986f398b93a2cb5bad..0000000000000000000000000000000000000000 --- a/.goreleaser.yaml +++ /dev/null @@ -1,24 +0,0 @@ -before: - hooks: - # You may remove this if you don't use go modules. - - go mod tidy -builds: - - env: - - CGO_ENABLED=0 - goos: - - linux - - darwin - - windows - main: ./cmd/weaviate-server - goarch: - - amd64 - - arm64 - ldflags: - - -w - - -extldflags "-static" - - -X github.com/weaviate/weaviate/usecases/config.GitHash={{ .Env.GIT_HASH }} - -# create a "fat" binary for MacOS -universal_binaries: - - replace: true - diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml deleted file mode 100644 index bac731fd437c6a26472597d5ff97bb5b904952a5..0000000000000000000000000000000000000000 --- a/.pre-commit-config.yaml +++ /dev/null @@ -1,13 +0,0 @@ -repos: - - repo: local - hooks: - - id: gofumpt - name: gofumpt - entry: gofumpt -w - language: system - types: [ go ] - - id: swagger - name: swagger - entry: ./tools/gen-code-from-swagger.sh - language: system - types: [ go ] diff --git a/.protolint.yaml b/.protolint.yaml deleted file mode 100644 index d720fd6f5940113e19fa744bf421b87d20397c10..0000000000000000000000000000000000000000 --- a/.protolint.yaml +++ /dev/null @@ -1,4 +0,0 @@ -lint: - rules_option: - max_line_length: - max_chars: 120 diff --git a/CITATION.cff b/CITATION.cff deleted file mode 100644 index b06edd7a9f9975c43c4354828256393a46d5d3de..0000000000000000000000000000000000000000 --- a/CITATION.cff +++ /dev/null @@ -1,23 +0,0 @@ -cff-version: 1.2.0 -message: "If you use this software, please cite it as below." -title: "Weaviate" -url: "https://github.com/weaviate/weaviate" -authors: -- family-names: "Dilocker" - given-names: "Etienne" -- family-names: "van Luijt" - given-names: "Bob" - orcid: "https://orcid.org/0000-0002-8239-7421" -- family-names: "Voorbach" - given-names: "Byron" -- family-names: "Hasan" - given-names: "Mohd Shukri" -- family-names: "Rodriguez" - given-names: "Abdel" -- family-names: "Kulawiak" - given-names: "Dirk Alexander" - orcid: "https://orcid.org/0000-0002-9848-3818" -- family-names: "Antas" - given-names: "Marcin" -- family-names: "Duckworth" - given-names: "Parker" diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md deleted file mode 100644 index 9487b02273793df66805338410743f45f9b97505..0000000000000000000000000000000000000000 --- a/CODE_OF_CONDUCT.md +++ /dev/null @@ -1,77 +0,0 @@ -# Contributor Covenant Code of Conduct - -## Our Pledge - -In the interest of fostering an open and welcoming environment, we as -contributors and maintainers pledge to make participation in our project and -our community a harassment-free experience for everyone, regardless of age, body -size, disability, ethnicity, sex characteristics, gender identity and expression, -level of experience, education, socio-economic status, nationality, personal -appearance, race, religion, or sexual identity and orientation. - -## Our Standards - -Examples of behavior that contributes to creating a positive environment -include: - -* Using welcoming and inclusive language -* Being respectful of differing viewpoints and experiences -* Gracefully accepting constructive criticism -* Focusing on what is best for the community -* Showing empathy towards other community members - -Examples of unacceptable behavior by participants include: - -* The use of sexualized language or imagery and unwelcome sexual attention or - advances -* Trolling, insulting/derogatory comments, and personal or political attacks -* Public or private harassment -* Publishing others' private information, such as a physical or electronic - address, without explicit permission -* Other conduct which could reasonably be considered inappropriate in a - professional setting - -## Our Responsibilities - -Project maintainers are responsible for clarifying the standards of acceptable -behavior and are expected to take appropriate and fair corrective action in -response to any instances of unacceptable behavior. - -Project maintainers have the right and responsibility to remove, edit, or -reject comments, commits, code, wiki edits, issues, and other contributions -that are not aligned to this Code of Conduct, or to ban temporarily or -permanently any contributor for other behaviors that they deem inappropriate, -threatening, offensive, or harmful. - -## Scope - -This Code of Conduct applies within all project spaces, and it also applies when -an individual is representing the project or its community in public spaces. -Examples of representing a project or community include using an official -project e-mail address, posting via an official social media account, or acting -as an appointed representative at an online or offline event. Representation of -a project may be further defined and clarified by project maintainers. - -## Enforcement - -Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported by contacting the project team at laura@semi.technology. All -complaints will be reviewed and investigated and will result in a response that -is deemed necessary and appropriate to the circumstances. The project team is -obligated to maintain confidentiality with regard to the reporter of an incident. -Further details of specific enforcement policies may be posted separately. - -Project maintainers who do not follow or enforce the Code of Conduct in good -faith may face temporary or permanent repercussions as determined by other -members of the project's leadership. - -## Attribution - -This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, -available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html - -[homepage]: https://www.contributor-covenant.org - -For answers to common questions about this code of conduct, see -https://www.contributor-covenant.org/faq - diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index b01e0fd9c68ca7ea3cb14d0372f1a64ed1314f7c..0000000000000000000000000000000000000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,43 +0,0 @@ -## Thanks for looking into contributing to Weaviate! -We really appreciate that you are willing to spend some time and effort to make Weaviate better for everyone! - -We have a detailed [contributor guide](https://weaviate.io/developers/contributor-guide/current/) as a part of our documentation. If you are new, we recommend reading through the [getting started guide for contributors](https://weaviate.io/developers/contributor-guide/current/getting-started/index.html) after reading this overview. - -## Finding a good issue to get started -We use the `good-first-issue` labels on issues that we think are great to get started. Typically, they are issues that are isolated to a specific area of Weaviate, e.g. only the API layer or only a specific package. - -## What do you need to contribute -To contribute to Weaviate Core you should have at least basic Golang skills. It is also tremendously helpful if you have successfully used Weaviate before, so that you have a rough understanding of the effects of changes etc. Any PR must include tests. We understand that not everyone is very familiar with testing strategies, and how we have setup our tests. This [guide on testing](https://weaviate.io/developers/contributor-guide/current/weaviate-core/tests.html) should be a great start. If you have completed a change, but are struggling with the test, you can also open a Draft PR and ask someone from our team for help. - -## If in doubt, ask. -The Weaviate team consists of some of the nicest people on this planet. If something is unclear or you'd like a second opinion, please don't hesitate to ask. We are glad that you want to help us, so naturally we will also do our best to help you on this journey - -## Development Environment -*Please note that the Weaviate team uses Linux and Mac (darwin/arm64) machines exclusively. Development on Windows may lead to unexpected issues.* - -Here are some guides to get started in no time: -* [Development Setup](https://weaviate.io/developers/contributor-guide/current/weaviate-core/setup.html) -* [Code Structure](https://weaviate.io/developers/contributor-guide/current/weaviate-core/structure.html) -* [CI/CD](https://weaviate.io/developers/contributor-guide/current/weaviate-core/cicd.html) -* [Testing Philosophy](https://weaviate.io/developers/contributor-guide/current/weaviate-core/tests.html) - -## Tagging your commit -Please tag your commit(s) with the appropriate GH issue that your change refers to, e.g. `gh-9001 reduce memory allocations of ACME widget`. Please also include something in your PR description to indicate which issue it will close, e.g. `fixes #9001` or `closes #9001`. - -## Pull Request -If you open an external pull request our CI pipeline will get started. This external run will not have access to secrets. This prevents people from submitting a malicious PR to steal secrets. As a result the CI run will be slightly different from an internal one. For example, it will not automatically push a Docker image. If your PR is merged, a container with your changes will be built from the trunk. - -## Agreements - -### Code of Conduct -Please note that this project is released with a Contributor Code of Conduct. By participating in this project you agree to abide by its terms. -[![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-v2.0%20adopted-ff69b4.svg)](CODE_OF_CONDUCT.md) - -### Contributor License Agreement - -Contributions to Weaviate must be accompanied by a Contributor License Agreement. You (or your employer) retain the copyright to your contribution; this simply gives us permission to use and redistribute your contributions as part of Weaviate. Go to [this page](https://www.semi.technology/playbooks/misc/contributor-license-agreement.html) to read the current agreement. - -The process works as follows: - -- You contribute by opening a [pull request](#pull-request). -- If you have not contributed before, our bot will ask to agree with the CLA diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index f80de2b9112a10da8433ade40e2fb61272e52f7e..0000000000000000000000000000000000000000 --- a/Dockerfile +++ /dev/null @@ -1,71 +0,0 @@ -############################################################################### -#python environment, main app and startup script. -FROM python:3.11.5 -#FROM python:3.11.9-slim -#FROM python:3.11.9-alpine -#FROM python:3.11-bookworm - -ENTRYPOINT ["/app/startup.sh"] -#RUN apt-get update && \ -# apt-get install -y libc6 && \ -# rm -rf /var/lib/apt/lists/* -WORKDIR /app - -#RUN ls -l / || ls -l /lib || ls -l /usr || ls -l /usr/lib6 || echo "### An ls failed." - -COPY ./requirements.txt /app/requirements.txt -COPY ./semsearch.py /app/semsearch.py -COPY ./startup.sh /app/startup.sh -RUN chmod 755 /app/startup.sh - -COPY ./multi-qa-MiniLM-L6-cos-v1 /app/multi-qa-MiniLM-L6-cos-v1 - -RUN mkdir -p /app/inputDocs -COPY ./inputDocs/* /app/inputDocs -RUN pip install --no-cache-dir --upgrade -r /app/requirements.txt -RUN pip install https://files.pythonhosted.org/packages/13/87/e0cb08c2d4bd7d38ab63816b306c8b1e7cfdc0e59bd54462e8b0df069078/semantic_text_splitter-0.6.3-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl -RUN pip show semantic-text-splitter - -RUN pip install llama_cpp_python - -############################################################################## -# Install Weaviate -WORKDIR /app/weaviate -RUN wget -qO- https://github.com/weaviate/weaviate/releases/download/v1.24.10/weaviate-v1.24.10-linux-amd64.tar.gz | tar -xzf - -RUN ls -al /app/weaviate - -# Set environment variables for Weaviate -ENV PATH="/app:/app/weaviate-v1.24.10-linux-x86_64:${PATH}" -# Expose the Weaviate port -EXPOSE 8080 - -#COPY Llama-2-7B-Chat-GGUF/llama-2-7b-chat.Q4_0.gguf /app -RUN cd /app; wget -v https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGUF/resolve/main/llama-2-7b-chat.Q4_0.gguf - -############################################################################## -# Install text2vec-transformers -WORKDIR /app/text2vec-transformers -COPY --from=semitechnologies/transformers-inference:sentence-transformers-multi-qa-MiniLM-L6-cos-v1 /app /app/text2vec-transformers -COPY --from=semitechnologies/transformers-inference:sentence-transformers-multi-qa-MiniLM-L6-cos-v1 /usr/local/bin /app/text2vec-transformers/bin - -COPY ./multi-qa-MiniLM-L6-cos-v1 /app/app/text2vec-transformers - -ENV PATH="/app/text2vec-transformers:/app/text2vec-transformers/bin:${PATH}" -#RUN pip install -r requirements.txt -#RUN pip install nltk==3.8.1 optimum==1.13.2 onnxruntime==1.16.1 onnx==1.14.1 -RUN ./custom_prerequisites.py - - -############################## -RUN useradd -m -u 1000 user - -############################################# -# Specify /data volume. -VOLUME /data - -############################################################################## -# Start the weaviate vector database, text2vec-transformers and the semantic search app. -#RUN /app/startup.sh -#RUN --mount=type=cache,target=/data,mode=777 /app/startup.sh -#RUN --mount=type=cache,target=/data,mode=777 echo "### Mounting /data" -CMD ["/app/startup.sh"] \ No newline at end of file diff --git a/DockerfileBuild b/DockerfileBuild deleted file mode 100644 index 2880c55d08f7daf6454cbf7381c0e6443f06a847..0000000000000000000000000000000000000000 --- a/DockerfileBuild +++ /dev/null @@ -1,93 +0,0 @@ -# Dockerfile for development purposes. -# Read docs/development.md for more information. -# vi: ft=dockerfile - -############################################################################### -# Base build image -FROM golang:1.21-alpine AS build_base -RUN apk add bash ca-certificates git gcc g++ libc-dev -WORKDIR /go/src/github.com/weaviate/weaviate -ENV GO111MODULE=on -# Populate the module cache based on the go.{mod,sum} files. -COPY go.mod . -COPY go.sum . -RUN go mod download - -############################################################################### -# This image builds the weaviate server -FROM build_base AS server_builder -ARG TARGETARCH -ARG GITHASH="unknown" -ARG EXTRA_BUILD_ARGS="" -COPY . . -RUN CGO_ENABLED=0 GOARCH=$TARGETARCH go build $EXTRA_BUILD_ARGS \ - -a -ldflags '-w -extldflags "-static" -X github.com/weaviate/weaviate/usecases/config.GitHash='"$GITHASH"'' \ - -o /weaviate-server ./cmd/weaviate-server - -############################################################################### -# This creates an image that can be used to fake an api for telemetry acceptance test purposes -FROM build_base AS telemetry_mock_api -COPY . . -ENTRYPOINT ["./tools/dev/telemetry_mock_api.sh"] - -############################################################################### -# This image gets grpc health check probe -FROM build_base AS grpc_health_probe_builder -ARG TARGETARCH -RUN GRPC_HEALTH_PROBE_VERSION=v0.4.22 && \ - wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-${TARGETARCH} && \ - chmod +x /bin/grpc_health_probe - -############################################################################### -# Weaviate (no differentiation between dev/test/prod - 12 factor!) -FROM alpine AS weaviate -#ENTRYPOINT ["/bin/weaviate"] -COPY --from=grpc_health_probe_builder /bin/grpc_health_probe /bin/ -COPY --from=server_builder /weaviate-server /bin/weaviate -RUN chmod 755 /bin/weaviate -RUN apk add --no-cache --upgrade ca-certificates openssl -RUN mkdir ./modules -RUN mkdir -p /var/lib/weaviate/data -RUN chmod -R 777 /var -ENV AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED true -ENV PERSISTENCE_DATA_PATH /var/lib/weaviate/data -#ENV ENABLE_MODULES text2vec-transformers -#CMD [ "--host", "0.0.0.0", "--port", "8080", "--scheme", "http", ""] - -############################################################################### -# python environment and app. -FROM python:3.11.5 -ENTRYPOINT ["/app/startup.sh"] -RUN apt update -WORKDIR /app - -COPY ./requirements.txt /app/requirements.txt -COPY ./semsearch.py /app/semsearch.py -COPY ./startup.sh /app/startup.sh -RUN chmod 755 /app/startup.sh - -COPY --from=weaviate /bin/weaviate /app/weaviate -COPY --from=weaviate ./modules ./ -COPY --from=server_builder /lib/libc.musl-x86_64.so.1 /lib -RUN mkdir -p /var/lib/weaviate/data y -RUN chmod -R 777 /var -COPY --from=weaviate /var/lib/weaviate/data /var/lib/weaviate/data -ENV AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED true -ENV AUTHENTICATION_OIDC_ENABLED false -ENV AUTHENTICATION_APIKEY_ENABLED false -ENV PERSISTENCE_DATA_PATH /var/lib/weaviate/data -#ENV ENABLE_MODULES text2vec-transformers -#ENV TRANSFORMERS_INFERENCE_API "http://127.0.0.1:999" - -RUN mkdir -p /app/inputDocs -COPY ./inputDocs/* /app/inputDocs - -RUN pip install --no-cache-dir --upgrade -r /app/requirements.txt -RUN pip install https://files.pythonhosted.org/packages/13/87/e0cb08c2d4bd7d38ab63816b306c8b1e7cfdc0e59bd54462e8b0df069078/semantic_text_splitter-0.6.3-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl -RUN pip show semantic-text-splitter -RUN useradd -m -u 1000 user - -############################################################################## -# Start the weaviate vector database and the semantic search app. -#RUN /app/startup.sh -CMD [] \ No newline at end of file diff --git a/DockerfileHld b/DockerfileHld deleted file mode 100644 index 20fdb2321d1995ef40516dcf920537056454a5d6..0000000000000000000000000000000000000000 --- a/DockerfileHld +++ /dev/null @@ -1,49 +0,0 @@ -# Dockerfile for development purposes. -# Read docs/development.md for more information -# vi: ft=dockerfile - -############################################################################### -# Base build image -FROM golang:1.21-alpine AS build_base -RUN apk add bash ca-certificates git gcc g++ libc-dev -WORKDIR /go/src/github.com/weaviate/weaviate -ENV GO111MODULE=on -# Populate the module cache based on the go.{mod,sum} files. -COPY go.mod . -COPY go.sum . -RUN go mod download - -############################################################################### -# This image builds the weaviate server -FROM build_base AS server_builder -ARG TARGETARCH -ARG GITHASH="unknown" -ARG EXTRA_BUILD_ARGS="" -COPY . . -RUN GOOS=linux GOARCH=$TARGETARCH go build $EXTRA_BUILD_ARGS \ - -ldflags '-w -extldflags "-static" -X github.com/weaviate/weaviate/usecases/config.GitHash='"$GITHASH"'' \ - -o /weaviate-server ./cmd/weaviate-server - -############################################################################### -# This creates an image that can be used to fake an api for telemetry acceptance test purposes -FROM build_base AS telemetry_mock_api -COPY . . -ENTRYPOINT ["./tools/dev/telemetry_mock_api.sh"] - -############################################################################### -# This image gets grpc health check probe -FROM build_base AS grpc_health_probe_builder -ARG TARGETARCH -RUN GRPC_HEALTH_PROBE_VERSION=v0.4.22 && \ - wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-${TARGETARCH} && \ - chmod +x /bin/grpc_health_probe - -############################################################################### -# Weaviate (no differentiation between dev/test/prod - 12 factor!) -FROM alpine AS weaviate -ENTRYPOINT ["/bin/weaviate"] -COPY --from=grpc_health_probe_builder /bin/grpc_health_probe /bin/ -COPY --from=server_builder /weaviate-server /bin/weaviate -RUN apk add --no-cache --upgrade ca-certificates openssl -RUN mkdir ./modules -CMD [ "--host", "0.0.0.0", "--port", "8080", "--scheme", "http"] diff --git a/DockerfilePythonWeaviate b/DockerfilePythonWeaviate deleted file mode 100644 index df6ba68216c48ac965ff8fc143ad31dc784eb5c2..0000000000000000000000000000000000000000 --- a/DockerfilePythonWeaviate +++ /dev/null @@ -1,59 +0,0 @@ -############################################################################### -#python environment, main app and startup script. -FROM python:3.11.5 -#FROM python:3.11.9-slim -#FROM python:3.11.9-alpine -#FROM python:3.11-bookworm - -ENTRYPOINT ["/app/startup.sh"] -#RUN apt-get update && \ -# apt-get install -y libc6 && \ -# rm -rf /var/lib/apt/lists/* -WORKDIR /app - -#RUN ls -l / || ls -l /lib || ls -l /usr || ls -l /usr/lib6 || echo "### An ls failed." - -COPY ./requirements.txt /app/requirements.txt -COPY ./semsearch.py /app/semsearch.py -COPY ./startup.sh /app/startup.sh -RUN chmod 755 /app/startup.sh - -COPY ./multi-qa-MiniLM-L6-cos-v1 /app/multi-qa-MiniLM-L6-cos-v1 - -RUN mkdir -p /app/inputDocs -COPY ./inputDocs/* /app/inputDocs -RUN pip install --no-cache-dir --upgrade -r /app/requirements.txt -RUN pip install https://files.pythonhosted.org/packages/13/87/e0cb08c2d4bd7d38ab63816b306c8b1e7cfdc0e59bd54462e8b0df069078/semantic_text_splitter-0.6.3-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl -RUN pip show semantic-text-splitter - -############################################################################## -# Install Weaviate -WORKDIR /app/weaviate -RUN wget -qO- https://github.com/weaviate/weaviate/releases/download/v1.24.10/weaviate-v1.24.10-linux-amd64.tar.gz | tar -xzf - -RUN ls -al /app/weaviate - -# Set environment variables for Weaviate -ENV PATH="/app:/app/weaviate-v1.24.10-linux-x86_64:${PATH}" -# Expose the Weaviate port -EXPOSE 8080 - -############################################################################## -# Install text2vec-transformers -WORKDIR /app/text2vec-transformers -COPY --from=semitechnologies/transformers-inference:sentence-transformers-multi-qa-MiniLM-L6-cos-v1 /app /app/text2vec-transformers -COPY --from=semitechnologies/transformers-inference:sentence-transformers-multi-qa-MiniLM-L6-cos-v1 /usr/local/bin /app/text2vec-transformers/bin - -COPY ./multi-qa-MiniLM-L6-cos-v1 /app/app/text2vec-transformers - -ENV PATH="/app/text2vec-transformers:/app/text2vec-transformers/bin:${PATH}" -#RUN pip install -r requirements.txt -#RUN pip install nltk==3.8.1 optimum==1.13.2 onnxruntime==1.16.1 onnx==1.14.1 -RUN ./custom_prerequisites.py - -############################## -RUN useradd -m -u 1000 user - -############################################################################## -# Start the weaviate vector database, text2vec-transformers and the semantic search app. -#RUN /app/startup.sh -CMD ["/app/startup.sh"] \ No newline at end of file diff --git a/DockerfileTestWvT2v b/DockerfileTestWvT2v deleted file mode 100644 index 63a19512411c99d66e701bad18e29684636a96cc..0000000000000000000000000000000000000000 --- a/DockerfileTestWvT2v +++ /dev/null @@ -1,15 +0,0 @@ -# Start with the official Weaviate image -FROM semitechnologies/weaviate:latest - -# Set environment variables -ENV WEAVIATE_SERVE_MODULES text2vec-transformers - -# Install Python and pip via apk, the package manager for Alpine -RUN apk update && apk add --no-cache python3 py3-pip transformers -#RUN pip3 install --no-cache-dir transformers - -# Expose the default port for Weaviate -EXPOSE 8080 - -# Start Weaviate -CMD ["weaviate", "start"] \ No newline at end of file diff --git a/Dockerfile_Orig b/Dockerfile_Orig deleted file mode 100644 index 4768cba033f96961e70e4ceecc8f41f49a7a3498..0000000000000000000000000000000000000000 --- a/Dockerfile_Orig +++ /dev/null @@ -1,86 +0,0 @@ -############################################################################### -# Use a specific version of Weaviate -FROM semitechnologies/weaviate:1.23.7 as weaviate - -# Set environment variables if needed -ENV WEAVIATE_ORIGIN http://localhost:8080 - -# Expose the default port for Weaviate -EXPOSE 8080 - -# Additional configuration or startup scripts can be added here -# For example, copying configuration files or scripts: -# COPY startup.sh /usr/local/bin/startup.sh -# RUN chmod +x /usr/local/bin/startup.sh - -# Define the default command -#CMD ["weaviate", "start"] - -############################################################################### -# Base build image -FROM golang:1.21-alpine AS build_base -RUN apk add bash ca-certificates git gcc g++ libc-dev -WORKDIR /go/src/github.com/weaviate/weaviate -ENV GO111MODULE=on -# Populate the module cache based on the go.{mod,sum} files. -COPY go.mod . -COPY go.sum . -RUN go mod download - -############################################################################### -# This image builds the weaviate server -FROM build_base AS server_builder -RUN apk add python3.11.5 - -ARG TARGETARCH -ARG GITHASH="unknown" -ARG EXTRA_BUILD_ARGS="" -COPY . . -RUN CGO_ENABLED=0 GOARCH=$TARGETARCH go build $EXTRA_BUILD_ARGS \ - -a -ldflags '-w -extldflags "-static" -X github.com/weaviate/weaviate/usecases/config.GitHash='"$GITHASH"'' \ - -o /weaviate-server ./cmd/weaviate-server - -############################################################################### -#python environment and app. -#FROM python:3.11.5 -#ENTRYPOINT ["/app/startup.sh"] -#RUN apt-get update && \ -# apt-get install -y libc6 && \ -# rm -rf /var/lib/apt/lists/* -WORKDIR /app - -#RUN ls -l / || ls -l /lib || ls -l /usr || ls -l /usr/lib6 || echo "### An ls failed." - -COPY ./requirements.txt /app/requirements.txt -COPY ./semsearch.py /app/semsearch.py -COPY ./startup.sh /app/startup.sh -RUN chmod 755 /app/startup.sh - -COPY --from=weaviate /bin/weaviate /app/weaviate -COPY --from=weaviate ./modules ./ - -COPY --from=server_builder /lib/libc.musl-x86_64.so.1 /lib -RUN mkdir -p /usr/lib64 y -#RUN ls -l /usr/lib64 -RUN ln -s /usr/lib64/libc.so.6 /usr/lib64/libc.musl-x86_64.so.1 - -RUN mkdir -p /var/lib/weaviate/data y -RUN chmod -R 777 /var -#COPY --from=weaviate /var/lib/weaviate/data /var/lib/weaviate/data -ENV AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED true -ENV AUTHENTICATION_OIDC_ENABLED false -ENV AUTHENTICATION_APIKEY_ENABLED false -ENV PERSISTENCE_DATA_PATH /var/lib/weaviate/data -#ENV ENABLE_MODULES text2vec-transformers -#ENV TRANSFORMERS_INFERENCE_API "http://127.0.0.1:999" - -RUN mkdir -p /app/inputDocs -COPY ./inputDocs/* /app/inputDocs -RUN pip install --no-cache-dir --upgrade -r /app/requirements.txt -RUN pip install https://files.pythonhosted.org/packages/13/87/e0cb08c2d4bd7d38ab63816b306c8b1e7cfdc0e59bd54462e8b0df069078/semantic_text_splitter-0.6.3-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl -RUN pip show semantic-text-splitter -RUN useradd -m -u 1000 user -############################################################################## -# Start the weaviate vector database and the semantic search app. -#RUN /app/startup.sh -CMD ["/app/startup.sh"] \ No newline at end of file diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 20b14c1dc48520c5f6db9d0d1834887ab80827a5..0000000000000000000000000000000000000000 --- a/LICENSE +++ /dev/null @@ -1,27 +0,0 @@ -Copyright (c) 2020-2024, Weaviate B.V. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Llama-2-7B-Chat-GGUF b/Llama-2-7B-Chat-GGUF deleted file mode 160000 index 191239b3e26b2882fb562ffccdd1cf0f65402adb..0000000000000000000000000000000000000000 --- a/Llama-2-7B-Chat-GGUF +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 191239b3e26b2882fb562ffccdd1cf0f65402adb diff --git a/README.md b/README.md deleted file mode 100644 index ab2113489f5cb9e30640b870de4fb602f57f30c1..0000000000000000000000000000000000000000 --- a/README.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -title: SemanticSearchPOC -emoji: 📉 -colorFrom: purple -colorTo: red -sdk: docker -pinned: false ---- - -Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference diff --git a/adapters/clients/client.go b/adapters/clients/client.go deleted file mode 100644 index af0b9d74d582422cabb15d482379ccac1e8863f3..0000000000000000000000000000000000000000 --- a/adapters/clients/client.go +++ /dev/null @@ -1,96 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "bytes" - "context" - "fmt" - "io" - "net/http" - "time" -) - -type retryClient struct { - client *http.Client - *retryer -} - -func (c *retryClient) doWithCustomMarshaller(timeout time.Duration, - req *http.Request, body []byte, decode func([]byte) error, -) (err error) { - ctx, cancel := context.WithTimeout(req.Context(), timeout) - defer cancel() - try := func(ctx context.Context) (bool, error) { - if body != nil { - req.Body = io.NopCloser(bytes.NewReader(body)) - } - res, err := c.client.Do(req) - if err != nil { - return ctx.Err() == nil, fmt.Errorf("connect: %w", err) - } - - respBody, err := io.ReadAll(res.Body) - if err != nil { - return shouldRetry(res.StatusCode), fmt.Errorf("read response: %w", err) - } - defer res.Body.Close() - - if code := res.StatusCode; code != http.StatusOK { - return shouldRetry(code), fmt.Errorf("status code: %v, error: %s", code, respBody) - } - - if err := decode(respBody); err != nil { - return false, fmt.Errorf("unmarshal response: %w", err) - } - - return false, nil - } - return c.retry(ctx, 9, try) -} - -type retryer struct { - minBackOff time.Duration - maxBackOff time.Duration - timeoutUnit time.Duration -} - -func newRetryer() *retryer { - return &retryer{ - minBackOff: time.Millisecond * 250, - maxBackOff: time.Second * 30, - timeoutUnit: time.Second, // used by unit tests - } -} - -func (r *retryer) retry(ctx context.Context, n int, work func(context.Context) (bool, error)) error { - delay := r.minBackOff - for { - keepTrying, err := work(ctx) - if !keepTrying || n < 1 || err == nil { - return err - } - - n-- - if delay = backOff(delay); delay > r.maxBackOff { - delay = r.maxBackOff - } - timer := time.NewTimer(delay) - select { - case <-ctx.Done(): - timer.Stop() - return fmt.Errorf("%v: %w", err, ctx.Err()) - case <-timer.C: - } - timer.Stop() - } -} diff --git a/adapters/clients/cluster_backups.go b/adapters/clients/cluster_backups.go deleted file mode 100644 index 6d1f9a7b715fbb99d4f6d7ea8abbe297f116d11e..0000000000000000000000000000000000000000 --- a/adapters/clients/cluster_backups.go +++ /dev/null @@ -1,178 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - - "github.com/weaviate/weaviate/usecases/backup" -) - -const ( - pathCanCommit = "/backups/can-commit" - pathCommit = "/backups/commit" - pathStatus = "/backups/status" - pathAbort = "/backups/abort" -) - -type ClusterBackups struct { - client *http.Client -} - -func NewClusterBackups(client *http.Client) *ClusterBackups { - return &ClusterBackups{client: client} -} - -func (c *ClusterBackups) CanCommit(ctx context.Context, - host string, req *backup.Request, -) (*backup.CanCommitResponse, error) { - url := url.URL{Scheme: "http", Host: host, Path: pathCanCommit} - - b, err := json.Marshal(req) - if err != nil { - return nil, fmt.Errorf("marshal can-commit request: %w", err) - } - - httpReq, err := http.NewRequest(http.MethodPost, url.String(), bytes.NewReader(b)) - if err != nil { - return nil, fmt.Errorf("new can-commit request: %w", err) - } - - respBody, statusCode, err := c.do(httpReq) - if err != nil { - return nil, fmt.Errorf("can-commit request: %w", err) - } - - if statusCode != http.StatusOK { - return nil, fmt.Errorf("unexpected status code %d (%s)", - statusCode, respBody) - } - - var resp backup.CanCommitResponse - err = json.Unmarshal(respBody, &resp) - if err != nil { - return nil, fmt.Errorf("unmarshal can-commit response: %w", err) - } - - return &resp, nil -} - -func (c *ClusterBackups) Commit(ctx context.Context, - host string, req *backup.StatusRequest, -) error { - url := url.URL{Scheme: "http", Host: host, Path: pathCommit} - - b, err := json.Marshal(req) - if err != nil { - return fmt.Errorf("marshal commit request: %w", err) - } - - httpReq, err := http.NewRequest(http.MethodPost, url.String(), bytes.NewReader(b)) - if err != nil { - return fmt.Errorf("new commit request: %w", err) - } - - respBody, statusCode, err := c.do(httpReq) - if err != nil { - return fmt.Errorf("commit request: %w", err) - } - - if statusCode != http.StatusCreated { - return fmt.Errorf("unexpected status code %d (%s)", - statusCode, respBody) - } - - return nil -} - -func (c *ClusterBackups) Status(ctx context.Context, - host string, req *backup.StatusRequest, -) (*backup.StatusResponse, error) { - url := url.URL{Scheme: "http", Host: host, Path: pathStatus} - - b, err := json.Marshal(req) - if err != nil { - return nil, fmt.Errorf("marshal status request: %w", err) - } - - httpReq, err := http.NewRequest(http.MethodPost, url.String(), bytes.NewReader(b)) - if err != nil { - return nil, fmt.Errorf("new status request: %w", err) - } - - respBody, statusCode, err := c.do(httpReq) - if err != nil { - return nil, fmt.Errorf("status request: %w", err) - } - - if statusCode != http.StatusOK { - return nil, fmt.Errorf("unexpected status code %d (%s)", - statusCode, respBody) - } - - var resp backup.StatusResponse - err = json.Unmarshal(respBody, &resp) - if err != nil { - return nil, fmt.Errorf("unmarshal status response: %w", err) - } - - return &resp, nil -} - -func (c *ClusterBackups) Abort(_ context.Context, - host string, req *backup.AbortRequest, -) error { - url := url.URL{Scheme: "http", Host: host, Path: pathAbort} - - b, err := json.Marshal(req) - if err != nil { - return fmt.Errorf("marshal abort request: %w", err) - } - - httpReq, err := http.NewRequest(http.MethodPost, url.String(), bytes.NewReader(b)) - if err != nil { - return fmt.Errorf("new abort request: %w", err) - } - - respBody, statusCode, err := c.do(httpReq) - if err != nil { - return fmt.Errorf("abort request: %w", err) - } - - if statusCode != http.StatusNoContent { - return fmt.Errorf("unexpected status code %d (%s)", - statusCode, respBody) - } - - return nil -} - -func (c *ClusterBackups) do(req *http.Request) (body []byte, statusCode int, err error) { - httpResp, err := c.client.Do(req) - if err != nil { - return nil, 0, fmt.Errorf("make request: %w", err) - } - - body, err = io.ReadAll(httpResp.Body) - if err != nil { - return nil, httpResp.StatusCode, fmt.Errorf("read response: %w", err) - } - defer httpResp.Body.Close() - - return body, httpResp.StatusCode, nil -} diff --git a/adapters/clients/cluster_classifications.go b/adapters/clients/cluster_classifications.go deleted file mode 100644 index 8d6dea21c866cb57c95174fa47b82b1f256d0a8b..0000000000000000000000000000000000000000 --- a/adapters/clients/cluster_classifications.go +++ /dev/null @@ -1,127 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "bytes" - "context" - "encoding/json" - "io" - "net/http" - "net/url" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/usecases/cluster" -) - -type ClusterClassifications struct { - client *http.Client -} - -func NewClusterClassifications(httpClient *http.Client) *ClusterClassifications { - return &ClusterClassifications{client: httpClient} -} - -func (c *ClusterClassifications) OpenTransaction(ctx context.Context, host string, - tx *cluster.Transaction, -) error { - path := "/classifications/transactions/" - method := http.MethodPost - url := url.URL{Scheme: "http", Host: host, Path: path} - - pl := txPayload{ - Type: tx.Type, - ID: tx.ID, - Payload: tx.Payload, - } - - jsonBytes, err := json.Marshal(pl) - if err != nil { - return errors.Wrap(err, "marshal transaction payload") - } - - req, err := http.NewRequestWithContext(ctx, method, url.String(), - bytes.NewReader(jsonBytes)) - if err != nil { - return errors.Wrap(err, "open http request") - } - - req.Header.Set("content-type", "application/json") - - res, err := c.client.Do(req) - if err != nil { - return errors.Wrap(err, "send http request") - } - - defer res.Body.Close() - if res.StatusCode != http.StatusCreated { - if res.StatusCode == http.StatusConflict { - return cluster.ErrConcurrentTransaction - } - - body, _ := io.ReadAll(res.Body) - return errors.Errorf("unexpected status code %d (%s)", res.StatusCode, - body) - } - - return nil -} - -func (c *ClusterClassifications) AbortTransaction(ctx context.Context, host string, - tx *cluster.Transaction, -) error { - path := "/classifications/transactions/" + tx.ID - method := http.MethodDelete - url := url.URL{Scheme: "http", Host: host, Path: path} - - req, err := http.NewRequestWithContext(ctx, method, url.String(), nil) - if err != nil { - return errors.Wrap(err, "open http request") - } - - res, err := c.client.Do(req) - if err != nil { - return errors.Wrap(err, "send http request") - } - - defer res.Body.Close() - if res.StatusCode != http.StatusNoContent { - return errors.Errorf("unexpected status code %d", res.StatusCode) - } - - return nil -} - -func (c *ClusterClassifications) CommitTransaction(ctx context.Context, host string, - tx *cluster.Transaction, -) error { - path := "/classifications/transactions/" + tx.ID + "/commit" - method := http.MethodPut - url := url.URL{Scheme: "http", Host: host, Path: path} - - req, err := http.NewRequestWithContext(ctx, method, url.String(), nil) - if err != nil { - return errors.Wrap(err, "open http request") - } - - res, err := c.client.Do(req) - if err != nil { - return errors.Wrap(err, "send http request") - } - - defer res.Body.Close() - if res.StatusCode != http.StatusNoContent { - return errors.Errorf("unexpected status code %d", res.StatusCode) - } - - return nil -} diff --git a/adapters/clients/cluster_node.go b/adapters/clients/cluster_node.go deleted file mode 100644 index 7da59a10b7f29436517c655e9c6fba31b04b3ad6..0000000000000000000000000000000000000000 --- a/adapters/clients/cluster_node.go +++ /dev/null @@ -1,66 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "context" - "encoding/json" - "io" - "net/http" - "net/url" - "path" - - enterrors "github.com/weaviate/weaviate/entities/errors" - "github.com/weaviate/weaviate/entities/models" -) - -type RemoteNode struct { - client *http.Client -} - -func NewRemoteNode(httpClient *http.Client) *RemoteNode { - return &RemoteNode{client: httpClient} -} - -func (c *RemoteNode) GetNodeStatus(ctx context.Context, hostName, className, output string) (*models.NodeStatus, error) { - p := "/nodes/status" - if className != "" { - p = path.Join(p, className) - } - method := http.MethodGet - params := url.Values{"output": []string{output}} - url := url.URL{Scheme: "http", Host: hostName, Path: p, RawQuery: params.Encode()} - - req, err := http.NewRequestWithContext(ctx, method, url.String(), nil) - if err != nil { - return nil, enterrors.NewErrOpenHttpRequest(err) - } - - res, err := c.client.Do(req) - if err != nil { - return nil, enterrors.NewErrSendHttpRequest(err) - } - - defer res.Body.Close() - body, _ := io.ReadAll(res.Body) - if res.StatusCode != http.StatusOK { - return nil, enterrors.NewErrUnexpectedStatusCode(res.StatusCode, body) - } - - var nodeStatus models.NodeStatus - err = json.Unmarshal(body, &nodeStatus) - if err != nil { - return nil, enterrors.NewErrUnmarshalBody(err) - } - - return &nodeStatus, nil -} diff --git a/adapters/clients/cluster_schema.go b/adapters/clients/cluster_schema.go deleted file mode 100644 index 066932c5eeb9cafa8b3b23e602d5326ac0d1deea..0000000000000000000000000000000000000000 --- a/adapters/clients/cluster_schema.go +++ /dev/null @@ -1,164 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - - "github.com/weaviate/weaviate/usecases/cluster" -) - -type ClusterSchema struct { - client *http.Client -} - -func NewClusterSchema(httpClient *http.Client) *ClusterSchema { - return &ClusterSchema{client: httpClient} -} - -func (c *ClusterSchema) OpenTransaction(ctx context.Context, host string, - tx *cluster.Transaction, -) error { - path := "/schema/transactions/" - method := http.MethodPost - url := url.URL{Scheme: "http", Host: host, Path: path} - - pl := txPayload{ - Type: tx.Type, - ID: tx.ID, - Payload: tx.Payload, - DeadlineMilli: tx.Deadline.UnixMilli(), - } - - jsonBytes, err := json.Marshal(pl) - if err != nil { - return fmt.Errorf("marshal transaction payload: %w", err) - } - - req, err := http.NewRequestWithContext(ctx, method, url.String(), - bytes.NewReader(jsonBytes)) - if err != nil { - return fmt.Errorf("open http request: %w", err) - } - - req.Header.Set("content-type", "application/json") - - res, err := c.client.Do(req) - if err != nil { - return fmt.Errorf("send http request: %w", err) - } - - defer res.Body.Close() - body, _ := io.ReadAll(res.Body) - if res.StatusCode != http.StatusCreated { - if res.StatusCode == http.StatusConflict { - return cluster.ErrConcurrentTransaction - } - - return fmt.Errorf("unexpected status code %d (%s)", res.StatusCode, - body) - } - - // optional for backward-compatibility before v1.17 where only - // write-transactions where supported. They had no return value other than - // the status code. With the introduction of read-transactions it is now - // possible to return the requested value - if len(body) == 0 { - return nil - } - - var txRes txResponsePayload - err = json.Unmarshal(body, &txRes) - if err != nil { - return fmt.Errorf("unexpected error unmarshalling tx response: %w", err) - } - - if tx.ID != txRes.ID { - return fmt.Errorf("unexpected mismatch between outgoing and incoming tx ids:"+ - "%s vs %s", tx.ID, txRes.ID) - } - - tx.Payload = txRes.Payload - - return nil -} - -func (c *ClusterSchema) AbortTransaction(ctx context.Context, host string, - tx *cluster.Transaction, -) error { - path := "/schema/transactions/" + tx.ID - method := http.MethodDelete - url := url.URL{Scheme: "http", Host: host, Path: path} - - req, err := http.NewRequestWithContext(ctx, method, url.String(), nil) - if err != nil { - return fmt.Errorf("open http request: %w", err) - } - - res, err := c.client.Do(req) - if err != nil { - return fmt.Errorf("send http request: %w", err) - } - - defer res.Body.Close() - if res.StatusCode != http.StatusNoContent { - errBody, _ := io.ReadAll(res.Body) - return fmt.Errorf("unexpected status code %d: %s", res.StatusCode, errBody) - } - - return nil -} - -func (c *ClusterSchema) CommitTransaction(ctx context.Context, host string, - tx *cluster.Transaction, -) error { - path := "/schema/transactions/" + tx.ID + "/commit" - method := http.MethodPut - url := url.URL{Scheme: "http", Host: host, Path: path} - - req, err := http.NewRequestWithContext(ctx, method, url.String(), nil) - if err != nil { - return fmt.Errorf("open http request: %w", err) - } - - res, err := c.client.Do(req) - if err != nil { - return fmt.Errorf("send http request: %w", err) - } - - defer res.Body.Close() - if res.StatusCode != http.StatusNoContent { - errBody, _ := io.ReadAll(res.Body) - return fmt.Errorf("unexpected status code %d: %s", res.StatusCode, errBody) - } - - return nil -} - -type txPayload struct { - Type cluster.TransactionType `json:"type"` - ID string `json:"id"` - Payload interface{} `json:"payload"` - DeadlineMilli int64 `json:"deadlineMilli"` -} - -type txResponsePayload struct { - Type cluster.TransactionType `json:"type"` - ID string `json:"id"` - Payload json.RawMessage `json:"payload"` -} diff --git a/adapters/clients/cluster_schema_test.go b/adapters/clients/cluster_schema_test.go deleted file mode 100644 index ef10c54d1a78c3b5ff8e3c98a6cdb403cc6e1be6..0000000000000000000000000000000000000000 --- a/adapters/clients/cluster_schema_test.go +++ /dev/null @@ -1,441 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "context" - "encoding/json" - "io" - "net/http" - "net/http/httptest" - "net/url" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/usecases/cluster" -) - -func TestOpenTransactionNoReturnPayload(t *testing.T) { - // The No-Return-Payload is the situation that existed prior to v1.17 where - // the only option for transactions was to broadcast updates. By keeping this - // test around, we can make sure that we are not breaking backward - // compatibility. - - handler := func(w http.ResponseWriter, r *http.Request) { - defer r.Body.Close() - body, err := io.ReadAll(r.Body) - require.Nil(t, err) - - var pl txPayload - err = json.Unmarshal(body, &pl) - require.Nil(t, err) - - assert.Equal(t, "mamma-mia-paylodia-belissima", pl.Payload.(string)) - w.WriteHeader(http.StatusCreated) - } - server := httptest.NewServer(http.HandlerFunc(handler)) - defer server.Close() - - u, _ := url.Parse(server.URL) - - c := NewClusterSchema(&http.Client{}) - - tx := &cluster.Transaction{ - ID: "12345", - Type: "the best", - Payload: "mamma-mia-paylodia-belissima", - } - - err := c.OpenTransaction(context.Background(), u.Host, tx) - assert.Nil(t, err) -} - -func TestOpenTransactionWithReturnPayload(t *testing.T) { - // Newly added as part of v1.17 where read-transactions were introduced which - // are used to sync up cluster schema state when nodes join - handler := func(w http.ResponseWriter, r *http.Request) { - resTx := txPayload{ - Type: "my-tx", - ID: "987", - Payload: "gracie-mille-per-payload", - } - resBody, err := json.Marshal(resTx) - require.Nil(t, err) - - w.WriteHeader(http.StatusCreated) - w.Write(resBody) - } - server := httptest.NewServer(http.HandlerFunc(handler)) - defer server.Close() - - u, _ := url.Parse(server.URL) - - c := NewClusterSchema(&http.Client{}) - - txIn := &cluster.Transaction{ - ID: "987", - Type: "my-tx", - } - - err := c.OpenTransaction(context.Background(), u.Host, txIn) - assert.Nil(t, err) - - expectedTxOut := &cluster.Transaction{ - ID: "987", - Type: "my-tx", - Payload: json.RawMessage("\"gracie-mille-per-payload\""), - } - - assert.Equal(t, expectedTxOut, txIn) -} - -func TestOpenTransactionWithTTL(t *testing.T) { - deadline, err := time.Parse(time.RFC3339Nano, "2040-01-02T15:04:05.00Z") - require.Nil(t, err) - - handler := func(w http.ResponseWriter, r *http.Request) { - defer r.Body.Close() - body, err := io.ReadAll(r.Body) - require.Nil(t, err) - - var pl txPayload - err = json.Unmarshal(body, &pl) - require.Nil(t, err) - - parsedDL := time.UnixMilli(pl.DeadlineMilli) - assert.Equal(t, deadline.UnixNano(), parsedDL.UnixNano()) - w.WriteHeader(http.StatusCreated) - } - server := httptest.NewServer(http.HandlerFunc(handler)) - defer server.Close() - - u, _ := url.Parse(server.URL) - - c := NewClusterSchema(&http.Client{}) - - tx := &cluster.Transaction{ - ID: "12345", - Type: "the best", - Payload: "mamma-mia-paylodia-belissima", - Deadline: deadline, - } - - err = c.OpenTransaction(context.Background(), u.Host, tx) - assert.Nil(t, err) -} - -func TestOpenTransactionUnhappyPaths(t *testing.T) { - type test struct { - name string - handler http.HandlerFunc - expectedErr error - expectedErrContains string - ctx context.Context - shutdownPrematurely bool - } - - expiredCtx, cancel := context.WithCancel(context.Background()) - cancel() - - tests := []test{ - { - name: "concurrent transaction", - handler: func(w http.ResponseWriter, r *http.Request) { - defer r.Body.Close() - w.WriteHeader(http.StatusConflict) - }, - expectedErr: cluster.ErrConcurrentTransaction, - }, - { - name: "arbitrary 500", - handler: func(w http.ResponseWriter, r *http.Request) { - defer r.Body.Close() - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte("nope!")) - }, - expectedErrContains: "nope!", - }, - { - name: "invalid json", - handler: func(w http.ResponseWriter, r *http.Request) { - defer r.Body.Close() - w.WriteHeader(http.StatusCreated) - w.Write([]byte("<< time.Nanosecond*50 { - t.Errorf("average time got %v", av) - } -} - -func newReplicationClient(httpClient *http.Client) *replicationClient { - c := NewReplicationClient(httpClient).(*replicationClient) - c.minBackOff = time.Millisecond * 1 - c.maxBackOff = time.Millisecond * 8 - c.timeoutUnit = time.Millisecond * 20 - return c -} diff --git a/adapters/handlers/graphql/common/fetch/filter.go b/adapters/handlers/graphql/common/fetch/filter.go deleted file mode 100644 index a447bbbcfede13d2efdea462b42902b2b39438d3..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/common/fetch/filter.go +++ /dev/null @@ -1,146 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package fetch - -import ( - "fmt" - - "github.com/tailor-inc/graphql" - "github.com/weaviate/weaviate/adapters/handlers/graphql/descriptions" - "github.com/weaviate/weaviate/adapters/handlers/graphql/local/common_filters" -) - -// FilterBuilder can build where filters for both local and -type FilterBuilder struct { - prefix string -} - -// NewFilterBuilder with kind and prefix -func NewFilterBuilder(prefix string) *FilterBuilder { - return &FilterBuilder{ - prefix: prefix, - } -} - -// Build a where filter ArgumentConfig -func (b *FilterBuilder) Build() *graphql.ArgumentConfig { - return &graphql.ArgumentConfig{ - Description: descriptions.FetchWhereFilterFields, - Type: graphql.NewNonNull(graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sFetchObjectWhereInpObj", b.prefix), - Fields: b.fields(), - Description: descriptions.FetchWhereFilterFieldsInpObj, - }, - )), - } -} - -func (b *FilterBuilder) fields() graphql.InputObjectConfigFieldMap { - return graphql.InputObjectConfigFieldMap{ - "class": &graphql.InputObjectFieldConfig{ - Type: graphql.NewNonNull(b.class()), - Description: descriptions.WhereClass, - }, - "properties": &graphql.InputObjectFieldConfig{ - Type: graphql.NewNonNull(graphql.NewList(b.properties())), - Description: descriptions.WhereProperties, - }, - "first": &graphql.InputObjectFieldConfig{ - Type: graphql.Int, - Description: descriptions.First, - }, - } -} - -func (b *FilterBuilder) properties() *graphql.InputObject { - elements := common_filters.BuildNew(fmt.Sprintf("%sFetchObject", b.prefix)) - - // Remove path and operands fields as they are not required here - delete(elements, "path") - delete(elements, "operands") - - // make operator required - elements["operator"].Type = graphql.NewNonNull(elements["operator"].Type) - - elements["certainty"] = &graphql.InputObjectFieldConfig{ - Type: graphql.NewNonNull(graphql.Float), - Description: descriptions.WhereCertainty, - } - elements["name"] = &graphql.InputObjectFieldConfig{ - Type: graphql.NewNonNull(graphql.String), - Description: descriptions.WhereName, - } - elements["keywords"] = &graphql.InputObjectFieldConfig{ - Type: graphql.NewList(b.keywordInpObj(fmt.Sprintf("%sFetchObjectWhereProperties", b.prefix))), - Description: descriptions.WhereKeywords, - } - - networkFetchWhereInpObjPropertiesObj := graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sFetchObjectWhereInpObjProperties", b.prefix), - Fields: elements, - Description: descriptions.WhereProperties, - }, - ) - - return networkFetchWhereInpObjPropertiesObj -} - -func (b *FilterBuilder) keywordInpObj(prefix string) *graphql.InputObject { - return graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sKeywordsInpObj", prefix), - Fields: graphql.InputObjectConfigFieldMap{ - "value": &graphql.InputObjectFieldConfig{ - Type: graphql.String, - Description: descriptions.WhereKeywordsValue, - }, - "weight": &graphql.InputObjectFieldConfig{ - Type: graphql.Float, - Description: descriptions.WhereKeywordsWeight, - }, - }, - Description: descriptions.WhereKeywordsInpObj, - }, - ) -} - -func (b *FilterBuilder) class() *graphql.InputObject { - filterClassElements := graphql.InputObjectConfigFieldMap{ - "name": &graphql.InputObjectFieldConfig{ - Type: graphql.String, - Description: descriptions.WhereName, - }, - "certainty": &graphql.InputObjectFieldConfig{ - Type: graphql.Float, - Description: descriptions.WhereCertainty, - }, - "keywords": &graphql.InputObjectFieldConfig{ - Type: graphql.NewList(b.keywordInpObj(fmt.Sprintf("%sFetchObjectWhereClass", b.prefix))), - Description: descriptions.WhereKeywords, - }, - "first": &graphql.InputObjectFieldConfig{ - Type: graphql.Int, - Description: descriptions.First, - }, - } - - networkFetchWhereInpObjClassInpObj := graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sFetchObjectWhereInpObjClassInpObj", b.prefix), - Fields: filterClassElements, - Description: descriptions.WhereClass, - }, - ) - return networkFetchWhereInpObjClassInpObj -} diff --git a/adapters/handlers/graphql/common/json_number.go b/adapters/handlers/graphql/common/json_number.go deleted file mode 100644 index c5a6738cea56f61053709dd114d888e48e277d7b..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/common/json_number.go +++ /dev/null @@ -1,60 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package common - -import ( - "encoding/json" - "fmt" - - "github.com/tailor-inc/graphql" - "github.com/weaviate/weaviate/entities/aggregation" -) - -// JSONNumberResolver turns json.Number types into number types usable by graphQL -func JSONNumberResolver(p graphql.ResolveParams) (interface{}, error) { - switch v := p.Source.(type) { - case map[string]interface{}: - field, ok := v[p.Info.FieldName] - if !ok { - return nil, nil - } - - switch n := field.(type) { - case json.Number: - return n.Float64() - case int64: - return float64(n), nil - case int: - return float64(n), nil - case float64: - return n, nil - } - - return nil, fmt.Errorf("unknown number type for %t", field) - - case map[string]float64: - return v[p.Info.FieldName], nil - - case aggregation.Text: - switch p.Info.FieldName { - // case "count": - // // TODO gh-974: Support Count in text aggregations - // return nil, nil - - default: - return nil, fmt.Errorf("fieldName '%s' does not match text aggregation", p.Info.FieldName) - } - - default: - return nil, fmt.Errorf("json number resolver: unusable type %T", p.Source) - } -} diff --git a/adapters/handlers/graphql/common/json_number_test.go b/adapters/handlers/graphql/common/json_number_test.go deleted file mode 100644 index 614f60acdfda6873c9823f7027c9b6ae3cfe251c..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/common/json_number_test.go +++ /dev/null @@ -1,79 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package common - -import ( - "encoding/json" - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/tailor-inc/graphql" -) - -type testCase struct { - input interface{} - expectedOutput float64 -} - -func TestJSONNumberResolver(t *testing.T) { - tests := []testCase{ - { - input: json.Number("10"), - expectedOutput: 10.0, - }, - { - input: int(10), - expectedOutput: 10.0, - }, - { - input: int64(10), - expectedOutput: 10.0, - }, - { - input: float64(10), - expectedOutput: 10.0, - }, - } - - for _, test := range tests { - name := fmt.Sprintf("%#v -> %#v", test.input, test.expectedOutput) - t.Run(name, func(t *testing.T) { - result, err := JSONNumberResolver(resolveParams(test.input)) - assert.Nil(t, err, "should not error") - assert.Equal(t, test.expectedOutput, result) - }) - } -} - -func resolveParams(input interface{}) graphql.ResolveParams { - return graphql.ResolveParams{ - Source: map[string]interface{}{ - "myField": input, - }, - Info: graphql.ResolveInfo{FieldName: "myField"}, - } -} - -func TestNumberFieldNotPresent(t *testing.T) { - // shouldn't return anything, but also not error. This can otherwise lead to - // odd behavior when no entries are present, yet we asked for int props and - // type, see https://github.com/weaviate/weaviate/issues/775 - params := graphql.ResolveParams{ - Source: map[string]interface{}{}, - Info: graphql.ResolveInfo{FieldName: "myField"}, - } - - result, err := JSONNumberResolver(params) - assert.Nil(t, err) - assert.Equal(t, nil, result) -} diff --git a/adapters/handlers/graphql/descriptions/aggregate.go b/adapters/handlers/graphql/descriptions/aggregate.go deleted file mode 100644 index ad666f739b2c1c5a685bb66146ddee0e2645f129..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/descriptions/aggregate.go +++ /dev/null @@ -1,83 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Package descriptions provides the descriptions as used by the graphql endpoint for Weaviate -package descriptions - -// AGGREGATE -const ( - AggregateProperty = "Aggregate this property" - AggregateObjects = "Aggregate Objects on a local Weaviate" -) - -const GroupBy = "Specify which properties to group by" - -const ( - AggregatePropertyObject = "An object containing Aggregation information about this property" -) - -const AggregateObjectsObj = "An object allowing Aggregation of %ss on a local Weaviate" - -const ( - AggregateMean = "Aggregate on the mean of numeric property values" - AggregateSum = "Aggregate on the sum of numeric property values" - AggregateMedian = "Aggregate on the median of numeric property values" - AggregateMode = "Aggregate on the mode of numeric property values" - AggregateMin = "Aggregate on the minimum of numeric property values" - AggregateMax = "Aggregate on the maximum of numeric property values" - AggregateCount = "Aggregate on the total amount of found property values" - AggregateGroupedBy = "Indicates the group of returned data" -) - -const AggregateNumericObj = "An object containing the %s of numeric properties" - -const AggregateCountObj = "An object containing countable properties" - -const AggregateGroupedByObj = "An object containing the path and value of the grouped property" - -const ( - AggregateGroupedByGroupedByPath = "The path of the grouped property" - AggregateGroupedByGroupedByValue = "The value of the grouped property" -) - -// NETWORK -const NetworkAggregateWeaviateObj = "An object containing Get Objects fields for network Weaviate instance: " - -const NetworkAggregate = "Perform Aggregation of Objects" - -const ( - NetworkAggregateObj = "An object allowing Aggregation of Objects" - NetworkAggregatePropertyObject = "An object containing Aggregation information about this property" -) - -const NetworkAggregateThingsActionsObj = "An object allowing Aggregation of %ss on a network Weaviate" - -const ( - NetworkAggregateMean = "Aggregate on the mean of numeric property values" - NetworkAggregateSum = "Aggregate on the sum of numeric property values" - NetworkAggregateMedian = "Aggregate on the median of numeric property values" - NetworkAggregateMode = "Aggregate on the mode of numeric property values" - NetworkAggregateMin = "Aggregate on the minimum of numeric property values" - NetworkAggregateMax = "Aggregate on the maximum of numeric property values" - NetworkAggregateCount = "Aggregate on the total amount of found property values" - NetworkAggregateGroupedBy = "Indicates the group of returned data" -) - -const NetworkAggregateNumericObj = "An object containing the %s of numeric properties" - -const NetworkAggregateCountObj = "An object containing countable properties" - -const NetworkAggregateGroupedByObj = "An object containing the path and value of the grouped property" - -const ( - NetworkAggregateGroupedByGroupedByPath = "The path of the grouped property" - NetworkAggregateGroupedByGroupedByValue = "The value of the grouped property" -) diff --git a/adapters/handlers/graphql/descriptions/explore.go b/adapters/handlers/graphql/descriptions/explore.go deleted file mode 100644 index ddd1bbcb32749ab12a457691cdcaf4596572f65e..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/descriptions/explore.go +++ /dev/null @@ -1,30 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Package descriptions provides the descriptions as used by the graphql endpoint for Weaviate -package descriptions - -const ( - LocalExplore = "Explore Concepts on a local weaviate with vector-aided search" - LocalExploreConcepts = "Explore Concepts on a local weaviate with vector-aided serach through keyword-based search terms" - VectorMovement = "Move your search term closer to or further away from another vector described by keywords" - Keywords = "Keywords are a list of search terms. Array type, e.g. [\"keyword 1\", \"keyword 2\"]" - Network = "Set to true, if the exploration should include remote peers" - Limit = "Limit the results set (usually fewer results mean faster queries)" - Offset = "Offset of the results set (usually fewer results mean faster queries)" - Certainty = "Normalized Distance between the result item and the search vector. Normalized to be between 0 (identical vectors) and 1 (perfect opposite)." - Distance = "The required degree of similarity between an object's characteristics and the provided filter values" - Vector = "Target vector to be used in kNN search" - Force = "The force to apply for a particular movements. Must be between 0 and 1 where 0 is equivalent to no movement and 1 is equivalent to largest movement possible" - ClassName = "Name of the Class" - ID = "Concept identifier in the uuid format" - Beacon = "Concept identifier in the beacon format, such as weaviate:////id" -) diff --git a/adapters/handlers/graphql/descriptions/fetch.go b/adapters/handlers/graphql/descriptions/fetch.go deleted file mode 100644 index f33c6ddd03d2725a6c410f7bb5b97486ea5a39cb..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/descriptions/fetch.go +++ /dev/null @@ -1,55 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Package descriptions provides the descriptions as used by the graphql endpoint for Weaviate -package descriptions - -// Local -const ( - LocalFetch = "Fetch Beacons that are similar to a specified concept from the Objects subsets on a Weaviate network" - LocalFetchObj = "An object used to perform a Fuzzy Fetch to search for Objects and Actions similar to a specified concept on a Weaviate network" -) - -const ( - LocalFetchObjects = "Perform a Fuzzy Fetch to Fetch Beacons similar to a specified concept on a Weaviate network from the Objects subset" - LocalFetchFuzzy = "Perform a Fuzzy Fetch to Fetch Beacons similar to a specified concept on a Weaviate network from both the Objects subsets" -) - -const ( - LocalFetchBeacon = "A Beacon result from a local Weaviate Local Fetch query" - LocalFetchClassName = "The class name of the result from a local Weaviate Local Fetch query" - LocalFetchCertainty = "The degree of similarity on a scale of 0-1 between the Beacon's characteristics and the provided concept" - LocalFetchActionsObj = "An object used to Fetch Beacons from the Actions subset of the dataset" -) - -const ( - LocalFetchFuzzyBeacon = "A Beacon result from a local Weaviate Fetch Fuzzy query from both the Objects subsets" - LocalFetchFuzzyClassName = "Class name of the result from a local Weaviate Fetch Fuzzy query from both the Objects subsets" - LocalFetchFuzzyCertainty = "The degree of similarity on a scale of 0-1 between the Beacon's characteristics and the provided concept" - LocalFetchFuzzyObj = "An object used to Fetch Beacons from both the Objects subsets" -) - -// NETWORK -const ( - NetworkFetch = "Fetch Beacons that are similar to a specified concept from the Objects subsets on a Weaviate network" - NetworkFetchObj = "An object used to perform a Fuzzy Fetch to search for Objects similar to a specified concept on a Weaviate network" -) - -const ( - NetworkFetchFuzzy = "Perform a Fuzzy Fetch to Fetch Beacons similar to a specified concept on a Weaviate network from both the Objects subsets" -) - -const ( - NetworkFetchFuzzyClassName = "The class name of the result from a network Weaviate Fetch Fuzzy query from both the Objects subsets" - NetworkFetchFuzzyBeacon = "A Beacon result from a network Weaviate Fetch Fuzzy query from both the Objects subsets" - NetworkFetchFuzzyCertainty = "The degree of similarity on a scale of 0-1 between the Beacon's characteristics and the provided concept" - NetworkFetchFuzzyObj = "An object used to Fetch Beacons from both the Objects subsets" -) diff --git a/adapters/handlers/graphql/descriptions/filters.go b/adapters/handlers/graphql/descriptions/filters.go deleted file mode 100644 index ab64d8c7483f1dcdc4c77f68d75a6384d274954b..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/descriptions/filters.go +++ /dev/null @@ -1,154 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Package descriptions provides the descriptions as used by the graphql endpoint for Weaviate -package descriptions - -// Where filter elements -const ( - GetWhere = "Filter options for a local Get query, used to convert the result to the specified filters" - GetWhereInpObj = "An object containing filter options for a local Get query, used to convert the result to the specified filters" -) - -const ( - LocalMetaWhere = "Filter options for a local Meta query, used to convert the result to the specified filters" - LocalMetaWhereInpObj = "An object containing filter options for a local Meta query, used to convert the result to the specified filters" -) - -const ( - AggregateWhere = "Filter options for a local Aggregate query, used to convert the result to the specified filters" - AggregateWhereInpObj = "An object containing filter options for a local Aggregate query, used to convert the result to the specified filters" -) - -const ( - NetworkGetWhere = "Filter options for a network Get query, used to convert the result to the specified filters" - NetworkGetWhereInpObj = "An object containing filter options for a network Get query, used to convert the result to the specified filters" -) - -const ( - NetworkMetaWhere = "Filter options for a network Meta query, used to convert the result to the specified filters" - NetworkMetaWhereInpObj = "An object containing filter options for a network Meta query, used to convert the result to the specified filters" -) - -const ( - NetworkAggregateWhere = "Filter options for a network Aggregate query, used to convert the result to the specified filters" - NetworkAggregateWhereInpObj = "An object containing filter options for a network Aggregate query, used to convert the result to the specified filters" -) - -const ( - WhereOperands = "Contains the Operands that can be applied to a 'where' filter" - WhereOperandsInpObj = "An object containing the Operands that can be applied to a 'where' filter" -) - -const ( - WhereOperator = "Contains the Operators that can be applied to a 'where' filter" - WhereOperatorEnum = "An object containing the Operators that can be applied to a 'where' filter" -) - -const WherePath = "Specify the path from the Objects fields to the property name (e.g. ['Things', 'City', 'population'] leads to the 'population' property of a 'City' object)" - -const ( - WhereValueInt = "Specify an Integer value that the target property will be compared to" - WhereValueNumber = "Specify a Float value that the target property will be compared to" - WhereValueBoolean = "Specify a Boolean value that the target property will be compared to" - WhereValueString = "Specify a String value that the target property will be compared to" - WhereValueRange = "Specify both geo-coordinates (latitude and longitude as decimals) and a maximum distance from the described coordinates. The search will return any result which is located less than or equal to the specified maximum distance in km away from the specified point." - WhereValueRangeGeoCoordinates = "The geoCoordinates that form the center point of the search." - WhereValueRangeGeoCoordinatesLatitude = "The latitude (in decimal format) of the geoCoordinates to search around." - WhereValueRangeGeoCoordinatesLongitude = "The longitude (in decimal format) of the geoCoordinates to search around." - WhereValueRangeDistance = "The distance from the point specified via geoCoordinates." - WhereValueRangeDistanceMax = "The maximum distance from the point specified geoCoordinates." - WhereValueText = "Specify a Text value that the target property will be compared to" - WhereValueDate = "Specify a Date value that the target property will be compared to" -) - -// Properties and Classes filter elements (used by Fetch and Introspect Where filters) -const ( - WhereProperties = "Specify which properties to filter on" - WherePropertiesObj = "Specify which properties to filter on" -) - -const ( - WherePropertiesPropertyName = "Specify which property name to filter properties on" - WhereCertainty = "Specify the required degree of similarity between an object's characteristics and the provided filter values on a scale of 0-1" - WhereName = "Specify the name of the property to filter on" -) - -const ( - WhereKeywords = "Specify which keywords to filter on" - WhereKeywordsInpObj = "Specify the value and the weight of a keyword" -) - -const ( - WhereKeywordsValue = "Specify the value of the keyword" - WhereKeywordsWeight = "Specify the weight of the keyword" -) - -const ( - WhereClass = "Specify which classes to filter on" - WhereInpObj = "Specify which classes and properties to filter on" -) - -// Unique Fetch filter elements -const ( - FetchWhereFilterFields = "An object containing filter options for a network Fetch search, used to convert the result to the specified filters" - FetchWhereFilterFieldsInpObj = "Filter options for a network Fetch search, used to convert the result to the specified filters" -) - -const ( - FetchFuzzyValue = "Specify the concept that will be used to fetch Objects on the network (e.g. 'Airplane', or 'City')" - FetchFuzzyCertainty = "Specify how much a Beacon's characteristics must match the provided concept on a scale of 0 to 1" -) - -// Unique Introspect filter elements -const ( - IntrospectWhereFilterFields = "An object containing filter options for a network Fetch search, used to convert the result to the specified filters" - IntrospectWhereFilterFieldsInpObj = "Filter options for a network Fetch search, used to convert the result to the specified filters" - IntrospectBeaconId = "The id of the Beacon" -) - -// GroupBy filter elements -const ( - GroupByGroup = "Specify the property of the class to group by" - GroupByCount = "Get the number of instances of a property in a group" - GroupBySum = "Get the sum of the values of a property in a group" - GroupByMin = "Get the minimum occurring value of a property in a group" - GroupByMax = "Get the maximum occurring value of a property in a group" - GroupByMean = "Get the mean value of a property in a group" - GroupByMedian = "Get the median of a property in a group" - GroupByMode = "Get the mode of a property in a group" -) - -// Request timeout filter elements -const NetworkTimeout = "Specify the time in seconds after which an unresolved request automatically fails" - -// Pagination filter elements -const ( - First = "Show the first x results (pagination option)" - After = "Show the results after the first x results (pagination option)" -) - -// Cursor API -const ( - AfterID = "Show the results after a given ID" -) - -const ( - SortPath = "Specify the path from the Objects fields to the property name (e.g. ['Get', 'City', 'population'] leads to the 'population' property of a 'City' object)" - SortOrder = "Specify the sort order, either ascending (asc) which is default or descending (desc)" -) - -const ( - GroupByFilter = "Specify the property of the class to group by" - GroupByPath = "Specify the path from the objects fields to the property name (e.g. ['Things', 'City', 'population'] leads to the 'population' property of a 'City' object)" - GroupByGroups = "Specify the number of groups to be created" - GroupByObjectsPerGroup = "Specify the number of max objects in group" -) diff --git a/adapters/handlers/graphql/descriptions/get.go b/adapters/handlers/graphql/descriptions/get.go deleted file mode 100644 index 606caa9bfe2a892bc39e24b4c2f6c7d04a62ae34..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/descriptions/get.go +++ /dev/null @@ -1,50 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Package descriptions provides the descriptions as used by the graphql endpoint for Weaviate -package descriptions - -// Local -const ( - GetObjects = "Get Objects on a local Weaviate" -) - -const ( - GetObj = "An object used to Get Objects on a local Weaviate" - Get = "Get Objects on a local Weaviate" -) - -const GetObjectsActionsObj = "An object used to get %ss on a local Weaviate" - -const GetClassUUID = "The UUID of a Object, assigned by its local Weaviate" - -// Network -const ( - NetworkGet = "Get Objects from a Weaviate in a network" - NetworkGetObj = "An object used to Get Objects from a Weaviate in a network" -) - -const NetworkGetWeaviateObj = "An object containing Get Objects fields for network Weaviate instance: " - -const ( - NetworkGetObjects = "Get Objects from a Weaviate in a network" -) - -const ( - NetworkGetObjectsObj = "An object containing the Objects objects on this network Weaviate instance." -) - -const NetworkGetClassUUID = "The UUID of a Object, assigned by the Weaviate network" // TODO check this with @lauraham - -const ConsistencyLevel = "Determines how many replicas must acknowledge a request " + - "before it is considered successful. Can be 'ONE', 'QUORUM', or 'ALL'" - -const Tenant = "The value by which a tenant is identified, specified in the class schema" diff --git a/adapters/handlers/graphql/descriptions/getMeta.go b/adapters/handlers/graphql/descriptions/getMeta.go deleted file mode 100644 index 45b040e8e3445f32e5481542a1e86f0e7959597d..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/descriptions/getMeta.go +++ /dev/null @@ -1,88 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Package descriptions provides the descriptions as used by the graphql endpoint for Weaviate -package descriptions - -// Local -const ( - LocalMetaObj = "An object used to Get Meta information about Objects on a local Weaviate" - LocalMeta = "Get Meta information about Objects on a local Weaviate" -) - -const ( - MetaPropertyType = "The datatype of this property" - MetaPropertyCount = "The total amount of found instances for this property" // TODO check this with @lauraham - MetaPropertyTopOccurrences = "An object containing data about the most frequently occurring values for this property" - MetaPropertyTopOccurrencesValue = "The most frequently occurring value for this property" - MetaPropertyTopOccurrencesOccurs = "How often the most frequently occurring value for this property occurs" // TODO check this with @lauraham - MetaPropertyMinimum = "The minimum value for this property" - MetaPropertyMaximum = "The maximum value for this property" - MetaPropertyMean = "The mean of all values for this property" - MetaPropertySum = "The sum of all values for this property" - MetaPropertyObject = "An object containing meta information about this property" -) - -const ( - AggregatePropertyType = "The datatype of this property" - AggregatePropertyCount = "The total amount of found instances for this property" // TODO check this with @lauraham - AggregatePropertyTopOccurrences = "An object containing data about the most frequently occurring values for this property" - AggregatePropertyTopOccurrencesValue = "The most frequently occurring value for this property" - AggregatePropertyTopOccurrencesOccurs = "How often the most frequently occurring value for this property occurs" // TODO check this with @lauraham - AggregatePropertyMinimum = "The minimum value for this property" - AggregatePropertyMaximum = "The maximum value for this property" - AggregatePropertyMean = "The mean of all values for this property" - AggregatePropertySum = "The sum of all values for this property" -) - -// Network -const ( - NetworkMeta = "Get meta information about Objects from a Weaviate in a network" - NetworkMetaObj = "An object used to Get meta information about Objects from a Weaviate in a network" - NetworkMetaWeaviateObj = "An object containing the Meta Objects fields for network Weaviate instance: " -) - -const ( - MetaMetaProperty = "Meta information about the object" - MetaProperty = "Meta information about the property " -) - -const ( - MetaClassPropertyTotalTrue = "How often this boolean property's value is true in the dataset" - MetaClassPropertyPercentageTrue = "The percentage of true values for this boolean property in the dataset" -) - -const ( - MetaClassPropertyTotalFalse = "How often this boolean property's value is false in the dataset" - MetaClassPropertyPercentageFalse = "The percentage of false values for this boolean property in the dataset" -) - -const ( - MetaClassPropertyPointingTo = "The classes that this object contains a reference to" - MetaClassMetaCount = "The total amount of found instances for a class" - MetaClassMetaObj = "An object containing Meta information about a class" -) - -const ( - AggregateClassPropertyTotalTrue = "How often this boolean property's value is true in the dataset" - AggregateClassPropertyPercentageTrue = "The percentage of true values for this boolean property in the dataset" -) - -const ( - AggregateClassPropertyTotalFalse = "How often this boolean property's value is false in the dataset" - AggregateClassPropertyPercentageFalse = "The percentage of false values for this boolean property in the dataset" -) - -const ( - AggregateClassPropertyPointingTo = "The classes that this object contains a reference to" - AggregateClassAggregateCount = "The total amount of found instances for a class" - AggregateClassAggregateObj = "An object containing Aggregate information about a class" -) diff --git a/adapters/handlers/graphql/descriptions/introspect.go b/adapters/handlers/graphql/descriptions/introspect.go deleted file mode 100644 index c5fd381184a6e3aeb13c6b7606a465aa629d2f93..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/descriptions/introspect.go +++ /dev/null @@ -1,30 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Package descriptions provides the descriptions as used by the graphql endpoint for Weaviate -package descriptions - -// NETWORK -const ( - NetworkIntrospect = "Get Introspection information about Objects and/or Beacons in a Weaviate network" - NetworkIntrospectObj = "An object used to perform an Introspection query on a Weaviate network" -) - -const ( - NetworkIntrospectWeaviate = "The Weaviate instance that the current Object or Beacon belongs to" - NetworkIntrospectClassName = "The name of the current Object or Beacon's class" - NetworkIntrospectCertainty = "The degree of similarity between a(n) Object or Beacon and the filter input" -) - -const ( - NetworkIntrospectBeaconProperties = "The properties of a Beacon" - NetworkIntrospectBeaconPropertiesPropertyName = "The names of the properties of a Beacon" -) diff --git a/adapters/handlers/graphql/descriptions/merge.go b/adapters/handlers/graphql/descriptions/merge.go deleted file mode 100644 index b2347fb20bca40449e804e12fc46f733e7c3e022..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/descriptions/merge.go +++ /dev/null @@ -1,31 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Package descriptions provides the descriptions as used by the graphql endpoint for Weaviate -package descriptions - -// Local -const ( - LocalMergeObj = "An object used to Merge Objects on a local Weaviate" - LocalMerge = "Merge Objects on a local Weaviate" -) - -const LocalMergeClassUUID = "The UUID of a Object, assigned by its local Weaviate" - -// Network -const ( - NetworkMerge = "Merge Objects from a Weaviate in a network" - NetworkMergeObj = "An object used to Merge Objects from a Weaviate in a network" -) - -const NetworkMergeWeaviateObj = "An object containing Merge Objects fields for network Weaviate instance: " - -const NetworkMergeClassUUID = "The UUID of a Thing or Action, assigned by the Weaviate network" // TODO check this with @lauraham diff --git a/adapters/handlers/graphql/descriptions/rootQuery.go b/adapters/handlers/graphql/descriptions/rootQuery.go deleted file mode 100644 index fb8ee9020e10bbf13d1746d5389ceb78c1b8bac5..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/descriptions/rootQuery.go +++ /dev/null @@ -1,28 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Package descriptions provides the descriptions as used by the graphql endpoint for Weaviate -package descriptions - -// ROOT -const ( - WeaviateObj = "The location of the root query" - WeaviateNetwork = "Query a Weaviate network" -) - -// LOCAL -const LocalObj = "A query on a local Weaviate" - -// NETWORK -const ( - NetworkWeaviate = "An object for the network Weaviate instance: " - NetworkObj = "An object used to perform queries on a Weaviate network" -) diff --git a/adapters/handlers/graphql/graphiql/graphiql.go b/adapters/handlers/graphql/graphiql/graphiql.go deleted file mode 100644 index b59845c5d066f999604ef739beed1a6146ccf1a2..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/graphiql/graphiql.go +++ /dev/null @@ -1,233 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Based on `graphiql.go` from https://github.com/graphql-go/handler -// only made RenderGraphiQL a public function. -package graphiql - -import ( - "encoding/json" - "html/template" - "net/http" - "strings" -) - -// graphiqlVersion is the current version of GraphiQL -const graphiqlVersion = "0.11.11" - -// graphiqlData is the page data structure of the rendered GraphiQL page -type graphiqlData struct { - GraphiqlVersion string - QueryString string - Variables string - OperationName string - AuthKey string - AuthToken string -} - -func AddMiddleware(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if strings.HasPrefix(r.URL.Path, "/v1/graphql") && r.Method == http.MethodGet { - renderGraphiQL(w, r) - } else { - next.ServeHTTP(w, r) - } - }) -} - -// renderGraphiQL renders the GraphiQL GUI -func renderGraphiQL(w http.ResponseWriter, r *http.Request) { - w.Header().Set("WWW-Authenticate", `Basic realm="Provide your key and token (as username as password respectively)"`) - - user, password, authOk := r.BasicAuth() - if !authOk { - http.Error(w, "Not authorized", 401) - return - } - - queryParams := r.URL.Query() - - t := template.New("GraphiQL") - t, err := t.Parse(graphiqlTemplate) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Attempt to deserialize the 'variables' query key to something reasonable. - var queryVars interface{} - err = json.Unmarshal([]byte(queryParams.Get("variables")), &queryVars) - - var varsString string - if err == nil { - vars, err := json.MarshalIndent(queryVars, "", " ") - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - varsString = string(vars) - if varsString == "null" { - varsString = "" - } - } - - // Create result string - d := graphiqlData{ - GraphiqlVersion: graphiqlVersion, - QueryString: queryParams.Get("query"), - Variables: varsString, - OperationName: queryParams.Get("operationName"), - AuthKey: user, - AuthToken: password, - } - err = t.ExecuteTemplate(w, "index", d) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - } -} - -// tmpl is the page template to render GraphiQL -const graphiqlTemplate = ` -{{ define "index" }} - - - - - - GraphiQL - - - - - - - - - - - -
Loading...
- - - -{{ end }} -` diff --git a/adapters/handlers/graphql/local/aggregate/aggregate.go b/adapters/handlers/graphql/local/aggregate/aggregate.go deleted file mode 100644 index 9bb7ecf36184e17be78ca5bfaa62f38ef3566da4..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/aggregate/aggregate.go +++ /dev/null @@ -1,281 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package aggregate - -import ( - "fmt" - - "github.com/tailor-inc/graphql" - "github.com/weaviate/weaviate/adapters/handlers/graphql/descriptions" - "github.com/weaviate/weaviate/adapters/handlers/graphql/local/common_filters" - "github.com/weaviate/weaviate/adapters/handlers/graphql/utils" - "github.com/weaviate/weaviate/entities/aggregation" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/usecases/config" -) - -type ModulesProvider interface { - AggregateArguments(class *models.Class) map[string]*graphql.ArgumentConfig - ExtractSearchParams(arguments map[string]interface{}, className string) map[string]interface{} -} - -// Build the Aggregate Kinds schema -func Build(dbSchema *schema.Schema, config config.Config, - modulesProvider ModulesProvider, -) (*graphql.Field, error) { - if len(dbSchema.Objects.Classes) == 0 { - return nil, utils.ErrEmptySchema - } - - var err error - var localAggregateObjects *graphql.Object - if len(dbSchema.Objects.Classes) > 0 { - localAggregateObjects, err = classFields(dbSchema.Objects.Classes, config, modulesProvider) - if err != nil { - return nil, err - } - } - - field := graphql.Field{ - Name: "Aggregate", - Description: descriptions.AggregateWhere, - Type: localAggregateObjects, - Resolve: passThroughResolver, - } - - return &field, nil -} - -func classFields(databaseSchema []*models.Class, - config config.Config, modulesProvider ModulesProvider, -) (*graphql.Object, error) { - fields := graphql.Fields{} - - for _, class := range databaseSchema { - field, err := classField(class, class.Description, config, modulesProvider) - if err != nil { - return nil, err - } - - fields[class.Class] = field - } - - return graphql.NewObject(graphql.ObjectConfig{ - Name: "AggregateObjectsObj", - Fields: fields, - Description: descriptions.AggregateObjectsObj, - }), nil -} - -func classField(class *models.Class, description string, - config config.Config, modulesProvider ModulesProvider, -) (*graphql.Field, error) { - metaClassName := fmt.Sprintf("Aggregate%s", class.Class) - - fields := graphql.ObjectConfig{ - Name: metaClassName, - Fields: (graphql.FieldsThunk)(func() graphql.Fields { - fields, err := classPropertyFields(class) - if err != nil { - // we cannot return an error in this FieldsThunk and have to panic unfortunately - panic(fmt.Sprintf("Failed to assemble single Local Aggregate Class field: %s", err)) - } - - return fields - }), - Description: description, - } - - fieldsObject := graphql.NewObject(fields) - fieldsField := &graphql.Field{ - Type: graphql.NewList(fieldsObject), - Description: description, - Args: graphql.FieldConfigArgument{ - "limit": &graphql.ArgumentConfig{ - Description: descriptions.First, - Type: graphql.Int, - }, - "where": &graphql.ArgumentConfig{ - Description: descriptions.GetWhere, - Type: graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("AggregateObjects%sWhereInpObj", class.Class), - Fields: common_filters.BuildNew(fmt.Sprintf("AggregateObjects%s", class.Class)), - Description: descriptions.GetWhereInpObj, - }, - ), - }, - "groupBy": &graphql.ArgumentConfig{ - Description: descriptions.GroupBy, - Type: graphql.NewList(graphql.String), - }, - "nearVector": nearVectorArgument(class.Class), - "nearObject": nearObjectArgument(class.Class), - "objectLimit": &graphql.ArgumentConfig{ - Description: descriptions.First, - Type: graphql.Int, - }, - "hybrid": hybridArgument(fieldsObject, class, modulesProvider), - }, - Resolve: makeResolveClass(modulesProvider, class), - } - - if modulesProvider != nil { - for name, argument := range modulesProvider.AggregateArguments(class) { - fieldsField.Args[name] = argument - } - } - - if schema.MultiTenancyEnabled(class) { - fieldsField.Args["tenant"] = tenantArgument() - } - - return fieldsField, nil -} - -func classPropertyFields(class *models.Class) (graphql.Fields, error) { - fields := graphql.Fields{} - for _, property := range class.Properties { - propertyType, err := schema.GetPropertyDataType(class, property.Name) - if err != nil { - return nil, fmt.Errorf("%s.%s: %s", class.Class, property.Name, err) - } - - convertedDataType, err := classPropertyField(*propertyType, class, property) - if err != nil { - return nil, err - } - - fields[property.Name] = convertedDataType - } - - // Special case: meta { count } appended to all regular props - fields["meta"] = &graphql.Field{ - Description: descriptions.LocalMetaObj, - Type: metaObject(fmt.Sprintf("Aggregate%s", class.Class)), - Resolve: func(p graphql.ResolveParams) (interface{}, error) { - // pass-through - return p.Source, nil - }, - } - - // Always append Grouped By field - fields["groupedBy"] = &graphql.Field{ - Description: descriptions.AggregateGroupedBy, - Type: groupedByProperty(class), - Resolve: func(p graphql.ResolveParams) (interface{}, error) { - switch typed := p.Source.(type) { - case aggregation.Group: - return typed.GroupedBy, nil - case map[string]interface{}: - return typed["groupedBy"], nil - default: - return nil, fmt.Errorf("groupedBy: unsupported type %T", p.Source) - } - }, - } - - return fields, nil -} - -func metaObject(prefix string) *graphql.Object { - return graphql.NewObject(graphql.ObjectConfig{ - Name: fmt.Sprintf("%sMetaObject", prefix), - Fields: graphql.Fields{ - "count": &graphql.Field{ - Type: graphql.Int, - Resolve: func(p graphql.ResolveParams) (interface{}, error) { - group, ok := p.Source.(aggregation.Group) - if !ok { - return nil, fmt.Errorf("meta count: expected aggregation.Group, got %T", p.Source) - } - - return group.Count, nil - }, - }, - }, - }) -} - -func classPropertyField(dataType schema.DataType, class *models.Class, property *models.Property) (*graphql.Field, error) { - switch dataType { - case schema.DataTypeText: - return makePropertyField(class, property, stringPropertyFields) - case schema.DataTypeInt: - return makePropertyField(class, property, numericPropertyFields) - case schema.DataTypeNumber: - return makePropertyField(class, property, numericPropertyFields) - case schema.DataTypeBoolean: - return makePropertyField(class, property, booleanPropertyFields) - case schema.DataTypeDate: - return makePropertyField(class, property, datePropertyFields) - case schema.DataTypeCRef: - return makePropertyField(class, property, referencePropertyFields) - case schema.DataTypeGeoCoordinates: - // simply skip for now, see gh-729 - return nil, nil - case schema.DataTypePhoneNumber: - // skipping for now, see gh-1088 where it was outscoped - return nil, nil - case schema.DataTypeBlob: - return makePropertyField(class, property, stringPropertyFields) - case schema.DataTypeTextArray: - return makePropertyField(class, property, stringPropertyFields) - case schema.DataTypeIntArray, schema.DataTypeNumberArray: - return makePropertyField(class, property, numericPropertyFields) - case schema.DataTypeBooleanArray: - return makePropertyField(class, property, booleanPropertyFields) - case schema.DataTypeDateArray: - return makePropertyField(class, property, datePropertyFields) - case schema.DataTypeUUID, schema.DataTypeUUIDArray: - // not aggregatable - return nil, nil - case schema.DataTypeObject, schema.DataTypeObjectArray: - // TODO: check if it's aggregable, skip for now - return nil, nil - default: - return nil, fmt.Errorf(schema.ErrorNoSuchDatatype+": %s", dataType) - } -} - -type propertyFieldMaker func(class *models.Class, - property *models.Property, prefix string) *graphql.Object - -func makePropertyField(class *models.Class, property *models.Property, - fieldMaker propertyFieldMaker, -) (*graphql.Field, error) { - prefix := "Aggregate" - return &graphql.Field{ - Description: fmt.Sprintf(`%s"%s"`, descriptions.AggregateProperty, property.Name), - Type: fieldMaker(class, property, prefix), - Resolve: func(p graphql.ResolveParams) (interface{}, error) { - switch typed := p.Source.(type) { - case aggregation.Group: - res, ok := typed.Properties[property.Name] - if !ok { - return nil, fmt.Errorf("missing property '%s'", property.Name) - } - - return res, nil - - default: - return nil, fmt.Errorf("property %s, unsupported type %T", property.Name, p.Source) - } - }, - }, nil -} - -func passThroughResolver(p graphql.ResolveParams) (interface{}, error) { - // bubble up root resolver - return p.Source, nil -} diff --git a/adapters/handlers/graphql/local/aggregate/explore_argument.go b/adapters/handlers/graphql/local/aggregate/explore_argument.go deleted file mode 100644 index 2e23ea6e828729a7bb178c688c3f11f24522e6cb..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/aggregate/explore_argument.go +++ /dev/null @@ -1,25 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package aggregate - -import ( - "github.com/tailor-inc/graphql" - "github.com/weaviate/weaviate/adapters/handlers/graphql/local/common_filters" -) - -func nearVectorArgument(className string) *graphql.ArgumentConfig { - return common_filters.NearVectorArgument("AggregateObjects", className) -} - -func nearObjectArgument(className string) *graphql.ArgumentConfig { - return common_filters.NearObjectArgument("AggregateObjects", className) -} diff --git a/adapters/handlers/graphql/local/aggregate/helpers_for_test.go b/adapters/handlers/graphql/local/aggregate/helpers_for_test.go deleted file mode 100644 index ca5de48c8b26dc6c80060d991a8859d72275fddc..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/aggregate/helpers_for_test.go +++ /dev/null @@ -1,56 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package aggregate - -import ( - "context" - "fmt" - - testhelper "github.com/weaviate/weaviate/adapters/handlers/graphql/test/helper" - "github.com/weaviate/weaviate/entities/aggregation" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/usecases/config" -) - -type mockRequestsLog struct{} - -func (m *mockRequestsLog) Register(first string, second string) { -} - -type mockResolver struct { - testhelper.MockResolver -} - -func newMockResolver(cfg config.Config) *mockResolver { - field, err := Build(&testhelper.CarSchema, cfg, nil) - if err != nil { - panic(fmt.Sprintf("could not build graphql test schema: %s", err)) - } - mockLog := &mockRequestsLog{} - mocker := &mockResolver{} - mocker.RootFieldName = "Aggregate" - mocker.RootField = field - mocker.RootObject = map[string]interface{}{ - "Resolver": Resolver(mocker), - "RequestsLog": mockLog, - "Config": cfg, - } - - return mocker -} - -func (m *mockResolver) Aggregate(ctx context.Context, principal *models.Principal, - params *aggregation.Params, -) (interface{}, error) { - args := m.Called(params) - return args.Get(0), args.Error(1) -} diff --git a/adapters/handlers/graphql/local/aggregate/hybrid_search.go b/adapters/handlers/graphql/local/aggregate/hybrid_search.go deleted file mode 100644 index bd6820f042368665659474e5508884b088fd12f6..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/aggregate/hybrid_search.go +++ /dev/null @@ -1,90 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package aggregate - -import ( - "fmt" - "os" - - "github.com/tailor-inc/graphql" - "github.com/weaviate/weaviate/entities/models" -) - -func hybridArgument(classObject *graphql.Object, - class *models.Class, modulesProvider ModulesProvider, -) *graphql.ArgumentConfig { - prefix := fmt.Sprintf("AggregateObjects%s", class.Class) - return &graphql.ArgumentConfig{ - Type: graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sHybridInpObj", prefix), - Fields: hybridOperands(classObject, class, modulesProvider), - Description: "Hybrid search", - }, - ), - } -} - -func hybridOperands(classObject *graphql.Object, - class *models.Class, modulesProvider ModulesProvider, -) graphql.InputObjectConfigFieldMap { - ss := graphql.NewInputObject(graphql.InputObjectConfig{ - Name: class.Class + "HybridSubSearch", - Fields: hybridSubSearch(classObject, class, modulesProvider), - }) - fieldMap := graphql.InputObjectConfigFieldMap{ - "query": &graphql.InputObjectFieldConfig{ - Description: "Query string", - Type: graphql.String, - }, - "alpha": &graphql.InputObjectFieldConfig{ - Description: "Search weight", - Type: graphql.Float, - }, - "vector": &graphql.InputObjectFieldConfig{ - Description: "Vector search", - Type: graphql.NewList(graphql.Float), - }, - } - - if os.Getenv("ENABLE_EXPERIMENTAL_HYBRID_OPERANDS") != "" { - fieldMap["operands"] = &graphql.InputObjectFieldConfig{ - Description: "Subsearch list", - Type: graphql.NewList(ss), - } - } - - return fieldMap -} - -func hybridSubSearch(classObject *graphql.Object, - class *models.Class, modulesProvider ModulesProvider, -) graphql.InputObjectConfigFieldMap { - prefixName := class.Class + "SubSearch" - - return graphql.InputObjectConfigFieldMap{ - "weight": &graphql.InputObjectFieldConfig{ - Description: "weight, 0 to 1", - Type: graphql.Float, - }, - "sparseSearch": &graphql.InputObjectFieldConfig{ - Description: "Sparse Search", - Type: graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sHybridAggregateBM25InpObj", prefixName), - Fields: bm25Fields(prefixName), - Description: "BM25f search", - }, - ), - }, - } -} diff --git a/adapters/handlers/graphql/local/aggregate/multi_tenancy.go b/adapters/handlers/graphql/local/aggregate/multi_tenancy.go deleted file mode 100644 index 53a75c43ad8dda6b9f8762d612f8584babe2c7b3..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/aggregate/multi_tenancy.go +++ /dev/null @@ -1,24 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package aggregate - -import ( - "github.com/tailor-inc/graphql" - "github.com/weaviate/weaviate/adapters/handlers/graphql/descriptions" -) - -func tenantArgument() *graphql.ArgumentConfig { - return &graphql.ArgumentConfig{ - Description: descriptions.Tenant, - Type: graphql.String, - } -} diff --git a/adapters/handlers/graphql/local/aggregate/properties.go b/adapters/handlers/graphql/local/aggregate/properties.go deleted file mode 100644 index a31a40ef6000945f26fd741f2a38ba00fc010690..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/aggregate/properties.go +++ /dev/null @@ -1,478 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package aggregate - -import ( - "fmt" - - "github.com/tailor-inc/graphql" - "github.com/weaviate/weaviate/adapters/handlers/graphql/descriptions" - "github.com/weaviate/weaviate/entities/aggregation" - "github.com/weaviate/weaviate/entities/models" -) - -func numericPropertyFields(class *models.Class, property *models.Property, prefix string) *graphql.Object { - getMetaIntFields := graphql.Fields{ - "sum": &graphql.Field{ - Name: fmt.Sprintf("%s%s%sSum", prefix, class.Class, property.Name), - Description: descriptions.AggregateSum, - Type: graphql.Float, - Resolve: makeResolveNumericFieldAggregator("sum"), - }, - "minimum": &graphql.Field{ - Name: fmt.Sprintf("%s%s%sMinimum", prefix, class.Class, property.Name), - Description: descriptions.AggregateMin, - Type: graphql.Float, - Resolve: makeResolveNumericFieldAggregator("minimum"), - }, - "maximum": &graphql.Field{ - Name: fmt.Sprintf("%s%s%sMaximum", prefix, class.Class, property.Name), - Description: descriptions.AggregateMax, - Type: graphql.Float, - Resolve: makeResolveNumericFieldAggregator("maximum"), - }, - "mean": &graphql.Field{ - Name: fmt.Sprintf("%s%s%sMean", prefix, class.Class, property.Name), - Description: descriptions.AggregateMean, - Type: graphql.Float, - Resolve: makeResolveNumericFieldAggregator("mean"), - }, - "mode": &graphql.Field{ - Name: fmt.Sprintf("%s%s%sMode", prefix, class.Class, property.Name), - Description: descriptions.AggregateMode, - Type: graphql.Float, - Resolve: makeResolveNumericFieldAggregator("mode"), - }, - "median": &graphql.Field{ - Name: fmt.Sprintf("%s%s%sMedian", prefix, class.Class, property.Name), - Description: descriptions.AggregateMedian, - Type: graphql.Float, - Resolve: makeResolveNumericFieldAggregator("median"), - }, - "count": &graphql.Field{ - Name: fmt.Sprintf("%s%s%sCount", prefix, class.Class, property.Name), - Description: descriptions.AggregateCount, - Type: graphql.Int, - Resolve: makeResolveNumericFieldAggregator("count"), - }, - "type": &graphql.Field{ - Name: fmt.Sprintf("%s%s%sType", prefix, class.Class, property.Name), - Description: descriptions.AggregateCount, - Type: graphql.String, - Resolve: func(p graphql.ResolveParams) (interface{}, error) { - prop, ok := p.Source.(aggregation.Property) - if !ok { - return nil, fmt.Errorf("numerical: type: expected aggregation.Property, got %T", p.Source) - } - - return prop.SchemaType, nil - }, - }, - } - - return graphql.NewObject(graphql.ObjectConfig{ - Name: fmt.Sprintf("%s%s%sObj", prefix, class.Class, property.Name), - Fields: getMetaIntFields, - Description: descriptions.AggregatePropertyObject, - }) -} - -func datePropertyFields(class *models.Class, - property *models.Property, prefix string, -) *graphql.Object { - getMetaDateFields := graphql.Fields{ - "count": &graphql.Field{ - Name: fmt.Sprintf("%s%sCount", prefix, class.Class), - Description: descriptions.AggregateCount, - Type: graphql.Int, - Resolve: makeResolveDateFieldAggregator("count"), - }, - "minimum": &graphql.Field{ - Name: fmt.Sprintf("%s%s%sMinimum", prefix, class.Class, property.Name), - Description: descriptions.AggregateMin, - Type: graphql.String, - Resolve: makeResolveDateFieldAggregator("minimum"), - }, - "maximum": &graphql.Field{ - Name: fmt.Sprintf("%s%s%sMaximum", prefix, class.Class, property.Name), - Description: descriptions.AggregateMax, - Type: graphql.String, - Resolve: makeResolveDateFieldAggregator("maximum"), - }, - "mode": &graphql.Field{ - Name: fmt.Sprintf("%s%s%sMode", prefix, class.Class, property.Name), - Description: descriptions.AggregateMode, - Type: graphql.String, - Resolve: makeResolveDateFieldAggregator("mode"), - }, - "median": &graphql.Field{ - Name: fmt.Sprintf("%s%s%sMedian", prefix, class.Class, property.Name), - Description: descriptions.AggregateMedian, - Type: graphql.String, - Resolve: makeResolveDateFieldAggregator("median"), - }, - } - - return graphql.NewObject(graphql.ObjectConfig{ - Name: fmt.Sprintf("%s%s%sObj", prefix, class.Class, property.Name), - Fields: getMetaDateFields, - Description: descriptions.AggregatePropertyObject, - }) -} - -func referencePropertyFields(class *models.Class, - property *models.Property, prefix string, -) *graphql.Object { - getMetaPointingFields := graphql.Fields{ - "type": &graphql.Field{ - Name: fmt.Sprintf("%s%sType", prefix, class.Class), - Description: descriptions.AggregatePropertyType, - Type: graphql.String, - Resolve: func(p graphql.ResolveParams) (interface{}, error) { - prop, ok := p.Source.(aggregation.Property) - if !ok { - return nil, fmt.Errorf("ref property type: expected aggregation.Property, got %T", - p.Source) - } - - return prop.SchemaType, nil - }, - }, - "pointingTo": &graphql.Field{ - Name: fmt.Sprintf("%s%sPointingTo", prefix, class.Class), - Description: descriptions.AggregateClassPropertyPointingTo, - Type: graphql.NewList(graphql.String), - Resolve: func(p graphql.ResolveParams) (interface{}, error) { - ref, err := extractReferenceAggregation(p.Source) - if err != nil { - return nil, fmt.Errorf("ref property pointingTo: %v", err) - } - - return ref.PointingTo, nil - }, - DeprecationReason: "Experimental, the format will change", - }, - } - - return graphql.NewObject(graphql.ObjectConfig{ - Name: fmt.Sprintf("%s%s%sObj", prefix, class.Class, property.Name), - Fields: getMetaPointingFields, - Description: descriptions.AggregatePropertyObject, - }) -} - -func extractReferenceAggregation(source interface{}) (*aggregation.Reference, error) { - property, ok := source.(aggregation.Property) - if !ok { - return nil, fmt.Errorf("expected aggregation.Property, got %T", source) - } - - if property.Type != aggregation.PropertyTypeReference { - return nil, fmt.Errorf("expected property to be of type reference, got %s", property.Type) - } - - return &property.ReferenceAggregation, nil -} - -func booleanPropertyFields(class *models.Class, - property *models.Property, prefix string, -) *graphql.Object { - getMetaPointingFields := graphql.Fields{ - "count": &graphql.Field{ - Name: fmt.Sprintf("%s%s%sCount", prefix, class.Class, property.Name), - Description: descriptions.AggregatePropertyCount, - Type: graphql.Int, - Resolve: booleanResolver(func(b aggregation.Boolean) interface{} { return b.Count }), - }, - "totalTrue": &graphql.Field{ - Name: fmt.Sprintf("%s%s%sTotalTrue", prefix, class.Class, property.Name), - Description: descriptions.AggregateClassPropertyTotalTrue, - Type: graphql.Int, - Resolve: booleanResolver(func(b aggregation.Boolean) interface{} { return b.TotalTrue }), - }, - "percentageTrue": &graphql.Field{ - Name: fmt.Sprintf("%s%s%sPercentageTrue", prefix, class.Class, property.Name), - Description: descriptions.AggregateClassPropertyPercentageTrue, - Type: graphql.Float, - Resolve: booleanResolver(func(b aggregation.Boolean) interface{} { return b.PercentageTrue }), - }, - "totalFalse": &graphql.Field{ - Name: fmt.Sprintf("%s%s%sTotalFalse", prefix, class.Class, property.Name), - Description: descriptions.AggregateClassPropertyTotalFalse, - Type: graphql.Int, - Resolve: booleanResolver(func(b aggregation.Boolean) interface{} { return b.TotalFalse }), - }, - "percentageFalse": &graphql.Field{ - Name: fmt.Sprintf("%s%s%sPercentageFalse", prefix, class.Class, property.Name), - Description: descriptions.AggregateClassPropertyPercentageFalse, - Type: graphql.Float, - Resolve: booleanResolver(func(b aggregation.Boolean) interface{} { return b.PercentageFalse }), - }, - "type": &graphql.Field{ - Name: fmt.Sprintf("%s%s%sType", prefix, class.Class, property.Name), - Description: descriptions.AggregateCount, - Type: graphql.String, - Resolve: func(p graphql.ResolveParams) (interface{}, error) { - prop, ok := p.Source.(aggregation.Property) - if !ok { - return nil, fmt.Errorf("boolean: type: expected aggregation.Property, got %T", p.Source) - } - - return prop.SchemaType, nil - }, - }, - } - - return graphql.NewObject(graphql.ObjectConfig{ - Name: fmt.Sprintf("%s%s%sObj", prefix, class.Class, property.Name), - Fields: getMetaPointingFields, - Description: descriptions.AggregatePropertyObject, - }) -} - -type booleanExtractorFunc func(aggregation.Boolean) interface{} - -func booleanResolver(extractor booleanExtractorFunc) func(p graphql.ResolveParams) (interface{}, error) { - return func(p graphql.ResolveParams) (interface{}, error) { - boolean, err := extractBooleanAggregation(p.Source) - if err != nil { - return nil, fmt.Errorf("boolean: %v", err) - } - - return extractor(*boolean), nil - } -} - -func extractBooleanAggregation(source interface{}) (*aggregation.Boolean, error) { - property, ok := source.(aggregation.Property) - if !ok { - return nil, fmt.Errorf("expected aggregation.Property, got %T", source) - } - - if property.Type != aggregation.PropertyTypeBoolean { - return nil, fmt.Errorf("expected property to be of type boolean, got %s", property.Type) - } - - return &property.BooleanAggregation, nil -} - -func stringPropertyFields(class *models.Class, - property *models.Property, prefix string, -) *graphql.Object { - getAggregatePointingFields := graphql.Fields{ - "count": &graphql.Field{ - Name: fmt.Sprintf("%s%sCount", prefix, class.Class), - Description: descriptions.AggregatePropertyCount, - Type: graphql.Int, - Resolve: textResolver(func(text aggregation.Text) (interface{}, error) { - return text.Count, nil - }), - }, - "type": &graphql.Field{ - Name: fmt.Sprintf("%s%s%sType", prefix, class.Class, property.Name), - Description: descriptions.AggregateCount, - Type: graphql.String, - Resolve: func(p graphql.ResolveParams) (interface{}, error) { - prop, ok := p.Source.(aggregation.Property) - if !ok { - return nil, fmt.Errorf("text type: expected aggregation.Property, got %T", p.Source) - } - - return prop.SchemaType, nil - }, - }, - "topOccurrences": &graphql.Field{ - Name: fmt.Sprintf("%s%sTopOccurrences", prefix, class.Class), - Description: descriptions.AggregatePropertyTopOccurrences, - Type: graphql.NewList(stringTopOccurrences(class, property, prefix)), - Resolve: textResolver(func(text aggregation.Text) (interface{}, error) { - list := make([]interface{}, len(text.Items)) - for i, to := range text.Items { - list[i] = to - } - - return list, nil - }), - Args: graphql.FieldConfigArgument{ - "limit": &graphql.ArgumentConfig{ - Description: descriptions.First, - Type: graphql.Int, - }, - }, - }, - } - - return graphql.NewObject(graphql.ObjectConfig{ - Name: fmt.Sprintf("%s%s%sObj", prefix, class.Class, property.Name), - Fields: getAggregatePointingFields, - Description: descriptions.AggregatePropertyObject, - }) -} - -type textExtractorFunc func(aggregation.Text) (interface{}, error) - -func textResolver(extractor textExtractorFunc) func(p graphql.ResolveParams) (interface{}, error) { - return func(p graphql.ResolveParams) (interface{}, error) { - text, err := extractTextAggregation(p.Source) - if err != nil { - return nil, fmt.Errorf("text: %v", err) - } - - return extractor(text) - } -} - -func stringTopOccurrences(class *models.Class, - property *models.Property, prefix string, -) *graphql.Object { - getAggregateAggregatePointingFields := graphql.Fields{ - "value": &graphql.Field{ - Name: fmt.Sprintf("%s%s%sTopOccurrencesValue", prefix, class.Class, property.Name), - Description: descriptions.AggregatePropertyTopOccurrencesValue, - Type: graphql.String, - Resolve: textOccurrenceResolver(func(t aggregation.TextOccurrence) interface{} { return t.Value }), - }, - "occurs": &graphql.Field{ - Name: fmt.Sprintf("%s%s%sTopOccurrencesOccurs", prefix, class.Class, property.Name), - Description: descriptions.AggregatePropertyTopOccurrencesOccurs, - Type: graphql.Int, - Resolve: textOccurrenceResolver(func(t aggregation.TextOccurrence) interface{} { return t.Occurs }), - }, - } - - getAggregateAggregatePointing := graphql.ObjectConfig{ - Name: fmt.Sprintf("%s%s%sTopOccurrencesObj", prefix, class.Class, property.Name), - Fields: getAggregateAggregatePointingFields, - Description: descriptions.AggregatePropertyTopOccurrences, - } - - return graphql.NewObject(getAggregateAggregatePointing) -} - -type textOccurrenceExtractorFunc func(aggregation.TextOccurrence) interface{} - -func textOccurrenceResolver(extractor textOccurrenceExtractorFunc) func(p graphql.ResolveParams) (interface{}, error) { - return func(p graphql.ResolveParams) (interface{}, error) { - textOccurrence, ok := p.Source.(aggregation.TextOccurrence) - if !ok { - return nil, fmt.Errorf("textOccurrence: %s: expected aggregation.TextOccurrence, but got %T", - p.Info.FieldName, p.Source) - } - - return extractor(textOccurrence), nil - } -} - -func extractTextAggregation(source interface{}) (aggregation.Text, error) { - property, ok := source.(aggregation.Property) - if !ok { - return aggregation.Text{}, fmt.Errorf("expected aggregation.Property, got %T", source) - } - - if property.Type == aggregation.PropertyTypeNumerical { - // in this case we can only use count - return aggregation.Text{ - Count: property.NumericalAggregations["count"].(int), - }, nil - } - - if property.Type != aggregation.PropertyTypeText { - return aggregation.Text{}, fmt.Errorf("expected property to be of type text, got %s (%#v)", property.Type, property) - } - - return property.TextAggregation, nil -} - -func groupedByProperty(class *models.Class) *graphql.Object { - classProperties := graphql.Fields{ - "path": &graphql.Field{ - Description: descriptions.AggregateGroupedByGroupedByPath, - Type: graphql.NewList(graphql.String), - Resolve: groupedByResolver(func(g *aggregation.GroupedBy) interface{} { return g.Path }), - }, - "value": &graphql.Field{ - Description: descriptions.AggregateGroupedByGroupedByValue, - Type: graphql.String, - Resolve: groupedByResolver(func(g *aggregation.GroupedBy) interface{} { return g.Value }), - }, - } - - classPropertiesObj := graphql.NewObject(graphql.ObjectConfig{ - Name: fmt.Sprintf("Aggregate%sGroupedByObj", class.Class), - Fields: classProperties, - Description: descriptions.AggregateGroupedByObj, - }) - - return classPropertiesObj -} - -type groupedByExtractorFunc func(*aggregation.GroupedBy) interface{} - -func groupedByResolver(extractor groupedByExtractorFunc) func(p graphql.ResolveParams) (interface{}, error) { - return func(p graphql.ResolveParams) (interface{}, error) { - groupedBy, ok := p.Source.(*aggregation.GroupedBy) - if !ok { - return nil, fmt.Errorf("groupedBy: %s: expected aggregation.GroupedBy, but got %T", - p.Info.FieldName, p.Source) - } - - return extractor(groupedBy), nil - } -} - -func makeResolveNumericFieldAggregator(aggregator string) func(p graphql.ResolveParams) (interface{}, error) { - return func(p graphql.ResolveParams) (interface{}, error) { - num, err := extractNumericAggregation(p.Source) - if err != nil { - return nil, fmt.Errorf("numerical aggregator %s: %v", aggregator, err) - } - - return num[aggregator], nil - } -} - -func extractNumericAggregation(source interface{}) (map[string]interface{}, error) { - property, ok := source.(aggregation.Property) - if !ok { - return nil, fmt.Errorf("expected aggregation.Property, got %T", source) - } - - if property.Type != aggregation.PropertyTypeNumerical { - return nil, fmt.Errorf("expected property to be of type numerical, got %s", property.Type) - } - - return property.NumericalAggregations, nil -} - -func makeResolveDateFieldAggregator(aggregator string) func(p graphql.ResolveParams) (interface{}, error) { - return func(p graphql.ResolveParams) (interface{}, error) { - date, err := extractDateAggregation(p.Source) - if err != nil { - return nil, fmt.Errorf("date aggregator %s: %v", aggregator, err) - } - - return date[aggregator], nil - } -} - -func extractDateAggregation(source interface{}) (map[string]interface{}, error) { - property, ok := source.(aggregation.Property) - if !ok { - return nil, fmt.Errorf("expected aggregation.Property, got %T", source) - } - - if property.Type != aggregation.PropertyTypeDate { - return nil, fmt.Errorf("expected property to be of type date, got %s", property.Type) - } - - return property.DateAggregations, nil -} diff --git a/adapters/handlers/graphql/local/aggregate/resolver.go b/adapters/handlers/graphql/local/aggregate/resolver.go deleted file mode 100644 index 5c813d9192d2ff0029c8189ae8846a049b612dca..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/aggregate/resolver.go +++ /dev/null @@ -1,326 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Package aggregate provides the local aggregate graphql endpoint for Weaviate -package aggregate - -import ( - "context" - "fmt" - "strconv" - "strings" - - "github.com/tailor-inc/graphql" - "github.com/tailor-inc/graphql/language/ast" - "github.com/weaviate/weaviate/adapters/handlers/graphql/local/common_filters" - "github.com/weaviate/weaviate/entities/aggregation" - enterrors "github.com/weaviate/weaviate/entities/errors" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/searchparams" -) - -// GroupedByFieldName is a special graphQL field that appears alongside the -// to-be-aggregated props, but doesn't require any processing by the connectors -// itself, as it just displays meta info about the overall aggregation. -const GroupedByFieldName = "groupedBy" - -// Resolver is a local interface that can be composed with other interfaces to -// form the overall GraphQL API main interface. All data-base connectors that -// want to support the Meta feature must implement this interface. -type Resolver interface { - Aggregate(ctx context.Context, principal *models.Principal, info *aggregation.Params) (interface{}, error) -} - -// RequestsLog is a local abstraction on the RequestsLog that needs to be -// provided to the graphQL API in order to log Local.Get queries. -type RequestsLog interface { - Register(requestType string, identifier string) -} - -func makeResolveClass(modulesProvider ModulesProvider, class *models.Class) graphql.FieldResolveFn { - return func(p graphql.ResolveParams) (interface{}, error) { - res, err := resolveAggregate(p, modulesProvider, class) - if err != nil { - return res, enterrors.NewErrGraphQLUser(err, "Aggregate", schema.ClassName(p.Info.FieldName).String()) - } - return res, nil - } -} - -func resolveAggregate(p graphql.ResolveParams, modulesProvider ModulesProvider, class *models.Class) (interface{}, error) { - className := schema.ClassName(p.Info.FieldName) - source, ok := p.Source.(map[string]interface{}) - if !ok { - return nil, fmt.Errorf("expected source to be a map, but was %t", p.Source) - } - - resolver, ok := source["Resolver"].(Resolver) - if !ok { - return nil, fmt.Errorf("expected source to contain a usable Resolver, but was %t", p.Source) - } - - // There can only be exactly one ast.Field; it is the class name. - if len(p.Info.FieldASTs) != 1 { - panic("Only one Field expected here") - } - - selections := p.Info.FieldASTs[0].SelectionSet - properties, includeMeta, err := extractProperties(selections) - if err != nil { - return nil, fmt.Errorf("could not extract properties for class '%s': %w", className, err) - } - - groupBy, err := extractGroupBy(p.Args, p.Info.FieldName) - if err != nil { - return nil, fmt.Errorf("could not extract groupBy path: %w", err) - } - - limit, err := extractLimit(p.Args) - if err != nil { - return nil, fmt.Errorf("could not extract limit: %w", err) - } - - objectLimit, err := extractObjectLimit(p.Args) - if objectLimit != nil && *objectLimit <= 0 { - return nil, fmt.Errorf("objectLimit must be a positive integer") - } - if err != nil { - return nil, fmt.Errorf("could not extract objectLimit: %w", err) - } - - filters, err := common_filters.ExtractFilters(p.Args, p.Info.FieldName) - if err != nil { - return nil, fmt.Errorf("could not extract filters: %w", err) - } - - var nearVectorParams *searchparams.NearVector - if nearVector, ok := p.Args["nearVector"]; ok { - p, err := common_filters.ExtractNearVector(nearVector.(map[string]interface{})) - if err != nil { - return nil, fmt.Errorf("failed to extract nearVector params: %w", err) - } - nearVectorParams = &p - } - - var nearObjectParams *searchparams.NearObject - if nearObject, ok := p.Args["nearObject"]; ok { - p, err := common_filters.ExtractNearObject(nearObject.(map[string]interface{})) - if err != nil { - return nil, fmt.Errorf("failed to extract nearObject params: %w", err) - } - nearObjectParams = &p - } - - var moduleParams map[string]interface{} - if modulesProvider != nil { - extractedParams := modulesProvider.ExtractSearchParams(p.Args, class.Class) - if len(extractedParams) > 0 { - moduleParams = extractedParams - } - } - - // Extract hybrid search params from the processed query - // Everything hybrid can go in another namespace AFTER modulesprovider is - // refactored - var hybridParams *searchparams.HybridSearch - if hybrid, ok := p.Args["hybrid"]; ok { - p, err := common_filters.ExtractHybridSearch(hybrid.(map[string]interface{}), false) - if err != nil { - return nil, fmt.Errorf("failed to extract hybrid params: %w", err) - } - hybridParams = p - } - - var tenant string - if tk, ok := p.Args["tenant"]; ok { - tenant = tk.(string) - } - - params := &aggregation.Params{ - Filters: filters, - ClassName: className, - Properties: properties, - GroupBy: groupBy, - IncludeMetaCount: includeMeta, - Limit: limit, - ObjectLimit: objectLimit, - NearVector: nearVectorParams, - NearObject: nearObjectParams, - ModuleParams: moduleParams, - Hybrid: hybridParams, - Tenant: tenant, - } - - // we might support objectLimit without nearMedia filters later, e.g. with sort - if params.ObjectLimit != nil && !validateObjectLimitUsage(params) { - return nil, fmt.Errorf("objectLimit can only be used with a near or hybrid filter") - } - - res, err := resolver.Aggregate(p.Context, principalFromContext(p.Context), params) - if err != nil { - return nil, err - } - - switch parsed := res.(type) { - case *aggregation.Result: - return parsed.Groups, nil - default: - return res, nil - } -} - -func extractProperties(selections *ast.SelectionSet) ([]aggregation.ParamProperty, bool, error) { - properties := []aggregation.ParamProperty{} - var includeMeta bool - - for _, selection := range selections.Selections { - field := selection.(*ast.Field) - name := field.Name.Value - if name == GroupedByFieldName { - // in the graphQL API we show the "groupedBy" field alongside various - // properties, however, we don't have to include it here, as we don't - // won't to perform aggregations on it. - // If we didn't exclude it we'd run into errors down the line, because - // the connector would look for a "groupedBy" prop on the specific class - // which doesn't exist. - - continue - } - - if name == "meta" { - includeMeta = true - continue - } - - if name == "__typename" { - continue - } - - name = strings.ToLower(string(name[0:1])) + string(name[1:]) - property := aggregation.ParamProperty{Name: schema.PropertyName(name)} - aggregators, err := extractAggregators(field.SelectionSet) - if err != nil { - return nil, false, err - } - - property.Aggregators = aggregators - properties = append(properties, property) - } - - return properties, includeMeta, nil -} - -func extractAggregators(selections *ast.SelectionSet) ([]aggregation.Aggregator, error) { - if selections == nil { - return nil, nil - } - analyses := []aggregation.Aggregator{} - for _, selection := range selections.Selections { - field := selection.(*ast.Field) - name := field.Name.Value - if name == "__typename" { - continue - } - property, err := aggregation.ParseAggregatorProp(name) - if err != nil { - return nil, err - } - - if property.String() == aggregation.NewTopOccurrencesAggregator(nil).String() { - // a top occurrence, so we need to check if we have a limit argument - if overwrite := extractLimitFromArgs(field.Arguments); overwrite != nil { - property.Limit = overwrite - } - } - - analyses = append(analyses, property) - } - - return analyses, nil -} - -func extractGroupBy(args map[string]interface{}, rootClass string) (*filters.Path, error) { - groupBy, ok := args["groupBy"] - if !ok { - // not set means the user is not interested in grouping (former Meta) - return nil, nil - } - - pathSegments, ok := groupBy.([]interface{}) - if !ok { - return nil, fmt.Errorf("no groupBy must be a list, instead got: %#v", groupBy) - } - - return filters.ParsePath(pathSegments, rootClass) -} - -func principalFromContext(ctx context.Context) *models.Principal { - principal := ctx.Value("principal") - if principal == nil { - return nil - } - - return principal.(*models.Principal) -} - -func extractLimit(args map[string]interface{}) (*int, error) { - limit, ok := args["limit"] - if !ok { - // not set means the user is not interested and the UC should use a reasonable default - return nil, nil - } - - limitInt, ok := limit.(int) - if !ok { - return nil, fmt.Errorf("limit must be an int, instead got: %#v", limit) - } - - return &limitInt, nil -} - -func extractObjectLimit(args map[string]interface{}) (*int, error) { - objectLimit, ok := args["objectLimit"] - if !ok { - return nil, nil - } - - objectLimitInt, ok := objectLimit.(int) - if !ok { - return nil, fmt.Errorf("objectLimit must be an int, instead got: %#v", objectLimit) - } - - return &objectLimitInt, nil -} - -func extractLimitFromArgs(args []*ast.Argument) *int { - for _, arg := range args { - if arg.Name.Value != "limit" { - continue - } - - v, ok := arg.Value.GetValue().(string) - if ok { - asInt, _ := strconv.Atoi(v) - return &asInt - } - } - - return nil -} - -func validateObjectLimitUsage(params *aggregation.Params) bool { - return params.NearObject != nil || - params.NearVector != nil || - len(params.ModuleParams) > 0 || - params.Hybrid != nil -} diff --git a/adapters/handlers/graphql/local/aggregate/resolver_test.go b/adapters/handlers/graphql/local/aggregate/resolver_test.go deleted file mode 100644 index e33ddb6fbea015e2c4c964fb99869134579df88d..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/aggregate/resolver_test.go +++ /dev/null @@ -1,967 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package aggregate - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/entities/aggregation" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/searchparams" - "github.com/weaviate/weaviate/usecases/config" -) - -type testCase struct { - name string - query string - expectedProps []aggregation.ParamProperty - resolverReturn interface{} - expectedResults []result - expectedGroupBy *filters.Path - expectedWhereFilter *filters.LocalFilter - expectedNearObjectFilter *searchparams.NearObject - expectedNearVectorFilter *searchparams.NearVector - expectedIncludeMetaCount bool - expectedLimit *int - expectedObjectLimit *int -} - -type testCases []testCase - -type result struct { - pathToField []string - expectedValue interface{} -} - -func groupCarByMadeByManufacturerName() *filters.Path { - return &filters.Path{ - Class: schema.ClassName("Car"), - Property: schema.PropertyName("madeBy"), - Child: &filters.Path{ - Class: schema.ClassName("Manufacturer"), - Property: schema.PropertyName("name"), - }, - } -} - -func Test_Resolve(t *testing.T) { - t.Parallel() - - tests := testCases{ - testCase{ - name: "for gh-758 (multiple operands)", - query: ` - { - Aggregate { - Car(where:{ - operator:Or - operands:[{ - valueText:"Fast", - operator:Equal, - path:["modelName"] - }, { - valueText:"Slow", - operator:Equal, - path:["modelName"] - }] - }) { - __typename - modelName { - __typename - count - } - } - } - }`, - expectedProps: []aggregation.ParamProperty{ - { - Name: "modelName", - Aggregators: []aggregation.Aggregator{aggregation.CountAggregator}, - }, - }, - resolverReturn: []aggregation.Group{ - { - Properties: map[string]aggregation.Property{ - "modelName": { - Type: aggregation.PropertyTypeText, - TextAggregation: aggregation.Text{ - Count: 20, - }, - }, - }, - }, - }, - expectedWhereFilter: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorOr, - Operands: []filters.Clause{ - { - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: schema.ClassName("Car"), - Property: schema.PropertyName("modelName"), - }, - Value: &filters.Value{ - Value: "Fast", - Type: schema.DataTypeText, - }, - }, - { - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: schema.ClassName("Car"), - Property: schema.PropertyName("modelName"), - }, - Value: &filters.Value{ - Value: "Slow", - Type: schema.DataTypeText, - }, - }, - }, - }, - }, - - expectedGroupBy: nil, - expectedResults: []result{{ - pathToField: []string{"Aggregate", "Car"}, - expectedValue: []interface{}{ - map[string]interface{}{ - "__typename": "AggregateCar", - "modelName": map[string]interface{}{ - "count": 20, - "__typename": "AggregateCarmodelNameObj", - }, - }, - }, - }}, - }, - testCase{ - name: "without grouping prop", - query: `{ Aggregate { Car { horsepower { mean } } } }`, - expectedProps: []aggregation.ParamProperty{ - { - Name: "horsepower", - Aggregators: []aggregation.Aggregator{aggregation.MeanAggregator}, - }, - }, - resolverReturn: []aggregation.Group{ - { - Properties: map[string]aggregation.Property{ - "horsepower": { - Type: aggregation.PropertyTypeNumerical, - NumericalAggregations: map[string]interface{}{ - "mean": 275.7773, - }, - }, - }, - }, - }, - - expectedGroupBy: nil, - expectedResults: []result{{ - pathToField: []string{"Aggregate", "Car"}, - expectedValue: []interface{}{ - map[string]interface{}{ - "horsepower": map[string]interface{}{"mean": 275.7773}, - }, - }, - }}, - }, - testCase{ - name: "setting limits overall", - query: `{ Aggregate { Car(limit:20) { horsepower { mean } } } }`, - expectedProps: []aggregation.ParamProperty{ - { - Name: "horsepower", - Aggregators: []aggregation.Aggregator{aggregation.MeanAggregator}, - }, - }, - resolverReturn: []aggregation.Group{ - { - Properties: map[string]aggregation.Property{ - "horsepower": { - Type: aggregation.PropertyTypeNumerical, - NumericalAggregations: map[string]interface{}{ - "mean": 275.7773, - }, - }, - }, - }, - }, - - expectedGroupBy: nil, - expectedLimit: ptInt(20), - expectedResults: []result{{ - pathToField: []string{"Aggregate", "Car"}, - expectedValue: []interface{}{ - map[string]interface{}{ - "horsepower": map[string]interface{}{"mean": 275.7773}, - }, - }, - }}, - }, - testCase{ - name: "with props formerly contained only in Meta", - query: `{ Aggregate { Car { - stillInProduction { type count totalTrue percentageTrue totalFalse percentageFalse } - modelName { type count topOccurrences { value occurs } } - madeBy { type pointingTo } - meta { count } - } } } `, - expectedIncludeMetaCount: true, - expectedProps: []aggregation.ParamProperty{ - { - Name: "stillInProduction", - Aggregators: []aggregation.Aggregator{ - aggregation.TypeAggregator, - aggregation.CountAggregator, - aggregation.TotalTrueAggregator, - aggregation.PercentageTrueAggregator, - aggregation.TotalFalseAggregator, - aggregation.PercentageFalseAggregator, - }, - }, - { - Name: "modelName", - Aggregators: []aggregation.Aggregator{ - aggregation.TypeAggregator, - aggregation.CountAggregator, - aggregation.NewTopOccurrencesAggregator(ptInt(5)), - }, - }, - { - Name: "madeBy", - Aggregators: []aggregation.Aggregator{ - aggregation.TypeAggregator, - aggregation.PointingToAggregator, - }, - }, - }, - resolverReturn: []aggregation.Group{ - { - Count: 10, - Properties: map[string]aggregation.Property{ - "stillInProduction": { - SchemaType: "boolean", - Type: aggregation.PropertyTypeBoolean, - BooleanAggregation: aggregation.Boolean{ - TotalTrue: 23, - TotalFalse: 17, - PercentageTrue: 60, - PercentageFalse: 40, - Count: 40, - }, - }, - "modelName": { - SchemaType: "string", - Type: aggregation.PropertyTypeText, - TextAggregation: aggregation.Text{ - Count: 40, - Items: []aggregation.TextOccurrence{ - { - Value: "fastcar", - Occurs: 39, - }, - { - Value: "slowcar", - Occurs: 1, - }, - }, - }, - }, - "madeBy": { - SchemaType: "cref", - Type: aggregation.PropertyTypeReference, - ReferenceAggregation: aggregation.Reference{ - PointingTo: []string{"Manufacturer"}, - }, - }, - }, - }, - }, - - expectedGroupBy: nil, - expectedResults: []result{{ - pathToField: []string{"Aggregate", "Car"}, - expectedValue: []interface{}{ - map[string]interface{}{ - "stillInProduction": map[string]interface{}{ - "type": "boolean", - "totalTrue": 23, - "totalFalse": 17, - "percentageTrue": 60.0, - "percentageFalse": 40.0, - "count": 40, - }, - "modelName": map[string]interface{}{ - "count": 40, - "type": "string", - "topOccurrences": []interface{}{ - map[string]interface{}{ - "value": "fastcar", - "occurs": 39, - }, - map[string]interface{}{ - "value": "slowcar", - "occurs": 1, - }, - }, - }, - "madeBy": map[string]interface{}{ - "type": "cref", - "pointingTo": []interface{}{"Manufacturer"}, - }, - "meta": map[string]interface{}{ - "count": 10, - }, - }, - }, - }}, - }, - testCase{ - name: "with custom limit in topOccurrences", - query: `{ Aggregate { Car { - modelName { topOccurrences(limit: 7) { value occurs } } - } } } `, - expectedProps: []aggregation.ParamProperty{ - { - Name: "modelName", - Aggregators: []aggregation.Aggregator{ - aggregation.NewTopOccurrencesAggregator(ptInt(7)), - }, - }, - }, - resolverReturn: []aggregation.Group{ - { - Count: 10, - Properties: map[string]aggregation.Property{ - "modelName": { - SchemaType: "string", - Type: aggregation.PropertyTypeText, - TextAggregation: aggregation.Text{ - Items: []aggregation.TextOccurrence{ - { - Value: "fastcar", - Occurs: 39, - }, - { - Value: "slowcar", - Occurs: 1, - }, - }, - }, - }, - }, - }, - }, - - expectedGroupBy: nil, - expectedResults: []result{{ - pathToField: []string{"Aggregate", "Car"}, - expectedValue: []interface{}{ - map[string]interface{}{ - "modelName": map[string]interface{}{ - "topOccurrences": []interface{}{ - map[string]interface{}{ - "value": "fastcar", - "occurs": 39, - }, - map[string]interface{}{ - "value": "slowcar", - "occurs": 1, - }, - }, - }, - }, - }, - }}, - }, - testCase{ - name: "single prop: mean (with type)", - query: `{ Aggregate { Car(groupBy:["madeBy", "Manufacturer", "name"]) { horsepower { mean type } } } }`, - expectedProps: []aggregation.ParamProperty{ - { - Name: "horsepower", - Aggregators: []aggregation.Aggregator{aggregation.MeanAggregator, aggregation.TypeAggregator}, - }, - }, - resolverReturn: []aggregation.Group{ - { - Properties: map[string]aggregation.Property{ - "horsepower": { - Type: aggregation.PropertyTypeNumerical, - SchemaType: "int", - NumericalAggregations: map[string]interface{}{ - "mean": 275.7773, - }, - }, - }, - }, - }, - - expectedGroupBy: groupCarByMadeByManufacturerName(), - expectedResults: []result{{ - pathToField: []string{"Aggregate", "Car"}, - expectedValue: []interface{}{ - map[string]interface{}{ - "horsepower": map[string]interface{}{ - "mean": 275.7773, - "type": "int", - }, - }, - }, - }}, - }, - - testCase{ - name: "single prop: mean with groupedBy path/value", - query: `{ Aggregate { Car(groupBy:["madeBy", "Manufacturer", "name"]) { horsepower { mean } groupedBy { value path } } } }`, - expectedProps: []aggregation.ParamProperty{ - { - Name: "horsepower", - Aggregators: []aggregation.Aggregator{aggregation.MeanAggregator}, - }, - }, - resolverReturn: []aggregation.Group{ - { - GroupedBy: &aggregation.GroupedBy{ - Path: []string{"madeBy", "Manufacturer", "name"}, - Value: "best-manufacturer", - }, - Properties: map[string]aggregation.Property{ - "horsepower": { - Type: aggregation.PropertyTypeNumerical, - NumericalAggregations: map[string]interface{}{ - "mean": 275.7773, - }, - }, - }, - }, - }, - expectedGroupBy: groupCarByMadeByManufacturerName(), - expectedResults: []result{{ - pathToField: []string{"Aggregate", "Car"}, - expectedValue: []interface{}{ - map[string]interface{}{ - "horsepower": map[string]interface{}{"mean": 275.7773}, - "groupedBy": map[string]interface{}{ - "path": []interface{}{"madeBy", "Manufacturer", "name"}, - "value": "best-manufacturer", - }, - }, - }, - }}, - }, - - testCase{ - name: "single prop: mean with a where filter", - query: `{ - Aggregate { - Car( - groupBy:["madeBy", "Manufacturer", "name"] - where: { - operator: LessThan, - valueInt: 200, - path: ["horsepower"], - } - ) { - horsepower { - mean - } - } - } - }`, - expectedProps: []aggregation.ParamProperty{ - { - Name: "horsepower", - Aggregators: []aggregation.Aggregator{aggregation.MeanAggregator}, - }, - }, - resolverReturn: []aggregation.Group{ - { - GroupedBy: &aggregation.GroupedBy{ - Path: []string{"madeBy", "Manufacturer", "name"}, - Value: "best-manufacturer", - }, - Properties: map[string]aggregation.Property{ - "horsepower": { - Type: aggregation.PropertyTypeNumerical, - NumericalAggregations: map[string]interface{}{ - "mean": 275.7773, - }, - }, - }, - }, - }, - expectedGroupBy: groupCarByMadeByManufacturerName(), - expectedResults: []result{{ - pathToField: []string{"Aggregate", "Car"}, - expectedValue: []interface{}{ - map[string]interface{}{ - "horsepower": map[string]interface{}{ - "mean": 275.7773, - }, - }, - }, - }}, - expectedWhereFilter: &filters.LocalFilter{ - Root: &filters.Clause{ - On: &filters.Path{ - Class: schema.ClassName("Car"), - Property: schema.PropertyName("horsepower"), - }, - Value: &filters.Value{ - Value: 200, - Type: schema.DataTypeInt, - }, - Operator: filters.OperatorLessThan, - }, - }, - }, - - testCase{ - name: "all int props", - query: `{ Aggregate { Car(groupBy:["madeBy", "Manufacturer", "name"]) { horsepower { mean, median, mode, maximum, minimum, count, sum } } } }`, - expectedProps: []aggregation.ParamProperty{ - { - Name: "horsepower", - Aggregators: []aggregation.Aggregator{aggregation.MeanAggregator, aggregation.MedianAggregator, aggregation.ModeAggregator, aggregation.MaximumAggregator, aggregation.MinimumAggregator, aggregation.CountAggregator, aggregation.SumAggregator}, - }, - }, - resolverReturn: []aggregation.Group{ - { - GroupedBy: &aggregation.GroupedBy{ - Path: []string{"madeBy", "Manufacturer", "name"}, - Value: "best-manufacturer", - }, - Properties: map[string]aggregation.Property{ - "horsepower": { - Type: aggregation.PropertyTypeNumerical, - NumericalAggregations: map[string]interface{}{ - "maximum": 610.0, - "minimum": 89.0, - "mean": 275.7, - "median": 289.0, - "mode": 115.0, - "count": 23, - "sum": 6343.0, - }, - }, - }, - }, - }, - expectedGroupBy: groupCarByMadeByManufacturerName(), - expectedResults: []result{ - { - pathToField: []string{"Aggregate", "Car"}, - expectedValue: []interface{}{ - map[string]interface{}{ - "horsepower": map[string]interface{}{ - "maximum": 610.0, - "minimum": 89.0, - "mean": 275.7, - "median": 289.0, - "mode": 115.0, - "count": 23, - "sum": 6343.0, - }, - }, - }, - }, - }, - }, - - testCase{ - name: "single prop: string", - query: `{ Aggregate { Car(groupBy:["madeBy", "Manufacturer", "name"]) { modelName { count } } } }`, - expectedProps: []aggregation.ParamProperty{ - { - Name: "modelName", - Aggregators: []aggregation.Aggregator{aggregation.CountAggregator}, - }, - }, - resolverReturn: []aggregation.Group{ - { - GroupedBy: &aggregation.GroupedBy{ - Path: []string{"madeBy", "Manufacturer", "name"}, - Value: "best-manufacturer", - }, - Properties: map[string]aggregation.Property{ - "modelName": { - Type: aggregation.PropertyTypeText, - TextAggregation: aggregation.Text{ - Count: 7, - }, - }, - }, - }, - }, - expectedGroupBy: groupCarByMadeByManufacturerName(), - expectedResults: []result{{ - pathToField: []string{"Aggregate", "Car"}, - expectedValue: []interface{}{ - map[string]interface{}{ - "modelName": map[string]interface{}{ - "count": 7, - }, - }, - }, - }}, - }, - - testCase{ - name: "with objectLimit + nearObject (distance)", - query: ` - { - Aggregate{ - Car( - objectLimit: 1 - nearObject: { - id: "123" - distance: 0.3 - } - ) { - modelName { - count - } - } - } - } - `, - expectedProps: []aggregation.ParamProperty{ - { - Name: "modelName", - Aggregators: []aggregation.Aggregator{aggregation.CountAggregator}, - }, - }, - expectedObjectLimit: ptInt(1), - expectedNearObjectFilter: &searchparams.NearObject{ - ID: "123", - Beacon: "", - Distance: 0.3, - WithDistance: true, - }, - resolverReturn: []aggregation.Group{ - { - Properties: map[string]aggregation.Property{ - "modelName": { - Type: aggregation.PropertyTypeText, - TextAggregation: aggregation.Text{ - Count: 7, - }, - }, - }, - }, - }, - expectedResults: []result{{ - pathToField: []string{"Aggregate", "Car"}, - expectedValue: []interface{}{ - map[string]interface{}{ - "modelName": map[string]interface{}{ - "count": 7, - }, - }, - }, - }}, - }, - - testCase{ - name: "with objectLimit + nearObject (certainty)", - query: ` - { - Aggregate{ - Car( - objectLimit: 1 - nearObject: { - id: "123" - certainty: 0.7 - } - ) { - modelName { - count - } - } - } - } - `, - expectedProps: []aggregation.ParamProperty{ - { - Name: "modelName", - Aggregators: []aggregation.Aggregator{aggregation.CountAggregator}, - }, - }, - expectedObjectLimit: ptInt(1), - expectedNearObjectFilter: &searchparams.NearObject{ - ID: "123", - Beacon: "", - Certainty: 0.7, - }, - resolverReturn: []aggregation.Group{ - { - Properties: map[string]aggregation.Property{ - "modelName": { - Type: aggregation.PropertyTypeText, - TextAggregation: aggregation.Text{ - Count: 7, - }, - }, - }, - }, - }, - expectedResults: []result{{ - pathToField: []string{"Aggregate", "Car"}, - expectedValue: []interface{}{ - map[string]interface{}{ - "modelName": map[string]interface{}{ - "count": 7, - }, - }, - }, - }}, - }, - - testCase{ - name: "with objectLimit + nearVector (distance)", - query: ` - { - Aggregate{ - Car( - objectLimit: 1 - nearVector: { - vector: [1, 2, 3] - distance: 0.3 - } - ) { - modelName { - count - } - } - } - } - `, - expectedProps: []aggregation.ParamProperty{ - { - Name: "modelName", - Aggregators: []aggregation.Aggregator{aggregation.CountAggregator}, - }, - }, - expectedObjectLimit: ptInt(1), - expectedNearVectorFilter: &searchparams.NearVector{ - Vector: []float32{1, 2, 3}, - Distance: 0.3, - WithDistance: true, - }, - resolverReturn: []aggregation.Group{ - { - Properties: map[string]aggregation.Property{ - "modelName": { - Type: aggregation.PropertyTypeText, - TextAggregation: aggregation.Text{ - Count: 7, - }, - }, - }, - }, - }, - expectedResults: []result{{ - pathToField: []string{"Aggregate", "Car"}, - expectedValue: []interface{}{ - map[string]interface{}{ - "modelName": map[string]interface{}{ - "count": 7, - }, - }, - }, - }}, - }, - - testCase{ - name: "with objectLimit + nearVector (certainty)", - query: ` - { - Aggregate{ - Car( - objectLimit: 1 - nearVector: { - vector: [1, 2, 3] - certainty: 0.7 - } - ) { - modelName { - count - } - } - } - } - `, - expectedProps: []aggregation.ParamProperty{ - { - Name: "modelName", - Aggregators: []aggregation.Aggregator{aggregation.CountAggregator}, - }, - }, - expectedObjectLimit: ptInt(1), - expectedNearVectorFilter: &searchparams.NearVector{ - Vector: []float32{1, 2, 3}, - Certainty: 0.7, - }, - resolverReturn: []aggregation.Group{ - { - Properties: map[string]aggregation.Property{ - "modelName": { - Type: aggregation.PropertyTypeText, - TextAggregation: aggregation.Text{ - Count: 7, - }, - }, - }, - }, - }, - expectedResults: []result{{ - pathToField: []string{"Aggregate", "Car"}, - expectedValue: []interface{}{ - map[string]interface{}{ - "modelName": map[string]interface{}{ - "count": 7, - }, - }, - }, - }}, - }, - testCase{ - name: "[deprecated string] for gh-758 (multiple operands)", - query: ` - { - Aggregate { - Car(where:{ - operator:Or - operands:[{ - valueString:"Fast", - operator:Equal, - path:["modelName"] - }, { - valueString:"Slow", - operator:Equal, - path:["modelName"] - }] - }) { - __typename - modelName { - __typename - count - } - } - } - }`, - expectedProps: []aggregation.ParamProperty{ - { - Name: "modelName", - Aggregators: []aggregation.Aggregator{aggregation.CountAggregator}, - }, - }, - resolverReturn: []aggregation.Group{ - { - Properties: map[string]aggregation.Property{ - "modelName": { - Type: aggregation.PropertyTypeText, - TextAggregation: aggregation.Text{ - Count: 20, - }, - }, - }, - }, - }, - expectedWhereFilter: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorOr, - Operands: []filters.Clause{ - { - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: schema.ClassName("Car"), - Property: schema.PropertyName("modelName"), - }, - Value: &filters.Value{ - Value: "Fast", - Type: schema.DataTypeString, - }, - }, - { - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: schema.ClassName("Car"), - Property: schema.PropertyName("modelName"), - }, - Value: &filters.Value{ - Value: "Slow", - Type: schema.DataTypeString, - }, - }, - }, - }, - }, - - expectedGroupBy: nil, - expectedResults: []result{{ - pathToField: []string{"Aggregate", "Car"}, - expectedValue: []interface{}{ - map[string]interface{}{ - "__typename": "AggregateCar", - "modelName": map[string]interface{}{ - "count": 20, - "__typename": "AggregateCarmodelNameObj", - }, - }, - }, - }}, - }, - } - - tests.AssertExtraction(t, "Car") -} - -func (tests testCases) AssertExtraction(t *testing.T, className string) { - for _, testCase := range tests { - t.Run(testCase.name, func(t *testing.T) { - resolver := newMockResolver(config.Config{}) - - expectedParams := &aggregation.Params{ - ClassName: schema.ClassName(className), - Properties: testCase.expectedProps, - GroupBy: testCase.expectedGroupBy, - Filters: testCase.expectedWhereFilter, - NearObject: testCase.expectedNearObjectFilter, - NearVector: testCase.expectedNearVectorFilter, - IncludeMetaCount: testCase.expectedIncludeMetaCount, - Limit: testCase.expectedLimit, - ObjectLimit: testCase.expectedObjectLimit, - } - - resolver.On("Aggregate", expectedParams). - Return(testCase.resolverReturn, nil).Once() - - result := resolver.AssertResolve(t, testCase.query) - - for _, expectedResult := range testCase.expectedResults { - value := result.Get(expectedResult.pathToField...).Result - - assert.Equal(t, expectedResult.expectedValue, value) - } - }) - } -} - -func ptInt(in int) *int { - return &in -} diff --git a/adapters/handlers/graphql/local/aggregate/sparse_search.go b/adapters/handlers/graphql/local/aggregate/sparse_search.go deleted file mode 100644 index 4529d0d50b5b56bb26e4b6bb411b273833d78def..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/aggregate/sparse_search.go +++ /dev/null @@ -1,29 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package aggregate - -import ( - "github.com/tailor-inc/graphql" -) - -func bm25Fields(prefix string) graphql.InputObjectConfigFieldMap { - return graphql.InputObjectConfigFieldMap{ - "query": &graphql.InputObjectFieldConfig{ - Description: "The query to search for", - Type: graphql.String, - }, - "properties": &graphql.InputObjectFieldConfig{ - Description: "The properties to search in", - Type: graphql.NewList(graphql.String), - }, - } -} diff --git a/adapters/handlers/graphql/local/common_filters/bm25.go b/adapters/handlers/graphql/local/common_filters/bm25.go deleted file mode 100644 index 2a089b38307521794f3d39b66e204752e5c990ad..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/common_filters/bm25.go +++ /dev/null @@ -1,38 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package common_filters - -import "github.com/weaviate/weaviate/entities/searchparams" - -// ExtractBM25 -func ExtractBM25(source map[string]interface{}, explainScore bool) searchparams.KeywordRanking { - var args searchparams.KeywordRanking - - p, ok := source["properties"] - if ok { - rawSlice := p.([]interface{}) - args.Properties = make([]string, len(rawSlice)) - for i, raw := range rawSlice { - args.Properties[i] = raw.(string) - } - } - - query, ok := source["query"] - if ok { - args.Query = query.(string) - } - - args.AdditionalExplanations = explainScore - args.Type = "bm25" - - return args -} diff --git a/adapters/handlers/graphql/local/common_filters/filters.go b/adapters/handlers/graphql/local/common_filters/filters.go deleted file mode 100644 index 16e5ab178ddc9138aff8bf52c7ed9035908ecd3a..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/common_filters/filters.go +++ /dev/null @@ -1,141 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Package common_filters provides the filters for the graphql endpoint for Weaviate -package common_filters - -import ( - "fmt" - - "github.com/tailor-inc/graphql" - "github.com/weaviate/weaviate/adapters/handlers/graphql/descriptions" -) - -// The filters common to Local->Get and Local->Meta queries. -func BuildNew(path string) graphql.InputObjectConfigFieldMap { - commonFilters := graphql.InputObjectConfigFieldMap{ - "operator": &graphql.InputObjectFieldConfig{ - Type: graphql.NewEnum(graphql.EnumConfig{ - Name: fmt.Sprintf("%sWhereOperatorEnum", path), - Values: graphql.EnumValueConfigMap{ - "And": &graphql.EnumValueConfig{}, - "Like": &graphql.EnumValueConfig{}, - "Or": &graphql.EnumValueConfig{}, - "Equal": &graphql.EnumValueConfig{}, - "Not": &graphql.EnumValueConfig{}, - "NotEqual": &graphql.EnumValueConfig{}, - "GreaterThan": &graphql.EnumValueConfig{}, - "GreaterThanEqual": &graphql.EnumValueConfig{}, - "LessThan": &graphql.EnumValueConfig{}, - "LessThanEqual": &graphql.EnumValueConfig{}, - "WithinGeoRange": &graphql.EnumValueConfig{}, - "IsNull": &graphql.EnumValueConfig{}, - "ContainsAny": &graphql.EnumValueConfig{}, - "ContainsAll": &graphql.EnumValueConfig{}, - }, - Description: descriptions.WhereOperatorEnum, - }), - Description: descriptions.WhereOperator, - }, - "path": &graphql.InputObjectFieldConfig{ - Type: graphql.NewList(graphql.String), - Description: descriptions.WherePath, - }, - "valueInt": &graphql.InputObjectFieldConfig{ - Type: newValueIntType(path), - Description: descriptions.WhereValueInt, - }, - "valueNumber": &graphql.InputObjectFieldConfig{ - Type: newValueNumberType(path), - Description: descriptions.WhereValueNumber, - }, - "valueBoolean": &graphql.InputObjectFieldConfig{ - Type: newValueBooleanType(path), - Description: descriptions.WhereValueBoolean, - }, - "valueString": &graphql.InputObjectFieldConfig{ - Type: newValueStringType(path), - Description: descriptions.WhereValueString, - }, - "valueText": &graphql.InputObjectFieldConfig{ - Type: newValueTextType(path), - Description: descriptions.WhereValueText, - }, - "valueDate": &graphql.InputObjectFieldConfig{ - Type: newValueDateType(path), - Description: descriptions.WhereValueString, - }, - "valueGeoRange": &graphql.InputObjectFieldConfig{ - Type: newGeoRangeInputObject(path), - Description: descriptions.WhereValueRange, - }, - } - - // Recurse into the same time. - commonFilters["operands"] = &graphql.InputObjectFieldConfig{ - Description: descriptions.WhereOperands, - Type: graphql.NewList(graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sWhereOperandsInpObj", path), - Description: descriptions.WhereOperandsInpObj, - Fields: (graphql.InputObjectConfigFieldMapThunk)(func() graphql.InputObjectConfigFieldMap { - return commonFilters - }), - }, - )), - } - - return commonFilters -} - -func newGeoRangeInputObject(path string) *graphql.InputObject { - return graphql.NewInputObject(graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sWhereGeoRangeInpObj", path), - Fields: graphql.InputObjectConfigFieldMap{ - "geoCoordinates": &graphql.InputObjectFieldConfig{ - Type: graphql.NewNonNull(newGeoRangeGeoCoordinatesInputObject(path)), - Description: descriptions.WhereValueRangeGeoCoordinates, - }, - "distance": &graphql.InputObjectFieldConfig{ - Type: graphql.NewNonNull(newGeoRangeDistanceInputObject(path)), - Description: descriptions.WhereValueRangeDistance, - }, - }, - }) -} - -func newGeoRangeGeoCoordinatesInputObject(path string) *graphql.InputObject { - return graphql.NewInputObject(graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sWhereGeoRangeGeoCoordinatesInpObj", path), - Fields: graphql.InputObjectConfigFieldMap{ - "latitude": &graphql.InputObjectFieldConfig{ - Type: graphql.NewNonNull(graphql.Float), - Description: descriptions.WhereValueRangeGeoCoordinatesLatitude, - }, - "longitude": &graphql.InputObjectFieldConfig{ - Type: graphql.NewNonNull(graphql.Float), - Description: descriptions.WhereValueRangeGeoCoordinatesLongitude, - }, - }, - }) -} - -func newGeoRangeDistanceInputObject(path string) *graphql.InputObject { - return graphql.NewInputObject(graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sWhereGeoRangeDistanceInpObj", path), - Fields: graphql.InputObjectConfigFieldMap{ - "max": &graphql.InputObjectFieldConfig{ - Type: graphql.NewNonNull(graphql.Float), - Description: descriptions.WhereValueRangeDistanceMax, - }, - }, - }) -} diff --git a/adapters/handlers/graphql/local/common_filters/filters_types.go b/adapters/handlers/graphql/local/common_filters/filters_types.go deleted file mode 100644 index 496af142e8e3671556c7170ead468d4940929f48..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/common_filters/filters_types.go +++ /dev/null @@ -1,148 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Package common_filters provides the filters for the graphql endpoint for Weaviate -package common_filters - -import ( - "fmt" - "strconv" - - "github.com/tailor-inc/graphql" - "github.com/tailor-inc/graphql/language/ast" -) - -func newValueTextType(path string) graphql.Input { - return graphql.NewScalar(graphql.ScalarConfig{ - Name: fmt.Sprintf("Text%v", path), - Description: "String or String[]", - Serialize: func(value interface{}) interface{} { - return graphql.String.Serialize(value) - }, - ParseValue: func(value interface{}) interface{} { - return graphql.String.ParseValue(value) - }, - ParseLiteral: func(valueAST ast.Value) interface{} { - switch valueAST := valueAST.(type) { - case *ast.StringValue: - return valueAST.Value - case *ast.ListValue: - result := make([]string, len(valueAST.Values)) - for i := range valueAST.Values { - result[i] = valueAST.Values[i].GetValue().(string) - } - return result - } - return nil - }, - }) -} - -func newValueStringType(path string) graphql.Input { - return newValueTextType(fmt.Sprintf("String%v", path)) -} - -func newValueDateType(path string) graphql.Input { - return newValueTextType(fmt.Sprintf("Date%v", path)) -} - -func newValueIntType(path string) graphql.Input { - return graphql.NewScalar(graphql.ScalarConfig{ - Name: fmt.Sprintf("Int%v", path), - Description: "Int or Int[]", - Serialize: func(value interface{}) interface{} { - return graphql.Int.Serialize(value) - }, - ParseValue: func(value interface{}) interface{} { - return graphql.Int.ParseValue(value) - }, - ParseLiteral: func(valueAST ast.Value) interface{} { - switch valueAST := valueAST.(type) { - case *ast.IntValue: - if intValue, err := strconv.Atoi(valueAST.Value); err == nil { - return intValue - } - case *ast.ListValue: - result := make([]int, len(valueAST.Values)) - for i := range valueAST.Values { - if intValue, err := strconv.Atoi(valueAST.Values[i].GetValue().(string)); err == nil { - result[i] = int(intValue) - } - } - return result - } - return nil - }, - }) -} - -func newValueNumberType(path string) graphql.Input { - return graphql.NewScalar(graphql.ScalarConfig{ - Name: fmt.Sprintf("Float%v", path), - Description: "Float or Float[]", - Serialize: func(value interface{}) interface{} { - return graphql.Float.Serialize(value) - }, - ParseValue: func(value interface{}) interface{} { - return graphql.Float.ParseValue(value) - }, - ParseLiteral: func(valueAST ast.Value) interface{} { - switch valueAST := valueAST.(type) { - case *ast.FloatValue: - if floatValue, err := strconv.ParseFloat(valueAST.Value, 64); err == nil { - return floatValue - } - case *ast.IntValue: - if floatValue, err := strconv.ParseFloat(valueAST.Value, 64); err == nil { - return floatValue - } - case *ast.ListValue: - result := make([]float64, len(valueAST.Values)) - for i := range valueAST.Values { - if floatValue, err := strconv.ParseFloat(valueAST.Values[i].GetValue().(string), 64); err == nil { - result[i] = floatValue - } - } - return result - } - return nil - }, - }) -} - -func newValueBooleanType(path string) graphql.Input { - return graphql.NewScalar(graphql.ScalarConfig{ - Name: fmt.Sprintf("Boolean%v", path), - Description: "Boolean or Boolean[]", - Serialize: func(value interface{}) interface{} { - return graphql.Boolean.Serialize(value) - }, - ParseValue: func(value interface{}) interface{} { - return graphql.Boolean.ParseValue(value) - }, - ParseLiteral: func(valueAST ast.Value) interface{} { - switch valueAST := valueAST.(type) { - case *ast.BooleanValue: - return valueAST.Value - case *ast.ListValue: - result := make([]bool, len(valueAST.Values)) - for i, val := range valueAST.Values { - switch v := val.(type) { - case *ast.BooleanValue: - result[i] = v.Value - } - } - return result - } - return nil - }, - }) -} diff --git a/adapters/handlers/graphql/local/common_filters/group_by.go b/adapters/handlers/graphql/local/common_filters/group_by.go deleted file mode 100644 index d01f987a45a31e7967eeff28e4a6eccb20e38f5d..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/common_filters/group_by.go +++ /dev/null @@ -1,39 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package common_filters - -import "github.com/weaviate/weaviate/entities/searchparams" - -// ExtractGroupBy -func ExtractGroupBy(source map[string]interface{}) searchparams.GroupBy { - var args searchparams.GroupBy - - p, ok := source["path"] - if ok { - rawSlice := p.([]interface{}) - if len(rawSlice) == 1 { - args.Property = rawSlice[0].(string) - } - } - - groups := source["groups"] - if groups != nil { - args.Groups = int(groups.(int)) - } - - objectsPerGroup := source["objectsPerGroup"] - if objectsPerGroup != nil { - args.ObjectsPerGroup = int(objectsPerGroup.(int)) - } - - return args -} diff --git a/adapters/handlers/graphql/local/common_filters/helper_test.go b/adapters/handlers/graphql/local/common_filters/helper_test.go deleted file mode 100644 index e8839f259f8c90a1a61e47324275fc79a214b286..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/common_filters/helper_test.go +++ /dev/null @@ -1,122 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package common_filters - -import ( - "testing" - - "github.com/tailor-inc/graphql" - test_helper "github.com/weaviate/weaviate/adapters/handlers/graphql/test/helper" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/searchparams" -) - -type mockResolver struct { - test_helper.MockResolver -} - -type mockParams struct { - reportFilter bool - reportNearVector bool - reportNearObject bool -} - -func newMockResolver(t *testing.T, params mockParams) *mockResolver { - if params.reportNearVector && params.reportNearObject { - t.Fatal("cannot provide both nearVector and nearObject") - } - - // Build a FakeGet. - fakeGet := &graphql.Field{ - Name: "SomeAction", - Description: "Fake Some Action", - Args: graphql.FieldConfigArgument{ - "where": &graphql.ArgumentConfig{ - Description: "Filter options for the Get search, to convert the data to the filter input", - Type: graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: "GetWhereInpObj", - Fields: BuildNew("Get"), - Description: "", - }, - ), - }, - "nearVector": NearVectorArgument("Get", "SomeAction"), - "nearObject": NearObjectArgument("Get", "SomeAction"), - }, - Type: graphql.Int, - Resolve: func(p graphql.ResolveParams) (interface{}, error) { - resolver := p.Source.(map[string]interface{})["Resolver"].(*mockResolver) - return resolver.ReportArgs(params, p.Args, p.Info.FieldName) - }, - } - - mocker := &mockResolver{} - mocker.RootFieldName = "SomeAction" - mocker.RootField = fakeGet - mocker.RootObject = map[string]interface{}{"Resolver": mocker} - return mocker -} - -func (m *mockResolver) ReportArgs(params mockParams, args map[string]interface{}, - fieldName string, -) (result interface{}, err error) { - if params.reportFilter { - filters, err := ExtractFilters(args, fieldName) - if err != nil { - return nil, err - } - result, err = m.ReportFilters(filters) - if err != nil { - return nil, err - } - } - - if params.reportNearVector { - nearVec, err := ExtractNearVector(args["nearVector"].(map[string]interface{})) - if err != nil { - return nil, err - } - result, err = m.ReportNearVector(nearVec) - if err != nil { - return nil, err - } - } - - if params.reportNearObject { - nearObj, err := ExtractNearObject(args["nearObject"].(map[string]interface{})) - if err != nil { - return nil, err - } - result, err = m.ReportNearObject(nearObj) - if err != nil { - return nil, err - } - } - - return -} - -func (m *mockResolver) ReportFilters(filter *filters.LocalFilter) (interface{}, error) { - args := m.Called(filter) - return args.Get(0), args.Error(1) -} - -func (m *mockResolver) ReportNearVector(params searchparams.NearVector) (interface{}, error) { - args := m.Called(params) - return args.Get(0), args.Error(1) -} - -func (m *mockResolver) ReportNearObject(params searchparams.NearObject) (interface{}, error) { - args := m.Called(params) - return args.Get(0), args.Error(1) -} diff --git a/adapters/handlers/graphql/local/common_filters/hybrid.go b/adapters/handlers/graphql/local/common_filters/hybrid.go deleted file mode 100644 index 63fea015876cd23c58969e0d1fd020649d1a0e36..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/common_filters/hybrid.go +++ /dev/null @@ -1,119 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package common_filters - -import ( - "fmt" - - "github.com/weaviate/weaviate/entities/searchparams" -) - -const DefaultAlpha = float64(0.75) -const ( - HybridRankedFusion = iota - HybridRelativeScoreFusion -) -const HybridFusionDefault = HybridRelativeScoreFusion - -func ExtractHybridSearch(source map[string]interface{}, explainScore bool) (*searchparams.HybridSearch, error) { - var subsearches []interface{} - operandsI := source["operands"] - if operandsI != nil { - operands := operandsI.([]interface{}) - for _, operand := range operands { - operandMap := operand.(map[string]interface{}) - subsearches = append(subsearches, operandMap) - } - } - - var weightedSearchResults []searchparams.WeightedSearchResult - var args searchparams.HybridSearch - for _, ss := range subsearches { - subsearch := ss.(map[string]interface{}) - switch { - case subsearch["sparseSearch"] != nil: - bm25 := subsearch["sparseSearch"].(map[string]interface{}) - arguments := ExtractBM25(bm25, explainScore) - - weightedSearchResults = append(weightedSearchResults, searchparams.WeightedSearchResult{ - SearchParams: arguments, - Weight: subsearch["weight"].(float64), - Type: "bm25", - }) - case subsearch["nearText"] != nil: - nearText := subsearch["nearText"].(map[string]interface{}) - arguments, _ := ExtractNearText(nearText) - - weightedSearchResults = append(weightedSearchResults, searchparams.WeightedSearchResult{ - SearchParams: arguments, - Weight: subsearch["weight"].(float64), - Type: "nearText", - }) - - case subsearch["nearVector"] != nil: - nearVector := subsearch["nearVector"].(map[string]interface{}) - arguments, _ := ExtractNearVector(nearVector) - - weightedSearchResults = append(weightedSearchResults, searchparams.WeightedSearchResult{ - SearchParams: arguments, - Weight: subsearch["weight"].(float64), - Type: "nearVector", - }) - - default: - return nil, fmt.Errorf("unknown subsearch type: %+v", subsearch) - } - } - - args.SubSearches = weightedSearchResults - - alpha, ok := source["alpha"] - if ok { - args.Alpha = alpha.(float64) - } else { - args.Alpha = DefaultAlpha - } - - if args.Alpha < 0 || args.Alpha > 1 { - return nil, fmt.Errorf("alpha should be between 0.0 and 1.0") - } - - query, ok := source["query"] - if ok { - args.Query = query.(string) - } - - fusionType, ok := source["fusionType"] - if ok { - args.FusionAlgorithm = fusionType.(int) - } else { - args.FusionAlgorithm = HybridFusionDefault - } - if _, ok := source["vector"]; ok { - vector := source["vector"].([]interface{}) - args.Vector = make([]float32, len(vector)) - for i, value := range vector { - args.Vector[i] = float32(value.(float64)) - } - } - - if _, ok := source["properties"]; ok { - properties := source["properties"].([]interface{}) - args.Properties = make([]string, len(properties)) - for i, value := range properties { - args.Properties[i] = value.(string) - } - } - - args.Type = "hybrid" - return &args, nil -} diff --git a/adapters/handlers/graphql/local/common_filters/nearText.go b/adapters/handlers/graphql/local/common_filters/nearText.go deleted file mode 100644 index e3481c2e8bae5ccbb595222c1a8bce20461cc450..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/common_filters/nearText.go +++ /dev/null @@ -1,114 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package common_filters - -import ( - "fmt" - - "github.com/weaviate/weaviate/entities/searchparams" -) - -// ExtractNearText arguments, such as "concepts", "moveTo", "moveAwayFrom", -// "limit", etc. -func ExtractNearText(source map[string]interface{}) (searchparams.NearTextParams, error) { - var args searchparams.NearTextParams - - // keywords is a required argument, so we don't need to check for its existing - keywords := source["concepts"].([]interface{}) - args.Values = make([]string, len(keywords)) - for i, value := range keywords { - args.Values[i] = value.(string) - } - - // autocorrect is an optional arg, so it could be nil - autocorrect, ok := source["autocorrect"] - if ok { - args.Autocorrect = autocorrect.(bool) - if args.Autocorrect { - return searchparams.NearTextParams{}, fmt.Errorf("autocorrect is not supported for hybrid nearText") - } - } - - // limit is an optional arg, so it could be nil - limit, ok := source["limit"] - if ok { - // the type is fixed through gql config, no need to catch incorrect type - // assumption - args.Limit = limit.(int) - } - - certainty, ok := source["certainty"] - if ok { - args.Certainty = certainty.(float64) - } - - distance, ok := source["distance"] - if ok { - args.Distance = distance.(float64) - args.WithDistance = true - } - - // moveTo is an optional arg, so it could be nil - moveTo, ok := source["moveTo"] - if ok { - args.MoveTo = extractMovement(moveTo) - } - - // network is an optional arg, so it could be nil - network, ok := source["network"] - if ok { - args.Network = network.(bool) - } - - // moveAwayFrom is an optional arg, so it could be nil - moveAwayFrom, ok := source["moveAwayFrom"] - if ok { - args.MoveAwayFrom = extractMovement(moveAwayFrom) - } - - return args, nil -} - -func extractMovement(input interface{}) searchparams.ExploreMove { - // the type is fixed through gql config, no need to catch incorrect type - // assumption, all fields are required so we don't need to check for their - // presence - moveToMap := input.(map[string]interface{}) - res := searchparams.ExploreMove{} - res.Force = float32(moveToMap["force"].(float64)) - - keywords, ok := moveToMap["concepts"].([]interface{}) - if ok { - res.Values = make([]string, len(keywords)) - for i, value := range keywords { - res.Values[i] = value.(string) - } - } - - objects, ok := moveToMap["objects"].([]interface{}) - if ok { - res.Objects = make([]searchparams.ObjectMove, len(objects)) - for i, value := range objects { - v, ok := value.(map[string]interface{}) - if ok { - if v["id"] != nil { - res.Objects[i].ID = v["id"].(string) - } - if v["beacon"] != nil { - res.Objects[i].Beacon = v["beacon"].(string) - } - } - } - } - - return res -} diff --git a/adapters/handlers/graphql/local/common_filters/near_filters.go b/adapters/handlers/graphql/local/common_filters/near_filters.go deleted file mode 100644 index 0be3c9168d151d570d7e52555e330fb5bcfaae2d..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/common_filters/near_filters.go +++ /dev/null @@ -1,82 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package common_filters - -import ( - "fmt" - - "github.com/tailor-inc/graphql" - "github.com/weaviate/weaviate/adapters/handlers/graphql/descriptions" -) - -func NearVectorArgument(argumentPrefix, className string) *graphql.ArgumentConfig { - prefix := fmt.Sprintf("%s%s", argumentPrefix, className) - return &graphql.ArgumentConfig{ - // Description: descriptions.GetExplore, - Type: graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sNearVectorInpObj", prefix), - Fields: nearVectorFields(prefix), - }, - ), - } -} - -func nearVectorFields(prefix string) graphql.InputObjectConfigFieldMap { - return graphql.InputObjectConfigFieldMap{ - "vector": &graphql.InputObjectFieldConfig{ - Description: descriptions.Vector, - Type: graphql.NewNonNull(graphql.NewList(graphql.Float)), - }, - "certainty": &graphql.InputObjectFieldConfig{ - Description: descriptions.Certainty, - Type: graphql.Float, - }, - "distance": &graphql.InputObjectFieldConfig{ - Description: descriptions.Distance, - Type: graphql.Float, - }, - } -} - -func NearObjectArgument(argumentPrefix, className string) *graphql.ArgumentConfig { - prefix := fmt.Sprintf("%s%s", argumentPrefix, className) - return &graphql.ArgumentConfig{ - Type: graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sNearObjectInpObj", prefix), - Fields: nearObjectFields(prefix), - }, - ), - } -} - -func nearObjectFields(prefix string) graphql.InputObjectConfigFieldMap { - return graphql.InputObjectConfigFieldMap{ - "id": &graphql.InputObjectFieldConfig{ - Description: descriptions.ID, - Type: graphql.String, - }, - "beacon": &graphql.InputObjectFieldConfig{ - Description: descriptions.Beacon, - Type: graphql.String, - }, - "certainty": &graphql.InputObjectFieldConfig{ - Description: descriptions.Certainty, - Type: graphql.Float, - }, - "distance": &graphql.InputObjectFieldConfig{ - Description: descriptions.Distance, - Type: graphql.Float, - }, - } -} diff --git a/adapters/handlers/graphql/local/common_filters/near_object.go b/adapters/handlers/graphql/local/common_filters/near_object.go deleted file mode 100644 index fae8be3427e4313e5131fad10c11066eea168d3f..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/common_filters/near_object.go +++ /dev/null @@ -1,51 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package common_filters - -import ( - "fmt" - - "github.com/weaviate/weaviate/entities/searchparams" -) - -// ExtractNearObject arguments, such as "vector" and "certainty" -func ExtractNearObject(source map[string]interface{}) (searchparams.NearObject, error) { - var args searchparams.NearObject - - id, ok := source["id"] - if ok { - args.ID = id.(string) - } - - beacon, ok := source["beacon"] - if ok { - args.Beacon = beacon.(string) - } - - certainty, certaintyOK := source["certainty"] - if certaintyOK { - args.Certainty = certainty.(float64) - } - - distance, distanceOK := source["distance"] - if distanceOK { - args.Distance = distance.(float64) - args.WithDistance = true - } - - if certaintyOK && distanceOK { - return searchparams.NearObject{}, - fmt.Errorf("cannot provide distance and certainty") - } - - return args, nil -} diff --git a/adapters/handlers/graphql/local/common_filters/near_vector.go b/adapters/handlers/graphql/local/common_filters/near_vector.go deleted file mode 100644 index 8c9293446d2ad414454e139006706b2b3f117add..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/common_filters/near_vector.go +++ /dev/null @@ -1,48 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package common_filters - -import ( - "fmt" - - "github.com/weaviate/weaviate/entities/searchparams" -) - -// ExtractNearVector arguments, such as "vector" and "distance" -func ExtractNearVector(source map[string]interface{}) (searchparams.NearVector, error) { - var args searchparams.NearVector - - // vector is a required argument, so we don't need to check for its existing - vector := source["vector"].([]interface{}) - args.Vector = make([]float32, len(vector)) - for i, value := range vector { - args.Vector[i] = float32(value.(float64)) - } - - certainty, certaintyOK := source["certainty"] - if certaintyOK { - args.Certainty = certainty.(float64) - } - - distance, distanceOK := source["distance"] - if distanceOK { - args.Distance = distance.(float64) - args.WithDistance = true - } - - if certaintyOK && distanceOK { - return searchparams.NearVector{}, - fmt.Errorf("cannot provide distance and certainty") - } - - return args, nil -} diff --git a/adapters/handlers/graphql/local/common_filters/parse_filters_into_ast.go b/adapters/handlers/graphql/local/common_filters/parse_filters_into_ast.go deleted file mode 100644 index c10984a6365018ac68d132f671c88cde7f055bfa..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/common_filters/parse_filters_into_ast.go +++ /dev/null @@ -1,176 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package common_filters - -import ( - "encoding/json" - "fmt" - - "github.com/weaviate/weaviate/adapters/handlers/rest/filterext" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" -) - -// Extract the filters from the arguments of a Local->Get or Local->Meta query. -func ExtractFilters(args map[string]interface{}, rootClass string) (*filters.LocalFilter, error) { - where, wherePresent := args["where"] - if !wherePresent { - // No filters; all is fine! - return nil, nil - } else { - whereMap := where.(map[string]interface{}) // guaranteed by GraphQL to be a map. - filter, err := filterMapToModel(whereMap) - if err != nil { - return nil, fmt.Errorf("failed to extract filters: %s", err) - } - - return filterext.Parse(filter, rootClass) - } -} - -func filterMapToModel(m map[string]interface{}) (*models.WhereFilter, error) { - b, err := json.Marshal(m) - if err != nil { - return nil, fmt.Errorf("failed convert map to models.WhereFilter: %s", err) - } - - var filter WhereFilter - err = json.Unmarshal(b, &filter) - if err != nil { - return nil, fmt.Errorf("failed convert map to models.WhereFilter: %s", err) - } - - return newConverter().do(&filter) -} - -type converter struct{} - -func newConverter() *converter { - return &converter{} -} - -func (c *converter) do(in *WhereFilter) (*models.WhereFilter, error) { - whereFilter := &models.WhereFilter{ - Operator: in.Operator, - Path: in.Path, - } - - if in.ValueInt != nil { - switch v := in.ValueInt.(type) { - case float64: - val := int64(v) - whereFilter.ValueInt = &val - case []interface{}: - ints := make([]int64, len(v)) - for i := range v { - ints[i] = int64(v[i].(float64)) - } - whereFilter.ValueIntArray = ints - default: - return nil, fmt.Errorf("unsupported type: '%T'", in.ValueInt) - } - } - if in.ValueNumber != nil { - switch v := in.ValueNumber.(type) { - case float64: - whereFilter.ValueNumber = &v - case []interface{}: - numbers := make([]float64, len(v)) - for i := range v { - numbers[i] = v[i].(float64) - } - whereFilter.ValueNumberArray = numbers - default: - return nil, fmt.Errorf("unsupported type: '%T'", in.ValueNumber) - } - } - if in.ValueBoolean != nil { - switch v := in.ValueBoolean.(type) { - case bool: - whereFilter.ValueBoolean = &v - case []interface{}: - bools := make([]bool, len(v)) - for i := range v { - bools[i] = v[i].(bool) - } - whereFilter.ValueBooleanArray = bools - default: - return nil, fmt.Errorf("unsupported type: '%T'", in.ValueBoolean) - } - } - if in.ValueString != nil { - value, valueArray, err := c.parseString(in.ValueString) - if err != nil { - return nil, err - } - whereFilter.ValueString = value - whereFilter.ValueStringArray = valueArray - } - if in.ValueText != nil { - value, valueArray, err := c.parseString(in.ValueText) - if err != nil { - return nil, err - } - whereFilter.ValueText = value - whereFilter.ValueTextArray = valueArray - } - if in.ValueDate != nil { - value, valueArray, err := c.parseString(in.ValueDate) - if err != nil { - return nil, err - } - whereFilter.ValueDate = value - whereFilter.ValueDateArray = valueArray - } - if in.ValueGeoRange != nil { - whereFilter.ValueGeoRange = in.ValueGeoRange - } - - // recursively build operands - for i, op := range in.Operands { - whereFilterOp, err := c.do(op) - if err != nil { - return nil, fmt.Errorf("operands[%v]: %w", i, err) - } - whereFilter.Operands = append(whereFilter.Operands, whereFilterOp) - } - - return whereFilter, nil -} - -func (c *converter) parseString(in interface{}) (value *string, valueArray []string, err error) { - switch v := in.(type) { - case string: - value = &v - case []interface{}: - valueArray = make([]string, len(v)) - for i := range v { - valueArray[i] = v[i].(string) - } - default: - err = fmt.Errorf("unsupported type: '%T'", in) - } - return -} - -type WhereFilter struct { - Operands []*WhereFilter `json:"operands"` - Operator string `json:"operator,omitempty"` - Path []string `json:"path"` - ValueBoolean interface{} `json:"valueBoolean,omitempty"` - ValueDate interface{} `json:"valueDate,omitempty"` - ValueInt interface{} `json:"valueInt,omitempty"` - ValueNumber interface{} `json:"valueNumber,omitempty"` - ValueString interface{} `json:"valueString,omitempty"` - ValueText interface{} `json:"valueText,omitempty"` - ValueGeoRange *models.WhereFilterGeoRange `json:"valueGeoRange,omitempty"` -} diff --git a/adapters/handlers/graphql/local/common_filters/parse_test.go b/adapters/handlers/graphql/local/common_filters/parse_test.go deleted file mode 100644 index 2eacc35ef8a818bb8735d9fe8d618337017b272d..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/common_filters/parse_test.go +++ /dev/null @@ -1,425 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package common_filters - -import ( - "testing" - - "github.com/tailor-inc/graphql/gqlerrors" - "github.com/tailor-inc/graphql/language/location" - test_helper "github.com/weaviate/weaviate/adapters/handlers/graphql/test/helper" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/searchparams" -) - -// Basic test on filter -func TestExtractFilterToplevelField(t *testing.T) { - t.Parallel() - - resolver := newMockResolver(t, mockParams{reportFilter: true}) - /*localfilter is a struct containing a clause struct - type filters.Clause struct { - Operator Operator - On *filters.Path - filters.Value *filters.Value - Operands []filters.Clause - }*/ - expectedParams := &filters.LocalFilter{Root: &filters.Clause{ - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: schema.AssertValidClassName("SomeAction"), - Property: schema.AssertValidPropertyName("intField"), - }, - Value: &filters.Value{ - Value: 42, - Type: schema.DataTypeInt, - }, - }} - - resolver.On("ReportFilters", expectedParams). - Return(test_helper.EmptyList(), nil).Once() - - query := `{ SomeAction(where: { path: ["intField"], operator: Equal, valueInt: 42}) }` - resolver.AssertResolve(t, query) -} - -func TestExtractFilterLike(t *testing.T) { - t.Parallel() - - t.Run("extracts with valueText", func(t *testing.T) { - resolver := newMockResolver(t, mockParams{reportFilter: true}) - expectedParams := &filters.LocalFilter{Root: &filters.Clause{ - Operator: filters.OperatorLike, - On: &filters.Path{ - Class: schema.AssertValidClassName("SomeAction"), - Property: schema.AssertValidPropertyName("name"), - }, - Value: &filters.Value{ - Value: "Schn*el", - Type: schema.DataTypeText, - }, - }} - - resolver.On("ReportFilters", expectedParams). - Return(test_helper.EmptyList(), nil).Once() - - query := `{ SomeAction(where: { - path: ["name"], - operator: Like, - valueText: "Schn*el", - }) }` - resolver.AssertResolve(t, query) - }) - - t.Run("[deprecated string] extracts with valueString", func(t *testing.T) { - resolver := newMockResolver(t, mockParams{reportFilter: true}) - expectedParams := &filters.LocalFilter{Root: &filters.Clause{ - Operator: filters.OperatorLike, - On: &filters.Path{ - Class: schema.AssertValidClassName("SomeAction"), - Property: schema.AssertValidPropertyName("name"), - }, - Value: &filters.Value{ - Value: "Schn*el", - Type: schema.DataTypeString, - }, - }} - - resolver.On("ReportFilters", expectedParams). - Return(test_helper.EmptyList(), nil).Once() - - query := `{ SomeAction(where: { - path: ["name"], - operator: Like, - valueString: "Schn*el", - }) }` - resolver.AssertResolve(t, query) - }) -} - -func TestExtractFilterLike_ValueText(t *testing.T) { - t.Parallel() - - resolver := newMockResolver(t, mockParams{reportFilter: true}) - expectedParams := &filters.LocalFilter{Root: &filters.Clause{ - Operator: filters.OperatorLike, - On: &filters.Path{ - Class: schema.AssertValidClassName("SomeAction"), - Property: schema.AssertValidPropertyName("name"), - }, - Value: &filters.Value{ - Value: "schn*el", - Type: schema.DataTypeText, - }, - }} - - resolver.On("ReportFilters", expectedParams). - Return(test_helper.EmptyList(), nil).Once() - - query := `{ SomeAction(where: { - path: ["name"], - operator: Like, - valueText: "schn*el", - }) }` - resolver.AssertResolve(t, query) -} - -func TestExtractFilterIsNull(t *testing.T) { - resolver := newMockResolver(t, mockParams{reportFilter: true}) - expectedParams := &filters.LocalFilter{Root: &filters.Clause{ - Operator: filters.OperatorIsNull, - On: &filters.Path{ - Class: schema.AssertValidClassName("SomeAction"), - Property: schema.AssertValidPropertyName("name"), - }, - Value: &filters.Value{ - Value: "true", - Type: schema.DataTypeText, - }, - }} - - resolver.On("ReportFilters", expectedParams). - Return(test_helper.EmptyList(), nil).Once() - - query := `{ SomeAction(where: { - path: ["name"], - operator: IsNull, - valueText: "true", - }) }` - resolver.AssertResolve(t, query) -} - -func TestExtractFilterGeoLocation(t *testing.T) { - t.Parallel() - - t.Run("with all fields set as required", func(t *testing.T) { - resolver := newMockResolver(t, mockParams{reportFilter: true}) - expectedParams := &filters.LocalFilter{Root: &filters.Clause{ - Operator: filters.OperatorWithinGeoRange, - On: &filters.Path{ - Class: schema.AssertValidClassName("SomeAction"), - Property: schema.AssertValidPropertyName("location"), - }, - Value: &filters.Value{ - Value: filters.GeoRange{ - GeoCoordinates: &models.GeoCoordinates{ - Latitude: ptFloat32(0.5), - Longitude: ptFloat32(0.6), - }, - Distance: 2.0, - }, - Type: schema.DataTypeGeoCoordinates, - }, - }} - - resolver.On("ReportFilters", expectedParams). - Return(test_helper.EmptyList(), nil).Once() - - query := `{ SomeAction(where: { - path: ["location"], - operator: WithinGeoRange, - valueGeoRange: {geoCoordinates: { latitude: 0.5, longitude: 0.6 }, distance: { max: 2.0 } } - }) }` - resolver.AssertResolve(t, query) - }) - - t.Run("with only some of the fields set", func(t *testing.T) { - resolver := newMockResolver(t, mockParams{reportFilter: true}) - expectedParams := &filters.LocalFilter{Root: &filters.Clause{ - Operator: filters.OperatorWithinGeoRange, - On: &filters.Path{ - Class: schema.AssertValidClassName("SomeAction"), - Property: schema.AssertValidPropertyName("location"), - }, - Value: &filters.Value{ - Value: filters.GeoRange{ - GeoCoordinates: &models.GeoCoordinates{ - Latitude: ptFloat32(0.5), - Longitude: ptFloat32(0.6), - }, - Distance: 2.0, - }, - Type: schema.DataTypeGeoCoordinates, - }, - }} - - resolver.On("ReportFilters", expectedParams). - Return(test_helper.EmptyList(), nil).Once() - - query := `{ SomeAction(where: { - path: ["location"], - operator: WithinGeoRange, - valueGeoRange: { geoCoordinates: { latitude: 0.5 }, distance: { max: 2.0} } - }) }` - - expectedErrors := []gqlerrors.FormattedError{ - { - Message: "Argument \"where\" has invalid value {path: [\"location\"], operator: WithinGeoRange, valueGeoRange: {geoCoordinates: {latitude: 0.5}, distance: {max: 2.0}}}.\nIn field \"valueGeoRange\": In field \"geoCoordinates\": In field \"longitude\": Expected \"Float!\", found null.", - Locations: []location.SourceLocation{{Line: 1, Column: 21}}, - }, - } - resolver.AssertErrors(t, query, expectedErrors) - }) -} - -func TestExtractFilterNestedField(t *testing.T) { - t.Parallel() - - resolver := newMockResolver(t, mockParams{reportFilter: true}) - - expectedParams := &filters.LocalFilter{Root: &filters.Clause{ - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: schema.AssertValidClassName("SomeAction"), - Property: schema.AssertValidPropertyName("hasAction"), - Child: &filters.Path{ - Class: schema.AssertValidClassName("SomeAction"), - Property: schema.AssertValidPropertyName("intField"), - }, - }, - Value: &filters.Value{ - Value: 42, - Type: schema.DataTypeInt, - }, - }} - - resolver.On("ReportFilters", expectedParams). - Return(test_helper.EmptyList(), nil).Once() - - query := `{ SomeAction(where: { path: ["hasAction", "SomeAction", "intField"], operator: Equal, valueInt: 42}) }` - resolver.AssertResolve(t, query) -} - -func TestExtractOperand(t *testing.T) { - t.Parallel() - - resolver := newMockResolver(t, mockParams{reportFilter: true}) - - expectedParams := &filters.LocalFilter{Root: &filters.Clause{ - Operator: filters.OperatorAnd, - Operands: []filters.Clause{ - { - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: schema.AssertValidClassName("SomeAction"), - Property: schema.AssertValidPropertyName("intField"), - }, - Value: &filters.Value{ - Value: 42, - Type: schema.DataTypeInt, - }, - }, - { - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: schema.AssertValidClassName("SomeAction"), - Property: schema.AssertValidPropertyName("hasAction"), - Child: &filters.Path{ - Class: schema.AssertValidClassName("SomeAction"), - Property: schema.AssertValidPropertyName("intField"), - }, - }, - Value: &filters.Value{ - Value: 4242, - Type: schema.DataTypeInt, - }, - }, - }, - }} - - resolver.On("ReportFilters", expectedParams). - Return(test_helper.EmptyList(), nil).Once() - - query := `{ SomeAction(where: { operator: And, operands: [ - { operator: Equal, valueInt: 42, path: ["intField"]}, - { operator: Equal, valueInt: 4242, path: ["hasAction", "SomeAction", "intField"] } - ]}) }` - resolver.AssertResolve(t, query) -} - -func TestExtractCompareOpFailsIfOperandPresent(t *testing.T) { - t.Parallel() - - resolver := newMockResolver(t, mockParams{reportFilter: true}) - - query := `{ SomeAction(where: { operator: Equal, operands: []}) }` - resolver.AssertFailToResolve(t, query) -} - -func TestExtractOperandFailsIfPathPresent(t *testing.T) { - t.Parallel() - - resolver := newMockResolver(t, mockParams{reportFilter: true}) - - query := `{ SomeAction(where: { path:["should", "not", "be", "present"], operator: And })}` - resolver.AssertFailToResolve(t, query) -} - -func TestExtractNearVector(t *testing.T) { - t.Parallel() - - t.Run("with certainty provided", func(t *testing.T) { - t.Parallel() - - query := `{ SomeAction(nearVector: {vector: [1, 2, 3], certainty: 0.7})}` - expectedparams := searchparams.NearVector{ - Vector: []float32{1, 2, 3}, - Certainty: 0.7, - } - - resolver := newMockResolver(t, mockParams{reportNearVector: true}) - - resolver.On("ReportNearVector", expectedparams). - Return(test_helper.EmptyList(), nil).Once() - - resolver.AssertResolve(t, query) - }) - - t.Run("with distance provided", func(t *testing.T) { - t.Parallel() - - query := `{ SomeAction(nearVector: {vector: [1, 2, 3], distance: 0.4})}` - expectedparams := searchparams.NearVector{ - Vector: []float32{1, 2, 3}, - Distance: 0.4, - WithDistance: true, - } - - resolver := newMockResolver(t, mockParams{reportNearVector: true}) - - resolver.On("ReportNearVector", expectedparams). - Return(test_helper.EmptyList(), nil).Once() - - resolver.AssertResolve(t, query) - }) - - t.Run("with distance and certainty provided", func(t *testing.T) { - t.Parallel() - - query := `{ SomeAction(nearVector: {vector: [1, 2, 3], distance: 0.4, certainty: 0.7})}` - resolver := newMockResolver(t, mockParams{reportNearVector: true}) - resolver.AssertFailToResolve(t, query) - }) -} - -func TestExtractNearObject(t *testing.T) { - t.Parallel() - - t.Run("with certainty provided", func(t *testing.T) { - t.Parallel() - - query := `{ SomeAction(nearObject: {id: "123", certainty: 0.7})}` - expectedparams := searchparams.NearObject{ - ID: "123", - Certainty: 0.7, - } - - resolver := newMockResolver(t, mockParams{reportNearObject: true}) - - resolver.On("ReportNearObject", expectedparams). - Return(test_helper.EmptyList(), nil).Once() - - resolver.AssertResolve(t, query) - }) - - t.Run("with distance provided", func(t *testing.T) { - t.Parallel() - - query := `{ SomeAction(nearObject: {id: "123", distance: 0.4})}` - expectedparams := searchparams.NearObject{ - ID: "123", - Distance: 0.4, - WithDistance: true, - } - - resolver := newMockResolver(t, mockParams{reportNearObject: true}) - - resolver.On("ReportNearObject", expectedparams). - Return(test_helper.EmptyList(), nil).Once() - - resolver.AssertResolve(t, query) - }) - - t.Run("with distance and certainty provided", func(t *testing.T) { - t.Parallel() - - query := `{ SomeAction(nearObject: {id: "123", distance: 0.4, certainty: 0.7})}` - resolver := newMockResolver(t, mockParams{reportNearObject: true}) - resolver.AssertFailToResolve(t, query) - }) -} - -func ptFloat32(in float32) *float32 { - return &in -} diff --git a/adapters/handlers/graphql/local/explore/concepts.go b/adapters/handlers/graphql/local/explore/concepts.go deleted file mode 100644 index 4f8a52242bf4edd7015ee187da48bbba55538976..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/explore/concepts.go +++ /dev/null @@ -1,186 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package explore - -import ( - "fmt" - - "github.com/tailor-inc/graphql" - "github.com/weaviate/weaviate/adapters/handlers/graphql/descriptions" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/search" -) - -type ModulesProvider interface { - ExploreArguments(schema *models.Schema) map[string]*graphql.ArgumentConfig - CrossClassExtractSearchParams(arguments map[string]interface{}) map[string]interface{} -} - -// Build builds the object containing the Local->Explore Fields, such as Objects -func Build(schema *models.Schema, modulesProvider ModulesProvider) *graphql.Field { - field := &graphql.Field{ - Name: "Explore", - Description: descriptions.LocalExplore, - Type: graphql.NewList(exploreObject()), - Resolve: newResolver(modulesProvider).resolve, - Args: graphql.FieldConfigArgument{ - "offset": &graphql.ArgumentConfig{ - Type: graphql.Int, - Description: descriptions.Offset, - }, - "limit": &graphql.ArgumentConfig{ - Type: graphql.Int, - Description: descriptions.Limit, - }, - - "nearVector": nearVectorArgument(), - "nearObject": nearObjectArgument(), - }, - } - - if modulesProvider != nil { - for name, argument := range modulesProvider.ExploreArguments(schema) { - field.Args[name] = argument - } - } - - return field -} - -func exploreObject() *graphql.Object { - getLocalExploreFields := graphql.Fields{ - "className": &graphql.Field{ - Name: "ExploreClassName", - Description: descriptions.ClassName, - Type: graphql.String, - Resolve: func(p graphql.ResolveParams) (interface{}, error) { - vsr, ok := p.Source.(search.Result) - if !ok { - return nil, fmt.Errorf("unknown type %T in Explore..className resolver", p.Source) - } - - return vsr.ClassName, nil - }, - }, - - "beacon": &graphql.Field{ - Name: "ExploreBeacon", - Description: descriptions.Beacon, - Type: graphql.String, - Resolve: func(p graphql.ResolveParams) (interface{}, error) { - vsr, ok := p.Source.(search.Result) - if !ok { - return nil, fmt.Errorf("unknown type %T in Explore..className resolver", p.Source) - } - - return vsr.Beacon, nil - }, - }, - - "certainty": &graphql.Field{ - Name: "ExploreCertainty", - Description: descriptions.Certainty, - Type: graphql.Float, - Resolve: func(p graphql.ResolveParams) (interface{}, error) { - vsr, ok := p.Source.(search.Result) - if !ok { - return nil, fmt.Errorf("unknown type %T in Explore..className resolver", p.Source) - } - - return vsr.Certainty, nil - }, - }, - - "distance": &graphql.Field{ - Name: "ExploreDistance", - Description: descriptions.Distance, - Type: graphql.Float, - Resolve: func(p graphql.ResolveParams) (interface{}, error) { - vsr, ok := p.Source.(search.Result) - if !ok { - return nil, fmt.Errorf("unknown type %T in Explore..className resolver", p.Source) - } - - return vsr.Dist, nil - }, - }, - } - - getLocalExploreFieldsObject := graphql.ObjectConfig{ - Name: "ExploreObj", - Fields: getLocalExploreFields, - Description: descriptions.LocalExplore, - } - - return graphql.NewObject(getLocalExploreFieldsObject) -} - -func nearVectorArgument() *graphql.ArgumentConfig { - return &graphql.ArgumentConfig{ - // Description: descriptions.GetExplore, - Type: graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: "ExploreNearVectorInpObj", - Fields: nearVectorFields(), - }, - ), - } -} - -func nearVectorFields() graphql.InputObjectConfigFieldMap { - return graphql.InputObjectConfigFieldMap{ - "vector": &graphql.InputObjectFieldConfig{ - Description: descriptions.Certainty, - Type: graphql.NewNonNull(graphql.NewList(graphql.Float)), - }, - "certainty": &graphql.InputObjectFieldConfig{ - Description: descriptions.Certainty, - Type: graphql.Float, - }, - "distance": &graphql.InputObjectFieldConfig{ - Description: descriptions.Distance, - Type: graphql.Float, - }, - } -} - -func nearObjectArgument() *graphql.ArgumentConfig { - return &graphql.ArgumentConfig{ - Type: graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: "ExploreNearObjectInpObj", - Fields: nearObjectFields(), - }, - ), - } -} - -func nearObjectFields() graphql.InputObjectConfigFieldMap { - return graphql.InputObjectConfigFieldMap{ - "id": &graphql.InputObjectFieldConfig{ - Description: descriptions.ID, - Type: graphql.String, - }, - "beacon": &graphql.InputObjectFieldConfig{ - Description: descriptions.Beacon, - Type: graphql.String, - }, - "certainty": &graphql.InputObjectFieldConfig{ - Description: descriptions.Certainty, - Type: graphql.Float, - }, - "distance": &graphql.InputObjectFieldConfig{ - Description: descriptions.Distance, - Type: graphql.Float, - }, - } -} diff --git a/adapters/handlers/graphql/local/explore/concepts_resolver.go b/adapters/handlers/graphql/local/explore/concepts_resolver.go deleted file mode 100644 index 7d01a9c622c620babc6d9bb35ca1d8d9e1bca9eb..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/explore/concepts_resolver.go +++ /dev/null @@ -1,147 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package explore - -import ( - "context" - "fmt" - - "github.com/tailor-inc/graphql" - "github.com/tailor-inc/graphql/language/ast" - "github.com/weaviate/weaviate/adapters/handlers/graphql/local/common_filters" - enterrors "github.com/weaviate/weaviate/entities/errors" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/search" - "github.com/weaviate/weaviate/usecases/traverser" -) - -// Resolver is a local interface that can be composed with other interfaces to -// form the overall GraphQL API main interface. All data-base connectors that -// want to support the Meta feature must implement this interface. -type Resolver interface { - Explore(ctx context.Context, principal *models.Principal, - params traverser.ExploreParams) ([]search.Result, error) -} - -// RequestsLog is a local abstraction on the RequestsLog that needs to be -// provided to the graphQL API in order to log Local.Fetch queries. -type RequestsLog interface { - Register(requestType string, identifier string) -} - -type resources struct { - resolver Resolver -} - -func newResources(s interface{}) (*resources, error) { - source, ok := s.(map[string]interface{}) - if !ok { - return nil, fmt.Errorf("expected source to be a map, but was %T", source) - } - - resolver, ok := source["Resolver"].(Resolver) - if !ok { - return nil, fmt.Errorf("expected source to contain a usable Resolver, but was %#v", source) - } - - return &resources{ - resolver: resolver, - }, nil -} - -type resolver struct { - modulesProvider ModulesProvider -} - -func newResolver(modulesProvider ModulesProvider) *resolver { - return &resolver{modulesProvider} -} - -func (r *resolver) resolve(p graphql.ResolveParams) (interface{}, error) { - result, err := r.resolveExplore(p) - if err != nil { - return result, enterrors.NewErrGraphQLUser(err, "Explore", "") - } - return result, nil -} - -func (r *resolver) resolveExplore(p graphql.ResolveParams) (interface{}, error) { - resources, err := newResources(p.Source) - if err != nil { - return nil, err - } - - params := traverser.ExploreParams{} - - if param, ok := p.Args["nearVector"]; ok { - extracted, err := common_filters.ExtractNearVector(param.(map[string]interface{})) - if err != nil { - return nil, fmt.Errorf("failed to extract nearVector params: %s", err) - } - params.NearVector = &extracted - } - - if param, ok := p.Args["nearObject"]; ok { - extracted, err := common_filters.ExtractNearObject(param.(map[string]interface{})) - if err != nil { - return nil, fmt.Errorf("failed to extract nearObject params: %s", err) - } - params.NearObject = &extracted - } - - if param, ok := p.Args["offset"]; ok { - params.Offset = param.(int) - } - - if param, ok := p.Args["limit"]; ok { - params.Limit = param.(int) - } - - if r.modulesProvider != nil { - extractedParams := r.modulesProvider.CrossClassExtractSearchParams(p.Args) - if len(extractedParams) > 0 { - params.ModuleParams = extractedParams - } - } - - if containsCertaintyProperty(p.Info) { - params.WithCertaintyProp = true - } - - return resources.resolver.Explore(p.Context, - principalFromContext(p.Context), params) -} - -func principalFromContext(ctx context.Context) *models.Principal { - principal := ctx.Value("principal") - if principal == nil { - return nil - } - - return principal.(*models.Principal) -} - -func containsCertaintyProperty(info graphql.ResolveInfo) bool { - if len(info.FieldASTs) == 0 { - return false - } - - for _, selection := range info.FieldASTs[0].SelectionSet.Selections { - field := selection.(*ast.Field) - name := field.Name.Value - if name == "certainty" { - return true - } - } - - return false -} diff --git a/adapters/handlers/graphql/local/explore/concepts_test.go b/adapters/handlers/graphql/local/explore/concepts_test.go deleted file mode 100644 index dfe533871e6562eac19321a715f590b45ab6dc7f..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/explore/concepts_test.go +++ /dev/null @@ -1,713 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package explore - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/search" - "github.com/weaviate/weaviate/entities/searchparams" - helper "github.com/weaviate/weaviate/test/helper" - "github.com/weaviate/weaviate/usecases/traverser" -) - -type testCase struct { - name string - query string - expectedParamsToTraverser traverser.ExploreParams - resolverReturn []search.Result - expectedResults []result -} - -type testCases []testCase - -type result struct { - pathToField []string - expectedValue interface{} -} - -func Test_ResolveExplore(t *testing.T) { - t.Parallel() - - testsNearText := testCases{ - testCase{ - name: "Resolve Explore with nearCustomText", - query: ` - { - Explore(nearCustomText: {concepts: ["car", "best brand"]}) { - beacon className certainty distance - } - }`, - expectedParamsToTraverser: traverser.ExploreParams{ - ModuleParams: map[string]interface{}{ - "nearCustomText": extractNearCustomTextParam(map[string]interface{}{ - "concepts": []interface{}{"car", "best brand"}, - }), - }, - WithCertaintyProp: true, - }, - resolverReturn: []search.Result{ - { - Beacon: "weaviate://localhost/some-uuid", - ClassName: "bestClass", - Certainty: 0.7, - Dist: helper.CertaintyToDist(t, 0.7), - }, - }, - expectedResults: []result{{ - pathToField: []string{"Explore"}, - expectedValue: []interface{}{ - map[string]interface{}{ - "beacon": "weaviate://localhost/some-uuid", - "className": "bestClass", - "certainty": float32(0.7), - "distance": helper.CertaintyToDist(t, 0.7), - }, - }, - }}, - }, - - testCase{ - name: "with nearCustomText with optional limit and distance set", - query: ` - { - Explore( - nearCustomText: {concepts: ["car", "best brand"], distance: 0.4}, limit: 17 - ){ - beacon className - } - }`, - expectedParamsToTraverser: traverser.ExploreParams{ - ModuleParams: map[string]interface{}{ - "nearCustomText": extractNearCustomTextParam(map[string]interface{}{ - "concepts": []interface{}{"car", "best brand"}, - "distance": float64(0.4), - }), - }, - Limit: 17, - }, - resolverReturn: []search.Result{ - { - Beacon: "weaviate://localhost/some-uuid", - ClassName: "bestClass", - }, - }, - expectedResults: []result{{ - pathToField: []string{"Explore"}, - expectedValue: []interface{}{ - map[string]interface{}{ - "beacon": "weaviate://localhost/some-uuid", - "className": "bestClass", - }, - }, - }}, - }, - - testCase{ - name: "with nearCustomText with optional limit and certainty set", - query: ` - { - Explore( - nearCustomText: {concepts: ["car", "best brand"], certainty: 0.6}, limit: 17 - ){ - beacon className - } - }`, - expectedParamsToTraverser: traverser.ExploreParams{ - ModuleParams: map[string]interface{}{ - "nearCustomText": extractNearCustomTextParam(map[string]interface{}{ - "concepts": []interface{}{"car", "best brand"}, - "certainty": float64(0.6), - }), - }, - Limit: 17, - }, - resolverReturn: []search.Result{ - { - Beacon: "weaviate://localhost/some-uuid", - ClassName: "bestClass", - }, - }, - expectedResults: []result{{ - pathToField: []string{"Explore"}, - expectedValue: []interface{}{ - map[string]interface{}{ - "beacon": "weaviate://localhost/some-uuid", - "className": "bestClass", - }, - }, - }}, - }, - - testCase{ - name: "with moveTo set", - query: ` - { - Explore( - limit: 17 - nearCustomText: { - concepts: ["car", "best brand"] - moveTo: { - concepts: ["mercedes"] - force: 0.7 - } - } - ) { - beacon className - } - }`, - expectedParamsToTraverser: traverser.ExploreParams{ - Limit: 17, - ModuleParams: map[string]interface{}{ - "nearCustomText": extractNearCustomTextParam(map[string]interface{}{ - "concepts": []interface{}{"car", "best brand"}, - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"mercedes"}, - "force": float64(0.7), - }, - }), - }, - }, - resolverReturn: []search.Result{ - { - Beacon: "weaviate://localhost/some-uuid", - ClassName: "bestClass", - }, - }, - expectedResults: []result{{ - pathToField: []string{"Explore"}, - expectedValue: []interface{}{ - map[string]interface{}{ - "beacon": "weaviate://localhost/some-uuid", - "className": "bestClass", - }, - }, - }}, - }, - - testCase{ - name: "with moveTo and moveAwayFrom set", - query: ` - { - Explore( - limit: 17 - nearCustomText: { - concepts: ["car", "best brand"] - moveTo: { - concepts: ["mercedes"] - force: 0.7 - } - moveAwayFrom: { - concepts: ["van"] - force: 0.7 - } - } - ) { - beacon className - } - }`, - expectedParamsToTraverser: traverser.ExploreParams{ - Limit: 17, - ModuleParams: map[string]interface{}{ - "nearCustomText": extractNearCustomTextParam(map[string]interface{}{ - "concepts": []interface{}{"car", "best brand"}, - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"mercedes"}, - "force": float64(0.7), - }, - "moveAwayFrom": map[string]interface{}{ - "concepts": []interface{}{"van"}, - "force": float64(0.7), - }, - }), - }, - }, - resolverReturn: []search.Result{ - { - Beacon: "weaviate://localhost/some-uuid", - ClassName: "bestClass", - }, - }, - expectedResults: []result{{ - pathToField: []string{"Explore"}, - expectedValue: []interface{}{ - map[string]interface{}{ - "beacon": "weaviate://localhost/some-uuid", - "className": "bestClass", - }, - }, - }}, - }, - - testCase{ - name: "with moveTo and objects set", - query: ` - { - Explore( - limit: 17 - nearCustomText: { - concepts: ["car", "best brand"] - moveTo: { - concepts: ["mercedes"] - force: 0.7 - objects: [ - {id: "moveto-uuid"}, - {beacon: "weaviate://localhost/other-moveto-uuid"}, - ] - } - } - ) { - beacon className - } - }`, - expectedParamsToTraverser: traverser.ExploreParams{ - Limit: 17, - ModuleParams: map[string]interface{}{ - "nearCustomText": extractNearCustomTextParam(map[string]interface{}{ - "concepts": []interface{}{"car", "best brand"}, - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"mercedes"}, - "force": float64(0.7), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveto-uuid", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/other-moveto-uuid", - }, - }, - }, - }), - }, - }, - resolverReturn: []search.Result{ - { - Beacon: "weaviate://localhost/some-uuid", - ClassName: "bestClass", - }, - }, - expectedResults: []result{{ - pathToField: []string{"Explore"}, - expectedValue: []interface{}{ - map[string]interface{}{ - "beacon": "weaviate://localhost/some-uuid", - "className": "bestClass", - }, - }, - }}, - }, - - testCase{ - name: "with moveTo and objects set", - query: ` - { - Explore( - limit: 17 - nearCustomText: { - concepts: ["car", "best brand"] - moveTo: { - concepts: ["mercedes"] - force: 0.7 - objects: [ - {id: "moveto-uuid1"}, - {beacon: "weaviate://localhost/moveto-uuid2"}, - ] - } - moveAwayFrom: { - concepts: ["van"] - force: 0.7 - objects: [ - {id: "moveAway-uuid1"}, - {beacon: "weaviate://localhost/moveAway-uuid2"}, - {id: "moveAway-uuid3"}, - {id: "moveAway-uuid4"}, - ] - } - } - ) { - beacon className - } - }`, - expectedParamsToTraverser: traverser.ExploreParams{ - Limit: 17, - ModuleParams: map[string]interface{}{ - "nearCustomText": extractNearCustomTextParam(map[string]interface{}{ - "concepts": []interface{}{"car", "best brand"}, - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"mercedes"}, - "force": float64(0.7), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveto-uuid1", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveto-uuid2", - }, - }, - }, - "moveAwayFrom": map[string]interface{}{ - "concepts": []interface{}{"van"}, - "force": float64(0.7), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveAway-uuid1", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveAway-uuid2", - }, - map[string]interface{}{ - "id": "moveAway-uuid3", - }, - map[string]interface{}{ - "id": "moveAway-uuid4", - }, - }, - }, - }), - }, - }, - resolverReturn: []search.Result{ - { - Beacon: "weaviate://localhost/some-uuid", - ClassName: "bestClass", - }, - }, - expectedResults: []result{{ - pathToField: []string{"Explore"}, - expectedValue: []interface{}{ - map[string]interface{}{ - "beacon": "weaviate://localhost/some-uuid", - "className": "bestClass", - }, - }, - }}, - }, - } - - tests := testCases{ - testCase{ - name: "Resolve Explore with nearVector", - query: ` - { - Explore(nearVector: {vector: [0, 1, 0.8]}) { - beacon className certainty distance - } - }`, - expectedParamsToTraverser: traverser.ExploreParams{ - NearVector: &searchparams.NearVector{ - Vector: []float32{0, 1, 0.8}, - }, - WithCertaintyProp: true, - }, - resolverReturn: []search.Result{ - { - Beacon: "weaviate://localhost/some-uuid", - ClassName: "bestClass", - Certainty: 0.7, - Dist: helper.CertaintyToDist(t, 0.7), - }, - }, - expectedResults: []result{{ - pathToField: []string{"Explore"}, - expectedValue: []interface{}{ - map[string]interface{}{ - "beacon": "weaviate://localhost/some-uuid", - "className": "bestClass", - "certainty": float32(0.7), - "distance": helper.CertaintyToDist(t, 0.7), - }, - }, - }}, - }, - - testCase{ - name: "with nearVector with optional limit", - query: ` - { - Explore(limit: 17, nearVector: {vector: [0, 1, 0.8]}) { - beacon className certainty distance - } - }`, - expectedParamsToTraverser: traverser.ExploreParams{ - NearVector: &searchparams.NearVector{ - Vector: []float32{0, 1, 0.8}, - }, - Limit: 17, - WithCertaintyProp: true, - }, - resolverReturn: []search.Result{ - { - Beacon: "weaviate://localhost/some-uuid", - ClassName: "bestClass", - Certainty: 0.7, - Dist: helper.CertaintyToDist(t, 0.7), - }, - }, - expectedResults: []result{{ - pathToField: []string{"Explore"}, - expectedValue: []interface{}{ - map[string]interface{}{ - "beacon": "weaviate://localhost/some-uuid", - "className": "bestClass", - "certainty": float32(0.7), - "distance": helper.CertaintyToDist(t, 0.7), - }, - }, - }}, - }, - - testCase{ - name: "Resolve Explore with nearObject, distance, and beacon set", - query: ` - { - Explore( - nearObject: { - beacon: "weaviate://localhost/27b5213d-e152-4fea-bd63-2063d529024d" - distance: 0.3 - }) { - beacon className certainty distance - } - }`, - expectedParamsToTraverser: traverser.ExploreParams{ - NearObject: &searchparams.NearObject{ - Beacon: "weaviate://localhost/27b5213d-e152-4fea-bd63-2063d529024d", - Distance: float64(0.3), - WithDistance: true, - }, - WithCertaintyProp: true, - }, - resolverReturn: []search.Result{ - { - Beacon: "weaviate://localhost/27b5213d-e152-4fea-bd63-2063d529024d", - ClassName: "bestClass", - Certainty: 0.7, - Dist: helper.CertaintyToDist(t, 0.7), - }, - }, - expectedResults: []result{{ - pathToField: []string{"Explore"}, - expectedValue: []interface{}{ - map[string]interface{}{ - "beacon": "weaviate://localhost/27b5213d-e152-4fea-bd63-2063d529024d", - "className": "bestClass", - "certainty": float32(0.7), - "distance": helper.CertaintyToDist(t, 0.7), - }, - }, - }}, - }, - - testCase{ - name: "Resolve Explore with nearObject, certainty, and beacon set", - query: ` - { - Explore( - nearObject: { - beacon: "weaviate://localhost/27b5213d-e152-4fea-bd63-2063d529024d" - certainty: 0.7 - }) { - beacon className certainty distance - } - }`, - expectedParamsToTraverser: traverser.ExploreParams{ - NearObject: &searchparams.NearObject{ - Beacon: "weaviate://localhost/27b5213d-e152-4fea-bd63-2063d529024d", - Certainty: 0.7, - }, - WithCertaintyProp: true, - }, - resolverReturn: []search.Result{ - { - Beacon: "weaviate://localhost/27b5213d-e152-4fea-bd63-2063d529024d", - ClassName: "bestClass", - Certainty: 0.7, - Dist: helper.CertaintyToDist(t, 0.7), - }, - }, - expectedResults: []result{{ - pathToField: []string{"Explore"}, - expectedValue: []interface{}{ - map[string]interface{}{ - "beacon": "weaviate://localhost/27b5213d-e152-4fea-bd63-2063d529024d", - "className": "bestClass", - "certainty": float32(0.7), - "distance": helper.CertaintyToDist(t, 0.7), - }, - }, - }}, - }, - - testCase{ - name: "Resolve Explore with nearObject, distance and id set", - query: ` - { - Explore( - limit: 17 - nearObject: { - id: "27b5213d-e152-4fea-bd63-2063d529024d" - distance: 0.3 - } - ) { - beacon className - } - }`, - expectedParamsToTraverser: traverser.ExploreParams{ - Limit: 17, - NearObject: &searchparams.NearObject{ - ID: "27b5213d-e152-4fea-bd63-2063d529024d", - Distance: 0.3, - WithDistance: true, - }, - }, - resolverReturn: []search.Result{ - { - Beacon: "weaviate://localhost/some-uuid", - ClassName: "bestClass", - }, - }, - expectedResults: []result{{ - pathToField: []string{"Explore"}, - expectedValue: []interface{}{ - map[string]interface{}{ - "beacon": "weaviate://localhost/some-uuid", - "className": "bestClass", - }, - }, - }}, - }, - - testCase{ - name: "Resolve Explore with nearObject, certainty and id set", - query: ` - { - Explore( - limit: 17 - nearObject: { - id: "27b5213d-e152-4fea-bd63-2063d529024d" - certainty: 0.7 - } - ) { - beacon className - } - }`, - expectedParamsToTraverser: traverser.ExploreParams{ - Limit: 17, - NearObject: &searchparams.NearObject{ - ID: "27b5213d-e152-4fea-bd63-2063d529024d", - Certainty: 0.7, - }, - }, - resolverReturn: []search.Result{ - { - Beacon: "weaviate://localhost/some-uuid", - ClassName: "bestClass", - }, - }, - expectedResults: []result{{ - pathToField: []string{"Explore"}, - expectedValue: []interface{}{ - map[string]interface{}{ - "beacon": "weaviate://localhost/some-uuid", - "className": "bestClass", - }, - }, - }}, - }, - } - - tests.AssertExtraction(t, newMockResolver()) - testsNearText.AssertExtraction(t, newMockResolver()) - tests.AssertExtraction(t, newMockResolverNoModules()) -} - -func Test_ExploreWithNoText2VecClasses(t *testing.T) { - t.Run("with distance", func(t *testing.T) { - resolver := newMockResolverEmptySchema() - query := ` - { - Explore( - nearCustomText: {concepts: ["car", "best brand"], distance: 0.6}, limit: 17 - ){ - beacon className - } - }` - res := resolver.Resolve(query) - require.Len(t, res.Errors, 1) - assert.Contains(t, res.Errors[0].Message, "Unknown argument \"nearCustomText\" on field \"Explore\"") - }) - - t.Run("with certainty", func(t *testing.T) { - resolver := newMockResolverEmptySchema() - query := ` - { - Explore( - nearCustomText: {concepts: ["car", "best brand"], certainty: 0.6}, limit: 17 - ){ - beacon className - } - }` - res := resolver.Resolve(query) - require.Len(t, res.Errors, 1) - assert.Contains(t, res.Errors[0].Message, "Unknown argument \"nearCustomText\" on field \"Explore\"") - }) -} - -func Test_ExploreWithNoModules(t *testing.T) { - t.Run("with distance", func(t *testing.T) { - resolver := newMockResolverNoModules() - query := ` - { - Explore( - nearCustomText: {concepts: ["car", "best brand"], distance: 0.6}, limit: 17 - ){ - beacon className - } - }` - res := resolver.Resolve(query) - require.Len(t, res.Errors, 1) - assert.Contains(t, res.Errors[0].Message, "Unknown argument \"nearCustomText\" on field \"Explore\"") - }) - - t.Run("with certainty", func(t *testing.T) { - resolver := newMockResolverNoModules() - query := ` - { - Explore( - nearCustomText: {concepts: ["car", "best brand"], certainty: 0.6}, limit: 17 - ){ - beacon className - } - }` - res := resolver.Resolve(query) - require.Len(t, res.Errors, 1) - assert.Contains(t, res.Errors[0].Message, "Unknown argument \"nearCustomText\" on field \"Explore\"") - }) -} - -func (tests testCases) AssertExtraction(t *testing.T, resolver *mockResolver) { - for _, testCase := range tests { - t.Run(testCase.name, func(t *testing.T) { - resolver.On("Explore", testCase.expectedParamsToTraverser). - Return(testCase.resolverReturn, nil).Once() - - result := resolver.AssertResolve(t, testCase.query) - - for _, expectedResult := range testCase.expectedResults { - value := result.Get(expectedResult.pathToField...).Result - - assert.Equal(t, expectedResult.expectedValue, value) - } - }) - } -} diff --git a/adapters/handlers/graphql/local/explore/helpers_for_test.go b/adapters/handlers/graphql/local/explore/helpers_for_test.go deleted file mode 100644 index 9a842307869f560cce9b79a1daa27c859cd3e129..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/explore/helpers_for_test.go +++ /dev/null @@ -1,338 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package explore - -import ( - "context" - "fmt" - "net/http" - - "github.com/tailor-inc/graphql" - "github.com/weaviate/weaviate/adapters/handlers/graphql/descriptions" - testhelper "github.com/weaviate/weaviate/adapters/handlers/graphql/test/helper" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/search" - "github.com/weaviate/weaviate/usecases/traverser" -) - -type mockRequestsLog struct{} - -func (m *mockRequestsLog) Register(first string, second string) { -} - -type mockResolver struct { - testhelper.MockResolver -} - -type fakeModulesProvider struct{} - -func (p *fakeModulesProvider) VectorFromInput(ctx context.Context, className string, input string) ([]float32, error) { - panic("not implemented") -} - -func (p *fakeModulesProvider) ExploreArguments(schema *models.Schema) map[string]*graphql.ArgumentConfig { - args := map[string]*graphql.ArgumentConfig{} - txt2vec := &nearCustomTextModule{} - for _, c := range schema.Classes { - if c.Vectorizer == txt2vec.Name() { - for name, argument := range txt2vec.Arguments() { - args[name] = argument.ExploreArgumentsFunction() - } - } - } - return args -} - -func (p *fakeModulesProvider) CrossClassExtractSearchParams(arguments map[string]interface{}) map[string]interface{} { - exractedParams := map[string]interface{}{} - if param, ok := arguments["nearCustomText"]; ok { - exractedParams["nearCustomText"] = extractNearCustomTextParam(param.(map[string]interface{})) - } - return exractedParams -} - -func extractNearCustomTextParam(param map[string]interface{}) interface{} { - nearCustomText := &nearCustomTextModule{} - argument := nearCustomText.Arguments()["nearCustomText"] - return argument.ExtractFunction(param) -} - -func getFakeModulesProvider() ModulesProvider { - return &fakeModulesProvider{} -} - -func newMockResolver() *mockResolver { - field := Build(testhelper.SimpleSchema.Objects, getFakeModulesProvider()) - mocker := &mockResolver{} - mockLog := &mockRequestsLog{} - mocker.RootFieldName = "Explore" - mocker.RootField = field - mocker.RootObject = map[string]interface{}{ - "Resolver": Resolver(mocker), - "RequestsLog": mockLog, - } - return mocker -} - -func newMockResolverNoModules() *mockResolver { - field := Build(testhelper.SimpleSchema.Objects, nil) - mocker := &mockResolver{} - mockLog := &mockRequestsLog{} - mocker.RootFieldName = "Explore" - mocker.RootField = field - mocker.RootObject = map[string]interface{}{ - "Resolver": Resolver(mocker), - "RequestsLog": mockLog, - } - return mocker -} - -func newMockResolverEmptySchema() *mockResolver { - field := Build(&models.Schema{}, getFakeModulesProvider()) - mocker := &mockResolver{} - mockLog := &mockRequestsLog{} - mocker.RootFieldName = "Explore" - mocker.RootField = field - mocker.RootObject = map[string]interface{}{ - "Resolver": Resolver(mocker), - "RequestsLog": mockLog, - } - return mocker -} - -func (m *mockResolver) Explore(ctx context.Context, - principal *models.Principal, params traverser.ExploreParams, -) ([]search.Result, error) { - args := m.Called(params) - return args.Get(0).([]search.Result), args.Error(1) -} - -type nearCustomTextParams struct { - Values []string - MoveTo nearExploreMove - MoveAwayFrom nearExploreMove - Certainty float64 - Distance float64 - WithDistance bool -} - -type nearExploreMove struct { - Values []string - Force float32 - Objects []nearObjectMove -} - -type nearObjectMove struct { - ID string - Beacon string -} - -type nearCustomTextModule struct{} - -func (m *nearCustomTextModule) Name() string { - return "text2vec-contextionary" -} - -func (m *nearCustomTextModule) Init(params moduletools.ModuleInitParams) error { - return nil -} - -func (m *nearCustomTextModule) RootHandler() http.Handler { - return nil -} - -func (m *nearCustomTextModule) Arguments() map[string]modulecapabilities.GraphQLArgument { - arguments := map[string]modulecapabilities.GraphQLArgument{} - // define nearCustomText argument - arguments["nearCustomText"] = modulecapabilities.GraphQLArgument{ - GetArgumentsFunction: func(classname string) *graphql.ArgumentConfig { - return m.getNearCustomTextArgument(classname) - }, - ExploreArgumentsFunction: func() *graphql.ArgumentConfig { - return m.getNearCustomTextArgument("") - }, - ExtractFunction: func(source map[string]interface{}) interface{} { - return m.extractNearCustomTextArgument(source) - }, - ValidateFunction: func(param interface{}) error { - // all is valid - return nil - }, - } - return arguments -} - -func (m *nearCustomTextModule) getNearCustomTextArgument(classname string) *graphql.ArgumentConfig { - prefix := classname - return &graphql.ArgumentConfig{ - Type: graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sNearCustomTextInpObj", prefix), - Fields: graphql.InputObjectConfigFieldMap{ - "concepts": &graphql.InputObjectFieldConfig{ - Type: graphql.NewNonNull(graphql.NewList(graphql.String)), - }, - "moveTo": &graphql.InputObjectFieldConfig{ - Description: descriptions.VectorMovement, - Type: graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sMoveTo", prefix), - Fields: graphql.InputObjectConfigFieldMap{ - "concepts": &graphql.InputObjectFieldConfig{ - Description: descriptions.Keywords, - Type: graphql.NewList(graphql.String), - }, - "objects": &graphql.InputObjectFieldConfig{ - Description: "objects", - Type: graphql.NewList(graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sMovementObjectsToInpObj", prefix), - Fields: graphql.InputObjectConfigFieldMap{ - "id": &graphql.InputObjectFieldConfig{ - Type: graphql.String, - Description: "id of an object", - }, - "beacon": &graphql.InputObjectFieldConfig{ - Type: graphql.String, - Description: descriptions.Beacon, - }, - }, - Description: "Movement Object", - }, - )), - }, - "force": &graphql.InputObjectFieldConfig{ - Description: descriptions.Force, - Type: graphql.NewNonNull(graphql.Float), - }, - }, - }), - }, - "moveAwayFrom": &graphql.InputObjectFieldConfig{ - Description: descriptions.VectorMovement, - Type: graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sMoveAway", prefix), - Fields: graphql.InputObjectConfigFieldMap{ - "concepts": &graphql.InputObjectFieldConfig{ - Description: descriptions.Keywords, - Type: graphql.NewList(graphql.String), - }, - "objects": &graphql.InputObjectFieldConfig{ - Description: "objects", - Type: graphql.NewList(graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sMovementObjectsAwayInpObj", prefix), - Fields: graphql.InputObjectConfigFieldMap{ - "id": &graphql.InputObjectFieldConfig{ - Type: graphql.String, - Description: "id of an object", - }, - "beacon": &graphql.InputObjectFieldConfig{ - Type: graphql.String, - Description: descriptions.Beacon, - }, - }, - Description: "Movement Object", - }, - )), - }, - "force": &graphql.InputObjectFieldConfig{ - Description: descriptions.Force, - Type: graphql.NewNonNull(graphql.Float), - }, - }, - }), - }, - "certainty": &graphql.InputObjectFieldConfig{ - Description: descriptions.Certainty, - Type: graphql.Float, - }, - "distance": &graphql.InputObjectFieldConfig{ - Description: descriptions.Distance, - Type: graphql.Float, - }, - }, - Description: descriptions.GetWhereInpObj, - }, - ), - } -} - -func (m *nearCustomTextModule) extractNearCustomTextArgument(source map[string]interface{}) *nearCustomTextParams { - var args nearCustomTextParams - - concepts := source["concepts"].([]interface{}) - args.Values = make([]string, len(concepts)) - for i, value := range concepts { - args.Values[i] = value.(string) - } - - certainty, ok := source["certainty"] - if ok { - args.Certainty = certainty.(float64) - } - - distance, ok := source["distance"] - if ok { - args.Distance = distance.(float64) - args.WithDistance = true - } - - // moveTo is an optional arg, so it could be nil - moveTo, ok := source["moveTo"] - if ok { - moveToMap := moveTo.(map[string]interface{}) - args.MoveTo = m.parseMoveParam(moveToMap) - } - - moveAwayFrom, ok := source["moveAwayFrom"] - if ok { - moveAwayFromMap := moveAwayFrom.(map[string]interface{}) - args.MoveAwayFrom = m.parseMoveParam(moveAwayFromMap) - } - - return &args -} - -func (m *nearCustomTextModule) parseMoveParam(source map[string]interface{}) nearExploreMove { - res := nearExploreMove{} - res.Force = float32(source["force"].(float64)) - - concepts, ok := source["concepts"].([]interface{}) - if ok { - res.Values = make([]string, len(concepts)) - for i, value := range concepts { - res.Values[i] = value.(string) - } - } - - objects, ok := source["objects"].([]interface{}) - if ok { - res.Objects = make([]nearObjectMove, len(objects)) - for i, value := range objects { - v, ok := value.(map[string]interface{}) - if ok { - if v["id"] != nil { - res.Objects[i].ID = v["id"].(string) - } - if v["beacon"] != nil { - res.Objects[i].Beacon = v["beacon"].(string) - } - } - } - } - - return res -} diff --git a/adapters/handlers/graphql/local/get/class_builder.go b/adapters/handlers/graphql/local/get/class_builder.go deleted file mode 100644 index 2da647861fb89b0e49204fb09006ec303c19e12e..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/get/class_builder.go +++ /dev/null @@ -1,302 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package get - -import ( - "fmt" - - "github.com/weaviate/weaviate/adapters/handlers/graphql/local/common_filters" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/tailor-inc/graphql" - "github.com/weaviate/weaviate/adapters/handlers/graphql/descriptions" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" -) - -type classBuilder struct { - schema *schema.Schema - knownClasses map[string]*graphql.Object - beaconClass *graphql.Object - logger logrus.FieldLogger - modulesProvider ModulesProvider -} - -func newClassBuilder(schema *schema.Schema, logger logrus.FieldLogger, - modulesProvider ModulesProvider, -) *classBuilder { - b := &classBuilder{} - - b.logger = logger - b.schema = schema - b.modulesProvider = modulesProvider - - b.initKnownClasses() - b.initBeaconClass() - - return b -} - -func (b *classBuilder) initKnownClasses() { - b.knownClasses = map[string]*graphql.Object{} -} - -func (b *classBuilder) initBeaconClass() { - b.beaconClass = graphql.NewObject(graphql.ObjectConfig{ - Name: "Beacon", - Fields: graphql.Fields{ - "beacon": &graphql.Field{ - Type: graphql.String, - }, - }, - }) -} - -func (b *classBuilder) objects() (*graphql.Object, error) { - return b.kinds(b.schema.Objects) -} - -func (b *classBuilder) kinds(kindSchema *models.Schema) (*graphql.Object, error) { - // needs to be defined outside the individual class as there can only be one definition of an enum - fusionAlgoEnum := graphql.NewEnum(graphql.EnumConfig{ - Name: "FusionEnum", - Values: graphql.EnumValueConfigMap{ - "rankedFusion": &graphql.EnumValueConfig{ - Value: common_filters.HybridRankedFusion, - }, - "relativeScoreFusion": &graphql.EnumValueConfig{ - Value: common_filters.HybridRelativeScoreFusion, - }, - }, - }) - classFields := graphql.Fields{} - for _, class := range kindSchema.Classes { - classField, err := b.classField(class, fusionAlgoEnum) - if err != nil { - return nil, fmt.Errorf("Could not build class for %s", class.Class) - } - classFields[class.Class] = classField - } - - classes := graphql.NewObject(graphql.ObjectConfig{ - Name: "GetObjectsObj", - Fields: classFields, - Description: descriptions.GetObjectsActionsObj, - }) - - return classes, nil -} - -func (b *classBuilder) classField(class *models.Class, fusionEnum *graphql.Enum) (*graphql.Field, error) { - classObject := b.classObject(class) - b.knownClasses[class.Class] = classObject - classField := buildGetClassField(classObject, class, b.modulesProvider, fusionEnum) - return &classField, nil -} - -func (b *classBuilder) classObject(class *models.Class) *graphql.Object { - return graphql.NewObject(graphql.ObjectConfig{ - Name: class.Class, - Fields: (graphql.FieldsThunk)(func() graphql.Fields { - classProperties := graphql.Fields{} - for _, property := range class.Properties { - propertyType, err := b.schema.FindPropertyDataType(property.DataType) - if err != nil { - if errors.Is(err, schema.ErrRefToNonexistentClass) { - // This is a common case when a class which is referenced - // by another class is deleted, leaving the referencing - // class with an invalid reference property. Panicking - // is not necessary here - b.logger.WithField("action", "graphql_rebuild"). - Warnf("ignoring ref prop %q on class %q, because it contains reference to nonexistent class %q", - property.Name, class.Class, property.DataType) - - continue - } else { - // We can't return an error in this FieldsThunk function, so we need to panic - panic(fmt.Sprintf("buildGetClass: wrong propertyType for %s.%s; %s", - class.Class, property.Name, err.Error())) - } - } - - if propertyType.IsPrimitive() { - classProperties[property.Name] = b.primitiveField(propertyType, property, - class.Class) - } else if propertyType.IsNested() { - classProperties[property.Name] = b.nestedField(propertyType, property, - class.Class) - } else { - classProperties[property.Name] = b.referenceField(propertyType, property, - class.Class) - } - } - - b.additionalFields(classProperties, class) - - return classProperties - }), - Description: class.Description, - }) -} - -func (b *classBuilder) additionalFields(classProperties graphql.Fields, class *models.Class) { - additionalProperties := graphql.Fields{} - additionalProperties["classification"] = b.additionalClassificationField(class) - additionalProperties["certainty"] = b.additionalCertaintyField(class) - additionalProperties["distance"] = b.additionalDistanceField(class) - additionalProperties["vector"] = b.additionalVectorField(class) - additionalProperties["id"] = b.additionalIDField() - additionalProperties["creationTimeUnix"] = b.additionalCreationTimeUnix() - additionalProperties["lastUpdateTimeUnix"] = b.additionalLastUpdateTimeUnix() - additionalProperties["score"] = b.additionalScoreField() - additionalProperties["explainScore"] = b.additionalExplainScoreField() - additionalProperties["group"] = b.additionalGroupField(classProperties, class) - if replicationEnabled(class) { - additionalProperties["isConsistent"] = b.isConsistentField() - } - // module specific additional properties - if b.modulesProvider != nil { - for name, field := range b.modulesProvider.GetAdditionalFields(class) { - additionalProperties[name] = field - } - } - classProperties["_additional"] = &graphql.Field{ - Type: graphql.NewObject(graphql.ObjectConfig{ - Name: fmt.Sprintf("%sAdditional", class.Class), - Fields: additionalProperties, - }), - } -} - -func (b *classBuilder) additionalIDField() *graphql.Field { - return &graphql.Field{ - Description: descriptions.GetClassUUID, - Type: graphql.String, - } -} - -func (b *classBuilder) additionalClassificationField(class *models.Class) *graphql.Field { - return &graphql.Field{ - Type: graphql.NewObject(graphql.ObjectConfig{ - Name: fmt.Sprintf("%sAdditionalClassification", class.Class), - Fields: graphql.Fields{ - "id": &graphql.Field{Type: graphql.String}, - "basedOn": &graphql.Field{Type: graphql.NewList(graphql.String)}, - "scope": &graphql.Field{Type: graphql.NewList(graphql.String)}, - "classifiedFields": &graphql.Field{Type: graphql.NewList(graphql.String)}, - "completed": &graphql.Field{Type: graphql.String}, - }, - }), - } -} - -func (b *classBuilder) additionalCertaintyField(class *models.Class) *graphql.Field { - return &graphql.Field{ - Type: graphql.Float, - } -} - -func (b *classBuilder) additionalDistanceField(class *models.Class) *graphql.Field { - return &graphql.Field{ - Type: graphql.Float, - } -} - -func (b *classBuilder) additionalVectorField(class *models.Class) *graphql.Field { - return &graphql.Field{ - Type: graphql.NewList(graphql.Float), - } -} - -func (b *classBuilder) additionalCreationTimeUnix() *graphql.Field { - return &graphql.Field{ - Type: graphql.String, - } -} - -func (b *classBuilder) additionalScoreField() *graphql.Field { - return &graphql.Field{ - Type: graphql.String, - } -} - -func (b *classBuilder) additionalExplainScoreField() *graphql.Field { - return &graphql.Field{ - Type: graphql.String, - } -} - -func (b *classBuilder) additionalLastUpdateTimeUnix() *graphql.Field { - return &graphql.Field{ - Type: graphql.String, - } -} - -func (b *classBuilder) isConsistentField() *graphql.Field { - return &graphql.Field{ - Type: graphql.Boolean, - } -} - -func (b *classBuilder) additionalGroupField(classProperties graphql.Fields, class *models.Class) *graphql.Field { - hitsFields := graphql.Fields{ - "_additional": &graphql.Field{ - Type: graphql.NewObject( - graphql.ObjectConfig{ - Name: fmt.Sprintf("%sAdditionalGroupHitsAdditional", class.Class), - Fields: graphql.Fields{ - "id": &graphql.Field{Type: graphql.String}, - "vector": &graphql.Field{Type: graphql.NewList(graphql.Float)}, - "distance": &graphql.Field{Type: graphql.Float}, - }, - }, - ), - }, - } - for name, field := range classProperties { - hitsFields[name] = field - } - return &graphql.Field{ - Type: graphql.NewObject(graphql.ObjectConfig{ - Name: fmt.Sprintf("%sAdditionalGroup", class.Class), - Fields: graphql.Fields{ - "id": &graphql.Field{Type: graphql.Int}, - "groupedBy": &graphql.Field{ - Type: graphql.NewObject(graphql.ObjectConfig{ - Name: fmt.Sprintf("%sAdditionalGroupGroupedBy", class.Class), - Fields: graphql.Fields{ - "path": &graphql.Field{ - Type: graphql.NewList(graphql.String), - }, - "value": &graphql.Field{ - Type: graphql.String, - }, - }, - }), - }, - - "minDistance": &graphql.Field{Type: graphql.Float}, - "maxDistance": &graphql.Field{Type: graphql.Float}, - "count": &graphql.Field{Type: graphql.Int}, - "hits": &graphql.Field{ - Type: graphql.NewList(graphql.NewObject( - graphql.ObjectConfig{ - Name: fmt.Sprintf("%sAdditionalGroupHits", class.Class), - Fields: hitsFields, - }, - )), - }, - }, - }), - } -} diff --git a/adapters/handlers/graphql/local/get/class_builder_fields.go b/adapters/handlers/graphql/local/get/class_builder_fields.go deleted file mode 100644 index b0b993ad8e9a78045f75bd7762586088c008fc68..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/get/class_builder_fields.go +++ /dev/null @@ -1,846 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package get - -import ( - "context" - "fmt" - "regexp" - "strings" - - "github.com/tailor-inc/graphql" - "github.com/tailor-inc/graphql/language/ast" - "github.com/weaviate/weaviate/adapters/handlers/graphql/descriptions" - "github.com/weaviate/weaviate/adapters/handlers/graphql/local/common_filters" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/dto" - enterrors "github.com/weaviate/weaviate/entities/errors" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/search" - "github.com/weaviate/weaviate/entities/searchparams" -) - -func (b *classBuilder) primitiveField(propertyType schema.PropertyDataType, - property *models.Property, className string, -) *graphql.Field { - switch propertyType.AsPrimitive() { - case schema.DataTypeText: - return &graphql.Field{ - Description: property.Description, - Name: property.Name, - Type: graphql.String, - } - case schema.DataTypeInt: - return &graphql.Field{ - Description: property.Description, - Name: property.Name, - Type: graphql.Int, - } - case schema.DataTypeNumber: - return &graphql.Field{ - Description: property.Description, - Name: property.Name, - Type: graphql.Float, - } - case schema.DataTypeBoolean: - return &graphql.Field{ - Description: property.Description, - Name: property.Name, - Type: graphql.Boolean, - } - case schema.DataTypeDate: - return &graphql.Field{ - Description: property.Description, - Name: property.Name, - Type: graphql.String, // String since no graphql date datatype exists - } - case schema.DataTypeGeoCoordinates: - obj := newGeoCoordinatesObject(className, property.Name) - - return &graphql.Field{ - Description: property.Description, - Name: property.Name, - Type: obj, - Resolve: resolveGeoCoordinates, - } - case schema.DataTypePhoneNumber: - obj := newPhoneNumberObject(className, property.Name) - - return &graphql.Field{ - Description: property.Description, - Name: property.Name, - Type: obj, - Resolve: resolvePhoneNumber, - } - case schema.DataTypeBlob: - return &graphql.Field{ - Description: property.Description, - Name: property.Name, - Type: graphql.String, - } - case schema.DataTypeTextArray: - return &graphql.Field{ - Description: property.Description, - Name: property.Name, - Type: graphql.NewList(graphql.String), - } - case schema.DataTypeIntArray: - return &graphql.Field{ - Description: property.Description, - Name: property.Name, - Type: graphql.NewList(graphql.Int), - } - case schema.DataTypeNumberArray: - return &graphql.Field{ - Description: property.Description, - Name: property.Name, - Type: graphql.NewList(graphql.Float), - } - case schema.DataTypeBooleanArray: - return &graphql.Field{ - Description: property.Description, - Name: property.Name, - Type: graphql.NewList(graphql.Boolean), - } - case schema.DataTypeDateArray: - return &graphql.Field{ - Description: property.Description, - Name: property.Name, - Type: graphql.NewList(graphql.String), // String since no graphql date datatype exists - } - case schema.DataTypeUUIDArray: - return &graphql.Field{ - Description: property.Description, - Name: property.Name, - Type: graphql.NewList(graphql.String), // Always return UUID as string representation to the user - } - case schema.DataTypeUUID: - return &graphql.Field{ - Description: property.Description, - Name: property.Name, - Type: graphql.String, // Always return UUID as string representation to the user - } - default: - panic(fmt.Sprintf("buildGetClass: unknown primitive type for %s.%s; %s", - className, property.Name, propertyType.AsPrimitive())) - } -} - -func newGeoCoordinatesObject(className string, propertyName string) *graphql.Object { - return graphql.NewObject(graphql.ObjectConfig{ - Description: "GeoCoordinates as latitude and longitude in decimal form", - Name: fmt.Sprintf("%s%sGeoCoordinatesObj", className, propertyName), - Fields: graphql.Fields{ - "latitude": &graphql.Field{ - Name: "Latitude", - Description: "The Latitude of the point in decimal form.", - Type: graphql.Float, - }, - "longitude": &graphql.Field{ - Name: "Longitude", - Description: "The Longitude of the point in decimal form.", - Type: graphql.Float, - }, - }, - }) -} - -func newPhoneNumberObject(className string, propertyName string) *graphql.Object { - return graphql.NewObject(graphql.ObjectConfig{ - Description: "PhoneNumber in various parsed formats", - Name: fmt.Sprintf("%s%sPhoneNumberObj", className, propertyName), - Fields: graphql.Fields{ - "input": &graphql.Field{ - Name: "Input", - Description: "The raw phone number as put in by the user prior to parsing", - Type: graphql.String, - }, - "internationalFormatted": &graphql.Field{ - Name: "Input", - Description: "The parsed phone number in the international format", - Type: graphql.String, - }, - "nationalFormatted": &graphql.Field{ - Name: "Input", - Description: "The parsed phone number in the national format", - Type: graphql.String, - }, - "national": &graphql.Field{ - Name: "Input", - Description: "The parsed phone number in the national format", - Type: graphql.Int, - }, - "valid": &graphql.Field{ - Name: "Input", - Description: "Whether the phone number could be successfully parsed and was considered valid by the parser", - Type: graphql.Boolean, - }, - "countryCode": &graphql.Field{ - Name: "Input", - Description: "The parsed country code, i.e. the leading numbers identifing the country in an international format", - Type: graphql.Int, - }, - "defaultCountry": &graphql.Field{ - Name: "Input", - Description: "The defaultCountry as put in by the user. (This is used to help parse national numbers into an international format)", - Type: graphql.String, - }, - }, - }) -} - -func buildGetClassField(classObject *graphql.Object, - class *models.Class, modulesProvider ModulesProvider, fusionEnum *graphql.Enum, -) graphql.Field { - field := graphql.Field{ - Type: graphql.NewList(classObject), - Description: class.Description, - Args: graphql.FieldConfigArgument{ - "after": &graphql.ArgumentConfig{ - Description: descriptions.AfterID, - Type: graphql.String, - }, - "limit": &graphql.ArgumentConfig{ - Description: descriptions.Limit, - Type: graphql.Int, - }, - "offset": &graphql.ArgumentConfig{ - Description: descriptions.After, - Type: graphql.Int, - }, - "autocut": &graphql.ArgumentConfig{ - Description: "Cut off number of results after the Nth extrema. Off by default, negative numbers mean off.", - Type: graphql.Int, - }, - - "sort": sortArgument(class.Class), - "nearVector": nearVectorArgument(class.Class), - "nearObject": nearObjectArgument(class.Class), - "where": whereArgument(class.Class), - "group": groupArgument(class.Class), - "groupBy": groupByArgument(class.Class), - }, - Resolve: newResolver(modulesProvider).makeResolveGetClass(class.Class), - } - - field.Args["bm25"] = bm25Argument(class.Class) - field.Args["hybrid"] = hybridArgument(classObject, class, modulesProvider, fusionEnum) - - if modulesProvider != nil { - for name, argument := range modulesProvider.GetArguments(class) { - field.Args[name] = argument - } - } - - if replicationEnabled(class) { - field.Args["consistencyLevel"] = consistencyLevelArgument(class) - } - - if schema.MultiTenancyEnabled(class) { - field.Args["tenant"] = tenantArgument() - } - - return field -} - -func resolveGeoCoordinates(p graphql.ResolveParams) (interface{}, error) { - field := p.Source.(map[string]interface{})[p.Info.FieldName] - if field == nil { - return nil, nil - } - - geo, ok := field.(*models.GeoCoordinates) - if !ok { - return nil, fmt.Errorf("expected a *models.GeoCoordinates, but got: %T", field) - } - - return map[string]interface{}{ - "latitude": geo.Latitude, - "longitude": geo.Longitude, - }, nil -} - -func resolvePhoneNumber(p graphql.ResolveParams) (interface{}, error) { - field := p.Source.(map[string]interface{})[p.Info.FieldName] - if field == nil { - return nil, nil - } - - phone, ok := field.(*models.PhoneNumber) - if !ok { - return nil, fmt.Errorf("expected a *models.PhoneNumber, but got: %T", field) - } - - return map[string]interface{}{ - "input": phone.Input, - "internationalFormatted": phone.InternationalFormatted, - "nationalFormatted": phone.NationalFormatted, - "national": phone.National, - "valid": phone.Valid, - "countryCode": phone.CountryCode, - "defaultCountry": phone.DefaultCountry, - }, nil -} - -func whereArgument(className string) *graphql.ArgumentConfig { - return &graphql.ArgumentConfig{ - Description: descriptions.GetWhere, - Type: graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("GetObjects%sWhereInpObj", className), - Fields: common_filters.BuildNew(fmt.Sprintf("GetObjects%s", className)), - Description: descriptions.GetWhereInpObj, - }, - ), - } -} - -type resolver struct { - modulesProvider ModulesProvider -} - -func newResolver(modulesProvider ModulesProvider) *resolver { - return &resolver{modulesProvider} -} - -func (r *resolver) makeResolveGetClass(className string) graphql.FieldResolveFn { - return func(p graphql.ResolveParams) (interface{}, error) { - result, err := r.resolveGet(p, className) - if err != nil { - return result, enterrors.NewErrGraphQLUser(err, "Get", className) - } - return result, nil - } -} - -func (r *resolver) resolveGet(p graphql.ResolveParams, className string) (interface{}, error) { - source, ok := p.Source.(map[string]interface{}) - if !ok { - return nil, fmt.Errorf("expected graphql root to be a map, but was %T", p.Source) - } - - resolver, ok := source["Resolver"].(Resolver) - if !ok { - return nil, fmt.Errorf("expected source map to have a usable Resolver, but got %#v", source["Resolver"]) - } - - pagination, err := filters.ExtractPaginationFromArgs(p.Args) - if err != nil { - return nil, err - } - - cursor, err := filters.ExtractCursorFromArgs(p.Args) - if err != nil { - return nil, err - } - - // There can only be exactly one ast.Field; it is the class name. - if len(p.Info.FieldASTs) != 1 { - panic("Only one Field expected here") - } - - selectionsOfClass := p.Info.FieldASTs[0].SelectionSet - - properties, addlProps, err := extractProperties( - className, selectionsOfClass, p.Info.Fragments, r.modulesProvider) - if err != nil { - return nil, err - } - - var sort []filters.Sort - if sortArg, ok := p.Args["sort"]; ok { - sort = filters.ExtractSortFromArgs(sortArg.([]interface{})) - } - - filters, err := common_filters.ExtractFilters(p.Args, p.Info.FieldName) - if err != nil { - return nil, fmt.Errorf("could not extract filters: %s", err) - } - - var nearVectorParams *searchparams.NearVector - if nearVector, ok := p.Args["nearVector"]; ok { - p, err := common_filters.ExtractNearVector(nearVector.(map[string]interface{})) - if err != nil { - return nil, fmt.Errorf("failed to extract nearVector params: %s", err) - } - nearVectorParams = &p - } - - var nearObjectParams *searchparams.NearObject - if nearObject, ok := p.Args["nearObject"]; ok { - p, err := common_filters.ExtractNearObject(nearObject.(map[string]interface{})) - if err != nil { - return nil, fmt.Errorf("failed to extract nearObject params: %s", err) - } - nearObjectParams = &p - } - - var moduleParams map[string]interface{} - if r.modulesProvider != nil { - extractedParams := r.modulesProvider.ExtractSearchParams(p.Args, className) - if len(extractedParams) > 0 { - moduleParams = extractedParams - } - } - - // extracts bm25 (sparseSearch) from the query - var keywordRankingParams *searchparams.KeywordRanking - if bm25, ok := p.Args["bm25"]; ok { - if len(sort) > 0 { - return nil, fmt.Errorf("bm25 search is not compatible with sort") - } - p := common_filters.ExtractBM25(bm25.(map[string]interface{}), addlProps.ExplainScore) - keywordRankingParams = &p - } - - // Extract hybrid search params from the processed query - // Everything hybrid can go in another namespace AFTER modulesprovider is - // refactored - var hybridParams *searchparams.HybridSearch - if hybrid, ok := p.Args["hybrid"]; ok { - if len(sort) > 0 { - return nil, fmt.Errorf("hybrid search is not compatible with sort") - } - p, err := common_filters.ExtractHybridSearch(hybrid.(map[string]interface{}), addlProps.ExplainScore) - if err != nil { - return nil, fmt.Errorf("failed to extract hybrid params: %w", err) - } - hybridParams = p - } - - var replProps *additional.ReplicationProperties - if cl, ok := p.Args["consistencyLevel"]; ok { - replProps = &additional.ReplicationProperties{ - ConsistencyLevel: cl.(string), - } - } - - group := extractGroup(p.Args) - - var groupByParams *searchparams.GroupBy - if groupBy, ok := p.Args["groupBy"]; ok { - p := common_filters.ExtractGroupBy(groupBy.(map[string]interface{})) - groupByParams = &p - } - - var tenant string - if tk, ok := p.Args["tenant"]; ok { - tenant = tk.(string) - } - - params := dto.GetParams{ - Filters: filters, - ClassName: className, - Pagination: pagination, - Cursor: cursor, - Properties: properties, - Sort: sort, - NearVector: nearVectorParams, - NearObject: nearObjectParams, - Group: group, - ModuleParams: moduleParams, - AdditionalProperties: addlProps, - KeywordRanking: keywordRankingParams, - HybridSearch: hybridParams, - ReplicationProperties: replProps, - GroupBy: groupByParams, - Tenant: tenant, - } - - // need to perform vector search by distance - // under certain conditions - setLimitBasedOnVectorSearchParams(¶ms) - - return func() (interface{}, error) { - result, err := resolver.GetClass(p.Context, principalFromContext(p.Context), params) - if err != nil { - return result, enterrors.NewErrGraphQLUser(err, "Get", params.ClassName) - } - return result, nil - }, nil -} - -// the limit needs to be set according to the vector search parameters. -// for example, if a certainty is provided by any of the near* options, -// and no limit was provided, weaviate will want to execute a vector -// search by distance. it knows to do this by watching for a limit -// flag, specifically filters.LimitFlagSearchByDistance -func setLimitBasedOnVectorSearchParams(params *dto.GetParams) { - setLimit := func(params *dto.GetParams) { - if params.Pagination == nil { - // limit was omitted entirely, implicitly - // indicating to do unlimited search - params.Pagination = &filters.Pagination{ - Limit: filters.LimitFlagSearchByDist, - } - } else if params.Pagination.Limit < 0 { - // a negative limit was set, explicitly - // indicating to do unlimited search - params.Pagination.Limit = filters.LimitFlagSearchByDist - } - } - - if params.NearVector != nil && - (params.NearVector.Certainty != 0 || params.NearVector.WithDistance) { - setLimit(params) - return - } - - if params.NearObject != nil && - (params.NearObject.Certainty != 0 || params.NearObject.WithDistance) { - setLimit(params) - return - } - - for _, param := range params.ModuleParams { - nearParam, ok := param.(modulecapabilities.NearParam) - if ok && nearParam.SimilarityMetricProvided() { - setLimit(params) - return - } - } -} - -func extractGroup(args map[string]interface{}) *dto.GroupParams { - group, ok := args["group"] - if !ok { - return nil - } - - asMap := group.(map[string]interface{}) // guaranteed by graphql - strategy := asMap["type"].(string) - force := asMap["force"].(float64) - return &dto.GroupParams{ - Strategy: strategy, - Force: float32(force), - } -} - -func principalFromContext(ctx context.Context) *models.Principal { - principal := ctx.Value("principal") - if principal == nil { - return nil - } - - return principal.(*models.Principal) -} - -func isPrimitive(selectionSet *ast.SelectionSet) bool { - if selectionSet == nil { - return true - } - - // if there is a selection set it could either be a cross-ref or a map-type - // field like GeoCoordinates or PhoneNumber - for _, subSelection := range selectionSet.Selections { - if subsectionField, ok := subSelection.(*ast.Field); ok { - if fieldNameIsOfObjectButNonReferenceType(subsectionField.Name.Value) { - return true - } - } - } - - // must be a ref field - return false -} - -type additionalCheck struct { - modulesProvider ModulesProvider -} - -func (ac *additionalCheck) isAdditional(parentName, name string) bool { - if parentName == "_additional" { - if name == "classification" || name == "certainty" || - name == "distance" || name == "id" || name == "vector" || - name == "creationTimeUnix" || name == "lastUpdateTimeUnix" || - name == "score" || name == "explainScore" || name == "isConsistent" || - name == "group" { - return true - } - if ac.isModuleAdditional(name) { - return true - } - } - return false -} - -func (ac *additionalCheck) isModuleAdditional(name string) bool { - if ac.modulesProvider != nil { - if len(ac.modulesProvider.GraphQLAdditionalFieldNames()) > 0 { - for _, moduleAdditionalProperty := range ac.modulesProvider.GraphQLAdditionalFieldNames() { - if name == moduleAdditionalProperty { - return true - } - } - } - } - return false -} - -func fieldNameIsOfObjectButNonReferenceType(field string) bool { - switch field { - case "latitude", "longitude": - // must be a geo prop - return true - case "input", "internationalFormatted", "nationalFormatted", "national", - "valid", "countryCode", "defaultCountry": - // must be a phone number - return true - default: - return false - } -} - -func extractProperties(className string, selections *ast.SelectionSet, - fragments map[string]ast.Definition, - modulesProvider ModulesProvider, -) ([]search.SelectProperty, additional.Properties, error) { - var properties []search.SelectProperty - var additionalProps additional.Properties - additionalCheck := &additionalCheck{modulesProvider} - - for _, selection := range selections.Selections { - field := selection.(*ast.Field) - name := field.Name.Value - property := search.SelectProperty{Name: name} - - property.IsPrimitive = isPrimitive(field.SelectionSet) - if !property.IsPrimitive { - // We can interpret this property in different ways - for _, subSelection := range field.SelectionSet.Selections { - switch s := subSelection.(type) { - case *ast.Field: - // Is it a field with the name __typename? - if s.Name.Value == "__typename" { - property.IncludeTypeName = true - continue - } else if additionalCheck.isAdditional(name, s.Name.Value) { - additionalProperty := s.Name.Value - if additionalProperty == "classification" { - additionalProps.Classification = true - continue - } - if additionalProperty == "certainty" { - additionalProps.Certainty = true - continue - } - if additionalProperty == "distance" { - additionalProps.Distance = true - continue - } - if additionalProperty == "id" { - additionalProps.ID = true - continue - } - if additionalProperty == "vector" { - additionalProps.Vector = true - continue - } - if additionalProperty == "creationTimeUnix" { - additionalProps.CreationTimeUnix = true - continue - } - if additionalProperty == "score" { - additionalProps.Score = true - continue - } - if additionalProperty == "explainScore" { - additionalProps.ExplainScore = true - continue - } - if additionalProperty == "lastUpdateTimeUnix" { - additionalProps.LastUpdateTimeUnix = true - continue - } - if additionalProperty == "isConsistent" { - additionalProps.IsConsistent = true - continue - } - if additionalProperty == "group" { - additionalProps.Group = true - additionalGroupHitProperties, err := extractGroupHitProperties(className, additionalProps, subSelection, fragments, modulesProvider) - if err != nil { - return nil, additionalProps, err - } - properties = append(properties, additionalGroupHitProperties...) - continue - } - if modulesProvider != nil { - if additionalCheck.isModuleAdditional(additionalProperty) { - additionalProps.ModuleParams = getModuleParams(additionalProps.ModuleParams) - additionalProps.ModuleParams[additionalProperty] = modulesProvider.ExtractAdditionalField(className, additionalProperty, s.Arguments) - continue - } - } - } else { - // It's an object / object array property - continue - } - - case *ast.FragmentSpread: - ref, err := extractFragmentSpread(className, s, fragments, modulesProvider) - if err != nil { - return nil, additionalProps, err - } - - property.Refs = append(property.Refs, ref) - - case *ast.InlineFragment: - ref, err := extractInlineFragment(className, s, fragments, modulesProvider) - if err != nil { - return nil, additionalProps, err - } - - property.Refs = append(property.Refs, ref) - - default: - return nil, additionalProps, fmt.Errorf("unrecoginzed type in subs-selection: %T", subSelection) - } - } - } - - if name == "_additional" { - continue - } - - properties = append(properties, property) - } - - return properties, additionalProps, nil -} - -func extractGroupHitProperties( - className string, - additionalProps additional.Properties, - subSelection ast.Selection, - fragments map[string]ast.Definition, - modulesProvider ModulesProvider, -) ([]search.SelectProperty, error) { - additionalGroupProperties := []search.SelectProperty{} - if subSelection != nil { - if selectionSet := subSelection.GetSelectionSet(); selectionSet != nil { - for _, groupSubSelection := range selectionSet.Selections { - if groupSubSelection != nil { - if groupSubSelectionField, ok := groupSubSelection.(*ast.Field); ok { - if groupSubSelectionField.Name.Value == "hits" && groupSubSelectionField.SelectionSet != nil { - for _, groupHitsSubSelection := range groupSubSelectionField.SelectionSet.Selections { - if hf, ok := groupHitsSubSelection.(*ast.Field); ok { - if hf.SelectionSet != nil { - for _, ss := range hf.SelectionSet.Selections { - if inlineFrag, ok := ss.(*ast.InlineFragment); ok { - ref, err := extractInlineFragment(className, inlineFrag, fragments, modulesProvider) - if err != nil { - return nil, err - } - - additionalGroupHitProp := search.SelectProperty{Name: fmt.Sprintf("_additional:group:hits:%v", hf.Name.Value)} - additionalGroupHitProp.Refs = append(additionalGroupHitProp.Refs, ref) - additionalGroupProperties = append(additionalGroupProperties, additionalGroupHitProp) - } - } - } - } - } - } - } - } - } - } - } - return additionalGroupProperties, nil -} - -func getModuleParams(moduleParams map[string]interface{}) map[string]interface{} { - if moduleParams == nil { - return map[string]interface{}{} - } - return moduleParams -} - -func extractInlineFragment(class string, fragment *ast.InlineFragment, - fragments map[string]ast.Definition, - modulesProvider ModulesProvider, -) (search.SelectClass, error) { - var className schema.ClassName - var err error - var result search.SelectClass - - if strings.Contains(fragment.TypeCondition.Name.Value, "__") { - // is a helper type for a network ref - // don't validate anything as of now - className = schema.ClassName(fragment.TypeCondition.Name.Value) - } else { - className, err = schema.ValidateClassName(fragment.TypeCondition.Name.Value) - if err != nil { - return result, fmt.Errorf("the inline fragment type name '%s' is not a valid class name", fragment.TypeCondition.Name.Value) - } - } - - if className == "Beacon" { - return result, fmt.Errorf("retrieving cross-refs by beacon is not supported yet - coming soon!") - } - - subProperties, additionalProperties, err := extractProperties(class, fragment.SelectionSet, fragments, modulesProvider) - if err != nil { - return result, err - } - - result.ClassName = string(className) - result.RefProperties = subProperties - result.AdditionalProperties = additionalProperties - return result, nil -} - -func extractFragmentSpread(class string, spread *ast.FragmentSpread, - fragments map[string]ast.Definition, - modulesProvider ModulesProvider, -) (search.SelectClass, error) { - var result search.SelectClass - name := spread.Name.Value - - def, ok := fragments[name] - if !ok { - return result, fmt.Errorf("spread fragment '%s' refers to unknown fragment", name) - } - - className, err := hackyWorkaroundToExtractClassName(def, name) - if err != nil { - return result, err - } - - subProperties, additionalProperties, err := extractProperties(class, def.GetSelectionSet(), fragments, modulesProvider) - if err != nil { - return result, err - } - - result.ClassName = string(className) - result.RefProperties = subProperties - result.AdditionalProperties = additionalProperties - return result, nil -} - -// It seems there's no proper way to extract this info unfortunately: -// https://github.com/tailor-inc/graphql/issues/455 -func hackyWorkaroundToExtractClassName(def ast.Definition, name string) (string, error) { - loc := def.GetLoc() - raw := loc.Source.Body[loc.Start:loc.End] - r := regexp.MustCompile(fmt.Sprintf(`fragment\s*%s\s*on\s*(\w*)\s*{`, name)) - matches := r.FindSubmatch(raw) - if len(matches) < 2 { - return "", fmt.Errorf("could not extract a className from fragment") - } - - return string(matches[1]), nil -} diff --git a/adapters/handlers/graphql/local/get/class_builder_nested.go b/adapters/handlers/graphql/local/get/class_builder_nested.go deleted file mode 100644 index 5392898922fb2c3b3fb4ed518dc9f6d0fcaf9b09..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/get/class_builder_nested.go +++ /dev/null @@ -1,87 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package get - -import ( - "fmt" - - "github.com/tailor-inc/graphql" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" -) - -func (b *classBuilder) nestedField(propertyType schema.PropertyDataType, - property *models.Property, className string, -) *graphql.Field { - return b.parseNestedProperties(property.NestedProperties, className, property.Name, property.DataType) -} - -func (b *classBuilder) parseNestedProperties(nestedProps []*models.NestedProperty, - className, prefix string, propDataType []string, -) *graphql.Field { - fields := graphql.Fields{} - for _, prop := range nestedProps { - if prop.NestedProperties != nil { - fields[prop.Name] = b.parseNestedProperties(prop.NestedProperties, - className, fmt.Sprintf("%s_%s", prefix, prop.Name), prop.DataType) - } else { - fields[prop.Name] = &graphql.Field{ - Name: fmt.Sprintf("%s_%s_%s_field", className, prefix, prop.Name), - Type: b.determinNestedPropertyType(prop.DataType, prop.Name), - } - } - } - - fieldType := graphql.NewObject(graphql.ObjectConfig{ - Name: fmt.Sprintf("%s_%s_object", className, prefix), - Fields: fields, - }) - - if len(propDataType) == 1 && propDataType[0] == schema.DataTypeObjectArray.String() { - return &graphql.Field{Type: graphql.NewList(fieldType)} - } - return &graphql.Field{Type: fieldType} -} - -func (b *classBuilder) determinNestedPropertyType(dataType []string, propName string) graphql.Output { - switch schema.DataType(dataType[0]) { - case schema.DataTypeText, schema.DataTypeString: - return graphql.String - case schema.DataTypeInt: - return graphql.Int - case schema.DataTypeNumber: - return graphql.Float - case schema.DataTypeBoolean: - return graphql.Boolean - case schema.DataTypeDate: - return graphql.String - case schema.DataTypeBlob: - return graphql.String - case schema.DataTypeUUID: - return graphql.String - case schema.DataTypeTextArray, schema.DataTypeStringArray: - return graphql.NewList(graphql.String) - case schema.DataTypeIntArray: - return graphql.NewList(graphql.Int) - case schema.DataTypeNumberArray: - return graphql.NewList(graphql.Float) - case schema.DataTypeBooleanArray: - return graphql.NewList(graphql.Boolean) - case schema.DataTypeDateArray: - return graphql.NewList(graphql.String) - case schema.DataTypeUUIDArray: - return graphql.NewList(graphql.String) - default: - panic(fmt.Sprintf("determinNestedPropertyType: unknown primitive type for property %s: %s", - propName, dataType[0])) - } -} diff --git a/adapters/handlers/graphql/local/get/class_builder_references.go b/adapters/handlers/graphql/local/get/class_builder_references.go deleted file mode 100644 index 1c6afcb6e5b8b47bc9b612d7b18e068dabe42a5b..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/get/class_builder_references.go +++ /dev/null @@ -1,118 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package get - -import ( - "fmt" - - "github.com/tailor-inc/graphql" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/search" - "golang.org/x/text/cases" - "golang.org/x/text/language" -) - -func (b *classBuilder) referenceField(propertyType schema.PropertyDataType, - property *models.Property, className string, -) *graphql.Field { - refClasses := propertyType.Classes() - propertyName := cases.Title(language.Und, cases.NoLower).String(property.Name) - dataTypeClasses := []*graphql.Object{} - - for _, refClassName := range refClasses { - // is a local ref - refClass, ok := b.knownClasses[string(refClassName)] - if !ok { - panic(fmt.Sprintf("buildGetClass: unknown referenced class type for %s.%s; %s", - className, property.Name, refClassName)) - } - - dataTypeClasses = append(dataTypeClasses, refClass) - } - - if (len(dataTypeClasses)) == 0 { - // this could be the case when we only have network-refs, but all network - // refs were invalid (e.g. because the peers are gone). In this case we - // must return (nil) early, otherwise graphql will error because it has a - // union field with an empty list of unions. - return nil - } - - dataTypeClasses = append(dataTypeClasses, b.beaconClass) - - classUnion := graphql.NewUnion(graphql.UnionConfig{ - Name: fmt.Sprintf("%s%s%s", className, propertyName, "Obj"), - Types: dataTypeClasses, - ResolveType: makeResolveClassUnionType(&b.knownClasses), - Description: property.Description, - }) - - return &graphql.Field{ - Type: graphql.NewList(classUnion), - Description: property.Description, - Resolve: makeResolveRefField(), - } -} - -func makeResolveClassUnionType(knownClasses *map[string]*graphql.Object) graphql.ResolveTypeFn { - return func(p graphql.ResolveTypeParams) *graphql.Object { - valueMap := p.Value.(map[string]interface{}) - refType := valueMap["__refClassType"].(string) - switch refType { - case "local": - className := valueMap["__refClassName"].(string) - classObj, ok := (*knownClasses)[className] - if !ok { - panic(fmt.Errorf( - "local ref refers to class '%s', but no such kind exists in the peer network", className)) - } - return classObj - default: - panic(fmt.Sprintf("unknown ref type %#v", refType)) - } - } -} - -func makeResolveRefField() graphql.FieldResolveFn { - return func(p graphql.ResolveParams) (interface{}, error) { - if p.Source.(map[string]interface{})[p.Info.FieldName] == nil { - return nil, nil - } - - items, ok := p.Source.(map[string]interface{})[p.Info.FieldName].([]interface{}) - if !ok { - // could be a models.MultipleRef which would indicate that we found only - // unresolved references, this is the case when accepts refs to types - // ClassA and ClassB and the object only contains refs to one type (e.g. - // ClassA). Now if the user only asks for resolving all of the other type - // (i.e. ClassB), then all results would be returned unresolved (as - // models.MultipleRef). - - return nil, nil - } - results := make([]interface{}, len(items)) - for i, item := range items { - switch v := item.(type) { - case search.LocalRef: - // inject some meta data so the ResolveType can determine the type - localRef := v.Fields - localRef["__refClassType"] = "local" - localRef["__refClassName"] = v.Class - results[i] = localRef - default: - return nil, fmt.Errorf("unsupported type, expected search.LocalRef or NetworkRef, got %T", v) - } - } - return results, nil - } -} diff --git a/adapters/handlers/graphql/local/get/class_builder_references_test.go b/adapters/handlers/graphql/local/get/class_builder_references_test.go deleted file mode 100644 index 5c8427d1388ba74d4e2b7d1a8bd277676026faab..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/get/class_builder_references_test.go +++ /dev/null @@ -1,61 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package get - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/dto" -) - -func TestGetNoNetworkRequestIsMadeWhenUserDoesntWantNetworkRef(t *testing.T) { - t.Parallel() - - resolver := newMockResolver() - - expectedParams := dto.GetParams{ - ClassName: "SomeThing", - AdditionalProperties: additional.Properties{ - ID: true, - }, - } - - resolverResponse := []interface{}{ - map[string]interface{}{ - "_additional": map[string]interface{}{ - "id": "some-uuid-for-the-local-class", - }, - }, - } - - resolver.On("GetClass", expectedParams). - Return(resolverResponse, nil).Once() - - query := "{ Get { SomeThing { _additional { id } } } }" - result := resolver.AssertResolve(t, query).Result - - expectedResult := map[string]interface{}{ - "Get": map[string]interface{}{ - "SomeThing": []interface{}{ - map[string]interface{}{ - "_additional": map[string]interface{}{ - "id": "some-uuid-for-the-local-class", - }, - }, - }, - }, - } - - assert.Equal(t, expectedResult, result, "should resolve the network cross-ref correctly") -} diff --git a/adapters/handlers/graphql/local/get/explore_argument.go b/adapters/handlers/graphql/local/get/explore_argument.go deleted file mode 100644 index 2f72564b4ab9047bf76512c2ac9f9bcd7f722c0e..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/get/explore_argument.go +++ /dev/null @@ -1,99 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package get - -import ( - "fmt" - - "github.com/tailor-inc/graphql" - "github.com/weaviate/weaviate/adapters/handlers/graphql/local/common_filters" - - "github.com/weaviate/weaviate/adapters/handlers/graphql/descriptions" -) - -func nearVectorArgument(className string) *graphql.ArgumentConfig { - return common_filters.NearVectorArgument("GetObjects", className) -} - -func nearObjectArgument(className string) *graphql.ArgumentConfig { - return common_filters.NearObjectArgument("GetObjects", className) -} - -func nearTextFields(prefix string) graphql.InputObjectConfigFieldMap { - nearTextFields := graphql.InputObjectConfigFieldMap{ - "concepts": &graphql.InputObjectFieldConfig{ - // Description: descriptions.Concepts, - Type: graphql.NewNonNull(graphql.NewList(graphql.String)), - }, - "moveTo": &graphql.InputObjectFieldConfig{ - Description: descriptions.VectorMovement, - Type: graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sMoveTo", prefix), - Fields: movementInp(fmt.Sprintf("%sMoveTo", prefix)), - }), - }, - "certainty": &graphql.InputObjectFieldConfig{ - Description: descriptions.Certainty, - Type: graphql.Float, - }, - "distance": &graphql.InputObjectFieldConfig{ - Description: descriptions.Distance, - Type: graphql.Float, - }, - "moveAwayFrom": &graphql.InputObjectFieldConfig{ - Description: descriptions.VectorMovement, - Type: graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sMoveAwayFrom", prefix), - Fields: movementInp(fmt.Sprintf("%sMoveAwayFrom", prefix)), - }), - }, - } - return nearTextFields -} - -func movementInp(prefix string) graphql.InputObjectConfigFieldMap { - return graphql.InputObjectConfigFieldMap{ - "concepts": &graphql.InputObjectFieldConfig{ - Description: descriptions.Keywords, - Type: graphql.NewList(graphql.String), - }, - "objects": &graphql.InputObjectFieldConfig{ - Description: "objects", - Type: graphql.NewList(objectsInpObj(prefix)), - }, - "force": &graphql.InputObjectFieldConfig{ - Description: descriptions.Force, - Type: graphql.NewNonNull(graphql.Float), - }, - } -} - -func objectsInpObj(prefix string) *graphql.InputObject { - return graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sMovementObjectsInpObj", prefix), - Fields: graphql.InputObjectConfigFieldMap{ - "id": &graphql.InputObjectFieldConfig{ - Type: graphql.String, - Description: "id of an object", - }, - "beacon": &graphql.InputObjectFieldConfig{ - Type: graphql.String, - Description: descriptions.Beacon, - }, - }, - Description: "Movement Object", - }, - ) -} diff --git a/adapters/handlers/graphql/local/get/get.go b/adapters/handlers/graphql/local/get/get.go deleted file mode 100644 index ce5b1fb5de812a7f8f6caca004d2b81b2ea4c974..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/get/get.go +++ /dev/null @@ -1,62 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package get - -import ( - "github.com/sirupsen/logrus" - "github.com/tailor-inc/graphql" - "github.com/tailor-inc/graphql/language/ast" - "github.com/weaviate/weaviate/adapters/handlers/graphql/descriptions" - "github.com/weaviate/weaviate/adapters/handlers/graphql/utils" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/schema" -) - -type ModulesProvider interface { - GetArguments(class *models.Class) map[string]*graphql.ArgumentConfig - ExtractSearchParams(arguments map[string]interface{}, className string) map[string]interface{} - GetAdditionalFields(class *models.Class) map[string]*graphql.Field - ExtractAdditionalField(className, name string, params []*ast.Argument) interface{} - GraphQLAdditionalFieldNames() []string - GetAll() []modulecapabilities.Module -} - -// Build the Local.Get part of the graphql tree -func Build(schema *schema.Schema, logger logrus.FieldLogger, - modulesProvider ModulesProvider, -) (*graphql.Field, error) { - if len(schema.Objects.Classes) == 0 { - return nil, utils.ErrEmptySchema - } - - cb := newClassBuilder(schema, logger, modulesProvider) - - var err error - var objects *graphql.Object - if len(schema.Objects.Classes) > 0 { - objects, err = cb.objects() - if err != nil { - return nil, err - } - } - - return &graphql.Field{ - Name: "Get", - Description: descriptions.GetObjects, - Type: objects, - Resolve: func(p graphql.ResolveParams) (interface{}, error) { - // Does nothing; pass through the filters - return p.Source, nil - }, - }, nil -} diff --git a/adapters/handlers/graphql/local/get/get_test.go b/adapters/handlers/graphql/local/get/get_test.go deleted file mode 100644 index 39ab73dab71e33b2cd098406c2622aa39e4bbdf3..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/get/get_test.go +++ /dev/null @@ -1,2300 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// These tests verify that the parameters to the resolver are properly extracted from a GraphQL query. - -package get - -import ( - "fmt" - "testing" - "time" - - "github.com/go-openapi/strfmt" - "github.com/google/uuid" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/tailor-inc/graphql/language/ast" - test_helper "github.com/weaviate/weaviate/adapters/handlers/graphql/test/helper" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/dto" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/search" - "github.com/weaviate/weaviate/entities/searchparams" - helper "github.com/weaviate/weaviate/test/helper" -) - -func TestSimpleFieldParamsOK(t *testing.T) { - t.Parallel() - resolver := newMockResolver() - expectedParams := dto.GetParams{ - ClassName: "SomeAction", - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - } - - resolver.On("GetClass", expectedParams). - Return(test_helper.EmptyList(), nil).Once() - - resolver.AssertResolve(t, "{ Get { SomeAction { intField } } }") -} - -func TestExtractIntField(t *testing.T) { - t.Parallel() - - resolver := newMockResolver() - - expectedParams := dto.GetParams{ - ClassName: "SomeAction", - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - } - - resolver.On("GetClass", expectedParams). - Return(test_helper.EmptyList(), nil).Once() - - query := "{ Get { SomeAction { intField } } }" - resolver.AssertResolve(t, query) -} - -func TestExtractGeoCoordinatesField(t *testing.T) { - t.Parallel() - - resolver := newMockResolver() - - expectedParams := dto.GetParams{ - ClassName: "SomeAction", - Properties: []search.SelectProperty{{Name: "location", IsPrimitive: true}}, - } - - resolverReturn := []interface{}{ - map[string]interface{}{ - "location": &models.GeoCoordinates{Latitude: ptFloat32(0.5), Longitude: ptFloat32(0.6)}, - }, - } - - resolver.On("GetClass", expectedParams). - Return(resolverReturn, nil).Once() - - query := "{ Get { SomeAction { location { latitude longitude } } } }" - result := resolver.AssertResolve(t, query) - - expectedLocation := map[string]interface{}{ - "location": map[string]interface{}{ - "latitude": float32(0.5), - "longitude": float32(0.6), - }, - } - - assert.Equal(t, expectedLocation, result.Get("Get", "SomeAction").Result.([]interface{})[0]) -} - -func TestExtractUUIDField(t *testing.T) { - t.Parallel() - - resolver := newMockResolver() - - expectedParams := dto.GetParams{ - ClassName: "SomeAction", - Properties: []search.SelectProperty{{Name: "uuidField", IsPrimitive: true}}, - } - - id := uuid.New() - - resolverReturn := []interface{}{ - map[string]interface{}{ - "uuidField": id, - }, - } - - resolver.On("GetClass", expectedParams). - Return(resolverReturn, nil).Once() - - query := "{ Get { SomeAction { uuidField } } }" - result := resolver.AssertResolve(t, query) - - expectedProps := map[string]interface{}{ - "uuidField": id.String(), - } - - assert.Equal(t, expectedProps, result.Get("Get", "SomeAction").Result.([]interface{})[0]) -} - -func TestExtractUUIDArrayField(t *testing.T) { - t.Parallel() - - resolver := newMockResolver() - - expectedParams := dto.GetParams{ - ClassName: "SomeAction", - Properties: []search.SelectProperty{{Name: "uuidArrayField", IsPrimitive: true}}, - } - - id1 := uuid.New() - id2 := uuid.New() - - resolverReturn := []interface{}{ - map[string]interface{}{ - "uuidArrayField": []uuid.UUID{id1, id2}, - }, - } - - resolver.On("GetClass", expectedParams). - Return(resolverReturn, nil).Once() - - query := "{ Get { SomeAction { uuidArrayField } } }" - result := resolver.AssertResolve(t, query) - - expectedProps := map[string]interface{}{ - "uuidArrayField": []any{id1.String(), id2.String()}, - } - - assert.Equal(t, expectedProps, result.Get("Get", "SomeAction").Result.([]interface{})[0]) -} - -func TestExtractPhoneNumberField(t *testing.T) { - // We need to explicitly test all cases of asking for just one sub-property - // at a time, because the AST-parsing uses known fields of known props to - // distinguish a complex primitive prop from a reference prop - // - // See "isPrimitive()" and "fieldNameIsOfObjectButNonReferenceType" in - // class_builder_fields.go for more details - - type test struct { - name string - query string - expectedParams dto.GetParams - resolverReturn interface{} - expectedResult interface{} - } - - tests := []test{ - { - name: "with only input requested", - query: "{ Get { SomeAction { phone { input } } } }", - expectedParams: dto.GetParams{ - ClassName: "SomeAction", - Properties: []search.SelectProperty{{Name: "phone", IsPrimitive: true}}, - }, - resolverReturn: []interface{}{ - map[string]interface{}{ - "phone": &models.PhoneNumber{Input: "+49 171 1234567"}, - }, - }, - expectedResult: map[string]interface{}{ - "phone": map[string]interface{}{ - "input": "+49 171 1234567", - }, - }, - }, - { - name: "with only internationalFormatted requested", - query: "{ Get { SomeAction { phone { internationalFormatted } } } }", - expectedParams: dto.GetParams{ - ClassName: "SomeAction", - Properties: []search.SelectProperty{{Name: "phone", IsPrimitive: true}}, - }, - resolverReturn: []interface{}{ - map[string]interface{}{ - "phone": &models.PhoneNumber{InternationalFormatted: "+49 171 1234567"}, - }, - }, - expectedResult: map[string]interface{}{ - "phone": map[string]interface{}{ - "internationalFormatted": "+49 171 1234567", - }, - }, - }, - { - name: "with only nationalFormatted requested", - query: "{ Get { SomeAction { phone { nationalFormatted } } } }", - expectedParams: dto.GetParams{ - ClassName: "SomeAction", - Properties: []search.SelectProperty{{Name: "phone", IsPrimitive: true}}, - }, - resolverReturn: []interface{}{ - map[string]interface{}{ - "phone": &models.PhoneNumber{NationalFormatted: "0171 1234567"}, - }, - }, - expectedResult: map[string]interface{}{ - "phone": map[string]interface{}{ - "nationalFormatted": "0171 1234567", - }, - }, - }, - { - name: "with only national requested", - query: "{ Get { SomeAction { phone { national } } } }", - expectedParams: dto.GetParams{ - ClassName: "SomeAction", - Properties: []search.SelectProperty{{Name: "phone", IsPrimitive: true}}, - }, - resolverReturn: []interface{}{ - map[string]interface{}{ - "phone": &models.PhoneNumber{National: 0o1711234567}, - }, - }, - expectedResult: map[string]interface{}{ - "phone": map[string]interface{}{ - "national": 0o1711234567, - }, - }, - }, - { - name: "with only valid requested", - query: "{ Get { SomeAction { phone { valid } } } }", - expectedParams: dto.GetParams{ - ClassName: "SomeAction", - Properties: []search.SelectProperty{{Name: "phone", IsPrimitive: true}}, - }, - resolverReturn: []interface{}{ - map[string]interface{}{ - "phone": &models.PhoneNumber{Valid: true}, - }, - }, - expectedResult: map[string]interface{}{ - "phone": map[string]interface{}{ - "valid": true, - }, - }, - }, - { - name: "with only countryCode requested", - query: "{ Get { SomeAction { phone { countryCode } } } }", - expectedParams: dto.GetParams{ - ClassName: "SomeAction", - Properties: []search.SelectProperty{{Name: "phone", IsPrimitive: true}}, - }, - resolverReturn: []interface{}{ - map[string]interface{}{ - "phone": &models.PhoneNumber{CountryCode: 49}, - }, - }, - expectedResult: map[string]interface{}{ - "phone": map[string]interface{}{ - "countryCode": 49, - }, - }, - }, - { - name: "with only defaultCountry requested", - query: "{ Get { SomeAction { phone { defaultCountry } } } }", - expectedParams: dto.GetParams{ - ClassName: "SomeAction", - Properties: []search.SelectProperty{{Name: "phone", IsPrimitive: true}}, - }, - resolverReturn: []interface{}{ - map[string]interface{}{ - "phone": &models.PhoneNumber{DefaultCountry: "DE"}, - }, - }, - expectedResult: map[string]interface{}{ - "phone": map[string]interface{}{ - "defaultCountry": "DE", - }, - }, - }, - { - name: "with multiple fields set", - query: "{ Get { SomeAction { phone { input internationalFormatted " + - "nationalFormatted defaultCountry national countryCode valid } } } }", - expectedParams: dto.GetParams{ - ClassName: "SomeAction", - Properties: []search.SelectProperty{{Name: "phone", IsPrimitive: true}}, - }, - resolverReturn: []interface{}{ - map[string]interface{}{ - "phone": &models.PhoneNumber{ - DefaultCountry: "DE", - CountryCode: 49, - NationalFormatted: "0171 123456", - InternationalFormatted: "+49 171 123456", - National: 171123456, - Input: "0171123456", - Valid: true, - }, - }, - }, - expectedResult: map[string]interface{}{ - "phone": map[string]interface{}{ - "defaultCountry": "DE", - "countryCode": 49, - "nationalFormatted": "0171 123456", - "internationalFormatted": "+49 171 123456", - "national": 171123456, - "input": "0171123456", - "valid": true, - }, - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - resolver := newMockResolver() - - resolver.On("GetClass", test.expectedParams). - Return(test.resolverReturn, nil).Once() - result := resolver.AssertResolve(t, test.query) - assert.Equal(t, test.expectedResult, result.Get("Get", "SomeAction").Result.([]interface{})[0]) - }) - } -} - -func TestExtractAdditionalFields(t *testing.T) { - // We don't need to explicitly test every subselection as we did on - // phoneNumber as these fields have fixed keys. So we can simply check for - // the prop - - type test struct { - name string - query string - expectedParams dto.GetParams - resolverReturn interface{} - expectedResult interface{} - } - - // To facilitate testing timestamps - nowString := fmt.Sprint(time.Now().UnixNano() / int64(time.Millisecond)) - - tests := []test{ - { - name: "with _additional distance", - query: "{ Get { SomeAction { _additional { distance } } } }", - expectedParams: dto.GetParams{ - ClassName: "SomeAction", - AdditionalProperties: additional.Properties{ - Distance: true, - }, - }, - resolverReturn: []interface{}{ - map[string]interface{}{ - "_additional": map[string]interface{}{ - "distance": helper.CertaintyToDist(t, 0.69), - }, - }, - }, - expectedResult: map[string]interface{}{ - "_additional": map[string]interface{}{ - "distance": helper.CertaintyToDist(t, 0.69), - }, - }, - }, - { - name: "with _additional certainty", - query: "{ Get { SomeAction { _additional { certainty } } } }", - expectedParams: dto.GetParams{ - ClassName: "SomeAction", - AdditionalProperties: additional.Properties{ - Certainty: true, - }, - }, - resolverReturn: []interface{}{ - map[string]interface{}{ - "_additional": map[string]interface{}{ - "certainty": 0.69, - "distance": helper.CertaintyToDist(t, 0.69), - }, - }, - }, - expectedResult: map[string]interface{}{ - "_additional": map[string]interface{}{ - "certainty": 0.69, - }, - }, - }, - { - name: "with _additional vector", - query: "{ Get { SomeAction { _additional { vector } } } }", - expectedParams: dto.GetParams{ - ClassName: "SomeAction", - AdditionalProperties: additional.Properties{ - Vector: true, - }, - }, - resolverReturn: []interface{}{ - map[string]interface{}{ - "_additional": map[string]interface{}{ - "vector": []float32{0.1, -0.3}, - }, - }, - }, - expectedResult: map[string]interface{}{ - "_additional": map[string]interface{}{ - "vector": []interface{}{float32(0.1), float32(-0.3)}, - }, - }, - }, - { - name: "with _additional creationTimeUnix", - query: "{ Get { SomeAction { _additional { creationTimeUnix } } } }", - expectedParams: dto.GetParams{ - ClassName: "SomeAction", - AdditionalProperties: additional.Properties{ - CreationTimeUnix: true, - }, - }, - resolverReturn: []interface{}{ - map[string]interface{}{ - "_additional": map[string]interface{}{ - "creationTimeUnix": nowString, - }, - }, - }, - expectedResult: map[string]interface{}{ - "_additional": map[string]interface{}{ - "creationTimeUnix": nowString, - }, - }, - }, - { - name: "with _additional lastUpdateTimeUnix", - query: "{ Get { SomeAction { _additional { lastUpdateTimeUnix } } } }", - expectedParams: dto.GetParams{ - ClassName: "SomeAction", - AdditionalProperties: additional.Properties{ - LastUpdateTimeUnix: true, - }, - }, - resolverReturn: []interface{}{ - map[string]interface{}{ - "_additional": map[string]interface{}{ - "lastUpdateTimeUnix": nowString, - }, - }, - }, - expectedResult: map[string]interface{}{ - "_additional": map[string]interface{}{ - "lastUpdateTimeUnix": nowString, - }, - }, - }, - { - name: "with _additional classification", - query: "{ Get { SomeAction { _additional { classification { id completed classifiedFields scope basedOn } } } } }", - expectedParams: dto.GetParams{ - ClassName: "SomeAction", - AdditionalProperties: additional.Properties{ - Classification: true, - }, - }, - resolverReturn: []interface{}{ - map[string]interface{}{ - "_additional": models.AdditionalProperties{ - "classification": &additional.Classification{ - ID: "12345", - BasedOn: []string{"primitiveProp"}, - Scope: []string{"refprop1", "refprop2", "refprop3"}, - ClassifiedFields: []string{"refprop3"}, - Completed: timeMust(strfmt.ParseDateTime("2006-01-02T15:04:05.000Z")), - }, - }, - }, - }, - expectedResult: map[string]interface{}{ - "_additional": map[string]interface{}{ - "classification": map[string]interface{}{ - "id": "12345", - "basedOn": []interface{}{"primitiveProp"}, - "scope": []interface{}{"refprop1", "refprop2", "refprop3"}, - "classifiedFields": []interface{}{"refprop3"}, - "completed": "2006-01-02T15:04:05.000Z", - }, - }, - }, - }, - { - name: "with _additional interpretation", - query: "{ Get { SomeAction { _additional { interpretation { source { concept weight occurrence } } } } } }", - expectedParams: dto.GetParams{ - ClassName: "SomeAction", - AdditionalProperties: additional.Properties{ - ModuleParams: map[string]interface{}{ - "interpretation": true, - }, - }, - }, - resolverReturn: []interface{}{ - map[string]interface{}{ - "_additional": map[string]interface{}{ - "interpretation": &Interpretation{ - Source: []*InterpretationSource{ - { - Concept: "foo", - Weight: 0.6, - Occurrence: 1200, - }, - { - Concept: "bar", - Weight: 0.9, - Occurrence: 800, - }, - }, - }, - }, - }, - }, - expectedResult: map[string]interface{}{ - "_additional": map[string]interface{}{ - "interpretation": map[string]interface{}{ - "source": []interface{}{ - map[string]interface{}{ - "concept": "foo", - "weight": 0.6, - "occurrence": 1200, - }, - map[string]interface{}{ - "concept": "bar", - "weight": 0.9, - "occurrence": 800, - }, - }, - }, - }, - }, - }, - { - name: "with _additional nearestNeighbors", - query: "{ Get { SomeAction { _additional { nearestNeighbors { neighbors { concept distance } } } } } }", - expectedParams: dto.GetParams{ - ClassName: "SomeAction", - AdditionalProperties: additional.Properties{ - ModuleParams: map[string]interface{}{ - "nearestNeighbors": true, - }, - }, - }, - resolverReturn: []interface{}{ - map[string]interface{}{ - "_additional": map[string]interface{}{ - "nearestNeighbors": &NearestNeighbors{ - Neighbors: []*NearestNeighbor{ - { - Concept: "foo", - Distance: 0.1, - }, - { - Concept: "bar", - Distance: 0.2, - }, - }, - }, - }, - }, - }, - expectedResult: map[string]interface{}{ - "_additional": map[string]interface{}{ - "nearestNeighbors": map[string]interface{}{ - "neighbors": []interface{}{ - map[string]interface{}{ - "concept": "foo", - "distance": float32(0.1), - }, - map[string]interface{}{ - "concept": "bar", - "distance": float32(0.2), - }, - }, - }, - }, - }, - }, - { - name: "with _additional featureProjection without any optional parameters", - query: "{ Get { SomeAction { _additional { featureProjection { vector } } } } }", - expectedParams: dto.GetParams{ - ClassName: "SomeAction", - AdditionalProperties: additional.Properties{ - ModuleParams: map[string]interface{}{ - "featureProjection": extractAdditionalParam("featureProjection", nil), - }, - }, - }, - resolverReturn: []interface{}{ - map[string]interface{}{ - "_additional": models.AdditionalProperties{ - "featureProjection": &FeatureProjection{ - Vector: []float32{0.0, 1.1, 2.2}, - }, - }, - }, - }, - expectedResult: map[string]interface{}{ - "_additional": map[string]interface{}{ - "featureProjection": map[string]interface{}{ - "vector": []interface{}{float32(0.0), float32(1.1), float32(2.2)}, - }, - }, - }, - }, - { - name: "with _additional featureProjection with optional parameters", - query: `{ Get { SomeAction { _additional { featureProjection(algorithm: "tsne", dimensions: 3, learningRate: 15, iterations: 100, perplexity: 10) { vector } } } } }`, - expectedParams: dto.GetParams{ - ClassName: "SomeAction", - AdditionalProperties: additional.Properties{ - ModuleParams: map[string]interface{}{ - "featureProjection": extractAdditionalParam("featureProjection", - []*ast.Argument{ - createArg("algorithm", "tsne"), - createArg("dimensions", "3"), - createArg("iterations", "100"), - createArg("learningRate", "15"), - createArg("perplexity", "10"), - }, - ), - }, - }, - }, - resolverReturn: []interface{}{ - map[string]interface{}{ - "_additional": map[string]interface{}{ - "featureProjection": &FeatureProjection{ - Vector: []float32{0.0, 1.1, 2.2}, - }, - }, - }, - }, - expectedResult: map[string]interface{}{ - "_additional": map[string]interface{}{ - "featureProjection": map[string]interface{}{ - "vector": []interface{}{float32(0.0), float32(1.1), float32(2.2)}, - }, - }, - }, - }, - { - name: "with _additional semanticPath set", - query: `{ Get { SomeAction { _additional { semanticPath { path { concept distanceToQuery distanceToResult distanceToPrevious distanceToNext } } } } } }`, - expectedParams: dto.GetParams{ - ClassName: "SomeAction", - AdditionalProperties: additional.Properties{ - ModuleParams: map[string]interface{}{ - "semanticPath": extractAdditionalParam("semanticPath", nil), - }, - }, - }, - resolverReturn: []interface{}{ - map[string]interface{}{ - "_additional": models.AdditionalProperties{ - "semanticPath": &SemanticPath{ - Path: []*SemanticPathElement{ - { - Concept: "foo", - DistanceToNext: ptFloat32(0.5), - DistanceToPrevious: nil, - DistanceToQuery: 0.1, - DistanceToResult: 0.1, - }, - { - Concept: "bar", - DistanceToPrevious: ptFloat32(0.5), - DistanceToNext: nil, - DistanceToQuery: 0.1, - DistanceToResult: 0.1, - }, - }, - }, - }, - }, - }, - expectedResult: map[string]interface{}{ - "_additional": map[string]interface{}{ - "semanticPath": map[string]interface{}{ - "path": []interface{}{ - map[string]interface{}{ - "concept": "foo", - "distanceToNext": float32(0.5), - "distanceToPrevious": nil, - "distanceToQuery": float32(0.1), - "distanceToResult": float32(0.1), - }, - map[string]interface{}{ - "concept": "bar", - "distanceToPrevious": float32(0.5), - "distanceToNext": nil, - "distanceToQuery": float32(0.1), - "distanceToResult": float32(0.1), - }, - }, - }, - }, - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - resolver := newMockResolverWithVectorizer("mock-custom-near-text-module") - - resolver.On("GetClass", test.expectedParams). - Return(test.resolverReturn, nil).Once() - result := resolver.AssertResolve(t, test.query) - assert.Equal(t, test.expectedResult, result.Get("Get", "SomeAction").Result.([]interface{})[0]) - }) - } -} - -func TestNearCustomTextRanker(t *testing.T) { - t.Parallel() - - resolver := newMockResolverWithVectorizer("mock-custom-near-text-module") - - t.Run("for actions", func(t *testing.T) { - query := `{ Get { SomeAction(nearCustomText: { - concepts: ["c1", "c2", "c3"], - moveTo: { - concepts:["positive"], - force: 0.5 - } - moveAwayFrom: { - concepts:["epic"] - force: 0.25 - } - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeAction", - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - ModuleParams: map[string]interface{}{ - "nearCustomText": extractNearTextParam(map[string]interface{}{ - "concepts": []interface{}{"c1", "c2", "c3"}, - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"positive"}, - "force": float64(0.5), - }, - "moveAwayFrom": map[string]interface{}{ - "concepts": []interface{}{"epic"}, - "force": float64(0.25), - }, - }), - }, - } - - resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - resolver.AssertResolve(t, query) - }) - - t.Run("for a class that does not have a text2vec module", func(t *testing.T) { - query := `{ Get { CustomVectorClass(nearCustomText: { - concepts: ["c1", "c2", "c3"], - moveTo: { - concepts:["positive"], - force: 0.5 - } - moveAwayFrom: { - concepts:["epic"] - force: 0.25 - } - }) { intField } } }` - - res := resolver.Resolve(query) - require.Len(t, res.Errors, 1) - assert.Contains(t, res.Errors[0].Message, "Unknown argument \"nearCustomText\" on field \"CustomVectorClass\"") - }) - - t.Run("for things with optional distance set", func(t *testing.T) { - query := `{ Get { SomeThing(nearCustomText: { - concepts: ["c1", "c2", "c3"], - distance: 0.6, - moveTo: { - concepts:["positive"], - force: 0.5 - } - moveAwayFrom: { - concepts:["epic"] - force: 0.25 - } - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeThing", - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - Pagination: &filters.Pagination{Limit: filters.LimitFlagSearchByDist}, - ModuleParams: map[string]interface{}{ - "nearCustomText": extractNearTextParam(map[string]interface{}{ - "concepts": []interface{}{"c1", "c2", "c3"}, - "distance": float64(0.6), - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"positive"}, - "force": float64(0.5), - }, - "moveAwayFrom": map[string]interface{}{ - "concepts": []interface{}{"epic"}, - "force": float64(0.25), - }, - }), - }, - } - resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - resolver.AssertResolve(t, query) - }) - - t.Run("for things with optional certainty set", func(t *testing.T) { - query := `{ Get { SomeThing(nearCustomText: { - concepts: ["c1", "c2", "c3"], - certainty: 0.4, - moveTo: { - concepts:["positive"], - force: 0.5 - } - moveAwayFrom: { - concepts:["epic"] - force: 0.25 - } - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeThing", - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - Pagination: &filters.Pagination{Limit: filters.LimitFlagSearchByDist}, - ModuleParams: map[string]interface{}{ - "nearCustomText": extractNearTextParam(map[string]interface{}{ - "concepts": []interface{}{"c1", "c2", "c3"}, - "certainty": float64(0.4), - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"positive"}, - "force": float64(0.5), - }, - "moveAwayFrom": map[string]interface{}{ - "concepts": []interface{}{"epic"}, - "force": float64(0.25), - }, - }), - }, - } - resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - resolver.AssertResolve(t, query) - }) - - t.Run("for things with optional distance and objects set", func(t *testing.T) { - query := `{ Get { SomeThing(nearCustomText: { - concepts: ["c1", "c2", "c3"], - distance: 0.4, - moveTo: { - concepts:["positive"], - force: 0.5 - objects: [ - { id: "moveTo-uuid1" } - { beacon: "weaviate://localhost/moveTo-uuid1" }, - { beacon: "weaviate://localhost/moveTo-uuid2" } - ] - } - moveAwayFrom: { - concepts:["epic"] - force: 0.25 - objects: [ - { id: "moveAway-uuid1" } - { beacon: "weaviate://localhost/moveAway-uuid2" } - ] - } - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeThing", - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - Pagination: &filters.Pagination{Limit: filters.LimitFlagSearchByDist}, - ModuleParams: map[string]interface{}{ - "nearCustomText": extractNearTextParam(map[string]interface{}{ - "concepts": []interface{}{"c1", "c2", "c3"}, - "distance": float64(0.4), - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"positive"}, - "force": float64(0.5), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveTo-uuid1", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveTo-uuid1", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveTo-uuid2", - }, - }, - }, - "moveAwayFrom": map[string]interface{}{ - "concepts": []interface{}{"epic"}, - "force": float64(0.25), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveAway-uuid1", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveAway-uuid2", - }, - }, - }, - }), - }, - } - resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - resolver.AssertResolve(t, query) - }) - - t.Run("for things with optional certainty and objects set", func(t *testing.T) { - query := `{ Get { SomeThing(nearCustomText: { - concepts: ["c1", "c2", "c3"], - certainty: 0.4, - moveTo: { - concepts:["positive"], - force: 0.5 - objects: [ - { id: "moveTo-uuid1" } - { beacon: "weaviate://localhost/moveTo-uuid1" }, - { beacon: "weaviate://localhost/moveTo-uuid2" } - ] - } - moveAwayFrom: { - concepts:["epic"] - force: 0.25 - objects: [ - { id: "moveAway-uuid1" } - { beacon: "weaviate://localhost/moveAway-uuid2" } - ] - } - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeThing", - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - Pagination: &filters.Pagination{Limit: filters.LimitFlagSearchByDist}, - ModuleParams: map[string]interface{}{ - "nearCustomText": extractNearTextParam(map[string]interface{}{ - "concepts": []interface{}{"c1", "c2", "c3"}, - "certainty": float64(0.4), - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"positive"}, - "force": float64(0.5), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveTo-uuid1", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveTo-uuid1", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveTo-uuid2", - }, - }, - }, - "moveAwayFrom": map[string]interface{}{ - "concepts": []interface{}{"epic"}, - "force": float64(0.25), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveAway-uuid1", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveAway-uuid2", - }, - }, - }, - }), - }, - } - resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - resolver.AssertResolve(t, query) - }) - - t.Run("for things with optional distance and limit set", func(t *testing.T) { - query := `{ Get { SomeThing( - limit: 6 - nearCustomText: { - concepts: ["c1", "c2", "c3"], - distance: 0.4, - moveTo: { - concepts:["positive"], - force: 0.5 - } - moveAwayFrom: { - concepts:["epic"] - force: 0.25 - } - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeThing", - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - Pagination: &filters.Pagination{Limit: 6}, - ModuleParams: map[string]interface{}{ - "nearCustomText": extractNearTextParam(map[string]interface{}{ - "concepts": []interface{}{"c1", "c2", "c3"}, - "distance": float64(0.4), - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"positive"}, - "force": float64(0.5), - }, - "moveAwayFrom": map[string]interface{}{ - "concepts": []interface{}{"epic"}, - "force": float64(0.25), - }, - }), - }, - } - resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - resolver.AssertResolve(t, query) - }) - - t.Run("for things with optional certainty and limit set", func(t *testing.T) { - query := `{ Get { SomeThing( - limit: 6 - nearCustomText: { - concepts: ["c1", "c2", "c3"], - certainty: 0.4, - moveTo: { - concepts:["positive"], - force: 0.5 - } - moveAwayFrom: { - concepts:["epic"] - force: 0.25 - } - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeThing", - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - Pagination: &filters.Pagination{Limit: 6}, - ModuleParams: map[string]interface{}{ - "nearCustomText": extractNearTextParam(map[string]interface{}{ - "concepts": []interface{}{"c1", "c2", "c3"}, - "certainty": float64(0.4), - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"positive"}, - "force": float64(0.5), - }, - "moveAwayFrom": map[string]interface{}{ - "concepts": []interface{}{"epic"}, - "force": float64(0.25), - }, - }), - }, - } - resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - resolver.AssertResolve(t, query) - }) - - t.Run("for things with optional distance and negative limit set", func(t *testing.T) { - query := `{ Get { SomeThing( - limit: -1 - nearCustomText: { - concepts: ["c1", "c2", "c3"], - distance: 0.4, - moveTo: { - concepts:["positive"], - force: 0.5 - } - moveAwayFrom: { - concepts:["epic"] - force: 0.25 - } - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeThing", - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - Pagination: &filters.Pagination{Limit: filters.LimitFlagSearchByDist}, - ModuleParams: map[string]interface{}{ - "nearCustomText": extractNearTextParam(map[string]interface{}{ - "concepts": []interface{}{"c1", "c2", "c3"}, - "distance": float64(0.4), - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"positive"}, - "force": float64(0.5), - }, - "moveAwayFrom": map[string]interface{}{ - "concepts": []interface{}{"epic"}, - "force": float64(0.25), - }, - }), - }, - } - resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - resolver.AssertResolve(t, query) - }) - - t.Run("for things with optional certainty and negative limit set", func(t *testing.T) { - query := `{ Get { SomeThing( - limit: -1 - nearCustomText: { - concepts: ["c1", "c2", "c3"], - certainty: 0.4, - moveTo: { - concepts:["positive"], - force: 0.5 - } - moveAwayFrom: { - concepts:["epic"] - force: 0.25 - } - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeThing", - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - Pagination: &filters.Pagination{Limit: filters.LimitFlagSearchByDist}, - ModuleParams: map[string]interface{}{ - "nearCustomText": extractNearTextParam(map[string]interface{}{ - "concepts": []interface{}{"c1", "c2", "c3"}, - "certainty": float64(0.4), - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"positive"}, - "force": float64(0.5), - }, - "moveAwayFrom": map[string]interface{}{ - "concepts": []interface{}{"epic"}, - "force": float64(0.25), - }, - }), - }, - } - resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - resolver.AssertResolve(t, query) - }) -} - -func TestNearVectorRanker(t *testing.T) { - t.Parallel() - - resolver := newMockResolver() - - t.Run("for actions", func(t *testing.T) { - query := `{ Get { SomeAction(nearVector: { - vector: [0.123, 0.984] - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeAction", - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - NearVector: &searchparams.NearVector{ - Vector: []float32{0.123, 0.984}, - }, - } - - resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - resolver.AssertResolve(t, query) - }) - - t.Run("for things with optional distance set", func(t *testing.T) { - query := `{ Get { SomeThing(nearVector: { - vector: [0.123, 0.984] - distance: 0.4 - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeThing", - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - Pagination: &filters.Pagination{Limit: filters.LimitFlagSearchByDist}, - NearVector: &searchparams.NearVector{ - Vector: []float32{0.123, 0.984}, - Distance: 0.4, - WithDistance: true, - }, - } - resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - resolver.AssertResolve(t, query) - }) - - t.Run("for things with optional certainty set", func(t *testing.T) { - query := `{ Get { SomeThing(nearVector: { - vector: [0.123, 0.984] - certainty: 0.4 - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeThing", - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - Pagination: &filters.Pagination{Limit: filters.LimitFlagSearchByDist}, - NearVector: &searchparams.NearVector{ - Vector: []float32{0.123, 0.984}, - Certainty: 0.4, - }, - } - resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - resolver.AssertResolve(t, query) - }) - - t.Run("for things with optional distance and limit set", func(t *testing.T) { - query := `{ Get { SomeThing( - limit: 4 - nearVector: { - vector: [0.123, 0.984] - distance: 0.1 - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeThing", - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - Pagination: &filters.Pagination{Limit: 4}, - NearVector: &searchparams.NearVector{ - Vector: []float32{0.123, 0.984}, - Distance: 0.1, - WithDistance: true, - }, - } - - resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - resolver.AssertResolve(t, query) - }) - - t.Run("for things with optional certainty and limit set", func(t *testing.T) { - query := `{ Get { SomeThing( - limit: 4 - nearVector: { - vector: [0.123, 0.984] - certainty: 0.1 - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeThing", - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - Pagination: &filters.Pagination{Limit: 4}, - NearVector: &searchparams.NearVector{ - Vector: []float32{0.123, 0.984}, - Certainty: 0.1, - }, - } - - resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - resolver.AssertResolve(t, query) - }) - - t.Run("for things with optional distance and negative limit set", func(t *testing.T) { - query := `{ Get { SomeThing( - limit: -1 - nearVector: { - vector: [0.123, 0.984] - distance: 0.1 - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeThing", - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - Pagination: &filters.Pagination{Limit: filters.LimitFlagSearchByDist}, - NearVector: &searchparams.NearVector{ - Vector: []float32{0.123, 0.984}, - Distance: 0.1, - WithDistance: true, - }, - } - - resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - resolver.AssertResolve(t, query) - }) - - t.Run("for things with optional certainty and negative limit set", func(t *testing.T) { - query := `{ Get { SomeThing( - limit: -1 - nearVector: { - vector: [0.123, 0.984] - certainty: 0.1 - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeThing", - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - Pagination: &filters.Pagination{Limit: filters.LimitFlagSearchByDist}, - NearVector: &searchparams.NearVector{ - Vector: []float32{0.123, 0.984}, - Certainty: 0.1, - }, - } - - resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - resolver.AssertResolve(t, query) - }) -} - -func TestExtractPagination(t *testing.T) { - t.Parallel() - - resolver := newMockResolver() - - expectedParams := dto.GetParams{ - ClassName: "SomeAction", - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - Pagination: &filters.Pagination{ - Limit: 10, - }, - } - - resolver.On("GetClass", expectedParams). - Return(test_helper.EmptyList(), nil).Once() - - query := "{ Get { SomeAction(limit: 10) { intField } } }" - resolver.AssertResolve(t, query) -} - -func TestExtractPaginationWithOffset(t *testing.T) { - t.Parallel() - - resolver := newMockResolver() - - expectedParams := dto.GetParams{ - ClassName: "SomeAction", - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - Pagination: &filters.Pagination{ - Offset: 5, - Limit: 10, - }, - } - - resolver.On("GetClass", expectedParams). - Return(test_helper.EmptyList(), nil).Once() - - query := "{ Get { SomeAction(offset: 5 limit: 10) { intField } } }" - resolver.AssertResolve(t, query) -} - -func TestExtractPaginationWithOnlyOffset(t *testing.T) { - t.Parallel() - - resolver := newMockResolver() - - expectedParams := dto.GetParams{ - ClassName: "SomeAction", - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - Pagination: &filters.Pagination{ - Offset: 5, - Limit: -1, - }, - } - - resolver.On("GetClass", expectedParams). - Return(test_helper.EmptyList(), nil).Once() - - query := "{ Get { SomeAction(offset: 5) { intField } } }" - resolver.AssertResolve(t, query) -} - -func TestExtractCursor(t *testing.T) { - t.Parallel() - - resolver := newMockResolver() - - expectedParams := dto.GetParams{ - ClassName: "SomeAction", - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - Cursor: &filters.Cursor{ - After: "8ef8d5cc-c101-4fbd-a016-84e766b93ecf", - Limit: 2, - }, - Pagination: &filters.Pagination{ - Offset: 0, - Limit: 2, - }, - } - - resolver.On("GetClass", expectedParams). - Return(test_helper.EmptyList(), nil).Once() - - query := `{ Get { SomeAction(after: "8ef8d5cc-c101-4fbd-a016-84e766b93ecf" limit: 2) { intField } } }` - resolver.AssertResolve(t, query) -} - -func TestExtractGroupParams(t *testing.T) { - t.Parallel() - - resolver := newMockResolver() - - expectedParams := dto.GetParams{ - ClassName: "SomeAction", - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - Group: &dto.GroupParams{ - Strategy: "closest", - Force: 0.3, - }, - } - - resolver.On("GetClass", expectedParams). - Return(test_helper.EmptyList(), nil).Once() - - query := "{ Get { SomeAction(group: {type: closest, force: 0.3}) { intField } } }" - resolver.AssertResolve(t, query) -} - -func TestGetRelation(t *testing.T) { - t.Parallel() - - t.Run("without using custom fragments", func(t *testing.T) { - resolver := newMockResolver() - - expectedParams := dto.GetParams{ - ClassName: "SomeAction", - Properties: []search.SelectProperty{ - { - Name: "hasAction", - IsPrimitive: false, - Refs: []search.SelectClass{ - { - ClassName: "SomeAction", - RefProperties: []search.SelectProperty{ - { - Name: "intField", - IsPrimitive: true, - }, - { - Name: "hasAction", - IsPrimitive: false, - Refs: []search.SelectClass{ - { - ClassName: "SomeAction", - RefProperties: []search.SelectProperty{ - { - Name: "intField", - IsPrimitive: true, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - } - - resolver.On("GetClass", expectedParams). - Return(test_helper.EmptyList(), nil).Once() - - query := "{ Get { SomeAction { hasAction { ... on SomeAction { intField, hasAction { ... on SomeAction { intField } } } } } } }" - resolver.AssertResolve(t, query) - }) - - t.Run("with a custom fragment one level deep", func(t *testing.T) { - resolver := newMockResolver() - - expectedParams := dto.GetParams{ - ClassName: "SomeAction", - Properties: []search.SelectProperty{ - { - Name: "hasAction", - IsPrimitive: false, - Refs: []search.SelectClass{ - { - ClassName: "SomeAction", - RefProperties: []search.SelectProperty{ - { - Name: "intField", - IsPrimitive: true, - }, - }, - }, - }, - }, - }, - } - - resolver.On("GetClass", expectedParams). - Return(test_helper.EmptyList(), nil).Once() - - query := "fragment actionFragment on SomeAction { intField } { Get { SomeAction { hasAction { ...actionFragment } } } }" - resolver.AssertResolve(t, query) - }) - - t.Run("with a custom fragment multiple levels deep", func(t *testing.T) { - resolver := newMockResolver() - - expectedParams := dto.GetParams{ - ClassName: "SomeAction", - Properties: []search.SelectProperty{ - { - Name: "hasAction", - IsPrimitive: false, - Refs: []search.SelectClass{ - { - ClassName: "SomeAction", - RefProperties: []search.SelectProperty{ - { - Name: "intField", - IsPrimitive: true, - }, - { - Name: "hasAction", - IsPrimitive: false, - Refs: []search.SelectClass{ - { - ClassName: "SomeAction", - RefProperties: []search.SelectProperty{ - { - Name: "intField", - IsPrimitive: true, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - } - - resolver.On("GetClass", expectedParams). - Return(test_helper.EmptyList(), nil).Once() - - query := ` - fragment innerFragment on SomeAction { intField } - fragment actionFragment on SomeAction { intField hasAction { ...innerFragment } } - - { Get { SomeAction { hasAction { ...actionFragment } } } }` - resolver.AssertResolve(t, query) - }) -} - -func TestNearObject(t *testing.T) { - t.Parallel() - - resolver := newMockResolver() - - t.Run("for objects with beacon", func(t *testing.T) { - query := `{ Get { SomeAction( - nearObject: { - beacon: "weaviate://localhost/some-uuid" - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeAction", - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - NearObject: &searchparams.NearObject{ - Beacon: "weaviate://localhost/some-uuid", - }, - } - - resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - resolver.AssertResolve(t, query) - }) - - t.Run("for objects with beacon and optional distance set", func(t *testing.T) { - query := `{ Get { SomeThing( - nearObject: { - beacon: "weaviate://localhost/some-other-uuid" - distance: 0.7 - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeThing", - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - Pagination: &filters.Pagination{Limit: filters.LimitFlagSearchByDist}, - NearObject: &searchparams.NearObject{ - Beacon: "weaviate://localhost/some-other-uuid", - Distance: 0.7, - WithDistance: true, - }, - } - resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - resolver.AssertResolve(t, query) - }) - - t.Run("for objects with beacon and optional certainty set", func(t *testing.T) { - query := `{ Get { SomeThing( - nearObject: { - beacon: "weaviate://localhost/some-other-uuid" - certainty: 0.7 - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeThing", - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - Pagination: &filters.Pagination{Limit: filters.LimitFlagSearchByDist}, - NearObject: &searchparams.NearObject{ - Beacon: "weaviate://localhost/some-other-uuid", - Certainty: 0.7, - }, - } - resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - resolver.AssertResolve(t, query) - }) - - t.Run("for objects with id set", func(t *testing.T) { - query := `{ Get { SomeAction( - nearObject: { - id: "some-uuid" - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeAction", - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - NearObject: &searchparams.NearObject{ - ID: "some-uuid", - }, - } - - resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - resolver.AssertResolve(t, query) - }) - - t.Run("for objects with id and optional distance set", func(t *testing.T) { - query := `{ Get { SomeThing( - nearObject: { - id: "some-other-uuid" - distance: 0.7 - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeThing", - Pagination: &filters.Pagination{Limit: filters.LimitFlagSearchByDist}, - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - NearObject: &searchparams.NearObject{ - ID: "some-other-uuid", - Distance: 0.7, - WithDistance: true, - }, - } - resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - resolver.AssertResolve(t, query) - }) - - t.Run("for objects with id and optional certainty set", func(t *testing.T) { - query := `{ Get { SomeThing( - nearObject: { - id: "some-other-uuid" - certainty: 0.7 - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeThing", - Pagination: &filters.Pagination{Limit: filters.LimitFlagSearchByDist}, - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - NearObject: &searchparams.NearObject{ - ID: "some-other-uuid", - Certainty: 0.7, - }, - } - resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - resolver.AssertResolve(t, query) - }) - - t.Run("for objects with optional distance and limit set", func(t *testing.T) { - query := `{ Get { SomeThing( - limit: 5 - nearObject: { - id: "some-other-uuid" - distance: 0.7 - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeThing", - Pagination: &filters.Pagination{Limit: 5}, - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - NearObject: &searchparams.NearObject{ - ID: "some-other-uuid", - Distance: 0.7, - WithDistance: true, - }, - } - resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - resolver.AssertResolve(t, query) - }) - - t.Run("for objects with optional certainty and limit set", func(t *testing.T) { - query := `{ Get { SomeThing( - limit: 5 - nearObject: { - id: "some-other-uuid" - certainty: 0.7 - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeThing", - Pagination: &filters.Pagination{Limit: 5}, - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - NearObject: &searchparams.NearObject{ - ID: "some-other-uuid", - Certainty: 0.7, - }, - } - resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - resolver.AssertResolve(t, query) - }) - - t.Run("for objects with optional distance and negative limit set", func(t *testing.T) { - query := `{ Get { SomeThing( - limit: -1 - nearObject: { - id: "some-other-uuid" - distance: 0.7 - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeThing", - Pagination: &filters.Pagination{Limit: filters.LimitFlagSearchByDist}, - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - NearObject: &searchparams.NearObject{ - ID: "some-other-uuid", - Distance: 0.7, - WithDistance: true, - }, - } - resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - resolver.AssertResolve(t, query) - }) - - t.Run("for objects with optional certainty and negative limit set", func(t *testing.T) { - query := `{ Get { SomeThing( - limit: -1 - nearObject: { - id: "some-other-uuid" - certainty: 0.7 - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeThing", - Pagination: &filters.Pagination{Limit: filters.LimitFlagSearchByDist}, - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - NearObject: &searchparams.NearObject{ - ID: "some-other-uuid", - Certainty: 0.7, - }, - } - resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - resolver.AssertResolve(t, query) - }) -} - -func TestNearTextNoNoModules(t *testing.T) { - t.Parallel() - - resolver := newMockResolverWithNoModules() - - t.Run("for nearText that is not available", func(t *testing.T) { - query := `{ Get { SomeAction(nearText: { - concepts: ["c1", "c2", "c3"], - moveTo: { - concepts:["positive"], - force: 0.5 - }, - moveAwayFrom: { - concepts:["epic"], - force: 0.25 - } - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeAction", - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - } - - resolver.On("GetClass", expectedParams). - Return(nil, nil).Once() - - resolver.AssertFailToResolve(t, query) - }) -} - -func TestBM25WithSort(t *testing.T) { - t.Parallel() - resolver := newMockResolverWithNoModules() - query := `{Get{SomeAction(bm25:{query:"apple",properties:["name"]},sort:[{path:["name"],order:desc}]){intField}}}` - resolver.AssertFailToResolve(t, query, "bm25 search is not compatible with sort") -} - -func TestHybridWithSort(t *testing.T) { - t.Parallel() - resolver := newMockResolverWithNoModules() - query := `{Get{SomeAction(hybrid:{query:"apple"},sort:[{path:["name"],order:desc}]){intField}}}` - resolver.AssertFailToResolve(t, query, "hybrid search is not compatible with sort") -} - -func TestNearObjectNoModules(t *testing.T) { - t.Parallel() - - resolver := newMockResolverWithNoModules() - - t.Run("for objects with beacon", func(t *testing.T) { - query := `{ Get { SomeAction( - nearObject: { - beacon: "weaviate://localhost/some-uuid" - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeAction", - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - NearObject: &searchparams.NearObject{ - Beacon: "weaviate://localhost/some-uuid", - }, - } - - resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - resolver.AssertResolve(t, query) - }) - - t.Run("for objects with ID and distance set", func(t *testing.T) { - query := `{ Get { SomeThing( - nearObject: { - id: "some-uuid" - distance: 0.7 - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeThing", - Pagination: &filters.Pagination{Limit: filters.LimitFlagSearchByDist}, - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - NearObject: &searchparams.NearObject{ - ID: "some-uuid", - Distance: 0.7, - WithDistance: true, - }, - } - - resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - resolver.AssertResolve(t, query) - }) - - t.Run("for objects with ID and certainty set", func(t *testing.T) { - query := `{ Get { SomeThing( - nearObject: { - id: "some-uuid" - certainty: 0.7 - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeThing", - Pagination: &filters.Pagination{Limit: filters.LimitFlagSearchByDist}, - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - NearObject: &searchparams.NearObject{ - ID: "some-uuid", - Certainty: 0.7, - }, - } - - resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - resolver.AssertResolve(t, query) - }) - - t.Run("for objects with distance and limit set", func(t *testing.T) { - query := `{ Get { SomeThing( - limit: 12 - nearObject: { - id: "some-uuid" - distance: 0.7 - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeThing", - Pagination: &filters.Pagination{Limit: 12}, - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - NearObject: &searchparams.NearObject{ - ID: "some-uuid", - Distance: 0.7, - WithDistance: true, - }, - } - - resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - resolver.AssertResolve(t, query) - }) - - t.Run("for objects with certainty and limit set", func(t *testing.T) { - query := `{ Get { SomeThing( - limit: 12 - nearObject: { - id: "some-uuid" - certainty: 0.7 - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeThing", - Pagination: &filters.Pagination{Limit: 12}, - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - NearObject: &searchparams.NearObject{ - ID: "some-uuid", - Certainty: 0.7, - }, - } - - resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - resolver.AssertResolve(t, query) - }) - - t.Run("for objects with distance and negative limit set", func(t *testing.T) { - query := `{ Get { SomeThing( - limit: -1 - nearObject: { - id: "some-uuid" - distance: 0.7 - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeThing", - Pagination: &filters.Pagination{Limit: filters.LimitFlagSearchByDist}, - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - NearObject: &searchparams.NearObject{ - ID: "some-uuid", - Distance: 0.7, - WithDistance: true, - }, - } - - resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - resolver.AssertResolve(t, query) - }) - - t.Run("for objects with certainty and negative limit set", func(t *testing.T) { - query := `{ Get { SomeThing( - limit: -1 - nearObject: { - id: "some-uuid" - certainty: 0.7 - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeThing", - Pagination: &filters.Pagination{Limit: filters.LimitFlagSearchByDist}, - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - NearObject: &searchparams.NearObject{ - ID: "some-uuid", - Certainty: 0.7, - }, - } - - resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - resolver.AssertResolve(t, query) - }) -} - -func TestNearVectorNoModules(t *testing.T) { - t.Parallel() - - resolver := newMockResolverWithNoModules() - - t.Run("for actions", func(t *testing.T) { - query := `{ Get { SomeAction(nearVector: { - vector: [0.123, 0.984] - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeAction", - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - NearVector: &searchparams.NearVector{ - Vector: []float32{0.123, 0.984}, - }, - } - - resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - resolver.AssertResolve(t, query) - }) - - t.Run("for things with optional distance set", func(t *testing.T) { - query := `{ Get { SomeThing(nearVector: { - vector: [0.123, 0.984] - distance: 0.4 - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeThing", - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - Pagination: &filters.Pagination{Limit: filters.LimitFlagSearchByDist}, - NearVector: &searchparams.NearVector{ - Vector: []float32{0.123, 0.984}, - Distance: 0.4, - WithDistance: true, - }, - } - resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - resolver.AssertResolve(t, query) - }) - - t.Run("for things with optional certainty set", func(t *testing.T) { - query := `{ Get { SomeThing(nearVector: { - vector: [0.123, 0.984] - certainty: 0.4 - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeThing", - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - Pagination: &filters.Pagination{Limit: filters.LimitFlagSearchByDist}, - NearVector: &searchparams.NearVector{ - Vector: []float32{0.123, 0.984}, - Certainty: 0.4, - }, - } - resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - resolver.AssertResolve(t, query) - }) - - t.Run("for things with optional certainty and limit set", func(t *testing.T) { - query := `{ Get { SomeThing( - limit: 4 - nearVector: { - vector: [0.123, 0.984] - certainty: 0.4 - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeThing", - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - Pagination: &filters.Pagination{Limit: 4}, - NearVector: &searchparams.NearVector{ - Vector: []float32{0.123, 0.984}, - Certainty: 0.4, - }, - } - resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - resolver.AssertResolve(t, query) - }) - - t.Run("for things with optional distance and negative limit set", func(t *testing.T) { - query := `{ Get { SomeThing( - limit: -1 - nearVector: { - vector: [0.123, 0.984] - distance: 0.4 - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeThing", - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - Pagination: &filters.Pagination{Limit: filters.LimitFlagSearchByDist}, - NearVector: &searchparams.NearVector{ - Vector: []float32{0.123, 0.984}, - Distance: 0.4, - WithDistance: true, - }, - } - resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - resolver.AssertResolve(t, query) - }) - - t.Run("for things with optional certainty and negative limit set", func(t *testing.T) { - query := `{ Get { SomeThing( - limit: -1 - nearVector: { - vector: [0.123, 0.984] - certainty: 0.4 - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeThing", - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - Pagination: &filters.Pagination{Limit: filters.LimitFlagSearchByDist}, - NearVector: &searchparams.NearVector{ - Vector: []float32{0.123, 0.984}, - Certainty: 0.4, - }, - } - resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - resolver.AssertResolve(t, query) - }) -} - -func TestSort(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - resolver *mockResolver - }{ - { - name: "with modules", - resolver: newMockResolver(), - }, - { - name: "with no modules", - resolver: newMockResolverWithNoModules(), - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Run("simple sort", func(t *testing.T) { - query := `{ Get { SomeAction(sort:[{ - path: ["path"] order: asc - }]) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeAction", - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - Sort: []filters.Sort{{Path: []string{"path"}, Order: "asc"}}, - } - - tt.resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - tt.resolver.AssertResolve(t, query) - }) - - t.Run("simple sort with two paths", func(t *testing.T) { - query := `{ Get { SomeAction(sort:[{ - path: ["path1", "path2"] order: desc - }]) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeAction", - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - Sort: []filters.Sort{{Path: []string{"path1", "path2"}, Order: "desc"}}, - } - - tt.resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - tt.resolver.AssertResolve(t, query) - }) - - t.Run("simple sort with two sort filters", func(t *testing.T) { - query := `{ Get { SomeAction(sort:[{ - path: ["first1", "first2", "first3", "first4"] order: asc - } { - path: ["second1"] order: desc - }]) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeAction", - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - Sort: []filters.Sort{ - {Path: []string{"first1", "first2", "first3", "first4"}, Order: "asc"}, - {Path: []string{"second1"}, Order: "desc"}, - }, - } - - tt.resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - tt.resolver.AssertResolve(t, query) - }) - }) - } -} - -func TestGroupBy(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - resolver *mockResolver - }{ - { - name: "with modules", - resolver: newMockResolver(), - }, - { - name: "with no modules", - resolver: newMockResolverWithNoModules(), - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Run("simple groupBy", func(t *testing.T) { - query := `{ Get { - SomeAction( - groupBy:{path: ["path"] groups: 2 objectsPerGroup:3} - ) { - _additional{group{count groupedBy {value path} maxDistance minDistance hits {_additional{distance}}} - } - } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeAction", - GroupBy: &searchparams.GroupBy{Property: "path", Groups: 2, ObjectsPerGroup: 3}, - AdditionalProperties: additional.Properties{Group: true}, - } - - tt.resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - tt.resolver.AssertResolve(t, query) - }) - }) - } -} - -func ptFloat32(in float32) *float32 { - return &in -} - -func timeMust(t strfmt.DateTime, err error) strfmt.DateTime { - if err != nil { - panic(err) - } - - return t -} diff --git a/adapters/handlers/graphql/local/get/group_argument.go b/adapters/handlers/graphql/local/get/group_argument.go deleted file mode 100644 index bfb37ccf2a15a47cd8dd7927914c73b30bf01c3c..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/get/group_argument.go +++ /dev/null @@ -1,52 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package get - -import ( - "fmt" - - "github.com/tailor-inc/graphql" - "github.com/weaviate/weaviate/adapters/handlers/graphql/descriptions" -) - -func groupArgument(className string) *graphql.ArgumentConfig { - prefix := fmt.Sprintf("GetObjects%s", className) - return &graphql.ArgumentConfig{ - // Description: descriptions.GetGroup, - Type: graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sGroupInpObj", prefix), - Fields: groupFields(prefix), - Description: descriptions.GetWhereInpObj, - }, - ), - } -} - -func groupFields(prefix string) graphql.InputObjectConfigFieldMap { - return graphql.InputObjectConfigFieldMap{ - "type": &graphql.InputObjectFieldConfig{ - // Description: descriptions.Concepts, - Type: graphql.NewEnum(graphql.EnumConfig{ - Name: fmt.Sprintf("%sGroupInpObjTypeEnum", prefix), - Values: graphql.EnumValueConfigMap{ - "closest": &graphql.EnumValueConfig{}, - "merge": &graphql.EnumValueConfig{}, - }, - }), - }, - "force": &graphql.InputObjectFieldConfig{ - Description: descriptions.Force, - Type: graphql.NewNonNull(graphql.Float), - }, - } -} diff --git a/adapters/handlers/graphql/local/get/group_by_argument.go b/adapters/handlers/graphql/local/get/group_by_argument.go deleted file mode 100644 index 706af9ccbd51cfddd1a6eb9295ac90ac8cab9e2f..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/get/group_by_argument.go +++ /dev/null @@ -1,49 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package get - -import ( - "fmt" - - "github.com/tailor-inc/graphql" - "github.com/weaviate/weaviate/adapters/handlers/graphql/descriptions" -) - -func groupByArgument(className string) *graphql.ArgumentConfig { - prefix := fmt.Sprintf("GetObjects%s", className) - return &graphql.ArgumentConfig{ - Type: graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sGroupByInpObj", prefix), - Fields: groupByFields(prefix), - Description: descriptions.GroupByFilter, - }, - ), - } -} - -func groupByFields(prefix string) graphql.InputObjectConfigFieldMap { - return graphql.InputObjectConfigFieldMap{ - "path": &graphql.InputObjectFieldConfig{ - Description: descriptions.GroupByPath, - Type: graphql.NewNonNull(graphql.NewList(graphql.String)), - }, - "groups": &graphql.InputObjectFieldConfig{ - Description: descriptions.GroupByGroups, - Type: graphql.NewNonNull(graphql.Int), - }, - "objectsPerGroup": &graphql.InputObjectFieldConfig{ - Description: descriptions.GroupByObjectsPerGroup, - Type: graphql.NewNonNull(graphql.Int), - }, - } -} diff --git a/adapters/handlers/graphql/local/get/helper_test.go b/adapters/handlers/graphql/local/get/helper_test.go deleted file mode 100644 index dde787505d0ad714d1fac8b4778322f59efd4c44..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/get/helper_test.go +++ /dev/null @@ -1,653 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package get - -import ( - "context" - "fmt" - "net/http" - - "github.com/sirupsen/logrus/hooks/test" - "github.com/tailor-inc/graphql" - "github.com/tailor-inc/graphql/language/ast" - "github.com/weaviate/weaviate/adapters/handlers/graphql/descriptions" - test_helper "github.com/weaviate/weaviate/adapters/handlers/graphql/test/helper" - "github.com/weaviate/weaviate/entities/dto" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/search" - "github.com/weaviate/weaviate/usecases/config" -) - -type mockRequestsLog struct{} - -func (m *mockRequestsLog) Register(first string, second string) { -} - -type mockResolver struct { - test_helper.MockResolver -} - -type fakeInterpretation struct { - returnArgs []search.Result -} - -func (f *fakeInterpretation) AdditionalPropertyFn(ctx context.Context, - in []search.Result, params interface{}, limit *int, - argumentModuleParams map[string]interface{}, -) ([]search.Result, error) { - return f.returnArgs, nil -} - -func (f *fakeInterpretation) ExtractAdditionalFn(param []*ast.Argument) interface{} { - return true -} - -func (f *fakeInterpretation) AdditonalPropertyDefaultValue() interface{} { - return true -} - -type fakeExtender struct { - returnArgs []search.Result -} - -func (f *fakeExtender) AdditionalPropertyFn(ctx context.Context, - in []search.Result, params interface{}, limit *int, - argumentModuleParams map[string]interface{}, -) ([]search.Result, error) { - return f.returnArgs, nil -} - -func (f *fakeExtender) ExtractAdditionalFn(param []*ast.Argument) interface{} { - return true -} - -func (f *fakeExtender) AdditonalPropertyDefaultValue() interface{} { - return true -} - -type fakeProjectorParams struct { - Enabled bool - Algorithm string - Dimensions int - Perplexity int - Iterations int - LearningRate int - IncludeNeighbors bool -} - -type fakeProjector struct { - returnArgs []search.Result -} - -func (f *fakeProjector) AdditionalPropertyFn(ctx context.Context, - in []search.Result, params interface{}, limit *int, - argumentModuleParams map[string]interface{}, -) ([]search.Result, error) { - return f.returnArgs, nil -} - -func (f *fakeProjector) ExtractAdditionalFn(param []*ast.Argument) interface{} { - if len(param) > 0 { - return &fakeProjectorParams{ - Enabled: true, - Algorithm: "tsne", - Dimensions: 3, - Iterations: 100, - LearningRate: 15, - Perplexity: 10, - } - } - return &fakeProjectorParams{ - Enabled: true, - } -} - -func (f *fakeProjector) AdditonalPropertyDefaultValue() interface{} { - return &fakeProjectorParams{} -} - -type pathBuilderParams struct{} - -type fakePathBuilder struct { - returnArgs []search.Result -} - -func (f *fakePathBuilder) AdditionalPropertyFn(ctx context.Context, - in []search.Result, params interface{}, limit *int, -) ([]search.Result, error) { - return f.returnArgs, nil -} - -func (f *fakePathBuilder) ExtractAdditionalFn(param []*ast.Argument) interface{} { - return &pathBuilderParams{} -} - -func (f *fakePathBuilder) AdditonalPropertyDefaultValue() interface{} { - return &pathBuilderParams{} -} - -type nearCustomTextParams struct { - Values []string - MoveTo nearExploreMove - MoveAwayFrom nearExploreMove - Certainty float64 - Distance float64 - WithDistance bool -} - -// implements the modulecapabilities.NearParam interface -func (n *nearCustomTextParams) GetCertainty() float64 { - return n.Certainty -} - -func (n nearCustomTextParams) GetDistance() float64 { - return n.Distance -} - -func (n nearCustomTextParams) SimilarityMetricProvided() bool { - return n.Certainty != 0 || n.WithDistance -} - -type nearExploreMove struct { - Values []string - Force float32 - Objects []nearObjectMove -} - -type nearObjectMove struct { - ID string - Beacon string -} - -type nearCustomTextModule struct { - fakePathBuilder *fakePathBuilder - fakeProjector *fakeProjector - fakeExtender *fakeExtender - fakeInterpretation *fakeInterpretation -} - -func newNearCustomTextModule() *nearCustomTextModule { - return &nearCustomTextModule{ - fakePathBuilder: &fakePathBuilder{}, - fakeProjector: &fakeProjector{}, - fakeExtender: &fakeExtender{}, - fakeInterpretation: &fakeInterpretation{}, - } -} - -func (m *nearCustomTextModule) Name() string { - return "mock-custom-near-text-module" -} - -func (m *nearCustomTextModule) Init(params moduletools.ModuleInitParams) error { - return nil -} - -func (m *nearCustomTextModule) RootHandler() http.Handler { - return nil -} - -func (m *nearCustomTextModule) getNearCustomTextArgument(classname string) *graphql.ArgumentConfig { - prefix := classname - return &graphql.ArgumentConfig{ - Type: graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sNearCustomTextInpObj", prefix), - Fields: graphql.InputObjectConfigFieldMap{ - "concepts": &graphql.InputObjectFieldConfig{ - Type: graphql.NewNonNull(graphql.NewList(graphql.String)), - }, - "moveTo": &graphql.InputObjectFieldConfig{ - Description: descriptions.VectorMovement, - Type: graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sMoveTo", prefix), - Fields: graphql.InputObjectConfigFieldMap{ - "concepts": &graphql.InputObjectFieldConfig{ - Description: descriptions.Keywords, - Type: graphql.NewList(graphql.String), - }, - "objects": &graphql.InputObjectFieldConfig{ - Description: "objects", - Type: graphql.NewList(graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sMovementObjectsToInpObj", prefix), - Fields: graphql.InputObjectConfigFieldMap{ - "id": &graphql.InputObjectFieldConfig{ - Type: graphql.String, - Description: "id of an object", - }, - "beacon": &graphql.InputObjectFieldConfig{ - Type: graphql.String, - Description: descriptions.Beacon, - }, - }, - Description: "Movement Object", - }, - )), - }, - "force": &graphql.InputObjectFieldConfig{ - Description: descriptions.Force, - Type: graphql.NewNonNull(graphql.Float), - }, - }, - }), - }, - "moveAwayFrom": &graphql.InputObjectFieldConfig{ - Description: descriptions.VectorMovement, - Type: graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sMoveAway", prefix), - Fields: graphql.InputObjectConfigFieldMap{ - "concepts": &graphql.InputObjectFieldConfig{ - Description: descriptions.Keywords, - Type: graphql.NewList(graphql.String), - }, - "objects": &graphql.InputObjectFieldConfig{ - Description: "objects", - Type: graphql.NewList(graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sMovementObjectsAwayInpObj", prefix), - Fields: graphql.InputObjectConfigFieldMap{ - "id": &graphql.InputObjectFieldConfig{ - Type: graphql.String, - Description: "id of an object", - }, - "beacon": &graphql.InputObjectFieldConfig{ - Type: graphql.String, - Description: descriptions.Beacon, - }, - }, - Description: "Movement Object", - }, - )), - }, - "force": &graphql.InputObjectFieldConfig{ - Description: descriptions.Force, - Type: graphql.NewNonNull(graphql.Float), - }, - }, - }), - }, - "certainty": &graphql.InputObjectFieldConfig{ - Description: descriptions.Certainty, - Type: graphql.Float, - }, - "distance": &graphql.InputObjectFieldConfig{ - Description: descriptions.Distance, - Type: graphql.Float, - }, - }, - Description: descriptions.GetWhereInpObj, - }, - ), - } -} - -func (m *nearCustomTextModule) extractNearCustomTextArgument(source map[string]interface{}) *nearCustomTextParams { - var args nearCustomTextParams - - concepts := source["concepts"].([]interface{}) - args.Values = make([]string, len(concepts)) - for i, value := range concepts { - args.Values[i] = value.(string) - } - - certainty, ok := source["certainty"] - if ok { - args.Certainty = certainty.(float64) - } - - distance, ok := source["distance"] - if ok { - args.Distance = distance.(float64) - args.WithDistance = true - } - - // moveTo is an optional arg, so it could be nil - moveTo, ok := source["moveTo"] - if ok { - moveToMap := moveTo.(map[string]interface{}) - args.MoveTo = m.parseMoveParam(moveToMap) - } - - moveAwayFrom, ok := source["moveAwayFrom"] - if ok { - moveAwayFromMap := moveAwayFrom.(map[string]interface{}) - args.MoveAwayFrom = m.parseMoveParam(moveAwayFromMap) - } - - return &args -} - -func (m *nearCustomTextModule) parseMoveParam(source map[string]interface{}) nearExploreMove { - res := nearExploreMove{} - res.Force = float32(source["force"].(float64)) - - concepts, ok := source["concepts"].([]interface{}) - if ok { - res.Values = make([]string, len(concepts)) - for i, value := range concepts { - res.Values[i] = value.(string) - } - } - - objects, ok := source["objects"].([]interface{}) - if ok { - res.Objects = make([]nearObjectMove, len(objects)) - for i, value := range objects { - v, ok := value.(map[string]interface{}) - if ok { - if v["id"] != nil { - res.Objects[i].ID = v["id"].(string) - } - if v["beacon"] != nil { - res.Objects[i].Beacon = v["beacon"].(string) - } - } - } - } - - return res -} - -func (m *nearCustomTextModule) Arguments() map[string]modulecapabilities.GraphQLArgument { - arguments := map[string]modulecapabilities.GraphQLArgument{} - // define nearCustomText argument - arguments["nearCustomText"] = modulecapabilities.GraphQLArgument{ - GetArgumentsFunction: func(classname string) *graphql.ArgumentConfig { - return m.getNearCustomTextArgument(classname) - }, - ExtractFunction: func(source map[string]interface{}) interface{} { - return m.extractNearCustomTextArgument(source) - }, - ValidateFunction: func(param interface{}) error { - // all is valid - return nil - }, - } - return arguments -} - -// additional properties -func (m *nearCustomTextModule) AdditionalProperties() map[string]modulecapabilities.AdditionalProperty { - additionalProperties := map[string]modulecapabilities.AdditionalProperty{} - additionalProperties["featureProjection"] = m.getFeatureProjection() - additionalProperties["nearestNeighbors"] = m.getNearestNeighbors() - additionalProperties["semanticPath"] = m.getSemanticPath() - additionalProperties["interpretation"] = m.getInterpretation() - return additionalProperties -} - -func (m *nearCustomTextModule) getFeatureProjection() modulecapabilities.AdditionalProperty { - return modulecapabilities.AdditionalProperty{ - DefaultValue: m.fakeProjector.AdditonalPropertyDefaultValue(), - GraphQLNames: []string{"featureProjection"}, - GraphQLFieldFunction: func(classname string) *graphql.Field { - return &graphql.Field{ - Args: graphql.FieldConfigArgument{ - "algorithm": &graphql.ArgumentConfig{ - Type: graphql.String, - DefaultValue: nil, - }, - "dimensions": &graphql.ArgumentConfig{ - Type: graphql.Int, - DefaultValue: nil, - }, - "learningRate": &graphql.ArgumentConfig{ - Type: graphql.Int, - DefaultValue: nil, - }, - "iterations": &graphql.ArgumentConfig{ - Type: graphql.Int, - DefaultValue: nil, - }, - "perplexity": &graphql.ArgumentConfig{ - Type: graphql.Int, - DefaultValue: nil, - }, - }, - Type: graphql.NewObject(graphql.ObjectConfig{ - Name: fmt.Sprintf("%sAdditionalFeatureProjection", classname), - Fields: graphql.Fields{ - "vector": &graphql.Field{Type: graphql.NewList(graphql.Float)}, - }, - }), - } - }, - GraphQLExtractFunction: m.fakeProjector.ExtractAdditionalFn, - } -} - -func (m *nearCustomTextModule) getNearestNeighbors() modulecapabilities.AdditionalProperty { - return modulecapabilities.AdditionalProperty{ - DefaultValue: m.fakeExtender.AdditonalPropertyDefaultValue(), - GraphQLNames: []string{"nearestNeighbors"}, - GraphQLFieldFunction: func(classname string) *graphql.Field { - return &graphql.Field{ - Type: graphql.NewObject(graphql.ObjectConfig{ - Name: fmt.Sprintf("%sAdditionalNearestNeighbors", classname), - Fields: graphql.Fields{ - "neighbors": &graphql.Field{Type: graphql.NewList(graphql.NewObject(graphql.ObjectConfig{ - Name: fmt.Sprintf("%sAdditionalNearestNeighborsNeighbors", classname), - Fields: graphql.Fields{ - "concept": &graphql.Field{Type: graphql.String}, - "distance": &graphql.Field{Type: graphql.Float}, - }, - }))}, - }, - }), - } - }, - GraphQLExtractFunction: m.fakeExtender.ExtractAdditionalFn, - } -} - -func (m *nearCustomTextModule) getSemanticPath() modulecapabilities.AdditionalProperty { - return modulecapabilities.AdditionalProperty{ - DefaultValue: m.fakePathBuilder.AdditonalPropertyDefaultValue(), - GraphQLNames: []string{"semanticPath"}, - GraphQLFieldFunction: func(classname string) *graphql.Field { - return &graphql.Field{ - Type: graphql.NewObject(graphql.ObjectConfig{ - Name: fmt.Sprintf("%sAdditionalSemanticPath", classname), - Fields: graphql.Fields{ - "path": &graphql.Field{Type: graphql.NewList(graphql.NewObject(graphql.ObjectConfig{ - Name: fmt.Sprintf("%sAdditionalSemanticPathElement", classname), - Fields: graphql.Fields{ - "concept": &graphql.Field{Type: graphql.String}, - "distanceToQuery": &graphql.Field{Type: graphql.Float}, - "distanceToResult": &graphql.Field{Type: graphql.Float}, - "distanceToNext": &graphql.Field{Type: graphql.Float}, - "distanceToPrevious": &graphql.Field{Type: graphql.Float}, - }, - }))}, - }, - }), - } - }, - GraphQLExtractFunction: m.fakePathBuilder.ExtractAdditionalFn, - } -} - -func (m *nearCustomTextModule) getInterpretation() modulecapabilities.AdditionalProperty { - return modulecapabilities.AdditionalProperty{ - DefaultValue: m.fakeInterpretation.AdditonalPropertyDefaultValue(), - GraphQLNames: []string{"interpretation"}, - GraphQLFieldFunction: func(classname string) *graphql.Field { - return &graphql.Field{ - Type: graphql.NewObject(graphql.ObjectConfig{ - Name: fmt.Sprintf("%sAdditionalInterpretation", classname), - Fields: graphql.Fields{ - "source": &graphql.Field{Type: graphql.NewList(graphql.NewObject(graphql.ObjectConfig{ - Name: fmt.Sprintf("%sAdditionalInterpretationSource", classname), - Fields: graphql.Fields{ - "concept": &graphql.Field{Type: graphql.String}, - "weight": &graphql.Field{Type: graphql.Float}, - "occurrence": &graphql.Field{Type: graphql.Int}, - }, - }))}, - }, - }), - } - }, - GraphQLExtractFunction: m.fakeInterpretation.ExtractAdditionalFn, - } -} - -type fakeModulesProvider struct { - nearCustomTextModule *nearCustomTextModule -} - -func newFakeModulesProvider() *fakeModulesProvider { - return &fakeModulesProvider{newNearCustomTextModule()} -} - -func (fmp *fakeModulesProvider) GetAll() []modulecapabilities.Module { - panic("implement me") -} - -func (fmp *fakeModulesProvider) VectorFromInput(ctx context.Context, className string, input string) ([]float32, error) { - panic("not implemented") -} - -func (fmp *fakeModulesProvider) GetArguments(class *models.Class) map[string]*graphql.ArgumentConfig { - args := map[string]*graphql.ArgumentConfig{} - if class.Vectorizer == fmp.nearCustomTextModule.Name() { - for name, argument := range fmp.nearCustomTextModule.Arguments() { - args[name] = argument.GetArgumentsFunction(class.Class) - } - } - return args -} - -func (fmp *fakeModulesProvider) ExtractSearchParams(arguments map[string]interface{}, className string) map[string]interface{} { - exractedParams := map[string]interface{}{} - if param, ok := arguments["nearCustomText"]; ok { - exractedParams["nearCustomText"] = extractNearTextParam(param.(map[string]interface{})) - } - return exractedParams -} - -func (fmp *fakeModulesProvider) GetAdditionalFields(class *models.Class) map[string]*graphql.Field { - additionalProperties := map[string]*graphql.Field{} - for name, additionalProperty := range fmp.nearCustomTextModule.AdditionalProperties() { - if additionalProperty.GraphQLFieldFunction != nil { - additionalProperties[name] = additionalProperty.GraphQLFieldFunction(class.Class) - } - } - return additionalProperties -} - -func (fmp *fakeModulesProvider) ExtractAdditionalField(className, name string, params []*ast.Argument) interface{} { - if additionalProperties := fmp.nearCustomTextModule.AdditionalProperties(); len(additionalProperties) > 0 { - if additionalProperty, ok := additionalProperties[name]; ok { - if additionalProperty.GraphQLExtractFunction != nil { - return additionalProperty.GraphQLExtractFunction(params) - } - } - } - return nil -} - -func (fmp *fakeModulesProvider) GraphQLAdditionalFieldNames() []string { - additionalPropertiesNames := []string{} - for _, additionalProperty := range fmp.nearCustomTextModule.AdditionalProperties() { - if additionalProperty.GraphQLNames != nil { - additionalPropertiesNames = append(additionalPropertiesNames, additionalProperty.GraphQLNames...) - } - } - return additionalPropertiesNames -} - -func extractNearTextParam(param map[string]interface{}) interface{} { - nearCustomTextModule := newNearCustomTextModule() - argument := nearCustomTextModule.Arguments()["nearCustomText"] - return argument.ExtractFunction(param) -} - -func createArg(name string, value string) *ast.Argument { - n := ast.Name{ - Value: name, - } - val := ast.StringValue{ - Kind: "Kind", - Value: value, - } - arg := ast.Argument{ - Name: ast.NewName(&n), - Kind: "Kind", - Value: ast.NewStringValue(&val), - } - a := ast.NewArgument(&arg) - return a -} - -func extractAdditionalParam(name string, args []*ast.Argument) interface{} { - nearCustomTextModule := newNearCustomTextModule() - additionalProperties := nearCustomTextModule.AdditionalProperties() - switch name { - case "semanticPath", "featureProjection": - if ap, ok := additionalProperties[name]; ok { - return ap.GraphQLExtractFunction(args) - } - return nil - default: - return nil - } -} - -func getFakeModulesProvider() ModulesProvider { - return newFakeModulesProvider() -} - -func newMockResolver() *mockResolver { - return newMockResolverWithVectorizer(config.VectorizerModuleText2VecContextionary) -} - -func newMockResolverWithVectorizer(vectorizer string) *mockResolver { - logger, _ := test.NewNullLogger() - simpleSchema := test_helper.CreateSimpleSchema(vectorizer) - field, err := Build(&simpleSchema, logger, getFakeModulesProvider()) - if err != nil { - panic(fmt.Sprintf("could not build graphql test schema: %s", err)) - } - mocker := &mockResolver{} - mockLog := &mockRequestsLog{} - mocker.RootFieldName = "Get" - mocker.RootField = field - mocker.RootObject = map[string]interface{}{"Resolver": Resolver(mocker), "RequestsLog": RequestsLog(mockLog)} - return mocker -} - -func newMockResolverWithNoModules() *mockResolver { - logger, _ := test.NewNullLogger() - field, err := Build(&test_helper.SimpleSchema, logger, nil) - if err != nil { - panic(fmt.Sprintf("could not build graphql test schema: %s", err)) - } - mocker := &mockResolver{} - mockLog := &mockRequestsLog{} - mocker.RootFieldName = "Get" - mocker.RootField = field - mocker.RootObject = map[string]interface{}{"Resolver": Resolver(mocker), "RequestsLog": RequestsLog(mockLog)} - return mocker -} - -func (m *mockResolver) GetClass(ctx context.Context, principal *models.Principal, - params dto.GetParams, -) ([]interface{}, error) { - args := m.Called(params) - return args.Get(0).([]interface{}), args.Error(1) -} diff --git a/adapters/handlers/graphql/local/get/hybrid_search.go b/adapters/handlers/graphql/local/get/hybrid_search.go deleted file mode 100644 index 810fbb01ddc2e37401a82ab5c4f2e63b3c994869..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/get/hybrid_search.go +++ /dev/null @@ -1,112 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package get - -import ( - "fmt" - "os" - - "github.com/tailor-inc/graphql" - "github.com/weaviate/weaviate/adapters/handlers/graphql/descriptions" - "github.com/weaviate/weaviate/entities/models" -) - -func hybridArgument(classObject *graphql.Object, - class *models.Class, modulesProvider ModulesProvider, fusionEnum *graphql.Enum, -) *graphql.ArgumentConfig { - prefix := fmt.Sprintf("GetObjects%s", class.Class) - return &graphql.ArgumentConfig{ - Type: graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sHybridInpObj", prefix), - Fields: hybridOperands(classObject, class, modulesProvider, fusionEnum), - Description: "Hybrid search", - }, - ), - } -} - -func hybridOperands(classObject *graphql.Object, - class *models.Class, modulesProvider ModulesProvider, fusionEnum *graphql.Enum, -) graphql.InputObjectConfigFieldMap { - ss := graphql.NewInputObject(graphql.InputObjectConfig{ - Name: class.Class + "SubSearch", - Fields: hybridSubSearch(classObject, class, modulesProvider), - }) - - fieldMap := graphql.InputObjectConfigFieldMap{ - "query": &graphql.InputObjectFieldConfig{ - Description: "Query string", - Type: graphql.String, - }, - "alpha": &graphql.InputObjectFieldConfig{ - Description: "Search weight", - Type: graphql.Float, - }, - "vector": &graphql.InputObjectFieldConfig{ - Description: "Vector search", - Type: graphql.NewList(graphql.Float), - }, - "properties": &graphql.InputObjectFieldConfig{ - Description: "Which properties should be included in the sparse search", - Type: graphql.NewList(graphql.String), - }, - "fusionType": &graphql.InputObjectFieldConfig{ - Description: "Algorithm used for fusing results from vector and keyword search", - Type: fusionEnum, - }, - } - - if os.Getenv("ENABLE_EXPERIMENTAL_HYBRID_OPERANDS") != "" { - fieldMap["operands"] = &graphql.InputObjectFieldConfig{ - Description: "Subsearch list", - Type: graphql.NewList(ss), - } - } - - return fieldMap -} - -func hybridSubSearch(classObject *graphql.Object, - class *models.Class, modulesProvider ModulesProvider, -) graphql.InputObjectConfigFieldMap { - prefixName := class.Class + "SubSearch" - - return graphql.InputObjectConfigFieldMap{ - "weight": &graphql.InputObjectFieldConfig{ - Description: "weight, 0 to 1", - Type: graphql.Float, - }, - "sparseSearch": &graphql.InputObjectFieldConfig{ - Description: "Sparse Search", - Type: graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sHybridGetBM25InpObj", prefixName), - Fields: bm25Fields(prefixName), - Description: "BM25f search", - }, - ), - }, - - "nearText": &graphql.InputObjectFieldConfig{ - Description: "nearText element", - - Type: graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sNearTextInpObj", prefixName), - Fields: nearTextFields(prefixName), - Description: descriptions.GetWhereInpObj, - }, - ), - }, - } -} diff --git a/adapters/handlers/graphql/local/get/models_for_test.go b/adapters/handlers/graphql/local/get/models_for_test.go deleted file mode 100644 index 57b0b3f4de57ce8dc2184f184e3aeccfc3be5ada..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/get/models_for_test.go +++ /dev/null @@ -1,48 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package get - -type FeatureProjection struct { - Vector []float32 `json:"vector"` -} - -type NearestNeighbors struct { - Neighbors []*NearestNeighbor `json:"neighbors"` -} - -type NearestNeighbor struct { - Concept string `json:"concept,omitempty"` - Distance float32 `json:"distance,omitempty"` - Vector []float32 `json:"vector"` -} - -type SemanticPath struct { - Path []*SemanticPathElement `json:"path"` -} - -type SemanticPathElement struct { - Concept string `json:"concept,omitempty"` - DistanceToNext *float32 `json:"distanceToNext,omitempty"` - DistanceToPrevious *float32 `json:"distanceToPrevious,omitempty"` - DistanceToQuery float32 `json:"distanceToQuery,omitempty"` - DistanceToResult float32 `json:"distanceToResult,omitempty"` -} - -type Interpretation struct { - Source []*InterpretationSource `json:"source"` -} - -type InterpretationSource struct { - Concept string `json:"concept,omitempty"` - Occurrence uint64 `json:"occurrence,omitempty"` - Weight float64 `json:"weight,omitempty"` -} diff --git a/adapters/handlers/graphql/local/get/multi_tenancy.go b/adapters/handlers/graphql/local/get/multi_tenancy.go deleted file mode 100644 index ea005c62e9c52244072871be9cb0a8de88d7e686..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/get/multi_tenancy.go +++ /dev/null @@ -1,24 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package get - -import ( - "github.com/tailor-inc/graphql" - "github.com/weaviate/weaviate/adapters/handlers/graphql/descriptions" -) - -func tenantArgument() *graphql.ArgumentConfig { - return &graphql.ArgumentConfig{ - Description: descriptions.Tenant, - Type: graphql.String, - } -} diff --git a/adapters/handlers/graphql/local/get/replication.go b/adapters/handlers/graphql/local/get/replication.go deleted file mode 100644 index 4718170500726cc4c28e6157720731508c928a53..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/get/replication.go +++ /dev/null @@ -1,39 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package get - -import ( - "fmt" - - "github.com/tailor-inc/graphql" - "github.com/weaviate/weaviate/adapters/handlers/graphql/descriptions" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/usecases/replica" -) - -func replicationEnabled(class *models.Class) bool { - return class.ReplicationConfig != nil && class.ReplicationConfig.Factor > 1 -} - -func consistencyLevelArgument(class *models.Class) *graphql.ArgumentConfig { - return &graphql.ArgumentConfig{ - Description: descriptions.ConsistencyLevel, - Type: graphql.NewEnum(graphql.EnumConfig{ - Name: fmt.Sprintf("%sConsistencyLevelEnum", class.Class), - Values: graphql.EnumValueConfigMap{ - string(replica.One): &graphql.EnumValueConfig{}, - string(replica.Quorum): &graphql.EnumValueConfig{}, - string(replica.All): &graphql.EnumValueConfig{}, - }, - }), - } -} diff --git a/adapters/handlers/graphql/local/get/resolver.go b/adapters/handlers/graphql/local/get/resolver.go deleted file mode 100644 index f5d69032f6de7196e46a1179e32753d9a9e202de..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/get/resolver.go +++ /dev/null @@ -1,30 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package get - -import ( - "context" - - "github.com/weaviate/weaviate/entities/dto" - "github.com/weaviate/weaviate/entities/models" -) - -// Resolver is a local abstraction of the required UC resolvers -type Resolver interface { - GetClass(ctx context.Context, principal *models.Principal, info dto.GetParams) ([]interface{}, error) -} - -// RequestsLog is a local abstraction on the RequestsLog that needs to be -// provided to the graphQL API in order to log Local.Get queries. -type RequestsLog interface { - Register(requestType string, identifier string) -} diff --git a/adapters/handlers/graphql/local/get/sort_argument.go b/adapters/handlers/graphql/local/get/sort_argument.go deleted file mode 100644 index de3dd5d45ace27da966e1c905c1fcec4caa9cf6b..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/get/sort_argument.go +++ /dev/null @@ -1,53 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package get - -import ( - "fmt" - - "github.com/tailor-inc/graphql" - "github.com/weaviate/weaviate/adapters/handlers/graphql/descriptions" -) - -func sortArgument(className string) *graphql.ArgumentConfig { - prefix := fmt.Sprintf("GetObjects%s", className) - return &graphql.ArgumentConfig{ - Type: graphql.NewList( - graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sSortInpObj", prefix), - Fields: sortFields(prefix), - Description: descriptions.GetWhereInpObj, - }, - ), - ), - } -} - -func sortFields(prefix string) graphql.InputObjectConfigFieldMap { - return graphql.InputObjectConfigFieldMap{ - "path": &graphql.InputObjectFieldConfig{ - Description: descriptions.SortPath, - Type: graphql.NewList(graphql.String), - }, - "order": &graphql.InputObjectFieldConfig{ - Description: descriptions.SortOrder, - Type: graphql.NewEnum(graphql.EnumConfig{ - Name: fmt.Sprintf("%sSortInpObjTypeEnum", prefix), - Values: graphql.EnumValueConfigMap{ - "asc": &graphql.EnumValueConfig{}, - "desc": &graphql.EnumValueConfig{}, - }, - }), - }, - } -} diff --git a/adapters/handlers/graphql/local/get/sparse_search.go b/adapters/handlers/graphql/local/get/sparse_search.go deleted file mode 100644 index dff56554cca9ae39ab91cd31e691b43f122addf1..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/get/sparse_search.go +++ /dev/null @@ -1,43 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package get - -import ( - "fmt" - - "github.com/tailor-inc/graphql" -) - -func bm25Argument(className string) *graphql.ArgumentConfig { - prefix := fmt.Sprintf("GetObjects%s", className) - return &graphql.ArgumentConfig{ - Type: graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sHybridGetBm25InpObj", prefix), - Fields: bm25Fields(prefix), - }, - ), - } -} - -func bm25Fields(prefix string) graphql.InputObjectConfigFieldMap { - return graphql.InputObjectConfigFieldMap{ - "query": &graphql.InputObjectFieldConfig{ - Description: "The query to search for", - Type: graphql.String, - }, - "properties": &graphql.InputObjectFieldConfig{ - Description: "The properties to search in", - Type: graphql.NewList(graphql.String), - }, - } -} diff --git a/adapters/handlers/graphql/local/local.go b/adapters/handlers/graphql/local/local.go deleted file mode 100644 index 9ffa4d4ffc4f5e45719d85b3cfc1e23a4e59c3b6..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/local.go +++ /dev/null @@ -1,57 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package local - -import ( - "github.com/sirupsen/logrus" - "github.com/tailor-inc/graphql" - "github.com/weaviate/weaviate/adapters/handlers/graphql/local/aggregate" - "github.com/weaviate/weaviate/adapters/handlers/graphql/local/explore" - "github.com/weaviate/weaviate/adapters/handlers/graphql/local/get" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/usecases/config" - "github.com/weaviate/weaviate/usecases/modules" -) - -// Build the local queries from the database schema. -func Build(dbSchema *schema.Schema, logger logrus.FieldLogger, - config config.Config, modulesProvider *modules.Provider, -) (graphql.Fields, error) { - getField, err := get.Build(dbSchema, logger, modulesProvider) - if err != nil { - return nil, err - } - - aggregateField, err := aggregate.Build(dbSchema, config, modulesProvider) - if err != nil { - return nil, err - } - - if modulesProvider.HasMultipleVectorizers() { - localFields := graphql.Fields{ - "Get": getField, - "Aggregate": aggregateField, - } - - return localFields, nil - } - - exploreField := explore.Build(dbSchema.Objects, modulesProvider) - - localFields := graphql.Fields{ - "Get": getField, - "Aggregate": aggregateField, - "Explore": exploreField, - } - - return localFields, nil -} diff --git a/adapters/handlers/graphql/local/local_component_test.go b/adapters/handlers/graphql/local/local_component_test.go deleted file mode 100644 index 360fc5e7422bffd426ea2df71c185c3138d35a64..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/local_component_test.go +++ /dev/null @@ -1,290 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package local - -import ( - "fmt" - "runtime/debug" - "testing" - - logrus "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/tailor-inc/graphql" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/usecases/config" - "github.com/weaviate/weaviate/usecases/modules" -) - -// These tests are component tests for the local package including all its -// subpackages, such as get, getmeta, etc.. However, they only assert that the -// graphql tree can be built under certain circumstances. This helps us to -// catch errors on edge cases like empty schemas, classes with empty -// properties, empty peer lists, peers with empty schemas, etc. However, we -// don't get any guarantee of whether the individual queries resolve -// correctly. For those cases we have unit tests in die individual subpackages -// (i.e. get, getmeta, aggregate, etc.). Additionally we have (a few) e2e -// tests. - -func TestBuild_GraphQLNetwork(t *testing.T) { - tests := testCases{ - // This tests asserts that an action-only schema doesn't lead to errors. - testCase{ - name: "with only objects locally", - localSchema: schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{ - { - Class: "BestLocalAction", - Properties: []*models.Property{ - { - DataType: schema.DataTypeText.PropString(), - Name: "myStringProp", - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - }, - }, - }, - }, - }, - - // This tests asserts that a things-only schema doesn't lead to errors. - testCase{ - name: "with only objects locally", - localSchema: schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{ - { - Class: "BestLocalThing", - Properties: []*models.Property{ - { - DataType: schema.DataTypeText.PropString(), - Name: "myStringProp", - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - }, - }, - }, - }, - }, - - // This tests asserts that a class without any properties doesn't lead to - // errors. - testCase{ - name: "with things without properties locally", - localSchema: schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{ - { - Class: "BestLocalThing", - Properties: []*models.Property{}, - }, - }, - }, - }, - }, - - testCase{ - name: "without any peers", - localSchema: validSchema(), - }, - } - - tests.AssertNoError(t) -} - -func TestBuild_RefProps(t *testing.T) { - t.Run("expected error logs", func(t *testing.T) { - tests := testCases{ - { - name: "build class with nonexistent ref prop", - localSchema: schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{ - { - Class: "ThisClassExists", - Properties: []*models.Property{ - { - DataType: []string{"ThisClassDoesNotExist"}, - Name: "ofNonexistentClass", - }, - }, - }, - }, - }, - }, - }, - } - - expectedLogMsg := "ignoring ref prop \"ofNonexistentClass\" on class \"ThisClassExists\", " + - "because it contains reference to nonexistent class [\"ThisClassDoesNotExist\"]" - - tests.AssertErrorLogs(t, expectedLogMsg) - }) - - t.Run("expected success", func(t *testing.T) { - tests := testCases{ - { - name: "build class with existing non-circular ref prop", - localSchema: schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{ - { - Class: "ThisClassExists", - Properties: []*models.Property{ - { - DataType: []string{"ThisClassAlsoExists"}, - Name: "ofExistingClass", - }, - }, - }, - { - Class: "ThisClassAlsoExists", - Properties: []*models.Property{ - { - DataType: schema.DataTypeText.PropString(), - Name: "stringProp", - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - }, - }, - }, - }, - }, - { - name: "build class with existing circular ref prop", - localSchema: schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{ - { - Class: "ThisClassExists", - Properties: []*models.Property{ - { - DataType: []string{"ThisClassAlsoExists"}, - Name: "ofExistingClass", - }, - }, - }, - { - Class: "ThisClassAlsoExists", - Properties: []*models.Property{ - { - DataType: []string{"ThisClassExists"}, - Name: "ofExistingClass", - }, - }, - }, - }, - }, - }, - }, - } - - tests.AssertNoError(t) - }) -} - -type testCase struct { - name string - localSchema schema.Schema -} - -type testCases []testCase - -func (tests testCases) AssertNoError(t *testing.T) { - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - modules := modules.NewProvider() - localSchema, err := Build(&test.localSchema, nil, config.Config{}, modules) - require.Nil(t, err, test.name) - - schemaObject := graphql.ObjectConfig{ - Name: "WeaviateObj", - Description: "Location of the root query", - Fields: localSchema, - } - - func() { - defer func() { - if r := recover(); r != nil { - err = fmt.Errorf("%v at %s", r, debug.Stack()) - } - }() - - _, err = graphql.NewSchema(graphql.SchemaConfig{ - Query: graphql.NewObject(schemaObject), - }) - }() - - assert.Nil(t, err, test.name) - }) - } -} - -// AssertErrorLogs still expects the test to pass without errors, -// but does expect the Build logger to contain errors messages -// from the GQL schema rebuilding thunk -func (tests testCases) AssertErrorLogs(t *testing.T, expectedMsg string) { - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - modules := modules.NewProvider() - logger, logsHook := logrus.NewNullLogger() - localSchema, err := Build(&test.localSchema, logger, config.Config{}, modules) - require.Nil(t, err, test.name) - - schemaObject := graphql.ObjectConfig{ - Name: "WeaviateObj", - Description: "Location of the root query", - Fields: localSchema, - } - - func() { - defer func() { - if r := recover(); r != nil { - err = fmt.Errorf("%v at %s", r, debug.Stack()) - } - }() - - _, err = graphql.NewSchema(graphql.SchemaConfig{ - Query: graphql.NewObject(schemaObject), - }) - }() - - last := logsHook.LastEntry() - assert.Contains(t, last.Message, expectedMsg) - assert.Nil(t, err) - }) - } -} - -func validSchema() schema.Schema { - return schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{ - { - Class: "BestLocalThing", - Properties: []*models.Property{ - { - DataType: schema.DataTypeText.PropString(), - Name: "myStringProp", - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - }, - }, - }, - } -} diff --git a/adapters/handlers/graphql/local/resolver.go b/adapters/handlers/graphql/local/resolver.go deleted file mode 100644 index 5c8479f4770d5abc32280bce656cf04282410394..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/local/resolver.go +++ /dev/null @@ -1,23 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package local - -import ( - "github.com/weaviate/weaviate/adapters/handlers/graphql/local/aggregate" - get "github.com/weaviate/weaviate/adapters/handlers/graphql/local/get" -) - -// Resolver for local GraphQL queries -type Resolver interface { - get.Resolver - aggregate.Resolver -} diff --git a/adapters/handlers/graphql/schema.go b/adapters/handlers/graphql/schema.go deleted file mode 100644 index f08c186680eb2522bcc15c4ff4410726701bea88..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/schema.go +++ /dev/null @@ -1,119 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Package graphql provides the graphql endpoint for Weaviate -package graphql - -import ( - "context" - "fmt" - "runtime/debug" - - "github.com/sirupsen/logrus" - "github.com/tailor-inc/graphql" - "github.com/weaviate/weaviate/adapters/handlers/graphql/local" - "github.com/weaviate/weaviate/adapters/handlers/graphql/local/get" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/usecases/config" - "github.com/weaviate/weaviate/usecases/modules" -) - -type Traverser interface { - local.Resolver -} - -type RequestsLogger interface { - get.RequestsLog -} - -// The communication interface between the REST API and the GraphQL API. -type GraphQL interface { - // Resolve the GraphQL query in 'query'. - Resolve(context context.Context, query string, operationName string, variables map[string]interface{}) *graphql.Result -} - -type graphQL struct { - schema graphql.Schema - traverser Traverser - config config.Config -} - -// Construct a GraphQL API from the database schema, and resolver interface. -func Build(schema *schema.Schema, traverser Traverser, - logger logrus.FieldLogger, config config.Config, modulesProvider *modules.Provider, -) (GraphQL, error) { - logger.WithField("action", "graphql_rebuild"). - WithField("schema", schema). - Debug("rebuilding the graphql schema") - - graphqlSchema, err := buildGraphqlSchema(schema, logger, config, modulesProvider) - if err != nil { - return nil, err - } - - return &graphQL{ - schema: graphqlSchema, - traverser: traverser, - config: config, - }, nil -} - -// Resolve at query time -func (g *graphQL) Resolve(context context.Context, query string, operationName string, variables map[string]interface{}) *graphql.Result { - return graphql.Do(graphql.Params{ - Schema: g.schema, - RootObject: map[string]interface{}{ - "Resolver": g.traverser, - "Config": g.config, - }, - RequestString: query, - OperationName: operationName, - VariableValues: variables, - Context: context, - }) -} - -func buildGraphqlSchema(dbSchema *schema.Schema, logger logrus.FieldLogger, - config config.Config, modulesProvider *modules.Provider, -) (graphql.Schema, error) { - localSchema, err := local.Build(dbSchema, logger, config, modulesProvider) - if err != nil { - return graphql.Schema{}, err - } - - schemaObject := graphql.ObjectConfig{ - Name: "WeaviateObj", - Description: "Location of the root query", - Fields: localSchema, - } - - // Run graphql.NewSchema in a sub-closure, so that we can recover from panics. - // We need to use panics to return errors deep inside the dynamic generation of the GraphQL schema, - // inside the FieldThunks. There is _no_ way to bubble up an error besides panicking. - var result graphql.Schema - func() { - defer func() { - if r := recover(); r != nil { - err = fmt.Errorf("%v at %s", r, debug.Stack()) - } - }() - - result, err = graphql.NewSchema(graphql.SchemaConfig{ - Query: graphql.NewObject(schemaObject), - }) - }() - - if err != nil { - return graphql.Schema{}, fmt.Errorf("Could not build GraphQL schema, because: %v", err) - } - - return result, nil -} diff --git a/adapters/handlers/graphql/test/helper/mock_resolver.go b/adapters/handlers/graphql/test/helper/mock_resolver.go deleted file mode 100644 index d7545528118e61aa90dc9c87cd9d912e2bf61682..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/test/helper/mock_resolver.go +++ /dev/null @@ -1,133 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package helper - -import ( - "context" - "encoding/json" - "fmt" - "sync" - "testing" - - "github.com/davecgh/go-spew/spew" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" - "github.com/tailor-inc/graphql" - "github.com/tailor-inc/graphql/gqlerrors" - "github.com/weaviate/weaviate/entities/schema" -) - -type MockResolver struct { - mock.Mock - Schema *schema.Schema - RootField *graphql.Field - RootFieldName string - RootObject map[string]interface{} -} - -var schemaBuildLock sync.Mutex - -func (mr *MockResolver) Resolve(query string) *graphql.Result { - fields := graphql.Fields{} - fields[mr.RootFieldName] = mr.RootField - schemaObject := graphql.ObjectConfig{ - Name: "RootObj", - Description: "Location of the root query", - Fields: fields, - } - - schemaBuildLock.Lock() // serialize creation of GraphQL schema. - schema, err := graphql.NewSchema(graphql.SchemaConfig{ - Query: graphql.NewObject(schemaObject), - }) - schemaBuildLock.Unlock() - - if err != nil { - panic(err) - } - - result := graphql.Do(graphql.Params{ - Schema: schema, - RequestString: query, - RootObject: mr.RootObject, - Context: context.Background(), - }) - - return result -} - -func (mr *MockResolver) AssertResolve(t *testing.T, query string) *GraphQLResult { - result := mr.Resolve(query) - if len(result.Errors) > 0 { - t.Fatalf("Failed to resolve; %s", spew.Sdump(result.Errors)) - } - - mr.AssertExpectations(t) - return &GraphQLResult{Result: result.Data} -} - -func (mr *MockResolver) AssertFailToResolve(t *testing.T, query string, errors ...string) { - result := mr.Resolve(query) - if len(result.Errors) == 0 { - t.Fatalf("Expected to not resolve; %#v", result.Errors) - } else { - t.Log("Resolve failed, as expected, with error", result.Errors) - } - if len(errors) > 0 { - require.Equal(t, errors[0], result.Errors[0].Error()) - } -} - -func (mr *MockResolver) AssertErrors(t *testing.T, query string, errors []gqlerrors.FormattedError) { - result := mr.Resolve(query) - for i, actual := range result.Errors { - assert.Equal(t, errors[i].Error(), actual.Error(), "should have failed in a specific way, but didnt") - } -} - -func (mr *MockResolver) AssertJSONResponse(t *testing.T, query string, expectedResponseString string) { - var expectedResponse map[string]interface{} - err := json.Unmarshal([]byte(expectedResponseString), &expectedResponse) - if err != nil { - t.Fatalf("Could not parse '%s' as json: %v", expectedResponseString, err) - } - - response := mr.AssertResolve(t, query) - - assert.Equal(t, expectedResponse, response) -} - -type GraphQLResult struct { - Result interface{} -} - -// Drill down in the result -func (g GraphQLResult) Get(paths ...string) *GraphQLResult { - current := g.Result - for _, path := range paths { - var ok bool - currentAsMap, ok := (current.(map[string]interface{})) - if !ok { - panic(fmt.Sprintf("Cannot get element %s in %#v; result: %#v", path, paths, g.Result)) - } - - current, ok = currentAsMap[path] - if !ok { - panic(fmt.Sprintf("Cannot get element %s in %#v; result: %#v", path, paths, g.Result)) - } - } - - return &GraphQLResult{ - Result: current, - } -} diff --git a/adapters/handlers/graphql/test/helper/schema_fixtures.go b/adapters/handlers/graphql/test/helper/schema_fixtures.go deleted file mode 100644 index 8481bc0ada939ca330b4329a97715cf58dd8d811..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/test/helper/schema_fixtures.go +++ /dev/null @@ -1,131 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package helper - -import ( - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/usecases/config" -) - -var SimpleSchema = CreateSimpleSchema(config.VectorizerModuleText2VecContextionary) - -func CreateSimpleSchema(vectorizer string) schema.Schema { - return schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{ - { - Class: "SomeThing", - Vectorizer: vectorizer, - Properties: []*models.Property{ - { - Name: "intField", - DataType: []string{"int"}, - }, - }, - }, - { - Class: "CustomVectorClass", - Vectorizer: config.VectorizerModuleNone, - Properties: []*models.Property{ - { - Name: "intField", - DataType: []string{"int"}, - }, - }, - }, - { - Vectorizer: vectorizer, - Class: "SomeAction", - Properties: []*models.Property{ - { - Name: "intField", - DataType: []string{"int"}, - }, - { - Name: "uuidField", - DataType: []string{"uuid"}, - }, - { - Name: "uuidArrayField", - DataType: []string{"uuid[]"}, - }, - { - Name: "location", - DataType: []string{"geoCoordinates"}, - }, - { - Name: "phone", - DataType: []string{"phoneNumber"}, - }, - { - Name: "hasAction", - DataType: []string{"SomeAction"}, - }, - { - Name: "hasActions", - DataType: []string{"SomeAction"}, - }, - }, - }, - }, - }, - } -} - -// CarSchema contains a car which has every primitive field and a ref field there is -var CarSchema = schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{ - { - Class: "Manufacturer", - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - }, - { - Class: "Car", - Properties: []*models.Property{ - { - Name: "horsepower", - DataType: []string{"int"}, - }, - { - Name: "weight", - DataType: []string{"number"}, - }, - { - Name: "modelName", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "madeBy", - DataType: []string{"Manufacturer"}, - }, - { - Name: "startOfProduction", - DataType: []string{"date"}, - }, - { - Name: "stillInProduction", - DataType: []string{"boolean"}, - }, - }, - }, - }, - }, -} diff --git a/adapters/handlers/graphql/test/helper/thunks.go b/adapters/handlers/graphql/test/helper/thunks.go deleted file mode 100644 index c1e6a6189bf087926435490b8069539497e2408a..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/test/helper/thunks.go +++ /dev/null @@ -1,41 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package helper - -func NilThunk() func() interface{} { - return func() interface{} { - return nil - } -} - -func IdentityThunk(x interface{}) func() interface{} { - return func() interface{} { - return x - } -} - -func EmptyListThunk() func() interface{} { - return func() interface{} { - list := []interface{}{} - return interface{}(list) - } -} - -func EmptyList() interface{} { - return []interface{}{} -} - -func SingletonThunk(x interface{}) func() interface{} { - return func() interface{} { - return interface{}(x) - } -} diff --git a/adapters/handlers/graphql/utils/helper_objects.go b/adapters/handlers/graphql/utils/helper_objects.go deleted file mode 100644 index aad80cb289f9c3dfb4ee54355c12b6b9f0fdd33e..0000000000000000000000000000000000000000 --- a/adapters/handlers/graphql/utils/helper_objects.go +++ /dev/null @@ -1,42 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Package utils provides utility methods and classes to support the graphql endpoint for Weaviate -package utils - -import ( - "errors" - - "github.com/tailor-inc/graphql" -) - -// GraphQLNetworkFieldContents contains all objects regarding GraphQL fields -type GraphQLNetworkFieldContents struct { - NetworkGetObject *graphql.Object // Object containing all fields for GraphQL Network Get schema generation - NetworkMetaObject *graphql.Object // Object containing all fields for GraphQL Network Meta schema generation - NetworkFetchObject *graphql.Object // Object containing all fields for GraphQL Network Fetch schema generation - NetworkIntrospectObject *graphql.Object // Object containing all fields for GraphQL Network Introspect schema generation - NetworkAggregateObject *graphql.Object // Object containing all fields for GraphQL Network Aggregate schema generation -} - -// FilterContainer contains all objects regarding GraphQL filters. Some filter elements are declared as global variables in the prototype, this struct achieves the same goal. -type FilterContainer struct { - WhereOperatorEnum *graphql.Enum // Object containing all fields for the Where filter - Operands *graphql.InputObject // Object containing all Operands - LocalFilterOptions map[string]*graphql.InputObject // Object containing all fields for Local filters - NetworkFilterOptions map[string]*graphql.InputObject // Object containing all fields for Network filters - FetchThingsActionsWhereFilterArgConf *graphql.ArgumentConfig // Object containing the Where filter fields for Fetch Objects - IntrospectThingsActionsWhereFilterArgConf *graphql.ArgumentConfig // Object containing the Where filter fields for Introspect Objects - WeaviateNetworkWhereKeywordsInpObj *graphql.InputObject // Object containing a global filter element - WeaviateNetworkIntrospectPropertiesObjField *graphql.Field // Object containing a global filter element -} - -var ErrEmptySchema = errors.New("there are no classes defined yet") diff --git a/adapters/handlers/grpc/server.go b/adapters/handlers/grpc/server.go deleted file mode 100644 index 1e9765eed814f9071bbb3df2a2a41482eb4e0293..0000000000000000000000000000000000000000 --- a/adapters/handlers/grpc/server.go +++ /dev/null @@ -1,85 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package grpc - -import ( - "fmt" - "net" - - "github.com/weaviate/weaviate/adapters/handlers/rest/state" - pbv0 "github.com/weaviate/weaviate/grpc/generated/protocol/v0" - pbv1 "github.com/weaviate/weaviate/grpc/generated/protocol/v1" - "github.com/weaviate/weaviate/usecases/auth/authentication/composer" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials" - _ "google.golang.org/grpc/encoding/gzip" // Install the gzip compressor - "google.golang.org/grpc/health/grpc_health_v1" - - v0 "github.com/weaviate/weaviate/adapters/handlers/grpc/v0" - v1 "github.com/weaviate/weaviate/adapters/handlers/grpc/v1" -) - -const maxMsgSize = 104858000 // 10mb, needs to be synchronized with clients - -func CreateGRPCServer(state *state.State) *GRPCServer { - o := []grpc.ServerOption{ - grpc.MaxRecvMsgSize(maxMsgSize), - grpc.MaxSendMsgSize(maxMsgSize), - } - - // Add TLS creds for the GRPC connection, if defined. - if len(state.ServerConfig.Config.GRPC.CertFile) > 0 || len(state.ServerConfig.Config.GRPC.KeyFile) > 0 { - c, err := credentials.NewServerTLSFromFile(state.ServerConfig.Config.GRPC.CertFile, - state.ServerConfig.Config.GRPC.KeyFile) - if err != nil { - state.Logger.WithField("action", "grpc_startup"). - Fatalf("grpc server TLS credential error: %s", err) - } - o = append(o, grpc.Creds(c)) - } - - s := grpc.NewServer(o...) - weaviateV0 := v0.NewService() - weaviateV1 := v1.NewService( - state.Traverser, - composer.New( - state.ServerConfig.Config.Authentication, - state.APIKey, state.OIDC), - state.ServerConfig.Config.Authentication.AnonymousAccess.Enabled, - state.SchemaManager, - state.BatchManager, - ) - pbv0.RegisterWeaviateServer(s, weaviateV0) - pbv1.RegisterWeaviateServer(s, weaviateV1) - grpc_health_v1.RegisterHealthServer(s, weaviateV1) - - return &GRPCServer{s} -} - -func StartAndListen(s *GRPCServer, state *state.State) error { - lis, err := net.Listen("tcp", fmt.Sprintf(":%d", - state.ServerConfig.Config.GRPC.Port)) - if err != nil { - return err - } - state.Logger.WithField("action", "grpc_startup"). - Infof("grpc server listening at %v", lis.Addr()) - if err := s.Serve(lis); err != nil { - return fmt.Errorf("failed to serve: %v", err) - } - - return nil -} - -type GRPCServer struct { - *grpc.Server -} diff --git a/adapters/handlers/grpc/v0/service.go b/adapters/handlers/grpc/v0/service.go deleted file mode 100644 index edaf12b32a7f05bfca588411128615e80ee72645..0000000000000000000000000000000000000000 --- a/adapters/handlers/grpc/v0/service.go +++ /dev/null @@ -1,40 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package weaviategrpc - -import ( - "context" - - "github.com/pkg/errors" - - pb "github.com/weaviate/weaviate/grpc/generated/protocol/v0" -) - -type Service struct { - pb.UnimplementedWeaviateServer -} - -func NewService() *Service { - return &Service{} -} - -func (s *Service) BatchObjects(ctx context.Context, req *pb.BatchObjectsRequest) (*pb.BatchObjectsReply, error) { - return nil, errors.New( - "The V0 gRPC API is deprecated and will be removed in the next major release. Please use the V1 API instead by upgrading your client.", - ) -} - -func (s *Service) Search(ctx context.Context, req *pb.SearchRequest) (*pb.SearchReply, error) { - return nil, errors.New( - "The V0 gRPC API is deprecated and will be removed in the next major release. Please use the V1 API instead by upgrading your client.", - ) -} diff --git a/adapters/handlers/grpc/v1/auth.go b/adapters/handlers/grpc/v1/auth.go deleted file mode 100644 index bb80098eccbd48e19f698dc678d9fa2686af713f..0000000000000000000000000000000000000000 --- a/adapters/handlers/grpc/v1/auth.go +++ /dev/null @@ -1,58 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package v1 - -import ( - "context" - "strings" - - "github.com/weaviate/weaviate/entities/models" - "google.golang.org/grpc/metadata" -) - -// This should probably be run as part of a middleware. In the initial gRPC -// implementation there is only a single endpoint, so it's fine to run this -// straight from the endpoint. But the moment we add a second endpoint, this -// should be called from a central place. This way we can make sure it's -// impossible to forget to add it to a new endpoint. -func (s *Service) principalFromContext(ctx context.Context) (*models.Principal, error) { - md, ok := metadata.FromIncomingContext(ctx) - if !ok { - return s.tryAnonymous() - } - - // the grpc library will lowercase all md keys, so we need to make sure to - // check a lowercase key - authValue, ok := md["authorization"] - if !ok { - return s.tryAnonymous() - } - - if len(authValue) == 0 { - return s.tryAnonymous() - } - - if !strings.HasPrefix(authValue[0], "Bearer ") { - return s.tryAnonymous() - } - - token := strings.TrimPrefix(authValue[0], "Bearer ") - return s.authComposer(token, nil) -} - -func (s *Service) tryAnonymous() (*models.Principal, error) { - if s.allowAnonymousAccess { - return nil, nil - } - - return s.authComposer("", nil) -} diff --git a/adapters/handlers/grpc/v1/auth_test.go b/adapters/handlers/grpc/v1/auth_test.go deleted file mode 100644 index ce5a0d93daae03a688060fc5f38b1f1bf54ccdac..0000000000000000000000000000000000000000 --- a/adapters/handlers/grpc/v1/auth_test.go +++ /dev/null @@ -1,128 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package v1 - -import ( - "context" - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" - "google.golang.org/grpc/metadata" -) - -func TestAuth(t *testing.T) { - tests := []struct { - name string - buildCtx func() context.Context - shouldErr bool - expectedOut *models.Principal - allowAnon bool - }{ - { - name: "nothing provided, anon allowed", - buildCtx: func() context.Context { - return context.Background() - }, - allowAnon: true, - shouldErr: false, - }, - { - name: "nothing provided, anon forbidden", - buildCtx: func() context.Context { - return context.Background() - }, - allowAnon: false, - shouldErr: true, - }, - { - name: "with md, but nothing usable, anon allowed", - buildCtx: func() context.Context { - md := metadata.Pairs("unrelated", "unrelated") - return metadata.NewIncomingContext(context.Background(), md) - }, - allowAnon: true, - shouldErr: false, - }, - { - name: "with md, but nothing usable, anon forbidden", - buildCtx: func() context.Context { - md := metadata.Pairs("unrelated", "unrelated") - return metadata.NewIncomingContext(context.Background(), md) - }, - allowAnon: false, - shouldErr: true, - }, - { - name: "with md, but nothing usable, anon allowed", - buildCtx: func() context.Context { - md := metadata.Pairs("authorization", "wrong-format") - return metadata.NewIncomingContext(context.Background(), md) - }, - allowAnon: true, - shouldErr: false, - }, - { - name: "with md, but nothing usable, anon forbidden", - buildCtx: func() context.Context { - md := metadata.Pairs("authorization", "wrong-format") - return metadata.NewIncomingContext(context.Background(), md) - }, - allowAnon: false, - shouldErr: true, - }, - { - name: "with md, and a token", - buildCtx: func() context.Context { - md := metadata.Pairs("authorization", "Bearer Foo") - return metadata.NewIncomingContext(context.Background(), md) - }, - shouldErr: false, - expectedOut: &models.Principal{Username: "Foo"}, - }, - { - name: "with a token that makes extraction error", - buildCtx: func() context.Context { - md := metadata.Pairs("authorization", "Bearer err") - return metadata.NewIncomingContext(context.Background(), md) - }, - shouldErr: true, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - s := &Service{ - allowAnonymousAccess: test.allowAnon, - authComposer: func(token string, scopes []string) (*models.Principal, error) { - if token == "" { - return nil, fmt.Errorf("not allowed") - } - if token == "err" { - return nil, fmt.Errorf("other error") - } - return &models.Principal{Username: token}, nil - }, - } - - p, err := s.principalFromContext(test.buildCtx()) - if test.shouldErr { - require.NotNil(t, err) - } else { - require.Nil(t, err) - assert.Equal(t, test.expectedOut, p) - } - }) - } -} diff --git a/adapters/handlers/grpc/v1/batch_delete.go b/adapters/handlers/grpc/v1/batch_delete.go deleted file mode 100644 index 1785967b0d3a9c81aa579a6802adad2958693bfd..0000000000000000000000000000000000000000 --- a/adapters/handlers/grpc/v1/batch_delete.go +++ /dev/null @@ -1,100 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package v1 - -import ( - "fmt" - "math/big" - "strings" - - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/schema" - pb "github.com/weaviate/weaviate/grpc/generated/protocol/v1" - "github.com/weaviate/weaviate/usecases/objects" -) - -func batchDeleteParamsFromProto(req *pb.BatchDeleteRequest, scheme schema.Schema) (objects.BatchDeleteParams, error) { - params := objects.BatchDeleteParams{} - - // make sure collection exists - _, err := schema.GetClassByName(scheme.Objects, req.Collection) - if err != nil { - return objects.BatchDeleteParams{}, err - } - - params.ClassName = schema.ClassName(req.Collection) - - if req.Verbose { - params.Output = "verbose" - } else { - params.Output = "minimal" - } - - params.DryRun = req.DryRun - - if req.Filters == nil { - return objects.BatchDeleteParams{}, fmt.Errorf("no filters in batch delete request") - } - - clause, err := extractFilters(req.Filters, scheme, req.Collection) - if err != nil { - return objects.BatchDeleteParams{}, err - } - filter := &filters.LocalFilter{Root: &clause} - if err := filters.ValidateFilters(scheme, filter); err != nil { - return objects.BatchDeleteParams{}, err - } - params.Filters = filter - - return params, nil -} - -func batchDeleteReplyFromObjects(response objects.BatchDeleteResult, verbose bool) (*pb.BatchDeleteReply, error) { - var successful, failed int64 - - var objs []*pb.BatchDeleteObject - if verbose { - objs = make([]*pb.BatchDeleteObject, 0, len(response.Objects)) - } - for _, obj := range response.Objects { - if obj.Err == nil { - successful += 1 - } else { - failed += 1 - } - if verbose { - hexInteger, success := new(big.Int).SetString(strings.Replace(obj.UUID.String(), "-", "", -1), 16) - if !success { - return nil, fmt.Errorf("failed to parse hex string to integer") - } - errorString := "" - if obj.Err != nil { - errorString = obj.Err.Error() - } - - resultObj := &pb.BatchDeleteObject{ - Uuid: hexInteger.Bytes(), - Successful: obj.Err == nil, - Error: &errorString, - } - objs = append(objs, resultObj) - } - } - reply := &pb.BatchDeleteReply{ - Successful: successful, - Failed: failed, - Matches: response.Matches, - Objects: objs, - } - - return reply, nil -} diff --git a/adapters/handlers/grpc/v1/batch_delete_test.go b/adapters/handlers/grpc/v1/batch_delete_test.go deleted file mode 100644 index 13953f312446c241ce9452d31dc59bc136805b57..0000000000000000000000000000000000000000 --- a/adapters/handlers/grpc/v1/batch_delete_test.go +++ /dev/null @@ -1,171 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package v1 - -import ( - "errors" - "fmt" - "testing" - - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - pb "github.com/weaviate/weaviate/grpc/generated/protocol/v1" - "github.com/weaviate/weaviate/usecases/objects" -) - -func TestBatchDeleteRequest(t *testing.T) { - collection := "TestClass" - scheme := schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{ - { - Class: collection, - Properties: []*models.Property{ - {Name: "name", DataType: schema.DataTypeText.PropString()}, - }, - }, - }, - }, - } - - simpleFilterOutput := &filters.LocalFilter{ - Root: &filters.Clause{ - On: &filters.Path{Class: schema.ClassName(collection), Property: "name"}, - Operator: filters.OperatorEqual, - Value: &filters.Value{Value: "test", Type: schema.DataTypeText}, - }, - } - simpleFilterInput := &pb.Filters{Operator: pb.Filters_OPERATOR_EQUAL, TestValue: &pb.Filters_ValueText{ValueText: "test"}, Target: &pb.FilterTarget{Target: &pb.FilterTarget_Property{Property: "name"}}} - - tests := []struct { - name string - req *pb.BatchDeleteRequest - out objects.BatchDeleteParams - error error - }{ - { - name: "simple filter", - req: &pb.BatchDeleteRequest{ - Collection: collection, - Filters: simpleFilterInput, - }, - out: objects.BatchDeleteParams{ - ClassName: schema.ClassName(collection), - DryRun: false, - Output: "minimal", - Filters: simpleFilterOutput, - }, - error: nil, - }, - { - name: "collection does not exist", - req: &pb.BatchDeleteRequest{Collection: "does not exist"}, - error: errors.New("no such class with name 'does not exist' found in the schema. Check your schema files for which classes are available"), - }, - { - name: "no filter", - req: &pb.BatchDeleteRequest{Collection: collection}, - error: fmt.Errorf("no filters in batch delete request"), - }, - { - name: "dry run", - req: &pb.BatchDeleteRequest{ - Collection: collection, - Filters: simpleFilterInput, - DryRun: true, - }, - out: objects.BatchDeleteParams{ - ClassName: schema.ClassName(collection), - DryRun: true, - Output: "minimal", - Filters: simpleFilterOutput, - }, - error: nil, - }, - { - name: "verbose", - req: &pb.BatchDeleteRequest{ - Collection: collection, - Filters: simpleFilterInput, - DryRun: false, - Verbose: true, - }, - out: objects.BatchDeleteParams{ - ClassName: schema.ClassName(collection), - DryRun: false, - Output: "verbose", - Filters: simpleFilterOutput, - }, - error: nil, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - out, err := batchDeleteParamsFromProto(tt.req, scheme) - require.Equal(t, tt.error, err) - - if tt.error == nil { - require.Equal(t, tt.out, out) - } - }) - } -} - -var ( - errorString = "error" - noErrorString = "" -) - -func TestBatchDeleteReply(t *testing.T) { - tests := []struct { - name string - response objects.BatchDeleteResult - verbose bool - out *pb.BatchDeleteReply - }{ - { - name: "single object", - response: objects.BatchDeleteResult{Matches: 1, Objects: objects.BatchSimpleObjects{{UUID: UUID1, Err: nil}}}, - out: &pb.BatchDeleteReply{Matches: 1, Successful: 1, Failed: 0}, - }, - { - name: "single object with err", - response: objects.BatchDeleteResult{Matches: 1, Objects: objects.BatchSimpleObjects{{UUID: UUID1, Err: errors.New("error")}}}, - out: &pb.BatchDeleteReply{Matches: 1, Successful: 0, Failed: 1}, - }, - { - name: "one error, one successful", - response: objects.BatchDeleteResult{Matches: 2, Objects: objects.BatchSimpleObjects{{UUID: UUID1, Err: errors.New("error")}, {UUID: UUID2, Err: nil}}}, - out: &pb.BatchDeleteReply{Matches: 2, Successful: 1, Failed: 1}, - }, - { - name: "one error, one successful - with verbosity", - response: objects.BatchDeleteResult{Matches: 2, Objects: objects.BatchSimpleObjects{{UUID: UUID1, Err: errors.New("error")}, {UUID: UUID2, Err: nil}}}, - verbose: true, - out: &pb.BatchDeleteReply{Matches: 2, Successful: 1, Failed: 1, Objects: []*pb.BatchDeleteObject{ - {Uuid: idByte(string(UUID1)), Successful: false, Error: &errorString}, - {Uuid: idByte(string(UUID2)), Successful: true, Error: &noErrorString}, - }}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - out, err := batchDeleteReplyFromObjects(tt.response, tt.verbose) - require.Nil(t, err) - require.Equal(t, tt.out, out) - }) - } -} diff --git a/adapters/handlers/grpc/v1/batch_parse_request.go b/adapters/handlers/grpc/v1/batch_parse_request.go deleted file mode 100644 index 4ef9516537eb47a1c4331f8846652f3f2f52f18d..0000000000000000000000000000000000000000 --- a/adapters/handlers/grpc/v1/batch_parse_request.go +++ /dev/null @@ -1,195 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package v1 - -import ( - "fmt" - - "github.com/go-openapi/strfmt" - "github.com/google/uuid" - "github.com/weaviate/weaviate/usecases/byteops" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - pb "github.com/weaviate/weaviate/grpc/generated/protocol/v1" -) - -const BEACON_START = "weaviate://localhost/" - -func sliceToInterface[T any](values []T) []interface{} { - tmpArray := make([]interface{}, len(values)) - for k := range values { - tmpArray[k] = values[k] - } - return tmpArray -} - -func batchFromProto(req *pb.BatchObjectsRequest, scheme schema.Schema) ([]*models.Object, map[int]int, map[int]error) { - objectsBatch := req.Objects - objs := make([]*models.Object, 0, len(objectsBatch)) - objOriginalIndex := make(map[int]int) - objectErrors := make(map[int]error, len(objectsBatch)) - - insertCounter := 0 - for i, obj := range objectsBatch { - class := scheme.GetClass(schema.ClassName(obj.Collection)) - var props map[string]interface{} - if obj.Properties != nil { - props = extractPrimitiveProperties(&pb.ObjectPropertiesValue{ - NonRefProperties: obj.Properties.NonRefProperties, - BooleanArrayProperties: obj.Properties.BooleanArrayProperties, - NumberArrayProperties: obj.Properties.NumberArrayProperties, - TextArrayProperties: obj.Properties.TextArrayProperties, - IntArrayProperties: obj.Properties.IntArrayProperties, - ObjectProperties: obj.Properties.ObjectProperties, - ObjectArrayProperties: obj.Properties.ObjectArrayProperties, - }) - if err := extractSingleRefTarget(class, obj.Properties.SingleTargetRefProps, props); err != nil { - objectErrors[i] = err - continue - } - if err := extractMultiRefTarget(class, obj.Properties.MultiTargetRefProps, props); err != nil { - objectErrors[i] = err - continue - } - } - - _, err := uuid.Parse(obj.Uuid) - if err != nil { - objectErrors[i] = err - continue - } - - var vector []float32 - // bytes vector has precedent for being more efficient - if len(obj.VectorBytes) > 0 { - vector = byteops.Float32FromByteVector(obj.VectorBytes) - } else if len(obj.Vector) > 0 { - vector = obj.Vector - } - - objOriginalIndex[insertCounter] = i - objs = append(objs, &models.Object{ - Class: obj.Collection, - Tenant: obj.Tenant, - Vector: vector, - Properties: props, - ID: strfmt.UUID(obj.Uuid), - }) - insertCounter += 1 - } - return objs[:insertCounter], objOriginalIndex, objectErrors -} - -func extractSingleRefTarget(class *models.Class, properties []*pb.BatchObject_SingleTargetRefProps, props map[string]interface{}) error { - for _, refSingle := range properties { - propName := refSingle.GetPropName() - prop, err := schema.GetPropertyByName(class, propName) - if err != nil { - return err - } - if len(prop.DataType) > 1 { - return fmt.Errorf("target is a multi-target reference, need single target %v", prop.DataType) - } - toClass := prop.DataType[0] - beacons := make([]interface{}, len(refSingle.Uuids)) - for j, uuid := range refSingle.Uuids { - beacons[j] = map[string]interface{}{"beacon": BEACON_START + toClass + "/" + uuid} - } - props[propName] = beacons - } - return nil -} - -func extractMultiRefTarget(class *models.Class, properties []*pb.BatchObject_MultiTargetRefProps, props map[string]interface{}) error { - for _, refMulti := range properties { - propName := refMulti.GetPropName() - prop, err := schema.GetPropertyByName(class, propName) - if err != nil { - return err - } - if len(prop.DataType) < 2 { - return fmt.Errorf("target is a single-target reference, need multi-target %v", prop.DataType) - } - beacons := make([]interface{}, len(refMulti.Uuids)) - for j, uuid := range refMulti.Uuids { - beacons[j] = map[string]interface{}{"beacon": BEACON_START + refMulti.TargetCollection + "/" + uuid} - } - props[propName] = beacons - } - return nil -} - -func extractPrimitiveProperties(properties *pb.ObjectPropertiesValue) map[string]interface{} { - var props map[string]interface{} - if properties.NonRefProperties != nil { - props = properties.NonRefProperties.AsMap() - } else { - props = make(map[string]interface{}) - } - - // arrays cannot be part of a GRPC map, so we need to handle each type separately - if properties.BooleanArrayProperties != nil { - for j := range properties.BooleanArrayProperties { - props[properties.BooleanArrayProperties[j].PropName] = sliceToInterface(properties.BooleanArrayProperties[j].Values) - } - } - - if properties.NumberArrayProperties != nil { - for j := range properties.NumberArrayProperties { - inputValuesBytes := properties.NumberArrayProperties[j].ValuesBytes - var values []float64 - - if len(inputValuesBytes) > 0 { - values = byteops.Float64FromByteVector(inputValuesBytes) - } else { - values = properties.NumberArrayProperties[j].Values - } - - props[properties.NumberArrayProperties[j].PropName] = sliceToInterface(values) - } - } - - if properties.TextArrayProperties != nil { - for j := range properties.TextArrayProperties { - props[properties.TextArrayProperties[j].PropName] = sliceToInterface(properties.TextArrayProperties[j].Values) - } - } - - if properties.IntArrayProperties != nil { - for j := range properties.IntArrayProperties { - props[properties.IntArrayProperties[j].PropName] = sliceToInterface(properties.IntArrayProperties[j].Values) - } - } - - if properties.ObjectProperties != nil { - for j := range properties.ObjectProperties { - props[properties.ObjectProperties[j].PropName] = extractPrimitiveProperties(properties.ObjectProperties[j].Value) - } - } - - if properties.ObjectArrayProperties != nil { - extractObjectArray(properties.ObjectArrayProperties, props) - } - - return props -} - -func extractObjectArray(propsArr []*pb.ObjectArrayProperties, props map[string]interface{}) { - for _, prop := range propsArr { - nested := make([]interface{}, len(prop.Values)) - for k := range prop.Values { - nested[k] = extractPrimitiveProperties(prop.Values[k]) - } - props[prop.PropName] = nested - } -} diff --git a/adapters/handlers/grpc/v1/batch_parse_request_test.go b/adapters/handlers/grpc/v1/batch_parse_request_test.go deleted file mode 100644 index 72fc38a299370a701d5cfc593ed146b798653729..0000000000000000000000000000000000000000 --- a/adapters/handlers/grpc/v1/batch_parse_request_test.go +++ /dev/null @@ -1,292 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package v1 - -import ( - "testing" - - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - pb "github.com/weaviate/weaviate/grpc/generated/protocol/v1" -) - -const ( - UUID3 = "a4de3ca0-6975-464f-b23b-adddd83630d7" - UUID4 = "7e10ec81-a26d-4ac7-8264-3e3e05397ddc" -) - -func TestGRPCBatchRequest(t *testing.T) { - collection := "TestClass" - refClass1 := "OtherClass" - refClass2 := "AnotherClass" - scheme := schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{ - { - Class: collection, - Properties: []*models.Property{ - {Name: "name", DataType: schema.DataTypeText.PropString()}, - {Name: "number", DataType: []string{"int"}}, - {Name: "ref", DataType: []string{refClass1}}, - {Name: "multiRef", DataType: []string{refClass1, refClass2}}, - }, - }, - { - Class: refClass1, - Properties: []*models.Property{ - {Name: "something", DataType: schema.DataTypeText.PropString()}, - {Name: "ref2", DataType: []string{refClass2}}, - }, - }, - { - Class: refClass2, - Properties: []*models.Property{ - {Name: "else", DataType: schema.DataTypeText.PropString()}, - {Name: "ref3", DataType: []string{refClass2}}, - }, - }, - }, - }, - } - - var nilMap map[string]interface{} - tests := []struct { - name string - req []*pb.BatchObject - out []*models.Object - outError []int - origIndex map[int]int - }{ - { - name: "empty object", - req: []*pb.BatchObject{{Collection: collection, Uuid: UUID4}}, - out: []*models.Object{{Class: collection, Properties: nilMap, ID: UUID4}}, - }, - { - name: "no UUID", - req: []*pb.BatchObject{{Collection: collection}}, - out: []*models.Object{}, - outError: []int{0}, - }, - { - name: "only normal props", - req: []*pb.BatchObject{{Collection: collection, Uuid: UUID4, Properties: &pb.BatchObject_Properties{ - NonRefProperties: newStruct(t, map[string]interface{}{ - "name": "something", - "age": 45, - }), - }}}, - out: []*models.Object{{Class: collection, ID: UUID4, Properties: map[string]interface{}{ - "name": "something", - "age": float64(45), - }}}, - }, - { - name: "only single refs", - req: []*pb.BatchObject{{Collection: collection, Uuid: UUID4, Properties: &pb.BatchObject_Properties{ - SingleTargetRefProps: []*pb.BatchObject_SingleTargetRefProps{ - {PropName: "ref", Uuids: []string{UUID3, UUID4}}, - }, - }}}, - out: []*models.Object{{Class: collection, ID: UUID4, Properties: map[string]interface{}{ - "ref": []interface{}{ - map[string]interface{}{"beacon": BEACON_START + refClass1 + "/" + UUID3}, - map[string]interface{}{"beacon": BEACON_START + refClass1 + "/" + UUID4}, - }, - }}}, - }, - { - name: "only mult ref", - req: []*pb.BatchObject{{Collection: collection, Uuid: UUID4, Properties: &pb.BatchObject_Properties{ - MultiTargetRefProps: []*pb.BatchObject_MultiTargetRefProps{ - {PropName: "multiRef", Uuids: []string{UUID3, UUID4}, TargetCollection: refClass2}, - }, - }}}, - out: []*models.Object{{Class: collection, ID: UUID4, Properties: map[string]interface{}{ - "multiRef": []interface{}{ - map[string]interface{}{"beacon": BEACON_START + refClass2 + "/" + UUID3}, - map[string]interface{}{"beacon": BEACON_START + refClass2 + "/" + UUID4}, - }, - }}}, - }, - { - name: "all property types", - req: []*pb.BatchObject{{Collection: collection, Uuid: UUID4, Properties: &pb.BatchObject_Properties{ - MultiTargetRefProps: []*pb.BatchObject_MultiTargetRefProps{ - {PropName: "multiRef", Uuids: []string{UUID4, UUID3}, TargetCollection: refClass2}, - }, - SingleTargetRefProps: []*pb.BatchObject_SingleTargetRefProps{ - {PropName: "ref", Uuids: []string{UUID4, UUID3}}, - }, - NonRefProperties: newStruct(t, map[string]interface{}{ - "name": "else", - "age": 46, - }), - }}}, - out: []*models.Object{{Class: collection, ID: UUID4, Properties: map[string]interface{}{ - "multiRef": []interface{}{ - map[string]interface{}{"beacon": BEACON_START + refClass2 + "/" + UUID4}, - map[string]interface{}{"beacon": BEACON_START + refClass2 + "/" + UUID3}, - }, - "ref": []interface{}{ - map[string]interface{}{"beacon": BEACON_START + refClass1 + "/" + UUID4}, - map[string]interface{}{"beacon": BEACON_START + refClass1 + "/" + UUID3}, - }, - "name": "else", - "age": float64(46), - }}}, - }, - { - name: "mult ref to single target", - req: []*pb.BatchObject{{Collection: collection, Uuid: UUID4, Properties: &pb.BatchObject_Properties{ - MultiTargetRefProps: []*pb.BatchObject_MultiTargetRefProps{ - {PropName: "ref", Uuids: []string{UUID3, UUID4}, TargetCollection: refClass2}, - }, - }}}, - out: []*models.Object{}, - outError: []int{0}, - }, - { - name: "single ref to multi target", - req: []*pb.BatchObject{{Collection: collection, Uuid: UUID4, Properties: &pb.BatchObject_Properties{ - SingleTargetRefProps: []*pb.BatchObject_SingleTargetRefProps{ - {PropName: "multiRef", Uuids: []string{UUID3, UUID4}}, - }, - }}}, - out: []*models.Object{}, - outError: []int{0}, - }, - { - name: "slice props", - req: []*pb.BatchObject{{Collection: collection, Uuid: UUID4, Properties: &pb.BatchObject_Properties{ - NonRefProperties: newStruct(t, map[string]interface{}{"name": "something"}), - BooleanArrayProperties: []*pb.BooleanArrayProperties{ - {PropName: "boolArray1", Values: []bool{true, true}}, - {PropName: "boolArray2", Values: []bool{false, true}}, - }, - IntArrayProperties: []*pb.IntArrayProperties{ - {PropName: "int1", Values: []int64{2, 3, 4}}, {PropName: "int2", Values: []int64{7, 8}}, - }, - NumberArrayProperties: []*pb.NumberArrayProperties{ - {PropName: "float1", Values: []float64{1, 2, 3}}, {PropName: "float2", Values: []float64{4, 5}}, - }, - TextArrayProperties: []*pb.TextArrayProperties{ - {PropName: "text1", Values: []string{"first", "second"}}, {PropName: "text2", Values: []string{"third"}}, - }, - }}}, - out: []*models.Object{{Class: collection, ID: UUID4, Properties: map[string]interface{}{ - "name": "something", - "boolArray1": []interface{}{true, true}, - "boolArray2": []interface{}{false, true}, - "int1": []interface{}{int64(2), int64(3), int64(4)}, - "int2": []interface{}{int64(7), int64(8)}, - "float1": []interface{}{1., 2., 3.}, - "float2": []interface{}{4., 5.}, - "text1": []interface{}{"first", "second"}, - "text2": []interface{}{"third"}, - }}}, - }, - { - name: "object props", - req: []*pb.BatchObject{{Collection: collection, Uuid: UUID4, Properties: &pb.BatchObject_Properties{ - ObjectProperties: []*pb.ObjectProperties{ - { - PropName: "simpleObj", Value: &pb.ObjectPropertiesValue{ - NonRefProperties: newStruct(t, map[string]interface{}{"name": "something"}), - }, - }, - { - PropName: "nestedObj", Value: &pb.ObjectPropertiesValue{ - ObjectProperties: []*pb.ObjectProperties{{ - PropName: "obj", Value: &pb.ObjectPropertiesValue{ - NonRefProperties: newStruct(t, map[string]interface{}{"name": "something"}), - }, - }}, - }, - }, - }, - }}}, - out: []*models.Object{{Class: collection, ID: UUID4, Properties: map[string]interface{}{ - "simpleObj": map[string]interface{}{"name": "something"}, - "nestedObj": map[string]interface{}{ - "obj": map[string]interface{}{"name": "something"}, - }, - }}}, - }, - { - name: "object array props", - req: []*pb.BatchObject{{Collection: collection, Uuid: UUID4, Properties: &pb.BatchObject_Properties{ - ObjectArrayProperties: []*pb.ObjectArrayProperties{ - { - PropName: "simpleObjs", Values: []*pb.ObjectPropertiesValue{ - { - NonRefProperties: newStruct(t, map[string]interface{}{"name": "something"}), - }, - { - NonRefProperties: newStruct(t, map[string]interface{}{"name": "something else"}), - }, - }, - }, - { - PropName: "nestedObjs", Values: []*pb.ObjectPropertiesValue{ - { - ObjectProperties: []*pb.ObjectProperties{{ - PropName: "obj", Value: &pb.ObjectPropertiesValue{ - NonRefProperties: newStruct(t, map[string]interface{}{"name": "something"}), - }, - }}, - }, - { - ObjectProperties: []*pb.ObjectProperties{{ - PropName: "obj", Value: &pb.ObjectPropertiesValue{ - NonRefProperties: newStruct(t, map[string]interface{}{"name": "something else"}), - }, - }}, - }, - }, - }, - }, - }}}, - out: []*models.Object{{Class: collection, ID: UUID4, Properties: map[string]interface{}{ - "simpleObjs": []interface{}{map[string]interface{}{"name": "something"}, map[string]interface{}{"name": "something else"}}, - "nestedObjs": []interface{}{ - map[string]interface{}{"obj": map[string]interface{}{"name": "something"}}, - map[string]interface{}{"obj": map[string]interface{}{"name": "something else"}}, - }, - }}}, - }, - { - name: "mix of errors and no errors", - req: []*pb.BatchObject{{Collection: collection, Uuid: UUID4}, {Collection: collection}, {Collection: collection}, {Collection: collection, Uuid: UUID3}}, - out: []*models.Object{{Class: collection, Properties: nilMap, ID: UUID4}, {Class: collection, Properties: nilMap, ID: UUID3}}, - outError: []int{1, 2}, - origIndex: map[int]int{0: 0, 1: 3}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - out, origIndex, batchErrors := batchFromProto(&pb.BatchObjectsRequest{Objects: tt.req}, scheme) - if len(tt.outError) > 0 { - require.NotNil(t, batchErrors) - if len(tt.out) > 0 { - require.Equal(t, tt.out, out) - require.Equal(t, tt.origIndex, origIndex) - } - } else { - require.Len(t, batchErrors, 0) - require.Equal(t, tt.out, out) - } - }) - } -} diff --git a/adapters/handlers/grpc/v1/filters.go b/adapters/handlers/grpc/v1/filters.go deleted file mode 100644 index 8c053eefafacefce2ed265565cb04fdf78acba87..0000000000000000000000000000000000000000 --- a/adapters/handlers/grpc/v1/filters.go +++ /dev/null @@ -1,283 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package v1 - -import ( - "fmt" - - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - pb "github.com/weaviate/weaviate/grpc/generated/protocol/v1" -) - -func extractFilters(filterIn *pb.Filters, scheme schema.Schema, className string) (filters.Clause, error) { - returnFilter := filters.Clause{} - if filterIn.Operator == pb.Filters_OPERATOR_AND || filterIn.Operator == pb.Filters_OPERATOR_OR { - if filterIn.Operator == pb.Filters_OPERATOR_AND { - returnFilter.Operator = filters.OperatorAnd - } else { - returnFilter.Operator = filters.OperatorOr - } - - clauses := make([]filters.Clause, len(filterIn.Filters)) - for i, clause := range filterIn.Filters { - retClause, err := extractFilters(clause, scheme, className) - if err != nil { - return filters.Clause{}, err - } - clauses[i] = retClause - } - - returnFilter.Operands = clauses - - } else { - if filterIn.Target == nil && len(filterIn.On)%2 != 1 { - return filters.Clause{}, fmt.Errorf( - "paths needs to have a uneven number of components: property, class, property, ...., got %v", filterIn.On, - ) - } - - switch filterIn.Operator { - case pb.Filters_OPERATOR_EQUAL: - returnFilter.Operator = filters.OperatorEqual - case pb.Filters_OPERATOR_NOT_EQUAL: - returnFilter.Operator = filters.OperatorNotEqual - case pb.Filters_OPERATOR_GREATER_THAN: - returnFilter.Operator = filters.OperatorGreaterThan - case pb.Filters_OPERATOR_GREATER_THAN_EQUAL: - returnFilter.Operator = filters.OperatorGreaterThanEqual - case pb.Filters_OPERATOR_LESS_THAN: - returnFilter.Operator = filters.OperatorLessThan - case pb.Filters_OPERATOR_LESS_THAN_EQUAL: - returnFilter.Operator = filters.OperatorLessThanEqual - case pb.Filters_OPERATOR_WITHIN_GEO_RANGE: - returnFilter.Operator = filters.OperatorWithinGeoRange - case pb.Filters_OPERATOR_LIKE: - returnFilter.Operator = filters.OperatorLike - case pb.Filters_OPERATOR_IS_NULL: - returnFilter.Operator = filters.OperatorIsNull - case pb.Filters_OPERATOR_CONTAINS_ANY: - returnFilter.Operator = filters.ContainsAny - case pb.Filters_OPERATOR_CONTAINS_ALL: - returnFilter.Operator = filters.ContainsAll - default: - return filters.Clause{}, fmt.Errorf("unknown filter operator %v", filterIn.Operator) - } - - var dataType schema.DataType - if filterIn.Target == nil { - path, err := extractPath(scheme, className, filterIn.On) - if err != nil { - return filters.Clause{}, err - } - returnFilter.On = path - - dataType, err = extractDataType(scheme, returnFilter.Operator, className, filterIn.On) - if err != nil { - return filters.Clause{}, err - } - } else { - path, dataType2, err := extractPathNew(scheme, className, filterIn.Target, returnFilter.Operator) - if err != nil { - return filters.Clause{}, err - } - dataType = dataType2 - returnFilter.On = path - } - - // datatype UUID is just a string - if dataType == schema.DataTypeUUID { - dataType = schema.DataTypeText - } - - var val interface{} - switch filterIn.TestValue.(type) { - case *pb.Filters_ValueText: - val = filterIn.GetValueText() - case *pb.Filters_ValueInt: - val = int(filterIn.GetValueInt()) - case *pb.Filters_ValueBoolean: - val = filterIn.GetValueBoolean() - case *pb.Filters_ValueNumber: - val = filterIn.GetValueNumber() - case *pb.Filters_ValueIntArray: - // convert from int32 GRPC to go int - valInt32 := filterIn.GetValueIntArray().Values - valInt := make([]int, len(valInt32)) - for i := 0; i < len(valInt32); i++ { - valInt[i] = int(valInt32[i]) - } - val = valInt - case *pb.Filters_ValueTextArray: - val = filterIn.GetValueTextArray().Values - case *pb.Filters_ValueNumberArray: - val = filterIn.GetValueNumberArray().Values - case *pb.Filters_ValueBooleanArray: - val = filterIn.GetValueBooleanArray().Values - case *pb.Filters_ValueGeo: - valueFilter := filterIn.GetValueGeo() - val = filters.GeoRange{ - GeoCoordinates: &models.GeoCoordinates{ - Latitude: &valueFilter.Latitude, - Longitude: &valueFilter.Longitude, - }, - Distance: valueFilter.Distance, - } - default: - return filters.Clause{}, fmt.Errorf("unknown value type %v", filterIn.TestValue) - } - - // correct the type of value when filtering on a float/int property but sending an int/float. This is easy to - // get wrong - if number, ok := val.(int); ok && dataType == schema.DataTypeNumber { - val = float64(number) - } - if number, ok := val.(float64); ok && dataType == schema.DataTypeInt { - val = int(number) - if float64(int(number)) != number { - return filters.Clause{}, fmt.Errorf("filtering for integer, but received a floating point number %v", number) - } - } - - // correct type for containsXXX in case users send int/float for a float/int array - if (returnFilter.Operator == filters.ContainsAll || returnFilter.Operator == filters.ContainsAny) && dataType == schema.DataTypeNumber { - valSlice, ok := val.([]int) - if ok { - val64 := make([]float64, len(valSlice)) - for i := 0; i < len(valSlice); i++ { - val64[i] = float64(valSlice[i]) - } - val = val64 - } - } - - if (returnFilter.Operator == filters.ContainsAll || returnFilter.Operator == filters.ContainsAny) && dataType == schema.DataTypeInt { - valSlice, ok := val.([]float64) - if ok { - valInt := make([]int, len(valSlice)) - for i := 0; i < len(valSlice); i++ { - if float64(int(valSlice[i])) != valSlice[i] { - return filters.Clause{}, fmt.Errorf("filtering for integer, but received a floating point number %v", valSlice[i]) - } - valInt[i] = int(valSlice[i]) - } - val = valInt - } - } - - value := filters.Value{Value: val, Type: dataType} - returnFilter.Value = &value - - } - - return returnFilter, nil -} - -func extractDataTypeProperty(scheme schema.Schema, operator filters.Operator, classname string, on []string) (schema.DataType, error) { - var dataType schema.DataType - if operator == filters.OperatorIsNull { - dataType = schema.DataTypeBoolean - } else if len(on) > 1 { - propToCheck := on[len(on)-1] - _, isPropLengthFilter := schema.IsPropertyLength(propToCheck, 0) - if isPropLengthFilter { - return schema.DataTypeInt, nil - } - - classOfProp := on[len(on)-2] - prop, err := scheme.GetProperty(schema.ClassName(classOfProp), schema.PropertyName(propToCheck)) - if err != nil { - return dataType, err - } - dataType = schema.DataType(prop.DataType[0]) - } else { - propToCheck := on[0] - _, isPropLengthFilter := schema.IsPropertyLength(propToCheck, 0) - if isPropLengthFilter { - return schema.DataTypeInt, nil - } - - prop, err := scheme.GetProperty(schema.ClassName(classname), schema.PropertyName(propToCheck)) - if err != nil { - return dataType, err - } - dataType = schema.DataType(prop.DataType[0]) - } - - // searches on array datatypes always need the base-type as value-type - if baseType, isArray := schema.IsArrayType(dataType); isArray { - return baseType, nil - } - return dataType, nil -} - -func extractDataType(scheme schema.Schema, operator filters.Operator, classname string, on []string) (schema.DataType, error) { - propToFilterOn := on[len(on)-1] - if propToFilterOn == filters.InternalPropID { - return schema.DataTypeText, nil - } else if propToFilterOn == filters.InternalPropCreationTimeUnix || propToFilterOn == filters.InternalPropLastUpdateTimeUnix { - return schema.DataTypeDate, nil - } else { - return extractDataTypeProperty(scheme, operator, classname, on) - } -} - -func extractPath(scheme schema.Schema, className string, on []string) (*filters.Path, error) { - if len(on) > 1 { - var err error - child, err := extractPath(scheme, on[1], on[2:]) - if err != nil { - return nil, err - } - return &filters.Path{Class: schema.ClassName(className), Property: schema.PropertyName(on[0]), Child: child}, nil - - } - return &filters.Path{Class: schema.ClassName(className), Property: schema.PropertyName(on[0]), Child: nil}, nil -} - -func extractPathNew(scheme schema.Schema, className string, target *pb.FilterTarget, operator filters.Operator) (*filters.Path, schema.DataType, error) { - class := scheme.GetClass(schema.ClassName(className)) - switch target.Target.(type) { - case *pb.FilterTarget_Property: - dt, err := extractDataType(scheme, operator, className, []string{target.GetProperty()}) - if err != nil { - return nil, "", err - } - return &filters.Path{Class: schema.ClassName(className), Property: schema.PropertyName(target.GetProperty()), Child: nil}, dt, nil - case *pb.FilterTarget_SingleTarget: - singleTarget := target.GetSingleTarget() - normalizedRefPropName := schema.LowercaseFirstLetter(singleTarget.On) - refProp, err := schema.GetPropertyByName(class, normalizedRefPropName) - if err != nil { - return nil, "", err - } - if len(refProp.DataType) != 1 { - return nil, "", fmt.Errorf("expected reference property with a single target, got %v for %v ", refProp.DataType, refProp.Name) - } - - child, property, err := extractPathNew(scheme, refProp.DataType[0], singleTarget.Target, operator) - if err != nil { - return nil, "", err - } - return &filters.Path{Class: schema.ClassName(className), Property: schema.PropertyName(normalizedRefPropName), Child: child}, property, nil - case *pb.FilterTarget_MultiTarget: - multiTarget := target.GetMultiTarget() - child, property, err := extractPathNew(scheme, multiTarget.TargetCollection, multiTarget.Target, operator) - if err != nil { - return nil, "", err - } - return &filters.Path{Class: schema.ClassName(className), Property: schema.PropertyName(schema.LowercaseFirstLetter(multiTarget.On)), Child: child}, property, nil - default: - return nil, "", fmt.Errorf("unknown target type %v", target) - } -} diff --git a/adapters/handlers/grpc/v1/health.go b/adapters/handlers/grpc/v1/health.go deleted file mode 100644 index b920c7f61ebbc48c9544ed805cb5fb7128cea532..0000000000000000000000000000000000000000 --- a/adapters/handlers/grpc/v1/health.go +++ /dev/null @@ -1,30 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package v1 - -import ( - "context" - - "google.golang.org/grpc/health/grpc_health_v1" -) - -func (s *Service) healthCheck() *grpc_health_v1.HealthCheckResponse { - return &grpc_health_v1.HealthCheckResponse{Status: grpc_health_v1.HealthCheckResponse_SERVING} -} - -func (s *Service) Check(ctx context.Context, request *grpc_health_v1.HealthCheckRequest) (*grpc_health_v1.HealthCheckResponse, error) { - return s.healthCheck(), nil -} - -func (s *Service) Watch(request *grpc_health_v1.HealthCheckRequest, server grpc_health_v1.Health_WatchServer) error { - return server.Send(s.healthCheck()) -} diff --git a/adapters/handlers/grpc/v1/mapping.go b/adapters/handlers/grpc/v1/mapping.go deleted file mode 100644 index ae67a65a46e736e2b6f9c643b8d13e23300370f3..0000000000000000000000000000000000000000 --- a/adapters/handlers/grpc/v1/mapping.go +++ /dev/null @@ -1,284 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package v1 - -import ( - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/search" - pb "github.com/weaviate/weaviate/grpc/generated/protocol/v1" - "google.golang.org/protobuf/runtime/protoimpl" -) - -func parseArray[T float64 | bool | string](v interface{}, innerDt schema.DataType) (*pb.Value, error) { - if _, ok := v.([]interface{}); ok { - return &pb.Value{Kind: &pb.Value_ListValue{ListValue: &pb.ListValue{Values: []*pb.Value{}}}}, nil - } - val, ok := v.([]T) - if !ok { - return nil, protoimpl.X.NewError("invalid type: %T when serializing %v", v, innerDt.String()) - } - list, err := NewPrimitiveList(val, innerDt) - if err != nil { - return nil, errors.Wrapf(err, "serializing array with type %v", innerDt) - } - return NewListValue(list), nil -} - -func NewPrimitiveValue(v interface{}, dt schema.DataType) (*pb.Value, error) { - innerDt, ok := schema.IsArrayType(dt) - if ok { - switch dt { - case schema.DataTypeBooleanArray: - return parseArray[bool](v, innerDt) - case schema.DataTypeDateArray: - return parseArray[string](v, innerDt) - case schema.DataTypeNumberArray: - return parseArray[float64](v, innerDt) - case schema.DataTypeIntArray: - return parseArray[float64](v, innerDt) - case schema.DataTypeStringArray: - return parseArray[string](v, innerDt) - case schema.DataTypeTextArray: - return parseArray[string](v, innerDt) - case schema.DataTypeUUIDArray: - return parseArray[string](v, innerDt) - default: - return nil, protoimpl.X.NewError("invalid type: %T", v) - } - } else { - switch dt { - case schema.DataTypeBoolean: - val, ok := v.(bool) - if !ok { - return nil, protoimpl.X.NewError("invalid type: %T expected bool when serializing bool", v) - } - return NewBoolValue(val), nil - case schema.DataTypeDate: - val, ok := v.(string) - if !ok { - return nil, protoimpl.X.NewError("invalid type: %T expected string when serializing date", v) - } - return NewDateValue(val), nil - case schema.DataTypeNumber: - val, ok := v.(float64) - if !ok { - return nil, protoimpl.X.NewError("invalid type: %T expected float64 when serializing number", v) - } - return NewNumberValue(val), nil - case schema.DataTypeInt: - val, ok := v.(float64) - if !ok { // integers are returned as float64 from search - return nil, protoimpl.X.NewError("invalid type: %T expected float64 when serializing int property", v) - } - return NewIntValue(int64(val)), nil - case schema.DataTypeString: - val, ok := v.(string) - if !ok { - return nil, protoimpl.X.NewError("invalid type: %T expected string when serializing string property", v) - } - return NewStringValue(val), nil - case schema.DataTypeText: - val, ok := v.(string) - if !ok { - return nil, protoimpl.X.NewError("invalid type: %T expected string when serializing text property", v) - } - return NewStringValue(val), nil - case schema.DataTypeUUID: - val, ok := v.(string) - if !ok { - return nil, protoimpl.X.NewError("invalid type: %T expected string when serializing uuid property", v) - } - return NewUuidValue(val), nil - case schema.DataTypeGeoCoordinates: - val, ok := v.(*models.GeoCoordinates) - if !ok { - return nil, protoimpl.X.NewError("invalid type: %T expected *models.GeoCoordinates when serializing geocoordinate property", v) - } - return NewGeoValue(val), nil - case schema.DataTypeBlob: - val, ok := v.(string) - if !ok { - return nil, protoimpl.X.NewError("invalid type: %T expected string when serializing blob property", v) - } - return NewBlobValue(val), nil - case schema.DataTypePhoneNumber: - val, ok := v.(*models.PhoneNumber) - if !ok { - return nil, protoimpl.X.NewError("invalid type: %T expected *models.PhoneNumber when serializing phone number property", v) - } - return NewPhoneNumberValue(val), nil - default: - return nil, protoimpl.X.NewError("invalid type: %T", v) - } - } -} - -func NewNestedValue[P schema.PropertyInterface](v interface{}, dt schema.DataType, parent P, prop search.SelectProperty) (*pb.Value, error) { - switch dt { - case schema.DataTypeObject: - if _, ok := v.(map[string]interface{}); !ok { - return nil, protoimpl.X.NewError("invalid type: %T expected map[string]interface{}", v) - } - obj, err := NewObject(v.(map[string]interface{}), parent, prop) - if err != nil { - return nil, errors.Wrap(err, "creating nested object") - } - return NewObjectValue(obj), nil - case schema.DataTypeObjectArray: - if _, ok := v.([]interface{}); !ok { - return nil, protoimpl.X.NewError("invalid type: %T expected []map[string]interface{}", v) - } - list, err := NewObjectList(v.([]interface{}), parent, prop) - if err != nil { - return nil, errors.Wrap(err, "creating nested object array") - } - return NewListValue(list), nil - default: - return nil, protoimpl.X.NewError("invalid type: %T", v) - } -} - -// NewStruct constructs a Struct from a general-purpose Go map. -// The map keys must be valid UTF-8. -// The map values are converted using NewValue. -func NewObject[P schema.PropertyInterface](v map[string]interface{}, parent P, selectProp search.SelectProperty) (*pb.Properties, error) { - if !selectProp.IsObject { - return nil, errors.New("select property is not an object") - } - x := &pb.Properties{Fields: make(map[string]*pb.Value, len(v))} - for _, selectProp := range selectProp.Props { - dt, err := schema.GetNestedPropertyDataType(parent, selectProp.Name) - if err != nil { - return nil, errors.Wrapf(err, "getting data type of nested property %s", selectProp.Name) - } - if *dt == schema.DataTypeObject || *dt == schema.DataTypeObjectArray { - nested, err := schema.GetNestedPropertyByName(parent, selectProp.Name) - if err != nil { - return nil, errors.Wrapf(err, "getting nested property %s", selectProp.Name) - } - x.Fields[selectProp.Name], err = NewNestedValue(v[selectProp.Name], *dt, &NestedProperty{NestedProperty: nested}, selectProp) - if err != nil { - return nil, errors.Wrapf(err, "creating nested object value %s", selectProp.Name) - } - } else { - x.Fields[selectProp.Name], err = NewPrimitiveValue(v[selectProp.Name], *dt) - if err != nil { - return nil, errors.Wrapf(err, "creating nested primitive value %s", selectProp.Name) - } - } - if err != nil { - return nil, errors.Wrapf(err, "serializing field %s", selectProp.Name) - } - } - return x, nil -} - -// NewList constructs a ListValue from a general-purpose Go slice. -// The slice elements are converted using NewValue. -func NewPrimitiveList[T bool | float64 | string](v []T, dt schema.DataType) (*pb.ListValue, error) { - var err error - x := &pb.ListValue{Values: make([]*pb.Value, len(v))} - for i, v := range v { - x.Values[i], err = NewPrimitiveValue(v, dt) - if err != nil { - return nil, err - } - } - return x, nil -} - -// NewList constructs a ListValue from a general-purpose Go slice. -// The slice elements are converted using NewValue. -func NewObjectList[P schema.PropertyInterface](v []interface{}, parent P, selectProp search.SelectProperty) (*pb.ListValue, error) { - if !selectProp.IsObject { - return nil, errors.New("select property is not an object") - } - x := &pb.ListValue{Values: make([]*pb.Value, len(v))} - for i, v := range v { - if _, ok := v.(map[string]interface{}); !ok { - return nil, protoimpl.X.NewError("invalid type: %T expected map[string]interface{}", v) - } - value, err := NewObject(v.(map[string]interface{}), parent, selectProp) - if err != nil { - return nil, err - } - x.Values[i] = NewObjectValue(value) - } - return x, nil -} - -// NewBoolValue constructs a new boolean Value. -func NewBoolValue(v bool) *pb.Value { - return &pb.Value{Kind: &pb.Value_BoolValue{BoolValue: v}} -} - -// NewNumberValue constructs a new number Value. -func NewNumberValue(v float64) *pb.Value { - return &pb.Value{Kind: &pb.Value_NumberValue{NumberValue: v}} -} - -// NewIntValue constructs a new number Value. -func NewIntValue(v int64) *pb.Value { - return &pb.Value{Kind: &pb.Value_IntValue{IntValue: v}} -} - -// NewStringValue constructs a new string Value. -func NewStringValue(v string) *pb.Value { - return &pb.Value{Kind: &pb.Value_StringValue{StringValue: v}} -} - -// NewDateValue constructs a new string Value. -func NewDateValue(v string) *pb.Value { - return &pb.Value{Kind: &pb.Value_DateValue{DateValue: v}} -} - -// NewUuidValue constructs a new string Value. -func NewUuidValue(v string) *pb.Value { - return &pb.Value{Kind: &pb.Value_UuidValue{UuidValue: v}} -} - -// NewGeoValue constructs a new geo Value. -func NewGeoValue(v *models.GeoCoordinates) *pb.Value { - return &pb.Value{Kind: &pb.Value_GeoValue{GeoValue: &pb.GeoCoordinate{Latitude: *v.Latitude, Longitude: *v.Longitude}}} -} - -// NewObjectValue constructs a new struct Value. -func NewObjectValue(v *pb.Properties) *pb.Value { - return &pb.Value{Kind: &pb.Value_ObjectValue{ObjectValue: v}} -} - -// NewListValue constructs a new list Value. -func NewListValue(v *pb.ListValue) *pb.Value { - return &pb.Value{Kind: &pb.Value_ListValue{ListValue: v}} -} - -// NewBlobValue constructs a new blob Value. -func NewBlobValue(v string) *pb.Value { - return &pb.Value{Kind: &pb.Value_BlobValue{BlobValue: v}} -} - -// NewPhoneNumberValue constructs a new phone number Value. -func NewPhoneNumberValue(v *models.PhoneNumber) *pb.Value { - return &pb.Value{Kind: &pb.Value_PhoneValue{ - PhoneValue: &pb.PhoneNumber{ - CountryCode: v.CountryCode, - DefaultCountry: v.DefaultCountry, - Input: v.Input, - InternationalFormatted: v.InternationalFormatted, - National: v.National, - NationalFormatted: v.NationalFormatted, - Valid: v.Valid, - }, - }} -} diff --git a/adapters/handlers/grpc/v1/mapping_test.go b/adapters/handlers/grpc/v1/mapping_test.go deleted file mode 100644 index a2188a2e15cd8e5627d22f3065eb9e2a88344fa9..0000000000000000000000000000000000000000 --- a/adapters/handlers/grpc/v1/mapping_test.go +++ /dev/null @@ -1,177 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package v1 - -import ( - "testing" - - "github.com/weaviate/weaviate/entities/models" - - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/schema" - pb "github.com/weaviate/weaviate/grpc/generated/protocol/v1" -) - -type innerTest struct { - datatype schema.DataType - out *pb.Value - shouldError bool -} - -func makeTestList(succeedingInnerTests map[schema.DataType]*pb.Value) []innerTest { - dtypes := append(schema.PrimitiveDataTypes, schema.DeprecatedPrimitiveDataTypes...) - list := make([]innerTest, len(dtypes)) - for idx := range dtypes { - out, ok := succeedingInnerTests[dtypes[idx]] - if ok { - list[idx] = innerTest{ - datatype: dtypes[idx], - out: out, - shouldError: false, - } - } else { - list[idx] = innerTest{ - datatype: dtypes[idx], - out: nil, - shouldError: true, - } - } - } - return list -} - -func TestNewPrimitiveValue(t *testing.T) { - float_val := float32(1.1) - - tests := []struct { - name string - in any - tests []innerTest - }{ - { - name: "bools", - in: []bool{true, false}, - tests: makeTestList(map[schema.DataType]*pb.Value{ - schema.DataTypeBooleanArray: {Kind: &pb.Value_ListValue{ListValue: &pb.ListValue{Values: []*pb.Value{ - {Kind: &pb.Value_BoolValue{BoolValue: true}}, - {Kind: &pb.Value_BoolValue{BoolValue: false}}, - }}}}, - }), - }, - { - name: "strings", - in: []string{"a string", "another string"}, - tests: makeTestList(map[schema.DataType]*pb.Value{ - schema.DataTypeDateArray: {Kind: &pb.Value_ListValue{ListValue: &pb.ListValue{Values: []*pb.Value{ - {Kind: &pb.Value_DateValue{DateValue: "a string"}}, - {Kind: &pb.Value_DateValue{DateValue: "another string"}}, - }}}}, - schema.DataTypeStringArray: {Kind: &pb.Value_ListValue{ListValue: &pb.ListValue{Values: []*pb.Value{ - {Kind: &pb.Value_StringValue{StringValue: "a string"}}, - {Kind: &pb.Value_StringValue{StringValue: "another string"}}, - }}}}, - schema.DataTypeTextArray: {Kind: &pb.Value_ListValue{ListValue: &pb.ListValue{Values: []*pb.Value{ - {Kind: &pb.Value_StringValue{StringValue: "a string"}}, - {Kind: &pb.Value_StringValue{StringValue: "another string"}}, - }}}}, - schema.DataTypeUUIDArray: {Kind: &pb.Value_ListValue{ListValue: &pb.ListValue{Values: []*pb.Value{ - {Kind: &pb.Value_UuidValue{UuidValue: "a string"}}, - {Kind: &pb.Value_UuidValue{UuidValue: "another string"}}, - }}}}, - }), - }, - { - name: "float64s", - in: []float64{1.1, 2.2, 3.3}, - tests: makeTestList(map[schema.DataType]*pb.Value{ - schema.DataTypeNumberArray: {Kind: &pb.Value_ListValue{ListValue: &pb.ListValue{Values: []*pb.Value{ - {Kind: &pb.Value_NumberValue{NumberValue: 1.1}}, - {Kind: &pb.Value_NumberValue{NumberValue: 2.2}}, - {Kind: &pb.Value_NumberValue{NumberValue: 3.3}}, - }}}}, - schema.DataTypeIntArray: {Kind: &pb.Value_ListValue{ListValue: &pb.ListValue{Values: []*pb.Value{ - {Kind: &pb.Value_IntValue{IntValue: 1}}, - {Kind: &pb.Value_IntValue{IntValue: 2}}, - {Kind: &pb.Value_IntValue{IntValue: 3}}, - }}}}, - }), - }, - { - name: "empty array", - in: []interface{}{}, - tests: makeTestList(map[schema.DataType]*pb.Value{ - schema.DataTypeBooleanArray: {Kind: &pb.Value_ListValue{ListValue: &pb.ListValue{Values: []*pb.Value{}}}}, - schema.DataTypeDateArray: {Kind: &pb.Value_ListValue{ListValue: &pb.ListValue{Values: []*pb.Value{}}}}, - schema.DataTypeNumberArray: {Kind: &pb.Value_ListValue{ListValue: &pb.ListValue{Values: []*pb.Value{}}}}, - schema.DataTypeIntArray: {Kind: &pb.Value_ListValue{ListValue: &pb.ListValue{Values: []*pb.Value{}}}}, - schema.DataTypeStringArray: {Kind: &pb.Value_ListValue{ListValue: &pb.ListValue{Values: []*pb.Value{}}}}, - schema.DataTypeTextArray: {Kind: &pb.Value_ListValue{ListValue: &pb.ListValue{Values: []*pb.Value{}}}}, - schema.DataTypeUUIDArray: {Kind: &pb.Value_ListValue{ListValue: &pb.ListValue{Values: []*pb.Value{}}}}, - }), - }, - { - name: "bool", - in: true, - tests: makeTestList(map[schema.DataType]*pb.Value{ - schema.DataTypeBoolean: {Kind: &pb.Value_BoolValue{BoolValue: true}}, - }), - }, - { - name: "string", - in: "a string", - tests: makeTestList(map[schema.DataType]*pb.Value{ - schema.DataTypeDate: {Kind: &pb.Value_DateValue{DateValue: "a string"}}, - schema.DataTypeString: {Kind: &pb.Value_StringValue{StringValue: "a string"}}, - schema.DataTypeText: {Kind: &pb.Value_StringValue{StringValue: "a string"}}, - schema.DataTypeUUID: {Kind: &pb.Value_UuidValue{UuidValue: "a string"}}, - schema.DataTypeBlob: {Kind: &pb.Value_BlobValue{BlobValue: "a string"}}, - }), - }, - { - name: "float64", - in: 1.1, - tests: makeTestList(map[schema.DataType]*pb.Value{ - schema.DataTypeNumber: {Kind: &pb.Value_NumberValue{NumberValue: 1.1}}, - schema.DataTypeInt: {Kind: &pb.Value_IntValue{IntValue: 1}}, - }), - }, - { - name: "geo", - in: &models.GeoCoordinates{Longitude: &float_val, Latitude: &float_val}, - tests: makeTestList(map[schema.DataType]*pb.Value{ - schema.DataTypeGeoCoordinates: {Kind: &pb.Value_GeoValue{GeoValue: &pb.GeoCoordinate{Latitude: float_val, Longitude: float_val}}}, - }), - }, - { - name: "phone number", - in: &models.PhoneNumber{Input: "1234567890"}, - tests: makeTestList(map[schema.DataType]*pb.Value{ - schema.DataTypePhoneNumber: {Kind: &pb.Value_PhoneValue{PhoneValue: &pb.PhoneNumber{Input: "1234567890"}}}, - }), - }, - } - - for _, tt := range tests { - for _, test := range tt.tests { - out, err := NewPrimitiveValue(tt.in, test.datatype) - if test.shouldError { - if err == nil { - t.Logf("expected an error for %v and %s", tt.in, test.datatype) - } - require.Error(t, err) - } else { - require.NoError(t, err) - require.Equal(t, test.out, out) - } - } - } -} diff --git a/adapters/handlers/grpc/v1/models.go b/adapters/handlers/grpc/v1/models.go deleted file mode 100644 index 60c77535c69091c505662bc8c648238e6d815789..0000000000000000000000000000000000000000 --- a/adapters/handlers/grpc/v1/models.go +++ /dev/null @@ -1,38 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package v1 - -import "github.com/weaviate/weaviate/entities/models" - -type Property struct { - *models.Property -} - -type NestedProperty struct { - *models.NestedProperty -} - -func (p *Property) GetName() string { - return p.Property.Name -} - -func (p *Property) GetNestedProperties() []*models.NestedProperty { - return p.Property.NestedProperties -} - -func (p *NestedProperty) GetName() string { - return p.NestedProperty.Name -} - -func (p *NestedProperty) GetNestedProperties() []*models.NestedProperty { - return p.NestedProperty.NestedProperties -} diff --git a/adapters/handlers/grpc/v1/parse_search_request.go b/adapters/handlers/grpc/v1/parse_search_request.go deleted file mode 100644 index 9be736cf0a830675fc2748be7b6ee8d4d7b59705..0000000000000000000000000000000000000000 --- a/adapters/handlers/grpc/v1/parse_search_request.go +++ /dev/null @@ -1,753 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package v1 - -import ( - "fmt" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/usecases/byteops" - - "github.com/weaviate/weaviate/usecases/modulecomponents/additional/generate" - "github.com/weaviate/weaviate/usecases/modulecomponents/additional/rank" - - "github.com/weaviate/weaviate/usecases/modulecomponents/nearVideo" - - "github.com/weaviate/weaviate/usecases/modulecomponents/nearAudio" - - "github.com/weaviate/weaviate/usecases/modulecomponents/nearImage" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/searchparams" - "github.com/weaviate/weaviate/entities/vectorindex/common" - nearText2 "github.com/weaviate/weaviate/usecases/modulecomponents/nearText" - - "github.com/go-openapi/strfmt" - "github.com/google/uuid" - "github.com/weaviate/weaviate/entities/schema/crossref" - - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/search" - - "github.com/weaviate/weaviate/adapters/handlers/graphql/local/common_filters" - "github.com/weaviate/weaviate/entities/dto" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/schema" - pb "github.com/weaviate/weaviate/grpc/generated/protocol/v1" -) - -func searchParamsFromProto(req *pb.SearchRequest, scheme schema.Schema) (dto.GetParams, error) { - out := dto.GetParams{} - class, err := schema.GetClassByName(scheme.Objects, req.Collection) - if err != nil { - return dto.GetParams{}, err - } - - out.ClassName = req.Collection - out.ReplicationProperties = extractReplicationProperties(req.ConsistencyLevel) - - out.Tenant = req.Tenant - - if req.Metadata != nil { - addProps, err := extractAdditionalPropsFromMetadata(class, req.Metadata) - if err != nil { - return dto.GetParams{}, errors.Wrap(err, "extract additional props") - } - out.AdditionalProperties = addProps - } - - out.Properties, err = extractPropertiesRequest(req.Properties, scheme, req.Collection, req.Uses_123Api) - if err != nil { - return dto.GetParams{}, errors.Wrap(err, "extract properties request") - } - if len(out.Properties) == 0 { - out.AdditionalProperties.NoProps = true - } - - if hs := req.HybridSearch; hs != nil { - fusionType := common_filters.HybridFusionDefault - if hs.FusionType == pb.Hybrid_FUSION_TYPE_RANKED { - fusionType = common_filters.HybridRankedFusion - } else if hs.FusionType == pb.Hybrid_FUSION_TYPE_RELATIVE_SCORE { - fusionType = common_filters.HybridRelativeScoreFusion - } - - var vector []float32 - // bytes vector has precedent for being more efficient - if len(hs.VectorBytes) > 0 { - vector = byteops.Float32FromByteVector(hs.VectorBytes) - } else if len(hs.Vector) > 0 { - vector = hs.Vector - } - - out.HybridSearch = &searchparams.HybridSearch{Query: hs.Query, Properties: schema.LowercaseFirstLetterOfStrings(hs.Properties), Vector: vector, Alpha: float64(hs.Alpha), FusionAlgorithm: fusionType} - } - - if bm25 := req.Bm25Search; bm25 != nil { - out.KeywordRanking = &searchparams.KeywordRanking{Query: bm25.Query, Properties: schema.LowercaseFirstLetterOfStrings(bm25.Properties), Type: "bm25", AdditionalExplanations: out.AdditionalProperties.ExplainScore} - } - - if nv := req.NearVector; nv != nil { - var vector []float32 - // bytes vector has precedent for being more efficient - if len(nv.VectorBytes) > 0 { - vector = byteops.Float32FromByteVector(nv.VectorBytes) - } else { - vector = nv.Vector - } - out.NearVector = &searchparams.NearVector{ - Vector: vector, - } - - // The following business logic should not sit in the API. However, it is - // also part of the GraphQL API, so we need to duplicate it in order to get - // the same behavior - if nv.Distance != nil && nv.Certainty != nil { - return out, fmt.Errorf("near_vector: cannot provide distance and certainty") - } - - if nv.Certainty != nil { - out.NearVector.Certainty = *nv.Certainty - } - - if nv.Distance != nil { - out.NearVector.Distance = *nv.Distance - out.NearVector.WithDistance = true - } - } - - if no := req.NearObject; no != nil { - out.NearObject = &searchparams.NearObject{ - ID: req.NearObject.Id, - } - - // The following business logic should not sit in the API. However, it is - // also part of the GraphQL API, so we need to duplicate it in order to get - // the same behavior - if no.Distance != nil && no.Certainty != nil { - return out, fmt.Errorf("near_object: cannot provide distance and certainty") - } - - if no.Certainty != nil { - out.NearObject.Certainty = *no.Certainty - } - - if no.Distance != nil { - out.NearObject.Distance = *no.Distance - out.NearObject.WithDistance = true - } - } - - if ni := req.NearImage; ni != nil { - nearImageOut, err := parseNearImage(ni) - if err != nil { - return dto.GetParams{}, err - } - - if out.ModuleParams == nil { - out.ModuleParams = make(map[string]interface{}) - } - out.ModuleParams["nearImage"] = nearImageOut - } - - if na := req.NearAudio; na != nil { - nearAudioOut, err := parseNearAudio(na) - if err != nil { - return dto.GetParams{}, err - } - if out.ModuleParams == nil { - out.ModuleParams = make(map[string]interface{}) - } - out.ModuleParams["nearAudio"] = nearAudioOut - } - - if nv := req.NearVideo; nv != nil { - nearVideoOut, err := parseNearVideo(nv) - if err != nil { - return dto.GetParams{}, err - } - if out.ModuleParams == nil { - out.ModuleParams = make(map[string]interface{}) - } - out.ModuleParams["nearVideo"] = nearVideoOut - } - - out.Pagination = &filters.Pagination{Offset: int(req.Offset), Autocut: int(req.Autocut)} - if req.Limit > 0 { - out.Pagination.Limit = int(req.Limit) - } else { - // TODO: align default with other APIs - out.Pagination.Limit = 10 - } - - if req.NearText != nil { - moveAwayOut, err := extractNearTextMove(req.Collection, req.NearText.MoveAway) - if err != nil { - return dto.GetParams{}, err - } - moveToOut, err := extractNearTextMove(req.Collection, req.NearText.MoveTo) - if err != nil { - return dto.GetParams{}, err - } - - nearText := &nearText2.NearTextParams{ - Values: req.NearText.Query, - Limit: out.Pagination.Limit, - MoveAwayFrom: moveAwayOut, - MoveTo: moveToOut, - } - - if req.NearText.Certainty != nil { - nearText.Certainty = *req.NearText.Certainty - } - if req.NearText.Distance != nil { - nearText.Distance = *req.NearText.Distance - nearText.WithDistance = true - } - if out.ModuleParams == nil { - out.ModuleParams = make(map[string]interface{}) - } - out.ModuleParams["nearText"] = nearText - } - - if req.Generative != nil { - if out.AdditionalProperties.ModuleParams == nil { - out.AdditionalProperties.ModuleParams = make(map[string]interface{}) - } - out.AdditionalProperties.ModuleParams["generate"] = extractGenerative(req) - } - - if req.Rerank != nil { - if out.AdditionalProperties.ModuleParams == nil { - out.AdditionalProperties.ModuleParams = make(map[string]interface{}) - } - out.AdditionalProperties.ModuleParams["rerank"] = extractRerank(req) - } - - if len(req.After) > 0 { - out.Cursor = &filters.Cursor{After: req.After, Limit: out.Pagination.Limit} - } - - if req.Filters != nil { - clause, err := extractFilters(req.Filters, scheme, req.Collection) - if err != nil { - return dto.GetParams{}, err - } - filter := &filters.LocalFilter{Root: &clause} - if err := filters.ValidateFilters(scheme, filter); err != nil { - return dto.GetParams{}, err - } - out.Filters = filter - } - - if len(req.SortBy) > 0 { - if req.NearText != nil || req.NearVideo != nil || req.NearAudio != nil || req.NearImage != nil || req.NearObject != nil || req.NearVector != nil || req.HybridSearch != nil || req.Bm25Search != nil || req.Generative != nil { - return dto.GetParams{}, errors.New("sorting cannot be combined with search") - } - out.Sort = extractSorting(req.SortBy) - } - - if req.GroupBy != nil { - groupBy, err := extractGroupBy(req.GroupBy, &out) - if err != nil { - return dto.GetParams{}, err - } - out.AdditionalProperties.Group = true - - out.GroupBy = groupBy - } - - return out, nil -} - -func extractGroupBy(groupIn *pb.GroupBy, out *dto.GetParams) (*searchparams.GroupBy, error) { - if len(groupIn.Path) != 1 { - return nil, fmt.Errorf("groupby path can only have one entry, received %v", groupIn.Path) - } - - groupOut := &searchparams.GroupBy{ - Property: groupIn.Path[0], - ObjectsPerGroup: int(groupIn.ObjectsPerGroup), - Groups: int(groupIn.NumberOfGroups), - } - - // add the property in case it was not requested as return prop - otherwise it is not resolved - if out.Properties.FindProperty(groupOut.Property) == nil { - out.Properties = append(out.Properties, search.SelectProperty{Name: groupOut.Property, IsPrimitive: true}) - } - out.AdditionalProperties.NoProps = false - - return groupOut, nil -} - -func extractSorting(sortIn []*pb.SortBy) []filters.Sort { - sortOut := make([]filters.Sort, len(sortIn)) - for i := range sortIn { - order := "asc" - if !sortIn[i].Ascending { - order = "desc" - } - sortOut[i] = filters.Sort{Order: order, Path: sortIn[i].Path} - } - return sortOut -} - -func extractGenerative(req *pb.SearchRequest) *generate.Params { - generative := generate.Params{} - if req.Generative.SingleResponsePrompt != "" { - generative.Prompt = &req.Generative.SingleResponsePrompt - } - if req.Generative.GroupedResponseTask != "" { - generative.Task = &req.Generative.GroupedResponseTask - } - if len(req.Generative.GroupedProperties) > 0 { - generative.Properties = req.Generative.GroupedProperties - } - return &generative -} - -func extractRerank(req *pb.SearchRequest) *rank.Params { - rerank := rank.Params{ - Property: &req.Rerank.Property, - } - if req.Rerank.Query != nil { - rerank.Query = req.Rerank.Query - } - return &rerank -} - -func extractNearTextMove(classname string, Move *pb.NearTextSearch_Move) (nearText2.ExploreMove, error) { - var moveAwayOut nearText2.ExploreMove - - if moveAwayReq := Move; moveAwayReq != nil { - moveAwayOut.Force = moveAwayReq.Force - if moveAwayReq.Uuids != nil && len(moveAwayReq.Uuids) > 0 { - moveAwayOut.Objects = make([]nearText2.ObjectMove, len(moveAwayReq.Uuids)) - for i, objUUid := range moveAwayReq.Uuids { - uuidFormat, err := uuid.Parse(objUUid) - if err != nil { - return moveAwayOut, err - } - moveAwayOut.Objects[i] = nearText2.ObjectMove{ - ID: objUUid, - Beacon: crossref.NewLocalhost(classname, strfmt.UUID(uuidFormat.String())).String(), - } - } - } - - moveAwayOut.Values = moveAwayReq.Concepts - } - return moveAwayOut, nil -} - -func extractPropertiesRequest(reqProps *pb.PropertiesRequest, scheme schema.Schema, className string, usesNewDefaultLogic bool) ([]search.SelectProperty, error) { - props := make([]search.SelectProperty, 0) - - if reqProps == nil { - // No properties selected at all, return all non-ref properties. - // Ignore blobs to not overload the response - nonRefProps, err := getAllNonRefNonBlobProperties(scheme, className) - if err != nil { - return nil, errors.Wrap(err, "get all non ref non blob properties") - } - return nonRefProps, nil - } - - if !usesNewDefaultLogic { - // Old stubs being used, use deprecated method - return extractPropertiesRequestDeprecated(reqProps, scheme, className) - } - - if reqProps.ReturnAllNonrefProperties { - // No non-ref return properties selected, return all non-ref properties. - // Ignore blobs to not overload the response - returnProps, err := getAllNonRefNonBlobProperties(scheme, className) - if err != nil { - return nil, errors.Wrap(err, "get all non ref non blob properties") - } - props = append(props, returnProps...) - } else if len(reqProps.NonRefProperties) > 0 { - // Non-ref properties are selected, return only those specified - // This catches the case where users send an empty list of non ref properties as their request, - // i.e. they want no non-ref properties - for _, prop := range reqProps.NonRefProperties { - props = append(props, search.SelectProperty{ - Name: schema.LowercaseFirstLetter(prop), - IsPrimitive: true, - IsObject: false, - }) - } - } - - if len(reqProps.RefProperties) > 0 { - class := scheme.GetClass(schema.ClassName(className)) - for _, prop := range reqProps.RefProperties { - normalizedRefPropName := schema.LowercaseFirstLetter(prop.ReferenceProperty) - schemaProp, err := schema.GetPropertyByName(class, normalizedRefPropName) - if err != nil { - return nil, err - } - - var linkedClassName string - if len(schemaProp.DataType) == 1 { - // use datatype of the reference property to get the name of the linked class - linkedClassName = schemaProp.DataType[0] - } else { - linkedClassName = prop.TargetCollection - if linkedClassName == "" { - return nil, fmt.Errorf( - "multi target references from collection %v and property %v with need an explicit"+ - "linked collection. Available linked collections are %v", - className, prop.ReferenceProperty, schemaProp.DataType) - } - } - var refProperties []search.SelectProperty - var addProps additional.Properties - if prop.Properties != nil { - refProperties, err = extractPropertiesRequest(prop.Properties, scheme, linkedClassName, usesNewDefaultLogic) - if err != nil { - return nil, errors.Wrap(err, "extract properties request") - } - } - if prop.Metadata != nil { - addProps, err = extractAdditionalPropsFromMetadata(class, prop.Metadata) - if err != nil { - return nil, errors.Wrap(err, "extract additional props for refs") - } - } - - if prop.Properties == nil { - refProperties, err = getAllNonRefNonBlobProperties(scheme, linkedClassName) - if err != nil { - return nil, errors.Wrap(err, "get all non ref non blob properties") - } - } - if len(refProperties) == 0 && isIdOnlyRequest(prop.Metadata) { - // This is a pure-ID query without any properties or additional metadata. - // Indicate this to the DB, so it can optimize accordingly - addProps.NoProps = true - } - - props = append(props, search.SelectProperty{ - Name: normalizedRefPropName, - IsPrimitive: false, - IsObject: false, - Refs: []search.SelectClass{{ - ClassName: linkedClassName, - RefProperties: refProperties, - AdditionalProperties: addProps, - }}, - }) - } - } - - if len(reqProps.ObjectProperties) > 0 { - props = append(props, extractNestedProperties(reqProps.ObjectProperties)...) - } - - return props, nil -} - -func extractPropertiesRequestDeprecated(reqProps *pb.PropertiesRequest, scheme schema.Schema, className string) ([]search.SelectProperty, error) { - if reqProps == nil { - return nil, nil - } - props := make([]search.SelectProperty, 0) - if reqProps.NonRefProperties != nil && len(reqProps.NonRefProperties) > 0 { - for _, prop := range reqProps.NonRefProperties { - props = append(props, search.SelectProperty{ - Name: schema.LowercaseFirstLetter(prop), - IsPrimitive: true, - IsObject: false, - }) - } - } - - if reqProps.RefProperties != nil && len(reqProps.RefProperties) > 0 { - class := scheme.GetClass(schema.ClassName(className)) - for _, prop := range reqProps.RefProperties { - normalizedRefPropName := schema.LowercaseFirstLetter(prop.ReferenceProperty) - schemaProp, err := schema.GetPropertyByName(class, normalizedRefPropName) - if err != nil { - return nil, err - } - - var linkedClassName string - if len(schemaProp.DataType) == 1 { - // use datatype of the reference property to get the name of the linked class - linkedClassName = schemaProp.DataType[0] - } else { - linkedClassName = prop.TargetCollection - if linkedClassName == "" { - return nil, fmt.Errorf( - "multi target references from collection %v and property %v with need an explicit"+ - "linked collection. Available linked collections are %v", - className, prop.ReferenceProperty, schemaProp.DataType) - } - } - var refProperties []search.SelectProperty - var addProps additional.Properties - if prop.Properties != nil { - refProperties, err = extractPropertiesRequestDeprecated(prop.Properties, scheme, linkedClassName) - if err != nil { - return nil, errors.Wrap(err, "extract properties request") - } - } - if prop.Metadata != nil { - addProps, err = extractAdditionalPropsFromMetadata(class, prop.Metadata) - if err != nil { - return nil, errors.Wrap(err, "extract additional props for refs") - } - } - - if prop.Properties == nil { - refProperties, err = getAllNonRefNonBlobProperties(scheme, linkedClassName) - if err != nil { - return nil, errors.Wrap(err, "get all non ref non blob properties") - } - } - if len(refProperties) == 0 && isIdOnlyRequest(prop.Metadata) { - // This is a pure-ID query without any properties or additional metadata. - // Indicate this to the DB, so it can optimize accordingly - addProps.NoProps = true - } - - props = append(props, search.SelectProperty{ - Name: normalizedRefPropName, - IsPrimitive: false, - IsObject: false, - Refs: []search.SelectClass{{ - ClassName: linkedClassName, - RefProperties: refProperties, - AdditionalProperties: addProps, - }}, - }) - } - } - - if reqProps.ObjectProperties != nil && len(reqProps.ObjectProperties) > 0 { - props = append(props, extractNestedProperties(reqProps.ObjectProperties)...) - } - - return props, nil -} - -func extractNestedProperties(props []*pb.ObjectPropertiesRequest) []search.SelectProperty { - selectProps := make([]search.SelectProperty, 0) - for _, prop := range props { - nestedProps := make([]search.SelectProperty, 0) - if prop.PrimitiveProperties != nil && len(prop.PrimitiveProperties) > 0 { - for _, primitive := range prop.PrimitiveProperties { - nestedProps = append(nestedProps, search.SelectProperty{ - Name: schema.LowercaseFirstLetter(primitive), - IsPrimitive: true, - IsObject: false, - }) - } - } - if prop.ObjectProperties != nil && len(prop.ObjectProperties) > 0 { - nestedProps = append(nestedProps, extractNestedProperties(prop.ObjectProperties)...) - } - selectProps = append(selectProps, search.SelectProperty{ - Name: schema.LowercaseFirstLetter(prop.PropName), - IsPrimitive: false, - IsObject: true, - Props: nestedProps, - }) - } - return selectProps -} - -func extractAdditionalPropsFromMetadata(class *models.Class, prop *pb.MetadataRequest) (additional.Properties, error) { - props := additional.Properties{ - Vector: prop.Vector, - ID: prop.Uuid, - CreationTimeUnix: prop.CreationTimeUnix, - LastUpdateTimeUnix: prop.LastUpdateTimeUnix, - Distance: prop.Distance, - Score: prop.Score, - ExplainScore: prop.ExplainScore, - IsConsistent: prop.IsConsistent, - } - - vectorIndex, err := schema.TypeAssertVectorIndex(class) - if err != nil { - return props, err - } - - // certainty is only compatible with cosine distance - if vectorIndex.DistanceName() == common.DistanceCosine && prop.Certainty { - props.Certainty = true - } else { - props.Certainty = false - } - - return props, nil -} - -func isIdOnlyRequest(metadata *pb.MetadataRequest) bool { - // could also use reflect here but this is more explicit - return (metadata != nil && - metadata.Uuid && - !metadata.Vector && - !metadata.CreationTimeUnix && - !metadata.LastUpdateTimeUnix && - !metadata.Distance && - !metadata.Certainty && - !metadata.Score && - !metadata.ExplainScore && - !metadata.IsConsistent) -} - -func getAllNonRefNonBlobProperties(scheme schema.Schema, className string) ([]search.SelectProperty, error) { - var props []search.SelectProperty - class := scheme.GetClass(schema.ClassName(className)) - - for _, prop := range class.Properties { - dt, err := schema.GetPropertyDataType(class, prop.Name) - if err != nil { - return []search.SelectProperty{}, errors.Wrap(err, "get property data type") - } - if *dt == schema.DataTypeCRef || *dt == schema.DataTypeBlob { - continue - } - if *dt == schema.DataTypeObject || *dt == schema.DataTypeObjectArray { - nested, err := schema.GetPropertyByName(class, prop.Name) - if err != nil { - return []search.SelectProperty{}, errors.Wrap(err, "get nested property by name") - } - nestedProps, err := getAllNonRefNonBlobNestedProperties(&Property{Property: nested}) - if err != nil { - return []search.SelectProperty{}, errors.Wrap(err, "get all non ref non blob nested properties") - } - props = append(props, search.SelectProperty{ - Name: prop.Name, - IsPrimitive: false, - IsObject: true, - Props: nestedProps, - }) - } else { - props = append(props, search.SelectProperty{ - Name: prop.Name, - IsPrimitive: true, - }) - } - } - return props, nil -} - -func getAllNonRefNonBlobNestedProperties[P schema.PropertyInterface](property P) ([]search.SelectProperty, error) { - var props []search.SelectProperty - for _, prop := range property.GetNestedProperties() { - dt, err := schema.GetNestedPropertyDataType(property, prop.Name) - if err != nil { - return []search.SelectProperty{}, errors.Wrap(err, "get nested property data type") - } - if *dt == schema.DataTypeCRef || *dt == schema.DataTypeBlob { - continue - } - if *dt == schema.DataTypeObject || *dt == schema.DataTypeObjectArray { - nested, err := schema.GetNestedPropertyByName(property, prop.Name) - if err != nil { - return []search.SelectProperty{}, errors.Wrap(err, "get nested property by name") - } - nestedProps, err := getAllNonRefNonBlobNestedProperties(&NestedProperty{NestedProperty: nested}) - if err != nil { - return []search.SelectProperty{}, errors.Wrap(err, "get all non ref non blob nested properties") - } - props = append(props, search.SelectProperty{ - Name: prop.Name, - IsPrimitive: false, - IsObject: true, - Props: nestedProps, - }) - } else { - props = append(props, search.SelectProperty{ - Name: prop.Name, - IsPrimitive: true, - }) - } - } - return props, nil -} - -func parseNearImage(ni *pb.NearImageSearch) (*nearImage.NearImageParams, error) { - nearImageOut := &nearImage.NearImageParams{ - Image: ni.Image, - } - - // The following business logic should not sit in the API. However, it is - // also part of the GraphQL API, so we need to duplicate it in order to get - // the same behavior - if ni.Distance != nil && ni.Certainty != nil { - return nil, fmt.Errorf("near_image: cannot provide distance and certainty") - } - - if ni.Certainty != nil { - nearImageOut.Certainty = *ni.Certainty - } - - if ni.Distance != nil { - nearImageOut.Distance = *ni.Distance - nearImageOut.WithDistance = true - } - - return nearImageOut, nil -} - -func parseNearAudio(na *pb.NearAudioSearch) (*nearAudio.NearAudioParams, error) { - nearAudioOut := &nearAudio.NearAudioParams{ - Audio: na.Audio, - } - - // The following business logic should not sit in the API. However, it is - // also part of the GraphQL API, so we need to duplicate it in order to get - // the same behavior - if na.Distance != nil && na.Certainty != nil { - return nil, fmt.Errorf("near_audio: cannot provide distance and certainty") - } - - if na.Certainty != nil { - nearAudioOut.Certainty = *na.Certainty - } - - if na.Distance != nil { - nearAudioOut.Distance = *na.Distance - nearAudioOut.WithDistance = true - } - - return nearAudioOut, nil -} - -func parseNearVideo(nv *pb.NearVideoSearch) (*nearVideo.NearVideoParams, error) { - nearVideoOut := &nearVideo.NearVideoParams{ - Video: nv.Video, - } - - // The following business logic should not sit in the API. However, it is - // also part of the GraphQL API, so we need to duplicate it in order to get - // the same behavior - if nv.Distance != nil && nv.Certainty != nil { - return nil, fmt.Errorf("near_video: cannot provide distance and certainty") - } - - if nv.Certainty != nil { - nearVideoOut.Certainty = *nv.Certainty - } - - if nv.Distance != nil { - nearVideoOut.Distance = *nv.Distance - nearVideoOut.WithDistance = true - } - - return nearVideoOut, nil -} diff --git a/adapters/handlers/grpc/v1/parse_search_request_test.go b/adapters/handlers/grpc/v1/parse_search_request_test.go deleted file mode 100644 index 088f81354bc02e09cac2c71ff2401936e68ed3f2..0000000000000000000000000000000000000000 --- a/adapters/handlers/grpc/v1/parse_search_request_test.go +++ /dev/null @@ -1,1136 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package v1 - -import ( - "testing" - - "github.com/weaviate/weaviate/entities/vectorindex/hnsw" - - "github.com/weaviate/weaviate/usecases/modulecomponents/additional/generate" - "github.com/weaviate/weaviate/usecases/modulecomponents/additional/rank" - - "github.com/weaviate/weaviate/usecases/modulecomponents/nearAudio" - "github.com/weaviate/weaviate/usecases/modulecomponents/nearImage" - "github.com/weaviate/weaviate/usecases/modulecomponents/nearVideo" - - "github.com/weaviate/weaviate/entities/schema/crossref" - nearText2 "github.com/weaviate/weaviate/usecases/modulecomponents/nearText" - - "github.com/weaviate/weaviate/adapters/handlers/graphql/local/common_filters" - "github.com/weaviate/weaviate/entities/searchparams" - vectorIndex "github.com/weaviate/weaviate/entities/vectorindex/common" - - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/dto" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/search" - - pb "github.com/weaviate/weaviate/grpc/generated/protocol/v1" -) - -func TestGRPCRequest(t *testing.T) { - classname := "TestClass" - refClass1 := "OtherClass" - refClass2 := "AnotherClass" - dotClass := "DotClass" - objClass := "ObjClass" - - defaultTestClassProps := search.SelectProperties{{Name: "name", IsPrimitive: true}, {Name: "number", IsPrimitive: true}, {Name: "floats", IsPrimitive: true}, {Name: "uuid", IsPrimitive: true}} - - scheme := schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{ - { - Class: classname, - Properties: []*models.Property{ - {Name: "name", DataType: schema.DataTypeText.PropString()}, - {Name: "number", DataType: schema.DataTypeInt.PropString()}, - {Name: "floats", DataType: schema.DataTypeNumberArray.PropString()}, - {Name: "uuid", DataType: schema.DataTypeUUID.PropString()}, - {Name: "ref", DataType: []string{refClass1}}, - {Name: "multiRef", DataType: []string{refClass1, refClass2}}, - }, - VectorIndexConfig: hnsw.UserConfig{Distance: vectorIndex.DefaultDistanceMetric}, - }, - { - Class: refClass1, - Properties: []*models.Property{ - {Name: "something", DataType: schema.DataTypeText.PropString()}, - {Name: "somethings", DataType: schema.DataTypeTextArray.PropString()}, - {Name: "ref2", DataType: []string{refClass2}}, - }, - VectorIndexConfig: hnsw.UserConfig{Distance: vectorIndex.DefaultDistanceMetric}, - }, - { - Class: refClass2, - Properties: []*models.Property{ - {Name: "else", DataType: schema.DataTypeText.PropString()}, - {Name: "ref3", DataType: []string{refClass2}}, - }, - }, - { - Class: dotClass, - Properties: []*models.Property{ - {Name: "something", DataType: schema.DataTypeText.PropString()}, - }, - VectorIndexConfig: hnsw.UserConfig{Distance: vectorIndex.DistanceDot}, - }, - { - Class: objClass, - Properties: []*models.Property{ - { - Name: "something", - DataType: schema.DataTypeObject.PropString(), - NestedProperties: []*models.NestedProperty{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - }, - { - Name: "else", - DataType: schema.DataTypeObject.PropString(), - NestedProperties: []*models.NestedProperty{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - }, - }, - }, - { - Name: "elses", - DataType: schema.DataTypeObjectArray.PropString(), - NestedProperties: []*models.NestedProperty{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - }, - }, - }, - }, - }, - }, - VectorIndexConfig: hnsw.UserConfig{Distance: vectorIndex.DefaultDistanceMetric}, - }, - }, - }, - } - defaultPagination := &filters.Pagination{Limit: 10} - quorum := pb.ConsistencyLevel_CONSISTENCY_LEVEL_QUORUM - someString1 := "a word" - someString2 := "other" - - tests := []struct { - name string - req *pb.SearchRequest - out dto.GetParams - error bool - }{ - { - name: "No classname", - req: &pb.SearchRequest{}, - out: dto.GetParams{}, - error: true, - }, - { - name: "No return values given", - req: &pb.SearchRequest{Collection: classname}, - out: dto.GetParams{ - ClassName: classname, Pagination: defaultPagination, Properties: defaultTestClassProps, - }, - error: false, - }, - { - name: "Empty return properties given", - req: &pb.SearchRequest{Collection: classname, Properties: &pb.PropertiesRequest{}}, - out: dto.GetParams{ - ClassName: classname, Pagination: defaultPagination, Properties: search.SelectProperties{}, AdditionalProperties: additional.Properties{ - NoProps: true, - }, - }, - error: false, - }, - { - name: "Empty return properties given with new default logic", - req: &pb.SearchRequest{Uses_123Api: true, Collection: classname, Properties: &pb.PropertiesRequest{}}, - out: dto.GetParams{ - ClassName: classname, Pagination: defaultPagination, Properties: search.SelectProperties{}, AdditionalProperties: additional.Properties{ - NoProps: true, - }, - }, - error: false, - }, - { - name: "No return values given for dot distance", - req: &pb.SearchRequest{Collection: dotClass}, - out: dto.GetParams{ - ClassName: dotClass, Pagination: defaultPagination, Properties: search.SelectProperties{{Name: "something", IsPrimitive: true}}, - }, - error: false, - }, - { - name: "Metadata return values", - req: &pb.SearchRequest{Collection: classname, Metadata: &pb.MetadataRequest{Vector: true, Certainty: false, IsConsistent: true}}, - out: dto.GetParams{ - ClassName: classname, Pagination: defaultPagination, - Properties: defaultTestClassProps, - AdditionalProperties: additional.Properties{ - Vector: true, - NoProps: false, - IsConsistent: true, - }, - }, - error: false, - }, - { - name: "Metadata ID only query", - req: &pb.SearchRequest{Collection: classname, Properties: &pb.PropertiesRequest{}, Metadata: &pb.MetadataRequest{Uuid: true}}, - out: dto.GetParams{ - ClassName: classname, Pagination: defaultPagination, - Properties: search.SelectProperties{}, - AdditionalProperties: additional.Properties{ - ID: true, - NoProps: true, - }, - }, - error: false, - }, - { - name: "Metadata ID only query using new default logic", - req: &pb.SearchRequest{Uses_123Api: true, Collection: classname, Properties: &pb.PropertiesRequest{}, Metadata: &pb.MetadataRequest{Uuid: true}}, - out: dto.GetParams{ - ClassName: classname, Pagination: defaultPagination, - Properties: search.SelectProperties{}, - AdditionalProperties: additional.Properties{ - ID: true, - NoProps: true, - }, - }, - error: false, - }, - { - name: "Properties return all nonref values", - req: &pb.SearchRequest{Collection: classname}, - out: dto.GetParams{ - ClassName: classname, Pagination: defaultPagination, Properties: defaultTestClassProps, - }, - error: false, - }, - { - name: "Properties return all nonref values with new default logic", - req: &pb.SearchRequest{Uses_123Api: true, Collection: classname, Properties: &pb.PropertiesRequest{ReturnAllNonrefProperties: true}}, - out: dto.GetParams{ - ClassName: classname, Pagination: defaultPagination, Properties: defaultTestClassProps, - }, - error: false, - }, - { - name: "Properties return all nonref values with ref and specific props using new default logic", - req: &pb.SearchRequest{Uses_123Api: true, Collection: classname, Properties: &pb.PropertiesRequest{ - ReturnAllNonrefProperties: true, - RefProperties: []*pb.RefPropertiesRequest{{ - ReferenceProperty: "ref", - TargetCollection: refClass1, - Metadata: &pb.MetadataRequest{Vector: true, Certainty: false}, - Properties: &pb.PropertiesRequest{NonRefProperties: []string{"something"}}, - }}, - }}, - out: dto.GetParams{ - ClassName: classname, Pagination: defaultPagination, Properties: search.SelectProperties{ - {Name: "name", IsPrimitive: true}, - {Name: "number", IsPrimitive: true}, - {Name: "floats", IsPrimitive: true}, - {Name: "uuid", IsPrimitive: true}, - {Name: "ref", IsPrimitive: false, Refs: []search.SelectClass{ - { - ClassName: refClass1, - RefProperties: search.SelectProperties{{Name: "something", IsPrimitive: true}}, - AdditionalProperties: additional.Properties{Vector: true}, - }, - }}, - }, - }, - error: false, - }, - { - name: "Properties return all nonref values with ref and all nonref props using new default logic", - req: &pb.SearchRequest{Uses_123Api: true, Collection: classname, Properties: &pb.PropertiesRequest{ - ReturnAllNonrefProperties: true, - RefProperties: []*pb.RefPropertiesRequest{{ - ReferenceProperty: "ref", - TargetCollection: refClass1, - Metadata: &pb.MetadataRequest{Vector: true, Certainty: false}, - Properties: &pb.PropertiesRequest{ReturnAllNonrefProperties: true}, - }}, - }}, - out: dto.GetParams{ - ClassName: classname, Pagination: defaultPagination, Properties: search.SelectProperties{ - {Name: "name", IsPrimitive: true}, - {Name: "number", IsPrimitive: true}, - {Name: "floats", IsPrimitive: true}, - {Name: "uuid", IsPrimitive: true}, - {Name: "ref", IsPrimitive: false, Refs: []search.SelectClass{ - { - ClassName: refClass1, - RefProperties: search.SelectProperties{ - {Name: "something", IsPrimitive: true}, - {Name: "somethings", IsPrimitive: true}, - }, - AdditionalProperties: additional.Properties{Vector: true}, - }, - }}, - }, - }, - error: false, - }, - { - name: "Properties return values only ref", - req: &pb.SearchRequest{Collection: classname, Properties: &pb.PropertiesRequest{ - RefProperties: []*pb.RefPropertiesRequest{ - { - ReferenceProperty: "ref", - TargetCollection: refClass1, - Metadata: &pb.MetadataRequest{Vector: true, Certainty: false}, - Properties: &pb.PropertiesRequest{NonRefProperties: []string{"something"}}, - }, - }, - }}, - out: dto.GetParams{ - ClassName: classname, Pagination: defaultPagination, Properties: search.SelectProperties{{Name: "ref", IsPrimitive: false, Refs: []search.SelectClass{{ClassName: refClass1, RefProperties: search.SelectProperties{{Name: "something", IsPrimitive: true}}, AdditionalProperties: additional.Properties{ - Vector: true, - }}}}}, - }, - error: false, - }, - { - name: "Properties return values only ref using new default logic", - req: &pb.SearchRequest{Uses_123Api: true, Collection: classname, Properties: &pb.PropertiesRequest{ - RefProperties: []*pb.RefPropertiesRequest{ - { - ReferenceProperty: "ref", - TargetCollection: refClass1, - Metadata: &pb.MetadataRequest{Vector: true, Certainty: false}, - Properties: &pb.PropertiesRequest{NonRefProperties: []string{"something"}}, - }, - }, - }}, - out: dto.GetParams{ - ClassName: classname, Pagination: defaultPagination, Properties: search.SelectProperties{{Name: "ref", IsPrimitive: false, Refs: []search.SelectClass{{ClassName: refClass1, RefProperties: search.SelectProperties{{Name: "something", IsPrimitive: true}}, AdditionalProperties: additional.Properties{ - Vector: true, - }}}}}, - }, - error: false, - }, - { - name: "Properties return values non-ref", - req: &pb.SearchRequest{Collection: classname, Properties: &pb.PropertiesRequest{NonRefProperties: []string{"name", "CapitalizedName"}}}, - out: dto.GetParams{ - ClassName: classname, Pagination: defaultPagination, Properties: search.SelectProperties{{Name: "name", IsPrimitive: true}, {Name: "capitalizedName", IsPrimitive: true}}, - }, - error: false, - }, - { - name: "Properties return values non-ref with new default logic", - req: &pb.SearchRequest{Uses_123Api: true, Collection: classname, Properties: &pb.PropertiesRequest{NonRefProperties: []string{"name", "CapitalizedName"}}}, - out: dto.GetParams{ - ClassName: classname, Pagination: defaultPagination, Properties: search.SelectProperties{{Name: "name", IsPrimitive: true}, {Name: "capitalizedName", IsPrimitive: true}}, - }, - error: false, - }, - { - name: "ref returns no values given", - req: &pb.SearchRequest{Collection: classname, Properties: &pb.PropertiesRequest{RefProperties: []*pb.RefPropertiesRequest{{ReferenceProperty: "ref", TargetCollection: refClass1}}}}, - out: dto.GetParams{ - ClassName: classname, Pagination: defaultPagination, Properties: search.SelectProperties{{Name: "ref", IsPrimitive: false, Refs: []search.SelectClass{{ClassName: refClass1, RefProperties: search.SelectProperties{{Name: "something", IsPrimitive: true}, {Name: "somethings", IsPrimitive: true}}}}}}, - }, - error: false, - }, - { - name: "Properties return values multi-ref (no linked class with error)", - req: &pb.SearchRequest{Collection: classname, Properties: &pb.PropertiesRequest{RefProperties: []*pb.RefPropertiesRequest{{ReferenceProperty: "multiRef", Metadata: &pb.MetadataRequest{Vector: true, Certainty: false}, Properties: &pb.PropertiesRequest{NonRefProperties: []string{"something"}}}}}}, - out: dto.GetParams{}, - error: true, - }, - { - name: "Properties return values multi-ref", - req: &pb.SearchRequest{Collection: classname, Properties: &pb.PropertiesRequest{RefProperties: []*pb.RefPropertiesRequest{ - {ReferenceProperty: "multiRef", TargetCollection: refClass1, Metadata: &pb.MetadataRequest{Vector: true, Certainty: false}, Properties: &pb.PropertiesRequest{NonRefProperties: []string{"something"}}}, - {ReferenceProperty: "MultiRef", TargetCollection: refClass2, Metadata: &pb.MetadataRequest{Uuid: true}, Properties: &pb.PropertiesRequest{NonRefProperties: []string{"Else"}}}, - }}}, - out: dto.GetParams{ - ClassName: classname, Pagination: defaultPagination, Properties: search.SelectProperties{ - {Name: "multiRef", IsPrimitive: false, Refs: []search.SelectClass{{ClassName: refClass1, RefProperties: search.SelectProperties{{Name: "something", IsPrimitive: true}}, AdditionalProperties: additional.Properties{Vector: true}}}}, - {Name: "multiRef", IsPrimitive: false, Refs: []search.SelectClass{{ClassName: refClass2, RefProperties: search.SelectProperties{{Name: "else", IsPrimitive: true}}, AdditionalProperties: additional.Properties{ID: true}}}}, - }, - }, - error: false, - }, - { - name: "hybrid ranked", - req: &pb.SearchRequest{ - Collection: classname, Metadata: &pb.MetadataRequest{Vector: true, Certainty: false}, - HybridSearch: &pb.Hybrid{Query: "query", FusionType: pb.Hybrid_FUSION_TYPE_RANKED, Alpha: 0.75, Properties: []string{"name", "CapitalizedName"}}, - }, - out: dto.GetParams{ - ClassName: classname, Pagination: defaultPagination, HybridSearch: &searchparams.HybridSearch{Query: "query", FusionAlgorithm: common_filters.HybridRankedFusion, Alpha: 0.75, Properties: []string{"name", "capitalizedName"}}, - Properties: defaultTestClassProps, - AdditionalProperties: additional.Properties{Vector: true, NoProps: false}, - }, - error: false, - }, - { - name: "hybrid relative", - req: &pb.SearchRequest{ - Collection: classname, Metadata: &pb.MetadataRequest{Vector: true, Certainty: false}, - HybridSearch: &pb.Hybrid{Query: "query", FusionType: pb.Hybrid_FUSION_TYPE_RELATIVE_SCORE}, - }, - out: dto.GetParams{ - ClassName: classname, Pagination: defaultPagination, HybridSearch: &searchparams.HybridSearch{Query: "query", FusionAlgorithm: common_filters.HybridRelativeScoreFusion}, - Properties: defaultTestClassProps, - AdditionalProperties: additional.Properties{Vector: true, NoProps: false}, - }, - error: false, - }, - { - name: "hybrid default", - req: &pb.SearchRequest{ - Collection: classname, Metadata: &pb.MetadataRequest{Vector: true, Certainty: false}, - HybridSearch: &pb.Hybrid{Query: "query"}, - }, - out: dto.GetParams{ - ClassName: classname, Pagination: defaultPagination, HybridSearch: &searchparams.HybridSearch{Query: "query", FusionAlgorithm: common_filters.HybridRelativeScoreFusion}, - Properties: defaultTestClassProps, - AdditionalProperties: additional.Properties{Vector: true, NoProps: false}, - }, - error: false, - }, - { - name: "bm25", - req: &pb.SearchRequest{ - Collection: classname, Metadata: &pb.MetadataRequest{Vector: true}, - Bm25Search: &pb.BM25{Query: "query", Properties: []string{"name", "CapitalizedName"}}, - }, - out: dto.GetParams{ - ClassName: classname, Pagination: defaultPagination, - KeywordRanking: &searchparams.KeywordRanking{Query: "query", Properties: []string{"name", "capitalizedName"}, Type: "bm25"}, - Properties: defaultTestClassProps, - AdditionalProperties: additional.Properties{Vector: true, NoProps: false}, - }, - error: false, - }, - { - name: "filter simple", - req: &pb.SearchRequest{ - Collection: classname, Metadata: &pb.MetadataRequest{Vector: true}, - Filters: &pb.Filters{Operator: pb.Filters_OPERATOR_EQUAL, TestValue: &pb.Filters_ValueText{ValueText: "test"}, On: []string{"name"}}, - }, - out: dto.GetParams{ - ClassName: classname, Pagination: defaultPagination, - Properties: defaultTestClassProps, - AdditionalProperties: additional.Properties{Vector: true, NoProps: false}, - Filters: &filters.LocalFilter{ - Root: &filters.Clause{ - On: &filters.Path{Class: schema.ClassName(classname), Property: "name"}, - Operator: filters.OperatorEqual, - Value: &filters.Value{Value: "test", Type: schema.DataTypeText}, - }, - }, - }, - error: false, - }, - { - name: "filter simple (new type)", - req: &pb.SearchRequest{ - Collection: classname, Metadata: &pb.MetadataRequest{Vector: true}, - Filters: &pb.Filters{Operator: pb.Filters_OPERATOR_EQUAL, TestValue: &pb.Filters_ValueText{ValueText: "test"}, Target: &pb.FilterTarget{Target: &pb.FilterTarget_Property{Property: "name"}}}, - }, - out: dto.GetParams{ - ClassName: classname, Pagination: defaultPagination, - Properties: defaultTestClassProps, - AdditionalProperties: additional.Properties{Vector: true, NoProps: false}, - Filters: &filters.LocalFilter{ - Root: &filters.Clause{ - On: &filters.Path{Class: schema.ClassName(classname), Property: "name"}, - Operator: filters.OperatorEqual, - Value: &filters.Value{Value: "test", Type: schema.DataTypeText}, - }, - }, - }, - error: false, - }, - { - name: "filter uuid", - req: &pb.SearchRequest{ - Collection: classname, Metadata: &pb.MetadataRequest{Vector: true}, - Filters: &pb.Filters{Operator: pb.Filters_OPERATOR_EQUAL, TestValue: &pb.Filters_ValueText{ValueText: UUID3}, On: []string{"uuid"}}, - }, - out: dto.GetParams{ - ClassName: classname, Pagination: defaultPagination, - Properties: defaultTestClassProps, - AdditionalProperties: additional.Properties{Vector: true, NoProps: false}, - Filters: &filters.LocalFilter{ - Root: &filters.Clause{ - On: &filters.Path{Class: schema.ClassName(classname), Property: "uuid"}, - Operator: filters.OperatorEqual, - Value: &filters.Value{Value: UUID3, Type: schema.DataTypeText}, - }, - }, - }, - error: false, - }, - { - name: "filter or", - req: &pb.SearchRequest{ - Collection: classname, Metadata: &pb.MetadataRequest{Vector: true}, - Filters: &pb.Filters{Operator: pb.Filters_OPERATOR_OR, Filters: []*pb.Filters{ - {Operator: pb.Filters_OPERATOR_EQUAL, TestValue: &pb.Filters_ValueText{ValueText: "test"}, On: []string{"name"}}, - {Operator: pb.Filters_OPERATOR_NOT_EQUAL, TestValue: &pb.Filters_ValueText{ValueText: "other"}, On: []string{"name"}}, - }}, - }, - out: dto.GetParams{ - ClassName: classname, Pagination: defaultPagination, - Properties: defaultTestClassProps, - AdditionalProperties: additional.Properties{Vector: true, NoProps: false}, - Filters: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorOr, - Operands: []filters.Clause{ - { - Value: &filters.Value{Value: "test", Type: schema.DataTypeText}, - On: &filters.Path{Class: schema.ClassName(classname), Property: "name"}, - Operator: filters.OperatorEqual, - }, - { - Value: &filters.Value{Value: "other", Type: schema.DataTypeText}, - On: &filters.Path{Class: schema.ClassName(classname), Property: "name"}, - Operator: filters.OperatorNotEqual, - }, - }, - }, - }, - }, - error: false, - }, - { - name: "filter reference", - req: &pb.SearchRequest{ - Collection: classname, Metadata: &pb.MetadataRequest{Vector: true}, - Filters: &pb.Filters{Operator: pb.Filters_OPERATOR_LESS_THAN, TestValue: &pb.Filters_ValueText{ValueText: "test"}, On: []string{"ref", refClass1, "something"}}, - }, - out: dto.GetParams{ - ClassName: classname, Pagination: defaultPagination, - Properties: defaultTestClassProps, - AdditionalProperties: additional.Properties{Vector: true, NoProps: false}, - Filters: &filters.LocalFilter{ - Root: &filters.Clause{ - On: &filters.Path{ - Class: schema.ClassName(classname), - Property: "ref", - Child: &filters.Path{Class: schema.ClassName(refClass1), Property: "something"}, - }, - Operator: filters.OperatorLessThan, - Value: &filters.Value{Value: "test", Type: schema.DataTypeText}, - }, - }, - }, - error: false, - }, - { - name: "filter reference (new filters)", - req: &pb.SearchRequest{ - Collection: classname, Metadata: &pb.MetadataRequest{Vector: true}, - Filters: &pb.Filters{Operator: pb.Filters_OPERATOR_LESS_THAN, TestValue: &pb.Filters_ValueText{ValueText: "test"}, Target: &pb.FilterTarget{Target: &pb.FilterTarget_SingleTarget{SingleTarget: &pb.FilterReferenceSingleTarget{On: "ref", Target: &pb.FilterTarget{Target: &pb.FilterTarget_Property{Property: "something"}}}}}}, - }, - out: dto.GetParams{ - ClassName: classname, Pagination: defaultPagination, - Properties: defaultTestClassProps, - AdditionalProperties: additional.Properties{Vector: true, NoProps: false}, - Filters: &filters.LocalFilter{ - Root: &filters.Clause{ - On: &filters.Path{ - Class: schema.ClassName(classname), - Property: "ref", - Child: &filters.Path{Class: schema.ClassName(refClass1), Property: "something"}, - }, - Operator: filters.OperatorLessThan, - Value: &filters.Value{Value: "test", Type: schema.DataTypeText}, - }, - }, - }, - error: false, - }, - { - name: "nested ref", - req: &pb.SearchRequest{ - Collection: classname, Metadata: &pb.MetadataRequest{Vector: true}, - Filters: &pb.Filters{Operator: pb.Filters_OPERATOR_LESS_THAN, TestValue: &pb.Filters_ValueText{ValueText: "test"}, On: []string{"ref", refClass1, "ref2", refClass2, "ref3", refClass2, "else"}}, - }, - out: dto.GetParams{ - ClassName: classname, Pagination: defaultPagination, - Properties: defaultTestClassProps, - AdditionalProperties: additional.Properties{Vector: true, NoProps: false}, - Filters: &filters.LocalFilter{ - Root: &filters.Clause{ - On: &filters.Path{ - Class: schema.ClassName(classname), - Property: "ref", - Child: &filters.Path{ - Class: schema.ClassName(refClass1), - Property: "ref2", - Child: &filters.Path{ - Class: schema.ClassName(refClass2), - Property: "ref3", - Child: &filters.Path{ - Class: schema.ClassName(refClass2), - Property: "else", - }, - }, - }, - }, - Operator: filters.OperatorLessThan, - Value: &filters.Value{Value: "test", Type: schema.DataTypeText}, - }, - }, - }, - error: false, - }, - { - name: "filter reference on array prop with contains", - req: &pb.SearchRequest{ - Collection: classname, Metadata: &pb.MetadataRequest{Vector: true}, - Filters: &pb.Filters{Operator: pb.Filters_OPERATOR_CONTAINS_ANY, TestValue: &pb.Filters_ValueTextArray{ValueTextArray: &pb.TextArray{Values: []string{"text"}}}, On: []string{"ref", refClass1, "somethings"}}, - }, - out: dto.GetParams{ - ClassName: classname, Pagination: defaultPagination, - Properties: defaultTestClassProps, - AdditionalProperties: additional.Properties{Vector: true, NoProps: false}, - Filters: &filters.LocalFilter{ - Root: &filters.Clause{ - On: &filters.Path{ - Class: schema.ClassName(classname), - Property: "ref", - Child: &filters.Path{ - Class: schema.ClassName(refClass1), - Property: "somethings", - }, - }, - Operator: filters.ContainsAny, - Value: &filters.Value{Value: []string{"text"}, Type: schema.DataTypeText}, - }, - }, - }, - error: false, - }, - { - name: "filter reference", - req: &pb.SearchRequest{ - Collection: classname, - Filters: &pb.Filters{ - Operator: pb.Filters_OPERATOR_LESS_THAN, - TestValue: &pb.Filters_ValueText{ValueText: "test"}, - On: []string{"ref", refClass1}, // two values do not work, property is missing - }, - }, - out: dto.GetParams{}, - error: true, - }, - { - name: "length filter ref", - req: &pb.SearchRequest{ - Collection: classname, Metadata: &pb.MetadataRequest{Vector: true}, - Filters: &pb.Filters{ - Operator: pb.Filters_OPERATOR_LESS_THAN, - TestValue: &pb.Filters_ValueInt{ValueInt: 3}, - On: []string{"ref", refClass1, "len(something)"}, - }, - }, - out: dto.GetParams{ - ClassName: classname, Pagination: defaultPagination, - Properties: defaultTestClassProps, - AdditionalProperties: additional.Properties{Vector: true, NoProps: false}, - Filters: &filters.LocalFilter{ - Root: &filters.Clause{ - On: &filters.Path{ - Class: schema.ClassName(classname), - Property: "ref", - Child: &filters.Path{ - Class: schema.ClassName(refClass1), - Property: "len(something)", - }, - }, - Operator: filters.OperatorLessThan, - Value: &filters.Value{Value: 3, Type: schema.DataTypeInt}, - }, - }, - }, - error: false, - }, - { - name: "length filter", - req: &pb.SearchRequest{ - Collection: classname, Metadata: &pb.MetadataRequest{Vector: true}, - Filters: &pb.Filters{ - Operator: pb.Filters_OPERATOR_LESS_THAN, - TestValue: &pb.Filters_ValueInt{ValueInt: 3}, - On: []string{"len(name)"}, - }, - }, - out: dto.GetParams{ - ClassName: classname, Pagination: defaultPagination, - Properties: defaultTestClassProps, - AdditionalProperties: additional.Properties{Vector: true, NoProps: false}, - Filters: &filters.LocalFilter{ - Root: &filters.Clause{ - On: &filters.Path{ - Class: schema.ClassName(classname), - Property: "len(name)", - }, - Operator: filters.OperatorLessThan, - Value: &filters.Value{Value: 3, Type: schema.DataTypeInt}, - }, - }, - }, - error: false, - }, - { - name: "contains filter with int value on float prop", - req: &pb.SearchRequest{ - Collection: classname, Metadata: &pb.MetadataRequest{Vector: true}, - Filters: &pb.Filters{ - Operator: pb.Filters_OPERATOR_CONTAINS_ALL, - TestValue: &pb.Filters_ValueIntArray{ValueIntArray: &pb.IntArray{Values: []int64{3}}}, - On: []string{"floats"}, - }, - }, - out: dto.GetParams{ - ClassName: classname, Pagination: defaultPagination, - Properties: defaultTestClassProps, - AdditionalProperties: additional.Properties{Vector: true, NoProps: false}, - Filters: &filters.LocalFilter{ - Root: &filters.Clause{ - On: &filters.Path{ - Class: schema.ClassName(classname), - Property: "floats", - }, - Operator: filters.ContainsAll, - Value: &filters.Value{Value: []float64{3}, Type: schema.DataTypeNumber}, - }, - }, - }, - error: false, - }, - { - name: "metadata filter id", - req: &pb.SearchRequest{ - Collection: classname, - Filters: &pb.Filters{ - Operator: pb.Filters_OPERATOR_EQUAL, - TestValue: &pb.Filters_ValueText{ValueText: UUID4}, - On: []string{filters.InternalPropID}, - }, - }, - out: dto.GetParams{ - ClassName: classname, Pagination: defaultPagination, - Properties: defaultTestClassProps, - AdditionalProperties: additional.Properties{NoProps: false}, - Filters: &filters.LocalFilter{ - Root: &filters.Clause{ - On: &filters.Path{ - Class: schema.ClassName(classname), - Property: filters.InternalPropID, - }, - Operator: filters.OperatorEqual, - Value: &filters.Value{Value: UUID4, Type: schema.DataTypeText}, - }, - }, - }, - error: false, - }, - { - name: "metadata filter time", - req: &pb.SearchRequest{ - Collection: classname, - Filters: &pb.Filters{ - Operator: pb.Filters_OPERATOR_EQUAL, - TestValue: &pb.Filters_ValueText{ValueText: "2022-03-18T20:26:34.586-05:00"}, - On: []string{filters.InternalPropCreationTimeUnix}, - }, - }, - out: dto.GetParams{ - ClassName: classname, Pagination: defaultPagination, - Properties: defaultTestClassProps, - AdditionalProperties: additional.Properties{NoProps: false}, - Filters: &filters.LocalFilter{ - Root: &filters.Clause{ - On: &filters.Path{ - Class: schema.ClassName(classname), - Property: filters.InternalPropCreationTimeUnix, - }, - Operator: filters.OperatorEqual, - Value: &filters.Value{Value: "2022-03-18T20:26:34.586-05:00", Type: schema.DataTypeDate}, - }, - }, - }, - error: false, - }, - { - name: "near text search", - req: &pb.SearchRequest{ - Collection: classname, Metadata: &pb.MetadataRequest{Vector: true}, - NearText: &pb.NearTextSearch{ - Query: []string{"first and", "second", "query"}, - MoveTo: &pb.NearTextSearch_Move{Force: 0.5, Concepts: []string{"first", "and second"}, Uuids: []string{UUID3, UUID4}}, - MoveAway: &pb.NearTextSearch_Move{Force: 0.3, Concepts: []string{"second to last", "really last"}, Uuids: []string{UUID4}}, - }, - }, - out: dto.GetParams{ - ClassName: classname, Pagination: defaultPagination, - Properties: defaultTestClassProps, - AdditionalProperties: additional.Properties{Vector: true, NoProps: false}, - ModuleParams: map[string]interface{}{ - "nearText": &nearText2.NearTextParams{ - Values: []string{"first and", "second", "query"}, - MoveTo: nearText2.ExploreMove{ - Force: 0.5, - Values: []string{"first", "and second"}, - Objects: []nearText2.ObjectMove{ - {ID: UUID3, Beacon: crossref.NewLocalhost(classname, UUID3).String()}, - {ID: UUID4, Beacon: crossref.NewLocalhost(classname, UUID4).String()}, - }, - }, - MoveAwayFrom: nearText2.ExploreMove{ - Force: 0.3, - Values: []string{"second to last", "really last"}, - Objects: []nearText2.ObjectMove{ - {ID: UUID4, Beacon: crossref.NewLocalhost(classname, UUID4).String()}, - }, - }, - Limit: 10, // default - }, - }, - }, - error: false, - }, - { - name: "near text wrong uuid format", - req: &pb.SearchRequest{ - Collection: classname, Metadata: &pb.MetadataRequest{Vector: true}, - NearText: &pb.NearTextSearch{ - Query: []string{"first"}, - MoveTo: &pb.NearTextSearch_Move{Force: 0.5, Uuids: []string{"not a uuid"}}, - }, - }, - out: dto.GetParams{}, - error: true, - }, - { - name: "near audio search", - req: &pb.SearchRequest{ - Collection: classname, Metadata: &pb.MetadataRequest{Vector: true}, - NearAudio: &pb.NearAudioSearch{ - Audio: "audio file", - }, - }, - out: dto.GetParams{ - ClassName: classname, Pagination: defaultPagination, - Properties: defaultTestClassProps, - AdditionalProperties: additional.Properties{Vector: true, NoProps: false}, - ModuleParams: map[string]interface{}{ - "nearAudio": &nearAudio.NearAudioParams{ - Audio: "audio file", - }, - }, - }, - error: false, - }, - { - name: "near video search", - req: &pb.SearchRequest{ - Collection: classname, Metadata: &pb.MetadataRequest{Vector: true}, - NearVideo: &pb.NearVideoSearch{ - Video: "video file", - }, - }, - out: dto.GetParams{ - ClassName: classname, Pagination: defaultPagination, - Properties: defaultTestClassProps, - AdditionalProperties: additional.Properties{Vector: true, NoProps: false}, - ModuleParams: map[string]interface{}{ - "nearVideo": &nearVideo.NearVideoParams{ - Video: "video file", - }, - }, - }, - error: false, - }, - { - name: "near image search", - req: &pb.SearchRequest{ - Collection: classname, Metadata: &pb.MetadataRequest{Vector: true}, - NearImage: &pb.NearImageSearch{ - Image: "image file", - }, - }, - out: dto.GetParams{ - ClassName: classname, Pagination: defaultPagination, - Properties: defaultTestClassProps, - AdditionalProperties: additional.Properties{Vector: true, NoProps: false}, - ModuleParams: map[string]interface{}{ - "nearImage": &nearImage.NearImageParams{ - Image: "image file", - }, - }, - }, - error: false, - }, - { - name: "Consistency", - req: &pb.SearchRequest{ - Collection: classname, Metadata: &pb.MetadataRequest{Vector: true}, - ConsistencyLevel: &quorum, - }, - out: dto.GetParams{ - ClassName: classname, Pagination: defaultPagination, - Properties: defaultTestClassProps, - AdditionalProperties: additional.Properties{Vector: true, NoProps: false}, - ReplicationProperties: &additional.ReplicationProperties{ConsistencyLevel: "QUORUM"}, - }, - error: false, - }, - { - name: "Generative", - req: &pb.SearchRequest{ - Collection: classname, Metadata: &pb.MetadataRequest{Vector: true}, - Generative: &pb.GenerativeSearch{SingleResponsePrompt: someString1, GroupedResponseTask: someString2, GroupedProperties: []string{"one", "two"}}, - }, - out: dto.GetParams{ - ClassName: classname, Pagination: defaultPagination, - Properties: defaultTestClassProps, - AdditionalProperties: additional.Properties{ - Vector: true, - NoProps: false, - ModuleParams: map[string]interface{}{ - "generate": &generate.Params{Prompt: &someString1, Task: &someString2, Properties: []string{"one", "two"}}, - }, - }, - }, - error: false, - }, - { - name: "Sort", - req: &pb.SearchRequest{ - Collection: classname, Metadata: &pb.MetadataRequest{Vector: true}, - SortBy: []*pb.SortBy{{Ascending: false, Path: []string{"name"}}}, - }, - out: dto.GetParams{ - ClassName: classname, Pagination: defaultPagination, - Properties: defaultTestClassProps, - AdditionalProperties: additional.Properties{ - Vector: true, - NoProps: false, - }, - Sort: []filters.Sort{{Order: "desc", Path: []string{"name"}}}, - }, - error: false, - }, - { - name: "Sort and vector search", - req: &pb.SearchRequest{ - Collection: classname, Metadata: &pb.MetadataRequest{Vector: true}, - SortBy: []*pb.SortBy{{Ascending: false, Path: []string{"name"}}}, - NearVector: &pb.NearVector{Vector: []float32{1, 2, 3}}, - }, - out: dto.GetParams{}, - error: true, - }, - { - name: "group by", - req: &pb.SearchRequest{ - Collection: classname, Metadata: &pb.MetadataRequest{Vector: true}, - GroupBy: &pb.GroupBy{Path: []string{"name"}, NumberOfGroups: 2, ObjectsPerGroup: 3}, - NearVector: &pb.NearVector{Vector: []float32{1, 2, 3}}, - }, - out: dto.GetParams{ - ClassName: classname, Pagination: defaultPagination, - Properties: defaultTestClassProps, - AdditionalProperties: additional.Properties{ - Vector: true, - NoProps: false, - Group: true, - }, - NearVector: &searchparams.NearVector{Vector: []float32{1, 2, 3}}, - GroupBy: &searchparams.GroupBy{Groups: 2, ObjectsPerGroup: 3, Property: "name"}, - }, - error: false, - }, - { - name: "group by with too long path", - req: &pb.SearchRequest{ - Collection: classname, Metadata: &pb.MetadataRequest{Vector: true}, - GroupBy: &pb.GroupBy{Path: []string{"ref", "Class"}, NumberOfGroups: 2, ObjectsPerGroup: 3}, - NearVector: &pb.NearVector{Vector: []float32{1, 2, 3}}, - }, - out: dto.GetParams{}, - error: true, - }, - { - name: "Object properties return", - req: &pb.SearchRequest{ - Collection: objClass, - Properties: &pb.PropertiesRequest{ - ObjectProperties: []*pb.ObjectPropertiesRequest{ - { - PropName: "something", - PrimitiveProperties: []string{"name"}, - ObjectProperties: []*pb.ObjectPropertiesRequest{ - { - PropName: "else", - PrimitiveProperties: []string{"name"}, - }, - { - PropName: "elses", - PrimitiveProperties: []string{"name"}, - }, - }, - }, - }, - }, - }, - out: dto.GetParams{ - ClassName: objClass, Pagination: defaultPagination, - Properties: search.SelectProperties{ - { - Name: "something", IsPrimitive: false, IsObject: true, - Props: search.SelectProperties{ - {Name: "name", IsPrimitive: true}, - { - Name: "else", IsPrimitive: false, IsObject: true, - Props: search.SelectProperties{{ - Name: "name", IsPrimitive: true, - }}, - }, - { - Name: "elses", IsPrimitive: false, IsObject: true, - Props: search.SelectProperties{{ - Name: "name", IsPrimitive: true, - }}, - }, - }, - }, - }, - }, - }, - { - name: "Empty return values given nested", - req: &pb.SearchRequest{Collection: objClass}, - out: dto.GetParams{ - ClassName: objClass, Pagination: defaultPagination, - Properties: search.SelectProperties{ - { - Name: "something", IsPrimitive: false, IsObject: true, - Props: search.SelectProperties{ - {Name: "name", IsPrimitive: true}, - { - Name: "else", IsPrimitive: false, IsObject: true, - Props: search.SelectProperties{{ - Name: "name", IsPrimitive: true, - }}, - }, - { - Name: "elses", IsPrimitive: false, IsObject: true, - Props: search.SelectProperties{{ - Name: "name", IsPrimitive: true, - }}, - }, - }, - }, - }, - }, - error: false, - }, - { - name: "No return values given nested with new default logic", - req: &pb.SearchRequest{Uses_123Api: true, Collection: objClass, Properties: &pb.PropertiesRequest{ReturnAllNonrefProperties: true}}, - out: dto.GetParams{ - ClassName: objClass, Pagination: defaultPagination, - Properties: search.SelectProperties{ - { - Name: "something", IsPrimitive: false, IsObject: true, - Props: search.SelectProperties{ - {Name: "name", IsPrimitive: true}, - { - Name: "else", IsPrimitive: false, IsObject: true, - Props: search.SelectProperties{{ - Name: "name", IsPrimitive: true, - }}, - }, - { - Name: "elses", IsPrimitive: false, IsObject: true, - Props: search.SelectProperties{{ - Name: "name", IsPrimitive: true, - }}, - }, - }, - }, - }, - }, - error: false, - }, - { - name: "Rerank without query", - req: &pb.SearchRequest{ - Collection: classname, - Rerank: &pb.Rerank{Property: someString1}, - }, - out: dto.GetParams{ - ClassName: classname, Pagination: defaultPagination, - Properties: defaultTestClassProps, - AdditionalProperties: additional.Properties{ - NoProps: false, - ModuleParams: map[string]interface{}{"rerank": &rank.Params{Property: &someString1}}, - }, - }, - error: false, - }, - { - name: "Rerank with query", - req: &pb.SearchRequest{ - Collection: classname, - Rerank: &pb.Rerank{Property: someString1, Query: &someString2}, - }, - out: dto.GetParams{ - ClassName: classname, Pagination: defaultPagination, - Properties: defaultTestClassProps, - AdditionalProperties: additional.Properties{ - NoProps: false, - ModuleParams: map[string]interface{}{"rerank": &rank.Params{Property: &someString1, Query: &someString2}}, - }, - }, - error: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - out, err := searchParamsFromProto(tt.req, scheme) - if tt.error { - require.NotNil(t, err) - } else { - require.Nil(t, err) - require.Equal(t, tt.out, out) - } - }) - } -} diff --git a/adapters/handlers/grpc/v1/prepare_reply.go b/adapters/handlers/grpc/v1/prepare_reply.go deleted file mode 100644 index 0d535529ec9034ffb76c9d004bfa14c3740d088b..0000000000000000000000000000000000000000 --- a/adapters/handlers/grpc/v1/prepare_reply.go +++ /dev/null @@ -1,799 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package v1 - -import ( - "fmt" - "math/big" - "strings" - "time" - - "github.com/weaviate/weaviate/usecases/byteops" - - "github.com/weaviate/weaviate/entities/schema" - generative "github.com/weaviate/weaviate/usecases/modulecomponents/additional/generate" - "github.com/weaviate/weaviate/usecases/modulecomponents/additional/models" - - "github.com/go-openapi/strfmt" - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/dto" - "github.com/weaviate/weaviate/entities/search" - pb "github.com/weaviate/weaviate/grpc/generated/protocol/v1" - "google.golang.org/protobuf/types/known/structpb" -) - -func searchResultsToProto(res []interface{}, start time.Time, searchParams dto.GetParams, scheme schema.Schema, usesPropertiesMessage bool) (*pb.SearchReply, error) { - tookSeconds := float64(time.Since(start)) / float64(time.Second) - out := &pb.SearchReply{ - Took: float32(tookSeconds), - GenerativeGroupedResult: new(string), // pointer to empty string - } - - if searchParams.GroupBy != nil { - out.GroupByResults = make([]*pb.GroupByResult, len(res)) - for i, raw := range res { - group, generativeGroupResponse, err := extractGroup(raw, searchParams, scheme, usesPropertiesMessage) - if err != nil { - return nil, err - } - if generativeGroupResponse != "" { - out.GenerativeGroupedResult = &generativeGroupResponse - } - out.GroupByResults[i] = group - } - } else { - objects, generativeGroupResponse, err := extractObjectsToResults(res, searchParams, scheme, false, usesPropertiesMessage) - if err != nil { - return nil, err - } - out.GenerativeGroupedResult = &generativeGroupResponse - out.Results = objects - } - return out, nil -} - -func extractObjectsToResults(res []interface{}, searchParams dto.GetParams, scheme schema.Schema, fromGroup, usesPropertiesMessage bool) ([]*pb.SearchResult, string, error) { - results := make([]*pb.SearchResult, len(res)) - generativeGroupResultsReturn := "" - for i, raw := range res { - asMap, ok := raw.(map[string]interface{}) - if !ok { - return nil, "", fmt.Errorf("could not parse returns %v", raw) - } - firstObject := i == 0 - - var props *pb.PropertiesResult - var err error - - if usesPropertiesMessage { - props, err = extractPropertiesAnswer(scheme, asMap, searchParams.Properties, searchParams.ClassName, searchParams.AdditionalProperties) - } else { - props, err = extractPropertiesAnswerDeprecated(scheme, asMap, searchParams.Properties, searchParams.ClassName, searchParams.AdditionalProperties) - } - if err != nil { - return nil, "", err - } - - additionalProps, generativeGroupResults, err := extractAdditionalProps(asMap, searchParams.AdditionalProperties, firstObject, fromGroup) - if err != nil { - return nil, "", err - } - - if generativeGroupResultsReturn == "" && generativeGroupResults != "" { - generativeGroupResultsReturn = generativeGroupResults - } - - result := &pb.SearchResult{ - Properties: props, - Metadata: additionalProps, - } - - results[i] = result - } - return results, generativeGroupResultsReturn, nil -} - -func extractAdditionalProps(asMap map[string]any, additionalPropsParams additional.Properties, firstObject, fromGroup bool) (*pb.MetadataResult, string, error) { - generativeSearchRaw, generativeSearchEnabled := additionalPropsParams.ModuleParams["generate"] - _, rerankEnabled := additionalPropsParams.ModuleParams["rerank"] - - metadata := &pb.MetadataResult{} - if additionalPropsParams.ID && !generativeSearchEnabled && !rerankEnabled && !fromGroup { - idRaw, ok := asMap["id"] - if !ok { - return nil, "", errors.New("could not extract get id in additional prop") - } - - idStrfmt, ok := idRaw.(strfmt.UUID) - if !ok { - return nil, "", errors.New("could not extract format id in additional prop") - } - metadata.Id = idStrfmt.String() - hexInteger, success := new(big.Int).SetString(strings.Replace(metadata.Id, "-", "", -1), 16) - if !success { - return nil, "", fmt.Errorf("failed to parse hex string to integer") - } - metadata.IdAsBytes = hexInteger.Bytes() - } - _, ok := asMap["_additional"] - if !ok { - return metadata, "", nil - } - - var additionalPropertiesMap map[string]interface{} - if !fromGroup { - additionalPropertiesMap = asMap["_additional"].(map[string]interface{}) - } else { - addPropertiesGroup := asMap["_additional"].(*additional.GroupHitAdditional) - additionalPropertiesMap = make(map[string]interface{}, 3) - additionalPropertiesMap["id"] = addPropertiesGroup.ID - additionalPropertiesMap["vector"] = addPropertiesGroup.Vector - additionalPropertiesMap["distance"] = addPropertiesGroup.Distance - } - generativeGroupResults := "" - // id is part of the _additional map in case of generative search, group, & rerank - don't aks me why - if additionalPropsParams.ID && (generativeSearchEnabled || fromGroup || rerankEnabled) { - idRaw, ok := additionalPropertiesMap["id"] - if !ok { - return nil, "", errors.New("could not extract get id generative in additional prop") - } - - idStrfmt, ok := idRaw.(strfmt.UUID) - if !ok { - return nil, "", errors.New("could not format id generative in additional prop") - } - metadata.Id = idStrfmt.String() - } - - if generativeSearchEnabled { - var generateFmt *models.GenerateResult - - generate, ok := additionalPropertiesMap["generate"] - if !ok { - generateFmt = &models.GenerateResult{} - } else { - generateFmt, ok = generate.(*models.GenerateResult) - if !ok { - return nil, "", errors.New("could not cast generative result additional prop") - } - } - - generativeSearch, ok := generativeSearchRaw.(*generative.Params) - if !ok { - return nil, "", errors.New("could not cast generative search params") - } - if generativeSearch.Prompt != nil && generateFmt.SingleResult == nil { - return nil, "", errors.New("No results for generative search despite a search request. Is a generative module enabled?") - } - - if generateFmt.Error != nil { - return nil, "", generateFmt.Error - } - - if generateFmt.SingleResult != nil && *generateFmt.SingleResult != "" { - metadata.Generative = *generateFmt.SingleResult - metadata.GenerativePresent = true - } - - // grouped results are only added to the first object for GQL reasons - // however, reranking can result in a different order, so we need to check every object - // recording the result if it's present assuming that it is at least somewhere and will be caught - if generateFmt.GroupedResult != nil && *generateFmt.GroupedResult != "" { - generativeGroupResults = *generateFmt.GroupedResult - } - } - - if rerankEnabled { - rerank, ok := additionalPropertiesMap["rerank"] - if !ok { - return nil, "", errors.New("No results for rerank despite a search request. Is a the rerank module enabled?") - } - rerankFmt, ok := rerank.([]*models.RankResult) - if !ok { - return nil, "", errors.New("could not cast rerank result additional prop") - } - metadata.RerankScore = *rerankFmt[0].Score - metadata.RerankScorePresent = true - } - - // additional properties are only present for certain searches/configs => don't return an error if not available - if additionalPropsParams.Vector { - vector, ok := additionalPropertiesMap["vector"] - if ok { - vectorfmt, ok2 := vector.([]float32) - if ok2 { - metadata.Vector = vectorfmt // deprecated, remove in a bit - metadata.VectorBytes = byteops.Float32ToByteVector(vectorfmt) - } - } - } - - if additionalPropsParams.Certainty { - metadata.CertaintyPresent = false - certainty, ok := additionalPropertiesMap["certainty"] - if ok { - certaintyfmt, ok2 := certainty.(float64) - if ok2 { - metadata.Certainty = float32(certaintyfmt) - metadata.CertaintyPresent = true - } - } - } - - if additionalPropsParams.Distance { - metadata.DistancePresent = false - distance, ok := additionalPropertiesMap["distance"] - if ok { - distancefmt, ok2 := distance.(float32) - if ok2 { - metadata.Distance = distancefmt - metadata.DistancePresent = true - } - } - } - - if additionalPropsParams.CreationTimeUnix { - metadata.CreationTimeUnixPresent = false - creationtime, ok := additionalPropertiesMap["creationTimeUnix"] - if ok { - creationtimefmt, ok2 := creationtime.(int64) - if ok2 { - metadata.CreationTimeUnix = creationtimefmt - metadata.CreationTimeUnixPresent = true - } - } - } - - if additionalPropsParams.LastUpdateTimeUnix { - metadata.LastUpdateTimeUnixPresent = false - lastUpdateTime, ok := additionalPropertiesMap["lastUpdateTimeUnix"] - if ok { - lastUpdateTimefmt, ok2 := lastUpdateTime.(int64) - if ok2 { - metadata.LastUpdateTimeUnix = lastUpdateTimefmt - metadata.LastUpdateTimeUnixPresent = true - } - } - } - - if additionalPropsParams.ExplainScore { - metadata.ExplainScorePresent = false - explainScore, ok := additionalPropertiesMap["explainScore"] - if ok { - explainScorefmt, ok2 := explainScore.(string) - if ok2 { - metadata.ExplainScore = explainScorefmt - metadata.ExplainScorePresent = true - } - } - } - - if additionalPropsParams.Score { - metadata.ScorePresent = false - score, ok := additionalPropertiesMap["score"] - if ok { - scorefmt, ok2 := score.(float32) - if ok2 { - metadata.Score = scorefmt - metadata.ScorePresent = true - } - } - } - - if additionalPropsParams.IsConsistent { - isConsistent, ok := additionalPropertiesMap["isConsistent"] - if ok { - isConsistentfmt, ok2 := isConsistent.(bool) - if ok2 { - metadata.IsConsistent = &isConsistentfmt - metadata.IsConsistentPresent = true - } - } - } - - return metadata, generativeGroupResults, nil -} - -func extractGroup(raw any, searchParams dto.GetParams, scheme schema.Schema, usesMarshalling bool) (*pb.GroupByResult, string, error) { - generativeSearchRaw, generativeSearchEnabled := searchParams.AdditionalProperties.ModuleParams["generate"] - _, rerankEnabled := searchParams.AdditionalProperties.ModuleParams["rerank"] - asMap, ok := raw.(map[string]interface{}) - if !ok { - return nil, "", fmt.Errorf("cannot parse result %v", raw) - } - add, ok := asMap["_additional"] - if !ok { - return nil, "", fmt.Errorf("_additional is required for groups %v", asMap) - } - addAsMap, ok := add.(map[string]interface{}) - if !ok { - return nil, "", fmt.Errorf("cannot parse _additional %v", add) - } - groupRaw, ok := addAsMap["group"] - if !ok { - return nil, "", fmt.Errorf("group is not present %v", addAsMap) - } - group, ok := groupRaw.(*additional.Group) - if !ok { - return nil, "", fmt.Errorf("cannot parse _additional %v", groupRaw) - } - - ret := &pb.GroupByResult{ - Name: group.GroupedBy.Value, - MaxDistance: group.MaxDistance, - MinDistance: group.MinDistance, - NumberOfObjects: int64(group.Count), - } - - groupedGenerativeResults := "" - if generativeSearchEnabled { - var generateFmt *models.GenerateResult - - generate, ok := addAsMap["generate"] - if !ok { - generateFmt = &models.GenerateResult{} - } else { - generateFmt, ok = generate.(*models.GenerateResult) - if !ok { - return nil, "", errors.New("could not cast generative result additional prop") - } - } - - generativeSearch, ok := generativeSearchRaw.(*generative.Params) - if !ok { - return nil, "", errors.New("could not cast generative search params") - } - if generativeSearch.Prompt != nil && generateFmt.SingleResult == nil { - return nil, "", errors.New("No results for generative search despite a search request. Is a generative module enabled?") - } - - if generateFmt.Error != nil { - return nil, "", generateFmt.Error - } - - if generateFmt.SingleResult != nil && *generateFmt.SingleResult != "" { - ret.Generative = &pb.GenerativeReply{Result: *generateFmt.SingleResult} - } - - // grouped results are only added to the first object for GQL reasons - // however, reranking can result in a different order, so we need to check every object - // recording the result if it's present assuming that it is at least somewhere and will be caught - if generateFmt.GroupedResult != nil && *generateFmt.GroupedResult != "" { - groupedGenerativeResults = *generateFmt.GroupedResult - } - } - - if rerankEnabled { - rerankRaw, ok := addAsMap["rerank"] - if !ok { - return nil, "", fmt.Errorf("rerank is not present %v", addAsMap) - } - rerank, ok := rerankRaw.([]*models.RankResult) - if !ok { - return nil, "", fmt.Errorf("cannot parse rerank %v", rerankRaw) - } - ret.Rerank = &pb.RerankReply{ - Score: *rerank[0].Score, - } - } - - // group results does not support more additional properties - searchParams.AdditionalProperties = additional.Properties{ - ID: searchParams.AdditionalProperties.ID, - Vector: searchParams.AdditionalProperties.Vector, - Distance: searchParams.AdditionalProperties.Distance, - } - - // group objects are returned as a different type than normal results ([]map[string]interface{} vs []interface). As - // the normal path is used much more often than groupBy, convert the []map[string]interface{} to []interface{}, even - // though we cast it to map[string]interface{} in the extraction function. - // This way we only do a copy for groupBy and not for the standard code-path which is used more often - returnObjectsUntyped := make([]interface{}, len(group.Hits)) - for i := range returnObjectsUntyped { - returnObjectsUntyped[i] = group.Hits[i] - } - - objects, _, err := extractObjectsToResults(returnObjectsUntyped, searchParams, scheme, true, usesMarshalling) - if err != nil { - return nil, "", errors.Wrap(err, "extracting hits from group") - } - - ret.Objects = objects - - return ret, groupedGenerativeResults, nil -} - -func extractPropertiesAnswerDeprecated(scheme schema.Schema, results map[string]interface{}, properties search.SelectProperties, className string, additionalPropsParams additional.Properties) (*pb.PropertiesResult, error) { - nonRefProps := make(map[string]interface{}, 0) - refProps := make([]*pb.RefPropertiesResult, 0) - objProps := make([]*pb.ObjectProperties, 0) - objArrayProps := make([]*pb.ObjectArrayProperties, 0) - for _, prop := range properties { - propRaw, ok := results[prop.Name] - if !ok { - continue - } - if prop.IsPrimitive { - nonRefProps[prop.Name] = propRaw - continue - } - if prop.IsObject { - nested, err := scheme.GetProperty(schema.ClassName(className), schema.PropertyName(prop.Name)) - if err != nil { - return nil, errors.Wrap(err, "getting property") - } - singleObj, ok := propRaw.(map[string]interface{}) - if ok { - extractedNestedProp, err := extractPropertiesNested(scheme, singleObj, prop, className, &Property{Property: nested}) - if err != nil { - return nil, errors.Wrap(err, "extracting nested properties") - } - objProps = append(objProps, &pb.ObjectProperties{ - PropName: prop.Name, - Value: extractedNestedProp, - }) - continue - } - arrayObjs, ok := propRaw.([]interface{}) - if ok { - extractedNestedProps := make([]*pb.ObjectPropertiesValue, 0, len(arrayObjs)) - for _, obj := range arrayObjs { - singleObj, ok := obj.(map[string]interface{}) - if !ok { - continue - } - extractedNestedProp, err := extractPropertiesNested(scheme, singleObj, prop, className, &Property{Property: nested}) - if err != nil { - return nil, err - } - extractedNestedProps = append(extractedNestedProps, extractedNestedProp) - } - objArrayProps = append(objArrayProps, - &pb.ObjectArrayProperties{ - PropName: prop.Name, - Values: extractedNestedProps, - }, - ) - continue - } - } - refs, ok := propRaw.([]interface{}) - if !ok { - continue - } - extractedRefProps := make([]*pb.PropertiesResult, 0, len(refs)) - for _, ref := range refs { - refLocal, ok := ref.(search.LocalRef) - if !ok { - continue - } - extractedRefProp, err := extractPropertiesAnswerDeprecated(scheme, refLocal.Fields, prop.Refs[0].RefProperties, refLocal.Class, additionalPropsParams) - if err != nil { - continue - } - additionalProps, _, err := extractAdditionalProps(refLocal.Fields, prop.Refs[0].AdditionalProperties, false, false) - if err != nil { - return nil, err - } - extractedRefProp.Metadata = additionalProps - extractedRefProps = append(extractedRefProps, extractedRefProp) - } - - refProp := pb.RefPropertiesResult{PropName: prop.Name, Properties: extractedRefProps} - refProps = append(refProps, &refProp) - } - props := pb.PropertiesResult{} - if len(nonRefProps) > 0 { - outProps := pb.ObjectPropertiesValue{} - if err := extractArrayTypesRoot(scheme, className, nonRefProps, &outProps); err != nil { - return nil, errors.Wrap(err, "extracting non-primitive types") - } - newStruct, err := structpb.NewStruct(nonRefProps) - if err != nil { - return nil, errors.Wrap(err, "creating non-ref-prop struct") - } - props.NonRefProperties = newStruct - props.IntArrayProperties = outProps.IntArrayProperties - props.NumberArrayProperties = outProps.NumberArrayProperties - props.TextArrayProperties = outProps.TextArrayProperties - props.BooleanArrayProperties = outProps.BooleanArrayProperties - props.ObjectProperties = outProps.ObjectProperties - props.ObjectArrayProperties = outProps.ObjectArrayProperties - } - if len(refProps) > 0 { - props.RefProps = refProps - } - if len(objProps) > 0 { - props.ObjectProperties = objProps - } - if len(objArrayProps) > 0 { - props.ObjectArrayProperties = objArrayProps - } - - props.TargetCollection = className - return &props, nil -} - -func extractPropertiesAnswer(scheme schema.Schema, results map[string]interface{}, properties search.SelectProperties, className string, additionalPropsParams additional.Properties) (*pb.PropertiesResult, error) { - nonRefProps := &pb.Properties{ - Fields: make(map[string]*pb.Value, 0), - } - refProps := make([]*pb.RefPropertiesResult, 0) - class := scheme.GetClass(schema.ClassName(className)) - for _, prop := range properties { - propRaw, ok := results[prop.Name] - if !ok { - continue - } - if prop.IsPrimitive { - dataType, err := schema.GetPropertyDataType(class, prop.Name) - if err != nil { - return nil, errors.Wrap(err, "getting primitive property datatype") - } - value, err := NewPrimitiveValue(propRaw, *dataType) - if err != nil { - return nil, errors.Wrapf(err, "creating primitive value for %v", prop.Name) - } - nonRefProps.Fields[prop.Name] = value - continue - } - if prop.IsObject { - nested, err := scheme.GetProperty(schema.ClassName(className), schema.PropertyName(prop.Name)) - if err != nil { - return nil, errors.Wrap(err, "getting nested property") - } - value, err := NewNestedValue(propRaw, schema.DataType(nested.DataType[0]), &Property{Property: nested}, prop) - if err != nil { - return nil, errors.Wrap(err, "creating object value") - } - nonRefProps.Fields[prop.Name] = value - continue - } - refs, ok := propRaw.([]interface{}) - if !ok { - continue - } - extractedRefProps := make([]*pb.PropertiesResult, 0, len(refs)) - for _, ref := range refs { - refLocal, ok := ref.(search.LocalRef) - if !ok { - continue - } - extractedRefProp, err := extractPropertiesAnswer(scheme, refLocal.Fields, prop.Refs[0].RefProperties, refLocal.Class, additionalPropsParams) - if err != nil { - continue - } - additionalProps, _, err := extractAdditionalProps(refLocal.Fields, prop.Refs[0].AdditionalProperties, false, false) - if err != nil { - return nil, err - } - extractedRefProp.Metadata = additionalProps - extractedRefProps = append(extractedRefProps, extractedRefProp) - } - - refProp := pb.RefPropertiesResult{PropName: prop.Name, Properties: extractedRefProps} - refProps = append(refProps, &refProp) - } - props := pb.PropertiesResult{} - if len(nonRefProps.Fields) != 0 { - props.NonRefProps = nonRefProps - } - if len(refProps) != 0 { - props.RefProps = refProps - } - props.RefPropsRequested = properties.HasRefs() - props.TargetCollection = className - return &props, nil -} - -func extractPropertiesNested[P schema.PropertyInterface](scheme schema.Schema, results map[string]interface{}, property search.SelectProperty, className string, parent P) (*pb.ObjectPropertiesValue, error) { - primitiveProps := make(map[string]interface{}, 0) - objProps := make([]*pb.ObjectProperties, 0) - objArrayProps := make([]*pb.ObjectArrayProperties, 0) - for _, prop := range property.Props { - propRaw, ok := results[prop.Name] - if !ok { - continue - } - if prop.IsPrimitive { - primitiveProps[prop.Name] = propRaw - continue - } - if prop.IsObject { - var err error - objProps, objArrayProps, err = extractObjectProperties(scheme, propRaw, prop, className, parent, objProps, objArrayProps) - if err != nil { - return nil, err - } - } - } - props := pb.ObjectPropertiesValue{} - if len(primitiveProps) > 0 { - if err := extractArrayTypesNested(scheme, className, primitiveProps, &props, parent); err != nil { - return nil, errors.Wrap(err, "extracting non-primitive types") - } - newStruct, err := structpb.NewStruct(primitiveProps) - if err != nil { - return nil, errors.Wrap(err, "creating non-ref-prop struct") - } - props.NonRefProperties = newStruct - } - if len(objProps) > 0 { - props.ObjectProperties = objProps - } - if len(objArrayProps) > 0 { - props.ObjectArrayProperties = objArrayProps - } - return &props, nil -} - -func extractObjectProperties[P schema.PropertyInterface](scheme schema.Schema, propRaw interface{}, property search.SelectProperty, className string, parent P, objProps []*pb.ObjectProperties, objArrayProps []*pb.ObjectArrayProperties) ([]*pb.ObjectProperties, []*pb.ObjectArrayProperties, error) { - prop, ok := propRaw.(map[string]interface{}) - if ok { - objProp, err := extractObjectSingleProperties(scheme, prop, property, className, parent) - if err != nil { - return objProps, objArrayProps, err - } - objProps = append(objProps, objProp) - } - propArray, ok := propRaw.([]interface{}) - if ok { - objArrayProp, err := extractObjectArrayProperties(scheme, propArray, property, className, parent) - if err != nil { - return objProps, objArrayProps, err - } - objArrayProps = append(objArrayProps, objArrayProp) - } - return objProps, objArrayProps, nil -} - -func extractObjectSingleProperties[P schema.PropertyInterface](scheme schema.Schema, prop map[string]interface{}, property search.SelectProperty, className string, parent P) (*pb.ObjectProperties, error) { - nested, err := schema.GetNestedPropertyByName(parent, property.Name) - if err != nil { - return nil, errors.Wrap(err, "getting property") - } - extractedNestedProp, err := extractPropertiesNested(scheme, prop, property, className, &NestedProperty{NestedProperty: nested}) - if err != nil { - return nil, errors.Wrap(err, fmt.Sprintf("extracting nested properties from %v", nested)) - } - return &pb.ObjectProperties{ - PropName: property.Name, - Value: extractedNestedProp, - }, nil -} - -func extractObjectArrayProperties[P schema.PropertyInterface](scheme schema.Schema, propObjs []interface{}, property search.SelectProperty, className string, parent P) (*pb.ObjectArrayProperties, error) { - extractedNestedProps := make([]*pb.ObjectPropertiesValue, 0, len(propObjs)) - for _, objRaw := range propObjs { - nested, err := schema.GetNestedPropertyByName(parent, property.Name) - if err != nil { - return nil, errors.Wrap(err, "getting property") - } - obj, ok := objRaw.(map[string]interface{}) - if !ok { - continue - } - extractedNestedProp, err := extractPropertiesNested(scheme, obj, property, className, &NestedProperty{NestedProperty: nested}) - if err != nil { - return nil, errors.Wrap(err, "extracting nested properties") - } - extractedNestedProps = append(extractedNestedProps, extractedNestedProp) - } - return &pb.ObjectArrayProperties{ - PropName: property.Name, - Values: extractedNestedProps, - }, nil -} - -func extractArrayTypesRoot(scheme schema.Schema, className string, rawProps map[string]interface{}, props *pb.ObjectPropertiesValue) error { - dataTypes := make(map[string]*schema.DataType, 0) - for propName := range rawProps { - dataType, err := schema.GetPropertyDataType(scheme.GetClass(schema.ClassName(className)), propName) - if err != nil { - return err - } - dataTypes[propName] = dataType - } - return extractArrayTypes(scheme, rawProps, props, dataTypes) -} - -func extractArrayTypesNested[P schema.PropertyInterface](scheme schema.Schema, className string, rawProps map[string]interface{}, props *pb.ObjectPropertiesValue, parent P) error { - dataTypes := make(map[string]*schema.DataType, 0) - for propName := range rawProps { - dataType, err := schema.GetNestedPropertyDataType(parent, propName) - if err != nil { - return err - } - dataTypes[propName] = dataType - } - return extractArrayTypes(scheme, rawProps, props, dataTypes) -} - -// slices cannot be part of a grpc struct, so we need to handle each of them separately -func extractArrayTypes(scheme schema.Schema, rawProps map[string]interface{}, props *pb.ObjectPropertiesValue, dataTypes map[string]*schema.DataType) error { - for propName, prop := range rawProps { - dataType := dataTypes[propName] - switch *dataType { - case schema.DataTypeIntArray: - propIntAsFloat, ok := prop.([]float64) - if !ok { - emptyArr, ok := prop.([]interface{}) - if ok && len(emptyArr) == 0 { - continue - } - return fmt.Errorf("property %v with datatype %v needs to be []float64, got %T", propName, dataType, prop) - } - propInt := make([]int64, len(propIntAsFloat)) - for i := range propIntAsFloat { - propInt[i] = int64(propIntAsFloat[i]) - } - if props.IntArrayProperties == nil { - props.IntArrayProperties = make([]*pb.IntArrayProperties, 0) - } - props.IntArrayProperties = append(props.IntArrayProperties, &pb.IntArrayProperties{PropName: propName, Values: propInt}) - delete(rawProps, propName) - case schema.DataTypeNumberArray: - propFloat, ok := prop.([]float64) - if !ok { - emptyArr, ok := prop.([]interface{}) - if ok && len(emptyArr) == 0 { - continue - } - return fmt.Errorf("property %v with datatype %v needs to be []float64, got %T", propName, dataType, prop) - } - - if props.NumberArrayProperties == nil { - props.NumberArrayProperties = make([]*pb.NumberArrayProperties, 0) - } - props.NumberArrayProperties = append( - props.NumberArrayProperties, - &pb.NumberArrayProperties{PropName: propName, ValuesBytes: byteops.Float64ToByteVector(propFloat), Values: propFloat}, - ) - delete(rawProps, propName) - case schema.DataTypeStringArray, schema.DataTypeTextArray, schema.DataTypeDateArray, schema.DataTypeUUIDArray: - propString, ok := prop.([]string) - if !ok { - emptyArr, ok := prop.([]interface{}) - if ok && len(emptyArr) == 0 { - continue - } - return fmt.Errorf("property %v with datatype %v needs to be []string, got %T", propName, dataType, prop) - } - if props.TextArrayProperties == nil { - props.TextArrayProperties = make([]*pb.TextArrayProperties, 0) - } - props.TextArrayProperties = append(props.TextArrayProperties, &pb.TextArrayProperties{PropName: propName, Values: propString}) - delete(rawProps, propName) - case schema.DataTypeBooleanArray: - propBool, ok := prop.([]bool) - if !ok { - emptyArr, ok := prop.([]interface{}) - if ok && len(emptyArr) == 0 { - continue - } - return fmt.Errorf("property %v with datatype %v needs to be []bool, got %T", propName, dataType, prop) - } - if props.BooleanArrayProperties == nil { - props.BooleanArrayProperties = make([]*pb.BooleanArrayProperties, 0) - } - props.BooleanArrayProperties = append(props.BooleanArrayProperties, &pb.BooleanArrayProperties{PropName: propName, Values: propBool}) - delete(rawProps, propName) - default: - _, isArray := schema.IsArrayType(*dataType) - if isArray { - return fmt.Errorf("property %v with array type not handled %v", propName, dataType) - } - } - } - return nil -} diff --git a/adapters/handlers/grpc/v1/prepare_reply_test.go b/adapters/handlers/grpc/v1/prepare_reply_test.go deleted file mode 100644 index cf000918a9997c571b380bd392bcd68453b69b71..0000000000000000000000000000000000000000 --- a/adapters/handlers/grpc/v1/prepare_reply_test.go +++ /dev/null @@ -1,1369 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package v1 - -import ( - "encoding/binary" - "encoding/json" - "math" - "math/big" - "strings" - "testing" - "time" - - pb "github.com/weaviate/weaviate/grpc/generated/protocol/v1" - - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/search" - "github.com/weaviate/weaviate/entities/searchparams" - - "github.com/go-openapi/strfmt" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/dto" - "github.com/weaviate/weaviate/usecases/modulecomponents/additional/generate" - addModels "github.com/weaviate/weaviate/usecases/modulecomponents/additional/models" - "google.golang.org/protobuf/encoding/protojson" - "google.golang.org/protobuf/types/known/structpb" -) - -const ( - UUID1 = strfmt.UUID("a4de3ca0-6975-464f-b23b-adddd83630d7") - UUID2 = strfmt.UUID("7e10ec81-a26d-4ac7-8264-3e3e05397ddc") -) - -func newStruct(t *testing.T, values map[string]interface{}) *structpb.Struct { - b, err := json.Marshal(values) - require.Nil(t, err) - s := &structpb.Struct{} - err = protojson.Unmarshal(b, s) - require.Nil(t, err) - return s -} - -func ignoreError[T any](val T, err error) T { - return val -} - -func byteVector(vec []float32) []byte { - vector := make([]byte, len(vec)*4) - - for i := 0; i < len(vec); i++ { - binary.LittleEndian.PutUint32(vector[i*4:i*4+4], math.Float32bits(vec[i])) - } - - return vector -} - -func idByte(id string) []byte { - hexInteger, _ := new(big.Int).SetString(strings.Replace(id, "-", "", -1), 16) - return hexInteger.Bytes() -} - -func TestGRPCReply(t *testing.T) { - allAdditional := dto.GetParams{AdditionalProperties: additional.Properties{ - Vector: true, - Certainty: true, - ID: true, - Distance: true, - CreationTimeUnix: true, - LastUpdateTimeUnix: true, - ExplainScore: true, - Score: true, - IsConsistent: true, - }} - truePointer := true - - someFloat64 := float64(0.1) - refClass1 := "RefClass1" - refClass2 := "RefClass2" - className := "className" - objClass := "objClass" - scheme := schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{ - { - Class: className, - Properties: []*models.Property{ - {Name: "word", DataType: schema.DataTypeText.PropString()}, - {Name: "other", DataType: []string{"int"}}, - {Name: "age", DataType: []string{"int"}}, - {Name: "nums", DataType: schema.DataTypeIntArray.PropString()}, - {Name: "ref", DataType: []string{refClass1}}, - {Name: "multiRef", DataType: []string{refClass1, refClass2}}, - }, - }, - { - Class: refClass1, - Properties: []*models.Property{ - {Name: "something", DataType: schema.DataTypeText.PropString()}, - {Name: "nums", DataType: schema.DataTypeIntArray.PropString()}, - {Name: "ref2", DataType: []string{refClass2}}, - }, - }, - { - Class: refClass2, - Properties: []*models.Property{ - {Name: "else", DataType: schema.DataTypeText.PropString()}, - {Name: "ref3", DataType: []string{refClass2}}, - }, - }, - { - Class: objClass, - Properties: []*models.Property{ - { - Name: "something", - DataType: schema.DataTypeObject.PropString(), - NestedProperties: []*models.NestedProperty{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - }, - { - Name: "names", - DataType: schema.DataTypeTextArray.PropString(), - }, - { - Name: "else", - DataType: schema.DataTypeObject.PropString(), - NestedProperties: []*models.NestedProperty{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - }, - { - Name: "names", - DataType: schema.DataTypeTextArray.PropString(), - }, - }, - }, - { - Name: "objs", - DataType: schema.DataTypeObjectArray.PropString(), - NestedProperties: []*models.NestedProperty{{ - Name: "name", - DataType: schema.DataTypeText.PropString(), - }}, - }, - }, - }, - }, - }, - }, - }, - } - - tests := []struct { - name string - res []any - searchParams dto.GetParams // only a few things are needed to control what is returned - outSearch []*pb.SearchResult - outGenerative string - outGroup []*pb.GroupByResult - usesWeaviateStruct bool - }{ - { - name: "vector only", - res: []interface{}{ - map[string]interface{}{ - "_additional": map[string]interface{}{"vector": []float32{1}}, - }, - map[string]interface{}{ - "_additional": map[string]interface{}{"vector": []float32{2}}, - }, - }, - searchParams: dto.GetParams{AdditionalProperties: additional.Properties{Vector: true}}, - outSearch: []*pb.SearchResult{ - {Metadata: &pb.MetadataResult{Vector: []float32{1}, VectorBytes: byteVector([]float32{1})}, Properties: &pb.PropertiesResult{}}, - {Metadata: &pb.MetadataResult{Vector: []float32{2}, VectorBytes: byteVector([]float32{2})}, Properties: &pb.PropertiesResult{}}, - }, - usesWeaviateStruct: true, - }, - { - name: "all additional", - res: []interface{}{ - map[string]interface{}{ - "id": UUID1, - "_additional": map[string]interface{}{ - "vector": []float32{1}, - "certainty": 0.4, - "distance": float32(0.01), - "creationTimeUnix": int64(123), - "lastUpdateTimeUnix": int64(345), - "explainScore": "other text", - "score": float32(0.25), - "isConsistent": true, - }, - }, - map[string]interface{}{ - "id": UUID2, - "_additional": map[string]interface{}{ - "vector": []float32{2}, - "certainty": 0.5, - "distance": float32(0.1), - "creationTimeUnix": int64(456), - "lastUpdateTimeUnix": int64(789), - "explainScore": "some text", - "score": float32(0.45), - "isConsistent": true, - }, - }, - }, - searchParams: allAdditional, - outSearch: []*pb.SearchResult{ - { - Metadata: &pb.MetadataResult{ - Vector: []float32{1}, - Id: string(UUID1), - Certainty: 0.4, - CertaintyPresent: true, - Distance: 0.01, - DistancePresent: true, - CreationTimeUnix: 123, - CreationTimeUnixPresent: true, - LastUpdateTimeUnix: 345, - LastUpdateTimeUnixPresent: true, - ExplainScore: "other text", - ExplainScorePresent: true, - Score: 0.25, - ScorePresent: true, - IsConsistent: &truePointer, - IsConsistentPresent: true, - VectorBytes: byteVector([]float32{1}), - IdAsBytes: idByte(string(UUID1)), - }, - Properties: &pb.PropertiesResult{}, - }, - { - Metadata: &pb.MetadataResult{ - Vector: []float32{2}, - Id: string(UUID2), - Certainty: 0.5, - CertaintyPresent: true, - Distance: 0.1, - DistancePresent: true, - CreationTimeUnix: 456, - CreationTimeUnixPresent: true, - LastUpdateTimeUnix: 789, - LastUpdateTimeUnixPresent: true, - ExplainScore: "some text", - ExplainScorePresent: true, - Score: 0.45, - ScorePresent: true, - IsConsistent: &truePointer, - IsConsistentPresent: true, - VectorBytes: byteVector([]float32{2}), - IdAsBytes: idByte(string(UUID2)), - }, - Properties: &pb.PropertiesResult{}, - }, - }, - }, - { - name: "primitive properties deprecated", - res: []interface{}{ - map[string]interface{}{ - "word": "word", - "age": 21, - }, - map[string]interface{}{ - "word": "other", - "age": 26, - }, - }, - searchParams: dto.GetParams{ - ClassName: className, - Properties: search.SelectProperties{{Name: "word", IsPrimitive: true}, {Name: "age", IsPrimitive: true}}, - }, - outSearch: []*pb.SearchResult{ - { - Metadata: &pb.MetadataResult{}, - Properties: &pb.PropertiesResult{ - TargetCollection: className, - NonRefProperties: newStruct(t, map[string]interface{}{ - "word": "word", - "age": 21, - }), - RefProps: []*pb.RefPropertiesResult{}, - RefPropsRequested: false, - }, - }, - { - Metadata: &pb.MetadataResult{}, - Properties: &pb.PropertiesResult{ - TargetCollection: className, - NonRefProperties: newStruct(t, map[string]interface{}{ - "word": "other", - "age": 26, - }), - RefProps: []*pb.RefPropertiesResult{}, - RefPropsRequested: false, - }, - }, - }, - }, - { - name: "primitive properties", - res: []interface{}{ - map[string]interface{}{ - "word": "word", - "age": float64(21), - }, - map[string]interface{}{ - "word": "other", - "age": float64(26), - }, - }, - searchParams: dto.GetParams{ - ClassName: className, - Properties: search.SelectProperties{{Name: "word", IsPrimitive: true}, {Name: "age", IsPrimitive: true}}, - }, - outSearch: []*pb.SearchResult{ - { - Metadata: &pb.MetadataResult{}, - Properties: &pb.PropertiesResult{ - TargetCollection: className, - NonRefProps: &pb.Properties{ - Fields: map[string]*pb.Value{ - "word": {Kind: &pb.Value_StringValue{StringValue: "word"}}, - "age": {Kind: &pb.Value_IntValue{IntValue: 21}}, - }, - }, - RefProps: []*pb.RefPropertiesResult{}, - RefPropsRequested: false, - }, - }, - { - Metadata: &pb.MetadataResult{}, - Properties: &pb.PropertiesResult{ - TargetCollection: className, - NonRefProps: &pb.Properties{ - Fields: map[string]*pb.Value{ - "word": {Kind: &pb.Value_StringValue{StringValue: "other"}}, - "age": {Kind: &pb.Value_IntValue{IntValue: 26}}, - }, - }, - RefProps: []*pb.RefPropertiesResult{}, - RefPropsRequested: false, - }, - }, - }, - usesWeaviateStruct: true, - }, - { - name: "array properties", - res: []interface{}{ - map[string]interface{}{"nums": []float64{1, 2, 3}}, // ints are encoded as float64 in json - }, - searchParams: dto.GetParams{ - ClassName: className, - Properties: search.SelectProperties{{Name: "nums", IsPrimitive: true}}, - }, - outSearch: []*pb.SearchResult{ - { - Metadata: &pb.MetadataResult{}, - Properties: &pb.PropertiesResult{ - TargetCollection: className, - NonRefProps: &pb.Properties{ - Fields: map[string]*pb.Value{ - "nums": {Kind: &pb.Value_ListValue{ListValue: ignoreError(NewPrimitiveList([]float64{1, 2, 3}, schema.DataTypeInt))}}, - }, - }, - }, - }, - }, - usesWeaviateStruct: true, - }, - { - name: "array properties deprecated", - res: []interface{}{ - map[string]interface{}{"nums": []float64{1, 2, 3}}, // ints are encoded as float64 in json - }, - searchParams: dto.GetParams{ - ClassName: className, - Properties: search.SelectProperties{{Name: "nums", IsPrimitive: true}}, - }, - outSearch: []*pb.SearchResult{ - { - Metadata: &pb.MetadataResult{}, - Properties: &pb.PropertiesResult{ - TargetCollection: className, - NonRefProperties: newStruct(t, map[string]interface{}{}), - IntArrayProperties: []*pb.IntArrayProperties{{ - PropName: "nums", - Values: []int64{1, 2, 3}, - }}, - }, - }, - }, - usesWeaviateStruct: false, - }, - { - name: "nested object properties", - res: []interface{}{ - map[string]interface{}{ - "something": map[string]interface{}{ - "name": "Bob", - "names": []string{"Jo", "Jill"}, - "else": map[string]interface{}{ - "name": "Bill", - "names": []string{"Jo", "Jill"}, - }, - "objs": []interface{}{ - map[string]interface{}{"name": "Bill"}, - }, - }, - }, - }, - searchParams: dto.GetParams{ - ClassName: objClass, - Properties: search.SelectProperties{{ - Name: "something", - IsPrimitive: false, - IsObject: true, - Props: []search.SelectProperty{ - { - Name: "name", - IsPrimitive: true, - }, - { - Name: "names", - IsPrimitive: true, - }, - { - Name: "else", - IsPrimitive: false, - IsObject: true, - Props: []search.SelectProperty{ - { - Name: "name", - IsPrimitive: true, - }, - { - Name: "names", - IsPrimitive: true, - }, - }, - }, - { - Name: "objs", - IsPrimitive: false, - IsObject: true, - Props: []search.SelectProperty{{ - Name: "name", - IsPrimitive: true, - }}, - }, - }, - }}, - }, - outSearch: []*pb.SearchResult{ - { - Metadata: &pb.MetadataResult{}, - Properties: &pb.PropertiesResult{ - TargetCollection: objClass, - NonRefProps: &pb.Properties{ - Fields: map[string]*pb.Value{ - "something": {Kind: &pb.Value_ObjectValue{ - ObjectValue: &pb.Properties{ - Fields: map[string]*pb.Value{ - "name": {Kind: &pb.Value_StringValue{StringValue: "Bob"}}, - "names": {Kind: &pb.Value_ListValue{ListValue: ignoreError(NewPrimitiveList([]string{"Jo", "Jill"}, schema.DataTypeString))}}, - "else": {Kind: &pb.Value_ObjectValue{ - ObjectValue: &pb.Properties{ - Fields: map[string]*pb.Value{ - "name": {Kind: &pb.Value_StringValue{StringValue: "Bill"}}, - "names": {Kind: &pb.Value_ListValue{ListValue: ignoreError(NewPrimitiveList([]string{"Jo", "Jill"}, schema.DataTypeString))}}, - }, - }, - }}, - "objs": {Kind: &pb.Value_ListValue{ListValue: &pb.ListValue{ - Values: []*pb.Value{{Kind: &pb.Value_ObjectValue{ - ObjectValue: &pb.Properties{ - Fields: map[string]*pb.Value{ - "name": {Kind: &pb.Value_StringValue{StringValue: "Bill"}}, - }, - }, - }}}, - }}}, - }, - }, - }}, - }, - }, - }, - }, - }, - usesWeaviateStruct: true, - }, - { - name: "nested object properties deprecated", - res: []interface{}{ - map[string]interface{}{ - "something": map[string]interface{}{ - "name": "Bob", - "names": []string{"Jo", "Jill"}, - "else": map[string]interface{}{ - "name": "Bill", - "names": []string{"Jo", "Jill"}, - }, - "objs": []interface{}{ - map[string]interface{}{"name": "Bill"}, - }, - }, - }, - }, - searchParams: dto.GetParams{ - ClassName: objClass, - Properties: search.SelectProperties{{ - Name: "something", - IsPrimitive: false, - IsObject: true, - Props: []search.SelectProperty{ - { - Name: "name", - IsPrimitive: true, - }, - { - Name: "names", - IsPrimitive: true, - }, - { - Name: "else", - IsPrimitive: false, - IsObject: true, - Props: []search.SelectProperty{ - { - Name: "name", - IsPrimitive: true, - }, - { - Name: "names", - IsPrimitive: true, - }, - }, - }, - { - Name: "objs", - IsPrimitive: false, - IsObject: true, - Props: []search.SelectProperty{{ - Name: "name", - IsPrimitive: true, - }}, - }, - }, - }}, - }, - outSearch: []*pb.SearchResult{ - { - Metadata: &pb.MetadataResult{}, - Properties: &pb.PropertiesResult{ - TargetCollection: objClass, - ObjectProperties: []*pb.ObjectProperties{ - { - PropName: "something", - Value: &pb.ObjectPropertiesValue{ - NonRefProperties: newStruct(t, map[string]interface{}{ - "name": "Bob", - }), - TextArrayProperties: []*pb.TextArrayProperties{{ - PropName: "names", - Values: []string{"Jo", "Jill"}, - }}, - ObjectProperties: []*pb.ObjectProperties{{ - PropName: "else", - Value: &pb.ObjectPropertiesValue{ - NonRefProperties: newStruct(t, map[string]interface{}{ - "name": "Bill", - }), - TextArrayProperties: []*pb.TextArrayProperties{{ - PropName: "names", - Values: []string{"Jo", "Jill"}, - }}, - }, - }}, - ObjectArrayProperties: []*pb.ObjectArrayProperties{{ - PropName: "objs", - Values: []*pb.ObjectPropertiesValue{{ - NonRefProperties: newStruct(t, map[string]interface{}{ - "name": "Bill", - }), - }}, - }}, - }, - }, - }, - }, - }, - }, - usesWeaviateStruct: false, - }, - { - name: "primitive and ref properties with no references", - res: []interface{}{ - map[string]interface{}{ - "word": "word", - }, - map[string]interface{}{ - "word": "other", - }, - }, - searchParams: dto.GetParams{ - ClassName: className, - Properties: search.SelectProperties{ - {Name: "word", IsPrimitive: true}, - {Name: "ref", IsPrimitive: false, Refs: []search.SelectClass{ - { - ClassName: refClass1, - RefProperties: search.SelectProperties{{Name: "something", IsPrimitive: true}}, - AdditionalProperties: additional.Properties{Vector: true}, - }, - }}, - }, - }, - outSearch: []*pb.SearchResult{ - { - Metadata: &pb.MetadataResult{}, - Properties: &pb.PropertiesResult{ - TargetCollection: className, - NonRefProps: &pb.Properties{ - Fields: map[string]*pb.Value{ - "word": {Kind: &pb.Value_StringValue{StringValue: "word"}}, - }, - }, - RefProps: []*pb.RefPropertiesResult{}, - RefPropsRequested: true, - }, - }, - { - Metadata: &pb.MetadataResult{}, - Properties: &pb.PropertiesResult{ - TargetCollection: className, - NonRefProps: &pb.Properties{ - Fields: map[string]*pb.Value{ - "word": {Kind: &pb.Value_StringValue{StringValue: "other"}}, - }, - }, - RefProps: []*pb.RefPropertiesResult{}, - RefPropsRequested: true, - }, - }, - }, - usesWeaviateStruct: true, - }, - { - name: "primitive and ref properties", - res: []interface{}{ - map[string]interface{}{ - "word": "word", - "ref": []interface{}{ - search.LocalRef{ - Class: refClass1, - Fields: map[string]interface{}{ - "something": "other", - "_additional": map[string]interface{}{"vector": []float32{3}}, - }, - }, - }, - }, - map[string]interface{}{ - "word": "other", - "ref": []interface{}{ - search.LocalRef{ - Class: refClass1, - Fields: map[string]interface{}{ - "something": "thing", - "_additional": map[string]interface{}{"vector": []float32{4}}, - }, - }, - }, - }, - }, - searchParams: dto.GetParams{ - ClassName: className, - Properties: search.SelectProperties{ - {Name: "word", IsPrimitive: true}, - {Name: "ref", IsPrimitive: false, Refs: []search.SelectClass{ - { - ClassName: refClass1, - RefProperties: search.SelectProperties{{Name: "something", IsPrimitive: true}}, - AdditionalProperties: additional.Properties{Vector: true}, - }, - }}, - }, - }, - outSearch: []*pb.SearchResult{ - { - Metadata: &pb.MetadataResult{}, - Properties: &pb.PropertiesResult{ - TargetCollection: className, - NonRefProps: &pb.Properties{ - Fields: map[string]*pb.Value{ - "word": {Kind: &pb.Value_StringValue{StringValue: "word"}}, - }, - }, - RefProps: []*pb.RefPropertiesResult{{ - PropName: "ref", - Properties: []*pb.PropertiesResult{ - { - TargetCollection: refClass1, - Metadata: &pb.MetadataResult{Vector: []float32{3}, VectorBytes: byteVector([]float32{3})}, - NonRefProps: &pb.Properties{ - Fields: map[string]*pb.Value{ - "something": {Kind: &pb.Value_StringValue{StringValue: "other"}}, - }, - }, - }, - }, - }}, - RefPropsRequested: true, - }, - }, - { - Metadata: &pb.MetadataResult{}, - Properties: &pb.PropertiesResult{ - TargetCollection: className, - NonRefProps: &pb.Properties{ - Fields: map[string]*pb.Value{ - "word": {Kind: &pb.Value_StringValue{StringValue: "other"}}, - }, - }, - RefProps: []*pb.RefPropertiesResult{{ - PropName: "ref", - Properties: []*pb.PropertiesResult{ - { - TargetCollection: refClass1, - Metadata: &pb.MetadataResult{Vector: []float32{4}, VectorBytes: byteVector([]float32{4})}, - NonRefProps: &pb.Properties{ - Fields: map[string]*pb.Value{ - "something": {Kind: &pb.Value_StringValue{StringValue: "thing"}}, - }, - }, - }, - }, - }}, - RefPropsRequested: true, - }, - }, - }, - usesWeaviateStruct: true, - }, - { - name: "nested ref properties", - res: []interface{}{ - map[string]interface{}{ - "word": "word", - "ref": []interface{}{ - search.LocalRef{ - Class: refClass1, - Fields: map[string]interface{}{ - "something": "other", - "ref2": []interface{}{ - search.LocalRef{ - Class: refClass2, - Fields: map[string]interface{}{ - "else": "thing", - }, - }, - }, - }, - }, - }, - }, - }, - searchParams: dto.GetParams{ - ClassName: className, - Properties: search.SelectProperties{ - {Name: "word", IsPrimitive: true}, - { - Name: "ref", IsPrimitive: false, Refs: []search.SelectClass{ - { - ClassName: refClass1, - RefProperties: search.SelectProperties{ - {Name: "something", IsPrimitive: true}, - { - Name: "ref2", IsPrimitive: false, Refs: []search.SelectClass{{ - ClassName: refClass2, - RefProperties: search.SelectProperties{{Name: "else", IsPrimitive: true}}, - }}, - }, - }, - }, - }, - }, - }, - }, - outSearch: []*pb.SearchResult{ - { - Metadata: &pb.MetadataResult{}, - Properties: &pb.PropertiesResult{ - TargetCollection: className, - NonRefProps: &pb.Properties{ - Fields: map[string]*pb.Value{ - "word": {Kind: &pb.Value_StringValue{StringValue: "word"}}, - }, - }, - RefProps: []*pb.RefPropertiesResult{{ - PropName: "ref", - Properties: []*pb.PropertiesResult{ - { - TargetCollection: refClass1, - Metadata: &pb.MetadataResult{}, - NonRefProps: &pb.Properties{ - Fields: map[string]*pb.Value{ - "something": {Kind: &pb.Value_StringValue{StringValue: "other"}}, - }, - }, - RefProps: []*pb.RefPropertiesResult{{ - PropName: "ref2", - Properties: []*pb.PropertiesResult{{ - TargetCollection: refClass2, - Metadata: &pb.MetadataResult{}, - NonRefProps: &pb.Properties{ - Fields: map[string]*pb.Value{ - "else": {Kind: &pb.Value_StringValue{StringValue: "thing"}}, - }, - }, - RefProps: []*pb.RefPropertiesResult{}, - RefPropsRequested: false, - }}, - }}, - RefPropsRequested: true, - }, - }, - }}, - RefPropsRequested: true, - }, - }, - }, - usesWeaviateStruct: true, - }, - { - name: "nested ref properties with no references", - res: []interface{}{ - map[string]interface{}{ - "word": "word", - "ref": []interface{}{ - search.LocalRef{ - Class: refClass1, - Fields: map[string]interface{}{ - "something": "other", - }, - }, - }, - }, - }, - searchParams: dto.GetParams{ - ClassName: className, - Properties: search.SelectProperties{ - {Name: "word", IsPrimitive: true}, - { - Name: "ref", IsPrimitive: false, Refs: []search.SelectClass{ - { - ClassName: refClass1, - RefProperties: search.SelectProperties{ - {Name: "something", IsPrimitive: true}, - { - Name: "ref2", IsPrimitive: false, Refs: []search.SelectClass{{ - ClassName: refClass2, - RefProperties: search.SelectProperties{{Name: "else", IsPrimitive: true}}, - }}, - }, - }, - }, - }, - }, - }, - }, - outSearch: []*pb.SearchResult{ - { - Metadata: &pb.MetadataResult{}, - Properties: &pb.PropertiesResult{ - TargetCollection: className, - NonRefProps: &pb.Properties{ - Fields: map[string]*pb.Value{ - "word": {Kind: &pb.Value_StringValue{StringValue: "word"}}, - }, - }, - RefProps: []*pb.RefPropertiesResult{{ - PropName: "ref", - Properties: []*pb.PropertiesResult{ - { - TargetCollection: refClass1, - Metadata: &pb.MetadataResult{}, - NonRefProps: &pb.Properties{ - Fields: map[string]*pb.Value{ - "something": {Kind: &pb.Value_StringValue{StringValue: "other"}}, - }, - }, - RefProps: []*pb.RefPropertiesResult{}, - RefPropsRequested: true, - }, - }, - }}, - RefPropsRequested: true, - }, - }, - }, - usesWeaviateStruct: true, - }, - { - name: "primitive and ref array properties", - res: []interface{}{ - map[string]interface{}{ - "word": "word", - "ref": []interface{}{ - search.LocalRef{ - Class: refClass1, - Fields: map[string]interface{}{ - "nums": []float64{1, 2, 3}, // ints are encoded as float64 in json - "_additional": map[string]interface{}{"vector": []float32{3}}, - }, - }, - }, - }, - }, - searchParams: dto.GetParams{ - ClassName: className, - Properties: search.SelectProperties{ - {Name: "word", IsPrimitive: true}, - {Name: "ref", IsPrimitive: false, Refs: []search.SelectClass{ - { - ClassName: refClass1, - RefProperties: search.SelectProperties{{Name: "nums", IsPrimitive: true}}, - AdditionalProperties: additional.Properties{Vector: true}, - }, - }}, - }, - }, - outSearch: []*pb.SearchResult{ - { - Metadata: &pb.MetadataResult{}, - Properties: &pb.PropertiesResult{ - TargetCollection: className, - NonRefProps: &pb.Properties{ - Fields: map[string]*pb.Value{ - "word": {Kind: &pb.Value_StringValue{StringValue: "word"}}, - }, - }, - RefProps: []*pb.RefPropertiesResult{{ - PropName: "ref", - Properties: []*pb.PropertiesResult{ - { - TargetCollection: refClass1, - Metadata: &pb.MetadataResult{Vector: []float32{3}, VectorBytes: byteVector([]float32{3})}, - NonRefProps: &pb.Properties{ - Fields: map[string]*pb.Value{ - "nums": {Kind: &pb.Value_ListValue{ListValue: ignoreError(NewPrimitiveList([]float64{1, 2, 3}, schema.DataTypeInt))}}, - }, - }, - }, - }, - }}, - RefPropsRequested: true, - }, - }, - }, - usesWeaviateStruct: true, - }, - { - name: "primitive and ref array properties deprecated", - res: []interface{}{ - map[string]interface{}{ - "word": "word", - "ref": []interface{}{ - search.LocalRef{ - Class: refClass1, - Fields: map[string]interface{}{ - "nums": []float64{1, 2, 3}, // ints are encoded as float64 in json - "_additional": map[string]interface{}{"vector": []float32{3}}, - }, - }, - }, - }, - }, - searchParams: dto.GetParams{ - ClassName: className, - Properties: search.SelectProperties{ - {Name: "word", IsPrimitive: true}, - {Name: "ref", IsPrimitive: false, Refs: []search.SelectClass{ - { - ClassName: refClass1, - RefProperties: search.SelectProperties{{Name: "nums", IsPrimitive: true}}, - AdditionalProperties: additional.Properties{Vector: true}, - }, - }}, - }, - }, - outSearch: []*pb.SearchResult{ - { - Metadata: &pb.MetadataResult{}, - Properties: &pb.PropertiesResult{ - TargetCollection: className, - NonRefProperties: newStruct(t, map[string]interface{}{ - "word": "word", - }), - RefProps: []*pb.RefPropertiesResult{{ - PropName: "ref", - Properties: []*pb.PropertiesResult{ - { - TargetCollection: refClass1, - Metadata: &pb.MetadataResult{Vector: []float32{3}, VectorBytes: byteVector([]float32{3})}, - NonRefProperties: newStruct(t, map[string]interface{}{}), - IntArrayProperties: []*pb.IntArrayProperties{{ - PropName: "nums", - Values: []int64{1, 2, 3}, - }}, - }, - }, - }}, - }, - }, - }, - usesWeaviateStruct: false, - }, - { - name: "generative single only with ID", - res: []interface{}{ - map[string]interface{}{ - "_additional": map[string]interface{}{ - "id": UUID1, // different place for generative - "generate": &addModels.GenerateResult{SingleResult: &refClass1}, // just use some string - }, - }, - map[string]interface{}{ - "_additional": map[string]interface{}{ - "id": UUID2, - "generate": &addModels.GenerateResult{SingleResult: &refClass2}, - }, - }, - }, - searchParams: dto.GetParams{AdditionalProperties: additional.Properties{ - ID: true, - ModuleParams: map[string]interface{}{ - "generate": &generate.Params{ - Prompt: &refClass1, - }, - }, - }}, - outSearch: []*pb.SearchResult{ - { - Metadata: &pb.MetadataResult{ - Id: string(UUID1), - Generative: refClass1, - GenerativePresent: true, - }, - Properties: &pb.PropertiesResult{}, - }, - { - Metadata: &pb.MetadataResult{ - Id: string(UUID2), - Generative: refClass2, - GenerativePresent: true, - }, - Properties: &pb.PropertiesResult{}, - }, - }, - }, - { - name: "generative single only without ID", - res: []interface{}{ - map[string]interface{}{ - "_additional": map[string]interface{}{ // different place for generative - "generate": &addModels.GenerateResult{SingleResult: &refClass1}, // just use some string - }, - }, - map[string]interface{}{ - "_additional": map[string]interface{}{ - "generate": &addModels.GenerateResult{SingleResult: &refClass2}, - }, - }, - }, - searchParams: dto.GetParams{AdditionalProperties: additional.Properties{ - ModuleParams: map[string]interface{}{ - "generate": &generate.Params{ - Prompt: &refClass1, - }, - }, - }}, - outSearch: []*pb.SearchResult{ - { - Metadata: &pb.MetadataResult{ - Generative: refClass1, - GenerativePresent: true, - }, - Properties: &pb.PropertiesResult{}, - }, - { - Metadata: &pb.MetadataResult{ - Generative: refClass2, - GenerativePresent: true, - }, - Properties: &pb.PropertiesResult{}, - }, - }, - }, - { - name: "generative group only", - res: []interface{}{ - map[string]interface{}{ - "_additional": map[string]interface{}{ - "id": UUID1, // different place for generative - "generate": &addModels.GenerateResult{GroupedResult: &refClass1}, // just use some string - }, - }, - map[string]interface{}{ - "_additional": map[string]interface{}{ - "id": UUID2, - "generate": &addModels.GenerateResult{}, - }, - }, - }, - searchParams: dto.GetParams{AdditionalProperties: additional.Properties{ - ID: true, - ModuleParams: map[string]interface{}{ - "generate": &generate.Params{ - Task: &refClass1, - }, - }, - }}, - outSearch: []*pb.SearchResult{ - { - Metadata: &pb.MetadataResult{ - Id: string(UUID1), - }, - Properties: &pb.PropertiesResult{}, - }, - { - Metadata: &pb.MetadataResult{ - Id: string(UUID2), - }, - Properties: &pb.PropertiesResult{}, - }, - }, - outGenerative: refClass1, - }, - { - name: "group by", - res: []interface{}{ - map[string]interface{}{ - "_additional": map[string]interface{}{ - "id": UUID2, - "group": &additional.Group{ - ID: 1, - MinDistance: 0.1, - MaxDistance: 0.2, - Count: 3, - GroupedBy: &additional.GroupedBy{Value: "GroupByValue1", Path: []string{"some_prop"}}, - Hits: []map[string]interface{}{ - { - "word": "word", - "ref": []interface{}{ - search.LocalRef{ - Class: refClass1, - Fields: map[string]interface{}{ - "something": "other", - "_additional": map[string]interface{}{"vector": []float32{2}, "id": UUID1}, - }, - }, - }, - "_additional": &additional.GroupHitAdditional{Vector: []float32{3}, ID: UUID2}, - }, - { - "word": "other", - "_additional": &additional.GroupHitAdditional{Vector: []float32{4}, ID: UUID1}, - }, - }, - }, - }, - }, - }, - searchParams: dto.GetParams{AdditionalProperties: additional.Properties{ - ID: true, - Vector: true, - }, GroupBy: &searchparams.GroupBy{Groups: 3, ObjectsPerGroup: 4, Property: "name"}}, - outGroup: []*pb.GroupByResult{{ - Name: "GroupByValue1", - MaxDistance: 0.2, - MinDistance: 0.1, - NumberOfObjects: 3, - Objects: []*pb.SearchResult{ - { - Properties: &pb.PropertiesResult{ - NonRefProperties: newStruct(t, map[string]interface{}{"word": "word"}), - RefProps: []*pb.RefPropertiesResult{ - { - PropName: "other", - Properties: []*pb.PropertiesResult{ - { - NonRefProperties: newStruct(t, map[string]interface{}{"something": "other"}), - Metadata: &pb.MetadataResult{Vector: []float32{2}, Id: UUID1.String()}, - }, - }, - }, - }, - RefPropsRequested: true, - }, - Metadata: &pb.MetadataResult{ - Id: string(UUID2), - Vector: []float32{3}, - }, - }, - { - Properties: &pb.PropertiesResult{ - NonRefProperties: newStruct(t, map[string]interface{}{"word": "other"}), - }, - Metadata: &pb.MetadataResult{ - Id: string(UUID1), - Vector: []float32{4}, - }, - }, - }, - }}, - }, - { - name: "rerank only", - res: []interface{}{ - map[string]interface{}{ - "_additional": map[string]interface{}{ - "id": UUID1, - "rerank": []*addModels.RankResult{{Score: &someFloat64}}, - }, - }, - }, - searchParams: dto.GetParams{AdditionalProperties: additional.Properties{ - ID: true, - ModuleParams: map[string]interface{}{"rerank": "must be present for extraction"}, - }}, - outSearch: []*pb.SearchResult{ - { - Metadata: &pb.MetadataResult{ - Id: string(UUID1), - RerankScore: someFloat64, - RerankScorePresent: true, - }, - Properties: &pb.PropertiesResult{}, - }, - }, - }, - { - name: "generate, group by, & rerank", - res: []interface{}{ - map[string]interface{}{ - "_additional": map[string]interface{}{ - "id": UUID2, - "generate": &addModels.GenerateResult{ - SingleResult: &refClass1, - GroupedResult: &refClass2, - }, - "rerank": []*addModels.RankResult{{Score: &someFloat64}}, - "group": &additional.Group{ - ID: 1, - MinDistance: 0.1, - MaxDistance: 0.2, - Count: 3, - GroupedBy: &additional.GroupedBy{Value: "GroupByValue1", Path: []string{"some_prop"}}, - Hits: []map[string]interface{}{ - { - "word": "word", - "ref": []interface{}{ - search.LocalRef{ - Class: refClass1, - Fields: map[string]interface{}{ - "something": "other", - "_additional": map[string]interface{}{"vector": []float32{2}, "id": UUID1}, - }, - }, - }, - "_additional": &additional.GroupHitAdditional{Vector: []float32{3}, ID: UUID2}, - }, - { - "word": "other", - "_additional": &additional.GroupHitAdditional{Vector: []float32{4}, ID: UUID1}, - }, - }, - }, - }, - }, - }, - searchParams: dto.GetParams{ - AdditionalProperties: additional.Properties{ - ID: true, - Vector: true, - ModuleParams: map[string]interface{}{ - "generate": &generate.Params{ - Prompt: &refClass1, - Task: &refClass2, - }, - "rerank": "must be present for extraction", - }, - }, - GroupBy: &searchparams.GroupBy{Groups: 3, ObjectsPerGroup: 4, Property: "name"}, - }, - outGroup: []*pb.GroupByResult{{ - Name: "GroupByValue1", - MaxDistance: 0.2, - MinDistance: 0.1, - NumberOfObjects: 3, - Generative: &pb.GenerativeReply{Result: refClass1}, - Rerank: &pb.RerankReply{Score: someFloat64}, - Objects: []*pb.SearchResult{ - { - Properties: &pb.PropertiesResult{ - NonRefProperties: newStruct(t, map[string]interface{}{"word": "word"}), - RefProps: []*pb.RefPropertiesResult{ - { - PropName: "other", - Properties: []*pb.PropertiesResult{ - { - NonRefProperties: newStruct(t, map[string]interface{}{"something": "other"}), - Metadata: &pb.MetadataResult{Vector: []float32{2}, Id: UUID1.String()}, - }, - }, - }, - }, - RefPropsRequested: true, - }, - Metadata: &pb.MetadataResult{ - Id: string(UUID2), - Vector: []float32{3}, - }, - }, - { - Properties: &pb.PropertiesResult{ - NonRefProperties: newStruct(t, map[string]interface{}{"word": "other"}), - }, - Metadata: &pb.MetadataResult{ - Id: string(UUID1), - Vector: []float32{4}, - }, - }, - }, - }}, - outGenerative: refClass2, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - out, err := searchResultsToProto(tt.res, time.Now(), tt.searchParams, scheme, tt.usesWeaviateStruct) - require.Nil(t, err) - for i := range tt.outSearch { - require.Equal(t, tt.outSearch[i].Properties.String(), out.Results[i].Properties.String()) - require.Equal(t, tt.outSearch[i].Metadata.String(), out.Results[i].Metadata.String()) - } - require.Equal(t, tt.outGenerative, *out.GenerativeGroupedResult) - }) - } -} diff --git a/adapters/handlers/grpc/v1/service.go b/adapters/handlers/grpc/v1/service.go deleted file mode 100644 index 9679564db27d756dabfc583c088b16ed772333d0..0000000000000000000000000000000000000000 --- a/adapters/handlers/grpc/v1/service.go +++ /dev/null @@ -1,212 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package v1 - -import ( - "context" - "fmt" - "time" - - "github.com/weaviate/weaviate/usecases/objects" - - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/dto" - "github.com/weaviate/weaviate/entities/schema" - pb "github.com/weaviate/weaviate/grpc/generated/protocol/v1" - "github.com/weaviate/weaviate/usecases/auth/authentication/composer" - schemaManager "github.com/weaviate/weaviate/usecases/schema" - "github.com/weaviate/weaviate/usecases/traverser" -) - -type Service struct { - pb.UnimplementedWeaviateServer - traverser *traverser.Traverser - authComposer composer.TokenFunc - allowAnonymousAccess bool - schemaManager *schemaManager.Manager - batchManager *objects.BatchManager -} - -func NewService(traverser *traverser.Traverser, authComposer composer.TokenFunc, - allowAnonymousAccess bool, schemaManager *schemaManager.Manager, - batchManager *objects.BatchManager, -) *Service { - return &Service{ - traverser: traverser, - authComposer: authComposer, - allowAnonymousAccess: allowAnonymousAccess, - schemaManager: schemaManager, - batchManager: batchManager, - } -} - -func (s *Service) BatchDelete(ctx context.Context, req *pb.BatchDeleteRequest) (*pb.BatchDeleteReply, error) { - before := time.Now() - principal, err := s.principalFromContext(ctx) - if err != nil { - return nil, fmt.Errorf("extract auth: %w", err) - } - replicationProperties := extractReplicationProperties(req.ConsistencyLevel) - scheme := s.schemaManager.GetSchemaSkipAuth() - - params, err := batchDeleteParamsFromProto(req, scheme) - if err != nil { - return nil, fmt.Errorf("batch delete params: %w", err) - } - - tenant := "" - if req.Tenant != nil { - tenant = *req.Tenant - } - - response, err := s.batchManager.DeleteObjectsFromGRPC(ctx, principal, params, replicationProperties, tenant) - if err != nil { - return nil, fmt.Errorf("batch delete: %w", err) - } - - result, err := batchDeleteReplyFromObjects(response, req.Verbose) - if err != nil { - return nil, fmt.Errorf("batch delete reply: %w", err) - } - result.Took = float32(time.Since(before).Seconds()) - - return result, nil -} - -func (s *Service) BatchObjects(ctx context.Context, req *pb.BatchObjectsRequest) (*pb.BatchObjectsReply, error) { - before := time.Now() - principal, err := s.principalFromContext(ctx) - if err != nil { - return nil, fmt.Errorf("extract auth: %w", err) - } - scheme := s.schemaManager.GetSchemaSkipAuth() - - objs, objOriginalIndex, objectParsingErrors := batchFromProto(req, scheme) - - replicationProperties := extractReplicationProperties(req.ConsistencyLevel) - - all := "ALL" - response, err := s.batchManager.AddObjects(ctx, principal, objs, []*string{&all}, replicationProperties) - if err != nil { - return nil, err - } - var objErrors []*pb.BatchObjectsReply_BatchError - - for i, obj := range response { - if obj.Err != nil { - objErrors = append(objErrors, &pb.BatchObjectsReply_BatchError{Index: int32(objOriginalIndex[i]), Error: obj.Err.Error()}) - } - } - - for i, err := range objectParsingErrors { - objErrors = append(objErrors, &pb.BatchObjectsReply_BatchError{Index: int32(i), Error: err.Error()}) - } - - result := &pb.BatchObjectsReply{ - Took: float32(time.Since(before).Seconds()), - Errors: objErrors, - } - return result, nil -} - -func (s *Service) Search(ctx context.Context, req *pb.SearchRequest) (*pb.SearchReply, error) { - before := time.Now() - - principal, err := s.principalFromContext(ctx) - if err != nil { - return nil, fmt.Errorf("extract auth: %w", err) - } - - scheme := s.schemaManager.GetSchemaSkipAuth() - - type reply struct { - Result *pb.SearchReply - Error error - } - - c := make(chan reply, 1) - go func() { - defer func() { - if err := recover(); err != nil { - c <- reply{ - Result: nil, - Error: fmt.Errorf("panic occurred: %v", err), - } - } - }() - - searchParams, err := searchParamsFromProto(req, scheme) - if err != nil { - c <- reply{ - Result: nil, - Error: fmt.Errorf("extract params: %w", err), - } - } - - if err := s.validateClassAndProperty(searchParams); err != nil { - c <- reply{ - Result: nil, - Error: err, - } - } - - res, err := s.traverser.GetClass(ctx, principal, searchParams) - if err != nil { - c <- reply{ - Result: nil, - Error: err, - } - } - - proto, err := searchResultsToProto(res, before, searchParams, scheme, req.Uses_123Api) - c <- reply{ - Result: proto, - Error: err, - } - }() - res := <-c - return res.Result, res.Error -} - -func (s *Service) validateClassAndProperty(searchParams dto.GetParams) error { - scheme := s.schemaManager.GetSchemaSkipAuth() - class, err := schema.GetClassByName(scheme.Objects, searchParams.ClassName) - if err != nil { - return err - } - - for _, prop := range searchParams.Properties { - _, err := schema.GetPropertyByName(class, prop.Name) - if err != nil { - return err - } - } - - return nil -} - -func extractReplicationProperties(level *pb.ConsistencyLevel) *additional.ReplicationProperties { - if level == nil { - return nil - } - - switch *level { - case pb.ConsistencyLevel_CONSISTENCY_LEVEL_ONE: - return &additional.ReplicationProperties{ConsistencyLevel: "ONE"} - case pb.ConsistencyLevel_CONSISTENCY_LEVEL_QUORUM: - return &additional.ReplicationProperties{ConsistencyLevel: "QUORUM"} - case pb.ConsistencyLevel_CONSISTENCY_LEVEL_ALL: - return &additional.ReplicationProperties{ConsistencyLevel: "ALL"} - default: - return nil - } -} diff --git a/adapters/handlers/rest/clusterapi/auth.go b/adapters/handlers/rest/clusterapi/auth.go deleted file mode 100644 index 93e014a87acaf8806fe3d3f14c4492f52dacb7e1..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/clusterapi/auth.go +++ /dev/null @@ -1,55 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clusterapi - -import ( - "net/http" - - "github.com/weaviate/weaviate/usecases/cluster" -) - -type auth interface { - handleFunc(handler http.HandlerFunc) http.HandlerFunc -} - -type basicAuthHandler struct { - basicAuth cluster.BasicAuth -} - -func NewBasicAuthHandler(authConfig cluster.AuthConfig) auth { - return &basicAuthHandler{authConfig.BasicAuth} -} - -func (h *basicAuthHandler) handleFunc(handler http.HandlerFunc) http.HandlerFunc { - if !h.basicAuth.Enabled() { - return handler - } - return func(w http.ResponseWriter, r *http.Request) { - u, p, ok := r.BasicAuth() - if ok && u == h.basicAuth.Username && p == h.basicAuth.Password { - handler(w, r) - return - } - // unauthorized request, send 401 - w.WriteHeader(401) - } -} - -type noopAuthHandler struct{} - -func NewNoopAuthHandler() auth { - return &noopAuthHandler{} -} - -func (h *noopAuthHandler) handleFunc(handler http.HandlerFunc) http.HandlerFunc { - return handler -} diff --git a/adapters/handlers/rest/clusterapi/backups.go b/adapters/handlers/rest/clusterapi/backups.go deleted file mode 100644 index 61a2cd5caaba1ffce9fb4128870e608d915cbfc7..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/clusterapi/backups.go +++ /dev/null @@ -1,156 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clusterapi - -import ( - "context" - "encoding/json" - "fmt" - "io" - "net/http" - - "github.com/weaviate/weaviate/usecases/backup" -) - -type backupManager interface { - OnCanCommit(ctx context.Context, req *backup.Request) *backup.CanCommitResponse - OnCommit(ctx context.Context, req *backup.StatusRequest) error - OnAbort(ctx context.Context, req *backup.AbortRequest) error - OnStatus(ctx context.Context, req *backup.StatusRequest) *backup.StatusResponse -} - -type backups struct { - manager backupManager - auth auth -} - -func NewBackups(manager backupManager, auth auth) *backups { - return &backups{manager: manager, auth: auth} -} - -func (b *backups) CanCommit() http.Handler { - return b.auth.handleFunc(b.canCommitHandler()) -} - -func (b *backups) canCommitHandler() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - body, err := io.ReadAll(r.Body) - if err != nil { - status := http.StatusInternalServerError - http.Error(w, fmt.Errorf("read request body: %w", err).Error(), status) - return - } - defer r.Body.Close() - - var req backup.Request - if err := json.Unmarshal(body, &req); err != nil { - status := http.StatusInternalServerError - http.Error(w, fmt.Errorf("unmarshal request: %w", err).Error(), status) - return - } - - resp := b.manager.OnCanCommit(r.Context(), &req) - b, err := json.Marshal(&resp) - if err != nil { - status := http.StatusInternalServerError - http.Error(w, fmt.Errorf("marshal response: %w", err).Error(), status) - return - } - - w.WriteHeader(http.StatusOK) - w.Write(b) - } -} - -func (b *backups) Commit() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - body, err := io.ReadAll(r.Body) - if err != nil { - status := http.StatusInternalServerError - http.Error(w, fmt.Errorf("read request body: %w", err).Error(), status) - return - } - defer r.Body.Close() - - var req backup.StatusRequest - if err := json.Unmarshal(body, &req); err != nil { - status := http.StatusInternalServerError - http.Error(w, fmt.Errorf("unmarshal request: %w", err).Error(), status) - return - } - - if err := b.manager.OnCommit(r.Context(), &req); err != nil { - status := http.StatusInternalServerError - http.Error(w, fmt.Errorf("commit: %w", err).Error(), status) - return - } - - w.WriteHeader(http.StatusCreated) - }) -} - -func (b *backups) Abort() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - body, err := io.ReadAll(r.Body) - if err != nil { - status := http.StatusInternalServerError - http.Error(w, fmt.Errorf("read request body: %w", err).Error(), status) - return - } - defer r.Body.Close() - - var req backup.AbortRequest - if err := json.Unmarshal(body, &req); err != nil { - status := http.StatusInternalServerError - http.Error(w, fmt.Errorf("unmarshal request: %w", err).Error(), status) - return - } - - if err := b.manager.OnAbort(r.Context(), &req); err != nil { - status := http.StatusInternalServerError - http.Error(w, fmt.Errorf("abort: %w", err).Error(), status) - return - } - - w.WriteHeader(http.StatusNoContent) - }) -} - -func (b *backups) Status() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - body, err := io.ReadAll(r.Body) - if err != nil { - status := http.StatusInternalServerError - http.Error(w, fmt.Errorf("read request body: %w", err).Error(), status) - return - } - defer r.Body.Close() - - var req backup.StatusRequest - if err := json.Unmarshal(body, &req); err != nil { - status := http.StatusInternalServerError - http.Error(w, fmt.Errorf("unmarshal request: %w", err).Error(), status) - return - } - - resp := b.manager.OnStatus(r.Context(), &req) - b, err := json.Marshal(&resp) - if err != nil { - status := http.StatusInternalServerError - http.Error(w, fmt.Errorf("marshal response: %w", err).Error(), status) - return - } - - w.WriteHeader(http.StatusOK) - w.Write(b) - }) -} diff --git a/adapters/handlers/rest/clusterapi/backups_test.go b/adapters/handlers/rest/clusterapi/backups_test.go deleted file mode 100644 index 1ab6ffdc6acb9826b54215ed03ab1cac4ea38136..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/clusterapi/backups_test.go +++ /dev/null @@ -1,180 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clusterapi_test - -import ( - "context" - "net/http" - "net/http/httptest" - "net/url" - "testing" - - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/adapters/clients" - "github.com/weaviate/weaviate/adapters/handlers/rest/clusterapi" - "github.com/weaviate/weaviate/usecases/backup" -) - -func TestInternalBackupsAPI(t *testing.T) { - nodes := []*backupNode{ - { - name: "node1", - backupManager: &fakeBackupManager{}, - }, - { - name: "node2", - backupManager: &fakeBackupManager{}, - }, - } - hosts := setupClusterAPI(t, nodes) - - for _, node := range nodes { - node.backupManager.On("OnCanCommit", &backup.Request{Method: backup.OpCreate}). - Return(&backup.CanCommitResponse{}) - node.backupManager.On("OnCommit", &backup.StatusRequest{}).Return(nil) - node.backupManager.On("OnAbort", &backup.AbortRequest{}).Return(nil) - } - - coord := newFakeCoordinator(newFakeNodeResolver(hosts)) - - t.Run("can commit, commit", func(t *testing.T) { - err := coord.Backup(context.Background(), &backup.Request{Method: backup.OpCreate}, false) - require.Nil(t, err) - }) - - t.Run("abort", func(t *testing.T) { - err := coord.Backup(context.Background(), &backup.Request{Method: backup.OpCreate}, true) - require.Nil(t, err) - }) -} - -func setupClusterAPI(t *testing.T, nodes []*backupNode) map[string]string { - hosts := make(map[string]string) - - for _, node := range nodes { - backupsHandler := clusterapi.NewBackups(node.backupManager, clusterapi.NewNoopAuthHandler()) - - mux := http.NewServeMux() - mux.Handle("/backups/can-commit", backupsHandler.CanCommit()) - mux.Handle("/backups/commit", backupsHandler.Commit()) - mux.Handle("/backups/abort", backupsHandler.Abort()) - mux.Handle("/backups/status", backupsHandler.Status()) - server := httptest.NewServer(mux) - - parsedURL, err := url.Parse(server.URL) - require.Nil(t, err) - - hosts[node.name] = parsedURL.Host - } - - return hosts -} - -type backupNode struct { - name string - backupManager *fakeBackupManager -} - -func newFakeNodeResolver(hosts map[string]string) *fakeNodeResolver { - return &fakeNodeResolver{hosts: hosts} -} - -type fakeNodeResolver struct { - hosts map[string]string -} - -func (r *fakeNodeResolver) NodeHostName(nodeName string) (string, bool) { - if host, ok := r.hosts[nodeName]; ok { - return host, true - } - return "", false -} - -func (r *fakeNodeResolver) HostNames() []string { - hosts := make([]string, len(r.hosts)) - count := 0 - for _, host := range r.hosts { - hosts[count] = host - count++ - } - return hosts -} - -func newFakeCoordinator(resolver *fakeNodeResolver) *fakeCoordinator { - return &fakeCoordinator{ - client: clients.NewClusterBackups(&http.Client{}), - nodeResolver: resolver, - } -} - -type fakeCoordinator struct { - client *clients.ClusterBackups - nodeResolver *fakeNodeResolver -} - -func (c *fakeCoordinator) Backup(ctx context.Context, req *backup.Request, abort bool) error { - if abort { - return c.abort(ctx) - } - - for _, host := range c.nodeResolver.HostNames() { - _, err := c.client.CanCommit(ctx, host, req) - if err != nil { - return err - } - } - - for _, host := range c.nodeResolver.HostNames() { - err := c.client.Commit(ctx, host, &backup.StatusRequest{}) - if err != nil { - return err - } - } - - return nil -} - -func (c *fakeCoordinator) abort(ctx context.Context) error { - for _, host := range c.nodeResolver.HostNames() { - err := c.client.Abort(ctx, host, &backup.AbortRequest{}) - if err != nil { - return err - } - } - - return nil -} - -type fakeBackupManager struct { - mock.Mock -} - -func (m *fakeBackupManager) OnCanCommit(ctx context.Context, req *backup.Request) *backup.CanCommitResponse { - args := m.Called(req) - return args.Get(0).(*backup.CanCommitResponse) -} - -func (m *fakeBackupManager) OnCommit(ctx context.Context, req *backup.StatusRequest) error { - args := m.Called(req) - return args.Error(0) -} - -func (m *fakeBackupManager) OnAbort(ctx context.Context, req *backup.AbortRequest) error { - args := m.Called(req) - return args.Error(0) -} - -func (m *fakeBackupManager) OnStatus(ctx context.Context, req *backup.StatusRequest) *backup.StatusResponse { - args := m.Called(req) - return args.Get(0).(*backup.StatusResponse) -} diff --git a/adapters/handlers/rest/clusterapi/classifications.go b/adapters/handlers/rest/clusterapi/classifications.go deleted file mode 100644 index 264bbdb454f3a19ece0265aab1f352331c0d0bd0..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/clusterapi/classifications.go +++ /dev/null @@ -1,20 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clusterapi - -type classifications struct { - txHandler -} - -func NewClassifications(manager txManager, auth auth) *classifications { - return &classifications{txHandler{manager: manager, auth: auth}} -} diff --git a/adapters/handlers/rest/clusterapi/fakes_for_test.go b/adapters/handlers/rest/clusterapi/fakes_for_test.go deleted file mode 100644 index 21d4026f965dcd3c892ddcce2eeb0694840ada5f..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/clusterapi/fakes_for_test.go +++ /dev/null @@ -1,279 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clusterapi_test - -import ( - "context" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/models" - schemaent "github.com/weaviate/weaviate/entities/schema" - ucs "github.com/weaviate/weaviate/usecases/schema" - "github.com/weaviate/weaviate/usecases/schema/migrate" - "github.com/weaviate/weaviate/usecases/sharding" -) - -type fakeRepo struct { - schema ucs.State -} - -func newFakeRepo() *fakeRepo { - return &fakeRepo{ - schema: ucs.NewState(1), - } -} - -func (f *fakeRepo) Save(ctx context.Context, schema ucs.State) error { - f.schema = schema - return nil -} - -func (f *fakeRepo) Load(context.Context) (ucs.State, error) { - return f.schema, nil -} - -func (f *fakeRepo) NewClass(context.Context, ucs.ClassPayload) error { - return nil -} - -func (f *fakeRepo) UpdateClass(context.Context, ucs.ClassPayload) error { - return nil -} - -func (f *fakeRepo) DeleteClass(ctx context.Context, class string) error { - return nil -} - -func (f *fakeRepo) NewShards(ctx context.Context, class string, shards []ucs.KeyValuePair) error { - return nil -} - -func (f *fakeRepo) UpdateShards(ctx context.Context, class string, shards []ucs.KeyValuePair) error { - return nil -} - -func (f *fakeRepo) DeleteShards(ctx context.Context, class string, shards []string) error { - return nil -} - -type fakeAuthorizer struct{} - -func (f *fakeAuthorizer) Authorize(principal *models.Principal, verb, resource string) error { - return nil -} - -type fakeVectorConfig map[string]interface{} - -func (f fakeVectorConfig) IndexType() string { - return "fake" -} - -func (f fakeVectorConfig) DistanceName() string { - return "fake" -} - -func dummyParseVectorConfig(in interface{}, indexType string) (schemaent.VectorIndexConfig, error) { - return fakeVectorConfig(in.(map[string]interface{})), nil -} - -func dummyValidateInvertedConfig(in *models.InvertedIndexConfig) error { - return nil -} - -type fakeVectorizerValidator struct { - valid []string -} - -func (f *fakeVectorizerValidator) ValidateVectorizer(moduleName string) error { - for _, valid := range f.valid { - if moduleName == valid { - return nil - } - } - - return errors.Errorf("invalid vectorizer %q", moduleName) -} - -type fakeModuleConfig struct{} - -func (f *fakeModuleConfig) SetClassDefaults(class *models.Class) { - defaultConfig := map[string]interface{}{ - "my-module1": map[string]interface{}{ - "my-setting": "default-value", - }, - } - - asMap, ok := class.ModuleConfig.(map[string]interface{}) - if !ok { - class.ModuleConfig = defaultConfig - return - } - - module, ok := asMap["my-module1"] - if !ok { - class.ModuleConfig = defaultConfig - return - } - - asMap, ok = module.(map[string]interface{}) - if !ok { - class.ModuleConfig = defaultConfig - return - } - - if _, ok := asMap["my-setting"]; !ok { - asMap["my-setting"] = "default-value" - defaultConfig["my-module1"] = asMap - class.ModuleConfig = defaultConfig - } -} - -func (f *fakeModuleConfig) SetSinglePropertyDefaults(class *models.Class, - prop *models.Property) { -} - -func (f *fakeModuleConfig) ValidateClass(ctx context.Context, class *models.Class) error { - return nil -} - -type fakeClusterState struct { - hosts []string -} - -func (f *fakeClusterState) SchemaSyncIgnored() bool { - return false -} - -func (f *fakeClusterState) SkipSchemaRepair() bool { - return false -} - -func (f *fakeClusterState) Hostnames() []string { - return f.hosts -} - -func (f *fakeClusterState) AllNames() []string { - return f.hosts -} - -func (f *fakeClusterState) Candidates() []string { - return f.hosts -} - -func (f *fakeClusterState) LocalName() string { - return f.hosts[0] -} - -func (f *fakeClusterState) NodeCount() int { - return len(f.hosts) -} - -func (f *fakeClusterState) ClusterHealthScore() int { - // 0 - healthy, >0 - unhealthy - return 0 -} - -func (f *fakeClusterState) ResolveParentNodes(string, string) (map[string]string, error) { - return nil, nil -} - -func (f *fakeClusterState) NodeHostname(nodeName string) (string, bool) { - return "", false -} - -type NilMigrator struct{} - -func (n *NilMigrator) AddClass(ctx context.Context, class *models.Class, - shardingState *sharding.State, -) error { - return nil -} - -func (n *NilMigrator) DropClass(ctx context.Context, className string) error { - return nil -} - -func (n *NilMigrator) UpdateClass(ctx context.Context, className string, newClassName *string) error { - return nil -} - -func (n *NilMigrator) GetShardsQueueSize(ctx context.Context, className, tenant string) (map[string]int64, error) { - return nil, nil -} - -func (n *NilMigrator) GetShardsStatus(ctx context.Context, className, tenant string) (map[string]string, error) { - return nil, nil -} - -func (n *NilMigrator) UpdateShardStatus(ctx context.Context, className, shardName, targetStatus string) error { - return nil -} - -func (n *NilMigrator) AddProperty(ctx context.Context, className string, prop *models.Property) error { - return nil -} - -func (n *NilMigrator) NewTenants(ctx context.Context, class *models.Class, creates []*migrate.CreateTenantPayload) (commit func(success bool), err error) { - return nil, nil -} - -func (n *NilMigrator) UpdateTenants(ctx context.Context, class *models.Class, updates []*migrate.UpdateTenantPayload) (commit func(success bool), err error) { - return nil, nil -} - -func (n *NilMigrator) DeleteTenants(ctx context.Context, class *models.Class, partitions []string) (commit func(success bool), err error) { - return nil, nil -} - -func (n *NilMigrator) UpdateProperty(ctx context.Context, className string, propName string, newName *string) error { - return nil -} - -func (n *NilMigrator) UpdatePropertyAddDataType(ctx context.Context, className string, propName string, newDataType string) error { - return nil -} - -func (n *NilMigrator) DropProperty(ctx context.Context, className string, propName string) error { - return nil -} - -func (n *NilMigrator) ValidateVectorIndexConfigUpdate(ctx context.Context, old, updated schemaent.VectorIndexConfig) error { - return nil -} - -func (n *NilMigrator) UpdateVectorIndexConfig(ctx context.Context, className string, updated schemaent.VectorIndexConfig) error { - return nil -} - -func (n *NilMigrator) ValidateInvertedIndexConfigUpdate(ctx context.Context, old, updated *models.InvertedIndexConfig) error { - return nil -} - -func (n *NilMigrator) UpdateInvertedIndexConfig(ctx context.Context, className string, updated *models.InvertedIndexConfig) error { - return nil -} - -func (n *NilMigrator) RecalculateVectorDimensions(ctx context.Context) error { - return nil -} - -func (n *NilMigrator) RecountProperties(ctx context.Context) error { - return nil -} - -func (n *NilMigrator) InvertedReindex(ctx context.Context, taskNames ...string) error { - return nil -} - -func (n *NilMigrator) AdjustFilterablePropSettings(ctx context.Context) error { - return nil -} diff --git a/adapters/handlers/rest/clusterapi/indices.go b/adapters/handlers/rest/clusterapi/indices.go deleted file mode 100644 index 6ae4be71689b9319f3821e0e53b00e2aa0d56ef0..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/clusterapi/indices.go +++ /dev/null @@ -1,1082 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clusterapi - -import ( - "context" - "encoding/base64" - "encoding/json" - "fmt" - "io" - "net/http" - "regexp" - - "github.com/go-openapi/strfmt" - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/aggregation" - "github.com/weaviate/weaviate/entities/filters" - entschema "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/search" - "github.com/weaviate/weaviate/entities/searchparams" - "github.com/weaviate/weaviate/entities/storobj" - "github.com/weaviate/weaviate/usecases/objects" - "github.com/weaviate/weaviate/usecases/replica" -) - -type indices struct { - shards shards - db db - auth auth - regexpObjects *regexp.Regexp - regexpObjectsOverwrite *regexp.Regexp - regexObjectsDigest *regexp.Regexp - regexpObjectsSearch *regexp.Regexp - regexpObjectsFind *regexp.Regexp - - regexpObjectsAggregations *regexp.Regexp - regexpObject *regexp.Regexp - regexpReferences *regexp.Regexp - regexpShardsQueueSize *regexp.Regexp - regexpShardsStatus *regexp.Regexp - regexpShardFiles *regexp.Regexp - regexpShard *regexp.Regexp - regexpShardReinit *regexp.Regexp -} - -const ( - cl = entschema.ClassNameRegexCore - sh = entschema.ShardNameRegexCore - ob = `[A-Za-z0-9_+-]+` - - urlPatternObjects = `\/indices\/(` + cl + `)` + - `\/shards\/(` + sh + `)\/objects` - urlPatternObjectsOverwrite = `\/indices\/(` + cl + `)` + - `\/shards\/(` + sh + `)\/objects:overwrite` - urlPatternObjectsDigest = `\/indices\/(` + cl + `)` + - `\/shards\/(` + sh + `)\/objects:digest` - urlPatternObjectsSearch = `\/indices\/(` + cl + `)` + - `\/shards\/(` + sh + `)\/objects\/_search` - urlPatternObjectsFind = `\/indices\/(` + cl + `)` + - `\/shards\/(` + sh + `)\/objects\/_find` - urlPatternObjectsAggregations = `\/indices\/(` + cl + `)` + - `\/shards\/(` + sh + `)\/objects\/_aggregations` - urlPatternObject = `\/indices\/(` + cl + `)` + - `\/shards\/(` + sh + `)\/objects\/(` + ob + `)` - urlPatternReferences = `\/indices\/(` + cl + `)` + - `\/shards\/(` + sh + `)\/references` - urlPatternShardsQueueSize = `\/indices\/(` + cl + `)` + - `\/shards\/(` + sh + `)\/queuesize` - urlPatternShardsStatus = `\/indices\/(` + cl + `)` + - `\/shards\/(` + sh + `)\/status` - urlPatternShardFiles = `\/indices\/(` + cl + `)` + - `\/shards\/(` + sh + `)\/files/(.*)` - urlPatternShard = `\/indices\/(` + cl + `)` + - `\/shards\/(` + sh + `)$` - urlPatternShardReinit = `\/indices\/(` + cl + `)` + - `\/shards\/(` + sh + `):reinit` -) - -type shards interface { - PutObject(ctx context.Context, indexName, shardName string, - obj *storobj.Object) error - BatchPutObjects(ctx context.Context, indexName, shardName string, - objs []*storobj.Object) []error - BatchAddReferences(ctx context.Context, indexName, shardName string, - refs objects.BatchReferences) []error - GetObject(ctx context.Context, indexName, shardName string, - id strfmt.UUID, selectProperties search.SelectProperties, - additional additional.Properties) (*storobj.Object, error) - Exists(ctx context.Context, indexName, shardName string, - id strfmt.UUID) (bool, error) - DeleteObject(ctx context.Context, indexName, shardName string, - id strfmt.UUID) error - MergeObject(ctx context.Context, indexName, shardName string, - mergeDoc objects.MergeDocument) error - MultiGetObjects(ctx context.Context, indexName, shardName string, - id []strfmt.UUID) ([]*storobj.Object, error) - Search(ctx context.Context, indexName, shardName string, - vector []float32, distance float32, limit int, filters *filters.LocalFilter, - keywordRanking *searchparams.KeywordRanking, sort []filters.Sort, - cursor *filters.Cursor, groupBy *searchparams.GroupBy, - additional additional.Properties, - ) ([]*storobj.Object, []float32, error) - Aggregate(ctx context.Context, indexName, shardName string, - params aggregation.Params) (*aggregation.Result, error) - FindUUIDs(ctx context.Context, indexName, shardName string, - filters *filters.LocalFilter) ([]strfmt.UUID, error) - DeleteObjectBatch(ctx context.Context, indexName, shardName string, - uuids []strfmt.UUID, dryRun bool) objects.BatchSimpleObjects - GetShardQueueSize(ctx context.Context, indexName, shardName string) (int64, error) - GetShardStatus(ctx context.Context, indexName, shardName string) (string, error) - UpdateShardStatus(ctx context.Context, indexName, shardName, - targetStatus string) error - - // Replication-specific - OverwriteObjects(ctx context.Context, indexName, shardName string, - vobjects []*objects.VObject) ([]replica.RepairResponse, error) - DigestObjects(ctx context.Context, indexName, shardName string, - ids []strfmt.UUID) (result []replica.RepairResponse, err error) - - // Scale-out Replication POC - FilePutter(ctx context.Context, indexName, shardName, - filePath string) (io.WriteCloser, error) - CreateShard(ctx context.Context, indexName, shardName string) error - ReInitShard(ctx context.Context, indexName, shardName string) error -} - -type db interface { - StartupComplete() bool -} - -func NewIndices(shards shards, db db, auth auth) *indices { - return &indices{ - regexpObjects: regexp.MustCompile(urlPatternObjects), - regexpObjectsOverwrite: regexp.MustCompile(urlPatternObjectsOverwrite), - regexObjectsDigest: regexp.MustCompile(urlPatternObjectsDigest), - regexpObjectsSearch: regexp.MustCompile(urlPatternObjectsSearch), - regexpObjectsFind: regexp.MustCompile(urlPatternObjectsFind), - - regexpObjectsAggregations: regexp.MustCompile(urlPatternObjectsAggregations), - regexpObject: regexp.MustCompile(urlPatternObject), - regexpReferences: regexp.MustCompile(urlPatternReferences), - regexpShardsQueueSize: regexp.MustCompile(urlPatternShardsQueueSize), - regexpShardsStatus: regexp.MustCompile(urlPatternShardsStatus), - regexpShardFiles: regexp.MustCompile(urlPatternShardFiles), - regexpShard: regexp.MustCompile(urlPatternShard), - regexpShardReinit: regexp.MustCompile(urlPatternShardReinit), - shards: shards, - db: db, - auth: auth, - } -} - -func (i *indices) Indices() http.Handler { - return i.auth.handleFunc(i.indicesHandler()) -} - -func (i *indices) indicesHandler() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - path := r.URL.Path - switch { - case i.regexpObjectsSearch.MatchString(path): - if r.Method != http.MethodPost { - http.Error(w, "405 Method not Allowed", http.StatusMethodNotAllowed) - return - } - - i.postSearchObjects().ServeHTTP(w, r) - return - case i.regexpObjectsFind.MatchString(path): - if r.Method != http.MethodPost { - http.Error(w, "405 Method not Allowed", http.StatusMethodNotAllowed) - return - } - - i.postFindUUIDs().ServeHTTP(w, r) - return - case i.regexpObjectsAggregations.MatchString(path): - if r.Method != http.MethodPost { - http.Error(w, "405 Method not Allowed", http.StatusMethodNotAllowed) - return - } - - i.postAggregateObjects().ServeHTTP(w, r) - return - case i.regexpObjectsOverwrite.MatchString(path): - if r.Method != http.MethodPut { - http.Error(w, "405 Method not Allowed", http.StatusMethodNotAllowed) - } - - i.putOverwriteObjects().ServeHTTP(w, r) - case i.regexObjectsDigest.MatchString(path): - if r.Method != http.MethodGet { - http.Error(w, "405 Method not Allowed", http.StatusMethodNotAllowed) - } - - i.getObjectsDigest().ServeHTTP(w, r) - case i.regexpObject.MatchString(path): - if r.Method == http.MethodGet { - i.getObject().ServeHTTP(w, r) - return - } - if r.Method == http.MethodDelete { - i.deleteObject().ServeHTTP(w, r) - return - } - if r.Method == http.MethodPatch { - i.mergeObject().ServeHTTP(w, r) - return - } - - http.Error(w, "405 Method not Allowed", http.StatusMethodNotAllowed) - return - - case i.regexpObjects.MatchString(path): - if r.Method == http.MethodGet { - i.getObjectsMulti().ServeHTTP(w, r) - return - } - if r.Method == http.MethodPost { - i.postObject().ServeHTTP(w, r) - return - } - if r.Method == http.MethodDelete { - i.deleteObjects().ServeHTTP(w, r) - return - } - http.Error(w, "405 Method not Allowed", http.StatusMethodNotAllowed) - return - - case i.regexpReferences.MatchString(path): - if r.Method != http.MethodPost { - http.Error(w, "405 Method not Allowed", http.StatusMethodNotAllowed) - return - } - - i.postReferences().ServeHTTP(w, r) - return - case i.regexpShardsQueueSize.MatchString(path): - if r.Method == http.MethodGet { - i.getGetShardQueueSize().ServeHTTP(w, r) - return - } - http.Error(w, "405 Method not Allowed", http.StatusMethodNotAllowed) - return - case i.regexpShardsStatus.MatchString(path): - if r.Method == http.MethodGet { - i.getGetShardStatus().ServeHTTP(w, r) - return - } - if r.Method == http.MethodPost { - i.postUpdateShardStatus().ServeHTTP(w, r) - return - } - http.Error(w, "405 Method not Allowed", http.StatusMethodNotAllowed) - return - - case i.regexpShardFiles.MatchString(path): - if r.Method == http.MethodPost { - i.postShardFile().ServeHTTP(w, r) - return - } - http.Error(w, "405 Method not Allowed", http.StatusMethodNotAllowed) - return - - case i.regexpShard.MatchString(path): - if r.Method == http.MethodPost { - i.postShard().ServeHTTP(w, r) - return - } - http.Error(w, "405 Method not Allowed", http.StatusMethodNotAllowed) - return - case i.regexpShardReinit.MatchString(path): - if r.Method == http.MethodPut { - i.putShardReinit().ServeHTTP(w, r) - return - } - http.Error(w, "405 Method not Allowed", http.StatusMethodNotAllowed) - return - - default: - http.NotFound(w, r) - return - } - } -} - -func (i *indices) postObject() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - args := i.regexpObjects.FindStringSubmatch(r.URL.Path) - if len(args) != 3 { - http.Error(w, "invalid URI", http.StatusBadRequest) - return - } - - index, shard := args[1], args[2] - - defer r.Body.Close() - - ct := r.Header.Get("content-type") - - switch ct { - case IndicesPayloads.ObjectList.MIME(): - i.postObjectBatch(w, r, index, shard) - return - - case IndicesPayloads.SingleObject.MIME(): - i.postObjectSingle(w, r, index, shard) - return - - default: - http.Error(w, "415 Unsupported Media Type", http.StatusUnsupportedMediaType) - return - } - }) -} - -func (i *indices) postObjectSingle(w http.ResponseWriter, r *http.Request, - index, shard string, -) { - bodyBytes, err := io.ReadAll(r.Body) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - obj, err := IndicesPayloads.SingleObject.Unmarshal(bodyBytes) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - if err := i.shards.PutObject(r.Context(), index, shard, obj); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - w.WriteHeader(http.StatusNoContent) -} - -func (i *indices) postObjectBatch(w http.ResponseWriter, r *http.Request, - index, shard string, -) { - bodyBytes, err := io.ReadAll(r.Body) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - objs, err := IndicesPayloads.ObjectList.Unmarshal(bodyBytes) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - errs := i.shards.BatchPutObjects(r.Context(), index, shard, objs) - errsJSON, err := IndicesPayloads.ErrorList.Marshal(errs) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - IndicesPayloads.ErrorList.SetContentTypeHeader(w) - w.Write(errsJSON) -} - -func (i *indices) getObject() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - args := i.regexpObject.FindStringSubmatch(r.URL.Path) - if len(args) != 4 { - http.Error(w, "invalid URI", http.StatusBadRequest) - return - } - - index, shard, id := args[1], args[2], args[3] - - defer r.Body.Close() - - if r.URL.Query().Get("check_exists") != "" { - i.checkExists(w, r, index, shard, id) - return - } - - additionalEncoded := r.URL.Query().Get("additional") - if additionalEncoded == "" { - http.Error(w, "missing required url param 'additional'", - http.StatusBadRequest) - return - } - - additionalBytes, err := base64.StdEncoding.DecodeString(additionalEncoded) - if err != nil { - http.Error(w, "base64 decode 'additional' param: "+err.Error(), - http.StatusBadRequest) - return - } - - selectPropertiesEncoded := r.URL.Query().Get("selectProperties") - if selectPropertiesEncoded == "" { - http.Error(w, "missing required url param 'selectProperties'", - http.StatusBadRequest) - return - } - - selectPropertiesBytes, err := base64.StdEncoding. - DecodeString(selectPropertiesEncoded) - if err != nil { - http.Error(w, "base64 decode 'selectProperties' param: "+err.Error(), - http.StatusBadRequest) - return - } - - var additional additional.Properties - if err := json.Unmarshal(additionalBytes, &additional); err != nil { - http.Error(w, "unmarshal 'additional' param from json: "+err.Error(), - http.StatusBadRequest) - return - } - - var selectProperties search.SelectProperties - if err := json.Unmarshal(selectPropertiesBytes, &selectProperties); err != nil { - http.Error(w, "unmarshal 'selectProperties' param from json: "+err.Error(), - http.StatusBadRequest) - return - } - if !i.db.StartupComplete() { - http.Error(w, "startup is not complete", http.StatusServiceUnavailable) - return - } - obj, err := i.shards.GetObject(r.Context(), index, shard, strfmt.UUID(id), - selectProperties, additional) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - - if obj == nil { - // this is a legitimate case - the requested ID doesn't exist, don't try - // to marshal anything - w.WriteHeader(http.StatusNotFound) - return - } - - objBytes, err := IndicesPayloads.SingleObject.Marshal(obj) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - - IndicesPayloads.SingleObject.SetContentTypeHeader(w) - w.Write(objBytes) - }) -} - -func (i *indices) checkExists(w http.ResponseWriter, r *http.Request, - index, shard, id string, -) { - ok, err := i.shards.Exists(r.Context(), index, shard, strfmt.UUID(id)) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - - if ok { - w.WriteHeader(http.StatusNoContent) - } else { - w.WriteHeader(http.StatusNotFound) - } -} - -func (i *indices) deleteObject() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - args := i.regexpObject.FindStringSubmatch(r.URL.Path) - if len(args) != 4 { - http.Error(w, "invalid URI", http.StatusBadRequest) - return - } - - index, shard, id := args[1], args[2], args[3] - - defer r.Body.Close() - - err := i.shards.DeleteObject(r.Context(), index, shard, strfmt.UUID(id)) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - - w.WriteHeader(http.StatusNoContent) - }) -} - -func (i *indices) mergeObject() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - args := i.regexpObject.FindStringSubmatch(r.URL.Path) - if len(args) != 4 { - http.Error(w, "invalid URI", http.StatusBadRequest) - return - } - - index, shard, _ := args[1], args[2], args[3] - - defer r.Body.Close() - ct, ok := IndicesPayloads.MergeDoc.CheckContentTypeHeaderReq(r) - if !ok { - http.Error(w, errors.Errorf("unexpected content type: %s", ct).Error(), - http.StatusUnsupportedMediaType) - return - } - - bodyBytes, err := io.ReadAll(r.Body) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - mergeDoc, err := IndicesPayloads.MergeDoc.Unmarshal(bodyBytes) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - if err := i.shards.MergeObject(r.Context(), index, shard, mergeDoc); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - w.WriteHeader(http.StatusNoContent) - }) -} - -func (i *indices) getObjectsMulti() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - args := i.regexpObjects.FindStringSubmatch(r.URL.Path) - if len(args) != 3 { - http.Error(w, fmt.Sprintf("invalid URI: %s", r.URL.Path), - http.StatusBadRequest) - return - } - - index, shard := args[1], args[2] - - defer r.Body.Close() - - idsEncoded := r.URL.Query().Get("ids") - if idsEncoded == "" { - http.Error(w, "missing required url param 'ids'", - http.StatusBadRequest) - return - } - - idsBytes, err := base64.StdEncoding.DecodeString(idsEncoded) - if err != nil { - http.Error(w, "base64 decode 'ids' param: "+err.Error(), - http.StatusBadRequest) - return - } - - var ids []strfmt.UUID - if err := json.Unmarshal(idsBytes, &ids); err != nil { - http.Error(w, "unmarshal 'ids' param from json: "+err.Error(), - http.StatusBadRequest) - return - } - - objs, err := i.shards.MultiGetObjects(r.Context(), index, shard, ids) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - - objsBytes, err := IndicesPayloads.ObjectList.Marshal(objs) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - - IndicesPayloads.ObjectList.SetContentTypeHeader(w) - w.Write(objsBytes) - }) -} - -func (i *indices) postSearchObjects() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - args := i.regexpObjectsSearch.FindStringSubmatch(r.URL.Path) - if len(args) != 3 { - http.Error(w, "invalid URI", http.StatusBadRequest) - return - } - - index, shard := args[1], args[2] - - defer r.Body.Close() - reqPayload, err := io.ReadAll(r.Body) - if err != nil { - http.Error(w, "read request body: "+err.Error(), http.StatusInternalServerError) - return - } - - ct, ok := IndicesPayloads.SearchParams.CheckContentTypeHeaderReq(r) - if !ok { - http.Error(w, errors.Errorf("unexpected content type: %s", ct).Error(), - http.StatusUnsupportedMediaType) - return - } - - vector, certainty, limit, filters, keywordRanking, sort, cursor, groupBy, additional, err := IndicesPayloads.SearchParams. - Unmarshal(reqPayload) - if err != nil { - http.Error(w, "unmarshal search params from json: "+err.Error(), - http.StatusBadRequest) - return - } - - results, dists, err := i.shards.Search(r.Context(), index, shard, - vector, certainty, limit, filters, keywordRanking, sort, cursor, groupBy, additional) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - resBytes, err := IndicesPayloads.SearchResults.Marshal(results, dists) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - IndicesPayloads.SearchResults.SetContentTypeHeader(w) - w.Write(resBytes) - }) -} - -func (i *indices) postReferences() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - args := i.regexpReferences.FindStringSubmatch(r.URL.Path) - if len(args) != 3 { - http.Error(w, "invalid URI", http.StatusBadRequest) - return - } - - index, shard := args[1], args[2] - - defer r.Body.Close() - reqPayload, err := io.ReadAll(r.Body) - if err != nil { - http.Error(w, "read request body: "+err.Error(), - http.StatusInternalServerError) - return - } - - ct, ok := IndicesPayloads.ReferenceList.CheckContentTypeHeaderReq(r) - if !ok { - http.Error(w, errors.Errorf("unexpected content type: %s", ct).Error(), - http.StatusUnsupportedMediaType) - return - } - - refs, err := IndicesPayloads.ReferenceList.Unmarshal(reqPayload) - if err != nil { - http.Error(w, "read request body: "+err.Error(), - http.StatusInternalServerError) - return - } - - errs := i.shards.BatchAddReferences(r.Context(), index, shard, refs) - errsJSON, err := IndicesPayloads.ErrorList.Marshal(errs) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - IndicesPayloads.ErrorList.SetContentTypeHeader(w) - w.Write(errsJSON) - }) -} - -func (i *indices) postAggregateObjects() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - args := i.regexpObjectsAggregations.FindStringSubmatch(r.URL.Path) - if len(args) != 3 { - http.Error(w, "invalid URI", http.StatusBadRequest) - return - } - - index, shard := args[1], args[2] - - defer r.Body.Close() - reqPayload, err := io.ReadAll(r.Body) - if err != nil { - http.Error(w, "read request body: "+err.Error(), - http.StatusInternalServerError) - return - } - - ct, ok := IndicesPayloads.AggregationParams.CheckContentTypeHeaderReq(r) - if !ok { - http.Error(w, errors.Errorf("unexpected content type: %s", ct).Error(), - http.StatusUnsupportedMediaType) - return - } - - params, err := IndicesPayloads.AggregationParams.Unmarshal(reqPayload) - if err != nil { - http.Error(w, "read request body: "+err.Error(), - http.StatusInternalServerError) - return - } - - aggRes, err := i.shards.Aggregate(r.Context(), index, shard, params) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - aggResBytes, err := IndicesPayloads.AggregationResult.Marshal(aggRes) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - IndicesPayloads.AggregationResult.SetContentTypeHeader(w) - w.Write(aggResBytes) - }) -} - -func (i *indices) postFindUUIDs() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - args := i.regexpObjectsFind.FindStringSubmatch(r.URL.Path) - if len(args) != 3 { - http.Error(w, "invalid URI", http.StatusBadRequest) - return - } - - index, shard := args[1], args[2] - - defer r.Body.Close() - reqPayload, err := io.ReadAll(r.Body) - if err != nil { - http.Error(w, "read request body: "+err.Error(), http.StatusInternalServerError) - return - } - - ct, ok := IndicesPayloads.FindUUIDsParams.CheckContentTypeHeaderReq(r) - if !ok { - http.Error(w, errors.Errorf("unexpected content type: %s", ct).Error(), - http.StatusUnsupportedMediaType) - return - } - - filters, err := IndicesPayloads.FindUUIDsParams. - Unmarshal(reqPayload) - if err != nil { - http.Error(w, "unmarshal find doc ids params from json: "+err.Error(), - http.StatusBadRequest) - return - } - - results, err := i.shards.FindUUIDs(r.Context(), index, shard, filters) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - resBytes, err := IndicesPayloads.FindUUIDsResults.Marshal(results) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - IndicesPayloads.FindUUIDsResults.SetContentTypeHeader(w) - w.Write(resBytes) - }) -} - -func (i *indices) putOverwriteObjects() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - args := i.regexpObjectsOverwrite.FindStringSubmatch(r.URL.Path) - if len(args) != 3 { - http.Error(w, "invalid URI", http.StatusBadRequest) - return - } - - index, shard := args[1], args[2] - - defer r.Body.Close() - reqPayload, err := io.ReadAll(r.Body) - if err != nil { - http.Error(w, "read request body: "+err.Error(), http.StatusInternalServerError) - return - } - - ct, ok := IndicesPayloads.VersionedObjectList.CheckContentTypeHeaderReq(r) - if !ok { - http.Error(w, errors.Errorf("unexpected content type: %s", ct).Error(), - http.StatusUnsupportedMediaType) - return - } - - vobjs, err := IndicesPayloads.VersionedObjectList.Unmarshal(reqPayload) - if err != nil { - http.Error(w, "unmarshal overwrite objects params from json: "+err.Error(), - http.StatusBadRequest) - return - } - - results, err := i.shards.OverwriteObjects(r.Context(), index, shard, vobjs) - if err != nil { - http.Error(w, "overwrite objects: "+err.Error(), - http.StatusInternalServerError) - return - } - - resBytes, err := json.Marshal(results) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - w.Write(resBytes) - }) -} - -func (i *indices) getObjectsDigest() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - args := i.regexObjectsDigest.FindStringSubmatch(r.URL.Path) - if len(args) != 3 { - http.Error(w, "invalid URI", http.StatusBadRequest) - return - } - - index, shard := args[1], args[2] - - defer r.Body.Close() - reqPayload, err := io.ReadAll(r.Body) - if err != nil { - http.Error(w, "read request body: "+err.Error(), http.StatusInternalServerError) - return - } - - var ids []strfmt.UUID - if err := json.Unmarshal(reqPayload, &ids); err != nil { - http.Error(w, "unmarshal digest objects params from json: "+err.Error(), - http.StatusBadRequest) - return - } - - results, err := i.shards.DigestObjects(r.Context(), index, shard, ids) - if err != nil { - http.Error(w, "digest objects: "+err.Error(), - http.StatusInternalServerError) - return - } - - resBytes, err := json.Marshal(results) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - w.Write(resBytes) - }) -} - -func (i *indices) deleteObjects() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - args := i.regexpObjects.FindStringSubmatch(r.URL.Path) - if len(args) != 3 { - http.Error(w, "invalid URI", http.StatusBadRequest) - return - } - - index, shard := args[1], args[2] - - defer r.Body.Close() - reqPayload, err := io.ReadAll(r.Body) - if err != nil { - http.Error(w, "read request body: "+err.Error(), http.StatusInternalServerError) - return - } - - ct, ok := IndicesPayloads.BatchDeleteParams.CheckContentTypeHeaderReq(r) - if !ok { - http.Error(w, errors.Errorf("unexpected content type: %s", ct).Error(), - http.StatusUnsupportedMediaType) - return - } - - uuids, dryRun, err := IndicesPayloads.BatchDeleteParams. - Unmarshal(reqPayload) - if err != nil { - http.Error(w, "unmarshal find doc ids params from json: "+err.Error(), - http.StatusBadRequest) - return - } - - results := i.shards.DeleteObjectBatch(r.Context(), index, shard, uuids, dryRun) - - resBytes, err := IndicesPayloads.BatchDeleteResults.Marshal(results) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - IndicesPayloads.BatchDeleteResults.SetContentTypeHeader(w) - w.Write(resBytes) - }) -} - -func (i *indices) getGetShardQueueSize() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - args := i.regexpShardsQueueSize.FindStringSubmatch(r.URL.Path) - if len(args) != 3 { - http.Error(w, "invalid URI", http.StatusBadRequest) - return - } - - index, shard := args[1], args[2] - - defer r.Body.Close() - - size, err := i.shards.GetShardQueueSize(r.Context(), index, shard) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - - sizeBytes, err := IndicesPayloads.GetShardQueueSizeResults.Marshal(size) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - - IndicesPayloads.GetShardQueueSizeResults.SetContentTypeHeader(w) - w.Write(sizeBytes) - }) -} - -func (i *indices) getGetShardStatus() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - args := i.regexpShardsStatus.FindStringSubmatch(r.URL.Path) - if len(args) != 3 { - http.Error(w, "invalid URI", http.StatusBadRequest) - return - } - - index, shard := args[1], args[2] - - defer r.Body.Close() - - status, err := i.shards.GetShardStatus(r.Context(), index, shard) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - - statusBytes, err := IndicesPayloads.GetShardStatusResults.Marshal(status) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - - IndicesPayloads.GetShardStatusResults.SetContentTypeHeader(w) - w.Write(statusBytes) - }) -} - -func (i *indices) postUpdateShardStatus() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - args := i.regexpShardsStatus.FindStringSubmatch(r.URL.Path) - if len(args) != 3 { - http.Error(w, "invalid URI", http.StatusBadRequest) - return - } - - index, shard := args[1], args[2] - - defer r.Body.Close() - reqPayload, err := io.ReadAll(r.Body) - if err != nil { - http.Error(w, "read request body: "+err.Error(), http.StatusInternalServerError) - return - } - - ct, ok := IndicesPayloads.UpdateShardStatusParams.CheckContentTypeHeaderReq(r) - if !ok { - http.Error(w, errors.Errorf("unexpected content type: %s", ct).Error(), - http.StatusUnsupportedMediaType) - return - } - - targetStatus, err := IndicesPayloads.UpdateShardStatusParams. - Unmarshal(reqPayload) - if err != nil { - http.Error(w, "unmarshal find doc ids params from json: "+err.Error(), - http.StatusBadRequest) - return - } - - err = i.shards.UpdateShardStatus(r.Context(), index, shard, targetStatus) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - }) -} - -func (i *indices) postShardFile() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - args := i.regexpShardFiles.FindStringSubmatch(r.URL.Path) - if len(args) != 4 { - http.Error(w, "invalid URI", http.StatusBadRequest) - return - } - - index, shard, filename := args[1], args[2], args[3] - - ct, ok := IndicesPayloads.ShardFiles.CheckContentTypeHeaderReq(r) - if !ok { - http.Error(w, errors.Errorf("unexpected content type: %s", ct).Error(), - http.StatusUnsupportedMediaType) - return - } - - fp, err := i.shards.FilePutter(r.Context(), index, shard, filename) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - defer fp.Close() - n, err := io.Copy(fp, r.Body) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - fmt.Printf("%s/%s/%s n=%d\n", index, shard, filename, n) - - w.WriteHeader(http.StatusNoContent) - }) -} - -func (i *indices) postShard() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - args := i.regexpShard.FindStringSubmatch(r.URL.Path) - fmt.Println(args) - if len(args) != 3 { - http.Error(w, "invalid URI", http.StatusBadRequest) - return - } - - index, shard := args[1], args[2] - - err := i.shards.CreateShard(r.Context(), index, shard) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - w.WriteHeader(http.StatusCreated) - }) -} - -func (i *indices) putShardReinit() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - args := i.regexpShardReinit.FindStringSubmatch(r.URL.Path) - fmt.Println(args) - if len(args) != 3 { - http.Error(w, "invalid URI", http.StatusBadRequest) - return - } - - index, shard := args[1], args[2] - - err := i.shards.ReInitShard(r.Context(), index, shard) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - w.WriteHeader(http.StatusNoContent) - }) -} diff --git a/adapters/handlers/rest/clusterapi/indices_payloads.go b/adapters/handlers/rest/clusterapi/indices_payloads.go deleted file mode 100644 index 0f1320c0b00d5c4646697d2b051d44f1729a14dc..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/clusterapi/indices_payloads.go +++ /dev/null @@ -1,816 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clusterapi - -import ( - "bytes" - "encoding/binary" - "encoding/json" - "fmt" - "io" - "math" - "net/http" - - "github.com/go-openapi/strfmt" - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/aggregation" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/searchparams" - "github.com/weaviate/weaviate/entities/storobj" - "github.com/weaviate/weaviate/usecases/objects" - "github.com/weaviate/weaviate/usecases/scaler" -) - -var IndicesPayloads = indicesPayloads{} - -type indicesPayloads struct { - ErrorList errorListPayload - SingleObject singleObjectPayload - MergeDoc mergeDocPayload - ObjectList objectListPayload - VersionedObjectList versionedObjectListPayload - SearchResults searchResultsPayload - SearchParams searchParamsPayload - ReferenceList referenceListPayload - AggregationParams aggregationParamsPayload - AggregationResult aggregationResultPayload - FindUUIDsParams findUUIDsParamsPayload - FindUUIDsResults findUUIDsResultsPayload - BatchDeleteParams batchDeleteParamsPayload - BatchDeleteResults batchDeleteResultsPayload - GetShardQueueSizeParams getShardQueueSizeParamsPayload - GetShardQueueSizeResults getShardQueueSizeResultsPayload - GetShardStatusParams getShardStatusParamsPayload - GetShardStatusResults getShardStatusResultsPayload - UpdateShardStatusParams updateShardStatusParamsPayload - UpdateShardsStatusResults updateShardsStatusResultsPayload - ShardFiles shardFilesPayload - IncreaseReplicationFactor increaseReplicationFactorPayload -} - -type increaseReplicationFactorPayload struct{} - -func (p increaseReplicationFactorPayload) Marshall(dist scaler.ShardDist) ([]byte, error) { - type payload struct { - ShardDist scaler.ShardDist `json:"shard_distribution"` - } - - pay := payload{ShardDist: dist} - return json.Marshal(pay) -} - -func (p increaseReplicationFactorPayload) Unmarshal(in []byte) (scaler.ShardDist, error) { - type payload struct { - ShardDist scaler.ShardDist `json:"shard_distribution"` - } - - pay := payload{} - if err := json.Unmarshal(in, &pay); err != nil { - return nil, fmt.Errorf("unmarshal replication factor payload: %w", err) - } - - return pay.ShardDist, nil -} - -type errorListPayload struct{} - -func (e errorListPayload) MIME() string { - return "application/vnd.weaviate.error.list+json" -} - -func (e errorListPayload) SetContentTypeHeader(w http.ResponseWriter) { - w.Header().Set("content-type", e.MIME()) -} - -func (e errorListPayload) CheckContentTypeHeader(r *http.Response) (string, bool) { - ct := r.Header.Get("content-type") - return ct, ct == e.MIME() -} - -func (e errorListPayload) Marshal(in []error) ([]byte, error) { - converted := make([]interface{}, len(in)) - for i, err := range in { - if err == nil { - continue - } - - converted[i] = err.Error() - } - - return json.Marshal(converted) -} - -func (e errorListPayload) Unmarshal(in []byte) []error { - var msgs []interface{} - json.Unmarshal(in, &msgs) - - converted := make([]error, len(msgs)) - - for i, msg := range msgs { - if msg == nil { - continue - } - - converted[i] = errors.New(msg.(string)) - } - - return converted -} - -type singleObjectPayload struct{} - -func (p singleObjectPayload) MIME() string { - return "application/vnd.weaviate.storobj+octet-stream" -} - -func (p singleObjectPayload) SetContentTypeHeader(w http.ResponseWriter) { - w.Header().Set("content-type", p.MIME()) -} - -func (p singleObjectPayload) SetContentTypeHeaderReq(r *http.Request) { - r.Header.Set("content-type", p.MIME()) -} - -func (p singleObjectPayload) CheckContentTypeHeader(r *http.Response) (string, bool) { - ct := r.Header.Get("content-type") - return ct, ct == p.MIME() -} - -func (p singleObjectPayload) Marshal(in *storobj.Object) ([]byte, error) { - return in.MarshalBinary() -} - -func (p singleObjectPayload) Unmarshal(in []byte) (*storobj.Object, error) { - return storobj.FromBinary(in) -} - -type objectListPayload struct{} - -func (p objectListPayload) MIME() string { - return "application/vnd.weaviate.storobj.list+octet-stream" -} - -func (p objectListPayload) CheckContentTypeHeader(r *http.Response) (string, bool) { - ct := r.Header.Get("content-type") - return ct, ct == p.MIME() -} - -func (p objectListPayload) SetContentTypeHeader(w http.ResponseWriter) { - w.Header().Set("content-type", p.MIME()) -} - -func (p objectListPayload) SetContentTypeHeaderReq(r *http.Request) { - r.Header.Set("content-type", p.MIME()) -} - -func (p objectListPayload) Marshal(in []*storobj.Object) ([]byte, error) { - // NOTE: This implementation is not optimized for allocation efficiency, - // reserve 1024 byte per object which is rather arbitrary - out := make([]byte, 0, 1024*len(in)) - - reusableLengthBuf := make([]byte, 8) - for _, ind := range in { - if ind != nil { - bytes, err := ind.MarshalBinary() - if err != nil { - return nil, err - } - - length := uint64(len(bytes)) - binary.LittleEndian.PutUint64(reusableLengthBuf, length) - - out = append(out, reusableLengthBuf...) - out = append(out, bytes...) - } - } - - return out, nil -} - -func (p objectListPayload) Unmarshal(in []byte) ([]*storobj.Object, error) { - var out []*storobj.Object - - reusableLengthBuf := make([]byte, 8) - r := bytes.NewReader(in) - - for { - _, err := r.Read(reusableLengthBuf) - if err == io.EOF { - break - } - if err != nil { - return nil, err - } - - payloadBytes := make([]byte, binary.LittleEndian.Uint64(reusableLengthBuf)) - _, err = r.Read(payloadBytes) - if err != nil { - return nil, err - } - - obj, err := storobj.FromBinary(payloadBytes) - if err != nil { - return nil, err - } - - out = append(out, obj) - } - - return out, nil -} - -type versionedObjectListPayload struct{} - -func (p versionedObjectListPayload) MIME() string { - return "application/vnd.weaviate.vobject.list+octet-stream" -} - -func (p versionedObjectListPayload) CheckContentTypeHeader(r *http.Response) (string, bool) { - ct := r.Header.Get("content-type") - return ct, ct == p.MIME() -} - -func (p versionedObjectListPayload) SetContentTypeHeader(w http.ResponseWriter) { - w.Header().Set("content-type", p.MIME()) -} - -func (p versionedObjectListPayload) SetContentTypeHeaderReq(r *http.Request) { - r.Header.Set("content-type", p.MIME()) -} - -func (p versionedObjectListPayload) CheckContentTypeHeaderReq(r *http.Request) (string, bool) { - ct := r.Header.Get("content-type") - return ct, ct == p.MIME() -} - -func (p versionedObjectListPayload) Marshal(in []*objects.VObject) ([]byte, error) { - // NOTE: This implementation is not optimized for allocation efficiency, - // reserve 1024 byte per object which is rather arbitrary - out := make([]byte, 0, 1024*len(in)) - - reusableLengthBuf := make([]byte, 8) - for _, ind := range in { - objBytes, err := ind.MarshalBinary() - if err != nil { - return nil, err - } - - length := uint64(len(objBytes)) - binary.LittleEndian.PutUint64(reusableLengthBuf, length) - - out = append(out, reusableLengthBuf...) - out = append(out, objBytes...) - } - - return out, nil -} - -func (p versionedObjectListPayload) Unmarshal(in []byte) ([]*objects.VObject, error) { - var out []*objects.VObject - - reusableLengthBuf := make([]byte, 8) - r := bytes.NewReader(in) - - for { - _, err := r.Read(reusableLengthBuf) - if err == io.EOF { - break - } - if err != nil { - return nil, err - } - - ln := binary.LittleEndian.Uint64(reusableLengthBuf) - payloadBytes := make([]byte, ln) - _, err = r.Read(payloadBytes) - if err != nil { - return nil, err - } - - var vobj objects.VObject - err = vobj.UnmarshalBinary(payloadBytes) - if err != nil { - return nil, err - } - - out = append(out, &vobj) - } - - return out, nil -} - -type mergeDocPayload struct{} - -func (p mergeDocPayload) MIME() string { - return "application/vnd.weaviate.mergedoc+json" -} - -func (p mergeDocPayload) SetContentTypeHeader(w http.ResponseWriter) { - w.Header().Set("content-type", p.MIME()) -} - -func (p mergeDocPayload) SetContentTypeHeaderReq(r *http.Request) { - r.Header.Set("content-type", p.MIME()) -} - -func (p mergeDocPayload) CheckContentTypeHeaderReq(r *http.Request) (string, bool) { - ct := r.Header.Get("content-type") - return ct, ct == p.MIME() -} - -func (p mergeDocPayload) Marshal(in objects.MergeDocument) ([]byte, error) { - // assumes that this type is fully json-marshable. Not the most - // bandwidth-efficient way, but this is unlikely to become a bottleneck. If it - // does, a custom binary marshaller might be more appropriate - return json.Marshal(in) -} - -func (p mergeDocPayload) Unmarshal(in []byte) (objects.MergeDocument, error) { - var mergeDoc objects.MergeDocument - err := json.Unmarshal(in, &mergeDoc) - return mergeDoc, err -} - -type searchParamsPayload struct{} - -func (p searchParamsPayload) Marshal(vector []float32, limit int, - filter *filters.LocalFilter, keywordRanking *searchparams.KeywordRanking, - sort []filters.Sort, cursor *filters.Cursor, groupBy *searchparams.GroupBy, - addP additional.Properties, -) ([]byte, error) { - type params struct { - SearchVector []float32 `json:"searchVector"` - Limit int `json:"limit"` - Filters *filters.LocalFilter `json:"filters"` - KeywordRanking *searchparams.KeywordRanking `json:"keywordRanking"` - Sort []filters.Sort `json:"sort"` - Cursor *filters.Cursor `json:"cursor"` - GroupBy *searchparams.GroupBy `json:"groupBy"` - Additional additional.Properties `json:"additional"` - } - - par := params{vector, limit, filter, keywordRanking, sort, cursor, groupBy, addP} - return json.Marshal(par) -} - -func (p searchParamsPayload) Unmarshal(in []byte) ([]float32, float32, int, - *filters.LocalFilter, *searchparams.KeywordRanking, []filters.Sort, - *filters.Cursor, *searchparams.GroupBy, additional.Properties, error, -) { - type searchParametersPayload struct { - SearchVector []float32 `json:"searchVector"` - Distance float32 `json:"distance"` - Limit int `json:"limit"` - Filters *filters.LocalFilter `json:"filters"` - KeywordRanking *searchparams.KeywordRanking `json:"keywordRanking"` - Sort []filters.Sort `json:"sort"` - Cursor *filters.Cursor `json:"cursor"` - GroupBy *searchparams.GroupBy `json:"groupBy"` - Additional additional.Properties `json:"additional"` - } - var par searchParametersPayload - err := json.Unmarshal(in, &par) - return par.SearchVector, par.Distance, par.Limit, - par.Filters, par.KeywordRanking, par.Sort, par.Cursor, par.GroupBy, par.Additional, err -} - -func (p searchParamsPayload) MIME() string { - return "vnd.weaviate.searchparams+json" -} - -func (p searchParamsPayload) CheckContentTypeHeaderReq(r *http.Request) (string, bool) { - ct := r.Header.Get("content-type") - return ct, ct == p.MIME() -} - -func (p searchParamsPayload) SetContentTypeHeaderReq(r *http.Request) { - r.Header.Set("content-type", p.MIME()) -} - -type searchResultsPayload struct{} - -func (p searchResultsPayload) Unmarshal(in []byte) ([]*storobj.Object, []float32, error) { - read := uint64(0) - - objsLength := binary.LittleEndian.Uint64(in[read : read+8]) - read += 8 - - objs, err := IndicesPayloads.ObjectList.Unmarshal(in[read : read+objsLength]) - if err != nil { - return nil, nil, err - } - read += objsLength - - distsLength := binary.LittleEndian.Uint64(in[read : read+8]) - read += 8 - - dists := make([]float32, distsLength) - for i := range dists { - dists[i] = math.Float32frombits(binary.LittleEndian.Uint32(in[read : read+4])) - read += 4 - } - - if read != uint64(len(in)) { - return nil, nil, errors.Errorf("corrupt read: %d != %d", read, len(in)) - } - - return objs, dists, nil -} - -func (p searchResultsPayload) Marshal(objs []*storobj.Object, - dists []float32, -) ([]byte, error) { - reusableLengthBuf := make([]byte, 8) - var out []byte - objsBytes, err := IndicesPayloads.ObjectList.Marshal(objs) - if err != nil { - return nil, err - } - - objsLength := uint64(len(objsBytes)) - binary.LittleEndian.PutUint64(reusableLengthBuf, objsLength) - - out = append(out, reusableLengthBuf...) - out = append(out, objsBytes...) - - distsLength := uint64(len(dists)) - binary.LittleEndian.PutUint64(reusableLengthBuf, distsLength) - out = append(out, reusableLengthBuf...) - - distsBuf := make([]byte, distsLength*4) - for i, dist := range dists { - distUint32 := math.Float32bits(dist) - binary.LittleEndian.PutUint32(distsBuf[(i*4):((i+1)*4)], distUint32) - } - out = append(out, distsBuf...) - - return out, nil -} - -func (p searchResultsPayload) MIME() string { - return "application/vnd.weaviate.shardsearchresults+octet-stream" -} - -func (p searchResultsPayload) SetContentTypeHeader(w http.ResponseWriter) { - w.Header().Set("content-type", p.MIME()) -} - -func (p searchResultsPayload) CheckContentTypeHeader(r *http.Response) (string, bool) { - ct := r.Header.Get("content-type") - return ct, ct == p.MIME() -} - -type referenceListPayload struct{} - -func (p referenceListPayload) MIME() string { - return "application/vnd.weaviate.references.list+json" -} - -func (p referenceListPayload) SetContentTypeHeaderReq(r *http.Request) { - r.Header.Set("content-type", p.MIME()) -} - -func (p referenceListPayload) CheckContentTypeHeaderReq(r *http.Request) (string, bool) { - ct := r.Header.Get("content-type") - return ct, ct == p.MIME() -} - -func (p referenceListPayload) Marshal(in objects.BatchReferences) ([]byte, error) { - // assumes that this type is fully json-marshable. Not the most - // bandwidth-efficient way, but this is unlikely to become a bottleneck. If it - // does, a custom binary marshaller might be more appropriate - return json.Marshal(in) -} - -func (p referenceListPayload) Unmarshal(in []byte) (objects.BatchReferences, error) { - var out objects.BatchReferences - err := json.Unmarshal(in, &out) - return out, err -} - -type aggregationParamsPayload struct{} - -func (p aggregationParamsPayload) Marshal(params aggregation.Params) ([]byte, error) { - // assumes that this type is fully json-marshable. Not the most - // bandwidth-efficient way, but this is unlikely to become a bottleneck. If it - // does, a custom binary marshaller might be more appropriate - return json.Marshal(params) -} - -func (p aggregationParamsPayload) Unmarshal(in []byte) (aggregation.Params, error) { - var out aggregation.Params - err := json.Unmarshal(in, &out) - return out, err -} - -func (p aggregationParamsPayload) MIME() string { - return "application/vnd.weaviate.aggregations.params+json" -} - -func (p aggregationParamsPayload) SetContentTypeHeaderReq(r *http.Request) { - r.Header.Set("content-type", p.MIME()) -} - -func (p aggregationParamsPayload) CheckContentTypeHeaderReq(r *http.Request) (string, bool) { - ct := r.Header.Get("content-type") - return ct, ct == p.MIME() -} - -type aggregationResultPayload struct{} - -func (p aggregationResultPayload) MIME() string { - return "application/vnd.weaviate.aggregations.result+json" -} - -func (p aggregationResultPayload) CheckContentTypeHeader(res *http.Response) (string, bool) { - ct := res.Header.Get("content-type") - return ct, ct == p.MIME() -} - -func (p aggregationResultPayload) SetContentTypeHeader(w http.ResponseWriter) { - w.Header().Set("content-type", p.MIME()) -} - -func (p aggregationResultPayload) Marshal(in *aggregation.Result) ([]byte, error) { - // assumes that this type is fully json-marshable. Not the most - // bandwidth-efficient way, but this is unlikely to become a bottleneck. If it - // does, a custom binary marshaller might be more appropriate - return json.Marshal(in) -} - -func (p aggregationResultPayload) Unmarshal(in []byte) (*aggregation.Result, error) { - var out aggregation.Result - err := json.Unmarshal(in, &out) - return &out, err -} - -type findUUIDsParamsPayload struct{} - -func (p findUUIDsParamsPayload) Marshal(filter *filters.LocalFilter) ([]byte, error) { - type params struct { - Filters *filters.LocalFilter `json:"filters"` - } - - par := params{filter} - return json.Marshal(par) -} - -func (p findUUIDsParamsPayload) Unmarshal(in []byte) (*filters.LocalFilter, error) { - type findUUIDsParametersPayload struct { - Filters *filters.LocalFilter `json:"filters"` - } - var par findUUIDsParametersPayload - err := json.Unmarshal(in, &par) - return par.Filters, err -} - -func (p findUUIDsParamsPayload) MIME() string { - return "vnd.weaviate.finduuidsparams+json" -} - -func (p findUUIDsParamsPayload) CheckContentTypeHeaderReq(r *http.Request) (string, bool) { - ct := r.Header.Get("content-type") - return ct, ct == p.MIME() -} - -func (p findUUIDsParamsPayload) SetContentTypeHeaderReq(r *http.Request) { - r.Header.Set("content-type", p.MIME()) -} - -type findUUIDsResultsPayload struct{} - -func (p findUUIDsResultsPayload) Unmarshal(in []byte) ([]strfmt.UUID, error) { - var out []strfmt.UUID - err := json.Unmarshal(in, &out) - return out, err -} - -func (p findUUIDsResultsPayload) Marshal(in []strfmt.UUID) ([]byte, error) { - return json.Marshal(in) -} - -func (p findUUIDsResultsPayload) MIME() string { - return "application/vnd.weaviate.findUUIDsresults+octet-stream" -} - -func (p findUUIDsResultsPayload) SetContentTypeHeader(w http.ResponseWriter) { - w.Header().Set("content-type", p.MIME()) -} - -func (p findUUIDsResultsPayload) CheckContentTypeHeader(r *http.Response) (string, bool) { - ct := r.Header.Get("content-type") - return ct, ct == p.MIME() -} - -type batchDeleteParamsPayload struct{} - -func (p batchDeleteParamsPayload) Marshal(uuids []strfmt.UUID, dryRun bool) ([]byte, error) { - type params struct { - UUIDs []strfmt.UUID `json:"uuids"` - DryRun bool `json:"dryRun"` - } - - par := params{uuids, dryRun} - return json.Marshal(par) -} - -func (p batchDeleteParamsPayload) Unmarshal(in []byte) ([]strfmt.UUID, bool, error) { - type batchDeleteParametersPayload struct { - UUIDs []strfmt.UUID `json:"uuids"` - DryRun bool `json:"dryRun"` - } - var par batchDeleteParametersPayload - err := json.Unmarshal(in, &par) - return par.UUIDs, par.DryRun, err -} - -func (p batchDeleteParamsPayload) MIME() string { - return "vnd.weaviate.batchdeleteparams+json" -} - -func (p batchDeleteParamsPayload) CheckContentTypeHeaderReq(r *http.Request) (string, bool) { - ct := r.Header.Get("content-type") - return ct, ct == p.MIME() -} - -func (p batchDeleteParamsPayload) SetContentTypeHeaderReq(r *http.Request) { - r.Header.Set("content-type", p.MIME()) -} - -type batchDeleteResultsPayload struct{} - -func (p batchDeleteResultsPayload) Unmarshal(in []byte) (objects.BatchSimpleObjects, error) { - var out objects.BatchSimpleObjects - err := json.Unmarshal(in, &out) - return out, err -} - -func (p batchDeleteResultsPayload) Marshal(in objects.BatchSimpleObjects) ([]byte, error) { - return json.Marshal(in) -} - -func (p batchDeleteResultsPayload) MIME() string { - return "application/vnd.weaviate.batchdeleteresults+octet-stream" -} - -func (p batchDeleteResultsPayload) SetContentTypeHeader(w http.ResponseWriter) { - w.Header().Set("content-type", p.MIME()) -} - -func (p batchDeleteResultsPayload) CheckContentTypeHeader(r *http.Response) (string, bool) { - ct := r.Header.Get("content-type") - return ct, ct == p.MIME() -} - -type getShardQueueSizeParamsPayload struct{} - -func (p getShardQueueSizeParamsPayload) MIME() string { - return "vnd.weaviate.getshardqueuesizeparams+json" -} - -func (p getShardQueueSizeParamsPayload) CheckContentTypeHeaderReq(r *http.Request) (string, bool) { - ct := r.Header.Get("content-type") - return ct, ct == p.MIME() -} - -func (p getShardQueueSizeParamsPayload) SetContentTypeHeaderReq(r *http.Request) { - r.Header.Set("content-type", p.MIME()) -} - -type getShardQueueSizeResultsPayload struct{} - -func (p getShardQueueSizeResultsPayload) Unmarshal(in []byte) (int64, error) { - var out int64 - err := json.Unmarshal(in, &out) - return out, err -} - -func (p getShardQueueSizeResultsPayload) Marshal(in int64) ([]byte, error) { - return json.Marshal(in) -} - -func (p getShardQueueSizeResultsPayload) MIME() string { - return "application/vnd.weaviate.getshardqueuesizeresults+octet-stream" -} - -func (p getShardQueueSizeResultsPayload) SetContentTypeHeader(w http.ResponseWriter) { - w.Header().Set("content-type", p.MIME()) -} - -func (p getShardQueueSizeResultsPayload) CheckContentTypeHeader(r *http.Response) (string, bool) { - ct := r.Header.Get("content-type") - return ct, ct == p.MIME() -} - -type getShardStatusParamsPayload struct{} - -func (p getShardStatusParamsPayload) MIME() string { - return "vnd.weaviate.getshardstatusparams+json" -} - -func (p getShardStatusParamsPayload) CheckContentTypeHeaderReq(r *http.Request) (string, bool) { - ct := r.Header.Get("content-type") - return ct, ct == p.MIME() -} - -func (p getShardStatusParamsPayload) SetContentTypeHeaderReq(r *http.Request) { - r.Header.Set("content-type", p.MIME()) -} - -type getShardStatusResultsPayload struct{} - -func (p getShardStatusResultsPayload) Unmarshal(in []byte) (string, error) { - var out string - err := json.Unmarshal(in, &out) - return out, err -} - -func (p getShardStatusResultsPayload) Marshal(in string) ([]byte, error) { - return json.Marshal(in) -} - -func (p getShardStatusResultsPayload) MIME() string { - return "application/vnd.weaviate.getshardstatusresults+octet-stream" -} - -func (p getShardStatusResultsPayload) SetContentTypeHeader(w http.ResponseWriter) { - w.Header().Set("content-type", p.MIME()) -} - -func (p getShardStatusResultsPayload) CheckContentTypeHeader(r *http.Response) (string, bool) { - ct := r.Header.Get("content-type") - return ct, ct == p.MIME() -} - -type updateShardStatusParamsPayload struct{} - -func (p updateShardStatusParamsPayload) Marshal(targetStatus string) ([]byte, error) { - type params struct { - TargetStatus string `json:"targetStatus"` - } - - par := params{targetStatus} - return json.Marshal(par) -} - -func (p updateShardStatusParamsPayload) Unmarshal(in []byte) (string, error) { - type updateShardStatusParametersPayload struct { - TargetStatus string `json:"targetStatus"` - } - var par updateShardStatusParametersPayload - err := json.Unmarshal(in, &par) - return par.TargetStatus, err -} - -func (p updateShardStatusParamsPayload) MIME() string { - return "vnd.weaviate.updateshardstatusparams+json" -} - -func (p updateShardStatusParamsPayload) CheckContentTypeHeaderReq(r *http.Request) (string, bool) { - ct := r.Header.Get("content-type") - return ct, ct == p.MIME() -} - -func (p updateShardStatusParamsPayload) SetContentTypeHeaderReq(r *http.Request) { - r.Header.Set("content-type", p.MIME()) -} - -type updateShardsStatusResultsPayload struct{} - -func (p updateShardsStatusResultsPayload) MIME() string { - return "application/vnd.weaviate.updateshardstatusresults+octet-stream" -} - -func (p updateShardsStatusResultsPayload) SetContentTypeHeader(w http.ResponseWriter) { - w.Header().Set("content-type", p.MIME()) -} - -func (p updateShardsStatusResultsPayload) CheckContentTypeHeader(r *http.Response) (string, bool) { - ct := r.Header.Get("content-type") - return ct, ct == p.MIME() -} - -type shardFilesPayload struct{} - -func (p shardFilesPayload) MIME() string { - return "application/vnd.weaviate.indexfiles+octet-stream" -} - -func (p shardFilesPayload) SetContentTypeHeaderReq(r *http.Request) { - r.Header.Set("content-type", p.MIME()) -} - -func (p shardFilesPayload) CheckContentTypeHeaderReq(r *http.Request) (string, bool) { - ct := r.Header.Get("content-type") - return ct, ct == p.MIME() -} diff --git a/adapters/handlers/rest/clusterapi/indices_payloads_test.go b/adapters/handlers/rest/clusterapi/indices_payloads_test.go deleted file mode 100644 index 19e550740fcf591d05290d3ab1b99ee5a94af16f..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/clusterapi/indices_payloads_test.go +++ /dev/null @@ -1,96 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clusterapi - -import ( - "testing" - "time" - - "github.com/go-openapi/strfmt" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema/crossref" - "github.com/weaviate/weaviate/entities/storobj" -) - -func Test_objectListPayload_Marshal(t *testing.T) { - now := time.Now() - vec1 := []float32{1, 2, 3, 4, 5} - vec2 := []float32{10, 20, 30, 40, 50} - id1 := strfmt.UUID("c6f85bf5-c3b7-4c1d-bd51-e899f9605336") - id2 := strfmt.UUID("88750a99-a72d-46c2-a582-89f02654391d") - - objs := []*storobj.Object{ - { - MarshallerVersion: 1, - Object: models.Object{ - ID: id1, - Class: "SomeClass", - CreationTimeUnix: now.UnixMilli(), - LastUpdateTimeUnix: now.Add(time.Hour).UnixMilli(), // time-traveling ;) - Properties: map[string]interface{}{ - "propA": "this is prop A", - "propB": "this is prop B", - "someDate": now.Format(time.RFC3339Nano), - "aNumber": 1e+06, - "crossRef": models.MultipleRef{ - crossref.NewLocalhost("OtherClass", id1). - SingleRef(), - }, - }, - Additional: map[string]interface{}{ - "score": 0.055465422484, - }, - }, - Vector: vec1, - VectorLen: 5, - }, - nil, - { - MarshallerVersion: 1, - Object: models.Object{ - ID: id2, - Class: "SomeClass", - CreationTimeUnix: now.UnixMilli(), - LastUpdateTimeUnix: now.Add(time.Hour).UnixMilli(), // time-traveling ;) - Properties: map[string]interface{}{ - "propA": "this is prop A", - "propB": "this is prop B", - "someDate": now.Format(time.RFC3339Nano), - "aNumber": 1e+06, - "crossRef": models.MultipleRef{ - crossref.NewLocalhost("OtherClass", id2). - SingleRef(), - }, - }, - Additional: map[string]interface{}{ - "score": 0.055465422484, - }, - }, - Vector: vec2, - VectorLen: 5, - }, - } - - payload := objectListPayload{} - b, err := payload.Marshal(objs) - require.Nil(t, err) - - received, err := payload.Unmarshal(b) - require.Nil(t, err) - assert.Len(t, received, 2) - assert.EqualValues(t, objs[0].Object, received[0].Object) - assert.EqualValues(t, objs[0].ID(), received[0].ID()) - assert.EqualValues(t, objs[2].Object, received[1].Object) - assert.EqualValues(t, objs[2].ID(), received[1].ID()) -} diff --git a/adapters/handlers/rest/clusterapi/indices_replicas.go b/adapters/handlers/rest/clusterapi/indices_replicas.go deleted file mode 100644 index 93c857527b16efa5ba789d3781c2e88c2a7c591b..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/clusterapi/indices_replicas.go +++ /dev/null @@ -1,701 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clusterapi - -import ( - "context" - "encoding/base64" - "encoding/json" - "fmt" - "io" - "net/http" - "regexp" - - "github.com/go-openapi/strfmt" - "github.com/weaviate/weaviate/entities/storobj" - "github.com/weaviate/weaviate/usecases/objects" - "github.com/weaviate/weaviate/usecases/replica" - "github.com/weaviate/weaviate/usecases/scaler" -) - -type replicator interface { - // Write endpoints - ReplicateObject(ctx context.Context, indexName, shardName, - requestID string, object *storobj.Object) replica.SimpleResponse - ReplicateObjects(ctx context.Context, indexName, shardName, - requestID string, objects []*storobj.Object) replica.SimpleResponse - ReplicateUpdate(ctx context.Context, indexName, shardName, - requestID string, mergeDoc *objects.MergeDocument) replica.SimpleResponse - ReplicateDeletion(ctx context.Context, indexName, shardName, - requestID string, uuid strfmt.UUID) replica.SimpleResponse - ReplicateDeletions(ctx context.Context, indexName, shardName, - requestID string, uuids []strfmt.UUID, dryRun bool) replica.SimpleResponse - ReplicateReferences(ctx context.Context, indexName, shardName, - requestID string, refs []objects.BatchReference) replica.SimpleResponse - CommitReplication(indexName, - shardName, requestID string) interface{} - AbortReplication(indexName, - shardName, requestID string) interface{} - OverwriteObjects(ctx context.Context, index, shard string, - vobjects []*objects.VObject) ([]replica.RepairResponse, error) - // Read endpoints - FetchObject(ctx context.Context, indexName, - shardName string, id strfmt.UUID) (objects.Replica, error) - FetchObjects(ctx context.Context, class, - shardName string, ids []strfmt.UUID) ([]objects.Replica, error) - DigestObjects(ctx context.Context, class, shardName string, - ids []strfmt.UUID) (result []replica.RepairResponse, err error) -} - -type localScaler interface { - LocalScaleOut(ctx context.Context, className string, - dist scaler.ShardDist) error -} - -type replicatedIndices struct { - shards replicator - scaler localScaler - auth auth -} - -var ( - regxObject = regexp.MustCompile(`\/replicas\/indices\/(` + cl + `)` + - `\/shards\/(` + sh + `)\/objects\/(` + ob + `)`) - regxOverwriteObjects = regexp.MustCompile(`\/indices\/(` + cl + `)` + - `\/shards\/(` + sh + `)\/objects/_overwrite`) - regxObjectsDigest = regexp.MustCompile(`\/indices\/(` + cl + `)` + - `\/shards\/(` + sh + `)\/objects/_digest`) - regxObjects = regexp.MustCompile(`\/replicas\/indices\/(` + cl + `)` + - `\/shards\/(` + sh + `)\/objects`) - regxReferences = regexp.MustCompile(`\/replicas\/indices\/(` + cl + `)` + - `\/shards\/(` + sh + `)\/objects/references`) - regxIncreaseRepFactor = regexp.MustCompile(`\/replicas\/indices\/(` + cl + `)` + - `\/replication-factor:increase`) - regxCommitPhase = regexp.MustCompile(`\/replicas\/indices\/(` + cl + `)` + - `\/shards\/(` + sh + `):(commit|abort)`) -) - -func NewReplicatedIndices(shards replicator, scaler localScaler, auth auth) *replicatedIndices { - return &replicatedIndices{ - shards: shards, - scaler: scaler, - auth: auth, - } -} - -func (i *replicatedIndices) Indices() http.Handler { - return i.auth.handleFunc(i.indicesHandler()) -} - -func (i *replicatedIndices) indicesHandler() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - path := r.URL.Path - switch { - case regxObjectsDigest.MatchString(path): - if r.Method == http.MethodGet { - i.getObjectsDigest().ServeHTTP(w, r) - return - } - - http.Error(w, "405 Method not Allowed", http.StatusMethodNotAllowed) - return - case regxOverwriteObjects.MatchString(path): - if r.Method == http.MethodPut { - i.putOverwriteObjects().ServeHTTP(w, r) - return - } - - http.Error(w, "405 Method not Allowed", http.StatusMethodNotAllowed) - return - case regxObject.MatchString(path): - if r.Method == http.MethodDelete { - i.deleteObject().ServeHTTP(w, r) - return - } - - if r.Method == http.MethodPatch { - i.patchObject().ServeHTTP(w, r) - return - } - - if r.Method == http.MethodGet { - i.getObject().ServeHTTP(w, r) - return - } - - if regxReferences.MatchString(path) { - if r.Method == http.MethodPost { - i.postRefs().ServeHTTP(w, r) - return - } - } - - http.Error(w, "405 Method not Allowed", http.StatusMethodNotAllowed) - return - - case regxObjects.MatchString(path): - if r.Method == http.MethodGet { - i.getObjectsMulti().ServeHTTP(w, r) - return - } - - if r.Method == http.MethodPost { - i.postObject().ServeHTTP(w, r) - return - } - - if r.Method == http.MethodDelete { - i.deleteObjects().ServeHTTP(w, r) - return - } - - http.Error(w, "405 Method not Allowed", http.StatusMethodNotAllowed) - return - - case regxIncreaseRepFactor.MatchString(path): - if r.Method == http.MethodPut { - i.increaseReplicationFactor().ServeHTTP(w, r) - return - } - - http.Error(w, "405 Method not Allowed", http.StatusMethodNotAllowed) - return - - case regxCommitPhase.MatchString(path): - if r.Method == http.MethodPost { - i.executeCommitPhase().ServeHTTP(w, r) - return - } - - http.Error(w, "405 Method not Allowed", http.StatusMethodNotAllowed) - return - - default: - http.NotFound(w, r) - return - } - } -} - -func (i *replicatedIndices) executeCommitPhase() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - args := regxCommitPhase.FindStringSubmatch(r.URL.Path) - if len(args) != 4 { - http.Error(w, "invalid URI", http.StatusBadRequest) - return - } - - requestID := r.URL.Query().Get(replica.RequestKey) - if requestID == "" { - http.Error(w, "request_id not provided", http.StatusBadRequest) - return - } - - index, shard, cmd := args[1], args[2], args[3] - - var resp interface{} - - switch cmd { - case "commit": - resp = i.shards.CommitReplication(index, shard, requestID) - case "abort": - resp = i.shards.AbortReplication(index, shard, requestID) - default: - http.Error(w, fmt.Sprintf("unrecognized command: %s", cmd), http.StatusNotImplemented) - return - } - if resp == nil { // could not find request with specified id - http.Error(w, "request not found", http.StatusNotFound) - return - } - b, err := json.Marshal(resp) - if err != nil { - http.Error(w, fmt.Sprintf("failed to marshal response: %+v, error: %v", resp, err), - http.StatusInternalServerError) - return - } - w.Write(b) - }) -} - -func (i *replicatedIndices) increaseReplicationFactor() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - args := regxIncreaseRepFactor.FindStringSubmatch(r.URL.Path) - fmt.Printf("path: %v, args: %+v", r.URL.Path, args) - if len(args) != 2 { - http.Error(w, "invalid URI", http.StatusBadRequest) - return - } - - index := args[1] - - bodyBytes, err := io.ReadAll(r.Body) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - dist, err := IndicesPayloads.IncreaseReplicationFactor.Unmarshal(bodyBytes) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - if err := i.scaler.LocalScaleOut(r.Context(), index, dist); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - w.WriteHeader(http.StatusNoContent) - }) -} - -func (i *replicatedIndices) postObject() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - args := regxObjects.FindStringSubmatch(r.URL.Path) - if len(args) != 3 { - http.Error(w, "invalid URI", http.StatusBadRequest) - return - } - - requestID := r.URL.Query().Get(replica.RequestKey) - if requestID == "" { - http.Error(w, "request_id not provided", http.StatusBadRequest) - return - } - - index, shard := args[1], args[2] - - defer r.Body.Close() - - ct := r.Header.Get("content-type") - - switch ct { - - case IndicesPayloads.SingleObject.MIME(): - i.postObjectSingle(w, r, index, shard, requestID) - return - case IndicesPayloads.ObjectList.MIME(): - i.postObjectBatch(w, r, index, shard, requestID) - return - default: - http.Error(w, "415 Unsupported Media Type", http.StatusUnsupportedMediaType) - return - } - }) -} - -func (i *replicatedIndices) patchObject() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - args := regxObjects.FindStringSubmatch(r.URL.Path) - if len(args) != 3 { - http.Error(w, "invalid URI", http.StatusBadRequest) - return - } - - requestID := r.URL.Query().Get(replica.RequestKey) - if requestID == "" { - http.Error(w, "request_id not provided", http.StatusBadRequest) - return - } - - index, shard := args[1], args[2] - - bodyBytes, err := io.ReadAll(r.Body) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - mergeDoc, err := IndicesPayloads.MergeDoc.Unmarshal(bodyBytes) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - resp := i.shards.ReplicateUpdate(r.Context(), index, shard, requestID, &mergeDoc) - if localIndexNotReady(resp) { - http.Error(w, resp.FirstError().Error(), http.StatusServiceUnavailable) - return - } - - b, err := json.Marshal(resp) - if err != nil { - http.Error(w, fmt.Sprintf("failed to marshal response: %+v, error: %v", resp, err), - http.StatusInternalServerError) - return - } - - w.Write(b) - }) -} - -func (i *replicatedIndices) getObjectsDigest() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - args := regxObjectsDigest.FindStringSubmatch(r.URL.Path) - if len(args) != 3 { - http.Error(w, "invalid URI", http.StatusBadRequest) - return - } - - index, shard := args[1], args[2] - - defer r.Body.Close() - reqPayload, err := io.ReadAll(r.Body) - if err != nil { - http.Error(w, "read request body: "+err.Error(), http.StatusInternalServerError) - return - } - - var ids []strfmt.UUID - if err := json.Unmarshal(reqPayload, &ids); err != nil { - http.Error(w, "unmarshal digest objects params from json: "+err.Error(), - http.StatusBadRequest) - return - } - - results, err := i.shards.DigestObjects(r.Context(), index, shard, ids) - if err != nil { - http.Error(w, "digest objects: "+err.Error(), - http.StatusInternalServerError) - return - } - - resBytes, err := json.Marshal(results) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - w.Write(resBytes) - }) -} - -func (i *replicatedIndices) putOverwriteObjects() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - args := regxOverwriteObjects.FindStringSubmatch(r.URL.Path) - if len(args) != 3 { - http.Error(w, "invalid URI", http.StatusBadRequest) - return - } - - index, shard := args[1], args[2] - - defer r.Body.Close() - reqPayload, err := io.ReadAll(r.Body) - if err != nil { - http.Error(w, "read request body: "+err.Error(), http.StatusInternalServerError) - return - } - - vobjs, err := IndicesPayloads.VersionedObjectList.Unmarshal(reqPayload) - if err != nil { - http.Error(w, "unmarshal overwrite objects params from json: "+err.Error(), - http.StatusBadRequest) - return - } - - results, err := i.shards.OverwriteObjects(r.Context(), index, shard, vobjs) - if err != nil { - http.Error(w, "overwrite objects: "+err.Error(), - http.StatusInternalServerError) - return - } - - resBytes, err := json.Marshal(results) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - w.Write(resBytes) - }) -} - -func (i *replicatedIndices) deleteObject() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - args := regxObject.FindStringSubmatch(r.URL.Path) - if len(args) != 4 { - http.Error(w, "invalid URI", http.StatusBadRequest) - return - } - - requestID := r.URL.Query().Get(replica.RequestKey) - if requestID == "" { - http.Error(w, "request_id not provided", http.StatusBadRequest) - return - } - - index, shard, id := args[1], args[2], args[3] - - defer r.Body.Close() - - resp := i.shards.ReplicateDeletion(r.Context(), index, shard, requestID, strfmt.UUID(id)) - if localIndexNotReady(resp) { - http.Error(w, resp.FirstError().Error(), http.StatusServiceUnavailable) - return - } - - b, err := json.Marshal(resp) - if err != nil { - http.Error(w, fmt.Sprintf("failed to marshal response: %+v, error: %v", resp, err), - http.StatusInternalServerError) - return - } - w.Write(b) - }) -} - -func (i *replicatedIndices) deleteObjects() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - args := regxObjects.FindStringSubmatch(r.URL.Path) - if len(args) != 3 { - http.Error(w, "invalid URI", http.StatusBadRequest) - return - } - - requestID := r.URL.Query().Get(replica.RequestKey) - if requestID == "" { - http.Error(w, "request_id not provided", http.StatusBadRequest) - return - } - - index, shard := args[1], args[2] - - bodyBytes, err := io.ReadAll(r.Body) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - defer r.Body.Close() - - uuids, dryRun, err := IndicesPayloads.BatchDeleteParams.Unmarshal(bodyBytes) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - resp := i.shards.ReplicateDeletions(r.Context(), index, shard, requestID, uuids, dryRun) - if localIndexNotReady(resp) { - http.Error(w, resp.FirstError().Error(), http.StatusServiceUnavailable) - return - } - - b, err := json.Marshal(resp) - if err != nil { - http.Error(w, fmt.Sprintf("failed to marshal response: %+v, error: %v", resp, err), - http.StatusInternalServerError) - return - } - w.Write(b) - }) -} - -func (i *replicatedIndices) postObjectSingle(w http.ResponseWriter, r *http.Request, - index, shard, requestID string, -) { - bodyBytes, err := io.ReadAll(r.Body) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - obj, err := IndicesPayloads.SingleObject.Unmarshal(bodyBytes) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - resp := i.shards.ReplicateObject(r.Context(), index, shard, requestID, obj) - if localIndexNotReady(resp) { - http.Error(w, resp.FirstError().Error(), http.StatusServiceUnavailable) - return - } - - b, err := json.Marshal(resp) - if err != nil { - http.Error(w, fmt.Sprintf("failed to marshal response: %+v, error: %v", resp, err), - http.StatusInternalServerError) - return - } - - w.Write(b) -} - -func (i *replicatedIndices) postObjectBatch(w http.ResponseWriter, r *http.Request, - index, shard, requestID string, -) { - bodyBytes, err := io.ReadAll(r.Body) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - objs, err := IndicesPayloads.ObjectList.Unmarshal(bodyBytes) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - resp := i.shards.ReplicateObjects(r.Context(), index, shard, requestID, objs) - if localIndexNotReady(resp) { - http.Error(w, resp.FirstError().Error(), http.StatusServiceUnavailable) - return - } - - b, err := json.Marshal(resp) - if err != nil { - http.Error(w, fmt.Sprintf("unmarshal resp: %+v, error: %v", resp, err), - http.StatusInternalServerError) - return - } - - w.Write(b) -} - -func (i *replicatedIndices) getObject() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - args := regxObject.FindStringSubmatch(r.URL.Path) - if len(args) != 4 { - http.Error(w, "invalid URI", http.StatusBadRequest) - return - } - - index, shard, id := args[1], args[2], args[3] - - defer r.Body.Close() - - var ( - resp objects.Replica - err error - ) - - resp, err = i.shards.FetchObject(r.Context(), index, shard, strfmt.UUID(id)) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - b, err := resp.MarshalBinary() - if err != nil { - http.Error(w, fmt.Sprintf("unmarshal resp: %+v, error: %v", resp, err), - http.StatusInternalServerError) - return - } - - w.Write(b) - }) -} - -func (i *replicatedIndices) getObjectsMulti() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - args := regxObjects.FindStringSubmatch(r.URL.Path) - if len(args) != 3 { - http.Error(w, fmt.Sprintf("invalid URI: %s", r.URL.Path), - http.StatusBadRequest) - return - } - - index, shard := args[1], args[2] - - defer r.Body.Close() - - idsEncoded := r.URL.Query().Get("ids") - if idsEncoded == "" { - http.Error(w, "missing required url param 'ids'", - http.StatusBadRequest) - return - } - - idsBytes, err := base64.StdEncoding.DecodeString(idsEncoded) - if err != nil { - http.Error(w, "base64 decode 'ids' param: "+err.Error(), - http.StatusBadRequest) - return - } - - var ids []strfmt.UUID - if err := json.Unmarshal(idsBytes, &ids); err != nil { - http.Error(w, "unmarshal 'ids' param from json: "+err.Error(), - http.StatusBadRequest) - return - } - - resp, err := i.shards.FetchObjects(r.Context(), index, shard, ids) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - - b, err := objects.Replicas(resp).MarshalBinary() - if err != nil { - http.Error(w, fmt.Sprintf("unmarshal resp: %+v, error: %v", resp, err), - http.StatusInternalServerError) - return - } - - w.Write(b) - }) -} - -func (i *replicatedIndices) postRefs() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - args := regxObjects.FindStringSubmatch(r.URL.Path) - if len(args) != 3 { - http.Error(w, "invalid URI", http.StatusBadRequest) - return - } - - requestID := r.URL.Query().Get(replica.RequestKey) - if requestID == "" { - http.Error(w, "request_id not provided", http.StatusBadRequest) - return - } - - index, shard := args[1], args[2] - bodyBytes, err := io.ReadAll(r.Body) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - refs, err := IndicesPayloads.ReferenceList.Unmarshal(bodyBytes) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - resp := i.shards.ReplicateReferences(r.Context(), index, shard, requestID, refs) - if localIndexNotReady(resp) { - http.Error(w, resp.FirstError().Error(), http.StatusServiceUnavailable) - return - } - - b, err := json.Marshal(resp) - if err != nil { - http.Error(w, fmt.Sprintf("unmarshal resp: %+v, error: %v", resp, err), - http.StatusInternalServerError) - return - } - - w.Write(b) - }) -} - -func localIndexNotReady(resp replica.SimpleResponse) bool { - if err := resp.FirstError(); err != nil { - re, ok := err.(*replica.Error) - if ok && re.IsStatusCode(replica.StatusNotReady) { - return true - } - } - return false -} diff --git a/adapters/handlers/rest/clusterapi/nodes.go b/adapters/handlers/rest/clusterapi/nodes.go deleted file mode 100644 index c882143c4672fead10d2625cfbcefcf4cc6207b8..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/clusterapi/nodes.go +++ /dev/null @@ -1,105 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clusterapi - -import ( - "context" - "encoding/json" - "fmt" - "net/http" - "regexp" - - "github.com/weaviate/weaviate/entities/models" - entschema "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/verbosity" -) - -type nodesManager interface { - GetNodeStatus(ctx context.Context, className, output string) (*models.NodeStatus, error) -} - -type nodes struct { - nodesManager nodesManager - auth auth -} - -func NewNodes(manager nodesManager, auth auth) *nodes { - return &nodes{nodesManager: manager, auth: auth} -} - -var ( - regxNodes = regexp.MustCompile(`/status`) - regxNodesClass = regexp.MustCompile(`/status/(` + entschema.ClassNameRegexCore + `)`) -) - -func (s *nodes) Nodes() http.Handler { - return s.auth.handleFunc(s.nodesHandler()) -} - -func (s *nodes) nodesHandler() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - path := r.URL.Path - switch { - case regxNodes.MatchString(path) || regxNodesClass.MatchString(path): - if r.Method != http.MethodGet { - msg := fmt.Sprintf("/nodes api path %q not found", path) - http.Error(w, msg, http.StatusMethodNotAllowed) - return - } - - s.incomingNodeStatus().ServeHTTP(w, r) - return - default: - http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) - return - } - } -} - -func (s *nodes) incomingNodeStatus() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - defer r.Body.Close() - - var className string - - args := regxNodesClass.FindStringSubmatch(r.URL.Path) - if len(args) == 3 { - className = args[2] - } - - output := verbosity.OutputMinimal - out, found := r.URL.Query()["output"] - if found && len(out) > 0 { - output = out[0] - } - - nodeStatus, err := s.nodesManager.GetNodeStatus(r.Context(), className, output) - if err != nil { - http.Error(w, "/nodes fulfill request: "+err.Error(), - http.StatusBadRequest) - return - } - - if nodeStatus == nil { - w.WriteHeader(http.StatusNotFound) - return - } - - nodeStatusBytes, err := json.Marshal(nodeStatus) - if err != nil { - http.Error(w, "/nodes marshal response: "+err.Error(), - http.StatusInternalServerError) - } - - w.Write(nodeStatusBytes) - }) -} diff --git a/adapters/handlers/rest/clusterapi/schema.go b/adapters/handlers/rest/clusterapi/schema.go deleted file mode 100644 index 3ac628523d4a37c4affc1af1942e2e62f026b22b..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/clusterapi/schema.go +++ /dev/null @@ -1,20 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clusterapi - -type schema struct { - txHandler -} - -func NewSchema(manager txManager, auth auth) *schema { - return &schema{txHandler{manager: manager, auth: auth}} -} diff --git a/adapters/handlers/rest/clusterapi/schema_component_test.go b/adapters/handlers/rest/clusterapi/schema_component_test.go deleted file mode 100644 index ede3876e6811fa70d702bfc1cf4677c7e82c4d50..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/clusterapi/schema_component_test.go +++ /dev/null @@ -1,212 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clusterapi_test - -import ( - "context" - "net/http" - "net/http/httptest" - "net/url" - "testing" - - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/adapters/clients" - "github.com/weaviate/weaviate/adapters/handlers/rest/clusterapi" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/usecases/cluster" - "github.com/weaviate/weaviate/usecases/config" - "github.com/weaviate/weaviate/usecases/scaler" - schemauc "github.com/weaviate/weaviate/usecases/schema" - "github.com/weaviate/weaviate/usecases/sharding" -) - -// This is a cross-package test that tests the schema manager in a distributed -// settings including some of its dependencies, such as the REST API and -// clients. This setup pretends that replication was one-way for simplicity -// sake, but uses the same components on either side to make sure that it -// would work in both directions. -func TestComponentCluster(t *testing.T) { - t.Run("add class", func(t *testing.T) { - localManager, remoteManager := setupManagers(t) - - ctx := context.Background() - - err := localManager.AddClass(ctx, nil, testClass()) - require.Nil(t, err) - - localClass, err := localManager.GetClass(ctx, nil, testClass().Class) - require.Nil(t, err) - remoteClass, err := remoteManager.GetClass(ctx, nil, testClass().Class) - require.Nil(t, err) - - assert.Equal(t, localClass, remoteClass) - }) - - t.Run("add class and extend property", func(t *testing.T) { - localManager, remoteManager := setupManagers(t) - - ctx := context.Background() - - err := localManager.AddClass(ctx, nil, testClass()) - require.Nil(t, err) - - err = localManager.AddClassProperty(ctx, nil, testClass().Class, testProperty()) - require.Nil(t, err) - - localClass, err := localManager.GetClass(ctx, nil, testClass().Class) - require.Nil(t, err) - remoteClass, err := remoteManager.GetClass(ctx, nil, testClass().Class) - require.Nil(t, err) - - assert.Equal(t, localClass, remoteClass) - }) - - t.Run("delete class", func(t *testing.T) { - localManager, remoteManager := setupManagers(t) - - ctx := context.Background() - - err := localManager.AddClass(ctx, nil, testClass()) - require.Nil(t, err) - - err = localManager.DeleteClass(ctx, nil, testClass().Class) - require.Nil(t, err) - - localSchema, err := localManager.GetSchema(nil) - require.Nil(t, err) - remoteSchema, err := remoteManager.GetSchema(nil) - require.Nil(t, err) - - assert.Equal(t, localSchema, remoteSchema) - }) - - t.Run("add class update config", func(t *testing.T) { - localManager, remoteManager := setupManagers(t) - - ctx := context.Background() - - err := localManager.AddClass(ctx, nil, testClass()) - require.Nil(t, err) - - updated := testClass() - updated.VectorIndexConfig.(map[string]interface{})["secondKey"] = "added" - - err = localManager.UpdateClass(ctx, nil, testClass().Class, updated) - require.Nil(t, err) - - localClass, err := localManager.GetClass(ctx, nil, testClass().Class) - require.Nil(t, err) - remoteClass, err := remoteManager.GetClass(ctx, nil, testClass().Class) - require.Nil(t, err) - - assert.Equal(t, localClass, remoteClass) - }) -} - -func setupManagers(t *testing.T) (*schemauc.Manager, *schemauc.Manager) { - remoteManager := newSchemaManagerWithClusterStateAndClient( - &fakeClusterState{hosts: []string{"node1"}}, nil) - - schemaHandlers := clusterapi.NewSchema(remoteManager.TxManager(), clusterapi.NewNoopAuthHandler()) - mux := http.NewServeMux() - mux.Handle("/schema/transactions/", http.StripPrefix("/schema/transactions/", - schemaHandlers.Transactions())) - server := httptest.NewServer(mux) - - client := clients.NewClusterSchema(&http.Client{}) - parsedURL, err := url.Parse(server.URL) - require.Nil(t, err) - state := &fakeClusterState{hosts: []string{parsedURL.Host}} - localManager := newSchemaManagerWithClusterStateAndClient(state, client) - - // this will also mark the tx managers as ready - localManager.StartServing(context.Background()) - remoteManager.StartServing(context.Background()) - - return localManager, remoteManager -} - -func testClass() *models.Class { - return &models.Class{ - Class: "MyClass", - Properties: []*models.Property{ - { - Name: "propOne", DataType: []string{"text"}, - }, - }, - VectorIndexConfig: map[string]interface{}{ - "foo": "bar", - }, - } -} - -func testProperty() *models.Property { - return &models.Property{ - Name: "propTwo", DataType: []string{"int"}, - } -} - -// New Local Schema *Manager -func newSchemaManagerWithClusterStateAndClient(clusterState *fakeClusterState, - client cluster.Client, -) *schemauc.Manager { - logger, _ := test.NewNullLogger() - vectorizerValidator := &fakeVectorizerValidator{ - valid: []string{"text2vec-contextionary", "model1", "model2"}, - } - sm, err := schemauc.NewManager(&NilMigrator{}, newFakeRepo(), logger, &fakeAuthorizer{}, - config.Config{DefaultVectorizerModule: config.VectorizerModuleNone}, - dummyParseVectorConfig, // only option for now - vectorizerValidator, dummyValidateInvertedConfig, - &fakeModuleConfig{}, clusterState, client, &fakeTxPersistence{}, - &fakeScaleOutManager{}, - ) - if err != nil { - panic(err.Error()) - } - - return sm -} - -type fakeScaleOutManager struct{} - -func (f *fakeScaleOutManager) Scale(ctx context.Context, - className string, updated sharding.Config, _, _ int64, -) (*sharding.State, error) { - return nil, nil -} - -func (f *fakeScaleOutManager) SetSchemaManager(sm scaler.SchemaManager) { -} - -// does nothing as this component test does not involve crashes -type fakeTxPersistence struct{} - -func (f *fakeTxPersistence) StoreTx(ctx context.Context, - tx *cluster.Transaction, -) error { - return nil -} - -func (f *fakeTxPersistence) DeleteTx(ctx context.Context, - txID string, -) error { - return nil -} - -func (f *fakeTxPersistence) IterateAll(ctx context.Context, - cb func(tx *cluster.Transaction), -) error { - return nil -} diff --git a/adapters/handlers/rest/clusterapi/serve.go b/adapters/handlers/rest/clusterapi/serve.go deleted file mode 100644 index 3a858fda32350f3e0e6c9caedce934206a250944..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/clusterapi/serve.go +++ /dev/null @@ -1,70 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clusterapi - -import ( - "encoding/json" - "fmt" - "net/http" - - "github.com/weaviate/weaviate/adapters/handlers/rest/state" -) - -func Serve(appState *state.State) { - port := appState.ServerConfig.Config.Cluster.DataBindPort - auth := NewBasicAuthHandler(appState.ServerConfig.Config.Cluster.AuthConfig) - - appState.Logger.WithField("port", port). - WithField("action", "cluster_api_startup"). - Debugf("serving cluster api on port %d", port) - - schema := NewSchema(appState.SchemaManager.TxManager(), auth) - indices := NewIndices(appState.RemoteIndexIncoming, appState.DB, auth) - replicatedIndices := NewReplicatedIndices(appState.RemoteReplicaIncoming, appState.Scaler, auth) - classifications := NewClassifications(appState.ClassificationRepo.TxManager(), auth) - nodes := NewNodes(appState.RemoteNodeIncoming, auth) - backups := NewBackups(appState.BackupManager, auth) - - mux := http.NewServeMux() - mux.Handle("/schema/transactions/", - http.StripPrefix("/schema/transactions/", schema.Transactions())) - mux.Handle("/classifications/transactions/", - http.StripPrefix("/classifications/transactions/", - classifications.Transactions())) - - mux.Handle("/nodes/", nodes.Nodes()) - mux.Handle("/indices/", indices.Indices()) - mux.Handle("/replicas/indices/", replicatedIndices.Indices()) - - mux.Handle("/backups/can-commit", backups.CanCommit()) - mux.Handle("/backups/commit", backups.Commit()) - mux.Handle("/backups/abort", backups.Abort()) - mux.Handle("/backups/status", backups.Status()) - - mux.Handle("/", index()) - http.ListenAndServe(fmt.Sprintf(":%d", port), mux) -} - -func index() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.String() != "" && r.URL.String() != "/" { - http.NotFound(w, r) - return - } - - payload := map[string]string{ - "description": "Weaviate's cluster-internal API for cross-node communication", - } - - json.NewEncoder(w).Encode(payload) - }) -} diff --git a/adapters/handlers/rest/clusterapi/transactions.go b/adapters/handlers/rest/clusterapi/transactions.go deleted file mode 100644 index 11344ae180ce41b7c320073f497066feb92cb907..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/clusterapi/transactions.go +++ /dev/null @@ -1,184 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clusterapi - -import ( - "context" - "encoding/json" - "net/http" - "strings" - "time" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/usecases/cluster" - ucs "github.com/weaviate/weaviate/usecases/schema" -) - -type txManager interface { - IncomingBeginTransaction(ctx context.Context, tx *cluster.Transaction) ([]byte, error) - IncomingCommitTransaction(ctx context.Context, tx *cluster.Transaction) error - IncomingAbortTransaction(ctx context.Context, tx *cluster.Transaction) -} - -type txPayload struct { - ID string `json:"id"` - Type cluster.TransactionType `json:"type"` - Payload json.RawMessage `json:"payload"` - DeadlineMilli int64 `json:"deadlineMilli"` -} - -type txHandler struct { - manager txManager - auth auth -} - -func (h *txHandler) Transactions() http.Handler { - return h.auth.handleFunc(h.transactionsHandler()) -} - -func (h *txHandler) transactionsHandler() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - path := r.URL.Path - switch { - case path == "": - if r.Method != http.MethodPost { - http.Error(w, "405 Method not Allowed", http.StatusMethodNotAllowed) - return - } - - h.incomingTransaction().ServeHTTP(w, r) - return - - case strings.HasSuffix(path, "/commit"): - if r.Method != http.MethodPut { - http.Error(w, "405 Method not Allowed", http.StatusMethodNotAllowed) - return - } - - h.incomingCommitTransaction().ServeHTTP(w, r) - return - default: - if r.Method != http.MethodDelete { - http.Error(w, "405 Method not Allowed", http.StatusMethodNotAllowed) - return - } - - h.incomingAbortTransaction().ServeHTTP(w, r) - return - } - } -} - -func (h *txHandler) incomingTransaction() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - defer r.Body.Close() - - if r.Header.Get("content-type") != "application/json" { - http.Error(w, "415 Unsupported Media Type", http.StatusUnsupportedMediaType) - return - } - - var payload txPayload - if err := json.NewDecoder(r.Body).Decode(&payload); err != nil { - http.Error(w, errors.Wrap(err, "decode body").Error(), - http.StatusInternalServerError) - return - } - - if len(payload.ID) == 0 { - http.Error(w, "id must be set", http.StatusBadRequest) - return - } - - if len(payload.Type) == 0 { - http.Error(w, "type must be set", http.StatusBadRequest) - return - } - - txPayload, err := ucs.UnmarshalTransaction(payload.Type, payload.Payload) - if err != nil { - http.Error(w, errors.Wrap(err, "decode tx payload").Error(), - http.StatusInternalServerError) - return - } - txType := payload.Type - tx := &cluster.Transaction{ - ID: payload.ID, - Type: txType, - Payload: txPayload, - Deadline: time.UnixMilli(payload.DeadlineMilli), - } - - data, err := h.manager.IncomingBeginTransaction(r.Context(), tx) - if err != nil { - status := http.StatusInternalServerError - if errors.Is(err, cluster.ErrConcurrentTransaction) { - status = http.StatusConflict - } - - http.Error(w, errors.Wrap(err, "open transaction").Error(), status) - return - } - if txType != ucs.ReadSchema { - w.WriteHeader(http.StatusCreated) - return - } - - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(err.Error())) - } - w.WriteHeader(http.StatusCreated) - w.Write(data) - }) -} - -func (h *txHandler) incomingAbortTransaction() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - defer r.Body.Close() - - path := r.URL.String() - tx := &cluster.Transaction{ - ID: path, - } - - h.manager.IncomingAbortTransaction(r.Context(), tx) - w.WriteHeader(http.StatusNoContent) - }) -} - -func (h *txHandler) incomingCommitTransaction() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - defer r.Body.Close() - - parts := strings.Split(r.URL.Path, "/") - if len(parts) != 2 { - http.NotFound(w, r) - return - } - - tx := &cluster.Transaction{ - ID: parts[0], - } - - if err := h.manager.IncomingCommitTransaction(r.Context(), tx); err != nil { - status := http.StatusInternalServerError - if errors.Is(err, cluster.ErrConcurrentTransaction) { - status = http.StatusConflict - } - - http.Error(w, errors.Wrap(err, "open transaction").Error(), status) - return - } - w.WriteHeader(http.StatusNoContent) - }) -} diff --git a/adapters/handlers/rest/configure_api.go b/adapters/handlers/rest/configure_api.go deleted file mode 100644 index 2d16b95df064fd6e67301cc6f9307e6bea38cc50..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/configure_api.go +++ /dev/null @@ -1,927 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package rest - -import ( - "context" - "encoding/json" - "fmt" - "net" - "net/http" - "os" - goruntime "runtime" - "runtime/debug" - "strings" - "time" - - _ "net/http/pprof" - - "github.com/KimMachineGun/automemlimit/memlimit" - openapierrors "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - "github.com/go-openapi/swag" - "github.com/pbnjay/memory" - "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus/promhttp" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/adapters/clients" - "github.com/weaviate/weaviate/adapters/handlers/rest/clusterapi" - "github.com/weaviate/weaviate/adapters/handlers/rest/operations" - "github.com/weaviate/weaviate/adapters/handlers/rest/state" - "github.com/weaviate/weaviate/adapters/repos/classifications" - "github.com/weaviate/weaviate/adapters/repos/db" - "github.com/weaviate/weaviate/adapters/repos/db/inverted" - modulestorage "github.com/weaviate/weaviate/adapters/repos/modules" - schemarepo "github.com/weaviate/weaviate/adapters/repos/schema" - txstore "github.com/weaviate/weaviate/adapters/repos/transactions" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/replication" - vectorIndex "github.com/weaviate/weaviate/entities/vectorindex" - modstgazure "github.com/weaviate/weaviate/modules/backup-azure" - modstgfs "github.com/weaviate/weaviate/modules/backup-filesystem" - modstggcs "github.com/weaviate/weaviate/modules/backup-gcs" - modstgs3 "github.com/weaviate/weaviate/modules/backup-s3" - modgenerativeanyscale "github.com/weaviate/weaviate/modules/generative-anyscale" - modgenerativeaws "github.com/weaviate/weaviate/modules/generative-aws" - modgenerativecohere "github.com/weaviate/weaviate/modules/generative-cohere" - modgenerativeopenai "github.com/weaviate/weaviate/modules/generative-openai" - modgenerativepalm "github.com/weaviate/weaviate/modules/generative-palm" - modimage "github.com/weaviate/weaviate/modules/img2vec-neural" - modbind "github.com/weaviate/weaviate/modules/multi2vec-bind" - modclip "github.com/weaviate/weaviate/modules/multi2vec-clip" - modner "github.com/weaviate/weaviate/modules/ner-transformers" - modqnaopenai "github.com/weaviate/weaviate/modules/qna-openai" - modqna "github.com/weaviate/weaviate/modules/qna-transformers" - modcentroid "github.com/weaviate/weaviate/modules/ref2vec-centroid" - modrerankercohere "github.com/weaviate/weaviate/modules/reranker-cohere" - modrerankertransformers "github.com/weaviate/weaviate/modules/reranker-transformers" - modsum "github.com/weaviate/weaviate/modules/sum-transformers" - modspellcheck "github.com/weaviate/weaviate/modules/text-spellcheck" - modtext2vecaws "github.com/weaviate/weaviate/modules/text2vec-aws" - modcohere "github.com/weaviate/weaviate/modules/text2vec-cohere" - modcontextionary "github.com/weaviate/weaviate/modules/text2vec-contextionary" - modgpt4all "github.com/weaviate/weaviate/modules/text2vec-gpt4all" - modhuggingface "github.com/weaviate/weaviate/modules/text2vec-huggingface" - modjinaai "github.com/weaviate/weaviate/modules/text2vec-jinaai" - modopenai "github.com/weaviate/weaviate/modules/text2vec-openai" - modtext2vecpalm "github.com/weaviate/weaviate/modules/text2vec-palm" - modtransformers "github.com/weaviate/weaviate/modules/text2vec-transformers" - "github.com/weaviate/weaviate/usecases/auth/authentication/composer" - "github.com/weaviate/weaviate/usecases/backup" - "github.com/weaviate/weaviate/usecases/classification" - "github.com/weaviate/weaviate/usecases/cluster" - "github.com/weaviate/weaviate/usecases/config" - "github.com/weaviate/weaviate/usecases/modules" - "github.com/weaviate/weaviate/usecases/monitoring" - "github.com/weaviate/weaviate/usecases/objects" - "github.com/weaviate/weaviate/usecases/replica" - "github.com/weaviate/weaviate/usecases/scaler" - schemaUC "github.com/weaviate/weaviate/usecases/schema" - "github.com/weaviate/weaviate/usecases/schema/migrate" - "github.com/weaviate/weaviate/usecases/sharding" - "github.com/weaviate/weaviate/usecases/traverser" -) - -const MinimumRequiredContextionaryVersion = "1.0.2" - -func makeConfigureServer(appState *state.State) func(*http.Server, string, string) { - return func(s *http.Server, scheme, addr string) { - // Add properties to the config - appState.ServerConfig.Hostname = addr - appState.ServerConfig.Scheme = scheme - } -} - -type vectorRepo interface { - objects.BatchVectorRepo - traverser.VectorSearcher - classification.VectorRepo - scaler.BackUpper - SetSchemaGetter(schemaUC.SchemaGetter) - WaitForStartup(ctx context.Context) error - Shutdown(ctx context.Context) error -} - -func getCores() (int, error) { - cpuset, err := os.ReadFile("/sys/fs/cgroup/cpuset/cpuset.cpus") - if err != nil { - return 0, errors.Wrap(err, "read cpuset") - } - - cores := strings.Split(strings.TrimSpace(string(cpuset)), ",") - return len(cores), nil -} - -func MakeAppState(ctx context.Context, options *swag.CommandLineOptionsGroup) *state.State { - appState := startupRoutine(ctx, options) - setupGoProfiling(appState.ServerConfig.Config) - - if appState.ServerConfig.Config.Monitoring.Enabled { - // only monitoring tool supported at the moment is prometheus - go func() { - mux := http.NewServeMux() - mux.Handle("/metrics", promhttp.Handler()) - http.ListenAndServe(fmt.Sprintf(":%d", appState.ServerConfig.Config.Monitoring.Port), mux) - }() - } - - limitResources(appState) - - err := registerModules(appState) - if err != nil { - appState.Logger. - WithField("action", "startup").WithError(err). - Fatal("modules didn't load") - } - - // now that modules are loaded we can run the remaining config validation - // which is module dependent - if err := appState.ServerConfig.Config.Validate(appState.Modules); err != nil { - appState.Logger. - WithField("action", "startup").WithError(err). - Fatal("invalid config") - } - - appState.ClusterHttpClient = reasonableHttpClient(appState.ServerConfig.Config.Cluster.AuthConfig) - - var vectorRepo vectorRepo - var vectorMigrator migrate.Migrator - var migrator migrate.Migrator - - if appState.ServerConfig.Config.Monitoring.Enabled { - promMetrics := monitoring.GetMetrics() - appState.Metrics = promMetrics - } - - // TODO: configure http transport for efficient intra-cluster comm - remoteIndexClient := clients.NewRemoteIndex(appState.ClusterHttpClient) - remoteNodesClient := clients.NewRemoteNode(appState.ClusterHttpClient) - replicationClient := clients.NewReplicationClient(appState.ClusterHttpClient) - repo, err := db.New(appState.Logger, db.Config{ - ServerVersion: config.ServerVersion, - GitHash: config.GitHash, - MemtablesFlushIdleAfter: appState.ServerConfig.Config.Persistence.FlushIdleMemtablesAfter, - MemtablesInitialSizeMB: 10, - MemtablesMaxSizeMB: appState.ServerConfig.Config.Persistence.MemtablesMaxSizeMB, - MemtablesMinActiveSeconds: appState.ServerConfig.Config.Persistence.MemtablesMinActiveDurationSeconds, - MemtablesMaxActiveSeconds: appState.ServerConfig.Config.Persistence.MemtablesMaxActiveDurationSeconds, - RootPath: appState.ServerConfig.Config.Persistence.DataPath, - QueryLimit: appState.ServerConfig.Config.QueryDefaults.Limit, - QueryMaximumResults: appState.ServerConfig.Config.QueryMaximumResults, - QueryNestedRefLimit: appState.ServerConfig.Config.QueryNestedCrossReferenceLimit, - MaxImportGoroutinesFactor: appState.ServerConfig.Config.MaxImportGoroutinesFactor, - TrackVectorDimensions: appState.ServerConfig.Config.TrackVectorDimensions, - ResourceUsage: appState.ServerConfig.Config.ResourceUsage, - AvoidMMap: appState.ServerConfig.Config.AvoidMmap, - DisableLazyLoadShards: appState.ServerConfig.Config.DisableLazyLoadShards, - // Pass dummy replication config with minimum factor 1. Otherwise the - // setting is not backward-compatible. The user may have created a class - // with factor=1 before the change was introduced. Now their setup would no - // longer start up if the required minimum is now higher than 1. We want - // the required minimum to only apply to newly created classes - not block - // loading existing ones. - Replication: replication.GlobalConfig{MinimumFactor: 1}, - }, remoteIndexClient, appState.Cluster, remoteNodesClient, replicationClient, appState.Metrics) // TODO client - if err != nil { - appState.Logger. - WithField("action", "startup").WithError(err). - Fatal("invalid new DB") - } - - appState.DB = repo - vectorMigrator = db.NewMigrator(repo, appState.Logger) - vectorRepo = repo - migrator = vectorMigrator - explorer := traverser.NewExplorer(repo, appState.Logger, appState.Modules, traverser.NewMetrics(appState.Metrics), appState.ServerConfig.Config) - schemaRepo := schemarepo.NewStore(appState.ServerConfig.Config.Persistence.DataPath, appState.Logger) - if err = schemaRepo.Open(); err != nil { - appState.Logger. - WithField("action", "startup").WithError(err). - Fatal("could not initialize schema repo") - os.Exit(1) - } - - localClassifierRepo, err := classifications.NewRepo( - appState.ServerConfig.Config.Persistence.DataPath, appState.Logger) - if err != nil { - appState.Logger. - WithField("action", "startup").WithError(err). - Fatal("could not initialize classifications repo") - os.Exit(1) - } - - // TODO: configure http transport for efficient intra-cluster comm - classificationsTxClient := clients.NewClusterClassifications(appState.ClusterHttpClient) - classifierRepo := classifications.NewDistributeRepo(classificationsTxClient, - appState.Cluster, localClassifierRepo, appState.Logger) - appState.ClassificationRepo = classifierRepo - - scaler := scaler.New(appState.Cluster, vectorRepo, - remoteIndexClient, appState.Logger, appState.ServerConfig.Config.Persistence.DataPath) - appState.Scaler = scaler - - // TODO: configure http transport for efficient intra-cluster comm - schemaTxClient := clients.NewClusterSchema(appState.ClusterHttpClient) - schemaTxPersistence := txstore.NewStore( - appState.ServerConfig.Config.Persistence.DataPath, appState.Logger) - schemaTxPersistence.SetUmarshalFn(schemaUC.UnmarshalTransaction) - if err := schemaTxPersistence.Open(); err != nil { - appState.Logger. - WithField("action", "startup").WithError(err). - Fatal("could not open tx repo") - os.Exit(1) - - } - - schemaManager, err := schemaUC.NewManager(migrator, schemaRepo, - appState.Logger, appState.Authorizer, appState.ServerConfig.Config, - vectorIndex.ParseAndValidateConfig, appState.Modules, inverted.ValidateConfig, - appState.Modules, appState.Cluster, schemaTxClient, - schemaTxPersistence, scaler, - ) - if err != nil { - appState.Logger. - WithField("action", "startup").WithError(err). - Fatal("could not initialize schema manager") - os.Exit(1) - } - - appState.SchemaManager = schemaManager - - appState.RemoteIndexIncoming = sharding.NewRemoteIndexIncoming(repo) - appState.RemoteNodeIncoming = sharding.NewRemoteNodeIncoming(repo) - appState.RemoteReplicaIncoming = replica.NewRemoteReplicaIncoming(repo) - - backupManager := backup.NewHandler(appState.Logger, appState.Authorizer, - schemaManager, repo, appState.Modules) - appState.BackupManager = backupManager - - go clusterapi.Serve(appState) - - vectorRepo.SetSchemaGetter(schemaManager) - explorer.SetSchemaGetter(schemaManager) - appState.Modules.SetSchemaGetter(schemaManager) - - err = vectorRepo.WaitForStartup(ctx) - if err != nil { - appState.Logger. - WithError(err). - WithField("action", "startup"). - Fatal("db didn't start up") - os.Exit(1) - } - - if err := schemaManager.StartServing(ctx); err != nil { - appState.Logger. - WithError(err). - WithField("action", "startup"). - Fatal("schema manager: resume dangling txs") - os.Exit(1) - - } - - batchManager := objects.NewBatchManager(vectorRepo, appState.Modules, - appState.Locks, schemaManager, appState.ServerConfig, appState.Logger, - appState.Authorizer, appState.Metrics) - appState.BatchManager = batchManager - objectsTraverser := traverser.NewTraverser(appState.ServerConfig, appState.Locks, - appState.Logger, appState.Authorizer, vectorRepo, explorer, schemaManager, - appState.Modules, traverser.NewMetrics(appState.Metrics), - appState.ServerConfig.Config.MaximumConcurrentGetRequests) - appState.Traverser = objectsTraverser - - updateSchemaCallback := makeUpdateSchemaCall(appState.Logger, appState, objectsTraverser) - schemaManager.RegisterSchemaUpdateCallback(updateSchemaCallback) - - err = migrator.AdjustFilterablePropSettings(ctx) - if err != nil { - appState.Logger. - WithError(err). - WithField("action", "adjustFilterablePropSettings"). - Fatal("migration failed") - os.Exit(1) - } - - // FIXME to avoid import cycles, tasks are passed as strings - reindexTaskNames := []string{} - var reindexCtx context.Context - reindexCtx, appState.ReindexCtxCancel = context.WithCancel(context.Background()) - reindexFinished := make(chan error, 1) - - if appState.ServerConfig.Config.ReindexSetToRoaringsetAtStartup { - reindexTaskNames = append(reindexTaskNames, "ShardInvertedReindexTaskSetToRoaringSet") - } - if appState.ServerConfig.Config.IndexMissingTextFilterableAtStartup { - reindexTaskNames = append(reindexTaskNames, "ShardInvertedReindexTaskMissingTextFilterable") - } - if len(reindexTaskNames) > 0 { - // start reindexing inverted indexes (if requested by user) in the background - // allowing db to complete api configuration and start handling requests - go func() { - appState.Logger. - WithField("action", "startup"). - Info("Reindexing inverted indexes") - reindexFinished <- migrator.InvertedReindex(reindexCtx, reindexTaskNames...) - }() - } - - configureServer = makeConfigureServer(appState) - - // while we accept an overall longer startup, e.g. due to a recovery, we - // still want to limit the module startup context, as that's mostly service - // discovery / dependency checking - moduleCtx, cancel := context.WithTimeout(ctx, 120*time.Second) - defer cancel() - - err = initModules(moduleCtx, appState) - if err != nil { - appState.Logger. - WithField("action", "startup").WithError(err). - Fatal("modules didn't initialize") - } - - // manually update schema once - schema := schemaManager.GetSchemaSkipAuth() - updateSchemaCallback(schema) - - // Add dimensions to all the objects in the database, if requested by the user - if appState.ServerConfig.Config.ReindexVectorDimensionsAtStartup { - appState.Logger. - WithField("action", "startup"). - Info("Reindexing dimensions") - migrator.RecalculateVectorDimensions(ctx) - } - - // Add recount properties of all the objects in the database, if requested by the user - if appState.ServerConfig.Config.RecountPropertiesAtStartup { - migrator.RecountProperties(ctx) - } - - return appState -} - -func configureAPI(api *operations.WeaviateAPI) http.Handler { - ctx := context.Background() - ctx, cancel := context.WithTimeout(ctx, 60*time.Minute) - defer cancel() - - config.ServerVersion = parseVersionFromSwaggerSpec() - appState := MakeAppState(ctx, connectorOptionGroup) - - api.ServeError = openapierrors.ServeError - - api.JSONConsumer = runtime.JSONConsumer() - - api.OidcAuth = composer.New( - appState.ServerConfig.Config.Authentication, - appState.APIKey, appState.OIDC) - - api.Logger = func(msg string, args ...interface{}) { - appState.Logger.WithField("action", "restapi_management").Infof(msg, args...) - } - - classifier := classification.New(appState.SchemaManager, appState.ClassificationRepo, appState.DB, // the DB is the vectorrepo - appState.Authorizer, - appState.Logger, appState.Modules) - - setupSchemaHandlers(api, appState.SchemaManager, appState.Metrics, appState.Logger) - objectsManager := objects.NewManager(appState.Locks, - appState.SchemaManager, appState.ServerConfig, appState.Logger, - appState.Authorizer, appState.DB, appState.Modules, - objects.NewMetrics(appState.Metrics)) - setupObjectHandlers(api, objectsManager, appState.ServerConfig.Config, appState.Logger, - appState.Modules, appState.Metrics) - setupObjectBatchHandlers(api, appState.BatchManager, appState.Metrics, appState.Logger) - setupGraphQLHandlers(api, appState, appState.SchemaManager, appState.ServerConfig.Config.DisableGraphQL, - appState.Metrics, appState.Logger) - setupMiscHandlers(api, appState.ServerConfig, appState.SchemaManager, appState.Modules, - appState.Metrics, appState.Logger) - setupClassificationHandlers(api, classifier, appState.Metrics, appState.Logger) - backupScheduler := backup.NewScheduler( - appState.Authorizer, - clients.NewClusterBackups(appState.ClusterHttpClient), - appState.DB, appState.Modules, - appState.Cluster, - appState.Logger) - setupBackupHandlers(api, backupScheduler, appState.Metrics, appState.Logger) - setupNodesHandlers(api, appState.SchemaManager, appState.DB, appState) - - grpcServer := createGrpcServer(appState) - setupMiddlewares := makeSetupMiddlewares(appState) - setupGlobalMiddleware := makeSetupGlobalMiddleware(appState) - - api.ServerShutdown = func() { - // stop reindexing on server shutdown - appState.ReindexCtxCancel() - - // gracefully stop gRPC server - grpcServer.GracefulStop() - - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) - defer cancel() - - if err := appState.SchemaManager.Shutdown(ctx); err != nil { - panic(err) - } - - if err := appState.DB.Shutdown(ctx); err != nil { - panic(err) - } - } - - startGrpcServer(grpcServer, appState) - - return setupGlobalMiddleware(api.Serve(setupMiddlewares)) -} - -// TODO: Split up and don't write into global variables. Instead return an appState -func startupRoutine(ctx context.Context, options *swag.CommandLineOptionsGroup) *state.State { - appState := &state.State{} - - logger := logger() - appState.Logger = logger - - logger.WithField("action", "startup").WithField("startup_time_left", timeTillDeadline(ctx)). - Debug("created startup context, nothing done so far") - - // Load the config using the flags - serverConfig := &config.WeaviateConfig{} - appState.ServerConfig = serverConfig - err := serverConfig.LoadConfig(options, logger) - if err != nil { - logger.WithField("action", "startup").WithError(err).Error("could not load config") - logger.Exit(1) - } - - monitoring.InitConfig(serverConfig.Config.Monitoring) - - if serverConfig.Config.DisableGraphQL { - logger.WithFields(logrus.Fields{ - "action": "startup", - "disable_graphql": true, - }).Warnf("GraphQL API disabled, relying only on gRPC API for querying. " + - "This is considered experimental and will likely experience breaking changes " + - "before reaching general availability") - } - - logger.WithFields(logrus.Fields{ - "action": "startup", - "default_vectorizer_module": serverConfig.Config.DefaultVectorizerModule, - }).Infof("the default vectorizer modules is set to %q, as a result all new "+ - "schema classes without an explicit vectorizer setting, will use this "+ - "vectorizer", serverConfig.Config.DefaultVectorizerModule) - - logger.WithFields(logrus.Fields{ - "action": "startup", - "auto_schema_enabled": serverConfig.Config.AutoSchema.Enabled, - }).Infof("auto schema enabled setting is set to \"%v\"", serverConfig.Config.AutoSchema.Enabled) - - logger.WithField("action", "startup").WithField("startup_time_left", timeTillDeadline(ctx)). - Debug("config loaded") - - appState.OIDC = configureOIDC(appState) - appState.APIKey = configureAPIKey(appState) - appState.AnonymousAccess = configureAnonymousAccess(appState) - appState.Authorizer = configureAuthorizer(appState) - - logger.WithField("action", "startup").WithField("startup_time_left", timeTillDeadline(ctx)). - Debug("configured OIDC and anonymous access client") - - appState.Locks = &dummyLock{} - - logger.WithField("action", "startup").WithField("startup_time_left", timeTillDeadline(ctx)). - Debug("initialized schema") - - clusterState, err := cluster.Init(serverConfig.Config.Cluster, serverConfig.Config.Persistence.DataPath, logger) - if err != nil { - logger.WithField("action", "startup").WithError(err). - Error("could not init cluster state") - logger.Exit(1) - } - - appState.Cluster = clusterState - - appState.Logger. - WithField("action", "startup"). - Debug("startup routine complete") - - return appState -} - -// logger does not parse the regular config object, as logging needs to be -// configured before the configuration is even loaded/parsed. We are thus -// "manually" reading the desired env vars and set reasonable defaults if they -// are not set. -// -// Defaults to log level info and json format -func logger() *logrus.Logger { - logger := logrus.New() - if os.Getenv("LOG_FORMAT") != "text" { - logger.SetFormatter(&logrus.JSONFormatter{}) - } - switch os.Getenv("LOG_LEVEL") { - case "debug": - logger.SetLevel(logrus.DebugLevel) - case "trace": - logger.SetLevel(logrus.TraceLevel) - default: - logger.SetLevel(logrus.InfoLevel) - } - - return logger -} - -type dummyLock struct{} - -func (d *dummyLock) LockConnector() (func() error, error) { - return func() error { return nil }, nil -} - -func (d *dummyLock) LockSchema() (func() error, error) { - return func() error { return nil }, nil -} - -// everything hard-coded right now, to be made dynamic (from go plugins later) -func registerModules(appState *state.State) error { - appState.Logger. - WithField("action", "startup"). - Debug("start registering modules") - - appState.Modules = modules.NewProvider() - - enabledModules := map[string]bool{} - if len(appState.ServerConfig.Config.EnableModules) > 0 { - modules := strings.Split(appState.ServerConfig.Config.EnableModules, ",") - for _, module := range modules { - enabledModules[strings.TrimSpace(module)] = true - } - } - - if _, ok := enabledModules["text2vec-contextionary"]; ok { - appState.Modules.Register(modcontextionary.New()) - appState.Logger. - WithField("action", "startup"). - WithField("module", "text2vec-contextionary"). - Debug("enabled module") - } - - if _, ok := enabledModules["text2vec-transformers"]; ok { - appState.Modules.Register(modtransformers.New()) - appState.Logger. - WithField("action", "startup"). - WithField("module", "text2vec-transformers"). - Debug("enabled module") - } - - if _, ok := enabledModules[modgpt4all.Name]; ok { - appState.Modules.Register(modgpt4all.New()) - appState.Logger. - WithField("action", "startup"). - WithField("module", modgpt4all.Name). - Debug("enabled module") - } - - if _, ok := enabledModules[modrerankertransformers.Name]; ok { - appState.Modules.Register(modrerankertransformers.New()) - appState.Logger. - WithField("action", "startup"). - WithField("module", modrerankertransformers.Name). - Debug("enabled module") - } - - if _, ok := enabledModules[modrerankercohere.Name]; ok { - appState.Modules.Register(modrerankercohere.New()) - appState.Logger. - WithField("action", "startup"). - WithField("module", modrerankercohere.Name). - Debug("enabled module") - } - - if _, ok := enabledModules["qna-transformers"]; ok { - appState.Modules.Register(modqna.New()) - appState.Logger. - WithField("action", "startup"). - WithField("module", "qna-transformers"). - Debug("enabled module") - } - - if _, ok := enabledModules["sum-transformers"]; ok { - appState.Modules.Register(modsum.New()) - appState.Logger. - WithField("action", "startup"). - WithField("module", "sum-transformers"). - Debug("enabled module") - } - - if _, ok := enabledModules["img2vec-neural"]; ok { - appState.Modules.Register(modimage.New()) - appState.Logger. - WithField("action", "startup"). - WithField("module", "img2vec-neural"). - Debug("enabled module") - } - - if _, ok := enabledModules["ner-transformers"]; ok { - appState.Modules.Register(modner.New()) - appState.Logger. - WithField("action", "startup"). - WithField("module", "ner-transformers"). - Debug("enabled module") - } - - if _, ok := enabledModules["text-spellcheck"]; ok { - appState.Modules.Register(modspellcheck.New()) - appState.Logger. - WithField("action", "startup"). - WithField("module", "text-spellcheck"). - Debug("enabled module") - } - - if _, ok := enabledModules["multi2vec-clip"]; ok { - appState.Modules.Register(modclip.New()) - appState.Logger. - WithField("action", "startup"). - WithField("module", "multi2vec-clip"). - Debug("enabled module") - } - - if _, ok := enabledModules["text2vec-openai"]; ok { - appState.Modules.Register(modopenai.New()) - appState.Logger. - WithField("action", "startup"). - WithField("module", "text2vec-openai"). - Debug("enabled module") - } - - if _, ok := enabledModules["qna-openai"]; ok { - appState.Modules.Register(modqnaopenai.New()) - appState.Logger. - WithField("action", "startup"). - WithField("module", "qna-openai"). - Debug("enabled module") - } - - if _, ok := enabledModules[modgenerativecohere.Name]; ok { - appState.Modules.Register(modgenerativecohere.New()) - appState.Logger. - WithField("action", "startup"). - WithField("module", modgenerativecohere.Name). - Debug("enabled module") - } - - if _, ok := enabledModules[modgenerativeopenai.Name]; ok { - appState.Modules.Register(modgenerativeopenai.New()) - appState.Logger. - WithField("action", "startup"). - WithField("module", modgenerativeopenai.Name). - Debug("enabled module") - } - - if _, ok := enabledModules[modgenerativeaws.Name]; ok { - appState.Modules.Register(modgenerativeaws.New()) - appState.Logger. - WithField("action", "startup"). - WithField("module", modgenerativeaws.Name). - Debug("enabled module") - } - - if _, ok := enabledModules[modhuggingface.Name]; ok { - appState.Modules.Register(modhuggingface.New()) - appState.Logger. - WithField("action", "startup"). - WithField("module", modhuggingface.Name). - Debug("enabled module") - } - - if _, ok := enabledModules[modgenerativepalm.Name]; ok { - appState.Modules.Register(modgenerativepalm.New()) - appState.Logger. - WithField("action", "startup"). - WithField("module", modgenerativepalm.Name). - Debug("enabled module") - } - - if _, ok := enabledModules[modgenerativeanyscale.Name]; ok { - appState.Modules.Register(modgenerativeanyscale.New()) - appState.Logger. - WithField("action", "startup"). - WithField("module", modgenerativeanyscale.Name). - Debug("enabled module") - } - - if _, ok := enabledModules[modtext2vecpalm.Name]; ok { - appState.Modules.Register(modtext2vecpalm.New()) - appState.Logger. - WithField("action", "startup"). - WithField("module", modtext2vecpalm.Name). - Debug("enabled module") - } - - if _, ok := enabledModules[modtext2vecaws.Name]; ok { - appState.Modules.Register(modtext2vecaws.New()) - appState.Logger. - WithField("action", "startup"). - WithField("module", modtext2vecaws.Name). - Debug("enabled module") - } - - if _, ok := enabledModules[modstgfs.Name]; ok { - appState.Modules.Register(modstgfs.New()) - appState.Logger. - WithField("action", "startup"). - WithField("module", modstgfs.Name). - Debug("enabled module") - } - - if _, ok := enabledModules[modstgs3.Name]; ok { - appState.Modules.Register(modstgs3.New()) - appState.Logger. - WithField("action", "startup"). - WithField("module", modstgs3.Name). - Debug("enabled module") - } - - if _, ok := enabledModules[modstggcs.Name]; ok { - appState.Modules.Register(modstggcs.New()) - appState.Logger. - WithField("action", "startup"). - WithField("module", modstggcs.Name). - Debug("enabled module") - } - - if _, ok := enabledModules[modstgazure.Name]; ok { - appState.Modules.Register(modstgazure.New()) - appState.Logger. - WithField("action", "startup"). - WithField("module", modstgazure.Name). - Debug("enabled module") - } - - if _, ok := enabledModules[modcentroid.Name]; ok { - appState.Modules.Register(modcentroid.New()) - appState.Logger. - WithField("action", "startup"). - WithField("module", modcentroid.Name). - Debug("enabled module") - } - - if _, ok := enabledModules[modcohere.Name]; ok { - appState.Modules.Register(modcohere.New()) - appState.Logger. - WithField("action", "startup"). - WithField("module", modcohere.Name). - Debug("enabled module") - } - - if _, ok := enabledModules[modbind.Name]; ok { - appState.Modules.Register(modbind.New()) - appState.Logger. - WithField("action", "startup"). - WithField("module", modbind.Name). - Debug("enabled module") - } - - if _, ok := enabledModules[modjinaai.Name]; ok { - appState.Modules.Register(modjinaai.New()) - appState.Logger. - WithField("action", "startup"). - WithField("module", modjinaai.Name). - Debug("enabled module") - } - - appState.Logger. - WithField("action", "startup"). - Debug("completed registering modules") - - return nil -} - -func initModules(ctx context.Context, appState *state.State) error { - storageProvider, err := modulestorage.NewRepo( - appState.ServerConfig.Config.Persistence.DataPath, appState.Logger) - if err != nil { - return errors.Wrap(err, "init storage provider") - } - - // TODO: gh-1481 don't pass entire appState in, but only what's needed. Probably only - // config? - moduleParams := moduletools.NewInitParams(storageProvider, appState, - appState.ServerConfig.Config, appState.Logger) - - appState.Logger. - WithField("action", "startup"). - Debug("start initializing modules") - if err := appState.Modules.Init(ctx, moduleParams, appState.Logger); err != nil { - return errors.Wrap(err, "init modules") - } - - appState.Logger. - WithField("action", "startup"). - Debug("finished initializing modules") - - return nil -} - -type clientWithAuth struct { - r http.RoundTripper - basicAuth cluster.BasicAuth -} - -func (c clientWithAuth) RoundTrip(r *http.Request) (*http.Response, error) { - r.SetBasicAuth(c.basicAuth.Username, c.basicAuth.Password) - return c.r.RoundTrip(r) -} - -func reasonableHttpClient(authConfig cluster.AuthConfig) *http.Client { - t := &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 120 * time.Second, - }).DialContext, - MaxIdleConnsPerHost: 100, - MaxIdleConns: 100, - IdleConnTimeout: 90 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - } - if authConfig.BasicAuth.Enabled() { - return &http.Client{Transport: clientWithAuth{r: t, basicAuth: authConfig.BasicAuth}} - } - return &http.Client{Transport: t} -} - -func setupGoProfiling(config config.Config) { - go func() { - fmt.Println(http.ListenAndServe(":6060", nil)) - }() - - if config.Profiling.BlockProfileRate > 0 { - goruntime.SetBlockProfileRate(config.Profiling.BlockProfileRate) - } - - if config.Profiling.MutexProfileFraction > 0 { - goruntime.SetMutexProfileFraction(config.Profiling.MutexProfileFraction) - } -} - -func parseVersionFromSwaggerSpec() string { - spec := struct { - Info struct { - Version string `json:"version"` - } `json:"info"` - }{} - - err := json.Unmarshal(SwaggerJSON, &spec) - if err != nil { - panic(err) - } - - return spec.Info.Version -} - -func limitResources(appState *state.State) { - if os.Getenv("LIMIT_RESOURCES") == "true" { - appState.Logger.Info("Limiting resources: memory: 80%, cores: all but one") - if os.Getenv("GOMAXPROCS") == "" { - // Fetch the number of cores from the cgroups cpuset - // and parse it into an int - cores, err := getCores() - if err == nil { - appState.Logger.WithField("cores", cores). - Warn("GOMAXPROCS not set, and unable to read from cgroups, setting to number of cores") - goruntime.GOMAXPROCS(cores) - } else { - cores = goruntime.NumCPU() - 1 - if cores > 0 { - appState.Logger.WithField("cores", cores). - Warnf("Unable to read from cgroups: %v, setting to max cores to: %v", err, cores) - goruntime.GOMAXPROCS(cores) - } - } - } - - limit, err := memlimit.SetGoMemLimit(0.8) - if err != nil { - appState.Logger.WithError(err).Warnf("Unable to set memory limit from cgroups: %v", err) - // Set memory limit to 90% of the available memory - limit := int64(float64(memory.TotalMemory()) * 0.8) - debug.SetMemoryLimit(limit) - appState.Logger.WithField("limit", limit).Info("Set memory limit based on available memory") - } else { - appState.Logger.WithField("limit", limit).Info("Set memory limit") - } - } else { - appState.Logger.Info("No resource limits set, weaviate will use all available memory and CPU. " + - "To limit resources, set LIMIT_RESOURCES=true") - } -} diff --git a/adapters/handlers/rest/configure_server.go b/adapters/handlers/rest/configure_server.go deleted file mode 100644 index f8921454189183a1d24ffd2a8767938d18afef5b..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/configure_server.go +++ /dev/null @@ -1,116 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package rest - -import ( - "context" - "net/http" - "os" - "time" - - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/adapters/handlers/graphql" - "github.com/weaviate/weaviate/adapters/handlers/graphql/utils" - "github.com/weaviate/weaviate/adapters/handlers/rest/state" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/usecases/auth/authentication/anonymous" - "github.com/weaviate/weaviate/usecases/auth/authentication/apikey" - "github.com/weaviate/weaviate/usecases/auth/authentication/oidc" - "github.com/weaviate/weaviate/usecases/auth/authorization" - "github.com/weaviate/weaviate/usecases/config" - "github.com/weaviate/weaviate/usecases/modules" - "github.com/weaviate/weaviate/usecases/traverser" -) - -// As soon as server is initialized but not run yet, this function will be called. -// If you need to modify a config, store server instance to stop it individually later, this is the place. -// This function can be called multiple times, depending on the number of serving schemes. -// scheme value will be set accordingly: "http", "https" or "unix" -// -// we will set it through configureAPI() as it needs access to resources that -// are only available within there -var configureServer func(*http.Server, string, string) - -func makeUpdateSchemaCall(logger logrus.FieldLogger, appState *state.State, traverser *traverser.Traverser) func(schema.Schema) { - return func(updatedSchema schema.Schema) { - if appState.ServerConfig.Config.DisableGraphQL { - return - } - - // Note that this is thread safe; we're running in a single go-routine, because the event - // handlers are called when the SchemaLock is still held. - - gql, err := rebuildGraphQL( - updatedSchema, - logger, - appState.ServerConfig.Config, - traverser, - appState.Modules, - ) - if err != nil && err != utils.ErrEmptySchema { - logger.WithField("action", "graphql_rebuild"). - WithError(err).Error("could not (re)build graphql provider") - } - appState.SetGraphQL(gql) - } -} - -func rebuildGraphQL(updatedSchema schema.Schema, logger logrus.FieldLogger, - config config.Config, traverser *traverser.Traverser, modulesProvider *modules.Provider, -) (graphql.GraphQL, error) { - updatedGraphQL, err := graphql.Build(&updatedSchema, traverser, logger, config, modulesProvider) - if err != nil { - return nil, err - } - - logger.WithField("action", "graphql_rebuild").Debug("successfully rebuild graphql schema") - return updatedGraphQL, nil -} - -// configureOIDC will always be called, even if OIDC is disabled, this way the -// middleware will still be able to provide the user with a valuable error -// message, even when OIDC is globally disabled. -func configureOIDC(appState *state.State) *oidc.Client { - c, err := oidc.New(appState.ServerConfig.Config) - if err != nil { - appState.Logger.WithField("action", "oidc_init").WithError(err).Fatal("oidc client could not start up") - os.Exit(1) - } - - return c -} - -func configureAPIKey(appState *state.State) *apikey.Client { - c, err := apikey.New(appState.ServerConfig.Config) - if err != nil { - appState.Logger.WithField("action", "oidc_init").WithError(err).Fatal("oidc client could not start up") - os.Exit(1) - } - - return c -} - -// configureAnonymousAccess will always be called, even if anonymous access is -// disabled. In this case the middleware provided by this client will block -// anonymous requests -func configureAnonymousAccess(appState *state.State) *anonymous.Client { - return anonymous.New(appState.ServerConfig.Config) -} - -func configureAuthorizer(appState *state.State) authorization.Authorizer { - return authorization.New(appState.ServerConfig.Config) -} - -func timeTillDeadline(ctx context.Context) string { - dl, _ := ctx.Deadline() - return time.Until(dl).String() -} diff --git a/adapters/handlers/rest/configure_weaviate.go b/adapters/handlers/rest/configure_weaviate.go deleted file mode 100644 index 6a7ce2bb56ff2fc17b9ce6d8563eada1efb6acc0..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/configure_weaviate.go +++ /dev/null @@ -1,41 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Package rest with all rest API functions. -package rest - -import ( - "crypto/tls" - - "github.com/go-openapi/swag" - - "github.com/weaviate/weaviate/adapters/handlers/rest/operations" - "github.com/weaviate/weaviate/usecases/config" -) - -var connectorOptionGroup *swag.CommandLineOptionsGroup - -// configureAPI -> see configure_api.go - -// configureServer -> see configure_server.go - -func configureFlags(api *operations.WeaviateAPI) { - connectorOptionGroup = config.GetConfigOptionGroup() - - api.CommandLineOptionsGroups = []swag.CommandLineOptionsGroup{ - *connectorOptionGroup, - } -} - -// The TLS configuration before HTTPS server starts. -func configureTLS(tlsConfig *tls.Config) { - // Make all necessary changes to the TLS configuration here. -} diff --git a/adapters/handlers/rest/configure_weaviate_test.go b/adapters/handlers/rest/configure_weaviate_test.go deleted file mode 100644 index 2b19237d7a8689b84957747ef9ad89b548dda588..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/configure_weaviate_test.go +++ /dev/null @@ -1,30 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package rest - -import ( - "reflect" - "testing" -) - -func TestCreateErrorResponseObject(t *testing.T) { - testResults := createErrorResponseObject("error message 1", "error message 2") - - // check which type is used - if typeName := reflect.TypeOf(testResults); typeName.Kind() == reflect.Ptr { - if typeName.Elem().Name() != "ErrorResponse" { - t.Error("Wrong struct used, should be ErrorResponse but is: ", typeName.Elem().Name()) - } - } else { - t.Error("Wrong struct used, should be ErrorResponse but is: ", typeName.Name()) - } -} diff --git a/adapters/handlers/rest/doc.go b/adapters/handlers/rest/doc.go deleted file mode 100644 index 2a0c256754f9c41751bc950409f7596ea08e2592..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/doc.go +++ /dev/null @@ -1,32 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -// Package rest Weaviate -// -// Cloud-native, modular vector database -// Schemes: -// https -// Host: localhost -// BasePath: /v1 -// Version: 1.23.3 -// Contact: Weaviate https://github.com/weaviate -// -// Consumes: -// - application/json -// - application/yaml -// -// Produces: -// - application/json -// -// swagger:meta -package rest diff --git a/adapters/handlers/rest/embedded_spec.go b/adapters/handlers/rest/embedded_spec.go deleted file mode 100644 index 8f4744140c159e05c2415ecb6808073903a39c8f..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/embedded_spec.go +++ /dev/null @@ -1,10255 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package rest - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "encoding/json" -) - -var ( - // SwaggerJSON embedded version of the swagger document used at generation time - SwaggerJSON json.RawMessage - // FlatSwaggerJSON embedded flattened version of the swagger document used at generation time - FlatSwaggerJSON json.RawMessage -) - -func init() { - SwaggerJSON = json.RawMessage([]byte(`{ - "consumes": [ - "application/yaml", - "application/json" - ], - "produces": [ - "application/json" - ], - "schemes": [ - "https" - ], - "swagger": "2.0", - "info": { - "description": "Cloud-native, modular vector database", - "title": "Weaviate", - "contact": { - "name": "Weaviate", - "url": "https://github.com/weaviate", - "email": "hello@weaviate.io" - }, - "version": "1.23.3" - }, - "basePath": "/v1", - "paths": { - "/": { - "get": { - "description": "Home. Discover the REST API", - "operationId": "weaviate.root", - "responses": { - "200": { - "description": "Weaviate is alive and ready to serve content", - "schema": { - "type": "object", - "properties": { - "links": { - "type": "array", - "items": { - "$ref": "#/definitions/Link" - } - } - } - } - } - } - } - }, - "/.well-known/live": { - "get": { - "description": "Determines whether the application is alive. Can be used for kubernetes liveness probe", - "operationId": "weaviate.wellknown.liveness", - "responses": { - "200": { - "description": "The application is able to respond to HTTP requests" - } - } - } - }, - "/.well-known/openid-configuration": { - "get": { - "description": "OIDC Discovery page, redirects to the token issuer if one is configured", - "tags": [ - "well-known", - "oidc", - "discovery" - ], - "summary": "OIDC discovery information if OIDC auth is enabled", - "responses": { - "200": { - "description": "Successful response, inspect body", - "schema": { - "type": "object", - "properties": { - "clientId": { - "description": "OAuth Client ID", - "type": "string" - }, - "href": { - "description": "The Location to redirect to", - "type": "string" - }, - "scopes": { - "description": "OAuth Scopes", - "type": "array", - "items": { - "type": "string" - }, - "x-omitempty": true - } - } - } - }, - "404": { - "description": "Not found, no oidc provider present" - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": false, - "x-available-in-websocket": false - } - }, - "/.well-known/ready": { - "get": { - "description": "Determines whether the application is ready to receive traffic. Can be used for kubernetes readiness probe.", - "operationId": "weaviate.wellknown.readiness", - "responses": { - "200": { - "description": "The application has completed its start-up routine and is ready to accept traffic." - }, - "503": { - "description": "The application is currently not able to serve traffic. If other horizontal replicas of weaviate are available and they are capable of receiving traffic, all traffic should be redirected there instead." - } - } - } - }, - "/backups/{backend}": { - "post": { - "description": "Starts a process of creating a backup for a set of classes", - "tags": [ - "backups" - ], - "operationId": "backups.create", - "parameters": [ - { - "type": "string", - "description": "Backup backend name e.g. filesystem, gcs, s3.", - "name": "backend", - "in": "path", - "required": true - }, - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/BackupCreateRequest" - } - } - ], - "responses": { - "200": { - "description": "Backup create process successfully started.", - "schema": { - "$ref": "#/definitions/BackupCreateResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Invalid backup creation attempt.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-serviceIds": [ - "weaviate.local.backup" - ] - } - }, - "/backups/{backend}/{id}": { - "get": { - "description": "Returns status of backup creation attempt for a set of classes", - "tags": [ - "backups" - ], - "operationId": "backups.create.status", - "parameters": [ - { - "type": "string", - "description": "Backup backend name e.g. filesystem, gcs, s3.", - "name": "backend", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "The ID of a backup. Must be URL-safe and work as a filesystem path, only lowercase, numbers, underscore, minus characters allowed.", - "name": "id", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "Backup creation status successfully returned", - "schema": { - "$ref": "#/definitions/BackupCreateStatusResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Not Found - Backup does not exist", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Invalid backup restoration status attempt.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-serviceIds": [ - "weaviate.local.backup" - ] - } - }, - "/backups/{backend}/{id}/restore": { - "get": { - "description": "Returns status of a backup restoration attempt for a set of classes", - "tags": [ - "backups" - ], - "operationId": "backups.restore.status", - "parameters": [ - { - "type": "string", - "description": "Backup backend name e.g. filesystem, gcs, s3.", - "name": "backend", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "The ID of a backup. Must be URL-safe and work as a filesystem path, only lowercase, numbers, underscore, minus characters allowed.", - "name": "id", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "Backup restoration status successfully returned", - "schema": { - "$ref": "#/definitions/BackupRestoreStatusResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Not Found - Backup does not exist", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-serviceIds": [ - "weaviate.local.backup" - ] - }, - "post": { - "description": "Starts a process of restoring a backup for a set of classes", - "tags": [ - "backups" - ], - "operationId": "backups.restore", - "parameters": [ - { - "type": "string", - "description": "Backup backend name e.g. filesystem, gcs, s3.", - "name": "backend", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "The ID of a backup. Must be URL-safe and work as a filesystem path, only lowercase, numbers, underscore, minus characters allowed.", - "name": "id", - "in": "path", - "required": true - }, - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/BackupRestoreRequest" - } - } - ], - "responses": { - "200": { - "description": "Backup restoration process successfully started.", - "schema": { - "$ref": "#/definitions/BackupRestoreResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Not Found - Backup does not exist", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Invalid backup restoration attempt.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-serviceIds": [ - "weaviate.local.backup" - ] - } - }, - "/batch/objects": { - "post": { - "description": "Register new Objects in bulk. Provided meta-data and schema values are validated.", - "tags": [ - "batch", - "objects" - ], - "summary": "Creates new Objects based on a Object template as a batch.", - "operationId": "batch.objects.create", - "parameters": [ - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "type": "object", - "properties": { - "fields": { - "description": "Define which fields need to be returned. Default value is ALL", - "type": "array", - "items": { - "type": "string", - "default": "ALL", - "enum": [ - "ALL", - "class", - "schema", - "id", - "creationTimeUnix" - ] - } - }, - "objects": { - "type": "array", - "items": { - "$ref": "#/definitions/Object" - } - } - } - } - }, - { - "$ref": "#/parameters/CommonConsistencyLevelParameterQuery" - } - ], - "responses": { - "200": { - "description": "Request succeeded, see response body to get detailed information about each batched item.", - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/ObjectsGetResponse" - } - } - }, - "400": { - "description": "Malformed request.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "x-serviceIds": [ - "weaviate.local.add" - ] - }, - "delete": { - "description": "Delete Objects in bulk that match a certain filter.", - "tags": [ - "batch", - "objects" - ], - "summary": "Deletes Objects based on a match filter as a batch.", - "operationId": "batch.objects.delete", - "parameters": [ - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/BatchDelete" - } - }, - { - "$ref": "#/parameters/CommonConsistencyLevelParameterQuery" - }, - { - "$ref": "#/parameters/CommonTenantParameterQuery" - } - ], - "responses": { - "200": { - "description": "Request succeeded, see response body to get detailed information about each batched item.", - "schema": { - "$ref": "#/definitions/BatchDeleteResponse" - } - }, - "400": { - "description": "Malformed request.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "x-serviceIds": [ - "weaviate.local.manipulate" - ] - } - }, - "/batch/references": { - "post": { - "description": "Register cross-references between any class items (objects or objects) in bulk.", - "tags": [ - "batch", - "references" - ], - "summary": "Creates new Cross-References between arbitrary classes in bulk.", - "operationId": "batch.references.create", - "parameters": [ - { - "description": "A list of references to be batched. The ideal size depends on the used database connector. Please see the documentation of the used connector for help", - "name": "body", - "in": "body", - "required": true, - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/BatchReference" - } - } - }, - { - "$ref": "#/parameters/CommonConsistencyLevelParameterQuery" - } - ], - "responses": { - "200": { - "description": "Request Successful. Warning: A successful request does not guarantee that every batched reference was successfully created. Inspect the response body to see which references succeeded and which failed.", - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/BatchReferenceResponse" - } - } - }, - "400": { - "description": "Malformed request.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "x-serviceIds": [ - "weaviate.local.add" - ] - } - }, - "/classifications/": { - "post": { - "description": "Trigger a classification based on the specified params. Classifications will run in the background, use GET /classifications/\u003cid\u003e to retrieve the status of your classification.", - "tags": [ - "classifications" - ], - "summary": "Starts a classification.", - "operationId": "classifications.post", - "parameters": [ - { - "description": "parameters to start a classification", - "name": "params", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/Classification" - } - } - ], - "responses": { - "201": { - "description": "Successfully started classification.", - "schema": { - "$ref": "#/definitions/Classification" - } - }, - "400": { - "description": "Incorrect request", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-serviceIds": [ - "weaviate.classifications.post" - ] - } - }, - "/classifications/{id}": { - "get": { - "description": "Get status, results and metadata of a previously created classification", - "tags": [ - "classifications" - ], - "summary": "View previously created classification", - "operationId": "classifications.get", - "parameters": [ - { - "type": "string", - "description": "classification id", - "name": "id", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "Found the classification, returned as body", - "schema": { - "$ref": "#/definitions/Classification" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Not Found - Classification does not exist" - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-serviceIds": [ - "weaviate.classifications.get" - ] - } - }, - "/graphql": { - "post": { - "description": "Get an object based on GraphQL", - "tags": [ - "graphql" - ], - "summary": "Get a response based on GraphQL", - "operationId": "graphql.post", - "parameters": [ - { - "description": "The GraphQL query request parameters.", - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/GraphQLQuery" - } - } - ], - "responses": { - "200": { - "description": "Successful query (with select).", - "schema": { - "$ref": "#/definitions/GraphQLResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "x-serviceIds": [ - "weaviate.local.query", - "weaviate.local.query.meta", - "weaviate.network.query", - "weaviate.network.query.meta" - ] - } - }, - "/graphql/batch": { - "post": { - "description": "Perform a batched GraphQL query", - "tags": [ - "graphql" - ], - "summary": "Get a response based on GraphQL.", - "operationId": "graphql.batch", - "parameters": [ - { - "description": "The GraphQL queries.", - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/GraphQLQueries" - } - } - ], - "responses": { - "200": { - "description": "Successful query (with select).", - "schema": { - "$ref": "#/definitions/GraphQLResponses" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "x-serviceIds": [ - "weaviate.local.query", - "weaviate.local.query.meta", - "weaviate.network.query", - "weaviate.network.query.meta" - ] - } - }, - "/meta": { - "get": { - "description": "Gives meta information about the server and can be used to provide information to another Weaviate instance that wants to interact with the current instance.", - "tags": [ - "meta" - ], - "summary": "Returns meta information of the current Weaviate instance.", - "operationId": "meta.get", - "responses": { - "200": { - "description": "Successful response.", - "schema": { - "$ref": "#/definitions/Meta" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "x-serviceIds": [ - "weaviate.local.query.meta" - ] - } - }, - "/nodes": { - "get": { - "description": "Returns status of Weaviate DB.", - "tags": [ - "nodes" - ], - "operationId": "nodes.get", - "parameters": [ - { - "$ref": "#/parameters/CommonOutputVerbosityParameterQuery" - } - ], - "responses": { - "200": { - "description": "Nodes status successfully returned", - "schema": { - "$ref": "#/definitions/NodesStatusResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Not Found - Backup does not exist", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Invalid backup restoration status attempt.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-serviceIds": [ - "weaviate.nodes.status.get" - ] - } - }, - "/nodes/{className}": { - "get": { - "description": "Returns status of Weaviate DB.", - "tags": [ - "nodes" - ], - "operationId": "nodes.get.class", - "parameters": [ - { - "type": "string", - "name": "className", - "in": "path", - "required": true - }, - { - "$ref": "#/parameters/CommonOutputVerbosityParameterQuery" - } - ], - "responses": { - "200": { - "description": "Nodes status successfully returned", - "schema": { - "$ref": "#/definitions/NodesStatusResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Not Found - Backup does not exist", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Invalid backup restoration status attempt.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-serviceIds": [ - "weaviate.nodes.status.get.class" - ] - } - }, - "/objects": { - "get": { - "description": "Lists all Objects in reverse order of creation, owned by the user that belongs to the used token.", - "tags": [ - "objects" - ], - "summary": "Get a list of Objects.", - "operationId": "objects.list", - "parameters": [ - { - "$ref": "#/parameters/CommonAfterParameterQuery" - }, - { - "$ref": "#/parameters/CommonOffsetParameterQuery" - }, - { - "$ref": "#/parameters/CommonLimitParameterQuery" - }, - { - "$ref": "#/parameters/CommonIncludeParameterQuery" - }, - { - "$ref": "#/parameters/CommonSortParameterQuery" - }, - { - "$ref": "#/parameters/CommonOrderParameterQuery" - }, - { - "$ref": "#/parameters/CommonClassParameterQuery" - }, - { - "$ref": "#/parameters/CommonTenantParameterQuery" - } - ], - "responses": { - "200": { - "description": "Successful response.", - "schema": { - "$ref": "#/definitions/ObjectsListResponse" - } - }, - "400": { - "description": "Malformed request.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Successful query result but no resource was found." - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "x-serviceIds": [ - "weaviate.local.query" - ] - }, - "post": { - "description": "Registers a new Object. Provided meta-data and schema values are validated.", - "tags": [ - "objects" - ], - "summary": "Create Objects between two Objects (object and subject).", - "operationId": "objects.create", - "parameters": [ - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/Object" - } - }, - { - "$ref": "#/parameters/CommonConsistencyLevelParameterQuery" - } - ], - "responses": { - "200": { - "description": "Object created.", - "schema": { - "$ref": "#/definitions/Object" - } - }, - "400": { - "description": "Malformed request.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "x-serviceIds": [ - "weaviate.local.add" - ] - } - }, - "/objects/validate": { - "post": { - "description": "Validate an Object's schema and meta-data. It has to be based on a schema, which is related to the given Object to be accepted by this validation.", - "tags": [ - "objects" - ], - "summary": "Validate an Object based on a schema.", - "operationId": "objects.validate", - "parameters": [ - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/Object" - } - } - ], - "responses": { - "200": { - "description": "Successfully validated." - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "x-serviceIds": [ - "weaviate.local.query.meta" - ] - } - }, - "/objects/{className}/{id}": { - "get": { - "description": "Get a single data object", - "tags": [ - "objects" - ], - "summary": "Get a specific Object based on its class and UUID. Also available as Websocket bus.", - "operationId": "objects.class.get", - "parameters": [ - { - "type": "string", - "name": "className", - "in": "path", - "required": true - }, - { - "type": "string", - "format": "uuid", - "description": "Unique ID of the Object.", - "name": "id", - "in": "path", - "required": true - }, - { - "$ref": "#/parameters/CommonIncludeParameterQuery" - }, - { - "$ref": "#/parameters/CommonConsistencyLevelParameterQuery" - }, - { - "$ref": "#/parameters/CommonNodeNameParameterQuery" - }, - { - "$ref": "#/parameters/CommonTenantParameterQuery" - } - ], - "responses": { - "200": { - "description": "Successful response.", - "schema": { - "$ref": "#/definitions/Object" - } - }, - "400": { - "description": "Malformed request.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Successful query result but no resource was found." - }, - "422": { - "description": "Request is well-formed (i.e., syntactically correct), but erroneous.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "x-serviceIds": [ - "weaviate.local.query" - ] - }, - "put": { - "description": "Update an individual data object based on its class and uuid.", - "tags": [ - "objects" - ], - "summary": "Update a class object based on its uuid", - "operationId": "objects.class.put", - "parameters": [ - { - "type": "string", - "name": "className", - "in": "path", - "required": true - }, - { - "type": "string", - "format": "uuid", - "description": "The uuid of the data object to update.", - "name": "id", - "in": "path", - "required": true - }, - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/Object" - } - }, - { - "$ref": "#/parameters/CommonConsistencyLevelParameterQuery" - } - ], - "responses": { - "200": { - "description": "Successfully received.", - "schema": { - "$ref": "#/definitions/Object" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Successful query result but no resource was found." - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "x-serviceIds": [ - "weaviate.local.manipulate" - ] - }, - "delete": { - "description": "Delete a single data object.", - "tags": [ - "objects" - ], - "summary": "Delete object based on its class and UUID.", - "operationId": "objects.class.delete", - "parameters": [ - { - "type": "string", - "name": "className", - "in": "path", - "required": true - }, - { - "type": "string", - "format": "uuid", - "description": "Unique ID of the Object.", - "name": "id", - "in": "path", - "required": true - }, - { - "$ref": "#/parameters/CommonConsistencyLevelParameterQuery" - }, - { - "$ref": "#/parameters/CommonTenantParameterQuery" - } - ], - "responses": { - "204": { - "description": "Successfully deleted." - }, - "400": { - "description": "Malformed request.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Successful query result but no resource was found." - }, - "422": { - "description": "Request is well-formed (i.e., syntactically correct), but erroneous.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": true, - "x-available-in-websocket": true, - "x-serviceIds": [ - "weaviate.local.manipulate" - ] - }, - "head": { - "description": "Checks if a data object exists without retrieving it.", - "tags": [ - "objects" - ], - "summary": "Checks object's existence based on its class and uuid.", - "operationId": "objects.class.head", - "parameters": [ - { - "type": "string", - "description": "The class name as defined in the schema", - "name": "className", - "in": "path", - "required": true - }, - { - "type": "string", - "format": "uuid", - "description": "The uuid of the data object", - "name": "id", - "in": "path", - "required": true - }, - { - "$ref": "#/parameters/CommonConsistencyLevelParameterQuery" - }, - { - "$ref": "#/parameters/CommonTenantParameterQuery" - } - ], - "responses": { - "204": { - "description": "Object exists." - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Object doesn't exist." - }, - "422": { - "description": "Request is well-formed (i.e., syntactically correct), but erroneous.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": true, - "x-available-in-websocket": true, - "x-serviceIds": [ - "weaviate.local.manipulate" - ] - }, - "patch": { - "description": "Update an individual data object based on its class and uuid. This method supports json-merge style patch semantics (RFC 7396). Provided meta-data and schema values are validated. LastUpdateTime is set to the time this function is called.", - "tags": [ - "objects" - ], - "summary": "Update an Object based on its UUID (using patch semantics).", - "operationId": "objects.class.patch", - "parameters": [ - { - "type": "string", - "description": "The class name as defined in the schema", - "name": "className", - "in": "path", - "required": true - }, - { - "type": "string", - "format": "uuid", - "description": "The uuid of the data object to update.", - "name": "id", - "in": "path", - "required": true - }, - { - "description": "RFC 7396-style patch, the body contains the object to merge into the existing object.", - "name": "body", - "in": "body", - "schema": { - "$ref": "#/definitions/Object" - } - }, - { - "$ref": "#/parameters/CommonConsistencyLevelParameterQuery" - } - ], - "responses": { - "204": { - "description": "Successfully applied. No content provided." - }, - "400": { - "description": "The patch-JSON is malformed.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Successful query result but no resource was found." - }, - "422": { - "description": "The patch-JSON is valid but unprocessable.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "x-serviceIds": [ - "weaviate.local.manipulate" - ] - } - }, - "/objects/{className}/{id}/references/{propertyName}": { - "put": { - "description": "Update all references of a property of a data object.", - "tags": [ - "objects" - ], - "summary": "Replace all references to a class-property.", - "operationId": "objects.class.references.put", - "parameters": [ - { - "type": "string", - "description": "The class name as defined in the schema", - "name": "className", - "in": "path", - "required": true - }, - { - "type": "string", - "format": "uuid", - "description": "Unique ID of the Object.", - "name": "id", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "Unique name of the property related to the Object.", - "name": "propertyName", - "in": "path", - "required": true - }, - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/MultipleRef" - } - }, - { - "$ref": "#/parameters/CommonConsistencyLevelParameterQuery" - }, - { - "$ref": "#/parameters/CommonTenantParameterQuery" - } - ], - "responses": { - "200": { - "description": "Successfully replaced all the references." - }, - "400": { - "description": "Malformed request.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Source object doesn't exist." - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the property exists or that it is a class?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "x-serviceIds": [ - "weaviate.local.manipulate" - ] - }, - "post": { - "description": "Add a single reference to a class-property.", - "tags": [ - "objects" - ], - "summary": "Add a single reference to a class-property.", - "operationId": "objects.class.references.create", - "parameters": [ - { - "type": "string", - "description": "The class name as defined in the schema", - "name": "className", - "in": "path", - "required": true - }, - { - "type": "string", - "format": "uuid", - "description": "Unique ID of the Object.", - "name": "id", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "Unique name of the property related to the Object.", - "name": "propertyName", - "in": "path", - "required": true - }, - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/SingleRef" - } - }, - { - "$ref": "#/parameters/CommonConsistencyLevelParameterQuery" - }, - { - "$ref": "#/parameters/CommonTenantParameterQuery" - } - ], - "responses": { - "200": { - "description": "Successfully added the reference." - }, - "400": { - "description": "Malformed request.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Source object doesn't exist." - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the property exists or that it is a class?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "x-serviceIds": [ - "weaviate.local.manipulate" - ] - }, - "delete": { - "description": "Delete the single reference that is given in the body from the list of references that this property of a data object has", - "tags": [ - "objects" - ], - "summary": "Delete the single reference that is given in the body from the list of references that this property has.", - "operationId": "objects.class.references.delete", - "parameters": [ - { - "type": "string", - "description": "The class name as defined in the schema", - "name": "className", - "in": "path", - "required": true - }, - { - "type": "string", - "format": "uuid", - "description": "Unique ID of the Object.", - "name": "id", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "Unique name of the property related to the Object.", - "name": "propertyName", - "in": "path", - "required": true - }, - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/SingleRef" - } - }, - { - "$ref": "#/parameters/CommonConsistencyLevelParameterQuery" - }, - { - "$ref": "#/parameters/CommonTenantParameterQuery" - } - ], - "responses": { - "204": { - "description": "Successfully deleted." - }, - "400": { - "description": "Malformed request.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Successful query result but no resource was found.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the property exists or that it is a class?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "x-serviceIds": [ - "weaviate.local.manipulate" - ] - } - }, - "/objects/{id}": { - "get": { - "description": "Lists Objects.", - "tags": [ - "objects" - ], - "summary": "Get a specific Object based on its UUID and a Object UUID. Also available as Websocket bus.", - "operationId": "objects.get", - "deprecated": true, - "parameters": [ - { - "type": "string", - "format": "uuid", - "description": "Unique ID of the Object.", - "name": "id", - "in": "path", - "required": true - }, - { - "$ref": "#/parameters/CommonIncludeParameterQuery" - } - ], - "responses": { - "200": { - "description": "Successful response.", - "schema": { - "$ref": "#/definitions/Object" - } - }, - "400": { - "description": "Malformed request.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Successful query result but no resource was found." - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "x-serviceIds": [ - "weaviate.local.query" - ] - }, - "put": { - "description": "Updates an Object's data. Given meta-data and schema values are validated. LastUpdateTime is set to the time this function is called.", - "tags": [ - "objects" - ], - "summary": "Update an Object based on its UUID.", - "operationId": "objects.update", - "deprecated": true, - "parameters": [ - { - "type": "string", - "format": "uuid", - "description": "Unique ID of the Object.", - "name": "id", - "in": "path", - "required": true - }, - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/Object" - } - }, - { - "$ref": "#/parameters/CommonConsistencyLevelParameterQuery" - } - ], - "responses": { - "200": { - "description": "Successfully received.", - "schema": { - "$ref": "#/definitions/Object" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Successful query result but no resource was found." - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "x-serviceIds": [ - "weaviate.local.manipulate" - ] - }, - "delete": { - "description": "Deletes an Object from the system.", - "tags": [ - "objects" - ], - "summary": "Delete an Object based on its UUID.", - "operationId": "objects.delete", - "deprecated": true, - "parameters": [ - { - "type": "string", - "format": "uuid", - "description": "Unique ID of the Object.", - "name": "id", - "in": "path", - "required": true - }, - { - "$ref": "#/parameters/CommonConsistencyLevelParameterQuery" - }, - { - "$ref": "#/parameters/CommonTenantParameterQuery" - } - ], - "responses": { - "204": { - "description": "Successfully deleted." - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Successful query result but no resource was found." - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": true, - "x-available-in-websocket": true, - "x-serviceIds": [ - "weaviate.local.manipulate" - ] - }, - "head": { - "description": "Checks if an Object exists in the system.", - "tags": [ - "objects" - ], - "summary": "Checks Object's existence based on its UUID.", - "operationId": "objects.head", - "deprecated": true, - "parameters": [ - { - "type": "string", - "format": "uuid", - "description": "Unique ID of the Object.", - "name": "id", - "in": "path", - "required": true - } - ], - "responses": { - "204": { - "description": "Object exists." - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Object doesn't exist." - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": true, - "x-available-in-websocket": true, - "x-serviceIds": [ - "weaviate.local.manipulate" - ] - }, - "patch": { - "description": "Updates an Object. This method supports json-merge style patch semantics (RFC 7396). Provided meta-data and schema values are validated. LastUpdateTime is set to the time this function is called.", - "tags": [ - "objects" - ], - "summary": "Update an Object based on its UUID (using patch semantics).", - "operationId": "objects.patch", - "deprecated": true, - "parameters": [ - { - "type": "string", - "format": "uuid", - "description": "Unique ID of the Object.", - "name": "id", - "in": "path", - "required": true - }, - { - "description": "RFC 7396-style patch, the body contains the object to merge into the existing object.", - "name": "body", - "in": "body", - "schema": { - "$ref": "#/definitions/Object" - } - }, - { - "$ref": "#/parameters/CommonConsistencyLevelParameterQuery" - } - ], - "responses": { - "204": { - "description": "Successfully applied. No content provided." - }, - "400": { - "description": "The patch-JSON is malformed." - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Successful query result but no resource was found." - }, - "422": { - "description": "The patch-JSON is valid but unprocessable.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "x-serviceIds": [ - "weaviate.local.manipulate" - ] - } - }, - "/objects/{id}/references/{propertyName}": { - "put": { - "description": "Replace all references to a class-property.", - "tags": [ - "objects" - ], - "summary": "Replace all references to a class-property.", - "operationId": "objects.references.update", - "deprecated": true, - "parameters": [ - { - "type": "string", - "format": "uuid", - "description": "Unique ID of the Object.", - "name": "id", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "Unique name of the property related to the Object.", - "name": "propertyName", - "in": "path", - "required": true - }, - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/MultipleRef" - } - }, - { - "$ref": "#/parameters/CommonTenantParameterQuery" - } - ], - "responses": { - "200": { - "description": "Successfully replaced all the references." - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the property exists or that it is a class?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "x-serviceIds": [ - "weaviate.local.manipulate" - ] - }, - "post": { - "description": "Add a single reference to a class-property.", - "tags": [ - "objects" - ], - "summary": "Add a single reference to a class-property.", - "operationId": "objects.references.create", - "deprecated": true, - "parameters": [ - { - "type": "string", - "format": "uuid", - "description": "Unique ID of the Object.", - "name": "id", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "Unique name of the property related to the Object.", - "name": "propertyName", - "in": "path", - "required": true - }, - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/SingleRef" - } - }, - { - "$ref": "#/parameters/CommonTenantParameterQuery" - } - ], - "responses": { - "200": { - "description": "Successfully added the reference." - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the property exists or that it is a class?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "x-serviceIds": [ - "weaviate.local.manipulate" - ] - }, - "delete": { - "description": "Delete the single reference that is given in the body from the list of references that this property has.", - "tags": [ - "objects" - ], - "summary": "Delete the single reference that is given in the body from the list of references that this property has.", - "operationId": "objects.references.delete", - "deprecated": true, - "parameters": [ - { - "type": "string", - "format": "uuid", - "description": "Unique ID of the Object.", - "name": "id", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "Unique name of the property related to the Object.", - "name": "propertyName", - "in": "path", - "required": true - }, - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/SingleRef" - } - }, - { - "$ref": "#/parameters/CommonTenantParameterQuery" - } - ], - "responses": { - "204": { - "description": "Successfully deleted." - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Successful query result but no resource was found.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "x-serviceIds": [ - "weaviate.local.manipulate" - ] - } - }, - "/schema": { - "get": { - "tags": [ - "schema" - ], - "summary": "Dump the current the database schema.", - "operationId": "schema.dump", - "responses": { - "200": { - "description": "Successfully dumped the database schema.", - "schema": { - "$ref": "#/definitions/Schema" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-serviceIds": [ - "weaviate.local.query.meta" - ] - }, - "post": { - "tags": [ - "schema" - ], - "summary": "Create a new Object class in the schema.", - "operationId": "schema.objects.create", - "parameters": [ - { - "name": "objectClass", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/Class" - } - } - ], - "responses": { - "200": { - "description": "Added the new Object class to the schema.", - "schema": { - "$ref": "#/definitions/Class" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Invalid Object class", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-serviceIds": [ - "weaviate.local.add.meta" - ] - } - }, - "/schema/cluster-status": { - "get": { - "tags": [ - "schema" - ], - "operationId": "schema.cluster.status", - "responses": { - "200": { - "description": "The schema in the cluster is in sync.", - "schema": { - "$ref": "#/definitions/SchemaClusterStatus" - } - }, - "500": { - "description": "The schema is either out of sync (see response body) or the sync check could not be completed.", - "schema": { - "$ref": "#/definitions/SchemaClusterStatus" - } - } - } - } - }, - "/schema/{className}": { - "get": { - "tags": [ - "schema" - ], - "summary": "Get a single class from the schema", - "operationId": "schema.objects.get", - "parameters": [ - { - "type": "string", - "name": "className", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "Found the Class, returned as body", - "schema": { - "$ref": "#/definitions/Class" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "This class does not exist" - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-serviceIds": [ - "weaviate.local.get.meta" - ] - }, - "put": { - "description": "Use this endpoint to alter an existing class in the schema. Note that not all settings are mutable. If an error about immutable fields is returned and you still need to update this particular setting, you will have to delete the class (and the underlying data) and recreate. This endpoint cannot be used to modify properties. Instead use POST /v1/schema/{className}/properties. A typical use case for this endpoint is to update configuration, such as the vectorIndexConfig. Note that even in mutable sections, such as vectorIndexConfig, some fields may be immutable.", - "tags": [ - "schema" - ], - "summary": "Update settings of an existing schema class", - "operationId": "schema.objects.update", - "parameters": [ - { - "type": "string", - "name": "className", - "in": "path", - "required": true - }, - { - "name": "objectClass", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/Class" - } - } - ], - "responses": { - "200": { - "description": "Class was updated successfully", - "schema": { - "$ref": "#/definitions/Class" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Class to be updated does not exist", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Invalid update attempt", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-serviceIds": [ - "weaviate.local.manipulate.meta" - ] - }, - "delete": { - "tags": [ - "schema" - ], - "summary": "Remove an Object class (and all data in the instances) from the schema.", - "operationId": "schema.objects.delete", - "parameters": [ - { - "type": "string", - "name": "className", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "Removed the Object class from the schema." - }, - "400": { - "description": "Could not delete the Object class.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-serviceIds": [ - "weaviate.local.manipulate.meta" - ] - } - }, - "/schema/{className}/properties": { - "post": { - "tags": [ - "schema" - ], - "summary": "Add a property to an Object class.", - "operationId": "schema.objects.properties.add", - "parameters": [ - { - "type": "string", - "name": "className", - "in": "path", - "required": true - }, - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/Property" - } - } - ], - "responses": { - "200": { - "description": "Added the property.", - "schema": { - "$ref": "#/definitions/Property" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Invalid property.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-serviceIds": [ - "weaviate.local.manipulate.meta" - ] - } - }, - "/schema/{className}/shards": { - "get": { - "tags": [ - "schema" - ], - "summary": "Get the shards status of an Object class", - "operationId": "schema.objects.shards.get", - "parameters": [ - { - "type": "string", - "name": "className", - "in": "path", - "required": true - }, - { - "type": "string", - "name": "tenant", - "in": "query" - } - ], - "responses": { - "200": { - "description": "Found the status of the shards, returned as body", - "schema": { - "$ref": "#/definitions/ShardStatusList" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "This class does not exist", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-serviceIds": [ - "weaviate.local.get.meta" - ] - } - }, - "/schema/{className}/shards/{shardName}": { - "put": { - "description": "Update shard status of an Object Class", - "tags": [ - "schema" - ], - "operationId": "schema.objects.shards.update", - "parameters": [ - { - "type": "string", - "name": "className", - "in": "path", - "required": true - }, - { - "type": "string", - "name": "shardName", - "in": "path", - "required": true - }, - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/ShardStatus" - } - } - ], - "responses": { - "200": { - "description": "Shard status was updated successfully", - "schema": { - "$ref": "#/definitions/ShardStatus" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Shard to be updated does not exist", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Invalid update attempt", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-serviceIds": [ - "weaviate.local.manipulate.meta" - ] - } - }, - "/schema/{className}/tenants": { - "get": { - "description": "get all tenants from a specific class", - "tags": [ - "schema" - ], - "operationId": "tenants.get", - "parameters": [ - { - "type": "string", - "name": "className", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "tenants from specified class.", - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/Tenant" - } - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Invalid Tenant class", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - } - }, - "put": { - "description": "Update tenant of a specific class", - "tags": [ - "schema" - ], - "operationId": "tenants.update", - "parameters": [ - { - "type": "string", - "name": "className", - "in": "path", - "required": true - }, - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/Tenant" - } - } - } - ], - "responses": { - "200": { - "description": "Updated tenants of the specified class", - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/Tenant" - } - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Invalid Tenant class", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - } - }, - "post": { - "description": "Create a new tenant for a specific class", - "tags": [ - "schema" - ], - "operationId": "tenants.create", - "parameters": [ - { - "type": "string", - "name": "className", - "in": "path", - "required": true - }, - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/Tenant" - } - } - } - ], - "responses": { - "200": { - "description": "Added new tenants to the specified class", - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/Tenant" - } - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Invalid Tenant class", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - } - }, - "delete": { - "description": "delete tenants from a specific class", - "tags": [ - "schema" - ], - "operationId": "tenants.delete", - "parameters": [ - { - "type": "string", - "name": "className", - "in": "path", - "required": true - }, - { - "name": "tenants", - "in": "body", - "required": true, - "schema": { - "type": "array", - "items": { - "type": "string" - } - } - } - ], - "responses": { - "200": { - "description": "Deleted tenants from specified class." - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Invalid Tenant class", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - } - } - } - }, - "definitions": { - "AdditionalProperties": { - "description": "Additional Meta information about a single object object.", - "type": "object", - "additionalProperties": { - "type": "object" - } - }, - "BM25Config": { - "description": "tuning parameters for the BM25 algorithm", - "type": "object", - "properties": { - "b": { - "description": "calibrates term-weight scaling based on the document length", - "type": "number", - "format": "float" - }, - "k1": { - "description": "calibrates term-weight scaling based on the term frequency within a document", - "type": "number", - "format": "float" - } - } - }, - "BackupConfig": { - "description": "Backup custom configuration", - "type": "object", - "properties": { - "CPUPercentage": { - "description": "Desired CPU core utilization ranging from 1%-80%", - "type": "integer", - "default": 50, - "maximum": 80, - "minimum": 1, - "x-nullable": false - }, - "ChunkSize": { - "description": "Weaviate will attempt to come close the specified size, with a minimum of 2MB, default of 128MB, and a maximum of 512MB", - "type": "integer", - "default": 128, - "maximum": 512, - "minimum": 2, - "x-nullable": false - }, - "CompressionLevel": { - "description": "compression level used by compression algorithm", - "type": "string", - "default": "DefaultCompression", - "enum": [ - "DefaultCompression", - "BestSpeed", - "BestCompression" - ], - "x-nullable": false - } - } - }, - "BackupCreateRequest": { - "description": "Request body for creating a backup of a set of classes", - "properties": { - "config": { - "description": "Custom configuration for the backup creation process", - "type": "object", - "$ref": "#/definitions/BackupConfig" - }, - "exclude": { - "description": "List of classes to exclude from the backup creation process", - "type": "array", - "items": { - "type": "string" - } - }, - "id": { - "description": "The ID of the backup. Must be URL-safe and work as a filesystem path, only lowercase, numbers, underscore, minus characters allowed.", - "type": "string" - }, - "include": { - "description": "List of classes to include in the backup creation process", - "type": "array", - "items": { - "type": "string" - } - } - } - }, - "BackupCreateResponse": { - "description": "The definition of a backup create response body", - "properties": { - "backend": { - "description": "Backup backend name e.g. filesystem, gcs, s3.", - "type": "string" - }, - "classes": { - "description": "The list of classes for which the backup creation process was started", - "type": "array", - "items": { - "type": "string" - } - }, - "error": { - "description": "error message if creation failed", - "type": "string" - }, - "id": { - "description": "The ID of the backup. Must be URL-safe and work as a filesystem path, only lowercase, numbers, underscore, minus characters allowed.", - "type": "string" - }, - "path": { - "description": "destination path of backup files proper to selected backend", - "type": "string" - }, - "status": { - "description": "phase of backup creation process", - "type": "string", - "default": "STARTED", - "enum": [ - "STARTED", - "TRANSFERRING", - "TRANSFERRED", - "SUCCESS", - "FAILED" - ] - } - } - }, - "BackupCreateStatusResponse": { - "description": "The definition of a backup create metadata", - "properties": { - "backend": { - "description": "Backup backend name e.g. filesystem, gcs, s3.", - "type": "string" - }, - "error": { - "description": "error message if creation failed", - "type": "string" - }, - "id": { - "description": "The ID of the backup. Must be URL-safe and work as a filesystem path, only lowercase, numbers, underscore, minus characters allowed.", - "type": "string" - }, - "path": { - "description": "destination path of backup files proper to selected backend", - "type": "string" - }, - "status": { - "description": "phase of backup creation process", - "type": "string", - "default": "STARTED", - "enum": [ - "STARTED", - "TRANSFERRING", - "TRANSFERRED", - "SUCCESS", - "FAILED" - ] - } - } - }, - "BackupRestoreRequest": { - "description": "Request body for restoring a backup for a set of classes", - "properties": { - "config": { - "description": "Custom configuration for the backup restoration process", - "type": "object", - "$ref": "#/definitions/RestoreConfig" - }, - "exclude": { - "description": "List of classes to exclude from the backup restoration process", - "type": "array", - "items": { - "type": "string" - } - }, - "include": { - "description": "List of classes to include in the backup restoration process", - "type": "array", - "items": { - "type": "string" - } - }, - "node_mapping": { - "description": "Allows overriding the node names stored in the backup with different ones. Useful when restoring backups to a different environment.", - "type": "object", - "additionalProperties": { - "type": "string" - } - } - } - }, - "BackupRestoreResponse": { - "description": "The definition of a backup restore response body", - "properties": { - "backend": { - "description": "Backup backend name e.g. filesystem, gcs, s3.", - "type": "string" - }, - "classes": { - "description": "The list of classes for which the backup restoration process was started", - "type": "array", - "items": { - "type": "string" - } - }, - "error": { - "description": "error message if restoration failed", - "type": "string" - }, - "id": { - "description": "The ID of the backup. Must be URL-safe and work as a filesystem path, only lowercase, numbers, underscore, minus characters allowed.", - "type": "string" - }, - "path": { - "description": "destination path of backup files proper to selected backend", - "type": "string" - }, - "status": { - "description": "phase of backup restoration process", - "type": "string", - "default": "STARTED", - "enum": [ - "STARTED", - "TRANSFERRING", - "TRANSFERRED", - "SUCCESS", - "FAILED" - ] - } - } - }, - "BackupRestoreStatusResponse": { - "description": "The definition of a backup restore metadata", - "properties": { - "backend": { - "description": "Backup backend name e.g. filesystem, gcs, s3.", - "type": "string" - }, - "error": { - "description": "error message if restoration failed", - "type": "string" - }, - "id": { - "description": "The ID of the backup. Must be URL-safe and work as a filesystem path, only lowercase, numbers, underscore, minus characters allowed.", - "type": "string" - }, - "path": { - "description": "destination path of backup files proper to selected backup backend", - "type": "string" - }, - "status": { - "description": "phase of backup restoration process", - "type": "string", - "default": "STARTED", - "enum": [ - "STARTED", - "TRANSFERRING", - "TRANSFERRED", - "SUCCESS", - "FAILED" - ] - } - } - }, - "BatchDelete": { - "type": "object", - "properties": { - "dryRun": { - "description": "If true, objects will not be deleted yet, but merely listed. Defaults to false.", - "type": "boolean", - "default": false - }, - "match": { - "description": "Outlines how to find the objects to be deleted.", - "type": "object", - "properties": { - "class": { - "description": "Class (name) which objects will be deleted.", - "type": "string", - "example": "City" - }, - "where": { - "description": "Filter to limit the objects to be deleted.", - "type": "object", - "$ref": "#/definitions/WhereFilter" - } - } - }, - "output": { - "description": "Controls the verbosity of the output, possible values are: \"minimal\", \"verbose\". Defaults to \"minimal\".", - "type": "string", - "default": "minimal" - } - } - }, - "BatchDeleteResponse": { - "description": "Delete Objects response.", - "type": "object", - "properties": { - "dryRun": { - "description": "If true, objects will not be deleted yet, but merely listed. Defaults to false.", - "type": "boolean", - "default": false - }, - "match": { - "description": "Outlines how to find the objects to be deleted.", - "type": "object", - "properties": { - "class": { - "description": "Class (name) which objects will be deleted.", - "type": "string", - "example": "City" - }, - "where": { - "description": "Filter to limit the objects to be deleted.", - "type": "object", - "$ref": "#/definitions/WhereFilter" - } - } - }, - "output": { - "description": "Controls the verbosity of the output, possible values are: \"minimal\", \"verbose\". Defaults to \"minimal\".", - "type": "string", - "default": "minimal" - }, - "results": { - "type": "object", - "properties": { - "failed": { - "description": "How many objects should have been deleted but could not be deleted.", - "type": "number", - "format": "int64", - "x-omitempty": false - }, - "limit": { - "description": "The most amount of objects that can be deleted in a single query, equals QUERY_MAXIMUM_RESULTS.", - "type": "number", - "format": "int64", - "x-omitempty": false - }, - "matches": { - "description": "How many objects were matched by the filter.", - "type": "number", - "format": "int64", - "x-omitempty": false - }, - "objects": { - "description": "With output set to \"minimal\" only objects with error occurred will the be described. Successfully deleted objects would be omitted. Output set to \"verbose\" will list all of the objets with their respective statuses.", - "type": "array", - "items": { - "description": "Results for this specific Object.", - "format": "object", - "properties": { - "errors": { - "$ref": "#/definitions/ErrorResponse" - }, - "id": { - "description": "ID of the Object.", - "type": "string", - "format": "uuid" - }, - "status": { - "type": "string", - "default": "SUCCESS", - "enum": [ - "SUCCESS", - "DRYRUN", - "FAILED" - ] - } - } - } - }, - "successful": { - "description": "How many objects were successfully deleted in this round.", - "type": "number", - "format": "int64", - "x-omitempty": false - } - } - } - } - }, - "BatchReference": { - "properties": { - "from": { - "description": "Long-form beacon-style URI to identify the source of the cross-ref including the property name. Should be in the form of weaviate://localhost/\u003ckinds\u003e/\u003cuuid\u003e/\u003cclassName\u003e/\u003cpropertyName\u003e, where \u003ckinds\u003e must be one of 'objects', 'objects' and \u003cclassName\u003e and \u003cpropertyName\u003e must represent the cross-ref property of source class to be used.", - "type": "string", - "format": "uri", - "example": "weaviate://localhost/Zoo/a5d09582-4239-4702-81c9-92a6e0122bb4/hasAnimals" - }, - "tenant": { - "description": "Name of the reference tenant.", - "type": "string" - }, - "to": { - "description": "Short-form URI to point to the cross-ref. Should be in the form of weaviate://localhost/\u003cuuid\u003e for the example of a local cross-ref to an object", - "type": "string", - "format": "uri", - "example": "weaviate://localhost/97525810-a9a5-4eb0-858a-71449aeb007f" - } - } - }, - "BatchReferenceResponse": { - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/BatchReference" - }, - { - "properties": { - "result": { - "description": "Results for this specific reference.", - "format": "object", - "properties": { - "errors": { - "$ref": "#/definitions/ErrorResponse" - }, - "status": { - "type": "string", - "default": "SUCCESS", - "enum": [ - "SUCCESS", - "PENDING", - "FAILED" - ] - } - } - } - } - } - ] - }, - "BatchStats": { - "description": "The summary of a nodes batch queue congestion status.", - "properties": { - "queueLength": { - "description": "How many objects are currently in the batch queue.", - "type": "number", - "format": "int", - "x-nullable": true, - "x-omitempty": true - }, - "ratePerSecond": { - "description": "How many objects are approximately processed from the batch queue per second.", - "type": "number", - "format": "int", - "x-omitempty": false - } - } - }, - "C11yExtension": { - "description": "A resource describing an extension to the contextinoary, containing both the identifier and the definition of the extension", - "properties": { - "concept": { - "description": "The new concept you want to extend. Must be an all-lowercase single word, or a space delimited compound word. Examples: 'foobarium', 'my custom concept'", - "type": "string", - "example": "foobarium" - }, - "definition": { - "description": "A list of space-delimited words or a sentence describing what the custom concept is about. Avoid using the custom concept itself. An Example definition for the custom concept 'foobarium': would be 'a naturally occurring element which can only be seen by programmers'", - "type": "string" - }, - "weight": { - "description": "Weight of the definition of the new concept where 1='override existing definition entirely' and 0='ignore custom definition'. Note that if the custom concept is not present in the contextionary yet, the weight cannot be less than 1.", - "type": "number", - "format": "float" - } - } - }, - "C11yNearestNeighbors": { - "description": "C11y function to show the nearest neighbors to a word.", - "type": "array", - "items": { - "type": "object", - "properties": { - "distance": { - "type": "number", - "format": "float" - }, - "word": { - "type": "string" - } - } - } - }, - "C11yVector": { - "description": "A Vector in the Contextionary", - "type": "array", - "items": { - "type": "number", - "format": "float" - } - }, - "C11yVectorBasedQuestion": { - "description": "Receive question based on array of classes, properties and values.", - "type": "array", - "items": { - "type": "object", - "properties": { - "classProps": { - "description": "Vectorized properties.", - "type": "array", - "maxItems": 300, - "minItems": 300, - "items": { - "type": "object", - "properties": { - "propsVectors": { - "type": "array", - "items": { - "type": "number", - "format": "float" - } - }, - "value": { - "description": "String with valuename.", - "type": "string" - } - } - } - }, - "classVectors": { - "description": "Vectorized classname.", - "type": "array", - "maxItems": 300, - "minItems": 300, - "items": { - "type": "number", - "format": "float" - } - } - } - } - }, - "C11yWordsResponse": { - "description": "An array of available words and contexts.", - "properties": { - "concatenatedWord": { - "description": "Weighted results for all words", - "type": "object", - "properties": { - "concatenatedNearestNeighbors": { - "$ref": "#/definitions/C11yNearestNeighbors" - }, - "concatenatedVector": { - "$ref": "#/definitions/C11yVector" - }, - "concatenatedWord": { - "type": "string" - }, - "singleWords": { - "type": "array", - "items": { - "format": "string" - } - } - } - }, - "individualWords": { - "description": "Weighted results for per individual word", - "type": "array", - "items": { - "type": "object", - "properties": { - "info": { - "type": "object", - "properties": { - "nearestNeighbors": { - "$ref": "#/definitions/C11yNearestNeighbors" - }, - "vector": { - "$ref": "#/definitions/C11yVector" - } - } - }, - "present": { - "type": "boolean" - }, - "word": { - "type": "string" - } - } - } - } - } - }, - "Class": { - "type": "object", - "properties": { - "class": { - "description": "Name of the class as URI relative to the schema URL.", - "type": "string" - }, - "description": { - "description": "Description of the class.", - "type": "string" - }, - "invertedIndexConfig": { - "$ref": "#/definitions/InvertedIndexConfig" - }, - "moduleConfig": { - "description": "Configuration specific to modules this Weaviate instance has installed", - "type": "object" - }, - "multiTenancyConfig": { - "$ref": "#/definitions/MultiTenancyConfig" - }, - "properties": { - "description": "The properties of the class.", - "type": "array", - "items": { - "$ref": "#/definitions/Property" - } - }, - "replicationConfig": { - "$ref": "#/definitions/ReplicationConfig" - }, - "shardingConfig": { - "description": "Manage how the index should be sharded and distributed in the cluster", - "type": "object" - }, - "vectorIndexConfig": { - "description": "Vector-index config, that is specific to the type of index selected in vectorIndexType", - "type": "object" - }, - "vectorIndexType": { - "description": "Name of the vector index to use, eg. (HNSW)", - "type": "string" - }, - "vectorizer": { - "description": "Specify how the vectors for this class should be determined. The options are either 'none' - this means you have to import a vector with each object yourself - or the name of a module that provides vectorization capabilities, such as 'text2vec-contextionary'. If left empty, it will use the globally configured default which can itself either be 'none' or a specific module.", - "type": "string" - } - } - }, - "Classification": { - "description": "Manage classifications, trigger them and view status of past classifications.", - "type": "object", - "properties": { - "basedOnProperties": { - "description": "base the text-based classification on these fields (of type text)", - "type": "array", - "items": { - "type": "string" - }, - "example": [ - "description" - ] - }, - "class": { - "description": "class (name) which is used in this classification", - "type": "string", - "example": "City" - }, - "classifyProperties": { - "description": "which ref-property to set as part of the classification", - "type": "array", - "items": { - "type": "string" - }, - "example": [ - "inCountry" - ] - }, - "error": { - "description": "error message if status == failed", - "type": "string", - "default": "", - "example": "classify xzy: something went wrong" - }, - "filters": { - "type": "object", - "properties": { - "sourceWhere": { - "description": "limit the objects to be classified", - "type": "object", - "$ref": "#/definitions/WhereFilter" - }, - "targetWhere": { - "description": "Limit the possible sources when using an algorithm which doesn't really on training data, e.g. 'contextual'. When using an algorithm with a training set, such as 'knn', limit the training set instead", - "type": "object", - "$ref": "#/definitions/WhereFilter" - }, - "trainingSetWhere": { - "description": "Limit the training objects to be considered during the classification. Can only be used on types with explicit training sets, such as 'knn'", - "type": "object", - "$ref": "#/definitions/WhereFilter" - } - } - }, - "id": { - "description": "ID to uniquely identify this classification run", - "type": "string", - "format": "uuid", - "example": "ee722219-b8ec-4db1-8f8d-5150bb1a9e0c" - }, - "meta": { - "description": "additional meta information about the classification", - "type": "object", - "$ref": "#/definitions/ClassificationMeta" - }, - "settings": { - "description": "classification-type specific settings", - "type": "object" - }, - "status": { - "description": "status of this classification", - "type": "string", - "enum": [ - "running", - "completed", - "failed" - ], - "example": "running" - }, - "type": { - "description": "which algorithm to use for classifications", - "type": "string" - } - } - }, - "ClassificationMeta": { - "description": "Additional information to a specific classification", - "type": "object", - "properties": { - "completed": { - "description": "time when this classification finished", - "type": "string", - "format": "date-time", - "example": "2017-07-21T17:32:28Z" - }, - "count": { - "description": "number of objects which were taken into consideration for classification", - "type": "integer", - "example": 147 - }, - "countFailed": { - "description": "number of objects which could not be classified - see error message for details", - "type": "integer", - "example": 7 - }, - "countSucceeded": { - "description": "number of objects successfully classified", - "type": "integer", - "example": 140 - }, - "started": { - "description": "time when this classification was started", - "type": "string", - "format": "date-time", - "example": "2017-07-21T17:32:28Z" - } - } - }, - "Deprecation": { - "type": "object", - "properties": { - "apiType": { - "description": "Describes which API is effected, usually one of: REST, GraphQL", - "type": "string" - }, - "id": { - "description": "The id that uniquely identifies this particular deprecations (mostly used internally)", - "type": "string" - }, - "locations": { - "description": "The locations within the specified API affected by this deprecation", - "type": "array", - "items": { - "type": "string" - } - }, - "mitigation": { - "description": "User-required object to not be affected by the (planned) removal", - "type": "string" - }, - "msg": { - "description": "What this deprecation is about", - "type": "string" - }, - "plannedRemovalVersion": { - "description": "A best-effort guess of which upcoming version will remove the feature entirely", - "type": "string" - }, - "removedIn": { - "description": "If the feature has already been removed, it was removed in this version", - "type": "string", - "x-nullable": true - }, - "removedTime": { - "description": "If the feature has already been removed, it was removed at this timestamp", - "type": "string", - "format": "date-time", - "x-nullable": true - }, - "sinceTime": { - "description": "The deprecation was introduced in this version", - "type": "string", - "format": "date-time" - }, - "sinceVersion": { - "description": "The deprecation was introduced in this version", - "type": "string" - }, - "status": { - "description": "Whether the problematic API functionality is deprecated (planned to be removed) or already removed", - "type": "string" - } - } - }, - "ErrorResponse": { - "description": "An error response given by Weaviate end-points.", - "type": "object", - "properties": { - "error": { - "type": "array", - "items": { - "type": "object", - "properties": { - "message": { - "type": "string" - } - } - } - } - } - }, - "GeoCoordinates": { - "properties": { - "latitude": { - "description": "The latitude of the point on earth in decimal form", - "type": "number", - "format": "float", - "x-nullable": true - }, - "longitude": { - "description": "The longitude of the point on earth in decimal form", - "type": "number", - "format": "float", - "x-nullable": true - } - } - }, - "GraphQLError": { - "description": "An error response caused by a GraphQL query.", - "properties": { - "locations": { - "type": "array", - "items": { - "type": "object", - "properties": { - "column": { - "type": "integer", - "format": "int64" - }, - "line": { - "type": "integer", - "format": "int64" - } - } - } - }, - "message": { - "type": "string" - }, - "path": { - "type": "array", - "items": { - "type": "string" - } - } - } - }, - "GraphQLQueries": { - "description": "A list of GraphQL queries.", - "type": "array", - "items": { - "$ref": "#/definitions/GraphQLQuery" - } - }, - "GraphQLQuery": { - "description": "GraphQL query based on: http://facebook.github.io/graphql/.", - "type": "object", - "properties": { - "operationName": { - "description": "The name of the operation if multiple exist in the query.", - "type": "string" - }, - "query": { - "description": "Query based on GraphQL syntax.", - "type": "string" - }, - "variables": { - "description": "Additional variables for the query.", - "type": "object" - } - } - }, - "GraphQLResponse": { - "description": "GraphQL based response: http://facebook.github.io/graphql/.", - "properties": { - "data": { - "description": "GraphQL data object.", - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/JsonObject" - } - }, - "errors": { - "description": "Array with errors.", - "type": "array", - "items": { - "$ref": "#/definitions/GraphQLError" - }, - "x-omitempty": true - } - } - }, - "GraphQLResponses": { - "description": "A list of GraphQL responses.", - "type": "array", - "items": { - "$ref": "#/definitions/GraphQLResponse" - } - }, - "InvertedIndexConfig": { - "description": "Configure the inverted index built into Weaviate", - "type": "object", - "properties": { - "bm25": { - "$ref": "#/definitions/BM25Config" - }, - "cleanupIntervalSeconds": { - "description": "Asynchronous index clean up happens every n seconds", - "type": "number", - "format": "int" - }, - "indexNullState": { - "description": "Index each object with the null state", - "type": "boolean" - }, - "indexPropertyLength": { - "description": "Index length of properties", - "type": "boolean" - }, - "indexTimestamps": { - "description": "Index each object by its internal timestamps", - "type": "boolean" - }, - "stopwords": { - "$ref": "#/definitions/StopwordConfig" - } - } - }, - "JsonObject": { - "description": "JSON object value.", - "type": "object" - }, - "Link": { - "type": "object", - "properties": { - "documentationHref": { - "description": "weaviate documentation about this resource group", - "type": "string" - }, - "href": { - "description": "target of the link", - "type": "string" - }, - "name": { - "description": "human readable name of the resource group", - "type": "string" - }, - "rel": { - "description": "relationship if both resources are related, e.g. 'next', 'previous', 'parent', etc.", - "type": "string" - } - } - }, - "Meta": { - "description": "Contains meta information of the current Weaviate instance.", - "type": "object", - "properties": { - "hostname": { - "description": "The url of the host.", - "type": "string", - "format": "url" - }, - "modules": { - "description": "Module-specific meta information", - "type": "object" - }, - "version": { - "description": "Version of weaviate you are currently running", - "type": "string" - } - } - }, - "MultiTenancyConfig": { - "description": "Configuration related to multi-tenancy within a class", - "properties": { - "enabled": { - "description": "Whether or not multi-tenancy is enabled for this class", - "type": "boolean", - "x-omitempty": false - } - } - }, - "MultipleRef": { - "description": "Multiple instances of references to other objects.", - "type": "array", - "items": { - "$ref": "#/definitions/SingleRef" - } - }, - "NestedProperty": { - "type": "object", - "properties": { - "dataType": { - "type": "array", - "items": { - "type": "string" - } - }, - "description": { - "type": "string" - }, - "indexFilterable": { - "type": "boolean", - "x-nullable": true - }, - "indexSearchable": { - "type": "boolean", - "x-nullable": true - }, - "name": { - "type": "string" - }, - "nestedProperties": { - "type": "array", - "items": { - "$ref": "#/definitions/NestedProperty" - }, - "x-omitempty": true - }, - "tokenization": { - "type": "string", - "enum": [ - "word", - "lowercase", - "whitespace", - "field" - ] - } - } - }, - "NodeShardStatus": { - "description": "The definition of a node shard status response body", - "properties": { - "class": { - "description": "The name of shard's class.", - "type": "string", - "x-omitempty": false - }, - "compressed": { - "description": "The status of vector compression/quantization.", - "format": "boolean", - "x-omitempty": false - }, - "name": { - "description": "The name of the shard.", - "type": "string", - "x-omitempty": false - }, - "objectCount": { - "description": "The number of objects in shard.", - "type": "number", - "format": "int64", - "x-omitempty": false - }, - "vectorIndexingStatus": { - "description": "The status of the vector indexing process.", - "format": "string", - "x-omitempty": false - }, - "vectorQueueLength": { - "description": "The length of the vector indexing queue.", - "type": "number", - "format": "int64", - "x-omitempty": false - } - } - }, - "NodeStats": { - "description": "The summary of Weaviate's statistics.", - "properties": { - "objectCount": { - "description": "The total number of objects in DB.", - "type": "number", - "format": "int64", - "x-omitempty": false - }, - "shardCount": { - "description": "The count of Weaviate's shards.", - "type": "number", - "format": "int", - "x-omitempty": false - } - } - }, - "NodeStatus": { - "description": "The definition of a backup node status response body", - "properties": { - "batchStats": { - "description": "Weaviate batch statistics.", - "type": "object", - "$ref": "#/definitions/BatchStats" - }, - "gitHash": { - "description": "The gitHash of Weaviate.", - "type": "string" - }, - "name": { - "description": "The name of the node.", - "type": "string" - }, - "shards": { - "description": "The list of the shards with it's statistics.", - "type": "array", - "items": { - "$ref": "#/definitions/NodeShardStatus" - } - }, - "stats": { - "description": "Weaviate overall statistics.", - "type": "object", - "$ref": "#/definitions/NodeStats" - }, - "status": { - "description": "Node's status.", - "type": "string", - "default": "HEALTHY", - "enum": [ - "HEALTHY", - "UNHEALTHY", - "UNAVAILABLE" - ] - }, - "version": { - "description": "The version of Weaviate.", - "type": "string" - } - } - }, - "NodesStatusResponse": { - "description": "The status of all of the Weaviate nodes", - "type": "object", - "properties": { - "nodes": { - "type": "array", - "items": { - "$ref": "#/definitions/NodeStatus" - } - } - } - }, - "Object": { - "type": "object", - "properties": { - "additional": { - "$ref": "#/definitions/AdditionalProperties" - }, - "class": { - "description": "Class of the Object, defined in the schema.", - "type": "string" - }, - "creationTimeUnix": { - "description": "Timestamp of creation of this Object in milliseconds since epoch UTC.", - "type": "integer", - "format": "int64" - }, - "id": { - "description": "ID of the Object.", - "type": "string", - "format": "uuid" - }, - "lastUpdateTimeUnix": { - "description": "Timestamp of the last Object update in milliseconds since epoch UTC.", - "type": "integer", - "format": "int64" - }, - "properties": { - "$ref": "#/definitions/PropertySchema" - }, - "tenant": { - "description": "Name of the Objects tenant.", - "type": "string" - }, - "vector": { - "description": "This object's position in the Contextionary vector space. Read-only if using a vectorizer other than 'none'. Writable and required if using 'none' as vectorizer.", - "$ref": "#/definitions/C11yVector" - }, - "vectorWeights": { - "$ref": "#/definitions/VectorWeights" - } - } - }, - "ObjectsGetResponse": { - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Object" - }, - { - "properties": { - "deprecations": { - "type": "array", - "items": { - "$ref": "#/definitions/Deprecation" - } - } - } - }, - { - "properties": { - "result": { - "description": "Results for this specific Object.", - "format": "object", - "properties": { - "errors": { - "$ref": "#/definitions/ErrorResponse" - }, - "status": { - "type": "string", - "default": "SUCCESS", - "enum": [ - "SUCCESS", - "PENDING", - "FAILED" - ] - } - } - } - } - } - ] - }, - "ObjectsListResponse": { - "description": "List of Objects.", - "type": "object", - "properties": { - "deprecations": { - "type": "array", - "items": { - "$ref": "#/definitions/Deprecation" - } - }, - "objects": { - "description": "The actual list of Objects.", - "type": "array", - "items": { - "$ref": "#/definitions/Object" - } - }, - "totalResults": { - "description": "The total number of Objects for the query. The number of items in a response may be smaller due to paging.", - "type": "integer", - "format": "int64" - } - } - }, - "PatchDocumentAction": { - "description": "Either a JSONPatch document as defined by RFC 6902 (from, op, path, value), or a merge document (RFC 7396).", - "required": [ - "op", - "path" - ], - "properties": { - "from": { - "description": "A string containing a JSON Pointer value.", - "type": "string" - }, - "merge": { - "$ref": "#/definitions/Object" - }, - "op": { - "description": "The operation to be performed.", - "type": "string", - "enum": [ - "add", - "remove", - "replace", - "move", - "copy", - "test" - ] - }, - "path": { - "description": "A JSON-Pointer.", - "type": "string" - }, - "value": { - "description": "The value to be used within the operations.", - "type": "object" - } - } - }, - "PatchDocumentObject": { - "description": "Either a JSONPatch document as defined by RFC 6902 (from, op, path, value), or a merge document (RFC 7396).", - "required": [ - "op", - "path" - ], - "properties": { - "from": { - "description": "A string containing a JSON Pointer value.", - "type": "string" - }, - "merge": { - "$ref": "#/definitions/Object" - }, - "op": { - "description": "The operation to be performed.", - "type": "string", - "enum": [ - "add", - "remove", - "replace", - "move", - "copy", - "test" - ] - }, - "path": { - "description": "A JSON-Pointer.", - "type": "string" - }, - "value": { - "description": "The value to be used within the operations.", - "type": "object" - } - } - }, - "PeerUpdate": { - "description": "A single peer in the network.", - "properties": { - "id": { - "description": "The session ID of the peer.", - "type": "string", - "format": "uuid" - }, - "name": { - "description": "Human readable name.", - "type": "string" - }, - "schemaHash": { - "description": "The latest known hash of the peer's schema.", - "type": "string" - }, - "uri": { - "description": "The location where the peer is exposed to the internet.", - "type": "string", - "format": "uri" - } - } - }, - "PeerUpdateList": { - "description": "List of known peers.", - "type": "array", - "items": { - "$ref": "#/definitions/PeerUpdate" - } - }, - "PhoneNumber": { - "properties": { - "countryCode": { - "description": "Read-only. The numerical country code (e.g. 49)", - "type": "number", - "format": "uint64" - }, - "defaultCountry": { - "description": "Optional. The ISO 3166-1 alpha-2 country code. This is used to figure out the correct countryCode and international format if only a national number (e.g. 0123 4567) is provided", - "type": "string" - }, - "input": { - "description": "The raw input as the phone number is present in your raw data set. It will be parsed into the standardized formats if valid.", - "type": "string" - }, - "internationalFormatted": { - "description": "Read-only. Parsed result in the international format (e.g. +49 123 ...)", - "type": "string" - }, - "national": { - "description": "Read-only. The numerical representation of the national part", - "type": "number", - "format": "uint64" - }, - "nationalFormatted": { - "description": "Read-only. Parsed result in the national format (e.g. 0123 456789)", - "type": "string" - }, - "valid": { - "description": "Read-only. Indicates whether the parsed number is a valid phone number", - "type": "boolean" - } - } - }, - "Principal": { - "type": "object", - "properties": { - "groups": { - "type": "array", - "items": { - "type": "string" - } - }, - "username": { - "description": "The username that was extracted either from the authentication information", - "type": "string" - } - } - }, - "Property": { - "type": "object", - "properties": { - "dataType": { - "description": "Can be a reference to another type when it starts with a capital (for example Person), otherwise \"string\" or \"int\".", - "type": "array", - "items": { - "type": "string" - } - }, - "description": { - "description": "Description of the property.", - "type": "string" - }, - "indexFilterable": { - "description": "Optional. Should this property be indexed in the inverted index. Defaults to true. If you choose false, you will not be able to use this property in where filters. This property has no affect on vectorization decisions done by modules", - "type": "boolean", - "x-nullable": true - }, - "indexInverted": { - "description": "Optional. Should this property be indexed in the inverted index. Defaults to true. If you choose false, you will not be able to use this property in where filters, bm25 or hybrid search. This property has no affect on vectorization decisions done by modules (deprecated as of v1.19; use indexFilterable or/and indexSearchable instead)", - "type": "boolean", - "x-nullable": true - }, - "indexSearchable": { - "description": "Optional. Should this property be indexed in the inverted index. Defaults to true. Applicable only to properties of data type text and text[]. If you choose false, you will not be able to use this property in bm25 or hybrid search. This property has no affect on vectorization decisions done by modules", - "type": "boolean", - "x-nullable": true - }, - "moduleConfig": { - "description": "Configuration specific to modules this Weaviate instance has installed", - "type": "object" - }, - "name": { - "description": "Name of the property as URI relative to the schema URL.", - "type": "string" - }, - "nestedProperties": { - "description": "The properties of the nested object(s). Applies to object and object[] data types.", - "type": "array", - "items": { - "$ref": "#/definitions/NestedProperty" - }, - "x-omitempty": true - }, - "tokenization": { - "description": "Determines tokenization of the property as separate words or whole field. Optional. Applies to text and text[] data types. Allowed values are ` + "`" + `word` + "`" + ` (default; splits on any non-alphanumerical, lowercases), ` + "`" + `lowercase` + "`" + ` (splits on white spaces, lowercases), ` + "`" + `whitespace` + "`" + ` (splits on white spaces), ` + "`" + `field` + "`" + ` (trims). Not supported for remaining data types", - "type": "string", - "enum": [ - "word", - "lowercase", - "whitespace", - "field" - ] - } - } - }, - "PropertySchema": { - "description": "This is an open object, with OpenAPI Specification 3.0 this will be more detailed. See Weaviate docs for more info. In the future this will become a key/value OR a SingleRef definition.", - "type": "object" - }, - "ReferenceMetaClassification": { - "description": "This meta field contains additional info about the classified reference property", - "properties": { - "closestLosingDistance": { - "description": "The lowest distance of a neighbor in the losing group. Optional. If k equals the size of the winning group, there is no losing group", - "type": "number", - "format": "float32", - "x-nullable": true - }, - "closestOverallDistance": { - "description": "The lowest distance of any neighbor, regardless of whether they were in the winning or losing group", - "type": "number", - "format": "float32" - }, - "closestWinningDistance": { - "description": "Closest distance of a neighbor from the winning group", - "type": "number", - "format": "float32" - }, - "losingCount": { - "description": "size of the losing group, can be 0 if the winning group size equals k", - "type": "number", - "format": "int64" - }, - "losingDistance": { - "description": "deprecated - do not use, to be removed in 0.23.0", - "type": "number", - "format": "float32", - "x-nullable": true - }, - "meanLosingDistance": { - "description": "Mean distance of all neighbors from the losing group. Optional. If k equals the size of the winning group, there is no losing group.", - "type": "number", - "format": "float32", - "x-nullable": true - }, - "meanWinningDistance": { - "description": "Mean distance of all neighbors from the winning group", - "type": "number", - "format": "float32" - }, - "overallCount": { - "description": "overall neighbors checked as part of the classification. In most cases this will equal k, but could be lower than k - for example if not enough data was present", - "type": "number", - "format": "int64" - }, - "winningCount": { - "description": "size of the winning group, a number between 1..k", - "type": "number", - "format": "int64" - }, - "winningDistance": { - "description": "deprecated - do not use, to be removed in 0.23.0", - "type": "number", - "format": "float32" - } - } - }, - "ReplicationConfig": { - "description": "Configure how replication is executed in a cluster", - "type": "object", - "properties": { - "factor": { - "description": "Number of times a class is replicated", - "type": "integer" - } - } - }, - "RestoreConfig": { - "description": "Backup custom configuration", - "type": "object", - "properties": { - "CPUPercentage": { - "description": "Desired CPU core utilization ranging from 1%-80%", - "type": "integer", - "default": 50, - "maximum": 80, - "minimum": 1, - "x-nullable": false - } - } - }, - "Schema": { - "description": "Definitions of semantic schemas (also see: https://github.com/weaviate/weaviate-semantic-schemas).", - "type": "object", - "properties": { - "classes": { - "description": "Semantic classes that are available.", - "type": "array", - "items": { - "$ref": "#/definitions/Class" - } - }, - "maintainer": { - "description": "Email of the maintainer.", - "type": "string", - "format": "email" - }, - "name": { - "description": "Name of the schema.", - "type": "string" - } - } - }, - "SchemaClusterStatus": { - "description": "Indicates the health of the schema in a cluster.", - "type": "object", - "properties": { - "error": { - "description": "Contains the sync check error if one occurred", - "type": "string", - "x-omitempty": true - }, - "healthy": { - "description": "True if the cluster is in sync, false if there is an issue (see error).", - "type": "boolean", - "x-omitempty": false - }, - "hostname": { - "description": "Hostname of the coordinating node, i.e. the one that received the cluster. This can be useful information if the error message contains phrases such as 'other nodes agree, but local does not', etc.", - "type": "string" - }, - "ignoreSchemaSync": { - "description": "The cluster check at startup can be ignored (to recover from an out-of-sync situation).", - "type": "boolean", - "x-omitempty": false - }, - "nodeCount": { - "description": "Number of nodes that participated in the sync check", - "type": "number", - "format": "int" - } - } - }, - "SchemaHistory": { - "description": "This is an open object, with OpenAPI Specification 3.0 this will be more detailed. See Weaviate docs for more info. In the future this will become a key/value OR a SingleRef definition.", - "type": "object" - }, - "ShardStatus": { - "description": "The status of a single shard", - "properties": { - "status": { - "description": "Status of the shard", - "type": "string" - } - } - }, - "ShardStatusGetResponse": { - "description": "Response body of shard status get request", - "properties": { - "name": { - "description": "Name of the shard", - "type": "string" - }, - "status": { - "description": "Status of the shard", - "type": "string" - }, - "vectorQueueSize": { - "description": "Size of the vector queue of the shard", - "type": "integer", - "x-omitempty": false - } - } - }, - "ShardStatusList": { - "description": "The status of all the shards of a Class", - "type": "array", - "items": { - "$ref": "#/definitions/ShardStatusGetResponse" - } - }, - "SingleRef": { - "description": "Either set beacon (direct reference) or set class and schema (concept reference)", - "properties": { - "beacon": { - "description": "If using a direct reference, specify the URI to point to the cross-ref here. Should be in the form of weaviate://localhost/\u003cuuid\u003e for the example of a local cross-ref to an object", - "type": "string", - "format": "uri" - }, - "class": { - "description": "If using a concept reference (rather than a direct reference), specify the desired class name here", - "type": "string", - "format": "uri" - }, - "classification": { - "description": "Additional Meta information about classifications if the item was part of one", - "$ref": "#/definitions/ReferenceMetaClassification" - }, - "href": { - "description": "If using a direct reference, this read-only fields provides a link to the referenced resource. If 'origin' is globally configured, an absolute URI is shown - a relative URI otherwise.", - "type": "string", - "format": "uri" - }, - "schema": { - "description": "If using a concept reference (rather than a direct reference), specify the desired properties here", - "$ref": "#/definitions/PropertySchema" - } - } - }, - "StopwordConfig": { - "description": "fine-grained control over stopword list usage", - "type": "object", - "properties": { - "additions": { - "description": "stopwords to be considered additionally", - "type": "array", - "items": { - "type": "string" - } - }, - "preset": { - "description": "pre-existing list of common words by language", - "type": "string" - }, - "removals": { - "description": "stopwords to be removed from consideration", - "type": "array", - "items": { - "type": "string" - } - } - } - }, - "Tenant": { - "description": "attributes representing a single tenant within weaviate", - "type": "object", - "properties": { - "activityStatus": { - "description": "activity status of the tenant's shard. Optional for creating tenant (implicit ` + "`" + `HOT` + "`" + `) and required for updating tenant. Allowed values are ` + "`" + `HOT` + "`" + ` - tenant is fully active, ` + "`" + `WARM` + "`" + ` - tenant is active, some restrictions are imposed (TBD; not supported yet), ` + "`" + `COLD` + "`" + ` - tenant is inactive; no actions can be performed on tenant, tenant's files are stored locally, ` + "`" + `FROZEN` + "`" + ` - as COLD, but files are stored on cloud storage (not supported yet)", - "type": "string", - "enum": [ - "HOT", - "WARM", - "COLD", - "FROZEN" - ] - }, - "name": { - "description": "name of the tenant", - "type": "string" - } - } - }, - "VectorWeights": { - "description": "Allow custom overrides of vector weights as math expressions. E.g. \"pancake\": \"7\" will set the weight for the word pancake to 7 in the vectorization, whereas \"w * 3\" would triple the originally calculated word. This is an open object, with OpenAPI Specification 3.0 this will be more detailed. See Weaviate docs for more info. In the future this will become a key/value (string/string) object.", - "type": "object" - }, - "WhereFilter": { - "description": "Filter search results using a where filter", - "type": "object", - "properties": { - "operands": { - "description": "combine multiple where filters, requires 'And' or 'Or' operator", - "type": "array", - "items": { - "$ref": "#/definitions/WhereFilter" - } - }, - "operator": { - "description": "operator to use", - "type": "string", - "enum": [ - "And", - "Or", - "Equal", - "Like", - "NotEqual", - "GreaterThan", - "GreaterThanEqual", - "LessThan", - "LessThanEqual", - "WithinGeoRange", - "IsNull", - "ContainsAny", - "ContainsAll" - ], - "example": "GreaterThanEqual" - }, - "path": { - "description": "path to the property currently being filtered", - "type": "array", - "items": { - "type": "string" - }, - "example": [ - "inCity", - "City", - "name" - ] - }, - "valueBoolean": { - "description": "value as boolean", - "type": "boolean", - "x-nullable": true, - "example": false - }, - "valueBooleanArray": { - "description": "value as boolean", - "type": "array", - "items": { - "type": "boolean" - }, - "x-nullable": true, - "x-omitempty": true, - "example": [ - true, - false - ] - }, - "valueDate": { - "description": "value as date (as string)", - "type": "string", - "x-nullable": true, - "example": "TODO" - }, - "valueDateArray": { - "description": "value as date (as string)", - "type": "array", - "items": { - "type": "string" - }, - "x-nullable": true, - "x-omitempty": true, - "example": "TODO" - }, - "valueGeoRange": { - "description": "value as geo coordinates and distance", - "type": "object", - "x-nullable": true, - "$ref": "#/definitions/WhereFilterGeoRange" - }, - "valueInt": { - "description": "value as integer", - "type": "integer", - "format": "int64", - "x-nullable": true, - "example": 2000 - }, - "valueIntArray": { - "description": "value as integer", - "type": "array", - "items": { - "type": "integer", - "format": "int64" - }, - "x-nullable": true, - "x-omitempty": true, - "example": "[100, 200]" - }, - "valueNumber": { - "description": "value as number/float", - "type": "number", - "format": "float64", - "x-nullable": true, - "example": 3.14 - }, - "valueNumberArray": { - "description": "value as number/float", - "type": "array", - "items": { - "type": "number", - "format": "float64" - }, - "x-nullable": true, - "x-omitempty": true, - "example": [ - 3.14 - ] - }, - "valueString": { - "description": "value as text (deprecated as of v1.19; alias for valueText)", - "type": "string", - "x-nullable": true, - "example": "my search term" - }, - "valueStringArray": { - "description": "value as text (deprecated as of v1.19; alias for valueText)", - "type": "array", - "items": { - "type": "string" - }, - "x-nullable": true, - "x-omitempty": true, - "example": [ - "my search term" - ] - }, - "valueText": { - "description": "value as text", - "type": "string", - "x-nullable": true, - "example": "my search term" - }, - "valueTextArray": { - "description": "value as text", - "type": "array", - "items": { - "type": "string" - }, - "x-nullable": true, - "x-omitempty": true, - "example": [ - "my search term" - ] - } - } - }, - "WhereFilterGeoRange": { - "description": "filter within a distance of a georange", - "type": "object", - "properties": { - "distance": { - "type": "object", - "properties": { - "max": { - "type": "number", - "format": "float64" - } - } - }, - "geoCoordinates": { - "x-nullable": false, - "$ref": "#/definitions/GeoCoordinates" - } - } - } - }, - "parameters": { - "CommonAfterParameterQuery": { - "type": "string", - "description": "The starting ID of the result window.", - "name": "after", - "in": "query" - }, - "CommonClassParameterQuery": { - "type": "string", - "description": "Class parameter specifies the class from which to query objects", - "name": "class", - "in": "query" - }, - "CommonConsistencyLevelParameterQuery": { - "type": "string", - "description": "Determines how many replicas must acknowledge a request before it is considered successful", - "name": "consistency_level", - "in": "query" - }, - "CommonIncludeParameterQuery": { - "type": "string", - "description": "Include additional information, such as classification infos. Allowed values include: classification, vector, interpretation", - "name": "include", - "in": "query" - }, - "CommonLimitParameterQuery": { - "type": "integer", - "format": "int64", - "description": "The maximum number of items to be returned per page. Default value is set in Weaviate config.", - "name": "limit", - "in": "query" - }, - "CommonNodeNameParameterQuery": { - "type": "string", - "description": "The target node which should fulfill the request", - "name": "node_name", - "in": "query" - }, - "CommonOffsetParameterQuery": { - "type": "integer", - "format": "int64", - "default": 0, - "description": "The starting index of the result window. Default value is 0.", - "name": "offset", - "in": "query" - }, - "CommonOrderParameterQuery": { - "type": "string", - "description": "Order parameter to tell how to order (asc or desc) data within given field", - "name": "order", - "in": "query" - }, - "CommonOutputVerbosityParameterQuery": { - "type": "string", - "default": "minimal", - "description": "Controls the verbosity of the output, possible values are: \"minimal\", \"verbose\". Defaults to \"minimal\".", - "name": "output", - "in": "query" - }, - "CommonSortParameterQuery": { - "type": "string", - "description": "Sort parameter to pass an information about the names of the sort fields", - "name": "sort", - "in": "query" - }, - "CommonTenantParameterQuery": { - "type": "string", - "description": "Specifies the tenant in a request targeting a multi-tenant class", - "name": "tenant", - "in": "query" - } - }, - "securityDefinitions": { - "oidc": { - "description": "OIDC (OpenConnect ID - based on OAuth2)", - "type": "oauth2", - "flow": "implicit", - "authorizationUrl": "http://to-be-configured-in-the-application-config" - } - }, - "security": [ - {}, - { - "oidc": [] - } - ], - "tags": [ - { - "name": "objects" - }, - { - "description": "These operations allow to execute batch requests for Objects and Objects. Mostly used for importing large datasets.", - "name": "batch" - }, - { - "name": "graphql" - }, - { - "name": "meta" - }, - { - "name": "P2P" - }, - { - "description": "All functions related to the Contextionary.", - "name": "contextionary-API" - }, - { - "description": "These operations enable manipulation of the schema in Weaviate schema.", - "name": "schema" - } - ], - "externalDocs": { - "url": "https://github.com/weaviate/weaviate" - } -}`)) - FlatSwaggerJSON = json.RawMessage([]byte(`{ - "consumes": [ - "application/json", - "application/yaml" - ], - "produces": [ - "application/json" - ], - "schemes": [ - "https" - ], - "swagger": "2.0", - "info": { - "description": "Cloud-native, modular vector database", - "title": "Weaviate", - "contact": { - "name": "Weaviate", - "url": "https://github.com/weaviate", - "email": "hello@weaviate.io" - }, - "version": "1.23.3" - }, - "basePath": "/v1", - "paths": { - "/": { - "get": { - "description": "Home. Discover the REST API", - "operationId": "weaviate.root", - "responses": { - "200": { - "description": "Weaviate is alive and ready to serve content", - "schema": { - "type": "object", - "properties": { - "links": { - "type": "array", - "items": { - "$ref": "#/definitions/Link" - } - } - } - } - } - } - } - }, - "/.well-known/live": { - "get": { - "description": "Determines whether the application is alive. Can be used for kubernetes liveness probe", - "operationId": "weaviate.wellknown.liveness", - "responses": { - "200": { - "description": "The application is able to respond to HTTP requests" - } - } - } - }, - "/.well-known/openid-configuration": { - "get": { - "description": "OIDC Discovery page, redirects to the token issuer if one is configured", - "tags": [ - "well-known", - "oidc", - "discovery" - ], - "summary": "OIDC discovery information if OIDC auth is enabled", - "responses": { - "200": { - "description": "Successful response, inspect body", - "schema": { - "type": "object", - "properties": { - "clientId": { - "description": "OAuth Client ID", - "type": "string" - }, - "href": { - "description": "The Location to redirect to", - "type": "string" - }, - "scopes": { - "description": "OAuth Scopes", - "type": "array", - "items": { - "type": "string" - }, - "x-omitempty": true - } - } - } - }, - "404": { - "description": "Not found, no oidc provider present" - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": false, - "x-available-in-websocket": false - } - }, - "/.well-known/ready": { - "get": { - "description": "Determines whether the application is ready to receive traffic. Can be used for kubernetes readiness probe.", - "operationId": "weaviate.wellknown.readiness", - "responses": { - "200": { - "description": "The application has completed its start-up routine and is ready to accept traffic." - }, - "503": { - "description": "The application is currently not able to serve traffic. If other horizontal replicas of weaviate are available and they are capable of receiving traffic, all traffic should be redirected there instead." - } - } - } - }, - "/backups/{backend}": { - "post": { - "description": "Starts a process of creating a backup for a set of classes", - "tags": [ - "backups" - ], - "operationId": "backups.create", - "parameters": [ - { - "type": "string", - "description": "Backup backend name e.g. filesystem, gcs, s3.", - "name": "backend", - "in": "path", - "required": true - }, - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/BackupCreateRequest" - } - } - ], - "responses": { - "200": { - "description": "Backup create process successfully started.", - "schema": { - "$ref": "#/definitions/BackupCreateResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Invalid backup creation attempt.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-serviceIds": [ - "weaviate.local.backup" - ] - } - }, - "/backups/{backend}/{id}": { - "get": { - "description": "Returns status of backup creation attempt for a set of classes", - "tags": [ - "backups" - ], - "operationId": "backups.create.status", - "parameters": [ - { - "type": "string", - "description": "Backup backend name e.g. filesystem, gcs, s3.", - "name": "backend", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "The ID of a backup. Must be URL-safe and work as a filesystem path, only lowercase, numbers, underscore, minus characters allowed.", - "name": "id", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "Backup creation status successfully returned", - "schema": { - "$ref": "#/definitions/BackupCreateStatusResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Not Found - Backup does not exist", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Invalid backup restoration status attempt.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-serviceIds": [ - "weaviate.local.backup" - ] - } - }, - "/backups/{backend}/{id}/restore": { - "get": { - "description": "Returns status of a backup restoration attempt for a set of classes", - "tags": [ - "backups" - ], - "operationId": "backups.restore.status", - "parameters": [ - { - "type": "string", - "description": "Backup backend name e.g. filesystem, gcs, s3.", - "name": "backend", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "The ID of a backup. Must be URL-safe and work as a filesystem path, only lowercase, numbers, underscore, minus characters allowed.", - "name": "id", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "Backup restoration status successfully returned", - "schema": { - "$ref": "#/definitions/BackupRestoreStatusResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Not Found - Backup does not exist", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-serviceIds": [ - "weaviate.local.backup" - ] - }, - "post": { - "description": "Starts a process of restoring a backup for a set of classes", - "tags": [ - "backups" - ], - "operationId": "backups.restore", - "parameters": [ - { - "type": "string", - "description": "Backup backend name e.g. filesystem, gcs, s3.", - "name": "backend", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "The ID of a backup. Must be URL-safe and work as a filesystem path, only lowercase, numbers, underscore, minus characters allowed.", - "name": "id", - "in": "path", - "required": true - }, - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/BackupRestoreRequest" - } - } - ], - "responses": { - "200": { - "description": "Backup restoration process successfully started.", - "schema": { - "$ref": "#/definitions/BackupRestoreResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Not Found - Backup does not exist", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Invalid backup restoration attempt.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-serviceIds": [ - "weaviate.local.backup" - ] - } - }, - "/batch/objects": { - "post": { - "description": "Register new Objects in bulk. Provided meta-data and schema values are validated.", - "tags": [ - "batch", - "objects" - ], - "summary": "Creates new Objects based on a Object template as a batch.", - "operationId": "batch.objects.create", - "parameters": [ - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "type": "object", - "properties": { - "fields": { - "description": "Define which fields need to be returned. Default value is ALL", - "type": "array", - "items": { - "type": "string", - "default": "ALL", - "enum": [ - "ALL", - "class", - "schema", - "id", - "creationTimeUnix" - ] - } - }, - "objects": { - "type": "array", - "items": { - "$ref": "#/definitions/Object" - } - } - } - } - }, - { - "type": "string", - "description": "Determines how many replicas must acknowledge a request before it is considered successful", - "name": "consistency_level", - "in": "query" - } - ], - "responses": { - "200": { - "description": "Request succeeded, see response body to get detailed information about each batched item.", - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/ObjectsGetResponse" - } - } - }, - "400": { - "description": "Malformed request.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "x-serviceIds": [ - "weaviate.local.add" - ] - }, - "delete": { - "description": "Delete Objects in bulk that match a certain filter.", - "tags": [ - "batch", - "objects" - ], - "summary": "Deletes Objects based on a match filter as a batch.", - "operationId": "batch.objects.delete", - "parameters": [ - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/BatchDelete" - } - }, - { - "type": "string", - "description": "Determines how many replicas must acknowledge a request before it is considered successful", - "name": "consistency_level", - "in": "query" - }, - { - "type": "string", - "description": "Specifies the tenant in a request targeting a multi-tenant class", - "name": "tenant", - "in": "query" - } - ], - "responses": { - "200": { - "description": "Request succeeded, see response body to get detailed information about each batched item.", - "schema": { - "$ref": "#/definitions/BatchDeleteResponse" - } - }, - "400": { - "description": "Malformed request.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "x-serviceIds": [ - "weaviate.local.manipulate" - ] - } - }, - "/batch/references": { - "post": { - "description": "Register cross-references between any class items (objects or objects) in bulk.", - "tags": [ - "batch", - "references" - ], - "summary": "Creates new Cross-References between arbitrary classes in bulk.", - "operationId": "batch.references.create", - "parameters": [ - { - "description": "A list of references to be batched. The ideal size depends on the used database connector. Please see the documentation of the used connector for help", - "name": "body", - "in": "body", - "required": true, - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/BatchReference" - } - } - }, - { - "type": "string", - "description": "Determines how many replicas must acknowledge a request before it is considered successful", - "name": "consistency_level", - "in": "query" - } - ], - "responses": { - "200": { - "description": "Request Successful. Warning: A successful request does not guarantee that every batched reference was successfully created. Inspect the response body to see which references succeeded and which failed.", - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/BatchReferenceResponse" - } - } - }, - "400": { - "description": "Malformed request.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "x-serviceIds": [ - "weaviate.local.add" - ] - } - }, - "/classifications/": { - "post": { - "description": "Trigger a classification based on the specified params. Classifications will run in the background, use GET /classifications/\u003cid\u003e to retrieve the status of your classification.", - "tags": [ - "classifications" - ], - "summary": "Starts a classification.", - "operationId": "classifications.post", - "parameters": [ - { - "description": "parameters to start a classification", - "name": "params", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/Classification" - } - } - ], - "responses": { - "201": { - "description": "Successfully started classification.", - "schema": { - "$ref": "#/definitions/Classification" - } - }, - "400": { - "description": "Incorrect request", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-serviceIds": [ - "weaviate.classifications.post" - ] - } - }, - "/classifications/{id}": { - "get": { - "description": "Get status, results and metadata of a previously created classification", - "tags": [ - "classifications" - ], - "summary": "View previously created classification", - "operationId": "classifications.get", - "parameters": [ - { - "type": "string", - "description": "classification id", - "name": "id", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "Found the classification, returned as body", - "schema": { - "$ref": "#/definitions/Classification" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Not Found - Classification does not exist" - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-serviceIds": [ - "weaviate.classifications.get" - ] - } - }, - "/graphql": { - "post": { - "description": "Get an object based on GraphQL", - "tags": [ - "graphql" - ], - "summary": "Get a response based on GraphQL", - "operationId": "graphql.post", - "parameters": [ - { - "description": "The GraphQL query request parameters.", - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/GraphQLQuery" - } - } - ], - "responses": { - "200": { - "description": "Successful query (with select).", - "schema": { - "$ref": "#/definitions/GraphQLResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "x-serviceIds": [ - "weaviate.local.query", - "weaviate.local.query.meta", - "weaviate.network.query", - "weaviate.network.query.meta" - ] - } - }, - "/graphql/batch": { - "post": { - "description": "Perform a batched GraphQL query", - "tags": [ - "graphql" - ], - "summary": "Get a response based on GraphQL.", - "operationId": "graphql.batch", - "parameters": [ - { - "description": "The GraphQL queries.", - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/GraphQLQueries" - } - } - ], - "responses": { - "200": { - "description": "Successful query (with select).", - "schema": { - "$ref": "#/definitions/GraphQLResponses" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "x-serviceIds": [ - "weaviate.local.query", - "weaviate.local.query.meta", - "weaviate.network.query", - "weaviate.network.query.meta" - ] - } - }, - "/meta": { - "get": { - "description": "Gives meta information about the server and can be used to provide information to another Weaviate instance that wants to interact with the current instance.", - "tags": [ - "meta" - ], - "summary": "Returns meta information of the current Weaviate instance.", - "operationId": "meta.get", - "responses": { - "200": { - "description": "Successful response.", - "schema": { - "$ref": "#/definitions/Meta" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "x-serviceIds": [ - "weaviate.local.query.meta" - ] - } - }, - "/nodes": { - "get": { - "description": "Returns status of Weaviate DB.", - "tags": [ - "nodes" - ], - "operationId": "nodes.get", - "parameters": [ - { - "type": "string", - "default": "minimal", - "description": "Controls the verbosity of the output, possible values are: \"minimal\", \"verbose\". Defaults to \"minimal\".", - "name": "output", - "in": "query" - } - ], - "responses": { - "200": { - "description": "Nodes status successfully returned", - "schema": { - "$ref": "#/definitions/NodesStatusResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Not Found - Backup does not exist", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Invalid backup restoration status attempt.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-serviceIds": [ - "weaviate.nodes.status.get" - ] - } - }, - "/nodes/{className}": { - "get": { - "description": "Returns status of Weaviate DB.", - "tags": [ - "nodes" - ], - "operationId": "nodes.get.class", - "parameters": [ - { - "type": "string", - "name": "className", - "in": "path", - "required": true - }, - { - "type": "string", - "default": "minimal", - "description": "Controls the verbosity of the output, possible values are: \"minimal\", \"verbose\". Defaults to \"minimal\".", - "name": "output", - "in": "query" - } - ], - "responses": { - "200": { - "description": "Nodes status successfully returned", - "schema": { - "$ref": "#/definitions/NodesStatusResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Not Found - Backup does not exist", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Invalid backup restoration status attempt.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-serviceIds": [ - "weaviate.nodes.status.get.class" - ] - } - }, - "/objects": { - "get": { - "description": "Lists all Objects in reverse order of creation, owned by the user that belongs to the used token.", - "tags": [ - "objects" - ], - "summary": "Get a list of Objects.", - "operationId": "objects.list", - "parameters": [ - { - "type": "string", - "description": "The starting ID of the result window.", - "name": "after", - "in": "query" - }, - { - "type": "integer", - "format": "int64", - "default": 0, - "description": "The starting index of the result window. Default value is 0.", - "name": "offset", - "in": "query" - }, - { - "type": "integer", - "format": "int64", - "description": "The maximum number of items to be returned per page. Default value is set in Weaviate config.", - "name": "limit", - "in": "query" - }, - { - "type": "string", - "description": "Include additional information, such as classification infos. Allowed values include: classification, vector, interpretation", - "name": "include", - "in": "query" - }, - { - "type": "string", - "description": "Sort parameter to pass an information about the names of the sort fields", - "name": "sort", - "in": "query" - }, - { - "type": "string", - "description": "Order parameter to tell how to order (asc or desc) data within given field", - "name": "order", - "in": "query" - }, - { - "type": "string", - "description": "Class parameter specifies the class from which to query objects", - "name": "class", - "in": "query" - }, - { - "type": "string", - "description": "Specifies the tenant in a request targeting a multi-tenant class", - "name": "tenant", - "in": "query" - } - ], - "responses": { - "200": { - "description": "Successful response.", - "schema": { - "$ref": "#/definitions/ObjectsListResponse" - } - }, - "400": { - "description": "Malformed request.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Successful query result but no resource was found." - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "x-serviceIds": [ - "weaviate.local.query" - ] - }, - "post": { - "description": "Registers a new Object. Provided meta-data and schema values are validated.", - "tags": [ - "objects" - ], - "summary": "Create Objects between two Objects (object and subject).", - "operationId": "objects.create", - "parameters": [ - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/Object" - } - }, - { - "type": "string", - "description": "Determines how many replicas must acknowledge a request before it is considered successful", - "name": "consistency_level", - "in": "query" - } - ], - "responses": { - "200": { - "description": "Object created.", - "schema": { - "$ref": "#/definitions/Object" - } - }, - "400": { - "description": "Malformed request.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "x-serviceIds": [ - "weaviate.local.add" - ] - } - }, - "/objects/validate": { - "post": { - "description": "Validate an Object's schema and meta-data. It has to be based on a schema, which is related to the given Object to be accepted by this validation.", - "tags": [ - "objects" - ], - "summary": "Validate an Object based on a schema.", - "operationId": "objects.validate", - "parameters": [ - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/Object" - } - } - ], - "responses": { - "200": { - "description": "Successfully validated." - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "x-serviceIds": [ - "weaviate.local.query.meta" - ] - } - }, - "/objects/{className}/{id}": { - "get": { - "description": "Get a single data object", - "tags": [ - "objects" - ], - "summary": "Get a specific Object based on its class and UUID. Also available as Websocket bus.", - "operationId": "objects.class.get", - "parameters": [ - { - "type": "string", - "name": "className", - "in": "path", - "required": true - }, - { - "type": "string", - "format": "uuid", - "description": "Unique ID of the Object.", - "name": "id", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "Include additional information, such as classification infos. Allowed values include: classification, vector, interpretation", - "name": "include", - "in": "query" - }, - { - "type": "string", - "description": "Determines how many replicas must acknowledge a request before it is considered successful", - "name": "consistency_level", - "in": "query" - }, - { - "type": "string", - "description": "The target node which should fulfill the request", - "name": "node_name", - "in": "query" - }, - { - "type": "string", - "description": "Specifies the tenant in a request targeting a multi-tenant class", - "name": "tenant", - "in": "query" - } - ], - "responses": { - "200": { - "description": "Successful response.", - "schema": { - "$ref": "#/definitions/Object" - } - }, - "400": { - "description": "Malformed request.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Successful query result but no resource was found." - }, - "422": { - "description": "Request is well-formed (i.e., syntactically correct), but erroneous.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "x-serviceIds": [ - "weaviate.local.query" - ] - }, - "put": { - "description": "Update an individual data object based on its class and uuid.", - "tags": [ - "objects" - ], - "summary": "Update a class object based on its uuid", - "operationId": "objects.class.put", - "parameters": [ - { - "type": "string", - "name": "className", - "in": "path", - "required": true - }, - { - "type": "string", - "format": "uuid", - "description": "The uuid of the data object to update.", - "name": "id", - "in": "path", - "required": true - }, - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/Object" - } - }, - { - "type": "string", - "description": "Determines how many replicas must acknowledge a request before it is considered successful", - "name": "consistency_level", - "in": "query" - } - ], - "responses": { - "200": { - "description": "Successfully received.", - "schema": { - "$ref": "#/definitions/Object" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Successful query result but no resource was found." - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "x-serviceIds": [ - "weaviate.local.manipulate" - ] - }, - "delete": { - "description": "Delete a single data object.", - "tags": [ - "objects" - ], - "summary": "Delete object based on its class and UUID.", - "operationId": "objects.class.delete", - "parameters": [ - { - "type": "string", - "name": "className", - "in": "path", - "required": true - }, - { - "type": "string", - "format": "uuid", - "description": "Unique ID of the Object.", - "name": "id", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "Determines how many replicas must acknowledge a request before it is considered successful", - "name": "consistency_level", - "in": "query" - }, - { - "type": "string", - "description": "Specifies the tenant in a request targeting a multi-tenant class", - "name": "tenant", - "in": "query" - } - ], - "responses": { - "204": { - "description": "Successfully deleted." - }, - "400": { - "description": "Malformed request.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Successful query result but no resource was found." - }, - "422": { - "description": "Request is well-formed (i.e., syntactically correct), but erroneous.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": true, - "x-available-in-websocket": true, - "x-serviceIds": [ - "weaviate.local.manipulate" - ] - }, - "head": { - "description": "Checks if a data object exists without retrieving it.", - "tags": [ - "objects" - ], - "summary": "Checks object's existence based on its class and uuid.", - "operationId": "objects.class.head", - "parameters": [ - { - "type": "string", - "description": "The class name as defined in the schema", - "name": "className", - "in": "path", - "required": true - }, - { - "type": "string", - "format": "uuid", - "description": "The uuid of the data object", - "name": "id", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "Determines how many replicas must acknowledge a request before it is considered successful", - "name": "consistency_level", - "in": "query" - }, - { - "type": "string", - "description": "Specifies the tenant in a request targeting a multi-tenant class", - "name": "tenant", - "in": "query" - } - ], - "responses": { - "204": { - "description": "Object exists." - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Object doesn't exist." - }, - "422": { - "description": "Request is well-formed (i.e., syntactically correct), but erroneous.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": true, - "x-available-in-websocket": true, - "x-serviceIds": [ - "weaviate.local.manipulate" - ] - }, - "patch": { - "description": "Update an individual data object based on its class and uuid. This method supports json-merge style patch semantics (RFC 7396). Provided meta-data and schema values are validated. LastUpdateTime is set to the time this function is called.", - "tags": [ - "objects" - ], - "summary": "Update an Object based on its UUID (using patch semantics).", - "operationId": "objects.class.patch", - "parameters": [ - { - "type": "string", - "description": "The class name as defined in the schema", - "name": "className", - "in": "path", - "required": true - }, - { - "type": "string", - "format": "uuid", - "description": "The uuid of the data object to update.", - "name": "id", - "in": "path", - "required": true - }, - { - "description": "RFC 7396-style patch, the body contains the object to merge into the existing object.", - "name": "body", - "in": "body", - "schema": { - "$ref": "#/definitions/Object" - } - }, - { - "type": "string", - "description": "Determines how many replicas must acknowledge a request before it is considered successful", - "name": "consistency_level", - "in": "query" - } - ], - "responses": { - "204": { - "description": "Successfully applied. No content provided." - }, - "400": { - "description": "The patch-JSON is malformed.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Successful query result but no resource was found." - }, - "422": { - "description": "The patch-JSON is valid but unprocessable.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "x-serviceIds": [ - "weaviate.local.manipulate" - ] - } - }, - "/objects/{className}/{id}/references/{propertyName}": { - "put": { - "description": "Update all references of a property of a data object.", - "tags": [ - "objects" - ], - "summary": "Replace all references to a class-property.", - "operationId": "objects.class.references.put", - "parameters": [ - { - "type": "string", - "description": "The class name as defined in the schema", - "name": "className", - "in": "path", - "required": true - }, - { - "type": "string", - "format": "uuid", - "description": "Unique ID of the Object.", - "name": "id", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "Unique name of the property related to the Object.", - "name": "propertyName", - "in": "path", - "required": true - }, - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/MultipleRef" - } - }, - { - "type": "string", - "description": "Determines how many replicas must acknowledge a request before it is considered successful", - "name": "consistency_level", - "in": "query" - }, - { - "type": "string", - "description": "Specifies the tenant in a request targeting a multi-tenant class", - "name": "tenant", - "in": "query" - } - ], - "responses": { - "200": { - "description": "Successfully replaced all the references." - }, - "400": { - "description": "Malformed request.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Source object doesn't exist." - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the property exists or that it is a class?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "x-serviceIds": [ - "weaviate.local.manipulate" - ] - }, - "post": { - "description": "Add a single reference to a class-property.", - "tags": [ - "objects" - ], - "summary": "Add a single reference to a class-property.", - "operationId": "objects.class.references.create", - "parameters": [ - { - "type": "string", - "description": "The class name as defined in the schema", - "name": "className", - "in": "path", - "required": true - }, - { - "type": "string", - "format": "uuid", - "description": "Unique ID of the Object.", - "name": "id", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "Unique name of the property related to the Object.", - "name": "propertyName", - "in": "path", - "required": true - }, - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/SingleRef" - } - }, - { - "type": "string", - "description": "Determines how many replicas must acknowledge a request before it is considered successful", - "name": "consistency_level", - "in": "query" - }, - { - "type": "string", - "description": "Specifies the tenant in a request targeting a multi-tenant class", - "name": "tenant", - "in": "query" - } - ], - "responses": { - "200": { - "description": "Successfully added the reference." - }, - "400": { - "description": "Malformed request.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Source object doesn't exist." - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the property exists or that it is a class?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "x-serviceIds": [ - "weaviate.local.manipulate" - ] - }, - "delete": { - "description": "Delete the single reference that is given in the body from the list of references that this property of a data object has", - "tags": [ - "objects" - ], - "summary": "Delete the single reference that is given in the body from the list of references that this property has.", - "operationId": "objects.class.references.delete", - "parameters": [ - { - "type": "string", - "description": "The class name as defined in the schema", - "name": "className", - "in": "path", - "required": true - }, - { - "type": "string", - "format": "uuid", - "description": "Unique ID of the Object.", - "name": "id", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "Unique name of the property related to the Object.", - "name": "propertyName", - "in": "path", - "required": true - }, - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/SingleRef" - } - }, - { - "type": "string", - "description": "Determines how many replicas must acknowledge a request before it is considered successful", - "name": "consistency_level", - "in": "query" - }, - { - "type": "string", - "description": "Specifies the tenant in a request targeting a multi-tenant class", - "name": "tenant", - "in": "query" - } - ], - "responses": { - "204": { - "description": "Successfully deleted." - }, - "400": { - "description": "Malformed request.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Successful query result but no resource was found.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the property exists or that it is a class?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "x-serviceIds": [ - "weaviate.local.manipulate" - ] - } - }, - "/objects/{id}": { - "get": { - "description": "Lists Objects.", - "tags": [ - "objects" - ], - "summary": "Get a specific Object based on its UUID and a Object UUID. Also available as Websocket bus.", - "operationId": "objects.get", - "deprecated": true, - "parameters": [ - { - "type": "string", - "format": "uuid", - "description": "Unique ID of the Object.", - "name": "id", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "Include additional information, such as classification infos. Allowed values include: classification, vector, interpretation", - "name": "include", - "in": "query" - } - ], - "responses": { - "200": { - "description": "Successful response.", - "schema": { - "$ref": "#/definitions/Object" - } - }, - "400": { - "description": "Malformed request.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Successful query result but no resource was found." - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "x-serviceIds": [ - "weaviate.local.query" - ] - }, - "put": { - "description": "Updates an Object's data. Given meta-data and schema values are validated. LastUpdateTime is set to the time this function is called.", - "tags": [ - "objects" - ], - "summary": "Update an Object based on its UUID.", - "operationId": "objects.update", - "deprecated": true, - "parameters": [ - { - "type": "string", - "format": "uuid", - "description": "Unique ID of the Object.", - "name": "id", - "in": "path", - "required": true - }, - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/Object" - } - }, - { - "type": "string", - "description": "Determines how many replicas must acknowledge a request before it is considered successful", - "name": "consistency_level", - "in": "query" - } - ], - "responses": { - "200": { - "description": "Successfully received.", - "schema": { - "$ref": "#/definitions/Object" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Successful query result but no resource was found." - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "x-serviceIds": [ - "weaviate.local.manipulate" - ] - }, - "delete": { - "description": "Deletes an Object from the system.", - "tags": [ - "objects" - ], - "summary": "Delete an Object based on its UUID.", - "operationId": "objects.delete", - "deprecated": true, - "parameters": [ - { - "type": "string", - "format": "uuid", - "description": "Unique ID of the Object.", - "name": "id", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "Determines how many replicas must acknowledge a request before it is considered successful", - "name": "consistency_level", - "in": "query" - }, - { - "type": "string", - "description": "Specifies the tenant in a request targeting a multi-tenant class", - "name": "tenant", - "in": "query" - } - ], - "responses": { - "204": { - "description": "Successfully deleted." - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Successful query result but no resource was found." - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": true, - "x-available-in-websocket": true, - "x-serviceIds": [ - "weaviate.local.manipulate" - ] - }, - "head": { - "description": "Checks if an Object exists in the system.", - "tags": [ - "objects" - ], - "summary": "Checks Object's existence based on its UUID.", - "operationId": "objects.head", - "deprecated": true, - "parameters": [ - { - "type": "string", - "format": "uuid", - "description": "Unique ID of the Object.", - "name": "id", - "in": "path", - "required": true - } - ], - "responses": { - "204": { - "description": "Object exists." - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Object doesn't exist." - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": true, - "x-available-in-websocket": true, - "x-serviceIds": [ - "weaviate.local.manipulate" - ] - }, - "patch": { - "description": "Updates an Object. This method supports json-merge style patch semantics (RFC 7396). Provided meta-data and schema values are validated. LastUpdateTime is set to the time this function is called.", - "tags": [ - "objects" - ], - "summary": "Update an Object based on its UUID (using patch semantics).", - "operationId": "objects.patch", - "deprecated": true, - "parameters": [ - { - "type": "string", - "format": "uuid", - "description": "Unique ID of the Object.", - "name": "id", - "in": "path", - "required": true - }, - { - "description": "RFC 7396-style patch, the body contains the object to merge into the existing object.", - "name": "body", - "in": "body", - "schema": { - "$ref": "#/definitions/Object" - } - }, - { - "type": "string", - "description": "Determines how many replicas must acknowledge a request before it is considered successful", - "name": "consistency_level", - "in": "query" - } - ], - "responses": { - "204": { - "description": "Successfully applied. No content provided." - }, - "400": { - "description": "The patch-JSON is malformed." - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Successful query result but no resource was found." - }, - "422": { - "description": "The patch-JSON is valid but unprocessable.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "x-serviceIds": [ - "weaviate.local.manipulate" - ] - } - }, - "/objects/{id}/references/{propertyName}": { - "put": { - "description": "Replace all references to a class-property.", - "tags": [ - "objects" - ], - "summary": "Replace all references to a class-property.", - "operationId": "objects.references.update", - "deprecated": true, - "parameters": [ - { - "type": "string", - "format": "uuid", - "description": "Unique ID of the Object.", - "name": "id", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "Unique name of the property related to the Object.", - "name": "propertyName", - "in": "path", - "required": true - }, - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/MultipleRef" - } - }, - { - "type": "string", - "description": "Specifies the tenant in a request targeting a multi-tenant class", - "name": "tenant", - "in": "query" - } - ], - "responses": { - "200": { - "description": "Successfully replaced all the references." - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the property exists or that it is a class?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "x-serviceIds": [ - "weaviate.local.manipulate" - ] - }, - "post": { - "description": "Add a single reference to a class-property.", - "tags": [ - "objects" - ], - "summary": "Add a single reference to a class-property.", - "operationId": "objects.references.create", - "deprecated": true, - "parameters": [ - { - "type": "string", - "format": "uuid", - "description": "Unique ID of the Object.", - "name": "id", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "Unique name of the property related to the Object.", - "name": "propertyName", - "in": "path", - "required": true - }, - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/SingleRef" - } - }, - { - "type": "string", - "description": "Specifies the tenant in a request targeting a multi-tenant class", - "name": "tenant", - "in": "query" - } - ], - "responses": { - "200": { - "description": "Successfully added the reference." - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the property exists or that it is a class?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "x-serviceIds": [ - "weaviate.local.manipulate" - ] - }, - "delete": { - "description": "Delete the single reference that is given in the body from the list of references that this property has.", - "tags": [ - "objects" - ], - "summary": "Delete the single reference that is given in the body from the list of references that this property has.", - "operationId": "objects.references.delete", - "deprecated": true, - "parameters": [ - { - "type": "string", - "format": "uuid", - "description": "Unique ID of the Object.", - "name": "id", - "in": "path", - "required": true - }, - { - "type": "string", - "description": "Unique name of the property related to the Object.", - "name": "propertyName", - "in": "path", - "required": true - }, - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/SingleRef" - } - }, - { - "type": "string", - "description": "Specifies the tenant in a request targeting a multi-tenant class", - "name": "tenant", - "in": "query" - } - ], - "responses": { - "204": { - "description": "Successfully deleted." - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Successful query result but no resource was found.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "x-serviceIds": [ - "weaviate.local.manipulate" - ] - } - }, - "/schema": { - "get": { - "tags": [ - "schema" - ], - "summary": "Dump the current the database schema.", - "operationId": "schema.dump", - "responses": { - "200": { - "description": "Successfully dumped the database schema.", - "schema": { - "$ref": "#/definitions/Schema" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-serviceIds": [ - "weaviate.local.query.meta" - ] - }, - "post": { - "tags": [ - "schema" - ], - "summary": "Create a new Object class in the schema.", - "operationId": "schema.objects.create", - "parameters": [ - { - "name": "objectClass", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/Class" - } - } - ], - "responses": { - "200": { - "description": "Added the new Object class to the schema.", - "schema": { - "$ref": "#/definitions/Class" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Invalid Object class", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-serviceIds": [ - "weaviate.local.add.meta" - ] - } - }, - "/schema/cluster-status": { - "get": { - "tags": [ - "schema" - ], - "operationId": "schema.cluster.status", - "responses": { - "200": { - "description": "The schema in the cluster is in sync.", - "schema": { - "$ref": "#/definitions/SchemaClusterStatus" - } - }, - "500": { - "description": "The schema is either out of sync (see response body) or the sync check could not be completed.", - "schema": { - "$ref": "#/definitions/SchemaClusterStatus" - } - } - } - } - }, - "/schema/{className}": { - "get": { - "tags": [ - "schema" - ], - "summary": "Get a single class from the schema", - "operationId": "schema.objects.get", - "parameters": [ - { - "type": "string", - "name": "className", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "Found the Class, returned as body", - "schema": { - "$ref": "#/definitions/Class" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "This class does not exist" - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-serviceIds": [ - "weaviate.local.get.meta" - ] - }, - "put": { - "description": "Use this endpoint to alter an existing class in the schema. Note that not all settings are mutable. If an error about immutable fields is returned and you still need to update this particular setting, you will have to delete the class (and the underlying data) and recreate. This endpoint cannot be used to modify properties. Instead use POST /v1/schema/{className}/properties. A typical use case for this endpoint is to update configuration, such as the vectorIndexConfig. Note that even in mutable sections, such as vectorIndexConfig, some fields may be immutable.", - "tags": [ - "schema" - ], - "summary": "Update settings of an existing schema class", - "operationId": "schema.objects.update", - "parameters": [ - { - "type": "string", - "name": "className", - "in": "path", - "required": true - }, - { - "name": "objectClass", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/Class" - } - } - ], - "responses": { - "200": { - "description": "Class was updated successfully", - "schema": { - "$ref": "#/definitions/Class" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Class to be updated does not exist", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Invalid update attempt", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-serviceIds": [ - "weaviate.local.manipulate.meta" - ] - }, - "delete": { - "tags": [ - "schema" - ], - "summary": "Remove an Object class (and all data in the instances) from the schema.", - "operationId": "schema.objects.delete", - "parameters": [ - { - "type": "string", - "name": "className", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "Removed the Object class from the schema." - }, - "400": { - "description": "Could not delete the Object class.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-serviceIds": [ - "weaviate.local.manipulate.meta" - ] - } - }, - "/schema/{className}/properties": { - "post": { - "tags": [ - "schema" - ], - "summary": "Add a property to an Object class.", - "operationId": "schema.objects.properties.add", - "parameters": [ - { - "type": "string", - "name": "className", - "in": "path", - "required": true - }, - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/Property" - } - } - ], - "responses": { - "200": { - "description": "Added the property.", - "schema": { - "$ref": "#/definitions/Property" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Invalid property.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-serviceIds": [ - "weaviate.local.manipulate.meta" - ] - } - }, - "/schema/{className}/shards": { - "get": { - "tags": [ - "schema" - ], - "summary": "Get the shards status of an Object class", - "operationId": "schema.objects.shards.get", - "parameters": [ - { - "type": "string", - "name": "className", - "in": "path", - "required": true - }, - { - "type": "string", - "name": "tenant", - "in": "query" - } - ], - "responses": { - "200": { - "description": "Found the status of the shards, returned as body", - "schema": { - "$ref": "#/definitions/ShardStatusList" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "This class does not exist", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-serviceIds": [ - "weaviate.local.get.meta" - ] - } - }, - "/schema/{className}/shards/{shardName}": { - "put": { - "description": "Update shard status of an Object Class", - "tags": [ - "schema" - ], - "operationId": "schema.objects.shards.update", - "parameters": [ - { - "type": "string", - "name": "className", - "in": "path", - "required": true - }, - { - "type": "string", - "name": "shardName", - "in": "path", - "required": true - }, - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/ShardStatus" - } - } - ], - "responses": { - "200": { - "description": "Shard status was updated successfully", - "schema": { - "$ref": "#/definitions/ShardStatus" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Shard to be updated does not exist", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Invalid update attempt", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "x-serviceIds": [ - "weaviate.local.manipulate.meta" - ] - } - }, - "/schema/{className}/tenants": { - "get": { - "description": "get all tenants from a specific class", - "tags": [ - "schema" - ], - "operationId": "tenants.get", - "parameters": [ - { - "type": "string", - "name": "className", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "tenants from specified class.", - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/Tenant" - } - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Invalid Tenant class", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - } - }, - "put": { - "description": "Update tenant of a specific class", - "tags": [ - "schema" - ], - "operationId": "tenants.update", - "parameters": [ - { - "type": "string", - "name": "className", - "in": "path", - "required": true - }, - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/Tenant" - } - } - } - ], - "responses": { - "200": { - "description": "Updated tenants of the specified class", - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/Tenant" - } - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Invalid Tenant class", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - } - }, - "post": { - "description": "Create a new tenant for a specific class", - "tags": [ - "schema" - ], - "operationId": "tenants.create", - "parameters": [ - { - "type": "string", - "name": "className", - "in": "path", - "required": true - }, - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/Tenant" - } - } - } - ], - "responses": { - "200": { - "description": "Added new tenants to the specified class", - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/Tenant" - } - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Invalid Tenant class", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - } - }, - "delete": { - "description": "delete tenants from a specific class", - "tags": [ - "schema" - ], - "operationId": "tenants.delete", - "parameters": [ - { - "type": "string", - "name": "className", - "in": "path", - "required": true - }, - { - "name": "tenants", - "in": "body", - "required": true, - "schema": { - "type": "array", - "items": { - "type": "string" - } - } - } - ], - "responses": { - "200": { - "description": "Deleted tenants from specified class." - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Invalid Tenant class", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - } - } - } - }, - "definitions": { - "AdditionalProperties": { - "description": "Additional Meta information about a single object object.", - "type": "object", - "additionalProperties": { - "type": "object" - } - }, - "BM25Config": { - "description": "tuning parameters for the BM25 algorithm", - "type": "object", - "properties": { - "b": { - "description": "calibrates term-weight scaling based on the document length", - "type": "number", - "format": "float" - }, - "k1": { - "description": "calibrates term-weight scaling based on the term frequency within a document", - "type": "number", - "format": "float" - } - } - }, - "BackupConfig": { - "description": "Backup custom configuration", - "type": "object", - "properties": { - "CPUPercentage": { - "description": "Desired CPU core utilization ranging from 1%-80%", - "type": "integer", - "default": 50, - "maximum": 80, - "minimum": 1, - "x-nullable": false - }, - "ChunkSize": { - "description": "Weaviate will attempt to come close the specified size, with a minimum of 2MB, default of 128MB, and a maximum of 512MB", - "type": "integer", - "default": 128, - "maximum": 512, - "minimum": 2, - "x-nullable": false - }, - "CompressionLevel": { - "description": "compression level used by compression algorithm", - "type": "string", - "default": "DefaultCompression", - "enum": [ - "DefaultCompression", - "BestSpeed", - "BestCompression" - ], - "x-nullable": false - } - } - }, - "BackupCreateRequest": { - "description": "Request body for creating a backup of a set of classes", - "properties": { - "config": { - "description": "Custom configuration for the backup creation process", - "type": "object", - "$ref": "#/definitions/BackupConfig" - }, - "exclude": { - "description": "List of classes to exclude from the backup creation process", - "type": "array", - "items": { - "type": "string" - } - }, - "id": { - "description": "The ID of the backup. Must be URL-safe and work as a filesystem path, only lowercase, numbers, underscore, minus characters allowed.", - "type": "string" - }, - "include": { - "description": "List of classes to include in the backup creation process", - "type": "array", - "items": { - "type": "string" - } - } - } - }, - "BackupCreateResponse": { - "description": "The definition of a backup create response body", - "properties": { - "backend": { - "description": "Backup backend name e.g. filesystem, gcs, s3.", - "type": "string" - }, - "classes": { - "description": "The list of classes for which the backup creation process was started", - "type": "array", - "items": { - "type": "string" - } - }, - "error": { - "description": "error message if creation failed", - "type": "string" - }, - "id": { - "description": "The ID of the backup. Must be URL-safe and work as a filesystem path, only lowercase, numbers, underscore, minus characters allowed.", - "type": "string" - }, - "path": { - "description": "destination path of backup files proper to selected backend", - "type": "string" - }, - "status": { - "description": "phase of backup creation process", - "type": "string", - "default": "STARTED", - "enum": [ - "STARTED", - "TRANSFERRING", - "TRANSFERRED", - "SUCCESS", - "FAILED" - ] - } - } - }, - "BackupCreateStatusResponse": { - "description": "The definition of a backup create metadata", - "properties": { - "backend": { - "description": "Backup backend name e.g. filesystem, gcs, s3.", - "type": "string" - }, - "error": { - "description": "error message if creation failed", - "type": "string" - }, - "id": { - "description": "The ID of the backup. Must be URL-safe and work as a filesystem path, only lowercase, numbers, underscore, minus characters allowed.", - "type": "string" - }, - "path": { - "description": "destination path of backup files proper to selected backend", - "type": "string" - }, - "status": { - "description": "phase of backup creation process", - "type": "string", - "default": "STARTED", - "enum": [ - "STARTED", - "TRANSFERRING", - "TRANSFERRED", - "SUCCESS", - "FAILED" - ] - } - } - }, - "BackupRestoreRequest": { - "description": "Request body for restoring a backup for a set of classes", - "properties": { - "config": { - "description": "Custom configuration for the backup restoration process", - "type": "object", - "$ref": "#/definitions/RestoreConfig" - }, - "exclude": { - "description": "List of classes to exclude from the backup restoration process", - "type": "array", - "items": { - "type": "string" - } - }, - "include": { - "description": "List of classes to include in the backup restoration process", - "type": "array", - "items": { - "type": "string" - } - }, - "node_mapping": { - "description": "Allows overriding the node names stored in the backup with different ones. Useful when restoring backups to a different environment.", - "type": "object", - "additionalProperties": { - "type": "string" - } - } - } - }, - "BackupRestoreResponse": { - "description": "The definition of a backup restore response body", - "properties": { - "backend": { - "description": "Backup backend name e.g. filesystem, gcs, s3.", - "type": "string" - }, - "classes": { - "description": "The list of classes for which the backup restoration process was started", - "type": "array", - "items": { - "type": "string" - } - }, - "error": { - "description": "error message if restoration failed", - "type": "string" - }, - "id": { - "description": "The ID of the backup. Must be URL-safe and work as a filesystem path, only lowercase, numbers, underscore, minus characters allowed.", - "type": "string" - }, - "path": { - "description": "destination path of backup files proper to selected backend", - "type": "string" - }, - "status": { - "description": "phase of backup restoration process", - "type": "string", - "default": "STARTED", - "enum": [ - "STARTED", - "TRANSFERRING", - "TRANSFERRED", - "SUCCESS", - "FAILED" - ] - } - } - }, - "BackupRestoreStatusResponse": { - "description": "The definition of a backup restore metadata", - "properties": { - "backend": { - "description": "Backup backend name e.g. filesystem, gcs, s3.", - "type": "string" - }, - "error": { - "description": "error message if restoration failed", - "type": "string" - }, - "id": { - "description": "The ID of the backup. Must be URL-safe and work as a filesystem path, only lowercase, numbers, underscore, minus characters allowed.", - "type": "string" - }, - "path": { - "description": "destination path of backup files proper to selected backup backend", - "type": "string" - }, - "status": { - "description": "phase of backup restoration process", - "type": "string", - "default": "STARTED", - "enum": [ - "STARTED", - "TRANSFERRING", - "TRANSFERRED", - "SUCCESS", - "FAILED" - ] - } - } - }, - "BatchDelete": { - "type": "object", - "properties": { - "dryRun": { - "description": "If true, objects will not be deleted yet, but merely listed. Defaults to false.", - "type": "boolean", - "default": false - }, - "match": { - "description": "Outlines how to find the objects to be deleted.", - "type": "object", - "properties": { - "class": { - "description": "Class (name) which objects will be deleted.", - "type": "string", - "example": "City" - }, - "where": { - "description": "Filter to limit the objects to be deleted.", - "type": "object", - "$ref": "#/definitions/WhereFilter" - } - } - }, - "output": { - "description": "Controls the verbosity of the output, possible values are: \"minimal\", \"verbose\". Defaults to \"minimal\".", - "type": "string", - "default": "minimal" - } - } - }, - "BatchDeleteMatch": { - "description": "Outlines how to find the objects to be deleted.", - "type": "object", - "properties": { - "class": { - "description": "Class (name) which objects will be deleted.", - "type": "string", - "example": "City" - }, - "where": { - "description": "Filter to limit the objects to be deleted.", - "type": "object", - "$ref": "#/definitions/WhereFilter" - } - } - }, - "BatchDeleteResponse": { - "description": "Delete Objects response.", - "type": "object", - "properties": { - "dryRun": { - "description": "If true, objects will not be deleted yet, but merely listed. Defaults to false.", - "type": "boolean", - "default": false - }, - "match": { - "description": "Outlines how to find the objects to be deleted.", - "type": "object", - "properties": { - "class": { - "description": "Class (name) which objects will be deleted.", - "type": "string", - "example": "City" - }, - "where": { - "description": "Filter to limit the objects to be deleted.", - "type": "object", - "$ref": "#/definitions/WhereFilter" - } - } - }, - "output": { - "description": "Controls the verbosity of the output, possible values are: \"minimal\", \"verbose\". Defaults to \"minimal\".", - "type": "string", - "default": "minimal" - }, - "results": { - "type": "object", - "properties": { - "failed": { - "description": "How many objects should have been deleted but could not be deleted.", - "type": "number", - "format": "int64", - "x-omitempty": false - }, - "limit": { - "description": "The most amount of objects that can be deleted in a single query, equals QUERY_MAXIMUM_RESULTS.", - "type": "number", - "format": "int64", - "x-omitempty": false - }, - "matches": { - "description": "How many objects were matched by the filter.", - "type": "number", - "format": "int64", - "x-omitempty": false - }, - "objects": { - "description": "With output set to \"minimal\" only objects with error occurred will the be described. Successfully deleted objects would be omitted. Output set to \"verbose\" will list all of the objets with their respective statuses.", - "type": "array", - "items": { - "$ref": "#/definitions/BatchDeleteResponseResultsObjectsItems0" - } - }, - "successful": { - "description": "How many objects were successfully deleted in this round.", - "type": "number", - "format": "int64", - "x-omitempty": false - } - } - } - } - }, - "BatchDeleteResponseMatch": { - "description": "Outlines how to find the objects to be deleted.", - "type": "object", - "properties": { - "class": { - "description": "Class (name) which objects will be deleted.", - "type": "string", - "example": "City" - }, - "where": { - "description": "Filter to limit the objects to be deleted.", - "type": "object", - "$ref": "#/definitions/WhereFilter" - } - } - }, - "BatchDeleteResponseResults": { - "type": "object", - "properties": { - "failed": { - "description": "How many objects should have been deleted but could not be deleted.", - "type": "number", - "format": "int64", - "x-omitempty": false - }, - "limit": { - "description": "The most amount of objects that can be deleted in a single query, equals QUERY_MAXIMUM_RESULTS.", - "type": "number", - "format": "int64", - "x-omitempty": false - }, - "matches": { - "description": "How many objects were matched by the filter.", - "type": "number", - "format": "int64", - "x-omitempty": false - }, - "objects": { - "description": "With output set to \"minimal\" only objects with error occurred will the be described. Successfully deleted objects would be omitted. Output set to \"verbose\" will list all of the objets with their respective statuses.", - "type": "array", - "items": { - "$ref": "#/definitions/BatchDeleteResponseResultsObjectsItems0" - } - }, - "successful": { - "description": "How many objects were successfully deleted in this round.", - "type": "number", - "format": "int64", - "x-omitempty": false - } - } - }, - "BatchDeleteResponseResultsObjectsItems0": { - "description": "Results for this specific Object.", - "format": "object", - "properties": { - "errors": { - "$ref": "#/definitions/ErrorResponse" - }, - "id": { - "description": "ID of the Object.", - "type": "string", - "format": "uuid" - }, - "status": { - "type": "string", - "default": "SUCCESS", - "enum": [ - "SUCCESS", - "DRYRUN", - "FAILED" - ] - } - } - }, - "BatchReference": { - "properties": { - "from": { - "description": "Long-form beacon-style URI to identify the source of the cross-ref including the property name. Should be in the form of weaviate://localhost/\u003ckinds\u003e/\u003cuuid\u003e/\u003cclassName\u003e/\u003cpropertyName\u003e, where \u003ckinds\u003e must be one of 'objects', 'objects' and \u003cclassName\u003e and \u003cpropertyName\u003e must represent the cross-ref property of source class to be used.", - "type": "string", - "format": "uri", - "example": "weaviate://localhost/Zoo/a5d09582-4239-4702-81c9-92a6e0122bb4/hasAnimals" - }, - "tenant": { - "description": "Name of the reference tenant.", - "type": "string" - }, - "to": { - "description": "Short-form URI to point to the cross-ref. Should be in the form of weaviate://localhost/\u003cuuid\u003e for the example of a local cross-ref to an object", - "type": "string", - "format": "uri", - "example": "weaviate://localhost/97525810-a9a5-4eb0-858a-71449aeb007f" - } - } - }, - "BatchReferenceResponse": { - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/BatchReference" - }, - { - "properties": { - "result": { - "description": "Results for this specific reference.", - "format": "object", - "properties": { - "errors": { - "$ref": "#/definitions/ErrorResponse" - }, - "status": { - "type": "string", - "default": "SUCCESS", - "enum": [ - "SUCCESS", - "PENDING", - "FAILED" - ] - } - } - } - } - } - ] - }, - "BatchReferenceResponseAO1Result": { - "description": "Results for this specific reference.", - "format": "object", - "properties": { - "errors": { - "$ref": "#/definitions/ErrorResponse" - }, - "status": { - "type": "string", - "default": "SUCCESS", - "enum": [ - "SUCCESS", - "PENDING", - "FAILED" - ] - } - } - }, - "BatchStats": { - "description": "The summary of a nodes batch queue congestion status.", - "properties": { - "queueLength": { - "description": "How many objects are currently in the batch queue.", - "type": "number", - "format": "int", - "x-nullable": true, - "x-omitempty": true - }, - "ratePerSecond": { - "description": "How many objects are approximately processed from the batch queue per second.", - "type": "number", - "format": "int", - "x-omitempty": false - } - } - }, - "C11yExtension": { - "description": "A resource describing an extension to the contextinoary, containing both the identifier and the definition of the extension", - "properties": { - "concept": { - "description": "The new concept you want to extend. Must be an all-lowercase single word, or a space delimited compound word. Examples: 'foobarium', 'my custom concept'", - "type": "string", - "example": "foobarium" - }, - "definition": { - "description": "A list of space-delimited words or a sentence describing what the custom concept is about. Avoid using the custom concept itself. An Example definition for the custom concept 'foobarium': would be 'a naturally occurring element which can only be seen by programmers'", - "type": "string" - }, - "weight": { - "description": "Weight of the definition of the new concept where 1='override existing definition entirely' and 0='ignore custom definition'. Note that if the custom concept is not present in the contextionary yet, the weight cannot be less than 1.", - "type": "number", - "format": "float" - } - } - }, - "C11yNearestNeighbors": { - "description": "C11y function to show the nearest neighbors to a word.", - "type": "array", - "items": { - "$ref": "#/definitions/C11yNearestNeighborsItems0" - } - }, - "C11yNearestNeighborsItems0": { - "type": "object", - "properties": { - "distance": { - "type": "number", - "format": "float" - }, - "word": { - "type": "string" - } - } - }, - "C11yVector": { - "description": "A Vector in the Contextionary", - "type": "array", - "items": { - "type": "number", - "format": "float" - } - }, - "C11yVectorBasedQuestion": { - "description": "Receive question based on array of classes, properties and values.", - "type": "array", - "items": { - "$ref": "#/definitions/C11yVectorBasedQuestionItems0" - } - }, - "C11yVectorBasedQuestionItems0": { - "type": "object", - "properties": { - "classProps": { - "description": "Vectorized properties.", - "type": "array", - "maxItems": 300, - "minItems": 300, - "items": { - "$ref": "#/definitions/C11yVectorBasedQuestionItems0ClassPropsItems0" - } - }, - "classVectors": { - "description": "Vectorized classname.", - "type": "array", - "maxItems": 300, - "minItems": 300, - "items": { - "type": "number", - "format": "float" - } - } - } - }, - "C11yVectorBasedQuestionItems0ClassPropsItems0": { - "type": "object", - "properties": { - "propsVectors": { - "type": "array", - "items": { - "type": "number", - "format": "float" - } - }, - "value": { - "description": "String with valuename.", - "type": "string" - } - } - }, - "C11yWordsResponse": { - "description": "An array of available words and contexts.", - "properties": { - "concatenatedWord": { - "description": "Weighted results for all words", - "type": "object", - "properties": { - "concatenatedNearestNeighbors": { - "$ref": "#/definitions/C11yNearestNeighbors" - }, - "concatenatedVector": { - "$ref": "#/definitions/C11yVector" - }, - "concatenatedWord": { - "type": "string" - }, - "singleWords": { - "type": "array", - "items": { - "format": "string" - } - } - } - }, - "individualWords": { - "description": "Weighted results for per individual word", - "type": "array", - "items": { - "$ref": "#/definitions/C11yWordsResponseIndividualWordsItems0" - } - } - } - }, - "C11yWordsResponseConcatenatedWord": { - "description": "Weighted results for all words", - "type": "object", - "properties": { - "concatenatedNearestNeighbors": { - "$ref": "#/definitions/C11yNearestNeighbors" - }, - "concatenatedVector": { - "$ref": "#/definitions/C11yVector" - }, - "concatenatedWord": { - "type": "string" - }, - "singleWords": { - "type": "array", - "items": { - "format": "string" - } - } - } - }, - "C11yWordsResponseIndividualWordsItems0": { - "type": "object", - "properties": { - "info": { - "type": "object", - "properties": { - "nearestNeighbors": { - "$ref": "#/definitions/C11yNearestNeighbors" - }, - "vector": { - "$ref": "#/definitions/C11yVector" - } - } - }, - "present": { - "type": "boolean" - }, - "word": { - "type": "string" - } - } - }, - "C11yWordsResponseIndividualWordsItems0Info": { - "type": "object", - "properties": { - "nearestNeighbors": { - "$ref": "#/definitions/C11yNearestNeighbors" - }, - "vector": { - "$ref": "#/definitions/C11yVector" - } - } - }, - "Class": { - "type": "object", - "properties": { - "class": { - "description": "Name of the class as URI relative to the schema URL.", - "type": "string" - }, - "description": { - "description": "Description of the class.", - "type": "string" - }, - "invertedIndexConfig": { - "$ref": "#/definitions/InvertedIndexConfig" - }, - "moduleConfig": { - "description": "Configuration specific to modules this Weaviate instance has installed", - "type": "object" - }, - "multiTenancyConfig": { - "$ref": "#/definitions/MultiTenancyConfig" - }, - "properties": { - "description": "The properties of the class.", - "type": "array", - "items": { - "$ref": "#/definitions/Property" - } - }, - "replicationConfig": { - "$ref": "#/definitions/ReplicationConfig" - }, - "shardingConfig": { - "description": "Manage how the index should be sharded and distributed in the cluster", - "type": "object" - }, - "vectorIndexConfig": { - "description": "Vector-index config, that is specific to the type of index selected in vectorIndexType", - "type": "object" - }, - "vectorIndexType": { - "description": "Name of the vector index to use, eg. (HNSW)", - "type": "string" - }, - "vectorizer": { - "description": "Specify how the vectors for this class should be determined. The options are either 'none' - this means you have to import a vector with each object yourself - or the name of a module that provides vectorization capabilities, such as 'text2vec-contextionary'. If left empty, it will use the globally configured default which can itself either be 'none' or a specific module.", - "type": "string" - } - } - }, - "Classification": { - "description": "Manage classifications, trigger them and view status of past classifications.", - "type": "object", - "properties": { - "basedOnProperties": { - "description": "base the text-based classification on these fields (of type text)", - "type": "array", - "items": { - "type": "string" - }, - "example": [ - "description" - ] - }, - "class": { - "description": "class (name) which is used in this classification", - "type": "string", - "example": "City" - }, - "classifyProperties": { - "description": "which ref-property to set as part of the classification", - "type": "array", - "items": { - "type": "string" - }, - "example": [ - "inCountry" - ] - }, - "error": { - "description": "error message if status == failed", - "type": "string", - "default": "", - "example": "classify xzy: something went wrong" - }, - "filters": { - "type": "object", - "properties": { - "sourceWhere": { - "description": "limit the objects to be classified", - "type": "object", - "$ref": "#/definitions/WhereFilter" - }, - "targetWhere": { - "description": "Limit the possible sources when using an algorithm which doesn't really on training data, e.g. 'contextual'. When using an algorithm with a training set, such as 'knn', limit the training set instead", - "type": "object", - "$ref": "#/definitions/WhereFilter" - }, - "trainingSetWhere": { - "description": "Limit the training objects to be considered during the classification. Can only be used on types with explicit training sets, such as 'knn'", - "type": "object", - "$ref": "#/definitions/WhereFilter" - } - } - }, - "id": { - "description": "ID to uniquely identify this classification run", - "type": "string", - "format": "uuid", - "example": "ee722219-b8ec-4db1-8f8d-5150bb1a9e0c" - }, - "meta": { - "description": "additional meta information about the classification", - "type": "object", - "$ref": "#/definitions/ClassificationMeta" - }, - "settings": { - "description": "classification-type specific settings", - "type": "object" - }, - "status": { - "description": "status of this classification", - "type": "string", - "enum": [ - "running", - "completed", - "failed" - ], - "example": "running" - }, - "type": { - "description": "which algorithm to use for classifications", - "type": "string" - } - } - }, - "ClassificationFilters": { - "type": "object", - "properties": { - "sourceWhere": { - "description": "limit the objects to be classified", - "type": "object", - "$ref": "#/definitions/WhereFilter" - }, - "targetWhere": { - "description": "Limit the possible sources when using an algorithm which doesn't really on training data, e.g. 'contextual'. When using an algorithm with a training set, such as 'knn', limit the training set instead", - "type": "object", - "$ref": "#/definitions/WhereFilter" - }, - "trainingSetWhere": { - "description": "Limit the training objects to be considered during the classification. Can only be used on types with explicit training sets, such as 'knn'", - "type": "object", - "$ref": "#/definitions/WhereFilter" - } - } - }, - "ClassificationMeta": { - "description": "Additional information to a specific classification", - "type": "object", - "properties": { - "completed": { - "description": "time when this classification finished", - "type": "string", - "format": "date-time", - "example": "2017-07-21T17:32:28Z" - }, - "count": { - "description": "number of objects which were taken into consideration for classification", - "type": "integer", - "example": 147 - }, - "countFailed": { - "description": "number of objects which could not be classified - see error message for details", - "type": "integer", - "example": 7 - }, - "countSucceeded": { - "description": "number of objects successfully classified", - "type": "integer", - "example": 140 - }, - "started": { - "description": "time when this classification was started", - "type": "string", - "format": "date-time", - "example": "2017-07-21T17:32:28Z" - } - } - }, - "Deprecation": { - "type": "object", - "properties": { - "apiType": { - "description": "Describes which API is effected, usually one of: REST, GraphQL", - "type": "string" - }, - "id": { - "description": "The id that uniquely identifies this particular deprecations (mostly used internally)", - "type": "string" - }, - "locations": { - "description": "The locations within the specified API affected by this deprecation", - "type": "array", - "items": { - "type": "string" - } - }, - "mitigation": { - "description": "User-required object to not be affected by the (planned) removal", - "type": "string" - }, - "msg": { - "description": "What this deprecation is about", - "type": "string" - }, - "plannedRemovalVersion": { - "description": "A best-effort guess of which upcoming version will remove the feature entirely", - "type": "string" - }, - "removedIn": { - "description": "If the feature has already been removed, it was removed in this version", - "type": "string", - "x-nullable": true - }, - "removedTime": { - "description": "If the feature has already been removed, it was removed at this timestamp", - "type": "string", - "format": "date-time", - "x-nullable": true - }, - "sinceTime": { - "description": "The deprecation was introduced in this version", - "type": "string", - "format": "date-time" - }, - "sinceVersion": { - "description": "The deprecation was introduced in this version", - "type": "string" - }, - "status": { - "description": "Whether the problematic API functionality is deprecated (planned to be removed) or already removed", - "type": "string" - } - } - }, - "ErrorResponse": { - "description": "An error response given by Weaviate end-points.", - "type": "object", - "properties": { - "error": { - "type": "array", - "items": { - "$ref": "#/definitions/ErrorResponseErrorItems0" - } - } - } - }, - "ErrorResponseErrorItems0": { - "type": "object", - "properties": { - "message": { - "type": "string" - } - } - }, - "GeoCoordinates": { - "properties": { - "latitude": { - "description": "The latitude of the point on earth in decimal form", - "type": "number", - "format": "float", - "x-nullable": true - }, - "longitude": { - "description": "The longitude of the point on earth in decimal form", - "type": "number", - "format": "float", - "x-nullable": true - } - } - }, - "GraphQLError": { - "description": "An error response caused by a GraphQL query.", - "properties": { - "locations": { - "type": "array", - "items": { - "$ref": "#/definitions/GraphQLErrorLocationsItems0" - } - }, - "message": { - "type": "string" - }, - "path": { - "type": "array", - "items": { - "type": "string" - } - } - } - }, - "GraphQLErrorLocationsItems0": { - "type": "object", - "properties": { - "column": { - "type": "integer", - "format": "int64" - }, - "line": { - "type": "integer", - "format": "int64" - } - } - }, - "GraphQLQueries": { - "description": "A list of GraphQL queries.", - "type": "array", - "items": { - "$ref": "#/definitions/GraphQLQuery" - } - }, - "GraphQLQuery": { - "description": "GraphQL query based on: http://facebook.github.io/graphql/.", - "type": "object", - "properties": { - "operationName": { - "description": "The name of the operation if multiple exist in the query.", - "type": "string" - }, - "query": { - "description": "Query based on GraphQL syntax.", - "type": "string" - }, - "variables": { - "description": "Additional variables for the query.", - "type": "object" - } - } - }, - "GraphQLResponse": { - "description": "GraphQL based response: http://facebook.github.io/graphql/.", - "properties": { - "data": { - "description": "GraphQL data object.", - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/JsonObject" - } - }, - "errors": { - "description": "Array with errors.", - "type": "array", - "items": { - "$ref": "#/definitions/GraphQLError" - }, - "x-omitempty": true - } - } - }, - "GraphQLResponses": { - "description": "A list of GraphQL responses.", - "type": "array", - "items": { - "$ref": "#/definitions/GraphQLResponse" - } - }, - "InvertedIndexConfig": { - "description": "Configure the inverted index built into Weaviate", - "type": "object", - "properties": { - "bm25": { - "$ref": "#/definitions/BM25Config" - }, - "cleanupIntervalSeconds": { - "description": "Asynchronous index clean up happens every n seconds", - "type": "number", - "format": "int" - }, - "indexNullState": { - "description": "Index each object with the null state", - "type": "boolean" - }, - "indexPropertyLength": { - "description": "Index length of properties", - "type": "boolean" - }, - "indexTimestamps": { - "description": "Index each object by its internal timestamps", - "type": "boolean" - }, - "stopwords": { - "$ref": "#/definitions/StopwordConfig" - } - } - }, - "JsonObject": { - "description": "JSON object value.", - "type": "object" - }, - "Link": { - "type": "object", - "properties": { - "documentationHref": { - "description": "weaviate documentation about this resource group", - "type": "string" - }, - "href": { - "description": "target of the link", - "type": "string" - }, - "name": { - "description": "human readable name of the resource group", - "type": "string" - }, - "rel": { - "description": "relationship if both resources are related, e.g. 'next', 'previous', 'parent', etc.", - "type": "string" - } - } - }, - "Meta": { - "description": "Contains meta information of the current Weaviate instance.", - "type": "object", - "properties": { - "hostname": { - "description": "The url of the host.", - "type": "string", - "format": "url" - }, - "modules": { - "description": "Module-specific meta information", - "type": "object" - }, - "version": { - "description": "Version of weaviate you are currently running", - "type": "string" - } - } - }, - "MultiTenancyConfig": { - "description": "Configuration related to multi-tenancy within a class", - "properties": { - "enabled": { - "description": "Whether or not multi-tenancy is enabled for this class", - "type": "boolean", - "x-omitempty": false - } - } - }, - "MultipleRef": { - "description": "Multiple instances of references to other objects.", - "type": "array", - "items": { - "$ref": "#/definitions/SingleRef" - } - }, - "NestedProperty": { - "type": "object", - "properties": { - "dataType": { - "type": "array", - "items": { - "type": "string" - } - }, - "description": { - "type": "string" - }, - "indexFilterable": { - "type": "boolean", - "x-nullable": true - }, - "indexSearchable": { - "type": "boolean", - "x-nullable": true - }, - "name": { - "type": "string" - }, - "nestedProperties": { - "type": "array", - "items": { - "$ref": "#/definitions/NestedProperty" - }, - "x-omitempty": true - }, - "tokenization": { - "type": "string", - "enum": [ - "word", - "lowercase", - "whitespace", - "field" - ] - } - } - }, - "NodeShardStatus": { - "description": "The definition of a node shard status response body", - "properties": { - "class": { - "description": "The name of shard's class.", - "type": "string", - "x-omitempty": false - }, - "compressed": { - "description": "The status of vector compression/quantization.", - "format": "boolean", - "x-omitempty": false - }, - "name": { - "description": "The name of the shard.", - "type": "string", - "x-omitempty": false - }, - "objectCount": { - "description": "The number of objects in shard.", - "type": "number", - "format": "int64", - "x-omitempty": false - }, - "vectorIndexingStatus": { - "description": "The status of the vector indexing process.", - "format": "string", - "x-omitempty": false - }, - "vectorQueueLength": { - "description": "The length of the vector indexing queue.", - "type": "number", - "format": "int64", - "x-omitempty": false - } - } - }, - "NodeStats": { - "description": "The summary of Weaviate's statistics.", - "properties": { - "objectCount": { - "description": "The total number of objects in DB.", - "type": "number", - "format": "int64", - "x-omitempty": false - }, - "shardCount": { - "description": "The count of Weaviate's shards.", - "type": "number", - "format": "int", - "x-omitempty": false - } - } - }, - "NodeStatus": { - "description": "The definition of a backup node status response body", - "properties": { - "batchStats": { - "description": "Weaviate batch statistics.", - "type": "object", - "$ref": "#/definitions/BatchStats" - }, - "gitHash": { - "description": "The gitHash of Weaviate.", - "type": "string" - }, - "name": { - "description": "The name of the node.", - "type": "string" - }, - "shards": { - "description": "The list of the shards with it's statistics.", - "type": "array", - "items": { - "$ref": "#/definitions/NodeShardStatus" - } - }, - "stats": { - "description": "Weaviate overall statistics.", - "type": "object", - "$ref": "#/definitions/NodeStats" - }, - "status": { - "description": "Node's status.", - "type": "string", - "default": "HEALTHY", - "enum": [ - "HEALTHY", - "UNHEALTHY", - "UNAVAILABLE" - ] - }, - "version": { - "description": "The version of Weaviate.", - "type": "string" - } - } - }, - "NodesStatusResponse": { - "description": "The status of all of the Weaviate nodes", - "type": "object", - "properties": { - "nodes": { - "type": "array", - "items": { - "$ref": "#/definitions/NodeStatus" - } - } - } - }, - "Object": { - "type": "object", - "properties": { - "additional": { - "$ref": "#/definitions/AdditionalProperties" - }, - "class": { - "description": "Class of the Object, defined in the schema.", - "type": "string" - }, - "creationTimeUnix": { - "description": "Timestamp of creation of this Object in milliseconds since epoch UTC.", - "type": "integer", - "format": "int64" - }, - "id": { - "description": "ID of the Object.", - "type": "string", - "format": "uuid" - }, - "lastUpdateTimeUnix": { - "description": "Timestamp of the last Object update in milliseconds since epoch UTC.", - "type": "integer", - "format": "int64" - }, - "properties": { - "$ref": "#/definitions/PropertySchema" - }, - "tenant": { - "description": "Name of the Objects tenant.", - "type": "string" - }, - "vector": { - "description": "This object's position in the Contextionary vector space. Read-only if using a vectorizer other than 'none'. Writable and required if using 'none' as vectorizer.", - "$ref": "#/definitions/C11yVector" - }, - "vectorWeights": { - "$ref": "#/definitions/VectorWeights" - } - } - }, - "ObjectsGetResponse": { - "type": "object", - "allOf": [ - { - "$ref": "#/definitions/Object" - }, - { - "properties": { - "deprecations": { - "type": "array", - "items": { - "$ref": "#/definitions/Deprecation" - } - } - } - }, - { - "properties": { - "result": { - "description": "Results for this specific Object.", - "format": "object", - "properties": { - "errors": { - "$ref": "#/definitions/ErrorResponse" - }, - "status": { - "type": "string", - "default": "SUCCESS", - "enum": [ - "SUCCESS", - "PENDING", - "FAILED" - ] - } - } - } - } - } - ] - }, - "ObjectsGetResponseAO2Result": { - "description": "Results for this specific Object.", - "format": "object", - "properties": { - "errors": { - "$ref": "#/definitions/ErrorResponse" - }, - "status": { - "type": "string", - "default": "SUCCESS", - "enum": [ - "SUCCESS", - "PENDING", - "FAILED" - ] - } - } - }, - "ObjectsListResponse": { - "description": "List of Objects.", - "type": "object", - "properties": { - "deprecations": { - "type": "array", - "items": { - "$ref": "#/definitions/Deprecation" - } - }, - "objects": { - "description": "The actual list of Objects.", - "type": "array", - "items": { - "$ref": "#/definitions/Object" - } - }, - "totalResults": { - "description": "The total number of Objects for the query. The number of items in a response may be smaller due to paging.", - "type": "integer", - "format": "int64" - } - } - }, - "PatchDocumentAction": { - "description": "Either a JSONPatch document as defined by RFC 6902 (from, op, path, value), or a merge document (RFC 7396).", - "required": [ - "op", - "path" - ], - "properties": { - "from": { - "description": "A string containing a JSON Pointer value.", - "type": "string" - }, - "merge": { - "$ref": "#/definitions/Object" - }, - "op": { - "description": "The operation to be performed.", - "type": "string", - "enum": [ - "add", - "remove", - "replace", - "move", - "copy", - "test" - ] - }, - "path": { - "description": "A JSON-Pointer.", - "type": "string" - }, - "value": { - "description": "The value to be used within the operations.", - "type": "object" - } - } - }, - "PatchDocumentObject": { - "description": "Either a JSONPatch document as defined by RFC 6902 (from, op, path, value), or a merge document (RFC 7396).", - "required": [ - "op", - "path" - ], - "properties": { - "from": { - "description": "A string containing a JSON Pointer value.", - "type": "string" - }, - "merge": { - "$ref": "#/definitions/Object" - }, - "op": { - "description": "The operation to be performed.", - "type": "string", - "enum": [ - "add", - "remove", - "replace", - "move", - "copy", - "test" - ] - }, - "path": { - "description": "A JSON-Pointer.", - "type": "string" - }, - "value": { - "description": "The value to be used within the operations.", - "type": "object" - } - } - }, - "PeerUpdate": { - "description": "A single peer in the network.", - "properties": { - "id": { - "description": "The session ID of the peer.", - "type": "string", - "format": "uuid" - }, - "name": { - "description": "Human readable name.", - "type": "string" - }, - "schemaHash": { - "description": "The latest known hash of the peer's schema.", - "type": "string" - }, - "uri": { - "description": "The location where the peer is exposed to the internet.", - "type": "string", - "format": "uri" - } - } - }, - "PeerUpdateList": { - "description": "List of known peers.", - "type": "array", - "items": { - "$ref": "#/definitions/PeerUpdate" - } - }, - "PhoneNumber": { - "properties": { - "countryCode": { - "description": "Read-only. The numerical country code (e.g. 49)", - "type": "number", - "format": "uint64" - }, - "defaultCountry": { - "description": "Optional. The ISO 3166-1 alpha-2 country code. This is used to figure out the correct countryCode and international format if only a national number (e.g. 0123 4567) is provided", - "type": "string" - }, - "input": { - "description": "The raw input as the phone number is present in your raw data set. It will be parsed into the standardized formats if valid.", - "type": "string" - }, - "internationalFormatted": { - "description": "Read-only. Parsed result in the international format (e.g. +49 123 ...)", - "type": "string" - }, - "national": { - "description": "Read-only. The numerical representation of the national part", - "type": "number", - "format": "uint64" - }, - "nationalFormatted": { - "description": "Read-only. Parsed result in the national format (e.g. 0123 456789)", - "type": "string" - }, - "valid": { - "description": "Read-only. Indicates whether the parsed number is a valid phone number", - "type": "boolean" - } - } - }, - "Principal": { - "type": "object", - "properties": { - "groups": { - "type": "array", - "items": { - "type": "string" - } - }, - "username": { - "description": "The username that was extracted either from the authentication information", - "type": "string" - } - } - }, - "Property": { - "type": "object", - "properties": { - "dataType": { - "description": "Can be a reference to another type when it starts with a capital (for example Person), otherwise \"string\" or \"int\".", - "type": "array", - "items": { - "type": "string" - } - }, - "description": { - "description": "Description of the property.", - "type": "string" - }, - "indexFilterable": { - "description": "Optional. Should this property be indexed in the inverted index. Defaults to true. If you choose false, you will not be able to use this property in where filters. This property has no affect on vectorization decisions done by modules", - "type": "boolean", - "x-nullable": true - }, - "indexInverted": { - "description": "Optional. Should this property be indexed in the inverted index. Defaults to true. If you choose false, you will not be able to use this property in where filters, bm25 or hybrid search. This property has no affect on vectorization decisions done by modules (deprecated as of v1.19; use indexFilterable or/and indexSearchable instead)", - "type": "boolean", - "x-nullable": true - }, - "indexSearchable": { - "description": "Optional. Should this property be indexed in the inverted index. Defaults to true. Applicable only to properties of data type text and text[]. If you choose false, you will not be able to use this property in bm25 or hybrid search. This property has no affect on vectorization decisions done by modules", - "type": "boolean", - "x-nullable": true - }, - "moduleConfig": { - "description": "Configuration specific to modules this Weaviate instance has installed", - "type": "object" - }, - "name": { - "description": "Name of the property as URI relative to the schema URL.", - "type": "string" - }, - "nestedProperties": { - "description": "The properties of the nested object(s). Applies to object and object[] data types.", - "type": "array", - "items": { - "$ref": "#/definitions/NestedProperty" - }, - "x-omitempty": true - }, - "tokenization": { - "description": "Determines tokenization of the property as separate words or whole field. Optional. Applies to text and text[] data types. Allowed values are ` + "`" + `word` + "`" + ` (default; splits on any non-alphanumerical, lowercases), ` + "`" + `lowercase` + "`" + ` (splits on white spaces, lowercases), ` + "`" + `whitespace` + "`" + ` (splits on white spaces), ` + "`" + `field` + "`" + ` (trims). Not supported for remaining data types", - "type": "string", - "enum": [ - "word", - "lowercase", - "whitespace", - "field" - ] - } - } - }, - "PropertySchema": { - "description": "This is an open object, with OpenAPI Specification 3.0 this will be more detailed. See Weaviate docs for more info. In the future this will become a key/value OR a SingleRef definition.", - "type": "object" - }, - "ReferenceMetaClassification": { - "description": "This meta field contains additional info about the classified reference property", - "properties": { - "closestLosingDistance": { - "description": "The lowest distance of a neighbor in the losing group. Optional. If k equals the size of the winning group, there is no losing group", - "type": "number", - "format": "float32", - "x-nullable": true - }, - "closestOverallDistance": { - "description": "The lowest distance of any neighbor, regardless of whether they were in the winning or losing group", - "type": "number", - "format": "float32" - }, - "closestWinningDistance": { - "description": "Closest distance of a neighbor from the winning group", - "type": "number", - "format": "float32" - }, - "losingCount": { - "description": "size of the losing group, can be 0 if the winning group size equals k", - "type": "number", - "format": "int64" - }, - "losingDistance": { - "description": "deprecated - do not use, to be removed in 0.23.0", - "type": "number", - "format": "float32", - "x-nullable": true - }, - "meanLosingDistance": { - "description": "Mean distance of all neighbors from the losing group. Optional. If k equals the size of the winning group, there is no losing group.", - "type": "number", - "format": "float32", - "x-nullable": true - }, - "meanWinningDistance": { - "description": "Mean distance of all neighbors from the winning group", - "type": "number", - "format": "float32" - }, - "overallCount": { - "description": "overall neighbors checked as part of the classification. In most cases this will equal k, but could be lower than k - for example if not enough data was present", - "type": "number", - "format": "int64" - }, - "winningCount": { - "description": "size of the winning group, a number between 1..k", - "type": "number", - "format": "int64" - }, - "winningDistance": { - "description": "deprecated - do not use, to be removed in 0.23.0", - "type": "number", - "format": "float32" - } - } - }, - "ReplicationConfig": { - "description": "Configure how replication is executed in a cluster", - "type": "object", - "properties": { - "factor": { - "description": "Number of times a class is replicated", - "type": "integer" - } - } - }, - "RestoreConfig": { - "description": "Backup custom configuration", - "type": "object", - "properties": { - "CPUPercentage": { - "description": "Desired CPU core utilization ranging from 1%-80%", - "type": "integer", - "default": 50, - "maximum": 80, - "minimum": 1, - "x-nullable": false - } - } - }, - "Schema": { - "description": "Definitions of semantic schemas (also see: https://github.com/weaviate/weaviate-semantic-schemas).", - "type": "object", - "properties": { - "classes": { - "description": "Semantic classes that are available.", - "type": "array", - "items": { - "$ref": "#/definitions/Class" - } - }, - "maintainer": { - "description": "Email of the maintainer.", - "type": "string", - "format": "email" - }, - "name": { - "description": "Name of the schema.", - "type": "string" - } - } - }, - "SchemaClusterStatus": { - "description": "Indicates the health of the schema in a cluster.", - "type": "object", - "properties": { - "error": { - "description": "Contains the sync check error if one occurred", - "type": "string", - "x-omitempty": true - }, - "healthy": { - "description": "True if the cluster is in sync, false if there is an issue (see error).", - "type": "boolean", - "x-omitempty": false - }, - "hostname": { - "description": "Hostname of the coordinating node, i.e. the one that received the cluster. This can be useful information if the error message contains phrases such as 'other nodes agree, but local does not', etc.", - "type": "string" - }, - "ignoreSchemaSync": { - "description": "The cluster check at startup can be ignored (to recover from an out-of-sync situation).", - "type": "boolean", - "x-omitempty": false - }, - "nodeCount": { - "description": "Number of nodes that participated in the sync check", - "type": "number", - "format": "int" - } - } - }, - "SchemaHistory": { - "description": "This is an open object, with OpenAPI Specification 3.0 this will be more detailed. See Weaviate docs for more info. In the future this will become a key/value OR a SingleRef definition.", - "type": "object" - }, - "ShardStatus": { - "description": "The status of a single shard", - "properties": { - "status": { - "description": "Status of the shard", - "type": "string" - } - } - }, - "ShardStatusGetResponse": { - "description": "Response body of shard status get request", - "properties": { - "name": { - "description": "Name of the shard", - "type": "string" - }, - "status": { - "description": "Status of the shard", - "type": "string" - }, - "vectorQueueSize": { - "description": "Size of the vector queue of the shard", - "type": "integer", - "x-omitempty": false - } - } - }, - "ShardStatusList": { - "description": "The status of all the shards of a Class", - "type": "array", - "items": { - "$ref": "#/definitions/ShardStatusGetResponse" - } - }, - "SingleRef": { - "description": "Either set beacon (direct reference) or set class and schema (concept reference)", - "properties": { - "beacon": { - "description": "If using a direct reference, specify the URI to point to the cross-ref here. Should be in the form of weaviate://localhost/\u003cuuid\u003e for the example of a local cross-ref to an object", - "type": "string", - "format": "uri" - }, - "class": { - "description": "If using a concept reference (rather than a direct reference), specify the desired class name here", - "type": "string", - "format": "uri" - }, - "classification": { - "description": "Additional Meta information about classifications if the item was part of one", - "$ref": "#/definitions/ReferenceMetaClassification" - }, - "href": { - "description": "If using a direct reference, this read-only fields provides a link to the referenced resource. If 'origin' is globally configured, an absolute URI is shown - a relative URI otherwise.", - "type": "string", - "format": "uri" - }, - "schema": { - "description": "If using a concept reference (rather than a direct reference), specify the desired properties here", - "$ref": "#/definitions/PropertySchema" - } - } - }, - "StopwordConfig": { - "description": "fine-grained control over stopword list usage", - "type": "object", - "properties": { - "additions": { - "description": "stopwords to be considered additionally", - "type": "array", - "items": { - "type": "string" - } - }, - "preset": { - "description": "pre-existing list of common words by language", - "type": "string" - }, - "removals": { - "description": "stopwords to be removed from consideration", - "type": "array", - "items": { - "type": "string" - } - } - } - }, - "Tenant": { - "description": "attributes representing a single tenant within weaviate", - "type": "object", - "properties": { - "activityStatus": { - "description": "activity status of the tenant's shard. Optional for creating tenant (implicit ` + "`" + `HOT` + "`" + `) and required for updating tenant. Allowed values are ` + "`" + `HOT` + "`" + ` - tenant is fully active, ` + "`" + `WARM` + "`" + ` - tenant is active, some restrictions are imposed (TBD; not supported yet), ` + "`" + `COLD` + "`" + ` - tenant is inactive; no actions can be performed on tenant, tenant's files are stored locally, ` + "`" + `FROZEN` + "`" + ` - as COLD, but files are stored on cloud storage (not supported yet)", - "type": "string", - "enum": [ - "HOT", - "WARM", - "COLD", - "FROZEN" - ] - }, - "name": { - "description": "name of the tenant", - "type": "string" - } - } - }, - "VectorWeights": { - "description": "Allow custom overrides of vector weights as math expressions. E.g. \"pancake\": \"7\" will set the weight for the word pancake to 7 in the vectorization, whereas \"w * 3\" would triple the originally calculated word. This is an open object, with OpenAPI Specification 3.0 this will be more detailed. See Weaviate docs for more info. In the future this will become a key/value (string/string) object.", - "type": "object" - }, - "WhereFilter": { - "description": "Filter search results using a where filter", - "type": "object", - "properties": { - "operands": { - "description": "combine multiple where filters, requires 'And' or 'Or' operator", - "type": "array", - "items": { - "$ref": "#/definitions/WhereFilter" - } - }, - "operator": { - "description": "operator to use", - "type": "string", - "enum": [ - "And", - "Or", - "Equal", - "Like", - "NotEqual", - "GreaterThan", - "GreaterThanEqual", - "LessThan", - "LessThanEqual", - "WithinGeoRange", - "IsNull", - "ContainsAny", - "ContainsAll" - ], - "example": "GreaterThanEqual" - }, - "path": { - "description": "path to the property currently being filtered", - "type": "array", - "items": { - "type": "string" - }, - "example": [ - "inCity", - "City", - "name" - ] - }, - "valueBoolean": { - "description": "value as boolean", - "type": "boolean", - "x-nullable": true, - "example": false - }, - "valueBooleanArray": { - "description": "value as boolean", - "type": "array", - "items": { - "type": "boolean" - }, - "x-nullable": true, - "x-omitempty": true, - "example": [ - true, - false - ] - }, - "valueDate": { - "description": "value as date (as string)", - "type": "string", - "x-nullable": true, - "example": "TODO" - }, - "valueDateArray": { - "description": "value as date (as string)", - "type": "array", - "items": { - "type": "string" - }, - "x-nullable": true, - "x-omitempty": true, - "example": "TODO" - }, - "valueGeoRange": { - "description": "value as geo coordinates and distance", - "type": "object", - "x-nullable": true, - "$ref": "#/definitions/WhereFilterGeoRange" - }, - "valueInt": { - "description": "value as integer", - "type": "integer", - "format": "int64", - "x-nullable": true, - "example": 2000 - }, - "valueIntArray": { - "description": "value as integer", - "type": "array", - "items": { - "type": "integer", - "format": "int64" - }, - "x-nullable": true, - "x-omitempty": true, - "example": "[100, 200]" - }, - "valueNumber": { - "description": "value as number/float", - "type": "number", - "format": "float64", - "x-nullable": true, - "example": 3.14 - }, - "valueNumberArray": { - "description": "value as number/float", - "type": "array", - "items": { - "type": "number", - "format": "float64" - }, - "x-nullable": true, - "x-omitempty": true, - "example": [ - 3.14 - ] - }, - "valueString": { - "description": "value as text (deprecated as of v1.19; alias for valueText)", - "type": "string", - "x-nullable": true, - "example": "my search term" - }, - "valueStringArray": { - "description": "value as text (deprecated as of v1.19; alias for valueText)", - "type": "array", - "items": { - "type": "string" - }, - "x-nullable": true, - "x-omitempty": true, - "example": [ - "my search term" - ] - }, - "valueText": { - "description": "value as text", - "type": "string", - "x-nullable": true, - "example": "my search term" - }, - "valueTextArray": { - "description": "value as text", - "type": "array", - "items": { - "type": "string" - }, - "x-nullable": true, - "x-omitempty": true, - "example": [ - "my search term" - ] - } - } - }, - "WhereFilterGeoRange": { - "description": "filter within a distance of a georange", - "type": "object", - "properties": { - "distance": { - "type": "object", - "properties": { - "max": { - "type": "number", - "format": "float64" - } - } - }, - "geoCoordinates": { - "x-nullable": false, - "$ref": "#/definitions/GeoCoordinates" - } - } - }, - "WhereFilterGeoRangeDistance": { - "type": "object", - "properties": { - "max": { - "type": "number", - "format": "float64" - } - } - } - }, - "parameters": { - "CommonAfterParameterQuery": { - "type": "string", - "description": "The starting ID of the result window.", - "name": "after", - "in": "query" - }, - "CommonClassParameterQuery": { - "type": "string", - "description": "Class parameter specifies the class from which to query objects", - "name": "class", - "in": "query" - }, - "CommonConsistencyLevelParameterQuery": { - "type": "string", - "description": "Determines how many replicas must acknowledge a request before it is considered successful", - "name": "consistency_level", - "in": "query" - }, - "CommonIncludeParameterQuery": { - "type": "string", - "description": "Include additional information, such as classification infos. Allowed values include: classification, vector, interpretation", - "name": "include", - "in": "query" - }, - "CommonLimitParameterQuery": { - "type": "integer", - "format": "int64", - "description": "The maximum number of items to be returned per page. Default value is set in Weaviate config.", - "name": "limit", - "in": "query" - }, - "CommonNodeNameParameterQuery": { - "type": "string", - "description": "The target node which should fulfill the request", - "name": "node_name", - "in": "query" - }, - "CommonOffsetParameterQuery": { - "type": "integer", - "format": "int64", - "default": 0, - "description": "The starting index of the result window. Default value is 0.", - "name": "offset", - "in": "query" - }, - "CommonOrderParameterQuery": { - "type": "string", - "description": "Order parameter to tell how to order (asc or desc) data within given field", - "name": "order", - "in": "query" - }, - "CommonOutputVerbosityParameterQuery": { - "type": "string", - "default": "minimal", - "description": "Controls the verbosity of the output, possible values are: \"minimal\", \"verbose\". Defaults to \"minimal\".", - "name": "output", - "in": "query" - }, - "CommonSortParameterQuery": { - "type": "string", - "description": "Sort parameter to pass an information about the names of the sort fields", - "name": "sort", - "in": "query" - }, - "CommonTenantParameterQuery": { - "type": "string", - "description": "Specifies the tenant in a request targeting a multi-tenant class", - "name": "tenant", - "in": "query" - } - }, - "securityDefinitions": { - "oidc": { - "description": "OIDC (OpenConnect ID - based on OAuth2)", - "type": "oauth2", - "flow": "implicit", - "authorizationUrl": "http://to-be-configured-in-the-application-config" - } - }, - "security": [ - {}, - { - "oidc": [] - } - ], - "tags": [ - { - "name": "objects" - }, - { - "description": "These operations allow to execute batch requests for Objects and Objects. Mostly used for importing large datasets.", - "name": "batch" - }, - { - "name": "graphql" - }, - { - "name": "meta" - }, - { - "name": "P2P" - }, - { - "description": "All functions related to the Contextionary.", - "name": "contextionary-API" - }, - { - "description": "These operations enable manipulation of the schema in Weaviate schema.", - "name": "schema" - } - ], - "externalDocs": { - "url": "https://github.com/weaviate/weaviate" - } -}`)) -} diff --git a/adapters/handlers/rest/filterext/parse.go b/adapters/handlers/rest/filterext/parse.go deleted file mode 100644 index 8d300fe40bc51715c80792e9f26e95ef0c76cba3..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/filterext/parse.go +++ /dev/null @@ -1,180 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package filterext - -import ( - "fmt" - - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" -) - -// Parse Filter from REST construct to entities filter -func Parse(in *models.WhereFilter, rootClass string) (*filters.LocalFilter, error) { - if in == nil { - return nil, nil - } - - operator, err := parseOperator(in.Operator) - if err != nil { - return nil, err - } - - if operator.OnValue() { - filter, err := parseValueFilter(in, operator, rootClass) - if err != nil { - return nil, fmt.Errorf("invalid where filter: %v", err) - } - return filter, nil - } - - filter, err := parseNestedFilter(in, operator, rootClass) - if err != nil { - return nil, fmt.Errorf("invalid where filter: %v", err) - } - return filter, nil -} - -func parseValueFilter(in *models.WhereFilter, - operator filters.Operator, rootClass string, -) (*filters.LocalFilter, error) { - value, err := parseValue(in) - if err != nil { - return nil, err - } - - path, err := parsePath(in.Path, rootClass) - if err != nil { - return nil, err - } - - return &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: operator, - Value: value, - On: path, - }, - }, nil -} - -func parseNestedFilter(in *models.WhereFilter, - operator filters.Operator, rootClass string, -) (*filters.LocalFilter, error) { - if in.Path != nil { - return nil, fmt.Errorf( - "operator '%s' not compatible with field 'path', remove 'path' "+ - "or switch to compare operator (eg. Equal, NotEqual, etc.)", - operator.Name()) - } - - if !allValuesNil(in) { - return nil, fmt.Errorf( - "operator '%s' not compatible with field 'value', "+ - "remove value field or switch to compare operator "+ - "(eg. Equal, NotEqual, etc.)", - operator.Name()) - } - - if in.Operands == nil || len(in.Operands) == 0 { - return nil, fmt.Errorf( - "operator '%s', but no operands set - add at least one operand", - operator.Name()) - } - - operands, err := parseOperands(in.Operands, rootClass) - if err != nil { - return nil, err - } - - return &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: operator, - Operands: operands, - }, - }, nil -} - -func parseOperands(ops []*models.WhereFilter, rootClass string) ([]filters.Clause, error) { - out := make([]filters.Clause, len(ops)) - for i, operand := range ops { - res, err := Parse(operand, rootClass) - if err != nil { - return nil, fmt.Errorf("operand %d: %v", i, err) - } - - out[i] = *res.Root - } - - return out, nil -} - -func parseOperator(in string) (filters.Operator, error) { - switch in { - case models.WhereFilterOperatorEqual: - return filters.OperatorEqual, nil - case models.WhereFilterOperatorLike: - return filters.OperatorLike, nil - case models.WhereFilterOperatorLessThan: - return filters.OperatorLessThan, nil - case models.WhereFilterOperatorLessThanEqual: - return filters.OperatorLessThanEqual, nil - case models.WhereFilterOperatorGreaterThan: - return filters.OperatorGreaterThan, nil - case models.WhereFilterOperatorGreaterThanEqual: - return filters.OperatorGreaterThanEqual, nil - case models.WhereFilterOperatorNotEqual: - return filters.OperatorNotEqual, nil - case models.WhereFilterOperatorWithinGeoRange: - return filters.OperatorWithinGeoRange, nil - case models.WhereFilterOperatorAnd: - return filters.OperatorAnd, nil - case models.WhereFilterOperatorOr: - return filters.OperatorOr, nil - case models.WhereFilterOperatorIsNull: - return filters.OperatorIsNull, nil - case models.WhereFilterOperatorContainsAny: - return filters.ContainsAny, nil - case models.WhereFilterOperatorContainsAll: - return filters.ContainsAll, nil - default: - return -1, fmt.Errorf("unrecognized operator: %s", in) - } -} - -func parsePath(in []string, rootClass string) (*filters.Path, error) { - if len(in) == 0 { - return nil, fmt.Errorf("field 'path': must have at least one element") - } - - pathElements := make([]interface{}, len(in)) - for i, elem := range in { - pathElements[i] = elem - } - - return filters.ParsePath(pathElements, rootClass) -} - -func allValuesNil(in *models.WhereFilter) bool { - return in.ValueBoolean == nil && - in.ValueDate == nil && - in.ValueString == nil && - in.ValueText == nil && - in.ValueInt == nil && - in.ValueNumber == nil && - in.ValueGeoRange == nil && - len(in.ValueBooleanArray) == 0 && - len(in.ValueDateArray) == 0 && - len(in.ValueStringArray) == 0 && - len(in.ValueTextArray) == 0 && - len(in.ValueIntArray) == 0 && - len(in.ValueNumberArray) == 0 -} diff --git a/adapters/handlers/rest/filterext/parse_test.go b/adapters/handlers/rest/filterext/parse_test.go deleted file mode 100644 index 1e06297ca0512cf88c1e4dcacc4cf03107ad29b6..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/filterext/parse_test.go +++ /dev/null @@ -1,520 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package filterext - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" -) - -func Test_ExtractFlatFilters(t *testing.T) { - t.Parallel() - - type test struct { - name string - input *models.WhereFilter - expectedFilter *filters.LocalFilter - expectedErr error - } - - t.Run("all value types", func(t *testing.T) { - tests := []test{ - { - name: "no filter", - }, - { - name: "valid int filter", - input: &models.WhereFilter{ - Operator: "Equal", - ValueInt: ptInt(42), - Path: []string{"intField"}, - }, - expectedFilter: &filters.LocalFilter{Root: &filters.Clause{ - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: schema.AssertValidClassName("Todo"), - Property: schema.AssertValidPropertyName("intField"), - }, - Value: &filters.Value{ - Value: 42, - Type: schema.DataTypeInt, - }, - }}, - }, - { - name: "valid date filter", - input: &models.WhereFilter{ - Operator: "Equal", - ValueDate: ptString("foo bar"), - Path: []string{"dateField"}, - }, - expectedFilter: &filters.LocalFilter{Root: &filters.Clause{ - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: schema.AssertValidClassName("Todo"), - Property: schema.AssertValidPropertyName("dateField"), - }, - Value: &filters.Value{ - Value: "foo bar", - Type: schema.DataTypeDate, - }, - }}, - }, - { - name: "valid text filter", - input: &models.WhereFilter{ - Operator: "Equal", - ValueText: ptString("foo bar"), - Path: []string{"textField"}, - }, - expectedFilter: &filters.LocalFilter{Root: &filters.Clause{ - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: schema.AssertValidClassName("Todo"), - Property: schema.AssertValidPropertyName("textField"), - }, - Value: &filters.Value{ - Value: "foo bar", - Type: schema.DataTypeText, - }, - }}, - }, - { - name: "valid number filter", - input: &models.WhereFilter{ - Operator: "Equal", - ValueNumber: ptFloat(20.20), - Path: []string{"numberField"}, - }, - expectedFilter: &filters.LocalFilter{Root: &filters.Clause{ - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: schema.AssertValidClassName("Todo"), - Property: schema.AssertValidPropertyName("numberField"), - }, - Value: &filters.Value{ - Value: 20.20, - Type: schema.DataTypeNumber, - }, - }}, - }, - { - name: "valid bool filter", - input: &models.WhereFilter{ - Operator: "Equal", - ValueBoolean: ptBool(true), - Path: []string{"booleanField"}, - }, - expectedFilter: &filters.LocalFilter{Root: &filters.Clause{ - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: schema.AssertValidClassName("Todo"), - Property: schema.AssertValidPropertyName("booleanField"), - }, - Value: &filters.Value{ - Value: true, - Type: schema.DataTypeBoolean, - }, - }}, - }, - { - name: "valid geo range filter", - input: &models.WhereFilter{ - Operator: "WithinGeoRange", - ValueGeoRange: inputGeoRangeFilter(0.5, 0.6, 2.0), - Path: []string{"geoField"}, - }, - expectedFilter: &filters.LocalFilter{Root: &filters.Clause{ - Operator: filters.OperatorWithinGeoRange, - On: &filters.Path{ - Class: schema.AssertValidClassName("Todo"), - Property: schema.AssertValidPropertyName("geoField"), - }, - Value: &filters.Value{ - Value: filters.GeoRange{ - GeoCoordinates: &models.GeoCoordinates{ - Latitude: ptFloat32(0.5), - Longitude: ptFloat32(0.6), - }, - Distance: 2.0, - }, - Type: schema.DataTypeGeoCoordinates, - }, - }}, - }, - { - name: "[deprecated string] valid string filter", - input: &models.WhereFilter{ - Operator: "Equal", - ValueString: ptString("foo bar"), - Path: []string{"stringField"}, - }, - expectedFilter: &filters.LocalFilter{Root: &filters.Clause{ - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: schema.AssertValidClassName("Todo"), - Property: schema.AssertValidPropertyName("stringField"), - }, - Value: &filters.Value{ - Value: "foo bar", - Type: schema.DataTypeString, - }, - }}, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - filter, err := Parse(test.input, "Todo") - assert.Equal(t, test.expectedErr, err) - assert.Equal(t, test.expectedFilter, filter) - }) - } - }) - - t.Run("invalid cases", func(t *testing.T) { - tests := []test{ - { - name: "geo missing coordinates", - input: &models.WhereFilter{ - Operator: "WithinGeoRange", - ValueGeoRange: &models.WhereFilterGeoRange{ - Distance: &models.WhereFilterGeoRangeDistance{ - Max: 20.0, - }, - }, - Path: []string{"geoField"}, - }, - expectedErr: fmt.Errorf("invalid where filter: valueGeoRange: " + - "field 'geoCoordinates' must be set"), - }, - { - name: "geo missing distance object", - input: &models.WhereFilter{ - Operator: "WithinGeoRange", - ValueGeoRange: &models.WhereFilterGeoRange{ - GeoCoordinates: &models.GeoCoordinates{ - Latitude: ptFloat32(4.5), - Longitude: ptFloat32(3.7), - }, - }, - Path: []string{"geoField"}, - }, - expectedErr: fmt.Errorf("invalid where filter: valueGeoRange: " + - "field 'distance' must be set"), - }, - { - name: "geo having negative distance", - input: &models.WhereFilter{ - Operator: "WithinGeoRange", - ValueGeoRange: &models.WhereFilterGeoRange{ - GeoCoordinates: &models.GeoCoordinates{ - Latitude: ptFloat32(4.5), - Longitude: ptFloat32(3.7), - }, - Distance: &models.WhereFilterGeoRangeDistance{ - Max: -20.0, - }, - }, - Path: []string{"geoField"}, - }, - expectedErr: fmt.Errorf("invalid where filter: valueGeoRange: " + - "field 'distance.max' must be a positive number"), - }, - { - name: "and operator and path set", - input: &models.WhereFilter{ - Operator: "And", - Path: []string{"some field"}, - }, - expectedErr: fmt.Errorf("invalid where filter: " + - "operator 'And' not compatible with field 'path', remove 'path' " + - "or switch to compare operator (eg. Equal, NotEqual, etc.)"), - }, - { - name: "and operator and value set", - input: &models.WhereFilter{ - Operator: "And", - ValueInt: ptInt(43), - }, - expectedErr: fmt.Errorf("invalid where filter: " + - "operator 'And' not compatible with field 'value', " + - "remove value field or switch to compare operator " + - "(eg. Equal, NotEqual, etc.)"), - }, - { - name: "and operator and no operands set", - input: &models.WhereFilter{ - Operator: "And", - }, - expectedErr: fmt.Errorf("invalid where filter: " + - "operator 'And', but no operands set - add at least one operand"), - }, - { - name: "equal operator and no values set", - input: &models.WhereFilter{ - Operator: "Equal", - }, - expectedErr: fmt.Errorf("invalid where filter: " + - "got operator 'Equal', but no value field set"), - }, - { - name: "equal operator and no path set", - input: &models.WhereFilter{ - Operator: "Equal", - ValueInt: ptInt(43), - }, - expectedErr: fmt.Errorf("invalid where filter: " + - "field 'path': must have at least one element"), - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - filter, err := Parse(test.input, "Todo") - assert.Equal(t, test.expectedErr, err) - assert.Equal(t, test.expectedFilter, filter) - }) - } - }) - - t.Run("all operator types", func(t *testing.T) { - // all tests use int as the value type, value types are tested separately - tests := []test{ - { - name: "equal", - input: inputIntFilterWithOp("Equal"), - expectedFilter: intFilterWithOp(filters.OperatorEqual), - }, - { - name: "like", // doesn't make sense on an int, but that's irrelevant for parsing - input: inputIntFilterWithOp("Like"), - expectedFilter: intFilterWithOp(filters.OperatorLike), - }, - { - name: "not equal", - input: inputIntFilterWithOp("NotEqual"), - expectedFilter: intFilterWithOp(filters.OperatorNotEqual), - }, - { - name: "greater than", - input: inputIntFilterWithOp("GreaterThan"), - expectedFilter: intFilterWithOp(filters.OperatorGreaterThan), - }, - { - name: "greater than/equal", - input: inputIntFilterWithOp("GreaterThanEqual"), - expectedFilter: intFilterWithOp(filters.OperatorGreaterThanEqual), - }, - { - name: "less than", - input: inputIntFilterWithOp("LessThan"), - expectedFilter: intFilterWithOp(filters.OperatorLessThan), - }, - { - name: "less than/equal", - input: inputIntFilterWithOp("LessThanEqual"), - expectedFilter: intFilterWithOp(filters.OperatorLessThanEqual), - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - filter, err := Parse(test.input, "Todo") - assert.Equal(t, test.expectedErr, err) - assert.Equal(t, test.expectedFilter, filter) - }) - } - }) - - t.Run("nested filters", func(t *testing.T) { - // all tests use int as the value type, value types are tested separately - tests := []test{ - { - name: "chained together using and", - input: &models.WhereFilter{ - Operator: "And", - Operands: []*models.WhereFilter{ - inputIntFilterWithValue(42), - inputIntFilterWithValueAndPath(43, - []string{"hasAction", "SomeAction", "intField"}), - }, - }, - expectedFilter: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorAnd, - Operands: []filters.Clause{ - { - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: schema.AssertValidClassName("Todo"), - Property: schema.AssertValidPropertyName("intField"), - }, - Value: &filters.Value{ - Value: 42, - Type: schema.DataTypeInt, - }, - }, - { - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: schema.AssertValidClassName("Todo"), - Property: schema.AssertValidPropertyName("hasAction"), - Child: &filters.Path{ - Class: schema.AssertValidClassName("SomeAction"), - Property: schema.AssertValidPropertyName("intField"), - }, - }, - Value: &filters.Value{ - Value: 43, - Type: schema.DataTypeInt, - }, - }, - }, - }, - }, - }, - { - name: "chained together using or", - input: &models.WhereFilter{ - Operator: "Or", - Operands: []*models.WhereFilter{ - inputIntFilterWithValue(42), - inputIntFilterWithValueAndPath(43, - []string{"hasAction", "SomeAction", "intField"}), - }, - }, - expectedFilter: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorOr, - Operands: []filters.Clause{ - { - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: schema.AssertValidClassName("Todo"), - Property: schema.AssertValidPropertyName("intField"), - }, - Value: &filters.Value{ - Value: 42, - Type: schema.DataTypeInt, - }, - }, - { - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: schema.AssertValidClassName("Todo"), - Property: schema.AssertValidPropertyName("hasAction"), - Child: &filters.Path{ - Class: schema.AssertValidClassName("SomeAction"), - Property: schema.AssertValidPropertyName("intField"), - }, - }, - Value: &filters.Value{ - Value: 43, - Type: schema.DataTypeInt, - }, - }, - }, - }, - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - filter, err := Parse(test.input, "Todo") - assert.Equal(t, test.expectedErr, err) - assert.Equal(t, test.expectedFilter, filter) - }) - } - }) -} - -func ptInt(in int) *int64 { - a := int64(in) - return &a -} - -func ptFloat(in float64) *float64 { - return &in -} - -func ptString(in string) *string { - return &in -} - -func ptBool(in bool) *bool { - return &in -} - -func intFilterWithOp(op filters.Operator) *filters.LocalFilter { - return &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: op, - On: &filters.Path{ - Class: schema.AssertValidClassName("Todo"), - Property: schema.AssertValidPropertyName("intField"), - }, - Value: &filters.Value{ - Value: 42, - Type: schema.DataTypeInt, - }, - }, - } -} - -func inputIntFilterWithOp(op string) *models.WhereFilter { - return &models.WhereFilter{ - Operator: op, - ValueInt: ptInt(42), - Path: []string{"intField"}, - } -} - -func inputIntFilterWithValue(value int) *models.WhereFilter { - return &models.WhereFilter{ - Operator: "Equal", - ValueInt: ptInt(value), - Path: []string{"intField"}, - } -} - -func inputIntFilterWithValueAndPath(value int, - path []string, -) *models.WhereFilter { - return &models.WhereFilter{ - Operator: "Equal", - ValueInt: ptInt(value), - Path: path, - } -} - -func inputGeoRangeFilter(lat, lon, max float64) *models.WhereFilterGeoRange { - return &models.WhereFilterGeoRange{ - Distance: &models.WhereFilterGeoRangeDistance{ - Max: max, - }, - GeoCoordinates: &models.GeoCoordinates{ - Latitude: ptFloat32(float32(lat)), - Longitude: ptFloat32(float32(lon)), - }, - } -} - -func ptFloat32(in float32) *float32 { - return &in -} diff --git a/adapters/handlers/rest/filterext/parse_value.go b/adapters/handlers/rest/filterext/parse_value.go deleted file mode 100644 index 05d367e5a7f02c1abb07ab30d0ef19cb27f3876e..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/filterext/parse_value.go +++ /dev/null @@ -1,194 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package filterext - -import ( - "encoding/json" - "fmt" - - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" -) - -func parseValue(in *models.WhereFilter) (*filters.Value, error) { - var value *filters.Value - - for _, extractor := range valueExtractors { - foundValue, err := extractor(in) - // Abort if we found a value, but it's for being passed a string to an int value. - if err != nil { - return nil, err - } - - if foundValue != nil { - if value != nil { - return nil, fmt.Errorf("found more than one values the clause '%s'", jsonify(in)) - } else { - value = foundValue - } - } - } - - if value == nil { - return nil, fmt.Errorf("got operator '%s', but no value field set", - in.Operator) - } - - return value, nil -} - -type valueExtractorFunc func(*models.WhereFilter) (*filters.Value, error) - -var valueExtractors = []valueExtractorFunc{ - // int - func(in *models.WhereFilter) (*filters.Value, error) { - if in.ValueInt == nil { - return nil, nil - } - - return valueFilter(int(*in.ValueInt), schema.DataTypeInt), nil - }, - // number - func(in *models.WhereFilter) (*filters.Value, error) { - if in.ValueNumber == nil { - return nil, nil - } - - return valueFilter(*in.ValueNumber, schema.DataTypeNumber), nil - }, - // text - func(in *models.WhereFilter) (*filters.Value, error) { - if in.ValueText == nil { - return nil, nil - } - - return valueFilter(*in.ValueText, schema.DataTypeText), nil - }, - // date (as string) - func(in *models.WhereFilter) (*filters.Value, error) { - if in.ValueDate == nil { - return nil, nil - } - - return valueFilter(*in.ValueDate, schema.DataTypeDate), nil - }, - // boolean - func(in *models.WhereFilter) (*filters.Value, error) { - if in.ValueBoolean == nil { - return nil, nil - } - - return valueFilter(*in.ValueBoolean, schema.DataTypeBoolean), nil - }, - - // int array - func(in *models.WhereFilter) (*filters.Value, error) { - if len(in.ValueIntArray) == 0 { - return nil, nil - } - - valueInts := make([]int, len(in.ValueIntArray)) - for i := range in.ValueIntArray { - valueInts[i] = int(in.ValueIntArray[i]) - } - return valueFilter(valueInts, schema.DataTypeInt), nil - }, - // number array - func(in *models.WhereFilter) (*filters.Value, error) { - if len(in.ValueNumberArray) == 0 { - return nil, nil - } - - return valueFilter(in.ValueNumberArray, schema.DataTypeNumber), nil - }, - // text array - func(in *models.WhereFilter) (*filters.Value, error) { - if len(in.ValueTextArray) == 0 { - return nil, nil - } - - return valueFilter(in.ValueTextArray, schema.DataTypeText), nil - }, - // date (as string) array - func(in *models.WhereFilter) (*filters.Value, error) { - if len(in.ValueDateArray) == 0 { - return nil, nil - } - - return valueFilter(in.ValueDateArray, schema.DataTypeDate), nil - }, - // boolean - func(in *models.WhereFilter) (*filters.Value, error) { - if len(in.ValueBooleanArray) == 0 { - return nil, nil - } - - return valueFilter(in.ValueBooleanArray, schema.DataTypeBoolean), nil - }, - - // geo range - func(in *models.WhereFilter) (*filters.Value, error) { - if in.ValueGeoRange == nil { - return nil, nil - } - - if in.ValueGeoRange.Distance == nil { - return nil, fmt.Errorf("valueGeoRange: field 'distance' must be set") - } - - if in.ValueGeoRange.Distance.Max < 0 { - return nil, fmt.Errorf("valueGeoRange: field 'distance.max' must be a positive number") - } - - if in.ValueGeoRange.GeoCoordinates == nil { - return nil, fmt.Errorf("valueGeoRange: field 'geoCoordinates' must be set") - } - - return valueFilter(filters.GeoRange{ - Distance: float32(in.ValueGeoRange.Distance.Max), - GeoCoordinates: &models.GeoCoordinates{ - Latitude: in.ValueGeoRange.GeoCoordinates.Latitude, - Longitude: in.ValueGeoRange.GeoCoordinates.Longitude, - }, - }, schema.DataTypeGeoCoordinates), nil - }, - // deprecated string - func(in *models.WhereFilter) (*filters.Value, error) { - if in.ValueString == nil { - return nil, nil - } - - return valueFilter(*in.ValueString, schema.DataTypeString), nil - }, - // deprecated string array - func(in *models.WhereFilter) (*filters.Value, error) { - if len(in.ValueStringArray) == 0 { - return nil, nil - } - - return valueFilter(in.ValueStringArray, schema.DataTypeString), nil - }, -} - -func valueFilter(value interface{}, dt schema.DataType) *filters.Value { - return &filters.Value{ - Type: dt, - Value: value, - } -} - -// Small utility function used in printing error messages. -func jsonify(stuff interface{}) string { - j, _ := json.Marshal(stuff) - return string(j) -} diff --git a/adapters/handlers/rest/grpc.go b/adapters/handlers/rest/grpc.go deleted file mode 100644 index b1f0a3db2ef3fbefe2f01cca803ab012db7262d3..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/grpc.go +++ /dev/null @@ -1,30 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package rest - -import ( - "github.com/weaviate/weaviate/adapters/handlers/grpc" - "github.com/weaviate/weaviate/adapters/handlers/rest/state" -) - -func createGrpcServer(state *state.State) *grpc.GRPCServer { - return grpc.CreateGRPCServer(state) -} - -func startGrpcServer(server *grpc.GRPCServer, state *state.State) { - go func() { - if err := grpc.StartAndListen(server, state); err != nil { - state.Logger.WithField("action", "grpc_startup").WithError(err). - Fatal("failed to start grpc server") - } - }() -} diff --git a/adapters/handlers/rest/handlers_backup.go b/adapters/handlers/rest/handlers_backup.go deleted file mode 100644 index f28e85e3b4dde5e56cee77c29de4de720154ddd2..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/handlers_backup.go +++ /dev/null @@ -1,254 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package rest - -import ( - "github.com/go-openapi/runtime/middleware" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/adapters/handlers/rest/operations" - "github.com/weaviate/weaviate/adapters/handlers/rest/operations/backups" - "github.com/weaviate/weaviate/entities/backup" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/usecases/auth/authorization/errors" - ubak "github.com/weaviate/weaviate/usecases/backup" - "github.com/weaviate/weaviate/usecases/monitoring" -) - -type backupHandlers struct { - manager *ubak.Scheduler - metricRequestsTotal restApiRequestsTotal -} - -// compressionFromCfg transforms model backup config to a backup compression config -func compressionFromBCfg(cfg *models.BackupConfig) ubak.Compression { - if cfg != nil { - if cfg.CPUPercentage == 0 { - cfg.CPUPercentage = ubak.DefaultCPUPercentage - } - - if cfg.ChunkSize == 0 { - cfg.ChunkSize = ubak.DefaultChunkSize - } - - if cfg.CompressionLevel == "" { - cfg.CompressionLevel = models.BackupConfigCompressionLevelDefaultCompression - } - - return ubak.Compression{ - CPUPercentage: int(cfg.CPUPercentage), - ChunkSize: int(cfg.ChunkSize), - Level: parseCompressionLevel(cfg.CompressionLevel), - } - } - - return ubak.Compression{ - Level: ubak.DefaultCompression, - CPUPercentage: ubak.DefaultCPUPercentage, - ChunkSize: ubak.DefaultChunkSize, - } -} - -func compressionFromRCfg(cfg *models.RestoreConfig) ubak.Compression { - if cfg != nil { - if cfg.CPUPercentage == 0 { - cfg.CPUPercentage = ubak.DefaultCPUPercentage - } - - return ubak.Compression{ - CPUPercentage: int(cfg.CPUPercentage), - Level: ubak.DefaultCompression, - ChunkSize: ubak.DefaultChunkSize, - } - } - - return ubak.Compression{ - Level: ubak.DefaultCompression, - CPUPercentage: ubak.DefaultCPUPercentage, - ChunkSize: ubak.DefaultChunkSize, - } -} - -func parseCompressionLevel(l string) ubak.CompressionLevel { - switch { - case l == models.BackupConfigCompressionLevelBestSpeed: - return ubak.BestSpeed - case l == models.BackupConfigCompressionLevelBestCompression: - return ubak.BestCompression - default: - return ubak.DefaultCompression - } -} - -func (s *backupHandlers) createBackup(params backups.BackupsCreateParams, - principal *models.Principal, -) middleware.Responder { - meta, err := s.manager.Backup(params.HTTPRequest.Context(), principal, &ubak.BackupRequest{ - ID: params.Body.ID, - Backend: params.Backend, - Include: params.Body.Include, - Exclude: params.Body.Exclude, - Compression: compressionFromBCfg(params.Body.Config), - }) - if err != nil { - s.metricRequestsTotal.logError("", err) - switch err.(type) { - case errors.Forbidden: - return backups.NewBackupsCreateForbidden(). - WithPayload(errPayloadFromSingleErr(err)) - case backup.ErrUnprocessable: - return backups.NewBackupsCreateUnprocessableEntity(). - WithPayload(errPayloadFromSingleErr(err)) - default: - return backups.NewBackupsCreateInternalServerError(). - WithPayload(errPayloadFromSingleErr(err)) - } - } - - s.metricRequestsTotal.logOk("") - return backups.NewBackupsCreateOK().WithPayload(meta) -} - -func (s *backupHandlers) createBackupStatus(params backups.BackupsCreateStatusParams, - principal *models.Principal, -) middleware.Responder { - status, err := s.manager.BackupStatus(params.HTTPRequest.Context(), principal, params.Backend, params.ID) - if err != nil { - s.metricRequestsTotal.logError("", err) - switch err.(type) { - case errors.Forbidden: - return backups.NewBackupsCreateStatusForbidden(). - WithPayload(errPayloadFromSingleErr(err)) - case backup.ErrUnprocessable: - return backups.NewBackupsCreateStatusUnprocessableEntity(). - WithPayload(errPayloadFromSingleErr(err)) - case backup.ErrNotFound: - return backups.NewBackupsCreateStatusNotFound(). - WithPayload(errPayloadFromSingleErr(err)) - default: - return backups.NewBackupsCreateStatusInternalServerError(). - WithPayload(errPayloadFromSingleErr(err)) - } - } - - strStatus := string(status.Status) - payload := models.BackupCreateStatusResponse{ - Status: &strStatus, - ID: params.ID, - Path: status.Path, - Backend: params.Backend, - Error: status.Err, - } - s.metricRequestsTotal.logOk("") - return backups.NewBackupsCreateStatusOK().WithPayload(&payload) -} - -func (s *backupHandlers) restoreBackup(params backups.BackupsRestoreParams, - principal *models.Principal, -) middleware.Responder { - meta, err := s.manager.Restore(params.HTTPRequest.Context(), principal, &ubak.BackupRequest{ - ID: params.ID, - Backend: params.Backend, - Include: params.Body.Include, - Exclude: params.Body.Exclude, - NodeMapping: params.Body.NodeMapping, - Compression: compressionFromRCfg(params.Body.Config), - }) - if err != nil { - s.metricRequestsTotal.logError("", err) - switch err.(type) { - case errors.Forbidden: - return backups.NewBackupsRestoreForbidden(). - WithPayload(errPayloadFromSingleErr(err)) - case backup.ErrNotFound: - return backups.NewBackupsRestoreNotFound(). - WithPayload(errPayloadFromSingleErr(err)) - case backup.ErrUnprocessable: - return backups.NewBackupsRestoreUnprocessableEntity(). - WithPayload(errPayloadFromSingleErr(err)) - default: - return backups.NewBackupsRestoreInternalServerError(). - WithPayload(errPayloadFromSingleErr(err)) - } - } - - s.metricRequestsTotal.logOk("") - return backups.NewBackupsRestoreOK().WithPayload(meta) -} - -func (s *backupHandlers) restoreBackupStatus(params backups.BackupsRestoreStatusParams, - principal *models.Principal, -) middleware.Responder { - status, err := s.manager.RestorationStatus( - params.HTTPRequest.Context(), principal, params.Backend, params.ID) - if err != nil { - s.metricRequestsTotal.logError("", err) - switch err.(type) { - case errors.Forbidden: - return backups.NewBackupsRestoreForbidden(). - WithPayload(errPayloadFromSingleErr(err)) - case backup.ErrNotFound: - return backups.NewBackupsRestoreNotFound(). - WithPayload(errPayloadFromSingleErr(err)) - case backup.ErrUnprocessable: - return backups.NewBackupsRestoreUnprocessableEntity(). - WithPayload(errPayloadFromSingleErr(err)) - default: - return backups.NewBackupsRestoreInternalServerError(). - WithPayload(errPayloadFromSingleErr(err)) - } - } - strStatus := string(status.Status) - payload := models.BackupRestoreStatusResponse{ - Status: &strStatus, - ID: params.ID, - Path: status.Path, - Backend: params.Backend, - Error: status.Err, - } - s.metricRequestsTotal.logOk("") - return backups.NewBackupsRestoreStatusOK().WithPayload(&payload) -} - -func setupBackupHandlers(api *operations.WeaviateAPI, - scheduler *ubak.Scheduler, metrics *monitoring.PrometheusMetrics, logger logrus.FieldLogger, -) { - h := &backupHandlers{scheduler, newBackupRequestsTotal(metrics, logger)} - api.BackupsBackupsCreateHandler = backups. - BackupsCreateHandlerFunc(h.createBackup) - api.BackupsBackupsCreateStatusHandler = backups. - BackupsCreateStatusHandlerFunc(h.createBackupStatus) - api.BackupsBackupsRestoreHandler = backups. - BackupsRestoreHandlerFunc(h.restoreBackup) - api.BackupsBackupsRestoreStatusHandler = backups. - BackupsRestoreStatusHandlerFunc(h.restoreBackupStatus) -} - -type backupRequestsTotal struct { - *restApiRequestsTotalImpl -} - -func newBackupRequestsTotal(metrics *monitoring.PrometheusMetrics, logger logrus.FieldLogger) restApiRequestsTotal { - return &backupRequestsTotal{ - restApiRequestsTotalImpl: &restApiRequestsTotalImpl{newRequestsTotalMetric(metrics, "rest"), "rest", "backup", logger}, - } -} - -func (e *backupRequestsTotal) logError(className string, err error) { - switch err.(type) { - case errors.Forbidden: - e.logUserError(className) - case backup.ErrUnprocessable, backup.ErrNotFound: - e.logUserError(className) - default: - e.logServerError(className, err) - } -} diff --git a/adapters/handlers/rest/handlers_backup_test.go b/adapters/handlers/rest/handlers_backup_test.go deleted file mode 100644 index b48baf600e6fec4ce189fb5e439631815df4869d..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/handlers_backup_test.go +++ /dev/null @@ -1,108 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package rest - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/entities/models" - ubak "github.com/weaviate/weaviate/usecases/backup" -) - -func TestCompressionBackupCfg(t *testing.T) { - tcs := map[string]struct { - cfg *models.BackupConfig - expectedCompression ubak.CompressionLevel - expectedCPU int - expectedChunkSize int - }{ - "without config": { - cfg: nil, - expectedCompression: ubak.DefaultCompression, - expectedCPU: ubak.DefaultCPUPercentage, - expectedChunkSize: ubak.DefaultChunkSize, - }, - "with config": { - cfg: &models.BackupConfig{ - CPUPercentage: 25, - ChunkSize: 512, - CompressionLevel: models.BackupConfigCompressionLevelBestSpeed, - }, - expectedCompression: ubak.BestSpeed, - expectedCPU: 25, - expectedChunkSize: 512, - }, - "with partial config [CPU]": { - cfg: &models.BackupConfig{ - CPUPercentage: 25, - }, - expectedCompression: ubak.DefaultCompression, - expectedCPU: 25, - expectedChunkSize: ubak.DefaultChunkSize, - }, - "with partial config [ChunkSize]": { - cfg: &models.BackupConfig{ - ChunkSize: 125, - }, - expectedCompression: ubak.DefaultCompression, - expectedCPU: ubak.DefaultCPUPercentage, - expectedChunkSize: 125, - }, - "with partial config [Compression]": { - cfg: &models.BackupConfig{ - CompressionLevel: models.BackupConfigCompressionLevelBestSpeed, - }, - expectedCompression: ubak.BestSpeed, - expectedCPU: ubak.DefaultCPUPercentage, - expectedChunkSize: ubak.DefaultChunkSize, - }, - } - - for n, tc := range tcs { - t.Run(n, func(t *testing.T) { - ccfg := compressionFromBCfg(tc.cfg) - assert.Equal(t, tc.expectedCompression, ccfg.Level) - assert.Equal(t, tc.expectedCPU, ccfg.CPUPercentage) - assert.Equal(t, tc.expectedChunkSize, ccfg.ChunkSize) - }) - } -} - -func TestCompressionRestoreCfg(t *testing.T) { - tcs := map[string]struct { - cfg *models.RestoreConfig - expectedCompression ubak.CompressionLevel - expectedCPU int - expectedChunkSize int - }{ - "without config": { - cfg: nil, - expectedCompression: ubak.DefaultCompression, - expectedCPU: ubak.DefaultCPUPercentage, - expectedChunkSize: ubak.DefaultChunkSize, - }, - "with config": { - cfg: &models.RestoreConfig{ - CPUPercentage: 25, - }, - expectedCPU: 25, - }, - } - - for n, tc := range tcs { - t.Run(n, func(t *testing.T) { - ccfg := compressionFromRCfg(tc.cfg) - assert.Equal(t, tc.expectedCPU, ccfg.CPUPercentage) - }) - } -} diff --git a/adapters/handlers/rest/handlers_batch_objects.go b/adapters/handlers/rest/handlers_batch_objects.go deleted file mode 100644 index 00e0fd315821ea22592ca930cb57dd9fc1ec3929..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/handlers_batch_objects.go +++ /dev/null @@ -1,277 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package rest - -import ( - "errors" - - middleware "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/adapters/handlers/rest/operations" - "github.com/weaviate/weaviate/adapters/handlers/rest/operations/batch" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/verbosity" - autherrs "github.com/weaviate/weaviate/usecases/auth/authorization/errors" - "github.com/weaviate/weaviate/usecases/monitoring" - "github.com/weaviate/weaviate/usecases/objects" -) - -type batchObjectHandlers struct { - manager *objects.BatchManager - metricRequestsTotal restApiRequestsTotal -} - -func (h *batchObjectHandlers) addObjects(params batch.BatchObjectsCreateParams, - principal *models.Principal, -) middleware.Responder { - repl, err := getReplicationProperties(params.ConsistencyLevel, nil) - if err != nil { - h.metricRequestsTotal.logError("", err) - return batch.NewBatchObjectsCreateBadRequest(). - WithPayload(errPayloadFromSingleErr(err)) - } - - objs, err := h.manager.AddObjects(params.HTTPRequest.Context(), principal, - params.Body.Objects, params.Body.Fields, repl) - if err != nil { - h.metricRequestsTotal.logError("", err) - switch err.(type) { - case autherrs.Forbidden: - return batch.NewBatchObjectsCreateForbidden(). - WithPayload(errPayloadFromSingleErr(err)) - case objects.ErrInvalidUserInput: - return batch.NewBatchObjectsCreateUnprocessableEntity(). - WithPayload(errPayloadFromSingleErr(err)) - case objects.ErrMultiTenancy: - return batch.NewBatchObjectsCreateUnprocessableEntity(). - WithPayload(errPayloadFromSingleErr(err)) - default: - return batch.NewBatchObjectsCreateInternalServerError(). - WithPayload(errPayloadFromSingleErr(err)) - } - } - - h.metricRequestsTotal.logOk("") - return batch.NewBatchObjectsCreateOK(). - WithPayload(h.objectsResponse(objs)) -} - -func (h *batchObjectHandlers) objectsResponse(input objects.BatchObjects) []*models.ObjectsGetResponse { - response := make([]*models.ObjectsGetResponse, len(input)) - for i, object := range input { - var errorResponse *models.ErrorResponse - status := models.ObjectsGetResponseAO2ResultStatusSUCCESS - if object.Err != nil { - errorResponse = errPayloadFromSingleErr(object.Err) - status = models.ObjectsGetResponseAO2ResultStatusFAILED - } - - object.Object.ID = object.UUID - response[i] = &models.ObjectsGetResponse{ - Object: *object.Object, - Result: &models.ObjectsGetResponseAO2Result{ - Errors: errorResponse, - Status: &status, - }, - } - } - - return response -} - -func (h *batchObjectHandlers) addReferences(params batch.BatchReferencesCreateParams, - principal *models.Principal, -) middleware.Responder { - repl, err := getReplicationProperties(params.ConsistencyLevel, nil) - if err != nil { - h.metricRequestsTotal.logError("", err) - return batch.NewBatchReferencesCreateBadRequest(). - WithPayload(errPayloadFromSingleErr(err)) - } - - references, err := h.manager.AddReferences(params.HTTPRequest.Context(), principal, params.Body, repl) - if err != nil { - h.metricRequestsTotal.logError("", err) - switch err.(type) { - case autherrs.Forbidden: - return batch.NewBatchReferencesCreateForbidden(). - WithPayload(errPayloadFromSingleErr(err)) - case objects.ErrInvalidUserInput: - return batch.NewBatchReferencesCreateUnprocessableEntity(). - WithPayload(errPayloadFromSingleErr(err)) - case objects.ErrMultiTenancy: - return batch.NewBatchReferencesCreateUnprocessableEntity(). - WithPayload(errPayloadFromSingleErr(err)) - default: - return batch.NewBatchReferencesCreateInternalServerError(). - WithPayload(errPayloadFromSingleErr(err)) - } - } - - h.metricRequestsTotal.logOk("") - return batch.NewBatchReferencesCreateOK(). - WithPayload(h.referencesResponse(references)) -} - -func (h *batchObjectHandlers) referencesResponse(input objects.BatchReferences) []*models.BatchReferenceResponse { - response := make([]*models.BatchReferenceResponse, len(input)) - for i, ref := range input { - var errorResponse *models.ErrorResponse - var reference models.BatchReference - - status := models.BatchReferenceResponseAO1ResultStatusSUCCESS - if ref.Err != nil { - errorResponse = errPayloadFromSingleErr(ref.Err) - status = models.BatchReferenceResponseAO1ResultStatusFAILED - } else { - reference.From = strfmt.URI(ref.From.String()) - reference.To = strfmt.URI(ref.To.String()) - } - - response[i] = &models.BatchReferenceResponse{ - BatchReference: reference, - Result: &models.BatchReferenceResponseAO1Result{ - Errors: errorResponse, - Status: &status, - }, - } - } - - return response -} - -func (h *batchObjectHandlers) deleteObjects(params batch.BatchObjectsDeleteParams, - principal *models.Principal, -) middleware.Responder { - repl, err := getReplicationProperties(params.ConsistencyLevel, nil) - if err != nil { - h.metricRequestsTotal.logError("", err) - return batch.NewBatchObjectsDeleteBadRequest(). - WithPayload(errPayloadFromSingleErr(err)) - } - - tenant := getTenant(params.Tenant) - - res, err := h.manager.DeleteObjects(params.HTTPRequest.Context(), principal, - params.Body.Match, params.Body.DryRun, params.Body.Output, repl, tenant) - if err != nil { - h.metricRequestsTotal.logError("", err) - if errors.As(err, &objects.ErrInvalidUserInput{}) { - return batch.NewBatchObjectsDeleteUnprocessableEntity(). - WithPayload(errPayloadFromSingleErr(err)) - } else if errors.As(err, &objects.ErrMultiTenancy{}) { - return batch.NewBatchObjectsDeleteUnprocessableEntity(). - WithPayload(errPayloadFromSingleErr(err)) - } else if errors.As(err, &autherrs.Forbidden{}) { - return batch.NewBatchObjectsDeleteForbidden(). - WithPayload(errPayloadFromSingleErr(err)) - } else { - return batch.NewBatchObjectsDeleteInternalServerError(). - WithPayload(errPayloadFromSingleErr(err)) - } - } - - h.metricRequestsTotal.logOk("") - return batch.NewBatchObjectsDeleteOK(). - WithPayload(h.objectsDeleteResponse(res)) -} - -func (h *batchObjectHandlers) objectsDeleteResponse(input *objects.BatchDeleteResponse) *models.BatchDeleteResponse { - var successful, failed int64 - output := input.Output - var objects []*models.BatchDeleteResponseResultsObjectsItems0 - for _, obj := range input.Result.Objects { - var errorResponse *models.ErrorResponse - - status := models.BatchDeleteResponseResultsObjectsItems0StatusSUCCESS - if input.DryRun { - status = models.BatchDeleteResponseResultsObjectsItems0StatusDRYRUN - } else if obj.Err != nil { - status = models.BatchDeleteResponseResultsObjectsItems0StatusFAILED - errorResponse = errPayloadFromSingleErr(obj.Err) - failed += 1 - } else { - successful += 1 - } - - if output == verbosity.OutputMinimal && - (status == models.BatchDeleteResponseResultsObjectsItems0StatusSUCCESS || - status == models.BatchDeleteResponseResultsObjectsItems0StatusDRYRUN) { - // only add SUCCESS and DRYRUN results if output is "verbose" - continue - } - - objects = append(objects, &models.BatchDeleteResponseResultsObjectsItems0{ - ID: obj.UUID, - Status: &status, - Errors: errorResponse, - }) - } - - response := &models.BatchDeleteResponse{ - Match: &models.BatchDeleteResponseMatch{ - Class: input.Match.Class, - Where: input.Match.Where, - }, - DryRun: &input.DryRun, - Output: &output, - Results: &models.BatchDeleteResponseResults{ - Matches: input.Result.Matches, - Limit: input.Result.Limit, - Successful: successful, - Failed: failed, - Objects: objects, - }, - } - return response -} - -func setupObjectBatchHandlers(api *operations.WeaviateAPI, manager *objects.BatchManager, metrics *monitoring.PrometheusMetrics, logger logrus.FieldLogger) { - h := &batchObjectHandlers{manager, newBatchRequestsTotal(metrics, logger)} - - api.BatchBatchObjectsCreateHandler = batch. - BatchObjectsCreateHandlerFunc(h.addObjects) - api.BatchBatchReferencesCreateHandler = batch. - BatchReferencesCreateHandlerFunc(h.addReferences) - api.BatchBatchObjectsDeleteHandler = batch. - BatchObjectsDeleteHandlerFunc(h.deleteObjects) -} - -type batchRequestsTotal struct { - *restApiRequestsTotalImpl -} - -func newBatchRequestsTotal(metrics *monitoring.PrometheusMetrics, logger logrus.FieldLogger) restApiRequestsTotal { - return &batchRequestsTotal{ - restApiRequestsTotalImpl: &restApiRequestsTotalImpl{newRequestsTotalMetric(metrics, "rest"), "rest", "batch", logger}, - } -} - -func (e *batchRequestsTotal) logError(className string, err error) { - switch err.(type) { - case errReplication: - e.logUserError(className) - case autherrs.Forbidden, objects.ErrInvalidUserInput: - e.logUserError(className) - case objects.ErrMultiTenancy: - e.logUserError(className) - default: - if errors.As(err, &objects.ErrMultiTenancy{}) || - errors.As(err, &objects.ErrInvalidUserInput{}) || - errors.As(err, &autherrs.Forbidden{}) { - e.logUserError(className) - } else { - e.logServerError(className, err) - } - } -} diff --git a/adapters/handlers/rest/handlers_classification.go b/adapters/handlers/rest/handlers_classification.go deleted file mode 100644 index ac97821c6292a29ed5f2671270fdcb3dd13eaf8f..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/handlers_classification.go +++ /dev/null @@ -1,76 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package rest - -import ( - middleware "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/adapters/handlers/rest/operations" - "github.com/weaviate/weaviate/adapters/handlers/rest/operations/classifications" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/usecases/classification" - "github.com/weaviate/weaviate/usecases/monitoring" -) - -func setupClassificationHandlers(api *operations.WeaviateAPI, - classifier *classification.Classifier, metrics *monitoring.PrometheusMetrics, logger logrus.FieldLogger, -) { - metricRequestsTotal := newClassificationRequestsTotal(metrics, logger) - api.ClassificationsClassificationsGetHandler = classifications.ClassificationsGetHandlerFunc( - func(params classifications.ClassificationsGetParams, principal *models.Principal) middleware.Responder { - res, err := classifier.Get(params.HTTPRequest.Context(), principal, strfmt.UUID(params.ID)) - if err != nil { - metricRequestsTotal.logError("", err) - return classifications.NewClassificationsGetInternalServerError().WithPayload(errPayloadFromSingleErr(err)) - } - - if res == nil { - metricRequestsTotal.logUserError("") - return classifications.NewClassificationsGetNotFound() - } - - metricRequestsTotal.logOk("") - return classifications.NewClassificationsGetOK().WithPayload(res) - }, - ) - - api.ClassificationsClassificationsPostHandler = classifications.ClassificationsPostHandlerFunc( - func(params classifications.ClassificationsPostParams, principal *models.Principal) middleware.Responder { - res, err := classifier.Schedule(params.HTTPRequest.Context(), principal, *params.Params) - if err != nil { - metricRequestsTotal.logUserError("") - return classifications.NewClassificationsPostBadRequest().WithPayload(errPayloadFromSingleErr(err)) - } - - metricRequestsTotal.logOk("") - return classifications.NewClassificationsPostCreated().WithPayload(res) - }, - ) -} - -type classificationRequestsTotal struct { - *restApiRequestsTotalImpl -} - -func newClassificationRequestsTotal(metrics *monitoring.PrometheusMetrics, logger logrus.FieldLogger) restApiRequestsTotal { - return &classificationRequestsTotal{ - restApiRequestsTotalImpl: &restApiRequestsTotalImpl{newRequestsTotalMetric(metrics, "rest"), "rest", "classification", logger}, - } -} - -func (e *classificationRequestsTotal) logError(className string, err error) { - switch err.(type) { - default: - e.logServerError(className, err) - } -} diff --git a/adapters/handlers/rest/handlers_graphql.go b/adapters/handlers/rest/handlers_graphql.go deleted file mode 100644 index e86eb5df972cebe81759c40dc17b32db09818a4b..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/handlers_graphql.go +++ /dev/null @@ -1,410 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package rest - -import ( - "context" - "encoding/json" - "fmt" - "strconv" - "strings" - "sync" - - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/usecases/auth/authorization/errors" - "github.com/weaviate/weaviate/usecases/monitoring" - "github.com/weaviate/weaviate/usecases/schema" - - middleware "github.com/go-openapi/runtime/middleware" - tailorincgraphql "github.com/tailor-inc/graphql" - "github.com/tailor-inc/graphql/gqlerrors" - libgraphql "github.com/weaviate/weaviate/adapters/handlers/graphql" - "github.com/weaviate/weaviate/adapters/handlers/rest/operations" - "github.com/weaviate/weaviate/adapters/handlers/rest/operations/graphql" - enterrors "github.com/weaviate/weaviate/entities/errors" - "github.com/weaviate/weaviate/entities/models" -) - -const error422 string = "The request is well-formed but was unable to be followed due to semantic errors." - -type gqlUnbatchedRequestResponse struct { - RequestIndex int - Response *models.GraphQLResponse -} - -type graphQLProvider interface { - GetGraphQL() libgraphql.GraphQL -} - -func setupGraphQLHandlers( - api *operations.WeaviateAPI, - gqlProvider graphQLProvider, - m *schema.Manager, - disabled bool, - metrics *monitoring.PrometheusMetrics, - logger logrus.FieldLogger, -) { - metricRequestsTotal := newGraphqlRequestsTotal(metrics, logger) - api.GraphqlGraphqlPostHandler = graphql.GraphqlPostHandlerFunc(func(params graphql.GraphqlPostParams, principal *models.Principal) middleware.Responder { - // All requests to the graphQL API need at least permissions to read the schema. Request might have further - // authorization requirements. - - err := m.Authorizer.Authorize(principal, "list", "schema/*") - if err != nil { - metricRequestsTotal.logUserError() - switch err.(type) { - case errors.Forbidden: - return graphql.NewGraphqlPostForbidden(). - WithPayload(errPayloadFromSingleErr(err)) - default: - return graphql.NewGraphqlPostUnprocessableEntity(). - WithPayload(errPayloadFromSingleErr(err)) - } - } - - if disabled { - metricRequestsTotal.logUserError() - err := fmt.Errorf("graphql api is disabled") - return graphql.NewGraphqlPostUnprocessableEntity(). - WithPayload(errPayloadFromSingleErr(err)) - } - - errorResponse := &models.ErrorResponse{} - - // Get all input from the body of the request, as it is a POST. - query := params.Body.Query - operationName := params.Body.OperationName - - // If query is empty, the request is unprocessable - if query == "" { - metricRequestsTotal.logUserError() - errorResponse.Error = []*models.ErrorResponseErrorItems0{ - { - Message: "query cannot be empty", - }, - } - return graphql.NewGraphqlPostUnprocessableEntity().WithPayload(errorResponse) - } - - // Only set variables if exists in request - var variables map[string]interface{} - if params.Body.Variables != nil { - variables = params.Body.Variables.(map[string]interface{}) - } - - graphQL := gqlProvider.GetGraphQL() - if graphQL == nil { - metricRequestsTotal.logUserError() - errorResponse.Error = []*models.ErrorResponseErrorItems0{ - { - Message: "no graphql provider present, this is most likely because no schema is present. Import a schema first!", - }, - } - return graphql.NewGraphqlPostUnprocessableEntity().WithPayload(errorResponse) - } - - ctx := params.HTTPRequest.Context() - ctx = context.WithValue(ctx, "principal", principal) - - result := graphQL.Resolve(ctx, query, - operationName, variables) - - // Marshal the JSON - resultJSON, jsonErr := json.Marshal(result) - if jsonErr != nil { - metricRequestsTotal.logUserError() - errorResponse.Error = []*models.ErrorResponseErrorItems0{ - { - Message: fmt.Sprintf("couldn't marshal json: %s", jsonErr), - }, - } - return graphql.NewGraphqlPostUnprocessableEntity().WithPayload(errorResponse) - } - - // Put the data in a response ready object - graphQLResponse := &models.GraphQLResponse{} - marshallErr := json.Unmarshal(resultJSON, graphQLResponse) - - // If json gave error, return nothing. - if marshallErr != nil { - metricRequestsTotal.logUserError() - errorResponse.Error = []*models.ErrorResponseErrorItems0{ - { - Message: fmt.Sprintf("couldn't unmarshal json: %s\noriginal result was %#v", marshallErr, result), - }, - } - return graphql.NewGraphqlPostUnprocessableEntity().WithPayload(errorResponse) - } - - metricRequestsTotal.log(result) - // Return the response - return graphql.NewGraphqlPostOK().WithPayload(graphQLResponse) - }) - - api.GraphqlGraphqlBatchHandler = graphql.GraphqlBatchHandlerFunc(func(params graphql.GraphqlBatchParams, principal *models.Principal) middleware.Responder { - amountOfBatchedRequests := len(params.Body) - errorResponse := &models.ErrorResponse{} - - if amountOfBatchedRequests == 0 { - metricRequestsTotal.logUserError() - return graphql.NewGraphqlBatchUnprocessableEntity().WithPayload(errorResponse) - } - requestResults := make(chan gqlUnbatchedRequestResponse, amountOfBatchedRequests) - - wg := new(sync.WaitGroup) - - ctx := params.HTTPRequest.Context() - ctx = context.WithValue(ctx, "principal", principal) - - graphQL := gqlProvider.GetGraphQL() - if graphQL == nil { - metricRequestsTotal.logUserError() - errRes := errPayloadFromSingleErr(fmt.Errorf("no graphql provider present, " + - "this is most likely because no schema is present. Import a schema first!")) - return graphql.NewGraphqlBatchUnprocessableEntity().WithPayload(errRes) - } - - // Generate a goroutine for each separate request - for requestIndex, unbatchedRequest := range params.Body { - wg.Add(1) - go handleUnbatchedGraphQLRequest(ctx, wg, graphQL, unbatchedRequest, requestIndex, &requestResults, metricRequestsTotal) - } - - wg.Wait() - - close(requestResults) - - batchedRequestResponse := make([]*models.GraphQLResponse, amountOfBatchedRequests) - - // Add the requests to the result array in the correct order - for unbatchedRequestResult := range requestResults { - batchedRequestResponse[unbatchedRequestResult.RequestIndex] = unbatchedRequestResult.Response - } - - return graphql.NewGraphqlBatchOK().WithPayload(batchedRequestResponse) - }) -} - -// Handle a single unbatched GraphQL request, return a tuple containing the index of the request in the batch and either the response or an error -func handleUnbatchedGraphQLRequest(ctx context.Context, wg *sync.WaitGroup, graphQL libgraphql.GraphQL, unbatchedRequest *models.GraphQLQuery, requestIndex int, requestResults *chan gqlUnbatchedRequestResponse, metricRequestsTotal *graphqlRequestsTotal) { - defer wg.Done() - - // Get all input from the body of the request - query := unbatchedRequest.Query - operationName := unbatchedRequest.OperationName - graphQLResponse := &models.GraphQLResponse{} - - // Return an unprocessable error if the query is empty - if query == "" { - metricRequestsTotal.logUserError() - // Regular error messages are returned as an error code in the request header, but that doesn't work for batched requests - errorCode := strconv.Itoa(graphql.GraphqlBatchUnprocessableEntityCode) - errorMessage := fmt.Sprintf("%s: %s", errorCode, error422) - errors := []*models.GraphQLError{{Message: errorMessage}} - graphQLResponse := models.GraphQLResponse{Data: nil, Errors: errors} - *requestResults <- gqlUnbatchedRequestResponse{ - requestIndex, - &graphQLResponse, - } - } else { - // Extract any variables from the request - var variables map[string]interface{} - if unbatchedRequest.Variables != nil { - var ok bool - variables, ok = unbatchedRequest.Variables.(map[string]interface{}) - if !ok { - errorCode := strconv.Itoa(graphql.GraphqlBatchUnprocessableEntityCode) - errorMessage := fmt.Sprintf("%s: %s", errorCode, fmt.Sprintf("expected map[string]interface{}, received %v", unbatchedRequest.Variables)) - - error := []*models.GraphQLError{{Message: errorMessage}} - graphQLResponse := models.GraphQLResponse{Data: nil, Errors: error} - *requestResults <- gqlUnbatchedRequestResponse{ - requestIndex, - &graphQLResponse, - } - return - } - } - - result := graphQL.Resolve(ctx, query, operationName, variables) - - // Marshal the JSON - resultJSON, jsonErr := json.Marshal(result) - - // Return an unprocessable error if marshalling the result to JSON failed - if jsonErr != nil { - metricRequestsTotal.logUserError() - // Regular error messages are returned as an error code in the request header, but that doesn't work for batched requests - errorCode := strconv.Itoa(graphql.GraphqlBatchUnprocessableEntityCode) - errorMessage := fmt.Sprintf("%s: %s", errorCode, error422) - errors := []*models.GraphQLError{{Message: errorMessage}} - graphQLResponse := models.GraphQLResponse{Data: nil, Errors: errors} - *requestResults <- gqlUnbatchedRequestResponse{ - requestIndex, - &graphQLResponse, - } - } else { - // Put the result data in a response ready object - marshallErr := json.Unmarshal(resultJSON, graphQLResponse) - - // Return an unprocessable error if unmarshalling the result to JSON failed - if marshallErr != nil { - metricRequestsTotal.logUserError() - // Regular error messages are returned as an error code in the request header, but that doesn't work for batched requests - errorCode := strconv.Itoa(graphql.GraphqlBatchUnprocessableEntityCode) - errorMessage := fmt.Sprintf("%s: %s", errorCode, error422) - errors := []*models.GraphQLError{{Message: errorMessage}} - graphQLResponse := models.GraphQLResponse{Data: nil, Errors: errors} - *requestResults <- gqlUnbatchedRequestResponse{ - requestIndex, - &graphQLResponse, - } - } else { - metricRequestsTotal.log(result) - // Return the GraphQL response - *requestResults <- gqlUnbatchedRequestResponse{ - requestIndex, - graphQLResponse, - } - } - } - } -} - -type graphqlRequestsTotal struct { - metrics *requestsTotalMetric - logger logrus.FieldLogger -} - -func newGraphqlRequestsTotal(metrics *monitoring.PrometheusMetrics, logger logrus.FieldLogger) *graphqlRequestsTotal { - return &graphqlRequestsTotal{newRequestsTotalMetric(metrics, "graphql"), logger} -} - -func (e *graphqlRequestsTotal) getQueryType(path []interface{}) string { - if len(path) > 0 { - return fmt.Sprintf("%v", path[0]) - } - return "" -} - -func (e *graphqlRequestsTotal) getClassName(path []interface{}) string { - if len(path) > 1 { - return fmt.Sprintf("%v", path[1]) - } - return "" -} - -func (e *graphqlRequestsTotal) getErrGraphQLUser(gqlError gqlerrors.FormattedError) (bool, *enterrors.ErrGraphQLUser) { - if gqlError.OriginalError() != nil { - if gqlOriginalErr, ok := gqlError.OriginalError().(*gqlerrors.Error); ok { - if gqlOriginalErr.OriginalError != nil { - switch err := gqlOriginalErr.OriginalError.(type) { - case enterrors.ErrGraphQLUser: - return e.getError(err) - default: - if gqlFormatted, ok := gqlOriginalErr.OriginalError.(gqlerrors.FormattedError); ok { - if gqlFormatted.OriginalError() != nil { - return e.getError(gqlFormatted.OriginalError()) - } - } - } - } - } - } - return false, nil -} - -func (e *graphqlRequestsTotal) isSyntaxRelatedError(gqlError gqlerrors.FormattedError) bool { - for _, prefix := range []string{"Syntax Error ", "Cannot query field"} { - if strings.HasPrefix(gqlError.Message, prefix) { - return true - } - } - return false -} - -func (e *graphqlRequestsTotal) getError(err error) (bool, *enterrors.ErrGraphQLUser) { - switch e := err.(type) { - case enterrors.ErrGraphQLUser: - return true, &e - default: - return false, nil - } -} - -func (e *graphqlRequestsTotal) log(result *tailorincgraphql.Result) { - if len(result.Errors) > 0 { - for _, gqlErr := range result.Errors { - if isUserError, err := e.getErrGraphQLUser(gqlErr); isUserError { - if e.metrics != nil { - e.metrics.RequestsTotalInc(UserError, err.ClassName(), err.QueryType()) - } - } else if e.isSyntaxRelatedError(gqlErr) { - if e.metrics != nil { - e.metrics.RequestsTotalInc(UserError, "", "") - } - } else { - e.logServerError(gqlErr, e.getClassName(gqlErr.Path), e.getQueryType(gqlErr.Path)) - } - } - } else if result.Data != nil { - e.logOk(result.Data) - } -} - -func (e *graphqlRequestsTotal) logServerError(err error, className, queryType string) { - e.logger.WithFields(logrus.Fields{ - "action": "requests_total", - "api": "graphql", - "query_type": queryType, - "class_name": className, - }).WithError(err).Error("unexpected error") - if e.metrics != nil { - e.metrics.RequestsTotalInc(ServerError, className, queryType) - } -} - -func (e *graphqlRequestsTotal) logUserError() { - if e.metrics != nil { - e.metrics.RequestsTotalInc(UserError, "", "") - } -} - -func (e *graphqlRequestsTotal) logOk(data interface{}) { - if e.metrics != nil { - className, queryType := e.getClassNameAndQueryType(data) - e.metrics.RequestsTotalInc(Ok, className, queryType) - } -} - -func (e *graphqlRequestsTotal) getClassNameAndQueryType(data interface{}) (className, queryType string) { - dataMap, ok := data.(map[string]interface{}) - if ok { - for query, value := range dataMap { - queryType = query - if queryType == "Explore" { - // Explore queries are cross class queries, we won't get a className in this case - // there's no sense in further value investigation - return - } - if value != nil { - if valueMap, ok := value.(map[string]interface{}); ok { - for class := range valueMap { - className = class - return - } - } - } - } - } - return -} diff --git a/adapters/handlers/rest/handlers_misc.go b/adapters/handlers/rest/handlers_misc.go deleted file mode 100644 index 23dec6db3bff5e2def616fd36ff0d3bffb75a513..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/handlers_misc.go +++ /dev/null @@ -1,152 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package rest - -import ( - "fmt" - "net/url" - - middleware "github.com/go-openapi/runtime/middleware" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/adapters/handlers/rest/operations" - "github.com/weaviate/weaviate/adapters/handlers/rest/operations/meta" - "github.com/weaviate/weaviate/adapters/handlers/rest/operations/well_known" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/usecases/config" - "github.com/weaviate/weaviate/usecases/monitoring" -) - -type schemaManager interface { - GetSchema(principal *models.Principal) (schema.Schema, error) - GetSchemaSkipAuth() schema.Schema -} - -func setupMiscHandlers(api *operations.WeaviateAPI, serverConfig *config.WeaviateConfig, - schemaManager schemaManager, modulesProvider ModulesProvider, metrics *monitoring.PrometheusMetrics, logger logrus.FieldLogger, -) { - metricRequestsTotal := newMiscRequestsTotal(metrics, logger) - api.MetaMetaGetHandler = meta.MetaGetHandlerFunc(func(params meta.MetaGetParams, principal *models.Principal) middleware.Responder { - var ( - metaInfos = map[string]interface{}{} - err error - ) - - if modulesProvider != nil { - metaInfos, err = modulesProvider.GetMeta() - if err != nil { - metricRequestsTotal.logError("", err) - return meta.NewMetaGetInternalServerError().WithPayload(errPayloadFromSingleErr(err)) - } - } - - res := &models.Meta{ - Hostname: serverConfig.GetHostAddress(), - Version: config.ServerVersion, - Modules: metaInfos, - } - metricRequestsTotal.logOk("") - return meta.NewMetaGetOK().WithPayload(res) - }) - - api.WellKnownGetWellKnownOpenidConfigurationHandler = well_known.GetWellKnownOpenidConfigurationHandlerFunc( - func(params well_known.GetWellKnownOpenidConfigurationParams, principal *models.Principal) middleware.Responder { - if !serverConfig.Config.Authentication.OIDC.Enabled { - metricRequestsTotal.logUserError("") - return well_known.NewGetWellKnownOpenidConfigurationNotFound() - } - - target, err := url.JoinPath(serverConfig.Config.Authentication.OIDC.Issuer, "/.well-known/openid-configuration") - if err != nil { - metricRequestsTotal.logError("", err) - return well_known.NewGetWellKnownOpenidConfigurationInternalServerError().WithPayload(errPayloadFromSingleErr(err)) - } - clientID := serverConfig.Config.Authentication.OIDC.ClientID - scopes := serverConfig.Config.Authentication.OIDC.Scopes - body := &well_known.GetWellKnownOpenidConfigurationOKBody{ - Href: target, - ClientID: clientID, - Scopes: scopes, - } - - metricRequestsTotal.logOk("") - return well_known.NewGetWellKnownOpenidConfigurationOK().WithPayload(body) - }) - - api.WeaviateRootHandler = operations.WeaviateRootHandlerFunc( - func(params operations.WeaviateRootParams, principal *models.Principal) middleware.Responder { - origin := serverConfig.Config.Origin - body := &operations.WeaviateRootOKBody{ - Links: []*models.Link{ - { - Name: "Meta information about this instance/cluster", - Href: fmt.Sprintf("%s/v1/meta", origin), - }, - { - Name: "view complete schema", - Href: fmt.Sprintf("%s/v1/schema", origin), - DocumentationHref: "https://weaviate.io/developers/weaviate/api/rest/schema", - }, - { - Name: "CRUD schema", - Href: fmt.Sprintf("%s/v1/schema{/:className}", origin), - DocumentationHref: "https://weaviate.io/developers/weaviate/api/rest/schema", - }, - { - Name: "CRUD objects", - Href: fmt.Sprintf("%s/v1/objects{/:id}", origin), - DocumentationHref: "https://weaviate.io/developers/weaviate/api/rest/objects", - }, - { - Name: "trigger and view status of classifications", - Href: fmt.Sprintf("%s/v1/classifications{/:id}", origin), - DocumentationHref: "https://weaviate.io/developers/weaviate/api/rest/classification,https://weaviate.io/developers/weaviate/api/rest/classification#knn-classification", - }, - { - Name: "check if Weaviate is live (returns 200 on GET when live)", - Href: fmt.Sprintf("%s/v1/.well-known/live", origin), - DocumentationHref: "https://weaviate.io/developers/weaviate/api/rest/well-known#liveness", - }, - { - Name: "check if Weaviate is ready (returns 200 on GET when ready)", - Href: fmt.Sprintf("%s/v1/.well-known/ready", origin), - DocumentationHref: "https://weaviate.io/developers/weaviate/api/rest/well-known#readiness", - }, - { - Name: "view link to openid configuration (returns 404 on GET if no openid is configured)", - Href: fmt.Sprintf("%s/v1/.well-known/openid-configuration", origin), - DocumentationHref: "https://weaviate.io/developers/weaviate/api/rest/well-known#openid-configuration", - }, - }, - } - - metricRequestsTotal.logOk("") - return operations.NewWeaviateRootOK().WithPayload(body) - }) -} - -type miscRequestsTotal struct { - *restApiRequestsTotalImpl -} - -func newMiscRequestsTotal(metrics *monitoring.PrometheusMetrics, logger logrus.FieldLogger) restApiRequestsTotal { - return &miscRequestsTotal{ - restApiRequestsTotalImpl: &restApiRequestsTotalImpl{newRequestsTotalMetric(metrics, "rest"), "rest", "misc", logger}, - } -} - -func (e *miscRequestsTotal) logError(className string, err error) { - switch err.(type) { - default: - e.logServerError(className, err) - } -} diff --git a/adapters/handlers/rest/handlers_nodes.go b/adapters/handlers/rest/handlers_nodes.go deleted file mode 100644 index 4b8641f7705f8536e1732ff9fc5912847454052b..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/handlers_nodes.go +++ /dev/null @@ -1,125 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package rest - -import ( - "errors" - - "github.com/go-openapi/runtime/middleware" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/adapters/handlers/rest/operations" - "github.com/weaviate/weaviate/adapters/handlers/rest/operations/nodes" - "github.com/weaviate/weaviate/adapters/handlers/rest/state" - "github.com/weaviate/weaviate/adapters/repos/db" - enterrors "github.com/weaviate/weaviate/entities/errors" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/verbosity" - autherrs "github.com/weaviate/weaviate/usecases/auth/authorization/errors" - "github.com/weaviate/weaviate/usecases/monitoring" - nodesUC "github.com/weaviate/weaviate/usecases/nodes" - schemaUC "github.com/weaviate/weaviate/usecases/schema" -) - -type nodesHandlers struct { - manager *nodesUC.Manager - metricRequestsTotal restApiRequestsTotal -} - -func (n *nodesHandlers) getNodesStatus(params nodes.NodesGetParams, principal *models.Principal) middleware.Responder { - output, err := verbosity.ParseOutput(params.Output) - if err != nil { - return nodes.NewNodesGetUnprocessableEntity().WithPayload(errPayloadFromSingleErr(err)) - } - - nodeStatuses, err := n.manager.GetNodeStatus(params.HTTPRequest.Context(), principal, "", output) - if err != nil { - return n.handleGetNodesError(err) - } - - status := &models.NodesStatusResponse{ - Nodes: nodeStatuses, - } - - n.metricRequestsTotal.logOk("") - return nodes.NewNodesGetOK().WithPayload(status) -} - -func (n *nodesHandlers) getNodesStatusByClass(params nodes.NodesGetClassParams, principal *models.Principal) middleware.Responder { - output, err := verbosity.ParseOutput(params.Output) - if err != nil { - return nodes.NewNodesGetUnprocessableEntity().WithPayload(errPayloadFromSingleErr(err)) - } - - nodeStatuses, err := n.manager.GetNodeStatus(params.HTTPRequest.Context(), principal, params.ClassName, output) - if err != nil { - return n.handleGetNodesError(err) - } - - status := &models.NodesStatusResponse{ - Nodes: nodeStatuses, - } - - n.metricRequestsTotal.logOk("") - return nodes.NewNodesGetOK().WithPayload(status) -} - -func (n *nodesHandlers) handleGetNodesError(err error) middleware.Responder { - n.metricRequestsTotal.logError("", err) - if errors.As(err, &enterrors.ErrNotFound{}) { - return nodes.NewNodesGetClassNotFound(). - WithPayload(errPayloadFromSingleErr(err)) - } - if errors.As(err, &autherrs.Forbidden{}) { - return nodes.NewNodesGetClassForbidden(). - WithPayload(errPayloadFromSingleErr(err)) - } - if errors.As(err, &enterrors.ErrUnprocessable{}) { - return nodes.NewNodesGetClassUnprocessableEntity(). - WithPayload(errPayloadFromSingleErr(err)) - } - return nodes.NewNodesGetClassInternalServerError(). - WithPayload(errPayloadFromSingleErr(err)) -} - -func setupNodesHandlers(api *operations.WeaviateAPI, - schemaManger *schemaUC.Manager, repo *db.DB, appState *state.State, -) { - nodesManager := nodesUC.NewManager(appState.Logger, appState.Authorizer, - repo, schemaManger) - - h := &nodesHandlers{nodesManager, newNodesRequestsTotal(appState.Metrics, appState.Logger)} - api.NodesNodesGetHandler = nodes. - NodesGetHandlerFunc(h.getNodesStatus) - api.NodesNodesGetClassHandler = nodes. - NodesGetClassHandlerFunc(h.getNodesStatusByClass) -} - -type nodesRequestsTotal struct { - *restApiRequestsTotalImpl -} - -func newNodesRequestsTotal(metrics *monitoring.PrometheusMetrics, logger logrus.FieldLogger) restApiRequestsTotal { - return &nodesRequestsTotal{ - restApiRequestsTotalImpl: &restApiRequestsTotalImpl{newRequestsTotalMetric(metrics, "rest"), "rest", "nodes", logger}, - } -} - -func (e *nodesRequestsTotal) logError(className string, err error) { - switch err.(type) { - case enterrors.ErrNotFound, enterrors.ErrUnprocessable: - e.logUserError(className) - case autherrs.Forbidden: - e.logUserError(className) - default: - e.logServerError(className, err) - } -} diff --git a/adapters/handlers/rest/handlers_objects.go b/adapters/handlers/rest/handlers_objects.go deleted file mode 100644 index 289b4f22763bdd3845dae8351a6df59ee48b1120..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/handlers_objects.go +++ /dev/null @@ -1,943 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package rest - -import ( - "context" - "errors" - "fmt" - "strings" - - middleware "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/adapters/handlers/rest/operations" - "github.com/weaviate/weaviate/adapters/handlers/rest/operations/objects" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema/crossref" - autherrs "github.com/weaviate/weaviate/usecases/auth/authorization/errors" - "github.com/weaviate/weaviate/usecases/config" - "github.com/weaviate/weaviate/usecases/monitoring" - uco "github.com/weaviate/weaviate/usecases/objects" - "github.com/weaviate/weaviate/usecases/replica" -) - -type objectHandlers struct { - manager objectsManager - logger logrus.FieldLogger - config config.Config - modulesProvider ModulesProvider - metricRequestsTotal restApiRequestsTotal -} - -type ModulesProvider interface { - RestApiAdditionalProperties(includeProp string, class *models.Class) map[string]interface{} - GetMeta() (map[string]interface{}, error) - HasMultipleVectorizers() bool -} - -type objectsManager interface { - AddObject(context.Context, *models.Principal, *models.Object, - *additional.ReplicationProperties) (*models.Object, error) - ValidateObject(context.Context, *models.Principal, - *models.Object, *additional.ReplicationProperties) error - GetObject(context.Context, *models.Principal, string, strfmt.UUID, - additional.Properties, *additional.ReplicationProperties, string) (*models.Object, error) - DeleteObject(context.Context, *models.Principal, string, - strfmt.UUID, *additional.ReplicationProperties, string) error - UpdateObject(context.Context, *models.Principal, string, strfmt.UUID, - *models.Object, *additional.ReplicationProperties) (*models.Object, error) - HeadObject(ctx context.Context, principal *models.Principal, class string, id strfmt.UUID, - repl *additional.ReplicationProperties, tenant string) (bool, *uco.Error) - GetObjects(context.Context, *models.Principal, *int64, *int64, - *string, *string, *string, additional.Properties, string) ([]*models.Object, error) - Query(ctx context.Context, principal *models.Principal, - params *uco.QueryParams) ([]*models.Object, *uco.Error) - MergeObject(context.Context, *models.Principal, *models.Object, - *additional.ReplicationProperties) *uco.Error - AddObjectReference(context.Context, *models.Principal, *uco.AddReferenceInput, - *additional.ReplicationProperties, string) *uco.Error - UpdateObjectReferences(context.Context, *models.Principal, - *uco.PutReferenceInput, *additional.ReplicationProperties, string) *uco.Error - DeleteObjectReference(context.Context, *models.Principal, *uco.DeleteReferenceInput, - *additional.ReplicationProperties, string) *uco.Error - GetObjectsClass(ctx context.Context, principal *models.Principal, id strfmt.UUID) (*models.Class, error) - GetObjectClassFromName(ctx context.Context, principal *models.Principal, className string) (*models.Class, error) -} - -func (h *objectHandlers) addObject(params objects.ObjectsCreateParams, - principal *models.Principal, -) middleware.Responder { - repl, err := getReplicationProperties(params.ConsistencyLevel, nil) - if err != nil { - h.metricRequestsTotal.logError("", err) - return objects.NewObjectsCreateBadRequest(). - WithPayload(errPayloadFromSingleErr(err)) - } - className := getClassName(params.Body) - - object, err := h.manager.AddObject(params.HTTPRequest.Context(), - principal, params.Body, repl) - if err != nil { - h.metricRequestsTotal.logError(className, err) - if errors.As(err, &uco.ErrInvalidUserInput{}) { - return objects.NewObjectsCreateUnprocessableEntity(). - WithPayload(errPayloadFromSingleErr(err)) - } else if errors.As(err, &uco.ErrMultiTenancy{}) { - return objects.NewObjectsCreateUnprocessableEntity(). - WithPayload(errPayloadFromSingleErr(err)) - } else if errors.As(err, &autherrs.Forbidden{}) { - return objects.NewObjectsCreateForbidden(). - WithPayload(errPayloadFromSingleErr(err)) - } else { - return objects.NewObjectsCreateInternalServerError(). - WithPayload(errPayloadFromSingleErr(err)) - } - } - - propertiesMap, ok := object.Properties.(map[string]interface{}) - if ok { - object.Properties = h.extendPropertiesWithAPILinks(propertiesMap) - } - - h.metricRequestsTotal.logOk(className) - return objects.NewObjectsCreateOK().WithPayload(object) -} - -func (h *objectHandlers) validateObject(params objects.ObjectsValidateParams, - principal *models.Principal, -) middleware.Responder { - className := getClassName(params.Body) - err := h.manager.ValidateObject(params.HTTPRequest.Context(), principal, params.Body, nil) - if err != nil { - h.metricRequestsTotal.logError(className, err) - switch err.(type) { - case autherrs.Forbidden: - return objects.NewObjectsValidateForbidden(). - WithPayload(errPayloadFromSingleErr(err)) - case uco.ErrInvalidUserInput: - return objects.NewObjectsValidateUnprocessableEntity(). - WithPayload(errPayloadFromSingleErr(err)) - case uco.ErrMultiTenancy: - return objects.NewObjectsValidateUnprocessableEntity(). - WithPayload(errPayloadFromSingleErr(err)) - default: - return objects.NewObjectsValidateInternalServerError(). - WithPayload(errPayloadFromSingleErr(err)) - } - } - - h.metricRequestsTotal.logOk(className) - return objects.NewObjectsValidateOK() -} - -// getObject gets object of a specific class -func (h *objectHandlers) getObject(params objects.ObjectsClassGetParams, - principal *models.Principal, -) middleware.Responder { - var additional additional.Properties - - // The process to extract additional params depends on knowing the schema - // which in turn requires a preflight load of the object. We can save this - // second db request if we know that the user did not specify any additional - // params. This could potentially be optimized further by checking if only - // non-module specific params are contained and decide then, but we do not - // know if this path is critical enough for this level of optimization. - if params.Include != nil { - var class *models.Class - var err error - if params.ClassName == "" { // deprecated request without classname - class, err = h.manager.GetObjectsClass(params.HTTPRequest.Context(), principal, params.ID) - } else { - class, err = h.manager.GetObjectClassFromName(params.HTTPRequest.Context(), principal, params.ClassName) - } - if err != nil { - h.metricRequestsTotal.logUserError(params.ClassName) - return objects.NewObjectsClassGetBadRequest(). - WithPayload(errPayloadFromSingleErr(err)) - } - - additional, err = parseIncludeParam(params.Include, h.modulesProvider, true, class) - if err != nil { - h.metricRequestsTotal.logError(params.ClassName, err) - return objects.NewObjectsClassGetBadRequest(). - WithPayload(errPayloadFromSingleErr(err)) - } - } - - replProps, err := getReplicationProperties(params.ConsistencyLevel, params.NodeName) - if err != nil { - h.metricRequestsTotal.logError(params.ClassName, err) - return objects.NewObjectsClassGetBadRequest(). - WithPayload(errPayloadFromSingleErr(err)) - } - - tenant := getTenant(params.Tenant) - - object, err := h.manager.GetObject(params.HTTPRequest.Context(), principal, - params.ClassName, params.ID, additional, replProps, tenant) - if err != nil { - h.metricRequestsTotal.logError(getClassName(object), err) - switch err.(type) { - case autherrs.Forbidden: - return objects.NewObjectsClassGetForbidden(). - WithPayload(errPayloadFromSingleErr(err)) - case uco.ErrNotFound: - return objects.NewObjectsClassGetNotFound() - case uco.ErrMultiTenancy: - return objects.NewObjectsClassGetUnprocessableEntity(). - WithPayload(errPayloadFromSingleErr(err)) - default: - return objects.NewObjectsClassGetInternalServerError(). - WithPayload(errPayloadFromSingleErr(err)) - } - } - - propertiesMap, ok := object.Properties.(map[string]interface{}) - if ok { - object.Properties = h.extendPropertiesWithAPILinks(propertiesMap) - } - - h.metricRequestsTotal.logOk(getClassName(object)) - return objects.NewObjectsClassGetOK().WithPayload(object) -} - -func (h *objectHandlers) getObjects(params objects.ObjectsListParams, - principal *models.Principal, -) middleware.Responder { - if params.Class != nil && *params.Class != "" { - return h.query(params, principal) - } - additional, err := parseIncludeParam(params.Include, h.modulesProvider, h.shouldIncludeGetObjectsModuleParams(), nil) - if err != nil { - h.metricRequestsTotal.logError("", err) - return objects.NewObjectsListBadRequest(). - WithPayload(errPayloadFromSingleErr(err)) - } - - var deprecationsRes []*models.Deprecation - - list, err := h.manager.GetObjects(params.HTTPRequest.Context(), principal, - params.Offset, params.Limit, params.Sort, params.Order, params.After, additional, - getTenant(params.Tenant)) - if err != nil { - h.metricRequestsTotal.logError("", err) - switch err.(type) { - case autherrs.Forbidden: - return objects.NewObjectsListForbidden(). - WithPayload(errPayloadFromSingleErr(err)) - case uco.ErrMultiTenancy: - return objects.NewObjectsListUnprocessableEntity(). - WithPayload(errPayloadFromSingleErr(err)) - default: - return objects.NewObjectsListInternalServerError(). - WithPayload(errPayloadFromSingleErr(err)) - } - } - - for i, object := range list { - propertiesMap, ok := object.Properties.(map[string]interface{}) - if ok { - list[i].Properties = h.extendPropertiesWithAPILinks(propertiesMap) - } - } - - h.metricRequestsTotal.logOk("") - return objects.NewObjectsListOK(). - WithPayload(&models.ObjectsListResponse{ - Objects: list, - TotalResults: int64(len(list)), - Deprecations: deprecationsRes, - }) -} - -func (h *objectHandlers) query(params objects.ObjectsListParams, - principal *models.Principal, -) middleware.Responder { - additional, err := parseIncludeParam(params.Include, h.modulesProvider, h.shouldIncludeGetObjectsModuleParams(), nil) - if err != nil { - h.metricRequestsTotal.logError(*params.Class, err) - return objects.NewObjectsListBadRequest(). - WithPayload(errPayloadFromSingleErr(err)) - } - req := uco.QueryParams{ - Class: *params.Class, - Offset: params.Offset, - Limit: params.Limit, - After: params.After, - Sort: params.Sort, - Order: params.Order, - Tenant: params.Tenant, - Additional: additional, - } - resultSet, rerr := h.manager.Query(params.HTTPRequest.Context(), principal, &req) - if rerr != nil { - h.metricRequestsTotal.logError(req.Class, rerr) - switch rerr.Code { - case uco.StatusForbidden: - return objects.NewObjectsListForbidden(). - WithPayload(errPayloadFromSingleErr(rerr)) - case uco.StatusNotFound: - return objects.NewObjectsListNotFound() - case uco.StatusBadRequest: - return objects.NewObjectsListUnprocessableEntity(). - WithPayload(errPayloadFromSingleErr(rerr)) - case uco.StatusUnprocessableEntity: - return objects.NewObjectsListUnprocessableEntity(). - WithPayload(errPayloadFromSingleErr(rerr)) - default: - return objects.NewObjectsListInternalServerError(). - WithPayload(errPayloadFromSingleErr(rerr)) - } - } - - for i, object := range resultSet { - propertiesMap, ok := object.Properties.(map[string]interface{}) - if ok { - resultSet[i].Properties = h.extendPropertiesWithAPILinks(propertiesMap) - } - } - - h.metricRequestsTotal.logOk(req.Class) - return objects.NewObjectsListOK(). - WithPayload(&models.ObjectsListResponse{ - Objects: resultSet, - TotalResults: int64(len(resultSet)), - Deprecations: []*models.Deprecation{}, - }) -} - -// deleteObject delete a single object of giving class -func (h *objectHandlers) deleteObject(params objects.ObjectsClassDeleteParams, - principal *models.Principal, -) middleware.Responder { - repl, err := getReplicationProperties(params.ConsistencyLevel, nil) - if err != nil { - h.metricRequestsTotal.logError(params.ClassName, err) - return objects.NewObjectsCreateBadRequest(). - WithPayload(errPayloadFromSingleErr(err)) - } - - tenant := getTenant(params.Tenant) - - err = h.manager.DeleteObject(params.HTTPRequest.Context(), - principal, params.ClassName, params.ID, repl, tenant) - if err != nil { - h.metricRequestsTotal.logError(params.ClassName, err) - switch err.(type) { - case autherrs.Forbidden: - return objects.NewObjectsClassDeleteForbidden(). - WithPayload(errPayloadFromSingleErr(err)) - case uco.ErrNotFound: - return objects.NewObjectsClassDeleteNotFound() - case uco.ErrMultiTenancy: - return objects.NewObjectsClassDeleteUnprocessableEntity(). - WithPayload(errPayloadFromSingleErr(err)) - default: - return objects.NewObjectsClassDeleteInternalServerError(). - WithPayload(errPayloadFromSingleErr(err)) - } - } - - h.metricRequestsTotal.logOk(params.ClassName) - return objects.NewObjectsClassDeleteNoContent() -} - -func (h *objectHandlers) updateObject(params objects.ObjectsClassPutParams, - principal *models.Principal, -) middleware.Responder { - className := getClassName(params.Body) - repl, err := getReplicationProperties(params.ConsistencyLevel, nil) - if err != nil { - h.metricRequestsTotal.logError(className, err) - return objects.NewObjectsCreateBadRequest(). - WithPayload(errPayloadFromSingleErr(err)) - } - - object, err := h.manager.UpdateObject(params.HTTPRequest.Context(), - principal, params.ClassName, params.ID, params.Body, repl) - if err != nil { - h.metricRequestsTotal.logError(className, err) - if errors.As(err, &uco.ErrInvalidUserInput{}) { - return objects.NewObjectsClassPutUnprocessableEntity(). - WithPayload(errPayloadFromSingleErr(err)) - } else if errors.As(err, &uco.ErrMultiTenancy{}) { - return objects.NewObjectsClassPutUnprocessableEntity(). - WithPayload(errPayloadFromSingleErr(err)) - } else if errors.As(err, &autherrs.Forbidden{}) { - return objects.NewObjectsClassPutForbidden(). - WithPayload(errPayloadFromSingleErr(err)) - } else { - return objects.NewObjectsClassPutInternalServerError(). - WithPayload(errPayloadFromSingleErr(err)) - } - } - - propertiesMap, ok := object.Properties.(map[string]interface{}) - if ok { - object.Properties = h.extendPropertiesWithAPILinks(propertiesMap) - } - - h.metricRequestsTotal.logOk(className) - return objects.NewObjectsClassPutOK().WithPayload(object) -} - -func (h *objectHandlers) headObject(params objects.ObjectsClassHeadParams, - principal *models.Principal, -) middleware.Responder { - repl, err := getReplicationProperties(params.ConsistencyLevel, nil) - if err != nil { - h.metricRequestsTotal.logError(params.ClassName, err) - return objects.NewObjectsCreateBadRequest(). - WithPayload(errPayloadFromSingleErr(err)) - } - - tenant := getTenant(params.Tenant) - - exists, objErr := h.manager.HeadObject(params.HTTPRequest.Context(), - principal, params.ClassName, params.ID, repl, tenant) - if objErr != nil { - h.metricRequestsTotal.logError(params.ClassName, objErr) - switch { - case objErr.Forbidden(): - return objects.NewObjectsClassHeadForbidden(). - WithPayload(errPayloadFromSingleErr(objErr)) - case objErr.UnprocessableEntity(): - return objects.NewObjectsClassHeadUnprocessableEntity(). - WithPayload(errPayloadFromSingleErr(objErr)) - default: - return objects.NewObjectsClassHeadInternalServerError(). - WithPayload(errPayloadFromSingleErr(objErr)) - } - } - - h.metricRequestsTotal.logOk(params.ClassName) - if !exists { - return objects.NewObjectsClassHeadNotFound() - } - return objects.NewObjectsClassHeadNoContent() -} - -func (h *objectHandlers) patchObject(params objects.ObjectsClassPatchParams, principal *models.Principal) middleware.Responder { - updates := params.Body - updates.ID = params.ID - updates.Class = params.ClassName - - repl, err := getReplicationProperties(params.ConsistencyLevel, nil) - if err != nil { - h.metricRequestsTotal.logError(getClassName(updates), err) - return objects.NewObjectsCreateBadRequest(). - WithPayload(errPayloadFromSingleErr(err)) - } - - objErr := h.manager.MergeObject(params.HTTPRequest.Context(), principal, updates, repl) - if objErr != nil { - h.metricRequestsTotal.logError(getClassName(updates), objErr) - switch { - case objErr.NotFound(): - return objects.NewObjectsClassPatchNotFound() - case objErr.Forbidden(): - return objects.NewObjectsClassPatchForbidden(). - WithPayload(errPayloadFromSingleErr(objErr)) - case objErr.BadRequest(): - return objects.NewObjectsClassPatchUnprocessableEntity(). - WithPayload(errPayloadFromSingleErr(objErr)) - case objErr.UnprocessableEntity(): - return objects.NewObjectsClassPatchUnprocessableEntity(). - WithPayload(errPayloadFromSingleErr(objErr)) - default: - return objects.NewObjectsClassPatchInternalServerError(). - WithPayload(errPayloadFromSingleErr(objErr)) - } - } - - h.metricRequestsTotal.logOk(getClassName(updates)) - return objects.NewObjectsClassPatchNoContent() -} - -func (h *objectHandlers) addObjectReference( - params objects.ObjectsClassReferencesCreateParams, - principal *models.Principal, -) middleware.Responder { - input := uco.AddReferenceInput{ - Class: params.ClassName, - ID: params.ID, - Property: params.PropertyName, - Ref: *params.Body, - } - - repl, err := getReplicationProperties(params.ConsistencyLevel, nil) - if err != nil { - h.metricRequestsTotal.logError(params.ClassName, err) - return objects.NewObjectsCreateBadRequest(). - WithPayload(errPayloadFromSingleErr(err)) - } - tenant := getTenant(params.Tenant) - - objErr := h.manager.AddObjectReference(params.HTTPRequest.Context(), principal, &input, repl, tenant) - if objErr != nil { - h.metricRequestsTotal.logError(params.ClassName, objErr) - switch { - case objErr.Forbidden(): - return objects.NewObjectsClassReferencesCreateForbidden(). - WithPayload(errPayloadFromSingleErr(objErr)) - case objErr.NotFound(): - return objects.NewObjectsClassReferencesCreateNotFound() - case objErr.BadRequest(): - return objects.NewObjectsClassReferencesCreateUnprocessableEntity(). - WithPayload(errPayloadFromSingleErr(objErr)) - case objErr.UnprocessableEntity(): - return objects.NewObjectsClassReferencesCreateUnprocessableEntity(). - WithPayload(errPayloadFromSingleErr(objErr)) - default: - return objects.NewObjectsClassReferencesCreateInternalServerError(). - WithPayload(errPayloadFromSingleErr(objErr)) - } - } - - h.metricRequestsTotal.logOk(params.ClassName) - return objects.NewObjectsClassReferencesCreateOK() -} - -func (h *objectHandlers) putObjectReferences(params objects.ObjectsClassReferencesPutParams, - principal *models.Principal, -) middleware.Responder { - input := uco.PutReferenceInput{ - Class: params.ClassName, - ID: params.ID, - Property: params.PropertyName, - Refs: params.Body, - } - - repl, err := getReplicationProperties(params.ConsistencyLevel, nil) - if err != nil { - h.metricRequestsTotal.logError(params.ClassName, err) - return objects.NewObjectsCreateBadRequest(). - WithPayload(errPayloadFromSingleErr(err)) - } - - tenant := getTenant(params.Tenant) - - objErr := h.manager.UpdateObjectReferences(params.HTTPRequest.Context(), principal, &input, repl, tenant) - if objErr != nil { - h.metricRequestsTotal.logError(params.ClassName, objErr) - switch { - case objErr.Forbidden(): - return objects.NewObjectsClassReferencesPutForbidden(). - WithPayload(errPayloadFromSingleErr(objErr)) - case objErr.NotFound(): - return objects.NewObjectsClassReferencesPutNotFound() - case objErr.BadRequest(): - return objects.NewObjectsClassReferencesPutUnprocessableEntity(). - WithPayload(errPayloadFromSingleErr(objErr)) - case objErr.UnprocessableEntity(): - return objects.NewObjectsClassReferencesPutUnprocessableEntity(). - WithPayload(errPayloadFromSingleErr(objErr)) - default: - return objects.NewObjectsClassReferencesPutInternalServerError(). - WithPayload(errPayloadFromSingleErr(objErr)) - } - } - - h.metricRequestsTotal.logOk(params.ClassName) - return objects.NewObjectsClassReferencesPutOK() -} - -func (h *objectHandlers) deleteObjectReference(params objects.ObjectsClassReferencesDeleteParams, - principal *models.Principal, -) middleware.Responder { - input := uco.DeleteReferenceInput{ - Class: params.ClassName, - ID: params.ID, - Property: params.PropertyName, - Reference: *params.Body, - } - - repl, err := getReplicationProperties(params.ConsistencyLevel, nil) - if err != nil { - h.metricRequestsTotal.logError(params.ClassName, err) - return objects.NewObjectsCreateBadRequest(). - WithPayload(errPayloadFromSingleErr(err)) - } - tenant := getTenant(params.Tenant) - - objErr := h.manager.DeleteObjectReference(params.HTTPRequest.Context(), principal, &input, repl, tenant) - if objErr != nil { - h.metricRequestsTotal.logError(params.ClassName, objErr) - switch objErr.Code { - case uco.StatusForbidden: - return objects.NewObjectsClassReferencesDeleteForbidden(). - WithPayload(errPayloadFromSingleErr(objErr)) - case uco.StatusNotFound: - return objects.NewObjectsClassReferencesDeleteNotFound() - case uco.StatusBadRequest: - return objects.NewObjectsClassReferencesDeleteUnprocessableEntity(). - WithPayload(errPayloadFromSingleErr(objErr)) - case uco.StatusUnprocessableEntity: - return objects.NewObjectsClassReferencesDeleteUnprocessableEntity(). - WithPayload(errPayloadFromSingleErr(objErr)) - default: - return objects.NewObjectsClassReferencesDeleteInternalServerError(). - WithPayload(errPayloadFromSingleErr(objErr)) - } - } - - h.metricRequestsTotal.logOk(params.ClassName) - return objects.NewObjectsClassReferencesDeleteNoContent() -} - -func setupObjectHandlers(api *operations.WeaviateAPI, - manager *uco.Manager, config config.Config, logger logrus.FieldLogger, - modulesProvider ModulesProvider, metrics *monitoring.PrometheusMetrics, -) { - h := &objectHandlers{manager, logger, config, modulesProvider, newObjectsRequestsTotal(metrics, logger)} - api.ObjectsObjectsCreateHandler = objects. - ObjectsCreateHandlerFunc(h.addObject) - api.ObjectsObjectsValidateHandler = objects. - ObjectsValidateHandlerFunc(h.validateObject) - api.ObjectsObjectsClassGetHandler = objects. - ObjectsClassGetHandlerFunc(h.getObject) - api.ObjectsObjectsClassHeadHandler = objects. - ObjectsClassHeadHandlerFunc(h.headObject) - api.ObjectsObjectsClassDeleteHandler = objects. - ObjectsClassDeleteHandlerFunc(h.deleteObject) - api.ObjectsObjectsListHandler = objects. - ObjectsListHandlerFunc(h.getObjects) - api.ObjectsObjectsClassPutHandler = objects. - ObjectsClassPutHandlerFunc(h.updateObject) - api.ObjectsObjectsClassPatchHandler = objects. - ObjectsClassPatchHandlerFunc(h.patchObject) - api.ObjectsObjectsClassReferencesCreateHandler = objects. - ObjectsClassReferencesCreateHandlerFunc(h.addObjectReference) - api.ObjectsObjectsClassReferencesDeleteHandler = objects. - ObjectsClassReferencesDeleteHandlerFunc(h.deleteObjectReference) - api.ObjectsObjectsClassReferencesPutHandler = objects. - ObjectsClassReferencesPutHandlerFunc(h.putObjectReferences) - // deprecated handlers - api.ObjectsObjectsGetHandler = objects. - ObjectsGetHandlerFunc(h.getObjectDeprecated) - api.ObjectsObjectsDeleteHandler = objects. - ObjectsDeleteHandlerFunc(h.deleteObjectDeprecated) - api.ObjectsObjectsHeadHandler = objects. - ObjectsHeadHandlerFunc(h.headObjectDeprecated) - api.ObjectsObjectsUpdateHandler = objects. - ObjectsUpdateHandlerFunc(h.updateObjectDeprecated) - api.ObjectsObjectsPatchHandler = objects. - ObjectsPatchHandlerFunc(h.patchObjectDeprecated) - api.ObjectsObjectsReferencesCreateHandler = objects. - ObjectsReferencesCreateHandlerFunc(h.addObjectReferenceDeprecated) - api.ObjectsObjectsReferencesUpdateHandler = objects. - ObjectsReferencesUpdateHandlerFunc(h.updateObjectReferencesDeprecated) - api.ObjectsObjectsReferencesDeleteHandler = objects. - ObjectsReferencesDeleteHandlerFunc(h.deleteObjectReferenceDeprecated) -} - -func (h *objectHandlers) getObjectDeprecated(params objects.ObjectsGetParams, - principal *models.Principal, -) middleware.Responder { - h.logger.Warn("deprecated endpoint: ", "GET "+params.HTTPRequest.URL.Path) - ps := objects.ObjectsClassGetParams{ - HTTPRequest: params.HTTPRequest, - ID: params.ID, - Include: params.Include, - } - return h.getObject(ps, principal) -} - -func (h *objectHandlers) headObjectDeprecated(params objects.ObjectsHeadParams, - principal *models.Principal, -) middleware.Responder { - h.logger.Warn("deprecated endpoint: ", "HEAD "+params.HTTPRequest.URL.Path) - r := objects.ObjectsClassHeadParams{ - HTTPRequest: params.HTTPRequest, - ID: params.ID, - } - return h.headObject(r, principal) -} - -func (h *objectHandlers) patchObjectDeprecated(params objects.ObjectsPatchParams, principal *models.Principal) middleware.Responder { - h.logger.Warn("deprecated endpoint: ", "PATCH "+params.HTTPRequest.URL.Path) - args := objects.ObjectsClassPatchParams{ - HTTPRequest: params.HTTPRequest, - ID: params.ID, - Body: params.Body, - } - if params.Body != nil { - args.ClassName = params.Body.Class - } - return h.patchObject(args, principal) -} - -func (h *objectHandlers) updateObjectDeprecated(params objects.ObjectsUpdateParams, - principal *models.Principal, -) middleware.Responder { - h.logger.Warn("deprecated endpoint: ", "PUT "+params.HTTPRequest.URL.Path) - ps := objects.ObjectsClassPutParams{ - HTTPRequest: params.HTTPRequest, - ClassName: params.Body.Class, - Body: params.Body, - ID: params.ID, - } - return h.updateObject(ps, principal) -} - -func (h *objectHandlers) deleteObjectDeprecated(params objects.ObjectsDeleteParams, - principal *models.Principal, -) middleware.Responder { - h.logger.Warn("deprecated endpoint: ", "DELETE "+params.HTTPRequest.URL.Path) - ps := objects.ObjectsClassDeleteParams{ - HTTPRequest: params.HTTPRequest, - ID: params.ID, - } - return h.deleteObject(ps, principal) -} - -func (h *objectHandlers) addObjectReferenceDeprecated(params objects.ObjectsReferencesCreateParams, - principal *models.Principal, -) middleware.Responder { - h.logger.Warn("deprecated endpoint: ", "POST "+params.HTTPRequest.URL.Path) - req := objects.ObjectsClassReferencesCreateParams{ - HTTPRequest: params.HTTPRequest, - Body: params.Body, - ID: params.ID, - PropertyName: params.PropertyName, - } - return h.addObjectReference(req, principal) -} - -func (h *objectHandlers) updateObjectReferencesDeprecated(params objects.ObjectsReferencesUpdateParams, - principal *models.Principal, -) middleware.Responder { - h.logger.Warn("deprecated endpoint: ", "PUT "+params.HTTPRequest.URL.Path) - req := objects.ObjectsClassReferencesPutParams{ - HTTPRequest: params.HTTPRequest, - ID: params.ID, - PropertyName: params.PropertyName, - Body: params.Body, - } - return h.putObjectReferences(req, principal) -} - -func (h *objectHandlers) deleteObjectReferenceDeprecated(params objects.ObjectsReferencesDeleteParams, - principal *models.Principal, -) middleware.Responder { - h.logger.Warn("deprecated endpoint: ", "DELETE "+params.HTTPRequest.URL.Path) - req := objects.ObjectsClassReferencesDeleteParams{ - HTTPRequest: params.HTTPRequest, - Body: params.Body, - ID: params.ID, - PropertyName: params.PropertyName, - } - return h.deleteObjectReference(req, principal) -} - -func (h *objectHandlers) extendPropertiesWithAPILinks(schema map[string]interface{}) map[string]interface{} { - if schema == nil { - return schema - } - - for key, value := range schema { - asMultiRef, ok := value.(models.MultipleRef) - if !ok { - continue - } - - schema[key] = h.extendReferencesWithAPILinks(asMultiRef) - } - return schema -} - -func (h *objectHandlers) extendReferencesWithAPILinks(refs models.MultipleRef) models.MultipleRef { - for i, ref := range refs { - refs[i] = h.extendReferenceWithAPILink(ref) - } - - return refs -} - -func (h *objectHandlers) extendReferenceWithAPILink(ref *models.SingleRef) *models.SingleRef { - parsed, err := crossref.Parse(ref.Beacon.String()) - if err != nil { - // ignore return unchanged - return ref - } - href := fmt.Sprintf("%s/v1/objects/%s/%s", h.config.Origin, parsed.Class, parsed.TargetID) - if parsed.Class == "" { - href = fmt.Sprintf("%s/v1/objects/%s", h.config.Origin, parsed.TargetID) - } - ref.Href = strfmt.URI(href) - return ref -} - -func (h *objectHandlers) shouldIncludeGetObjectsModuleParams() bool { - if h.modulesProvider == nil || !h.modulesProvider.HasMultipleVectorizers() { - return true - } - return false -} - -type objectsRequestsTotal struct { - *restApiRequestsTotalImpl -} - -func newObjectsRequestsTotal(metrics *monitoring.PrometheusMetrics, logger logrus.FieldLogger) restApiRequestsTotal { - return &objectsRequestsTotal{ - restApiRequestsTotalImpl: &restApiRequestsTotalImpl{newRequestsTotalMetric(metrics, "rest"), "rest", "objects", logger}, - } -} - -func (e *objectsRequestsTotal) logError(className string, err error) { - switch err := err.(type) { - case uco.ErrMultiTenancy: - e.logUserError(className) - case errReplication, errUnregonizedProperty: - e.logUserError(className) - case autherrs.Forbidden: - e.logUserError(className) - case uco.ErrInvalidUserInput, uco.ErrNotFound: - e.logUserError(className) - case *uco.Error: - switch err.Code { - case uco.StatusInternalServerError: - e.logServerError(className, err) - default: - e.logUserError(className) - } - default: - if errors.As(err, &uco.ErrInvalidUserInput{}) || - errors.As(err, &uco.ErrMultiTenancy{}) || - errors.As(err, &autherrs.Forbidden{}) { - e.logUserError(className) - } else { - e.logServerError(className, err) - } - } -} - -func parseIncludeParam(in *string, modulesProvider ModulesProvider, includeModuleParams bool, - class *models.Class, -) (additional.Properties, error) { - out := additional.Properties{} - if in == nil { - return out, nil - } - - parts := strings.Split(*in, ",") - - for _, prop := range parts { - if prop == "classification" { - out.Classification = true - out.RefMeta = true - continue - } - if prop == "vector" { - out.Vector = true - continue - } - if includeModuleParams && modulesProvider != nil { - moduleParams := modulesProvider.RestApiAdditionalProperties(prop, class) - if len(moduleParams) > 0 { - out.ModuleParams = getModuleParams(out.ModuleParams) - for param, value := range moduleParams { - out.ModuleParams[param] = value - } - continue - } - } - return out, newErrUnregonizedProperty(fmt.Errorf("unrecognized property '%s' in ?include list", prop)) - } - - return out, nil -} - -func getModuleParams(moduleParams map[string]interface{}) map[string]interface{} { - if moduleParams == nil { - return map[string]interface{}{} - } - return moduleParams -} - -func getReplicationProperties(consistencyLvl, nodeName *string) (*additional.ReplicationProperties, error) { - if nodeName == nil && consistencyLvl == nil { - return nil, nil - } - - repl := additional.ReplicationProperties{} - if nodeName != nil { - repl.NodeName = *nodeName - } - - cl, err := getConsistencyLevel(consistencyLvl) - if err != nil { - return nil, newErrReplication(err) - } - repl.ConsistencyLevel = cl - - if repl.ConsistencyLevel != "" && repl.NodeName != "" { - return nil, newErrReplication(fmt.Errorf("consistency_level and node_name are mutually exclusive")) - } - - return &repl, nil -} - -func getConsistencyLevel(lvl *string) (string, error) { - if lvl != nil { - switch replica.ConsistencyLevel(*lvl) { - case replica.One, replica.Quorum, replica.All: - return *lvl, nil - default: - return "", fmt.Errorf("unrecognized consistency level '%v', "+ - "try one of the following: ['ONE', 'QUORUM', 'ALL']", *lvl) - } - } - - return "", nil -} - -func getTenant(maybeKey *string) string { - if maybeKey != nil { - return *maybeKey - } - return "" -} - -func getClassName(obj *models.Object) string { - if obj != nil { - return obj.Class - } - return "" -} - -type errReplication struct { - err error -} - -func newErrReplication(err error) errReplication { - return errReplication{err} -} - -func (e errReplication) Error() string { - return fmt.Sprintf("%v", e.err) -} - -type errUnregonizedProperty struct { - err error -} - -func newErrUnregonizedProperty(err error) errUnregonizedProperty { - return errUnregonizedProperty{err} -} - -func (e errUnregonizedProperty) Error() string { - return fmt.Sprintf("%v", e.err) -} diff --git a/adapters/handlers/rest/handlers_objects_test.go b/adapters/handlers/rest/handlers_objects_test.go deleted file mode 100644 index c2c81160fafc07186a229700f77e9855b259fd3a..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/handlers_objects_test.go +++ /dev/null @@ -1,1155 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package rest - -import ( - "context" - stderrors "errors" - "net/http/httptest" - "testing" - - "github.com/go-openapi/strfmt" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/adapters/handlers/rest/operations/objects" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/usecases/auth/authorization/errors" - "github.com/weaviate/weaviate/usecases/config" - uco "github.com/weaviate/weaviate/usecases/objects" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestEnrichObjectsWithLinks(t *testing.T) { - t.Run("add object", func(t *testing.T) { - type test struct { - name string - object *models.Object - expectedResult *models.Object - } - - tests := []test{ - { - name: "without props - nothing changes", - object: &models.Object{Class: "Foo", Properties: nil}, - expectedResult: &models.Object{Class: "Foo", Properties: nil}, - }, - { - name: "without ref props - nothing changes", - object: &models.Object{Class: "Foo", Properties: map[string]interface{}{ - "name": "hello world", - "numericalField": 134, - }}, - expectedResult: &models.Object{Class: "Foo", Properties: map[string]interface{}{ - "name": "hello world", - "numericalField": 134, - }}, - }, - { - name: "with a ref prop - no origin configured", - object: &models.Object{Class: "Foo", Properties: map[string]interface{}{ - "name": "hello world", - "numericalField": 134, - "someRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: "weaviate://localhost/85f78e29-5937-4390-a121-5379f262b4e5", - }, - }, - }}, - expectedResult: &models.Object{Class: "Foo", Properties: map[string]interface{}{ - "name": "hello world", - "numericalField": 134, - "someRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: "weaviate://localhost/85f78e29-5937-4390-a121-5379f262b4e5", - Href: "/v1/objects/85f78e29-5937-4390-a121-5379f262b4e5", - }, - }, - }}, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - fakeManager := &fakeManager{ - addObjectReturn: test.object, - } - h := &objectHandlers{manager: fakeManager, metricRequestsTotal: &fakeMetricRequestsTotal{}} - res := h.addObject(objects.ObjectsCreateParams{ - HTTPRequest: httptest.NewRequest("POST", "/v1/objects", nil), - Body: test.object, - }, nil) - parsed, ok := res.(*objects.ObjectsCreateOK) - require.True(t, ok) - assert.Equal(t, test.expectedResult, parsed.Payload) - }) - } - }) - - // This test "with an origin configured" is not repeated for every handler, - // as testing this feature once was deemed sufficient - t.Run("add object - with an origin configured", func(t *testing.T) { - type test struct { - name string - object *models.Object - expectedResult *models.Object - } - - tests := []test{ - { - name: "without props - nothing changes", - object: &models.Object{Class: "Foo", Properties: nil}, - expectedResult: &models.Object{Class: "Foo", Properties: nil}, - }, - { - name: "without ref props - nothing changes", - object: &models.Object{Class: "Foo", Properties: map[string]interface{}{ - "name": "hello world", - "numericalField": 134, - }}, - expectedResult: &models.Object{Class: "Foo", Properties: map[string]interface{}{ - "name": "hello world", - "numericalField": 134, - }}, - }, - { - name: "with a ref prop - no origin configured", - object: &models.Object{Class: "Foo", Properties: map[string]interface{}{ - "name": "hello world", - "numericalField": 134, - "someRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: "weaviate://localhost/85f78e29-5937-4390-a121-5379f262b4e5", - }, - }, - }}, - expectedResult: &models.Object{Class: "Foo", Properties: map[string]interface{}{ - "name": "hello world", - "numericalField": 134, - "someRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: "weaviate://localhost/85f78e29-5937-4390-a121-5379f262b4e5", - Href: "https://awesomehost.com/v1/objects/85f78e29-5937-4390-a121-5379f262b4e5", - }, - }, - }}, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - fakeManager := &fakeManager{ - addObjectReturn: test.object, - } - config := config.Config{Origin: "https://awesomehost.com"} - h := &objectHandlers{manager: fakeManager, config: config, metricRequestsTotal: &fakeMetricRequestsTotal{}} - res := h.addObject(objects.ObjectsCreateParams{ - HTTPRequest: httptest.NewRequest("POST", "/v1/objects", nil), - Body: test.object, - }, nil) - parsed, ok := res.(*objects.ObjectsCreateOK) - require.True(t, ok) - assert.Equal(t, test.expectedResult, parsed.Payload) - }) - } - }) - - t.Run("get object deprecated", func(t *testing.T) { - type test struct { - name string - object *models.Object - expectedResult *models.Object - } - - tests := []test{ - { - name: "without props - nothing changes", - object: &models.Object{Class: "Foo", Properties: nil}, - expectedResult: &models.Object{Class: "Foo", Properties: nil}, - }, - { - name: "without ref props - nothing changes", - object: &models.Object{Class: "Foo", Properties: map[string]interface{}{ - "name": "hello world", - "numericalField": 134, - }}, - expectedResult: &models.Object{Class: "Foo", Properties: map[string]interface{}{ - "name": "hello world", - "numericalField": 134, - }}, - }, - { - name: "with a ref prop - no origin configured", - object: &models.Object{Class: "Foo", Properties: map[string]interface{}{ - "name": "hello world", - "numericalField": 134, - "someRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: "weaviate://localhost/85f78e29-5937-4390-a121-5379f262b4e5", - }, - }, - }}, - expectedResult: &models.Object{Class: "Foo", Properties: map[string]interface{}{ - "name": "hello world", - "numericalField": 134, - "someRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: "weaviate://localhost/85f78e29-5937-4390-a121-5379f262b4e5", - Href: "/v1/objects/85f78e29-5937-4390-a121-5379f262b4e5", - }, - }, - }}, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - fakeManager := &fakeManager{ - getObjectReturn: test.object, - } - h := &objectHandlers{manager: fakeManager, logger: &logrus.Logger{}, metricRequestsTotal: &fakeMetricRequestsTotal{}} - res := h.getObjectDeprecated(objects.ObjectsGetParams{HTTPRequest: httptest.NewRequest("GET", "/v1/objects", nil)}, nil) - parsed, ok := res.(*objects.ObjectsClassGetOK) - require.True(t, ok) - assert.Equal(t, test.expectedResult, parsed.Payload) - }) - } - }) - - t.Run("get objects", func(t *testing.T) { - type test struct { - name string - object []*models.Object - expectedResult []*models.Object - } - - tests := []test{ - { - name: "without props - nothing changes", - object: []*models.Object{{Class: "Foo", Properties: nil}}, - expectedResult: []*models.Object{{Class: "Foo", Properties: nil}}, - }, - { - name: "without ref props - nothing changes", - object: []*models.Object{ - {Class: "Foo", Properties: map[string]interface{}{ - "name": "hello world", - "numericalField": 134, - }}, - {Class: "Bar", Properties: map[string]interface{}{ - "name": "hello world", - "numericalField": 234, - }}, - }, - expectedResult: []*models.Object{ - {Class: "Foo", Properties: map[string]interface{}{ - "name": "hello world", - "numericalField": 134, - }}, - {Class: "Bar", Properties: map[string]interface{}{ - "name": "hello world", - "numericalField": 234, - }}, - }, - }, - { - name: "with a ref prop - no origin configured", - object: []*models.Object{ - {Class: "Foo", Properties: map[string]interface{}{ - "name": "hello world", - "numericalField": 134, - }}, - {Class: "Bar", Properties: map[string]interface{}{ - "name": "hello world", - "numericalField": 234, - "someRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: "weaviate://localhost/85f78e29-5937-4390-a121-5379f262b4e5", - }, - }, - }}, - }, - expectedResult: []*models.Object{ - {Class: "Foo", Properties: map[string]interface{}{ - "name": "hello world", - "numericalField": 134, - }}, - {Class: "Bar", Properties: map[string]interface{}{ - "name": "hello world", - "numericalField": 234, - "someRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: "weaviate://localhost/85f78e29-5937-4390-a121-5379f262b4e5", - Href: "/v1/objects/85f78e29-5937-4390-a121-5379f262b4e5", - }, - }, - }}, - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - fakeManager := &fakeManager{ - queryResult: test.object, - } - h := &objectHandlers{manager: fakeManager, metricRequestsTotal: &fakeMetricRequestsTotal{}} - res := h.getObjects(objects.ObjectsListParams{HTTPRequest: httptest.NewRequest("GET", "/v1/objects", nil)}, nil) - parsed, ok := res.(*objects.ObjectsListOK) - require.True(t, ok) - assert.Equal(t, test.expectedResult, parsed.Payload.Objects) - }) - } - }) - - t.Run("update object deprecated", func(t *testing.T) { - type test struct { - name string - object *models.Object - expectedResult *models.Object - } - - tests := []test{ - { - name: "without props - nothing changes", - object: &models.Object{Class: "Foo", Properties: nil}, - expectedResult: &models.Object{Class: "Foo", Properties: nil}, - }, - { - name: "without ref props - nothing changes", - object: &models.Object{Class: "Foo", Properties: map[string]interface{}{ - "name": "hello world", - "numericalField": 134, - }}, - expectedResult: &models.Object{Class: "Foo", Properties: map[string]interface{}{ - "name": "hello world", - "numericalField": 134, - }}, - }, - { - name: "with a ref prop - no origin configured", - object: &models.Object{Class: "Foo", Properties: map[string]interface{}{ - "name": "hello world", - "numericalField": 134, - "someRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: "weaviate://localhost/85f78e29-5937-4390-a121-5379f262b4e5", - }, - }, - }}, - expectedResult: &models.Object{Class: "Foo", Properties: map[string]interface{}{ - "name": "hello world", - "numericalField": 134, - "someRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: "weaviate://localhost/85f78e29-5937-4390-a121-5379f262b4e5", - Href: "/v1/objects/85f78e29-5937-4390-a121-5379f262b4e5", - }, - }, - }}, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - fakeManager := &fakeManager{ - updateObjectReturn: test.object, - } - h := &objectHandlers{manager: fakeManager, logger: &logrus.Logger{}, metricRequestsTotal: &fakeMetricRequestsTotal{}} - res := h.updateObjectDeprecated(objects.ObjectsUpdateParams{ - HTTPRequest: httptest.NewRequest("POST", "/v1/objects", nil), - Body: test.object, - }, nil) - parsed, ok := res.(*objects.ObjectsClassPutOK) - require.True(t, ok) - assert.Equal(t, test.expectedResult, parsed.Payload) - }) - } - }) - - t.Run("add object", func(t *testing.T) { - type test struct { - name string - object *models.Object - expectedResult *models.Object - } - - tests := []test{ - { - name: "without props - noaction changes", - object: &models.Object{Class: "Foo", Properties: nil}, - expectedResult: &models.Object{Class: "Foo", Properties: nil}, - }, - { - name: "without ref props - noaction changes", - object: &models.Object{Class: "Foo", Properties: map[string]interface{}{ - "name": "hello world", - "numericalField": 134, - }}, - expectedResult: &models.Object{Class: "Foo", Properties: map[string]interface{}{ - "name": "hello world", - "numericalField": 134, - }}, - }, - { - name: "with a ref prop - no origin configured", - object: &models.Object{Class: "Foo", Properties: map[string]interface{}{ - "name": "hello world", - "numericalField": 134, - "someRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: "weaviate://localhost/85f78e29-5937-4390-a121-5379f262b4e5", - }, - }, - }}, - expectedResult: &models.Object{Class: "Foo", Properties: map[string]interface{}{ - "name": "hello world", - "numericalField": 134, - "someRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: "weaviate://localhost/85f78e29-5937-4390-a121-5379f262b4e5", - Href: "/v1/objects/85f78e29-5937-4390-a121-5379f262b4e5", - }, - }, - }}, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - fakeManager := &fakeManager{ - addObjectReturn: test.object, - } - h := &objectHandlers{manager: fakeManager, metricRequestsTotal: &fakeMetricRequestsTotal{}} - res := h.addObject(objects.ObjectsCreateParams{ - HTTPRequest: httptest.NewRequest("POST", "/v1/objects", nil), - Body: test.object, - }, nil) - parsed, ok := res.(*objects.ObjectsCreateOK) - require.True(t, ok) - assert.Equal(t, test.expectedResult, parsed.Payload) - }) - } - }) - - t.Run("get objects", func(t *testing.T) { - type test struct { - name string - object []*models.Object - expectedResult []*models.Object - } - - tests := []test{ - { - name: "without props - noaction changes", - object: []*models.Object{{Class: "Foo", Properties: nil}}, - expectedResult: []*models.Object{{Class: "Foo", Properties: nil}}, - }, - { - name: "without ref props - noaction changes", - object: []*models.Object{ - {Class: "Foo", Properties: map[string]interface{}{ - "name": "hello world", - "numericalField": 134, - }}, - {Class: "Bar", Properties: map[string]interface{}{ - "name": "hello world", - "numericalField": 234, - }}, - }, - expectedResult: []*models.Object{ - {Class: "Foo", Properties: map[string]interface{}{ - "name": "hello world", - "numericalField": 134, - }}, - {Class: "Bar", Properties: map[string]interface{}{ - "name": "hello world", - "numericalField": 234, - }}, - }, - }, - { - name: "with a ref prop - no origin configured", - object: []*models.Object{ - {Class: "Foo", Properties: map[string]interface{}{ - "name": "hello world", - "numericalField": 134, - }}, - {Class: "Bar", Properties: map[string]interface{}{ - "name": "hello world", - "numericalField": 234, - "someRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: "weaviate://localhost/85f78e29-5937-4390-a121-5379f262b4e5", - }, - }, - }}, - }, - expectedResult: []*models.Object{ - {Class: "Foo", Properties: map[string]interface{}{ - "name": "hello world", - "numericalField": 134, - }}, - {Class: "Bar", Properties: map[string]interface{}{ - "name": "hello world", - "numericalField": 234, - "someRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: "weaviate://localhost/85f78e29-5937-4390-a121-5379f262b4e5", - Href: "/v1/objects/85f78e29-5937-4390-a121-5379f262b4e5", - }, - }, - }}, - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - fakeManager := &fakeManager{ - queryResult: test.object, - } - h := &objectHandlers{manager: fakeManager, metricRequestsTotal: &fakeMetricRequestsTotal{}} - res := h.getObjects(objects.ObjectsListParams{HTTPRequest: httptest.NewRequest("GET", "/v1/objects", nil)}, nil) - parsed, ok := res.(*objects.ObjectsListOK) - require.True(t, ok) - assert.Equal(t, test.expectedResult, parsed.Payload.Objects) - }) - } - }) - - // New endpoints which uniquely identify objects of a class - t.Run("UpdateObject", func(t *testing.T) { - cls := "MyClass" - type test struct { - name string - object *models.Object - expectedResult *models.Object - err error - } - - tests := []test{ - { - name: "without props - noaction changes", - object: &models.Object{Class: cls, Properties: nil}, - expectedResult: &models.Object{Class: cls, Properties: nil}, - }, - { - name: "without ref props - noaction changes", - object: &models.Object{Class: cls, Properties: map[string]interface{}{ - "name": "hello world", - "numericalField": 134, - }}, - expectedResult: &models.Object{Class: cls, Properties: map[string]interface{}{ - "name": "hello world", - "numericalField": 134, - }}, - }, - { - name: "with a ref prop - no origin configured", - object: &models.Object{Class: cls, Properties: map[string]interface{}{ - "name": "hello world", - "numericalField": 134, - "someRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: "weaviate://localhost/85f78e29-5937-4390-a121-5379f262b4e5", - }, - }, - }}, - expectedResult: &models.Object{Class: cls, Properties: map[string]interface{}{ - "name": "hello world", - "numericalField": 134, - "someRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: "weaviate://localhost/85f78e29-5937-4390-a121-5379f262b4e5", - Href: "/v1/objects/85f78e29-5937-4390-a121-5379f262b4e5", - }, - }, - }}, - }, - { - name: "forbidden", - err: errors.NewForbidden(&models.Principal{}, "get", "Myclass/123"), - }, - { - name: "validation", - err: uco.ErrInvalidUserInput{}, - }, - { - name: "not found", - err: uco.ErrNotFound{}, - }, - { - name: "unknown error", - err: stderrors.New("any error"), - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - fakeManager := &fakeManager{ - updateObjectReturn: test.object, - updateObjectErr: test.err, - } - h := &objectHandlers{manager: fakeManager, metricRequestsTotal: &fakeMetricRequestsTotal{}} - res := h.updateObject(objects.ObjectsClassPutParams{ - HTTPRequest: httptest.NewRequest("POST", "/v1/objects/123", nil), - Body: test.object, - ID: "123", - ClassName: cls, - }, nil) - parsed, ok := res.(*objects.ObjectsClassPutOK) - if test.err != nil { - require.False(t, ok) - return - } - require.True(t, ok) - assert.Equal(t, test.expectedResult, parsed.Payload) - }) - } - }) - - t.Run("PatchObject", func(t *testing.T) { - var ( - fakeManager = &fakeManager{} - fakeMetricRequestsTotal = &fakeMetricRequestsTotal{} - h = &objectHandlers{ - manager: fakeManager, - logger: &logrus.Logger{}, - metricRequestsTotal: fakeMetricRequestsTotal, - } - req = objects.ObjectsClassPatchParams{ - HTTPRequest: httptest.NewRequest("PATCH", "/v1/objects/MyClass/123", nil), - ClassName: "MyClass", - ID: "123", - Body: &models.Object{Properties: map[string]interface{}{"name": "hello world"}}, - } - ) - res := h.patchObject(req, nil) - if _, ok := res.(*objects.ObjectsClassPatchNoContent); !ok { - t.Errorf("unexpected result %v", res) - } - fakeManager.patchObjectReturn = &uco.Error{Code: uco.StatusBadRequest} - res = h.patchObject(req, nil) - if _, ok := res.(*objects.ObjectsClassPatchUnprocessableEntity); !ok { - t.Errorf("expected: %T got: %T", objects.ObjectsClassPatchUnprocessableEntity{}, res) - } - fakeManager.patchObjectReturn = &uco.Error{Code: uco.StatusNotFound} - res = h.patchObject(req, nil) - if _, ok := res.(*objects.ObjectsClassPatchNotFound); !ok { - t.Errorf("expected: %T got: %T", objects.ObjectsClassPatchNotFound{}, res) - } - fakeManager.patchObjectReturn = &uco.Error{Code: uco.StatusForbidden} - res = h.patchObject(req, nil) - if _, ok := res.(*objects.ObjectsClassPatchForbidden); !ok { - t.Errorf("expected: %T got: %T", objects.ObjectsClassPatchForbidden{}, res) - } - fakeManager.patchObjectReturn = &uco.Error{Code: uco.StatusInternalServerError} - res = h.patchObject(req, nil) - if _, ok := res.(*objects.ObjectsClassPatchInternalServerError); !ok { - t.Errorf("expected: %T got: %T", objects.ObjectsClassPatchInternalServerError{}, res) - } - - // test deprecated function - fakeManager.patchObjectReturn = nil - res = h.patchObjectDeprecated(objects.ObjectsPatchParams{ - HTTPRequest: httptest.NewRequest("PATCH", "/v1/objects/123", nil), - ID: "123", - Body: &models.Object{ - Class: "MyClass", - Properties: map[string]interface{}{"name": "hello world"}, - }, - }, nil) - if _, ok := res.(*objects.ObjectsClassPatchNoContent); !ok { - t.Errorf("unexpected result %v", res) - } - }) - - t.Run("GetObject", func(t *testing.T) { - cls := "MyClass" - type test struct { - name string - object *models.Object - err error - expectedResult *models.Object - } - - tests := []test{ - { - name: "without props - noaction changes", - object: &models.Object{Class: cls, Properties: nil}, - expectedResult: &models.Object{Class: cls, Properties: nil}, - }, - { - name: "without ref props - noaction changes", - object: &models.Object{Class: cls, Properties: map[string]interface{}{ - "name": "hello world", - "numericalField": 134, - }}, - expectedResult: &models.Object{Class: cls, Properties: map[string]interface{}{ - "name": "hello world", - "numericalField": 134, - }}, - }, - { - name: "with a ref prop - no origin configured", - object: &models.Object{Class: cls, Properties: map[string]interface{}{ - "name": "hello world", - "numericalField": 134, - "someRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: "weaviate://localhost/85f78e29-5937-4390-a121-5379f262b4e5", - }, - }, - }}, - expectedResult: &models.Object{Class: cls, Properties: map[string]interface{}{ - "name": "hello world", - "numericalField": 134, - "someRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: "weaviate://localhost/85f78e29-5937-4390-a121-5379f262b4e5", - Href: "/v1/objects/85f78e29-5937-4390-a121-5379f262b4e5", - }, - }, - }}, - }, - { - name: "error forbidden", - err: errors.NewForbidden(&models.Principal{}, "get", "Myclass/123"), - }, - { - name: "use case err not found", - err: uco.ErrNotFound{}, - }, - { - name: "any other error", - err: stderrors.New("unknown error"), - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - fakeManager := &fakeManager{ - getObjectReturn: test.object, - getObjectErr: test.err, - } - h := &objectHandlers{manager: fakeManager, metricRequestsTotal: &fakeMetricRequestsTotal{}} - req := objects.ObjectsClassGetParams{ - HTTPRequest: httptest.NewRequest("GET", "/v1/objects/MyClass/123", nil), - ClassName: cls, - ID: "123", - } - res := h.getObject(req, nil) - parsed, ok := res.(*objects.ObjectsClassGetOK) - if test.err != nil { - require.False(t, ok) - return - } - require.True(t, ok) - assert.Equal(t, test.expectedResult, parsed.Payload) - }) - } - }) - - t.Run("DeleteObject", func(t *testing.T) { - cls := "MyClass" - type test struct { - name string - err error - } - - tests := []test{ - { - name: "without props - noaction changes", - }, - { - name: "error forbidden", - err: errors.NewForbidden(&models.Principal{}, "get", "Myclass/123"), - }, - { - name: "use case err not found", - err: uco.ErrNotFound{}, - }, - { - name: "unknown error", - err: stderrors.New("any error"), - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - fakeManager := &fakeManager{ - deleteObjectReturn: test.err, - } - h := &objectHandlers{manager: fakeManager, metricRequestsTotal: &fakeMetricRequestsTotal{}} - req := objects.ObjectsClassDeleteParams{ - HTTPRequest: httptest.NewRequest("GET", "/v1/objects/MyClass/123", nil), - ClassName: cls, - ID: "123", - } - res := h.deleteObject(req, nil) - _, ok := res.(*objects.ObjectsClassDeleteNoContent) - if test.err != nil { - require.False(t, ok) - return - } - require.True(t, ok) - }) - } - }) - - t.Run("HeadObject", func(t *testing.T) { - m := &fakeManager{ - headObjectReturn: true, - } - h := &objectHandlers{manager: m, logger: &logrus.Logger{}, metricRequestsTotal: &fakeMetricRequestsTotal{}} - req := objects.ObjectsClassHeadParams{ - HTTPRequest: httptest.NewRequest("HEAD", "/v1/objects/MyClass/123", nil), - ClassName: "MyClass", - ID: "123", - } - res := h.headObject(req, nil) - if _, ok := res.(*objects.ObjectsClassHeadNoContent); !ok { - t.Errorf("unexpected result %v", res) - } - - m.headObjectErr = &uco.Error{Code: uco.StatusForbidden} - res = h.headObject(req, nil) - if _, ok := res.(*objects.ObjectsClassHeadForbidden); !ok { - t.Errorf("expected: %T got: %T", objects.ObjectsClassHeadForbidden{}, res) - } - m.headObjectErr = &uco.Error{Code: uco.StatusInternalServerError} - res = h.headObject(req, nil) - if _, ok := res.(*objects.ObjectsClassHeadInternalServerError); !ok { - t.Errorf("expected: %T got: %T", objects.ObjectsClassHeadInternalServerError{}, res) - } - m.headObjectErr = nil - m.headObjectReturn = false - res = h.headObject(req, nil) - if _, ok := res.(*objects.ObjectsClassHeadNotFound); !ok { - t.Errorf("expected: %T got: %T", objects.ObjectsClassHeadNotFound{}, res) - } - // same test as before but using old request - oldRequest := objects.ObjectsHeadParams{HTTPRequest: req.HTTPRequest} - res = h.headObjectDeprecated(oldRequest, nil) - if _, ok := res.(*objects.ObjectsClassHeadNotFound); !ok { - t.Errorf("expected: %T got: %T", objects.ObjectsClassHeadNotFound{}, res) - } - }) - - t.Run("PostReference", func(t *testing.T) { - m := &fakeManager{} - h := &objectHandlers{manager: m, logger: &logrus.Logger{}, metricRequestsTotal: &fakeMetricRequestsTotal{}} - req := objects.ObjectsClassReferencesCreateParams{ - HTTPRequest: httptest.NewRequest("HEAD", "/v1/objects/MyClass/123/references/prop", nil), - ClassName: "MyClass", - ID: "123", - Body: new(models.SingleRef), - PropertyName: "prop", - } - res := h.addObjectReference(req, nil) - if _, ok := res.(*objects.ObjectsClassReferencesCreateOK); !ok { - t.Errorf("unexpected result %v", res) - } - - m.addRefErr = &uco.Error{Code: uco.StatusForbidden} - res = h.addObjectReference(req, nil) - if _, ok := res.(*objects.ObjectsClassReferencesCreateForbidden); !ok { - t.Errorf("expected: %T got: %T", objects.ObjectsClassReferencesCreateForbidden{}, res) - } - // source object not found - m.addRefErr = &uco.Error{Code: uco.StatusNotFound} - res = h.addObjectReference(req, nil) - if _, ok := res.(*objects.ObjectsClassReferencesCreateNotFound); !ok { - t.Errorf("expected: %T got: %T", objects.ObjectsClassReferencesCreateNotFound{}, res) - } - - m.addRefErr = &uco.Error{Code: uco.StatusInternalServerError} - res = h.addObjectReference(req, nil) - if _, ok := res.(*objects.ObjectsClassReferencesCreateInternalServerError); !ok { - t.Errorf("expected: %T got: %T", objects.ObjectsClassReferencesCreateInternalServerError{}, res) - } - m.addRefErr = &uco.Error{Code: uco.StatusBadRequest} - res = h.addObjectReference(req, nil) - if _, ok := res.(*objects.ObjectsClassReferencesCreateUnprocessableEntity); !ok { - t.Errorf("expected: %T got: %T", objects.ObjectsClassReferencesCreateUnprocessableEntity{}, res) - } - // same test as before but using old request - oldRequest := objects.ObjectsReferencesCreateParams{ - HTTPRequest: req.HTTPRequest, - Body: req.Body, - ID: req.ID, - PropertyName: req.ClassName, - } - res = h.addObjectReferenceDeprecated(oldRequest, nil) - if _, ok := res.(*objects.ObjectsClassReferencesCreateUnprocessableEntity); !ok { - t.Errorf("expected: %T got: %T", objects.ObjectsClassReferencesCreateUnprocessableEntity{}, res) - } - }) - - t.Run("PutReferences", func(t *testing.T) { - m := &fakeManager{} - h := &objectHandlers{manager: m, logger: &logrus.Logger{}, metricRequestsTotal: &fakeMetricRequestsTotal{}} - req := objects.ObjectsClassReferencesPutParams{ - HTTPRequest: httptest.NewRequest("HEAD", "/v1/objects/MyClass/123/references/prop", nil), - ClassName: "MyClass", - ID: "123", - Body: models.MultipleRef{}, - PropertyName: "prop", - } - res := h.putObjectReferences(req, nil) - if _, ok := res.(*objects.ObjectsClassReferencesPutOK); !ok { - t.Errorf("unexpected result %v", res) - } - - m.putRefErr = &uco.Error{Code: uco.StatusForbidden} - res = h.putObjectReferences(req, nil) - if _, ok := res.(*objects.ObjectsClassReferencesPutForbidden); !ok { - t.Errorf("expected: %T got: %T", objects.ObjectsClassReferencesPutForbidden{}, res) - } - m.putRefErr = &uco.Error{Code: uco.StatusInternalServerError} - res = h.putObjectReferences(req, nil) - if _, ok := res.(*objects.ObjectsClassReferencesPutInternalServerError); !ok { - t.Errorf("expected: %T got: %T", objects.ObjectsClassReferencesPutInternalServerError{}, res) - } - m.putRefErr = &uco.Error{Code: uco.StatusBadRequest} - res = h.putObjectReferences(req, nil) - if _, ok := res.(*objects.ObjectsClassReferencesPutUnprocessableEntity); !ok { - t.Errorf("expected: %T got: %T", objects.ObjectsClassReferencesPutUnprocessableEntity{}, res) - } - // same test as before but using old request - oldRequest := objects.ObjectsReferencesUpdateParams{ - HTTPRequest: req.HTTPRequest, - Body: req.Body, - ID: req.ID, - PropertyName: req.ClassName, - } - res = h.updateObjectReferencesDeprecated(oldRequest, nil) - if _, ok := res.(*objects.ObjectsClassReferencesPutUnprocessableEntity); !ok { - t.Errorf("expected: %T got: %T", objects.ObjectsClassReferencesPutUnprocessableEntity{}, res) - } - }) - - t.Run("DeleteReference", func(t *testing.T) { - m := &fakeManager{} - h := &objectHandlers{manager: m, logger: &logrus.Logger{}, metricRequestsTotal: &fakeMetricRequestsTotal{}} - req := objects.ObjectsClassReferencesDeleteParams{ - HTTPRequest: httptest.NewRequest("HEAD", "/v1/objects/MyClass/123/references/prop", nil), - ClassName: "MyClass", - ID: "123", - Body: new(models.SingleRef), - PropertyName: "prop", - } - res := h.deleteObjectReference(req, nil) - if _, ok := res.(*objects.ObjectsClassReferencesDeleteNoContent); !ok { - t.Errorf("unexpected result %v", res) - } - - m.deleteRefErr = &uco.Error{Code: uco.StatusForbidden} - res = h.deleteObjectReference(req, nil) - if _, ok := res.(*objects.ObjectsClassReferencesDeleteForbidden); !ok { - t.Errorf("expected: %T got: %T", objects.ObjectsClassReferencesDeleteForbidden{}, res) - } - // source object not found - m.deleteRefErr = &uco.Error{Code: uco.StatusNotFound} - res = h.deleteObjectReference(req, nil) - if _, ok := res.(*objects.ObjectsClassReferencesDeleteNotFound); !ok { - t.Errorf("expected: %T got: %T", objects.ObjectsClassReferencesDeleteNotFound{}, res) - } - - m.deleteRefErr = &uco.Error{Code: uco.StatusInternalServerError} - res = h.deleteObjectReference(req, nil) - if _, ok := res.(*objects.ObjectsClassReferencesDeleteInternalServerError); !ok { - t.Errorf("expected: %T got: %T", objects.ObjectsClassReferencesDeleteInternalServerError{}, res) - } - m.deleteRefErr = &uco.Error{Code: uco.StatusBadRequest} - res = h.deleteObjectReference(req, nil) - if _, ok := res.(*objects.ObjectsClassReferencesDeleteUnprocessableEntity); !ok { - t.Errorf("expected: %T got: %T", objects.ObjectsClassReferencesDeleteUnprocessableEntity{}, res) - } - // same test as before but using old request - oldRequest := objects.ObjectsReferencesDeleteParams{ - HTTPRequest: req.HTTPRequest, - Body: req.Body, - ID: req.ID, - PropertyName: req.ClassName, - } - res = h.deleteObjectReferenceDeprecated(oldRequest, nil) - if _, ok := res.(*objects.ObjectsClassReferencesDeleteUnprocessableEntity); !ok { - t.Errorf("expected: %T got: %T", objects.ObjectsClassReferencesDeleteUnprocessableEntity{}, res) - } - }) - - t.Run("Query", func(t *testing.T) { - var ( - cls = "MyClass" - m = &fakeManager{ - queryErr: nil, - queryResult: []*models.Object{{ - Properties: map[string]interface{}{"name": "John"}, - }}, - } - fakeMetricRequestsTotal = &fakeMetricRequestsTotal{} - h = &objectHandlers{ - manager: m, - logger: &logrus.Logger{}, - metricRequestsTotal: fakeMetricRequestsTotal, - } - req = objects.ObjectsListParams{ - HTTPRequest: httptest.NewRequest("HEAD", "/v1/objects/", nil), - Class: &cls, - } - ) - - res := h.query(req, nil) - if _, ok := res.(*objects.ObjectsListOK); !ok { - t.Errorf("unexpected result %v", res) - } - - m.queryErr = &uco.Error{Code: uco.StatusForbidden} - res = h.query(req, nil) - if _, ok := res.(*objects.ObjectsListForbidden); !ok { - t.Errorf("expected: %T got: %T", objects.ObjectsListForbidden{}, res) - } - m.queryErr = &uco.Error{Code: uco.StatusNotFound} - res = h.query(req, nil) - if _, ok := res.(*objects.ObjectsListNotFound); !ok { - t.Errorf("expected: %T got: %T", objects.ObjectsListNotFound{}, res) - } - m.queryErr = &uco.Error{Code: uco.StatusBadRequest} - res = h.query(req, nil) - if _, ok := res.(*objects.ObjectsListUnprocessableEntity); !ok { - t.Errorf("expected: %T got: %T", objects.ObjectsListUnprocessableEntity{}, res) - } - m.queryErr = &uco.Error{Code: uco.StatusInternalServerError} - res = h.query(req, nil) - if _, ok := res.(*objects.ObjectsListInternalServerError); !ok { - t.Errorf("expected: %T got: %T", objects.ObjectsListInternalServerError{}, res) - } - }) -} - -type fakeManager struct { - getObjectReturn *models.Object - getObjectErr error - - addObjectReturn *models.Object - queryResult []*models.Object - queryErr *uco.Error - updateObjectReturn *models.Object - updateObjectErr error - deleteObjectReturn error - patchObjectReturn *uco.Error - headObjectReturn bool - headObjectErr *uco.Error - addRefErr *uco.Error - putRefErr *uco.Error - deleteRefErr *uco.Error -} - -func (f *fakeManager) HeadObject(context.Context, *models.Principal, - string, strfmt.UUID, *additional.ReplicationProperties, string, -) (bool, *uco.Error) { - return f.headObjectReturn, f.headObjectErr -} - -func (f *fakeManager) AddObject(_ context.Context, _ *models.Principal, - object *models.Object, _ *additional.ReplicationProperties, -) (*models.Object, error) { - return object, nil -} - -func (f *fakeManager) ValidateObject(_ context.Context, _ *models.Principal, - _ *models.Object, _ *additional.ReplicationProperties, -) error { - panic("not implemented") // TODO: Implement -} - -func (f *fakeManager) GetObject(_ context.Context, _ *models.Principal, class string, - _ strfmt.UUID, _ additional.Properties, _ *additional.ReplicationProperties, _ string, -) (*models.Object, error) { - return f.getObjectReturn, f.getObjectErr -} - -func (f *fakeManager) GetObjectsClass(ctx context.Context, - principal *models.Principal, id strfmt.UUID, -) (*models.Class, error) { - class := &models.Class{ - Class: f.getObjectReturn.Class, - Vectorizer: "text2vec-contextionary", - } - return class, nil -} - -func (f *fakeManager) GetObjectClassFromName(ctx context.Context, principal *models.Principal, - className string, -) (*models.Class, error) { - class := &models.Class{ - Class: f.getObjectReturn.Class, - Vectorizer: "text2vec-contextionary", - } - return class, nil -} - -func (f *fakeManager) GetObjects(ctx context.Context, principal *models.Principal, offset *int64, limit *int64, sort *string, order *string, after *string, addl additional.Properties, tenant string) ([]*models.Object, error) { - return f.queryResult, nil -} - -func (f *fakeManager) Query(_ context.Context, - _ *models.Principal, _ *uco.QueryParams, -) ([]*models.Object, *uco.Error) { - return f.queryResult, f.queryErr -} - -func (f *fakeManager) UpdateObject(_ context.Context, _ *models.Principal, _ string, - _ strfmt.UUID, updates *models.Object, _ *additional.ReplicationProperties, -) (*models.Object, error) { - return updates, f.updateObjectErr -} - -func (f *fakeManager) MergeObject(_ context.Context, _ *models.Principal, - _ *models.Object, _ *additional.ReplicationProperties, -) *uco.Error { - return f.patchObjectReturn -} - -func (f *fakeManager) DeleteObject(_ context.Context, _ *models.Principal, - class string, _ strfmt.UUID, _ *additional.ReplicationProperties, _ string, -) error { - return f.deleteObjectReturn -} - -func (f *fakeManager) AddObjectReference(context.Context, *models.Principal, - *uco.AddReferenceInput, *additional.ReplicationProperties, string, -) *uco.Error { - return f.addRefErr -} - -func (f *fakeManager) UpdateObjectReferences(context.Context, *models.Principal, - *uco.PutReferenceInput, *additional.ReplicationProperties, string, -) *uco.Error { - return f.putRefErr -} - -func (f *fakeManager) DeleteObjectReference(context.Context, *models.Principal, - *uco.DeleteReferenceInput, *additional.ReplicationProperties, string, -) *uco.Error { - return f.deleteRefErr -} - -type fakeMetricRequestsTotal struct{} - -func (f *fakeMetricRequestsTotal) logError(className string, err error) {} -func (f *fakeMetricRequestsTotal) logOk(className string) {} -func (f *fakeMetricRequestsTotal) logUserError(className string) {} -func (f *fakeMetricRequestsTotal) logServerError(className string, err error) {} diff --git a/adapters/handlers/rest/handlers_schema.go b/adapters/handlers/rest/handlers_schema.go deleted file mode 100644 index 969ad81856fe375172de0acd28623d775c5c530e..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/handlers_schema.go +++ /dev/null @@ -1,354 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package rest - -import ( - "github.com/go-openapi/runtime/middleware" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/adapters/handlers/rest/operations" - "github.com/weaviate/weaviate/adapters/handlers/rest/operations/schema" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/usecases/auth/authorization/errors" - "github.com/weaviate/weaviate/usecases/monitoring" - uco "github.com/weaviate/weaviate/usecases/objects" - schemaUC "github.com/weaviate/weaviate/usecases/schema" -) - -type schemaHandlers struct { - manager *schemaUC.Manager - metricRequestsTotal restApiRequestsTotal -} - -func (s *schemaHandlers) addClass(params schema.SchemaObjectsCreateParams, - principal *models.Principal, -) middleware.Responder { - err := s.manager.AddClass(params.HTTPRequest.Context(), principal, params.ObjectClass) - if err != nil { - s.metricRequestsTotal.logError(params.ObjectClass.Class, err) - switch err.(type) { - case errors.Forbidden: - return schema.NewSchemaObjectsCreateForbidden(). - WithPayload(errPayloadFromSingleErr(err)) - default: - return schema.NewSchemaObjectsCreateUnprocessableEntity(). - WithPayload(errPayloadFromSingleErr(err)) - } - } - - s.metricRequestsTotal.logOk(params.ObjectClass.Class) - return schema.NewSchemaObjectsCreateOK().WithPayload(params.ObjectClass) -} - -func (s *schemaHandlers) updateClass(params schema.SchemaObjectsUpdateParams, - principal *models.Principal, -) middleware.Responder { - err := s.manager.UpdateClass(params.HTTPRequest.Context(), principal, params.ClassName, - params.ObjectClass) - if err != nil { - s.metricRequestsTotal.logError(params.ClassName, err) - if err == schemaUC.ErrNotFound { - return schema.NewSchemaObjectsUpdateNotFound() - } - - switch err.(type) { - case errors.Forbidden: - return schema.NewSchemaObjectsUpdateForbidden(). - WithPayload(errPayloadFromSingleErr(err)) - default: - return schema.NewSchemaObjectsUpdateUnprocessableEntity(). - WithPayload(errPayloadFromSingleErr(err)) - } - } - - s.metricRequestsTotal.logOk(params.ClassName) - return schema.NewSchemaObjectsUpdateOK().WithPayload(params.ObjectClass) -} - -func (s *schemaHandlers) getClass(params schema.SchemaObjectsGetParams, - principal *models.Principal, -) middleware.Responder { - class, err := s.manager.GetClass(params.HTTPRequest.Context(), principal, params.ClassName) - if err != nil { - s.metricRequestsTotal.logError(params.ClassName, err) - switch err.(type) { - case errors.Forbidden: - return schema.NewSchemaObjectsGetForbidden(). - WithPayload(errPayloadFromSingleErr(err)) - default: - return schema.NewSchemaObjectsGetInternalServerError(). - WithPayload(errPayloadFromSingleErr(err)) - } - } - - if class == nil { - s.metricRequestsTotal.logUserError(params.ClassName) - return schema.NewSchemaObjectsGetNotFound() - } - - s.metricRequestsTotal.logOk(params.ClassName) - return schema.NewSchemaObjectsGetOK().WithPayload(class) -} - -func (s *schemaHandlers) deleteClass(params schema.SchemaObjectsDeleteParams, principal *models.Principal) middleware.Responder { - err := s.manager.DeleteClass(params.HTTPRequest.Context(), principal, params.ClassName) - if err != nil { - s.metricRequestsTotal.logError(params.ClassName, err) - switch err.(type) { - case errors.Forbidden: - return schema.NewSchemaObjectsDeleteForbidden(). - WithPayload(errPayloadFromSingleErr(err)) - default: - return schema.NewSchemaObjectsDeleteBadRequest().WithPayload(errPayloadFromSingleErr(err)) - } - } - - s.metricRequestsTotal.logOk(params.ClassName) - return schema.NewSchemaObjectsDeleteOK() -} - -func (s *schemaHandlers) addClassProperty(params schema.SchemaObjectsPropertiesAddParams, - principal *models.Principal, -) middleware.Responder { - err := s.manager.AddClassProperty(params.HTTPRequest.Context(), principal, params.ClassName, params.Body) - if err != nil { - s.metricRequestsTotal.logError(params.ClassName, err) - switch err.(type) { - case errors.Forbidden: - return schema.NewSchemaObjectsPropertiesAddForbidden(). - WithPayload(errPayloadFromSingleErr(err)) - default: - return schema.NewSchemaObjectsPropertiesAddUnprocessableEntity(). - WithPayload(errPayloadFromSingleErr(err)) - } - } - - s.metricRequestsTotal.logOk(params.ClassName) - return schema.NewSchemaObjectsPropertiesAddOK().WithPayload(params.Body) -} - -func (s *schemaHandlers) getSchema(params schema.SchemaDumpParams, principal *models.Principal) middleware.Responder { - dbSchema, err := s.manager.GetSchema(principal) - if err != nil { - s.metricRequestsTotal.logError("", err) - switch err.(type) { - case errors.Forbidden: - return schema.NewSchemaDumpForbidden(). - WithPayload(errPayloadFromSingleErr(err)) - default: - return schema.NewSchemaDumpForbidden().WithPayload(errPayloadFromSingleErr(err)) - } - } - - payload := dbSchema.Objects - - s.metricRequestsTotal.logOk("") - return schema.NewSchemaDumpOK().WithPayload(payload) -} - -func (s *schemaHandlers) getClusterStatus(params schema.SchemaClusterStatusParams, principal *models.Principal) middleware.Responder { - status, err := s.manager.ClusterStatus(params.HTTPRequest.Context()) - if err == nil { - s.metricRequestsTotal.logOk("") - return schema.NewSchemaClusterStatusOK().WithPayload(status) - } else { - s.metricRequestsTotal.logServerError("", err) - return schema.NewSchemaClusterStatusInternalServerError().WithPayload(status) - } -} - -func (s *schemaHandlers) getShardsStatus(params schema.SchemaObjectsShardsGetParams, - principal *models.Principal, -) middleware.Responder { - var tenant string - if params.Tenant == nil { - tenant = "" - } else { - tenant = *params.Tenant - } - status, err := s.manager.GetShardsStatus(params.HTTPRequest.Context(), principal, params.ClassName, tenant) - if err != nil { - s.metricRequestsTotal.logError("", err) - switch err.(type) { - case errors.Forbidden: - return schema.NewSchemaObjectsShardsGetForbidden(). - WithPayload(errPayloadFromSingleErr(err)) - default: - return schema.NewSchemaObjectsShardsGetNotFound(). - WithPayload(errPayloadFromSingleErr(err)) - } - } - - payload := status - - s.metricRequestsTotal.logOk("") - return schema.NewSchemaObjectsShardsGetOK().WithPayload(payload) -} - -func (s *schemaHandlers) updateShardStatus(params schema.SchemaObjectsShardsUpdateParams, - principal *models.Principal, -) middleware.Responder { - err := s.manager.UpdateShardStatus( - params.HTTPRequest.Context(), principal, params.ClassName, params.ShardName, params.Body.Status) - if err != nil { - s.metricRequestsTotal.logError("", err) - switch err.(type) { - case errors.Forbidden: - return schema.NewSchemaObjectsShardsGetForbidden(). - WithPayload(errPayloadFromSingleErr(err)) - default: - return schema.NewSchemaObjectsShardsUpdateUnprocessableEntity(). - WithPayload(errPayloadFromSingleErr(err)) - } - } - - payload := params.Body - - s.metricRequestsTotal.logOk("") - return schema.NewSchemaObjectsShardsUpdateOK().WithPayload(payload) -} - -func (s *schemaHandlers) createTenants(params schema.TenantsCreateParams, - principal *models.Principal, -) middleware.Responder { - created, err := s.manager.AddTenants( - params.HTTPRequest.Context(), principal, params.ClassName, params.Body) - if err != nil { - s.metricRequestsTotal.logError(params.ClassName, err) - switch err.(type) { - case errors.Forbidden: - return schema.NewTenantsCreateForbidden(). - WithPayload(errPayloadFromSingleErr(err)) - default: - return schema.NewTenantsCreateUnprocessableEntity(). - WithPayload(errPayloadFromSingleErr(err)) - } - } - - s.metricRequestsTotal.logOk(params.ClassName) - return schema.NewTenantsCreateOK().WithPayload(created) -} - -func (s *schemaHandlers) updateTenants(params schema.TenantsUpdateParams, - principal *models.Principal, -) middleware.Responder { - err := s.manager.UpdateTenants( - params.HTTPRequest.Context(), principal, params.ClassName, params.Body) - if err != nil { - s.metricRequestsTotal.logError(params.ClassName, err) - switch err.(type) { - case errors.Forbidden: - return schema.NewTenantsUpdateForbidden(). - WithPayload(errPayloadFromSingleErr(err)) - default: - return schema.NewTenantsUpdateUnprocessableEntity(). - WithPayload(errPayloadFromSingleErr(err)) - } - } - - payload := params.Body - - s.metricRequestsTotal.logOk(params.ClassName) - return schema.NewTenantsUpdateOK().WithPayload(payload) -} - -func (s *schemaHandlers) deleteTenants(params schema.TenantsDeleteParams, - principal *models.Principal, -) middleware.Responder { - err := s.manager.DeleteTenants( - params.HTTPRequest.Context(), principal, params.ClassName, params.Tenants) - if err != nil { - s.metricRequestsTotal.logError(params.ClassName, err) - switch err.(type) { - case errors.Forbidden: - return schema.NewTenantsDeleteForbidden(). - WithPayload(errPayloadFromSingleErr(err)) - default: - return schema.NewTenantsDeleteUnprocessableEntity(). - WithPayload(errPayloadFromSingleErr(err)) - } - } - - s.metricRequestsTotal.logOk(params.ClassName) - return schema.NewTenantsDeleteOK() -} - -func (s *schemaHandlers) getTenants(params schema.TenantsGetParams, - principal *models.Principal, -) middleware.Responder { - tenants, err := s.manager.GetTenants(params.HTTPRequest.Context(), principal, params.ClassName) - if err != nil { - s.metricRequestsTotal.logError(params.ClassName, err) - switch err.(type) { - case errors.Forbidden: - return schema.NewTenantsGetForbidden(). - WithPayload(errPayloadFromSingleErr(err)) - default: - return schema.NewTenantsGetUnprocessableEntity(). - WithPayload(errPayloadFromSingleErr(err)) - } - } - - s.metricRequestsTotal.logOk(params.ClassName) - return schema.NewTenantsGetOK().WithPayload(tenants) -} - -func setupSchemaHandlers(api *operations.WeaviateAPI, manager *schemaUC.Manager, metrics *monitoring.PrometheusMetrics, logger logrus.FieldLogger) { - h := &schemaHandlers{manager, newSchemaRequestsTotal(metrics, logger)} - - api.SchemaSchemaObjectsCreateHandler = schema. - SchemaObjectsCreateHandlerFunc(h.addClass) - api.SchemaSchemaObjectsDeleteHandler = schema. - SchemaObjectsDeleteHandlerFunc(h.deleteClass) - api.SchemaSchemaObjectsPropertiesAddHandler = schema. - SchemaObjectsPropertiesAddHandlerFunc(h.addClassProperty) - - api.SchemaSchemaObjectsUpdateHandler = schema. - SchemaObjectsUpdateHandlerFunc(h.updateClass) - - api.SchemaSchemaObjectsGetHandler = schema. - SchemaObjectsGetHandlerFunc(h.getClass) - api.SchemaSchemaDumpHandler = schema. - SchemaDumpHandlerFunc(h.getSchema) - api.SchemaSchemaClusterStatusHandler = schema. - SchemaClusterStatusHandlerFunc(h.getClusterStatus) - - api.SchemaSchemaObjectsShardsGetHandler = schema. - SchemaObjectsShardsGetHandlerFunc(h.getShardsStatus) - api.SchemaSchemaObjectsShardsUpdateHandler = schema. - SchemaObjectsShardsUpdateHandlerFunc(h.updateShardStatus) - - api.SchemaTenantsCreateHandler = schema.TenantsCreateHandlerFunc(h.createTenants) - api.SchemaTenantsUpdateHandler = schema.TenantsUpdateHandlerFunc(h.updateTenants) - api.SchemaTenantsDeleteHandler = schema.TenantsDeleteHandlerFunc(h.deleteTenants) - api.SchemaTenantsGetHandler = schema.TenantsGetHandlerFunc(h.getTenants) -} - -type schemaRequestsTotal struct { - *restApiRequestsTotalImpl -} - -func newSchemaRequestsTotal(metrics *monitoring.PrometheusMetrics, logger logrus.FieldLogger) restApiRequestsTotal { - return &schemaRequestsTotal{ - restApiRequestsTotalImpl: &restApiRequestsTotalImpl{newRequestsTotalMetric(metrics, "rest"), "rest", "schema", logger}, - } -} - -func (e *schemaRequestsTotal) logError(className string, err error) { - switch err.(type) { - case uco.ErrMultiTenancy: - e.logUserError(className) - case errors.Forbidden: - e.logUserError(className) - default: - e.logUserError(className) - } -} diff --git a/adapters/handlers/rest/helpers.go b/adapters/handlers/rest/helpers.go deleted file mode 100644 index 67ceeefb21b1f0c49f1ea58e931c1526ba68d7af..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/helpers.go +++ /dev/null @@ -1,39 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package rest - -import ( - "fmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// createErrorResponseObject is a common function to create an error response -func createErrorResponseObject(messages ...string) *models.ErrorResponse { - // Initialize return value - er := &models.ErrorResponse{} - - // appends all error messages to the error - for _, message := range messages { - er.Error = append(er.Error, &models.ErrorResponseErrorItems0{ - Message: message, - }) - } - - return er -} - -func errPayloadFromSingleErr(err error) *models.ErrorResponse { - return &models.ErrorResponse{Error: []*models.ErrorResponseErrorItems0{{ - Message: fmt.Sprintf("%s", err), - }}} -} diff --git a/adapters/handlers/rest/middlewares.go b/adapters/handlers/rest/middlewares.go deleted file mode 100644 index 19b203341f24699242ec2c53d7deb93633958938..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/middlewares.go +++ /dev/null @@ -1,197 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package rest - -import ( - "context" - "fmt" - "net/http" - "strings" - "time" - - "github.com/prometheus/client_golang/prometheus" - "github.com/rs/cors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/adapters/handlers/rest/state" - "github.com/weaviate/weaviate/adapters/handlers/rest/swagger_middleware" - "github.com/weaviate/weaviate/usecases/config" - "github.com/weaviate/weaviate/usecases/modules" - "github.com/weaviate/weaviate/usecases/monitoring" -) - -// The middleware configuration is for the handler executors. These do not apply to the swagger.json document. -// The middleware executes after routing but before authentication, binding and validation -// -// we are setting the middlewares from within configureAPI, as we need access -// to some resources which are not exposed -func makeSetupMiddlewares(appState *state.State) func(http.Handler) http.Handler { - return func(handler http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.String() == "/v1/.well-known/openid-configuration" || r.URL.String() == "/v1" { - handler.ServeHTTP(w, r) - return - } - appState.AnonymousAccess.Middleware(handler).ServeHTTP(w, r) - }) - } -} - -func addHandleRoot(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.String() == "/" { - w.Header().Add("Location", "/v1") - w.WriteHeader(http.StatusMovedPermanently) - w.Write([]byte(`{"links":{"href":"/v1","name":"api v1","documentationHref":` + - `"https://weaviate.io/developers/weaviate/current/"}}`)) - return - } - - next.ServeHTTP(w, r) - }) -} - -func makeAddModuleHandlers(modules *modules.Provider) func(http.Handler) http.Handler { - return func(next http.Handler) http.Handler { - mux := http.NewServeMux() - - for _, mod := range modules.GetAll() { - prefix := fmt.Sprintf("/v1/modules/%s", mod.Name()) - mux.Handle(fmt.Sprintf("%s/", prefix), - http.StripPrefix(prefix, mod.RootHandler())) - } - - prefix := "/v1/modules" - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if url := r.URL.String(); len(url) > len(prefix) && url[:len(prefix)] == prefix { - mux.ServeHTTP(w, r) - return - } - - next.ServeHTTP(w, r) - }) - } -} - -// The middleware configuration happens before anything, this middleware also applies to serving the swagger.json document. -// So this is a good place to plug in a panic handling middleware, logging and metrics -// Contains "x-api-key", "x-api-token" for legacy reasons, older interfaces might need these headers. -func makeSetupGlobalMiddleware(appState *state.State) func(http.Handler) http.Handler { - return func(handler http.Handler) http.Handler { - handleCORS := cors.New(cors.Options{ - OptionsPassthrough: true, - AllowedMethods: strings.Split(appState.ServerConfig.Config.CORS.AllowMethods, ","), - AllowedHeaders: strings.Split(appState.ServerConfig.Config.CORS.AllowHeaders, ","), - AllowedOrigins: strings.Split(appState.ServerConfig.Config.CORS.AllowOrigin, ","), - }).Handler - handler = handleCORS(handler) - handler = swagger_middleware.AddMiddleware([]byte(SwaggerJSON), handler) - handler = makeAddLogging(appState.Logger)(handler) - if appState.ServerConfig.Config.Monitoring.Enabled { - handler = makeAddMonitoring(appState.Metrics)(handler) - } - handler = addPreflight(handler, appState.ServerConfig.Config.CORS) - handler = addLiveAndReadyness(appState, handler) - handler = addHandleRoot(handler) - handler = makeAddModuleHandlers(appState.Modules)(handler) - handler = addInjectHeadersIntoContext(handler) - handler = makeCatchPanics(appState.Logger, - newPanicsRequestsTotal(appState.Metrics, appState.Logger))(handler) - - return handler - } -} - -func makeAddLogging(logger logrus.FieldLogger) func(http.Handler) http.Handler { - return func(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - logger. - WithField("action", "restapi_request"). - WithField("method", r.Method). - WithField("url", r.URL). - Debug("received HTTP request") - next.ServeHTTP(w, r) - }) - } -} - -func makeAddMonitoring(metrics *monitoring.PrometheusMetrics) func(http.Handler) http.Handler { - return func(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - before := time.Now() - method := r.Method - path := r.URL.Path - next.ServeHTTP(w, r) - - if strings.HasPrefix(path, "/v1/batch/objects") && method == http.MethodPost { - metrics.BatchTime.With(prometheus.Labels{ - "operation": "total_api_level", - "class_name": "n/a", - "shard_name": "n/a", - }). - Observe(float64(time.Since(before) / time.Millisecond)) - } - }) - } -} - -func addPreflight(next http.Handler, cfg config.CORS) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Access-Control-Allow-Origin", cfg.AllowOrigin) - w.Header().Set("Access-Control-Allow-Methods", cfg.AllowMethods) - w.Header().Set("Access-Control-Allow-Headers", cfg.AllowHeaders) - - if r.Method == "OPTIONS" { - return - } - - next.ServeHTTP(w, r) - }) -} - -func addInjectHeadersIntoContext(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - changed := false - for k, v := range r.Header { - if strings.HasPrefix(k, "X-") { - ctx = context.WithValue(ctx, k, v) - changed = true - } - } - - if changed { - next.ServeHTTP(w, r.Clone(ctx)) - } else { - next.ServeHTTP(w, r) - } - }) -} - -func addLiveAndReadyness(state *state.State, next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.String() == "/v1/.well-known/live" { - w.WriteHeader(http.StatusOK) - return - } - - if r.URL.String() == "/v1/.well-known/ready" { - code := http.StatusServiceUnavailable - if state.DB.StartupComplete() && state.Cluster.ClusterHealthScore() == 0 { - code = http.StatusOK - } - w.WriteHeader(code) - return - } - - next.ServeHTTP(w, r) - }) -} diff --git a/adapters/handlers/rest/operations/backups/backups_create.go b/adapters/handlers/rest/operations/backups/backups_create.go deleted file mode 100644 index b833adf3d8cd42a0c44f4b32ece0e17aef86b74e..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/backups/backups_create.go +++ /dev/null @@ -1,82 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package backups - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// BackupsCreateHandlerFunc turns a function with the right signature into a backups create handler -type BackupsCreateHandlerFunc func(BackupsCreateParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn BackupsCreateHandlerFunc) Handle(params BackupsCreateParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// BackupsCreateHandler interface for that can handle valid backups create params -type BackupsCreateHandler interface { - Handle(BackupsCreateParams, *models.Principal) middleware.Responder -} - -// NewBackupsCreate creates a new http.Handler for the backups create operation -func NewBackupsCreate(ctx *middleware.Context, handler BackupsCreateHandler) *BackupsCreate { - return &BackupsCreate{Context: ctx, Handler: handler} -} - -/* - BackupsCreate swagger:route POST /backups/{backend} backups backupsCreate - -Starts a process of creating a backup for a set of classes -*/ -type BackupsCreate struct { - Context *middleware.Context - Handler BackupsCreateHandler -} - -func (o *BackupsCreate) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewBackupsCreateParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/backups/backups_create_parameters.go b/adapters/handlers/rest/operations/backups/backups_create_parameters.go deleted file mode 100644 index 862611ae88958a03983517f66f0fbe97e8bc5025..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/backups/backups_create_parameters.go +++ /dev/null @@ -1,120 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package backups - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "io" - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/validate" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewBackupsCreateParams creates a new BackupsCreateParams object -// -// There are no default values defined in the spec. -func NewBackupsCreateParams() BackupsCreateParams { - - return BackupsCreateParams{} -} - -// BackupsCreateParams contains all the bound params for the backups create operation -// typically these are obtained from a http.Request -// -// swagger:parameters backups.create -type BackupsCreateParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` - - /*Backup backend name e.g. filesystem, gcs, s3. - Required: true - In: path - */ - Backend string - /* - Required: true - In: body - */ - Body *models.BackupCreateRequest -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewBackupsCreateParams() beforehand. -func (o *BackupsCreateParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - rBackend, rhkBackend, _ := route.Params.GetOK("backend") - if err := o.bindBackend(rBackend, rhkBackend, route.Formats); err != nil { - res = append(res, err) - } - - if runtime.HasBody(r) { - defer r.Body.Close() - var body models.BackupCreateRequest - if err := route.Consumer.Consume(r.Body, &body); err != nil { - if err == io.EOF { - res = append(res, errors.Required("body", "body", "")) - } else { - res = append(res, errors.NewParseError("body", "body", "", err)) - } - } else { - // validate body object - if err := body.Validate(route.Formats); err != nil { - res = append(res, err) - } - - ctx := validate.WithOperationRequest(r.Context()) - if err := body.ContextValidate(ctx, route.Formats); err != nil { - res = append(res, err) - } - - if len(res) == 0 { - o.Body = &body - } - } - } else { - res = append(res, errors.Required("body", "body", "")) - } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// bindBackend binds and validates parameter Backend from path. -func (o *BackupsCreateParams) bindBackend(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - o.Backend = raw - - return nil -} diff --git a/adapters/handlers/rest/operations/backups/backups_create_responses.go b/adapters/handlers/rest/operations/backups/backups_create_responses.go deleted file mode 100644 index c7d487c746d2a969fe83f661565c0ead06fa95ec..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/backups/backups_create_responses.go +++ /dev/null @@ -1,230 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package backups - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// BackupsCreateOKCode is the HTTP code returned for type BackupsCreateOK -const BackupsCreateOKCode int = 200 - -/* -BackupsCreateOK Backup create process successfully started. - -swagger:response backupsCreateOK -*/ -type BackupsCreateOK struct { - - /* - In: Body - */ - Payload *models.BackupCreateResponse `json:"body,omitempty"` -} - -// NewBackupsCreateOK creates BackupsCreateOK with default headers values -func NewBackupsCreateOK() *BackupsCreateOK { - - return &BackupsCreateOK{} -} - -// WithPayload adds the payload to the backups create o k response -func (o *BackupsCreateOK) WithPayload(payload *models.BackupCreateResponse) *BackupsCreateOK { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the backups create o k response -func (o *BackupsCreateOK) SetPayload(payload *models.BackupCreateResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *BackupsCreateOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(200) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// BackupsCreateUnauthorizedCode is the HTTP code returned for type BackupsCreateUnauthorized -const BackupsCreateUnauthorizedCode int = 401 - -/* -BackupsCreateUnauthorized Unauthorized or invalid credentials. - -swagger:response backupsCreateUnauthorized -*/ -type BackupsCreateUnauthorized struct { -} - -// NewBackupsCreateUnauthorized creates BackupsCreateUnauthorized with default headers values -func NewBackupsCreateUnauthorized() *BackupsCreateUnauthorized { - - return &BackupsCreateUnauthorized{} -} - -// WriteResponse to the client -func (o *BackupsCreateUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// BackupsCreateForbiddenCode is the HTTP code returned for type BackupsCreateForbidden -const BackupsCreateForbiddenCode int = 403 - -/* -BackupsCreateForbidden Forbidden - -swagger:response backupsCreateForbidden -*/ -type BackupsCreateForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewBackupsCreateForbidden creates BackupsCreateForbidden with default headers values -func NewBackupsCreateForbidden() *BackupsCreateForbidden { - - return &BackupsCreateForbidden{} -} - -// WithPayload adds the payload to the backups create forbidden response -func (o *BackupsCreateForbidden) WithPayload(payload *models.ErrorResponse) *BackupsCreateForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the backups create forbidden response -func (o *BackupsCreateForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *BackupsCreateForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// BackupsCreateUnprocessableEntityCode is the HTTP code returned for type BackupsCreateUnprocessableEntity -const BackupsCreateUnprocessableEntityCode int = 422 - -/* -BackupsCreateUnprocessableEntity Invalid backup creation attempt. - -swagger:response backupsCreateUnprocessableEntity -*/ -type BackupsCreateUnprocessableEntity struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewBackupsCreateUnprocessableEntity creates BackupsCreateUnprocessableEntity with default headers values -func NewBackupsCreateUnprocessableEntity() *BackupsCreateUnprocessableEntity { - - return &BackupsCreateUnprocessableEntity{} -} - -// WithPayload adds the payload to the backups create unprocessable entity response -func (o *BackupsCreateUnprocessableEntity) WithPayload(payload *models.ErrorResponse) *BackupsCreateUnprocessableEntity { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the backups create unprocessable entity response -func (o *BackupsCreateUnprocessableEntity) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *BackupsCreateUnprocessableEntity) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(422) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// BackupsCreateInternalServerErrorCode is the HTTP code returned for type BackupsCreateInternalServerError -const BackupsCreateInternalServerErrorCode int = 500 - -/* -BackupsCreateInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response backupsCreateInternalServerError -*/ -type BackupsCreateInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewBackupsCreateInternalServerError creates BackupsCreateInternalServerError with default headers values -func NewBackupsCreateInternalServerError() *BackupsCreateInternalServerError { - - return &BackupsCreateInternalServerError{} -} - -// WithPayload adds the payload to the backups create internal server error response -func (o *BackupsCreateInternalServerError) WithPayload(payload *models.ErrorResponse) *BackupsCreateInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the backups create internal server error response -func (o *BackupsCreateInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *BackupsCreateInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/backups/backups_create_status.go b/adapters/handlers/rest/operations/backups/backups_create_status.go deleted file mode 100644 index 289f2cf01a2602242004eb3ee53081ef6a5ceb8e..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/backups/backups_create_status.go +++ /dev/null @@ -1,82 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package backups - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// BackupsCreateStatusHandlerFunc turns a function with the right signature into a backups create status handler -type BackupsCreateStatusHandlerFunc func(BackupsCreateStatusParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn BackupsCreateStatusHandlerFunc) Handle(params BackupsCreateStatusParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// BackupsCreateStatusHandler interface for that can handle valid backups create status params -type BackupsCreateStatusHandler interface { - Handle(BackupsCreateStatusParams, *models.Principal) middleware.Responder -} - -// NewBackupsCreateStatus creates a new http.Handler for the backups create status operation -func NewBackupsCreateStatus(ctx *middleware.Context, handler BackupsCreateStatusHandler) *BackupsCreateStatus { - return &BackupsCreateStatus{Context: ctx, Handler: handler} -} - -/* - BackupsCreateStatus swagger:route GET /backups/{backend}/{id} backups backupsCreateStatus - -Returns status of backup creation attempt for a set of classes -*/ -type BackupsCreateStatus struct { - Context *middleware.Context - Handler BackupsCreateStatusHandler -} - -func (o *BackupsCreateStatus) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewBackupsCreateStatusParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/backups/backups_create_status_parameters.go b/adapters/handlers/rest/operations/backups/backups_create_status_parameters.go deleted file mode 100644 index 7fcae3db4684ef9ea96814ad3978171821cef88d..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/backups/backups_create_status_parameters.go +++ /dev/null @@ -1,106 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package backups - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" -) - -// NewBackupsCreateStatusParams creates a new BackupsCreateStatusParams object -// -// There are no default values defined in the spec. -func NewBackupsCreateStatusParams() BackupsCreateStatusParams { - - return BackupsCreateStatusParams{} -} - -// BackupsCreateStatusParams contains all the bound params for the backups create status operation -// typically these are obtained from a http.Request -// -// swagger:parameters backups.create.status -type BackupsCreateStatusParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` - - /*Backup backend name e.g. filesystem, gcs, s3. - Required: true - In: path - */ - Backend string - /*The ID of a backup. Must be URL-safe and work as a filesystem path, only lowercase, numbers, underscore, minus characters allowed. - Required: true - In: path - */ - ID string -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewBackupsCreateStatusParams() beforehand. -func (o *BackupsCreateStatusParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - rBackend, rhkBackend, _ := route.Params.GetOK("backend") - if err := o.bindBackend(rBackend, rhkBackend, route.Formats); err != nil { - res = append(res, err) - } - - rID, rhkID, _ := route.Params.GetOK("id") - if err := o.bindID(rID, rhkID, route.Formats); err != nil { - res = append(res, err) - } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// bindBackend binds and validates parameter Backend from path. -func (o *BackupsCreateStatusParams) bindBackend(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - o.Backend = raw - - return nil -} - -// bindID binds and validates parameter ID from path. -func (o *BackupsCreateStatusParams) bindID(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - o.ID = raw - - return nil -} diff --git a/adapters/handlers/rest/operations/backups/backups_create_status_responses.go b/adapters/handlers/rest/operations/backups/backups_create_status_responses.go deleted file mode 100644 index e3488a28fc51e9b4453b0f5caeefc1bce845f2ea..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/backups/backups_create_status_responses.go +++ /dev/null @@ -1,275 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package backups - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// BackupsCreateStatusOKCode is the HTTP code returned for type BackupsCreateStatusOK -const BackupsCreateStatusOKCode int = 200 - -/* -BackupsCreateStatusOK Backup creation status successfully returned - -swagger:response backupsCreateStatusOK -*/ -type BackupsCreateStatusOK struct { - - /* - In: Body - */ - Payload *models.BackupCreateStatusResponse `json:"body,omitempty"` -} - -// NewBackupsCreateStatusOK creates BackupsCreateStatusOK with default headers values -func NewBackupsCreateStatusOK() *BackupsCreateStatusOK { - - return &BackupsCreateStatusOK{} -} - -// WithPayload adds the payload to the backups create status o k response -func (o *BackupsCreateStatusOK) WithPayload(payload *models.BackupCreateStatusResponse) *BackupsCreateStatusOK { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the backups create status o k response -func (o *BackupsCreateStatusOK) SetPayload(payload *models.BackupCreateStatusResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *BackupsCreateStatusOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(200) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// BackupsCreateStatusUnauthorizedCode is the HTTP code returned for type BackupsCreateStatusUnauthorized -const BackupsCreateStatusUnauthorizedCode int = 401 - -/* -BackupsCreateStatusUnauthorized Unauthorized or invalid credentials. - -swagger:response backupsCreateStatusUnauthorized -*/ -type BackupsCreateStatusUnauthorized struct { -} - -// NewBackupsCreateStatusUnauthorized creates BackupsCreateStatusUnauthorized with default headers values -func NewBackupsCreateStatusUnauthorized() *BackupsCreateStatusUnauthorized { - - return &BackupsCreateStatusUnauthorized{} -} - -// WriteResponse to the client -func (o *BackupsCreateStatusUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// BackupsCreateStatusForbiddenCode is the HTTP code returned for type BackupsCreateStatusForbidden -const BackupsCreateStatusForbiddenCode int = 403 - -/* -BackupsCreateStatusForbidden Forbidden - -swagger:response backupsCreateStatusForbidden -*/ -type BackupsCreateStatusForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewBackupsCreateStatusForbidden creates BackupsCreateStatusForbidden with default headers values -func NewBackupsCreateStatusForbidden() *BackupsCreateStatusForbidden { - - return &BackupsCreateStatusForbidden{} -} - -// WithPayload adds the payload to the backups create status forbidden response -func (o *BackupsCreateStatusForbidden) WithPayload(payload *models.ErrorResponse) *BackupsCreateStatusForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the backups create status forbidden response -func (o *BackupsCreateStatusForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *BackupsCreateStatusForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// BackupsCreateStatusNotFoundCode is the HTTP code returned for type BackupsCreateStatusNotFound -const BackupsCreateStatusNotFoundCode int = 404 - -/* -BackupsCreateStatusNotFound Not Found - Backup does not exist - -swagger:response backupsCreateStatusNotFound -*/ -type BackupsCreateStatusNotFound struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewBackupsCreateStatusNotFound creates BackupsCreateStatusNotFound with default headers values -func NewBackupsCreateStatusNotFound() *BackupsCreateStatusNotFound { - - return &BackupsCreateStatusNotFound{} -} - -// WithPayload adds the payload to the backups create status not found response -func (o *BackupsCreateStatusNotFound) WithPayload(payload *models.ErrorResponse) *BackupsCreateStatusNotFound { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the backups create status not found response -func (o *BackupsCreateStatusNotFound) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *BackupsCreateStatusNotFound) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(404) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// BackupsCreateStatusUnprocessableEntityCode is the HTTP code returned for type BackupsCreateStatusUnprocessableEntity -const BackupsCreateStatusUnprocessableEntityCode int = 422 - -/* -BackupsCreateStatusUnprocessableEntity Invalid backup restoration status attempt. - -swagger:response backupsCreateStatusUnprocessableEntity -*/ -type BackupsCreateStatusUnprocessableEntity struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewBackupsCreateStatusUnprocessableEntity creates BackupsCreateStatusUnprocessableEntity with default headers values -func NewBackupsCreateStatusUnprocessableEntity() *BackupsCreateStatusUnprocessableEntity { - - return &BackupsCreateStatusUnprocessableEntity{} -} - -// WithPayload adds the payload to the backups create status unprocessable entity response -func (o *BackupsCreateStatusUnprocessableEntity) WithPayload(payload *models.ErrorResponse) *BackupsCreateStatusUnprocessableEntity { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the backups create status unprocessable entity response -func (o *BackupsCreateStatusUnprocessableEntity) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *BackupsCreateStatusUnprocessableEntity) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(422) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// BackupsCreateStatusInternalServerErrorCode is the HTTP code returned for type BackupsCreateStatusInternalServerError -const BackupsCreateStatusInternalServerErrorCode int = 500 - -/* -BackupsCreateStatusInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response backupsCreateStatusInternalServerError -*/ -type BackupsCreateStatusInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewBackupsCreateStatusInternalServerError creates BackupsCreateStatusInternalServerError with default headers values -func NewBackupsCreateStatusInternalServerError() *BackupsCreateStatusInternalServerError { - - return &BackupsCreateStatusInternalServerError{} -} - -// WithPayload adds the payload to the backups create status internal server error response -func (o *BackupsCreateStatusInternalServerError) WithPayload(payload *models.ErrorResponse) *BackupsCreateStatusInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the backups create status internal server error response -func (o *BackupsCreateStatusInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *BackupsCreateStatusInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/backups/backups_create_status_urlbuilder.go b/adapters/handlers/rest/operations/backups/backups_create_status_urlbuilder.go deleted file mode 100644 index 519622ec363ed3356641403c1d86fbdba1998dbd..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/backups/backups_create_status_urlbuilder.go +++ /dev/null @@ -1,118 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package backups - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" - "strings" -) - -// BackupsCreateStatusURL generates an URL for the backups create status operation -type BackupsCreateStatusURL struct { - Backend string - ID string - - _basePath string - // avoid unkeyed usage - _ struct{} -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *BackupsCreateStatusURL) WithBasePath(bp string) *BackupsCreateStatusURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *BackupsCreateStatusURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *BackupsCreateStatusURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/backups/{backend}/{id}" - - backend := o.Backend - if backend != "" { - _path = strings.Replace(_path, "{backend}", backend, -1) - } else { - return nil, errors.New("backend is required on BackupsCreateStatusURL") - } - - id := o.ID - if id != "" { - _path = strings.Replace(_path, "{id}", id, -1) - } else { - return nil, errors.New("id is required on BackupsCreateStatusURL") - } - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *BackupsCreateStatusURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *BackupsCreateStatusURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *BackupsCreateStatusURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on BackupsCreateStatusURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on BackupsCreateStatusURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *BackupsCreateStatusURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/backups/backups_create_urlbuilder.go b/adapters/handlers/rest/operations/backups/backups_create_urlbuilder.go deleted file mode 100644 index 5e729e8c83f20ae8c7b26623e073817389e1df0f..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/backups/backups_create_urlbuilder.go +++ /dev/null @@ -1,110 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package backups - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" - "strings" -) - -// BackupsCreateURL generates an URL for the backups create operation -type BackupsCreateURL struct { - Backend string - - _basePath string - // avoid unkeyed usage - _ struct{} -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *BackupsCreateURL) WithBasePath(bp string) *BackupsCreateURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *BackupsCreateURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *BackupsCreateURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/backups/{backend}" - - backend := o.Backend - if backend != "" { - _path = strings.Replace(_path, "{backend}", backend, -1) - } else { - return nil, errors.New("backend is required on BackupsCreateURL") - } - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *BackupsCreateURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *BackupsCreateURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *BackupsCreateURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on BackupsCreateURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on BackupsCreateURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *BackupsCreateURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/backups/backups_restore.go b/adapters/handlers/rest/operations/backups/backups_restore.go deleted file mode 100644 index 29c8a384965b8ee9e6716496c4a66568ad426544..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/backups/backups_restore.go +++ /dev/null @@ -1,82 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package backups - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// BackupsRestoreHandlerFunc turns a function with the right signature into a backups restore handler -type BackupsRestoreHandlerFunc func(BackupsRestoreParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn BackupsRestoreHandlerFunc) Handle(params BackupsRestoreParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// BackupsRestoreHandler interface for that can handle valid backups restore params -type BackupsRestoreHandler interface { - Handle(BackupsRestoreParams, *models.Principal) middleware.Responder -} - -// NewBackupsRestore creates a new http.Handler for the backups restore operation -func NewBackupsRestore(ctx *middleware.Context, handler BackupsRestoreHandler) *BackupsRestore { - return &BackupsRestore{Context: ctx, Handler: handler} -} - -/* - BackupsRestore swagger:route POST /backups/{backend}/{id}/restore backups backupsRestore - -Starts a process of restoring a backup for a set of classes -*/ -type BackupsRestore struct { - Context *middleware.Context - Handler BackupsRestoreHandler -} - -func (o *BackupsRestore) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewBackupsRestoreParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/backups/backups_restore_parameters.go b/adapters/handlers/rest/operations/backups/backups_restore_parameters.go deleted file mode 100644 index b19c70f24833544d6044a0520b59b4760124993d..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/backups/backups_restore_parameters.go +++ /dev/null @@ -1,144 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package backups - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "io" - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/validate" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewBackupsRestoreParams creates a new BackupsRestoreParams object -// -// There are no default values defined in the spec. -func NewBackupsRestoreParams() BackupsRestoreParams { - - return BackupsRestoreParams{} -} - -// BackupsRestoreParams contains all the bound params for the backups restore operation -// typically these are obtained from a http.Request -// -// swagger:parameters backups.restore -type BackupsRestoreParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` - - /*Backup backend name e.g. filesystem, gcs, s3. - Required: true - In: path - */ - Backend string - /* - Required: true - In: body - */ - Body *models.BackupRestoreRequest - /*The ID of a backup. Must be URL-safe and work as a filesystem path, only lowercase, numbers, underscore, minus characters allowed. - Required: true - In: path - */ - ID string -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewBackupsRestoreParams() beforehand. -func (o *BackupsRestoreParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - rBackend, rhkBackend, _ := route.Params.GetOK("backend") - if err := o.bindBackend(rBackend, rhkBackend, route.Formats); err != nil { - res = append(res, err) - } - - if runtime.HasBody(r) { - defer r.Body.Close() - var body models.BackupRestoreRequest - if err := route.Consumer.Consume(r.Body, &body); err != nil { - if err == io.EOF { - res = append(res, errors.Required("body", "body", "")) - } else { - res = append(res, errors.NewParseError("body", "body", "", err)) - } - } else { - // validate body object - if err := body.Validate(route.Formats); err != nil { - res = append(res, err) - } - - ctx := validate.WithOperationRequest(r.Context()) - if err := body.ContextValidate(ctx, route.Formats); err != nil { - res = append(res, err) - } - - if len(res) == 0 { - o.Body = &body - } - } - } else { - res = append(res, errors.Required("body", "body", "")) - } - - rID, rhkID, _ := route.Params.GetOK("id") - if err := o.bindID(rID, rhkID, route.Formats); err != nil { - res = append(res, err) - } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// bindBackend binds and validates parameter Backend from path. -func (o *BackupsRestoreParams) bindBackend(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - o.Backend = raw - - return nil -} - -// bindID binds and validates parameter ID from path. -func (o *BackupsRestoreParams) bindID(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - o.ID = raw - - return nil -} diff --git a/adapters/handlers/rest/operations/backups/backups_restore_responses.go b/adapters/handlers/rest/operations/backups/backups_restore_responses.go deleted file mode 100644 index eb1d8d2c5e1329c5d94cc15954cb35caeaae48c6..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/backups/backups_restore_responses.go +++ /dev/null @@ -1,275 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package backups - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// BackupsRestoreOKCode is the HTTP code returned for type BackupsRestoreOK -const BackupsRestoreOKCode int = 200 - -/* -BackupsRestoreOK Backup restoration process successfully started. - -swagger:response backupsRestoreOK -*/ -type BackupsRestoreOK struct { - - /* - In: Body - */ - Payload *models.BackupRestoreResponse `json:"body,omitempty"` -} - -// NewBackupsRestoreOK creates BackupsRestoreOK with default headers values -func NewBackupsRestoreOK() *BackupsRestoreOK { - - return &BackupsRestoreOK{} -} - -// WithPayload adds the payload to the backups restore o k response -func (o *BackupsRestoreOK) WithPayload(payload *models.BackupRestoreResponse) *BackupsRestoreOK { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the backups restore o k response -func (o *BackupsRestoreOK) SetPayload(payload *models.BackupRestoreResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *BackupsRestoreOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(200) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// BackupsRestoreUnauthorizedCode is the HTTP code returned for type BackupsRestoreUnauthorized -const BackupsRestoreUnauthorizedCode int = 401 - -/* -BackupsRestoreUnauthorized Unauthorized or invalid credentials. - -swagger:response backupsRestoreUnauthorized -*/ -type BackupsRestoreUnauthorized struct { -} - -// NewBackupsRestoreUnauthorized creates BackupsRestoreUnauthorized with default headers values -func NewBackupsRestoreUnauthorized() *BackupsRestoreUnauthorized { - - return &BackupsRestoreUnauthorized{} -} - -// WriteResponse to the client -func (o *BackupsRestoreUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// BackupsRestoreForbiddenCode is the HTTP code returned for type BackupsRestoreForbidden -const BackupsRestoreForbiddenCode int = 403 - -/* -BackupsRestoreForbidden Forbidden - -swagger:response backupsRestoreForbidden -*/ -type BackupsRestoreForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewBackupsRestoreForbidden creates BackupsRestoreForbidden with default headers values -func NewBackupsRestoreForbidden() *BackupsRestoreForbidden { - - return &BackupsRestoreForbidden{} -} - -// WithPayload adds the payload to the backups restore forbidden response -func (o *BackupsRestoreForbidden) WithPayload(payload *models.ErrorResponse) *BackupsRestoreForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the backups restore forbidden response -func (o *BackupsRestoreForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *BackupsRestoreForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// BackupsRestoreNotFoundCode is the HTTP code returned for type BackupsRestoreNotFound -const BackupsRestoreNotFoundCode int = 404 - -/* -BackupsRestoreNotFound Not Found - Backup does not exist - -swagger:response backupsRestoreNotFound -*/ -type BackupsRestoreNotFound struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewBackupsRestoreNotFound creates BackupsRestoreNotFound with default headers values -func NewBackupsRestoreNotFound() *BackupsRestoreNotFound { - - return &BackupsRestoreNotFound{} -} - -// WithPayload adds the payload to the backups restore not found response -func (o *BackupsRestoreNotFound) WithPayload(payload *models.ErrorResponse) *BackupsRestoreNotFound { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the backups restore not found response -func (o *BackupsRestoreNotFound) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *BackupsRestoreNotFound) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(404) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// BackupsRestoreUnprocessableEntityCode is the HTTP code returned for type BackupsRestoreUnprocessableEntity -const BackupsRestoreUnprocessableEntityCode int = 422 - -/* -BackupsRestoreUnprocessableEntity Invalid backup restoration attempt. - -swagger:response backupsRestoreUnprocessableEntity -*/ -type BackupsRestoreUnprocessableEntity struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewBackupsRestoreUnprocessableEntity creates BackupsRestoreUnprocessableEntity with default headers values -func NewBackupsRestoreUnprocessableEntity() *BackupsRestoreUnprocessableEntity { - - return &BackupsRestoreUnprocessableEntity{} -} - -// WithPayload adds the payload to the backups restore unprocessable entity response -func (o *BackupsRestoreUnprocessableEntity) WithPayload(payload *models.ErrorResponse) *BackupsRestoreUnprocessableEntity { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the backups restore unprocessable entity response -func (o *BackupsRestoreUnprocessableEntity) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *BackupsRestoreUnprocessableEntity) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(422) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// BackupsRestoreInternalServerErrorCode is the HTTP code returned for type BackupsRestoreInternalServerError -const BackupsRestoreInternalServerErrorCode int = 500 - -/* -BackupsRestoreInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response backupsRestoreInternalServerError -*/ -type BackupsRestoreInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewBackupsRestoreInternalServerError creates BackupsRestoreInternalServerError with default headers values -func NewBackupsRestoreInternalServerError() *BackupsRestoreInternalServerError { - - return &BackupsRestoreInternalServerError{} -} - -// WithPayload adds the payload to the backups restore internal server error response -func (o *BackupsRestoreInternalServerError) WithPayload(payload *models.ErrorResponse) *BackupsRestoreInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the backups restore internal server error response -func (o *BackupsRestoreInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *BackupsRestoreInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/backups/backups_restore_status.go b/adapters/handlers/rest/operations/backups/backups_restore_status.go deleted file mode 100644 index b41ef87cb52bc1cf33b0ca6fc9baa1e5325bac84..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/backups/backups_restore_status.go +++ /dev/null @@ -1,82 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package backups - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// BackupsRestoreStatusHandlerFunc turns a function with the right signature into a backups restore status handler -type BackupsRestoreStatusHandlerFunc func(BackupsRestoreStatusParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn BackupsRestoreStatusHandlerFunc) Handle(params BackupsRestoreStatusParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// BackupsRestoreStatusHandler interface for that can handle valid backups restore status params -type BackupsRestoreStatusHandler interface { - Handle(BackupsRestoreStatusParams, *models.Principal) middleware.Responder -} - -// NewBackupsRestoreStatus creates a new http.Handler for the backups restore status operation -func NewBackupsRestoreStatus(ctx *middleware.Context, handler BackupsRestoreStatusHandler) *BackupsRestoreStatus { - return &BackupsRestoreStatus{Context: ctx, Handler: handler} -} - -/* - BackupsRestoreStatus swagger:route GET /backups/{backend}/{id}/restore backups backupsRestoreStatus - -Returns status of a backup restoration attempt for a set of classes -*/ -type BackupsRestoreStatus struct { - Context *middleware.Context - Handler BackupsRestoreStatusHandler -} - -func (o *BackupsRestoreStatus) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewBackupsRestoreStatusParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/backups/backups_restore_status_parameters.go b/adapters/handlers/rest/operations/backups/backups_restore_status_parameters.go deleted file mode 100644 index 8f7df4c659d9cce2f48a6152ccefba57a832f469..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/backups/backups_restore_status_parameters.go +++ /dev/null @@ -1,106 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package backups - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" -) - -// NewBackupsRestoreStatusParams creates a new BackupsRestoreStatusParams object -// -// There are no default values defined in the spec. -func NewBackupsRestoreStatusParams() BackupsRestoreStatusParams { - - return BackupsRestoreStatusParams{} -} - -// BackupsRestoreStatusParams contains all the bound params for the backups restore status operation -// typically these are obtained from a http.Request -// -// swagger:parameters backups.restore.status -type BackupsRestoreStatusParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` - - /*Backup backend name e.g. filesystem, gcs, s3. - Required: true - In: path - */ - Backend string - /*The ID of a backup. Must be URL-safe and work as a filesystem path, only lowercase, numbers, underscore, minus characters allowed. - Required: true - In: path - */ - ID string -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewBackupsRestoreStatusParams() beforehand. -func (o *BackupsRestoreStatusParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - rBackend, rhkBackend, _ := route.Params.GetOK("backend") - if err := o.bindBackend(rBackend, rhkBackend, route.Formats); err != nil { - res = append(res, err) - } - - rID, rhkID, _ := route.Params.GetOK("id") - if err := o.bindID(rID, rhkID, route.Formats); err != nil { - res = append(res, err) - } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// bindBackend binds and validates parameter Backend from path. -func (o *BackupsRestoreStatusParams) bindBackend(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - o.Backend = raw - - return nil -} - -// bindID binds and validates parameter ID from path. -func (o *BackupsRestoreStatusParams) bindID(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - o.ID = raw - - return nil -} diff --git a/adapters/handlers/rest/operations/backups/backups_restore_status_responses.go b/adapters/handlers/rest/operations/backups/backups_restore_status_responses.go deleted file mode 100644 index 5f2f921d494b51baa88b6aa522fd99f545bb5d54..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/backups/backups_restore_status_responses.go +++ /dev/null @@ -1,230 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package backups - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// BackupsRestoreStatusOKCode is the HTTP code returned for type BackupsRestoreStatusOK -const BackupsRestoreStatusOKCode int = 200 - -/* -BackupsRestoreStatusOK Backup restoration status successfully returned - -swagger:response backupsRestoreStatusOK -*/ -type BackupsRestoreStatusOK struct { - - /* - In: Body - */ - Payload *models.BackupRestoreStatusResponse `json:"body,omitempty"` -} - -// NewBackupsRestoreStatusOK creates BackupsRestoreStatusOK with default headers values -func NewBackupsRestoreStatusOK() *BackupsRestoreStatusOK { - - return &BackupsRestoreStatusOK{} -} - -// WithPayload adds the payload to the backups restore status o k response -func (o *BackupsRestoreStatusOK) WithPayload(payload *models.BackupRestoreStatusResponse) *BackupsRestoreStatusOK { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the backups restore status o k response -func (o *BackupsRestoreStatusOK) SetPayload(payload *models.BackupRestoreStatusResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *BackupsRestoreStatusOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(200) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// BackupsRestoreStatusUnauthorizedCode is the HTTP code returned for type BackupsRestoreStatusUnauthorized -const BackupsRestoreStatusUnauthorizedCode int = 401 - -/* -BackupsRestoreStatusUnauthorized Unauthorized or invalid credentials. - -swagger:response backupsRestoreStatusUnauthorized -*/ -type BackupsRestoreStatusUnauthorized struct { -} - -// NewBackupsRestoreStatusUnauthorized creates BackupsRestoreStatusUnauthorized with default headers values -func NewBackupsRestoreStatusUnauthorized() *BackupsRestoreStatusUnauthorized { - - return &BackupsRestoreStatusUnauthorized{} -} - -// WriteResponse to the client -func (o *BackupsRestoreStatusUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// BackupsRestoreStatusForbiddenCode is the HTTP code returned for type BackupsRestoreStatusForbidden -const BackupsRestoreStatusForbiddenCode int = 403 - -/* -BackupsRestoreStatusForbidden Forbidden - -swagger:response backupsRestoreStatusForbidden -*/ -type BackupsRestoreStatusForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewBackupsRestoreStatusForbidden creates BackupsRestoreStatusForbidden with default headers values -func NewBackupsRestoreStatusForbidden() *BackupsRestoreStatusForbidden { - - return &BackupsRestoreStatusForbidden{} -} - -// WithPayload adds the payload to the backups restore status forbidden response -func (o *BackupsRestoreStatusForbidden) WithPayload(payload *models.ErrorResponse) *BackupsRestoreStatusForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the backups restore status forbidden response -func (o *BackupsRestoreStatusForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *BackupsRestoreStatusForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// BackupsRestoreStatusNotFoundCode is the HTTP code returned for type BackupsRestoreStatusNotFound -const BackupsRestoreStatusNotFoundCode int = 404 - -/* -BackupsRestoreStatusNotFound Not Found - Backup does not exist - -swagger:response backupsRestoreStatusNotFound -*/ -type BackupsRestoreStatusNotFound struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewBackupsRestoreStatusNotFound creates BackupsRestoreStatusNotFound with default headers values -func NewBackupsRestoreStatusNotFound() *BackupsRestoreStatusNotFound { - - return &BackupsRestoreStatusNotFound{} -} - -// WithPayload adds the payload to the backups restore status not found response -func (o *BackupsRestoreStatusNotFound) WithPayload(payload *models.ErrorResponse) *BackupsRestoreStatusNotFound { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the backups restore status not found response -func (o *BackupsRestoreStatusNotFound) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *BackupsRestoreStatusNotFound) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(404) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// BackupsRestoreStatusInternalServerErrorCode is the HTTP code returned for type BackupsRestoreStatusInternalServerError -const BackupsRestoreStatusInternalServerErrorCode int = 500 - -/* -BackupsRestoreStatusInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response backupsRestoreStatusInternalServerError -*/ -type BackupsRestoreStatusInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewBackupsRestoreStatusInternalServerError creates BackupsRestoreStatusInternalServerError with default headers values -func NewBackupsRestoreStatusInternalServerError() *BackupsRestoreStatusInternalServerError { - - return &BackupsRestoreStatusInternalServerError{} -} - -// WithPayload adds the payload to the backups restore status internal server error response -func (o *BackupsRestoreStatusInternalServerError) WithPayload(payload *models.ErrorResponse) *BackupsRestoreStatusInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the backups restore status internal server error response -func (o *BackupsRestoreStatusInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *BackupsRestoreStatusInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/backups/backups_restore_status_urlbuilder.go b/adapters/handlers/rest/operations/backups/backups_restore_status_urlbuilder.go deleted file mode 100644 index 1fe01e50a3b1cdbb60aa811595dc5a592b11a071..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/backups/backups_restore_status_urlbuilder.go +++ /dev/null @@ -1,118 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package backups - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" - "strings" -) - -// BackupsRestoreStatusURL generates an URL for the backups restore status operation -type BackupsRestoreStatusURL struct { - Backend string - ID string - - _basePath string - // avoid unkeyed usage - _ struct{} -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *BackupsRestoreStatusURL) WithBasePath(bp string) *BackupsRestoreStatusURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *BackupsRestoreStatusURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *BackupsRestoreStatusURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/backups/{backend}/{id}/restore" - - backend := o.Backend - if backend != "" { - _path = strings.Replace(_path, "{backend}", backend, -1) - } else { - return nil, errors.New("backend is required on BackupsRestoreStatusURL") - } - - id := o.ID - if id != "" { - _path = strings.Replace(_path, "{id}", id, -1) - } else { - return nil, errors.New("id is required on BackupsRestoreStatusURL") - } - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *BackupsRestoreStatusURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *BackupsRestoreStatusURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *BackupsRestoreStatusURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on BackupsRestoreStatusURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on BackupsRestoreStatusURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *BackupsRestoreStatusURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/backups/backups_restore_urlbuilder.go b/adapters/handlers/rest/operations/backups/backups_restore_urlbuilder.go deleted file mode 100644 index 1a4be1c3b0d3ebe69d7a53bcc703667d284112c8..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/backups/backups_restore_urlbuilder.go +++ /dev/null @@ -1,118 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package backups - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" - "strings" -) - -// BackupsRestoreURL generates an URL for the backups restore operation -type BackupsRestoreURL struct { - Backend string - ID string - - _basePath string - // avoid unkeyed usage - _ struct{} -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *BackupsRestoreURL) WithBasePath(bp string) *BackupsRestoreURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *BackupsRestoreURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *BackupsRestoreURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/backups/{backend}/{id}/restore" - - backend := o.Backend - if backend != "" { - _path = strings.Replace(_path, "{backend}", backend, -1) - } else { - return nil, errors.New("backend is required on BackupsRestoreURL") - } - - id := o.ID - if id != "" { - _path = strings.Replace(_path, "{id}", id, -1) - } else { - return nil, errors.New("id is required on BackupsRestoreURL") - } - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *BackupsRestoreURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *BackupsRestoreURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *BackupsRestoreURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on BackupsRestoreURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on BackupsRestoreURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *BackupsRestoreURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/batch/batch_objects_create.go b/adapters/handlers/rest/operations/batch/batch_objects_create.go deleted file mode 100644 index 2f392ce53487da383042b6b292e112a815c61fc8..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/batch/batch_objects_create.go +++ /dev/null @@ -1,238 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package batch - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "context" - "encoding/json" - "net/http" - "strconv" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" - "github.com/go-openapi/validate" - - "github.com/weaviate/weaviate/entities/models" -) - -// BatchObjectsCreateHandlerFunc turns a function with the right signature into a batch objects create handler -type BatchObjectsCreateHandlerFunc func(BatchObjectsCreateParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn BatchObjectsCreateHandlerFunc) Handle(params BatchObjectsCreateParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// BatchObjectsCreateHandler interface for that can handle valid batch objects create params -type BatchObjectsCreateHandler interface { - Handle(BatchObjectsCreateParams, *models.Principal) middleware.Responder -} - -// NewBatchObjectsCreate creates a new http.Handler for the batch objects create operation -func NewBatchObjectsCreate(ctx *middleware.Context, handler BatchObjectsCreateHandler) *BatchObjectsCreate { - return &BatchObjectsCreate{Context: ctx, Handler: handler} -} - -/* - BatchObjectsCreate swagger:route POST /batch/objects batch objects batchObjectsCreate - -Creates new Objects based on a Object template as a batch. - -Register new Objects in bulk. Provided meta-data and schema values are validated. -*/ -type BatchObjectsCreate struct { - Context *middleware.Context - Handler BatchObjectsCreateHandler -} - -func (o *BatchObjectsCreate) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewBatchObjectsCreateParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} - -// BatchObjectsCreateBody batch objects create body -// -// swagger:model BatchObjectsCreateBody -type BatchObjectsCreateBody struct { - - // Define which fields need to be returned. Default value is ALL - Fields []*string `json:"fields" yaml:"fields"` - - // objects - Objects []*models.Object `json:"objects" yaml:"objects"` -} - -// Validate validates this batch objects create body -func (o *BatchObjectsCreateBody) Validate(formats strfmt.Registry) error { - var res []error - - if err := o.validateFields(formats); err != nil { - res = append(res, err) - } - - if err := o.validateObjects(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -var batchObjectsCreateBodyFieldsItemsEnum []interface{} - -func init() { - var res []string - if err := json.Unmarshal([]byte(`["ALL","class","schema","id","creationTimeUnix"]`), &res); err != nil { - panic(err) - } - for _, v := range res { - batchObjectsCreateBodyFieldsItemsEnum = append(batchObjectsCreateBodyFieldsItemsEnum, v) - } -} - -func (o *BatchObjectsCreateBody) validateFieldsItemsEnum(path, location string, value string) error { - if err := validate.EnumCase(path, location, value, batchObjectsCreateBodyFieldsItemsEnum, true); err != nil { - return err - } - return nil -} - -func (o *BatchObjectsCreateBody) validateFields(formats strfmt.Registry) error { - if swag.IsZero(o.Fields) { // not required - return nil - } - - for i := 0; i < len(o.Fields); i++ { - if swag.IsZero(o.Fields[i]) { // not required - continue - } - - // value enum - if err := o.validateFieldsItemsEnum("body"+"."+"fields"+"."+strconv.Itoa(i), "body", *o.Fields[i]); err != nil { - return err - } - - } - - return nil -} - -func (o *BatchObjectsCreateBody) validateObjects(formats strfmt.Registry) error { - if swag.IsZero(o.Objects) { // not required - return nil - } - - for i := 0; i < len(o.Objects); i++ { - if swag.IsZero(o.Objects[i]) { // not required - continue - } - - if o.Objects[i] != nil { - if err := o.Objects[i].Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("body" + "." + "objects" + "." + strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("body" + "." + "objects" + "." + strconv.Itoa(i)) - } - return err - } - } - - } - - return nil -} - -// ContextValidate validate this batch objects create body based on the context it is used -func (o *BatchObjectsCreateBody) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - if err := o.contextValidateObjects(ctx, formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (o *BatchObjectsCreateBody) contextValidateObjects(ctx context.Context, formats strfmt.Registry) error { - - for i := 0; i < len(o.Objects); i++ { - - if o.Objects[i] != nil { - if err := o.Objects[i].ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("body" + "." + "objects" + "." + strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("body" + "." + "objects" + "." + strconv.Itoa(i)) - } - return err - } - } - - } - - return nil -} - -// MarshalBinary interface implementation -func (o *BatchObjectsCreateBody) MarshalBinary() ([]byte, error) { - if o == nil { - return nil, nil - } - return swag.WriteJSON(o) -} - -// UnmarshalBinary interface implementation -func (o *BatchObjectsCreateBody) UnmarshalBinary(b []byte) error { - var res BatchObjectsCreateBody - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *o = res - return nil -} diff --git a/adapters/handlers/rest/operations/batch/batch_objects_create_parameters.go b/adapters/handlers/rest/operations/batch/batch_objects_create_parameters.go deleted file mode 100644 index a576a7f214210e281461478318a82ce7d9eb945a..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/batch/batch_objects_create_parameters.go +++ /dev/null @@ -1,123 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package batch - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "io" - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/validate" -) - -// NewBatchObjectsCreateParams creates a new BatchObjectsCreateParams object -// -// There are no default values defined in the spec. -func NewBatchObjectsCreateParams() BatchObjectsCreateParams { - - return BatchObjectsCreateParams{} -} - -// BatchObjectsCreateParams contains all the bound params for the batch objects create operation -// typically these are obtained from a http.Request -// -// swagger:parameters batch.objects.create -type BatchObjectsCreateParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` - - /* - Required: true - In: body - */ - Body BatchObjectsCreateBody - /*Determines how many replicas must acknowledge a request before it is considered successful - In: query - */ - ConsistencyLevel *string -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewBatchObjectsCreateParams() beforehand. -func (o *BatchObjectsCreateParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - qs := runtime.Values(r.URL.Query()) - - if runtime.HasBody(r) { - defer r.Body.Close() - var body BatchObjectsCreateBody - if err := route.Consumer.Consume(r.Body, &body); err != nil { - if err == io.EOF { - res = append(res, errors.Required("body", "body", "")) - } else { - res = append(res, errors.NewParseError("body", "body", "", err)) - } - } else { - // validate body object - if err := body.Validate(route.Formats); err != nil { - res = append(res, err) - } - - ctx := validate.WithOperationRequest(r.Context()) - if err := body.ContextValidate(ctx, route.Formats); err != nil { - res = append(res, err) - } - - if len(res) == 0 { - o.Body = body - } - } - } else { - res = append(res, errors.Required("body", "body", "")) - } - - qConsistencyLevel, qhkConsistencyLevel, _ := qs.GetOK("consistency_level") - if err := o.bindConsistencyLevel(qConsistencyLevel, qhkConsistencyLevel, route.Formats); err != nil { - res = append(res, err) - } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// bindConsistencyLevel binds and validates parameter ConsistencyLevel from query. -func (o *BatchObjectsCreateParams) bindConsistencyLevel(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: false - // AllowEmptyValue: false - - if raw == "" { // empty values pass all other validations - return nil - } - o.ConsistencyLevel = &raw - - return nil -} diff --git a/adapters/handlers/rest/operations/batch/batch_objects_create_responses.go b/adapters/handlers/rest/operations/batch/batch_objects_create_responses.go deleted file mode 100644 index 36542d90c71d4bd8181cf10bf40360625b15e08d..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/batch/batch_objects_create_responses.go +++ /dev/null @@ -1,278 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package batch - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// BatchObjectsCreateOKCode is the HTTP code returned for type BatchObjectsCreateOK -const BatchObjectsCreateOKCode int = 200 - -/* -BatchObjectsCreateOK Request succeeded, see response body to get detailed information about each batched item. - -swagger:response batchObjectsCreateOK -*/ -type BatchObjectsCreateOK struct { - - /* - In: Body - */ - Payload []*models.ObjectsGetResponse `json:"body,omitempty"` -} - -// NewBatchObjectsCreateOK creates BatchObjectsCreateOK with default headers values -func NewBatchObjectsCreateOK() *BatchObjectsCreateOK { - - return &BatchObjectsCreateOK{} -} - -// WithPayload adds the payload to the batch objects create o k response -func (o *BatchObjectsCreateOK) WithPayload(payload []*models.ObjectsGetResponse) *BatchObjectsCreateOK { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the batch objects create o k response -func (o *BatchObjectsCreateOK) SetPayload(payload []*models.ObjectsGetResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *BatchObjectsCreateOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(200) - payload := o.Payload - if payload == nil { - // return empty array - payload = make([]*models.ObjectsGetResponse, 0, 50) - } - - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } -} - -// BatchObjectsCreateBadRequestCode is the HTTP code returned for type BatchObjectsCreateBadRequest -const BatchObjectsCreateBadRequestCode int = 400 - -/* -BatchObjectsCreateBadRequest Malformed request. - -swagger:response batchObjectsCreateBadRequest -*/ -type BatchObjectsCreateBadRequest struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewBatchObjectsCreateBadRequest creates BatchObjectsCreateBadRequest with default headers values -func NewBatchObjectsCreateBadRequest() *BatchObjectsCreateBadRequest { - - return &BatchObjectsCreateBadRequest{} -} - -// WithPayload adds the payload to the batch objects create bad request response -func (o *BatchObjectsCreateBadRequest) WithPayload(payload *models.ErrorResponse) *BatchObjectsCreateBadRequest { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the batch objects create bad request response -func (o *BatchObjectsCreateBadRequest) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *BatchObjectsCreateBadRequest) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(400) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// BatchObjectsCreateUnauthorizedCode is the HTTP code returned for type BatchObjectsCreateUnauthorized -const BatchObjectsCreateUnauthorizedCode int = 401 - -/* -BatchObjectsCreateUnauthorized Unauthorized or invalid credentials. - -swagger:response batchObjectsCreateUnauthorized -*/ -type BatchObjectsCreateUnauthorized struct { -} - -// NewBatchObjectsCreateUnauthorized creates BatchObjectsCreateUnauthorized with default headers values -func NewBatchObjectsCreateUnauthorized() *BatchObjectsCreateUnauthorized { - - return &BatchObjectsCreateUnauthorized{} -} - -// WriteResponse to the client -func (o *BatchObjectsCreateUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// BatchObjectsCreateForbiddenCode is the HTTP code returned for type BatchObjectsCreateForbidden -const BatchObjectsCreateForbiddenCode int = 403 - -/* -BatchObjectsCreateForbidden Forbidden - -swagger:response batchObjectsCreateForbidden -*/ -type BatchObjectsCreateForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewBatchObjectsCreateForbidden creates BatchObjectsCreateForbidden with default headers values -func NewBatchObjectsCreateForbidden() *BatchObjectsCreateForbidden { - - return &BatchObjectsCreateForbidden{} -} - -// WithPayload adds the payload to the batch objects create forbidden response -func (o *BatchObjectsCreateForbidden) WithPayload(payload *models.ErrorResponse) *BatchObjectsCreateForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the batch objects create forbidden response -func (o *BatchObjectsCreateForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *BatchObjectsCreateForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// BatchObjectsCreateUnprocessableEntityCode is the HTTP code returned for type BatchObjectsCreateUnprocessableEntity -const BatchObjectsCreateUnprocessableEntityCode int = 422 - -/* -BatchObjectsCreateUnprocessableEntity Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file? - -swagger:response batchObjectsCreateUnprocessableEntity -*/ -type BatchObjectsCreateUnprocessableEntity struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewBatchObjectsCreateUnprocessableEntity creates BatchObjectsCreateUnprocessableEntity with default headers values -func NewBatchObjectsCreateUnprocessableEntity() *BatchObjectsCreateUnprocessableEntity { - - return &BatchObjectsCreateUnprocessableEntity{} -} - -// WithPayload adds the payload to the batch objects create unprocessable entity response -func (o *BatchObjectsCreateUnprocessableEntity) WithPayload(payload *models.ErrorResponse) *BatchObjectsCreateUnprocessableEntity { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the batch objects create unprocessable entity response -func (o *BatchObjectsCreateUnprocessableEntity) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *BatchObjectsCreateUnprocessableEntity) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(422) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// BatchObjectsCreateInternalServerErrorCode is the HTTP code returned for type BatchObjectsCreateInternalServerError -const BatchObjectsCreateInternalServerErrorCode int = 500 - -/* -BatchObjectsCreateInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response batchObjectsCreateInternalServerError -*/ -type BatchObjectsCreateInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewBatchObjectsCreateInternalServerError creates BatchObjectsCreateInternalServerError with default headers values -func NewBatchObjectsCreateInternalServerError() *BatchObjectsCreateInternalServerError { - - return &BatchObjectsCreateInternalServerError{} -} - -// WithPayload adds the payload to the batch objects create internal server error response -func (o *BatchObjectsCreateInternalServerError) WithPayload(payload *models.ErrorResponse) *BatchObjectsCreateInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the batch objects create internal server error response -func (o *BatchObjectsCreateInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *BatchObjectsCreateInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/batch/batch_objects_create_urlbuilder.go b/adapters/handlers/rest/operations/batch/batch_objects_create_urlbuilder.go deleted file mode 100644 index 0451350920c556b7c101388f62379bb3758d5e47..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/batch/batch_objects_create_urlbuilder.go +++ /dev/null @@ -1,114 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package batch - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" -) - -// BatchObjectsCreateURL generates an URL for the batch objects create operation -type BatchObjectsCreateURL struct { - ConsistencyLevel *string - - _basePath string - // avoid unkeyed usage - _ struct{} -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *BatchObjectsCreateURL) WithBasePath(bp string) *BatchObjectsCreateURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *BatchObjectsCreateURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *BatchObjectsCreateURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/batch/objects" - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - qs := make(url.Values) - - var consistencyLevelQ string - if o.ConsistencyLevel != nil { - consistencyLevelQ = *o.ConsistencyLevel - } - if consistencyLevelQ != "" { - qs.Set("consistency_level", consistencyLevelQ) - } - - _result.RawQuery = qs.Encode() - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *BatchObjectsCreateURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *BatchObjectsCreateURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *BatchObjectsCreateURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on BatchObjectsCreateURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on BatchObjectsCreateURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *BatchObjectsCreateURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/batch/batch_objects_delete.go b/adapters/handlers/rest/operations/batch/batch_objects_delete.go deleted file mode 100644 index 2168f3185c43fbbf67ddab48f94a29ddf1f64496..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/batch/batch_objects_delete.go +++ /dev/null @@ -1,84 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package batch - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// BatchObjectsDeleteHandlerFunc turns a function with the right signature into a batch objects delete handler -type BatchObjectsDeleteHandlerFunc func(BatchObjectsDeleteParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn BatchObjectsDeleteHandlerFunc) Handle(params BatchObjectsDeleteParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// BatchObjectsDeleteHandler interface for that can handle valid batch objects delete params -type BatchObjectsDeleteHandler interface { - Handle(BatchObjectsDeleteParams, *models.Principal) middleware.Responder -} - -// NewBatchObjectsDelete creates a new http.Handler for the batch objects delete operation -func NewBatchObjectsDelete(ctx *middleware.Context, handler BatchObjectsDeleteHandler) *BatchObjectsDelete { - return &BatchObjectsDelete{Context: ctx, Handler: handler} -} - -/* - BatchObjectsDelete swagger:route DELETE /batch/objects batch objects batchObjectsDelete - -Deletes Objects based on a match filter as a batch. - -Delete Objects in bulk that match a certain filter. -*/ -type BatchObjectsDelete struct { - Context *middleware.Context - Handler BatchObjectsDeleteHandler -} - -func (o *BatchObjectsDelete) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewBatchObjectsDeleteParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/batch/batch_objects_delete_parameters.go b/adapters/handlers/rest/operations/batch/batch_objects_delete_parameters.go deleted file mode 100644 index ea647121ddcd1467801ee3c862af0eb745e54c61..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/batch/batch_objects_delete_parameters.go +++ /dev/null @@ -1,152 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package batch - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "io" - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/validate" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewBatchObjectsDeleteParams creates a new BatchObjectsDeleteParams object -// -// There are no default values defined in the spec. -func NewBatchObjectsDeleteParams() BatchObjectsDeleteParams { - - return BatchObjectsDeleteParams{} -} - -// BatchObjectsDeleteParams contains all the bound params for the batch objects delete operation -// typically these are obtained from a http.Request -// -// swagger:parameters batch.objects.delete -type BatchObjectsDeleteParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` - - /* - Required: true - In: body - */ - Body *models.BatchDelete - /*Determines how many replicas must acknowledge a request before it is considered successful - In: query - */ - ConsistencyLevel *string - /*Specifies the tenant in a request targeting a multi-tenant class - In: query - */ - Tenant *string -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewBatchObjectsDeleteParams() beforehand. -func (o *BatchObjectsDeleteParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - qs := runtime.Values(r.URL.Query()) - - if runtime.HasBody(r) { - defer r.Body.Close() - var body models.BatchDelete - if err := route.Consumer.Consume(r.Body, &body); err != nil { - if err == io.EOF { - res = append(res, errors.Required("body", "body", "")) - } else { - res = append(res, errors.NewParseError("body", "body", "", err)) - } - } else { - // validate body object - if err := body.Validate(route.Formats); err != nil { - res = append(res, err) - } - - ctx := validate.WithOperationRequest(r.Context()) - if err := body.ContextValidate(ctx, route.Formats); err != nil { - res = append(res, err) - } - - if len(res) == 0 { - o.Body = &body - } - } - } else { - res = append(res, errors.Required("body", "body", "")) - } - - qConsistencyLevel, qhkConsistencyLevel, _ := qs.GetOK("consistency_level") - if err := o.bindConsistencyLevel(qConsistencyLevel, qhkConsistencyLevel, route.Formats); err != nil { - res = append(res, err) - } - - qTenant, qhkTenant, _ := qs.GetOK("tenant") - if err := o.bindTenant(qTenant, qhkTenant, route.Formats); err != nil { - res = append(res, err) - } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// bindConsistencyLevel binds and validates parameter ConsistencyLevel from query. -func (o *BatchObjectsDeleteParams) bindConsistencyLevel(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: false - // AllowEmptyValue: false - - if raw == "" { // empty values pass all other validations - return nil - } - o.ConsistencyLevel = &raw - - return nil -} - -// bindTenant binds and validates parameter Tenant from query. -func (o *BatchObjectsDeleteParams) bindTenant(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: false - // AllowEmptyValue: false - - if raw == "" { // empty values pass all other validations - return nil - } - o.Tenant = &raw - - return nil -} diff --git a/adapters/handlers/rest/operations/batch/batch_objects_delete_responses.go b/adapters/handlers/rest/operations/batch/batch_objects_delete_responses.go deleted file mode 100644 index da11938985c00b12dbddef22612343fa2967eb8b..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/batch/batch_objects_delete_responses.go +++ /dev/null @@ -1,275 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package batch - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// BatchObjectsDeleteOKCode is the HTTP code returned for type BatchObjectsDeleteOK -const BatchObjectsDeleteOKCode int = 200 - -/* -BatchObjectsDeleteOK Request succeeded, see response body to get detailed information about each batched item. - -swagger:response batchObjectsDeleteOK -*/ -type BatchObjectsDeleteOK struct { - - /* - In: Body - */ - Payload *models.BatchDeleteResponse `json:"body,omitempty"` -} - -// NewBatchObjectsDeleteOK creates BatchObjectsDeleteOK with default headers values -func NewBatchObjectsDeleteOK() *BatchObjectsDeleteOK { - - return &BatchObjectsDeleteOK{} -} - -// WithPayload adds the payload to the batch objects delete o k response -func (o *BatchObjectsDeleteOK) WithPayload(payload *models.BatchDeleteResponse) *BatchObjectsDeleteOK { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the batch objects delete o k response -func (o *BatchObjectsDeleteOK) SetPayload(payload *models.BatchDeleteResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *BatchObjectsDeleteOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(200) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// BatchObjectsDeleteBadRequestCode is the HTTP code returned for type BatchObjectsDeleteBadRequest -const BatchObjectsDeleteBadRequestCode int = 400 - -/* -BatchObjectsDeleteBadRequest Malformed request. - -swagger:response batchObjectsDeleteBadRequest -*/ -type BatchObjectsDeleteBadRequest struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewBatchObjectsDeleteBadRequest creates BatchObjectsDeleteBadRequest with default headers values -func NewBatchObjectsDeleteBadRequest() *BatchObjectsDeleteBadRequest { - - return &BatchObjectsDeleteBadRequest{} -} - -// WithPayload adds the payload to the batch objects delete bad request response -func (o *BatchObjectsDeleteBadRequest) WithPayload(payload *models.ErrorResponse) *BatchObjectsDeleteBadRequest { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the batch objects delete bad request response -func (o *BatchObjectsDeleteBadRequest) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *BatchObjectsDeleteBadRequest) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(400) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// BatchObjectsDeleteUnauthorizedCode is the HTTP code returned for type BatchObjectsDeleteUnauthorized -const BatchObjectsDeleteUnauthorizedCode int = 401 - -/* -BatchObjectsDeleteUnauthorized Unauthorized or invalid credentials. - -swagger:response batchObjectsDeleteUnauthorized -*/ -type BatchObjectsDeleteUnauthorized struct { -} - -// NewBatchObjectsDeleteUnauthorized creates BatchObjectsDeleteUnauthorized with default headers values -func NewBatchObjectsDeleteUnauthorized() *BatchObjectsDeleteUnauthorized { - - return &BatchObjectsDeleteUnauthorized{} -} - -// WriteResponse to the client -func (o *BatchObjectsDeleteUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// BatchObjectsDeleteForbiddenCode is the HTTP code returned for type BatchObjectsDeleteForbidden -const BatchObjectsDeleteForbiddenCode int = 403 - -/* -BatchObjectsDeleteForbidden Forbidden - -swagger:response batchObjectsDeleteForbidden -*/ -type BatchObjectsDeleteForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewBatchObjectsDeleteForbidden creates BatchObjectsDeleteForbidden with default headers values -func NewBatchObjectsDeleteForbidden() *BatchObjectsDeleteForbidden { - - return &BatchObjectsDeleteForbidden{} -} - -// WithPayload adds the payload to the batch objects delete forbidden response -func (o *BatchObjectsDeleteForbidden) WithPayload(payload *models.ErrorResponse) *BatchObjectsDeleteForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the batch objects delete forbidden response -func (o *BatchObjectsDeleteForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *BatchObjectsDeleteForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// BatchObjectsDeleteUnprocessableEntityCode is the HTTP code returned for type BatchObjectsDeleteUnprocessableEntity -const BatchObjectsDeleteUnprocessableEntityCode int = 422 - -/* -BatchObjectsDeleteUnprocessableEntity Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file? - -swagger:response batchObjectsDeleteUnprocessableEntity -*/ -type BatchObjectsDeleteUnprocessableEntity struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewBatchObjectsDeleteUnprocessableEntity creates BatchObjectsDeleteUnprocessableEntity with default headers values -func NewBatchObjectsDeleteUnprocessableEntity() *BatchObjectsDeleteUnprocessableEntity { - - return &BatchObjectsDeleteUnprocessableEntity{} -} - -// WithPayload adds the payload to the batch objects delete unprocessable entity response -func (o *BatchObjectsDeleteUnprocessableEntity) WithPayload(payload *models.ErrorResponse) *BatchObjectsDeleteUnprocessableEntity { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the batch objects delete unprocessable entity response -func (o *BatchObjectsDeleteUnprocessableEntity) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *BatchObjectsDeleteUnprocessableEntity) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(422) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// BatchObjectsDeleteInternalServerErrorCode is the HTTP code returned for type BatchObjectsDeleteInternalServerError -const BatchObjectsDeleteInternalServerErrorCode int = 500 - -/* -BatchObjectsDeleteInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response batchObjectsDeleteInternalServerError -*/ -type BatchObjectsDeleteInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewBatchObjectsDeleteInternalServerError creates BatchObjectsDeleteInternalServerError with default headers values -func NewBatchObjectsDeleteInternalServerError() *BatchObjectsDeleteInternalServerError { - - return &BatchObjectsDeleteInternalServerError{} -} - -// WithPayload adds the payload to the batch objects delete internal server error response -func (o *BatchObjectsDeleteInternalServerError) WithPayload(payload *models.ErrorResponse) *BatchObjectsDeleteInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the batch objects delete internal server error response -func (o *BatchObjectsDeleteInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *BatchObjectsDeleteInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/batch/batch_objects_delete_urlbuilder.go b/adapters/handlers/rest/operations/batch/batch_objects_delete_urlbuilder.go deleted file mode 100644 index 115ec8e1cfe9246d27e45472c01c349d3e74636d..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/batch/batch_objects_delete_urlbuilder.go +++ /dev/null @@ -1,123 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package batch - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" -) - -// BatchObjectsDeleteURL generates an URL for the batch objects delete operation -type BatchObjectsDeleteURL struct { - ConsistencyLevel *string - Tenant *string - - _basePath string - // avoid unkeyed usage - _ struct{} -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *BatchObjectsDeleteURL) WithBasePath(bp string) *BatchObjectsDeleteURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *BatchObjectsDeleteURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *BatchObjectsDeleteURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/batch/objects" - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - qs := make(url.Values) - - var consistencyLevelQ string - if o.ConsistencyLevel != nil { - consistencyLevelQ = *o.ConsistencyLevel - } - if consistencyLevelQ != "" { - qs.Set("consistency_level", consistencyLevelQ) - } - - var tenantQ string - if o.Tenant != nil { - tenantQ = *o.Tenant - } - if tenantQ != "" { - qs.Set("tenant", tenantQ) - } - - _result.RawQuery = qs.Encode() - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *BatchObjectsDeleteURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *BatchObjectsDeleteURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *BatchObjectsDeleteURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on BatchObjectsDeleteURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on BatchObjectsDeleteURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *BatchObjectsDeleteURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/batch/batch_references_create.go b/adapters/handlers/rest/operations/batch/batch_references_create.go deleted file mode 100644 index f9705142cca186b9cf9da4f9c08ced638773c442..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/batch/batch_references_create.go +++ /dev/null @@ -1,84 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package batch - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// BatchReferencesCreateHandlerFunc turns a function with the right signature into a batch references create handler -type BatchReferencesCreateHandlerFunc func(BatchReferencesCreateParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn BatchReferencesCreateHandlerFunc) Handle(params BatchReferencesCreateParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// BatchReferencesCreateHandler interface for that can handle valid batch references create params -type BatchReferencesCreateHandler interface { - Handle(BatchReferencesCreateParams, *models.Principal) middleware.Responder -} - -// NewBatchReferencesCreate creates a new http.Handler for the batch references create operation -func NewBatchReferencesCreate(ctx *middleware.Context, handler BatchReferencesCreateHandler) *BatchReferencesCreate { - return &BatchReferencesCreate{Context: ctx, Handler: handler} -} - -/* - BatchReferencesCreate swagger:route POST /batch/references batch references batchReferencesCreate - -Creates new Cross-References between arbitrary classes in bulk. - -Register cross-references between any class items (objects or objects) in bulk. -*/ -type BatchReferencesCreate struct { - Context *middleware.Context - Handler BatchReferencesCreateHandler -} - -func (o *BatchReferencesCreate) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewBatchReferencesCreateParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/batch/batch_references_create_parameters.go b/adapters/handlers/rest/operations/batch/batch_references_create_parameters.go deleted file mode 100644 index f6c58f2eb9563fcdee9b5d238a575d6a00c18a99..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/batch/batch_references_create_parameters.go +++ /dev/null @@ -1,126 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package batch - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "io" - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewBatchReferencesCreateParams creates a new BatchReferencesCreateParams object -// -// There are no default values defined in the spec. -func NewBatchReferencesCreateParams() BatchReferencesCreateParams { - - return BatchReferencesCreateParams{} -} - -// BatchReferencesCreateParams contains all the bound params for the batch references create operation -// typically these are obtained from a http.Request -// -// swagger:parameters batch.references.create -type BatchReferencesCreateParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` - - /*A list of references to be batched. The ideal size depends on the used database connector. Please see the documentation of the used connector for help - Required: true - In: body - */ - Body []*models.BatchReference - /*Determines how many replicas must acknowledge a request before it is considered successful - In: query - */ - ConsistencyLevel *string -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewBatchReferencesCreateParams() beforehand. -func (o *BatchReferencesCreateParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - qs := runtime.Values(r.URL.Query()) - - if runtime.HasBody(r) { - defer r.Body.Close() - var body []*models.BatchReference - if err := route.Consumer.Consume(r.Body, &body); err != nil { - if err == io.EOF { - res = append(res, errors.Required("body", "body", "")) - } else { - res = append(res, errors.NewParseError("body", "body", "", err)) - } - } else { - - // validate array of body objects - for i := range body { - if body[i] == nil { - continue - } - if err := body[i].Validate(route.Formats); err != nil { - res = append(res, err) - break - } - } - - if len(res) == 0 { - o.Body = body - } - } - } else { - res = append(res, errors.Required("body", "body", "")) - } - - qConsistencyLevel, qhkConsistencyLevel, _ := qs.GetOK("consistency_level") - if err := o.bindConsistencyLevel(qConsistencyLevel, qhkConsistencyLevel, route.Formats); err != nil { - res = append(res, err) - } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// bindConsistencyLevel binds and validates parameter ConsistencyLevel from query. -func (o *BatchReferencesCreateParams) bindConsistencyLevel(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: false - // AllowEmptyValue: false - - if raw == "" { // empty values pass all other validations - return nil - } - o.ConsistencyLevel = &raw - - return nil -} diff --git a/adapters/handlers/rest/operations/batch/batch_references_create_responses.go b/adapters/handlers/rest/operations/batch/batch_references_create_responses.go deleted file mode 100644 index 087817c12ef115d889e09f4a47e7b6e2999668af..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/batch/batch_references_create_responses.go +++ /dev/null @@ -1,278 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package batch - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// BatchReferencesCreateOKCode is the HTTP code returned for type BatchReferencesCreateOK -const BatchReferencesCreateOKCode int = 200 - -/* -BatchReferencesCreateOK Request Successful. Warning: A successful request does not guarantee that every batched reference was successfully created. Inspect the response body to see which references succeeded and which failed. - -swagger:response batchReferencesCreateOK -*/ -type BatchReferencesCreateOK struct { - - /* - In: Body - */ - Payload []*models.BatchReferenceResponse `json:"body,omitempty"` -} - -// NewBatchReferencesCreateOK creates BatchReferencesCreateOK with default headers values -func NewBatchReferencesCreateOK() *BatchReferencesCreateOK { - - return &BatchReferencesCreateOK{} -} - -// WithPayload adds the payload to the batch references create o k response -func (o *BatchReferencesCreateOK) WithPayload(payload []*models.BatchReferenceResponse) *BatchReferencesCreateOK { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the batch references create o k response -func (o *BatchReferencesCreateOK) SetPayload(payload []*models.BatchReferenceResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *BatchReferencesCreateOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(200) - payload := o.Payload - if payload == nil { - // return empty array - payload = make([]*models.BatchReferenceResponse, 0, 50) - } - - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } -} - -// BatchReferencesCreateBadRequestCode is the HTTP code returned for type BatchReferencesCreateBadRequest -const BatchReferencesCreateBadRequestCode int = 400 - -/* -BatchReferencesCreateBadRequest Malformed request. - -swagger:response batchReferencesCreateBadRequest -*/ -type BatchReferencesCreateBadRequest struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewBatchReferencesCreateBadRequest creates BatchReferencesCreateBadRequest with default headers values -func NewBatchReferencesCreateBadRequest() *BatchReferencesCreateBadRequest { - - return &BatchReferencesCreateBadRequest{} -} - -// WithPayload adds the payload to the batch references create bad request response -func (o *BatchReferencesCreateBadRequest) WithPayload(payload *models.ErrorResponse) *BatchReferencesCreateBadRequest { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the batch references create bad request response -func (o *BatchReferencesCreateBadRequest) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *BatchReferencesCreateBadRequest) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(400) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// BatchReferencesCreateUnauthorizedCode is the HTTP code returned for type BatchReferencesCreateUnauthorized -const BatchReferencesCreateUnauthorizedCode int = 401 - -/* -BatchReferencesCreateUnauthorized Unauthorized or invalid credentials. - -swagger:response batchReferencesCreateUnauthorized -*/ -type BatchReferencesCreateUnauthorized struct { -} - -// NewBatchReferencesCreateUnauthorized creates BatchReferencesCreateUnauthorized with default headers values -func NewBatchReferencesCreateUnauthorized() *BatchReferencesCreateUnauthorized { - - return &BatchReferencesCreateUnauthorized{} -} - -// WriteResponse to the client -func (o *BatchReferencesCreateUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// BatchReferencesCreateForbiddenCode is the HTTP code returned for type BatchReferencesCreateForbidden -const BatchReferencesCreateForbiddenCode int = 403 - -/* -BatchReferencesCreateForbidden Forbidden - -swagger:response batchReferencesCreateForbidden -*/ -type BatchReferencesCreateForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewBatchReferencesCreateForbidden creates BatchReferencesCreateForbidden with default headers values -func NewBatchReferencesCreateForbidden() *BatchReferencesCreateForbidden { - - return &BatchReferencesCreateForbidden{} -} - -// WithPayload adds the payload to the batch references create forbidden response -func (o *BatchReferencesCreateForbidden) WithPayload(payload *models.ErrorResponse) *BatchReferencesCreateForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the batch references create forbidden response -func (o *BatchReferencesCreateForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *BatchReferencesCreateForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// BatchReferencesCreateUnprocessableEntityCode is the HTTP code returned for type BatchReferencesCreateUnprocessableEntity -const BatchReferencesCreateUnprocessableEntityCode int = 422 - -/* -BatchReferencesCreateUnprocessableEntity Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file? - -swagger:response batchReferencesCreateUnprocessableEntity -*/ -type BatchReferencesCreateUnprocessableEntity struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewBatchReferencesCreateUnprocessableEntity creates BatchReferencesCreateUnprocessableEntity with default headers values -func NewBatchReferencesCreateUnprocessableEntity() *BatchReferencesCreateUnprocessableEntity { - - return &BatchReferencesCreateUnprocessableEntity{} -} - -// WithPayload adds the payload to the batch references create unprocessable entity response -func (o *BatchReferencesCreateUnprocessableEntity) WithPayload(payload *models.ErrorResponse) *BatchReferencesCreateUnprocessableEntity { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the batch references create unprocessable entity response -func (o *BatchReferencesCreateUnprocessableEntity) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *BatchReferencesCreateUnprocessableEntity) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(422) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// BatchReferencesCreateInternalServerErrorCode is the HTTP code returned for type BatchReferencesCreateInternalServerError -const BatchReferencesCreateInternalServerErrorCode int = 500 - -/* -BatchReferencesCreateInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response batchReferencesCreateInternalServerError -*/ -type BatchReferencesCreateInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewBatchReferencesCreateInternalServerError creates BatchReferencesCreateInternalServerError with default headers values -func NewBatchReferencesCreateInternalServerError() *BatchReferencesCreateInternalServerError { - - return &BatchReferencesCreateInternalServerError{} -} - -// WithPayload adds the payload to the batch references create internal server error response -func (o *BatchReferencesCreateInternalServerError) WithPayload(payload *models.ErrorResponse) *BatchReferencesCreateInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the batch references create internal server error response -func (o *BatchReferencesCreateInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *BatchReferencesCreateInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/batch/batch_references_create_urlbuilder.go b/adapters/handlers/rest/operations/batch/batch_references_create_urlbuilder.go deleted file mode 100644 index ab60161e45632a7f3dfaab835161d9a2de35fd10..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/batch/batch_references_create_urlbuilder.go +++ /dev/null @@ -1,114 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package batch - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" -) - -// BatchReferencesCreateURL generates an URL for the batch references create operation -type BatchReferencesCreateURL struct { - ConsistencyLevel *string - - _basePath string - // avoid unkeyed usage - _ struct{} -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *BatchReferencesCreateURL) WithBasePath(bp string) *BatchReferencesCreateURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *BatchReferencesCreateURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *BatchReferencesCreateURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/batch/references" - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - qs := make(url.Values) - - var consistencyLevelQ string - if o.ConsistencyLevel != nil { - consistencyLevelQ = *o.ConsistencyLevel - } - if consistencyLevelQ != "" { - qs.Set("consistency_level", consistencyLevelQ) - } - - _result.RawQuery = qs.Encode() - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *BatchReferencesCreateURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *BatchReferencesCreateURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *BatchReferencesCreateURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on BatchReferencesCreateURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on BatchReferencesCreateURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *BatchReferencesCreateURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/classifications/classifications_get.go b/adapters/handlers/rest/operations/classifications/classifications_get.go deleted file mode 100644 index 5aa9952e8bc2e9bc27ff9ffd188a482f99760f7a..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/classifications/classifications_get.go +++ /dev/null @@ -1,84 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package classifications - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// ClassificationsGetHandlerFunc turns a function with the right signature into a classifications get handler -type ClassificationsGetHandlerFunc func(ClassificationsGetParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn ClassificationsGetHandlerFunc) Handle(params ClassificationsGetParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// ClassificationsGetHandler interface for that can handle valid classifications get params -type ClassificationsGetHandler interface { - Handle(ClassificationsGetParams, *models.Principal) middleware.Responder -} - -// NewClassificationsGet creates a new http.Handler for the classifications get operation -func NewClassificationsGet(ctx *middleware.Context, handler ClassificationsGetHandler) *ClassificationsGet { - return &ClassificationsGet{Context: ctx, Handler: handler} -} - -/* - ClassificationsGet swagger:route GET /classifications/{id} classifications classificationsGet - -# View previously created classification - -Get status, results and metadata of a previously created classification -*/ -type ClassificationsGet struct { - Context *middleware.Context - Handler ClassificationsGetHandler -} - -func (o *ClassificationsGet) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewClassificationsGetParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/classifications/classifications_get_parameters.go b/adapters/handlers/rest/operations/classifications/classifications_get_parameters.go deleted file mode 100644 index 841f068eca6dce4e455a7e794178ab4f3f7f3b60..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/classifications/classifications_get_parameters.go +++ /dev/null @@ -1,82 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package classifications - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" -) - -// NewClassificationsGetParams creates a new ClassificationsGetParams object -// -// There are no default values defined in the spec. -func NewClassificationsGetParams() ClassificationsGetParams { - - return ClassificationsGetParams{} -} - -// ClassificationsGetParams contains all the bound params for the classifications get operation -// typically these are obtained from a http.Request -// -// swagger:parameters classifications.get -type ClassificationsGetParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` - - /*classification id - Required: true - In: path - */ - ID string -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewClassificationsGetParams() beforehand. -func (o *ClassificationsGetParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - rID, rhkID, _ := route.Params.GetOK("id") - if err := o.bindID(rID, rhkID, route.Formats); err != nil { - res = append(res, err) - } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// bindID binds and validates parameter ID from path. -func (o *ClassificationsGetParams) bindID(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - o.ID = raw - - return nil -} diff --git a/adapters/handlers/rest/operations/classifications/classifications_get_responses.go b/adapters/handlers/rest/operations/classifications/classifications_get_responses.go deleted file mode 100644 index bc16b02fe68c3b7e2d54e13f6eb18fc7e9bfcb6d..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/classifications/classifications_get_responses.go +++ /dev/null @@ -1,210 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package classifications - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// ClassificationsGetOKCode is the HTTP code returned for type ClassificationsGetOK -const ClassificationsGetOKCode int = 200 - -/* -ClassificationsGetOK Found the classification, returned as body - -swagger:response classificationsGetOK -*/ -type ClassificationsGetOK struct { - - /* - In: Body - */ - Payload *models.Classification `json:"body,omitempty"` -} - -// NewClassificationsGetOK creates ClassificationsGetOK with default headers values -func NewClassificationsGetOK() *ClassificationsGetOK { - - return &ClassificationsGetOK{} -} - -// WithPayload adds the payload to the classifications get o k response -func (o *ClassificationsGetOK) WithPayload(payload *models.Classification) *ClassificationsGetOK { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the classifications get o k response -func (o *ClassificationsGetOK) SetPayload(payload *models.Classification) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ClassificationsGetOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(200) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ClassificationsGetUnauthorizedCode is the HTTP code returned for type ClassificationsGetUnauthorized -const ClassificationsGetUnauthorizedCode int = 401 - -/* -ClassificationsGetUnauthorized Unauthorized or invalid credentials. - -swagger:response classificationsGetUnauthorized -*/ -type ClassificationsGetUnauthorized struct { -} - -// NewClassificationsGetUnauthorized creates ClassificationsGetUnauthorized with default headers values -func NewClassificationsGetUnauthorized() *ClassificationsGetUnauthorized { - - return &ClassificationsGetUnauthorized{} -} - -// WriteResponse to the client -func (o *ClassificationsGetUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// ClassificationsGetForbiddenCode is the HTTP code returned for type ClassificationsGetForbidden -const ClassificationsGetForbiddenCode int = 403 - -/* -ClassificationsGetForbidden Forbidden - -swagger:response classificationsGetForbidden -*/ -type ClassificationsGetForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewClassificationsGetForbidden creates ClassificationsGetForbidden with default headers values -func NewClassificationsGetForbidden() *ClassificationsGetForbidden { - - return &ClassificationsGetForbidden{} -} - -// WithPayload adds the payload to the classifications get forbidden response -func (o *ClassificationsGetForbidden) WithPayload(payload *models.ErrorResponse) *ClassificationsGetForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the classifications get forbidden response -func (o *ClassificationsGetForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ClassificationsGetForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ClassificationsGetNotFoundCode is the HTTP code returned for type ClassificationsGetNotFound -const ClassificationsGetNotFoundCode int = 404 - -/* -ClassificationsGetNotFound Not Found - Classification does not exist - -swagger:response classificationsGetNotFound -*/ -type ClassificationsGetNotFound struct { -} - -// NewClassificationsGetNotFound creates ClassificationsGetNotFound with default headers values -func NewClassificationsGetNotFound() *ClassificationsGetNotFound { - - return &ClassificationsGetNotFound{} -} - -// WriteResponse to the client -func (o *ClassificationsGetNotFound) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(404) -} - -// ClassificationsGetInternalServerErrorCode is the HTTP code returned for type ClassificationsGetInternalServerError -const ClassificationsGetInternalServerErrorCode int = 500 - -/* -ClassificationsGetInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response classificationsGetInternalServerError -*/ -type ClassificationsGetInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewClassificationsGetInternalServerError creates ClassificationsGetInternalServerError with default headers values -func NewClassificationsGetInternalServerError() *ClassificationsGetInternalServerError { - - return &ClassificationsGetInternalServerError{} -} - -// WithPayload adds the payload to the classifications get internal server error response -func (o *ClassificationsGetInternalServerError) WithPayload(payload *models.ErrorResponse) *ClassificationsGetInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the classifications get internal server error response -func (o *ClassificationsGetInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ClassificationsGetInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/classifications/classifications_get_urlbuilder.go b/adapters/handlers/rest/operations/classifications/classifications_get_urlbuilder.go deleted file mode 100644 index 9239c97ce004f62302b753b30506d5bf0b38b948..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/classifications/classifications_get_urlbuilder.go +++ /dev/null @@ -1,110 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package classifications - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" - "strings" -) - -// ClassificationsGetURL generates an URL for the classifications get operation -type ClassificationsGetURL struct { - ID string - - _basePath string - // avoid unkeyed usage - _ struct{} -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *ClassificationsGetURL) WithBasePath(bp string) *ClassificationsGetURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *ClassificationsGetURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *ClassificationsGetURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/classifications/{id}" - - id := o.ID - if id != "" { - _path = strings.Replace(_path, "{id}", id, -1) - } else { - return nil, errors.New("id is required on ClassificationsGetURL") - } - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *ClassificationsGetURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *ClassificationsGetURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *ClassificationsGetURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on ClassificationsGetURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on ClassificationsGetURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *ClassificationsGetURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/classifications/classifications_post.go b/adapters/handlers/rest/operations/classifications/classifications_post.go deleted file mode 100644 index 06f9b6b67a1707c2ee7bef30e411d9e2849cad75..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/classifications/classifications_post.go +++ /dev/null @@ -1,84 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package classifications - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// ClassificationsPostHandlerFunc turns a function with the right signature into a classifications post handler -type ClassificationsPostHandlerFunc func(ClassificationsPostParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn ClassificationsPostHandlerFunc) Handle(params ClassificationsPostParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// ClassificationsPostHandler interface for that can handle valid classifications post params -type ClassificationsPostHandler interface { - Handle(ClassificationsPostParams, *models.Principal) middleware.Responder -} - -// NewClassificationsPost creates a new http.Handler for the classifications post operation -func NewClassificationsPost(ctx *middleware.Context, handler ClassificationsPostHandler) *ClassificationsPost { - return &ClassificationsPost{Context: ctx, Handler: handler} -} - -/* - ClassificationsPost swagger:route POST /classifications/ classifications classificationsPost - -Starts a classification. - -Trigger a classification based on the specified params. Classifications will run in the background, use GET /classifications/ to retrieve the status of your classification. -*/ -type ClassificationsPost struct { - Context *middleware.Context - Handler ClassificationsPostHandler -} - -func (o *ClassificationsPost) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewClassificationsPostParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/classifications/classifications_post_parameters.go b/adapters/handlers/rest/operations/classifications/classifications_post_parameters.go deleted file mode 100644 index cb7054190e9e7bdb4934f3c64bfeb029c98d66ed..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/classifications/classifications_post_parameters.go +++ /dev/null @@ -1,95 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package classifications - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "io" - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/validate" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewClassificationsPostParams creates a new ClassificationsPostParams object -// -// There are no default values defined in the spec. -func NewClassificationsPostParams() ClassificationsPostParams { - - return ClassificationsPostParams{} -} - -// ClassificationsPostParams contains all the bound params for the classifications post operation -// typically these are obtained from a http.Request -// -// swagger:parameters classifications.post -type ClassificationsPostParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` - - /*parameters to start a classification - Required: true - In: body - */ - Params *models.Classification -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewClassificationsPostParams() beforehand. -func (o *ClassificationsPostParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - if runtime.HasBody(r) { - defer r.Body.Close() - var body models.Classification - if err := route.Consumer.Consume(r.Body, &body); err != nil { - if err == io.EOF { - res = append(res, errors.Required("params", "body", "")) - } else { - res = append(res, errors.NewParseError("params", "body", "", err)) - } - } else { - // validate body object - if err := body.Validate(route.Formats); err != nil { - res = append(res, err) - } - - ctx := validate.WithOperationRequest(r.Context()) - if err := body.ContextValidate(ctx, route.Formats); err != nil { - res = append(res, err) - } - - if len(res) == 0 { - o.Params = &body - } - } - } else { - res = append(res, errors.Required("params", "body", "")) - } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/adapters/handlers/rest/operations/classifications/classifications_post_responses.go b/adapters/handlers/rest/operations/classifications/classifications_post_responses.go deleted file mode 100644 index 858f887b6065e5149056ff11ff8bef897063f128..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/classifications/classifications_post_responses.go +++ /dev/null @@ -1,230 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package classifications - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// ClassificationsPostCreatedCode is the HTTP code returned for type ClassificationsPostCreated -const ClassificationsPostCreatedCode int = 201 - -/* -ClassificationsPostCreated Successfully started classification. - -swagger:response classificationsPostCreated -*/ -type ClassificationsPostCreated struct { - - /* - In: Body - */ - Payload *models.Classification `json:"body,omitempty"` -} - -// NewClassificationsPostCreated creates ClassificationsPostCreated with default headers values -func NewClassificationsPostCreated() *ClassificationsPostCreated { - - return &ClassificationsPostCreated{} -} - -// WithPayload adds the payload to the classifications post created response -func (o *ClassificationsPostCreated) WithPayload(payload *models.Classification) *ClassificationsPostCreated { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the classifications post created response -func (o *ClassificationsPostCreated) SetPayload(payload *models.Classification) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ClassificationsPostCreated) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(201) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ClassificationsPostBadRequestCode is the HTTP code returned for type ClassificationsPostBadRequest -const ClassificationsPostBadRequestCode int = 400 - -/* -ClassificationsPostBadRequest Incorrect request - -swagger:response classificationsPostBadRequest -*/ -type ClassificationsPostBadRequest struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewClassificationsPostBadRequest creates ClassificationsPostBadRequest with default headers values -func NewClassificationsPostBadRequest() *ClassificationsPostBadRequest { - - return &ClassificationsPostBadRequest{} -} - -// WithPayload adds the payload to the classifications post bad request response -func (o *ClassificationsPostBadRequest) WithPayload(payload *models.ErrorResponse) *ClassificationsPostBadRequest { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the classifications post bad request response -func (o *ClassificationsPostBadRequest) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ClassificationsPostBadRequest) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(400) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ClassificationsPostUnauthorizedCode is the HTTP code returned for type ClassificationsPostUnauthorized -const ClassificationsPostUnauthorizedCode int = 401 - -/* -ClassificationsPostUnauthorized Unauthorized or invalid credentials. - -swagger:response classificationsPostUnauthorized -*/ -type ClassificationsPostUnauthorized struct { -} - -// NewClassificationsPostUnauthorized creates ClassificationsPostUnauthorized with default headers values -func NewClassificationsPostUnauthorized() *ClassificationsPostUnauthorized { - - return &ClassificationsPostUnauthorized{} -} - -// WriteResponse to the client -func (o *ClassificationsPostUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// ClassificationsPostForbiddenCode is the HTTP code returned for type ClassificationsPostForbidden -const ClassificationsPostForbiddenCode int = 403 - -/* -ClassificationsPostForbidden Forbidden - -swagger:response classificationsPostForbidden -*/ -type ClassificationsPostForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewClassificationsPostForbidden creates ClassificationsPostForbidden with default headers values -func NewClassificationsPostForbidden() *ClassificationsPostForbidden { - - return &ClassificationsPostForbidden{} -} - -// WithPayload adds the payload to the classifications post forbidden response -func (o *ClassificationsPostForbidden) WithPayload(payload *models.ErrorResponse) *ClassificationsPostForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the classifications post forbidden response -func (o *ClassificationsPostForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ClassificationsPostForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ClassificationsPostInternalServerErrorCode is the HTTP code returned for type ClassificationsPostInternalServerError -const ClassificationsPostInternalServerErrorCode int = 500 - -/* -ClassificationsPostInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response classificationsPostInternalServerError -*/ -type ClassificationsPostInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewClassificationsPostInternalServerError creates ClassificationsPostInternalServerError with default headers values -func NewClassificationsPostInternalServerError() *ClassificationsPostInternalServerError { - - return &ClassificationsPostInternalServerError{} -} - -// WithPayload adds the payload to the classifications post internal server error response -func (o *ClassificationsPostInternalServerError) WithPayload(payload *models.ErrorResponse) *ClassificationsPostInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the classifications post internal server error response -func (o *ClassificationsPostInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ClassificationsPostInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/classifications/classifications_post_urlbuilder.go b/adapters/handlers/rest/operations/classifications/classifications_post_urlbuilder.go deleted file mode 100644 index c021d496fc605123a4635b4bff242983008654e7..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/classifications/classifications_post_urlbuilder.go +++ /dev/null @@ -1,98 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package classifications - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" -) - -// ClassificationsPostURL generates an URL for the classifications post operation -type ClassificationsPostURL struct { - _basePath string -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *ClassificationsPostURL) WithBasePath(bp string) *ClassificationsPostURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *ClassificationsPostURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *ClassificationsPostURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/classifications/" - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *ClassificationsPostURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *ClassificationsPostURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *ClassificationsPostURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on ClassificationsPostURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on ClassificationsPostURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *ClassificationsPostURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/graphql/graphql_batch.go b/adapters/handlers/rest/operations/graphql/graphql_batch.go deleted file mode 100644 index 281088f2d17230131ab9a6188a97cc42240acd31..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/graphql/graphql_batch.go +++ /dev/null @@ -1,84 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package graphql - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// GraphqlBatchHandlerFunc turns a function with the right signature into a graphql batch handler -type GraphqlBatchHandlerFunc func(GraphqlBatchParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn GraphqlBatchHandlerFunc) Handle(params GraphqlBatchParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// GraphqlBatchHandler interface for that can handle valid graphql batch params -type GraphqlBatchHandler interface { - Handle(GraphqlBatchParams, *models.Principal) middleware.Responder -} - -// NewGraphqlBatch creates a new http.Handler for the graphql batch operation -func NewGraphqlBatch(ctx *middleware.Context, handler GraphqlBatchHandler) *GraphqlBatch { - return &GraphqlBatch{Context: ctx, Handler: handler} -} - -/* - GraphqlBatch swagger:route POST /graphql/batch graphql graphqlBatch - -Get a response based on GraphQL. - -Perform a batched GraphQL query -*/ -type GraphqlBatch struct { - Context *middleware.Context - Handler GraphqlBatchHandler -} - -func (o *GraphqlBatch) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewGraphqlBatchParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/graphql/graphql_batch_parameters.go b/adapters/handlers/rest/operations/graphql/graphql_batch_parameters.go deleted file mode 100644 index 3c4307e8cb3fdea5159475b95260eedee70ed47b..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/graphql/graphql_batch_parameters.go +++ /dev/null @@ -1,95 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package graphql - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "io" - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/validate" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewGraphqlBatchParams creates a new GraphqlBatchParams object -// -// There are no default values defined in the spec. -func NewGraphqlBatchParams() GraphqlBatchParams { - - return GraphqlBatchParams{} -} - -// GraphqlBatchParams contains all the bound params for the graphql batch operation -// typically these are obtained from a http.Request -// -// swagger:parameters graphql.batch -type GraphqlBatchParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` - - /*The GraphQL queries. - Required: true - In: body - */ - Body models.GraphQLQueries -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewGraphqlBatchParams() beforehand. -func (o *GraphqlBatchParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - if runtime.HasBody(r) { - defer r.Body.Close() - var body models.GraphQLQueries - if err := route.Consumer.Consume(r.Body, &body); err != nil { - if err == io.EOF { - res = append(res, errors.Required("body", "body", "")) - } else { - res = append(res, errors.NewParseError("body", "body", "", err)) - } - } else { - // validate body object - if err := body.Validate(route.Formats); err != nil { - res = append(res, err) - } - - ctx := validate.WithOperationRequest(r.Context()) - if err := body.ContextValidate(ctx, route.Formats); err != nil { - res = append(res, err) - } - - if len(res) == 0 { - o.Body = body - } - } - } else { - res = append(res, errors.Required("body", "body", "")) - } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/adapters/handlers/rest/operations/graphql/graphql_batch_responses.go b/adapters/handlers/rest/operations/graphql/graphql_batch_responses.go deleted file mode 100644 index 5dff9ccc4c0941ffd369f8c1a950d88ec8f4433a..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/graphql/graphql_batch_responses.go +++ /dev/null @@ -1,233 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package graphql - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// GraphqlBatchOKCode is the HTTP code returned for type GraphqlBatchOK -const GraphqlBatchOKCode int = 200 - -/* -GraphqlBatchOK Successful query (with select). - -swagger:response graphqlBatchOK -*/ -type GraphqlBatchOK struct { - - /* - In: Body - */ - Payload models.GraphQLResponses `json:"body,omitempty"` -} - -// NewGraphqlBatchOK creates GraphqlBatchOK with default headers values -func NewGraphqlBatchOK() *GraphqlBatchOK { - - return &GraphqlBatchOK{} -} - -// WithPayload adds the payload to the graphql batch o k response -func (o *GraphqlBatchOK) WithPayload(payload models.GraphQLResponses) *GraphqlBatchOK { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the graphql batch o k response -func (o *GraphqlBatchOK) SetPayload(payload models.GraphQLResponses) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *GraphqlBatchOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(200) - payload := o.Payload - if payload == nil { - // return empty array - payload = models.GraphQLResponses{} - } - - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } -} - -// GraphqlBatchUnauthorizedCode is the HTTP code returned for type GraphqlBatchUnauthorized -const GraphqlBatchUnauthorizedCode int = 401 - -/* -GraphqlBatchUnauthorized Unauthorized or invalid credentials. - -swagger:response graphqlBatchUnauthorized -*/ -type GraphqlBatchUnauthorized struct { -} - -// NewGraphqlBatchUnauthorized creates GraphqlBatchUnauthorized with default headers values -func NewGraphqlBatchUnauthorized() *GraphqlBatchUnauthorized { - - return &GraphqlBatchUnauthorized{} -} - -// WriteResponse to the client -func (o *GraphqlBatchUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// GraphqlBatchForbiddenCode is the HTTP code returned for type GraphqlBatchForbidden -const GraphqlBatchForbiddenCode int = 403 - -/* -GraphqlBatchForbidden Forbidden - -swagger:response graphqlBatchForbidden -*/ -type GraphqlBatchForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewGraphqlBatchForbidden creates GraphqlBatchForbidden with default headers values -func NewGraphqlBatchForbidden() *GraphqlBatchForbidden { - - return &GraphqlBatchForbidden{} -} - -// WithPayload adds the payload to the graphql batch forbidden response -func (o *GraphqlBatchForbidden) WithPayload(payload *models.ErrorResponse) *GraphqlBatchForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the graphql batch forbidden response -func (o *GraphqlBatchForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *GraphqlBatchForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// GraphqlBatchUnprocessableEntityCode is the HTTP code returned for type GraphqlBatchUnprocessableEntity -const GraphqlBatchUnprocessableEntityCode int = 422 - -/* -GraphqlBatchUnprocessableEntity Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file? - -swagger:response graphqlBatchUnprocessableEntity -*/ -type GraphqlBatchUnprocessableEntity struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewGraphqlBatchUnprocessableEntity creates GraphqlBatchUnprocessableEntity with default headers values -func NewGraphqlBatchUnprocessableEntity() *GraphqlBatchUnprocessableEntity { - - return &GraphqlBatchUnprocessableEntity{} -} - -// WithPayload adds the payload to the graphql batch unprocessable entity response -func (o *GraphqlBatchUnprocessableEntity) WithPayload(payload *models.ErrorResponse) *GraphqlBatchUnprocessableEntity { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the graphql batch unprocessable entity response -func (o *GraphqlBatchUnprocessableEntity) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *GraphqlBatchUnprocessableEntity) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(422) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// GraphqlBatchInternalServerErrorCode is the HTTP code returned for type GraphqlBatchInternalServerError -const GraphqlBatchInternalServerErrorCode int = 500 - -/* -GraphqlBatchInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response graphqlBatchInternalServerError -*/ -type GraphqlBatchInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewGraphqlBatchInternalServerError creates GraphqlBatchInternalServerError with default headers values -func NewGraphqlBatchInternalServerError() *GraphqlBatchInternalServerError { - - return &GraphqlBatchInternalServerError{} -} - -// WithPayload adds the payload to the graphql batch internal server error response -func (o *GraphqlBatchInternalServerError) WithPayload(payload *models.ErrorResponse) *GraphqlBatchInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the graphql batch internal server error response -func (o *GraphqlBatchInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *GraphqlBatchInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/graphql/graphql_batch_urlbuilder.go b/adapters/handlers/rest/operations/graphql/graphql_batch_urlbuilder.go deleted file mode 100644 index 2304571de55f441a139ae62cd804283910fdfc82..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/graphql/graphql_batch_urlbuilder.go +++ /dev/null @@ -1,98 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package graphql - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" -) - -// GraphqlBatchURL generates an URL for the graphql batch operation -type GraphqlBatchURL struct { - _basePath string -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *GraphqlBatchURL) WithBasePath(bp string) *GraphqlBatchURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *GraphqlBatchURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *GraphqlBatchURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/graphql/batch" - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *GraphqlBatchURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *GraphqlBatchURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *GraphqlBatchURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on GraphqlBatchURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on GraphqlBatchURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *GraphqlBatchURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/graphql/graphql_post.go b/adapters/handlers/rest/operations/graphql/graphql_post.go deleted file mode 100644 index 998dbacb1c1dadbec260b08a2f99217b55083555..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/graphql/graphql_post.go +++ /dev/null @@ -1,84 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package graphql - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// GraphqlPostHandlerFunc turns a function with the right signature into a graphql post handler -type GraphqlPostHandlerFunc func(GraphqlPostParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn GraphqlPostHandlerFunc) Handle(params GraphqlPostParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// GraphqlPostHandler interface for that can handle valid graphql post params -type GraphqlPostHandler interface { - Handle(GraphqlPostParams, *models.Principal) middleware.Responder -} - -// NewGraphqlPost creates a new http.Handler for the graphql post operation -func NewGraphqlPost(ctx *middleware.Context, handler GraphqlPostHandler) *GraphqlPost { - return &GraphqlPost{Context: ctx, Handler: handler} -} - -/* - GraphqlPost swagger:route POST /graphql graphql graphqlPost - -# Get a response based on GraphQL - -Get an object based on GraphQL -*/ -type GraphqlPost struct { - Context *middleware.Context - Handler GraphqlPostHandler -} - -func (o *GraphqlPost) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewGraphqlPostParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/graphql/graphql_post_parameters.go b/adapters/handlers/rest/operations/graphql/graphql_post_parameters.go deleted file mode 100644 index 2a3a2fb5076f895b177ff991b72fcac53940e9dd..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/graphql/graphql_post_parameters.go +++ /dev/null @@ -1,95 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package graphql - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "io" - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/validate" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewGraphqlPostParams creates a new GraphqlPostParams object -// -// There are no default values defined in the spec. -func NewGraphqlPostParams() GraphqlPostParams { - - return GraphqlPostParams{} -} - -// GraphqlPostParams contains all the bound params for the graphql post operation -// typically these are obtained from a http.Request -// -// swagger:parameters graphql.post -type GraphqlPostParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` - - /*The GraphQL query request parameters. - Required: true - In: body - */ - Body *models.GraphQLQuery -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewGraphqlPostParams() beforehand. -func (o *GraphqlPostParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - if runtime.HasBody(r) { - defer r.Body.Close() - var body models.GraphQLQuery - if err := route.Consumer.Consume(r.Body, &body); err != nil { - if err == io.EOF { - res = append(res, errors.Required("body", "body", "")) - } else { - res = append(res, errors.NewParseError("body", "body", "", err)) - } - } else { - // validate body object - if err := body.Validate(route.Formats); err != nil { - res = append(res, err) - } - - ctx := validate.WithOperationRequest(r.Context()) - if err := body.ContextValidate(ctx, route.Formats); err != nil { - res = append(res, err) - } - - if len(res) == 0 { - o.Body = &body - } - } - } else { - res = append(res, errors.Required("body", "body", "")) - } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/adapters/handlers/rest/operations/graphql/graphql_post_responses.go b/adapters/handlers/rest/operations/graphql/graphql_post_responses.go deleted file mode 100644 index 94a039a7466227cfb55c3fd9b6491fb2ca666d45..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/graphql/graphql_post_responses.go +++ /dev/null @@ -1,230 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package graphql - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// GraphqlPostOKCode is the HTTP code returned for type GraphqlPostOK -const GraphqlPostOKCode int = 200 - -/* -GraphqlPostOK Successful query (with select). - -swagger:response graphqlPostOK -*/ -type GraphqlPostOK struct { - - /* - In: Body - */ - Payload *models.GraphQLResponse `json:"body,omitempty"` -} - -// NewGraphqlPostOK creates GraphqlPostOK with default headers values -func NewGraphqlPostOK() *GraphqlPostOK { - - return &GraphqlPostOK{} -} - -// WithPayload adds the payload to the graphql post o k response -func (o *GraphqlPostOK) WithPayload(payload *models.GraphQLResponse) *GraphqlPostOK { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the graphql post o k response -func (o *GraphqlPostOK) SetPayload(payload *models.GraphQLResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *GraphqlPostOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(200) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// GraphqlPostUnauthorizedCode is the HTTP code returned for type GraphqlPostUnauthorized -const GraphqlPostUnauthorizedCode int = 401 - -/* -GraphqlPostUnauthorized Unauthorized or invalid credentials. - -swagger:response graphqlPostUnauthorized -*/ -type GraphqlPostUnauthorized struct { -} - -// NewGraphqlPostUnauthorized creates GraphqlPostUnauthorized with default headers values -func NewGraphqlPostUnauthorized() *GraphqlPostUnauthorized { - - return &GraphqlPostUnauthorized{} -} - -// WriteResponse to the client -func (o *GraphqlPostUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// GraphqlPostForbiddenCode is the HTTP code returned for type GraphqlPostForbidden -const GraphqlPostForbiddenCode int = 403 - -/* -GraphqlPostForbidden Forbidden - -swagger:response graphqlPostForbidden -*/ -type GraphqlPostForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewGraphqlPostForbidden creates GraphqlPostForbidden with default headers values -func NewGraphqlPostForbidden() *GraphqlPostForbidden { - - return &GraphqlPostForbidden{} -} - -// WithPayload adds the payload to the graphql post forbidden response -func (o *GraphqlPostForbidden) WithPayload(payload *models.ErrorResponse) *GraphqlPostForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the graphql post forbidden response -func (o *GraphqlPostForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *GraphqlPostForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// GraphqlPostUnprocessableEntityCode is the HTTP code returned for type GraphqlPostUnprocessableEntity -const GraphqlPostUnprocessableEntityCode int = 422 - -/* -GraphqlPostUnprocessableEntity Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file? - -swagger:response graphqlPostUnprocessableEntity -*/ -type GraphqlPostUnprocessableEntity struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewGraphqlPostUnprocessableEntity creates GraphqlPostUnprocessableEntity with default headers values -func NewGraphqlPostUnprocessableEntity() *GraphqlPostUnprocessableEntity { - - return &GraphqlPostUnprocessableEntity{} -} - -// WithPayload adds the payload to the graphql post unprocessable entity response -func (o *GraphqlPostUnprocessableEntity) WithPayload(payload *models.ErrorResponse) *GraphqlPostUnprocessableEntity { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the graphql post unprocessable entity response -func (o *GraphqlPostUnprocessableEntity) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *GraphqlPostUnprocessableEntity) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(422) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// GraphqlPostInternalServerErrorCode is the HTTP code returned for type GraphqlPostInternalServerError -const GraphqlPostInternalServerErrorCode int = 500 - -/* -GraphqlPostInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response graphqlPostInternalServerError -*/ -type GraphqlPostInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewGraphqlPostInternalServerError creates GraphqlPostInternalServerError with default headers values -func NewGraphqlPostInternalServerError() *GraphqlPostInternalServerError { - - return &GraphqlPostInternalServerError{} -} - -// WithPayload adds the payload to the graphql post internal server error response -func (o *GraphqlPostInternalServerError) WithPayload(payload *models.ErrorResponse) *GraphqlPostInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the graphql post internal server error response -func (o *GraphqlPostInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *GraphqlPostInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/graphql/graphql_post_urlbuilder.go b/adapters/handlers/rest/operations/graphql/graphql_post_urlbuilder.go deleted file mode 100644 index 2030045771aa05419659e7385811752cf9dcb0a5..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/graphql/graphql_post_urlbuilder.go +++ /dev/null @@ -1,98 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package graphql - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" -) - -// GraphqlPostURL generates an URL for the graphql post operation -type GraphqlPostURL struct { - _basePath string -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *GraphqlPostURL) WithBasePath(bp string) *GraphqlPostURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *GraphqlPostURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *GraphqlPostURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/graphql" - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *GraphqlPostURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *GraphqlPostURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *GraphqlPostURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on GraphqlPostURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on GraphqlPostURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *GraphqlPostURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/meta/meta_get.go b/adapters/handlers/rest/operations/meta/meta_get.go deleted file mode 100644 index e0db78688afef4d32ca2d86e176373c3d2fedd7e..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/meta/meta_get.go +++ /dev/null @@ -1,84 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package meta - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// MetaGetHandlerFunc turns a function with the right signature into a meta get handler -type MetaGetHandlerFunc func(MetaGetParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn MetaGetHandlerFunc) Handle(params MetaGetParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// MetaGetHandler interface for that can handle valid meta get params -type MetaGetHandler interface { - Handle(MetaGetParams, *models.Principal) middleware.Responder -} - -// NewMetaGet creates a new http.Handler for the meta get operation -func NewMetaGet(ctx *middleware.Context, handler MetaGetHandler) *MetaGet { - return &MetaGet{Context: ctx, Handler: handler} -} - -/* - MetaGet swagger:route GET /meta meta metaGet - -Returns meta information of the current Weaviate instance. - -Gives meta information about the server and can be used to provide information to another Weaviate instance that wants to interact with the current instance. -*/ -type MetaGet struct { - Context *middleware.Context - Handler MetaGetHandler -} - -func (o *MetaGet) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewMetaGetParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/meta/meta_get_parameters.go b/adapters/handlers/rest/operations/meta/meta_get_parameters.go deleted file mode 100644 index 8f04a62667a9bbef5a8e0349c6e8c0d9534629ec..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/meta/meta_get_parameters.go +++ /dev/null @@ -1,57 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package meta - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime/middleware" -) - -// NewMetaGetParams creates a new MetaGetParams object -// -// There are no default values defined in the spec. -func NewMetaGetParams() MetaGetParams { - - return MetaGetParams{} -} - -// MetaGetParams contains all the bound params for the meta get operation -// typically these are obtained from a http.Request -// -// swagger:parameters meta.get -type MetaGetParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewMetaGetParams() beforehand. -func (o *MetaGetParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/adapters/handlers/rest/operations/meta/meta_get_responses.go b/adapters/handlers/rest/operations/meta/meta_get_responses.go deleted file mode 100644 index 09333fa062cf3686daf277755a7af337b4ff74a8..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/meta/meta_get_responses.go +++ /dev/null @@ -1,185 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package meta - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// MetaGetOKCode is the HTTP code returned for type MetaGetOK -const MetaGetOKCode int = 200 - -/* -MetaGetOK Successful response. - -swagger:response metaGetOK -*/ -type MetaGetOK struct { - - /* - In: Body - */ - Payload *models.Meta `json:"body,omitempty"` -} - -// NewMetaGetOK creates MetaGetOK with default headers values -func NewMetaGetOK() *MetaGetOK { - - return &MetaGetOK{} -} - -// WithPayload adds the payload to the meta get o k response -func (o *MetaGetOK) WithPayload(payload *models.Meta) *MetaGetOK { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the meta get o k response -func (o *MetaGetOK) SetPayload(payload *models.Meta) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *MetaGetOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(200) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// MetaGetUnauthorizedCode is the HTTP code returned for type MetaGetUnauthorized -const MetaGetUnauthorizedCode int = 401 - -/* -MetaGetUnauthorized Unauthorized or invalid credentials. - -swagger:response metaGetUnauthorized -*/ -type MetaGetUnauthorized struct { -} - -// NewMetaGetUnauthorized creates MetaGetUnauthorized with default headers values -func NewMetaGetUnauthorized() *MetaGetUnauthorized { - - return &MetaGetUnauthorized{} -} - -// WriteResponse to the client -func (o *MetaGetUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// MetaGetForbiddenCode is the HTTP code returned for type MetaGetForbidden -const MetaGetForbiddenCode int = 403 - -/* -MetaGetForbidden Forbidden - -swagger:response metaGetForbidden -*/ -type MetaGetForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewMetaGetForbidden creates MetaGetForbidden with default headers values -func NewMetaGetForbidden() *MetaGetForbidden { - - return &MetaGetForbidden{} -} - -// WithPayload adds the payload to the meta get forbidden response -func (o *MetaGetForbidden) WithPayload(payload *models.ErrorResponse) *MetaGetForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the meta get forbidden response -func (o *MetaGetForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *MetaGetForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// MetaGetInternalServerErrorCode is the HTTP code returned for type MetaGetInternalServerError -const MetaGetInternalServerErrorCode int = 500 - -/* -MetaGetInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response metaGetInternalServerError -*/ -type MetaGetInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewMetaGetInternalServerError creates MetaGetInternalServerError with default headers values -func NewMetaGetInternalServerError() *MetaGetInternalServerError { - - return &MetaGetInternalServerError{} -} - -// WithPayload adds the payload to the meta get internal server error response -func (o *MetaGetInternalServerError) WithPayload(payload *models.ErrorResponse) *MetaGetInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the meta get internal server error response -func (o *MetaGetInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *MetaGetInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/meta/meta_get_urlbuilder.go b/adapters/handlers/rest/operations/meta/meta_get_urlbuilder.go deleted file mode 100644 index af6261433685e5253cf2597a35f802be0bfcfe0e..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/meta/meta_get_urlbuilder.go +++ /dev/null @@ -1,98 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package meta - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" -) - -// MetaGetURL generates an URL for the meta get operation -type MetaGetURL struct { - _basePath string -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *MetaGetURL) WithBasePath(bp string) *MetaGetURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *MetaGetURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *MetaGetURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/meta" - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *MetaGetURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *MetaGetURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *MetaGetURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on MetaGetURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on MetaGetURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *MetaGetURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/nodes/nodes_get.go b/adapters/handlers/rest/operations/nodes/nodes_get.go deleted file mode 100644 index eb6a0cf6a215d065e6f2f146462947ea5c4abf0f..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/nodes/nodes_get.go +++ /dev/null @@ -1,82 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package nodes - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// NodesGetHandlerFunc turns a function with the right signature into a nodes get handler -type NodesGetHandlerFunc func(NodesGetParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn NodesGetHandlerFunc) Handle(params NodesGetParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// NodesGetHandler interface for that can handle valid nodes get params -type NodesGetHandler interface { - Handle(NodesGetParams, *models.Principal) middleware.Responder -} - -// NewNodesGet creates a new http.Handler for the nodes get operation -func NewNodesGet(ctx *middleware.Context, handler NodesGetHandler) *NodesGet { - return &NodesGet{Context: ctx, Handler: handler} -} - -/* - NodesGet swagger:route GET /nodes nodes nodesGet - -Returns status of Weaviate DB. -*/ -type NodesGet struct { - Context *middleware.Context - Handler NodesGetHandler -} - -func (o *NodesGet) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewNodesGetParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/nodes/nodes_get_class.go b/adapters/handlers/rest/operations/nodes/nodes_get_class.go deleted file mode 100644 index 4d3a37d0c4f0a6b6276295588219402b4855c7d0..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/nodes/nodes_get_class.go +++ /dev/null @@ -1,82 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package nodes - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// NodesGetClassHandlerFunc turns a function with the right signature into a nodes get class handler -type NodesGetClassHandlerFunc func(NodesGetClassParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn NodesGetClassHandlerFunc) Handle(params NodesGetClassParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// NodesGetClassHandler interface for that can handle valid nodes get class params -type NodesGetClassHandler interface { - Handle(NodesGetClassParams, *models.Principal) middleware.Responder -} - -// NewNodesGetClass creates a new http.Handler for the nodes get class operation -func NewNodesGetClass(ctx *middleware.Context, handler NodesGetClassHandler) *NodesGetClass { - return &NodesGetClass{Context: ctx, Handler: handler} -} - -/* - NodesGetClass swagger:route GET /nodes/{className} nodes nodesGetClass - -Returns status of Weaviate DB. -*/ -type NodesGetClass struct { - Context *middleware.Context - Handler NodesGetClassHandler -} - -func (o *NodesGetClass) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewNodesGetClassParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/nodes/nodes_get_class_parameters.go b/adapters/handlers/rest/operations/nodes/nodes_get_class_parameters.go deleted file mode 100644 index 209d37b9837bb2ec389853ba5c28c6ef21639327..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/nodes/nodes_get_class_parameters.go +++ /dev/null @@ -1,121 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package nodes - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" -) - -// NewNodesGetClassParams creates a new NodesGetClassParams object -// with the default values initialized. -func NewNodesGetClassParams() NodesGetClassParams { - - var ( - // initialize parameters with default values - - outputDefault = string("minimal") - ) - - return NodesGetClassParams{ - Output: &outputDefault, - } -} - -// NodesGetClassParams contains all the bound params for the nodes get class operation -// typically these are obtained from a http.Request -// -// swagger:parameters nodes.get.class -type NodesGetClassParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` - - /* - Required: true - In: path - */ - ClassName string - /*Controls the verbosity of the output, possible values are: "minimal", "verbose". Defaults to "minimal". - In: query - Default: "minimal" - */ - Output *string -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewNodesGetClassParams() beforehand. -func (o *NodesGetClassParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - qs := runtime.Values(r.URL.Query()) - - rClassName, rhkClassName, _ := route.Params.GetOK("className") - if err := o.bindClassName(rClassName, rhkClassName, route.Formats); err != nil { - res = append(res, err) - } - - qOutput, qhkOutput, _ := qs.GetOK("output") - if err := o.bindOutput(qOutput, qhkOutput, route.Formats); err != nil { - res = append(res, err) - } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// bindClassName binds and validates parameter ClassName from path. -func (o *NodesGetClassParams) bindClassName(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - o.ClassName = raw - - return nil -} - -// bindOutput binds and validates parameter Output from query. -func (o *NodesGetClassParams) bindOutput(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: false - // AllowEmptyValue: false - - if raw == "" { // empty values pass all other validations - // Default values have been previously initialized by NewNodesGetClassParams() - return nil - } - o.Output = &raw - - return nil -} diff --git a/adapters/handlers/rest/operations/nodes/nodes_get_class_responses.go b/adapters/handlers/rest/operations/nodes/nodes_get_class_responses.go deleted file mode 100644 index 0213178bb08e421ce3a6e117768ed72de665c094..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/nodes/nodes_get_class_responses.go +++ /dev/null @@ -1,275 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package nodes - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// NodesGetClassOKCode is the HTTP code returned for type NodesGetClassOK -const NodesGetClassOKCode int = 200 - -/* -NodesGetClassOK Nodes status successfully returned - -swagger:response nodesGetClassOK -*/ -type NodesGetClassOK struct { - - /* - In: Body - */ - Payload *models.NodesStatusResponse `json:"body,omitempty"` -} - -// NewNodesGetClassOK creates NodesGetClassOK with default headers values -func NewNodesGetClassOK() *NodesGetClassOK { - - return &NodesGetClassOK{} -} - -// WithPayload adds the payload to the nodes get class o k response -func (o *NodesGetClassOK) WithPayload(payload *models.NodesStatusResponse) *NodesGetClassOK { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the nodes get class o k response -func (o *NodesGetClassOK) SetPayload(payload *models.NodesStatusResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *NodesGetClassOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(200) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// NodesGetClassUnauthorizedCode is the HTTP code returned for type NodesGetClassUnauthorized -const NodesGetClassUnauthorizedCode int = 401 - -/* -NodesGetClassUnauthorized Unauthorized or invalid credentials. - -swagger:response nodesGetClassUnauthorized -*/ -type NodesGetClassUnauthorized struct { -} - -// NewNodesGetClassUnauthorized creates NodesGetClassUnauthorized with default headers values -func NewNodesGetClassUnauthorized() *NodesGetClassUnauthorized { - - return &NodesGetClassUnauthorized{} -} - -// WriteResponse to the client -func (o *NodesGetClassUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// NodesGetClassForbiddenCode is the HTTP code returned for type NodesGetClassForbidden -const NodesGetClassForbiddenCode int = 403 - -/* -NodesGetClassForbidden Forbidden - -swagger:response nodesGetClassForbidden -*/ -type NodesGetClassForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewNodesGetClassForbidden creates NodesGetClassForbidden with default headers values -func NewNodesGetClassForbidden() *NodesGetClassForbidden { - - return &NodesGetClassForbidden{} -} - -// WithPayload adds the payload to the nodes get class forbidden response -func (o *NodesGetClassForbidden) WithPayload(payload *models.ErrorResponse) *NodesGetClassForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the nodes get class forbidden response -func (o *NodesGetClassForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *NodesGetClassForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// NodesGetClassNotFoundCode is the HTTP code returned for type NodesGetClassNotFound -const NodesGetClassNotFoundCode int = 404 - -/* -NodesGetClassNotFound Not Found - Backup does not exist - -swagger:response nodesGetClassNotFound -*/ -type NodesGetClassNotFound struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewNodesGetClassNotFound creates NodesGetClassNotFound with default headers values -func NewNodesGetClassNotFound() *NodesGetClassNotFound { - - return &NodesGetClassNotFound{} -} - -// WithPayload adds the payload to the nodes get class not found response -func (o *NodesGetClassNotFound) WithPayload(payload *models.ErrorResponse) *NodesGetClassNotFound { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the nodes get class not found response -func (o *NodesGetClassNotFound) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *NodesGetClassNotFound) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(404) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// NodesGetClassUnprocessableEntityCode is the HTTP code returned for type NodesGetClassUnprocessableEntity -const NodesGetClassUnprocessableEntityCode int = 422 - -/* -NodesGetClassUnprocessableEntity Invalid backup restoration status attempt. - -swagger:response nodesGetClassUnprocessableEntity -*/ -type NodesGetClassUnprocessableEntity struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewNodesGetClassUnprocessableEntity creates NodesGetClassUnprocessableEntity with default headers values -func NewNodesGetClassUnprocessableEntity() *NodesGetClassUnprocessableEntity { - - return &NodesGetClassUnprocessableEntity{} -} - -// WithPayload adds the payload to the nodes get class unprocessable entity response -func (o *NodesGetClassUnprocessableEntity) WithPayload(payload *models.ErrorResponse) *NodesGetClassUnprocessableEntity { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the nodes get class unprocessable entity response -func (o *NodesGetClassUnprocessableEntity) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *NodesGetClassUnprocessableEntity) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(422) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// NodesGetClassInternalServerErrorCode is the HTTP code returned for type NodesGetClassInternalServerError -const NodesGetClassInternalServerErrorCode int = 500 - -/* -NodesGetClassInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response nodesGetClassInternalServerError -*/ -type NodesGetClassInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewNodesGetClassInternalServerError creates NodesGetClassInternalServerError with default headers values -func NewNodesGetClassInternalServerError() *NodesGetClassInternalServerError { - - return &NodesGetClassInternalServerError{} -} - -// WithPayload adds the payload to the nodes get class internal server error response -func (o *NodesGetClassInternalServerError) WithPayload(payload *models.ErrorResponse) *NodesGetClassInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the nodes get class internal server error response -func (o *NodesGetClassInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *NodesGetClassInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/nodes/nodes_get_class_urlbuilder.go b/adapters/handlers/rest/operations/nodes/nodes_get_class_urlbuilder.go deleted file mode 100644 index 2d3bfcf115cb92030f61ab23c3c6ce89fb0774b2..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/nodes/nodes_get_class_urlbuilder.go +++ /dev/null @@ -1,124 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package nodes - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" - "strings" -) - -// NodesGetClassURL generates an URL for the nodes get class operation -type NodesGetClassURL struct { - ClassName string - - Output *string - - _basePath string - // avoid unkeyed usage - _ struct{} -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *NodesGetClassURL) WithBasePath(bp string) *NodesGetClassURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *NodesGetClassURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *NodesGetClassURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/nodes/{className}" - - className := o.ClassName - if className != "" { - _path = strings.Replace(_path, "{className}", className, -1) - } else { - return nil, errors.New("className is required on NodesGetClassURL") - } - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - qs := make(url.Values) - - var outputQ string - if o.Output != nil { - outputQ = *o.Output - } - if outputQ != "" { - qs.Set("output", outputQ) - } - - _result.RawQuery = qs.Encode() - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *NodesGetClassURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *NodesGetClassURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *NodesGetClassURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on NodesGetClassURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on NodesGetClassURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *NodesGetClassURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/nodes/nodes_get_parameters.go b/adapters/handlers/rest/operations/nodes/nodes_get_parameters.go deleted file mode 100644 index 5161d92be4b07db1d7e914f0f3dc76d9fed0be0b..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/nodes/nodes_get_parameters.go +++ /dev/null @@ -1,97 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package nodes - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" -) - -// NewNodesGetParams creates a new NodesGetParams object -// with the default values initialized. -func NewNodesGetParams() NodesGetParams { - - var ( - // initialize parameters with default values - - outputDefault = string("minimal") - ) - - return NodesGetParams{ - Output: &outputDefault, - } -} - -// NodesGetParams contains all the bound params for the nodes get operation -// typically these are obtained from a http.Request -// -// swagger:parameters nodes.get -type NodesGetParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` - - /*Controls the verbosity of the output, possible values are: "minimal", "verbose". Defaults to "minimal". - In: query - Default: "minimal" - */ - Output *string -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewNodesGetParams() beforehand. -func (o *NodesGetParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - qs := runtime.Values(r.URL.Query()) - - qOutput, qhkOutput, _ := qs.GetOK("output") - if err := o.bindOutput(qOutput, qhkOutput, route.Formats); err != nil { - res = append(res, err) - } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// bindOutput binds and validates parameter Output from query. -func (o *NodesGetParams) bindOutput(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: false - // AllowEmptyValue: false - - if raw == "" { // empty values pass all other validations - // Default values have been previously initialized by NewNodesGetParams() - return nil - } - o.Output = &raw - - return nil -} diff --git a/adapters/handlers/rest/operations/nodes/nodes_get_responses.go b/adapters/handlers/rest/operations/nodes/nodes_get_responses.go deleted file mode 100644 index 0f8377120b7b6feb8ae40066fd030822b81f6389..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/nodes/nodes_get_responses.go +++ /dev/null @@ -1,275 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package nodes - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// NodesGetOKCode is the HTTP code returned for type NodesGetOK -const NodesGetOKCode int = 200 - -/* -NodesGetOK Nodes status successfully returned - -swagger:response nodesGetOK -*/ -type NodesGetOK struct { - - /* - In: Body - */ - Payload *models.NodesStatusResponse `json:"body,omitempty"` -} - -// NewNodesGetOK creates NodesGetOK with default headers values -func NewNodesGetOK() *NodesGetOK { - - return &NodesGetOK{} -} - -// WithPayload adds the payload to the nodes get o k response -func (o *NodesGetOK) WithPayload(payload *models.NodesStatusResponse) *NodesGetOK { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the nodes get o k response -func (o *NodesGetOK) SetPayload(payload *models.NodesStatusResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *NodesGetOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(200) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// NodesGetUnauthorizedCode is the HTTP code returned for type NodesGetUnauthorized -const NodesGetUnauthorizedCode int = 401 - -/* -NodesGetUnauthorized Unauthorized or invalid credentials. - -swagger:response nodesGetUnauthorized -*/ -type NodesGetUnauthorized struct { -} - -// NewNodesGetUnauthorized creates NodesGetUnauthorized with default headers values -func NewNodesGetUnauthorized() *NodesGetUnauthorized { - - return &NodesGetUnauthorized{} -} - -// WriteResponse to the client -func (o *NodesGetUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// NodesGetForbiddenCode is the HTTP code returned for type NodesGetForbidden -const NodesGetForbiddenCode int = 403 - -/* -NodesGetForbidden Forbidden - -swagger:response nodesGetForbidden -*/ -type NodesGetForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewNodesGetForbidden creates NodesGetForbidden with default headers values -func NewNodesGetForbidden() *NodesGetForbidden { - - return &NodesGetForbidden{} -} - -// WithPayload adds the payload to the nodes get forbidden response -func (o *NodesGetForbidden) WithPayload(payload *models.ErrorResponse) *NodesGetForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the nodes get forbidden response -func (o *NodesGetForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *NodesGetForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// NodesGetNotFoundCode is the HTTP code returned for type NodesGetNotFound -const NodesGetNotFoundCode int = 404 - -/* -NodesGetNotFound Not Found - Backup does not exist - -swagger:response nodesGetNotFound -*/ -type NodesGetNotFound struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewNodesGetNotFound creates NodesGetNotFound with default headers values -func NewNodesGetNotFound() *NodesGetNotFound { - - return &NodesGetNotFound{} -} - -// WithPayload adds the payload to the nodes get not found response -func (o *NodesGetNotFound) WithPayload(payload *models.ErrorResponse) *NodesGetNotFound { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the nodes get not found response -func (o *NodesGetNotFound) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *NodesGetNotFound) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(404) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// NodesGetUnprocessableEntityCode is the HTTP code returned for type NodesGetUnprocessableEntity -const NodesGetUnprocessableEntityCode int = 422 - -/* -NodesGetUnprocessableEntity Invalid backup restoration status attempt. - -swagger:response nodesGetUnprocessableEntity -*/ -type NodesGetUnprocessableEntity struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewNodesGetUnprocessableEntity creates NodesGetUnprocessableEntity with default headers values -func NewNodesGetUnprocessableEntity() *NodesGetUnprocessableEntity { - - return &NodesGetUnprocessableEntity{} -} - -// WithPayload adds the payload to the nodes get unprocessable entity response -func (o *NodesGetUnprocessableEntity) WithPayload(payload *models.ErrorResponse) *NodesGetUnprocessableEntity { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the nodes get unprocessable entity response -func (o *NodesGetUnprocessableEntity) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *NodesGetUnprocessableEntity) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(422) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// NodesGetInternalServerErrorCode is the HTTP code returned for type NodesGetInternalServerError -const NodesGetInternalServerErrorCode int = 500 - -/* -NodesGetInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response nodesGetInternalServerError -*/ -type NodesGetInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewNodesGetInternalServerError creates NodesGetInternalServerError with default headers values -func NewNodesGetInternalServerError() *NodesGetInternalServerError { - - return &NodesGetInternalServerError{} -} - -// WithPayload adds the payload to the nodes get internal server error response -func (o *NodesGetInternalServerError) WithPayload(payload *models.ErrorResponse) *NodesGetInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the nodes get internal server error response -func (o *NodesGetInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *NodesGetInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/nodes/nodes_get_urlbuilder.go b/adapters/handlers/rest/operations/nodes/nodes_get_urlbuilder.go deleted file mode 100644 index 07a591482e706ce29b1f9fba3aa47a2487110473..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/nodes/nodes_get_urlbuilder.go +++ /dev/null @@ -1,114 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package nodes - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" -) - -// NodesGetURL generates an URL for the nodes get operation -type NodesGetURL struct { - Output *string - - _basePath string - // avoid unkeyed usage - _ struct{} -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *NodesGetURL) WithBasePath(bp string) *NodesGetURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *NodesGetURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *NodesGetURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/nodes" - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - qs := make(url.Values) - - var outputQ string - if o.Output != nil { - outputQ = *o.Output - } - if outputQ != "" { - qs.Set("output", outputQ) - } - - _result.RawQuery = qs.Encode() - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *NodesGetURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *NodesGetURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *NodesGetURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on NodesGetURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on NodesGetURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *NodesGetURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/objects/objects_class_delete.go b/adapters/handlers/rest/operations/objects/objects_class_delete.go deleted file mode 100644 index 7128be4cd6c80df76b4ee1b83a6d3d015ab2655b..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_class_delete.go +++ /dev/null @@ -1,84 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsClassDeleteHandlerFunc turns a function with the right signature into a objects class delete handler -type ObjectsClassDeleteHandlerFunc func(ObjectsClassDeleteParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn ObjectsClassDeleteHandlerFunc) Handle(params ObjectsClassDeleteParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// ObjectsClassDeleteHandler interface for that can handle valid objects class delete params -type ObjectsClassDeleteHandler interface { - Handle(ObjectsClassDeleteParams, *models.Principal) middleware.Responder -} - -// NewObjectsClassDelete creates a new http.Handler for the objects class delete operation -func NewObjectsClassDelete(ctx *middleware.Context, handler ObjectsClassDeleteHandler) *ObjectsClassDelete { - return &ObjectsClassDelete{Context: ctx, Handler: handler} -} - -/* - ObjectsClassDelete swagger:route DELETE /objects/{className}/{id} objects objectsClassDelete - -Delete object based on its class and UUID. - -Delete a single data object. -*/ -type ObjectsClassDelete struct { - Context *middleware.Context - Handler ObjectsClassDeleteHandler -} - -func (o *ObjectsClassDelete) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewObjectsClassDeleteParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/objects/objects_class_delete_parameters.go b/adapters/handlers/rest/operations/objects/objects_class_delete_parameters.go deleted file mode 100644 index 5e6075ab4701969460b52133b078160c5a80536a..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_class_delete_parameters.go +++ /dev/null @@ -1,183 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/validate" -) - -// NewObjectsClassDeleteParams creates a new ObjectsClassDeleteParams object -// -// There are no default values defined in the spec. -func NewObjectsClassDeleteParams() ObjectsClassDeleteParams { - - return ObjectsClassDeleteParams{} -} - -// ObjectsClassDeleteParams contains all the bound params for the objects class delete operation -// typically these are obtained from a http.Request -// -// swagger:parameters objects.class.delete -type ObjectsClassDeleteParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` - - /* - Required: true - In: path - */ - ClassName string - /*Determines how many replicas must acknowledge a request before it is considered successful - In: query - */ - ConsistencyLevel *string - /*Unique ID of the Object. - Required: true - In: path - */ - ID strfmt.UUID - /*Specifies the tenant in a request targeting a multi-tenant class - In: query - */ - Tenant *string -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewObjectsClassDeleteParams() beforehand. -func (o *ObjectsClassDeleteParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - qs := runtime.Values(r.URL.Query()) - - rClassName, rhkClassName, _ := route.Params.GetOK("className") - if err := o.bindClassName(rClassName, rhkClassName, route.Formats); err != nil { - res = append(res, err) - } - - qConsistencyLevel, qhkConsistencyLevel, _ := qs.GetOK("consistency_level") - if err := o.bindConsistencyLevel(qConsistencyLevel, qhkConsistencyLevel, route.Formats); err != nil { - res = append(res, err) - } - - rID, rhkID, _ := route.Params.GetOK("id") - if err := o.bindID(rID, rhkID, route.Formats); err != nil { - res = append(res, err) - } - - qTenant, qhkTenant, _ := qs.GetOK("tenant") - if err := o.bindTenant(qTenant, qhkTenant, route.Formats); err != nil { - res = append(res, err) - } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// bindClassName binds and validates parameter ClassName from path. -func (o *ObjectsClassDeleteParams) bindClassName(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - o.ClassName = raw - - return nil -} - -// bindConsistencyLevel binds and validates parameter ConsistencyLevel from query. -func (o *ObjectsClassDeleteParams) bindConsistencyLevel(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: false - // AllowEmptyValue: false - - if raw == "" { // empty values pass all other validations - return nil - } - o.ConsistencyLevel = &raw - - return nil -} - -// bindID binds and validates parameter ID from path. -func (o *ObjectsClassDeleteParams) bindID(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - - // Format: uuid - value, err := formats.Parse("uuid", raw) - if err != nil { - return errors.InvalidType("id", "path", "strfmt.UUID", raw) - } - o.ID = *(value.(*strfmt.UUID)) - - if err := o.validateID(formats); err != nil { - return err - } - - return nil -} - -// validateID carries on validations for parameter ID -func (o *ObjectsClassDeleteParams) validateID(formats strfmt.Registry) error { - - if err := validate.FormatOf("id", "path", "uuid", o.ID.String(), formats); err != nil { - return err - } - return nil -} - -// bindTenant binds and validates parameter Tenant from query. -func (o *ObjectsClassDeleteParams) bindTenant(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: false - // AllowEmptyValue: false - - if raw == "" { // empty values pass all other validations - return nil - } - o.Tenant = &raw - - return nil -} diff --git a/adapters/handlers/rest/operations/objects/objects_class_delete_responses.go b/adapters/handlers/rest/operations/objects/objects_class_delete_responses.go deleted file mode 100644 index 18b5f96422dd9b5721e2145e1d380c0f5ebadeee..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_class_delete_responses.go +++ /dev/null @@ -1,280 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsClassDeleteNoContentCode is the HTTP code returned for type ObjectsClassDeleteNoContent -const ObjectsClassDeleteNoContentCode int = 204 - -/* -ObjectsClassDeleteNoContent Successfully deleted. - -swagger:response objectsClassDeleteNoContent -*/ -type ObjectsClassDeleteNoContent struct { -} - -// NewObjectsClassDeleteNoContent creates ObjectsClassDeleteNoContent with default headers values -func NewObjectsClassDeleteNoContent() *ObjectsClassDeleteNoContent { - - return &ObjectsClassDeleteNoContent{} -} - -// WriteResponse to the client -func (o *ObjectsClassDeleteNoContent) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(204) -} - -// ObjectsClassDeleteBadRequestCode is the HTTP code returned for type ObjectsClassDeleteBadRequest -const ObjectsClassDeleteBadRequestCode int = 400 - -/* -ObjectsClassDeleteBadRequest Malformed request. - -swagger:response objectsClassDeleteBadRequest -*/ -type ObjectsClassDeleteBadRequest struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsClassDeleteBadRequest creates ObjectsClassDeleteBadRequest with default headers values -func NewObjectsClassDeleteBadRequest() *ObjectsClassDeleteBadRequest { - - return &ObjectsClassDeleteBadRequest{} -} - -// WithPayload adds the payload to the objects class delete bad request response -func (o *ObjectsClassDeleteBadRequest) WithPayload(payload *models.ErrorResponse) *ObjectsClassDeleteBadRequest { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects class delete bad request response -func (o *ObjectsClassDeleteBadRequest) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsClassDeleteBadRequest) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(400) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsClassDeleteUnauthorizedCode is the HTTP code returned for type ObjectsClassDeleteUnauthorized -const ObjectsClassDeleteUnauthorizedCode int = 401 - -/* -ObjectsClassDeleteUnauthorized Unauthorized or invalid credentials. - -swagger:response objectsClassDeleteUnauthorized -*/ -type ObjectsClassDeleteUnauthorized struct { -} - -// NewObjectsClassDeleteUnauthorized creates ObjectsClassDeleteUnauthorized with default headers values -func NewObjectsClassDeleteUnauthorized() *ObjectsClassDeleteUnauthorized { - - return &ObjectsClassDeleteUnauthorized{} -} - -// WriteResponse to the client -func (o *ObjectsClassDeleteUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// ObjectsClassDeleteForbiddenCode is the HTTP code returned for type ObjectsClassDeleteForbidden -const ObjectsClassDeleteForbiddenCode int = 403 - -/* -ObjectsClassDeleteForbidden Forbidden - -swagger:response objectsClassDeleteForbidden -*/ -type ObjectsClassDeleteForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsClassDeleteForbidden creates ObjectsClassDeleteForbidden with default headers values -func NewObjectsClassDeleteForbidden() *ObjectsClassDeleteForbidden { - - return &ObjectsClassDeleteForbidden{} -} - -// WithPayload adds the payload to the objects class delete forbidden response -func (o *ObjectsClassDeleteForbidden) WithPayload(payload *models.ErrorResponse) *ObjectsClassDeleteForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects class delete forbidden response -func (o *ObjectsClassDeleteForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsClassDeleteForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsClassDeleteNotFoundCode is the HTTP code returned for type ObjectsClassDeleteNotFound -const ObjectsClassDeleteNotFoundCode int = 404 - -/* -ObjectsClassDeleteNotFound Successful query result but no resource was found. - -swagger:response objectsClassDeleteNotFound -*/ -type ObjectsClassDeleteNotFound struct { -} - -// NewObjectsClassDeleteNotFound creates ObjectsClassDeleteNotFound with default headers values -func NewObjectsClassDeleteNotFound() *ObjectsClassDeleteNotFound { - - return &ObjectsClassDeleteNotFound{} -} - -// WriteResponse to the client -func (o *ObjectsClassDeleteNotFound) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(404) -} - -// ObjectsClassDeleteUnprocessableEntityCode is the HTTP code returned for type ObjectsClassDeleteUnprocessableEntity -const ObjectsClassDeleteUnprocessableEntityCode int = 422 - -/* -ObjectsClassDeleteUnprocessableEntity Request is well-formed (i.e., syntactically correct), but erroneous. - -swagger:response objectsClassDeleteUnprocessableEntity -*/ -type ObjectsClassDeleteUnprocessableEntity struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsClassDeleteUnprocessableEntity creates ObjectsClassDeleteUnprocessableEntity with default headers values -func NewObjectsClassDeleteUnprocessableEntity() *ObjectsClassDeleteUnprocessableEntity { - - return &ObjectsClassDeleteUnprocessableEntity{} -} - -// WithPayload adds the payload to the objects class delete unprocessable entity response -func (o *ObjectsClassDeleteUnprocessableEntity) WithPayload(payload *models.ErrorResponse) *ObjectsClassDeleteUnprocessableEntity { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects class delete unprocessable entity response -func (o *ObjectsClassDeleteUnprocessableEntity) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsClassDeleteUnprocessableEntity) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(422) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsClassDeleteInternalServerErrorCode is the HTTP code returned for type ObjectsClassDeleteInternalServerError -const ObjectsClassDeleteInternalServerErrorCode int = 500 - -/* -ObjectsClassDeleteInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response objectsClassDeleteInternalServerError -*/ -type ObjectsClassDeleteInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsClassDeleteInternalServerError creates ObjectsClassDeleteInternalServerError with default headers values -func NewObjectsClassDeleteInternalServerError() *ObjectsClassDeleteInternalServerError { - - return &ObjectsClassDeleteInternalServerError{} -} - -// WithPayload adds the payload to the objects class delete internal server error response -func (o *ObjectsClassDeleteInternalServerError) WithPayload(payload *models.ErrorResponse) *ObjectsClassDeleteInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects class delete internal server error response -func (o *ObjectsClassDeleteInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsClassDeleteInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/objects/objects_class_delete_urlbuilder.go b/adapters/handlers/rest/operations/objects/objects_class_delete_urlbuilder.go deleted file mode 100644 index 99403102a36084a33ae7303c373cdcb7e0b67618..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_class_delete_urlbuilder.go +++ /dev/null @@ -1,143 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" - "strings" - - "github.com/go-openapi/strfmt" -) - -// ObjectsClassDeleteURL generates an URL for the objects class delete operation -type ObjectsClassDeleteURL struct { - ClassName string - ID strfmt.UUID - - ConsistencyLevel *string - Tenant *string - - _basePath string - // avoid unkeyed usage - _ struct{} -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *ObjectsClassDeleteURL) WithBasePath(bp string) *ObjectsClassDeleteURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *ObjectsClassDeleteURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *ObjectsClassDeleteURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/objects/{className}/{id}" - - className := o.ClassName - if className != "" { - _path = strings.Replace(_path, "{className}", className, -1) - } else { - return nil, errors.New("className is required on ObjectsClassDeleteURL") - } - - id := o.ID.String() - if id != "" { - _path = strings.Replace(_path, "{id}", id, -1) - } else { - return nil, errors.New("id is required on ObjectsClassDeleteURL") - } - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - qs := make(url.Values) - - var consistencyLevelQ string - if o.ConsistencyLevel != nil { - consistencyLevelQ = *o.ConsistencyLevel - } - if consistencyLevelQ != "" { - qs.Set("consistency_level", consistencyLevelQ) - } - - var tenantQ string - if o.Tenant != nil { - tenantQ = *o.Tenant - } - if tenantQ != "" { - qs.Set("tenant", tenantQ) - } - - _result.RawQuery = qs.Encode() - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *ObjectsClassDeleteURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *ObjectsClassDeleteURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *ObjectsClassDeleteURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on ObjectsClassDeleteURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on ObjectsClassDeleteURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *ObjectsClassDeleteURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/objects/objects_class_get.go b/adapters/handlers/rest/operations/objects/objects_class_get.go deleted file mode 100644 index 6deaed9e24262a0991256204264a84a3424bd68e..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_class_get.go +++ /dev/null @@ -1,84 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsClassGetHandlerFunc turns a function with the right signature into a objects class get handler -type ObjectsClassGetHandlerFunc func(ObjectsClassGetParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn ObjectsClassGetHandlerFunc) Handle(params ObjectsClassGetParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// ObjectsClassGetHandler interface for that can handle valid objects class get params -type ObjectsClassGetHandler interface { - Handle(ObjectsClassGetParams, *models.Principal) middleware.Responder -} - -// NewObjectsClassGet creates a new http.Handler for the objects class get operation -func NewObjectsClassGet(ctx *middleware.Context, handler ObjectsClassGetHandler) *ObjectsClassGet { - return &ObjectsClassGet{Context: ctx, Handler: handler} -} - -/* - ObjectsClassGet swagger:route GET /objects/{className}/{id} objects objectsClassGet - -Get a specific Object based on its class and UUID. Also available as Websocket bus. - -Get a single data object -*/ -type ObjectsClassGet struct { - Context *middleware.Context - Handler ObjectsClassGetHandler -} - -func (o *ObjectsClassGet) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewObjectsClassGetParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/objects/objects_class_get_parameters.go b/adapters/handlers/rest/operations/objects/objects_class_get_parameters.go deleted file mode 100644 index ef8acf86b716a4dcf10bc4d819ec5d0678135c1e..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_class_get_parameters.go +++ /dev/null @@ -1,237 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/validate" -) - -// NewObjectsClassGetParams creates a new ObjectsClassGetParams object -// -// There are no default values defined in the spec. -func NewObjectsClassGetParams() ObjectsClassGetParams { - - return ObjectsClassGetParams{} -} - -// ObjectsClassGetParams contains all the bound params for the objects class get operation -// typically these are obtained from a http.Request -// -// swagger:parameters objects.class.get -type ObjectsClassGetParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` - - /* - Required: true - In: path - */ - ClassName string - /*Determines how many replicas must acknowledge a request before it is considered successful - In: query - */ - ConsistencyLevel *string - /*Unique ID of the Object. - Required: true - In: path - */ - ID strfmt.UUID - /*Include additional information, such as classification infos. Allowed values include: classification, vector, interpretation - In: query - */ - Include *string - /*The target node which should fulfill the request - In: query - */ - NodeName *string - /*Specifies the tenant in a request targeting a multi-tenant class - In: query - */ - Tenant *string -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewObjectsClassGetParams() beforehand. -func (o *ObjectsClassGetParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - qs := runtime.Values(r.URL.Query()) - - rClassName, rhkClassName, _ := route.Params.GetOK("className") - if err := o.bindClassName(rClassName, rhkClassName, route.Formats); err != nil { - res = append(res, err) - } - - qConsistencyLevel, qhkConsistencyLevel, _ := qs.GetOK("consistency_level") - if err := o.bindConsistencyLevel(qConsistencyLevel, qhkConsistencyLevel, route.Formats); err != nil { - res = append(res, err) - } - - rID, rhkID, _ := route.Params.GetOK("id") - if err := o.bindID(rID, rhkID, route.Formats); err != nil { - res = append(res, err) - } - - qInclude, qhkInclude, _ := qs.GetOK("include") - if err := o.bindInclude(qInclude, qhkInclude, route.Formats); err != nil { - res = append(res, err) - } - - qNodeName, qhkNodeName, _ := qs.GetOK("node_name") - if err := o.bindNodeName(qNodeName, qhkNodeName, route.Formats); err != nil { - res = append(res, err) - } - - qTenant, qhkTenant, _ := qs.GetOK("tenant") - if err := o.bindTenant(qTenant, qhkTenant, route.Formats); err != nil { - res = append(res, err) - } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// bindClassName binds and validates parameter ClassName from path. -func (o *ObjectsClassGetParams) bindClassName(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - o.ClassName = raw - - return nil -} - -// bindConsistencyLevel binds and validates parameter ConsistencyLevel from query. -func (o *ObjectsClassGetParams) bindConsistencyLevel(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: false - // AllowEmptyValue: false - - if raw == "" { // empty values pass all other validations - return nil - } - o.ConsistencyLevel = &raw - - return nil -} - -// bindID binds and validates parameter ID from path. -func (o *ObjectsClassGetParams) bindID(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - - // Format: uuid - value, err := formats.Parse("uuid", raw) - if err != nil { - return errors.InvalidType("id", "path", "strfmt.UUID", raw) - } - o.ID = *(value.(*strfmt.UUID)) - - if err := o.validateID(formats); err != nil { - return err - } - - return nil -} - -// validateID carries on validations for parameter ID -func (o *ObjectsClassGetParams) validateID(formats strfmt.Registry) error { - - if err := validate.FormatOf("id", "path", "uuid", o.ID.String(), formats); err != nil { - return err - } - return nil -} - -// bindInclude binds and validates parameter Include from query. -func (o *ObjectsClassGetParams) bindInclude(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: false - // AllowEmptyValue: false - - if raw == "" { // empty values pass all other validations - return nil - } - o.Include = &raw - - return nil -} - -// bindNodeName binds and validates parameter NodeName from query. -func (o *ObjectsClassGetParams) bindNodeName(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: false - // AllowEmptyValue: false - - if raw == "" { // empty values pass all other validations - return nil - } - o.NodeName = &raw - - return nil -} - -// bindTenant binds and validates parameter Tenant from query. -func (o *ObjectsClassGetParams) bindTenant(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: false - // AllowEmptyValue: false - - if raw == "" { // empty values pass all other validations - return nil - } - o.Tenant = &raw - - return nil -} diff --git a/adapters/handlers/rest/operations/objects/objects_class_get_responses.go b/adapters/handlers/rest/operations/objects/objects_class_get_responses.go deleted file mode 100644 index 1ece6cdfcb14a6ab76759573edb8c20e5986114b..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_class_get_responses.go +++ /dev/null @@ -1,300 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsClassGetOKCode is the HTTP code returned for type ObjectsClassGetOK -const ObjectsClassGetOKCode int = 200 - -/* -ObjectsClassGetOK Successful response. - -swagger:response objectsClassGetOK -*/ -type ObjectsClassGetOK struct { - - /* - In: Body - */ - Payload *models.Object `json:"body,omitempty"` -} - -// NewObjectsClassGetOK creates ObjectsClassGetOK with default headers values -func NewObjectsClassGetOK() *ObjectsClassGetOK { - - return &ObjectsClassGetOK{} -} - -// WithPayload adds the payload to the objects class get o k response -func (o *ObjectsClassGetOK) WithPayload(payload *models.Object) *ObjectsClassGetOK { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects class get o k response -func (o *ObjectsClassGetOK) SetPayload(payload *models.Object) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsClassGetOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(200) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsClassGetBadRequestCode is the HTTP code returned for type ObjectsClassGetBadRequest -const ObjectsClassGetBadRequestCode int = 400 - -/* -ObjectsClassGetBadRequest Malformed request. - -swagger:response objectsClassGetBadRequest -*/ -type ObjectsClassGetBadRequest struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsClassGetBadRequest creates ObjectsClassGetBadRequest with default headers values -func NewObjectsClassGetBadRequest() *ObjectsClassGetBadRequest { - - return &ObjectsClassGetBadRequest{} -} - -// WithPayload adds the payload to the objects class get bad request response -func (o *ObjectsClassGetBadRequest) WithPayload(payload *models.ErrorResponse) *ObjectsClassGetBadRequest { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects class get bad request response -func (o *ObjectsClassGetBadRequest) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsClassGetBadRequest) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(400) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsClassGetUnauthorizedCode is the HTTP code returned for type ObjectsClassGetUnauthorized -const ObjectsClassGetUnauthorizedCode int = 401 - -/* -ObjectsClassGetUnauthorized Unauthorized or invalid credentials. - -swagger:response objectsClassGetUnauthorized -*/ -type ObjectsClassGetUnauthorized struct { -} - -// NewObjectsClassGetUnauthorized creates ObjectsClassGetUnauthorized with default headers values -func NewObjectsClassGetUnauthorized() *ObjectsClassGetUnauthorized { - - return &ObjectsClassGetUnauthorized{} -} - -// WriteResponse to the client -func (o *ObjectsClassGetUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// ObjectsClassGetForbiddenCode is the HTTP code returned for type ObjectsClassGetForbidden -const ObjectsClassGetForbiddenCode int = 403 - -/* -ObjectsClassGetForbidden Forbidden - -swagger:response objectsClassGetForbidden -*/ -type ObjectsClassGetForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsClassGetForbidden creates ObjectsClassGetForbidden with default headers values -func NewObjectsClassGetForbidden() *ObjectsClassGetForbidden { - - return &ObjectsClassGetForbidden{} -} - -// WithPayload adds the payload to the objects class get forbidden response -func (o *ObjectsClassGetForbidden) WithPayload(payload *models.ErrorResponse) *ObjectsClassGetForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects class get forbidden response -func (o *ObjectsClassGetForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsClassGetForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsClassGetNotFoundCode is the HTTP code returned for type ObjectsClassGetNotFound -const ObjectsClassGetNotFoundCode int = 404 - -/* -ObjectsClassGetNotFound Successful query result but no resource was found. - -swagger:response objectsClassGetNotFound -*/ -type ObjectsClassGetNotFound struct { -} - -// NewObjectsClassGetNotFound creates ObjectsClassGetNotFound with default headers values -func NewObjectsClassGetNotFound() *ObjectsClassGetNotFound { - - return &ObjectsClassGetNotFound{} -} - -// WriteResponse to the client -func (o *ObjectsClassGetNotFound) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(404) -} - -// ObjectsClassGetUnprocessableEntityCode is the HTTP code returned for type ObjectsClassGetUnprocessableEntity -const ObjectsClassGetUnprocessableEntityCode int = 422 - -/* -ObjectsClassGetUnprocessableEntity Request is well-formed (i.e., syntactically correct), but erroneous. - -swagger:response objectsClassGetUnprocessableEntity -*/ -type ObjectsClassGetUnprocessableEntity struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsClassGetUnprocessableEntity creates ObjectsClassGetUnprocessableEntity with default headers values -func NewObjectsClassGetUnprocessableEntity() *ObjectsClassGetUnprocessableEntity { - - return &ObjectsClassGetUnprocessableEntity{} -} - -// WithPayload adds the payload to the objects class get unprocessable entity response -func (o *ObjectsClassGetUnprocessableEntity) WithPayload(payload *models.ErrorResponse) *ObjectsClassGetUnprocessableEntity { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects class get unprocessable entity response -func (o *ObjectsClassGetUnprocessableEntity) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsClassGetUnprocessableEntity) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(422) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsClassGetInternalServerErrorCode is the HTTP code returned for type ObjectsClassGetInternalServerError -const ObjectsClassGetInternalServerErrorCode int = 500 - -/* -ObjectsClassGetInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response objectsClassGetInternalServerError -*/ -type ObjectsClassGetInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsClassGetInternalServerError creates ObjectsClassGetInternalServerError with default headers values -func NewObjectsClassGetInternalServerError() *ObjectsClassGetInternalServerError { - - return &ObjectsClassGetInternalServerError{} -} - -// WithPayload adds the payload to the objects class get internal server error response -func (o *ObjectsClassGetInternalServerError) WithPayload(payload *models.ErrorResponse) *ObjectsClassGetInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects class get internal server error response -func (o *ObjectsClassGetInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsClassGetInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/objects/objects_class_get_urlbuilder.go b/adapters/handlers/rest/operations/objects/objects_class_get_urlbuilder.go deleted file mode 100644 index a2742682e3cca3751a8dbe71e3c862f81158c94c..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_class_get_urlbuilder.go +++ /dev/null @@ -1,161 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" - "strings" - - "github.com/go-openapi/strfmt" -) - -// ObjectsClassGetURL generates an URL for the objects class get operation -type ObjectsClassGetURL struct { - ClassName string - ID strfmt.UUID - - ConsistencyLevel *string - Include *string - NodeName *string - Tenant *string - - _basePath string - // avoid unkeyed usage - _ struct{} -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *ObjectsClassGetURL) WithBasePath(bp string) *ObjectsClassGetURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *ObjectsClassGetURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *ObjectsClassGetURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/objects/{className}/{id}" - - className := o.ClassName - if className != "" { - _path = strings.Replace(_path, "{className}", className, -1) - } else { - return nil, errors.New("className is required on ObjectsClassGetURL") - } - - id := o.ID.String() - if id != "" { - _path = strings.Replace(_path, "{id}", id, -1) - } else { - return nil, errors.New("id is required on ObjectsClassGetURL") - } - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - qs := make(url.Values) - - var consistencyLevelQ string - if o.ConsistencyLevel != nil { - consistencyLevelQ = *o.ConsistencyLevel - } - if consistencyLevelQ != "" { - qs.Set("consistency_level", consistencyLevelQ) - } - - var includeQ string - if o.Include != nil { - includeQ = *o.Include - } - if includeQ != "" { - qs.Set("include", includeQ) - } - - var nodeNameQ string - if o.NodeName != nil { - nodeNameQ = *o.NodeName - } - if nodeNameQ != "" { - qs.Set("node_name", nodeNameQ) - } - - var tenantQ string - if o.Tenant != nil { - tenantQ = *o.Tenant - } - if tenantQ != "" { - qs.Set("tenant", tenantQ) - } - - _result.RawQuery = qs.Encode() - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *ObjectsClassGetURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *ObjectsClassGetURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *ObjectsClassGetURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on ObjectsClassGetURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on ObjectsClassGetURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *ObjectsClassGetURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/objects/objects_class_head.go b/adapters/handlers/rest/operations/objects/objects_class_head.go deleted file mode 100644 index cd2bee0f6a2b04886e0597305294a780256bb23e..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_class_head.go +++ /dev/null @@ -1,84 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsClassHeadHandlerFunc turns a function with the right signature into a objects class head handler -type ObjectsClassHeadHandlerFunc func(ObjectsClassHeadParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn ObjectsClassHeadHandlerFunc) Handle(params ObjectsClassHeadParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// ObjectsClassHeadHandler interface for that can handle valid objects class head params -type ObjectsClassHeadHandler interface { - Handle(ObjectsClassHeadParams, *models.Principal) middleware.Responder -} - -// NewObjectsClassHead creates a new http.Handler for the objects class head operation -func NewObjectsClassHead(ctx *middleware.Context, handler ObjectsClassHeadHandler) *ObjectsClassHead { - return &ObjectsClassHead{Context: ctx, Handler: handler} -} - -/* - ObjectsClassHead swagger:route HEAD /objects/{className}/{id} objects objectsClassHead - -Checks object's existence based on its class and uuid. - -Checks if a data object exists without retrieving it. -*/ -type ObjectsClassHead struct { - Context *middleware.Context - Handler ObjectsClassHeadHandler -} - -func (o *ObjectsClassHead) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewObjectsClassHeadParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/objects/objects_class_head_parameters.go b/adapters/handlers/rest/operations/objects/objects_class_head_parameters.go deleted file mode 100644 index 08299515cbe904e5295adc7586e84937aedce8d3..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_class_head_parameters.go +++ /dev/null @@ -1,183 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/validate" -) - -// NewObjectsClassHeadParams creates a new ObjectsClassHeadParams object -// -// There are no default values defined in the spec. -func NewObjectsClassHeadParams() ObjectsClassHeadParams { - - return ObjectsClassHeadParams{} -} - -// ObjectsClassHeadParams contains all the bound params for the objects class head operation -// typically these are obtained from a http.Request -// -// swagger:parameters objects.class.head -type ObjectsClassHeadParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` - - /*The class name as defined in the schema - Required: true - In: path - */ - ClassName string - /*Determines how many replicas must acknowledge a request before it is considered successful - In: query - */ - ConsistencyLevel *string - /*The uuid of the data object - Required: true - In: path - */ - ID strfmt.UUID - /*Specifies the tenant in a request targeting a multi-tenant class - In: query - */ - Tenant *string -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewObjectsClassHeadParams() beforehand. -func (o *ObjectsClassHeadParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - qs := runtime.Values(r.URL.Query()) - - rClassName, rhkClassName, _ := route.Params.GetOK("className") - if err := o.bindClassName(rClassName, rhkClassName, route.Formats); err != nil { - res = append(res, err) - } - - qConsistencyLevel, qhkConsistencyLevel, _ := qs.GetOK("consistency_level") - if err := o.bindConsistencyLevel(qConsistencyLevel, qhkConsistencyLevel, route.Formats); err != nil { - res = append(res, err) - } - - rID, rhkID, _ := route.Params.GetOK("id") - if err := o.bindID(rID, rhkID, route.Formats); err != nil { - res = append(res, err) - } - - qTenant, qhkTenant, _ := qs.GetOK("tenant") - if err := o.bindTenant(qTenant, qhkTenant, route.Formats); err != nil { - res = append(res, err) - } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// bindClassName binds and validates parameter ClassName from path. -func (o *ObjectsClassHeadParams) bindClassName(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - o.ClassName = raw - - return nil -} - -// bindConsistencyLevel binds and validates parameter ConsistencyLevel from query. -func (o *ObjectsClassHeadParams) bindConsistencyLevel(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: false - // AllowEmptyValue: false - - if raw == "" { // empty values pass all other validations - return nil - } - o.ConsistencyLevel = &raw - - return nil -} - -// bindID binds and validates parameter ID from path. -func (o *ObjectsClassHeadParams) bindID(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - - // Format: uuid - value, err := formats.Parse("uuid", raw) - if err != nil { - return errors.InvalidType("id", "path", "strfmt.UUID", raw) - } - o.ID = *(value.(*strfmt.UUID)) - - if err := o.validateID(formats); err != nil { - return err - } - - return nil -} - -// validateID carries on validations for parameter ID -func (o *ObjectsClassHeadParams) validateID(formats strfmt.Registry) error { - - if err := validate.FormatOf("id", "path", "uuid", o.ID.String(), formats); err != nil { - return err - } - return nil -} - -// bindTenant binds and validates parameter Tenant from query. -func (o *ObjectsClassHeadParams) bindTenant(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: false - // AllowEmptyValue: false - - if raw == "" { // empty values pass all other validations - return nil - } - o.Tenant = &raw - - return nil -} diff --git a/adapters/handlers/rest/operations/objects/objects_class_head_responses.go b/adapters/handlers/rest/operations/objects/objects_class_head_responses.go deleted file mode 100644 index b63df74b75f1b843ebf855d518fb310dbfd183ff..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_class_head_responses.go +++ /dev/null @@ -1,235 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsClassHeadNoContentCode is the HTTP code returned for type ObjectsClassHeadNoContent -const ObjectsClassHeadNoContentCode int = 204 - -/* -ObjectsClassHeadNoContent Object exists. - -swagger:response objectsClassHeadNoContent -*/ -type ObjectsClassHeadNoContent struct { -} - -// NewObjectsClassHeadNoContent creates ObjectsClassHeadNoContent with default headers values -func NewObjectsClassHeadNoContent() *ObjectsClassHeadNoContent { - - return &ObjectsClassHeadNoContent{} -} - -// WriteResponse to the client -func (o *ObjectsClassHeadNoContent) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(204) -} - -// ObjectsClassHeadUnauthorizedCode is the HTTP code returned for type ObjectsClassHeadUnauthorized -const ObjectsClassHeadUnauthorizedCode int = 401 - -/* -ObjectsClassHeadUnauthorized Unauthorized or invalid credentials. - -swagger:response objectsClassHeadUnauthorized -*/ -type ObjectsClassHeadUnauthorized struct { -} - -// NewObjectsClassHeadUnauthorized creates ObjectsClassHeadUnauthorized with default headers values -func NewObjectsClassHeadUnauthorized() *ObjectsClassHeadUnauthorized { - - return &ObjectsClassHeadUnauthorized{} -} - -// WriteResponse to the client -func (o *ObjectsClassHeadUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// ObjectsClassHeadForbiddenCode is the HTTP code returned for type ObjectsClassHeadForbidden -const ObjectsClassHeadForbiddenCode int = 403 - -/* -ObjectsClassHeadForbidden Forbidden - -swagger:response objectsClassHeadForbidden -*/ -type ObjectsClassHeadForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsClassHeadForbidden creates ObjectsClassHeadForbidden with default headers values -func NewObjectsClassHeadForbidden() *ObjectsClassHeadForbidden { - - return &ObjectsClassHeadForbidden{} -} - -// WithPayload adds the payload to the objects class head forbidden response -func (o *ObjectsClassHeadForbidden) WithPayload(payload *models.ErrorResponse) *ObjectsClassHeadForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects class head forbidden response -func (o *ObjectsClassHeadForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsClassHeadForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsClassHeadNotFoundCode is the HTTP code returned for type ObjectsClassHeadNotFound -const ObjectsClassHeadNotFoundCode int = 404 - -/* -ObjectsClassHeadNotFound Object doesn't exist. - -swagger:response objectsClassHeadNotFound -*/ -type ObjectsClassHeadNotFound struct { -} - -// NewObjectsClassHeadNotFound creates ObjectsClassHeadNotFound with default headers values -func NewObjectsClassHeadNotFound() *ObjectsClassHeadNotFound { - - return &ObjectsClassHeadNotFound{} -} - -// WriteResponse to the client -func (o *ObjectsClassHeadNotFound) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(404) -} - -// ObjectsClassHeadUnprocessableEntityCode is the HTTP code returned for type ObjectsClassHeadUnprocessableEntity -const ObjectsClassHeadUnprocessableEntityCode int = 422 - -/* -ObjectsClassHeadUnprocessableEntity Request is well-formed (i.e., syntactically correct), but erroneous. - -swagger:response objectsClassHeadUnprocessableEntity -*/ -type ObjectsClassHeadUnprocessableEntity struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsClassHeadUnprocessableEntity creates ObjectsClassHeadUnprocessableEntity with default headers values -func NewObjectsClassHeadUnprocessableEntity() *ObjectsClassHeadUnprocessableEntity { - - return &ObjectsClassHeadUnprocessableEntity{} -} - -// WithPayload adds the payload to the objects class head unprocessable entity response -func (o *ObjectsClassHeadUnprocessableEntity) WithPayload(payload *models.ErrorResponse) *ObjectsClassHeadUnprocessableEntity { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects class head unprocessable entity response -func (o *ObjectsClassHeadUnprocessableEntity) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsClassHeadUnprocessableEntity) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(422) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsClassHeadInternalServerErrorCode is the HTTP code returned for type ObjectsClassHeadInternalServerError -const ObjectsClassHeadInternalServerErrorCode int = 500 - -/* -ObjectsClassHeadInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response objectsClassHeadInternalServerError -*/ -type ObjectsClassHeadInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsClassHeadInternalServerError creates ObjectsClassHeadInternalServerError with default headers values -func NewObjectsClassHeadInternalServerError() *ObjectsClassHeadInternalServerError { - - return &ObjectsClassHeadInternalServerError{} -} - -// WithPayload adds the payload to the objects class head internal server error response -func (o *ObjectsClassHeadInternalServerError) WithPayload(payload *models.ErrorResponse) *ObjectsClassHeadInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects class head internal server error response -func (o *ObjectsClassHeadInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsClassHeadInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/objects/objects_class_head_urlbuilder.go b/adapters/handlers/rest/operations/objects/objects_class_head_urlbuilder.go deleted file mode 100644 index b52494e73fc86bb9c584e9fba085908b80752bf1..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_class_head_urlbuilder.go +++ /dev/null @@ -1,143 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" - "strings" - - "github.com/go-openapi/strfmt" -) - -// ObjectsClassHeadURL generates an URL for the objects class head operation -type ObjectsClassHeadURL struct { - ClassName string - ID strfmt.UUID - - ConsistencyLevel *string - Tenant *string - - _basePath string - // avoid unkeyed usage - _ struct{} -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *ObjectsClassHeadURL) WithBasePath(bp string) *ObjectsClassHeadURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *ObjectsClassHeadURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *ObjectsClassHeadURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/objects/{className}/{id}" - - className := o.ClassName - if className != "" { - _path = strings.Replace(_path, "{className}", className, -1) - } else { - return nil, errors.New("className is required on ObjectsClassHeadURL") - } - - id := o.ID.String() - if id != "" { - _path = strings.Replace(_path, "{id}", id, -1) - } else { - return nil, errors.New("id is required on ObjectsClassHeadURL") - } - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - qs := make(url.Values) - - var consistencyLevelQ string - if o.ConsistencyLevel != nil { - consistencyLevelQ = *o.ConsistencyLevel - } - if consistencyLevelQ != "" { - qs.Set("consistency_level", consistencyLevelQ) - } - - var tenantQ string - if o.Tenant != nil { - tenantQ = *o.Tenant - } - if tenantQ != "" { - qs.Set("tenant", tenantQ) - } - - _result.RawQuery = qs.Encode() - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *ObjectsClassHeadURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *ObjectsClassHeadURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *ObjectsClassHeadURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on ObjectsClassHeadURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on ObjectsClassHeadURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *ObjectsClassHeadURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/objects/objects_class_patch.go b/adapters/handlers/rest/operations/objects/objects_class_patch.go deleted file mode 100644 index ab895a2991be42ffc2118d76bc88db9d45787029..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_class_patch.go +++ /dev/null @@ -1,84 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsClassPatchHandlerFunc turns a function with the right signature into a objects class patch handler -type ObjectsClassPatchHandlerFunc func(ObjectsClassPatchParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn ObjectsClassPatchHandlerFunc) Handle(params ObjectsClassPatchParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// ObjectsClassPatchHandler interface for that can handle valid objects class patch params -type ObjectsClassPatchHandler interface { - Handle(ObjectsClassPatchParams, *models.Principal) middleware.Responder -} - -// NewObjectsClassPatch creates a new http.Handler for the objects class patch operation -func NewObjectsClassPatch(ctx *middleware.Context, handler ObjectsClassPatchHandler) *ObjectsClassPatch { - return &ObjectsClassPatch{Context: ctx, Handler: handler} -} - -/* - ObjectsClassPatch swagger:route PATCH /objects/{className}/{id} objects objectsClassPatch - -Update an Object based on its UUID (using patch semantics). - -Update an individual data object based on its class and uuid. This method supports json-merge style patch semantics (RFC 7396). Provided meta-data and schema values are validated. LastUpdateTime is set to the time this function is called. -*/ -type ObjectsClassPatch struct { - Context *middleware.Context - Handler ObjectsClassPatchHandler -} - -func (o *ObjectsClassPatch) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewObjectsClassPatchParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/objects/objects_class_patch_parameters.go b/adapters/handlers/rest/operations/objects/objects_class_patch_parameters.go deleted file mode 100644 index 3b753f5b1fcdd21a21bae9286cf7ec60f5dd0497..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_class_patch_parameters.go +++ /dev/null @@ -1,184 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/validate" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewObjectsClassPatchParams creates a new ObjectsClassPatchParams object -// -// There are no default values defined in the spec. -func NewObjectsClassPatchParams() ObjectsClassPatchParams { - - return ObjectsClassPatchParams{} -} - -// ObjectsClassPatchParams contains all the bound params for the objects class patch operation -// typically these are obtained from a http.Request -// -// swagger:parameters objects.class.patch -type ObjectsClassPatchParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` - - /*RFC 7396-style patch, the body contains the object to merge into the existing object. - In: body - */ - Body *models.Object - /*The class name as defined in the schema - Required: true - In: path - */ - ClassName string - /*Determines how many replicas must acknowledge a request before it is considered successful - In: query - */ - ConsistencyLevel *string - /*The uuid of the data object to update. - Required: true - In: path - */ - ID strfmt.UUID -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewObjectsClassPatchParams() beforehand. -func (o *ObjectsClassPatchParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - qs := runtime.Values(r.URL.Query()) - - if runtime.HasBody(r) { - defer r.Body.Close() - var body models.Object - if err := route.Consumer.Consume(r.Body, &body); err != nil { - res = append(res, errors.NewParseError("body", "body", "", err)) - } else { - // validate body object - if err := body.Validate(route.Formats); err != nil { - res = append(res, err) - } - - ctx := validate.WithOperationRequest(r.Context()) - if err := body.ContextValidate(ctx, route.Formats); err != nil { - res = append(res, err) - } - - if len(res) == 0 { - o.Body = &body - } - } - } - - rClassName, rhkClassName, _ := route.Params.GetOK("className") - if err := o.bindClassName(rClassName, rhkClassName, route.Formats); err != nil { - res = append(res, err) - } - - qConsistencyLevel, qhkConsistencyLevel, _ := qs.GetOK("consistency_level") - if err := o.bindConsistencyLevel(qConsistencyLevel, qhkConsistencyLevel, route.Formats); err != nil { - res = append(res, err) - } - - rID, rhkID, _ := route.Params.GetOK("id") - if err := o.bindID(rID, rhkID, route.Formats); err != nil { - res = append(res, err) - } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// bindClassName binds and validates parameter ClassName from path. -func (o *ObjectsClassPatchParams) bindClassName(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - o.ClassName = raw - - return nil -} - -// bindConsistencyLevel binds and validates parameter ConsistencyLevel from query. -func (o *ObjectsClassPatchParams) bindConsistencyLevel(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: false - // AllowEmptyValue: false - - if raw == "" { // empty values pass all other validations - return nil - } - o.ConsistencyLevel = &raw - - return nil -} - -// bindID binds and validates parameter ID from path. -func (o *ObjectsClassPatchParams) bindID(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - - // Format: uuid - value, err := formats.Parse("uuid", raw) - if err != nil { - return errors.InvalidType("id", "path", "strfmt.UUID", raw) - } - o.ID = *(value.(*strfmt.UUID)) - - if err := o.validateID(formats); err != nil { - return err - } - - return nil -} - -// validateID carries on validations for parameter ID -func (o *ObjectsClassPatchParams) validateID(formats strfmt.Registry) error { - - if err := validate.FormatOf("id", "path", "uuid", o.ID.String(), formats); err != nil { - return err - } - return nil -} diff --git a/adapters/handlers/rest/operations/objects/objects_class_patch_responses.go b/adapters/handlers/rest/operations/objects/objects_class_patch_responses.go deleted file mode 100644 index 01c4dd96cac0c3ed0ef7c531963961317c495202..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_class_patch_responses.go +++ /dev/null @@ -1,280 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsClassPatchNoContentCode is the HTTP code returned for type ObjectsClassPatchNoContent -const ObjectsClassPatchNoContentCode int = 204 - -/* -ObjectsClassPatchNoContent Successfully applied. No content provided. - -swagger:response objectsClassPatchNoContent -*/ -type ObjectsClassPatchNoContent struct { -} - -// NewObjectsClassPatchNoContent creates ObjectsClassPatchNoContent with default headers values -func NewObjectsClassPatchNoContent() *ObjectsClassPatchNoContent { - - return &ObjectsClassPatchNoContent{} -} - -// WriteResponse to the client -func (o *ObjectsClassPatchNoContent) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(204) -} - -// ObjectsClassPatchBadRequestCode is the HTTP code returned for type ObjectsClassPatchBadRequest -const ObjectsClassPatchBadRequestCode int = 400 - -/* -ObjectsClassPatchBadRequest The patch-JSON is malformed. - -swagger:response objectsClassPatchBadRequest -*/ -type ObjectsClassPatchBadRequest struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsClassPatchBadRequest creates ObjectsClassPatchBadRequest with default headers values -func NewObjectsClassPatchBadRequest() *ObjectsClassPatchBadRequest { - - return &ObjectsClassPatchBadRequest{} -} - -// WithPayload adds the payload to the objects class patch bad request response -func (o *ObjectsClassPatchBadRequest) WithPayload(payload *models.ErrorResponse) *ObjectsClassPatchBadRequest { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects class patch bad request response -func (o *ObjectsClassPatchBadRequest) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsClassPatchBadRequest) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(400) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsClassPatchUnauthorizedCode is the HTTP code returned for type ObjectsClassPatchUnauthorized -const ObjectsClassPatchUnauthorizedCode int = 401 - -/* -ObjectsClassPatchUnauthorized Unauthorized or invalid credentials. - -swagger:response objectsClassPatchUnauthorized -*/ -type ObjectsClassPatchUnauthorized struct { -} - -// NewObjectsClassPatchUnauthorized creates ObjectsClassPatchUnauthorized with default headers values -func NewObjectsClassPatchUnauthorized() *ObjectsClassPatchUnauthorized { - - return &ObjectsClassPatchUnauthorized{} -} - -// WriteResponse to the client -func (o *ObjectsClassPatchUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// ObjectsClassPatchForbiddenCode is the HTTP code returned for type ObjectsClassPatchForbidden -const ObjectsClassPatchForbiddenCode int = 403 - -/* -ObjectsClassPatchForbidden Forbidden - -swagger:response objectsClassPatchForbidden -*/ -type ObjectsClassPatchForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsClassPatchForbidden creates ObjectsClassPatchForbidden with default headers values -func NewObjectsClassPatchForbidden() *ObjectsClassPatchForbidden { - - return &ObjectsClassPatchForbidden{} -} - -// WithPayload adds the payload to the objects class patch forbidden response -func (o *ObjectsClassPatchForbidden) WithPayload(payload *models.ErrorResponse) *ObjectsClassPatchForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects class patch forbidden response -func (o *ObjectsClassPatchForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsClassPatchForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsClassPatchNotFoundCode is the HTTP code returned for type ObjectsClassPatchNotFound -const ObjectsClassPatchNotFoundCode int = 404 - -/* -ObjectsClassPatchNotFound Successful query result but no resource was found. - -swagger:response objectsClassPatchNotFound -*/ -type ObjectsClassPatchNotFound struct { -} - -// NewObjectsClassPatchNotFound creates ObjectsClassPatchNotFound with default headers values -func NewObjectsClassPatchNotFound() *ObjectsClassPatchNotFound { - - return &ObjectsClassPatchNotFound{} -} - -// WriteResponse to the client -func (o *ObjectsClassPatchNotFound) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(404) -} - -// ObjectsClassPatchUnprocessableEntityCode is the HTTP code returned for type ObjectsClassPatchUnprocessableEntity -const ObjectsClassPatchUnprocessableEntityCode int = 422 - -/* -ObjectsClassPatchUnprocessableEntity The patch-JSON is valid but unprocessable. - -swagger:response objectsClassPatchUnprocessableEntity -*/ -type ObjectsClassPatchUnprocessableEntity struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsClassPatchUnprocessableEntity creates ObjectsClassPatchUnprocessableEntity with default headers values -func NewObjectsClassPatchUnprocessableEntity() *ObjectsClassPatchUnprocessableEntity { - - return &ObjectsClassPatchUnprocessableEntity{} -} - -// WithPayload adds the payload to the objects class patch unprocessable entity response -func (o *ObjectsClassPatchUnprocessableEntity) WithPayload(payload *models.ErrorResponse) *ObjectsClassPatchUnprocessableEntity { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects class patch unprocessable entity response -func (o *ObjectsClassPatchUnprocessableEntity) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsClassPatchUnprocessableEntity) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(422) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsClassPatchInternalServerErrorCode is the HTTP code returned for type ObjectsClassPatchInternalServerError -const ObjectsClassPatchInternalServerErrorCode int = 500 - -/* -ObjectsClassPatchInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response objectsClassPatchInternalServerError -*/ -type ObjectsClassPatchInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsClassPatchInternalServerError creates ObjectsClassPatchInternalServerError with default headers values -func NewObjectsClassPatchInternalServerError() *ObjectsClassPatchInternalServerError { - - return &ObjectsClassPatchInternalServerError{} -} - -// WithPayload adds the payload to the objects class patch internal server error response -func (o *ObjectsClassPatchInternalServerError) WithPayload(payload *models.ErrorResponse) *ObjectsClassPatchInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects class patch internal server error response -func (o *ObjectsClassPatchInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsClassPatchInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/objects/objects_class_patch_urlbuilder.go b/adapters/handlers/rest/operations/objects/objects_class_patch_urlbuilder.go deleted file mode 100644 index 2e017bc304afae84f662dc8370c54b18aeec204a..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_class_patch_urlbuilder.go +++ /dev/null @@ -1,134 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" - "strings" - - "github.com/go-openapi/strfmt" -) - -// ObjectsClassPatchURL generates an URL for the objects class patch operation -type ObjectsClassPatchURL struct { - ClassName string - ID strfmt.UUID - - ConsistencyLevel *string - - _basePath string - // avoid unkeyed usage - _ struct{} -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *ObjectsClassPatchURL) WithBasePath(bp string) *ObjectsClassPatchURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *ObjectsClassPatchURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *ObjectsClassPatchURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/objects/{className}/{id}" - - className := o.ClassName - if className != "" { - _path = strings.Replace(_path, "{className}", className, -1) - } else { - return nil, errors.New("className is required on ObjectsClassPatchURL") - } - - id := o.ID.String() - if id != "" { - _path = strings.Replace(_path, "{id}", id, -1) - } else { - return nil, errors.New("id is required on ObjectsClassPatchURL") - } - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - qs := make(url.Values) - - var consistencyLevelQ string - if o.ConsistencyLevel != nil { - consistencyLevelQ = *o.ConsistencyLevel - } - if consistencyLevelQ != "" { - qs.Set("consistency_level", consistencyLevelQ) - } - - _result.RawQuery = qs.Encode() - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *ObjectsClassPatchURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *ObjectsClassPatchURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *ObjectsClassPatchURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on ObjectsClassPatchURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on ObjectsClassPatchURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *ObjectsClassPatchURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/objects/objects_class_put.go b/adapters/handlers/rest/operations/objects/objects_class_put.go deleted file mode 100644 index 7962f945f0ad54db6fc4895e5d93c1590ce11b71..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_class_put.go +++ /dev/null @@ -1,84 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsClassPutHandlerFunc turns a function with the right signature into a objects class put handler -type ObjectsClassPutHandlerFunc func(ObjectsClassPutParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn ObjectsClassPutHandlerFunc) Handle(params ObjectsClassPutParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// ObjectsClassPutHandler interface for that can handle valid objects class put params -type ObjectsClassPutHandler interface { - Handle(ObjectsClassPutParams, *models.Principal) middleware.Responder -} - -// NewObjectsClassPut creates a new http.Handler for the objects class put operation -func NewObjectsClassPut(ctx *middleware.Context, handler ObjectsClassPutHandler) *ObjectsClassPut { - return &ObjectsClassPut{Context: ctx, Handler: handler} -} - -/* - ObjectsClassPut swagger:route PUT /objects/{className}/{id} objects objectsClassPut - -# Update a class object based on its uuid - -Update an individual data object based on its class and uuid. -*/ -type ObjectsClassPut struct { - Context *middleware.Context - Handler ObjectsClassPutHandler -} - -func (o *ObjectsClassPut) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewObjectsClassPutParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/objects/objects_class_put_parameters.go b/adapters/handlers/rest/operations/objects/objects_class_put_parameters.go deleted file mode 100644 index 1a3ace9f9621662ab84a0211c00e43e41ac793f1..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_class_put_parameters.go +++ /dev/null @@ -1,192 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "io" - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/validate" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewObjectsClassPutParams creates a new ObjectsClassPutParams object -// -// There are no default values defined in the spec. -func NewObjectsClassPutParams() ObjectsClassPutParams { - - return ObjectsClassPutParams{} -} - -// ObjectsClassPutParams contains all the bound params for the objects class put operation -// typically these are obtained from a http.Request -// -// swagger:parameters objects.class.put -type ObjectsClassPutParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` - - /* - Required: true - In: body - */ - Body *models.Object - /* - Required: true - In: path - */ - ClassName string - /*Determines how many replicas must acknowledge a request before it is considered successful - In: query - */ - ConsistencyLevel *string - /*The uuid of the data object to update. - Required: true - In: path - */ - ID strfmt.UUID -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewObjectsClassPutParams() beforehand. -func (o *ObjectsClassPutParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - qs := runtime.Values(r.URL.Query()) - - if runtime.HasBody(r) { - defer r.Body.Close() - var body models.Object - if err := route.Consumer.Consume(r.Body, &body); err != nil { - if err == io.EOF { - res = append(res, errors.Required("body", "body", "")) - } else { - res = append(res, errors.NewParseError("body", "body", "", err)) - } - } else { - // validate body object - if err := body.Validate(route.Formats); err != nil { - res = append(res, err) - } - - ctx := validate.WithOperationRequest(r.Context()) - if err := body.ContextValidate(ctx, route.Formats); err != nil { - res = append(res, err) - } - - if len(res) == 0 { - o.Body = &body - } - } - } else { - res = append(res, errors.Required("body", "body", "")) - } - - rClassName, rhkClassName, _ := route.Params.GetOK("className") - if err := o.bindClassName(rClassName, rhkClassName, route.Formats); err != nil { - res = append(res, err) - } - - qConsistencyLevel, qhkConsistencyLevel, _ := qs.GetOK("consistency_level") - if err := o.bindConsistencyLevel(qConsistencyLevel, qhkConsistencyLevel, route.Formats); err != nil { - res = append(res, err) - } - - rID, rhkID, _ := route.Params.GetOK("id") - if err := o.bindID(rID, rhkID, route.Formats); err != nil { - res = append(res, err) - } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// bindClassName binds and validates parameter ClassName from path. -func (o *ObjectsClassPutParams) bindClassName(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - o.ClassName = raw - - return nil -} - -// bindConsistencyLevel binds and validates parameter ConsistencyLevel from query. -func (o *ObjectsClassPutParams) bindConsistencyLevel(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: false - // AllowEmptyValue: false - - if raw == "" { // empty values pass all other validations - return nil - } - o.ConsistencyLevel = &raw - - return nil -} - -// bindID binds and validates parameter ID from path. -func (o *ObjectsClassPutParams) bindID(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - - // Format: uuid - value, err := formats.Parse("uuid", raw) - if err != nil { - return errors.InvalidType("id", "path", "strfmt.UUID", raw) - } - o.ID = *(value.(*strfmt.UUID)) - - if err := o.validateID(formats); err != nil { - return err - } - - return nil -} - -// validateID carries on validations for parameter ID -func (o *ObjectsClassPutParams) validateID(formats strfmt.Registry) error { - - if err := validate.FormatOf("id", "path", "uuid", o.ID.String(), formats); err != nil { - return err - } - return nil -} diff --git a/adapters/handlers/rest/operations/objects/objects_class_put_responses.go b/adapters/handlers/rest/operations/objects/objects_class_put_responses.go deleted file mode 100644 index 250162adf12c8508bb23d6d04d4aa93932144f98..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_class_put_responses.go +++ /dev/null @@ -1,255 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsClassPutOKCode is the HTTP code returned for type ObjectsClassPutOK -const ObjectsClassPutOKCode int = 200 - -/* -ObjectsClassPutOK Successfully received. - -swagger:response objectsClassPutOK -*/ -type ObjectsClassPutOK struct { - - /* - In: Body - */ - Payload *models.Object `json:"body,omitempty"` -} - -// NewObjectsClassPutOK creates ObjectsClassPutOK with default headers values -func NewObjectsClassPutOK() *ObjectsClassPutOK { - - return &ObjectsClassPutOK{} -} - -// WithPayload adds the payload to the objects class put o k response -func (o *ObjectsClassPutOK) WithPayload(payload *models.Object) *ObjectsClassPutOK { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects class put o k response -func (o *ObjectsClassPutOK) SetPayload(payload *models.Object) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsClassPutOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(200) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsClassPutUnauthorizedCode is the HTTP code returned for type ObjectsClassPutUnauthorized -const ObjectsClassPutUnauthorizedCode int = 401 - -/* -ObjectsClassPutUnauthorized Unauthorized or invalid credentials. - -swagger:response objectsClassPutUnauthorized -*/ -type ObjectsClassPutUnauthorized struct { -} - -// NewObjectsClassPutUnauthorized creates ObjectsClassPutUnauthorized with default headers values -func NewObjectsClassPutUnauthorized() *ObjectsClassPutUnauthorized { - - return &ObjectsClassPutUnauthorized{} -} - -// WriteResponse to the client -func (o *ObjectsClassPutUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// ObjectsClassPutForbiddenCode is the HTTP code returned for type ObjectsClassPutForbidden -const ObjectsClassPutForbiddenCode int = 403 - -/* -ObjectsClassPutForbidden Forbidden - -swagger:response objectsClassPutForbidden -*/ -type ObjectsClassPutForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsClassPutForbidden creates ObjectsClassPutForbidden with default headers values -func NewObjectsClassPutForbidden() *ObjectsClassPutForbidden { - - return &ObjectsClassPutForbidden{} -} - -// WithPayload adds the payload to the objects class put forbidden response -func (o *ObjectsClassPutForbidden) WithPayload(payload *models.ErrorResponse) *ObjectsClassPutForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects class put forbidden response -func (o *ObjectsClassPutForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsClassPutForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsClassPutNotFoundCode is the HTTP code returned for type ObjectsClassPutNotFound -const ObjectsClassPutNotFoundCode int = 404 - -/* -ObjectsClassPutNotFound Successful query result but no resource was found. - -swagger:response objectsClassPutNotFound -*/ -type ObjectsClassPutNotFound struct { -} - -// NewObjectsClassPutNotFound creates ObjectsClassPutNotFound with default headers values -func NewObjectsClassPutNotFound() *ObjectsClassPutNotFound { - - return &ObjectsClassPutNotFound{} -} - -// WriteResponse to the client -func (o *ObjectsClassPutNotFound) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(404) -} - -// ObjectsClassPutUnprocessableEntityCode is the HTTP code returned for type ObjectsClassPutUnprocessableEntity -const ObjectsClassPutUnprocessableEntityCode int = 422 - -/* -ObjectsClassPutUnprocessableEntity Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file? - -swagger:response objectsClassPutUnprocessableEntity -*/ -type ObjectsClassPutUnprocessableEntity struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsClassPutUnprocessableEntity creates ObjectsClassPutUnprocessableEntity with default headers values -func NewObjectsClassPutUnprocessableEntity() *ObjectsClassPutUnprocessableEntity { - - return &ObjectsClassPutUnprocessableEntity{} -} - -// WithPayload adds the payload to the objects class put unprocessable entity response -func (o *ObjectsClassPutUnprocessableEntity) WithPayload(payload *models.ErrorResponse) *ObjectsClassPutUnprocessableEntity { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects class put unprocessable entity response -func (o *ObjectsClassPutUnprocessableEntity) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsClassPutUnprocessableEntity) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(422) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsClassPutInternalServerErrorCode is the HTTP code returned for type ObjectsClassPutInternalServerError -const ObjectsClassPutInternalServerErrorCode int = 500 - -/* -ObjectsClassPutInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response objectsClassPutInternalServerError -*/ -type ObjectsClassPutInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsClassPutInternalServerError creates ObjectsClassPutInternalServerError with default headers values -func NewObjectsClassPutInternalServerError() *ObjectsClassPutInternalServerError { - - return &ObjectsClassPutInternalServerError{} -} - -// WithPayload adds the payload to the objects class put internal server error response -func (o *ObjectsClassPutInternalServerError) WithPayload(payload *models.ErrorResponse) *ObjectsClassPutInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects class put internal server error response -func (o *ObjectsClassPutInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsClassPutInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/objects/objects_class_put_urlbuilder.go b/adapters/handlers/rest/operations/objects/objects_class_put_urlbuilder.go deleted file mode 100644 index 95e200195ab7cba674fb10abf8e9cfd651cdd629..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_class_put_urlbuilder.go +++ /dev/null @@ -1,134 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" - "strings" - - "github.com/go-openapi/strfmt" -) - -// ObjectsClassPutURL generates an URL for the objects class put operation -type ObjectsClassPutURL struct { - ClassName string - ID strfmt.UUID - - ConsistencyLevel *string - - _basePath string - // avoid unkeyed usage - _ struct{} -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *ObjectsClassPutURL) WithBasePath(bp string) *ObjectsClassPutURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *ObjectsClassPutURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *ObjectsClassPutURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/objects/{className}/{id}" - - className := o.ClassName - if className != "" { - _path = strings.Replace(_path, "{className}", className, -1) - } else { - return nil, errors.New("className is required on ObjectsClassPutURL") - } - - id := o.ID.String() - if id != "" { - _path = strings.Replace(_path, "{id}", id, -1) - } else { - return nil, errors.New("id is required on ObjectsClassPutURL") - } - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - qs := make(url.Values) - - var consistencyLevelQ string - if o.ConsistencyLevel != nil { - consistencyLevelQ = *o.ConsistencyLevel - } - if consistencyLevelQ != "" { - qs.Set("consistency_level", consistencyLevelQ) - } - - _result.RawQuery = qs.Encode() - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *ObjectsClassPutURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *ObjectsClassPutURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *ObjectsClassPutURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on ObjectsClassPutURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on ObjectsClassPutURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *ObjectsClassPutURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/objects/objects_class_references_create.go b/adapters/handlers/rest/operations/objects/objects_class_references_create.go deleted file mode 100644 index 0e900f6425c7ce35b399d487c11fc005319e3965..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_class_references_create.go +++ /dev/null @@ -1,84 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsClassReferencesCreateHandlerFunc turns a function with the right signature into a objects class references create handler -type ObjectsClassReferencesCreateHandlerFunc func(ObjectsClassReferencesCreateParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn ObjectsClassReferencesCreateHandlerFunc) Handle(params ObjectsClassReferencesCreateParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// ObjectsClassReferencesCreateHandler interface for that can handle valid objects class references create params -type ObjectsClassReferencesCreateHandler interface { - Handle(ObjectsClassReferencesCreateParams, *models.Principal) middleware.Responder -} - -// NewObjectsClassReferencesCreate creates a new http.Handler for the objects class references create operation -func NewObjectsClassReferencesCreate(ctx *middleware.Context, handler ObjectsClassReferencesCreateHandler) *ObjectsClassReferencesCreate { - return &ObjectsClassReferencesCreate{Context: ctx, Handler: handler} -} - -/* - ObjectsClassReferencesCreate swagger:route POST /objects/{className}/{id}/references/{propertyName} objects objectsClassReferencesCreate - -Add a single reference to a class-property. - -Add a single reference to a class-property. -*/ -type ObjectsClassReferencesCreate struct { - Context *middleware.Context - Handler ObjectsClassReferencesCreateHandler -} - -func (o *ObjectsClassReferencesCreate) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewObjectsClassReferencesCreateParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/objects/objects_class_references_create_parameters.go b/adapters/handlers/rest/operations/objects/objects_class_references_create_parameters.go deleted file mode 100644 index ba05847bd8644e403f04d7cb5a20099c2b8fe966..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_class_references_create_parameters.go +++ /dev/null @@ -1,243 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "io" - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/validate" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewObjectsClassReferencesCreateParams creates a new ObjectsClassReferencesCreateParams object -// -// There are no default values defined in the spec. -func NewObjectsClassReferencesCreateParams() ObjectsClassReferencesCreateParams { - - return ObjectsClassReferencesCreateParams{} -} - -// ObjectsClassReferencesCreateParams contains all the bound params for the objects class references create operation -// typically these are obtained from a http.Request -// -// swagger:parameters objects.class.references.create -type ObjectsClassReferencesCreateParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` - - /* - Required: true - In: body - */ - Body *models.SingleRef - /*The class name as defined in the schema - Required: true - In: path - */ - ClassName string - /*Determines how many replicas must acknowledge a request before it is considered successful - In: query - */ - ConsistencyLevel *string - /*Unique ID of the Object. - Required: true - In: path - */ - ID strfmt.UUID - /*Unique name of the property related to the Object. - Required: true - In: path - */ - PropertyName string - /*Specifies the tenant in a request targeting a multi-tenant class - In: query - */ - Tenant *string -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewObjectsClassReferencesCreateParams() beforehand. -func (o *ObjectsClassReferencesCreateParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - qs := runtime.Values(r.URL.Query()) - - if runtime.HasBody(r) { - defer r.Body.Close() - var body models.SingleRef - if err := route.Consumer.Consume(r.Body, &body); err != nil { - if err == io.EOF { - res = append(res, errors.Required("body", "body", "")) - } else { - res = append(res, errors.NewParseError("body", "body", "", err)) - } - } else { - // validate body object - if err := body.Validate(route.Formats); err != nil { - res = append(res, err) - } - - ctx := validate.WithOperationRequest(r.Context()) - if err := body.ContextValidate(ctx, route.Formats); err != nil { - res = append(res, err) - } - - if len(res) == 0 { - o.Body = &body - } - } - } else { - res = append(res, errors.Required("body", "body", "")) - } - - rClassName, rhkClassName, _ := route.Params.GetOK("className") - if err := o.bindClassName(rClassName, rhkClassName, route.Formats); err != nil { - res = append(res, err) - } - - qConsistencyLevel, qhkConsistencyLevel, _ := qs.GetOK("consistency_level") - if err := o.bindConsistencyLevel(qConsistencyLevel, qhkConsistencyLevel, route.Formats); err != nil { - res = append(res, err) - } - - rID, rhkID, _ := route.Params.GetOK("id") - if err := o.bindID(rID, rhkID, route.Formats); err != nil { - res = append(res, err) - } - - rPropertyName, rhkPropertyName, _ := route.Params.GetOK("propertyName") - if err := o.bindPropertyName(rPropertyName, rhkPropertyName, route.Formats); err != nil { - res = append(res, err) - } - - qTenant, qhkTenant, _ := qs.GetOK("tenant") - if err := o.bindTenant(qTenant, qhkTenant, route.Formats); err != nil { - res = append(res, err) - } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// bindClassName binds and validates parameter ClassName from path. -func (o *ObjectsClassReferencesCreateParams) bindClassName(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - o.ClassName = raw - - return nil -} - -// bindConsistencyLevel binds and validates parameter ConsistencyLevel from query. -func (o *ObjectsClassReferencesCreateParams) bindConsistencyLevel(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: false - // AllowEmptyValue: false - - if raw == "" { // empty values pass all other validations - return nil - } - o.ConsistencyLevel = &raw - - return nil -} - -// bindID binds and validates parameter ID from path. -func (o *ObjectsClassReferencesCreateParams) bindID(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - - // Format: uuid - value, err := formats.Parse("uuid", raw) - if err != nil { - return errors.InvalidType("id", "path", "strfmt.UUID", raw) - } - o.ID = *(value.(*strfmt.UUID)) - - if err := o.validateID(formats); err != nil { - return err - } - - return nil -} - -// validateID carries on validations for parameter ID -func (o *ObjectsClassReferencesCreateParams) validateID(formats strfmt.Registry) error { - - if err := validate.FormatOf("id", "path", "uuid", o.ID.String(), formats); err != nil { - return err - } - return nil -} - -// bindPropertyName binds and validates parameter PropertyName from path. -func (o *ObjectsClassReferencesCreateParams) bindPropertyName(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - o.PropertyName = raw - - return nil -} - -// bindTenant binds and validates parameter Tenant from query. -func (o *ObjectsClassReferencesCreateParams) bindTenant(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: false - // AllowEmptyValue: false - - if raw == "" { // empty values pass all other validations - return nil - } - o.Tenant = &raw - - return nil -} diff --git a/adapters/handlers/rest/operations/objects/objects_class_references_create_responses.go b/adapters/handlers/rest/operations/objects/objects_class_references_create_responses.go deleted file mode 100644 index 2de737175f377954bc95747dcb8ba9ec9f4aa17a..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_class_references_create_responses.go +++ /dev/null @@ -1,280 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsClassReferencesCreateOKCode is the HTTP code returned for type ObjectsClassReferencesCreateOK -const ObjectsClassReferencesCreateOKCode int = 200 - -/* -ObjectsClassReferencesCreateOK Successfully added the reference. - -swagger:response objectsClassReferencesCreateOK -*/ -type ObjectsClassReferencesCreateOK struct { -} - -// NewObjectsClassReferencesCreateOK creates ObjectsClassReferencesCreateOK with default headers values -func NewObjectsClassReferencesCreateOK() *ObjectsClassReferencesCreateOK { - - return &ObjectsClassReferencesCreateOK{} -} - -// WriteResponse to the client -func (o *ObjectsClassReferencesCreateOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(200) -} - -// ObjectsClassReferencesCreateBadRequestCode is the HTTP code returned for type ObjectsClassReferencesCreateBadRequest -const ObjectsClassReferencesCreateBadRequestCode int = 400 - -/* -ObjectsClassReferencesCreateBadRequest Malformed request. - -swagger:response objectsClassReferencesCreateBadRequest -*/ -type ObjectsClassReferencesCreateBadRequest struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsClassReferencesCreateBadRequest creates ObjectsClassReferencesCreateBadRequest with default headers values -func NewObjectsClassReferencesCreateBadRequest() *ObjectsClassReferencesCreateBadRequest { - - return &ObjectsClassReferencesCreateBadRequest{} -} - -// WithPayload adds the payload to the objects class references create bad request response -func (o *ObjectsClassReferencesCreateBadRequest) WithPayload(payload *models.ErrorResponse) *ObjectsClassReferencesCreateBadRequest { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects class references create bad request response -func (o *ObjectsClassReferencesCreateBadRequest) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsClassReferencesCreateBadRequest) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(400) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsClassReferencesCreateUnauthorizedCode is the HTTP code returned for type ObjectsClassReferencesCreateUnauthorized -const ObjectsClassReferencesCreateUnauthorizedCode int = 401 - -/* -ObjectsClassReferencesCreateUnauthorized Unauthorized or invalid credentials. - -swagger:response objectsClassReferencesCreateUnauthorized -*/ -type ObjectsClassReferencesCreateUnauthorized struct { -} - -// NewObjectsClassReferencesCreateUnauthorized creates ObjectsClassReferencesCreateUnauthorized with default headers values -func NewObjectsClassReferencesCreateUnauthorized() *ObjectsClassReferencesCreateUnauthorized { - - return &ObjectsClassReferencesCreateUnauthorized{} -} - -// WriteResponse to the client -func (o *ObjectsClassReferencesCreateUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// ObjectsClassReferencesCreateForbiddenCode is the HTTP code returned for type ObjectsClassReferencesCreateForbidden -const ObjectsClassReferencesCreateForbiddenCode int = 403 - -/* -ObjectsClassReferencesCreateForbidden Forbidden - -swagger:response objectsClassReferencesCreateForbidden -*/ -type ObjectsClassReferencesCreateForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsClassReferencesCreateForbidden creates ObjectsClassReferencesCreateForbidden with default headers values -func NewObjectsClassReferencesCreateForbidden() *ObjectsClassReferencesCreateForbidden { - - return &ObjectsClassReferencesCreateForbidden{} -} - -// WithPayload adds the payload to the objects class references create forbidden response -func (o *ObjectsClassReferencesCreateForbidden) WithPayload(payload *models.ErrorResponse) *ObjectsClassReferencesCreateForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects class references create forbidden response -func (o *ObjectsClassReferencesCreateForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsClassReferencesCreateForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsClassReferencesCreateNotFoundCode is the HTTP code returned for type ObjectsClassReferencesCreateNotFound -const ObjectsClassReferencesCreateNotFoundCode int = 404 - -/* -ObjectsClassReferencesCreateNotFound Source object doesn't exist. - -swagger:response objectsClassReferencesCreateNotFound -*/ -type ObjectsClassReferencesCreateNotFound struct { -} - -// NewObjectsClassReferencesCreateNotFound creates ObjectsClassReferencesCreateNotFound with default headers values -func NewObjectsClassReferencesCreateNotFound() *ObjectsClassReferencesCreateNotFound { - - return &ObjectsClassReferencesCreateNotFound{} -} - -// WriteResponse to the client -func (o *ObjectsClassReferencesCreateNotFound) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(404) -} - -// ObjectsClassReferencesCreateUnprocessableEntityCode is the HTTP code returned for type ObjectsClassReferencesCreateUnprocessableEntity -const ObjectsClassReferencesCreateUnprocessableEntityCode int = 422 - -/* -ObjectsClassReferencesCreateUnprocessableEntity Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the property exists or that it is a class? - -swagger:response objectsClassReferencesCreateUnprocessableEntity -*/ -type ObjectsClassReferencesCreateUnprocessableEntity struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsClassReferencesCreateUnprocessableEntity creates ObjectsClassReferencesCreateUnprocessableEntity with default headers values -func NewObjectsClassReferencesCreateUnprocessableEntity() *ObjectsClassReferencesCreateUnprocessableEntity { - - return &ObjectsClassReferencesCreateUnprocessableEntity{} -} - -// WithPayload adds the payload to the objects class references create unprocessable entity response -func (o *ObjectsClassReferencesCreateUnprocessableEntity) WithPayload(payload *models.ErrorResponse) *ObjectsClassReferencesCreateUnprocessableEntity { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects class references create unprocessable entity response -func (o *ObjectsClassReferencesCreateUnprocessableEntity) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsClassReferencesCreateUnprocessableEntity) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(422) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsClassReferencesCreateInternalServerErrorCode is the HTTP code returned for type ObjectsClassReferencesCreateInternalServerError -const ObjectsClassReferencesCreateInternalServerErrorCode int = 500 - -/* -ObjectsClassReferencesCreateInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response objectsClassReferencesCreateInternalServerError -*/ -type ObjectsClassReferencesCreateInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsClassReferencesCreateInternalServerError creates ObjectsClassReferencesCreateInternalServerError with default headers values -func NewObjectsClassReferencesCreateInternalServerError() *ObjectsClassReferencesCreateInternalServerError { - - return &ObjectsClassReferencesCreateInternalServerError{} -} - -// WithPayload adds the payload to the objects class references create internal server error response -func (o *ObjectsClassReferencesCreateInternalServerError) WithPayload(payload *models.ErrorResponse) *ObjectsClassReferencesCreateInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects class references create internal server error response -func (o *ObjectsClassReferencesCreateInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsClassReferencesCreateInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/objects/objects_class_references_create_urlbuilder.go b/adapters/handlers/rest/operations/objects/objects_class_references_create_urlbuilder.go deleted file mode 100644 index 844dd6510ef413e5721dd3c7d6019e3c85fadcfd..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_class_references_create_urlbuilder.go +++ /dev/null @@ -1,151 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" - "strings" - - "github.com/go-openapi/strfmt" -) - -// ObjectsClassReferencesCreateURL generates an URL for the objects class references create operation -type ObjectsClassReferencesCreateURL struct { - ClassName string - ID strfmt.UUID - PropertyName string - - ConsistencyLevel *string - Tenant *string - - _basePath string - // avoid unkeyed usage - _ struct{} -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *ObjectsClassReferencesCreateURL) WithBasePath(bp string) *ObjectsClassReferencesCreateURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *ObjectsClassReferencesCreateURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *ObjectsClassReferencesCreateURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/objects/{className}/{id}/references/{propertyName}" - - className := o.ClassName - if className != "" { - _path = strings.Replace(_path, "{className}", className, -1) - } else { - return nil, errors.New("className is required on ObjectsClassReferencesCreateURL") - } - - id := o.ID.String() - if id != "" { - _path = strings.Replace(_path, "{id}", id, -1) - } else { - return nil, errors.New("id is required on ObjectsClassReferencesCreateURL") - } - - propertyName := o.PropertyName - if propertyName != "" { - _path = strings.Replace(_path, "{propertyName}", propertyName, -1) - } else { - return nil, errors.New("propertyName is required on ObjectsClassReferencesCreateURL") - } - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - qs := make(url.Values) - - var consistencyLevelQ string - if o.ConsistencyLevel != nil { - consistencyLevelQ = *o.ConsistencyLevel - } - if consistencyLevelQ != "" { - qs.Set("consistency_level", consistencyLevelQ) - } - - var tenantQ string - if o.Tenant != nil { - tenantQ = *o.Tenant - } - if tenantQ != "" { - qs.Set("tenant", tenantQ) - } - - _result.RawQuery = qs.Encode() - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *ObjectsClassReferencesCreateURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *ObjectsClassReferencesCreateURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *ObjectsClassReferencesCreateURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on ObjectsClassReferencesCreateURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on ObjectsClassReferencesCreateURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *ObjectsClassReferencesCreateURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/objects/objects_class_references_delete.go b/adapters/handlers/rest/operations/objects/objects_class_references_delete.go deleted file mode 100644 index 98ce49b41b09f88aeead17a01519a058abef3bd1..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_class_references_delete.go +++ /dev/null @@ -1,84 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsClassReferencesDeleteHandlerFunc turns a function with the right signature into a objects class references delete handler -type ObjectsClassReferencesDeleteHandlerFunc func(ObjectsClassReferencesDeleteParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn ObjectsClassReferencesDeleteHandlerFunc) Handle(params ObjectsClassReferencesDeleteParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// ObjectsClassReferencesDeleteHandler interface for that can handle valid objects class references delete params -type ObjectsClassReferencesDeleteHandler interface { - Handle(ObjectsClassReferencesDeleteParams, *models.Principal) middleware.Responder -} - -// NewObjectsClassReferencesDelete creates a new http.Handler for the objects class references delete operation -func NewObjectsClassReferencesDelete(ctx *middleware.Context, handler ObjectsClassReferencesDeleteHandler) *ObjectsClassReferencesDelete { - return &ObjectsClassReferencesDelete{Context: ctx, Handler: handler} -} - -/* - ObjectsClassReferencesDelete swagger:route DELETE /objects/{className}/{id}/references/{propertyName} objects objectsClassReferencesDelete - -Delete the single reference that is given in the body from the list of references that this property has. - -Delete the single reference that is given in the body from the list of references that this property of a data object has -*/ -type ObjectsClassReferencesDelete struct { - Context *middleware.Context - Handler ObjectsClassReferencesDeleteHandler -} - -func (o *ObjectsClassReferencesDelete) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewObjectsClassReferencesDeleteParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/objects/objects_class_references_delete_parameters.go b/adapters/handlers/rest/operations/objects/objects_class_references_delete_parameters.go deleted file mode 100644 index c98f363bb342ca261fa8025b86a3e1d8c99a8851..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_class_references_delete_parameters.go +++ /dev/null @@ -1,243 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "io" - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/validate" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewObjectsClassReferencesDeleteParams creates a new ObjectsClassReferencesDeleteParams object -// -// There are no default values defined in the spec. -func NewObjectsClassReferencesDeleteParams() ObjectsClassReferencesDeleteParams { - - return ObjectsClassReferencesDeleteParams{} -} - -// ObjectsClassReferencesDeleteParams contains all the bound params for the objects class references delete operation -// typically these are obtained from a http.Request -// -// swagger:parameters objects.class.references.delete -type ObjectsClassReferencesDeleteParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` - - /* - Required: true - In: body - */ - Body *models.SingleRef - /*The class name as defined in the schema - Required: true - In: path - */ - ClassName string - /*Determines how many replicas must acknowledge a request before it is considered successful - In: query - */ - ConsistencyLevel *string - /*Unique ID of the Object. - Required: true - In: path - */ - ID strfmt.UUID - /*Unique name of the property related to the Object. - Required: true - In: path - */ - PropertyName string - /*Specifies the tenant in a request targeting a multi-tenant class - In: query - */ - Tenant *string -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewObjectsClassReferencesDeleteParams() beforehand. -func (o *ObjectsClassReferencesDeleteParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - qs := runtime.Values(r.URL.Query()) - - if runtime.HasBody(r) { - defer r.Body.Close() - var body models.SingleRef - if err := route.Consumer.Consume(r.Body, &body); err != nil { - if err == io.EOF { - res = append(res, errors.Required("body", "body", "")) - } else { - res = append(res, errors.NewParseError("body", "body", "", err)) - } - } else { - // validate body object - if err := body.Validate(route.Formats); err != nil { - res = append(res, err) - } - - ctx := validate.WithOperationRequest(r.Context()) - if err := body.ContextValidate(ctx, route.Formats); err != nil { - res = append(res, err) - } - - if len(res) == 0 { - o.Body = &body - } - } - } else { - res = append(res, errors.Required("body", "body", "")) - } - - rClassName, rhkClassName, _ := route.Params.GetOK("className") - if err := o.bindClassName(rClassName, rhkClassName, route.Formats); err != nil { - res = append(res, err) - } - - qConsistencyLevel, qhkConsistencyLevel, _ := qs.GetOK("consistency_level") - if err := o.bindConsistencyLevel(qConsistencyLevel, qhkConsistencyLevel, route.Formats); err != nil { - res = append(res, err) - } - - rID, rhkID, _ := route.Params.GetOK("id") - if err := o.bindID(rID, rhkID, route.Formats); err != nil { - res = append(res, err) - } - - rPropertyName, rhkPropertyName, _ := route.Params.GetOK("propertyName") - if err := o.bindPropertyName(rPropertyName, rhkPropertyName, route.Formats); err != nil { - res = append(res, err) - } - - qTenant, qhkTenant, _ := qs.GetOK("tenant") - if err := o.bindTenant(qTenant, qhkTenant, route.Formats); err != nil { - res = append(res, err) - } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// bindClassName binds and validates parameter ClassName from path. -func (o *ObjectsClassReferencesDeleteParams) bindClassName(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - o.ClassName = raw - - return nil -} - -// bindConsistencyLevel binds and validates parameter ConsistencyLevel from query. -func (o *ObjectsClassReferencesDeleteParams) bindConsistencyLevel(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: false - // AllowEmptyValue: false - - if raw == "" { // empty values pass all other validations - return nil - } - o.ConsistencyLevel = &raw - - return nil -} - -// bindID binds and validates parameter ID from path. -func (o *ObjectsClassReferencesDeleteParams) bindID(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - - // Format: uuid - value, err := formats.Parse("uuid", raw) - if err != nil { - return errors.InvalidType("id", "path", "strfmt.UUID", raw) - } - o.ID = *(value.(*strfmt.UUID)) - - if err := o.validateID(formats); err != nil { - return err - } - - return nil -} - -// validateID carries on validations for parameter ID -func (o *ObjectsClassReferencesDeleteParams) validateID(formats strfmt.Registry) error { - - if err := validate.FormatOf("id", "path", "uuid", o.ID.String(), formats); err != nil { - return err - } - return nil -} - -// bindPropertyName binds and validates parameter PropertyName from path. -func (o *ObjectsClassReferencesDeleteParams) bindPropertyName(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - o.PropertyName = raw - - return nil -} - -// bindTenant binds and validates parameter Tenant from query. -func (o *ObjectsClassReferencesDeleteParams) bindTenant(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: false - // AllowEmptyValue: false - - if raw == "" { // empty values pass all other validations - return nil - } - o.Tenant = &raw - - return nil -} diff --git a/adapters/handlers/rest/operations/objects/objects_class_references_delete_responses.go b/adapters/handlers/rest/operations/objects/objects_class_references_delete_responses.go deleted file mode 100644 index 06765ef357e9db72908a31dc49c79cac841deb8d..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_class_references_delete_responses.go +++ /dev/null @@ -1,300 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsClassReferencesDeleteNoContentCode is the HTTP code returned for type ObjectsClassReferencesDeleteNoContent -const ObjectsClassReferencesDeleteNoContentCode int = 204 - -/* -ObjectsClassReferencesDeleteNoContent Successfully deleted. - -swagger:response objectsClassReferencesDeleteNoContent -*/ -type ObjectsClassReferencesDeleteNoContent struct { -} - -// NewObjectsClassReferencesDeleteNoContent creates ObjectsClassReferencesDeleteNoContent with default headers values -func NewObjectsClassReferencesDeleteNoContent() *ObjectsClassReferencesDeleteNoContent { - - return &ObjectsClassReferencesDeleteNoContent{} -} - -// WriteResponse to the client -func (o *ObjectsClassReferencesDeleteNoContent) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(204) -} - -// ObjectsClassReferencesDeleteBadRequestCode is the HTTP code returned for type ObjectsClassReferencesDeleteBadRequest -const ObjectsClassReferencesDeleteBadRequestCode int = 400 - -/* -ObjectsClassReferencesDeleteBadRequest Malformed request. - -swagger:response objectsClassReferencesDeleteBadRequest -*/ -type ObjectsClassReferencesDeleteBadRequest struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsClassReferencesDeleteBadRequest creates ObjectsClassReferencesDeleteBadRequest with default headers values -func NewObjectsClassReferencesDeleteBadRequest() *ObjectsClassReferencesDeleteBadRequest { - - return &ObjectsClassReferencesDeleteBadRequest{} -} - -// WithPayload adds the payload to the objects class references delete bad request response -func (o *ObjectsClassReferencesDeleteBadRequest) WithPayload(payload *models.ErrorResponse) *ObjectsClassReferencesDeleteBadRequest { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects class references delete bad request response -func (o *ObjectsClassReferencesDeleteBadRequest) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsClassReferencesDeleteBadRequest) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(400) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsClassReferencesDeleteUnauthorizedCode is the HTTP code returned for type ObjectsClassReferencesDeleteUnauthorized -const ObjectsClassReferencesDeleteUnauthorizedCode int = 401 - -/* -ObjectsClassReferencesDeleteUnauthorized Unauthorized or invalid credentials. - -swagger:response objectsClassReferencesDeleteUnauthorized -*/ -type ObjectsClassReferencesDeleteUnauthorized struct { -} - -// NewObjectsClassReferencesDeleteUnauthorized creates ObjectsClassReferencesDeleteUnauthorized with default headers values -func NewObjectsClassReferencesDeleteUnauthorized() *ObjectsClassReferencesDeleteUnauthorized { - - return &ObjectsClassReferencesDeleteUnauthorized{} -} - -// WriteResponse to the client -func (o *ObjectsClassReferencesDeleteUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// ObjectsClassReferencesDeleteForbiddenCode is the HTTP code returned for type ObjectsClassReferencesDeleteForbidden -const ObjectsClassReferencesDeleteForbiddenCode int = 403 - -/* -ObjectsClassReferencesDeleteForbidden Forbidden - -swagger:response objectsClassReferencesDeleteForbidden -*/ -type ObjectsClassReferencesDeleteForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsClassReferencesDeleteForbidden creates ObjectsClassReferencesDeleteForbidden with default headers values -func NewObjectsClassReferencesDeleteForbidden() *ObjectsClassReferencesDeleteForbidden { - - return &ObjectsClassReferencesDeleteForbidden{} -} - -// WithPayload adds the payload to the objects class references delete forbidden response -func (o *ObjectsClassReferencesDeleteForbidden) WithPayload(payload *models.ErrorResponse) *ObjectsClassReferencesDeleteForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects class references delete forbidden response -func (o *ObjectsClassReferencesDeleteForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsClassReferencesDeleteForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsClassReferencesDeleteNotFoundCode is the HTTP code returned for type ObjectsClassReferencesDeleteNotFound -const ObjectsClassReferencesDeleteNotFoundCode int = 404 - -/* -ObjectsClassReferencesDeleteNotFound Successful query result but no resource was found. - -swagger:response objectsClassReferencesDeleteNotFound -*/ -type ObjectsClassReferencesDeleteNotFound struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsClassReferencesDeleteNotFound creates ObjectsClassReferencesDeleteNotFound with default headers values -func NewObjectsClassReferencesDeleteNotFound() *ObjectsClassReferencesDeleteNotFound { - - return &ObjectsClassReferencesDeleteNotFound{} -} - -// WithPayload adds the payload to the objects class references delete not found response -func (o *ObjectsClassReferencesDeleteNotFound) WithPayload(payload *models.ErrorResponse) *ObjectsClassReferencesDeleteNotFound { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects class references delete not found response -func (o *ObjectsClassReferencesDeleteNotFound) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsClassReferencesDeleteNotFound) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(404) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsClassReferencesDeleteUnprocessableEntityCode is the HTTP code returned for type ObjectsClassReferencesDeleteUnprocessableEntity -const ObjectsClassReferencesDeleteUnprocessableEntityCode int = 422 - -/* -ObjectsClassReferencesDeleteUnprocessableEntity Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the property exists or that it is a class? - -swagger:response objectsClassReferencesDeleteUnprocessableEntity -*/ -type ObjectsClassReferencesDeleteUnprocessableEntity struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsClassReferencesDeleteUnprocessableEntity creates ObjectsClassReferencesDeleteUnprocessableEntity with default headers values -func NewObjectsClassReferencesDeleteUnprocessableEntity() *ObjectsClassReferencesDeleteUnprocessableEntity { - - return &ObjectsClassReferencesDeleteUnprocessableEntity{} -} - -// WithPayload adds the payload to the objects class references delete unprocessable entity response -func (o *ObjectsClassReferencesDeleteUnprocessableEntity) WithPayload(payload *models.ErrorResponse) *ObjectsClassReferencesDeleteUnprocessableEntity { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects class references delete unprocessable entity response -func (o *ObjectsClassReferencesDeleteUnprocessableEntity) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsClassReferencesDeleteUnprocessableEntity) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(422) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsClassReferencesDeleteInternalServerErrorCode is the HTTP code returned for type ObjectsClassReferencesDeleteInternalServerError -const ObjectsClassReferencesDeleteInternalServerErrorCode int = 500 - -/* -ObjectsClassReferencesDeleteInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response objectsClassReferencesDeleteInternalServerError -*/ -type ObjectsClassReferencesDeleteInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsClassReferencesDeleteInternalServerError creates ObjectsClassReferencesDeleteInternalServerError with default headers values -func NewObjectsClassReferencesDeleteInternalServerError() *ObjectsClassReferencesDeleteInternalServerError { - - return &ObjectsClassReferencesDeleteInternalServerError{} -} - -// WithPayload adds the payload to the objects class references delete internal server error response -func (o *ObjectsClassReferencesDeleteInternalServerError) WithPayload(payload *models.ErrorResponse) *ObjectsClassReferencesDeleteInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects class references delete internal server error response -func (o *ObjectsClassReferencesDeleteInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsClassReferencesDeleteInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/objects/objects_class_references_delete_urlbuilder.go b/adapters/handlers/rest/operations/objects/objects_class_references_delete_urlbuilder.go deleted file mode 100644 index 3ff76bada67796d60753848ac2fae8c637ce6f91..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_class_references_delete_urlbuilder.go +++ /dev/null @@ -1,151 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" - "strings" - - "github.com/go-openapi/strfmt" -) - -// ObjectsClassReferencesDeleteURL generates an URL for the objects class references delete operation -type ObjectsClassReferencesDeleteURL struct { - ClassName string - ID strfmt.UUID - PropertyName string - - ConsistencyLevel *string - Tenant *string - - _basePath string - // avoid unkeyed usage - _ struct{} -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *ObjectsClassReferencesDeleteURL) WithBasePath(bp string) *ObjectsClassReferencesDeleteURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *ObjectsClassReferencesDeleteURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *ObjectsClassReferencesDeleteURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/objects/{className}/{id}/references/{propertyName}" - - className := o.ClassName - if className != "" { - _path = strings.Replace(_path, "{className}", className, -1) - } else { - return nil, errors.New("className is required on ObjectsClassReferencesDeleteURL") - } - - id := o.ID.String() - if id != "" { - _path = strings.Replace(_path, "{id}", id, -1) - } else { - return nil, errors.New("id is required on ObjectsClassReferencesDeleteURL") - } - - propertyName := o.PropertyName - if propertyName != "" { - _path = strings.Replace(_path, "{propertyName}", propertyName, -1) - } else { - return nil, errors.New("propertyName is required on ObjectsClassReferencesDeleteURL") - } - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - qs := make(url.Values) - - var consistencyLevelQ string - if o.ConsistencyLevel != nil { - consistencyLevelQ = *o.ConsistencyLevel - } - if consistencyLevelQ != "" { - qs.Set("consistency_level", consistencyLevelQ) - } - - var tenantQ string - if o.Tenant != nil { - tenantQ = *o.Tenant - } - if tenantQ != "" { - qs.Set("tenant", tenantQ) - } - - _result.RawQuery = qs.Encode() - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *ObjectsClassReferencesDeleteURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *ObjectsClassReferencesDeleteURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *ObjectsClassReferencesDeleteURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on ObjectsClassReferencesDeleteURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on ObjectsClassReferencesDeleteURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *ObjectsClassReferencesDeleteURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/objects/objects_class_references_put.go b/adapters/handlers/rest/operations/objects/objects_class_references_put.go deleted file mode 100644 index 9d0a266560c99bc15db293795db257e951c9f219..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_class_references_put.go +++ /dev/null @@ -1,84 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsClassReferencesPutHandlerFunc turns a function with the right signature into a objects class references put handler -type ObjectsClassReferencesPutHandlerFunc func(ObjectsClassReferencesPutParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn ObjectsClassReferencesPutHandlerFunc) Handle(params ObjectsClassReferencesPutParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// ObjectsClassReferencesPutHandler interface for that can handle valid objects class references put params -type ObjectsClassReferencesPutHandler interface { - Handle(ObjectsClassReferencesPutParams, *models.Principal) middleware.Responder -} - -// NewObjectsClassReferencesPut creates a new http.Handler for the objects class references put operation -func NewObjectsClassReferencesPut(ctx *middleware.Context, handler ObjectsClassReferencesPutHandler) *ObjectsClassReferencesPut { - return &ObjectsClassReferencesPut{Context: ctx, Handler: handler} -} - -/* - ObjectsClassReferencesPut swagger:route PUT /objects/{className}/{id}/references/{propertyName} objects objectsClassReferencesPut - -Replace all references to a class-property. - -Update all references of a property of a data object. -*/ -type ObjectsClassReferencesPut struct { - Context *middleware.Context - Handler ObjectsClassReferencesPutHandler -} - -func (o *ObjectsClassReferencesPut) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewObjectsClassReferencesPutParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/objects/objects_class_references_put_parameters.go b/adapters/handlers/rest/operations/objects/objects_class_references_put_parameters.go deleted file mode 100644 index 6508df282eab8beee98d805ac814ae4df1e1e997..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_class_references_put_parameters.go +++ /dev/null @@ -1,243 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "io" - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/validate" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewObjectsClassReferencesPutParams creates a new ObjectsClassReferencesPutParams object -// -// There are no default values defined in the spec. -func NewObjectsClassReferencesPutParams() ObjectsClassReferencesPutParams { - - return ObjectsClassReferencesPutParams{} -} - -// ObjectsClassReferencesPutParams contains all the bound params for the objects class references put operation -// typically these are obtained from a http.Request -// -// swagger:parameters objects.class.references.put -type ObjectsClassReferencesPutParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` - - /* - Required: true - In: body - */ - Body models.MultipleRef - /*The class name as defined in the schema - Required: true - In: path - */ - ClassName string - /*Determines how many replicas must acknowledge a request before it is considered successful - In: query - */ - ConsistencyLevel *string - /*Unique ID of the Object. - Required: true - In: path - */ - ID strfmt.UUID - /*Unique name of the property related to the Object. - Required: true - In: path - */ - PropertyName string - /*Specifies the tenant in a request targeting a multi-tenant class - In: query - */ - Tenant *string -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewObjectsClassReferencesPutParams() beforehand. -func (o *ObjectsClassReferencesPutParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - qs := runtime.Values(r.URL.Query()) - - if runtime.HasBody(r) { - defer r.Body.Close() - var body models.MultipleRef - if err := route.Consumer.Consume(r.Body, &body); err != nil { - if err == io.EOF { - res = append(res, errors.Required("body", "body", "")) - } else { - res = append(res, errors.NewParseError("body", "body", "", err)) - } - } else { - // validate body object - if err := body.Validate(route.Formats); err != nil { - res = append(res, err) - } - - ctx := validate.WithOperationRequest(r.Context()) - if err := body.ContextValidate(ctx, route.Formats); err != nil { - res = append(res, err) - } - - if len(res) == 0 { - o.Body = body - } - } - } else { - res = append(res, errors.Required("body", "body", "")) - } - - rClassName, rhkClassName, _ := route.Params.GetOK("className") - if err := o.bindClassName(rClassName, rhkClassName, route.Formats); err != nil { - res = append(res, err) - } - - qConsistencyLevel, qhkConsistencyLevel, _ := qs.GetOK("consistency_level") - if err := o.bindConsistencyLevel(qConsistencyLevel, qhkConsistencyLevel, route.Formats); err != nil { - res = append(res, err) - } - - rID, rhkID, _ := route.Params.GetOK("id") - if err := o.bindID(rID, rhkID, route.Formats); err != nil { - res = append(res, err) - } - - rPropertyName, rhkPropertyName, _ := route.Params.GetOK("propertyName") - if err := o.bindPropertyName(rPropertyName, rhkPropertyName, route.Formats); err != nil { - res = append(res, err) - } - - qTenant, qhkTenant, _ := qs.GetOK("tenant") - if err := o.bindTenant(qTenant, qhkTenant, route.Formats); err != nil { - res = append(res, err) - } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// bindClassName binds and validates parameter ClassName from path. -func (o *ObjectsClassReferencesPutParams) bindClassName(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - o.ClassName = raw - - return nil -} - -// bindConsistencyLevel binds and validates parameter ConsistencyLevel from query. -func (o *ObjectsClassReferencesPutParams) bindConsistencyLevel(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: false - // AllowEmptyValue: false - - if raw == "" { // empty values pass all other validations - return nil - } - o.ConsistencyLevel = &raw - - return nil -} - -// bindID binds and validates parameter ID from path. -func (o *ObjectsClassReferencesPutParams) bindID(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - - // Format: uuid - value, err := formats.Parse("uuid", raw) - if err != nil { - return errors.InvalidType("id", "path", "strfmt.UUID", raw) - } - o.ID = *(value.(*strfmt.UUID)) - - if err := o.validateID(formats); err != nil { - return err - } - - return nil -} - -// validateID carries on validations for parameter ID -func (o *ObjectsClassReferencesPutParams) validateID(formats strfmt.Registry) error { - - if err := validate.FormatOf("id", "path", "uuid", o.ID.String(), formats); err != nil { - return err - } - return nil -} - -// bindPropertyName binds and validates parameter PropertyName from path. -func (o *ObjectsClassReferencesPutParams) bindPropertyName(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - o.PropertyName = raw - - return nil -} - -// bindTenant binds and validates parameter Tenant from query. -func (o *ObjectsClassReferencesPutParams) bindTenant(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: false - // AllowEmptyValue: false - - if raw == "" { // empty values pass all other validations - return nil - } - o.Tenant = &raw - - return nil -} diff --git a/adapters/handlers/rest/operations/objects/objects_class_references_put_responses.go b/adapters/handlers/rest/operations/objects/objects_class_references_put_responses.go deleted file mode 100644 index ad702c7c0bf956a56d6715968603898b6c9ec914..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_class_references_put_responses.go +++ /dev/null @@ -1,280 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsClassReferencesPutOKCode is the HTTP code returned for type ObjectsClassReferencesPutOK -const ObjectsClassReferencesPutOKCode int = 200 - -/* -ObjectsClassReferencesPutOK Successfully replaced all the references. - -swagger:response objectsClassReferencesPutOK -*/ -type ObjectsClassReferencesPutOK struct { -} - -// NewObjectsClassReferencesPutOK creates ObjectsClassReferencesPutOK with default headers values -func NewObjectsClassReferencesPutOK() *ObjectsClassReferencesPutOK { - - return &ObjectsClassReferencesPutOK{} -} - -// WriteResponse to the client -func (o *ObjectsClassReferencesPutOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(200) -} - -// ObjectsClassReferencesPutBadRequestCode is the HTTP code returned for type ObjectsClassReferencesPutBadRequest -const ObjectsClassReferencesPutBadRequestCode int = 400 - -/* -ObjectsClassReferencesPutBadRequest Malformed request. - -swagger:response objectsClassReferencesPutBadRequest -*/ -type ObjectsClassReferencesPutBadRequest struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsClassReferencesPutBadRequest creates ObjectsClassReferencesPutBadRequest with default headers values -func NewObjectsClassReferencesPutBadRequest() *ObjectsClassReferencesPutBadRequest { - - return &ObjectsClassReferencesPutBadRequest{} -} - -// WithPayload adds the payload to the objects class references put bad request response -func (o *ObjectsClassReferencesPutBadRequest) WithPayload(payload *models.ErrorResponse) *ObjectsClassReferencesPutBadRequest { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects class references put bad request response -func (o *ObjectsClassReferencesPutBadRequest) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsClassReferencesPutBadRequest) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(400) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsClassReferencesPutUnauthorizedCode is the HTTP code returned for type ObjectsClassReferencesPutUnauthorized -const ObjectsClassReferencesPutUnauthorizedCode int = 401 - -/* -ObjectsClassReferencesPutUnauthorized Unauthorized or invalid credentials. - -swagger:response objectsClassReferencesPutUnauthorized -*/ -type ObjectsClassReferencesPutUnauthorized struct { -} - -// NewObjectsClassReferencesPutUnauthorized creates ObjectsClassReferencesPutUnauthorized with default headers values -func NewObjectsClassReferencesPutUnauthorized() *ObjectsClassReferencesPutUnauthorized { - - return &ObjectsClassReferencesPutUnauthorized{} -} - -// WriteResponse to the client -func (o *ObjectsClassReferencesPutUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// ObjectsClassReferencesPutForbiddenCode is the HTTP code returned for type ObjectsClassReferencesPutForbidden -const ObjectsClassReferencesPutForbiddenCode int = 403 - -/* -ObjectsClassReferencesPutForbidden Forbidden - -swagger:response objectsClassReferencesPutForbidden -*/ -type ObjectsClassReferencesPutForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsClassReferencesPutForbidden creates ObjectsClassReferencesPutForbidden with default headers values -func NewObjectsClassReferencesPutForbidden() *ObjectsClassReferencesPutForbidden { - - return &ObjectsClassReferencesPutForbidden{} -} - -// WithPayload adds the payload to the objects class references put forbidden response -func (o *ObjectsClassReferencesPutForbidden) WithPayload(payload *models.ErrorResponse) *ObjectsClassReferencesPutForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects class references put forbidden response -func (o *ObjectsClassReferencesPutForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsClassReferencesPutForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsClassReferencesPutNotFoundCode is the HTTP code returned for type ObjectsClassReferencesPutNotFound -const ObjectsClassReferencesPutNotFoundCode int = 404 - -/* -ObjectsClassReferencesPutNotFound Source object doesn't exist. - -swagger:response objectsClassReferencesPutNotFound -*/ -type ObjectsClassReferencesPutNotFound struct { -} - -// NewObjectsClassReferencesPutNotFound creates ObjectsClassReferencesPutNotFound with default headers values -func NewObjectsClassReferencesPutNotFound() *ObjectsClassReferencesPutNotFound { - - return &ObjectsClassReferencesPutNotFound{} -} - -// WriteResponse to the client -func (o *ObjectsClassReferencesPutNotFound) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(404) -} - -// ObjectsClassReferencesPutUnprocessableEntityCode is the HTTP code returned for type ObjectsClassReferencesPutUnprocessableEntity -const ObjectsClassReferencesPutUnprocessableEntityCode int = 422 - -/* -ObjectsClassReferencesPutUnprocessableEntity Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the property exists or that it is a class? - -swagger:response objectsClassReferencesPutUnprocessableEntity -*/ -type ObjectsClassReferencesPutUnprocessableEntity struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsClassReferencesPutUnprocessableEntity creates ObjectsClassReferencesPutUnprocessableEntity with default headers values -func NewObjectsClassReferencesPutUnprocessableEntity() *ObjectsClassReferencesPutUnprocessableEntity { - - return &ObjectsClassReferencesPutUnprocessableEntity{} -} - -// WithPayload adds the payload to the objects class references put unprocessable entity response -func (o *ObjectsClassReferencesPutUnprocessableEntity) WithPayload(payload *models.ErrorResponse) *ObjectsClassReferencesPutUnprocessableEntity { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects class references put unprocessable entity response -func (o *ObjectsClassReferencesPutUnprocessableEntity) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsClassReferencesPutUnprocessableEntity) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(422) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsClassReferencesPutInternalServerErrorCode is the HTTP code returned for type ObjectsClassReferencesPutInternalServerError -const ObjectsClassReferencesPutInternalServerErrorCode int = 500 - -/* -ObjectsClassReferencesPutInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response objectsClassReferencesPutInternalServerError -*/ -type ObjectsClassReferencesPutInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsClassReferencesPutInternalServerError creates ObjectsClassReferencesPutInternalServerError with default headers values -func NewObjectsClassReferencesPutInternalServerError() *ObjectsClassReferencesPutInternalServerError { - - return &ObjectsClassReferencesPutInternalServerError{} -} - -// WithPayload adds the payload to the objects class references put internal server error response -func (o *ObjectsClassReferencesPutInternalServerError) WithPayload(payload *models.ErrorResponse) *ObjectsClassReferencesPutInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects class references put internal server error response -func (o *ObjectsClassReferencesPutInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsClassReferencesPutInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/objects/objects_class_references_put_urlbuilder.go b/adapters/handlers/rest/operations/objects/objects_class_references_put_urlbuilder.go deleted file mode 100644 index 69b10849467d3a965ca58230ee743eb4082e95ae..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_class_references_put_urlbuilder.go +++ /dev/null @@ -1,151 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" - "strings" - - "github.com/go-openapi/strfmt" -) - -// ObjectsClassReferencesPutURL generates an URL for the objects class references put operation -type ObjectsClassReferencesPutURL struct { - ClassName string - ID strfmt.UUID - PropertyName string - - ConsistencyLevel *string - Tenant *string - - _basePath string - // avoid unkeyed usage - _ struct{} -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *ObjectsClassReferencesPutURL) WithBasePath(bp string) *ObjectsClassReferencesPutURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *ObjectsClassReferencesPutURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *ObjectsClassReferencesPutURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/objects/{className}/{id}/references/{propertyName}" - - className := o.ClassName - if className != "" { - _path = strings.Replace(_path, "{className}", className, -1) - } else { - return nil, errors.New("className is required on ObjectsClassReferencesPutURL") - } - - id := o.ID.String() - if id != "" { - _path = strings.Replace(_path, "{id}", id, -1) - } else { - return nil, errors.New("id is required on ObjectsClassReferencesPutURL") - } - - propertyName := o.PropertyName - if propertyName != "" { - _path = strings.Replace(_path, "{propertyName}", propertyName, -1) - } else { - return nil, errors.New("propertyName is required on ObjectsClassReferencesPutURL") - } - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - qs := make(url.Values) - - var consistencyLevelQ string - if o.ConsistencyLevel != nil { - consistencyLevelQ = *o.ConsistencyLevel - } - if consistencyLevelQ != "" { - qs.Set("consistency_level", consistencyLevelQ) - } - - var tenantQ string - if o.Tenant != nil { - tenantQ = *o.Tenant - } - if tenantQ != "" { - qs.Set("tenant", tenantQ) - } - - _result.RawQuery = qs.Encode() - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *ObjectsClassReferencesPutURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *ObjectsClassReferencesPutURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *ObjectsClassReferencesPutURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on ObjectsClassReferencesPutURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on ObjectsClassReferencesPutURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *ObjectsClassReferencesPutURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/objects/objects_create.go b/adapters/handlers/rest/operations/objects/objects_create.go deleted file mode 100644 index 236af56205bd5281b073f11fdb7fcda1f567fe80..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_create.go +++ /dev/null @@ -1,84 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsCreateHandlerFunc turns a function with the right signature into a objects create handler -type ObjectsCreateHandlerFunc func(ObjectsCreateParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn ObjectsCreateHandlerFunc) Handle(params ObjectsCreateParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// ObjectsCreateHandler interface for that can handle valid objects create params -type ObjectsCreateHandler interface { - Handle(ObjectsCreateParams, *models.Principal) middleware.Responder -} - -// NewObjectsCreate creates a new http.Handler for the objects create operation -func NewObjectsCreate(ctx *middleware.Context, handler ObjectsCreateHandler) *ObjectsCreate { - return &ObjectsCreate{Context: ctx, Handler: handler} -} - -/* - ObjectsCreate swagger:route POST /objects objects objectsCreate - -Create Objects between two Objects (object and subject). - -Registers a new Object. Provided meta-data and schema values are validated. -*/ -type ObjectsCreate struct { - Context *middleware.Context - Handler ObjectsCreateHandler -} - -func (o *ObjectsCreate) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewObjectsCreateParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/objects/objects_create_parameters.go b/adapters/handlers/rest/operations/objects/objects_create_parameters.go deleted file mode 100644 index 4e8e6951b3a8056a372dbfa93f4f659b9a998cbb..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_create_parameters.go +++ /dev/null @@ -1,125 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "io" - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/validate" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewObjectsCreateParams creates a new ObjectsCreateParams object -// -// There are no default values defined in the spec. -func NewObjectsCreateParams() ObjectsCreateParams { - - return ObjectsCreateParams{} -} - -// ObjectsCreateParams contains all the bound params for the objects create operation -// typically these are obtained from a http.Request -// -// swagger:parameters objects.create -type ObjectsCreateParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` - - /* - Required: true - In: body - */ - Body *models.Object - /*Determines how many replicas must acknowledge a request before it is considered successful - In: query - */ - ConsistencyLevel *string -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewObjectsCreateParams() beforehand. -func (o *ObjectsCreateParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - qs := runtime.Values(r.URL.Query()) - - if runtime.HasBody(r) { - defer r.Body.Close() - var body models.Object - if err := route.Consumer.Consume(r.Body, &body); err != nil { - if err == io.EOF { - res = append(res, errors.Required("body", "body", "")) - } else { - res = append(res, errors.NewParseError("body", "body", "", err)) - } - } else { - // validate body object - if err := body.Validate(route.Formats); err != nil { - res = append(res, err) - } - - ctx := validate.WithOperationRequest(r.Context()) - if err := body.ContextValidate(ctx, route.Formats); err != nil { - res = append(res, err) - } - - if len(res) == 0 { - o.Body = &body - } - } - } else { - res = append(res, errors.Required("body", "body", "")) - } - - qConsistencyLevel, qhkConsistencyLevel, _ := qs.GetOK("consistency_level") - if err := o.bindConsistencyLevel(qConsistencyLevel, qhkConsistencyLevel, route.Formats); err != nil { - res = append(res, err) - } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// bindConsistencyLevel binds and validates parameter ConsistencyLevel from query. -func (o *ObjectsCreateParams) bindConsistencyLevel(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: false - // AllowEmptyValue: false - - if raw == "" { // empty values pass all other validations - return nil - } - o.ConsistencyLevel = &raw - - return nil -} diff --git a/adapters/handlers/rest/operations/objects/objects_create_responses.go b/adapters/handlers/rest/operations/objects/objects_create_responses.go deleted file mode 100644 index f117f388a1cd98a88b29285565a40a79590a6624..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_create_responses.go +++ /dev/null @@ -1,275 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsCreateOKCode is the HTTP code returned for type ObjectsCreateOK -const ObjectsCreateOKCode int = 200 - -/* -ObjectsCreateOK Object created. - -swagger:response objectsCreateOK -*/ -type ObjectsCreateOK struct { - - /* - In: Body - */ - Payload *models.Object `json:"body,omitempty"` -} - -// NewObjectsCreateOK creates ObjectsCreateOK with default headers values -func NewObjectsCreateOK() *ObjectsCreateOK { - - return &ObjectsCreateOK{} -} - -// WithPayload adds the payload to the objects create o k response -func (o *ObjectsCreateOK) WithPayload(payload *models.Object) *ObjectsCreateOK { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects create o k response -func (o *ObjectsCreateOK) SetPayload(payload *models.Object) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsCreateOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(200) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsCreateBadRequestCode is the HTTP code returned for type ObjectsCreateBadRequest -const ObjectsCreateBadRequestCode int = 400 - -/* -ObjectsCreateBadRequest Malformed request. - -swagger:response objectsCreateBadRequest -*/ -type ObjectsCreateBadRequest struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsCreateBadRequest creates ObjectsCreateBadRequest with default headers values -func NewObjectsCreateBadRequest() *ObjectsCreateBadRequest { - - return &ObjectsCreateBadRequest{} -} - -// WithPayload adds the payload to the objects create bad request response -func (o *ObjectsCreateBadRequest) WithPayload(payload *models.ErrorResponse) *ObjectsCreateBadRequest { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects create bad request response -func (o *ObjectsCreateBadRequest) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsCreateBadRequest) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(400) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsCreateUnauthorizedCode is the HTTP code returned for type ObjectsCreateUnauthorized -const ObjectsCreateUnauthorizedCode int = 401 - -/* -ObjectsCreateUnauthorized Unauthorized or invalid credentials. - -swagger:response objectsCreateUnauthorized -*/ -type ObjectsCreateUnauthorized struct { -} - -// NewObjectsCreateUnauthorized creates ObjectsCreateUnauthorized with default headers values -func NewObjectsCreateUnauthorized() *ObjectsCreateUnauthorized { - - return &ObjectsCreateUnauthorized{} -} - -// WriteResponse to the client -func (o *ObjectsCreateUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// ObjectsCreateForbiddenCode is the HTTP code returned for type ObjectsCreateForbidden -const ObjectsCreateForbiddenCode int = 403 - -/* -ObjectsCreateForbidden Forbidden - -swagger:response objectsCreateForbidden -*/ -type ObjectsCreateForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsCreateForbidden creates ObjectsCreateForbidden with default headers values -func NewObjectsCreateForbidden() *ObjectsCreateForbidden { - - return &ObjectsCreateForbidden{} -} - -// WithPayload adds the payload to the objects create forbidden response -func (o *ObjectsCreateForbidden) WithPayload(payload *models.ErrorResponse) *ObjectsCreateForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects create forbidden response -func (o *ObjectsCreateForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsCreateForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsCreateUnprocessableEntityCode is the HTTP code returned for type ObjectsCreateUnprocessableEntity -const ObjectsCreateUnprocessableEntityCode int = 422 - -/* -ObjectsCreateUnprocessableEntity Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file? - -swagger:response objectsCreateUnprocessableEntity -*/ -type ObjectsCreateUnprocessableEntity struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsCreateUnprocessableEntity creates ObjectsCreateUnprocessableEntity with default headers values -func NewObjectsCreateUnprocessableEntity() *ObjectsCreateUnprocessableEntity { - - return &ObjectsCreateUnprocessableEntity{} -} - -// WithPayload adds the payload to the objects create unprocessable entity response -func (o *ObjectsCreateUnprocessableEntity) WithPayload(payload *models.ErrorResponse) *ObjectsCreateUnprocessableEntity { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects create unprocessable entity response -func (o *ObjectsCreateUnprocessableEntity) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsCreateUnprocessableEntity) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(422) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsCreateInternalServerErrorCode is the HTTP code returned for type ObjectsCreateInternalServerError -const ObjectsCreateInternalServerErrorCode int = 500 - -/* -ObjectsCreateInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response objectsCreateInternalServerError -*/ -type ObjectsCreateInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsCreateInternalServerError creates ObjectsCreateInternalServerError with default headers values -func NewObjectsCreateInternalServerError() *ObjectsCreateInternalServerError { - - return &ObjectsCreateInternalServerError{} -} - -// WithPayload adds the payload to the objects create internal server error response -func (o *ObjectsCreateInternalServerError) WithPayload(payload *models.ErrorResponse) *ObjectsCreateInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects create internal server error response -func (o *ObjectsCreateInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsCreateInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/objects/objects_create_urlbuilder.go b/adapters/handlers/rest/operations/objects/objects_create_urlbuilder.go deleted file mode 100644 index 5da955cf8271f89ba02674be956e31e1a430dc75..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_create_urlbuilder.go +++ /dev/null @@ -1,114 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" -) - -// ObjectsCreateURL generates an URL for the objects create operation -type ObjectsCreateURL struct { - ConsistencyLevel *string - - _basePath string - // avoid unkeyed usage - _ struct{} -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *ObjectsCreateURL) WithBasePath(bp string) *ObjectsCreateURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *ObjectsCreateURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *ObjectsCreateURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/objects" - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - qs := make(url.Values) - - var consistencyLevelQ string - if o.ConsistencyLevel != nil { - consistencyLevelQ = *o.ConsistencyLevel - } - if consistencyLevelQ != "" { - qs.Set("consistency_level", consistencyLevelQ) - } - - _result.RawQuery = qs.Encode() - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *ObjectsCreateURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *ObjectsCreateURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *ObjectsCreateURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on ObjectsCreateURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on ObjectsCreateURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *ObjectsCreateURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/objects/objects_delete.go b/adapters/handlers/rest/operations/objects/objects_delete.go deleted file mode 100644 index 37a7908f8f959a363ea84e98326a37358ad46a42..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_delete.go +++ /dev/null @@ -1,84 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsDeleteHandlerFunc turns a function with the right signature into a objects delete handler -type ObjectsDeleteHandlerFunc func(ObjectsDeleteParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn ObjectsDeleteHandlerFunc) Handle(params ObjectsDeleteParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// ObjectsDeleteHandler interface for that can handle valid objects delete params -type ObjectsDeleteHandler interface { - Handle(ObjectsDeleteParams, *models.Principal) middleware.Responder -} - -// NewObjectsDelete creates a new http.Handler for the objects delete operation -func NewObjectsDelete(ctx *middleware.Context, handler ObjectsDeleteHandler) *ObjectsDelete { - return &ObjectsDelete{Context: ctx, Handler: handler} -} - -/* - ObjectsDelete swagger:route DELETE /objects/{id} objects objectsDelete - -Delete an Object based on its UUID. - -Deletes an Object from the system. -*/ -type ObjectsDelete struct { - Context *middleware.Context - Handler ObjectsDeleteHandler -} - -func (o *ObjectsDelete) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewObjectsDeleteParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/objects/objects_delete_parameters.go b/adapters/handlers/rest/operations/objects/objects_delete_parameters.go deleted file mode 100644 index 9321d0e14d9e03c673aa038ca5869aa571dd3f47..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_delete_parameters.go +++ /dev/null @@ -1,159 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/validate" -) - -// NewObjectsDeleteParams creates a new ObjectsDeleteParams object -// -// There are no default values defined in the spec. -func NewObjectsDeleteParams() ObjectsDeleteParams { - - return ObjectsDeleteParams{} -} - -// ObjectsDeleteParams contains all the bound params for the objects delete operation -// typically these are obtained from a http.Request -// -// swagger:parameters objects.delete -type ObjectsDeleteParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` - - /*Determines how many replicas must acknowledge a request before it is considered successful - In: query - */ - ConsistencyLevel *string - /*Unique ID of the Object. - Required: true - In: path - */ - ID strfmt.UUID - /*Specifies the tenant in a request targeting a multi-tenant class - In: query - */ - Tenant *string -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewObjectsDeleteParams() beforehand. -func (o *ObjectsDeleteParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - qs := runtime.Values(r.URL.Query()) - - qConsistencyLevel, qhkConsistencyLevel, _ := qs.GetOK("consistency_level") - if err := o.bindConsistencyLevel(qConsistencyLevel, qhkConsistencyLevel, route.Formats); err != nil { - res = append(res, err) - } - - rID, rhkID, _ := route.Params.GetOK("id") - if err := o.bindID(rID, rhkID, route.Formats); err != nil { - res = append(res, err) - } - - qTenant, qhkTenant, _ := qs.GetOK("tenant") - if err := o.bindTenant(qTenant, qhkTenant, route.Formats); err != nil { - res = append(res, err) - } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// bindConsistencyLevel binds and validates parameter ConsistencyLevel from query. -func (o *ObjectsDeleteParams) bindConsistencyLevel(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: false - // AllowEmptyValue: false - - if raw == "" { // empty values pass all other validations - return nil - } - o.ConsistencyLevel = &raw - - return nil -} - -// bindID binds and validates parameter ID from path. -func (o *ObjectsDeleteParams) bindID(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - - // Format: uuid - value, err := formats.Parse("uuid", raw) - if err != nil { - return errors.InvalidType("id", "path", "strfmt.UUID", raw) - } - o.ID = *(value.(*strfmt.UUID)) - - if err := o.validateID(formats); err != nil { - return err - } - - return nil -} - -// validateID carries on validations for parameter ID -func (o *ObjectsDeleteParams) validateID(formats strfmt.Registry) error { - - if err := validate.FormatOf("id", "path", "uuid", o.ID.String(), formats); err != nil { - return err - } - return nil -} - -// bindTenant binds and validates parameter Tenant from query. -func (o *ObjectsDeleteParams) bindTenant(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: false - // AllowEmptyValue: false - - if raw == "" { // empty values pass all other validations - return nil - } - o.Tenant = &raw - - return nil -} diff --git a/adapters/handlers/rest/operations/objects/objects_delete_responses.go b/adapters/handlers/rest/operations/objects/objects_delete_responses.go deleted file mode 100644 index f308b4101803b9adcfa4a71b81061d879463bcb0..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_delete_responses.go +++ /dev/null @@ -1,190 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsDeleteNoContentCode is the HTTP code returned for type ObjectsDeleteNoContent -const ObjectsDeleteNoContentCode int = 204 - -/* -ObjectsDeleteNoContent Successfully deleted. - -swagger:response objectsDeleteNoContent -*/ -type ObjectsDeleteNoContent struct { -} - -// NewObjectsDeleteNoContent creates ObjectsDeleteNoContent with default headers values -func NewObjectsDeleteNoContent() *ObjectsDeleteNoContent { - - return &ObjectsDeleteNoContent{} -} - -// WriteResponse to the client -func (o *ObjectsDeleteNoContent) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(204) -} - -// ObjectsDeleteUnauthorizedCode is the HTTP code returned for type ObjectsDeleteUnauthorized -const ObjectsDeleteUnauthorizedCode int = 401 - -/* -ObjectsDeleteUnauthorized Unauthorized or invalid credentials. - -swagger:response objectsDeleteUnauthorized -*/ -type ObjectsDeleteUnauthorized struct { -} - -// NewObjectsDeleteUnauthorized creates ObjectsDeleteUnauthorized with default headers values -func NewObjectsDeleteUnauthorized() *ObjectsDeleteUnauthorized { - - return &ObjectsDeleteUnauthorized{} -} - -// WriteResponse to the client -func (o *ObjectsDeleteUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// ObjectsDeleteForbiddenCode is the HTTP code returned for type ObjectsDeleteForbidden -const ObjectsDeleteForbiddenCode int = 403 - -/* -ObjectsDeleteForbidden Forbidden - -swagger:response objectsDeleteForbidden -*/ -type ObjectsDeleteForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsDeleteForbidden creates ObjectsDeleteForbidden with default headers values -func NewObjectsDeleteForbidden() *ObjectsDeleteForbidden { - - return &ObjectsDeleteForbidden{} -} - -// WithPayload adds the payload to the objects delete forbidden response -func (o *ObjectsDeleteForbidden) WithPayload(payload *models.ErrorResponse) *ObjectsDeleteForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects delete forbidden response -func (o *ObjectsDeleteForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsDeleteForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsDeleteNotFoundCode is the HTTP code returned for type ObjectsDeleteNotFound -const ObjectsDeleteNotFoundCode int = 404 - -/* -ObjectsDeleteNotFound Successful query result but no resource was found. - -swagger:response objectsDeleteNotFound -*/ -type ObjectsDeleteNotFound struct { -} - -// NewObjectsDeleteNotFound creates ObjectsDeleteNotFound with default headers values -func NewObjectsDeleteNotFound() *ObjectsDeleteNotFound { - - return &ObjectsDeleteNotFound{} -} - -// WriteResponse to the client -func (o *ObjectsDeleteNotFound) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(404) -} - -// ObjectsDeleteInternalServerErrorCode is the HTTP code returned for type ObjectsDeleteInternalServerError -const ObjectsDeleteInternalServerErrorCode int = 500 - -/* -ObjectsDeleteInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response objectsDeleteInternalServerError -*/ -type ObjectsDeleteInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsDeleteInternalServerError creates ObjectsDeleteInternalServerError with default headers values -func NewObjectsDeleteInternalServerError() *ObjectsDeleteInternalServerError { - - return &ObjectsDeleteInternalServerError{} -} - -// WithPayload adds the payload to the objects delete internal server error response -func (o *ObjectsDeleteInternalServerError) WithPayload(payload *models.ErrorResponse) *ObjectsDeleteInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects delete internal server error response -func (o *ObjectsDeleteInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsDeleteInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/objects/objects_delete_urlbuilder.go b/adapters/handlers/rest/operations/objects/objects_delete_urlbuilder.go deleted file mode 100644 index 8447b9572f9fac7419938f48000ebeece5ff45f2..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_delete_urlbuilder.go +++ /dev/null @@ -1,135 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" - "strings" - - "github.com/go-openapi/strfmt" -) - -// ObjectsDeleteURL generates an URL for the objects delete operation -type ObjectsDeleteURL struct { - ID strfmt.UUID - - ConsistencyLevel *string - Tenant *string - - _basePath string - // avoid unkeyed usage - _ struct{} -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *ObjectsDeleteURL) WithBasePath(bp string) *ObjectsDeleteURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *ObjectsDeleteURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *ObjectsDeleteURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/objects/{id}" - - id := o.ID.String() - if id != "" { - _path = strings.Replace(_path, "{id}", id, -1) - } else { - return nil, errors.New("id is required on ObjectsDeleteURL") - } - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - qs := make(url.Values) - - var consistencyLevelQ string - if o.ConsistencyLevel != nil { - consistencyLevelQ = *o.ConsistencyLevel - } - if consistencyLevelQ != "" { - qs.Set("consistency_level", consistencyLevelQ) - } - - var tenantQ string - if o.Tenant != nil { - tenantQ = *o.Tenant - } - if tenantQ != "" { - qs.Set("tenant", tenantQ) - } - - _result.RawQuery = qs.Encode() - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *ObjectsDeleteURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *ObjectsDeleteURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *ObjectsDeleteURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on ObjectsDeleteURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on ObjectsDeleteURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *ObjectsDeleteURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/objects/objects_get.go b/adapters/handlers/rest/operations/objects/objects_get.go deleted file mode 100644 index cf8427313a798288209a50a3201a1a2af0afc204..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_get.go +++ /dev/null @@ -1,84 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsGetHandlerFunc turns a function with the right signature into a objects get handler -type ObjectsGetHandlerFunc func(ObjectsGetParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn ObjectsGetHandlerFunc) Handle(params ObjectsGetParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// ObjectsGetHandler interface for that can handle valid objects get params -type ObjectsGetHandler interface { - Handle(ObjectsGetParams, *models.Principal) middleware.Responder -} - -// NewObjectsGet creates a new http.Handler for the objects get operation -func NewObjectsGet(ctx *middleware.Context, handler ObjectsGetHandler) *ObjectsGet { - return &ObjectsGet{Context: ctx, Handler: handler} -} - -/* - ObjectsGet swagger:route GET /objects/{id} objects objectsGet - -Get a specific Object based on its UUID and a Object UUID. Also available as Websocket bus. - -Lists Objects. -*/ -type ObjectsGet struct { - Context *middleware.Context - Handler ObjectsGetHandler -} - -func (o *ObjectsGet) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewObjectsGetParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/objects/objects_get_parameters.go b/adapters/handlers/rest/operations/objects/objects_get_parameters.go deleted file mode 100644 index 530b481920226fc4415527e6450907d549eef080..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_get_parameters.go +++ /dev/null @@ -1,132 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/validate" -) - -// NewObjectsGetParams creates a new ObjectsGetParams object -// -// There are no default values defined in the spec. -func NewObjectsGetParams() ObjectsGetParams { - - return ObjectsGetParams{} -} - -// ObjectsGetParams contains all the bound params for the objects get operation -// typically these are obtained from a http.Request -// -// swagger:parameters objects.get -type ObjectsGetParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` - - /*Unique ID of the Object. - Required: true - In: path - */ - ID strfmt.UUID - /*Include additional information, such as classification infos. Allowed values include: classification, vector, interpretation - In: query - */ - Include *string -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewObjectsGetParams() beforehand. -func (o *ObjectsGetParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - qs := runtime.Values(r.URL.Query()) - - rID, rhkID, _ := route.Params.GetOK("id") - if err := o.bindID(rID, rhkID, route.Formats); err != nil { - res = append(res, err) - } - - qInclude, qhkInclude, _ := qs.GetOK("include") - if err := o.bindInclude(qInclude, qhkInclude, route.Formats); err != nil { - res = append(res, err) - } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// bindID binds and validates parameter ID from path. -func (o *ObjectsGetParams) bindID(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - - // Format: uuid - value, err := formats.Parse("uuid", raw) - if err != nil { - return errors.InvalidType("id", "path", "strfmt.UUID", raw) - } - o.ID = *(value.(*strfmt.UUID)) - - if err := o.validateID(formats); err != nil { - return err - } - - return nil -} - -// validateID carries on validations for parameter ID -func (o *ObjectsGetParams) validateID(formats strfmt.Registry) error { - - if err := validate.FormatOf("id", "path", "uuid", o.ID.String(), formats); err != nil { - return err - } - return nil -} - -// bindInclude binds and validates parameter Include from query. -func (o *ObjectsGetParams) bindInclude(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: false - // AllowEmptyValue: false - - if raw == "" { // empty values pass all other validations - return nil - } - o.Include = &raw - - return nil -} diff --git a/adapters/handlers/rest/operations/objects/objects_get_responses.go b/adapters/handlers/rest/operations/objects/objects_get_responses.go deleted file mode 100644 index 36d95f70dacb61cbf09ac71b56aebdcac84b680b..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_get_responses.go +++ /dev/null @@ -1,255 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsGetOKCode is the HTTP code returned for type ObjectsGetOK -const ObjectsGetOKCode int = 200 - -/* -ObjectsGetOK Successful response. - -swagger:response objectsGetOK -*/ -type ObjectsGetOK struct { - - /* - In: Body - */ - Payload *models.Object `json:"body,omitempty"` -} - -// NewObjectsGetOK creates ObjectsGetOK with default headers values -func NewObjectsGetOK() *ObjectsGetOK { - - return &ObjectsGetOK{} -} - -// WithPayload adds the payload to the objects get o k response -func (o *ObjectsGetOK) WithPayload(payload *models.Object) *ObjectsGetOK { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects get o k response -func (o *ObjectsGetOK) SetPayload(payload *models.Object) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsGetOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(200) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsGetBadRequestCode is the HTTP code returned for type ObjectsGetBadRequest -const ObjectsGetBadRequestCode int = 400 - -/* -ObjectsGetBadRequest Malformed request. - -swagger:response objectsGetBadRequest -*/ -type ObjectsGetBadRequest struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsGetBadRequest creates ObjectsGetBadRequest with default headers values -func NewObjectsGetBadRequest() *ObjectsGetBadRequest { - - return &ObjectsGetBadRequest{} -} - -// WithPayload adds the payload to the objects get bad request response -func (o *ObjectsGetBadRequest) WithPayload(payload *models.ErrorResponse) *ObjectsGetBadRequest { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects get bad request response -func (o *ObjectsGetBadRequest) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsGetBadRequest) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(400) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsGetUnauthorizedCode is the HTTP code returned for type ObjectsGetUnauthorized -const ObjectsGetUnauthorizedCode int = 401 - -/* -ObjectsGetUnauthorized Unauthorized or invalid credentials. - -swagger:response objectsGetUnauthorized -*/ -type ObjectsGetUnauthorized struct { -} - -// NewObjectsGetUnauthorized creates ObjectsGetUnauthorized with default headers values -func NewObjectsGetUnauthorized() *ObjectsGetUnauthorized { - - return &ObjectsGetUnauthorized{} -} - -// WriteResponse to the client -func (o *ObjectsGetUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// ObjectsGetForbiddenCode is the HTTP code returned for type ObjectsGetForbidden -const ObjectsGetForbiddenCode int = 403 - -/* -ObjectsGetForbidden Forbidden - -swagger:response objectsGetForbidden -*/ -type ObjectsGetForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsGetForbidden creates ObjectsGetForbidden with default headers values -func NewObjectsGetForbidden() *ObjectsGetForbidden { - - return &ObjectsGetForbidden{} -} - -// WithPayload adds the payload to the objects get forbidden response -func (o *ObjectsGetForbidden) WithPayload(payload *models.ErrorResponse) *ObjectsGetForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects get forbidden response -func (o *ObjectsGetForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsGetForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsGetNotFoundCode is the HTTP code returned for type ObjectsGetNotFound -const ObjectsGetNotFoundCode int = 404 - -/* -ObjectsGetNotFound Successful query result but no resource was found. - -swagger:response objectsGetNotFound -*/ -type ObjectsGetNotFound struct { -} - -// NewObjectsGetNotFound creates ObjectsGetNotFound with default headers values -func NewObjectsGetNotFound() *ObjectsGetNotFound { - - return &ObjectsGetNotFound{} -} - -// WriteResponse to the client -func (o *ObjectsGetNotFound) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(404) -} - -// ObjectsGetInternalServerErrorCode is the HTTP code returned for type ObjectsGetInternalServerError -const ObjectsGetInternalServerErrorCode int = 500 - -/* -ObjectsGetInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response objectsGetInternalServerError -*/ -type ObjectsGetInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsGetInternalServerError creates ObjectsGetInternalServerError with default headers values -func NewObjectsGetInternalServerError() *ObjectsGetInternalServerError { - - return &ObjectsGetInternalServerError{} -} - -// WithPayload adds the payload to the objects get internal server error response -func (o *ObjectsGetInternalServerError) WithPayload(payload *models.ErrorResponse) *ObjectsGetInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects get internal server error response -func (o *ObjectsGetInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsGetInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/objects/objects_get_urlbuilder.go b/adapters/handlers/rest/operations/objects/objects_get_urlbuilder.go deleted file mode 100644 index b62eb167287e405ea271adfa0fd46995c808ef4e..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_get_urlbuilder.go +++ /dev/null @@ -1,126 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" - "strings" - - "github.com/go-openapi/strfmt" -) - -// ObjectsGetURL generates an URL for the objects get operation -type ObjectsGetURL struct { - ID strfmt.UUID - - Include *string - - _basePath string - // avoid unkeyed usage - _ struct{} -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *ObjectsGetURL) WithBasePath(bp string) *ObjectsGetURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *ObjectsGetURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *ObjectsGetURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/objects/{id}" - - id := o.ID.String() - if id != "" { - _path = strings.Replace(_path, "{id}", id, -1) - } else { - return nil, errors.New("id is required on ObjectsGetURL") - } - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - qs := make(url.Values) - - var includeQ string - if o.Include != nil { - includeQ = *o.Include - } - if includeQ != "" { - qs.Set("include", includeQ) - } - - _result.RawQuery = qs.Encode() - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *ObjectsGetURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *ObjectsGetURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *ObjectsGetURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on ObjectsGetURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on ObjectsGetURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *ObjectsGetURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/objects/objects_head.go b/adapters/handlers/rest/operations/objects/objects_head.go deleted file mode 100644 index 95800d1814185078c90fd6142c96769c1b49664f..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_head.go +++ /dev/null @@ -1,84 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsHeadHandlerFunc turns a function with the right signature into a objects head handler -type ObjectsHeadHandlerFunc func(ObjectsHeadParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn ObjectsHeadHandlerFunc) Handle(params ObjectsHeadParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// ObjectsHeadHandler interface for that can handle valid objects head params -type ObjectsHeadHandler interface { - Handle(ObjectsHeadParams, *models.Principal) middleware.Responder -} - -// NewObjectsHead creates a new http.Handler for the objects head operation -func NewObjectsHead(ctx *middleware.Context, handler ObjectsHeadHandler) *ObjectsHead { - return &ObjectsHead{Context: ctx, Handler: handler} -} - -/* - ObjectsHead swagger:route HEAD /objects/{id} objects objectsHead - -Checks Object's existence based on its UUID. - -Checks if an Object exists in the system. -*/ -type ObjectsHead struct { - Context *middleware.Context - Handler ObjectsHeadHandler -} - -func (o *ObjectsHead) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewObjectsHeadParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/objects/objects_head_parameters.go b/adapters/handlers/rest/operations/objects/objects_head_parameters.go deleted file mode 100644 index 0ba97fac802a53c278f34665a12e868a5f84f39b..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_head_parameters.go +++ /dev/null @@ -1,102 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/validate" -) - -// NewObjectsHeadParams creates a new ObjectsHeadParams object -// -// There are no default values defined in the spec. -func NewObjectsHeadParams() ObjectsHeadParams { - - return ObjectsHeadParams{} -} - -// ObjectsHeadParams contains all the bound params for the objects head operation -// typically these are obtained from a http.Request -// -// swagger:parameters objects.head -type ObjectsHeadParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` - - /*Unique ID of the Object. - Required: true - In: path - */ - ID strfmt.UUID -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewObjectsHeadParams() beforehand. -func (o *ObjectsHeadParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - rID, rhkID, _ := route.Params.GetOK("id") - if err := o.bindID(rID, rhkID, route.Formats); err != nil { - res = append(res, err) - } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// bindID binds and validates parameter ID from path. -func (o *ObjectsHeadParams) bindID(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - - // Format: uuid - value, err := formats.Parse("uuid", raw) - if err != nil { - return errors.InvalidType("id", "path", "strfmt.UUID", raw) - } - o.ID = *(value.(*strfmt.UUID)) - - if err := o.validateID(formats); err != nil { - return err - } - - return nil -} - -// validateID carries on validations for parameter ID -func (o *ObjectsHeadParams) validateID(formats strfmt.Registry) error { - - if err := validate.FormatOf("id", "path", "uuid", o.ID.String(), formats); err != nil { - return err - } - return nil -} diff --git a/adapters/handlers/rest/operations/objects/objects_head_responses.go b/adapters/handlers/rest/operations/objects/objects_head_responses.go deleted file mode 100644 index fe8d861d899b32c7067a6f421eb2d40f7dc01356..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_head_responses.go +++ /dev/null @@ -1,190 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsHeadNoContentCode is the HTTP code returned for type ObjectsHeadNoContent -const ObjectsHeadNoContentCode int = 204 - -/* -ObjectsHeadNoContent Object exists. - -swagger:response objectsHeadNoContent -*/ -type ObjectsHeadNoContent struct { -} - -// NewObjectsHeadNoContent creates ObjectsHeadNoContent with default headers values -func NewObjectsHeadNoContent() *ObjectsHeadNoContent { - - return &ObjectsHeadNoContent{} -} - -// WriteResponse to the client -func (o *ObjectsHeadNoContent) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(204) -} - -// ObjectsHeadUnauthorizedCode is the HTTP code returned for type ObjectsHeadUnauthorized -const ObjectsHeadUnauthorizedCode int = 401 - -/* -ObjectsHeadUnauthorized Unauthorized or invalid credentials. - -swagger:response objectsHeadUnauthorized -*/ -type ObjectsHeadUnauthorized struct { -} - -// NewObjectsHeadUnauthorized creates ObjectsHeadUnauthorized with default headers values -func NewObjectsHeadUnauthorized() *ObjectsHeadUnauthorized { - - return &ObjectsHeadUnauthorized{} -} - -// WriteResponse to the client -func (o *ObjectsHeadUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// ObjectsHeadForbiddenCode is the HTTP code returned for type ObjectsHeadForbidden -const ObjectsHeadForbiddenCode int = 403 - -/* -ObjectsHeadForbidden Forbidden - -swagger:response objectsHeadForbidden -*/ -type ObjectsHeadForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsHeadForbidden creates ObjectsHeadForbidden with default headers values -func NewObjectsHeadForbidden() *ObjectsHeadForbidden { - - return &ObjectsHeadForbidden{} -} - -// WithPayload adds the payload to the objects head forbidden response -func (o *ObjectsHeadForbidden) WithPayload(payload *models.ErrorResponse) *ObjectsHeadForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects head forbidden response -func (o *ObjectsHeadForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsHeadForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsHeadNotFoundCode is the HTTP code returned for type ObjectsHeadNotFound -const ObjectsHeadNotFoundCode int = 404 - -/* -ObjectsHeadNotFound Object doesn't exist. - -swagger:response objectsHeadNotFound -*/ -type ObjectsHeadNotFound struct { -} - -// NewObjectsHeadNotFound creates ObjectsHeadNotFound with default headers values -func NewObjectsHeadNotFound() *ObjectsHeadNotFound { - - return &ObjectsHeadNotFound{} -} - -// WriteResponse to the client -func (o *ObjectsHeadNotFound) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(404) -} - -// ObjectsHeadInternalServerErrorCode is the HTTP code returned for type ObjectsHeadInternalServerError -const ObjectsHeadInternalServerErrorCode int = 500 - -/* -ObjectsHeadInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response objectsHeadInternalServerError -*/ -type ObjectsHeadInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsHeadInternalServerError creates ObjectsHeadInternalServerError with default headers values -func NewObjectsHeadInternalServerError() *ObjectsHeadInternalServerError { - - return &ObjectsHeadInternalServerError{} -} - -// WithPayload adds the payload to the objects head internal server error response -func (o *ObjectsHeadInternalServerError) WithPayload(payload *models.ErrorResponse) *ObjectsHeadInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects head internal server error response -func (o *ObjectsHeadInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsHeadInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/objects/objects_head_urlbuilder.go b/adapters/handlers/rest/operations/objects/objects_head_urlbuilder.go deleted file mode 100644 index c655524e7596fb03a24e0a5ec1e9ad1bc346305c..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_head_urlbuilder.go +++ /dev/null @@ -1,112 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" - "strings" - - "github.com/go-openapi/strfmt" -) - -// ObjectsHeadURL generates an URL for the objects head operation -type ObjectsHeadURL struct { - ID strfmt.UUID - - _basePath string - // avoid unkeyed usage - _ struct{} -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *ObjectsHeadURL) WithBasePath(bp string) *ObjectsHeadURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *ObjectsHeadURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *ObjectsHeadURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/objects/{id}" - - id := o.ID.String() - if id != "" { - _path = strings.Replace(_path, "{id}", id, -1) - } else { - return nil, errors.New("id is required on ObjectsHeadURL") - } - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *ObjectsHeadURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *ObjectsHeadURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *ObjectsHeadURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on ObjectsHeadURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on ObjectsHeadURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *ObjectsHeadURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/objects/objects_list.go b/adapters/handlers/rest/operations/objects/objects_list.go deleted file mode 100644 index f438f971d0bba18b80444a88a34ddf1c337db125..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_list.go +++ /dev/null @@ -1,84 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsListHandlerFunc turns a function with the right signature into a objects list handler -type ObjectsListHandlerFunc func(ObjectsListParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn ObjectsListHandlerFunc) Handle(params ObjectsListParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// ObjectsListHandler interface for that can handle valid objects list params -type ObjectsListHandler interface { - Handle(ObjectsListParams, *models.Principal) middleware.Responder -} - -// NewObjectsList creates a new http.Handler for the objects list operation -func NewObjectsList(ctx *middleware.Context, handler ObjectsListHandler) *ObjectsList { - return &ObjectsList{Context: ctx, Handler: handler} -} - -/* - ObjectsList swagger:route GET /objects objects objectsList - -Get a list of Objects. - -Lists all Objects in reverse order of creation, owned by the user that belongs to the used token. -*/ -type ObjectsList struct { - Context *middleware.Context - Handler ObjectsListHandler -} - -func (o *ObjectsList) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewObjectsListParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/objects/objects_list_parameters.go b/adapters/handlers/rest/operations/objects/objects_list_parameters.go deleted file mode 100644 index a73b34451ddd0e394ef585f8545319dc10beceba..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_list_parameters.go +++ /dev/null @@ -1,297 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" -) - -// NewObjectsListParams creates a new ObjectsListParams object -// with the default values initialized. -func NewObjectsListParams() ObjectsListParams { - - var ( - // initialize parameters with default values - - offsetDefault = int64(0) - ) - - return ObjectsListParams{ - Offset: &offsetDefault, - } -} - -// ObjectsListParams contains all the bound params for the objects list operation -// typically these are obtained from a http.Request -// -// swagger:parameters objects.list -type ObjectsListParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` - - /*The starting ID of the result window. - In: query - */ - After *string - /*Class parameter specifies the class from which to query objects - In: query - */ - Class *string - /*Include additional information, such as classification infos. Allowed values include: classification, vector, interpretation - In: query - */ - Include *string - /*The maximum number of items to be returned per page. Default value is set in Weaviate config. - In: query - */ - Limit *int64 - /*The starting index of the result window. Default value is 0. - In: query - Default: 0 - */ - Offset *int64 - /*Order parameter to tell how to order (asc or desc) data within given field - In: query - */ - Order *string - /*Sort parameter to pass an information about the names of the sort fields - In: query - */ - Sort *string - /*Specifies the tenant in a request targeting a multi-tenant class - In: query - */ - Tenant *string -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewObjectsListParams() beforehand. -func (o *ObjectsListParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - qs := runtime.Values(r.URL.Query()) - - qAfter, qhkAfter, _ := qs.GetOK("after") - if err := o.bindAfter(qAfter, qhkAfter, route.Formats); err != nil { - res = append(res, err) - } - - qClass, qhkClass, _ := qs.GetOK("class") - if err := o.bindClass(qClass, qhkClass, route.Formats); err != nil { - res = append(res, err) - } - - qInclude, qhkInclude, _ := qs.GetOK("include") - if err := o.bindInclude(qInclude, qhkInclude, route.Formats); err != nil { - res = append(res, err) - } - - qLimit, qhkLimit, _ := qs.GetOK("limit") - if err := o.bindLimit(qLimit, qhkLimit, route.Formats); err != nil { - res = append(res, err) - } - - qOffset, qhkOffset, _ := qs.GetOK("offset") - if err := o.bindOffset(qOffset, qhkOffset, route.Formats); err != nil { - res = append(res, err) - } - - qOrder, qhkOrder, _ := qs.GetOK("order") - if err := o.bindOrder(qOrder, qhkOrder, route.Formats); err != nil { - res = append(res, err) - } - - qSort, qhkSort, _ := qs.GetOK("sort") - if err := o.bindSort(qSort, qhkSort, route.Formats); err != nil { - res = append(res, err) - } - - qTenant, qhkTenant, _ := qs.GetOK("tenant") - if err := o.bindTenant(qTenant, qhkTenant, route.Formats); err != nil { - res = append(res, err) - } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// bindAfter binds and validates parameter After from query. -func (o *ObjectsListParams) bindAfter(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: false - // AllowEmptyValue: false - - if raw == "" { // empty values pass all other validations - return nil - } - o.After = &raw - - return nil -} - -// bindClass binds and validates parameter Class from query. -func (o *ObjectsListParams) bindClass(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: false - // AllowEmptyValue: false - - if raw == "" { // empty values pass all other validations - return nil - } - o.Class = &raw - - return nil -} - -// bindInclude binds and validates parameter Include from query. -func (o *ObjectsListParams) bindInclude(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: false - // AllowEmptyValue: false - - if raw == "" { // empty values pass all other validations - return nil - } - o.Include = &raw - - return nil -} - -// bindLimit binds and validates parameter Limit from query. -func (o *ObjectsListParams) bindLimit(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: false - // AllowEmptyValue: false - - if raw == "" { // empty values pass all other validations - return nil - } - - value, err := swag.ConvertInt64(raw) - if err != nil { - return errors.InvalidType("limit", "query", "int64", raw) - } - o.Limit = &value - - return nil -} - -// bindOffset binds and validates parameter Offset from query. -func (o *ObjectsListParams) bindOffset(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: false - // AllowEmptyValue: false - - if raw == "" { // empty values pass all other validations - // Default values have been previously initialized by NewObjectsListParams() - return nil - } - - value, err := swag.ConvertInt64(raw) - if err != nil { - return errors.InvalidType("offset", "query", "int64", raw) - } - o.Offset = &value - - return nil -} - -// bindOrder binds and validates parameter Order from query. -func (o *ObjectsListParams) bindOrder(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: false - // AllowEmptyValue: false - - if raw == "" { // empty values pass all other validations - return nil - } - o.Order = &raw - - return nil -} - -// bindSort binds and validates parameter Sort from query. -func (o *ObjectsListParams) bindSort(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: false - // AllowEmptyValue: false - - if raw == "" { // empty values pass all other validations - return nil - } - o.Sort = &raw - - return nil -} - -// bindTenant binds and validates parameter Tenant from query. -func (o *ObjectsListParams) bindTenant(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: false - // AllowEmptyValue: false - - if raw == "" { // empty values pass all other validations - return nil - } - o.Tenant = &raw - - return nil -} diff --git a/adapters/handlers/rest/operations/objects/objects_list_responses.go b/adapters/handlers/rest/operations/objects/objects_list_responses.go deleted file mode 100644 index 8dff5b8106b6da7d0626cb462544b517116417ce..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_list_responses.go +++ /dev/null @@ -1,300 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsListOKCode is the HTTP code returned for type ObjectsListOK -const ObjectsListOKCode int = 200 - -/* -ObjectsListOK Successful response. - -swagger:response objectsListOK -*/ -type ObjectsListOK struct { - - /* - In: Body - */ - Payload *models.ObjectsListResponse `json:"body,omitempty"` -} - -// NewObjectsListOK creates ObjectsListOK with default headers values -func NewObjectsListOK() *ObjectsListOK { - - return &ObjectsListOK{} -} - -// WithPayload adds the payload to the objects list o k response -func (o *ObjectsListOK) WithPayload(payload *models.ObjectsListResponse) *ObjectsListOK { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects list o k response -func (o *ObjectsListOK) SetPayload(payload *models.ObjectsListResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsListOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(200) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsListBadRequestCode is the HTTP code returned for type ObjectsListBadRequest -const ObjectsListBadRequestCode int = 400 - -/* -ObjectsListBadRequest Malformed request. - -swagger:response objectsListBadRequest -*/ -type ObjectsListBadRequest struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsListBadRequest creates ObjectsListBadRequest with default headers values -func NewObjectsListBadRequest() *ObjectsListBadRequest { - - return &ObjectsListBadRequest{} -} - -// WithPayload adds the payload to the objects list bad request response -func (o *ObjectsListBadRequest) WithPayload(payload *models.ErrorResponse) *ObjectsListBadRequest { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects list bad request response -func (o *ObjectsListBadRequest) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsListBadRequest) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(400) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsListUnauthorizedCode is the HTTP code returned for type ObjectsListUnauthorized -const ObjectsListUnauthorizedCode int = 401 - -/* -ObjectsListUnauthorized Unauthorized or invalid credentials. - -swagger:response objectsListUnauthorized -*/ -type ObjectsListUnauthorized struct { -} - -// NewObjectsListUnauthorized creates ObjectsListUnauthorized with default headers values -func NewObjectsListUnauthorized() *ObjectsListUnauthorized { - - return &ObjectsListUnauthorized{} -} - -// WriteResponse to the client -func (o *ObjectsListUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// ObjectsListForbiddenCode is the HTTP code returned for type ObjectsListForbidden -const ObjectsListForbiddenCode int = 403 - -/* -ObjectsListForbidden Forbidden - -swagger:response objectsListForbidden -*/ -type ObjectsListForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsListForbidden creates ObjectsListForbidden with default headers values -func NewObjectsListForbidden() *ObjectsListForbidden { - - return &ObjectsListForbidden{} -} - -// WithPayload adds the payload to the objects list forbidden response -func (o *ObjectsListForbidden) WithPayload(payload *models.ErrorResponse) *ObjectsListForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects list forbidden response -func (o *ObjectsListForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsListForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsListNotFoundCode is the HTTP code returned for type ObjectsListNotFound -const ObjectsListNotFoundCode int = 404 - -/* -ObjectsListNotFound Successful query result but no resource was found. - -swagger:response objectsListNotFound -*/ -type ObjectsListNotFound struct { -} - -// NewObjectsListNotFound creates ObjectsListNotFound with default headers values -func NewObjectsListNotFound() *ObjectsListNotFound { - - return &ObjectsListNotFound{} -} - -// WriteResponse to the client -func (o *ObjectsListNotFound) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(404) -} - -// ObjectsListUnprocessableEntityCode is the HTTP code returned for type ObjectsListUnprocessableEntity -const ObjectsListUnprocessableEntityCode int = 422 - -/* -ObjectsListUnprocessableEntity Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file? - -swagger:response objectsListUnprocessableEntity -*/ -type ObjectsListUnprocessableEntity struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsListUnprocessableEntity creates ObjectsListUnprocessableEntity with default headers values -func NewObjectsListUnprocessableEntity() *ObjectsListUnprocessableEntity { - - return &ObjectsListUnprocessableEntity{} -} - -// WithPayload adds the payload to the objects list unprocessable entity response -func (o *ObjectsListUnprocessableEntity) WithPayload(payload *models.ErrorResponse) *ObjectsListUnprocessableEntity { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects list unprocessable entity response -func (o *ObjectsListUnprocessableEntity) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsListUnprocessableEntity) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(422) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsListInternalServerErrorCode is the HTTP code returned for type ObjectsListInternalServerError -const ObjectsListInternalServerErrorCode int = 500 - -/* -ObjectsListInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response objectsListInternalServerError -*/ -type ObjectsListInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsListInternalServerError creates ObjectsListInternalServerError with default headers values -func NewObjectsListInternalServerError() *ObjectsListInternalServerError { - - return &ObjectsListInternalServerError{} -} - -// WithPayload adds the payload to the objects list internal server error response -func (o *ObjectsListInternalServerError) WithPayload(payload *models.ErrorResponse) *ObjectsListInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects list internal server error response -func (o *ObjectsListInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsListInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/objects/objects_list_urlbuilder.go b/adapters/handlers/rest/operations/objects/objects_list_urlbuilder.go deleted file mode 100644 index e06867cf0cfc8ecfbccf9ad6981810a44b39b973..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_list_urlbuilder.go +++ /dev/null @@ -1,179 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" - - "github.com/go-openapi/swag" -) - -// ObjectsListURL generates an URL for the objects list operation -type ObjectsListURL struct { - After *string - Class *string - Include *string - Limit *int64 - Offset *int64 - Order *string - Sort *string - Tenant *string - - _basePath string - // avoid unkeyed usage - _ struct{} -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *ObjectsListURL) WithBasePath(bp string) *ObjectsListURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *ObjectsListURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *ObjectsListURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/objects" - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - qs := make(url.Values) - - var afterQ string - if o.After != nil { - afterQ = *o.After - } - if afterQ != "" { - qs.Set("after", afterQ) - } - - var classQ string - if o.Class != nil { - classQ = *o.Class - } - if classQ != "" { - qs.Set("class", classQ) - } - - var includeQ string - if o.Include != nil { - includeQ = *o.Include - } - if includeQ != "" { - qs.Set("include", includeQ) - } - - var limitQ string - if o.Limit != nil { - limitQ = swag.FormatInt64(*o.Limit) - } - if limitQ != "" { - qs.Set("limit", limitQ) - } - - var offsetQ string - if o.Offset != nil { - offsetQ = swag.FormatInt64(*o.Offset) - } - if offsetQ != "" { - qs.Set("offset", offsetQ) - } - - var orderQ string - if o.Order != nil { - orderQ = *o.Order - } - if orderQ != "" { - qs.Set("order", orderQ) - } - - var sortQ string - if o.Sort != nil { - sortQ = *o.Sort - } - if sortQ != "" { - qs.Set("sort", sortQ) - } - - var tenantQ string - if o.Tenant != nil { - tenantQ = *o.Tenant - } - if tenantQ != "" { - qs.Set("tenant", tenantQ) - } - - _result.RawQuery = qs.Encode() - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *ObjectsListURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *ObjectsListURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *ObjectsListURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on ObjectsListURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on ObjectsListURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *ObjectsListURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/objects/objects_patch.go b/adapters/handlers/rest/operations/objects/objects_patch.go deleted file mode 100644 index 58e6bbdbbee67e350f80c9b3f1ad71c93569008a..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_patch.go +++ /dev/null @@ -1,84 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsPatchHandlerFunc turns a function with the right signature into a objects patch handler -type ObjectsPatchHandlerFunc func(ObjectsPatchParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn ObjectsPatchHandlerFunc) Handle(params ObjectsPatchParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// ObjectsPatchHandler interface for that can handle valid objects patch params -type ObjectsPatchHandler interface { - Handle(ObjectsPatchParams, *models.Principal) middleware.Responder -} - -// NewObjectsPatch creates a new http.Handler for the objects patch operation -func NewObjectsPatch(ctx *middleware.Context, handler ObjectsPatchHandler) *ObjectsPatch { - return &ObjectsPatch{Context: ctx, Handler: handler} -} - -/* - ObjectsPatch swagger:route PATCH /objects/{id} objects objectsPatch - -Update an Object based on its UUID (using patch semantics). - -Updates an Object. This method supports json-merge style patch semantics (RFC 7396). Provided meta-data and schema values are validated. LastUpdateTime is set to the time this function is called. -*/ -type ObjectsPatch struct { - Context *middleware.Context - Handler ObjectsPatchHandler -} - -func (o *ObjectsPatch) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewObjectsPatchParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/objects/objects_patch_parameters.go b/adapters/handlers/rest/operations/objects/objects_patch_parameters.go deleted file mode 100644 index 2d64047d957447a803ec302680411ec3be356222..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_patch_parameters.go +++ /dev/null @@ -1,160 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/validate" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewObjectsPatchParams creates a new ObjectsPatchParams object -// -// There are no default values defined in the spec. -func NewObjectsPatchParams() ObjectsPatchParams { - - return ObjectsPatchParams{} -} - -// ObjectsPatchParams contains all the bound params for the objects patch operation -// typically these are obtained from a http.Request -// -// swagger:parameters objects.patch -type ObjectsPatchParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` - - /*RFC 7396-style patch, the body contains the object to merge into the existing object. - In: body - */ - Body *models.Object - /*Determines how many replicas must acknowledge a request before it is considered successful - In: query - */ - ConsistencyLevel *string - /*Unique ID of the Object. - Required: true - In: path - */ - ID strfmt.UUID -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewObjectsPatchParams() beforehand. -func (o *ObjectsPatchParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - qs := runtime.Values(r.URL.Query()) - - if runtime.HasBody(r) { - defer r.Body.Close() - var body models.Object - if err := route.Consumer.Consume(r.Body, &body); err != nil { - res = append(res, errors.NewParseError("body", "body", "", err)) - } else { - // validate body object - if err := body.Validate(route.Formats); err != nil { - res = append(res, err) - } - - ctx := validate.WithOperationRequest(r.Context()) - if err := body.ContextValidate(ctx, route.Formats); err != nil { - res = append(res, err) - } - - if len(res) == 0 { - o.Body = &body - } - } - } - - qConsistencyLevel, qhkConsistencyLevel, _ := qs.GetOK("consistency_level") - if err := o.bindConsistencyLevel(qConsistencyLevel, qhkConsistencyLevel, route.Formats); err != nil { - res = append(res, err) - } - - rID, rhkID, _ := route.Params.GetOK("id") - if err := o.bindID(rID, rhkID, route.Formats); err != nil { - res = append(res, err) - } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// bindConsistencyLevel binds and validates parameter ConsistencyLevel from query. -func (o *ObjectsPatchParams) bindConsistencyLevel(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: false - // AllowEmptyValue: false - - if raw == "" { // empty values pass all other validations - return nil - } - o.ConsistencyLevel = &raw - - return nil -} - -// bindID binds and validates parameter ID from path. -func (o *ObjectsPatchParams) bindID(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - - // Format: uuid - value, err := formats.Parse("uuid", raw) - if err != nil { - return errors.InvalidType("id", "path", "strfmt.UUID", raw) - } - o.ID = *(value.(*strfmt.UUID)) - - if err := o.validateID(formats); err != nil { - return err - } - - return nil -} - -// validateID carries on validations for parameter ID -func (o *ObjectsPatchParams) validateID(formats strfmt.Registry) error { - - if err := validate.FormatOf("id", "path", "uuid", o.ID.String(), formats); err != nil { - return err - } - return nil -} diff --git a/adapters/handlers/rest/operations/objects/objects_patch_responses.go b/adapters/handlers/rest/operations/objects/objects_patch_responses.go deleted file mode 100644 index b4b6b4dc23c6cac61ffb90519fc13c2669ae01af..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_patch_responses.go +++ /dev/null @@ -1,260 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsPatchNoContentCode is the HTTP code returned for type ObjectsPatchNoContent -const ObjectsPatchNoContentCode int = 204 - -/* -ObjectsPatchNoContent Successfully applied. No content provided. - -swagger:response objectsPatchNoContent -*/ -type ObjectsPatchNoContent struct { -} - -// NewObjectsPatchNoContent creates ObjectsPatchNoContent with default headers values -func NewObjectsPatchNoContent() *ObjectsPatchNoContent { - - return &ObjectsPatchNoContent{} -} - -// WriteResponse to the client -func (o *ObjectsPatchNoContent) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(204) -} - -// ObjectsPatchBadRequestCode is the HTTP code returned for type ObjectsPatchBadRequest -const ObjectsPatchBadRequestCode int = 400 - -/* -ObjectsPatchBadRequest The patch-JSON is malformed. - -swagger:response objectsPatchBadRequest -*/ -type ObjectsPatchBadRequest struct { -} - -// NewObjectsPatchBadRequest creates ObjectsPatchBadRequest with default headers values -func NewObjectsPatchBadRequest() *ObjectsPatchBadRequest { - - return &ObjectsPatchBadRequest{} -} - -// WriteResponse to the client -func (o *ObjectsPatchBadRequest) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(400) -} - -// ObjectsPatchUnauthorizedCode is the HTTP code returned for type ObjectsPatchUnauthorized -const ObjectsPatchUnauthorizedCode int = 401 - -/* -ObjectsPatchUnauthorized Unauthorized or invalid credentials. - -swagger:response objectsPatchUnauthorized -*/ -type ObjectsPatchUnauthorized struct { -} - -// NewObjectsPatchUnauthorized creates ObjectsPatchUnauthorized with default headers values -func NewObjectsPatchUnauthorized() *ObjectsPatchUnauthorized { - - return &ObjectsPatchUnauthorized{} -} - -// WriteResponse to the client -func (o *ObjectsPatchUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// ObjectsPatchForbiddenCode is the HTTP code returned for type ObjectsPatchForbidden -const ObjectsPatchForbiddenCode int = 403 - -/* -ObjectsPatchForbidden Forbidden - -swagger:response objectsPatchForbidden -*/ -type ObjectsPatchForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsPatchForbidden creates ObjectsPatchForbidden with default headers values -func NewObjectsPatchForbidden() *ObjectsPatchForbidden { - - return &ObjectsPatchForbidden{} -} - -// WithPayload adds the payload to the objects patch forbidden response -func (o *ObjectsPatchForbidden) WithPayload(payload *models.ErrorResponse) *ObjectsPatchForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects patch forbidden response -func (o *ObjectsPatchForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsPatchForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsPatchNotFoundCode is the HTTP code returned for type ObjectsPatchNotFound -const ObjectsPatchNotFoundCode int = 404 - -/* -ObjectsPatchNotFound Successful query result but no resource was found. - -swagger:response objectsPatchNotFound -*/ -type ObjectsPatchNotFound struct { -} - -// NewObjectsPatchNotFound creates ObjectsPatchNotFound with default headers values -func NewObjectsPatchNotFound() *ObjectsPatchNotFound { - - return &ObjectsPatchNotFound{} -} - -// WriteResponse to the client -func (o *ObjectsPatchNotFound) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(404) -} - -// ObjectsPatchUnprocessableEntityCode is the HTTP code returned for type ObjectsPatchUnprocessableEntity -const ObjectsPatchUnprocessableEntityCode int = 422 - -/* -ObjectsPatchUnprocessableEntity The patch-JSON is valid but unprocessable. - -swagger:response objectsPatchUnprocessableEntity -*/ -type ObjectsPatchUnprocessableEntity struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsPatchUnprocessableEntity creates ObjectsPatchUnprocessableEntity with default headers values -func NewObjectsPatchUnprocessableEntity() *ObjectsPatchUnprocessableEntity { - - return &ObjectsPatchUnprocessableEntity{} -} - -// WithPayload adds the payload to the objects patch unprocessable entity response -func (o *ObjectsPatchUnprocessableEntity) WithPayload(payload *models.ErrorResponse) *ObjectsPatchUnprocessableEntity { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects patch unprocessable entity response -func (o *ObjectsPatchUnprocessableEntity) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsPatchUnprocessableEntity) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(422) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsPatchInternalServerErrorCode is the HTTP code returned for type ObjectsPatchInternalServerError -const ObjectsPatchInternalServerErrorCode int = 500 - -/* -ObjectsPatchInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response objectsPatchInternalServerError -*/ -type ObjectsPatchInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsPatchInternalServerError creates ObjectsPatchInternalServerError with default headers values -func NewObjectsPatchInternalServerError() *ObjectsPatchInternalServerError { - - return &ObjectsPatchInternalServerError{} -} - -// WithPayload adds the payload to the objects patch internal server error response -func (o *ObjectsPatchInternalServerError) WithPayload(payload *models.ErrorResponse) *ObjectsPatchInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects patch internal server error response -func (o *ObjectsPatchInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsPatchInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/objects/objects_patch_urlbuilder.go b/adapters/handlers/rest/operations/objects/objects_patch_urlbuilder.go deleted file mode 100644 index 57afc03bb477fd3b928a6fb8439a8a141e7e08fd..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_patch_urlbuilder.go +++ /dev/null @@ -1,126 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" - "strings" - - "github.com/go-openapi/strfmt" -) - -// ObjectsPatchURL generates an URL for the objects patch operation -type ObjectsPatchURL struct { - ID strfmt.UUID - - ConsistencyLevel *string - - _basePath string - // avoid unkeyed usage - _ struct{} -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *ObjectsPatchURL) WithBasePath(bp string) *ObjectsPatchURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *ObjectsPatchURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *ObjectsPatchURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/objects/{id}" - - id := o.ID.String() - if id != "" { - _path = strings.Replace(_path, "{id}", id, -1) - } else { - return nil, errors.New("id is required on ObjectsPatchURL") - } - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - qs := make(url.Values) - - var consistencyLevelQ string - if o.ConsistencyLevel != nil { - consistencyLevelQ = *o.ConsistencyLevel - } - if consistencyLevelQ != "" { - qs.Set("consistency_level", consistencyLevelQ) - } - - _result.RawQuery = qs.Encode() - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *ObjectsPatchURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *ObjectsPatchURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *ObjectsPatchURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on ObjectsPatchURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on ObjectsPatchURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *ObjectsPatchURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/objects/objects_references_create.go b/adapters/handlers/rest/operations/objects/objects_references_create.go deleted file mode 100644 index 5ba4009f182d9d9b0747be26adfc9ccf59cb89ea..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_references_create.go +++ /dev/null @@ -1,84 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsReferencesCreateHandlerFunc turns a function with the right signature into a objects references create handler -type ObjectsReferencesCreateHandlerFunc func(ObjectsReferencesCreateParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn ObjectsReferencesCreateHandlerFunc) Handle(params ObjectsReferencesCreateParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// ObjectsReferencesCreateHandler interface for that can handle valid objects references create params -type ObjectsReferencesCreateHandler interface { - Handle(ObjectsReferencesCreateParams, *models.Principal) middleware.Responder -} - -// NewObjectsReferencesCreate creates a new http.Handler for the objects references create operation -func NewObjectsReferencesCreate(ctx *middleware.Context, handler ObjectsReferencesCreateHandler) *ObjectsReferencesCreate { - return &ObjectsReferencesCreate{Context: ctx, Handler: handler} -} - -/* - ObjectsReferencesCreate swagger:route POST /objects/{id}/references/{propertyName} objects objectsReferencesCreate - -Add a single reference to a class-property. - -Add a single reference to a class-property. -*/ -type ObjectsReferencesCreate struct { - Context *middleware.Context - Handler ObjectsReferencesCreateHandler -} - -func (o *ObjectsReferencesCreate) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewObjectsReferencesCreateParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/objects/objects_references_create_parameters.go b/adapters/handlers/rest/operations/objects/objects_references_create_parameters.go deleted file mode 100644 index d42a3ba7b1b89096d6281291900e87c1f63196b0..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_references_create_parameters.go +++ /dev/null @@ -1,192 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "io" - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/validate" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewObjectsReferencesCreateParams creates a new ObjectsReferencesCreateParams object -// -// There are no default values defined in the spec. -func NewObjectsReferencesCreateParams() ObjectsReferencesCreateParams { - - return ObjectsReferencesCreateParams{} -} - -// ObjectsReferencesCreateParams contains all the bound params for the objects references create operation -// typically these are obtained from a http.Request -// -// swagger:parameters objects.references.create -type ObjectsReferencesCreateParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` - - /* - Required: true - In: body - */ - Body *models.SingleRef - /*Unique ID of the Object. - Required: true - In: path - */ - ID strfmt.UUID - /*Unique name of the property related to the Object. - Required: true - In: path - */ - PropertyName string - /*Specifies the tenant in a request targeting a multi-tenant class - In: query - */ - Tenant *string -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewObjectsReferencesCreateParams() beforehand. -func (o *ObjectsReferencesCreateParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - qs := runtime.Values(r.URL.Query()) - - if runtime.HasBody(r) { - defer r.Body.Close() - var body models.SingleRef - if err := route.Consumer.Consume(r.Body, &body); err != nil { - if err == io.EOF { - res = append(res, errors.Required("body", "body", "")) - } else { - res = append(res, errors.NewParseError("body", "body", "", err)) - } - } else { - // validate body object - if err := body.Validate(route.Formats); err != nil { - res = append(res, err) - } - - ctx := validate.WithOperationRequest(r.Context()) - if err := body.ContextValidate(ctx, route.Formats); err != nil { - res = append(res, err) - } - - if len(res) == 0 { - o.Body = &body - } - } - } else { - res = append(res, errors.Required("body", "body", "")) - } - - rID, rhkID, _ := route.Params.GetOK("id") - if err := o.bindID(rID, rhkID, route.Formats); err != nil { - res = append(res, err) - } - - rPropertyName, rhkPropertyName, _ := route.Params.GetOK("propertyName") - if err := o.bindPropertyName(rPropertyName, rhkPropertyName, route.Formats); err != nil { - res = append(res, err) - } - - qTenant, qhkTenant, _ := qs.GetOK("tenant") - if err := o.bindTenant(qTenant, qhkTenant, route.Formats); err != nil { - res = append(res, err) - } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// bindID binds and validates parameter ID from path. -func (o *ObjectsReferencesCreateParams) bindID(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - - // Format: uuid - value, err := formats.Parse("uuid", raw) - if err != nil { - return errors.InvalidType("id", "path", "strfmt.UUID", raw) - } - o.ID = *(value.(*strfmt.UUID)) - - if err := o.validateID(formats); err != nil { - return err - } - - return nil -} - -// validateID carries on validations for parameter ID -func (o *ObjectsReferencesCreateParams) validateID(formats strfmt.Registry) error { - - if err := validate.FormatOf("id", "path", "uuid", o.ID.String(), formats); err != nil { - return err - } - return nil -} - -// bindPropertyName binds and validates parameter PropertyName from path. -func (o *ObjectsReferencesCreateParams) bindPropertyName(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - o.PropertyName = raw - - return nil -} - -// bindTenant binds and validates parameter Tenant from query. -func (o *ObjectsReferencesCreateParams) bindTenant(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: false - // AllowEmptyValue: false - - if raw == "" { // empty values pass all other validations - return nil - } - o.Tenant = &raw - - return nil -} diff --git a/adapters/handlers/rest/operations/objects/objects_references_create_responses.go b/adapters/handlers/rest/operations/objects/objects_references_create_responses.go deleted file mode 100644 index c0349bd01aa5bc4cd1f2350a89a3e40052df3dc8..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_references_create_responses.go +++ /dev/null @@ -1,210 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsReferencesCreateOKCode is the HTTP code returned for type ObjectsReferencesCreateOK -const ObjectsReferencesCreateOKCode int = 200 - -/* -ObjectsReferencesCreateOK Successfully added the reference. - -swagger:response objectsReferencesCreateOK -*/ -type ObjectsReferencesCreateOK struct { -} - -// NewObjectsReferencesCreateOK creates ObjectsReferencesCreateOK with default headers values -func NewObjectsReferencesCreateOK() *ObjectsReferencesCreateOK { - - return &ObjectsReferencesCreateOK{} -} - -// WriteResponse to the client -func (o *ObjectsReferencesCreateOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(200) -} - -// ObjectsReferencesCreateUnauthorizedCode is the HTTP code returned for type ObjectsReferencesCreateUnauthorized -const ObjectsReferencesCreateUnauthorizedCode int = 401 - -/* -ObjectsReferencesCreateUnauthorized Unauthorized or invalid credentials. - -swagger:response objectsReferencesCreateUnauthorized -*/ -type ObjectsReferencesCreateUnauthorized struct { -} - -// NewObjectsReferencesCreateUnauthorized creates ObjectsReferencesCreateUnauthorized with default headers values -func NewObjectsReferencesCreateUnauthorized() *ObjectsReferencesCreateUnauthorized { - - return &ObjectsReferencesCreateUnauthorized{} -} - -// WriteResponse to the client -func (o *ObjectsReferencesCreateUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// ObjectsReferencesCreateForbiddenCode is the HTTP code returned for type ObjectsReferencesCreateForbidden -const ObjectsReferencesCreateForbiddenCode int = 403 - -/* -ObjectsReferencesCreateForbidden Forbidden - -swagger:response objectsReferencesCreateForbidden -*/ -type ObjectsReferencesCreateForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsReferencesCreateForbidden creates ObjectsReferencesCreateForbidden with default headers values -func NewObjectsReferencesCreateForbidden() *ObjectsReferencesCreateForbidden { - - return &ObjectsReferencesCreateForbidden{} -} - -// WithPayload adds the payload to the objects references create forbidden response -func (o *ObjectsReferencesCreateForbidden) WithPayload(payload *models.ErrorResponse) *ObjectsReferencesCreateForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects references create forbidden response -func (o *ObjectsReferencesCreateForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsReferencesCreateForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsReferencesCreateUnprocessableEntityCode is the HTTP code returned for type ObjectsReferencesCreateUnprocessableEntity -const ObjectsReferencesCreateUnprocessableEntityCode int = 422 - -/* -ObjectsReferencesCreateUnprocessableEntity Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the property exists or that it is a class? - -swagger:response objectsReferencesCreateUnprocessableEntity -*/ -type ObjectsReferencesCreateUnprocessableEntity struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsReferencesCreateUnprocessableEntity creates ObjectsReferencesCreateUnprocessableEntity with default headers values -func NewObjectsReferencesCreateUnprocessableEntity() *ObjectsReferencesCreateUnprocessableEntity { - - return &ObjectsReferencesCreateUnprocessableEntity{} -} - -// WithPayload adds the payload to the objects references create unprocessable entity response -func (o *ObjectsReferencesCreateUnprocessableEntity) WithPayload(payload *models.ErrorResponse) *ObjectsReferencesCreateUnprocessableEntity { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects references create unprocessable entity response -func (o *ObjectsReferencesCreateUnprocessableEntity) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsReferencesCreateUnprocessableEntity) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(422) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsReferencesCreateInternalServerErrorCode is the HTTP code returned for type ObjectsReferencesCreateInternalServerError -const ObjectsReferencesCreateInternalServerErrorCode int = 500 - -/* -ObjectsReferencesCreateInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response objectsReferencesCreateInternalServerError -*/ -type ObjectsReferencesCreateInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsReferencesCreateInternalServerError creates ObjectsReferencesCreateInternalServerError with default headers values -func NewObjectsReferencesCreateInternalServerError() *ObjectsReferencesCreateInternalServerError { - - return &ObjectsReferencesCreateInternalServerError{} -} - -// WithPayload adds the payload to the objects references create internal server error response -func (o *ObjectsReferencesCreateInternalServerError) WithPayload(payload *models.ErrorResponse) *ObjectsReferencesCreateInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects references create internal server error response -func (o *ObjectsReferencesCreateInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsReferencesCreateInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/objects/objects_references_create_urlbuilder.go b/adapters/handlers/rest/operations/objects/objects_references_create_urlbuilder.go deleted file mode 100644 index b293d2feaa20d6134d9451849a98bfd997629a91..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_references_create_urlbuilder.go +++ /dev/null @@ -1,134 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" - "strings" - - "github.com/go-openapi/strfmt" -) - -// ObjectsReferencesCreateURL generates an URL for the objects references create operation -type ObjectsReferencesCreateURL struct { - ID strfmt.UUID - PropertyName string - - Tenant *string - - _basePath string - // avoid unkeyed usage - _ struct{} -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *ObjectsReferencesCreateURL) WithBasePath(bp string) *ObjectsReferencesCreateURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *ObjectsReferencesCreateURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *ObjectsReferencesCreateURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/objects/{id}/references/{propertyName}" - - id := o.ID.String() - if id != "" { - _path = strings.Replace(_path, "{id}", id, -1) - } else { - return nil, errors.New("id is required on ObjectsReferencesCreateURL") - } - - propertyName := o.PropertyName - if propertyName != "" { - _path = strings.Replace(_path, "{propertyName}", propertyName, -1) - } else { - return nil, errors.New("propertyName is required on ObjectsReferencesCreateURL") - } - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - qs := make(url.Values) - - var tenantQ string - if o.Tenant != nil { - tenantQ = *o.Tenant - } - if tenantQ != "" { - qs.Set("tenant", tenantQ) - } - - _result.RawQuery = qs.Encode() - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *ObjectsReferencesCreateURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *ObjectsReferencesCreateURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *ObjectsReferencesCreateURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on ObjectsReferencesCreateURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on ObjectsReferencesCreateURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *ObjectsReferencesCreateURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/objects/objects_references_delete.go b/adapters/handlers/rest/operations/objects/objects_references_delete.go deleted file mode 100644 index 3cc4265f2e11ce857346b6c4b2baa3a4a9e2d1aa..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_references_delete.go +++ /dev/null @@ -1,84 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsReferencesDeleteHandlerFunc turns a function with the right signature into a objects references delete handler -type ObjectsReferencesDeleteHandlerFunc func(ObjectsReferencesDeleteParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn ObjectsReferencesDeleteHandlerFunc) Handle(params ObjectsReferencesDeleteParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// ObjectsReferencesDeleteHandler interface for that can handle valid objects references delete params -type ObjectsReferencesDeleteHandler interface { - Handle(ObjectsReferencesDeleteParams, *models.Principal) middleware.Responder -} - -// NewObjectsReferencesDelete creates a new http.Handler for the objects references delete operation -func NewObjectsReferencesDelete(ctx *middleware.Context, handler ObjectsReferencesDeleteHandler) *ObjectsReferencesDelete { - return &ObjectsReferencesDelete{Context: ctx, Handler: handler} -} - -/* - ObjectsReferencesDelete swagger:route DELETE /objects/{id}/references/{propertyName} objects objectsReferencesDelete - -Delete the single reference that is given in the body from the list of references that this property has. - -Delete the single reference that is given in the body from the list of references that this property has. -*/ -type ObjectsReferencesDelete struct { - Context *middleware.Context - Handler ObjectsReferencesDeleteHandler -} - -func (o *ObjectsReferencesDelete) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewObjectsReferencesDeleteParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/objects/objects_references_delete_parameters.go b/adapters/handlers/rest/operations/objects/objects_references_delete_parameters.go deleted file mode 100644 index c96ffd28e8d53f26637643bf1bc0b4b8c224e1b1..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_references_delete_parameters.go +++ /dev/null @@ -1,192 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "io" - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/validate" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewObjectsReferencesDeleteParams creates a new ObjectsReferencesDeleteParams object -// -// There are no default values defined in the spec. -func NewObjectsReferencesDeleteParams() ObjectsReferencesDeleteParams { - - return ObjectsReferencesDeleteParams{} -} - -// ObjectsReferencesDeleteParams contains all the bound params for the objects references delete operation -// typically these are obtained from a http.Request -// -// swagger:parameters objects.references.delete -type ObjectsReferencesDeleteParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` - - /* - Required: true - In: body - */ - Body *models.SingleRef - /*Unique ID of the Object. - Required: true - In: path - */ - ID strfmt.UUID - /*Unique name of the property related to the Object. - Required: true - In: path - */ - PropertyName string - /*Specifies the tenant in a request targeting a multi-tenant class - In: query - */ - Tenant *string -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewObjectsReferencesDeleteParams() beforehand. -func (o *ObjectsReferencesDeleteParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - qs := runtime.Values(r.URL.Query()) - - if runtime.HasBody(r) { - defer r.Body.Close() - var body models.SingleRef - if err := route.Consumer.Consume(r.Body, &body); err != nil { - if err == io.EOF { - res = append(res, errors.Required("body", "body", "")) - } else { - res = append(res, errors.NewParseError("body", "body", "", err)) - } - } else { - // validate body object - if err := body.Validate(route.Formats); err != nil { - res = append(res, err) - } - - ctx := validate.WithOperationRequest(r.Context()) - if err := body.ContextValidate(ctx, route.Formats); err != nil { - res = append(res, err) - } - - if len(res) == 0 { - o.Body = &body - } - } - } else { - res = append(res, errors.Required("body", "body", "")) - } - - rID, rhkID, _ := route.Params.GetOK("id") - if err := o.bindID(rID, rhkID, route.Formats); err != nil { - res = append(res, err) - } - - rPropertyName, rhkPropertyName, _ := route.Params.GetOK("propertyName") - if err := o.bindPropertyName(rPropertyName, rhkPropertyName, route.Formats); err != nil { - res = append(res, err) - } - - qTenant, qhkTenant, _ := qs.GetOK("tenant") - if err := o.bindTenant(qTenant, qhkTenant, route.Formats); err != nil { - res = append(res, err) - } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// bindID binds and validates parameter ID from path. -func (o *ObjectsReferencesDeleteParams) bindID(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - - // Format: uuid - value, err := formats.Parse("uuid", raw) - if err != nil { - return errors.InvalidType("id", "path", "strfmt.UUID", raw) - } - o.ID = *(value.(*strfmt.UUID)) - - if err := o.validateID(formats); err != nil { - return err - } - - return nil -} - -// validateID carries on validations for parameter ID -func (o *ObjectsReferencesDeleteParams) validateID(formats strfmt.Registry) error { - - if err := validate.FormatOf("id", "path", "uuid", o.ID.String(), formats); err != nil { - return err - } - return nil -} - -// bindPropertyName binds and validates parameter PropertyName from path. -func (o *ObjectsReferencesDeleteParams) bindPropertyName(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - o.PropertyName = raw - - return nil -} - -// bindTenant binds and validates parameter Tenant from query. -func (o *ObjectsReferencesDeleteParams) bindTenant(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: false - // AllowEmptyValue: false - - if raw == "" { // empty values pass all other validations - return nil - } - o.Tenant = &raw - - return nil -} diff --git a/adapters/handlers/rest/operations/objects/objects_references_delete_responses.go b/adapters/handlers/rest/operations/objects/objects_references_delete_responses.go deleted file mode 100644 index f4660f63e05d35963747970a460d800b1036d016..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_references_delete_responses.go +++ /dev/null @@ -1,210 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsReferencesDeleteNoContentCode is the HTTP code returned for type ObjectsReferencesDeleteNoContent -const ObjectsReferencesDeleteNoContentCode int = 204 - -/* -ObjectsReferencesDeleteNoContent Successfully deleted. - -swagger:response objectsReferencesDeleteNoContent -*/ -type ObjectsReferencesDeleteNoContent struct { -} - -// NewObjectsReferencesDeleteNoContent creates ObjectsReferencesDeleteNoContent with default headers values -func NewObjectsReferencesDeleteNoContent() *ObjectsReferencesDeleteNoContent { - - return &ObjectsReferencesDeleteNoContent{} -} - -// WriteResponse to the client -func (o *ObjectsReferencesDeleteNoContent) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(204) -} - -// ObjectsReferencesDeleteUnauthorizedCode is the HTTP code returned for type ObjectsReferencesDeleteUnauthorized -const ObjectsReferencesDeleteUnauthorizedCode int = 401 - -/* -ObjectsReferencesDeleteUnauthorized Unauthorized or invalid credentials. - -swagger:response objectsReferencesDeleteUnauthorized -*/ -type ObjectsReferencesDeleteUnauthorized struct { -} - -// NewObjectsReferencesDeleteUnauthorized creates ObjectsReferencesDeleteUnauthorized with default headers values -func NewObjectsReferencesDeleteUnauthorized() *ObjectsReferencesDeleteUnauthorized { - - return &ObjectsReferencesDeleteUnauthorized{} -} - -// WriteResponse to the client -func (o *ObjectsReferencesDeleteUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// ObjectsReferencesDeleteForbiddenCode is the HTTP code returned for type ObjectsReferencesDeleteForbidden -const ObjectsReferencesDeleteForbiddenCode int = 403 - -/* -ObjectsReferencesDeleteForbidden Forbidden - -swagger:response objectsReferencesDeleteForbidden -*/ -type ObjectsReferencesDeleteForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsReferencesDeleteForbidden creates ObjectsReferencesDeleteForbidden with default headers values -func NewObjectsReferencesDeleteForbidden() *ObjectsReferencesDeleteForbidden { - - return &ObjectsReferencesDeleteForbidden{} -} - -// WithPayload adds the payload to the objects references delete forbidden response -func (o *ObjectsReferencesDeleteForbidden) WithPayload(payload *models.ErrorResponse) *ObjectsReferencesDeleteForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects references delete forbidden response -func (o *ObjectsReferencesDeleteForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsReferencesDeleteForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsReferencesDeleteNotFoundCode is the HTTP code returned for type ObjectsReferencesDeleteNotFound -const ObjectsReferencesDeleteNotFoundCode int = 404 - -/* -ObjectsReferencesDeleteNotFound Successful query result but no resource was found. - -swagger:response objectsReferencesDeleteNotFound -*/ -type ObjectsReferencesDeleteNotFound struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsReferencesDeleteNotFound creates ObjectsReferencesDeleteNotFound with default headers values -func NewObjectsReferencesDeleteNotFound() *ObjectsReferencesDeleteNotFound { - - return &ObjectsReferencesDeleteNotFound{} -} - -// WithPayload adds the payload to the objects references delete not found response -func (o *ObjectsReferencesDeleteNotFound) WithPayload(payload *models.ErrorResponse) *ObjectsReferencesDeleteNotFound { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects references delete not found response -func (o *ObjectsReferencesDeleteNotFound) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsReferencesDeleteNotFound) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(404) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsReferencesDeleteInternalServerErrorCode is the HTTP code returned for type ObjectsReferencesDeleteInternalServerError -const ObjectsReferencesDeleteInternalServerErrorCode int = 500 - -/* -ObjectsReferencesDeleteInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response objectsReferencesDeleteInternalServerError -*/ -type ObjectsReferencesDeleteInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsReferencesDeleteInternalServerError creates ObjectsReferencesDeleteInternalServerError with default headers values -func NewObjectsReferencesDeleteInternalServerError() *ObjectsReferencesDeleteInternalServerError { - - return &ObjectsReferencesDeleteInternalServerError{} -} - -// WithPayload adds the payload to the objects references delete internal server error response -func (o *ObjectsReferencesDeleteInternalServerError) WithPayload(payload *models.ErrorResponse) *ObjectsReferencesDeleteInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects references delete internal server error response -func (o *ObjectsReferencesDeleteInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsReferencesDeleteInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/objects/objects_references_delete_urlbuilder.go b/adapters/handlers/rest/operations/objects/objects_references_delete_urlbuilder.go deleted file mode 100644 index a84ab4eb33facb707bd5b82e2800b945eab25f9b..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_references_delete_urlbuilder.go +++ /dev/null @@ -1,134 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" - "strings" - - "github.com/go-openapi/strfmt" -) - -// ObjectsReferencesDeleteURL generates an URL for the objects references delete operation -type ObjectsReferencesDeleteURL struct { - ID strfmt.UUID - PropertyName string - - Tenant *string - - _basePath string - // avoid unkeyed usage - _ struct{} -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *ObjectsReferencesDeleteURL) WithBasePath(bp string) *ObjectsReferencesDeleteURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *ObjectsReferencesDeleteURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *ObjectsReferencesDeleteURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/objects/{id}/references/{propertyName}" - - id := o.ID.String() - if id != "" { - _path = strings.Replace(_path, "{id}", id, -1) - } else { - return nil, errors.New("id is required on ObjectsReferencesDeleteURL") - } - - propertyName := o.PropertyName - if propertyName != "" { - _path = strings.Replace(_path, "{propertyName}", propertyName, -1) - } else { - return nil, errors.New("propertyName is required on ObjectsReferencesDeleteURL") - } - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - qs := make(url.Values) - - var tenantQ string - if o.Tenant != nil { - tenantQ = *o.Tenant - } - if tenantQ != "" { - qs.Set("tenant", tenantQ) - } - - _result.RawQuery = qs.Encode() - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *ObjectsReferencesDeleteURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *ObjectsReferencesDeleteURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *ObjectsReferencesDeleteURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on ObjectsReferencesDeleteURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on ObjectsReferencesDeleteURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *ObjectsReferencesDeleteURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/objects/objects_references_update.go b/adapters/handlers/rest/operations/objects/objects_references_update.go deleted file mode 100644 index 6209747719f510a31a5a650f460950baa1c915cd..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_references_update.go +++ /dev/null @@ -1,84 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsReferencesUpdateHandlerFunc turns a function with the right signature into a objects references update handler -type ObjectsReferencesUpdateHandlerFunc func(ObjectsReferencesUpdateParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn ObjectsReferencesUpdateHandlerFunc) Handle(params ObjectsReferencesUpdateParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// ObjectsReferencesUpdateHandler interface for that can handle valid objects references update params -type ObjectsReferencesUpdateHandler interface { - Handle(ObjectsReferencesUpdateParams, *models.Principal) middleware.Responder -} - -// NewObjectsReferencesUpdate creates a new http.Handler for the objects references update operation -func NewObjectsReferencesUpdate(ctx *middleware.Context, handler ObjectsReferencesUpdateHandler) *ObjectsReferencesUpdate { - return &ObjectsReferencesUpdate{Context: ctx, Handler: handler} -} - -/* - ObjectsReferencesUpdate swagger:route PUT /objects/{id}/references/{propertyName} objects objectsReferencesUpdate - -Replace all references to a class-property. - -Replace all references to a class-property. -*/ -type ObjectsReferencesUpdate struct { - Context *middleware.Context - Handler ObjectsReferencesUpdateHandler -} - -func (o *ObjectsReferencesUpdate) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewObjectsReferencesUpdateParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/objects/objects_references_update_parameters.go b/adapters/handlers/rest/operations/objects/objects_references_update_parameters.go deleted file mode 100644 index 6129241db7db08b4d4234e54d6d3012ca2f7df55..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_references_update_parameters.go +++ /dev/null @@ -1,192 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "io" - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/validate" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewObjectsReferencesUpdateParams creates a new ObjectsReferencesUpdateParams object -// -// There are no default values defined in the spec. -func NewObjectsReferencesUpdateParams() ObjectsReferencesUpdateParams { - - return ObjectsReferencesUpdateParams{} -} - -// ObjectsReferencesUpdateParams contains all the bound params for the objects references update operation -// typically these are obtained from a http.Request -// -// swagger:parameters objects.references.update -type ObjectsReferencesUpdateParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` - - /* - Required: true - In: body - */ - Body models.MultipleRef - /*Unique ID of the Object. - Required: true - In: path - */ - ID strfmt.UUID - /*Unique name of the property related to the Object. - Required: true - In: path - */ - PropertyName string - /*Specifies the tenant in a request targeting a multi-tenant class - In: query - */ - Tenant *string -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewObjectsReferencesUpdateParams() beforehand. -func (o *ObjectsReferencesUpdateParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - qs := runtime.Values(r.URL.Query()) - - if runtime.HasBody(r) { - defer r.Body.Close() - var body models.MultipleRef - if err := route.Consumer.Consume(r.Body, &body); err != nil { - if err == io.EOF { - res = append(res, errors.Required("body", "body", "")) - } else { - res = append(res, errors.NewParseError("body", "body", "", err)) - } - } else { - // validate body object - if err := body.Validate(route.Formats); err != nil { - res = append(res, err) - } - - ctx := validate.WithOperationRequest(r.Context()) - if err := body.ContextValidate(ctx, route.Formats); err != nil { - res = append(res, err) - } - - if len(res) == 0 { - o.Body = body - } - } - } else { - res = append(res, errors.Required("body", "body", "")) - } - - rID, rhkID, _ := route.Params.GetOK("id") - if err := o.bindID(rID, rhkID, route.Formats); err != nil { - res = append(res, err) - } - - rPropertyName, rhkPropertyName, _ := route.Params.GetOK("propertyName") - if err := o.bindPropertyName(rPropertyName, rhkPropertyName, route.Formats); err != nil { - res = append(res, err) - } - - qTenant, qhkTenant, _ := qs.GetOK("tenant") - if err := o.bindTenant(qTenant, qhkTenant, route.Formats); err != nil { - res = append(res, err) - } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// bindID binds and validates parameter ID from path. -func (o *ObjectsReferencesUpdateParams) bindID(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - - // Format: uuid - value, err := formats.Parse("uuid", raw) - if err != nil { - return errors.InvalidType("id", "path", "strfmt.UUID", raw) - } - o.ID = *(value.(*strfmt.UUID)) - - if err := o.validateID(formats); err != nil { - return err - } - - return nil -} - -// validateID carries on validations for parameter ID -func (o *ObjectsReferencesUpdateParams) validateID(formats strfmt.Registry) error { - - if err := validate.FormatOf("id", "path", "uuid", o.ID.String(), formats); err != nil { - return err - } - return nil -} - -// bindPropertyName binds and validates parameter PropertyName from path. -func (o *ObjectsReferencesUpdateParams) bindPropertyName(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - o.PropertyName = raw - - return nil -} - -// bindTenant binds and validates parameter Tenant from query. -func (o *ObjectsReferencesUpdateParams) bindTenant(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: false - // AllowEmptyValue: false - - if raw == "" { // empty values pass all other validations - return nil - } - o.Tenant = &raw - - return nil -} diff --git a/adapters/handlers/rest/operations/objects/objects_references_update_responses.go b/adapters/handlers/rest/operations/objects/objects_references_update_responses.go deleted file mode 100644 index f5151be5892cff9bd80c03d384197318ec2ac1f2..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_references_update_responses.go +++ /dev/null @@ -1,210 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsReferencesUpdateOKCode is the HTTP code returned for type ObjectsReferencesUpdateOK -const ObjectsReferencesUpdateOKCode int = 200 - -/* -ObjectsReferencesUpdateOK Successfully replaced all the references. - -swagger:response objectsReferencesUpdateOK -*/ -type ObjectsReferencesUpdateOK struct { -} - -// NewObjectsReferencesUpdateOK creates ObjectsReferencesUpdateOK with default headers values -func NewObjectsReferencesUpdateOK() *ObjectsReferencesUpdateOK { - - return &ObjectsReferencesUpdateOK{} -} - -// WriteResponse to the client -func (o *ObjectsReferencesUpdateOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(200) -} - -// ObjectsReferencesUpdateUnauthorizedCode is the HTTP code returned for type ObjectsReferencesUpdateUnauthorized -const ObjectsReferencesUpdateUnauthorizedCode int = 401 - -/* -ObjectsReferencesUpdateUnauthorized Unauthorized or invalid credentials. - -swagger:response objectsReferencesUpdateUnauthorized -*/ -type ObjectsReferencesUpdateUnauthorized struct { -} - -// NewObjectsReferencesUpdateUnauthorized creates ObjectsReferencesUpdateUnauthorized with default headers values -func NewObjectsReferencesUpdateUnauthorized() *ObjectsReferencesUpdateUnauthorized { - - return &ObjectsReferencesUpdateUnauthorized{} -} - -// WriteResponse to the client -func (o *ObjectsReferencesUpdateUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// ObjectsReferencesUpdateForbiddenCode is the HTTP code returned for type ObjectsReferencesUpdateForbidden -const ObjectsReferencesUpdateForbiddenCode int = 403 - -/* -ObjectsReferencesUpdateForbidden Forbidden - -swagger:response objectsReferencesUpdateForbidden -*/ -type ObjectsReferencesUpdateForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsReferencesUpdateForbidden creates ObjectsReferencesUpdateForbidden with default headers values -func NewObjectsReferencesUpdateForbidden() *ObjectsReferencesUpdateForbidden { - - return &ObjectsReferencesUpdateForbidden{} -} - -// WithPayload adds the payload to the objects references update forbidden response -func (o *ObjectsReferencesUpdateForbidden) WithPayload(payload *models.ErrorResponse) *ObjectsReferencesUpdateForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects references update forbidden response -func (o *ObjectsReferencesUpdateForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsReferencesUpdateForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsReferencesUpdateUnprocessableEntityCode is the HTTP code returned for type ObjectsReferencesUpdateUnprocessableEntity -const ObjectsReferencesUpdateUnprocessableEntityCode int = 422 - -/* -ObjectsReferencesUpdateUnprocessableEntity Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the property exists or that it is a class? - -swagger:response objectsReferencesUpdateUnprocessableEntity -*/ -type ObjectsReferencesUpdateUnprocessableEntity struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsReferencesUpdateUnprocessableEntity creates ObjectsReferencesUpdateUnprocessableEntity with default headers values -func NewObjectsReferencesUpdateUnprocessableEntity() *ObjectsReferencesUpdateUnprocessableEntity { - - return &ObjectsReferencesUpdateUnprocessableEntity{} -} - -// WithPayload adds the payload to the objects references update unprocessable entity response -func (o *ObjectsReferencesUpdateUnprocessableEntity) WithPayload(payload *models.ErrorResponse) *ObjectsReferencesUpdateUnprocessableEntity { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects references update unprocessable entity response -func (o *ObjectsReferencesUpdateUnprocessableEntity) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsReferencesUpdateUnprocessableEntity) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(422) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsReferencesUpdateInternalServerErrorCode is the HTTP code returned for type ObjectsReferencesUpdateInternalServerError -const ObjectsReferencesUpdateInternalServerErrorCode int = 500 - -/* -ObjectsReferencesUpdateInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response objectsReferencesUpdateInternalServerError -*/ -type ObjectsReferencesUpdateInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsReferencesUpdateInternalServerError creates ObjectsReferencesUpdateInternalServerError with default headers values -func NewObjectsReferencesUpdateInternalServerError() *ObjectsReferencesUpdateInternalServerError { - - return &ObjectsReferencesUpdateInternalServerError{} -} - -// WithPayload adds the payload to the objects references update internal server error response -func (o *ObjectsReferencesUpdateInternalServerError) WithPayload(payload *models.ErrorResponse) *ObjectsReferencesUpdateInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects references update internal server error response -func (o *ObjectsReferencesUpdateInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsReferencesUpdateInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/objects/objects_references_update_urlbuilder.go b/adapters/handlers/rest/operations/objects/objects_references_update_urlbuilder.go deleted file mode 100644 index 67cf952df1a7ab6fbb88805e4aab47ec086ea9a5..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_references_update_urlbuilder.go +++ /dev/null @@ -1,134 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" - "strings" - - "github.com/go-openapi/strfmt" -) - -// ObjectsReferencesUpdateURL generates an URL for the objects references update operation -type ObjectsReferencesUpdateURL struct { - ID strfmt.UUID - PropertyName string - - Tenant *string - - _basePath string - // avoid unkeyed usage - _ struct{} -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *ObjectsReferencesUpdateURL) WithBasePath(bp string) *ObjectsReferencesUpdateURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *ObjectsReferencesUpdateURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *ObjectsReferencesUpdateURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/objects/{id}/references/{propertyName}" - - id := o.ID.String() - if id != "" { - _path = strings.Replace(_path, "{id}", id, -1) - } else { - return nil, errors.New("id is required on ObjectsReferencesUpdateURL") - } - - propertyName := o.PropertyName - if propertyName != "" { - _path = strings.Replace(_path, "{propertyName}", propertyName, -1) - } else { - return nil, errors.New("propertyName is required on ObjectsReferencesUpdateURL") - } - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - qs := make(url.Values) - - var tenantQ string - if o.Tenant != nil { - tenantQ = *o.Tenant - } - if tenantQ != "" { - qs.Set("tenant", tenantQ) - } - - _result.RawQuery = qs.Encode() - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *ObjectsReferencesUpdateURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *ObjectsReferencesUpdateURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *ObjectsReferencesUpdateURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on ObjectsReferencesUpdateURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on ObjectsReferencesUpdateURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *ObjectsReferencesUpdateURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/objects/objects_update.go b/adapters/handlers/rest/operations/objects/objects_update.go deleted file mode 100644 index 33f82cd5a4aa3147476a2c175c36a1560adffb0e..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_update.go +++ /dev/null @@ -1,84 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsUpdateHandlerFunc turns a function with the right signature into a objects update handler -type ObjectsUpdateHandlerFunc func(ObjectsUpdateParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn ObjectsUpdateHandlerFunc) Handle(params ObjectsUpdateParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// ObjectsUpdateHandler interface for that can handle valid objects update params -type ObjectsUpdateHandler interface { - Handle(ObjectsUpdateParams, *models.Principal) middleware.Responder -} - -// NewObjectsUpdate creates a new http.Handler for the objects update operation -func NewObjectsUpdate(ctx *middleware.Context, handler ObjectsUpdateHandler) *ObjectsUpdate { - return &ObjectsUpdate{Context: ctx, Handler: handler} -} - -/* - ObjectsUpdate swagger:route PUT /objects/{id} objects objectsUpdate - -Update an Object based on its UUID. - -Updates an Object's data. Given meta-data and schema values are validated. LastUpdateTime is set to the time this function is called. -*/ -type ObjectsUpdate struct { - Context *middleware.Context - Handler ObjectsUpdateHandler -} - -func (o *ObjectsUpdate) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewObjectsUpdateParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/objects/objects_update_parameters.go b/adapters/handlers/rest/operations/objects/objects_update_parameters.go deleted file mode 100644 index 6c24f4018c3f91d59c9d46560c565c0b923bc084..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_update_parameters.go +++ /dev/null @@ -1,168 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "io" - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/validate" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewObjectsUpdateParams creates a new ObjectsUpdateParams object -// -// There are no default values defined in the spec. -func NewObjectsUpdateParams() ObjectsUpdateParams { - - return ObjectsUpdateParams{} -} - -// ObjectsUpdateParams contains all the bound params for the objects update operation -// typically these are obtained from a http.Request -// -// swagger:parameters objects.update -type ObjectsUpdateParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` - - /* - Required: true - In: body - */ - Body *models.Object - /*Determines how many replicas must acknowledge a request before it is considered successful - In: query - */ - ConsistencyLevel *string - /*Unique ID of the Object. - Required: true - In: path - */ - ID strfmt.UUID -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewObjectsUpdateParams() beforehand. -func (o *ObjectsUpdateParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - qs := runtime.Values(r.URL.Query()) - - if runtime.HasBody(r) { - defer r.Body.Close() - var body models.Object - if err := route.Consumer.Consume(r.Body, &body); err != nil { - if err == io.EOF { - res = append(res, errors.Required("body", "body", "")) - } else { - res = append(res, errors.NewParseError("body", "body", "", err)) - } - } else { - // validate body object - if err := body.Validate(route.Formats); err != nil { - res = append(res, err) - } - - ctx := validate.WithOperationRequest(r.Context()) - if err := body.ContextValidate(ctx, route.Formats); err != nil { - res = append(res, err) - } - - if len(res) == 0 { - o.Body = &body - } - } - } else { - res = append(res, errors.Required("body", "body", "")) - } - - qConsistencyLevel, qhkConsistencyLevel, _ := qs.GetOK("consistency_level") - if err := o.bindConsistencyLevel(qConsistencyLevel, qhkConsistencyLevel, route.Formats); err != nil { - res = append(res, err) - } - - rID, rhkID, _ := route.Params.GetOK("id") - if err := o.bindID(rID, rhkID, route.Formats); err != nil { - res = append(res, err) - } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// bindConsistencyLevel binds and validates parameter ConsistencyLevel from query. -func (o *ObjectsUpdateParams) bindConsistencyLevel(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: false - // AllowEmptyValue: false - - if raw == "" { // empty values pass all other validations - return nil - } - o.ConsistencyLevel = &raw - - return nil -} - -// bindID binds and validates parameter ID from path. -func (o *ObjectsUpdateParams) bindID(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - - // Format: uuid - value, err := formats.Parse("uuid", raw) - if err != nil { - return errors.InvalidType("id", "path", "strfmt.UUID", raw) - } - o.ID = *(value.(*strfmt.UUID)) - - if err := o.validateID(formats); err != nil { - return err - } - - return nil -} - -// validateID carries on validations for parameter ID -func (o *ObjectsUpdateParams) validateID(formats strfmt.Registry) error { - - if err := validate.FormatOf("id", "path", "uuid", o.ID.String(), formats); err != nil { - return err - } - return nil -} diff --git a/adapters/handlers/rest/operations/objects/objects_update_responses.go b/adapters/handlers/rest/operations/objects/objects_update_responses.go deleted file mode 100644 index 6f8caaba4e64e84f7dbf0448ccc7cb1c374142bb..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_update_responses.go +++ /dev/null @@ -1,255 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsUpdateOKCode is the HTTP code returned for type ObjectsUpdateOK -const ObjectsUpdateOKCode int = 200 - -/* -ObjectsUpdateOK Successfully received. - -swagger:response objectsUpdateOK -*/ -type ObjectsUpdateOK struct { - - /* - In: Body - */ - Payload *models.Object `json:"body,omitempty"` -} - -// NewObjectsUpdateOK creates ObjectsUpdateOK with default headers values -func NewObjectsUpdateOK() *ObjectsUpdateOK { - - return &ObjectsUpdateOK{} -} - -// WithPayload adds the payload to the objects update o k response -func (o *ObjectsUpdateOK) WithPayload(payload *models.Object) *ObjectsUpdateOK { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects update o k response -func (o *ObjectsUpdateOK) SetPayload(payload *models.Object) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsUpdateOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(200) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsUpdateUnauthorizedCode is the HTTP code returned for type ObjectsUpdateUnauthorized -const ObjectsUpdateUnauthorizedCode int = 401 - -/* -ObjectsUpdateUnauthorized Unauthorized or invalid credentials. - -swagger:response objectsUpdateUnauthorized -*/ -type ObjectsUpdateUnauthorized struct { -} - -// NewObjectsUpdateUnauthorized creates ObjectsUpdateUnauthorized with default headers values -func NewObjectsUpdateUnauthorized() *ObjectsUpdateUnauthorized { - - return &ObjectsUpdateUnauthorized{} -} - -// WriteResponse to the client -func (o *ObjectsUpdateUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// ObjectsUpdateForbiddenCode is the HTTP code returned for type ObjectsUpdateForbidden -const ObjectsUpdateForbiddenCode int = 403 - -/* -ObjectsUpdateForbidden Forbidden - -swagger:response objectsUpdateForbidden -*/ -type ObjectsUpdateForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsUpdateForbidden creates ObjectsUpdateForbidden with default headers values -func NewObjectsUpdateForbidden() *ObjectsUpdateForbidden { - - return &ObjectsUpdateForbidden{} -} - -// WithPayload adds the payload to the objects update forbidden response -func (o *ObjectsUpdateForbidden) WithPayload(payload *models.ErrorResponse) *ObjectsUpdateForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects update forbidden response -func (o *ObjectsUpdateForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsUpdateForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsUpdateNotFoundCode is the HTTP code returned for type ObjectsUpdateNotFound -const ObjectsUpdateNotFoundCode int = 404 - -/* -ObjectsUpdateNotFound Successful query result but no resource was found. - -swagger:response objectsUpdateNotFound -*/ -type ObjectsUpdateNotFound struct { -} - -// NewObjectsUpdateNotFound creates ObjectsUpdateNotFound with default headers values -func NewObjectsUpdateNotFound() *ObjectsUpdateNotFound { - - return &ObjectsUpdateNotFound{} -} - -// WriteResponse to the client -func (o *ObjectsUpdateNotFound) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(404) -} - -// ObjectsUpdateUnprocessableEntityCode is the HTTP code returned for type ObjectsUpdateUnprocessableEntity -const ObjectsUpdateUnprocessableEntityCode int = 422 - -/* -ObjectsUpdateUnprocessableEntity Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file? - -swagger:response objectsUpdateUnprocessableEntity -*/ -type ObjectsUpdateUnprocessableEntity struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsUpdateUnprocessableEntity creates ObjectsUpdateUnprocessableEntity with default headers values -func NewObjectsUpdateUnprocessableEntity() *ObjectsUpdateUnprocessableEntity { - - return &ObjectsUpdateUnprocessableEntity{} -} - -// WithPayload adds the payload to the objects update unprocessable entity response -func (o *ObjectsUpdateUnprocessableEntity) WithPayload(payload *models.ErrorResponse) *ObjectsUpdateUnprocessableEntity { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects update unprocessable entity response -func (o *ObjectsUpdateUnprocessableEntity) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsUpdateUnprocessableEntity) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(422) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsUpdateInternalServerErrorCode is the HTTP code returned for type ObjectsUpdateInternalServerError -const ObjectsUpdateInternalServerErrorCode int = 500 - -/* -ObjectsUpdateInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response objectsUpdateInternalServerError -*/ -type ObjectsUpdateInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsUpdateInternalServerError creates ObjectsUpdateInternalServerError with default headers values -func NewObjectsUpdateInternalServerError() *ObjectsUpdateInternalServerError { - - return &ObjectsUpdateInternalServerError{} -} - -// WithPayload adds the payload to the objects update internal server error response -func (o *ObjectsUpdateInternalServerError) WithPayload(payload *models.ErrorResponse) *ObjectsUpdateInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects update internal server error response -func (o *ObjectsUpdateInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsUpdateInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/objects/objects_update_urlbuilder.go b/adapters/handlers/rest/operations/objects/objects_update_urlbuilder.go deleted file mode 100644 index 9f5cc63aa2a3f4f2737bc3076ef9c44949e92fa1..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_update_urlbuilder.go +++ /dev/null @@ -1,126 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" - "strings" - - "github.com/go-openapi/strfmt" -) - -// ObjectsUpdateURL generates an URL for the objects update operation -type ObjectsUpdateURL struct { - ID strfmt.UUID - - ConsistencyLevel *string - - _basePath string - // avoid unkeyed usage - _ struct{} -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *ObjectsUpdateURL) WithBasePath(bp string) *ObjectsUpdateURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *ObjectsUpdateURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *ObjectsUpdateURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/objects/{id}" - - id := o.ID.String() - if id != "" { - _path = strings.Replace(_path, "{id}", id, -1) - } else { - return nil, errors.New("id is required on ObjectsUpdateURL") - } - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - qs := make(url.Values) - - var consistencyLevelQ string - if o.ConsistencyLevel != nil { - consistencyLevelQ = *o.ConsistencyLevel - } - if consistencyLevelQ != "" { - qs.Set("consistency_level", consistencyLevelQ) - } - - _result.RawQuery = qs.Encode() - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *ObjectsUpdateURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *ObjectsUpdateURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *ObjectsUpdateURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on ObjectsUpdateURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on ObjectsUpdateURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *ObjectsUpdateURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/objects/objects_validate.go b/adapters/handlers/rest/operations/objects/objects_validate.go deleted file mode 100644 index cd7feba4175cbcd9a394605c32bed3e968b3a8a0..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_validate.go +++ /dev/null @@ -1,84 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsValidateHandlerFunc turns a function with the right signature into a objects validate handler -type ObjectsValidateHandlerFunc func(ObjectsValidateParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn ObjectsValidateHandlerFunc) Handle(params ObjectsValidateParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// ObjectsValidateHandler interface for that can handle valid objects validate params -type ObjectsValidateHandler interface { - Handle(ObjectsValidateParams, *models.Principal) middleware.Responder -} - -// NewObjectsValidate creates a new http.Handler for the objects validate operation -func NewObjectsValidate(ctx *middleware.Context, handler ObjectsValidateHandler) *ObjectsValidate { - return &ObjectsValidate{Context: ctx, Handler: handler} -} - -/* - ObjectsValidate swagger:route POST /objects/validate objects objectsValidate - -Validate an Object based on a schema. - -Validate an Object's schema and meta-data. It has to be based on a schema, which is related to the given Object to be accepted by this validation. -*/ -type ObjectsValidate struct { - Context *middleware.Context - Handler ObjectsValidateHandler -} - -func (o *ObjectsValidate) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewObjectsValidateParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/objects/objects_validate_parameters.go b/adapters/handlers/rest/operations/objects/objects_validate_parameters.go deleted file mode 100644 index 84e940dd68fc2f6f395ed776b8f62e3d1e329d9d..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_validate_parameters.go +++ /dev/null @@ -1,95 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "io" - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/validate" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewObjectsValidateParams creates a new ObjectsValidateParams object -// -// There are no default values defined in the spec. -func NewObjectsValidateParams() ObjectsValidateParams { - - return ObjectsValidateParams{} -} - -// ObjectsValidateParams contains all the bound params for the objects validate operation -// typically these are obtained from a http.Request -// -// swagger:parameters objects.validate -type ObjectsValidateParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` - - /* - Required: true - In: body - */ - Body *models.Object -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewObjectsValidateParams() beforehand. -func (o *ObjectsValidateParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - if runtime.HasBody(r) { - defer r.Body.Close() - var body models.Object - if err := route.Consumer.Consume(r.Body, &body); err != nil { - if err == io.EOF { - res = append(res, errors.Required("body", "body", "")) - } else { - res = append(res, errors.NewParseError("body", "body", "", err)) - } - } else { - // validate body object - if err := body.Validate(route.Formats); err != nil { - res = append(res, err) - } - - ctx := validate.WithOperationRequest(r.Context()) - if err := body.ContextValidate(ctx, route.Formats); err != nil { - res = append(res, err) - } - - if len(res) == 0 { - o.Body = &body - } - } - } else { - res = append(res, errors.Required("body", "body", "")) - } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/adapters/handlers/rest/operations/objects/objects_validate_responses.go b/adapters/handlers/rest/operations/objects/objects_validate_responses.go deleted file mode 100644 index d74fd82cd0832a50008bc3ea96d61daffd1c46e4..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_validate_responses.go +++ /dev/null @@ -1,210 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsValidateOKCode is the HTTP code returned for type ObjectsValidateOK -const ObjectsValidateOKCode int = 200 - -/* -ObjectsValidateOK Successfully validated. - -swagger:response objectsValidateOK -*/ -type ObjectsValidateOK struct { -} - -// NewObjectsValidateOK creates ObjectsValidateOK with default headers values -func NewObjectsValidateOK() *ObjectsValidateOK { - - return &ObjectsValidateOK{} -} - -// WriteResponse to the client -func (o *ObjectsValidateOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(200) -} - -// ObjectsValidateUnauthorizedCode is the HTTP code returned for type ObjectsValidateUnauthorized -const ObjectsValidateUnauthorizedCode int = 401 - -/* -ObjectsValidateUnauthorized Unauthorized or invalid credentials. - -swagger:response objectsValidateUnauthorized -*/ -type ObjectsValidateUnauthorized struct { -} - -// NewObjectsValidateUnauthorized creates ObjectsValidateUnauthorized with default headers values -func NewObjectsValidateUnauthorized() *ObjectsValidateUnauthorized { - - return &ObjectsValidateUnauthorized{} -} - -// WriteResponse to the client -func (o *ObjectsValidateUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// ObjectsValidateForbiddenCode is the HTTP code returned for type ObjectsValidateForbidden -const ObjectsValidateForbiddenCode int = 403 - -/* -ObjectsValidateForbidden Forbidden - -swagger:response objectsValidateForbidden -*/ -type ObjectsValidateForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsValidateForbidden creates ObjectsValidateForbidden with default headers values -func NewObjectsValidateForbidden() *ObjectsValidateForbidden { - - return &ObjectsValidateForbidden{} -} - -// WithPayload adds the payload to the objects validate forbidden response -func (o *ObjectsValidateForbidden) WithPayload(payload *models.ErrorResponse) *ObjectsValidateForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects validate forbidden response -func (o *ObjectsValidateForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsValidateForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsValidateUnprocessableEntityCode is the HTTP code returned for type ObjectsValidateUnprocessableEntity -const ObjectsValidateUnprocessableEntityCode int = 422 - -/* -ObjectsValidateUnprocessableEntity Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file? - -swagger:response objectsValidateUnprocessableEntity -*/ -type ObjectsValidateUnprocessableEntity struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsValidateUnprocessableEntity creates ObjectsValidateUnprocessableEntity with default headers values -func NewObjectsValidateUnprocessableEntity() *ObjectsValidateUnprocessableEntity { - - return &ObjectsValidateUnprocessableEntity{} -} - -// WithPayload adds the payload to the objects validate unprocessable entity response -func (o *ObjectsValidateUnprocessableEntity) WithPayload(payload *models.ErrorResponse) *ObjectsValidateUnprocessableEntity { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects validate unprocessable entity response -func (o *ObjectsValidateUnprocessableEntity) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsValidateUnprocessableEntity) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(422) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// ObjectsValidateInternalServerErrorCode is the HTTP code returned for type ObjectsValidateInternalServerError -const ObjectsValidateInternalServerErrorCode int = 500 - -/* -ObjectsValidateInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response objectsValidateInternalServerError -*/ -type ObjectsValidateInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewObjectsValidateInternalServerError creates ObjectsValidateInternalServerError with default headers values -func NewObjectsValidateInternalServerError() *ObjectsValidateInternalServerError { - - return &ObjectsValidateInternalServerError{} -} - -// WithPayload adds the payload to the objects validate internal server error response -func (o *ObjectsValidateInternalServerError) WithPayload(payload *models.ErrorResponse) *ObjectsValidateInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the objects validate internal server error response -func (o *ObjectsValidateInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *ObjectsValidateInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/objects/objects_validate_urlbuilder.go b/adapters/handlers/rest/operations/objects/objects_validate_urlbuilder.go deleted file mode 100644 index 8bf216ee4087996edcaed5e214893eb241b7f93f..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/objects/objects_validate_urlbuilder.go +++ /dev/null @@ -1,98 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" -) - -// ObjectsValidateURL generates an URL for the objects validate operation -type ObjectsValidateURL struct { - _basePath string -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *ObjectsValidateURL) WithBasePath(bp string) *ObjectsValidateURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *ObjectsValidateURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *ObjectsValidateURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/objects/validate" - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *ObjectsValidateURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *ObjectsValidateURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *ObjectsValidateURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on ObjectsValidateURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on ObjectsValidateURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *ObjectsValidateURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/schema/schema_cluster_status.go b/adapters/handlers/rest/operations/schema/schema_cluster_status.go deleted file mode 100644 index a05351acb08803831d3bda5e3bd7a92a474d8223..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/schema_cluster_status.go +++ /dev/null @@ -1,82 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// SchemaClusterStatusHandlerFunc turns a function with the right signature into a schema cluster status handler -type SchemaClusterStatusHandlerFunc func(SchemaClusterStatusParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn SchemaClusterStatusHandlerFunc) Handle(params SchemaClusterStatusParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// SchemaClusterStatusHandler interface for that can handle valid schema cluster status params -type SchemaClusterStatusHandler interface { - Handle(SchemaClusterStatusParams, *models.Principal) middleware.Responder -} - -// NewSchemaClusterStatus creates a new http.Handler for the schema cluster status operation -func NewSchemaClusterStatus(ctx *middleware.Context, handler SchemaClusterStatusHandler) *SchemaClusterStatus { - return &SchemaClusterStatus{Context: ctx, Handler: handler} -} - -/* - SchemaClusterStatus swagger:route GET /schema/cluster-status schema schemaClusterStatus - -SchemaClusterStatus schema cluster status API -*/ -type SchemaClusterStatus struct { - Context *middleware.Context - Handler SchemaClusterStatusHandler -} - -func (o *SchemaClusterStatus) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewSchemaClusterStatusParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/schema/schema_cluster_status_parameters.go b/adapters/handlers/rest/operations/schema/schema_cluster_status_parameters.go deleted file mode 100644 index c5582235e5dfc4acd518ca6bc5bb0b2ae9eb0f22..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/schema_cluster_status_parameters.go +++ /dev/null @@ -1,57 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime/middleware" -) - -// NewSchemaClusterStatusParams creates a new SchemaClusterStatusParams object -// -// There are no default values defined in the spec. -func NewSchemaClusterStatusParams() SchemaClusterStatusParams { - - return SchemaClusterStatusParams{} -} - -// SchemaClusterStatusParams contains all the bound params for the schema cluster status operation -// typically these are obtained from a http.Request -// -// swagger:parameters schema.cluster.status -type SchemaClusterStatusParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewSchemaClusterStatusParams() beforehand. -func (o *SchemaClusterStatusParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/adapters/handlers/rest/operations/schema/schema_cluster_status_responses.go b/adapters/handlers/rest/operations/schema/schema_cluster_status_responses.go deleted file mode 100644 index 45c7dcbb3087c25394b058396f4dbf906e52605a..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/schema_cluster_status_responses.go +++ /dev/null @@ -1,115 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// SchemaClusterStatusOKCode is the HTTP code returned for type SchemaClusterStatusOK -const SchemaClusterStatusOKCode int = 200 - -/* -SchemaClusterStatusOK The schema in the cluster is in sync. - -swagger:response schemaClusterStatusOK -*/ -type SchemaClusterStatusOK struct { - - /* - In: Body - */ - Payload *models.SchemaClusterStatus `json:"body,omitempty"` -} - -// NewSchemaClusterStatusOK creates SchemaClusterStatusOK with default headers values -func NewSchemaClusterStatusOK() *SchemaClusterStatusOK { - - return &SchemaClusterStatusOK{} -} - -// WithPayload adds the payload to the schema cluster status o k response -func (o *SchemaClusterStatusOK) WithPayload(payload *models.SchemaClusterStatus) *SchemaClusterStatusOK { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the schema cluster status o k response -func (o *SchemaClusterStatusOK) SetPayload(payload *models.SchemaClusterStatus) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *SchemaClusterStatusOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(200) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// SchemaClusterStatusInternalServerErrorCode is the HTTP code returned for type SchemaClusterStatusInternalServerError -const SchemaClusterStatusInternalServerErrorCode int = 500 - -/* -SchemaClusterStatusInternalServerError The schema is either out of sync (see response body) or the sync check could not be completed. - -swagger:response schemaClusterStatusInternalServerError -*/ -type SchemaClusterStatusInternalServerError struct { - - /* - In: Body - */ - Payload *models.SchemaClusterStatus `json:"body,omitempty"` -} - -// NewSchemaClusterStatusInternalServerError creates SchemaClusterStatusInternalServerError with default headers values -func NewSchemaClusterStatusInternalServerError() *SchemaClusterStatusInternalServerError { - - return &SchemaClusterStatusInternalServerError{} -} - -// WithPayload adds the payload to the schema cluster status internal server error response -func (o *SchemaClusterStatusInternalServerError) WithPayload(payload *models.SchemaClusterStatus) *SchemaClusterStatusInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the schema cluster status internal server error response -func (o *SchemaClusterStatusInternalServerError) SetPayload(payload *models.SchemaClusterStatus) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *SchemaClusterStatusInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/schema/schema_cluster_status_urlbuilder.go b/adapters/handlers/rest/operations/schema/schema_cluster_status_urlbuilder.go deleted file mode 100644 index d31e4e59fef1121470ebe84100619e5eba93a877..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/schema_cluster_status_urlbuilder.go +++ /dev/null @@ -1,98 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" -) - -// SchemaClusterStatusURL generates an URL for the schema cluster status operation -type SchemaClusterStatusURL struct { - _basePath string -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *SchemaClusterStatusURL) WithBasePath(bp string) *SchemaClusterStatusURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *SchemaClusterStatusURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *SchemaClusterStatusURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/schema/cluster-status" - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *SchemaClusterStatusURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *SchemaClusterStatusURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *SchemaClusterStatusURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on SchemaClusterStatusURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on SchemaClusterStatusURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *SchemaClusterStatusURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/schema/schema_dump.go b/adapters/handlers/rest/operations/schema/schema_dump.go deleted file mode 100644 index fd2345e3124da3f66f3d0971778e07b9be40b94d..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/schema_dump.go +++ /dev/null @@ -1,82 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// SchemaDumpHandlerFunc turns a function with the right signature into a schema dump handler -type SchemaDumpHandlerFunc func(SchemaDumpParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn SchemaDumpHandlerFunc) Handle(params SchemaDumpParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// SchemaDumpHandler interface for that can handle valid schema dump params -type SchemaDumpHandler interface { - Handle(SchemaDumpParams, *models.Principal) middleware.Responder -} - -// NewSchemaDump creates a new http.Handler for the schema dump operation -func NewSchemaDump(ctx *middleware.Context, handler SchemaDumpHandler) *SchemaDump { - return &SchemaDump{Context: ctx, Handler: handler} -} - -/* - SchemaDump swagger:route GET /schema schema schemaDump - -Dump the current the database schema. -*/ -type SchemaDump struct { - Context *middleware.Context - Handler SchemaDumpHandler -} - -func (o *SchemaDump) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewSchemaDumpParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/schema/schema_dump_parameters.go b/adapters/handlers/rest/operations/schema/schema_dump_parameters.go deleted file mode 100644 index 25af60c498174de83ba855c8afb5e27038ac1819..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/schema_dump_parameters.go +++ /dev/null @@ -1,57 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime/middleware" -) - -// NewSchemaDumpParams creates a new SchemaDumpParams object -// -// There are no default values defined in the spec. -func NewSchemaDumpParams() SchemaDumpParams { - - return SchemaDumpParams{} -} - -// SchemaDumpParams contains all the bound params for the schema dump operation -// typically these are obtained from a http.Request -// -// swagger:parameters schema.dump -type SchemaDumpParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewSchemaDumpParams() beforehand. -func (o *SchemaDumpParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/adapters/handlers/rest/operations/schema/schema_dump_responses.go b/adapters/handlers/rest/operations/schema/schema_dump_responses.go deleted file mode 100644 index 3e5abae8aa54e9f1d3a3b15ee7f43d09d03930d1..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/schema_dump_responses.go +++ /dev/null @@ -1,185 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// SchemaDumpOKCode is the HTTP code returned for type SchemaDumpOK -const SchemaDumpOKCode int = 200 - -/* -SchemaDumpOK Successfully dumped the database schema. - -swagger:response schemaDumpOK -*/ -type SchemaDumpOK struct { - - /* - In: Body - */ - Payload *models.Schema `json:"body,omitempty"` -} - -// NewSchemaDumpOK creates SchemaDumpOK with default headers values -func NewSchemaDumpOK() *SchemaDumpOK { - - return &SchemaDumpOK{} -} - -// WithPayload adds the payload to the schema dump o k response -func (o *SchemaDumpOK) WithPayload(payload *models.Schema) *SchemaDumpOK { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the schema dump o k response -func (o *SchemaDumpOK) SetPayload(payload *models.Schema) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *SchemaDumpOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(200) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// SchemaDumpUnauthorizedCode is the HTTP code returned for type SchemaDumpUnauthorized -const SchemaDumpUnauthorizedCode int = 401 - -/* -SchemaDumpUnauthorized Unauthorized or invalid credentials. - -swagger:response schemaDumpUnauthorized -*/ -type SchemaDumpUnauthorized struct { -} - -// NewSchemaDumpUnauthorized creates SchemaDumpUnauthorized with default headers values -func NewSchemaDumpUnauthorized() *SchemaDumpUnauthorized { - - return &SchemaDumpUnauthorized{} -} - -// WriteResponse to the client -func (o *SchemaDumpUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// SchemaDumpForbiddenCode is the HTTP code returned for type SchemaDumpForbidden -const SchemaDumpForbiddenCode int = 403 - -/* -SchemaDumpForbidden Forbidden - -swagger:response schemaDumpForbidden -*/ -type SchemaDumpForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewSchemaDumpForbidden creates SchemaDumpForbidden with default headers values -func NewSchemaDumpForbidden() *SchemaDumpForbidden { - - return &SchemaDumpForbidden{} -} - -// WithPayload adds the payload to the schema dump forbidden response -func (o *SchemaDumpForbidden) WithPayload(payload *models.ErrorResponse) *SchemaDumpForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the schema dump forbidden response -func (o *SchemaDumpForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *SchemaDumpForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// SchemaDumpInternalServerErrorCode is the HTTP code returned for type SchemaDumpInternalServerError -const SchemaDumpInternalServerErrorCode int = 500 - -/* -SchemaDumpInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response schemaDumpInternalServerError -*/ -type SchemaDumpInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewSchemaDumpInternalServerError creates SchemaDumpInternalServerError with default headers values -func NewSchemaDumpInternalServerError() *SchemaDumpInternalServerError { - - return &SchemaDumpInternalServerError{} -} - -// WithPayload adds the payload to the schema dump internal server error response -func (o *SchemaDumpInternalServerError) WithPayload(payload *models.ErrorResponse) *SchemaDumpInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the schema dump internal server error response -func (o *SchemaDumpInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *SchemaDumpInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/schema/schema_dump_urlbuilder.go b/adapters/handlers/rest/operations/schema/schema_dump_urlbuilder.go deleted file mode 100644 index 6453f7ba34f9d004381da19765d855ce6ffdf083..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/schema_dump_urlbuilder.go +++ /dev/null @@ -1,98 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" -) - -// SchemaDumpURL generates an URL for the schema dump operation -type SchemaDumpURL struct { - _basePath string -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *SchemaDumpURL) WithBasePath(bp string) *SchemaDumpURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *SchemaDumpURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *SchemaDumpURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/schema" - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *SchemaDumpURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *SchemaDumpURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *SchemaDumpURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on SchemaDumpURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on SchemaDumpURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *SchemaDumpURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/schema/schema_objects_create.go b/adapters/handlers/rest/operations/schema/schema_objects_create.go deleted file mode 100644 index 14ab25739600762ab9fd2818771c6e1809dcbe86..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/schema_objects_create.go +++ /dev/null @@ -1,82 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// SchemaObjectsCreateHandlerFunc turns a function with the right signature into a schema objects create handler -type SchemaObjectsCreateHandlerFunc func(SchemaObjectsCreateParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn SchemaObjectsCreateHandlerFunc) Handle(params SchemaObjectsCreateParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// SchemaObjectsCreateHandler interface for that can handle valid schema objects create params -type SchemaObjectsCreateHandler interface { - Handle(SchemaObjectsCreateParams, *models.Principal) middleware.Responder -} - -// NewSchemaObjectsCreate creates a new http.Handler for the schema objects create operation -func NewSchemaObjectsCreate(ctx *middleware.Context, handler SchemaObjectsCreateHandler) *SchemaObjectsCreate { - return &SchemaObjectsCreate{Context: ctx, Handler: handler} -} - -/* - SchemaObjectsCreate swagger:route POST /schema schema schemaObjectsCreate - -Create a new Object class in the schema. -*/ -type SchemaObjectsCreate struct { - Context *middleware.Context - Handler SchemaObjectsCreateHandler -} - -func (o *SchemaObjectsCreate) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewSchemaObjectsCreateParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/schema/schema_objects_create_parameters.go b/adapters/handlers/rest/operations/schema/schema_objects_create_parameters.go deleted file mode 100644 index b65687ff1da63500ff69852e7dc29aec29fb39ea..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/schema_objects_create_parameters.go +++ /dev/null @@ -1,95 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "io" - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/validate" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewSchemaObjectsCreateParams creates a new SchemaObjectsCreateParams object -// -// There are no default values defined in the spec. -func NewSchemaObjectsCreateParams() SchemaObjectsCreateParams { - - return SchemaObjectsCreateParams{} -} - -// SchemaObjectsCreateParams contains all the bound params for the schema objects create operation -// typically these are obtained from a http.Request -// -// swagger:parameters schema.objects.create -type SchemaObjectsCreateParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` - - /* - Required: true - In: body - */ - ObjectClass *models.Class -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewSchemaObjectsCreateParams() beforehand. -func (o *SchemaObjectsCreateParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - if runtime.HasBody(r) { - defer r.Body.Close() - var body models.Class - if err := route.Consumer.Consume(r.Body, &body); err != nil { - if err == io.EOF { - res = append(res, errors.Required("objectClass", "body", "")) - } else { - res = append(res, errors.NewParseError("objectClass", "body", "", err)) - } - } else { - // validate body object - if err := body.Validate(route.Formats); err != nil { - res = append(res, err) - } - - ctx := validate.WithOperationRequest(r.Context()) - if err := body.ContextValidate(ctx, route.Formats); err != nil { - res = append(res, err) - } - - if len(res) == 0 { - o.ObjectClass = &body - } - } - } else { - res = append(res, errors.Required("objectClass", "body", "")) - } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/adapters/handlers/rest/operations/schema/schema_objects_create_responses.go b/adapters/handlers/rest/operations/schema/schema_objects_create_responses.go deleted file mode 100644 index 2c62dd447b5ce97fdae2ac0b0651a9d53b5a38d9..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/schema_objects_create_responses.go +++ /dev/null @@ -1,230 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// SchemaObjectsCreateOKCode is the HTTP code returned for type SchemaObjectsCreateOK -const SchemaObjectsCreateOKCode int = 200 - -/* -SchemaObjectsCreateOK Added the new Object class to the schema. - -swagger:response schemaObjectsCreateOK -*/ -type SchemaObjectsCreateOK struct { - - /* - In: Body - */ - Payload *models.Class `json:"body,omitempty"` -} - -// NewSchemaObjectsCreateOK creates SchemaObjectsCreateOK with default headers values -func NewSchemaObjectsCreateOK() *SchemaObjectsCreateOK { - - return &SchemaObjectsCreateOK{} -} - -// WithPayload adds the payload to the schema objects create o k response -func (o *SchemaObjectsCreateOK) WithPayload(payload *models.Class) *SchemaObjectsCreateOK { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the schema objects create o k response -func (o *SchemaObjectsCreateOK) SetPayload(payload *models.Class) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *SchemaObjectsCreateOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(200) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// SchemaObjectsCreateUnauthorizedCode is the HTTP code returned for type SchemaObjectsCreateUnauthorized -const SchemaObjectsCreateUnauthorizedCode int = 401 - -/* -SchemaObjectsCreateUnauthorized Unauthorized or invalid credentials. - -swagger:response schemaObjectsCreateUnauthorized -*/ -type SchemaObjectsCreateUnauthorized struct { -} - -// NewSchemaObjectsCreateUnauthorized creates SchemaObjectsCreateUnauthorized with default headers values -func NewSchemaObjectsCreateUnauthorized() *SchemaObjectsCreateUnauthorized { - - return &SchemaObjectsCreateUnauthorized{} -} - -// WriteResponse to the client -func (o *SchemaObjectsCreateUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// SchemaObjectsCreateForbiddenCode is the HTTP code returned for type SchemaObjectsCreateForbidden -const SchemaObjectsCreateForbiddenCode int = 403 - -/* -SchemaObjectsCreateForbidden Forbidden - -swagger:response schemaObjectsCreateForbidden -*/ -type SchemaObjectsCreateForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewSchemaObjectsCreateForbidden creates SchemaObjectsCreateForbidden with default headers values -func NewSchemaObjectsCreateForbidden() *SchemaObjectsCreateForbidden { - - return &SchemaObjectsCreateForbidden{} -} - -// WithPayload adds the payload to the schema objects create forbidden response -func (o *SchemaObjectsCreateForbidden) WithPayload(payload *models.ErrorResponse) *SchemaObjectsCreateForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the schema objects create forbidden response -func (o *SchemaObjectsCreateForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *SchemaObjectsCreateForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// SchemaObjectsCreateUnprocessableEntityCode is the HTTP code returned for type SchemaObjectsCreateUnprocessableEntity -const SchemaObjectsCreateUnprocessableEntityCode int = 422 - -/* -SchemaObjectsCreateUnprocessableEntity Invalid Object class - -swagger:response schemaObjectsCreateUnprocessableEntity -*/ -type SchemaObjectsCreateUnprocessableEntity struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewSchemaObjectsCreateUnprocessableEntity creates SchemaObjectsCreateUnprocessableEntity with default headers values -func NewSchemaObjectsCreateUnprocessableEntity() *SchemaObjectsCreateUnprocessableEntity { - - return &SchemaObjectsCreateUnprocessableEntity{} -} - -// WithPayload adds the payload to the schema objects create unprocessable entity response -func (o *SchemaObjectsCreateUnprocessableEntity) WithPayload(payload *models.ErrorResponse) *SchemaObjectsCreateUnprocessableEntity { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the schema objects create unprocessable entity response -func (o *SchemaObjectsCreateUnprocessableEntity) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *SchemaObjectsCreateUnprocessableEntity) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(422) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// SchemaObjectsCreateInternalServerErrorCode is the HTTP code returned for type SchemaObjectsCreateInternalServerError -const SchemaObjectsCreateInternalServerErrorCode int = 500 - -/* -SchemaObjectsCreateInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response schemaObjectsCreateInternalServerError -*/ -type SchemaObjectsCreateInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewSchemaObjectsCreateInternalServerError creates SchemaObjectsCreateInternalServerError with default headers values -func NewSchemaObjectsCreateInternalServerError() *SchemaObjectsCreateInternalServerError { - - return &SchemaObjectsCreateInternalServerError{} -} - -// WithPayload adds the payload to the schema objects create internal server error response -func (o *SchemaObjectsCreateInternalServerError) WithPayload(payload *models.ErrorResponse) *SchemaObjectsCreateInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the schema objects create internal server error response -func (o *SchemaObjectsCreateInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *SchemaObjectsCreateInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/schema/schema_objects_create_urlbuilder.go b/adapters/handlers/rest/operations/schema/schema_objects_create_urlbuilder.go deleted file mode 100644 index 349bcccdd236ce4f6fdab89f0cbdcf55a90c2467..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/schema_objects_create_urlbuilder.go +++ /dev/null @@ -1,98 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" -) - -// SchemaObjectsCreateURL generates an URL for the schema objects create operation -type SchemaObjectsCreateURL struct { - _basePath string -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *SchemaObjectsCreateURL) WithBasePath(bp string) *SchemaObjectsCreateURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *SchemaObjectsCreateURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *SchemaObjectsCreateURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/schema" - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *SchemaObjectsCreateURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *SchemaObjectsCreateURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *SchemaObjectsCreateURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on SchemaObjectsCreateURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on SchemaObjectsCreateURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *SchemaObjectsCreateURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/schema/schema_objects_delete.go b/adapters/handlers/rest/operations/schema/schema_objects_delete.go deleted file mode 100644 index b5af991694bf3996a7bde1401973a6e7f02806c4..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/schema_objects_delete.go +++ /dev/null @@ -1,82 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// SchemaObjectsDeleteHandlerFunc turns a function with the right signature into a schema objects delete handler -type SchemaObjectsDeleteHandlerFunc func(SchemaObjectsDeleteParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn SchemaObjectsDeleteHandlerFunc) Handle(params SchemaObjectsDeleteParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// SchemaObjectsDeleteHandler interface for that can handle valid schema objects delete params -type SchemaObjectsDeleteHandler interface { - Handle(SchemaObjectsDeleteParams, *models.Principal) middleware.Responder -} - -// NewSchemaObjectsDelete creates a new http.Handler for the schema objects delete operation -func NewSchemaObjectsDelete(ctx *middleware.Context, handler SchemaObjectsDeleteHandler) *SchemaObjectsDelete { - return &SchemaObjectsDelete{Context: ctx, Handler: handler} -} - -/* - SchemaObjectsDelete swagger:route DELETE /schema/{className} schema schemaObjectsDelete - -Remove an Object class (and all data in the instances) from the schema. -*/ -type SchemaObjectsDelete struct { - Context *middleware.Context - Handler SchemaObjectsDeleteHandler -} - -func (o *SchemaObjectsDelete) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewSchemaObjectsDeleteParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/schema/schema_objects_delete_parameters.go b/adapters/handlers/rest/operations/schema/schema_objects_delete_parameters.go deleted file mode 100644 index 4da2a075c4ee96a3a7aa439711a22e8bf162dc17..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/schema_objects_delete_parameters.go +++ /dev/null @@ -1,82 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" -) - -// NewSchemaObjectsDeleteParams creates a new SchemaObjectsDeleteParams object -// -// There are no default values defined in the spec. -func NewSchemaObjectsDeleteParams() SchemaObjectsDeleteParams { - - return SchemaObjectsDeleteParams{} -} - -// SchemaObjectsDeleteParams contains all the bound params for the schema objects delete operation -// typically these are obtained from a http.Request -// -// swagger:parameters schema.objects.delete -type SchemaObjectsDeleteParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` - - /* - Required: true - In: path - */ - ClassName string -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewSchemaObjectsDeleteParams() beforehand. -func (o *SchemaObjectsDeleteParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - rClassName, rhkClassName, _ := route.Params.GetOK("className") - if err := o.bindClassName(rClassName, rhkClassName, route.Formats); err != nil { - res = append(res, err) - } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// bindClassName binds and validates parameter ClassName from path. -func (o *SchemaObjectsDeleteParams) bindClassName(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - o.ClassName = raw - - return nil -} diff --git a/adapters/handlers/rest/operations/schema/schema_objects_delete_responses.go b/adapters/handlers/rest/operations/schema/schema_objects_delete_responses.go deleted file mode 100644 index 2ac18469ad1707d678a8e6beaabcffc9d7133ad8..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/schema_objects_delete_responses.go +++ /dev/null @@ -1,210 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// SchemaObjectsDeleteOKCode is the HTTP code returned for type SchemaObjectsDeleteOK -const SchemaObjectsDeleteOKCode int = 200 - -/* -SchemaObjectsDeleteOK Removed the Object class from the schema. - -swagger:response schemaObjectsDeleteOK -*/ -type SchemaObjectsDeleteOK struct { -} - -// NewSchemaObjectsDeleteOK creates SchemaObjectsDeleteOK with default headers values -func NewSchemaObjectsDeleteOK() *SchemaObjectsDeleteOK { - - return &SchemaObjectsDeleteOK{} -} - -// WriteResponse to the client -func (o *SchemaObjectsDeleteOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(200) -} - -// SchemaObjectsDeleteBadRequestCode is the HTTP code returned for type SchemaObjectsDeleteBadRequest -const SchemaObjectsDeleteBadRequestCode int = 400 - -/* -SchemaObjectsDeleteBadRequest Could not delete the Object class. - -swagger:response schemaObjectsDeleteBadRequest -*/ -type SchemaObjectsDeleteBadRequest struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewSchemaObjectsDeleteBadRequest creates SchemaObjectsDeleteBadRequest with default headers values -func NewSchemaObjectsDeleteBadRequest() *SchemaObjectsDeleteBadRequest { - - return &SchemaObjectsDeleteBadRequest{} -} - -// WithPayload adds the payload to the schema objects delete bad request response -func (o *SchemaObjectsDeleteBadRequest) WithPayload(payload *models.ErrorResponse) *SchemaObjectsDeleteBadRequest { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the schema objects delete bad request response -func (o *SchemaObjectsDeleteBadRequest) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *SchemaObjectsDeleteBadRequest) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(400) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// SchemaObjectsDeleteUnauthorizedCode is the HTTP code returned for type SchemaObjectsDeleteUnauthorized -const SchemaObjectsDeleteUnauthorizedCode int = 401 - -/* -SchemaObjectsDeleteUnauthorized Unauthorized or invalid credentials. - -swagger:response schemaObjectsDeleteUnauthorized -*/ -type SchemaObjectsDeleteUnauthorized struct { -} - -// NewSchemaObjectsDeleteUnauthorized creates SchemaObjectsDeleteUnauthorized with default headers values -func NewSchemaObjectsDeleteUnauthorized() *SchemaObjectsDeleteUnauthorized { - - return &SchemaObjectsDeleteUnauthorized{} -} - -// WriteResponse to the client -func (o *SchemaObjectsDeleteUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// SchemaObjectsDeleteForbiddenCode is the HTTP code returned for type SchemaObjectsDeleteForbidden -const SchemaObjectsDeleteForbiddenCode int = 403 - -/* -SchemaObjectsDeleteForbidden Forbidden - -swagger:response schemaObjectsDeleteForbidden -*/ -type SchemaObjectsDeleteForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewSchemaObjectsDeleteForbidden creates SchemaObjectsDeleteForbidden with default headers values -func NewSchemaObjectsDeleteForbidden() *SchemaObjectsDeleteForbidden { - - return &SchemaObjectsDeleteForbidden{} -} - -// WithPayload adds the payload to the schema objects delete forbidden response -func (o *SchemaObjectsDeleteForbidden) WithPayload(payload *models.ErrorResponse) *SchemaObjectsDeleteForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the schema objects delete forbidden response -func (o *SchemaObjectsDeleteForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *SchemaObjectsDeleteForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// SchemaObjectsDeleteInternalServerErrorCode is the HTTP code returned for type SchemaObjectsDeleteInternalServerError -const SchemaObjectsDeleteInternalServerErrorCode int = 500 - -/* -SchemaObjectsDeleteInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response schemaObjectsDeleteInternalServerError -*/ -type SchemaObjectsDeleteInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewSchemaObjectsDeleteInternalServerError creates SchemaObjectsDeleteInternalServerError with default headers values -func NewSchemaObjectsDeleteInternalServerError() *SchemaObjectsDeleteInternalServerError { - - return &SchemaObjectsDeleteInternalServerError{} -} - -// WithPayload adds the payload to the schema objects delete internal server error response -func (o *SchemaObjectsDeleteInternalServerError) WithPayload(payload *models.ErrorResponse) *SchemaObjectsDeleteInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the schema objects delete internal server error response -func (o *SchemaObjectsDeleteInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *SchemaObjectsDeleteInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/schema/schema_objects_delete_urlbuilder.go b/adapters/handlers/rest/operations/schema/schema_objects_delete_urlbuilder.go deleted file mode 100644 index d50e4a41b2ad9b42b68e88b4b0ada7e30d9f298f..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/schema_objects_delete_urlbuilder.go +++ /dev/null @@ -1,110 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" - "strings" -) - -// SchemaObjectsDeleteURL generates an URL for the schema objects delete operation -type SchemaObjectsDeleteURL struct { - ClassName string - - _basePath string - // avoid unkeyed usage - _ struct{} -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *SchemaObjectsDeleteURL) WithBasePath(bp string) *SchemaObjectsDeleteURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *SchemaObjectsDeleteURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *SchemaObjectsDeleteURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/schema/{className}" - - className := o.ClassName - if className != "" { - _path = strings.Replace(_path, "{className}", className, -1) - } else { - return nil, errors.New("className is required on SchemaObjectsDeleteURL") - } - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *SchemaObjectsDeleteURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *SchemaObjectsDeleteURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *SchemaObjectsDeleteURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on SchemaObjectsDeleteURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on SchemaObjectsDeleteURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *SchemaObjectsDeleteURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/schema/schema_objects_get.go b/adapters/handlers/rest/operations/schema/schema_objects_get.go deleted file mode 100644 index b4cd3f5c1e467ffa2622784fb31d341bbeb7bfd6..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/schema_objects_get.go +++ /dev/null @@ -1,82 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// SchemaObjectsGetHandlerFunc turns a function with the right signature into a schema objects get handler -type SchemaObjectsGetHandlerFunc func(SchemaObjectsGetParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn SchemaObjectsGetHandlerFunc) Handle(params SchemaObjectsGetParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// SchemaObjectsGetHandler interface for that can handle valid schema objects get params -type SchemaObjectsGetHandler interface { - Handle(SchemaObjectsGetParams, *models.Principal) middleware.Responder -} - -// NewSchemaObjectsGet creates a new http.Handler for the schema objects get operation -func NewSchemaObjectsGet(ctx *middleware.Context, handler SchemaObjectsGetHandler) *SchemaObjectsGet { - return &SchemaObjectsGet{Context: ctx, Handler: handler} -} - -/* - SchemaObjectsGet swagger:route GET /schema/{className} schema schemaObjectsGet - -Get a single class from the schema -*/ -type SchemaObjectsGet struct { - Context *middleware.Context - Handler SchemaObjectsGetHandler -} - -func (o *SchemaObjectsGet) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewSchemaObjectsGetParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/schema/schema_objects_get_parameters.go b/adapters/handlers/rest/operations/schema/schema_objects_get_parameters.go deleted file mode 100644 index 5b59d7f76844470e5400d1210643ad002c253add..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/schema_objects_get_parameters.go +++ /dev/null @@ -1,82 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" -) - -// NewSchemaObjectsGetParams creates a new SchemaObjectsGetParams object -// -// There are no default values defined in the spec. -func NewSchemaObjectsGetParams() SchemaObjectsGetParams { - - return SchemaObjectsGetParams{} -} - -// SchemaObjectsGetParams contains all the bound params for the schema objects get operation -// typically these are obtained from a http.Request -// -// swagger:parameters schema.objects.get -type SchemaObjectsGetParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` - - /* - Required: true - In: path - */ - ClassName string -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewSchemaObjectsGetParams() beforehand. -func (o *SchemaObjectsGetParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - rClassName, rhkClassName, _ := route.Params.GetOK("className") - if err := o.bindClassName(rClassName, rhkClassName, route.Formats); err != nil { - res = append(res, err) - } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// bindClassName binds and validates parameter ClassName from path. -func (o *SchemaObjectsGetParams) bindClassName(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - o.ClassName = raw - - return nil -} diff --git a/adapters/handlers/rest/operations/schema/schema_objects_get_responses.go b/adapters/handlers/rest/operations/schema/schema_objects_get_responses.go deleted file mode 100644 index a19a5a0587f09a0744e7f00f0cca6afcf613935e..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/schema_objects_get_responses.go +++ /dev/null @@ -1,210 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// SchemaObjectsGetOKCode is the HTTP code returned for type SchemaObjectsGetOK -const SchemaObjectsGetOKCode int = 200 - -/* -SchemaObjectsGetOK Found the Class, returned as body - -swagger:response schemaObjectsGetOK -*/ -type SchemaObjectsGetOK struct { - - /* - In: Body - */ - Payload *models.Class `json:"body,omitempty"` -} - -// NewSchemaObjectsGetOK creates SchemaObjectsGetOK with default headers values -func NewSchemaObjectsGetOK() *SchemaObjectsGetOK { - - return &SchemaObjectsGetOK{} -} - -// WithPayload adds the payload to the schema objects get o k response -func (o *SchemaObjectsGetOK) WithPayload(payload *models.Class) *SchemaObjectsGetOK { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the schema objects get o k response -func (o *SchemaObjectsGetOK) SetPayload(payload *models.Class) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *SchemaObjectsGetOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(200) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// SchemaObjectsGetUnauthorizedCode is the HTTP code returned for type SchemaObjectsGetUnauthorized -const SchemaObjectsGetUnauthorizedCode int = 401 - -/* -SchemaObjectsGetUnauthorized Unauthorized or invalid credentials. - -swagger:response schemaObjectsGetUnauthorized -*/ -type SchemaObjectsGetUnauthorized struct { -} - -// NewSchemaObjectsGetUnauthorized creates SchemaObjectsGetUnauthorized with default headers values -func NewSchemaObjectsGetUnauthorized() *SchemaObjectsGetUnauthorized { - - return &SchemaObjectsGetUnauthorized{} -} - -// WriteResponse to the client -func (o *SchemaObjectsGetUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// SchemaObjectsGetForbiddenCode is the HTTP code returned for type SchemaObjectsGetForbidden -const SchemaObjectsGetForbiddenCode int = 403 - -/* -SchemaObjectsGetForbidden Forbidden - -swagger:response schemaObjectsGetForbidden -*/ -type SchemaObjectsGetForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewSchemaObjectsGetForbidden creates SchemaObjectsGetForbidden with default headers values -func NewSchemaObjectsGetForbidden() *SchemaObjectsGetForbidden { - - return &SchemaObjectsGetForbidden{} -} - -// WithPayload adds the payload to the schema objects get forbidden response -func (o *SchemaObjectsGetForbidden) WithPayload(payload *models.ErrorResponse) *SchemaObjectsGetForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the schema objects get forbidden response -func (o *SchemaObjectsGetForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *SchemaObjectsGetForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// SchemaObjectsGetNotFoundCode is the HTTP code returned for type SchemaObjectsGetNotFound -const SchemaObjectsGetNotFoundCode int = 404 - -/* -SchemaObjectsGetNotFound This class does not exist - -swagger:response schemaObjectsGetNotFound -*/ -type SchemaObjectsGetNotFound struct { -} - -// NewSchemaObjectsGetNotFound creates SchemaObjectsGetNotFound with default headers values -func NewSchemaObjectsGetNotFound() *SchemaObjectsGetNotFound { - - return &SchemaObjectsGetNotFound{} -} - -// WriteResponse to the client -func (o *SchemaObjectsGetNotFound) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(404) -} - -// SchemaObjectsGetInternalServerErrorCode is the HTTP code returned for type SchemaObjectsGetInternalServerError -const SchemaObjectsGetInternalServerErrorCode int = 500 - -/* -SchemaObjectsGetInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response schemaObjectsGetInternalServerError -*/ -type SchemaObjectsGetInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewSchemaObjectsGetInternalServerError creates SchemaObjectsGetInternalServerError with default headers values -func NewSchemaObjectsGetInternalServerError() *SchemaObjectsGetInternalServerError { - - return &SchemaObjectsGetInternalServerError{} -} - -// WithPayload adds the payload to the schema objects get internal server error response -func (o *SchemaObjectsGetInternalServerError) WithPayload(payload *models.ErrorResponse) *SchemaObjectsGetInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the schema objects get internal server error response -func (o *SchemaObjectsGetInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *SchemaObjectsGetInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/schema/schema_objects_get_urlbuilder.go b/adapters/handlers/rest/operations/schema/schema_objects_get_urlbuilder.go deleted file mode 100644 index 4fc2b35e4b586db9bb3961784c4febcb585e2d9d..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/schema_objects_get_urlbuilder.go +++ /dev/null @@ -1,110 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" - "strings" -) - -// SchemaObjectsGetURL generates an URL for the schema objects get operation -type SchemaObjectsGetURL struct { - ClassName string - - _basePath string - // avoid unkeyed usage - _ struct{} -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *SchemaObjectsGetURL) WithBasePath(bp string) *SchemaObjectsGetURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *SchemaObjectsGetURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *SchemaObjectsGetURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/schema/{className}" - - className := o.ClassName - if className != "" { - _path = strings.Replace(_path, "{className}", className, -1) - } else { - return nil, errors.New("className is required on SchemaObjectsGetURL") - } - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *SchemaObjectsGetURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *SchemaObjectsGetURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *SchemaObjectsGetURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on SchemaObjectsGetURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on SchemaObjectsGetURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *SchemaObjectsGetURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/schema/schema_objects_properties_add.go b/adapters/handlers/rest/operations/schema/schema_objects_properties_add.go deleted file mode 100644 index 86fb79f251c6b0fad113bea10670dc1a6081d08f..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/schema_objects_properties_add.go +++ /dev/null @@ -1,82 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// SchemaObjectsPropertiesAddHandlerFunc turns a function with the right signature into a schema objects properties add handler -type SchemaObjectsPropertiesAddHandlerFunc func(SchemaObjectsPropertiesAddParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn SchemaObjectsPropertiesAddHandlerFunc) Handle(params SchemaObjectsPropertiesAddParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// SchemaObjectsPropertiesAddHandler interface for that can handle valid schema objects properties add params -type SchemaObjectsPropertiesAddHandler interface { - Handle(SchemaObjectsPropertiesAddParams, *models.Principal) middleware.Responder -} - -// NewSchemaObjectsPropertiesAdd creates a new http.Handler for the schema objects properties add operation -func NewSchemaObjectsPropertiesAdd(ctx *middleware.Context, handler SchemaObjectsPropertiesAddHandler) *SchemaObjectsPropertiesAdd { - return &SchemaObjectsPropertiesAdd{Context: ctx, Handler: handler} -} - -/* - SchemaObjectsPropertiesAdd swagger:route POST /schema/{className}/properties schema schemaObjectsPropertiesAdd - -Add a property to an Object class. -*/ -type SchemaObjectsPropertiesAdd struct { - Context *middleware.Context - Handler SchemaObjectsPropertiesAddHandler -} - -func (o *SchemaObjectsPropertiesAdd) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewSchemaObjectsPropertiesAddParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/schema/schema_objects_properties_add_parameters.go b/adapters/handlers/rest/operations/schema/schema_objects_properties_add_parameters.go deleted file mode 100644 index cc6f43d569fbf00b5b542819d1f041a065872196..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/schema_objects_properties_add_parameters.go +++ /dev/null @@ -1,120 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "io" - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/validate" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewSchemaObjectsPropertiesAddParams creates a new SchemaObjectsPropertiesAddParams object -// -// There are no default values defined in the spec. -func NewSchemaObjectsPropertiesAddParams() SchemaObjectsPropertiesAddParams { - - return SchemaObjectsPropertiesAddParams{} -} - -// SchemaObjectsPropertiesAddParams contains all the bound params for the schema objects properties add operation -// typically these are obtained from a http.Request -// -// swagger:parameters schema.objects.properties.add -type SchemaObjectsPropertiesAddParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` - - /* - Required: true - In: body - */ - Body *models.Property - /* - Required: true - In: path - */ - ClassName string -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewSchemaObjectsPropertiesAddParams() beforehand. -func (o *SchemaObjectsPropertiesAddParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - if runtime.HasBody(r) { - defer r.Body.Close() - var body models.Property - if err := route.Consumer.Consume(r.Body, &body); err != nil { - if err == io.EOF { - res = append(res, errors.Required("body", "body", "")) - } else { - res = append(res, errors.NewParseError("body", "body", "", err)) - } - } else { - // validate body object - if err := body.Validate(route.Formats); err != nil { - res = append(res, err) - } - - ctx := validate.WithOperationRequest(r.Context()) - if err := body.ContextValidate(ctx, route.Formats); err != nil { - res = append(res, err) - } - - if len(res) == 0 { - o.Body = &body - } - } - } else { - res = append(res, errors.Required("body", "body", "")) - } - - rClassName, rhkClassName, _ := route.Params.GetOK("className") - if err := o.bindClassName(rClassName, rhkClassName, route.Formats); err != nil { - res = append(res, err) - } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// bindClassName binds and validates parameter ClassName from path. -func (o *SchemaObjectsPropertiesAddParams) bindClassName(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - o.ClassName = raw - - return nil -} diff --git a/adapters/handlers/rest/operations/schema/schema_objects_properties_add_responses.go b/adapters/handlers/rest/operations/schema/schema_objects_properties_add_responses.go deleted file mode 100644 index a7b5fe10a91cadcdd3c1876b6c02b436cd86d580..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/schema_objects_properties_add_responses.go +++ /dev/null @@ -1,230 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// SchemaObjectsPropertiesAddOKCode is the HTTP code returned for type SchemaObjectsPropertiesAddOK -const SchemaObjectsPropertiesAddOKCode int = 200 - -/* -SchemaObjectsPropertiesAddOK Added the property. - -swagger:response schemaObjectsPropertiesAddOK -*/ -type SchemaObjectsPropertiesAddOK struct { - - /* - In: Body - */ - Payload *models.Property `json:"body,omitempty"` -} - -// NewSchemaObjectsPropertiesAddOK creates SchemaObjectsPropertiesAddOK with default headers values -func NewSchemaObjectsPropertiesAddOK() *SchemaObjectsPropertiesAddOK { - - return &SchemaObjectsPropertiesAddOK{} -} - -// WithPayload adds the payload to the schema objects properties add o k response -func (o *SchemaObjectsPropertiesAddOK) WithPayload(payload *models.Property) *SchemaObjectsPropertiesAddOK { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the schema objects properties add o k response -func (o *SchemaObjectsPropertiesAddOK) SetPayload(payload *models.Property) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *SchemaObjectsPropertiesAddOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(200) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// SchemaObjectsPropertiesAddUnauthorizedCode is the HTTP code returned for type SchemaObjectsPropertiesAddUnauthorized -const SchemaObjectsPropertiesAddUnauthorizedCode int = 401 - -/* -SchemaObjectsPropertiesAddUnauthorized Unauthorized or invalid credentials. - -swagger:response schemaObjectsPropertiesAddUnauthorized -*/ -type SchemaObjectsPropertiesAddUnauthorized struct { -} - -// NewSchemaObjectsPropertiesAddUnauthorized creates SchemaObjectsPropertiesAddUnauthorized with default headers values -func NewSchemaObjectsPropertiesAddUnauthorized() *SchemaObjectsPropertiesAddUnauthorized { - - return &SchemaObjectsPropertiesAddUnauthorized{} -} - -// WriteResponse to the client -func (o *SchemaObjectsPropertiesAddUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// SchemaObjectsPropertiesAddForbiddenCode is the HTTP code returned for type SchemaObjectsPropertiesAddForbidden -const SchemaObjectsPropertiesAddForbiddenCode int = 403 - -/* -SchemaObjectsPropertiesAddForbidden Forbidden - -swagger:response schemaObjectsPropertiesAddForbidden -*/ -type SchemaObjectsPropertiesAddForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewSchemaObjectsPropertiesAddForbidden creates SchemaObjectsPropertiesAddForbidden with default headers values -func NewSchemaObjectsPropertiesAddForbidden() *SchemaObjectsPropertiesAddForbidden { - - return &SchemaObjectsPropertiesAddForbidden{} -} - -// WithPayload adds the payload to the schema objects properties add forbidden response -func (o *SchemaObjectsPropertiesAddForbidden) WithPayload(payload *models.ErrorResponse) *SchemaObjectsPropertiesAddForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the schema objects properties add forbidden response -func (o *SchemaObjectsPropertiesAddForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *SchemaObjectsPropertiesAddForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// SchemaObjectsPropertiesAddUnprocessableEntityCode is the HTTP code returned for type SchemaObjectsPropertiesAddUnprocessableEntity -const SchemaObjectsPropertiesAddUnprocessableEntityCode int = 422 - -/* -SchemaObjectsPropertiesAddUnprocessableEntity Invalid property. - -swagger:response schemaObjectsPropertiesAddUnprocessableEntity -*/ -type SchemaObjectsPropertiesAddUnprocessableEntity struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewSchemaObjectsPropertiesAddUnprocessableEntity creates SchemaObjectsPropertiesAddUnprocessableEntity with default headers values -func NewSchemaObjectsPropertiesAddUnprocessableEntity() *SchemaObjectsPropertiesAddUnprocessableEntity { - - return &SchemaObjectsPropertiesAddUnprocessableEntity{} -} - -// WithPayload adds the payload to the schema objects properties add unprocessable entity response -func (o *SchemaObjectsPropertiesAddUnprocessableEntity) WithPayload(payload *models.ErrorResponse) *SchemaObjectsPropertiesAddUnprocessableEntity { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the schema objects properties add unprocessable entity response -func (o *SchemaObjectsPropertiesAddUnprocessableEntity) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *SchemaObjectsPropertiesAddUnprocessableEntity) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(422) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// SchemaObjectsPropertiesAddInternalServerErrorCode is the HTTP code returned for type SchemaObjectsPropertiesAddInternalServerError -const SchemaObjectsPropertiesAddInternalServerErrorCode int = 500 - -/* -SchemaObjectsPropertiesAddInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response schemaObjectsPropertiesAddInternalServerError -*/ -type SchemaObjectsPropertiesAddInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewSchemaObjectsPropertiesAddInternalServerError creates SchemaObjectsPropertiesAddInternalServerError with default headers values -func NewSchemaObjectsPropertiesAddInternalServerError() *SchemaObjectsPropertiesAddInternalServerError { - - return &SchemaObjectsPropertiesAddInternalServerError{} -} - -// WithPayload adds the payload to the schema objects properties add internal server error response -func (o *SchemaObjectsPropertiesAddInternalServerError) WithPayload(payload *models.ErrorResponse) *SchemaObjectsPropertiesAddInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the schema objects properties add internal server error response -func (o *SchemaObjectsPropertiesAddInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *SchemaObjectsPropertiesAddInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/schema/schema_objects_properties_add_urlbuilder.go b/adapters/handlers/rest/operations/schema/schema_objects_properties_add_urlbuilder.go deleted file mode 100644 index af1107f5be8588e3b7f680d2fb2dc5dbd13cdb35..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/schema_objects_properties_add_urlbuilder.go +++ /dev/null @@ -1,110 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" - "strings" -) - -// SchemaObjectsPropertiesAddURL generates an URL for the schema objects properties add operation -type SchemaObjectsPropertiesAddURL struct { - ClassName string - - _basePath string - // avoid unkeyed usage - _ struct{} -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *SchemaObjectsPropertiesAddURL) WithBasePath(bp string) *SchemaObjectsPropertiesAddURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *SchemaObjectsPropertiesAddURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *SchemaObjectsPropertiesAddURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/schema/{className}/properties" - - className := o.ClassName - if className != "" { - _path = strings.Replace(_path, "{className}", className, -1) - } else { - return nil, errors.New("className is required on SchemaObjectsPropertiesAddURL") - } - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *SchemaObjectsPropertiesAddURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *SchemaObjectsPropertiesAddURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *SchemaObjectsPropertiesAddURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on SchemaObjectsPropertiesAddURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on SchemaObjectsPropertiesAddURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *SchemaObjectsPropertiesAddURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/schema/schema_objects_shards_get.go b/adapters/handlers/rest/operations/schema/schema_objects_shards_get.go deleted file mode 100644 index 8d2bdb6b689e2cc909a4f3994a5c21e5b8fd69f5..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/schema_objects_shards_get.go +++ /dev/null @@ -1,82 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// SchemaObjectsShardsGetHandlerFunc turns a function with the right signature into a schema objects shards get handler -type SchemaObjectsShardsGetHandlerFunc func(SchemaObjectsShardsGetParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn SchemaObjectsShardsGetHandlerFunc) Handle(params SchemaObjectsShardsGetParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// SchemaObjectsShardsGetHandler interface for that can handle valid schema objects shards get params -type SchemaObjectsShardsGetHandler interface { - Handle(SchemaObjectsShardsGetParams, *models.Principal) middleware.Responder -} - -// NewSchemaObjectsShardsGet creates a new http.Handler for the schema objects shards get operation -func NewSchemaObjectsShardsGet(ctx *middleware.Context, handler SchemaObjectsShardsGetHandler) *SchemaObjectsShardsGet { - return &SchemaObjectsShardsGet{Context: ctx, Handler: handler} -} - -/* - SchemaObjectsShardsGet swagger:route GET /schema/{className}/shards schema schemaObjectsShardsGet - -Get the shards status of an Object class -*/ -type SchemaObjectsShardsGet struct { - Context *middleware.Context - Handler SchemaObjectsShardsGetHandler -} - -func (o *SchemaObjectsShardsGet) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewSchemaObjectsShardsGetParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/schema/schema_objects_shards_get_parameters.go b/adapters/handlers/rest/operations/schema/schema_objects_shards_get_parameters.go deleted file mode 100644 index 4de4e21af57591edbc978ac702ceca9521ba7d9d..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/schema_objects_shards_get_parameters.go +++ /dev/null @@ -1,112 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" -) - -// NewSchemaObjectsShardsGetParams creates a new SchemaObjectsShardsGetParams object -// -// There are no default values defined in the spec. -func NewSchemaObjectsShardsGetParams() SchemaObjectsShardsGetParams { - - return SchemaObjectsShardsGetParams{} -} - -// SchemaObjectsShardsGetParams contains all the bound params for the schema objects shards get operation -// typically these are obtained from a http.Request -// -// swagger:parameters schema.objects.shards.get -type SchemaObjectsShardsGetParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` - - /* - Required: true - In: path - */ - ClassName string - /* - In: query - */ - Tenant *string -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewSchemaObjectsShardsGetParams() beforehand. -func (o *SchemaObjectsShardsGetParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - qs := runtime.Values(r.URL.Query()) - - rClassName, rhkClassName, _ := route.Params.GetOK("className") - if err := o.bindClassName(rClassName, rhkClassName, route.Formats); err != nil { - res = append(res, err) - } - - qTenant, qhkTenant, _ := qs.GetOK("tenant") - if err := o.bindTenant(qTenant, qhkTenant, route.Formats); err != nil { - res = append(res, err) - } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// bindClassName binds and validates parameter ClassName from path. -func (o *SchemaObjectsShardsGetParams) bindClassName(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - o.ClassName = raw - - return nil -} - -// bindTenant binds and validates parameter Tenant from query. -func (o *SchemaObjectsShardsGetParams) bindTenant(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: false - // AllowEmptyValue: false - - if raw == "" { // empty values pass all other validations - return nil - } - o.Tenant = &raw - - return nil -} diff --git a/adapters/handlers/rest/operations/schema/schema_objects_shards_get_responses.go b/adapters/handlers/rest/operations/schema/schema_objects_shards_get_responses.go deleted file mode 100644 index 8349102499fa38240151760e944e4b0e87089d9f..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/schema_objects_shards_get_responses.go +++ /dev/null @@ -1,233 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// SchemaObjectsShardsGetOKCode is the HTTP code returned for type SchemaObjectsShardsGetOK -const SchemaObjectsShardsGetOKCode int = 200 - -/* -SchemaObjectsShardsGetOK Found the status of the shards, returned as body - -swagger:response schemaObjectsShardsGetOK -*/ -type SchemaObjectsShardsGetOK struct { - - /* - In: Body - */ - Payload models.ShardStatusList `json:"body,omitempty"` -} - -// NewSchemaObjectsShardsGetOK creates SchemaObjectsShardsGetOK with default headers values -func NewSchemaObjectsShardsGetOK() *SchemaObjectsShardsGetOK { - - return &SchemaObjectsShardsGetOK{} -} - -// WithPayload adds the payload to the schema objects shards get o k response -func (o *SchemaObjectsShardsGetOK) WithPayload(payload models.ShardStatusList) *SchemaObjectsShardsGetOK { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the schema objects shards get o k response -func (o *SchemaObjectsShardsGetOK) SetPayload(payload models.ShardStatusList) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *SchemaObjectsShardsGetOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(200) - payload := o.Payload - if payload == nil { - // return empty array - payload = models.ShardStatusList{} - } - - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } -} - -// SchemaObjectsShardsGetUnauthorizedCode is the HTTP code returned for type SchemaObjectsShardsGetUnauthorized -const SchemaObjectsShardsGetUnauthorizedCode int = 401 - -/* -SchemaObjectsShardsGetUnauthorized Unauthorized or invalid credentials. - -swagger:response schemaObjectsShardsGetUnauthorized -*/ -type SchemaObjectsShardsGetUnauthorized struct { -} - -// NewSchemaObjectsShardsGetUnauthorized creates SchemaObjectsShardsGetUnauthorized with default headers values -func NewSchemaObjectsShardsGetUnauthorized() *SchemaObjectsShardsGetUnauthorized { - - return &SchemaObjectsShardsGetUnauthorized{} -} - -// WriteResponse to the client -func (o *SchemaObjectsShardsGetUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// SchemaObjectsShardsGetForbiddenCode is the HTTP code returned for type SchemaObjectsShardsGetForbidden -const SchemaObjectsShardsGetForbiddenCode int = 403 - -/* -SchemaObjectsShardsGetForbidden Forbidden - -swagger:response schemaObjectsShardsGetForbidden -*/ -type SchemaObjectsShardsGetForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewSchemaObjectsShardsGetForbidden creates SchemaObjectsShardsGetForbidden with default headers values -func NewSchemaObjectsShardsGetForbidden() *SchemaObjectsShardsGetForbidden { - - return &SchemaObjectsShardsGetForbidden{} -} - -// WithPayload adds the payload to the schema objects shards get forbidden response -func (o *SchemaObjectsShardsGetForbidden) WithPayload(payload *models.ErrorResponse) *SchemaObjectsShardsGetForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the schema objects shards get forbidden response -func (o *SchemaObjectsShardsGetForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *SchemaObjectsShardsGetForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// SchemaObjectsShardsGetNotFoundCode is the HTTP code returned for type SchemaObjectsShardsGetNotFound -const SchemaObjectsShardsGetNotFoundCode int = 404 - -/* -SchemaObjectsShardsGetNotFound This class does not exist - -swagger:response schemaObjectsShardsGetNotFound -*/ -type SchemaObjectsShardsGetNotFound struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewSchemaObjectsShardsGetNotFound creates SchemaObjectsShardsGetNotFound with default headers values -func NewSchemaObjectsShardsGetNotFound() *SchemaObjectsShardsGetNotFound { - - return &SchemaObjectsShardsGetNotFound{} -} - -// WithPayload adds the payload to the schema objects shards get not found response -func (o *SchemaObjectsShardsGetNotFound) WithPayload(payload *models.ErrorResponse) *SchemaObjectsShardsGetNotFound { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the schema objects shards get not found response -func (o *SchemaObjectsShardsGetNotFound) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *SchemaObjectsShardsGetNotFound) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(404) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// SchemaObjectsShardsGetInternalServerErrorCode is the HTTP code returned for type SchemaObjectsShardsGetInternalServerError -const SchemaObjectsShardsGetInternalServerErrorCode int = 500 - -/* -SchemaObjectsShardsGetInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response schemaObjectsShardsGetInternalServerError -*/ -type SchemaObjectsShardsGetInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewSchemaObjectsShardsGetInternalServerError creates SchemaObjectsShardsGetInternalServerError with default headers values -func NewSchemaObjectsShardsGetInternalServerError() *SchemaObjectsShardsGetInternalServerError { - - return &SchemaObjectsShardsGetInternalServerError{} -} - -// WithPayload adds the payload to the schema objects shards get internal server error response -func (o *SchemaObjectsShardsGetInternalServerError) WithPayload(payload *models.ErrorResponse) *SchemaObjectsShardsGetInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the schema objects shards get internal server error response -func (o *SchemaObjectsShardsGetInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *SchemaObjectsShardsGetInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/schema/schema_objects_shards_get_urlbuilder.go b/adapters/handlers/rest/operations/schema/schema_objects_shards_get_urlbuilder.go deleted file mode 100644 index e7828e785d72b3f67d5c79ec4a7454615cb945a0..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/schema_objects_shards_get_urlbuilder.go +++ /dev/null @@ -1,124 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" - "strings" -) - -// SchemaObjectsShardsGetURL generates an URL for the schema objects shards get operation -type SchemaObjectsShardsGetURL struct { - ClassName string - - Tenant *string - - _basePath string - // avoid unkeyed usage - _ struct{} -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *SchemaObjectsShardsGetURL) WithBasePath(bp string) *SchemaObjectsShardsGetURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *SchemaObjectsShardsGetURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *SchemaObjectsShardsGetURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/schema/{className}/shards" - - className := o.ClassName - if className != "" { - _path = strings.Replace(_path, "{className}", className, -1) - } else { - return nil, errors.New("className is required on SchemaObjectsShardsGetURL") - } - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - qs := make(url.Values) - - var tenantQ string - if o.Tenant != nil { - tenantQ = *o.Tenant - } - if tenantQ != "" { - qs.Set("tenant", tenantQ) - } - - _result.RawQuery = qs.Encode() - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *SchemaObjectsShardsGetURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *SchemaObjectsShardsGetURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *SchemaObjectsShardsGetURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on SchemaObjectsShardsGetURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on SchemaObjectsShardsGetURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *SchemaObjectsShardsGetURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/schema/schema_objects_shards_update.go b/adapters/handlers/rest/operations/schema/schema_objects_shards_update.go deleted file mode 100644 index 1323e4e3958ec01e86dd56148859cfd446fc1a8c..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/schema_objects_shards_update.go +++ /dev/null @@ -1,82 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// SchemaObjectsShardsUpdateHandlerFunc turns a function with the right signature into a schema objects shards update handler -type SchemaObjectsShardsUpdateHandlerFunc func(SchemaObjectsShardsUpdateParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn SchemaObjectsShardsUpdateHandlerFunc) Handle(params SchemaObjectsShardsUpdateParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// SchemaObjectsShardsUpdateHandler interface for that can handle valid schema objects shards update params -type SchemaObjectsShardsUpdateHandler interface { - Handle(SchemaObjectsShardsUpdateParams, *models.Principal) middleware.Responder -} - -// NewSchemaObjectsShardsUpdate creates a new http.Handler for the schema objects shards update operation -func NewSchemaObjectsShardsUpdate(ctx *middleware.Context, handler SchemaObjectsShardsUpdateHandler) *SchemaObjectsShardsUpdate { - return &SchemaObjectsShardsUpdate{Context: ctx, Handler: handler} -} - -/* - SchemaObjectsShardsUpdate swagger:route PUT /schema/{className}/shards/{shardName} schema schemaObjectsShardsUpdate - -Update shard status of an Object Class -*/ -type SchemaObjectsShardsUpdate struct { - Context *middleware.Context - Handler SchemaObjectsShardsUpdateHandler -} - -func (o *SchemaObjectsShardsUpdate) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewSchemaObjectsShardsUpdateParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/schema/schema_objects_shards_update_parameters.go b/adapters/handlers/rest/operations/schema/schema_objects_shards_update_parameters.go deleted file mode 100644 index af65673907bc73a6309cfa7f4950be3ac3106994..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/schema_objects_shards_update_parameters.go +++ /dev/null @@ -1,144 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "io" - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/validate" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewSchemaObjectsShardsUpdateParams creates a new SchemaObjectsShardsUpdateParams object -// -// There are no default values defined in the spec. -func NewSchemaObjectsShardsUpdateParams() SchemaObjectsShardsUpdateParams { - - return SchemaObjectsShardsUpdateParams{} -} - -// SchemaObjectsShardsUpdateParams contains all the bound params for the schema objects shards update operation -// typically these are obtained from a http.Request -// -// swagger:parameters schema.objects.shards.update -type SchemaObjectsShardsUpdateParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` - - /* - Required: true - In: body - */ - Body *models.ShardStatus - /* - Required: true - In: path - */ - ClassName string - /* - Required: true - In: path - */ - ShardName string -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewSchemaObjectsShardsUpdateParams() beforehand. -func (o *SchemaObjectsShardsUpdateParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - if runtime.HasBody(r) { - defer r.Body.Close() - var body models.ShardStatus - if err := route.Consumer.Consume(r.Body, &body); err != nil { - if err == io.EOF { - res = append(res, errors.Required("body", "body", "")) - } else { - res = append(res, errors.NewParseError("body", "body", "", err)) - } - } else { - // validate body object - if err := body.Validate(route.Formats); err != nil { - res = append(res, err) - } - - ctx := validate.WithOperationRequest(r.Context()) - if err := body.ContextValidate(ctx, route.Formats); err != nil { - res = append(res, err) - } - - if len(res) == 0 { - o.Body = &body - } - } - } else { - res = append(res, errors.Required("body", "body", "")) - } - - rClassName, rhkClassName, _ := route.Params.GetOK("className") - if err := o.bindClassName(rClassName, rhkClassName, route.Formats); err != nil { - res = append(res, err) - } - - rShardName, rhkShardName, _ := route.Params.GetOK("shardName") - if err := o.bindShardName(rShardName, rhkShardName, route.Formats); err != nil { - res = append(res, err) - } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// bindClassName binds and validates parameter ClassName from path. -func (o *SchemaObjectsShardsUpdateParams) bindClassName(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - o.ClassName = raw - - return nil -} - -// bindShardName binds and validates parameter ShardName from path. -func (o *SchemaObjectsShardsUpdateParams) bindShardName(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - o.ShardName = raw - - return nil -} diff --git a/adapters/handlers/rest/operations/schema/schema_objects_shards_update_responses.go b/adapters/handlers/rest/operations/schema/schema_objects_shards_update_responses.go deleted file mode 100644 index a255c209882a98df7abb0e66e0e41818e335fcea..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/schema_objects_shards_update_responses.go +++ /dev/null @@ -1,275 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// SchemaObjectsShardsUpdateOKCode is the HTTP code returned for type SchemaObjectsShardsUpdateOK -const SchemaObjectsShardsUpdateOKCode int = 200 - -/* -SchemaObjectsShardsUpdateOK Shard status was updated successfully - -swagger:response schemaObjectsShardsUpdateOK -*/ -type SchemaObjectsShardsUpdateOK struct { - - /* - In: Body - */ - Payload *models.ShardStatus `json:"body,omitempty"` -} - -// NewSchemaObjectsShardsUpdateOK creates SchemaObjectsShardsUpdateOK with default headers values -func NewSchemaObjectsShardsUpdateOK() *SchemaObjectsShardsUpdateOK { - - return &SchemaObjectsShardsUpdateOK{} -} - -// WithPayload adds the payload to the schema objects shards update o k response -func (o *SchemaObjectsShardsUpdateOK) WithPayload(payload *models.ShardStatus) *SchemaObjectsShardsUpdateOK { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the schema objects shards update o k response -func (o *SchemaObjectsShardsUpdateOK) SetPayload(payload *models.ShardStatus) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *SchemaObjectsShardsUpdateOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(200) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// SchemaObjectsShardsUpdateUnauthorizedCode is the HTTP code returned for type SchemaObjectsShardsUpdateUnauthorized -const SchemaObjectsShardsUpdateUnauthorizedCode int = 401 - -/* -SchemaObjectsShardsUpdateUnauthorized Unauthorized or invalid credentials. - -swagger:response schemaObjectsShardsUpdateUnauthorized -*/ -type SchemaObjectsShardsUpdateUnauthorized struct { -} - -// NewSchemaObjectsShardsUpdateUnauthorized creates SchemaObjectsShardsUpdateUnauthorized with default headers values -func NewSchemaObjectsShardsUpdateUnauthorized() *SchemaObjectsShardsUpdateUnauthorized { - - return &SchemaObjectsShardsUpdateUnauthorized{} -} - -// WriteResponse to the client -func (o *SchemaObjectsShardsUpdateUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// SchemaObjectsShardsUpdateForbiddenCode is the HTTP code returned for type SchemaObjectsShardsUpdateForbidden -const SchemaObjectsShardsUpdateForbiddenCode int = 403 - -/* -SchemaObjectsShardsUpdateForbidden Forbidden - -swagger:response schemaObjectsShardsUpdateForbidden -*/ -type SchemaObjectsShardsUpdateForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewSchemaObjectsShardsUpdateForbidden creates SchemaObjectsShardsUpdateForbidden with default headers values -func NewSchemaObjectsShardsUpdateForbidden() *SchemaObjectsShardsUpdateForbidden { - - return &SchemaObjectsShardsUpdateForbidden{} -} - -// WithPayload adds the payload to the schema objects shards update forbidden response -func (o *SchemaObjectsShardsUpdateForbidden) WithPayload(payload *models.ErrorResponse) *SchemaObjectsShardsUpdateForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the schema objects shards update forbidden response -func (o *SchemaObjectsShardsUpdateForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *SchemaObjectsShardsUpdateForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// SchemaObjectsShardsUpdateNotFoundCode is the HTTP code returned for type SchemaObjectsShardsUpdateNotFound -const SchemaObjectsShardsUpdateNotFoundCode int = 404 - -/* -SchemaObjectsShardsUpdateNotFound Shard to be updated does not exist - -swagger:response schemaObjectsShardsUpdateNotFound -*/ -type SchemaObjectsShardsUpdateNotFound struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewSchemaObjectsShardsUpdateNotFound creates SchemaObjectsShardsUpdateNotFound with default headers values -func NewSchemaObjectsShardsUpdateNotFound() *SchemaObjectsShardsUpdateNotFound { - - return &SchemaObjectsShardsUpdateNotFound{} -} - -// WithPayload adds the payload to the schema objects shards update not found response -func (o *SchemaObjectsShardsUpdateNotFound) WithPayload(payload *models.ErrorResponse) *SchemaObjectsShardsUpdateNotFound { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the schema objects shards update not found response -func (o *SchemaObjectsShardsUpdateNotFound) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *SchemaObjectsShardsUpdateNotFound) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(404) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// SchemaObjectsShardsUpdateUnprocessableEntityCode is the HTTP code returned for type SchemaObjectsShardsUpdateUnprocessableEntity -const SchemaObjectsShardsUpdateUnprocessableEntityCode int = 422 - -/* -SchemaObjectsShardsUpdateUnprocessableEntity Invalid update attempt - -swagger:response schemaObjectsShardsUpdateUnprocessableEntity -*/ -type SchemaObjectsShardsUpdateUnprocessableEntity struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewSchemaObjectsShardsUpdateUnprocessableEntity creates SchemaObjectsShardsUpdateUnprocessableEntity with default headers values -func NewSchemaObjectsShardsUpdateUnprocessableEntity() *SchemaObjectsShardsUpdateUnprocessableEntity { - - return &SchemaObjectsShardsUpdateUnprocessableEntity{} -} - -// WithPayload adds the payload to the schema objects shards update unprocessable entity response -func (o *SchemaObjectsShardsUpdateUnprocessableEntity) WithPayload(payload *models.ErrorResponse) *SchemaObjectsShardsUpdateUnprocessableEntity { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the schema objects shards update unprocessable entity response -func (o *SchemaObjectsShardsUpdateUnprocessableEntity) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *SchemaObjectsShardsUpdateUnprocessableEntity) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(422) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// SchemaObjectsShardsUpdateInternalServerErrorCode is the HTTP code returned for type SchemaObjectsShardsUpdateInternalServerError -const SchemaObjectsShardsUpdateInternalServerErrorCode int = 500 - -/* -SchemaObjectsShardsUpdateInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response schemaObjectsShardsUpdateInternalServerError -*/ -type SchemaObjectsShardsUpdateInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewSchemaObjectsShardsUpdateInternalServerError creates SchemaObjectsShardsUpdateInternalServerError with default headers values -func NewSchemaObjectsShardsUpdateInternalServerError() *SchemaObjectsShardsUpdateInternalServerError { - - return &SchemaObjectsShardsUpdateInternalServerError{} -} - -// WithPayload adds the payload to the schema objects shards update internal server error response -func (o *SchemaObjectsShardsUpdateInternalServerError) WithPayload(payload *models.ErrorResponse) *SchemaObjectsShardsUpdateInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the schema objects shards update internal server error response -func (o *SchemaObjectsShardsUpdateInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *SchemaObjectsShardsUpdateInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/schema/schema_objects_shards_update_urlbuilder.go b/adapters/handlers/rest/operations/schema/schema_objects_shards_update_urlbuilder.go deleted file mode 100644 index 4cc831029c5ad0984193bd1f27e297b46d39a9a3..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/schema_objects_shards_update_urlbuilder.go +++ /dev/null @@ -1,118 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" - "strings" -) - -// SchemaObjectsShardsUpdateURL generates an URL for the schema objects shards update operation -type SchemaObjectsShardsUpdateURL struct { - ClassName string - ShardName string - - _basePath string - // avoid unkeyed usage - _ struct{} -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *SchemaObjectsShardsUpdateURL) WithBasePath(bp string) *SchemaObjectsShardsUpdateURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *SchemaObjectsShardsUpdateURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *SchemaObjectsShardsUpdateURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/schema/{className}/shards/{shardName}" - - className := o.ClassName - if className != "" { - _path = strings.Replace(_path, "{className}", className, -1) - } else { - return nil, errors.New("className is required on SchemaObjectsShardsUpdateURL") - } - - shardName := o.ShardName - if shardName != "" { - _path = strings.Replace(_path, "{shardName}", shardName, -1) - } else { - return nil, errors.New("shardName is required on SchemaObjectsShardsUpdateURL") - } - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *SchemaObjectsShardsUpdateURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *SchemaObjectsShardsUpdateURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *SchemaObjectsShardsUpdateURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on SchemaObjectsShardsUpdateURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on SchemaObjectsShardsUpdateURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *SchemaObjectsShardsUpdateURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/schema/schema_objects_update.go b/adapters/handlers/rest/operations/schema/schema_objects_update.go deleted file mode 100644 index e18d3a09b1f5a2c36a23619d0f6a612f0e7218ff..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/schema_objects_update.go +++ /dev/null @@ -1,84 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// SchemaObjectsUpdateHandlerFunc turns a function with the right signature into a schema objects update handler -type SchemaObjectsUpdateHandlerFunc func(SchemaObjectsUpdateParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn SchemaObjectsUpdateHandlerFunc) Handle(params SchemaObjectsUpdateParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// SchemaObjectsUpdateHandler interface for that can handle valid schema objects update params -type SchemaObjectsUpdateHandler interface { - Handle(SchemaObjectsUpdateParams, *models.Principal) middleware.Responder -} - -// NewSchemaObjectsUpdate creates a new http.Handler for the schema objects update operation -func NewSchemaObjectsUpdate(ctx *middleware.Context, handler SchemaObjectsUpdateHandler) *SchemaObjectsUpdate { - return &SchemaObjectsUpdate{Context: ctx, Handler: handler} -} - -/* - SchemaObjectsUpdate swagger:route PUT /schema/{className} schema schemaObjectsUpdate - -# Update settings of an existing schema class - -Use this endpoint to alter an existing class in the schema. Note that not all settings are mutable. If an error about immutable fields is returned and you still need to update this particular setting, you will have to delete the class (and the underlying data) and recreate. This endpoint cannot be used to modify properties. Instead use POST /v1/schema/{className}/properties. A typical use case for this endpoint is to update configuration, such as the vectorIndexConfig. Note that even in mutable sections, such as vectorIndexConfig, some fields may be immutable. -*/ -type SchemaObjectsUpdate struct { - Context *middleware.Context - Handler SchemaObjectsUpdateHandler -} - -func (o *SchemaObjectsUpdate) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewSchemaObjectsUpdateParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/schema/schema_objects_update_parameters.go b/adapters/handlers/rest/operations/schema/schema_objects_update_parameters.go deleted file mode 100644 index 866e464b0f6e31e6eca7e0535dbcda96056905f1..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/schema_objects_update_parameters.go +++ /dev/null @@ -1,120 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "io" - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/validate" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewSchemaObjectsUpdateParams creates a new SchemaObjectsUpdateParams object -// -// There are no default values defined in the spec. -func NewSchemaObjectsUpdateParams() SchemaObjectsUpdateParams { - - return SchemaObjectsUpdateParams{} -} - -// SchemaObjectsUpdateParams contains all the bound params for the schema objects update operation -// typically these are obtained from a http.Request -// -// swagger:parameters schema.objects.update -type SchemaObjectsUpdateParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` - - /* - Required: true - In: path - */ - ClassName string - /* - Required: true - In: body - */ - ObjectClass *models.Class -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewSchemaObjectsUpdateParams() beforehand. -func (o *SchemaObjectsUpdateParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - rClassName, rhkClassName, _ := route.Params.GetOK("className") - if err := o.bindClassName(rClassName, rhkClassName, route.Formats); err != nil { - res = append(res, err) - } - - if runtime.HasBody(r) { - defer r.Body.Close() - var body models.Class - if err := route.Consumer.Consume(r.Body, &body); err != nil { - if err == io.EOF { - res = append(res, errors.Required("objectClass", "body", "")) - } else { - res = append(res, errors.NewParseError("objectClass", "body", "", err)) - } - } else { - // validate body object - if err := body.Validate(route.Formats); err != nil { - res = append(res, err) - } - - ctx := validate.WithOperationRequest(r.Context()) - if err := body.ContextValidate(ctx, route.Formats); err != nil { - res = append(res, err) - } - - if len(res) == 0 { - o.ObjectClass = &body - } - } - } else { - res = append(res, errors.Required("objectClass", "body", "")) - } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// bindClassName binds and validates parameter ClassName from path. -func (o *SchemaObjectsUpdateParams) bindClassName(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - o.ClassName = raw - - return nil -} diff --git a/adapters/handlers/rest/operations/schema/schema_objects_update_responses.go b/adapters/handlers/rest/operations/schema/schema_objects_update_responses.go deleted file mode 100644 index cc35dca04ef3b03b756bb7d2d3cc8d6c302390a3..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/schema_objects_update_responses.go +++ /dev/null @@ -1,275 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// SchemaObjectsUpdateOKCode is the HTTP code returned for type SchemaObjectsUpdateOK -const SchemaObjectsUpdateOKCode int = 200 - -/* -SchemaObjectsUpdateOK Class was updated successfully - -swagger:response schemaObjectsUpdateOK -*/ -type SchemaObjectsUpdateOK struct { - - /* - In: Body - */ - Payload *models.Class `json:"body,omitempty"` -} - -// NewSchemaObjectsUpdateOK creates SchemaObjectsUpdateOK with default headers values -func NewSchemaObjectsUpdateOK() *SchemaObjectsUpdateOK { - - return &SchemaObjectsUpdateOK{} -} - -// WithPayload adds the payload to the schema objects update o k response -func (o *SchemaObjectsUpdateOK) WithPayload(payload *models.Class) *SchemaObjectsUpdateOK { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the schema objects update o k response -func (o *SchemaObjectsUpdateOK) SetPayload(payload *models.Class) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *SchemaObjectsUpdateOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(200) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// SchemaObjectsUpdateUnauthorizedCode is the HTTP code returned for type SchemaObjectsUpdateUnauthorized -const SchemaObjectsUpdateUnauthorizedCode int = 401 - -/* -SchemaObjectsUpdateUnauthorized Unauthorized or invalid credentials. - -swagger:response schemaObjectsUpdateUnauthorized -*/ -type SchemaObjectsUpdateUnauthorized struct { -} - -// NewSchemaObjectsUpdateUnauthorized creates SchemaObjectsUpdateUnauthorized with default headers values -func NewSchemaObjectsUpdateUnauthorized() *SchemaObjectsUpdateUnauthorized { - - return &SchemaObjectsUpdateUnauthorized{} -} - -// WriteResponse to the client -func (o *SchemaObjectsUpdateUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// SchemaObjectsUpdateForbiddenCode is the HTTP code returned for type SchemaObjectsUpdateForbidden -const SchemaObjectsUpdateForbiddenCode int = 403 - -/* -SchemaObjectsUpdateForbidden Forbidden - -swagger:response schemaObjectsUpdateForbidden -*/ -type SchemaObjectsUpdateForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewSchemaObjectsUpdateForbidden creates SchemaObjectsUpdateForbidden with default headers values -func NewSchemaObjectsUpdateForbidden() *SchemaObjectsUpdateForbidden { - - return &SchemaObjectsUpdateForbidden{} -} - -// WithPayload adds the payload to the schema objects update forbidden response -func (o *SchemaObjectsUpdateForbidden) WithPayload(payload *models.ErrorResponse) *SchemaObjectsUpdateForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the schema objects update forbidden response -func (o *SchemaObjectsUpdateForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *SchemaObjectsUpdateForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// SchemaObjectsUpdateNotFoundCode is the HTTP code returned for type SchemaObjectsUpdateNotFound -const SchemaObjectsUpdateNotFoundCode int = 404 - -/* -SchemaObjectsUpdateNotFound Class to be updated does not exist - -swagger:response schemaObjectsUpdateNotFound -*/ -type SchemaObjectsUpdateNotFound struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewSchemaObjectsUpdateNotFound creates SchemaObjectsUpdateNotFound with default headers values -func NewSchemaObjectsUpdateNotFound() *SchemaObjectsUpdateNotFound { - - return &SchemaObjectsUpdateNotFound{} -} - -// WithPayload adds the payload to the schema objects update not found response -func (o *SchemaObjectsUpdateNotFound) WithPayload(payload *models.ErrorResponse) *SchemaObjectsUpdateNotFound { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the schema objects update not found response -func (o *SchemaObjectsUpdateNotFound) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *SchemaObjectsUpdateNotFound) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(404) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// SchemaObjectsUpdateUnprocessableEntityCode is the HTTP code returned for type SchemaObjectsUpdateUnprocessableEntity -const SchemaObjectsUpdateUnprocessableEntityCode int = 422 - -/* -SchemaObjectsUpdateUnprocessableEntity Invalid update attempt - -swagger:response schemaObjectsUpdateUnprocessableEntity -*/ -type SchemaObjectsUpdateUnprocessableEntity struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewSchemaObjectsUpdateUnprocessableEntity creates SchemaObjectsUpdateUnprocessableEntity with default headers values -func NewSchemaObjectsUpdateUnprocessableEntity() *SchemaObjectsUpdateUnprocessableEntity { - - return &SchemaObjectsUpdateUnprocessableEntity{} -} - -// WithPayload adds the payload to the schema objects update unprocessable entity response -func (o *SchemaObjectsUpdateUnprocessableEntity) WithPayload(payload *models.ErrorResponse) *SchemaObjectsUpdateUnprocessableEntity { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the schema objects update unprocessable entity response -func (o *SchemaObjectsUpdateUnprocessableEntity) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *SchemaObjectsUpdateUnprocessableEntity) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(422) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// SchemaObjectsUpdateInternalServerErrorCode is the HTTP code returned for type SchemaObjectsUpdateInternalServerError -const SchemaObjectsUpdateInternalServerErrorCode int = 500 - -/* -SchemaObjectsUpdateInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response schemaObjectsUpdateInternalServerError -*/ -type SchemaObjectsUpdateInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewSchemaObjectsUpdateInternalServerError creates SchemaObjectsUpdateInternalServerError with default headers values -func NewSchemaObjectsUpdateInternalServerError() *SchemaObjectsUpdateInternalServerError { - - return &SchemaObjectsUpdateInternalServerError{} -} - -// WithPayload adds the payload to the schema objects update internal server error response -func (o *SchemaObjectsUpdateInternalServerError) WithPayload(payload *models.ErrorResponse) *SchemaObjectsUpdateInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the schema objects update internal server error response -func (o *SchemaObjectsUpdateInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *SchemaObjectsUpdateInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/schema/schema_objects_update_urlbuilder.go b/adapters/handlers/rest/operations/schema/schema_objects_update_urlbuilder.go deleted file mode 100644 index dc9ec0a65d5edacb69de499fea9cc04e25b7b141..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/schema_objects_update_urlbuilder.go +++ /dev/null @@ -1,110 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" - "strings" -) - -// SchemaObjectsUpdateURL generates an URL for the schema objects update operation -type SchemaObjectsUpdateURL struct { - ClassName string - - _basePath string - // avoid unkeyed usage - _ struct{} -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *SchemaObjectsUpdateURL) WithBasePath(bp string) *SchemaObjectsUpdateURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *SchemaObjectsUpdateURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *SchemaObjectsUpdateURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/schema/{className}" - - className := o.ClassName - if className != "" { - _path = strings.Replace(_path, "{className}", className, -1) - } else { - return nil, errors.New("className is required on SchemaObjectsUpdateURL") - } - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *SchemaObjectsUpdateURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *SchemaObjectsUpdateURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *SchemaObjectsUpdateURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on SchemaObjectsUpdateURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on SchemaObjectsUpdateURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *SchemaObjectsUpdateURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/schema/tenants_create.go b/adapters/handlers/rest/operations/schema/tenants_create.go deleted file mode 100644 index 6c7d65d3a082c63d51088f29c61157712f7493d7..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/tenants_create.go +++ /dev/null @@ -1,82 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// TenantsCreateHandlerFunc turns a function with the right signature into a tenants create handler -type TenantsCreateHandlerFunc func(TenantsCreateParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn TenantsCreateHandlerFunc) Handle(params TenantsCreateParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// TenantsCreateHandler interface for that can handle valid tenants create params -type TenantsCreateHandler interface { - Handle(TenantsCreateParams, *models.Principal) middleware.Responder -} - -// NewTenantsCreate creates a new http.Handler for the tenants create operation -func NewTenantsCreate(ctx *middleware.Context, handler TenantsCreateHandler) *TenantsCreate { - return &TenantsCreate{Context: ctx, Handler: handler} -} - -/* - TenantsCreate swagger:route POST /schema/{className}/tenants schema tenantsCreate - -Create a new tenant for a specific class -*/ -type TenantsCreate struct { - Context *middleware.Context - Handler TenantsCreateHandler -} - -func (o *TenantsCreate) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewTenantsCreateParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/schema/tenants_create_parameters.go b/adapters/handlers/rest/operations/schema/tenants_create_parameters.go deleted file mode 100644 index 30a17e1b0d9b7ddaf0045e31954188a67544853e..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/tenants_create_parameters.go +++ /dev/null @@ -1,121 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "io" - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewTenantsCreateParams creates a new TenantsCreateParams object -// -// There are no default values defined in the spec. -func NewTenantsCreateParams() TenantsCreateParams { - - return TenantsCreateParams{} -} - -// TenantsCreateParams contains all the bound params for the tenants create operation -// typically these are obtained from a http.Request -// -// swagger:parameters tenants.create -type TenantsCreateParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` - - /* - Required: true - In: body - */ - Body []*models.Tenant - /* - Required: true - In: path - */ - ClassName string -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewTenantsCreateParams() beforehand. -func (o *TenantsCreateParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - if runtime.HasBody(r) { - defer r.Body.Close() - var body []*models.Tenant - if err := route.Consumer.Consume(r.Body, &body); err != nil { - if err == io.EOF { - res = append(res, errors.Required("body", "body", "")) - } else { - res = append(res, errors.NewParseError("body", "body", "", err)) - } - } else { - - // validate array of body objects - for i := range body { - if body[i] == nil { - continue - } - if err := body[i].Validate(route.Formats); err != nil { - res = append(res, err) - break - } - } - - if len(res) == 0 { - o.Body = body - } - } - } else { - res = append(res, errors.Required("body", "body", "")) - } - - rClassName, rhkClassName, _ := route.Params.GetOK("className") - if err := o.bindClassName(rClassName, rhkClassName, route.Formats); err != nil { - res = append(res, err) - } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// bindClassName binds and validates parameter ClassName from path. -func (o *TenantsCreateParams) bindClassName(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - o.ClassName = raw - - return nil -} diff --git a/adapters/handlers/rest/operations/schema/tenants_create_responses.go b/adapters/handlers/rest/operations/schema/tenants_create_responses.go deleted file mode 100644 index 66b6d8c8cf3576d2df6d5e79ed21c3e4d1bb54bb..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/tenants_create_responses.go +++ /dev/null @@ -1,233 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// TenantsCreateOKCode is the HTTP code returned for type TenantsCreateOK -const TenantsCreateOKCode int = 200 - -/* -TenantsCreateOK Added new tenants to the specified class - -swagger:response tenantsCreateOK -*/ -type TenantsCreateOK struct { - - /* - In: Body - */ - Payload []*models.Tenant `json:"body,omitempty"` -} - -// NewTenantsCreateOK creates TenantsCreateOK with default headers values -func NewTenantsCreateOK() *TenantsCreateOK { - - return &TenantsCreateOK{} -} - -// WithPayload adds the payload to the tenants create o k response -func (o *TenantsCreateOK) WithPayload(payload []*models.Tenant) *TenantsCreateOK { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the tenants create o k response -func (o *TenantsCreateOK) SetPayload(payload []*models.Tenant) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *TenantsCreateOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(200) - payload := o.Payload - if payload == nil { - // return empty array - payload = make([]*models.Tenant, 0, 50) - } - - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } -} - -// TenantsCreateUnauthorizedCode is the HTTP code returned for type TenantsCreateUnauthorized -const TenantsCreateUnauthorizedCode int = 401 - -/* -TenantsCreateUnauthorized Unauthorized or invalid credentials. - -swagger:response tenantsCreateUnauthorized -*/ -type TenantsCreateUnauthorized struct { -} - -// NewTenantsCreateUnauthorized creates TenantsCreateUnauthorized with default headers values -func NewTenantsCreateUnauthorized() *TenantsCreateUnauthorized { - - return &TenantsCreateUnauthorized{} -} - -// WriteResponse to the client -func (o *TenantsCreateUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// TenantsCreateForbiddenCode is the HTTP code returned for type TenantsCreateForbidden -const TenantsCreateForbiddenCode int = 403 - -/* -TenantsCreateForbidden Forbidden - -swagger:response tenantsCreateForbidden -*/ -type TenantsCreateForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewTenantsCreateForbidden creates TenantsCreateForbidden with default headers values -func NewTenantsCreateForbidden() *TenantsCreateForbidden { - - return &TenantsCreateForbidden{} -} - -// WithPayload adds the payload to the tenants create forbidden response -func (o *TenantsCreateForbidden) WithPayload(payload *models.ErrorResponse) *TenantsCreateForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the tenants create forbidden response -func (o *TenantsCreateForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *TenantsCreateForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// TenantsCreateUnprocessableEntityCode is the HTTP code returned for type TenantsCreateUnprocessableEntity -const TenantsCreateUnprocessableEntityCode int = 422 - -/* -TenantsCreateUnprocessableEntity Invalid Tenant class - -swagger:response tenantsCreateUnprocessableEntity -*/ -type TenantsCreateUnprocessableEntity struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewTenantsCreateUnprocessableEntity creates TenantsCreateUnprocessableEntity with default headers values -func NewTenantsCreateUnprocessableEntity() *TenantsCreateUnprocessableEntity { - - return &TenantsCreateUnprocessableEntity{} -} - -// WithPayload adds the payload to the tenants create unprocessable entity response -func (o *TenantsCreateUnprocessableEntity) WithPayload(payload *models.ErrorResponse) *TenantsCreateUnprocessableEntity { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the tenants create unprocessable entity response -func (o *TenantsCreateUnprocessableEntity) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *TenantsCreateUnprocessableEntity) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(422) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// TenantsCreateInternalServerErrorCode is the HTTP code returned for type TenantsCreateInternalServerError -const TenantsCreateInternalServerErrorCode int = 500 - -/* -TenantsCreateInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response tenantsCreateInternalServerError -*/ -type TenantsCreateInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewTenantsCreateInternalServerError creates TenantsCreateInternalServerError with default headers values -func NewTenantsCreateInternalServerError() *TenantsCreateInternalServerError { - - return &TenantsCreateInternalServerError{} -} - -// WithPayload adds the payload to the tenants create internal server error response -func (o *TenantsCreateInternalServerError) WithPayload(payload *models.ErrorResponse) *TenantsCreateInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the tenants create internal server error response -func (o *TenantsCreateInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *TenantsCreateInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/schema/tenants_create_urlbuilder.go b/adapters/handlers/rest/operations/schema/tenants_create_urlbuilder.go deleted file mode 100644 index 9cecf1b6689e05b3459fd6806461a22600fdbc0e..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/tenants_create_urlbuilder.go +++ /dev/null @@ -1,110 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" - "strings" -) - -// TenantsCreateURL generates an URL for the tenants create operation -type TenantsCreateURL struct { - ClassName string - - _basePath string - // avoid unkeyed usage - _ struct{} -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *TenantsCreateURL) WithBasePath(bp string) *TenantsCreateURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *TenantsCreateURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *TenantsCreateURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/schema/{className}/tenants" - - className := o.ClassName - if className != "" { - _path = strings.Replace(_path, "{className}", className, -1) - } else { - return nil, errors.New("className is required on TenantsCreateURL") - } - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *TenantsCreateURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *TenantsCreateURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *TenantsCreateURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on TenantsCreateURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on TenantsCreateURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *TenantsCreateURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/schema/tenants_delete.go b/adapters/handlers/rest/operations/schema/tenants_delete.go deleted file mode 100644 index 4fc86eaa0268c72088cf911a3306fc24593a4dd1..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/tenants_delete.go +++ /dev/null @@ -1,82 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// TenantsDeleteHandlerFunc turns a function with the right signature into a tenants delete handler -type TenantsDeleteHandlerFunc func(TenantsDeleteParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn TenantsDeleteHandlerFunc) Handle(params TenantsDeleteParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// TenantsDeleteHandler interface for that can handle valid tenants delete params -type TenantsDeleteHandler interface { - Handle(TenantsDeleteParams, *models.Principal) middleware.Responder -} - -// NewTenantsDelete creates a new http.Handler for the tenants delete operation -func NewTenantsDelete(ctx *middleware.Context, handler TenantsDeleteHandler) *TenantsDelete { - return &TenantsDelete{Context: ctx, Handler: handler} -} - -/* - TenantsDelete swagger:route DELETE /schema/{className}/tenants schema tenantsDelete - -delete tenants from a specific class -*/ -type TenantsDelete struct { - Context *middleware.Context - Handler TenantsDeleteHandler -} - -func (o *TenantsDelete) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewTenantsDeleteParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/schema/tenants_delete_parameters.go b/adapters/handlers/rest/operations/schema/tenants_delete_parameters.go deleted file mode 100644 index bb5b90750bd81486f2888377d45d7bc113cda77e..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/tenants_delete_parameters.go +++ /dev/null @@ -1,106 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "io" - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" -) - -// NewTenantsDeleteParams creates a new TenantsDeleteParams object -// -// There are no default values defined in the spec. -func NewTenantsDeleteParams() TenantsDeleteParams { - - return TenantsDeleteParams{} -} - -// TenantsDeleteParams contains all the bound params for the tenants delete operation -// typically these are obtained from a http.Request -// -// swagger:parameters tenants.delete -type TenantsDeleteParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` - - /* - Required: true - In: path - */ - ClassName string - /* - Required: true - In: body - */ - Tenants []string -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewTenantsDeleteParams() beforehand. -func (o *TenantsDeleteParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - rClassName, rhkClassName, _ := route.Params.GetOK("className") - if err := o.bindClassName(rClassName, rhkClassName, route.Formats); err != nil { - res = append(res, err) - } - - if runtime.HasBody(r) { - defer r.Body.Close() - var body []string - if err := route.Consumer.Consume(r.Body, &body); err != nil { - if err == io.EOF { - res = append(res, errors.Required("tenants", "body", "")) - } else { - res = append(res, errors.NewParseError("tenants", "body", "", err)) - } - } else { - // no validation required on inline body - o.Tenants = body - } - } else { - res = append(res, errors.Required("tenants", "body", "")) - } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// bindClassName binds and validates parameter ClassName from path. -func (o *TenantsDeleteParams) bindClassName(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - o.ClassName = raw - - return nil -} diff --git a/adapters/handlers/rest/operations/schema/tenants_delete_responses.go b/adapters/handlers/rest/operations/schema/tenants_delete_responses.go deleted file mode 100644 index cc08d942cf514823fe7bfce361adffab7df0d4b5..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/tenants_delete_responses.go +++ /dev/null @@ -1,210 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// TenantsDeleteOKCode is the HTTP code returned for type TenantsDeleteOK -const TenantsDeleteOKCode int = 200 - -/* -TenantsDeleteOK Deleted tenants from specified class. - -swagger:response tenantsDeleteOK -*/ -type TenantsDeleteOK struct { -} - -// NewTenantsDeleteOK creates TenantsDeleteOK with default headers values -func NewTenantsDeleteOK() *TenantsDeleteOK { - - return &TenantsDeleteOK{} -} - -// WriteResponse to the client -func (o *TenantsDeleteOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(200) -} - -// TenantsDeleteUnauthorizedCode is the HTTP code returned for type TenantsDeleteUnauthorized -const TenantsDeleteUnauthorizedCode int = 401 - -/* -TenantsDeleteUnauthorized Unauthorized or invalid credentials. - -swagger:response tenantsDeleteUnauthorized -*/ -type TenantsDeleteUnauthorized struct { -} - -// NewTenantsDeleteUnauthorized creates TenantsDeleteUnauthorized with default headers values -func NewTenantsDeleteUnauthorized() *TenantsDeleteUnauthorized { - - return &TenantsDeleteUnauthorized{} -} - -// WriteResponse to the client -func (o *TenantsDeleteUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// TenantsDeleteForbiddenCode is the HTTP code returned for type TenantsDeleteForbidden -const TenantsDeleteForbiddenCode int = 403 - -/* -TenantsDeleteForbidden Forbidden - -swagger:response tenantsDeleteForbidden -*/ -type TenantsDeleteForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewTenantsDeleteForbidden creates TenantsDeleteForbidden with default headers values -func NewTenantsDeleteForbidden() *TenantsDeleteForbidden { - - return &TenantsDeleteForbidden{} -} - -// WithPayload adds the payload to the tenants delete forbidden response -func (o *TenantsDeleteForbidden) WithPayload(payload *models.ErrorResponse) *TenantsDeleteForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the tenants delete forbidden response -func (o *TenantsDeleteForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *TenantsDeleteForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// TenantsDeleteUnprocessableEntityCode is the HTTP code returned for type TenantsDeleteUnprocessableEntity -const TenantsDeleteUnprocessableEntityCode int = 422 - -/* -TenantsDeleteUnprocessableEntity Invalid Tenant class - -swagger:response tenantsDeleteUnprocessableEntity -*/ -type TenantsDeleteUnprocessableEntity struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewTenantsDeleteUnprocessableEntity creates TenantsDeleteUnprocessableEntity with default headers values -func NewTenantsDeleteUnprocessableEntity() *TenantsDeleteUnprocessableEntity { - - return &TenantsDeleteUnprocessableEntity{} -} - -// WithPayload adds the payload to the tenants delete unprocessable entity response -func (o *TenantsDeleteUnprocessableEntity) WithPayload(payload *models.ErrorResponse) *TenantsDeleteUnprocessableEntity { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the tenants delete unprocessable entity response -func (o *TenantsDeleteUnprocessableEntity) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *TenantsDeleteUnprocessableEntity) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(422) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// TenantsDeleteInternalServerErrorCode is the HTTP code returned for type TenantsDeleteInternalServerError -const TenantsDeleteInternalServerErrorCode int = 500 - -/* -TenantsDeleteInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response tenantsDeleteInternalServerError -*/ -type TenantsDeleteInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewTenantsDeleteInternalServerError creates TenantsDeleteInternalServerError with default headers values -func NewTenantsDeleteInternalServerError() *TenantsDeleteInternalServerError { - - return &TenantsDeleteInternalServerError{} -} - -// WithPayload adds the payload to the tenants delete internal server error response -func (o *TenantsDeleteInternalServerError) WithPayload(payload *models.ErrorResponse) *TenantsDeleteInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the tenants delete internal server error response -func (o *TenantsDeleteInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *TenantsDeleteInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/schema/tenants_delete_urlbuilder.go b/adapters/handlers/rest/operations/schema/tenants_delete_urlbuilder.go deleted file mode 100644 index 32f72656a1834f1d4f3386614a2c3e414d7e88a0..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/tenants_delete_urlbuilder.go +++ /dev/null @@ -1,110 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" - "strings" -) - -// TenantsDeleteURL generates an URL for the tenants delete operation -type TenantsDeleteURL struct { - ClassName string - - _basePath string - // avoid unkeyed usage - _ struct{} -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *TenantsDeleteURL) WithBasePath(bp string) *TenantsDeleteURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *TenantsDeleteURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *TenantsDeleteURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/schema/{className}/tenants" - - className := o.ClassName - if className != "" { - _path = strings.Replace(_path, "{className}", className, -1) - } else { - return nil, errors.New("className is required on TenantsDeleteURL") - } - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *TenantsDeleteURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *TenantsDeleteURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *TenantsDeleteURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on TenantsDeleteURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on TenantsDeleteURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *TenantsDeleteURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/schema/tenants_get.go b/adapters/handlers/rest/operations/schema/tenants_get.go deleted file mode 100644 index a1effe91d0e7ef74b7280b002eac333c61a1548d..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/tenants_get.go +++ /dev/null @@ -1,82 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// TenantsGetHandlerFunc turns a function with the right signature into a tenants get handler -type TenantsGetHandlerFunc func(TenantsGetParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn TenantsGetHandlerFunc) Handle(params TenantsGetParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// TenantsGetHandler interface for that can handle valid tenants get params -type TenantsGetHandler interface { - Handle(TenantsGetParams, *models.Principal) middleware.Responder -} - -// NewTenantsGet creates a new http.Handler for the tenants get operation -func NewTenantsGet(ctx *middleware.Context, handler TenantsGetHandler) *TenantsGet { - return &TenantsGet{Context: ctx, Handler: handler} -} - -/* - TenantsGet swagger:route GET /schema/{className}/tenants schema tenantsGet - -get all tenants from a specific class -*/ -type TenantsGet struct { - Context *middleware.Context - Handler TenantsGetHandler -} - -func (o *TenantsGet) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewTenantsGetParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/schema/tenants_get_parameters.go b/adapters/handlers/rest/operations/schema/tenants_get_parameters.go deleted file mode 100644 index 311dd27dd9ce61dd61c7b4f91428e54662bfb146..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/tenants_get_parameters.go +++ /dev/null @@ -1,82 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" -) - -// NewTenantsGetParams creates a new TenantsGetParams object -// -// There are no default values defined in the spec. -func NewTenantsGetParams() TenantsGetParams { - - return TenantsGetParams{} -} - -// TenantsGetParams contains all the bound params for the tenants get operation -// typically these are obtained from a http.Request -// -// swagger:parameters tenants.get -type TenantsGetParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` - - /* - Required: true - In: path - */ - ClassName string -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewTenantsGetParams() beforehand. -func (o *TenantsGetParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - rClassName, rhkClassName, _ := route.Params.GetOK("className") - if err := o.bindClassName(rClassName, rhkClassName, route.Formats); err != nil { - res = append(res, err) - } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// bindClassName binds and validates parameter ClassName from path. -func (o *TenantsGetParams) bindClassName(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - o.ClassName = raw - - return nil -} diff --git a/adapters/handlers/rest/operations/schema/tenants_get_responses.go b/adapters/handlers/rest/operations/schema/tenants_get_responses.go deleted file mode 100644 index 51ee30c3d3da5e5c850b5d9f7603f51100d01e61..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/tenants_get_responses.go +++ /dev/null @@ -1,233 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// TenantsGetOKCode is the HTTP code returned for type TenantsGetOK -const TenantsGetOKCode int = 200 - -/* -TenantsGetOK tenants from specified class. - -swagger:response tenantsGetOK -*/ -type TenantsGetOK struct { - - /* - In: Body - */ - Payload []*models.Tenant `json:"body,omitempty"` -} - -// NewTenantsGetOK creates TenantsGetOK with default headers values -func NewTenantsGetOK() *TenantsGetOK { - - return &TenantsGetOK{} -} - -// WithPayload adds the payload to the tenants get o k response -func (o *TenantsGetOK) WithPayload(payload []*models.Tenant) *TenantsGetOK { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the tenants get o k response -func (o *TenantsGetOK) SetPayload(payload []*models.Tenant) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *TenantsGetOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(200) - payload := o.Payload - if payload == nil { - // return empty array - payload = make([]*models.Tenant, 0, 50) - } - - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } -} - -// TenantsGetUnauthorizedCode is the HTTP code returned for type TenantsGetUnauthorized -const TenantsGetUnauthorizedCode int = 401 - -/* -TenantsGetUnauthorized Unauthorized or invalid credentials. - -swagger:response tenantsGetUnauthorized -*/ -type TenantsGetUnauthorized struct { -} - -// NewTenantsGetUnauthorized creates TenantsGetUnauthorized with default headers values -func NewTenantsGetUnauthorized() *TenantsGetUnauthorized { - - return &TenantsGetUnauthorized{} -} - -// WriteResponse to the client -func (o *TenantsGetUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// TenantsGetForbiddenCode is the HTTP code returned for type TenantsGetForbidden -const TenantsGetForbiddenCode int = 403 - -/* -TenantsGetForbidden Forbidden - -swagger:response tenantsGetForbidden -*/ -type TenantsGetForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewTenantsGetForbidden creates TenantsGetForbidden with default headers values -func NewTenantsGetForbidden() *TenantsGetForbidden { - - return &TenantsGetForbidden{} -} - -// WithPayload adds the payload to the tenants get forbidden response -func (o *TenantsGetForbidden) WithPayload(payload *models.ErrorResponse) *TenantsGetForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the tenants get forbidden response -func (o *TenantsGetForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *TenantsGetForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// TenantsGetUnprocessableEntityCode is the HTTP code returned for type TenantsGetUnprocessableEntity -const TenantsGetUnprocessableEntityCode int = 422 - -/* -TenantsGetUnprocessableEntity Invalid Tenant class - -swagger:response tenantsGetUnprocessableEntity -*/ -type TenantsGetUnprocessableEntity struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewTenantsGetUnprocessableEntity creates TenantsGetUnprocessableEntity with default headers values -func NewTenantsGetUnprocessableEntity() *TenantsGetUnprocessableEntity { - - return &TenantsGetUnprocessableEntity{} -} - -// WithPayload adds the payload to the tenants get unprocessable entity response -func (o *TenantsGetUnprocessableEntity) WithPayload(payload *models.ErrorResponse) *TenantsGetUnprocessableEntity { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the tenants get unprocessable entity response -func (o *TenantsGetUnprocessableEntity) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *TenantsGetUnprocessableEntity) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(422) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// TenantsGetInternalServerErrorCode is the HTTP code returned for type TenantsGetInternalServerError -const TenantsGetInternalServerErrorCode int = 500 - -/* -TenantsGetInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response tenantsGetInternalServerError -*/ -type TenantsGetInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewTenantsGetInternalServerError creates TenantsGetInternalServerError with default headers values -func NewTenantsGetInternalServerError() *TenantsGetInternalServerError { - - return &TenantsGetInternalServerError{} -} - -// WithPayload adds the payload to the tenants get internal server error response -func (o *TenantsGetInternalServerError) WithPayload(payload *models.ErrorResponse) *TenantsGetInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the tenants get internal server error response -func (o *TenantsGetInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *TenantsGetInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/schema/tenants_get_urlbuilder.go b/adapters/handlers/rest/operations/schema/tenants_get_urlbuilder.go deleted file mode 100644 index 3de80e159bb93d74f0a97ab759f75f57642cce45..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/tenants_get_urlbuilder.go +++ /dev/null @@ -1,110 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" - "strings" -) - -// TenantsGetURL generates an URL for the tenants get operation -type TenantsGetURL struct { - ClassName string - - _basePath string - // avoid unkeyed usage - _ struct{} -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *TenantsGetURL) WithBasePath(bp string) *TenantsGetURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *TenantsGetURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *TenantsGetURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/schema/{className}/tenants" - - className := o.ClassName - if className != "" { - _path = strings.Replace(_path, "{className}", className, -1) - } else { - return nil, errors.New("className is required on TenantsGetURL") - } - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *TenantsGetURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *TenantsGetURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *TenantsGetURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on TenantsGetURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on TenantsGetURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *TenantsGetURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/schema/tenants_update.go b/adapters/handlers/rest/operations/schema/tenants_update.go deleted file mode 100644 index b62fc9ba9632735c9320b2effdb1daa11467ea3a..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/tenants_update.go +++ /dev/null @@ -1,82 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// TenantsUpdateHandlerFunc turns a function with the right signature into a tenants update handler -type TenantsUpdateHandlerFunc func(TenantsUpdateParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn TenantsUpdateHandlerFunc) Handle(params TenantsUpdateParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// TenantsUpdateHandler interface for that can handle valid tenants update params -type TenantsUpdateHandler interface { - Handle(TenantsUpdateParams, *models.Principal) middleware.Responder -} - -// NewTenantsUpdate creates a new http.Handler for the tenants update operation -func NewTenantsUpdate(ctx *middleware.Context, handler TenantsUpdateHandler) *TenantsUpdate { - return &TenantsUpdate{Context: ctx, Handler: handler} -} - -/* - TenantsUpdate swagger:route PUT /schema/{className}/tenants schema tenantsUpdate - -Update tenant of a specific class -*/ -type TenantsUpdate struct { - Context *middleware.Context - Handler TenantsUpdateHandler -} - -func (o *TenantsUpdate) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewTenantsUpdateParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/schema/tenants_update_parameters.go b/adapters/handlers/rest/operations/schema/tenants_update_parameters.go deleted file mode 100644 index ced2f775e0876a41a44eb8e8ac9b5523401940f2..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/tenants_update_parameters.go +++ /dev/null @@ -1,121 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "io" - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewTenantsUpdateParams creates a new TenantsUpdateParams object -// -// There are no default values defined in the spec. -func NewTenantsUpdateParams() TenantsUpdateParams { - - return TenantsUpdateParams{} -} - -// TenantsUpdateParams contains all the bound params for the tenants update operation -// typically these are obtained from a http.Request -// -// swagger:parameters tenants.update -type TenantsUpdateParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` - - /* - Required: true - In: body - */ - Body []*models.Tenant - /* - Required: true - In: path - */ - ClassName string -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewTenantsUpdateParams() beforehand. -func (o *TenantsUpdateParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - if runtime.HasBody(r) { - defer r.Body.Close() - var body []*models.Tenant - if err := route.Consumer.Consume(r.Body, &body); err != nil { - if err == io.EOF { - res = append(res, errors.Required("body", "body", "")) - } else { - res = append(res, errors.NewParseError("body", "body", "", err)) - } - } else { - - // validate array of body objects - for i := range body { - if body[i] == nil { - continue - } - if err := body[i].Validate(route.Formats); err != nil { - res = append(res, err) - break - } - } - - if len(res) == 0 { - o.Body = body - } - } - } else { - res = append(res, errors.Required("body", "body", "")) - } - - rClassName, rhkClassName, _ := route.Params.GetOK("className") - if err := o.bindClassName(rClassName, rhkClassName, route.Formats); err != nil { - res = append(res, err) - } - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// bindClassName binds and validates parameter ClassName from path. -func (o *TenantsUpdateParams) bindClassName(rawData []string, hasKey bool, formats strfmt.Registry) error { - var raw string - if len(rawData) > 0 { - raw = rawData[len(rawData)-1] - } - - // Required: true - // Parameter is provided by construction from the route - o.ClassName = raw - - return nil -} diff --git a/adapters/handlers/rest/operations/schema/tenants_update_responses.go b/adapters/handlers/rest/operations/schema/tenants_update_responses.go deleted file mode 100644 index 0329de1920f7cfcbf6c73e308a8e72615ba4bd01..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/tenants_update_responses.go +++ /dev/null @@ -1,233 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// TenantsUpdateOKCode is the HTTP code returned for type TenantsUpdateOK -const TenantsUpdateOKCode int = 200 - -/* -TenantsUpdateOK Updated tenants of the specified class - -swagger:response tenantsUpdateOK -*/ -type TenantsUpdateOK struct { - - /* - In: Body - */ - Payload []*models.Tenant `json:"body,omitempty"` -} - -// NewTenantsUpdateOK creates TenantsUpdateOK with default headers values -func NewTenantsUpdateOK() *TenantsUpdateOK { - - return &TenantsUpdateOK{} -} - -// WithPayload adds the payload to the tenants update o k response -func (o *TenantsUpdateOK) WithPayload(payload []*models.Tenant) *TenantsUpdateOK { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the tenants update o k response -func (o *TenantsUpdateOK) SetPayload(payload []*models.Tenant) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *TenantsUpdateOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(200) - payload := o.Payload - if payload == nil { - // return empty array - payload = make([]*models.Tenant, 0, 50) - } - - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } -} - -// TenantsUpdateUnauthorizedCode is the HTTP code returned for type TenantsUpdateUnauthorized -const TenantsUpdateUnauthorizedCode int = 401 - -/* -TenantsUpdateUnauthorized Unauthorized or invalid credentials. - -swagger:response tenantsUpdateUnauthorized -*/ -type TenantsUpdateUnauthorized struct { -} - -// NewTenantsUpdateUnauthorized creates TenantsUpdateUnauthorized with default headers values -func NewTenantsUpdateUnauthorized() *TenantsUpdateUnauthorized { - - return &TenantsUpdateUnauthorized{} -} - -// WriteResponse to the client -func (o *TenantsUpdateUnauthorized) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(401) -} - -// TenantsUpdateForbiddenCode is the HTTP code returned for type TenantsUpdateForbidden -const TenantsUpdateForbiddenCode int = 403 - -/* -TenantsUpdateForbidden Forbidden - -swagger:response tenantsUpdateForbidden -*/ -type TenantsUpdateForbidden struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewTenantsUpdateForbidden creates TenantsUpdateForbidden with default headers values -func NewTenantsUpdateForbidden() *TenantsUpdateForbidden { - - return &TenantsUpdateForbidden{} -} - -// WithPayload adds the payload to the tenants update forbidden response -func (o *TenantsUpdateForbidden) WithPayload(payload *models.ErrorResponse) *TenantsUpdateForbidden { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the tenants update forbidden response -func (o *TenantsUpdateForbidden) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *TenantsUpdateForbidden) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(403) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// TenantsUpdateUnprocessableEntityCode is the HTTP code returned for type TenantsUpdateUnprocessableEntity -const TenantsUpdateUnprocessableEntityCode int = 422 - -/* -TenantsUpdateUnprocessableEntity Invalid Tenant class - -swagger:response tenantsUpdateUnprocessableEntity -*/ -type TenantsUpdateUnprocessableEntity struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewTenantsUpdateUnprocessableEntity creates TenantsUpdateUnprocessableEntity with default headers values -func NewTenantsUpdateUnprocessableEntity() *TenantsUpdateUnprocessableEntity { - - return &TenantsUpdateUnprocessableEntity{} -} - -// WithPayload adds the payload to the tenants update unprocessable entity response -func (o *TenantsUpdateUnprocessableEntity) WithPayload(payload *models.ErrorResponse) *TenantsUpdateUnprocessableEntity { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the tenants update unprocessable entity response -func (o *TenantsUpdateUnprocessableEntity) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *TenantsUpdateUnprocessableEntity) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(422) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// TenantsUpdateInternalServerErrorCode is the HTTP code returned for type TenantsUpdateInternalServerError -const TenantsUpdateInternalServerErrorCode int = 500 - -/* -TenantsUpdateInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response tenantsUpdateInternalServerError -*/ -type TenantsUpdateInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewTenantsUpdateInternalServerError creates TenantsUpdateInternalServerError with default headers values -func NewTenantsUpdateInternalServerError() *TenantsUpdateInternalServerError { - - return &TenantsUpdateInternalServerError{} -} - -// WithPayload adds the payload to the tenants update internal server error response -func (o *TenantsUpdateInternalServerError) WithPayload(payload *models.ErrorResponse) *TenantsUpdateInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the tenants update internal server error response -func (o *TenantsUpdateInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *TenantsUpdateInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/schema/tenants_update_urlbuilder.go b/adapters/handlers/rest/operations/schema/tenants_update_urlbuilder.go deleted file mode 100644 index 769cb075cd6a9d0bae732017aa8a34aee1e36af2..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/schema/tenants_update_urlbuilder.go +++ /dev/null @@ -1,110 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" - "strings" -) - -// TenantsUpdateURL generates an URL for the tenants update operation -type TenantsUpdateURL struct { - ClassName string - - _basePath string - // avoid unkeyed usage - _ struct{} -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *TenantsUpdateURL) WithBasePath(bp string) *TenantsUpdateURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *TenantsUpdateURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *TenantsUpdateURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/schema/{className}/tenants" - - className := o.ClassName - if className != "" { - _path = strings.Replace(_path, "{className}", className, -1) - } else { - return nil, errors.New("className is required on TenantsUpdateURL") - } - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *TenantsUpdateURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *TenantsUpdateURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *TenantsUpdateURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on TenantsUpdateURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on TenantsUpdateURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *TenantsUpdateURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/weaviate_api.go b/adapters/handlers/rest/operations/weaviate_api.go deleted file mode 100644 index 3eaf97559df99f723003e8a41359c54c3090490c..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/weaviate_api.go +++ /dev/null @@ -1,948 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package operations - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "net/http" - "strings" - - "github.com/go-openapi/errors" - "github.com/go-openapi/loads" - "github.com/go-openapi/runtime" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/runtime/security" - "github.com/go-openapi/runtime/yamlpc" - "github.com/go-openapi/spec" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" - - "github.com/weaviate/weaviate/adapters/handlers/rest/operations/backups" - "github.com/weaviate/weaviate/adapters/handlers/rest/operations/batch" - "github.com/weaviate/weaviate/adapters/handlers/rest/operations/classifications" - "github.com/weaviate/weaviate/adapters/handlers/rest/operations/graphql" - "github.com/weaviate/weaviate/adapters/handlers/rest/operations/meta" - "github.com/weaviate/weaviate/adapters/handlers/rest/operations/nodes" - "github.com/weaviate/weaviate/adapters/handlers/rest/operations/objects" - "github.com/weaviate/weaviate/adapters/handlers/rest/operations/schema" - "github.com/weaviate/weaviate/adapters/handlers/rest/operations/well_known" - "github.com/weaviate/weaviate/entities/models" -) - -// NewWeaviateAPI creates a new Weaviate instance -func NewWeaviateAPI(spec *loads.Document) *WeaviateAPI { - return &WeaviateAPI{ - handlers: make(map[string]map[string]http.Handler), - formats: strfmt.Default, - defaultConsumes: "application/json", - defaultProduces: "application/json", - customConsumers: make(map[string]runtime.Consumer), - customProducers: make(map[string]runtime.Producer), - PreServerShutdown: func() {}, - ServerShutdown: func() {}, - spec: spec, - useSwaggerUI: false, - ServeError: errors.ServeError, - BasicAuthenticator: security.BasicAuth, - APIKeyAuthenticator: security.APIKeyAuth, - BearerAuthenticator: security.BearerAuth, - - JSONConsumer: runtime.JSONConsumer(), - YamlConsumer: yamlpc.YAMLConsumer(), - - JSONProducer: runtime.JSONProducer(), - - WellKnownGetWellKnownOpenidConfigurationHandler: well_known.GetWellKnownOpenidConfigurationHandlerFunc(func(params well_known.GetWellKnownOpenidConfigurationParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation well_known.GetWellKnownOpenidConfiguration has not yet been implemented") - }), - BackupsBackupsCreateHandler: backups.BackupsCreateHandlerFunc(func(params backups.BackupsCreateParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation backups.BackupsCreate has not yet been implemented") - }), - BackupsBackupsCreateStatusHandler: backups.BackupsCreateStatusHandlerFunc(func(params backups.BackupsCreateStatusParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation backups.BackupsCreateStatus has not yet been implemented") - }), - BackupsBackupsRestoreHandler: backups.BackupsRestoreHandlerFunc(func(params backups.BackupsRestoreParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation backups.BackupsRestore has not yet been implemented") - }), - BackupsBackupsRestoreStatusHandler: backups.BackupsRestoreStatusHandlerFunc(func(params backups.BackupsRestoreStatusParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation backups.BackupsRestoreStatus has not yet been implemented") - }), - BatchBatchObjectsCreateHandler: batch.BatchObjectsCreateHandlerFunc(func(params batch.BatchObjectsCreateParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation batch.BatchObjectsCreate has not yet been implemented") - }), - BatchBatchObjectsDeleteHandler: batch.BatchObjectsDeleteHandlerFunc(func(params batch.BatchObjectsDeleteParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation batch.BatchObjectsDelete has not yet been implemented") - }), - BatchBatchReferencesCreateHandler: batch.BatchReferencesCreateHandlerFunc(func(params batch.BatchReferencesCreateParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation batch.BatchReferencesCreate has not yet been implemented") - }), - ClassificationsClassificationsGetHandler: classifications.ClassificationsGetHandlerFunc(func(params classifications.ClassificationsGetParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation classifications.ClassificationsGet has not yet been implemented") - }), - ClassificationsClassificationsPostHandler: classifications.ClassificationsPostHandlerFunc(func(params classifications.ClassificationsPostParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation classifications.ClassificationsPost has not yet been implemented") - }), - GraphqlGraphqlBatchHandler: graphql.GraphqlBatchHandlerFunc(func(params graphql.GraphqlBatchParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation graphql.GraphqlBatch has not yet been implemented") - }), - GraphqlGraphqlPostHandler: graphql.GraphqlPostHandlerFunc(func(params graphql.GraphqlPostParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation graphql.GraphqlPost has not yet been implemented") - }), - MetaMetaGetHandler: meta.MetaGetHandlerFunc(func(params meta.MetaGetParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation meta.MetaGet has not yet been implemented") - }), - NodesNodesGetHandler: nodes.NodesGetHandlerFunc(func(params nodes.NodesGetParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation nodes.NodesGet has not yet been implemented") - }), - NodesNodesGetClassHandler: nodes.NodesGetClassHandlerFunc(func(params nodes.NodesGetClassParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation nodes.NodesGetClass has not yet been implemented") - }), - ObjectsObjectsClassDeleteHandler: objects.ObjectsClassDeleteHandlerFunc(func(params objects.ObjectsClassDeleteParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation objects.ObjectsClassDelete has not yet been implemented") - }), - ObjectsObjectsClassGetHandler: objects.ObjectsClassGetHandlerFunc(func(params objects.ObjectsClassGetParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation objects.ObjectsClassGet has not yet been implemented") - }), - ObjectsObjectsClassHeadHandler: objects.ObjectsClassHeadHandlerFunc(func(params objects.ObjectsClassHeadParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation objects.ObjectsClassHead has not yet been implemented") - }), - ObjectsObjectsClassPatchHandler: objects.ObjectsClassPatchHandlerFunc(func(params objects.ObjectsClassPatchParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation objects.ObjectsClassPatch has not yet been implemented") - }), - ObjectsObjectsClassPutHandler: objects.ObjectsClassPutHandlerFunc(func(params objects.ObjectsClassPutParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation objects.ObjectsClassPut has not yet been implemented") - }), - ObjectsObjectsClassReferencesCreateHandler: objects.ObjectsClassReferencesCreateHandlerFunc(func(params objects.ObjectsClassReferencesCreateParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation objects.ObjectsClassReferencesCreate has not yet been implemented") - }), - ObjectsObjectsClassReferencesDeleteHandler: objects.ObjectsClassReferencesDeleteHandlerFunc(func(params objects.ObjectsClassReferencesDeleteParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation objects.ObjectsClassReferencesDelete has not yet been implemented") - }), - ObjectsObjectsClassReferencesPutHandler: objects.ObjectsClassReferencesPutHandlerFunc(func(params objects.ObjectsClassReferencesPutParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation objects.ObjectsClassReferencesPut has not yet been implemented") - }), - ObjectsObjectsCreateHandler: objects.ObjectsCreateHandlerFunc(func(params objects.ObjectsCreateParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation objects.ObjectsCreate has not yet been implemented") - }), - ObjectsObjectsDeleteHandler: objects.ObjectsDeleteHandlerFunc(func(params objects.ObjectsDeleteParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation objects.ObjectsDelete has not yet been implemented") - }), - ObjectsObjectsGetHandler: objects.ObjectsGetHandlerFunc(func(params objects.ObjectsGetParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation objects.ObjectsGet has not yet been implemented") - }), - ObjectsObjectsHeadHandler: objects.ObjectsHeadHandlerFunc(func(params objects.ObjectsHeadParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation objects.ObjectsHead has not yet been implemented") - }), - ObjectsObjectsListHandler: objects.ObjectsListHandlerFunc(func(params objects.ObjectsListParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation objects.ObjectsList has not yet been implemented") - }), - ObjectsObjectsPatchHandler: objects.ObjectsPatchHandlerFunc(func(params objects.ObjectsPatchParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation objects.ObjectsPatch has not yet been implemented") - }), - ObjectsObjectsReferencesCreateHandler: objects.ObjectsReferencesCreateHandlerFunc(func(params objects.ObjectsReferencesCreateParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation objects.ObjectsReferencesCreate has not yet been implemented") - }), - ObjectsObjectsReferencesDeleteHandler: objects.ObjectsReferencesDeleteHandlerFunc(func(params objects.ObjectsReferencesDeleteParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation objects.ObjectsReferencesDelete has not yet been implemented") - }), - ObjectsObjectsReferencesUpdateHandler: objects.ObjectsReferencesUpdateHandlerFunc(func(params objects.ObjectsReferencesUpdateParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation objects.ObjectsReferencesUpdate has not yet been implemented") - }), - ObjectsObjectsUpdateHandler: objects.ObjectsUpdateHandlerFunc(func(params objects.ObjectsUpdateParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation objects.ObjectsUpdate has not yet been implemented") - }), - ObjectsObjectsValidateHandler: objects.ObjectsValidateHandlerFunc(func(params objects.ObjectsValidateParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation objects.ObjectsValidate has not yet been implemented") - }), - SchemaSchemaClusterStatusHandler: schema.SchemaClusterStatusHandlerFunc(func(params schema.SchemaClusterStatusParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation schema.SchemaClusterStatus has not yet been implemented") - }), - SchemaSchemaDumpHandler: schema.SchemaDumpHandlerFunc(func(params schema.SchemaDumpParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation schema.SchemaDump has not yet been implemented") - }), - SchemaSchemaObjectsCreateHandler: schema.SchemaObjectsCreateHandlerFunc(func(params schema.SchemaObjectsCreateParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation schema.SchemaObjectsCreate has not yet been implemented") - }), - SchemaSchemaObjectsDeleteHandler: schema.SchemaObjectsDeleteHandlerFunc(func(params schema.SchemaObjectsDeleteParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation schema.SchemaObjectsDelete has not yet been implemented") - }), - SchemaSchemaObjectsGetHandler: schema.SchemaObjectsGetHandlerFunc(func(params schema.SchemaObjectsGetParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation schema.SchemaObjectsGet has not yet been implemented") - }), - SchemaSchemaObjectsPropertiesAddHandler: schema.SchemaObjectsPropertiesAddHandlerFunc(func(params schema.SchemaObjectsPropertiesAddParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation schema.SchemaObjectsPropertiesAdd has not yet been implemented") - }), - SchemaSchemaObjectsShardsGetHandler: schema.SchemaObjectsShardsGetHandlerFunc(func(params schema.SchemaObjectsShardsGetParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation schema.SchemaObjectsShardsGet has not yet been implemented") - }), - SchemaSchemaObjectsShardsUpdateHandler: schema.SchemaObjectsShardsUpdateHandlerFunc(func(params schema.SchemaObjectsShardsUpdateParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation schema.SchemaObjectsShardsUpdate has not yet been implemented") - }), - SchemaSchemaObjectsUpdateHandler: schema.SchemaObjectsUpdateHandlerFunc(func(params schema.SchemaObjectsUpdateParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation schema.SchemaObjectsUpdate has not yet been implemented") - }), - SchemaTenantsCreateHandler: schema.TenantsCreateHandlerFunc(func(params schema.TenantsCreateParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation schema.TenantsCreate has not yet been implemented") - }), - SchemaTenantsDeleteHandler: schema.TenantsDeleteHandlerFunc(func(params schema.TenantsDeleteParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation schema.TenantsDelete has not yet been implemented") - }), - SchemaTenantsGetHandler: schema.TenantsGetHandlerFunc(func(params schema.TenantsGetParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation schema.TenantsGet has not yet been implemented") - }), - SchemaTenantsUpdateHandler: schema.TenantsUpdateHandlerFunc(func(params schema.TenantsUpdateParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation schema.TenantsUpdate has not yet been implemented") - }), - WeaviateRootHandler: WeaviateRootHandlerFunc(func(params WeaviateRootParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation WeaviateRoot has not yet been implemented") - }), - WeaviateWellknownLivenessHandler: WeaviateWellknownLivenessHandlerFunc(func(params WeaviateWellknownLivenessParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation WeaviateWellknownLiveness has not yet been implemented") - }), - WeaviateWellknownReadinessHandler: WeaviateWellknownReadinessHandlerFunc(func(params WeaviateWellknownReadinessParams, principal *models.Principal) middleware.Responder { - return middleware.NotImplemented("operation WeaviateWellknownReadiness has not yet been implemented") - }), - - OidcAuth: func(token string, scopes []string) (*models.Principal, error) { - return nil, errors.NotImplemented("oauth2 bearer auth (oidc) has not yet been implemented") - }, - // default authorizer is authorized meaning no requests are blocked - APIAuthorizer: security.Authorized(), - } -} - -/*WeaviateAPI Cloud-native, modular vector database */ -type WeaviateAPI struct { - spec *loads.Document - context *middleware.Context - handlers map[string]map[string]http.Handler - formats strfmt.Registry - customConsumers map[string]runtime.Consumer - customProducers map[string]runtime.Producer - defaultConsumes string - defaultProduces string - Middleware func(middleware.Builder) http.Handler - useSwaggerUI bool - - // BasicAuthenticator generates a runtime.Authenticator from the supplied basic auth function. - // It has a default implementation in the security package, however you can replace it for your particular usage. - BasicAuthenticator func(security.UserPassAuthentication) runtime.Authenticator - - // APIKeyAuthenticator generates a runtime.Authenticator from the supplied token auth function. - // It has a default implementation in the security package, however you can replace it for your particular usage. - APIKeyAuthenticator func(string, string, security.TokenAuthentication) runtime.Authenticator - - // BearerAuthenticator generates a runtime.Authenticator from the supplied bearer token auth function. - // It has a default implementation in the security package, however you can replace it for your particular usage. - BearerAuthenticator func(string, security.ScopedTokenAuthentication) runtime.Authenticator - - // JSONConsumer registers a consumer for the following mime types: - // - application/json - JSONConsumer runtime.Consumer - // YamlConsumer registers a consumer for the following mime types: - // - application/yaml - YamlConsumer runtime.Consumer - - // JSONProducer registers a producer for the following mime types: - // - application/json - JSONProducer runtime.Producer - - // OidcAuth registers a function that takes an access token and a collection of required scopes and returns a principal - // it performs authentication based on an oauth2 bearer token provided in the request - OidcAuth func(string, []string) (*models.Principal, error) - - // APIAuthorizer provides access control (ACL/RBAC/ABAC) by providing access to the request and authenticated principal - APIAuthorizer runtime.Authorizer - - // WellKnownGetWellKnownOpenidConfigurationHandler sets the operation handler for the get well known openid configuration operation - WellKnownGetWellKnownOpenidConfigurationHandler well_known.GetWellKnownOpenidConfigurationHandler - // BackupsBackupsCreateHandler sets the operation handler for the backups create operation - BackupsBackupsCreateHandler backups.BackupsCreateHandler - // BackupsBackupsCreateStatusHandler sets the operation handler for the backups create status operation - BackupsBackupsCreateStatusHandler backups.BackupsCreateStatusHandler - // BackupsBackupsRestoreHandler sets the operation handler for the backups restore operation - BackupsBackupsRestoreHandler backups.BackupsRestoreHandler - // BackupsBackupsRestoreStatusHandler sets the operation handler for the backups restore status operation - BackupsBackupsRestoreStatusHandler backups.BackupsRestoreStatusHandler - // BatchBatchObjectsCreateHandler sets the operation handler for the batch objects create operation - BatchBatchObjectsCreateHandler batch.BatchObjectsCreateHandler - // BatchBatchObjectsDeleteHandler sets the operation handler for the batch objects delete operation - BatchBatchObjectsDeleteHandler batch.BatchObjectsDeleteHandler - // BatchBatchReferencesCreateHandler sets the operation handler for the batch references create operation - BatchBatchReferencesCreateHandler batch.BatchReferencesCreateHandler - // ClassificationsClassificationsGetHandler sets the operation handler for the classifications get operation - ClassificationsClassificationsGetHandler classifications.ClassificationsGetHandler - // ClassificationsClassificationsPostHandler sets the operation handler for the classifications post operation - ClassificationsClassificationsPostHandler classifications.ClassificationsPostHandler - // GraphqlGraphqlBatchHandler sets the operation handler for the graphql batch operation - GraphqlGraphqlBatchHandler graphql.GraphqlBatchHandler - // GraphqlGraphqlPostHandler sets the operation handler for the graphql post operation - GraphqlGraphqlPostHandler graphql.GraphqlPostHandler - // MetaMetaGetHandler sets the operation handler for the meta get operation - MetaMetaGetHandler meta.MetaGetHandler - // NodesNodesGetHandler sets the operation handler for the nodes get operation - NodesNodesGetHandler nodes.NodesGetHandler - // NodesNodesGetClassHandler sets the operation handler for the nodes get class operation - NodesNodesGetClassHandler nodes.NodesGetClassHandler - // ObjectsObjectsClassDeleteHandler sets the operation handler for the objects class delete operation - ObjectsObjectsClassDeleteHandler objects.ObjectsClassDeleteHandler - // ObjectsObjectsClassGetHandler sets the operation handler for the objects class get operation - ObjectsObjectsClassGetHandler objects.ObjectsClassGetHandler - // ObjectsObjectsClassHeadHandler sets the operation handler for the objects class head operation - ObjectsObjectsClassHeadHandler objects.ObjectsClassHeadHandler - // ObjectsObjectsClassPatchHandler sets the operation handler for the objects class patch operation - ObjectsObjectsClassPatchHandler objects.ObjectsClassPatchHandler - // ObjectsObjectsClassPutHandler sets the operation handler for the objects class put operation - ObjectsObjectsClassPutHandler objects.ObjectsClassPutHandler - // ObjectsObjectsClassReferencesCreateHandler sets the operation handler for the objects class references create operation - ObjectsObjectsClassReferencesCreateHandler objects.ObjectsClassReferencesCreateHandler - // ObjectsObjectsClassReferencesDeleteHandler sets the operation handler for the objects class references delete operation - ObjectsObjectsClassReferencesDeleteHandler objects.ObjectsClassReferencesDeleteHandler - // ObjectsObjectsClassReferencesPutHandler sets the operation handler for the objects class references put operation - ObjectsObjectsClassReferencesPutHandler objects.ObjectsClassReferencesPutHandler - // ObjectsObjectsCreateHandler sets the operation handler for the objects create operation - ObjectsObjectsCreateHandler objects.ObjectsCreateHandler - // ObjectsObjectsDeleteHandler sets the operation handler for the objects delete operation - ObjectsObjectsDeleteHandler objects.ObjectsDeleteHandler - // ObjectsObjectsGetHandler sets the operation handler for the objects get operation - ObjectsObjectsGetHandler objects.ObjectsGetHandler - // ObjectsObjectsHeadHandler sets the operation handler for the objects head operation - ObjectsObjectsHeadHandler objects.ObjectsHeadHandler - // ObjectsObjectsListHandler sets the operation handler for the objects list operation - ObjectsObjectsListHandler objects.ObjectsListHandler - // ObjectsObjectsPatchHandler sets the operation handler for the objects patch operation - ObjectsObjectsPatchHandler objects.ObjectsPatchHandler - // ObjectsObjectsReferencesCreateHandler sets the operation handler for the objects references create operation - ObjectsObjectsReferencesCreateHandler objects.ObjectsReferencesCreateHandler - // ObjectsObjectsReferencesDeleteHandler sets the operation handler for the objects references delete operation - ObjectsObjectsReferencesDeleteHandler objects.ObjectsReferencesDeleteHandler - // ObjectsObjectsReferencesUpdateHandler sets the operation handler for the objects references update operation - ObjectsObjectsReferencesUpdateHandler objects.ObjectsReferencesUpdateHandler - // ObjectsObjectsUpdateHandler sets the operation handler for the objects update operation - ObjectsObjectsUpdateHandler objects.ObjectsUpdateHandler - // ObjectsObjectsValidateHandler sets the operation handler for the objects validate operation - ObjectsObjectsValidateHandler objects.ObjectsValidateHandler - // SchemaSchemaClusterStatusHandler sets the operation handler for the schema cluster status operation - SchemaSchemaClusterStatusHandler schema.SchemaClusterStatusHandler - // SchemaSchemaDumpHandler sets the operation handler for the schema dump operation - SchemaSchemaDumpHandler schema.SchemaDumpHandler - // SchemaSchemaObjectsCreateHandler sets the operation handler for the schema objects create operation - SchemaSchemaObjectsCreateHandler schema.SchemaObjectsCreateHandler - // SchemaSchemaObjectsDeleteHandler sets the operation handler for the schema objects delete operation - SchemaSchemaObjectsDeleteHandler schema.SchemaObjectsDeleteHandler - // SchemaSchemaObjectsGetHandler sets the operation handler for the schema objects get operation - SchemaSchemaObjectsGetHandler schema.SchemaObjectsGetHandler - // SchemaSchemaObjectsPropertiesAddHandler sets the operation handler for the schema objects properties add operation - SchemaSchemaObjectsPropertiesAddHandler schema.SchemaObjectsPropertiesAddHandler - // SchemaSchemaObjectsShardsGetHandler sets the operation handler for the schema objects shards get operation - SchemaSchemaObjectsShardsGetHandler schema.SchemaObjectsShardsGetHandler - // SchemaSchemaObjectsShardsUpdateHandler sets the operation handler for the schema objects shards update operation - SchemaSchemaObjectsShardsUpdateHandler schema.SchemaObjectsShardsUpdateHandler - // SchemaSchemaObjectsUpdateHandler sets the operation handler for the schema objects update operation - SchemaSchemaObjectsUpdateHandler schema.SchemaObjectsUpdateHandler - // SchemaTenantsCreateHandler sets the operation handler for the tenants create operation - SchemaTenantsCreateHandler schema.TenantsCreateHandler - // SchemaTenantsDeleteHandler sets the operation handler for the tenants delete operation - SchemaTenantsDeleteHandler schema.TenantsDeleteHandler - // SchemaTenantsGetHandler sets the operation handler for the tenants get operation - SchemaTenantsGetHandler schema.TenantsGetHandler - // SchemaTenantsUpdateHandler sets the operation handler for the tenants update operation - SchemaTenantsUpdateHandler schema.TenantsUpdateHandler - // WeaviateRootHandler sets the operation handler for the weaviate root operation - WeaviateRootHandler WeaviateRootHandler - // WeaviateWellknownLivenessHandler sets the operation handler for the weaviate wellknown liveness operation - WeaviateWellknownLivenessHandler WeaviateWellknownLivenessHandler - // WeaviateWellknownReadinessHandler sets the operation handler for the weaviate wellknown readiness operation - WeaviateWellknownReadinessHandler WeaviateWellknownReadinessHandler - - // ServeError is called when an error is received, there is a default handler - // but you can set your own with this - ServeError func(http.ResponseWriter, *http.Request, error) - - // PreServerShutdown is called before the HTTP(S) server is shutdown - // This allows for custom functions to get executed before the HTTP(S) server stops accepting traffic - PreServerShutdown func() - - // ServerShutdown is called when the HTTP(S) server is shut down and done - // handling all active connections and does not accept connections any more - ServerShutdown func() - - // Custom command line argument groups with their descriptions - CommandLineOptionsGroups []swag.CommandLineOptionsGroup - - // User defined logger function. - Logger func(string, ...interface{}) -} - -// UseRedoc for documentation at /docs -func (o *WeaviateAPI) UseRedoc() { - o.useSwaggerUI = false -} - -// UseSwaggerUI for documentation at /docs -func (o *WeaviateAPI) UseSwaggerUI() { - o.useSwaggerUI = true -} - -// SetDefaultProduces sets the default produces media type -func (o *WeaviateAPI) SetDefaultProduces(mediaType string) { - o.defaultProduces = mediaType -} - -// SetDefaultConsumes returns the default consumes media type -func (o *WeaviateAPI) SetDefaultConsumes(mediaType string) { - o.defaultConsumes = mediaType -} - -// SetSpec sets a spec that will be served for the clients. -func (o *WeaviateAPI) SetSpec(spec *loads.Document) { - o.spec = spec -} - -// DefaultProduces returns the default produces media type -func (o *WeaviateAPI) DefaultProduces() string { - return o.defaultProduces -} - -// DefaultConsumes returns the default consumes media type -func (o *WeaviateAPI) DefaultConsumes() string { - return o.defaultConsumes -} - -// Formats returns the registered string formats -func (o *WeaviateAPI) Formats() strfmt.Registry { - return o.formats -} - -// RegisterFormat registers a custom format validator -func (o *WeaviateAPI) RegisterFormat(name string, format strfmt.Format, validator strfmt.Validator) { - o.formats.Add(name, format, validator) -} - -// Validate validates the registrations in the WeaviateAPI -func (o *WeaviateAPI) Validate() error { - var unregistered []string - - if o.JSONConsumer == nil { - unregistered = append(unregistered, "JSONConsumer") - } - if o.YamlConsumer == nil { - unregistered = append(unregistered, "YamlConsumer") - } - - if o.JSONProducer == nil { - unregistered = append(unregistered, "JSONProducer") - } - - if o.OidcAuth == nil { - unregistered = append(unregistered, "OidcAuth") - } - - if o.WellKnownGetWellKnownOpenidConfigurationHandler == nil { - unregistered = append(unregistered, "well_known.GetWellKnownOpenidConfigurationHandler") - } - if o.BackupsBackupsCreateHandler == nil { - unregistered = append(unregistered, "backups.BackupsCreateHandler") - } - if o.BackupsBackupsCreateStatusHandler == nil { - unregistered = append(unregistered, "backups.BackupsCreateStatusHandler") - } - if o.BackupsBackupsRestoreHandler == nil { - unregistered = append(unregistered, "backups.BackupsRestoreHandler") - } - if o.BackupsBackupsRestoreStatusHandler == nil { - unregistered = append(unregistered, "backups.BackupsRestoreStatusHandler") - } - if o.BatchBatchObjectsCreateHandler == nil { - unregistered = append(unregistered, "batch.BatchObjectsCreateHandler") - } - if o.BatchBatchObjectsDeleteHandler == nil { - unregistered = append(unregistered, "batch.BatchObjectsDeleteHandler") - } - if o.BatchBatchReferencesCreateHandler == nil { - unregistered = append(unregistered, "batch.BatchReferencesCreateHandler") - } - if o.ClassificationsClassificationsGetHandler == nil { - unregistered = append(unregistered, "classifications.ClassificationsGetHandler") - } - if o.ClassificationsClassificationsPostHandler == nil { - unregistered = append(unregistered, "classifications.ClassificationsPostHandler") - } - if o.GraphqlGraphqlBatchHandler == nil { - unregistered = append(unregistered, "graphql.GraphqlBatchHandler") - } - if o.GraphqlGraphqlPostHandler == nil { - unregistered = append(unregistered, "graphql.GraphqlPostHandler") - } - if o.MetaMetaGetHandler == nil { - unregistered = append(unregistered, "meta.MetaGetHandler") - } - if o.NodesNodesGetHandler == nil { - unregistered = append(unregistered, "nodes.NodesGetHandler") - } - if o.NodesNodesGetClassHandler == nil { - unregistered = append(unregistered, "nodes.NodesGetClassHandler") - } - if o.ObjectsObjectsClassDeleteHandler == nil { - unregistered = append(unregistered, "objects.ObjectsClassDeleteHandler") - } - if o.ObjectsObjectsClassGetHandler == nil { - unregistered = append(unregistered, "objects.ObjectsClassGetHandler") - } - if o.ObjectsObjectsClassHeadHandler == nil { - unregistered = append(unregistered, "objects.ObjectsClassHeadHandler") - } - if o.ObjectsObjectsClassPatchHandler == nil { - unregistered = append(unregistered, "objects.ObjectsClassPatchHandler") - } - if o.ObjectsObjectsClassPutHandler == nil { - unregistered = append(unregistered, "objects.ObjectsClassPutHandler") - } - if o.ObjectsObjectsClassReferencesCreateHandler == nil { - unregistered = append(unregistered, "objects.ObjectsClassReferencesCreateHandler") - } - if o.ObjectsObjectsClassReferencesDeleteHandler == nil { - unregistered = append(unregistered, "objects.ObjectsClassReferencesDeleteHandler") - } - if o.ObjectsObjectsClassReferencesPutHandler == nil { - unregistered = append(unregistered, "objects.ObjectsClassReferencesPutHandler") - } - if o.ObjectsObjectsCreateHandler == nil { - unregistered = append(unregistered, "objects.ObjectsCreateHandler") - } - if o.ObjectsObjectsDeleteHandler == nil { - unregistered = append(unregistered, "objects.ObjectsDeleteHandler") - } - if o.ObjectsObjectsGetHandler == nil { - unregistered = append(unregistered, "objects.ObjectsGetHandler") - } - if o.ObjectsObjectsHeadHandler == nil { - unregistered = append(unregistered, "objects.ObjectsHeadHandler") - } - if o.ObjectsObjectsListHandler == nil { - unregistered = append(unregistered, "objects.ObjectsListHandler") - } - if o.ObjectsObjectsPatchHandler == nil { - unregistered = append(unregistered, "objects.ObjectsPatchHandler") - } - if o.ObjectsObjectsReferencesCreateHandler == nil { - unregistered = append(unregistered, "objects.ObjectsReferencesCreateHandler") - } - if o.ObjectsObjectsReferencesDeleteHandler == nil { - unregistered = append(unregistered, "objects.ObjectsReferencesDeleteHandler") - } - if o.ObjectsObjectsReferencesUpdateHandler == nil { - unregistered = append(unregistered, "objects.ObjectsReferencesUpdateHandler") - } - if o.ObjectsObjectsUpdateHandler == nil { - unregistered = append(unregistered, "objects.ObjectsUpdateHandler") - } - if o.ObjectsObjectsValidateHandler == nil { - unregistered = append(unregistered, "objects.ObjectsValidateHandler") - } - if o.SchemaSchemaClusterStatusHandler == nil { - unregistered = append(unregistered, "schema.SchemaClusterStatusHandler") - } - if o.SchemaSchemaDumpHandler == nil { - unregistered = append(unregistered, "schema.SchemaDumpHandler") - } - if o.SchemaSchemaObjectsCreateHandler == nil { - unregistered = append(unregistered, "schema.SchemaObjectsCreateHandler") - } - if o.SchemaSchemaObjectsDeleteHandler == nil { - unregistered = append(unregistered, "schema.SchemaObjectsDeleteHandler") - } - if o.SchemaSchemaObjectsGetHandler == nil { - unregistered = append(unregistered, "schema.SchemaObjectsGetHandler") - } - if o.SchemaSchemaObjectsPropertiesAddHandler == nil { - unregistered = append(unregistered, "schema.SchemaObjectsPropertiesAddHandler") - } - if o.SchemaSchemaObjectsShardsGetHandler == nil { - unregistered = append(unregistered, "schema.SchemaObjectsShardsGetHandler") - } - if o.SchemaSchemaObjectsShardsUpdateHandler == nil { - unregistered = append(unregistered, "schema.SchemaObjectsShardsUpdateHandler") - } - if o.SchemaSchemaObjectsUpdateHandler == nil { - unregistered = append(unregistered, "schema.SchemaObjectsUpdateHandler") - } - if o.SchemaTenantsCreateHandler == nil { - unregistered = append(unregistered, "schema.TenantsCreateHandler") - } - if o.SchemaTenantsDeleteHandler == nil { - unregistered = append(unregistered, "schema.TenantsDeleteHandler") - } - if o.SchemaTenantsGetHandler == nil { - unregistered = append(unregistered, "schema.TenantsGetHandler") - } - if o.SchemaTenantsUpdateHandler == nil { - unregistered = append(unregistered, "schema.TenantsUpdateHandler") - } - if o.WeaviateRootHandler == nil { - unregistered = append(unregistered, "WeaviateRootHandler") - } - if o.WeaviateWellknownLivenessHandler == nil { - unregistered = append(unregistered, "WeaviateWellknownLivenessHandler") - } - if o.WeaviateWellknownReadinessHandler == nil { - unregistered = append(unregistered, "WeaviateWellknownReadinessHandler") - } - - if len(unregistered) > 0 { - return fmt.Errorf("missing registration: %s", strings.Join(unregistered, ", ")) - } - - return nil -} - -// ServeErrorFor gets a error handler for a given operation id -func (o *WeaviateAPI) ServeErrorFor(operationID string) func(http.ResponseWriter, *http.Request, error) { - return o.ServeError -} - -// AuthenticatorsFor gets the authenticators for the specified security schemes -func (o *WeaviateAPI) AuthenticatorsFor(schemes map[string]spec.SecurityScheme) map[string]runtime.Authenticator { - result := make(map[string]runtime.Authenticator) - for name := range schemes { - switch name { - case "oidc": - result[name] = o.BearerAuthenticator(name, func(token string, scopes []string) (interface{}, error) { - return o.OidcAuth(token, scopes) - }) - - } - } - return result -} - -// Authorizer returns the registered authorizer -func (o *WeaviateAPI) Authorizer() runtime.Authorizer { - return o.APIAuthorizer -} - -// ConsumersFor gets the consumers for the specified media types. -// MIME type parameters are ignored here. -func (o *WeaviateAPI) ConsumersFor(mediaTypes []string) map[string]runtime.Consumer { - result := make(map[string]runtime.Consumer, len(mediaTypes)) - for _, mt := range mediaTypes { - switch mt { - case "application/json": - result["application/json"] = o.JSONConsumer - case "application/yaml": - result["application/yaml"] = o.YamlConsumer - } - - if c, ok := o.customConsumers[mt]; ok { - result[mt] = c - } - } - return result -} - -// ProducersFor gets the producers for the specified media types. -// MIME type parameters are ignored here. -func (o *WeaviateAPI) ProducersFor(mediaTypes []string) map[string]runtime.Producer { - result := make(map[string]runtime.Producer, len(mediaTypes)) - for _, mt := range mediaTypes { - switch mt { - case "application/json": - result["application/json"] = o.JSONProducer - } - - if p, ok := o.customProducers[mt]; ok { - result[mt] = p - } - } - return result -} - -// HandlerFor gets a http.Handler for the provided operation method and path -func (o *WeaviateAPI) HandlerFor(method, path string) (http.Handler, bool) { - if o.handlers == nil { - return nil, false - } - um := strings.ToUpper(method) - if _, ok := o.handlers[um]; !ok { - return nil, false - } - if path == "/" { - path = "" - } - h, ok := o.handlers[um][path] - return h, ok -} - -// Context returns the middleware context for the weaviate API -func (o *WeaviateAPI) Context() *middleware.Context { - if o.context == nil { - o.context = middleware.NewRoutableContext(o.spec, o, nil) - } - - return o.context -} - -func (o *WeaviateAPI) initHandlerCache() { - o.Context() // don't care about the result, just that the initialization happened - if o.handlers == nil { - o.handlers = make(map[string]map[string]http.Handler) - } - - if o.handlers["GET"] == nil { - o.handlers["GET"] = make(map[string]http.Handler) - } - o.handlers["GET"]["/.well-known/openid-configuration"] = well_known.NewGetWellKnownOpenidConfiguration(o.context, o.WellKnownGetWellKnownOpenidConfigurationHandler) - if o.handlers["POST"] == nil { - o.handlers["POST"] = make(map[string]http.Handler) - } - o.handlers["POST"]["/backups/{backend}"] = backups.NewBackupsCreate(o.context, o.BackupsBackupsCreateHandler) - if o.handlers["GET"] == nil { - o.handlers["GET"] = make(map[string]http.Handler) - } - o.handlers["GET"]["/backups/{backend}/{id}"] = backups.NewBackupsCreateStatus(o.context, o.BackupsBackupsCreateStatusHandler) - if o.handlers["POST"] == nil { - o.handlers["POST"] = make(map[string]http.Handler) - } - o.handlers["POST"]["/backups/{backend}/{id}/restore"] = backups.NewBackupsRestore(o.context, o.BackupsBackupsRestoreHandler) - if o.handlers["GET"] == nil { - o.handlers["GET"] = make(map[string]http.Handler) - } - o.handlers["GET"]["/backups/{backend}/{id}/restore"] = backups.NewBackupsRestoreStatus(o.context, o.BackupsBackupsRestoreStatusHandler) - if o.handlers["POST"] == nil { - o.handlers["POST"] = make(map[string]http.Handler) - } - o.handlers["POST"]["/batch/objects"] = batch.NewBatchObjectsCreate(o.context, o.BatchBatchObjectsCreateHandler) - if o.handlers["DELETE"] == nil { - o.handlers["DELETE"] = make(map[string]http.Handler) - } - o.handlers["DELETE"]["/batch/objects"] = batch.NewBatchObjectsDelete(o.context, o.BatchBatchObjectsDeleteHandler) - if o.handlers["POST"] == nil { - o.handlers["POST"] = make(map[string]http.Handler) - } - o.handlers["POST"]["/batch/references"] = batch.NewBatchReferencesCreate(o.context, o.BatchBatchReferencesCreateHandler) - if o.handlers["GET"] == nil { - o.handlers["GET"] = make(map[string]http.Handler) - } - o.handlers["GET"]["/classifications/{id}"] = classifications.NewClassificationsGet(o.context, o.ClassificationsClassificationsGetHandler) - if o.handlers["POST"] == nil { - o.handlers["POST"] = make(map[string]http.Handler) - } - o.handlers["POST"]["/classifications"] = classifications.NewClassificationsPost(o.context, o.ClassificationsClassificationsPostHandler) - if o.handlers["POST"] == nil { - o.handlers["POST"] = make(map[string]http.Handler) - } - o.handlers["POST"]["/graphql/batch"] = graphql.NewGraphqlBatch(o.context, o.GraphqlGraphqlBatchHandler) - if o.handlers["POST"] == nil { - o.handlers["POST"] = make(map[string]http.Handler) - } - o.handlers["POST"]["/graphql"] = graphql.NewGraphqlPost(o.context, o.GraphqlGraphqlPostHandler) - if o.handlers["GET"] == nil { - o.handlers["GET"] = make(map[string]http.Handler) - } - o.handlers["GET"]["/meta"] = meta.NewMetaGet(o.context, o.MetaMetaGetHandler) - if o.handlers["GET"] == nil { - o.handlers["GET"] = make(map[string]http.Handler) - } - o.handlers["GET"]["/nodes"] = nodes.NewNodesGet(o.context, o.NodesNodesGetHandler) - if o.handlers["GET"] == nil { - o.handlers["GET"] = make(map[string]http.Handler) - } - o.handlers["GET"]["/nodes/{className}"] = nodes.NewNodesGetClass(o.context, o.NodesNodesGetClassHandler) - if o.handlers["DELETE"] == nil { - o.handlers["DELETE"] = make(map[string]http.Handler) - } - o.handlers["DELETE"]["/objects/{className}/{id}"] = objects.NewObjectsClassDelete(o.context, o.ObjectsObjectsClassDeleteHandler) - if o.handlers["GET"] == nil { - o.handlers["GET"] = make(map[string]http.Handler) - } - o.handlers["GET"]["/objects/{className}/{id}"] = objects.NewObjectsClassGet(o.context, o.ObjectsObjectsClassGetHandler) - if o.handlers["HEAD"] == nil { - o.handlers["HEAD"] = make(map[string]http.Handler) - } - o.handlers["HEAD"]["/objects/{className}/{id}"] = objects.NewObjectsClassHead(o.context, o.ObjectsObjectsClassHeadHandler) - if o.handlers["PATCH"] == nil { - o.handlers["PATCH"] = make(map[string]http.Handler) - } - o.handlers["PATCH"]["/objects/{className}/{id}"] = objects.NewObjectsClassPatch(o.context, o.ObjectsObjectsClassPatchHandler) - if o.handlers["PUT"] == nil { - o.handlers["PUT"] = make(map[string]http.Handler) - } - o.handlers["PUT"]["/objects/{className}/{id}"] = objects.NewObjectsClassPut(o.context, o.ObjectsObjectsClassPutHandler) - if o.handlers["POST"] == nil { - o.handlers["POST"] = make(map[string]http.Handler) - } - o.handlers["POST"]["/objects/{className}/{id}/references/{propertyName}"] = objects.NewObjectsClassReferencesCreate(o.context, o.ObjectsObjectsClassReferencesCreateHandler) - if o.handlers["DELETE"] == nil { - o.handlers["DELETE"] = make(map[string]http.Handler) - } - o.handlers["DELETE"]["/objects/{className}/{id}/references/{propertyName}"] = objects.NewObjectsClassReferencesDelete(o.context, o.ObjectsObjectsClassReferencesDeleteHandler) - if o.handlers["PUT"] == nil { - o.handlers["PUT"] = make(map[string]http.Handler) - } - o.handlers["PUT"]["/objects/{className}/{id}/references/{propertyName}"] = objects.NewObjectsClassReferencesPut(o.context, o.ObjectsObjectsClassReferencesPutHandler) - if o.handlers["POST"] == nil { - o.handlers["POST"] = make(map[string]http.Handler) - } - o.handlers["POST"]["/objects"] = objects.NewObjectsCreate(o.context, o.ObjectsObjectsCreateHandler) - if o.handlers["DELETE"] == nil { - o.handlers["DELETE"] = make(map[string]http.Handler) - } - o.handlers["DELETE"]["/objects/{id}"] = objects.NewObjectsDelete(o.context, o.ObjectsObjectsDeleteHandler) - if o.handlers["GET"] == nil { - o.handlers["GET"] = make(map[string]http.Handler) - } - o.handlers["GET"]["/objects/{id}"] = objects.NewObjectsGet(o.context, o.ObjectsObjectsGetHandler) - if o.handlers["HEAD"] == nil { - o.handlers["HEAD"] = make(map[string]http.Handler) - } - o.handlers["HEAD"]["/objects/{id}"] = objects.NewObjectsHead(o.context, o.ObjectsObjectsHeadHandler) - if o.handlers["GET"] == nil { - o.handlers["GET"] = make(map[string]http.Handler) - } - o.handlers["GET"]["/objects"] = objects.NewObjectsList(o.context, o.ObjectsObjectsListHandler) - if o.handlers["PATCH"] == nil { - o.handlers["PATCH"] = make(map[string]http.Handler) - } - o.handlers["PATCH"]["/objects/{id}"] = objects.NewObjectsPatch(o.context, o.ObjectsObjectsPatchHandler) - if o.handlers["POST"] == nil { - o.handlers["POST"] = make(map[string]http.Handler) - } - o.handlers["POST"]["/objects/{id}/references/{propertyName}"] = objects.NewObjectsReferencesCreate(o.context, o.ObjectsObjectsReferencesCreateHandler) - if o.handlers["DELETE"] == nil { - o.handlers["DELETE"] = make(map[string]http.Handler) - } - o.handlers["DELETE"]["/objects/{id}/references/{propertyName}"] = objects.NewObjectsReferencesDelete(o.context, o.ObjectsObjectsReferencesDeleteHandler) - if o.handlers["PUT"] == nil { - o.handlers["PUT"] = make(map[string]http.Handler) - } - o.handlers["PUT"]["/objects/{id}/references/{propertyName}"] = objects.NewObjectsReferencesUpdate(o.context, o.ObjectsObjectsReferencesUpdateHandler) - if o.handlers["PUT"] == nil { - o.handlers["PUT"] = make(map[string]http.Handler) - } - o.handlers["PUT"]["/objects/{id}"] = objects.NewObjectsUpdate(o.context, o.ObjectsObjectsUpdateHandler) - if o.handlers["POST"] == nil { - o.handlers["POST"] = make(map[string]http.Handler) - } - o.handlers["POST"]["/objects/validate"] = objects.NewObjectsValidate(o.context, o.ObjectsObjectsValidateHandler) - if o.handlers["GET"] == nil { - o.handlers["GET"] = make(map[string]http.Handler) - } - o.handlers["GET"]["/schema/cluster-status"] = schema.NewSchemaClusterStatus(o.context, o.SchemaSchemaClusterStatusHandler) - if o.handlers["GET"] == nil { - o.handlers["GET"] = make(map[string]http.Handler) - } - o.handlers["GET"]["/schema"] = schema.NewSchemaDump(o.context, o.SchemaSchemaDumpHandler) - if o.handlers["POST"] == nil { - o.handlers["POST"] = make(map[string]http.Handler) - } - o.handlers["POST"]["/schema"] = schema.NewSchemaObjectsCreate(o.context, o.SchemaSchemaObjectsCreateHandler) - if o.handlers["DELETE"] == nil { - o.handlers["DELETE"] = make(map[string]http.Handler) - } - o.handlers["DELETE"]["/schema/{className}"] = schema.NewSchemaObjectsDelete(o.context, o.SchemaSchemaObjectsDeleteHandler) - if o.handlers["GET"] == nil { - o.handlers["GET"] = make(map[string]http.Handler) - } - o.handlers["GET"]["/schema/{className}"] = schema.NewSchemaObjectsGet(o.context, o.SchemaSchemaObjectsGetHandler) - if o.handlers["POST"] == nil { - o.handlers["POST"] = make(map[string]http.Handler) - } - o.handlers["POST"]["/schema/{className}/properties"] = schema.NewSchemaObjectsPropertiesAdd(o.context, o.SchemaSchemaObjectsPropertiesAddHandler) - if o.handlers["GET"] == nil { - o.handlers["GET"] = make(map[string]http.Handler) - } - o.handlers["GET"]["/schema/{className}/shards"] = schema.NewSchemaObjectsShardsGet(o.context, o.SchemaSchemaObjectsShardsGetHandler) - if o.handlers["PUT"] == nil { - o.handlers["PUT"] = make(map[string]http.Handler) - } - o.handlers["PUT"]["/schema/{className}/shards/{shardName}"] = schema.NewSchemaObjectsShardsUpdate(o.context, o.SchemaSchemaObjectsShardsUpdateHandler) - if o.handlers["PUT"] == nil { - o.handlers["PUT"] = make(map[string]http.Handler) - } - o.handlers["PUT"]["/schema/{className}"] = schema.NewSchemaObjectsUpdate(o.context, o.SchemaSchemaObjectsUpdateHandler) - if o.handlers["POST"] == nil { - o.handlers["POST"] = make(map[string]http.Handler) - } - o.handlers["POST"]["/schema/{className}/tenants"] = schema.NewTenantsCreate(o.context, o.SchemaTenantsCreateHandler) - if o.handlers["DELETE"] == nil { - o.handlers["DELETE"] = make(map[string]http.Handler) - } - o.handlers["DELETE"]["/schema/{className}/tenants"] = schema.NewTenantsDelete(o.context, o.SchemaTenantsDeleteHandler) - if o.handlers["GET"] == nil { - o.handlers["GET"] = make(map[string]http.Handler) - } - o.handlers["GET"]["/schema/{className}/tenants"] = schema.NewTenantsGet(o.context, o.SchemaTenantsGetHandler) - if o.handlers["PUT"] == nil { - o.handlers["PUT"] = make(map[string]http.Handler) - } - o.handlers["PUT"]["/schema/{className}/tenants"] = schema.NewTenantsUpdate(o.context, o.SchemaTenantsUpdateHandler) - if o.handlers["GET"] == nil { - o.handlers["GET"] = make(map[string]http.Handler) - } - o.handlers["GET"][""] = NewWeaviateRoot(o.context, o.WeaviateRootHandler) - if o.handlers["GET"] == nil { - o.handlers["GET"] = make(map[string]http.Handler) - } - o.handlers["GET"]["/.well-known/live"] = NewWeaviateWellknownLiveness(o.context, o.WeaviateWellknownLivenessHandler) - if o.handlers["GET"] == nil { - o.handlers["GET"] = make(map[string]http.Handler) - } - o.handlers["GET"]["/.well-known/ready"] = NewWeaviateWellknownReadiness(o.context, o.WeaviateWellknownReadinessHandler) -} - -// Serve creates a http handler to serve the API over HTTP -// can be used directly in http.ListenAndServe(":8000", api.Serve(nil)) -func (o *WeaviateAPI) Serve(builder middleware.Builder) http.Handler { - o.Init() - - if o.Middleware != nil { - return o.Middleware(builder) - } - if o.useSwaggerUI { - return o.context.APIHandlerSwaggerUI(builder) - } - return o.context.APIHandler(builder) -} - -// Init allows you to just initialize the handler cache, you can then recompose the middleware as you see fit -func (o *WeaviateAPI) Init() { - if len(o.handlers) == 0 { - o.initHandlerCache() - } -} - -// RegisterConsumer allows you to add (or override) a consumer for a media type. -func (o *WeaviateAPI) RegisterConsumer(mediaType string, consumer runtime.Consumer) { - o.customConsumers[mediaType] = consumer -} - -// RegisterProducer allows you to add (or override) a producer for a media type. -func (o *WeaviateAPI) RegisterProducer(mediaType string, producer runtime.Producer) { - o.customProducers[mediaType] = producer -} - -// AddMiddlewareFor adds a http middleware to existing handler -func (o *WeaviateAPI) AddMiddlewareFor(method, path string, builder middleware.Builder) { - um := strings.ToUpper(method) - if path == "/" { - path = "" - } - o.Init() - if h, ok := o.handlers[um][path]; ok { - o.handlers[method][path] = builder(h) - } -} diff --git a/adapters/handlers/rest/operations/weaviate_root.go b/adapters/handlers/rest/operations/weaviate_root.go deleted file mode 100644 index 8f08ce82afeac965e9aa8563d72d5443e675ab76..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/weaviate_root.go +++ /dev/null @@ -1,188 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package operations - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "context" - "net/http" - "strconv" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" - - "github.com/weaviate/weaviate/entities/models" -) - -// WeaviateRootHandlerFunc turns a function with the right signature into a weaviate root handler -type WeaviateRootHandlerFunc func(WeaviateRootParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn WeaviateRootHandlerFunc) Handle(params WeaviateRootParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// WeaviateRootHandler interface for that can handle valid weaviate root params -type WeaviateRootHandler interface { - Handle(WeaviateRootParams, *models.Principal) middleware.Responder -} - -// NewWeaviateRoot creates a new http.Handler for the weaviate root operation -func NewWeaviateRoot(ctx *middleware.Context, handler WeaviateRootHandler) *WeaviateRoot { - return &WeaviateRoot{Context: ctx, Handler: handler} -} - -/* - WeaviateRoot swagger:route GET / weaviateRoot - -Home. Discover the REST API -*/ -type WeaviateRoot struct { - Context *middleware.Context - Handler WeaviateRootHandler -} - -func (o *WeaviateRoot) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewWeaviateRootParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} - -// WeaviateRootOKBody weaviate root o k body -// -// swagger:model WeaviateRootOKBody -type WeaviateRootOKBody struct { - - // links - Links []*models.Link `json:"links" yaml:"links"` -} - -// Validate validates this weaviate root o k body -func (o *WeaviateRootOKBody) Validate(formats strfmt.Registry) error { - var res []error - - if err := o.validateLinks(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (o *WeaviateRootOKBody) validateLinks(formats strfmt.Registry) error { - if swag.IsZero(o.Links) { // not required - return nil - } - - for i := 0; i < len(o.Links); i++ { - if swag.IsZero(o.Links[i]) { // not required - continue - } - - if o.Links[i] != nil { - if err := o.Links[i].Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("weaviateRootOK" + "." + "links" + "." + strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("weaviateRootOK" + "." + "links" + "." + strconv.Itoa(i)) - } - return err - } - } - - } - - return nil -} - -// ContextValidate validate this weaviate root o k body based on the context it is used -func (o *WeaviateRootOKBody) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - if err := o.contextValidateLinks(ctx, formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (o *WeaviateRootOKBody) contextValidateLinks(ctx context.Context, formats strfmt.Registry) error { - - for i := 0; i < len(o.Links); i++ { - - if o.Links[i] != nil { - if err := o.Links[i].ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("weaviateRootOK" + "." + "links" + "." + strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("weaviateRootOK" + "." + "links" + "." + strconv.Itoa(i)) - } - return err - } - } - - } - - return nil -} - -// MarshalBinary interface implementation -func (o *WeaviateRootOKBody) MarshalBinary() ([]byte, error) { - if o == nil { - return nil, nil - } - return swag.WriteJSON(o) -} - -// UnmarshalBinary interface implementation -func (o *WeaviateRootOKBody) UnmarshalBinary(b []byte) error { - var res WeaviateRootOKBody - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *o = res - return nil -} diff --git a/adapters/handlers/rest/operations/weaviate_root_parameters.go b/adapters/handlers/rest/operations/weaviate_root_parameters.go deleted file mode 100644 index 6b8f68bccf2238635e40f1310ce835f389ce7428..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/weaviate_root_parameters.go +++ /dev/null @@ -1,57 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package operations - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime/middleware" -) - -// NewWeaviateRootParams creates a new WeaviateRootParams object -// -// There are no default values defined in the spec. -func NewWeaviateRootParams() WeaviateRootParams { - - return WeaviateRootParams{} -} - -// WeaviateRootParams contains all the bound params for the weaviate root operation -// typically these are obtained from a http.Request -// -// swagger:parameters weaviate.root -type WeaviateRootParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewWeaviateRootParams() beforehand. -func (o *WeaviateRootParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/adapters/handlers/rest/operations/weaviate_root_responses.go b/adapters/handlers/rest/operations/weaviate_root_responses.go deleted file mode 100644 index bfe34ca03b334f3aa9cdce59e6b0b8b1c28b3888..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/weaviate_root_responses.go +++ /dev/null @@ -1,68 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package operations - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" -) - -// WeaviateRootOKCode is the HTTP code returned for type WeaviateRootOK -const WeaviateRootOKCode int = 200 - -/* -WeaviateRootOK Weaviate is alive and ready to serve content - -swagger:response weaviateRootOK -*/ -type WeaviateRootOK struct { - - /* - In: Body - */ - Payload *WeaviateRootOKBody `json:"body,omitempty"` -} - -// NewWeaviateRootOK creates WeaviateRootOK with default headers values -func NewWeaviateRootOK() *WeaviateRootOK { - - return &WeaviateRootOK{} -} - -// WithPayload adds the payload to the weaviate root o k response -func (o *WeaviateRootOK) WithPayload(payload *WeaviateRootOKBody) *WeaviateRootOK { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the weaviate root o k response -func (o *WeaviateRootOK) SetPayload(payload *WeaviateRootOKBody) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *WeaviateRootOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(200) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/weaviate_root_urlbuilder.go b/adapters/handlers/rest/operations/weaviate_root_urlbuilder.go deleted file mode 100644 index 8d56aa828cafb8b0e89fa2daaff7a0a15d55533f..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/weaviate_root_urlbuilder.go +++ /dev/null @@ -1,98 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package operations - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" -) - -// WeaviateRootURL generates an URL for the weaviate root operation -type WeaviateRootURL struct { - _basePath string -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *WeaviateRootURL) WithBasePath(bp string) *WeaviateRootURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *WeaviateRootURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *WeaviateRootURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/" - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *WeaviateRootURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *WeaviateRootURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *WeaviateRootURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on WeaviateRootURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on WeaviateRootURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *WeaviateRootURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/weaviate_wellknown_liveness.go b/adapters/handlers/rest/operations/weaviate_wellknown_liveness.go deleted file mode 100644 index d58c680d29900813031128fed589f98b27fb22ad..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/weaviate_wellknown_liveness.go +++ /dev/null @@ -1,82 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package operations - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// WeaviateWellknownLivenessHandlerFunc turns a function with the right signature into a weaviate wellknown liveness handler -type WeaviateWellknownLivenessHandlerFunc func(WeaviateWellknownLivenessParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn WeaviateWellknownLivenessHandlerFunc) Handle(params WeaviateWellknownLivenessParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// WeaviateWellknownLivenessHandler interface for that can handle valid weaviate wellknown liveness params -type WeaviateWellknownLivenessHandler interface { - Handle(WeaviateWellknownLivenessParams, *models.Principal) middleware.Responder -} - -// NewWeaviateWellknownLiveness creates a new http.Handler for the weaviate wellknown liveness operation -func NewWeaviateWellknownLiveness(ctx *middleware.Context, handler WeaviateWellknownLivenessHandler) *WeaviateWellknownLiveness { - return &WeaviateWellknownLiveness{Context: ctx, Handler: handler} -} - -/* - WeaviateWellknownLiveness swagger:route GET /.well-known/live weaviateWellknownLiveness - -Determines whether the application is alive. Can be used for kubernetes liveness probe -*/ -type WeaviateWellknownLiveness struct { - Context *middleware.Context - Handler WeaviateWellknownLivenessHandler -} - -func (o *WeaviateWellknownLiveness) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewWeaviateWellknownLivenessParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/weaviate_wellknown_liveness_parameters.go b/adapters/handlers/rest/operations/weaviate_wellknown_liveness_parameters.go deleted file mode 100644 index 60627271ff348415f71d7cf479b8233e6470e6b7..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/weaviate_wellknown_liveness_parameters.go +++ /dev/null @@ -1,57 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package operations - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime/middleware" -) - -// NewWeaviateWellknownLivenessParams creates a new WeaviateWellknownLivenessParams object -// -// There are no default values defined in the spec. -func NewWeaviateWellknownLivenessParams() WeaviateWellknownLivenessParams { - - return WeaviateWellknownLivenessParams{} -} - -// WeaviateWellknownLivenessParams contains all the bound params for the weaviate wellknown liveness operation -// typically these are obtained from a http.Request -// -// swagger:parameters weaviate.wellknown.liveness -type WeaviateWellknownLivenessParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewWeaviateWellknownLivenessParams() beforehand. -func (o *WeaviateWellknownLivenessParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/adapters/handlers/rest/operations/weaviate_wellknown_liveness_responses.go b/adapters/handlers/rest/operations/weaviate_wellknown_liveness_responses.go deleted file mode 100644 index bc6f86fa4a5a394b71fb830593d26ff73311c645..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/weaviate_wellknown_liveness_responses.go +++ /dev/null @@ -1,48 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package operations - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" -) - -// WeaviateWellknownLivenessOKCode is the HTTP code returned for type WeaviateWellknownLivenessOK -const WeaviateWellknownLivenessOKCode int = 200 - -/* -WeaviateWellknownLivenessOK The application is able to respond to HTTP requests - -swagger:response weaviateWellknownLivenessOK -*/ -type WeaviateWellknownLivenessOK struct { -} - -// NewWeaviateWellknownLivenessOK creates WeaviateWellknownLivenessOK with default headers values -func NewWeaviateWellknownLivenessOK() *WeaviateWellknownLivenessOK { - - return &WeaviateWellknownLivenessOK{} -} - -// WriteResponse to the client -func (o *WeaviateWellknownLivenessOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(200) -} diff --git a/adapters/handlers/rest/operations/weaviate_wellknown_liveness_urlbuilder.go b/adapters/handlers/rest/operations/weaviate_wellknown_liveness_urlbuilder.go deleted file mode 100644 index 8d6325db59b43df9b0b7109e96e3d6bd18e8f555..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/weaviate_wellknown_liveness_urlbuilder.go +++ /dev/null @@ -1,98 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package operations - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" -) - -// WeaviateWellknownLivenessURL generates an URL for the weaviate wellknown liveness operation -type WeaviateWellknownLivenessURL struct { - _basePath string -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *WeaviateWellknownLivenessURL) WithBasePath(bp string) *WeaviateWellknownLivenessURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *WeaviateWellknownLivenessURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *WeaviateWellknownLivenessURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/.well-known/live" - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *WeaviateWellknownLivenessURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *WeaviateWellknownLivenessURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *WeaviateWellknownLivenessURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on WeaviateWellknownLivenessURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on WeaviateWellknownLivenessURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *WeaviateWellknownLivenessURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/weaviate_wellknown_readiness.go b/adapters/handlers/rest/operations/weaviate_wellknown_readiness.go deleted file mode 100644 index becf878b78d10c781c1a8ab234e9530cc1537a30..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/weaviate_wellknown_readiness.go +++ /dev/null @@ -1,82 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package operations - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime/middleware" - - "github.com/weaviate/weaviate/entities/models" -) - -// WeaviateWellknownReadinessHandlerFunc turns a function with the right signature into a weaviate wellknown readiness handler -type WeaviateWellknownReadinessHandlerFunc func(WeaviateWellknownReadinessParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn WeaviateWellknownReadinessHandlerFunc) Handle(params WeaviateWellknownReadinessParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// WeaviateWellknownReadinessHandler interface for that can handle valid weaviate wellknown readiness params -type WeaviateWellknownReadinessHandler interface { - Handle(WeaviateWellknownReadinessParams, *models.Principal) middleware.Responder -} - -// NewWeaviateWellknownReadiness creates a new http.Handler for the weaviate wellknown readiness operation -func NewWeaviateWellknownReadiness(ctx *middleware.Context, handler WeaviateWellknownReadinessHandler) *WeaviateWellknownReadiness { - return &WeaviateWellknownReadiness{Context: ctx, Handler: handler} -} - -/* - WeaviateWellknownReadiness swagger:route GET /.well-known/ready weaviateWellknownReadiness - -Determines whether the application is ready to receive traffic. Can be used for kubernetes readiness probe. -*/ -type WeaviateWellknownReadiness struct { - Context *middleware.Context - Handler WeaviateWellknownReadinessHandler -} - -func (o *WeaviateWellknownReadiness) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewWeaviateWellknownReadinessParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} diff --git a/adapters/handlers/rest/operations/weaviate_wellknown_readiness_parameters.go b/adapters/handlers/rest/operations/weaviate_wellknown_readiness_parameters.go deleted file mode 100644 index 3fa6bf1e29d84de8cc7d6e65fab6a288dabdc023..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/weaviate_wellknown_readiness_parameters.go +++ /dev/null @@ -1,57 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package operations - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime/middleware" -) - -// NewWeaviateWellknownReadinessParams creates a new WeaviateWellknownReadinessParams object -// -// There are no default values defined in the spec. -func NewWeaviateWellknownReadinessParams() WeaviateWellknownReadinessParams { - - return WeaviateWellknownReadinessParams{} -} - -// WeaviateWellknownReadinessParams contains all the bound params for the weaviate wellknown readiness operation -// typically these are obtained from a http.Request -// -// swagger:parameters weaviate.wellknown.readiness -type WeaviateWellknownReadinessParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewWeaviateWellknownReadinessParams() beforehand. -func (o *WeaviateWellknownReadinessParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/adapters/handlers/rest/operations/weaviate_wellknown_readiness_responses.go b/adapters/handlers/rest/operations/weaviate_wellknown_readiness_responses.go deleted file mode 100644 index f4a6ed8b9cda664bf7a5fa159b5a25a74bf6dca1..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/weaviate_wellknown_readiness_responses.go +++ /dev/null @@ -1,73 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package operations - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" -) - -// WeaviateWellknownReadinessOKCode is the HTTP code returned for type WeaviateWellknownReadinessOK -const WeaviateWellknownReadinessOKCode int = 200 - -/* -WeaviateWellknownReadinessOK The application has completed its start-up routine and is ready to accept traffic. - -swagger:response weaviateWellknownReadinessOK -*/ -type WeaviateWellknownReadinessOK struct { -} - -// NewWeaviateWellknownReadinessOK creates WeaviateWellknownReadinessOK with default headers values -func NewWeaviateWellknownReadinessOK() *WeaviateWellknownReadinessOK { - - return &WeaviateWellknownReadinessOK{} -} - -// WriteResponse to the client -func (o *WeaviateWellknownReadinessOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(200) -} - -// WeaviateWellknownReadinessServiceUnavailableCode is the HTTP code returned for type WeaviateWellknownReadinessServiceUnavailable -const WeaviateWellknownReadinessServiceUnavailableCode int = 503 - -/* -WeaviateWellknownReadinessServiceUnavailable The application is currently not able to serve traffic. If other horizontal replicas of weaviate are available and they are capable of receiving traffic, all traffic should be redirected there instead. - -swagger:response weaviateWellknownReadinessServiceUnavailable -*/ -type WeaviateWellknownReadinessServiceUnavailable struct { -} - -// NewWeaviateWellknownReadinessServiceUnavailable creates WeaviateWellknownReadinessServiceUnavailable with default headers values -func NewWeaviateWellknownReadinessServiceUnavailable() *WeaviateWellknownReadinessServiceUnavailable { - - return &WeaviateWellknownReadinessServiceUnavailable{} -} - -// WriteResponse to the client -func (o *WeaviateWellknownReadinessServiceUnavailable) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(503) -} diff --git a/adapters/handlers/rest/operations/weaviate_wellknown_readiness_urlbuilder.go b/adapters/handlers/rest/operations/weaviate_wellknown_readiness_urlbuilder.go deleted file mode 100644 index ea7a43e5d2422a5594c35e7ae18d2a6203d6a617..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/weaviate_wellknown_readiness_urlbuilder.go +++ /dev/null @@ -1,98 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package operations - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" -) - -// WeaviateWellknownReadinessURL generates an URL for the weaviate wellknown readiness operation -type WeaviateWellknownReadinessURL struct { - _basePath string -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *WeaviateWellknownReadinessURL) WithBasePath(bp string) *WeaviateWellknownReadinessURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *WeaviateWellknownReadinessURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *WeaviateWellknownReadinessURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/.well-known/ready" - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *WeaviateWellknownReadinessURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *WeaviateWellknownReadinessURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *WeaviateWellknownReadinessURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on WeaviateWellknownReadinessURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on WeaviateWellknownReadinessURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *WeaviateWellknownReadinessURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/operations/well_known/get_well_known_openid_configuration.go b/adapters/handlers/rest/operations/well_known/get_well_known_openid_configuration.go deleted file mode 100644 index e14af4c015ed74c446086cb996810d2416bc08e5..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/well_known/get_well_known_openid_configuration.go +++ /dev/null @@ -1,130 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package well_known - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "context" - "net/http" - - "github.com/go-openapi/runtime/middleware" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" - - "github.com/weaviate/weaviate/entities/models" -) - -// GetWellKnownOpenidConfigurationHandlerFunc turns a function with the right signature into a get well known openid configuration handler -type GetWellKnownOpenidConfigurationHandlerFunc func(GetWellKnownOpenidConfigurationParams, *models.Principal) middleware.Responder - -// Handle executing the request and returning a response -func (fn GetWellKnownOpenidConfigurationHandlerFunc) Handle(params GetWellKnownOpenidConfigurationParams, principal *models.Principal) middleware.Responder { - return fn(params, principal) -} - -// GetWellKnownOpenidConfigurationHandler interface for that can handle valid get well known openid configuration params -type GetWellKnownOpenidConfigurationHandler interface { - Handle(GetWellKnownOpenidConfigurationParams, *models.Principal) middleware.Responder -} - -// NewGetWellKnownOpenidConfiguration creates a new http.Handler for the get well known openid configuration operation -func NewGetWellKnownOpenidConfiguration(ctx *middleware.Context, handler GetWellKnownOpenidConfigurationHandler) *GetWellKnownOpenidConfiguration { - return &GetWellKnownOpenidConfiguration{Context: ctx, Handler: handler} -} - -/* - GetWellKnownOpenidConfiguration swagger:route GET /.well-known/openid-configuration well-known oidc discovery getWellKnownOpenidConfiguration - -# OIDC discovery information if OIDC auth is enabled - -OIDC Discovery page, redirects to the token issuer if one is configured -*/ -type GetWellKnownOpenidConfiguration struct { - Context *middleware.Context - Handler GetWellKnownOpenidConfigurationHandler -} - -func (o *GetWellKnownOpenidConfiguration) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - route, rCtx, _ := o.Context.RouteInfo(r) - if rCtx != nil { - *r = *rCtx - } - var Params = NewGetWellKnownOpenidConfigurationParams() - uprinc, aCtx, err := o.Context.Authorize(r, route) - if err != nil { - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - if aCtx != nil { - *r = *aCtx - } - var principal *models.Principal - if uprinc != nil { - principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise - } - - if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params - o.Context.Respond(rw, r, route.Produces, route, err) - return - } - - res := o.Handler.Handle(Params, principal) // actually handle the request - o.Context.Respond(rw, r, route.Produces, route, res) - -} - -// GetWellKnownOpenidConfigurationOKBody get well known openid configuration o k body -// -// swagger:model GetWellKnownOpenidConfigurationOKBody -type GetWellKnownOpenidConfigurationOKBody struct { - - // OAuth Client ID - ClientID string `json:"clientId,omitempty" yaml:"clientId,omitempty"` - - // The Location to redirect to - Href string `json:"href,omitempty" yaml:"href,omitempty"` - - // OAuth Scopes - Scopes []string `json:"scopes,omitempty" yaml:"scopes,omitempty"` -} - -// Validate validates this get well known openid configuration o k body -func (o *GetWellKnownOpenidConfigurationOKBody) Validate(formats strfmt.Registry) error { - return nil -} - -// ContextValidate validates this get well known openid configuration o k body based on context it is used -func (o *GetWellKnownOpenidConfigurationOKBody) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - return nil -} - -// MarshalBinary interface implementation -func (o *GetWellKnownOpenidConfigurationOKBody) MarshalBinary() ([]byte, error) { - if o == nil { - return nil, nil - } - return swag.WriteJSON(o) -} - -// UnmarshalBinary interface implementation -func (o *GetWellKnownOpenidConfigurationOKBody) UnmarshalBinary(b []byte) error { - var res GetWellKnownOpenidConfigurationOKBody - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *o = res - return nil -} diff --git a/adapters/handlers/rest/operations/well_known/get_well_known_openid_configuration_parameters.go b/adapters/handlers/rest/operations/well_known/get_well_known_openid_configuration_parameters.go deleted file mode 100644 index 4697be81f7041d7408fe2c2e7cf9f81f8dac6934..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/well_known/get_well_known_openid_configuration_parameters.go +++ /dev/null @@ -1,57 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package well_known - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime/middleware" -) - -// NewGetWellKnownOpenidConfigurationParams creates a new GetWellKnownOpenidConfigurationParams object -// -// There are no default values defined in the spec. -func NewGetWellKnownOpenidConfigurationParams() GetWellKnownOpenidConfigurationParams { - - return GetWellKnownOpenidConfigurationParams{} -} - -// GetWellKnownOpenidConfigurationParams contains all the bound params for the get well known openid configuration operation -// typically these are obtained from a http.Request -// -// swagger:parameters GetWellKnownOpenidConfiguration -type GetWellKnownOpenidConfigurationParams struct { - - // HTTP Request Object - HTTPRequest *http.Request `json:"-"` -} - -// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface -// for simple values it will use straight method calls. -// -// To ensure default values, the struct must have been initialized with NewGetWellKnownOpenidConfigurationParams() beforehand. -func (o *GetWellKnownOpenidConfigurationParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { - var res []error - - o.HTTPRequest = r - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/adapters/handlers/rest/operations/well_known/get_well_known_openid_configuration_responses.go b/adapters/handlers/rest/operations/well_known/get_well_known_openid_configuration_responses.go deleted file mode 100644 index 445a35d5f5753dd6d70dc7fe74bf37e402ec3565..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/well_known/get_well_known_openid_configuration_responses.go +++ /dev/null @@ -1,140 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package well_known - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "net/http" - - "github.com/go-openapi/runtime" - - "github.com/weaviate/weaviate/entities/models" -) - -// GetWellKnownOpenidConfigurationOKCode is the HTTP code returned for type GetWellKnownOpenidConfigurationOK -const GetWellKnownOpenidConfigurationOKCode int = 200 - -/* -GetWellKnownOpenidConfigurationOK Successful response, inspect body - -swagger:response getWellKnownOpenidConfigurationOK -*/ -type GetWellKnownOpenidConfigurationOK struct { - - /* - In: Body - */ - Payload *GetWellKnownOpenidConfigurationOKBody `json:"body,omitempty"` -} - -// NewGetWellKnownOpenidConfigurationOK creates GetWellKnownOpenidConfigurationOK with default headers values -func NewGetWellKnownOpenidConfigurationOK() *GetWellKnownOpenidConfigurationOK { - - return &GetWellKnownOpenidConfigurationOK{} -} - -// WithPayload adds the payload to the get well known openid configuration o k response -func (o *GetWellKnownOpenidConfigurationOK) WithPayload(payload *GetWellKnownOpenidConfigurationOKBody) *GetWellKnownOpenidConfigurationOK { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the get well known openid configuration o k response -func (o *GetWellKnownOpenidConfigurationOK) SetPayload(payload *GetWellKnownOpenidConfigurationOKBody) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *GetWellKnownOpenidConfigurationOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(200) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} - -// GetWellKnownOpenidConfigurationNotFoundCode is the HTTP code returned for type GetWellKnownOpenidConfigurationNotFound -const GetWellKnownOpenidConfigurationNotFoundCode int = 404 - -/* -GetWellKnownOpenidConfigurationNotFound Not found, no oidc provider present - -swagger:response getWellKnownOpenidConfigurationNotFound -*/ -type GetWellKnownOpenidConfigurationNotFound struct { -} - -// NewGetWellKnownOpenidConfigurationNotFound creates GetWellKnownOpenidConfigurationNotFound with default headers values -func NewGetWellKnownOpenidConfigurationNotFound() *GetWellKnownOpenidConfigurationNotFound { - - return &GetWellKnownOpenidConfigurationNotFound{} -} - -// WriteResponse to the client -func (o *GetWellKnownOpenidConfigurationNotFound) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses - - rw.WriteHeader(404) -} - -// GetWellKnownOpenidConfigurationInternalServerErrorCode is the HTTP code returned for type GetWellKnownOpenidConfigurationInternalServerError -const GetWellKnownOpenidConfigurationInternalServerErrorCode int = 500 - -/* -GetWellKnownOpenidConfigurationInternalServerError An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. - -swagger:response getWellKnownOpenidConfigurationInternalServerError -*/ -type GetWellKnownOpenidConfigurationInternalServerError struct { - - /* - In: Body - */ - Payload *models.ErrorResponse `json:"body,omitempty"` -} - -// NewGetWellKnownOpenidConfigurationInternalServerError creates GetWellKnownOpenidConfigurationInternalServerError with default headers values -func NewGetWellKnownOpenidConfigurationInternalServerError() *GetWellKnownOpenidConfigurationInternalServerError { - - return &GetWellKnownOpenidConfigurationInternalServerError{} -} - -// WithPayload adds the payload to the get well known openid configuration internal server error response -func (o *GetWellKnownOpenidConfigurationInternalServerError) WithPayload(payload *models.ErrorResponse) *GetWellKnownOpenidConfigurationInternalServerError { - o.Payload = payload - return o -} - -// SetPayload sets the payload to the get well known openid configuration internal server error response -func (o *GetWellKnownOpenidConfigurationInternalServerError) SetPayload(payload *models.ErrorResponse) { - o.Payload = payload -} - -// WriteResponse to the client -func (o *GetWellKnownOpenidConfigurationInternalServerError) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { - - rw.WriteHeader(500) - if o.Payload != nil { - payload := o.Payload - if err := producer.Produce(rw, payload); err != nil { - panic(err) // let the recovery middleware deal with this - } - } -} diff --git a/adapters/handlers/rest/operations/well_known/get_well_known_openid_configuration_urlbuilder.go b/adapters/handlers/rest/operations/well_known/get_well_known_openid_configuration_urlbuilder.go deleted file mode 100644 index 18d3c0d4f27a86abd565d222c067bf8417f408ba..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/operations/well_known/get_well_known_openid_configuration_urlbuilder.go +++ /dev/null @@ -1,98 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package well_known - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the generate command - -import ( - "errors" - "net/url" - golangswaggerpaths "path" -) - -// GetWellKnownOpenidConfigurationURL generates an URL for the get well known openid configuration operation -type GetWellKnownOpenidConfigurationURL struct { - _basePath string -} - -// WithBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *GetWellKnownOpenidConfigurationURL) WithBasePath(bp string) *GetWellKnownOpenidConfigurationURL { - o.SetBasePath(bp) - return o -} - -// SetBasePath sets the base path for this url builder, only required when it's different from the -// base path specified in the swagger spec. -// When the value of the base path is an empty string -func (o *GetWellKnownOpenidConfigurationURL) SetBasePath(bp string) { - o._basePath = bp -} - -// Build a url path and query string -func (o *GetWellKnownOpenidConfigurationURL) Build() (*url.URL, error) { - var _result url.URL - - var _path = "/.well-known/openid-configuration" - - _basePath := o._basePath - if _basePath == "" { - _basePath = "/v1" - } - _result.Path = golangswaggerpaths.Join(_basePath, _path) - - return &_result, nil -} - -// Must is a helper function to panic when the url builder returns an error -func (o *GetWellKnownOpenidConfigurationURL) Must(u *url.URL, err error) *url.URL { - if err != nil { - panic(err) - } - if u == nil { - panic("url can't be nil") - } - return u -} - -// String returns the string representation of the path with query string -func (o *GetWellKnownOpenidConfigurationURL) String() string { - return o.Must(o.Build()).String() -} - -// BuildFull builds a full url with scheme, host, path and query string -func (o *GetWellKnownOpenidConfigurationURL) BuildFull(scheme, host string) (*url.URL, error) { - if scheme == "" { - return nil, errors.New("scheme is required for a full url on GetWellKnownOpenidConfigurationURL") - } - if host == "" { - return nil, errors.New("host is required for a full url on GetWellKnownOpenidConfigurationURL") - } - - base, err := o.Build() - if err != nil { - return nil, err - } - - base.Scheme = scheme - base.Host = host - return base, nil -} - -// StringFull returns the string representation of a complete url -func (o *GetWellKnownOpenidConfigurationURL) StringFull(scheme, host string) string { - return o.Must(o.BuildFull(scheme, host)).String() -} diff --git a/adapters/handlers/rest/panics_middleware.go b/adapters/handlers/rest/panics_middleware.go deleted file mode 100644 index 2346184f39716025c0f7fd380509886fd486ce4a..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/panics_middleware.go +++ /dev/null @@ -1,99 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package rest - -import ( - "fmt" - "net" - "net/http" - "runtime/debug" - "syscall" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" -) - -func makeCatchPanics(logger logrus.FieldLogger, metricRequestsTotal restApiRequestsTotal) func(http.Handler) http.Handler { - return func(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - defer handlePanics(logger, metricRequestsTotal, r) - next.ServeHTTP(w, r) - }) - } -} - -func handlePanics(logger logrus.FieldLogger, metricRequestsTotal restApiRequestsTotal, r *http.Request) { - recovered := recover() - if recovered == nil { - return - } - - err, ok := recovered.(error) - if !ok { - // not a typed error, we can not handle this error other returning it to - // the user - logger.WithFields(logrus.Fields{ - "error": recovered, - "method": r.Method, - "path": r.URL, - }).Errorf("%v", recovered) - - // This was not expected, so we want to print the stack, this will help us - // find the source of the issue if the user sends their logs - metricRequestsTotal.logServerError("", fmt.Errorf("%v", recovered)) - debug.PrintStack() - return - } - - if errors.Is(err, syscall.EPIPE) { - metricRequestsTotal.logUserError("") - handleBrokenPipe(err, logger, r) - return - } - - var netErr net.Error - if errors.As(err, &netErr) { - if netErr.Timeout() { - metricRequestsTotal.logUserError("") - handleTimeout(netErr, logger, r) - return - } - } - - // typed as error, but none we are currently handling explicitly - logger.WithError(err).WithFields(logrus.Fields{ - "method": r.Method, - "path": r.URL, - }).Errorf(err.Error()) - // This was not expected, so we want to print the stack, this will help us - // find the source of the issue if the user sends their logs - metricRequestsTotal.logServerError("", err) - debug.PrintStack() -} - -func handleBrokenPipe(err error, logger logrus.FieldLogger, r *http.Request) { - logger.WithError(err).WithFields(logrus.Fields{ - "method": r.Method, - "path": r.URL, - "description": "A broken pipe error occurs when Weaviate tries to write a response onto a connection that has already been closed or reset by the client. Typically, this is the case when the server was not able to respond within the configured client-side timeout.", - "hint": "Either try increasing the client-side timeout, or sending a computationally cheaper request, for example by reducing a batch size, reducing a limit, using less complex filters, etc.", - }).Errorf("broken pipe") -} - -func handleTimeout(err net.Error, logger logrus.FieldLogger, r *http.Request) { - logger.WithError(err).WithFields(logrus.Fields{ - "method": r.Method, - "path": r.URL, - "description": "An I/O timeout occurs when the request takes longer than the specified server-side timeout.", - "hint": "Either try increasing the server-side timeout using e.g. '--write-timeout=600s' as a command line flag when starting Weaviate, or try sending a computationally cheaper request, for example by reducing a batch size, reducing a limit, using less complex filters, etc. Note that this error is only thrown if client-side and server-side timeouts are not in sync, more precisely if the client-side timeout is longer than the server side timeout.", - }).Errorf("i/o timeout") -} diff --git a/adapters/handlers/rest/requests_total_metrics.go b/adapters/handlers/rest/requests_total_metrics.go deleted file mode 100644 index e7a47d3c1dbd9fee88470ee9c1d2a276ae794f4f..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/requests_total_metrics.go +++ /dev/null @@ -1,123 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package rest - -import ( - "github.com/prometheus/client_golang/prometheus" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/usecases/monitoring" -) - -type RequestStatus int - -const ( - Ok RequestStatus = iota - UserError - ServerError -) - -func (s RequestStatus) String() string { - switch s { - case Ok: - return "ok" - case UserError: - return "user_error" - case ServerError: - return "server_error" - } - return "unknown" -} - -type requestsTotalMetric struct { - requestsTotal *prometheus.GaugeVec - groupClasses bool - api string -} - -func newRequestsTotalMetric(prom *monitoring.PrometheusMetrics, api string) *requestsTotalMetric { - if prom == nil { - return nil - } - return &requestsTotalMetric{ - requestsTotal: prom.RequestsTotal, - groupClasses: prom.Group, - api: api, - } -} - -func (m *requestsTotalMetric) RequestsTotalInc(status RequestStatus, className, queryType string) { - if m == nil { - return - } - - if m.groupClasses { - className = "n/a" - } - - m.requestsTotal.With(prometheus.Labels{ - "status": status.String(), - "class_name": className, - "api": m.api, - "query_type": queryType, - }).Inc() -} - -type restApiRequestsTotal interface { - logError(className string, err error) - logOk(className string) - logUserError(className string) - logServerError(className string, err error) -} - -type restApiRequestsTotalImpl struct { - metrics *requestsTotalMetric - api, queryType string - logger logrus.FieldLogger -} - -func (e *restApiRequestsTotalImpl) logOk(className string) { - if e.metrics != nil { - e.metrics.RequestsTotalInc(Ok, className, e.queryType) - } -} - -func (e *restApiRequestsTotalImpl) logUserError(className string) { - if e.metrics != nil { - e.metrics.RequestsTotalInc(UserError, className, e.queryType) - } -} - -func (e *restApiRequestsTotalImpl) logServerError(className string, err error) { - e.logger.WithFields(logrus.Fields{ - "action": "requests_total", - "api": e.api, - "query_type": e.queryType, - "class_name": className, - }).WithError(err).Error("unexpected error") - if e.metrics != nil { - e.metrics.RequestsTotalInc(ServerError, className, e.queryType) - } -} - -type panicsRequestsTotal struct { - *restApiRequestsTotalImpl -} - -func newPanicsRequestsTotal(metrics *monitoring.PrometheusMetrics, logger logrus.FieldLogger) restApiRequestsTotal { - return &panicsRequestsTotal{ - restApiRequestsTotalImpl: &restApiRequestsTotalImpl{newRequestsTotalMetric(metrics, "rest"), "rest", "", logger}, - } -} - -func (e *panicsRequestsTotal) logError(className string, err error) { - e.logServerError(className, err) -} diff --git a/adapters/handlers/rest/server.go b/adapters/handlers/rest/server.go deleted file mode 100644 index cb3ef9310982804851f319099fca5629c7688809..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/server.go +++ /dev/null @@ -1,518 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package rest - -import ( - "context" - "crypto/tls" - "crypto/x509" - "errors" - "fmt" - "log" - "net" - "net/http" - "os" - "os/signal" - "strconv" - "sync" - "sync/atomic" - "syscall" - "time" - - "github.com/go-openapi/runtime/flagext" - "github.com/go-openapi/swag" - flags "github.com/jessevdk/go-flags" - "golang.org/x/net/netutil" - - "github.com/weaviate/weaviate/adapters/handlers/rest/operations" -) - -const ( - schemeHTTP = "http" - schemeHTTPS = "https" - schemeUnix = "unix" -) - -var defaultSchemes []string - -func init() { - defaultSchemes = []string{ - schemeHTTPS, - } -} - -// NewServer creates a new api weaviate server but does not configure it -func NewServer(api *operations.WeaviateAPI) *Server { - s := new(Server) - - s.shutdown = make(chan struct{}) - s.api = api - s.interrupt = make(chan os.Signal, 1) - return s -} - -// ConfigureAPI configures the API and handlers. -func (s *Server) ConfigureAPI() { - if s.api != nil { - s.handler = configureAPI(s.api) - } -} - -// ConfigureFlags configures the additional flags defined by the handlers. Needs to be called before the parser.Parse -func (s *Server) ConfigureFlags() { - if s.api != nil { - configureFlags(s.api) - } -} - -// Server for the weaviate API -type Server struct { - EnabledListeners []string `long:"scheme" description:"the listeners to enable, this can be repeated and defaults to the schemes in the swagger spec"` - CleanupTimeout time.Duration `long:"cleanup-timeout" description:"grace period for which to wait before killing idle connections" default:"10s"` - GracefulTimeout time.Duration `long:"graceful-timeout" description:"grace period for which to wait before shutting down the server" default:"15s"` - MaxHeaderSize flagext.ByteSize `long:"max-header-size" description:"controls the maximum number of bytes the server will read parsing the request header's keys and values, including the request line. It does not limit the size of the request body." default:"1MiB"` - - SocketPath flags.Filename `long:"socket-path" description:"the unix socket to listen on" default:"/var/run/weaviate.sock"` - domainSocketL net.Listener - - Host string `long:"host" description:"the IP to listen on" default:"localhost" env:"HOST"` - Port int `long:"port" description:"the port to listen on for insecure connections, defaults to a random value" env:"PORT"` - ListenLimit int `long:"listen-limit" description:"limit the number of outstanding requests"` - KeepAlive time.Duration `long:"keep-alive" description:"sets the TCP keep-alive timeouts on accepted connections. It prunes dead TCP connections ( e.g. closing laptop mid-download)" default:"3m"` - ReadTimeout time.Duration `long:"read-timeout" description:"maximum duration before timing out read of the request" default:"30s"` - WriteTimeout time.Duration `long:"write-timeout" description:"maximum duration before timing out write of the response" default:"60s"` - httpServerL net.Listener - - TLSHost string `long:"tls-host" description:"the IP to listen on for tls, when not specified it's the same as --host" env:"TLS_HOST"` - TLSPort int `long:"tls-port" description:"the port to listen on for secure connections, defaults to a random value" env:"TLS_PORT"` - TLSCertificate flags.Filename `long:"tls-certificate" description:"the certificate to use for secure connections" env:"TLS_CERTIFICATE"` - TLSCertificateKey flags.Filename `long:"tls-key" description:"the private key to use for secure connections" env:"TLS_PRIVATE_KEY"` - TLSCACertificate flags.Filename `long:"tls-ca" description:"the certificate authority file to be used with mutual tls auth" env:"TLS_CA_CERTIFICATE"` - TLSListenLimit int `long:"tls-listen-limit" description:"limit the number of outstanding requests"` - TLSKeepAlive time.Duration `long:"tls-keep-alive" description:"sets the TCP keep-alive timeouts on accepted connections. It prunes dead TCP connections ( e.g. closing laptop mid-download)"` - TLSReadTimeout time.Duration `long:"tls-read-timeout" description:"maximum duration before timing out read of the request"` - TLSWriteTimeout time.Duration `long:"tls-write-timeout" description:"maximum duration before timing out write of the response"` - httpsServerL net.Listener - - api *operations.WeaviateAPI - handler http.Handler - hasListeners bool - shutdown chan struct{} - shuttingDown int32 - interrupted bool - interrupt chan os.Signal -} - -// Logf logs message either via defined user logger or via system one if no user logger is defined. -func (s *Server) Logf(f string, args ...interface{}) { - if s.api != nil && s.api.Logger != nil { - s.api.Logger(f, args...) - } else { - log.Printf(f, args...) - } -} - -// Fatalf logs message either via defined user logger or via system one if no user logger is defined. -// Exits with non-zero status after printing -func (s *Server) Fatalf(f string, args ...interface{}) { - if s.api != nil && s.api.Logger != nil { - s.api.Logger(f, args...) - os.Exit(1) - } else { - log.Fatalf(f, args...) - } -} - -// SetAPI configures the server with the specified API. Needs to be called before Serve -func (s *Server) SetAPI(api *operations.WeaviateAPI) { - if api == nil { - s.api = nil - s.handler = nil - return - } - - s.api = api - s.handler = configureAPI(api) -} - -func (s *Server) hasScheme(scheme string) bool { - schemes := s.EnabledListeners - if len(schemes) == 0 { - schemes = defaultSchemes - } - - for _, v := range schemes { - if v == scheme { - return true - } - } - return false -} - -// Serve the api -func (s *Server) Serve() (err error) { - if !s.hasListeners { - if err = s.Listen(); err != nil { - return err - } - } - - // set default handler, if none is set - if s.handler == nil { - if s.api == nil { - return errors.New("can't create the default handler, as no api is set") - } - - s.SetHandler(s.api.Serve(nil)) - } - - wg := new(sync.WaitGroup) - once := new(sync.Once) - signalNotify(s.interrupt) - go handleInterrupt(once, s) - - servers := []*http.Server{} - - if s.hasScheme(schemeUnix) { - domainSocket := new(http.Server) - domainSocket.MaxHeaderBytes = int(s.MaxHeaderSize) - domainSocket.Handler = s.handler - if int64(s.CleanupTimeout) > 0 { - domainSocket.IdleTimeout = s.CleanupTimeout - } - - configureServer(domainSocket, "unix", string(s.SocketPath)) - - servers = append(servers, domainSocket) - wg.Add(1) - s.Logf("Serving weaviate at unix://%s", s.SocketPath) - go func(l net.Listener) { - defer wg.Done() - if err := domainSocket.Serve(l); err != nil && err != http.ErrServerClosed { - s.Fatalf("%v", err) - } - s.Logf("Stopped serving weaviate at unix://%s", s.SocketPath) - }(s.domainSocketL) - } - - if s.hasScheme(schemeHTTP) { - httpServer := new(http.Server) - httpServer.MaxHeaderBytes = int(s.MaxHeaderSize) - httpServer.ReadTimeout = s.ReadTimeout - httpServer.WriteTimeout = s.WriteTimeout - httpServer.SetKeepAlivesEnabled(int64(s.KeepAlive) > 0) - if s.ListenLimit > 0 { - s.httpServerL = netutil.LimitListener(s.httpServerL, s.ListenLimit) - } - - if int64(s.CleanupTimeout) > 0 { - httpServer.IdleTimeout = s.CleanupTimeout - } - - httpServer.Handler = s.handler - - configureServer(httpServer, "http", s.httpServerL.Addr().String()) - - servers = append(servers, httpServer) - wg.Add(1) - s.Logf("Serving weaviate at http://%s", s.httpServerL.Addr()) - go func(l net.Listener) { - defer wg.Done() - if err := httpServer.Serve(l); err != nil && err != http.ErrServerClosed { - s.Fatalf("%v", err) - } - s.Logf("Stopped serving weaviate at http://%s", l.Addr()) - }(s.httpServerL) - } - - if s.hasScheme(schemeHTTPS) { - httpsServer := new(http.Server) - httpsServer.MaxHeaderBytes = int(s.MaxHeaderSize) - httpsServer.ReadTimeout = s.TLSReadTimeout - httpsServer.WriteTimeout = s.TLSWriteTimeout - httpsServer.SetKeepAlivesEnabled(int64(s.TLSKeepAlive) > 0) - if s.TLSListenLimit > 0 { - s.httpsServerL = netutil.LimitListener(s.httpsServerL, s.TLSListenLimit) - } - if int64(s.CleanupTimeout) > 0 { - httpsServer.IdleTimeout = s.CleanupTimeout - } - httpsServer.Handler = s.handler - - // Inspired by https://blog.bracebin.com/achieving-perfect-ssl-labs-score-with-go - httpsServer.TLSConfig = &tls.Config{ - // Causes servers to use Go's default ciphersuite preferences, - // which are tuned to avoid attacks. Does nothing on clients. - PreferServerCipherSuites: true, - // Only use curves which have assembly implementations - // https://github.com/golang/go/tree/master/src/crypto/elliptic - CurvePreferences: []tls.CurveID{tls.CurveP256}, - // Use modern tls mode https://wiki.mozilla.org/Security/Server_Side_TLS#Modern_compatibility - NextProtos: []string{"h2", "http/1.1"}, - // https://www.owasp.org/index.php/Transport_Layer_Protection_Cheat_Sheet#Rule_-_Only_Support_Strong_Protocols - MinVersion: tls.VersionTLS12, - // These ciphersuites support Forward Secrecy: https://en.wikipedia.org/wiki/Forward_secrecy - CipherSuites: []uint16{ - tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, - tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, - tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, - tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, - tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, - tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, - }, - } - - // build standard config from server options - if s.TLSCertificate != "" && s.TLSCertificateKey != "" { - httpsServer.TLSConfig.Certificates = make([]tls.Certificate, 1) - httpsServer.TLSConfig.Certificates[0], err = tls.LoadX509KeyPair(string(s.TLSCertificate), string(s.TLSCertificateKey)) - if err != nil { - return err - } - } - - if s.TLSCACertificate != "" { - // include specified CA certificate - caCert, caCertErr := os.ReadFile(string(s.TLSCACertificate)) - if caCertErr != nil { - return caCertErr - } - caCertPool := x509.NewCertPool() - ok := caCertPool.AppendCertsFromPEM(caCert) - if !ok { - return fmt.Errorf("cannot parse CA certificate") - } - httpsServer.TLSConfig.ClientCAs = caCertPool - httpsServer.TLSConfig.ClientAuth = tls.RequireAndVerifyClientCert - } - - // call custom TLS configurator - configureTLS(httpsServer.TLSConfig) - - if len(httpsServer.TLSConfig.Certificates) == 0 && httpsServer.TLSConfig.GetCertificate == nil { - // after standard and custom config are passed, this ends up with no certificate - if s.TLSCertificate == "" { - if s.TLSCertificateKey == "" { - s.Fatalf("the required flags `--tls-certificate` and `--tls-key` were not specified") - } - s.Fatalf("the required flag `--tls-certificate` was not specified") - } - if s.TLSCertificateKey == "" { - s.Fatalf("the required flag `--tls-key` was not specified") - } - // this happens with a wrong custom TLS configurator - s.Fatalf("no certificate was configured for TLS") - } - - configureServer(httpsServer, "https", s.httpsServerL.Addr().String()) - - servers = append(servers, httpsServer) - wg.Add(1) - s.Logf("Serving weaviate at https://%s", s.httpsServerL.Addr()) - go func(l net.Listener) { - defer wg.Done() - if err := httpsServer.Serve(l); err != nil && err != http.ErrServerClosed { - s.Fatalf("%v", err) - } - s.Logf("Stopped serving weaviate at https://%s", l.Addr()) - }(tls.NewListener(s.httpsServerL, httpsServer.TLSConfig)) - } - - wg.Add(1) - go s.handleShutdown(wg, &servers) - - wg.Wait() - return nil -} - -// Listen creates the listeners for the server -func (s *Server) Listen() error { - if s.hasListeners { // already done this - return nil - } - - if s.hasScheme(schemeHTTPS) { - // Use http host if https host wasn't defined - if s.TLSHost == "" { - s.TLSHost = s.Host - } - // Use http listen limit if https listen limit wasn't defined - if s.TLSListenLimit == 0 { - s.TLSListenLimit = s.ListenLimit - } - // Use http tcp keep alive if https tcp keep alive wasn't defined - if int64(s.TLSKeepAlive) == 0 { - s.TLSKeepAlive = s.KeepAlive - } - // Use http read timeout if https read timeout wasn't defined - if int64(s.TLSReadTimeout) == 0 { - s.TLSReadTimeout = s.ReadTimeout - } - // Use http write timeout if https write timeout wasn't defined - if int64(s.TLSWriteTimeout) == 0 { - s.TLSWriteTimeout = s.WriteTimeout - } - } - - if s.hasScheme(schemeUnix) { - domSockListener, err := net.Listen("unix", string(s.SocketPath)) - if err != nil { - return err - } - s.domainSocketL = domSockListener - } - - if s.hasScheme(schemeHTTP) { - listener, err := net.Listen("tcp", net.JoinHostPort(s.Host, strconv.Itoa(s.Port))) - if err != nil { - return err - } - - h, p, err := swag.SplitHostPort(listener.Addr().String()) - if err != nil { - return err - } - s.Host = h - s.Port = p - s.httpServerL = listener - } - - if s.hasScheme(schemeHTTPS) { - tlsListener, err := net.Listen("tcp", net.JoinHostPort(s.TLSHost, strconv.Itoa(s.TLSPort))) - if err != nil { - return err - } - - sh, sp, err := swag.SplitHostPort(tlsListener.Addr().String()) - if err != nil { - return err - } - s.TLSHost = sh - s.TLSPort = sp - s.httpsServerL = tlsListener - } - - s.hasListeners = true - return nil -} - -// Shutdown server and clean up resources -func (s *Server) Shutdown() error { - if atomic.CompareAndSwapInt32(&s.shuttingDown, 0, 1) { - close(s.shutdown) - } - return nil -} - -func (s *Server) handleShutdown(wg *sync.WaitGroup, serversPtr *[]*http.Server) { - // wg.Done must occur last, after s.api.ServerShutdown() - // (to preserve old behaviour) - defer wg.Done() - - <-s.shutdown - - servers := *serversPtr - - ctx, cancel := context.WithTimeout(context.TODO(), s.GracefulTimeout) - defer cancel() - - // first execute the pre-shutdown hook - s.api.PreServerShutdown() - - shutdownChan := make(chan bool) - for i := range servers { - server := servers[i] - go func() { - var success bool - defer func() { - shutdownChan <- success - }() - if err := server.Shutdown(ctx); err != nil { - // Error from closing listeners, or context timeout: - s.Logf("HTTP server Shutdown: %v", err) - } else { - success = true - } - }() - } - - // Wait until all listeners have successfully shut down before calling ServerShutdown - success := true - for range servers { - success = success && <-shutdownChan - } - if success { - s.api.ServerShutdown() - } -} - -// GetHandler returns a handler useful for testing -func (s *Server) GetHandler() http.Handler { - return s.handler -} - -// SetHandler allows for setting a http handler on this server -func (s *Server) SetHandler(handler http.Handler) { - s.handler = handler -} - -// UnixListener returns the domain socket listener -func (s *Server) UnixListener() (net.Listener, error) { - if !s.hasListeners { - if err := s.Listen(); err != nil { - return nil, err - } - } - return s.domainSocketL, nil -} - -// HTTPListener returns the http listener -func (s *Server) HTTPListener() (net.Listener, error) { - if !s.hasListeners { - if err := s.Listen(); err != nil { - return nil, err - } - } - return s.httpServerL, nil -} - -// TLSListener returns the https listener -func (s *Server) TLSListener() (net.Listener, error) { - if !s.hasListeners { - if err := s.Listen(); err != nil { - return nil, err - } - } - return s.httpsServerL, nil -} - -func handleInterrupt(once *sync.Once, s *Server) { - once.Do(func() { - for range s.interrupt { - if s.interrupted { - s.Logf("Server already shutting down") - continue - } - s.interrupted = true - s.Logf("Shutting down... ") - if err := s.Shutdown(); err != nil { - s.Logf("HTTP server Shutdown: %v", err) - } - } - }) -} - -func signalNotify(interrupt chan<- os.Signal) { - signal.Notify(interrupt, syscall.SIGINT, syscall.SIGTERM) -} diff --git a/adapters/handlers/rest/state/state.go b/adapters/handlers/rest/state/state.go deleted file mode 100644 index 3c2e6ad8bfc323c7a0e4a7015bdab647aff09b49..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/state/state.go +++ /dev/null @@ -1,88 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package state - -import ( - "context" - "net/http" - "sync" - - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/adapters/handlers/graphql" - "github.com/weaviate/weaviate/adapters/repos/classifications" - "github.com/weaviate/weaviate/adapters/repos/db" - "github.com/weaviate/weaviate/usecases/auth/authentication/anonymous" - "github.com/weaviate/weaviate/usecases/auth/authentication/apikey" - "github.com/weaviate/weaviate/usecases/auth/authentication/oidc" - "github.com/weaviate/weaviate/usecases/auth/authorization" - "github.com/weaviate/weaviate/usecases/backup" - "github.com/weaviate/weaviate/usecases/cluster" - "github.com/weaviate/weaviate/usecases/config" - "github.com/weaviate/weaviate/usecases/locks" - "github.com/weaviate/weaviate/usecases/modules" - "github.com/weaviate/weaviate/usecases/monitoring" - "github.com/weaviate/weaviate/usecases/objects" - "github.com/weaviate/weaviate/usecases/replica" - "github.com/weaviate/weaviate/usecases/scaler" - "github.com/weaviate/weaviate/usecases/schema" - "github.com/weaviate/weaviate/usecases/sharding" - "github.com/weaviate/weaviate/usecases/traverser" -) - -// State is the only source of application-wide state -// NOTE: This is not true yet, see gh-723 -// TODO: remove dependencies to anything that's not an ent or uc -type State struct { - OIDC *oidc.Client - AnonymousAccess *anonymous.Client - APIKey *apikey.Client - Authorizer authorization.Authorizer - ServerConfig *config.WeaviateConfig - Locks locks.ConnectorSchemaLock - Logger *logrus.Logger - gqlMutex sync.Mutex - GraphQL graphql.GraphQL - Modules *modules.Provider - SchemaManager *schema.Manager - Scaler *scaler.Scaler - Cluster *cluster.State - RemoteIndexIncoming *sharding.RemoteIndexIncoming - RemoteNodeIncoming *sharding.RemoteNodeIncoming - RemoteReplicaIncoming *replica.RemoteReplicaIncoming - Traverser *traverser.Traverser - - ClassificationRepo *classifications.DistributedRepo - Metrics *monitoring.PrometheusMetrics - BackupManager *backup.Handler - DB *db.DB - BatchManager *objects.BatchManager - ClusterHttpClient *http.Client - ReindexCtxCancel context.CancelFunc -} - -// GetGraphQL is the safe way to retrieve GraphQL from the state as it can be -// replaced at runtime. Instead of passing appState.GraphQL to your adapters, -// pass appState itself which you can abstract with a local interface such as: -// -// type gqlProvider interface { GetGraphQL graphql.GraphQL } -func (s *State) GetGraphQL() graphql.GraphQL { - s.gqlMutex.Lock() - gql := s.GraphQL - s.gqlMutex.Unlock() - return gql -} - -func (s *State) SetGraphQL(gql graphql.GraphQL) { - s.gqlMutex.Lock() - s.GraphQL = gql - s.gqlMutex.Unlock() -} diff --git a/adapters/handlers/rest/swagger_middleware/swagger_middleware.go b/adapters/handlers/rest/swagger_middleware/swagger_middleware.go deleted file mode 100644 index 88ba4b1839d62f6684420e11d6504b0fdb6980bf..0000000000000000000000000000000000000000 --- a/adapters/handlers/rest/swagger_middleware/swagger_middleware.go +++ /dev/null @@ -1,154 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package swagger_middleware - -import ( - "fmt" - "html/template" - "net/http" - "strings" -) - -const swaggerUIVersion = "3.19.4" - -type templateData struct { - Prefix string - APIKey string - APIToken string -} - -func AddMiddleware(swaggerJSON []byte, next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if strings.HasPrefix(r.URL.Path, "/v1/swagger.json") && r.Method == http.MethodGet { - w.Header().Set("Content-Type", "application/json") - w.Write(swaggerJSON) - } else if strings.HasPrefix(r.URL.Path, "/v1/swagger") && r.Method == http.MethodGet { - renderSwagger(w, r) - } else { - next.ServeHTTP(w, r) - } - }) -} - -// renderswagger renders the swagger GUI -func renderSwagger(w http.ResponseWriter, r *http.Request) { - w.Header().Set("WWW-Authenticate", `Basic realm="Provide your key and token (as username as password respectively)"`) - - user, password, authOk := r.BasicAuth() - if !authOk { - http.Error(w, "Not authorized", 401) - return - } - - t := template.New("Swagger") - t, err := t.Parse(swaggerTemplate) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - // Create result string - d := templateData{ - Prefix: fmt.Sprintf("https://cdn.jsdelivr.net/npm/swagger-ui-dist@%s", swaggerUIVersion), - APIKey: user, - APIToken: password, - } - - err = t.ExecuteTemplate(w, "index", d) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - } -} - -// tmpl is the page template to render GraphiQL -const swaggerTemplate = ` -{{ define "index" }} - - - - - - Weaviate API - - - - - - - -
- - - - - - -{{ end }} -` diff --git a/adapters/repos/classifications/distributed_repo.go b/adapters/repos/classifications/distributed_repo.go deleted file mode 100644 index 0477248877d2d4d41a975946a80b882bac34c3a1..0000000000000000000000000000000000000000 --- a/adapters/repos/classifications/distributed_repo.go +++ /dev/null @@ -1,121 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package classifications - -import ( - "context" - "sync" - "time" - - "github.com/go-openapi/strfmt" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/usecases/classification" - "github.com/weaviate/weaviate/usecases/cluster" -) - -const DefaultTxTTL = 60 * time.Second - -type DistributedRepo struct { - sync.RWMutex - txRemote *cluster.TxManager - localRepo localRepo -} - -type localRepo interface { - Get(ctx context.Context, id strfmt.UUID) (*models.Classification, error) - Put(ctx context.Context, classification models.Classification) error -} - -func NewDistributeRepo(remoteClient cluster.Client, - memberLister cluster.MemberLister, localRepo localRepo, - logger logrus.FieldLogger, -) *DistributedRepo { - broadcaster := cluster.NewTxBroadcaster(memberLister, remoteClient) - txRemote := cluster.NewTxManager(broadcaster, &dummyTxPersistence{}, logger) - txRemote.StartAcceptIncoming() - repo := &DistributedRepo{ - txRemote: txRemote, - localRepo: localRepo, - } - - repo.txRemote.SetCommitFn(repo.incomingCommit) - - return repo -} - -func (r *DistributedRepo) Get(ctx context.Context, - id strfmt.UUID, -) (*models.Classification, error) { - r.RLock() - defer r.RUnlock() - - return r.localRepo.Get(ctx, id) -} - -func (r *DistributedRepo) Put(ctx context.Context, - pl models.Classification, -) error { - r.Lock() - defer r.Unlock() - - tx, err := r.txRemote.BeginTransaction(ctx, classification.TransactionPut, - classification.TransactionPutPayload{ - Classification: pl, - }, DefaultTxTTL) - if err != nil { - return errors.Wrap(err, "open cluster-wide transaction") - } - - err = r.txRemote.CommitWriteTransaction(ctx, tx) - if err != nil { - return errors.Wrap(err, "commit cluster-wide transaction") - } - - return r.localRepo.Put(ctx, pl) -} - -func (r *DistributedRepo) incomingCommit(ctx context.Context, - tx *cluster.Transaction, -) error { - if tx.Type != classification.TransactionPut { - return errors.Errorf("unrecognized tx type: %s", tx.Type) - } - - return r.localRepo.Put(ctx, tx.Payload.(classification.TransactionPutPayload). - Classification) -} - -func (r *DistributedRepo) TxManager() *cluster.TxManager { - return r.txRemote -} - -// NOTE: classifications do not yet make use of the new durability guarantees -// introduced by the txManager as part of v1.21.3. The reasoning behind this is -// that the classification itself is not crash-safe anyway, so there is no -// point. We need to decide down the line what to do with this? It is a rarely -// used, but not used feature. For now we are not aware of anyone having any -// issues with its stability. -type dummyTxPersistence struct{} - -func (d *dummyTxPersistence) StoreTx(ctx context.Context, tx *cluster.Transaction) error { - return nil -} - -func (d *dummyTxPersistence) DeleteTx(ctx context.Context, txID string) error { - return nil -} - -func (d *dummyTxPersistence) IterateAll(ctx context.Context, cb func(tx *cluster.Transaction)) error { - return nil -} diff --git a/adapters/repos/classifications/repo.go b/adapters/repos/classifications/repo.go deleted file mode 100644 index 147fe28fe90d3266139f35c061be0356e67e282d..0000000000000000000000000000000000000000 --- a/adapters/repos/classifications/repo.go +++ /dev/null @@ -1,114 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package classifications - -import ( - "context" - "encoding/json" - "fmt" - "os" - - "github.com/go-openapi/strfmt" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/usecases/classification" - bolt "go.etcd.io/bbolt" -) - -var classificationsBucket = []byte("classifications") - -type Repo struct { - logger logrus.FieldLogger - baseDir string - db *bolt.DB -} - -func NewRepo(baseDir string, logger logrus.FieldLogger) (*Repo, error) { - r := &Repo{ - baseDir: baseDir, - logger: logger, - } - - err := r.init() - return r, err -} - -func (r *Repo) DBPath() string { - return fmt.Sprintf("%s/classifications.db", r.baseDir) -} - -func (r *Repo) keyFromID(id strfmt.UUID) []byte { - return []byte(id) -} - -func (r *Repo) init() error { - if err := os.MkdirAll(r.baseDir, 0o777); err != nil { - return errors.Wrapf(err, "create root path directory at %s", r.baseDir) - } - - boltdb, err := bolt.Open(r.DBPath(), 0o600, nil) - if err != nil { - return errors.Wrapf(err, "open bolt at %s", r.DBPath()) - } - - err = boltdb.Update(func(tx *bolt.Tx) error { - if _, err := tx.CreateBucketIfNotExists(classificationsBucket); err != nil { - return errors.Wrapf(err, "create classifications bucket '%s'", - string(helpers.ObjectsBucket)) - } - return nil - }) - if err != nil { - return errors.Wrapf(err, "create bolt buckets") - } - - r.db = boltdb - - return nil -} - -func (r *Repo) Put(ctx context.Context, classification models.Classification) error { - classificationJSON, err := json.Marshal(classification) - if err != nil { - return errors.Wrap(err, "marshal classification to JSON") - } - - return r.db.Update(func(tx *bolt.Tx) error { - b := tx.Bucket(classificationsBucket) - return b.Put(r.keyFromID(classification.ID), classificationJSON) - }) -} - -func (r *Repo) Get(ctx context.Context, id strfmt.UUID) (*models.Classification, error) { - var classificationJSON []byte - r.db.View(func(tx *bolt.Tx) error { - b := tx.Bucket(classificationsBucket) - classificationJSON = b.Get(r.keyFromID(id)) - return nil - }) - - if len(classificationJSON) == 0 { - return nil, nil - } - - var c models.Classification - err := json.Unmarshal(classificationJSON, &c) - if err != nil { - return nil, errors.Wrapf(err, "parse classification from JSON") - } - - return &c, nil -} - -var _ = classification.Repo(&Repo{}) diff --git a/adapters/repos/classifications/repo_integration_test.go b/adapters/repos/classifications/repo_integration_test.go deleted file mode 100644 index 211c5cf760874676f98d95584afe5e2d52ca82e3..0000000000000000000000000000000000000000 --- a/adapters/repos/classifications/repo_integration_test.go +++ /dev/null @@ -1,78 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package classifications - -import ( - "context" - "testing" - - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" -) - -func Test_ClassificationsRepo(t *testing.T) { - dirName := t.TempDir() - - logger, _ := test.NewNullLogger() - - r, err := NewRepo(dirName, logger) - require.Nil(t, err) - _ = r - - t.Run("asking for a non-existing classification", func(t *testing.T) { - res, err := r.Get(context.Background(), "wrong-id") - require.Nil(t, err) - assert.Nil(t, res) - }) - - t.Run("storing classifications", func(t *testing.T) { - err := r.Put(context.Background(), exampleOne()) - require.Nil(t, err) - - err = r.Put(context.Background(), exampleTwo()) - require.Nil(t, err) - }) - - t.Run("retrieving stored classifications", func(t *testing.T) { - expectedOne := exampleOne() - expectedTwo := exampleTwo() - - res, err := r.Get(context.Background(), expectedOne.ID) - require.Nil(t, err) - assert.Equal(t, &expectedOne, res) - - res, err = r.Get(context.Background(), expectedTwo.ID) - require.Nil(t, err) - assert.Equal(t, &expectedTwo, res) - }) -} - -func exampleOne() models.Classification { - return models.Classification{ - ID: "01ed111a-919c-4dd5-ab9e-7b247b11e18c", - Class: "ExampleClassOne", - BasedOnProperties: []string{"prop1"}, - } -} - -func exampleTwo() models.Classification { - return models.Classification{ - ID: "4fbaebf3-41a9-414b-ac1d-433d74d4ef2c", - Class: "ExampleClassTwo", - BasedOnProperties: []string{"prop2"}, - } -} diff --git a/adapters/repos/db/aggregations_fixtures_for_test.go b/adapters/repos/db/aggregations_fixtures_for_test.go deleted file mode 100644 index 43a3ec3ad1e962f4a822c3f3f4196fd7273be053..0000000000000000000000000000000000000000 --- a/adapters/repos/db/aggregations_fixtures_for_test.go +++ /dev/null @@ -1,367 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package db - -import ( - "fmt" - - "github.com/go-openapi/strfmt" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - enthnsw "github.com/weaviate/weaviate/entities/vectorindex/hnsw" -) - -var productClass = &models.Class{ - Class: "AggregationsTestProduct", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, -} - -var companyClass = &models.Class{ - Class: "AggregationsTestCompany", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{ - { - Name: "sector", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "location", - DataType: []string{"text"}, - Tokenization: "word", - }, - { - Name: "dividendYield", - DataType: []string{"number"}, - }, - { - Name: "price", - DataType: []string{"int"}, // unrealistic for this to be an int, but - // we've already tested another number prop ;-) - }, - { - Name: "listedInIndex", - DataType: []string{"boolean"}, - }, - { - Name: "makesProduct", - DataType: []string{"AggregationsTestProduct"}, - }, - }, -} - -var arrayTypesClass = &models.Class{ - Class: "AggregationsTestArrayTypes", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{ - { - Name: "strings", - DataType: schema.DataTypeTextArray.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "numbers", - DataType: []string{"number[]"}, - }, - }, -} - -var customerClass = &models.Class{ - Class: "AggregationsTestCustomer", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{ - { - Name: "internalId", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "timeArrived", - DataType: []string{"date"}, - }, - { - Name: "countryOfOrigin", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, -} - -var products = []map[string]interface{}{ - { - "name": "Superbread", - }, -} - -var productsIds = []strfmt.UUID{ - "1295c052-263d-4aae-99dd-920c5a370d06", -} - -// company objects are imported not just once each, but each is imported -// importFactor times. This should even out shard imbalances a bit better. -var importFactor = 10 - -var companies = []map[string]interface{}{ - { - "sector": "Financials", - "location": "New York", - "dividendYield": 1.3, - "price": int64(150), - "listedInIndex": true, - }, - { - "sector": "Financials", - "location": "New York", - "dividendYield": 4.0, - "price": int64(600), - "listedInIndex": true, - }, - { - "sector": "Financials", - "location": "San Francisco", - "dividendYield": 1.3, - "price": int64(47), - "listedInIndex": true, - }, - { - "sector": "Food", - "location": "Atlanta", - "dividendYield": 1.3, - "price": int64(160), - "listedInIndex": true, - }, - { - "sector": "Food", - "location": "Atlanta", - "dividendYield": 2.0, - "price": int64(70), - "listedInIndex": true, - }, - { - "sector": "Food", - "location": "Los Angeles", - "dividendYield": 0.0, - "price": int64(800), - "listedInIndex": false, - }, - { - "sector": "Food", - "location": "Detroit", - "dividendYield": 8.0, - "price": int64(10), - "listedInIndex": true, - "makesProduct": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", productsIds[0])), - }, - }, - }, - { - "sector": "Food", - "location": "San Francisco", - "dividendYield": 0.0, - "price": int64(200), - "listedInIndex": true, - }, - { - "sector": "Food", - "location": "New York", - "dividendYield": 1.1, - "price": int64(70), - "listedInIndex": true, - }, -} - -// Use fixed ids to make the test deterministic. The length of this must match -// the len(companies)*importFactor These are somewhat carefully arranged to -// make sure that we prevent the flakiness that was described in -// https://github.com/weaviate/weaviate/issues/1884 -var companyIDs = []strfmt.UUID{ - "9ee7640f-b4fc-45f1-b502-580e79062c99", - "f745485b-f6ef-4785-bd54-574cc4923899", - "0109289b-88d5-48d5-9f3e-1e5edb7e56c1", - "ce7c2930-2af8-44ec-8789-eceee930c37d", - "3f7b1ea2-e6e8-49d4-b44c-30916dbab43a", - "ed3b417f-2b7e-456e-9599-744541728950", - "175c22ea-190a-4b8c-9fd0-fbffa5b1b8be", - "e0165a52-f504-47d3-b947-b3829eca3563", - "19ede86d-86d6-4ea8-9ae3-a10a1607b433", - "034c8d0b-50d7-4753-811c-abbfad44c653", - "99564c6b-8529-4f56-9c3f-f808385f034a", - "adc362ea-105d-4544-b981-dedc100d25b9", - "21e1dc5b-49e1-4d35-acb4-be969e4e3a30", - "cb278f74-c632-4460-ae78-2c2c140c0e12", - "4d552f33-552a-41aa-b732-da3d0f708b79", - "faf509ad-bdb6-4aa1-ae9f-e9f410210419", - "59992957-65cf-44bd-8894-d8b2364f080f", - "529217e6-9ec3-41c6-bb6d-13e8f380960a", - "058ec709-782d-4b58-a38f-01593d97f181", - "d027204b-805c-46ae-8a61-7d35f9c6eaba", - "0fce8fea-6ca7-4c80-bc81-55d26e1fd0bd", - "1f4832c7-d164-441e-b197-aa193b4d128f", - "15ad080d-49fc-4b8b-b621-d47f98aa5fdb", - "cb40966d-963d-4283-94be-7da5de70992e", - "6516f7d9-c505-40b3-94de-a42498eea22d", - "9dbcbd08-1067-4bec-84a4-4f3ad19286d3", - "dabd68eb-27b9-462b-a271-300058c5798b", - "9a33f431-cb28-49ae-b9c9-94e97f752a2a", - "4aa27f5c-f475-444b-9279-94b8a5da14c9", - "e71cf490-9a59-4475-80c1-b6c872f3b33c", - "a8e7e8bf-237b-4d95-99be-6af332bdf942", - "08c239c3-e19e-4d88-b0fd-861853dd5e36", - "0209ab7c-2573-4d66-9917-942450e02750", - "0a71a4da-d9c9-423e-8dd8-ebd5c2a86a2d", - "f312aa16-992e-4505-82aa-04da30ea5fe3", - "5b9f9449-1336-44e3-88f9-979f3c752bd7", - "dcec96ab-9377-4272-8a48-07568c4ed533", - "3f28352d-9960-4251-aa05-ce4c612e1ab7", - "4a08101e-f280-41f9-9e7b-fe12d7216c3c", - "0dd7383f-c71b-483c-9253-e180f8763405", - "cfb83c85-cf8f-49da-952f-5bd954b7e616", - "b016bb0f-9e07-4d40-9878-a6bbaa67d866", - "311d7423-552b-4d4c-b7b6-cdcd15e1009b", - "895877d6-9cf3-4d79-989e-4d89f6867e09", - "92bdb79c-6870-44e2-ab71-c3137c78cb2d", - "b16cb9c4-5a6c-444c-bbf5-7b0ef2c1ac12", - "efb09d97-09c4-4976-aa14-abfd520c114d", - "6431f59d-9ed8-4963-9ed7-7336a5795d8b", - "1ad26844-6f6b-4007-834d-09ec8668fe7b", - "6d83b349-6ec8-49bb-b438-7f29a16216d3", - "68156360-1cae-406e-8baf-177178f0815e", - "3c726a50-ec82-4828-8967-f704477dfef7", - "46f702b2-e1c3-4e4a-868e-10ec1e260c75", - "51ff679a-87d8-4bef-830d-327e7b4c8f8e", - "aea6fc5c-8eb0-4cd7-8cfe-1285239d16bd", - "b70bbf68-5ebc-4315-9819-deb65b241f3f", - "6069853f-8822-434f-9d59-c881800e0a27", - "9a287ef6-6920-4d01-a44a-7561d2fb627d", - "fa057d95-9ba8-418a-aeb2-fe4b2acd31b0", - "9b0fb28f-21f1-4df1-a55b-67bf6be79911", - "044403fb-25f6-461f-ad92-8f9533908070", - "35d09151-c469-4901-8092-2114060cb86b", - "85884aa2-5d0b-4dff-80f6-8ca7cbab9fef", - "bd36a31a-f14e-4040-ad11-0ec5b6217b6a", - "fe20620f-c612-4712-9475-d4cfc59e8bba", - "09ba0773-e81d-4cb9-968a-552e1cbaf143", - "7a7378a5-2d05-4457-b2d4-1fe068535167", - "6867d1e5-2d30-4f91-90d2-2b453ffd5cd5", - "2fef1b16-3dd1-4ad5-bc55-cf8f9783b40c", - "0590f51b-7c9f-41a0-b81e-cdbc205ebdd9", - "7ed55b94-86d5-440a-9a8a-5f83dabcb69b", - "2daca92a-c8a6-4ab4-a528-3d99ce6f72f2", - "24187e67-947d-436d-ae7f-d20b03874b56", - "864ff42d-00fe-44a8-8163-8af459dc1c0c", - "0c2cc9a5-089a-4d10-882a-837c154117ea", - "fb256f18-e812-4355-b41a-c69d933f2a61", - "b631c4df-8229-43c0-9e5e-189c5d666ac2", - "8da03018-3272-4bd3-987c-1dd1e807bc1d", - "bf736b76-fccc-4d1b-8d9f-2e78fdb0d972", - "1fc9dffc-da23-4b99-8330-44c5598919db", - "1ed74402-bc81-4245-8275-c2862a4d6a86", - "6a91adb4-23df-43bd-9564-f97e76382a52", - "d1661202-c568-4032-8ec6-99fe4238de84", - "e4b4186d-f02e-47d7-8214-d3854ee475fd", - "664e6157-bfcc-4513-8f04-c095d3ecb2d5", - "b3b9951f-0867-453d-bf20-9f22bdb5a38b", - "52adfeae-ab75-4250-ae45-61af9e231e86", - "e994c378-ac0b-4f08-bd56-462990be36dd", - "a012f65a-28a7-4005-95bb-f55783bcdda0", - "0cbc7fb6-843c-4aad-b540-e37ddb7c84c6", -} - -var arrayTypes = []map[string]interface{}{ - { - "strings": []string{"a", "b", "c"}, - "numbers": []float64{1.0, 2.0, 2.0, 3.0, 3.0}, - }, - { - "strings": []string{"a"}, - "numbers": []float64{1.0, 2.0}, - }, -} - -var customers = []map[string]interface{}{ - { - "internalId": "customer 1", - "countryOfOrigin": "US", - "timeArrived": mustStringToTime("2022-06-16T17:30:17.231346Z"), - "isNewCustomer": false, - }, - { - "internalId": "customer 2", - "countryOfOrigin": "US", - "timeArrived": mustStringToTime("2022-06-16T17:30:17.231346Z"), - "isNewCustomer": false, - }, - { - "internalId": "customer 3", - "countryOfOrigin": "US", - "timeArrived": mustStringToTime("2022-06-16T17:30:17.231346Z"), - "isNewCustomer": false, - }, - { - "internalId": "customer 4", - "countryOfOrigin": "US", - "timeArrived": mustStringToTime("2022-06-16T17:30:20.123546Z"), - "isNewCustomer": true, - }, - { - "internalId": "customer 5", - "countryOfOrigin": "US", - "timeArrived": mustStringToTime("2022-06-16T17:30:20.123546Z"), - "isNewCustomer": true, - }, - { - "internalId": "customer 6", - "countryOfOrigin": "US", - "timeArrived": mustStringToTime("2022-06-16T17:30:22.112435Z"), - "isNewCustomer": false, - }, - { - "internalId": "customer 7", - "countryOfOrigin": "US", - "timeArrived": mustStringToTime("2022-06-16T17:30:23.754272Z"), - "isNewCustomer": false, - }, - { - "internalId": "customer 8", - "countryOfOrigin": "US", - "timeArrived": mustStringToTime("2022-06-16T17:30:24.325698Z"), - "isNewCustomer": true, - }, - { - "internalId": "customer 9", - "countryOfOrigin": "US", - "timeArrived": mustStringToTime("2022-06-16T17:30:25.524536Z"), - "isNewCustomer": false, - }, - { - "internalId": "customer 10", - "countryOfOrigin": "US", - "timeArrived": mustStringToTime("2022-06-16T17:30:26.451235Z"), - "isNewCustomer": true, - }, -} diff --git a/adapters/repos/db/aggregations_integration_test.go b/adapters/repos/db/aggregations_integration_test.go deleted file mode 100644 index 75f75141ee5bed1569e83cac85067dc08f940101..0000000000000000000000000000000000000000 --- a/adapters/repos/db/aggregations_integration_test.go +++ /dev/null @@ -1,2266 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package db - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/go-openapi/strfmt" - "github.com/google/uuid" - "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/aggregation" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" -) - -func Test_Aggregations(t *testing.T) { - dirName := t.TempDir() - - shardState := singleShardState() - logger := logrus.New() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: shardState, - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - migrator := NewMigrator(repo, logger) - - t.Run("prepare test schema and data ", - prepareCompanyTestSchemaAndData(repo, migrator, schemaGetter)) - - t.Run("numerical aggregations with grouping", - testNumericalAggregationsWithGrouping(repo, true)) - - t.Run("numerical aggregations without grouping (formerly Meta)", - testNumericalAggregationsWithoutGrouping(repo, true)) - - t.Run("numerical aggregations with filters", - testNumericalAggregationsWithFilters(repo)) - - t.Run("date aggregations with grouping", - testDateAggregationsWithGrouping(repo, true)) - - t.Run("date aggregations without grouping", - testDateAggregationsWithoutGrouping(repo, true)) - - t.Run("date aggregations with filters", - testDateAggregationsWithFilters(repo)) - - t.Run("clean up", - cleanupCompanyTestSchemaAndData(repo, migrator)) -} - -func Test_Aggregations_MultiShard(t *testing.T) { - dirName := t.TempDir() - - shardState := fixedMultiShardState() - logger := logrus.New() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: shardState, - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - migrator := NewMigrator(repo, logger) - - t.Run("prepare test schema and data ", - prepareCompanyTestSchemaAndData(repo, migrator, schemaGetter)) - - t.Run("numerical aggregations with grouping", - testNumericalAggregationsWithGrouping(repo, false)) - - t.Run("numerical aggregations without grouping (formerly Meta)", - testNumericalAggregationsWithoutGrouping(repo, false)) - - t.Run("numerical aggregations with filters", - testNumericalAggregationsWithFilters(repo)) - - t.Run("date aggregations with grouping", - testDateAggregationsWithGrouping(repo, true)) - - t.Run("date aggregations without grouping", - testDateAggregationsWithoutGrouping(repo, true)) - - t.Run("date aggregations with filters", - testDateAggregationsWithFilters(repo)) - - t.Run("clean up", - cleanupCompanyTestSchemaAndData(repo, migrator)) -} - -func prepareCompanyTestSchemaAndData(repo *DB, - migrator *Migrator, schemaGetter *fakeSchemaGetter, -) func(t *testing.T) { - return func(t *testing.T) { - schema := schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{ - productClass, - companyClass, - arrayTypesClass, - customerClass, - }, - }, - } - - schemaGetter.schema = schema - - t.Run("creating the class", func(t *testing.T) { - require.Nil(t, - migrator.AddClass(context.Background(), productClass, schemaGetter.shardState)) - require.Nil(t, - migrator.AddClass(context.Background(), companyClass, schemaGetter.shardState)) - require.Nil(t, - migrator.AddClass(context.Background(), arrayTypesClass, schemaGetter.shardState)) - require.Nil(t, - migrator.AddClass(context.Background(), customerClass, schemaGetter.shardState)) - }) - - schemaGetter.schema = schema - - t.Run("import products", func(t *testing.T) { - for i, schema := range products { - t.Run(fmt.Sprintf("importing product %d", i), func(t *testing.T) { - fixture := models.Object{ - Class: productClass.Class, - ID: productsIds[i], - Properties: schema, - } - require.Nil(t, - repo.PutObject(context.Background(), &fixture, []float32{0.1, 0.2, 0.01, 0.2}, nil)) - }) - } - }) - - t.Run("import companies", func(t *testing.T) { - for j := 0; j < importFactor; j++ { - for i, schema := range companies { - t.Run(fmt.Sprintf("importing company %d", i), func(t *testing.T) { - fixture := models.Object{ - Class: companyClass.Class, - ID: companyIDs[j*(importFactor-1)+i], - Properties: schema, - } - - require.Nil(t, - repo.PutObject(context.Background(), &fixture, []float32{0.1, 0.1, 0.1, 0.1}, nil)) - }) - } - } - }) - - t.Run("import array types", func(t *testing.T) { - for i, schema := range arrayTypes { - t.Run(fmt.Sprintf("importing array type %d", i), func(t *testing.T) { - fixture := models.Object{ - Class: arrayTypesClass.Class, - ID: strfmt.UUID(uuid.Must(uuid.NewRandom()).String()), - Properties: schema, - } - require.Nil(t, - repo.PutObject(context.Background(), &fixture, []float32{0.1, 0.1, 0.1, 0.1}, nil)) - }) - } - }) - - t.Run("import customers", func(t *testing.T) { - for i, schema := range customers { - t.Run(fmt.Sprintf("importing customer #%d", i), func(t *testing.T) { - fixture := models.Object{ - Class: customerClass.Class, - ID: strfmt.UUID(uuid.Must(uuid.NewRandom()).String()), - Properties: schema, - } - require.Nil(t, - repo.PutObject(context.Background(), &fixture, []float32{0.1, 0.1, 0.1, 0.1}, nil)) - }) - } - }) - } -} - -func cleanupCompanyTestSchemaAndData(repo *DB, - migrator *Migrator, -) func(t *testing.T) { - return func(t *testing.T) { - assert.Nil(t, repo.Shutdown(context.Background())) - } -} - -func testNumericalAggregationsWithGrouping(repo *DB, exact bool) func(t *testing.T) { - return func(t *testing.T) { - epsilon := 0.1 - if !exact { - epsilon = 1.0 - } - - t.Run("single field, single aggregator", func(t *testing.T) { - params := aggregation.Params{ - ClassName: schema.ClassName(companyClass.Class), - GroupBy: &filters.Path{ - Class: schema.ClassName(companyClass.Class), - Property: schema.PropertyName("sector"), - }, - IncludeMetaCount: true, - Properties: []aggregation.ParamProperty{ - { - Name: schema.PropertyName("dividendYield"), - Aggregators: []aggregation.Aggregator{aggregation.MeanAggregator}, - }, - }, - } - - res, err := repo.Aggregate(context.Background(), params) - require.Nil(t, err) - - expectedResult := &aggregation.Result{ - Groups: []aggregation.Group{ - { - Count: 60, - GroupedBy: &aggregation.GroupedBy{ - Path: []string{"sector"}, - Value: "Food", - }, - Properties: map[string]aggregation.Property{ - "dividendYield": { - Type: aggregation.PropertyTypeNumerical, - NumericalAggregations: map[string]interface{}{ - "mean": 2.066666666666666, - }, - }, - }, - }, - { - Count: 30, - GroupedBy: &aggregation.GroupedBy{ - Path: []string{"sector"}, - Value: "Financials", - }, - Properties: map[string]aggregation.Property{ - "dividendYield": { - Type: aggregation.PropertyTypeNumerical, - NumericalAggregations: map[string]interface{}{ - "mean": 2.1999999999999999, - }, - }, - }, - }, - }, - } - - require.Equal(t, len(expectedResult.Groups), len(res.Groups)) - - for i := 0; i <= 1; i++ { - assert.Equal(t, expectedResult.Groups[i].Count, - res.Groups[i].Count) - - expectedDivYield := expectedResult.Groups[i].Properties["dividendYield"] - actualDivYield := res.Groups[i].Properties["dividendYield"] - - assert.InEpsilon(t, expectedDivYield.NumericalAggregations["mean"], - actualDivYield.NumericalAggregations["mean"], epsilon) - } - }) - - t.Run("grouping by a non-numerical, non-string prop", func(t *testing.T) { - params := aggregation.Params{ - ClassName: schema.ClassName(companyClass.Class), - GroupBy: &filters.Path{ - Class: schema.ClassName(companyClass.Class), - Property: schema.PropertyName("listedInIndex"), - }, - Properties: []aggregation.ParamProperty{ - { - Name: schema.PropertyName("dividendYield"), - Aggregators: []aggregation.Aggregator{aggregation.MeanAggregator}, - }, - }, - } - - res, err := repo.Aggregate(context.Background(), params) - require.Nil(t, err) - - expectedResult := &aggregation.Result{ - Groups: []aggregation.Group{ - { - Count: 80, - GroupedBy: &aggregation.GroupedBy{ - Path: []string{"listedInIndex"}, - Value: true, - }, - Properties: map[string]aggregation.Property{ - "dividendYield": { - Type: aggregation.PropertyTypeNumerical, - NumericalAggregations: map[string]interface{}{ - "mean": 2.375, - }, - }, - }, - }, - { - Count: 10, - GroupedBy: &aggregation.GroupedBy{ - Path: []string{"listedInIndex"}, - Value: false, - }, - Properties: map[string]aggregation.Property{ - "dividendYield": { - Type: aggregation.PropertyTypeNumerical, - NumericalAggregations: map[string]interface{}{ - "mean": 0.0, - }, - }, - }, - }, - }, - } - - // there is now way to use InEpsilon or InDelta on nested structs with - // testify, so unfortunately we have to do a manual deep equal: - assert.Equal(t, len(res.Groups), len(expectedResult.Groups)) - assert.Equal(t, expectedResult.Groups[0].Count, res.Groups[0].Count) - assert.Equal(t, expectedResult.Groups[0].GroupedBy, res.Groups[0].GroupedBy) - assert.InEpsilon(t, expectedResult.Groups[0].Properties["dividendYield"]. - NumericalAggregations["mean"], - res.Groups[0].Properties["dividendYield"].NumericalAggregations["mean"], - epsilon) - assert.Equal(t, len(res.Groups), len(expectedResult.Groups)) - assert.Equal(t, expectedResult.Groups[1].Count, res.Groups[1].Count) - assert.Equal(t, expectedResult.Groups[1].GroupedBy, res.Groups[1].GroupedBy) - assert.InDelta(t, expectedResult.Groups[1].Properties["dividendYield"]. - NumericalAggregations["mean"], - res.Groups[1].Properties["dividendYield"].NumericalAggregations["mean"], - epsilon) - }) - - t.Run("multiple fields, multiple aggregators, grouped by string", func(t *testing.T) { - params := aggregation.Params{ - ClassName: schema.ClassName(companyClass.Class), - GroupBy: &filters.Path{ - Class: schema.ClassName(companyClass.Class), - Property: schema.PropertyName("sector"), - }, - Properties: []aggregation.ParamProperty{ - { - Name: schema.PropertyName("dividendYield"), - Aggregators: []aggregation.Aggregator{ - aggregation.MeanAggregator, - aggregation.MaximumAggregator, - aggregation.MinimumAggregator, - aggregation.SumAggregator, - aggregation.ModeAggregator, - aggregation.MedianAggregator, - aggregation.CountAggregator, - aggregation.TypeAggregator, - }, - }, - { - Name: schema.PropertyName("price"), - Aggregators: []aggregation.Aggregator{ - aggregation.TypeAggregator, - aggregation.MeanAggregator, - aggregation.MaximumAggregator, - aggregation.MinimumAggregator, - aggregation.SumAggregator, - // aggregation.ModeAggregator, // ignore as there is no most common value - aggregation.MedianAggregator, - aggregation.CountAggregator, - }, - }, - { - Name: schema.PropertyName("listedInIndex"), - Aggregators: []aggregation.Aggregator{ - aggregation.TypeAggregator, - aggregation.PercentageTrueAggregator, - aggregation.PercentageFalseAggregator, - aggregation.TotalTrueAggregator, - aggregation.TotalFalseAggregator, - }, - }, - { - Name: schema.PropertyName("location"), - Aggregators: []aggregation.Aggregator{ - aggregation.TypeAggregator, - aggregation.NewTopOccurrencesAggregator(ptInt(5)), - }, - }, - }, - } - - res, err := repo.Aggregate(context.Background(), params) - require.Nil(t, err) - - expectedResult := &aggregation.Result{ - Groups: []aggregation.Group{ - { - Count: 60, - GroupedBy: &aggregation.GroupedBy{ - Path: []string{"sector"}, - Value: "Food", - }, - Properties: map[string]aggregation.Property{ - "dividendYield": { - Type: aggregation.PropertyTypeNumerical, - NumericalAggregations: map[string]interface{}{ - "mean": 2.06667, - "maximum": 8.0, - "minimum": 0.0, - "sum": 124, - "mode": 0., - "median": 1.1, - "count": 60, - }, - }, - "price": { - Type: aggregation.PropertyTypeNumerical, - NumericalAggregations: map[string]interface{}{ - "mean": 218.33333, - "maximum": 800., - "minimum": 10., - "sum": 13100., - // "mode": 70, - "median": 115, - "count": 60, - }, - }, - "listedInIndex": { - Type: aggregation.PropertyTypeBoolean, - BooleanAggregation: aggregation.Boolean{ - TotalTrue: 50, - TotalFalse: 10, - PercentageTrue: 0.8333333333333334, - PercentageFalse: 0.16666666666666666, - Count: 60, - }, - }, - "location": { - Type: aggregation.PropertyTypeText, - TextAggregation: aggregation.Text{ - Count: 60, - Items: []aggregation.TextOccurrence{ - { - Value: "Atlanta", - Occurs: 20, - }, - { - Value: "Detroit", - Occurs: 10, - }, - { - Value: "Los Angeles", - Occurs: 10, - }, - { - Value: "New York", - Occurs: 10, - }, - { - Value: "San Francisco", - Occurs: 10, - }, - }, - }, - }, - }, - }, - { - Count: 30, - GroupedBy: &aggregation.GroupedBy{ - Path: []string{"sector"}, - Value: "Financials", - }, - Properties: map[string]aggregation.Property{ - "dividendYield": { - Type: aggregation.PropertyTypeNumerical, - NumericalAggregations: map[string]interface{}{ - "mean": 2.2, - "maximum": 4.0, - "minimum": 1.3, - "sum": 66., - "mode": 1.3, - "median": 1.3, - "count": 30, - }, - }, - "price": { - Type: aggregation.PropertyTypeNumerical, - NumericalAggregations: map[string]interface{}{ - "mean": 265.66667, - "maximum": 600., - "minimum": 47., - "sum": 7970., - // "mode": 47, - "median": 150., - "count": 30., - }, - }, - "listedInIndex": { - Type: aggregation.PropertyTypeBoolean, - BooleanAggregation: aggregation.Boolean{ - TotalTrue: 30, - TotalFalse: 0, - PercentageTrue: 1, - PercentageFalse: 0, - Count: 30, - }, - }, - "location": { - Type: aggregation.PropertyTypeText, - TextAggregation: aggregation.Text{ - Count: 30, - Items: []aggregation.TextOccurrence{ - { - Value: "New York", - Occurs: 20, - }, - { - Value: "San Francisco", - Occurs: 10, - }, - }, - }, - }, - }, - }, - }, - } - - // there is now way to use InEpsilon or InDelta on nested structs with - // testify, so unfortunately we have to do a manual deep equal: - assert.Equal(t, len(res.Groups), len(expectedResult.Groups)) - assert.Equal(t, expectedResult.Groups[0].Count, res.Groups[0].Count) - assert.Equal(t, expectedResult.Groups[0].GroupedBy, res.Groups[0].GroupedBy) - expectedProps := expectedResult.Groups[0].Properties - actualProps := res.Groups[0].Properties - assert.Equal(t, expectedProps["location"].TextAggregation.Count, - actualProps["location"].TextAggregation.Count) - assert.ElementsMatch(t, expectedProps["location"].TextAggregation.Items, - actualProps["location"].TextAggregation.Items) - assert.Equal(t, expectedProps["listedInIndex"], actualProps["listedInIndex"]) - assert.InDeltaMapValues(t, expectedProps["dividendYield"].NumericalAggregations, - actualProps["dividendYield"].NumericalAggregations, epsilon*100) - assert.InDeltaMapValues(t, expectedProps["price"].NumericalAggregations, - actualProps["price"].NumericalAggregations, epsilon*100) - - assert.Equal(t, len(res.Groups), len(expectedResult.Groups)) - assert.Equal(t, expectedResult.Groups[1].Count, res.Groups[1].Count) - assert.Equal(t, expectedResult.Groups[1].GroupedBy, res.Groups[1].GroupedBy) - expectedProps = expectedResult.Groups[1].Properties - actualProps = res.Groups[1].Properties - assert.Equal(t, expectedProps["location"], actualProps["location"]) - assert.Equal(t, expectedProps["listedInIndex"], actualProps["listedInIndex"]) - assert.InDeltaMapValues(t, expectedProps["dividendYield"].NumericalAggregations, - actualProps["dividendYield"].NumericalAggregations, epsilon*100) - assert.InDeltaMapValues(t, expectedProps["price"].NumericalAggregations, - actualProps["price"].NumericalAggregations, epsilon*500) - }) - - t.Run("with filters, grouped by string", func(t *testing.T) { - params := aggregation.Params{ - ClassName: schema.ClassName(companyClass.Class), - GroupBy: &filters.Path{ - Class: schema.ClassName(companyClass.Class), - Property: schema.PropertyName("sector"), - }, - Filters: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorLessThan, - Value: &filters.Value{ - Type: schema.DataTypeInt, - Value: 600, - }, - On: &filters.Path{ - Property: "price", - }, - }, - }, - Properties: []aggregation.ParamProperty{ - { - Name: schema.PropertyName("dividendYield"), - Aggregators: []aggregation.Aggregator{ - aggregation.MeanAggregator, - aggregation.MaximumAggregator, - aggregation.MinimumAggregator, - aggregation.SumAggregator, - // aggregation.ModeAggregator, - aggregation.MedianAggregator, - aggregation.CountAggregator, - aggregation.TypeAggregator, - }, - }, - { - Name: schema.PropertyName("price"), - Aggregators: []aggregation.Aggregator{ - aggregation.TypeAggregator, - aggregation.MeanAggregator, - aggregation.MaximumAggregator, - aggregation.MinimumAggregator, - aggregation.SumAggregator, - // aggregation.ModeAggregator, // ignore as there is no most common value - aggregation.MedianAggregator, - aggregation.CountAggregator, - }, - }, - { - Name: schema.PropertyName("listedInIndex"), - Aggregators: []aggregation.Aggregator{ - aggregation.TypeAggregator, - aggregation.PercentageTrueAggregator, - aggregation.PercentageFalseAggregator, - aggregation.TotalTrueAggregator, - aggregation.TotalFalseAggregator, - }, - }, - { - Name: schema.PropertyName("location"), - Aggregators: []aggregation.Aggregator{ - aggregation.TypeAggregator, - aggregation.NewTopOccurrencesAggregator(ptInt(5)), - }, - }, - }, - } - - res, err := repo.Aggregate(context.Background(), params) - require.Nil(t, err) - - expectedResult := &aggregation.Result{ - Groups: []aggregation.Group{ - { - Count: 50, - GroupedBy: &aggregation.GroupedBy{ - Path: []string{"sector"}, - Value: "Food", - }, - Properties: map[string]aggregation.Property{ - "dividendYield": { - Type: aggregation.PropertyTypeNumerical, - NumericalAggregations: map[string]interface{}{ - "mean": 2.48, - "maximum": 8.0, - "minimum": 0.0, - "sum": 124., - "median": 1.3, - "count": 50, - }, - }, - "price": { - Type: aggregation.PropertyTypeNumerical, - NumericalAggregations: map[string]interface{}{ - "mean": 102., - "maximum": 200., - "minimum": 10., - "sum": 5100., - "median": 70., - "count": 50., - }, - }, - "listedInIndex": { - Type: aggregation.PropertyTypeBoolean, - BooleanAggregation: aggregation.Boolean{ - TotalTrue: 50, - TotalFalse: 0, - PercentageTrue: 1, - PercentageFalse: 0, - Count: 50, - }, - }, - "location": { - Type: aggregation.PropertyTypeText, - TextAggregation: aggregation.Text{ - Count: 50, - Items: []aggregation.TextOccurrence{ - { - Value: "Atlanta", - Occurs: 20, - }, - { - Value: "Detroit", - Occurs: 10, - }, - { - Value: "New York", - Occurs: 10, - }, - { - Value: "San Francisco", - Occurs: 10, - }, - }, - }, - }, - }, - }, - { - Count: 20, - GroupedBy: &aggregation.GroupedBy{ - Path: []string{"sector"}, - Value: "Financials", - }, - Properties: map[string]aggregation.Property{ - "dividendYield": { - Type: aggregation.PropertyTypeNumerical, - NumericalAggregations: map[string]interface{}{ - "mean": 1.3, - "maximum": 1.3, - "minimum": 1.3, - "sum": 26., - "median": 1.3, - "count": 20., - }, - }, - "price": { - Type: aggregation.PropertyTypeNumerical, - NumericalAggregations: map[string]interface{}{ - "mean": 98.5, - "maximum": 150., - "minimum": 47., - "sum": 1970., - "median": 98.5, - "count": 20., - }, - }, - "listedInIndex": { - Type: aggregation.PropertyTypeBoolean, - BooleanAggregation: aggregation.Boolean{ - TotalTrue: 20, - TotalFalse: 0, - PercentageTrue: 1, - PercentageFalse: 0, - Count: 20, - }, - }, - "location": { - Type: aggregation.PropertyTypeText, - TextAggregation: aggregation.Text{ - Count: 20, - Items: []aggregation.TextOccurrence{ - { - Value: "New York", - Occurs: 10, - }, - { - Value: "San Francisco", - Occurs: 10, - }, - }, - }, - }, - }, - }, - }, - } - - // there is now way to use InEpsilon or InDelta on nested structs with - // testify, so unfortunately we have to do a manual deep equal: - assert.Equal(t, len(res.Groups), len(expectedResult.Groups)) - assert.Equal(t, expectedResult.Groups[0].Count, res.Groups[0].Count) - assert.Equal(t, expectedResult.Groups[0].GroupedBy, res.Groups[0].GroupedBy) - expectedProps := expectedResult.Groups[0].Properties - actualProps := res.Groups[0].Properties - assert.Equal(t, expectedProps["location"].TextAggregation.Count, - actualProps["location"].TextAggregation.Count) - assert.ElementsMatch(t, expectedProps["location"].TextAggregation.Items, - actualProps["location"].TextAggregation.Items) - assert.Equal(t, expectedProps["listedInIndex"], actualProps["listedInIndex"]) - assert.InDeltaMapValues(t, expectedProps["dividendYield"].NumericalAggregations, - actualProps["dividendYield"].NumericalAggregations, epsilon*100) - assert.InDeltaMapValues(t, expectedProps["price"].NumericalAggregations, - actualProps["price"].NumericalAggregations, epsilon*100) - - assert.Equal(t, len(res.Groups), len(expectedResult.Groups)) - assert.Equal(t, expectedResult.Groups[1].Count, res.Groups[1].Count) - assert.Equal(t, expectedResult.Groups[1].GroupedBy, res.Groups[1].GroupedBy) - expectedProps = expectedResult.Groups[1].Properties - actualProps = res.Groups[1].Properties - assert.Equal(t, expectedProps["location"].TextAggregation.Count, - actualProps["location"].TextAggregation.Count) - assert.ElementsMatch(t, expectedProps["location"].TextAggregation.Items, - actualProps["location"].TextAggregation.Items) - assert.Equal(t, expectedProps["listedInIndex"], actualProps["listedInIndex"]) - assert.InDeltaMapValues(t, expectedProps["dividendYield"].NumericalAggregations, - actualProps["dividendYield"].NumericalAggregations, epsilon*100) - assert.InDeltaMapValues(t, expectedProps["price"].NumericalAggregations, - actualProps["price"].NumericalAggregations, epsilon*100) - }) - - t.Run("no filters, grouped by ref prop", func(t *testing.T) { - params := aggregation.Params{ - ClassName: schema.ClassName(companyClass.Class), - GroupBy: &filters.Path{ - Class: schema.ClassName(companyClass.Class), - Property: schema.PropertyName("makesProduct"), - }, - Properties: []aggregation.ParamProperty{ - { - Name: schema.PropertyName("dividendYield"), - Aggregators: []aggregation.Aggregator{ - aggregation.MeanAggregator, - aggregation.MaximumAggregator, - aggregation.MinimumAggregator, - aggregation.SumAggregator, - // aggregation.ModeAggregator, - aggregation.MedianAggregator, - aggregation.CountAggregator, - aggregation.TypeAggregator, - }, - }, - { - Name: schema.PropertyName("price"), - Aggregators: []aggregation.Aggregator{ - aggregation.TypeAggregator, - aggregation.MeanAggregator, - aggregation.MaximumAggregator, - aggregation.MinimumAggregator, - aggregation.SumAggregator, - // aggregation.ModeAggregator, // ignore as there is no most common value - aggregation.MedianAggregator, - aggregation.CountAggregator, - }, - }, - { - Name: schema.PropertyName("listedInIndex"), - Aggregators: []aggregation.Aggregator{ - aggregation.TypeAggregator, - aggregation.PercentageTrueAggregator, - aggregation.PercentageFalseAggregator, - aggregation.TotalTrueAggregator, - aggregation.TotalFalseAggregator, - }, - }, - { - Name: schema.PropertyName("location"), - Aggregators: []aggregation.Aggregator{ - aggregation.TypeAggregator, - aggregation.NewTopOccurrencesAggregator(ptInt(5)), - }, - }, - }, - } - - res, err := repo.Aggregate(context.Background(), params) - require.Nil(t, err) - - expectedResult := &aggregation.Result{ - Groups: []aggregation.Group{ - { - Count: 10, - GroupedBy: &aggregation.GroupedBy{ - Path: []string{"makesProduct"}, - Value: strfmt.URI("weaviate://localhost/1295c052-263d-4aae-99dd-920c5a370d06"), - }, - Properties: map[string]aggregation.Property{ - "dividendYield": { - Type: aggregation.PropertyTypeNumerical, - NumericalAggregations: map[string]interface{}{ - "mean": 8.0, - "maximum": 8.0, - "minimum": 8.0, - "sum": 80.0, - "median": 8.0, - "count": 10., - }, - }, - "price": { - Type: aggregation.PropertyTypeNumerical, - NumericalAggregations: map[string]interface{}{ - "mean": 10., - "maximum": 10., - "minimum": 10., - "sum": 100., - "median": 10., - "count": 10., - }, - }, - "listedInIndex": { - Type: aggregation.PropertyTypeBoolean, - BooleanAggregation: aggregation.Boolean{ - TotalTrue: 10, - TotalFalse: 0, - PercentageTrue: 1, - PercentageFalse: 0, - Count: 10, - }, - }, - "location": { - Type: aggregation.PropertyTypeText, - TextAggregation: aggregation.Text{ - Count: 10, - Items: []aggregation.TextOccurrence{ - { - Value: "Detroit", - Occurs: 10, - }, - }, - }, - }, - }, - }, - }, - } - - // there is now way to use InEpsilon or InDelta on nested structs with - // testify, so unfortunately we have to do a manual deep equal: - assert.Equal(t, len(res.Groups), len(expectedResult.Groups)) - assert.Equal(t, expectedResult.Groups[0].Count, res.Groups[0].Count) - assert.Equal(t, expectedResult.Groups[0].GroupedBy, res.Groups[0].GroupedBy) - expectedProps := expectedResult.Groups[0].Properties - actualProps := res.Groups[0].Properties - assert.Equal(t, expectedProps["location"].TextAggregation.Count, - actualProps["location"].TextAggregation.Count) - assert.ElementsMatch(t, expectedProps["location"].TextAggregation.Items, - actualProps["location"].TextAggregation.Items) - assert.Equal(t, expectedProps["listedInIndex"], actualProps["listedInIndex"]) - assert.InDeltaMapValues(t, expectedProps["dividendYield"].NumericalAggregations, - actualProps["dividendYield"].NumericalAggregations, epsilon*100) - assert.InDeltaMapValues(t, expectedProps["price"].NumericalAggregations, - actualProps["price"].NumericalAggregations, epsilon*100) - }) - - t.Run("with ref filter, grouped by string", func(t *testing.T) { - params := aggregation.Params{ - ClassName: schema.ClassName(companyClass.Class), - GroupBy: &filters.Path{ - Class: schema.ClassName(companyClass.Class), - Property: schema.PropertyName("sector"), - }, - Filters: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorEqual, - Value: &filters.Value{ - Type: schema.DataTypeText, - Value: "Superbread", - }, - On: &filters.Path{ - Property: "makesProduct", - Child: &filters.Path{ - Class: "AggregationsTestProduct", - Property: "name", - }, - }, - }, - }, - Properties: []aggregation.ParamProperty{ - { - Name: schema.PropertyName("dividendYield"), - Aggregators: []aggregation.Aggregator{ - aggregation.MeanAggregator, - aggregation.MaximumAggregator, - aggregation.MinimumAggregator, - aggregation.SumAggregator, - // aggregation.ModeAggregator, - aggregation.MedianAggregator, - aggregation.CountAggregator, - aggregation.TypeAggregator, - }, - }, - { - Name: schema.PropertyName("price"), - Aggregators: []aggregation.Aggregator{ - aggregation.TypeAggregator, - aggregation.MeanAggregator, - aggregation.MaximumAggregator, - aggregation.MinimumAggregator, - aggregation.SumAggregator, - // aggregation.ModeAggregator, // ignore as there is no most common value - aggregation.MedianAggregator, - aggregation.CountAggregator, - }, - }, - { - Name: schema.PropertyName("listedInIndex"), - Aggregators: []aggregation.Aggregator{ - aggregation.TypeAggregator, - aggregation.PercentageTrueAggregator, - aggregation.PercentageFalseAggregator, - aggregation.TotalTrueAggregator, - aggregation.TotalFalseAggregator, - }, - }, - { - Name: schema.PropertyName("location"), - Aggregators: []aggregation.Aggregator{ - aggregation.TypeAggregator, - aggregation.NewTopOccurrencesAggregator(ptInt(5)), - }, - }, - }, - } - - res, err := repo.Aggregate(context.Background(), params) - require.Nil(t, err) - require.NotNil(t, res) - - expectedResult := &aggregation.Result{ - Groups: []aggregation.Group{ - { - Count: 10, - GroupedBy: &aggregation.GroupedBy{ - Path: []string{"sector"}, - Value: "Food", - }, - Properties: map[string]aggregation.Property{ - "dividendYield": { - Type: aggregation.PropertyTypeNumerical, - NumericalAggregations: map[string]interface{}{ - "mean": 8.0, - "maximum": 8.0, - "minimum": 8.0, - "sum": 80., - "median": 8.0, - "count": 10., - }, - }, - "price": { - Type: aggregation.PropertyTypeNumerical, - NumericalAggregations: map[string]interface{}{ - "mean": 10., - "maximum": 10., - "minimum": 10., - "sum": 100., - "median": 10., - "count": 10., - }, - }, - "listedInIndex": { - Type: aggregation.PropertyTypeBoolean, - BooleanAggregation: aggregation.Boolean{ - TotalTrue: 10, - TotalFalse: 0, - PercentageTrue: 1, - PercentageFalse: 0, - Count: 10, - }, - }, - "location": { - Type: aggregation.PropertyTypeText, - TextAggregation: aggregation.Text{ - Count: 10, - Items: []aggregation.TextOccurrence{ - { - Value: "Detroit", - Occurs: 10, - }, - }, - }, - }, - }, - }, - }, - } - - // there is now way to use InEpsilon or InDelta on nested structs with - // testify, so unfortunately we have to do a manual deep equal: - assert.Equal(t, len(res.Groups), len(expectedResult.Groups)) - assert.Equal(t, expectedResult.Groups[0].Count, res.Groups[0].Count) - assert.Equal(t, expectedResult.Groups[0].GroupedBy, res.Groups[0].GroupedBy) - expectedProps := expectedResult.Groups[0].Properties - actualProps := res.Groups[0].Properties - assert.Equal(t, expectedProps["location"], actualProps["location"]) - assert.Equal(t, expectedProps["listedInIndex"], actualProps["listedInIndex"]) - assert.InDeltaMapValues(t, expectedProps["dividendYield"].NumericalAggregations, - actualProps["dividendYield"].NumericalAggregations, 0.001) - assert.InDeltaMapValues(t, expectedProps["price"].NumericalAggregations, - actualProps["price"].NumericalAggregations, 0.001) - }) - - t.Run("array types, single aggregator strings", func(t *testing.T) { - if !exact { - t.Skip() - } - params := aggregation.Params{ - ClassName: schema.ClassName(arrayTypesClass.Class), - GroupBy: &filters.Path{ - Class: schema.ClassName(arrayTypesClass.Class), - Property: schema.PropertyName("strings"), - }, - IncludeMetaCount: true, - } - - res, err := repo.Aggregate(context.Background(), params) - require.Nil(t, err) - - expectedResult := &aggregation.Result{ - Groups: []aggregation.Group{ - { - Count: 2, - GroupedBy: &aggregation.GroupedBy{ - Path: []string{"strings"}, - Value: "a", - }, - Properties: map[string]aggregation.Property{}, - }, - { - Count: 1, - GroupedBy: &aggregation.GroupedBy{ - Path: []string{"strings"}, - Value: "b", - }, - Properties: map[string]aggregation.Property{}, - }, - { - Count: 1, - GroupedBy: &aggregation.GroupedBy{ - Path: []string{"strings"}, - Value: "c", - }, - Properties: map[string]aggregation.Property{}, - }, - }, - } - - assert.ElementsMatch(t, expectedResult.Groups, res.Groups) - }) - - t.Run("array types, single aggregator numbers", func(t *testing.T) { - if !exact { - t.Skip() - } - params := aggregation.Params{ - ClassName: schema.ClassName(arrayTypesClass.Class), - GroupBy: &filters.Path{ - Class: schema.ClassName(arrayTypesClass.Class), - Property: schema.PropertyName("numbers"), - }, - IncludeMetaCount: true, - } - - res, err := repo.Aggregate(context.Background(), params) - require.Nil(t, err) - - expectedResult := &aggregation.Result{ - Groups: []aggregation.Group{ - { - Count: 2, - GroupedBy: &aggregation.GroupedBy{ - Path: []string{"numbers"}, - Value: float64(1.0), - }, - Properties: map[string]aggregation.Property{}, - }, - { - Count: 2, - GroupedBy: &aggregation.GroupedBy{ - Path: []string{"numbers"}, - Value: float64(2.0), - }, - Properties: map[string]aggregation.Property{}, - }, - { - Count: 1, - GroupedBy: &aggregation.GroupedBy{ - Path: []string{"numbers"}, - Value: float64(3.0), - }, - Properties: map[string]aggregation.Property{}, - }, - }, - } - - assert.ElementsMatch(t, expectedResult.Groups, res.Groups) - }) - } -} - -func testDateAggregationsWithFilters(repo *DB) func(t *testing.T) { - return func(t *testing.T) { - t.Run("Aggregations with filter that matches nothing", func(t *testing.T) { - params := aggregation.Params{ - ClassName: schema.ClassName(customerClass.Class), - Filters: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorGreaterThan, - Value: &filters.Value{ - Type: schema.DataTypeDate, - Value: "0312-06-16T17:30:17.231346Z", // hello roman empire! - }, - On: &filters.Path{ - Property: "timeArrived", - }, - }, - }, - IncludeMetaCount: true, - Properties: []aggregation.ParamProperty{ - { - Name: schema.PropertyName("timeArrived"), - Aggregators: []aggregation.Aggregator{aggregation.MeanAggregator, aggregation.CountAggregator, aggregation.MaximumAggregator, aggregation.MedianAggregator, aggregation.MinimumAggregator, aggregation.ModeAggregator, aggregation.TypeAggregator}, - }, - }, - } - res, err := repo.Aggregate(context.Background(), params) - - // No results match the filter, so only a count of 0 is included - require.Nil(t, err) - require.Equal(t, 1, len(res.Groups)) - require.Equal(t, 1, len(res.Groups[0].Properties)) - require.Equal(t, 1, len(res.Groups[0].Properties["timeArrived"].DateAggregations)) - require.Equal(t, int64(0), res.Groups[0].Properties["timeArrived"].DateAggregations["count"].(int64)) - }) - } -} - -func testNumericalAggregationsWithFilters(repo *DB) func(t *testing.T) { - return func(t *testing.T) { - t.Run("Aggregations with filter that matches nothing", func(t *testing.T) { - params := aggregation.Params{ - ClassName: schema.ClassName(companyClass.Class), - Filters: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorLessThan, - Value: &filters.Value{ - Type: schema.DataTypeInt, - Value: -5, // price is positive everywhere - }, - On: &filters.Path{ - Property: "price", - }, - }, - }, - IncludeMetaCount: true, - Properties: []aggregation.ParamProperty{ - { - Name: schema.PropertyName("dividendYield"), - Aggregators: []aggregation.Aggregator{aggregation.MeanAggregator, aggregation.CountAggregator, aggregation.MaximumAggregator, aggregation.MedianAggregator, aggregation.MinimumAggregator, aggregation.ModeAggregator, aggregation.TypeAggregator}, - }, - }, - } - res, err := repo.Aggregate(context.Background(), params) - - // No results match the filter, so only a count of 0 is included - require.Nil(t, err) - require.Equal(t, 1, len(res.Groups)) - require.Equal(t, 1, len(res.Groups[0].Properties)) - require.Equal(t, 1, len(res.Groups[0].Properties["dividendYield"].NumericalAggregations)) - require.Equal(t, float64(0), res.Groups[0].Properties["dividendYield"].NumericalAggregations["count"].(float64)) - }) - } -} - -func testNumericalAggregationsWithoutGrouping(repo *DB, - exact bool, -) func(t *testing.T) { - return func(t *testing.T) { - t.Run("only meta count, no other aggregations", func(t *testing.T) { - params := aggregation.Params{ - ClassName: schema.ClassName(companyClass.Class), - IncludeMetaCount: true, - GroupBy: nil, // explicitly set to nil - } - - res, err := repo.Aggregate(context.Background(), params) - require.Nil(t, err) - - expectedResult := &aggregation.Result{ - Groups: []aggregation.Group{ - { - GroupedBy: nil, - Count: 90, - }, - }, - } - - require.NotNil(t, res) - assert.Equal(t, expectedResult.Groups, res.Groups) - }) - - t.Run("single field, single aggregator", func(t *testing.T) { - params := aggregation.Params{ - ClassName: schema.ClassName(companyClass.Class), - GroupBy: nil, // explicitly set to nil - Properties: []aggregation.ParamProperty{ - { - Name: schema.PropertyName("dividendYield"), - Aggregators: []aggregation.Aggregator{aggregation.MeanAggregator}, - }, - }, - } - - res, err := repo.Aggregate(context.Background(), params) - require.Nil(t, err) - - if exact { - expectedResult := &aggregation.Result{ - Groups: []aggregation.Group{ - { - GroupedBy: nil, - Properties: map[string]aggregation.Property{ - "dividendYield": { - Type: aggregation.PropertyTypeNumerical, - NumericalAggregations: map[string]interface{}{ - "mean": 2.111111111111111, - }, - }, - }, - }, - }, - } - - assert.Equal(t, expectedResult.Groups, res.Groups) - } else { - require.Len(t, res.Groups, 1) - divYield := res.Groups[0].Properties["dividendYield"] - assert.Equal(t, aggregation.PropertyTypeNumerical, divYield.Type) - assert.InDelta(t, 2.1111, divYield.NumericalAggregations["mean"], 2) - } - }) - - t.Run("multiple fields, multiple aggregators", func(t *testing.T) { - params := aggregation.Params{ - ClassName: schema.ClassName(companyClass.Class), - GroupBy: nil, // explicitly set to nil, - IncludeMetaCount: true, - Properties: []aggregation.ParamProperty{ - { - Name: schema.PropertyName("dividendYield"), - Aggregators: []aggregation.Aggregator{ - aggregation.MeanAggregator, - aggregation.MaximumAggregator, - aggregation.MinimumAggregator, - aggregation.SumAggregator, - aggregation.ModeAggregator, - aggregation.MedianAggregator, - aggregation.CountAggregator, - aggregation.TypeAggregator, // ignored in the repo, but can't block - }, - }, - { - Name: schema.PropertyName("price"), - Aggregators: []aggregation.Aggregator{ - aggregation.MeanAggregator, - aggregation.MaximumAggregator, - aggregation.MinimumAggregator, - aggregation.SumAggregator, - aggregation.ModeAggregator, - aggregation.MedianAggregator, - aggregation.CountAggregator, - aggregation.TypeAggregator, // ignored in the repo, but can't block - }, - }, - { - Name: schema.PropertyName("listedInIndex"), - Aggregators: []aggregation.Aggregator{ - aggregation.PercentageTrueAggregator, - aggregation.PercentageFalseAggregator, - aggregation.TotalTrueAggregator, - aggregation.TotalFalseAggregator, - aggregation.TypeAggregator, // ignored in the repo, but can't block - }, - }, - { - Name: schema.PropertyName("location"), - Aggregators: []aggregation.Aggregator{ - // limit is so high, it's not really restrictive - aggregation.NewTopOccurrencesAggregator(ptInt(10)), - aggregation.TypeAggregator, // ignored in the repo, but can't block - }, - }, - { - Name: schema.PropertyName("sector"), - Aggregators: []aggregation.Aggregator{ - // limit is very restrictive - aggregation.NewTopOccurrencesAggregator(ptInt(1)), - aggregation.TypeAggregator, // ignored in the repo, but can't block - }, - }, - // we are not expecting any result from the following agg, as this is - // handled in the usecase. However, we at least want to make sure it - // doesn't block or lead to any errors - { - Name: schema.PropertyName("makesProduct"), - Aggregators: []aggregation.Aggregator{ - aggregation.PointingToAggregator, - aggregation.TypeAggregator, - }, - }, - }, - } - - res, err := repo.Aggregate(context.Background(), params) - require.Nil(t, err) - - expectedResult := &aggregation.Result{ - Groups: []aggregation.Group{ - { - Count: 90, // because includeMetaCount was set - Properties: map[string]aggregation.Property{ - "dividendYield": { - Type: aggregation.PropertyTypeNumerical, - NumericalAggregations: map[string]interface{}{ - "mean": 2.111111111111111, - "maximum": 8.0, - "minimum": 0.0, - "sum": 190., - "mode": 1.3, - "median": 1.3, - "count": 90., - }, - }, - "price": { - Type: aggregation.PropertyTypeNumerical, - NumericalAggregations: map[string]interface{}{ - "mean": 234.11111111111111, - "maximum": 800., - "minimum": 10., - "sum": 21070., - "mode": 70., - "median": 150., - "count": 90., - }, - }, - "listedInIndex": { - Type: aggregation.PropertyTypeBoolean, - BooleanAggregation: aggregation.Boolean{ - TotalTrue: 80, - TotalFalse: 10, - PercentageTrue: 0.8888888888888888, - PercentageFalse: 0.1111111111111111, - Count: 90, - }, - }, - "location": { - Type: aggregation.PropertyTypeText, - TextAggregation: aggregation.Text{ - Count: 90, - Items: []aggregation.TextOccurrence{ - { - Value: "New York", - Occurs: 30, - }, - { - Value: "Atlanta", - Occurs: 20, - }, - { - Value: "San Francisco", - Occurs: 20, - }, - { - Value: "Detroit", - Occurs: 10, - }, - { - Value: "Los Angeles", - Occurs: 10, - }, - }, - }, - }, - "sector": { - Type: aggregation.PropertyTypeText, - TextAggregation: aggregation.Text{ - Count: 90, - Items: []aggregation.TextOccurrence{ - { - Value: "Food", - Occurs: 60, - }, - }, - }, - }, - }, - }, - }, - } - - if exact { - assert.Equal(t, expectedResult.Groups, res.Groups) - } else { - t.Run("numerical fields", func(t *testing.T) { - aggs := res.Groups[0].Properties["dividendYield"].NumericalAggregations - expextedAggs := expectedResult.Groups[0].Properties["dividendYield"].NumericalAggregations - - // max, min, count, sum are always exact matches, but we need an - // epsilon check because of floating point arithmetics - assert.InEpsilon(t, expextedAggs["maximum"], aggs["maximum"], 0.1) - assert.Equal(t, expextedAggs["minimum"], aggs["minimum"]) // equal because the result == 0 - assert.InEpsilon(t, expextedAggs["count"], aggs["count"], 0.1) - assert.InEpsilon(t, expextedAggs["sum"], aggs["sum"], 0.1) - - // mean, mode, median are always fuzzy - assert.InDelta(t, expextedAggs["mean"], aggs["mean"], 2) - assert.InDelta(t, expextedAggs["mode"], aggs["mode"], 2) - assert.InDelta(t, expextedAggs["median"], aggs["median"], 2) - }) - - t.Run("int fields", func(t *testing.T) { - aggs := res.Groups[0].Properties["price"].NumericalAggregations - expextedAggs := expectedResult.Groups[0].Properties["price"].NumericalAggregations - - // max, min, count, sum are always exact matches, but we need an - // epsilon check because of floating point arithmetics - assert.InEpsilon(t, expextedAggs["maximum"], aggs["maximum"], 0.1) - assert.InEpsilon(t, expextedAggs["minimum"], aggs["minimum"], 0.1) - assert.InEpsilon(t, expextedAggs["count"], aggs["count"], 0.1) - assert.InEpsilon(t, expextedAggs["sum"], aggs["sum"], 0.1) - - // mean, mode, median are always fuzzy - assert.InEpsilon(t, expextedAggs["mean"], aggs["mean"], 0.5, "mean") - assert.InEpsilon(t, expextedAggs["mode"], aggs["mode"], 10, "mode") - assert.InEpsilon(t, expextedAggs["median"], aggs["median"], 0.5, "median") - }) - - t.Run("boolean fields", func(t *testing.T) { - aggs := res.Groups[0].Properties["listedInIndex"].BooleanAggregation - expectedAggs := expectedResult.Groups[0].Properties["listedInIndex"].BooleanAggregation - - assert.InEpsilon(t, expectedAggs.TotalTrue, aggs.TotalTrue, 0.1) - assert.InEpsilon(t, expectedAggs.TotalFalse, aggs.TotalFalse, 0.1) - assert.InEpsilon(t, expectedAggs.PercentageTrue, aggs.PercentageTrue, 0.1) - assert.InEpsilon(t, expectedAggs.PercentageFalse, aggs.PercentageFalse, 0.1) - assert.InEpsilon(t, expectedAggs.Count, aggs.Count, 0.1) - }) - - t.Run("text fields (location)", func(t *testing.T) { - aggs := res.Groups[0].Properties["location"].TextAggregation - expectedAggs := expectedResult.Groups[0].Properties["location"].TextAggregation - - assert.Equal(t, expectedAggs.Count, aggs.Count) - assert.ElementsMatch(t, expectedAggs.Items, aggs.Items) - }) - t.Run("text fields (sector)", func(t *testing.T) { - aggs := res.Groups[0].Properties["sector"].TextAggregation - expectedAggs := expectedResult.Groups[0].Properties["sector"].TextAggregation - - assert.Equal(t, expectedAggs.Count, aggs.Count) - assert.ElementsMatch(t, expectedAggs.Items, aggs.Items) - }) - } - }) - - t.Run("multiple fields, multiple aggregators, single-level filter", func(t *testing.T) { - if !exact { - // filtering is happening inside a shard, so there is no need to test - // this again for multi-sharding. This saves us from adapting all the - // assertions to work with fuzzy values - t.Skip() - } - - params := aggregation.Params{ - ClassName: schema.ClassName(companyClass.Class), - GroupBy: nil, // explicitly set to nil, - Filters: sectorEqualsFoodFilter(), - IncludeMetaCount: true, - Properties: []aggregation.ParamProperty{ - { - Name: schema.PropertyName("dividendYield"), - Aggregators: []aggregation.Aggregator{ - aggregation.MeanAggregator, - aggregation.MaximumAggregator, - aggregation.MinimumAggregator, - aggregation.SumAggregator, - aggregation.ModeAggregator, - aggregation.MedianAggregator, - aggregation.CountAggregator, - aggregation.TypeAggregator, // ignored in the repo, but can't block - }, - }, - { - Name: schema.PropertyName("price"), - Aggregators: []aggregation.Aggregator{ - aggregation.MeanAggregator, - aggregation.MaximumAggregator, - aggregation.MinimumAggregator, - aggregation.SumAggregator, - aggregation.ModeAggregator, - aggregation.MedianAggregator, - aggregation.CountAggregator, - aggregation.TypeAggregator, // ignored in the repo, but can't block - }, - }, - { - Name: schema.PropertyName("listedInIndex"), - Aggregators: []aggregation.Aggregator{ - aggregation.PercentageTrueAggregator, - aggregation.PercentageFalseAggregator, - aggregation.TotalTrueAggregator, - aggregation.TotalFalseAggregator, - aggregation.TypeAggregator, // ignored in the repo, but can't block - }, - }, - { - Name: schema.PropertyName("location"), - Aggregators: []aggregation.Aggregator{ - // limit is so high, it's not really restrictive - aggregation.NewTopOccurrencesAggregator(ptInt(10)), - aggregation.TypeAggregator, // ignored in the repo, but can't block - }, - }, - { - Name: schema.PropertyName("sector"), - Aggregators: []aggregation.Aggregator{ - // limit is very restrictive - aggregation.NewTopOccurrencesAggregator(ptInt(1)), - aggregation.TypeAggregator, // ignored in the repo, but can't block - }, - }, - // we are not expecting any result from the following agg, as this is - // handled in the usecase. However, we at least want to make sure it - // doesn't block or lead to any errors - { - Name: schema.PropertyName("makesProduct"), - Aggregators: []aggregation.Aggregator{ - aggregation.PointingToAggregator, - aggregation.TypeAggregator, - }, - }, - }, - } - - res, err := repo.Aggregate(context.Background(), params) - require.Nil(t, err) - - actualDivYield := res.Groups[0].Properties["dividendYield"] - delete(res.Groups[0].Properties, "dividendYield") - actualPrice := res.Groups[0].Properties["price"] - delete(res.Groups[0].Properties, "price") - actualMakesProduct := res.Groups[0].Properties["makesProduct"] - delete(res.Groups[0].Properties, "makesProduct") - - expectedDivYield := aggregation.Property{ - Type: aggregation.PropertyTypeNumerical, - NumericalAggregations: map[string]interface{}{ - "mean": 2.066666666666666, - "maximum": 8.0, - "minimum": 0.0, - "sum": 124, - "mode": 0.0, - "median": 1.2, - "count": 60., - }, - } - - expectedPrice := aggregation.Property{ - Type: aggregation.PropertyTypeNumerical, - NumericalAggregations: map[string]interface{}{ - "mean": 218.33333333333334, - "maximum": 800., - "minimum": 10., - "sum": 13100., - "mode": 70., - "median": 115., - "count": 60., - }, - } - - expectedMakesProduct := aggregation.Property{ - Type: aggregation.PropertyTypeReference, - ReferenceAggregation: aggregation.Reference{ - PointingTo: []string{"weaviate://localhost/1295c052-263d-4aae-99dd-920c5a370d06"}, - }, - } - - expectedResult := &aggregation.Result{ - Groups: []aggregation.Group{ - { - Count: 60, // because includeMetaCount was set - Properties: map[string]aggregation.Property{ - "listedInIndex": { - Type: aggregation.PropertyTypeBoolean, - BooleanAggregation: aggregation.Boolean{ - TotalTrue: 50, - TotalFalse: 10, - PercentageTrue: 0.8333333333333334, - PercentageFalse: 0.16666666666666666, - Count: 60, - }, - }, - "location": { - Type: aggregation.PropertyTypeText, - TextAggregation: aggregation.Text{ - Count: 60, - Items: []aggregation.TextOccurrence{ - { - Value: "Atlanta", - Occurs: 20, - }, - { - Value: "Detroit", - Occurs: 10, - }, - { - Value: "Los Angeles", - Occurs: 10, - }, - { - Value: "New York", - Occurs: 10, - }, - { - Value: "San Francisco", - Occurs: 10, - }, - }, - }, - }, - "sector": { - Type: aggregation.PropertyTypeText, - TextAggregation: aggregation.Text{ - Count: 60, - Items: []aggregation.TextOccurrence{ - { - Value: "Food", - Occurs: 60, - }, - }, - }, - }, - }, - }, - }, - } - - assert.Equal(t, expectedResult.Groups, res.Groups) - - // floating point arithmetic for numerical fields - - assert.InEpsilon(t, expectedDivYield.NumericalAggregations["mean"], - actualDivYield.NumericalAggregations["mean"], 0.1) - assert.InEpsilon(t, expectedPrice.NumericalAggregations["mean"], - actualPrice.NumericalAggregations["mean"], 0.1) - - assert.InEpsilon(t, expectedDivYield.NumericalAggregations["maximum"], - actualDivYield.NumericalAggregations["maximum"], 0.1) - assert.InEpsilon(t, expectedPrice.NumericalAggregations["maximum"], - actualPrice.NumericalAggregations["maximum"], 0.1) - - assert.Equal(t, expectedDivYield.NumericalAggregations["minimum"], - actualDivYield.NumericalAggregations["minimum"]) - assert.Equal(t, expectedPrice.NumericalAggregations["minimum"], - actualPrice.NumericalAggregations["minimum"]) - - assert.Equal(t, expectedDivYield.NumericalAggregations["mode"], - actualDivYield.NumericalAggregations["mode"]) - assert.Equal(t, expectedPrice.NumericalAggregations["mode"], - actualPrice.NumericalAggregations["mode"]) - - assert.InEpsilon(t, expectedDivYield.NumericalAggregations["median"], - actualDivYield.NumericalAggregations["median"], 0.1) - assert.InEpsilon(t, expectedPrice.NumericalAggregations["median"], - actualPrice.NumericalAggregations["median"], 0.1) - - assert.InEpsilon(t, expectedDivYield.NumericalAggregations["count"], - actualDivYield.NumericalAggregations["count"], 0.1) - assert.InEpsilon(t, expectedPrice.NumericalAggregations["count"], - actualPrice.NumericalAggregations["count"], 0.1) - - assert.Equal(t, expectedMakesProduct.ReferenceAggregation.PointingTo, - actualMakesProduct.ReferenceAggregation.PointingTo) - }) - - t.Run("multiple fields, multiple aggregators, ref filter", func(t *testing.T) { - if !exact { - // filtering is happening inside a shard, so there is no need to test - // this again for multi-sharding. This saves us from adapting all the - // assertions to work with fuzzy values - t.Skip() - } - - params := aggregation.Params{ - ClassName: schema.ClassName(companyClass.Class), - GroupBy: nil, // explicitly set to nil, - Filters: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorEqual, - Value: &filters.Value{ - Type: schema.DataTypeText, - Value: "Superbread", - }, - On: &filters.Path{ - Property: "makesProduct", - Child: &filters.Path{ - Class: "AggregationsTestProduct", - Property: "name", - }, - }, - }, - }, - IncludeMetaCount: true, - Properties: []aggregation.ParamProperty{ - { - Name: schema.PropertyName("dividendYield"), - Aggregators: []aggregation.Aggregator{ - aggregation.MeanAggregator, - aggregation.MaximumAggregator, - aggregation.MinimumAggregator, - aggregation.SumAggregator, - aggregation.ModeAggregator, - aggregation.MedianAggregator, - aggregation.CountAggregator, - aggregation.TypeAggregator, // ignored in the repo, but can't block - }, - }, - { - Name: schema.PropertyName("price"), - Aggregators: []aggregation.Aggregator{ - aggregation.MeanAggregator, - aggregation.MaximumAggregator, - aggregation.MinimumAggregator, - aggregation.SumAggregator, - aggregation.ModeAggregator, - aggregation.MedianAggregator, - aggregation.CountAggregator, - aggregation.TypeAggregator, // ignored in the repo, but can't block - }, - }, - { - Name: schema.PropertyName("listedInIndex"), - Aggregators: []aggregation.Aggregator{ - aggregation.PercentageTrueAggregator, - aggregation.PercentageFalseAggregator, - aggregation.TotalTrueAggregator, - aggregation.TotalFalseAggregator, - aggregation.TypeAggregator, // ignored in the repo, but can't block - }, - }, - { - Name: schema.PropertyName("location"), - Aggregators: []aggregation.Aggregator{ - // limit is so high, it's not really restrictive - aggregation.NewTopOccurrencesAggregator(ptInt(10)), - aggregation.TypeAggregator, // ignored in the repo, but can't block - }, - }, - { - Name: schema.PropertyName("sector"), - Aggregators: []aggregation.Aggregator{ - // limit is very restrictive - aggregation.NewTopOccurrencesAggregator(ptInt(1)), - aggregation.TypeAggregator, // ignored in the repo, but can't block - }, - }, - // we are not expecting any result from the following agg, as this is - // handled in the usecase. However, we at least want to make sure it - // doesn't block or lead to any errors - { - Name: schema.PropertyName("makesProduct"), - Aggregators: []aggregation.Aggregator{ - aggregation.PointingToAggregator, - aggregation.TypeAggregator, - }, - }, - }, - } - - res, err := repo.Aggregate(context.Background(), params) - require.Nil(t, err) - - expectedResult := &aggregation.Result{ - Groups: []aggregation.Group{ - { - Count: 10, - Properties: map[string]aggregation.Property{ - "makesProduct": { - Type: aggregation.PropertyTypeReference, - ReferenceAggregation: aggregation.Reference{PointingTo: []string{"weaviate://localhost/1295c052-263d-4aae-99dd-920c5a370d06"}}, - }, - "dividendYield": { - Type: aggregation.PropertyTypeNumerical, - NumericalAggregations: map[string]interface{}{ - "mean": 8.0, - "maximum": 8.0, - "minimum": 8.0, - "sum": 80., - "mode": 8.0, - "median": 8.0, - "count": 10., - }, - }, - "price": { - Type: aggregation.PropertyTypeNumerical, - NumericalAggregations: map[string]interface{}{ - "mean": 10., - "maximum": 10., - "minimum": 10., - "sum": 100., - "mode": 10., - "median": 10., - "count": 10., - }, - }, - "listedInIndex": { - Type: aggregation.PropertyTypeBoolean, - BooleanAggregation: aggregation.Boolean{ - TotalTrue: 10, - TotalFalse: 0, - PercentageTrue: 1, - PercentageFalse: 0, - Count: 10, - }, - }, - "location": { - Type: aggregation.PropertyTypeText, - TextAggregation: aggregation.Text{ - Count: 10, - Items: []aggregation.TextOccurrence{ - { - Value: "Detroit", - Occurs: 10, - }, - }, - }, - }, - "sector": { - Type: aggregation.PropertyTypeText, - TextAggregation: aggregation.Text{ - Count: 10, - Items: []aggregation.TextOccurrence{ - { - Value: "Food", - Occurs: 10, - }, - }, - }, - }, - }, - }, - }, - } - - assert.Equal(t, expectedResult.Groups, res.Groups) - }) - - t.Run("array types, only meta count, no other aggregations", func(t *testing.T) { - params := aggregation.Params{ - ClassName: schema.ClassName(arrayTypesClass.Class), - IncludeMetaCount: true, - GroupBy: nil, // explicitly set to nil - } - - res, err := repo.Aggregate(context.Background(), params) - require.Nil(t, err) - - expectedResult := &aggregation.Result{ - Groups: []aggregation.Group{ - { - GroupedBy: nil, - Count: 2, - }, - }, - } - - require.NotNil(t, res) - assert.Equal(t, expectedResult.Groups, res.Groups) - }) - - t.Run("array types, single aggregator numbers", func(t *testing.T) { - params := aggregation.Params{ - ClassName: schema.ClassName(arrayTypesClass.Class), - GroupBy: nil, // explicitly set to nil - Properties: []aggregation.ParamProperty{ - { - Name: schema.PropertyName("numbers"), - Aggregators: []aggregation.Aggregator{ - aggregation.MeanAggregator, - aggregation.MaximumAggregator, - aggregation.MinimumAggregator, - aggregation.SumAggregator, - aggregation.ModeAggregator, - aggregation.MedianAggregator, - aggregation.CountAggregator, - aggregation.TypeAggregator, // ignored in the repo, but can't block - }, - }, - }, - } - - res, err := repo.Aggregate(context.Background(), params) - require.Nil(t, err) - - expectedResult := &aggregation.Result{ - Groups: []aggregation.Group{ - { - GroupedBy: nil, - Properties: map[string]aggregation.Property{ - "numbers": { - Type: aggregation.PropertyTypeNumerical, - NumericalAggregations: map[string]interface{}{ - "mean": 2.0, - "maximum": 3.0, - "minimum": 1.0, - "sum": 14.0, - "mode": 2.0, - "median": 2.0, - "count": 7., - }, - }, - }, - }, - }, - } - - assert.Equal(t, expectedResult.Groups, res.Groups) - }) - - t.Run("array types, single aggregator strings", func(t *testing.T) { - if !exact { - t.Skip() - } - params := aggregation.Params{ - ClassName: schema.ClassName(arrayTypesClass.Class), - GroupBy: nil, // explicitly set to nil - Properties: []aggregation.ParamProperty{ - { - Name: schema.PropertyName("strings"), - Aggregators: []aggregation.Aggregator{ - // limit is very restrictive - aggregation.NewTopOccurrencesAggregator(ptInt(1)), - aggregation.TypeAggregator, // ignored in the repo, but can't block - }, - }, - }, - } - - res, err := repo.Aggregate(context.Background(), params) - require.Nil(t, err) - - expectedResult := &aggregation.Result{ - Groups: []aggregation.Group{ - { - GroupedBy: nil, - Properties: map[string]aggregation.Property{ - "strings": { - Type: aggregation.PropertyTypeText, - TextAggregation: aggregation.Text{ - Count: 4, - Items: []aggregation.TextOccurrence{ - { - Value: "a", - Occurs: 2, - }, - }, - }, - }, - }, - }, - }, - } - - assert.Equal(t, expectedResult.Groups, res.Groups) - }) - } -} - -func testDateAggregationsWithGrouping(repo *DB, exact bool) func(t *testing.T) { - return func(t *testing.T) { - t.Run("group on only unique values", func(t *testing.T) { - params := aggregation.Params{ - ClassName: schema.ClassName(customerClass.Class), - IncludeMetaCount: true, - GroupBy: &filters.Path{ - Class: schema.ClassName(customerClass.Class), - // Each customer obj has a unique value for the `internalId` field - Property: schema.PropertyName("internalId"), - }, - } - - res, err := repo.Aggregate(context.Background(), params) - require.Nil(t, err) - - require.NotNil(t, res) - assert.Len(t, res.Groups, len(customers)) - }) - - t.Run("group on only identical values", func(t *testing.T) { - params := aggregation.Params{ - ClassName: schema.ClassName(customerClass.Class), - IncludeMetaCount: true, - GroupBy: &filters.Path{ - Class: schema.ClassName(customerClass.Class), - // Each customer obj has the same value for the `countryOfOrigin` field - Property: schema.PropertyName("countryOfOrigin"), - }, - Properties: []aggregation.ParamProperty{ - { - Name: "timeArrived", - Aggregators: []aggregation.Aggregator{ - aggregation.CountAggregator, - aggregation.MinimumAggregator, - aggregation.MaximumAggregator, - aggregation.MedianAggregator, - aggregation.ModeAggregator, - }, - }, - }, - } - - res, err := repo.Aggregate(context.Background(), params) - require.Nil(t, err) - - require.NotNil(t, res) - assert.Len(t, res.Groups, 1) - - expectedProperties := map[string]interface{}{ - "count": int64(10), - "minimum": "2022-06-16T17:30:17.231346Z", - "maximum": "2022-06-16T17:30:26.451235Z", - "median": "2022-06-16T17:30:21.1179905Z", - "mode": "2022-06-16T17:30:17.231346Z", - } - receivedProperties := res.Groups[0].Properties["timeArrived"].DateAggregations - assert.EqualValues(t, expectedProperties, receivedProperties) - }) - - t.Run("group on some unique values", func(t *testing.T) { - params := aggregation.Params{ - ClassName: schema.ClassName(customerClass.Class), - IncludeMetaCount: true, - GroupBy: &filters.Path{ - Class: schema.ClassName(customerClass.Class), - // should result in two groups due to bool value - Property: schema.PropertyName("isNewCustomer"), - }, - Properties: []aggregation.ParamProperty{ - { - Name: "timeArrived", - Aggregators: []aggregation.Aggregator{ - aggregation.CountAggregator, - aggregation.MinimumAggregator, - aggregation.MaximumAggregator, - aggregation.MedianAggregator, - aggregation.ModeAggregator, - }, - }, - }, - } - - res, err := repo.Aggregate(context.Background(), params) - require.Nil(t, err) - - require.NotNil(t, res) - assert.Len(t, res.Groups, 2) - - expectedResult := []aggregation.Group{ - { - Properties: map[string]aggregation.Property{ - "timeArrived": { - Type: "date", - DateAggregations: map[string]interface{}{ - "count": int64(6), - "maximum": "2022-06-16T17:30:25.524536Z", - "median": "2022-06-16T17:30:19.6718905Z", - "minimum": "2022-06-16T17:30:17.231346Z", - "mode": "2022-06-16T17:30:17.231346Z", - }, - }, - }, - GroupedBy: &aggregation.GroupedBy{ - Value: false, - Path: []string{"isNewCustomer"}, - }, - Count: 6, - }, - { - Properties: map[string]aggregation.Property{ - "timeArrived": { - Type: "date", - DateAggregations: map[string]interface{}{ - "count": int64(4), - "maximum": "2022-06-16T17:30:26.451235Z", - "median": "2022-06-16T17:30:22.224622Z", - "minimum": "2022-06-16T17:30:20.123546Z", - "mode": "2022-06-16T17:30:20.123546Z", - }, - }, - }, - GroupedBy: &aggregation.GroupedBy{ - Value: true, - Path: []string{"isNewCustomer"}, - }, - Count: 4, - }, - } - - assert.EqualValues(t, expectedResult, res.Groups) - }) - } -} - -func testDateAggregationsWithoutGrouping(repo *DB, exact bool) func(t *testing.T) { - return func(t *testing.T) { - t.Run("without grouping", func(t *testing.T) { - params := aggregation.Params{ - ClassName: schema.ClassName(customerClass.Class), - GroupBy: nil, - Properties: []aggregation.ParamProperty{ - { - Name: "timeArrived", - Aggregators: []aggregation.Aggregator{ - aggregation.CountAggregator, - aggregation.MinimumAggregator, - aggregation.MaximumAggregator, - aggregation.MedianAggregator, - aggregation.ModeAggregator, - }, - }, - }, - } - - res, err := repo.Aggregate(context.Background(), params) - require.Nil(t, err) - - require.NotNil(t, res) - require.Len(t, res.Groups, 1) - }) - } -} - -func ptInt(in int) *int { - return &in -} - -func sectorEqualsFoodFilter() *filters.LocalFilter { - return &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: "Company", - Property: "sector", - }, - Value: &filters.Value{ - Value: "Food", - Type: schema.DataTypeText, - }, - }, - } -} - -func mustStringToTime(s string) time.Time { - asTime, err := time.ParseInLocation(time.RFC3339Nano, s, time.UTC) - if err != nil { - panic(fmt.Sprintf("failed to parse time: %s, %s", s, err)) - } - return asTime -} diff --git a/adapters/repos/db/aggregator/aggregator.go b/adapters/repos/db/aggregator/aggregator.go deleted file mode 100644 index f180bbfa55292fd7b5aea5a8fcdb053c1d86ae73..0000000000000000000000000000000000000000 --- a/adapters/repos/db/aggregator/aggregator.go +++ /dev/null @@ -1,121 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package aggregator - -import ( - "context" - "fmt" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/adapters/repos/db/inverted" - "github.com/weaviate/weaviate/adapters/repos/db/inverted/stopwords" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv" - "github.com/weaviate/weaviate/entities/aggregation" - "github.com/weaviate/weaviate/entities/schema" - schemaUC "github.com/weaviate/weaviate/usecases/schema" -) - -type vectorIndex interface { - SearchByVectorDistance(vector []float32, targetDistance float32, maxLimit int64, - allowList helpers.AllowList) ([]uint64, []float32, error) - SearchByVector(vector []float32, k int, allowList helpers.AllowList) ([]uint64, []float32, error) -} - -type Aggregator struct { - logger logrus.FieldLogger - store *lsmkv.Store - params aggregation.Params - getSchema schemaUC.SchemaGetter - classSearcher inverted.ClassSearcher // to support ref-filters - vectorIndex vectorIndex - stopwords stopwords.StopwordDetector - shardVersion uint16 - propLenTracker *inverted.JsonShardMetaData - isFallbackToSearchable inverted.IsFallbackToSearchable - tenant string - nestedCrossRefLimit int64 -} - -func New(store *lsmkv.Store, params aggregation.Params, - getSchema schemaUC.SchemaGetter, classSearcher inverted.ClassSearcher, - stopwords stopwords.StopwordDetector, shardVersion uint16, - vectorIndex vectorIndex, logger logrus.FieldLogger, - propLenTracker *inverted.JsonShardMetaData, - isFallbackToSearchable inverted.IsFallbackToSearchable, - tenant string, nestedCrossRefLimit int64, -) *Aggregator { - return &Aggregator{ - logger: logger, - store: store, - params: params, - getSchema: getSchema, - classSearcher: classSearcher, - stopwords: stopwords, - shardVersion: shardVersion, - vectorIndex: vectorIndex, - propLenTracker: propLenTracker, - isFallbackToSearchable: isFallbackToSearchable, - tenant: tenant, - nestedCrossRefLimit: nestedCrossRefLimit, - } -} - -func (a *Aggregator) GetPropertyLengthTracker() *inverted.JsonShardMetaData { - return a.propLenTracker -} - -func (a *Aggregator) Do(ctx context.Context) (*aggregation.Result, error) { - if a.params.GroupBy != nil { - return newGroupedAggregator(a).Do(ctx) - } - - if a.params.Filters != nil || len(a.params.SearchVector) > 0 || a.params.Hybrid != nil { - return newFilteredAggregator(a).Do(ctx) - } - - return newUnfilteredAggregator(a).Do(ctx) -} - -func (a *Aggregator) aggTypeOfProperty( - name schema.PropertyName, -) (aggregation.PropertyType, schema.DataType, error) { - s := a.getSchema.GetSchemaSkipAuth() - schemaProp, err := s.GetProperty(a.params.ClassName, name) - if err != nil { - return "", "", errors.Wrapf(err, "property %s", name) - } - - if schema.IsRefDataType(schemaProp.DataType) { - return aggregation.PropertyTypeReference, schema.DataTypeCRef, nil - } - - dt := schema.DataType(schemaProp.DataType[0]) - switch dt { - case schema.DataTypeInt, schema.DataTypeNumber, schema.DataTypeIntArray, - schema.DataTypeNumberArray: - return aggregation.PropertyTypeNumerical, dt, nil - case schema.DataTypeBoolean, schema.DataTypeBooleanArray: - return aggregation.PropertyTypeBoolean, dt, nil - case schema.DataTypeText, schema.DataTypeTextArray: - return aggregation.PropertyTypeText, dt, nil - case schema.DataTypeDate, schema.DataTypeDateArray: - return aggregation.PropertyTypeDate, dt, nil - case schema.DataTypeGeoCoordinates: - return "", "", fmt.Errorf("dataType geoCoordinates can't be aggregated") - case schema.DataTypePhoneNumber: - return "", "", fmt.Errorf("dataType phoneNumber can't be aggregated") - default: - return "", "", fmt.Errorf("unrecoginzed dataType %v", schemaProp.DataType[0]) - } -} diff --git a/adapters/repos/db/aggregator/boolean.go b/adapters/repos/db/aggregator/boolean.go deleted file mode 100644 index 195fd235727a9bef4cc93dd2fa1000b2567b7ffb..0000000000000000000000000000000000000000 --- a/adapters/repos/db/aggregator/boolean.go +++ /dev/null @@ -1,78 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package aggregator - -import ( - "bytes" - "encoding/binary" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/aggregation" -) - -func newBoolAggregator() *boolAggregator { - return &boolAggregator{} -} - -type boolAggregator struct { - countTrue uint64 - countFalse uint64 -} - -func (a *boolAggregator) AddBoolRow(value []byte, count uint64) error { - var valueParsed bool - - if err := binary.Read(bytes.NewReader(value), binary.LittleEndian, - &valueParsed); err != nil { - return errors.Wrap(err, "read bool") - } - - if count == 0 { - // skip - return nil - } - - if valueParsed { - a.countTrue += count - } else { - a.countFalse += count - } - - return nil -} - -func (a *boolAggregator) AddBool(value bool) error { - if value { - a.countTrue++ - } else { - a.countFalse++ - } - - return nil -} - -func (a *boolAggregator) Res() aggregation.Boolean { - out := aggregation.Boolean{} - - count := int(a.countTrue) + int(a.countFalse) - if count == 0 { - return out - } - - out.Count = count - out.TotalFalse = int(a.countFalse) - out.TotalTrue = int(a.countTrue) - out.PercentageTrue = float64(a.countTrue) / float64(count) - out.PercentageFalse = float64(a.countFalse) / float64(count) - - return out -} diff --git a/adapters/repos/db/aggregator/date.go b/adapters/repos/db/aggregator/date.go deleted file mode 100644 index f9b28c790149464978c91f1de968b0b1e24a270c..0000000000000000000000000000000000000000 --- a/adapters/repos/db/aggregator/date.go +++ /dev/null @@ -1,211 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package aggregator - -import ( - "fmt" - "math" - "sort" - "time" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/adapters/repos/db/inverted" - "github.com/weaviate/weaviate/entities/aggregation" -) - -func addDateAggregations(prop *aggregation.Property, - aggs []aggregation.Aggregator, agg *dateAggregator, -) { - if prop.DateAggregations == nil { - prop.DateAggregations = map[string]interface{}{} - } - agg.buildPairsFromCounts() - - // if there are no elements to aggregate over because a filter does not match anything, calculating median etc. makes - // no sense. Non-existent entries evaluate to nil with an interface{} map - if agg.count == 0 { - for _, entry := range aggs { - if entry == aggregation.CountAggregator { - prop.DateAggregations["count"] = int64(agg.count) - break - } - } - return - } - - // when combining the results from different shards, we need the raw dates to recompute the mode and median. - // Therefore we add a reference later which needs to be cleared out before returning the results to a user - for _, aProp := range aggs { - switch aProp { - case aggregation.ModeAggregator, aggregation.MedianAggregator: - prop.DateAggregations["_dateAggregator"] = agg - } - } - - for _, aProp := range aggs { - switch aProp { - case aggregation.MinimumAggregator: - prop.DateAggregations[aProp.String()] = agg.Min() - case aggregation.MaximumAggregator: - prop.DateAggregations[aProp.String()] = agg.Max() - case aggregation.ModeAggregator: - prop.DateAggregations[aProp.String()] = agg.Mode() - case aggregation.CountAggregator: - prop.DateAggregations[aProp.String()] = agg.Count() - case aggregation.MedianAggregator: - prop.DateAggregations[aProp.String()] = agg.Median() - - default: - continue - } - } -} - -type dateAggregator struct { - count uint64 - maxCount uint64 - min timestamp - max timestamp - mode timestamp - pairs []timestampCountPair // for row-based median calculation - valueCounter map[timestamp]uint64 // for individual median calculation -} - -func newDateAggregator() *dateAggregator { - return &dateAggregator{ - min: timestamp{epochNano: math.MaxInt64}, - valueCounter: map[timestamp]uint64{}, - pairs: make([]timestampCountPair, 0), - } -} - -// timestamp allows us to contain multiple representations of a datetime -// the nanosecs value is needed for the numerical comparisons, and the -// string value is what the user expects to see -type timestamp struct { - epochNano int64 - rfc3339 string -} - -func newTimestamp(epochNano int64) timestamp { - return timestamp{ - epochNano: epochNano, - rfc3339: time.Unix(0, epochNano).UTC().Format(time.RFC3339Nano), - } -} - -type timestampCountPair struct { - value timestamp - count uint64 -} - -func (a *dateAggregator) AddTimestamp(rfc3339 string) error { - t, err := time.Parse(time.RFC3339Nano, rfc3339) - if err != nil { - return fmt.Errorf("failed to parse timestamp: %s", err) - } - - ts := timestamp{ - epochNano: t.UnixNano(), - rfc3339: rfc3339, - } - return a.addRow(ts, 1) -} - -func (a *dateAggregator) AddTimestampRow(b []byte, count uint64) error { - nsec, err := inverted.ParseLexicographicallySortableInt64(b) - if err != nil { - return errors.Wrap(err, "read int64") - } - - ts := newTimestamp(nsec) - - return a.addRow(ts, count) -} - -func (a *dateAggregator) addRow(ts timestamp, count uint64) error { - if count == 0 { - // skip - return nil - } - - a.count += count - if ts.epochNano < a.min.epochNano { - a.min = ts - } - if ts.epochNano > a.max.epochNano { - a.max = ts - } - - currentCount := a.valueCounter[ts] - currentCount += count - a.valueCounter[ts] = currentCount - - return nil -} - -func (a *dateAggregator) Max() string { - return a.max.rfc3339 -} - -func (a *dateAggregator) Min() string { - return a.min.rfc3339 -} - -// Mode does not require preparation if build from rows, but requires a call of -// buildPairsFromCounts() if it was built using individual objects -func (a *dateAggregator) Mode() string { - return a.mode.rfc3339 -} - -func (a *dateAggregator) Count() int64 { - return int64(a.count) -} - -// Median does not require preparation if build from rows, but requires a call of -// buildPairsFromCounts() if it was built using individual objects -// -// Check the numericalAggregator.Median() for details about the calculation -func (a *dateAggregator) Median() string { - middleIndex := a.count / 2 - count := uint64(0) - for index, pair := range a.pairs { - count += pair.count - if a.count%2 == 1 && count > middleIndex { - return pair.value.rfc3339 // case a) - } else if a.count%2 == 0 { - if count == middleIndex { - MedianEpochNano := pair.value.epochNano + (a.pairs[index+1].value.epochNano-pair.value.epochNano)/2 - return time.Unix(0, MedianEpochNano).UTC().Format(time.RFC3339Nano) // case b2) - } else if count > middleIndex { - return pair.value.rfc3339 // case b1) - } - } - } - panic("Couldn't determine median. This should never happen. Did you add values and call buildRows before?") -} - -// turns the value counter into a sorted list, as well as identifying the mode -func (a *dateAggregator) buildPairsFromCounts() { - a.pairs = a.pairs[:0] // clear out old values in case this function called more than once - for value, count := range a.valueCounter { - if count > a.maxCount { - a.maxCount = count - a.mode = value - } - a.pairs = append(a.pairs, timestampCountPair{value: value, count: count}) - } - - sort.Slice(a.pairs, func(x, y int) bool { - return a.pairs[x].value.epochNano < a.pairs[y].value.epochNano - }) -} diff --git a/adapters/repos/db/aggregator/date_test.go b/adapters/repos/db/aggregator/date_test.go deleted file mode 100644 index 0f332bb95a5522594031220a9a9d46d38397f4e6..0000000000000000000000000000000000000000 --- a/adapters/repos/db/aggregator/date_test.go +++ /dev/null @@ -1,78 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package aggregator - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -const ( - DateYearMonthDayHourMinute = "2022-06-16T17:30:" - DateNanoSecondsTimeZone = ".451235Z" -) - -func TestDateAggregator(t *testing.T) { - tests := []struct { - name string - seconds []string - expectedMedian string - expectedMode string - }{ - { - name: "Single value", - seconds: []string{"17"}, - expectedMedian: "17", - expectedMode: "17", - }, - { - name: "Even number of values", - seconds: []string{"18", "18", "20", "25"}, - expectedMedian: "19", - expectedMode: "18", - }, - { - name: "Uneven number of values", - seconds: []string{"18", "18", "19", "20", "25"}, - expectedMedian: "19", - expectedMode: "18", - }, - } - names := []string{"AddTimestamp", "AddRow"} - for _, tt := range tests { - for _, name := range names { // test two ways of adding the value to the aggregator - t.Run(tt.name+" "+name, func(t *testing.T) { - agg := newDateAggregator() - for _, second := range tt.seconds { - fullDate := DateYearMonthDayHourMinute + second + DateNanoSecondsTimeZone - if name == names[0] { - err := agg.AddTimestamp(fullDate) - assert.Nil(t, err) - } else { - timeParsed, err := time.Parse(time.RFC3339, fullDate) - assert.Nil(t, err) - ts := newTimestamp(timeParsed.UnixNano()) - err = agg.addRow(ts, 1) - assert.Nil(t, err) - } - } - agg.buildPairsFromCounts() // needed to populate all required info - assert.Equal(t, DateYearMonthDayHourMinute+tt.expectedMedian+DateNanoSecondsTimeZone, agg.Median()) - if len(tt.expectedMode) > 0 { // if there is no value that appears more often than other values - assert.Equal(t, DateYearMonthDayHourMinute+tt.expectedMode+DateNanoSecondsTimeZone, agg.Mode()) - } - }) - } - } -} diff --git a/adapters/repos/db/aggregator/filtered.go b/adapters/repos/db/aggregator/filtered.go deleted file mode 100644 index e9472e4c879f45b52e7ecd645dc691595ca6b980..0000000000000000000000000000000000000000 --- a/adapters/repos/db/aggregator/filtered.go +++ /dev/null @@ -1,455 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package aggregator - -import ( - "context" - "fmt" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/adapters/repos/db/docid" - "github.com/weaviate/weaviate/adapters/repos/db/inverted" - "github.com/weaviate/weaviate/adapters/repos/db/propertyspecific" - "github.com/weaviate/weaviate/entities/aggregation" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/searchparams" - "github.com/weaviate/weaviate/entities/storobj" - "github.com/weaviate/weaviate/usecases/traverser/hybrid" -) - -type filteredAggregator struct { - *Aggregator -} - -func newFilteredAggregator(agg *Aggregator) *filteredAggregator { - return &filteredAggregator{Aggregator: agg} -} - -func (fa *filteredAggregator) GetPropertyLengthTracker() *inverted.JsonShardMetaData { - return fa.propLenTracker -} - -func (fa *filteredAggregator) Do(ctx context.Context) (*aggregation.Result, error) { - if fa.params.Hybrid != nil { - return fa.hybrid(ctx) - } - - return fa.filtered(ctx) -} - -func (fa *filteredAggregator) hybrid(ctx context.Context) (*aggregation.Result, error) { - sparseSearch := func() ([]*storobj.Object, []float32, error) { - kw, err := fa.buildHybridKeywordRanking() - if err != nil { - return nil, nil, fmt.Errorf("build hybrid keyword ranking: %w", err) - } - - if fa.params.ObjectLimit == nil { - limit := hybrid.DefaultLimit - fa.params.ObjectLimit = &limit - } - - sparse, dists, err := fa.bm25Objects(ctx, kw) - if err != nil { - return nil, nil, fmt.Errorf("aggregate sparse search: %w", err) - } - - return sparse, dists, nil - } - - denseSearch := func(vec []float32) ([]*storobj.Object, []float32, error) { - allowList, err := fa.buildAllowList(ctx) - if err != nil { - return nil, nil, err - } - - res, dists, err := fa.objectVectorSearch(vec, allowList) - if err != nil { - return nil, nil, fmt.Errorf("aggregate dense search: %w", err) - } - - return res, dists, nil - } - - res, err := hybrid.Search(ctx, &hybrid.Params{ - HybridSearch: fa.params.Hybrid, - Class: fa.params.ClassName.String(), - }, fa.logger, sparseSearch, denseSearch, nil, nil) - if err != nil { - return nil, err - } - - ids := make([]uint64, len(res)) - for i, r := range res { - ids[i] = r.DocID - } - - return fa.prepareResult(ctx, ids) -} - -func (fa *filteredAggregator) filtered(ctx context.Context) (*aggregation.Result, error) { - var foundIDs []uint64 - - allowList, err := fa.buildAllowList(ctx) - if err != nil { - return nil, err - } - - if len(fa.params.SearchVector) > 0 { - foundIDs, _, err = fa.vectorSearch(allowList, fa.params.SearchVector) - if err != nil { - return nil, err - } - } else { - foundIDs = allowList.Slice() - } - - return fa.prepareResult(ctx, foundIDs) -} - -func (fa *filteredAggregator) bm25Objects(ctx context.Context, kw *searchparams.KeywordRanking) ([]*storobj.Object, []float32, error) { - var ( - s = fa.getSchema.GetSchemaSkipAuth() - class = s.GetClass(fa.params.ClassName) - cfg = inverted.ConfigFromModel(class.InvertedIndexConfig) - ) - objs, dists, err := inverted.NewBM25Searcher(cfg.BM25, fa.store, s, - propertyspecific.Indices{}, fa.classSearcher, - fa.GetPropertyLengthTracker(), fa.logger, fa.shardVersion, - ).BM25F(ctx, nil, fa.params.ClassName, *fa.params.ObjectLimit, *kw) - if err != nil { - return nil, nil, fmt.Errorf("bm25 objects: %w", err) - } - return objs, dists, nil -} - -func (fa *filteredAggregator) properties(ctx context.Context, - ids []uint64, -) (map[string]aggregation.Property, error) { - propAggs, err := fa.prepareAggregatorsForProps() - if err != nil { - return nil, errors.Wrap(err, "prepare aggregators for props") - } - - scan := func(properties *models.PropertySchema, docID uint64) (bool, error) { - if err := fa.AnalyzeObject(ctx, properties, propAggs); err != nil { - return false, errors.Wrapf(err, "analyze object %d", docID) - } - return true, nil - } - propertyNames := make([]string, 0, len(propAggs)) - for k := range propAggs { - propertyNames = append(propertyNames, k) - } - - err = docid.ScanObjectsLSM(fa.store, ids, scan, propertyNames) - if err != nil { - return nil, errors.Wrap(err, "properties view tx") - } - - return propAggs.results() -} - -func (fa *filteredAggregator) AnalyzeObject(ctx context.Context, - properties *models.PropertySchema, propAggs map[string]propAgg, -) error { - if err := ctx.Err(); err != nil { - return err - } - - if properties == nil { - return nil - } - - for propName, prop := range propAggs { - value, ok := (*properties).(map[string]interface{})[propName] - if !ok { - continue - } - - if err := fa.addPropValue(prop, value); err != nil { - return fmt.Errorf("failed to add prop value: %s", err) - } - } - - return nil -} - -func (fa *filteredAggregator) addPropValue(prop propAgg, value interface{}) error { - switch prop.aggType { - case aggregation.PropertyTypeBoolean: - analyzeBool := func(value interface{}) error { - asBool, ok := value.(bool) - if !ok { - return fmt.Errorf("expected property type boolean, received %T", value) - } - if err := prop.boolAgg.AddBool(asBool); err != nil { - return err - } - return nil - } - switch prop.dataType { - case schema.DataTypeBoolean: - if err := analyzeBool(value); err != nil { - return err - } - case schema.DataTypeBooleanArray: - valueStruct, ok := value.([]interface{}) - if !ok { - return fmt.Errorf("expected property type []boolean, received %T", valueStruct) - } - for _, val := range valueStruct { - if err := analyzeBool(val); err != nil { - return err - } - } - default: - return fmt.Errorf("unknown datatype %v for aggregation %v", prop.dataType, aggregation.PropertyTypeText) - } - case aggregation.PropertyTypeNumerical: - analyzeFloat := func(value interface{}) error { - asFloat, ok := value.(float64) - if !ok { - return fmt.Errorf("expected property type float64, received %T", value) - } - if err := prop.numericalAgg.AddFloat64(asFloat); err != nil { - return err - } - return nil - } - switch prop.dataType { - case schema.DataTypeNumber, schema.DataTypeInt: - if err := analyzeFloat(value); err != nil { - return err - } - case schema.DataTypeNumberArray, schema.DataTypeIntArray: - valueStruct, ok := value.([]interface{}) - if !ok { - return fmt.Errorf("expected property type []float* or []int*, received %T", valueStruct) - } - for _, val := range valueStruct { - if err := analyzeFloat(val); err != nil { - return err - } - } - default: - return fmt.Errorf("unknown datatype %v for aggregation %v", prop.dataType, aggregation.PropertyTypeText) - } - case aggregation.PropertyTypeText: - analyzeString := func(value interface{}) error { - asString, ok := value.(string) - if !ok { - return fmt.Errorf("expected property type string, received %T", value) - } - if err := prop.textAgg.AddText(asString); err != nil { - return err - } - return nil - } - switch prop.dataType { - case schema.DataTypeText: - if err := analyzeString(value); err != nil { - return err - } - case schema.DataTypeTextArray: - valueStruct, ok := value.([]interface{}) - if !ok { - return fmt.Errorf("expected property type []text or []string, received %T", valueStruct) - } - for _, val := range valueStruct { - if err := analyzeString(val); err != nil { - return err - } - } - default: - return fmt.Errorf("unknown datatype %v for aggregation %v", prop.dataType, aggregation.PropertyTypeText) - } - case aggregation.PropertyTypeDate: - analyzeDate := func(value interface{}) error { - asString, ok := value.(string) - if !ok { - return fmt.Errorf("expected property type date, received %T", value) - } - if err := prop.dateAgg.AddTimestamp(asString); err != nil { - return err - } - return nil - } - switch prop.dataType { - case schema.DataTypeDate: - if err := analyzeDate(value); err != nil { - return err - } - case schema.DataTypeDateArray: - valueStruct, ok := value.([]interface{}) - if !ok { - return fmt.Errorf("expected property type []date, received %T", valueStruct) - } - for _, val := range valueStruct { - if err := analyzeDate(val); err != nil { - return err - } - } - default: - return fmt.Errorf("unknown datatype %v for aggregation %v", prop.dataType, aggregation.PropertyTypeText) - } - case aggregation.PropertyTypeReference: - if prop.dataType != schema.DataTypeCRef { - return errors.New(string("unknown datatype for aggregation type reference: " + prop.dataType)) - } - - analyzeRef := func(value interface{}) error { - referenceList, ok := value.([]interface{}) - if !ok { - return fmt.Errorf("expected property type reference, received %T", value) - } - if len(referenceList) != 1 { - return fmt.Errorf("expected list with length 1, got %T", len(referenceList)) - } - refMap, ok := referenceList[0].(map[string]interface{}) - if !ok { - return fmt.Errorf("expected property type reference, received %T", value) - } - - if err := prop.refAgg.AddReference(refMap); err != nil { - return err - } - return nil - } - if err := analyzeRef(value); err != nil { - return err - } - default: - return errors.New(string("Unknown aggregation type " + prop.aggType)) - - } - - return nil -} - -func (fa *filteredAggregator) prepareResult(ctx context.Context, foundIDs []uint64) (*aggregation.Result, error) { - var out aggregation.Result - // without grouping there is always exactly one group - out.Groups = make([]aggregation.Group, 1) - - if fa.params.IncludeMetaCount { - out.Groups[0].Count = len(foundIDs) - } - - props, err := fa.properties(ctx, foundIDs) - if err != nil { - return nil, errors.Wrap(err, "aggregate properties") - } - - out.Groups[0].Properties = props - return &out, nil -} - -// a helper type to select the right aggregator for a prop -type propAgg struct { - name schema.PropertyName - - // the user is interested in those specific aggregations - specifiedAggregators []aggregation.Aggregator - - // underlying data type of prop - dataType schema.DataType - - // use aggType to chose with agg to use - aggType aggregation.PropertyType - - boolAgg *boolAggregator - textAgg *textAggregator - numericalAgg *numericalAggregator - dateAgg *dateAggregator - refAgg *refAggregator -} - -// propAggs groups propAgg helpers by prop name -type propAggs map[string]propAgg - -func (pa *propAgg) initAggregator() { - switch pa.aggType { - case aggregation.PropertyTypeText: - limit := extractLimitFromTopOccs(pa.specifiedAggregators) - pa.textAgg = newTextAggregator(limit) - case aggregation.PropertyTypeBoolean: - pa.boolAgg = newBoolAggregator() - case aggregation.PropertyTypeNumerical: - pa.numericalAgg = newNumericalAggregator() - case aggregation.PropertyTypeDate: - pa.dateAgg = newDateAggregator() - case aggregation.PropertyTypeReference: - pa.refAgg = newRefAggregator() - default: - panic("Unknown aggregation type: " + pa.aggType) - } -} - -func (pa propAggs) results() (map[string]aggregation.Property, error) { - out := map[string]aggregation.Property{} - - for _, prop := range pa { - aggProp := aggregation.Property{ - Type: prop.aggType, - } - - switch prop.aggType { - case aggregation.PropertyTypeBoolean: - aggProp.BooleanAggregation = prop.boolAgg.Res() - out[prop.name.String()] = aggProp - case aggregation.PropertyTypeText: - aggProp.TextAggregation = prop.textAgg.Res() - out[prop.name.String()] = aggProp - case aggregation.PropertyTypeNumerical: - addNumericalAggregations(&aggProp, prop.specifiedAggregators, - prop.numericalAgg) - out[prop.name.String()] = aggProp - case aggregation.PropertyTypeDate: - addDateAggregations(&aggProp, prop.specifiedAggregators, - prop.dateAgg) - out[prop.name.String()] = aggProp - case aggregation.PropertyTypeReference: - addReferenceAggregations(&aggProp, prop.specifiedAggregators, - prop.refAgg) - out[prop.name.String()] = aggProp - default: - return nil, errors.New(string("unknown aggregation type " + prop.aggType)) - } - } - - return out, nil -} - -func (fa *filteredAggregator) prepareAggregatorsForProps() (propAggs, error) { - out := propAggs{} - - for _, prop := range fa.params.Properties { - pa := propAgg{ - name: prop.Name, - specifiedAggregators: prop.Aggregators, - } - - at, dt, err := fa.aggTypeOfProperty(prop.Name) - if err != nil { - return nil, errors.Wrapf(err, "property %s", prop.Name) - } - - pa.aggType = at - pa.dataType = dt - pa.initAggregator() - out[prop.Name.String()] = pa - } - - return out, nil -} diff --git a/adapters/repos/db/aggregator/grouped.go b/adapters/repos/db/aggregator/grouped.go deleted file mode 100644 index 406725e82155c280747cec94c33ca155bdc6fe35..0000000000000000000000000000000000000000 --- a/adapters/repos/db/aggregator/grouped.go +++ /dev/null @@ -1,83 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package aggregator - -import ( - "context" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/aggregation" -) - -// groupedAggregator performs aggregation in groups. This is a two-step -// process. First a whole-db scan is performed to identify the groups, then -// the top-n groups are selected (the rest is discarded). Only for those top -// groups an actual aggregation is performed -type groupedAggregator struct { - *Aggregator -} - -func newGroupedAggregator(agg *Aggregator) *groupedAggregator { - return &groupedAggregator{Aggregator: agg} -} - -func (ga *groupedAggregator) Do(ctx context.Context) (*aggregation.Result, error) { - out := aggregation.Result{} - - groups, err := ga.identifyGroups(ctx) - if err != nil { - return nil, errors.Wrap(err, "identify groups") - } - - out.Groups = make([]aggregation.Group, len(groups)) - for i, g := range groups { - res, err := ga.aggregateGroup(ctx, g.res, g.docIDs) - if err != nil { - return nil, errors.Wrapf(err, "aggregate group %d (%v)", i, - g.res.GroupedBy.Value) - } - out.Groups[i] = res - } - - return &out, nil -} - -// group is a helper construct that contains the final aggregation.Group which -// will eventually be served to the user. But it also contains the list of -// docIDs in that group, so we can use those to perform the actual aggregation -// (for each group) in a second step -type group struct { - res aggregation.Group - docIDs []uint64 -} - -func (ga *groupedAggregator) identifyGroups(ctx context.Context) ([]group, error) { - limit := 100 // reasonable default in case we get none - if ga.params.Limit != nil { - limit = *ga.params.Limit - } - return newGrouper(ga.Aggregator, limit).Do(ctx) -} - -func (ga *groupedAggregator) aggregateGroup(ctx context.Context, - in aggregation.Group, ids []uint64, -) (aggregation.Group, error) { - out := in - fa := newFilteredAggregator(ga.Aggregator) - props, err := fa.properties(ctx, ids) - if err != nil { - return out, errors.Wrap(err, "aggregate properties") - } - - out.Properties = props - return out, nil -} diff --git a/adapters/repos/db/aggregator/grouper.go b/adapters/repos/db/aggregator/grouper.go deleted file mode 100644 index 2530b05091c00a56f157cec856e8a7f55a1d29f6..0000000000000000000000000000000000000000 --- a/adapters/repos/db/aggregator/grouper.go +++ /dev/null @@ -1,309 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package aggregator - -import ( - "context" - "fmt" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/adapters/repos/db/docid" - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv" - "github.com/weaviate/weaviate/entities/aggregation" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/storobj" - "github.com/weaviate/weaviate/usecases/traverser/hybrid" - bolt "go.etcd.io/bbolt" -) - -// grouper is the component which identifies the top-n groups for a specific -// group-by parameter. It is used as part of the grouped aggregator, which then -// additionally performs an aggregation for each group. -type grouper struct { - *Aggregator - values map[interface{}]map[uint64]struct{} // map[value][docID]struct, to keep docIds unique - topGroups []group - limit int -} - -func newGrouper(a *Aggregator, limit int) *grouper { - return &grouper{ - Aggregator: a, - values: map[interface{}]map[uint64]struct{}{}, - limit: limit, - } -} - -func (g *grouper) Do(ctx context.Context) ([]group, error) { - if len(g.params.GroupBy.Slice()) > 1 { - return nil, fmt.Errorf("grouping by cross-refs not supported") - } - - if g.params.Filters == nil && len(g.params.SearchVector) == 0 && g.params.Hybrid == nil { - return g.groupAll(ctx) - } else { - return g.groupFiltered(ctx) - } -} - -func (g *grouper) groupAll(ctx context.Context) ([]group, error) { - err := ScanAllLSM(g.store, func(prop *models.PropertySchema, docID uint64) (bool, error) { - return true, g.addElementById(prop, docID) - }) - if err != nil { - return nil, errors.Wrap(err, "group all (unfiltered)") - } - - return g.aggregateAndSelect() -} - -func (g *grouper) groupFiltered(ctx context.Context) ([]group, error) { - ids, err := g.fetchDocIDs(ctx) - if err != nil { - return nil, err - } - - if err := docid.ScanObjectsLSM(g.store, ids, - func(prop *models.PropertySchema, docID uint64) (bool, error) { - return true, g.addElementById(prop, docID) - }, []string{g.params.GroupBy.Property.String()}); err != nil { - return nil, err - } - - return g.aggregateAndSelect() -} - -func (g *grouper) fetchDocIDs(ctx context.Context) (ids []uint64, err error) { - allowList, err := g.buildAllowList(ctx) - if err != nil { - return nil, err - } - - if len(g.params.SearchVector) > 0 { - ids, _, err = g.vectorSearch(allowList, g.params.SearchVector) - if err != nil { - return nil, fmt.Errorf("failed to perform vector search: %w", err) - } - } else if g.params.Hybrid != nil { - ids, err = g.hybrid(ctx, allowList) - if err != nil { - return nil, fmt.Errorf("hybrid search: %w", err) - } - } else { - ids = allowList.Slice() - } - - return -} - -func (g *grouper) hybrid(ctx context.Context, allowList helpers.AllowList) ([]uint64, error) { - sparseSearch := func() ([]*storobj.Object, []float32, error) { - kw, err := g.buildHybridKeywordRanking() - if err != nil { - return nil, nil, fmt.Errorf("build hybrid keyword ranking: %w", err) - } - - if g.params.ObjectLimit == nil { - limit := hybrid.DefaultLimit - g.params.ObjectLimit = &limit - } - - sparse, dists, err := g.bm25Objects(ctx, kw) - if err != nil { - return nil, nil, fmt.Errorf("aggregate sparse search: %w", err) - } - - return sparse, dists, nil - } - - denseSearch := func(vec []float32) ([]*storobj.Object, []float32, error) { - res, dists, err := g.objectVectorSearch(vec, allowList) - if err != nil { - return nil, nil, fmt.Errorf("aggregate grouped dense search: %w", err) - } - - return res, dists, nil - } - - res, err := hybrid.Search(ctx, &hybrid.Params{ - HybridSearch: g.params.Hybrid, - Keyword: nil, - Class: g.params.ClassName.String(), - }, g.logger, sparseSearch, denseSearch, nil, nil) - if err != nil { - return nil, err - } - - ids := make([]uint64, len(res)) - for i, r := range res { - ids[i] = r.DocID - } - - return ids, nil -} - -func (g *grouper) addElementById(s *models.PropertySchema, docID uint64) error { - if s == nil { - return nil - } - - item, ok := (*s).(map[string]interface{})[g.params.GroupBy.Property.String()] - if !ok { - return nil - } - - switch val := item.(type) { - case []string: - for i := range val { - g.addItem(val[i], docID) - } - case []float64: - for i := range val { - g.addItem(val[i], docID) - } - case []bool: - for i := range val { - g.addItem(val[i], docID) - } - case []interface{}: - for i := range val { - g.addItem(val[i], docID) - } - case models.MultipleRef: - for i := range val { - g.addItem(val[i].Beacon, docID) - } - default: - g.addItem(val, docID) - } - - return nil -} - -func (g *grouper) addItem(item interface{}, docID uint64) { - idsMap, ok := g.values[item] - if !ok { - idsMap = map[uint64]struct{}{} - } - idsMap[docID] = struct{}{} - g.values[item] = idsMap -} - -func (g *grouper) aggregateAndSelect() ([]group, error) { - for value, idsMap := range g.values { - count := len(idsMap) - ids := make([]uint64, count) - - i := 0 - for id := range idsMap { - ids[i] = id - i++ - } - - g.insertOrdered(group{ - res: aggregation.Group{ - GroupedBy: &aggregation.GroupedBy{ - Path: g.params.GroupBy.Slice(), - Value: value, - }, - Count: count, - }, - docIDs: ids, - }) - } - - return g.topGroups, nil -} - -func (g *grouper) insertOrdered(elem group) { - if len(g.topGroups) == 0 { - g.topGroups = []group{elem} - return - } - - added := false - for i, existing := range g.topGroups { - if existing.res.Count > elem.res.Count { - continue - } - - // we have found the first one that's smaller so we must insert before i - g.topGroups = append( - g.topGroups[:i], append( - []group{elem}, - g.topGroups[i:]..., - )..., - ) - - added = true - break - } - - if len(g.topGroups) > g.limit { - g.topGroups = g.topGroups[:len(g.topGroups)-1] - } - - if !added && len(g.topGroups) < g.limit { - g.topGroups = append(g.topGroups, elem) - } -} - -// ScanAll iterates over every row in the object buckets -// TODO: where should this live? -func ScanAll(tx *bolt.Tx, scan docid.ObjectScanFn) error { - b := tx.Bucket(helpers.ObjectsBucket) - if b == nil { - return fmt.Errorf("objects bucket not found") - } - - b.ForEach(func(_, v []byte) error { - elem, err := storobj.FromBinary(v) - if err != nil { - return errors.Wrapf(err, "unmarshal data object") - } - - // scanAll has no abort, so we can ignore the first arg - properties := elem.Properties() - _, err = scan(&properties, elem.DocID()) - return err - }) - - return nil -} - -// ScanAllLSM iterates over every row in the object buckets -func ScanAllLSM(store *lsmkv.Store, scan docid.ObjectScanFn) error { - b := store.Bucket(helpers.ObjectsBucketLSM) - if b == nil { - return fmt.Errorf("objects bucket not found") - } - - c := b.Cursor() - defer c.Close() - - for k, v := c.First(); k != nil; k, v = c.Next() { - elem, err := storobj.FromBinary(v) - if err != nil { - return errors.Wrapf(err, "unmarshal data object") - } - - // scanAll has no abort, so we can ignore the first arg - properties := elem.Properties() - _, err = scan(&properties, elem.DocID()) - if err != nil { - return err - } - } - - return nil -} diff --git a/adapters/repos/db/aggregator/hybrid.go b/adapters/repos/db/aggregator/hybrid.go deleted file mode 100644 index 0b6273b74b10b4be71b117e8af357407669156e1..0000000000000000000000000000000000000000 --- a/adapters/repos/db/aggregator/hybrid.go +++ /dev/null @@ -1,62 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package aggregator - -import ( - "context" - "fmt" - - "github.com/weaviate/weaviate/adapters/repos/db/inverted" - "github.com/weaviate/weaviate/adapters/repos/db/propertyspecific" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/searchparams" - "github.com/weaviate/weaviate/entities/storobj" -) - -func (a *Aggregator) buildHybridKeywordRanking() (*searchparams.KeywordRanking, error) { - kw := &searchparams.KeywordRanking{ - Type: "bm25", - Query: a.params.Hybrid.Query, - } - - cl, err := schema.GetClassByName( - a.getSchema.GetSchemaSkipAuth().Objects, - a.params.ClassName.String()) - if err != nil { - return nil, err - } - - for _, v := range cl.Properties { - if v.DataType[0] == "text" || v.DataType[0] == "string" { // TODO: Also the array types? - kw.Properties = append(kw.Properties, v.Name) - } - } - - return kw, nil -} - -func (a *Aggregator) bm25Objects(ctx context.Context, kw *searchparams.KeywordRanking) ([]*storobj.Object, []float32, error) { - var ( - s = a.getSchema.GetSchemaSkipAuth() - class = s.GetClass(a.params.ClassName) - cfg = inverted.ConfigFromModel(class.InvertedIndexConfig) - ) - - objs, dists, err := inverted.NewBM25Searcher(cfg.BM25, a.store, s, - propertyspecific.Indices{}, a.classSearcher, - a.GetPropertyLengthTracker(), a.logger, a.shardVersion, - ).BM25F(ctx, nil, a.params.ClassName, *a.params.ObjectLimit, *kw) - if err != nil { - return nil, nil, fmt.Errorf("bm25 objects: %w", err) - } - return objs, dists, nil -} diff --git a/adapters/repos/db/aggregator/numerical.go b/adapters/repos/db/aggregator/numerical.go deleted file mode 100644 index 0e3b25066992c4d8b4f6f1441acc618381be7a80..0000000000000000000000000000000000000000 --- a/adapters/repos/db/aggregator/numerical.go +++ /dev/null @@ -1,226 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package aggregator - -import ( - "math" - "sort" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/adapters/repos/db/inverted" - "github.com/weaviate/weaviate/entities/aggregation" -) - -func addNumericalAggregations(prop *aggregation.Property, - aggs []aggregation.Aggregator, agg *numericalAggregator, -) { - if prop.NumericalAggregations == nil { - prop.NumericalAggregations = map[string]interface{}{} - } - agg.buildPairsFromCounts() - - // if there are no elements to aggregate over because a filter does not match anything, calculating mean etc. makes - // no sense. Non-existent entries evaluate to nil with an interface{} map - if agg.count == 0 { - for _, entry := range aggs { - if entry == aggregation.CountAggregator { - prop.NumericalAggregations["count"] = float64(agg.count) - break - } - } - return - } - - // when combining the results from different shards, we need the raw numbers to recompute the mode, mean and median. - // Therefore we add a reference later which needs to be cleared out before returning the results to a user -loop: - for _, aProp := range aggs { - switch aProp { - case aggregation.ModeAggregator, aggregation.MedianAggregator, aggregation.MeanAggregator: - prop.NumericalAggregations["_numericalAggregator"] = agg - break loop - } - } - - for _, aProp := range aggs { - switch aProp { - case aggregation.MeanAggregator: - prop.NumericalAggregations[aProp.String()] = agg.Mean() - case aggregation.MinimumAggregator: - prop.NumericalAggregations[aProp.String()] = agg.Min() - case aggregation.MaximumAggregator: - prop.NumericalAggregations[aProp.String()] = agg.Max() - case aggregation.MedianAggregator: - prop.NumericalAggregations[aProp.String()] = agg.Median() - case aggregation.ModeAggregator: - prop.NumericalAggregations[aProp.String()] = agg.Mode() - case aggregation.SumAggregator: - prop.NumericalAggregations[aProp.String()] = agg.Sum() - case aggregation.CountAggregator: - prop.NumericalAggregations[aProp.String()] = agg.Count() - default: - continue - } - } -} - -func newNumericalAggregator() *numericalAggregator { - return &numericalAggregator{ - min: math.MaxFloat64, - max: -math.MaxFloat64, - valueCounter: map[float64]uint64{}, - pairs: make([]floatCountPair, 0), - } -} - -type numericalAggregator struct { - count uint64 - min float64 - max float64 - sum float64 - maxCount uint64 - mode float64 - pairs []floatCountPair // for row-based median calculation - valueCounter map[float64]uint64 // for individual median calculation -} - -type floatCountPair struct { - value float64 - count uint64 -} - -func (a *numericalAggregator) AddFloat64(value float64) error { - return a.AddNumberRow(value, 1) -} - -// turns the value counter into a sorted list, as well as identifying the mode. Must be called before calling median etc -func (a *numericalAggregator) buildPairsFromCounts() { - a.pairs = a.pairs[:0] // clear out old values in case this function called more than once - a.pairs = append(a.pairs, make([]floatCountPair, 0, len(a.valueCounter))...) - - for value, count := range a.valueCounter { - // get one with higher count or lower value if counts are equal - if count > a.maxCount || (count == a.maxCount && value < a.mode) { - a.maxCount = count - a.mode = value - } - a.pairs = append(a.pairs, floatCountPair{value: value, count: count}) - } - - sort.Slice(a.pairs, func(x, y int) bool { - return a.pairs[x].value < a.pairs[y].value - }) -} - -func (a *numericalAggregator) AddFloat64Row(number []byte, - count uint64, -) error { - numberParsed, err := inverted.ParseLexicographicallySortableFloat64(number) - if err != nil { - return errors.Wrap(err, "read float64") - } - - return a.AddNumberRow(numberParsed, count) -} - -func (a *numericalAggregator) AddInt64Row(number []byte, count uint64) error { - numberParsed, err := inverted.ParseLexicographicallySortableInt64(number) - if err != nil { - return errors.Wrap(err, "read int64") - } - - return a.AddNumberRow(float64(numberParsed), count) -} - -func (a *numericalAggregator) AddNumberRow(number float64, count uint64) error { - if count == 0 { - // skip - return nil - } - - a.count += count - a.sum += number * float64(count) - if number < a.min { - a.min = number - } - if number > a.max { - a.max = number - } - - currentCount := a.valueCounter[number] - currentCount += count - a.valueCounter[number] = currentCount - - return nil -} - -func (a *numericalAggregator) Mean() float64 { - if a.count == 0 { - return 0 - } - return a.sum / float64(a.count) -} - -func (a *numericalAggregator) Max() float64 { - return a.max -} - -func (a *numericalAggregator) Min() float64 { - return a.min -} - -func (a *numericalAggregator) Sum() float64 { - return a.sum -} - -func (a *numericalAggregator) Count() float64 { - return float64(a.count) -} - -// Mode does not require preparation if build from rows, but requires a call of -// buildPairsFromCounts() if it was built using individual objects -func (a *numericalAggregator) Mode() float64 { - return a.mode -} - -// Median does not require preparation if build from rows, but requires a call of -// buildPairsFromCounts() if it was built using individual objects. The call will panic -// if called without adding at least one element or without calling buildPairsFromCounts() -// -// since the pairs are read from an inverted index, which is in turn -// lexicographically sorted, we know that our pairs must also be sorted -// -// There are two cases: -// a) There is an uneven number of elements, then the median element is at index N/2 -// b) There is an even number of elements, then the median element is (elem_(N/2) + elem_(N/2+1))/2. -// -// with two sub-cases: -// b1) element N/2 and N/2 + 1 are within the same pair, then the median is the value of this pair -// b2) element N/2 and N/2 are part of different pairs, then the average of these pairs is the median and the -// median value is not part of the collection itself -func (a *numericalAggregator) Median() float64 { - middleIndex := a.count / 2 - count := uint64(0) - for index, pair := range a.pairs { - count += pair.count - if a.count%2 == 1 && count > middleIndex { - return pair.value // case a) - } else if a.count%2 == 0 { - if count == middleIndex { - return (pair.value + a.pairs[index+1].value) / 2 // case b2) - } else if count > middleIndex { - return pair.value // case b1) - } - } - } - panic("Couldn't determine median. This should never happen. Did you add values and call buildRows before?") -} diff --git a/adapters/repos/db/aggregator/numerical_test.go b/adapters/repos/db/aggregator/numerical_test.go deleted file mode 100644 index f0e4d4c526f8f829834af28773743a61fc8cf509..0000000000000000000000000000000000000000 --- a/adapters/repos/db/aggregator/numerical_test.go +++ /dev/null @@ -1,204 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package aggregator - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestNumericalAggregator_MedianCalculation(t *testing.T) { - tests := []struct { - name string - numbers []float64 - expectedMedian float64 - }{ - { - name: "Uneven number of elements", - numbers: []float64{1, 2, 3, 4, 5, 6, 7}, - expectedMedian: 4, - }, - { - name: "Uneven number of elements with double elements", - numbers: []float64{2, 4, 4, 4, 5, 5, 7}, - expectedMedian: 4, - }, - { - name: "Even number of elements", - numbers: []float64{1, 2, 3, 5, 7, 7}, - expectedMedian: 4, - }, - { - name: "Even number of elements with double elements, median is within double entries", - numbers: []float64{1, 2, 3, 3, 6, 7}, - expectedMedian: 3, - }, - { - name: "Even number of elements with double elements, median is after double entries", - numbers: []float64{3, 3, 3, 5, 5, 7}, - expectedMedian: 4, - }, - { - name: "Single value", - numbers: []float64{42}, - expectedMedian: 42, - }, - } - for _, tt := range tests { - for i := 0; i < 2; i++ { // test two ways of adding the value to the aggregator - t.Run(tt.name, func(t *testing.T) { - agg := newNumericalAggregator() - for _, num := range tt.numbers { - if i == 0 { - agg.AddFloat64(num) - } else { - agg.AddNumberRow(num, 1) - } - } - agg.buildPairsFromCounts() // needed to populate all required info - assert.Equal(t, tt.expectedMedian, agg.Median()) - }) - } - } -} - -func TestNumericalAggregator_ModeCalculation(t *testing.T) { - tests := []struct { - name string - numbers []float64 - expectedMode float64 - }{ - { - name: "Different elements (asc)", - numbers: []float64{1, 2, 3, 4, 5, 6, 7}, - expectedMode: 1, - }, - { - name: "Different elements (desc)", - numbers: []float64{7, 6, 5, 4, 3, 2, 1}, - expectedMode: 1, - }, - { - name: "Elements with different number of duplicates", - numbers: []float64{2, 4, 4, 5, 5, 5, 7}, - expectedMode: 5, - }, - { - name: "Elements with same number of duplicates (asc)", - numbers: []float64{1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7}, - expectedMode: 1, - }, - { - name: "Elements with same number of duplicates (desc)", - numbers: []float64{7, 7, 6, 6, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1}, - expectedMode: 1, - }, - { - name: "Elements with same number of duplicates (mixed oreder)", - numbers: []float64{5, 2, 6, 2, 1, 4, 7, 5, 7, 4, 3, 1, 6, 3}, - expectedMode: 1, - }, - { - name: "Single element", - numbers: []float64{42}, - expectedMode: 42, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - agg := newNumericalAggregator() - for _, num := range tt.numbers { - agg.AddFloat64(num) - } - agg.buildPairsFromCounts() // needed to populate all required info - - assert.Equal(t, tt.expectedMode, agg.Mode()) - }) - - t.Run(tt.name, func(t *testing.T) { - agg := newNumericalAggregator() - for _, num := range tt.numbers { - agg.AddNumberRow(num, 1) - } - agg.buildPairsFromCounts() // needed to populate all required info - - assert.Equal(t, tt.expectedMode, agg.Mode()) - }) - } -} - -func TestNumericalAggregator_MinMaxCalculation(t *testing.T) { - tests := []struct { - name string - numbers []float64 - expectedMin float64 - expectedMax float64 - }{ - { - name: "Different positive elements (asc)", - numbers: []float64{0, 1, 2, 3, 4, 5, 6, 7}, - expectedMin: 0, - expectedMax: 7, - }, - { - name: "Different positive elements (desc)", - numbers: []float64{7, 6, 5, 4, 3, 2, 1, 0}, - expectedMin: 0, - expectedMax: 7, - }, - { - name: "Different negative elements (desc)", - numbers: []float64{-1, -2, -3, -4, -5, -6, -7}, - expectedMin: -7, - expectedMax: -1, - }, - { - name: "Different negative elements (asc)", - numbers: []float64{-7, -6, -5, -4, -3, -2, -1}, - expectedMin: -7, - expectedMax: -1, - }, - { - name: "Different elements (mixed order)", - numbers: []float64{-7, 6, -5, 4, -3, 2, -1, 0, 1, -2, 3, -4, 5, -6, 7}, - expectedMin: -7, - expectedMax: 7, - }, - { - name: "Single element", - numbers: []float64{-42}, - expectedMin: -42, - expectedMax: -42, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - agg := newNumericalAggregator() - for _, num := range tt.numbers { - agg.AddFloat64(num) - } - assert.Equal(t, tt.expectedMin, agg.Min()) - assert.Equal(t, tt.expectedMax, agg.Max()) - }) - - t.Run(tt.name, func(t *testing.T) { - agg := newNumericalAggregator() - for _, num := range tt.numbers { - agg.AddNumberRow(num, 1) - } - assert.Equal(t, tt.expectedMin, agg.Min()) - assert.Equal(t, tt.expectedMax, agg.Max()) - }) - } -} diff --git a/adapters/repos/db/aggregator/references.go b/adapters/repos/db/aggregator/references.go deleted file mode 100644 index 61944fe7641aca8d8e67d86eacf790821c7abfdf..0000000000000000000000000000000000000000 --- a/adapters/repos/db/aggregator/references.go +++ /dev/null @@ -1,64 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package aggregator - -import ( - "errors" - - "github.com/weaviate/weaviate/entities/aggregation" -) - -func addReferenceAggregations(prop *aggregation.Property, - aggs []aggregation.Aggregator, agg *refAggregator, -) { - prop.ReferenceAggregation = aggregation.Reference{} - prop.ReferenceAggregation.PointingTo = agg.PointingTo() - - for _, aProp := range aggs { - switch aProp { - case aggregation.PointingToAggregator: - prop.ReferenceAggregation.PointingTo = agg.PointingTo() - default: - continue - } - } -} - -func newRefAggregator() *refAggregator { - return &refAggregator{valueCounter: map[string]uint64{}} -} - -type refAggregator struct { - count uint64 - valueCounter map[string]uint64 -} - -func (a *refAggregator) AddReference(ref map[string]interface{}) error { - a.count++ - - beacon, ok := ref["beacon"].(string) - if !ok { - return errors.New("not a reference" + beacon) - } - count := a.valueCounter[beacon] - count++ - a.valueCounter[beacon] = count - return nil -} - -func (a *refAggregator) PointingTo() []string { - keys := make([]string, 0, len(a.valueCounter)) - for pointingTo := range a.valueCounter { - keys = append(keys, pointingTo) - } - return keys -} diff --git a/adapters/repos/db/aggregator/shard_combiner.go b/adapters/repos/db/aggregator/shard_combiner.go deleted file mode 100644 index dcb35cd47c230876ea228c4abace2d28846d752f..0000000000000000000000000000000000000000 --- a/adapters/repos/db/aggregator/shard_combiner.go +++ /dev/null @@ -1,341 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package aggregator - -import ( - "sort" - "time" - - "github.com/weaviate/weaviate/entities/aggregation" -) - -type ShardCombiner struct{} - -func NewShardCombiner() *ShardCombiner { - return &ShardCombiner{} -} - -func (sc *ShardCombiner) Do(results []*aggregation.Result) *aggregation.Result { - allResultsAreNil := true - firstNonNilRes := 0 - for i, res := range results { - if res == nil || len(res.Groups) < 1 { - continue - } - allResultsAreNil = false - firstNonNilRes = i - } - - if allResultsAreNil { - return &aggregation.Result{} - } - - if results[firstNonNilRes].Groups[0].GroupedBy == nil { - return sc.combineUngrouped(results) - } - - return sc.combineGrouped(results) -} - -func (sc *ShardCombiner) combineUngrouped(results []*aggregation.Result) *aggregation.Result { - combined := aggregation.Result{ - Groups: make([]aggregation.Group, 1), - } - - for _, shard := range results { - if len(shard.Groups) == 0 { // not every shard has results - continue - } - sc.mergeIntoCombinedGroupAtPos(combined.Groups, 0, shard.Groups[0]) - } - - sc.finalizeGroup(&combined.Groups[0]) - return &combined -} - -func (sc *ShardCombiner) combineGrouped(results []*aggregation.Result) *aggregation.Result { - combined := aggregation.Result{} - - for _, shard := range results { - for _, shardGroup := range shard.Groups { - pos := getPosOfGroup(combined.Groups, shardGroup.GroupedBy.Value) - if pos < 0 { - combined.Groups = append(combined.Groups, shardGroup) - } else { - sc.mergeIntoCombinedGroupAtPos(combined.Groups, pos, shardGroup) - } - } - } - - for i := range combined.Groups { - sc.finalizeGroup(&combined.Groups[i]) - } - - sort.Slice(combined.Groups, func(a, b int) bool { - return combined.Groups[a].Count > combined.Groups[b].Count - }) - return &combined -} - -func (sc *ShardCombiner) mergeIntoCombinedGroupAtPos(combinedGroups []aggregation.Group, - pos int, shardGroup aggregation.Group, -) { - combinedGroups[pos].Count += shardGroup.Count - - for propName, prop := range shardGroup.Properties { - if combinedGroups[pos].Properties == nil { - combinedGroups[pos].Properties = map[string]aggregation.Property{} - } - - combinedProp := combinedGroups[pos].Properties[propName] - - combinedProp.Type = prop.Type - - switch prop.Type { - case aggregation.PropertyTypeNumerical: - if combinedProp.NumericalAggregations == nil { - combinedProp.NumericalAggregations = map[string]interface{}{} - } - sc.mergeNumericalProp( - combinedProp.NumericalAggregations, prop.NumericalAggregations) - case aggregation.PropertyTypeDate: - if combinedProp.DateAggregations == nil { - combinedProp.DateAggregations = map[string]interface{}{} - } - sc.mergeDateProp( - combinedProp.DateAggregations, prop.DateAggregations) - case aggregation.PropertyTypeBoolean: - sc.mergeBooleanProp( - &combinedProp.BooleanAggregation, &prop.BooleanAggregation) - case aggregation.PropertyTypeText: - sc.mergeTextProp( - &combinedProp.TextAggregation, &prop.TextAggregation) - case aggregation.PropertyTypeReference: - sc.mergeRefProp( - &combinedProp.ReferenceAggregation, &prop.ReferenceAggregation) - default: - panic("unknown prop type: " + prop.Type) - } - combinedGroups[pos].Properties[propName] = combinedProp - - } -} - -func (sc *ShardCombiner) mergeDateProp(first, second map[string]interface{}) { - if len(second) == 0 { - return - } - - // add all values from the second map to the first one. This is needed to compute median and mode correctly - for propType := range second { - switch propType { - case "_dateAggregator": - dateAggSource := second[propType].(*dateAggregator) - if dateAggCombined, ok := first[propType]; ok { - dateAggCombinedTyped := dateAggCombined.(*dateAggregator) - for _, pair := range dateAggSource.pairs { - for i := uint64(0); i < pair.count; i++ { - dateAggCombinedTyped.AddTimestamp(pair.value.rfc3339) - } - } - dateAggCombinedTyped.buildPairsFromCounts() - first[propType] = dateAggCombinedTyped - - } else { - first[propType] = second[propType] - } - } - } - - for propType, value := range second { - switch propType { - case "count": - if val, ok := first[propType]; ok { - first[propType] = val.(int64) + value.(int64) - } else { - first[propType] = value - } - case "mode": - dateAggCombined := first["_dateAggregator"].(*dateAggregator) - first[propType] = dateAggCombined.Mode() - case "median": - dateAggCombined := first["_dateAggregator"].(*dateAggregator) - first[propType] = dateAggCombined.Median() - case "minimum": - val, ok := first["minimum"] - if !ok { - first["minimum"] = value - } else { - source1Time, _ := time.Parse(time.RFC3339, val.(string)) - source2Time, _ := time.Parse(time.RFC3339, value.(string)) - if source2Time.Before(source1Time) { - first["minimum"] = value - } - } - case "maximum": - val, ok := first["maximum"] - if !ok { - first["maximum"] = value - } else { - source1Time, _ := time.Parse(time.RFC3339, val.(string)) - source2Time, _ := time.Parse(time.RFC3339, value.(string)) - if source2Time.After(source1Time) { - first["maximum"] = value - } - } - case "_dateAggregator": - continue - default: - panic("unknown map entry: " + propType) - } - } -} - -func (sc *ShardCombiner) mergeNumericalProp(first, second map[string]interface{}) { - if len(second) == 0 { - return - } - - // add all values from the second map to the first one. This is needed to compute median, mean and mode correctly - for propType := range second { - switch propType { - case "_numericalAggregator": - numAggSecondTyped := second[propType].(*numericalAggregator) - if numAggFirst, ok := first[propType]; ok { - numAggFirstTyped := numAggFirst.(*numericalAggregator) - for _, pair := range numAggSecondTyped.pairs { - for i := uint64(0); i < pair.count; i++ { - numAggFirstTyped.AddFloat64(pair.value) - } - } - numAggFirstTyped.buildPairsFromCounts() - first[propType] = numAggFirstTyped - } else { - first[propType] = second[propType] - } - } - } - - for propType, value := range second { - switch propType { - case "count", "sum": - if val, ok := first[propType]; ok { - first[propType] = val.(float64) + value.(float64) - } else { - first[propType] = value - } - case "mode": - numAggFirst := first["_numericalAggregator"].(*numericalAggregator) - first[propType] = numAggFirst.Mode() - case "mean": - numAggFirst := first["_numericalAggregator"].(*numericalAggregator) - first[propType] = numAggFirst.Mean() - case "median": - numAggFirst := first["_numericalAggregator"].(*numericalAggregator) - first[propType] = numAggFirst.Median() - case "minimum": - if _, ok := first["minimum"]; !ok || value.(float64) < first["minimum"].(float64) { - first["minimum"] = value - } - case "maximum": - if _, ok := first["maximum"]; !ok || value.(float64) > first["maximum"].(float64) { - first["maximum"] = value - } - case "_numericalAggregator": - continue - default: - panic("unknown map entry: " + propType) - } - } -} - -func (sc *ShardCombiner) finalizeDateProp(combined map[string]interface{}) { - delete(combined, "_dateAggregator") -} - -func (sc *ShardCombiner) finalizeNumerical(combined map[string]interface{}) { - delete(combined, "_numericalAggregator") -} - -func (sc *ShardCombiner) mergeBooleanProp(combined, source *aggregation.Boolean) { - combined.Count += source.Count - combined.TotalFalse += source.TotalFalse - combined.TotalTrue += source.TotalTrue -} - -func (sc *ShardCombiner) finalizeBoolean(combined *aggregation.Boolean) { - combined.PercentageFalse = float64(combined.TotalFalse) / float64(combined.Count) - combined.PercentageTrue = float64(combined.TotalTrue) / float64(combined.Count) -} - -func (sc *ShardCombiner) mergeTextProp(first, second *aggregation.Text) { - first.Count += second.Count - - for _, textOcc := range second.Items { - pos := getPosOfTextOcc(first.Items, textOcc.Value) - if pos < 0 { - first.Items = append(first.Items, textOcc) - } else { - first.Items[pos].Occurs += textOcc.Occurs - } - } -} - -func (sc *ShardCombiner) mergeRefProp(first, second *aggregation.Reference) { - first.PointingTo = append(first.PointingTo, second.PointingTo...) -} - -func (sc *ShardCombiner) finalizeText(combined *aggregation.Text) { - sort.Slice(combined.Items, func(a, b int) bool { - return combined.Items[a].Occurs > combined.Items[b].Occurs - }) -} - -func getPosOfTextOcc(haystack []aggregation.TextOccurrence, needle string) int { - for i, elem := range haystack { - if elem.Value == needle { - return i - } - } - - return -1 -} - -func (sc *ShardCombiner) finalizeGroup(group *aggregation.Group) { - for propName, prop := range group.Properties { - switch prop.Type { - case aggregation.PropertyTypeNumerical: - sc.finalizeNumerical(prop.NumericalAggregations) - case aggregation.PropertyTypeBoolean: - sc.finalizeBoolean(&prop.BooleanAggregation) - case aggregation.PropertyTypeText: - sc.finalizeText(&prop.TextAggregation) - case aggregation.PropertyTypeDate: - sc.finalizeDateProp(prop.DateAggregations) - case aggregation.PropertyTypeReference: - continue - default: - panic("Unknown prop type: " + prop.Type) - } - group.Properties[propName] = prop - } -} - -func getPosOfGroup(haystack []aggregation.Group, needle interface{}) int { - for i, elem := range haystack { - if elem.GroupedBy.Value == needle { - return i - } - } - - return -1 -} diff --git a/adapters/repos/db/aggregator/shard_combiner_test.go b/adapters/repos/db/aggregator/shard_combiner_test.go deleted file mode 100644 index f2ec27f26e5256128ccf04816355104f0d08d260..0000000000000000000000000000000000000000 --- a/adapters/repos/db/aggregator/shard_combiner_test.go +++ /dev/null @@ -1,249 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package aggregator - -import ( - "math/rand" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/entities/aggregation" -) - -const ( - YearMonthDayHourMinute = "2022-06-16T18:30:" - NanoSecondsTimeZone = ".451235Z" -) - -type TestStructDates struct { - name string - dates1 []string - dates2 []string - expectedMedian string - expectedMaximum string - expectedMode string - expectedMinimum string -} - -func TestShardCombinerMergeDates(t *testing.T) { - tests := []TestStructDates{ - { - name: "Many values", - dates1: []string{"55", "26", "10"}, - dates2: []string{"15", "26", "45", "26"}, - expectedMaximum: "55", - expectedMinimum: "10", - expectedMedian: "26", - expectedMode: "26", - }, - { - name: "Struct with single element", - dates1: []string{"45"}, - dates2: []string{"00", "26", "45", "27"}, - expectedMaximum: "45", - expectedMinimum: "00", - expectedMedian: "27", - expectedMode: "45", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - testDates(t, tt.dates1, tt.dates2, tt) - testDates(t, tt.dates2, tt.dates1, tt) - }) - } -} - -func testDates(t *testing.T, dates1, dates2 []string, tt TestStructDates) { - sc := NewShardCombiner() - dateMap1 := createDateAgg(dates1) - dateMap2 := createDateAgg(dates2) - - sc.mergeDateProp(dateMap1, dateMap2) - sc.finalizeDateProp(dateMap1) - assert.Equal(t, YearMonthDayHourMinute+tt.expectedMinimum+NanoSecondsTimeZone, dateMap1["minimum"]) - assert.Equal(t, YearMonthDayHourMinute+tt.expectedMaximum+NanoSecondsTimeZone, dateMap1["maximum"]) - assert.Equal(t, YearMonthDayHourMinute+tt.expectedMedian+NanoSecondsTimeZone, dateMap1["median"]) - assert.Equal(t, int64(len(tt.dates1)+len(tt.dates2)), dateMap1["count"]) - assert.Equal(t, YearMonthDayHourMinute+tt.expectedMode+NanoSecondsTimeZone, dateMap1["mode"]) -} - -func createDateAgg(dates []string) map[string]interface{} { - agg := newDateAggregator() - for _, date := range dates { - agg.AddTimestamp(YearMonthDayHourMinute + date + NanoSecondsTimeZone) - } - agg.buildPairsFromCounts() // needed to populate all required info - - prop := aggregation.Property{} - aggs := []aggregation.Aggregator{aggregation.MedianAggregator, aggregation.MinimumAggregator, aggregation.MaximumAggregator, aggregation.CountAggregator, aggregation.ModeAggregator} - addDateAggregations(&prop, aggs, agg) - return prop.DateAggregations -} - -type TestStructNumbers struct { - name string - numbers1 []float64 - numbers2 []float64 - testMode bool -} - -func TestShardCombinerMergeNumerical(t *testing.T) { - tests := []TestStructNumbers{ - { - name: "Uneven number of elements for both", - numbers1: []float64{0, 9, 9}, - numbers2: []float64{2}, - testMode: true, - }, - { - name: "Even number of elements for both", - numbers1: []float64{0, 5, 10, 15}, - numbers2: []float64{15, 15}, - testMode: true, - }, - { - name: "Mode is affected by merge", - numbers1: []float64{2.5, 2.5, 10, 15}, - numbers2: []float64{15, 15}, - testMode: true, - }, - { - name: "random", - numbers1: createRandomSlice(), - numbers2: createRandomSlice(), - testMode: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - testNumbers(t, tt.numbers1, tt.numbers2, tt.testMode) - testNumbers(t, tt.numbers2, tt.numbers1, tt.testMode) - }) - } -} - -func TestShardCombinerMergeNil(t *testing.T) { - tests := []struct { - name string - results []*aggregation.Result - totalResults int - }{ - { - name: "First is nil", - results: []*aggregation.Result{ - { - Groups: []aggregation.Group{}, - }, - { - Groups: []aggregation.Group{{GroupedBy: &aggregation.GroupedBy{Value: 10, Path: []string{"something"}}}}, - }, - }, - totalResults: 1, - }, - { - name: "Second is nil", - results: []*aggregation.Result{ - { - Groups: []aggregation.Group{{GroupedBy: &aggregation.GroupedBy{Value: 10, Path: []string{"something"}}}}, - }, - { - Groups: []aggregation.Group{}, - }, - }, - totalResults: 1, - }, - { - name: "Both are nil", - results: []*aggregation.Result{ - { - Groups: []aggregation.Group{}, - }, - { - Groups: []aggregation.Group{}, - }, - }, - totalResults: 0, - }, - { - name: "Non are nil", - results: []*aggregation.Result{ - { - Groups: []aggregation.Group{{GroupedBy: &aggregation.GroupedBy{Value: 9, Path: []string{"other thing"}}}}, - }, - { - Groups: []aggregation.Group{{GroupedBy: &aggregation.GroupedBy{Value: 10, Path: []string{"something"}}}}, - }, - }, - totalResults: 2, - }, - { - name: "Ungrouped with nil", - results: []*aggregation.Result{ - { - Groups: []aggregation.Group{{Count: 1}}, - }, - { - Groups: []aggregation.Group{}, - }, - }, - totalResults: 1, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - combinedResults := NewShardCombiner().Do(tt.results) - assert.Equal(t, len(combinedResults.Groups), tt.totalResults) - }) - } -} - -func testNumbers(t *testing.T, numbers1, numbers2 []float64, testMode bool) { - sc := NewShardCombiner() - numberMap1 := createNumericalAgg(numbers1) - numberMap2 := createNumericalAgg(numbers2) - - combinedMap := createNumericalAgg(append(numbers1, numbers2...)) - - sc.mergeNumericalProp(numberMap1, numberMap2) - sc.finalizeNumerical(numberMap1) - - assert.Equal(t, len(numbers1)+len(numbers2), int(numberMap1["count"].(float64))) - assert.InDelta(t, combinedMap["mean"], numberMap1["mean"], 0.0001) - assert.InDelta(t, combinedMap["median"], numberMap1["median"], 0.0001) - if testMode { // for random numbers the mode is flaky as there is no guaranteed order if several values have the same count - assert.Equal(t, combinedMap["mode"], numberMap1["mode"]) - } -} - -func createNumericalAgg(numbers []float64) map[string]interface{} { - agg := newNumericalAggregator() - for _, num := range numbers { - agg.AddFloat64(num) - } - agg.buildPairsFromCounts() // needed to populate all required info - - prop := aggregation.Property{} - aggs := []aggregation.Aggregator{aggregation.MedianAggregator, aggregation.MeanAggregator, aggregation.ModeAggregator, aggregation.CountAggregator} - addNumericalAggregations(&prop, aggs, agg) - return prop.NumericalAggregations -} - -func createRandomSlice() []float64 { - size := rand.Intn(100) + 1 // at least one entry - array := make([]float64, size) - for i := 0; i < size; i++ { - array[i] = rand.Float64() * 1000 - } - return array -} diff --git a/adapters/repos/db/aggregator/text.go b/adapters/repos/db/aggregator/text.go deleted file mode 100644 index c0093c1b21e43aead47a0a429844a7fbca1ec76c..0000000000000000000000000000000000000000 --- a/adapters/repos/db/aggregator/text.go +++ /dev/null @@ -1,149 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package aggregator - -import ( - "sort" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/aggregation" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/storobj" -) - -func extractLimitFromTopOccs(aggs []aggregation.Aggregator) int { - for _, agg := range aggs { - if agg.Type == aggregation.TopOccurrencesType && agg.Limit != nil { - return *agg.Limit - } - } - - // we couldn't extract a limit, default to something reasonable - return 5 -} - -func newTextAggregator(limit int) *textAggregator { - return &textAggregator{itemCounter: map[string]int{}, max: limit} -} - -type textAggregator struct { - max int - count uint64 - - itemCounter map[string]int - - // always keep sorted, so we can cut off the last elem, when it grows larger - // than max - topPairs []aggregation.TextOccurrence -} - -func (a *Aggregator) parseAndAddTextRow(agg *textAggregator, - v []byte, propName schema.PropertyName, -) error { - items, ok, err := storobj.ParseAndExtractTextProp(v, propName.String()) - if err != nil { - return errors.Wrap(err, "parse and extract prop") - } - - if !ok { - return nil - } - - for i := range items { - if err := agg.AddText(items[i]); err != nil { - return err - } - } - return nil -} - -func (a *textAggregator) AddText(value string) error { - a.count++ - - itemCount := a.itemCounter[value] - itemCount++ - a.itemCounter[value] = itemCount - return nil -} - -func (a *textAggregator) insertOrdered(elem aggregation.TextOccurrence) { - if len(a.topPairs) == 0 { - a.topPairs = []aggregation.TextOccurrence{elem} - return - } - - added := false - for i, pair := range a.topPairs { - if pair.Occurs > elem.Occurs { - continue - } - // if number of occurrences is the same, - // skip if string is after one in topPairs - if pair.Occurs == elem.Occurs && pair.Value < elem.Value { - continue - } - - // we have found the first one that's smaller so me must insert before i - a.topPairs = append( - a.topPairs[:i], append( - []aggregation.TextOccurrence{elem}, - a.topPairs[i:]..., - )..., - ) - - added = true - break - } - - if len(a.topPairs) > a.max { - a.topPairs = a.topPairs[:len(a.topPairs)-1] - } - - if !added && len(a.topPairs) < a.max { - a.topPairs = append(a.topPairs, elem) - } -} - -func (a *textAggregator) Res() aggregation.Text { - out := aggregation.Text{} - if a.count == 0 { - return out - } - - for value, count := range a.itemCounter { - a.insertOrdered(aggregation.TextOccurrence{ - Value: value, - Occurs: count, - }) - } - - out.Items = a.topPairs - sort.SliceStable(out.Items, func(a, b int) bool { - countA := out.Items[a].Occurs - countB := out.Items[b].Occurs - - if countA != countB { - return countA > countB - } - - valueA := out.Items[a].Value - valueB := out.Items[b].Value - if len(valueA) == 0 || len(valueB) == 0 { - return false // order doesn't matter in this case, just prevent a panic - } - - return valueA[0] < valueB[0] - }) - - out.Count = int(a.count) - return out -} diff --git a/adapters/repos/db/aggregator/text_test.go b/adapters/repos/db/aggregator/text_test.go deleted file mode 100644 index af0dbc65ab89dc6132d0f3430fe64bfa8d9f93b8..0000000000000000000000000000000000000000 --- a/adapters/repos/db/aggregator/text_test.go +++ /dev/null @@ -1,98 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package aggregator - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/entities/aggregation" -) - -func TestTextAggregator_TopOccurrencesCalculation(t *testing.T) { - testCases := []struct { - name string - texts []string - expectedCount int - expectedTopOccurrences []aggregation.TextOccurrence - }{ - { - name: "All texts occurring once", - texts: []string{"b_occurs1", "c_occurs1", "g_occurs1", "f_occurs1", "a_occurs1", "d_occurs1", "e_occurs1"}, - expectedCount: 7, - expectedTopOccurrences: []aggregation.TextOccurrence{ - {Value: "a_occurs1", Occurs: 1}, - {Value: "b_occurs1", Occurs: 1}, - {Value: "c_occurs1", Occurs: 1}, - {Value: "d_occurs1", Occurs: 1}, - {Value: "e_occurs1", Occurs: 1}, - }, - }, - { - name: "All texts occurring different number of times", - texts: []string{ - "b_occurs2", "e_occurs5", "d_occurs4", "c_occurs3", "g_occurs7", "e_occurs5", "d_occurs4", - "f_occurs6", "g_occurs7", "c_occurs3", "b_occurs2", "g_occurs7", "f_occurs6", "g_occurs7", "d_occurs4", - "a_occurs1", "f_occurs6", "g_occurs7", "g_occurs7", "f_occurs6", "d_occurs4", "e_occurs5", "g_occurs7", - "c_occurs3", "f_occurs6", "e_occurs5", "f_occurs6", "e_occurs5", - }, - expectedCount: 28, - expectedTopOccurrences: []aggregation.TextOccurrence{ - {Value: "g_occurs7", Occurs: 7}, - {Value: "f_occurs6", Occurs: 6}, - {Value: "e_occurs5", Occurs: 5}, - {Value: "d_occurs4", Occurs: 4}, - {Value: "c_occurs3", Occurs: 3}, - }, - }, - { - name: "Some texts occurring same number of times", - texts: []string{ - "a_occurs4", "b_occurs3", "g_occurs4", "f_occurs3", "a_occurs4", "e_occurs2", "a_occurs4", - "c_occurs2", "g_occurs4", "f_occurs3", "b_occurs3", "c_occurs2", "a_occurs4", "f_occurs3", "g_occurs4", - "b_occurs3", "d_occurs1", "e_occurs2", "g_occurs4", - }, - expectedCount: 19, - expectedTopOccurrences: []aggregation.TextOccurrence{ - {Value: "a_occurs4", Occurs: 4}, - {Value: "g_occurs4", Occurs: 4}, - {Value: "b_occurs3", Occurs: 3}, - {Value: "f_occurs3", Occurs: 3}, - {Value: "c_occurs2", Occurs: 2}, - }, - }, - { - name: "Fewer texts than limit", - texts: []string{"b_occurs3", "d_occurs3", "c_occurs1", "d_occurs3", "b_occurs3", "b_occurs3", "a_occurs1", "d_occurs3"}, - expectedCount: 8, - expectedTopOccurrences: []aggregation.TextOccurrence{ - {Value: "b_occurs3", Occurs: 3}, - {Value: "d_occurs3", Occurs: 3}, - {Value: "a_occurs1", Occurs: 1}, - {Value: "c_occurs1", Occurs: 1}, - }, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - agg := newTextAggregator(5) - for _, text := range tc.texts { - agg.AddText(text) - } - - res := agg.Res() - assert.Equal(t, tc.expectedCount, res.Count) - assert.Equal(t, tc.expectedTopOccurrences, res.Items) - }) - } -} diff --git a/adapters/repos/db/aggregator/unfiltered.go b/adapters/repos/db/aggregator/unfiltered.go deleted file mode 100644 index d31855955748a0beaf2755a782301a5c38cc97a3..0000000000000000000000000000000000000000 --- a/adapters/repos/db/aggregator/unfiltered.go +++ /dev/null @@ -1,145 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package aggregator - -import ( - "context" - "fmt" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/entities/aggregation" - "github.com/weaviate/weaviate/entities/schema" -) - -// unfilteredAggregator allows for relatively efficient whole-dataset -// aggregations, because it uses the invert index which is already grouped and -// ordered. Numerical aggregations can therefore be relatively efficient. -// -// As opposed to reading n objects, the unfiltered aggregator read x rows per -// props. X can be different for each prop. -// -// However, this aggregator does not work with subselections of the dataset, -// such as when grouping or a filter is set. -type unfilteredAggregator struct { - *Aggregator -} - -func newUnfilteredAggregator(agg *Aggregator) *unfilteredAggregator { - return &unfilteredAggregator{Aggregator: agg} -} - -func (ua *unfilteredAggregator) Do(ctx context.Context) (*aggregation.Result, error) { - out := aggregation.Result{} - - // without grouping there is always exactly one group - out.Groups = make([]aggregation.Group, 1) - - if ua.params.IncludeMetaCount { - if err := ua.addMetaCount(ctx, &out); err != nil { - return nil, errors.Wrap(err, "add meta count") - } - } - - props, err := ua.properties(ctx) - if err != nil { - return nil, errors.Wrap(err, "aggregate properties") - } - - out.Groups[0].Properties = props - - return &out, nil -} - -func (ua *unfilteredAggregator) addMetaCount(ctx context.Context, - out *aggregation.Result, -) error { - b := ua.store.Bucket(helpers.ObjectsBucketLSM) - if b == nil { - return errors.Errorf("objects bucket is nil") - } - - out.Groups[0].Count = b.Count() - - return nil -} - -func (ua unfilteredAggregator) properties( - ctx context.Context, -) (map[string]aggregation.Property, error) { - if len(ua.params.Properties) == 0 { - return nil, nil - } - - out := map[string]aggregation.Property{} - - for _, prop := range ua.params.Properties { - if err := ctx.Err(); err != nil { - return nil, errors.Wrapf(err, "start property %s", prop.Name) - } - - analyzed, err := ua.property(ctx, prop) - if err != nil { - return nil, errors.Wrapf(err, "property %s", prop.Name) - } - - if analyzed == nil { - continue - } - - out[prop.Name.String()] = *analyzed - } - - return out, nil -} - -func (ua unfilteredAggregator) property(ctx context.Context, - prop aggregation.ParamProperty, -) (*aggregation.Property, error) { - aggType, dt, err := ua.aggTypeOfProperty(prop.Name) - if err != nil { - return nil, err - } - - switch aggType { - case aggregation.PropertyTypeNumerical: - switch dt { - case schema.DataTypeNumber: - return ua.floatProperty(ctx, prop) - case schema.DataTypeNumberArray, schema.DataTypeIntArray: - return ua.numberArrayProperty(ctx, prop) - default: - return ua.intProperty(ctx, prop) - } - case aggregation.PropertyTypeBoolean: - switch dt { - case schema.DataTypeBooleanArray: - return ua.boolArrayProperty(ctx, prop) - default: - return ua.boolProperty(ctx, prop) - } - case aggregation.PropertyTypeText: - return ua.textProperty(ctx, prop) - case aggregation.PropertyTypeDate: - switch dt { - case schema.DataTypeDateArray: - return ua.dateArrayProperty(ctx, prop) - default: - return ua.dateProperty(ctx, prop) - } - case aggregation.PropertyTypeReference: - // ignore, as this is handled outside the repo in the uc - return nil, nil - default: - return nil, fmt.Errorf("aggreation type %s not supported yet", aggType) - } -} diff --git a/adapters/repos/db/aggregator/unfiltered_type_specific.go b/adapters/repos/db/aggregator/unfiltered_type_specific.go deleted file mode 100644 index 15aef4d5931dd07548a6ffa0d3e9e4d4a53fb774..0000000000000000000000000000000000000000 --- a/adapters/repos/db/aggregator/unfiltered_type_specific.go +++ /dev/null @@ -1,503 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package aggregator - -import ( - "context" - "fmt" - - "github.com/pkg/errors" - "github.com/weaviate/sroar" - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv" - "github.com/weaviate/weaviate/entities/aggregation" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/storobj" -) - -func (ua unfilteredAggregator) boolProperty(ctx context.Context, - prop aggregation.ParamProperty, -) (*aggregation.Property, error) { - out := aggregation.Property{ - Type: aggregation.PropertyTypeBoolean, - } - - b := ua.store.Bucket(helpers.BucketFromPropNameLSM(prop.Name.String())) - if b == nil { - return nil, errors.Errorf("could not find bucket for prop %s", prop.Name) - } - - agg := newBoolAggregator() - - // bool never has a frequency, so it's either a Set or RoaringSet - if b.Strategy() == lsmkv.StrategyRoaringSet { - c := b.CursorRoaringSet() - defer c.Close() - - for k, v := c.First(); k != nil; k, v = c.Next() { - err := ua.parseAndAddBoolRowRoaringSet(agg, k, v) - if err != nil { - return nil, err - } - } - } else { - c := b.SetCursor() // bool never has a frequency, so it's always a Set - defer c.Close() - - for k, v := c.First(); k != nil; k, v = c.Next() { - err := ua.parseAndAddBoolRowSet(agg, k, v) - if err != nil { - return nil, err - } - } - } - - out.BooleanAggregation = agg.Res() - - return &out, nil -} - -func (ua unfilteredAggregator) boolArrayProperty(ctx context.Context, - prop aggregation.ParamProperty, -) (*aggregation.Property, error) { - out := aggregation.Property{ - Type: aggregation.PropertyTypeBoolean, - } - - b := ua.store.Bucket(helpers.ObjectsBucketLSM) - if b == nil { - return nil, errors.Errorf("could not find bucket for prop %s", prop.Name) - } - - agg := newBoolAggregator() - - c := b.Cursor() - defer c.Close() - - for k, v := c.First(); k != nil; k, v = c.Next() { - err := ua.parseAndAddBoolArrayRow(agg, v, prop.Name) - if err != nil { - return nil, err - } - } - - out.BooleanAggregation = agg.Res() - - return &out, nil -} - -func (ua unfilteredAggregator) parseAndAddBoolRowSet(agg *boolAggregator, k []byte, v [][]byte) error { - if len(k) != 1 { - // we expect to see a single byte for a marshalled bool - return fmt.Errorf("unexpected key length on inverted index, "+ - "expected 1: got %d", len(k)) - } - - if err := agg.AddBoolRow(k, uint64(len(v))); err != nil { - return err - } - - return nil -} - -func (ua unfilteredAggregator) parseAndAddBoolRowRoaringSet(agg *boolAggregator, k []byte, v *sroar.Bitmap) error { - if len(k) != 1 { - // we expect to see a single byte for a marshalled bool - return fmt.Errorf("unexpected key length on inverted index, "+ - "expected 1: got %d", len(k)) - } - - if err := agg.AddBoolRow(k, uint64(v.GetCardinality())); err != nil { - return err - } - - return nil -} - -func (ua unfilteredAggregator) parseAndAddBoolArrayRow(agg *boolAggregator, - v []byte, propName schema.PropertyName, -) error { - items, ok, err := storobj.ParseAndExtractBoolArrayProp(v, propName.String()) - if err != nil { - return errors.Wrap(err, "parse and extract prop") - } - - if !ok { - return nil - } - - for i := range items { - if err := agg.AddBool(items[i]); err != nil { - return err - } - } - - return nil -} - -func (ua unfilteredAggregator) floatProperty(ctx context.Context, - prop aggregation.ParamProperty, -) (*aggregation.Property, error) { - out := aggregation.Property{ - Type: aggregation.PropertyTypeNumerical, - NumericalAggregations: map[string]interface{}{}, - } - - b := ua.store.Bucket(helpers.BucketFromPropNameLSM(prop.Name.String())) - if b == nil { - return nil, errors.Errorf("could not find bucket for prop %s", prop.Name) - } - - agg := newNumericalAggregator() - - // flat never has a frequency, so it's either a Set or RoaringSet - if b.Strategy() == lsmkv.StrategyRoaringSet { - c := b.CursorRoaringSet() - defer c.Close() - - for k, v := c.First(); k != nil; k, v = c.Next() { - if err := ua.parseAndAddFloatRowRoaringSet(agg, k, v); err != nil { - return nil, err - } - } - } else { - c := b.SetCursor() - defer c.Close() - - for k, v := c.First(); k != nil; k, v = c.Next() { - if err := ua.parseAndAddFloatRowSet(agg, k, v); err != nil { - return nil, err - } - } - } - - addNumericalAggregations(&out, prop.Aggregators, agg) - - return &out, nil -} - -func (ua unfilteredAggregator) intProperty(ctx context.Context, - prop aggregation.ParamProperty, -) (*aggregation.Property, error) { - out := aggregation.Property{ - Type: aggregation.PropertyTypeNumerical, - NumericalAggregations: map[string]interface{}{}, - } - - b := ua.store.Bucket(helpers.BucketFromPropNameLSM(prop.Name.String())) - if b == nil { - return nil, errors.Errorf("could not find bucket for prop %s", prop.Name) - } - - agg := newNumericalAggregator() - - // int never has a frequency, so it's either a Set or RoaringSet - if b.Strategy() == lsmkv.StrategyRoaringSet { - c := b.CursorRoaringSet() - defer c.Close() - - for k, v := c.First(); k != nil; k, v = c.Next() { - if err := ua.parseAndAddIntRowRoaringSet(agg, k, v); err != nil { - return nil, err - } - } - } else { - - c := b.SetCursor() - defer c.Close() - - for k, v := c.First(); k != nil; k, v = c.Next() { - if err := ua.parseAndAddIntRowSet(agg, k, v); err != nil { - return nil, err - } - } - } - - addNumericalAggregations(&out, prop.Aggregators, agg) - - return &out, nil -} - -func (ua unfilteredAggregator) dateProperty(ctx context.Context, - prop aggregation.ParamProperty, -) (*aggregation.Property, error) { - out := aggregation.Property{ - Type: aggregation.PropertyTypeDate, - DateAggregations: map[string]interface{}{}, - } - - b := ua.store.Bucket(helpers.BucketFromPropNameLSM(prop.Name.String())) - if b == nil { - return nil, errors.Errorf("could not find bucket for prop %s", prop.Name) - } - - agg := newDateAggregator() - - // dates don't have frequency, so it's either a Set or RoaringSet - if b.Strategy() == lsmkv.StrategyRoaringSet { - c := b.CursorRoaringSet() - defer c.Close() - - for k, v := c.First(); k != nil; k, v = c.Next() { - if err := ua.parseAndAddDateRowRoaringSet(agg, k, v); err != nil { - return nil, err - } - } - } else { - c := b.SetCursor() - defer c.Close() - - for k, v := c.First(); k != nil; k, v = c.Next() { - if err := ua.parseAndAddDateRowSet(agg, k, v); err != nil { - return nil, err - } - } - } - - addDateAggregations(&out, prop.Aggregators, agg) - - return &out, nil -} - -func (ua unfilteredAggregator) parseAndAddDateRowSet(agg *dateAggregator, k []byte, - v [][]byte, -) error { - if len(k) != 8 { - // dates are stored as epoch nanoseconds, we expect to see an int64 - return fmt.Errorf("unexpected key length on inverted index, "+ - "expected 8: got %d", len(k)) - } - - if err := agg.AddTimestampRow(k, uint64(len(v))); err != nil { - return err - } - - return nil -} - -func (ua unfilteredAggregator) parseAndAddDateRowRoaringSet(agg *dateAggregator, k []byte, - v *sroar.Bitmap, -) error { - if len(k) != 8 { - // dates are stored as epoch nanoseconds, we expect to see an int64 - return fmt.Errorf("unexpected key length on inverted index, "+ - "expected 8: got %d", len(k)) - } - - if err := agg.AddTimestampRow(k, uint64(v.GetCardinality())); err != nil { - return err - } - - return nil -} - -func (ua unfilteredAggregator) dateArrayProperty(ctx context.Context, - prop aggregation.ParamProperty, -) (*aggregation.Property, error) { - out := aggregation.Property{ - Type: aggregation.PropertyTypeDate, - DateAggregations: map[string]interface{}{}, - } - - b := ua.store.Bucket(helpers.ObjectsBucketLSM) - if b == nil { - return nil, errors.Errorf("could not find bucket for prop %s", prop.Name) - } - - agg := newDateAggregator() - - c := b.Cursor() - defer c.Close() - - for k, v := c.First(); k != nil; k, v = c.Next() { - if err := ua.parseAndAddDateArrayRow(agg, v, prop.Name); err != nil { - return nil, err - } - } - - addDateAggregations(&out, prop.Aggregators, agg) - - return &out, nil -} - -func (ua unfilteredAggregator) parseAndAddDateArrayRow(agg *dateAggregator, - v []byte, propName schema.PropertyName, -) error { - items, ok, err := storobj.ParseAndExtractProperty(v, propName.String()) - if err != nil { - return errors.Wrap(err, "parse and extract prop") - } - - if !ok { - return nil - } - - for i := range items { - if err := agg.AddTimestamp(items[i]); err != nil { - return err - } - } - - return nil -} - -func (ua unfilteredAggregator) parseAndAddFloatRowSet(agg *numericalAggregator, k []byte, - v [][]byte, -) error { - if len(k) != 8 { - // we expect to see either an int64 or a float64, so any non-8 length - // is unexpected - return fmt.Errorf("unexpected key length on inverted index, "+ - "expected 8: got %d", len(k)) - } - - if err := agg.AddFloat64Row(k, uint64(len(v))); err != nil { - return err - } - - return nil -} - -func (ua unfilteredAggregator) parseAndAddFloatRowRoaringSet(agg *numericalAggregator, k []byte, - v *sroar.Bitmap, -) error { - if len(k) != 8 { - // we expect to see either an int64 or a float64, so any non-8 length - // is unexpected - return fmt.Errorf("unexpected key length on inverted index, "+ - "expected 8: got %d", len(k)) - } - - if err := agg.AddFloat64Row(k, uint64(v.GetCardinality())); err != nil { - return err - } - - return nil -} - -func (ua unfilteredAggregator) parseAndAddIntRowSet(agg *numericalAggregator, k []byte, - v [][]byte, -) error { - if len(k) != 8 { - // we expect to see either an int64 or a float64, so any non-8 length - // is unexpected - return fmt.Errorf("unexpected key length on inverted index, "+ - "expected 8: got %d", len(k)) - } - - if err := agg.AddInt64Row(k, uint64(len(v))); err != nil { - return err - } - - return nil -} - -func (ua unfilteredAggregator) parseAndAddIntRowRoaringSet(agg *numericalAggregator, k []byte, - v *sroar.Bitmap, -) error { - if len(k) != 8 { - // we expect to see either an int64 or a float64, so any non-8 length - // is unexpected - return fmt.Errorf("unexpected key length on inverted index, "+ - "expected 8: got %d", len(k)) - } - - if err := agg.AddInt64Row(k, uint64(v.GetCardinality())); err != nil { - return err - } - - return nil -} - -func (ua unfilteredAggregator) parseAndAddNumberArrayRow(agg *numericalAggregator, - v []byte, propName schema.PropertyName, -) error { - items, ok, err := storobj.ParseAndExtractNumberArrayProp(v, propName.String()) - if err != nil { - return errors.Wrap(err, "parse and extract prop") - } - - if !ok { - return nil - } - - for i := range items { - err := agg.AddNumberRow(items[i], 1) - if err != nil { - return err - } - } - - return nil -} - -func (ua unfilteredAggregator) textProperty(ctx context.Context, - prop aggregation.ParamProperty, -) (*aggregation.Property, error) { - out := aggregation.Property{ - Type: aggregation.PropertyTypeText, - TextAggregation: aggregation.Text{}, - } - - limit := extractLimitFromTopOccs(prop.Aggregators) - - b := ua.store.Bucket(helpers.ObjectsBucketLSM) - if b == nil { - return nil, errors.Errorf("could not find bucket for prop %s", prop.Name) - } - - agg := newTextAggregator(limit) - - // we're looking at the whole object, so this is neither a Set, nor a Map, but - // a Replace strategy - c := b.Cursor() - defer c.Close() - - for k, v := c.First(); k != nil; k, v = c.Next() { - if err := ua.parseAndAddTextRow(agg, v, prop.Name); err != nil { - return nil, err - } - } - - out.TextAggregation = agg.Res() - - return &out, nil -} - -func (ua unfilteredAggregator) numberArrayProperty(ctx context.Context, - prop aggregation.ParamProperty, -) (*aggregation.Property, error) { - out := aggregation.Property{ - Type: aggregation.PropertyTypeNumerical, - NumericalAggregations: map[string]interface{}{}, - } - - b := ua.store.Bucket(helpers.ObjectsBucketLSM) - if b == nil { - return nil, errors.Errorf("could not find bucket for prop %s", prop.Name) - } - - agg := newNumericalAggregator() - - c := b.Cursor() - defer c.Close() - - for k, v := c.First(); k != nil; k, v = c.Next() { - if err := ua.parseAndAddNumberArrayRow(agg, v, prop.Name); err != nil { - return nil, err - } - } - - addNumericalAggregations(&out, prop.Aggregators, agg) - - return &out, nil -} diff --git a/adapters/repos/db/aggregator/vector_search.go b/adapters/repos/db/aggregator/vector_search.go deleted file mode 100644 index bb7178f804bd0da8f9dee63127396c883fa3ff8a..0000000000000000000000000000000000000000 --- a/adapters/repos/db/aggregator/vector_search.go +++ /dev/null @@ -1,104 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package aggregator - -import ( - "context" - "fmt" - - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/adapters/repos/db/inverted" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/storobj" -) - -func (a *Aggregator) vectorSearch(allow helpers.AllowList, vec []float32) ([]uint64, []float32, error) { - if a.params.ObjectLimit != nil { - return a.searchByVector(vec, a.params.ObjectLimit, allow) - } - - return a.searchByVectorDistance(vec, allow) -} - -func (a *Aggregator) searchByVector(searchVector []float32, limit *int, ids helpers.AllowList) ([]uint64, []float32, error) { - idsFound, dists, err := a.vectorIndex.SearchByVector(searchVector, *limit, ids) - if err != nil { - return idsFound, nil, err - } - - if a.params.Certainty > 0 { - targetDist := float32(1-a.params.Certainty) * 2 - - i := 0 - for _, dist := range dists { - if dist > targetDist { - break - } - i++ - } - - return idsFound[:i], dists, nil - - } - return idsFound, dists, nil -} - -func (a *Aggregator) searchByVectorDistance(searchVector []float32, ids helpers.AllowList) ([]uint64, []float32, error) { - if a.params.Certainty <= 0 { - return nil, nil, fmt.Errorf("must provide certainty or objectLimit with vector search") - } - - targetDist := float32(1-a.params.Certainty) * 2 - idsFound, dists, err := a.vectorIndex.SearchByVectorDistance(searchVector, targetDist, -1, ids) - if err != nil { - return nil, nil, fmt.Errorf("aggregate search by vector: %w", err) - } - - return idsFound, dists, nil -} - -func (a *Aggregator) objectVectorSearch(searchVector []float32, - allowList helpers.AllowList, -) ([]*storobj.Object, []float32, error) { - ids, dists, err := a.vectorSearch(allowList, searchVector) - if err != nil { - return nil, nil, err - } - - bucket := a.store.Bucket(helpers.ObjectsBucketLSM) - objs, err := storobj.ObjectsByDocID(bucket, ids, additional.Properties{}) - if err != nil { - return nil, nil, fmt.Errorf("get objects by doc id: %w", err) - } - return objs, dists, nil -} - -func (a *Aggregator) buildAllowList(ctx context.Context) (helpers.AllowList, error) { - var ( - allow helpers.AllowList - err error - ) - - if a.params.Filters != nil { - s := a.getSchema.GetSchemaSkipAuth() - allow, err = inverted.NewSearcher(a.logger, a.store, s, nil, - a.classSearcher, a.stopwords, a.shardVersion, a.isFallbackToSearchable, - a.tenant, a.nestedCrossRefLimit). - DocIDs(ctx, a.params.Filters, additional.Properties{}, - a.params.ClassName) - if err != nil { - return nil, fmt.Errorf("retrieve doc IDs from searcher: %w", err) - } - } - - return allow, nil -} diff --git a/adapters/repos/db/backup.go b/adapters/repos/db/backup.go deleted file mode 100644 index 8526bdac4356bf2426da868aab0f4b50f6ae66f0..0000000000000000000000000000000000000000 --- a/adapters/repos/db/backup.go +++ /dev/null @@ -1,355 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "context" - "fmt" - "sync" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/backup" - "github.com/weaviate/weaviate/entities/schema" -) - -type BackupState struct { - BackupID string - InProgress bool -} - -// Backupable returns whether all given class can be backed up. -func (db *DB) Backupable(ctx context.Context, classes []string) error { - for _, c := range classes { - className := schema.ClassName(c) - idx := db.GetIndex(className) - if idx == nil || idx.Config.ClassName != className { - return fmt.Errorf("class %v doesn't exist", c) - } - } - return nil -} - -// ListBackupable returns a list of all classes which can be backed up. -func (db *DB) ListBackupable() []string { - cs := make([]string, 0, len(db.indices)) - db.indexLock.RLock() - defer db.indexLock.RUnlock() - for _, idx := range db.indices { - cls := string(idx.Config.ClassName) - cs = append(cs, cls) - } - return cs -} - -// BackupDescriptors returns a channel of class descriptors. -// Class descriptor records everything needed to restore a class -// If an error happens a descriptor with an error will be written to the channel just before closing it. -func (db *DB) BackupDescriptors(ctx context.Context, bakid string, classes []string, -) <-chan backup.ClassDescriptor { - ds := make(chan backup.ClassDescriptor, len(classes)) - go func() { - for _, c := range classes { - desc := backup.ClassDescriptor{Name: c} - idx := db.GetIndex(schema.ClassName(c)) - if idx == nil { - desc.Error = fmt.Errorf("class %v doesn't exist any more", c) - } else if err := idx.descriptor(ctx, bakid, &desc); err != nil { - desc.Error = fmt.Errorf("backup class %v descriptor: %w", c, err) - } else { - desc.Error = ctx.Err() - } - - ds <- desc - if desc.Error != nil { - break - } - } - close(ds) - }() - return ds -} - -func (db *DB) ShardsBackup( - ctx context.Context, bakID, class string, shards []string, -) (_ backup.ClassDescriptor, err error) { - cd := backup.ClassDescriptor{Name: class} - idx := db.GetIndex(schema.ClassName(class)) - if idx == nil { - return cd, fmt.Errorf("no index for class %q", class) - } - - if err := idx.initBackup(bakID); err != nil { - return cd, fmt.Errorf("init backup state for class %q: %w", class, err) - } - - defer func() { - if err != nil { - go idx.ReleaseBackup(ctx, bakID) - } - }() - - sm := make(map[string]ShardLike, len(shards)) - for _, shardName := range shards { - shard := idx.shards.Load(shardName) - if shard == nil { - return cd, fmt.Errorf("no shard %q for class %q", shardName, class) - } - sm[shardName] = shard - } - - // prevent writing into the index during collection of metadata - idx.backupMutex.Lock() - defer idx.backupMutex.Unlock() - for shardName, shard := range sm { - if err := shard.BeginBackup(ctx); err != nil { - return cd, fmt.Errorf("class %q: shard %q: begin backup: %w", class, shardName, err) - } - - sd := backup.ShardDescriptor{Name: shardName} - if err := shard.ListBackupFiles(ctx, &sd); err != nil { - return cd, fmt.Errorf("class %q: shard %q: list backup files: %w", class, shardName, err) - } - - cd.Shards = append(cd.Shards, &sd) - } - - return cd, nil -} - -// ReleaseBackup release resources acquired by the index during backup -func (db *DB) ReleaseBackup(ctx context.Context, bakID, class string) (err error) { - fields := logrus.Fields{ - "op": "release_backup", - "class": class, - "id": bakID, - } - db.logger.WithFields(fields).Debug("starting") - begin := time.Now() - defer func() { - l := db.logger.WithFields(fields).WithField("took", time.Since(begin)) - if err != nil { - l.Error(err) - return - } - l.Debug("finish") - }() - - idx := db.GetIndex(schema.ClassName(class)) - if idx != nil { - return idx.ReleaseBackup(ctx, bakID) - } - return nil -} - -func (db *DB) ClassExists(name string) bool { - return db.IndexExists(schema.ClassName(name)) -} - -// Returns the list of nodes where shards of class are contained. -// If there are no shards for the class, returns an empty list -// If there are shards for the class but no nodes are found, return an error -func (db *DB) Shards(ctx context.Context, class string) ([]string, error) { - unique := make(map[string]struct{}) - - ss := db.schemaGetter.CopyShardingState(class) - if len(ss.Physical) == 0 { - return []string{}, nil - } - - for _, shard := range ss.Physical { - for _, node := range shard.BelongsToNodes { - unique[node] = struct{}{} - } - } - - var ( - nodes = make([]string, len(unique)) - counter = 0 - ) - - for node := range unique { - nodes[counter] = node - counter++ - } - if len(nodes) == 0 { - return nil, fmt.Errorf("found %v shards, but has 0 nodes", len(ss.Physical)) - } - - return nodes, nil -} - -func (db *DB) ListClasses(ctx context.Context) []string { - classes := db.schemaGetter.GetSchemaSkipAuth().Objects.Classes - classNames := make([]string, len(classes)) - - for i, class := range classes { - classNames[i] = class.Class - } - - return classNames -} - -// descriptor record everything needed to restore a class -func (i *Index) descriptor(ctx context.Context, backupID string, desc *backup.ClassDescriptor) (err error) { - if err := i.initBackup(backupID); err != nil { - return err - } - - defer func() { - if err != nil { - go i.ReleaseBackup(ctx, backupID) - } - }() - - // prevent writing into the index during collection of metadata - i.backupMutex.Lock() - defer i.backupMutex.Unlock() - - if err = i.ForEachShard(func(name string, s ShardLike) error { - if err = s.BeginBackup(ctx); err != nil { - return fmt.Errorf("pause compaction and flush: %w", err) - } - var sd backup.ShardDescriptor - if err := s.ListBackupFiles(ctx, &sd); err != nil { - return fmt.Errorf("list shard %v files: %w", s.Name(), err) - } - - desc.Shards = append(desc.Shards, &sd) - return nil - }); err != nil { - return err - } - - if desc.ShardingState, err = i.marshalShardingState(); err != nil { - return fmt.Errorf("marshal sharding state %w", err) - } - if desc.Schema, err = i.marshalSchema(); err != nil { - return fmt.Errorf("marshal schema %w", err) - } - return nil -} - -// ReleaseBackup marks the specified backup as inactive and restarts all -// async background and maintenance processes. It errors if the backup does not exist -// or is already inactive. -func (i *Index) ReleaseBackup(ctx context.Context, id string) error { - defer i.resetBackupState() - if err := i.resumeMaintenanceCycles(ctx); err != nil { - return err - } - return nil -} - -func (i *Index) initBackup(id string) error { - new := &BackupState{ - BackupID: id, - InProgress: true, - } - if !i.lastBackup.CompareAndSwap(nil, new) { - bid := "" - if x := i.lastBackup.Load(); x != nil { - bid = x.BackupID - } - return errors.Errorf( - "cannot create new backup, backup ‘%s’ is not yet released, this "+ - "means its contents have not yet been fully copied to its destination, "+ - "try again later", bid) - } - - return nil -} - -func (i *Index) resetBackupState() { - i.lastBackup.Store(nil) -} - -func (i *Index) resumeMaintenanceCycles(ctx context.Context) (lastErr error) { - i.ForEachShard(func(name string, shard ShardLike) error { - if err := shard.resumeMaintenanceCycles(ctx); err != nil { - lastErr = err - i.logger.WithField("shard", name).WithField("op", "resume_maintenance").Error(err) - } - time.Sleep(time.Millisecond * 10) - return nil - }) - return lastErr -} - -func (i *Index) marshalShardingState() ([]byte, error) { - b, err := i.getSchema.CopyShardingState(i.Config.ClassName.String()).JSON() - if err != nil { - return nil, errors.Wrap(err, "marshal sharding state") - } - - return b, nil -} - -func (i *Index) marshalSchema() ([]byte, error) { - schema := i.getSchema.GetSchemaSkipAuth() - - b, err := schema.GetClass(i.Config.ClassName).MarshalBinary() - if err != nil { - return nil, errors.Wrap(err, "marshal schema") - } - - return b, err -} - -const ( - mutexRetryDuration = time.Millisecond * 500 - mutexNotifyDuration = 20 * time.Second -) - -// backupMutex is an adapter built around rwmutex that facilitates cooperative blocking between write and read locks -type backupMutex struct { - sync.RWMutex - log logrus.FieldLogger - retryDuration time.Duration - notifyDuration time.Duration -} - -// LockWithContext attempts to acquire a write lock while respecting the provided context. -// It reports whether the lock acquisition was successful or if the context has been cancelled. -func (m *backupMutex) LockWithContext(ctx context.Context) error { - return m.lock(ctx, m.TryLock) -} - -func (m *backupMutex) lock(ctx context.Context, tryLock func() bool) error { - if tryLock() { - return nil - } - curTime := time.Now() - t := time.NewTicker(m.retryDuration) - defer t.Stop() - for { - select { - case <-ctx.Done(): - return ctx.Err() - case <-t.C: - if tryLock() { - return nil - } - if time.Since(curTime) > m.notifyDuration { - curTime = time.Now() - m.log.Info("backup process waiting for ongoing writes to finish") - } - } - } -} - -func (s *backupMutex) RLockGuard(reader func() error) error { - s.RLock() - defer s.RUnlock() - return reader() -} diff --git a/adapters/repos/db/backup_integration_test.go b/adapters/repos/db/backup_integration_test.go deleted file mode 100644 index 2e948396b82a0baa0ffd492b7aff4872d18321d5..0000000000000000000000000000000000000000 --- a/adapters/repos/db/backup_integration_test.go +++ /dev/null @@ -1,302 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package db - -import ( - "context" - "fmt" - "os" - "path" - "path/filepath" - "regexp" - "testing" - "time" - - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/storobj" - enthnsw "github.com/weaviate/weaviate/entities/vectorindex/hnsw" -) - -func TestBackup_DBLevel(t *testing.T) { - t.Run("successful backup creation", func(t *testing.T) { - ctx := testCtx() - dirName := t.TempDir() - className := "DBLevelBackupClass" - backupID := "backup1" - now := time.Now() - - db := setupTestDB(t, dirName, makeTestClass(className)) - defer func() { - require.Nil(t, db.Shutdown(context.Background())) - }() - - t.Run("insert data", func(t *testing.T) { - require.Nil(t, db.PutObject(ctx, &models.Object{ - Class: className, - CreationTimeUnix: now.UnixNano(), - ID: "ff9fcae5-57b8-431c-b8e2-986fd78f5809", - LastUpdateTimeUnix: now.UnixNano(), - Vector: []float32{1, 2, 3}, - VectorWeights: nil, - }, []float32{1, 2, 3}, nil)) - }) - - expectedNodeName := "node1" - expectedShardName := db.schemaGetter. - CopyShardingState(className). - AllPhysicalShards()[0] - testShd := db.GetIndex(schema.ClassName(className)). - shards.Load(expectedShardName) - expectedCounterPath, _ := filepath.Rel(testShd.Index().Config.RootPath, testShd.Counter().FileName()) - expectedCounter, err := os.ReadFile(testShd.Counter().FileName()) - require.Nil(t, err) - expectedPropLengthPath, _ := filepath.Rel(testShd.Index().Config.RootPath, testShd.GetPropertyLengthTracker().FileName()) - expectedShardVersionPath, _ := filepath.Rel(testShd.Index().Config.RootPath, testShd.Versioner().path) - expectedShardVersion, err := os.ReadFile(testShd.Versioner().path) - require.Nil(t, err) - expectedPropLength, err := os.ReadFile(testShd.GetPropertyLengthTracker().FileName()) - require.Nil(t, err) - expectedShardState, err := testShd.Index().getSchema.CopyShardingState(className).JSON() - require.Nil(t, err) - expectedSchema, err := testShd.Index().getSchema.GetSchemaSkipAuth(). - Objects.Classes[0].MarshalBinary() - require.Nil(t, err) - - classes := db.ListBackupable() - - t.Run("doesn't fail on casing permutation of existing class", func(t *testing.T) { - err := db.Backupable(ctx, []string{"DBLeVELBackupClass"}) - require.NotNil(t, err) - require.Equal(t, "class DBLeVELBackupClass doesn't exist", err.Error()) - }) - - t.Run("create backup", func(t *testing.T) { - err := db.Backupable(ctx, classes) - assert.Nil(t, err) - - ch := db.BackupDescriptors(ctx, backupID, classes) - - for d := range ch { - assert.Equal(t, className, d.Name) - assert.Len(t, d.Shards, len(classes)) - for _, shd := range d.Shards { - assert.Equal(t, expectedShardName, shd.Name) - assert.Equal(t, expectedNodeName, shd.Node) - assert.NotEmpty(t, shd.Files) - for _, f := range shd.Files { - assert.NotEmpty(t, f) - } - assert.Equal(t, expectedCounterPath, shd.DocIDCounterPath) - assert.Equal(t, expectedCounter, shd.DocIDCounter) - assert.Equal(t, expectedPropLengthPath, shd.PropLengthTrackerPath) - assert.Equal(t, expectedPropLength, shd.PropLengthTracker) - assert.Equal(t, expectedShardVersionPath, shd.ShardVersionPath) - assert.Equal(t, expectedShardVersion, shd.Version) - } - assert.Equal(t, expectedShardState, d.ShardingState) - assert.Equal(t, expectedSchema, d.Schema) - } - }) - - t.Run("release backup", func(t *testing.T) { - for _, class := range classes { - err := db.ReleaseBackup(ctx, backupID, class) - assert.Nil(t, err) - } - }) - - t.Run("node names from shards", func(t *testing.T) { - res, err := db.Shards(ctx, className) - assert.NoError(t, err) - assert.Len(t, res, 1) - assert.Equal(t, "node1", res[0]) - }) - - t.Run("get all classes", func(t *testing.T) { - res := db.ListClasses(ctx) - assert.Len(t, res, 1) - assert.Equal(t, className, res[0]) - }) - }) - - t.Run("failed backup creation from expired context", func(t *testing.T) { - ctx := testCtx() - dirName := t.TempDir() - className := "DBLevelBackupClass" - backupID := "backup1" - now := time.Now() - - db := setupTestDB(t, dirName, makeTestClass(className)) - defer func() { - require.Nil(t, db.Shutdown(context.Background())) - }() - - t.Run("insert data", func(t *testing.T) { - require.Nil(t, db.PutObject(ctx, &models.Object{ - Class: className, - CreationTimeUnix: now.UnixNano(), - ID: "ff9fcae5-57b8-431c-b8e2-986fd78f5809", - LastUpdateTimeUnix: now.UnixNano(), - Vector: []float32{1, 2, 3}, - VectorWeights: nil, - }, []float32{1, 2, 3}, nil)) - }) - - t.Run("fail with expired context", func(t *testing.T) { - classes := db.ListBackupable() - - err := db.Backupable(ctx, classes) - assert.Nil(t, err) - - timeoutCtx, cancel := context.WithTimeout(context.Background(), 0) - defer cancel() - - ch := db.BackupDescriptors(timeoutCtx, backupID, classes) - for d := range ch { - require.NotNil(t, d.Error) - assert.Contains(t, d.Error.Error(), "context deadline exceeded") - } - }) - }) -} - -func TestBackup_BucketLevel(t *testing.T) { - ctx := testCtx() - className := "BucketLevelBackup" - shard, _ := testShard(t, ctx, className) - - t.Run("insert data", func(t *testing.T) { - err := shard.PutObject(ctx, &storobj.Object{ - MarshallerVersion: 1, - Object: models.Object{ - ID: "8c29da7a-600a-43dc-85fb-83ab2b08c294", - Class: className, - Properties: map[string]interface{}{ - "stringField": "somevalue", - }, - }, - }, - ) - require.Nil(t, err) - }) - - t.Run("perform backup sequence", func(t *testing.T) { - objBucket := shard.Store().Bucket("objects") - require.NotNil(t, objBucket) - - err := shard.Store().PauseCompaction(ctx) - require.Nil(t, err) - - err = objBucket.FlushMemtable() - require.Nil(t, err) - - files, err := objBucket.ListFiles(ctx, shard.Index().Config.RootPath) - require.Nil(t, err) - - t.Run("check ListFiles, results", func(t *testing.T) { - assert.Len(t, files, 4) - - // build regex to get very close approximation to the expected - // contents of the ListFiles result. the only thing we can't - // know for sure is the actual name of the segment group, hence - // the `.*` - re := path.Clean(fmt.Sprintf("%s\\/.*\\.(wal|db|bloom|cna)", shard.Index().Config.RootPath)) - - // we expect to see only four files inside the bucket at this point: - // 1. a *.db file - the segment itself - // 2. a *.bloom file - the segments' bloom filter (only since v1.17) - // 3. a *.secondary.0.bloom file - the bloom filter for the secondary index at pos 0 (only since v1.17) - // 4. a *.cna file - th segment's count net additions (only since v1.17) - // - // These files are created when the memtable is flushed, and the new - // segment is initialized. Both happens as a result of calling - // FlushMemtable(). - for i := range files { - isMatch, err := regexp.MatchString(re, files[i]) - assert.Nil(t, err) - assert.True(t, isMatch, files[i]) - } - - // check that we have one of each: *.db - exts := make([]string, 4) - for i, file := range files { - exts[i] = filepath.Ext(file) - } - assert.Contains(t, exts, ".db") // the main segment - assert.Contains(t, exts, ".cna") // the segment's count net additions - assert.Contains(t, exts, ".bloom") // matches both bloom filters (primary+secondary) - }) - - err = shard.Store().ResumeCompaction(ctx) - require.Nil(t, err) - }) - - t.Run("cleanup", func(t *testing.T) { - require.Nil(t, shard.Shutdown(ctx)) - require.Nil(t, os.RemoveAll(shard.Index().Config.RootPath)) - }) -} - -func setupTestDB(t *testing.T, rootDir string, classes ...*models.Class) *DB { - logger, _ := test.NewNullLogger() - - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - db, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: rootDir, - QueryMaximumResults: 10, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - db.SetSchemaGetter(schemaGetter) - require.Nil(t, db.WaitForStartup(testCtx())) - migrator := NewMigrator(db, logger) - - for _, class := range classes { - require.Nil(t, - migrator.AddClass(context.Background(), class, schemaGetter.shardState)) - } - - // update schema getter so it's in sync with class - schemaGetter.schema = schema.Schema{ - Objects: &models.Schema{ - Classes: classes, - }, - } - - return db -} - -func makeTestClass(className string) *models.Class { - return &models.Class{ - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Class: className, - Properties: []*models.Property{ - { - Name: "stringProp", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - } -} diff --git a/adapters/repos/db/backup_test.go b/adapters/repos/db/backup_test.go deleted file mode 100644 index a8cd7fc99d9f96e2806e69ea2d6cdb74cd8b76b0..0000000000000000000000000000000000000000 --- a/adapters/repos/db/backup_test.go +++ /dev/null @@ -1,57 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "context" - "errors" - "testing" - "time" - - tlog "github.com/sirupsen/logrus/hooks/test" -) - -func TestBackupMutex(t *testing.T) { - l, _ := tlog.NewNullLogger() - t.Run("success first time", func(t *testing.T) { - m := backupMutex{log: l, retryDuration: time.Millisecond, notifyDuration: 5 * time.Millisecond} - ctx, cancel := context.WithTimeout(context.Background(), 12*time.Millisecond) - defer cancel() - if err := m.LockWithContext(ctx); err != nil { - t.Errorf("error want:nil got:%v ", err) - } - }) - t.Run("success after retry", func(t *testing.T) { - m := backupMutex{log: l, retryDuration: 2 * time.Millisecond, notifyDuration: 5 * time.Millisecond} - m.RLock() - go func() { - defer m.RUnlock() - time.Sleep(time.Millisecond * 15) - }() - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - if err := m.LockWithContext(ctx); err != nil { - t.Errorf("error want:nil got:%v ", err) - } - }) - t.Run("cancelled context", func(t *testing.T) { - m := backupMutex{log: l, retryDuration: time.Millisecond, notifyDuration: 5 * time.Millisecond} - m.RLock() - defer m.RUnlock() - ctx, cancel := context.WithTimeout(context.Background(), 12*time.Millisecond) - defer cancel() - err := m.LockWithContext(ctx) - if !errors.Is(err, context.DeadlineExceeded) { - t.Errorf("error want:%v got:%v", err, context.DeadlineExceeded) - } - }) -} diff --git a/adapters/repos/db/batch.go b/adapters/repos/db/batch.go deleted file mode 100644 index d20a817cfb355ee26c4d84416436ae88b3bfabbe..0000000000000000000000000000000000000000 --- a/adapters/repos/db/batch.go +++ /dev/null @@ -1,222 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "context" - "fmt" - - "github.com/go-openapi/strfmt" - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/storobj" - "github.com/weaviate/weaviate/usecases/objects" -) - -type batchQueue struct { - objects []*storobj.Object - originalIndex []int -} - -func (db *DB) BatchPutObjects(ctx context.Context, objs objects.BatchObjects, - repl *additional.ReplicationProperties, -) (objects.BatchObjects, error) { - objectByClass := make(map[string]batchQueue) - indexByClass := make(map[string]*Index) - - if err := db.memMonitor.CheckAlloc(estimateBatchMemory(objs)); err != nil { - return nil, fmt.Errorf("cannot process batch: %w", err) - } - - for _, item := range objs { - if item.Err != nil { - // item has a validation error or another reason to ignore - continue - } - queue := objectByClass[item.Object.Class] - queue.objects = append(queue.objects, storobj.FromObject(item.Object, item.Vector)) - queue.originalIndex = append(queue.originalIndex, item.OriginalIndex) - objectByClass[item.Object.Class] = queue - } - - // wrapped by func to acquire and safely release indexLock only for duration of loop - func() { - db.indexLock.RLock() - defer db.indexLock.RUnlock() - - for class, queue := range objectByClass { - index, ok := db.indices[indexID(schema.ClassName(class))] - if !ok { - msg := fmt.Sprintf("could not find index for class %v. It might have been deleted in the meantime", class) - db.logger.Warn(msg) - for _, origIdx := range queue.originalIndex { - if origIdx >= len(objs) { - db.logger.Errorf( - "batch add queue index out of bounds. len(objs) == %d, queue.originalIndex == %d", - len(objs), origIdx) - break - } - objs[origIdx].Err = fmt.Errorf(msg) - } - continue - } - index.dropIndex.RLock() - indexByClass[class] = index - } - }() - - // safely release remaining locks (in case of panic) - defer func() { - for _, index := range indexByClass { - if index != nil { - index.dropIndex.RUnlock() - } - } - }() - - for class, index := range indexByClass { - queue := objectByClass[class] - errs := index.putObjectBatch(ctx, queue.objects, repl) - // remove index from map to skip releasing its lock in defer - indexByClass[class] = nil - index.dropIndex.RUnlock() - for i, err := range errs { - if err != nil { - objs[queue.originalIndex[i]].Err = err - } - } - } - - return objs, nil -} - -func (db *DB) AddBatchReferences(ctx context.Context, references objects.BatchReferences, - repl *additional.ReplicationProperties, -) (objects.BatchReferences, error) { - refByClass := make(map[schema.ClassName]objects.BatchReferences) - indexByClass := make(map[schema.ClassName]*Index) - - for _, item := range references { - if item.Err != nil { - // item has a validation error or another reason to ignore - continue - } - refByClass[item.From.Class] = append(refByClass[item.From.Class], item) - } - - // wrapped by func to acquire and safely release indexLock only for duration of loop - func() { - db.indexLock.RLock() - defer db.indexLock.RUnlock() - - for class, queue := range refByClass { - index, ok := db.indices[indexID(class)] - if !ok { - for _, item := range queue { - references[item.OriginalIndex].Err = fmt.Errorf("could not find index for class %v. It might have been deleted in the meantime", class) - } - continue - } - index.dropIndex.RLock() - indexByClass[class] = index - } - }() - - // safely release remaining locks (in case of panic) - defer func() { - for _, index := range indexByClass { - if index != nil { - index.dropIndex.RUnlock() - } - } - }() - - for class, index := range indexByClass { - queue := refByClass[class] - errs := index.AddReferencesBatch(ctx, queue, repl) - // remove index from map to skip releasing its lock in defer - indexByClass[class] = nil - index.dropIndex.RUnlock() - for i, err := range errs { - if err != nil { - references[queue[i].OriginalIndex].Err = err - } - } - } - - return references, nil -} - -func (db *DB) BatchDeleteObjects(ctx context.Context, params objects.BatchDeleteParams, - repl *additional.ReplicationProperties, tenant string, -) (objects.BatchDeleteResult, error) { - // get index for a given class - className := params.ClassName - idx := db.GetIndex(className) - if idx == nil { - return objects.BatchDeleteResult{}, errors.Errorf("cannot find index for class %v", className) - } - - // find all DocIDs in all shards that match the filter - shardDocIDs, err := idx.findUUIDs(ctx, params.Filters, tenant) - if err != nil { - return objects.BatchDeleteResult{}, errors.Wrapf(err, "cannot find objects") - } - // prepare to be deleted list of DocIDs from all shards - toDelete := map[string][]strfmt.UUID{} - limit := db.config.QueryMaximumResults - - matches := int64(0) - for shardName, docIDs := range shardDocIDs { - docIDsLength := int64(len(docIDs)) - if matches <= limit { - if matches+docIDsLength <= limit { - toDelete[shardName] = docIDs - } else { - toDelete[shardName] = docIDs[:limit-matches] - } - } - matches += docIDsLength - } - // delete the DocIDs in given shards - deletedObjects, err := idx.batchDeleteObjects(ctx, toDelete, params.DryRun, repl) - if err != nil { - return objects.BatchDeleteResult{}, errors.Wrapf(err, "cannot delete objects") - } - - result := objects.BatchDeleteResult{ - Matches: matches, - Limit: db.config.QueryMaximumResults, - DryRun: params.DryRun, - Objects: deletedObjects, - } - return result, nil -} - -func estimateBatchMemory(objs objects.BatchObjects) int64 { - var sum int64 - for _, item := range objs { - // Note: This is very much oversimplified. It assumes that we always need - // the footprint of the full vector and it assumes a fixed overhead of 30B - // per vector. In reality this depends on the HNSW settings - and possibly - // in the future we might have completely different index types. - // - // However, in the meantime this should be a fairly reasonable estimate, as - // it's not meant to fail exactly on the last available byte, but rather - // prevent OOM crashes. Given the fuzziness and async style of the - // memtrackinga somewhat decent estimate should be go good enough. - sum += int64(len(item.Vector)*4 + 30) - } - - return sum -} diff --git a/adapters/repos/db/batch_integration_test.go b/adapters/repos/db/batch_integration_test.go deleted file mode 100644 index 1d95ed1e1c6c7bfaf07a4851bb54376ccbacaff0..0000000000000000000000000000000000000000 --- a/adapters/repos/db/batch_integration_test.go +++ /dev/null @@ -1,1273 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package db - -import ( - "context" - "fmt" - "math/rand" - "sort" - "testing" - "time" - - "github.com/go-openapi/strfmt" - "github.com/google/uuid" - "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" - "github.com/weaviate/weaviate/entities/dto" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/search" - enthnsw "github.com/weaviate/weaviate/entities/vectorindex/hnsw" - "github.com/weaviate/weaviate/usecases/objects" -) - -func TestBatchPutObjectsWithDimensions(t *testing.T) { - dirName := t.TempDir() - - logger := logrus.New() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - TrackVectorDimensions: true, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - - defer func() { - require.Nil(t, repo.Shutdown(context.Background())) - }() - migrator := NewMigrator(repo, logger) - - t.Run("creating the thing class", testAddBatchObjectClass(repo, migrator, schemaGetter)) - - dimBefore := GetDimensionsFromRepo(repo, "ThingForBatching") - require.Equal(t, 0, dimBefore, "Dimensions are empty before import") - - simpleInsertObjects(t, repo, "ThingForBatching", 123) - - dimAfter := GetDimensionsFromRepo(repo, "ThingForBatching") - require.Equal(t, 369, dimAfter, "Dimensions are present after import") -} - -func TestBatchPutObjects(t *testing.T) { - dirName := t.TempDir() - - logger := logrus.New() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - TrackVectorDimensions: true, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - - defer func() { - require.Nil(t, repo.Shutdown(context.Background())) - }() - migrator := NewMigrator(repo, logger) - - t.Run("creating the thing class", testAddBatchObjectClass(repo, migrator, schemaGetter)) - - t.Run("batch import things", testBatchImportObjects(repo)) - t.Run("batch import things with geo props", testBatchImportGeoObjects(repo)) -} - -func TestBatchPutObjectsNoVectorsWithDimensions(t *testing.T) { - dirName := t.TempDir() - - logger := logrus.New() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - TrackVectorDimensions: true, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - - defer func() { - require.Nil(t, repo.Shutdown(context.Background())) - }() - migrator := NewMigrator(repo, logger) - - t.Run("creating the thing class", testAddBatchObjectClass(repo, migrator, - schemaGetter)) - - dimensions := GetDimensionsFromRepo(repo, "ThingForBatching") - require.Equal(t, 0, dimensions, "Dimensions are empty before import") - - t.Run("batch import things", testBatchImportObjectsNoVector(repo)) - - dimAfter := GetDimensionsFromRepo(repo, "ThingForBatching") - require.Equal(t, 0, dimAfter, "Dimensions are empty after import (no vectors in import)") -} - -func TestBatchPutObjectsNoVectors(t *testing.T) { - dirName := t.TempDir() - - logger := logrus.New() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - - defer func() { - require.Nil(t, repo.Shutdown(context.Background())) - }() - migrator := NewMigrator(repo, logger) - - t.Run("creating the thing class", testAddBatchObjectClass(repo, migrator, schemaGetter)) - - t.Run("batch import things", testBatchImportObjectsNoVector(repo)) -} - -func TestBatchDeleteObjectsWithDimensions(t *testing.T) { - className := "ThingForBatching" - dirName := t.TempDir() - - logger := logrus.New() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 1, - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - TrackVectorDimensions: true, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - defer func() { - require.Nil(t, repo.Shutdown(context.Background())) - }() - - migrator := NewMigrator(repo, logger) - - t.Run("creating the test class", testAddBatchObjectClass(repo, migrator, schemaGetter)) - - dimBefore := GetDimensionsFromRepo(repo, className) - require.Equal(t, 0, dimBefore, "Dimensions are empty before import") - - simpleInsertObjects(t, repo, className, 103) - - dimAfter := GetDimensionsFromRepo(repo, className) - require.Equal(t, 309, dimAfter, "Dimensions are present before delete") - - delete2Objects(t, repo, className) - - dimFinal := GetDimensionsFromRepo(repo, className) - require.Equal(t, 303, dimFinal, "2 objects have been deleted") -} - -func delete2Objects(t *testing.T, repo *DB, className string) { - batchDeleteRes, err := repo.BatchDeleteObjects(context.Background(), objects.BatchDeleteParams{ - ClassName: "ThingForBatching", - Filters: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorOr, - Operands: []filters.Clause{ - { - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: "ThingForBatching", - Property: schema.PropertyName("id"), - }, - Value: &filters.Value{ - Value: "8d5a3aa2-3c8d-4589-9ae1-3f638f506003", - Type: schema.DataTypeText, - }, - }, - { - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: "ThingForBatching", - Property: schema.PropertyName("id"), - }, - Value: &filters.Value{ - Value: "8d5a3aa2-3c8d-4589-9ae1-3f638f506004", - Type: schema.DataTypeText, - }, - }, - }, - }, - }, - DryRun: false, - Output: "verbose", - }, nil, "") - require.Nil(t, err) - require.Equal(t, 2, len(batchDeleteRes.Objects), "Objects deleted") -} - -func TestBatchDeleteObjects(t *testing.T) { - dirName := t.TempDir() - - logger := logrus.New() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - TrackVectorDimensions: true, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - defer func() { - require.Nil(t, repo.Shutdown(context.Background())) - }() - migrator := NewMigrator(repo, logger) - - t.Run("creating the thing class", testAddBatchObjectClass(repo, migrator, schemaGetter)) - - t.Run("batch import things", testBatchImportObjects(repo)) - - t.Run("batch delete things", testBatchDeleteObjects(repo)) -} - -func TestBatchDeleteObjects_JourneyWithDimensions(t *testing.T) { - dirName := t.TempDir() - - queryMaximumResults := int64(200) - logger := logrus.New() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: queryMaximumResults, - MaxImportGoroutinesFactor: 1, - TrackVectorDimensions: true, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - defer func() { - require.Nil(t, repo.Shutdown(context.Background())) - }() - migrator := NewMigrator(repo, logger) - - t.Run("creating the thing class", testAddBatchObjectClass(repo, migrator, schemaGetter)) - - dimBefore := GetDimensionsFromRepo(repo, "ThingForBatching") - require.Equal(t, 0, dimBefore, "Dimensions are empty before import") - - simpleInsertObjects(t, repo, "ThingForBatching", 103) - - dimAfter := GetDimensionsFromRepo(repo, "ThingForBatching") - require.Equal(t, 309, dimAfter, "Dimensions are present before delete") - - delete2Objects(t, repo, "ThingForBatching") - - dimFinal := GetDimensionsFromRepo(repo, "ThingForBatching") - require.Equal(t, 303, dimFinal, "Dimensions have been deleted") -} - -func TestBatchDeleteObjects_Journey(t *testing.T) { - dirName := t.TempDir() - - queryMaximumResults := int64(20) - logger := logrus.New() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: queryMaximumResults, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - defer func() { - require.Nil(t, repo.Shutdown(context.Background())) - }() - migrator := NewMigrator(repo, logger) - - t.Run("creating the thing class", testAddBatchObjectClass(repo, migrator, - schemaGetter)) - t.Run("batch import things", testBatchImportObjects(repo)) - t.Run("batch delete journey things", testBatchDeleteObjectsJourney(repo, queryMaximumResults)) -} - -func testAddBatchObjectClass(repo *DB, migrator *Migrator, - schemaGetter *fakeSchemaGetter, -) func(t *testing.T) { - return func(t *testing.T) { - class := &models.Class{ - Class: "ThingForBatching", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{ - { - Name: "stringProp", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "location", - DataType: []string{string(schema.DataTypeGeoCoordinates)}, - }, - }, - } - - require.Nil(t, migrator.AddClass(context.Background(), class, schemaGetter.shardState)) - - schemaGetter.schema.Objects = &models.Schema{ - Classes: []*models.Class{class}, - } - } -} - -func testBatchImportObjectsNoVector(repo *DB) func(t *testing.T) { - return func(t *testing.T) { - t.Run("with a prior validation error, but nothing to cause errors in the db", func(t *testing.T) { - batch := objects.BatchObjects{ - objects.BatchObject{ - OriginalIndex: 0, - Err: nil, - Object: &models.Object{ - Class: "ThingForBatching", - Properties: map[string]interface{}{ - "stringProp": "first element", - }, - ID: "8d5a3aa2-3c8d-4589-9ae1-3f638f506970", - }, - UUID: "8d5a3aa2-3c8d-4589-9ae1-3f638f506970", - }, - objects.BatchObject{ - OriginalIndex: 1, - Err: fmt.Errorf("already has a validation error"), - Object: &models.Object{ - Class: "ThingForBatching", - Properties: map[string]interface{}{ - "stringProp": "second element", - }, - ID: "86a380e9-cb60-4b2a-bc48-51f52acd72d6", - }, - UUID: "86a380e9-cb60-4b2a-bc48-51f52acd72d6", - }, - objects.BatchObject{ - OriginalIndex: 2, - Err: nil, - Object: &models.Object{ - Class: "ThingForBatching", - Properties: map[string]interface{}{ - "stringProp": "third element", - }, - ID: "90ade18e-2b99-4903-aa34-1d5d648c932d", - }, - UUID: "90ade18e-2b99-4903-aa34-1d5d648c932d", - }, - } - - t.Run("can import", func(t *testing.T) { - batchRes, err := repo.BatchPutObjects(context.Background(), batch, nil) - require.Nil(t, err) - - assert.Nil(t, batchRes[0].Err) - assert.Nil(t, batchRes[2].Err) - }) - - params := dto.GetParams{ - ClassName: "ThingForBatching", - Pagination: &filters.Pagination{Limit: 10}, - Filters: nil, - } - _, err := repo.Search(context.Background(), params) - require.Nil(t, err) - }) - } -} - -func simpleInsertObjects(t *testing.T, repo *DB, class string, count int) { - batch := make(objects.BatchObjects, count) - for i := 0; i < count; i++ { - batch[i] = objects.BatchObject{ - OriginalIndex: i, - Err: nil, - Object: &models.Object{ - Class: class, - Properties: map[string]interface{}{ - "stringProp": fmt.Sprintf("element %d", i), - }, - ID: strfmt.UUID(fmt.Sprintf("8d5a3aa2-3c8d-4589-9ae1-3f638f506%03d", i)), - }, - UUID: strfmt.UUID(fmt.Sprintf("8d5a3aa2-3c8d-4589-9ae1-3f638f506%03d", i)), - Vector: []float32{1, 2, 3}, - } - } - - repo.BatchPutObjects(context.Background(), batch, nil) -} - -func testBatchImportObjects(repo *DB) func(t *testing.T) { - return func(t *testing.T) { - t.Run("with a prior validation error, but nothing to cause errors in the db", func(t *testing.T) { - batch := objects.BatchObjects{ - objects.BatchObject{ - OriginalIndex: 0, - Err: nil, - Object: &models.Object{ - Class: "ThingForBatching", - Properties: map[string]interface{}{ - "stringProp": "first element", - }, - ID: "8d5a3aa2-3c8d-4589-9ae1-3f638f506970", - }, - UUID: "8d5a3aa2-3c8d-4589-9ae1-3f638f506970", - Vector: []float32{1, 2, 3}, - }, - objects.BatchObject{ - OriginalIndex: 1, - Err: fmt.Errorf("already has a validation error"), - Object: &models.Object{ - Class: "ThingForBatching", - Properties: map[string]interface{}{ - "stringProp": "second element", - }, - ID: "86a380e9-cb60-4b2a-bc48-51f52acd72d6", - }, - UUID: "86a380e9-cb60-4b2a-bc48-51f52acd72d6", - Vector: []float32{1, 2, 3}, - }, - objects.BatchObject{ - OriginalIndex: 2, - Err: nil, - Object: &models.Object{ - Class: "ThingForBatching", - Properties: map[string]interface{}{ - "stringProp": "third element", - }, - ID: "90ade18e-2b99-4903-aa34-1d5d648c932d", - }, - UUID: "90ade18e-2b99-4903-aa34-1d5d648c932d", - Vector: []float32{1, 2, 3}, - }, - } - - t.Run("can import", func(t *testing.T) { - batchRes, err := repo.BatchPutObjects(context.Background(), batch, nil) - require.Nil(t, err) - - assert.Nil(t, batchRes[0].Err) - assert.Nil(t, batchRes[2].Err) - }) - - params := dto.GetParams{ - ClassName: "ThingForBatching", - Pagination: &filters.Pagination{Limit: 10}, - Filters: nil, - } - res, err := repo.Search(context.Background(), params) - require.Nil(t, err) - - t.Run("contains first element", func(t *testing.T) { - item, ok := findID(res, batch[0].Object.ID) - require.Equal(t, true, ok, "results should contain our desired id") - assert.Equal(t, "first element", item.Schema.(map[string]interface{})["stringProp"]) - }) - - t.Run("contains third element", func(t *testing.T) { - item, ok := findID(res, batch[2].Object.ID) - require.Equal(t, true, ok, "results should contain our desired id") - assert.Equal(t, "third element", item.Schema.(map[string]interface{})["stringProp"]) - }) - - t.Run("can be queried through the inverted index", func(t *testing.T) { - filter := buildFilter("stringProp", "third", eq, schema.DataTypeText) - params := dto.GetParams{ - ClassName: "ThingForBatching", - Pagination: &filters.Pagination{Limit: 10}, - Filters: filter, - } - res, err := repo.Search(context.Background(), params) - require.Nil(t, err) - - require.Len(t, res, 1) - assert.Equal(t, strfmt.UUID("90ade18e-2b99-4903-aa34-1d5d648c932d"), - res[0].ID) - }) - }) - - t.Run("with an import which will fail", func(t *testing.T) { - batch := objects.BatchObjects{ - objects.BatchObject{ - OriginalIndex: 0, - Err: nil, - Object: &models.Object{ - Class: "ThingForBatching", - Properties: map[string]interface{}{ - "stringProp": "first element", - }, - ID: "79aebd44-7486-4fed-9334-3a74cc09a1c3", - }, - UUID: "79aebd44-7486-4fed-9334-3a74cc09a1c3", - }, - objects.BatchObject{ - OriginalIndex: 1, - Err: fmt.Errorf("already had a prior error"), - Object: &models.Object{ - Class: "ThingForBatching", - Properties: map[string]interface{}{ - "stringProp": "second element", - }, - ID: "1c2d8ce6-32da-4081-9794-a81e23e673e4", - }, - UUID: "1c2d8ce6-32da-4081-9794-a81e23e673e4", - }, - objects.BatchObject{ - OriginalIndex: 2, - Err: nil, - Object: &models.Object{ - Class: "ThingForBatching", - Properties: map[string]interface{}{ - "stringProp": "third element", - }, - ID: "", // ID can't be empty in es, this should produce an error - }, - UUID: "", - }, - } - - t.Run("can import", func(t *testing.T) { - batchRes, err := repo.BatchPutObjects(context.Background(), batch, nil) - require.Nil(t, err, "there shouldn't be an overall error, only individual ones") - - t.Run("element errors are marked correctly", func(t *testing.T) { - require.Len(t, batchRes, 3) - assert.NotNil(t, batchRes[1].Err) // from validation - assert.NotNil(t, batchRes[2].Err) // from db - }) - }) - - params := dto.GetParams{ - ClassName: "ThingForBatching", - Pagination: &filters.Pagination{Limit: 10}, - Filters: nil, - } - res, err := repo.Search(context.Background(), params) - require.Nil(t, err) - - t.Run("does not contain second element (validation error)", func(t *testing.T) { - _, ok := findID(res, batch[1].Object.ID) - require.Equal(t, false, ok, "results should not contain our desired id") - }) - - t.Run("does not contain third element (es error)", func(t *testing.T) { - _, ok := findID(res, batch[2].Object.ID) - require.Equal(t, false, ok, "results should not contain our desired id") - }) - }) - - t.Run("upserting the same objects over and over again", func(t *testing.T) { - for i := 0; i < 20; i++ { - batch := objects.BatchObjects{ - objects.BatchObject{ - OriginalIndex: 0, - Err: nil, - Object: &models.Object{ - Class: "ThingForBatching", - Properties: map[string]interface{}{ - "stringProp": "first element", - }, - ID: "8d5a3aa2-3c8d-4589-9ae1-3f638f506970", - }, - UUID: "8d5a3aa2-3c8d-4589-9ae1-3f638f506970", - Vector: []float32{1, 2, 3}, - }, - objects.BatchObject{ - OriginalIndex: 1, - Err: nil, - Object: &models.Object{ - Class: "ThingForBatching", - Properties: map[string]interface{}{ - "stringProp": "third element", - }, - ID: "90ade18e-2b99-4903-aa34-1d5d648c932d", - }, - UUID: "90ade18e-2b99-4903-aa34-1d5d648c932d", - Vector: []float32{1, 1, -3}, - }, - } - - t.Run("can import", func(t *testing.T) { - batchRes, err := repo.BatchPutObjects(context.Background(), batch, nil) - require.Nil(t, err) - - assert.Nil(t, batchRes[0].Err) - assert.Nil(t, batchRes[1].Err) - }) - - t.Run("a vector search returns the correct number of elements", func(t *testing.T) { - res, err := repo.VectorSearch(context.Background(), dto.GetParams{ - ClassName: "ThingForBatching", - Pagination: &filters.Pagination{ - Offset: 0, - Limit: 10, - }, - SearchVector: []float32{1, 2, 3}, - }) - require.Nil(t, err) - assert.Len(t, res, 2) - }) - - } - }) - - t.Run("with a duplicate UUID", func(t *testing.T) { - // it should ignore the first one as the second one would overwrite the - // first one anyway - batch := make(objects.BatchObjects, 53) - - batch[0] = objects.BatchObject{ - OriginalIndex: 0, - Err: nil, - Vector: []float32{7, 8, 9}, - Object: &models.Object{ - Class: "ThingForBatching", - Properties: map[string]interface{}{ - "stringProp": "first element", - }, - ID: "79aebd44-7486-4fed-9334-3a74cc09a1c3", - }, - UUID: "79aebd44-7486-4fed-9334-3a74cc09a1c3", - } - - // add 50 more nonsensical items, so we cross the transaction threshold - - for i := 1; i < 51; i++ { - uid, err := uuid.NewRandom() - require.Nil(t, err) - id := strfmt.UUID(uid.String()) - batch[i] = objects.BatchObject{ - OriginalIndex: i, - Err: nil, - Vector: []float32{0.05, 0.1, 0.2}, - Object: &models.Object{ - Class: "ThingForBatching", - Properties: map[string]interface{}{ - "stringProp": "ignore me", - }, - ID: id, - }, - UUID: id, - } - } - - batch[51] = objects.BatchObject{ - OriginalIndex: 51, - Err: fmt.Errorf("already had a prior error"), - Vector: []float32{3, 2, 1}, - Object: &models.Object{ - Class: "ThingForBatching", - Properties: map[string]interface{}{ - "stringProp": "first element", - }, - ID: "1c2d8ce6-32da-4081-9794-a81e23e673e4", - }, - UUID: "1c2d8ce6-32da-4081-9794-a81e23e673e4", - } - batch[52] = objects.BatchObject{ - OriginalIndex: 52, - Err: nil, - Vector: []float32{1, 2, 3}, - Object: &models.Object{ - Class: "ThingForBatching", - Properties: map[string]interface{}{ - "stringProp": "first element, imported a second time", - }, - ID: "79aebd44-7486-4fed-9334-3a74cc09a1c3", // note the duplicate id with item 1 - }, - UUID: "79aebd44-7486-4fed-9334-3a74cc09a1c3", // note the duplicate id with item 1 - } - - t.Run("can import", func(t *testing.T) { - batchRes, err := repo.BatchPutObjects(context.Background(), batch, nil) - require.Nil(t, err, "there shouldn't be an overall error, only individual ones") - - t.Run("element errors are marked correctly", func(t *testing.T) { - require.Len(t, batchRes, 53) - assert.NotNil(t, batchRes[51].Err) // from validation - }) - }) - - params := dto.GetParams{ - ClassName: "ThingForBatching", - Pagination: &filters.Pagination{Limit: 10}, - Filters: nil, - } - res, err := repo.Search(context.Background(), params) - require.Nil(t, err) - - t.Run("does not contain second element (validation error)", func(t *testing.T) { - _, ok := findID(res, batch[51].Object.ID) - require.Equal(t, false, ok, "results should not contain our desired id") - }) - - t.Run("does not contain third element (es error)", func(t *testing.T) { - _, ok := findID(res, batch[52].Object.ID) - require.Equal(t, false, ok, "results should not contain our desired id") - }) - }) - - t.Run("when a context expires", func(t *testing.T) { - // it should ignore the first one as the second one would overwrite the - // first one anyway - size := 50 - batch := make(objects.BatchObjects, size) - // add 50 more nonsensical items, so we cross the transaction threshold - - for i := 0; i < size; i++ { - uid, err := uuid.NewRandom() - require.Nil(t, err) - id := strfmt.UUID(uid.String()) - batch[i] = objects.BatchObject{ - Err: nil, - Vector: []float32{0.05, 0.1, 0.2}, - Object: &models.Object{ - Class: "ThingForBatching", - Properties: map[string]interface{}{ - "stringProp": "ignore me", - }, - ID: id, - }, - UUID: id, - } - } - - t.Run("can import", func(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Millisecond) - defer cancel() - - batchRes, err := repo.BatchPutObjects(ctx, batch, nil) - require.Nil(t, err, "there shouldn't be an overall error, only individual ones") - - t.Run("some elements have error'd due to context", func(t *testing.T) { - require.Len(t, batchRes, 50) - - errCount := 0 - for _, elem := range batchRes { - if elem.Err != nil { - errCount++ - assert.Contains(t, elem.Err.Error(), "context deadline exceeded") - } - } - - assert.True(t, errCount > 0) - }) - }) - }) - } -} - -// geo props are the first props with property specific indices, so making sure -// that they work with batches at scale adds value beyond the regular batch -// import tests -func testBatchImportGeoObjects(repo *DB) func(t *testing.T) { - r := getRandomSeed() - return func(t *testing.T) { - size := 500 - batchSize := 50 - - objs := make([]*models.Object, size) - - t.Run("generate random vectors", func(t *testing.T) { - for i := 0; i < size; i++ { - id, _ := uuid.NewRandom() - objs[i] = &models.Object{ - Class: "ThingForBatching", - ID: strfmt.UUID(id.String()), - Properties: map[string]interface{}{ - "location": randGeoCoordinates(r), - }, - Vector: []float32{0.123, 0.234, rand.Float32()}, // does not matter for this test - } - } - }) - - t.Run("import vectors in batches", func(t *testing.T) { - for i := 0; i < size; i += batchSize { - batch := make(objects.BatchObjects, batchSize) - for j := 0; j < batchSize; j++ { - batch[j] = objects.BatchObject{ - OriginalIndex: j, - Object: objs[i+j], - Vector: objs[i+j].Vector, - } - } - - res, err := repo.BatchPutObjects(context.Background(), batch, nil) - require.Nil(t, err) - assertAllItemsErrorFree(t, res) - } - }) - - const km = 1000 - distances := []float32{ - 0.1, - 1, - 10, - 100, - 1000, - 2000, - 5000, - 7500, - 10000, - 12500, - 15000, - 20000, - 35000, - 100000, // larger than the circumference of the earth, should contain all - } - - t.Run("query for expected results", func(t *testing.T) { - queryGeo := randGeoCoordinates(r) - - for _, maxDist := range distances { - t.Run(fmt.Sprintf("with maxDist=%f", maxDist), func(t *testing.T) { - var relevant int - var retrieved int - - controlList := bruteForceMaxDist(objs, []float32{ - *queryGeo.Latitude, - *queryGeo.Longitude, - }, maxDist*km) - - res, err := repo.Search(context.Background(), dto.GetParams{ - ClassName: "ThingForBatching", - Pagination: &filters.Pagination{Limit: 500}, - Filters: buildFilter("location", filters.GeoRange{ - GeoCoordinates: queryGeo, - Distance: maxDist * km, - }, filters.OperatorWithinGeoRange, schema.DataTypeGeoCoordinates), - }) - require.Nil(t, err) - - retrieved += len(res) - relevant += matchesInUUIDLists(controlList, resToUUIDs(res)) - - if relevant == 0 { - // skip, as we risk dividing by zero, if both relevant and retrieved - // are zero, however, we want to fail with a divide-by-zero if only - // retrieved is 0 and relevant was more than 0 - return - } - recall := float32(relevant) / float32(retrieved) - assert.True(t, recall >= 0.99) - }) - } - }) - - t.Run("renew vector positions to test batch geo updates", func(t *testing.T) { - for i, obj := range objs { - obj.Properties = map[string]interface{}{ - "location": randGeoCoordinates(r), - } - objs[i] = obj - } - }) - - t.Run("import in batches again (as update - same IDs!)", func(t *testing.T) { - for i := 0; i < size; i += batchSize { - batch := make(objects.BatchObjects, batchSize) - for j := 0; j < batchSize; j++ { - batch[j] = objects.BatchObject{ - OriginalIndex: j, - Object: objs[i+j], - Vector: objs[i+j].Vector, - } - } - - res, err := repo.BatchPutObjects(context.Background(), batch, nil) - require.Nil(t, err) - assertAllItemsErrorFree(t, res) - } - }) - - t.Run("query again to verify updates worked", func(t *testing.T) { - queryGeo := randGeoCoordinates(r) - - for _, maxDist := range distances { - t.Run(fmt.Sprintf("with maxDist=%f", maxDist), func(t *testing.T) { - var relevant int - var retrieved int - - controlList := bruteForceMaxDist(objs, []float32{ - *queryGeo.Latitude, - *queryGeo.Longitude, - }, maxDist*km) - - res, err := repo.Search(context.Background(), dto.GetParams{ - ClassName: "ThingForBatching", - Pagination: &filters.Pagination{Limit: 500}, - Filters: buildFilter("location", filters.GeoRange{ - GeoCoordinates: queryGeo, - Distance: maxDist * km, - }, filters.OperatorWithinGeoRange, schema.DataTypeGeoCoordinates), - }) - require.Nil(t, err) - - retrieved += len(res) - relevant += matchesInUUIDLists(controlList, resToUUIDs(res)) - - if relevant == 0 { - // skip, as we risk dividing by zero, if both relevant and retrieved - // are zero, however, we want to fail with a divide-by-zero if only - // retrieved is 0 and relevant was more than 0 - return - } - recall := float32(relevant) / float32(retrieved) - fmt.Printf("recall is %f\n", recall) - assert.True(t, recall >= 0.99) - }) - } - }) - } -} - -func testBatchDeleteObjects(repo *DB) func(t *testing.T) { - return func(t *testing.T) { - getParams := func(dryRun bool, output string) objects.BatchDeleteParams { - return objects.BatchDeleteParams{ - ClassName: "ThingForBatching", - Filters: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorLike, - Value: &filters.Value{ - Value: "*", - Type: schema.DataTypeText, - }, - On: &filters.Path{ - Property: schema.PropertyName("id"), - }, - }, - }, - DryRun: dryRun, - Output: output, - } - } - performClassSearch := func() ([]search.Result, error) { - return repo.Search(context.Background(), dto.GetParams{ - ClassName: "ThingForBatching", - Pagination: &filters.Pagination{Limit: 10000}, - }) - } - t.Run("batch delete with dryRun set to true", func(t *testing.T) { - // get the initial count of the objects - res, err := performClassSearch() - require.Nil(t, err) - beforeDelete := len(res) - require.True(t, beforeDelete > 0) - // dryRun == true, only test how many objects can be deleted - batchDeleteRes, err := repo.BatchDeleteObjects(context.Background(), getParams(true, "verbose"), nil, "") - require.Nil(t, err) - require.Equal(t, int64(beforeDelete), batchDeleteRes.Matches) - require.Equal(t, beforeDelete, len(batchDeleteRes.Objects)) - for _, batchRes := range batchDeleteRes.Objects { - require.Nil(t, batchRes.Err) - } - res, err = performClassSearch() - require.Nil(t, err) - assert.Equal(t, beforeDelete, len(res)) - }) - - t.Run("batch delete with dryRun set to true and output to minimal", func(t *testing.T) { - // get the initial count of the objects - res, err := performClassSearch() - require.Nil(t, err) - beforeDelete := len(res) - require.True(t, beforeDelete > 0) - // dryRun == true, only test how many objects can be deleted - batchDeleteRes, err := repo.BatchDeleteObjects(context.Background(), getParams(true, "minimal"), nil, "") - require.Nil(t, err) - require.Equal(t, int64(beforeDelete), batchDeleteRes.Matches) - require.Equal(t, beforeDelete, len(batchDeleteRes.Objects)) - for _, batchRes := range batchDeleteRes.Objects { - require.Nil(t, batchRes.Err) - } - res, err = performClassSearch() - require.Nil(t, err) - assert.Equal(t, beforeDelete, len(res)) - }) - - t.Run("batch delete only 2 given objects", func(t *testing.T) { - // get the initial count of the objects - res, err := performClassSearch() - require.Nil(t, err) - beforeDelete := len(res) - require.True(t, beforeDelete > 0) - // dryRun == true, only test how many objects can be deleted - batchDeleteRes, err := repo.BatchDeleteObjects(context.Background(), objects.BatchDeleteParams{ - ClassName: "ThingForBatching", - Filters: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorOr, - Operands: []filters.Clause{ - { - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: "ThingForBatching", - Property: schema.PropertyName("id"), - }, - Value: &filters.Value{ - Value: "8d5a3aa2-3c8d-4589-9ae1-3f638f506970", - Type: schema.DataTypeText, - }, - }, - { - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: "ThingForBatching", - Property: schema.PropertyName("id"), - }, - Value: &filters.Value{ - Value: "90ade18e-2b99-4903-aa34-1d5d648c932d", - Type: schema.DataTypeText, - }, - }, - }, - }, - }, - DryRun: false, - Output: "verbose", - }, nil, "") - require.Nil(t, err) - require.Equal(t, int64(2), batchDeleteRes.Matches) - require.Equal(t, 2, len(batchDeleteRes.Objects)) - for _, batchRes := range batchDeleteRes.Objects { - require.Nil(t, batchRes.Err) - } - res, err = performClassSearch() - require.Nil(t, err) - assert.Equal(t, beforeDelete-2, len(res)) - }) - - t.Run("batch delete with dryRun set to false", func(t *testing.T) { - // get the initial count of the objects - res, err := performClassSearch() - require.Nil(t, err) - beforeDelete := len(res) - require.True(t, beforeDelete > 0) - // dryRun == true, only test how many objects can be deleted - batchDeleteRes, err := repo.BatchDeleteObjects(context.Background(), getParams(false, "verbose"), nil, "") - require.Nil(t, err) - require.Equal(t, int64(beforeDelete), batchDeleteRes.Matches) - require.Equal(t, beforeDelete, len(batchDeleteRes.Objects)) - for _, batchRes := range batchDeleteRes.Objects { - require.Nil(t, batchRes.Err) - } - res, err = performClassSearch() - require.Nil(t, err) - assert.Equal(t, 0, len(res)) - }) - } -} - -func testBatchDeleteObjectsJourney(repo *DB, queryMaximumResults int64) func(t *testing.T) { - return func(t *testing.T) { - getParams := func(dryRun bool, output string) objects.BatchDeleteParams { - return objects.BatchDeleteParams{ - ClassName: "ThingForBatching", - Filters: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorLike, - Value: &filters.Value{ - Value: "*", - Type: schema.DataTypeText, - }, - On: &filters.Path{ - Property: schema.PropertyName("id"), - }, - }, - }, - DryRun: dryRun, - Output: output, - } - } - performClassSearch := func() ([]search.Result, error) { - return repo.Search(context.Background(), dto.GetParams{ - ClassName: "ThingForBatching", - Pagination: &filters.Pagination{Limit: 20}, - }) - } - t.Run("batch delete journey", func(t *testing.T) { - // delete objects to limit - batchDeleteRes, err := repo.BatchDeleteObjects(context.Background(), getParams(true, "verbose"), nil, "") - require.Nil(t, err) - objectsMatches := batchDeleteRes.Matches - - leftToDelete := objectsMatches - deleteIterationCount := 0 - deletedObjectsCount := 0 - for { - // delete objects to limit - batchDeleteRes, err := repo.BatchDeleteObjects(context.Background(), getParams(false, "verbose"), nil, "") - require.Nil(t, err) - matches, deleted := batchDeleteRes.Matches, len(batchDeleteRes.Objects) - require.Equal(t, leftToDelete, matches) - require.True(t, deleted > 0) - deletedObjectsCount += deleted - - batchDeleteRes, err = repo.BatchDeleteObjects(context.Background(), getParams(true, "verbose"), nil, "") - require.Nil(t, err) - leftToDelete = batchDeleteRes.Matches - - res, err := performClassSearch() - require.Nil(t, err) - afterDelete := len(res) - require.True(t, afterDelete >= 0) - if afterDelete == 0 { - // where have deleted all objects - break - } - deleteIterationCount += 1 - if deleteIterationCount > 100 { - // something went wrong - break - } - } - require.False(t, deleteIterationCount > 100, "Batch delete journey tests didn't stop properly") - require.True(t, objectsMatches/int64(queryMaximumResults) <= int64(deleteIterationCount)) - require.Equal(t, objectsMatches, int64(deletedObjectsCount)) - }) - } -} - -func assertAllItemsErrorFree(t *testing.T, res objects.BatchObjects) { - for _, elem := range res { - assert.Nil(t, elem.Err) - } -} - -func bruteForceMaxDist(inputs []*models.Object, query []float32, maxDist float32) []strfmt.UUID { - type distanceAndIndex struct { - distance float32 - index int - } - - distances := make([]distanceAndIndex, len(inputs)) - - distancer := distancer.NewGeoProvider().New(query) - for i, elem := range inputs { - coord := elem.Properties.(map[string]interface{})["location"].(*models.GeoCoordinates) - vec := []float32{*coord.Latitude, *coord.Longitude} - - dist, _, _ := distancer.Distance(vec) - distances[i] = distanceAndIndex{ - index: i, - distance: dist, - } - } - - sort.Slice(distances, func(a, b int) bool { - return distances[a].distance < distances[b].distance - }) - - out := make([]strfmt.UUID, len(distances)) - i := 0 - for _, elem := range distances { - if elem.distance > maxDist { - break - } - out[i] = inputs[distances[i].index].ID - i++ - } - - return out[:i] -} - -func randGeoCoordinates(r *rand.Rand) *models.GeoCoordinates { - maxLat := float32(90.0) - minLat := float32(-90.0) - maxLon := float32(180) - minLon := float32(-180) - - lat := minLat + (maxLat-minLat)*r.Float32() - lon := minLon + (maxLon-minLon)*r.Float32() - return &models.GeoCoordinates{ - Latitude: &lat, - Longitude: &lon, - } -} - -func resToUUIDs(in []search.Result) []strfmt.UUID { - out := make([]strfmt.UUID, len(in)) - for i, obj := range in { - out[i] = obj.ID - } - - return out -} - -func matchesInUUIDLists(control []strfmt.UUID, results []strfmt.UUID) int { - desired := map[strfmt.UUID]struct{}{} - for _, relevant := range control { - desired[relevant] = struct{}{} - } - - var matches int - for _, candidate := range results { - _, ok := desired[candidate] - if ok { - matches++ - } - } - - return matches -} diff --git a/adapters/repos/db/batch_reference_integration_test.go b/adapters/repos/db/batch_reference_integration_test.go deleted file mode 100644 index e8d866ab10ebf9d468f39be9ba561a3f1f0adfb2..0000000000000000000000000000000000000000 --- a/adapters/repos/db/batch_reference_integration_test.go +++ /dev/null @@ -1,402 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest - -package db - -import ( - "context" - "fmt" - "testing" - - "github.com/go-openapi/strfmt" - "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/dto" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/schema/crossref" - enthnsw "github.com/weaviate/weaviate/entities/vectorindex/hnsw" - "github.com/weaviate/weaviate/usecases/objects" -) - -func Test_AddingReferencesInBatches(t *testing.T) { - dirName := t.TempDir() - - logger := logrus.New() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - - defer repo.Shutdown(context.Background()) - - migrator := NewMigrator(repo, logger) - - s := schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{ - { - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Class: "AddingBatchReferencesTestTarget", - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - }, - { - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Class: "AddingBatchReferencesTestSource", - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "toTarget", - DataType: []string{"AddingBatchReferencesTestTarget"}, - }, - }, - }, - }, - }, - } - - t.Run("add required classes", func(t *testing.T) { - for _, class := range s.Objects.Classes { - t.Run(fmt.Sprintf("add %s", class.Class), func(t *testing.T) { - err := migrator.AddClass(context.Background(), class, schemaGetter.shardState) - require.Nil(t, err) - }) - } - }) - schemaGetter.schema = s - - target1 := strfmt.UUID("7b395e5c-cf4d-4297-b8cc-1d849a057de3") - target2 := strfmt.UUID("8f9f54f3-a7db-415e-881a-0e6fb79a7ec7") - target3 := strfmt.UUID("046251cf-cb02-4102-b854-c7c4691cf16f") - target4 := strfmt.UUID("bc7d8875-3a24-4137-8203-e152096dea4f") - sourceID := strfmt.UUID("a3c98a66-be4a-4eaf-8cf3-04648a11d0f7") - - t.Run("add objects", func(t *testing.T) { - err := repo.PutObject(context.Background(), &models.Object{ - ID: sourceID, - Class: "AddingBatchReferencesTestSource", - Properties: map[string]interface{}{ - "name": "source item", - }, - }, []float32{0.5}, nil) - require.Nil(t, err) - - targets := []strfmt.UUID{target1, target2, target3, target4} - - for i, target := range targets { - err = repo.PutObject(context.Background(), &models.Object{ - ID: target, - Class: "AddingBatchReferencesTestTarget", - Properties: map[string]interface{}{ - "name": fmt.Sprintf("target item %d", i), - }, - }, []float32{0.7}, nil) - require.Nil(t, err) - } - }) - - t.Run("verify ref count through filters", func(t *testing.T) { - t.Run("count==0 should return the source", func(t *testing.T) { - filter := buildFilter("toTarget", 0, eq, schema.DataTypeInt) - res, err := repo.Search(context.Background(), dto.GetParams{ - Filters: filter, - ClassName: "AddingBatchReferencesTestSource", - Pagination: &filters.Pagination{ - Limit: 10, - }, - }) - - require.Nil(t, err) - require.Len(t, res, 1) - assert.Equal(t, res[0].ID, sourceID) - }) - - t.Run("count>0 should not return anything", func(t *testing.T) { - filter := buildFilter("toTarget", 0, gt, schema.DataTypeInt) - res, err := repo.Search(context.Background(), dto.GetParams{ - Filters: filter, - ClassName: "AddingBatchReferencesTestSource", - Pagination: &filters.Pagination{ - Limit: 10, - }, - }) - - require.Nil(t, err) - require.Len(t, res, 0) - }) - }) - - t.Run("add reference between them - first batch", func(t *testing.T) { - source, err := crossref.ParseSource(fmt.Sprintf( - "weaviate://localhost/AddingBatchReferencesTestSource/%s/toTarget", - sourceID)) - require.Nil(t, err) - targets := []strfmt.UUID{target1, target2} - refs := make(objects.BatchReferences, len(targets)) - for i, target := range targets { - to, err := crossref.Parse(fmt.Sprintf("weaviate://localhost/%s", - target)) - require.Nil(t, err) - refs[i] = objects.BatchReference{ - Err: nil, - From: source, - To: to, - OriginalIndex: i, - } - } - _, err = repo.AddBatchReferences(context.Background(), refs, nil) - assert.Nil(t, err) - }) - - t.Run("verify ref count through filters", func(t *testing.T) { - // so far we have imported two refs (!) - t.Run("count==2 should return the source", func(t *testing.T) { - filter := buildFilter("toTarget", 2, eq, schema.DataTypeInt) - res, err := repo.Search(context.Background(), dto.GetParams{ - Filters: filter, - ClassName: "AddingBatchReferencesTestSource", - Pagination: &filters.Pagination{ - Limit: 10, - }, - }) - - require.Nil(t, err) - require.Len(t, res, 1) - assert.Equal(t, res[0].ID, sourceID) - }) - - t.Run("count==0 should not return anything", func(t *testing.T) { - filter := buildFilter("toTarget", 0, eq, schema.DataTypeInt) - res, err := repo.Search(context.Background(), dto.GetParams{ - Filters: filter, - ClassName: "AddingBatchReferencesTestSource", - Pagination: &filters.Pagination{ - Limit: 10, - }, - }) - - require.Nil(t, err) - require.Len(t, res, 0) - }) - }) - - t.Run("add reference between them - second batch including errors", func(t *testing.T) { - source, err := crossref.ParseSource(fmt.Sprintf( - "weaviate://localhost/AddingBatchReferencesTestSource/%s/toTarget", - sourceID)) - require.Nil(t, err) - sourceNonExistingClass, err := crossref.ParseSource(fmt.Sprintf( - "weaviate://localhost/NonExistingClass/%s/toTarget", - sourceID)) - require.Nil(t, err) - sourceNonExistingProp, err := crossref.ParseSource(fmt.Sprintf( - "weaviate://localhost/AddingBatchReferencesTestSource/%s/nonExistingProp", - sourceID)) - require.Nil(t, err) - - targets := []strfmt.UUID{target3, target4} - refs := make(objects.BatchReferences, 3*len(targets)) - for i, target := range targets { - to, err := crossref.Parse(fmt.Sprintf("weaviate://localhost/%s", target)) - require.Nil(t, err) - - refs[3*i] = objects.BatchReference{ - Err: nil, - From: source, - To: to, - OriginalIndex: 3 * i, - } - refs[3*i+1] = objects.BatchReference{ - Err: nil, - From: sourceNonExistingClass, - To: to, - OriginalIndex: 3*i + 1, - } - refs[3*i+2] = objects.BatchReference{ - Err: nil, - From: sourceNonExistingProp, - To: to, - OriginalIndex: 3*i + 2, - } - } - batchRefs, err := repo.AddBatchReferences(context.Background(), refs, nil) - assert.Nil(t, err) - require.Len(t, batchRefs, 6) - assert.Nil(t, batchRefs[0].Err) - assert.Nil(t, batchRefs[3].Err) - assert.Contains(t, batchRefs[1].Err.Error(), "NonExistingClass") - assert.Contains(t, batchRefs[4].Err.Error(), "NonExistingClass") - assert.Contains(t, batchRefs[2].Err.Error(), "nonExistingProp") - assert.Contains(t, batchRefs[5].Err.Error(), "nonExistingProp") - }) - - t.Run("check all references are now present", func(t *testing.T) { - source, err := repo.ObjectByID(context.Background(), sourceID, nil, additional.Properties{}, "") - require.Nil(t, err) - - refs := source.Object().Properties.(map[string]interface{})["toTarget"] - refsSlice, ok := refs.(models.MultipleRef) - require.True(t, ok, fmt.Sprintf("toTarget must be models.MultipleRef, but got %#v", refs)) - - foundBeacons := []string{} - for _, ref := range refsSlice { - foundBeacons = append(foundBeacons, ref.Beacon.String()) - } - expectedBeacons := []string{ - fmt.Sprintf("weaviate://localhost/%s", target1), - fmt.Sprintf("weaviate://localhost/%s", target2), - fmt.Sprintf("weaviate://localhost/%s", target3), - fmt.Sprintf("weaviate://localhost/%s", target4), - } - - assert.ElementsMatch(t, foundBeacons, expectedBeacons) - }) - - t.Run("verify ref count through filters", func(t *testing.T) { - // so far we have imported two refs (!) - t.Run("count==4 should return the source", func(t *testing.T) { - filter := buildFilter("toTarget", 4, eq, schema.DataTypeInt) - res, err := repo.Search(context.Background(), dto.GetParams{ - Filters: filter, - ClassName: "AddingBatchReferencesTestSource", - Pagination: &filters.Pagination{ - Limit: 10, - }, - }) - - require.Nil(t, err) - require.Len(t, res, 1) - assert.Equal(t, res[0].ID, sourceID) - }) - - t.Run("count==0 should not return anything", func(t *testing.T) { - filter := buildFilter("toTarget", 0, eq, schema.DataTypeInt) - res, err := repo.Search(context.Background(), dto.GetParams{ - Filters: filter, - ClassName: "AddingBatchReferencesTestSource", - Pagination: &filters.Pagination{ - Limit: 10, - }, - }) - - require.Nil(t, err) - require.Len(t, res, 0) - }) - - t.Run("count==2 should not return anything", func(t *testing.T) { - filter := buildFilter("toTarget", 2, eq, schema.DataTypeInt) - res, err := repo.Search(context.Background(), dto.GetParams{ - Filters: filter, - ClassName: "AddingBatchReferencesTestSource", - Pagination: &filters.Pagination{ - Limit: 10, - }, - }) - - require.Nil(t, err) - require.Len(t, res, 0) - }) - }) - - t.Run("verify search by cross-ref", func(t *testing.T) { - filter := &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: eq, - On: &filters.Path{ - Class: schema.ClassName("AddingBatchReferencesTestSource"), - Property: schema.PropertyName("toTarget"), - Child: &filters.Path{ - Class: schema.ClassName("AddingBatchReferencesTestTarget"), - Property: schema.PropertyName("name"), - }, - }, - Value: &filters.Value{ - Value: "item", - Type: schema.DataTypeText, - }, - }, - } - res, err := repo.Search(context.Background(), dto.GetParams{ - Filters: filter, - ClassName: "AddingBatchReferencesTestSource", - Pagination: &filters.Pagination{ - Limit: 10, - }, - }) - - require.Nil(t, err) - require.Len(t, res, 1) - assert.Equal(t, res[0].ID, sourceID) - }) - - t.Run("verify objects are still searchable through the vector index", - func(t *testing.T) { - // prior to making the inverted index and its docIDs immutable, a ref - // update would not change the doc ID, therefore the batch reference - // never had to interact with the vector index. Now that they're - // immutable, the updated doc ID needs to be "re-inserted" even if the - // vector is still the same - // UPDATE gh-1334: Since batch refs are now a special case where we - // tolerate a re-use of the doc id, the above assumption is no longer - // correct. However, this test still adds value, since we were now able - // to remove the additional storage updates. By still including this - // test we verify that such an update is indeed no longer necessary - res, err := repo.VectorSearch(context.Background(), dto.GetParams{ - ClassName: "AddingBatchReferencesTestSource", - SearchVector: []float32{0.49}, - Pagination: &filters.Pagination{ - Limit: 1, - }, - }) - - require.Nil(t, err) - require.Len(t, res, 1) - assert.Equal(t, sourceID, res[0].ID) - }) - - t.Run("remove source and target classes", func(t *testing.T) { - err := repo.DeleteIndex("AddingBatchReferencesTestSource") - assert.Nil(t, err) - err = repo.DeleteIndex("AddingBatchReferencesTestTarget") - assert.Nil(t, err) - - t.Run("verify classes do not exist", func(t *testing.T) { - assert.False(t, repo.IndexExists("AddingBatchReferencesTestSource")) - assert.False(t, repo.IndexExists("AddingBatchReferencesTestTarget")) - }) - }) -} diff --git a/adapters/repos/db/bm25f_test.go b/adapters/repos/db/bm25f_test.go deleted file mode 100644 index 75bc685127ab140f77b5bf324f1a28f775c58458..0000000000000000000000000000000000000000 --- a/adapters/repos/db/bm25f_test.go +++ /dev/null @@ -1,1045 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest - -package db - -import ( - "context" - "fmt" - "testing" - - "github.com/go-openapi/strfmt" - "github.com/google/uuid" - "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/adapters/repos/db/inverted" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/searchparams" - enthnsw "github.com/weaviate/weaviate/entities/vectorindex/hnsw" -) - -func BM25FinvertedConfig(k1, b float32, stopWordPreset string) *models.InvertedIndexConfig { - return &models.InvertedIndexConfig{ - Bm25: &models.BM25Config{ - K1: k1, - B: b, - }, - CleanupIntervalSeconds: 60, - Stopwords: &models.StopwordConfig{ - Preset: stopWordPreset, - }, - IndexNullState: true, - IndexPropertyLength: true, - } -} - -func SetupClass(t require.TestingT, repo *DB, schemaGetter *fakeSchemaGetter, logger logrus.FieldLogger, k1, b float32, -) { - vFalse := false - vTrue := true - - class := &models.Class{ - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: BM25FinvertedConfig(k1, b, "none"), - Class: "MyClass", - - Properties: []*models.Property{ - { - Name: "title", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWord, - IndexFilterable: &vFalse, - IndexSearchable: &vTrue, - }, - { - Name: "description", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWord, - IndexFilterable: &vFalse, - IndexSearchable: &vTrue, - }, - { - Name: "review", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWord, - IndexFilterable: &vFalse, - IndexSearchable: &vTrue, - }, - { - Name: "textField", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationField, - IndexFilterable: &vFalse, - IndexSearchable: &vTrue, - }, - { - Name: "textWhitespace", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - IndexFilterable: &vFalse, - IndexSearchable: &vTrue, - }, - { - Name: "relatedToGolf", - DataType: schema.DataTypeBoolean.PropString(), - IndexFilterable: &vFalse, - IndexSearchable: &vTrue, - }, - { - Name: "multiTitles", - DataType: schema.DataTypeTextArray.PropString(), - Tokenization: models.PropertyTokenizationWord, - IndexFilterable: &vFalse, - IndexSearchable: &vTrue, - }, - { - Name: "multiTextWhitespace", - DataType: schema.DataTypeTextArray.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - IndexFilterable: &vFalse, - IndexSearchable: &vTrue, - }, - }, - } - - schema := schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{class}, - }, - } - - schemaGetter.schema = schema - - migrator := NewMigrator(repo, logger) - migrator.AddClass(context.Background(), class, schemaGetter.shardState) - - testData := []map[string]interface{}{} - testData = append(testData, map[string]interface{}{"title": "Our journey to BM25F", "description": "This is how we get to BM25F", "review": "none none none", "multiTitles": []string{"breakfast", "dinner"}}) - testData = append(testData, map[string]interface{}{"title": "Why I dont like journey", "description": "This is about how we get somewhere", "multiTitles": []string{"going to a restaurant for dinner", "sandwiches and desert are a great lunch"}}) - testData = append(testData, map[string]interface{}{"title": "My journeys in Journey", "description": "A journey story about journeying"}) - testData = append(testData, map[string]interface{}{"title": "An unrelated title", "description": "Actually all about journey"}) - testData = append(testData, map[string]interface{}{"title": "journey journey", "description": "journey journey journey"}) - testData = append(testData, map[string]interface{}{"title": "journey", "description": "journey journey", "multiTextWhitespace": []string{"totally irrelevant:)", "we all MuuultiYell! together"}}) - testData = append(testData, map[string]interface{}{"title": "JOURNEY", "description": "A LOUD JOURNEY", "multiTextWhitespace": []string{"MuuultiYell!", "is fun"}}) - testData = append(testData, map[string]interface{}{"title": "An unrelated title", "description": "Absolutely nothing to do with the topic", "textField": "*&^$@#$%^&*()(Offtopic!!!!"}) - testData = append(testData, map[string]interface{}{"title": "none", "description": "other", "textField": "YELLING IS FUN"}) - testData = append(testData, map[string]interface{}{"title": "something", "description": "none none", "review": "none none none none none none"}) - - for i, data := range testData { - id := strfmt.UUID(uuid.MustParse(fmt.Sprintf("%032d", i)).String()) - - obj := &models.Object{Class: "MyClass", ID: id, Properties: data, CreationTimeUnix: 1565612833955, LastUpdateTimeUnix: 10000020} - vector := []float32{1, 3, 5, 0.4} - //{title: "Our journey to BM25F", description: " This is how we get to BM25F"}} - err := repo.PutObject(context.Background(), obj, vector, nil) - require.Nil(t, err) - } -} - -// DuplicatedFrom SetupClass to make sure this new test does not alter the results of the existing one -func SetupClassForFilterScoringTest(t require.TestingT, repo *DB, schemaGetter *fakeSchemaGetter, logger logrus.FieldLogger, k1, b float32, -) { - vFalse := false - vTrue := true - - class := &models.Class{ - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: BM25FinvertedConfig(k1, b, "none"), - Class: "FilterClass", - - Properties: []*models.Property{ - { - Name: "description", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWord, - IndexFilterable: &vFalse, - IndexSearchable: &vTrue, - }, - { - Name: "relatedToGolf", - DataType: schema.DataTypeBoolean.PropString(), - IndexFilterable: &vTrue, - }, - }, - } - - schema := schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{class}, - }, - } - - schemaGetter.schema = schema - - migrator := NewMigrator(repo, logger) - migrator.AddClass(context.Background(), class, schemaGetter.shardState) - - testData := []map[string]interface{}{} - testData = append(testData, map[string]interface{}{"description": "Brooks Koepka appeared a lot in the ms marco dataset. I was surprised to see golf content in there. I assume if the dataset was newer, we'd see a lot more Rory though.", "relatedToGolf": true}) - testData = append(testData, map[string]interface{}{"description": "While one would expect Koepka to be a somewhat rare name, it did appear in msmarco also outside the context of Brooks.", "relatedToGolf": false}) - - for i, data := range testData { - id := strfmt.UUID(uuid.MustParse(fmt.Sprintf("%032d", i)).String()) - - obj := &models.Object{Class: "FilterClass", ID: id, Properties: data, CreationTimeUnix: 1565612833955, LastUpdateTimeUnix: 10000020} - vector := []float32{1, 3, 5, 0.4} - err := repo.PutObject(context.Background(), obj, vector, nil) - require.Nil(t, err) - } -} - -func TestBM25FJourney(t *testing.T) { - dirName := t.TempDir() - - logger := logrus.New() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, nil, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(context.TODO())) - defer repo.Shutdown(context.Background()) - - SetupClass(t, repo, schemaGetter, logger, 1.2, 0.75) - - idx := repo.GetIndex("MyClass") - require.NotNil(t, idx) - - // Check basic search - addit := additional.Properties{} - - t.Run("bm25f journey", func(t *testing.T) { - kwr := &searchparams.KeywordRanking{Type: "bm25", Properties: []string{"title", "description", "textField"}, Query: "journey"} - res, _, err := idx.objectSearch(context.TODO(), 1000, nil, kwr, nil, nil, addit, nil, "", 0) - require.Nil(t, err) - - // Print results - t.Log("--- Start results for basic search ---") - for _, r := range res { - t.Logf("Result id: %v, score: %v, title: %v, description: %v, additional %+v\n", r.DocID(), r.Score(), r.Object.Properties.(map[string]interface{})["title"], r.Object.Properties.(map[string]interface{})["description"], r.Object.Additional) - } - - // Check results in correct order - require.Equal(t, uint64(4), res[0].DocID()) - require.Equal(t, uint64(5), res[1].DocID()) - require.Equal(t, uint64(6), res[2].DocID()) - require.Equal(t, uint64(3), res[3].DocID()) - require.Equal(t, uint64(0), res[4].DocID()) - require.Equal(t, uint64(2), res[5].DocID()) - - // Without additionalExplanations no explainScore entry should be present - require.Contains(t, res[0].Object.Additional, "score") - require.NotContains(t, res[0].Object.Additional, "explainScore") - }) - - // Check non-alpha search on string field - - // text/field are tokenized entirely, so we can search for non-alpha characters - t.Run("bm25f textField non-alpha", func(t *testing.T) { - kwrTextField := &searchparams.KeywordRanking{Type: "bm25", Properties: []string{"title", "description", "textField"}, Query: "*&^$@#$%^&*()(Offtopic!!!!"} - addit = additional.Properties{} - resTextField, _, err := idx.objectSearch(context.TODO(), 1000, nil, kwrTextField, nil, nil, addit, nil, "", 0) - require.Nil(t, err) - - // Print results - t.Log("--- Start results for textField search ---") - for _, r := range resTextField { - t.Logf("Result id: %v, score: %v, title: %v, description: %v, additional %+v\n", r.DocID(), r.Score(), r.Object.Properties.(map[string]interface{})["title"], r.Object.Properties.(map[string]interface{})["description"], r.Object.Additional) - } - - // Check results in correct order - require.Equal(t, uint64(7), resTextField[0].DocID()) - }) - - // text/field are not lower-cased before indexing, so upper case searches must be passed through unchanged. - t.Run("bm25f textField caps", func(t *testing.T) { - kwrTextField := &searchparams.KeywordRanking{Type: "bm25", Properties: []string{"textField"}, Query: "YELLING IS FUN"} - addit := additional.Properties{} - resTextField, _, err := idx.objectSearch(context.TODO(), 1000, nil, kwrTextField, nil, nil, addit, nil, "", 0) - require.Nil(t, err) - - // Print results - t.Log("--- Start results for textField caps search ---") - for _, r := range resTextField { - t.Logf("Result id: %v, score: %v, title: %v, description: %v, additional %+v\n", r.DocID(), r.Score(), r.Object.Properties.(map[string]interface{})["title"], r.Object.Properties.(map[string]interface{})["description"], r.Object.Additional) - } - - // Check results in correct order - require.Equal(t, uint64(8), resTextField[0].DocID()) - }) - - // Check basic text search WITH CAPS - t.Run("bm25f text with caps", func(t *testing.T) { - kwr := &searchparams.KeywordRanking{Type: "bm25", Properties: []string{"title", "description"}, Query: "JOURNEY"} - res, _, err := idx.objectSearch(context.TODO(), 1000, nil, kwr, nil, nil, addit, nil, "", 0) - // Print results - t.Log("--- Start results for search with caps ---") - for _, r := range res { - t.Logf("Result id: %v, score: %v, title: %v, description: %v, additional %+v\n", r.DocID(), r.Score(), r.Object.Properties.(map[string]interface{})["title"], r.Object.Properties.(map[string]interface{})["description"], r.Object.Additional) - } - require.Nil(t, err) - - // Check results in correct order - require.Equal(t, uint64(4), res[0].DocID()) - require.Equal(t, uint64(5), res[1].DocID()) - require.Equal(t, uint64(6), res[2].DocID()) - require.Equal(t, uint64(2), res[3].DocID()) - require.Equal(t, uint64(3), res[4].DocID()) - require.Equal(t, uint64(0), res[5].DocID()) - require.Equal(t, uint64(1), res[6].DocID()) - }) - - t.Run("bm25f journey boosted", func(t *testing.T) { - kwr := &searchparams.KeywordRanking{Type: "bm25", Properties: []string{"title^3", "description"}, Query: "journey"} - res, _, err := idx.objectSearch(context.TODO(), 1000, nil, kwr, nil, nil, addit, nil, "", 0) - - require.Nil(t, err) - // Print results - t.Log("--- Start results for boosted search ---") - for _, r := range res { - t.Logf("Result id: %v, score: %v, title: %v, description: %v, additional %+v\n", r.DocID(), r.Score(), r.Object.Properties.(map[string]interface{})["title"], r.Object.Properties.(map[string]interface{})["description"], r.Object.Additional) - } - - // Check results in correct order - require.Equal(t, uint64(4), res[0].DocID()) - require.Equal(t, uint64(5), res[1].DocID()) - require.Equal(t, uint64(6), res[2].DocID()) - require.Equal(t, uint64(0), res[3].DocID()) - require.Equal(t, uint64(1), res[4].DocID()) - require.Equal(t, uint64(2), res[5].DocID()) - require.Equal(t, uint64(3), res[6].DocID()) - }) - - t.Run("Check search with two terms", func(t *testing.T) { - kwr := &searchparams.KeywordRanking{Type: "bm25", Properties: []string{"title", "description"}, Query: "journey somewhere"} - res, _, err := idx.objectSearch(context.TODO(), 1000, nil, kwr, nil, nil, addit, nil, "", 0) - require.Nil(t, err) - // Check results in correct order - require.Equal(t, uint64(1), res[0].DocID()) - require.Equal(t, uint64(4), res[1].DocID()) - require.Equal(t, uint64(5), res[2].DocID()) - require.Equal(t, uint64(6), res[3].DocID()) - require.Equal(t, uint64(2), res[4].DocID()) - }) - - t.Run("bm25f journey somewhere no properties", func(t *testing.T) { - // Check search with no properties (should include all properties) - kwr := &searchparams.KeywordRanking{Type: "bm25", Properties: []string{}, Query: "journey somewhere"} - res, _, err := idx.objectSearch(context.TODO(), 1000, nil, kwr, nil, nil, addit, nil, "", 0) - require.Nil(t, err) - - // Check results in correct order - require.Equal(t, uint64(1), res[0].DocID()) - require.Equal(t, uint64(4), res[1].DocID()) - require.Equal(t, uint64(5), res[2].DocID()) - require.Equal(t, uint64(6), res[3].DocID()) - }) - - t.Run("bm25f non alphanums", func(t *testing.T) { - // Check search with no properties (should include all properties) - kwr := &searchparams.KeywordRanking{Type: "bm25", Properties: []string{}, Query: "*&^$@#$%^&*()(Offtopic!!!!"} - res, _, err := idx.objectSearch(context.TODO(), 1000, nil, kwr, nil, nil, addit, nil, "", 0) - require.Nil(t, err) - require.Equal(t, uint64(7), res[0].DocID()) - }) - - t.Run("First result has high score", func(t *testing.T) { - kwr := &searchparams.KeywordRanking{Type: "bm25", Properties: []string{"description"}, Query: "about BM25F"} - res, _, err := idx.objectSearch(context.TODO(), 5, nil, kwr, nil, nil, addit, nil, "", 0) - require.Nil(t, err) - - require.Equal(t, uint64(0), res[0].DocID()) - require.Len(t, res, 4) // four results have one of the terms - }) - - t.Run("More results than limit", func(t *testing.T) { - kwr := &searchparams.KeywordRanking{Type: "bm25", Properties: []string{"description"}, Query: "journey"} - res, _, err := idx.objectSearch(context.TODO(), 5, nil, kwr, nil, nil, addit, nil, "", 0) - require.Nil(t, err) - - require.Equal(t, uint64(4), res[0].DocID()) - require.Equal(t, uint64(5), res[1].DocID()) - require.Equal(t, uint64(6), res[2].DocID()) - require.Equal(t, uint64(3), res[3].DocID()) - require.Equal(t, uint64(2), res[4].DocID()) - require.Len(t, res, 5) // four results have one of the terms - }) - - t.Run("Results from three properties", func(t *testing.T) { - kwr := &searchparams.KeywordRanking{Type: "bm25", Query: "none"} - res, _, err := idx.objectSearch(context.TODO(), 5, nil, kwr, nil, nil, addit, nil, "", 0) - require.Nil(t, err) - - require.Equal(t, uint64(9), res[0].DocID()) - require.Equal(t, uint64(0), res[1].DocID()) - require.Equal(t, uint64(8), res[2].DocID()) - require.Len(t, res, 3) - }) - - t.Run("Include additional explanations", func(t *testing.T) { - kwr := &searchparams.KeywordRanking{Type: "bm25", Properties: []string{"description"}, Query: "journey", AdditionalExplanations: true} - res, _, err := idx.objectSearch(context.TODO(), 5, nil, kwr, nil, nil, addit, nil, "", 0) - require.Nil(t, err) - - // With additionalExplanations explainScore entry should be present - require.Contains(t, res[0].Object.Additional, "score") - require.Contains(t, res[0].Object.Additional, "explainScore") - require.Contains(t, res[0].Object.Additional["explainScore"], "BM25") - }) - - t.Run("Array fields text", func(t *testing.T) { - kwr := &searchparams.KeywordRanking{Type: "bm25", Properties: []string{"multiTitles"}, Query: "dinner"} - res, _, err := idx.objectSearch(context.TODO(), 5, nil, kwr, nil, nil, addit, nil, "", 0) - require.Nil(t, err) - - require.Len(t, res, 2) - require.Equal(t, uint64(0), res[0].DocID()) - require.Equal(t, uint64(1), res[1].DocID()) - }) - - t.Run("Array fields string", func(t *testing.T) { - kwr := &searchparams.KeywordRanking{Type: "bm25", Properties: []string{"multiTextWhitespace"}, Query: "MuuultiYell!"} - res, _, err := idx.objectSearch(context.TODO(), 5, nil, kwr, nil, nil, addit, nil, "", 0) - require.Nil(t, err) - - require.Len(t, res, 2) - require.Equal(t, uint64(6), res[0].DocID()) - require.Equal(t, uint64(5), res[1].DocID()) - }) - - t.Run("With autocut", func(t *testing.T) { - kwr := &searchparams.KeywordRanking{Type: "bm25", Query: "journey", Properties: []string{"description"}} - resNoAutoCut, _, err := idx.objectSearch(context.TODO(), 10, nil, kwr, nil, nil, addit, nil, "", 0) - require.Nil(t, err) - - resAutoCut, _, err := idx.objectSearch(context.TODO(), 10, nil, kwr, nil, nil, addit, nil, "", 1) - require.Nil(t, err) - - require.Less(t, len(resAutoCut), len(resNoAutoCut)) - - require.EqualValues(t, 0.5868752, resNoAutoCut[0].Score()) - require.EqualValues(t, 0.5450892, resNoAutoCut[1].Score()) // <= autocut last element - require.EqualValues(t, 0.34149727, resNoAutoCut[2].Score()) - require.EqualValues(t, 0.3049518, resNoAutoCut[3].Score()) - require.EqualValues(t, 0.27547202, resNoAutoCut[4].Score()) - - require.Len(t, resAutoCut, 2) - require.EqualValues(t, 0.5868752, resAutoCut[0].Score()) - require.EqualValues(t, 0.5450892, resAutoCut[1].Score()) - }) -} - -func TestBM25FSingleProp(t *testing.T) { - dirName := t.TempDir() - - logger := logrus.New() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, nil, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(context.TODO())) - defer repo.Shutdown(context.Background()) - - SetupClass(t, repo, schemaGetter, logger, 0.5, 100) - - idx := repo.GetIndex("MyClass") - require.NotNil(t, idx) - - // Check boosted - kwr := &searchparams.KeywordRanking{Type: "bm25", Properties: []string{"description"}, Query: "journey"} - addit := additional.Properties{} - res, _, err := idx.objectSearch(context.TODO(), 1000, nil, kwr, nil, nil, addit, nil, "", 0) - t.Log("--- Start results for singleprop search ---") - for _, r := range res { - t.Logf("Result id: %v, score: %v, title: %v, description: %v, additional %+v\n", r.DocID(), r.Score(), r.Object.Properties.(map[string]interface{})["title"], r.Object.Properties.(map[string]interface{})["description"], r.Object.Additional) - } - require.Nil(t, err) - // Check results in correct order - require.Equal(t, uint64(3), res[0].DocID()) - require.Equal(t, uint64(4), res[3].DocID()) - - // Check scores - EqualFloats(t, float32(0.1248), res[0].Score(), 5) - EqualFloats(t, float32(0.0363), res[1].Score(), 5) -} - -func TestBM25FWithFilters(t *testing.T) { - dirName := t.TempDir() - - logger := logrus.New() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, nil, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(context.TODO())) - defer repo.Shutdown(context.Background()) - - SetupClass(t, repo, schemaGetter, logger, 0.5, 100) - - idx := repo.GetIndex("MyClass") - require.NotNil(t, idx) - - filter := &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorOr, - Operands: []filters.Clause{ - { - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: schema.ClassName("MyClass"), - Property: schema.PropertyName("title"), - }, - Value: &filters.Value{ - Value: "My", - Type: schema.DataType("text"), - }, - }, - { - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: schema.ClassName("MyClass"), - Property: schema.PropertyName("title"), - }, - Value: &filters.Value{ - Value: "journeys", - Type: schema.DataType("text"), - }, - }, - }, - }, - } - - kwr := &searchparams.KeywordRanking{Type: "bm25", Properties: []string{"description"}, Query: "journey"} - addit := additional.Properties{} - res, _, err := idx.objectSearch(context.TODO(), 1000, filter, kwr, nil, nil, addit, nil, "", 0) - - require.Nil(t, err) - require.True(t, len(res) == 1) - require.Equal(t, uint64(2), res[0].DocID()) -} - -func TestBM25FWithFilters_ScoreIsIdenticalWithOrWithoutFilter(t *testing.T) { - dirName := t.TempDir() - - logger := logrus.New() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, nil, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(context.TODO())) - defer repo.Shutdown(context.Background()) - - SetupClassForFilterScoringTest(t, repo, schemaGetter, logger, 1.2, 0.75) - - idx := repo.GetIndex("FilterClass") - require.NotNil(t, idx) - - filter := &filters.LocalFilter{ - Root: &filters.Clause{ - On: &filters.Path{ - Class: schema.ClassName("FilterClass"), - Property: schema.PropertyName("relatedToGolf"), - }, - Operator: filters.OperatorEqual, - Value: &filters.Value{ - Value: true, - Type: dtBool, - }, - }, - } - - kwr := &searchparams.KeywordRanking{ - Type: "bm25", - Properties: []string{"description"}, - Query: "koepka golf", - } - - addit := additional.Properties{} - filtered, _, err := idx.objectSearch(context.TODO(), 1000, filter, kwr, nil, nil, addit, nil, "", 0) - require.Nil(t, err) - unfiltered, _, err := idx.objectSearch(context.TODO(), 1000, nil, kwr, nil, nil, addit, nil, "", 0) - require.Nil(t, err) - - require.Len(t, filtered, 1) // should match exactly one element - require.Len(t, unfiltered, 2) // contains irrelevant result - - assert.Equal(t, uint64(0), filtered[0].DocID()) // brooks koepka result - assert.Equal(t, uint64(0), unfiltered[0].DocID()) // brooks koepka result - - assert.Equal(t, filtered[0].Score(), unfiltered[0].Score()) -} - -func TestBM25FDifferentParamsJourney(t *testing.T) { - dirName := t.TempDir() - - logger := logrus.New() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, nil, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(context.TODO())) - defer repo.Shutdown(context.Background()) - - SetupClass(t, repo, schemaGetter, logger, 0.5, 100) - - idx := repo.GetIndex("MyClass") - require.NotNil(t, idx) - - // Check boosted - kwr := &searchparams.KeywordRanking{Type: "bm25", Properties: []string{"title^2", "description"}, Query: "journey"} - addit := additional.Properties{} - res, _, err := idx.objectSearch(context.TODO(), 1000, nil, kwr, nil, nil, addit, nil, "", 0) - - // Print results - t.Log("--- Start results for boosted search ---") - for _, r := range res { - t.Logf("Result id: %v, score: %v, title: %v, description: %v, additional %+v\n", r.DocID(), r.Score(), r.Object.Properties.(map[string]interface{})["title"], r.Object.Properties.(map[string]interface{})["description"], r.Object.Additional) - } - - require.Nil(t, err) - - // Check results in correct order - require.Equal(t, uint64(6), res[0].DocID()) - require.Equal(t, uint64(1), res[3].DocID()) - - // Print results - t.Log("--- Start results for boosted search ---") - for _, r := range res { - t.Logf("Result id: %v, score: %v, title: %v, description: %v, additional %+v\n", r.DocID(), r.Score(), r.Object.Properties.(map[string]interface{})["title"], r.Object.Properties.(map[string]interface{})["description"], r.Object.Additional) - } - - // Check scores - EqualFloats(t, float32(0.06023), res[0].Score(), 6) - EqualFloats(t, float32(0.04238), res[1].Score(), 6) -} - -func EqualFloats(t *testing.T, expected, actual float32, significantFigures int) { - s1 := fmt.Sprintf("%v", expected) - s2 := fmt.Sprintf("%v", actual) - if len(s1) < 2 || len(s2) < 2 { - t.Fail() - } - if len(s1) <= significantFigures { - significantFigures = len(s1) - 1 - } - if len(s2) <= significantFigures { - significantFigures = len(s2) - 1 - } - require.Equal(t, s1[:significantFigures+1], s2[:significantFigures+1]) -} - -// Compare with previous BM25 version to ensure the algorithm functions correctly -func TestBM25FCompare(t *testing.T) { - dirName := t.TempDir() - - logger := logrus.New() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, nil, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(context.TODO())) - defer repo.Shutdown(context.Background()) - - SetupClass(t, repo, schemaGetter, logger, 0.5, 100) - - idx := repo.GetIndex("MyClass") - require.NotNil(t, idx) - - shardNames := idx.getSchema.CopyShardingState(idx.Config.ClassName.String()).AllPhysicalShards() - - for _, shardName := range shardNames { - shard := idx.shards.Load(shardName) - t.Logf("------ BM25F --------\n") - kwr := &searchparams.KeywordRanking{Type: "bm25", Properties: []string{"title"}, Query: "journey"} - addit := additional.Properties{} - - withBM25Fobjs, withBM25Fscores, err := shard.ObjectSearch(context.TODO(), 1000, nil, kwr, nil, nil, addit) - require.Nil(t, err) - - for i, r := range withBM25Fobjs { - t.Logf("Result id: %v, score: %v, title: %v, description: %v, additional %+v\n", r.DocID(), withBM25Fscores[i], r.Object.Properties.(map[string]interface{})["title"], r.Object.Properties.(map[string]interface{})["description"], r.Object.Additional) - } - - t.Logf("------ BM25 --------\n") - kwr.Type = "" - - objs, scores, err := shard.ObjectSearch(context.TODO(), 1000, nil, kwr, nil, nil, addit) - require.Nil(t, err) - - for i, r := range objs { - t.Logf("Result id: %v, score: %v, title: %v, description: %v, additional %+v\n", r.DocID(), scores[i], r.Object.Properties.(map[string]interface{})["title"], r.Object.Properties.(map[string]interface{})["description"], r.Object.Additional) - } - - require.Equal(t, len(withBM25Fobjs), len(objs)) - for i := range objs { - t.Logf("%v: BM25F score: %v, BM25 score: %v", i, withBM25Fscores[i], scores[i]) - EqualFloats(t, withBM25Fscores[i], scores[i], 9) - } - - // Not all the scores are unique and the search is not stable, so pick ones that don't move - require.Equal(t, uint64(4), objs[0].DocID()) - require.Equal(t, uint64(5), objs[1].DocID()) - require.Equal(t, uint64(6), objs[2].DocID()) - require.Equal(t, uint64(1), objs[3].DocID()) - require.Equal(t, uint64(2), objs[4].DocID()) - require.Equal(t, uint64(0), objs[5].DocID()) - - require.Equal(t, uint64(4), withBM25Fobjs[0].DocID()) - require.Equal(t, uint64(5), withBM25Fobjs[1].DocID()) - require.Equal(t, uint64(6), withBM25Fobjs[2].DocID()) - require.Equal(t, uint64(1), withBM25Fobjs[3].DocID()) - require.Equal(t, uint64(2), withBM25Fobjs[4].DocID()) - require.Equal(t, uint64(0), withBM25Fobjs[5].DocID()) - - } -} - -func Test_propertyHasSearchableIndex(t *testing.T) { - vFalse := false - vTrue := true - - class := &models.Class{ - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: BM25FinvertedConfig(1, 1, "none"), - Class: "MyClass", - - Properties: []*models.Property{ - { - Name: "title", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWord, - IndexFilterable: &vFalse, - IndexSearchable: nil, - }, - { - Name: "description", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWord, - IndexFilterable: &vFalse, - IndexSearchable: &vTrue, - }, - { - Name: "textField", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationField, - IndexFilterable: &vFalse, - IndexSearchable: &vFalse, - }, - }, - } - - ClassSchema := &models.Schema{ - Classes: []*models.Class{class}, - } - t.Run("Property index", func(t *testing.T) { - if got := inverted.PropertyHasSearchableIndex(ClassSchema, "MyClass", "description"); got != true { - t.Errorf("PropertyHasSearchableIndex() = %v, want %v", got, true) - } - - if got := inverted.PropertyHasSearchableIndex(ClassSchema, "MyClass", "description^2"); got != true { - t.Errorf("PropertyHasSearchableIndex() = %v, want %v", got, true) - } - - if got := inverted.PropertyHasSearchableIndex(ClassSchema, "MyClass", "textField"); got != false { - t.Errorf("PropertyHasSearchableIndex() = %v, want %v", got, false) - } - - if got := inverted.PropertyHasSearchableIndex(ClassSchema, "MyClass", "title"); got != true { - t.Errorf("PropertyHasSearchableIndex() = %v, want %v", got, true) - } - }) -} - -func SetupClassDocuments(t require.TestingT, repo *DB, schemaGetter *fakeSchemaGetter, logger logrus.FieldLogger, k1, b float32, preset string, -) string { - vFalse := false - vTrue := true - - className := "DocumentsPreset_" + preset - class := &models.Class{ - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: BM25FinvertedConfig(k1, b, preset), - Class: className, - - Properties: []*models.Property{ - { - Name: "document", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWord, - IndexFilterable: &vFalse, - IndexSearchable: &vTrue, - }, - }, - } - schemaGetter.schema = schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{class}, - }, - } - - migrator := NewMigrator(repo, logger) - migrator.AddClass(context.Background(), class, schemaGetter.shardState) - - testData := []map[string]interface{}{} - testData = append(testData, map[string]interface{}{"document": "No matter what you do, the question of \"\"what is income\"\" is *always* going to be an extremely complex question. To use this particular example, is paying a royalty fee to an external party a legitimate business expense that is part of the cost of doing business and which subtracts from your \"\"income\"\"?"}) - testData = append(testData, map[string]interface{}{"document": "test"}) - testData = append(testData, map[string]interface{}{"document": "As long as the losing business is not considered \"\"passive activity\"\" or \"\"hobby\"\", then yes. Passive Activity is an activity where you do not have to actively do anything to generate income. For example - royalties or rentals. Hobby is an activity that doesn't generate profit. Generally, if your business doesn't consistently generate profit (the IRS looks at 3 out of the last 5 years), it may be characterized as hobby. For hobby, loss deduction is limited by the hobby income and the 2% AGI threshold."}) - testData = append(testData, map[string]interface{}{"document": "So you're basically saying that average market fluctuations have an affect on individual stocks, because individual stocks are often priced in relation to the growth of the market as a whole? Also, what kinds of investments would be considered \"\"risk free\"\" in this nomenclature?"}) - - for i, data := range testData { - id := strfmt.UUID(uuid.MustParse(fmt.Sprintf("%032d", i)).String()) - - obj := &models.Object{Class: className, ID: id, Properties: data, CreationTimeUnix: 1565612833955, LastUpdateTimeUnix: 10000020} - vector := []float32{1, 3, 5, 0.4} - //{title: "Our journey to BM25F", description: " This is how we get to BM25F"}} - err := repo.PutObject(context.Background(), obj, vector, nil) - require.Nil(t, err) - } - return className -} - -func TestBM25F_ComplexDocuments(t *testing.T) { - dirName := t.TempDir() - - logger := logrus.New() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - schemaGetter.schema = schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{}, - }, - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, nil, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(context.TODO())) - defer repo.Shutdown(context.Background()) - - classNone := SetupClassDocuments(t, repo, schemaGetter, logger, 0.5, 0.75, "none") - idxNone := repo.GetIndex(schema.ClassName(classNone)) - require.NotNil(t, idxNone) - - addit := additional.Properties{} - - t.Run("single term", func(t *testing.T) { - kwr := &searchparams.KeywordRanking{Type: "bm25", Query: "considered a"} - res, _, err := idxNone.objectSearch(context.TODO(), 10, nil, kwr, nil, nil, addit, nil, "", 0) - require.Nil(t, err) - - // Print results - t.Log("--- Start results for boosted search ---") - for _, r := range res { - t.Logf("Result id: %v, score: %v, \n", r.DocID(), r.Score()) - } - - // Check results in correct order - require.Equal(t, uint64(3), res[0].DocID()) - require.Equal(t, uint64(0), res[1].DocID()) - require.Equal(t, uint64(2), res[2].DocID()) - require.Len(t, res, 3) - - // Check scores - EqualFloats(t, float32(0.8914), res[0].Score(), 5) - EqualFloats(t, float32(0.5425), res[1].Score(), 5) - EqualFloats(t, float32(0.3952), res[2].Score(), 5) - }) - - t.Run("Results without stopwords", func(t *testing.T) { - kwrNoStopwords := &searchparams.KeywordRanking{Type: "bm25", Query: "example losing business"} - resNoStopwords, _, err := idxNone.objectSearch(context.TODO(), 10, nil, kwrNoStopwords, nil, nil, addit, nil, "", 0) - require.Nil(t, err) - - classEn := SetupClassDocuments(t, repo, schemaGetter, logger, 0.5, 0.75, "en") - idxEn := repo.GetIndex(schema.ClassName(classEn)) - require.NotNil(t, idxEn) - kwrStopwords := &searchparams.KeywordRanking{Type: "bm25", Query: "an example on losing the business"} - resStopwords, _, err := idxEn.objectSearch(context.TODO(), 10, nil, kwrStopwords, nil, nil, addit, nil, "", 0) - require.Nil(t, err) - - require.Equal(t, len(resNoStopwords), len(resStopwords)) - for i, resNo := range resNoStopwords { - resYes := resStopwords[i] - require.Equal(t, resNo.DocID(), resYes.DocID()) - require.Equal(t, resNo.Score(), resYes.Score()) - } - - kwrStopwordsDuplicate := &searchparams.KeywordRanking{Type: "bm25", Query: "on an example on losing the business on"} - resStopwordsDuplicate, _, err := idxEn.objectSearch(context.TODO(), 10, nil, kwrStopwordsDuplicate, nil, nil, addit, nil, "", 0) - require.Nil(t, err) - require.Equal(t, len(resNoStopwords), len(resStopwordsDuplicate)) - for i, resNo := range resNoStopwords { - resYes := resStopwordsDuplicate[i] - require.Equal(t, resNo.DocID(), resYes.DocID()) - require.Equal(t, resNo.Score(), resYes.Score()) - } - }) -} - -func MultiPropClass(t require.TestingT, repo *DB, schemaGetter *fakeSchemaGetter, logger logrus.FieldLogger, k1, b float32) string { - vFalse := false - vTrue := true - - className := "MultiProps" - class := &models.Class{ - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: BM25FinvertedConfig(k1, b, "none"), - Class: className, - - Properties: []*models.Property{ - { - Name: "document", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWord, - IndexFilterable: &vFalse, - IndexSearchable: &vTrue, - }, - { - Name: "title", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWord, - IndexFilterable: &vFalse, - IndexSearchable: &vTrue, - }, - }, - } - schemaGetter.schema = schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{class}, - }, - } - - migrator := NewMigrator(repo, logger) - migrator.AddClass(context.Background(), class, schemaGetter.shardState) - - testData := []map[string]interface{}{} - testData = append(testData, map[string]interface{}{"document": "test", "title": "pepper"}) - testData = append(testData, map[string]interface{}{"document": "banana", "title": "pepper"}) - testData = append(testData, map[string]interface{}{"document": "apple", "title": "banana taste great"}) - testData = append(testData, map[string]interface{}{"document": "banana burger", "title": "test"}) - testData = append(testData, map[string]interface{}{"document": "carotte", "title": "great"}) - - for i, data := range testData { - id := strfmt.UUID(uuid.MustParse(fmt.Sprintf("%032d", i)).String()) - - obj := &models.Object{Class: className, ID: id, Properties: data, CreationTimeUnix: 1565612833955, LastUpdateTimeUnix: 10000020} - vector := []float32{1, 3, 5, 0.4} - err := repo.PutObject(context.Background(), obj, vector, nil) - require.Nil(t, err) - } - return className -} - -func TestBM25F_SortMultiProp(t *testing.T) { - t.Skip("Currently failing") - dirName := t.TempDir() - - logger := logrus.New() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - schemaGetter.schema = schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{}, - }, - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, nil, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(context.TODO())) - defer repo.Shutdown(context.Background()) - - idx := repo.GetIndex(schema.ClassName(MultiPropClass(t, repo, schemaGetter, logger, 0.5, 0.75))) - require.NotNil(t, idx) - - addit := additional.Properties{} - - t.Run("single term", func(t *testing.T) { - kwr := &searchparams.KeywordRanking{Type: "bm25", Query: "pepper banana"} - res, _, err := idx.objectSearch(context.TODO(), 1, nil, kwr, nil, nil, addit, nil, "", 0) - require.Nil(t, err) - - // Print results - t.Log("--- Start results for boosted search ---") - for _, r := range res { - t.Logf("Result id: %v, score: %v, \n", r.DocID(), r.Score()) - } - - // Document 1 is a result for both terms - require.Len(t, res, 1) - require.Equal(t, uint64(1), res[0].DocID()) - }) -} diff --git a/adapters/repos/db/classification.go b/adapters/repos/db/classification.go deleted file mode 100644 index c1107a4cc6d52dcebd3fcd44c018b88c425b8290..0000000000000000000000000000000000000000 --- a/adapters/repos/db/classification.go +++ /dev/null @@ -1,293 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "context" - "fmt" - "math" - - "github.com/go-openapi/strfmt" - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/dto" - "github.com/weaviate/weaviate/entities/filters" - libfilters "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/search" - "github.com/weaviate/weaviate/usecases/classification" - "github.com/weaviate/weaviate/usecases/vectorizer" -) - -// TODO: why is this logic in the persistence package? This is business-logic, -// move out of here! -func (db *DB) GetUnclassified(ctx context.Context, class string, - properties []string, filter *libfilters.LocalFilter, -) ([]search.Result, error) { - mergedFilter := mergeUserFilterWithRefCountFilter(filter, class, properties, - libfilters.OperatorEqual, 0) - res, err := db.Search(ctx, dto.GetParams{ - ClassName: class, - Filters: mergedFilter, - Pagination: &libfilters.Pagination{ - Limit: 10000, // TODO: gh-1219 increase - }, - AdditionalProperties: additional.Properties{ - Classification: true, - Vector: true, - ModuleParams: map[string]interface{}{ - "interpretation": true, - }, - }, - }) - - return res, err -} - -// TODO: why is this logic in the persistence package? This is business-logic, -// move out of here! -func (db *DB) ZeroShotSearch(ctx context.Context, vector []float32, - class string, properties []string, - filter *libfilters.LocalFilter, -) ([]search.Result, error) { - res, err := db.VectorSearch(ctx, dto.GetParams{ - ClassName: class, - SearchVector: vector, - Pagination: &filters.Pagination{ - Limit: 1, - }, - Filters: filter, - AdditionalProperties: additional.Properties{ - Vector: true, - }, - }) - - return res, err -} - -// TODO: why is this logic in the persistence package? This is business-logic, -// move out of here! -func (db *DB) AggregateNeighbors(ctx context.Context, vector []float32, - class string, properties []string, k int, - filter *libfilters.LocalFilter, -) ([]classification.NeighborRef, error) { - mergedFilter := mergeUserFilterWithRefCountFilter(filter, class, properties, - libfilters.OperatorGreaterThan, 0) - res, err := db.VectorSearch(ctx, dto.GetParams{ - ClassName: class, - SearchVector: vector, - Pagination: &filters.Pagination{ - Limit: k, - }, - Filters: mergedFilter, - AdditionalProperties: additional.Properties{ - Vector: true, - }, - }) - if err != nil { - return nil, errors.Wrap(err, "aggregate neighbors: search neighbors") - } - - return NewKnnAggregator(res, vector).Aggregate(k, properties) -} - -// TODO: this is business logic, move out of here -type KnnAggregator struct { - input search.Results - sourceVector []float32 -} - -func NewKnnAggregator(input search.Results, sourceVector []float32) *KnnAggregator { - return &KnnAggregator{input: input, sourceVector: sourceVector} -} - -func (a *KnnAggregator) Aggregate(k int, properties []string) ([]classification.NeighborRef, error) { - neighbors, err := a.extractBeacons(properties) - if err != nil { - return nil, errors.Wrap(err, "aggregate: extract beacons from neighbors") - } - - return a.aggregateBeacons(neighbors) -} - -func (a *KnnAggregator) extractBeacons(properties []string) (neighborProps, error) { - neighbors := neighborProps{} - for i, elem := range a.input { - schemaMap, ok := elem.Schema.(map[string]interface{}) - if !ok { - return nil, fmt.Errorf("expecteded element[%d].Schema to be map, got: %T", i, elem.Schema) - } - - for _, prop := range properties { - refProp, ok := schemaMap[prop] - if !ok { - return nil, fmt.Errorf("expecteded element[%d].Schema to have property %q, but didn't", i, prop) - } - - refTyped, ok := refProp.(models.MultipleRef) - if !ok { - return nil, fmt.Errorf("expecteded element[%d].Schema.%s to be models.MultipleRef, got: %T", i, prop, refProp) - } - - if len(refTyped) != 1 { - return nil, fmt.Errorf("a knn training data object needs to have exactly one label: "+ - "expecteded element[%d].Schema.%s to have exactly one reference, got: %d", - i, prop, len(refTyped)) - } - - distance, err := vectorizer.NormalizedDistance(a.sourceVector, elem.Vector) - if err != nil { - return nil, errors.Wrap(err, "calculate distance between source and candidate") - } - - beacon := refTyped[0].Beacon.String() - neighborProp := neighbors[prop] - if neighborProp.beacons == nil { - neighborProp.beacons = neighborBeacons{} - } - neighborProp.beacons[beacon] = append(neighborProp.beacons[beacon], distance) - neighbors[prop] = neighborProp - } - } - - return neighbors, nil -} - -func (a *KnnAggregator) aggregateBeacons(props neighborProps) ([]classification.NeighborRef, error) { - var out []classification.NeighborRef - for propName, prop := range props { - var winningBeacon string - var winningCount int - var totalCount int - - for beacon, distances := range prop.beacons { - totalCount += len(distances) - if len(distances) > winningCount { - winningBeacon = beacon - winningCount = len(distances) - } - } - - distances := a.distances(prop.beacons, winningBeacon) - out = append(out, classification.NeighborRef{ - Beacon: strfmt.URI(winningBeacon), - WinningCount: winningCount, - OverallCount: totalCount, - LosingCount: totalCount - winningCount, - Property: propName, - Distances: distances, - }) - } - - return out, nil -} - -func (a *KnnAggregator) distances(beacons neighborBeacons, - winner string, -) classification.NeighborRefDistances { - out := classification.NeighborRefDistances{} - - var winningDistances []float32 - var losingDistances []float32 - - for beacon, distances := range beacons { - if beacon == winner { - winningDistances = distances - } else { - losingDistances = append(losingDistances, distances...) - } - } - - if len(losingDistances) > 0 { - mean := mean(losingDistances) - out.MeanLosingDistance = &mean - - closest := min(losingDistances) - out.ClosestLosingDistance = &closest - } - - out.ClosestOverallDistance = min(append(winningDistances, losingDistances...)) - out.ClosestWinningDistance = min(winningDistances) - out.MeanWinningDistance = mean(winningDistances) - - return out -} - -type neighborProps map[string]neighborProp - -type neighborProp struct { - beacons neighborBeacons -} - -type neighborBeacons map[string][]float32 - -func mergeUserFilterWithRefCountFilter(userFilter *libfilters.LocalFilter, className string, - properties []string, op libfilters.Operator, refCount int, -) *libfilters.LocalFilter { - countFilters := make([]libfilters.Clause, len(properties)) - for i, prop := range properties { - countFilters[i] = libfilters.Clause{ - Operator: op, - Value: &libfilters.Value{ - Type: schema.DataTypeInt, - Value: refCount, - }, - On: &libfilters.Path{ - Class: schema.ClassName(className), - Property: schema.PropertyName(prop), - }, - } - } - - var countRootClause libfilters.Clause - if len(countFilters) == 1 { - countRootClause = countFilters[0] - } else { - countRootClause = libfilters.Clause{ - Operands: countFilters, - Operator: libfilters.OperatorAnd, - } - } - - rootFilter := &libfilters.LocalFilter{} - if userFilter == nil { - rootFilter.Root = &countRootClause - } else { - rootFilter.Root = &libfilters.Clause{ - Operator: libfilters.OperatorAnd, // so we can AND the refcount requirements and whatever custom filters, the user has - Operands: []libfilters.Clause{*userFilter.Root, countRootClause}, - } - } - - return rootFilter -} - -func mean(in []float32) float32 { - sum := float32(0) - for _, v := range in { - sum += v - } - - return sum / float32(len(in)) -} - -func min(in []float32) float32 { - min := float32(math.MaxFloat32) - for _, dist := range in { - if dist < min { - min = dist - } - } - - return min -} diff --git a/adapters/repos/db/classification_integration_test.go b/adapters/repos/db/classification_integration_test.go deleted file mode 100644 index a92295c5bf2e433f2777fca41f4e46a9e458a830..0000000000000000000000000000000000000000 --- a/adapters/repos/db/classification_integration_test.go +++ /dev/null @@ -1,435 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package db - -import ( - "context" - "fmt" - "testing" - - "github.com/go-openapi/strfmt" - "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/search" - enthnsw "github.com/weaviate/weaviate/entities/vectorindex/hnsw" - "github.com/weaviate/weaviate/usecases/classification" -) - -func TestClassifications(t *testing.T) { - dirName := t.TempDir() - - logger := logrus.New() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - defer repo.Shutdown(context.Background()) - migrator := NewMigrator(repo, logger) - - t.Run("importing classification schema", func(t *testing.T) { - for _, class := range classificationTestSchema() { - err := migrator.AddClass(context.Background(), class, schemaGetter.shardState) - require.Nil(t, err) - } - }) - - // update schema getter so it's in sync with class - schemaGetter.schema = schema.Schema{Objects: &models.Schema{Classes: classificationTestSchema()}} - - t.Run("importing categories", func(t *testing.T) { - for _, res := range classificationTestCategories() { - thing := res.Object() - err := repo.PutObject(context.Background(), thing, res.Vector, nil) - require.Nil(t, err) - } - }) - - t.Run("importing articles", func(t *testing.T) { - for _, res := range classificationTestArticles() { - thing := res.Object() - err := repo.PutObject(context.Background(), thing, res.Vector, nil) - require.Nil(t, err) - } - }) - - t.Run("finding all unclassified (no filters)", func(t *testing.T) { - res, err := repo.GetUnclassified(context.Background(), - "Article", []string{"exactCategory", "mainCategory"}, nil) - require.Nil(t, err) - require.Len(t, res, 6) - }) - - t.Run("finding all unclassified (with filters)", func(t *testing.T) { - filter := &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorEqual, - On: &filters.Path{ - Property: "description", - }, - Value: &filters.Value{ - Value: "johnny", - Type: schema.DataTypeText, - }, - }, - } - - res, err := repo.GetUnclassified(context.Background(), - "Article", []string{"exactCategory", "mainCategory"}, filter) - require.Nil(t, err) - require.Len(t, res, 1) - assert.Equal(t, strfmt.UUID("a2bbcbdc-76e1-477d-9e72-a6d2cfb50109"), res[0].ID) - }) - - t.Run("aggregating over item neighbors", func(t *testing.T) { - t.Run("close to politics (no filters)", func(t *testing.T) { - res, err := repo.AggregateNeighbors(context.Background(), - []float32{0.7, 0.01, 0.01}, "Article", - []string{"exactCategory", "mainCategory"}, 1, nil) - - expectedRes := []classification.NeighborRef{ - { - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", idCategoryPolitics)), - Property: "exactCategory", - OverallCount: 1, - WinningCount: 1, - LosingCount: 0, - Distances: classification.NeighborRefDistances{ - MeanWinningDistance: 0.00010201335, - ClosestWinningDistance: 0.00010201335, - ClosestOverallDistance: 0.00010201335, - }, - }, - { - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", idMainCategoryPoliticsAndSociety)), - Property: "mainCategory", - OverallCount: 1, - WinningCount: 1, - LosingCount: 0, - Distances: classification.NeighborRefDistances{ - MeanWinningDistance: 0.00010201335, - ClosestWinningDistance: 0.00010201335, - ClosestOverallDistance: 0.00010201335, - }, - }, - } - - require.Nil(t, err) - assert.ElementsMatch(t, expectedRes, res) - }) - - t.Run("close to food and drink (no filters)", func(t *testing.T) { - res, err := repo.AggregateNeighbors(context.Background(), - []float32{0.01, 0.01, 0.66}, "Article", - []string{"exactCategory", "mainCategory"}, 1, nil) - - expectedRes := []classification.NeighborRef{ - { - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", idCategoryFoodAndDrink)), - Property: "exactCategory", - OverallCount: 1, - WinningCount: 1, - LosingCount: 0, - Distances: classification.NeighborRefDistances{ - MeanWinningDistance: 0.00011473894, - ClosestWinningDistance: 0.00011473894, - ClosestOverallDistance: 0.00011473894, - }, - }, - { - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", idMainCategoryFoodAndDrink)), - Property: "mainCategory", - OverallCount: 1, - WinningCount: 1, - LosingCount: 0, - Distances: classification.NeighborRefDistances{ - MeanWinningDistance: 0.00011473894, - ClosestWinningDistance: 0.00011473894, - ClosestOverallDistance: 0.00011473894, - }, - }, - } - - require.Nil(t, err) - assert.ElementsMatch(t, expectedRes, res) - }) - - t.Run("close to food and drink (but limiting to politics through filter)", func(t *testing.T) { - filter := &filters.LocalFilter{ - Root: &filters.Clause{ - On: &filters.Path{ - Property: "description", - }, - Value: &filters.Value{ - Value: "politics", - Type: schema.DataTypeText, - }, - Operator: filters.OperatorEqual, - }, - } - res, err := repo.AggregateNeighbors(context.Background(), - []float32{0.01, 0.01, 0.66}, "Article", - []string{"exactCategory", "mainCategory"}, 1, filter) - - expectedRes := []classification.NeighborRef{ - { - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", idCategoryPolitics)), - Property: "exactCategory", - OverallCount: 1, - WinningCount: 1, - LosingCount: 0, - Distances: classification.NeighborRefDistances{ - MeanWinningDistance: 0.49242598, - ClosestWinningDistance: 0.49242598, - ClosestOverallDistance: 0.49242598, - }, - }, - { - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", idMainCategoryPoliticsAndSociety)), - Property: "mainCategory", - OverallCount: 1, - WinningCount: 1, - LosingCount: 0, - Distances: classification.NeighborRefDistances{ - MeanWinningDistance: 0.49242598, - ClosestWinningDistance: 0.49242598, - ClosestOverallDistance: 0.49242598, - }, - }, - } - - require.Nil(t, err) - assert.ElementsMatch(t, expectedRes, res) - }) - }) -} - -// test fixtures -func classificationTestSchema() []*models.Class { - return []*models.Class{ - { - Class: "ExactCategory", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - }, - { - Class: "MainCategory", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - }, - { - Class: "Article", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{ - { - Name: "description", - DataType: []string{string(schema.DataTypeText)}, - Tokenization: "word", - }, - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "exactCategory", - DataType: []string{"ExactCategory"}, - }, - { - Name: "mainCategory", - DataType: []string{"MainCategory"}, - }, - }, - }, - } -} - -const ( - idMainCategoryPoliticsAndSociety = "39c6abe3-4bbe-4c4e-9e60-ca5e99ec6b4e" - idMainCategoryFoodAndDrink = "5a3d909a-4f0d-4168-8f5c-cd3074d1e79a" - idCategoryPolitics = "1b204f16-7da6-44fd-bbd2-8cc4a7414bc3" - idCategorySociety = "ec500f39-1dc9-4580-9bd1-55a8ea8e37a2" - idCategoryFoodAndDrink = "027b708a-31ca-43ea-9001-88bec864c79c" -) - -func beaconRef(target string) *models.SingleRef { - beacon := fmt.Sprintf("weaviate://localhost/%s", target) - return &models.SingleRef{Beacon: strfmt.URI(beacon)} -} - -func classificationTestCategories() search.Results { - // using search.Results, because it's the perfect grouping of object and - // vector - return search.Results{ - // exact categories - search.Result{ - ID: idCategoryPolitics, - ClassName: "ExactCategory", - Vector: []float32{1, 0, 0}, - Schema: map[string]interface{}{ - "name": "Politics", - }, - }, - search.Result{ - ID: idCategorySociety, - ClassName: "ExactCategory", - Vector: []float32{0, 1, 0}, - Schema: map[string]interface{}{ - "name": "Society", - }, - }, - search.Result{ - ID: idCategoryFoodAndDrink, - ClassName: "ExactCategory", - Vector: []float32{0, 0, 1}, - Schema: map[string]interface{}{ - "name": "Food and Drink", - }, - }, - - // main categories - search.Result{ - ID: idMainCategoryPoliticsAndSociety, - ClassName: "MainCategory", - Vector: []float32{0, 1, 0}, - Schema: map[string]interface{}{ - "name": "Politics and Society", - }, - }, - search.Result{ - ID: idMainCategoryFoodAndDrink, - ClassName: "MainCategory", - Vector: []float32{0, 0, 1}, - Schema: map[string]interface{}{ - "name": "Food and Drink", - }, - }, - } -} - -func classificationTestArticles() search.Results { - // using search.Results, because it's the perfect grouping of object and - // vector - return search.Results{ - // classified - search.Result{ - ID: "8aeecd06-55a0-462c-9853-81b31a284d80", - ClassName: "Article", - Vector: []float32{1, 0, 0}, - Schema: map[string]interface{}{ - "description": "This article talks about politics", - "exactCategory": models.MultipleRef{beaconRef(idCategoryPolitics)}, - "mainCategory": models.MultipleRef{beaconRef(idMainCategoryPoliticsAndSociety)}, - }, - }, - search.Result{ - ID: "9f4c1847-2567-4de7-8861-34cf47a071ae", - ClassName: "Article", - Vector: []float32{0, 1, 0}, - Schema: map[string]interface{}{ - "description": "This articles talks about society", - "exactCategory": models.MultipleRef{beaconRef(idCategorySociety)}, - "mainCategory": models.MultipleRef{beaconRef(idMainCategoryPoliticsAndSociety)}, - }, - }, - search.Result{ - ID: "926416ec-8fb1-4e40-ab8c-37b226b3d68e", - ClassName: "Article", - Vector: []float32{0, 0, 1}, - Schema: map[string]interface{}{ - "description": "This article talks about food", - "exactCategory": models.MultipleRef{beaconRef(idCategoryFoodAndDrink)}, - "mainCategory": models.MultipleRef{beaconRef(idMainCategoryFoodAndDrink)}, - }, - }, - - // unclassified - search.Result{ - ID: "75ba35af-6a08-40ae-b442-3bec69b355f9", - ClassName: "Article", - Vector: []float32{0.78, 0, 0}, - Schema: map[string]interface{}{ - "description": "Barack Obama is a former US president", - }, - }, - search.Result{ - ID: "f850439a-d3cd-4f17-8fbf-5a64405645cd", - ClassName: "Article", - Vector: []float32{0.90, 0, 0}, - Schema: map[string]interface{}{ - "description": "Michelle Obama is Barack Obamas wife", - }, - }, - search.Result{ - ID: "a2bbcbdc-76e1-477d-9e72-a6d2cfb50109", - ClassName: "Article", - Vector: []float32{0, 0.78, 0}, - Schema: map[string]interface{}{ - "description": "Johnny Depp is an actor", - }, - }, - search.Result{ - ID: "069410c3-4b9e-4f68-8034-32a066cb7997", - ClassName: "Article", - Vector: []float32{0, 0.90, 0}, - Schema: map[string]interface{}{ - "description": "Brad Pitt starred in a Quentin Tarantino movie", - }, - }, - search.Result{ - ID: "06a1e824-889c-4649-97f9-1ed3fa401d8e", - ClassName: "Article", - Vector: []float32{0, 0, 0.78}, - Schema: map[string]interface{}{ - "description": "Ice Cream often contains a lot of sugar", - }, - }, - search.Result{ - ID: "6402e649-b1e0-40ea-b192-a64eab0d5e56", - ClassName: "Article", - Vector: []float32{0, 0, 0.90}, - Schema: map[string]interface{}{ - "description": "French Fries are more common in Belgium and the US than in France", - }, - }, - } -} diff --git a/adapters/repos/db/clusterintegrationtest/backup_coordinator_integration_test.go b/adapters/repos/db/clusterintegrationtest/backup_coordinator_integration_test.go deleted file mode 100644 index fd5f3062ec9d0af49df870b6a09c520b2f02d5bd..0000000000000000000000000000000000000000 --- a/adapters/repos/db/clusterintegrationtest/backup_coordinator_integration_test.go +++ /dev/null @@ -1,208 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package clusterintegrationtest - -import ( - "context" - "encoding/json" - "fmt" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" - modstgfs "github.com/weaviate/weaviate/modules/backup-filesystem" - "github.com/weaviate/weaviate/usecases/backup" -) - -var backend *fakeBackupBackend - -func TestDistributedBackups(t *testing.T) { - var ( - dirName = setupDirectory(t) - rnd = getRandomSeed() - numObjs = 100 - numNodes = 3 - backupID = "new-backup" - nodes []*node - ) - - t.Run("setup", func(t *testing.T) { - overallShardState := multiShardState(numNodes) - shardStateSerialized, err := json.Marshal(overallShardState) - require.Nil(t, err) - - backend = &fakeBackupBackend{ - backupsPath: dirName, - backupID: backupID, - startedAt: time.Now(), - } - - for i := 0; i < numNodes; i++ { - node := &node{ - name: fmt.Sprintf("node-%d", i), - } - - node.init(dirName, shardStateSerialized, &nodes) - nodes = append(nodes, node) - } - }) - - t.Run("apply schema", func(t *testing.T) { - for i := range nodes { - err := nodes[i].migrator.AddClass(context.Background(), class(), - nodes[i].schemaManager.shardState) - require.Nil(t, err) - err = nodes[i].migrator.AddClass(context.Background(), secondClassWithRef(), - nodes[i].schemaManager.shardState) - require.Nil(t, err) - nodes[i].schemaManager.schema.Objects.Classes = append(nodes[i].schemaManager.schema.Objects.Classes, - class(), secondClassWithRef()) - } - }) - - data := exampleData(numObjs) - refData := exampleDataWithRefs(numObjs, 5, data) - - t.Run("import data", func(t *testing.T) { - t.Run("import first class into random node", func(t *testing.T) { - for _, obj := range data { - node := nodes[rnd.Intn(len(nodes))] - - err := node.repo.PutObject(context.Background(), obj, obj.Vector, nil) - require.Nil(t, err) - } - }) - - t.Run("import second class into random node", func(t *testing.T) { - for _, obj := range refData { - node := nodes[rnd.Intn(len(nodes))] - - err := node.repo.PutObject(context.Background(), obj, obj.Vector, nil) - require.Nil(t, err) - } - }) - }) - - t.Run("should fail backup with local filesystem backend", func(t *testing.T) { - req := &backup.BackupRequest{ - ID: backupID, Backend: modstgfs.Name, - Include: []string{distributedClass}, - } - - resp, err := nodes[0].scheduler.Backup(context.Background(), &models.Principal{}, req) - assert.Nil(t, resp) - assert.Contains(t, err.Error(), "local filesystem backend is not viable for backing up a node cluster") - }) - - t.Run("should fail restore with local filesystem backend", func(t *testing.T) { - req := &backup.BackupRequest{ - ID: backupID, Backend: modstgfs.Name, - Include: []string{distributedClass}, - } - - resp, err := nodes[0].scheduler.Restore(context.Background(), &models.Principal{}, req) - assert.Nil(t, resp) - assert.Contains(t, err.Error(), "local filesystem backend is not viable for backing up a node cluster") - }) - - t.Run("let each node be the coordinator", func(t *testing.T) { - for _, node := range nodes { - t.Run(fmt.Sprintf("%s: coordinate backup", node.name), func(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) - defer cancel() - - req := &backup.BackupRequest{ - ID: backupID, Backend: "fake-backend", - Include: []string{distributedClass}, - } - - resp, err := node.scheduler.Backup(ctx, &models.Principal{}, req) - assert.Nil(t, err, "expected nil err, got: %s", err) - assert.Empty(t, resp.Error, "expected empty, got: %s", resp.Error) - assert.NotEmpty(t, resp.Path) - assert.Contains(t, resp.Classes, distributedClass) - }) - - t.Run(fmt.Sprintf("%s: get backup status", node.name), func(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) - defer cancel() - - start := time.Now() - - for { - if time.Now().After(start.Add(30 * time.Second)) { - t.Fatal("backup deadline exceeded") - } - resp, err := node.scheduler.BackupStatus(ctx, &models.Principal{}, "fake-backend", backupID) - assert.Nil(t, err, "expected nil err, got: %s", err) - if resp != nil && string(resp.Status) == "SUCCESS" { - break - } - if resp != nil && string(resp.Status) == "FAILED" { - t.Fatalf("backup failed: %q", resp.Err) - } - } - }) - - t.Run(fmt.Sprintf("%s: restore to cluster", node.name), func(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) - defer cancel() - - req := &backup.BackupRequest{ - ID: backupID, Backend: "fake-backend", - Include: []string{distributedClass}, - } - - resp, err := node.scheduler.Restore(ctx, &models.Principal{}, req) - assert.Nil(t, err, "expected nil err, got: %s", err) - assert.Empty(t, resp.Error, "expected empty, got: %s", resp.Error) - assert.NotEmpty(t, resp.Path) - assert.Contains(t, resp.Classes, distributedClass) - }) - - t.Run(fmt.Sprintf("%s: get restore status", node.name), func(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) - defer cancel() - - start := time.Now() - - for { - if time.Now().After(start.Add(30 * time.Second)) { - t.Fatal("restore deadline exceeded") - } - resp, err := node.scheduler.RestorationStatus(ctx, &models.Principal{}, "fake-backend", backupID) - assert.Nil(t, err, "expected nil err, got: %s", err) - if resp != nil && string(resp.Status) == "SUCCESS" { - break - } - if resp != nil && string(resp.Status) == "FAILED" { - t.Fatalf("restore failed: %q", resp.Err) - } - } - }) - - backend.reset() - } - }) - - t.Run("shutdown", func(t *testing.T) { - for _, node := range nodes { - err := node.repo.Shutdown(context.Background()) - require.Nil(t, err, "expected nil, got: %v", err) - } - }) -} diff --git a/adapters/repos/db/clusterintegrationtest/cluster_integration_test.go b/adapters/repos/db/clusterintegrationtest/cluster_integration_test.go deleted file mode 100644 index c7f9661cdcdf28fd3f45e48cb8082357ab9a35d7..0000000000000000000000000000000000000000 --- a/adapters/repos/db/clusterintegrationtest/cluster_integration_test.go +++ /dev/null @@ -1,671 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package clusterintegrationtest - -import ( - "context" - "encoding/json" - "fmt" - "math/rand" - "strconv" - "strings" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/adapters/repos/db" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/aggregation" - "github.com/weaviate/weaviate/entities/dto" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/search" - "github.com/weaviate/weaviate/entities/searchparams" - "github.com/weaviate/weaviate/usecases/objects" -) - -const ( - vectorDims = 20 - numberOfNodes = 10 - distributedClass = "Distributed" -) - -// TestDistributedSetup uses as many real components and only mocks out -// non-essential parts. Essentially we fix the shard/cluster state and schema -// as they aren't critical to this test, but use real repos and real HTTP APIs -// between the repos. -func TestDistributedSetup(t *testing.T) { - t.Run("individual imports", func(t *testing.T) { - dirName := setupDirectory(t) - r := getRandomSeed() - testDistributed(t, dirName, r, false) - }) - t.Run("batched imports", func(t *testing.T) { - dirName := setupDirectory(t) - r := getRandomSeed() - testDistributed(t, dirName, r, true) - }) -} - -func testDistributed(t *testing.T, dirName string, rnd *rand.Rand, batch bool) { - var nodes []*node - numberOfObjects := 200 - - t.Run("setup", func(t *testing.T) { - overallShardState := multiShardState(numberOfNodes) - shardStateSerialized, err := json.Marshal(overallShardState) - require.Nil(t, err) - - for i := 0; i < numberOfNodes; i++ { - node := &node{ - name: fmt.Sprintf("node-%d", i), - } - - node.init(dirName, shardStateSerialized, &nodes) - nodes = append(nodes, node) - } - }) - - t.Run("apply schema", func(t *testing.T) { - for i := range nodes { - err := nodes[i].migrator.AddClass(context.Background(), class(), - nodes[i].schemaManager.shardState) - require.Nil(t, err) - err = nodes[i].migrator.AddClass(context.Background(), secondClassWithRef(), - nodes[i].schemaManager.shardState) - require.Nil(t, err) - nodes[i].schemaManager.schema.Objects.Classes = append(nodes[i].schemaManager.schema.Objects.Classes, - class(), secondClassWithRef()) - } - }) - - data := exampleData(numberOfObjects) - refData := exampleDataWithRefs(numberOfObjects, 5, data) - - if batch { - t.Run("import large batch from random node", func(t *testing.T) { - // pick a random node, but send the entire batch to this node - node := nodes[rnd.Intn(len(nodes))] - - batchObjs := dataAsBatch(data) - res, err := node.repo.BatchPutObjects(context.Background(), batchObjs, nil) - require.Nil(t, err) - for _, ind := range res { - require.Nil(t, ind.Err) - } - }) - - t.Run("import second class without refs", func(t *testing.T) { - // pick a random node, but send the entire batch to this node - node := nodes[rnd.Intn(len(nodes))] - - batchObjs := dataAsBatchWithProps(refData, []string{"description"}) - res, err := node.repo.BatchPutObjects(context.Background(), batchObjs, nil) - require.Nil(t, err) - for _, ind := range res { - require.Nil(t, ind.Err) - } - }) - - t.Run("import refs as batch", func(t *testing.T) { - // pick a random node, but send the entire batch to this node - node := nodes[rnd.Intn(len(nodes))] - - batch := refsAsBatch(refData, "toFirst") - res, err := node.repo.AddBatchReferences(context.Background(), batch, nil) - require.Nil(t, err) - for _, ind := range res { - require.Nil(t, ind.Err) - } - }) - } else { - t.Run("import first class by picking a random node", func(t *testing.T) { - for _, obj := range data { - node := nodes[rnd.Intn(len(nodes))] - - err := node.repo.PutObject(context.Background(), obj, obj.Vector, nil) - require.Nil(t, err) - } - }) - t.Run("import second class with refs by picking a random node", func(t *testing.T) { - for _, obj := range refData { - node := nodes[rnd.Intn(len(nodes))] - - err := node.repo.PutObject(context.Background(), obj, obj.Vector, nil) - require.Nil(t, err) - - } - }) - } - t.Run("query individually to check if all exist using random nodes", func(t *testing.T) { - for _, obj := range data { - node := nodes[rnd.Intn(len(nodes))] - - ok, err := node.repo.Exists(context.Background(), distributedClass, obj.ID, nil, "") - require.Nil(t, err) - assert.True(t, ok) - } - }) - - t.Run("query individually using random node", func(t *testing.T) { - for _, obj := range data { - node := nodes[rnd.Intn(len(nodes))] - - res, err := node.repo.ObjectByID(context.Background(), obj.ID, search.SelectProperties{}, additional.Properties{}, "") - require.Nil(t, err) - require.NotNil(t, res) - - // only compare string prop to avoid having to deal with parsing time - // props - assert.Equal(t, obj.Properties.(map[string]interface{})["description"], - res.Object().Properties.(map[string]interface{})["description"]) - } - }) - - t.Run("perform vector searches", func(t *testing.T) { - // note this test assumes a recall of 100% which only works with HNSW on - // small sizes, so if we use this test suite with massive sizes, we should - // not expect this test to succeed 100% of times anymore. - runs := 10 - - for i := 0; i < runs; i++ { - query := make([]float32, vectorDims) - for i := range query { - query[i] = rnd.Float32() - } - - groundTruth := bruteForceObjectsByQuery(data, query) - - node := nodes[rnd.Intn(len(nodes))] - res, err := node.repo.VectorSearch(context.Background(), dto.GetParams{ - SearchVector: query, - Pagination: &filters.Pagination{ - Limit: 25, - }, - ClassName: distributedClass, - }) - assert.Nil(t, err) - for i, obj := range res { - assert.Equal(t, groundTruth[i].ID, obj.ID, fmt.Sprintf("at pos %d", i)) - } - } - - for _, obj := range data { - node := nodes[rnd.Intn(len(nodes))] - - res, err := node.repo.ObjectByID(context.Background(), obj.ID, search.SelectProperties{}, additional.Properties{}, "") - require.Nil(t, err) - require.NotNil(t, res) - - // only compare string prop to avoid having to deal with parsing time - // props - assert.Equal(t, obj.Properties.(map[string]interface{})["description"], - res.Object().Properties.(map[string]interface{})["description"]) - } - }) - - t.Run("query individually and resolve references", func(t *testing.T) { - for _, obj := range refData { - // if i == 5 { - // break - // } - node := nodes[rnd.Intn(len(nodes))] - - res, err := node.repo.ObjectByID(context.Background(), obj.ID, search.SelectProperties{ - search.SelectProperty{ - Name: "toFirst", - IsPrimitive: false, - Refs: []search.SelectClass{ - { - ClassName: distributedClass, - RefProperties: search.SelectProperties{ - search.SelectProperty{ - Name: "description", - IsPrimitive: true, - }, - }, - }, - }, - }, - }, additional.Properties{}, "") - require.Nil(t, err) - require.NotNil(t, res) - props := res.Object().Properties.(map[string]interface{}) - refProp, ok := props["toFirst"].([]interface{}) - require.True(t, ok) - - var refPayload []map[string]interface{} - for _, res := range refProp { - parsed, ok := res.(search.LocalRef) - require.True(t, ok) - refPayload = append(refPayload, map[string]interface{}{ - "description": parsed.Fields["description"], - }) - } - - actual := manuallyResolveRef(t, obj, data, "toFirst", "description", nil) - assert.Equal(t, actual, refPayload) - } - }) - - t.Run("query individually with cross-ref vectors and resolve references", func(t *testing.T) { - for _, obj := range refData { - // if i == 1 { - // break - // } - node := nodes[rnd.Intn(len(nodes))] - - res, err := node.repo.Object(context.Background(), obj.Class, obj.ID, search.SelectProperties{ - search.SelectProperty{ - Name: "toFirst", - IsPrimitive: false, - Refs: []search.SelectClass{ - { - ClassName: distributedClass, - RefProperties: search.SelectProperties{ - search.SelectProperty{ - Name: "description", - IsPrimitive: true, - }, - }, - AdditionalProperties: additional.Properties{ - Vector: true, - }, - }, - }, - }, - }, additional.Properties{}, nil, "") - require.Nil(t, err) - require.NotNil(t, res) - props := res.Object().Properties.(map[string]interface{}) - refProp, ok := props["toFirst"].([]interface{}) - require.True(t, ok) - - var refPayload []map[string]interface{} - var refVector []map[string]interface{} - for _, ref := range refProp { - parsed, ok := ref.(search.LocalRef) - require.True(t, ok) - refPayload = append(refPayload, map[string]interface{}{ - "description": parsed.Fields["description"], - }) - vector, ok := parsed.Fields["vector"].([]float32) - require.True(t, ok) - require.NotEmpty(t, vector) - refVector = append(refVector, map[string]interface{}{ - "vector": vector, - }) - } - - actual := manuallyResolveRef(t, obj, data, "toFirst", "description", nil) - assert.Equal(t, actual, refPayload) - actual = manuallyResolveRef(t, obj, data, "toFirst", "vector", node.repo) - assert.Equal(t, actual, refVector) - } - }) - - t.Run("ranked keyword search", func(t *testing.T) { - for i := 0; i < numberOfObjects; i++ { - description := fmt.Sprintf("object %d", i) - keywordRanking := &searchparams.KeywordRanking{ - Query: description, - Properties: []string{"description"}, - } - - params := dto.GetParams{ - ClassName: distributedClass, - KeywordRanking: keywordRanking, - Pagination: &filters.Pagination{Limit: 100}, - } - - node := nodes[rnd.Intn(len(nodes))] - res, err := node.repo.Search(context.Background(), params) - require.Nil(t, err) - require.NotEmpty(t, res) - - expected := strings.Join(strings.Split(description, " "), "-") - received := res[0].Object().Properties.(map[string]interface{})["description"] - assert.Equal(t, expected, received) - } - }) - - t.Run("aggregate count", func(t *testing.T) { - params := aggregation.Params{ - ClassName: schema.ClassName(distributedClass), - IncludeMetaCount: true, - } - - node := nodes[rnd.Intn(len(nodes))] - res, err := node.repo.Aggregate(context.Background(), params) - require.Nil(t, err) - - expectedResult := &aggregation.Result{ - Groups: []aggregation.Group{ - { - Count: numberOfObjects, - }, - }, - } - - assert.Equal(t, expectedResult, res) - }) - - t.Run("modify an object using patch", func(t *testing.T) { - obj := data[0] - - node := nodes[rnd.Intn(len(nodes))] - err := node.repo.Merge(context.Background(), objects.MergeDocument{ - Class: distributedClass, - ID: obj.ID, - PrimitiveSchema: map[string]interface{}{ - "other_property": "a-value-inserted-through-merge", - }, - }, nil, "") - - require.Nil(t, err) - }) - - t.Run("verify the patched object contains the additions and orig", func(t *testing.T) { - obj := data[0] - - node := nodes[rnd.Intn(len(nodes))] - res, err := node.repo.ObjectByID(context.Background(), obj.ID, search.SelectProperties{}, additional.Properties{}, "") - - require.Nil(t, err) - previousMap := obj.Properties.(map[string]interface{}) - assert.Equal(t, "a-value-inserted-through-merge", res.Object().Properties.(map[string]interface{})["other_property"]) - assert.Equal(t, previousMap["description"], res.Object().Properties.(map[string]interface{})["description"]) - }) - - // This test prevents a regression on - // https://github.com/weaviate/weaviate/issues/1775 - t.Run("query items by date filter with regular field", func(t *testing.T) { - count := len(data) / 2 // try to match half the data objects present - cutoff := time.Unix(0, 0).Add(time.Duration(count) * time.Hour) - node := nodes[rnd.Intn(len(nodes))] - res, err := node.repo.Search(context.Background(), dto.GetParams{ - Filters: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorLessThan, - On: &filters.Path{ - Class: distributedClass, - Property: schema.PropertyName("date_property"), - }, - Value: &filters.Value{ - Value: cutoff, - Type: schema.DataTypeDate, - }, - }, - }, - ClassName: distributedClass, - Pagination: &filters.Pagination{ - Limit: len(data), - }, - }) - - require.Nil(t, err) - assert.Equal(t, count, len(res)) - }) - - // This test prevents a regression on - // https://github.com/weaviate/weaviate/issues/1775 - t.Run("query items by date filter with array field", func(t *testing.T) { - count := len(data) / 2 // try to match half the data objects present - cutoff := time.Unix(0, 0).Add(time.Duration(count) * time.Hour) - node := nodes[rnd.Intn(len(nodes))] - res, err := node.repo.Search(context.Background(), dto.GetParams{ - Filters: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorLessThan, - On: &filters.Path{ - Class: distributedClass, - Property: schema.PropertyName("date_array_property"), - }, - Value: &filters.Value{ - Value: cutoff, - Type: schema.DataTypeDate, - }, - }, - }, - ClassName: distributedClass, - Pagination: &filters.Pagination{ - Limit: len(data), - }, - }) - - require.Nil(t, err) - assert.Equal(t, count, len(res)) - }) - - t.Run("sort by", func(t *testing.T) { - getPhoneNumber := func(a search.Result) *float64 { - prop := a.Object().Properties.(map[string]interface{})["phone_property"] - if phoneNumber, ok := prop.(*models.PhoneNumber); ok { - phoneStr := fmt.Sprintf("%v%v", phoneNumber.CountryCode, phoneNumber.National) - if phone, err := strconv.ParseFloat(phoneStr, 64); err == nil { - return &phone - } - } - return nil - } - getDate := func(a search.Result) *time.Time { - asString := a.Object().Properties.(map[string]interface{})["date_property"].(string) - if date, err := time.Parse(time.RFC3339, asString); err == nil { - return &date - } - return nil - } - testData := []struct { - name string - sort []filters.Sort - compareFn func(a, b search.Result) bool - }{ - { - name: "description asc", - sort: []filters.Sort{{Path: []string{"description"}, Order: "asc"}}, - compareFn: func(a, b search.Result) bool { - descriptionA := a.Object().Properties.(map[string]interface{})["description"].(string) - descriptionB := b.Object().Properties.(map[string]interface{})["description"].(string) - return strings.ToLower(descriptionA) <= strings.ToLower(descriptionB) - }, - }, - { - name: "description desc", - sort: []filters.Sort{{Path: []string{"description"}, Order: "desc"}}, - compareFn: func(a, b search.Result) bool { - descriptionA := a.Object().Properties.(map[string]interface{})["description"].(string) - descriptionB := b.Object().Properties.(map[string]interface{})["description"].(string) - return strings.ToLower(descriptionA) >= strings.ToLower(descriptionB) - }, - }, - { - name: "date_property asc", - sort: []filters.Sort{{Path: []string{"date_property"}, Order: "asc"}}, - compareFn: func(a, b search.Result) bool { - datePropA, datePropB := getDate(a), getDate(b) - if datePropA != nil && datePropB != nil { - return datePropA.Before(*datePropB) - } - return false - }, - }, - { - name: "date_property desc", - sort: []filters.Sort{{Path: []string{"date_property"}, Order: "desc"}}, - compareFn: func(a, b search.Result) bool { - datePropA, datePropB := getDate(a), getDate(b) - if datePropA != nil && datePropB != nil { - return datePropA.After(*datePropB) - } - return false - }, - }, - { - name: "int_property asc", - sort: []filters.Sort{{Path: []string{"int_property"}, Order: "asc"}}, - compareFn: func(a, b search.Result) bool { - intPropertyA := a.Object().Properties.(map[string]interface{})["int_property"].(float64) - intPropertyB := b.Object().Properties.(map[string]interface{})["int_property"].(float64) - return intPropertyA <= intPropertyB - }, - }, - { - name: "int_property desc", - sort: []filters.Sort{{Path: []string{"int_property"}, Order: "desc"}}, - compareFn: func(a, b search.Result) bool { - intPropertyA := a.Object().Properties.(map[string]interface{})["int_property"].(float64) - intPropertyB := b.Object().Properties.(map[string]interface{})["int_property"].(float64) - return intPropertyA >= intPropertyB - }, - }, - { - name: "phone_property asc", - sort: []filters.Sort{{Path: []string{"phone_property"}, Order: "asc"}}, - compareFn: func(a, b search.Result) bool { - phoneA, phoneB := getPhoneNumber(a), getPhoneNumber(b) - if phoneA != nil && phoneB != nil { - return *phoneA <= *phoneB - } - return false - }, - }, - { - name: "phone_property desc", - sort: []filters.Sort{{Path: []string{"phone_property"}, Order: "desc"}}, - compareFn: func(a, b search.Result) bool { - phoneA, phoneB := getPhoneNumber(a), getPhoneNumber(b) - if phoneA != nil && phoneB != nil { - return *phoneA >= *phoneB - } - return false - }, - }, - } - for _, td := range testData { - t.Run(td.name, func(t *testing.T) { - params := dto.GetParams{ - ClassName: distributedClass, - Sort: td.sort, - Pagination: &filters.Pagination{Limit: 100}, - } - - node := nodes[rnd.Intn(len(nodes))] - res, err := node.repo.Search(context.Background(), params) - require.Nil(t, err) - require.NotEmpty(t, res) - - if len(res) > 1 { - for i := 1; i < len(res); i++ { - assert.True(t, td.compareFn(res[i-1], res[i])) - } - } - }) - } - }) - - t.Run("node names by shard", func(t *testing.T) { - for _, n := range nodes { - nodeSet := make(map[string]bool) - foundNodes, err := n.repo.Shards(context.Background(), distributedClass) - assert.NoError(t, err) - for _, found := range foundNodes { - nodeSet[found] = true - } - assert.Len(t, nodeSet, numberOfNodes, "expected %d nodes, got %d", - numberOfNodes, len(foundNodes)) - } - }) - - t.Run("delete a third of the data from random nodes", func(t *testing.T) { - for i, obj := range data { - if i%3 != 0 { - // keep this item - continue - } - - node := nodes[rnd.Intn(len(nodes))] - err := node.repo.DeleteObject(context.Background(), distributedClass, obj.ID, nil, "") - require.Nil(t, err) - } - }) - - t.Run("make sure 2/3 exist, 1/3 no longer exists", func(t *testing.T) { - for i, obj := range data { - expected := true - if i%3 == 0 { - expected = false - } - - node := nodes[rnd.Intn(len(nodes))] - actual, err := node.repo.Exists(context.Background(), distributedClass, obj.ID, nil, "") - require.Nil(t, err) - assert.Equal(t, expected, actual) - } - }) - - t.Run("batch delete the remaining 2/3 of data", func(t *testing.T) { - getParams := func(className string, dryRun bool) objects.BatchDeleteParams { - return objects.BatchDeleteParams{ - ClassName: schema.ClassName(className), - Filters: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorLike, - Value: &filters.Value{ - Value: "*", - Type: schema.DataTypeText, - }, - On: &filters.Path{ - Property: "id", - }, - }, - }, - DryRun: dryRun, - Output: "verbose", - } - } - performClassSearch := func(repo *db.DB, className string) ([]search.Result, error) { - return repo.Search(context.Background(), dto.GetParams{ - ClassName: className, - Pagination: &filters.Pagination{Limit: 10000}, - }) - } - node := nodes[rnd.Intn(len(nodes))] - // get the initial count of the objects - res, err := performClassSearch(node.repo, distributedClass) - require.Nil(t, err) - beforeDelete := len(res) - require.True(t, beforeDelete > 0) - // dryRun == false, perform actual delete - batchDeleteRes, err := node.repo.BatchDeleteObjects(context.Background(), getParams(distributedClass, false), nil, "") - require.Nil(t, err) - require.Equal(t, int64(beforeDelete), batchDeleteRes.Matches) - require.Equal(t, beforeDelete, len(batchDeleteRes.Objects)) - for _, batchRes := range batchDeleteRes.Objects { - require.Nil(t, batchRes.Err) - } - // check that every object is deleted - res, err = performClassSearch(node.repo, distributedClass) - require.Nil(t, err) - require.Equal(t, 0, len(res)) - }) - - t.Run("shutdown", func(t *testing.T) { - for _, node := range nodes { - node.repo.Shutdown(context.Background()) - } - }) -} diff --git a/adapters/repos/db/clusterintegrationtest/doc.go b/adapters/repos/db/clusterintegrationtest/doc.go deleted file mode 100644 index 1ee601746f1e7538438aa37338c71233d0eff18a..0000000000000000000000000000000000000000 --- a/adapters/repos/db/clusterintegrationtest/doc.go +++ /dev/null @@ -1,17 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// clusterintegrationtest acts as a test package to provide a component test -// spanning multiple parts of the application, including everything that's -// required for a distributed setup. It thus acts like a mini "main" page and -// must be separated from the rest of the package to avoid circular import -// issues, etc. -package clusterintegrationtest diff --git a/adapters/repos/db/clusterintegrationtest/fakes_for_test.go b/adapters/repos/db/clusterintegrationtest/fakes_for_test.go deleted file mode 100644 index 8cd316049183e5dcb1b86704a1e81a2b821dfbcb..0000000000000000000000000000000000000000 --- a/adapters/repos/db/clusterintegrationtest/fakes_for_test.go +++ /dev/null @@ -1,394 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package clusterintegrationtest - -import ( - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "net/http/httptest" - "net/url" - "path" - "sync" - "time" - - "github.com/sirupsen/logrus/hooks/test" - "github.com/weaviate/weaviate/adapters/clients" - "github.com/weaviate/weaviate/adapters/handlers/rest/clusterapi" - "github.com/weaviate/weaviate/adapters/repos/db" - "github.com/weaviate/weaviate/entities/backup" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/schema" - modstgfs "github.com/weaviate/weaviate/modules/backup-filesystem" - ubak "github.com/weaviate/weaviate/usecases/backup" - "github.com/weaviate/weaviate/usecases/sharding" -) - -type node struct { - name string - repo *db.DB - schemaManager *fakeSchemaManager - backupManager *ubak.Handler - scheduler *ubak.Scheduler - migrator *db.Migrator - hostname string -} - -func (n *node) init(dirName string, shardStateRaw []byte, - allNodes *[]*node, -) { - localDir := path.Join(dirName, n.name) - logger, _ := test.NewNullLogger() - - nodeResolver := &nodeResolver{ - nodes: allNodes, - local: n.name, - } - - shardState, err := sharding.StateFromJSON(shardStateRaw, nodeResolver) - if err != nil { - panic(err) - } - - client := clients.NewRemoteIndex(&http.Client{}) - nodesClient := clients.NewRemoteNode(&http.Client{}) - replicaClient := clients.NewReplicationClient(&http.Client{}) - n.repo, err = db.New(logger, db.Config{ - MemtablesFlushIdleAfter: 60, - RootPath: localDir, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - }, client, nodeResolver, nodesClient, replicaClient, nil) - if err != nil { - panic(err) - } - n.schemaManager = &fakeSchemaManager{ - shardState: shardState, - schema: schema.Schema{Objects: &models.Schema{}}, - nodeResolver: nodeResolver, - } - - n.repo.SetSchemaGetter(n.schemaManager) - err = n.repo.WaitForStartup(context.Background()) - if err != nil { - panic(err) - } - - backendProvider := newFakeBackupBackendProvider(localDir) - n.backupManager = ubak.NewHandler( - logger, &fakeAuthorizer{}, n.schemaManager, n.repo, backendProvider) - - backupClient := clients.NewClusterBackups(&http.Client{}) - n.scheduler = ubak.NewScheduler( - &fakeAuthorizer{}, backupClient, n.repo, backendProvider, nodeResolver, logger) - - n.migrator = db.NewMigrator(n.repo, logger) - - indices := clusterapi.NewIndices(sharding.NewRemoteIndexIncoming(n.repo), n.repo, clusterapi.NewNoopAuthHandler()) - mux := http.NewServeMux() - mux.Handle("/indices/", indices.Indices()) - - backups := clusterapi.NewBackups(n.backupManager, clusterapi.NewNoopAuthHandler()) - mux.Handle("/backups/can-commit", backups.CanCommit()) - mux.Handle("/backups/commit", backups.Commit()) - mux.Handle("/backups/abort", backups.Abort()) - mux.Handle("/backups/status", backups.Status()) - - srv := httptest.NewServer(mux) - u, err := url.Parse(srv.URL) - if err != nil { - panic(err) - } - n.hostname = u.Host -} - -type fakeNodes struct { - nodes []string -} - -func (f fakeNodes) Candidates() []string { - return f.nodes -} - -func (f fakeNodes) LocalName() string { - return f.nodes[0] -} - -type fakeSchemaManager struct { - schema schema.Schema - shardState *sharding.State - nodeResolver *nodeResolver -} - -func (f *fakeSchemaManager) GetSchemaSkipAuth() schema.Schema { - return f.schema -} - -func (f *fakeSchemaManager) CopyShardingState(class string) *sharding.State { - return f.shardState -} - -func (f *fakeSchemaManager) ShardOwner(class, shard string) (string, error) { - ss := f.shardState - x, ok := ss.Physical[shard] - if !ok { - return "", fmt.Errorf("shard not found") - } - if len(x.BelongsToNodes) < 1 || x.BelongsToNodes[0] == "" { - return "", fmt.Errorf("owner node not found") - } - return ss.Physical[shard].BelongsToNodes[0], nil -} - -func (f *fakeSchemaManager) ShardReplicas(class, shard string) ([]string, error) { - ss := f.shardState - x, ok := ss.Physical[shard] - if !ok { - return nil, fmt.Errorf("shard not found") - } - return x.BelongsToNodes, nil -} - -func (f *fakeSchemaManager) TenantShard(class, tenant string) (string, string) { - return tenant, models.TenantActivityStatusHOT -} - -func (f *fakeSchemaManager) ShardFromUUID(class string, uuid []byte) string { - ss := f.shardState - return ss.Shard("", string(uuid)) -} - -func (f *fakeSchemaManager) RestoreClass(ctx context.Context, d *backup.ClassDescriptor, nodeMapping map[string]string) error { - return nil -} - -func (f *fakeSchemaManager) Nodes() []string { - return []string{"NOT SET"} -} - -func (f *fakeSchemaManager) NodeName() string { - return f.nodeResolver.local -} - -func (f *fakeSchemaManager) ClusterHealthScore() int { - return 0 -} - -func (f *fakeSchemaManager) ResolveParentNodes(_ string, shard string, -) (map[string]string, error) { - return nil, nil -} - -type nodeResolver struct { - nodes *[]*node - local string -} - -func (r nodeResolver) AllNames() []string { - panic("node resolving not implemented yet") -} - -func (r nodeResolver) Candidates() []string { - return nil -} - -func (r nodeResolver) LocalName() string { - return r.local -} - -func (r nodeResolver) NodeCount() int { - return len(*r.nodes) -} - -func (r nodeResolver) NodeHostname(nodeName string) (string, bool) { - for _, node := range *r.nodes { - if node.name == nodeName { - return node.hostname, true - } - } - - return "", false -} - -func newFakeBackupBackendProvider(backupsPath string) *fakeBackupBackendProvider { - return &fakeBackupBackendProvider{ - backupsPath: backupsPath, - } -} - -type fakeBackupBackendProvider struct { - backupsPath string -} - -func (f *fakeBackupBackendProvider) BackupBackend(name string) (modulecapabilities.BackupBackend, error) { - backend.setLocal(name == modstgfs.Name) - return backend, nil -} - -type fakeBackupBackend struct { - sync.Mutex - backupsPath string - backupID string - counter int - isLocal bool - startedAt time.Time -} - -func (f *fakeBackupBackend) HomeDir(backupID string) string { - f.Lock() - defer f.Unlock() - return f.backupsPath -} - -func (f *fakeBackupBackend) GetObject(ctx context.Context, backupID, key string) ([]byte, error) { - f.Lock() - defer f.Unlock() - - f.counter++ - - if f.counter <= 2 { - return nil, backup.ErrNotFound{} - } - - var resp interface{} - - if key == ubak.GlobalBackupFile { - resp = f.successGlobalMeta() - } else { - resp = f.successLocalMeta() - } - - b, _ := json.Marshal(resp) - return b, nil -} - -func (f *fakeBackupBackend) WriteToFile(ctx context.Context, backupID, key, destPath string) error { - f.Lock() - defer f.Unlock() - return nil -} - -func (f *fakeBackupBackend) Write(ctx context.Context, backupID, key string, r io.ReadCloser) (int64, error) { - f.Lock() - defer f.Unlock() - defer r.Close() - return 0, nil -} - -func (f *fakeBackupBackend) Read(ctx context.Context, backupID, key string, w io.WriteCloser) (int64, error) { - f.Lock() - defer f.Unlock() - defer w.Close() - return 0, nil -} - -func (f *fakeBackupBackend) SourceDataPath() string { - f.Lock() - defer f.Unlock() - return f.backupsPath -} - -func (f *fakeBackupBackend) setLocal(v bool) { - f.Lock() - defer f.Unlock() - f.isLocal = v -} - -func (f *fakeBackupBackend) IsExternal() bool { - f.Lock() - defer f.Unlock() - return !f.isLocal -} - -func (f *fakeBackupBackend) Name() string { - return "fakeBackupBackend" -} - -func (f *fakeBackupBackend) PutFile(ctx context.Context, backupID, key, srcPath string) error { - f.Lock() - defer f.Unlock() - return nil -} - -func (f *fakeBackupBackend) PutObject(ctx context.Context, backupID, key string, byes []byte) error { - f.Lock() - defer f.Unlock() - return nil -} - -func (f *fakeBackupBackend) Initialize(ctx context.Context, backupID string) error { - f.Lock() - defer f.Unlock() - return nil -} - -func (f *fakeBackupBackend) successGlobalMeta() backup.DistributedBackupDescriptor { - return backup.DistributedBackupDescriptor{ - StartedAt: f.startedAt, - ID: f.backupID, - Nodes: map[string]*backup.NodeDescriptor{ - "node-0": { - Classes: []string{distributedClass}, - Status: "SUCCESS", - }, - }, - Status: "SUCCESS", - Version: ubak.Version, - ServerVersion: "x.x.x", - } -} - -func (f *fakeBackupBackend) successLocalMeta() backup.BackupDescriptor { - return backup.BackupDescriptor{ - ID: f.backupID, - Status: "SUCCESS", - ServerVersion: "x.x.x", - Version: ubak.Version, - StartedAt: f.startedAt, - Classes: []backup.ClassDescriptor{ - { - Name: distributedClass, - Shards: []*backup.ShardDescriptor{ - { - Name: "123", - Node: "node-0", - Files: []string{"some-file.db"}, - DocIDCounter: []byte("1"), - DocIDCounterPath: ".", - Version: []byte("1"), - ShardVersionPath: ".", - PropLengthTracker: []byte("1"), - PropLengthTrackerPath: ".", - }, - }, - ShardingState: []byte("sharding state!"), - Schema: []byte("schema!"), - }, - }, - } -} - -func (f *fakeBackupBackend) reset() { - f.counter = 0 -} - -type fakeAuthorizer struct{} - -func (f *fakeAuthorizer) Authorize(_ *models.Principal, _, _ string) error { - return nil -} diff --git a/adapters/repos/db/clusterintegrationtest/helpers_for_test.go b/adapters/repos/db/clusterintegrationtest/helpers_for_test.go deleted file mode 100644 index 8a4d91f8749d333ad0a5eaeb0c7cdaeac5b5b4d4..0000000000000000000000000000000000000000 --- a/adapters/repos/db/clusterintegrationtest/helpers_for_test.go +++ /dev/null @@ -1,366 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package clusterintegrationtest - -import ( - "context" - "encoding/json" - "fmt" - "math" - "math/rand" - "sort" - "testing" - "time" - - "github.com/go-openapi/strfmt" - "github.com/google/uuid" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/adapters/repos/db" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/schema/crossref" - enthnsw "github.com/weaviate/weaviate/entities/vectorindex/hnsw" - "github.com/weaviate/weaviate/usecases/objects" - "github.com/weaviate/weaviate/usecases/sharding" -) - -func getRandomSeed() *rand.Rand { - return rand.New(rand.NewSource(time.Now().UnixNano())) -} - -func setupDirectory(t *testing.T) string { - dirName := t.TempDir() - return dirName -} - -func dataAsBatch(data []*models.Object) objects.BatchObjects { - batchObjs := make(objects.BatchObjects, len(data)) - for i := range data { - batchObjs[i] = objects.BatchObject{ - OriginalIndex: i, - Err: nil, - Object: data[i], - UUID: data[i].ID, - Vector: data[i].Vector, - } - } - - return batchObjs -} - -func dataAsBatchWithProps(data []*models.Object, props []string) objects.BatchObjects { - batchObjs := make(objects.BatchObjects, len(data)) - for i := range data { - batchObjs[i] = objects.BatchObject{ - OriginalIndex: i, - Err: nil, - Object: copyObjectWithProp(data[i], props), - UUID: data[i].ID, - Vector: data[i].Vector, - } - } - - return batchObjs -} - -// copyObjectWithProp is not a 100% copy. It may still contain the same -// pointers in some properties, it does however guarantee that it does not -// alter the existing input - this guarantee is lost, if you modify the output -func copyObjectWithProp(in *models.Object, propsToCopy []string) *models.Object { - out := &models.Object{} - - out.Additional = in.Additional - out.Class = in.Class - out.Vector = in.Vector - out.CreationTimeUnix = in.CreationTimeUnix - out.LastUpdateTimeUnix = in.LastUpdateTimeUnix - out.ID = in.ID - props := map[string]interface{}{} - - for _, propName := range propsToCopy { - props[propName] = in.Properties.(map[string]interface{})[propName] - } - - out.Properties = props - return out -} - -func multiShardState(nodeCount int) *sharding.State { - config, err := sharding.ParseConfig(map[string]interface{}{ - "desiredCount": json.Number(fmt.Sprintf("%d", nodeCount)), - }, 1) - if err != nil { - panic(err) - } - - nodeList := make([]string, nodeCount) - for i := range nodeList { - nodeList[i] = fmt.Sprintf("node-%d", i) - } - - s, err := sharding.InitState("multi-shard-test-index", config, - fakeNodes{nodeList}, 1, false) - if err != nil { - panic(err) - } - - return s -} - -func class() *models.Class { - cfg := enthnsw.NewDefaultUserConfig() - cfg.EF = 500 - return &models.Class{ - Class: distributedClass, - VectorIndexConfig: cfg, - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{ - { - Name: "description", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWord, - }, - { - Name: "other_property", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWord, - }, - { - Name: "date_property", - DataType: schema.DataTypeDate.PropString(), - }, - { - Name: "date_array_property", - DataType: schema.DataTypeDateArray.PropString(), - }, - { - Name: "int_property", - DataType: schema.DataTypeInt.PropString(), - }, - { - Name: "phone_property", - DataType: schema.DataTypePhoneNumber.PropString(), - }, - }, - } -} - -func secondClassWithRef() *models.Class { - cfg := enthnsw.NewDefaultUserConfig() - cfg.EF = 500 - return &models.Class{ - Class: "SecondDistributed", - VectorIndexConfig: cfg, - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{ - { - Name: "description", - DataType: []string{string(schema.DataTypeText)}, - }, - { - Name: "toFirst", - DataType: []string{distributedClass}, - }, - }, - } -} - -func invertedConfig() *models.InvertedIndexConfig { - return &models.InvertedIndexConfig{ - CleanupIntervalSeconds: 60, - } -} - -func exampleData(size int) []*models.Object { - out := make([]*models.Object, size) - - for i := range out { - vec := make([]float32, vectorDims) - for i := range vec { - vec[i] = rand.Float32() - } - - timestamp := time.Unix(0, 0).Add(time.Duration(i) * time.Hour) - phoneNumber := uint64(1000000 + rand.Intn(10000)) - - out[i] = &models.Object{ - Class: distributedClass, - ID: strfmt.UUID(uuid.New().String()), - Properties: map[string]interface{}{ - "description": fmt.Sprintf("object-%d", i), - "date_property": timestamp, - "date_array_property": []interface{}{timestamp}, - "int_property": rand.Intn(1000), - "phone_property": &models.PhoneNumber{ - CountryCode: 49, - DefaultCountry: "DE", - Input: fmt.Sprintf("0171 %d", phoneNumber), - Valid: true, - InternationalFormatted: fmt.Sprintf("+49 171 %d", phoneNumber), - National: phoneNumber, - NationalFormatted: fmt.Sprintf("0171 %d", phoneNumber), - }, - }, - Vector: vec, - } - } - - return out -} - -func exampleDataWithRefs(size int, refCount int, targetObjs []*models.Object) []*models.Object { - out := make([]*models.Object, size) - - for i := range out { - vec := make([]float32, vectorDims) - for i := range vec { - vec[i] = rand.Float32() - } - - refs := make(models.MultipleRef, refCount) - for i := range refs { - randomTarget := targetObjs[rand.Intn(len(targetObjs))] - refs[i] = crossref.New("localhost", distributedClass, randomTarget.ID).SingleRef() - } - - out[i] = &models.Object{ - Class: "SecondDistributed", - ID: strfmt.UUID(uuid.New().String()), - Properties: map[string]interface{}{ - "description": fmt.Sprintf("second-object-%d", i), - "toFirst": refs, - }, - Vector: vec, - } - } - - return out -} - -func bruteForceObjectsByQuery(objs []*models.Object, - query []float32, -) []*models.Object { - type distanceAndObj struct { - distance float32 - obj *models.Object - } - - distProv := distancer.NewCosineDistanceProvider() - distances := make([]distanceAndObj, len(objs)) - - for i := range objs { - dist, _, _ := distProv.SingleDist(normalize(query), normalize(objs[i].Vector)) - distances[i] = distanceAndObj{ - distance: dist, - obj: objs[i], - } - } - - sort.Slice(distances, func(a, b int) bool { - return distances[a].distance < distances[b].distance - }) - - out := make([]*models.Object, len(objs)) - for i := range out { - out[i] = distances[i].obj - } - - return out -} - -func normalize(v []float32) []float32 { - var norm float32 - for i := range v { - norm += v[i] * v[i] - } - - norm = float32(math.Sqrt(float64(norm))) - for i := range v { - v[i] = v[i] / norm - } - - return v -} - -func manuallyResolveRef(t *testing.T, obj *models.Object, - possibleTargets []*models.Object, localPropName, - referencedPropName string, - repo *db.DB, -) []map[string]interface{} { - beacons := obj.Properties.(map[string]interface{})[localPropName].(models.MultipleRef) - out := make([]map[string]interface{}, len(beacons)) - - for i, ref := range beacons { - parsed, err := crossref.Parse(ref.Beacon.String()) - require.Nil(t, err) - target := findId(possibleTargets, parsed.TargetID) - require.NotNil(t, target, "target not found") - if referencedPropName == "vector" { - // find referenced object to get his actual vector from DB - require.NotNil(t, repo) - res, err := repo.Object(context.Background(), parsed.Class, parsed.TargetID, - nil, additional.Properties{Vector: true}, nil, "") - require.Nil(t, err) - require.NotNil(t, res) - out[i] = map[string]interface{}{ - referencedPropName: res.Vector, - } - } else { - out[i] = map[string]interface{}{ - referencedPropName: target.Properties.(map[string]interface{})[referencedPropName], - } - } - } - - return out -} - -func findId(list []*models.Object, id strfmt.UUID) *models.Object { - for _, obj := range list { - if obj.ID == id { - return obj - } - } - - return nil -} - -func refsAsBatch(in []*models.Object, propName string) objects.BatchReferences { - out := objects.BatchReferences{} - - originalIndex := 0 - for _, obj := range in { - beacons := obj.Properties.(map[string]interface{})[propName].(models.MultipleRef) - current := make(objects.BatchReferences, len(beacons)) - for i, beacon := range beacons { - to, err := crossref.Parse(beacon.Beacon.String()) - if err != nil { - panic(err) - } - current[i] = objects.BatchReference{ - OriginalIndex: originalIndex, - To: to, - From: crossref.NewSource(schema.ClassName(obj.Class), - schema.PropertyName(propName), obj.ID), - } - originalIndex++ - } - out = append(out, current...) - } - - return out -} diff --git a/adapters/repos/db/crud.go b/adapters/repos/db/crud.go deleted file mode 100644 index c902640468a69d695908fad4c7d7743690313ea8..0000000000000000000000000000000000000000 --- a/adapters/repos/db/crud.go +++ /dev/null @@ -1,271 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "context" - "fmt" - "time" - - "github.com/go-openapi/strfmt" - "github.com/pkg/errors" - "github.com/weaviate/weaviate/adapters/repos/db/refcache" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/multi" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/schema/crossref" - "github.com/weaviate/weaviate/entities/search" - "github.com/weaviate/weaviate/entities/storobj" - "github.com/weaviate/weaviate/usecases/objects" -) - -func (db *DB) PutObject(ctx context.Context, obj *models.Object, - vector []float32, repl *additional.ReplicationProperties, -) error { - object := storobj.FromObject(obj, vector) - idx := db.GetIndex(object.Class()) - if idx == nil { - return fmt.Errorf("import into non-existing index for %s", object.Class()) - } - - if err := idx.putObject(ctx, object, repl); err != nil { - return fmt.Errorf("import into index %s: %w", idx.ID(), err) - } - - return nil -} - -// DeleteObject from of a specific class giving its ID -func (db *DB) DeleteObject(ctx context.Context, class string, id strfmt.UUID, - repl *additional.ReplicationProperties, tenant string, -) error { - idx := db.GetIndex(schema.ClassName(class)) - if idx == nil { - return fmt.Errorf("delete from non-existing index for %s", class) - } - - err := idx.deleteObject(ctx, id, repl, tenant) - if err != nil { - return fmt.Errorf("delete from index %q: %w", idx.ID(), err) - } - - return nil -} - -func (db *DB) MultiGet(ctx context.Context, query []multi.Identifier, - additional additional.Properties, tenant string, -) ([]search.Result, error) { - byIndex := map[string][]multi.Identifier{} - db.indexLock.RLock() - defer db.indexLock.RUnlock() - - for i, q := range query { - // store original position to make assembly easier later - q.OriginalPosition = i - - for _, index := range db.indices { - if index.Config.ClassName != schema.ClassName(q.ClassName) { - continue - } - - queue := byIndex[index.ID()] - queue = append(queue, q) - byIndex[index.ID()] = queue - } - } - - out := make(search.Results, len(query)) - for indexID, queries := range byIndex { - indexRes, err := db.indices[indexID].multiObjectByID(ctx, queries, tenant) - if err != nil { - return nil, errors.Wrapf(err, "index %q", indexID) - } - - for i, obj := range indexRes { - if obj == nil { - continue - } - res := obj.SearchResult(additional, tenant) - out[queries[i].OriginalPosition] = *res - } - } - - return out, nil -} - -// ObjectByID checks every index of the particular kind for the ID -// -// @warning: this function is deprecated by Object() -func (db *DB) ObjectByID(ctx context.Context, id strfmt.UUID, - props search.SelectProperties, additional additional.Properties, - tenant string, -) (*search.Result, error) { - results, err := db.ObjectsByID(ctx, id, props, additional, tenant) - if err != nil { - return nil, err - } - if len(results) == 0 { - return nil, nil - } - return &results[0], nil -} - -// ObjectsByID checks every index of the particular kind for the ID -// this method is only used for Explore queries where we don't have -// a class context -func (db *DB) ObjectsByID(ctx context.Context, id strfmt.UUID, - props search.SelectProperties, additional additional.Properties, - tenant string, -) (search.Results, error) { - var result []*storobj.Object - // TODO: Search in parallel, rather than sequentially or this will be - // painfully slow on large schemas - db.indexLock.RLock() - - for _, index := range db.indices { - res, err := index.objectByID(ctx, id, props, additional, nil, tenant) - if err != nil { - db.indexLock.RUnlock() - switch err.(type) { - case objects.ErrMultiTenancy: - return nil, objects.NewErrMultiTenancy(fmt.Errorf("search index %s: %w", index.ID(), err)) - default: - return nil, errors.Wrapf(err, "search index %s", index.ID()) - } - } - - if res != nil { - result = append(result, res) - } - } - db.indexLock.RUnlock() - - if result == nil { - return nil, nil - } - - return db.ResolveReferences(ctx, - storobj.SearchResults(result, additional, tenant), props, nil, additional, tenant) -} - -// Object gets object with id from index of specified class. -func (db *DB) Object(ctx context.Context, class string, id strfmt.UUID, - props search.SelectProperties, addl additional.Properties, - repl *additional.ReplicationProperties, tenant string, -) (*search.Result, error) { - idx := db.GetIndex(schema.ClassName(class)) - if idx == nil { - return nil, nil - } - - obj, err := idx.objectByID(ctx, id, props, addl, repl, tenant) - if err != nil { - switch err.(type) { - case objects.ErrMultiTenancy: - return nil, objects.NewErrMultiTenancy(fmt.Errorf("search index %s: %w", idx.ID(), err)) - default: - return nil, errors.Wrapf(err, "search index %s", idx.ID()) - } - } - var r *search.Result - if obj != nil { - r = obj.SearchResult(addl, tenant) - } - if r == nil { - return nil, nil - } - return db.enrichRefsForSingle(ctx, r, props, addl, tenant) -} - -func (db *DB) enrichRefsForSingle(ctx context.Context, obj *search.Result, - props search.SelectProperties, additional additional.Properties, tenant string, -) (*search.Result, error) { - res, err := refcache.NewResolver(refcache.NewCacher(db, db.logger, tenant)). - Do(ctx, []search.Result{*obj}, props, additional) - if err != nil { - return nil, errors.Wrap(err, "resolve cross-refs") - } - - return &res[0], nil -} - -func (db *DB) Exists(ctx context.Context, class string, id strfmt.UUID, - repl *additional.ReplicationProperties, tenant string, -) (bool, error) { - if class == "" { - return db.anyExists(ctx, id, repl) - } - index := db.GetIndex(schema.ClassName(class)) - if index == nil { - return false, nil - } - return index.exists(ctx, id, repl, tenant) -} - -func (db *DB) anyExists(ctx context.Context, id strfmt.UUID, - repl *additional.ReplicationProperties, -) (bool, error) { - // TODO: Search in parallel, rather than sequentially or this will be - // painfully slow on large schemas - db.indexLock.RLock() - defer db.indexLock.RUnlock() - - for _, index := range db.indices { - ok, err := index.exists(ctx, id, repl, "") - if err != nil { - switch err.(type) { - case objects.ErrMultiTenancy: - return false, objects.NewErrMultiTenancy(fmt.Errorf("search index %s: %w", index.ID(), err)) - default: - return false, errors.Wrapf(err, "search index %s", index.ID()) - } - } - if ok { - return true, nil - } - } - - return false, nil -} - -func (db *DB) AddReference(ctx context.Context, source *crossref.RefSource, target *crossref.Ref, - repl *additional.ReplicationProperties, tenant string, -) error { - return db.Merge(ctx, objects.MergeDocument{ - Class: source.Class.String(), - ID: source.TargetID, - UpdateTime: time.Now().UnixMilli(), - References: objects.BatchReferences{ - objects.BatchReference{ - From: source, - To: target, - }, - }, - }, repl, tenant) -} - -func (db *DB) Merge(ctx context.Context, merge objects.MergeDocument, - repl *additional.ReplicationProperties, tenant string, -) error { - idx := db.GetIndex(schema.ClassName(merge.Class)) - if idx == nil { - return fmt.Errorf("merge from non-existing index for %s", merge.Class) - } - - err := idx.mergeObject(ctx, merge, repl, tenant) - if err != nil { - return errors.Wrapf(err, "merge into index %s", idx.ID()) - } - - return nil -} diff --git a/adapters/repos/db/crud_deletion_integration_test.go b/adapters/repos/db/crud_deletion_integration_test.go deleted file mode 100644 index 32c60e7ae63fa9fa76c960a1ac29e2a745536313..0000000000000000000000000000000000000000 --- a/adapters/repos/db/crud_deletion_integration_test.go +++ /dev/null @@ -1,241 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package db - -import ( - "context" - "testing" - - "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/dto" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - libschema "github.com/weaviate/weaviate/entities/schema" -) - -func TestDeleteJourney(t *testing.T) { - dirName := t.TempDir() - - logger := logrus.New() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - defer repo.Shutdown(context.Background()) - migrator := NewMigrator(repo, logger) - - schema := libschema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{updateTestClass()}, - }, - } - - t.Run("add schema", func(t *testing.T) { - err := migrator.AddClass(context.Background(), updateTestClass(), - schemaGetter.CopyShardingState(updateTestClass().Class)) - require.Nil(t, err) - }) - schemaGetter.schema = schema - - t.Run("import some objects", func(t *testing.T) { - for _, res := range updateTestData() { - err := repo.PutObject(context.Background(), res.Object(), res.Vector, nil) - require.Nil(t, err) - } - }) - - searchVector := []float32{0.1, 0.1, 0.1} - - t.Run("verify vector search results are initially as expected", - func(t *testing.T) { - res, err := repo.VectorSearch(context.Background(), dto.GetParams{ - ClassName: "UpdateTestClass", - SearchVector: searchVector, - Pagination: &filters.Pagination{ - Limit: 100, - }, - }) - - expectedOrder := []interface{}{ - "element-0", "element-2", "element-3", "element-1", - } - - require.Nil(t, err) - require.Len(t, res, 4) - assert.Equal(t, expectedOrder, extractPropValues(res, "name")) - }) - - searchInv := func(t *testing.T, op filters.Operator, value int) []interface{} { - res, err := repo.ObjectSearch(context.Background(), 0, 100, - &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: op, - On: &filters.Path{ - Class: "UpdateTestClass", - Property: libschema.PropertyName("intProp"), - }, - Value: &filters.Value{ - Type: libschema.DataTypeInt, - Value: value, - }, - }, - }, nil, additional.Properties{}, "") - require.Nil(t, err) - return extractPropValues(res, "name") - } - - t.Run("verify invert index results are initially as expected", - func(t *testing.T) { - expectedOrder := []interface{}{ - "element-0", "element-1", "element-2", "element-3", - } - assert.Equal(t, expectedOrder, searchInv(t, filters.OperatorGreaterThanEqual, 0)) - - expectedOrder = []interface{}{"element-0"} - assert.Equal(t, expectedOrder, searchInv(t, filters.OperatorEqual, 0)) - - expectedOrder = []interface{}{"element-1"} - assert.Equal(t, expectedOrder, searchInv(t, filters.OperatorEqual, 10)) - - expectedOrder = []interface{}{"element-2"} - assert.Equal(t, expectedOrder, searchInv(t, filters.OperatorEqual, 20)) - - expectedOrder = []interface{}{"element-3"} - assert.Equal(t, expectedOrder, searchInv(t, filters.OperatorEqual, 30)) - }) - - t.Run("delete element-0", - func(t *testing.T) { - id := updateTestData()[0].ID - - err := repo.DeleteObject(context.Background(), "UpdateTestClass", id, nil, "") - require.Nil(t, err) - }) - - t.Run("verify new vector search results are as expected", func(t *testing.T) { - res, err := repo.VectorSearch(context.Background(), dto.GetParams{ - ClassName: "UpdateTestClass", - SearchVector: searchVector, - Pagination: &filters.Pagination{ - Limit: 100, - }, - }) - - expectedOrder := []interface{}{ - "element-2", "element-3", "element-1", - } - - require.Nil(t, err) - require.Len(t, res, 3) - assert.Equal(t, expectedOrder, extractPropValues(res, "name")) - }) - - t.Run("verify invert results still work properly", func(t *testing.T) { - expectedOrder := []interface{}{ - "element-1", "element-2", "element-3", - } - assert.Equal(t, expectedOrder, searchInv(t, filters.OperatorGreaterThanEqual, 0)) - - expectedOrder = []interface{}{"element-1"} - assert.Equal(t, expectedOrder, searchInv(t, filters.OperatorEqual, 10)) - - expectedOrder = []interface{}{"element-2"} - assert.Equal(t, expectedOrder, searchInv(t, filters.OperatorEqual, 20)) - - expectedOrder = []interface{}{"element-3"} - assert.Equal(t, expectedOrder, searchInv(t, filters.OperatorEqual, 30)) - }) - - t.Run("delete element-1", - func(t *testing.T) { - id := updateTestData()[1].ID - - err := repo.DeleteObject(context.Background(), "UpdateTestClass", id, nil, "") - require.Nil(t, err) - }) - - t.Run("verify new vector search results are as expected", func(t *testing.T) { - res, err := repo.VectorSearch(context.Background(), dto.GetParams{ - ClassName: "UpdateTestClass", - SearchVector: searchVector, - Pagination: &filters.Pagination{ - Limit: 100, - }, - }) - - expectedOrder := []interface{}{ - "element-2", "element-3", - } - - require.Nil(t, err) - require.Len(t, res, 2) - assert.Equal(t, expectedOrder, extractPropValues(res, "name")) - }) - - t.Run("verify invert results have been updated correctly", func(t *testing.T) { - expectedOrder := []interface{}{ - "element-2", "element-3", - } - assert.Equal(t, expectedOrder, searchInv(t, filters.OperatorGreaterThanEqual, 0)) - - expectedOrder = []interface{}{"element-2"} - assert.Equal(t, expectedOrder, searchInv(t, filters.OperatorEqual, 20)) - - expectedOrder = []interface{}{"element-3"} - assert.Equal(t, expectedOrder, searchInv(t, filters.OperatorEqual, 30)) - }) - - t.Run("delete the index", func(t *testing.T) { - res, err := repo.VectorSearch(context.Background(), dto.GetParams{ - ClassName: "UpdateTestClass", - SearchVector: searchVector, - Pagination: &filters.Pagination{ - Limit: 100, - }, - }) - - expectedOrder := []interface{}{ - "element-2", "element-3", - } - - require.Nil(t, err) - require.Len(t, res, 2) - assert.Equal(t, expectedOrder, extractPropValues(res, "name")) - - id := updateTestData()[2].ID - - err = repo.DeleteObject(context.Background(), "UpdateTestClass", id, nil, "") - require.Nil(t, err) - - index := repo.GetIndex("UpdateTestClass") - require.NotNil(t, index) - - err = repo.DeleteIndex("UpdateTestClass") - assert.Nil(t, err) - }) -} diff --git a/adapters/repos/db/crud_integration_test.go b/adapters/repos/db/crud_integration_test.go deleted file mode 100644 index 21c20bc83661c258b108410cfa2371cbd2ca17f5..0000000000000000000000000000000000000000 --- a/adapters/repos/db/crud_integration_test.go +++ /dev/null @@ -1,2592 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest - -package db - -import ( - "context" - "fmt" - "math/rand" - "testing" - "time" - - "github.com/go-openapi/strfmt" - "github.com/google/uuid" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/dto" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/multi" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/schema/crossref" - "github.com/weaviate/weaviate/entities/search" - "github.com/weaviate/weaviate/entities/searchparams" - enthnsw "github.com/weaviate/weaviate/entities/vectorindex/hnsw" - "github.com/weaviate/weaviate/usecases/objects" - "github.com/weaviate/weaviate/usecases/replica" -) - -func TestCRUD(t *testing.T) { - dirName := t.TempDir() - - logger, _ := test.NewNullLogger() - thingclass := &models.Class{ - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Class: "TheBestThingClass", - Properties: []*models.Property{ - { - Name: "stringProp", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "location", - DataType: []string{string(schema.DataTypeGeoCoordinates)}, - }, - { - Name: "phone", - DataType: []string{string(schema.DataTypePhoneNumber)}, - }, - }, - } - actionclass := &models.Class{ - Class: "TheBestActionClass", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{ - { - Name: "stringProp", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "refProp", - DataType: []string{"TheBestThingClass"}, - }, - { - Name: "phone", - DataType: []string{string(schema.DataTypePhoneNumber)}, - }, - }, - } - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 10, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - defer repo.Shutdown(context.Background()) - migrator := NewMigrator(repo, logger) - - t.Run("creating the thing class", func(t *testing.T) { - require.Nil(t, - migrator.AddClass(context.Background(), thingclass, schemaGetter.shardState)) - }) - - t.Run("creating the action class", func(t *testing.T) { - require.Nil(t, - migrator.AddClass(context.Background(), actionclass, schemaGetter.shardState)) - }) - - // update schema getter so it's in sync with class - schemaGetter.schema = schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{actionclass, thingclass}, - }, - } - - thingID := strfmt.UUID("a0b55b05-bc5b-4cc9-b646-1452d1390a62") - - t.Run("validating that the thing doesn't exist prior", func(t *testing.T) { - ok, err := repo.Exists(context.Background(), "TheBestThingClass", thingID, nil, "") - require.Nil(t, err) - assert.False(t, ok) - }) - - t.Run("adding a thing", func(t *testing.T) { - thing := &models.Object{ - CreationTimeUnix: 1565612833955, - LastUpdateTimeUnix: 1000001, - ID: thingID, - Class: "TheBestThingClass", - Properties: map[string]interface{}{ - "stringProp": "some value", - "phone": &models.PhoneNumber{ - CountryCode: 49, - DefaultCountry: "DE", - Input: "0171 1234567", - Valid: true, - InternationalFormatted: "+49 171 1234567", - National: 1234567, - NationalFormatted: "0171 1234567", - }, - "location": &models.GeoCoordinates{ - Latitude: ptFloat32(1), - Longitude: ptFloat32(2), - }, - }, - Additional: models.AdditionalProperties{ - "interpretation": map[string]interface{}{ - "source": []interface{}{ - map[string]interface{}{ - "concept": "some", - "occurrence": float64(1), - "weight": float64(1), - }, - map[string]interface{}{ - "concept": "value", - "occurrence": float64(1), - "weight": float64(1), - }, - }, - }, - }, - } - vector := []float32{1, 3, 5, 0.4} - - err := repo.PutObject(context.Background(), thing, vector, nil) - - assert.Nil(t, err) - }) - - t.Run("validating that the thing exists now", func(t *testing.T) { - ok, err := repo.Exists(context.Background(), "TheBestThingClass", thingID, nil, "") - require.Nil(t, err) - assert.True(t, ok) - }) - - t.Run("trying to add a thing to a non-existing class", func(t *testing.T) { - thing := &models.Object{ - CreationTimeUnix: 1565612833955, - LastUpdateTimeUnix: 1000001, - ID: thingID, - Class: "WrongClass", - Properties: map[string]interface{}{ - "stringProp": "some value", - }, - } - vector := []float32{1, 3, 5, 0.4} - - err := repo.PutObject(context.Background(), thing, vector, nil) - assert.Equal(t, - fmt.Errorf("import into non-existing index for WrongClass"), err) - }) - - timeMust := func(t strfmt.DateTime, err error) strfmt.DateTime { - if err != nil { - panic(err) - } - - return t - } - - t.Run("updating the thing", func(t *testing.T) { - thing := &models.Object{ - CreationTimeUnix: 1565612833955, - LastUpdateTimeUnix: 10000020, - ID: thingID, - Class: "TheBestThingClass", - Properties: map[string]interface{}{ - "stringProp": "updated value", - "phone": &models.PhoneNumber{ - CountryCode: 49, - DefaultCountry: "DE", - Input: "0171 1234567", - Valid: true, - InternationalFormatted: "+49 171 1234567", - National: 1234567, - NationalFormatted: "0171 1234567", - }, - "location": &models.GeoCoordinates{ - Latitude: ptFloat32(1), - Longitude: ptFloat32(2), - }, - }, - } - vector := []float32{1, 3, 5, 0.4} - - err := repo.PutObject(context.Background(), thing, vector, nil) - assert.Nil(t, err) - }) - - t.Run("validating the updates are reflected", func(t *testing.T) { - expected := &models.Object{ - CreationTimeUnix: 1565612833955, - LastUpdateTimeUnix: 10000020, - ID: thingID, - Class: "TheBestThingClass", - VectorWeights: map[string]string(nil), - Properties: map[string]interface{}{ - "stringProp": "updated value", - "phone": &models.PhoneNumber{ - CountryCode: 49, - DefaultCountry: "DE", - Input: "0171 1234567", - Valid: true, - InternationalFormatted: "+49 171 1234567", - National: 1234567, - NationalFormatted: "0171 1234567", - }, - "location": &models.GeoCoordinates{ - Latitude: ptFloat32(1), - Longitude: ptFloat32(2), - }, - }, - Additional: models.AdditionalProperties{}, - } - res, err := repo.ObjectByID(context.Background(), thingID, nil, additional.Properties{}, "") - require.Nil(t, err) - assert.Equal(t, expected, res.ObjectWithVector(false)) - - res, err = repo.Object(context.Background(), expected.Class, thingID, nil, - additional.Properties{}, nil, "") - require.Nil(t, err) - assert.Equal(t, expected, res.ObjectWithVector(false)) - }) - - t.Run("finding the updated object by querying for an updated value", - func(t *testing.T) { - // This is to verify the inverted index was updated correctly - res, err := repo.Search(context.Background(), dto.GetParams{ - ClassName: "TheBestThingClass", - Pagination: &filters.Pagination{Limit: 10}, - Filters: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: "TheBestThingClass", - Property: "stringProp", - }, - Value: &filters.Value{ - // we would not have found this object before using "updated", as - // this string was only introduced as part of the update - Value: "updated", - Type: schema.DataTypeText, - }, - }, - }, - }) - require.Nil(t, err) - require.Len(t, res, 1) - assert.Equal(t, thingID, res[0].ID) - }) - - t.Run("NOT finding the previous version by querying for an outdated value", - func(t *testing.T) { - // This is to verify the inverted index was cleaned up correctly - res, err := repo.Search(context.Background(), dto.GetParams{ - ClassName: "TheBestThingClass", - Pagination: &filters.Pagination{Limit: 10}, - Filters: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: "TheBestThingClass", - Property: "stringProp", - }, - Value: &filters.Value{ - Value: "some", - Type: schema.DataTypeText, - }, - }, - }, - }) - require.Nil(t, err) - require.Len(t, res, 0) - }) - - t.Run("still finding it for an unchanged term", - func(t *testing.T) { - // This is to verify that while we're adding new links and cleaning up - // old ones, we don't actually touch those that were present and still - // should be - res, err := repo.Search(context.Background(), dto.GetParams{ - ClassName: "TheBestThingClass", - Pagination: &filters.Pagination{Limit: 10}, - Filters: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: "TheBestThingClass", - Property: "stringProp", - }, - Value: &filters.Value{ - // we would not have found this object before using "updated", as - // this string was only introduced as part of the update - Value: "value", - Type: schema.DataTypeText, - }, - }, - }, - }) - require.Nil(t, err) - require.Len(t, res, 1) - assert.Equal(t, thingID, res[0].ID) - }) - - t.Run("updating the thing back to its original value", func(t *testing.T) { - thing := &models.Object{ - CreationTimeUnix: 1565612833955, - LastUpdateTimeUnix: 1000001, - ID: thingID, - Class: "TheBestThingClass", - Properties: map[string]interface{}{ - "stringProp": "some value", - "phone": &models.PhoneNumber{ - CountryCode: 49, - DefaultCountry: "DE", - Input: "0171 1234567", - Valid: true, - InternationalFormatted: "+49 171 1234567", - National: 1234567, - NationalFormatted: "0171 1234567", - }, - "location": &models.GeoCoordinates{ - Latitude: ptFloat32(1), - Longitude: ptFloat32(2), - }, - }, - } - vector := []float32{1, 3, 5, 0.4} - - err := repo.PutObject(context.Background(), thing, vector, nil) - assert.Nil(t, err) - }) - - actionID := strfmt.UUID("022ca5ba-7c0b-4a78-85bf-26346bbcfae7") - t.Run("adding an action", func(t *testing.T) { - action := &models.Object{ - CreationTimeUnix: 1000002, - LastUpdateTimeUnix: 1000003, - ID: actionID, - Class: "TheBestActionClass", - Properties: map[string]interface{}{ - "stringProp": "some act-citing value", - "refProp": models.MultipleRef{ - &models.SingleRef{ - Classification: &models.ReferenceMetaClassification{ - LosingDistance: ptFloat64(0.7), - MeanLosingDistance: ptFloat64(0.7), - ClosestLosingDistance: ptFloat64(0.65), - WinningDistance: 0.3, - MeanWinningDistance: 0.3, - ClosestWinningDistance: 0.25, - ClosestOverallDistance: 0.25, - OverallCount: 3, - WinningCount: 2, - LosingCount: 1, - }, - Beacon: strfmt.URI( - crossref.NewLocalhost("", thingID).String()), - }, - }, - }, - Additional: models.AdditionalProperties{ - "classification": &additional.Classification{ - ID: "foo", - Scope: []string{"scope1", "scope2"}, - ClassifiedFields: []string{"field1", "field2"}, - Completed: timeMust(strfmt.ParseDateTime("2006-01-02T15:04:05.000Z")), - }, - }, - } - vector := []float32{3, 1, 0.3, 12} - - err := repo.PutObject(context.Background(), action, vector, nil) - - assert.Nil(t, err) - }) - - t.Run("searching by vector", func(t *testing.T) { - // the search vector is designed to be very close to the action, but - // somewhat far from the thing. So it should match the action closer - searchVector := []float32{2.9, 1.1, 0.5, 8.01} - - res, err := repo.CrossClassVectorSearch(context.Background(), searchVector, 0, 10, nil) - - require.Nil(t, err) - require.Equal(t, true, len(res) >= 2) - assert.Equal(t, actionID, res[0].ID) - assert.Equal(t, "TheBestActionClass", res[0].ClassName) - assert.Equal(t, "TheBestActionClass", res[0].ClassName) - assert.Equal(t, int64(1000002), res[0].Created) - assert.Equal(t, int64(1000003), res[0].Updated) - assert.Equal(t, thingID, res[1].ID) - - assert.Equal(t, "TheBestThingClass", res[1].ClassName) - assert.Equal(t, int64(1565612833955), res[1].Created) - assert.Equal(t, int64(1000001), res[1].Updated) - }) - - t.Run("searching by vector for a single class", func(t *testing.T) { - // the search vector is designed to be very close to the action, but - // somewhat far from the thing. So it should match the action closer - searchVector := []float32{2.9, 1.1, 0.5, 8.01} - - params := dto.GetParams{ - SearchVector: searchVector, - ClassName: "TheBestThingClass", - Pagination: &filters.Pagination{Limit: 10}, - Filters: nil, - } - res, err := repo.VectorSearch(context.Background(), params) - - require.Nil(t, err) - require.Len(t, res, 1, "got exactly one result") - assert.Equal(t, thingID, res[0].ID, "extracted the ID") - assert.Equal(t, "TheBestThingClass", res[0].ClassName, "matches the class name") - schema := res[0].Schema.(map[string]interface{}) - assert.Equal(t, "some value", schema["stringProp"], "has correct string prop") - assert.Equal(t, &models.GeoCoordinates{ptFloat32(1), ptFloat32(2)}, schema["location"], "has correct geo prop") - assert.Equal(t, &models.PhoneNumber{ - CountryCode: 49, - DefaultCountry: "DE", - Input: "0171 1234567", - Valid: true, - InternationalFormatted: "+49 171 1234567", - National: 1234567, - NationalFormatted: "0171 1234567", - }, schema["phone"], "has correct phone prop") - assert.Equal(t, models.AdditionalProperties{}, res[0].AdditionalProperties, "no meta information should be included unless explicitly asked for") - assert.Equal(t, thingID, schema["id"], "has id in schema as uuid field") - }) - - t.Run("searching by class type", func(t *testing.T) { - params := dto.GetParams{ - SearchVector: nil, - ClassName: "TheBestThingClass", - Pagination: &filters.Pagination{Limit: 10}, - Filters: nil, - } - res, err := repo.Search(context.Background(), params) - - require.Nil(t, err) - require.Len(t, res, 1, "got exactly one result") - assert.Equal(t, thingID, res[0].ID, "extracted the ID") - assert.Equal(t, "TheBestThingClass", res[0].ClassName, "matches the class name") - schema := res[0].Schema.(map[string]interface{}) - assert.Equal(t, "some value", schema["stringProp"], "has correct string prop") - assert.Equal(t, &models.GeoCoordinates{ptFloat32(1), ptFloat32(2)}, schema["location"], "has correct geo prop") - assert.Equal(t, thingID, schema["id"], "has id in schema as uuid field") - }) - - t.Run("adding a thing with interpretation additional property", func(t *testing.T) { - thing := &models.Object{ - CreationTimeUnix: 1565612833955, - LastUpdateTimeUnix: 1000001, - ID: thingID, - Class: "TheBestThingClass", - Properties: map[string]interface{}{ - "stringProp": "some value", - "phone": &models.PhoneNumber{ - CountryCode: 49, - DefaultCountry: "DE", - Input: "0171 1234567", - Valid: true, - InternationalFormatted: "+49 171 1234567", - National: 1234567, - NationalFormatted: "0171 1234567", - }, - "location": &models.GeoCoordinates{ - Latitude: ptFloat32(1), - Longitude: ptFloat32(2), - }, - }, - Additional: models.AdditionalProperties{ - "interpretation": map[string]interface{}{ - "source": []interface{}{ - map[string]interface{}{ - "concept": "some", - "occurrence": float64(1), - "weight": float64(1), - }, - map[string]interface{}{ - "concept": "value", - "occurrence": float64(1), - "weight": float64(1), - }, - }, - }, - }, - } - vector := []float32{1, 3, 5, 0.4} - - err := repo.PutObject(context.Background(), thing, vector, nil) - - assert.Nil(t, err) - }) - - t.Run("searching all things", func(t *testing.T) { - // as the test suits grow we might have to extend the limit - res, err := repo.ObjectSearch(context.Background(), 0, 100, nil, nil, additional.Properties{}, "") - require.Nil(t, err) - - item, ok := findID(res, thingID) - require.Equal(t, true, ok, "results should contain our desired thing id") - - assert.Equal(t, thingID, item.ID, "extracted the ID") - assert.Equal(t, "TheBestThingClass", item.ClassName, "matches the class name") - schema := item.Schema.(map[string]interface{}) - assert.Equal(t, "some value", schema["stringProp"], "has correct string prop") - assert.Equal(t, &models.GeoCoordinates{ptFloat32(1), ptFloat32(2)}, schema["location"], "has correct geo prop") - assert.Equal(t, thingID, schema["id"], "has id in schema as uuid field") - assert.Equal(t, models.AdditionalProperties{}, item.AdditionalProperties, "has no additional properties unless explicitly asked for") - }) - - t.Run("searching all things with Vector additional props", func(t *testing.T) { - // as the test suits grow we might have to extend the limit - res, err := repo.ObjectSearch(context.Background(), 0, 100, nil, nil, additional.Properties{Vector: true}, "") - require.Nil(t, err) - - item, ok := findID(res, thingID) - require.Equal(t, true, ok, "results should contain our desired thing id") - - assert.Equal(t, thingID, item.ID, "extracted the ID") - assert.Equal(t, "TheBestThingClass", item.ClassName, "matches the class name") - schema := item.Schema.(map[string]interface{}) - assert.Equal(t, "some value", schema["stringProp"], "has correct string prop") - assert.Equal(t, &models.GeoCoordinates{ptFloat32(1), ptFloat32(2)}, schema["location"], "has correct geo prop") - assert.Equal(t, thingID, schema["id"], "has id in schema as uuid field") - assert.Equal(t, []float32{1, 3, 5, 0.4}, item.Vector, "has Vector property") - }) - - t.Run("searching all things with Vector and Interpretation additional props", func(t *testing.T) { - // as the test suits grow we might have to extend the limit - params := additional.Properties{ - Vector: true, - ModuleParams: map[string]interface{}{ - "interpretation": true, - }, - } - res, err := repo.ObjectSearch(context.Background(), 0, 100, nil, nil, params, "") - require.Nil(t, err) - - item, ok := findID(res, thingID) - require.Equal(t, true, ok, "results should contain our desired thing id") - - assert.Equal(t, thingID, item.ID, "extracted the ID") - assert.Equal(t, "TheBestThingClass", item.ClassName, "matches the class name") - schema := item.Schema.(map[string]interface{}) - assert.Equal(t, "some value", schema["stringProp"], "has correct string prop") - assert.Equal(t, &models.GeoCoordinates{ptFloat32(1), ptFloat32(2)}, schema["location"], "has correct geo prop") - assert.Equal(t, thingID, schema["id"], "has id in schema as uuid field") - assert.Equal(t, []float32{1, 3, 5, 0.4}, item.Vector, "has Vector property") - assert.Equal(t, models.AdditionalProperties{ - "interpretation": map[string]interface{}{ - "source": []interface{}{ - map[string]interface{}{ - "concept": "some", - "occurrence": float64(1), - "weight": float64(1), - }, - map[string]interface{}{ - "concept": "value", - "occurrence": float64(1), - "weight": float64(1), - }, - }, - }, - }, item.AdditionalProperties, "has Vector and Interpretation additional property") - }) - - t.Run("searching a thing by ID", func(t *testing.T) { - item, err := repo.ObjectByID(context.Background(), thingID, search.SelectProperties{}, additional.Properties{}, "") - require.Nil(t, err) - require.NotNil(t, item, "must have a result") - - assert.Equal(t, thingID, item.ID, "extracted the ID") - assert.Equal(t, "TheBestThingClass", item.ClassName, "matches the class name") - schema := item.Schema.(map[string]interface{}) - assert.Equal(t, "some value", schema["stringProp"], "has correct string prop") - assert.Equal(t, &models.GeoCoordinates{ptFloat32(1), ptFloat32(2)}, schema["location"], "has correct geo prop") - assert.Equal(t, thingID, schema["id"], "has id in schema as uuid field") - }) - - // Check the same, but with Object() - t.Run("searching a thing by ID", func(t *testing.T) { - item, err := repo.Object(context.Background(), "TheBestThingClass", - thingID, search.SelectProperties{}, additional.Properties{}, nil, "") - require.Nil(t, err) - require.NotNil(t, item, "must have a result") - - assert.Equal(t, thingID, item.ID, "extracted the ID") - assert.Equal(t, "TheBestThingClass", item.ClassName, "matches the class name") - schema := item.Schema.(map[string]interface{}) - assert.Equal(t, "some value", schema["stringProp"], "has correct string prop") - assert.Equal(t, &models.GeoCoordinates{ptFloat32(1), ptFloat32(2)}, schema["location"], "has correct geo prop") - assert.Equal(t, thingID, schema["id"], "has id in schema as uuid field") - }) - - t.Run("listing multiple things by IDs (MultiGet)", func(t *testing.T) { - query := []multi.Identifier{ - { - ID: "be685717-e61e-450d-8d5c-f44f32d0336c", // this id does not exist - ClassName: "TheBestThingClass", - }, - { - ID: thingID.String(), - ClassName: "TheBestThingClass", - }, - } - res, err := repo.MultiGet(context.Background(), query, additional.Properties{}, "") - require.Nil(t, err) - require.Len(t, res, 2, "length must match even with nil-items") - - assert.Equal(t, strfmt.UUID(""), res[0].ID, "empty object for the not-found item") - - item := res[1] - assert.Equal(t, thingID, item.ID, "extracted the ID") - assert.Equal(t, "TheBestThingClass", item.ClassName, "matches the class name") - schema := item.Schema.(map[string]interface{}) - assert.Equal(t, "some value", schema["stringProp"], "has correct string prop") - assert.Equal(t, &models.GeoCoordinates{ptFloat32(1), ptFloat32(2)}, schema["location"], "has correct geo prop") - assert.Equal(t, thingID, schema["id"], "has id in schema as uuid field") - }) - - t.Run("searching an action by ID without meta", func(t *testing.T) { - item, err := repo.ObjectByID(context.Background(), actionID, search.SelectProperties{}, additional.Properties{}, "") - require.Nil(t, err) - require.NotNil(t, item, "must have a result") - - assert.Equal(t, actionID, item.ID, "extracted the ID") - assert.Equal(t, "TheBestActionClass", item.ClassName, "matches the class name") - schema := item.Schema.(map[string]interface{}) - assert.Equal(t, "some act-citing value", schema["stringProp"], "has correct string prop") - assert.Equal(t, models.AdditionalProperties{}, item.AdditionalProperties, "not meta information should be included unless explicitly asked for") - expectedRefProp := models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI( - crossref.NewLocalhost("", thingID).String()), - }, - } - assert.Equal(t, expectedRefProp, schema["refProp"]) - }) - - t.Run("searching an action by ID with Classification and Vector additional properties", func(t *testing.T) { - item, err := repo.ObjectByID(context.Background(), actionID, search.SelectProperties{}, additional.Properties{Classification: true, Vector: true, RefMeta: true}, "") - require.Nil(t, err) - require.NotNil(t, item, "must have a result") - - assert.Equal(t, actionID, item.ID, "extracted the ID") - assert.Equal(t, "TheBestActionClass", item.ClassName, "matches the class name") - schema := item.Schema.(map[string]interface{}) - assert.Equal(t, "some act-citing value", schema["stringProp"], "has correct string prop") - assert.Equal(t, models.AdditionalProperties{ - "classification": &additional.Classification{ - ID: "foo", - Scope: []string{"scope1", "scope2"}, - ClassifiedFields: []string{"field1", "field2"}, - Completed: timeMust(strfmt.ParseDateTime("2006-01-02T15:04:05.000Z")), - }, - }, item.AdditionalProperties, "it should include the object meta as it was explicitly specified") - assert.Equal(t, []float32{3, 1, 0.3, 12}, item.Vector, "has Vector property") - - expectedRefProp := models.MultipleRef{ - &models.SingleRef{ - Classification: &models.ReferenceMetaClassification{ - LosingDistance: ptFloat64(0.7), - MeanLosingDistance: ptFloat64(0.7), - ClosestLosingDistance: ptFloat64(0.65), - WinningDistance: 0.3, - MeanWinningDistance: 0.3, - ClosestWinningDistance: 0.25, - ClosestOverallDistance: 0.25, - OverallCount: 3, - WinningCount: 2, - LosingCount: 1, - }, - Beacon: strfmt.URI( - crossref.NewLocalhost("", thingID).String()), - }, - } - assert.Equal(t, expectedRefProp, schema["refProp"]) - }) - - t.Run("searching an action by ID with only Vector additional property", func(t *testing.T) { - item, err := repo.ObjectByID(context.Background(), actionID, search.SelectProperties{}, additional.Properties{Vector: true}, "") - require.Nil(t, err) - require.NotNil(t, item, "must have a result") - - assert.Equal(t, actionID, item.ID, "extracted the ID") - assert.Equal(t, "TheBestActionClass", item.ClassName, "matches the class name") - schema := item.Schema.(map[string]interface{}) - assert.Equal(t, "some act-citing value", schema["stringProp"], "has correct string prop") - assert.Equal(t, []float32{3, 1, 0.3, 12}, item.Vector, "it should include the object meta as it was explicitly specified") - }) - - t.Run("searching all actions", func(t *testing.T) { - res, err := repo.ObjectSearch(context.Background(), 0, 10, nil, nil, additional.Properties{}, "") - require.Nil(t, err) - - item, ok := findID(res, actionID) - require.Equal(t, true, ok, "results should contain our desired action id") - - assert.Equal(t, actionID, item.ID, "extracted the ID") - assert.Equal(t, "TheBestActionClass", item.ClassName, "matches the class name") - schema := item.Schema.(map[string]interface{}) - assert.Equal(t, "some act-citing value", schema["stringProp"], "has correct string prop") - }) - - t.Run("sorting all objects", func(t *testing.T) { - // prepare - thingID1 := strfmt.UUID("7c8183ae-150d-433f-92b6-ed095b000001") - thingID2 := strfmt.UUID("7c8183ae-150d-433f-92b6-ed095b000002") - thingID3 := strfmt.UUID("7c8183ae-150d-433f-92b6-ed095b000003") - thingID4 := strfmt.UUID("7c8183ae-150d-433f-92b6-ed095b000004") - actionID1 := strfmt.UUID("7c8183ae-150d-433f-92b6-ed095b100001") - actionID2 := strfmt.UUID("7c8183ae-150d-433f-92b6-ed095b100002") - actionID3 := strfmt.UUID("7c8183ae-150d-433f-92b6-ed095b100003") - testData := []struct { - id strfmt.UUID - className string - stringProp string - phone uint64 - longitude float32 - }{ - { - id: thingID1, - className: "TheBestThingClass", - stringProp: "a very short text", - phone: 1234900, - longitude: 10, - }, - { - id: thingID2, - className: "TheBestThingClass", - stringProp: "zebra lives in Zoo", - phone: 1234800, - longitude: 111, - }, - { - id: thingID3, - className: "TheBestThingClass", - stringProp: "the best thing class", - phone: 1234910, - longitude: 2, - }, - { - id: thingID4, - className: "TheBestThingClass", - stringProp: "car", - phone: 1234901, - longitude: 11, - }, - { - id: actionID1, - className: "TheBestActionClass", - stringProp: "a very short text", - phone: 1234000, - longitude: 10, - }, - { - id: actionID2, - className: "TheBestActionClass", - stringProp: "zebra lives in Zoo", - phone: 1234002, - longitude: 5, - }, - { - id: actionID3, - className: "TheBestActionClass", - stringProp: "fossil fuels", - phone: 1234010, - longitude: 6, - }, - } - for _, td := range testData { - object := &models.Object{ - CreationTimeUnix: 1565612833990, - LastUpdateTimeUnix: 1000001, - ID: td.id, - Class: td.className, - Properties: map[string]interface{}{ - "stringProp": td.stringProp, - "phone": &models.PhoneNumber{ - CountryCode: 49, - DefaultCountry: "DE", - Input: fmt.Sprintf("0171 %d", td.phone), - Valid: true, - InternationalFormatted: fmt.Sprintf("+49 171 %d", td.phone), - National: td.phone, - NationalFormatted: fmt.Sprintf("0171 %d", td.phone), - }, - "location": &models.GeoCoordinates{ - Latitude: ptFloat32(1), - Longitude: ptFloat32(td.longitude), - }, - }, - } - vector := []float32{1.1, 1.3, 1.5, 1.4} - err := repo.PutObject(context.Background(), object, vector, nil) - assert.Nil(t, err) - } - // run sorting tests - tests := []struct { - name string - sort []filters.Sort - expectedThingIDs []strfmt.UUID - expectedActionIDs []strfmt.UUID - constainsErrorMsgs []string - }{ - { - name: "by stringProp asc", - sort: []filters.Sort{{Path: []string{"stringProp"}, Order: "asc"}}, - expectedThingIDs: []strfmt.UUID{thingID1, thingID4, thingID, thingID3, thingID2}, - expectedActionIDs: []strfmt.UUID{actionID1, actionID3, actionID, actionID2}, - }, - { - name: "by stringProp desc", - sort: []filters.Sort{{Path: []string{"stringProp"}, Order: "desc"}}, - expectedThingIDs: []strfmt.UUID{thingID2, thingID3, thingID, thingID4, thingID1}, - expectedActionIDs: []strfmt.UUID{actionID2, actionID, actionID3, actionID1}, - }, - { - name: "by phone asc", - sort: []filters.Sort{{Path: []string{"phone"}, Order: "asc"}}, - expectedThingIDs: []strfmt.UUID{thingID, thingID2, thingID1, thingID4, thingID3}, - expectedActionIDs: []strfmt.UUID{actionID, actionID1, actionID2, actionID3}, - }, - { - name: "by phone desc", - sort: []filters.Sort{{Path: []string{"phone"}, Order: "desc"}}, - expectedThingIDs: []strfmt.UUID{thingID3, thingID4, thingID1, thingID2, thingID}, - expectedActionIDs: []strfmt.UUID{actionID3, actionID2, actionID1, actionID}, - }, - { - name: "by phone and stringProp asc", - sort: []filters.Sort{ - {Path: []string{"phone"}, Order: "asc"}, - {Path: []string{"stringProp"}, Order: "asc"}, - }, - expectedThingIDs: []strfmt.UUID{thingID, thingID2, thingID1, thingID4, thingID3}, - expectedActionIDs: []strfmt.UUID{actionID, actionID1, actionID2, actionID3}, - }, - { - name: "by location asc", - sort: []filters.Sort{{Path: []string{"location"}, Order: "asc"}}, - constainsErrorMsgs: []string{"search: search index thebestactionclass: sort parameter at position 0: " + - "no such prop with name 'location' found in class 'TheBestActionClass' in the schema. " + - "Check your schema files for which properties in this class are available"}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - res, err := repo.ObjectSearch(context.Background(), 0, 100, nil, tt.sort, additional.Properties{Vector: true}, "") - if len(tt.constainsErrorMsgs) > 0 { - require.NotNil(t, err) - for _, errorMsg := range tt.constainsErrorMsgs { - assert.Contains(t, err.Error(), errorMsg) - } - } else { - require.Nil(t, err) - require.Len(t, res, 9) - - var thingIds, actionIds []strfmt.UUID - for i := range res { - if res[i].ClassName == "TheBestThingClass" { - thingIds = append(thingIds, res[i].ID) - } else { - actionIds = append(actionIds, res[i].ID) - } - } - assert.EqualValues(t, thingIds, tt.expectedThingIDs, "thing ids don't match") - assert.EqualValues(t, actionIds, tt.expectedActionIDs, "action ids don't match") - } - }) - } - // clean up - for _, td := range testData { - err := repo.DeleteObject(context.Background(), td.className, td.id, nil, "") - assert.Nil(t, err) - } - }) - - t.Run("verifying the thing is indexed in the inverted index", func(t *testing.T) { - // This is a control for the upcoming deletion, after the deletion it should not - // be indexed anymore. - res, err := repo.Search(context.Background(), dto.GetParams{ - ClassName: "TheBestThingClass", - Pagination: &filters.Pagination{Limit: 10}, - Filters: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: "TheBestThingClass", - Property: "stringProp", - }, - Value: &filters.Value{ - Value: "some", - Type: schema.DataTypeText, - }, - }, - }, - }) - require.Nil(t, err) - require.Len(t, res, 1) - }) - - t.Run("verifying the action is indexed in the inverted index", func(t *testing.T) { - // This is a control for the upcoming deletion, after the deletion it should not - // be indexed anymore. - res, err := repo.Search(context.Background(), dto.GetParams{ - ClassName: "TheBestActionClass", - Pagination: &filters.Pagination{Limit: 10}, - Filters: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: "TheBestActionClass", - Property: "stringProp", - }, - Value: &filters.Value{ - Value: "some", - Type: schema.DataTypeText, - }, - }, - }, - }) - require.Nil(t, err) - require.Len(t, res, 1) - }) - - t.Run("deleting a thing again", func(t *testing.T) { - err := repo.DeleteObject(context.Background(), "TheBestThingClass", thingID, nil, "") - - assert.Nil(t, err) - }) - - t.Run("deleting a action again", func(t *testing.T) { - err := repo.DeleteObject(context.Background(), "TheBestActionClass", actionID, nil, "") - - assert.Nil(t, err) - }) - - t.Run("trying to delete from a non-existing class", func(t *testing.T) { - err := repo.DeleteObject(context.Background(), "WrongClass", thingID, nil, "") - - assert.Equal(t, fmt.Errorf( - "delete from non-existing index for WrongClass"), err) - }) - - t.Run("verifying the thing is NOT indexed in the inverted index", - func(t *testing.T) { - res, err := repo.Search(context.Background(), dto.GetParams{ - ClassName: "TheBestThingClass", - Pagination: &filters.Pagination{Limit: 10}, - Filters: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: "TheBestThingClass", - Property: "stringProp", - }, - Value: &filters.Value{ - Value: "some", - Type: schema.DataTypeText, - }, - }, - }, - }) - require.Nil(t, err) - require.Len(t, res, 0) - }) - - t.Run("verifying the action is NOT indexed in the inverted index", - func(t *testing.T) { - res, err := repo.Search(context.Background(), dto.GetParams{ - ClassName: "TheBestActionClass", - Pagination: &filters.Pagination{Limit: 10}, - Filters: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: "TheBestActionClass", - Property: "stringProp", - }, - Value: &filters.Value{ - Value: "some", - Type: schema.DataTypeText, - }, - }, - }, - }) - require.Nil(t, err) - require.Len(t, res, 0) - }) - - t.Run("trying to get the deleted thing by ID", func(t *testing.T) { - item, err := repo.ObjectByID(context.Background(), thingID, search.SelectProperties{}, additional.Properties{}, "") - require.Nil(t, err) - require.Nil(t, item, "must not have a result") - }) - - t.Run("trying to get the deleted action by ID", func(t *testing.T) { - item, err := repo.ObjectByID(context.Background(), actionID, search.SelectProperties{}, additional.Properties{}, "") - require.Nil(t, err) - require.Nil(t, item, "must not have a result") - }) - - t.Run("searching by vector for a single thing class again after deletion", - func(t *testing.T) { - searchVector := []float32{2.9, 1.1, 0.5, 8.01} - params := dto.GetParams{ - SearchVector: searchVector, - ClassName: "TheBestThingClass", - Pagination: &filters.Pagination{Limit: 10}, - Filters: nil, - } - - res, err := repo.VectorSearch(context.Background(), params) - - require.Nil(t, err) - assert.Len(t, res, 0) - }) - - t.Run("searching by vector for a single action class again after deletion", func(t *testing.T) { - searchVector := []float32{2.9, 1.1, 0.5, 8.01} - params := dto.GetParams{ - SearchVector: searchVector, - ClassName: "TheBestActionClass", - Pagination: &filters.Pagination{Limit: 10}, - Filters: nil, - } - - res, err := repo.VectorSearch(context.Background(), params) - - require.Nil(t, err) - assert.Len(t, res, 0) - }) - - t.Run("ensure referenced class searches are not limited", func(t *testing.T) { - numThings := int(repo.config.QueryMaximumResults * 10) - createdActionIDs := make([]strfmt.UUID, numThings) - createdThingIDs := make([]strfmt.UUID, numThings) - - t.Run("add new action objects", func(t *testing.T) { - actionBatch := make([]objects.BatchObject, numThings) - for i := 0; i < len(createdActionIDs); i++ { - newID := strfmt.UUID(uuid.NewString()) - actionBatch[i] = objects.BatchObject{ - UUID: newID, - Object: &models.Object{ - ID: newID, - Class: "TheBestActionClass", - Properties: map[string]interface{}{ - "stringProp": fmt.Sprintf("action#%d", i), - }, - }, - } - createdActionIDs[i] = newID - } - batchObjResp, err := repo.BatchPutObjects(context.Background(), actionBatch, nil) - require.Len(t, batchObjResp, numThings) - require.Nil(t, err) - for _, r := range batchObjResp { - require.Nil(t, r.Err) - } - }) - - t.Run("add more thing objects to reference", func(t *testing.T) { - thingBatch := make([]objects.BatchObject, numThings) - for i := 0; i < len(createdThingIDs); i++ { - newID := strfmt.UUID(uuid.NewString()) - thingBatch[i] = objects.BatchObject{ - UUID: newID, - Object: &models.Object{ - ID: newID, - Class: "TheBestThingClass", - Properties: map[string]interface{}{ - "stringProp": fmt.Sprintf("thing#%d", i), - }, - }, - } - createdThingIDs[i] = newID - } - batchObjResp, err := repo.BatchPutObjects(context.Background(), thingBatch, nil) - require.Len(t, batchObjResp, numThings) - require.Nil(t, err) - for _, r := range batchObjResp { - require.Nil(t, r.Err) - } - }) - - t.Run("reference each thing from an action", func(t *testing.T) { - refBatch := make([]objects.BatchReference, numThings) - for i := range refBatch { - ref := objects.BatchReference{ - From: &crossref.RefSource{ - Local: true, - PeerName: "localhost", - Class: "TheBestActionClass", - Property: schema.PropertyName("refProp"), - TargetID: createdActionIDs[i], - }, - To: &crossref.Ref{ - Local: true, - PeerName: "localhost", - TargetID: createdThingIDs[i], - }, - } - refBatch[i] = ref - } - batchRefResp, err := repo.AddBatchReferences(context.Background(), refBatch, nil) - require.Nil(t, err) - require.Len(t, batchRefResp, numThings) - for _, r := range batchRefResp { - require.Nil(t, r.Err) - } - }) - - t.Run("query every action for its referenced thing", func(t *testing.T) { - for i := range createdActionIDs { - resp, err := repo.Search(context.Background(), dto.GetParams{ - ClassName: "TheBestActionClass", - Pagination: &filters.Pagination{Limit: 5}, - AdditionalProperties: additional.Properties{ID: true}, - Properties: search.SelectProperties{ - { - Name: "refProp", - Refs: []search.SelectClass{ - { - ClassName: "TheBestThingClass", - RefProperties: search.SelectProperties{ - { - Name: "stringProp", - IsPrimitive: true, - }, - }, - }, - }, - }, - }, - Filters: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorAnd, - Operands: []filters.Clause{ - { - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: "TheBestActionClass", - Property: "stringProp", - }, - Value: &filters.Value{ - Value: fmt.Sprintf("action#%d", i), - Type: schema.DataTypeText, - }, - }, - { - Operator: filters.OperatorLike, - On: &filters.Path{ - Class: "TheBestActionClass", - Property: "refProp", - Child: &filters.Path{ - Class: "TheBestThingClass", - Property: "stringProp", - }, - }, - Value: &filters.Value{ - Value: "thing#*", - Type: schema.DataTypeText, - }, - }, - }, - }, - }, - }) - - require.Nil(t, err) - require.Len(t, resp, 1) - assert.Len(t, resp[0].Schema.(map[string]interface{})["refProp"], 1) - } - }) - }) - - t.Run("query obj by id which has no props", func(t *testing.T) { - id := strfmt.UUID("2cd8a381-6568-4724-9d5c-1ef28d439e94") - - t.Run("insert test obj", func(t *testing.T) { - vec := []float32{0.1, 0.2, 0.3, 0.4} - obj := &models.Object{ - ID: id, - Class: "TheBestActionClass", - Vector: vec, - } - require.Nil(t, repo.PutObject(context.Background(), obj, vec, nil)) - }) - - t.Run("perform search with id filter", func(t *testing.T) { - res, err := repo.Search(context.Background(), dto.GetParams{ - Pagination: &filters.Pagination{Limit: 10}, - ClassName: "TheBestActionClass", - Filters: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: "TheBestActionClass", - Property: filters.InternalPropID, - }, - Value: &filters.Value{ - Value: id.String(), - Type: schema.DataTypeText, - }, - }, - }, - }) - - require.Nil(t, err) - - expected := []search.Result{ - { - ID: id, - ClassName: "TheBestActionClass", - Schema: map[string]interface{}{ - "id": id, - }, - Score: 0, - AdditionalProperties: models.AdditionalProperties{}, - Dims: 4, - }, - } - - assert.Equal(t, expected, res) - }) - }) -} - -func TestCRUD_Query(t *testing.T) { - dirName := t.TempDir() - - logger, _ := test.NewNullLogger() - thingclass := &models.Class{ - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Class: "TheBestThingClass", - Properties: []*models.Property{ - { - Name: "stringProp", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - } - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 10, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - defer repo.Shutdown(context.Background()) - migrator := NewMigrator(repo, logger) - - t.Run("creating the thing class", func(t *testing.T) { - require.Nil(t, - migrator.AddClass(context.Background(), thingclass, schemaGetter.shardState)) - }) - - // update schema getter so it's in sync with class - schemaGetter.schema = schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{thingclass}, - }, - } - - t.Run("scroll through all objects", func(t *testing.T) { - // prepare - className := "TheBestThingClass" - thingID1 := strfmt.UUID("7c8183ae-150d-433f-92b6-ed095b000001") - thingID2 := strfmt.UUID("7c8183ae-150d-433f-92b6-ed095b000002") - thingID3 := strfmt.UUID("7c8183ae-150d-433f-92b6-ed095b000003") - thingID4 := strfmt.UUID("7c8183ae-150d-433f-92b6-ed095b000004") - thingID5 := strfmt.UUID("7c8183ae-150d-433f-92b6-ed095b000005") - thingID6 := strfmt.UUID("7c8183ae-150d-433f-92b6-ed095b000006") - thingID7 := strfmt.UUID("7c8183ae-150d-433f-92b6-ed095b000007") - testData := []struct { - id strfmt.UUID - className string - stringProp string - phone uint64 - longitude float32 - }{ - { - id: thingID1, - className: className, - stringProp: "a very short text", - }, - { - id: thingID2, - className: className, - stringProp: "zebra lives in Zoo", - }, - { - id: thingID3, - className: className, - stringProp: "the best thing class", - }, - { - id: thingID4, - className: className, - stringProp: "car", - }, - { - id: thingID5, - className: className, - stringProp: "a very short text", - }, - { - id: thingID6, - className: className, - stringProp: "zebra lives in Zoo", - }, - { - id: thingID7, - className: className, - stringProp: "fossil fuels", - }, - } - for _, td := range testData { - object := &models.Object{ - CreationTimeUnix: 1565612833990, - LastUpdateTimeUnix: 1000001, - ID: td.id, - Class: td.className, - Properties: map[string]interface{}{ - "stringProp": td.stringProp, - }, - } - vector := []float32{1.1, 1.3, 1.5, 1.4} - err := repo.PutObject(context.Background(), object, vector, nil) - assert.Nil(t, err) - } - // toParams helper method - toParams := func(className string, offset, limit int, - cursor *filters.Cursor, filters *filters.LocalFilter, sort []filters.Sort, - ) *objects.QueryInput { - return &objects.QueryInput{ - Class: className, - Offset: offset, - Limit: limit, - Cursor: cursor, - Filters: filters, - Sort: sort, - Additional: additional.Properties{}, - } - } - // run scrolling through all results - tests := []struct { - name string - className string - cursor *filters.Cursor - query *objects.QueryInput - expectedThingIDs []strfmt.UUID - constainsErrorMsgs []string - }{ - { - name: "all results with step limit: 100", - query: toParams(className, 0, 100, &filters.Cursor{After: "", Limit: 100}, nil, nil), - expectedThingIDs: []strfmt.UUID{thingID1, thingID2, thingID3, thingID4, thingID5, thingID6, thingID7}, - }, - { - name: "all results with step limit: 1", - query: toParams(className, 0, 1, &filters.Cursor{After: "", Limit: 1}, nil, nil), - expectedThingIDs: []strfmt.UUID{thingID1, thingID2, thingID3, thingID4, thingID5, thingID6, thingID7}, - }, - { - name: "all results with step limit: 1 after: thingID4", - query: toParams(className, 0, 1, &filters.Cursor{After: thingID4.String(), Limit: 1}, nil, nil), - expectedThingIDs: []strfmt.UUID{thingID5, thingID6, thingID7}, - }, - { - name: "all results with step limit: 1 after: thingID7", - query: toParams(className, 0, 1, &filters.Cursor{After: thingID7.String(), Limit: 1}, nil, nil), - expectedThingIDs: []strfmt.UUID{}, - }, - { - name: "all results with step limit: 3", - query: toParams(className, 0, 3, &filters.Cursor{After: "", Limit: 3}, nil, nil), - expectedThingIDs: []strfmt.UUID{thingID1, thingID2, thingID3, thingID4, thingID5, thingID6, thingID7}, - }, - { - name: "all results with step limit: 7", - query: toParams(className, 0, 7, &filters.Cursor{After: "", Limit: 7}, nil, nil), - expectedThingIDs: []strfmt.UUID{thingID1, thingID2, thingID3, thingID4, thingID5, thingID6, thingID7}, - }, - { - name: "error on empty class", - query: toParams("", 0, 7, &filters.Cursor{After: "", Limit: 7}, nil, nil), - constainsErrorMsgs: []string{"class not found"}, - }, - { - name: "error on sort parameter", - query: toParams(className, 0, 7, - &filters.Cursor{After: "", Limit: 7}, nil, - []filters.Sort{{Path: []string{"stringProp"}, Order: "asc"}}, - ), - cursor: &filters.Cursor{After: "", Limit: 7}, - constainsErrorMsgs: []string{"sort cannot be set with after and limit parameters"}, - }, - { - name: "error on offset parameter", - query: toParams(className, 10, 7, - &filters.Cursor{After: "", Limit: 7}, nil, - nil, - ), - cursor: &filters.Cursor{After: "", Limit: 7}, - constainsErrorMsgs: []string{"offset cannot be set with after and limit parameters"}, - }, - { - name: "error on offset and sort parameter", - query: toParams(className, 10, 7, - &filters.Cursor{After: "", Limit: 7}, nil, - []filters.Sort{{Path: []string{"stringProp"}, Order: "asc"}}, - ), - cursor: &filters.Cursor{After: "", Limit: 7}, - constainsErrorMsgs: []string{"offset,sort cannot be set with after and limit parameters"}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if len(tt.constainsErrorMsgs) > 0 { - res, err := repo.Query(context.Background(), tt.query) - require.NotNil(t, err) - assert.Nil(t, res) - for _, errorMsg := range tt.constainsErrorMsgs { - assert.Contains(t, err.Error(), errorMsg) - } - } else { - cursorSearch := func(t *testing.T, className string, cursor *filters.Cursor) []strfmt.UUID { - res, err := repo.Query(context.Background(), toParams(className, 0, cursor.Limit, cursor, nil, nil)) - require.Nil(t, err) - var ids []strfmt.UUID - for i := range res { - ids = append(ids, res[i].ID) - } - return ids - } - - var thingIds []strfmt.UUID - cursor := tt.query.Cursor - for { - result := cursorSearch(t, tt.query.Class, cursor) - thingIds = append(thingIds, result...) - if len(result) == 0 { - break - } - after := result[len(result)-1] - cursor = &filters.Cursor{After: after.String(), Limit: cursor.Limit} - } - - require.Equal(t, len(tt.expectedThingIDs), len(thingIds)) - for i := range tt.expectedThingIDs { - assert.Equal(t, tt.expectedThingIDs[i], thingIds[i]) - } - } - }) - } - // clean up - for _, td := range testData { - err := repo.DeleteObject(context.Background(), td.className, td.id, nil, "") - assert.Nil(t, err) - } - }) -} - -func Test_ImportWithoutVector_UpdateWithVectorLater(t *testing.T) { - r := getRandomSeed() - total := 100 - individual := total / 4 - className := "DeferredVector" - var data []*models.Object - var class *models.Class - - dirName := t.TempDir() - logger, _ := test.NewNullLogger() - - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - defer repo.Shutdown(context.Background()) - migrator := NewMigrator(repo, logger) - - t.Run("prepare data for test", func(t *testing.T) { - data = make([]*models.Object, total) - for i := range data { - data[i] = &models.Object{ - ID: strfmt.UUID(uuid.Must(uuid.NewRandom()).String()), - Class: className, - Properties: map[string]interface{}{ - "int_prop": int64(i), - }, - Vector: nil, - } - } - }) - - t.Run("create required schema", func(t *testing.T) { - class = &models.Class{ - Class: className, - Properties: []*models.Property{ - { - DataType: []string{string(schema.DataTypeInt)}, - Name: "int_prop", - }, - }, - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - } - require.Nil(t, - migrator.AddClass(context.Background(), class, schemaGetter.shardState)) - }) - - // update schema getter so it's in sync with class - schemaGetter.schema = schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{class}, - }, - } - - t.Run("import individual objects without vector", func(t *testing.T) { - for i := 0; i < individual; i++ { - err := repo.PutObject(context.Background(), data[i], nil, nil) // nil vector ! - require.Nil(t, err) - } - }) - - t.Run("import batch objects without vector", func(t *testing.T) { - batch := make(objects.BatchObjects, total-individual) - - for i := range batch { - batch[i] = objects.BatchObject{ - OriginalIndex: i, - Err: nil, - Vector: nil, - Object: data[i+individual], - UUID: data[i+individual].ID, - } - } - - res, err := repo.BatchPutObjects(context.Background(), batch, nil) - require.Nil(t, err) - - for _, obj := range res { - require.Nil(t, obj.Err) - } - }) - - t.Run("verify inverted index works correctly", func(t *testing.T) { - res, err := repo.Search(context.Background(), dto.GetParams{ - Filters: buildFilter("int_prop", total+1, lte, dtInt), - ClassName: className, - Pagination: &filters.Pagination{ - Offset: 0, - Limit: total, - }, - }) - require.Nil(t, err) - assert.Len(t, res, total) - }) - - t.Run("perform unfiltered vector search and verify there are no matches", func(t *testing.T) { - res, err := repo.VectorSearch(context.Background(), dto.GetParams{ - Filters: nil, - ClassName: className, - Pagination: &filters.Pagination{ - Offset: 0, - Limit: total, - }, - SearchVector: randomVector(r, 7), - }) - require.Nil(t, err) - assert.Len(t, res, 0) // we skipped the vector on half the elements, so we should now match half - }) - - t.Run("update some of the objects to add vectors", func(t *testing.T) { - for i := range data { - if i%2 == 1 { - continue - } - - data[i].Vector = randomVector(r, 7) - err := repo.PutObject(context.Background(), data[i], data[i].Vector, nil) - require.Nil(t, err) - } - }) - - t.Run("perform unfiltered vector search and verify correct matches", func(t *testing.T) { - res, err := repo.VectorSearch(context.Background(), dto.GetParams{ - Filters: nil, - ClassName: className, - Pagination: &filters.Pagination{ - Offset: 0, - Limit: total, - }, - SearchVector: randomVector(r, 7), - }) - require.Nil(t, err) - assert.Len(t, res, total/2) // we skipped the vector on half the elements, so we should now match half - }) - - t.Run("perform filtered vector search and verify correct matches", func(t *testing.T) { - res, err := repo.VectorSearch(context.Background(), dto.GetParams{ - Filters: buildFilter("int_prop", 50, lt, dtInt), - ClassName: className, - Pagination: &filters.Pagination{ - Offset: 0, - Limit: total, - }, - SearchVector: randomVector(r, 7), - }) - require.Nil(t, err) - // we skipped the vector on half the elements, and cut the list in half with - // the filter, so we're only expected a quarter of the total size now - assert.Len(t, res, total/4) - }) -} - -func TestVectorSearch_ByDistance(t *testing.T) { - className := "SomeClass" - var class *models.Class - - dirName := t.TempDir() - logger, _ := test.NewNullLogger() - - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - // this is set really low to ensure that search - // by distance is conducted, which executes - // without regard to this value - QueryMaximumResults: 1, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - defer repo.Shutdown(context.Background()) - migrator := NewMigrator(repo, logger) - - t.Run("create required schema", func(t *testing.T) { - class = &models.Class{ - Class: className, - Properties: []*models.Property{ - { - DataType: []string{string(schema.DataTypeInt)}, - Name: "int_prop", - }, - }, - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - } - require.Nil(t, - migrator.AddClass(context.Background(), class, schemaGetter.shardState)) - }) - - // update schema getter so it's in sync with class - schemaGetter.schema = schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{class}, - }, - } - - searchVector := []float32{-0.10190568, -0.06259751, 0.05616188, -0.19249836, 0.09714927, -0.1902525, -0.064424865, -0.0387358, 0.17581701, 0.4476738, 0.29261824, 0.12026761, -0.19975126, 0.023600178, 0.17348698, 0.12701738, -0.36018127, -0.12051587, -0.17620522, 0.060741074, -0.064512916, 0.18640806, -0.1529852, 0.08211839, -0.02558465, -0.11369845, 0.0924098, -0.10544433, -0.14728987, -0.041860342, -0.08533595, 0.25886244, 0.2963937, 0.26010615, 0.2111097, 0.029396622, 0.01429563, 0.06410264, -0.119665794, 0.33583277, -0.05802661, 0.023306102, 0.14435922, -0.003951336, -0.13870825, 0.07140894, 0.10469943, -0.059021875, -0.065911904, 0.024216041, -0.26282874, 0.04896568, -0.08291928, -0.12793182, -0.077824734, 0.08843151, 0.31247458, -0.066301286, 0.006904921, -0.08277095, 0.13936226, -0.64392364, -0.19566211, 0.047227614, 0.086121306, -0.20725192, -0.096485816, -0.16436341, -0.06559169, -0.019639932, -0.012729637, 0.08901619, 0.0015896161, -0.24789932, 0.35496348, -0.16272856, -0.01648429, 0.11247674, 0.08099968, 0.13339259, 0.055829972, -0.34662855, 0.068509, 0.13880715, 0.3201848, -0.055557363, 0.22142135, -0.12867308, 0.0037871755, 0.24888979, -0.007443307, 0.08906625, -0.02022331, 0.11510742, -0.2385861, 0.16177008, -0.16214795, -0.28715602, 0.016784908, 0.19386634, -0.07731616, -0.100485384, 0.4100712, 0.061834496, -0.2325293, -0.026056025, -0.11632323, -0.17040555, -0.081960455, -0.0061040106, -0.05949373, 0.044952348, -0.079565264, 0.024430245, -0.09375341, -0.30249637, 0.115205586, -0.13083287, -0.04264671, -0.089810364, 0.16227561, 0.07318055, -0.10496504, 0.00063501706, -0.04936106, -0.0022282854, 1.0893154, 0.1698662, -0.019563455, -0.011128426, 0.04477475, -0.15656771, -0.056911886, -0.5759019, -0.1881429, 0.17088258, 0.24124439, 0.111288875, -0.0015475494, -0.021278847, -0.08362156, 0.09997524, -0.094385885, -0.1674031, 0.061180864, 0.28517494, -0.016217072, 0.025866214, -0.22854298, -0.17924422, -0.037767246, 0.12252907, -0.31698978, -0.038031228, 0.055408552, 0.1743545, -0.040576655, 0.1293942, -0.56650764, -0.10306195, -0.19548112, -0.245544, -0.018241389, -0.039024632, -0.31659162, 0.1565075, 0.08412337, 0.13177724, -0.13766576, -0.15355161, -0.16960397, -0.012436442, 0.04828157, 0.12566057, -0.35308784, -0.37520224, -0.1265899, -0.13991497, 0.14402144, 0.117542416, -0.20750546, -0.5849919, -0.010469457, -0.19677396, 0.011365964, 0.00666846, -0.083470255, 0.24928358, 0.07026387, 0.19082819, 0.24557637, 0.014292963, 0.14846677, 0.031625308, -0.20398879, 0.19507346, -0.18119761, -0.045725327, -0.042455163, -0.099733196, -0.33636123, -0.28447086, 0.30274838, -0.01603988, -0.0529655, 0.15784146, 0.08746072, -0.1703993, 0.2414512, 0.060322937, -0.00812057, 0.031162385, -0.1764905, 0.22107981, -0.016657066, 0.31948856, 0.07282925, -0.036991462, 0.01266936, -0.009106514, -0.038732465, 0.20973183, 0.033236098, -0.10673938, -0.06880061, 0.115524575, -0.39688373, 0.08749971, -0.21816005, -0.22100002, -0.3716853, -0.14720486, 0.24316181, 0.29673144, 0.020808747, 0.07658521, 0.16310681, 0.38785335, 0.0992224, 0.14177811, 0.025954131, -0.08690783, 0.19653428, 0.09584941, 0.040072605, -0.00038361162, -0.094546966, 0.1910902, 0.13217318, 0.060072783, -0.0655816, 0.2777626, 0.1799169, 0.20187178, -0.0996889, -0.01932122, -0.13133621, 0.057482753, -0.36892185, -0.032093313, 0.14607865, 0.12033318, -0.041683596, -0.2048406, -0.041777443, -0.14975598, -0.2526341, 0.12659752, 0.010567178, -0.297333, -0.27522174, 0.06923473, 0.043150593, -0.017045585, -0.2400216, 0.11413547, -0.40081662, -0.0018820907, 0.13800722, 0.085972115, -0.01519989, -0.10491216, 0.09170084, 0.063085504, 0.046743374, -0.014466267, 0.09880224, 0.027706565, 0.09951337, 0.17317492, -0.025654864, 0.14658073, 0.042377427, -0.08402882, -0.12423425, 0.32714987, -0.1527207, 0.106094465, 0.017378228, -0.06302387} - searchObject := strfmt.UUID("fe687bf4-f10f-4c23-948d-0746ea2927b3") - - tests := map[strfmt.UUID]struct { - inputVec []float32 - expected bool - }{ - strfmt.UUID("88460290-03b2-44a3-9adb-9fa3ae11d9e6"): { - inputVec: []float32{-0.11015724, -0.05380307, 0.027512914, -0.16925375, 0.08306809, -0.19312492, -0.08910436, -0.011051652, 0.17981204, 0.40469593, 0.28226805, 0.09381516, -0.18380599, 0.03102771, 0.1645333, 0.1530153, -0.3187937, -0.10800173, -0.18466279, 0.0004336393, -0.0495677, 0.19905856, -0.11614494, 0.08834681, -0.011200292, -0.11969374, 0.12497086, -0.12427251, -0.13395442, -0.0060353535, -0.07504816, 0.23205791, 0.2982508, 0.2517544, 0.176147, -0.036871903, 0.017852835, 0.040007118, -0.118621, 0.3648693, -0.058933854, 0.04004229, 0.11871147, -0.019860389, -0.12701912, 0.106662825, 0.086498804, -0.04303973, -0.0742352, 0.018250324, -0.26544014, 0.029228423, -0.087171465, -0.1282789, -0.06403083, 0.09680911, 0.31433868, -0.081510685, -0.011283603, -0.041624587, 0.16530018, -0.6714878, -0.2436993, 0.03173918, 0.106117725, -0.20803581, -0.10429562, -0.16975354, -0.078582145, -0.0065962705, -0.06840946, 0.094937086, -0.020617036, -0.23795949, 0.34785536, -0.19834635, -0.015064479, 0.11930141, 0.090962164, 0.120560184, 0.054095767, -0.38602966, 0.057141174, 0.12039684, 0.32000408, -0.05146908, 0.20762976, -0.09342379, 0.037577383, 0.23894139, -0.0075003104, 0.104791366, -0.015841056, 0.102840215, -0.20813248, 0.1855997, -0.12594056, -0.27132365, -0.0055563124, 0.21954241, -0.10798524, -0.111896284, 0.44049335, 0.049884494, -0.22339955, -0.005374135, -0.120713554, -0.22275059, -0.09146004, 0.017188415, -0.106493734, 0.045247544, -0.07725446, 0.056848228, -0.10294392, -0.2896642, 0.112891, -0.13773362, -0.089911595, -0.13500965, 0.14051703, 0.040092673, -0.13896292, 0.04580957, -0.014300959, 0.03737215, 1.0661443, 0.19767477, -0.07703914, -0.012910904, -0.0037716173, -0.14437087, -0.06938004, -0.5348036, -0.16047458, 0.19416414, 0.21938956, 0.092242256, -0.012630808, -0.021863988, -0.051702406, 0.08780951, -0.0815602, -0.15332024, 0.077632725, 0.25709584, -0.025725808, 0.042116437, -0.22687604, -0.11791685, -0.028626656, 0.16734225, -0.3017483, -0.03236202, 0.02888077, 0.18193199, -0.009032297, 0.14454253, -0.511494, -0.12119192, -0.20757924, -0.2561716, -0.03904554, -0.07348411, -0.28547177, 0.15967208, 0.079396725, 0.14358875, -0.12829632, -0.18175666, -0.15540425, -0.020419862, 0.019070208, 0.12168836, -0.3428434, -0.357543, -0.11218741, -0.12834033, 0.13564876, 0.12768728, -0.1817461, -0.61235875, -0.029409664, -0.19765733, 0.03872163, 0.0074950717, -0.10025679, 0.2872255, 0.033995092, 0.12945211, 0.21831632, 0.04666009, 0.14233032, 0.016767867, -0.2039244, 0.2000191, -0.13099428, -0.020210614, -0.06286195, -0.0948797, -0.34830436, -0.21595824, 0.32722405, -0.024735296, -0.07859145, 0.16975155, 0.08186461, -0.19249061, 0.23405583, 0.04837592, 0.021467948, -0.022215014, -0.14892808, 0.23908867, -0.050126728, 0.2867907, 0.07740656, -0.01714987, -0.0046314416, -0.0048108613, -0.007407311, 0.1807499, 0.049772616, -0.14680666, -0.07335314, 0.09023705, -0.40600133, 0.05522128, -0.20085222, -0.20410904, -0.34319055, -0.10792269, 0.2297779, 0.30397663, 0.05230268, 0.06408224, 0.13797496, 0.3691112, 0.083033495, 0.13695791, -0.015612457, -0.06413475, 0.18117142, 0.12928344, 0.049171276, 0.016104931, -0.102417335, 0.19589683, 0.14380622, 0.0748065, -0.005402455, 0.27243868, 0.14925551, 0.19564849, -0.10738364, -0.054175537, -0.10068278, 0.06004795, -0.38755924, -0.01654251, 0.1394104, 0.0968949, 0.004271706, -0.17105898, -0.050423585, -0.15311627, -0.24458972, 0.12665795, -0.022814916, -0.23887472, -0.289588, 0.05521137, 0.041259795, -0.021133862, -0.23674431, 0.08424598, -0.37863016, 0.017239956, 0.13776784, 0.060790475, 0.057887543, -0.08784489, 0.08803934, 0.027996546, 0.085972995, -0.014455558, 0.11668073, 0.03812387, 0.088413864, 0.22228678, -0.015599858, 0.11000236, 0.035271563, -0.08088438, -0.13092226, 0.29378533, -0.12311522, 0.09377676, 0.02948718, -0.09136077}, - expected: true, - }, - strfmt.UUID("c99bc97d-7035-4311-94f3-947dc6471f51"): { - inputVec: []float32{-0.07545987, -0.046643265, 0.044445477, -0.18531442, 0.07922216, -0.19388637, -0.069393866, -0.036144026, 0.1713317, 0.41803706, 0.23576374, 0.073170714, -0.14085358, 0.012535708, 0.17439325, 0.10057567, -0.33506152, -0.06800867, -0.18882714, 0.002687021, -0.03276807, 0.17267752, -0.13951558, 0.071382746, 0.020254405, -0.10178502, 0.13977699, -0.107296936, -0.113307, -0.002506761, -0.092065684, 0.21008658, 0.31157792, 0.19640765, 0.15769793, -0.0050196033, 0.0022481605, 0.015436289, -0.11822955, 0.31494477, -0.07425527, 0.051401984, 0.11648046, -0.00016831602, -0.12758006, 0.06814693, 0.06108981, -0.025454175, -0.018695071, 0.041827776, -0.23480764, 0.06652185, -0.078328855, -0.121668324, -0.04341973, 0.114403985, 0.32614416, -0.07992741, -0.019665314, -0.017408244, 0.12615794, -0.6350545, -0.17056493, 0.07171332, 0.047071394, -0.18428493, -0.09011123, -0.15995751, -0.03345579, -0.014678, -0.038699757, 0.044125225, 0.0042562615, -0.29445595, 0.30460796, -0.13630153, 0.00014055961, 0.08996278, 0.08948901, 0.12164838, 0.079090506, -0.36153567, 0.02817218, 0.11914518, 0.29805067, -0.07431765, 0.16793592, -0.099549234, 0.045226075, 0.22235383, -0.045654725, 0.09233901, -0.004902314, 0.08621588, -0.19723448, 0.19557382, -0.13199815, -0.22924824, -0.015981175, 0.19762704, -0.08940076, -0.084909916, 0.43372774, 0.026998578, -0.20827708, 0.037450224, -0.078997016, -0.18065391, -0.071308024, 0.00870316, -0.114981964, 0.017085023, -0.07696264, 0.009330409, -0.097458, -0.25530958, 0.118254915, -0.12516825, -0.008301536, -0.20432962, 0.15235707, 0.012840041, -0.18034773, 0.039270073, -0.03131139, 0.013706253, 0.98688674, 0.18840753, -0.055119563, 0.00836046, 0.019445436, -0.10701598, -0.024610046, -0.50088006, -0.15488546, 0.14209819, 0.1798376, 0.08615982, -0.0119235935, -0.0060070297, -0.08406098, 0.10096481, -0.09077014, -0.15957798, 0.10556352, 0.2100476, -0.036947068, 0.05316554, -0.20480183, -0.14873864, -0.0069070593, 0.16027303, -0.288908, -0.04487129, 0.0705415, 0.11973847, -0.0017247469, 0.14092937, -0.5262047, -0.094283305, -0.19120996, -0.2816572, -0.010916339, -0.07984056, -0.28659204, 0.13706332, 0.07364347, 0.12300072, -0.17554194, -0.16378267, -0.15244205, 0.00075927645, 0.017289847, 0.12072629, -0.33452734, -0.33727616, -0.12780978, -0.09350711, 0.105674624, 0.10770573, -0.17278843, -0.5760599, -0.013741414, -0.15395893, 0.009837732, 0.015417911, -0.11384676, 0.24567491, 0.04905973, 0.10762609, 0.2131752, 0.019281652, 0.11665857, 0.022718405, -0.2234067, 0.23241606, -0.12194457, -0.049972955, -0.012225418, -0.14856412, -0.386102, -0.23018965, 0.28920102, -0.023396742, -0.114672944, 0.12130062, 0.05654803, -0.16194181, 0.24095012, 0.03644393, 0.028024165, -0.008832254, -0.16496961, 0.19496499, -0.035887964, 0.25981775, 0.0970074, 0.0013458093, -0.009548204, 0.040741496, -0.019192837, 0.20718361, -0.004034228, -0.1343262, -0.06990001, 0.09888768, -0.35942966, 0.043895893, -0.19182123, -0.17963983, -0.3222771, -0.10223457, 0.23866613, 0.25855777, 0.04051543, 0.08756274, 0.15683484, 0.37856522, 0.04853359, 0.10198129, -0.0061066896, -0.049892712, 0.17087941, 0.14563805, 0.06984385, 0.0071270005, -0.11838641, 0.18716812, 0.14013803, 0.05242403, 0.034357738, 0.3083466, 0.14742611, 0.17841975, -0.124118194, -0.014102871, -0.052544866, 0.037493005, -0.33485797, -0.013164912, 0.1066288, 0.11141791, -0.04029921, -0.16429856, -0.032241724, -0.15965424, -0.2430594, 0.13654563, 0.009401224, -0.2045843, -0.28467956, 0.07325551, 0.027996557, -0.033877768, -0.24350801, 0.08329816, -0.35555813, 0.006908567, 0.07227365, 0.03188268, 0.032559503, -0.09180395, 0.05601515, 0.0047281734, 0.06878795, -0.018943194, 0.08251342, 0.042039152, 0.12902294, 0.20526606, -0.014881293, 0.11723917, 0.0115632, -0.09016013, -0.12117223, 0.31020245, -0.111444525, 0.077845715, 0.00046715315, -0.104099475}, - expected: true, - }, - strfmt.UUID("fe687bf4-f10f-4c23-948d-0746ea2927b3"): { - inputVec: []float32{-0.20739016, -0.19551805, 0.06645163, 0.008650202, 0.03700748, -0.04132599, -0.029881354, 0.04684896, 0.096614264, 0.42888844, 0.10003969, 0.026234219, -0.051639702, -0.118660435, 0.14473079, 0.2911885, -0.1180539, -0.16804434, -0.48081538, 0.021702053, 0.12612472, 0.15442817, -0.05836532, 0.074295096, -0.28077397, -0.24297802, 0.047836643, -0.36753318, -0.30482984, 0.09265357, 0.25571078, 0.41130066, 0.46177864, 0.34033778, 0.20721313, -0.37726295, 0.07721501, 0.08009689, 0.00027321206, 0.5168123, -0.15305339, 0.0937765, 0.096195236, -0.21120761, 0.014014921, 0.3133104, 0.20773117, 0.08483507, -0.27784437, -0.17281856, -0.6050923, -0.22439326, -0.16914369, -0.3149047, -0.13828672, 0.16334395, -0.0018224253, -0.024342008, 0.3511251, 0.04979151, 0.34223744, -0.6965703, -0.36211932, -0.27092442, 0.34418032, -0.09667905, 0.13344757, -0.15622364, -0.24129291, 0.06958589, -0.2681816, -0.09497071, -0.08923615, -0.06642436, 0.48688608, -0.33535984, 0.014242731, 0.079838976, 0.32949054, 0.09051045, -0.2653392, -0.47393548, 0.07508276, 0.0062832804, 0.724184, -0.18929236, 0.11718613, 0.049603477, 0.08766128, 0.31040704, 0.04038693, -0.0017023507, -0.18986607, 0.056264438, -0.20978904, -0.107441366, -0.30505633, -0.45781082, -0.11571784, 0.32160303, -0.1347523, -0.08090298, 0.51651996, -0.023250414, -0.18725531, -0.14222279, 0.009277832, -0.49789724, -0.25156206, 0.0042495225, 0.0038805408, -0.031416763, 0.10277136, 0.14383446, -0.23241928, -0.42357358, 0.027033398, -0.2262604, -0.2685295, -0.14510548, 0.18256307, 0.063297585, 0.027636252, 0.081166506, 0.06726344, 0.1677495, 1.5217289, 0.33152232, -0.2209926, 0.051426213, 0.15640806, -0.30210486, -0.32857975, -0.4170022, -0.028293105, 0.28772062, 0.50510746, 0.09162247, -0.12383193, -0.25066972, -0.1441897, 0.107192926, -0.07404076, 0.0042472635, 0.11014519, 0.22332853, 0.09434378, -0.3278343, 0.041899726, 0.06838457, 0.10983681, 0.11864574, -0.25336757, -0.047530346, -0.027303243, 0.37403497, 0.13420461, 0.14946426, -0.41996637, -0.037703935, -0.47961184, -0.29839846, -0.103934005, -0.12058302, -0.12806267, 0.22814582, 0.3904893, -0.16044962, -0.17479864, -0.33139735, -0.29185295, 0.0653074, 0.042426735, 0.06092335, -0.18776153, -0.52555144, -0.15889317, -0.20644087, 0.2293067, 0.26668283, -0.15607063, -0.696593, -0.08224992, -0.4283747, 0.26883888, -0.031052848, -0.1311875, 0.26636878, 0.16457985, 0.15660451, 0.10629464, 0.17345549, 0.23963387, 0.22997221, -0.111713186, -0.08499592, -0.2274625, 0.19285984, -0.08285016, -0.02692149, -0.3426618, -0.13361897, 0.2870389, -0.12032792, -0.22944619, 0.25588584, 0.24607788, -0.2762531, 0.30983892, 0.011088746, -0.15739818, 0.053215, -0.21660997, 0.033805694, -0.17886437, 0.2979239, 0.2163545, -0.08381542, 0.19666128, -0.28977823, -0.20994817, -0.012160099, 0.057499636, -0.12549455, 0.19303595, -0.14420606, -0.51937664, 0.23400985, -0.27893808, -0.2660984, -0.27870297, -0.32149136, 0.19958079, 0.34468395, 0.18947665, -0.16529581, 0.101419374, 0.30195153, 0.09030288, 0.12496541, 0.02999903, -0.016697621, 0.15314853, 0.27848768, 0.24102053, 0.06933273, 0.08923653, 0.10477832, 0.4389032, 0.15679164, -0.11119637, 0.134823, 0.30230528, 0.20818473, -0.005579584, -0.3474488, -0.44394243, 0.22270252, -0.3668763, 0.07474772, 0.011691334, 0.088187896, 0.23832949, -0.07960201, 0.066471875, 0.034641538, -0.39984587, 0.0032980456, -0.28492525, -0.46358657, -0.2148288, -0.107226945, 0.02734428, -0.24686679, -0.123900555, 0.18174778, -0.31248868, 0.13808723, 0.31549984, 0.21521719, 0.13966985, -0.27272752, 0.12091104, 0.14257833, 0.23175247, 0.15639938, 0.40828535, 0.31916845, 0.023645567, 0.20658277, -0.20365283, 0.113746524, 0.13173752, -0.050343305, -0.31581175, 0.09704622, -0.014172505, 0.16924341, 0.30327854, -0.17770194}, - expected: false, - }, - strfmt.UUID("e7bf6c45-de72-493a-b273-5ef198974d61"): { - inputVec: []float32{0.089313604, -0.050221898, 0.18352903, 0.16257699, 0.14520381, 0.17993976, 0.14594483, 0.019256027, -0.15505213, 0.23606326, -0.14456263, 0.2679586, -0.112208664, 0.12997514, 0.0051072896, 0.28151348, -0.10495799, 0.026782967, -0.38603118, 0.16190273, -0.0428943, -0.16265322, -0.17910561, 0.0746288, -0.3117934, -0.15871756, -0.11377734, -0.06822346, -0.13829489, 0.13019162, 0.30741218, 0.16194165, 0.013218932, 0.054517113, 0.12490437, -0.07709048, 0.02556826, -0.21159878, -0.09082174, 0.24629511, 0.05013666, 0.25168124, -0.14423938, -0.0937688, -0.07811525, -0.049346007, 0.3592527, 0.30411252, -0.1168557, 0.18870471, 0.06614835, -0.20099068, -0.084436245, 0.073036775, -0.03448665, -0.11147946, -0.10862863, -0.012393957, 0.18990599, 0.060957544, 0.19518377, -0.027541652, -0.26750082, -0.12780671, 0.09570065, -0.03541132, 0.094820626, -0.13539355, -0.09468136, 0.18476579, -0.20970085, -0.20989786, -0.12084438, -0.04517079, -0.008074663, 0.02824076, 0.114496395, -0.20462593, 0.103516705, -0.101554185, -0.1374868, -0.24884155, -0.08101618, -0.016105993, 0.22608215, -0.007247754, -0.17246912, 0.058247145, -0.041018173, 0.19471274, -0.022576109, 0.032828204, -0.079321206, -0.09259324, 0.041115705, -0.25280195, -0.28517374, -0.19496292, 0.18070905, 0.06384923, -0.004056949, 0.1536253, 0.17861623, -0.033833142, 0.12039968, 0.04458716, 0.08793809, -0.15683243, -0.1087904, 0.1741014, 0.007256374, -0.20265253, 0.034111258, 0.03311363, -0.09449356, -0.13161612, -0.026084669, 0.07609202, 0.03452338, 0.08840356, -0.044566724, 0.1507175, 0.089273594, 0.18872644, 0.18333815, -0.023196407, 0.63831943, 0.20309874, 0.10217627, 0.11445079, 0.18965706, -0.16809432, -0.343172, -0.06439529, 0.08362327, 0.32746288, 0.38483366, 0.020372175, -0.25239283, 0.019468365, -0.016367752, 0.016749177, 0.024621855, 0.030529505, 0.20601188, -0.100692995, -0.16414656, -0.23193358, 0.26616478, 0.06166736, 0.14341855, 0.1294041, 0.045133967, 0.0014262896, -0.0194398, 0.040737696, 0.10099013, -0.10838136, -0.28768313, -0.073719576, -0.15836753, -0.10482511, -0.1349642, -0.107005455, 0.01957546, 0.13799994, 0.056444198, -0.38841644, -0.07585945, -0.018703599, -0.19934878, 0.15176265, 0.04133126, 0.063531734, 0.09720055, -0.29999572, 0.04765686, -0.23604262, 0.081500284, 0.056092553, -0.13664724, -0.37729686, 0.031137427, -0.052083906, 0.117984496, -0.14562207, -0.029609507, 0.13725121, 0.090367764, 0.12787215, 0.11026589, 0.25123242, 0.12911159, 0.055398554, 0.0032232201, 0.026706887, 0.14584258, 0.019900957, -0.12197998, -0.087177716, -0.24649806, -0.17869286, 0.07139921, -0.09633085, -0.16027117, 0.23617831, 0.05429949, -0.061085824, 0.040451035, 0.052443117, -0.14255014, 0.15598148, -0.2336374, 0.08394173, -0.34318882, 0.3419207, 0.18282516, -0.03709172, 0.10525048, -0.1871602, -0.22663523, 0.01635051, 0.16996534, -0.18056048, -0.169894, -0.18467705, -0.3641231, 0.060861763, -0.080082566, -0.08888943, 0.11629789, -0.00973362, 0.07452957, 0.25680214, 0.042024083, -0.024963235, 0.1743134, 0.10921186, 0.25191578, 0.028438354, 0.004781374, -0.08364819, 0.051807538, 0.1165724, 0.29184434, -0.21512283, 0.12515399, -0.08803969, 0.41930157, -0.10181762, 0.038189832, 0.085555896, -0.026453126, 0.04717047, 0.12667313, 0.023158737, -0.45877644, 0.18732828, 0.062374037, -0.21956007, -0.04449947, 0.19028638, 0.1359094, 0.26384917, 0.077602044, 0.35136092, 0.069637895, 0.048263475, -0.02498448, -0.09221205, -0.012142404, -0.124592446, 0.14599627, -0.050875153, -0.25454503, -0.069588415, -0.29793787, -0.13407284, 0.25388947, 0.35565627, -0.034204755, 0.0024766966, 0.086427726, -0.054318108, 0.063218184, -0.037823644, 0.108287826, 0.14440496, 0.025134278, 0.14978257, -0.03355889, 0.02980915, -0.13764386, 0.4167542, -0.03938922, 0.026970355, 0.24595529, 0.111741625, -0.074567944, -0.057232533}, - expected: false, - }, - strfmt.UUID("0999d109-1d5f-465a-bd8b-e3fbd46f10aa"): { - inputVec: []float32{-0.10486144, -0.07437922, 0.069469325, -0.1438278, 0.07740161, -0.18606456, -0.09991434, -0.020051572, 0.19863395, 0.4347328, 0.297606, 0.07853262, -0.16025662, 0.023596637, 0.16935731, 0.17052403, -0.29870638, -0.10309007, -0.20055692, 0.0027809117, -0.03928043, 0.21178603, -0.13793766, 0.08118157, 0.006693433, -0.13829204, 0.14778963, -0.13180175, -0.21128704, -0.0026104634, -0.076393716, 0.22200249, 0.32417125, 0.26045212, 0.1783609, -0.114116184, 0.0100981165, 0.07233143, -0.15913877, 0.4238603, -0.036907215, 0.0595873, 0.0807002, -0.07637312, -0.12889846, 0.111177936, 0.091114685, -0.018454906, -0.12132672, 0.056664582, -0.30461523, 0.020763714, -0.10992191, -0.14430659, -0.092879646, 0.13615008, 0.33039626, -0.115675874, 0.03607886, -0.027918883, 0.19531779, -0.7211654, -0.23073879, 0.011791817, 0.1315166, -0.22779183, -0.13773227, -0.1814997, -0.09008116, 0.021698939, -0.102921166, 0.090760864, 0.011856942, -0.25561005, 0.40769714, -0.21286584, -0.018059848, 0.13812906, 0.079457305, 0.12631191, 0.0024881593, -0.4282836, 0.0619608, 0.12207897, 0.39083096, -0.009502015, 0.19990632, -0.06503092, 0.0635979, 0.27579078, -0.020699967, 0.068474516, 0.0043831975, 0.10303624, -0.1885405, 0.22989234, -0.15952443, -0.29842895, 0.006752088, 0.22831629, -0.13150804, -0.13695218, 0.5357904, 0.050116863, -0.24064547, -0.01375713, -0.096647836, -0.24984525, -0.10429946, 0.002098812, -0.08113263, 0.05237009, -0.10246039, 0.05234802, -0.13899775, -0.3439524, 0.12522809, -0.18406768, -0.09022853, -0.19954625, 0.15810682, 0.039185096, -0.13576287, 0.045047805, 0.0035671506, 0.055920787, 1.1730403, 0.24019612, -0.13423051, -0.008052084, -0.00431602, -0.17079304, -0.09064658, -0.58728856, -0.1365065, 0.22919424, 0.22795208, 0.13396585, 0.018962797, -0.0075796233, -0.072394304, 0.10908417, -0.10881145, -0.16565171, 0.10378018, 0.27296618, -0.059810717, 0.03355443, -0.22429268, -0.12499127, -0.0441017, 0.20800696, -0.29992488, -0.003536096, 0.0026575085, 0.2427503, -0.007395092, 0.13233404, -0.5494433, -0.13144702, -0.2899963, -0.27367246, -0.05257514, -0.0939783, -0.267614, 0.16651331, 0.13891254, 0.08047202, -0.14046521, -0.19062972, -0.1433134, 0.0067776316, 0.00207368, 0.12986982, -0.35847133, -0.41852546, -0.15541135, -0.09865207, 0.14805861, 0.17072491, -0.22655731, -0.6473966, -0.007884447, -0.2060257, 0.035390265, 0.02781265, -0.09760371, 0.30535778, 0.047540557, 0.14565119, 0.21733035, 0.06558403, 0.13184759, 0.044231005, -0.22218557, 0.1897204, -0.1596938, 0.017510587, -0.030249557, -0.082377456, -0.39669412, -0.18365891, 0.34806964, -0.024830062, -0.06955674, 0.21521395, 0.1201222, -0.21855503, 0.23522708, 0.038058903, -0.019610198, -0.025448406, -0.18122384, 0.26068974, -0.055872105, 0.29595166, 0.11005987, -0.00841942, 0.006325112, -0.0013332894, -0.025598384, 0.17320716, 0.03480282, -0.1504056, -0.07133905, 0.08367911, -0.41866872, 0.062191408, -0.14972427, -0.18488628, -0.37027854, -0.14803104, 0.23587811, 0.33285886, 0.059688937, 0.030515533, 0.16795416, 0.3813925, 0.0755207, 0.15504116, -0.003507182, -0.08249321, 0.24292688, 0.13771294, 0.08057683, 0.016365156, -0.12878628, 0.1833687, 0.17496476, 0.050333332, 0.008188007, 0.32129762, 0.15476923, 0.2052587, -0.060781036, -0.1502798, -0.10187848, 0.11062117, -0.41137248, 0.016532877, 0.107270226, 0.08759128, 0.011842419, -0.17039144, -0.0139911, -0.13244899, -0.23845059, 0.075682834, -0.052250806, -0.30011725, -0.28581655, -0.00055503653, 0.022204043, -0.08598292, -0.24763824, 0.08245162, -0.39607832, 0.008443992, 0.16124122, 0.08812278, 0.0335653, -0.09692297, 0.07613783, 0.033542078, 0.11447116, -0.0069911424, 0.09004892, 0.09898015, 0.14595516, 0.24977732, -0.0018444546, 0.06290809, 0.013354713, -0.10336537, -0.1028908, 0.31109008, -0.110210516, 0.07165067, 0.050161615, -0.11413514}, - expected: true, - }, - } - - t.Run("insert test objects", func(t *testing.T) { - for id, props := range tests { - err := repo.PutObject(context.Background(), &models.Object{Class: className, ID: id}, props.inputVec, nil) - require.Nil(t, err) - } - }) - - t.Run("perform nearVector search by distance", func(t *testing.T) { - results, err := repo.VectorSearch(context.Background(), dto.GetParams{ - ClassName: className, - Pagination: &filters.Pagination{Limit: filters.LimitFlagSearchByDist}, - NearVector: &searchparams.NearVector{ - Distance: 0.1, - }, - SearchVector: searchVector, - AdditionalProperties: additional.Properties{Distance: true}, - }) - require.Nil(t, err) - require.NotEmpty(t, results) - // ensure that we receive more results than - // the `QueryMaximumResults`, as this should - // only apply to limited vector searches - require.Greater(t, len(results), 1) - - for _, res := range results { - if props, ok := tests[res.ID]; !ok { - t.Fatalf("received unexpected result: %+v", res) - } else { - assert.True(t, props.expected, "result id was not intended to meet threshold %s", res.ID) - } - } - }) - - t.Run("perform nearObject search by distance", func(t *testing.T) { - results, err := repo.VectorSearch(context.Background(), dto.GetParams{ - ClassName: className, - Pagination: &filters.Pagination{Limit: filters.LimitFlagSearchByDist}, - NearObject: &searchparams.NearObject{ - Distance: 0.1, - ID: searchObject.String(), - }, - SearchVector: searchVector, - AdditionalProperties: additional.Properties{Distance: true}, - }) - require.Nil(t, err) - require.NotEmpty(t, results) - // ensure that we receive more results than - // the `QueryMaximumResults`, as this should - // only apply to limited vector searches - require.Greater(t, len(results), 1) - - for _, res := range results { - if props, ok := tests[res.ID]; !ok { - t.Fatalf("received unexpected result: %+v", res) - } else { - assert.True(t, props.expected, "result id was not intended to meet threshold %s", res.ID) - } - } - }) -} - -func TestVectorSearch_ByCertainty(t *testing.T) { - className := "SomeClass" - var class *models.Class - - dirName := t.TempDir() - logger, _ := test.NewNullLogger() - - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - // this is set really low to ensure that search - // by distance is conducted, which executes - // without regard to this value - QueryMaximumResults: 1, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - defer repo.Shutdown(context.Background()) - migrator := NewMigrator(repo, logger) - - t.Run("create required schema", func(t *testing.T) { - class = &models.Class{ - Class: className, - Properties: []*models.Property{ - { - DataType: []string{string(schema.DataTypeInt)}, - Name: "int_prop", - }, - }, - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - } - require.Nil(t, - migrator.AddClass(context.Background(), class, schemaGetter.shardState)) - }) - - // update schema getter so it's in sync with class - schemaGetter.schema = schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{class}, - }, - } - - searchVector := []float32{-0.10190568, -0.06259751, 0.05616188, -0.19249836, 0.09714927, -0.1902525, -0.064424865, -0.0387358, 0.17581701, 0.4476738, 0.29261824, 0.12026761, -0.19975126, 0.023600178, 0.17348698, 0.12701738, -0.36018127, -0.12051587, -0.17620522, 0.060741074, -0.064512916, 0.18640806, -0.1529852, 0.08211839, -0.02558465, -0.11369845, 0.0924098, -0.10544433, -0.14728987, -0.041860342, -0.08533595, 0.25886244, 0.2963937, 0.26010615, 0.2111097, 0.029396622, 0.01429563, 0.06410264, -0.119665794, 0.33583277, -0.05802661, 0.023306102, 0.14435922, -0.003951336, -0.13870825, 0.07140894, 0.10469943, -0.059021875, -0.065911904, 0.024216041, -0.26282874, 0.04896568, -0.08291928, -0.12793182, -0.077824734, 0.08843151, 0.31247458, -0.066301286, 0.006904921, -0.08277095, 0.13936226, -0.64392364, -0.19566211, 0.047227614, 0.086121306, -0.20725192, -0.096485816, -0.16436341, -0.06559169, -0.019639932, -0.012729637, 0.08901619, 0.0015896161, -0.24789932, 0.35496348, -0.16272856, -0.01648429, 0.11247674, 0.08099968, 0.13339259, 0.055829972, -0.34662855, 0.068509, 0.13880715, 0.3201848, -0.055557363, 0.22142135, -0.12867308, 0.0037871755, 0.24888979, -0.007443307, 0.08906625, -0.02022331, 0.11510742, -0.2385861, 0.16177008, -0.16214795, -0.28715602, 0.016784908, 0.19386634, -0.07731616, -0.100485384, 0.4100712, 0.061834496, -0.2325293, -0.026056025, -0.11632323, -0.17040555, -0.081960455, -0.0061040106, -0.05949373, 0.044952348, -0.079565264, 0.024430245, -0.09375341, -0.30249637, 0.115205586, -0.13083287, -0.04264671, -0.089810364, 0.16227561, 0.07318055, -0.10496504, 0.00063501706, -0.04936106, -0.0022282854, 1.0893154, 0.1698662, -0.019563455, -0.011128426, 0.04477475, -0.15656771, -0.056911886, -0.5759019, -0.1881429, 0.17088258, 0.24124439, 0.111288875, -0.0015475494, -0.021278847, -0.08362156, 0.09997524, -0.094385885, -0.1674031, 0.061180864, 0.28517494, -0.016217072, 0.025866214, -0.22854298, -0.17924422, -0.037767246, 0.12252907, -0.31698978, -0.038031228, 0.055408552, 0.1743545, -0.040576655, 0.1293942, -0.56650764, -0.10306195, -0.19548112, -0.245544, -0.018241389, -0.039024632, -0.31659162, 0.1565075, 0.08412337, 0.13177724, -0.13766576, -0.15355161, -0.16960397, -0.012436442, 0.04828157, 0.12566057, -0.35308784, -0.37520224, -0.1265899, -0.13991497, 0.14402144, 0.117542416, -0.20750546, -0.5849919, -0.010469457, -0.19677396, 0.011365964, 0.00666846, -0.083470255, 0.24928358, 0.07026387, 0.19082819, 0.24557637, 0.014292963, 0.14846677, 0.031625308, -0.20398879, 0.19507346, -0.18119761, -0.045725327, -0.042455163, -0.099733196, -0.33636123, -0.28447086, 0.30274838, -0.01603988, -0.0529655, 0.15784146, 0.08746072, -0.1703993, 0.2414512, 0.060322937, -0.00812057, 0.031162385, -0.1764905, 0.22107981, -0.016657066, 0.31948856, 0.07282925, -0.036991462, 0.01266936, -0.009106514, -0.038732465, 0.20973183, 0.033236098, -0.10673938, -0.06880061, 0.115524575, -0.39688373, 0.08749971, -0.21816005, -0.22100002, -0.3716853, -0.14720486, 0.24316181, 0.29673144, 0.020808747, 0.07658521, 0.16310681, 0.38785335, 0.0992224, 0.14177811, 0.025954131, -0.08690783, 0.19653428, 0.09584941, 0.040072605, -0.00038361162, -0.094546966, 0.1910902, 0.13217318, 0.060072783, -0.0655816, 0.2777626, 0.1799169, 0.20187178, -0.0996889, -0.01932122, -0.13133621, 0.057482753, -0.36892185, -0.032093313, 0.14607865, 0.12033318, -0.041683596, -0.2048406, -0.041777443, -0.14975598, -0.2526341, 0.12659752, 0.010567178, -0.297333, -0.27522174, 0.06923473, 0.043150593, -0.017045585, -0.2400216, 0.11413547, -0.40081662, -0.0018820907, 0.13800722, 0.085972115, -0.01519989, -0.10491216, 0.09170084, 0.063085504, 0.046743374, -0.014466267, 0.09880224, 0.027706565, 0.09951337, 0.17317492, -0.025654864, 0.14658073, 0.042377427, -0.08402882, -0.12423425, 0.32714987, -0.1527207, 0.106094465, 0.017378228, -0.06302387} - searchObject := strfmt.UUID("fe687bf4-f10f-4c23-948d-0746ea2927b3") - - tests := map[strfmt.UUID]struct { - inputVec []float32 - expected bool - }{ - strfmt.UUID("88460290-03b2-44a3-9adb-9fa3ae11d9e6"): { - inputVec: []float32{-0.11015724, -0.05380307, 0.027512914, -0.16925375, 0.08306809, -0.19312492, -0.08910436, -0.011051652, 0.17981204, 0.40469593, 0.28226805, 0.09381516, -0.18380599, 0.03102771, 0.1645333, 0.1530153, -0.3187937, -0.10800173, -0.18466279, 0.0004336393, -0.0495677, 0.19905856, -0.11614494, 0.08834681, -0.011200292, -0.11969374, 0.12497086, -0.12427251, -0.13395442, -0.0060353535, -0.07504816, 0.23205791, 0.2982508, 0.2517544, 0.176147, -0.036871903, 0.017852835, 0.040007118, -0.118621, 0.3648693, -0.058933854, 0.04004229, 0.11871147, -0.019860389, -0.12701912, 0.106662825, 0.086498804, -0.04303973, -0.0742352, 0.018250324, -0.26544014, 0.029228423, -0.087171465, -0.1282789, -0.06403083, 0.09680911, 0.31433868, -0.081510685, -0.011283603, -0.041624587, 0.16530018, -0.6714878, -0.2436993, 0.03173918, 0.106117725, -0.20803581, -0.10429562, -0.16975354, -0.078582145, -0.0065962705, -0.06840946, 0.094937086, -0.020617036, -0.23795949, 0.34785536, -0.19834635, -0.015064479, 0.11930141, 0.090962164, 0.120560184, 0.054095767, -0.38602966, 0.057141174, 0.12039684, 0.32000408, -0.05146908, 0.20762976, -0.09342379, 0.037577383, 0.23894139, -0.0075003104, 0.104791366, -0.015841056, 0.102840215, -0.20813248, 0.1855997, -0.12594056, -0.27132365, -0.0055563124, 0.21954241, -0.10798524, -0.111896284, 0.44049335, 0.049884494, -0.22339955, -0.005374135, -0.120713554, -0.22275059, -0.09146004, 0.017188415, -0.106493734, 0.045247544, -0.07725446, 0.056848228, -0.10294392, -0.2896642, 0.112891, -0.13773362, -0.089911595, -0.13500965, 0.14051703, 0.040092673, -0.13896292, 0.04580957, -0.014300959, 0.03737215, 1.0661443, 0.19767477, -0.07703914, -0.012910904, -0.0037716173, -0.14437087, -0.06938004, -0.5348036, -0.16047458, 0.19416414, 0.21938956, 0.092242256, -0.012630808, -0.021863988, -0.051702406, 0.08780951, -0.0815602, -0.15332024, 0.077632725, 0.25709584, -0.025725808, 0.042116437, -0.22687604, -0.11791685, -0.028626656, 0.16734225, -0.3017483, -0.03236202, 0.02888077, 0.18193199, -0.009032297, 0.14454253, -0.511494, -0.12119192, -0.20757924, -0.2561716, -0.03904554, -0.07348411, -0.28547177, 0.15967208, 0.079396725, 0.14358875, -0.12829632, -0.18175666, -0.15540425, -0.020419862, 0.019070208, 0.12168836, -0.3428434, -0.357543, -0.11218741, -0.12834033, 0.13564876, 0.12768728, -0.1817461, -0.61235875, -0.029409664, -0.19765733, 0.03872163, 0.0074950717, -0.10025679, 0.2872255, 0.033995092, 0.12945211, 0.21831632, 0.04666009, 0.14233032, 0.016767867, -0.2039244, 0.2000191, -0.13099428, -0.020210614, -0.06286195, -0.0948797, -0.34830436, -0.21595824, 0.32722405, -0.024735296, -0.07859145, 0.16975155, 0.08186461, -0.19249061, 0.23405583, 0.04837592, 0.021467948, -0.022215014, -0.14892808, 0.23908867, -0.050126728, 0.2867907, 0.07740656, -0.01714987, -0.0046314416, -0.0048108613, -0.007407311, 0.1807499, 0.049772616, -0.14680666, -0.07335314, 0.09023705, -0.40600133, 0.05522128, -0.20085222, -0.20410904, -0.34319055, -0.10792269, 0.2297779, 0.30397663, 0.05230268, 0.06408224, 0.13797496, 0.3691112, 0.083033495, 0.13695791, -0.015612457, -0.06413475, 0.18117142, 0.12928344, 0.049171276, 0.016104931, -0.102417335, 0.19589683, 0.14380622, 0.0748065, -0.005402455, 0.27243868, 0.14925551, 0.19564849, -0.10738364, -0.054175537, -0.10068278, 0.06004795, -0.38755924, -0.01654251, 0.1394104, 0.0968949, 0.004271706, -0.17105898, -0.050423585, -0.15311627, -0.24458972, 0.12665795, -0.022814916, -0.23887472, -0.289588, 0.05521137, 0.041259795, -0.021133862, -0.23674431, 0.08424598, -0.37863016, 0.017239956, 0.13776784, 0.060790475, 0.057887543, -0.08784489, 0.08803934, 0.027996546, 0.085972995, -0.014455558, 0.11668073, 0.03812387, 0.088413864, 0.22228678, -0.015599858, 0.11000236, 0.035271563, -0.08088438, -0.13092226, 0.29378533, -0.12311522, 0.09377676, 0.02948718, -0.09136077}, - expected: true, - }, - strfmt.UUID("c99bc97d-7035-4311-94f3-947dc6471f51"): { - inputVec: []float32{-0.07545987, -0.046643265, 0.044445477, -0.18531442, 0.07922216, -0.19388637, -0.069393866, -0.036144026, 0.1713317, 0.41803706, 0.23576374, 0.073170714, -0.14085358, 0.012535708, 0.17439325, 0.10057567, -0.33506152, -0.06800867, -0.18882714, 0.002687021, -0.03276807, 0.17267752, -0.13951558, 0.071382746, 0.020254405, -0.10178502, 0.13977699, -0.107296936, -0.113307, -0.002506761, -0.092065684, 0.21008658, 0.31157792, 0.19640765, 0.15769793, -0.0050196033, 0.0022481605, 0.015436289, -0.11822955, 0.31494477, -0.07425527, 0.051401984, 0.11648046, -0.00016831602, -0.12758006, 0.06814693, 0.06108981, -0.025454175, -0.018695071, 0.041827776, -0.23480764, 0.06652185, -0.078328855, -0.121668324, -0.04341973, 0.114403985, 0.32614416, -0.07992741, -0.019665314, -0.017408244, 0.12615794, -0.6350545, -0.17056493, 0.07171332, 0.047071394, -0.18428493, -0.09011123, -0.15995751, -0.03345579, -0.014678, -0.038699757, 0.044125225, 0.0042562615, -0.29445595, 0.30460796, -0.13630153, 0.00014055961, 0.08996278, 0.08948901, 0.12164838, 0.079090506, -0.36153567, 0.02817218, 0.11914518, 0.29805067, -0.07431765, 0.16793592, -0.099549234, 0.045226075, 0.22235383, -0.045654725, 0.09233901, -0.004902314, 0.08621588, -0.19723448, 0.19557382, -0.13199815, -0.22924824, -0.015981175, 0.19762704, -0.08940076, -0.084909916, 0.43372774, 0.026998578, -0.20827708, 0.037450224, -0.078997016, -0.18065391, -0.071308024, 0.00870316, -0.114981964, 0.017085023, -0.07696264, 0.009330409, -0.097458, -0.25530958, 0.118254915, -0.12516825, -0.008301536, -0.20432962, 0.15235707, 0.012840041, -0.18034773, 0.039270073, -0.03131139, 0.013706253, 0.98688674, 0.18840753, -0.055119563, 0.00836046, 0.019445436, -0.10701598, -0.024610046, -0.50088006, -0.15488546, 0.14209819, 0.1798376, 0.08615982, -0.0119235935, -0.0060070297, -0.08406098, 0.10096481, -0.09077014, -0.15957798, 0.10556352, 0.2100476, -0.036947068, 0.05316554, -0.20480183, -0.14873864, -0.0069070593, 0.16027303, -0.288908, -0.04487129, 0.0705415, 0.11973847, -0.0017247469, 0.14092937, -0.5262047, -0.094283305, -0.19120996, -0.2816572, -0.010916339, -0.07984056, -0.28659204, 0.13706332, 0.07364347, 0.12300072, -0.17554194, -0.16378267, -0.15244205, 0.00075927645, 0.017289847, 0.12072629, -0.33452734, -0.33727616, -0.12780978, -0.09350711, 0.105674624, 0.10770573, -0.17278843, -0.5760599, -0.013741414, -0.15395893, 0.009837732, 0.015417911, -0.11384676, 0.24567491, 0.04905973, 0.10762609, 0.2131752, 0.019281652, 0.11665857, 0.022718405, -0.2234067, 0.23241606, -0.12194457, -0.049972955, -0.012225418, -0.14856412, -0.386102, -0.23018965, 0.28920102, -0.023396742, -0.114672944, 0.12130062, 0.05654803, -0.16194181, 0.24095012, 0.03644393, 0.028024165, -0.008832254, -0.16496961, 0.19496499, -0.035887964, 0.25981775, 0.0970074, 0.0013458093, -0.009548204, 0.040741496, -0.019192837, 0.20718361, -0.004034228, -0.1343262, -0.06990001, 0.09888768, -0.35942966, 0.043895893, -0.19182123, -0.17963983, -0.3222771, -0.10223457, 0.23866613, 0.25855777, 0.04051543, 0.08756274, 0.15683484, 0.37856522, 0.04853359, 0.10198129, -0.0061066896, -0.049892712, 0.17087941, 0.14563805, 0.06984385, 0.0071270005, -0.11838641, 0.18716812, 0.14013803, 0.05242403, 0.034357738, 0.3083466, 0.14742611, 0.17841975, -0.124118194, -0.014102871, -0.052544866, 0.037493005, -0.33485797, -0.013164912, 0.1066288, 0.11141791, -0.04029921, -0.16429856, -0.032241724, -0.15965424, -0.2430594, 0.13654563, 0.009401224, -0.2045843, -0.28467956, 0.07325551, 0.027996557, -0.033877768, -0.24350801, 0.08329816, -0.35555813, 0.006908567, 0.07227365, 0.03188268, 0.032559503, -0.09180395, 0.05601515, 0.0047281734, 0.06878795, -0.018943194, 0.08251342, 0.042039152, 0.12902294, 0.20526606, -0.014881293, 0.11723917, 0.0115632, -0.09016013, -0.12117223, 0.31020245, -0.111444525, 0.077845715, 0.00046715315, -0.104099475}, - expected: true, - }, - strfmt.UUID("fe687bf4-f10f-4c23-948d-0746ea2927b3"): { - inputVec: []float32{-0.20739016, -0.19551805, 0.06645163, 0.008650202, 0.03700748, -0.04132599, -0.029881354, 0.04684896, 0.096614264, 0.42888844, 0.10003969, 0.026234219, -0.051639702, -0.118660435, 0.14473079, 0.2911885, -0.1180539, -0.16804434, -0.48081538, 0.021702053, 0.12612472, 0.15442817, -0.05836532, 0.074295096, -0.28077397, -0.24297802, 0.047836643, -0.36753318, -0.30482984, 0.09265357, 0.25571078, 0.41130066, 0.46177864, 0.34033778, 0.20721313, -0.37726295, 0.07721501, 0.08009689, 0.00027321206, 0.5168123, -0.15305339, 0.0937765, 0.096195236, -0.21120761, 0.014014921, 0.3133104, 0.20773117, 0.08483507, -0.27784437, -0.17281856, -0.6050923, -0.22439326, -0.16914369, -0.3149047, -0.13828672, 0.16334395, -0.0018224253, -0.024342008, 0.3511251, 0.04979151, 0.34223744, -0.6965703, -0.36211932, -0.27092442, 0.34418032, -0.09667905, 0.13344757, -0.15622364, -0.24129291, 0.06958589, -0.2681816, -0.09497071, -0.08923615, -0.06642436, 0.48688608, -0.33535984, 0.014242731, 0.079838976, 0.32949054, 0.09051045, -0.2653392, -0.47393548, 0.07508276, 0.0062832804, 0.724184, -0.18929236, 0.11718613, 0.049603477, 0.08766128, 0.31040704, 0.04038693, -0.0017023507, -0.18986607, 0.056264438, -0.20978904, -0.107441366, -0.30505633, -0.45781082, -0.11571784, 0.32160303, -0.1347523, -0.08090298, 0.51651996, -0.023250414, -0.18725531, -0.14222279, 0.009277832, -0.49789724, -0.25156206, 0.0042495225, 0.0038805408, -0.031416763, 0.10277136, 0.14383446, -0.23241928, -0.42357358, 0.027033398, -0.2262604, -0.2685295, -0.14510548, 0.18256307, 0.063297585, 0.027636252, 0.081166506, 0.06726344, 0.1677495, 1.5217289, 0.33152232, -0.2209926, 0.051426213, 0.15640806, -0.30210486, -0.32857975, -0.4170022, -0.028293105, 0.28772062, 0.50510746, 0.09162247, -0.12383193, -0.25066972, -0.1441897, 0.107192926, -0.07404076, 0.0042472635, 0.11014519, 0.22332853, 0.09434378, -0.3278343, 0.041899726, 0.06838457, 0.10983681, 0.11864574, -0.25336757, -0.047530346, -0.027303243, 0.37403497, 0.13420461, 0.14946426, -0.41996637, -0.037703935, -0.47961184, -0.29839846, -0.103934005, -0.12058302, -0.12806267, 0.22814582, 0.3904893, -0.16044962, -0.17479864, -0.33139735, -0.29185295, 0.0653074, 0.042426735, 0.06092335, -0.18776153, -0.52555144, -0.15889317, -0.20644087, 0.2293067, 0.26668283, -0.15607063, -0.696593, -0.08224992, -0.4283747, 0.26883888, -0.031052848, -0.1311875, 0.26636878, 0.16457985, 0.15660451, 0.10629464, 0.17345549, 0.23963387, 0.22997221, -0.111713186, -0.08499592, -0.2274625, 0.19285984, -0.08285016, -0.02692149, -0.3426618, -0.13361897, 0.2870389, -0.12032792, -0.22944619, 0.25588584, 0.24607788, -0.2762531, 0.30983892, 0.011088746, -0.15739818, 0.053215, -0.21660997, 0.033805694, -0.17886437, 0.2979239, 0.2163545, -0.08381542, 0.19666128, -0.28977823, -0.20994817, -0.012160099, 0.057499636, -0.12549455, 0.19303595, -0.14420606, -0.51937664, 0.23400985, -0.27893808, -0.2660984, -0.27870297, -0.32149136, 0.19958079, 0.34468395, 0.18947665, -0.16529581, 0.101419374, 0.30195153, 0.09030288, 0.12496541, 0.02999903, -0.016697621, 0.15314853, 0.27848768, 0.24102053, 0.06933273, 0.08923653, 0.10477832, 0.4389032, 0.15679164, -0.11119637, 0.134823, 0.30230528, 0.20818473, -0.005579584, -0.3474488, -0.44394243, 0.22270252, -0.3668763, 0.07474772, 0.011691334, 0.088187896, 0.23832949, -0.07960201, 0.066471875, 0.034641538, -0.39984587, 0.0032980456, -0.28492525, -0.46358657, -0.2148288, -0.107226945, 0.02734428, -0.24686679, -0.123900555, 0.18174778, -0.31248868, 0.13808723, 0.31549984, 0.21521719, 0.13966985, -0.27272752, 0.12091104, 0.14257833, 0.23175247, 0.15639938, 0.40828535, 0.31916845, 0.023645567, 0.20658277, -0.20365283, 0.113746524, 0.13173752, -0.050343305, -0.31581175, 0.09704622, -0.014172505, 0.16924341, 0.30327854, -0.17770194}, - expected: false, - }, - strfmt.UUID("e7bf6c45-de72-493a-b273-5ef198974d61"): { - inputVec: []float32{0.089313604, -0.050221898, 0.18352903, 0.16257699, 0.14520381, 0.17993976, 0.14594483, 0.019256027, -0.15505213, 0.23606326, -0.14456263, 0.2679586, -0.112208664, 0.12997514, 0.0051072896, 0.28151348, -0.10495799, 0.026782967, -0.38603118, 0.16190273, -0.0428943, -0.16265322, -0.17910561, 0.0746288, -0.3117934, -0.15871756, -0.11377734, -0.06822346, -0.13829489, 0.13019162, 0.30741218, 0.16194165, 0.013218932, 0.054517113, 0.12490437, -0.07709048, 0.02556826, -0.21159878, -0.09082174, 0.24629511, 0.05013666, 0.25168124, -0.14423938, -0.0937688, -0.07811525, -0.049346007, 0.3592527, 0.30411252, -0.1168557, 0.18870471, 0.06614835, -0.20099068, -0.084436245, 0.073036775, -0.03448665, -0.11147946, -0.10862863, -0.012393957, 0.18990599, 0.060957544, 0.19518377, -0.027541652, -0.26750082, -0.12780671, 0.09570065, -0.03541132, 0.094820626, -0.13539355, -0.09468136, 0.18476579, -0.20970085, -0.20989786, -0.12084438, -0.04517079, -0.008074663, 0.02824076, 0.114496395, -0.20462593, 0.103516705, -0.101554185, -0.1374868, -0.24884155, -0.08101618, -0.016105993, 0.22608215, -0.007247754, -0.17246912, 0.058247145, -0.041018173, 0.19471274, -0.022576109, 0.032828204, -0.079321206, -0.09259324, 0.041115705, -0.25280195, -0.28517374, -0.19496292, 0.18070905, 0.06384923, -0.004056949, 0.1536253, 0.17861623, -0.033833142, 0.12039968, 0.04458716, 0.08793809, -0.15683243, -0.1087904, 0.1741014, 0.007256374, -0.20265253, 0.034111258, 0.03311363, -0.09449356, -0.13161612, -0.026084669, 0.07609202, 0.03452338, 0.08840356, -0.044566724, 0.1507175, 0.089273594, 0.18872644, 0.18333815, -0.023196407, 0.63831943, 0.20309874, 0.10217627, 0.11445079, 0.18965706, -0.16809432, -0.343172, -0.06439529, 0.08362327, 0.32746288, 0.38483366, 0.020372175, -0.25239283, 0.019468365, -0.016367752, 0.016749177, 0.024621855, 0.030529505, 0.20601188, -0.100692995, -0.16414656, -0.23193358, 0.26616478, 0.06166736, 0.14341855, 0.1294041, 0.045133967, 0.0014262896, -0.0194398, 0.040737696, 0.10099013, -0.10838136, -0.28768313, -0.073719576, -0.15836753, -0.10482511, -0.1349642, -0.107005455, 0.01957546, 0.13799994, 0.056444198, -0.38841644, -0.07585945, -0.018703599, -0.19934878, 0.15176265, 0.04133126, 0.063531734, 0.09720055, -0.29999572, 0.04765686, -0.23604262, 0.081500284, 0.056092553, -0.13664724, -0.37729686, 0.031137427, -0.052083906, 0.117984496, -0.14562207, -0.029609507, 0.13725121, 0.090367764, 0.12787215, 0.11026589, 0.25123242, 0.12911159, 0.055398554, 0.0032232201, 0.026706887, 0.14584258, 0.019900957, -0.12197998, -0.087177716, -0.24649806, -0.17869286, 0.07139921, -0.09633085, -0.16027117, 0.23617831, 0.05429949, -0.061085824, 0.040451035, 0.052443117, -0.14255014, 0.15598148, -0.2336374, 0.08394173, -0.34318882, 0.3419207, 0.18282516, -0.03709172, 0.10525048, -0.1871602, -0.22663523, 0.01635051, 0.16996534, -0.18056048, -0.169894, -0.18467705, -0.3641231, 0.060861763, -0.080082566, -0.08888943, 0.11629789, -0.00973362, 0.07452957, 0.25680214, 0.042024083, -0.024963235, 0.1743134, 0.10921186, 0.25191578, 0.028438354, 0.004781374, -0.08364819, 0.051807538, 0.1165724, 0.29184434, -0.21512283, 0.12515399, -0.08803969, 0.41930157, -0.10181762, 0.038189832, 0.085555896, -0.026453126, 0.04717047, 0.12667313, 0.023158737, -0.45877644, 0.18732828, 0.062374037, -0.21956007, -0.04449947, 0.19028638, 0.1359094, 0.26384917, 0.077602044, 0.35136092, 0.069637895, 0.048263475, -0.02498448, -0.09221205, -0.012142404, -0.124592446, 0.14599627, -0.050875153, -0.25454503, -0.069588415, -0.29793787, -0.13407284, 0.25388947, 0.35565627, -0.034204755, 0.0024766966, 0.086427726, -0.054318108, 0.063218184, -0.037823644, 0.108287826, 0.14440496, 0.025134278, 0.14978257, -0.03355889, 0.02980915, -0.13764386, 0.4167542, -0.03938922, 0.026970355, 0.24595529, 0.111741625, -0.074567944, -0.057232533}, - expected: false, - }, - strfmt.UUID("0999d109-1d5f-465a-bd8b-e3fbd46f10aa"): { - inputVec: []float32{-0.10486144, -0.07437922, 0.069469325, -0.1438278, 0.07740161, -0.18606456, -0.09991434, -0.020051572, 0.19863395, 0.4347328, 0.297606, 0.07853262, -0.16025662, 0.023596637, 0.16935731, 0.17052403, -0.29870638, -0.10309007, -0.20055692, 0.0027809117, -0.03928043, 0.21178603, -0.13793766, 0.08118157, 0.006693433, -0.13829204, 0.14778963, -0.13180175, -0.21128704, -0.0026104634, -0.076393716, 0.22200249, 0.32417125, 0.26045212, 0.1783609, -0.114116184, 0.0100981165, 0.07233143, -0.15913877, 0.4238603, -0.036907215, 0.0595873, 0.0807002, -0.07637312, -0.12889846, 0.111177936, 0.091114685, -0.018454906, -0.12132672, 0.056664582, -0.30461523, 0.020763714, -0.10992191, -0.14430659, -0.092879646, 0.13615008, 0.33039626, -0.115675874, 0.03607886, -0.027918883, 0.19531779, -0.7211654, -0.23073879, 0.011791817, 0.1315166, -0.22779183, -0.13773227, -0.1814997, -0.09008116, 0.021698939, -0.102921166, 0.090760864, 0.011856942, -0.25561005, 0.40769714, -0.21286584, -0.018059848, 0.13812906, 0.079457305, 0.12631191, 0.0024881593, -0.4282836, 0.0619608, 0.12207897, 0.39083096, -0.009502015, 0.19990632, -0.06503092, 0.0635979, 0.27579078, -0.020699967, 0.068474516, 0.0043831975, 0.10303624, -0.1885405, 0.22989234, -0.15952443, -0.29842895, 0.006752088, 0.22831629, -0.13150804, -0.13695218, 0.5357904, 0.050116863, -0.24064547, -0.01375713, -0.096647836, -0.24984525, -0.10429946, 0.002098812, -0.08113263, 0.05237009, -0.10246039, 0.05234802, -0.13899775, -0.3439524, 0.12522809, -0.18406768, -0.09022853, -0.19954625, 0.15810682, 0.039185096, -0.13576287, 0.045047805, 0.0035671506, 0.055920787, 1.1730403, 0.24019612, -0.13423051, -0.008052084, -0.00431602, -0.17079304, -0.09064658, -0.58728856, -0.1365065, 0.22919424, 0.22795208, 0.13396585, 0.018962797, -0.0075796233, -0.072394304, 0.10908417, -0.10881145, -0.16565171, 0.10378018, 0.27296618, -0.059810717, 0.03355443, -0.22429268, -0.12499127, -0.0441017, 0.20800696, -0.29992488, -0.003536096, 0.0026575085, 0.2427503, -0.007395092, 0.13233404, -0.5494433, -0.13144702, -0.2899963, -0.27367246, -0.05257514, -0.0939783, -0.267614, 0.16651331, 0.13891254, 0.08047202, -0.14046521, -0.19062972, -0.1433134, 0.0067776316, 0.00207368, 0.12986982, -0.35847133, -0.41852546, -0.15541135, -0.09865207, 0.14805861, 0.17072491, -0.22655731, -0.6473966, -0.007884447, -0.2060257, 0.035390265, 0.02781265, -0.09760371, 0.30535778, 0.047540557, 0.14565119, 0.21733035, 0.06558403, 0.13184759, 0.044231005, -0.22218557, 0.1897204, -0.1596938, 0.017510587, -0.030249557, -0.082377456, -0.39669412, -0.18365891, 0.34806964, -0.024830062, -0.06955674, 0.21521395, 0.1201222, -0.21855503, 0.23522708, 0.038058903, -0.019610198, -0.025448406, -0.18122384, 0.26068974, -0.055872105, 0.29595166, 0.11005987, -0.00841942, 0.006325112, -0.0013332894, -0.025598384, 0.17320716, 0.03480282, -0.1504056, -0.07133905, 0.08367911, -0.41866872, 0.062191408, -0.14972427, -0.18488628, -0.37027854, -0.14803104, 0.23587811, 0.33285886, 0.059688937, 0.030515533, 0.16795416, 0.3813925, 0.0755207, 0.15504116, -0.003507182, -0.08249321, 0.24292688, 0.13771294, 0.08057683, 0.016365156, -0.12878628, 0.1833687, 0.17496476, 0.050333332, 0.008188007, 0.32129762, 0.15476923, 0.2052587, -0.060781036, -0.1502798, -0.10187848, 0.11062117, -0.41137248, 0.016532877, 0.107270226, 0.08759128, 0.011842419, -0.17039144, -0.0139911, -0.13244899, -0.23845059, 0.075682834, -0.052250806, -0.30011725, -0.28581655, -0.00055503653, 0.022204043, -0.08598292, -0.24763824, 0.08245162, -0.39607832, 0.008443992, 0.16124122, 0.08812278, 0.0335653, -0.09692297, 0.07613783, 0.033542078, 0.11447116, -0.0069911424, 0.09004892, 0.09898015, 0.14595516, 0.24977732, -0.0018444546, 0.06290809, 0.013354713, -0.10336537, -0.1028908, 0.31109008, -0.110210516, 0.07165067, 0.050161615, -0.11413514}, - expected: true, - }, - } - - t.Run("insert test objects", func(t *testing.T) { - for id, props := range tests { - err := repo.PutObject(context.Background(), &models.Object{Class: className, ID: id}, props.inputVec, nil) - require.Nil(t, err) - } - }) - - t.Run("perform nearVector search by distance", func(t *testing.T) { - results, err := repo.VectorSearch(context.Background(), dto.GetParams{ - ClassName: className, - Pagination: &filters.Pagination{Limit: filters.LimitFlagSearchByDist}, - NearVector: &searchparams.NearVector{ - Certainty: 0.9, - }, - SearchVector: searchVector, - AdditionalProperties: additional.Properties{Certainty: true}, - }) - require.Nil(t, err) - require.NotEmpty(t, results) - // ensure that we receive more results than - // the `QueryMaximumResults`, as this should - // only apply to limited vector searches - require.Greater(t, len(results), 1) - - for _, res := range results { - if props, ok := tests[res.ID]; !ok { - t.Fatalf("received unexpected result: %+v", res) - } else { - assert.True(t, props.expected, "result id was not intended to meet threshold %s", res.ID) - } - } - }) - - t.Run("perform nearObject search by distance", func(t *testing.T) { - results, err := repo.VectorSearch(context.Background(), dto.GetParams{ - ClassName: className, - Pagination: &filters.Pagination{Limit: filters.LimitFlagSearchByDist}, - NearObject: &searchparams.NearObject{ - Certainty: 0.9, - ID: searchObject.String(), - }, - SearchVector: searchVector, - AdditionalProperties: additional.Properties{Certainty: true}, - }) - require.Nil(t, err) - require.NotEmpty(t, results) - // ensure that we receive more results than - // the `QueryMaximumResults`, as this should - // only apply to limited vector searches - require.Greater(t, len(results), 1) - - for _, res := range results { - if props, ok := tests[res.ID]; !ok { - t.Fatalf("received unexpected result: %+v", res) - } else { - assert.True(t, props.expected, "result id was not intended to meet threshold %s", res.ID) - } - } - }) -} - -func Test_PutPatchRestart(t *testing.T) { - dirName := t.TempDir() - logger, _ := test.NewNullLogger() - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) - defer cancel() - - testClass := &models.Class{ - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Class: "PutPatchRestart", - Properties: []*models.Property{ - { - Name: "description", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - } - - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 100, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - defer repo.Shutdown(context.Background()) - require.Nil(t, repo.WaitForStartup(ctx)) - migrator := NewMigrator(repo, logger) - - require.Nil(t, - migrator.AddClass(ctx, testClass, schemaGetter.shardState)) - - // update schema getter so it's in sync with class - schemaGetter.schema = schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{testClass}, - }, - } - - testID := strfmt.UUID("93c31577-922e-4184-87a5-5ac6db12f73c") - testVec := []float32{0.1, 0.2, 0.1, 0.3} - - t.Run("create initial object", func(t *testing.T) { - err = repo.PutObject(ctx, &models.Object{ - ID: testID, - Class: testClass.Class, - Properties: map[string]interface{}{"description": "test object init"}, - }, testVec, nil) - require.Nil(t, err) - }) - - t.Run("repeatedly put with nil vec, patch with vec, and restart", func(t *testing.T) { - for i := 0; i < 10; i++ { - err = repo.PutObject(ctx, &models.Object{ - ID: testID, - Class: testClass.Class, - Properties: map[string]interface{}{ - "description": fmt.Sprintf("test object, put #%d", i+1), - }, - }, nil, nil) - require.Nil(t, err) - - err = repo.Merge(ctx, objects.MergeDocument{ - ID: testID, - Class: testClass.Class, - PrimitiveSchema: map[string]interface{}{ - "description": fmt.Sprintf("test object, patch #%d", i+1), - }, - Vector: testVec, - UpdateTime: time.Now().UnixNano() / int64(time.Millisecond), - }, nil, "") - require.Nil(t, err) - - require.Nil(t, repo.Shutdown(ctx)) - require.Nil(t, repo.WaitForStartup(ctx)) - } - }) - - t.Run("assert the final result is correct", func(t *testing.T) { - findByIDFilter := &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: schema.ClassName(testClass.Class), - Property: filters.InternalPropID, - }, - Value: &filters.Value{ - Value: testID.String(), - Type: schema.DataTypeText, - }, - }, - } - res, err := repo.ObjectSearch(ctx, 0, 10, findByIDFilter, - nil, additional.Properties{}, "") - require.Nil(t, err) - assert.Len(t, res, 1) - - expectedDescription := "test object, patch #10" - resultDescription := res[0].Schema.(map[string]interface{})["description"] - assert.Equal(t, expectedDescription, resultDescription) - }) -} - -func TestCRUDWithEmptyArrays(t *testing.T) { - dirName := t.TempDir() - - logger, _ := test.NewNullLogger() - - class := &models.Class{ - Class: "TestClass", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{ - { - Name: "textArray", - DataType: schema.DataTypeTextArray.PropString(), - }, - { - Name: "numberArray", - DataType: []string{string(schema.DataTypeNumberArray)}, - }, - { - Name: "boolArray", - DataType: []string{string(schema.DataTypeBooleanArray)}, - }, - }, - } - classRefName := "TestRefClass" - classRef := &models.Class{ - Class: classRefName, - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{ - { - Name: "stringProp", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - } - classNameWithRefs := "TestClassWithRefs" - classWithRefs := &models.Class{ - Class: classNameWithRefs, - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{ - { - Name: "stringProp", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "refProp", - DataType: []string{classRefName}, - }, - }, - } - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 100, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - defer repo.Shutdown(context.Background()) - migrator := NewMigrator(repo, logger) - require.Nil(t, - migrator.AddClass(context.Background(), class, schemaGetter.shardState)) - require.Nil(t, - migrator.AddClass(context.Background(), classRef, schemaGetter.shardState)) - require.Nil(t, - migrator.AddClass(context.Background(), classWithRefs, schemaGetter.shardState)) - // update schema getter so it's in sync with class - schemaGetter.schema = schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{class, classRef, classWithRefs}, - }, - } - - t.Run("empty arrays", func(t *testing.T) { - objID := strfmt.UUID("a0b55b05-bc5b-4cc9-b646-1452d1390a62") - obj1 := &models.Object{ - ID: objID, - Class: "TestClass", - Properties: map[string]interface{}{ - "textArray": []string{}, - "numberArray": []float64{}, - "boolArray": []bool{}, - }, - } - obj2 := &models.Object{ - ID: objID, - Class: "TestClass", - Properties: map[string]interface{}{ - "textArray": []string{"value"}, - "numberArray": []float64{0.5}, - "boolArray": []bool{true}, - }, - } - - assert.Nil(t, repo.PutObject(context.Background(), obj1, []float32{1, 3, 5, 0.4}, nil)) - assert.Nil(t, repo.PutObject(context.Background(), obj2, []float32{1, 3, 5, 0.4}, nil)) - - res, err := repo.ObjectByID(context.Background(), objID, nil, additional.Properties{}, "") - require.Nil(t, err) - assert.Equal(t, obj2.Properties, res.ObjectWithVector(false).Properties) - }) - - t.Run("empty references", func(t *testing.T) { - objRefID := strfmt.UUID("a0b55b05-bc5b-4cc9-b646-1452d1390000") - objRef := &models.Object{ - ID: objRefID, - Class: classRefName, - Properties: map[string]interface{}{ - "stringProp": "string prop value", - }, - } - assert.Nil(t, repo.PutObject(context.Background(), objRef, []float32{1, 3, 5, 0.4}, nil)) - - obj1ID := strfmt.UUID("a0b55b05-bc5b-4cc9-b646-1452d1390a62") - obj1 := &models.Object{ - ID: obj1ID, - Class: classNameWithRefs, - Properties: map[string]interface{}{ - "stringProp": "some prop", - // due to the fix introduced in https://github.com/weaviate/weaviate/pull/2320, - // MultipleRef's can appear as empty []interface{} when no actual refs are provided for - // an object's reference property. - // - // when obj1 is unmarshalled from storage, refProp will be represented as []interface{}, - // because it is an empty reference property. so when comparing obj1 with the result of - // repo.Object, we need this refProp here to be a []interface{}. Note that this is due - // to our usage of storobj.Object.MarshallerVersion 1, and future MarshallerVersions may - // not have this ambiguous property type limitation. - "refProp": []interface{}{}, - }, - } - obj2ID := strfmt.UUID("a0b55b05-bc5b-4cc9-b646-1452d1390a63") - obj2 := &models.Object{ - ID: obj2ID, - Class: classNameWithRefs, - Properties: map[string]interface{}{ - "stringProp": "some second prop", - "refProp": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI( - crossref.NewLocalhost(classRefName, objRefID).String()), - }, - }, - }, - } - - assert.Nil(t, repo.PutObject(context.Background(), obj1, []float32{1, 3, 5, 0.4}, nil)) - assert.Nil(t, repo.PutObject(context.Background(), obj2, []float32{1, 3, 5, 0.4}, nil)) - - res, err := repo.Object(context.Background(), classNameWithRefs, obj1ID, nil, - additional.Properties{}, nil, "") - require.Nil(t, err) - assert.NotNil(t, res) - assert.Equal(t, obj1.Properties, res.ObjectWithVector(false).Properties) - - res, err = repo.Object(context.Background(), classNameWithRefs, obj2ID, nil, - additional.Properties{}, nil, "") - require.Nil(t, err) - assert.NotNil(t, res) - assert.Equal(t, obj2.Properties, res.ObjectWithVector(false).Properties) - }) -} - -func TestOverwriteObjects(t *testing.T) { - dirName := t.TempDir() - logger, _ := test.NewNullLogger() - class := &models.Class{ - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Class: "SomeClass", - Properties: []*models.Property{ - { - Name: "stringProp", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - } - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 10, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, - &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - defer repo.Shutdown(context.Background()) - migrator := NewMigrator(repo, logger) - t.Run("create the class", func(t *testing.T) { - require.Nil(t, - migrator.AddClass(context.Background(), class, schemaGetter.shardState)) - }) - // update schema getter so it's in sync with class - schemaGetter.schema = schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{class}, - }, - } - - now := time.Now() - later := now.Add(time.Hour) // time-traveling ;) - stale := &models.Object{ - ID: "981c09f9-67f3-4e6e-a988-c53eaefbd58e", - Class: class.Class, - CreationTimeUnix: now.UnixMilli(), - LastUpdateTimeUnix: now.UnixMilli(), - Properties: map[string]interface{}{ - "oldValue": "how things used to be", - }, - Vector: []float32{1, 2, 3}, - VectorWeights: (map[string]string)(nil), - Additional: models.AdditionalProperties{}, - } - - fresh := &models.Object{ - ID: "981c09f9-67f3-4e6e-a988-c53eaefbd58e", - Class: class.Class, - CreationTimeUnix: now.UnixMilli(), - LastUpdateTimeUnix: later.UnixMilli(), - Properties: map[string]interface{}{ - "oldValue": "how things used to be", - "newValue": "how they are now", - }, - Vector: []float32{4, 5, 6}, - VectorWeights: (map[string]string)(nil), - Additional: models.AdditionalProperties{}, - } - - t.Run("insert stale object", func(t *testing.T) { - err := repo.PutObject(context.Background(), stale, stale.Vector, nil) - require.Nil(t, err) - }) - - t.Run("overwrite with fresh object", func(t *testing.T) { - input := []*objects.VObject{ - { - LatestObject: fresh, - Vector: []float32{4, 5, 6}, - StaleUpdateTime: stale.LastUpdateTimeUnix, - }, - } - - idx := repo.GetIndex(schema.ClassName(class.Class)) - shd, err := idx.determineObjectShard(fresh.ID, "") - require.Nil(t, err) - - received, err := idx.overwriteObjects(context.Background(), shd, input) - assert.Nil(t, err) - assert.ElementsMatch(t, nil, received) - }) - - t.Run("assert data was overwritten", func(t *testing.T) { - found, err := repo.Object(context.Background(), stale.Class, - stale.ID, nil, additional.Properties{}, nil, "") - assert.Nil(t, err) - assert.EqualValues(t, fresh, found.Object()) - }) -} - -func TestIndexDigestObjects(t *testing.T) { - dirName := t.TempDir() - logger, _ := test.NewNullLogger() - class := &models.Class{ - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Class: "SomeClass", - Properties: []*models.Property{ - { - Name: "stringProp", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - } - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 10, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, - &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - defer repo.Shutdown(context.Background()) - migrator := NewMigrator(repo, logger) - t.Run("create the class", func(t *testing.T) { - require.Nil(t, - migrator.AddClass(context.Background(), class, schemaGetter.shardState)) - }) - // update schema getter so it's in sync with class - schemaGetter.schema = schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{class}, - }, - } - - now := time.Now() - later := now.Add(time.Hour) // time-traveling ;) - obj1 := &models.Object{ - ID: "ae48fda2-866a-4c90-94fc-fce40d5f3767", - Class: class.Class, - CreationTimeUnix: now.UnixMilli(), - LastUpdateTimeUnix: now.UnixMilli(), - Properties: map[string]interface{}{ - "oldValue": "how things used to be", - }, - Vector: []float32{1, 2, 3}, - VectorWeights: (map[string]string)(nil), - Additional: models.AdditionalProperties{}, - } - - obj2 := &models.Object{ - ID: "b71ffac8-6534-4368-9718-5410ca89ce16", - Class: class.Class, - CreationTimeUnix: later.UnixMilli(), - LastUpdateTimeUnix: later.UnixMilli(), - Properties: map[string]interface{}{ - "oldValue": "how things used to be", - }, - Vector: []float32{1, 2, 3}, - VectorWeights: (map[string]string)(nil), - Additional: models.AdditionalProperties{}, - } - - t.Run("insert test objects", func(t *testing.T) { - err := repo.PutObject(context.Background(), obj1, obj1.Vector, nil) - require.Nil(t, err) - err = repo.PutObject(context.Background(), obj2, obj2.Vector, nil) - require.Nil(t, err) - }) - - t.Run("get digest object", func(t *testing.T) { - idx := repo.GetIndex(schema.ClassName(class.Class)) - shd, err := idx.determineObjectShard(obj1.ID, "") - require.Nil(t, err) - - input := []strfmt.UUID{obj1.ID, obj2.ID} - - expected := []replica.RepairResponse{ - { - ID: obj1.ID.String(), - UpdateTime: obj1.LastUpdateTimeUnix, - }, - { - ID: obj2.ID.String(), - UpdateTime: obj2.LastUpdateTimeUnix, - }, - } - - res, err := idx.digestObjects(context.Background(), shd, input) - require.Nil(t, err) - assert.Equal(t, expected, res) - }) -} - -func findID(list []search.Result, id strfmt.UUID) (search.Result, bool) { - for _, item := range list { - if item.ID == id { - return item, true - } - } - - return search.Result{}, false -} - -func ptFloat32(in float32) *float32 { - return &in -} - -func ptFloat64(in float64) *float64 { - return &in -} - -func randomVector(r *rand.Rand, dim int) []float32 { - out := make([]float32, dim) - for i := range out { - out[i] = r.Float32() - } - - return out -} - -func TestIndexDifferentVectorLength(t *testing.T) { - logger, _ := test.NewNullLogger() - class := &models.Class{ - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Class: "SomeClass", - Properties: []*models.Property{ - { - Name: "stringProp", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - } - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: t.TempDir(), - QueryMaximumResults: 10, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, - &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - defer repo.Shutdown(context.Background()) - migrator := NewMigrator(repo, logger) - require.Nil(t, migrator.AddClass(context.Background(), class, schemaGetter.shardState)) - // update schema getter so it's in sync with class - schemaGetter.schema = schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{class}, - }, - } - - obj1ID := strfmt.UUID("ae48fda2-866a-4c90-94fc-fce40d5f3767") - objNilID := strfmt.UUID("b71ffac9-6534-4368-9718-5410ca89ce16") - - t.Run("Add object with nil vector", func(t *testing.T) { - objNil := &models.Object{ - ID: objNilID, - Class: class.Class, - Vector: nil, - } - require.Nil(t, repo.PutObject(context.Background(), objNil, objNil.Vector, nil)) - found, err := repo.Object(context.Background(), class.Class, objNil.ID, nil, - additional.Properties{}, nil, "") - require.Nil(t, err) - require.Equal(t, found.Vector, []float32{}) - require.Equal(t, objNil.ID, found.ID) - }) - - t.Run("Add object with non-nil vector after nil vector", func(t *testing.T) { - obj1 := &models.Object{ - ID: obj1ID, - Class: class.Class, - Vector: []float32{1, 2, 3}, - } - require.Nil(t, repo.PutObject(context.Background(), obj1, obj1.Vector, nil)) - }) - - t.Run("Add object with different vector length", func(t *testing.T) { - obj2 := &models.Object{ - ID: "b71ffac8-6534-4368-9718-5410ca89ce16", - Class: class.Class, - Vector: []float32{1, 2, 3, 4}, - } - require.NotNil(t, repo.PutObject(context.Background(), obj2, obj2.Vector, nil)) - found, err := repo.Object(context.Background(), class.Class, obj2.ID, nil, - additional.Properties{}, nil, "") - require.Nil(t, err) - require.Nil(t, found) - }) - - t.Run("Update object with different vector length", func(t *testing.T) { - err = repo.Merge(context.Background(), objects.MergeDocument{ - ID: obj1ID, - Class: class.Class, - PrimitiveSchema: map[string]interface{}{}, - Vector: []float32{1, 2, 3, 4}, - UpdateTime: time.Now().UnixNano() / int64(time.Millisecond), - }, nil, "") - require.NotNil(t, err) - found, err := repo.Object(context.Background(), class.Class, - obj1ID, nil, additional.Properties{}, nil, "") - require.Nil(t, err) - require.Len(t, found.Vector, 3) - }) - - t.Run("Update nil object with fitting vector", func(t *testing.T) { - err = repo.Merge(context.Background(), objects.MergeDocument{ - ID: objNilID, - Class: class.Class, - PrimitiveSchema: map[string]interface{}{}, - Vector: []float32{1, 2, 3}, - UpdateTime: time.Now().UnixNano() / int64(time.Millisecond), - }, nil, "") - require.Nil(t, err) - found, err := repo.Object(context.Background(), class.Class, objNilID, nil, - additional.Properties{}, nil, "") - require.Nil(t, err) - require.Len(t, found.Vector, 3) - }) - - t.Run("Add nil object after objects with vector", func(t *testing.T) { - obj2Nil := &models.Object{ - ID: "b71ffac8-6534-4368-9718-5410ca89ce16", - Class: class.Class, - Vector: nil, - } - require.Nil(t, repo.PutObject(context.Background(), obj2Nil, obj2Nil.Vector, nil)) - found, err := repo.Object(context.Background(), class.Class, obj2Nil.ID, nil, - additional.Properties{}, nil, "") - require.Nil(t, err) - require.Equal(t, obj2Nil.ID, found.ID) - require.Equal(t, []float32{}, found.Vector) - }) -} diff --git a/adapters/repos/db/crud_noindex_property_integration_test.go b/adapters/repos/db/crud_noindex_property_integration_test.go deleted file mode 100644 index 2549882030962a29b24cd2a584af99c8dbba6dba..0000000000000000000000000000000000000000 --- a/adapters/repos/db/crud_noindex_property_integration_test.go +++ /dev/null @@ -1,157 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package db - -import ( - "context" - "testing" - - "github.com/go-openapi/strfmt" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/dto" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/search" - enthnsw "github.com/weaviate/weaviate/entities/vectorindex/hnsw" -) - -func TestCRUD_NoIndexProp(t *testing.T) { - dirName := t.TempDir() - - vFalse := false - logger, _ := test.NewNullLogger() - thingclass := &models.Class{ - Class: "ThingClassWithNoIndexProps", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{{ - Name: "stringProp", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, { - Name: "hiddenStringProp", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - IndexFilterable: &vFalse, - IndexSearchable: &vFalse, - }}, - } - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - MemtablesFlushIdleAfter: 60, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - defer repo.Shutdown(context.Background()) - - migrator := NewMigrator(repo, logger) - - t.Run("creating the thing class", func(t *testing.T) { - require.Nil(t, - migrator.AddClass(context.Background(), thingclass, schemaGetter.shardState)) - - // update schema getter so it's in sync with class - schemaGetter.schema = schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{thingclass}, - }, - } - }) - - thingID := strfmt.UUID("9f119c4f-80da-4ae5-bfd1-e4b63054125f") - - t.Run("adding a thing", func(t *testing.T) { - thing := &models.Object{ - CreationTimeUnix: 1565612833955, - LastUpdateTimeUnix: 1000001, - ID: thingID, - Class: "ThingClassWithNoIndexProps", - Properties: map[string]interface{}{ - "stringProp": "some value", - "hiddenStringProp": "some hidden value", - }, - } - vector := []float32{1, 3, 5, 0.4} - err := repo.PutObject(context.Background(), thing, vector, nil) - - assert.Nil(t, err) - }) - - t.Run("all props are present when getting by id", func(t *testing.T) { - res, err := repo.ObjectByID(context.Background(), thingID, search.SelectProperties{}, additional.Properties{}, "") - expectedSchema := map[string]interface{}{ - "stringProp": "some value", - "hiddenStringProp": "some hidden value", - "id": thingID, - } - - require.Nil(t, err) - assert.Equal(t, expectedSchema, res.Schema) - }) - - // Same as above, but with Object() - t.Run("all props are present when getting by id and class", func(t *testing.T) { - res, err := repo.Object(context.Background(), "ThingClassWithNoIndexProps", thingID, - search.SelectProperties{}, additional.Properties{}, nil, "") - expectedSchema := map[string]interface{}{ - "stringProp": "some value", - "hiddenStringProp": "some hidden value", - "id": thingID, - } - - require.Nil(t, err) - assert.Equal(t, expectedSchema, res.Schema) - }) - - t.Run("class search on the noindex prop errors", func(t *testing.T) { - _, err := repo.Search(context.Background(), dto.GetParams{ - ClassName: "ThingClassWithNoIndexProps", - Pagination: &filters.Pagination{ - Limit: 10, - }, - Filters: buildFilter("hiddenStringProp", "hidden", eq, schema.DataTypeText), - }) - - require.NotNil(t, err) - assert.Contains(t, err.Error(), "Filtering by property 'hiddenStringProp' requires inverted index. "+ - "Is `indexFilterable` option of property 'hiddenStringProp' enabled? "+ - "Set it to `true` or leave empty") - }) - - t.Run("class search on timestamp prop with no timestamp indexing error", func(t *testing.T) { - _, err := repo.Search(context.Background(), dto.GetParams{ - ClassName: "ThingClassWithNoIndexProps", - Pagination: &filters.Pagination{ - Limit: 10, - }, - Filters: buildFilter("_creationTimeUnix", "1234567891011", eq, schema.DataTypeText), - }) - - require.NotNil(t, err) - assert.Contains(t, err.Error(), - "Timestamps must be indexed to be filterable! Add `IndexTimestamps: true` to the InvertedIndexConfig in") - }) -} diff --git a/adapters/repos/db/crud_null_objects_integration_test.go b/adapters/repos/db/crud_null_objects_integration_test.go deleted file mode 100644 index 37b10f7f670f27073c6d9058b376ab0c6eeb1ee1..0000000000000000000000000000000000000000 --- a/adapters/repos/db/crud_null_objects_integration_test.go +++ /dev/null @@ -1,234 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest - -package db - -import ( - "context" - "testing" - - "github.com/weaviate/weaviate/entities/dto" - "github.com/weaviate/weaviate/entities/filters" - enthnsw "github.com/weaviate/weaviate/entities/vectorindex/hnsw" - - "github.com/go-openapi/strfmt" - "github.com/stretchr/testify/assert" - - "github.com/google/uuid" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/usecases/objects" -) - -// Cannot filter for null state without enabling in the InvertedIndexConfig -func TestFilterNullStateError(t *testing.T) { - class := createClassWithEverything(false, false) - migrator, repo, schemaGetter := createRepo(t) - defer repo.Shutdown(context.Background()) - err := migrator.AddClass(context.Background(), class, schemaGetter.shardState) - require.Nil(t, err) - // update schema getter so it's in sync with class - schemaGetter.schema = schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{class}, - }, - } - - nilFilter := &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorIsNull, - On: &filters.Path{ - Class: schema.ClassName(carClass.Class), - Property: schema.PropertyName(class.Properties[0].Name), - }, - Value: &filters.Value{ - Value: true, - Type: schema.DataTypeBoolean, - }, - }, - } - - params := dto.GetParams{ - SearchVector: []float32{0.1, 0.1, 0.1, 1.1, 0.1}, - ClassName: class.Class, - Pagination: &filters.Pagination{Limit: 5}, - Filters: nilFilter, - } - _, err = repo.Search(context.Background(), params) - require.NotNil(t, err) -} - -func TestNullArrayClass(t *testing.T) { - arrayClass := createClassWithEverything(true, false) - - names := []string{"elements", "batches"} - for _, name := range names { - t.Run("add nil object via "+name, func(t *testing.T) { - migrator, repo, schemaGetter := createRepo(t) - defer repo.Shutdown(context.Background()) - err := migrator.AddClass(context.Background(), arrayClass, schemaGetter.shardState) - require.Nil(t, err) - // update schema getter so it's in sync with class - schemaGetter.schema = schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{arrayClass}, - }, - } - - ObjectUuid1 := uuid.New() - arrayObjNil := &models.Object{ - ID: strfmt.UUID(ObjectUuid1.String()), - Class: "EverythingClass", - Properties: map[string]interface{}{ - "strings": nil, - "ints": nil, - "datesAsStrings": nil, - "numbers": nil, - "booleans": nil, - "texts": nil, - "number": nil, - "boolean": nil, - "int": nil, - "string": nil, - "text": nil, - "phoneNumber": nil, - "phoneNumbers": nil, - }, - } - - ObjectUuid2 := uuid.New() - arrayObjEmpty := &models.Object{ - ID: strfmt.UUID(ObjectUuid2.String()), - Class: "EverythingClass", - Properties: map[string]interface{}{}, - } - - if name == names[0] { - assert.Nil(t, repo.PutObject(context.Background(), arrayObjNil, []float32{1}, nil)) - assert.Nil(t, repo.PutObject(context.Background(), arrayObjEmpty, []float32{1}, nil)) - - } else { - batch := make([]objects.BatchObject, 2) - batch[0] = objects.BatchObject{Object: arrayObjNil, UUID: arrayObjNil.ID} - batch[1] = objects.BatchObject{Object: arrayObjEmpty, UUID: arrayObjEmpty.ID} - _, err := repo.BatchPutObjects(context.Background(), batch, nil) - assert.Nil(t, err) - } - - item1, err := repo.ObjectByID(context.Background(), arrayObjNil.ID, nil, additional.Properties{}, "") - assert.Nil(t, err) - item2, err := repo.ObjectByID(context.Background(), arrayObjEmpty.ID, nil, additional.Properties{}, "") - assert.Nil(t, err) - - item1Schema := item1.Schema.(map[string]interface{}) - item2Schema := item2.Schema.(map[string]interface{}) - delete(item1Schema, "id") - delete(item2Schema, "id") - assert.Equal(t, item1Schema, item2Schema) - }) - } -} - -func createRepo(t *testing.T) (*Migrator, *DB, *fakeSchemaGetter) { - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - logger, _ := test.NewNullLogger() - dirName := t.TempDir() - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 10, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - return NewMigrator(repo, logger), repo, schemaGetter -} - -func createClassWithEverything(IndexNullState bool, IndexPropertyLength bool) *models.Class { - return &models.Class{ - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: &models.InvertedIndexConfig{ - CleanupIntervalSeconds: 60, - Stopwords: &models.StopwordConfig{ - Preset: "none", - }, - IndexNullState: IndexNullState, - IndexPropertyLength: IndexPropertyLength, - }, - Class: "EverythingClass", - Properties: []*models.Property{ - { - Name: "strings", - DataType: schema.DataTypeTextArray.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "texts", - DataType: []string{"text[]"}, - Tokenization: models.PropertyTokenizationWord, - }, - { - Name: "numbers", - DataType: []string{"number[]"}, - }, - { - Name: "ints", - DataType: []string{"int[]"}, - }, - { - Name: "booleans", - DataType: []string{"boolean[]"}, - }, - { - Name: "datesAsStrings", - DataType: []string{"date[]"}, - }, - { - Name: "number", - DataType: []string{"number"}, - }, - { - Name: "bool", - DataType: []string{"boolean"}, - }, - { - Name: "int", - DataType: []string{"int"}, - }, - { - Name: "string", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "text", - DataType: []string{"text"}, - }, - { - Name: "phoneNumber", - DataType: []string{"phoneNumber"}, - }, - { - Name: "phoneNumbers", - DataType: []string{"phoneNumber[]"}, - }, - }, - } -} diff --git a/adapters/repos/db/crud_references_integration_test.go b/adapters/repos/db/crud_references_integration_test.go deleted file mode 100644 index de3a6be56259b1d37edb63d9963ce2b1d5a65159..0000000000000000000000000000000000000000 --- a/adapters/repos/db/crud_references_integration_test.go +++ /dev/null @@ -1,710 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package db - -import ( - "context" - "fmt" - "log" - "testing" - - "github.com/go-openapi/strfmt" - "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/schema/crossref" - "github.com/weaviate/weaviate/entities/search" - enthnsw "github.com/weaviate/weaviate/entities/vectorindex/hnsw" -) - -func TestNestedReferences(t *testing.T) { - dirName := t.TempDir() - - refSchema := schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{ - { - Class: "Planet", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - }, - { - Class: "Continent", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "onPlanet", - DataType: []string{"Planet"}, - }, - }, - }, - { - Class: "Country", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "onContinent", - DataType: []string{"Continent"}, - }, - }, - }, - { - Class: "City", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "inCountry", - DataType: []string{"Country"}, - }, - }, - }, - { - Class: "Place", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "inCity", - DataType: []string{"City"}, - }, - }, - }, - }, - }, - } - logger := logrus.New() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - defer repo.Shutdown(context.Background()) - migrator := NewMigrator(repo, logger) - - t.Run("adding all classes to the schema", func(t *testing.T) { - for _, class := range refSchema.Objects.Classes { - t.Run(fmt.Sprintf("add %s", class.Class), func(t *testing.T) { - err := migrator.AddClass(context.Background(), class, schemaGetter.shardState) - require.Nil(t, err) - }) - } - }) - - // update schema getter so it's in sync with class - schemaGetter.schema = refSchema - - t.Run("importing some thing objects with references", func(t *testing.T) { - objects := []models.Object{ - { - Class: "Planet", - Properties: map[string]interface{}{ - "name": "Earth", - }, - ID: "32c69af9-cbbe-4ec9-bf6c-365cd6c22fdf", - CreationTimeUnix: 1566464889, - }, - { - Class: "Continent", - Properties: map[string]interface{}{ - "name": "North America", - "onPlanet": models.MultipleRef{ - &models.SingleRef{ - Beacon: "weaviate://localhost/32c69af9-cbbe-4ec9-bf6c-365cd6c22fdf", - }, - }, - }, - ID: "4aad8154-e7f3-45b8-81a6-725171419e55", - CreationTimeUnix: 1566464892, - }, - { - Class: "Country", - Properties: map[string]interface{}{ - "name": "USA", - "onContinent": models.MultipleRef{ - &models.SingleRef{ - Beacon: "weaviate://localhost/4aad8154-e7f3-45b8-81a6-725171419e55", - }, - }, - }, - ID: "18c80a16-346a-477d-849d-9d92e5040ac9", - CreationTimeUnix: 1566464896, - }, - { - Class: "City", - Properties: map[string]interface{}{ - "name": "San Francisco", - "inCountry": models.MultipleRef{ - &models.SingleRef{ - Beacon: "weaviate://localhost/18c80a16-346a-477d-849d-9d92e5040ac9", - }, - }, - }, - ID: "2297e094-6218-43d4-85b1-3d20af752f23", - CreationTimeUnix: 1566464899, - }, - { - Class: "Place", - Properties: map[string]interface{}{ - "name": "Tim Apple's Fruit Bar", - "inCity": models.MultipleRef{ - &models.SingleRef{ - Beacon: "weaviate://localhost/2297e094-6218-43d4-85b1-3d20af752f23", - }, - }, - }, - ID: "4ef47fb0-3cf5-44fc-b378-9e217dff13ac", - CreationTimeUnix: 1566464904, - }, - } - - for _, thing := range objects { - t.Run(fmt.Sprintf("add %s", thing.ID), func(t *testing.T) { - err := repo.PutObject(context.Background(), &thing, []float32{1, 2, 3, 4, 5, 6, 7}, nil) - require.Nil(t, err) - }) - } - }) - - t.Run("fully resolving the place", func(t *testing.T) { - expectedSchema := map[string]interface{}{ - "inCity": []interface{}{ - search.LocalRef{ - Class: "City", - Fields: map[string]interface{}{ - "inCountry": []interface{}{ - search.LocalRef{ - Class: "Country", - Fields: map[string]interface{}{ - "onContinent": []interface{}{ - search.LocalRef{ - Class: "Continent", - Fields: map[string]interface{}{ - "onPlanet": []interface{}{ - search.LocalRef{ - Class: "Planet", - Fields: map[string]interface{}{ - "name": "Earth", - "id": strfmt.UUID("32c69af9-cbbe-4ec9-bf6c-365cd6c22fdf"), - }, - }, - }, - "name": "North America", - "id": strfmt.UUID("4aad8154-e7f3-45b8-81a6-725171419e55"), - }, - }, - }, - "name": "USA", - "id": strfmt.UUID("18c80a16-346a-477d-849d-9d92e5040ac9"), - }, - }, - }, - "name": "San Francisco", - "id": strfmt.UUID("2297e094-6218-43d4-85b1-3d20af752f23"), - }, - }, - }, - "name": "Tim Apple's Fruit Bar", - "id": strfmt.UUID("4ef47fb0-3cf5-44fc-b378-9e217dff13ac"), - } - - res, err := repo.ObjectByID(context.Background(), "4ef47fb0-3cf5-44fc-b378-9e217dff13ac", fullyNestedSelectProperties(), additional.Properties{}, "") - require.Nil(t, err) - assert.Equal(t, expectedSchema, res.Schema) - }) - - t.Run("fully resolving the place with vectors", func(t *testing.T) { - expectedSchema := map[string]interface{}{ - "inCity": []interface{}{ - search.LocalRef{ - Class: "City", - Fields: map[string]interface{}{ - "inCountry": []interface{}{ - search.LocalRef{ - Class: "Country", - Fields: map[string]interface{}{ - "onContinent": []interface{}{ - search.LocalRef{ - Class: "Continent", - Fields: map[string]interface{}{ - "onPlanet": []interface{}{ - search.LocalRef{ - Class: "Planet", - Fields: map[string]interface{}{ - "name": "Earth", - "id": strfmt.UUID("32c69af9-cbbe-4ec9-bf6c-365cd6c22fdf"), - "vector": []float32{1, 2, 3, 4, 5, 6, 7}, - }, - }, - }, - "name": "North America", - "id": strfmt.UUID("4aad8154-e7f3-45b8-81a6-725171419e55"), - "vector": []float32{1, 2, 3, 4, 5, 6, 7}, - }, - }, - }, - "name": "USA", - "id": strfmt.UUID("18c80a16-346a-477d-849d-9d92e5040ac9"), - "vector": []float32{1, 2, 3, 4, 5, 6, 7}, - }, - }, - }, - "name": "San Francisco", - "id": strfmt.UUID("2297e094-6218-43d4-85b1-3d20af752f23"), - "vector": []float32{1, 2, 3, 4, 5, 6, 7}, - }, - }, - }, - "name": "Tim Apple's Fruit Bar", - "id": strfmt.UUID("4ef47fb0-3cf5-44fc-b378-9e217dff13ac"), - } - - res, err := repo.ObjectByID(context.Background(), "4ef47fb0-3cf5-44fc-b378-9e217dff13ac", fullyNestedSelectPropertiesWithVector(), additional.Properties{}, "") - require.Nil(t, err) - assert.Equal(t, expectedSchema, res.Schema) - }) - - t.Run("partially resolving the place", func(t *testing.T) { - expectedSchema := map[string]interface{}{ - "inCity": []interface{}{ - search.LocalRef{ - Class: "City", - Fields: map[string]interface{}{ - "name": "San Francisco", - "id": strfmt.UUID("2297e094-6218-43d4-85b1-3d20af752f23"), - // why is inCountry present here? We didn't specify it our select - // properties. Note it is "inCountry" with a lowercase letter - // (meaning unresolved) whereas "inCountry" would mean it was - // resolved. In GraphQL this property would simply be hidden (as - // the GQL is unaware of unresolved properties) - // However, for caching and other queries it is helpful that this - // info is still present, the important thing is that we're - // avoiding the costly resolving of it, if we don't need it. - "inCountry": models.MultipleRef{ - &models.SingleRef{ - Beacon: "weaviate://localhost/18c80a16-346a-477d-849d-9d92e5040ac9", - }, - }, - }, - }, - }, - "name": "Tim Apple's Fruit Bar", - "id": strfmt.UUID("4ef47fb0-3cf5-44fc-b378-9e217dff13ac"), - } - - res, err := repo.ObjectByID(context.Background(), "4ef47fb0-3cf5-44fc-b378-9e217dff13ac", partiallyNestedSelectProperties(), additional.Properties{}, "") - require.Nil(t, err) - assert.Equal(t, expectedSchema, res.Schema) - }) - - t.Run("resolving without any refs", func(t *testing.T) { - res, err := repo.ObjectByID(context.Background(), "4ef47fb0-3cf5-44fc-b378-9e217dff13ac", search.SelectProperties{}, additional.Properties{}, "") - - expectedSchema := map[string]interface{}{ - "id": strfmt.UUID("4ef47fb0-3cf5-44fc-b378-9e217dff13ac"), - "inCity": models.MultipleRef{ - &models.SingleRef{ - Beacon: "weaviate://localhost/2297e094-6218-43d4-85b1-3d20af752f23", - }, - }, - "name": "Tim Apple's Fruit Bar", - } - - require.Nil(t, err) - - assert.Equal(t, expectedSchema, res.Schema, "does not contain any resolved refs") - }) - - t.Run("adding a new place to verify idnexing is constantly happening in the background", func(t *testing.T) { - newPlace := models.Object{ - Class: "Place", - Properties: map[string]interface{}{ - "name": "John Oliver's Avocados", - "inCity": models.MultipleRef{ - &models.SingleRef{ - Beacon: "weaviate://localhost/2297e094-6218-43d4-85b1-3d20af752f23", - }, - }, - }, - ID: "0f02d525-902d-4dc0-8052-647cb420c1a6", - CreationTimeUnix: 1566464912, - } - - err := repo.PutObject(context.Background(), &newPlace, []float32{1, 2, 3, 4, 5, 6, 7}, nil) - require.Nil(t, err) - }) -} - -func fullyNestedSelectProperties() search.SelectProperties { - return search.SelectProperties{ - search.SelectProperty{ - Name: "inCity", - IsPrimitive: false, - Refs: []search.SelectClass{ - { - ClassName: "City", - RefProperties: search.SelectProperties{ - search.SelectProperty{ - Name: "inCountry", - IsPrimitive: false, - Refs: []search.SelectClass{ - { - ClassName: "Country", - RefProperties: search.SelectProperties{ - search.SelectProperty{ - Name: "onContinent", - IsPrimitive: false, - Refs: []search.SelectClass{ - { - ClassName: "Continent", - RefProperties: search.SelectProperties{ - search.SelectProperty{ - Name: "onPlanet", - IsPrimitive: false, - Refs: []search.SelectClass{ - { - ClassName: "Planet", - RefProperties: nil, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - } -} - -func fullyNestedSelectPropertiesWithVector() search.SelectProperties { - return search.SelectProperties{ - search.SelectProperty{ - Name: "inCity", - IsPrimitive: false, - Refs: []search.SelectClass{ - { - ClassName: "City", - RefProperties: search.SelectProperties{ - search.SelectProperty{ - Name: "inCountry", - IsPrimitive: false, - Refs: []search.SelectClass{ - { - ClassName: "Country", - RefProperties: search.SelectProperties{ - search.SelectProperty{ - Name: "onContinent", - IsPrimitive: false, - Refs: []search.SelectClass{ - { - ClassName: "Continent", - RefProperties: search.SelectProperties{ - search.SelectProperty{ - Name: "onPlanet", - IsPrimitive: false, - Refs: []search.SelectClass{ - { - ClassName: "Planet", - RefProperties: nil, - AdditionalProperties: additional.Properties{ - Vector: true, - }, - }, - }, - }, - }, - AdditionalProperties: additional.Properties{ - Vector: true, - }, - }, - }, - }, - }, - AdditionalProperties: additional.Properties{ - Vector: true, - }, - }, - }, - }, - }, - AdditionalProperties: additional.Properties{ - Vector: true, - }, - }, - }, - }, - } -} - -func partiallyNestedSelectProperties() search.SelectProperties { - return search.SelectProperties{ - search.SelectProperty{ - Name: "inCity", - IsPrimitive: false, - Refs: []search.SelectClass{ - { - ClassName: "City", - RefProperties: search.SelectProperties{}, - }, - }, - }, - } -} - -func GetDimensionsFromRepo(repo *DB, className string) int { - if !repo.config.TrackVectorDimensions { - log.Printf("Vector dimensions tracking is disabled, returning 0") - return 0 - } - index := repo.GetIndex(schema.ClassName(className)) - sum := 0 - index.ForEachShard(func(name string, shard ShardLike) error { - sum += shard.Dimensions() - return nil - }) - return sum -} - -func GetQuantizedDimensionsFromRepo(repo *DB, className string, segments int) int { - if !repo.config.TrackVectorDimensions { - log.Printf("Vector dimensions tracking is disabled, returning 0") - return 0 - } - index := repo.GetIndex(schema.ClassName(className)) - sum := 0 - index.ForEachShard(func(name string, shard ShardLike) error { - sum += shard.QuantizedDimensions(segments) - return nil - }) - return sum -} - -func Test_AddingReferenceOneByOne(t *testing.T) { - dirName := t.TempDir() - - sch := schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{ - { - Class: "AddingReferencesTestTarget", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - }, - { - Class: "AddingReferencesTestSource", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "toTarget", - DataType: []string{"AddingReferencesTestTarget"}, - }, - }, - }, - }, - }, - } - logger := logrus.New() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - MaxImportGoroutinesFactor: 1, - TrackVectorDimensions: true, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - defer repo.Shutdown(context.Background()) - migrator := NewMigrator(repo, logger) - - t.Run("add required classes", func(t *testing.T) { - for _, class := range sch.Objects.Classes { - t.Run(fmt.Sprintf("add %s", class.Class), func(t *testing.T) { - err := migrator.AddClass(context.Background(), class, schemaGetter.shardState) - require.Nil(t, err) - }) - } - }) - - schemaGetter.schema = sch - targetID := strfmt.UUID("a4a92239-e748-4e55-bbbd-f606926619a7") - target2ID := strfmt.UUID("325084e7-4faa-43a5-b2b1-56e207be169a") - sourceID := strfmt.UUID("0826c61b-85c1-44ac-aebb-cfd07ace6a57") - - t.Run("add objects", func(t *testing.T) { - err := repo.PutObject(context.Background(), &models.Object{ - ID: sourceID, - Class: "AddingReferencesTestSource", - Properties: map[string]interface{}{ - "name": "source item", - }, - }, []float32{0.5}, nil) - require.Nil(t, err) - - err = repo.PutObject(context.Background(), &models.Object{ - ID: targetID, - Class: "AddingReferencesTestTarget", - Properties: map[string]interface{}{ - "name": "target item", - }, - }, []float32{0.5}, nil) - require.Nil(t, err) - - err = repo.PutObject(context.Background(), &models.Object{ - ID: target2ID, - Class: "AddingReferencesTestTarget", - Properties: map[string]interface{}{ - "name": "another target item", - }, - }, []float32{0.5}, nil) - require.Nil(t, err) - }) - - t.Run("add reference between them", func(t *testing.T) { - // Get dimensions before adding reference - sourceShardDimension := GetDimensionsFromRepo(repo, "AddingReferencesTestSource") - targetShardDimension := GetDimensionsFromRepo(repo, "AddingReferencesTestTarget") - - source := crossref.NewSource("AddingReferencesTestSource", "toTarget", sourceID) - target := crossref.New("localhost", "", targetID) - - err := repo.AddReference(context.Background(), source, target, nil, "") - assert.Nil(t, err) - - // Check dimensions after adding reference - sourceDimensionAfter := GetDimensionsFromRepo(repo, "AddingReferencesTestSource") - targetDimensionAfter := GetDimensionsFromRepo(repo, "AddingReferencesTestTarget") - - require.Equalf(t, sourceShardDimension, sourceDimensionAfter, "dimensions of source should not change") - require.Equalf(t, targetShardDimension, targetDimensionAfter, "dimensions of target should not change") - }) - - t.Run("check reference was added", func(t *testing.T) { - source, err := repo.ObjectByID(context.Background(), sourceID, nil, additional.Properties{}, "") - require.Nil(t, err) - require.NotNil(t, source) - require.NotNil(t, source.Object()) - require.NotNil(t, source.Object().Properties) - - refs := source.Object().Properties.(map[string]interface{})["toTarget"] - refsSlice, ok := refs.(models.MultipleRef) - require.True(t, ok, - fmt.Sprintf("toTarget must be models.MultipleRef, but got %#v", refs)) - - foundBeacons := []string{} - for _, ref := range refsSlice { - foundBeacons = append(foundBeacons, ref.Beacon.String()) - } - expectedBeacons := []string{ - fmt.Sprintf("weaviate://localhost/%s", targetID), - } - - assert.ElementsMatch(t, foundBeacons, expectedBeacons) - }) - - t.Run("reference a second target", func(t *testing.T) { - source := crossref.NewSource("AddingReferencesTestSource", "toTarget", sourceID) - target := crossref.New("localhost", "", target2ID) - - err := repo.AddReference(context.Background(), source, target, nil, "") - assert.Nil(t, err) - }) - - t.Run("check both references are now present", func(t *testing.T) { - source, err := repo.ObjectByID(context.Background(), sourceID, nil, additional.Properties{}, "") - require.Nil(t, err) - require.NotNil(t, source) - require.NotNil(t, source.Object()) - require.NotNil(t, source.Object().Properties) - - refs := source.Object().Properties.(map[string]interface{})["toTarget"] - refsSlice, ok := refs.(models.MultipleRef) - require.True(t, ok, - fmt.Sprintf("toTarget must be models.MultipleRef, but got %#v", refs)) - - foundBeacons := []string{} - for _, ref := range refsSlice { - foundBeacons = append(foundBeacons, ref.Beacon.String()) - } - expectedBeacons := []string{ - fmt.Sprintf("weaviate://localhost/%s", targetID), - fmt.Sprintf("weaviate://localhost/%s", target2ID), - } - - assert.ElementsMatch(t, foundBeacons, expectedBeacons) - }) -} diff --git a/adapters/repos/db/crud_references_multiple_types_integration_test.go b/adapters/repos/db/crud_references_multiple_types_integration_test.go deleted file mode 100644 index 8d89732cf2c8c2cdd76b8e264faf0a82dc1ab016..0000000000000000000000000000000000000000 --- a/adapters/repos/db/crud_references_multiple_types_integration_test.go +++ /dev/null @@ -1,669 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package db - -import ( - "context" - "fmt" - "testing" - - "github.com/go-openapi/strfmt" - "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/search" -) - -func TestMultipleCrossRefTypes(t *testing.T) { - dirName := t.TempDir() - - logger := logrus.New() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - defer repo.Shutdown(context.Background()) - migrator := NewMigrator(repo, logger) - - t.Run("adding all classes to the schema", func(t *testing.T) { - for _, class := range parkingGaragesSchema().Objects.Classes { - t.Run(fmt.Sprintf("add %s", class.Class), func(t *testing.T) { - err := migrator.AddClass(context.Background(), class, schemaGetter.shardState) - require.Nil(t, err) - }) - } - }) - - // update schema getter so it's in sync with class - schemaGetter.schema = parkingGaragesSchema() - - t.Run("importing with various combinations of props", func(t *testing.T) { - objects := []models.Object{ - { - Class: "MultiRefParkingGarage", - Properties: map[string]interface{}{ - "name": "Luxury Parking Garage", - "location": &models.GeoCoordinates{ - Latitude: ptFloat32(48.864716), - Longitude: ptFloat32(2.349014), - }, - }, - ID: "a7e10b55-1ac4-464f-80df-82508eea1951", - CreationTimeUnix: 1566469890, - }, - { - Class: "MultiRefParkingGarage", - Properties: map[string]interface{}{ - "name": "Crappy Parking Garage", - "location": &models.GeoCoordinates{ - Latitude: ptFloat32(42.331429), - Longitude: ptFloat32(-83.045753), - }, - }, - ID: "ba2232cf-bb0e-413d-b986-6aa996d34d2e", - CreationTimeUnix: 1566469892, - }, - { - Class: "MultiRefParkingLot", - Properties: map[string]interface{}{ - "name": "Fancy Parking Lot", - }, - ID: "1023967b-9512-475b-8ef9-673a110b695d", - CreationTimeUnix: 1566469894, - }, - { - Class: "MultiRefParkingLot", - Properties: map[string]interface{}{ - "name": "The worst parking lot youve ever seen", - }, - ID: "901859d8-69bf-444c-bf43-498963d798d2", - CreationTimeUnix: 1566469897, - }, - { - Class: "MultiRefCar", - Properties: map[string]interface{}{ - "name": "Car which is parked no where", - }, - ID: "329c306b-c912-4ec7-9b1d-55e5e0ca8dea", - CreationTimeUnix: 1566469899, - }, - { - Class: "MultiRefCar", - Properties: map[string]interface{}{ - "name": "Car which is parked in a garage", - "parkedAt": models.MultipleRef{ - &models.SingleRef{ - Beacon: "weaviate://localhost/a7e10b55-1ac4-464f-80df-82508eea1951", - }, - }, - }, - ID: "fe3ca25d-8734-4ede-9a81-bc1ed8c3ea43", - CreationTimeUnix: 1566469902, - }, - { - Class: "MultiRefCar", - Properties: map[string]interface{}{ - "name": "Car which is parked in a lot", - "parkedAt": models.MultipleRef{ - &models.SingleRef{ - Beacon: "weaviate://localhost/1023967b-9512-475b-8ef9-673a110b695d", - }, - }, - }, - ID: "21ab5130-627a-4268-baef-1a516bd6cad4", - CreationTimeUnix: 1566469906, - }, - { - Class: "MultiRefCar", - Properties: map[string]interface{}{ - "name": "Car which is parked in two places at the same time (magic!)", - "parkedAt": models.MultipleRef{ - &models.SingleRef{ - Beacon: "weaviate://localhost/a7e10b55-1ac4-464f-80df-82508eea1951", - }, - &models.SingleRef{ - Beacon: "weaviate://localhost/1023967b-9512-475b-8ef9-673a110b695d", - }, - }, - }, - ID: "533673a7-2a5c-4e1c-b35d-a3809deabace", - CreationTimeUnix: 1566469909, - }, - { - Class: "MultiRefDriver", - Properties: map[string]interface{}{ - "name": "Johny Drivemuch", - "drives": models.MultipleRef{ - &models.SingleRef{ - Beacon: "weaviate://localhost/533673a7-2a5c-4e1c-b35d-a3809deabace", - }, - }, - }, - ID: "9653ab38-c16b-4561-80df-7a7e19300dd0", - CreationTimeUnix: 1566469912, - }, - { - Class: "MultiRefPerson", - Properties: map[string]interface{}{ - "name": "Jane Doughnut", - "friendsWith": models.MultipleRef{ - &models.SingleRef{ - Beacon: "weaviate://localhost/9653ab38-c16b-4561-80df-7a7e19300dd0", - }, - }, - }, - ID: "91ad23a3-07ba-4d4c-9836-76c57094f734", - CreationTimeUnix: 1566469915, - }, - { - Class: "MultiRefSociety", - Properties: map[string]interface{}{ - "name": "Cool People", - "hasMembers": models.MultipleRef{ - &models.SingleRef{ - Beacon: "weaviate://localhost/91ad23a3-07ba-4d4c-9836-76c57094f734", - }, - }, - }, - ID: "5cd9afa6-f3df-4f57-a204-840d6b256dba", - CreationTimeUnix: 1566469918, - }, - } - - for _, thing := range objects { - t.Run(fmt.Sprintf("add %s", thing.ID), func(t *testing.T) { - err := repo.PutObject(context.Background(), &thing, []float32{1, 2, 3, 4, 5, 6, 7}, nil) - require.Nil(t, err) - }) - } - }) - - t.Run("car with no refs", func(t *testing.T) { - var id strfmt.UUID = "329c306b-c912-4ec7-9b1d-55e5e0ca8dea" - expectedSchema := map[string]interface{}{ - "name": "Car which is parked no where", - "id": id, - } - - t.Run("asking for no refs", func(t *testing.T) { - res, err := repo.ObjectByID(context.Background(), id, nil, additional.Properties{}, "") - require.Nil(t, err) - - assert.Equal(t, expectedSchema, res.Schema) - }) - - t.Run("asking for refs of type garage", func(t *testing.T) { - res, err := repo.ObjectByID(context.Background(), id, parkedAtGarage(), additional.Properties{}, "") - require.Nil(t, err) - - assert.Equal(t, expectedSchema, res.Schema) - }) - - t.Run("asking for refs of type lot", func(t *testing.T) { - res, err := repo.ObjectByID(context.Background(), id, parkedAtLot(), additional.Properties{}, "") - require.Nil(t, err) - - assert.Equal(t, expectedSchema, res.Schema) - }) - - t.Run("asking for refs of both types", func(t *testing.T) { - res, err := repo.ObjectByID(context.Background(), id, parkedAtEither(), additional.Properties{}, "") - require.Nil(t, err) - - assert.Equal(t, expectedSchema, res.Schema) - }) - }) - - t.Run("car with single ref to garage", func(t *testing.T) { - var id strfmt.UUID = "fe3ca25d-8734-4ede-9a81-bc1ed8c3ea43" - expectedSchemaUnresolved := map[string]interface{}{ - "name": "Car which is parked in a garage", - "id": id, - // ref is present, but unresolved, therefore the lowercase letter - "parkedAt": models.MultipleRef{ - &models.SingleRef{ - Beacon: "weaviate://localhost/a7e10b55-1ac4-464f-80df-82508eea1951", - }, - }, - } - - getExpectedSchema := func(withVector bool) map[string]interface{} { - fields := map[string]interface{}{ - "name": "Luxury Parking Garage", - "location": &models.GeoCoordinates{ - Latitude: ptFloat32(48.864716), - Longitude: ptFloat32(2.349014), - }, - "id": strfmt.UUID("a7e10b55-1ac4-464f-80df-82508eea1951"), - } - if withVector { - fields["vector"] = []float32{1, 2, 3, 4, 5, 6, 7} - } - return map[string]interface{}{ - "name": "Car which is parked in a garage", - "id": id, - "parkedAt": []interface{}{ - search.LocalRef{ - Class: "MultiRefParkingGarage", - Fields: fields, - }, - }, - } - } - - expectedSchemaWithRefs := getExpectedSchema(false) - expectedSchemaWithRefsWithVector := getExpectedSchema(true) - - t.Run("asking for no refs", func(t *testing.T) { - res, err := repo.ObjectByID(context.Background(), id, nil, additional.Properties{}, "") - require.Nil(t, err) - - assert.Equal(t, expectedSchemaUnresolved, res.Schema) - }) - - t.Run("asking for refs of type garage", func(t *testing.T) { - res, err := repo.ObjectByID(context.Background(), id, parkedAtGarage(), additional.Properties{}, "") - require.Nil(t, err) - - assert.Equal(t, expectedSchemaWithRefs, res.Schema) - }) - - t.Run("asking for refs of type garage with vector", func(t *testing.T) { - res, err := repo.ObjectByID(context.Background(), id, parkedAtGarageWithVector(true), additional.Properties{}, "") - require.Nil(t, err) - - assert.Equal(t, expectedSchemaWithRefsWithVector, res.Schema) - }) - - t.Run("asking for refs of type lot", func(t *testing.T) { - res, err := repo.ObjectByID(context.Background(), id, parkedAtLot(), additional.Properties{}, "") - require.Nil(t, err) - - assert.Equal(t, expectedSchemaUnresolved, res.Schema) - }) - - t.Run("asking for refs of both types", func(t *testing.T) { - res, err := repo.ObjectByID(context.Background(), id, parkedAtEither(), additional.Properties{}, "") - require.Nil(t, err) - - assert.Equal(t, expectedSchemaWithRefs, res.Schema) - }) - }) - - t.Run("car with single ref to lot", func(t *testing.T) { - var id strfmt.UUID = "21ab5130-627a-4268-baef-1a516bd6cad4" - expectedSchemaUnresolved := map[string]interface{}{ - "name": "Car which is parked in a lot", - "id": id, - // ref is present, but unresolved, therefore the lowercase letter - "parkedAt": models.MultipleRef{ - &models.SingleRef{ - Beacon: "weaviate://localhost/1023967b-9512-475b-8ef9-673a110b695d", - }, - }, - } - - getSchemaWithRefs := func(withVector bool) map[string]interface{} { - fields := map[string]interface{}{ - "name": "Fancy Parking Lot", - "id": strfmt.UUID("1023967b-9512-475b-8ef9-673a110b695d"), - } - if withVector { - fields["vector"] = []float32{1, 2, 3, 4, 5, 6, 7} - } - return map[string]interface{}{ - "name": "Car which is parked in a lot", - "id": id, - "parkedAt": []interface{}{ - search.LocalRef{ - Class: "MultiRefParkingLot", - Fields: fields, - }, - }, - } - } - - expectedSchemaWithRefs := getSchemaWithRefs(false) - expectedSchemaWithRefsWithVector := getSchemaWithRefs(true) - - t.Run("asking for no refs", func(t *testing.T) { - res, err := repo.ObjectByID(context.Background(), id, nil, additional.Properties{}, "") - require.Nil(t, err) - - assert.Equal(t, expectedSchemaUnresolved, res.Schema) - }) - - t.Run("asking for refs of type garage", func(t *testing.T) { - res, err := repo.ObjectByID(context.Background(), id, parkedAtGarage(), additional.Properties{}, "") - require.Nil(t, err) - - assert.Equal(t, expectedSchemaUnresolved, res.Schema) - }) - - t.Run("asking for refs of type lot", func(t *testing.T) { - res, err := repo.ObjectByID(context.Background(), id, parkedAtLot(), additional.Properties{}, "") - require.Nil(t, err) - - assert.Equal(t, expectedSchemaWithRefs, res.Schema) - }) - - t.Run("asking for refs with vector of type lot", func(t *testing.T) { - res, err := repo.ObjectByID(context.Background(), id, parkedAtLotWithVector(), additional.Properties{}, "") - require.Nil(t, err) - - assert.Equal(t, expectedSchemaWithRefsWithVector, res.Schema) - }) - - t.Run("asking for refs of both types", func(t *testing.T) { - res, err := repo.ObjectByID(context.Background(), id, parkedAtEither(), additional.Properties{}, "") - require.Nil(t, err) - - assert.Equal(t, expectedSchemaWithRefs, res.Schema) - }) - }) - - t.Run("car with refs to both", func(t *testing.T) { - var id strfmt.UUID = "533673a7-2a5c-4e1c-b35d-a3809deabace" - expectedSchemaUnresolved := map[string]interface{}{ - "name": "Car which is parked in two places at the same time (magic!)", - "id": id, - // ref is present, but unresolved, therefore the lowercase letter - "parkedAt": models.MultipleRef{ - &models.SingleRef{ - Beacon: "weaviate://localhost/a7e10b55-1ac4-464f-80df-82508eea1951", - }, - &models.SingleRef{ - Beacon: "weaviate://localhost/1023967b-9512-475b-8ef9-673a110b695d", - }, - }, - } - getExpectedSchemaWithLotRef := func(withVector bool) map[string]interface{} { - fields := map[string]interface{}{ - "name": "Fancy Parking Lot", - "id": strfmt.UUID("1023967b-9512-475b-8ef9-673a110b695d"), - } - if withVector { - fields["vector"] = []float32{1, 2, 3, 4, 5, 6, 7} - } - return map[string]interface{}{ - "name": "Car which is parked in two places at the same time (magic!)", - "id": id, - "parkedAt": []interface{}{ - search.LocalRef{ - Class: "MultiRefParkingLot", - Fields: fields, - }, - }, - } - } - expectedSchemaWithLotRef := getExpectedSchemaWithLotRef(false) - expectedSchemaWithLotRefWithVector := getExpectedSchemaWithLotRef(true) - getExpectedSchemaWithGarageRef := func(withVector bool) map[string]interface{} { - fields := map[string]interface{}{ - "name": "Luxury Parking Garage", - "location": &models.GeoCoordinates{ - Latitude: ptFloat32(48.864716), - Longitude: ptFloat32(2.349014), - }, - "id": strfmt.UUID("a7e10b55-1ac4-464f-80df-82508eea1951"), - } - if withVector { - fields["vector"] = []float32{1, 2, 3, 4, 5, 6, 7} - } - return map[string]interface{}{ - "name": "Car which is parked in two places at the same time (magic!)", - "id": id, - "parkedAt": []interface{}{ - search.LocalRef{ - Class: "MultiRefParkingGarage", - Fields: fields, - }, - }, - } - } - expectedSchemaWithGarageRef := getExpectedSchemaWithGarageRef(false) - expectedSchemaWithGarageRefWithVector := getExpectedSchemaWithGarageRef(true) - getExpectedSchemaWithAllRefs := func(withVector bool) map[string]interface{} { - fieldsParkingLot := map[string]interface{}{ - "name": "Fancy Parking Lot", - "id": strfmt.UUID("1023967b-9512-475b-8ef9-673a110b695d"), - } - if withVector { - fieldsParkingLot["vector"] = []float32{1, 2, 3, 4, 5, 6, 7} - } - fieldsParkingGarage := map[string]interface{}{ - "name": "Luxury Parking Garage", - "location": &models.GeoCoordinates{ - Latitude: ptFloat32(48.864716), - Longitude: ptFloat32(2.349014), - }, - "id": strfmt.UUID("a7e10b55-1ac4-464f-80df-82508eea1951"), - } - if withVector { - fieldsParkingGarage["vector"] = []float32{1, 2, 3, 4, 5, 6, 7} - } - return map[string]interface{}{ - "name": "Car which is parked in two places at the same time (magic!)", - "id": id, - "parkedAt": []interface{}{ - search.LocalRef{ - Class: "MultiRefParkingLot", - Fields: fieldsParkingLot, - }, - search.LocalRef{ - Class: "MultiRefParkingGarage", - Fields: fieldsParkingGarage, - }, - }, - } - } - expectedSchemaWithAllRefs := getExpectedSchemaWithAllRefs(false) - expectedSchemaWithAllRefsWithVector := getExpectedSchemaWithAllRefs(true) - - t.Run("asking for no refs", func(t *testing.T) { - res, err := repo.ObjectByID(context.Background(), id, nil, additional.Properties{}, "") - require.Nil(t, err) - - assert.Equal(t, expectedSchemaUnresolved, res.Schema) - }) - - t.Run("asking for refs of type garage", func(t *testing.T) { - res, err := repo.ObjectByID(context.Background(), id, parkedAtGarage(), additional.Properties{}, "") - require.Nil(t, err) - - assert.Equal(t, expectedSchemaWithGarageRef, res.Schema) - }) - - t.Run("asking for refs with vector of type garage", func(t *testing.T) { - res, err := repo.ObjectByID(context.Background(), id, parkedAtGarageWithVector(true), additional.Properties{}, "") - require.Nil(t, err) - - assert.Equal(t, expectedSchemaWithGarageRefWithVector, res.Schema) - }) - - t.Run("asking for refs of type lot", func(t *testing.T) { - res, err := repo.ObjectByID(context.Background(), id, parkedAtLot(), additional.Properties{}, "") - require.Nil(t, err) - - assert.Equal(t, expectedSchemaWithLotRef, res.Schema) - }) - - t.Run("asking for refs with vector of type lot", func(t *testing.T) { - res, err := repo.ObjectByID(context.Background(), id, parkedAtLotWithVector(), additional.Properties{}, "") - require.Nil(t, err) - - assert.Equal(t, expectedSchemaWithLotRefWithVector, res.Schema) - }) - - t.Run("asking for refs of both types", func(t *testing.T) { - res, err := repo.ObjectByID(context.Background(), id, parkedAtEither(), additional.Properties{}, "") - require.Nil(t, err) - - assert.Equal(t, expectedSchemaWithAllRefs, res.Schema) - }) - - t.Run("asking for refs with vectors of both types", func(t *testing.T) { - res, err := repo.ObjectByID(context.Background(), id, parkedAtEitherWithVector(), additional.Properties{}, "") - require.Nil(t, err) - - assert.Equal(t, expectedSchemaWithAllRefsWithVector, res.Schema) - }) - }) -} - -func parkedAtGarage() search.SelectProperties { - return parkedAtGarageWithVector(false) -} - -func parkedAtGarageWithVector(withVector bool) search.SelectProperties { - return search.SelectProperties{ - search.SelectProperty{ - Name: "parkedAt", - IsPrimitive: false, - Refs: []search.SelectClass{ - { - ClassName: "MultiRefParkingGarage", - RefProperties: search.SelectProperties{ - search.SelectProperty{ - Name: "name", - IsPrimitive: true, - }, - }, - AdditionalProperties: additional.Properties{ - Vector: withVector, - }, - }, - }, - }, - } -} - -func parkedAtLot() search.SelectProperties { - return search.SelectProperties{ - search.SelectProperty{ - Name: "parkedAt", - IsPrimitive: false, - Refs: []search.SelectClass{ - { - ClassName: "MultiRefParkingLot", - RefProperties: search.SelectProperties{ - search.SelectProperty{ - Name: "name", - IsPrimitive: true, - }, - }, - }, - }, - }, - } -} - -func parkedAtLotWithVector() search.SelectProperties { - return search.SelectProperties{ - search.SelectProperty{ - Name: "parkedAt", - IsPrimitive: false, - Refs: []search.SelectClass{ - { - ClassName: "MultiRefParkingLot", - RefProperties: search.SelectProperties{ - search.SelectProperty{ - Name: "name", - IsPrimitive: true, - }, - }, - AdditionalProperties: additional.Properties{ - Vector: true, - }, - }, - }, - }, - } -} - -func parkedAtEither() search.SelectProperties { - return search.SelectProperties{ - search.SelectProperty{ - Name: "parkedAt", - IsPrimitive: false, - Refs: []search.SelectClass{ - { - ClassName: "MultiRefParkingLot", - RefProperties: search.SelectProperties{ - search.SelectProperty{ - Name: "name", - IsPrimitive: true, - }, - }, - }, - { - ClassName: "MultiRefParkingGarage", - RefProperties: search.SelectProperties{ - search.SelectProperty{ - Name: "name", - IsPrimitive: true, - }, - }, - }, - }, - }, - } -} - -func parkedAtEitherWithVector() search.SelectProperties { - return search.SelectProperties{ - search.SelectProperty{ - Name: "parkedAt", - IsPrimitive: false, - Refs: []search.SelectClass{ - { - ClassName: "MultiRefParkingLot", - RefProperties: search.SelectProperties{ - search.SelectProperty{ - Name: "name", - IsPrimitive: true, - }, - }, - AdditionalProperties: additional.Properties{ - Vector: true, - }, - }, - { - ClassName: "MultiRefParkingGarage", - RefProperties: search.SelectProperties{ - search.SelectProperty{ - Name: "name", - IsPrimitive: true, - }, - }, - AdditionalProperties: additional.Properties{ - Vector: true, - }, - }, - }, - }, - } -} diff --git a/adapters/repos/db/crud_update_integration_test.go b/adapters/repos/db/crud_update_integration_test.go deleted file mode 100644 index 7227c924b648728aacdd0c5a01a9a3ad04d09463..0000000000000000000000000000000000000000 --- a/adapters/repos/db/crud_update_integration_test.go +++ /dev/null @@ -1,391 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package db - -import ( - "context" - "testing" - - "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/adapters/repos/db/inverted" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/dto" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - libschema "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/search" - enthnsw "github.com/weaviate/weaviate/entities/vectorindex/hnsw" -) - -// Updates are non trivial, because vector indices are built under the -// assumption that items are immutable (this is true for HNSW, the assumption -// is that this is generally true in the majority of cases). Therefore an -// update is essentially a delete and a new import with a new doc ID. This -// needs to be tested extensively because there's a lot of room for error -// regarding the clean up of Doc ID pointers in the inverted indices, etc. -func TestUpdateJourney(t *testing.T) { - dirName := t.TempDir() - - logger := logrus.New() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - defer repo.Shutdown(context.Background()) - migrator := NewMigrator(repo, logger) - - schema := libschema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{updateTestClass()}, - }, - } - - t.Run("add schema", func(t *testing.T) { - err := migrator.AddClass(context.Background(), updateTestClass(), schemaGetter.shardState) - require.Nil(t, err) - }) - schemaGetter.schema = schema - - t.Run("import some objects", func(t *testing.T) { - for _, res := range updateTestData() { - err := repo.PutObject(context.Background(), res.Object(), res.Vector, nil) - require.Nil(t, err) - } - - tracker := getTracker(repo, "UpdateTestClass") - - require.Nil(t, err) - - sum, count, mean, err := tracker.PropertyTally("name") - require.Nil(t, err) - assert.Equal(t, 4, sum) - assert.Equal(t, 4, count) - assert.InEpsilon(t, 1, mean, 0.1) - }) - - searchVector := []float32{0.1, 0.1, 0.1} - - t.Run("verify vector search results are initially as expected", - func(t *testing.T) { - res, err := repo.VectorSearch(context.Background(), dto.GetParams{ - ClassName: "UpdateTestClass", - SearchVector: searchVector, - Pagination: &filters.Pagination{ - Limit: 100, - }, - }) - - expectedInAnyOrder := []interface{}{ - "element-0", "element-1", "element-2", "element-3", - } - - require.Nil(t, err) - require.Len(t, res, 4) - assert.ElementsMatch(t, expectedInAnyOrder, extractPropValues(res, "name")) - }) - - searchInv := func(t *testing.T, op filters.Operator, value int) []interface{} { - res, err := repo.ObjectSearch(context.Background(), 0, 100, - &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: op, - On: &filters.Path{ - Class: "UpdateTestClass", - Property: libschema.PropertyName("intProp"), - }, - Value: &filters.Value{ - Type: libschema.DataTypeInt, - Value: value, - }, - }, - }, nil, additional.Properties{}, "") - require.Nil(t, err) - return extractPropValues(res, "name") - } - - t.Run("verify invert index results are initially as expected", - func(t *testing.T) { - expectedInAnyOrder := []interface{}{ - "element-0", "element-1", "element-2", "element-3", - } - assert.ElementsMatch(t, expectedInAnyOrder, searchInv(t, filters.OperatorGreaterThanEqual, 0)) - - expectedInAnyOrder = []interface{}{"element-0"} - assert.ElementsMatch(t, expectedInAnyOrder, searchInv(t, filters.OperatorEqual, 0)) - - expectedInAnyOrder = []interface{}{"element-1"} - assert.ElementsMatch(t, expectedInAnyOrder, searchInv(t, filters.OperatorEqual, 10)) - - expectedInAnyOrder = []interface{}{"element-2"} - assert.ElementsMatch(t, expectedInAnyOrder, searchInv(t, filters.OperatorEqual, 20)) - - expectedInAnyOrder = []interface{}{"element-3"} - assert.ElementsMatch(t, expectedInAnyOrder, searchInv(t, filters.OperatorEqual, 30)) - }) - - t.Run("update vector position of one item to move it into a different direction", - func(t *testing.T) { - // updating element-0 to be very far away from our search vector - updatedVec := []float32{-0.1, -0.12, -0.105} - id := updateTestData()[0].ID - - old, err := repo.ObjectByID(context.Background(), id, search.SelectProperties{}, additional.Properties{}, "") - require.Nil(t, err) - - err = repo.PutObject(context.Background(), old.Object(), updatedVec, nil) - require.Nil(t, err) - - tracker := getTracker(repo, "UpdateTestClass") - - require.Nil(t, err) - - sum, count, mean, err := tracker.PropertyTally("name") - require.Nil(t, err) - assert.Equal(t, 4, sum) - assert.Equal(t, 4, count) - assert.InEpsilon(t, 1, mean, 0.1) - }) - - t.Run("verify new vector search results are as expected", func(t *testing.T) { - res, err := repo.VectorSearch(context.Background(), dto.GetParams{ - ClassName: "UpdateTestClass", - SearchVector: searchVector, - Pagination: &filters.Pagination{ - Limit: 100, - }, - }) - - expectedInAnyOrder := []interface{}{ - "element-0", "element-1", "element-2", "element-3", - } - - require.Nil(t, err) - require.Len(t, res, 4) - assert.ElementsMatch(t, expectedInAnyOrder, extractPropValues(res, "name")) - }) - - t.Run("verify invert results still work properly", func(t *testing.T) { - expectedInAnyOrder := []interface{}{ - "element-0", "element-1", "element-2", "element-3", - } - assert.ElementsMatch(t, expectedInAnyOrder, searchInv(t, filters.OperatorGreaterThanEqual, 0)) - - expectedInAnyOrder = []interface{}{"element-0"} - assert.ElementsMatch(t, expectedInAnyOrder, searchInv(t, filters.OperatorEqual, 0)) - - expectedInAnyOrder = []interface{}{"element-1"} - assert.ElementsMatch(t, expectedInAnyOrder, searchInv(t, filters.OperatorEqual, 10)) - - expectedInAnyOrder = []interface{}{"element-2"} - assert.ElementsMatch(t, expectedInAnyOrder, searchInv(t, filters.OperatorEqual, 20)) - - expectedInAnyOrder = []interface{}{"element-3"} - assert.ElementsMatch(t, expectedInAnyOrder, searchInv(t, filters.OperatorEqual, 30)) - }) - - t.Run("update a second object and modify vector and invert props at the same time", - func(t *testing.T) { - // this time we are updating element-2 and move it away from the search - // vector, as well as updating an invert prop - - updatedVec := []float32{-0.1, -0.12, -0.105123} - id := updateTestData()[2].ID - - old, err := repo.ObjectByID(context.Background(), id, search.SelectProperties{}, additional.Properties{}, "") - require.Nil(t, err) - - old.Schema.(map[string]interface{})["intProp"] = int64(21) - err = repo.PutObject(context.Background(), old.Object(), updatedVec, nil) - require.Nil(t, err) - - tracker := getTracker(repo, "UpdateTestClass") - - require.Nil(t, err) - - sum, count, mean, err := tracker.PropertyTally("name") - require.Nil(t, err) - assert.Equal(t, 4, sum) - assert.Equal(t, 4, count) - assert.InEpsilon(t, 1, mean, 0.1) - }) - - t.Run("verify new vector search results are as expected", func(t *testing.T) { - res, err := repo.VectorSearch(context.Background(), dto.GetParams{ - ClassName: "UpdateTestClass", - SearchVector: searchVector, - Pagination: &filters.Pagination{ - Limit: 100, - }, - }) - - expectedInAnyOrder := []interface{}{ - "element-0", "element-1", "element-2", "element-3", - } - - require.Nil(t, err) - require.Len(t, res, 4) - assert.ElementsMatch(t, expectedInAnyOrder, extractPropValues(res, "name")) - }) - - t.Run("verify invert results have been updated correctly", func(t *testing.T) { - expectedInAnyOrder := []interface{}{ - "element-0", "element-1", "element-2", "element-3", - } - assert.ElementsMatch(t, expectedInAnyOrder, searchInv(t, filters.OperatorGreaterThanEqual, 0)) - - expectedInAnyOrder = []interface{}{"element-0"} - assert.ElementsMatch(t, expectedInAnyOrder, searchInv(t, filters.OperatorEqual, 0)) - - expectedInAnyOrder = []interface{}{"element-1"} - assert.ElementsMatch(t, expectedInAnyOrder, searchInv(t, filters.OperatorEqual, 10)) - - expectedInAnyOrder = []interface{}{} // value is no longer 20, but 21 - assert.ElementsMatch(t, expectedInAnyOrder, searchInv(t, filters.OperatorEqual, 20)) - - expectedInAnyOrder = []interface{}{"element-2"} - assert.ElementsMatch(t, expectedInAnyOrder, searchInv(t, filters.OperatorEqual, 21)) - - expectedInAnyOrder = []interface{}{"element-3"} - assert.ElementsMatch(t, expectedInAnyOrder, searchInv(t, filters.OperatorEqual, 30)) - }) - - t.Run("test recount", func(t *testing.T) { - tracker := getTracker(repo, "UpdateTestClass") - - require.Nil(t, err) - - sum, count, mean, err := tracker.PropertyTally("name") - require.Nil(t, err) - assert.Equal(t, 4, sum) - assert.Equal(t, 4, count) - assert.InEpsilon(t, 1, mean, 0.1) - - tracker.Clear() - sum, count, mean, err = tracker.PropertyTally("name") - require.Nil(t, err) - assert.Equal(t, 0, sum) - assert.Equal(t, 0, count) - assert.Equal(t, float64(0), mean) - - logger := logrus.New() - migrator := NewMigrator(repo, logger) - migrator.RecountProperties(context.Background()) - - sum, count, mean, err = tracker.PropertyTally("name") - require.Nil(t, err) - assert.Equal(t, 4, sum) - assert.Equal(t, 4, count) - assert.Equal(t, float64(1), mean) - }) -} - -func updateTestClass() *models.Class { - return &models.Class{ - Class: "UpdateTestClass", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: &models.InvertedIndexConfig{ - CleanupIntervalSeconds: 3, - }, - Properties: []*models.Property{ - { - DataType: []string{string(schema.DataTypeInt)}, - Name: "intProp", - }, - { - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - Name: "name", - }, - }, - } -} - -func updateTestData() search.Results { - return search.Results{ - search.Result{ - ClassName: "UpdateTestClass", - ID: "426b0b29-9ded-40b6-b786-da3d1fec412f", - Schema: map[string]interface{}{ - "intProp": int64(0), - "name": "element-0", - }, - Vector: []float32{0.89379513, 0.67022973, 0.57360715}, - }, - search.Result{ - ClassName: "UpdateTestClass", - ID: "a1560f12-f0f0-4439-b5b8-b7bcecf5fed7", - - Schema: map[string]interface{}{ - "intProp": int64(10), - "name": "element-1", - }, - Vector: []float32{0.9660323, 0.35887036, 0.6072966}, - }, - search.Result{ - ClassName: "UpdateTestClass", - ID: "0c73f145-5dc4-49a9-bd58-82725f8b13fa", - - Schema: map[string]interface{}{ - "intProp": int64(20), - "name": "element-2", - }, - Vector: []float32{0.8194746, 0.56142205, 0.5130103}, - }, - search.Result{ - ClassName: "UpdateTestClass", - ID: "aec8462e-276a-4989-a612-8314c35d163a", - Schema: map[string]interface{}{ - "intProp": int64(30), - "name": "element-3", - }, - Vector: []float32{0.42401955, 0.8278863, 0.5952888}, - }, - } -} - -func extractPropValues(in search.Results, propName string) []interface{} { - out := make([]interface{}, len(in)) - - for i, res := range in { - out[i] = res.Schema.(map[string]interface{})[propName] - } - - return out -} - -func getTracker(repo *DB, className string) *inverted.JsonShardMetaData { - index := repo.GetIndex("UpdateTestClass") - var shard ShardLike - index.ForEachShard(func(name string, shardv ShardLike) error { - shard = shardv - return nil - }) - - tracker := shard.GetPropertyLengthTracker() - - return tracker -} diff --git a/adapters/repos/db/delete_filter_integration_test.go b/adapters/repos/db/delete_filter_integration_test.go deleted file mode 100644 index 8d1bc8d66441d9367a9e14b318270e8e6f59c78c..0000000000000000000000000000000000000000 --- a/adapters/repos/db/delete_filter_integration_test.go +++ /dev/null @@ -1,264 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package db - -import ( - "context" - "testing" - - "github.com/go-openapi/strfmt" - "github.com/google/uuid" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/dto" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/search" - enthnsw "github.com/weaviate/weaviate/entities/vectorindex/hnsw" -) - -// This test aims to prevent a regression on -// https://github.com/weaviate/weaviate/issues/1308 where we -// discovered that if the first n doc ids are deleted and a filter would return -// <= n doc ids, it would return no results instead of skipping the deleted ids -// and returning the next ones -func Test_FilterSearchesOnDeletedDocIDsWithLimits(t *testing.T) { - className := "DeletedDocIDLimitTestClass" - dirName := t.TempDir() - - logger, _ := test.NewNullLogger() - thingclass := &models.Class{ - Class: className, - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{{ - Name: "unrelatedProp", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, { - Name: "boolProp", - DataType: []string{string(schema.DataTypeBoolean)}, - }}, - } - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - migrator := NewMigrator(repo, logger) - defer repo.Shutdown(testCtx()) - - t.Run("creating the thing class", func(t *testing.T) { - require.Nil(t, - migrator.AddClass(context.Background(), thingclass, schemaGetter.shardState)) - - // update schema getter so it's in sync with class - schemaGetter.schema = schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{thingclass}, - }, - } - }) - - var things []*models.Object - t.Run("importing 10 initial items", func(t *testing.T) { - things = make([]*models.Object, 10) - for i := 0; i < 10; i++ { - things[i] = &models.Object{ - Class: className, - ID: mustNewUUID(), - Properties: map[string]interface{}{ - "boolProp": i < 5, - "unrelatedProp": "initialValue", - }, - Vector: []float32{0.1}, - } - - err := repo.PutObject(context.Background(), things[i], things[i].Vector, nil) - require.Nil(t, err) - } - }) - - t.Run("updating the first 5 elements", func(t *testing.T) { - // The idea is that the first 5 elements can be found with a boolProp==true - // search, however, the bug occurred if those items all had received an - // update - - for i := 0; i < 5; i++ { - things[i].Properties.(map[string]interface{})["unrelatedProp"] = "updatedValue" - - err := repo.PutObject(context.Background(), things[i], things[i].Vector, nil) - require.Nil(t, err) - } - }) - - t.Run("searching for boolProp == true with a strict limit", func(t *testing.T) { - res, err := repo.Search(context.Background(), dto.GetParams{ - ClassName: className, - Pagination: &filters.Pagination{ - // important as the first 5 doc ids we encounter now should all be - // deleted - Limit: 5, - }, - Filters: buildFilter("boolProp", true, eq, dtBool), - }) - expectedIDs := []strfmt.UUID{ - things[0].ID, things[1].ID, things[2].ID, things[3].ID, things[4].ID, - } - - require.Nil(t, err) - - require.Len(t, res, 5) - actualIDs := extractIDs(res) - assert.Equal(t, expectedIDs, actualIDs) - }) -} - -func mustNewUUID() strfmt.UUID { - id, err := uuid.NewRandom() - if err != nil { - panic(err) - } - - return strfmt.UUID(id.String()) -} - -func extractIDs(in []search.Result) []strfmt.UUID { - out := make([]strfmt.UUID, len(in)) - for i, res := range in { - out[i] = res.ID - } - - return out -} - -// This bug aims to prevent a regression on -// https://github.com/weaviate/weaviate/issues/1765 -func TestLimitOneAfterDeletion(t *testing.T) { - className := "Test" - dirName := t.TempDir() - - logger, _ := test.NewNullLogger() - class := &models.Class{ - Class: className, - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{{ - Name: "author", - DataType: []string{string(schema.DataTypeText)}, - Tokenization: "word", - }}, - } - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - defer repo.Shutdown(testCtx()) - migrator := NewMigrator(repo, logger) - - t.Run("creating the class", func(t *testing.T) { - require.Nil(t, - migrator.AddClass(context.Background(), class, schemaGetter.shardState)) - - // update schema getter so it's in sync with class - schemaGetter.schema = schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{class}, - }, - } - }) - - firstID := strfmt.UUID("114c8f57-f244-4419-b5c1-cb2f635b76d0") - - t.Run("import single object", func(t *testing.T) { - err := repo.PutObject(context.Background(), &models.Object{ - Class: "Test", - ID: firstID, - Properties: map[string]interface{}{ - "author": "Simon", - }, - }, []float32{0, 1}, nil) - - require.Nil(t, err) - }) - - t.Run("delete first object", func(t *testing.T) { - err := repo.DeleteObject(context.Background(), "Test", firstID, nil, "") - require.Nil(t, err) - }) - - t.Run("create another object", func(t *testing.T) { - // new object has a different ID, but the same inverted props as the - // previously deleted one - err := repo.PutObject(context.Background(), &models.Object{ - Class: "Test", - ID: "74776bbd-2de0-421d-8cef-757e16466dd9", - Properties: map[string]interface{}{ - "author": "Simon", - }, - }, []float32{0, 1}, nil) - - require.Nil(t, err) - }) - - t.Run("query with high limit", func(t *testing.T) { - res, err := repo.Search(context.Background(), dto.GetParams{ - Filters: buildFilter("author", "Simon", eq, dtText), - ClassName: "Test", - Pagination: &filters.Pagination{ - Offset: 0, - Limit: 100, - }, - }) - - require.Nil(t, err) - require.Len(t, res, 1) - assert.Equal(t, "Simon", res[0].Object().Properties.(map[string]interface{})["author"]) - }) - - t.Run("query with limit 1", func(t *testing.T) { - res, err := repo.Search(context.Background(), dto.GetParams{ - Filters: buildFilter("author", "Simon", eq, dtText), - ClassName: "Test", - Pagination: &filters.Pagination{ - Offset: 0, - Limit: 1, - }, - }) - - require.Nil(t, err) - require.Len(t, res, 1) - assert.Equal(t, "Simon", res[0].Object().Properties.(map[string]interface{})["author"]) - }) -} diff --git a/adapters/repos/db/disk_use_unix.go b/adapters/repos/db/disk_use_unix.go deleted file mode 100644 index ca848cfe6106b3f0b8fb1f5af0112918464b6212..0000000000000000000000000000000000000000 --- a/adapters/repos/db/disk_use_unix.go +++ /dev/null @@ -1,35 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build !windows - -package db - -import ( - "syscall" -) - -func (db *DB) getDiskUse(diskPath string) diskUse { - fs := syscall.Statfs_t{} - - err := syscall.Statfs(diskPath, &fs) - if err != nil { - db.logger.WithField("action", "read_disk_use"). - WithField("path", diskPath). - Errorf("failed to read disk usage: %s", err) - } - - return diskUse{ - total: fs.Blocks * uint64(fs.Bsize), - free: fs.Bfree * uint64(fs.Bsize), - avail: fs.Bfree * uint64(fs.Bsize), - } -} diff --git a/adapters/repos/db/disk_use_windows.go b/adapters/repos/db/disk_use_windows.go deleted file mode 100644 index d54ec817b765881377e27bbfbf64943f94d37343..0000000000000000000000000000000000000000 --- a/adapters/repos/db/disk_use_windows.go +++ /dev/null @@ -1,44 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build windows - -package db - -import ( - "syscall" - - "golang.org/x/sys/windows" -) - -func (db *DB) getDiskUse(diskPath string) diskUse { - var freeBytesAvailable, totalNumberOfBytes, totalNumberOfFreeBytes uint64 - - _, _ = syscall.UTF16PtrFromString(diskPath) - - err := windows.GetDiskFreeSpaceEx( - syscall.StringToUTF16Ptr(diskPath), - &freeBytesAvailable, - &totalNumberOfBytes, - &totalNumberOfFreeBytes, - ) - if err != nil { - db.logger.WithField("action", "read_disk_use"). - WithField("path", diskPath). - Errorf("failed to read disk usage: %s", err) - } - - return diskUse{ - total: totalNumberOfBytes, - free: totalNumberOfFreeBytes, - avail: freeBytesAvailable, - } -} diff --git a/adapters/repos/db/docid/scan.go b/adapters/repos/db/docid/scan.go deleted file mode 100644 index 54707996df773b2f9ab66c2a61cb9cc0bf870a33..0000000000000000000000000000000000000000 --- a/adapters/repos/db/docid/scan.go +++ /dev/null @@ -1,127 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package docid - -import ( - "encoding/binary" - - "github.com/weaviate/weaviate/entities/storobj" - - "github.com/weaviate/weaviate/entities/models" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv" -) - -// ObjectScanFn is called once per object, if false or an error is returned, -// the scanning will stop -type ObjectScanFn func(prop *models.PropertySchema, docID uint64) (bool, error) - -// ScanObjectsLSM calls the provided scanFn on each object for the -// specified pointer. If a pointer does not resolve to an object-id, the item -// will be skipped. The number of times scanFn is called can therefore be -// smaller than the input length of pointers. -func ScanObjectsLSM(store *lsmkv.Store, pointers []uint64, scan ObjectScanFn, properties []string) error { - return newObjectScannerLSM(store, pointers, scan, properties).Do() -} - -type objectScannerLSM struct { - store *lsmkv.Store - pointers []uint64 - scanFn ObjectScanFn - objectsBucket *lsmkv.Bucket - properties []string -} - -func newObjectScannerLSM(store *lsmkv.Store, pointers []uint64, - scan ObjectScanFn, properties []string, -) *objectScannerLSM { - return &objectScannerLSM{ - store: store, - pointers: pointers, - scanFn: scan, - properties: properties, - } -} - -func (os *objectScannerLSM) Do() error { - if err := os.init(); err != nil { - return errors.Wrap(err, "init object scanner") - } - - if err := os.scan(); err != nil { - return errors.Wrap(err, "scan") - } - - return nil -} - -func (os *objectScannerLSM) init() error { - bucket := os.store.Bucket(helpers.ObjectsBucketLSM) - if bucket == nil { - return errors.Errorf("objects bucket not found") - } - os.objectsBucket = bucket - - return nil -} - -func (os *objectScannerLSM) scan() error { - // each object is scanned one after the other, so we can reuse the same memory allocations for all objects - docIDBytes := make([]byte, 8) - - // Preallocate strings needed for json unmarshalling - propStrings := make([][]string, len(os.properties)) - for i := range os.properties { - propStrings[i] = []string{os.properties[i]} - } - - // The typed properties are needed for extraction from json - var properties models.PropertySchema - propertiesTyped := map[string]interface{}{} - - for _, prop := range os.properties { - propertiesTyped[prop] = nil - } - - for _, id := range os.pointers { - binary.LittleEndian.PutUint64(docIDBytes, id) - res, err := os.objectsBucket.GetBySecondary(0, docIDBytes) - if err != nil { - return err - } - - if res == nil { - continue - } - - if len(os.properties) > 0 { - err = storobj.UnmarshalPropertiesFromObject(res, &propertiesTyped, os.properties, propStrings) - if err != nil { - return errors.Wrapf(err, "unmarshal data object") - } - properties = propertiesTyped - } - - continueScan, err := os.scanFn(&properties, id) - if err != nil { - return errors.Wrapf(err, "scan") - } - - if !continueScan { - break - } - } - - return nil -} diff --git a/adapters/repos/db/fakes_for_test.go b/adapters/repos/db/fakes_for_test.go deleted file mode 100644 index 59237357bc840242d5fb8604740e1ac94f2cecf9..0000000000000000000000000000000000000000 --- a/adapters/repos/db/fakes_for_test.go +++ /dev/null @@ -1,341 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package db - -import ( - "context" - "encoding/json" - "fmt" - "io" - - "github.com/go-openapi/strfmt" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/aggregation" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/search" - "github.com/weaviate/weaviate/entities/searchparams" - "github.com/weaviate/weaviate/entities/storobj" - "github.com/weaviate/weaviate/usecases/objects" - "github.com/weaviate/weaviate/usecases/replica" - "github.com/weaviate/weaviate/usecases/sharding" -) - -type fakeSchemaGetter struct { - schema schema.Schema - shardState *sharding.State -} - -func (f *fakeSchemaGetter) GetSchemaSkipAuth() schema.Schema { - return f.schema -} - -func (f *fakeSchemaGetter) CopyShardingState(class string) *sharding.State { - return f.shardState -} - -func (f *fakeSchemaGetter) ShardOwner(class, shard string) (string, error) { - ss := f.shardState - x, ok := ss.Physical[shard] - if !ok { - return "", fmt.Errorf("shard not found") - } - if len(x.BelongsToNodes) < 1 || x.BelongsToNodes[0] == "" { - return "", fmt.Errorf("owner node not found") - } - return x.BelongsToNodes[0], nil -} - -func (f *fakeSchemaGetter) ShardReplicas(class, shard string) ([]string, error) { - ss := f.shardState - x, ok := ss.Physical[shard] - if !ok { - return nil, fmt.Errorf("shard not found") - } - return x.BelongsToNodes, nil -} - -func (f *fakeSchemaGetter) TenantShard(class, tenant string) (string, string) { - return tenant, models.TenantActivityStatusHOT -} - -func (f *fakeSchemaGetter) ShardFromUUID(class string, uuid []byte) string { - ss := f.shardState - return ss.Shard("", string(uuid)) -} - -func (f *fakeSchemaGetter) Nodes() []string { - return []string{"node1"} -} - -func (f *fakeSchemaGetter) NodeName() string { - return "node1" -} - -func (f *fakeSchemaGetter) ClusterHealthScore() int { - return 0 -} - -func (f fakeSchemaGetter) ResolveParentNodes(_ string, shard string) (map[string]string, error) { - return nil, nil -} - -func singleShardState() *sharding.State { - config, err := sharding.ParseConfig(nil, 1) - if err != nil { - panic(err) - } - - s, err := sharding.InitState("test-index", config, fakeNodes{[]string{"node1"}}, 1, false) - if err != nil { - panic(err) - } - - return s -} - -func multiShardState() *sharding.State { - config, err := sharding.ParseConfig(map[string]interface{}{ - "desiredCount": json.Number("3"), - }, 1) - if err != nil { - panic(err) - } - - s, err := sharding.InitState("multi-shard-test-index", config, - fakeNodes{[]string{"node1"}}, 1, false) - if err != nil { - panic(err) - } - - return s -} - -func fixedMultiShardState() *sharding.State { - raw := []byte(`{"indexID":"multi-shard-test-index","config":{"virtualPerPhysical":128,"desiredCount":3,"actualCount":0,"desiredVirtualCount":384,"actualVirtualCount":0,"key":"_id","strategy":"hash","function":"murmur3"},"physical":{"10HAiobndwqQ":{"name":"10HAiobndwqQ","ownsVirtual":["P1ib1Jb4kcVl","cJnMLj5AELbO","OXKlgrTvR2oT","Wwn8XeDuTnEq","aFlA2RXHaTZq","oeibhOxhslJ4","xNWB5azVw7oD","wXr2DKiiO4rQ","6IcKuxi0brIT","Ib0bC21DkrSY","xLBqexoUmcoS","k7s6GojJDHw4","Ij8HLoZKUIGh","cYXwzjMjpL2p","18xE7smej6jn","wKvtkC7ISgqS","JQoplEib8uSu","CXmIRbTMCrVg","xeqIBs9YD2cA","48RZ8vWnLnYd","N2liOQZHORRd","FLYoUFplOmxQ","yPsCEegMk4Q2","MDVwNcNC9Gra","2PgGKvp3Nl9s","NKgE3HgdeGQm","NEObTyTfjbkD","MKp04TysWuOK","aV4XlXJRhzIX","jcaPi3kHStaK","8BgRFjMgSMe4","z6T8lD2o39SQ","ioHJiTM0L2b0","rblINp3n1sZ0","c5elvzxRiyRR","3W0h1o0dwd3u","JSuWUzk63Inq","2AmfpIT0zs3p","XS08CGL3mnm6","Uvrp2Des8Ab3","L215BrEJZez6","8FlXfnU8F53r","mYSXTeMJNg5X","LE68IkTZ1KBq","3DpBKBAyrh49","comyKg1nNYkV","VEUsNjgJ1M4k","nMusHd6e2Fkf","mbwwycSRznHa","UdvpOIbYGk3E","w54br5SJaUNd","glARwN18WtJG","DUFmLbD5AQdB","OyAthRrzjE9C","aRIS8UTR2sMs","dtl4DVfMpjM2","YceXk2KF4xnV","ntXhJG4ehh97","6JQHfQJXq9tC","81UYRgmxCxvy","UUvwR3z3SjqT","hVoXGtgS5V4b","pJQ37OFnd0h0","7jZlJg79Tx3C","81Tr7kFVuWFO","uoVfvn3j1vVs","si39KY4GMFCu","opTmmfmor5Xi","jUJHmkXSdjJM","taj02QCfJJ2v","vCn9VOgxbsvC","BhoBBmk2O5Fx","0Ym87WlhMWSH","8v1uPEAZANss","KcyNwToNc5uf","uTwHaNDD5s4i","nN3HYdDnNFvW","wBKHrHzpqkXI","DSeQOWKAAfPy","Dr8dRry1VNRG","qvDj32cx0hSO","FrFb8tNKNyUW","pgxcn0O1NKoW","LQdu95vAEz5N","TR2Tz7fHYqme","KoUT5JzcEigR","EAnJxQirsjKw","R8oWsRMA8Z9G","Q8PJeKnlygDF","KpWXuBwKB3Np","BpVhOJVXrM9b","zPbNJqGLV31c","nXTYGKAWzMD3","DfDu4yl3r1qn","VExbdXg0Ek0R","KISjTsevB5iE","3qa8pPUU95kc","iWYrenDncRMZ","Cqju2FPfqDzN","X4f5TfQ2Zh3w","Ol4Jrf6D4hI8","sTxMnH5hC27G","f1r8QwYMUnF2","M8Xxly7aXuVg","JprYyU27D3SD","1pZrOBR3Mp0n","7dAlRglk4gUq","meZAtRpcJSGj","RAKQqUg9tkVy","evXAOdPjRLLO","ZhRSfF5kiST2","K2KoDH0smQno","97FKS8uAgBGZ","A6uDhUMxgGzC","JcvKnGUxrjbi","7oYBFzAQhylH","0qi2orWv2yBQ","HruSBzNO6D2X","Pw1ZyGgzqr4w","Ua0wIIuK7MVF","0yiZLUGHfCfg","aTgCTGMPZTPv","04uFLfQjFQ5M","MaWgxbZRgif0","SPWsrRGsxBtQ","CMG5X2zuOP8T","rjKzLkrjnadI","3zdVkbUS6vRy"],"ownsPercentage":0.3115328681081298},"5UnDHhImGGus":{"name":"5UnDHhImGGus","ownsVirtual":["kk3uy33qESSP","AYNqxnC70vWf","1L72JpPhK3AH","N5YYIOLBJsT5","Kdpf7oo9rHRd","2zc4yEUgcsxU","04flFfZJX9y4","siSzkkL8ch9H","vSuf6j765jqp","PT37knJ3yCVo","IfrQL0OScc3u","CGR9oTerTPfV","saP105LXdjTN","wJbYdeQm8NxO","tHpmYN4knZgr","7KyVTpba2SB8","cghDTFg7bkKV","9SKvNyPOO16k","mpyKh0Uvq9pK","RBxCMlncsWP4","wzCL9BumCknu","SliMd6ggZIIT","MRj5CdiSSHkh","A9NHb9n0ZxAu","o0I69xAYCy8l","cOB2G6rQQDMp","fq4nrOTRJSFl","MVHggjKS2WIh","f7KnlZipyMoU","kC8TDVP2qpDU","RSCYtxYOBl4u","FRl5xH4a7yhi","Pn8nYsbPoQXd","VawtjdDTNeP4","xWK39CVQOq7c","IiRbFljPDBMc","DInND1HhGkNp","odhRcafbqKku","wMSpj5IJ6Jr2","urILYmbzLgFM","f8XCDnms7AuS","85iBn44HwzLG","4sh42JIRZ5x4","RlZ9POMVYbei","9DuGADJSRpBx","zY4rPaerd1ys","PA9vlkamSA8b","wgvpTSabR2Gp","lUb5dLRn4Q7i","CIohP98pYrEU","uydQQfk9x5dv","U0547wfcqSsu","OejkNdX5mxtz","ndz1d2vCwoIq","uImQxXnTgMUt","73XdZtGlJTnb","jTVVZexNxBtp","w0MlBKADJImt","0s0krALwyqS4","tq85nEKxsXd6","itKBOdSzdcD2","T1LLWfgdGEtq","vBUqaK2WYdna","t0KeI7GeoLhD","rohnBjf5J9D8","SQ4Xh9ZRBYnc","VhJWJyF49q7N","hsdU1XjrtTL0","EgXtER7wworu","xp4mWs2aVMbB","hK5JiEF1Odsf","vxlhQDXcCDoc","Ytl1gLqLrmUF","7JIKqhHrk8gs","fnaBlG1IB2tO","UtdVz5Nl280P","S2fMEGZ6FK5b","HYBYL0oNGwfa","jPIF7y5qI3fh","H6BtD1FRUcEb","PeiLrsaz0A1Z","58AEK168LCOI","b2T01LA7C14D","ofCP6twUePhY","gLBhIXTrzmM5","ZdNfpA9QwBPW","I6lAuYCU2JkE","6BoSzEVYohm2","iH2Nxldim4CZ","WCrjXqYEOGO3","3CBHlZozw0Z1","KnwHXiKEEMD5","u2NQ8vav4GGA","AJexcI7np4a6","xY2JLJ1fxlZP","aGQsUOxlSSI7","6mcznERQIGgE","WqDu1P0hsKi2","vQ8Nr3MMIvqX","cCuxaZc2WPTR","e2bHBdxkqCWf","AVApjgCPtUh6","h03hg1MFr8UK","bd7afitI4wfb","mA1JyNKe3vCB","94dJ1MbkYawY","rkAx8oOGiGp5","5dkf0GDkoFS3","ULSfcu1j23rT","IXPatqSYHjfd","Od8MgXRVCkCj","qJn82gKq9WyS","0suDv1qp3ndv","ptSPQhrhvHUx","bFNikojhtkrf","EVmTAbpjGR6I","wAmFbdhLWtEt","3XGWS3W2ZWY1","JPenxeL41DVm","WYWMO6T7W33P","gzKXC2EEuP0y","wrXomqZRYWlP","3xb5qWUIdy3l","q6uOaqi6vhFp","1tkGefsVhdMZ","VxASALmWn2ch","hiizguisdpTg","tfavKaxeSt8e"],"ownsPercentage":0.34078263030984246},"qc4oYcGtIr3y":{"name":"qc4oYcGtIr3y","ownsVirtual":["ZpSrDlljAmhU","PoixZZemlqGZ","uq0LzAAY5iuw","hDhqptRj4wek","f2c8q7U2y0Wc","zO4HfORWFYQx","E4OaMLfzLI3l","qjwhfQLQXRtk","2iX5gmw2cSrS","uMkCwD02ZjTb","UixcolZ7TGh7","PzFLgH6tNojh","a0D06UHaaNTQ","EfHluzACLDad","zLp00SBesBQM","MWsu1YNw7GyZ","EcEpTU5Z1N2m","FVhhe4CrETSN","oAGIG611zPBS","93O50DIDBjvT","psuAMG40uLmP","4ZKvAdv7y3W3","2hY6C71VJZa6","TsFT7FF0NLc9","mdW6a1iyMMQB","Pqolehn1Kgzx","6lS4e7DEBTSK","cTPggh41RFbY","IeTvnejJYwTE","shwsWnArjWTe","NPiIKetDYl31","fgTWGQMwfW8m","AmEGOYK67uL5","3KocyAIgLFK9","NtPJZ0DCROIt","qPo0bL1DanzJ","S7pN5St5UtGx","WmgtIhiclMxv","iZr9zB8YgfBS","Ec4HWsyJ98Pj","Vbw2kZ8Pzb10","W1p8DeaOGQwa","tf20ZJKNTYuN","VYlWIOMzTG6Q","E7qcPMaKvpuY","HnNcLsgB4Qle","9UENJD0jVfQJ","QEzoqtGH5L3q","c3Fmgjy0kiGM","nPdFzMMAl8e1","vPsB4hMU3u1Q","zcHMGe7BCirf","jlzvsigHmfto","6t35oHrP3E89","Un0x4I8BmBxf","ENBKLdnBVJMj","PPBVWwBeAt88","6OVVNvUPztYL","4mW6gvrMBHur","lGwgDbeyOBH7","nQF4tc5aCEj6","YhUygFdgZypN","bYeSTo07NtUQ","8Di0Ol5egcop","k0raV8wB89r3","YZq35jzzcmLB","IFXmevuG7Zrh","3C5qZO15G38a","2RvYTa9iW6Om","Xgg7Xi7IPaxF","KoRSyfj9NHxw","ngQi7Ikjaotp","pZxtjHxpnAP7","gbdKhYN535Uf","Xaqm44zLtfaI","qT0JE0lB5nbG","iHFJaeDAyc3y","VQjkAf5kPk7D","6DXk6xSSav8y","M3kWMrkyMrGT","XZG9YcJJQOQB","6MSMwtkaDKYl","Nk6OCjX40weL","uLUgLpz1cxlj","PquK03kCX2s6","gtoqy7jnEc10","rX2vYIXPVa18","8irbpjVkihDh","OH4vVMfkcJJp","tcrH84kQAesz","qhFDFwxgRnVT","IiO65JAUXrTl","g7Up5jauHf3w","aOm3ikcxu3yr","s4qjXxgUNJsT","gtk7Xsi7whtY","qcgtLYQ2qEet","HKmvdMeJ7ZUM","wkkemoIF2SPl","jv8aLeLUQAs3","XqtbcXDoer5y","6QrhJx5gbnzD","a6g520corlxT","UXRmLgtMDrXc","10sk2R5hdJ3w","7dRVmMSfU9TF","BVUpofeiRXJ0","Fbu1Q7R1XE6Q","KhlfU1eDGz8J","asfBwHMBj3DG","h10KwfryF0Hs","gpqm1I3ZXfFw","Jrt8yXZwyfFi","l7bTeARR9sxT","SF2wURCU9LAf","TEVjUfLfnSxk","WadYBzIWkqkJ","MPs72GrXF80t","XIK3cSdZyVhk","5G1XAnbIxrsI","THznNzcBVet8","scRBGW6E4EA4","maRmPi8FWELR","LKWfqTsgLJ2h","ThChEv0oUVF8","BwQaadmEuOWL","xQhsuMpqsB9Y","90E1fTakTcmi"],"ownsPercentage":0.3476845015820278}},"virtual":[{"name":"wMSpj5IJ6Jr2","upper":890026480952456,"ownsPercentage":0.0009179861551301902,"assignedToPhysical":"5UnDHhImGGus"},{"name":"DInND1HhGkNp","upper":20777246139474581,"ownsPercentage":0.001078088337923306,"assignedToPhysical":"5UnDHhImGGus"},{"name":"c5elvzxRiyRR","upper":160478001236453307,"ownsPercentage":0.007573193108700487,"assignedToPhysical":"10HAiobndwqQ"},{"name":"aOm3ikcxu3yr","upper":169380448435422008,"ownsPercentage":0.00048260262967796797,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"gtoqy7jnEc10","upper":196373783656105797,"ownsPercentage":0.0014633116344447423,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"3qa8pPUU95kc","upper":212860879729245124,"ownsPercentage":0.0008937672690237444,"assignedToPhysical":"10HAiobndwqQ"},{"name":"wrXomqZRYWlP","upper":214005801208299024,"ownsPercentage":0.00006206631774577777,"assignedToPhysical":"5UnDHhImGGus"},{"name":"psuAMG40uLmP","upper":251004648148060861,"ownsPercentage":0.002005711511577422,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"BVUpofeiRXJ0","upper":299839797586403218,"ownsPercentage":0.0026473587557352524,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"PoixZZemlqGZ","upper":312733734053195943,"ownsPercentage":0.000698981696459337,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"SPWsrRGsxBtQ","upper":332381381418126825,"ownsPercentage":0.0010651010978643578,"assignedToPhysical":"10HAiobndwqQ"},{"name":"kk3uy33qESSP","upper":348216885392409783,"ownsPercentage":0.0008584443905660211,"assignedToPhysical":"5UnDHhImGGus"},{"name":"qT0JE0lB5nbG","upper":454179755158415031,"ownsPercentage":0.005744258680155073,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"1pZrOBR3Mp0n","upper":493292167589083547,"ownsPercentage":0.0021202881264239925,"assignedToPhysical":"10HAiobndwqQ"},{"name":"RSCYtxYOBl4u","upper":560609404341315928,"ownsPercentage":0.0036492747166245697,"assignedToPhysical":"5UnDHhImGGus"},{"name":"HruSBzNO6D2X","upper":577899831252128784,"ownsPercentage":0.0009373159209952564,"assignedToPhysical":"10HAiobndwqQ"},{"name":"ENBKLdnBVJMj","upper":638066638177577744,"ownsPercentage":0.003261649139004383,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"2RvYTa9iW6Om","upper":685076307063247990,"ownsPercentage":0.0025483992566833953,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"UXRmLgtMDrXc","upper":698279650281856233,"ownsPercentage":0.0007157546700843405,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"PA9vlkamSA8b","upper":723564314835311292,"ownsPercentage":0.0013706844119711601,"assignedToPhysical":"5UnDHhImGGus"},{"name":"7JIKqhHrk8gs","upper":806465637273563321,"ownsPercentage":0.004494089694473707,"assignedToPhysical":"5UnDHhImGGus"},{"name":"b2T01LA7C14D","upper":833505543382924548,"ownsPercentage":0.0014658362473786754,"assignedToPhysical":"5UnDHhImGGus"},{"name":"odhRcafbqKku","upper":859042461960569242,"ownsPercentage":0.0013843591300233909,"assignedToPhysical":"5UnDHhImGGus"},{"name":"73XdZtGlJTnb","upper":909598291510966844,"ownsPercentage":0.002740637011517397,"assignedToPhysical":"5UnDHhImGGus"},{"name":"U0547wfcqSsu","upper":925074112165663730,"ownsPercentage":0.0008389459187409203,"assignedToPhysical":"5UnDHhImGGus"},{"name":"oAGIG611zPBS","upper":960831498707513441,"ownsPercentage":0.0019384118085538698,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"aFlA2RXHaTZq","upper":986327436288144592,"ownsPercentage":0.0013821375457237555,"assignedToPhysical":"10HAiobndwqQ"},{"name":"nQF4tc5aCEj6","upper":1025597420805531024,"ownsPercentage":0.0021288301263611246,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"Xgg7Xi7IPaxF","upper":1085798861384560310,"ownsPercentage":0.003263526633127028,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"2zc4yEUgcsxU","upper":1157579081722514043,"ownsPercentage":0.0038912135415948815,"assignedToPhysical":"5UnDHhImGGus"},{"name":"WYWMO6T7W33P","upper":1203198582761505852,"ownsPercentage":0.0024730381067089822,"assignedToPhysical":"5UnDHhImGGus"},{"name":"rjKzLkrjnadI","upper":1270538940193839560,"ownsPercentage":0.00365052809120433,"assignedToPhysical":"10HAiobndwqQ"},{"name":"shwsWnArjWTe","upper":1370658063932621614,"ownsPercentage":0.005427468573246627,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"Un0x4I8BmBxf","upper":1399326713180590975,"ownsPercentage":0.0015541305898436652,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"MaWgxbZRgif0","upper":1403710789048654944,"ownsPercentage":0.00023766122902481144,"assignedToPhysical":"10HAiobndwqQ"},{"name":"si39KY4GMFCu","upper":1452671883616463620,"ownsPercentage":0.0026541862548843196,"assignedToPhysical":"10HAiobndwqQ"},{"name":"EcEpTU5Z1N2m","upper":1457938633566877424,"ownsPercentage":0.00028551108690882845,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"MKp04TysWuOK","upper":1489663018037007097,"ownsPercentage":0.0017197823281639994,"assignedToPhysical":"10HAiobndwqQ"},{"name":"5G1XAnbIxrsI","upper":1500937357544703297,"ownsPercentage":0.0006111831693791686,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"A6uDhUMxgGzC","upper":1534480789179842443,"ownsPercentage":0.0018183930725718428,"assignedToPhysical":"10HAiobndwqQ"},{"name":"R8oWsRMA8Z9G","upper":1636736621003898917,"ownsPercentage":0.005543299750647721,"assignedToPhysical":"10HAiobndwqQ"},{"name":"6mcznERQIGgE","upper":1672314219256841825,"ownsPercentage":0.0019286654658828594,"assignedToPhysical":"5UnDHhImGGus"},{"name":"iHFJaeDAyc3y","upper":1824597043576342080,"ownsPercentage":0.00825526844797153,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"jv8aLeLUQAs3","upper":1845224475960403555,"ownsPercentage":0.001118215350179865,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"uoVfvn3j1vVs","upper":1849612883575629432,"ownsPercentage":0.00023789605350899137,"assignedToPhysical":"10HAiobndwqQ"},{"name":"fnaBlG1IB2tO","upper":2005466986221275098,"ownsPercentage":0.008448867833959392,"assignedToPhysical":"5UnDHhImGGus"},{"name":"Ec4HWsyJ98Pj","upper":2032295005301287389,"ownsPercentage":0.0014543498285015945,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"iZr9zB8YgfBS","upper":2141312756521615279,"ownsPercentage":0.005909864135628187,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"evXAOdPjRLLO","upper":2198349852896446041,"ownsPercentage":0.00309198719009283,"assignedToPhysical":"10HAiobndwqQ"},{"name":"BpVhOJVXrM9b","upper":2253577169221936466,"ownsPercentage":0.002993878817032044,"assignedToPhysical":"10HAiobndwqQ"},{"name":"9DuGADJSRpBx","upper":2257868273525488385,"ownsPercentage":0.0002326212304136444,"assignedToPhysical":"5UnDHhImGGus"},{"name":"asfBwHMBj3DG","upper":2290812012486862222,"ownsPercentage":0.0017858836675858434,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"hiizguisdpTg","upper":2311656255664796327,"ownsPercentage":0.0011299686868666155,"assignedToPhysical":"5UnDHhImGGus"},{"name":"Ib0bC21DkrSY","upper":2330298822826987746,"ownsPercentage":0.0010106155908977431,"assignedToPhysical":"10HAiobndwqQ"},{"name":"I6lAuYCU2JkE","upper":2364963278411676435,"ownsPercentage":0.0018791639026473376,"assignedToPhysical":"5UnDHhImGGus"},{"name":"uLUgLpz1cxlj","upper":2368186543570048126,"ownsPercentage":0.00017473355436017104,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"MVHggjKS2WIh","upper":2388232167176925479,"ownsPercentage":0.001086675433170157,"assignedToPhysical":"5UnDHhImGGus"},{"name":"qhFDFwxgRnVT","upper":2399872610948814081,"ownsPercentage":0.0006310297213088491,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"meZAtRpcJSGj","upper":2471390138583843854,"ownsPercentage":0.003876972941634569,"assignedToPhysical":"10HAiobndwqQ"},{"name":"K2KoDH0smQno","upper":2490612946772682019,"ownsPercentage":0.0010420705199805241,"assignedToPhysical":"10HAiobndwqQ"},{"name":"6OVVNvUPztYL","upper":2564700834413600126,"ownsPercentage":0.0040163124367572686,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"zLp00SBesBQM","upper":2599202103553865989,"ownsPercentage":0.001870317547769167,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"UtdVz5Nl280P","upper":2610648603275380327,"ownsPercentage":0.0006205159932710283,"assignedToPhysical":"5UnDHhImGGus"},{"name":"4ZKvAdv7y3W3","upper":2766853347457801461,"ownsPercentage":0.008467876149756172,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"HYBYL0oNGwfa","upper":2768980890630006328,"ownsPercentage":0.00011533434646806092,"assignedToPhysical":"5UnDHhImGGus"},{"name":"Kdpf7oo9rHRd","upper":2818247286096185879,"ownsPercentage":0.002670736649748094,"assignedToPhysical":"5UnDHhImGGus"},{"name":"bd7afitI4wfb","upper":2886637236964887853,"ownsPercentage":0.0037074266654011797,"assignedToPhysical":"5UnDHhImGGus"},{"name":"XIK3cSdZyVhk","upper":2964061555958206534,"ownsPercentage":0.004197180742788341,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"wkkemoIF2SPl","upper":2978179529135431872,"ownsPercentage":0.0007653368594919895,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"l7bTeARR9sxT","upper":3050093338750210702,"ownsPercentage":0.0038984554308026083,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"A9NHb9n0ZxAu","upper":3055274346561046711,"ownsPercentage":0.0002808629962086384,"assignedToPhysical":"5UnDHhImGGus"},{"name":"8BgRFjMgSMe4","upper":3057496754161327476,"ownsPercentage":0.0001204769574186351,"assignedToPhysical":"10HAiobndwqQ"},{"name":"fq4nrOTRJSFl","upper":3062424065410077862,"ownsPercentage":0.0002671100780203716,"assignedToPhysical":"5UnDHhImGGus"},{"name":"uImQxXnTgMUt","upper":3219508588015719193,"ownsPercentage":0.008515569033644234,"assignedToPhysical":"5UnDHhImGGus"},{"name":"3KocyAIgLFK9","upper":3284826999312512617,"ownsPercentage":0.0035409181715642573,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"t0KeI7GeoLhD","upper":3299284401243240197,"ownsPercentage":0.0007837373290895484,"assignedToPhysical":"5UnDHhImGGus"},{"name":"Vbw2kZ8Pzb10","upper":3328043137210732710,"ownsPercentage":0.0015590142006946198,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"ZpSrDlljAmhU","upper":3351786810141955740,"ownsPercentage":0.0012871470887408638,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"UixcolZ7TGh7","upper":3412825665246073188,"ownsPercentage":0.003308922965495603,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"jUJHmkXSdjJM","upper":3467954473586604049,"ownsPercentage":0.002988538688467028,"assignedToPhysical":"10HAiobndwqQ"},{"name":"VawtjdDTNeP4","upper":3475360162886164233,"ownsPercentage":0.00040146322136679026,"assignedToPhysical":"5UnDHhImGGus"},{"name":"gtk7Xsi7whtY","upper":3528953607002574204,"ownsPercentage":0.002905306427099608,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"nMusHd6e2Fkf","upper":3574259978294764044,"ownsPercentage":0.0024560633091213558,"assignedToPhysical":"10HAiobndwqQ"},{"name":"iH2Nxldim4CZ","upper":3603054592879595105,"ownsPercentage":0.0015609591844378313,"assignedToPhysical":"5UnDHhImGGus"},{"name":"7dAlRglk4gUq","upper":3675694468025527315,"ownsPercentage":0.003937815522114775,"assignedToPhysical":"10HAiobndwqQ"},{"name":"6MSMwtkaDKYl","upper":3762304686931209074,"ownsPercentage":0.004695149374849264,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"Ytl1gLqLrmUF","upper":3823342486002833461,"ownsPercentage":0.003308865717859443,"assignedToPhysical":"5UnDHhImGGus"},{"name":"8Di0Ol5egcop","upper":3881547047175440066,"ownsPercentage":0.003155275583595276,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"maRmPi8FWELR","upper":3955364129023922481,"ownsPercentage":0.0040016320253332465,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"wJbYdeQm8NxO","upper":4044968890060555644,"ownsPercentage":0.004857483829048108,"assignedToPhysical":"5UnDHhImGGus"},{"name":"aTgCTGMPZTPv","upper":4049380745841614516,"ownsPercentage":0.00023916718112583805,"assignedToPhysical":"10HAiobndwqQ"},{"name":"lGwgDbeyOBH7","upper":4111968510537073276,"ownsPercentage":0.003392889522691397,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"DSeQOWKAAfPy","upper":4204527092302830545,"ownsPercentage":0.005017610771630561,"assignedToPhysical":"10HAiobndwqQ"},{"name":"10sk2R5hdJ3w","upper":4224041044327154580,"ownsPercentage":0.0010578534589275013,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"IeTvnejJYwTE","upper":4235756052132322162,"ownsPercentage":0.0006350718456523667,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"nPdFzMMAl8e1","upper":4236420495176881018,"ownsPercentage":0.000036019529620179725,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"LKWfqTsgLJ2h","upper":4263385304921123528,"ownsPercentage":0.0014617652652683015,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"vPsB4hMU3u1Q","upper":4292354302930656194,"ownsPercentage":0.0015704125288331784,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"FRl5xH4a7yhi","upper":4322318686813382506,"ownsPercentage":0.001624372505142075,"assignedToPhysical":"5UnDHhImGGus"},{"name":"3zdVkbUS6vRy","upper":4343664613540309444,"ownsPercentage":0.0011571650065525288,"assignedToPhysical":"10HAiobndwqQ"},{"name":"FrFb8tNKNyUW","upper":4496485165952040014,"ownsPercentage":0.008284418746261658,"assignedToPhysical":"10HAiobndwqQ"},{"name":"u2NQ8vav4GGA","upper":4532019006561781396,"ownsPercentage":0.0019262933592917625,"assignedToPhysical":"5UnDHhImGGus"},{"name":"itKBOdSzdcD2","upper":4537357993340643413,"ownsPercentage":0.0002894270532256792,"assignedToPhysical":"5UnDHhImGGus"},{"name":"f2c8q7U2y0Wc","upper":4555516035749608803,"ownsPercentage":0.00098434945139421,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"EVmTAbpjGR6I","upper":4572129041729378275,"ownsPercentage":0.0009005928587390369,"assignedToPhysical":"5UnDHhImGGus"},{"name":"xNWB5azVw7oD","upper":4613011128090221427,"ownsPercentage":0.0022162223424083078,"assignedToPhysical":"10HAiobndwqQ"},{"name":"Pqolehn1Kgzx","upper":4651985171511437890,"ownsPercentage":0.0021127871273913636,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"Ua0wIIuK7MVF","upper":4696630627126192084,"ownsPercentage":0.002420234998456083,"assignedToPhysical":"10HAiobndwqQ"},{"name":"3C5qZO15G38a","upper":4784907021305312069,"ownsPercentage":0.004785472917409431,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"h03hg1MFr8UK","upper":4796223604197399554,"ownsPercentage":0.0006134731878356771,"assignedToPhysical":"5UnDHhImGGus"},{"name":"THznNzcBVet8","upper":4915944216646993407,"ownsPercentage":0.006490067405457239,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"ioHJiTM0L2b0","upper":4957842579607055296,"ownsPercentage":0.0022713148072442645,"assignedToPhysical":"10HAiobndwqQ"},{"name":"aV4XlXJRhzIX","upper":4993935340105560932,"ownsPercentage":0.00195659246717394,"assignedToPhysical":"10HAiobndwqQ"},{"name":"0qi2orWv2yBQ","upper":5024894406509825712,"ownsPercentage":0.0016782943526813434,"assignedToPhysical":"10HAiobndwqQ"},{"name":"CGR9oTerTPfV","upper":5158458025247331609,"ownsPercentage":0.007240498280011476,"assignedToPhysical":"5UnDHhImGGus"},{"name":"97FKS8uAgBGZ","upper":5174323415549741130,"ownsPercentage":0.0008600645316601428,"assignedToPhysical":"10HAiobndwqQ"},{"name":"BhoBBmk2O5Fx","upper":5187434264602339602,"ownsPercentage":0.000710740551297839,"assignedToPhysical":"10HAiobndwqQ"},{"name":"nXTYGKAWzMD3","upper":5195193606611309874,"ownsPercentage":0.00042063477315918037,"assignedToPhysical":"10HAiobndwqQ"},{"name":"WqDu1P0hsKi2","upper":5224777598831476475,"ownsPercentage":0.0016037514317949445,"assignedToPhysical":"5UnDHhImGGus"},{"name":"PT37knJ3yCVo","upper":5375620358405560754,"ownsPercentage":0.00817720238169654,"assignedToPhysical":"5UnDHhImGGus"},{"name":"f8XCDnms7AuS","upper":5495283831850589643,"ownsPercentage":0.00648696989381309,"assignedToPhysical":"5UnDHhImGGus"},{"name":"7jZlJg79Tx3C","upper":5506419823422963041,"ownsPercentage":0.0006036833127773753,"assignedToPhysical":"10HAiobndwqQ"},{"name":"Uvrp2Des8Ab3","upper":5542895169334733013,"ownsPercentage":0.001977332463985063,"assignedToPhysical":"10HAiobndwqQ"},{"name":"k0raV8wB89r3","upper":5552370617218929180,"ownsPercentage":0.0005136650590659331,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"MWsu1YNw7GyZ","upper":5663161801830126264,"ownsPercentage":0.0060060021523851235,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"qjwhfQLQXRtk","upper":5698664742506073073,"ownsPercentage":0.0019246182705242756,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"opTmmfmor5Xi","upper":5721258211780544165,"ownsPercentage":0.0012247944235683026,"assignedToPhysical":"10HAiobndwqQ"},{"name":"gLBhIXTrzmM5","upper":5774022938275618813,"ownsPercentage":0.0028603815548281695,"assignedToPhysical":"5UnDHhImGGus"},{"name":"dtl4DVfMpjM2","upper":5819402277572133063,"ownsPercentage":0.0024600189125618785,"assignedToPhysical":"10HAiobndwqQ"},{"name":"3CBHlZozw0Z1","upper":5827388583406330893,"ownsPercentage":0.0004329385067785473,"assignedToPhysical":"5UnDHhImGGus"},{"name":"PzFLgH6tNojh","upper":5831627539924522089,"ownsPercentage":0.00022979429330472422,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"pZxtjHxpnAP7","upper":5872740343819337424,"ownsPercentage":0.0022287295649864648,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"81Tr7kFVuWFO","upper":5894639010637646722,"ownsPercentage":0.0011871291069473584,"assignedToPhysical":"10HAiobndwqQ"},{"name":"WadYBzIWkqkJ","upper":5920889332354674332,"ownsPercentage":0.0014230327917022375,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"9UENJD0jVfQJ","upper":5950482002594647221,"ownsPercentage":0.001604221868191287,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"cJnMLj5AELbO","upper":5962987215025360822,"ownsPercentage":0.000677908924238621,"assignedToPhysical":"10HAiobndwqQ"},{"name":"KISjTsevB5iE","upper":5986387043408926511,"ownsPercentage":0.0012685072384624944,"assignedToPhysical":"10HAiobndwqQ"},{"name":"IfrQL0OScc3u","upper":6084330577947103338,"ownsPercentage":0.005309529646360018,"assignedToPhysical":"5UnDHhImGGus"},{"name":"tfavKaxeSt8e","upper":6101649858778365729,"ownsPercentage":0.0009388800951570619,"assignedToPhysical":"5UnDHhImGGus"},{"name":"Q8PJeKnlygDF","upper":6120544210147852964,"ownsPercentage":0.0010242648401251264,"assignedToPhysical":"10HAiobndwqQ"},{"name":"1L72JpPhK3AH","upper":6139009236851798467,"ownsPercentage":0.0010009911033710284,"assignedToPhysical":"5UnDHhImGGus"},{"name":"58AEK168LCOI","upper":6203537455014676060,"ownsPercentage":0.0034980817159405236,"assignedToPhysical":"5UnDHhImGGus"},{"name":"ntXhJG4ehh97","upper":6250479863873075726,"ownsPercentage":0.002544753083298985,"assignedToPhysical":"10HAiobndwqQ"},{"name":"EfHluzACLDad","upper":6257881409784994672,"ownsPercentage":0.00040123860787268626,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"Pw1ZyGgzqr4w","upper":6276168982882809802,"ownsPercentage":0.0009913713241069314,"assignedToPhysical":"10HAiobndwqQ"},{"name":"hDhqptRj4wek","upper":6281860672615602714,"ownsPercentage":0.00030854711867037577,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"Nk6OCjX40weL","upper":6284740492439475368,"ownsPercentage":0.0001561153454704777,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"IiRbFljPDBMc","upper":6322365517697796882,"ownsPercentage":0.002039656706244708,"assignedToPhysical":"5UnDHhImGGus"},{"name":"81UYRgmxCxvy","upper":6374289897584535219,"ownsPercentage":0.0028148262739082167,"assignedToPhysical":"10HAiobndwqQ"},{"name":"ULSfcu1j23rT","upper":6442652778775914548,"ownsPercentage":0.003705959215253095,"assignedToPhysical":"5UnDHhImGGus"},{"name":"aGQsUOxlSSI7","upper":6628874745657227247,"ownsPercentage":0.010095113052862145,"assignedToPhysical":"5UnDHhImGGus"},{"name":"xY2JLJ1fxlZP","upper":6635203822280934241,"ownsPercentage":0.0003430999312625172,"assignedToPhysical":"5UnDHhImGGus"},{"name":"zO4HfORWFYQx","upper":6699485210543020577,"ownsPercentage":0.0034847010402069105,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"DfDu4yl3r1qn","upper":6727354121803122774,"ownsPercentage":0.001510776706650427,"assignedToPhysical":"10HAiobndwqQ"},{"name":"M3kWMrkyMrGT","upper":6740502144735913698,"ownsPercentage":0.0007127557513810577,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"PquK03kCX2s6","upper":6747123146248742074,"ownsPercentage":0.00035892521121191684,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"CMG5X2zuOP8T","upper":6980966254492504618,"ownsPercentage":0.012676660298932517,"assignedToPhysical":"10HAiobndwqQ"},{"name":"zPbNJqGLV31c","upper":6987529689123558289,"ownsPercentage":0.00035580450429774927,"assignedToPhysical":"10HAiobndwqQ"},{"name":"85iBn44HwzLG","upper":7008758635743647494,"ownsPercentage":0.001150823502253976,"assignedToPhysical":"5UnDHhImGGus"},{"name":"jTVVZexNxBtp","upper":7184765512514992474,"ownsPercentage":0.009541351908394035,"assignedToPhysical":"5UnDHhImGGus"},{"name":"scRBGW6E4EA4","upper":7288564078426028941,"ownsPercentage":0.005626931533081278,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"E7qcPMaKvpuY","upper":7317924845818590019,"ownsPercentage":0.0015916503896428141,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"VxASALmWn2ch","upper":7320272934945058874,"ownsPercentage":0.00012729016660535615,"assignedToPhysical":"5UnDHhImGGus"},{"name":"W1p8DeaOGQwa","upper":7329379115336767830,"ownsPercentage":0.0004936470281867876,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"siSzkkL8ch9H","upper":7365032735264167895,"ownsPercentage":0.0019327866091129811,"assignedToPhysical":"5UnDHhImGGus"},{"name":"h10KwfryF0Hs","upper":7437357468956214082,"ownsPercentage":0.003920731669667602,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"JSuWUzk63Inq","upper":7474332566751363072,"ownsPercentage":0.002004424067868226,"assignedToPhysical":"10HAiobndwqQ"},{"name":"nN3HYdDnNFvW","upper":7485158231506008231,"ownsPercentage":0.0005868604622793019,"assignedToPhysical":"10HAiobndwqQ"},{"name":"BwQaadmEuOWL","upper":7507090743021187083,"ownsPercentage":0.0011889638316410127,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"Wwn8XeDuTnEq","upper":7528762709048575525,"ownsPercentage":0.0011748396324463297,"assignedToPhysical":"10HAiobndwqQ"},{"name":"f1r8QwYMUnF2","upper":7530763667960730121,"ownsPercentage":0.00010847219998061223,"assignedToPhysical":"10HAiobndwqQ"},{"name":"4mW6gvrMBHur","upper":7545234120228279501,"ownsPercentage":0.0007844447892662416,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"urILYmbzLgFM","upper":7773164834661528343,"ownsPercentage":0.012356148788235075,"assignedToPhysical":"5UnDHhImGGus"},{"name":"TsFT7FF0NLc9","upper":7773394258423625280,"ownsPercentage":0.000012437087064264831,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"04flFfZJX9y4","upper":7854961957136350003,"ownsPercentage":0.004421793807448961,"assignedToPhysical":"5UnDHhImGGus"},{"name":"qvDj32cx0hSO","upper":7883310802927429864,"ownsPercentage":0.0015367940097072666,"assignedToPhysical":"10HAiobndwqQ"},{"name":"xLBqexoUmcoS","upper":7905257767824779877,"ownsPercentage":0.0011897473510584994,"assignedToPhysical":"10HAiobndwqQ"},{"name":"JprYyU27D3SD","upper":7910815885169914432,"ownsPercentage":0.00030130614502621244,"assignedToPhysical":"10HAiobndwqQ"},{"name":"jlzvsigHmfto","upper":7933413828155010817,"ownsPercentage":0.0012250369439072532,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"93O50DIDBjvT","upper":7993784483485729679,"ownsPercentage":0.0032726997831969496,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"YceXk2KF4xnV","upper":7996374436592219689,"ownsPercentage":0.0001404016392346025,"assignedToPhysical":"10HAiobndwqQ"},{"name":"VYlWIOMzTG6Q","upper":8006107196738942655,"ownsPercentage":0.0005276139847678688,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"vSuf6j765jqp","upper":8017216651818234722,"ownsPercentage":0.0006022447666049291,"assignedToPhysical":"5UnDHhImGGus"},{"name":"NEObTyTfjbkD","upper":8050138084024281148,"ownsPercentage":0.0017846744159564893,"assignedToPhysical":"10HAiobndwqQ"},{"name":"ndz1d2vCwoIq","upper":8066848342316863678,"ownsPercentage":0.0009058649171805948,"assignedToPhysical":"5UnDHhImGGus"},{"name":"vQ8Nr3MMIvqX","upper":8069602844311102736,"ownsPercentage":0.00014932185231348206,"assignedToPhysical":"5UnDHhImGGus"},{"name":"uTwHaNDD5s4i","upper":8130579194898972165,"ownsPercentage":0.00330553458888029,"assignedToPhysical":"10HAiobndwqQ"},{"name":"mdW6a1iyMMQB","upper":8250566081955192821,"ownsPercentage":0.006504502180806364,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"S7pN5St5UtGx","upper":8340682427773445817,"ownsPercentage":0.004885216895630245,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"ofCP6twUePhY","upper":8447921142937480046,"ownsPercentage":0.005813422397770006,"assignedToPhysical":"5UnDHhImGGus"},{"name":"RAKQqUg9tkVy","upper":8504599523728357916,"ownsPercentage":0.0030725411793215236,"assignedToPhysical":"10HAiobndwqQ"},{"name":"6QrhJx5gbnzD","upper":8513094344746041558,"ownsPercentage":0.0004605051701124064,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"OejkNdX5mxtz","upper":8539594040314563465,"ownsPercentage":0.0014365513752797974,"assignedToPhysical":"5UnDHhImGGus"},{"name":"6lS4e7DEBTSK","upper":8575073699044782028,"ownsPercentage":0.001923356153717363,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"xp4mWs2aVMbB","upper":8602429679782843757,"ownsPercentage":0.001482970687333907,"assignedToPhysical":"5UnDHhImGGus"},{"name":"tHpmYN4knZgr","upper":8635392688536345460,"ownsPercentage":0.0017869282851102623,"assignedToPhysical":"5UnDHhImGGus"},{"name":"2AmfpIT0zs3p","upper":8670860499000032734,"ownsPercentage":0.0019227138579016923,"assignedToPhysical":"10HAiobndwqQ"},{"name":"mA1JyNKe3vCB","upper":8676513197675841415,"ownsPercentage":0.0003064334092358853,"assignedToPhysical":"5UnDHhImGGus"},{"name":"jPIF7y5qI3fh","upper":8706429952052375065,"ownsPercentage":0.0016217905044376502,"assignedToPhysical":"5UnDHhImGGus"},{"name":"gbdKhYN535Uf","upper":8799657348866550157,"ownsPercentage":0.005053867308054841,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"8FlXfnU8F53r","upper":8899248676446159866,"ownsPercentage":0.005398856686126419,"assignedToPhysical":"10HAiobndwqQ"},{"name":"M8Xxly7aXuVg","upper":9005461565059215222,"ownsPercentage":0.005757812229011776,"assignedToPhysical":"10HAiobndwqQ"},{"name":"2hY6C71VJZa6","upper":9067556512448386258,"ownsPercentage":0.003366173842985617,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"comyKg1nNYkV","upper":9070826503298640431,"ownsPercentage":0.0001772665591926648,"assignedToPhysical":"10HAiobndwqQ"},{"name":"XS08CGL3mnm6","upper":9127646470634449826,"ownsPercentage":0.003080216601301997,"assignedToPhysical":"10HAiobndwqQ"},{"name":"CXmIRbTMCrVg","upper":9160013201600391890,"ownsPercentage":0.0017546040014764118,"assignedToPhysical":"10HAiobndwqQ"},{"name":"JQoplEib8uSu","upper":9242936734742863962,"ownsPercentage":0.004495293739162097,"assignedToPhysical":"10HAiobndwqQ"},{"name":"3XGWS3W2ZWY1","upper":9275480562589952859,"ownsPercentage":0.0017642044426404019,"assignedToPhysical":"5UnDHhImGGus"},{"name":"SliMd6ggZIIT","upper":9291129844433481316,"ownsPercentage":0.0008483492686295756,"assignedToPhysical":"5UnDHhImGGus"},{"name":"3W0h1o0dwd3u","upper":9351332456188418356,"ownsPercentage":0.0032635901227002053,"assignedToPhysical":"10HAiobndwqQ"},{"name":"z6T8lD2o39SQ","upper":9370501884911054932,"ownsPercentage":0.0010391768133194302,"assignedToPhysical":"10HAiobndwqQ"},{"name":"ptSPQhrhvHUx","upper":9557106538402328786,"ownsPercentage":0.010115858535557194,"assignedToPhysical":"5UnDHhImGGus"},{"name":"Fbu1Q7R1XE6Q","upper":9581667180131361575,"ownsPercentage":0.0013314350560127742,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"fgTWGQMwfW8m","upper":9616806305989840223,"ownsPercentage":0.0019048958297502058,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"WCrjXqYEOGO3","upper":9633001025346879568,"ownsPercentage":0.0008779174954847554,"assignedToPhysical":"5UnDHhImGGus"},{"name":"tq85nEKxsXd6","upper":9753345111991751138,"ownsPercentage":0.006523866009307677,"assignedToPhysical":"5UnDHhImGGus"},{"name":"zY4rPaerd1ys","upper":9769847995880917641,"ownsPercentage":0.0008946231282455177,"assignedToPhysical":"5UnDHhImGGus"},{"name":"OXKlgrTvR2oT","upper":9784001926533888562,"ownsPercentage":0.0007672861181580123,"assignedToPhysical":"10HAiobndwqQ"},{"name":"qJn82gKq9WyS","upper":9804919859287427473,"ownsPercentage":0.001133963406764629,"assignedToPhysical":"5UnDHhImGGus"},{"name":"AmEGOYK67uL5","upper":9837139302196143865,"ownsPercentage":0.0017466194998951496,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"cYXwzjMjpL2p","upper":9844833667801973668,"ownsPercentage":0.00041711239528692084,"assignedToPhysical":"10HAiobndwqQ"},{"name":"Ol4Jrf6D4hI8","upper":10005914080522808168,"ownsPercentage":0.008732186670839522,"assignedToPhysical":"10HAiobndwqQ"},{"name":"EAnJxQirsjKw","upper":10253565889542214993,"ownsPercentage":0.013425231467940306,"assignedToPhysical":"10HAiobndwqQ"},{"name":"UdvpOIbYGk3E","upper":10281661603144781072,"ownsPercentage":0.0015230716862716341,"assignedToPhysical":"10HAiobndwqQ"},{"name":"uMkCwD02ZjTb","upper":10369513924269020804,"ownsPercentage":0.004762483871039744,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"wzCL9BumCknu","upper":10436218359642224243,"ownsPercentage":0.003616054687302305,"assignedToPhysical":"5UnDHhImGGus"},{"name":"xeqIBs9YD2cA","upper":10556580804488844452,"ownsPercentage":0.006524861209418617,"assignedToPhysical":"10HAiobndwqQ"},{"name":"rohnBjf5J9D8","upper":10561798726114405432,"ownsPercentage":0.00028286409811461547,"assignedToPhysical":"5UnDHhImGGus"},{"name":"5dkf0GDkoFS3","upper":10565577061354284927,"ownsPercentage":0.0002048239637727944,"assignedToPhysical":"5UnDHhImGGus"},{"name":"1tkGefsVhdMZ","upper":10640136392647408019,"ownsPercentage":0.004041869448353525,"assignedToPhysical":"5UnDHhImGGus"},{"name":"E4OaMLfzLI3l","upper":10734061622421226437,"ownsPercentage":0.0050916969085987055,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"FLYoUFplOmxQ","upper":10799040812602198121,"ownsPercentage":0.0035225289580279127,"assignedToPhysical":"10HAiobndwqQ"},{"name":"LE68IkTZ1KBq","upper":10816649793406241553,"ownsPercentage":0.0009545847621499717,"assignedToPhysical":"10HAiobndwqQ"},{"name":"vBUqaK2WYdna","upper":10915645950426407919,"ownsPercentage":0.005366592425449025,"assignedToPhysical":"5UnDHhImGGus"},{"name":"k7s6GojJDHw4","upper":11001855601817753716,"ownsPercentage":0.004673434566385755,"assignedToPhysical":"10HAiobndwqQ"},{"name":"DUFmLbD5AQdB","upper":11008974609608993012,"ownsPercentage":0.00038592218566014386,"assignedToPhysical":"10HAiobndwqQ"},{"name":"a6g520corlxT","upper":11021271569193417168,"ownsPercentage":0.0006666195148199558,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"RBxCMlncsWP4","upper":11052047453477516654,"ownsPercentage":0.001668364030049158,"assignedToPhysical":"5UnDHhImGGus"},{"name":"90E1fTakTcmi","upper":11188766378182331430,"ownsPercentage":0.007411547759242115,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"wBKHrHzpqkXI","upper":11243260275836301483,"ownsPercentage":0.0029541201111818534,"assignedToPhysical":"10HAiobndwqQ"},{"name":"YZq35jzzcmLB","upper":11356751102912206819,"ownsPercentage":0.006152350063643664,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"6BoSzEVYohm2","upper":11467132392555628345,"ownsPercentage":0.005983781701657466,"assignedToPhysical":"5UnDHhImGGus"},{"name":"RlZ9POMVYbei","upper":11534987860118061834,"ownsPercentage":0.003678452267310503,"assignedToPhysical":"5UnDHhImGGus"},{"name":"mYSXTeMJNg5X","upper":11584157949447943989,"ownsPercentage":0.0026655158836382275,"assignedToPhysical":"10HAiobndwqQ"},{"name":"a0D06UHaaNTQ","upper":11686754521353283792,"ownsPercentage":0.0055617713074667344,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"uydQQfk9x5dv","upper":11719597803869881397,"ownsPercentage":0.0017804379128025154,"assignedToPhysical":"5UnDHhImGGus"},{"name":"WmgtIhiclMxv","upper":11783499227615313778,"ownsPercentage":0.0034641031224857295,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"yPsCEegMk4Q2","upper":11785036598019323645,"ownsPercentage":0.00008334101659712077,"assignedToPhysical":"10HAiobndwqQ"},{"name":"hVoXGtgS5V4b","upper":11822411649275226527,"ownsPercentage":0.002026105588420349,"assignedToPhysical":"10HAiobndwqQ"},{"name":"rX2vYIXPVa18","upper":11839341004889018262,"ownsPercentage":0.0009177422067626335,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"VEUsNjgJ1M4k","upper":11891663786092253089,"ownsPercentage":0.0028364236525515456,"assignedToPhysical":"10HAiobndwqQ"},{"name":"7KyVTpba2SB8","upper":12025520187007287877,"ownsPercentage":0.007256370033658569,"assignedToPhysical":"5UnDHhImGGus"},{"name":"0yiZLUGHfCfg","upper":12054389308461989689,"ownsPercentage":0.0015649982099467795,"assignedToPhysical":"10HAiobndwqQ"},{"name":"w54br5SJaUNd","upper":12070914180643214570,"ownsPercentage":0.0008958151159464646,"assignedToPhysical":"10HAiobndwqQ"},{"name":"KhlfU1eDGz8J","upper":12112182337465455065,"ownsPercentage":0.002237151264057282,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"w0MlBKADJImt","upper":12241122797128861048,"ownsPercentage":0.006989876324417216,"assignedToPhysical":"5UnDHhImGGus"},{"name":"TR2Tz7fHYqme","upper":12287552841407224373,"ownsPercentage":0.0025169777437599843,"assignedToPhysical":"10HAiobndwqQ"},{"name":"jcaPi3kHStaK","upper":12488886731702542352,"ownsPercentage":0.010914332062657099,"assignedToPhysical":"10HAiobndwqQ"},{"name":"L215BrEJZez6","upper":12528467364563959494,"ownsPercentage":0.0021456704068349808,"assignedToPhysical":"10HAiobndwqQ"},{"name":"SF2wURCU9LAf","upper":12560487892291319550,"ownsPercentage":0.0017358362863068052,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"VhJWJyF49q7N","upper":12605819633176064544,"ownsPercentage":0.002457438597489524,"assignedToPhysical":"5UnDHhImGGus"},{"name":"8irbpjVkihDh","upper":12607449777738158082,"ownsPercentage":0.00008837031378436226,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"vxlhQDXcCDoc","upper":12623878231076967528,"ownsPercentage":0.000890588240025697,"assignedToPhysical":"5UnDHhImGGus"},{"name":"tf20ZJKNTYuN","upper":12632610065963450389,"ownsPercentage":0.0004733537176854718,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"18xE7smej6jn","upper":12818842642211020687,"ownsPercentage":0.010095688187759404,"assignedToPhysical":"10HAiobndwqQ"},{"name":"Xaqm44zLtfaI","upper":12839153738477675750,"ownsPercentage":0.001101066734893482,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"AVApjgCPtUh6","upper":12939045257719913228,"ownsPercentage":0.005415130108765572,"assignedToPhysical":"5UnDHhImGGus"},{"name":"kC8TDVP2qpDU","upper":12941012840812649932,"ownsPercentage":0.00010666289318454411,"assignedToPhysical":"5UnDHhImGGus"},{"name":"lUb5dLRn4Q7i","upper":12975402347204344357,"ownsPercentage":0.0018642588770289618,"assignedToPhysical":"5UnDHhImGGus"},{"name":"VQjkAf5kPk7D","upper":13015759331929071073,"ownsPercentage":0.0021877565256756513,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"6DXk6xSSav8y","upper":13034693447105073367,"ownsPercentage":0.0010264204403956224,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"vCn9VOgxbsvC","upper":13049209090978829828,"ownsPercentage":0.0007868946311476329,"assignedToPhysical":"10HAiobndwqQ"},{"name":"0s0krALwyqS4","upper":13077149257960491748,"ownsPercentage":0.0015146394870562806,"assignedToPhysical":"5UnDHhImGGus"},{"name":"pgxcn0O1NKoW","upper":13176230253235278906,"ownsPercentage":0.0053711915164475115,"assignedToPhysical":"10HAiobndwqQ"},{"name":"gpqm1I3ZXfFw","upper":13232741920389130436,"ownsPercentage":0.0030635036149491775,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"04uFLfQjFQ5M","upper":13283819730558978214,"ownsPercentage":0.0027689336375975575,"assignedToPhysical":"10HAiobndwqQ"},{"name":"OH4vVMfkcJJp","upper":13344918619775115132,"ownsPercentage":0.0033121774212293404,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"xQhsuMpqsB9Y","upper":13434661038335714055,"ownsPercentage":0.004864946258375241,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"cTPggh41RFbY","upper":13540465127958032066,"ownsPercentage":0.00573565119131841,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"ThChEv0oUVF8","upper":13608899141929009629,"ownsPercentage":0.0037098153309618616,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"tcrH84kQAesz","upper":13751466221448629825,"ownsPercentage":0.007728576867004294,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"bFNikojhtkrf","upper":13791329741974145281,"ownsPercentage":0.0021610057778342177,"assignedToPhysical":"5UnDHhImGGus"},{"name":"AJexcI7np4a6","upper":13794193112533578183,"ownsPercentage":0.00015522362905840933,"assignedToPhysical":"5UnDHhImGGus"},{"name":"KoUT5JzcEigR","upper":13800391291566166029,"ownsPercentage":0.00033600395862929224,"assignedToPhysical":"10HAiobndwqQ"},{"name":"c3Fmgjy0kiGM","upper":13807991780268228783,"ownsPercentage":0.0004120233181363985,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"hK5JiEF1Odsf","upper":13822883361746399217,"ownsPercentage":0.0008072742495188642,"assignedToPhysical":"5UnDHhImGGus"},{"name":"bYeSTo07NtUQ","upper":14015079807354872646,"ownsPercentage":0.01041899019363495,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"aRIS8UTR2sMs","upper":14090228842807991479,"ownsPercentage":0.004073837374923082,"assignedToPhysical":"10HAiobndwqQ"},{"name":"rkAx8oOGiGp5","upper":14101487892057637593,"ownsPercentage":0.0006103542828293803,"assignedToPhysical":"5UnDHhImGGus"},{"name":"cOB2G6rQQDMp","upper":14213755249803946573,"ownsPercentage":0.006086025658387776,"assignedToPhysical":"5UnDHhImGGus"},{"name":"ngQi7Ikjaotp","upper":14228350724118087400,"ownsPercentage":0.0007912222479923932,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"sTxMnH5hC27G","upper":14241811106918739693,"ownsPercentage":0.0007296888137476867,"assignedToPhysical":"10HAiobndwqQ"},{"name":"hsdU1XjrtTL0","upper":14243251849196632867,"ownsPercentage":0.00007810279538417468,"assignedToPhysical":"5UnDHhImGGus"},{"name":"xWK39CVQOq7c","upper":14260628115736981572,"ownsPercentage":0.0009419692966366622,"assignedToPhysical":"5UnDHhImGGus"},{"name":"N2liOQZHORRd","upper":14295405224153487552,"ownsPercentage":0.0018852708248969852,"assignedToPhysical":"10HAiobndwqQ"},{"name":"gzKXC2EEuP0y","upper":14373776598404986114,"ownsPercentage":0.004248520711207463,"assignedToPhysical":"5UnDHhImGGus"},{"name":"HnNcLsgB4Qle","upper":14436355817013137488,"ownsPercentage":0.0033924262383701513,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"g7Up5jauHf3w","upper":14449061804126616310,"ownsPercentage":0.0006887929416003281,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"qPo0bL1DanzJ","upper":14466288443411053553,"ownsPercentage":0.0009338579868405498,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"e2bHBdxkqCWf","upper":14522703947576021700,"ownsPercentage":0.0030582906088761743,"assignedToPhysical":"5UnDHhImGGus"},{"name":"SQ4Xh9ZRBYnc","upper":14646673551193670540,"ownsPercentage":0.006720405678221086,"assignedToPhysical":"5UnDHhImGGus"},{"name":"zcHMGe7BCirf","upper":14679858498593792044,"ownsPercentage":0.0017989596032514463,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"wKvtkC7ISgqS","upper":14689830528749515078,"ownsPercentage":0.0005405848379462938,"assignedToPhysical":"10HAiobndwqQ"},{"name":"PeiLrsaz0A1Z","upper":14714795221021913078,"ownsPercentage":0.0013533386798582998,"assignedToPhysical":"5UnDHhImGGus"},{"name":"Dr8dRry1VNRG","upper":14775254950618810772,"ownsPercentage":0.0032775285088421317,"assignedToPhysical":"10HAiobndwqQ"},{"name":"0Ym87WlhMWSH","upper":14833826949240357903,"ownsPercentage":0.0031751944076149684,"assignedToPhysical":"10HAiobndwqQ"},{"name":"pJQ37OFnd0h0","upper":14860665055930654367,"ownsPercentage":0.0014548966789508589,"assignedToPhysical":"10HAiobndwqQ"},{"name":"3xb5qWUIdy3l","upper":14949938709884717460,"ownsPercentage":0.004839534478135717,"assignedToPhysical":"5UnDHhImGGus"},{"name":"P1ib1Jb4kcVl","upper":14974180759437663598,"ownsPercentage":0.0013141641395402726,"assignedToPhysical":"10HAiobndwqQ"},{"name":"6t35oHrP3E89","upper":15157689149190409377,"ownsPercentage":0.009948009741962183,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"IFXmevuG7Zrh","upper":15179330466362727829,"ownsPercentage":0.001173178154683776,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"Ij8HLoZKUIGh","upper":15222784680299792327,"ownsPercentage":0.0023556576577107606,"assignedToPhysical":"10HAiobndwqQ"},{"name":"ZhRSfF5kiST2","upper":15269456053544292690,"ownsPercentage":0.002530060213228457,"assignedToPhysical":"10HAiobndwqQ"},{"name":"AYNqxnC70vWf","upper":15311513209867171019,"ownsPercentage":0.0022799230126913576,"assignedToPhysical":"5UnDHhImGGus"},{"name":"YhUygFdgZypN","upper":15313859598534298546,"ownsPercentage":0.0001271979845197516,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"0suDv1qp3ndv","upper":15378307640662597300,"ownsPercentage":0.003493735364396941,"assignedToPhysical":"5UnDHhImGGus"},{"name":"PPBVWwBeAt88","upper":15385273392099262806,"ownsPercentage":0.00037761414203133826,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"wXr2DKiiO4rQ","upper":15398475090070389854,"ownsPercentage":0.0007156654810396711,"assignedToPhysical":"10HAiobndwqQ"},{"name":"HKmvdMeJ7ZUM","upper":15401238888200171185,"ownsPercentage":0.00014982579683101466,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"48RZ8vWnLnYd","upper":15410079619103447657,"ownsPercentage":0.00047925698258460435,"assignedToPhysical":"10HAiobndwqQ"},{"name":"oeibhOxhslJ4","upper":15414157861976474291,"ownsPercentage":0.0002210819891429501,"assignedToPhysical":"10HAiobndwqQ"},{"name":"Cqju2FPfqDzN","upper":15424911264355003597,"ownsPercentage":0.0005829431110206132,"assignedToPhysical":"10HAiobndwqQ"},{"name":"H6BtD1FRUcEb","upper":15510388510461004250,"ownsPercentage":0.004633730796310202,"assignedToPhysical":"5UnDHhImGGus"},{"name":"94dJ1MbkYawY","upper":15520430127780613983,"ownsPercentage":0.000544357165659447,"assignedToPhysical":"5UnDHhImGGus"},{"name":"KcyNwToNc5uf","upper":15531135481528689161,"ownsPercentage":0.0005803383895444473,"assignedToPhysical":"10HAiobndwqQ"},{"name":"IXPatqSYHjfd","upper":15577751823830575066,"ownsPercentage":0.002527076979851631,"assignedToPhysical":"5UnDHhImGGus"},{"name":"Jrt8yXZwyfFi","upper":15630618935322638641,"ownsPercentage":0.0028659318566364353,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"OyAthRrzjE9C","upper":15636043160678880161,"ownsPercentage":0.00029404784576440076,"assignedToPhysical":"10HAiobndwqQ"},{"name":"LQdu95vAEz5N","upper":15694615554822090470,"ownsPercentage":0.003175215848887293,"assignedToPhysical":"10HAiobndwqQ"},{"name":"2iX5gmw2cSrS","upper":15834882014397728570,"ownsPercentage":0.0076038600099378505,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"wAmFbdhLWtEt","upper":15923976449109514542,"ownsPercentage":0.004829818983544314,"assignedToPhysical":"5UnDHhImGGus"},{"name":"JPenxeL41DVm","upper":16062655725361741270,"ownsPercentage":0.007517818629569082,"assignedToPhysical":"5UnDHhImGGus"},{"name":"wgvpTSabR2Gp","upper":16095714196141917796,"ownsPercentage":0.001792103291945798,"assignedToPhysical":"5UnDHhImGGus"},{"name":"q6uOaqi6vhFp","upper":16096660802391646118,"ownsPercentage":0.00005131562762219013,"assignedToPhysical":"5UnDHhImGGus"},{"name":"QEzoqtGH5L3q","upper":16119600637102882747,"ownsPercentage":0.0012435709315190568,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"ZdNfpA9QwBPW","upper":16184079002913190227,"ownsPercentage":0.003495379214492522,"assignedToPhysical":"5UnDHhImGGus"},{"name":"o0I69xAYCy8l","upper":16202714695155389065,"ownsPercentage":0.0010102429007381622,"assignedToPhysical":"5UnDHhImGGus"},{"name":"NtPJZ0DCROIt","upper":16335259737813065899,"ownsPercentage":0.007185281160081854,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"FVhhe4CrETSN","upper":16363533504010903771,"ownsPercentage":0.001532723936802152,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"Od8MgXRVCkCj","upper":16372836885229567459,"ownsPercentage":0.0005043373064368005,"assignedToPhysical":"5UnDHhImGGus"},{"name":"iWYrenDncRMZ","upper":16399843699729802176,"ownsPercentage":0.001464042347653375,"assignedToPhysical":"10HAiobndwqQ"},{"name":"JcvKnGUxrjbi","upper":16445456059619812276,"ownsPercentage":0.0024726509842469815,"assignedToPhysical":"10HAiobndwqQ"},{"name":"qcgtLYQ2qEet","upper":16479360271815554726,"ownsPercentage":0.0018379510259516748,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"rblINp3n1sZ0","upper":16487866384862819249,"ownsPercentage":0.0004611173122625745,"assignedToPhysical":"10HAiobndwqQ"},{"name":"XZG9YcJJQOQB","upper":16521085219067707559,"ownsPercentage":0.0018007966106187846,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"7dRVmMSfU9TF","upper":16595274498779891691,"ownsPercentage":0.0040218089119542395,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"f7KnlZipyMoU","upper":16684900004595745633,"ownsPercentage":0.0048586084057830525,"assignedToPhysical":"5UnDHhImGGus"},{"name":"KoRSyfj9NHxw","upper":16691189783006387724,"ownsPercentage":0.0003409695708635289,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"NKgE3HgdeGQm","upper":16839333850387189639,"ownsPercentage":0.00803090598475522,"assignedToPhysical":"10HAiobndwqQ"},{"name":"4sh42JIRZ5x4","upper":16853708534942197307,"ownsPercentage":0.000779253211166657,"assignedToPhysical":"5UnDHhImGGus"},{"name":"N5YYIOLBJsT5","upper":16912253308197006536,"ownsPercentage":0.0031737185175267713,"assignedToPhysical":"5UnDHhImGGus"},{"name":"MDVwNcNC9Gra","upper":16913676074504809068,"ownsPercentage":0.00007712831609293425,"assignedToPhysical":"10HAiobndwqQ"},{"name":"KnwHXiKEEMD5","upper":16942230975172303110,"ownsPercentage":0.001547964266940241,"assignedToPhysical":"5UnDHhImGGus"},{"name":"6JQHfQJXq9tC","upper":16984712941502358546,"ownsPercentage":0.002302952009325108,"assignedToPhysical":"10HAiobndwqQ"},{"name":"7oYBFzAQhylH","upper":17037846590624227824,"ownsPercentage":0.0028803808905006594,"assignedToPhysical":"10HAiobndwqQ"},{"name":"6IcKuxi0brIT","upper":17068513180173499251,"ownsPercentage":0.0016624391506020674,"assignedToPhysical":"10HAiobndwqQ"},{"name":"CIohP98pYrEU","upper":17175059447197700381,"ownsPercentage":0.005775884708892977,"assignedToPhysical":"5UnDHhImGGus"},{"name":"X4f5TfQ2Zh3w","upper":17191937029992434765,"ownsPercentage":0.0009149355966177495,"assignedToPhysical":"10HAiobndwqQ"},{"name":"XqtbcXDoer5y","upper":17295841694861764475,"ownsPercentage":0.005632683169135277,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"mpyKh0Uvq9pK","upper":17296806499840116196,"ownsPercentage":0.0000523021826776883,"assignedToPhysical":"5UnDHhImGGus"},{"name":"mbwwycSRznHa","upper":17300383715923936993,"ownsPercentage":0.00019392127247642982,"assignedToPhysical":"10HAiobndwqQ"},{"name":"glARwN18WtJG","upper":17305101748749341512,"ownsPercentage":0.0002557650719580751,"assignedToPhysical":"10HAiobndwqQ"},{"name":"taj02QCfJJ2v","upper":17404497849161967856,"ownsPercentage":0.005388273400197841,"assignedToPhysical":"10HAiobndwqQ"},{"name":"KpWXuBwKB3Np","upper":17414490882508683665,"ownsPercentage":0.0005417234232114685,"assignedToPhysical":"10HAiobndwqQ"},{"name":"s4qjXxgUNJsT","upper":17416265334764561276,"ownsPercentage":0.0000961932495397155,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"8v1uPEAZANss","upper":17467684639236524850,"ownsPercentage":0.002787446080809826,"assignedToPhysical":"10HAiobndwqQ"},{"name":"NPiIKetDYl31","upper":17513502977661685481,"ownsPercentage":0.002483817103011761,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"Pn8nYsbPoQXd","upper":17545583939057489619,"ownsPercentage":0.0017391124020377222,"assignedToPhysical":"5UnDHhImGGus"},{"name":"cCuxaZc2WPTR","upper":17551748894130738275,"ownsPercentage":0.00033420288418458625,"assignedToPhysical":"5UnDHhImGGus"},{"name":"saP105LXdjTN","upper":17666884407730275401,"ownsPercentage":0.006241508698742624,"assignedToPhysical":"5UnDHhImGGus"},{"name":"3DpBKBAyrh49","upper":17697646293978287087,"ownsPercentage":0.0016676051949923117,"assignedToPhysical":"10HAiobndwqQ"},{"name":"MPs72GrXF80t","upper":17857969619417060675,"ownsPercentage":0.008691144887040944,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"VExbdXg0Ek0R","upper":17904914075552982977,"ownsPercentage":0.0025448640664358715,"assignedToPhysical":"10HAiobndwqQ"},{"name":"TEVjUfLfnSxk","upper":17954419263366483034,"ownsPercentage":0.002683681608834984,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"uq0LzAAY5iuw","upper":18090967361863579046,"ownsPercentage":0.007402287251965807,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"S2fMEGZ6FK5b","upper":18099625189226614089,"ownsPercentage":0.000469341761800352,"assignedToPhysical":"5UnDHhImGGus"},{"name":"9SKvNyPOO16k","upper":18126877605085664304,"ownsPercentage":0.001477356423993033,"assignedToPhysical":"5UnDHhImGGus"},{"name":"UUvwR3z3SjqT","upper":18281219283787153712,"ownsPercentage":0.008366879167660726,"assignedToPhysical":"10HAiobndwqQ"},{"name":"MRj5CdiSSHkh","upper":18295367407612657153,"ownsPercentage":0.0007669713294102378,"assignedToPhysical":"5UnDHhImGGus"},{"name":"2PgGKvp3Nl9s","upper":18318403431528121668,"ownsPercentage":0.0012487853587287332,"assignedToPhysical":"10HAiobndwqQ"},{"name":"cghDTFg7bkKV","upper":18341280274520652486,"ownsPercentage":0.001240156143605585,"assignedToPhysical":"5UnDHhImGGus"},{"name":"IiO65JAUXrTl","upper":18364923769868434758,"ownsPercentage":0.0012817164510608228,"assignedToPhysical":"qc4oYcGtIr3y"},{"name":"EgXtER7wworu","upper":18388782832305629848,"ownsPercentage":0.0012934023663937104,"assignedToPhysical":"5UnDHhImGGus"},{"name":"T1LLWfgdGEtq","upper":18430700244523608817,"ownsPercentage":0.002272347469585161,"assignedToPhysical":"5UnDHhImGGus"}]} -`) - - s, err := sharding.StateFromJSON(raw, &fakeNodes{[]string{"node1"}}) - if err != nil { - panic(err) - } - - for name, shard := range s.Physical { - shard.BelongsToNodes = []string{"node1"} - s.Physical[name] = shard - } - return s -} - -type fakeNodes struct { - nodes []string -} - -func (f fakeNodes) Candidates() []string { - return f.nodes -} - -func (f fakeNodes) LocalName() string { - return f.nodes[0] -} - -type fakeRemoteClient struct{} - -func (f *fakeRemoteClient) BatchPutObjects(ctx context.Context, hostName, indexName, shardName string, objs []*storobj.Object, repl *additional.ReplicationProperties) []error { - return nil -} - -func (f *fakeRemoteClient) PutObject(ctx context.Context, hostName, indexName, - shardName string, obj *storobj.Object, -) error { - return nil -} - -func (f *fakeRemoteClient) GetObject(ctx context.Context, hostName, indexName, - shardName string, id strfmt.UUID, props search.SelectProperties, - additional additional.Properties, -) (*storobj.Object, error) { - return nil, nil -} - -func (f *fakeRemoteClient) Exists(ctx context.Context, hostName, indexName, - shardName string, id strfmt.UUID, -) (bool, error) { - return false, nil -} - -func (f *fakeRemoteClient) DeleteObject(ctx context.Context, hostName, indexName, - shardName string, id strfmt.UUID, -) error { - return nil -} - -func (f *fakeRemoteClient) MergeObject(ctx context.Context, hostName, indexName, - shardName string, mergeDoc objects.MergeDocument, -) error { - return nil -} - -func (f *fakeRemoteClient) MultiGetObjects(ctx context.Context, hostName, indexName, - shardName string, ids []strfmt.UUID, -) ([]*storobj.Object, error) { - return nil, nil -} - -func (f *fakeRemoteClient) SearchShard(ctx context.Context, hostName, indexName, - shardName string, vector []float32, limit int, - filters *filters.LocalFilter, _ *searchparams.KeywordRanking, sort []filters.Sort, - cursor *filters.Cursor, groupBy *searchparams.GroupBy, additional additional.Properties, -) ([]*storobj.Object, []float32, error) { - return nil, nil, nil -} - -func (f *fakeRemoteClient) Aggregate(ctx context.Context, hostName, indexName, - shardName string, params aggregation.Params, -) (*aggregation.Result, error) { - return nil, nil -} - -func (f *fakeRemoteClient) BatchAddReferences(ctx context.Context, hostName, - indexName, shardName string, refs objects.BatchReferences, -) []error { - return nil -} - -func (f *fakeRemoteClient) FindUUIDs(ctx context.Context, hostName, indexName, shardName string, - filters *filters.LocalFilter, -) ([]strfmt.UUID, error) { - return nil, nil -} - -func (f *fakeRemoteClient) DeleteObjectBatch(ctx context.Context, hostName, indexName, shardName string, - uuids []strfmt.UUID, dryRun bool, -) objects.BatchSimpleObjects { - return nil -} - -func (f *fakeRemoteClient) GetShardQueueSize(ctx context.Context, - hostName, indexName, shardName string, -) (int64, error) { - return 0, nil -} - -func (f *fakeRemoteClient) GetShardStatus(ctx context.Context, - hostName, indexName, shardName string, -) (string, error) { - return "", nil -} - -func (f *fakeRemoteClient) UpdateShardStatus(ctx context.Context, hostName, indexName, shardName, - targetStatus string, -) error { - return nil -} - -func (f *fakeRemoteClient) PutFile(ctx context.Context, hostName, indexName, shardName, - fileName string, payload io.ReadSeekCloser, -) error { - return nil -} - -type fakeNodeResolver struct{} - -func (f *fakeNodeResolver) NodeHostname(string) (string, bool) { - return "", false -} - -type fakeRemoteNodeClient struct{} - -func (f *fakeRemoteNodeClient) GetNodeStatus(ctx context.Context, hostName, className, output string) (*models.NodeStatus, error) { - return &models.NodeStatus{}, nil -} - -type fakeReplicationClient struct{} - -func (f *fakeReplicationClient) PutObject(ctx context.Context, host, index, shard, requestID string, - obj *storobj.Object, -) (replica.SimpleResponse, error) { - return replica.SimpleResponse{}, nil -} - -func (f *fakeReplicationClient) DeleteObject(ctx context.Context, host, index, shard, requestID string, - id strfmt.UUID, -) (replica.SimpleResponse, error) { - return replica.SimpleResponse{}, nil -} - -func (f *fakeReplicationClient) PutObjects(ctx context.Context, host, index, shard, requestID string, - objs []*storobj.Object, -) (replica.SimpleResponse, error) { - return replica.SimpleResponse{}, nil -} - -func (f *fakeReplicationClient) MergeObject(ctx context.Context, host, index, shard, requestID string, - mergeDoc *objects.MergeDocument, -) (replica.SimpleResponse, error) { - return replica.SimpleResponse{}, nil -} - -func (f *fakeReplicationClient) DeleteObjects(ctx context.Context, host, index, shard, requestID string, - uuids []strfmt.UUID, dryRun bool, -) (replica.SimpleResponse, error) { - return replica.SimpleResponse{}, nil -} - -func (f *fakeReplicationClient) AddReferences(ctx context.Context, host, index, shard, requestID string, - refs []objects.BatchReference, -) (replica.SimpleResponse, error) { - return replica.SimpleResponse{}, nil -} - -func (f *fakeReplicationClient) Commit(ctx context.Context, host, index, shard, requestID string, resp interface{}) error { - return nil -} - -func (f *fakeReplicationClient) Abort(ctx context.Context, host, index, shard, requestID string) (replica.SimpleResponse, error) { - return replica.SimpleResponse{}, nil -} - -func (fakeReplicationClient) Exists(ctx context.Context, hostName, indexName, - shardName string, id strfmt.UUID, -) (bool, error) { - return false, nil -} - -func (*fakeReplicationClient) FetchObject(ctx context.Context, hostName, indexName, - shardName string, id strfmt.UUID, props search.SelectProperties, - additional additional.Properties, -) (objects.Replica, error) { - return objects.Replica{}, nil -} - -func (*fakeReplicationClient) DigestObjects(ctx context.Context, - hostName, indexName, shardName string, ids []strfmt.UUID, -) (result []replica.RepairResponse, err error) { - return nil, nil -} - -func (*fakeReplicationClient) FetchObjects(ctx context.Context, host, - index, shard string, ids []strfmt.UUID, -) ([]objects.Replica, error) { - return nil, nil -} - -func (*fakeReplicationClient) OverwriteObjects(ctx context.Context, - host, index, shard string, objects []*objects.VObject, -) ([]replica.RepairResponse, error) { - return nil, nil -} diff --git a/adapters/repos/db/file_structure_migration.go b/adapters/repos/db/file_structure_migration.go deleted file mode 100644 index 66caafa3198bda026df80f7e0101e247c5156b30..0000000000000000000000000000000000000000 --- a/adapters/repos/db/file_structure_migration.go +++ /dev/null @@ -1,310 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "fmt" - "os" - "path" - "path/filepath" - "strings" - "time" - - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - entschema "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/usecases/schema" -) - -const vectorIndexCommitLog = `hnsw.commitlog.d` - -func (db *DB) migrateFileStructureIfNecessary() error { - fsMigrationPath := path.Join(db.config.RootPath, "migration1.22.fs.hierarchy") - exists, err := fileExists(fsMigrationPath) - if err != nil { - return err - } - if !exists { - if err = db.migrateToHierarchicalFS(); err != nil { - return fmt.Errorf("migrate to hierarchical fs: %w", err) - } - if _, err = os.Create(fsMigrationPath); err != nil { - return fmt.Errorf("create hierarchical fs indicator: %w", err) - } - } - return nil -} - -func (db *DB) migrateToHierarchicalFS() error { - before := time.Now() - - root, err := os.ReadDir(db.config.RootPath) - if err != nil { - return fmt.Errorf("read db root: %w", err) - } - - plan, err := db.assembleFSMigrationPlan(root) - if err != nil { - return err - } - - for newRoot, parts := range plan.partsByShard { - for _, part := range parts { - newPath := path.Join(newRoot, part.newRelPath) - absDir, _ := filepath.Split(newPath) - if err := os.MkdirAll(absDir, os.ModePerm); err != nil { - return fmt.Errorf("mkdir %q: %w", absDir, err) - } - if err = os.Rename(part.oldAbsPath, newPath); err != nil { - return fmt.Errorf("mv %s %s: %w", part.oldAbsPath, newPath, err) - } - } - } - - db.logger.WithField("action", "hierarchical_fs_migration"). - Debugf("fs migration took %s\n", time.Since(before)) - - return nil -} - -type migrationPart struct { - oldAbsPath string - newRelPath string -} - -type shardRoot = string - -type migrationPlan struct { - rootPath string - partsByShard map[shardRoot][]migrationPart -} - -func newMigrationPlan(rootPath string) *migrationPlan { - return &migrationPlan{rootPath: rootPath, partsByShard: make(map[string][]migrationPart)} -} - -func (p *migrationPlan) append(class, shard, oldRootRelPath, newShardRelPath string) { - shardRoot := path.Join(p.rootPath, strings.ToLower(class), shard) - p.partsByShard[shardRoot] = append(p.partsByShard[shardRoot], migrationPart{ - oldAbsPath: path.Join(p.rootPath, oldRootRelPath), - newRelPath: newShardRelPath, - }) -} - -func (p *migrationPlan) prepend(class, shard, oldRootRelPath, newShardRelPath string) { - shardRoot := path.Join(p.rootPath, strings.ToLower(class), shard) - p.partsByShard[shardRoot] = append([]migrationPart{{ - oldAbsPath: path.Join(p.rootPath, oldRootRelPath), - newRelPath: newShardRelPath, - }}, p.partsByShard[shardRoot]...) -} - -func (db *DB) assembleFSMigrationPlan(entries []os.DirEntry) (*migrationPlan, error) { - fm := newFileMatcher(db.schemaGetter, db.config.RootPath) - plan := newMigrationPlan(db.config.RootPath) - - for _, entry := range entries { - if ok, cs := fm.isShardLsmDir(entry); ok { - // make sure lsm dir is moved first, otherwise os.Rename may fail - // if directory already exists (created by other files/dirs moved before) - plan.prepend(cs.class, cs.shard, - entry.Name(), - "lsm") - } else if ok, cs, suffix := fm.isShardFile(entry); ok { - plan.append(cs.class, cs.shard, - entry.Name(), - suffix) - } else if ok, cs := fm.isShardCommitLogDir(entry); ok { - plan.append(cs.class, cs.shard, - entry.Name(), - fmt.Sprintf("main.%s", vectorIndexCommitLog)) - } else if ok, csp := fm.isShardGeoCommitLogDir(entry); ok { - plan.append(csp.class, csp.shard, - entry.Name(), - fmt.Sprintf("geo.%s.%s", csp.geoProp, vectorIndexCommitLog)) - } else if ok, css := fm.isPqDir(entry); ok { - for _, cs := range css { - plan.append(cs.class, cs.shard, - path.Join(strings.ToLower(entry.Name()), cs.shard, "compressed_objects"), - path.Join("lsm", helpers.VectorsCompressedBucketLSM)) - } - - // explicitly rename Class directory starting with uppercase to lowercase - // as MkdirAll will not create lowercased dir if uppercased one exists - oldClassRoot := path.Join(db.config.RootPath, entry.Name()) - newClassRoot := path.Join(db.config.RootPath, strings.ToLower(entry.Name())) - if err := os.Rename(oldClassRoot, newClassRoot); err != nil { - return nil, fmt.Errorf( - "rename pq index dir to avoid collision, old: %q, new: %q, err: %w", - oldClassRoot, newClassRoot, err) - } - } - } - return plan, nil -} - -type classShard struct { - class string - shard string -} - -type classShardGeoProp struct { - class string - shard string - geoProp string -} - -type fileMatcher struct { - rootPath string - shardLsmDirs map[string]*classShard - shardFilePrefixes map[string]*classShard - shardGeoDirPrefixes map[string]*classShardGeoProp - classes map[string][]*classShard -} - -func newFileMatcher(schemaGetter schema.SchemaGetter, rootPath string) *fileMatcher { - shardLsmDirs := make(map[string]*classShard) - shardFilePrefixes := make(map[string]*classShard) - shardGeoDirPrefixes := make(map[string]*classShardGeoProp) - classes := make(map[string][]*classShard) - - sch := schemaGetter.GetSchemaSkipAuth() - for _, class := range sch.Objects.Classes { - shards := schemaGetter.CopyShardingState(class.Class).AllLocalPhysicalShards() - lowercasedClass := strings.ToLower(class.Class) - - var geoProps []string - for _, prop := range class.Properties { - if dt, ok := entschema.AsPrimitive(prop.DataType); ok && dt == entschema.DataTypeGeoCoordinates { - geoProps = append(geoProps, prop.Name) - } - } - - classes[class.Class] = make([]*classShard, 0, len(shards)) - for _, shard := range shards { - cs := &classShard{class: class.Class, shard: shard} - shardLsmDirs[fmt.Sprintf("%s_%s_lsm", lowercasedClass, shard)] = cs - shardFilePrefixes[fmt.Sprintf("%s_%s", lowercasedClass, shard)] = cs - classes[class.Class] = append(classes[class.Class], cs) - - for _, geoProp := range geoProps { - csp := &classShardGeoProp{class: class.Class, shard: shard, geoProp: geoProp} - shardGeoDirPrefixes[fmt.Sprintf("%s_%s_%s", lowercasedClass, shard, geoProp)] = csp - } - } - } - - return &fileMatcher{ - rootPath: rootPath, - shardLsmDirs: shardLsmDirs, - shardFilePrefixes: shardFilePrefixes, - shardGeoDirPrefixes: shardGeoDirPrefixes, - classes: classes, - } -} - -// Checks if entry is directory with name (class is lowercased): -// class_shard_lsm -func (fm *fileMatcher) isShardLsmDir(entry os.DirEntry) (bool, *classShard) { - if !entry.IsDir() { - return false, nil - } - if cs, ok := fm.shardLsmDirs[entry.Name()]; ok { - return true, cs - } - return false, nil -} - -// Checks if entry is file with name (class is lowercased): -// class_shard.* -// (e.g. class_shard.version, class_shard.indexcount) -func (fm *fileMatcher) isShardFile(entry os.DirEntry) (bool, *classShard, string) { - if !entry.Type().IsRegular() { - return false, nil, "" - } - parts := strings.SplitN(entry.Name(), ".", 2) - if len(parts) != 2 { - return false, nil, "" - } - if cs, ok := fm.shardFilePrefixes[parts[0]]; ok { - return true, cs, parts[1] - } - return false, nil, "" -} - -// Checks if entry is directory with name (class is lowercased): -// class_shard.hnsw.commitlog.d -func (fm *fileMatcher) isShardCommitLogDir(entry os.DirEntry) (bool, *classShard) { - if !entry.IsDir() { - return false, nil - } - parts := strings.SplitN(entry.Name(), ".", 2) - if len(parts) != 2 { - return false, nil - } - if parts[1] != vectorIndexCommitLog { - return false, nil - } - if cs, ok := fm.shardFilePrefixes[parts[0]]; ok { - return true, cs - } - return false, nil -} - -// Checks if entry is directory with name (class is lowercased): -// class_shard_prop.hnsw.commitlog.d -func (fm *fileMatcher) isShardGeoCommitLogDir(entry os.DirEntry) (bool, *classShardGeoProp) { - if !entry.IsDir() { - return false, nil - } - parts := strings.SplitN(entry.Name(), ".", 2) - if len(parts) != 2 { - return false, nil - } - if parts[1] != vectorIndexCommitLog { - return false, nil - } - if csp, ok := fm.shardGeoDirPrefixes[parts[0]]; ok { - return true, csp - } - return false, nil -} - -// Checks if entry is directory containing PQ index: -// Class/shard/compressed_object -func (fm *fileMatcher) isPqDir(entry os.DirEntry) (bool, []*classShard) { - if !entry.IsDir() { - return false, nil - } - - resultcss := []*classShard{} - if css, ok := fm.classes[entry.Name()]; ok { - for _, cs := range css { - pqDir := path.Join(fm.rootPath, cs.class, cs.shard, "compressed_objects") - if info, err := os.Stat(pqDir); err == nil && info.IsDir() { - resultcss = append(resultcss, cs) - } - } - return true, resultcss - } - return false, nil -} - -func fileExists(file string) (bool, error) { - _, err := os.Stat(file) - if os.IsNotExist(err) { - return false, nil - } - if err != nil { - return false, err - } - return true, nil -} diff --git a/adapters/repos/db/file_structure_migration_test.go b/adapters/repos/db/file_structure_migration_test.go deleted file mode 100644 index de4d51431023223a8480d84cceb2079f8389425f..0000000000000000000000000000000000000000 --- a/adapters/repos/db/file_structure_migration_test.go +++ /dev/null @@ -1,305 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "fmt" - "math/rand" - "os" - "path" - "strings" - "testing" - - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/usecases/sharding" -) - -const ( - numClasses = 100 - numShards = 10 - uppercase = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - lowercase = "abcdefghijklmnopqrstuvwxyz" - digits = "0123456789" - chars = uppercase + lowercase + digits - localNode = "node1" -) - -var ( - rootFiles = []string{ - "classifications.db", - "modules.db", - "schema.db", - } - indexDirExts = []string{ - ".hnsw.commitlog.d", - "_someGeoProp.hnsw.commitlog.d", - "_lsm", - } - indexFileExts = []string{ - ".indexcount", - ".proplengths", - ".version", - } - migratedRootFiles = append(rootFiles, - "migration1.22.fs.hierarchy") -) - -func TestFileStructureMigration(t *testing.T) { - shardsByClass := make(map[string][]string, numClasses) - - t.Run("generate index and shard names", func(t *testing.T) { - for i := 0; i < numClasses; i++ { - c := randClassName() - shardsByClass[c] = make([]string, numShards) - for j := 0; j < numShards; j++ { - s := randShardName() - shardsByClass[c][j] = s - } - } - }) - - root := t.TempDir() - - t.Run("write test db files", func(t *testing.T) { - for _, f := range rootFiles { - require.Nil(t, os.WriteFile(path.Join(root, f), nil, os.ModePerm)) - } - - for class, shards := range shardsByClass { - for _, shard := range shards { - idx := path.Join(root, fmt.Sprintf("%s_%s", strings.ToLower(class), shard)) - for _, ext := range indexDirExts { - require.Nil(t, os.MkdirAll(idx+ext, os.ModePerm)) - } - for _, ext := range indexFileExts { - require.Nil(t, os.WriteFile(idx+ext, nil, os.ModePerm)) - } - - pqDir := path.Join(root, class, shard, "compressed_objects") - require.Nil(t, os.MkdirAll(pqDir, os.ModePerm)) - } - } - }) - - files, err := os.ReadDir(root) - require.Nil(t, err) - - t.Run("assert expected flat contents length", func(t *testing.T) { - // Flat structure root contains: - // - (3 dirs + 3 files) per shard per index - // - dirs: main commilog, geo prop commitlog, lsm store - // - files: indexcount, proplengths, version - // - 1 dir per index; shards dirs are nested - // - pq store - // - 3 root db files - expectedLen := numClasses*(numShards*(len(indexDirExts)+len(indexFileExts))+1) + len(rootFiles) - require.Len(t, files, expectedLen) - }) - - t.Run("migrate the db", func(t *testing.T) { - classes := make([]*models.Class, numClasses) - states := make(map[string]*sharding.State, numClasses) - - i := 0 - for class, shards := range shardsByClass { - classes[i] = &models.Class{ - Class: class, - Properties: []*models.Property{{ - Name: "someGeoProp", - DataType: schema.DataTypeGeoCoordinates.PropString(), - }}, - } - states[class] = &sharding.State{ - Physical: make(map[string]sharding.Physical), - } - states[class].SetLocalName(localNode) - - for _, shard := range shards { - states[class].Physical[shard] = sharding.Physical{ - Name: shard, - BelongsToNodes: []string{localNode}, - } - } - - i++ - } - - db := testDB(root, classes, states) - require.Nil(t, db.migrateFileStructureIfNecessary()) - }) - - files, err = os.ReadDir(root) - require.Nil(t, err) - - t.Run("assert expected hierarchical contents length", func(t *testing.T) { - // After migration, the hierarchical structure root contains: - // - one dir per index - // - 3 original root db files, and one additional which is the FS migration indicator - expectedLen := numClasses + len(migratedRootFiles) - require.Len(t, files, expectedLen) - }) - - t.Run("assert all db files were migrated", func(t *testing.T) { - var foundRootFiles []string - for _, f := range files { - if f.IsDir() { - idx := f - shardsRoot, err := os.ReadDir(path.Join(root, idx.Name())) - require.Nil(t, err) - for _, shard := range shardsRoot { - assertShardRootContents(t, shardsByClass, root, idx, shard) - } - } else { - foundRootFiles = append(foundRootFiles, f.Name()) - } - } - - assert.ElementsMatch(t, migratedRootFiles, foundRootFiles) - }) -} - -func assertShardRootContents(t *testing.T, shardsByClass map[string][]string, root string, idx, shard os.DirEntry) { - assert.True(t, shard.IsDir()) - - // Whatever we find in this shard directory, it should be able to - // be mapped back to the original flat structure root contents - lowercasedClasses := make(map[string]string, len(shardsByClass)) - for class := range shardsByClass { - lowercasedClasses[strings.ToLower(class)] = class - } - require.Contains(t, lowercasedClasses, idx.Name()) - assert.Contains(t, shardsByClass[lowercasedClasses[idx.Name()]], shard.Name()) - - // Now we will get a set of all expected files within the shard dir. - // Check to see if all of these files are found. - expected := expectedShardContents() - shardFiles, err := os.ReadDir(path.Join(root, idx.Name(), shard.Name())) - require.Nil(t, err) - for _, sf := range shardFiles { - expected[sf.Name()] = true - } - expected.assert(t) - - // Check if pq store was migrated to main store as "vectors_compressed" subdir - pqDir := path.Join(root, idx.Name(), shard.Name(), "lsm", helpers.VectorsCompressedBucketLSM) - info, err := os.Stat(pqDir) - require.NoError(t, err) - assert.True(t, info.IsDir()) -} - -func testDB(root string, classes []*models.Class, states map[string]*sharding.State) *DB { - logger, _ := test.NewNullLogger() - return &DB{ - config: Config{RootPath: root}, - logger: logger, - schemaGetter: &fakeMigrationSchemaGetter{ - sch: schema.Schema{Objects: &models.Schema{Classes: classes}}, - states: states, - }, - } -} - -func randClassName() string { - return randStringBytes(16) -} - -func randShardName() string { - return randStringBytes(8) -} - -func randStringBytes(n int) string { - b := make([]byte, n) - for i := range b { - switch { - case i == 0: - b[i] = randChar(uppercase) - case i == n/2: - b[i] = []byte("_")[0] - default: - b[i] = randChar(chars) - } - } - return string(b) -} - -func randChar(str string) byte { - return str[rand.Intn(len(str))] -} - -type shardContents map[string]bool - -func expectedShardContents() shardContents { - return shardContents{ - "main.hnsw.commitlog.d": false, - "geo.someGeoProp.hnsw.commitlog.d": false, - "lsm": false, - "indexcount": false, - "proplengths": false, - "version": false, - } -} - -func (c shardContents) assert(t *testing.T) { - for name, found := range c { - assert.True(t, found, "didn't find %q in shard contents", name) - } -} - -type fakeMigrationSchemaGetter struct { - sch schema.Schema - states map[string]*sharding.State -} - -func (sg *fakeMigrationSchemaGetter) GetSchemaSkipAuth() schema.Schema { - return sg.sch -} - -func (sg *fakeMigrationSchemaGetter) Nodes() []string { - return nil -} - -func (sg *fakeMigrationSchemaGetter) NodeName() string { - return "" -} - -func (sg *fakeMigrationSchemaGetter) ClusterHealthScore() int { - return 0 -} - -func (sg *fakeMigrationSchemaGetter) ResolveParentNodes(string, string) (map[string]string, error) { - return nil, nil -} - -func (sg *fakeMigrationSchemaGetter) CopyShardingState(class string) *sharding.State { - return sg.states[class] -} - -func (sg *fakeMigrationSchemaGetter) ShardOwner(class, shard string) (string, error) { - return "", nil -} - -func (sg *fakeMigrationSchemaGetter) TenantShard(class, tenant string) (string, string) { - return "", "" -} - -func (sg *fakeMigrationSchemaGetter) ShardFromUUID(class string, uuid []byte) string { - return "" -} - -func (sg *fakeMigrationSchemaGetter) ShardReplicas(class, shard string) ([]string, error) { - return nil, nil -} diff --git a/adapters/repos/db/filters_integration_test.go b/adapters/repos/db/filters_integration_test.go deleted file mode 100644 index 248687741e0cf3f01577561973a9895452e4249a..0000000000000000000000000000000000000000 --- a/adapters/repos/db/filters_integration_test.go +++ /dev/null @@ -1,1657 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest - -package db - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/go-openapi/strfmt" - "github.com/google/uuid" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/dto" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - enthnsw "github.com/weaviate/weaviate/entities/vectorindex/hnsw" -) - -func TestFilters(t *testing.T) { - dirName := t.TempDir() - - logger, _ := test.NewNullLogger() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - defer repo.Shutdown(testCtx()) - - migrator := NewMigrator(repo, logger) - t.Run("prepare test schema and data ", prepareCarTestSchemaAndData(repo, migrator, schemaGetter)) - - t.Run("primitive props without nesting", testPrimitiveProps(repo)) - - t.Run("primitive props with limit", testPrimitivePropsWithLimit(repo)) - - t.Run("chained primitive props", testChainedPrimitiveProps(repo, migrator)) - - t.Run("sort props", testSortProperties(repo)) -} - -func TestFiltersNoLengthIndex(t *testing.T) { - dirName := t.TempDir() - - logger, _ := test.NewNullLogger() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - defer repo.Shutdown(testCtx()) - migrator := NewMigrator(repo, logger) - t.Run("prepare test schema and data ", prepareCarTestSchemaAndDataNoLength(repo, migrator, schemaGetter)) - t.Run("primitive props without nesting", testPrimitivePropsWithNoLengthIndex(repo)) -} - -var ( - // operators - eq = filters.OperatorEqual - neq = filters.OperatorNotEqual - lt = filters.OperatorLessThan - lte = filters.OperatorLessThanEqual - like = filters.OperatorLike - gt = filters.OperatorGreaterThan - gte = filters.OperatorGreaterThanEqual - wgr = filters.OperatorWithinGeoRange - and = filters.OperatorAnd - null = filters.OperatorIsNull - - // datatypes - dtInt = schema.DataTypeInt - dtBool = schema.DataTypeBoolean - dtNumber = schema.DataTypeNumber - dtText = schema.DataTypeText - dtDate = schema.DataTypeDate - dtGeoCoordinates = schema.DataTypeGeoCoordinates -) - -func prepareCarTestSchemaAndData(repo *DB, - migrator *Migrator, schemaGetter *fakeSchemaGetter, -) func(t *testing.T) { - return func(t *testing.T) { - t.Run("creating the class", func(t *testing.T) { - require.Nil(t, - migrator.AddClass(context.Background(), carClass, schemaGetter.shardState)) - schemaGetter.schema.Objects = &models.Schema{ - Classes: []*models.Class{ - carClass, - }, - } - }) - - for i, fixture := range cars { - t.Run(fmt.Sprintf("importing car %d", i), func(t *testing.T) { - require.Nil(t, - repo.PutObject(context.Background(), &fixture, carVectors[i], nil)) - }) - } - } -} - -func prepareCarTestSchemaAndDataNoLength(repo *DB, - migrator *Migrator, schemaGetter *fakeSchemaGetter, -) func(t *testing.T) { - return func(t *testing.T) { - t.Run("creating the class", func(t *testing.T) { - require.Nil(t, - migrator.AddClass(context.Background(), carClassNoLengthIndex, schemaGetter.shardState)) - schemaGetter.schema.Objects = &models.Schema{ - Classes: []*models.Class{ - carClassNoLengthIndex, - }, - } - }) - - for i, fixture := range cars { - t.Run(fmt.Sprintf("importing car %d", i), func(t *testing.T) { - require.Nil(t, - repo.PutObject(context.Background(), &fixture, carVectors[i], nil)) - }) - } - } -} - -func testPrimitivePropsWithNoLengthIndex(repo *DB) func(t *testing.T) { - return func(t *testing.T) { - type test struct { - name string - filter *filters.LocalFilter - expectedIDs []strfmt.UUID - limit int - ErrMsg string - } - - tests := []test{ - { - name: "Filter by unsupported geo-coordinates", - filter: buildFilter("len(parkedAt)", 0, eq, dtInt), - expectedIDs: []strfmt.UUID{}, - ErrMsg: "Property length must be indexed to be filterable! add `IndexPropertyLength: true` to the invertedIndexConfig in", - }, - { - name: "Filter by unsupported number", - filter: buildFilter("len(horsepower)", 1, eq, dtInt), - expectedIDs: []strfmt.UUID{}, - ErrMsg: "Property length must be indexed to be filterable", - }, - { - name: "Filter by unsupported date", - filter: buildFilter("len(released)", 1, eq, dtInt), - expectedIDs: []strfmt.UUID{}, - ErrMsg: "Property length must be indexed to be filterable! add `IndexPropertyLength: true` to the invertedIndexConfig in", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - if test.limit == 0 { - test.limit = 100 - } - params := dto.GetParams{ - SearchVector: []float32{0.1, 0.1, 0.1, 1.1, 0.1}, - ClassName: carClass.Class, - Pagination: &filters.Pagination{Limit: test.limit}, - Filters: test.filter, - } - res, err := repo.Search(context.Background(), params) - if len(test.ErrMsg) > 0 { - require.Contains(t, err.Error(), test.ErrMsg) - } else { - require.Nil(t, err) - require.Len(t, res, len(test.expectedIDs)) - - ids := make([]strfmt.UUID, len(test.expectedIDs)) - for pos, concept := range res { - ids[pos] = concept.ID - } - assert.ElementsMatch(t, ids, test.expectedIDs, "ids don't match") - - } - }) - } - } -} - -func testPrimitiveProps(repo *DB) func(t *testing.T) { - return func(t *testing.T) { - type test struct { - name string - filter *filters.LocalFilter - expectedIDs []strfmt.UUID - limit int - ErrMsg string - } - - tests := []test{ - { - name: "horsepower == 130", - filter: buildFilter("horsepower", 130, eq, dtInt), - expectedIDs: []strfmt.UUID{carSprinterID}, - }, - { - name: "horsepower < 200", - filter: buildFilter("horsepower", 200, lt, dtInt), - expectedIDs: []strfmt.UUID{carSprinterID, carPoloID}, - }, - { - name: "horsepower <= 130", - filter: buildFilter("horsepower", 130, lte, dtInt), - expectedIDs: []strfmt.UUID{carSprinterID, carPoloID}, - }, - { - name: "horsepower > 200", - filter: buildFilter("horsepower", 200, gt, dtInt), - expectedIDs: []strfmt.UUID{carE63sID}, - }, - { - name: "horsepower >= 612", - filter: buildFilter("horsepower", 612, gte, dtInt), - expectedIDs: []strfmt.UUID{carE63sID}, - }, - { - name: "modelName != sprinter", - filter: buildFilter("modelName", "sprinter", neq, dtText), - expectedIDs: []strfmt.UUID{carE63sID, carPoloID, carNilID}, - }, - { - name: "modelName = spr*er (optimizable) dtText", - filter: buildFilter("modelName", "spr*er", like, dtText), - expectedIDs: []strfmt.UUID{carSprinterID}, - }, - { - name: "modelName = *rinte? (non-optimizable) dtText", - filter: buildFilter("modelName", "*rinte?", like, dtText), - expectedIDs: []strfmt.UUID{carSprinterID}, - }, - { - name: "modelName = spr*er (optimizable) dtText", - filter: buildFilter("modelName", "spr*er", like, dtText), - expectedIDs: []strfmt.UUID{carSprinterID}, - }, - { - name: "modelName = *rinte? (non-optimizable) dtText", - filter: buildFilter("modelName", "*rinte?", like, dtText), - expectedIDs: []strfmt.UUID{carSprinterID}, - }, - { - name: "weight == 3499.90", - filter: buildFilter("weight", 3499.90, eq, dtNumber), - expectedIDs: []strfmt.UUID{carSprinterID}, - }, - { - name: "weight <= 3499.90", - filter: buildFilter("weight", 3499.90, lte, dtNumber), - expectedIDs: []strfmt.UUID{carSprinterID, carE63sID, carPoloID}, - }, - { - name: "weight < 3499.90", - filter: buildFilter("weight", 3499.90, lt, dtNumber), - expectedIDs: []strfmt.UUID{carE63sID, carPoloID}, - }, - { - name: "weight > 3000", - filter: buildFilter("weight", 3000.0, gt, dtNumber), - expectedIDs: []strfmt.UUID{carSprinterID}, - }, - { - name: "weight == 2069.4", - filter: buildFilter("weight", 2069.4, eq, dtNumber), - expectedIDs: []strfmt.UUID{}, - }, - { - name: "weight == 2069.5", - filter: buildFilter("weight", 2069.5, eq, dtNumber), - expectedIDs: []strfmt.UUID{carE63sID}, - }, - { - name: "weight >= 2069.5", - filter: buildFilter("weight", 2069.5, gte, dtNumber), - expectedIDs: []strfmt.UUID{carSprinterID, carE63sID}, - }, - { - name: "before or equal 2017", - filter: buildFilter("released", mustParseTime("2017-02-17T09:47:00+02:00"), lte, dtDate), - expectedIDs: []strfmt.UUID{carPoloID, carE63sID, carSprinterID}, - }, - { - name: "before 1980", - filter: buildFilter("released", mustParseTime("1980-01-01T00:00:00+02:00"), lt, dtDate), - expectedIDs: []strfmt.UUID{carPoloID}, - }, - { - name: "from or equal 1995 on", - filter: buildFilter("released", mustParseTime("1995-08-17T12:47:00+02:00"), gte, dtDate), - expectedIDs: []strfmt.UUID{carSprinterID, carE63sID}, - }, - { - name: "from 1995 on", - filter: buildFilter("released", mustParseTime("1995-08-17T12:47:00+02:00"), gt, dtDate), - expectedIDs: []strfmt.UUID{carE63sID}, - }, - { - name: "equal to 1995-08-17T12:47:00+02:00", - filter: buildFilter("released", mustParseTime("1995-08-17T12:47:00+02:00"), eq, dtDate), - expectedIDs: []strfmt.UUID{carSprinterID}, - }, - { - name: "not equal to 1995-08-17T12:47:00+02:00", - filter: buildFilter("released", mustParseTime("1995-08-17T12:47:00+02:00"), neq, dtDate), - expectedIDs: []strfmt.UUID{carPoloID, carE63sID}, - }, - { - name: "exactly matching a specific contact email", - filter: buildFilter("contact", "john@heavycars.example.com", eq, dtText), - expectedIDs: []strfmt.UUID{carSprinterID}, - }, - { - name: "matching an email from within a text (not string) field", - filter: buildFilter("description", "john@heavycars.example.com", eq, dtText), - expectedIDs: []strfmt.UUID{carSprinterID}, - }, - { - name: "full-text matching the word engine", - filter: buildFilter("description", "engine", eq, dtText), - expectedIDs: []strfmt.UUID{carPoloID}, - }, - { - name: "matching two words", - filter: buildFilter("description", "this car", eq, dtText), - expectedIDs: []strfmt.UUID{carSprinterID, carPoloID, carE63sID}, - }, - { - name: "matching three words", - filter: buildFilter("description", "but car has", eq, dtText), - expectedIDs: []strfmt.UUID{carPoloID, carE63sID}, - }, - { - name: "matching words with special characters", - filter: buildFilter("description", "it's also not exactly lightweight.", eq, dtText), - expectedIDs: []strfmt.UUID{carE63sID}, - }, - { - name: "matching words without special characters", - filter: buildFilter("description", "also not exactly lightweight", eq, dtText), - expectedIDs: []strfmt.UUID{carE63sID}, - }, - { - name: "by id", - filter: buildFilter("id", carPoloID.String(), eq, dtText), - expectedIDs: []strfmt.UUID{carPoloID}, - }, - { - name: "by id not equal", - filter: buildFilter("id", carE63sID.String(), neq, dtText), - expectedIDs: []strfmt.UUID{carPoloID, carSprinterID, carNilID, carEmpty}, - }, - { - name: "by id less then equal", - filter: buildFilter("id", carPoloID.String(), lte, dtText), - expectedIDs: []strfmt.UUID{carPoloID, carE63sID}, - }, - { - name: "by id less then", - filter: buildFilter("id", carPoloID.String(), lt, dtText), - expectedIDs: []strfmt.UUID{carE63sID}, - }, - { - name: "by id greater then equal", - filter: buildFilter("id", carPoloID.String(), gte, dtText), - expectedIDs: []strfmt.UUID{carPoloID, carSprinterID, carNilID, carEmpty}, - }, - { - name: "by id greater then", - filter: buildFilter("id", carPoloID.String(), gt, dtText), - expectedIDs: []strfmt.UUID{carSprinterID, carNilID, carEmpty}, - }, - { - name: "within 600km of San Francisco", - filter: buildFilter("parkedAt", filters.GeoRange{ - GeoCoordinates: &models.GeoCoordinates{ - Latitude: ptFloat32(37.733795), - Longitude: ptFloat32(-122.446747), - }, - Distance: 600000, - }, wgr, dtGeoCoordinates), - expectedIDs: []strfmt.UUID{carSprinterID}, - }, - // { - // name: "by id like", - // filter: buildFilter("id", carPoloID.String(), like, dtText), - // expectedIDs: []strfmt.UUID{carPoloID}, - // }, - { - name: "by color with word tokenization", - filter: buildFilter("colorWhitespace", "grey", eq, dtText), - expectedIDs: []strfmt.UUID{carE63sID, carSprinterID, carPoloID}, - }, - { - name: "by color with word tokenization multiword (1)", - filter: buildFilter("colorWhitespace", "light grey", eq, dtText), - expectedIDs: []strfmt.UUID{carE63sID, carSprinterID}, - }, - { - name: "by color with word tokenization multiword (2)", - filter: buildFilter("colorWhitespace", "dark grey", eq, dtText), - expectedIDs: []strfmt.UUID{carPoloID}, - }, - { - name: "by color with field tokenization", - filter: buildFilter("colorField", "grey", eq, dtText), - expectedIDs: []strfmt.UUID{}, - }, - { - name: "by color with field tokenization multiword (1)", - filter: buildFilter("colorField", "light grey", eq, dtText), - expectedIDs: []strfmt.UUID{carSprinterID}, - }, - { - name: "by color with field tokenization multiword (2)", - filter: buildFilter("colorField", "dark grey", eq, dtText), - expectedIDs: []strfmt.UUID{carPoloID}, - }, - { - name: "by color array with word tokenization", - filter: buildFilter("colorArrayWhitespace", "grey", eq, dtText), - expectedIDs: []strfmt.UUID{carE63sID, carSprinterID, carPoloID}, - }, - { - name: "by color array with word tokenization multiword (1)", - filter: buildFilter("colorArrayWhitespace", "light grey", eq, dtText), - expectedIDs: []strfmt.UUID{carE63sID, carSprinterID}, - }, - { - name: "by color array with word tokenization multiword (2)", - filter: buildFilter("colorArrayWhitespace", "dark grey", eq, dtText), - expectedIDs: []strfmt.UUID{carPoloID}, - }, - { - name: "by color array with field tokenization", - filter: buildFilter("colorArrayField", "grey", eq, dtText), - expectedIDs: []strfmt.UUID{carE63sID, carPoloID}, - }, - { - name: "by color with array field tokenization multiword (1)", - filter: buildFilter("colorArrayField", "light grey", eq, dtText), - expectedIDs: []strfmt.UUID{carSprinterID}, - }, - { - name: "by color with array field tokenization multiword (2)", - filter: buildFilter("colorArrayField", "dark grey", eq, dtText), - expectedIDs: []strfmt.UUID{}, - }, - { - name: "by null value", - filter: buildFilter("colorArrayField", true, null, dtBool), - expectedIDs: []strfmt.UUID{carNilID, carEmpty}, - }, - { - name: "by value not null", - filter: buildFilter("colorArrayField", false, null, dtBool), - expectedIDs: []strfmt.UUID{carE63sID, carPoloID, carSprinterID}, - }, - { - name: "by string length", - filter: buildFilter("len(colorField)", 10, eq, dtInt), - expectedIDs: []strfmt.UUID{carSprinterID}, - }, - { - name: "by array length", - filter: buildFilter("len(colorArrayField)", 2, eq, dtInt), - expectedIDs: []strfmt.UUID{carE63sID, carPoloID}, - }, - { - name: "by text length (equal)", - filter: buildFilter("len(description)", 65, eq, dtInt), - expectedIDs: []strfmt.UUID{carE63sID}, - }, - { - name: "by text length (lte)", - filter: buildFilter("len(description)", 65, lte, dtInt), - expectedIDs: []strfmt.UUID{carE63sID, carNilID, carEmpty}, - }, - { - name: "by text length (gte)", - filter: buildFilter("len(description)", 65, gte, dtInt), - expectedIDs: []strfmt.UUID{carE63sID, carPoloID, carSprinterID}, - }, - { - name: "length 0 (not added and empty)", - filter: buildFilter("len(colorArrayWhitespace)", 0, eq, dtInt), - expectedIDs: []strfmt.UUID{carNilID, carEmpty}, - }, - { - name: "Filter unicode strings", - filter: buildFilter("len(contact)", 30, eq, dtInt), - expectedIDs: []strfmt.UUID{carE63sID}, - }, - { - name: "Filter unicode texts", - filter: buildFilter("len(description)", 110, eq, dtInt), - expectedIDs: []strfmt.UUID{carPoloID}, - }, - { - name: "Empty string properties", - filter: buildFilter("modelName", true, null, dtBool), - expectedIDs: []strfmt.UUID{carEmpty}, - }, - { - name: "Empty string by length", - filter: buildFilter("len(description)", 0, eq, dtInt), - expectedIDs: []strfmt.UUID{carEmpty, carNilID}, - }, - { - name: "Empty array by length", - filter: buildFilter("len(colorArrayWhitespace)", 0, eq, dtInt), - expectedIDs: []strfmt.UUID{carEmpty, carNilID}, - }, - { - name: "made by Mercedes ... I mean manufacturer1", - filter: buildFilter("manufacturerId", manufacturer1.String(), eq, dtText), - expectedIDs: []strfmt.UUID{carE63sID, carSprinterID}, - }, - { - name: "made by manufacturer2", - filter: buildFilter("manufacturerId", manufacturer2.String(), eq, dtText), - expectedIDs: []strfmt.UUID{carPoloID}, - }, - { - name: "available at the north dealership", - filter: buildFilter("availableAtDealerships", dealershipNorth.String(), eq, dtText), - expectedIDs: []strfmt.UUID{carE63sID, carSprinterID}, - }, - { - name: "available at the south dealership", - filter: buildFilter("availableAtDealerships", dealershipSouth.String(), eq, dtText), - expectedIDs: []strfmt.UUID{carPoloID, carSprinterID}, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - if test.limit == 0 { - test.limit = 100 - } - params := dto.GetParams{ - SearchVector: []float32{0.1, 0.1, 0.1, 1.1, 0.1}, - ClassName: carClass.Class, - Pagination: &filters.Pagination{Limit: test.limit}, - Filters: test.filter, - } - res, err := repo.Search(context.Background(), params) - if len(test.ErrMsg) > 0 { - require.Contains(t, err.Error(), test.ErrMsg) - } else { - require.Nil(t, err) - require.Len(t, res, len(test.expectedIDs)) - - ids := make([]strfmt.UUID, len(test.expectedIDs)) - for pos, concept := range res { - ids[pos] = concept.ID - } - assert.ElementsMatch(t, ids, test.expectedIDs, "ids don't match") - - } - }) - } - } -} - -func testPrimitivePropsWithLimit(repo *DB) func(t *testing.T) { - return func(t *testing.T) { - t.Run("greater than", func(t *testing.T) { - limit := 1 - - params := dto.GetParams{ - SearchVector: []float32{0.1, 0.1, 0.1, 1.1, 0.1}, - ClassName: carClass.Class, - Pagination: &filters.Pagination{Limit: limit}, - Filters: buildFilter("horsepower", 2, gt, dtInt), // would otherwise return 3 results - } - res, err := repo.Search(context.Background(), params) - require.Nil(t, err) - assert.Len(t, res, limit) - }) - - t.Run("less than", func(t *testing.T) { - limit := 1 - - params := dto.GetParams{ - SearchVector: []float32{0.1, 0.1, 0.1, 1.1, 0.1}, - ClassName: carClass.Class, - Pagination: &filters.Pagination{Limit: limit}, - Filters: buildFilter("horsepower", 20000, lt, dtInt), // would otherwise return 3 results - } - res, err := repo.Search(context.Background(), params) - require.Nil(t, err) - assert.Len(t, res, limit) - }) - } -} - -func testChainedPrimitiveProps(repo *DB, - migrator *Migrator, -) func(t *testing.T) { - return func(t *testing.T) { - type test struct { - name string - filter *filters.LocalFilter - expectedIDs []strfmt.UUID - } - - tests := []test{ - { - name: "modelName == sprinter AND weight > 3000", - filter: filterAnd( - buildFilter("modelName", "sprinter", eq, dtText), - buildFilter("weight", float64(3000), gt, dtNumber), - ), - expectedIDs: []strfmt.UUID{carSprinterID}, - }, - { - name: "modelName == sprinter OR modelName == e63s", - filter: filterOr( - buildFilter("modelName", "sprinter", eq, dtText), - buildFilter("modelName", "e63s", eq, dtText), - ), - expectedIDs: []strfmt.UUID{carSprinterID, carE63sID}, - }, - // test{ - // name: "NOT modelName == sprinter, modelName == e63s", - // filter: filterNot( - // buildFilter("modelName", "sprinter", eq, dtText), - // buildFilter("modelName", "e63s", eq, dtText), - // ), - // expectedIDs: []strfmt.UUID{carPoloID}, - // }, - // test{ - // name: "NOT horsepower < 200 , weight > 3000", - // filter: filterNot( - // buildFilter("horsepower", 200, lt, dtNumber), - // buildFilter("weight", 3000, gt, dtNumber), - // ), - // expectedIDs: []strfmt.UUID{carE63sID}, - // }, - { - name: "(heavy AND powerful) OR light", - filter: filterOr( - filterAnd( - buildFilter("horsepower", 200, gt, dtInt), - buildFilter("weight", float64(1500), gt, dtNumber), - ), - buildFilter("weight", float64(1500), lt, dtNumber), - ), - expectedIDs: []strfmt.UUID{carE63sID, carPoloID}, - }, - - // this test prevents a regression on - // https://github.com/weaviate/weaviate/issues/1638 - { - name: "Like ca* AND Like eng*", - filter: filterAnd( - buildFilter("description", "ca*", like, dtText), - buildFilter("description", "eng*", like, dtText), - ), - expectedIDs: []strfmt.UUID{carPoloID}, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - params := dto.GetParams{ - // SearchVector: []float32{0.1, 0.1, 0.1, 1.1, 0.1}, - ClassName: carClass.Class, - Pagination: &filters.Pagination{Limit: 100}, - Filters: test.filter, - } - res, err := repo.Search(context.Background(), params) - require.Nil(t, err) - require.Len(t, res, len(test.expectedIDs)) - - ids := make([]strfmt.UUID, len(test.expectedIDs)) - for pos, concept := range res { - ids[pos] = concept.ID - } - assert.ElementsMatch(t, ids, test.expectedIDs, "ids dont match") - }) - } - } -} - -func buildFilter(propName string, value interface{}, operator filters.Operator, schemaType schema.DataType) *filters.LocalFilter { - return &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: operator, - On: &filters.Path{ - Class: schema.ClassName(carClass.Class), - Property: schema.PropertyName(propName), - }, - Value: &filters.Value{ - Value: value, - Type: schemaType, - }, - }, - } -} - -func buildSortFilter(path []string, order string) filters.Sort { - return filters.Sort{Path: path, Order: order} -} - -func compoundFilter(operator filters.Operator, - operands ...*filters.LocalFilter, -) *filters.LocalFilter { - clauses := make([]filters.Clause, len(operands)) - for i, filter := range operands { - clauses[i] = *filter.Root - } - - return &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: operator, - Operands: clauses, - }, - } -} - -func filterAnd(operands ...*filters.LocalFilter) *filters.LocalFilter { - return compoundFilter(filters.OperatorAnd, operands...) -} - -func filterOr(operands ...*filters.LocalFilter) *filters.LocalFilter { - return compoundFilter(filters.OperatorOr, operands...) -} - -// test data -var carClass = &models.Class{ - Class: "FilterTestCar", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{ - { - DataType: schema.DataTypeText.PropString(), - Name: "modelName", - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - DataType: schema.DataTypeText.PropString(), - Name: "contact", - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - DataType: schema.DataTypeText.PropString(), - Name: "description", - Tokenization: models.PropertyTokenizationWord, - }, - { - DataType: []string{string(schema.DataTypeInt)}, - Name: "horsepower", - }, - { - DataType: []string{string(schema.DataTypeNumber)}, - Name: "weight", - }, - { - DataType: []string{string(schema.DataTypeGeoCoordinates)}, - Name: "parkedAt", - }, - { - DataType: []string{string(schema.DataTypeDate)}, - Name: "released", - }, - { - DataType: schema.DataTypeText.PropString(), - Name: "colorWhitespace", - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - DataType: schema.DataTypeText.PropString(), - Name: "colorField", - Tokenization: models.PropertyTokenizationField, - }, - { - DataType: schema.DataTypeTextArray.PropString(), - Name: "colorArrayWhitespace", - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - DataType: schema.DataTypeTextArray.PropString(), - Name: "colorArrayField", - Tokenization: models.PropertyTokenizationField, - }, - { - DataType: []string{string(schema.DataTypeUUID)}, - Name: "manufacturerId", - }, - { - DataType: []string{string(schema.DataTypeUUIDArray)}, - Name: "availableAtDealerships", - }, - }, -} - -// test data -var carClassNoLengthIndex = &models.Class{ - Class: "FilterTestCar", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: &models.InvertedIndexConfig{ - CleanupIntervalSeconds: 60, - Stopwords: &models.StopwordConfig{ - Preset: "none", - }, - IndexNullState: true, - IndexPropertyLength: false, - }, - Properties: []*models.Property{ - { - DataType: schema.DataTypeText.PropString(), - Name: "modelName", - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - DataType: schema.DataTypeText.PropString(), - Name: "contact", - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - DataType: schema.DataTypeText.PropString(), - Name: "description", - Tokenization: models.PropertyTokenizationWord, - }, - { - DataType: []string{string(schema.DataTypeInt)}, - Name: "horsepower", - }, - { - DataType: []string{string(schema.DataTypeNumber)}, - Name: "weight", - }, - { - DataType: []string{string(schema.DataTypeGeoCoordinates)}, - Name: "parkedAt", - }, - { - DataType: []string{string(schema.DataTypeDate)}, - Name: "released", - }, - { - DataType: schema.DataTypeText.PropString(), - Name: "colorWhitespace", - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - DataType: schema.DataTypeText.PropString(), - Name: "colorField", - Tokenization: models.PropertyTokenizationField, - }, - { - DataType: schema.DataTypeTextArray.PropString(), - Name: "colorArrayWhitespace", - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - DataType: schema.DataTypeTextArray.PropString(), - Name: "colorArrayField", - Tokenization: models.PropertyTokenizationField, - }, - { - DataType: []string{string(schema.DataTypeUUID)}, - Name: "manufacturerId", - }, - { - DataType: []string{string(schema.DataTypeUUIDArray)}, - Name: "availableAtDealerships", - }, - }, -} - -var ( - carSprinterID strfmt.UUID = "d4c48788-7798-4bdd-bca9-5cd5012a5271" - carE63sID strfmt.UUID = "62906c61-f92f-4f2c-874f-842d4fb9d80b" - carPoloID strfmt.UUID = "b444e1d8-d73a-4d53-a417-8d6501c27f2e" - carNilID strfmt.UUID = "b444e1d8-d73a-4d53-a417-8d6501c27f3e" - carEmpty strfmt.UUID = "b444e1d8-d73a-4d53-a417-8d6501c27f4e" - - // these UUIDs are not primary IDs of objects, but rather values for uuid and - // uuid[] fields - manufacturer1 = uuid.MustParse("11111111-2222-3333-4444-000000000001") - manufacturer2 = uuid.MustParse("11111111-2222-3333-4444-000000000002") - dealershipNorth = uuid.MustParse("99999999-9999-9999-9999-000000000001") - dealershipSouth = uuid.MustParse("99999999-9999-9999-9999-000000000002") -) - -func mustParseTime(in string) time.Time { - asTime, err := time.Parse(time.RFC3339, in) - if err != nil { - panic(err) - } - return asTime -} - -var cars = []models.Object{ - { - Class: carClass.Class, - ID: carSprinterID, - Properties: map[string]interface{}{ - "modelName": "sprinter", - "horsepower": int64(130), - "weight": 3499.90, - "released": mustParseTime("1995-08-17T12:47:00+02:00"), - "parkedAt": &models.GeoCoordinates{ - Latitude: ptFloat32(34.052235), - Longitude: ptFloat32(-118.243683), - }, - "contact": "john@heavycars.example.com", - "description": "This car resembles a large van that can still be driven with a regular license. Contact john@heavycars.example.com for details", - "colorWhitespace": "light grey", - "colorField": "light grey", - "colorArrayWhitespace": []interface{}{"light grey"}, - "colorArrayField": []interface{}{"light grey"}, - "manufacturerId": manufacturer1, - "availableAtDealerships": []uuid.UUID{dealershipNorth, dealershipSouth}, - }, - }, - { - Class: carClass.Class, - ID: carE63sID, - Properties: map[string]interface{}{ - "modelName": "e63s", - "horsepower": int64(612), - "weight": 2069.5, - "released": mustParseTime("2017-02-17T09:47:00+02:00"), - "parkedAt": &models.GeoCoordinates{ - Latitude: ptFloat32(40.730610), - Longitude: ptFloat32(-73.935242), - }, - "contact": "jessica-世界@unicode.example.com", - "description": "This car has a huge motor, but it's also not exactly lightweight.", - "colorWhitespace": "very light grey", - "colorField": "very light grey", - "colorArrayWhitespace": []interface{}{"very light", "grey"}, - "colorArrayField": []interface{}{"very light", "grey"}, - "manufacturerId": manufacturer1, - "availableAtDealerships": []uuid.UUID{dealershipNorth}, - }, - }, - { - Class: carClass.Class, - ID: carPoloID, - Properties: map[string]interface{}{ - "released": mustParseTime("1975-01-01T10:12:00+02:00"), - "modelName": "polo", - "horsepower": int64(100), - "weight": 1200.0, - "contact": "sandra@efficientcars.example.com", - "description": "This small car has a small engine and unicode labels (ąę), but it's very light, so it feels faster than it is.", - "colorWhitespace": "dark grey", - "colorField": "dark grey", - "colorArrayWhitespace": []interface{}{"dark", "grey"}, - "colorArrayField": []interface{}{"dark", "grey"}, - "manufacturerId": manufacturer2, - "availableAtDealerships": []uuid.UUID{dealershipSouth}, - }, - }, - { - Class: carClass.Class, - ID: carNilID, - Properties: map[string]interface{}{ - "modelName": "NilCar", - }, - }, - { - Class: carClass.Class, - ID: carEmpty, - Properties: map[string]interface{}{ - "modelName": "", - "contact": "", - "description": "", - "colorWhitespace": "", - "colorField": "", - "colorArrayWhitespace": []interface{}{}, - "colorArrayField": []interface{}{}, - }, - }, -} - -var carVectors = [][]float32{ - {1.1, 0, 0, 0, 0}, - {0, 1.1, 0, 0, 0}, - {0, 0, 1.1, 0, 0}, - {0, 0, 0, 1.1, 0}, - {0, 0, 0, 0, 1.1}, -} - -func TestGeoPropUpdateJourney(t *testing.T) { - dirName := t.TempDir() - - logger, _ := test.NewNullLogger() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - defer repo.Shutdown(context.Background()) - - migrator := NewMigrator(repo, logger) - - t.Run("import schema", func(t *testing.T) { - class := &models.Class{ - Class: "GeoUpdateTestClass", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{ - { - Name: "location", - DataType: []string{string(schema.DataTypeGeoCoordinates)}, - }, - }, - } - - migrator.AddClass(context.Background(), class, schemaGetter.shardState) - schemaGetter.schema.Objects = &models.Schema{ - Classes: []*models.Class{class}, - } - }) - - ids := []strfmt.UUID{ - "4002609e-ee57-4404-a0ad-798af7da0004", - "1477aed8-f677-4131-a3ad-4deef6176066", - } - - searchQuery := filters.GeoRange{ - GeoCoordinates: &models.GeoCoordinates{ - Latitude: ptFloat32(6.0), - Longitude: ptFloat32(-2.0), - }, - Distance: 400000, // distance to filter only 1 closest object in both test cases - } - - upsertFn := func(coordinates [][]float32) func(t *testing.T) { - return func(t *testing.T) { - for i, id := range ids { - repo.PutObject(context.Background(), &models.Object{ - Class: "GeoUpdateTestClass", - ID: id, - Properties: map[string]interface{}{ - "location": &models.GeoCoordinates{ - Latitude: &coordinates[i][0], - Longitude: &coordinates[i][1], - }, - }, - }, []float32{0.5}, nil) - } - } - } - - t.Run("import items", upsertFn([][]float32{ - {7, 1}, - {8, 2}, - })) - - t.Run("verify 1st object found", func(t *testing.T) { - res, err := repo.Search(context.Background(), - getParamsWithFilter("GeoUpdateTestClass", buildFilter( - "location", searchQuery, wgr, schema.DataTypeGeoCoordinates, - ))) - - require.Nil(t, err) - require.Len(t, res, 1) - assert.Equal(t, ids[0], res[0].ID) - }) - - t.Run("import items", upsertFn([][]float32{ - // move item 0 farther away from the search query and item 1 closer to it - {23, 14}, - {6.5, -1}, - })) - - t.Run("verify 2nd object found", func(t *testing.T) { - res, err := repo.Search(context.Background(), - getParamsWithFilter("GeoUpdateTestClass", buildFilter( - "location", searchQuery, wgr, schema.DataTypeGeoCoordinates, - ))) - - require.Nil(t, err) - require.Len(t, res, 1) - - // notice the opposite order - assert.Equal(t, ids[1], res[0].ID) - }) -} - -// This test prevents a regression on -// https://github.com/weaviate/weaviate/issues/1426 -func TestCasingOfOperatorCombinations(t *testing.T) { - dirName := t.TempDir() - - logger, _ := test.NewNullLogger() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - defer repo.Shutdown(context.Background()) - - migrator := NewMigrator(repo, logger) - - class := &models.Class{ - Class: "FilterCasingBug", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "textPropWord", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWord, - }, - { - Name: "textPropWhitespace", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - } - - objects := []*models.Object{ - { - Class: class.Class, - ID: strfmt.UUID(uuid.New().String()), - Properties: map[string]interface{}{ - "name": "all lowercase", - "textPropWhitespace": "apple banana orange", - "textPropWord": "apple banana orange", - }, - Vector: []float32{0.1}, - }, - { - Class: class.Class, - ID: strfmt.UUID(uuid.New().String()), - Properties: map[string]interface{}{ - "name": "mixed case", - "textPropWhitespace": "apple Banana ORANGE", - "textPropWord": "apple Banana ORANGE", - }, - Vector: []float32{0.1}, - }, - { - Class: class.Class, - ID: strfmt.UUID(uuid.New().String()), - Properties: map[string]interface{}{ - "name": "first letter uppercase", - "textPropWhitespace": "Apple Banana Orange", - "textPropWord": "Apple Banana Orange", - }, - Vector: []float32{0.1}, - }, - { - Class: class.Class, - ID: strfmt.UUID(uuid.New().String()), - Properties: map[string]interface{}{ - "name": "all uppercase", - "textPropWhitespace": "APPLE BANANA ORANGE", - "textPropWord": "APPLE BANANA ORANGE", - }, - Vector: []float32{0.1}, - }, - } - - t.Run("creating the class", func(t *testing.T) { - require.Nil(t, - migrator.AddClass(context.Background(), class, schemaGetter.shardState)) - schemaGetter.schema.Objects = &models.Schema{ - Classes: []*models.Class{ - class, - }, - } - }) - - t.Run("importing the objects", func(t *testing.T) { - for i, obj := range objects { - t.Run(fmt.Sprintf("importing object %d", i), func(t *testing.T) { - require.Nil(t, - repo.PutObject(context.Background(), obj, obj.Vector, nil)) - }) - } - }) - - t.Run("verifying combinations", func(t *testing.T) { - type test struct { - name string - filter *filters.LocalFilter - expectedNames []string - limit int - } - - tests := []test{ - { - name: "text word, lowercase, single word, should match all", - filter: buildFilter("textPropWord", "apple", eq, dtText), - expectedNames: []string{ - "all uppercase", "all lowercase", "mixed case", - "first letter uppercase", - }, - }, - { - name: "text word, lowercase, multiple words, should match all", - filter: buildFilter("textPropWord", "apple banana orange", eq, dtText), - expectedNames: []string{ - "all uppercase", "all lowercase", "mixed case", - "first letter uppercase", - }, - }, - { - name: "text word, mixed case, single word, should match all", - filter: buildFilter("textPropWord", "Apple", eq, dtText), - expectedNames: []string{ - "all uppercase", "all lowercase", "mixed case", - "first letter uppercase", - }, - }, - { - name: "text word, mixed case, multiple words, should match all", - filter: buildFilter("textPropWord", "Apple Banana Orange", eq, dtText), - expectedNames: []string{ - "all uppercase", "all lowercase", "mixed case", - "first letter uppercase", - }, - }, - { - name: "text word, uppercase, single word, should match all", - filter: buildFilter("textPropWord", "APPLE", eq, dtText), - expectedNames: []string{ - "all uppercase", "all lowercase", "mixed case", - "first letter uppercase", - }, - }, - { - name: "text word, uppercase, multiple words, should match all", - filter: buildFilter("textPropWord", "APPLE BANANA ORANGE", eq, dtText), - expectedNames: []string{ - "all uppercase", "all lowercase", "mixed case", - "first letter uppercase", - }, - }, - { - name: "text whitespace, lowercase, single word, should match exact casing", - filter: buildFilter("textPropWhitespace", "apple", eq, dtText), - expectedNames: []string{ - "all lowercase", "mixed case", // mixed matches because the first word is all lowercase - }, - }, - { - name: "text whitespace, lowercase, multiple words, should match all-lowercase", - filter: buildFilter("textPropWhitespace", "apple banana orange", eq, dtText), - expectedNames: []string{"all lowercase"}, - }, - { - name: "text whitespace, mixed case, single word, should match exact matches", - filter: buildFilter("textPropWhitespace", "Banana", eq, dtText), - expectedNames: []string{ - "mixed case", "first letter uppercase", - }, - }, - { - name: "text whitespace, mixed case, multiple words, should match exact matches", - filter: buildFilter("textPropWhitespace", "apple Banana ORANGE", eq, dtText), - expectedNames: []string{ - "mixed case", - }, - }, - { - name: "text whitespace, uppercase, single word, should match all upper", - filter: buildFilter("textPropWhitespace", "APPLE", eq, dtText), - expectedNames: []string{ - "all uppercase", - }, - }, - { - name: "text whitespace, uppercase, multiple words, should match only all upper", - filter: buildFilter("textPropWhitespace", "APPLE BANANA ORANGE", eq, dtText), - expectedNames: []string{ - "all uppercase", - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - if test.limit == 0 { - test.limit = 100 - } - params := dto.GetParams{ - ClassName: class.Class, - Pagination: &filters.Pagination{Limit: test.limit}, - Filters: test.filter, - } - res, err := repo.Search(context.Background(), params) - require.Nil(t, err) - require.Len(t, res, len(test.expectedNames)) - - names := make([]string, len(test.expectedNames)) - for pos, obj := range res { - names[pos] = obj.Schema.(map[string]interface{})["name"].(string) - } - assert.ElementsMatch(t, names, test.expectedNames, "names don't match") - }) - } - }) -} - -func testSortProperties(repo *DB) func(t *testing.T) { - return func(t *testing.T) { - type test struct { - name string - sort []filters.Sort - expectedIDs []strfmt.UUID - wantErr bool - errMessage string - } - tests := []test{ - { - name: "modelName asc", - sort: []filters.Sort{ - buildSortFilter([]string{"modelName"}, "asc"), - }, - expectedIDs: []strfmt.UUID{carEmpty, carE63sID, carNilID, carPoloID, carSprinterID}, - }, - { - name: "modelName desc", - sort: []filters.Sort{ - buildSortFilter([]string{"modelName"}, "desc"), - }, - expectedIDs: []strfmt.UUID{carSprinterID, carPoloID, carNilID, carE63sID, carEmpty}, - }, - { - name: "horsepower asc", - sort: []filters.Sort{ - buildSortFilter([]string{"horsepower"}, "asc"), - }, - expectedIDs: []strfmt.UUID{carNilID, carEmpty, carPoloID, carSprinterID, carE63sID}, - }, - { - name: "horsepower desc", - sort: []filters.Sort{ - buildSortFilter([]string{"horsepower"}, "desc"), - }, - expectedIDs: []strfmt.UUID{carE63sID, carSprinterID, carPoloID, carNilID, carEmpty}, - }, - { - name: "weight asc", - sort: []filters.Sort{ - buildSortFilter([]string{"weight"}, "asc"), - }, - expectedIDs: []strfmt.UUID{carNilID, carEmpty, carPoloID, carE63sID, carSprinterID}, - }, - { - name: "weight desc", - sort: []filters.Sort{ - buildSortFilter([]string{"weight"}, "desc"), - }, - expectedIDs: []strfmt.UUID{carSprinterID, carE63sID, carPoloID, carNilID, carEmpty}, - }, - { - name: "released asc", - sort: []filters.Sort{ - buildSortFilter([]string{"released"}, "asc"), - }, - expectedIDs: []strfmt.UUID{carNilID, carEmpty, carPoloID, carSprinterID, carE63sID}, - }, - { - name: "released desc", - sort: []filters.Sort{ - buildSortFilter([]string{"released"}, "desc"), - }, - expectedIDs: []strfmt.UUID{carE63sID, carSprinterID, carPoloID, carNilID, carEmpty}, - }, - { - name: "parkedAt asc", - sort: []filters.Sort{ - buildSortFilter([]string{"parkedAt"}, "asc"), - }, - expectedIDs: []strfmt.UUID{carPoloID, carNilID, carEmpty, carSprinterID, carE63sID}, - }, - { - name: "parkedAt desc", - sort: []filters.Sort{ - buildSortFilter([]string{"parkedAt"}, "desc"), - }, - expectedIDs: []strfmt.UUID{carE63sID, carSprinterID, carPoloID, carNilID, carEmpty}, - }, - { - name: "contact asc", - sort: []filters.Sort{ - buildSortFilter([]string{"contact"}, "asc"), - }, - expectedIDs: []strfmt.UUID{carNilID, carEmpty, carE63sID, carSprinterID, carPoloID}, - }, - { - name: "contact desc", - sort: []filters.Sort{ - buildSortFilter([]string{"contact"}, "desc"), - }, - expectedIDs: []strfmt.UUID{carPoloID, carSprinterID, carE63sID, carEmpty, carNilID}, - }, - { - name: "description asc", - sort: []filters.Sort{ - buildSortFilter([]string{"description"}, "asc"), - }, - expectedIDs: []strfmt.UUID{carNilID, carEmpty, carE63sID, carSprinterID, carPoloID}, - }, - { - name: "description desc", - sort: []filters.Sort{ - buildSortFilter([]string{"description"}, "desc"), - }, - expectedIDs: []strfmt.UUID{carPoloID, carSprinterID, carE63sID, carEmpty, carNilID}, - }, - { - name: "colorArrayWhitespace asc", - sort: []filters.Sort{ - buildSortFilter([]string{"colorArrayWhitespace"}, "asc"), - }, - expectedIDs: []strfmt.UUID{carNilID, carEmpty, carPoloID, carSprinterID, carE63sID}, - }, - { - name: "colorArrayWhitespace desc", - sort: []filters.Sort{ - buildSortFilter([]string{"colorArrayWhitespace"}, "desc"), - }, - expectedIDs: []strfmt.UUID{carE63sID, carSprinterID, carPoloID, carNilID, carEmpty}, - }, - { - name: "modelName and horsepower asc", - sort: []filters.Sort{ - buildSortFilter([]string{"modelName"}, "asc"), - buildSortFilter([]string{"horsepower"}, "asc"), - }, - expectedIDs: []strfmt.UUID{carEmpty, carE63sID, carNilID, carPoloID, carSprinterID}, - }, - { - name: "horsepower and modelName asc", - sort: []filters.Sort{ - buildSortFilter([]string{"horsepower"}, "asc"), - buildSortFilter([]string{"modelName"}, "asc"), - }, - expectedIDs: []strfmt.UUID{carEmpty, carNilID, carPoloID, carSprinterID, carE63sID}, - }, - { - name: "horsepower and modelName asc invalid sort", - sort: []filters.Sort{ - buildSortFilter([]string{"horsepower", "modelName"}, "asc"), - }, - expectedIDs: nil, - wantErr: true, - errMessage: "sort object list: sorting by reference not supported, path must have exactly one argument", - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - params := dto.GetParams{ - ClassName: carClass.Class, - Pagination: &filters.Pagination{Limit: 100}, - Sort: test.sort, - } - res, err := repo.Search(context.Background(), params) - if test.wantErr { - require.NotNil(t, err) - require.Contains(t, err.Error(), test.errMessage) - } else { - require.Nil(t, err) - require.Len(t, res, len(test.expectedIDs)) - - ids := make([]strfmt.UUID, len(test.expectedIDs)) - for pos, concept := range res { - ids[pos] = concept.ID - } - assert.EqualValues(t, test.expectedIDs, ids, "ids dont match") - } - }) - } - } -} - -func TestFilteringAfterDeletion(t *testing.T) { - dirName := t.TempDir() - - logger, _ := test.NewNullLogger() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - defer repo.Shutdown(context.Background()) - - migrator := NewMigrator(repo, logger) - class := &models.Class{ - Class: "DeletionClass", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "other", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - } - UUID1 := strfmt.UUID(uuid.New().String()) - UUID2 := strfmt.UUID(uuid.New().String()) - objects := []*models.Object{ - { - Class: class.Class, - ID: UUID1, - Properties: map[string]interface{}{ - "name": "otherthing", - "other": "not nil", - }, - }, - { - Class: class.Class, - ID: UUID2, - Properties: map[string]interface{}{ - "name": "something", - "other": nil, - }, - }, - } - - t.Run("creating the class and add objects", func(t *testing.T) { - require.Nil(t, - migrator.AddClass(context.Background(), class, schemaGetter.shardState)) - schemaGetter.schema.Objects = &models.Schema{ - Classes: []*models.Class{ - class, - }, - } - for i, obj := range objects { - t.Run(fmt.Sprintf("importing object %d", i), func(t *testing.T) { - require.Nil(t, - repo.PutObject(context.Background(), obj, obj.Vector, nil)) - }) - } - }) - - t.Run("Filter before deletion", func(t *testing.T) { - filterNil := buildFilter("other", true, null, dtBool) - paramsNil := dto.GetParams{ - ClassName: class.Class, - Pagination: &filters.Pagination{Limit: 2}, - Filters: filterNil, - } - resNil, err := repo.Search(context.Background(), paramsNil) - assert.Nil(t, err) - assert.Equal(t, 1, len(resNil)) - assert.Equal(t, UUID2, resNil[0].ID) - - filterLen := buildFilter("len(name)", 9, eq, dtInt) - paramsLen := dto.GetParams{ - ClassName: class.Class, - Pagination: &filters.Pagination{Limit: 2}, - Filters: filterLen, - } - resLen, err := repo.Search(context.Background(), paramsLen) - assert.Nil(t, err) - assert.Equal(t, 1, len(resLen)) - assert.Equal(t, UUID2, resLen[0].ID) - }) - - t.Run("Delete object and filter again", func(t *testing.T) { - repo.DeleteObject(context.Background(), "DeletionClass", UUID2, nil, "") - - filterNil := buildFilter("other", true, null, dtBool) - paramsNil := dto.GetParams{ - ClassName: class.Class, - Pagination: &filters.Pagination{Limit: 2}, - Filters: filterNil, - } - resNil, err := repo.Search(context.Background(), paramsNil) - assert.Nil(t, err) - assert.Equal(t, 0, len(resNil)) - - filterLen := buildFilter("len(name)", 9, eq, dtInt) - paramsLen := dto.GetParams{ - ClassName: class.Class, - Pagination: &filters.Pagination{Limit: 2}, - Filters: filterLen, - } - resLen, err := repo.Search(context.Background(), paramsLen) - assert.Nil(t, err) - assert.Equal(t, 0, len(resLen)) - }) -} diff --git a/adapters/repos/db/filters_limits_integration_test.go b/adapters/repos/db/filters_limits_integration_test.go deleted file mode 100644 index 65e9ac3489fbdceb1818b0b65632728b0acd1571..0000000000000000000000000000000000000000 --- a/adapters/repos/db/filters_limits_integration_test.go +++ /dev/null @@ -1,370 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package db - -import ( - "context" - "fmt" - "testing" - - "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/aggregation" - "github.com/weaviate/weaviate/entities/dto" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" -) - -// This test aims to prevent a regression on -// https://github.com/weaviate/weaviate/issues/1352 -// -// It reuses the company-schema from the regular filters test, but runs them in -// isolation as to not interfere with the existing tests -func Test_LimitsOnChainedFilters(t *testing.T) { - dirName := t.TempDir() - - logger := logrus.New() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - defer repo.Shutdown(context.Background()) - - migrator := NewMigrator(repo, logger) - - t.Run("creating the class", func(t *testing.T) { - schema := schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{ - productClass, - companyClass, - }, - }, - } - - require.Nil(t, - migrator.AddClass(context.Background(), productClass, schemaGetter.shardState)) - require.Nil(t, - migrator.AddClass(context.Background(), companyClass, schemaGetter.shardState)) - - schemaGetter.schema = schema - }) - - data := chainedFilterCompanies(100) - - t.Run("import companies", func(t *testing.T) { - for i, company := range data { - t.Run(fmt.Sprintf("importing product %d", i), func(t *testing.T) { - require.Nil(t, - repo.PutObject(context.Background(), company, []float32{0.1, 0.2, 0.01, 0.2}, nil)) - }) - } - }) - - t.Run("combine two filters with a strict limit", func(t *testing.T) { - limit := 20 - - filter := filterAnd( - buildFilter("price", 20, gte, dtInt), - buildFilter("price", 100, lt, dtInt), - ) - - res, err := repo.Search(context.Background(), dto.GetParams{ - ClassName: companyClass.Class, - Filters: filter, - Pagination: &filters.Pagination{ - Limit: limit, - }, - }) - - require.Nil(t, err) - assert.Len(t, res, limit) - - for _, obj := range res { - assert.Less(t, obj.Schema.(map[string]interface{})["price"].(float64), - float64(100)) - assert.GreaterOrEqual(t, - obj.Schema.(map[string]interface{})["price"].(float64), float64(20)) - } - }) -} - -func chainedFilterCompanies(size int) []*models.Object { - out := make([]*models.Object, size) - - for i := range out { - out[i] = &models.Object{ - ID: mustNewUUID(), - Class: companyClass.Class, - Properties: map[string]interface{}{ - "price": int64(i), - }, - } - } - - return out -} - -// This test aims to prevent a regression on -// https://github.com/weaviate/weaviate/issues/1355 -// -// It reuses the company-schema from the regular filters test, but runs them in -// isolation as to not interfere with the existing tests -func Test_FilterLimitsAfterUpdates(t *testing.T) { - dirName := t.TempDir() - - logger := logrus.New() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - defer repo.Shutdown(context.Background()) - - migrator := NewMigrator(repo, logger) - - t.Run("creating the class", func(t *testing.T) { - schema := schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{ - productClass, - companyClass, - }, - }, - } - - require.Nil(t, - migrator.AddClass(context.Background(), productClass, schemaGetter.shardState)) - require.Nil(t, - migrator.AddClass(context.Background(), companyClass, schemaGetter.shardState)) - - schemaGetter.schema = schema - }) - - data := chainedFilterCompanies(100) - - t.Run("import companies", func(t *testing.T) { - for i, company := range data { - t.Run(fmt.Sprintf("importing product %d", i), func(t *testing.T) { - require.Nil(t, - repo.PutObject(context.Background(), company, []float32{0.1, 0.2, 0.01, 0.2}, nil)) - }) - } - }) - - t.Run("verify all with ref count 0 are found", func(t *testing.T) { - limit := 100 - filter := buildFilter("makesProduct", 0, eq, dtInt) - res, err := repo.Search(context.Background(), dto.GetParams{ - ClassName: companyClass.Class, - Filters: filter, - Pagination: &filters.Pagination{ - Limit: limit, - }, - }) - - require.Nil(t, err) - assert.Len(t, res, limit) - }) - - t.Run("verify a non refcount prop", func(t *testing.T) { - limit := 100 - filter := buildFilter("price", float64(0), gte, dtNumber) - res, err := repo.Search(context.Background(), dto.GetParams{ - ClassName: companyClass.Class, - Filters: filter, - Pagination: &filters.Pagination{ - Limit: limit, - }, - }) - - require.Nil(t, err) - assert.Len(t, res, limit) - }) - - t.Run("perform updates on each company", func(t *testing.T) { - // in this case we're altering the vector position, but it doesn't really - // matter - what we want to provoke is to fill up our index with deleted - // doc ids - for i, company := range data { - t.Run(fmt.Sprintf("importing product %d", i), func(t *testing.T) { - require.Nil(t, - repo.PutObject(context.Background(), company, []float32{0.1, 0.21, 0.01, 0.2}, nil)) - }) - } - }) - - t.Run("verify all with ref count 0 are found", func(t *testing.T) { - limit := 100 - filter := buildFilter("makesProduct", 0, eq, dtInt) - res, err := repo.Search(context.Background(), dto.GetParams{ - ClassName: companyClass.Class, - Filters: filter, - Pagination: &filters.Pagination{ - Limit: limit, - }, - }) - - require.Nil(t, err) - assert.Len(t, res, limit) - }) - - t.Run("verify a non refcount prop", func(t *testing.T) { - limit := 100 - filter := buildFilter("price", float64(0), gte, dtNumber) - res, err := repo.Search(context.Background(), dto.GetParams{ - ClassName: companyClass.Class, - Filters: filter, - Pagination: &filters.Pagination{ - Limit: limit, - }, - }) - - require.Nil(t, err) - assert.Len(t, res, limit) - }) -} - -// This test aims to prevent a regression on -// https://github.com/weaviate/weaviate/issues/1356 -// -// It reuses the company-schema from the regular filters test, but runs them in -// isolation as to not interfere with the existing tests -func Test_AggregationsAfterUpdates(t *testing.T) { - dirName := t.TempDir() - - logger := logrus.New() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - defer repo.Shutdown(context.Background()) - - migrator := NewMigrator(repo, logger) - - t.Run("creating the class", func(t *testing.T) { - schema := schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{ - productClass, - companyClass, - }, - }, - } - - require.Nil(t, - migrator.AddClass(context.Background(), productClass, schemaGetter.shardState)) - require.Nil(t, - migrator.AddClass(context.Background(), companyClass, schemaGetter.shardState)) - - schemaGetter.schema = schema - }) - - data := chainedFilterCompanies(100) - - t.Run("import companies", func(t *testing.T) { - for i, company := range data { - t.Run(fmt.Sprintf("importing product %d", i), func(t *testing.T) { - require.Nil(t, - repo.PutObject(context.Background(), company, []float32{0.1, 0.2, 0.01, 0.2}, nil)) - }) - } - }) - - t.Run("verify all with ref count 0 are correctly aggregated", - func(t *testing.T) { - filter := buildFilter("makesProduct", 0, eq, dtInt) - res, err := repo.Aggregate(context.Background(), - aggregation.Params{ - ClassName: schema.ClassName(companyClass.Class), - Filters: filter, - IncludeMetaCount: true, - }) - - require.Nil(t, err) - require.Len(t, res.Groups, 1) - assert.Equal(t, res.Groups[0].Count, 100) - }) - - t.Run("perform updates on each company", func(t *testing.T) { - // in this case we're altering the vector position, but it doesn't really - // matter - what we want to provoke is to fill up our index with deleted - // doc ids - for i, company := range data { - t.Run(fmt.Sprintf("importing product %d", i), func(t *testing.T) { - require.Nil(t, - repo.PutObject(context.Background(), company, []float32{0.1, 0.21, 0.01, 0.2}, nil)) - }) - } - }) - - t.Run("verify all with ref count 0 are correctly aggregated", - func(t *testing.T) { - filter := buildFilter("makesProduct", 0, eq, dtInt) - res, err := repo.Aggregate(context.Background(), - aggregation.Params{ - ClassName: schema.ClassName(companyClass.Class), - Filters: filter, - IncludeMetaCount: true, - }) - - require.Nil(t, err) - require.Len(t, res.Groups, 1) - assert.Equal(t, res.Groups[0].Count, 100) - }) - - t.Run("verify all with ref count 0 are correctly aggregated", - func(t *testing.T) { - filter := buildFilter("makesProduct", 0, eq, dtInt) - res, err := repo.Aggregate(context.Background(), - aggregation.Params{ - ClassName: schema.ClassName(companyClass.Class), - Filters: filter, - IncludeMetaCount: true, - }) - - require.Nil(t, err) - require.Len(t, res.Groups, 1) - assert.Equal(t, 100, res.Groups[0].Count) - }) -} diff --git a/adapters/repos/db/filters_on_refs_integration_test.go b/adapters/repos/db/filters_on_refs_integration_test.go deleted file mode 100644 index afc01c4fcee11853fc481c224385ca706ce0711e..0000000000000000000000000000000000000000 --- a/adapters/repos/db/filters_on_refs_integration_test.go +++ /dev/null @@ -1,804 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package db - -import ( - "context" - "fmt" - "testing" - - "github.com/go-openapi/strfmt" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/dto" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/schema/crossref" - "github.com/weaviate/weaviate/entities/search" -) - -func TestRefFilters(t *testing.T) { - dirName := t.TempDir() - - logger, _ := test.NewNullLogger() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryLimit: 20, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - defer repo.Shutdown(testCtx()) - migrator := NewMigrator(repo, logger) - - t.Run("adding all classes to the schema", func(t *testing.T) { - schemaGetter.schema.Objects = &models.Schema{} - for _, class := range parkingGaragesSchema().Objects.Classes { - t.Run(fmt.Sprintf("add %s", class.Class), func(t *testing.T) { - err := migrator.AddClass(context.Background(), class, schemaGetter.shardState) - require.Nil(t, err) - schemaGetter.schema.Objects.Classes = append(schemaGetter.schema.Objects.Classes, class) - }) - } - }) - - t.Run("importing with various combinations of props", func(t *testing.T) { - objects := []models.Object{ - { - Class: "MultiRefParkingGarage", - Properties: map[string]interface{}{ - "name": "Luxury Parking Garage", - "location": &models.GeoCoordinates{ - Latitude: ptFloat32(48.864716), - Longitude: ptFloat32(2.349014), - }, - }, - ID: "a7e10b55-1ac4-464f-80df-82508eea1951", - CreationTimeUnix: 1566469890, - }, - { - Class: "MultiRefParkingGarage", - Properties: map[string]interface{}{ - "name": "Crappy Parking Garage", - "location": &models.GeoCoordinates{ - Latitude: ptFloat32(42.331429), - Longitude: ptFloat32(-83.045753), - }, - }, - ID: "ba2232cf-bb0e-413d-b986-6aa996d34d2e", - CreationTimeUnix: 1566469892, - }, - { - Class: "MultiRefParkingLot", - Properties: map[string]interface{}{ - "name": "Fancy Parking Lot", - }, - ID: "1023967b-9512-475b-8ef9-673a110b695d", - CreationTimeUnix: 1566469894, - }, - { - Class: "MultiRefParkingLot", - Properties: map[string]interface{}{ - "name": "The worst parking lot youve ever seen", - }, - ID: "901859d8-69bf-444c-bf43-498963d798d2", - CreationTimeUnix: 1566469897, - }, - { - Class: "MultiRefCar", - Properties: map[string]interface{}{ - "name": "Car which is parked no where", - }, - ID: "329c306b-c912-4ec7-9b1d-55e5e0ca8dea", - CreationTimeUnix: 1566469899, - }, - { - Class: "MultiRefCar", - Properties: map[string]interface{}{ - "name": "Car which is parked in a garage", - "parkedAt": models.MultipleRef{ - &models.SingleRef{ - Beacon: "weaviate://localhost/MultiRefParkingGarage/a7e10b55-1ac4-464f-80df-82508eea1951", - }, - }, - }, - ID: "fe3ca25d-8734-4ede-9a81-bc1ed8c3ea43", - CreationTimeUnix: 1566469902, - }, - { - Class: "MultiRefCar", - Properties: map[string]interface{}{ - "name": "Car which is parked in a lot", - "parkedAt": models.MultipleRef{ - &models.SingleRef{ - Beacon: "weaviate://localhost/1023967b-9512-475b-8ef9-673a110b695d", - }, - }, - }, - ID: "21ab5130-627a-4268-baef-1a516bd6cad4", - CreationTimeUnix: 1566469906, - }, - { - Class: "MultiRefCar", - Properties: map[string]interface{}{ - "name": "Car which is parked in two places at the same time (magic!)", - "parkedAt": models.MultipleRef{ - &models.SingleRef{ - Beacon: "weaviate://localhost/a7e10b55-1ac4-464f-80df-82508eea1951", - }, - &models.SingleRef{ - Beacon: "weaviate://localhost/MultiRefParkingLot/1023967b-9512-475b-8ef9-673a110b695d", - }, - }, - }, - ID: "533673a7-2a5c-4e1c-b35d-a3809deabace", - CreationTimeUnix: 1566469909, - }, - { - Class: "MultiRefDriver", - Properties: map[string]interface{}{ - "name": "Johny Drivemuch", - "drives": models.MultipleRef{ - &models.SingleRef{ - Beacon: "weaviate://localhost/533673a7-2a5c-4e1c-b35d-a3809deabace", - }, - }, - }, - ID: "9653ab38-c16b-4561-80df-7a7e19300dd0", - CreationTimeUnix: 1566469912, - }, - { - Class: "MultiRefPerson", - Properties: map[string]interface{}{ - "name": "Jane Doughnut", - "friendsWith": models.MultipleRef{ - &models.SingleRef{ - Beacon: "weaviate://localhost/9653ab38-c16b-4561-80df-7a7e19300dd0", - }, - }, - }, - ID: "91ad23a3-07ba-4d4c-9836-76c57094f734", - CreationTimeUnix: 1566469915, - }, - { - Class: "MultiRefSociety", - Properties: map[string]interface{}{ - "name": "Cool People", - "hasMembers": models.MultipleRef{ - &models.SingleRef{ - Beacon: "weaviate://localhost/91ad23a3-07ba-4d4c-9836-76c57094f734", - }, - }, - }, - ID: "5cd9afa6-f3df-4f57-a204-840d6b256dba", - CreationTimeUnix: 1566469918, - }, - } - - for _, thing := range objects { - t.Run(fmt.Sprintf("add %s", thing.ID), func(t *testing.T) { - err := repo.PutObject(context.Background(), &thing, []float32{1, 2, 3, 4, 5, 6, 7}, nil) - require.Nil(t, err) - }) - } - }) - - t.Run("filtering", func(t *testing.T) { - t.Run("one level deep", func(t *testing.T) { - t.Run("ref name matches", func(t *testing.T) { - filter := filterCarParkedAtGarage(schema.DataTypeText, - "name", filters.OperatorEqual, "Luxury Parking Garage") - params := getParamsWithFilter("MultiRefCar", filter) - - res, err := repo.Search(context.Background(), params) - require.Nil(t, err) - require.Len(t, res, 2) - }) - - t.Run("ref id matches", func(t *testing.T) { - filter := filterCarParkedAtGarage(schema.DataTypeText, - "id", filters.OperatorEqual, "a7e10b55-1ac4-464f-80df-82508eea1951") - params := getParamsWithFilter("MultiRefCar", filter) - - res, err := repo.Search(context.Background(), params) - require.Nil(t, err) - require.Len(t, res, 2) - }) - - t.Run("ref name doesn't match", func(t *testing.T) { - filter := filterCarParkedAtGarage(schema.DataTypeText, - "name", filters.OperatorEqual, "There is no parking garage with this name") - params := getParamsWithFilter("MultiRefCar", filter) - - res, err := repo.Search(context.Background(), params) - require.Nil(t, err) - require.Len(t, res, 0) - }) - - t.Run("within geo range", func(t *testing.T) { - filter := filterCarParkedAtGarage(schema.DataTypeGeoCoordinates, - "location", filters.OperatorWithinGeoRange, filters.GeoRange{ - GeoCoordinates: &models.GeoCoordinates{ - Latitude: ptFloat32(48.801407), - Longitude: ptFloat32(2.130122), - }, - Distance: 100000, - }) - params := getParamsWithFilter("MultiRefCar", filter) - - res, err := repo.Search(context.Background(), params) - require.Nil(t, err) - require.Len(t, res, 2) - - names := extractNames(res) - expectedNames := []string{ - "Car which is parked in a garage", - "Car which is parked in two places at the same time (magic!)", - } - - assert.ElementsMatch(t, names, expectedNames) - }) - - t.Run("outside of geo range", func(t *testing.T) { - filter := filterCarParkedAtGarage(schema.DataTypeGeoCoordinates, - "location", filters.OperatorWithinGeoRange, filters.GeoRange{ - GeoCoordinates: &models.GeoCoordinates{ - Latitude: ptFloat32(42.279594), - Longitude: ptFloat32(-83.732124), - }, - Distance: 100000, - }) - params := getParamsWithFilter("MultiRefCar", filter) - - res, err := repo.Search(context.Background(), params) - require.Nil(t, err) - require.Len(t, res, 0) - }) - - t.Run("combining ref filter with primitive root filter", func(t *testing.T) { - parkedAtFilter := filterCarParkedAtGarage(schema.DataTypeGeoCoordinates, - "location", filters.OperatorWithinGeoRange, filters.GeoRange{ - GeoCoordinates: &models.GeoCoordinates{ - Latitude: ptFloat32(48.801407), - Longitude: ptFloat32(2.130122), - }, - Distance: 100000, - }) - - filter := &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorAnd, - Operands: []filters.Clause{ - *(parkedAtFilter.Root), - { - On: &filters.Path{ - Class: schema.ClassName("MultiRefCar"), - Property: schema.PropertyName("name"), - }, - Value: &filters.Value{ - Value: "Car which is parked in a garage", - Type: schema.DataTypeText, - }, - Operator: filters.OperatorEqual, - }, - }, - }, - } - params := getParamsWithFilter("MultiRefCar", filter) - - res, err := repo.Search(context.Background(), params) - require.Nil(t, err) - require.Len(t, res, 1) - - names := extractNames(res) - expectedNames := []string{ - "Car which is parked in a garage", - } - - assert.ElementsMatch(t, names, expectedNames) - }) - }) - - t.Run("multiple levels deep", func(t *testing.T) { - t.Run("ref name matches", func(t *testing.T) { - filter := filterDrivesCarParkedAtGarage(schema.DataTypeText, - "name", filters.OperatorEqual, "Luxury Parking Garage") - params := getParamsWithFilter("MultiRefDriver", filter) - - res, err := repo.Search(context.Background(), params) - require.Nil(t, err) - require.Len(t, res, 1) - - assert.Equal(t, "Johny Drivemuch", res[0].Schema.(map[string]interface{})["name"]) - }) - - t.Run("ref name doesn't match", func(t *testing.T) { - filter := filterDrivesCarParkedAtGarage(schema.DataTypeText, - "name", filters.OperatorEqual, "There is no parking garage with this name") - params := getParamsWithFilter("MultiRefDriver", filter) - - res, err := repo.Search(context.Background(), params) - require.Nil(t, err) - require.Len(t, res, 0) - }) - - t.Run("within geo range", func(t *testing.T) { - filter := filterDrivesCarParkedAtGarage(schema.DataTypeGeoCoordinates, - "location", filters.OperatorWithinGeoRange, filters.GeoRange{ - GeoCoordinates: &models.GeoCoordinates{ - Latitude: ptFloat32(48.801407), - Longitude: ptFloat32(2.130122), - }, - Distance: 100000, - }) - params := getParamsWithFilter("MultiRefDriver", filter) - - res, err := repo.Search(context.Background(), params) - require.Nil(t, err) - require.Len(t, res, 1) - - assert.Equal(t, "Johny Drivemuch", res[0].Schema.(map[string]interface{})["name"]) - }) - - t.Run("outside of geo range", func(t *testing.T) { - filter := filterDrivesCarParkedAtGarage(schema.DataTypeGeoCoordinates, - "location", filters.OperatorWithinGeoRange, filters.GeoRange{ - GeoCoordinates: &models.GeoCoordinates{ - Latitude: ptFloat32(42.279594), - Longitude: ptFloat32(-83.732124), - }, - Distance: 100000, - }) - params := getParamsWithFilter("MultiRefDriver", filter) - - res, err := repo.Search(context.Background(), params) - require.Nil(t, err) - require.Len(t, res, 0) - }) - }) - - t.Run("by reference count", func(t *testing.T) { - t.Run("equal to zero", func(t *testing.T) { - filter := filterCarParkedCount(filters.OperatorEqual, 0) - params := getParamsWithFilter("MultiRefCar", filter) - res, err := repo.Search(context.Background(), params) - require.Nil(t, err) - require.Len(t, res, 1) // there is just one car parked nowhere - assert.Equal(t, "Car which is parked no where", res[0].Schema.(map[string]interface{})["name"]) - }) - - t.Run("equal to one", func(t *testing.T) { - filter := filterCarParkedCount(filters.OperatorEqual, 1) - params := getParamsWithFilter("MultiRefCar", filter) - res, err := repo.Search(context.Background(), params) - require.Nil(t, err) - expectedNames := []string{ - "Car which is parked in a garage", - "Car which is parked in a lot", - } - assert.ElementsMatch(t, expectedNames, extractNames(res)) - }) - - t.Run("equal to more than one", func(t *testing.T) { - filter := filterCarParkedCount(filters.OperatorGreaterThan, 1) - params := getParamsWithFilter("MultiRefCar", filter) - res, err := repo.Search(context.Background(), params) - require.Nil(t, err) - expectedNames := []string{ - "Car which is parked in two places at the same time (magic!)", - } - assert.ElementsMatch(t, expectedNames, extractNames(res)) - }) - - t.Run("greater or equal one", func(t *testing.T) { - filter := filterCarParkedCount(filters.OperatorGreaterThanEqual, 1) - params := getParamsWithFilter("MultiRefCar", filter) - res, err := repo.Search(context.Background(), params) - require.Nil(t, err) - expectedNames := []string{ - "Car which is parked in a garage", - "Car which is parked in a lot", - "Car which is parked in two places at the same time (magic!)", - } - assert.ElementsMatch(t, expectedNames, extractNames(res)) - }) - - t.Run("less than one", func(t *testing.T) { - filter := filterCarParkedCount(filters.OperatorLessThan, 1) - params := getParamsWithFilter("MultiRefCar", filter) - res, err := repo.Search(context.Background(), params) - require.Nil(t, err) - expectedNames := []string{ - "Car which is parked no where", - } - assert.ElementsMatch(t, expectedNames, extractNames(res)) - }) - - t.Run("less than or equal one", func(t *testing.T) { - filter := filterCarParkedCount(filters.OperatorLessThanEqual, 1) - params := getParamsWithFilter("MultiRefCar", filter) - res, err := repo.Search(context.Background(), params) - require.Nil(t, err) - expectedNames := []string{ - "Car which is parked in a garage", - "Car which is parked in a lot", - "Car which is parked no where", - } - assert.ElementsMatch(t, expectedNames, extractNames(res)) - }) - }) - }) -} - -func TestRefFilters_MergingWithAndOperator(t *testing.T) { - // This test is to prevent a regression where checksums get lost on an AND - // operator, which was discovered through a journey test as part of gh-1286. - // The schema is modelled after the journey test, as the regular tests suites - // above do not seem to run into this issue on their own - dirName := t.TempDir() - - logger, _ := test.NewNullLogger() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - defer repo.Shutdown(testCtx()) - migrator := NewMigrator(repo, logger) - - t.Run("adding all classes to the schema", func(t *testing.T) { - schemaGetter.schema.Objects = &models.Schema{} - for _, class := range cityCountryAirportSchema().Objects.Classes { - t.Run(fmt.Sprintf("add %s", class.Class), func(t *testing.T) { - err := migrator.AddClass(context.Background(), class, schemaGetter.shardState) - require.Nil(t, err) - schemaGetter.schema.Objects.Classes = append(schemaGetter.schema.Objects.Classes, class) - }) - } - }) - - const ( - netherlands strfmt.UUID = "67b79643-cf8b-4b22-b206-6e63dbb4e57a" - germany strfmt.UUID = "561eea29-b733-4079-b50b-cfabd78190b7" - amsterdam strfmt.UUID = "8f5f8e44-d348-459c-88b1-c1a44bb8f8be" - rotterdam strfmt.UUID = "660db307-a163-41d2-8182-560782cd018f" - berlin strfmt.UUID = "9b9cbea5-e87e-4cd0-89af-e2f424fd52d6" - dusseldorf strfmt.UUID = "6ffb03f8-a853-4ec5-a5d8-302e45aaaf13" - nullisland strfmt.UUID = "823abeca-eef3-41c7-b587-7a6977b08003" - airport1 strfmt.UUID = "4770bb19-20fd-406e-ac64-9dac54c27a0f" - airport2 strfmt.UUID = "cad6ab9b-5bb9-4388-a933-a5bdfd23db37" - airport3 strfmt.UUID = "55a4dbbb-e2af-4b2a-901d-98146d1eeca7" - airport4 strfmt.UUID = "62d15920-b546-4844-bc87-3ae33268fab5" - ) - - t.Run("import all data objects", func(t *testing.T) { - objects := []*models.Object{ - { - Class: "Country", - ID: netherlands, - Properties: map[string]interface{}{ - "name": "Netherlands", - }, - }, - { - Class: "Country", - ID: germany, - Properties: map[string]interface{}{ - "name": "Germany", - }, - }, - - // cities - { - Class: "City", - ID: amsterdam, - Properties: map[string]interface{}{ - "name": "Amsterdam", - "population": int64(1800000), - "location": &models.GeoCoordinates{ - Latitude: ptFloat32(52.366667), - Longitude: ptFloat32(4.9), - }, - "inCountry": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI( - strfmt.URI(crossref.NewLocalhost("", netherlands).String()), - ), - }, - }, - }, - }, - { - Class: "City", - ID: rotterdam, - Properties: map[string]interface{}{ - "name": "Rotterdam", - "population": int64(600000), - "inCountry": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(crossref.NewLocalhost("", netherlands).String()), - }, - }, - }, - }, - { - Class: "City", - ID: berlin, - Properties: map[string]interface{}{ - "name": "Berlin", - "population": int64(3470000), - "inCountry": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(crossref.NewLocalhost("", germany).String()), - }, - }, - }, - }, - { - Class: "City", - ID: dusseldorf, - Properties: map[string]interface{}{ - "name": "Dusseldorf", - "population": int64(600000), - "inCountry": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(crossref.NewLocalhost("", germany).String()), - }, - }, - "location": &models.GeoCoordinates{ - Latitude: ptFloat32(51.225556), - Longitude: ptFloat32(6.782778), - }, - }, - }, - - { - Class: "City", - ID: nullisland, - Properties: map[string]interface{}{ - "name": "Null Island", - "population": 0, - "location": &models.GeoCoordinates{ - Latitude: ptFloat32(0), - Longitude: ptFloat32(0), - }, - }, - }, - - // airports - { - Class: "Airport", - ID: airport1, - Properties: map[string]interface{}{ - "code": "10000", - "phone": map[string]interface{}{ - "input": "+311234567", - }, - "inCity": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(crossref.NewLocalhost("", amsterdam).String()), - }, - }, - }, - }, - { - Class: "Airport", - ID: airport2, - Properties: map[string]interface{}{ - "code": "20000", - "inCity": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(crossref.NewLocalhost("", rotterdam).String()), - }, - }, - }, - }, - { - Class: "Airport", - ID: airport3, - Properties: map[string]interface{}{ - "code": "30000", - "inCity": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(crossref.NewLocalhost("", dusseldorf).String()), - }, - }, - }, - }, - { - Class: "Airport", - ID: airport4, - Properties: map[string]interface{}{ - "code": "40000", - "inCity": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(crossref.NewLocalhost("", berlin).String()), - }, - }, - }, - }, - } - - for _, obj := range objects { - require.Nil(t, repo.PutObject(context.Background(), obj, []float32{0.1}, nil)) - } - }) - - t.Run("combining multi-level ref filters with AND", func(t *testing.T) { - // In gh-1286 we discovered that on this query the checksum was missing and - // we somehow didn't perform a merge, but rather always took the first set - // of ids - - filter := filterAirportsInGermanCitiesOver600k() - res, err := repo.Search(context.Background(), - getParamsWithFilter("Airport", filter)) - require.Nil(t, err) - - expectedCodes := []string{"40000"} - actualCodes := extractCodes(res) - - assert.Equal(t, expectedCodes, actualCodes) - }) -} - -func filterCarParkedAtGarage(dataType schema.DataType, - prop string, operator filters.Operator, value interface{}, -) *filters.LocalFilter { - return &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: operator, - On: &filters.Path{ - Class: schema.ClassName("MultiRefCar"), - Property: schema.PropertyName("parkedAt"), - Child: &filters.Path{ - Class: schema.ClassName("MultiRefParkingGarage"), - Property: schema.PropertyName(prop), - }, - }, - Value: &filters.Value{ - Value: value, - Type: dataType, - }, - }, - } -} - -func filterCarParkedCount(operator filters.Operator, value int) *filters.LocalFilter { - return &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: operator, - On: &filters.Path{ - Class: schema.ClassName("MultiRefCar"), - Property: schema.PropertyName("parkedAt"), - }, - Value: &filters.Value{ - Value: value, - Type: schema.DataTypeInt, - }, - }, - } -} - -func filterDrivesCarParkedAtGarage(dataType schema.DataType, - prop string, operator filters.Operator, value interface{}, -) *filters.LocalFilter { - return &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: operator, - On: &filters.Path{ - Class: schema.ClassName("MultiRefDriver"), - Property: schema.PropertyName("drives"), - Child: filterCarParkedAtGarage(dataType, prop, operator, value).Root.On, - }, - Value: &filters.Value{ - Value: value, - Type: dataType, - }, - }, - } -} - -func filterAirportsInGermanCitiesOver600k() *filters.LocalFilter { - return &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: and, - Operands: []filters.Clause{ - { - Operator: gt, - On: &filters.Path{ - Class: schema.ClassName("Airport"), - Property: schema.PropertyName("inCity"), - Child: &filters.Path{ - Class: schema.ClassName("City"), - Property: schema.PropertyName("population"), - }, - }, - Value: &filters.Value{ - Value: 600000, - Type: dtInt, - }, - }, - { - Operator: eq, - On: &filters.Path{ - Class: schema.ClassName("Airport"), - Property: schema.PropertyName("inCity"), - Child: &filters.Path{ - Class: schema.ClassName("City"), - Property: schema.PropertyName("inCountry"), - Child: &filters.Path{ - Class: schema.ClassName("Country"), - Property: schema.PropertyName("name"), - }, - }, - }, - Value: &filters.Value{ - Value: "Germany", - Type: schema.DataTypeText, - }, - }, - }, - }, - } -} - -func getParamsWithFilter(className string, filter *filters.LocalFilter) dto.GetParams { - return dto.GetParams{ - Filters: filter, - // we don't care about actually resolving the ref as long as filtering - // on it worked - Properties: nil, - Pagination: &filters.Pagination{ - Offset: 0, - Limit: 10, - }, - ClassName: className, - } -} - -func extractNames(in []search.Result) []string { - out := make([]string, len(in)) - for i, res := range in { - out[i] = res.Schema.(map[string]interface{})["name"].(string) - } - - return out -} - -func extractCodes(in []search.Result) []string { - out := make([]string, len(in)) - for i, res := range in { - out[i] = res.Schema.(map[string]interface{})["code"].(string) - } - - return out -} diff --git a/adapters/repos/db/group_merger.go b/adapters/repos/db/group_merger.go deleted file mode 100644 index d42df94007d76ceb708fc0af5150185addd068c7..0000000000000000000000000000000000000000 --- a/adapters/repos/db/group_merger.go +++ /dev/null @@ -1,122 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "fmt" - "sort" - - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/searchparams" - "github.com/weaviate/weaviate/entities/storobj" -) - -type groupMerger struct { - objects []*storobj.Object - dists []float32 - groupBy *searchparams.GroupBy -} - -func newGroupMerger(objects []*storobj.Object, dists []float32, - groupBy *searchparams.GroupBy, -) *groupMerger { - return &groupMerger{objects, dists, groupBy} -} - -func (gm *groupMerger) Do() ([]*storobj.Object, []float32, error) { - groups := map[string][]*additional.Group{} - objects := map[string][]int{} - - for i, obj := range gm.objects { - g, ok := obj.AdditionalProperties()["group"] - if !ok { - return nil, nil, fmt.Errorf("group not found for object: %v", obj.ID()) - } - group, ok := g.(*additional.Group) - if !ok { - return nil, nil, fmt.Errorf("wrong group type for object: %v", obj.ID()) - } - groups[group.GroupedBy.Value] = append(groups[group.GroupedBy.Value], group) - objects[group.GroupedBy.Value] = append(objects[group.GroupedBy.Value], i) - } - - getMinDistance := func(groups []*additional.Group) float32 { - min := groups[0].MinDistance - for i := range groups { - if groups[i].MinDistance < min { - min = groups[i].MinDistance - } - } - return min - } - - type groupMinDistance struct { - value string - distance float32 - } - - groupDistances := []groupMinDistance{} - for val, group := range groups { - groupDistances = append(groupDistances, groupMinDistance{ - value: val, distance: getMinDistance(group), - }) - } - - sort.Slice(groupDistances, func(i, j int) bool { - return groupDistances[i].distance < groupDistances[j].distance - }) - - desiredLength := len(groups) - if desiredLength > gm.groupBy.Groups { - desiredLength = gm.groupBy.Groups - } - - objs := make([]*storobj.Object, desiredLength) - dists := make([]float32, desiredLength) - for i, groupDistance := range groupDistances[:desiredLength] { - val := groupDistance.value - group := groups[groupDistance.value] - count := 0 - hits := []map[string]interface{}{} - for _, g := range group { - count += g.Count - hits = append(hits, g.Hits...) - } - - sort.Slice(hits, func(i, j int) bool { - return hits[i]["_additional"].(*additional.GroupHitAdditional).Distance < - hits[j]["_additional"].(*additional.GroupHitAdditional).Distance - }) - - if len(hits) > gm.groupBy.ObjectsPerGroup { - hits = hits[:gm.groupBy.ObjectsPerGroup] - count = len(hits) - } - - indx := objects[val][0] - obj, dist := gm.objects[indx], gm.dists[indx] - obj.AdditionalProperties()["group"] = &additional.Group{ - ID: i, - GroupedBy: &additional.GroupedBy{ - Value: val, - Path: []string{gm.groupBy.Property}, - }, - Count: count, - Hits: hits, - MaxDistance: hits[0]["_additional"].(*additional.GroupHitAdditional).Distance, - MinDistance: hits[len(hits)-1]["_additional"].(*additional.GroupHitAdditional).Distance, - } - objs[i], dists[i] = obj, dist - } - - return objs, dists, nil -} diff --git a/adapters/repos/db/helper_for_test.go b/adapters/repos/db/helper_for_test.go deleted file mode 100644 index 7f01756cd7c19e359e49102e897843d13d6277d1..0000000000000000000000000000000000000000 --- a/adapters/repos/db/helper_for_test.go +++ /dev/null @@ -1,307 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package db - -import ( - "context" - "math/rand" - "testing" - "time" - - "github.com/go-openapi/strfmt" - "github.com/google/uuid" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/adapters/repos/db/indexcheckpoint" - "github.com/weaviate/weaviate/adapters/repos/db/inverted" - "github.com/weaviate/weaviate/adapters/repos/db/inverted/stopwords" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/storobj" - enthnsw "github.com/weaviate/weaviate/entities/vectorindex/hnsw" -) - -func parkingGaragesSchema() schema.Schema { - return schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{ - { - Class: "MultiRefParkingGarage", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "location", - DataType: []string{string(schema.DataTypeGeoCoordinates)}, - }, - }, - }, - { - Class: "MultiRefParkingLot", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - }, - { - Class: "MultiRefCar", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "parkedAt", - DataType: []string{"MultiRefParkingGarage", "MultiRefParkingLot"}, - }, - }, - }, - { - Class: "MultiRefDriver", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "drives", - DataType: []string{"MultiRefCar"}, - }, - }, - }, - { - Class: "MultiRefPerson", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "friendsWith", - DataType: []string{"MultiRefDriver"}, - }, - }, - }, - { - Class: "MultiRefSociety", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "hasMembers", - DataType: []string{"MultiRefPerson"}, - }, - }, - }, - - // for classifications test - { - Class: "ExactCategory", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - }, - { - Class: "MainCategory", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - }, - }, - }, - } -} - -func cityCountryAirportSchema() schema.Schema { - return schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{ - { - Class: "Country", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{ - {Name: "name", DataType: schema.DataTypeText.PropString(), Tokenization: models.PropertyTokenizationWhitespace}, - }, - }, - { - Class: "City", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{ - {Name: "name", DataType: schema.DataTypeText.PropString(), Tokenization: models.PropertyTokenizationWhitespace}, - {Name: "inCountry", DataType: []string{"Country"}}, - {Name: "population", DataType: []string{"int"}}, - {Name: "location", DataType: []string{"geoCoordinates"}}, - }, - }, - { - Class: "Airport", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{ - {Name: "code", DataType: schema.DataTypeText.PropString(), Tokenization: models.PropertyTokenizationWhitespace}, - {Name: "phone", DataType: []string{"phoneNumber"}}, - {Name: "inCity", DataType: []string{"City"}}, - }, - }, - }, - }, - } -} - -func testCtx() context.Context { - //nolint:govet - ctx, _ := context.WithTimeout(context.Background(), 30*time.Second) - return ctx -} - -func getRandomSeed() *rand.Rand { - return rand.New(rand.NewSource(time.Now().UnixNano())) -} - -func testShard(t *testing.T, ctx context.Context, className string, indexOpts ...func(*Index)) (ShardLike, *Index) { - return testShardWithSettings(t, ctx, &models.Class{Class: className}, enthnsw.UserConfig{Skip: true}, - false, false, indexOpts...) -} - -func testShardWithSettings(t *testing.T, ctx context.Context, class *models.Class, - vic schema.VectorIndexConfig, withStopwords, withCheckpoints bool, indexOpts ...func(*Index), -) (ShardLike, *Index) { - tmpDir := t.TempDir() - logger, _ := test.NewNullLogger() - maxResults := int64(10_000) - - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: tmpDir, - QueryMaximumResults: maxResults, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - - shardState := singleShardState() - sch := schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{class}, - }, - } - schemaGetter := &fakeSchemaGetter{shardState: shardState, schema: sch} - - iic := schema.InvertedIndexConfig{} - if class.InvertedIndexConfig != nil { - iic = inverted.ConfigFromModel(class.InvertedIndexConfig) - } - var sd *stopwords.Detector - if withStopwords { - sd, err = stopwords.NewDetectorFromConfig(iic.Stopwords) - require.NoError(t, err) - } - var checkpts *indexcheckpoint.Checkpoints - if withCheckpoints { - checkpts, err = indexcheckpoint.New(tmpDir, logger) - require.NoError(t, err) - } - - idx := &Index{ - Config: IndexConfig{ - RootPath: tmpDir, - ClassName: schema.ClassName(class.Class), - QueryMaximumResults: maxResults, - }, - invertedIndexConfig: iic, - vectorIndexUserConfig: vic, - logger: logger, - getSchema: schemaGetter, - centralJobQueue: repo.jobQueueCh, - stopwords: sd, - indexCheckpoints: checkpts, - } - idx.closingCtx, idx.closingCancel = context.WithCancel(context.Background()) - idx.initCycleCallbacksNoop() - for _, opt := range indexOpts { - opt(idx) - } - - shardName := shardState.AllPhysicalShards()[0] - shard, err := idx.initShard(ctx, shardName, class, nil) - require.NoError(t, err) - - idx.shards.Store(shardName, shard) - return shard, idx -} - -func testObject(className string) *storobj.Object { - return &storobj.Object{ - MarshallerVersion: 1, - Object: models.Object{ - ID: strfmt.UUID(uuid.NewString()), - Class: className, - }, - Vector: []float32{1, 2, 3}, - } -} - -func createRandomObjects(r *rand.Rand, className string, numObj int) []*storobj.Object { - obj := make([]*storobj.Object, numObj) - - for i := 0; i < numObj; i++ { - obj[i] = &storobj.Object{ - MarshallerVersion: 1, - Object: models.Object{ - ID: strfmt.UUID(uuid.NewString()), - Class: className, - }, - Vector: []float32{r.Float32(), r.Float32(), r.Float32(), r.Float32()}, - } - } - return obj -} diff --git a/adapters/repos/db/helpers/allow_list.go b/adapters/repos/db/helpers/allow_list.go deleted file mode 100644 index db8f9bbc8e8c0a90644ce5198548294f142f4d5c..0000000000000000000000000000000000000000 --- a/adapters/repos/db/helpers/allow_list.go +++ /dev/null @@ -1,130 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package helpers - -import ( - "github.com/weaviate/sroar" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv/roaringset" -) - -type AllowList interface { - Insert(ids ...uint64) - Contains(id uint64) bool - DeepCopy() AllowList - Slice() []uint64 - - IsEmpty() bool - Len() int - Min() uint64 - Max() uint64 - Size() uint64 - - Iterator() AllowListIterator - LimitedIterator(limit int) AllowListIterator -} - -type AllowListIterator interface { - Next() (uint64, bool) - Len() int -} - -func NewAllowList(ids ...uint64) AllowList { - return NewAllowListFromBitmap(roaringset.NewBitmap(ids...)) -} - -func NewAllowListFromBitmap(bm *sroar.Bitmap) AllowList { - return &bitmapAllowList{bm: bm} -} - -func NewAllowListFromBitmapDeepCopy(bm *sroar.Bitmap) AllowList { - return NewAllowListFromBitmap(bm.Clone()) -} - -type bitmapAllowList struct { - bm *sroar.Bitmap -} - -func (al *bitmapAllowList) Insert(ids ...uint64) { - al.bm.SetMany(ids) -} - -func (al *bitmapAllowList) Contains(id uint64) bool { - return al.bm.Contains(id) -} - -func (al *bitmapAllowList) DeepCopy() AllowList { - return NewAllowListFromBitmapDeepCopy(al.bm) -} - -func (al *bitmapAllowList) Slice() []uint64 { - return al.bm.ToArray() -} - -func (al *bitmapAllowList) IsEmpty() bool { - return al.bm.IsEmpty() -} - -func (al *bitmapAllowList) Len() int { - return al.bm.GetCardinality() -} - -func (al *bitmapAllowList) Min() uint64 { - return al.bm.Minimum() -} - -func (al *bitmapAllowList) Max() uint64 { - return al.bm.Maximum() -} - -func (al *bitmapAllowList) Size() uint64 { - // TODO provide better size estimation - return uint64(1.5 * float64(len(al.bm.ToBuffer()))) -} - -func (al *bitmapAllowList) Iterator() AllowListIterator { - return al.LimitedIterator(0) -} - -func (al *bitmapAllowList) LimitedIterator(limit int) AllowListIterator { - return newBitmapAllowListIterator(al.bm, limit) -} - -type bitmapAllowListIterator struct { - len int - counter int - it *sroar.Iterator -} - -func newBitmapAllowListIterator(bm *sroar.Bitmap, limit int) AllowListIterator { - len := bm.GetCardinality() - if limit > 0 && limit < len { - len = limit - } - - return &bitmapAllowListIterator{ - len: len, - counter: 0, - it: bm.NewIterator(), - } -} - -func (i *bitmapAllowListIterator) Next() (uint64, bool) { - if i.counter >= i.len { - return 0, false - } - i.counter++ - return i.it.Next(), true -} - -func (i *bitmapAllowListIterator) Len() int { - return i.len -} diff --git a/adapters/repos/db/helpers/allow_list_test.go b/adapters/repos/db/helpers/allow_list_test.go deleted file mode 100644 index 9ee834f051532711e6046ea3d56bba25adae77b4..0000000000000000000000000000000000000000 --- a/adapters/repos/db/helpers/allow_list_test.go +++ /dev/null @@ -1,284 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package helpers - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv/roaringset" -) - -func TestAllowList(t *testing.T) { - t.Run("allowlist created with no values", func(t *testing.T) { - al := NewAllowList() - - assert.Equal(t, 0, al.Len()) - assert.True(t, al.IsEmpty()) - - assert.Equal(t, uint64(0), al.Min()) - assert.Equal(t, uint64(0), al.Max()) - }) - - t.Run("allowlist created with initial values", func(t *testing.T) { - al := NewAllowList(1, 2, 3) - - assert.Equal(t, 3, al.Len()) - assert.False(t, al.IsEmpty()) - - assert.True(t, al.Contains(1)) - assert.True(t, al.Contains(2)) - assert.True(t, al.Contains(3)) - - assert.Equal(t, uint64(1), al.Min()) - assert.Equal(t, uint64(3), al.Max()) - }) - - t.Run("allowlist with inserted values", func(t *testing.T) { - al := NewAllowList(1, 2, 3) - al.Insert(4, 5) - - assert.Equal(t, 5, al.Len()) - assert.False(t, al.IsEmpty()) - - assert.True(t, al.Contains(1)) - assert.True(t, al.Contains(2)) - assert.True(t, al.Contains(3)) - assert.True(t, al.Contains(4)) - assert.True(t, al.Contains(5)) - - assert.Equal(t, uint64(1), al.Min()) - assert.Equal(t, uint64(5), al.Max()) - }) - - t.Run("allowlist exported to slice", func(t *testing.T) { - al := NewAllowList(1, 2, 3) - al.Insert(4, 5) - - assert.ElementsMatch(t, []uint64{1, 2, 3, 4, 5}, al.Slice()) - }) - - t.Run("allowlist deepcopy", func(t *testing.T) { - al := NewAllowList(1, 2, 3) - copy := al.DeepCopy() - al.Insert(4, 5) - - assert.Equal(t, 5, al.Len()) - assert.False(t, al.IsEmpty()) - - assert.True(t, al.Contains(1)) - assert.True(t, al.Contains(2)) - assert.True(t, al.Contains(3)) - assert.True(t, al.Contains(4)) - assert.True(t, al.Contains(5)) - - assert.Equal(t, uint64(1), al.Min()) - assert.Equal(t, uint64(5), al.Max()) - - assert.Equal(t, 3, copy.Len()) - assert.False(t, copy.IsEmpty()) - - assert.True(t, copy.Contains(1)) - assert.True(t, copy.Contains(2)) - assert.True(t, copy.Contains(3)) - - assert.Equal(t, uint64(1), copy.Min()) - assert.Equal(t, uint64(3), copy.Max()) - }) - - t.Run("allowlist created from bitmap", func(t *testing.T) { - bm := roaringset.NewBitmap(1, 2, 3) - - al := NewAllowListFromBitmap(bm) - bm.SetMany([]uint64{4, 5}) - - assert.Equal(t, 5, al.Len()) - assert.False(t, al.IsEmpty()) - - assert.True(t, al.Contains(1)) - assert.True(t, al.Contains(2)) - assert.True(t, al.Contains(3)) - assert.True(t, al.Contains(4)) - assert.True(t, al.Contains(5)) - - assert.Equal(t, uint64(1), al.Min()) - assert.Equal(t, uint64(5), al.Max()) - }) - - t.Run("allowlist created from bitmap deepcopy", func(t *testing.T) { - bm := roaringset.NewBitmap(1, 2, 3) - - al := NewAllowListFromBitmapDeepCopy(bm) - bm.SetMany([]uint64{4, 5}) - - assert.Equal(t, 3, al.Len()) - assert.False(t, al.IsEmpty()) - - assert.True(t, al.Contains(1)) - assert.True(t, al.Contains(2)) - assert.True(t, al.Contains(3)) - - assert.Equal(t, uint64(1), al.Min()) - assert.Equal(t, uint64(3), al.Max()) - }) -} - -func TestAllowList_Iterator(t *testing.T) { - t.Run("empty bitmap iterator", func(t *testing.T) { - it := NewAllowList().Iterator() - - id1, ok1 := it.Next() - id2, ok2 := it.Next() - - assert.Equal(t, 0, it.Len()) - assert.False(t, ok1) - assert.Equal(t, uint64(0), id1) - assert.False(t, ok2) - assert.Equal(t, uint64(0), id2) - }) - - t.Run("iterating step by step", func(t *testing.T) { - it := NewAllowList(3, 2, 1).Iterator() - - id1, ok1 := it.Next() - id2, ok2 := it.Next() - id3, ok3 := it.Next() - id4, ok4 := it.Next() - - assert.Equal(t, 3, it.Len()) - assert.True(t, ok1) - assert.Equal(t, uint64(1), id1) - assert.True(t, ok2) - assert.Equal(t, uint64(2), id2) - assert.True(t, ok3) - assert.Equal(t, uint64(3), id3) - assert.False(t, ok4) - assert.Equal(t, uint64(0), id4) - }) - - t.Run("iterating in loop", func(t *testing.T) { - it := NewAllowList(3, 2, 1).Iterator() - ids := []uint64{} - - for id, ok := it.Next(); ok; id, ok = it.Next() { - ids = append(ids, id) - } - - assert.Equal(t, 3, it.Len()) - assert.Equal(t, []uint64{1, 2, 3}, ids) - }) -} - -func TestAllowList_LimitedIterator(t *testing.T) { - t.Run("empty bitmap iterator", func(t *testing.T) { - it := NewAllowList().LimitedIterator(2) - - id1, ok1 := it.Next() - id2, ok2 := it.Next() - - assert.Equal(t, 0, it.Len()) - assert.False(t, ok1) - assert.Equal(t, uint64(0), id1) - assert.False(t, ok2) - assert.Equal(t, uint64(0), id2) - }) - - t.Run("iterating step by step (higher limit)", func(t *testing.T) { - it := NewAllowList(3, 2, 1).LimitedIterator(4) - - id1, ok1 := it.Next() - id2, ok2 := it.Next() - id3, ok3 := it.Next() - id4, ok4 := it.Next() - - assert.Equal(t, 3, it.Len()) - assert.True(t, ok1) - assert.Equal(t, uint64(1), id1) - assert.True(t, ok2) - assert.Equal(t, uint64(2), id2) - assert.True(t, ok3) - assert.Equal(t, uint64(3), id3) - assert.False(t, ok4) - assert.Equal(t, uint64(0), id4) - }) - - t.Run("iterating step by step (equal limit)", func(t *testing.T) { - it := NewAllowList(3, 2, 1).LimitedIterator(3) - - id1, ok1 := it.Next() - id2, ok2 := it.Next() - id3, ok3 := it.Next() - id4, ok4 := it.Next() - - assert.Equal(t, 3, it.Len()) - assert.True(t, ok1) - assert.Equal(t, uint64(1), id1) - assert.True(t, ok2) - assert.Equal(t, uint64(2), id2) - assert.True(t, ok3) - assert.Equal(t, uint64(3), id3) - assert.False(t, ok4) - assert.Equal(t, uint64(0), id4) - }) - - t.Run("iterating step by step (lower limit)", func(t *testing.T) { - it := NewAllowList(3, 2, 1).LimitedIterator(2) - - id1, ok1 := it.Next() - id2, ok2 := it.Next() - id3, ok3 := it.Next() - - assert.Equal(t, 2, it.Len()) - assert.True(t, ok1) - assert.Equal(t, uint64(1), id1) - assert.True(t, ok2) - assert.Equal(t, uint64(2), id2) - assert.False(t, ok3) - assert.Equal(t, uint64(0), id3) - }) - - t.Run("iterating in loop (higher limit)", func(t *testing.T) { - it := NewAllowList(3, 2, 1).LimitedIterator(4) - ids := []uint64{} - - for id, ok := it.Next(); ok; id, ok = it.Next() { - ids = append(ids, id) - } - - assert.Equal(t, 3, it.Len()) - assert.Equal(t, []uint64{1, 2, 3}, ids) - }) - - t.Run("iterating in loop (equal limit)", func(t *testing.T) { - it := NewAllowList(3, 2, 1).LimitedIterator(3) - ids := []uint64{} - - for id, ok := it.Next(); ok; id, ok = it.Next() { - ids = append(ids, id) - } - - assert.Equal(t, 3, it.Len()) - assert.Equal(t, []uint64{1, 2, 3}, ids) - }) - - t.Run("iterating in loop (lower limit)", func(t *testing.T) { - it := NewAllowList(3, 2, 1).LimitedIterator(2) - ids := []uint64{} - - for id, ok := it.Next(); ok; id, ok = it.Next() { - ids = append(ids, id) - } - - assert.Equal(t, 2, it.Len()) - assert.Equal(t, []uint64{1, 2}, ids) - }) -} diff --git a/adapters/repos/db/helpers/helpers.go b/adapters/repos/db/helpers/helpers.go deleted file mode 100644 index 1d706befb59ef233c776deef0f04966ec6170b32..0000000000000000000000000000000000000000 --- a/adapters/repos/db/helpers/helpers.go +++ /dev/null @@ -1,67 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package helpers - -import ( - "fmt" - - "github.com/weaviate/weaviate/entities/filters" -) - -var ( - ObjectsBucket = []byte("objects") - ObjectsBucketLSM = "objects" - VectorsCompressedBucketLSM = "vectors_compressed" - VectorsBucketLSM = "vectors" - DimensionsBucketLSM = "dimensions" -) - -// MetaCountProp helps create an internally used propName for meta props that -// don't explicitly exist in the user schema, but are required for proper -// indexing, such as the count of arrays. -func MetaCountProp(propName string) string { - return fmt.Sprintf("%s__meta_count", propName) -} - -func PropLength(propName string) string { - return propName + filters.InternalPropertyLength -} - -func PropNull(propName string) string { - return propName + filters.InternalNullIndex -} - -// BucketFromPropNameLSM creates string used as the bucket name -// for a particular prop in the inverted index -func BucketFromPropNameLSM(propName string) string { - return fmt.Sprintf("property_%s", propName) -} - -func BucketFromPropNameLengthLSM(propName string) string { - return BucketFromPropNameLSM(PropLength(propName)) -} - -func BucketFromPropNameNullLSM(propName string) string { - return BucketFromPropNameLSM(PropNull(propName)) -} - -func BucketFromPropNameMetaCountLSM(propName string) string { - return BucketFromPropNameLSM(MetaCountProp(propName)) -} - -func TempBucketFromBucketName(bucketName string) string { - return bucketName + "_temp" -} - -func BucketSearchableFromPropNameLSM(propName string) string { - return BucketFromPropNameLSM(propName + "_searchable") -} diff --git a/adapters/repos/db/helpers/tokenizer.go b/adapters/repos/db/helpers/tokenizer.go deleted file mode 100644 index 1caff3e86afcc20a7a6e02994850476b153b3e79..0000000000000000000000000000000000000000 --- a/adapters/repos/db/helpers/tokenizer.go +++ /dev/null @@ -1,118 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package helpers - -import ( - "strings" - "unicode" - - "github.com/weaviate/weaviate/entities/models" -) - -var Tokenizations []string = []string{ - models.PropertyTokenizationWord, - models.PropertyTokenizationLowercase, - models.PropertyTokenizationWhitespace, - models.PropertyTokenizationField, -} - -func Tokenize(tokenization string, in string) []string { - switch tokenization { - case models.PropertyTokenizationWord: - return tokenizeWord(in) - case models.PropertyTokenizationLowercase: - return tokenizeLowercase(in) - case models.PropertyTokenizationWhitespace: - return tokenizeWhitespace(in) - case models.PropertyTokenizationField: - return tokenizeField(in) - default: - return []string{} - } -} - -func TokenizeWithWildcards(tokenization string, in string) []string { - switch tokenization { - case models.PropertyTokenizationWord: - return tokenizeWordWithWildcards(in) - case models.PropertyTokenizationLowercase: - return tokenizeLowercase(in) - case models.PropertyTokenizationWhitespace: - return tokenizeWhitespace(in) - case models.PropertyTokenizationField: - return tokenizeField(in) - default: - return []string{} - } -} - -// tokenizeField trims white spaces -// (former DataTypeString/Field) -func tokenizeField(in string) []string { - return []string{strings.TrimFunc(in, unicode.IsSpace)} -} - -// tokenizeWhitespace splits on white spaces, does not alter casing -// (former DataTypeString/Word) -func tokenizeWhitespace(in string) []string { - return strings.FieldsFunc(in, unicode.IsSpace) -} - -// tokenizeLowercase splits on white spaces and lowercases the words -func tokenizeLowercase(in string) []string { - terms := tokenizeWhitespace(in) - return lowercase(terms) -} - -// tokenizeWord splits on any non-alphanumerical and lowercases the words -// (former DataTypeText/Word) -func tokenizeWord(in string) []string { - terms := strings.FieldsFunc(in, func(r rune) bool { - return !unicode.IsLetter(r) && !unicode.IsNumber(r) - }) - return lowercase(terms) -} - -// tokenizeWordWithWildcards splits on any non-alphanumerical except wildcard-symbols and -// lowercases the words -func tokenizeWordWithWildcards(in string) []string { - terms := strings.FieldsFunc(in, func(r rune) bool { - return !unicode.IsLetter(r) && !unicode.IsNumber(r) && r != '?' && r != '*' - }) - return lowercase(terms) -} - -func lowercase(terms []string) []string { - for i := range terms { - terms[i] = strings.ToLower(terms[i]) - } - return terms -} - -func TokenizeAndCountDuplicates(tokenization string, in string) ([]string, []int) { - counts := map[string]int{} - for _, term := range Tokenize(tokenization, in) { - counts[term]++ - } - - unique := make([]string, len(counts)) - boosts := make([]int, len(counts)) - - i := 0 - for term, boost := range counts { - unique[i] = term - boosts[i] = boost - i++ - } - - return unique, boosts -} diff --git a/adapters/repos/db/helpers/tokenizer_test.go b/adapters/repos/db/helpers/tokenizer_test.go deleted file mode 100644 index dc327bf64f7eb89db88132d075eedee889fb720a..0000000000000000000000000000000000000000 --- a/adapters/repos/db/helpers/tokenizer_test.go +++ /dev/null @@ -1,143 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package helpers - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/entities/models" -) - -func TestTokenize(t *testing.T) { - input := " Hello You*-beautiful_world?!" - - type testCase struct { - tokenization string - expected []string - } - - t.Run("tokenize", func(t *testing.T) { - testCases := []testCase{ - { - tokenization: models.PropertyTokenizationField, - expected: []string{"Hello You*-beautiful_world?!"}, - }, - { - tokenization: models.PropertyTokenizationWhitespace, - expected: []string{"Hello", "You*-beautiful_world?!"}, - }, - { - tokenization: models.PropertyTokenizationLowercase, - expected: []string{"hello", "you*-beautiful_world?!"}, - }, - { - tokenization: models.PropertyTokenizationWord, - expected: []string{"hello", "you", "beautiful", "world"}, - }, - } - - for _, tc := range testCases { - terms := Tokenize(tc.tokenization, input) - assert.ElementsMatch(t, tc.expected, terms) - } - }) - - t.Run("tokenize with wildcards", func(t *testing.T) { - testCases := []testCase{ - { - tokenization: models.PropertyTokenizationField, - expected: []string{"Hello You*-beautiful_world?!"}, - }, - { - tokenization: models.PropertyTokenizationWhitespace, - expected: []string{"Hello", "You*-beautiful_world?!"}, - }, - { - tokenization: models.PropertyTokenizationLowercase, - expected: []string{"hello", "you*-beautiful_world?!"}, - }, - { - tokenization: models.PropertyTokenizationWord, - expected: []string{"hello", "you*", "beautiful", "world?"}, - }, - } - - for _, tc := range testCases { - terms := TokenizeWithWildcards(tc.tokenization, input) - assert.ElementsMatch(t, tc.expected, terms) - } - }) -} - -func TestTokenizeAndCountDuplicates(t *testing.T) { - input := "Hello You Beautiful World! hello you beautiful world!" - - type testCase struct { - tokenization string - expected map[string]int - } - - testCases := []testCase{ - { - tokenization: models.PropertyTokenizationField, - expected: map[string]int{ - "Hello You Beautiful World! hello you beautiful world!": 1, - }, - }, - { - tokenization: models.PropertyTokenizationWhitespace, - expected: map[string]int{ - "Hello": 1, - "You": 1, - "Beautiful": 1, - "World!": 1, - "hello": 1, - "you": 1, - "beautiful": 1, - "world!": 1, - }, - }, - { - tokenization: models.PropertyTokenizationLowercase, - expected: map[string]int{ - "hello": 2, - "you": 2, - "beautiful": 2, - "world!": 2, - }, - }, - { - tokenization: models.PropertyTokenizationWord, - expected: map[string]int{ - "hello": 2, - "you": 2, - "beautiful": 2, - "world": 2, - }, - }, - } - - for _, tc := range testCases { - t.Run(tc.tokenization, func(t *testing.T) { - terms, dups := TokenizeAndCountDuplicates(tc.tokenization, input) - - assert.Len(t, terms, len(tc.expected)) - assert.Len(t, dups, len(tc.expected)) - - for i := range terms { - assert.Contains(t, tc.expected, terms[i]) - assert.Equal(t, tc.expected[terms[i]], dups[i]) - } - }) - } -} diff --git a/adapters/repos/db/index.go b/adapters/repos/db/index.go deleted file mode 100644 index 413905320e13be38a98389d6f604a32965e44fa6..0000000000000000000000000000000000000000 --- a/adapters/repos/db/index.go +++ /dev/null @@ -1,2039 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "context" - "fmt" - "os" - "path" - "runtime" - "runtime/debug" - golangSort "sort" - "strings" - "sync" - "sync/atomic" - "time" - - "github.com/go-openapi/strfmt" - "github.com/google/uuid" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/adapters/repos/db/aggregator" - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/adapters/repos/db/indexcheckpoint" - "github.com/weaviate/weaviate/adapters/repos/db/inverted" - "github.com/weaviate/weaviate/adapters/repos/db/inverted/stopwords" - "github.com/weaviate/weaviate/adapters/repos/db/sorter" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/aggregation" - "github.com/weaviate/weaviate/entities/autocut" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/multi" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/search" - "github.com/weaviate/weaviate/entities/searchparams" - "github.com/weaviate/weaviate/entities/storobj" - "github.com/weaviate/weaviate/usecases/config" - "github.com/weaviate/weaviate/usecases/monitoring" - "github.com/weaviate/weaviate/usecases/objects" - "github.com/weaviate/weaviate/usecases/replica" - schemaUC "github.com/weaviate/weaviate/usecases/schema" - "github.com/weaviate/weaviate/usecases/sharding" - "golang.org/x/sync/errgroup" -) - -var ( - errTenantNotFound = errors.New("tenant not found") - errTenantNotActive = errors.New("tenant not active") - - // Use runtime.GOMAXPROCS instead of runtime.NumCPU because NumCPU returns - // the physical CPU cores. However, in a containerization context, that might - // not be what we want. The physical node could have 128 cores, but we could - // be cgroup-limited to 2 cores. In that case, we want 2 to be our limit, not - // 128. It isn't guaranteed that MAXPROCS reflects the cgroup limit, but at - // least there is a chance that it was set correctly. If not, it defaults to - // NumCPU anyway, so we're not any worse off. - _NUMCPU = runtime.GOMAXPROCS(0) - errShardNotFound = errors.New("shard not found") -) - -// shardMap is a syn.Map which specialized in storing shards -type shardMap sync.Map - -// Range calls f sequentially for each key and value present in the map. -// If f returns an error, range stops the iteration -func (m *shardMap) Range(f func(name string, shard ShardLike) error) (err error) { - (*sync.Map)(m).Range(func(key, value any) bool { - err = f(key.(string), value.(ShardLike)) - return err == nil - }) - return err -} - -// RangeConcurrently calls f for each key and value present in the map with at -// most _NUMCPU executors running in parallel. As opposed to [Range] it does -// not guarantee an exit on the first error. -func (m *shardMap) RangeConcurrently(f func(name string, shard ShardLike) error) (err error) { - eg := errgroup.Group{} - eg.SetLimit(_NUMCPU) - (*sync.Map)(m).Range(func(key, value any) bool { - name, shard := key.(string), value.(ShardLike) - eg.Go(func() error { - return f(name, shard) - }) - return true - }) - - return eg.Wait() -} - -// Load returns the shard or nil if no shard is present. -func (m *shardMap) Load(name string) ShardLike { - v, ok := (*sync.Map)(m).Load(name) - if !ok { - return nil - } - return v.(ShardLike) -} - -// Store sets a shard giving its name and value -func (m *shardMap) Store(name string, shard ShardLike) { - (*sync.Map)(m).Store(name, shard) -} - -// Swap swaps the shard for a key and returns the previous value if any. -// The loaded result reports whether the key was present. -func (m *shardMap) Swap(name string, shard ShardLike) (previous ShardLike, loaded bool) { - v, ok := (*sync.Map)(m).Swap(name, shard) - if v == nil || !ok { - return nil, ok - } - return v.(ShardLike), ok -} - -// CompareAndSwap swaps the old and new values for key if the value stored in the map is equal to old. -func (m *shardMap) CompareAndSwap(name string, old, new ShardLike) bool { - return (*sync.Map)(m).CompareAndSwap(name, old, new) -} - -// LoadAndDelete deletes the value for a key, returning the previous value if any. -// The loaded result reports whether the key was present. -func (m *shardMap) LoadAndDelete(name string) (ShardLike, bool) { - v, ok := (*sync.Map)(m).LoadAndDelete(name) - if v == nil || !ok { - return nil, ok - } - return v.(ShardLike), ok -} - -// Index is the logical unit which contains all the data for one particular -// class. An index can be further broken up into self-contained units, called -// Shards, to allow for easy distribution across Nodes -type Index struct { - classSearcher inverted.ClassSearcher // to allow for nested by-references searches - shards shardMap - Config IndexConfig - vectorIndexUserConfig schema.VectorIndexConfig - vectorIndexUserConfigLock sync.Mutex - getSchema schemaUC.SchemaGetter - logger logrus.FieldLogger - remote *sharding.RemoteIndex - stopwords *stopwords.Detector - replicator *replica.Replicator - - invertedIndexConfig schema.InvertedIndexConfig - invertedIndexConfigLock sync.Mutex - - // This lock should be used together with the db indexLock. - // - // The db indexlock locks the map that contains all indices against changes and should be used while iterating. - // This lock protects this specific index form being deleted while in use. Use Rlock to signal that it is in use. - // This way many goroutines can use a specific index in parallel. The delete-routine will try to acquire a RWlock. - // - // Usage: - // Lock the whole db using db.indexLock - // pick the indices you want and Rlock them - // unlock db.indexLock - // Use the indices - // RUnlock all picked indices - dropIndex sync.RWMutex - - metrics *Metrics - centralJobQueue chan job - indexCheckpoints *indexcheckpoint.Checkpoints - - partitioningEnabled bool - - cycleCallbacks *indexCycleCallbacks - - backupMutex backupMutex - lastBackup atomic.Pointer[BackupState] - - // canceled when either Shutdown or Drop called - closingCtx context.Context - closingCancel context.CancelFunc -} - -func (i *Index) GetShards() []ShardLike { - var out []ShardLike - i.shards.Range(func(_ string, shard ShardLike) error { - out = append(out, shard) - return nil - }) - - return out -} - -func (i *Index) ID() string { - return indexID(i.Config.ClassName) -} - -func (i *Index) path() string { - return path.Join(i.Config.RootPath, i.ID()) -} - -type nodeResolver interface { - NodeHostname(nodeName string) (string, bool) -} - -// NewIndex creates an index with the specified amount of shards, using only -// the shards that are local to a node -func NewIndex(ctx context.Context, cfg IndexConfig, - shardState *sharding.State, invertedIndexConfig schema.InvertedIndexConfig, - vectorIndexUserConfig schema.VectorIndexConfig, sg schemaUC.SchemaGetter, - cs inverted.ClassSearcher, logger logrus.FieldLogger, - nodeResolver nodeResolver, remoteClient sharding.RemoteIndexClient, - replicaClient replica.Client, - promMetrics *monitoring.PrometheusMetrics, class *models.Class, jobQueueCh chan job, - indexCheckpoints *indexcheckpoint.Checkpoints, -) (*Index, error) { - sd, err := stopwords.NewDetectorFromConfig(invertedIndexConfig.Stopwords) - if err != nil { - return nil, errors.Wrap(err, "failed to create new index") - } - - repl := replica.NewReplicator(cfg.ClassName.String(), - sg, nodeResolver, replicaClient, logger) - - if cfg.QueryNestedRefLimit == 0 { - cfg.QueryNestedRefLimit = config.DefaultQueryNestedCrossReferenceLimit - } - - index := &Index{ - Config: cfg, - getSchema: sg, - logger: logger, - classSearcher: cs, - vectorIndexUserConfig: vectorIndexUserConfig, - invertedIndexConfig: invertedIndexConfig, - stopwords: sd, - replicator: repl, - remote: sharding.NewRemoteIndex(cfg.ClassName.String(), sg, - nodeResolver, remoteClient), - metrics: NewMetrics(logger, promMetrics, cfg.ClassName.String(), "n/a"), - centralJobQueue: jobQueueCh, - partitioningEnabled: shardState.PartitioningEnabled, - backupMutex: backupMutex{log: logger, retryDuration: mutexRetryDuration, notifyDuration: mutexNotifyDuration}, - indexCheckpoints: indexCheckpoints, - } - index.closingCtx, index.closingCancel = context.WithCancel(context.Background()) - - index.initCycleCallbacks() - - if err := index.checkSingleShardMigration(shardState); err != nil { - return nil, errors.Wrap(err, "migrating sharding state from previous version") - } - - eg := errgroup.Group{} - eg.SetLimit(_NUMCPU) - - if err := os.MkdirAll(index.path(), os.ModePerm); err != nil { - return nil, fmt.Errorf("init index %q: %w", index.ID(), err) - } - - if err := index.initAndStoreShards(ctx, shardState, class, promMetrics); err != nil { - return nil, err - } - - index.cycleCallbacks.compactionCycle.Start() - index.cycleCallbacks.flushCycle.Start() - - return index, nil -} - -func (i *Index) initAndStoreShards(ctx context.Context, shardState *sharding.State, class *models.Class, - promMetrics *monitoring.PrometheusMetrics, -) error { - if i.Config.DisableLazyLoadShards { - eg := errgroup.Group{} - eg.SetLimit(_NUMCPU) - - for _, shardName := range shardState.AllLocalPhysicalShards() { - physical := shardState.Physical[shardName] - if physical.ActivityStatus() != models.TenantActivityStatusHOT { - // do not instantiate inactive shard - continue - } - - shardName := shardName // prevent loop variable capture - eg.Go(func() error { - shard, err := NewShard(ctx, promMetrics, shardName, i, class, i.centralJobQueue, i.indexCheckpoints, nil) - if err != nil { - return fmt.Errorf("init shard %s of index %s: %w", shardName, i.ID(), err) - } - - i.shards.Store(shardName, shard) - return nil - }) - } - - return eg.Wait() - } - - for _, shardName := range shardState.AllLocalPhysicalShards() { - physical := shardState.Physical[shardName] - if physical.ActivityStatus() != models.TenantActivityStatusHOT { - // do not instantiate inactive shard - continue - } - - shard := NewLazyLoadShard(ctx, promMetrics, shardName, i, class, i.centralJobQueue, i.indexCheckpoints) - i.shards.Store(shardName, shard) - } - - go func() { - ticker := time.NewTicker(time.Second) - defer ticker.Stop() - - i.ForEachShard(func(name string, shard ShardLike) error { - // prioritize closingCtx over ticker: - // check closing again in case of ticker was selected when both - // cases where available - select { - case <-i.closingCtx.Done(): - // break loop by returning error - return i.closingCtx.Err() - case <-ticker.C: - select { - case <-i.closingCtx.Done(): - // break loop by returning error - return i.closingCtx.Err() - default: - shard.(*LazyLoadShard).Load(context.Background()) - return nil - } - } - }) - }() - - return nil -} - -func (i *Index) initAndStoreShard(ctx context.Context, shardName string, class *models.Class, - promMetrics *monitoring.PrometheusMetrics, -) error { - shard, err := i.initShard(ctx, shardName, class, promMetrics) - if err != nil { - return err - } - i.shards.Store(shardName, shard) - return nil -} - -func (i *Index) initShard(ctx context.Context, shardName string, class *models.Class, - promMetrics *monitoring.PrometheusMetrics, -) (ShardLike, error) { - if i.Config.DisableLazyLoadShards { - shard, err := NewShard(ctx, promMetrics, shardName, i, class, i.centralJobQueue, i.indexCheckpoints, nil) - if err != nil { - return nil, fmt.Errorf("init shard %s of index %s: %w", shardName, i.ID(), err) - } - return shard, nil - } - - shard := NewLazyLoadShard(ctx, promMetrics, shardName, i, class, i.centralJobQueue, i.indexCheckpoints) - return shard, nil -} - -// Iterate over all objects in the index, applying the callback function to each one. Adding or removing objects during iteration is not supported. -func (i *Index) IterateObjects(ctx context.Context, cb func(index *Index, shard ShardLike, object *storobj.Object) error) (err error) { - return i.ForEachShard(func(_ string, shard ShardLike) error { - wrapper := func(object *storobj.Object) error { - return cb(i, shard, object) - } - bucket := shard.Store().Bucket(helpers.ObjectsBucketLSM) - return bucket.IterateObjects(ctx, wrapper) - }) -} - -func (i *Index) ForEachShard(f func(name string, shard ShardLike) error) error { - return i.shards.Range(f) -} - -func (i *Index) ForEachShardConcurrently(f func(name string, shard ShardLike) error) error { - return i.shards.RangeConcurrently(f) -} - -// Iterate over all objects in the shard, applying the callback function to each one. Adding or removing objects during iteration is not supported. -func (i *Index) IterateShards(ctx context.Context, cb func(index *Index, shard ShardLike) error) (err error) { - return i.ForEachShard(func(key string, shard ShardLike) error { - return cb(i, shard) - }) -} - -func (i *Index) addProperty(ctx context.Context, prop *models.Property) error { - eg := &errgroup.Group{} - eg.SetLimit(_NUMCPU) - - i.ForEachShard(func(key string, shard ShardLike) error { - shard.createPropertyIndex(ctx, prop, eg) - return nil - }) - if err := eg.Wait(); err != nil { - return errors.Wrapf(err, "extend idx '%s' with property '%s", i.ID(), prop.Name) - } - return nil -} - -func (i *Index) addUUIDProperty(ctx context.Context) error { - return i.ForEachShard(func(name string, shard ShardLike) error { - err := shard.addIDProperty(ctx) - if err != nil { - return errors.Wrapf(err, "add id property to shard %q", name) - } - return nil - }) -} - -func (i *Index) addDimensionsProperty(ctx context.Context) error { - return i.ForEachShard(func(name string, shard ShardLike) error { - if err := shard.addDimensionsProperty(ctx); err != nil { - return errors.Wrapf(err, "add dimensions property to shard %q", name) - } - return nil - }) -} - -func (i *Index) addTimestampProperties(ctx context.Context) error { - return i.ForEachShard(func(name string, shard ShardLike) error { - if err := shard.addTimestampProperties(ctx); err != nil { - return errors.Wrapf(err, "add timestamp properties to shard %q", name) - } - return nil - }) -} - -func (i *Index) updateVectorIndexConfig(ctx context.Context, - updated schema.VectorIndexConfig, -) error { - // an updated is not specific to one shard, but rather all - err := i.ForEachShard(func(name string, shard ShardLike) error { - // At the moment, we don't do anything in an update that could fail, but - // technically this should be part of some sort of a two-phase commit or - // have another way to rollback if we have updates that could potentially - // fail in the future. For now that's not a realistic risk. - if err := shard.UpdateVectorIndexConfig(ctx, updated); err != nil { - return errors.Wrapf(err, "shard %s", name) - } - return nil - }) - if err != nil { - return err - } - i.vectorIndexUserConfigLock.Lock() - defer i.vectorIndexUserConfigLock.Unlock() - - i.vectorIndexUserConfig = updated - - return nil -} - -func (i *Index) getInvertedIndexConfig() schema.InvertedIndexConfig { - i.invertedIndexConfigLock.Lock() - defer i.invertedIndexConfigLock.Unlock() - - return i.invertedIndexConfig -} - -func (i *Index) updateInvertedIndexConfig(ctx context.Context, - updated schema.InvertedIndexConfig, -) error { - i.invertedIndexConfigLock.Lock() - defer i.invertedIndexConfigLock.Unlock() - - i.invertedIndexConfig = updated - - return nil -} - -type IndexConfig struct { - RootPath string - ClassName schema.ClassName - QueryMaximumResults int64 - QueryNestedRefLimit int64 - ResourceUsage config.ResourceUsage - MemtablesFlushIdleAfter int - MemtablesInitialSizeMB int - MemtablesMaxSizeMB int - MemtablesMinActiveSeconds int - MemtablesMaxActiveSeconds int - ReplicationFactor int64 - AvoidMMap bool - DisableLazyLoadShards bool - - TrackVectorDimensions bool -} - -func indexID(class schema.ClassName) string { - return strings.ToLower(string(class)) -} - -func (i *Index) determineObjectShard(id strfmt.UUID, tenant string) (string, error) { - className := i.Config.ClassName.String() - if tenant != "" { - if shard, status := i.getSchema.TenantShard(className, tenant); shard != "" { - if status == models.TenantActivityStatusHOT { - return shard, nil - } - return "", objects.NewErrMultiTenancy(fmt.Errorf("%w: '%s'", errTenantNotActive, tenant)) - } - return "", objects.NewErrMultiTenancy(fmt.Errorf("%w: %q", errTenantNotFound, tenant)) - } - - uuid, err := uuid.Parse(id.String()) - if err != nil { - return "", fmt.Errorf("parse uuid: %q", id.String()) - } - - uuidBytes, err := uuid.MarshalBinary() // cannot error - if err != nil { - return "", fmt.Errorf("marshal uuid: %q", id.String()) - } - - return i.getSchema.ShardFromUUID(className, uuidBytes), nil -} - -func (i *Index) putObject(ctx context.Context, object *storobj.Object, - replProps *additional.ReplicationProperties, -) error { - if err := i.validateMultiTenancy(object.Object.Tenant); err != nil { - return err - } - - if i.Config.ClassName != object.Class() { - return fmt.Errorf("cannot import object of class %s into index of class %s", - object.Class(), i.Config.ClassName) - } - - shardName, err := i.determineObjectShard(object.ID(), object.Object.Tenant) - if err != nil { - return objects.NewErrInvalidUserInput("determine shard: %v", err) - } - - if i.replicationEnabled() { - if replProps == nil { - replProps = defaultConsistency() - } - cl := replica.ConsistencyLevel(replProps.ConsistencyLevel) - if err := i.replicator.PutObject(ctx, shardName, object, cl); err != nil { - return fmt.Errorf("replicate insertion: shard=%q: %w", shardName, err) - } - return nil - } - - // no replication, remote shard - if i.localShard(shardName) == nil { - if err := i.remote.PutObject(ctx, shardName, object); err != nil { - return fmt.Errorf("put remote object: shard=%q: %w", shardName, err) - } - return nil - } - - // no replication, local shard - i.backupMutex.RLock() - defer i.backupMutex.RUnlock() - err = errShardNotFound - if shard := i.localShard(shardName); shard != nil { // does shard still exist - err = shard.PutObject(ctx, object) - } - if err != nil { - return fmt.Errorf("put local object: shard=%q: %w", shardName, err) - } - - return nil -} - -func (i *Index) IncomingPutObject(ctx context.Context, shardName string, - object *storobj.Object, -) error { - i.backupMutex.RLock() - defer i.backupMutex.RUnlock() - localShard := i.localShard(shardName) - if localShard == nil { - return errShardNotFound - } - - // This is a bit hacky, the problem here is that storobj.Parse() currently - // misses date fields as it has no way of knowing that a date-formatted - // string was actually a date type. However, adding this functionality to - // Parse() would break a lot of code, because it currently - // schema-independent. To find out if a field is a date or date[], we need to - // involve the schema, thus why we are doing it here. This was discovered as - // part of https://github.com/weaviate/weaviate/issues/1775 - if err := i.parseDateFieldsInProps(object.Object.Properties); err != nil { - return err - } - - if err := localShard.PutObject(ctx, object); err != nil { - return err - } - - return nil -} - -func (i *Index) replicationEnabled() bool { - return i.Config.ReplicationFactor > 1 -} - -// parseDateFieldsInProps checks the schema for the current class for which -// fields are date fields, then - if they are set - parses them accordingly. -// Works for both date and date[]. -func (i *Index) parseDateFieldsInProps(props interface{}) error { - if props == nil { - return nil - } - - propMap, ok := props.(map[string]interface{}) - if !ok { - // don't know what to do with this - return nil - } - - schemaModel := i.getSchema.GetSchemaSkipAuth().Objects - c, err := schema.GetClassByName(schemaModel, i.Config.ClassName.String()) - if err != nil { - return err - } - - for _, prop := range c.Properties { - if prop.DataType[0] == string(schema.DataTypeDate) { - raw, ok := propMap[prop.Name] - if !ok { - // prop is not set, nothing to do - continue - } - - parsed, err := parseAsStringToTime(raw) - if err != nil { - return errors.Wrapf(err, "time prop %q", prop.Name) - } - - propMap[prop.Name] = parsed - } - - if prop.DataType[0] == string(schema.DataTypeDateArray) { - raw, ok := propMap[prop.Name] - if !ok { - // prop is not set, nothing to do - continue - } - - asSlice, ok := raw.([]string) - if !ok { - return errors.Errorf("parse as time array, expected []interface{} got %T", - raw) - } - parsedSlice := make([]interface{}, len(asSlice)) - for j := range asSlice { - parsed, err := parseAsStringToTime(interface{}(asSlice[j])) - if err != nil { - return errors.Wrapf(err, "time array prop %q at pos %d", prop.Name, j) - } - - parsedSlice[j] = parsed - } - propMap[prop.Name] = parsedSlice - - } - } - - return nil -} - -func parseAsStringToTime(in interface{}) (time.Time, error) { - var parsed time.Time - var err error - - asString, ok := in.(string) - if !ok { - return parsed, errors.Errorf("parse as time, expected string got %T", in) - } - - parsed, err = time.Parse(time.RFC3339, asString) - if err != nil { - return parsed, err - } - - return parsed, nil -} - -// return value []error gives the error for the index with the positions -// matching the inputs -func (i *Index) putObjectBatch(ctx context.Context, objects []*storobj.Object, - replProps *additional.ReplicationProperties, -) []error { - type objsAndPos struct { - objects []*storobj.Object - pos []int - } - out := make([]error, len(objects)) - if i.replicationEnabled() && replProps == nil { - replProps = defaultConsistency() - } - - byShard := map[string]objsAndPos{} - for pos, obj := range objects { - if err := i.validateMultiTenancy(obj.Object.Tenant); err != nil { - out[pos] = err - continue - } - shardName, err := i.determineObjectShard(obj.ID(), obj.Object.Tenant) - if err != nil { - out[pos] = err - continue - } - - group := byShard[shardName] - group.objects = append(group.objects, obj) - group.pos = append(group.pos, pos) - byShard[shardName] = group - } - - wg := &sync.WaitGroup{} - for shardName, group := range byShard { - wg.Add(1) - go func(shardName string, group objsAndPos) { - defer wg.Done() - - defer func() { - err := recover() - if err != nil { - for pos := range group.pos { - out[pos] = fmt.Errorf("an unexpected error occurred: %s", err) - } - fmt.Fprintf(os.Stderr, "panic: %s\n", err) - debug.PrintStack() - } - }() - var errs []error - if replProps != nil { - errs = i.replicator.PutObjects(ctx, shardName, group.objects, - replica.ConsistencyLevel(replProps.ConsistencyLevel)) - } else if i.localShard(shardName) == nil { - errs = i.remote.BatchPutObjects(ctx, shardName, group.objects) - } else { - i.backupMutex.RLockGuard(func() error { - if shard := i.localShard(shardName); shard != nil { - errs = shard.PutObjectBatch(ctx, group.objects) - } else { - errs = duplicateErr(errShardNotFound, len(group.objects)) - } - return nil - }) - } - for i, err := range errs { - desiredPos := group.pos[i] - out[desiredPos] = err - } - }(shardName, group) - } - - wg.Wait() - - return out -} - -func duplicateErr(in error, count int) []error { - out := make([]error, count) - for i := range out { - out[i] = in - } - - return out -} - -func (i *Index) IncomingBatchPutObjects(ctx context.Context, shardName string, - objects []*storobj.Object, -) []error { - i.backupMutex.RLock() - defer i.backupMutex.RUnlock() - localShard := i.localShard(shardName) - if localShard == nil { - return duplicateErr(errShardNotFound, len(objects)) - } - - // This is a bit hacky, the problem here is that storobj.Parse() currently - // misses date fields as it has no way of knowing that a date-formatted - // string was actually a date type. However, adding this functionality to - // Parse() would break a lot of code, because it currently - // schema-independent. To find out if a field is a date or date[], we need to - // involve the schema, thus why we are doing it here. This was discovered as - // part of https://github.com/weaviate/weaviate/issues/1775 - for j := range objects { - if err := i.parseDateFieldsInProps(objects[j].Object.Properties); err != nil { - return duplicateErr(err, len(objects)) - } - } - return localShard.PutObjectBatch(ctx, objects) -} - -// return value map[int]error gives the error for the index as it received it -func (i *Index) AddReferencesBatch(ctx context.Context, refs objects.BatchReferences, - replProps *additional.ReplicationProperties, -) []error { - type refsAndPos struct { - refs objects.BatchReferences - pos []int - } - if i.replicationEnabled() && replProps == nil { - replProps = defaultConsistency() - } - - byShard := map[string]refsAndPos{} - out := make([]error, len(refs)) - - for pos, ref := range refs { - if err := i.validateMultiTenancy(ref.Tenant); err != nil { - out[pos] = err - continue - } - shardName, err := i.determineObjectShard(ref.From.TargetID, ref.Tenant) - if err != nil { - out[pos] = err - continue - } - - group := byShard[shardName] - group.refs = append(group.refs, ref) - group.pos = append(group.pos, pos) - byShard[shardName] = group - } - - for shardName, group := range byShard { - var errs []error - if i.replicationEnabled() { - errs = i.replicator.AddReferences(ctx, shardName, group.refs, - replica.ConsistencyLevel(replProps.ConsistencyLevel)) - } else if i.localShard(shardName) == nil { - errs = i.remote.BatchAddReferences(ctx, shardName, group.refs) - } else { - i.backupMutex.RLockGuard(func() error { - if shard := i.localShard(shardName); shard != nil { - errs = shard.AddReferencesBatch(ctx, group.refs) - } else { - errs = duplicateErr(errShardNotFound, len(group.refs)) - } - return nil - }) - } - for i, err := range errs { - desiredPos := group.pos[i] - out[desiredPos] = err - } - } - - return out -} - -func (i *Index) IncomingBatchAddReferences(ctx context.Context, shardName string, - refs objects.BatchReferences, -) []error { - i.backupMutex.RLock() - defer i.backupMutex.RUnlock() - localShard := i.localShard(shardName) - if localShard == nil { - return duplicateErr(errShardNotFound, len(refs)) - } - - return localShard.AddReferencesBatch(ctx, refs) -} - -func (i *Index) objectByID(ctx context.Context, id strfmt.UUID, - props search.SelectProperties, addl additional.Properties, - replProps *additional.ReplicationProperties, tenant string, -) (*storobj.Object, error) { - if err := i.validateMultiTenancy(tenant); err != nil { - return nil, err - } - - shardName, err := i.determineObjectShard(id, tenant) - if err != nil { - switch err.(type) { - case objects.ErrMultiTenancy: - return nil, objects.NewErrMultiTenancy(fmt.Errorf("determine shard: %w", err)) - default: - return nil, objects.NewErrInvalidUserInput("determine shard: %v", err) - } - } - - var obj *storobj.Object - - if i.replicationEnabled() { - if replProps == nil { - replProps = defaultConsistency() - } - if replProps.NodeName != "" { - obj, err = i.replicator.NodeObject(ctx, replProps.NodeName, shardName, id, props, addl) - } else { - obj, err = i.replicator.GetOne(ctx, - replica.ConsistencyLevel(replProps.ConsistencyLevel), shardName, id, props, addl) - } - return obj, err - } - - if shard := i.localShard(shardName); shard != nil { - obj, err = shard.ObjectByID(ctx, id, props, addl) - if err != nil { - return obj, fmt.Errorf("get local object: shard=%s: %w", shardName, err) - } - } else { - obj, err = i.remote.GetObject(ctx, shardName, id, props, addl) - if err != nil { - return obj, fmt.Errorf("get remote object: shard=%s: %w", shardName, err) - } - } - - return obj, nil -} - -func (i *Index) IncomingGetObject(ctx context.Context, shardName string, - id strfmt.UUID, props search.SelectProperties, - additional additional.Properties, -) (*storobj.Object, error) { - shard := i.localShard(shardName) - if shard == nil { - return nil, errShardNotFound - } - - return shard.ObjectByID(ctx, id, props, additional) -} - -func (i *Index) IncomingMultiGetObjects(ctx context.Context, shardName string, - ids []strfmt.UUID, -) ([]*storobj.Object, error) { - shard := i.localShard(shardName) - if shard == nil { - return nil, errShardNotFound - } - - return shard.MultiObjectByID(ctx, wrapIDsInMulti(ids)) -} - -func (i *Index) multiObjectByID(ctx context.Context, - query []multi.Identifier, tenant string, -) ([]*storobj.Object, error) { - if err := i.validateMultiTenancy(tenant); err != nil { - return nil, err - } - - type idsAndPos struct { - ids []multi.Identifier - pos []int - } - - byShard := map[string]idsAndPos{} - for pos, id := range query { - shardName, err := i.determineObjectShard(strfmt.UUID(id.ID), tenant) - if err != nil { - return nil, objects.NewErrInvalidUserInput("determine shard: %v", err) - } - - group := byShard[shardName] - group.ids = append(group.ids, id) - group.pos = append(group.pos, pos) - byShard[shardName] = group - } - - out := make([]*storobj.Object, len(query)) - - for shardName, group := range byShard { - - var objects []*storobj.Object - var err error - - if shard := i.localShard(shardName); shard != nil { - objects, err = shard.MultiObjectByID(ctx, group.ids) - if err != nil { - return nil, errors.Wrapf(err, "shard %s", shard.ID()) - } - } else { - objects, err = i.remote.MultiGetObjects(ctx, shardName, extractIDsFromMulti(group.ids)) - if err != nil { - return nil, errors.Wrapf(err, "remote shard %s", shardName) - } - } - - for i, obj := range objects { - desiredPos := group.pos[i] - out[desiredPos] = obj - } - } - - return out, nil -} - -func extractIDsFromMulti(in []multi.Identifier) []strfmt.UUID { - out := make([]strfmt.UUID, len(in)) - - for i, id := range in { - out[i] = strfmt.UUID(id.ID) - } - - return out -} - -func wrapIDsInMulti(in []strfmt.UUID) []multi.Identifier { - out := make([]multi.Identifier, len(in)) - - for i, id := range in { - out[i] = multi.Identifier{ID: string(id)} - } - - return out -} - -func (i *Index) exists(ctx context.Context, id strfmt.UUID, - replProps *additional.ReplicationProperties, tenant string, -) (bool, error) { - if err := i.validateMultiTenancy(tenant); err != nil { - return false, err - } - - shardName, err := i.determineObjectShard(id, tenant) - if err != nil { - switch err.(type) { - case objects.ErrMultiTenancy: - return false, objects.NewErrMultiTenancy(fmt.Errorf("determine shard: %w", err)) - default: - return false, objects.NewErrInvalidUserInput("determine shard: %v", err) - } - } - - var exists bool - if i.replicationEnabled() { - if replProps == nil { - replProps = defaultConsistency() - } - cl := replica.ConsistencyLevel(replProps.ConsistencyLevel) - return i.replicator.Exists(ctx, cl, shardName, id) - - } - if shard := i.localShard(shardName); shard != nil { - exists, err = shard.Exists(ctx, id) - if err != nil { - err = fmt.Errorf("exists locally: shard=%q: %w", shardName, err) - } - } else { - exists, err = i.remote.Exists(ctx, shardName, id) - if err != nil { - err = fmt.Errorf("exists remotely: shard=%q: %w", shardName, err) - } - } - return exists, err -} - -func (i *Index) IncomingExists(ctx context.Context, shardName string, - id strfmt.UUID, -) (bool, error) { - shard := i.localShard(shardName) - if shard == nil { - return false, errShardNotFound - } - - return shard.Exists(ctx, id) -} - -func (i *Index) objectSearch(ctx context.Context, limit int, filters *filters.LocalFilter, - keywordRanking *searchparams.KeywordRanking, sort []filters.Sort, cursor *filters.Cursor, - addlProps additional.Properties, replProps *additional.ReplicationProperties, tenant string, autoCut int, -) ([]*storobj.Object, []float32, error) { - if err := i.validateMultiTenancy(tenant); err != nil { - return nil, nil, err - } - - shardNames, err := i.targetShardNames(tenant) - if err != nil || len(shardNames) == 0 { - return nil, nil, err - } - - // If the request is a BM25F with no properties selected, use all possible properties - if keywordRanking != nil && keywordRanking.Type == "bm25" && len(keywordRanking.Properties) == 0 { - - cl, err := schema.GetClassByName( - i.getSchema.GetSchemaSkipAuth().Objects, - i.Config.ClassName.String()) - if err != nil { - return nil, nil, err - } - - propHash := cl.Properties - // Get keys of hash - for _, v := range propHash { - if inverted.PropertyHasSearchableIndex(i.getSchema.GetSchemaSkipAuth().Objects, - i.Config.ClassName.String(), v.Name) { - - keywordRanking.Properties = append(keywordRanking.Properties, v.Name) - } - } - - // WEAVIATE-471 - error if we can't find a property to search - if len(keywordRanking.Properties) == 0 { - return nil, []float32{}, errors.New( - "No properties provided, and no indexed properties found in class") - } - } - - outObjects, outScores, err := i.objectSearchByShard(ctx, limit, - filters, keywordRanking, sort, cursor, addlProps, shardNames) - if err != nil { - return nil, nil, err - } - - if len(outObjects) == len(outScores) { - if keywordRanking != nil && keywordRanking.Type == "bm25" { - for ii := range outObjects { - oo := outObjects[ii] - os := outScores[ii] - - if oo.AdditionalProperties() == nil { - oo.Object.Additional = make(map[string]interface{}) - } - oo.Object.Additional["score"] = os - - // Collect all keys starting with "BM25F" and add them to the Additional - if keywordRanking.AdditionalExplanations { - explainScore := "" - for k, v := range oo.Object.Additional { - if strings.HasPrefix(k, "BM25F") { - - explainScore = fmt.Sprintf("%v, %v:%v", explainScore, k, v) - delete(oo.Object.Additional, k) - } - } - oo.Object.Additional["explainScore"] = explainScore - } - } - } - } - - if len(sort) > 0 { - if len(shardNames) > 1 { - var err error - outObjects, outScores, err = i.sort(outObjects, outScores, sort, limit) - if err != nil { - return nil, nil, errors.Wrap(err, "sort") - } - } - } else if keywordRanking != nil { - outObjects, outScores = i.sortKeywordRanking(outObjects, outScores) - } else if len(shardNames) > 1 && !addlProps.ReferenceQuery { - // sort only for multiple shards (already sorted for single) - // and for not reference nested query (sort is applied for root query) - outObjects, outScores = i.sortByID(outObjects, outScores) - } - - if autoCut > 0 { - cutOff := autocut.Autocut(outScores, autoCut) - outObjects = outObjects[:cutOff] - outScores = outScores[:cutOff] - } - - // if this search was caused by a reference property - // search, we should not limit the number of results. - // for example, if the query contains a where filter - // whose operator is `And`, and one of the operands - // contains a path to a reference prop, the Search - // caused by such a ref prop being limited can cause - // the `And` to return no results where results would - // be expected. we won't know that unless we search - // and return all referenced object properties. - if !addlProps.ReferenceQuery && len(outObjects) > limit { - if len(outObjects) == len(outScores) { - outScores = outScores[:limit] - } - outObjects = outObjects[:limit] - } - - if i.replicationEnabled() { - if replProps == nil { - replProps = defaultConsistency(replica.One) - } - l := replica.ConsistencyLevel(replProps.ConsistencyLevel) - err = i.replicator.CheckConsistency(ctx, l, outObjects) - if err != nil { - i.logger.WithField("action", "object_search"). - Errorf("failed to check consistency of search results: %v", err) - } - } - - return outObjects, outScores, nil -} - -func (i *Index) objectSearchByShard(ctx context.Context, limit int, filters *filters.LocalFilter, - keywordRanking *searchparams.KeywordRanking, sort []filters.Sort, cursor *filters.Cursor, - addlProps additional.Properties, shards []string, -) ([]*storobj.Object, []float32, error) { - resultObjects, resultScores := objectSearchPreallocate(limit, shards) - - eg := errgroup.Group{} - eg.SetLimit(_NUMCPU * 2) - shardResultLock := sync.Mutex{} - for _, shardName := range shards { - shardName := shardName - - eg.Go(func() error { - var ( - objs []*storobj.Object - scores []float32 - nodeName string - err error - ) - - if shard := i.localShard(shardName); shard != nil { - nodeName = i.getSchema.NodeName() - objs, scores, err = shard.ObjectSearch(ctx, limit, filters, keywordRanking, sort, cursor, addlProps) - if err != nil { - return fmt.Errorf( - "local shard object search %s: %w", shard.ID(), err) - } - } else { - objs, scores, nodeName, err = i.remote.SearchShard( - ctx, shardName, nil, limit, filters, keywordRanking, - sort, cursor, nil, addlProps, i.replicationEnabled()) - if err != nil { - return fmt.Errorf( - "remote shard object search %s: %w", shardName, err) - } - } - - if i.replicationEnabled() { - storobj.AddOwnership(objs, nodeName, shardName) - } - - shardResultLock.Lock() - resultObjects = append(resultObjects, objs...) - resultScores = append(resultScores, scores...) - shardResultLock.Unlock() - - return nil - }) - } - if err := eg.Wait(); err != nil { - return nil, nil, err - } - if len(resultObjects) == len(resultScores) { - - // Force a stable sort order by UUID - - type resultSortable struct { - object *storobj.Object - score float32 - } - objs := resultObjects - scores := resultScores - var results []resultSortable = make([]resultSortable, len(objs)) - for i := range objs { - results[i] = resultSortable{ - object: objs[i], - score: scores[i], - } - } - - golangSort.Slice(results, func(i, j int) bool { - if results[i].score == results[j].score { - return results[i].object.Object.ID > results[j].object.Object.ID - } - - return results[i].score > results[j].score - }) - - var finalObjs []*storobj.Object = make([]*storobj.Object, len(results)) - var finalScores []float32 = make([]float32, len(results)) - for i, result := range results { - - finalObjs[i] = result.object - finalScores[i] = result.score - } - - return finalObjs, finalScores, nil - } - - return resultObjects, resultScores, nil -} - -func (i *Index) sortByID(objects []*storobj.Object, scores []float32, -) ([]*storobj.Object, []float32) { - return newIDSorter().sort(objects, scores) -} - -func (i *Index) sortKeywordRanking(objects []*storobj.Object, - scores []float32, -) ([]*storobj.Object, []float32) { - return newScoresSorter().sort(objects, scores) -} - -func (i *Index) sort(objects []*storobj.Object, scores []float32, - sort []filters.Sort, limit int, -) ([]*storobj.Object, []float32, error) { - return sorter.NewObjectsSorter(i.getSchema.GetSchemaSkipAuth()). - Sort(objects, scores, limit, sort) -} - -func (i *Index) mergeGroups(objects []*storobj.Object, dists []float32, - groupBy *searchparams.GroupBy, limit, shardCount int, -) ([]*storobj.Object, []float32, error) { - return newGroupMerger(objects, dists, groupBy).Do() -} - -func (i *Index) singleLocalShardObjectVectorSearch(ctx context.Context, searchVector []float32, - dist float32, limit int, filters *filters.LocalFilter, - sort []filters.Sort, groupBy *searchparams.GroupBy, additional additional.Properties, - shardName string, -) ([]*storobj.Object, []float32, error) { - shard := i.localShard(shardName) - res, resDists, err := shard.ObjectVectorSearch( - ctx, searchVector, dist, limit, filters, sort, groupBy, additional) - if err != nil { - return nil, nil, errors.Wrapf(err, "shard %s", shard.ID()) - } - - return res, resDists, nil -} - -// to be called after validating multi-tenancy -func (i *Index) targetShardNames(tenant string) ([]string, error) { - className := i.Config.ClassName.String() - if !i.partitioningEnabled { - shardingState := i.getSchema.CopyShardingState(className) - return shardingState.AllPhysicalShards(), nil - } - if tenant != "" { - if shard, status := i.getSchema.TenantShard(className, tenant); shard != "" { - if status == models.TenantActivityStatusHOT { - return []string{shard}, nil - } - return nil, objects.NewErrMultiTenancy(fmt.Errorf("%w: '%s'", errTenantNotActive, tenant)) - } - } - return nil, objects.NewErrMultiTenancy(fmt.Errorf("%w: %q", errTenantNotFound, tenant)) -} - -func (i *Index) objectVectorSearch(ctx context.Context, searchVector []float32, - dist float32, limit int, filters *filters.LocalFilter, sort []filters.Sort, - groupBy *searchparams.GroupBy, additional additional.Properties, - replProps *additional.ReplicationProperties, tenant string, -) ([]*storobj.Object, []float32, error) { - if err := i.validateMultiTenancy(tenant); err != nil { - return nil, nil, err - } - shardNames, err := i.targetShardNames(tenant) - if err != nil || len(shardNames) == 0 { - return nil, nil, err - } - - if len(shardNames) == 1 { - if i.localShard(shardNames[0]) != nil { - return i.singleLocalShardObjectVectorSearch(ctx, searchVector, dist, limit, filters, - sort, groupBy, additional, shardNames[0]) - } - } - - // a limit of -1 is used to signal a search by distance. if that is - // the case we have to adjust how we calculate the output capacity - var shardCap int - if limit < 0 { - shardCap = len(shardNames) * hnsw.DefaultSearchByDistInitialLimit - } else { - shardCap = len(shardNames) * limit - } - - eg := &errgroup.Group{} - eg.SetLimit(_NUMCPU * 2) - m := &sync.Mutex{} - - out := make([]*storobj.Object, 0, shardCap) - dists := make([]float32, 0, shardCap) - for _, shardName := range shardNames { - shardName := shardName - eg.Go(func() error { - var ( - res []*storobj.Object - resDists []float32 - nodeName string - err error - ) - - if shard := i.localShard(shardName); shard != nil { - nodeName = i.getSchema.NodeName() - res, resDists, err = shard.ObjectVectorSearch( - ctx, searchVector, dist, limit, filters, sort, groupBy, additional) - if err != nil { - return errors.Wrapf(err, "shard %s", shard.ID()) - } - - } else { - res, resDists, nodeName, err = i.remote.SearchShard(ctx, - shardName, searchVector, limit, filters, - nil, sort, nil, groupBy, additional, i.replicationEnabled()) - if err != nil { - return errors.Wrapf(err, "remote shard %s", shardName) - } - } - if i.replicationEnabled() { - storobj.AddOwnership(res, nodeName, shardName) - } - - m.Lock() - out = append(out, res...) - dists = append(dists, resDists...) - m.Unlock() - - return nil - }) - } - - if err := eg.Wait(); err != nil { - return nil, nil, err - } - - if len(shardNames) == 1 { - return out, dists, nil - } - - if len(shardNames) > 1 && groupBy != nil { - return i.mergeGroups(out, dists, groupBy, limit, len(shardNames)) - } - - if len(shardNames) > 1 && len(sort) > 0 { - return i.sort(out, dists, sort, limit) - } - - out, dists = newDistancesSorter().sort(out, dists) - if limit > 0 && len(out) > limit { - out = out[:limit] - dists = dists[:limit] - } - - if i.replicationEnabled() { - if replProps == nil { - replProps = defaultConsistency(replica.One) - } - l := replica.ConsistencyLevel(replProps.ConsistencyLevel) - err = i.replicator.CheckConsistency(ctx, l, out) - if err != nil { - i.logger.WithField("action", "object_vector_search"). - Errorf("failed to check consistency of search results: %v", err) - } - } - - return out, dists, nil -} - -func (i *Index) IncomingSearch(ctx context.Context, shardName string, - searchVector []float32, distance float32, limit int, filters *filters.LocalFilter, - keywordRanking *searchparams.KeywordRanking, sort []filters.Sort, - cursor *filters.Cursor, groupBy *searchparams.GroupBy, - additional additional.Properties, -) ([]*storobj.Object, []float32, error) { - shard := i.localShard(shardName) - if shard == nil { - return nil, nil, errShardNotFound - } - - if searchVector == nil { - res, scores, err := shard.ObjectSearch(ctx, limit, filters, keywordRanking, sort, cursor, additional) - if err != nil { - return nil, nil, err - } - - return res, scores, nil - } - - res, resDists, err := shard.ObjectVectorSearch( - ctx, searchVector, distance, limit, filters, sort, groupBy, additional) - if err != nil { - return nil, nil, errors.Wrapf(err, "shard %s", shard.ID()) - } - - return res, resDists, nil -} - -func (i *Index) deleteObject(ctx context.Context, id strfmt.UUID, - replProps *additional.ReplicationProperties, tenant string, -) error { - if err := i.validateMultiTenancy(tenant); err != nil { - return err - } - - shardName, err := i.determineObjectShard(id, tenant) - if err != nil { - return objects.NewErrInvalidUserInput("determine shard: %v", err) - } - - if i.replicationEnabled() { - if replProps == nil { - replProps = defaultConsistency() - } - cl := replica.ConsistencyLevel(replProps.ConsistencyLevel) - if err := i.replicator.DeleteObject(ctx, shardName, id, cl); err != nil { - return fmt.Errorf("replicate deletion: shard=%q %w", shardName, err) - } - return nil - } - - // no replication, remote shard - if i.localShard(shardName) == nil { - if err := i.remote.DeleteObject(ctx, shardName, id); err != nil { - return fmt.Errorf("delete remote object: shard=%q: %w", shardName, err) - } - return nil - } - - // no replication, local shard - i.backupMutex.RLock() - defer i.backupMutex.RUnlock() - err = errShardNotFound - if shard := i.localShard(shardName); shard != nil { - err = shard.DeleteObject(ctx, id) - } - if err != nil { - return fmt.Errorf("delete local object: shard=%q: %w", shardName, err) - } - return nil -} - -func (i *Index) IncomingDeleteObject(ctx context.Context, shardName string, - id strfmt.UUID, -) error { - i.backupMutex.RLock() - defer i.backupMutex.RUnlock() - shard := i.localShard(shardName) - if shard == nil { - return errShardNotFound - } - return shard.DeleteObject(ctx, id) -} - -func (i *Index) localShard(name string) ShardLike { - return i.shards.Load(name) -} - -func (i *Index) mergeObject(ctx context.Context, merge objects.MergeDocument, - replProps *additional.ReplicationProperties, tenant string, -) error { - if err := i.validateMultiTenancy(tenant); err != nil { - return err - } - - shardName, err := i.determineObjectShard(merge.ID, tenant) - if err != nil { - return objects.NewErrInvalidUserInput("determine shard: %v", err) - } - - if i.replicationEnabled() { - if replProps == nil { - replProps = defaultConsistency() - } - cl := replica.ConsistencyLevel(replProps.ConsistencyLevel) - if err := i.replicator.MergeObject(ctx, shardName, &merge, cl); err != nil { - return fmt.Errorf("replicate single update: %w", err) - } - return nil - } - - // no replication, remote shard - if i.localShard(shardName) == nil { - if err := i.remote.MergeObject(ctx, shardName, merge); err != nil { - return fmt.Errorf("update remote object: shard=%q: %w", shardName, err) - } - return nil - } - - // no replication, local shard - i.backupMutex.RLock() - defer i.backupMutex.RUnlock() - err = errShardNotFound - if shard := i.localShard(shardName); shard != nil { - err = shard.MergeObject(ctx, merge) - } - if err != nil { - return fmt.Errorf("update local object: shard=%q: %w", shardName, err) - } - - return nil -} - -func (i *Index) IncomingMergeObject(ctx context.Context, shardName string, - mergeDoc objects.MergeDocument, -) error { - i.backupMutex.RLock() - defer i.backupMutex.RUnlock() - shard := i.localShard(shardName) - if shard == nil { - return errShardNotFound - } - - return shard.MergeObject(ctx, mergeDoc) -} - -func (i *Index) aggregate(ctx context.Context, - params aggregation.Params, -) (*aggregation.Result, error) { - if err := i.validateMultiTenancy(params.Tenant); err != nil { - return nil, err - } - - shardNames, err := i.targetShardNames(params.Tenant) - if err != nil || len(shardNames) == 0 { - return nil, err - } - - results := make([]*aggregation.Result, len(shardNames)) - for j, shardName := range shardNames { - var err error - var res *aggregation.Result - if shard := i.localShard(shardName); shard != nil { - res, err = shard.Aggregate(ctx, params) - } else { - res, err = i.remote.Aggregate(ctx, shardName, params) - } - if err != nil { - return nil, errors.Wrapf(err, "shard %s", shardName) - } - - results[j] = res - } - - return aggregator.NewShardCombiner().Do(results), nil -} - -func (i *Index) IncomingAggregate(ctx context.Context, shardName string, - params aggregation.Params, -) (*aggregation.Result, error) { - shard := i.localShard(shardName) - if shard == nil { - return nil, errShardNotFound - } - - return shard.Aggregate(ctx, params) -} - -func (i *Index) drop() error { - i.closingCancel() - - var eg errgroup.Group - eg.SetLimit(_NUMCPU * 2) - fields := logrus.Fields{"action": "drop_shard", "class": i.Config.ClassName} - dropShard := func(name string, shard ShardLike) error { - if shard == nil { - return nil - } - eg.Go(func() error { - if err := shard.drop(); err != nil { - logrus.WithFields(fields).WithField("id", shard.ID()).Error(err) - } - return nil - }) - return nil - } - - i.backupMutex.RLock() - defer i.backupMutex.RUnlock() - - i.shards.Range(dropShard) - if err := eg.Wait(); err != nil { - return err - } - - // Dropping the shards only unregisters the shards callbacks, but we still - // need to stop the cycle managers that those shards used to register with. - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) - defer cancel() - - if err := i.stopCycleManagers(ctx, "drop"); err != nil { - return err - } - - return os.RemoveAll(i.path()) -} - -// dropShards deletes shards in a transactional manner. -// To confirm the deletion, the user must call Commit(true). -// To roll back the deletion, the user must call Commit(false) -func (i *Index) dropShards(names []string) (commit func(success bool), err error) { - shards := make(map[string]ShardLike, len(names)) - i.backupMutex.RLock() - defer i.backupMutex.RUnlock() - - // mark deleted shards - for _, name := range names { - prev, ok := i.shards.Swap(name, nil) // mark - if !ok { // shard doesn't exit - i.shards.LoadAndDelete(name) // rollback nil value created by swap() - continue - } - if prev != nil { - shards[name] = prev - } - } - - rollback := func() { - for name, shard := range shards { - i.shards.CompareAndSwap(name, nil, shard) - } - } - - var eg errgroup.Group - eg.SetLimit(_NUMCPU * 2) - commit = func(success bool) { - if !success { - rollback() - return - } - // detach shards - for name := range shards { - i.shards.LoadAndDelete(name) - } - - // drop shards - for _, shard := range shards { - shard := shard - eg.Go(func() error { - if err := shard.drop(); err != nil { - i.logger.WithField("action", "drop_shard"). - WithField("shard", shard.ID()).Error(err) - } - return nil - }) - } - } - - return commit, eg.Wait() -} - -func (i *Index) Shutdown(ctx context.Context) error { - i.closingCancel() - - i.backupMutex.RLock() - defer i.backupMutex.RUnlock() - - // TODO allow every resource cleanup to run, before returning early with error - if err := i.ForEachShardConcurrently(func(name string, shard ShardLike) error { - if err := shard.Shutdown(ctx); err != nil { - return errors.Wrapf(err, "shutdown shard %q", name) - } - return nil - }); err != nil { - return err - } - if err := i.stopCycleManagers(ctx, "shutdown"); err != nil { - return err - } - - return nil -} - -func (i *Index) stopCycleManagers(ctx context.Context, usecase string) error { - if err := i.cycleCallbacks.compactionCycle.StopAndWait(ctx); err != nil { - return fmt.Errorf("%s: stop compaction cycle: %w", usecase, err) - } - if err := i.cycleCallbacks.flushCycle.StopAndWait(ctx); err != nil { - return fmt.Errorf("%s: stop flush cycle: %w", usecase, err) - } - if err := i.cycleCallbacks.vectorCommitLoggerCycle.StopAndWait(ctx); err != nil { - return fmt.Errorf("%s: stop vector commit logger cycle: %w", usecase, err) - } - if err := i.cycleCallbacks.vectorTombstoneCleanupCycle.StopAndWait(ctx); err != nil { - return fmt.Errorf("%s: stop vector tombstone cleanup cycle: %w", usecase, err) - } - if err := i.cycleCallbacks.geoPropsCommitLoggerCycle.StopAndWait(ctx); err != nil { - return fmt.Errorf("%s: stop geo props commit logger cycle: %w", usecase, err) - } - if err := i.cycleCallbacks.geoPropsTombstoneCleanupCycle.StopAndWait(ctx); err != nil { - return fmt.Errorf("%s: stop geo props tombstone cleanup cycle: %w", usecase, err) - } - return nil -} - -func (i *Index) getShardsQueueSize(ctx context.Context, tenant string) (map[string]int64, error) { - shardsQueueSize := make(map[string]int64) - - shardState := i.getSchema.CopyShardingState(i.Config.ClassName.String()) - shardNames := shardState.AllPhysicalShards() - - for _, shardName := range shardNames { - if tenant != "" && shardName != tenant { - continue - } - var err error - var size int64 - if !shardState.IsLocalShard(shardName) { - size, err = i.remote.GetShardQueueSize(ctx, shardName) - } else { - shard := i.localShard(shardName) - if shard == nil { - err = errors.Errorf("shard %s does not exist", shardName) - } else { - size = shard.Queue().Size() - } - } - if err != nil { - return nil, errors.Wrapf(err, "shard %s", shardName) - } - - shardsQueueSize[shardName] = size - } - - return shardsQueueSize, nil -} - -func (i *Index) IncomingGetShardQueueSize(ctx context.Context, shardName string) (int64, error) { - shard := i.localShard(shardName) - if shard == nil { - return 0, errShardNotFound - } - return shard.Queue().Size(), nil -} - -func (i *Index) getShardsStatus(ctx context.Context, tenant string) (map[string]string, error) { - shardsStatus := make(map[string]string) - - shardState := i.getSchema.CopyShardingState(i.Config.ClassName.String()) - shardNames := shardState.AllPhysicalShards() - - for _, shardName := range shardNames { - if tenant != "" && shardName != tenant { - continue - } - var err error - var status string - if !shardState.IsLocalShard(shardName) { - status, err = i.remote.GetShardStatus(ctx, shardName) - } else { - shard := i.localShard(shardName) - if shard == nil { - err = errors.Errorf("shard %s does not exist", shardName) - } else { - status = shard.GetStatus().String() - } - } - if err != nil { - return nil, errors.Wrapf(err, "shard %s", shardName) - } - - shardsStatus[shardName] = status - } - - return shardsStatus, nil -} - -func (i *Index) IncomingGetShardStatus(ctx context.Context, shardName string) (string, error) { - shard := i.localShard(shardName) - if shard == nil { - return "", errShardNotFound - } - return shard.GetStatus().String(), nil -} - -func (i *Index) updateShardStatus(ctx context.Context, shardName, targetStatus string) error { - if shard := i.localShard(shardName); shard != nil { - return shard.UpdateStatus(targetStatus) - } - return i.remote.UpdateShardStatus(ctx, shardName, targetStatus) -} - -func (i *Index) IncomingUpdateShardStatus(ctx context.Context, shardName, targetStatus string) error { - shard := i.localShard(shardName) - if shard == nil { - return errShardNotFound - } - return shard.UpdateStatus(targetStatus) -} - -func (i *Index) notifyReady() { - i.ForEachShard(func(name string, shard ShardLike) error { - shard.NotifyReady() - return nil - }) -} - -func (i *Index) findUUIDs(ctx context.Context, - filters *filters.LocalFilter, tenant string, -) (map[string][]strfmt.UUID, error) { - before := time.Now() - defer i.metrics.BatchDelete(before, "filter_total") - - if err := i.validateMultiTenancy(tenant); err != nil { - return nil, err - } - - shardNames, err := i.targetShardNames(tenant) - if err != nil { - return nil, err - } - - results := make(map[string][]strfmt.UUID) - for _, shardName := range shardNames { - var err error - var res []strfmt.UUID - if shard := i.localShard(shardName); shard != nil { - res, err = shard.FindUUIDs(ctx, filters) - } else { - res, err = i.remote.FindUUIDs(ctx, shardName, filters) - } - if err != nil { - return nil, fmt.Errorf("find matching doc ids in shard %q: %w", shardName, err) - } - - results[shardName] = res - } - - return results, nil -} - -func (i *Index) IncomingFindUUIDs(ctx context.Context, shardName string, - filters *filters.LocalFilter, -) ([]strfmt.UUID, error) { - shard := i.localShard(shardName) - if shard == nil { - return nil, errShardNotFound - } - - return shard.FindUUIDs(ctx, filters) -} - -func (i *Index) batchDeleteObjects(ctx context.Context, shardUUIDs map[string][]strfmt.UUID, - dryRun bool, replProps *additional.ReplicationProperties, -) (objects.BatchSimpleObjects, error) { - before := time.Now() - defer i.metrics.BatchDelete(before, "delete_from_shards_total") - - type result struct { - objs objects.BatchSimpleObjects - } - - if i.replicationEnabled() && replProps == nil { - replProps = defaultConsistency() - } - - wg := &sync.WaitGroup{} - ch := make(chan result, len(shardUUIDs)) - for shardName, uuids := range shardUUIDs { - uuids := uuids - wg.Add(1) - go func(shardName string, uuids []strfmt.UUID) { - defer wg.Done() - - var objs objects.BatchSimpleObjects - if i.replicationEnabled() { - objs = i.replicator.DeleteObjects(ctx, shardName, uuids, - dryRun, replica.ConsistencyLevel(replProps.ConsistencyLevel)) - } else if i.localShard(shardName) == nil { - objs = i.remote.DeleteObjectBatch(ctx, shardName, uuids, dryRun) - } else { - i.backupMutex.RLockGuard(func() error { - if shard := i.localShard(shardName); shard != nil { - objs = shard.DeleteObjectBatch(ctx, uuids, dryRun) - } else { - objs = objects.BatchSimpleObjects{objects.BatchSimpleObject{Err: errShardNotFound}} - } - return nil - }) - } - ch <- result{objs} - }(shardName, uuids) - } - - wg.Wait() - close(ch) - - var out objects.BatchSimpleObjects - for res := range ch { - out = append(out, res.objs...) - } - - return out, nil -} - -func (i *Index) IncomingDeleteObjectBatch(ctx context.Context, shardName string, - uuids []strfmt.UUID, dryRun bool, -) objects.BatchSimpleObjects { - i.backupMutex.RLock() - defer i.backupMutex.RUnlock() - shard := i.localShard(shardName) - if shard == nil { - return objects.BatchSimpleObjects{ - objects.BatchSimpleObject{Err: errShardNotFound}, - } - } - - return shard.DeleteObjectBatch(ctx, uuids, dryRun) -} - -func defaultConsistency(l ...replica.ConsistencyLevel) *additional.ReplicationProperties { - rp := &additional.ReplicationProperties{} - if len(l) != 0 { - rp.ConsistencyLevel = string(l[0]) - } else { - rp.ConsistencyLevel = string(replica.Quorum) - } - return rp -} - -func objectSearchPreallocate(limit int, shards []string) ([]*storobj.Object, []float32) { - perShardLimit := config.DefaultQueryMaximumResults - if perShardLimit > int64(limit) { - perShardLimit = int64(limit) - } - capacity := perShardLimit * int64(len(shards)) - objects := make([]*storobj.Object, 0, capacity) - scores := make([]float32, 0, capacity) - - return objects, scores -} - -func (i *Index) addNewShard(ctx context.Context, - class *models.Class, shardName string, -) error { - if shard := i.localShard(shardName); shard != nil { - return fmt.Errorf("shard %q exists already", shardName) - } - - // TODO: metrics - return i.initAndStoreShard(ctx, shardName, class, i.metrics.baseMetrics) -} - -func (i *Index) validateMultiTenancy(tenant string) error { - if i.partitioningEnabled && tenant == "" { - return objects.NewErrMultiTenancy( - fmt.Errorf("class %s has multi-tenancy enabled, but request was without tenant", i.Config.ClassName), - ) - } else if !i.partitioningEnabled && tenant != "" { - return objects.NewErrMultiTenancy( - fmt.Errorf("class %s has multi-tenancy disabled, but request was with tenant", i.Config.ClassName), - ) - } - return nil -} diff --git a/adapters/repos/db/index_cyclecallbacks.go b/adapters/repos/db/index_cyclecallbacks.go deleted file mode 100644 index 6999eab09cb511038c38efe7130e6acee20f551a..0000000000000000000000000000000000000000 --- a/adapters/repos/db/index_cyclecallbacks.go +++ /dev/null @@ -1,133 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "strings" - "time" - - "github.com/weaviate/weaviate/entities/cyclemanager" - "github.com/weaviate/weaviate/entities/vectorindex/hnsw" - enthnsw "github.com/weaviate/weaviate/entities/vectorindex/hnsw" -) - -type indexCycleCallbacks struct { - compactionCallbacks cyclemanager.CycleCallbackGroup - compactionCycle cyclemanager.CycleManager - - flushCallbacks cyclemanager.CycleCallbackGroup - flushCycle cyclemanager.CycleManager - - vectorCommitLoggerCallbacks cyclemanager.CycleCallbackGroup - vectorCommitLoggerCycle cyclemanager.CycleManager - vectorTombstoneCleanupCallbacks cyclemanager.CycleCallbackGroup - vectorTombstoneCleanupCycle cyclemanager.CycleManager - - geoPropsCommitLoggerCallbacks cyclemanager.CycleCallbackGroup - geoPropsCommitLoggerCycle cyclemanager.CycleManager - geoPropsTombstoneCleanupCallbacks cyclemanager.CycleCallbackGroup - geoPropsTombstoneCleanupCycle cyclemanager.CycleManager -} - -func (index *Index) initCycleCallbacks() { - vectorTombstoneCleanupIntervalSeconds := hnsw.DefaultCleanupIntervalSeconds - if hnswUserConfig, ok := index.vectorIndexUserConfig.(hnsw.UserConfig); ok { - vectorTombstoneCleanupIntervalSeconds = hnswUserConfig.CleanupIntervalSeconds - } - - id := func(elems ...string) string { - elems = append([]string{"index", index.ID()}, elems...) - return strings.Join(elems, "/") - } - - compactionCallbacks := cyclemanager.NewCallbackGroup(id("compaction"), index.logger, _NUMCPU*2) - compactionCycle := cyclemanager.NewManager( - cyclemanager.CompactionCycleTicker(), - compactionCallbacks.CycleCallback) - - flushCallbacks := cyclemanager.NewCallbackGroup(id("flush"), index.logger, _NUMCPU*2) - flushCycle := cyclemanager.NewManager( - cyclemanager.MemtableFlushCycleTicker(), - flushCallbacks.CycleCallback) - - vectorCommitLoggerCallbacks := cyclemanager.NewCallbackGroup(id("vector", "commit_logger"), index.logger, _NUMCPU*2) - // Previously we had an interval of 10s in here, which was changed to - // 0.5s as part of gh-1867. There's really no way to wait so long in - // between checks: If you are running on a low-powered machine, the - // interval will simply find that there is no work and do nothing in - // each iteration. However, if you are running on a very powerful - // machine within 10s you could have potentially created two units of - // work, but we'll only be handling one every 10s. This means - // uncombined/uncondensed hnsw commit logs will keep piling up can only - // be processes long after the initial insert is complete. This also - // means that if there is a crash during importing a lot of work needs - // to be done at startup, since the commit logs still contain too many - // redundancies. So as of now it seems there are only advantages to - // running the cleanup checks and work much more often. - // - // update: switched to dynamic intervals with values between 500ms and 10s - // introduced to address https://github.com/weaviate/weaviate/issues/2783 - vectorCommitLoggerCycle := cyclemanager.NewManager( - cyclemanager.HnswCommitLoggerCycleTicker(), - vectorCommitLoggerCallbacks.CycleCallback) - - vectorTombstoneCleanupCallbacks := cyclemanager.NewCallbackGroup(id("vector", "tombstone_cleanup"), index.logger, _NUMCPU*2) - vectorTombstoneCleanupCycle := cyclemanager.NewManager( - cyclemanager.NewFixedTicker(time.Duration(vectorTombstoneCleanupIntervalSeconds)*time.Second), - vectorTombstoneCleanupCallbacks.CycleCallback) - - geoPropsCommitLoggerCallbacks := cyclemanager.NewCallbackGroup(id("geo_props", "commit_logger"), index.logger, _NUMCPU*2) - geoPropsCommitLoggerCycle := cyclemanager.NewManager( - cyclemanager.GeoCommitLoggerCycleTicker(), - geoPropsCommitLoggerCallbacks.CycleCallback) - - geoPropsTombstoneCleanupCallbacks := cyclemanager.NewCallbackGroup(id("geo_props", "tombstone_cleanup"), index.logger, _NUMCPU*2) - geoPropsTombstoneCleanupCycle := cyclemanager.NewManager( - cyclemanager.NewFixedTicker(enthnsw.DefaultCleanupIntervalSeconds*time.Second), - geoPropsTombstoneCleanupCallbacks.CycleCallback) - - index.cycleCallbacks = &indexCycleCallbacks{ - compactionCallbacks: compactionCallbacks, - compactionCycle: compactionCycle, - flushCallbacks: flushCallbacks, - flushCycle: flushCycle, - - vectorCommitLoggerCallbacks: vectorCommitLoggerCallbacks, - vectorCommitLoggerCycle: vectorCommitLoggerCycle, - vectorTombstoneCleanupCallbacks: vectorTombstoneCleanupCallbacks, - vectorTombstoneCleanupCycle: vectorTombstoneCleanupCycle, - - geoPropsCommitLoggerCallbacks: geoPropsCommitLoggerCallbacks, - geoPropsCommitLoggerCycle: geoPropsCommitLoggerCycle, - geoPropsTombstoneCleanupCallbacks: geoPropsTombstoneCleanupCallbacks, - geoPropsTombstoneCleanupCycle: geoPropsTombstoneCleanupCycle, - } -} - -func (index *Index) initCycleCallbacksNoop() { - index.cycleCallbacks = &indexCycleCallbacks{ - compactionCallbacks: cyclemanager.NewCallbackGroupNoop(), - compactionCycle: cyclemanager.NewManagerNoop(), - flushCallbacks: cyclemanager.NewCallbackGroupNoop(), - flushCycle: cyclemanager.NewManagerNoop(), - - vectorCommitLoggerCallbacks: cyclemanager.NewCallbackGroupNoop(), - vectorCommitLoggerCycle: cyclemanager.NewManagerNoop(), - vectorTombstoneCleanupCallbacks: cyclemanager.NewCallbackGroupNoop(), - vectorTombstoneCleanupCycle: cyclemanager.NewManagerNoop(), - - geoPropsCommitLoggerCallbacks: cyclemanager.NewCallbackGroupNoop(), - geoPropsCommitLoggerCycle: cyclemanager.NewManagerNoop(), - geoPropsTombstoneCleanupCallbacks: cyclemanager.NewCallbackGroupNoop(), - geoPropsTombstoneCleanupCycle: cyclemanager.NewManagerNoop(), - } -} diff --git a/adapters/repos/db/index_integration_test.go b/adapters/repos/db/index_integration_test.go deleted file mode 100644 index 9a18c039214289f2cc7affb8bfa86320b42c845e..0000000000000000000000000000000000000000 --- a/adapters/repos/db/index_integration_test.go +++ /dev/null @@ -1,372 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package db - -import ( - "context" - "os" - "path" - "testing" - - "github.com/go-openapi/strfmt" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/adapters/repos/db/inverted" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/storagestate" - "github.com/weaviate/weaviate/entities/storobj" - "github.com/weaviate/weaviate/entities/vectorindex/hnsw" -) - -func TestIndex_DropIndex(t *testing.T) { - dirName := t.TempDir() - class := &models.Class{Class: "deletetest"} - index := emptyIdx(t, dirName, class) - - indexFilesBeforeDelete, err := getIndexFilenames(dirName, class.Class) - require.Nil(t, err) - - err = index.drop() - require.Nil(t, err) - - indexFilesAfterDelete, err := getIndexFilenames(dirName, class.Class) - require.Nil(t, err) - - assert.Equal(t, 6, len(indexFilesBeforeDelete)) - assert.Equal(t, 0, len(indexFilesAfterDelete)) -} - -func TestIndex_DropEmptyAndRecreateEmptyIndex(t *testing.T) { - dirName := t.TempDir() - class := &models.Class{Class: "deletetest"} - index := emptyIdx(t, dirName, class) - - indexFilesBeforeDelete, err := getIndexFilenames(dirName, class.Class) - require.Nil(t, err) - - // drop the index - err = index.drop() - require.Nil(t, err) - - indexFilesAfterDelete, err := getIndexFilenames(dirName, class.Class) - require.Nil(t, err) - - index = emptyIdx(t, dirName, class) - - indexFilesAfterRecreate, err := getIndexFilenames(dirName, class.Class) - require.Nil(t, err) - - assert.Equal(t, 6, len(indexFilesBeforeDelete)) - assert.Equal(t, 0, len(indexFilesAfterDelete)) - assert.Equal(t, 6, len(indexFilesAfterRecreate)) - - err = index.drop() - require.Nil(t, err) -} - -func TestIndex_DropWithDataAndRecreateWithDataIndex(t *testing.T) { - dirName := t.TempDir() - logger, _ := test.NewNullLogger() - class := &models.Class{ - Class: "deletetest", - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - InvertedIndexConfig: &models.InvertedIndexConfig{}, - } - fakeSchema := schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{ - class, - }, - }, - } - // create index with data - shardState := singleShardState() - index, err := NewIndex(testCtx(), IndexConfig{ - RootPath: dirName, - ClassName: schema.ClassName(class.Class), - }, shardState, inverted.ConfigFromModel(class.InvertedIndexConfig), - hnsw.NewDefaultUserConfig(), &fakeSchemaGetter{ - schema: fakeSchema, shardState: shardState, - }, nil, logger, nil, nil, nil, nil, class, nil, nil) - require.Nil(t, err) - - productsIds := []strfmt.UUID{ - "1295c052-263d-4aae-99dd-920c5a370d06", - "1295c052-263d-4aae-99dd-920c5a370d07", - } - - products := []map[string]interface{}{ - {"name": "one"}, - {"name": "two"}, - } - - err = index.addUUIDProperty(context.TODO()) - require.Nil(t, err) - - err = index.addProperty(context.TODO(), &models.Property{ - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }) - require.Nil(t, err) - - for i, p := range products { - product := models.Object{ - Class: class.Class, - ID: productsIds[i], - Properties: p, - } - - err := index.putObject(context.TODO(), storobj.FromObject( - &product, []float32{0.1, 0.2, 0.01, 0.2}), nil) - require.Nil(t, err) - } - - indexFilesBeforeDelete, err := getIndexFilenames(dirName, class.Class) - require.Nil(t, err) - - beforeDeleteObj1, err := index.objectByID(context.TODO(), - productsIds[0], nil, additional.Properties{}, nil, "") - require.Nil(t, err) - - beforeDeleteObj2, err := index.objectByID(context.TODO(), - productsIds[1], nil, additional.Properties{}, nil, "") - require.Nil(t, err) - - // drop the index - err = index.drop() - require.Nil(t, err) - - indexFilesAfterDelete, err := getIndexFilenames(dirName, class.Class) - require.Nil(t, err) - - // recreate the index - index, err = NewIndex(testCtx(), IndexConfig{ - RootPath: dirName, - ClassName: schema.ClassName(class.Class), - }, shardState, inverted.ConfigFromModel(class.InvertedIndexConfig), - hnsw.NewDefaultUserConfig(), &fakeSchemaGetter{ - schema: fakeSchema, - shardState: shardState, - }, nil, logger, nil, nil, nil, nil, class, nil, nil) - require.Nil(t, err) - - err = index.addUUIDProperty(context.TODO()) - require.Nil(t, err) - err = index.addProperty(context.TODO(), &models.Property{ - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }) - require.Nil(t, err) - - indexFilesAfterRecreate, err := getIndexFilenames(dirName, class.Class) - require.Nil(t, err) - - afterRecreateObj1, err := index.objectByID(context.TODO(), - productsIds[0], nil, additional.Properties{}, nil, "") - require.Nil(t, err) - - afterRecreateObj2, err := index.objectByID(context.TODO(), - productsIds[1], nil, additional.Properties{}, nil, "") - require.Nil(t, err) - - // insert some data in the recreated index - for i, p := range products { - thing := models.Object{ - Class: class.Class, - ID: productsIds[i], - Properties: p, - } - - err := index.putObject(context.TODO(), storobj.FromObject( - &thing, []float32{0.1, 0.2, 0.01, 0.2}), nil) - require.Nil(t, err) - } - - afterRecreateAndInsertObj1, err := index.objectByID(context.TODO(), - productsIds[0], nil, additional.Properties{}, nil, "") - require.Nil(t, err) - - afterRecreateAndInsertObj2, err := index.objectByID(context.TODO(), - productsIds[1], nil, additional.Properties{}, nil, "") - require.Nil(t, err) - - // update the index vectorIndexUserConfig - beforeVectorConfig, ok := index.vectorIndexUserConfig.(hnsw.UserConfig) - require.Equal(t, -1, beforeVectorConfig.EF) - require.True(t, ok) - beforeVectorConfig.EF = 99 - err = index.updateVectorIndexConfig(context.TODO(), beforeVectorConfig) - require.Nil(t, err) - afterVectorConfig, ok := index.vectorIndexUserConfig.(hnsw.UserConfig) - require.True(t, ok) - require.Equal(t, 99, afterVectorConfig.EF) - - assert.Equal(t, 6, len(indexFilesBeforeDelete)) - assert.Equal(t, 0, len(indexFilesAfterDelete)) - assert.Equal(t, 6, len(indexFilesAfterRecreate)) - assert.Equal(t, indexFilesBeforeDelete, indexFilesAfterRecreate) - assert.NotNil(t, beforeDeleteObj1) - assert.NotNil(t, beforeDeleteObj2) - assert.Empty(t, afterRecreateObj1) - assert.Empty(t, afterRecreateObj2) - assert.NotNil(t, afterRecreateAndInsertObj1) - assert.NotNil(t, afterRecreateAndInsertObj2) -} - -func TestIndex_DropReadOnlyEmptyIndex(t *testing.T) { - ctx := testCtx() - class := &models.Class{Class: "deletetest"} - shard, index := testShard(t, ctx, class.Class) - - err := index.updateShardStatus(ctx, shard.Name(), storagestate.StatusReadOnly.String()) - require.Nil(t, err) - - err = index.drop() - require.Nil(t, err) -} - -func TestIndex_DropReadOnlyIndexWithData(t *testing.T) { - ctx := testCtx() - dirName := t.TempDir() - logger, _ := test.NewNullLogger() - class := &models.Class{ - Class: "deletetest", - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - InvertedIndexConfig: &models.InvertedIndexConfig{}, - } - fakeSchema := schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{ - class, - }, - }, - } - - shardState := singleShardState() - index, err := NewIndex(ctx, IndexConfig{ - RootPath: dirName, - ClassName: schema.ClassName(class.Class), - }, shardState, inverted.ConfigFromModel(class.InvertedIndexConfig), - hnsw.NewDefaultUserConfig(), &fakeSchemaGetter{ - schema: fakeSchema, shardState: shardState, - }, nil, logger, nil, nil, nil, nil, class, nil, nil) - require.Nil(t, err) - - productsIds := []strfmt.UUID{ - "1295c052-263d-4aae-99dd-920c5a370d06", - "1295c052-263d-4aae-99dd-920c5a370d07", - } - - products := []map[string]interface{}{ - {"name": "one"}, - {"name": "two"}, - } - - err = index.addUUIDProperty(ctx) - require.Nil(t, err) - - err = index.addProperty(ctx, &models.Property{ - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }) - require.Nil(t, err) - - for i, p := range products { - product := models.Object{ - Class: class.Class, - ID: productsIds[i], - Properties: p, - } - - err := index.putObject(ctx, storobj.FromObject( - &product, []float32{0.1, 0.2, 0.01, 0.2}), nil) - require.Nil(t, err) - } - - // set all shards to readonly - index.ForEachShard(func(name string, shard ShardLike) error { - err = shard.UpdateStatus(storagestate.StatusReadOnly.String()) - require.Nil(t, err) - return nil - }) - - err = index.drop() - require.Nil(t, err) -} - -func emptyIdx(t *testing.T, rootDir string, class *models.Class) *Index { - logger, _ := test.NewNullLogger() - shardState := singleShardState() - - idx, err := NewIndex(testCtx(), IndexConfig{ - RootPath: rootDir, - ClassName: schema.ClassName(class.Class), - DisableLazyLoadShards: true, - }, shardState, inverted.ConfigFromModel(invertedConfig()), - hnsw.NewDefaultUserConfig(), &fakeSchemaGetter{ - shardState: shardState, - }, nil, logger, nil, nil, nil, nil, class, nil, nil) - require.Nil(t, err) - return idx -} - -func invertedConfig() *models.InvertedIndexConfig { - return &models.InvertedIndexConfig{ - CleanupIntervalSeconds: 60, - Stopwords: &models.StopwordConfig{ - Preset: "none", - }, - IndexNullState: true, - IndexPropertyLength: true, - } -} - -func getIndexFilenames(rootDir, indexName string) ([]string, error) { - var filenames []string - indexRoot, err := os.ReadDir(path.Join(rootDir, indexName)) - if err != nil { - if os.IsNotExist(err) { - // index was dropped, or never existed - return filenames, nil - } - return nil, err - } - shardFiles, err := os.ReadDir(path.Join(rootDir, indexName, indexRoot[0].Name())) - if err != nil { - return filenames, err - } - for _, f := range shardFiles { - filenames = append(filenames, f.Name()) - } - return filenames, nil -} diff --git a/adapters/repos/db/index_queue.go b/adapters/repos/db/index_queue.go deleted file mode 100644 index cf07b10c22209220adefd810f212163741a9ed30..0000000000000000000000000000000000000000 --- a/adapters/repos/db/index_queue.go +++ /dev/null @@ -1,994 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "container/list" - "context" - "encoding/binary" - "math" - "runtime" - "sync" - "sync/atomic" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/adapters/repos/db/indexcheckpoint" - "github.com/weaviate/weaviate/adapters/repos/db/priorityqueue" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" - "github.com/weaviate/weaviate/entities/storagestate" - "github.com/weaviate/weaviate/entities/storobj" -) - -// IndexQueue is an in-memory queue of vectors to index. -// It batches vectors together before sending them to the indexing workers. -// It is safe to use concurrently. -type IndexQueue struct { - Shard shardStatusUpdater - Index batchIndexer - shardID string - - IndexQueueOptions - - // indexCh is the channel used to send vectors to the shared indexing workers. - indexCh chan job - - // context used to close pending tasks - // if canceled, prevents new vectors from being added to the queue. - ctx context.Context - cancelFn context.CancelFunc - - // tracks the background workers - wg sync.WaitGroup - - // queue of not-yet-indexed vectors - queue *vectorQueue - - // keeps track of the last call to Push() - lastPushed atomic.Pointer[time.Time] - - pqMaxPool *pqMaxPool - - checkpoints *indexcheckpoint.Checkpoints - - paused atomic.Bool -} - -type vectorDescriptor struct { - id uint64 - vector []float32 -} - -type IndexQueueOptions struct { - // BatchSize is the number of vectors to batch together - // before sending them to the indexing worker. - BatchSize int - - // IndexInterval is the maximum time to wait before sending - // the pending vectors to the indexing worker. - IndexInterval time.Duration - - // Max time a vector can stay in the queue before being indexed. - StaleTimeout time.Duration - - // Logger is the logger used by the queue. - Logger logrus.FieldLogger - - // Maximum number of vectors to use for brute force search - // when vectors are not indexed. - BruteForceSearchLimit int -} - -type batchIndexer interface { - AddBatch(ctx context.Context, id []uint64, vector [][]float32) error - SearchByVector(vector []float32, k int, allowList helpers.AllowList) ([]uint64, []float32, error) - SearchByVectorDistance(vector []float32, dist float32, - maxLimit int64, allow helpers.AllowList) ([]uint64, []float32, error) - DistanceBetweenVectors(x, y []float32) (float32, bool, error) - ContainsNode(id uint64) bool - Delete(id ...uint64) error - DistancerProvider() distancer.Provider -} - -type compressedIndexer interface { - Compressed() bool - AlreadyIndexed() uint64 - TurnOnCompression(callback func()) error - ShouldCompress() (bool, int) -} - -type shardStatusUpdater interface { - compareAndSwapStatus(old, new string) (storagestate.Status, error) -} - -func NewIndexQueue( - shardID string, - shard shardStatusUpdater, - index batchIndexer, - centralJobQueue chan job, - checkpoints *indexcheckpoint.Checkpoints, - opts IndexQueueOptions, -) (*IndexQueue, error) { - if opts.Logger == nil { - opts.Logger = logrus.New() - } - opts.Logger = opts.Logger.WithField("component", "index_queue") - - if opts.BatchSize == 0 { - opts.BatchSize = 1000 - } - - if opts.IndexInterval == 0 { - opts.IndexInterval = 1 * time.Second - } - - if opts.BruteForceSearchLimit == 0 { - opts.BruteForceSearchLimit = 100_000 - } - - if opts.StaleTimeout == 0 { - opts.StaleTimeout = 1 * time.Minute - } - - q := IndexQueue{ - shardID: shardID, - IndexQueueOptions: opts, - Shard: shard, - Index: index, - indexCh: centralJobQueue, - pqMaxPool: newPqMaxPool(0), - checkpoints: checkpoints, - } - - q.queue = newVectorQueue(&q) - - q.ctx, q.cancelFn = context.WithCancel(context.Background()) - - if !asyncEnabled() { - return &q, nil - } - - q.wg.Add(1) - go func() { - defer q.wg.Done() - - q.indexer() - }() - - return &q, nil -} - -// Close immediately closes the queue and waits for workers to finish their current tasks. -// Any pending vectors are discarded. -func (q *IndexQueue) Close() error { - // check if the queue is closed - if q.ctx.Err() != nil { - return nil - } - - // prevent new jobs from being added - q.cancelFn() - - q.wg.Wait() - - // loop over the chunks of the queue - // wait for the done chan to be closed - // then return - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - q.queue.wait(ctx) - - return nil -} - -// Push adds a list of vectors to the queue. -func (q *IndexQueue) Push(ctx context.Context, vectors ...vectorDescriptor) error { - if ctx.Err() != nil { - return ctx.Err() - } - if q.ctx.Err() != nil { - return errors.New("index queue closed") - } - - // store the time of the last push - now := time.Now() - q.lastPushed.Store(&now) - - q.queue.Add(vectors) - return nil -} - -// Size returns the number of vectors waiting to be indexed. -func (q *IndexQueue) Size() int64 { - var count int64 - q.queue.fullChunks.Lock() - e := q.queue.fullChunks.list.Front() - for e != nil { - c := e.Value.(*chunk) - count += int64(c.cursor) - - e = e.Next() - } - q.queue.fullChunks.Unlock() - - q.queue.curBatch.Lock() - if q.queue.curBatch.c != nil { - count += int64(q.queue.curBatch.c.cursor) - } - q.queue.curBatch.Unlock() - - return count -} - -// Delete marks the given vectors as deleted. -// This method can be called even if the async indexing is disabled. -func (q *IndexQueue) Delete(ids ...uint64) error { - if !asyncEnabled() { - return q.Index.Delete(ids...) - } - - remaining := make([]uint64, 0, len(ids)) - indexed := make([]uint64, 0, len(ids)) - - for _, id := range ids { - if q.Index.ContainsNode(id) { - indexed = append(indexed, id) - - // is it already marked as deleted in the queue? - if q.queue.IsDeleted(id) { - q.queue.ResetDeleted(id) - } - - continue - } - - remaining = append(remaining, id) - } - - err := q.Index.Delete(indexed...) - if err != nil { - return errors.Wrap(err, "delete node from index") - } - - q.queue.Delete(remaining) - - return nil -} - -// PreloadShard goes through the LSM store from the last checkpoint -// and enqueues any unindexed vector. -func (q *IndexQueue) PreloadShard(shard ShardLike) error { - if !asyncEnabled() { - return nil - } - - // load non-indexed vectors and add them to the queue - checkpoint, err := q.checkpoints.Get(q.shardID) - if err != nil { - return errors.Wrap(err, "get last indexed id") - } - if checkpoint == 0 { - return nil - } - - start := time.Now() - - maxDocID := shard.Counter().Get() - - var counter int - - ctx := context.Background() - - buf := make([]byte, 8) - for i := checkpoint; i < maxDocID; i++ { - binary.LittleEndian.PutUint64(buf, i) - - v, err := shard.Store().Bucket(helpers.ObjectsBucketLSM).GetBySecondary(0, buf) - if err != nil { - return errors.Wrap(err, "get last indexed object") - } - if v == nil { - continue - } - obj, err := storobj.FromBinary(v) - if err != nil { - return errors.Wrap(err, "unmarshal last indexed object") - } - id := obj.DocID() - if shard.VectorIndex().ContainsNode(id) { - continue - } - if len(obj.Vector) == 0 { - continue - } - counter++ - - desc := vectorDescriptor{ - id: id, - vector: obj.Vector, - } - err = q.Push(ctx, desc) - if err != nil { - return err - } - } - - q.Logger. - WithField("checkpoint", checkpoint). - WithField("last_stored_id", maxDocID). - WithField("count", counter). - WithField("took", time.Since(start)). - WithField("shard_id", q.shardID). - Debug("enqueued vectors from last indexed checkpoint") - - return nil -} - -// Drop removes all persisted data related to the queue. -// It closes the queue if not already. -// It does not remove the index. -// It should be called only when the index is dropped. -func (q *IndexQueue) Drop() error { - _ = q.Close() - - if q.checkpoints != nil { - return q.checkpoints.Delete(q.shardID) - } - - return nil -} - -func (q *IndexQueue) indexer() { - t := time.NewTicker(q.IndexInterval) - - workerNb := runtime.GOMAXPROCS(0) - 1 - - for { - select { - case <-t.C: - if q.Size() == 0 { - _, _ = q.Shard.compareAndSwapStatus(storagestate.StatusIndexing.String(), storagestate.StatusReady.String()) - continue - } - if q.paused.Load() { - continue - } - status, err := q.Shard.compareAndSwapStatus(storagestate.StatusReady.String(), storagestate.StatusIndexing.String()) - if status != storagestate.StatusIndexing || err != nil { - q.Logger.WithField("status", status).WithError(err).Warn("failed to set shard status to 'indexing', trying again in " + q.IndexInterval.String()) - continue - } - - lastPushed := q.lastPushed.Load() - if lastPushed == nil || time.Since(*lastPushed) > time.Second { - // send at most 2 times the number of workers in one go, - // then wait for the next tick in case more vectors - // are added to the queue - q.pushToWorkers(2*workerNb, false) - } else { - // send only one batch at a time and wait for it to be indexed - // to avoid competing for resources with the Push() method. - // This ensures the resources are used for queueing vectors in priority, - // then for indexing them. - q.pushToWorkers(1, true) - } - q.checkCompressionSettings() - case <-q.ctx.Done(): - // stop the ticker - t.Stop() - return - } - } -} - -func (q *IndexQueue) pushToWorkers(max int, wait bool) { - chunks := q.queue.borrowChunks(max) - for i, c := range chunks { - select { - case <-q.ctx.Done(): - // release unsent borrowed chunks - for _, c := range chunks[i:] { - q.queue.releaseChunk(c) - } - - return - case q.indexCh <- job{ - chunk: c, - indexer: q.Index, - queue: q.queue, - ctx: q.ctx, - }: - } - } - - if wait { - q.queue.wait(q.ctx) - } -} - -// SearchByVector performs the search through the index first, then uses brute force to -// query unindexed vectors. -func (q *IndexQueue) SearchByVector(vector []float32, k int, allowList helpers.AllowList) ([]uint64, []float32, error) { - return q.search(vector, -1, k, allowList) -} - -// SearchByVectorDistance performs the search through the index first, then uses brute force to -// query unindexed vectors. -func (q *IndexQueue) SearchByVectorDistance(vector []float32, dist float32, maxLimit int64, allowList helpers.AllowList) ([]uint64, []float32, error) { - return q.search(vector, dist, int(maxLimit), allowList) -} - -func (q *IndexQueue) search(vector []float32, dist float32, maxLimit int, allowList helpers.AllowList) ([]uint64, []float32, error) { - var indexedResults []uint64 - var distances []float32 - var err error - if dist == -1 { - indexedResults, distances, err = q.Index.SearchByVector(vector, maxLimit, allowList) - } else { - indexedResults, distances, err = q.Index.SearchByVectorDistance(vector, dist, int64(maxLimit), allowList) - } - if err != nil { - return nil, nil, err - } - - if !asyncEnabled() { - return indexedResults, distances, nil - } - - if q.Index.DistancerProvider().Type() == "cosine-dot" { - // cosine-dot requires normalized vectors, as the dot product and cosine - // similarity are only identical if the vector is normalized - vector = distancer.Normalize(vector) - } - - var results *priorityqueue.Queue[any] - var seen map[uint64]struct{} - - err = q.queue.Iterate(allowList, func(objects []vectorDescriptor) error { - if results == nil { - results = q.pqMaxPool.GetMax(maxLimit) - seen = make(map[uint64]struct{}, len(indexedResults)) - for i := range indexedResults { - seen[indexedResults[i]] = struct{}{} - results.Insert(indexedResults[i], distances[i]) - } - } - - return q.bruteForce(vector, objects, maxLimit, results, allowList, dist, seen) - }) - if results != nil { - defer q.pqMaxPool.Put(results) - } - if err != nil { - return nil, nil, err - } - if results == nil { - return indexedResults, distances, nil - } - - ids := make([]uint64, results.Len()) - dists := make([]float32, results.Len()) - - i := results.Len() - 1 - for results.Len() > 0 { - element := results.Pop() - ids[i] = element.ID - dists[i] = element.Dist - i-- - } - - return ids, dists, nil -} - -func (q *IndexQueue) checkCompressionSettings() { - ci, ok := q.Index.(compressedIndexer) - if !ok { - return - } - - shouldCompress, shouldCompressAt := ci.ShouldCompress() - if !shouldCompress || ci.Compressed() { - return - } - - if ci.AlreadyIndexed() > uint64(shouldCompressAt) { - q.pauseIndexing() - err := ci.TurnOnCompression(q.resumeIndexing) - if err != nil { - q.Logger.WithError(err).Error("failed to turn on compression") - } - } -} - -// pause indexing and wait for the workers to finish their current tasks -// related to this queue. -func (q *IndexQueue) pauseIndexing() { - q.Logger.Debug("pausing indexing, waiting for the current tasks to finish") - q.paused.Store(true) - q.queue.wait(q.ctx) - q.Logger.Debug("indexing paused") -} - -// resume indexing -func (q *IndexQueue) resumeIndexing() { - q.paused.Store(false) - q.Logger.Debug("indexing resumed") -} - -func (q *IndexQueue) bruteForce(vector []float32, snapshot []vectorDescriptor, k int, - results *priorityqueue.Queue[any], allowList helpers.AllowList, - maxDistance float32, seen map[uint64]struct{}, -) error { - for i := range snapshot { - // skip indexed data - if _, ok := seen[snapshot[i].id]; ok { - continue - } - - // skip filtered data - if allowList != nil && !allowList.Contains(snapshot[i].id) { - continue - } - - v := snapshot[i].vector - if q.Index.DistancerProvider().Type() == "cosine-dot" { - // cosine-dot requires normalized vectors, as the dot product and cosine - // similarity are only identical if the vector is normalized - v = distancer.Normalize(v) - } - - dist, _, err := q.Index.DistanceBetweenVectors(vector, v) - if err != nil { - return err - } - - // skip vectors that are too far away - if maxDistance > 0 && dist > maxDistance { - continue - } - - if k < 0 || results.Len() < k || dist < results.Top().Dist { - results.Insert(snapshot[i].id, dist) - if k > 0 { - for results.Len() > k { - results.Pop() - } - } - } - } - return nil -} - -type pqMaxPool struct { - pool *sync.Pool -} - -func newPqMaxPool(defaultCap int) *pqMaxPool { - return &pqMaxPool{ - pool: &sync.Pool{ - New: func() interface{} { - return priorityqueue.NewMax[any](defaultCap) - }, - }, - } -} - -func (pqh *pqMaxPool) GetMax(capacity int) *priorityqueue.Queue[any] { - pq := pqh.pool.Get().(*priorityqueue.Queue[any]) - if pq.Cap() < capacity { - pq.ResetCap(capacity) - } else { - pq.Reset() - } - - return pq -} - -func (pqh *pqMaxPool) Put(pq *priorityqueue.Queue[any]) { - pqh.pool.Put(pq) -} - -type vectorQueue struct { - IndexQueue *IndexQueue - pool sync.Pool - curBatch struct { - sync.Mutex - - c *chunk - } - fullChunks struct { - sync.Mutex - - list *list.List - } - deleted struct { - sync.RWMutex - - m map[uint64]struct{} - } -} - -func newVectorQueue(iq *IndexQueue) *vectorQueue { - q := vectorQueue{ - IndexQueue: iq, - pool: sync.Pool{ - New: func() any { - buf := make([]vectorDescriptor, iq.BatchSize) - return &buf - }, - }, - } - - q.fullChunks.list = list.New() - q.deleted.m = make(map[uint64]struct{}) - - return &q -} - -func (q *vectorQueue) getBuffer() []vectorDescriptor { - buff := *(q.pool.Get().(*[]vectorDescriptor)) - return buff[:q.IndexQueue.BatchSize] -} - -func (q *vectorQueue) getFreeChunk() *chunk { - c := chunk{ - data: q.getBuffer(), - } - c.indexed = make(chan struct{}) - return &c -} - -func (q *vectorQueue) wait(ctx context.Context) { - for { - // get first non-closed channel - var ch chan struct{} - - q.fullChunks.Lock() - e := q.fullChunks.list.Front() - LOOP: - for e != nil { - c := e.Value.(*chunk) - if c.borrowed { - select { - case <-c.indexed: - default: - ch = c.indexed - break LOOP - } - } - - e = e.Next() - } - q.fullChunks.Unlock() - - if ch == nil { - return - } - - select { - case <-ch: - case <-time.After(5 * time.Second): - case <-ctx.Done(): - return - } - } -} - -func (q *vectorQueue) Add(vectors []vectorDescriptor) { - var full []*chunk - - q.curBatch.Lock() - f := q.ensureHasSpace() - if f != nil { - full = append(full, f) - } - - for len(vectors) != 0 { - curBatch := q.curBatch.c - n := copy(curBatch.data[curBatch.cursor:], vectors) - curBatch.cursor += n - - vectors = vectors[n:] - - f := q.ensureHasSpace() - if f != nil { - full = append(full, f) - } - } - q.curBatch.Unlock() - - if len(full) > 0 { - q.fullChunks.Lock() - for _, f := range full { - f.elem = q.fullChunks.list.PushBack(f) - } - q.fullChunks.Unlock() - } -} - -func (q *vectorQueue) ensureHasSpace() *chunk { - if q.curBatch.c == nil { - q.curBatch.c = q.getFreeChunk() - } - - if q.curBatch.c.cursor == 0 { - now := time.Now() - q.curBatch.c.createdAt = &now - } - - if q.curBatch.c.cursor < q.IndexQueue.BatchSize { - return nil - } - - c := q.curBatch.c - q.curBatch.c = q.getFreeChunk() - now := time.Now() - q.curBatch.c.createdAt = &now - return c -} - -func (q *vectorQueue) borrowChunks(max int) []*chunk { - if max <= 0 { - max = math.MaxInt64 - } - - q.fullChunks.Lock() - var chunks []*chunk - e := q.fullChunks.list.Front() - count := 0 - for e != nil && count < max { - c := e.Value.(*chunk) - if !c.borrowed { - count++ - c.borrowed = true - chunks = append(chunks, c) - } - - e = e.Next() - } - q.fullChunks.Unlock() - - if count < max { - var incompleteChunk *chunk - q.curBatch.Lock() - if q.curBatch.c != nil && time.Since(*q.curBatch.c.createdAt) > q.IndexQueue.StaleTimeout && q.curBatch.c.cursor > 0 { - q.curBatch.c.borrowed = true - chunks = append(chunks, q.curBatch.c) - incompleteChunk = q.curBatch.c - q.curBatch.c = nil - } - q.curBatch.Unlock() - - // add the incomplete chunk to the full chunks list - if incompleteChunk != nil { - q.fullChunks.Lock() - q.fullChunks.list.PushBack(incompleteChunk) - q.fullChunks.Unlock() - } - } - - return chunks -} - -func (q *vectorQueue) releaseChunk(c *chunk) { - if c == nil { - return - } - - if c.indexed != nil { - close(c.indexed) - } - - q.fullChunks.Lock() - if c.elem != nil { - q.fullChunks.list.Remove(c.elem) - } - - // reset the chunk to notify the search - // that it was released - c.borrowed = false - c.cursor = 0 - c.elem = nil - c.createdAt = nil - c.indexed = nil - data := c.data - c.data = nil - - q.fullChunks.Unlock() - - if len(data) == q.IndexQueue.BatchSize { - q.pool.Put(&data) - } -} - -// persistCheckpoint update the on-disk checkpoint that tracks the last indexed id -// optimistically. It is not guaranteed to be accurate but it is guaranteed to be lower -// than any vector in the queue. -// To calculate the checkpoint, we use the lowest id in the current batch -// minus the number of vectors in the queue (delta), which is capped at 10k vectors. -// The calculation looks like this: -// checkpoint = min(ids) - max(queueSize, 10_000) -func (q *vectorQueue) persistCheckpoint(ids []uint64) { - if len(ids) == 0 { - return - } - - q.fullChunks.Lock() - cl := q.fullChunks.list.Len() - q.fullChunks.Unlock() - - // get the lowest id in the current batch - var minID uint64 - for _, id := range ids { - if minID == 0 || id < minID { - minID = id - } - } - - delta := uint64(cl * q.IndexQueue.BatchSize) - // cap the delta to 10k vectors - if delta > 10_000 { - delta = 10_000 - } - var checkpoint uint64 - if minID > delta { - checkpoint = minID - delta - } else { - checkpoint = 0 - } - - err := q.IndexQueue.checkpoints.Update(q.IndexQueue.shardID, checkpoint) - if err != nil { - q.IndexQueue.Logger.WithError(err).Error("update checkpoint") - } -} - -// Iterate through all chunks in the queue and call the given function. -// Deleted vectors are skipped, and if an allowlist is provided, only vectors -// in the allowlist are returned. -func (q *vectorQueue) Iterate(allowlist helpers.AllowList, fn func(objects []vectorDescriptor) error) error { - buf := q.getBuffer() - defer q.pool.Put(&buf) - - var count int - - // since chunks can get released concurrently, - // we first get the pointers to all chunks. - // then iterate over them. - // This will not give us the latest data, but - // will prevent us from losing access to the rest - // of the linked list if an intermediate chunk is released. - var elems []*list.Element - q.fullChunks.Lock() - e := q.fullChunks.list.Front() - for e != nil { - elems = append(elems, e) - e = e.Next() - } - q.fullChunks.Unlock() - - for i := 0; i < len(elems); i++ { - // we need to lock the list to prevent the chunk from being released - q.fullChunks.Lock() - c := elems[i].Value.(*chunk) - if c.data == nil { - // the chunk was released in the meantime, - // skip it - q.fullChunks.Unlock() - continue - } - - buf = buf[:0] - for i := 0; i < c.cursor; i++ { - if allowlist != nil && !allowlist.Contains(c.data[i].id) { - continue - } - - if q.IsDeleted(c.data[i].id) { - continue - } - - buf = append(buf, c.data[i]) - count++ - if count >= q.IndexQueue.BruteForceSearchLimit { - break - } - } - q.fullChunks.Unlock() - - if len(buf) == 0 { - continue - } - - err := fn(buf) - if err != nil { - return err - } - - if count >= q.IndexQueue.BruteForceSearchLimit { - break - } - } - - if count >= q.IndexQueue.BruteForceSearchLimit { - return nil - } - - buf = buf[:0] - q.curBatch.Lock() - if q.curBatch.c != nil { - for i := 0; i < q.curBatch.c.cursor; i++ { - c := q.curBatch.c - - if allowlist != nil && !allowlist.Contains(c.data[i].id) { - continue - } - - if q.IsDeleted(c.data[i].id) { - continue - } - - buf = append(buf, c.data[i]) - count++ - - if count >= q.IndexQueue.BruteForceSearchLimit { - break - } - } - } - q.curBatch.Unlock() - - if len(buf) == 0 { - return nil - } - - err := fn(buf) - if err != nil { - return err - } - - return nil -} - -func (q *vectorQueue) Delete(ids []uint64) { - q.deleted.Lock() - for _, id := range ids { - q.deleted.m[id] = struct{}{} - } - q.deleted.Unlock() -} - -func (q *vectorQueue) IsDeleted(id uint64) bool { - q.deleted.RLock() - _, ok := q.deleted.m[id] - q.deleted.RUnlock() - return ok -} - -func (q *vectorQueue) ResetDeleted(id ...uint64) { - q.deleted.Lock() - for _, id := range id { - delete(q.deleted.m, id) - } - q.deleted.Unlock() -} - -type chunk struct { - cursor int - borrowed bool - data []vectorDescriptor - elem *list.Element - createdAt *time.Time - indexed chan struct{} -} diff --git a/adapters/repos/db/index_queue_test.go b/adapters/repos/db/index_queue_test.go deleted file mode 100644 index 78bb673d59c4ac5bf958a935466550bdfeae2b23..0000000000000000000000000000000000000000 --- a/adapters/repos/db/index_queue_test.go +++ /dev/null @@ -1,1003 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "context" - "fmt" - "math/rand" - "os" - "sort" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/sirupsen/logrus" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/adapters/repos/db/indexcheckpoint" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv" - "github.com/weaviate/weaviate/adapters/repos/db/vector/common" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" - "github.com/weaviate/weaviate/entities/cyclemanager" - "github.com/weaviate/weaviate/entities/storagestate" - ent "github.com/weaviate/weaviate/entities/vectorindex/hnsw" -) - -func startWorker(t testing.TB, retryInterval ...time.Duration) chan job { - t.Helper() - ch := make(chan job) - t.Cleanup(func() { - close(ch) - }) - - itv := time.Millisecond - if len(retryInterval) > 0 { - itv = retryInterval[0] - } - - go func() { - logger := logrus.New() - logger.Level = logrus.ErrorLevel - asyncWorker(ch, logger, itv) - }() - - return ch -} - -func newCheckpointManager(t testing.TB) *indexcheckpoint.Checkpoints { - t.Helper() - - return newCheckpointManagerWithDir(t, t.TempDir()) -} - -func newCheckpointManagerWithDir(t testing.TB, dir string) *indexcheckpoint.Checkpoints { - t.Helper() - - c, err := indexcheckpoint.New(dir, logrus.New()) - require.NoError(t, err) - - return c -} - -func pushVector(t testing.TB, ctx context.Context, q *IndexQueue, id uint64, vector []float32) { - err := q.Push(ctx, vectorDescriptor{ - id: id, - vector: vector, - }) - require.NoError(t, err) -} - -func randVector(dim int) []float32 { - vec := make([]float32, dim) - for i := range vec { - vec[i] = rand.Float32() - } - - return vec -} - -func TestIndexQueue(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - - os.Setenv("ASYNC_INDEXING", "true") - defer os.Unsetenv("ASYNC_INDEXING") - - writeIDs := func(q *IndexQueue, from, to uint64) { - vectors := make([]vectorDescriptor, 0, to-from) - for i := from; i < to; i++ { - vectors = append(vectors, vectorDescriptor{ - id: i, - vector: []float32{1, 2, 3}, - }) - } - err := q.Push(ctx, vectors...) - require.NoError(t, err) - } - - getLastUpdate := func(q *IndexQueue) time.Time { - fi, err := os.Stat(q.checkpoints.Filename()) - require.NoError(t, err) - return fi.ModTime() - } - - waitForUpdate := func(q *IndexQueue) func(timeout ...time.Duration) bool { - lastUpdate := getLastUpdate(q) - - return func(timeout ...time.Duration) bool { - start := time.Now() - - if len(timeout) == 0 { - timeout = []time.Duration{500 * time.Millisecond} - } - for { - cur := getLastUpdate(q) - if cur.Equal(lastUpdate) { - if time.Since(start) > timeout[0] { - return false - } - time.Sleep(5 * time.Millisecond) - continue - } - - lastUpdate = cur - return true - } - } - } - - t.Run("pushes to indexer if batch is full", func(t *testing.T) { - var idx mockBatchIndexer - idsCh := make(chan []uint64, 1) - idx.addBatchFn = func(ids []uint64, vector [][]float32) error { - idsCh <- ids - return nil - } - - q, err := NewIndexQueue("1", new(mockShard), &idx, startWorker(t), newCheckpointManager(t), IndexQueueOptions{ - BatchSize: 2, - }) - require.NoError(t, err) - defer q.Close() - - pushVector(t, ctx, q, 1, []float32{1, 2, 3}) - select { - case <-idsCh: - t.Fatal("should not have been called") - case <-time.After(100 * time.Millisecond): - } - - pushVector(t, ctx, q, 2, []float32{4, 5, 6}) - ids := <-idsCh - - require.Equal(t, []uint64{1, 2}, ids) - }) - - t.Run("doesn't index if batch is not null", func(t *testing.T) { - var idx mockBatchIndexer - called := make(chan struct{}) - idx.addBatchFn = func(id []uint64, vector [][]float32) error { - called <- struct{}{} - return nil - } - q, err := NewIndexQueue("1", new(mockShard), &idx, startWorker(t), newCheckpointManager(t), IndexQueueOptions{ - BatchSize: 100, - IndexInterval: time.Microsecond, - }) - require.NoError(t, err) - defer q.Close() - - pushVector(t, ctx, q, 1, []float32{1, 2, 3}) - select { - case <-called: - t.Fatal("should not have been called") - case <-time.After(100 * time.Millisecond): - } - - pushVector(t, ctx, q, 2, []float32{4, 5, 6}) - - select { - case <-called: - t.Fatal("should not have been called") - case <-time.After(100 * time.Millisecond): - } - }) - - t.Run("retry on indexing error", func(t *testing.T) { - var idx mockBatchIndexer - i := int32(0) - called := make(chan struct{}) - idx.addBatchFn = func(id []uint64, vector [][]float32) error { - if atomic.AddInt32(&i, 1) < 3 { - return fmt.Errorf("indexing error: %d", i) - } - - close(called) - - return nil - } - - q, err := NewIndexQueue("1", new(mockShard), &idx, startWorker(t), newCheckpointManager(t), IndexQueueOptions{ - BatchSize: 1, - }) - require.NoError(t, err) - defer q.Close() - - pushVector(t, ctx, q, 1, []float32{1, 2, 3}) - <-called - }) - - t.Run("merges results from queries", func(t *testing.T) { - var idx mockBatchIndexer - called := make(chan struct{}) - idx.addBatchFn = func(id []uint64, vector [][]float32) error { - close(called) - return nil - } - - q, err := NewIndexQueue("1", new(mockShard), &idx, startWorker(t), newCheckpointManager(t), IndexQueueOptions{ - BatchSize: 3, - IndexInterval: 100 * time.Millisecond, - }) - require.NoError(t, err) - defer q.Close() - - pushVector(t, ctx, q, 1, []float32{1, 2, 3}) - pushVector(t, ctx, q, 2, []float32{4, 5, 6}) - pushVector(t, ctx, q, 3, []float32{7, 8, 9}) - pushVector(t, ctx, q, 4, []float32{1, 2, 3}) - - <-called - - time.Sleep(500 * time.Millisecond) - res, _, err := q.SearchByVector([]float32{1, 2, 3}, 2, nil) - require.NoError(t, err) - require.Equal(t, []uint64{1, 4}, res) - }) - - t.Run("search with empty index", func(t *testing.T) { - var idx mockBatchIndexer - - q, err := NewIndexQueue("1", new(mockShard), &idx, startWorker(t), newCheckpointManager(t), IndexQueueOptions{ - BatchSize: 6, - }) - require.NoError(t, err) - defer q.Close() - - for i := 0; i < 10; i++ { - pushVector(t, ctx, q, uint64(i+1), []float32{float32(i) + 1, float32(i) + 2, float32(i) + 3}) - } - - res, _, err := q.SearchByVector([]float32{1, 2, 3}, 2, nil) - require.NoError(t, err) - require.Equal(t, []uint64{1, 2}, res) - }) - - t.Run("queue size", func(t *testing.T) { - var idx mockBatchIndexer - closeCh := make(chan struct{}) - idx.addBatchFn = func(id []uint64, vector [][]float32) error { - <-closeCh - return nil - } - - q, err := NewIndexQueue("1", new(mockShard), &idx, startWorker(t), newCheckpointManager(t), IndexQueueOptions{ - BatchSize: 5, - }) - require.NoError(t, err) - defer q.Close() - - for i := uint64(0); i < 101; i++ { - pushVector(t, ctx, q, i+1, []float32{1, 2, 3}) - } - - time.Sleep(100 * time.Millisecond) - require.EqualValues(t, 101, q.Size()) - close(closeCh) - }) - - t.Run("deletion", func(t *testing.T) { - var idx mockBatchIndexer - var count int32 - indexingDone := make(chan struct{}) - idx.addBatchFn = func(id []uint64, vector [][]float32) error { - if atomic.AddInt32(&count, 1) == 5 { - close(indexingDone) - } - - return nil - } - - q, err := NewIndexQueue("1", new(mockShard), &idx, startWorker(t), newCheckpointManager(t), IndexQueueOptions{ - BatchSize: 4, - IndexInterval: 100 * time.Millisecond, - }) - require.NoError(t, err) - defer q.Close() - - for i := uint64(0); i < 20; i++ { - pushVector(t, ctx, q, i, []float32{1, 2, 3}) - } - - err = q.Delete(5, 10, 15) - require.NoError(t, err) - - wait := waitForUpdate(q) - <-indexingDone - - // wait for the checkpoint file to be written to disk - wait() - - // check what has been indexed - require.Equal(t, []uint64{0, 1, 2, 3, 4, 6, 7, 8, 9, 11, 12, 13, 14, 16, 17, 18, 19}, idx.IDs()) - - // the "deleted" mask should be empty - q.queue.deleted.Lock() - require.Empty(t, q.queue.deleted.m) - q.queue.deleted.Unlock() - - // now delete something that's already indexed - err = q.Delete(0, 4, 8) - require.NoError(t, err) - - // the "deleted" mask should still be empty - q.queue.deleted.Lock() - require.Empty(t, q.queue.deleted.m) - q.queue.deleted.Unlock() - - // check what's in the index - require.Equal(t, []uint64{1, 2, 3, 6, 7, 9, 11, 12, 13, 14, 16, 17, 18, 19}, idx.IDs()) - - // delete something that's not indexed yet - err = q.Delete(20, 21, 22) - require.NoError(t, err) - - // the "deleted" mask should contain the deleted ids - q.queue.deleted.Lock() - var ids []int - for id := range q.queue.deleted.m { - ids = append(ids, int(id)) - } - q.queue.deleted.Unlock() - sort.Ints(ids) - require.Equal(t, []int{20, 21, 22}, ids) - }) - - t.Run("brute force upper limit", func(t *testing.T) { - var idx mockBatchIndexer - - q, err := NewIndexQueue("1", new(mockShard), &idx, startWorker(t), newCheckpointManager(t), IndexQueueOptions{ - BatchSize: 1000, - BruteForceSearchLimit: 2, - }) - require.NoError(t, err) - defer q.Close() - - pushVector(t, ctx, q, 1, []float32{1, 2, 3}) - pushVector(t, ctx, q, 2, []float32{4, 5, 6}) - pushVector(t, ctx, q, 3, []float32{7, 8, 9}) - pushVector(t, ctx, q, 4, []float32{1, 2, 3}) - - res, _, err := q.SearchByVector([]float32{7, 8, 9}, 2, nil) - require.NoError(t, err) - // despite having 4 vectors in the queue - // only the first two are used for brute force search - require.Equal(t, []uint64{2, 1}, res) - }) - - t.Run("stores a safe checkpoint", func(t *testing.T) { - var idx mockBatchIndexer - - dir := t.TempDir() - q, err := NewIndexQueue("1", new(mockShard), &idx, startWorker(t), newCheckpointManagerWithDir(t, dir), IndexQueueOptions{ - BatchSize: 5, - IndexInterval: time.Hour, - }) - require.NoError(t, err) - defer q.Close() - - wait := waitForUpdate(q) - writeIDs(q, 5, 7) // [5, 6] - writeIDs(q, 9, 13) // [5, 6, 9, 10, 11], [12] - writeIDs(q, 0, 5) // [5, 6, 9, 10, 11], [12, 0, 1, 2, 3], [4] - time.Sleep(100 * time.Millisecond) - before, err := q.checkpoints.Get("1") - require.NoError(t, err) - q.pushToWorkers(-1, false) - // the checkpoint should be: 0, then 0 - // the cursor should not be updated - wait(100 * time.Millisecond) - after, err := q.checkpoints.Get("1") - require.NoError(t, err) - require.Equal(t, before, after) - - writeIDs(q, 15, 25) // [4, 15, 16, 17, 18], [19, 20, 21, 22, 23], [24] - writeIDs(q, 30, 40) // [4, 15, 16, 17, 18], [19, 20, 21, 22, 23], [24, 30, 31, 32, 33], [34, 35, 36, 37, 38], [39] - time.Sleep(100 * time.Millisecond) - // the checkpoint should be: 0, then 4, then 14, then 29 - q.pushToWorkers(-1, false) - // 0 - wait() - // 4 - wait() - // 14 - wait() - // 29 - wait() - v, err := q.checkpoints.Get("1") - require.NoError(t, err) - require.Equal(t, 29, int(v)) - }) - - t.Run("stale vectors", func(t *testing.T) { - var idx mockBatchIndexer - closeCh := make(chan struct{}) - idx.addBatchFn = func(id []uint64, vector [][]float32) error { - close(closeCh) - return nil - } - - q, err := NewIndexQueue("1", new(mockShard), &idx, startWorker(t), newCheckpointManager(t), IndexQueueOptions{ - BatchSize: 5, - StaleTimeout: 100 * time.Millisecond, - IndexInterval: 10 * time.Millisecond, - }) - require.NoError(t, err) - defer q.Close() - - for i := uint64(0); i < 3; i++ { - pushVector(t, ctx, q, i+1, []float32{1, 2, 3}) - } - - select { - case <-closeCh: - case <-time.After(500 * time.Millisecond): - t.Fatal("should have been indexed after 100ms") - } - - require.EqualValues(t, []uint64{1, 2, 3}, idx.IDs()) - }) - - t.Run("updates the shard state", func(t *testing.T) { - var idx mockBatchIndexer - indexed := make(chan struct{}) - idx.addBatchFn = func(id []uint64, vector [][]float32) error { - close(indexed) - return nil - } - - updated := make(chan string) - shard := mockShard{ - compareAndSwapStatusFn: func(old, new string) (storagestate.Status, error) { - updated <- new - return storagestate.Status(new), nil - }, - } - - q, err := NewIndexQueue("1", &shard, &idx, startWorker(t), newCheckpointManager(t), IndexQueueOptions{ - BatchSize: 2, - IndexInterval: 100 * time.Millisecond, - }) - require.NoError(t, err) - defer q.Close() - - for i := uint64(0); i < 2; i++ { - pushVector(t, ctx, q, i+1, []float32{1, 2, 3}) - } - - select { - case newState := <-updated: - require.Equal(t, storagestate.StatusIndexing.String(), newState) - case <-time.After(200 * time.Millisecond): - t.Fatal("shard state should have been updated after 100ms") - } - - select { - case <-indexed: - case <-time.After(200 * time.Millisecond): - t.Fatal("should have been indexed after 100ms") - } - - select { - case newState := <-updated: - require.Equal(t, storagestate.StatusReady.String(), newState) - case <-time.After(200 * time.Millisecond): - t.Fatal("shard state should have been updated after 100ms") - } - }) - - t.Run("close waits for indexing to be done", func(t *testing.T) { - var idx mockBatchIndexer - var count int - idx.addBatchFn = func(id []uint64, vector [][]float32) error { - <-time.After(10 * time.Millisecond) - count++ - return nil - } - - q, err := NewIndexQueue("1", new(mockShard), &idx, startWorker(t), newCheckpointManager(t), IndexQueueOptions{ - BatchSize: 5, - }) - require.NoError(t, err) - defer q.Close() - - for i := uint64(0); i < 100; i++ { - pushVector(t, ctx, q, i+1, []float32{1, 2, 3}) - } - - q.pushToWorkers(-1, false) - q.Close() - - require.EqualValues(t, 20, count) - }) - - t.Run("cos: normalized the query vector", func(t *testing.T) { - var idx mockBatchIndexer - idx.distancerProvider = distancer.NewCosineDistanceProvider() - - q, err := NewIndexQueue("1", new(mockShard), &idx, startWorker(t), newCheckpointManager(t), IndexQueueOptions{ - BatchSize: 7, // 300 is not divisible by 7 - IndexInterval: 100 * time.Second, - }) - require.NoError(t, err) - defer q.Close() - - for i := uint64(0); i < 300; i++ { - pushVector(t, ctx, q, i+1, randVector(1536)) - } - - q.pushToWorkers(-1, false) - - _, distances, err := q.SearchByVector(randVector(1536), 10, nil) - require.NoError(t, err) - - // all distances should be between 0 and 1 - for _, dist := range distances { - require.True(t, dist >= 0 && dist <= 1) - } - }) - - t.Run("pause/resume indexing", func(t *testing.T) { - var idx mockBatchIndexer - called := make(chan struct{}) - idx.addBatchFn = func(id []uint64, vector [][]float32) error { - called <- struct{}{} - // simulate work - <-time.After(100 * time.Millisecond) - return nil - } - - q, err := NewIndexQueue("1", new(mockShard), &idx, startWorker(t), newCheckpointManager(t), IndexQueueOptions{ - BatchSize: 2, - IndexInterval: 10 * time.Millisecond, - }) - require.NoError(t, err) - defer q.Close() - - pushVector(t, ctx, q, 1, []float32{1, 2, 3}) - pushVector(t, ctx, q, 2, []float32{4, 5, 6}) - - // batch indexed - <-called - - // pause indexing: this will block until the batch is indexed - q.pauseIndexing() - - // add more vectors - pushVector(t, ctx, q, 3, []float32{7, 8, 9}) - pushVector(t, ctx, q, 4, []float32{1, 2, 3}) - - // wait enough time to make sure the indexing is not happening - <-time.After(200 * time.Millisecond) - - select { - case <-called: - t.Fatal("should not have been called") - default: - } - - // resume indexing - q.resumeIndexing() - - // wait for the indexing to be done - <-called - }) - - t.Run("compression", func(t *testing.T) { - var idx mockBatchIndexer - called := make(chan struct{}) - idx.shouldCompress = true - idx.threshold = 4 - idx.alreadyIndexed = 6 - - release := make(chan struct{}) - idx.onCompressionTurnedOn = func(callback func()) error { - go func() { - <-release - callback() - }() - - close(called) - return nil - } - - q, err := NewIndexQueue("1", new(mockShard), &idx, startWorker(t), newCheckpointManager(t), IndexQueueOptions{ - BatchSize: 2, - IndexInterval: 10 * time.Millisecond, - }) - require.NoError(t, err) - defer q.Close() - - pushVector(t, ctx, q, 1, []float32{1, 2, 3}) - pushVector(t, ctx, q, 2, []float32{4, 5, 6}) - - // compression requested - <-called - - // indexing should be paused - require.True(t, q.paused.Load()) - - // release the compression - idx.compressed = true - close(release) - - // indexing should be resumed eventually - time.Sleep(100 * time.Millisecond) - require.False(t, q.paused.Load()) - - indexed := make(chan struct{}) - idx.addBatchFn = func(id []uint64, vector [][]float32) error { - close(indexed) - return nil - } - - // add more vectors - pushVector(t, ctx, q, 3, []float32{7, 8, 9}) - pushVector(t, ctx, q, 4, []float32{1, 2, 3}) - - // indexing should happen - <-indexed - }) - - t.Run("compression does not occur at the indexing if async is enabled", func(t *testing.T) { - vectors := [][]float32{{0, 1, 3, 4, 5, 6}, {0, 1, 3, 4, 5, 6}, {0, 1, 3, 4, 5, 6}} - distancer := distancer.NewL2SquaredProvider() - uc := ent.UserConfig{} - uc.MaxConnections = 112 - uc.EFConstruction = 112 - uc.EF = 10 - uc.VectorCacheMaxObjects = 10e12 - index, _ := hnsw.New( - hnsw.Config{ - RootPath: t.TempDir(), - ID: "recallbenchmark", - MakeCommitLoggerThunk: hnsw.MakeNoopCommitLogger, - DistanceProvider: distancer, - VectorForIDThunk: func(ctx context.Context, id uint64) ([]float32, error) { - return vectors[int(id)], nil - }, - TempVectorForIDThunk: func(ctx context.Context, id uint64, container *common.VectorSlice) ([]float32, error) { - copy(container.Slice, vectors[int(id)]) - return container.Slice, nil - }, - }, uc, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), newDummyStore(t)) - defer index.Shutdown(context.Background()) - - q, err := NewIndexQueue("1", new(mockShard), index, startWorker(t), newCheckpointManager(t), IndexQueueOptions{ - BatchSize: 2, - IndexInterval: 10 * time.Millisecond, - }) - require.NoError(t, err) - defer q.Close() - - uc.PQ = ent.PQConfig{Enabled: true, Encoder: ent.PQEncoder{Type: "please break...", Distribution: "normal"}} - err = index.UpdateUserConfig(uc, func() {}) - require.Nil(t, err) - }) - - t.Run("sending batch with deleted ids to worker", func(t *testing.T) { - var idx mockBatchIndexer - idx.addBatchFn = func(id []uint64, vector [][]float32) error { - t.Fatal("should not have been called") - return nil - } - - q, err := NewIndexQueue("1", new(mockShard), &idx, startWorker(t), newCheckpointManager(t), IndexQueueOptions{ - BatchSize: 2, - IndexInterval: 100 * time.Second, - }) - require.NoError(t, err) - defer q.Close() - - pushVector(t, ctx, q, 0, []float32{1, 2, 3}) - pushVector(t, ctx, q, 1, []float32{1, 2, 3}) - - err = q.Delete(0, 1) - require.NoError(t, err) - - q.pushToWorkers(-1, true) - }) - - t.Run("release twice", func(t *testing.T) { - var idx mockBatchIndexer - - q, err := NewIndexQueue("1", new(mockShard), &idx, startWorker(t), newCheckpointManager(t), IndexQueueOptions{ - BatchSize: 10, - IndexInterval: time.Hour, // do not index automatically - }) - require.NoError(t, err) - - for i := uint64(0); i < 35; i++ { - pushVector(t, ctx, q, i+1, []float32{1, 2, 3}) - } - - chunks := q.queue.borrowChunks(10) - require.Equal(t, 3, len(chunks)) - - // release once - for _, chunk := range chunks { - q.queue.releaseChunk(chunk) - } - - // release again - for _, chunk := range chunks { - q.queue.releaseChunk(chunk) - } - }) -} - -func BenchmarkPush(b *testing.B) { - var idx mockBatchIndexer - - idx.addBatchFn = func(id []uint64, vector [][]float32) error { - time.Sleep(1 * time.Second) - return nil - } - - q, err := NewIndexQueue("1", new(mockShard), &idx, startWorker(b), newCheckpointManager(b), IndexQueueOptions{ - BatchSize: 1000, - IndexInterval: 1 * time.Millisecond, - }) - require.NoError(b, err) - defer q.Close() - - vecs := make([]vectorDescriptor, 100) - for j := range vecs { - vecs[j] = vectorDescriptor{ - id: uint64(j), - vector: []float32{1, 2, 3}, - } - } - - b.ResetTimer() - for i := 0; i < b.N; i++ { - for j := 0; j < 100; j++ { - err = q.Push(context.Background(), vecs...) - require.NoError(b, err) - } - } -} - -type mockShard struct { - compareAndSwapStatusFn func(old, new string) (storagestate.Status, error) -} - -func (m *mockShard) compareAndSwapStatus(old, new string) (storagestate.Status, error) { - if m.compareAndSwapStatusFn == nil { - return storagestate.Status(new), nil - } - - return m.compareAndSwapStatusFn(old, new) -} - -type mockBatchIndexer struct { - sync.Mutex - addBatchFn func(id []uint64, vector [][]float32) error - vectors map[uint64][]float32 - containsNodeFn func(id uint64) bool - deleteFn func(ids ...uint64) error - distancerProvider distancer.Provider - shouldCompress bool - threshold int - compressed bool - alreadyIndexed uint64 - onCompressionTurnedOn func(func()) error -} - -func (m *mockBatchIndexer) AddBatch(ctx context.Context, ids []uint64, vector [][]float32) (err error) { - m.Lock() - defer m.Unlock() - - if m.addBatchFn != nil { - err = m.addBatchFn(ids, vector) - } - - if m.vectors == nil { - m.vectors = make(map[uint64][]float32) - } - - for i, id := range ids { - m.vectors[id] = vector[i] - } - - return -} - -func (m *mockBatchIndexer) SearchByVector(vector []float32, k int, allowList helpers.AllowList) ([]uint64, []float32, error) { - m.Lock() - defer m.Unlock() - - results := newPqMaxPool(k).GetMax(k) - - if m.DistancerProvider().Type() == "cosine-dot" { - vector = distancer.Normalize(vector) - } - - for id, v := range m.vectors { - // skip filtered data - if allowList != nil && allowList.Contains(id) { - continue - } - - if m.DistancerProvider().Type() == "cosine-dot" { - v = distancer.Normalize(v) - } - - dist, _, err := m.DistanceBetweenVectors(vector, v) - if err != nil { - return nil, nil, err - } - - if results.Len() < k || dist < results.Top().Dist { - results.Insert(id, dist) - for results.Len() > k { - results.Pop() - } - } - } - var ids []uint64 - var distances []float32 - - for i := k - 1; i >= 0; i-- { - if results.Len() == 0 { - break - } - element := results.Pop() - ids = append(ids, element.ID) - distances = append(distances, element.Dist) - } - - return ids, distances, nil -} - -func (m *mockBatchIndexer) SearchByVectorDistance(vector []float32, maxDistance float32, maxLimit int64, allowList helpers.AllowList) ([]uint64, []float32, error) { - m.Lock() - defer m.Unlock() - - results := newPqMaxPool(int(maxLimit)).GetMax(int(maxLimit)) - - if m.DistancerProvider().Type() == "cosine-dot" { - vector = distancer.Normalize(vector) - } - - for id, v := range m.vectors { - // skip filtered data - if allowList != nil && allowList.Contains(id) { - continue - } - - if m.DistancerProvider().Type() == "cosine-dot" { - v = distancer.Normalize(v) - } - - dist, _, err := m.DistanceBetweenVectors(vector, v) - if err != nil { - return nil, nil, err - } - - if dist > maxDistance { - continue - } - - if results.Len() < int(maxLimit) || dist < results.Top().Dist { - results.Insert(id, dist) - for results.Len() > int(maxLimit) { - results.Pop() - } - } - } - var ids []uint64 - var distances []float32 - - for i := maxLimit - 1; i >= 0; i-- { - if results.Len() == 0 { - break - } - element := results.Pop() - ids = append(ids, element.ID) - distances = append(distances, element.Dist) - } - - return ids, distances, nil -} - -func (m *mockBatchIndexer) DistanceBetweenVectors(x, y []float32) (float32, bool, error) { - res := float32(0) - for i := range x { - diff := x[i] - y[i] - res += diff * diff - } - return res, true, nil -} - -func (m *mockBatchIndexer) ContainsNode(id uint64) bool { - m.Lock() - defer m.Unlock() - if m.containsNodeFn != nil { - return m.containsNodeFn(id) - } - - _, ok := m.vectors[id] - return ok -} - -func (m *mockBatchIndexer) Delete(ids ...uint64) error { - m.Lock() - defer m.Unlock() - if m.deleteFn != nil { - return m.deleteFn(ids...) - } - - for _, id := range ids { - delete(m.vectors, id) - } - - return nil -} - -func (m *mockBatchIndexer) IDs() []uint64 { - m.Lock() - defer m.Unlock() - - ids := make([]uint64, 0, len(m.vectors)) - for id := range m.vectors { - ids = append(ids, id) - } - - sort.Slice(ids, func(i, j int) bool { - return ids[i] < ids[j] - }) - - return ids -} - -func (m *mockBatchIndexer) DistancerProvider() distancer.Provider { - if m.distancerProvider == nil { - return distancer.NewL2SquaredProvider() - } - - return m.distancerProvider -} - -func (m *mockBatchIndexer) ShouldCompress() (bool, int) { - return m.shouldCompress, m.threshold -} - -func (m *mockBatchIndexer) Compressed() bool { - return m.compressed -} - -func (m *mockBatchIndexer) AlreadyIndexed() uint64 { - return m.alreadyIndexed -} - -func (m *mockBatchIndexer) TurnOnCompression(callback func()) error { - if m.onCompressionTurnedOn != nil { - return m.onCompressionTurnedOn(callback) - } - - return nil -} - -func newDummyStore(t *testing.T) *lsmkv.Store { - logger, _ := test.NewNullLogger() - storeDir := t.TempDir() - store, err := lsmkv.New(storeDir, storeDir, logger, nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop()) - require.Nil(t, err) - return store -} diff --git a/adapters/repos/db/index_sharding_backward_compatibility.go b/adapters/repos/db/index_sharding_backward_compatibility.go deleted file mode 100644 index 1ae53d638579d2a006b320993fa2032f149e50bb..0000000000000000000000000000000000000000 --- a/adapters/repos/db/index_sharding_backward_compatibility.go +++ /dev/null @@ -1,59 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "os" - "path/filepath" - "strings" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/usecases/sharding" -) - -func (i *Index) checkSingleShardMigration(shardState *sharding.State) error { - res, err := os.ReadDir(i.Config.RootPath) - if err != nil { - return err - } - - for _, entry := range res { - if !strings.HasPrefix(entry.Name(), i.ID()+"_single") { - // either not part of this index, or not a "_single" shard - continue - } - - // whatever is left now, needs to be migrated - shards := shardState.AllPhysicalShards() - if len(shards) != 1 { - return errors.Errorf("cannot migrate '_single' shard into config with %d "+ - "desired shards", len(shards)) - } - - shardName := shards[0] - newName := i.ID() + "_" + shardName + strings.TrimPrefix(entry.Name(), i.ID()+"_single") - oldPath := filepath.Join(i.Config.RootPath, entry.Name()) - newPath := filepath.Join(i.Config.RootPath, newName) - - if err := os.Rename(oldPath, newPath); err != nil { - return errors.Wrapf(err, "migrate shard %q to %q", oldPath, newPath) - } - - i.logger.WithField("action", "index_startup_migrate_shards_successful"). - WithField("old_shard", oldPath). - WithField("new_shard", newPath). - Infof("successfully migrated shard file %q (created in an earlier version) to %q", - oldPath, newPath) - } - - return nil -} diff --git a/adapters/repos/db/indexcheckpoint/checkpoint.go b/adapters/repos/db/indexcheckpoint/checkpoint.go deleted file mode 100644 index 599c19254e1b72b719d6ad511391a7ba9619bd54..0000000000000000000000000000000000000000 --- a/adapters/repos/db/indexcheckpoint/checkpoint.go +++ /dev/null @@ -1,113 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package indexcheckpoint - -import ( - "encoding/binary" - "path/filepath" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - bolt "go.etcd.io/bbolt" -) - -var checkpointBucket = []byte("checkpoint") - -// Checkpoints keeps track of the last indexed vector id for each shard. -// It stores the ids in a BoltDB file. -type Checkpoints struct { - db *bolt.DB -} - -func New(dir string, logger logrus.FieldLogger) (*Checkpoints, error) { - path := filepath.Join(dir, "index.db") - db, err := bolt.Open(path, 0o600, nil) - if err != nil { - return nil, errors.Wrapf(err, "open %q", path) - } - - ic := Checkpoints{ - db: db, - } - - err = ic.initDB() - if err != nil { - return nil, err - } - - return &ic, nil -} - -func (c *Checkpoints) initDB() error { - err := c.db.Update(func(tx *bolt.Tx) error { - _, err := tx.CreateBucketIfNotExists(checkpointBucket) - return err - }) - - return errors.Wrap(err, "init db") -} - -// Close the underlying DB -func (c *Checkpoints) Close() { - c.db.Close() -} - -func (c *Checkpoints) Get(shardID string) (uint64, error) { - var count uint64 - err := c.db.View(func(tx *bolt.Tx) error { - b := tx.Bucket(checkpointBucket) - v := b.Get([]byte(shardID)) - if v == nil { - return nil - } - - count = binary.LittleEndian.Uint64(v) - - return nil - }) - if err != nil { - return 0, errors.Wrap(err, "get checkpoint") - } - - return count, nil -} - -func (c *Checkpoints) Update(shardID string, id uint64) error { - buf := make([]byte, 8) - binary.LittleEndian.PutUint64(buf, id) - - err := c.db.Update(func(tx *bolt.Tx) error { - b := tx.Bucket(checkpointBucket) - return b.Put([]byte(shardID), buf) - }) - if err != nil { - return errors.Wrap(err, "update checkpoint") - } - - return nil -} - -func (c *Checkpoints) Delete(shardID string) error { - err := c.db.Update(func(tx *bolt.Tx) error { - b := tx.Bucket(checkpointBucket) - return b.Delete([]byte(shardID)) - }) - if err != nil { - return errors.Wrap(err, "delete checkpoint") - } - - return nil -} - -func (c *Checkpoints) Filename() string { - return c.db.Path() -} diff --git a/adapters/repos/db/indexcounter/counter.go b/adapters/repos/db/indexcounter/counter.go deleted file mode 100644 index f785d0bde6959c45402dfffabcf750aab1d62382..0000000000000000000000000000000000000000 --- a/adapters/repos/db/indexcounter/counter.go +++ /dev/null @@ -1,103 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package indexcounter - -import ( - "encoding/binary" - "fmt" - "os" - "sync" - - "github.com/pkg/errors" -) - -type Counter struct { - count uint64 - sync.Mutex - f *os.File -} - -func New(shardPath string) (*Counter, error) { - fileName := fmt.Sprintf("%s/indexcount", shardPath) - f, err := os.OpenFile(fileName, os.O_RDWR|os.O_CREATE, 0o666) - if err != nil { - return nil, err - } - - stat, err := f.Stat() - if err != nil { - return nil, err - } - - var initialCount uint64 = 0 - if stat.Size() > 0 { - // the file has existed before, we need to initialize with its content - err := binary.Read(f, binary.LittleEndian, &initialCount) - if err != nil { - return nil, errors.Wrap(err, "read initial count from file") - } - - } - - return &Counter{ - count: initialCount, - f: f, - }, nil -} - -func (c *Counter) Get() uint64 { - c.Lock() - defer c.Unlock() - return c.count -} - -func (c *Counter) GetAndInc() (uint64, error) { - c.Lock() - defer c.Unlock() - before := c.count - c.count++ - c.f.Seek(0, 0) - err := binary.Write(c.f, binary.LittleEndian, &c.count) - if err != nil { - return 0, errors.Wrap(err, "increase counter on disk") - } - c.f.Seek(0, 0) - return before, nil -} - -// PreviewNext can be used to check if there is data present in the index, if -// it returns 0, you can be certain that no data exists -func (c *Counter) PreviewNext() uint64 { - c.Lock() - defer c.Unlock() - - return c.count -} - -func (c *Counter) Drop() error { - c.Lock() - defer c.Unlock() - if c.f == nil { - return nil - } - filename := c.FileName() - c.f.Close() - err := os.Remove(filename) - if err != nil { - return errors.Wrap(err, "drop counter file") - } - return nil -} - -func (c *Counter) FileName() string { - return c.f.Name() -} diff --git a/adapters/repos/db/init.go b/adapters/repos/db/init.go deleted file mode 100644 index baf3bedc92ad726a11d4b5644e8f661e6f1f8c2c..0000000000000000000000000000000000000000 --- a/adapters/repos/db/init.go +++ /dev/null @@ -1,106 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "context" - "fmt" - "os" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/adapters/repos/db/indexcheckpoint" - "github.com/weaviate/weaviate/adapters/repos/db/inverted" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/usecases/config" - "github.com/weaviate/weaviate/usecases/replica" -) - -// init gets the current schema and creates one index object per class. -// The indices will in turn create shards, which will either read an -// existing db file from disk, or create a new one if none exists -func (db *DB) init(ctx context.Context) error { - if err := os.MkdirAll(db.config.RootPath, 0o777); err != nil { - return fmt.Errorf("create root path directory at %s: %w", db.config.RootPath, err) - } - - // As of v1.22, db files are stored in a hierarchical structure - // rather than a flat one. If weaviate is started with files - // that are still in the flat structure, we will migrate them - // over. - if err := db.migrateFileStructureIfNecessary(); err != nil { - return err - } - - if asyncEnabled() { - // init the index checkpoint file - var err error - db.indexCheckpoints, err = indexcheckpoint.New(db.config.RootPath, db.logger) - if err != nil { - return errors.Wrap(err, "init index checkpoint") - } - } - - objects := db.schemaGetter.GetSchemaSkipAuth().Objects - if objects != nil { - for _, class := range objects.Classes { - invertedConfig := class.InvertedIndexConfig - if invertedConfig == nil { - // for backward compatibility, this field was introduced in v1.0.4, - // prior schemas will not yet have the field. Init with the defaults - // which were previously hard-coded. - // In this method we are essentially reading the schema from disk, so - // it could have been created before v1.0.4 - invertedConfig = &models.InvertedIndexConfig{ - CleanupIntervalSeconds: config.DefaultCleanupIntervalSeconds, - Bm25: &models.BM25Config{ - K1: config.DefaultBM25k1, - B: config.DefaultBM25b, - }, - } - } - if err := replica.ValidateConfig(class, db.config.Replication); err != nil { - return fmt.Errorf("replication config: %w", err) - } - - idx, err := NewIndex(ctx, IndexConfig{ - ClassName: schema.ClassName(class.Class), - RootPath: db.config.RootPath, - ResourceUsage: db.config.ResourceUsage, - QueryMaximumResults: db.config.QueryMaximumResults, - QueryNestedRefLimit: db.config.QueryNestedRefLimit, - MemtablesFlushIdleAfter: db.config.MemtablesFlushIdleAfter, - MemtablesInitialSizeMB: db.config.MemtablesInitialSizeMB, - MemtablesMaxSizeMB: db.config.MemtablesMaxSizeMB, - MemtablesMinActiveSeconds: db.config.MemtablesMinActiveSeconds, - MemtablesMaxActiveSeconds: db.config.MemtablesMaxActiveSeconds, - TrackVectorDimensions: db.config.TrackVectorDimensions, - AvoidMMap: db.config.AvoidMMap, - DisableLazyLoadShards: db.config.DisableLazyLoadShards, - ReplicationFactor: class.ReplicationConfig.Factor, - }, db.schemaGetter.CopyShardingState(class.Class), - inverted.ConfigFromModel(invertedConfig), - class.VectorIndexConfig.(schema.VectorIndexConfig), - db.schemaGetter, db, db.logger, db.nodeResolver, db.remoteIndex, - db.replicaClient, db.promMetrics, class, db.jobQueueCh, db.indexCheckpoints) - if err != nil { - return errors.Wrap(err, "create index") - } - - db.indexLock.Lock() - db.indices[idx.ID()] = idx - db.indexLock.Unlock() - } - } - - return nil -} diff --git a/adapters/repos/db/inverted/analyzer.go b/adapters/repos/db/inverted/analyzer.go deleted file mode 100644 index ff9b3e7372a1f261a4a472d032a0fe6d8f0642b9..0000000000000000000000000000000000000000 --- a/adapters/repos/db/inverted/analyzer.go +++ /dev/null @@ -1,246 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package inverted - -import ( - "bytes" - "encoding/binary" - - "github.com/google/uuid" - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/entities/models" -) - -type IsFallbackToSearchable func() bool - -type Countable struct { - Data []byte - TermFrequency float32 -} - -type Property struct { - Name string - Items []Countable - Length int - HasFilterableIndex bool // roaring set index - HasSearchableIndex bool // map index (with frequencies) -} - -type NilProperty struct { - Name string - AddToPropertyLength bool -} - -func DedupItems(props []Property) []Property { - for i := range props { - seen := map[string]struct{}{} - items := props[i].Items - - var key string - // reverse order to keep latest elements - for j := len(items) - 1; j >= 0; j-- { - key = string(items[j].Data) - if _, ok := seen[key]; ok { - // remove element already seen - items = append(items[:j], items[j+1:]...) - } - seen[key] = struct{}{} - } - props[i].Items = items - } - return props -} - -type Analyzer struct { - isFallbackToSearchable IsFallbackToSearchable -} - -// Text tokenizes given input according to selected tokenization, -// then aggregates duplicates -func (a *Analyzer) Text(tokenization, in string) []Countable { - return a.TextArray(tokenization, []string{in}) -} - -// TextArray tokenizes given input according to selected tokenization, -// then aggregates duplicates -func (a *Analyzer) TextArray(tokenization string, inArr []string) []Countable { - var terms []string - for _, in := range inArr { - terms = append(terms, helpers.Tokenize(tokenization, in)...) - } - - counts := map[string]uint64{} - for _, term := range terms { - counts[term]++ - } - - countable := make([]Countable, len(counts)) - i := 0 - for term, count := range counts { - countable[i] = Countable{ - Data: []byte(term), - TermFrequency: float32(count), - } - i++ - } - return countable -} - -// Int requires no analysis, so it's actually just a simple conversion to a -// string-formatted byte slice of the int -func (a *Analyzer) Int(in int64) ([]Countable, error) { - data, err := LexicographicallySortableInt64(in) - if err != nil { - return nil, err - } - - return []Countable{ - { - Data: data, - }, - }, nil -} - -// UUID requires no analysis, so it's just dumping the raw binary representation -func (a *Analyzer) UUID(in uuid.UUID) ([]Countable, error) { - return []Countable{ - { - Data: in[:], - }, - }, nil -} - -// UUID array requires no analysis, so it's just dumping the raw binary -// representation of each contained element -func (a *Analyzer) UUIDArray(in []uuid.UUID) ([]Countable, error) { - out := make([]Countable, len(in)) - for i := range in { - out[i] = Countable{ - Data: in[i][:], - } - } - - return out, nil -} - -// Int array requires no analysis, so it's actually just a simple conversion to a -// string-formatted byte slice of the int -func (a *Analyzer) IntArray(in []int64) ([]Countable, error) { - out := make([]Countable, len(in)) - for i := range in { - data, err := LexicographicallySortableInt64(in[i]) - if err != nil { - return nil, err - } - out[i] = Countable{Data: data} - } - - return out, nil -} - -// Float requires no analysis, so it's actually just a simple conversion to a -// lexicographically sortable byte slice. -func (a *Analyzer) Float(in float64) ([]Countable, error) { - data, err := LexicographicallySortableFloat64(in) - if err != nil { - return nil, err - } - - return []Countable{ - { - Data: data, - }, - }, nil -} - -// Float array requires no analysis, so it's actually just a simple conversion to a -// lexicographically sortable byte slice. -func (a *Analyzer) FloatArray(in []float64) ([]Countable, error) { - out := make([]Countable, len(in)) - for i := range in { - data, err := LexicographicallySortableFloat64(in[i]) - if err != nil { - return nil, err - } - out[i] = Countable{Data: data} - } - - return out, nil -} - -// BoolArray requires no analysis, so it's actually just a simple conversion to a -// little-endian ordered byte slice -func (a *Analyzer) BoolArray(in []bool) ([]Countable, error) { - out := make([]Countable, len(in)) - for i := range in { - b := bytes.NewBuffer(nil) - err := binary.Write(b, binary.LittleEndian, &in[i]) - if err != nil { - return nil, err - } - out[i] = Countable{Data: b.Bytes()} - } - - return out, nil -} - -// Bool requires no analysis, so it's actually just a simple conversion to a -// little-endian ordered byte slice -func (a *Analyzer) Bool(in bool) ([]Countable, error) { - b := bytes.NewBuffer(nil) - err := binary.Write(b, binary.LittleEndian, &in) - if err != nil { - return nil, err - } - - return []Countable{ - { - Data: b.Bytes(), - }, - }, nil -} - -// RefCount does not index the content of the refs, but only the count with 0 -// being an explicitly allowed value as well. -func (a *Analyzer) RefCount(in models.MultipleRef) ([]Countable, error) { - length := uint64(len(in)) - data, err := LexicographicallySortableUint64(length) - if err != nil { - return nil, err - } - - return []Countable{ - { - Data: data, - }, - }, nil -} - -// Ref indexes references as beacon-strings -func (a *Analyzer) Ref(in models.MultipleRef) ([]Countable, error) { - out := make([]Countable, len(in)) - - for i, ref := range in { - out[i] = Countable{ - Data: []byte(ref.Beacon), - } - } - - return out, nil -} - -func NewAnalyzer(isFallbackToSearchable IsFallbackToSearchable) *Analyzer { - if isFallbackToSearchable == nil { - isFallbackToSearchable = func() bool { return false } - } - return &Analyzer{isFallbackToSearchable: isFallbackToSearchable} -} diff --git a/adapters/repos/db/inverted/analyzer_test.go b/adapters/repos/db/inverted/analyzer_test.go deleted file mode 100644 index e608f361f1d6568a0c4cb60e8080200ee183c66c..0000000000000000000000000000000000000000 --- a/adapters/repos/db/inverted/analyzer_test.go +++ /dev/null @@ -1,598 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package inverted - -import ( - "bytes" - "math" - "sort" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" -) - -func TestAnalyzer(t *testing.T) { - a := NewAnalyzer(nil) - - countable := func(data []string, freq []int) []Countable { - countable := make([]Countable, len(data)) - for i := range data { - countable[i] = Countable{ - Data: []byte(data[i]), - TermFrequency: float32(freq[i]), - } - } - return countable - } - - t.Run("with text", func(t *testing.T) { - type testCase struct { - name string - input string - tokenization string - expectedCountable []Countable - } - - testCases := []testCase{ - { - name: "tokenization word, unique words", - input: "Hello, my name is John Doe", - tokenization: models.PropertyTokenizationWord, - expectedCountable: countable( - []string{"hello", "my", "name", "is", "john", "doe"}, - []int{1, 1, 1, 1, 1, 1}, - ), - }, - { - name: "tokenization word, duplicated words", - input: "Du. Du hast. Du hast. Du hast mich gefragt.", - tokenization: models.PropertyTokenizationWord, - expectedCountable: countable( - []string{"du", "hast", "mich", "gefragt"}, - []int{4, 3, 1, 1}, - ), - }, - { - name: "tokenization lowercase, unique words", - input: "My email is john-thats-jay.ohh.age.n+alloneword@doe.com", - tokenization: models.PropertyTokenizationLowercase, - expectedCountable: countable( - []string{"my", "email", "is", "john-thats-jay.ohh.age.n+alloneword@doe.com"}, - []int{1, 1, 1, 1}, - ), - }, - { - name: "tokenization lowercase, duplicated words", - input: "Du. Du hast. Du hast. Du hast mich gefragt.", - tokenization: models.PropertyTokenizationLowercase, - expectedCountable: countable( - []string{"du.", "du", "hast.", "hast", "mich", "gefragt."}, - []int{1, 3, 2, 1, 1, 1}, - ), - }, - { - name: "tokenization whitespace, unique words", - input: "My email is john-thats-jay.ohh.age.n+alloneword@doe.com", - tokenization: models.PropertyTokenizationWhitespace, - expectedCountable: countable( - []string{"My", "email", "is", "john-thats-jay.ohh.age.n+alloneword@doe.com"}, - []int{1, 1, 1, 1}, - ), - }, - { - name: "tokenization whitespace, duplicated words", - input: "Du. Du hast. Du hast. Du hast mich gefragt.", - tokenization: models.PropertyTokenizationWhitespace, - expectedCountable: countable( - []string{"Du.", "Du", "hast.", "hast", "mich", "gefragt."}, - []int{1, 3, 2, 1, 1, 1}, - ), - }, - { - name: "tokenization field", - input: "\n Du. Du hast. Du hast. Du hast mich gefragt.\t ", - tokenization: models.PropertyTokenizationField, - expectedCountable: countable( - []string{"Du. Du hast. Du hast. Du hast mich gefragt."}, - []int{1}, - ), - }, - { - name: "non existing tokenization", - input: "Du. Du hast. Du hast. Du hast mich gefragt.", - tokenization: "non_existing", - expectedCountable: []Countable{}, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - countable := a.Text(tc.tokenization, tc.input) - assert.ElementsMatch(t, tc.expectedCountable, countable) - }) - } - }) - - t.Run("with text array", func(t *testing.T) { - type testCase struct { - name string - input []string - tokenization string - expectedCountable []Countable - } - - testCases := []testCase{ - { - name: "tokenization word, unique words", - input: []string{"Hello,", "my name is John Doe"}, - tokenization: models.PropertyTokenizationWord, - expectedCountable: countable( - []string{"hello", "my", "name", "is", "john", "doe"}, - []int{1, 1, 1, 1, 1, 1}, - ), - }, - { - name: "tokenization word, duplicated words", - input: []string{"Du. Du hast. Du hast.", "Du hast mich gefragt."}, - tokenization: models.PropertyTokenizationWord, - expectedCountable: countable( - []string{"du", "hast", "mich", "gefragt"}, - []int{4, 3, 1, 1}, - ), - }, - { - name: "tokenization lowercase, unique words", - input: []string{"My email", "is john-thats-jay.ohh.age.n+alloneword@doe.com"}, - tokenization: models.PropertyTokenizationLowercase, - expectedCountable: countable( - []string{"my", "email", "is", "john-thats-jay.ohh.age.n+alloneword@doe.com"}, - []int{1, 1, 1, 1}, - ), - }, - { - name: "tokenization lowercase, duplicated words", - input: []string{"Du. Du hast. Du hast.", "Du hast mich gefragt."}, - tokenization: models.PropertyTokenizationLowercase, - expectedCountable: countable( - []string{"du.", "du", "hast.", "hast", "mich", "gefragt."}, - []int{1, 3, 2, 1, 1, 1}, - ), - }, - { - name: "tokenization whitespace, unique words", - input: []string{"My email", "is john-thats-jay.ohh.age.n+alloneword@doe.com"}, - tokenization: models.PropertyTokenizationWhitespace, - expectedCountable: countable( - []string{"My", "email", "is", "john-thats-jay.ohh.age.n+alloneword@doe.com"}, - []int{1, 1, 1, 1}, - ), - }, - { - name: "tokenization whitespace, duplicated words", - input: []string{"Du. Du hast. Du hast.", "Du hast mich gefragt."}, - tokenization: models.PropertyTokenizationWhitespace, - expectedCountable: countable( - []string{"Du.", "Du", "hast.", "hast", "mich", "gefragt."}, - []int{1, 3, 2, 1, 1, 1}, - ), - }, - { - name: "tokenization field", - input: []string{"\n Du. Du hast. Du hast.", "Du hast mich gefragt.\t "}, - tokenization: models.PropertyTokenizationField, - expectedCountable: countable( - []string{"Du. Du hast. Du hast.", "Du hast mich gefragt."}, - []int{1, 1}, - ), - }, - { - name: "non existing tokenization", - input: []string{"Du. Du hast. Du hast.", "Du hast mich gefragt."}, - tokenization: "non_existing", - expectedCountable: []Countable{}, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - countable := a.TextArray(tc.tokenization, tc.input) - assert.ElementsMatch(t, tc.expectedCountable, countable) - }) - } - }) - - t.Run("with int it stays sortable", func(t *testing.T) { - getData := func(in []Countable, err error) []byte { - require.Nil(t, err) - return in[0].Data - } - - results := [][]byte{ - getData(a.Float(math.MinInt64)), - getData(a.Int(-1000000)), - getData(a.Int(-400000)), - getData(a.Int(-20000)), - getData(a.Int(-9000)), - getData(a.Int(-301)), - getData(a.Int(-300)), - getData(a.Int(-299)), - getData(a.Int(-1)), - getData(a.Int(0)), - getData(a.Int(1)), - getData(a.Int(299)), - getData(a.Int(300)), - getData(a.Int(301)), - getData(a.Int(9000)), - getData(a.Int(20000)), - getData(a.Int(400000)), - getData(a.Int(1000000)), - getData(a.Float(math.MaxInt64)), - } - - afterSort := make([][]byte, len(results)) - copy(afterSort, results) - sort.Slice(afterSort, func(a, b int) bool { return bytes.Compare(afterSort[a], afterSort[b]) == -1 }) - assert.Equal(t, results, afterSort) - }) - - t.Run("with float it stays sortable", func(t *testing.T) { - getData := func(in []Countable, err error) []byte { - require.Nil(t, err) - return in[0].Data - } - - results := [][]byte{ - getData(a.Float(-math.MaxFloat64)), - getData(a.Float(-1000000)), - getData(a.Float(-400000)), - getData(a.Float(-20000)), - getData(a.Float(-9000.9)), - getData(a.Float(-9000.8999)), - getData(a.Float(-9000.8998)), - getData(a.Float(-9000.79999)), - getData(a.Float(-301)), - getData(a.Float(-300)), - getData(a.Float(-299)), - getData(a.Float(-1)), - getData(a.Float(-0.09)), - getData(a.Float(-0.01)), - getData(a.Float(-0.009)), - getData(a.Float(0)), - getData(a.Float(math.SmallestNonzeroFloat64)), - getData(a.Float(0.009)), - getData(a.Float(0.01)), - getData(a.Float(0.09)), - getData(a.Float(0.1)), - getData(a.Float(0.9)), - getData(a.Float(1)), - getData(a.Float(299)), - getData(a.Float(300)), - getData(a.Float(301)), - getData(a.Float(9000)), - getData(a.Float(20000)), - getData(a.Float(400000)), - getData(a.Float(1000000)), - getData(a.Float(math.MaxFloat64)), - } - - afterSort := make([][]byte, len(results)) - copy(afterSort, results) - sort.Slice(afterSort, func(a, b int) bool { return bytes.Compare(afterSort[a], afterSort[b]) == -1 }) - assert.Equal(t, results, afterSort) - }) - - t.Run("with refCount it stays sortable", func(t *testing.T) { - getData := func(in []Countable, err error) []byte { - require.Nil(t, err) - return in[0].Data - } - - results := [][]byte{ - getData(a.RefCount(make(models.MultipleRef, 0))), - getData(a.RefCount(make(models.MultipleRef, 1))), - getData(a.RefCount(make(models.MultipleRef, 2))), - getData(a.RefCount(make(models.MultipleRef, 99))), - getData(a.RefCount(make(models.MultipleRef, 100))), - getData(a.RefCount(make(models.MultipleRef, 101))), - getData(a.RefCount(make(models.MultipleRef, 256))), - getData(a.RefCount(make(models.MultipleRef, 300))), - getData(a.RefCount(make(models.MultipleRef, 456))), - } - - afterSort := make([][]byte, len(results)) - copy(afterSort, results) - sort.Slice(afterSort, func(a, b int) bool { return bytes.Compare(afterSort[a], afterSort[b]) == -1 }) - assert.Equal(t, results, afterSort) - }) - - byteTrue := []byte{0x1} - byteFalse := []byte{0x0} - - t.Run("analyze bool", func(t *testing.T) { - t.Run("true", func(t *testing.T) { - countable, err := a.Bool(true) - require.Nil(t, err) - require.Len(t, countable, 1) - - c := countable[0] - assert.Equal(t, byteTrue, c.Data) - assert.Equal(t, float32(0), c.TermFrequency) - }) - - t.Run("false", func(t *testing.T) { - countable, err := a.Bool(false) - require.Nil(t, err) - require.Len(t, countable, 1) - - c := countable[0] - assert.Equal(t, byteFalse, c.Data) - assert.Equal(t, float32(0), c.TermFrequency) - }) - }) - - t.Run("analyze bool array", func(t *testing.T) { - type testCase struct { - name string - values []bool - expected [][]byte - } - - testCases := []testCase{ - { - name: "[true]", - values: []bool{true}, - expected: [][]byte{byteTrue}, - }, - { - name: "[false]", - values: []bool{false}, - expected: [][]byte{byteFalse}, - }, - { - name: "[true, true, true]", - values: []bool{true, true, true}, - expected: [][]byte{byteTrue, byteTrue, byteTrue}, - }, - { - name: "[false, false, false]", - values: []bool{false, false, false}, - expected: [][]byte{byteFalse, byteFalse, byteFalse}, - }, - { - name: "[false, true, false, true]", - values: []bool{false, true, false, true}, - expected: [][]byte{byteFalse, byteTrue, byteFalse, byteTrue}, - }, - { - name: "[]", - values: []bool{}, - expected: [][]byte{}, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - countable, err := a.BoolArray(tc.values) - require.Nil(t, err) - require.Len(t, countable, len(tc.expected)) - - for i := range countable { - assert.Equal(t, tc.expected[i], countable[i].Data) - assert.Equal(t, float32(0), countable[i].TermFrequency) - } - }) - } - }) -} - -func TestAnalyzer_DefaultEngPreset(t *testing.T) { - countable := func(data []string, freq []int) []Countable { - countable := make([]Countable, len(data)) - for i := range data { - countable[i] = Countable{ - Data: []byte(data[i]), - TermFrequency: float32(freq[i]), - } - } - return countable - } - - a := NewAnalyzer(nil) - input := "Hello you-beautiful_World" - - t.Run("with text", func(t *testing.T) { - type testCase struct { - name string - tokenization string - input string - expectedCountable []Countable - } - - testCases := []testCase{ - { - name: "tokenization word", - tokenization: models.PropertyTokenizationWord, - input: input, - expectedCountable: countable( - []string{"hello", "you", "beautiful", "world"}, - []int{1, 1, 1, 1}, - ), - }, - { - name: "tokenization lowercase", - tokenization: models.PropertyTokenizationLowercase, - input: input, - expectedCountable: countable( - []string{"hello", "you-beautiful_world"}, - []int{1, 1}, - ), - }, - { - name: "tokenization whitespace", - tokenization: models.PropertyTokenizationWhitespace, - input: input, - expectedCountable: countable( - []string{"Hello", "you-beautiful_World"}, - []int{1, 1}, - ), - }, - { - name: "tokenization field", - tokenization: models.PropertyTokenizationField, - input: input, - expectedCountable: countable( - []string{"Hello you-beautiful_World"}, - []int{1}, - ), - }, - { - name: "non existing tokenization", - tokenization: "non_existing", - input: input, - expectedCountable: []Countable{}, - }, - } - - for _, tc := range testCases { - countable := a.Text(tc.tokenization, tc.input) - assert.ElementsMatch(t, tc.expectedCountable, countable) - } - }) - - t.Run("with text array", func(t *testing.T) { - type testCase struct { - name string - tokenization string - input []string - expectedCountable []Countable - } - - testCases := []testCase{ - { - name: "tokenization word", - tokenization: models.PropertyTokenizationWord, - input: []string{input, input}, - expectedCountable: countable( - []string{"hello", "you", "beautiful", "world"}, - []int{2, 2, 2, 2}, - ), - }, - { - name: "tokenization lowercase", - tokenization: models.PropertyTokenizationLowercase, - input: []string{input, input}, - expectedCountable: countable( - []string{"hello", "you-beautiful_world"}, - []int{2, 2}, - ), - }, - { - name: "tokenization whitespace", - tokenization: models.PropertyTokenizationWhitespace, - input: []string{input, input}, - expectedCountable: countable( - []string{"Hello", "you-beautiful_World"}, - []int{2, 2}, - ), - }, - { - name: "tokenization field", - tokenization: models.PropertyTokenizationField, - input: []string{input, input}, - expectedCountable: countable( - []string{"Hello you-beautiful_World"}, - []int{2}, - ), - }, - { - name: "non existing tokenization", - tokenization: "non_existing", - input: []string{input, input}, - expectedCountable: []Countable{}, - }, - } - - for _, tc := range testCases { - countable := a.TextArray(tc.tokenization, tc.input) - assert.ElementsMatch(t, tc.expectedCountable, countable) - } - }) -} - -type fakeStopwordDetector struct{} - -func (fsd fakeStopwordDetector) IsStopword(word string) bool { - return false -} - -func TestDedupItems(t *testing.T) { - props := []Property{ - { - Name: "propNothingToDo", - Items: []Countable{ - {Data: []byte("fff"), TermFrequency: 3}, - {Data: []byte("eee"), TermFrequency: 2}, - {Data: []byte("ddd"), TermFrequency: 1}, - }, - }, - { - Name: "propToDedup1", - Items: []Countable{ - {Data: []byte("aaa"), TermFrequency: 1}, - {Data: []byte("bbb"), TermFrequency: 2}, - {Data: []byte("ccc"), TermFrequency: 3}, - {Data: []byte("aaa"), TermFrequency: 4}, - {Data: []byte("ccc"), TermFrequency: 0}, - }, - }, - { - Name: "propToDedup2", - Items: []Countable{ - {Data: []uint8{1}, TermFrequency: 5}, - {Data: []uint8{1}, TermFrequency: 4}, - {Data: []uint8{1}, TermFrequency: 3}, - {Data: []uint8{1}, TermFrequency: 2}, - {Data: []uint8{1}, TermFrequency: 1}, - }, - }, - } - - expectedProps := []Property{ - { - Name: "propNothingToDo", - Items: []Countable{ - {Data: []byte("fff"), TermFrequency: 3}, - {Data: []byte("eee"), TermFrequency: 2}, - {Data: []byte("ddd"), TermFrequency: 1}, - }, - }, - { - Name: "propToDedup1", - Items: []Countable{ - {Data: []byte("bbb"), TermFrequency: 2}, - {Data: []byte("aaa"), TermFrequency: 4}, - {Data: []byte("ccc"), TermFrequency: 0}, - }, - }, - { - Name: "propToDedup2", - Items: []Countable{ - {Data: []uint8{1}, TermFrequency: 1}, - }, - }, - } - - dedupProps := DedupItems(props) - assert.Equal(t, expectedProps, dedupProps) -} diff --git a/adapters/repos/db/inverted/bm25_searcher.go b/adapters/repos/db/inverted/bm25_searcher.go deleted file mode 100644 index de043e494a2bbcfd8e942ce5de5e3e65e5f78671..0000000000000000000000000000000000000000 --- a/adapters/repos/db/inverted/bm25_searcher.go +++ /dev/null @@ -1,627 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package inverted - -import ( - "context" - "encoding/binary" - "fmt" - "math" - "sort" - "strconv" - "strings" - - "github.com/weaviate/weaviate/adapters/repos/db/inverted/stopwords" - "golang.org/x/sync/errgroup" - - "github.com/weaviate/sroar" - "github.com/weaviate/weaviate/adapters/repos/db/priorityqueue" - - "github.com/weaviate/weaviate/entities/inverted" - "github.com/weaviate/weaviate/entities/models" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv" - "github.com/weaviate/weaviate/adapters/repos/db/propertyspecific" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/searchparams" - "github.com/weaviate/weaviate/entities/storobj" -) - -type BM25Searcher struct { - config schema.BM25Config - store *lsmkv.Store - schema schema.Schema - classSearcher ClassSearcher // to allow recursive searches on ref-props - propIndices propertyspecific.Indices - propLenTracker propLengthRetriever - logger logrus.FieldLogger - shardVersion uint16 -} - -type propLengthRetriever interface { - PropertyMean(prop string) (float32, error) -} - -func NewBM25Searcher(config schema.BM25Config, store *lsmkv.Store, - schema schema.Schema, propIndices propertyspecific.Indices, - classSearcher ClassSearcher, propLenTracker propLengthRetriever, - logger logrus.FieldLogger, shardVersion uint16, -) *BM25Searcher { - return &BM25Searcher{ - config: config, - store: store, - schema: schema, - propIndices: propIndices, - classSearcher: classSearcher, - propLenTracker: propLenTracker, - logger: logger.WithField("action", "bm25_search"), - shardVersion: shardVersion, - } -} - -func (b *BM25Searcher) BM25F(ctx context.Context, filterDocIds helpers.AllowList, className schema.ClassName, limit int, keywordRanking searchparams.KeywordRanking) ([]*storobj.Object, []float32, error) { - // WEAVIATE-471 - If a property is not searchable, return an error - for _, property := range keywordRanking.Properties { - if !PropertyHasSearchableIndex(b.schema.Objects, string(className), property) { - return nil, nil, inverted.NewMissingSearchableIndexError(property) - } - } - class, err := schema.GetClassByName(b.schema.Objects, string(className)) - if err != nil { - return nil, nil, err - } - - objs, scores, err := b.wand(ctx, filterDocIds, class, keywordRanking, limit) - if err != nil { - return nil, nil, errors.Wrap(err, "wand") - } - - return objs, scores, nil -} - -func (b *BM25Searcher) GetPropertyLengthTracker() *JsonShardMetaData { - return b.propLenTracker.(*JsonShardMetaData) -} - -func (b *BM25Searcher) wand( - ctx context.Context, filterDocIds helpers.AllowList, class *models.Class, params searchparams.KeywordRanking, limit int, -) ([]*storobj.Object, []float32, error) { - N := float64(b.store.Bucket(helpers.ObjectsBucketLSM).Count()) - - var stopWordDetector *stopwords.Detector - if class.InvertedIndexConfig != nil && class.InvertedIndexConfig.Stopwords != nil { - var err error - stopWordDetector, err = stopwords.NewDetectorFromConfig(*(class.InvertedIndexConfig.Stopwords)) - if err != nil { - return nil, nil, err - } - } - - // There are currently cases, for different tokenization: - // word, lowercase, whitespace and field. - // Query is tokenized and respective properties are then searched for the search terms, - // results at the end are combined using WAND - tokenizationsOrdered := []string{ - models.PropertyTokenizationWord, - models.PropertyTokenizationLowercase, - models.PropertyTokenizationWhitespace, - models.PropertyTokenizationField, - } - - queryTermsByTokenization := map[string][]string{} - duplicateBoostsByTokenization := map[string][]int{} - propNamesByTokenization := map[string][]string{} - propertyBoosts := make(map[string]float32, len(params.Properties)) - - for _, tokenization := range tokenizationsOrdered { - queryTermsByTokenization[tokenization], duplicateBoostsByTokenization[tokenization] = helpers.TokenizeAndCountDuplicates(tokenization, params.Query) - - // stopword filtering for word tokenization - if tokenization == models.PropertyTokenizationWord { - queryTermsByTokenization[tokenization], duplicateBoostsByTokenization[tokenization] = b.removeStopwordsFromQueryTerms(queryTermsByTokenization[tokenization], duplicateBoostsByTokenization[tokenization], stopWordDetector) - } - - propNamesByTokenization[tokenization] = make([]string, 0) - } - - averagePropLength := 0. - for _, propertyWithBoost := range params.Properties { - property := propertyWithBoost - propBoost := 1 - if strings.Contains(propertyWithBoost, "^") { - property = strings.Split(propertyWithBoost, "^")[0] - boostStr := strings.Split(propertyWithBoost, "^")[1] - propBoost, _ = strconv.Atoi(boostStr) - } - propertyBoosts[property] = float32(propBoost) - - propMean, err := b.GetPropertyLengthTracker().PropertyMean(property) - if err != nil { - return nil, nil, err - } - averagePropLength += float64(propMean) - - prop, err := schema.GetPropertyByName(class, property) - if err != nil { - return nil, nil, err - } - - switch dt, _ := schema.AsPrimitive(prop.DataType); dt { - case schema.DataTypeText, schema.DataTypeTextArray: - if _, exists := propNamesByTokenization[prop.Tokenization]; !exists { - return nil, nil, fmt.Errorf("cannot handle tokenization '%v' of property '%s'", - prop.Tokenization, prop.Name) - } - propNamesByTokenization[prop.Tokenization] = append(propNamesByTokenization[prop.Tokenization], property) - default: - return nil, nil, fmt.Errorf("cannot handle datatype '%v' of property '%s'", dt, prop.Name) - } - } - - averagePropLength = averagePropLength / float64(len(params.Properties)) - - // preallocate the results - lengthAllResults := 0 - for tokenization, propNames := range propNamesByTokenization { - if len(propNames) > 0 { - lengthAllResults += len(queryTermsByTokenization[tokenization]) - } - } - results := make(terms, lengthAllResults) - indices := make([]map[uint64]int, lengthAllResults) - - var eg errgroup.Group - eg.SetLimit(_NUMCPU) - offset := 0 - - for _, tokenization := range tokenizationsOrdered { - propNames := propNamesByTokenization[tokenization] - if len(propNames) > 0 { - queryTerms := queryTermsByTokenization[tokenization] - duplicateBoosts := duplicateBoostsByTokenization[tokenization] - - for i := range queryTerms { - j := i - k := i + offset - - eg.Go(func() error { - termResult, docIndices, err := b.createTerm(N, filterDocIds, queryTerms[j], propNames, - propertyBoosts, duplicateBoosts[j], params.AdditionalExplanations) - if err != nil { - return err - } - results[k] = termResult - indices[k] = docIndices - return nil - }) - } - offset += len(queryTerms) - } - } - - if err := eg.Wait(); err != nil { - return nil, nil, err - } - // all results. Sum up the length of the results from all terms to get an upper bound of how many results there are - if limit == 0 { - for _, ind := range indices { - limit += len(ind) - } - } - - // the results are needed in the original order to be able to locate frequency/property length for the top-results - resultsOriginalOrder := make(terms, len(results)) - copy(resultsOriginalOrder, results) - - topKHeap := b.getTopKHeap(limit, results, averagePropLength) - return b.getTopKObjects(topKHeap, resultsOriginalOrder, indices, params.AdditionalExplanations) -} - -func (b *BM25Searcher) removeStopwordsFromQueryTerms(queryTerms []string, duplicateBoost []int, detector *stopwords.Detector) ([]string, []int) { - if detector == nil || len(queryTerms) == 0 { - return queryTerms, duplicateBoost - } - - i := 0 -WordLoop: - for { - if i == len(queryTerms) { - return queryTerms, duplicateBoost - } - queryTerm := queryTerms[i] - if detector.IsStopword(queryTerm) { - queryTerms[i] = queryTerms[len(queryTerms)-1] - queryTerms = queryTerms[:len(queryTerms)-1] - duplicateBoost[i] = duplicateBoost[len(duplicateBoost)-1] - duplicateBoost = duplicateBoost[:len(duplicateBoost)-1] - - continue WordLoop - } - - i++ - } -} - -func (b *BM25Searcher) getTopKObjects(topKHeap *priorityqueue.Queue[any], - results terms, indices []map[uint64]int, additionalExplanations bool, -) ([]*storobj.Object, []float32, error) { - objectsBucket := b.store.Bucket(helpers.ObjectsBucketLSM) - if objectsBucket == nil { - return nil, nil, errors.Errorf("objects bucket not found") - } - - objects := make([]*storobj.Object, 0, topKHeap.Len()) - scores := make([]float32, 0, topKHeap.Len()) - - buf := make([]byte, 8) - for topKHeap.Len() > 0 { - res := topKHeap.Pop() - binary.LittleEndian.PutUint64(buf, res.ID) - objectByte, err := objectsBucket.GetBySecondary(0, buf) - if err != nil { - return nil, nil, err - } - - if len(objectByte) == 0 { - b.logger.Warnf("Skipping object in BM25: object with id %v has a length of 0 bytes.", res.ID) - continue - } - - obj, err := storobj.FromBinary(objectByte) - if err != nil { - return nil, nil, err - } - - if additionalExplanations { - // add score explanation - if obj.AdditionalProperties() == nil { - obj.Object.Additional = make(map[string]interface{}) - } - for j, result := range results { - if termIndice, ok := indices[j][res.ID]; ok { - queryTerm := result.queryTerm - obj.Object.Additional["BM25F_"+queryTerm+"_frequency"] = result.data[termIndice].frequency - obj.Object.Additional["BM25F_"+queryTerm+"_propLength"] = result.data[termIndice].propLength - } - } - } - objects = append(objects, obj) - scores = append(scores, res.Dist) - - } - return objects, scores, nil -} - -func (b *BM25Searcher) getTopKHeap(limit int, results terms, averagePropLength float64, -) *priorityqueue.Queue[any] { - topKHeap := priorityqueue.NewMin[any](limit) - worstDist := float64(-10000) // tf score can be negative - sort.Sort(results) - for { - if results.completelyExhausted() || results.pivot(worstDist) { - return topKHeap - } - - id, score := results.scoreNext(averagePropLength, b.config) - - if topKHeap.Len() < limit || topKHeap.Top().Dist < float32(score) { - topKHeap.Insert(id, float32(score)) - for topKHeap.Len() > limit { - topKHeap.Pop() - } - // only update the worst distance when the queue is full, otherwise results can be missing if the first - // entry that is checked already has a very high score - if topKHeap.Len() >= limit { - worstDist = float64(topKHeap.Top().Dist) - } - } - } -} - -func (b *BM25Searcher) createTerm(N float64, filterDocIds helpers.AllowList, query string, propertyNames []string, propertyBoosts map[string]float32, duplicateTextBoost int, additionalExplanations bool) (term, map[uint64]int, error) { - termResult := term{queryTerm: query} - filteredDocIDs := sroar.NewBitmap() // to build the global n if there is a filter - - allMsAndProps := make(AllMapPairsAndPropName, 0, len(propertyNames)) - for _, propName := range propertyNames { - - bucket := b.store.Bucket(helpers.BucketSearchableFromPropNameLSM(propName)) - if bucket == nil { - return termResult, nil, fmt.Errorf("could not find bucket for property %v", propName) - } - preM, err := bucket.MapList([]byte(query)) - if err != nil { - return termResult, nil, err - } - - var m []lsmkv.MapPair - if filterDocIds != nil { - m = make([]lsmkv.MapPair, 0, len(preM)) - for _, val := range preM { - docID := binary.BigEndian.Uint64(val.Key) - if filterDocIds.Contains(docID) { - m = append(m, val) - } else { - filteredDocIDs.Set(docID) - } - } - } else { - m = preM - } - if len(m) == 0 { - continue - } - - allMsAndProps = append(allMsAndProps, MapPairsAndPropName{MapPairs: m, propname: propName}) - } - - // sort ascending, this code has two effects - // 1) We can skip writing the indices from the last property to the map (see next comment). Therefore, having the - // biggest property at the end will save us most writes on average - // 2) For the first property all entries are new, and we can create the map with the respective size. When choosing - // the second-biggest entry as the first property we save additional allocations later - sort.Sort(allMsAndProps) - if len(allMsAndProps) > 2 { - allMsAndProps[len(allMsAndProps)-2], allMsAndProps[0] = allMsAndProps[0], allMsAndProps[len(allMsAndProps)-2] - } - - var docMapPairs []docPointerWithScore = nil - var docMapPairsIndices map[uint64]int = nil - for i, mAndProps := range allMsAndProps { - m := mAndProps.MapPairs - propName := mAndProps.propname - - // The indices are needed for two things: - // a) combining the results of different properties - // b) Retrieve additional information that helps to understand the results when debugging. The retrieval is done - // in a later step, after it is clear which objects are the most relevant - // - // When b) is not needed the results from the last property do not need to be added to the index-map as there - // won't be any follow-up combinations. - includeIndicesForLastElement := false - if additionalExplanations || i < len(allMsAndProps)-1 { - includeIndicesForLastElement = true - } - - // only create maps/slices if we know how many entries there are - if docMapPairs == nil { - docMapPairs = make([]docPointerWithScore, 0, len(m)) - docMapPairsIndices = make(map[uint64]int, len(m)) - for k, val := range m { - if len(val.Value) < 8 { - b.logger.Warnf("Skipping pair in BM25: MapPair.Value should be 8 bytes long, but is %d.", len(val.Value)) - continue - } - freqBits := binary.LittleEndian.Uint32(val.Value[0:4]) - propLenBits := binary.LittleEndian.Uint32(val.Value[4:8]) - docMapPairs = append(docMapPairs, - docPointerWithScore{ - id: binary.BigEndian.Uint64(val.Key), - frequency: math.Float32frombits(freqBits) * propertyBoosts[propName], - propLength: math.Float32frombits(propLenBits), - }) - if includeIndicesForLastElement { - docMapPairsIndices[binary.BigEndian.Uint64(val.Key)] = k - } - } - } else { - for _, val := range m { - if len(val.Value) < 8 { - b.logger.Warnf("Skipping pair in BM25: MapPair.Value should be 8 bytes long, but is %d.", len(val.Value)) - continue - } - key := binary.BigEndian.Uint64(val.Key) - ind, ok := docMapPairsIndices[key] - freqBits := binary.LittleEndian.Uint32(val.Value[0:4]) - propLenBits := binary.LittleEndian.Uint32(val.Value[4:8]) - if ok { - docMapPairs[ind].propLength += math.Float32frombits(propLenBits) - docMapPairs[ind].frequency += math.Float32frombits(freqBits) * propertyBoosts[propName] - } else { - docMapPairs = append(docMapPairs, - docPointerWithScore{ - id: binary.BigEndian.Uint64(val.Key), - frequency: math.Float32frombits(freqBits) * propertyBoosts[propName], - propLength: math.Float32frombits(propLenBits), - }) - if includeIndicesForLastElement { - docMapPairsIndices[binary.BigEndian.Uint64(val.Key)] = len(docMapPairs) - 1 // current last entry - } - } - } - } - } - if docMapPairs == nil { - termResult.exhausted = true - return termResult, docMapPairsIndices, nil - } - termResult.data = docMapPairs - - n := float64(len(docMapPairs)) - if filterDocIds != nil { - n += float64(filteredDocIDs.GetCardinality()) - } - termResult.idf = math.Log(float64(1)+(N-n+0.5)/(n+0.5)) * float64(duplicateTextBoost) - - termResult.posPointer = 0 - termResult.idPointer = termResult.data[0].id - return termResult, docMapPairsIndices, nil -} - -type term struct { - // doubles as max impact (with tf=1, the max impact would be 1*idf), if there - // is a boost for a queryTerm, simply apply it here once - idf float64 - - idPointer uint64 - posPointer uint64 - data []docPointerWithScore - exhausted bool - queryTerm string -} - -func (t *term) scoreAndAdvance(averagePropLength float64, config schema.BM25Config) (uint64, float64) { - id := t.idPointer - pair := t.data[t.posPointer] - freq := float64(pair.frequency) - tf := freq / (freq + config.K1*(1-config.B+config.B*float64(pair.propLength)/averagePropLength)) - - // advance - t.posPointer++ - if t.posPointer >= uint64(len(t.data)) { - t.exhausted = true - } else { - t.idPointer = t.data[t.posPointer].id - } - - return id, tf * t.idf -} - -func (t *term) advanceAtLeast(minID uint64) { - for t.idPointer < minID { - t.posPointer++ - if t.posPointer >= uint64(len(t.data)) { - t.exhausted = true - return - } - t.idPointer = t.data[t.posPointer].id - } -} - -type terms []term - -func (t terms) completelyExhausted() bool { - for i := range t { - if !t[i].exhausted { - return false - } - } - return true -} - -func (t terms) pivot(minScore float64) bool { - minID, pivotPoint, abort := t.findMinID(minScore) - if abort { - return true - } - if pivotPoint == 0 { - return false - } - - t.advanceAllAtLeast(minID) - sort.Sort(t) - return false -} - -func (t terms) advanceAllAtLeast(minID uint64) { - for i := range t { - t[i].advanceAtLeast(minID) - } -} - -func (t terms) findMinID(minScore float64) (uint64, int, bool) { - cumScore := float64(0) - - for i, term := range t { - if term.exhausted { - continue - } - cumScore += term.idf - if cumScore >= minScore { - return term.idPointer, i, false - } - } - - return 0, 0, true -} - -func (t terms) findFirstNonExhausted() (int, bool) { - for i := range t { - if !t[i].exhausted { - return i, true - } - } - - return -1, false -} - -func (t terms) scoreNext(averagePropLength float64, config schema.BM25Config) (uint64, float64) { - pos, ok := t.findFirstNonExhausted() - if !ok { - // done, nothing left to score - return 0, 0 - } - - id := t[pos].idPointer - var cumScore float64 - for i := pos; i < len(t); i++ { - if t[i].idPointer != id || t[i].exhausted { - continue - } - _, score := t[i].scoreAndAdvance(averagePropLength, config) - cumScore += score - } - - sort.Sort(t) // pointer was advanced in scoreAndAdvance - - return id, cumScore -} - -// provide sort interface -func (t terms) Len() int { - return len(t) -} - -func (t terms) Less(i, j int) bool { - return t[i].idPointer < t[j].idPointer -} - -func (t terms) Swap(i, j int) { - t[i], t[j] = t[j], t[i] -} - -type MapPairsAndPropName struct { - propname string - MapPairs []lsmkv.MapPair -} - -type AllMapPairsAndPropName []MapPairsAndPropName - -// provide sort interface -func (m AllMapPairsAndPropName) Len() int { - return len(m) -} - -func (m AllMapPairsAndPropName) Less(i, j int) bool { - return len(m[i].MapPairs) < len(m[j].MapPairs) -} - -func (m AllMapPairsAndPropName) Swap(i, j int) { - m[i], m[j] = m[j], m[i] -} - -func PropertyHasSearchableIndex(schemaDefinition *models.Schema, className, tentativePropertyName string) bool { - propertyName := strings.Split(tentativePropertyName, "^")[0] - c, err := schema.GetClassByName(schemaDefinition, string(className)) - if err != nil { - return false - } - p, err := schema.GetPropertyByName(c, propertyName) - if err != nil { - return false - } - return HasSearchableIndex(p) -} diff --git a/adapters/repos/db/inverted/config.go b/adapters/repos/db/inverted/config.go deleted file mode 100644 index 42d2eabb8f2647aa66fe7dbf00469e3e8bcad8f2..0000000000000000000000000000000000000000 --- a/adapters/repos/db/inverted/config.go +++ /dev/null @@ -1,166 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package inverted - -import ( - "runtime" - "strings" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/adapters/repos/db/inverted/stopwords" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/usecases/config" -) - -var _NUMCPU = runtime.NumCPU() - -func ValidateConfig(conf *models.InvertedIndexConfig) error { - if conf.CleanupIntervalSeconds < 0 { - return errors.Errorf("cleanup interval seconds must be > 0") - } - - err := validateBM25Config(conf.Bm25) - if err != nil { - return err - } - - err = validateStopwordConfig(conf.Stopwords) - if err != nil { - return err - } - - return nil -} - -func ConfigFromModel(iicm *models.InvertedIndexConfig) schema.InvertedIndexConfig { - var conf schema.InvertedIndexConfig - - conf.IndexTimestamps = iicm.IndexTimestamps - conf.IndexNullState = iicm.IndexNullState - conf.IndexPropertyLength = iicm.IndexPropertyLength - - if iicm.Bm25 == nil { - conf.BM25.K1 = float64(config.DefaultBM25k1) - conf.BM25.B = float64(config.DefaultBM25b) - } else { - conf.BM25.K1 = float64(iicm.Bm25.K1) - conf.BM25.B = float64(iicm.Bm25.B) - } - - if iicm.Stopwords == nil { - conf.Stopwords = models.StopwordConfig{ - Preset: stopwords.EnglishPreset, - } - } else { - conf.Stopwords.Preset = iicm.Stopwords.Preset - conf.Stopwords.Additions = iicm.Stopwords.Additions - conf.Stopwords.Removals = iicm.Stopwords.Removals - } - - return conf -} - -func validateBM25Config(conf *models.BM25Config) error { - if conf == nil { - return nil - } - - if conf.K1 < 0 { - return errors.Errorf("BM25.k1 must be >= 0") - } - if conf.B < 0 || conf.B > 1 { - return errors.Errorf("BM25.b must be <= 0 and <= 1") - } - - return nil -} - -func validateStopwordConfig(conf *models.StopwordConfig) error { - if conf == nil { - conf = &models.StopwordConfig{} - } - - if conf.Preset == "" { - conf.Preset = stopwords.EnglishPreset - } - - if _, ok := stopwords.Presets[conf.Preset]; !ok { - return errors.Errorf("stopwordPreset '%s' does not exist", conf.Preset) - } - - err := validateStopwordAdditionsRemovals(conf) - if err != nil { - return err - } - - return nil -} - -func validateStopwordAdditionsRemovals(conf *models.StopwordConfig) error { - // the same stopword cannot exist - // in both additions and removals - foundAdditions := make(map[string]int) - - for idx, add := range conf.Additions { - if strings.TrimSpace(add) == "" { - return errors.Errorf("cannot use whitespace in stopword.additions") - } - - // save the index of the addition since it - // is readily available here. we will need - // this below when trimming additions that - // already exist in the selected preset - foundAdditions[add] = idx - } - - for _, rem := range conf.Removals { - if strings.TrimSpace(rem) == "" { - return errors.Errorf("cannot use whitespace in stopword.removals") - } - - if _, ok := foundAdditions[rem]; ok { - return errors.Errorf( - "found '%s' in both stopwords.additions and stopwords.removals", rem) - } - } - - removeStopwordAdditionsIfInPreset(conf, foundAdditions) - return nil -} - -func removeStopwordAdditionsIfInPreset(conf *models.StopwordConfig, foundAdditions map[string]int) { - presets := stopwords.Presets[conf.Preset] - - // if any of the elements in stopwords.additions - // already exist in the preset, mark it as to - // be removed - indicesToRemove := make(map[int]bool) - for _, preset := range presets { - if idx, ok := foundAdditions[preset]; ok { - indicesToRemove[idx] = true - } - } - - if len(indicesToRemove) == 0 { - return - } - - // take remaining additions, build new list - var trimmedAdditions []string - for idx, add := range conf.Additions { - if _, ok := indicesToRemove[idx]; !ok { - trimmedAdditions = append(trimmedAdditions, add) - } - } - conf.Additions = trimmedAdditions -} diff --git a/adapters/repos/db/inverted/config_test.go b/adapters/repos/db/inverted/config_test.go deleted file mode 100644 index 73786d67948265e5475df358b2e03f329dba9666..0000000000000000000000000000000000000000 --- a/adapters/repos/db/inverted/config_test.go +++ /dev/null @@ -1,234 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package inverted - -import ( - "math" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/usecases/config" -) - -const float64EqualityThreshold = 1e-6 - -func almostEqual(t *testing.T, a, b float64) bool { - closeEnough := math.Abs(a-b) <= float64EqualityThreshold - if !closeEnough { - t.Logf("%f and %f differ by more than a threshold of %f", - a, b, float64EqualityThreshold) - } - return closeEnough -} - -func TestValidateConfig(t *testing.T) { - t.Run("with invalid BM25.k1", func(t *testing.T) { - in := &models.InvertedIndexConfig{ - Bm25: &models.BM25Config{ - K1: -1, - B: 0.7, - }, - } - - err := ValidateConfig(in) - assert.EqualError(t, err, "BM25.k1 must be >= 0") - }) - - t.Run("with invalid BM25.b", func(t *testing.T) { - in := &models.InvertedIndexConfig{ - Bm25: &models.BM25Config{ - K1: 1, - B: 1.001, - }, - } - - err := ValidateConfig(in) - assert.EqualError(t, err, "BM25.b must be <= 0 and <= 1") - }) - - t.Run("with valid config", func(t *testing.T) { - in := &models.InvertedIndexConfig{ - Bm25: &models.BM25Config{ - K1: 1, - B: 0.1, - }, - } - - err := ValidateConfig(in) - assert.Nil(t, err) - }) - - t.Run("with nonexistent stopword preset", func(t *testing.T) { - in := &models.InvertedIndexConfig{ - Stopwords: &models.StopwordConfig{ - Preset: "DNE", - }, - } - - err := ValidateConfig(in) - assert.EqualError(t, err, "stopwordPreset 'DNE' does not exist") - }) - - t.Run("with whitespace stopword additions", func(t *testing.T) { - additions := [][]string{ - {"bats", " "}, - {""}, - {"something", " ", "skippable"}, - } - - for _, addList := range additions { - in := &models.InvertedIndexConfig{ - Stopwords: &models.StopwordConfig{ - Additions: addList, - }, - } - - err := ValidateConfig(in) - assert.EqualError(t, err, "cannot use whitespace in stopword.additions") - } - }) - - t.Run("with whitespace stopword removals", func(t *testing.T) { - removals := [][]string{ - {"bats", " "}, - {""}, - {"something", " ", "skippable"}, - } - - for _, remList := range removals { - in := &models.InvertedIndexConfig{ - Stopwords: &models.StopwordConfig{ - Removals: remList, - }, - } - - err := ValidateConfig(in) - assert.EqualError(t, err, "cannot use whitespace in stopword.removals") - } - }) - - t.Run("with shared additions/removals items", func(t *testing.T) { - in := &models.InvertedIndexConfig{ - Stopwords: &models.StopwordConfig{ - Additions: []string{"some", "words", "are", "different"}, - Removals: []string{"and", "some", "the", "same"}, - }, - } - - err := ValidateConfig(in) - assert.EqualError(t, err, - "found 'some' in both stopwords.additions and stopwords.removals") - }) - - t.Run("with additions that exist in preset", func(t *testing.T) { - tests := []struct { - additions []string - expectedLength int - }{ - { - additions: []string{"superfluous", "extravagant", "a"}, - expectedLength: 2, - }, - { - additions: []string{"a", "are", "the"}, - expectedLength: 0, - }, - { - additions: []string{"everyone", "sleeps", "eventually"}, - expectedLength: 3, - }, - } - - for _, test := range tests { - in := &models.InvertedIndexConfig{ - Stopwords: &models.StopwordConfig{ - Preset: "en", - Additions: test.additions, - }, - } - - err := ValidateConfig(in) - assert.Nil(t, err) - assert.Equal(t, test.expectedLength, len(in.Stopwords.Additions)) - } - }) -} - -func TestConfigFromModel(t *testing.T) { - t.Run("with all fields set", func(t *testing.T) { - k1 := 1.12 - b := 0.7 - - in := &models.InvertedIndexConfig{ - Bm25: &models.BM25Config{ - K1: float32(k1), - B: float32(b), - }, - Stopwords: &models.StopwordConfig{ - Preset: "en", - }, - } - - expected := schema.InvertedIndexConfig{ - BM25: schema.BM25Config{ - K1: k1, - B: b, - }, - Stopwords: models.StopwordConfig{ - Preset: "en", - }, - } - - conf := ConfigFromModel(in) - assert.True(t, almostEqual(t, conf.BM25.K1, expected.BM25.K1)) - assert.True(t, almostEqual(t, conf.BM25.B, expected.BM25.B)) - assert.Equal(t, expected.Stopwords, conf.Stopwords) - }) - - t.Run("with no BM25 params set", func(t *testing.T) { - interval := int64(1) - - in := &models.InvertedIndexConfig{ - CleanupIntervalSeconds: interval, - } - - expected := schema.InvertedIndexConfig{ - BM25: schema.BM25Config{ - K1: float64(config.DefaultBM25k1), - B: float64(config.DefaultBM25b), - }, - } - - conf := ConfigFromModel(in) - assert.True(t, almostEqual(t, conf.BM25.K1, expected.BM25.K1)) - assert.True(t, almostEqual(t, conf.BM25.B, expected.BM25.B)) - }) - - t.Run("with no Stopword config set", func(t *testing.T) { - interval := int64(1) - - in := &models.InvertedIndexConfig{ - CleanupIntervalSeconds: interval, - } - - expected := schema.InvertedIndexConfig{ - Stopwords: models.StopwordConfig{ - Preset: "en", - }, - } - - conf := ConfigFromModel(in) - assert.Equal(t, expected.Stopwords, conf.Stopwords) - }) -} diff --git a/adapters/repos/db/inverted/config_update.go b/adapters/repos/db/inverted/config_update.go deleted file mode 100644 index ddc5940ed8d688266a29aa7c2acd0bfee87ecb5c..0000000000000000000000000000000000000000 --- a/adapters/repos/db/inverted/config_update.go +++ /dev/null @@ -1,87 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package inverted - -import ( - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/models" -) - -func ValidateUserConfigUpdate(initial, updated *models.InvertedIndexConfig) error { - if updated.CleanupIntervalSeconds < 0 { - return errors.Errorf("cleanup interval seconds must be > 0") - } - - err := validateBM25ConfigUpdate(initial, updated) - if err != nil { - return err - } - - err = validateInvertedIndexConfigUpdate(initial, updated) - if err != nil { - return err - } - - err = validateStopwordsConfigUpdate(initial, updated) - if err != nil { - return err - } - - return nil -} - -func validateBM25ConfigUpdate(initial, updated *models.InvertedIndexConfig) error { - if updated.Bm25 == nil { - updated.Bm25 = &models.BM25Config{ - K1: initial.Bm25.K1, - B: initial.Bm25.B, - } - return nil - } - - err := validateBM25Config(updated.Bm25) - if err != nil { - return err - } - - return nil -} - -func validateInvertedIndexConfigUpdate(initial, updated *models.InvertedIndexConfig) error { - if updated.IndexPropertyLength != initial.IndexPropertyLength { - return errors.New("IndexPropertyLength cannot be changed when updating a schema") - } - - if updated.IndexNullState != initial.IndexNullState { - return errors.New("IndexNullState cannot be changed when updating a schema") - } - - return nil -} - -func validateStopwordsConfigUpdate(initial, updated *models.InvertedIndexConfig) error { - if updated.Stopwords == nil { - updated.Stopwords = &models.StopwordConfig{ - Preset: initial.Stopwords.Preset, - Additions: initial.Stopwords.Additions, - Removals: initial.Stopwords.Removals, - } - return nil - } - - err := validateStopwordConfig(updated.Stopwords) - if err != nil { - return err - } - - return nil -} diff --git a/adapters/repos/db/inverted/config_update_test.go b/adapters/repos/db/inverted/config_update_test.go deleted file mode 100644 index 99cf88094568e4982dfa5fa7c251db0a87e2ac94..0000000000000000000000000000000000000000 --- a/adapters/repos/db/inverted/config_update_test.go +++ /dev/null @@ -1,141 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package inverted - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/adapters/repos/db/inverted/stopwords" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/usecases/config" -) - -func TestValidateUserConfigUpdate(t *testing.T) { - validInitial := &models.InvertedIndexConfig{ - CleanupIntervalSeconds: 1, - Bm25: &models.BM25Config{ - K1: config.DefaultBM25k1, - B: config.DefaultBM25b, - }, - Stopwords: &models.StopwordConfig{ - Preset: stopwords.EnglishPreset, - }, - } - - t.Run("with valid updated config all fields", func(t *testing.T) { - updated := &models.InvertedIndexConfig{ - CleanupIntervalSeconds: 2, - Bm25: &models.BM25Config{ - K1: 1.3, - B: 0.778, - }, - Stopwords: &models.StopwordConfig{ - Preset: "en", - Additions: []string{"star", "nebula"}, - Removals: []string{"the", "a"}, - }, - } - - err := ValidateUserConfigUpdate(validInitial, updated) - require.Nil(t, err) - }) - - t.Run("with valid updated config missing BM25", func(t *testing.T) { - updated := &models.InvertedIndexConfig{ - CleanupIntervalSeconds: 2, - } - - err := ValidateUserConfigUpdate(validInitial, updated) - require.Nil(t, err) - assert.Equal(t, validInitial.Bm25.K1, updated.Bm25.K1) - assert.Equal(t, validInitial.Bm25.B, updated.Bm25.B) - }) - - t.Run("with valid updated config missing Stopwords", func(t *testing.T) { - updated := &models.InvertedIndexConfig{ - CleanupIntervalSeconds: 2, - } - - err := ValidateUserConfigUpdate(validInitial, updated) - require.Nil(t, err) - assert.Equal(t, validInitial.Stopwords.Preset, updated.Stopwords.Preset) - assert.Equal(t, validInitial.Stopwords.Additions, updated.Stopwords.Additions) - assert.Equal(t, validInitial.Stopwords.Removals, updated.Stopwords.Removals) - }) - - t.Run("with invalid cleanup interval", func(t *testing.T) { - updated := &models.InvertedIndexConfig{ - CleanupIntervalSeconds: -1, - } - - err := ValidateUserConfigUpdate(validInitial, updated) - require.EqualError(t, err, "cleanup interval seconds must be > 0") - }) - - t.Run("with invalid updated Bm25 config", func(t *testing.T) { - updated := &models.InvertedIndexConfig{ - CleanupIntervalSeconds: 1, - Bm25: &models.BM25Config{ - K1: 1.2, - B: 1.2, - }, - } - - err := ValidateUserConfigUpdate(validInitial, updated) - require.EqualError(t, err, "BM25.b must be <= 0 and <= 1") - }) - - t.Run("with invalid updated Stopwords preset config", func(t *testing.T) { - updated := &models.InvertedIndexConfig{ - CleanupIntervalSeconds: 1, - Stopwords: &models.StopwordConfig{ - Preset: "mongolian", - }, - } - - err := ValidateUserConfigUpdate(validInitial, updated) - require.EqualError(t, err, "stopwordPreset 'mongolian' does not exist") - }) - - t.Run("with invalid updated Stopwords addition/removal config", func(t *testing.T) { - updated := &models.InvertedIndexConfig{ - CleanupIntervalSeconds: 1, - Stopwords: &models.StopwordConfig{ - Additions: []string{"duplicate"}, - Removals: []string{"duplicate"}, - }, - } - - err := ValidateUserConfigUpdate(validInitial, updated) - require.EqualError(t, err, "found 'duplicate' in both stopwords.additions and stopwords.removals") - }) - - t.Run("with invalid updated inverted index null state change", func(t *testing.T) { - updated := &models.InvertedIndexConfig{ - IndexNullState: true, - } - - err := ValidateUserConfigUpdate(validInitial, updated) - require.EqualError(t, err, "IndexNullState cannot be changed when updating a schema") - }) - - t.Run("with invalid updated inverted index property length change", func(t *testing.T) { - updated := &models.InvertedIndexConfig{ - IndexPropertyLength: true, - } - - err := ValidateUserConfigUpdate(validInitial, updated) - require.EqualError(t, err, "IndexPropertyLength cannot be changed when updating a schema") - }) -} diff --git a/adapters/repos/db/inverted/delta_analyzer.go b/adapters/repos/db/inverted/delta_analyzer.go deleted file mode 100644 index 4cca7ef7561925ab17e0228da7da54e05a723059..0000000000000000000000000000000000000000 --- a/adapters/repos/db/inverted/delta_analyzer.go +++ /dev/null @@ -1,188 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package inverted - -import "bytes" - -type DeltaResults struct { - ToDelete []Property - ToAdd []Property -} - -func Delta(previous, next []Property) DeltaResults { - out := DeltaResults{} - - previous = DedupItems(previous) - next = DedupItems(next) - - if previous == nil { - out.ToAdd = next - return out - } - - previousByProp := map[string]Property{} - for _, prevProp := range previous { - previousByProp[prevProp.Name] = prevProp - } - - for _, nextProp := range next { - prevProp, ok := previousByProp[nextProp.Name] - if !ok { - // this prop didn't exist before so we can add all of it - out.ToAdd = append(out.ToAdd, nextProp) - continue - } - delete(previousByProp, nextProp.Name) - - // there is a chance they're identical, such a check is pretty cheap and - // it could prevent us from running an expensive merge, so let's try our - // luck - if listsIdentical(prevProp.Items, nextProp.Items) { - // then we don't need to do anything about this prop - continue - } - - toAdd, toDelete := countableDelta(prevProp.Items, nextProp.Items) - if len(toAdd) > 0 { - out.ToAdd = append(out.ToAdd, Property{ - Name: nextProp.Name, - Items: toAdd, - Length: nextProp.Length, - HasFilterableIndex: nextProp.HasFilterableIndex, - HasSearchableIndex: nextProp.HasSearchableIndex, - }) - } - if len(toDelete) > 0 { - out.ToDelete = append(out.ToDelete, Property{ - Name: nextProp.Name, - Items: toDelete, - Length: prevProp.Length, - HasFilterableIndex: nextProp.HasFilterableIndex, - HasSearchableIndex: nextProp.HasSearchableIndex, - }) - } - // special case to update optional length/nil indexes on - // all values removed - if len(toAdd) == 0 && len(toDelete) > 0 && - nextProp.Length == 0 && prevProp.Length > 0 { - out.ToAdd = append(out.ToAdd, Property{ - Name: nextProp.Name, - Items: []Countable{}, - Length: 0, - HasFilterableIndex: nextProp.HasFilterableIndex, - HasSearchableIndex: nextProp.HasSearchableIndex, - }) - } - } - - // extend ToDelete with props from previous missing in next - for _, prevProp := range previous { - if _, ok := previousByProp[prevProp.Name]; ok { - out.ToDelete = append(out.ToDelete, prevProp) - } - } - - return out -} - -func countableDelta(prev, next []Countable) ([]Countable, []Countable) { - var ( - add []Countable - del []Countable - ) - - seenInPrev := map[string]Countable{} - - for _, prevItem := range prev { - seenInPrev[string(prevItem.Data)] = prevItem - } - - for _, nextItem := range next { - prev, ok := seenInPrev[string(nextItem.Data)] - if ok && prev.TermFrequency == nextItem.TermFrequency { - // we have an identical overlap, delete from old list - delete(seenInPrev, string(nextItem.Data)) - // don't add to new list - continue - } - - add = append(add, nextItem) - } - - // anything that's now left on the seenInPrev map must be deleted because - // it either - // - is no longer present - // - is still present, but with updated values - for _, prevItem := range prev { - if _, ok := seenInPrev[string(prevItem.Data)]; ok { - del = append(del, prevItem) - } - } - - return add, del -} - -func listsIdentical(a []Countable, b []Countable) bool { - if len(a) != len(b) { - // can't possibly be identical if they have different lengths, exit early - return false - } - - for i := range a { - if !bytes.Equal(a[i].Data, b[i].Data) || - a[i].TermFrequency != b[i].TermFrequency { - // return as soon as an item didn't match - return false - } - } - - // we have proven in O(n) time that both lists are identical - // while O(n) is the worst case for this check it prevents us from running a - // considerably more expensive merge - return true -} - -type DeltaNilResults struct { - ToDelete []NilProperty - ToAdd []NilProperty -} - -func DeltaNil(previous, next []NilProperty) DeltaNilResults { - out := DeltaNilResults{} - - if previous == nil { - out.ToAdd = next - return out - } - - previousByProp := map[string]NilProperty{} - for _, prevProp := range previous { - previousByProp[prevProp.Name] = prevProp - } - - for _, nextProp := range next { - if _, ok := previousByProp[nextProp.Name]; !ok { - out.ToAdd = append(out.ToAdd, nextProp) - continue - } - delete(previousByProp, nextProp.Name) - } - - // extend ToDelete with props from previous missing in next - for _, prevProp := range previous { - if _, ok := previousByProp[prevProp.Name]; ok { - out.ToDelete = append(out.ToDelete, prevProp) - } - } - - return out -} diff --git a/adapters/repos/db/inverted/delta_analyzer_test.go b/adapters/repos/db/inverted/delta_analyzer_test.go deleted file mode 100644 index 855ba5076fc6adee334efa2b5430b372995df930..0000000000000000000000000000000000000000 --- a/adapters/repos/db/inverted/delta_analyzer_test.go +++ /dev/null @@ -1,562 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package inverted - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestDeltaAnalyzer(t *testing.T) { - t.Run("without previous indexing", func(t *testing.T) { - previous := []Property(nil) - next := []Property{ - { - Name: "prop1", - Items: []Countable{ - { - Data: []byte("value1"), - TermFrequency: 7, - }, - { - Data: []byte("value2"), - TermFrequency: 3, - }, - }, - }, - { - Name: "prop2", - Items: []Countable{ - { - Data: []byte("value3"), - TermFrequency: 7, - }, - { - Data: []byte("value4"), - TermFrequency: 3, - }, - }, - }, - } - - res := Delta(previous, next) - assert.Equal(t, next, res.ToAdd) - assert.Len(t, res.ToDelete, 0) - }) - - t.Run("with previous indexing and no changes", func(t *testing.T) { - previous := []Property{ - { - Name: "prop1", - Items: []Countable{ - { - Data: []byte("value1"), - TermFrequency: 7, - }, - { - Data: []byte("value2"), - TermFrequency: 3, - }, - }, - }, - { - Name: "prop2", - Items: []Countable{ - { - Data: []byte("value3"), - TermFrequency: 7, - }, - { - Data: []byte("value4"), - TermFrequency: 3, - }, - }, - }, - } - next := []Property{ - { - Name: "prop1", - Items: []Countable{ - { - Data: []byte("value1"), - TermFrequency: 7, - }, - { - Data: []byte("value2"), - TermFrequency: 3, - }, - }, - }, - { - Name: "prop2", - Items: []Countable{ - { - Data: []byte("value3"), - TermFrequency: 7, - }, - { - Data: []byte("value4"), - TermFrequency: 3, - }, - }, - }, - } - - res := Delta(previous, next) - assert.Len(t, res.ToDelete, 0) - assert.Len(t, res.ToAdd, 0) - }) - - t.Run("with previous indexing - only additions", func(t *testing.T) { - previous := []Property{ - { - Name: "prop1", - Items: []Countable{ - { - Data: []byte("value2"), - TermFrequency: 3, - }, - }, - }, - { - Name: "prop2", - Items: []Countable{ - { - Data: []byte("value4"), - TermFrequency: 3, - }, - }, - }, - } - next := []Property{ - { - Name: "prop1", - Items: []Countable{ - { - Data: []byte("value1"), - TermFrequency: 7, - }, - { - Data: []byte("value2"), - TermFrequency: 3, - }, - }, - }, - { - Name: "prop2", - Items: []Countable{ - { - Data: []byte("value3"), - TermFrequency: 7, - }, - { - Data: []byte("value4"), - TermFrequency: 3, - }, - }, - }, - } - - expectedAdd := []Property{ - { - Name: "prop1", - Items: []Countable{ - { - Data: []byte("value1"), - TermFrequency: 7, - }, - }, - }, - { - Name: "prop2", - Items: []Countable{ - { - Data: []byte("value3"), - TermFrequency: 7, - }, - }, - }, - } - - res := Delta(previous, next) - assert.Equal(t, expectedAdd, res.ToAdd) - assert.Len(t, res.ToDelete, 0) - }) - - t.Run("with previous indexing - both additions and deletions", func(t *testing.T) { - previous := []Property{ - { - Name: "prop1", - Items: []Countable{ - { - Data: []byte("value2"), - TermFrequency: 3, - }, - }, - }, - { - Name: "prop2", - Items: []Countable{ - { - Data: []byte("value4"), - TermFrequency: 3, - }, - }, - }, - } - next := []Property{ - { - Name: "prop1", - Items: []Countable{ - { - Data: []byte("value1"), - TermFrequency: 7, - }, - }, - }, - { - Name: "prop2", - Items: []Countable{ - { - Data: []byte("value3"), - TermFrequency: 7, - }, - { - Data: []byte("value4"), - TermFrequency: 3, - }, - }, - }, - } - - expectedAdd := []Property{ - { - Name: "prop1", - Items: []Countable{ - { - Data: []byte("value1"), - TermFrequency: 7, - }, - }, - }, - { - Name: "prop2", - Items: []Countable{ - { - Data: []byte("value3"), - TermFrequency: 7, - }, - }, - }, - } - - expectedDelete := []Property{ - { - Name: "prop1", - Items: []Countable{ - { - Data: []byte("value2"), - TermFrequency: 3, - }, - }, - }, - } - - res := Delta(previous, next) - assert.Equal(t, expectedAdd, res.ToAdd) - assert.Equal(t, expectedDelete, res.ToDelete) - }) -} - -func TestDeltaAnalyzer_Arrays(t *testing.T) { - lexInt64 := func(val int64) []byte { - bytes, _ := LexicographicallySortableInt64(val) - return bytes - } - lexBool := func(val bool) []byte { - if val { - return []uint8{1} - } - return []uint8{0} - } - - t.Run("with previous indexing - both additions and deletions", func(t *testing.T) { - previous := []Property{ - { - Name: "ints", - Items: []Countable{ - {Data: lexInt64(101)}, - {Data: lexInt64(101)}, - {Data: lexInt64(101)}, - {Data: lexInt64(101)}, - {Data: lexInt64(101)}, - {Data: lexInt64(101)}, - {Data: lexInt64(102)}, - {Data: lexInt64(103)}, - {Data: lexInt64(104)}, - }, - Length: 9, - HasFilterableIndex: true, - HasSearchableIndex: false, - }, - { - Name: "booleans", - Items: []Countable{ - {Data: lexBool(true)}, - {Data: lexBool(true)}, - {Data: lexBool(true)}, - {Data: lexBool(false)}, - }, - Length: 4, - HasFilterableIndex: true, - HasSearchableIndex: false, - }, - { - Name: "numbers", - Items: []Countable{}, - Length: 0, - HasFilterableIndex: true, - HasSearchableIndex: false, - }, - { - Name: "texts", - Items: []Countable{ - {Data: []byte("aaa")}, - {Data: []byte("bbb")}, - {Data: []byte("ccc")}, - }, - Length: 3, - HasFilterableIndex: true, - HasSearchableIndex: false, - }, - { - Name: "dates", - Items: []Countable{ - {Data: []byte("2021-06-01T22:18:59.640162Z")}, - {Data: []byte("2022-06-01T22:18:59.640162Z")}, - }, - Length: 2, - HasFilterableIndex: true, - HasSearchableIndex: false, - }, - { - Name: "_creationTimeUnix", - Items: []Countable{ - {Data: []byte("1703778000000")}, - }, - Length: 0, - HasFilterableIndex: true, - HasSearchableIndex: false, - }, - { - Name: "_lastUpdateTimeUnix", - Items: []Countable{ - {Data: []byte("1703778000000")}, - }, - Length: 0, - HasFilterableIndex: true, - HasSearchableIndex: false, - }, - } - next := []Property{ - { - Name: "ints", - Items: []Countable{ - {Data: lexInt64(101)}, - {Data: lexInt64(101)}, - {Data: lexInt64(101)}, - {Data: lexInt64(101)}, - {Data: lexInt64(103)}, - {Data: lexInt64(104)}, - {Data: lexInt64(105)}, - }, - Length: 7, - HasFilterableIndex: true, - HasSearchableIndex: false, - }, - { - Name: "booleans", - Items: []Countable{ - {Data: lexBool(true)}, - {Data: lexBool(true)}, - {Data: lexBool(true)}, - {Data: lexBool(false)}, - }, - Length: 4, - HasFilterableIndex: true, - HasSearchableIndex: false, - }, - { - Name: "texts", - Items: []Countable{}, - Length: 0, - HasFilterableIndex: true, - HasSearchableIndex: false, - }, - { - Name: "_creationTimeUnix", - Items: []Countable{ - {Data: []byte("1703778000000")}, - }, - Length: 0, - HasFilterableIndex: true, - HasSearchableIndex: false, - }, - { - Name: "_lastUpdateTimeUnix", - Items: []Countable{ - {Data: []byte("1703778500000")}, - }, - Length: 0, - HasFilterableIndex: true, - HasSearchableIndex: false, - }, - } - - expectedAdd := []Property{ - { - Name: "ints", - Items: []Countable{ - {Data: lexInt64(105)}, - }, - Length: 7, - HasFilterableIndex: true, - HasSearchableIndex: false, - }, - { - Name: "texts", - Items: []Countable{}, - Length: 0, - HasFilterableIndex: true, - HasSearchableIndex: false, - }, - { - Name: "_lastUpdateTimeUnix", - Items: []Countable{ - {Data: []byte("1703778500000")}, - }, - Length: 0, - HasFilterableIndex: true, - HasSearchableIndex: false, - }, - } - expectedDelete := []Property{ - { - Name: "ints", - Items: []Countable{ - {Data: lexInt64(102)}, - }, - Length: 9, - HasFilterableIndex: true, - HasSearchableIndex: false, - }, - { - Name: "texts", - Items: []Countable{ - {Data: []byte("aaa")}, - {Data: []byte("bbb")}, - {Data: []byte("ccc")}, - }, - Length: 3, - HasFilterableIndex: true, - HasSearchableIndex: false, - }, - { - Name: "_lastUpdateTimeUnix", - Items: []Countable{ - {Data: []byte("1703778000000")}, - }, - Length: 0, - HasFilterableIndex: true, - HasSearchableIndex: false, - }, - { - Name: "numbers", - Items: []Countable{}, - Length: 0, - HasFilterableIndex: true, - HasSearchableIndex: false, - }, - { - Name: "dates", - Items: []Countable{ - {Data: []byte("2021-06-01T22:18:59.640162Z")}, - {Data: []byte("2022-06-01T22:18:59.640162Z")}, - }, - Length: 2, - HasFilterableIndex: true, - HasSearchableIndex: false, - }, - } - - delta := Delta(previous, next) - assert.Equal(t, expectedAdd, delta.ToAdd) - assert.Equal(t, expectedDelete, delta.ToDelete) - }) -} - -func TestDeltaNilAnalyzer(t *testing.T) { - previous := []NilProperty{ - { - Name: "ints", - AddToPropertyLength: false, - }, - { - Name: "booleans", - AddToPropertyLength: true, - }, - { - Name: "numbers", - AddToPropertyLength: true, - }, - } - next := []NilProperty{ - { - Name: "booleans", - AddToPropertyLength: true, - }, - { - Name: "texts", - AddToPropertyLength: true, - }, - { - Name: "dates", - AddToPropertyLength: false, - }, - } - - expectedAdd := []NilProperty{ - { - Name: "texts", - AddToPropertyLength: true, - }, - { - Name: "dates", - AddToPropertyLength: false, - }, - } - expectedDelete := []NilProperty{ - { - Name: "ints", - AddToPropertyLength: false, - }, - { - Name: "numbers", - AddToPropertyLength: true, - }, - } - - deltaNil := DeltaNil(previous, next) - assert.Equal(t, expectedAdd, deltaNil.ToAdd) - assert.Equal(t, expectedDelete, deltaNil.ToDelete) -} diff --git a/adapters/repos/db/inverted/delta_merger.go b/adapters/repos/db/inverted/delta_merger.go deleted file mode 100644 index b9df16c2fb7752a7a52b7bbdd26116e28d04836e..0000000000000000000000000000000000000000 --- a/adapters/repos/db/inverted/delta_merger.go +++ /dev/null @@ -1,226 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package inverted - -// DeltaMerger can be used to condense the number of single writes into one big -// one. Additionally it removes overlaps between additions and deletions. It is -// meant to be used in batch situation, where 5 ref objects in a row might each -// increase the doc count by one. Instead of writing 5 additions and 4 -// deletions, this can be condensed to write just one addition -type DeltaMerger struct { - additions propsByName - deletions propsByName -} - -func NewDeltaMerger() *DeltaMerger { - return &DeltaMerger{ - additions: propsByName{}, - deletions: propsByName{}, - } -} - -func (dm *DeltaMerger) AddAdditions(props []Property, docID uint64) { - for _, prop := range props { - storedProp := dm.additions.getOrCreate(prop.Name) - storedProp.hasFilterableIndex = prop.HasFilterableIndex - storedProp.hasSearchableIndex = prop.HasSearchableIndex - for _, item := range prop.Items { - storedItem := storedProp.getOrCreateItem(item.Data) - storedItem.addDocIDAndFrequency(docID, item.TermFrequency) - } - } -} - -func (dm *DeltaMerger) AddDeletions(props []Property, docID uint64) { - for _, prop := range props { - additionProp := dm.additions.getOrCreate(prop.Name) - for _, item := range prop.Items { - additionItem := additionProp.getOrCreateItem(item.Data) - ok := additionItem.deleteIfPresent(docID) - if ok { - // we are done with this prop, no need to register an explicit deletion - continue - } - - // this was not added by us, we need to remove it - deletionProp := dm.deletions.getOrCreate(prop.Name) - deletionProp.hasFilterableIndex = prop.HasFilterableIndex - deletionProp.hasSearchableIndex = prop.HasSearchableIndex - deletionItem := deletionProp.getOrCreateItem(item.Data) - deletionItem.addDocIDAndFrequency(docID, 0) // frequency does not matter on deletion - } - } -} - -func (dm *DeltaMerger) Merge() DeltaMergeResult { - return DeltaMergeResult{ - Additions: dm.additions.merge(), - Deletions: dm.deletions.merge(), - } -} - -type DeltaMergeResult struct { - Additions []MergeProperty - Deletions []MergeProperty -} - -type MergeProperty struct { - Name string - MergeItems []MergeItem - HasFilterableIndex bool - HasSearchableIndex bool -} - -type MergeItem struct { - Data []byte - DocIDs []MergeDocIDWithFrequency -} - -// IDs is meant for cases such as deletion, where the frequency is irrelevant, -// but the expected format is a []docID -func (mi MergeItem) IDs() []uint64 { - out := make([]uint64, len(mi.DocIDs)) - for i, tuple := range mi.DocIDs { - out[i] = tuple.DocID - } - - return out -} - -// Countable converts the merge item to a regular (non-merge) Countable. Note -// that this loses the IDs and Frequency information, so IDs have to be passed -// separately using .IDs() -func (mi MergeItem) Countable() Countable { - return Countable{ - Data: mi.Data, - } -} - -type MergeDocIDWithFrequency struct { - DocID uint64 - Frequency float32 -} - -type propsByName map[string]*propWithDocIDs - -func (pbn propsByName) getOrCreate(name string) *propWithDocIDs { - prop, ok := pbn[name] - if ok { - return prop - } - prop = &propWithDocIDs{name: name, items: map[string]*countableWithDocIDs{}} - pbn[name] = prop - return prop -} - -func (pbn propsByName) merge() []MergeProperty { - out := make([]MergeProperty, len(pbn)) - i := 0 - for _, prop := range pbn { - mergedProp := prop.merge() - if mergedProp == nil { - continue - } - out[i] = *mergedProp - i++ - } - - if i == 0 { - return nil - } - - return out[:i] -} - -type propWithDocIDs struct { - name string - items map[string]*countableWithDocIDs - hasFilterableIndex bool - hasSearchableIndex bool -} - -func (pwd *propWithDocIDs) getOrCreateItem(data []byte) *countableWithDocIDs { - name := string(data) - item, ok := pwd.items[name] - if ok { - return item - } - item = &countableWithDocIDs{ - value: data, - docIDs: map[uint64]float32{}, - } - pwd.items[name] = item - return item -} - -func (pwd *propWithDocIDs) merge() *MergeProperty { - items := make([]MergeItem, len(pwd.items)) - - i := 0 - for _, item := range pwd.items { - mergedItem := item.merge() - if mergedItem == nil { - continue - } - - items[i] = *mergedItem - i++ - } - - if i == 0 { - return nil - } - - return &MergeProperty{ - Name: pwd.name, - MergeItems: items[:i], - HasFilterableIndex: pwd.hasFilterableIndex, - HasSearchableIndex: pwd.hasSearchableIndex, - } -} - -type countableWithDocIDs struct { - value []byte - docIDs map[uint64]float32 // map[docid]frequency -} - -func (cwd *countableWithDocIDs) addDocIDAndFrequency(docID uint64, freq float32) { - cwd.docIDs[docID] = freq -} - -func (cwd *countableWithDocIDs) deleteIfPresent(docID uint64) bool { - _, ok := cwd.docIDs[docID] - if !ok { - return false - } - - delete(cwd.docIDs, docID) - return true -} - -func (cwd *countableWithDocIDs) merge() *MergeItem { - if len(cwd.docIDs) == 0 { - return nil - } - - ids := make([]MergeDocIDWithFrequency, len(cwd.docIDs)) - i := 0 - for docID, freq := range cwd.docIDs { - ids[i] = MergeDocIDWithFrequency{DocID: docID, Frequency: freq} - i++ - } - - return &MergeItem{ - Data: cwd.value, - DocIDs: ids, - } -} diff --git a/adapters/repos/db/inverted/delta_merger_test.go b/adapters/repos/db/inverted/delta_merger_test.go deleted file mode 100644 index c415d3c8995fbb212485a32e124f64e83e386e03..0000000000000000000000000000000000000000 --- a/adapters/repos/db/inverted/delta_merger_test.go +++ /dev/null @@ -1,62 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package inverted - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestDeltaMerger(t *testing.T) { - dm := NewDeltaMerger() - - t.Run("a simple add and delete with one prop and one doc id", func(t *testing.T) { - dm.AddAdditions([]Property{{ - Name: "field1", Items: []Countable{ - {Data: []byte("a")}, - {Data: []byte("b")}, - }, - HasFilterableIndex: false, - HasSearchableIndex: true, - }}, 0) - - dm.AddDeletions([]Property{{ - Name: "field1", Items: []Countable{ - {Data: []byte("a")}, - }, - }}, 0) - - expected := DeltaMergeResult{ - Additions: []MergeProperty{ - { - Name: "field1", - HasFilterableIndex: false, - HasSearchableIndex: true, - MergeItems: []MergeItem{ - { - Data: []byte("b"), - DocIDs: []MergeDocIDWithFrequency{ - { - DocID: 0, - }, - }, - }, - }, - }, - }, - } - - actual := dm.Merge() - assert.Equal(t, expected, actual) - }) -} diff --git a/adapters/repos/db/inverted/filters_integration_test.go b/adapters/repos/db/inverted/filters_integration_test.go deleted file mode 100644 index dfac928f9fa9f16fd5358620a17098da4aec70ac..0000000000000000000000000000000000000000 --- a/adapters/repos/db/inverted/filters_integration_test.go +++ /dev/null @@ -1,672 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package inverted - -import ( - "context" - "encoding/binary" - "testing" - - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/cyclemanager" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/usecases/config" -) - -const ( - className = "TestClass" -) - -func Test_Filters_String(t *testing.T) { - dirName := t.TempDir() - - logger, _ := test.NewNullLogger() - store, err := lsmkv.New(dirName, dirName, logger, nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop()) - require.Nil(t, err) - - propName := "inverted-with-frequency" - bucketName := helpers.BucketSearchableFromPropNameLSM(propName) - require.Nil(t, store.CreateOrLoadBucket(context.Background(), - bucketName, lsmkv.WithStrategy(lsmkv.StrategyMapCollection))) - bWithFrequency := store.Bucket(bucketName) - - defer store.Shutdown(context.Background()) - - fakeInvertedIndex := map[string][]uint64{ - "modulo-2": {2, 4, 6, 8, 10, 12, 14, 16}, - "modulo-3": {3, 6, 9, 12, 15}, - "modulo-4": {4, 8, 12, 16}, - "modulo-5": {5, 10, 15}, - "modulo-6": {6, 12}, - "modulo-7": {7, 14}, - "modulo-8": {8, 16}, - "modulo-9": {9}, - "modulo-10": {10}, - "modulo-11": {11}, - "modulo-12": {12}, - "modulo-13": {13}, - "modulo-14": {14}, - "modulo-15": {15}, - "modulo-16": {16}, - } - - t.Run("import data", func(t *testing.T) { - for value, ids := range fakeInvertedIndex { - idsMapValues := idsToBinaryMapValues(ids) - for _, pair := range idsMapValues { - require.Nil(t, bWithFrequency.MapSet([]byte(value), pair)) - } - } - - require.Nil(t, bWithFrequency.FlushAndSwitch()) - }) - - searcher := NewSearcher(logger, store, createSchema(), nil, nil, - fakeStopwordDetector{}, 2, func() bool { return false }, "", config.DefaultQueryNestedCrossReferenceLimit) - - type test struct { - name string - filter *filters.LocalFilter - expectedListBeforeUpdate helpers.AllowList - expectedListAfterUpdate helpers.AllowList - } - - tests := []test{ - { - name: "exact match - single level", - filter: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: "foo", - Property: schema.PropertyName(propName), - }, - Value: &filters.Value{ - Value: "modulo-7", - Type: schema.DataTypeText, - }, - }, - }, - expectedListBeforeUpdate: helpers.NewAllowList(7, 14), - expectedListAfterUpdate: helpers.NewAllowList(7, 14, 21), - }, - { - name: "like operator", - filter: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorLike, - On: &filters.Path{ - Class: "foo", - Property: schema.PropertyName(propName), - }, - Value: &filters.Value{ - Value: "modulo-1*", - Type: schema.DataTypeText, - }, - }, - }, - expectedListBeforeUpdate: helpers.NewAllowList(10, 11, 12, 13, 14, 15, 16), - expectedListAfterUpdate: helpers.NewAllowList(10, 11, 12, 13, 14, 15, 16, 17), - }, - { - name: "exact match - or filter", - filter: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorOr, - Operands: []filters.Clause{ - { - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: "foo", - Property: schema.PropertyName(propName), - }, - Value: &filters.Value{ - Value: "modulo-7", - Type: schema.DataTypeText, - }, - }, - { - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: "foo", - Property: schema.PropertyName(propName), - }, - Value: &filters.Value{ - Value: "modulo-8", - Type: schema.DataTypeText, - }, - }, - }, - }, - }, - expectedListBeforeUpdate: helpers.NewAllowList(7, 8, 14, 16), - expectedListAfterUpdate: helpers.NewAllowList(7, 8, 14, 16, 21), - }, - { - name: "exact match - and filter", - filter: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorAnd, - Operands: []filters.Clause{ - { - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: "foo", - Property: schema.PropertyName(propName), - }, - Value: &filters.Value{ - Value: "modulo-7", - Type: schema.DataTypeText, - }, - }, - { - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: "foo", - Property: schema.PropertyName(propName), - }, - Value: &filters.Value{ - Value: "modulo-14", - Type: schema.DataTypeText, - }, - }, - }, - }, - }, - expectedListBeforeUpdate: helpers.NewAllowList(14), - expectedListAfterUpdate: helpers.NewAllowList(14), - }, - { - // This test prevents a regression on - // https://github.com/weaviate/weaviate/issues/1770 - name: "combined and/or filter, see gh-1770", - filter: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorAnd, - Operands: []filters.Clause{ - // This part will produce results - { - Operator: filters.OperatorOr, - Operands: []filters.Clause{ - { - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: "foo", - Property: schema.PropertyName(propName), - }, - Value: &filters.Value{ - Value: "modulo-7", - Type: schema.DataTypeText, - }, - }, - { - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: "foo", - Property: schema.PropertyName(propName), - }, - Value: &filters.Value{ - Value: "modulo-8", - Type: schema.DataTypeText, - }, - }, - }, - }, - - // This part will produce no results - { - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: "foo", - Property: schema.PropertyName(propName), - }, - Value: &filters.Value{ - Value: "modulo-7000000", - Type: schema.DataTypeText, - }, - }, - }, - }, - }, - // prior to the fix of gh-1770 the second AND operand was ignored due to - // a missing hash in the merge and we would get results here, when we - // shouldn't - expectedListBeforeUpdate: helpers.NewAllowList(), - expectedListAfterUpdate: helpers.NewAllowList(), - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - t.Run("before update", func(t *testing.T) { - res, err := searcher.DocIDs(context.Background(), test.filter, - additional.Properties{}, className) - assert.Nil(t, err) - assert.Equal(t, test.expectedListBeforeUpdate.Slice(), res.Slice()) - }) - - t.Run("update", func(t *testing.T) { - value := []byte("modulo-7") - idsMapValues := idsToBinaryMapValues([]uint64{21}) - for _, pair := range idsMapValues { - require.Nil(t, bWithFrequency.MapSet([]byte(value), pair)) - } - - // for like filter - value = []byte("modulo-17") - idsMapValues = idsToBinaryMapValues([]uint64{17}) - for _, pair := range idsMapValues { - require.Nil(t, bWithFrequency.MapSet([]byte(value), pair)) - } - }) - - t.Run("after update", func(t *testing.T) { - res, err := searcher.DocIDs(context.Background(), test.filter, - additional.Properties{}, className) - assert.Nil(t, err) - assert.Equal(t, test.expectedListAfterUpdate.Slice(), res.Slice()) - }) - - t.Run("restore inverted index, so test suite can be run again", - func(t *testing.T) { - idsMapValues := idsToBinaryMapValues([]uint64{21}) - require.Nil(t, bWithFrequency.MapDeleteKey([]byte("modulo-7"), - idsMapValues[0].Key)) - - idsMapValues = idsToBinaryMapValues([]uint64{17}) - require.Nil(t, bWithFrequency.MapDeleteKey([]byte("modulo-17"), - idsMapValues[0].Key)) - }) - }) - } -} - -func Test_Filters_Int(t *testing.T) { - dirName := t.TempDir() - - logger, _ := test.NewNullLogger() - store, err := lsmkv.New(dirName, dirName, logger, nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop()) - require.Nil(t, err) - - propName := "inverted-without-frequency" - bucketName := helpers.BucketFromPropNameLSM(propName) - require.Nil(t, store.CreateOrLoadBucket(context.Background(), - bucketName, lsmkv.WithStrategy(lsmkv.StrategySetCollection))) - bucket := store.Bucket(bucketName) - - defer store.Shutdown(context.Background()) - - fakeInvertedIndex := map[int64][]uint64{ - 2: {2, 4, 6, 8, 10, 12, 14, 16}, - 3: {3, 6, 9, 12, 15}, - 4: {4, 8, 12, 16}, - 5: {5, 10, 15}, - 6: {6, 12}, - 7: {7, 14}, - 8: {8, 16}, - 9: {9}, - 10: {10}, - 11: {11}, - 12: {12}, - 13: {13}, - 14: {14}, - 15: {15}, - 16: {16}, - } - - t.Run("import data", func(t *testing.T) { - for value, ids := range fakeInvertedIndex { - idValues := idsToBinaryList(ids) - valueBytes, err := LexicographicallySortableInt64(value) - require.Nil(t, err) - require.Nil(t, bucket.SetAdd(valueBytes, idValues)) - } - - require.Nil(t, bucket.FlushAndSwitch()) - }) - - searcher := NewSearcher(logger, store, createSchema(), nil, nil, - fakeStopwordDetector{}, 2, func() bool { return false }, "", config.DefaultQueryNestedCrossReferenceLimit) - - type test struct { - name string - filter *filters.LocalFilter - expectedListBeforeUpdate helpers.AllowList - expectedListAfterUpdate helpers.AllowList - } - - tests := []test{ - { - name: "exact match - single level", - filter: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: "foo", - Property: schema.PropertyName(propName), - }, - Value: &filters.Value{ - Value: 7, - Type: schema.DataTypeInt, - }, - }, - }, - expectedListBeforeUpdate: helpers.NewAllowList(7, 14), - expectedListAfterUpdate: helpers.NewAllowList(7, 14, 21), - }, - { - name: "not equal", - filter: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorNotEqual, - On: &filters.Path{ - Class: "foo", - Property: schema.PropertyName(propName), - }, - Value: &filters.Value{ - Value: 13, - Type: schema.DataTypeInt, - }, - }, - }, - expectedListBeforeUpdate: helpers.NewAllowList(2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 15, 16), - expectedListAfterUpdate: helpers.NewAllowList(2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 15, 16, 21), - }, - { - name: "exact match - or filter", - filter: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorOr, - Operands: []filters.Clause{ - { - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: "foo", - Property: schema.PropertyName(propName), - }, - Value: &filters.Value{ - Value: 7, - Type: schema.DataTypeInt, - }, - }, - { - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: "foo", - Property: schema.PropertyName(propName), - }, - Value: &filters.Value{ - Value: 8, - Type: schema.DataTypeInt, - }, - }, - }, - }, - }, - expectedListBeforeUpdate: helpers.NewAllowList(7, 8, 14, 16), - expectedListAfterUpdate: helpers.NewAllowList(7, 8, 14, 16, 21), - }, - { - name: "exact match - and filter", - filter: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorAnd, - Operands: []filters.Clause{ - { - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: "foo", - Property: schema.PropertyName(propName), - }, - Value: &filters.Value{ - Value: 7, - Type: schema.DataTypeInt, - }, - }, - { - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: "foo", - Property: schema.PropertyName(propName), - }, - Value: &filters.Value{ - Value: 14, - Type: schema.DataTypeInt, - }, - }, - }, - }, - }, - expectedListBeforeUpdate: helpers.NewAllowList(14), - expectedListAfterUpdate: helpers.NewAllowList(14), - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - t.Run("before update", func(t *testing.T) { - res, err := searcher.DocIDs(context.Background(), test.filter, - additional.Properties{}, className) - assert.Nil(t, err) - assert.Equal(t, test.expectedListBeforeUpdate.Slice(), res.Slice()) - }) - - t.Run("update", func(t *testing.T) { - value, _ := LexicographicallySortableInt64(7) - idsBinary := idsToBinaryList([]uint64{21}) - require.Nil(t, bucket.SetAdd([]byte(value), idsBinary)) - }) - - t.Run("after update", func(t *testing.T) { - res, err := searcher.DocIDs(context.Background(), test.filter, - additional.Properties{}, className) - assert.Nil(t, err) - assert.Equal(t, test.expectedListAfterUpdate.Slice(), res.Slice()) - }) - - t.Run("restore inverted index, so we can run test suite again", - func(t *testing.T) { - idsList := idsToBinaryList([]uint64{21}) - value, _ := LexicographicallySortableInt64(7) - require.Nil(t, bucket.SetDeleteSingle(value, idsList[0])) - }) - }) - } -} - -// This prevents a regression on -// https://github.com/weaviate/weaviate/issues/1772 -func Test_Filters_String_DuplicateEntriesInAnd(t *testing.T) { - dirName := t.TempDir() - - logger, _ := test.NewNullLogger() - store, err := lsmkv.New(dirName, dirName, logger, nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop()) - require.Nil(t, err) - - propName := "inverted-with-frequency" - bucketName := helpers.BucketSearchableFromPropNameLSM(propName) - require.Nil(t, store.CreateOrLoadBucket(context.Background(), - bucketName, lsmkv.WithStrategy(lsmkv.StrategyMapCollection))) - bWithFrequency := store.Bucket(bucketName) - - defer store.Shutdown(context.Background()) - - fakeInvertedIndex := map[string][]uint64{ - "list_a": {0, 1}, - "list_b": {1, 1, 1, 1, 1}, - } - - t.Run("import data", func(t *testing.T) { - for value, ids := range fakeInvertedIndex { - idsMapValues := idsToBinaryMapValues(ids) - for _, pair := range idsMapValues { - require.Nil(t, bWithFrequency.MapSet([]byte(value), pair)) - } - } - require.Nil(t, bWithFrequency.FlushAndSwitch()) - }) - - searcher := NewSearcher(logger, store, createSchema(), nil, nil, - fakeStopwordDetector{}, 2, func() bool { return false }, "", config.DefaultQueryNestedCrossReferenceLimit) - - type test struct { - name string - filter *filters.LocalFilter - expectedListBeforeUpdate helpers.AllowList - expectedListAfterUpdate helpers.AllowList - } - - tests := []test{ - { - name: "exact match - and filter", - filter: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorAnd, - Operands: []filters.Clause{ - { - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: "foo", - Property: schema.PropertyName(propName), - }, - Value: &filters.Value{ - Value: "list_a", - Type: schema.DataTypeText, - }, - }, - { - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: "foo", - Property: schema.PropertyName(propName), - }, - Value: &filters.Value{ - Value: "list_b", - Type: schema.DataTypeText, - }, - }, - }, - }, - }, - expectedListBeforeUpdate: helpers.NewAllowList(1), - expectedListAfterUpdate: helpers.NewAllowList(1, 3), - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - t.Run("before update", func(t *testing.T) { - res, err := searcher.DocIDs(context.Background(), test.filter, - additional.Properties{}, className) - assert.Nil(t, err) - assert.Equal(t, test.expectedListBeforeUpdate.Slice(), res.Slice()) - }) - - t.Run("update", func(t *testing.T) { - value := []byte("list_a") - idsMapValues := idsToBinaryMapValues([]uint64{3}) - for _, pair := range idsMapValues { - require.Nil(t, bWithFrequency.MapSet([]byte(value), pair)) - } - - value = []byte("list_b") - idsMapValues = idsToBinaryMapValues([]uint64{3}) - for _, pair := range idsMapValues { - require.Nil(t, bWithFrequency.MapSet([]byte(value), pair)) - } - }) - - t.Run("after update", func(t *testing.T) { - res, err := searcher.DocIDs(context.Background(), test.filter, - additional.Properties{}, className) - assert.Nil(t, err) - assert.Equal(t, test.expectedListAfterUpdate.Slice(), res.Slice()) - }) - - t.Run("restore inverted index, so we can run test suite again", - func(t *testing.T) { - idsMapValues := idsToBinaryMapValues([]uint64{3}) - require.Nil(t, bWithFrequency.MapDeleteKey([]byte("list_a"), - idsMapValues[0].Key)) - require.Nil(t, bWithFrequency.MapDeleteKey([]byte("list_b"), - idsMapValues[0].Key)) - }) - }) - } -} - -func idsToBinaryList(ids []uint64) [][]byte { - out := make([][]byte, len(ids)) - for i, id := range ids { - out[i] = make([]byte, 8) - binary.LittleEndian.PutUint64(out[i], id) - } - - return out -} - -func idsToBinaryMapValues(ids []uint64) []lsmkv.MapPair { - out := make([]lsmkv.MapPair, len(ids)) - for i, id := range ids { - out[i] = lsmkv.MapPair{ - Key: make([]byte, 8), - Value: make([]byte, 8), - } - binary.BigEndian.PutUint64(out[i].Key, id) - // leave frequency empty for now - } - - return out -} - -func createSchema() schema.Schema { - vFalse := false - vTrue := true - - return schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{ - { - Class: className, - Properties: []*models.Property{ - { - Name: "inverted-with-frequency", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - IndexFilterable: &vFalse, - IndexSearchable: &vTrue, - }, - { - Name: "inverted-without-frequency", - DataType: schema.DataTypeInt.PropString(), - IndexFilterable: &vTrue, - IndexSearchable: &vFalse, - }, - }, - }, - }, - }, - } -} diff --git a/adapters/repos/db/inverted/like_regexp.go b/adapters/repos/db/inverted/like_regexp.go deleted file mode 100644 index 79477542cda5094193d794c1fd72d13938897925..0000000000000000000000000000000000000000 --- a/adapters/repos/db/inverted/like_regexp.go +++ /dev/null @@ -1,61 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package inverted - -import ( - "bytes" - "regexp" - - "github.com/pkg/errors" -) - -type likeRegexp struct { - optimizable bool - min []byte - regexp *regexp.Regexp -} - -func parseLikeRegexp(in []byte) (*likeRegexp, error) { - r, err := regexp.Compile(transformLikeStringToRegexp(in)) - if err != nil { - return nil, errors.Wrap(err, "compile regex from 'like' string") - } - - min, ok := optimizable(in) - return &likeRegexp{ - regexp: r, - min: min, - optimizable: ok, - }, nil -} - -func transformLikeStringToRegexp(in []byte) string { - in = bytes.ReplaceAll(in, []byte("?"), []byte(".")) - in = bytes.ReplaceAll(in, []byte("*"), []byte(".*")) - return "^" + string(in) + "$" -} - -func optimizable(in []byte) ([]byte, bool) { - maxCharsWithoutWildcard := 0 - for _, char := range in { - if isWildcardCharacter(char) { - break - } - maxCharsWithoutWildcard++ - } - - return in[:maxCharsWithoutWildcard], maxCharsWithoutWildcard > 0 -} - -func isWildcardCharacter(in byte) bool { - return in == '?' || in == '*' -} diff --git a/adapters/repos/db/inverted/like_regexp_test.go b/adapters/repos/db/inverted/like_regexp_test.go deleted file mode 100644 index 3319c524a16755c2002b05e7e6df9f5e4306a2e2..0000000000000000000000000000000000000000 --- a/adapters/repos/db/inverted/like_regexp_test.go +++ /dev/null @@ -1,127 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package inverted - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestLikeRegexp(t *testing.T) { - type test struct { - input []byte - subject []byte - shouldMatch bool - expectedError error - } - - run := func(t *testing.T, tests []test) { - for _, test := range tests { - t.Run(fmt.Sprintf("for input %q and subject %q", string(test.input), - string(test.subject)), func(t *testing.T) { - res, err := parseLikeRegexp(test.input) - if test.expectedError != nil { - assert.Equal(t, test.expectedError, err) - return - } - - require.Nil(t, err) - assert.Equal(t, test.shouldMatch, res.regexp.Match(test.subject)) - }) - } - } - - t.Run("without a wildcard", func(t *testing.T) { - input := []byte("car") - tests := []test{ - {input: input, subject: []byte("car"), shouldMatch: true}, - {input: input, subject: []byte("care"), shouldMatch: false}, - {input: input, subject: []byte("supercar"), shouldMatch: false}, - } - - run(t, tests) - }) - - t.Run("with a single-character wildcard", func(t *testing.T) { - input := []byte("car?") - tests := []test{ - {input: input, subject: []byte("car"), shouldMatch: false}, - {input: input, subject: []byte("cap"), shouldMatch: false}, - {input: input, subject: []byte("care"), shouldMatch: true}, - {input: input, subject: []byte("supercar"), shouldMatch: false}, - {input: input, subject: []byte("carer"), shouldMatch: false}, - } - - run(t, tests) - }) - - t.Run("with a multi-character wildcard", func(t *testing.T) { - input := []byte("car*") - tests := []test{ - {input: input, subject: []byte("car"), shouldMatch: true}, - {input: input, subject: []byte("cap"), shouldMatch: false}, - {input: input, subject: []byte("care"), shouldMatch: true}, - {input: input, subject: []byte("supercar"), shouldMatch: false}, - {input: input, subject: []byte("carer"), shouldMatch: true}, - } - - run(t, tests) - }) - - t.Run("with several wildcards", func(t *testing.T) { - input := []byte("*c?r*") - tests := []test{ - {input: input, subject: []byte("car"), shouldMatch: true}, - {input: input, subject: []byte("cap"), shouldMatch: false}, - {input: input, subject: []byte("care"), shouldMatch: true}, - {input: input, subject: []byte("supercar"), shouldMatch: true}, - {input: input, subject: []byte("carer"), shouldMatch: true}, - } - - run(t, tests) - }) -} - -func TestLikeRegexp_ForOptimizability(t *testing.T) { - type test struct { - input []byte - shouldBeOptimizable bool - expectedMin []byte - } - - run := func(t *testing.T, tests []test) { - for _, test := range tests { - t.Run(fmt.Sprintf("for input %q", string(test.input)), func(t *testing.T) { - res, err := parseLikeRegexp(test.input) - require.Nil(t, err) - assert.Equal(t, test.shouldBeOptimizable, res.optimizable) - assert.Equal(t, test.expectedMin, res.min) - }) - } - } - - tests := []test{ - {input: []byte("car"), shouldBeOptimizable: true, expectedMin: []byte("car")}, - {input: []byte("car*"), shouldBeOptimizable: true, expectedMin: []byte("car")}, - {input: []byte("car?"), shouldBeOptimizable: true, expectedMin: []byte("car")}, - {input: []byte("c?r"), shouldBeOptimizable: true, expectedMin: []byte("c")}, - {input: []byte("car*taker"), shouldBeOptimizable: true, expectedMin: []byte("car")}, - {input: []byte("car?tak*?*er"), shouldBeOptimizable: true, expectedMin: []byte("car")}, - {input: []byte("?car"), shouldBeOptimizable: false, expectedMin: []byte{}}, - {input: []byte("*car"), shouldBeOptimizable: false, expectedMin: []byte{}}, - } - - run(t, tests) -} diff --git a/adapters/repos/db/inverted/merge_benchmarks_test.go b/adapters/repos/db/inverted/merge_benchmarks_test.go deleted file mode 100644 index cdb2fa913bc9fd8ce15312952c9ab3ff0f2d69c7..0000000000000000000000000000000000000000 --- a/adapters/repos/db/inverted/merge_benchmarks_test.go +++ /dev/null @@ -1,223 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package inverted - -import ( - "math" - "math/rand" - "sort" - "testing" -) - -// func BenchmarkAnd10k1m_Old(b *testing.B) { -// b.StopTimer() - -// list1 := propValuePair{ -// docIDs: docPointers{ -// docIDs: randomIDs(1e4), -// checksum: []byte{0x01}, -// }, -// operator: filters.OperatorEqual, -// } - -// list2 := propValuePair{ -// docIDs: docPointers{ -// docIDs: randomIDs(1e6), -// checksum: []byte{0x02}, -// }, -// operator: filters.OperatorEqual, -// } - -// b.StartTimer() -// for i := 0; i < b.N; i++ { -// mergeAnd([]*propValuePair{&list1, &list2}, false) -// } -// } - -// func BenchmarkAnd10k1m_Optimized(b *testing.B) { -// b.StopTimer() - -// list1 := propValuePair{ -// docIDs: docPointers{ -// docIDs: randomIDs(1e4), -// checksum: []byte{0x01}, -// }, -// operator: filters.OperatorEqual, -// } - -// list2 := propValuePair{ -// docIDs: docPointers{ -// docIDs: randomIDs(1e6), -// checksum: []byte{0x02}, -// }, -// operator: filters.OperatorEqual, -// } - -// b.StartTimer() -// for i := 0; i < b.N; i++ { -// mergeAndOptimized([]*propValuePair{&list1, &list2}, false) -// } -// } - -// func BenchmarkMultipleListsOf20k_Old(b *testing.B) { -// b.StopTimer() - -// lists := make([]*propValuePair, 10) -// for i := range lists { -// lists[i] = &propValuePair{ -// docIDs: docPointers{ -// docIDs: randomIDs(2e4), -// checksum: []byte{uint8(i)}, -// }, -// operator: filters.OperatorEqual, -// } -// } - -// b.StartTimer() -// for i := 0; i < b.N; i++ { -// mergeAnd(lists, false) -// } -// } - -// func BenchmarkMultipleListsOf20k_Optimized(b *testing.B) { -// b.StopTimer() - -// lists := make([]*propValuePair, 10) -// for i := range lists { -// lists[i] = &propValuePair{ -// docIDs: docPointers{ -// docIDs: randomIDs(2e4), -// checksum: []byte{uint8(i)}, -// }, -// operator: filters.OperatorEqual, -// } -// } - -// b.StartTimer() -// for i := 0; i < b.N; i++ { -// mergeAndOptimized(lists, false) -// } -// } - -func BenchmarkSort10k(b *testing.B) { - for i := 0; i < b.N; i++ { - b.StopTimer() - list := randomIDs(1e4) - b.StartTimer() - - sort.Slice(list, func(a, b int) bool { - return list[a] < list[b] - }) - } -} - -func BenchmarkUnsortedLinearSearch(b *testing.B) { - searchTargets := randomIDs(1e5) - - for i := 0; i < b.N; i++ { - b.StopTimer() - list := randomIDs(1e5) - b.StartTimer() - - for i := range searchTargets { - linearSearchUnsorted(list, searchTargets[i]) - } - } -} - -func BenchmarkSortedBinarySearch(b *testing.B) { - searchTargets := randomIDs(1e6) - - for i := 0; i < b.N; i++ { - b.StopTimer() - list := randomIDs(1e4) - b.StartTimer() - - sort.Slice(list, func(a, b int) bool { - return list[a] < list[b] - }) - - for i := range searchTargets { - binarySearch(list, searchTargets[i]) - } - } -} - -func BenchmarkHashmap(b *testing.B) { - searchTargets := randomIDs(1e6) - - for i := 0; i < b.N; i++ { - b.StopTimer() - list := randomIDs(1e4) - b.StartTimer() - - lookup := make(map[uint64]struct{}, len(list)) - for i := range list { - lookup[list[i]] = struct{}{} - } - - for i := range searchTargets { - _, ok := lookup[searchTargets[i]] - _ = ok - } - } -} - -func randomIDs(count int) []uint64 { - out := make([]uint64, count) - for i := range out { - out[i] = rand.Uint64() - } - - return out -} - -func linearSearchUnsorted(in []uint64, needle uint64) bool { - for i := range in { - if in[i] == needle { - return true - } - } - - return false -} - -// function binary_search(A, n, T) is -// L := 0 -// R := n − 1 -// while L ≤ R do -// m := floor((L + R) / 2) -// if A[m] < T then -// L := m + 1 -// else if A[m] > T then -// R := m − 1 -// else: -// return m -// return unsuccessful - -func binarySearch(in []uint64, needle uint64) bool { - left := 0 - right := len(in) - 1 - - for left <= right { - m := int(math.Floor(float64((left + right)) / float64(2))) - if in[m] < needle { - left = m + 1 - } else if in[m] > needle { - right = m - 1 - } else { - return true - } - } - - return false -} diff --git a/adapters/repos/db/inverted/new_prop_length_tracker.go b/adapters/repos/db/inverted/new_prop_length_tracker.go deleted file mode 100644 index 1eedf264f5f2a0c9ef0812475b118ee630faadc3..0000000000000000000000000000000000000000 --- a/adapters/repos/db/inverted/new_prop_length_tracker.go +++ /dev/null @@ -1,379 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package inverted - -import ( - "encoding/json" - "math" - "os" - "sync" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" -) - -var MAX_BUCKETS = 64 - -type ShardMetaData struct { - BucketedData map[string]map[int]int - SumData map[string]int - CountData map[string]int - ObjectCount int -} - -type JsonShardMetaData struct { - path string - data *ShardMetaData // Only this part is saved in the file - sync.Mutex - UnlimitedBuckets bool - logger logrus.FieldLogger -} - -// This class replaces the old PropertyLengthTracker. It fixes a bug and provides a -// simpler, easier to maintain implementation. The format is future-proofed, new -// data can be added to the file without breaking old versions of Weaviate. -// -// * We need to know the mean length of all properties for BM25 calculations -// * The prop length tracker is an approximate tracker that uses buckets and simply counts the entries in the buckets -// * There is a precise global counter for the sum of all lengths and a precise global counter for the number of entries -// * It only exists for string/text (and their array forms) because these are the only prop types that can be used with BM25 -// * It should probably always exist when indexSearchable is set on a text prop going forward -// -// Property lengths are put into one of 64 buckets. The value of a bucket is given by the formula: -// -// float32(4 * math.Pow(1.25, float64(bucket)-3.5)) -// -// Which as implemented gives bucket values of 0,1,2,3,4,5,6,8,10,13,17,21,26,33,41,52,65,81,101,127,158,198,248,310,387,484,606,757,947,1183,1479,1849,2312,2890,3612,4515,5644,7055,8819,11024,13780,17226,21532,26915,33644,42055,52569,65712,82140,102675,128344,160430,200537,250671,313339,391674,489593,611991,764989,956237,1195296,1494120,1867651,2334564 -// -// These buckets are then recorded to disk. The original implementation was a binary format where all the data was tracked using manual pointer arithmetic. The new version tracks the statistics in a go map, and marshals that into JSON before writing it to disk. There is no measurable difference in speed between these two implementations while importing data, however it appears to slow the queries by about 15% (while improving recall by ~25%). -// -// The new tracker is exactly compatible with the old format to enable migration, which is why there is a -1 bucket. Altering the number of buckets or their values will break compatibility. -// -// Set UnlimitedBuckets to true for precise length tracking -// -// Note that some of the code in this file is forced by the need to be backwards-compatible with the old format. Once we are confident that all users have migrated to the new format, we can remove the old format code and simplify this file. - -// NewJsonShardMetaData creates a new tracker and loads the data from the given path. If the file is in the old format, it will be converted to the new format. -func NewJsonShardMetaData(path string, logger logrus.FieldLogger) (t *JsonShardMetaData, err error) { - // Recover and return empty tracker on panic - defer func() { - if r := recover(); r != nil { - t.logger.Printf("Recovered from panic in NewJsonShardMetaData, original error: %v", r) - t = &JsonShardMetaData{ - data: &ShardMetaData{make(map[string]map[int]int), make(map[string]int), make(map[string]int), 0}, - path: path, - UnlimitedBuckets: false, - } - err = errors.Errorf("Recovered from panic in NewJsonShardMetaData, original error: %v", r) - } - }() - - t = &JsonShardMetaData{ - data: &ShardMetaData{make(map[string]map[int]int), make(map[string]int), make(map[string]int), 0}, - path: path, - UnlimitedBuckets: false, - logger: logger, - } - - // read the file into memory - bytes, err := os.ReadFile(path) - if err != nil { - if os.IsNotExist(err) { // File doesn't exist, probably a new class(or a recount), return empty tracker - logger.Printf("WARNING: prop len tracker file %s does not exist, creating new tracker", path) - t.Flush(false) - return t, nil - } - return nil, errors.Wrap(err, "read property length tracker file:"+path) - } - - if len(bytes) == 0 { - return nil, errors.Errorf("failed sanity check, empty prop len tracker file %s has length 0. Delete file and set environment variable RECOUNT_PROPERTIES_AT_STARTUP to true", path) - } - - // We don't have data file versioning, so we try to parse it as json. If the parse fails, it is probably the old format file, so we call the old format loader and copy everything across. - if err = json.Unmarshal(bytes, &t.data); err != nil { - // It's probably the old format file, load the old format and convert it to the new format - plt, err := NewPropertyLengthTracker(path) - if err != nil { - return nil, errors.Wrap(err, "convert old property length tracker") - } - - propertyNames := plt.PropertyNames() - data := &ShardMetaData{make(map[string]map[int]int), make(map[string]int), make(map[string]int), 0} - // Loop over every page and bucket in the old tracker and add it to the new tracker - for _, name := range propertyNames { - data.BucketedData[name] = make(map[int]int, MAX_BUCKETS) - data.CountData[name] = 0 - data.SumData[name] = 0 - for i := 0; i <= MAX_BUCKETS; i++ { - fromBucket := i - if i == MAX_BUCKETS { - fromBucket = -1 - } - count, err := plt.BucketCount(name, uint16(fromBucket)) - if err != nil { - return nil, errors.Wrap(err, "convert old property length tracker") - } - data.BucketedData[name][fromBucket] = int(count) - value := float32(0) - if fromBucket == -1 { - value = 0 - } else { - value = plt.valueFromBucket(uint16(fromBucket)) - } - - data.SumData[name] = data.SumData[name] + int(value)*int(count) - data.CountData[name] = data.CountData[name] + int(count) - } - } - t.data = data - t.Flush(true) - plt.Close() - plt.Drop() - t.Flush(false) - } - t.path = path - - // Make really sure we aren't going to crash on a nil pointer - if t.data == nil { - return nil, errors.Errorf("failed sanity check, prop len tracker file %s has nil data. Delete file and set environment variable RECOUNT_PROPERTIES_AT_STARTUP to true", path) - } - return t, nil -} - -func (t *JsonShardMetaData) Clear() { - if t == nil { - return - } - t.Lock() - defer t.Unlock() - - t.data = &ShardMetaData{make(map[string]map[int]int), make(map[string]int), make(map[string]int), 0} -} - -// Path to the file on disk -func (t *JsonShardMetaData) FileName() string { - if t == nil { - return "" - } - return t.path -} - -func (t *JsonShardMetaData) TrackObjects(delta int) error { - if t == nil { - return nil - } - t.Lock() - defer t.Unlock() - - t.data.ObjectCount = t.data.ObjectCount + delta - return nil -} - -// Adds a new value to the tracker -func (t *JsonShardMetaData) TrackProperty(propName string, value float32) error { - if t == nil { - return nil - } - t.Lock() - defer t.Unlock() - - // Remove this check once we are confident that all users have migrated to the new format - if t.data == nil { - t.logger.Print("WARNING: t.data is nil in TrackProperty, initializing to empty tracker") - t.data = &ShardMetaData{make(map[string]map[int]int), make(map[string]int), make(map[string]int), 0} - } - t.data.SumData[propName] = t.data.SumData[propName] + int(value) - t.data.CountData[propName] = t.data.CountData[propName] + 1 - - bucketId := t.bucketFromValue(value) - if _, ok := t.data.BucketedData[propName]; ok { - t.data.BucketedData[propName][int(bucketId)] = t.data.BucketedData[propName][int(bucketId)] + 1 - } else { - - t.data.BucketedData[propName] = make(map[int]int, 64+1) - t.data.BucketedData[propName][int(bucketId)] = 1 - } - - return nil -} - -// Removes a value from the tracker -func (t *JsonShardMetaData) UnTrackProperty(propName string, value float32) error { - if t == nil { - return nil - } - t.Lock() - defer t.Unlock() - - // Remove this check once we are confident that all users have migrated to the new format - if t.data == nil { - t.logger.Print("WARNING: t.data is nil in TrackProperty, initializing to empty tracker") - t.data = &ShardMetaData{make(map[string]map[int]int), make(map[string]int), make(map[string]int), 0} - } - t.data.SumData[propName] = t.data.SumData[propName] - int(value) - t.data.CountData[propName] = t.data.CountData[propName] - 1 - - bucketId := t.bucketFromValue(value) - if _, ok := t.data.BucketedData[propName]; ok { - t.data.BucketedData[propName][int(bucketId)] = t.data.BucketedData[propName][int(bucketId)] - 1 - } else { - return errors.New("property not found") - } - - return nil -} - -// Returns the bucket that the given value belongs to -func (t *JsonShardMetaData) bucketFromValue(value float32) int { - if t == nil { - return 0 - } - if t.UnlimitedBuckets { - return int(value) - } - if value <= 5.00 { - return int(value) - 1 - } - - bucket := int(math.Log(float64(value)/4.0)/math.Log(1.25) + 4) - if bucket > MAX_BUCKETS-1 { - return MAX_BUCKETS - } - return int(bucket) -} - -// Returns the average length of the given property -func (t *JsonShardMetaData) PropertyMean(propName string) (float32, error) { - if t == nil { - return 0, nil - } - t.Lock() - defer t.Unlock() - - sum, ok := t.data.SumData[propName] - if !ok { - return 0, nil - } - count, ok := t.data.CountData[propName] - if !ok { - return 0, nil - } - - return float32(sum) / float32(count), nil -} - -// returns totalPropertyLength, totalCount, average propertyLength = sum / totalCount, total propertylength, totalCount, error -func (t *JsonShardMetaData) PropertyTally(propName string) (int, int, float64, error) { - if t == nil { - return 0, 0, 0, nil - } - t.Lock() - defer t.Unlock() - sum, ok := t.data.SumData[propName] - if !ok { - return 0, 0, 0, nil // Required to match the old prop tracker (for now) - } - count, ok := t.data.CountData[propName] - if !ok { - return 0, 0, 0, nil // Required to match the old prop tracker (for now) - } - return sum, count, float64(sum) / float64(count), nil -} - -// Returns the number of documents stored in the shard -func (t *JsonShardMetaData) ObjectTally() int { - if t == nil { - return 0 - } - t.Lock() - defer t.Unlock() - - return t.data.ObjectCount -} - -// Writes the current state of the tracker to disk. (flushBackup = true) will only write the backup file -func (t *JsonShardMetaData) Flush(flushBackup bool) error { - if t == nil { - return nil - } - if !flushBackup { // Write the backup file first - t.Flush(true) - } - - t.Lock() - defer t.Unlock() - - bytes, err := json.Marshal(t.data) - if err != nil { - return err - } - - filename := t.path - if flushBackup { - filename = t.path + ".bak" - } - - // Do a write+rename to avoid corrupting the file if we crash while writing - tempfile := filename + ".tmp" - - err = os.WriteFile(tempfile, bytes, 0o666) - if err != nil { - return err - } - - err = os.Rename(tempfile, filename) - if err != nil { - return err - } - - return nil -} - -// Closes the tracker and removes the backup file -func (t *JsonShardMetaData) Close() error { - if t == nil { - return nil - } - if err := t.Flush(false); err != nil { - return errors.Wrap(err, "flush before closing") - } - - t.Lock() - defer t.Unlock() - - t.data.BucketedData = nil - - return nil -} - -// Drop removes the tracker from disk -func (t *JsonShardMetaData) Drop() error { - if t == nil { - return nil - } - t.Close() - - t.Lock() - defer t.Unlock() - - t.data.BucketedData = nil - - if err := os.Remove(t.path); err != nil { - return errors.Wrap(err, "remove prop length tracker state from disk:"+t.path) - } - if err := os.Remove(t.path + ".bak"); err != nil { - return errors.Wrap(err, "remove prop length tracker state from disk:"+t.path+".bak") - } - - return nil -} diff --git a/adapters/repos/db/inverted/objects.go b/adapters/repos/db/inverted/objects.go deleted file mode 100644 index eab7ce7f214e0f5a5b1b3e1be0b6f73f568fff65..0000000000000000000000000000000000000000 --- a/adapters/repos/db/inverted/objects.go +++ /dev/null @@ -1,622 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package inverted - -import ( - "encoding/json" - "fmt" - "time" - "unicode/utf8" - - "github.com/go-openapi/strfmt" - "github.com/google/uuid" - "github.com/pkg/errors" - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/usecases/objects/validation" -) - -func (a *Analyzer) Object(input map[string]any, props []*models.Property, - uuid strfmt.UUID, -) ([]Property, error) { - propsMap := map[string]*models.Property{} - for _, prop := range props { - propsMap[prop.Name] = prop - } - - properties, err := a.analyzeProps(propsMap, input) - if err != nil { - return nil, fmt.Errorf("analyze props: %w", err) - } - - idProp, err := a.analyzeIDProp(uuid) - if err != nil { - return nil, fmt.Errorf("analyze uuid prop: %w", err) - } - properties = append(properties, *idProp) - - tsProps, err := a.analyzeTimestampProps(input) - if err != nil { - return nil, fmt.Errorf("analyze timestamp props: %w", err) - } - // tsProps will be nil here if weaviate is - // not setup to index by timestamps - if tsProps != nil { - properties = append(properties, tsProps...) - } - - return properties, nil -} - -func (a *Analyzer) analyzeProps(propsMap map[string]*models.Property, - input map[string]any, -) ([]Property, error) { - var out []Property - for key, prop := range propsMap { - if len(prop.DataType) < 1 { - return nil, fmt.Errorf("prop %q has no datatype", prop.Name) - } - - if !HasInvertedIndex(prop) { - continue - } - - if schema.IsBlobDataType(prop.DataType) { - continue - } - - if schema.IsRefDataType(prop.DataType) { - if err := a.extendPropertiesWithReference(&out, prop, input, key); err != nil { - return nil, err - } - } else if schema.IsArrayDataType(prop.DataType) { - if err := a.extendPropertiesWithArrayType(&out, prop, input, key); err != nil { - return nil, err - } - } else { - if err := a.extendPropertiesWithPrimitive(&out, prop, input, key); err != nil { - return nil, err - } - } - - } - return out, nil -} - -func (a *Analyzer) analyzeIDProp(id strfmt.UUID) (*Property, error) { - value, err := id.MarshalText() - if err != nil { - return nil, fmt.Errorf("marshal id prop: %w", err) - } - return &Property{ - Name: filters.InternalPropID, - Items: []Countable{ - { - Data: value, - }, - }, - HasFilterableIndex: HasFilterableIndexIdProp, - HasSearchableIndex: HasSearchableIndexIdProp, - }, nil -} - -func (a *Analyzer) analyzeTimestampProps(input map[string]any) ([]Property, error) { - createTime, createTimeOK := input[filters.InternalPropCreationTimeUnix] - updateTime, updateTimeOK := input[filters.InternalPropLastUpdateTimeUnix] - - var props []Property - if createTimeOK { - b, err := json.Marshal(createTime) - if err != nil { - return nil, fmt.Errorf("analyze create timestamp prop: %w", err) - } - props = append(props, Property{ - Name: filters.InternalPropCreationTimeUnix, - Items: []Countable{{Data: b}}, - HasFilterableIndex: HasFilterableIndexTimestampProp, - HasSearchableIndex: HasSearchableIndexTimestampProp, - }) - } - - if updateTimeOK { - b, err := json.Marshal(updateTime) - if err != nil { - return nil, fmt.Errorf("analyze update timestamp prop: %w", err) - } - props = append(props, Property{ - Name: filters.InternalPropLastUpdateTimeUnix, - Items: []Countable{{Data: b}}, - HasFilterableIndex: HasFilterableIndexTimestampProp, - HasSearchableIndex: HasSearchableIndexTimestampProp, - }) - } - - return props, nil -} - -func (a *Analyzer) extendPropertiesWithArrayType(properties *[]Property, - prop *models.Property, input map[string]any, propName string, -) error { - value, ok := input[propName] - if !ok { - // skip any primitive prop that's not set - return nil - } - - var err error - value, err = typedSliceToUntyped(value) - if err != nil { - return fmt.Errorf("extend properties with array type: %w", err) - } - - values, ok := value.([]any) - if !ok { - // skip any primitive prop that's not set - return errors.New("analyze array prop: expected array prop") - } - - property, err := a.analyzeArrayProp(prop, values) - if err != nil { - return fmt.Errorf("analyze array prop: %w", err) - } - if property == nil { - return nil - } - - *properties = append(*properties, *property) - return nil -} - -// extendPropertiesWithPrimitive mutates the passed in properties, by extending -// it with an additional property - if applicable -func (a *Analyzer) extendPropertiesWithPrimitive(properties *[]Property, - prop *models.Property, input map[string]any, propName string, -) error { - var property *Property - var err error - - value, ok := input[propName] - if !ok { - // skip any primitive prop that's not set - return nil - } - property, err = a.analyzePrimitiveProp(prop, value) - if err != nil { - return fmt.Errorf("analyze primitive prop: %w", err) - } - if property == nil { - return nil - } - - *properties = append(*properties, *property) - return nil -} - -func (a *Analyzer) analyzeArrayProp(prop *models.Property, values []any) (*Property, error) { - var items []Countable - hasFilterableIndex := HasFilterableIndex(prop) - hasSearchableIndex := HasSearchableIndex(prop) - - switch dt := schema.DataType(prop.DataType[0]); dt { - case schema.DataTypeTextArray: - hasFilterableIndex = hasFilterableIndex && !a.isFallbackToSearchable() - in, err := stringsFromValues(prop, values) - if err != nil { - return nil, err - } - items = a.TextArray(prop.Tokenization, in) - case schema.DataTypeIntArray: - in := make([]int64, len(values)) - for i, value := range values { - if asJsonNumber, ok := value.(json.Number); ok { - var err error - value, err = asJsonNumber.Float64() - if err != nil { - return nil, err - } - } - - if asFloat, ok := value.(float64); ok { - // unmarshaling from json into a dynamic schema will assume every number - // is a float64 - value = int64(asFloat) - } - - asInt, ok := value.(int64) - if !ok { - return nil, fmt.Errorf("expected property %s to be of type int64, but got %T", prop.Name, value) - } - in[i] = asInt - } - - var err error - items, err = a.IntArray(in) - if err != nil { - return nil, fmt.Errorf("analyze property %s: %w", prop.Name, err) - } - case schema.DataTypeNumberArray: - in := make([]float64, len(values)) - for i, value := range values { - if asJsonNumber, ok := value.(json.Number); ok { - var err error - value, err = asJsonNumber.Float64() - if err != nil { - return nil, err - } - } - - asFloat, ok := value.(float64) - if !ok { - return nil, fmt.Errorf("expected property %s to be of type float64, but got %T", prop.Name, value) - } - in[i] = asFloat - } - - var err error - items, err = a.FloatArray(in) // convert to int before analyzing - if err != nil { - return nil, fmt.Errorf("analyze property %s: %w", prop.Name, err) - } - case schema.DataTypeBooleanArray: - in := make([]bool, len(values)) - for i, value := range values { - asBool, ok := value.(bool) - if !ok { - return nil, fmt.Errorf("expected property %s to be of type bool, but got %T", prop.Name, value) - } - in[i] = asBool - } - - var err error - items, err = a.BoolArray(in) // convert to int before analyzing - if err != nil { - return nil, fmt.Errorf("analyze property %s: %w", prop.Name, err) - } - case schema.DataTypeDateArray: - in := make([]int64, len(values)) - for i, value := range values { - // dates can be either a date-string or directly a time object. Try to parse both - if asTime, okTime := value.(time.Time); okTime { - in[i] = asTime.UnixNano() - } else if asString, okString := value.(string); okString { - parsedTime, err := time.Parse(time.RFC3339Nano, asString) - if err != nil { - return nil, fmt.Errorf("parse time: %w", err) - } - in[i] = parsedTime.UnixNano() - } else { - return nil, fmt.Errorf("expected property %s to be a time-string or time object, but got %T", prop.Name, value) - } - } - - var err error - items, err = a.IntArray(in) - if err != nil { - return nil, fmt.Errorf("analyze property %s: %w", prop.Name, err) - } - case schema.DataTypeUUIDArray: - parsed, err := validation.ParseUUIDArray(values) - if err != nil { - return nil, fmt.Errorf("parse uuid array: %w", err) - } - - items, err = a.UUIDArray(parsed) - if err != nil { - return nil, fmt.Errorf("analyze property %s: %w", prop.Name, err) - } - - default: - // ignore unsupported prop type - return nil, nil - } - - return &Property{ - Name: prop.Name, - Items: items, - Length: len(values), - HasFilterableIndex: hasFilterableIndex, - HasSearchableIndex: hasSearchableIndex, - }, nil -} - -func stringsFromValues(prop *models.Property, values []any) ([]string, error) { - in := make([]string, len(values)) - for i, value := range values { - asString, ok := value.(string) - if !ok { - return nil, fmt.Errorf("expected property %s to be of type string, but got %T", prop.Name, value) - } - in[i] = asString - } - return in, nil -} - -func (a *Analyzer) analyzePrimitiveProp(prop *models.Property, value any) (*Property, error) { - var items []Countable - propertyLength := -1 // will be overwritten for string/text, signals not to add the other types. - hasFilterableIndex := HasFilterableIndex(prop) - hasSearchableIndex := HasSearchableIndex(prop) - - switch dt := schema.DataType(prop.DataType[0]); dt { - case schema.DataTypeText: - hasFilterableIndex = hasFilterableIndex && !a.isFallbackToSearchable() - asString, ok := value.(string) - if !ok { - return nil, fmt.Errorf("expected property %s to be of type string, but got %T", prop.Name, value) - } - items = a.Text(prop.Tokenization, asString) - propertyLength = utf8.RuneCountInString(asString) - case schema.DataTypeInt: - if asFloat, ok := value.(float64); ok { - // unmarshaling from json into a dynamic schema will assume every number - // is a float64 - value = int64(asFloat) - } - - if asInt, ok := value.(int); ok { - // when merging an existing object we may retrieve an untyped int - value = int64(asInt) - } - - asInt, ok := value.(int64) - if !ok { - return nil, fmt.Errorf("expected property %s to be of type int64, but got %T", prop.Name, value) - } - - var err error - items, err = a.Int(asInt) - if err != nil { - return nil, fmt.Errorf("analyze property %s: %w", prop.Name, err) - } - case schema.DataTypeNumber: - asFloat, ok := value.(float64) - if !ok { - return nil, fmt.Errorf("expected property %s to be of type float64, but got %T", prop.Name, value) - } - - var err error - items, err = a.Float(asFloat) // convert to int before analyzing - if err != nil { - return nil, fmt.Errorf("analyze property %s: %w", prop.Name, err) - } - case schema.DataTypeBoolean: - asBool, ok := value.(bool) - if !ok { - return nil, fmt.Errorf("expected property %s to be of type bool, but got %T", prop.Name, value) - } - - var err error - items, err = a.Bool(asBool) // convert to int before analyzing - if err != nil { - return nil, fmt.Errorf("analyze property %s: %w", prop.Name, err) - } - case schema.DataTypeDate: - var err error - if asString, ok := value.(string); ok { - // for example when patching the date may have been loaded as a string - value, err = time.Parse(time.RFC3339Nano, asString) - if err != nil { - return nil, fmt.Errorf("parse stringified timestamp: %w", err) - } - } - asTime, ok := value.(time.Time) - if !ok { - return nil, fmt.Errorf("expected property %s to be time.Time, but got %T", prop.Name, value) - } - - items, err = a.Int(asTime.UnixNano()) - if err != nil { - return nil, fmt.Errorf("analyze property %s: %w", prop.Name, err) - } - case schema.DataTypeUUID: - var err error - - if asString, ok := value.(string); ok { - // for example when patching the uuid may have been loaded as a string - value, err = uuid.Parse(asString) - if err != nil { - return nil, fmt.Errorf("parse stringified uuid: %w", err) - } - } - - asUUID, ok := value.(uuid.UUID) - if !ok { - return nil, fmt.Errorf("expected property %s to be uuid.UUID, but got %T", prop.Name, value) - } - - items, err = a.UUID(asUUID) - if err != nil { - return nil, fmt.Errorf("analyze property %s: %w", prop.Name, err) - } - default: - // ignore unsupported prop type - return nil, nil - } - - return &Property{ - Name: prop.Name, - Items: items, - Length: propertyLength, - HasFilterableIndex: hasFilterableIndex, - HasSearchableIndex: hasSearchableIndex, - }, nil -} - -// extendPropertiesWithReference extends the specified properties arrays with -// either 1 or 2 entries: If the ref is not set, only the ref-count property -// will be added. If the ref is set the ref-prop itself will also be added and -// contain all references as values -func (a *Analyzer) extendPropertiesWithReference(properties *[]Property, - prop *models.Property, input map[string]any, propName string, -) error { - value, ok := input[propName] - if !ok { - // explicitly set zero-value, so we can index for "ref not set" - value = make(models.MultipleRef, 0) - } - - var asRefs models.MultipleRef - asRefs, ok = value.(models.MultipleRef) - if !ok { - // due to the fix introduced in https://github.com/weaviate/weaviate/pull/2320, - // MultipleRef's can appear as empty []any when no actual refs are provided for - // an object's reference property. - // - // if we encounter []any, assume it indicates an empty ref prop, and skip it. - _, ok := value.([]any) - if !ok { - return fmt.Errorf("expected property %q to be of type models.MutlipleRef,"+ - " but got %T", prop.Name, value) - } - return nil - } - - property, err := a.analyzeRefPropCount(prop, asRefs) - if err != nil { - return fmt.Errorf("ref count: %w", err) - } - - *properties = append(*properties, *property) - - if len(asRefs) == 0 { - return nil - } - - property, err = a.analyzeRefProp(prop, asRefs) - if err != nil { - return fmt.Errorf("refs: %w", err) - } - - *properties = append(*properties, *property) - return nil -} - -func (a *Analyzer) analyzeRefPropCount(prop *models.Property, - value models.MultipleRef, -) (*Property, error) { - items, err := a.RefCount(value) - if err != nil { - return nil, fmt.Errorf("analyze ref-property %q: %w", prop.Name, err) - } - - return &Property{ - Name: helpers.MetaCountProp(prop.Name), - Items: items, - Length: len(value), - HasFilterableIndex: HasFilterableIndex(prop), - HasSearchableIndex: HasSearchableIndex(prop), - }, nil -} - -func (a *Analyzer) analyzeRefProp(prop *models.Property, - value models.MultipleRef, -) (*Property, error) { - items, err := a.Ref(value) - if err != nil { - return nil, fmt.Errorf("analyze ref-property %q: %w", prop.Name, err) - } - - return &Property{ - Name: prop.Name, - Items: items, - HasFilterableIndex: HasFilterableIndex(prop), - HasSearchableIndex: HasSearchableIndex(prop), - }, nil -} - -func typedSliceToUntyped(in any) ([]any, error) { - switch typed := in.(type) { - case []any: - // nothing to do - return typed, nil - case []string: - return convertToUntyped[string](typed), nil - case []int: - return convertToUntyped[int](typed), nil - case []time.Time: - return convertToUntyped[time.Time](typed), nil - case []bool: - return convertToUntyped[bool](typed), nil - case []float64: - return convertToUntyped[float64](typed), nil - case []uuid.UUID: - return convertToUntyped[uuid.UUID](typed), nil - default: - return nil, errors.Errorf("unsupported type %T", in) - } -} - -func convertToUntyped[T comparable](in []T) []any { - out := make([]any, len(in)) - for i := range out { - out[i] = in[i] - } - return out -} - -// Indicates whether property should be indexed -// Index holds document ids with property of/containing particular value -// and number of its occurrences in that property -// (index created using bucket of StrategyMapCollection) -func HasSearchableIndex(prop *models.Property) bool { - switch dt, _ := schema.AsPrimitive(prop.DataType); dt { - case schema.DataTypeText, schema.DataTypeTextArray: - // by default property has searchable index only for text/text[] props - if prop.IndexSearchable == nil { - return true - } - return *prop.IndexSearchable - default: - return false - } -} - -// Indicates whether property should be indexed -// Index holds document ids with property of/containing particular value -// (index created using bucket of StrategyRoaringSet) -func HasFilterableIndex(prop *models.Property) bool { - // by default property has filterable index - if prop.IndexFilterable == nil { - return true - } - return *prop.IndexFilterable -} - -func HasInvertedIndex(prop *models.Property) bool { - return HasFilterableIndex(prop) || HasSearchableIndex(prop) -} - -const ( - // always - HasFilterableIndexIdProp = true - HasSearchableIndexIdProp = false - - // only if index.invertedIndexConfig.IndexTimestamps set - HasFilterableIndexTimestampProp = true - HasSearchableIndexTimestampProp = false - - // only if property.indexFilterable or property.indexSearchable set - HasFilterableIndexMetaCount = true - HasSearchableIndexMetaCount = false - - // only if index.invertedIndexConfig.IndexNullState set - // and either property.indexFilterable or property.indexSearchable set - HasFilterableIndexPropNull = true - HasSearchableIndexPropNull = false - - // only if index.invertedIndexConfig.IndexPropertyLength set - // and either property.indexFilterable or property.indexSearchable set - HasFilterableIndexPropLength = true - HasSearchableIndexPropLength = false -) diff --git a/adapters/repos/db/inverted/objects_test.go b/adapters/repos/db/inverted/objects_test.go deleted file mode 100644 index 8ba0e6aec3f5e0ef1c0bc888d1a75c67df0a276a..0000000000000000000000000000000000000000 --- a/adapters/repos/db/inverted/objects_test.go +++ /dev/null @@ -1,931 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package inverted - -import ( - "fmt" - "reflect" - "testing" - "time" - - "github.com/go-openapi/strfmt" - "github.com/google/uuid" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" -) - -func TestAnalyzeObject(t *testing.T) { - a := NewAnalyzer(nil) - - t.Run("with multiple properties", func(t *testing.T) { - id1 := uuid.New() - id2 := uuid.New() - sch := map[string]interface{}{ - "description": "I am great!", - "email": "john@doe.com", - "about_me": "I like reading sci-fi books", - "profession": "Mechanical Engineer", - "id1": id1, // correctly parsed - "id2": id2.String(), // untyped - "idArray1": []uuid.UUID{id1}, // correctly parsed - "idArray2": []any{id2.String()}, // untyped - } - - uuid := "2609f1bc-7693-48f3-b531-6ddc52cd2501" - props := []*models.Property{ - { - Name: "description", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWord, - }, - { - Name: "email", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "about_me", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationLowercase, - }, - { - Name: "profession", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationField, - }, - { - Name: "id1", - DataType: []string{"uuid"}, - }, - { - Name: "id2", - DataType: []string{"uuid"}, - }, - { - Name: "idArray1", - DataType: []string{"uuid[]"}, - }, - { - Name: "idArray2", - DataType: []string{"uuid[]"}, - }, - } - res, err := a.Object(sch, props, strfmt.UUID(uuid)) - require.Nil(t, err) - - expectedDescription := []Countable{ - { - Data: []byte("i"), - TermFrequency: float32(1), - }, - { - Data: []byte("am"), - TermFrequency: float32(1), - }, - { - Data: []byte("great"), - TermFrequency: float32(1), - }, - } - - expectedEmail := []Countable{ - { - Data: []byte("john@doe.com"), - TermFrequency: float32(1), - }, - } - - expectedAboutMe := []Countable{ - { - Data: []byte("i"), - TermFrequency: float32(1), - }, - { - Data: []byte("like"), - TermFrequency: float32(1), - }, - { - Data: []byte("reading"), - TermFrequency: float32(1), - }, - { - Data: []byte("sci-fi"), - TermFrequency: float32(1), - }, - { - Data: []byte("books"), - TermFrequency: float32(1), - }, - } - - expectedProfession := []Countable{ - { - Data: []byte("Mechanical Engineer"), - TermFrequency: float32(1), - }, - } - - expectedUUID := []Countable{ - { - Data: []byte(uuid), - TermFrequency: 0, - }, - } - - expectedID1 := []Countable{ - { - Data: []byte(id1[:]), - TermFrequency: 0, - }, - } - - expectedID2 := []Countable{ - { - Data: []byte(id2[:]), - TermFrequency: 0, - }, - } - - expectedIDArray1 := []Countable{ - { - Data: []byte(id1[:]), - TermFrequency: 0, - }, - } - - expectedIDArray2 := []Countable{ - { - Data: []byte(id2[:]), - TermFrequency: 0, - }, - } - - require.Len(t, res, 9) - var actualDescription []Countable - var actualEmail []Countable - var actualAboutMe []Countable - var actualProfession []Countable - var actualUUID []Countable - var actualID1 []Countable - var actualID2 []Countable - var actualIDArray1 []Countable - var actualIDArray2 []Countable - - for _, elem := range res { - if elem.Name == "email" { - actualEmail = elem.Items - } - - if elem.Name == "description" { - actualDescription = elem.Items - } - - if elem.Name == "about_me" { - actualAboutMe = elem.Items - } - - if elem.Name == "profession" { - actualProfession = elem.Items - } - - if elem.Name == "_id" { - actualUUID = elem.Items - } - - if elem.Name == "id1" { - actualID1 = elem.Items - } - - if elem.Name == "id2" { - actualID2 = elem.Items - } - - if elem.Name == "idArray1" { - actualIDArray1 = elem.Items - } - - if elem.Name == "idArray2" { - actualIDArray2 = elem.Items - } - } - - assert.ElementsMatch(t, expectedEmail, actualEmail, res) - assert.ElementsMatch(t, expectedDescription, actualDescription, res) - assert.ElementsMatch(t, expectedAboutMe, actualAboutMe, res) - assert.ElementsMatch(t, expectedProfession, actualProfession, res) - assert.ElementsMatch(t, expectedUUID, actualUUID, res) - assert.ElementsMatch(t, expectedID1, actualID1, res) - assert.ElementsMatch(t, expectedID2, actualID2, res) - assert.ElementsMatch(t, expectedIDArray1, actualIDArray1, res) - assert.ElementsMatch(t, expectedIDArray2, actualIDArray2, res) - }) - - t.Run("with array properties", func(t *testing.T) { - sch := map[string]interface{}{ - "descriptions": []interface{}{"I am great!", "I am also great!"}, - "emails": []interface{}{"john@doe.com", "john2@doe.com"}, - "about_me": []interface{}{"I like reading sci-fi books", "I like playing piano"}, - "professions": []interface{}{"Mechanical Engineer", "Marketing Analyst"}, - "integers": []interface{}{int64(1), int64(2), int64(3), int64(4)}, - "numbers": []interface{}{float64(1.1), float64(2.2), float64(3.0), float64(4)}, - } - - uuid := "2609f1bc-7693-48f3-b531-6ddc52cd2501" - props := []*models.Property{ - { - Name: "descriptions", - DataType: schema.DataTypeTextArray.PropString(), - Tokenization: models.PropertyTokenizationWord, - }, - { - Name: "emails", - DataType: schema.DataTypeTextArray.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "about_me", - DataType: schema.DataTypeTextArray.PropString(), - Tokenization: models.PropertyTokenizationLowercase, - }, - { - Name: "professions", - DataType: schema.DataTypeTextArray.PropString(), - Tokenization: models.PropertyTokenizationField, - }, - { - Name: "integers", - DataType: []string{"int[]"}, - }, - { - Name: "numbers", - DataType: []string{"number[]"}, - }, - } - res, err := a.Object(sch, props, strfmt.UUID(uuid)) - require.Nil(t, err) - - expectedDescriptions := []Countable{ - { - Data: []byte("i"), - TermFrequency: float32(2), - }, - { - Data: []byte("am"), - TermFrequency: float32(2), - }, - { - Data: []byte("great"), - TermFrequency: float32(2), - }, - { - Data: []byte("also"), - TermFrequency: float32(1), - }, - } - - expectedEmails := []Countable{ - { - Data: []byte("john@doe.com"), - TermFrequency: float32(1), - }, - { - Data: []byte("john2@doe.com"), - TermFrequency: float32(1), - }, - } - - expectedAboutMe := []Countable{ - { - Data: []byte("i"), - TermFrequency: float32(2), - }, - { - Data: []byte("like"), - TermFrequency: float32(2), - }, - { - Data: []byte("reading"), - TermFrequency: float32(1), - }, - { - Data: []byte("sci-fi"), - TermFrequency: float32(1), - }, - { - Data: []byte("books"), - TermFrequency: float32(1), - }, - { - Data: []byte("playing"), - TermFrequency: float32(1), - }, - { - Data: []byte("piano"), - TermFrequency: float32(1), - }, - } - - expectedProfessions := []Countable{ - { - Data: []byte("Mechanical Engineer"), - TermFrequency: float32(1), - }, - { - Data: []byte("Marketing Analyst"), - TermFrequency: float32(1), - }, - } - - expectedIntegers := []Countable{ - { - Data: mustGetByteIntNumber(1), - }, - { - Data: mustGetByteIntNumber(2), - }, - { - Data: mustGetByteIntNumber(3), - }, - { - Data: mustGetByteIntNumber(4), - }, - } - - expectedNumbers := []Countable{ - { - Data: mustGetByteFloatNumber(1.1), - }, - { - Data: mustGetByteFloatNumber(2.2), - }, - { - Data: mustGetByteFloatNumber(3.0), - }, - { - Data: mustGetByteFloatNumber(4), - }, - } - - expectedUUID := []Countable{ - { - Data: []byte(uuid), - TermFrequency: 0, - }, - } - - assert.Len(t, res, 7) - var actualDescriptions []Countable - var actualEmails []Countable - var actualAboutMe []Countable - var actualProfessions []Countable - var actualIntegers []Countable - var actualNumbers []Countable - var actualUUID []Countable - - for _, elem := range res { - if elem.Name == "emails" { - actualEmails = elem.Items - } - - if elem.Name == "descriptions" { - actualDescriptions = elem.Items - } - - if elem.Name == "about_me" { - actualAboutMe = elem.Items - } - - if elem.Name == "professions" { - actualProfessions = elem.Items - } - - if elem.Name == "integers" { - actualIntegers = elem.Items - } - - if elem.Name == "numbers" { - actualNumbers = elem.Items - } - - if elem.Name == "_id" { - actualUUID = elem.Items - } - } - - assert.ElementsMatch(t, expectedEmails, actualEmails, res) - assert.ElementsMatch(t, expectedDescriptions, actualDescriptions, res) - assert.ElementsMatch(t, expectedAboutMe, actualAboutMe, res) - assert.ElementsMatch(t, expectedProfessions, actualProfessions, res) - assert.ElementsMatch(t, expectedIntegers, actualIntegers, res) - assert.ElementsMatch(t, expectedNumbers, actualNumbers, res) - assert.ElementsMatch(t, expectedUUID, actualUUID, res) - }) - - t.Run("with refProps", func(t *testing.T) { - t.Run("with a single ref set in the object schema", func(t *testing.T) { - beacon := strfmt.URI( - "weaviate://localhost/c563d7fa-4a36-4eff-9f39-af1e1db276c4") - schema := map[string]interface{}{ - "myRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: beacon, - }, - }, - } - - uuid := "2609f1bc-7693-48f3-b531-6ddc52cd2501" - props := []*models.Property{ - { - Name: "myRef", - DataType: []string{"RefClass"}, - }, - } - res, err := a.Object(schema, props, strfmt.UUID(uuid)) - require.Nil(t, err) - - expectedRefCount := []Countable{ - {Data: []uint8{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01}}, - } - - expectedRef := []Countable{ - {Data: []byte(beacon)}, - } - - expectedUUID := []Countable{ - { - Data: []byte(uuid), - TermFrequency: 0, - }, - } - - require.Len(t, res, 3) - var actualRefCount []Countable - var actualUUID []Countable - var actualRef []Countable - - for _, elem := range res { - switch elem.Name { - case helpers.MetaCountProp("myRef"): - actualRefCount = elem.Items - case "_id": - actualUUID = elem.Items - case "myRef": - actualRef = elem.Items - } - } - - assert.ElementsMatch(t, expectedRefCount, actualRefCount, res) - assert.ElementsMatch(t, expectedUUID, actualUUID, res) - assert.ElementsMatch(t, expectedRef, actualRef, res) - }) - - t.Run("with multiple refs set in the object schema", func(t *testing.T) { - beacon1 := strfmt.URI( - "weaviate://localhost/c563d7fa-4a36-4eff-9f39-af1e1db276c4") - beacon2 := strfmt.URI( - "weaviate://localhost/49fe5d33-0b52-4189-8e8d-4268427c4317") - - schema := map[string]interface{}{ - "myRef": models.MultipleRef{ - {Beacon: beacon1}, - {Beacon: beacon2}, - }, - } - - uuid := "2609f1bc-7693-48f3-b531-6ddc52cd2501" - props := []*models.Property{ - { - Name: "myRef", - DataType: []string{"RefClass"}, - }, - } - res, err := a.Object(schema, props, strfmt.UUID(uuid)) - require.Nil(t, err) - - expectedRefCount := []Countable{ - {Data: []uint8{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02}}, - } - - expectedRef := []Countable{ - {Data: []byte(beacon1)}, - {Data: []byte(beacon2)}, - } - - expectedUUID := []Countable{ - { - Data: []byte(uuid), - TermFrequency: 0, - }, - } - - require.Len(t, res, 3) - var actualRefCount []Countable - var actualUUID []Countable - var actualRef []Countable - - for _, elem := range res { - switch elem.Name { - case helpers.MetaCountProp("myRef"): - actualRefCount = elem.Items - case "_id": - actualUUID = elem.Items - case "myRef": - actualRef = elem.Items - } - } - - assert.ElementsMatch(t, expectedRefCount, actualRefCount, res) - assert.ElementsMatch(t, expectedUUID, actualUUID, res) - assert.ElementsMatch(t, expectedRef, actualRef, res) - }) - - t.Run("with the ref omitted in the object schema", func(t *testing.T) { - schema := map[string]interface{}{} - - uuid := "2609f1bc-7693-48f3-b531-6ddc52cd2501" - props := []*models.Property{ - { - Name: "myRef", - DataType: []string{"RefClass"}, - }, - } - res, err := a.Object(schema, props, strfmt.UUID(uuid)) - require.Nil(t, err) - - expectedRefCount := []Countable{ - {Data: []uint8{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}}, - } - - expectedUUID := []Countable{ - { - Data: []byte(uuid), - TermFrequency: 0, - }, - } - - require.Len(t, res, 2) - var actualRefCount []Countable - var actualUUID []Countable - - for _, elem := range res { - if elem.Name == helpers.MetaCountProp("myRef") { - actualRefCount = elem.Items - } - if elem.Name == "_id" { - actualUUID = elem.Items - } - } - - assert.ElementsMatch(t, expectedRefCount, actualRefCount, res) - assert.ElementsMatch(t, expectedUUID, actualUUID, res) - }) - - // due to the fix introduced in https://github.com/weaviate/weaviate/pull/2320, - // MultipleRef's can appear as empty []interface{} when no actual refs are provided for - // an object's reference property. - // - // this test asserts that reference properties do not break when they are unmarshalled - // as empty interface{} slices. - t.Run("when rep prop is stored as empty interface{} slice", func(t *testing.T) { - uuid := "cf768bb0-03d8-4464-8f54-f787cf174c01" - name := "Transformers" - sch := map[string]interface{}{ - "name": name, - "reference": []interface{}{}, - } - - props := []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "reference", - DataType: []string{"SomeClass"}, - }, - } - res, err := a.Object(sch, props, strfmt.UUID(uuid)) - require.Nil(t, err) - - expectedUUID := []Countable{ - { - Data: []byte(uuid), - TermFrequency: 0, - }, - } - - expectedName := []Countable{ - { - Data: []byte(name), - TermFrequency: 1, - }, - } - - require.Len(t, res, 2) - var actualUUID []Countable - var actualName []Countable - - for _, elem := range res { - switch elem.Name { - case "_id": - actualUUID = elem.Items - case "name": - actualName = elem.Items - } - } - - assert.ElementsMatch(t, expectedUUID, actualUUID, res) - assert.ElementsMatch(t, expectedName, actualName, res) - }) - }) - - t.Run("when objects are indexed by timestamps", func(t *testing.T) { - sch := map[string]interface{}{ - "description": "pretty ok if you ask me", - "_creationTimeUnix": 1650551406404, - "_lastUpdateTimeUnix": 1650551406404, - } - - uuid := strfmt.UUID("2609f1bc-7693-48f3-b531-6ddc52cd2501") - props := []*models.Property{ - { - Name: "description", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWord, - }, - } - - res, err := a.Object(sch, props, uuid) - require.Nil(t, err) - require.Len(t, res, 4) - - expected := []Property{ - { - Name: "description", - Items: []Countable{ - {Data: []byte("pretty"), TermFrequency: 1}, - {Data: []byte("ok"), TermFrequency: 1}, - {Data: []byte("if"), TermFrequency: 1}, - {Data: []byte("you"), TermFrequency: 1}, - {Data: []byte("ask"), TermFrequency: 1}, - {Data: []byte("me"), TermFrequency: 1}, - }, - HasFilterableIndex: true, - HasSearchableIndex: true, - }, - { - Name: "_id", - Items: []Countable{{Data: []byte("2609f1bc-7693-48f3-b531-6ddc52cd2501")}}, - HasFilterableIndex: true, - HasSearchableIndex: false, - }, - { - Name: "_creationTimeUnix", - Items: []Countable{{Data: []byte("1650551406404")}}, - HasFilterableIndex: true, - HasSearchableIndex: false, - }, - { - Name: "_lastUpdateTimeUnix", - Items: []Countable{{Data: []byte("1650551406404")}}, - HasFilterableIndex: true, - HasSearchableIndex: false, - }, - } - - for i := range res { - assert.Equal(t, expected[i].Name, res[i].Name) - assert.Equal(t, expected[i].HasFilterableIndex, res[i].HasFilterableIndex) - assert.Equal(t, expected[i].HasSearchableIndex, res[i].HasSearchableIndex) - assert.ElementsMatch(t, expected[i].Items, res[i].Items) - } - }) -} - -func TestConvertSliceToUntyped(t *testing.T) { - tests := []struct { - name string - input interface{} - expectedErr error - }{ - { - name: "interface{} slice", - input: []interface{}{map[string]interface{}{}}, - }, - { - name: "string slice", - input: []string{"some", "slice"}, - }, - { - name: "int slice", - input: []int{1, 2, 3, 4, 5}, - }, - { - name: "time slice", - input: []time.Time{time.Now(), time.Now(), time.Now()}, - }, - { - name: "bool slice", - input: []bool{false}, - }, - { - name: "float64 slice", - input: []float64{1.2, 53555, 4.123, 2, 7.8877887, 0.0001}, - }, - { - name: "empty slice", - input: []string{}, - }, - { - name: "unsupported uint8 slice", - input: []uint8{1, 2, 3, 4, 5}, - expectedErr: fmt.Errorf("unsupported type []uint8"), - }, - { - name: "unsupported struct{}", - input: struct{}{}, - expectedErr: fmt.Errorf("unsupported type struct {}"), - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - output, err := typedSliceToUntyped(test.input) - if test.expectedErr != nil { - assert.EqualError(t, err, test.expectedErr.Error()) - } else { - require.Nil(t, err) - assert.Len(t, output, reflect.ValueOf(test.input).Len()) - assert.IsType(t, []interface{}{}, output) - } - }) - } -} - -func TestIndexInverted(t *testing.T) { - vFalse := false - vTrue := true - - t.Run("has filterable index", func(t *testing.T) { - type testCase struct { - name string - indexFilterable *bool - dataType schema.DataType - - expextedFilterable bool - } - - testCases := []testCase{ - { - name: "int, filterable null", - indexFilterable: nil, - dataType: schema.DataTypeInt, - - expextedFilterable: true, - }, - { - name: "int, filterable false", - indexFilterable: &vFalse, - dataType: schema.DataTypeInt, - - expextedFilterable: false, - }, - { - name: "int, filterable true", - indexFilterable: &vTrue, - dataType: schema.DataTypeInt, - - expextedFilterable: true, - }, - { - name: "text, filterable null", - indexFilterable: nil, - dataType: schema.DataTypeText, - - expextedFilterable: true, - }, - { - name: "text, filterable false", - indexFilterable: &vFalse, - dataType: schema.DataTypeText, - - expextedFilterable: false, - }, - { - name: "text, filterable true", - indexFilterable: &vTrue, - dataType: schema.DataTypeText, - - expextedFilterable: true, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - hasFilterableIndex := HasFilterableIndex(&models.Property{ - Name: "prop", - DataType: tc.dataType.PropString(), - IndexFilterable: tc.indexFilterable, - }) - - assert.Equal(t, tc.expextedFilterable, hasFilterableIndex) - }) - } - }) - - t.Run("has searchable index", func(t *testing.T) { - type testCase struct { - name string - indexSearchable *bool - dataType schema.DataType - - expextedSearchable bool - } - - testCases := []testCase{ - { - name: "int, searchable null", - indexSearchable: nil, - dataType: schema.DataTypeInt, - - expextedSearchable: false, - }, - { - name: "int, searchable false", - indexSearchable: &vFalse, - dataType: schema.DataTypeInt, - - expextedSearchable: false, - }, - { - name: "int, searchable true", - indexSearchable: &vTrue, - dataType: schema.DataTypeInt, - - expextedSearchable: false, - }, - { - name: "text, searchable null", - indexSearchable: nil, - dataType: schema.DataTypeText, - - expextedSearchable: true, - }, - { - name: "text, searchable false", - indexSearchable: &vFalse, - dataType: schema.DataTypeText, - - expextedSearchable: false, - }, - { - name: "text, searchable true", - indexSearchable: &vTrue, - dataType: schema.DataTypeText, - - expextedSearchable: true, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - hasSearchableIndex := HasSearchableIndex(&models.Property{ - Name: "prop", - DataType: tc.dataType.PropString(), - IndexSearchable: tc.indexSearchable, - }) - - assert.Equal(t, tc.expextedSearchable, hasSearchableIndex) - }) - } - }) -} - -func mustGetByteIntNumber(in int) []byte { - out, err := LexicographicallySortableInt64(int64(in)) - if err != nil { - panic(err) - } - return out -} - -func mustGetByteFloatNumber(in float64) []byte { - out, err := LexicographicallySortableFloat64(in) - if err != nil { - panic(err) - } - return out -} diff --git a/adapters/repos/db/inverted/prop_length_tracker.go b/adapters/repos/db/inverted/prop_length_tracker.go deleted file mode 100644 index 6f92ecfc9c7bac2488fc17c8b228bfe0d41b3080..0000000000000000000000000000000000000000 --- a/adapters/repos/db/inverted/prop_length_tracker.go +++ /dev/null @@ -1,431 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package inverted - -import ( - "encoding/binary" - "fmt" - "io" - "math" - "os" - "sync" - - "github.com/pkg/errors" -) - -// Page Design -// | Bytes | Description | -// | --------- | ------------------------------------------------ | -// | start | page is now 0 -// | 0-1 | uint16 pointer to last index byte -// | 2-3 | uint16 pointer for property name length -// | 4-n | property name -// | ... | repeat length+pointer pattern -// | 3584-3840 | second property buckets (64 buckets of float32) -// | 3840-4096 | first property buckets -// | repeat | page is now 1, repeat all of above -// -// Fixed Assumptions: -// - First two bytes always used to indicate end of index, minimal value is 02, -// as the first possible value with index length=0 is after the two bytes -// themselves. -// - 64 buckets of float32 per property (=256B per prop), excluding the index -// - One index row is always 4+len(propName), consisting of a uint16 prop name -// length pointer, the name itself and an offset pointer pointing to the start -// (first byte) of the buckets -// -// The counter to the last index byte is only an uint16, so it can at maximum address 65535. This will overflow when the -// 16th page is added (eg at page=15). To avoid a crash an error is returned in this case, but we will need to change -// the byteformat to fix this. -type PropertyLengthTracker struct { - file *os.File - path string - pages []byte - sync.Mutex -} - -func NewPropertyLengthTracker(path string) (*PropertyLengthTracker, error) { - f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0o666) - if err != nil { - return nil, err - } - - stat, err := f.Stat() - if err != nil { - return nil, err - } - - t := &PropertyLengthTracker{ - pages: nil, - file: f, - path: path, - } - - if stat.Size() > 0 { - // the file has existed before, we need to initialize with its content, we - // can read the entire contents into memory - existingPages, err := io.ReadAll(f) - if err != nil { - return nil, errors.Wrap(err, "read initial count from file") - } - - if len(existingPages)%4096 != 0 { - return nil, errors.Errorf( - "failed sanity check, prop len tracker file %s has length %d", path, - len(existingPages)) - } - - t.pages = existingPages - } else { - // this is the first time this is being created, initialize with an empty - // page - t.pages = make([]byte, 4096) - // set initial end-of-index offset to 2 - binary.LittleEndian.PutUint16(t.pages[0:2], 2) - } - - return t, nil -} - -func (t *PropertyLengthTracker) BucketCount(propName string, bucket uint16) (uint16, error) { - t.Lock() - defer t.Unlock() - - page, offset, ok := t.propExists(propName) - if !ok { - return 0, fmt.Errorf("property %v does not exist in OldPropertyLengthTracker", propName) - } - - offset = offset + page*4096 - - o := offset + (bucket * 4) - v := binary.LittleEndian.Uint32(t.pages[o : o+4]) - count := math.Float32frombits(v) - - return uint16(count), nil -} - -func (t *PropertyLengthTracker) PropertyNames() []string { - var names []string - pages := len(t.pages) / int(4096) - for page := 0; page < pages; page++ { - pageStart := page * int(4096) - - relativeEOI := binary.LittleEndian.Uint16(t.pages[pageStart : pageStart+2]) // t.uint16At(pageStart) - EOI := pageStart + int(relativeEOI) - - offset := int(pageStart) + 2 - for offset < EOI { - propNameLength := int(binary.LittleEndian.Uint16(t.pages[offset : offset+2])) // int(t.uint16At(offset)) - offset += 2 - - propName := t.pages[offset : offset+propNameLength] - offset += propNameLength - - offset += 2 - - names = append(names, string(propName)) - } - } - return names -} - -func (t *PropertyLengthTracker) TrackProperty(propName string, value float32) error { - t.Lock() - defer t.Unlock() - - var page uint16 - var relBucketOffset uint16 - if p, o, ok := t.propExists(propName); ok { - page = p - relBucketOffset = o - } else { - var err error - page, relBucketOffset, err = t.addProperty(propName) - if err != nil { - return err - } - } - - bucketOffset := page*4096 + relBucketOffset + t.bucketFromValue(value)*4 - - v := binary.LittleEndian.Uint32(t.pages[bucketOffset : bucketOffset+4]) - currentValue := math.Float32frombits(v) - currentValue += 1 - v = math.Float32bits(currentValue) - binary.LittleEndian.PutUint32(t.pages[bucketOffset:bucketOffset+4], v) - return nil -} - -func (t *PropertyLengthTracker) UnTrackProperty(propName string, value float32) error { - t.Lock() - defer t.Unlock() - - var page uint16 - var relBucketOffset uint16 - if p, o, ok := t.propExists(propName); ok { - page = p - relBucketOffset = o - } else { - return fmt.Errorf("property %v does not exist in OldPropertyLengthTracker", propName) - } - - bucketOffset := page*4096 + relBucketOffset + t.bucketFromValue(value)*4 - - v := binary.LittleEndian.Uint32(t.pages[bucketOffset : bucketOffset+4]) - currentValue := math.Float32frombits(v) - currentValue -= 1 - v = math.Float32bits(currentValue) - binary.LittleEndian.PutUint32(t.pages[bucketOffset:bucketOffset+4], v) - return nil -} - -// propExists returns page number, relative offset on page, and a bool whether -// the prop existed at all. The first to values have no meaning if the latter -// is false -func (t *PropertyLengthTracker) propExists(needle string) (uint16, uint16, bool) { - pages := len(t.pages) / 4096 - for page := 0; page < pages; page++ { - pageStart := page * 4096 - - relativeEOI := binary.LittleEndian.Uint16(t.pages[pageStart : pageStart+2]) - EOI := pageStart + int(relativeEOI) - - offset := int(pageStart) + 2 - for offset < EOI { - propNameLength := int(binary.LittleEndian.Uint16( - t.pages[offset : offset+2])) - offset += 2 - - propName := t.pages[offset : offset+propNameLength] - offset += propNameLength - bucketPointer := binary.LittleEndian.Uint16( - t.pages[offset : offset+2]) - offset += 2 - - if string(propName) == needle { - return uint16(page), bucketPointer, true - } - - } - } - return 0, 0, false -} - -func (t *PropertyLengthTracker) addProperty(propName string) (uint16, uint16, error) { - page := uint16(0) - - for { - propNameBytes := []byte(propName) - t.createPageIfNotExists(page) - pageStart := page * 4096 - lastBucketOffset := pageStart + 4096 - - relativeOffset := binary.LittleEndian.Uint16(t.pages[pageStart : pageStart+2]) - offset := pageStart + relativeOffset - if relativeOffset != 2 { - // relative offset is other than 2, so there are also props in. This - // means we can take the value of offset-2 to read the bucket offset - lastBucketOffset = pageStart + binary.LittleEndian. - Uint16(t.pages[offset-2:offset]) - } - - if !t.canPageFit(propNameBytes, offset, lastBucketOffset) { - page++ - // overflow of uint16 variable that tracks the size of the tracker - if page > 15 { - return 0, 0, fmt.Errorf("could not add property %v, to PropertyLengthTracker, because the total"+ - "length of all properties is too long", propName) - } - continue - } - - propNameLength := uint16(len(propNameBytes)) - binary.LittleEndian.PutUint16(t.pages[offset:offset+2], propNameLength) - offset += 2 - copy(t.pages[offset:offset+propNameLength], propNameBytes) - offset += propNameLength - - newBucketOffset := lastBucketOffset - 256 - pageStart - binary.LittleEndian.PutUint16(t.pages[offset:offset+2], newBucketOffset) - offset += 2 - - // update end of index offset for page, since the prop name index has - // now grown - binary.LittleEndian.PutUint16(t.pages[pageStart:pageStart+2], offset-pageStart) - return page, newBucketOffset, nil - } -} - -func (t *PropertyLengthTracker) canPageFit(propName []byte, - offset uint16, lastBucketOffset uint16, -) bool { - // lastBucketOffset represents the end of the writable area, offset - // represents the start, which means we can take the delta to see // how - // much space is left on this page - spaceLeft := lastBucketOffset - offset - - // we need to write 256 bytes for the buckets, plus two pointers of uint16 - spaceNeeded := uint16(len(propName)+4) + 256 - - return spaceLeft >= spaceNeeded -} - -func (t *PropertyLengthTracker) bucketFromValue(value float32) uint16 { - if value <= 5.00 { - return uint16(value) - 1 - } - - bucket := int(math.Log(float64(value)/4.0)/math.Log(1.25) + 4) - if bucket > 63 { - return 64 - } - return uint16(bucket) -} - -func (t *PropertyLengthTracker) valueFromBucket(bucket uint16) float32 { - if bucket <= 5 { - return float32(bucket + 1) - } - - return float32(4 * math.Pow(1.25, float64(bucket)-3.5)) -} - -func (t *PropertyLengthTracker) PropertyMean(propName string) (float32, error) { - t.Lock() - defer t.Unlock() - - page, offset, ok := t.propExists(propName) - if !ok { - return 0, nil - } - - sum := float32(0) - totalCount := float32(0) - bucket := uint16(0) - - offset = offset + page*4096 - for o := offset; o < offset+256; o += 4 { - v := binary.LittleEndian.Uint32(t.pages[o : o+4]) - count := math.Float32frombits(v) - sum += float32(t.valueFromBucket(bucket)) * count - totalCount += count - - bucket++ - } - - if totalCount == 0 { - return 0, nil - } - - return sum / totalCount, nil -} - -func (t *PropertyLengthTracker) PropertyTally(propName string) (int, int, float32, error) { - t.Lock() - defer t.Unlock() - - page, offset, ok := t.propExists(propName) - if !ok { - return 0, 0, 0, nil - } - - sum := float32(0) - totalCount := float32(0) - bucket := uint16(0) - - offset = offset + page*4096 - for o := offset; o < offset+256; o += 4 { - v := binary.LittleEndian.Uint32(t.pages[o : o+4]) - count := math.Float32frombits(v) - sum += float32(t.valueFromBucket(bucket)) * count - totalCount += count - - bucket++ - } - - if totalCount == 0 { - return 0, 0, 0, nil - } - - return int(sum), int(totalCount), sum / totalCount, nil -} - -func (t *PropertyLengthTracker) createPageIfNotExists(page uint16) { - if uint16(len(t.pages))/4096-1 < page { - // we need to grow the page buffer - newPages := make([]byte, uint64(page)*4096+4096) - copy(newPages[:len(t.pages)], t.pages) - - // the new page must have the correct offset initialized - binary.LittleEndian.PutUint16(newPages[page*4096:page*4096+2], 2) - t.pages = newPages - } -} - -func (t *PropertyLengthTracker) Flush() error { - t.Lock() - defer t.Unlock() - - if err := t.file.Truncate(int64(len(t.pages))); err != nil { - return errors.Wrap(err, "truncate prop tracker file to correct length") - } - - if _, err := t.file.Seek(0, io.SeekStart); err != nil { - return errors.Wrap(err, "seek to beginning of prop tracker file") - } - - if _, err := t.file.Write(t.pages); err != nil { - return errors.Wrap(err, "flush page content to disk") - } - - return nil -} - -func (t *PropertyLengthTracker) Close() error { - if err := t.Flush(); err != nil { - return errors.Wrap(err, "flush before closing") - } - - t.Lock() - defer t.Unlock() - - if err := t.file.Close(); err != nil { - return errors.Wrap(err, "close prop length tracker file") - } - - t.pages = nil - - return nil -} - -func (t *PropertyLengthTracker) Drop() error { - t.Lock() - defer t.Unlock() - - if err := t.file.Close(); err != nil { - _ = err - // explicitly ignore error - } - - t.pages = nil - - if err := os.Remove(t.path); err != nil { - return errors.Wrap(err, "remove prop length tracker state from disk") - } - - return nil -} - -func (t *PropertyLengthTracker) FileName() string { - return t.file.Name() -} diff --git a/adapters/repos/db/inverted/prop_length_tracker_test.go b/adapters/repos/db/inverted/prop_length_tracker_test.go deleted file mode 100644 index 8b386a8ed707fdf35c2cb0f3b67e615f94174927..0000000000000000000000000000000000000000 --- a/adapters/repos/db/inverted/prop_length_tracker_test.go +++ /dev/null @@ -1,690 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package inverted - -import ( - "fmt" - "path" - "testing" - - "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func Test_PropertyLengthTracker(t *testing.T) { - dirName := t.TempDir() - trackerPath := path.Join(dirName, "my_test_shard") - l := logrus.New() - - // This test suite doesn't actually test persistence, there is a separate - // one. However, we still need to supply a valid path. Since nothing is ever - // written, we can use the same one for each sub-test without them - // accidentally sharing state. - - t.Run("single prop", func(t *testing.T) { - type test struct { - values []float32 - name string - floatCompare bool - } - - tests := []test{ - { - values: []float32{2, 2, 3, 100, 100, 500, 7}, - name: "mixed_values", - floatCompare: true, - }, - { - values: []float32{ - 1000, 1200, 1000, 1300, 800, 2000, 2050, - 2070, 900, - }, - name: "high_values", - floatCompare: true, - }, - { - values: []float32{ - 60000, 50000, 65000, - }, - name: "very_high_values", - floatCompare: true, - }, - { - values: []float32{ - 1, 2, 4, 3, 4, 2, 1, 5, 6, 7, 8, 2, 7, 2, 3, 5, - 6, 3, 5, 9, 3, 4, 8, - }, - name: "very_low_values", - floatCompare: true, - }, - { - values: []float32{0, 0}, - name: "zeros", - floatCompare: false, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - tracker, err := NewJsonShardMetaData(trackerPath+test.name, l) - require.Nil(t, err) - - actualMean := float32(0) - for _, v := range test.values { - tracker.TrackProperty("my-very-first-prop", v) - actualMean += v - } - actualMean = actualMean / float32(len(test.values)) - - res, err := tracker.PropertyMean("my-very-first-prop") - require.Nil(t, err) - - if test.floatCompare { - assert.InEpsilon(t, actualMean, res, 0.1) - } else { - assert.Equal(t, actualMean, res) - } - require.Nil(t, tracker.Close()) - }) - } - }) - - t.Run("test untrack", func(t *testing.T) { - tracker, err := NewJsonShardMetaData(trackerPath, l) - require.Nil(t, err) - - tracker.TrackProperty("test-prop", 1) - tracker.TrackProperty("test-prop", 2) - tracker.TrackProperty("test-prop", 3) - tracker.Flush(false) - - sum, count, mean, err := tracker.PropertyTally("test-prop") - require.Nil(t, err) - assert.Equal(t, 6, sum) - assert.Equal(t, 3, count) - assert.InEpsilon(t, 2, mean, 0.1) - - tracker.UnTrackProperty("test-prop", 2) - sum, count, mean, err = tracker.PropertyTally("test-prop") - require.Nil(t, err) - assert.Equal(t, 4, sum) - assert.Equal(t, 2, count) - assert.InEpsilon(t, 2, mean, 0.1) - - tracker.UnTrackProperty("test-prop", 1) - sum, count, mean, err = tracker.PropertyTally("test-prop") - require.Nil(t, err) - assert.Equal(t, 3, sum) - assert.Equal(t, 1, count) - assert.InEpsilon(t, 3, mean, 0.1) - - require.Nil(t, tracker.Close()) - }) - - t.Run("multiple properties (can all fit on one page)", func(t *testing.T) { - type prop struct { - values []float32 - propName string - } - - props := []prop{ - { - values: []float32{2, 2, 3, 100, 100, 500, 7}, - propName: "property-numero-uno", - }, { - values: []float32{ - 1000, 1200, 1000, 1300, 800, 2000, 2050, - 2070, 900, - }, - propName: "the-second-of-the-properties", - }, { - values: []float32{ - 60000, 50000, 65000, - }, - propName: "property_nummer_DREI", - }, - } - - // This time we use a single tracker - tracker, err := NewJsonShardMetaData(trackerPath, l) - require.Nil(t, err) - - for _, prop := range props { - for _, v := range prop.values { - tracker.TrackProperty(prop.propName, v) - } - } - - for _, prop := range props { - actualMean := float32(0) - for _, v := range prop.values { - actualMean += v - } - actualMean = actualMean / float32(len(prop.values)) - - res, err := tracker.PropertyMean(prop.propName) - require.Nil(t, err) - - assert.InEpsilon(t, actualMean, res, 0.1) - } - - require.Nil(t, tracker.Close()) - }) - - t.Run("with more properties that can fit on one page", func(t *testing.T) { - // This time we use a single tracker - tracker, err := NewJsonShardMetaData(trackerPath, l) - require.Nil(t, err) - - create20PropsAndVerify(t, tracker) - - require.Nil(t, tracker.Close()) - }) -} - -func create20PropsAndVerify(t *testing.T, tracker *JsonShardMetaData) { - type prop struct { - values []float32 - propName string - } - - // the most props we could ever fit on a single page is 16 if there was no - // index, which is impossible. This means the practical max is 15, so at - // least 5 props should overflow to the second page. - propCount := 20 - props := make([]prop, propCount) - - for i := range props { - props[i] = prop{ - values: []float32{1, 4, 3, 17}, - propName: fmt.Sprintf("prop_%d", i), - } - } - - for _, prop := range props { - for _, v := range prop.values { - tracker.TrackProperty(prop.propName, v) - } - } - - for _, prop := range props { - actualMean := float32(0) - for _, v := range prop.values { - actualMean += v - } - actualMean = actualMean / float32(len(prop.values)) - - res, err := tracker.PropertyMean(prop.propName) - require.Nil(t, err) - - assert.InEpsilon(t, actualMean, res, 0.1) - } - - // modify a prop on page 2 and verify - tracker.TrackProperty("prop_19", 24) - actualMeanForProp20 := float32(1+4+3+17+25) / 5.0 - res, err := tracker.PropertyMean("prop_19") - require.Nil(t, err) - - assert.InEpsilon(t, actualMeanForProp20, res, 0.1) -} - -func Test_PropertyLengthTracker_Persistence(t *testing.T) { - dirName := t.TempDir() - - path := path.Join(dirName, "my_test_shard") - - var tracker *JsonShardMetaData - l := logrus.New() - - t.Run("initializing an empty tracker, no file present", func(t *testing.T) { - tr, err := NewJsonShardMetaData(path, l) - require.Nil(t, err) - tracker = tr - }) - - t.Run("importing multi-page data and verifying", func(t *testing.T) { - create20PropsAndVerify(t, tracker) - }) - - t.Run("commit the state to disk", func(t *testing.T) { - require.Nil(t, tracker.Flush(false)) - }) - - t.Run("shut down the tracker", func(t *testing.T) { - require.Nil(t, tracker.Close()) - }) - - var secondTracker *JsonShardMetaData - t.Run("initializing a new tracker from the same file", func(t *testing.T) { - tr, err := NewJsonShardMetaData(path, l) - require.Nil(t, err) - secondTracker = tr - }) - - t.Run("verify data is correct after read from disk", func(t *testing.T) { - // root page - actualMeanForProp0 := float32(1+4+3+17) / 4.0 - res, err := secondTracker.PropertyMean("prop_0") - require.Nil(t, err) - assert.InEpsilon(t, actualMeanForProp0, res, 0.1) - - // later page - actualMeanForProp20 := float32(1+4+3+17+25) / 5.0 - res, err = secondTracker.PropertyMean("prop_19") - require.Nil(t, err) - assert.InEpsilon(t, actualMeanForProp20, res, 0.1) - }) -} - -// Testing the switch from the old property length tracker to the new one -func TestFormatConversion(t *testing.T) { - dirName := t.TempDir() - - path := path.Join(dirName, "my_test_shard") - - var tracker *PropertyLengthTracker - - t.Run("initializing an empty tracker, no file present", func(t *testing.T) { - tr, err := NewPropertyLengthTracker(path) - require.Nil(t, err) - tracker = tr - }) - - t.Run("importing multi-page data and verifying", func(t *testing.T) { - create20PropsAndVerify_old(t, tracker) - }) - - t.Run("commit the state to disk", func(t *testing.T) { - require.Nil(t, tracker.Flush()) - }) - - t.Run("shut down the tracker", func(t *testing.T) { - require.Nil(t, tracker.Close()) - }) - - var newTracker *JsonShardMetaData - l := logrus.New() - - t.Run("initializing a new tracker from the same file", func(t *testing.T) { - tr, err := NewJsonShardMetaData(path, l) - require.Nil(t, err) - newTracker = tr - }) - - t.Run("verify data is correct after read from disk", func(t *testing.T) { - // root page - actualMeanForProp0 := float32(1+4+3+17) / 4.0 - res, err := newTracker.PropertyMean("prop_0") - require.Nil(t, err) - assert.InEpsilon(t, actualMeanForProp0, res, 0.1) - - // later page - actualMeanForProp20 := float32(1+4+3+17+25) / 5.0 - res, err = newTracker.PropertyMean("prop_19") - require.Nil(t, err) - assert.InEpsilon(t, actualMeanForProp20, res, 0.1) - - res, err = newTracker.PropertyMean("prop_22") - require.Nil(t, err) - assert.EqualValues(t, res, 0) - sum, count, average, _ := newTracker.PropertyTally("prop_22") - assert.EqualValues(t, 0, sum) - assert.EqualValues(t, 3, count) - assert.EqualValues(t, 0, average) - }) -} - -func create20PropsAndVerify_old(t *testing.T, tracker *PropertyLengthTracker) { - type prop struct { - values []float32 - propName string - } - - // the most props we could ever fit on a single page is 16 if there was no - // index, which is impossible. This means the practical max is 15, so at - // least 5 props should overflow to the second page. - propCount := 20 - props := make([]prop, propCount) - - for i := range props { - props[i] = prop{ - values: []float32{1, 4, 3, 17}, - propName: fmt.Sprintf("prop_%d", i), - } - } - - for _, prop := range props { - for _, v := range prop.values { - tracker.TrackProperty(prop.propName, v) - } - } - - tracker.TrackProperty("prop_22", 0) - tracker.TrackProperty("prop_22", 0) - tracker.TrackProperty("prop_22", 0) - - for _, prop := range props { - actualMean := float32(0) - for _, v := range prop.values { - actualMean += v - } - actualMean = actualMean / float32(len(prop.values)) - - res, err := tracker.PropertyMean(prop.propName) - require.Nil(t, err) - - assert.InEpsilon(t, actualMean, res, 0.1) - } - - // modify a prop on page 2 and verify - tracker.TrackProperty("prop_19", 24) - actualMeanForProp20 := float32(1+4+3+17+25) / 5.0 - res, err := tracker.PropertyMean("prop_19") - require.Nil(t, err) - - assert.InEpsilon(t, actualMeanForProp20, res, 0.1) - - res, err = tracker.PropertyMean("prop_22") - require.Nil(t, err) - assert.EqualValues(t, res, 0) - - sum, _, average, _ := tracker.PropertyTally("prop_22") - assert.EqualValues(t, 0, sum) - // assert.EqualValues(t, 3, count) - assert.EqualValues(t, 0, average) -} - -// Test the old property length tracker - -func TestOldPropertyLengthTracker(t *testing.T) { - dirName := t.TempDir() - trackerPath := path.Join(dirName, "my_test_shard") - - // This test suite doesn't actually test persistence, there is a separate - // one. However, we still need to supply a valid path. Since nothing is ever - // written, we can use the same one for each sub-test without them - // accidentally sharing state. - - t.Run("single prop", func(t *testing.T) { - type test struct { - values []float32 - name string - floatCompare bool - } - - tests := []test{ - { - values: []float32{2, 2, 3, 100, 100, 500, 7}, - name: "mixed_values", - floatCompare: true, - }, { - values: []float32{ - 1000, 1200, 1000, 1300, 800, 2000, 2050, - 2070, 900, - }, - name: "high_values", - floatCompare: true, - }, { - values: []float32{ - 60000, 50000, 65000, - }, - name: "very_high_values", - floatCompare: true, - }, { - values: []float32{ - 1, 2, 4, 3, 4, 2, 1, 5, 6, 7, 8, 2, 7, 2, 3, 5, - 6, 3, 5, 9, 3, 4, 8, - }, - name: "very_low_values", - floatCompare: true, - }, { - values: []float32{0, 0}, - name: "zeros", - floatCompare: false, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - tracker, err := NewPropertyLengthTracker(trackerPath + test.name) - require.Nil(t, err) - - actualMean := float32(0) - for _, v := range test.values { - tracker.TrackProperty("my-very-first-prop", v) - actualMean += v - } - actualMean = actualMean / float32(len(test.values)) - - res, err := tracker.PropertyMean("my-very-first-prop") - require.Nil(t, err) - - if test.floatCompare { - assert.InEpsilon(t, actualMean, res, 0.1) - } else { - assert.Equal(t, actualMean, res) - } - require.Nil(t, tracker.Close()) - }) - } - }) - - t.Run("test untrack", func(t *testing.T) { - tracker, err := NewPropertyLengthTracker(trackerPath) - require.Nil(t, err) - - tracker.TrackProperty("test-prop", 1) - tracker.TrackProperty("test-prop", 2) - tracker.TrackProperty("test-prop", 3) - tracker.Flush() - - sum, count, mean, err := tracker.PropertyTally("test-prop") - require.Nil(t, err) - assert.Equal(t, 6, sum) - assert.Equal(t, 3, count) - assert.InEpsilon(t, 2, mean, 0.1) - - tracker.UnTrackProperty("test-prop", 2) - sum, count, mean, err = tracker.PropertyTally("test-prop") - require.Nil(t, err) - assert.Equal(t, 4, sum) - assert.Equal(t, 2, count) - assert.InEpsilon(t, 2, mean, 0.1) - - tracker.UnTrackProperty("test-prop", 1) - sum, count, mean, err = tracker.PropertyTally("test-prop") - require.Nil(t, err) - assert.Equal(t, 3, sum) - assert.Equal(t, 1, count) - assert.InEpsilon(t, 3, mean, 0.1) - - require.Nil(t, tracker.Close()) - }) - - t.Run("multiple properties (can all fit on one page)", func(t *testing.T) { - type prop struct { - values []float32 - propName string - } - - props := []prop{ - { - values: []float32{2, 2, 3, 100, 100, 500, 7}, - propName: "property-numero-uno", - }, { - values: []float32{ - 1000, 1200, 1000, 1300, 800, 2000, 2050, - 2070, 900, - }, - propName: "the-second-of-the-properties", - }, { - values: []float32{ - 60000, 50000, 65000, - }, - propName: "property_nummer_DREI", - }, - } - - // This time we use a single tracker - tracker, err := NewPropertyLengthTracker(trackerPath) - require.Nil(t, err) - - for _, prop := range props { - for _, v := range prop.values { - tracker.TrackProperty(prop.propName, v) - } - } - - for _, prop := range props { - actualMean := float32(0) - for _, v := range prop.values { - actualMean += v - } - actualMean = actualMean / float32(len(prop.values)) - - res, err := tracker.PropertyMean(prop.propName) - require.Nil(t, err) - - assert.InEpsilon(t, actualMean, res, 0.1) - } - - require.Nil(t, tracker.Close()) - }) - - t.Run("with more properties that can fit on one page", func(t *testing.T) { - // This time we use a single tracker - tracker, err := NewPropertyLengthTracker(trackerPath) - require.Nil(t, err) - - create20PropsAndVerify_old(t, tracker) - - require.Nil(t, tracker.Close()) - }) -} - -func TestOldPropertyLengthTracker_Persistence(t *testing.T) { - dirName := t.TempDir() - - path := path.Join(dirName, "my_test_shard") - - var tracker *PropertyLengthTracker - - t.Run("initializing an empty tracker, no file present", func(t *testing.T) { - tr, err := NewPropertyLengthTracker(path) - require.Nil(t, err) - tracker = tr - }) - - t.Run("importing multi-page data and verifying", func(t *testing.T) { - create20PropsAndVerify_old(t, tracker) - }) - - t.Run("commit the state to disk", func(t *testing.T) { - require.Nil(t, tracker.Flush()) - }) - - t.Run("shut down the tracker", func(t *testing.T) { - require.Nil(t, tracker.Close()) - }) - - var secondTracker *PropertyLengthTracker - t.Run("initializing a new tracker from the same file", func(t *testing.T) { - tr, err := NewPropertyLengthTracker(path) - require.Nil(t, err) - secondTracker = tr - }) - - t.Run("verify data is correct after read from disk", func(t *testing.T) { - // root page - actualMeanForProp0 := float32(1+4+3+17) / 4.0 - res, err := secondTracker.PropertyMean("prop_0") - require.Nil(t, err) - assert.InEpsilon(t, actualMeanForProp0, res, 0.1) - - // later page - actualMeanForProp20 := float32(1+4+3+17+25) / 5.0 - res, err = secondTracker.PropertyMean("prop_19") - require.Nil(t, err) - assert.InEpsilon(t, actualMeanForProp20, res, 0.1) - }) - - t.Run("shut down the second tracker", func(t *testing.T) { - require.Nil(t, secondTracker.Close()) - }) -} - -func Test_PropertyLengthTracker_Overflow(t *testing.T) { - dirName := t.TempDir() - path := path.Join(dirName, "my_test_shard") - - tracker, err := NewPropertyLengthTracker(path) - require.Nil(t, err) - - for i := 0; i < 16*15; i++ { - err := tracker.TrackProperty(fmt.Sprintf("prop_%v", i), float32(i)) - require.Nil(t, err) - } - - // Check that property that would cause the internal counter to overflow is not added - err = tracker.TrackProperty("OVERFLOW", float32(123)) - require.NotNil(t, err) - - require.Nil(t, tracker.Close()) -} - -// Test that object racking works -func Test_PropertyLengthTracker_ObjectTracking(t *testing.T) { - dirName := t.TempDir() - - path := path.Join(dirName, "my_test_shard") - - var tracker *JsonShardMetaData - - l := logrus.New() - - t.Run("initializing an empty tracker, no file present", func(t *testing.T) { - tr, err := NewJsonShardMetaData(path, l) - require.Nil(t, err) - tracker = tr - }) - - t.Run("test object tracking", func(t *testing.T) { - start := tracker.ObjectTally() - require.Equal(t, start, 0) - - tracker.TrackObjects(1) - require.Equal(t, tracker.ObjectTally(), 1) - - tracker.TrackObjects(1) - require.Equal(t, tracker.ObjectTally(), 2) - - tracker.TrackObjects(-1) - require.Equal(t, tracker.ObjectTally(), 1) - - tracker.TrackObjects(-1) - require.Equal(t, tracker.ObjectTally(), 0) - - tracker.TrackObjects(2) - require.Equal(t, tracker.ObjectTally(), 2) - - err := tracker.Close() - require.Nil(t, err) - - tr, err := NewJsonShardMetaData(path, l) - require.Nil(t, err) - tracker = tr - - require.Equal(t, tracker.ObjectTally(), 2) - }) -} diff --git a/adapters/repos/db/inverted/prop_value_pairs.go b/adapters/repos/db/inverted/prop_value_pairs.go deleted file mode 100644 index 434700b217450d10802f42b9ad9ace29682e02ac..0000000000000000000000000000000000000000 --- a/adapters/repos/db/inverted/prop_value_pairs.go +++ /dev/null @@ -1,158 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package inverted - -import ( - "context" - "fmt" - "strings" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv/roaringset" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "golang.org/x/sync/errgroup" -) - -type propValuePair struct { - prop string - operator filters.Operator - - // set for all values that can be served by an inverted index, i.e. anything - // that's not a geoRange - value []byte - - // only set if operator=OperatorWithinGeoRange, as that cannot be served by a - // byte value from an inverted index - valueGeoRange *filters.GeoRange - docIDs docBitmap - children []*propValuePair - hasFilterableIndex bool - hasSearchableIndex bool - Class *models.Class // The schema -} - -func newPropValuePair(class *models.Class) (*propValuePair, error) { - if class == nil { - return nil, errors.Errorf("class must not be nil") - } - return &propValuePair{docIDs: newDocBitmap(), Class: class}, nil -} - -func (pv *propValuePair) fetchDocIDs(s *Searcher, limit int) error { - if pv.operator.OnValue() { - - // TODO text_rbm_inverted_index find better way check whether prop len - if strings.HasSuffix(pv.prop, filters.InternalPropertyLength) && - !pv.Class.InvertedIndexConfig.IndexPropertyLength { - return errors.Errorf("Property length must be indexed to be filterable! add `IndexPropertyLength: true` to the invertedIndexConfig in %v. Geo-coordinates, phone numbers and data blobs are not supported by property length.", pv.Class.Class) - } - - if pv.operator == filters.OperatorIsNull && !pv.Class.InvertedIndexConfig.IndexNullState { - return errors.Errorf("Nullstate must be indexed to be filterable! Add `indexNullState: true` to the invertedIndexConfig") - } - - if (pv.prop == filters.InternalPropCreationTimeUnix || - pv.prop == filters.InternalPropLastUpdateTimeUnix) && - !pv.Class.InvertedIndexConfig.IndexTimestamps { - return errors.Errorf("Timestamps must be indexed to be filterable! Add `IndexTimestamps: true` to the InvertedIndexConfig in %v", pv.Class.Class) - } - - var bucketName string - if pv.hasFilterableIndex { - bucketName = helpers.BucketFromPropNameLSM(pv.prop) - } else if pv.hasSearchableIndex { - bucketName = helpers.BucketSearchableFromPropNameLSM(pv.prop) - } else { - return errors.Errorf("bucket for prop %s not found - is it indexed?", pv.prop) - } - - b := s.store.Bucket(bucketName) - - // TODO: I think we can delete this check entirely. The bucket will never be nill, and routines should now check if their particular feature is active in the schema. However, not all those routines have checks yet. - if b == nil && pv.operator != filters.OperatorWithinGeoRange { - // a nil bucket is ok for a WithinGeoRange filter, as this query is not - // served by the inverted index, but propagated to a secondary index in - // .docPointers() - return errors.Errorf("bucket for prop %s not found - is it indexed?", pv.prop) - } - - ctx := context.TODO() // TODO: pass through instead of spawning new - dbm, err := s.docBitmap(ctx, b, limit, pv) - if err != nil { - return err - } - pv.docIDs = dbm - } else { - eg := errgroup.Group{} - // prevent unbounded concurrency, see - // https://github.com/weaviate/weaviate/issues/3179 for details - eg.SetLimit(2 * _NUMCPU) - for i, child := range pv.children { - i, child := i, child - eg.Go(func() error { - // Explicitly set the limit to 0 (=unlimited) as this is a nested filter, - // otherwise we run into situations where each subfilter on their own - // runs into the limit, possibly yielding in "less than limit" results - // after merging. - err := child.fetchDocIDs(s, 0) - if err != nil { - return errors.Wrapf(err, "nested child %d", i) - } - - return nil - }) - } - if err := eg.Wait(); err != nil { - return fmt.Errorf("nested query: %w", err) - } - } - - return nil -} - -func (pv *propValuePair) mergeDocIDs() (*docBitmap, error) { - if pv.operator.OnValue() { - return &pv.docIDs, nil - } - - if pv.operator != filters.OperatorAnd && pv.operator != filters.OperatorOr { - return nil, fmt.Errorf("unsupported operator: %s", pv.operator.Name()) - } - if len(pv.children) == 0 { - return nil, fmt.Errorf("no children for operator: %s", pv.operator.Name()) - } - - dbms := make([]*docBitmap, len(pv.children)) - for i, child := range pv.children { - dbm, err := child.mergeDocIDs() - if err != nil { - return nil, errors.Wrapf(err, "retrieve doc bitmap of child %d", i) - } - dbms[i] = dbm - } - - mergeRes := dbms[0].docIDs.Clone() - mergeFn := mergeRes.And - if pv.operator == filters.OperatorOr { - mergeFn = mergeRes.Or - } - - for i := 1; i < len(dbms); i++ { - mergeFn(dbms[i].docIDs) - } - - return &docBitmap{ - docIDs: roaringset.Condense(mergeRes), - }, nil -} diff --git a/adapters/repos/db/inverted/prop_value_pairs_test.go b/adapters/repos/db/inverted/prop_value_pairs_test.go deleted file mode 100644 index 3930dd29976157b4c4208ca4144b00b819eb4524..0000000000000000000000000000000000000000 --- a/adapters/repos/db/inverted/prop_value_pairs_test.go +++ /dev/null @@ -1,111 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package inverted - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/sroar" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv/roaringset" - "github.com/weaviate/weaviate/entities/filters" -) - -func TestPropValuePairs_Merging(t *testing.T) { - t.Run("always creates new underlying bitmap", func(t *testing.T) { - type testCase struct { - name string - - bitmaps []*sroar.Bitmap - operator filters.Operator - - expectedIds []uint64 - } - - testCases := []testCase{ - { - name: "AND; different sets", - - bitmaps: []*sroar.Bitmap{ - roaringset.NewBitmap(7, 8, 9, 10, 11), - roaringset.NewBitmap(1, 3, 5, 7, 9, 11), - roaringset.NewBitmap(1, 3, 5, 7, 9), - }, - operator: filters.OperatorAnd, - - expectedIds: []uint64{7, 9}, - }, - { - name: "OR; different sets", - - bitmaps: []*sroar.Bitmap{ - roaringset.NewBitmap(7, 8, 9, 10, 11), - roaringset.NewBitmap(1, 3, 5, 7, 9, 11), - roaringset.NewBitmap(1, 3, 5, 7, 9), - }, - operator: filters.OperatorOr, - - expectedIds: []uint64{1, 3, 5, 7, 8, 9, 10, 11}, - }, - { - name: "AND; same sets", - - bitmaps: []*sroar.Bitmap{ - roaringset.NewBitmap(7, 8, 9, 10, 11), - roaringset.NewBitmap(7, 8, 9, 10, 11), - roaringset.NewBitmap(7, 8, 9, 10, 11), - }, - operator: filters.OperatorAnd, - - expectedIds: []uint64{7, 8, 9, 10, 11}, - }, - { - name: "OR; same sets", - - bitmaps: []*sroar.Bitmap{ - roaringset.NewBitmap(7, 8, 9, 10, 11), - roaringset.NewBitmap(7, 8, 9, 10, 11), - roaringset.NewBitmap(7, 8, 9, 10, 11), - }, - operator: filters.OperatorOr, - - expectedIds: []uint64{7, 8, 9, 10, 11}, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - pv := &propValuePair{ - operator: tc.operator, - children: make([]*propValuePair, len(tc.bitmaps)), - } - for i := range tc.bitmaps { - pv.children[i] = &propValuePair{ - operator: filters.OperatorEqual, - docIDs: docBitmap{ - docIDs: tc.bitmaps[i], - }, - } - } - - dbm, err := pv.mergeDocIDs() - - require.Nil(t, err) - assert.ElementsMatch(t, tc.expectedIds, dbm.IDs()) - assert.False(t, tc.bitmaps[0] == dbm.docIDs) - assert.False(t, tc.bitmaps[1] == dbm.docIDs) - assert.False(t, tc.bitmaps[2] == dbm.docIDs) - }) - } - }) -} diff --git a/adapters/repos/db/inverted/row_reader.go b/adapters/repos/db/inverted/row_reader.go deleted file mode 100644 index 6259d9c3c8940dc1afa009257845865cebf5261c..0000000000000000000000000000000000000000 --- a/adapters/repos/db/inverted/row_reader.go +++ /dev/null @@ -1,257 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package inverted - -import ( - "bytes" - "context" - "fmt" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv" - "github.com/weaviate/weaviate/entities/filters" -) - -// RowReader reads one or many row(s) depending on the specified operator -type RowReader struct { - value []byte - bucket *lsmkv.Bucket - operator filters.Operator - - keyOnly bool -} - -// If keyOnly is set, the RowReader will request key-only cursors wherever -// cursors are used, the specified value arguments in the ReadFn will always be -// nil -func NewRowReader(bucket *lsmkv.Bucket, value []byte, - operator filters.Operator, keyOnly bool, -) *RowReader { - return &RowReader{ - bucket: bucket, - value: value, - operator: operator, - keyOnly: keyOnly, - } -} - -// ReadFn will be called 1..n times per match. This means it will also be -// called on a non-match, in this case v == nil. -// It is up to the caller to decide if that is an error case or not. -// -// Note that because what we are parsing is an inverted index row, it can -// sometimes become confusing what a key and value actually resembles. The -// variables k and v are the literal row key and value. So this means, the -// data-value as in "less than 17" where 17 would be the "value" is in the key -// variable "k". The value will contain the docCount, hash and list of pointers -// (with optional frequency) to the docIDs -// -// The boolean return argument is a way to stop iteration (e.g. when a limit is -// reached) without producing an error. In normal operation always return true, -// if false is returned once, the loop is broken. -type ReadFn func(k []byte, values [][]byte) (bool, error) - -// Read a row using the specified ReadFn. If RowReader was created with -// keysOnly==true, the values argument in the readFn will always be nil on all -// requests involving cursors -func (rr *RowReader) Read(ctx context.Context, readFn ReadFn) error { - switch rr.operator { - case filters.OperatorEqual: - return rr.equal(ctx, readFn) - case filters.OperatorNotEqual: - return rr.notEqual(ctx, readFn) - case filters.OperatorGreaterThan: - return rr.greaterThan(ctx, readFn, false) - case filters.OperatorGreaterThanEqual: - return rr.greaterThan(ctx, readFn, true) - case filters.OperatorLessThan: - return rr.lessThan(ctx, readFn, false) - case filters.OperatorLessThanEqual: - return rr.lessThan(ctx, readFn, true) - case filters.OperatorLike: - return rr.like(ctx, readFn) - case filters.OperatorIsNull: // we need to fetch a row with a given value (there is only nil and !nil) and can reuse equal to get the correct row - return rr.equal(ctx, readFn) - default: - return fmt.Errorf("operator %v not supported", rr.operator) - } -} - -// equal is a special case, as we don't need to iterate, but just read a single -// row -func (rr *RowReader) equal(ctx context.Context, readFn ReadFn) error { - if err := ctx.Err(); err != nil { - return err - } - - v, err := rr.bucket.SetList(rr.value) - if err != nil { - return err - } - - _, err = readFn(rr.value, v) - return err -} - -// greaterThan reads from the specified value to the end. The first row is only -// included if allowEqual==true, otherwise it starts with the next one -func (rr *RowReader) greaterThan(ctx context.Context, readFn ReadFn, - allowEqual bool, -) error { - c := rr.newCursor() - defer c.Close() - - for k, v := c.Seek(rr.value); k != nil; k, v = c.Next() { - if err := ctx.Err(); err != nil { - return err - } - - if bytes.Equal(k, rr.value) && !allowEqual { - continue - } - - continueReading, err := readFn(k, v) - if err != nil { - return err - } - - if !continueReading { - break - } - } - - return nil -} - -// lessThan reads from the very begging to the specified value. The last -// matching row is only included if allowEqual==true, otherwise it ends one -// prior to that. -func (rr *RowReader) lessThan(ctx context.Context, readFn ReadFn, - allowEqual bool, -) error { - c := rr.newCursor() - defer c.Close() - - for k, v := c.First(); k != nil && bytes.Compare(k, rr.value) != 1; k, v = c.Next() { - if err := ctx.Err(); err != nil { - return err - } - - if bytes.Equal(k, rr.value) && !allowEqual { - continue - } - - continueReading, err := readFn(k, v) - if err != nil { - return err - } - - if !continueReading { - break - } - } - - return nil -} - -// notEqual is another special case, as it's the opposite of equal. So instead -// of reading just one row, we read all but one row. -func (rr *RowReader) notEqual(ctx context.Context, readFn ReadFn) error { - c := rr.newCursor() - defer c.Close() - - for k, v := c.First(); k != nil; k, v = c.Next() { - if err := ctx.Err(); err != nil { - return err - } - - if bytes.Equal(k, rr.value) { - continue - } - - continueReading, err := readFn(k, v) - if err != nil { - return err - } - - if !continueReading { - break - } - } - - return nil -} - -func (rr *RowReader) like(ctx context.Context, readFn ReadFn) error { - like, err := parseLikeRegexp(rr.value) - if err != nil { - return errors.Wrapf(err, "parse like value") - } - - c := rr.newCursor() - defer c.Close() - - var ( - initialK []byte - initialV [][]byte - ) - - if like.optimizable { - initialK, initialV = c.Seek(like.min) - } else { - initialK, initialV = c.First() - } - - for k, v := initialK, initialV; k != nil; k, v = c.Next() { - if err := ctx.Err(); err != nil { - return err - } - - if like.optimizable { - // if the query is optimizable, i.e. it doesn't start with a wildcard, we - // can abort once we've moved past the point where the fixed characters - // no longer match - if len(k) < len(like.min) { - break - } - - if bytes.Compare(like.min, k[:len(like.min)]) == -1 { - break - } - } - - if !like.regexp.Match(k) { - continue - } - - continueReading, err := readFn(k, v) - if err != nil { - return err - } - - if !continueReading { - break - } - } - - return nil -} - -// newCursor will either return a regular cursor - or a key-only cursor if -// keyOnly==true -func (rr *RowReader) newCursor() *lsmkv.CursorSet { - if rr.keyOnly { - return rr.bucket.SetCursorKeyOnly() - } - - return rr.bucket.SetCursor() -} diff --git a/adapters/repos/db/inverted/row_reader_frequency.go b/adapters/repos/db/inverted/row_reader_frequency.go deleted file mode 100644 index bf667527cc6d5b42f7b376dc983d3e8c8541f8d5..0000000000000000000000000000000000000000 --- a/adapters/repos/db/inverted/row_reader_frequency.go +++ /dev/null @@ -1,270 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package inverted - -import ( - "bytes" - "context" - "fmt" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv" - "github.com/weaviate/weaviate/entities/filters" -) - -// RowReaderFrequency reads one or many row(s) depending on the specified operator -type RowReaderFrequency struct { - value []byte - bucket *lsmkv.Bucket - operator filters.Operator - keyOnly bool - shardVersion uint16 -} - -func NewRowReaderFrequency(bucket *lsmkv.Bucket, value []byte, - operator filters.Operator, keyOnly bool, shardVersion uint16, -) *RowReaderFrequency { - return &RowReaderFrequency{ - bucket: bucket, - value: value, - operator: operator, - keyOnly: keyOnly, - shardVersion: shardVersion, - } -} - -// ReadFnFrequency will be called 1..n times per match. This means it will also be -// called on a non-match, in this case v == nil. -// It is up to the caller to decide if that is an error case or not. -// -// Note that because what we are parsing is an inverted index row, it can -// sometimes become confusing what a key and value actually resembles. The -// variables k and v are the literal row key and value. So this means, the -// data-value as in "less than 17" where 17 would be the "value" is in the key -// variable "k". The value will contain the docCount, hash and list of pointers -// (with optional frequency) to the docIDs -// -// The boolean return argument is a way to stop iteration (e.g. when a limit is -// reached) without producing an error. In normal operation always return true, -// if false is returned once, the loop is broken. -type ReadFnFrequency func(k []byte, values []lsmkv.MapPair) (bool, error) - -func (rr *RowReaderFrequency) Read(ctx context.Context, readFn ReadFnFrequency) error { - switch rr.operator { - case filters.OperatorEqual: - return rr.equal(ctx, readFn) - case filters.OperatorNotEqual: - return rr.notEqual(ctx, readFn) - case filters.OperatorGreaterThan: - return rr.greaterThan(ctx, readFn, false) - case filters.OperatorGreaterThanEqual: - return rr.greaterThan(ctx, readFn, true) - case filters.OperatorLessThan: - return rr.lessThan(ctx, readFn, false) - case filters.OperatorLessThanEqual: - return rr.lessThan(ctx, readFn, true) - case filters.OperatorLike: - return rr.like(ctx, readFn) - default: - return fmt.Errorf("operator %v supported", rr.operator) - } -} - -// equal is a special case, as we don't need to iterate, but just read a single -// row -func (rr *RowReaderFrequency) equal(ctx context.Context, readFn ReadFnFrequency) error { - if err := ctx.Err(); err != nil { - return err - } - - var v []lsmkv.MapPair - var err error - if rr.shardVersion < 2 { - v, err = rr.bucket.MapList(rr.value, lsmkv.MapListAcceptDuplicates(), - lsmkv.MapListLegacySortingRequired()) - if err != nil { - return err - } - } else { - v, err = rr.bucket.MapList(rr.value, lsmkv.MapListAcceptDuplicates()) - if err != nil { - return err - } - } - // TODO: don't we need to check here if this is a doc id vs a object search? - // Or is this not a problem because the latter removes duplicates anyway? - - _, err = readFn(rr.value, v) - return err -} - -// greaterThan reads from the specified value to the end. The first row is only -// included if allowEqual==true, otherwise it starts with the next one -func (rr *RowReaderFrequency) greaterThan(ctx context.Context, readFn ReadFnFrequency, - allowEqual bool, -) error { - c := rr.newCursor() - defer c.Close() - - for k, v := c.Seek(rr.value); k != nil; k, v = c.Next() { - if err := ctx.Err(); err != nil { - return err - } - - if bytes.Equal(k, rr.value) && !allowEqual { - continue - } - - continueReading, err := readFn(k, v) - if err != nil { - return err - } - - if !continueReading { - break - } - } - - return nil -} - -// lessThan reads from the very begging to the specified value. The last -// matching row is only included if allowEqual==true, otherwise it ends one -// prior to that. -func (rr *RowReaderFrequency) lessThan(ctx context.Context, readFn ReadFnFrequency, - allowEqual bool, -) error { - c := rr.newCursor() - defer c.Close() - - for k, v := c.First(); k != nil && bytes.Compare(k, rr.value) != 1; k, v = c.Next() { - if err := ctx.Err(); err != nil { - return err - } - - if bytes.Equal(k, rr.value) && !allowEqual { - continue - } - - continueReading, err := readFn(k, v) - if err != nil { - return err - } - - if !continueReading { - break - } - } - - return nil -} - -// notEqual is another special case, as it's the opposite of equal. So instead -// of reading just one row, we read all but one row. -func (rr *RowReaderFrequency) notEqual(ctx context.Context, readFn ReadFnFrequency) error { - c := rr.newCursor() - defer c.Close() - - for k, v := c.First(); k != nil; k, v = c.Next() { - if err := ctx.Err(); err != nil { - return err - } - - if bytes.Equal(k, rr.value) { - continue - } - - continueReading, err := readFn(k, v) - if err != nil { - return err - } - - if !continueReading { - break - } - } - - return nil -} - -func (rr *RowReaderFrequency) like(ctx context.Context, readFn ReadFnFrequency) error { - like, err := parseLikeRegexp(rr.value) - if err != nil { - return errors.Wrapf(err, "parse like value") - } - - // TODO: don't we need to check here if this is a doc id vs a object search? - // Or is this not a problem because the latter removes duplicates anyway? - c := rr.newCursor(lsmkv.MapListAcceptDuplicates()) - defer c.Close() - - var ( - initialK []byte - initialV []lsmkv.MapPair - ) - - if like.optimizable { - initialK, initialV = c.Seek(like.min) - } else { - initialK, initialV = c.First() - } - - for k, v := initialK, initialV; k != nil; k, v = c.Next() { - if err := ctx.Err(); err != nil { - return err - } - - if like.optimizable { - // if the query is optimizable, i.e. it doesn't start with a wildcard, we - // can abort once we've moved past the point where the fixed characters - // no longer match - if len(k) < len(like.min) { - break - } - - if bytes.Compare(like.min, k[:len(like.min)]) == -1 { - break - } - } - - if !like.regexp.Match(k) { - continue - } - - continueReading, err := readFn(k, v) - if err != nil { - return err - } - - if !continueReading { - break - } - } - - return nil -} - -// newCursor will either return a regular cursor - or a key-only cursor if -// keyOnly==true -func (rr *RowReaderFrequency) newCursor( - opts ...lsmkv.MapListOption, -) *lsmkv.CursorMap { - if rr.shardVersion < 2 { - opts = append(opts, lsmkv.MapListLegacySortingRequired()) - } - - if rr.keyOnly { - return rr.bucket.MapCursorKeyOnly(opts...) - } - - return rr.bucket.MapCursor(opts...) -} diff --git a/adapters/repos/db/inverted/row_reader_roaring_set.go b/adapters/repos/db/inverted/row_reader_roaring_set.go deleted file mode 100644 index fc69a27a16935074a46e3c2ff92641cd85b1e254..0000000000000000000000000000000000000000 --- a/adapters/repos/db/inverted/row_reader_roaring_set.go +++ /dev/null @@ -1,246 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package inverted - -import ( - "bytes" - "context" - "fmt" - - "github.com/pkg/errors" - "github.com/weaviate/sroar" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv" - "github.com/weaviate/weaviate/entities/filters" -) - -// RowReaderRoaringSet reads one or many row(s) depending on the specified -// operator -type RowReaderRoaringSet struct { - value []byte - operator filters.Operator - newCursor func() lsmkv.CursorRoaringSet - getter func(key []byte) (*sroar.Bitmap, error) -} - -// If keyOnly is set, the RowReaderRoaringSet will request key-only cursors -// wherever cursors are used, the specified value arguments in the -// RoaringSetReadFn will always be empty -func NewRowReaderRoaringSet(bucket *lsmkv.Bucket, value []byte, - operator filters.Operator, keyOnly bool, -) *RowReaderRoaringSet { - getter := bucket.RoaringSetGet - newCursor := bucket.CursorRoaringSet - if keyOnly { - newCursor = bucket.CursorRoaringSetKeyOnly - } - - return &RowReaderRoaringSet{ - value: value, - operator: operator, - newCursor: newCursor, - getter: getter, - } -} - -// RoaringSetReadFn will be called 1..n times per match. This means it will also -// be called on a non-match, in this case v == empty bitmap. -// It is up to the caller to decide if that is an error case or not. -// -// Note that because what we are parsing is an inverted index row, it can -// sometimes become confusing what a key and value actually resembles. The -// variables k and v are the literal row key and value. So this means, the -// data-value as in "less than 17" where 17 would be the "value" is in the key -// variable "k". The value will contain bitmap with docIDs having value "k" -// -// The boolean return argument is a way to stop iteration (e.g. when a limit is -// reached) without producing an error. In normal operation always return true, -// if false is returned once, the loop is broken. -type RoaringSetReadFn func(k []byte, v *sroar.Bitmap) (bool, error) - -// Read a row using the specified ReadFn. If RowReader was created with -// keysOnly==true, the values argument in the readFn will always be nil on all -// requests involving cursors -func (rr *RowReaderRoaringSet) Read(ctx context.Context, readFn RoaringSetReadFn) error { - switch rr.operator { - case filters.OperatorEqual, filters.OperatorIsNull: - return rr.equal(ctx, readFn) - case filters.OperatorNotEqual: - return rr.notEqual(ctx, readFn) - case filters.OperatorGreaterThan: - return rr.greaterThan(ctx, readFn, false) - case filters.OperatorGreaterThanEqual: - return rr.greaterThan(ctx, readFn, true) - case filters.OperatorLessThan: - return rr.lessThan(ctx, readFn, false) - case filters.OperatorLessThanEqual: - return rr.lessThan(ctx, readFn, true) - case filters.OperatorLike: - return rr.like(ctx, readFn) - default: - return fmt.Errorf("operator %v not supported", rr.operator) - } -} - -// equal is a special case, as we don't need to iterate, but just read a single -// row -func (rr *RowReaderRoaringSet) equal(ctx context.Context, - readFn RoaringSetReadFn, -) error { - if err := ctx.Err(); err != nil { - return err - } - - v, err := rr.getter(rr.value) - if err != nil { - return err - } - - _, err = readFn(rr.value, v) - return err -} - -// greaterThan reads from the specified value to the end. The first row is only -// included if allowEqual==true, otherwise it starts with the next one -func (rr *RowReaderRoaringSet) greaterThan(ctx context.Context, - readFn RoaringSetReadFn, allowEqual bool, -) error { - c := rr.newCursor() - defer c.Close() - - for k, v := c.Seek(rr.value); k != nil; k, v = c.Next() { - if err := ctx.Err(); err != nil { - return err - } - - if bytes.Equal(k, rr.value) && !allowEqual { - continue - } - - if continueReading, err := readFn(k, v); err != nil { - return err - } else if !continueReading { - break - } - } - - return nil -} - -// lessThan reads from the very begging to the specified value. The last -// matching row is only included if allowEqual==true, otherwise it ends one -// prior to that. -func (rr *RowReaderRoaringSet) lessThan(ctx context.Context, - readFn RoaringSetReadFn, allowEqual bool, -) error { - c := rr.newCursor() - defer c.Close() - - for k, v := c.First(); k != nil && bytes.Compare(k, rr.value) < 1; k, v = c.Next() { - if err := ctx.Err(); err != nil { - return err - } - - if bytes.Equal(k, rr.value) && !allowEqual { - continue - } - - if continueReading, err := readFn(k, v); err != nil { - return err - } else if !continueReading { - break - } - } - - return nil -} - -// notEqual is another special case, as it's the opposite of equal. So instead -// of reading just one row, we read all but one row. -func (rr *RowReaderRoaringSet) notEqual(ctx context.Context, - readFn RoaringSetReadFn, -) error { - c := rr.newCursor() - defer c.Close() - - for k, v := c.First(); k != nil; k, v = c.Next() { - if err := ctx.Err(); err != nil { - return err - } - - if bytes.Equal(k, rr.value) { - continue - } - - if continueReading, err := readFn(k, v); err != nil { - return err - } else if !continueReading { - break - } - } - - return nil -} - -func (rr *RowReaderRoaringSet) like(ctx context.Context, - readFn RoaringSetReadFn, -) error { - like, err := parseLikeRegexp(rr.value) - if err != nil { - return errors.Wrapf(err, "parse like value") - } - - c := rr.newCursor() - defer c.Close() - - var ( - initialK []byte - initialV *sroar.Bitmap - likeMinLen int - ) - - if like.optimizable { - initialK, initialV = c.Seek(like.min) - likeMinLen = len(like.min) - } else { - initialK, initialV = c.First() - } - - for k, v := initialK, initialV; k != nil; k, v = c.Next() { - if err := ctx.Err(); err != nil { - return err - } - - if like.optimizable { - // if the query is optimizable, i.e. it doesn't start with a wildcard, we - // can abort once we've moved past the point where the fixed characters - // no longer match - if len(k) < likeMinLen { - break - } - if bytes.Compare(like.min, k[:likeMinLen]) == -1 { - break - } - } - - if !like.regexp.Match(k) { - continue - } - - if continueReading, err := readFn(k, v); err != nil { - return err - } else if !continueReading { - break - } - } - - return nil -} diff --git a/adapters/repos/db/inverted/row_reader_roaring_set_test.go b/adapters/repos/db/inverted/row_reader_roaring_set_test.go deleted file mode 100644 index b02bf6b6665df0639458eed82fecbf8261166615..0000000000000000000000000000000000000000 --- a/adapters/repos/db/inverted/row_reader_roaring_set_test.go +++ /dev/null @@ -1,304 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package inverted - -import ( - "bytes" - "context" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/sroar" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv/roaringset" - "github.com/weaviate/weaviate/entities/filters" - entlsmkv "github.com/weaviate/weaviate/entities/lsmkv" -) - -func TestRowReaderRoaringSet(t *testing.T) { - data := []kvData{ - {"aaa", []uint64{1, 2, 3}}, - {"bbb", []uint64{11, 22, 33}}, - {"ccc", []uint64{111, 222, 333}}, - {"ddd", []uint64{1111, 2222, 3333}}, - {"eee", []uint64{11111, 22222, 33333}}, - {"fff", []uint64{111111, 222222, 333333}}, - {"ggg", []uint64{1111111, 2222222, 3333333}}, - {"hhh", []uint64{11111111, 2222222, 33333333}}, - } - ctx := context.Background() - - testcases := []struct { - name string - value string - operator filters.Operator - expected []kvData - }{ - { - name: "equal 'ggg' value", - value: "ggg", - operator: filters.OperatorEqual, - expected: []kvData{ - {"ggg", []uint64{1111111, 2222222, 3333333}}, - }, - }, - { - name: "not equal 'ccc' value", - value: "ccc", - operator: filters.OperatorNotEqual, - expected: []kvData{ - {"aaa", []uint64{1, 2, 3}}, - {"bbb", []uint64{11, 22, 33}}, - {"ddd", []uint64{1111, 2222, 3333}}, - {"eee", []uint64{11111, 22222, 33333}}, - {"fff", []uint64{111111, 222222, 333333}}, - {"ggg", []uint64{1111111, 2222222, 3333333}}, - {"hhh", []uint64{11111111, 2222222, 33333333}}, - }, - }, - { - name: "not equal non-matching value", - value: "fgh", - operator: filters.OperatorNotEqual, - expected: []kvData{ - {"aaa", []uint64{1, 2, 3}}, - {"bbb", []uint64{11, 22, 33}}, - {"ccc", []uint64{111, 222, 333}}, - {"ddd", []uint64{1111, 2222, 3333}}, - {"eee", []uint64{11111, 22222, 33333}}, - {"fff", []uint64{111111, 222222, 333333}}, - {"ggg", []uint64{1111111, 2222222, 3333333}}, - {"hhh", []uint64{11111111, 2222222, 33333333}}, - }, - }, - { - name: "greater than 'ddd' value", - value: "ddd", - operator: filters.OperatorGreaterThan, - expected: []kvData{ - {"eee", []uint64{11111, 22222, 33333}}, - {"fff", []uint64{111111, 222222, 333333}}, - {"ggg", []uint64{1111111, 2222222, 3333333}}, - {"hhh", []uint64{11111111, 2222222, 33333333}}, - }, - }, - { - name: "greater than equal 'ddd' value", - value: "ddd", - operator: filters.OperatorGreaterThanEqual, - expected: []kvData{ - {"ddd", []uint64{1111, 2222, 3333}}, - {"eee", []uint64{11111, 22222, 33333}}, - {"fff", []uint64{111111, 222222, 333333}}, - {"ggg", []uint64{1111111, 2222222, 3333333}}, - {"hhh", []uint64{11111111, 2222222, 33333333}}, - }, - }, - { - name: "greater than non-matching value", - value: "fgh", - operator: filters.OperatorGreaterThan, - expected: []kvData{ - {"ggg", []uint64{1111111, 2222222, 3333333}}, - {"hhh", []uint64{11111111, 2222222, 33333333}}, - }, - }, - { - name: "greater than equal non-matching value", - value: "fgh", - operator: filters.OperatorGreaterThanEqual, - expected: []kvData{ - {"ggg", []uint64{1111111, 2222222, 3333333}}, - {"hhh", []uint64{11111111, 2222222, 33333333}}, - }, - }, - { - name: "less than 'eee' value", - value: "eee", - operator: filters.OperatorLessThan, - expected: []kvData{ - {"aaa", []uint64{1, 2, 3}}, - {"bbb", []uint64{11, 22, 33}}, - {"ccc", []uint64{111, 222, 333}}, - {"ddd", []uint64{1111, 2222, 3333}}, - }, - }, - { - name: "less than equal 'eee' value", - value: "eee", - operator: filters.OperatorLessThanEqual, - expected: []kvData{ - {"aaa", []uint64{1, 2, 3}}, - {"bbb", []uint64{11, 22, 33}}, - {"ccc", []uint64{111, 222, 333}}, - {"ddd", []uint64{1111, 2222, 3333}}, - {"eee", []uint64{11111, 22222, 33333}}, - }, - }, - { - name: "less than non-matching value", - value: "fgh", - operator: filters.OperatorLessThan, - expected: []kvData{ - {"aaa", []uint64{1, 2, 3}}, - {"bbb", []uint64{11, 22, 33}}, - {"ccc", []uint64{111, 222, 333}}, - {"ddd", []uint64{1111, 2222, 3333}}, - {"eee", []uint64{11111, 22222, 33333}}, - {"fff", []uint64{111111, 222222, 333333}}, - }, - }, - { - name: "less than equal non-matching value", - value: "fgh", - operator: filters.OperatorLessThanEqual, - expected: []kvData{ - {"aaa", []uint64{1, 2, 3}}, - {"bbb", []uint64{11, 22, 33}}, - {"ccc", []uint64{111, 222, 333}}, - {"ddd", []uint64{1111, 2222, 3333}}, - {"eee", []uint64{11111, 22222, 33333}}, - {"fff", []uint64{111111, 222222, 333333}}, - }, - }, - { - name: "like '*b' value", - value: "*b", - operator: filters.OperatorLike, - expected: []kvData{ - {"bbb", []uint64{11, 22, 33}}, - }, - }, - { - name: "like 'h*' value", - value: "h*", - operator: filters.OperatorLike, - expected: []kvData{ - {"hhh", []uint64{11111111, 2222222, 33333333}}, - }, - }, - } - - for _, tc := range testcases { - type readResult struct { - k []byte - v *sroar.Bitmap - } - - t.Run(tc.name, func(t *testing.T) { - result := []readResult{} - rowReader := createRowReaderRoaringSet([]byte(tc.value), tc.operator, data) - rowReader.Read(ctx, func(k []byte, v *sroar.Bitmap) (bool, error) { - result = append(result, readResult{k, v}) - return true, nil - }) - - assert.Len(t, result, len(tc.expected)) - for i, expectedKV := range tc.expected { - assert.Equal(t, []byte(expectedKV.k), result[i].k) - assert.Equal(t, len(expectedKV.v), result[i].v.GetCardinality()) - for _, expectedV := range expectedKV.v { - assert.True(t, result[i].v.Contains(expectedV)) - } - } - }) - - t.Run(tc.name+" with 3 results limit", func(t *testing.T) { - limit := 3 - expected := tc.expected - if len(tc.expected) > limit { - expected = tc.expected[:limit] - } - - result := []readResult{} - rowReader := createRowReaderRoaringSet([]byte(tc.value), tc.operator, data) - rowReader.Read(ctx, func(k []byte, v *sroar.Bitmap) (bool, error) { - result = append(result, readResult{k, v}) - if len(result) >= limit { - return false, nil - } - return true, nil - }) - - assert.Len(t, result, len(expected)) - for i, expectedKV := range expected { - assert.Equal(t, []byte(expectedKV.k), result[i].k) - assert.Equal(t, len(expectedKV.v), result[i].v.GetCardinality()) - for _, expectedV := range expectedKV.v { - assert.True(t, result[i].v.Contains(expectedV)) - } - } - }) - } -} - -type kvData struct { - k string - v []uint64 -} - -type dummyCursorRoaringSet struct { - data []kvData - pos int - closed bool -} - -func (c *dummyCursorRoaringSet) First() ([]byte, *sroar.Bitmap) { - c.pos = 0 - return c.Next() -} - -func (c *dummyCursorRoaringSet) Next() ([]byte, *sroar.Bitmap) { - bm := sroar.NewBitmap() - if c.pos >= len(c.data) { - return nil, bm - } - pos := c.pos - c.pos++ - bm.SetMany(c.data[pos].v) - return []byte(c.data[pos].k), bm -} - -func (c *dummyCursorRoaringSet) Seek(key []byte) ([]byte, *sroar.Bitmap) { - pos := -1 - for i := 0; i < len(c.data); i++ { - if bytes.Compare([]byte(c.data[i].k), key) >= 0 { - pos = i - break - } - } - if pos < 0 { - return nil, sroar.NewBitmap() - } - c.pos = pos - return c.Next() -} - -func (c *dummyCursorRoaringSet) Close() { - c.closed = true -} - -func createRowReaderRoaringSet(value []byte, operator filters.Operator, data []kvData) *RowReaderRoaringSet { - return &RowReaderRoaringSet{ - value: value, - operator: operator, - newCursor: func() lsmkv.CursorRoaringSet { return &dummyCursorRoaringSet{data: data} }, - getter: func(key []byte) (*sroar.Bitmap, error) { - for i := 0; i < len(data); i++ { - if bytes.Equal([]byte(data[i].k), key) { - return roaringset.NewBitmap(data[i].v...), nil - } - } - return nil, entlsmkv.NotFound - }, - } -} diff --git a/adapters/repos/db/inverted/searcher.go b/adapters/repos/db/inverted/searcher.go deleted file mode 100644 index a6635ebaa4209e5cd257bb923ba9fc381d7cdde3..0000000000000000000000000000000000000000 --- a/adapters/repos/db/inverted/searcher.go +++ /dev/null @@ -1,883 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package inverted - -import ( - "context" - "encoding/binary" - "fmt" - "strconv" - "time" - - "github.com/google/uuid" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/sroar" - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/adapters/repos/db/inverted/stopwords" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv" - "github.com/weaviate/weaviate/adapters/repos/db/propertyspecific" - "github.com/weaviate/weaviate/adapters/repos/db/sorter" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/inverted" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/storobj" - "golang.org/x/sync/errgroup" -) - -type Searcher struct { - logger logrus.FieldLogger - store *lsmkv.Store - schema schema.Schema - classSearcher ClassSearcher // to allow recursive searches on ref-props - propIndices propertyspecific.Indices - stopwords stopwords.StopwordDetector - shardVersion uint16 - isFallbackToSearchable IsFallbackToSearchable - tenant string - // nestedCrossRefLimit limits the number of nested cross refs returned for a query - nestedCrossRefLimit int64 -} - -func NewSearcher(logger logrus.FieldLogger, store *lsmkv.Store, - schema schema.Schema, propIndices propertyspecific.Indices, - classSearcher ClassSearcher, stopwords stopwords.StopwordDetector, - shardVersion uint16, isFallbackToSearchable IsFallbackToSearchable, - tenant string, nestedCrossRefLimit int64, -) *Searcher { - return &Searcher{ - logger: logger, - store: store, - schema: schema, - propIndices: propIndices, - classSearcher: classSearcher, - stopwords: stopwords, - shardVersion: shardVersion, - isFallbackToSearchable: isFallbackToSearchable, - tenant: tenant, - nestedCrossRefLimit: nestedCrossRefLimit, - } -} - -// Objects returns a list of full objects -func (s *Searcher) Objects(ctx context.Context, limit int, - filter *filters.LocalFilter, sort []filters.Sort, additional additional.Properties, - className schema.ClassName, -) ([]*storobj.Object, error) { - allowList, err := s.docIDs(ctx, filter, additional, className, limit) - if err != nil { - return nil, err - } - - var it docIDsIterator - if len(sort) > 0 { - docIDs, err := s.sort(ctx, limit, sort, allowList, additional, className) - if err != nil { - return nil, errors.Wrap(err, "sort doc ids") - } - it = newSliceDocIDsIterator(docIDs) - } else { - it = allowList.LimitedIterator(limit) - } - - return s.objectsByDocID(it, additional) -} - -func (s *Searcher) sort(ctx context.Context, limit int, sort []filters.Sort, docIDs helpers.AllowList, - additional additional.Properties, className schema.ClassName, -) ([]uint64, error) { - lsmSorter, err := sorter.NewLSMSorter(s.store, s.schema, className) - if err != nil { - return nil, err - } - return lsmSorter.SortDocIDs(ctx, limit, sort, docIDs) -} - -func (s *Searcher) objectsByDocID(it docIDsIterator, - additional additional.Properties, -) ([]*storobj.Object, error) { - bucket := s.store.Bucket(helpers.ObjectsBucketLSM) - if bucket == nil { - return nil, errors.Errorf("objects bucket not found") - } - - out := make([]*storobj.Object, it.Len()) - docIDBytes := make([]byte, 8) - - i := 0 - for docID, ok := it.Next(); ok; docID, ok = it.Next() { - binary.LittleEndian.PutUint64(docIDBytes, docID) - res, err := bucket.GetBySecondary(0, docIDBytes) - if err != nil { - return nil, err - } - - if res == nil { - continue - } - - var unmarshalled *storobj.Object - if additional.ReferenceQuery { - unmarshalled, err = storobj.FromBinaryUUIDOnly(res) - } else { - unmarshalled, err = storobj.FromBinaryOptional(res, additional) - } - if err != nil { - return nil, errors.Wrapf(err, "unmarshal data object at position %d", i) - } - - out[i] = unmarshalled - i++ - } - - return out[:i], nil -} - -// DocIDs is similar to Objects, but does not actually resolve the docIDs to -// full objects. Instead it returns the pure object id pointers. They can then -// be used in a secondary index (e.g. vector index) -// -// DocID queries does not contain a limit by design, as we won't know if the limit -// wouldn't remove the item that is most important for the follow up query. -// Imagine the user sets the limit to 1 and the follow-up is a vector search. -// If we already limited the allowList to 1, the vector search would be -// pointless, as only the first element would be allowed, regardless of which -// had the shortest distance -func (s *Searcher) DocIDs(ctx context.Context, filter *filters.LocalFilter, - additional additional.Properties, className schema.ClassName, -) (helpers.AllowList, error) { - return s.docIDs(ctx, filter, additional, className, 0) -} - -func (s *Searcher) docIDs(ctx context.Context, filter *filters.LocalFilter, - additional additional.Properties, className schema.ClassName, - limit int, -) (helpers.AllowList, error) { - pv, err := s.extractPropValuePair(filter.Root, className) - if err != nil { - return nil, err - } - - if err := pv.fetchDocIDs(s, limit); err != nil { - return nil, errors.Wrap(err, "fetch doc ids for prop/value pair") - } - - dbm, err := pv.mergeDocIDs() - if err != nil { - return nil, errors.Wrap(err, "merge doc ids by operator") - } - - return helpers.NewAllowListFromBitmap(dbm.docIDs), nil -} - -func (s *Searcher) extractPropValuePair(filter *filters.Clause, - className schema.ClassName, -) (*propValuePair, error) { - class := s.schema.FindClassByName(schema.ClassName(className)) - if class == nil { - return nil, fmt.Errorf("class %q not found", className) - } - out, err := newPropValuePair(class) - if err != nil { - return nil, errors.Wrap(err, "new prop value pair") - } - if filter.Operands != nil { - // nested filter - children, err := s.extractPropValuePairs(filter.Operands, className) - if err != nil { - return nil, err - } - out.children = children - out.operator = filter.Operator - return out, nil - } - - if filter.Operator == filters.ContainsAny || filter.Operator == filters.ContainsAll { - return s.extractContains(filter.On, filter.Value.Type, filter.Value.Value, filter.Operator, class) - } - - // on value or non-nested filter - props := filter.On.Slice() - propName := props[0] - - if s.onInternalProp(propName) { - return s.extractInternalProp(propName, filter.Value.Type, filter.Value.Value, filter.Operator, class) - } - - if extractedPropName, ok := schema.IsPropertyLength(propName, 0); ok { - property, err := s.schema.GetProperty(className, schema.PropertyName(extractedPropName)) - if err != nil { - return nil, err - } - return s.extractPropertyLength(property, filter.Value.Type, filter.Value.Value, filter.Operator, class) - } - - property, err := s.schema.GetProperty(className, schema.PropertyName(propName)) - if err != nil { - return nil, err - } - - if s.onRefProp(property) && len(props) != 1 { - return s.extractReferenceFilter(property, filter, class) - } - - if s.onRefProp(property) && filter.Value.Type == schema.DataTypeInt { - // ref prop and int type is a special case, the user is looking for the - // reference count as opposed to the content - return s.extractReferenceCount(property, filter.Value.Value, filter.Operator, class) - } - - if filter.Operator == filters.OperatorIsNull { - return s.extractPropertyNull(property, filter.Value.Type, filter.Value.Value, filter.Operator, class) - } - - if s.onGeoProp(property) { - return s.extractGeoFilter(property, filter.Value.Value, filter.Value.Type, filter.Operator, class) - } - - if s.onUUIDProp(property) { - return s.extractUUIDFilter(property, filter.Value.Value, filter.Value.Type, filter.Operator, class) - } - - if s.onTokenizableProp(property) { - return s.extractTokenizableProp(property, filter.Value.Type, filter.Value.Value, filter.Operator, class) - } - - return s.extractPrimitiveProp(property, filter.Value.Type, filter.Value.Value, filter.Operator, class) -} - -func (s *Searcher) extractPropValuePairs(operands []filters.Clause, className schema.ClassName) ([]*propValuePair, error) { - children := make([]*propValuePair, len(operands)) - eg := errgroup.Group{} - // prevent unbounded concurrency, see - // https://github.com/weaviate/weaviate/issues/3179 for details - eg.SetLimit(2 * _NUMCPU) - - for i, clause := range operands { - i, clause := i, clause - eg.Go(func() error { - child, err := s.extractPropValuePair(&clause, className) - if err != nil { - return errors.Wrapf(err, "nested clause at pos %d", i) - } - children[i] = child - - return nil - }) - } - if err := eg.Wait(); err != nil { - return nil, fmt.Errorf("nested query: %w", err) - } - return children, nil -} - -func (s *Searcher) extractReferenceFilter(prop *models.Property, - filter *filters.Clause, class *models.Class, -) (*propValuePair, error) { - ctx := context.TODO() - return newRefFilterExtractor(s.logger, s.classSearcher, filter, class, prop, s.tenant, s.nestedCrossRefLimit). - Do(ctx) -} - -func (s *Searcher) extractPrimitiveProp(prop *models.Property, propType schema.DataType, - value interface{}, operator filters.Operator, class *models.Class, -) (*propValuePair, error) { - var extractValueFn func(in interface{}) ([]byte, error) - switch propType { - case schema.DataTypeBoolean: - extractValueFn = s.extractBoolValue - case schema.DataTypeInt: - extractValueFn = s.extractIntValue - case schema.DataTypeNumber: - extractValueFn = s.extractNumberValue - case schema.DataTypeDate: - extractValueFn = s.extractDateValue - case "": - return nil, fmt.Errorf("data type cannot be empty") - default: - return nil, fmt.Errorf("data type %q not supported in query", propType) - } - - byteValue, err := extractValueFn(value) - if err != nil { - return nil, err - } - - hasFilterableIndex := HasFilterableIndex(prop) - hasSearchableIndex := HasSearchableIndex(prop) - - if !hasFilterableIndex && !hasSearchableIndex { - return nil, inverted.NewMissingFilterableIndexError(prop.Name) - } - - return &propValuePair{ - value: byteValue, - prop: prop.Name, - operator: operator, - hasFilterableIndex: hasFilterableIndex, - hasSearchableIndex: hasSearchableIndex, - Class: class, - }, nil -} - -func (s *Searcher) extractReferenceCount(prop *models.Property, value interface{}, - operator filters.Operator, class *models.Class, -) (*propValuePair, error) { - byteValue, err := s.extractIntCountValue(value) - if err != nil { - return nil, err - } - - hasFilterableIndex := HasFilterableIndexMetaCount && HasInvertedIndex(prop) - hasSearchableIndex := HasSearchableIndexMetaCount && HasInvertedIndex(prop) - - if !hasFilterableIndex && !hasSearchableIndex { - return nil, inverted.NewMissingFilterableMetaCountIndexError(prop.Name) - } - - return &propValuePair{ - value: byteValue, - prop: helpers.MetaCountProp(prop.Name), - operator: operator, - hasFilterableIndex: hasFilterableIndex, - hasSearchableIndex: hasSearchableIndex, - Class: class, - }, nil -} - -func (s *Searcher) extractGeoFilter(prop *models.Property, value interface{}, - valueType schema.DataType, operator filters.Operator, class *models.Class, -) (*propValuePair, error) { - if valueType != schema.DataTypeGeoCoordinates { - return nil, fmt.Errorf("prop %q is of type geoCoordinates, it can only"+ - "be used with geoRange filters", prop.Name) - } - - parsed := value.(filters.GeoRange) - - return &propValuePair{ - value: nil, // not going to be served by an inverted index - valueGeoRange: &parsed, - prop: prop.Name, - operator: operator, - hasFilterableIndex: HasFilterableIndex(prop), - hasSearchableIndex: HasSearchableIndex(prop), - Class: class, - }, nil -} - -func (s *Searcher) extractUUIDFilter(prop *models.Property, value interface{}, - valueType schema.DataType, operator filters.Operator, class *models.Class, -) (*propValuePair, error) { - var byteValue []byte - - switch valueType { - case schema.DataTypeText: - asStr, ok := value.(string) - if !ok { - return nil, fmt.Errorf("expected to see uuid as string in filter, got %T", value) - } - parsed, err := uuid.Parse(asStr) - if err != nil { - return nil, fmt.Errorf("parse uuid string: %w", err) - } - byteValue = parsed[:] - default: - return nil, fmt.Errorf("prop %q is of type uuid, the uuid to filter "+ - "on must be specified as a string (e.g. valueText:)", prop.Name) - } - - hasFilterableIndex := HasFilterableIndex(prop) - hasSearchableIndex := HasSearchableIndex(prop) - - if !hasFilterableIndex && !hasSearchableIndex { - return nil, inverted.NewMissingFilterableIndexError(prop.Name) - } - - return &propValuePair{ - value: byteValue, - prop: prop.Name, - operator: operator, - hasFilterableIndex: hasFilterableIndex, - hasSearchableIndex: hasSearchableIndex, - Class: class, - }, nil -} - -func (s *Searcher) extractInternalProp(propName string, propType schema.DataType, value interface{}, - operator filters.Operator, class *models.Class, -) (*propValuePair, error) { - switch propName { - case filters.InternalPropBackwardsCompatID, filters.InternalPropID: - return s.extractIDProp(propName, propType, value, operator, class) - case filters.InternalPropCreationTimeUnix, filters.InternalPropLastUpdateTimeUnix: - return s.extractTimestampProp(propName, propType, value, operator, class) - default: - return nil, fmt.Errorf( - "failed to extract internal prop, unsupported internal prop '%s'", propName) - } -} - -func (s *Searcher) extractIDProp(propName string, propType schema.DataType, - value interface{}, operator filters.Operator, class *models.Class, -) (*propValuePair, error) { - var byteValue []byte - - switch propType { - case schema.DataTypeText: - v, ok := value.(string) - if !ok { - return nil, fmt.Errorf("expected value to be string, got '%T'", value) - } - byteValue = []byte(v) - default: - return nil, fmt.Errorf( - "failed to extract id prop, unsupported type '%T' for prop '%s'", propType, propName) - } - - return &propValuePair{ - value: byteValue, - prop: filters.InternalPropID, - operator: operator, - hasFilterableIndex: HasFilterableIndexIdProp, - hasSearchableIndex: HasSearchableIndexIdProp, - Class: class, - }, nil -} - -func (s *Searcher) extractTimestampProp(propName string, propType schema.DataType, value interface{}, - operator filters.Operator, class *models.Class, -) (*propValuePair, error) { - var byteValue []byte - - switch propType { - case schema.DataTypeText: - v, ok := value.(string) - if !ok { - return nil, fmt.Errorf("expected value to be string, got '%T'", value) - } - _, err := strconv.ParseInt(v, 10, 64) - if err != nil { - return nil, fmt.Errorf("expected value to be timestamp, got '%s'", v) - } - byteValue = []byte(v) - case schema.DataTypeDate: - v, ok := value.(string) - if !ok { - return nil, fmt.Errorf("expected value to be string, got '%T'", value) - } - t, err := time.Parse(time.RFC3339, v) - if err != nil { - return nil, errors.Wrap(err, "trying parse time as RFC3339 string") - } - - // if propType is a `valueDate`, we need to convert - // it to ms before fetching. this is the format by - // which our timestamps are indexed - byteValue = []byte(strconv.FormatInt(t.UnixMilli(), 10)) - default: - return nil, fmt.Errorf( - "failed to extract timestamp prop, unsupported type '%T' for prop '%s'", propType, propName) - } - - return &propValuePair{ - value: byteValue, - prop: propName, - operator: operator, - hasFilterableIndex: HasFilterableIndexTimestampProp, // TODO text_rbm_inverted_index & with settings - hasSearchableIndex: HasSearchableIndexTimestampProp, // TODO text_rbm_inverted_index & with settings - Class: class, - }, nil -} - -func (s *Searcher) extractTokenizableProp(prop *models.Property, propType schema.DataType, - value interface{}, operator filters.Operator, class *models.Class, -) (*propValuePair, error) { - var terms []string - - valueString, ok := value.(string) - if !ok { - return nil, fmt.Errorf("expected value to be string, got '%T'", value) - } - - switch propType { - case schema.DataTypeText: - // if the operator is like, we cannot apply the regular text-splitting - // logic as it would remove all wildcard symbols - if operator == filters.OperatorLike { - terms = helpers.TokenizeWithWildcards(prop.Tokenization, valueString) - } else { - terms = helpers.Tokenize(prop.Tokenization, valueString) - } - default: - return nil, fmt.Errorf("expected value type to be text, got %v", propType) - } - - hasFilterableIndex := HasFilterableIndex(prop) && !s.isFallbackToSearchable() - hasSearchableIndex := HasSearchableIndex(prop) - - if !hasFilterableIndex && !hasSearchableIndex { - return nil, inverted.NewMissingFilterableIndexError(prop.Name) - } - - propValuePairs := make([]*propValuePair, 0, len(terms)) - for _, term := range terms { - if s.stopwords.IsStopword(term) { - continue - } - propValuePairs = append(propValuePairs, &propValuePair{ - value: []byte(term), - prop: prop.Name, - operator: operator, - hasFilterableIndex: hasFilterableIndex, - hasSearchableIndex: hasSearchableIndex, - Class: class, - }) - } - - if len(propValuePairs) > 1 { - return &propValuePair{operator: filters.OperatorAnd, children: propValuePairs, Class: class}, nil - } - if len(propValuePairs) == 1 { - return propValuePairs[0], nil - } - return nil, errors.Errorf("invalid search term, only stopwords provided. Stopwords can be configured in class.invertedIndexConfig.stopwords") -} - -func (s *Searcher) extractPropertyLength(prop *models.Property, propType schema.DataType, - value interface{}, operator filters.Operator, class *models.Class, -) (*propValuePair, error) { - var byteValue []byte - - switch propType { - case schema.DataTypeInt: - b, err := s.extractIntValue(value) - if err != nil { - return nil, err - } - byteValue = b - default: - return nil, fmt.Errorf( - "failed to extract length of prop, unsupported type '%T' for length of prop '%s'", propType, prop.Name) - } - - return &propValuePair{ - value: byteValue, - prop: helpers.PropLength(prop.Name), - operator: operator, - hasFilterableIndex: HasFilterableIndexPropLength, // TODO text_rbm_inverted_index & with settings - hasSearchableIndex: HasSearchableIndexPropLength, // TODO text_rbm_inverted_index & with settings - Class: class, - }, nil -} - -func (s *Searcher) extractPropertyNull(prop *models.Property, propType schema.DataType, - value interface{}, operator filters.Operator, class *models.Class, -) (*propValuePair, error) { - var valResult []byte - - switch propType { - case schema.DataTypeBoolean: - b, err := s.extractBoolValue(value) - if err != nil { - return nil, err - } - valResult = b - default: - return nil, fmt.Errorf( - "failed to extract null prop, unsupported type '%T' for null prop '%s'", propType, prop.Name) - } - - return &propValuePair{ - value: valResult, - prop: helpers.PropNull(prop.Name), - operator: operator, - hasFilterableIndex: HasFilterableIndexPropNull, // TODO text_rbm_inverted_index & with settings - hasSearchableIndex: HasSearchableIndexPropNull, // TODO text_rbm_inverted_index & with settings - Class: class, - }, nil -} - -func (s *Searcher) extractContains(path *filters.Path, propType schema.DataType, value interface{}, - operator filters.Operator, class *models.Class, -) (*propValuePair, error) { - var operands []filters.Clause - switch propType { - case schema.DataTypeText, schema.DataTypeTextArray: - valueStringArray, err := s.extractStringArray(value) - if err != nil { - return nil, err - } - operands = getContainsOperands(propType, path, valueStringArray) - case schema.DataTypeInt, schema.DataTypeIntArray: - valueIntArray, err := s.extractIntArray(value) - if err != nil { - return nil, err - } - operands = getContainsOperands(propType, path, valueIntArray) - case schema.DataTypeNumber, schema.DataTypeNumberArray: - valueFloat64Array, err := s.extractFloat64Array(value) - if err != nil { - return nil, err - } - operands = getContainsOperands(propType, path, valueFloat64Array) - case schema.DataTypeBoolean, schema.DataTypeBooleanArray: - valueBooleanArray, err := s.extractBoolArray(value) - if err != nil { - return nil, err - } - operands = getContainsOperands(propType, path, valueBooleanArray) - case schema.DataTypeDate, schema.DataTypeDateArray: - valueDateArray, err := s.extractStringArray(value) - if err != nil { - return nil, err - } - operands = getContainsOperands(propType, path, valueDateArray) - default: - return nil, fmt.Errorf("unsupported type '%T' for '%v' operator", propType, operator) - } - - children, err := s.extractPropValuePairs(operands, schema.ClassName(class.Class)) - if err != nil { - return nil, err - } - out, err := newPropValuePair(class) - if err != nil { - return nil, errors.Wrap(err, "new prop value pair") - } - out.children = children - // filters.ContainsAny - out.operator = filters.OperatorOr - if operator == filters.ContainsAll { - out.operator = filters.OperatorAnd - } - out.Class = class - return out, nil -} - -// TODO: repeated calls to on... aren't too efficient because we iterate over -// the schema each time, might be smarter to have a single method that -// determines the type and then we switch based on the result. However, the -// effect of that should be very small unless the schema is absolutely massive. -func (s *Searcher) onRefProp(property *models.Property) bool { - return schema.IsRefDataType(property.DataType) -} - -// TODO: repeated calls to on... aren't too efficient because we iterate over -// the schema each time, might be smarter to have a single method that -// determines the type and then we switch based on the result. However, the -// effect of that should be very small unless the schema is absolutely massive. -func (s *Searcher) onGeoProp(prop *models.Property) bool { - return schema.DataType(prop.DataType[0]) == schema.DataTypeGeoCoordinates -} - -// Note: A UUID prop is a user-specified prop of type UUID. This has nothing to -// do with the primary ID of an object which happens to always be a UUID in -// Weaviate v1 -// -// TODO: repeated calls to on... aren't too efficient because we iterate over -// the schema each time, might be smarter to have a single method that -// determines the type and then we switch based on the result. However, the -// effect of that should be very small unless the schema is absolutely massive. -func (s *Searcher) onUUIDProp(prop *models.Property) bool { - switch dt, _ := schema.AsPrimitive(prop.DataType); dt { - case schema.DataTypeUUID, schema.DataTypeUUIDArray: - return true - default: - return false - } -} - -func (s *Searcher) onInternalProp(propName string) bool { - return filters.IsInternalProperty(schema.PropertyName(propName)) -} - -func (s *Searcher) onTokenizableProp(prop *models.Property) bool { - switch dt, _ := schema.AsPrimitive(prop.DataType); dt { - case schema.DataTypeText, schema.DataTypeTextArray: - return true - default: - return false - } -} - -func (s *Searcher) extractStringArray(value interface{}) ([]string, error) { - switch v := value.(type) { - case []string: - return v, nil - case []interface{}: - vals := make([]string, len(v)) - for i := range v { - val, ok := v[i].(string) - if !ok { - return nil, fmt.Errorf("value[%d] type should be string but is %T", i, v[i]) - } - vals[i] = val - } - return vals, nil - default: - return nil, fmt.Errorf("value type should be []string but is %T", value) - } -} - -func (s *Searcher) extractIntArray(value interface{}) ([]int, error) { - switch v := value.(type) { - case []int: - return v, nil - case []interface{}: - vals := make([]int, len(v)) - for i := range v { - // in this case all number values are unmarshalled to float64, so we need to cast to float64 - // and then make int - val, ok := v[i].(float64) - if !ok { - return nil, fmt.Errorf("value[%d] type should be float64 but is %T", i, v[i]) - } - vals[i] = int(val) - } - return vals, nil - default: - return nil, fmt.Errorf("value type should be []int but is %T", value) - } -} - -func (s *Searcher) extractFloat64Array(value interface{}) ([]float64, error) { - switch v := value.(type) { - case []float64: - return v, nil - case []interface{}: - vals := make([]float64, len(v)) - for i := range v { - val, ok := v[i].(float64) - if !ok { - return nil, fmt.Errorf("value[%d] type should be float64 but is %T", i, v[i]) - } - vals[i] = val - } - return vals, nil - default: - return nil, fmt.Errorf("value type should be []float64 but is %T", value) - } -} - -func (s *Searcher) extractBoolArray(value interface{}) ([]bool, error) { - switch v := value.(type) { - case []bool: - return v, nil - case []interface{}: - vals := make([]bool, len(v)) - for i := range v { - val, ok := v[i].(bool) - if !ok { - return nil, fmt.Errorf("value[%d] type should be bool but is %T", i, v[i]) - } - vals[i] = val - } - return vals, nil - default: - return nil, fmt.Errorf("value type should be []bool but is %T", value) - } -} - -func getContainsOperands[T any](propType schema.DataType, path *filters.Path, values []T) []filters.Clause { - operands := make([]filters.Clause, len(values)) - for i := range values { - operands[i] = filters.Clause{ - Operator: filters.OperatorEqual, - On: path, - Value: &filters.Value{ - Type: propType, - Value: values[i], - }, - } - } - return operands -} - -type docIDsIterator interface { - Next() (uint64, bool) - Len() int -} - -type sliceDocIDsIterator struct { - docIDs []uint64 - pos int -} - -func newSliceDocIDsIterator(docIDs []uint64) docIDsIterator { - return &sliceDocIDsIterator{docIDs: docIDs, pos: 0} -} - -func (it *sliceDocIDsIterator) Next() (uint64, bool) { - if it.pos >= len(it.docIDs) { - return 0, false - } - pos := it.pos - it.pos++ - return it.docIDs[pos], true -} - -func (it *sliceDocIDsIterator) Len() int { - return len(it.docIDs) -} - -type docBitmap struct { - docIDs *sroar.Bitmap -} - -// newUninitializedDocBitmap can be used whenever we can be sure that the first -// user of the docBitmap will set or replace the bitmap, such as a row reader -func newUninitializedDocBitmap() docBitmap { - return docBitmap{docIDs: nil} -} - -func newDocBitmap() docBitmap { - return docBitmap{docIDs: sroar.NewBitmap()} -} - -func (dbm *docBitmap) count() int { - if dbm.docIDs == nil { - return 0 - } - return dbm.docIDs.GetCardinality() -} - -func (dbm *docBitmap) IDs() []uint64 { - if dbm.docIDs == nil { - return []uint64{} - } - return dbm.docIDs.ToArray() -} - -func (dbm *docBitmap) IDsWithLimit(limit int) []uint64 { - card := dbm.docIDs.GetCardinality() - if limit >= card { - return dbm.IDs() - } - - out := make([]uint64, limit) - for i := range out { - // safe to ignore error, it can only error if the index is >= cardinality - // which we have already ruled out - out[i], _ = dbm.docIDs.Select(uint64(i)) - } - - return out -} - -type docPointerWithScore struct { - id uint64 - frequency float32 - propLength float32 -} diff --git a/adapters/repos/db/inverted/searcher_doc_bitmap.go b/adapters/repos/db/inverted/searcher_doc_bitmap.go deleted file mode 100644 index 45626799312603017b87413e58416bc50ef6a660..0000000000000000000000000000000000000000 --- a/adapters/repos/db/inverted/searcher_doc_bitmap.go +++ /dev/null @@ -1,153 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package inverted - -import ( - "context" - "encoding/binary" - "fmt" - - "github.com/pkg/errors" - "github.com/weaviate/sroar" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv" - "github.com/weaviate/weaviate/entities/filters" -) - -func (s *Searcher) docBitmap(ctx context.Context, b *lsmkv.Bucket, limit int, - pv *propValuePair, -) (docBitmap, error) { - // geo props cannot be served by the inverted index and they require an - // external index. So, instead of trying to serve this chunk of the filter - // request internally, we can pass it to an external geo index - if pv.operator == filters.OperatorWithinGeoRange { - return s.docBitmapGeo(ctx, pv) - } - // all other operators perform operations on the inverted index which we - // can serve directly - - if pv.hasFilterableIndex { - // bucket with strategy roaring set serves bitmaps directly - if b.Strategy() == lsmkv.StrategyRoaringSet { - return s.docBitmapInvertedRoaringSet(ctx, b, limit, pv) - } - - // bucket with strategy set serves docIds used to build bitmap - return s.docBitmapInvertedSet(ctx, b, limit, pv) - } - - if pv.hasSearchableIndex { - // bucket with strategy map serves docIds used to build bitmap - // and frequencies, which are ignored for filtering - return s.docBitmapInvertedMap(ctx, b, limit, pv) - } - - return docBitmap{}, fmt.Errorf("property '%s' is neither filterable nor searchable", pv.prop) -} - -func (s *Searcher) docBitmapInvertedRoaringSet(ctx context.Context, b *lsmkv.Bucket, - limit int, pv *propValuePair, -) (docBitmap, error) { - out := newUninitializedDocBitmap() - isEmpty := true - var readFn RoaringSetReadFn = func(k []byte, docIDs *sroar.Bitmap) (bool, error) { - if isEmpty { - out.docIDs = docIDs - isEmpty = false - } else { - out.docIDs.Or(docIDs) - } - - if limit > 0 && out.docIDs.GetCardinality() >= limit { - return false, nil - } - return true, nil - } - - rr := NewRowReaderRoaringSet(b, pv.value, pv.operator, false) - if err := rr.Read(ctx, readFn); err != nil { - return out, errors.Wrap(err, "read row") - } - - if isEmpty { - return newDocBitmap(), nil - } - return out, nil -} - -func (s *Searcher) docBitmapInvertedSet(ctx context.Context, b *lsmkv.Bucket, - limit int, pv *propValuePair, -) (docBitmap, error) { - out := newDocBitmap() - var readFn ReadFn = func(k []byte, ids [][]byte) (bool, error) { - for _, asBytes := range ids { - out.docIDs.Set(binary.LittleEndian.Uint64(asBytes)) - } - - if limit > 0 && out.docIDs.GetCardinality() >= limit { - return false, nil - } - return true, nil - } - - rr := NewRowReader(b, pv.value, pv.operator, false) - if err := rr.Read(ctx, readFn); err != nil { - return out, errors.Wrap(err, "read row") - } - - return out, nil -} - -func (s *Searcher) docBitmapInvertedMap(ctx context.Context, b *lsmkv.Bucket, - limit int, pv *propValuePair, -) (docBitmap, error) { - out := newDocBitmap() - var readFn ReadFnFrequency = func(k []byte, pairs []lsmkv.MapPair) (bool, error) { - for _, pair := range pairs { - // this entry has a frequency, but that's only used for bm25, not for - // pure filtering, so we can ignore it here - if s.shardVersion < 2 { - out.docIDs.Set(binary.LittleEndian.Uint64(pair.Key)) - } else { - out.docIDs.Set(binary.BigEndian.Uint64(pair.Key)) - } - } - - if limit > 0 && out.docIDs.GetCardinality() >= limit { - return false, nil - } - return true, nil - } - - rr := NewRowReaderFrequency(b, pv.value, pv.operator, false, s.shardVersion) - if err := rr.Read(ctx, readFn); err != nil { - return out, errors.Wrap(err, "read row") - } - - return out, nil -} - -func (s *Searcher) docBitmapGeo(ctx context.Context, pv *propValuePair) (docBitmap, error) { - out := newDocBitmap() - propIndex, ok := s.propIndices.ByProp(pv.prop) - - if !ok { - return out, nil - } - - res, err := propIndex.GeoIndex.WithinRange(ctx, *pv.valueGeoRange) - if err != nil { - return out, errors.Wrapf(err, "geo index range search on prop %q", pv.prop) - } - - out.docIDs.SetMany(res) - return out, nil -} diff --git a/adapters/repos/db/inverted/searcher_ref_filter.go b/adapters/repos/db/inverted/searcher_ref_filter.go deleted file mode 100644 index 0c63e782f7696fda1aa13a476de99ed5b33da077..0000000000000000000000000000000000000000 --- a/adapters/repos/db/inverted/searcher_ref_filter.go +++ /dev/null @@ -1,250 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package inverted - -import ( - "context" - "fmt" - - "github.com/go-openapi/strfmt" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/dto" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema/crossref" - "github.com/weaviate/weaviate/entities/search" -) - -// a helper tool to extract the uuid beacon for any matching reference -type refFilterExtractor struct { - logger logrus.FieldLogger - classSearcher ClassSearcher - filter *filters.Clause - class *models.Class - property *models.Property - tenant string - limit int64 -} - -// ClassSearcher is anything that allows a root-level ClassSearch -type ClassSearcher interface { - Search(ctx context.Context, - params dto.GetParams) ([]search.Result, error) - GetQueryMaximumResults() int -} - -func newRefFilterExtractor(logger logrus.FieldLogger, classSearcher ClassSearcher, - filter *filters.Clause, class *models.Class, property *models.Property, tenant string, limit int64, -) *refFilterExtractor { - return &refFilterExtractor{ - logger: logger, - classSearcher: classSearcher, - filter: filter, - class: class, - property: property, - tenant: tenant, - limit: limit, - } -} - -func (r *refFilterExtractor) Do(ctx context.Context) (*propValuePair, error) { - if err := r.validate(); err != nil { - return nil, errors.Wrap(err, "invalid usage") - } - - ids, err := r.fetchIDs(ctx) - if err != nil { - return nil, errors.Wrap(err, "nested request to fetch matching IDs") - } - - if len(ids) > r.classSearcher.GetQueryMaximumResults() { - r.logger. - WithField("nested_reference_results", len(ids)). - WithField("query_maximum_results", r.classSearcher.GetQueryMaximumResults()). - Warnf("Number of found nested reference results exceeds configured QUERY_MAXIMUM_RESULTS. " + - "This may result in search performance degradation or even out of memory errors.") - } - - return r.resultsToPropValuePairs(ids) -} - -func (r *refFilterExtractor) paramsForNestedRequest() (dto.GetParams, error) { - return dto.GetParams{ - Filters: r.innerFilter(), - ClassName: r.filter.On.Child.Class.String(), - Pagination: &filters.Pagination{ - Offset: 0, - // Limit can be set to dynamically with QUERY_NESTED_CROSS_REFERENCE_LIMIT - Limit: int(r.limit), - }, - // set this to indicate that this is a sub-query, so we do not need - // to perform the same search limits cutoff check that we do with - // the root query - AdditionalProperties: additional.Properties{ReferenceQuery: true}, - Tenant: r.tenant, - IsRefOrigin: true, - }, nil -} - -func (r *refFilterExtractor) innerFilter() *filters.LocalFilter { - return &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: r.filter.Operator, - On: r.filter.On.Child, - Value: r.filter.Value, - }, - } -} - -type classUUIDPair struct { - class string - id strfmt.UUID -} - -func (r *refFilterExtractor) fetchIDs(ctx context.Context) ([]classUUIDPair, error) { - params, err := r.paramsForNestedRequest() - if err != nil { - return nil, err - } - - res, err := r.classSearcher.Search(ctx, params) - if err != nil { - return nil, err - } - - out := make([]classUUIDPair, len(res)) - for i, elem := range res { - out[i] = classUUIDPair{class: elem.ClassName, id: elem.ID} - } - - return out, nil -} - -func (r *refFilterExtractor) resultsToPropValuePairs(ids []classUUIDPair, -) (*propValuePair, error) { - switch len(ids) { - case 0: - return r.emptyPropValuePair(), nil - case 1: - return r.backwardCompatibleIDToPropValuePair(ids[0]) - default: - return r.chainedIDsToPropValuePair(ids) - } -} - -func (r *refFilterExtractor) emptyPropValuePair() *propValuePair { - return &propValuePair{ - prop: r.property.Name, - value: nil, - operator: filters.OperatorEqual, - hasFilterableIndex: HasFilterableIndex(r.property), - hasSearchableIndex: HasSearchableIndex(r.property), - Class: r.class, - } -} - -// Because we still support the old beacon format that did not include the -// class yet, we cannot be sure about which format we will find in the -// database. Typically we would now build a filter, such as value==beacon. -// However, this beacon would either have the old format or the new format. -// Depending on which format was used during importing, one would match and the -// other wouldn't. -// -// As a workaround we can use an OR filter to allow both, such as -// ( value==beacon_old_format OR value==beacon_new_format ) -func (r *refFilterExtractor) backwardCompatibleIDToPropValuePair(p classUUIDPair) (*propValuePair, error) { - // this workaround is already implemented in the chained ID case, so we can - // simply pass it through: - return r.chainedIDsToPropValuePair([]classUUIDPair{p}) -} - -func (r *refFilterExtractor) idToPropValuePairWithValue(v []byte, - hasFilterableIndex, hasSearchableIndex bool, -) (*propValuePair, error) { - return &propValuePair{ - prop: r.property.Name, - value: v, - operator: filters.OperatorEqual, - hasFilterableIndex: hasFilterableIndex, - hasSearchableIndex: hasSearchableIndex, - Class: r.class, - }, nil -} - -// chain multiple alternatives using an OR operator -func (r *refFilterExtractor) chainedIDsToPropValuePair(ids []classUUIDPair) (*propValuePair, error) { - hasFilterableIndex := HasFilterableIndex(r.property) - hasSearchableIndex := HasSearchableIndex(r.property) - - children, err := r.idsToPropValuePairs(ids, hasFilterableIndex, hasSearchableIndex) - if err != nil { - return nil, err - } - - return &propValuePair{ - prop: r.property.Name, - operator: filters.OperatorOr, - children: children, - hasFilterableIndex: hasFilterableIndex, - hasSearchableIndex: hasSearchableIndex, - Class: r.class, - }, nil -} - -// Use both new format with class name in the beacon, as well as the old -// format. Since the results will be OR'ed anyway, this is safe todo. -// -// The additional lookups and OR-merge operations have a cost, therefore this -// backward-compatible logic should be removed, as soon as we can be sure that -// no more class-less beacons exist. Most likely this will be the case with the -// next breaking change, such as v2.0.0. -func (r *refFilterExtractor) idsToPropValuePairs(ids []classUUIDPair, - hasFilterableIndex, hasSearchableIndex bool, -) ([]*propValuePair, error) { - // This makes it safe to access the first element later on without further - // checks - if len(ids) == 0 { - return nil, nil - } - - out := make([]*propValuePair, len(ids)*2) - bb := crossref.NewBulkBuilderWithEstimates(len(ids)*2, ids[0].class, 1.25) - for i, id := range ids { - // future-proof way - pv, err := r.idToPropValuePairWithValue(bb.ClassAndID(id.class, id.id), hasFilterableIndex, hasSearchableIndex) - if err != nil { - return nil, err - } - - out[i*2] = pv - - // backward-compatible way - pv, err = r.idToPropValuePairWithValue(bb.LegacyIDOnly(id.id), hasFilterableIndex, hasSearchableIndex) - if err != nil { - return nil, err - } - - out[(i*2)+1] = pv - } - - return out, nil -} - -func (r *refFilterExtractor) validate() error { - if len(r.filter.On.Slice())%2 != 1 { - return fmt.Errorf("path must have an odd number of segments") - } - - return nil -} diff --git a/adapters/repos/db/inverted/searcher_test.go b/adapters/repos/db/inverted/searcher_test.go deleted file mode 100644 index 15682f038f4c50302adf045ce8b534510e4aec57..0000000000000000000000000000000000000000 --- a/adapters/repos/db/inverted/searcher_test.go +++ /dev/null @@ -1,133 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package inverted - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/sroar" -) - -func TestDocBitmap(t *testing.T) { - t.Run("empty doc bitmap", func(t *testing.T) { - dbm := newDocBitmap() - - assert.Equal(t, 0, dbm.count()) - assert.Empty(t, dbm.IDs()) - }) - - t.Run("filled doc bitmap", func(t *testing.T) { - ids := []uint64{1, 2, 3, 4, 5} - - dbm := newDocBitmap() - dbm.docIDs.SetMany(ids) - - assert.Equal(t, 5, dbm.count()) - assert.ElementsMatch(t, ids, dbm.IDs()) - }) -} - -func TestDocBitmap_IDsWithLimit(t *testing.T) { - type test struct { - name string - limit int - input []uint64 - expectedOutput []uint64 - } - - tests := []test{ - { - name: "empty bitmap, positive limit", - input: []uint64{}, - limit: 7, - expectedOutput: []uint64{}, - }, - { - name: "limit matches bitmap cardinality", - input: []uint64{2, 4, 6, 8, 10}, - limit: 5, - expectedOutput: []uint64{2, 4, 6, 8, 10}, - }, - { - name: "limit less than cardinality", - input: []uint64{2, 4, 6, 8, 10}, - limit: 3, - expectedOutput: []uint64{2, 4, 6}, - }, - { - name: "limit higher than cardinality", - input: []uint64{2, 4, 6, 8, 10}, - limit: 10, - expectedOutput: []uint64{2, 4, 6, 8, 10}, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - dbm := docBitmap{ - docIDs: sroar.NewBitmap(), - } - - dbm.docIDs.SetMany(test.input) - - res := dbm.IDsWithLimit(test.limit) - assert.Equal(t, test.expectedOutput, res) - }) - } -} - -func TestDocIDsIterator_Slice(t *testing.T) { - t.Run("iterator empty slice", func(t *testing.T) { - it := newSliceDocIDsIterator([]uint64{}) - - id1, ok1 := it.Next() - - assert.Equal(t, 0, it.Len()) - assert.False(t, ok1) - assert.Equal(t, uint64(0), id1) - }) - - t.Run("iterator step by step", func(t *testing.T) { - it := newSliceDocIDsIterator([]uint64{3, 1, 0, 2}) - - id1, ok1 := it.Next() - id2, ok2 := it.Next() - id3, ok3 := it.Next() - id4, ok4 := it.Next() - id5, ok5 := it.Next() - - assert.Equal(t, 4, it.Len()) - assert.True(t, ok1) - assert.Equal(t, uint64(3), id1) - assert.True(t, ok2) - assert.Equal(t, uint64(1), id2) - assert.True(t, ok3) - assert.Equal(t, uint64(0), id3) - assert.True(t, ok4) - assert.Equal(t, uint64(2), id4) - assert.False(t, ok5) - assert.Equal(t, uint64(0), id5) - }) - - t.Run("iterator in loop", func(t *testing.T) { - it := newSliceDocIDsIterator([]uint64{3, 1, 0, 2}) - ids := []uint64{} - - for id, ok := it.Next(); ok; id, ok = it.Next() { - ids = append(ids, id) - } - - assert.Equal(t, 4, it.Len()) - assert.Equal(t, []uint64{3, 1, 0, 2}, ids) - }) -} diff --git a/adapters/repos/db/inverted/searcher_value_extractors.go b/adapters/repos/db/inverted/searcher_value_extractors.go deleted file mode 100644 index 6d930bf4f684328f781e06c4142a821a9c94ee9b..0000000000000000000000000000000000000000 --- a/adapters/repos/db/inverted/searcher_value_extractors.go +++ /dev/null @@ -1,90 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package inverted - -import ( - "bytes" - "encoding/binary" - "fmt" - "time" - - "github.com/pkg/errors" -) - -func (s *Searcher) extractNumberValue(in interface{}) ([]byte, error) { - value, ok := in.(float64) - if !ok { - return nil, fmt.Errorf("expected value to be float64, got %T", in) - } - - return LexicographicallySortableFloat64(value) -} - -// assumes an untyped int and stores as string-formatted int64 -func (s *Searcher) extractIntValue(in interface{}) ([]byte, error) { - value, ok := in.(int) - if !ok { - return nil, fmt.Errorf("expected value to be int, got %T", in) - } - - return LexicographicallySortableInt64(int64(value)) -} - -// assumes an untyped int and stores as string-formatted int64 -func (s *Searcher) extractIntCountValue(in interface{}) ([]byte, error) { - value, ok := in.(int) - if !ok { - return nil, fmt.Errorf("expected value to be int, got %T", in) - } - - return LexicographicallySortableUint64(uint64(value)) -} - -// assumes an untyped bool and stores as bool64 -func (s *Searcher) extractBoolValue(in interface{}) ([]byte, error) { - value, ok := in.(bool) - if !ok { - return nil, fmt.Errorf("expected value to be bool, got %T", in) - } - - buf := bytes.NewBuffer(nil) - if err := binary.Write(buf, binary.LittleEndian, value); err != nil { - return nil, errors.Wrap(err, "encode bool as binary") - } - - return buf.Bytes(), nil -} - -// assumes a time.Time date and stores as string-formatted int64, if it -// encounters a string it tries to parse it as a time.Time -func (s *Searcher) extractDateValue(in interface{}) ([]byte, error) { - var asInt64 int64 - - switch t := in.(type) { - case string: - parsed, err := time.Parse(time.RFC3339, t) - if err != nil { - return nil, errors.Wrap(err, "trying parse time as RFC3339 string") - } - - asInt64 = parsed.UnixNano() - - case time.Time: - asInt64 = t.UnixNano() - - default: - return nil, fmt.Errorf("expected value to be time.Time (or parseable string)"+ - ", got %T", in) - } - - return LexicographicallySortableInt64(asInt64) -} diff --git a/adapters/repos/db/inverted/serialization.go b/adapters/repos/db/inverted/serialization.go deleted file mode 100644 index 08d30ddc19ed7259bbde60d769e7eaf379aac452..0000000000000000000000000000000000000000 --- a/adapters/repos/db/inverted/serialization.go +++ /dev/null @@ -1,155 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package inverted - -import ( - "bytes" - "encoding/binary" - "fmt" - "math" - - "github.com/pkg/errors" -) - -// LexicographicallySortableFloat64 transforms a conversion to a -// lexicographically sortable byte slice. In general, for lexicographical -// sorting big endian notatino is required. Additionally the sign needs to be -// flipped in any case, but additionally each remaining byte also needs to be -// flipped if the number is negative -func LexicographicallySortableFloat64(in float64) ([]byte, error) { - buf := bytes.NewBuffer(nil) - - err := binary.Write(buf, binary.BigEndian, in) - if err != nil { - return nil, errors.Wrap(err, "serialize float64 value as big endian") - } - - var out []byte - if in >= 0 { - // on positive numbers only flip the sign - out = buf.Bytes() - firstByte := out[0] ^ 0x80 - out = append([]byte{firstByte}, out[1:]...) - } else { - // on negative numbers flip every bit - out = make([]byte, 8) - for i, b := range buf.Bytes() { - out[i] = b ^ 0xFF - } - } - - return out, nil -} - -// ParseLexicographicallySortableFloat64 reverses the changes in -// LexicographicallySortableFloat64 -func ParseLexicographicallySortableFloat64(in []byte) (float64, error) { - if len(in) != 8 { - return 0, fmt.Errorf("float64 must be 8 bytes long, got: %d", len(in)) - } - - flipped := make([]byte, 8) - if in[0]&0x80 == 0x80 { - // encoded as negative means it was originally positive, so we only need to - // flip the sign - flipped[0] = in[0] ^ 0x80 - - // the remainder can be copied - for i := 1; i < 8; i++ { - flipped[i] = in[i] - } - } else { - // encoded as positive means it was originally negative, so we need to flip - // everything - for i := 0; i < 8; i++ { - flipped[i] = in[i] ^ 0xFF - } - } - - r := bytes.NewReader(flipped) - var value float64 - - err := binary.Read(r, binary.BigEndian, &value) - if err != nil { - return 0, errors.Wrap(err, "deserialize float64 value as big endian") - } - - return value, nil -} - -// LexicographicallySortableInt64 performs a conversion to a lexicographically -// sortable byte slice. For this, big endian notation is required and the sign -// must be flipped -func LexicographicallySortableInt64(in int64) ([]byte, error) { - buf := bytes.NewBuffer(nil) - asInt64 := int64(in) - - // flip the sign - asInt64 = asInt64 ^ math.MinInt64 - - err := binary.Write(buf, binary.BigEndian, asInt64) - if err != nil { - return nil, errors.Wrap(err, "serialize int value as big endian") - } - - return buf.Bytes(), nil -} - -// ParseLexicographicallySortableInt64 reverses the changes in -// LexicographicallySortableInt64 -func ParseLexicographicallySortableInt64(in []byte) (int64, error) { - if len(in) != 8 { - return 0, fmt.Errorf("int64 must be 8 bytes long, got: %d", len(in)) - } - - r := bytes.NewReader(in) - var value int64 - - err := binary.Read(r, binary.BigEndian, &value) - if err != nil { - return 0, errors.Wrap(err, "deserialize int64 value as big endian") - } - - return value ^ math.MinInt64, nil -} - -// LexicographicallySortableUint64 performs a conversion to a lexicographically -// sortable byte slice. For this, big endian notation is required. -func LexicographicallySortableUint64(in uint64) ([]byte, error) { - buf := bytes.NewBuffer(nil) - - // no signs to flip as this is a uint - err := binary.Write(buf, binary.BigEndian, in) - if err != nil { - return nil, errors.Wrap(err, "serialize int value as big endian") - } - - return buf.Bytes(), nil -} - -// ParseLexicographicallySortableUint64 reverses the changes in -// LexicographicallySortableUint64 -func ParseLexicographicallySortableUint64(in []byte) (uint64, error) { - if len(in) != 8 { - return 0, fmt.Errorf("uint64 must be 8 bytes long, got: %d", len(in)) - } - - r := bytes.NewReader(in) - var value uint64 - - err := binary.Read(r, binary.BigEndian, &value) - if err != nil { - return 0, errors.Wrap(err, "deserialize uint64 value as big endian") - } - - return value, nil -} diff --git a/adapters/repos/db/inverted/serialization_test.go b/adapters/repos/db/inverted/serialization_test.go deleted file mode 100644 index 36b3c238d4ed1136b10fbd305f643059e2c7c639..0000000000000000000000000000000000000000 --- a/adapters/repos/db/inverted/serialization_test.go +++ /dev/null @@ -1,96 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package inverted - -import ( - "fmt" - "math" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// TestSerialization makes sure that writing and reading into the -// lexicographically sortable types byte slices ends up with the same values as -// original. There is no focus on the sortability itself, as that is already -// tested extensively in analyzer_test.go -func TestSerialization(t *testing.T) { - t.Run("float64", func(t *testing.T) { - subjects := []float64{ - math.SmallestNonzeroFloat64, - -400.0001, - -21, - 0, - 21, - 400.0001, - math.MaxFloat64, - } - - for _, sub := range subjects { - t.Run(fmt.Sprintf("with %f", sub), func(t *testing.T) { - bytes, err := LexicographicallySortableFloat64(sub) - require.Nil(t, err) - - parsed, err := ParseLexicographicallySortableFloat64(bytes) - require.Nil(t, err) - - assert.Equal(t, sub, parsed, "before and after must match") - }) - } - }) - - t.Run("int64", func(t *testing.T) { - subjects := []int64{ - math.MinInt64, - -400, - -21, - 0, - 21, - 400, - math.MaxInt64, - } - - for _, sub := range subjects { - t.Run(fmt.Sprintf("with %d", sub), func(t *testing.T) { - bytes, err := LexicographicallySortableInt64(sub) - require.Nil(t, err) - - parsed, err := ParseLexicographicallySortableInt64(bytes) - require.Nil(t, err) - - assert.Equal(t, sub, parsed, "before and after must match") - }) - } - }) - - t.Run("uint64", func(t *testing.T) { - subjects := []uint64{ - 0, - 21, - 400, - math.MaxUint64, - } - - for _, sub := range subjects { - t.Run(fmt.Sprintf("with %d", sub), func(t *testing.T) { - bytes, err := LexicographicallySortableUint64(sub) - require.Nil(t, err) - - parsed, err := ParseLexicographicallySortableUint64(bytes) - require.Nil(t, err) - - assert.Equal(t, sub, parsed, "before and after must match") - }) - } - }) -} diff --git a/adapters/repos/db/inverted/stopwords/detector.go b/adapters/repos/db/inverted/stopwords/detector.go deleted file mode 100644 index 6155100d8c08136c3a7b538d05685563ea6f3195..0000000000000000000000000000000000000000 --- a/adapters/repos/db/inverted/stopwords/detector.go +++ /dev/null @@ -1,89 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package stopwords - -import ( - "sync" - - "github.com/weaviate/weaviate/entities/models" - - "github.com/pkg/errors" -) - -type StopwordDetector interface { - IsStopword(string) bool -} - -type Detector struct { - sync.Mutex - stopwords map[string]struct{} -} - -func NewDetectorFromConfig(config models.StopwordConfig) (*Detector, error) { - d, err := NewDetectorFromPreset(config.Preset) - if err != nil { - return nil, errors.Wrap(err, "failed to create new detector from config") - } - - d.SetAdditions(config.Additions) - d.SetRemovals(config.Removals) - - return d, nil -} - -func NewDetectorFromPreset(preset string) (*Detector, error) { - var list []string - var ok bool - - if preset != "" { - list, ok = Presets[preset] - if !ok { - return nil, errors.Errorf("preset %q not known to stopword detector", preset) - } - } - - d := &Detector{ - stopwords: map[string]struct{}{}, - } - - for _, word := range list { - d.stopwords[word] = struct{}{} - } - - return d, nil -} - -func (d *Detector) SetAdditions(additions []string) { - d.Lock() - defer d.Unlock() - - for _, add := range additions { - d.stopwords[add] = struct{}{} - } -} - -func (d *Detector) SetRemovals(removals []string) { - d.Lock() - defer d.Unlock() - - for _, rem := range removals { - delete(d.stopwords, rem) - } -} - -func (d *Detector) IsStopword(word string) bool { - d.Lock() - defer d.Unlock() - - _, ok := d.stopwords[word] - return ok -} diff --git a/adapters/repos/db/inverted/stopwords/detector_test.go b/adapters/repos/db/inverted/stopwords/detector_test.go deleted file mode 100644 index b657f88c98e81bf5887c20e0315d59e40f0f45e8..0000000000000000000000000000000000000000 --- a/adapters/repos/db/inverted/stopwords/detector_test.go +++ /dev/null @@ -1,159 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package stopwords - -import ( - "testing" - - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" -) - -func TestStopwordDetector(t *testing.T) { - type testcase struct { - cfg models.StopwordConfig - input []string - expectedCountable int - } - - runTest := func(t *testing.T, tests []testcase) { - for _, test := range tests { - sd, err := NewDetectorFromConfig(test.cfg) - require.Nil(t, err) - - var result []string - for _, word := range test.input { - if !sd.IsStopword(word) { - result = append(result, word) - } - } - require.Equal(t, test.expectedCountable, len(result)) - } - } - - t.Run("with en preset, additions", func(t *testing.T) { - tests := []testcase{ - { - cfg: models.StopwordConfig{ - Preset: "en", - Additions: []string{"dog"}, - }, - input: []string{"dog", "dog", "dog", "dog"}, - expectedCountable: 0, - }, - { - cfg: models.StopwordConfig{ - Preset: "en", - Additions: []string{"dog"}, - }, - input: []string{"dog", "dog", "dog", "cat"}, - expectedCountable: 1, - }, - { - cfg: models.StopwordConfig{ - Preset: "en", - Additions: []string{"dog"}, - }, - input: []string{"a", "dog", "is", "the", "best"}, - expectedCountable: 1, - }, - } - - runTest(t, tests) - }) - - t.Run("with no preset, additions", func(t *testing.T) { - tests := []testcase{ - { - cfg: models.StopwordConfig{ - Preset: "none", - Additions: []string{"dog"}, - }, - input: []string{"a", "dog", "is", "the", "best"}, - expectedCountable: 4, - }, - } - - runTest(t, tests) - }) - - t.Run("with en preset, removals", func(t *testing.T) { - tests := []testcase{ - { - cfg: models.StopwordConfig{ - Preset: "en", - Removals: []string{"a"}, - }, - input: []string{"a", "dog", "is", "the", "best"}, - expectedCountable: 3, - }, - { - cfg: models.StopwordConfig{ - Preset: "en", - Removals: []string{"a", "is", "the"}, - }, - input: []string{"a", "dog", "is", "the", "best"}, - expectedCountable: 5, - }, - } - - runTest(t, tests) - }) - - t.Run("with en preset, removals", func(t *testing.T) { - tests := []testcase{ - { - cfg: models.StopwordConfig{ - Preset: "en", - Removals: []string{"a"}, - }, - input: []string{"a", "dog", "is", "the", "best"}, - expectedCountable: 3, - }, - { - cfg: models.StopwordConfig{ - Preset: "en", - Removals: []string{"a", "is", "the"}, - }, - input: []string{"a", "dog", "is", "the", "best"}, - expectedCountable: 5, - }, - } - - runTest(t, tests) - }) - - t.Run("with en preset, additions, removals", func(t *testing.T) { - tests := []testcase{ - { - cfg: models.StopwordConfig{ - Preset: "en", - Additions: []string{"dog"}, - Removals: []string{"a"}, - }, - input: []string{"a", "dog", "is", "the", "best"}, - expectedCountable: 2, - }, - { - cfg: models.StopwordConfig{ - Preset: "en", - Additions: []string{"dog", "best"}, - Removals: []string{"a", "the", "is"}, - }, - input: []string{"a", "dog", "is", "the", "best"}, - expectedCountable: 3, - }, - } - - runTest(t, tests) - }) -} diff --git a/adapters/repos/db/inverted/stopwords/presets.go b/adapters/repos/db/inverted/stopwords/presets.go deleted file mode 100644 index 65d4dc7a49750155580c2963232df1e0f4184bf2..0000000000000000000000000000000000000000 --- a/adapters/repos/db/inverted/stopwords/presets.go +++ /dev/null @@ -1,27 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package stopwords - -const ( - EnglishPreset = "en" - NoPreset = "none" -) - -var Presets = map[string][]string{ - EnglishPreset: { - "a", "an", "and", "are", "as", "at", "be", "but", "by", "for", - "if", "in", "into", "is", "it", "no", "not", "of", "on", "or", "such", "that", - "the", "their", "then", "there", "these", "they", "this", "to", "was", "will", - "with", - }, - NoPreset: {}, -} diff --git a/adapters/repos/db/inverted_index_integration_test.go b/adapters/repos/db/inverted_index_integration_test.go deleted file mode 100644 index 5cec4ae2edfd7c09deb18aa4dcce1261ff6343a9..0000000000000000000000000000000000000000 --- a/adapters/repos/db/inverted_index_integration_test.go +++ /dev/null @@ -1,1356 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest - -package db - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/go-openapi/strfmt" - "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/entities/dto" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/vectorindex/hnsw" - enthnsw "github.com/weaviate/weaviate/entities/vectorindex/hnsw" -) - -func TestIndexByTimestampsNullStatePropLength_AddClass(t *testing.T) { - dirName := t.TempDir() - vFalse := false - vTrue := true - - class := &models.Class{ - Class: "TestClass", - VectorIndexConfig: hnsw.NewDefaultUserConfig(), - InvertedIndexConfig: &models.InvertedIndexConfig{ - CleanupIntervalSeconds: 60, - Stopwords: &models.StopwordConfig{ - Preset: "none", - }, - IndexTimestamps: true, - IndexNullState: true, - IndexPropertyLength: true, - }, - Properties: []*models.Property{ - { - Name: "initialWithIINil", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "initialWithIITrue", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - IndexFilterable: &vTrue, - IndexSearchable: &vTrue, - }, - { - Name: "initialWithoutII", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - IndexFilterable: &vFalse, - IndexSearchable: &vFalse, - }, - }, - } - shardState := singleShardState() - logger := logrus.New() - schemaGetter := &fakeSchemaGetter{shardState: shardState, schema: schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{class}, - }, - }} - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - defer repo.Shutdown(context.Background()) - - migrator := NewMigrator(repo, logger) - require.Nil(t, migrator.AddClass(context.Background(), class, schemaGetter.shardState)) - - require.Nil(t, migrator.AddProperty(context.Background(), class.Class, &models.Property{ - Name: "updateWithIINil", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - })) - require.Nil(t, migrator.AddProperty(context.Background(), class.Class, &models.Property{ - Name: "updateWithIITrue", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - IndexFilterable: &vTrue, - IndexSearchable: &vTrue, - })) - require.Nil(t, migrator.AddProperty(context.Background(), class.Class, &models.Property{ - Name: "updateWithoutII", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - IndexFilterable: &vFalse, - IndexSearchable: &vFalse, - })) - - t.Run("check for additional buckets", func(t *testing.T) { - for _, idx := range migrator.db.indices { - idx.ForEachShard(func(_ string, shd ShardLike) error { - createBucket := shd.Store().Bucket("property__creationTimeUnix") - assert.NotNil(t, createBucket) - - updateBucket := shd.Store().Bucket("property__lastUpdateTimeUnix") - assert.NotNil(t, updateBucket) - - cases := []struct { - prop string - compareFunc func(t assert.TestingT, object interface{}, msgAndArgs ...interface{}) bool - }{ - {prop: "initialWithIINil", compareFunc: assert.NotNil}, - {prop: "initialWithIITrue", compareFunc: assert.NotNil}, - {prop: "initialWithoutII", compareFunc: assert.Nil}, - {prop: "updateWithIINil", compareFunc: assert.NotNil}, - {prop: "updateWithIITrue", compareFunc: assert.NotNil}, - {prop: "updateWithoutII", compareFunc: assert.Nil}, - } - for _, tt := range cases { - tt.compareFunc(t, shd.Store().Bucket("property_"+tt.prop+filters.InternalNullIndex)) - tt.compareFunc(t, shd.Store().Bucket("property_"+tt.prop+filters.InternalPropertyLength)) - } - return nil - }) - } - }) - - t.Run("Add Objects", func(t *testing.T) { - testID1 := strfmt.UUID("a0b55b05-bc5b-4cc9-b646-1452d1390a62") - objWithProperty := &models.Object{ - ID: testID1, - Class: "TestClass", - Properties: map[string]interface{}{"initialWithIINil": "0", "initialWithIITrue": "0", "initialWithoutII": "1", "updateWithIINil": "2", "updateWithIITrue": "2", "updateWithoutII": "3"}, - } - vec := []float32{1, 2, 3} - require.Nil(t, repo.PutObject(context.Background(), objWithProperty, vec, nil)) - - testID2 := strfmt.UUID("a0b55b05-bc5b-4cc9-b646-1452d1390a63") - objWithoutProperty := &models.Object{ - ID: testID2, - Class: "TestClass", - Properties: map[string]interface{}{}, - } - require.Nil(t, repo.PutObject(context.Background(), objWithoutProperty, vec, nil)) - - testID3 := strfmt.UUID("a0b55b05-bc5b-4cc9-b646-1452d1390a64") - objWithNilProperty := &models.Object{ - ID: testID3, - Class: "TestClass", - Properties: map[string]interface{}{"initialWithIINil": nil, "initialWithIITrue": nil, "initialWithoutII": nil, "updateWithIINil": nil, "updateWithIITrue": nil, "updateWithoutII": nil}, - } - require.Nil(t, repo.PutObject(context.Background(), objWithNilProperty, vec, nil)) - }) - - t.Run("delete class", func(t *testing.T) { - require.Nil(t, migrator.DropClass(context.Background(), class.Class)) - for _, idx := range migrator.db.indices { - idx.ForEachShard(func(name string, shd ShardLike) error { - require.Nil(t, shd.Store().Bucket("property__creationTimeUnix")) - require.Nil(t, shd.Store().Bucket("property_name"+filters.InternalNullIndex)) - require.Nil(t, shd.Store().Bucket("property_name"+filters.InternalPropertyLength)) - return nil - }) - } - }) -} - -func TestIndexNullState_GetClass(t *testing.T) { - dirName := t.TempDir() - - testID1 := strfmt.UUID("a0b55b05-bc5b-4cc9-b646-1452d1390a62") - testID2 := strfmt.UUID("65be32cc-bb74-49c7-833e-afb14f957eae") - refID1 := strfmt.UUID("f2e42a9f-e0b5-46bd-8a9c-e70b6330622c") - refID2 := strfmt.UUID("92d5920c-1c20-49da-9cdc-b765813e175b") - - var repo *DB - var schemaGetter *fakeSchemaGetter - - t.Run("init repo", func(t *testing.T) { - schemaGetter = &fakeSchemaGetter{ - shardState: singleShardState(), - schema: schema.Schema{ - Objects: &models.Schema{}, - }, - } - var err error - repo, err = New(logrus.New(), Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - }) - - defer repo.Shutdown(testCtx()) - - t.Run("add classes", func(t *testing.T) { - class := &models.Class{ - Class: "TestClass", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: &models.InvertedIndexConfig{ - IndexNullState: true, - IndexTimestamps: true, - IndexPropertyLength: true, - }, - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationField, - }, - }, - } - - refClass := &models.Class{ - Class: "RefClass", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: &models.InvertedIndexConfig{ - IndexTimestamps: true, - IndexPropertyLength: true, - }, - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationField, - }, - { - Name: "toTest", - DataType: []string{"TestClass"}, - }, - }, - } - - migrator := NewMigrator(repo, repo.logger) - err := migrator.AddClass(context.Background(), class, schemaGetter.shardState) - require.Nil(t, err) - err = migrator.AddClass(context.Background(), refClass, schemaGetter.shardState) - require.Nil(t, err) - schemaGetter.schema.Objects.Classes = append(schemaGetter.schema.Objects.Classes, class, refClass) - }) - - t.Run("insert test objects", func(t *testing.T) { - vec := []float32{1, 2, 3} - for _, obj := range []*models.Object{ - { - ID: testID1, - Class: "TestClass", - Properties: map[string]interface{}{ - "name": "object1", - }, - }, - { - ID: testID2, - Class: "TestClass", - Properties: map[string]interface{}{ - "name": nil, - }, - }, - { - ID: refID1, - Class: "RefClass", - Properties: map[string]interface{}{ - "name": "ref1", - "toTest": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/TestClass/%s", testID1)), - }, - }, - }, - }, - { - ID: refID2, - Class: "RefClass", - Properties: map[string]interface{}{ - "name": "ref2", - "toTest": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/TestClass/%s", testID2)), - }, - }, - }, - }, - } { - err := repo.PutObject(context.Background(), obj, vec, nil) - require.Nil(t, err) - } - }) - - t.Run("check buckets exist", func(t *testing.T) { - index := repo.indices["testclass"] - n := 0 - index.ForEachShard(func(_ string, shard ShardLike) error { - bucketNull := shard.Store().Bucket(helpers.BucketFromPropNameNullLSM("name")) - require.NotNil(t, bucketNull) - n++ - return nil - }) - require.Equal(t, 1, n) - }) - - type testCase struct { - name string - filter *filters.LocalFilter - expectedIds []strfmt.UUID - } - - t.Run("get object with null filters", func(t *testing.T) { - testCases := []testCase{ - { - name: "is null", - filter: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorIsNull, - On: &filters.Path{ - Class: "TestClass", - Property: "name", - }, - Value: &filters.Value{ - Value: false, - Type: schema.DataTypeBoolean, - }, - }, - }, - expectedIds: []strfmt.UUID{testID1}, - }, - { - name: "is not null", - filter: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorIsNull, - On: &filters.Path{ - Class: "TestClass", - Property: "name", - }, - Value: &filters.Value{ - Value: true, - Type: schema.DataTypeBoolean, - }, - }, - }, - expectedIds: []strfmt.UUID{testID2}, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - res, err := repo.Search(context.Background(), dto.GetParams{ - ClassName: "TestClass", - Pagination: &filters.Pagination{Limit: 10}, - Filters: tc.filter, - }) - require.Nil(t, err) - require.Len(t, res, len(tc.expectedIds)) - - ids := make([]strfmt.UUID, len(res)) - for i := range res { - ids[i] = res[i].ID - } - assert.ElementsMatch(t, ids, tc.expectedIds) - }) - } - }) - - t.Run("get referencing object with null filters", func(t *testing.T) { - testCases := []testCase{ - { - name: "is null", - filter: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorIsNull, - On: &filters.Path{ - Class: "RefClass", - Property: "toTest", - Child: &filters.Path{ - Class: "TestClass", - Property: "name", - }, - }, - Value: &filters.Value{ - Value: false, - Type: schema.DataTypeBoolean, - }, - }, - }, - expectedIds: []strfmt.UUID{refID1}, - }, - { - name: "is not null", - filter: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorIsNull, - On: &filters.Path{ - Class: "RefClass", - Property: "toTest", - Child: &filters.Path{ - Class: "TestClass", - Property: "name", - }, - }, - Value: &filters.Value{ - Value: true, - Type: schema.DataTypeBoolean, - }, - }, - }, - expectedIds: []strfmt.UUID{refID2}, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - res, err := repo.Search(context.Background(), dto.GetParams{ - ClassName: "RefClass", - Pagination: &filters.Pagination{Limit: 10}, - Filters: tc.filter, - }) - require.Nil(t, err) - require.Len(t, res, len(tc.expectedIds)) - - ids := make([]strfmt.UUID, len(res)) - for i := range res { - ids[i] = res[i].ID - } - assert.ElementsMatch(t, ids, tc.expectedIds) - }) - } - }) -} - -func TestIndexPropLength_GetClass(t *testing.T) { - dirName := t.TempDir() - - testID1 := strfmt.UUID("a0b55b05-bc5b-4cc9-b646-1452d1390a62") - testID2 := strfmt.UUID("65be32cc-bb74-49c7-833e-afb14f957eae") - refID1 := strfmt.UUID("f2e42a9f-e0b5-46bd-8a9c-e70b6330622c") - refID2 := strfmt.UUID("92d5920c-1c20-49da-9cdc-b765813e175b") - - var repo *DB - var schemaGetter *fakeSchemaGetter - - t.Run("init repo", func(t *testing.T) { - schemaGetter = &fakeSchemaGetter{ - shardState: singleShardState(), - schema: schema.Schema{ - Objects: &models.Schema{}, - }, - } - var err error - repo, err = New(logrus.New(), Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - }) - - defer repo.Shutdown(testCtx()) - - t.Run("add classes", func(t *testing.T) { - class := &models.Class{ - Class: "TestClass", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: &models.InvertedIndexConfig{ - IndexPropertyLength: true, - IndexTimestamps: true, - }, - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationField, - }, - { - Name: "int_array", - DataType: schema.DataTypeIntArray.PropString(), - }, - }, - } - - refClass := &models.Class{ - Class: "RefClass", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: &models.InvertedIndexConfig{ - IndexTimestamps: true, - }, - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationField, - }, - { - Name: "toTest", - DataType: []string{"TestClass"}, - }, - }, - } - - migrator := NewMigrator(repo, repo.logger) - err := migrator.AddClass(context.Background(), class, schemaGetter.shardState) - require.Nil(t, err) - err = migrator.AddClass(context.Background(), refClass, schemaGetter.shardState) - require.Nil(t, err) - schemaGetter.schema.Objects.Classes = append(schemaGetter.schema.Objects.Classes, class, refClass) - }) - - t.Run("insert test objects", func(t *testing.T) { - vec := []float32{1, 2, 3} - for _, obj := range []*models.Object{ - { - ID: testID1, - Class: "TestClass", - Properties: map[string]interface{}{ - "name": "short", - "int_array": []float64{}, - }, - }, - { - ID: testID2, - Class: "TestClass", - Properties: map[string]interface{}{ - "name": "muchLongerName", - "int_array": []float64{1, 2, 3}, - }, - }, - { - ID: refID1, - Class: "RefClass", - Properties: map[string]interface{}{ - "name": "ref1", - "toTest": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/TestClass/%s", testID1)), - }, - }, - }, - }, - { - ID: refID2, - Class: "RefClass", - Properties: map[string]interface{}{ - "name": "ref2", - "toTest": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/TestClass/%s", testID2)), - }, - }, - }, - }, - } { - err := repo.PutObject(context.Background(), obj, vec, nil) - require.Nil(t, err) - } - }) - - t.Run("check buckets exist", func(t *testing.T) { - index := repo.indices["testclass"] - n := 0 - index.ForEachShard(func(_ string, shard ShardLike) error { - bucketPropLengthName := shard.Store().Bucket(helpers.BucketFromPropNameLengthLSM("name")) - require.NotNil(t, bucketPropLengthName) - bucketPropLengthIntArray := shard.Store().Bucket(helpers.BucketFromPropNameLengthLSM("int_array")) - require.NotNil(t, bucketPropLengthIntArray) - n++ - return nil - }) - require.Equal(t, 1, n) - }) - - type testCase struct { - name string - filter *filters.LocalFilter - expectedIds []strfmt.UUID - } - - t.Run("get object with prop length filters", func(t *testing.T) { - testCases := []testCase{ - { - name: "name length = 5", - filter: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: "TestClass", - Property: "len(name)", - }, - Value: &filters.Value{ - Value: 5, - Type: schema.DataTypeInt, - }, - }, - }, - expectedIds: []strfmt.UUID{testID1}, - }, - { - name: "name length >= 6", - filter: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorGreaterThanEqual, - On: &filters.Path{ - Class: "TestClass", - Property: "len(name)", - }, - Value: &filters.Value{ - Value: 6, - Type: schema.DataTypeInt, - }, - }, - }, - expectedIds: []strfmt.UUID{testID2}, - }, - { - name: "array length = 0", - filter: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: "TestClass", - Property: "len(int_array)", - }, - Value: &filters.Value{ - Value: 0, - Type: schema.DataTypeInt, - }, - }, - }, - expectedIds: []strfmt.UUID{testID1}, - }, - { - name: "array length < 4", - filter: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorLessThan, - On: &filters.Path{ - Class: "TestClass", - Property: "len(int_array)", - }, - Value: &filters.Value{ - Value: 4, - Type: schema.DataTypeInt, - }, - }, - }, - expectedIds: []strfmt.UUID{testID1, testID2}, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - res, err := repo.Search(context.Background(), dto.GetParams{ - ClassName: "TestClass", - Pagination: &filters.Pagination{Limit: 10}, - Filters: tc.filter, - }) - require.Nil(t, err) - require.Len(t, res, len(tc.expectedIds)) - - ids := make([]strfmt.UUID, len(res)) - for i := range res { - ids[i] = res[i].ID - } - assert.ElementsMatch(t, ids, tc.expectedIds) - }) - } - }) - - t.Run("get referencing object with prop length filters", func(t *testing.T) { - testCases := []testCase{ - { - name: "name length = 5", - filter: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: "RefClass", - Property: "toTest", - Child: &filters.Path{ - Class: "TestClass", - Property: "len(name)", - }, - }, - Value: &filters.Value{ - Value: 5, - Type: schema.DataTypeInt, - }, - }, - }, - expectedIds: []strfmt.UUID{refID1}, - }, - { - name: "name length >= 6", - filter: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorGreaterThanEqual, - On: &filters.Path{ - Class: "RefClass", - Property: "toTest", - Child: &filters.Path{ - Class: "TestClass", - Property: "len(name)", - }, - }, - Value: &filters.Value{ - Value: 6, - Type: schema.DataTypeInt, - }, - }, - }, - expectedIds: []strfmt.UUID{refID2}, - }, - { - name: "array length = 0", - filter: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: "RefClass", - Property: "toTest", - Child: &filters.Path{ - Class: "TestClass", - Property: "len(int_array)", - }, - }, - Value: &filters.Value{ - Value: 0, - Type: schema.DataTypeInt, - }, - }, - }, - expectedIds: []strfmt.UUID{refID1}, - }, - { - name: "array length < 4", - filter: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorLessThan, - On: &filters.Path{ - Class: "RefClass", - Property: "toTest", - Child: &filters.Path{ - Class: "TestClass", - Property: "len(int_array)", - }, - }, - Value: &filters.Value{ - Value: 4, - Type: schema.DataTypeInt, - }, - }, - }, - expectedIds: []strfmt.UUID{refID1, refID2}, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - res, err := repo.Search(context.Background(), dto.GetParams{ - ClassName: "RefClass", - Pagination: &filters.Pagination{Limit: 10}, - Filters: tc.filter, - }) - require.Nil(t, err) - require.Len(t, res, len(tc.expectedIds)) - - ids := make([]strfmt.UUID, len(res)) - for i := range res { - ids[i] = res[i].ID - } - assert.ElementsMatch(t, ids, tc.expectedIds) - }) - } - }) -} - -func TestIndexByTimestamps_GetClass(t *testing.T) { - dirName := t.TempDir() - - time1 := time.Now() - time2 := time1.Add(-time.Hour) - timestamp1 := time1.UnixMilli() - timestamp2 := time2.UnixMilli() - - testID1 := strfmt.UUID("a0b55b05-bc5b-4cc9-b646-1452d1390a62") - testID2 := strfmt.UUID("65be32cc-bb74-49c7-833e-afb14f957eae") - refID1 := strfmt.UUID("f2e42a9f-e0b5-46bd-8a9c-e70b6330622c") - refID2 := strfmt.UUID("92d5920c-1c20-49da-9cdc-b765813e175b") - - var repo *DB - var schemaGetter *fakeSchemaGetter - - t.Run("init repo", func(t *testing.T) { - schemaGetter = &fakeSchemaGetter{ - shardState: singleShardState(), - schema: schema.Schema{ - Objects: &models.Schema{}, - }, - } - var err error - repo, err = New(logrus.New(), Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - }) - - defer repo.Shutdown(testCtx()) - - t.Run("add classes", func(t *testing.T) { - class := &models.Class{ - Class: "TestClass", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: &models.InvertedIndexConfig{ - IndexTimestamps: true, - IndexPropertyLength: true, - }, - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationField, - }, - }, - } - - refClass := &models.Class{ - Class: "RefClass", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: &models.InvertedIndexConfig{ - IndexTimestamps: true, - IndexPropertyLength: true, - }, - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationField, - }, - { - Name: "toTest", - DataType: []string{"TestClass"}, - }, - }, - } - - migrator := NewMigrator(repo, repo.logger) - err := migrator.AddClass(context.Background(), class, schemaGetter.shardState) - require.Nil(t, err) - err = migrator.AddClass(context.Background(), refClass, schemaGetter.shardState) - require.Nil(t, err) - schemaGetter.schema.Objects.Classes = append(schemaGetter.schema.Objects.Classes, class, refClass) - }) - - t.Run("insert test objects", func(t *testing.T) { - vec := []float32{1, 2, 3} - for _, obj := range []*models.Object{ - { - ID: testID1, - Class: "TestClass", - CreationTimeUnix: timestamp1, - LastUpdateTimeUnix: timestamp1, - Properties: map[string]interface{}{ - "name": "object1", - }, - }, - { - ID: testID2, - Class: "TestClass", - CreationTimeUnix: timestamp2, - LastUpdateTimeUnix: timestamp2, - Properties: map[string]interface{}{ - "name": "object2", - }, - }, - { - ID: refID1, - Class: "RefClass", - Properties: map[string]interface{}{ - "name": "ref1", - "toTest": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/TestClass/%s", testID1)), - }, - }, - }, - }, - { - ID: refID2, - Class: "RefClass", - Properties: map[string]interface{}{ - "name": "ref2", - "toTest": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/TestClass/%s", testID2)), - }, - }, - }, - }, - } { - err := repo.PutObject(context.Background(), obj, vec, nil) - require.Nil(t, err) - } - }) - - t.Run("check buckets exist", func(t *testing.T) { - index := repo.indices["testclass"] - n := 0 - index.ForEachShard(func(_ string, shard ShardLike) error { - bucketCreated := shard.Store().Bucket("property_" + filters.InternalPropCreationTimeUnix) - require.NotNil(t, bucketCreated) - bucketUpdated := shard.Store().Bucket("property_" + filters.InternalPropLastUpdateTimeUnix) - require.NotNil(t, bucketUpdated) - n++ - return nil - }) - require.Equal(t, 1, n) - }) - - type testCase struct { - name string - filter *filters.LocalFilter - expectedIds []strfmt.UUID - } - - t.Run("get object with timestamp filters", func(t *testing.T) { - testCases := []testCase{ - { - name: "by creation timestamp 1", - filter: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: "TestClass", - Property: "_creationTimeUnix", - }, - Value: &filters.Value{ - Value: fmt.Sprint(timestamp1), - Type: schema.DataTypeText, - }, - }, - }, - expectedIds: []strfmt.UUID{testID1}, - }, - { - name: "by creation timestamp 2", - filter: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: "TestClass", - Property: "_creationTimeUnix", - }, - Value: &filters.Value{ - Value: fmt.Sprint(timestamp2), - Type: schema.DataTypeText, - }, - }, - }, - expectedIds: []strfmt.UUID{testID2}, - }, - { - name: "by creation date 1", - filter: &filters.LocalFilter{ - Root: &filters.Clause{ - // since RFC3339 is limited to seconds, - // >= operator is used to match object with timestamp containing milliseconds - Operator: filters.OperatorGreaterThanEqual, - On: &filters.Path{ - Class: "TestClass", - Property: "_creationTimeUnix", - }, - Value: &filters.Value{ - Value: time1.Format(time.RFC3339), - Type: schema.DataTypeDate, - }, - }, - }, - expectedIds: []strfmt.UUID{testID1}, - }, - { - name: "by creation date 2", - filter: &filters.LocalFilter{ - Root: &filters.Clause{ - // since RFC3339 is limited to seconds, - // >= operator is used to match object with timestamp containing milliseconds - Operator: filters.OperatorGreaterThanEqual, - On: &filters.Path{ - Class: "TestClass", - Property: "_creationTimeUnix", - }, - Value: &filters.Value{ - Value: time2.Format(time.RFC3339), - Type: schema.DataTypeDate, - }, - }, - }, - expectedIds: []strfmt.UUID{testID1, testID2}, - }, - - { - name: "by updated timestamp 1", - filter: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: "TestClass", - Property: "_lastUpdateTimeUnix", - }, - Value: &filters.Value{ - Value: fmt.Sprint(timestamp1), - Type: schema.DataTypeText, - }, - }, - }, - expectedIds: []strfmt.UUID{testID1}, - }, - { - name: "by updated timestamp 2", - filter: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: "TestClass", - Property: "_lastUpdateTimeUnix", - }, - Value: &filters.Value{ - Value: fmt.Sprint(timestamp2), - Type: schema.DataTypeText, - }, - }, - }, - expectedIds: []strfmt.UUID{testID2}, - }, - { - name: "by updated date 1", - filter: &filters.LocalFilter{ - Root: &filters.Clause{ - // since RFC3339 is limited to seconds, - // >= operator is used to match object with timestamp containing milliseconds - Operator: filters.OperatorGreaterThanEqual, - On: &filters.Path{ - Class: "TestClass", - Property: "_lastUpdateTimeUnix", - }, - Value: &filters.Value{ - Value: time1.Format(time.RFC3339), - Type: schema.DataTypeDate, - }, - }, - }, - expectedIds: []strfmt.UUID{testID1}, - }, - { - name: "by updated date 2", - filter: &filters.LocalFilter{ - Root: &filters.Clause{ - // since RFC3339 is limited to seconds, - // >= operator is used to match object with timestamp containing milliseconds - Operator: filters.OperatorGreaterThanEqual, - On: &filters.Path{ - Class: "TestClass", - Property: "_lastUpdateTimeUnix", - }, - Value: &filters.Value{ - Value: time2.Format(time.RFC3339), - Type: schema.DataTypeDate, - }, - }, - }, - expectedIds: []strfmt.UUID{testID1, testID2}, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - res, err := repo.Search(context.Background(), dto.GetParams{ - ClassName: "TestClass", - Pagination: &filters.Pagination{Limit: 10}, - Filters: tc.filter, - }) - require.Nil(t, err) - require.Len(t, res, len(tc.expectedIds)) - - ids := make([]strfmt.UUID, len(res)) - for i := range res { - ids[i] = res[i].ID - } - assert.ElementsMatch(t, ids, tc.expectedIds) - }) - } - }) - - t.Run("get referencing object with timestamp filters", func(t *testing.T) { - testCases := []testCase{ - { - name: "by creation timestamp 1", - filter: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: "RefClass", - Property: "toTest", - Child: &filters.Path{ - Class: "TestClass", - Property: "_creationTimeUnix", - }, - }, - Value: &filters.Value{ - Value: fmt.Sprint(timestamp1), - Type: schema.DataTypeText, - }, - }, - }, - expectedIds: []strfmt.UUID{refID1}, - }, - { - name: "by creation timestamp 2", - filter: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: "RefClass", - Property: "toTest", - Child: &filters.Path{ - Class: "TestClass", - Property: "_creationTimeUnix", - }, - }, - Value: &filters.Value{ - Value: fmt.Sprint(timestamp2), - Type: schema.DataTypeText, - }, - }, - }, - expectedIds: []strfmt.UUID{refID2}, - }, - { - name: "by creation date 1", - filter: &filters.LocalFilter{ - Root: &filters.Clause{ - // since RFC3339 is limited to seconds, - // >= operator is used to match object with timestamp containing milliseconds - Operator: filters.OperatorGreaterThanEqual, - On: &filters.Path{ - Class: "RefClass", - Property: "toTest", - Child: &filters.Path{ - Class: "TestClass", - Property: "_creationTimeUnix", - }, - }, - Value: &filters.Value{ - Value: time1.Format(time.RFC3339), - Type: schema.DataTypeDate, - }, - }, - }, - expectedIds: []strfmt.UUID{refID1}, - }, - { - name: "by creation date 2", - filter: &filters.LocalFilter{ - Root: &filters.Clause{ - // since RFC3339 is limited to seconds, - // >= operator is used to match object with timestamp containing milliseconds - Operator: filters.OperatorGreaterThanEqual, - On: &filters.Path{ - Class: "RefClass", - Property: "toTest", - Child: &filters.Path{ - Class: "TestClass", - Property: "_creationTimeUnix", - }, - }, - Value: &filters.Value{ - Value: time2.Format(time.RFC3339), - Type: schema.DataTypeDate, - }, - }, - }, - expectedIds: []strfmt.UUID{refID1, refID2}, - }, - - { - name: "by updated timestamp 1", - filter: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: "RefClass", - Property: "toTest", - Child: &filters.Path{ - Class: "TestClass", - Property: "_lastUpdateTimeUnix", - }, - }, - Value: &filters.Value{ - Value: fmt.Sprint(timestamp1), - Type: schema.DataTypeText, - }, - }, - }, - expectedIds: []strfmt.UUID{refID1}, - }, - { - name: "by updated timestamp 2", - filter: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: "RefClass", - Property: "toTest", - Child: &filters.Path{ - Class: "TestClass", - Property: "_lastUpdateTimeUnix", - }, - }, - Value: &filters.Value{ - Value: fmt.Sprint(timestamp2), - Type: schema.DataTypeText, - }, - }, - }, - expectedIds: []strfmt.UUID{refID2}, - }, - { - name: "by updated date 1", - filter: &filters.LocalFilter{ - Root: &filters.Clause{ - // since RFC3339 is limited to seconds, - // >= operator is used to match object with timestamp containing milliseconds - Operator: filters.OperatorGreaterThanEqual, - On: &filters.Path{ - Class: "RefClass", - Property: "toTest", - Child: &filters.Path{ - Class: "TestClass", - Property: "_lastUpdateTimeUnix", - }, - }, - Value: &filters.Value{ - Value: time1.Format(time.RFC3339), - Type: schema.DataTypeDate, - }, - }, - }, - expectedIds: []strfmt.UUID{refID1}, - }, - { - name: "by updated date 2", - filter: &filters.LocalFilter{ - Root: &filters.Clause{ - // since RFC3339 is limited to seconds, - // >= operator is used to match object with timestamp containing milliseconds - Operator: filters.OperatorGreaterThanEqual, - On: &filters.Path{ - Class: "RefClass", - Property: "toTest", - Child: &filters.Path{ - Class: "TestClass", - Property: "_lastUpdateTimeUnix", - }, - }, - Value: &filters.Value{ - Value: time2.Format(time.RFC3339), - Type: schema.DataTypeDate, - }, - }, - }, - expectedIds: []strfmt.UUID{refID1, refID2}, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - res, err := repo.Search(context.Background(), dto.GetParams{ - ClassName: "RefClass", - Pagination: &filters.Pagination{Limit: 10}, - Filters: tc.filter, - }) - require.Nil(t, err) - require.Len(t, res, len(tc.expectedIds)) - - ids := make([]strfmt.UUID, len(res)) - for i := range res { - ids[i] = res[i].ID - } - assert.ElementsMatch(t, ids, tc.expectedIds) - }) - } - }) -} - -// Cannot filter for property length without enabling in the InvertedIndexConfig -func TestFilterPropertyLengthError(t *testing.T) { - class := createClassWithEverything(false, false) - migrator, repo, schemaGetter := createRepo(t) - defer repo.Shutdown(context.Background()) - err := migrator.AddClass(context.Background(), class, schemaGetter.shardState) - require.Nil(t, err) - // update schema getter so it's in sync with class - schemaGetter.schema = schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{class}, - }, - } - - LengthFilter := &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: schema.ClassName(carClass.Class), - Property: "len(" + schema.PropertyName(class.Properties[0].Name) + ")", - }, - Value: &filters.Value{ - Value: 1, - Type: dtInt, - }, - }, - } - - params := dto.GetParams{ - SearchVector: []float32{0.1, 0.1, 0.1, 1.1, 0.1}, - ClassName: class.Class, - Pagination: &filters.Pagination{Limit: 5}, - Filters: LengthFilter, - } - _, err = repo.Search(context.Background(), params) - require.NotNil(t, err) -} diff --git a/adapters/repos/db/inverted_migrator_filter_to_search.go b/adapters/repos/db/inverted_migrator_filter_to_search.go deleted file mode 100644 index 4cc809ed6fc13df3281329c0d0df13bb00e71ffa..0000000000000000000000000000000000000000 --- a/adapters/repos/db/inverted_migrator_filter_to_search.go +++ /dev/null @@ -1,406 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "os" - "path" - "sync" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/adapters/repos/db/inverted" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv" - "github.com/weaviate/weaviate/entities/models" - ucschema "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/storagestate" - "github.com/weaviate/weaviate/usecases/schema" - "golang.org/x/sync/errgroup" -) - -type filterableToSearchableMigrator struct { - logger logrus.FieldLogger - files *filterableToSearchableMigrationFiles - schemaGetter schema.SchemaGetter - indexes map[string]*Index -} - -func newFilterableToSearchableMigrator(migrator *Migrator) *filterableToSearchableMigrator { - return &filterableToSearchableMigrator{ - logger: migrator.logger, - files: newFilterableToSearchableMigrationFiles(migrator.db.config.RootPath), - schemaGetter: migrator.db.schemaGetter, - indexes: migrator.db.indices, - } -} - -func (m *filterableToSearchableMigrator) migrate(ctx context.Context) error { - // only properties with both Filterable and Searchable indexing enabled - // filterable bucket has map strategy (when it should have roaring set strategy) - // searchable bucket does not exist (in fact weaviate will create empty one with map strategy) - - // if flag exists, no class/property needs fixing - if m.files.existsMigrationSkipFlag() { - m.log().Debug("migration skip flag set, skipping migration") - return nil - } - - migrationState, err := m.files.loadMigrationState() - if err != nil { - m.log().WithError(err).Error("loading migrated state") - return errors.Wrap(err, "loading migrated state") - } - - migrationStateUpdated := false - updateLock := new(sync.Mutex) - sch := m.schemaGetter.GetSchemaSkipAuth().Objects - - m.log().Debug("starting migration") - - eg := &errgroup.Group{} - eg.SetLimit(_NUMCPU * 2) - for _, index := range m.indexes { - index := index - - eg.Go(func() error { - migratedProps, err := m.migrateClass(ctx, index, sch) - if err != nil { - m.logIndex(index).WithError(err).Error("failed migrating class") - return errors.Wrap(err, "failed migrating class") - } - if len(migratedProps) == 0 { - return nil - } - - updateLock.Lock() - defer updateLock.Unlock() - - migrationState.MissingFilterableClass2Props[index.Config.ClassName.String()] = migratedProps - migrationStateUpdated = true - return nil - }) - } - - err = eg.Wait() - if err != nil { - m.log().WithError(err).Error("failed migrating classes") - } - - // save state regardless of previous error - if migrationStateUpdated { - m.log().Debug("saving migration state") - if err := m.files.saveMigrationState(migrationState); err != nil { - m.log().WithError(err).Error("failed saving migration state") - return errors.Wrap(err, "failed saving migration state") - } - } - - if err != nil { - return errors.Wrap(err, "failed migrating classes") - } - - if err := m.files.createMigrationSkipFlag(); err != nil { - m.log().WithError(err).Error("failed creating migration skip flag") - return errors.Wrap(err, "failed creating migration skip flag") - } - - m.log().Debug("finished migration") - return nil -} - -func (m *filterableToSearchableMigrator) switchShardsToFallbackMode(ctx context.Context) error { - m.log().Debug("starting switching fallback mode") - - migrationState, err := m.files.loadMigrationState() - if err != nil { - m.log().WithError(err).Error("loading migrated state") - return errors.Wrap(err, "loading migrated state") - } - - if len(migrationState.MissingFilterableClass2Props) == 0 { - m.log().Debug("no missing filterable indexes, fallback mode skipped") - return nil - } - - for _, index := range m.indexes { - if _, ok := migrationState.MissingFilterableClass2Props[index.Config.ClassName.String()]; !ok { - continue - } - index.ForEachShard(func(name string, shard ShardLike) error { - m.logShard(shard).Debug("setting fallback mode for shard") - shard.setFallbackToSearchable(true) - return nil - }) - } - - m.log().Debug("finished switching fallback mode") - return nil -} - -func (m *filterableToSearchableMigrator) migrateClass(ctx context.Context, index *Index, - sch *models.Schema, -) (map[string]struct{}, error) { - m.logIndex(index).Debug("started migration of index") - - className := index.Config.ClassName.String() - class, err := ucschema.GetClassByName(sch, className) - if err != nil { - return nil, err - } - - shard2PropsToFix := map[string]map[string]struct{}{} - uniquePropsToFix := map[string]struct{}{} - for _, prop := range class.Properties { - if !(inverted.HasFilterableIndex(prop) && inverted.HasSearchableIndex(prop)) { - continue - } - if err := index.ForEachShard(func(name string, shard ShardLike) error { - if toFix, err := m.isPropToFix(prop, shard); toFix { - if _, ok := shard2PropsToFix[shard.Name()]; !ok { - shard2PropsToFix[shard.Name()] = map[string]struct{}{} - } - shard2PropsToFix[shard.Name()][prop.Name] = struct{}{} - uniquePropsToFix[prop.Name] = struct{}{} - } else if err != nil { - m.logShard(shard).WithError(err).Error("failed discovering props to fix") - return errors.Wrap(err, "failed discovering props to fix") - } - return nil - }); err != nil { - return nil, err - } - } - - m.logIndex(index). - WithField("number_of_props", len(uniquePropsToFix)). - WithField("props", m.uniquePropsToSlice(uniquePropsToFix)). - Debug("found properties to fix") - - if len(uniquePropsToFix) == 0 { - return nil, nil - } - - eg := &errgroup.Group{} - eg.SetLimit(_NUMCPU) - for shardName, props := range shard2PropsToFix { - shard := index.shards.Load(shardName) - props := props - - eg.Go(func() error { - if err := m.migrateShard(ctx, shard, props); err != nil { - m.logShard(shard).WithError(err).Error("failed migrating shard") - return errors.Wrap(err, "failed migrating shard") - } - return nil - }) - } - if err := eg.Wait(); err != nil { - return nil, err - } - - m.logIndex(index).Debug("finished migration of index") - return uniquePropsToFix, nil -} - -func (m *filterableToSearchableMigrator) migrateShard(ctx context.Context, shard ShardLike, - props map[string]struct{}, -) error { - m.logShard(shard).Debug("started migration of shard") - - m.pauseStoreActivity(ctx, shard) - defer m.resumeStoreActivity(ctx, shard) - - for propName := range props { - srcBucketName := helpers.BucketFromPropNameLSM(propName) - dstBucketName := helpers.BucketSearchableFromPropNameLSM(propName) - - m.logShard(shard). - WithField("bucketSrc", srcBucketName). - WithField("bucketDst", dstBucketName). - WithField("prop", propName). - Debug("replacing buckets") - - if err := shard.Store().ReplaceBuckets(ctx, dstBucketName, srcBucketName); err != nil { - return err - } - } - - m.logShard(shard).Debug("finished migration of shard") - return nil -} - -func (m *filterableToSearchableMigrator) isPropToFix(prop *models.Property, shard ShardLike) (bool, error) { - bucketFilterable := shard.Store().Bucket(helpers.BucketFromPropNameLSM(prop.Name)) - if bucketFilterable != nil && - bucketFilterable.Strategy() == lsmkv.StrategyMapCollection && - bucketFilterable.DesiredStrategy() == lsmkv.StrategyRoaringSet { - - bucketSearchable := shard.Store().Bucket(helpers.BucketSearchableFromPropNameLSM(prop.Name)) - if bucketSearchable != nil && - bucketSearchable.Strategy() == lsmkv.StrategyMapCollection { - - if m.isEmptyMapBucket(bucketSearchable) { - return true, nil - } - return false, fmt.Errorf("searchable bucket is not empty") - } else { - return false, fmt.Errorf("searchable bucket should have map strategy") - } - } - return false, nil -} - -func (m *filterableToSearchableMigrator) isEmptyMapBucket(bucket *lsmkv.Bucket) bool { - cur := bucket.MapCursorKeyOnly() - defer cur.Close() - - key, _ := cur.First() - return key == nil -} - -func (m *filterableToSearchableMigrator) pauseStoreActivity( - ctx context.Context, shard ShardLike, -) error { - m.logShard(shard).Debug("pausing store activity") - - if err := shard.Store().PauseCompaction(ctx); err != nil { - return errors.Wrapf(err, "failed pausing compaction for shard '%s'", shard.ID()) - } - if err := shard.Store().FlushMemtables(ctx); err != nil { - return errors.Wrapf(err, "failed flushing memtables for shard '%s'", shard.ID()) - } - shard.Store().UpdateBucketsStatus(storagestate.StatusReadOnly) - - m.logShard(shard).Debug("paused store activity") - return nil -} - -func (m *filterableToSearchableMigrator) resumeStoreActivity( - ctx context.Context, shard ShardLike, -) error { - m.logShard(shard).Debug("resuming store activity") - - if err := shard.Store().ResumeCompaction(ctx); err != nil { - return errors.Wrapf(err, "failed resuming compaction for shard '%s'", shard.ID()) - } - shard.Store().UpdateBucketsStatus(storagestate.StatusReady) - - m.logShard(shard).Debug("resumed store activity") - return nil -} - -func (m *filterableToSearchableMigrator) log() *logrus.Entry { - return m.logger.WithField("action", "inverted filter2search migration") -} - -func (m *filterableToSearchableMigrator) logIndex(index *Index) *logrus.Entry { - return m.log().WithField("index", index.ID()) -} - -func (m *filterableToSearchableMigrator) logShard(shard ShardLike) *logrus.Entry { - return m.logIndex(shard.Index()).WithField("shard", shard.ID()) -} - -func (m *filterableToSearchableMigrator) uniquePropsToSlice(uniqueProps map[string]struct{}) []string { - props := make([]string, 0, len(uniqueProps)) - for prop := range uniqueProps { - props = append(props, prop) - } - return props -} - -type filterableToSearchableMigrationState struct { - MissingFilterableClass2Props map[string]map[string]struct{} - CreatedFilterableClass2Props map[string]map[string]struct{} -} - -type filterableToSearchableMigrationFiles struct { - flagFileName string - stateFileName string -} - -func newFilterableToSearchableMigrationFiles(rootPath string) *filterableToSearchableMigrationFiles { - return &filterableToSearchableMigrationFiles{ - flagFileName: path.Join(rootPath, "migration1.19.filter2search.skip.flag"), - stateFileName: path.Join(rootPath, "migration1.19.filter2search.state"), - } -} - -func (mf *filterableToSearchableMigrationFiles) loadMigrationState() (*filterableToSearchableMigrationState, error) { - f, err := os.OpenFile(mf.stateFileName, os.O_RDWR|os.O_CREATE, 0o666) - if err != nil { - return nil, err - } - defer f.Close() - - buf := new(bytes.Buffer) - if _, err := buf.ReadFrom(f); err != nil { - return nil, err - } - bytes := buf.Bytes() - - state := filterableToSearchableMigrationState{ - MissingFilterableClass2Props: map[string]map[string]struct{}{}, - CreatedFilterableClass2Props: map[string]map[string]struct{}{}, - } - - if len(bytes) > 0 { - if err := json.Unmarshal(bytes, &state); err != nil { - return nil, err - } - } - return &state, nil -} - -func (mf *filterableToSearchableMigrationFiles) saveMigrationState(state *filterableToSearchableMigrationState) error { - bytes, err := json.Marshal(state) - if err != nil { - return err - } - - fileNameTemp := mf.stateFileName + ".temp" - f, err := os.Create(fileNameTemp) - if err != nil { - return err - } - - _, err = f.Write(bytes) - f.Close() - if err != nil { - return err - } - - err = os.Rename(fileNameTemp, mf.stateFileName) - if err != nil { - return err - } - return nil -} - -func (mf *filterableToSearchableMigrationFiles) existsMigrationSkipFlag() bool { - _, err := os.Stat(mf.flagFileName) - return err == nil -} - -func (mf *filterableToSearchableMigrationFiles) createMigrationSkipFlag() error { - f, err := os.Create(mf.flagFileName) - if err != nil { - return err - } - f.Close() - return nil -} diff --git a/adapters/repos/db/inverted_reindexer.go b/adapters/repos/db/inverted_reindexer.go deleted file mode 100644 index 100caffebf34899697fade04edc9634abc814b95..0000000000000000000000000000000000000000 --- a/adapters/repos/db/inverted_reindexer.go +++ /dev/null @@ -1,444 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "context" - "fmt" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/adapters/repos/db/inverted" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/storagestate" - "github.com/weaviate/weaviate/entities/storobj" -) - -type ShardInvertedReindexTask interface { - GetPropertiesToReindex(ctx context.Context, shard ShardLike, - ) ([]ReindexableProperty, error) - // right now only OnResume is needed, but in the future more - // callbacks could be added - // (like OnPrePauseStore, OnPostPauseStore, OnPreResumeStore, etc) - OnPostResumeStore(ctx context.Context, shard ShardLike) error -} - -type ReindexableProperty struct { - PropertyName string - IndexType PropertyIndexType - NewIndex bool // is new index, there is no bucket to replace with - DesiredStrategy string - BucketOptions []lsmkv.BucketOption -} - -type ShardInvertedReindexer struct { - logger logrus.FieldLogger - shard ShardLike - - tasks []ShardInvertedReindexTask - class *models.Class -} - -func NewShardInvertedReindexer(shard ShardLike, logger logrus.FieldLogger) *ShardInvertedReindexer { - class, _ := schema.GetClassByName(shard.Index().getSchema.GetSchemaSkipAuth().Objects, - shard.Index().Config.ClassName.String()) - - return &ShardInvertedReindexer{ - logger: logger, - shard: shard, - tasks: []ShardInvertedReindexTask{}, - class: class, - } -} - -func (r *ShardInvertedReindexer) AddTask(task ShardInvertedReindexTask) { - r.tasks = append(r.tasks, task) -} - -func (r *ShardInvertedReindexer) Do(ctx context.Context) error { - for _, task := range r.tasks { - if err := r.checkContextExpired(ctx, "remaining tasks skipped due to context canceled"); err != nil { - return err - } - if err := r.doTask(ctx, task); err != nil { - return err - } - } - return nil -} - -func (r *ShardInvertedReindexer) doTask(ctx context.Context, task ShardInvertedReindexTask) error { - reindexProperties, err := task.GetPropertiesToReindex(ctx, r.shard) - if err != nil { - r.logError(err, "failed getting reindex properties") - return errors.Wrapf(err, "failed getting reindex properties") - } - if len(reindexProperties) == 0 { - r.logger. - WithField("action", "inverted reindex"). - WithField("index", r.shard.Index().ID()). - WithField("shard", r.shard.ID()). - Debug("no properties to reindex") - return nil - } - - if err := r.checkContextExpired(ctx, "pausing store stopped due to context canceled"); err != nil { - return err - } - - if err := r.pauseStoreActivity(ctx); err != nil { - r.logError(err, "failed pausing store activity") - return err - } - - bucketsToReindex := make([]string, len(reindexProperties)) - for i, reindexProperty := range reindexProperties { - if err := r.checkContextExpired(ctx, "creating temp buckets stopped due to context canceled"); err != nil { - return err - } - - if !isIndexTypeSupportedByStrategy(reindexProperty.IndexType, reindexProperty.DesiredStrategy) { - err := fmt.Errorf("strategy '%s' is not supported for given index type '%d", - reindexProperty.DesiredStrategy, reindexProperty.IndexType) - r.logError(err, "invalid strategy") - return err - } - - // TODO verify if property indeed need reindex before creating buckets - // (is filterable / is searchable / null or prop length index enabled) - bucketsToReindex[i] = r.bucketName(reindexProperty.PropertyName, reindexProperty.IndexType) - if err := r.createTempBucket(ctx, bucketsToReindex[i], reindexProperty.DesiredStrategy, - reindexProperty.BucketOptions...); err != nil { - r.logError(err, "failed creating temporary bucket") - return err - } - r.logger. - WithField("action", "inverted reindex"). - WithField("shard", r.shard.Name()). - WithField("property", reindexProperty.PropertyName). - WithField("strategy", reindexProperty.DesiredStrategy). - WithField("index_type", reindexProperty.IndexType). - Debug("created temporary bucket") - } - - if err := r.reindexProperties(ctx, reindexProperties); err != nil { - r.logError(err, "failed reindexing properties") - return errors.Wrapf(err, "failed reindexing properties on shard '%s'", r.shard.Name()) - } - - for i := range bucketsToReindex { - if err := r.checkContextExpired(ctx, "replacing buckets stopped due to context canceled"); err != nil { - return err - } - tempBucketName := helpers.TempBucketFromBucketName(bucketsToReindex[i]) - tempBucket := r.shard.Store().Bucket(tempBucketName) - tempBucket.FlushMemtable() - tempBucket.UpdateStatus(storagestate.StatusReadOnly) - - if reindexProperties[i].NewIndex { - if err := r.shard.Store().RenameBucket(ctx, tempBucketName, bucketsToReindex[i]); err != nil { - r.logError(err, "failed renaming buckets") - return err - } - - r.logger. - WithField("action", "inverted reindex"). - WithField("shard", r.shard.Name()). - WithField("bucket", bucketsToReindex[i]). - WithField("temp_bucket", tempBucketName). - Debug("renamed bucket") - } else { - if err := r.shard.Store().ReplaceBuckets(ctx, bucketsToReindex[i], tempBucketName); err != nil { - r.logError(err, "failed replacing buckets") - return err - } - - r.logger. - WithField("action", "inverted reindex"). - WithField("shard", r.shard.Name()). - WithField("bucket", bucketsToReindex[i]). - WithField("temp_bucket", tempBucketName). - Debug("replaced buckets") - } - } - - if err := r.checkContextExpired(ctx, "resuming store stopped due to context canceled"); err != nil { - return err - } - - if err := r.resumeStoreActivity(ctx, task); err != nil { - r.logError(err, "failed resuming store activity") - return err - } - - return nil -} - -func (r *ShardInvertedReindexer) pauseStoreActivity(ctx context.Context) error { - if err := r.shard.Store().PauseCompaction(ctx); err != nil { - return errors.Wrapf(err, "failed pausing compaction for shard '%s'", r.shard.Name()) - } - if err := r.shard.Store().FlushMemtables(ctx); err != nil { - return errors.Wrapf(err, "failed flushing memtables for shard '%s'", r.shard.Name()) - } - r.shard.Store().UpdateBucketsStatus(storagestate.StatusReadOnly) - - r.logger. - WithField("action", "inverted reindex"). - WithField("shard", r.shard.Name()). - Debug("paused store activity") - - return nil -} - -func (r *ShardInvertedReindexer) resumeStoreActivity(ctx context.Context, task ShardInvertedReindexTask) error { - if err := r.shard.Store().ResumeCompaction(ctx); err != nil { - return errors.Wrapf(err, "failed resuming compaction for shard '%s'", r.shard.Name()) - } - r.shard.Store().UpdateBucketsStatus(storagestate.StatusReady) - if err := task.OnPostResumeStore(ctx, r.shard); err != nil { - return errors.Wrap(err, "failed OnPostResumeStore") - } - - r.logger. - WithField("action", "inverted reindex"). - WithField("shard", r.shard.Name()). - Debug("resumed store activity") - - return nil -} - -func (r *ShardInvertedReindexer) createTempBucket(ctx context.Context, name string, - strategy string, options ...lsmkv.BucketOption, -) error { - tempName := helpers.TempBucketFromBucketName(name) - bucketOptions := append(options, lsmkv.WithStrategy(strategy)) - - if err := r.shard.Store().CreateBucket(ctx, tempName, bucketOptions...); err != nil { - return errors.Wrapf(err, "failed creating temp bucket '%s'", tempName) - } - return nil -} - -func (r *ShardInvertedReindexer) reindexProperties(ctx context.Context, reindexableProperties []ReindexableProperty) error { - checker := newReindexablePropertyChecker(reindexableProperties, r.class) - objectsBucket := r.shard.Store().Bucket(helpers.ObjectsBucketLSM) - - r.logger. - WithField("action", "inverted reindex"). - WithField("shard", r.shard.Name()). - Debug("starting populating indexes") - - i := 0 - if err := objectsBucket.IterateObjects(ctx, func(object *storobj.Object) error { - // check context expired every 100k objects - if i%100_000 == 0 && i != 0 { - if err := r.checkContextExpired(ctx, "iterating through objects stopped due to context canceled"); err != nil { - return err - } - r.logger. - WithField("action", "inverted reindex"). - WithField("shard", r.shard.Name()). - Debugf("iterating through objects: %d done", i) - } - docID := object.DocID() - properties, nilProperties, err := r.shard.AnalyzeObject(object) - if err != nil { - return errors.Wrapf(err, "failed analyzying object") - } - - for _, property := range properties { - if err := r.handleProperty(ctx, checker, docID, property); err != nil { - return errors.Wrapf(err, "failed reindexing property '%s' of object '%d'", property.Name, docID) - } - } - for _, nilProperty := range nilProperties { - if err := r.handleNilProperty(ctx, checker, docID, nilProperty); err != nil { - return errors.Wrapf(err, "failed reindexing property '%s' of object '%d'", nilProperty.Name, docID) - } - } - - i++ - return nil - }); err != nil { - return err - } - - r.logger. - WithField("action", "inverted reindex"). - WithField("shard", r.shard.Name()). - Debugf("iterating through objects: %d done", i) - - return nil -} - -func (r *ShardInvertedReindexer) handleProperty(ctx context.Context, checker *reindexablePropertyChecker, - docID uint64, property inverted.Property, -) error { - reindexablePropValue := checker.isReindexable(property.Name, IndexTypePropValue) - reindexablePropSearchableValue := checker.isReindexable(property.Name, IndexTypePropSearchableValue) - - if reindexablePropValue || reindexablePropSearchableValue { - schemaProp := checker.getSchemaProp(property.Name) - - var bucketValue, bucketSearchableValue *lsmkv.Bucket - - if reindexablePropValue { - bucketValue = r.tempBucket(property.Name, IndexTypePropValue) - if bucketValue == nil { - return fmt.Errorf("no bucket for prop '%s' value found", property.Name) - } - } - if reindexablePropSearchableValue { - bucketSearchableValue = r.tempBucket(property.Name, IndexTypePropSearchableValue) - if bucketSearchableValue == nil { - return fmt.Errorf("no bucket searchable for prop '%s' value found", property.Name) - } - } - - propLen := float32(len(property.Items)) - for _, item := range property.Items { - key := item.Data - if reindexablePropSearchableValue && inverted.HasSearchableIndex(schemaProp) { - pair := r.shard.pairPropertyWithFrequency(docID, item.TermFrequency, propLen) - if err := r.shard.addToPropertyMapBucket(bucketSearchableValue, pair, key); err != nil { - return errors.Wrapf(err, "failed adding to prop '%s' value bucket", property.Name) - } - } - if reindexablePropValue && inverted.HasFilterableIndex(schemaProp) { - if err := r.shard.addToPropertySetBucket(bucketValue, docID, key); err != nil { - return errors.Wrapf(err, "failed adding to prop '%s' value bucket", property.Name) - } - } - } - } - - // add non-nil properties to the null-state inverted index, - // but skip internal properties (__meta_count, _id etc) - if isMetaCountProperty(property) || isInternalProperty(property) { - return nil - } - - // properties where defining a length does not make sense (floats etc.) have a negative entry as length - if r.shard.Index().invertedIndexConfig.IndexPropertyLength && property.Length >= 0 { - key, err := bucketKeyPropertyLength(property.Length) - if err != nil { - return errors.Wrapf(err, "failed creating key for prop '%s' length", property.Name) - } - if checker.isReindexable(property.Name, IndexTypePropLength) { - bucketLength := r.tempBucket(property.Name, IndexTypePropLength) - if bucketLength == nil { - return fmt.Errorf("no bucket for prop '%s' length found", property.Name) - } - if err := r.shard.addToPropertySetBucket(bucketLength, docID, key); err != nil { - return errors.Wrapf(err, "failed adding to prop '%s' length bucket", property.Name) - } - } - } - - if r.shard.Index().invertedIndexConfig.IndexNullState { - key, err := bucketKeyPropertyNull(property.Length == 0) - if err != nil { - return errors.Wrapf(err, "failed creating key for prop '%s' null", property.Name) - } - if checker.isReindexable(property.Name, IndexTypePropNull) { - bucketNull := r.tempBucket(property.Name, IndexTypePropNull) - if bucketNull == nil { - return fmt.Errorf("no bucket for prop '%s' null found", property.Name) - } - if err := r.shard.addToPropertySetBucket(bucketNull, docID, key); err != nil { - return errors.Wrapf(err, "failed adding to prop '%s' null bucket", property.Name) - } - } - } - - return nil -} - -func (r *ShardInvertedReindexer) handleNilProperty(ctx context.Context, checker *reindexablePropertyChecker, - docID uint64, nilProperty inverted.NilProperty, -) error { - if r.shard.Index().invertedIndexConfig.IndexPropertyLength && nilProperty.AddToPropertyLength { - key, err := bucketKeyPropertyLength(0) - if err != nil { - return errors.Wrapf(err, "failed creating key for prop '%s' length", nilProperty.Name) - } - if checker.isReindexable(nilProperty.Name, IndexTypePropLength) { - bucketLength := r.tempBucket(nilProperty.Name, IndexTypePropLength) - if bucketLength == nil { - return fmt.Errorf("no bucket for prop '%s' length found", nilProperty.Name) - } - if err := r.shard.addToPropertySetBucket(bucketLength, docID, key); err != nil { - return errors.Wrapf(err, "failed adding to prop '%s' length bucket", nilProperty.Name) - } - } - } - - if r.shard.Index().invertedIndexConfig.IndexNullState { - key, err := bucketKeyPropertyNull(true) - if err != nil { - return errors.Wrapf(err, "failed creating key for prop '%s' null", nilProperty.Name) - } - if checker.isReindexable(nilProperty.Name, IndexTypePropNull) { - bucketNull := r.tempBucket(nilProperty.Name, IndexTypePropNull) - if bucketNull == nil { - return fmt.Errorf("no bucket for prop '%s' null found", nilProperty.Name) - } - if err := r.shard.addToPropertySetBucket(bucketNull, docID, key); err != nil { - return errors.Wrapf(err, "failed adding to prop '%s' null bucket", nilProperty.Name) - } - } - } - - return nil -} - -func (r *ShardInvertedReindexer) bucketName(propName string, indexType PropertyIndexType) string { - checkSupportedPropertyIndexType(indexType) - - switch indexType { - case IndexTypePropValue: - return helpers.BucketFromPropNameLSM(propName) - case IndexTypePropSearchableValue: - return helpers.BucketSearchableFromPropNameLSM(propName) - case IndexTypePropLength: - return helpers.BucketFromPropNameLengthLSM(propName) - case IndexTypePropNull: - return helpers.BucketFromPropNameNullLSM(propName) - default: - return "" - } -} - -func (r *ShardInvertedReindexer) tempBucket(propName string, indexType PropertyIndexType) *lsmkv.Bucket { - tempBucketName := helpers.TempBucketFromBucketName(r.bucketName(propName, indexType)) - return r.shard.Store().Bucket(tempBucketName) -} - -func (r *ShardInvertedReindexer) checkContextExpired(ctx context.Context, msg string) error { - if ctx.Err() != nil { - r.logError(ctx.Err(), msg) - return errors.Wrapf(ctx.Err(), msg) - } - return nil -} - -func (r *ShardInvertedReindexer) logError(err error, msg string, args ...interface{}) { - r.logger. - WithField("action", "inverted reindex"). - WithField("shard", r.shard.Name()). - WithError(err). - Errorf(msg, args...) -} diff --git a/adapters/repos/db/inverted_reindexer_index_types.go b/adapters/repos/db/inverted_reindexer_index_types.go deleted file mode 100644 index 0e6990bf0b751fd01e6a3e6577f384079793cda7..0000000000000000000000000000000000000000 --- a/adapters/repos/db/inverted_reindexer_index_types.go +++ /dev/null @@ -1,55 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import "github.com/weaviate/weaviate/adapters/repos/db/lsmkv" - -type PropertyIndexType uint8 - -const ( - IndexTypePropValue PropertyIndexType = iota + 1 - IndexTypePropLength - IndexTypePropNull - IndexTypePropSearchableValue -) - -func isSupportedPropertyIndexType(indexType PropertyIndexType) bool { - switch indexType { - case IndexTypePropValue, - IndexTypePropLength, - IndexTypePropNull, - IndexTypePropSearchableValue: - return true - default: - return false - } -} - -func checkSupportedPropertyIndexType(indexType PropertyIndexType) { - if !isSupportedPropertyIndexType(indexType) { - panic("unsupported property index type") - } -} - -// Some index types are supported by specific strategies only -// Method ensures both index type and strategy work together -func isIndexTypeSupportedByStrategy(indexType PropertyIndexType, strategy string) bool { - switch indexType { - case IndexTypePropLength, - IndexTypePropNull, - IndexTypePropValue: - return lsmkv.IsExpectedStrategy(strategy, lsmkv.StrategySetCollection, lsmkv.StrategyRoaringSet) - case IndexTypePropSearchableValue: - return lsmkv.IsExpectedStrategy(strategy, lsmkv.StrategyMapCollection) - } - return false -} diff --git a/adapters/repos/db/inverted_reindexer_missing_text_filterable.go b/adapters/repos/db/inverted_reindexer_missing_text_filterable.go deleted file mode 100644 index 60f0ff0f3659c2afe0b0548f674a57be7a1eab7c..0000000000000000000000000000000000000000 --- a/adapters/repos/db/inverted_reindexer_missing_text_filterable.go +++ /dev/null @@ -1,120 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "context" - "sync" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv" -) - -type shardInvertedReindexTaskMissingTextFilterable struct { - logger logrus.FieldLogger - files *filterableToSearchableMigrationFiles - stateLock *sync.RWMutex - - migrationState *filterableToSearchableMigrationState -} - -func newShardInvertedReindexTaskMissingTextFilterable(migrator *Migrator, -) *shardInvertedReindexTaskMissingTextFilterable { - return &shardInvertedReindexTaskMissingTextFilterable{ - logger: migrator.logger, - files: newFilterableToSearchableMigrationFiles(migrator.db.config.RootPath), - stateLock: new(sync.RWMutex), - } -} - -func (t *shardInvertedReindexTaskMissingTextFilterable) init() error { - migrationState, err := t.files.loadMigrationState() - if err != nil { - return errors.Wrap(err, "failed loading migration state") - } - - t.migrationState = migrationState - return nil -} - -func (t *shardInvertedReindexTaskMissingTextFilterable) GetPropertiesToReindex(ctx context.Context, - shard ShardLike, -) ([]ReindexableProperty, error) { - reindexableProperties := []ReindexableProperty{} - - t.stateLock.RLock() - className := shard.Index().Config.ClassName.String() - props, ok := t.migrationState.MissingFilterableClass2Props[className] - t.stateLock.RUnlock() - - if !ok || len(props) == 0 { - return reindexableProperties, nil - } - - bucketOptions := []lsmkv.BucketOption{ - lsmkv.WithIdleThreshold(time.Duration(shard.Index().Config.MemtablesFlushIdleAfter) * time.Second), - } - - for propName := range props { - bucketNameSearchable := helpers.BucketSearchableFromPropNameLSM(propName) - bucketNameFilterable := helpers.BucketFromPropNameLSM(propName) - - bucketSearchable := shard.Store().Bucket(bucketNameSearchable) - bucketFilterable := shard.Store().Bucket(bucketNameFilterable) - - // exists bucket searchable of strategy map and either of - // - exists empty filterable bucket of strategy roaring set - // (weaviate was restrated after filterable to searchable migration) - // - filterable bucket does not exist - // (indexing comes right after filterable to searchable migration) - if bucketSearchable != nil && - bucketSearchable.Strategy() == lsmkv.StrategyMapCollection { - - if bucketFilterable == nil { - reindexableProperties = append(reindexableProperties, ReindexableProperty{ - PropertyName: propName, - IndexType: IndexTypePropValue, - DesiredStrategy: lsmkv.StrategyRoaringSet, - NewIndex: true, - BucketOptions: bucketOptions, - }) - } else if bucketFilterable.Strategy() == lsmkv.StrategyRoaringSet { - reindexableProperties = append(reindexableProperties, ReindexableProperty{ - PropertyName: propName, - IndexType: IndexTypePropValue, - DesiredStrategy: lsmkv.StrategyRoaringSet, - BucketOptions: bucketOptions, - }) - } - } - } - - return reindexableProperties, nil -} - -func (t *shardInvertedReindexTaskMissingTextFilterable) updateMigrationStateAndSave(classCreatedFilterable string) error { - t.stateLock.Lock() - defer t.stateLock.Unlock() - - t.migrationState.CreatedFilterableClass2Props[classCreatedFilterable] = t.migrationState.MissingFilterableClass2Props[classCreatedFilterable] - delete(t.migrationState.MissingFilterableClass2Props, classCreatedFilterable) - return t.files.saveMigrationState(t.migrationState) -} - -func (t *shardInvertedReindexTaskMissingTextFilterable) OnPostResumeStore(ctx context.Context, shard ShardLike) error { - // turn off fallback mode immediately after creating filterable index and resuming store's activity - shard.setFallbackToSearchable(false) - return nil -} diff --git a/adapters/repos/db/inverted_reindexer_set_to_roaringset.go b/adapters/repos/db/inverted_reindexer_set_to_roaringset.go deleted file mode 100644 index 1eeb31cc3e724190132b19144c872187ef872bcd..0000000000000000000000000000000000000000 --- a/adapters/repos/db/inverted_reindexer_set_to_roaringset.go +++ /dev/null @@ -1,76 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "context" - "time" - - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv" -) - -type ShardInvertedReindexTaskSetToRoaringSet struct{} - -func (t *ShardInvertedReindexTaskSetToRoaringSet) GetPropertiesToReindex(ctx context.Context, - shard ShardLike, -) ([]ReindexableProperty, error) { - reindexableProperties := []ReindexableProperty{} - - bucketOptions := []lsmkv.BucketOption{ - lsmkv.WithIdleThreshold(time.Duration(shard.Index().Config.MemtablesFlushIdleAfter) * time.Second), - } - - for name, bucket := range shard.Store().GetBucketsByName() { - if bucket.Strategy() == lsmkv.StrategySetCollection && - bucket.DesiredStrategy() == lsmkv.StrategyRoaringSet { - - propName, indexType := GetPropNameAndIndexTypeFromBucketName(name) - switch indexType { - case IndexTypePropValue: - reindexableProperties = append(reindexableProperties, - ReindexableProperty{ - PropertyName: propName, - IndexType: IndexTypePropValue, - DesiredStrategy: lsmkv.StrategyRoaringSet, - BucketOptions: bucketOptions, - }, - ) - case IndexTypePropLength: - reindexableProperties = append(reindexableProperties, - ReindexableProperty{ - PropertyName: propName, - IndexType: IndexTypePropLength, - DesiredStrategy: lsmkv.StrategyRoaringSet, - BucketOptions: bucketOptions, - }, - ) - case IndexTypePropNull: - reindexableProperties = append(reindexableProperties, - ReindexableProperty{ - PropertyName: propName, - IndexType: IndexTypePropNull, - DesiredStrategy: lsmkv.StrategyRoaringSet, - BucketOptions: bucketOptions, - }, - ) - default: - // skip remaining - } - } - } - - return reindexableProperties, nil -} - -func (t *ShardInvertedReindexTaskSetToRoaringSet) OnPostResumeStore(ctx context.Context, shard ShardLike) error { - return nil -} diff --git a/adapters/repos/db/inverted_reindexer_utils.go b/adapters/repos/db/inverted_reindexer_utils.go deleted file mode 100644 index 363b252a193be08b74afa3efdf9a03a4144af8c3..0000000000000000000000000000000000000000 --- a/adapters/repos/db/inverted_reindexer_utils.go +++ /dev/null @@ -1,88 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "regexp" - - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" -) - -func GetPropNameAndIndexTypeFromBucketName(bucketName string) (string, PropertyIndexType) { - propRegexpGroup := "(?P.*)" - - types := []struct { - indexType PropertyIndexType - bucketNameFn func(string) string - }{ - { - IndexTypePropNull, - helpers.BucketFromPropNameNullLSM, - }, - { - IndexTypePropLength, - helpers.BucketFromPropNameLengthLSM, - }, - { - IndexTypePropSearchableValue, - helpers.BucketSearchableFromPropNameLSM, - }, - { - IndexTypePropValue, - helpers.BucketFromPropNameLSM, - }, - } - - for _, t := range types { - r, err := regexp.Compile("^" + t.bucketNameFn(propRegexpGroup) + "$") - if err != nil { - continue - } - matches := r.FindStringSubmatch(bucketName) - if len(matches) > 0 { - return matches[r.SubexpIndex("propName")], t.indexType - } - } - return "", 0 -} - -type reindexablePropertyChecker struct { - reindexables map[string]map[PropertyIndexType]struct{} - props map[string]*models.Property -} - -func newReindexablePropertyChecker(reindexableProperties []ReindexableProperty, class *models.Class) *reindexablePropertyChecker { - reindexables := map[string]map[PropertyIndexType]struct{}{} - props := map[string]*models.Property{} - for _, property := range reindexableProperties { - if _, ok := reindexables[property.PropertyName]; !ok { - reindexables[property.PropertyName] = map[PropertyIndexType]struct{}{} - } - reindexables[property.PropertyName][property.IndexType] = struct{}{} - props[property.PropertyName], _ = schema.GetPropertyByName(class, property.PropertyName) - } - return &reindexablePropertyChecker{reindexables, props} -} - -func (c *reindexablePropertyChecker) isReindexable(propName string, indexType PropertyIndexType) bool { - if _, ok := c.reindexables[propName]; ok { - _, ok := c.reindexables[propName][indexType] - return ok - } - return false -} - -func (c *reindexablePropertyChecker) getSchemaProp(propName string) *models.Property { - return c.props[propName] -} diff --git a/adapters/repos/db/lsmkv/.gitignore b/adapters/repos/db/lsmkv/.gitignore deleted file mode 100644 index 401de141bbf5f9ec284d2ff0412feed3b083a61f..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/.gitignore +++ /dev/null @@ -1 +0,0 @@ -my-bucket diff --git a/adapters/repos/db/lsmkv/binary_search_tree.go b/adapters/repos/db/lsmkv/binary_search_tree.go deleted file mode 100644 index 9a91616f319f76c4bb1404d8e8706f22da599ae0..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/binary_search_tree.go +++ /dev/null @@ -1,380 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "bytes" - - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv/rbtree" - "github.com/weaviate/weaviate/entities/lsmkv" -) - -type binarySearchTree struct { - root *binarySearchNode -} - -// returns net additions of insert in bytes, and previous secondary keys -func (t *binarySearchTree) insert(key, value []byte, secondaryKeys [][]byte) (int, [][]byte) { - if t.root == nil { - t.root = &binarySearchNode{ - key: key, - value: value, - secondaryKeys: secondaryKeys, - colourIsRed: false, // root node is always black - } - return len(key) + len(value), nil - } - - addition, newRoot, previousSecondaryKeys := t.root.insert(key, value, secondaryKeys) - if newRoot != nil { - t.root = newRoot - } - t.root.colourIsRed = false // Can be flipped in the process of balancing, but root is always black - - return addition, previousSecondaryKeys -} - -func (t *binarySearchTree) get(key []byte) ([]byte, error) { - if t.root == nil { - return nil, lsmkv.NotFound - } - - return t.root.get(key) -} - -func (t *binarySearchTree) setTombstone(key []byte, secondaryKeys [][]byte) { - if t.root == nil { - // we need to actively insert a node with a tombstone, even if this node is - // not present because we still need to propagate the delete into the disk - // segments. It could refer to an entity which was created in a previous - // segment and is thus unknown to this memtable - t.root = &binarySearchNode{ - key: key, - value: nil, - tombstone: true, - secondaryKeys: secondaryKeys, - colourIsRed: false, // root node is always black - } - return - } - - newRoot := t.root.setTombstone(key, secondaryKeys) - if newRoot != nil { - t.root = newRoot - } - t.root.colourIsRed = false // Can be flipped in the process of balancing, but root is always black -} - -func (t *binarySearchTree) flattenInOrder() []*binarySearchNode { - if t.root == nil { - return nil - } - - return t.root.flattenInOrder() -} - -type countStats struct { - upsertKeys [][]byte - tombstonedKeys [][]byte -} - -func (c *countStats) hasUpsert(needle []byte) bool { - if c == nil { - return false - } - - for _, hay := range c.upsertKeys { - if bytes.Equal(needle, hay) { - return true - } - } - - return false -} - -func (c *countStats) hasTombstone(needle []byte) bool { - if c == nil { - return false - } - - for _, hay := range c.tombstonedKeys { - if bytes.Equal(needle, hay) { - return true - } - } - - return false -} - -func (t *binarySearchTree) countStats() *countStats { - stats := &countStats{} - if t.root == nil { - return stats - } - - t.root.countStats(stats) - return stats -} - -type binarySearchNode struct { - key []byte - value []byte - secondaryKeys [][]byte - left *binarySearchNode - right *binarySearchNode - parent *binarySearchNode - tombstone bool - colourIsRed bool -} - -func (n *binarySearchNode) Parent() rbtree.Node { - if n == nil { - return nil - } - return n.parent -} - -func (n *binarySearchNode) SetParent(parent rbtree.Node) { - if n == nil { - addNewSearchNodeReceiver(&n) - } - - if parent == nil { - n.parent = nil - return - } - - n.parent = parent.(*binarySearchNode) -} - -func (n *binarySearchNode) Left() rbtree.Node { - if n == nil { - return nil - } - return n.left -} - -func (n *binarySearchNode) SetLeft(left rbtree.Node) { - if n == nil { - addNewSearchNodeReceiver(&n) - } - - if left == nil { - n.left = nil - return - } - - n.left = left.(*binarySearchNode) -} - -func (n *binarySearchNode) Right() rbtree.Node { - if n == nil { - return nil - } - return n.right -} - -func (n *binarySearchNode) SetRight(right rbtree.Node) { - if n == nil { - addNewSearchNodeReceiver(&n) - } - - if right == nil { - n.right = nil - return - } - - n.right = right.(*binarySearchNode) -} - -func (n *binarySearchNode) IsRed() bool { - if n == nil { - return false - } - return n.colourIsRed -} - -func (n *binarySearchNode) SetRed(isRed bool) { - n.colourIsRed = isRed -} - -func (n *binarySearchNode) IsNil() bool { - return n == nil -} - -func addNewSearchNodeReceiver(nodePtr **binarySearchNode) { - *nodePtr = &binarySearchNode{} -} - -// returns net additions of insert in bytes -func (n *binarySearchNode) insert(key, value []byte, secondaryKeys [][]byte) (netAdditions int, newRoot *binarySearchNode, previousSecondaryKeys [][]byte) { - if bytes.Equal(key, n.key) { - // since the key already exists, we only need to take the difference - // between the existing value and the new one to determine net change - netAdditions = len(n.value) - len(value) - if netAdditions < 0 { - netAdditions *= -1 - } - - // assign new value to node - n.value = value - - // reset tombstone in case it had one - n.tombstone = false - previousSecondaryKeys = n.secondaryKeys - n.secondaryKeys = secondaryKeys - - newRoot = nil // tree root does not change when replacing node - return - } - - if bytes.Compare(key, n.key) < 0 { - if n.left != nil { - netAdditions, newRoot, previousSecondaryKeys = n.left.insert(key, value, secondaryKeys) - return - } else { - n.left = &binarySearchNode{ - key: key, - value: value, - secondaryKeys: secondaryKeys, - parent: n, - colourIsRed: true, // new nodes are always red, except root node which is handled in the tree itself - } - newRoot = binarySearchNodeFromRB(rbtree.Rebalance(n.left)) - netAdditions = len(key) + len(value) - return - } - } else { - if n.right != nil { - netAdditions, newRoot, previousSecondaryKeys = n.right.insert(key, value, secondaryKeys) - return - } else { - n.right = &binarySearchNode{ - key: key, - value: value, - secondaryKeys: secondaryKeys, - parent: n, - colourIsRed: true, - } - netAdditions = len(key) + len(value) - newRoot = binarySearchNodeFromRB(rbtree.Rebalance(n.right)) - return - } - } -} - -func (n *binarySearchNode) get(key []byte) ([]byte, error) { - if bytes.Equal(n.key, key) { - if !n.tombstone { - return n.value, nil - } else { - return nil, lsmkv.Deleted - } - } - - if bytes.Compare(key, n.key) < 0 { - if n.left == nil { - return nil, lsmkv.NotFound - } - - return n.left.get(key) - } else { - if n.right == nil { - return nil, lsmkv.NotFound - } - - return n.right.get(key) - } -} - -func (n *binarySearchNode) setTombstone(key []byte, secondaryKeys [][]byte) *binarySearchNode { - if bytes.Equal(n.key, key) { - n.value = nil - n.tombstone = true - n.secondaryKeys = secondaryKeys - return nil - } - - if bytes.Compare(key, n.key) < 0 { - if n.left == nil { - n.left = &binarySearchNode{ - key: key, - value: nil, - tombstone: true, - secondaryKeys: secondaryKeys, - parent: n, - colourIsRed: true, - } - return binarySearchNodeFromRB(rbtree.Rebalance(n.left)) - - } - return n.left.setTombstone(key, secondaryKeys) - } else { - if n.right == nil { - n.right = &binarySearchNode{ - key: key, - value: nil, - tombstone: true, - secondaryKeys: secondaryKeys, - parent: n, - colourIsRed: true, - } - return binarySearchNodeFromRB(rbtree.Rebalance(n.right)) - } - return n.right.setTombstone(key, secondaryKeys) - } -} - -func (n *binarySearchNode) flattenInOrder() []*binarySearchNode { - var left []*binarySearchNode - var right []*binarySearchNode - - if n.left != nil { - left = n.left.flattenInOrder() - } - - if n.right != nil { - right = n.right.flattenInOrder() - } - - right = append([]*binarySearchNode{n}, right...) - return append(left, right...) -} - -// This is not very allocation friendly, since we basically need to allocate -// once for each element in the memtable. However, these results can -// potentially be cached, as we don't care about the intermediary results, just -// the net additions. -func (n *binarySearchNode) countStats(stats *countStats) { - if n.tombstone { - stats.tombstonedKeys = append(stats.tombstonedKeys, n.key) - } else { - stats.upsertKeys = append(stats.upsertKeys, n.key) - } - - if n.left != nil { - n.left.countStats(stats) - } - - if n.right != nil { - n.right.countStats(stats) - } -} - -func binarySearchNodeFromRB(rbNode rbtree.Node) (bsNode *binarySearchNode) { - if rbNode == nil { - bsNode = nil - return - } - bsNode = rbNode.(*binarySearchNode) - return -} diff --git a/adapters/repos/db/lsmkv/binary_search_tree_map.go b/adapters/repos/db/lsmkv/binary_search_tree_map.go deleted file mode 100644 index f695e9aca60e670014797bdfdbd4ef46b628eed6..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/binary_search_tree_map.go +++ /dev/null @@ -1,260 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "bytes" - "sort" - - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv/rbtree" - "github.com/weaviate/weaviate/entities/lsmkv" -) - -type binarySearchTreeMap struct { - root *binarySearchNodeMap -} - -func (t *binarySearchTreeMap) insert(key []byte, pair MapPair) { - if t.root == nil { - t.root = &binarySearchNodeMap{ - key: key, - values: []MapPair{pair}, - colourIsRed: false, // root node is always black - } - return - } - - if newRoot := t.root.insert(key, pair); newRoot != nil { - t.root = newRoot - } - t.root.colourIsRed = false // Can be flipped in the process of balancing, but root is always black -} - -func (t *binarySearchTreeMap) get(key []byte) ([]MapPair, error) { - if t.root == nil { - return nil, lsmkv.NotFound - } - - return t.root.get(key) -} - -func (t *binarySearchTreeMap) flattenInOrder() []*binarySearchNodeMap { - if t.root == nil { - return nil - } - - return t.root.flattenInOrder() -} - -type binarySearchNodeMap struct { - key []byte - values []MapPair - left *binarySearchNodeMap - right *binarySearchNodeMap - parent *binarySearchNodeMap - colourIsRed bool -} - -func (n *binarySearchNodeMap) Parent() rbtree.Node { - if n == nil { - return nil - } - return n.parent -} - -func (n *binarySearchNodeMap) SetParent(parent rbtree.Node) { - if n == nil { - addNewSearchNodeMapReceiver(&n) - } - - if parent == nil { - n.parent = nil - return - } - - n.parent = parent.(*binarySearchNodeMap) -} - -func (n *binarySearchNodeMap) Left() rbtree.Node { - if n == nil { - return nil - } - return n.left -} - -func (n *binarySearchNodeMap) SetLeft(left rbtree.Node) { - if n == nil { - addNewSearchNodeMapReceiver(&n) - } - - if left == nil { - n.left = nil - return - } - - n.left = left.(*binarySearchNodeMap) -} - -func (n *binarySearchNodeMap) Right() rbtree.Node { - if n == nil { - return nil - } - return n.right -} - -func (n *binarySearchNodeMap) SetRight(right rbtree.Node) { - if n == nil { - addNewSearchNodeMapReceiver(&n) - } - - if right == nil { - n.right = nil - return - } - - n.right = right.(*binarySearchNodeMap) -} - -func (n *binarySearchNodeMap) IsRed() bool { - if n == nil { - return false - } - return n.colourIsRed -} - -func (n *binarySearchNodeMap) SetRed(isRed bool) { - n.colourIsRed = isRed -} - -func (n *binarySearchNodeMap) IsNil() bool { - return n == nil -} - -func addNewSearchNodeMapReceiver(nodePtr **binarySearchNodeMap) { - *nodePtr = &binarySearchNodeMap{} -} - -func (n *binarySearchNodeMap) insert(key []byte, pair MapPair) *binarySearchNodeMap { - if bytes.Equal(key, n.key) { - n.values = append(n.values, pair) - return nil // tree root does not change when replacing node - } - - if bytes.Compare(key, n.key) < 0 { - if n.left != nil { - return n.left.insert(key, pair) - } else { - n.left = &binarySearchNodeMap{ - key: key, - parent: n, - colourIsRed: true, - values: []MapPair{pair}, - } - return binarySearchNodeMapFromRB(rbtree.Rebalance(n.left)) - } - } else { - if n.right != nil { - return n.right.insert(key, pair) - } else { - n.right = &binarySearchNodeMap{ - key: key, - parent: n, - colourIsRed: true, - values: []MapPair{pair}, - } - return binarySearchNodeMapFromRB(rbtree.Rebalance(n.right)) - } - } -} - -func (n *binarySearchNodeMap) get(key []byte) ([]MapPair, error) { - if bytes.Equal(n.key, key) { - return sortAndDedupValues(n.values), nil - } - - if bytes.Compare(key, n.key) < 0 { - if n.left == nil { - return nil, lsmkv.NotFound - } - - return n.left.get(key) - } else { - if n.right == nil { - return nil, lsmkv.NotFound - } - - return n.right.get(key) - } -} - -func (n *binarySearchNodeMap) flattenInOrder() []*binarySearchNodeMap { - var left []*binarySearchNodeMap - var right []*binarySearchNodeMap - - if n.left != nil { - left = n.left.flattenInOrder() - } - - if n.right != nil { - right = n.right.flattenInOrder() - } - - // the values are sorted on read for performance reasons, the assumption is - // that while a memtable is open writes a much more common, thus we write map - // KVs unsorted and only sort/dedup them on read. - right = append([]*binarySearchNodeMap{{ - key: n.key, - values: sortAndDedupValues(n.values), - colourIsRed: n.colourIsRed, - }}, right...) - return append(left, right...) -} - -// takes a list of MapPair and sorts it while keeping the original order. Then -// removes redundancies (from updates or deletes after previous inserts) using -// a simple deduplication process. -func sortAndDedupValues(in []MapPair) []MapPair { - out := make([]MapPair, len(in)) - copy(out, in) - - // use SliceStable so that we keep the insert order on duplicates. This is - // important because otherwise we can't dedup them correctly if we don't know - // in which order they came in. - sort.SliceStable(out, func(a, b int) bool { - return bytes.Compare(out[a].Key, out[b].Key) < 0 - }) - - // now deduping is as simple as looking one key ahead - if it's the same key - // simply skip the current element. Meaning "out" will be a subset of - // (sorted) "in". - outIndex := 0 - for inIndex, pair := range out { - // look ahead - if inIndex+1 < len(out) && bytes.Equal(out[inIndex+1].Key, pair.Key) { - continue - } - - out[outIndex] = pair - outIndex++ - } - - return out[:outIndex] -} - -func binarySearchNodeMapFromRB(rbNode rbtree.Node) (bsNode *binarySearchNodeMap) { - if rbNode == nil { - bsNode = nil - return - } - bsNode = rbNode.(*binarySearchNodeMap) - return -} diff --git a/adapters/repos/db/lsmkv/binary_search_tree_map_test.go b/adapters/repos/db/lsmkv/binary_search_tree_map_test.go deleted file mode 100644 index 0db5d56aa5edbea76b6758aa5b9b5b41dab9e3ec..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/binary_search_tree_map_test.go +++ /dev/null @@ -1,212 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func Test_BinarySearchTreeMap(t *testing.T) { - t.Run("single row key, single map key", func(t *testing.T) { - tree := &binarySearchTreeMap{} - rowKey := []byte("rowkey") - - pair1 := MapPair{ - Key: []byte("map-key-1"), - Value: []byte("map-value-1"), - } - - tree.insert(rowKey, pair1) - - res, err := tree.get(rowKey) - require.Nil(t, err) - assert.Equal(t, []MapPair{ - { - Key: []byte("map-key-1"), - Value: []byte("map-value-1"), - }, - }, res) - }) - - t.Run("single row key, updated map value", func(t *testing.T) { - tree := &binarySearchTreeMap{} - rowKey := []byte("rowkey") - - tree.insert(rowKey, MapPair{ - Key: []byte("c"), - Value: []byte("c1"), - }) - - tree.insert(rowKey, MapPair{ - Key: []byte("a"), - Value: []byte("a1"), - }) - - tree.insert(rowKey, MapPair{ - Key: []byte("b"), - Value: []byte("b1"), - }) - - tree.insert(rowKey, MapPair{ - Key: []byte("b"), - Value: []byte("b2"), - }) - - tree.insert(rowKey, MapPair{ - Key: []byte("a"), - Value: []byte("a2"), - }) - - res, err := tree.get(rowKey) - require.Nil(t, err) - assert.Equal(t, []MapPair{ - { - Key: []byte("a"), - Value: []byte("a2"), - }, - { - Key: []byte("b"), - Value: []byte("b2"), - }, - { - Key: []byte("c"), - Value: []byte("c1"), - }, - }, res) - }) - - t.Run("two row keys, updated map value", func(t *testing.T) { - tree := &binarySearchTreeMap{} - rowKey1 := []byte("rowkey") - rowKey2 := []byte("other-rowkey") - - tree.insert(rowKey1, MapPair{ - Key: []byte("c"), - Value: []byte("c1"), - }) - - tree.insert(rowKey1, MapPair{ - Key: []byte("a"), - Value: []byte("a1"), - }) - - tree.insert(rowKey2, MapPair{ - Key: []byte("z"), - Value: []byte("z1"), - }) - - tree.insert(rowKey1, MapPair{ - Key: []byte("b"), - Value: []byte("b1"), - }) - - tree.insert(rowKey2, MapPair{ - Key: []byte("x"), - Value: []byte("x1"), - }) - - tree.insert(rowKey1, MapPair{ - Key: []byte("b"), - Value: []byte("b2"), - }) - - tree.insert(rowKey1, MapPair{ - Key: []byte("a"), - Value: []byte("a2"), - }) - - tree.insert(rowKey2, MapPair{ - Key: []byte("x"), - Value: []byte("x2"), - }) - - res, err := tree.get(rowKey1) - require.Nil(t, err) - assert.Equal(t, []MapPair{ - { - Key: []byte("a"), - Value: []byte("a2"), - }, - { - Key: []byte("b"), - Value: []byte("b2"), - }, - { - Key: []byte("c"), - Value: []byte("c1"), - }, - }, res) - - res, err = tree.get(rowKey2) - require.Nil(t, err) - assert.Equal(t, []MapPair{ - { - Key: []byte("x"), - Value: []byte("x2"), - }, - { - Key: []byte("z"), - Value: []byte("z1"), - }, - }, res) - }) - - t.Run("single row key, deleted map values", func(t *testing.T) { - tree := &binarySearchTreeMap{} - rowKey := []byte("rowkey") - - tree.insert(rowKey, MapPair{ - Key: []byte("c"), - Value: []byte("c1"), - }) - - tree.insert(rowKey, MapPair{ - Key: []byte("a"), - Value: []byte("a1"), - }) - - tree.insert(rowKey, MapPair{ - Key: []byte("b"), - Value: []byte("b1"), - }) - - tree.insert(rowKey, MapPair{ - Key: []byte("b"), - Tombstone: true, - }) - - tree.insert(rowKey, MapPair{ - Key: []byte("a"), - Tombstone: true, - }) - - res, err := tree.get(rowKey) - require.Nil(t, err) - assert.Equal(t, []MapPair{ - { - Key: []byte("a"), - Tombstone: true, - }, - { - Key: []byte("b"), - Tombstone: true, - }, - { - Key: []byte("c"), - Value: []byte("c1"), - }, - }, res) - }) -} diff --git a/adapters/repos/db/lsmkv/binary_search_tree_multi.go b/adapters/repos/db/lsmkv/binary_search_tree_multi.go deleted file mode 100644 index d808e5a46d3b27e6535eb736971dfa19579da552..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/binary_search_tree_multi.go +++ /dev/null @@ -1,276 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "bytes" - - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv/rbtree" - "github.com/weaviate/weaviate/entities/lsmkv" -) - -type binarySearchTreeMulti struct { - root *binarySearchNodeMulti -} - -type value struct { - value []byte - tombstone bool -} - -func (t *binarySearchTreeMulti) insert(key []byte, values []value) { - if t.root == nil { - t.root = &binarySearchNodeMulti{ - key: key, - values: values, - colourIsRed: false, // root node is always black - } - return - } - - if newRoot := t.root.insert(key, values); newRoot != nil { - t.root = newRoot - } - t.root.colourIsRed = false // Can be flipped in the process of balancing, but root is always black -} - -func (t *binarySearchTreeMulti) get(key []byte) ([]value, error) { - if t.root == nil { - return nil, lsmkv.NotFound - } - - return t.root.get(key) -} - -// // set Tombstone for the entire entry, i.e. all values for this key -// func (t *binarySearchTreeMulti) setTombstone(key []byte) { -// if t.root == nil { -// // we need to actively insert a node with a tombstone, even if this node is -// // not present because we still need to propagate the delete into the disk -// // segments. It could refer to an entity which was created in a previous -// // segment and is thus unknown to this memtable -// t.root = &binarySearchNodeMulti{ -// key: key, -// value: nil, -// tombstone: true, -// } -// } - -// t.root.setTombstone(key) -// } - -func (t *binarySearchTreeMulti) flattenInOrder() []*binarySearchNodeMulti { - if t.root == nil { - return nil - } - - return t.root.flattenInOrder() -} - -type binarySearchNodeMulti struct { - key []byte - values []value - left *binarySearchNodeMulti - right *binarySearchNodeMulti - parent *binarySearchNodeMulti - colourIsRed bool -} - -func (n *binarySearchNodeMulti) Parent() rbtree.Node { - if n == nil { - return nil - } - return n.parent -} - -func (n *binarySearchNodeMulti) SetParent(parent rbtree.Node) { - if n == nil { - addNewSearchNodeMultiReceiver(&n) - } - - if parent == nil { - n.parent = nil - return - } - - n.parent = parent.(*binarySearchNodeMulti) -} - -func (n *binarySearchNodeMulti) Left() rbtree.Node { - if n == nil { - return nil - } - return n.left -} - -func (n *binarySearchNodeMulti) SetLeft(left rbtree.Node) { - if n == nil { - addNewSearchNodeMultiReceiver(&n) - } - - if left == nil { - n.left = nil - return - } - - n.left = left.(*binarySearchNodeMulti) -} - -func (n *binarySearchNodeMulti) Right() rbtree.Node { - if n == nil { - return nil - } - return n.right -} - -func (n *binarySearchNodeMulti) SetRight(right rbtree.Node) { - if n == nil { - addNewSearchNodeMultiReceiver(&n) - } - - if right == nil { - n.right = nil - return - } - - n.right = right.(*binarySearchNodeMulti) -} - -func (n *binarySearchNodeMulti) IsRed() bool { - if n == nil { - return false - } - return n.colourIsRed -} - -func (n *binarySearchNodeMulti) SetRed(isRed bool) { - n.colourIsRed = isRed -} - -func (n *binarySearchNodeMulti) IsNil() bool { - return n == nil -} - -func addNewSearchNodeMultiReceiver(nodePtr **binarySearchNodeMulti) { - *nodePtr = &binarySearchNodeMulti{} -} - -func (n *binarySearchNodeMulti) insert(key []byte, values []value) *binarySearchNodeMulti { - if bytes.Equal(key, n.key) { - n.values = append(n.values, values...) - return nil - } - - if bytes.Compare(key, n.key) < 0 { - if n.left != nil { - return n.left.insert(key, values) - } else { - n.left = &binarySearchNodeMulti{ - key: key, - values: values, - parent: n, - colourIsRed: true, - } - return binarySearchNodeMultiFromRB(rbtree.Rebalance(n.left)) - } - } else { - if n.right != nil { - return n.right.insert(key, values) - } else { - n.right = &binarySearchNodeMulti{ - key: key, - values: values, - parent: n, - colourIsRed: true, - } - return binarySearchNodeMultiFromRB(rbtree.Rebalance(n.right)) - } - } -} - -func (n *binarySearchNodeMulti) get(key []byte) ([]value, error) { - if bytes.Equal(n.key, key) { - return n.values, nil - } - - if bytes.Compare(key, n.key) < 0 { - if n.left == nil { - return nil, lsmkv.NotFound - } - - return n.left.get(key) - } else { - if n.right == nil { - return nil, lsmkv.NotFound - } - - return n.right.get(key) - } -} - -func binarySearchNodeMultiFromRB(rbNode rbtree.Node) (bsNode *binarySearchNodeMulti) { - if rbNode == nil { - bsNode = nil - return - } - bsNode = rbNode.(*binarySearchNodeMulti) - return -} - -// func (n *binarySearchNodeMulti) setTombstone(key []byte) { -// if bytes.Equal(n.key, key) { -// n.value = nil -// n.tombstone = true -// } - -// if bytes.Compare(key, n.key) < 0 { -// if n.left == nil { -// n.left = &binarySearchNodeMulti{ -// key: key, -// value: nil, -// tombstone: true, -// } -// return -// } - -// n.left.setTombstone(key) -// return -// } else { -// if n.right == nil { -// n.right = &binarySearchNodeMulti{ -// key: key, -// value: nil, -// tombstone: true, -// } -// return -// } - -// n.right.setTombstone(key) -// return -// } -// } - -func (n *binarySearchNodeMulti) flattenInOrder() []*binarySearchNodeMulti { - var left []*binarySearchNodeMulti - var right []*binarySearchNodeMulti - - if n.left != nil { - left = n.left.flattenInOrder() - } - - if n.right != nil { - right = n.right.flattenInOrder() - } - - right = append([]*binarySearchNodeMulti{n}, right...) - return append(left, right...) -} diff --git a/adapters/repos/db/lsmkv/binary_search_tree_test.go b/adapters/repos/db/lsmkv/binary_search_tree_test.go deleted file mode 100644 index 37cb67fe4e9357965d9f21b74b733507539575a9..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/binary_search_tree_test.go +++ /dev/null @@ -1,133 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "crypto/rand" - "testing" - - "github.com/google/uuid" - "github.com/stretchr/testify/require" -) - -// This test asserts that the *binarySearchTree.insert -// method properly calculates the net additions of a -// new node into the tree -func TestInsertNetAdditions_Replace(t *testing.T) { - t.Run("single node entry", func(t *testing.T) { - tree := &binarySearchTree{} - - key := make([]byte, 8) - val := make([]byte, 8) - - rand.Read(key) - rand.Read(val) - - n, _ := tree.insert(key, val, nil) - require.Equal(t, len(key)+len(val), n) - }) - - t.Run("multiple unique node entries", func(t *testing.T) { - tree := &binarySearchTree{} - - amount := 100 - size := 8 - - var n int - for i := 0; i < amount; i++ { - key := make([]byte, size) - val := make([]byte, size) - - rand.Read(key) - rand.Read(val) - - newAdditions, _ := tree.insert(key, val, nil) - n += newAdditions - } - - require.Equal(t, amount*size*2, n) - }) - - t.Run("multiple non-unique node entries", func(t *testing.T) { - tree := &binarySearchTree{} - - var ( - amount = 100 - keySize = 100 - origValSize = 100 - newValSize = origValSize * 100 - keys = make([][]byte, amount) - vals = make([][]byte, amount) - - netAdditions int - ) - - // write the keys and original values - for i := range keys { - key := make([]byte, keySize) - rand.Read(key) - - val := make([]byte, origValSize) - rand.Read(val) - - keys[i], vals[i] = key, val - } - - // make initial inserts - for i := range keys { - currentNetAddition, _ := tree.insert(keys[i], vals[i], nil) - netAdditions += currentNetAddition - } - - // change the values of the existing keys - // with new values of different length - for i := 0; i < amount; i++ { - val := make([]byte, newValSize) - rand.Read(val) - - vals[i] = val - } - - for i := 0; i < amount; i++ { - currentNetAddition, _ := tree.insert(keys[i], vals[i], nil) - netAdditions += currentNetAddition - } - - // Formulas for calculating the total net additions after - // updating the keys with differently sized values - expectedFirstNetAdd := amount * (keySize + origValSize) - expectedSecondNetAdd := (amount * (keySize + newValSize)) - (amount * keySize) - (amount * origValSize) - expectedNetAdditions := expectedFirstNetAdd + expectedSecondNetAdd - - require.Equal(t, expectedNetAdditions, netAdditions) - }) - - // test to assure multiple tombstone nodes are not created when same value is added and deleted multiple times - // https://semi-technology.atlassian.net/browse/WEAVIATE-31 - t.Run("consecutive adding and deleting value does not multiply nodes", func(t *testing.T) { - tree := &binarySearchTree{} - - key := []byte(uuid.New().String()) - value := make([]byte, 100) - rand.Read(value) - - for i := 0; i < 10; i++ { - tree.insert(key, value, nil) - tree.setTombstone(key, nil) - } - - flat := tree.flattenInOrder() - - require.Equal(t, 1, len(flat)) - require.True(t, flat[0].tombstone) - }) -} diff --git a/adapters/repos/db/lsmkv/bloom_filter_metrics.go b/adapters/repos/db/lsmkv/bloom_filter_metrics.go deleted file mode 100644 index cba3a96284fbcc0ef2a319c3121f2b308860efff..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/bloom_filter_metrics.go +++ /dev/null @@ -1,28 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -type bloomFilterMetrics struct { - trueNegative TimeObserver - falsePositive TimeObserver - truePositive TimeObserver -} - -// newBloomFilterMetrics curries the prometheus metrics just once at -// initialization to prevent further allocs on the hot path -func newBloomFilterMetrics(metrics *Metrics) *bloomFilterMetrics { - return &bloomFilterMetrics{ - trueNegative: metrics.BloomFilterObserver("replace", "get_true_negative"), - falsePositive: metrics.BloomFilterObserver("replace", "get_false_positive"), - truePositive: metrics.BloomFilterObserver("replace", "get_true_positive"), - } -} diff --git a/adapters/repos/db/lsmkv/bucket.go b/adapters/repos/db/lsmkv/bucket.go deleted file mode 100644 index fbabb86bffaf46fd809b0992cf59c3fc1be3f9f2..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/bucket.go +++ /dev/null @@ -1,1018 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "bytes" - "context" - "errors" - "fmt" - "os" - "path/filepath" - "sort" - "sync" - "time" - - "github.com/prometheus/client_golang/prometheus" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv/segmentindex" - "github.com/weaviate/weaviate/entities/cyclemanager" - "github.com/weaviate/weaviate/entities/interval" - "github.com/weaviate/weaviate/entities/lsmkv" - "github.com/weaviate/weaviate/entities/storagestate" - "github.com/weaviate/weaviate/entities/storobj" -) - -type Bucket struct { - dir string - rootDir string - active *Memtable - flushing *Memtable - disk *SegmentGroup - logger logrus.FieldLogger - - // Lock() means a move from active to flushing is happening, RLock() is - // normal operation - flushLock sync.RWMutex - haltedFlushTimer *interval.BackoffTimer - - walThreshold uint64 - flushAfterIdle time.Duration - memtableThreshold uint64 - memtableResizer *memtableSizeAdvisor - strategy string - // Strategy inverted index is supposed to be created with, but existing - // segment files were created with different one. - // It can happen when new strategy were introduced to weaviate, but - // files are already created using old implementation. - // Example: RoaringSet strategy replaces CollectionSet strategy. - // Field can be used for migration files of old strategy to newer one. - desiredStrategy string - secondaryIndices uint16 - - // Optional to avoid syscalls - mmapContents bool - - // for backward compatibility - legacyMapSortingBeforeCompaction bool - - flushCallbackCtrl cyclemanager.CycleCallbackCtrl - - status storagestate.Status - statusLock sync.RWMutex - - metrics *Metrics - - // all "replace" buckets support counting through net additions, but not all - // produce a meaningful count. Typically, the only count we're interested in - // is that of the bucket that holds objects - monitorCount bool - - pauseTimer *prometheus.Timer // Times the pause - - // Whether tombstones (set/map/replace types) or deletions (roaringset type) - // should be kept in root segment during compaction process. - // Since segments are immutable, deletions are added as new entries with - // tombstones. Tombstones are by default copied to merged segment, as they - // can refer to keys/values present in previous segments. - // Those tombstones can be removed entirely when merging with root (1st) segment, - // due to lack of previous segments, tombstones may relate to. - // As info about key/value being deleted (based on tombstone presence) may be important - // for some use cases (e.g. replication needs to know if object(ObjectsBucketLSM) was deleted) - // keeping tombstones on compaction is optional - keepTombstones bool - - // Init and use bloom filter for getting key from bucket segments. - // As some buckets can be accessed only with cursor (see flat index), - // where bloom filter is not applicable, it can be disabled. - // ON by default - useBloomFilter bool - - // Net additions keep track of number of elements stored in bucket (of type replace). - // As some buckets don't have to provide Count info (see flat index), - // tracking additions can be disabled. - // ON by default - calcCountNetAdditions bool - - forceCompaction bool -} - -// NewBucket initializes a new bucket. It either loads the state from disk if -// it exists, or initializes new state. -// -// You do not need to ever call NewBucket() yourself, if you are using a -// [Store]. In this case the [Store] can manage buckets for you, using methods -// such as CreateOrLoadBucket(). -func NewBucket(ctx context.Context, dir, rootDir string, logger logrus.FieldLogger, - metrics *Metrics, compactionCallbacks, flushCallbacks cyclemanager.CycleCallbackGroup, - opts ...BucketOption, -) (*Bucket, error) { - beforeAll := time.Now() - defaultMemTableThreshold := uint64(10 * 1024 * 1024) - defaultWalThreshold := uint64(1024 * 1024 * 1024) - defaultFlushAfterIdle := 60 * time.Second - defaultStrategy := StrategyReplace - - if err := os.MkdirAll(dir, 0o700); err != nil { - return nil, err - } - - b := &Bucket{ - dir: dir, - rootDir: rootDir, - memtableThreshold: defaultMemTableThreshold, - walThreshold: defaultWalThreshold, - flushAfterIdle: defaultFlushAfterIdle, - strategy: defaultStrategy, - mmapContents: true, - logger: logger, - metrics: metrics, - useBloomFilter: true, - calcCountNetAdditions: true, - haltedFlushTimer: interval.NewBackoffTimer(), - } - - for _, opt := range opts { - if err := opt(b); err != nil { - return nil, err - } - } - - if b.memtableResizer != nil { - b.memtableThreshold = uint64(b.memtableResizer.Initial()) - } - - sg, err := newSegmentGroup(logger, metrics, compactionCallbacks, - sgConfig{ - dir: dir, - strategy: b.strategy, - mapRequiresSorting: b.legacyMapSortingBeforeCompaction, - monitorCount: b.monitorCount, - mmapContents: b.mmapContents, - keepTombstones: b.keepTombstones, - forceCompaction: b.forceCompaction, - useBloomFilter: b.useBloomFilter, - calcCountNetAdditions: b.calcCountNetAdditions, - }) - if err != nil { - return nil, fmt.Errorf("init disk segments: %w", err) - } - - // Actual strategy is stored in segment files. In case it is SetCollection, - // while new implementation uses bitmaps and supposed to be RoaringSet, - // bucket and segmentgroup strategy is changed back to SetCollection - // (memtables will be created later on, with already modified strategy) - // TODO what if only WAL files exists, and there is no segment to get actual strategy? - if b.strategy == StrategyRoaringSet && len(sg.segments) > 0 && - sg.segments[0].strategy == segmentindex.StrategySetCollection { - b.strategy = StrategySetCollection - b.desiredStrategy = StrategyRoaringSet - sg.strategy = StrategySetCollection - } - // As of v1.19 property's IndexInterval setting is replaced with - // IndexFilterable (roaring set) + IndexSearchable (map) and enabled by default. - // Buckets for text/text[] inverted indexes created before 1.19 have strategy - // map and name that since 1.19 is used by filterable indeverted index. - // Those buckets (roaring set by configuration, but in fact map) have to be - // renamed on startup by migrator. Here actual strategy is set based on - // data found in segment files - if b.strategy == StrategyRoaringSet && len(sg.segments) > 0 && - sg.segments[0].strategy == segmentindex.StrategyMapCollection { - b.strategy = StrategyMapCollection - b.desiredStrategy = StrategyRoaringSet - sg.strategy = StrategyMapCollection - } - - b.disk = sg - - if err := b.setNewActiveMemtable(); err != nil { - return nil, err - } - - if err := b.recoverFromCommitLogs(ctx); err != nil { - return nil, err - } - - id := "bucket/flush/" + b.dir - b.flushCallbackCtrl = flushCallbacks.Register(id, b.flushAndSwitchIfThresholdsMet) - - b.metrics.TrackStartupBucket(beforeAll) - - return b, nil -} - -func (b *Bucket) GetDir() string { - return b.dir -} - -func (b *Bucket) GetRootDir() string { - return b.rootDir -} - -func (b *Bucket) GetStrategy() string { - return b.strategy -} - -func (b *Bucket) GetDesiredStrategy() string { - return b.desiredStrategy -} - -func (b *Bucket) GetSecondaryIndices() uint16 { - return b.secondaryIndices -} - -func (b *Bucket) GetStatus() storagestate.Status { - b.statusLock.RLock() - defer b.statusLock.RUnlock() - - return b.status -} - -func (b *Bucket) GetMemtableThreshold() uint64 { - return b.memtableThreshold -} - -func (b *Bucket) GetWalThreshold() uint64 { - return b.walThreshold -} - -func (b *Bucket) GetFlushAfterIdle() time.Duration { - return b.flushAfterIdle -} - -func (b *Bucket) GetFlushCallbackCtrl() cyclemanager.CycleCallbackCtrl { - return b.flushCallbackCtrl -} - -func (b *Bucket) IterateObjects(ctx context.Context, f func(object *storobj.Object) error) error { - i := 0 - cursor := b.Cursor() - defer cursor.Close() - - for k, v := cursor.First(); k != nil; k, v = cursor.Next() { - obj, err := storobj.FromBinary(v) - if err != nil { - return fmt.Errorf("cannot unmarshal object %d, %v", i, err) - } - if err := f(obj); err != nil { - return fmt.Errorf("callback on object '%d' failed: %w", obj.DocID(), err) - } - - i++ - } - - return nil -} - -func (b *Bucket) IterateMapObjects(ctx context.Context, f func([]byte, []byte, []byte, bool) error) error { - cursor := b.MapCursor() - defer cursor.Close() - - for kList, vList := cursor.First(); kList != nil; kList, vList = cursor.Next() { - for _, v := range vList { - if err := f(kList, v.Key, v.Value, v.Tombstone); err != nil { - return fmt.Errorf("callback on object '%v' failed: %w", v, err) - } - } - } - - return nil -} - -func (b *Bucket) SetMemtableThreshold(size uint64) { - b.memtableThreshold = size -} - -// Get retrieves the single value for the given key. -// -// Get is specific to ReplaceStrategy and cannot be used with any of the other -// strategies. Use [Bucket.SetList] or [Bucket.MapList] instead. -// -// Get uses the regular or "primary" key for an object. If a bucket has -// secondary indexes, use [Bucket.GetBySecondary] to retrieve an object using -// its secondary key -func (b *Bucket) Get(key []byte) ([]byte, error) { - b.flushLock.RLock() - defer b.flushLock.RUnlock() - - v, err := b.active.get(key) - if err == nil { - // item found and no error, return and stop searching, since the strategy - // is replace - return v, nil - } - if errors.Is(err, lsmkv.Deleted) { - // deleted in the mem-table (which is always the latest) means we don't - // have to check the disk segments, return nil now - return nil, nil - } - - if err != lsmkv.NotFound { - panic(fmt.Sprintf("unsupported error in bucket.Get: %v\n", err)) - } - - if b.flushing != nil { - v, err := b.flushing.get(key) - if err == nil { - // item found and no error, return and stop searching, since the strategy - // is replace - return v, nil - } - if errors.Is(err, lsmkv.Deleted) { - // deleted in the now most recent memtable means we don't have to check - // the disk segments, return nil now - return nil, nil - } - - if err != lsmkv.NotFound { - panic("unsupported error in bucket.Get") - } - } - - return b.disk.get(key) -} - -// GetBySecondary retrieves an object using one of its secondary keys. A bucket -// can have an infinite number of secondary keys. Specify the secondary key -// position as the first argument. -// -// A real-life example of secondary keys is the Weaviate object store. Objects -// are stored with the user-facing ID as their primary key and with the doc-id -// (an ever-increasing uint64) as the secondary key. -// -// Similar to [Bucket.Get], GetBySecondary is limited to ReplaceStrategy. No -// equivalent exists for Set and Map, as those do not support secondary -// indexes. -func (b *Bucket) GetBySecondary(pos int, key []byte) ([]byte, error) { - bytes, _, err := b.GetBySecondaryIntoMemory(pos, key, nil) - return bytes, err -} - -// GetBySecondaryWithBuffer is like [Bucket.GetBySecondary], but also takes a -// buffer. It's in the response of the caller to pool the buffer, since the -// bucket does not know when the caller is done using it. The return bytes will -// likely point to the same memory that's part of the buffer. However, if the -// buffer is to small, a larger buffer may also be returned (second arg). -func (b *Bucket) GetBySecondaryWithBuffer(pos int, key []byte, buf []byte) ([]byte, []byte, error) { - bytes, newBuf, err := b.GetBySecondaryIntoMemory(pos, key, buf) - return bytes, newBuf, err -} - -// GetBySecondaryIntoMemory copies into the specified memory, and retrieves -// an object using one of its secondary keys. A bucket -// can have an infinite number of secondary keys. Specify the secondary key -// position as the first argument. -// -// A real-life example of secondary keys is the Weaviate object store. Objects -// are stored with the user-facing ID as their primary key and with the doc-id -// (an ever-increasing uint64) as the secondary key. -// -// Similar to [Bucket.Get], GetBySecondary is limited to ReplaceStrategy. No -// equivalent exists for Set and Map, as those do not support secondary -// indexes. -func (b *Bucket) GetBySecondaryIntoMemory(pos int, key []byte, buffer []byte) ([]byte, []byte, error) { - b.flushLock.RLock() - defer b.flushLock.RUnlock() - - v, err := b.active.getBySecondary(pos, key) - if err == nil { - // item found and no error, return and stop searching, since the strategy - // is replace - return v, buffer, nil - } - if errors.Is(err, lsmkv.Deleted) { - // deleted in the mem-table (which is always the latest) means we don't - // have to check the disk segments, return nil now - return nil, buffer, nil - } - - if err != lsmkv.NotFound { - panic("unsupported error in bucket.Get") - } - - if b.flushing != nil { - v, err := b.flushing.getBySecondary(pos, key) - if err == nil { - // item found and no error, return and stop searching, since the strategy - // is replace - return v, buffer, nil - } - if errors.Is(err, lsmkv.Deleted) { - // deleted in the now most recent memtable means we don't have to check - // the disk segments, return nil now - return nil, buffer, nil - } - - if err != lsmkv.NotFound { - panic("unsupported error in bucket.Get") - } - } - - return b.disk.getBySecondaryIntoMemory(pos, key, buffer) -} - -// SetList returns all Set entries for a given key. -// -// SetList is specific to the Set Strategy, for Map use [Bucket.MapList], and -// for Replace use [Bucket.Get]. -func (b *Bucket) SetList(key []byte) ([][]byte, error) { - b.flushLock.RLock() - defer b.flushLock.RUnlock() - - var out []value - - v, err := b.disk.getCollection(key) - if err != nil { - if err != nil && err != lsmkv.NotFound { - return nil, err - } - } - out = v - - if b.flushing != nil { - v, err = b.flushing.getCollection(key) - if err != nil { - if err != nil && err != lsmkv.NotFound { - return nil, err - } - } - out = append(out, v...) - - } - - v, err = b.active.getCollection(key) - if err != nil { - if err != nil && err != lsmkv.NotFound { - return nil, err - } - } - if len(v) > 0 { - // skip the expensive append operation if there was no memtable - out = append(out, v...) - } - - return newSetDecoder().Do(out), nil -} - -// Put creates or replaces a single value for a given key. -// -// err := bucket.Put([]byte("my_key"), []byte("my_value")) -// if err != nil { -// /* do something */ -// } -// -// If a bucket has a secondary index configured, you can also specify one or -// more secondary keys, like so: -// -// err := bucket.Put([]byte("my_key"), []byte("my_value"), -// WithSecondaryKey(0, []byte("my_alternative_key")), -// ) -// if err != nil { -// /* do something */ -// } -// -// Put is limited to ReplaceStrategy, use [Bucket.SetAdd] for Set or -// [Bucket.MapSet] and [Bucket.MapSetMulti]. -func (b *Bucket) Put(key, value []byte, opts ...SecondaryKeyOption) error { - b.flushLock.RLock() - defer b.flushLock.RUnlock() - - return b.active.put(key, value, opts...) -} - -// SetAdd adds one or more Set-Entries to a Set for the given key. SetAdd is -// entirely agnostic of existing entries, it acts as append-only. This also -// makes it agnostic of whether the key already exists or not. -// -// Example to add two entries to a set: -// -// err := bucket.SetAdd([]byte("my_key"), [][]byte{ -// []byte("one-set-element"), []byte("another-set-element"), -// }) -// if err != nil { -// /* do something */ -// } -// -// SetAdd is specific to the Set strategy. For Replace, use [Bucket.Put], for -// Map use either [Bucket.MapSet] or [Bucket.MapSetMulti]. -func (b *Bucket) SetAdd(key []byte, values [][]byte) error { - b.flushLock.RLock() - defer b.flushLock.RUnlock() - - return b.active.append(key, newSetEncoder().Do(values)) -} - -// SetDeleteSingle removes one Set element from the given key. Note that LSM -// stores are append only, thus internally this action appends a tombstone. The -// entry will not be removed until a compaction has run, and even then a -// compaction does not guarantee the removal of the data right away. This is -// because an entry could have been created in an older segment than those -// present in the compaction. This can be seen as an implementation detail, -// unless the caller expects to free disk space by calling this method. Such -// freeing is not guaranteed. -// -// SetDeleteSingle is specific to the Set Strategy. For Replace, you can use -// [Bucket.Delete] to delete the entire row, for Maps use [Bucket.MapDeleteKey] -// to delete a single map entry. -func (b *Bucket) SetDeleteSingle(key []byte, valueToDelete []byte) error { - b.flushLock.RLock() - defer b.flushLock.RUnlock() - - return b.active.append(key, []value{ - { - value: valueToDelete, - tombstone: true, - }, - }) -} - -// WasDeleted determines if an object used to exist in the LSM store -// -// There are 3 different locations that we need to check for the key -// in this order: active memtable, flushing memtable, and disk -// segment -func (b *Bucket) WasDeleted(key []byte) (bool, error) { - if !b.keepTombstones { - return false, fmt.Errorf("Bucket requires option `keepTombstones` set to check deleted keys") - } - - b.flushLock.RLock() - defer b.flushLock.RUnlock() - - _, err := b.active.get(key) - switch err { - case nil: - return false, nil - case lsmkv.Deleted: - return true, nil - case lsmkv.NotFound: - // We can still check flushing and disk - default: - return false, fmt.Errorf("unsupported bucket error: %w", err) - } - - if b.flushing != nil { - _, err := b.flushing.get(key) - switch err { - case nil: - return false, nil - case lsmkv.Deleted: - return true, nil - case lsmkv.NotFound: - // We can still check disk - default: - return false, fmt.Errorf("unsupported bucket error: %w", err) - } - } - - _, err = b.disk.get(key) - switch err { - case nil, lsmkv.NotFound: - return false, nil - case lsmkv.Deleted: - return true, nil - default: - return false, fmt.Errorf("unsupported bucket error: %w", err) - } -} - -type MapListOptionConfig struct { - acceptDuplicates bool - legacyRequireManualSorting bool -} - -type MapListOption func(c *MapListOptionConfig) - -func MapListAcceptDuplicates() MapListOption { - return func(c *MapListOptionConfig) { - c.acceptDuplicates = true - } -} - -func MapListLegacySortingRequired() MapListOption { - return func(c *MapListOptionConfig) { - c.legacyRequireManualSorting = true - } -} - -// MapList returns all map entries for a given row key. The order of map pairs -// has no specific meaning. For efficient merge operations, pair entries are -// stored sorted on disk, however that is an implementation detail and not a -// caller-facing guarantee. -// -// MapList is specific to the Map strategy, for Sets use [Bucket.SetList], for -// Replace use [Bucket.Get]. -func (b *Bucket) MapList(key []byte, cfgs ...MapListOption) ([]MapPair, error) { - b.flushLock.RLock() - defer b.flushLock.RUnlock() - - c := MapListOptionConfig{} - for _, cfg := range cfgs { - cfg(&c) - } - - segments := [][]MapPair{} - // before := time.Now() - disk, err := b.disk.getCollectionBySegments(key) - if err != nil { - if err != nil && err != lsmkv.NotFound { - return nil, err - } - } - - for i := range disk { - segmentDecoded := make([]MapPair, len(disk[i])) - for j, v := range disk[i] { - if err := segmentDecoded[j].FromBytes(v.value, false); err != nil { - return nil, err - } - segmentDecoded[j].Tombstone = v.tombstone - } - segments = append(segments, segmentDecoded) - } - - // fmt.Printf("--map-list: get all disk segments took %s\n", time.Since(before)) - - // before = time.Now() - // fmt.Printf("--map-list: append all disk segments took %s\n", time.Since(before)) - - if b.flushing != nil { - v, err := b.flushing.getMap(key) - if err != nil { - if err != nil && err != lsmkv.NotFound { - return nil, err - } - } - - segments = append(segments, v) - } - - // before = time.Now() - v, err := b.active.getMap(key) - if err != nil { - if err != nil && err != lsmkv.NotFound { - return nil, err - } - } - segments = append(segments, v) - // fmt.Printf("--map-list: get all active segments took %s\n", time.Since(before)) - - // before = time.Now() - // defer func() { - // fmt.Printf("--map-list: run decoder took %s\n", time.Since(before)) - // }() - - if c.legacyRequireManualSorting { - // Sort to support segments which were stored in an unsorted fashion - for i := range segments { - sort.Slice(segments[i], func(a, b int) bool { - return bytes.Compare(segments[i][a].Key, segments[i][b].Key) == -1 - }) - } - } - - return newSortedMapMerger().do(segments) -} - -// MapSet writes one [MapPair] into the map for the given row key. It is -// agnostic of whether the row key already exists, as well as agnostic of -// whether the map key already exists. In both cases it will create the entry -// if it does not exist or override if it does. -// -// Example to add a new MapPair: -// -// pair := MapPair{Key: []byte("Jane"), Value: []byte("Backend")} -// err := bucket.MapSet([]byte("developers"), pair) -// if err != nil { -// /* do something */ -// } -// -// MapSet is specific to the Map Strategy, for Replace use [Bucket.Put], and for Set use [Bucket.SetAdd] instead. -func (b *Bucket) MapSet(rowKey []byte, kv MapPair) error { - b.flushLock.RLock() - defer b.flushLock.RUnlock() - - return b.active.appendMapSorted(rowKey, kv) -} - -// MapSetMulti is the same as [Bucket.MapSet], except that it takes in multiple -// [MapPair] objects at the same time. -func (b *Bucket) MapSetMulti(rowKey []byte, kvs []MapPair) error { - b.flushLock.RLock() - defer b.flushLock.RUnlock() - - for _, kv := range kvs { - if err := b.active.appendMapSorted(rowKey, kv); err != nil { - return err - } - } - - return nil -} - -// MapDeleteKey removes one key-value pair from the given map row. Note that -// LSM stores are append only, thus internally this action appends a tombstone. -// The entry will not be removed until a compaction has run, and even then a -// compaction does not guarantee the removal of the data right away. This is -// because an entry could have been created in an older segment than those -// present in the compaction. This can be seen as an implementation detail, -// unless the caller expects to free disk space by calling this method. Such -// freeing is not guaranteed. -// -// MapDeleteKey is specific to the Map Strategy. For Replace, you can use -// [Bucket.Delete] to delete the entire row, for Sets use [Bucket.SetDeleteSingle] to delete a single set element. -func (b *Bucket) MapDeleteKey(rowKey, mapKey []byte) error { - b.flushLock.RLock() - defer b.flushLock.RUnlock() - - pair := MapPair{ - Key: mapKey, - Tombstone: true, - } - - return b.active.appendMapSorted(rowKey, pair) -} - -// Delete removes the given row. Note that LSM stores are append only, thus -// internally this action appends a tombstone. The entry will not be removed -// until a compaction has run, and even then a compaction does not guarantee -// the removal of the data right away. This is because an entry could have been -// created in an older segment than those present in the compaction. This can -// be seen as an implementation detail, unless the caller expects to free disk -// space by calling this method. Such freeing is not guaranteed. -// -// Delete is specific to the Replace Strategy. For Maps, you can use -// [Bucket.MapDeleteKey] to delete a single key-value pair, for Sets use -// [Bucket.SetDeleteSingle] to delete a single set element. -func (b *Bucket) Delete(key []byte, opts ...SecondaryKeyOption) error { - b.flushLock.RLock() - defer b.flushLock.RUnlock() - - return b.active.setTombstone(key, opts...) -} - -// meant to be called from situations where a lock is already held, does not -// lock on its own -func (b *Bucket) setNewActiveMemtable() error { - mt, err := newMemtable(filepath.Join(b.dir, fmt.Sprintf("segment-%d", - time.Now().UnixNano())), b.strategy, b.secondaryIndices, b.metrics) - if err != nil { - return err - } - - b.active = mt - return nil -} - -func (b *Bucket) Count() int { - b.flushLock.RLock() - defer b.flushLock.RUnlock() - - if b.strategy != StrategyReplace { - panic("Count() called on strategy other than 'replace'") - } - - memtableCount := 0 - if b.flushing == nil { - // only consider active - memtableCount += b.memtableNetCount(b.active.countStats(), nil) - } else { - flushingCountStats := b.flushing.countStats() - activeCountStats := b.active.countStats() - deltaActive := b.memtableNetCount(activeCountStats, flushingCountStats) - deltaFlushing := b.memtableNetCount(flushingCountStats, nil) - - memtableCount = deltaActive + deltaFlushing - } - - diskCount := b.disk.count() - - if b.monitorCount { - b.metrics.ObjectCount(memtableCount + diskCount) - } - return memtableCount + diskCount -} - -func (b *Bucket) memtableNetCount(stats *countStats, previousMemtable *countStats) int { - netCount := 0 - - // TODO: this uses regular get, given that this may be called quite commonly, - // we might consider building a pure Exists(), which skips reading the value - // and only checks for tombstones, etc. - for _, key := range stats.upsertKeys { - if !b.existsOnDiskAndPreviousMemtable(previousMemtable, key) { - netCount++ - } - } - - for _, key := range stats.tombstonedKeys { - if b.existsOnDiskAndPreviousMemtable(previousMemtable, key) { - netCount-- - } - } - - return netCount -} - -func (b *Bucket) existsOnDiskAndPreviousMemtable(previous *countStats, key []byte) bool { - v, _ := b.disk.get(key) // current implementation can't error - if v == nil { - // not on disk, but it could still be in the previous memtable - return previous.hasUpsert(key) - } - - // it exists on disk ,but it could still have been deleted in the previous memtable - return !previous.hasTombstone(key) -} - -func (b *Bucket) Shutdown(ctx context.Context) error { - if err := b.disk.shutdown(ctx); err != nil { - return err - } - - if err := b.flushCallbackCtrl.Unregister(ctx); err != nil { - return fmt.Errorf("long-running flush in progress: %w", ctx.Err()) - } - - b.flushLock.Lock() - if err := b.active.flush(); err != nil { - return err - } - b.flushLock.Unlock() - - if b.flushing == nil { - // active has flushing, no one else was currently flushing, it's safe to - // exit - return nil - } - - // it seems we still need to wait for someone to finish flushing - t := time.NewTicker(50 * time.Millisecond) - defer t.Stop() - for { - select { - case <-ctx.Done(): - return ctx.Err() - case <-t.C: - if b.flushing == nil { - return nil - } - } - } -} - -func (b *Bucket) flushAndSwitchIfThresholdsMet(shouldAbort cyclemanager.ShouldAbortCallback) bool { - b.flushLock.RLock() - commitLogSize := b.active.commitlog.Size() - memtableTooLarge := b.active.Size() >= b.memtableThreshold - walTooLarge := uint64(commitLogSize) >= b.walThreshold - dirtyButIdle := (b.active.Size() > 0 || commitLogSize > 0) && - b.active.IdleDuration() >= b.flushAfterIdle - shouldSwitch := memtableTooLarge || walTooLarge || dirtyButIdle - - // If true, the parent shard has indicated that it has - // entered an immutable state. During this time, the - // bucket should refrain from flushing until its shard - // indicates otherwise - if shouldSwitch && b.isReadOnly() { - if b.haltedFlushTimer.IntervalElapsed() { - b.logger.WithField("action", "lsm_memtable_flush"). - WithField("path", b.dir). - Warn("flush halted due to shard READONLY status") - b.haltedFlushTimer.IncreaseInterval() - } - - b.flushLock.RUnlock() - return false - } - - b.flushLock.RUnlock() - if shouldSwitch { - b.haltedFlushTimer.Reset() - cycleLength := b.active.ActiveDuration() - if err := b.FlushAndSwitch(); err != nil { - b.logger.WithField("action", "lsm_memtable_flush"). - WithField("path", b.dir). - WithError(err). - Errorf("flush and switch failed") - } - - if b.memtableResizer != nil { - next, ok := b.memtableResizer.NextTarget(int(b.memtableThreshold), cycleLength) - if ok { - b.memtableThreshold = uint64(next) - } - } - return true - } - return false -} - -// UpdateStatus is used by the parent shard to communicate to the bucket -// when the shard has been set to readonly, or when it is ready for -// writes. -func (b *Bucket) UpdateStatus(status storagestate.Status) { - b.statusLock.Lock() - defer b.statusLock.Unlock() - - b.status = status - b.disk.UpdateStatus(status) -} - -func (b *Bucket) isReadOnly() bool { - b.statusLock.Lock() - defer b.statusLock.Unlock() - - return b.status == storagestate.StatusReadOnly -} - -// FlushAndSwitch is typically called periodically and does not require manual -// calling, but there are some situations where this might be intended, such as -// in test scenarios or when a force flush is desired. -func (b *Bucket) FlushAndSwitch() error { - before := time.Now() - - b.logger.WithField("action", "lsm_memtable_flush_start"). - WithField("path", b.dir). - Trace("start flush and switch") - if err := b.atomicallySwitchMemtable(); err != nil { - return fmt.Errorf("switch active memtable: %w", err) - } - - if err := b.flushing.flush(); err != nil { - return fmt.Errorf("flush: %w", err) - } - - if err := b.atomicallyAddDiskSegmentAndRemoveFlushing(); err != nil { - return fmt.Errorf("add segment and remove flushing: %w", err) - } - - took := time.Since(before) - b.logger.WithField("action", "lsm_memtable_flush_complete"). - WithField("path", b.dir). - Trace("finish flush and switch") - - b.logger.WithField("action", "lsm_memtable_flush_complete"). - WithField("path", b.dir). - WithField("took", took). - Debugf("flush and switch took %s\n", took) - - return nil -} - -func (b *Bucket) atomicallyAddDiskSegmentAndRemoveFlushing() error { - b.flushLock.Lock() - defer b.flushLock.Unlock() - - path := b.flushing.path - if err := b.disk.add(path + ".db"); err != nil { - return err - } - b.flushing = nil - - if b.strategy == StrategyReplace && b.monitorCount { - // having just flushed the memtable we now have the most up2date count which - // is a good place to update the metric - b.metrics.ObjectCount(b.disk.count()) - } - - return nil -} - -func (b *Bucket) atomicallySwitchMemtable() error { - b.flushLock.Lock() - defer b.flushLock.Unlock() - - b.flushing = b.active - return b.setNewActiveMemtable() -} - -func (b *Bucket) Strategy() string { - return b.strategy -} - -func (b *Bucket) DesiredStrategy() string { - return b.desiredStrategy -} - -// the WAL uses a buffer and isn't written until the buffer size is crossed or -// this function explicitly called. This allows to avoid unnecessary disk -// writes in larger operations, such as batches. It is sufficient to call write -// on the WAL just once. This does not make a batch atomic, but it guarantees -// that the WAL is written before a successful response is returned to the -// user. -func (b *Bucket) WriteWAL() error { - b.flushLock.RLock() - defer b.flushLock.RUnlock() - - return b.active.writeWAL() -} diff --git a/adapters/repos/db/lsmkv/bucket_backup.go b/adapters/repos/db/lsmkv/bucket_backup.go deleted file mode 100644 index 21a636096ada2a042b715499591e5b18c74443a6..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/bucket_backup.go +++ /dev/null @@ -1,94 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "context" - "io/fs" - "path" - "path/filepath" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/storagestate" -) - -// FlushMemtable flushes any active memtable and returns only once the memtable -// has been fully flushed and a stable state on disk has been reached. -// -// This is a preparatory stage for creating backups. -// -// Method should be run only if flushCycle is not running -// (was not started, is stopped, or noop impl is provided) -func (b *Bucket) FlushMemtable() error { - if b.isReadOnly() { - return errors.Wrap(storagestate.ErrStatusReadOnly, "flush memtable") - } - - // this lock does not currently _need_ to be - // obtained, as the only other place that - // grabs this lock is the flush cycle, which - // has just been stopped above. - // - // that being said, we will lock here anyway - // as flushLock may be added elsewhere in the - // future - b.flushLock.Lock() - if b.active == nil && b.flushing == nil { - b.flushLock.Unlock() - return nil - } - b.flushLock.Unlock() - - stat, err := b.active.commitlog.file.Stat() - if err != nil { - b.logger.WithField("action", "lsm_wal_stat"). - WithField("path", b.dir). - WithError(err). - Fatal("bucket backup memtable flush failed") - } - - // attempting a flush&switch on when the active memtable - // or WAL is empty results in a corrupted backup attempt - if b.active.Size() > 0 || stat.Size() > 0 { - if err := b.FlushAndSwitch(); err != nil { - return err - } - } - return nil -} - -// ListFiles lists all files that currently exist in the Bucket. The files are only -// in a stable state if the memtable is empty, and if compactions are paused. If one -// of those conditions is not given, it errors -func (b *Bucket) ListFiles(ctx context.Context, basePath string) ([]string, error) { - var ( - bucketRoot = b.disk.dir - files []string - ) - - err := filepath.WalkDir(bucketRoot, func(currPath string, d fs.DirEntry, err error) error { - if d.IsDir() { - return nil - } - // ignore .wal files because they are not immutable - if filepath.Ext(currPath) == ".wal" { - return nil - } - files = append(files, path.Join(basePath, path.Base(currPath))) - return nil - }) - if err != nil { - return nil, errors.Errorf("failed to list files for bucket: %s", err) - } - - return files, nil -} diff --git a/adapters/repos/db/lsmkv/bucket_backup_test.go b/adapters/repos/db/lsmkv/bucket_backup_test.go deleted file mode 100644 index df089cb9ed789c514085c093f8c7a1293f99d27e..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/bucket_backup_test.go +++ /dev/null @@ -1,95 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "context" - "fmt" - "path/filepath" - "testing" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/cyclemanager" - "github.com/weaviate/weaviate/entities/storagestate" -) - -func Test_BucketBackup(t *testing.T) { - ctx := context.Background() - tests := bucketTests{ - { - name: "bucketBackup_FlushMemtable", - f: bucketBackup_FlushMemtable, - opts: []BucketOption{WithStrategy(StrategyReplace)}, - }, - { - name: "bucketBackup_ListFiles", - f: bucketBackup_ListFiles, - opts: []BucketOption{WithStrategy(StrategyReplace)}, - }, - } - tests.run(ctx, t) -} - -func bucketBackup_FlushMemtable(ctx context.Context, t *testing.T, opts []BucketOption) { - t.Run("assert that readonly bucket fails to flush", func(t *testing.T) { - dirName := t.TempDir() - - b, err := NewBucket(ctx, dirName, dirName, logrus.New(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - b.UpdateStatus(storagestate.StatusReadOnly) - - err = b.FlushMemtable() - require.NotNil(t, err) - expectedErr := errors.Wrap(storagestate.ErrStatusReadOnly, "flush memtable") - assert.EqualError(t, expectedErr, err.Error()) - - err = b.Shutdown(context.Background()) - require.Nil(t, err) - }) -} - -func bucketBackup_ListFiles(ctx context.Context, t *testing.T, opts []BucketOption) { - dirName := t.TempDir() - - b, err := NewBucket(ctx, dirName, dirName, logrus.New(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - t.Run("insert contents into bucket", func(t *testing.T) { - for i := 0; i < 10; i++ { - err := b.Put([]byte(fmt.Sprint(i)), []byte(fmt.Sprint(i))) - require.Nil(t, err) - } - b.FlushMemtable() // flush memtable to generate .db files - }) - - t.Run("assert expected bucket contents", func(t *testing.T) { - files, err := b.ListFiles(ctx, dirName) - assert.Nil(t, err) - assert.Len(t, files, 3) - - exts := make([]string, 3) - for i, file := range files { - exts[i] = filepath.Ext(file) - } - assert.Contains(t, exts, ".db") // the segment itself - assert.Contains(t, exts, ".bloom") // the segment's bloom filter - assert.Contains(t, exts, ".cna") // the segment's count net additions - }) - - err = b.Shutdown(context.Background()) - require.Nil(t, err) -} diff --git a/adapters/repos/db/lsmkv/bucket_options.go b/adapters/repos/db/lsmkv/bucket_options.go deleted file mode 100644 index 085a43ecac6315952feb5305541098f7717fe532..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/bucket_options.go +++ /dev/null @@ -1,161 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "time" - - "github.com/pkg/errors" -) - -type BucketOption func(b *Bucket) error - -func WithStrategy(strategy string) BucketOption { - return func(b *Bucket) error { - switch strategy { - case StrategyReplace, StrategyMapCollection, StrategySetCollection, - StrategyRoaringSet: - default: - return errors.Errorf("unrecognized strategy %q", strategy) - } - - b.strategy = strategy - return nil - } -} - -func WithMemtableThreshold(threshold uint64) BucketOption { - return func(b *Bucket) error { - b.memtableThreshold = threshold - return nil - } -} - -func WithWalThreshold(threshold uint64) BucketOption { - return func(b *Bucket) error { - b.walThreshold = threshold - return nil - } -} - -func WithIdleThreshold(threshold time.Duration) BucketOption { - return func(b *Bucket) error { - b.flushAfterIdle = threshold - return nil - } -} - -func WithSecondaryIndices(count uint16) BucketOption { - return func(b *Bucket) error { - b.secondaryIndices = count - return nil - } -} - -func WithLegacyMapSorting() BucketOption { - return func(b *Bucket) error { - b.legacyMapSortingBeforeCompaction = true - return nil - } -} - -func WithPread(with bool) BucketOption { - return func(b *Bucket) error { - b.mmapContents = !with - return nil - } -} - -func WithDynamicMemtableSizing( - initialMB, maxMB, minActiveSeconds, maxActiveSeconds int, -) BucketOption { - return func(b *Bucket) error { - mb := 1024 * 1024 - cfg := memtableSizeAdvisorCfg{ - initial: initialMB * mb, - stepSize: 10 * mb, - maxSize: maxMB * mb, - minDuration: time.Duration(minActiveSeconds) * time.Second, - maxDuration: time.Duration(maxActiveSeconds) * time.Second, - } - b.memtableResizer = newMemtableSizeAdvisor(cfg) - return nil - } -} - -type secondaryIndexKeys [][]byte - -type SecondaryKeyOption func(s secondaryIndexKeys) error - -func WithSecondaryKey(pos int, key []byte) SecondaryKeyOption { - return func(s secondaryIndexKeys) error { - if pos > len(s) { - return errors.Errorf("set secondary index %d on an index of length %d", - pos, len(s)) - } - - s[pos] = key - - return nil - } -} - -func WithMonitorCount() BucketOption { - return func(b *Bucket) error { - if b.strategy != StrategyReplace { - return errors.Errorf("count monitoring only supported on 'replace' buckets") - } - b.monitorCount = true - return nil - } -} - -func WithKeepTombstones(keepTombstones bool) BucketOption { - return func(b *Bucket) error { - b.keepTombstones = keepTombstones - return nil - } -} - -func WithUseBloomFilter(useBloomFilter bool) BucketOption { - return func(b *Bucket) error { - b.useBloomFilter = useBloomFilter - return nil - } -} - -func WithCalcCountNetAdditions(calcCountNetAdditions bool) BucketOption { - return func(b *Bucket) error { - b.calcCountNetAdditions = calcCountNetAdditions - return nil - } -} - -/* -Background for this option: - -We use the LSM store in two places: -Our existing key/value and inverted buckets -As part of the new brute-force based index (to be built this week). - -Brute-force index -This is a simple disk-index where we use a cursor to iterate over all objects. This is what we need the force-compaction for. The experimentation so far has shown that the cursor is much more performant on a single segment than it is on multiple segments. This is because with a single segment it’s essentially just one conitiguuous chunk of data on disk that we read through. But with multiple segments (and an unpredicatable order) it ends up being many tiny reads (inefficient). -Existing uses of the LSM store -For existing uses, e.g. the object store, we don’t want to force-compact. This is because they can grow massive. For example, you could have a 100GB segment, then a new write leads to a new segment that is just a few bytes. If we would force-compact those two we would write 100GB every time the user sends a few bytes to Weaviate. In this case, the existing tiered compaction strategy makes more sense. -Configurability of buckets -*/ -func WithForceCompation(opt bool) BucketOption { - return func(b *Bucket) error { - b.forceCompaction = opt - return nil - } -} diff --git a/adapters/repos/db/lsmkv/bucket_recover_from_wal.go b/adapters/repos/db/lsmkv/bucket_recover_from_wal.go deleted file mode 100644 index 838ddaee99c189fd2e9a6295d09e4080ad22e7a5..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/bucket_recover_from_wal.go +++ /dev/null @@ -1,117 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "context" - "io" - "os" - "path/filepath" - "time" - - "github.com/pkg/errors" -) - -func (b *Bucket) recoverFromCommitLogs(ctx context.Context) error { - beforeAll := time.Now() - defer b.metrics.TrackStartupBucketRecovery(beforeAll) - - // the context is only ever checked once at the beginning, as there is no - // point in aborting an ongoing recovery. It makes more sense to let it - // complete and have the next recovery (this is called once per bucket) run - // into this error. This way in a crashloop we'd eventually recover each - // bucket until there is nothing left to recover and startup could complete - // in time - if err := ctx.Err(); err != nil { - return errors.Wrap(err, "recover commit log") - } - - list, err := os.ReadDir(b.dir) - if err != nil { - return err - } - - var walFileNames []string - for _, fileInfo := range list { - if filepath.Ext(fileInfo.Name()) != ".wal" { - // skip, this could be disk segments, etc. - continue - } - - if filepath.Join(b.dir, fileInfo.Name()) == b.active.path+".wal" { - // this is the new one which was just created - continue - } - - walFileNames = append(walFileNames, fileInfo.Name()) - } - - if len(walFileNames) == 0 { - // nothing to recover from - return nil - } - - // recover from each log - for _, fname := range walFileNames { - b.logger.WithField("action", "lsm_recover_from_active_wal"). - WithField("path", filepath.Join(b.dir, fname)). - Warning("active write-ahead-log found. Did weaviate crash prior to this? Trying to recover...") - - if err := b.parseWALIntoMemtable(filepath.Join(b.dir, fname)); err != nil { - return errors.Wrapf(err, "ingest wal %q", fname) - } - - b.logger.WithField("action", "lsm_recover_from_active_wal_success"). - WithField("path", filepath.Join(b.dir, fname)). - Info("successfully recovered from write-ahead-log") - } - - if b.active.size > 0 { - if err := b.FlushAndSwitch(); err != nil { - return errors.Wrap(err, "flush memtable after WAL recovery") - } - } - - // delete the commit logs as we can now be sure that they are part of a disk - // segment - for _, fname := range walFileNames { - if err := os.RemoveAll(filepath.Join(b.dir, fname)); err != nil { - return errors.Wrap(err, "clean up commit log") - } - } - - return nil -} - -func (b *Bucket) parseWALIntoMemtable(fname string) error { - // pause commit logging while reading the old log to avoid creating a - // duplicate of the log - b.active.commitlog.pause() - defer b.active.commitlog.unpause() - - err := newCommitLoggerParser(fname, b.active, b.strategy, b.metrics).Do() - if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { - // we need to check for both EOF or UnexpectedEOF, as we don't know where - // the commit log got corrupted, a field ending that weset a longer - // encoding for would return EOF, whereas a field read with binary.Read - // with a fixed size would return UnexpectedEOF. From our perspective both - // are unexpected. - - b.logger.WithField("action", "lsm_recover_from_active_wal_corruption"). - WithField("path", filepath.Join(b.dir, fname)). - Error("write-ahead-log ended abruptly, some elements may not have been recovered") - - return nil - } - - return err -} diff --git a/adapters/repos/db/lsmkv/bucket_roaring_set.go b/adapters/repos/db/lsmkv/bucket_roaring_set.go deleted file mode 100644 index bf6fed12405f4e514ce8fdf162a631fd795479d8..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/bucket_roaring_set.go +++ /dev/null @@ -1,108 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "fmt" - - "github.com/weaviate/sroar" - "github.com/weaviate/weaviate/entities/lsmkv" -) - -func (b *Bucket) RoaringSetAddOne(key []byte, value uint64) error { - if err := checkStrategyRoaringSet(b.strategy); err != nil { - return err - } - - b.flushLock.RLock() - defer b.flushLock.RUnlock() - - return b.active.roaringSetAddOne(key, value) -} - -func (b *Bucket) RoaringSetRemoveOne(key []byte, value uint64) error { - if err := checkStrategyRoaringSet(b.strategy); err != nil { - return err - } - - b.flushLock.RLock() - defer b.flushLock.RUnlock() - - return b.active.roaringSetRemoveOne(key, value) -} - -func (b *Bucket) RoaringSetAddList(key []byte, values []uint64) error { - if err := checkStrategyRoaringSet(b.strategy); err != nil { - return err - } - - b.flushLock.RLock() - defer b.flushLock.RUnlock() - - return b.active.roaringSetAddList(key, values) -} - -func (b *Bucket) RoaringSetAddBitmap(key []byte, bm *sroar.Bitmap) error { - if err := checkStrategyRoaringSet(b.strategy); err != nil { - return err - } - - b.flushLock.RLock() - defer b.flushLock.RUnlock() - - return b.active.roaringSetAddBitmap(key, bm) -} - -func (b *Bucket) RoaringSetGet(key []byte) (*sroar.Bitmap, error) { - if err := checkStrategyRoaringSet(b.strategy); err != nil { - return nil, err - } - - b.flushLock.RLock() - defer b.flushLock.RUnlock() - - segments, err := b.disk.roaringSetGet(key) - if err != nil { - return nil, err - } - - if b.flushing != nil { - flushing, err := b.flushing.roaringSetGet(key) - if err != nil { - if err != lsmkv.NotFound { - return nil, err - } - } else { - segments = append(segments, flushing) - } - } - - memtable, err := b.active.roaringSetGet(key) - if err != nil { - if err != lsmkv.NotFound { - return nil, err - } - } else { - segments = append(segments, memtable) - } - - return segments.Flatten(), nil -} - -func checkStrategyRoaringSet(bucketStrat string) error { - if bucketStrat == StrategyRoaringSet { - return nil - } - - return fmt.Errorf("this method requires a roaring set strategy, got: %s", - bucketStrat) -} diff --git a/adapters/repos/db/lsmkv/bucket_test.go b/adapters/repos/db/lsmkv/bucket_test.go deleted file mode 100644 index 55142965083a2377c70db93611f8e600d4a2c46d..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/bucket_test.go +++ /dev/null @@ -1,261 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "context" - "os" - "testing" - - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/cyclemanager" -) - -type bucketTest struct { - name string - f func(context.Context, *testing.T, []BucketOption) - opts []BucketOption -} - -type bucketTests []bucketTest - -func (tests bucketTests) run(ctx context.Context, t *testing.T) { - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - t.Run("mmap", func(t *testing.T) { - test.f(ctx, t, test.opts) - }) - t.Run("pread", func(t *testing.T) { - test.f(ctx, t, append([]BucketOption{WithPread(true)}, test.opts...)) - }) - }) - } -} - -func TestBucket(t *testing.T) { - ctx := context.Background() - tests := bucketTests{ - { - name: "bucket_WasDeleted_KeepTombstones", - f: bucket_WasDeleted_KeepTombstones, - opts: []BucketOption{ - WithStrategy(StrategyReplace), - WithKeepTombstones(true), - }, - }, - { - name: "bucket_WasDeleted_CleanupTombstones", - f: bucket_WasDeleted_CleanupTombstones, - opts: []BucketOption{ - WithStrategy(StrategyReplace), - }, - }, - { - name: "bucketReadsIntoMemory", - f: bucketReadsIntoMemory, - opts: []BucketOption{ - WithStrategy(StrategyReplace), - WithSecondaryIndices(1), - }, - }, - } - tests.run(ctx, t) -} - -func bucket_WasDeleted_KeepTombstones(ctx context.Context, t *testing.T, opts []BucketOption) { - tmpDir := t.TempDir() - logger, _ := test.NewNullLogger() - - b, err := NewBucket(ctx, tmpDir, "", logger, nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - t.Cleanup(func() { - require.Nil(t, b.Shutdown(context.Background())) - }) - - var ( - key = []byte("key") - val = []byte("value") - ) - - t.Run("insert object", func(t *testing.T) { - err = b.Put(key, val) - require.Nil(t, err) - }) - - t.Run("assert object was not deleted yet", func(t *testing.T) { - deleted, err := b.WasDeleted(key) - require.Nil(t, err) - assert.False(t, deleted) - }) - - t.Run("delete object", func(t *testing.T) { - err = b.Delete(key) - require.Nil(t, err) - }) - - t.Run("assert object was deleted", func(t *testing.T) { - deleted, err := b.WasDeleted(key) - require.Nil(t, err) - assert.True(t, deleted) - }) - - t.Run("assert a nonexistent object is not detected as deleted", func(t *testing.T) { - deleted, err := b.WasDeleted([]byte("DNE")) - require.Nil(t, err) - assert.False(t, deleted) - }) -} - -func bucket_WasDeleted_CleanupTombstones(ctx context.Context, t *testing.T, opts []BucketOption) { - tmpDir := t.TempDir() - logger, _ := test.NewNullLogger() - - b, err := NewBucket(ctx, tmpDir, "", logger, nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.NoError(t, err) - t.Cleanup(func() { - require.NoError(t, b.Shutdown(context.Background())) - }) - - var ( - key = []byte("key") - val = []byte("value") - ) - - t.Run("insert object", func(t *testing.T) { - err = b.Put(key, val) - require.Nil(t, err) - }) - - t.Run("fails on WasDeleted without keepTombstones set (before delete)", func(t *testing.T) { - deleted, err := b.WasDeleted(key) - require.ErrorContains(t, err, "keepTombstones") - require.False(t, deleted) - }) - - t.Run("delete object", func(t *testing.T) { - err = b.Delete(key) - require.Nil(t, err) - }) - - t.Run("fails on WasDeleted without keepTombstones set (after delete)", func(t *testing.T) { - deleted, err := b.WasDeleted(key) - require.ErrorContains(t, err, "keepTombstones") - require.False(t, deleted) - }) - - t.Run("fails on WasDeleted without keepTombstones set (non-existent key)", func(t *testing.T) { - deleted, err := b.WasDeleted([]byte("DNE")) - require.ErrorContains(t, err, "keepTombstones") - require.False(t, deleted) - }) -} - -func bucketReadsIntoMemory(ctx context.Context, t *testing.T, opts []BucketOption) { - dirName := t.TempDir() - logger, _ := test.NewNullLogger() - - b, err := NewBucket(ctx, dirName, "", logger, nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - defer b.Shutdown(ctx) - - require.Nil(t, b.Put([]byte("hello"), []byte("world"), - WithSecondaryKey(0, []byte("bonjour")))) - require.Nil(t, b.FlushMemtable()) - - files, err := os.ReadDir(b.dir) - require.Nil(t, err) - - _, ok := findFileWithExt(files, ".bloom") - assert.True(t, ok) - - _, ok = findFileWithExt(files, "secondary.0.bloom") - assert.True(t, ok) - - b2, err := NewBucket(ctx, b.dir, "", logger, nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - defer b2.Shutdown(ctx) - - valuePrimary, err := b2.Get([]byte("hello")) - require.Nil(t, err) - valueSecondary := make([]byte, 5) - valueSecondary, _, err = b2.GetBySecondaryIntoMemory(0, []byte("bonjour"), valueSecondary) - require.Nil(t, err) - - assert.Equal(t, []byte("world"), valuePrimary) - assert.Equal(t, []byte("world"), valueSecondary) -} - -func TestBucket_MemtableCountWithFlushing(t *testing.T) { - b := Bucket{ - // by using an empty segment group for the disk portion, we can test the - // memtable portion in isolation - disk: &SegmentGroup{}, - } - - tests := []struct { - name string - current *countStats - previous *countStats - expectedNetActive int - expectedNetPrevious int - expectedNetTotal int - }{ - { - name: "only active, only additions", - current: &countStats{ - upsertKeys: [][]byte{[]byte("key-1")}, - }, - expectedNetActive: 1, - }, - { - name: "only active, both additions and deletions", - current: &countStats{ - upsertKeys: [][]byte{[]byte("key-1")}, - // no key with key-2 ever existed, so this does not alter the net count - tombstonedKeys: [][]byte{[]byte("key-2")}, - }, - expectedNetActive: 1, - }, - { - name: "an deletion that was previously added", - current: &countStats{ - tombstonedKeys: [][]byte{[]byte("key-a")}, - }, - previous: &countStats{ - upsertKeys: [][]byte{[]byte("key-a")}, - }, - expectedNetActive: -1, - expectedNetPrevious: 1, - expectedNetTotal: 0, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - actualActive := b.memtableNetCount(tt.current, tt.previous) - assert.Equal(t, tt.expectedNetActive, actualActive) - - if tt.previous != nil { - actualPrevious := b.memtableNetCount(tt.previous, nil) - assert.Equal(t, tt.expectedNetPrevious, actualPrevious) - - assert.Equal(t, tt.expectedNetTotal, actualPrevious+actualActive) - } - }) - } -} diff --git a/adapters/repos/db/lsmkv/bucket_threshold_test.go b/adapters/repos/db/lsmkv/bucket_threshold_test.go deleted file mode 100644 index 8885735c03d0e50db674324a9c480f0729685490..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/bucket_threshold_test.go +++ /dev/null @@ -1,395 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package lsmkv - -import ( - "context" - "crypto/rand" - "encoding/json" - "sync" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/cyclemanager" -) - -// This test ensures that the WAL threshold is being adhered to, and that a -// flush to segment followed by a switch to a new WAL is being performed -// once the threshold is reached -func TestWriteAheadLogThreshold_Replace(t *testing.T) { - dirName := t.TempDir() - - amount := 100 - keys := make([][]byte, amount) - values := make([][]byte, amount) - - walThreshold := uint64(4096) - tolerance := 4. - - flushCallbacks := cyclemanager.NewCallbackGroup("flush", nullLogger(), 1) - flushCycle := cyclemanager.NewManager(cyclemanager.MemtableFlushCycleTicker(), flushCallbacks.CycleCallback) - flushCycle.Start() - - bucket, err := NewBucket(testCtx(), dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), flushCallbacks, - WithStrategy(StrategyReplace), - WithMemtableThreshold(1024*1024*1024), - WithWalThreshold(walThreshold)) - require.Nil(t, err) - - // generate only a small amount of sequential values. this allows - // us to keep the memtable small (the net additions will be close - // to zero), and focus on testing the WAL threshold - t.Run("generate sequential data", func(t *testing.T) { - for i := range keys { - n, err := json.Marshal(i) - require.Nil(t, err) - - keys[i], values[i] = n, n - } - }) - - t.Run("check switchover during insertion", func(t *testing.T) { - // Importing data for over 10s with 1.6ms break between each object - // should result in ~100kB of commitlog data in total. - // With couple of flush attempts happening during this 10s period, - // and with threshold set to 4kB, first .wal size should be much smaller than 100kb - // when commitlog switched to new .wal file. - ctxTimeout, cancelTimeout := context.WithTimeout(context.Background(), 10*time.Second) - - wg := &sync.WaitGroup{} - wg.Add(1) - go func() { - for { - for i := range keys { - if i%100 == 0 && ctxTimeout.Err() != nil { - wg.Done() - return - } - assert.Nil(t, bucket.Put(keys[i], values[i])) - time.Sleep(1600 * time.Microsecond) - } - } - }() - - var firstWalFile string - var firstWalSize int64 - out: - for { - time.Sleep(time.Millisecond) - if ctxTimeout.Err() != nil { - t.Fatalf("Import finished without flushing in the meantime. Size of first WAL file was (%d)", firstWalSize) - } - - bucket.flushLock.RLock() - walFile := bucket.active.commitlog.path - walSize := bucket.active.commitlog.Size() - bucket.flushLock.RUnlock() - - if firstWalFile == "" { - firstWalFile = walFile - } - - if firstWalFile == walFile { - firstWalSize = walSize - } else { - // new path found; flush must have occurred - stop import and exit loop - cancelTimeout() - break out - } - } - - wg.Wait() - if !isSizeWithinTolerance(t, uint64(firstWalSize), walThreshold, tolerance) { - t.Fatalf("WAL size (%d) was allowed to increase beyond threshold (%d) with tolerance of (%f)%%", - firstWalSize, walThreshold, tolerance*100) - } - }) - - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) - defer cancel() - - require.Nil(t, bucket.Shutdown(ctx)) - require.Nil(t, flushCycle.StopAndWait(ctx)) -} - -// This test ensures that the Memtable threshold is being adhered to, and -// that a flush to segment followed by a switch to a new WAL is being -// performed once the threshold is reached -func TestMemtableThreshold_Replace(t *testing.T) { - dirName := t.TempDir() - - amount := 10000 - sizePerValue := 8 - - keys := make([][]byte, amount) - values := make([][]byte, amount) - - memtableThreshold := uint64(4096) - tolerance := 4. - - flushCallbacks := cyclemanager.NewCallbackGroup("flush", nullLogger(), 1) - flushCycle := cyclemanager.NewManager(cyclemanager.MemtableFlushCycleTicker(), flushCallbacks.CycleCallback) - flushCycle.Start() - - bucket, err := NewBucket(testCtx(), dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), flushCallbacks, - WithStrategy(StrategyReplace), - WithMemtableThreshold(memtableThreshold)) - require.Nil(t, err) - - t.Run("generate random data", func(t *testing.T) { - for i := range keys { - n, err := json.Marshal(i) - require.Nil(t, err) - - keys[i] = n - values[i] = make([]byte, sizePerValue) - rand.Read(values[i]) - } - }) - - t.Run("check switchover during insertion", func(t *testing.T) { - // Importing data for over 10s with 0.8ms break between each object - // should result in ~100kB of memtable data. - // With couple of flush attempts happening during this 10s period, - // and with threshold set to 4kB, first memtable size should be much smaller than 100kb - // when memtable flushed and replaced with new one - ctxTimeout, cancelTimeout := context.WithTimeout(context.Background(), 10*time.Second) - - wg := &sync.WaitGroup{} - wg.Add(1) - go func() { - for { - for i := range keys { - if i%100 == 0 && ctxTimeout.Err() != nil { - wg.Done() - return - } - assert.Nil(t, bucket.Put(keys[i], values[i])) - time.Sleep(800 * time.Microsecond) - } - } - }() - - var firstMemtablePath string - var firstMemtableSize uint64 - out: - for { - time.Sleep(time.Millisecond) - if ctxTimeout.Err() != nil { - t.Fatalf("Import finished without flushing in the meantime. Size of first memtable was (%d)", firstMemtableSize) - } - - bucket.flushLock.RLock() - activePath := bucket.active.path - activeSize := bucket.active.Size() - bucket.flushLock.RUnlock() - - if firstMemtablePath == "" { - firstMemtablePath = activePath - } - - if firstMemtablePath == activePath { - firstMemtableSize = activeSize - } else { - // new path found; flush must have occurred - stop import and exit loop - cancelTimeout() - break out - } - } - - wg.Wait() - if !isSizeWithinTolerance(t, uint64(firstMemtableSize), memtableThreshold, tolerance) { - t.Fatalf("Memtable size (%d) was allowed to increase beyond threshold (%d) with tolerance of (%f)%%", - firstMemtableSize, memtableThreshold, tolerance*100) - } - }) - - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) - defer cancel() - - require.Nil(t, bucket.Shutdown(ctx)) - require.Nil(t, flushCycle.StopAndWait(ctx)) -} - -func isSizeWithinTolerance(t *testing.T, detectedSize uint64, threshold uint64, tolerance float64) bool { - return detectedSize > 0 && float64(detectedSize) <= float64(threshold)*(tolerance+1) -} - -func TestMemtableFlushesIfIdle(t *testing.T) { - t.Run("an empty memtable is not flushed", func(t *testing.T) { - dirName := t.TempDir() - - flushCallbacks := cyclemanager.NewCallbackGroup("flush", nullLogger(), 1) - flushCycle := cyclemanager.NewManager(cyclemanager.MemtableFlushCycleTicker(), flushCallbacks.CycleCallback) - flushCycle.Start() - - bucket, err := NewBucket(testCtx(), dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), flushCallbacks, - WithStrategy(StrategyReplace), - WithMemtableThreshold(1e12), // large enough to not affect this test - WithWalThreshold(1e12), // large enough to not affect this test - WithIdleThreshold(10*time.Millisecond), - ) - require.Nil(t, err) - - t.Run("assert no segments exist initially", func(t *testing.T) { - bucket.disk.maintenanceLock.RLock() - defer bucket.disk.maintenanceLock.RUnlock() - - assert.Equal(t, 0, len(bucket.disk.segments)) - }) - - t.Run("wait until idle threshold has passed", func(t *testing.T) { - // First flush attempt should occur after ~100ms after creating bucket. - // Buffer of 200ms guarantees, flush will be called during sleep period. - time.Sleep(200 * time.Millisecond) - }) - - t.Run("assert no segments exist even after passing the idle threshold", func(t *testing.T) { - bucket.disk.maintenanceLock.RLock() - defer bucket.disk.maintenanceLock.RUnlock() - - assert.Equal(t, 0, len(bucket.disk.segments)) - }) - - t.Run("shutdown bucket", func(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) - defer cancel() - require.Nil(t, bucket.Shutdown(ctx)) - require.Nil(t, flushCycle.StopAndWait(ctx)) - }) - }) - - t.Run("a dirty memtable is flushed once the idle period is over", func(t *testing.T) { - dirName := t.TempDir() - - flushCallbacks := cyclemanager.NewCallbackGroup("flush", nullLogger(), 1) - flushCycle := cyclemanager.NewManager(cyclemanager.MemtableFlushCycleTicker(), flushCallbacks.CycleCallback) - flushCycle.Start() - - bucket, err := NewBucket(testCtx(), dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), flushCallbacks, - WithStrategy(StrategyReplace), - WithMemtableThreshold(1e12), // large enough to not affect this test - WithWalThreshold(1e12), // large enough to not affect this test - WithIdleThreshold(50*time.Millisecond), - ) - require.Nil(t, err) - - t.Run("import something to make it dirty", func(t *testing.T) { - require.Nil(t, bucket.Put([]byte("some-key"), []byte("some-value"))) - }) - - t.Run("assert no segments exist initially", func(t *testing.T) { - bucket.disk.maintenanceLock.RLock() - defer bucket.disk.maintenanceLock.RUnlock() - - assert.Equal(t, 0, len(bucket.disk.segments)) - }) - - t.Run("wait until idle threshold has passed", func(t *testing.T) { - // First flush attempt should occur after ~100ms after creating bucket. - // Buffer of 200ms guarantees, flush will be called during sleep period. - time.Sleep(200 * time.Millisecond) - }) - - t.Run("assert that a flush has occurred (and one segment exists)", func(t *testing.T) { - bucket.disk.maintenanceLock.RLock() - defer bucket.disk.maintenanceLock.RUnlock() - - assert.Equal(t, 1, len(bucket.disk.segments)) - }) - - t.Run("shutdown bucket", func(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) - defer cancel() - require.Nil(t, bucket.Shutdown(ctx)) - require.Nil(t, flushCycle.StopAndWait(ctx)) - }) - }) - - t.Run("a dirty memtable is not flushed as long as the next write occurs before the idle threshold", func(t *testing.T) { - dirName := t.TempDir() - - flushCallbacks := cyclemanager.NewCallbackGroup("flush", nullLogger(), 1) - flushCycle := cyclemanager.NewManager(cyclemanager.MemtableFlushCycleTicker(), flushCallbacks.CycleCallback) - flushCycle.Start() - - bucket, err := NewBucket(testCtx(), dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), flushCallbacks, - WithStrategy(StrategyReplace), - WithMemtableThreshold(1e12), // large enough to not affect this test - WithWalThreshold(1e12), // large enough to not affect this test - WithIdleThreshold(50*time.Millisecond), - ) - require.Nil(t, err) - - t.Run("import something to make it dirty", func(t *testing.T) { - require.Nil(t, bucket.Put([]byte("some-key"), []byte("some-value"))) - }) - - t.Run("assert no segments exist initially", func(t *testing.T) { - bucket.disk.maintenanceLock.RLock() - defer bucket.disk.maintenanceLock.RUnlock() - - assert.Equal(t, 0, len(bucket.disk.segments)) - }) - - t.Run("keep importing without ever crossing the idle threshold", func(t *testing.T) { - rounds := 20 - data := make([]byte, rounds*4) - _, err := rand.Read(data) - require.Nil(t, err) - - for i := 0; i < rounds; i++ { - key := data[(i * 4) : (i+1)*4] - bucket.Put(key, []byte("value")) - time.Sleep(25 * time.Millisecond) - } - }) - - t.Run("assert that no flushing has occurred", func(t *testing.T) { - bucket.disk.maintenanceLock.RLock() - defer bucket.disk.maintenanceLock.RUnlock() - - assert.Equal(t, 0, len(bucket.disk.segments)) - }) - - t.Run("wait until idle threshold has passed", func(t *testing.T) { - // At that point 2 flush attempt have already occurred. - // 3rd attempt should occur after ~930ms after creating bucket. - // Buffer of 500ms guarantees, flush will be called during sleep period. - time.Sleep(500 * time.Millisecond) - }) - - t.Run("assert that a flush has occurred (and one segment exists)", func(t *testing.T) { - bucket.disk.maintenanceLock.RLock() - defer bucket.disk.maintenanceLock.RUnlock() - - assert.Equal(t, 1, len(bucket.disk.segments)) - }) - - t.Run("shutdown bucket", func(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) - defer cancel() - require.Nil(t, bucket.Shutdown(ctx)) - require.Nil(t, flushCycle.StopAndWait(ctx)) - }) - }) -} diff --git a/adapters/repos/db/lsmkv/commitlogger.go b/adapters/repos/db/lsmkv/commitlogger.go deleted file mode 100644 index 0043ae1d3128ef67d24b5bad42a80e6c0b88786f..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/commitlogger.go +++ /dev/null @@ -1,177 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "bufio" - "encoding/binary" - "os" - "sync/atomic" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv/roaringset" -) - -type commitLogger struct { - file *os.File - writer *bufio.Writer - n atomic.Int64 - path string - - // e.g. when recovering from an existing log, we do not want to write into a - // new log again - paused bool -} - -type CommitType uint16 - -const ( - CommitTypeReplace CommitType = iota // replace strategy - - // collection strategy - this can handle all cases as updates and deletes are - // only appends in a collection strategy - CommitTypeCollection - CommitTypeRoaringSet -) - -func (ct CommitType) String() string { - switch ct { - case CommitTypeReplace: - return "replace" - case CommitTypeCollection: - return "collection" - case CommitTypeRoaringSet: - return "roaringset" - default: - return "unknown" - } -} - -func (ct CommitType) Is(checkedCommitType CommitType) bool { - return ct == checkedCommitType -} - -func newCommitLogger(path string) (*commitLogger, error) { - out := &commitLogger{ - path: path + ".wal", - } - - f, err := os.Create(out.path) - if err != nil { - return nil, err - } - - out.file = f - - out.writer = bufio.NewWriter(f) - return out, nil -} - -func (cl *commitLogger) put(node segmentReplaceNode) error { - if cl.paused { - return nil - } - - // TODO: do we need a timestamp? if so, does it need to be a vector clock? - if err := binary.Write(cl.writer, binary.LittleEndian, CommitTypeReplace); err != nil { - return err - } - n := 1 - - if ki, err := node.KeyIndexAndWriteTo(cl.writer); err != nil { - return err - } else { - n += ki.ValueEnd - ki.ValueStart - } - - cl.n.Add(int64(n)) - - return nil -} - -func (cl *commitLogger) append(node segmentCollectionNode) error { - if cl.paused { - return nil - } - - // TODO: do we need a timestamp? if so, does it need to be a vector clock? - if err := binary.Write(cl.writer, binary.LittleEndian, CommitTypeCollection); err != nil { - return err - } - n := 1 - - if ki, err := node.KeyIndexAndWriteTo(cl.writer); err != nil { - return err - } else { - n += ki.ValueEnd - ki.ValueStart - } - - cl.n.Add(int64(n)) - - return nil -} - -func (cl *commitLogger) add(node *roaringset.SegmentNode) error { - if cl.paused { - return nil - } - - if err := binary.Write(cl.writer, binary.LittleEndian, CommitTypeRoaringSet); err != nil { - return err - } - n := 1 - - if ki, err := node.KeyIndexAndWriteTo(cl.writer, 0); err != nil { - return err - } else { - n += ki.ValueEnd - ki.ValueStart - } - - cl.n.Add(int64(n)) - - return nil -} - -// Size returns the amount of data that has been written since the commit -// logger was initialized. After a flush a new logger is initialized which -// automatically resets the logger. -func (cl *commitLogger) Size() int64 { - return cl.n.Load() -} - -func (cl *commitLogger) close() error { - if cl.paused { - return errors.Errorf("attempting to close a paused commit logger") - } - - if err := cl.writer.Flush(); err != nil { - return err - } - - return cl.file.Close() -} - -func (cl *commitLogger) pause() { - cl.paused = true -} - -func (cl *commitLogger) unpause() { - cl.paused = false -} - -func (cl *commitLogger) delete() error { - return os.Remove(cl.path) -} - -func (cl *commitLogger) flushBuffers() error { - return cl.writer.Flush() -} diff --git a/adapters/repos/db/lsmkv/commitlogger_parser.go b/adapters/repos/db/lsmkv/commitlogger_parser.go deleted file mode 100644 index 626e117e0ca325fb6d2dd20e7629fac2ca70696e..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/commitlogger_parser.go +++ /dev/null @@ -1,140 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "bufio" - "encoding/binary" - "io" - "os" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/diskio" -) - -type commitloggerParser struct { - path string - strategy string - memtable *Memtable - reader io.Reader - metrics *Metrics - replaceCache map[string]segmentReplaceNode -} - -func newCommitLoggerParser(path string, activeMemtable *Memtable, - strategy string, metrics *Metrics, -) *commitloggerParser { - return &commitloggerParser{ - path: path, - memtable: activeMemtable, - strategy: strategy, - metrics: metrics, - replaceCache: map[string]segmentReplaceNode{}, - } -} - -func (p *commitloggerParser) Do() error { - switch p.strategy { - case StrategyReplace: - return p.doReplace() - case StrategyMapCollection, StrategySetCollection: - return p.doCollection() - case StrategyRoaringSet: - return p.doRoaringSet() - default: - return errors.Errorf("unknown strategy %s on commit log parse", p.strategy) - } -} - -// doReplace parsers all entries into a cache for deduplication first and only -// imports unique entries into the actual memtable as a final step. -func (p *commitloggerParser) doReplace() error { - f, err := os.Open(p.path) - if err != nil { - return err - } - - metered := diskio.NewMeteredReader(f, p.metrics.TrackStartupReadWALDiskIO) - p.reader = bufio.NewReaderSize(metered, 1*1024*1024) - - // errUnexpectedLength indicates that we could not read the commit log to the - // end, for example because the last element on the log was corrupt. - var errUnexpectedLength error - - for { - var commitType CommitType - - err := binary.Read(p.reader, binary.LittleEndian, &commitType) - if err == io.EOF { - break - } - - if err != nil { - errUnexpectedLength = errors.Wrap(err, "read commit type") - break - } - - if CommitTypeReplace.Is(commitType) { - if err := p.parseReplaceNode(); err != nil { - errUnexpectedLength = errors.Wrap(err, "read replace node") - break - } - } else { - f.Close() - return errors.Errorf("found a %s commit on a replace bucket", commitType.String()) - } - } - - for _, node := range p.replaceCache { - var opts []SecondaryKeyOption - if p.memtable.secondaryIndices > 0 { - for i, secKey := range node.secondaryKeys { - opts = append(opts, WithSecondaryKey(i, secKey)) - } - } - if node.tombstone { - p.memtable.setTombstone(node.primaryKey, opts...) - } else { - p.memtable.put(node.primaryKey, node.value, opts...) - } - } - - if errUnexpectedLength != nil { - f.Close() - return errUnexpectedLength - } - - return f.Close() -} - -// parseReplaceNode only parses into the deduplication cache, not into the -// final memtable yet. A second step is required to parse from the cache into -// the actual memtable. -func (p *commitloggerParser) parseReplaceNode() error { - n, err := ParseReplaceNode(p.reader, p.memtable.secondaryIndices) - if err != nil { - return err - } - - if !n.tombstone { - p.replaceCache[string(n.primaryKey)] = n - } else { - if existing, ok := p.replaceCache[string(n.primaryKey)]; ok { - existing.tombstone = true - p.replaceCache[string(n.primaryKey)] = existing - } else { - p.replaceCache[string(n.primaryKey)] = n - } - } - - return nil -} diff --git a/adapters/repos/db/lsmkv/commitlogger_parser_collection.go b/adapters/repos/db/lsmkv/commitlogger_parser_collection.go deleted file mode 100644 index 1a51ae73c53dc0feff079f07fdc967abf35f258c..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/commitlogger_parser_collection.go +++ /dev/null @@ -1,86 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "bufio" - "encoding/binary" - "io" - "os" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/diskio" -) - -func (p *commitloggerParser) doCollection() error { - f, err := os.Open(p.path) - if err != nil { - return err - } - - metered := diskio.NewMeteredReader(f, p.metrics.TrackStartupReadWALDiskIO) - p.reader = bufio.NewReaderSize(metered, 1*1024*1024) - - for { - var commitType CommitType - - err := binary.Read(p.reader, binary.LittleEndian, &commitType) - if err == io.EOF { - break - } - - if err != nil { - f.Close() - return errors.Wrap(err, "read commit type") - } - - if CommitTypeCollection.Is(commitType) { - if err := p.parseCollectionNode(); err != nil { - f.Close() - return errors.Wrap(err, "read collection node") - } - } else { - f.Close() - return errors.Errorf("found a %s commit on collection bucket", commitType.String()) - } - } - - return f.Close() -} - -func (p *commitloggerParser) parseCollectionNode() error { - n, err := ParseCollectionNode(p.reader) - if err != nil { - return err - } - - if p.strategy == StrategyMapCollection { - return p.parseMapNode(n) - } - return p.memtable.append(n.primaryKey, n.values) -} - -func (p *commitloggerParser) parseMapNode(n segmentCollectionNode) error { - for _, val := range n.values { - mp := MapPair{} - if err := mp.FromBytes(val.value, false); err != nil { - return err - } - mp.Tombstone = val.tombstone - - if err := p.memtable.appendMapSorted(n.primaryKey, mp); err != nil { - return err - } - } - - return nil -} diff --git a/adapters/repos/db/lsmkv/commitlogger_parser_roaringset.go b/adapters/repos/db/lsmkv/commitlogger_parser_roaringset.go deleted file mode 100644 index aa4e195e81c7a7d579a6d3ddb5f9ff36c3690ecc..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/commitlogger_parser_roaringset.go +++ /dev/null @@ -1,81 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "bufio" - "encoding/binary" - "io" - "os" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv/roaringset" - "github.com/weaviate/weaviate/entities/diskio" -) - -func (p *commitloggerParser) doRoaringSet() error { - f, err := os.Open(p.path) - if err != nil { - return err - } - - metered := diskio.NewMeteredReader(f, p.metrics.TrackStartupReadWALDiskIO) - p.reader = bufio.NewReaderSize(metered, 1*1024*1024) - - for { - var commitType CommitType - - err := binary.Read(p.reader, binary.LittleEndian, &commitType) - if err == io.EOF { - break - } - - if err != nil { - f.Close() - return errors.Wrap(err, "read commit type") - } - - if CommitTypeRoaringSet.Is(commitType) { - if err := p.parseRoaringSetNode(); err != nil { - f.Close() - return errors.Wrap(err, "read collection node") - } - } else { - f.Close() - return errors.Errorf("found a %s commit on collection bucket", commitType.String()) - } - } - - return f.Close() -} - -func (p *commitloggerParser) parseRoaringSetNode() error { - lenBuf := make([]byte, 8) - if _, err := io.ReadFull(p.reader, lenBuf); err != nil { - return errors.Wrap(err, "read segment len") - } - segmentLen := binary.LittleEndian.Uint64(lenBuf) - - segBuf := make([]byte, segmentLen) - copy(segBuf, lenBuf) - if _, err := io.ReadFull(p.reader, segBuf[8:]); err != nil { - return errors.Wrap(err, "read segment contents") - } - - segment := roaringset.NewSegmentNodeFromBuffer(segBuf) - key := segment.PrimaryKey() - if err := p.memtable.roaringSetAddRemoveBitmaps(key, segment.Additions(), segment.Deletions()); err != nil { - return errors.Wrap(err, "add/remove bitmaps") - } - - return nil -} diff --git a/adapters/repos/db/lsmkv/compaction_integration2_test.go b/adapters/repos/db/lsmkv/compaction_integration2_test.go deleted file mode 100644 index c33931a13eca7b67b23c67c390aa5f7cdc04a254..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/compaction_integration2_test.go +++ /dev/null @@ -1,269 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "context" - "fmt" - "math/rand" - "testing" - - "github.com/sirupsen/logrus" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/cyclemanager" -) - -func TestCompactionReplaceStrategyStraggler(t *testing.T) { - opts := []BucketOption{WithStrategy(StrategyReplace)} - size := 200 - - type kv struct { - key []byte - value []byte - delete bool - } - - var segment1 []kv - var segment2 []kv - var segment3 []kv - var expected []kv - var bucket *Bucket - - dirName := t.TempDir() - - t.Run("create test data", func(t *testing.T) { - // The test data is split into 4 scenarios evenly: - // - // 1.) created in the first segment, never touched again - // 2.) created in the first segment, updated in the second - // 3.) created in the first segment, deleted in the second - // 4.) not present in the first segment, created in the second - for i := 0; i < size; i++ { - key := []byte(fmt.Sprintf("key-%3d", i)) - originalValue := []byte(fmt.Sprintf("value-%3d-original", i)) - - switch i % 4 { - case 0: - // add to segment 1 - segment1 = append(segment1, kv{ - key: key, - value: originalValue, - }) - - // leave this element untouched in the second segment - expected = append(expected, kv{ - key: key, - value: originalValue, - }) - case 1: - // add to segment 1 - segment1 = append(segment1, kv{ - key: key, - value: originalValue, - }) - - // update in the second segment - updatedValue := []byte(fmt.Sprintf("value-%3d-updated", i)) - segment2 = append(segment2, kv{ - key: key, - value: updatedValue, - }) - // update in the third segment - updatedValue = []byte(fmt.Sprintf("value-%3d-updated-twice", i)) - segment3 = append(segment3, kv{ - key: key, - value: updatedValue, - }) - - expected = append(expected, kv{ - key: key, - value: updatedValue, - }) - case 2: - // add to segment 1 - segment1 = append(segment1, kv{ - key: key, - value: originalValue, - }) - - // delete in the third segment - segment3 = append(segment3, kv{ - key: key, - delete: true, - }) - - // do not add to expected at all - - case 3: - // do not add to segment 1 - - // only add to segment 3 (first entry) - segment3 = append(segment3, kv{ - key: key, - value: originalValue, - }) - - expected = append(expected, kv{ - key: key, - value: originalValue, - }) - } - } - }) - - t.Run("shuffle the import order for each segment", func(t *testing.T) { - // this is to make sure we don't accidentally rely on the import order - rand.Shuffle(len(segment1), func(i, j int) { - segment1[i], segment1[j] = segment1[j], segment1[i] - }) - rand.Shuffle(len(segment2), func(i, j int) { - segment2[i], segment2[j] = segment2[j], segment2[i] - }) - }) - - t.Run("init bucket", func(t *testing.T) { - b, err := NewBucket(context.TODO(), dirName, "", nullLogger2(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - bucket = b - }) - - t.Run("import segment 1", func(t *testing.T) { - for _, pair := range segment1 { - if !pair.delete { - err := bucket.Put(pair.key, pair.value) - require.Nil(t, err) - } else { - err := bucket.Delete(pair.key) - require.Nil(t, err) - - } - } - }) - - t.Run("flush to disk", func(t *testing.T) { - require.Nil(t, bucket.FlushAndSwitch()) - }) - - t.Run("import segment 2", func(t *testing.T) { - for _, pair := range segment2 { - if !pair.delete { - err := bucket.Put(pair.key, pair.value) - require.Nil(t, err) - } else { - err := bucket.Delete(pair.key) - require.Nil(t, err) - - } - } - }) - - t.Run("flush to disk", func(t *testing.T) { - require.Nil(t, bucket.FlushAndSwitch()) - }) - - t.Run("import segment 3", func(t *testing.T) { - for _, pair := range segment3 { - if !pair.delete { - err := bucket.Put(pair.key, pair.value) - require.Nil(t, err) - } else { - err := bucket.Delete(pair.key) - require.Nil(t, err) - - } - } - }) - - t.Run("flush to disk", func(t *testing.T) { - require.Nil(t, bucket.FlushAndSwitch()) - }) - - t.Run("verify control before compaction", func(t *testing.T) { - var retrieved []kv - - c := bucket.Cursor() - defer c.Close() - - for k, v := c.First(); k != nil; k, v = c.Next() { - keyCopy := copyByteSlice2(k) - valueCopy := copyByteSlice2(v) - retrieved = append(retrieved, kv{ - key: keyCopy, - value: valueCopy, - }) - } - - assert.Equal(t, expected, retrieved) - }) - - t.Run("verify count control before compaction", func(*testing.T) { - assert.Equal(t, len(expected), bucket.Count()) - }) - - t.Run("compact until no longer eligible", func(t *testing.T) { - var compacted bool - var err error - for compacted, err = bucket.disk.compactOnce(); err == nil && compacted; compacted, err = bucket.disk.compactOnce() { - } - require.Nil(t, err) - }) - - t.Run("verify control after compaction", func(t *testing.T) { - var retrieved []kv - - c := bucket.Cursor() - defer c.Close() - - for k, v := c.First(); k != nil; k, v = c.Next() { - keyCopy := copyByteSlice2(k) - valueCopy := copyByteSlice2(v) - retrieved = append(retrieved, kv{ - key: keyCopy, - value: valueCopy, - }) - } - - assert.Equal(t, expected, retrieved) - }) - - t.Run("verify control using individual get operations", - func(t *testing.T) { - for _, pair := range expected { - retrieved, err := bucket.Get(pair.key) - require.NoError(t, err) - - assert.Equal(t, pair.value, retrieved) - } - }) - - t.Run("verify count after compaction", func(*testing.T) { - assert.Equal(t, len(expected), bucket.Count()) - }) -} - -func nullLogger2() logrus.FieldLogger { - log, _ := test.NewNullLogger() - return log -} - -func copyByteSlice2(src []byte) []byte { - dst := make([]byte, len(src)) - copy(dst, src) - return dst -} diff --git a/adapters/repos/db/lsmkv/compaction_integration_test.go b/adapters/repos/db/lsmkv/compaction_integration_test.go deleted file mode 100644 index 3eccfa1296e948f6d30dfc8fb52116f9fefde6ad..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/compaction_integration_test.go +++ /dev/null @@ -1,369 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package lsmkv - -import ( - "context" - "os" - "path/filepath" - "testing" - - "github.com/sirupsen/logrus" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func testCtx() context.Context { - return context.Background() -} - -type bucketIntegrationTest struct { - name string - f func(context.Context, *testing.T, []BucketOption) - opts []BucketOption -} - -type bucketIntegrationTests []bucketIntegrationTest - -func (tests bucketIntegrationTests) run(ctx context.Context, t *testing.T) { - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - t.Run("mmap", func(t *testing.T) { - test.f(ctx, t, test.opts) - }) - t.Run("pread", func(t *testing.T) { - test.f(ctx, t, append([]BucketOption{WithPread(true)}, test.opts...)) - }) - }) - } -} - -func TestCompaction(t *testing.T) { - ctx := testCtx() - tests := bucketIntegrationTests{ - { - name: "compactionReplaceStrategy", - f: func(ctx context.Context, t *testing.T, opts []BucketOption) { - compactionReplaceStrategy(ctx, t, opts, 12116, 12116) - }, - opts: []BucketOption{ - WithStrategy(StrategyReplace), - }, - }, - { - name: "compactionReplaceStrategy_KeepTombstones", - f: func(ctx context.Context, t *testing.T, opts []BucketOption) { - compactionReplaceStrategy(ctx, t, opts, 15266, 15266) - }, - opts: []BucketOption{ - WithStrategy(StrategyReplace), - WithKeepTombstones(true), - }, - }, - { - name: "compactionReplaceStrategy_WithSecondaryKeys", - f: compactionReplaceStrategy_WithSecondaryKeys, - opts: []BucketOption{ - WithStrategy(StrategyReplace), - WithSecondaryIndices(1), - }, - }, - { - name: "compactionReplaceStrategy_WithSecondaryKeys_KeepTombstones", - f: compactionReplaceStrategy_WithSecondaryKeys, - opts: []BucketOption{ - WithStrategy(StrategyReplace), - WithSecondaryIndices(1), - WithKeepTombstones(true), - }, - }, - { - name: "compactionReplaceStrategy_RemoveUnnecessaryDeletes", - f: compactionReplaceStrategy_RemoveUnnecessaryDeletes, - opts: []BucketOption{ - WithStrategy(StrategyReplace), - }, - }, - { - name: "compactionReplaceStrategy_RemoveUnnecessaryDeletes_KeepTombstones", - f: compactionReplaceStrategy_RemoveUnnecessaryDeletes, - opts: []BucketOption{ - WithStrategy(StrategyReplace), - WithKeepTombstones(true), - }, - }, - { - name: "compactionReplaceStrategy_RemoveUnnecessaryUpdates", - f: compactionReplaceStrategy_RemoveUnnecessaryUpdates, - opts: []BucketOption{ - WithStrategy(StrategyReplace), - }, - }, - { - name: "compactionReplaceStrategy_RemoveUnnecessaryUpdates_KeepTombstones", - f: compactionReplaceStrategy_RemoveUnnecessaryUpdates, - opts: []BucketOption{ - WithStrategy(StrategyReplace), - WithKeepTombstones(true), - }, - }, - { - name: "compactionReplaceStrategy_FrequentPutDeleteOperations", - f: compactionReplaceStrategy_FrequentPutDeleteOperations, - opts: []BucketOption{ - WithStrategy(StrategyReplace), - }, - }, - { - name: "compactionReplaceStrategy_FrequentPutDeleteOperations_KeepTombstones", - f: compactionReplaceStrategy_FrequentPutDeleteOperations, - opts: []BucketOption{ - WithStrategy(StrategyReplace), - WithKeepTombstones(true), - }, - }, - { - name: "compactionReplaceStrategy_FrequentPutDeleteOperations_WithSecondaryKeys", - f: compactionReplaceStrategy_FrequentPutDeleteOperations_WithSecondaryKeys, - opts: []BucketOption{ - WithStrategy(StrategyReplace), - WithSecondaryIndices(1), - }, - }, - { - name: "compactionReplaceStrategy_FrequentPutDeleteOperations_WithSecondaryKeys_KeepTombstones", - f: compactionReplaceStrategy_FrequentPutDeleteOperations_WithSecondaryKeys, - opts: []BucketOption{ - WithStrategy(StrategyReplace), - WithSecondaryIndices(1), - WithKeepTombstones(true), - }, - }, - - { - name: "compactionSetStrategy", - f: func(ctx context.Context, t *testing.T, opts []BucketOption) { - compactionSetStrategy(ctx, t, opts, 6836, 6836) - }, - opts: []BucketOption{ - WithStrategy(StrategySetCollection), - }, - }, - { - name: "compactionSetStrategy_KeepTombstones", - f: func(ctx context.Context, t *testing.T, opts []BucketOption) { - compactionSetStrategy(ctx, t, opts, 9756, 9756) - }, - opts: []BucketOption{ - WithStrategy(StrategySetCollection), - WithKeepTombstones(true), - }, - }, - { - name: "compactionSetStrategy_RemoveUnnecessary", - f: compactionSetStrategy_RemoveUnnecessary, - opts: []BucketOption{ - WithStrategy(StrategySetCollection), - }, - }, - { - name: "compactionSetStrategy_RemoveUnnecessary_KeepTombstones", - f: compactionSetStrategy_RemoveUnnecessary, - opts: []BucketOption{ - WithStrategy(StrategySetCollection), - WithKeepTombstones(true), - }, - }, - { - name: "compactionSetStrategy_FrequentPutDeleteOperations", - f: compactionSetStrategy_FrequentPutDeleteOperations, - opts: []BucketOption{ - WithStrategy(StrategySetCollection), - }, - }, - { - name: "compactionSetStrategy_FrequentPutDeleteOperations_KeepTombstones", - f: compactionSetStrategy_FrequentPutDeleteOperations, - opts: []BucketOption{ - WithStrategy(StrategySetCollection), - WithKeepTombstones(true), - }, - }, - - { - name: "compactionMapStrategy", - f: func(ctx context.Context, t *testing.T, opts []BucketOption) { - compactionMapStrategy(ctx, t, opts, 10676, 10676) - }, - opts: []BucketOption{ - WithStrategy(StrategyMapCollection), - }, - }, - { - name: "compactionMapStrategy_KeepTombstones", - f: func(ctx context.Context, t *testing.T, opts []BucketOption) { - compactionMapStrategy(ctx, t, opts, 13416, 13416) - }, - opts: []BucketOption{ - WithStrategy(StrategyMapCollection), - WithKeepTombstones(true), - }, - }, - { - name: "compactionMapStrategy_RemoveUnnecessary", - f: compactionMapStrategy_RemoveUnnecessary, - opts: []BucketOption{ - WithStrategy(StrategyMapCollection), - }, - }, - { - name: "compactionMapStrategy_RemoveUnnecessary_KeepTombstones", - f: compactionMapStrategy_RemoveUnnecessary, - opts: []BucketOption{ - WithStrategy(StrategyMapCollection), - WithKeepTombstones(true), - }, - }, - { - name: "compactionMapStrategy_FrequentPutDeleteOperations", - f: compactionMapStrategy_FrequentPutDeleteOperations, - opts: []BucketOption{ - WithStrategy(StrategyMapCollection), - }, - }, - { - name: "compactionMapStrategy_FrequentPutDeleteOperations_KeepTombstones", - f: compactionMapStrategy_FrequentPutDeleteOperations, - opts: []BucketOption{ - WithStrategy(StrategyMapCollection), - WithKeepTombstones(true), - }, - }, - - { - name: "compactionRoaringSetStrategy_Random", - f: compactionRoaringSetStrategy_Random, - opts: []BucketOption{ - WithStrategy(StrategyRoaringSet), - }, - }, - { - name: "compactionRoaringSetStrategy_Random_KeepTombstones", - f: compactionRoaringSetStrategy_Random, - opts: []BucketOption{ - WithStrategy(StrategyRoaringSet), - WithKeepTombstones(true), - }, - }, - { - name: "compactionRoaringSetStrategy", - f: func(ctx context.Context, t *testing.T, opts []BucketOption) { - compactionRoaringSetStrategy(ctx, t, opts, 19168, 19168) - }, - opts: []BucketOption{ - WithStrategy(StrategyRoaringSet), - }, - }, - { - name: "compactionRoaringSetStrategy_KeepTombstones", - f: func(ctx context.Context, t *testing.T, opts []BucketOption) { - compactionRoaringSetStrategy(ctx, t, opts, 29792, 29792) - }, - opts: []BucketOption{ - WithStrategy(StrategyRoaringSet), - WithKeepTombstones(true), - }, - }, - { - name: "compactionRoaringSetStrategy_RemoveUnnecessary", - f: compactionRoaringSetStrategy_RemoveUnnecessary, - opts: []BucketOption{ - WithStrategy(StrategyRoaringSet), - }, - }, - { - name: "compactionRoaringSetStrategy_RemoveUnnecessary_KeepTombstones", - f: compactionRoaringSetStrategy_RemoveUnnecessary, - opts: []BucketOption{ - WithStrategy(StrategyRoaringSet), - WithKeepTombstones(true), - }, - }, - { - name: "compactionRoaringSetStrategy_FrequentPutDeleteOperations", - f: compactionRoaringSetStrategy_FrequentPutDeleteOperations, - opts: []BucketOption{ - WithStrategy(StrategyRoaringSet), - }, - }, - { - name: "compactionRoaringSetStrategy_FrequentPutDeleteOperations_KeepTombstones", - f: compactionRoaringSetStrategy_FrequentPutDeleteOperations, - opts: []BucketOption{ - WithStrategy(StrategyRoaringSet), - WithKeepTombstones(true), - }, - }, - } - tests.run(ctx, t) -} - -func nullLogger() logrus.FieldLogger { - log, _ := test.NewNullLogger() - return log -} - -func copyByteSlice(src []byte) []byte { - dst := make([]byte, len(src)) - copy(dst, src) - return dst -} - -func assertSingleSegmentOfSize(t *testing.T, bucket *Bucket, expectedMinSize, expectedMaxSize int64) { - files, err := bucket.ListFiles(context.Background(), bucket.dir) - require.NoError(t, err) - - dbFiles := make([]string, 0, len(files)) - for _, f := range files { - if filepath.Ext(f) == ".db" { - dbFiles = append(dbFiles, f) - } - } - require.Len(t, dbFiles, 1) - - fi, err := os.Stat(dbFiles[0]) - require.NoError(t, err) - assert.LessOrEqual(t, expectedMinSize, fi.Size()) - assert.GreaterOrEqual(t, expectedMaxSize, fi.Size()) -} - -func assertSecondSegmentOfSize(t *testing.T, bucket *Bucket, expectedMinSize, expectedMaxSize int64) { - files, err := bucket.ListFiles(context.Background(), bucket.dir) - require.NoError(t, err) - - dbFiles := make([]string, 0, len(files)) - for _, f := range files { - if filepath.Ext(f) == ".db" { - dbFiles = append(dbFiles, f) - } - } - require.Len(t, dbFiles, 2) - - fi, err := os.Stat(dbFiles[1]) - require.NoError(t, err) - assert.LessOrEqual(t, expectedMinSize, fi.Size()) - assert.GreaterOrEqual(t, expectedMaxSize, fi.Size()) -} diff --git a/adapters/repos/db/lsmkv/compaction_map_integration_test.go b/adapters/repos/db/lsmkv/compaction_map_integration_test.go deleted file mode 100644 index fd10852ee6e8eb54879e444afcfef340980fc5db..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/compaction_map_integration_test.go +++ /dev/null @@ -1,640 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package lsmkv - -import ( - "bytes" - "context" - "fmt" - "math/rand" - "sort" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/cyclemanager" -) - -func compactionMapStrategy(ctx context.Context, t *testing.T, opts []BucketOption, - expectedMinSize, expectedMaxSize int64, -) { - size := 100 - - type kv struct { - key []byte - values []MapPair - } - - // this segment is not part of the merge, but might still play a role in - // overall results. For example if one of the later segments has a tombstone - // for it - var previous1 []kv - var previous2 []kv - - var segment1 []kv - var segment2 []kv - var expected []kv - var bucket *Bucket - - dirName := t.TempDir() - - t.Run("create test data", func(t *testing.T) { - // The test data is split into 4 scenarios evenly: - // - // 0.) created in the first segment, never touched again - // 1.) created in the first segment, appended to it in the second - // 2.) created in the first segment, first element updated in the second - // 3.) created in the first segment, second element updated in the second - // 4.) created in the first segment, first element deleted in the second - // 5.) created in the first segment, second element deleted in the second - // 6.) not present in the first segment, created in the second - // 7.) present in an unrelated previous segment, deleted in the first - // 8.) present in an unrelated previous segment, deleted in the second - // 9.) present in an unrelated previous segment, never touched again - for i := 0; i < size; i++ { - rowKey := []byte(fmt.Sprintf("row-%03d", i)) - - pair1 := MapPair{ - Key: []byte(fmt.Sprintf("value-%03d-01", i)), - Value: []byte(fmt.Sprintf("value-%03d-01-original", i)), - } - pair2 := MapPair{ - Key: []byte(fmt.Sprintf("value-%03d-02", i)), - Value: []byte(fmt.Sprintf("value-%03d-02-original", i)), - } - pairs := []MapPair{pair1, pair2} - - switch i % 10 { - case 0: - // add to segment 1 - segment1 = append(segment1, kv{ - key: rowKey, - values: pairs[:1], - }) - - // leave this element untouched in the second segment - expected = append(expected, kv{ - key: rowKey, - values: pairs[:1], - }) - case 1: - // add to segment 1 - segment1 = append(segment1, kv{ - key: rowKey, - values: pairs[:1], - }) - - // add extra pair in the second segment - segment2 = append(segment2, kv{ - key: rowKey, - values: pairs[1:2], - }) - - expected = append(expected, kv{ - key: rowKey, - values: pairs, - }) - case 2: - // add both to segment 1 - segment1 = append(segment1, kv{ - key: rowKey, - values: pairs, - }) - - // update first key in the second segment - updated := pair1 - updated.Value = []byte("updated") - - segment2 = append(segment2, kv{ - key: rowKey, - values: []MapPair{updated}, - }) - - expected = append(expected, kv{ - key: rowKey, - values: []MapPair{pair2, updated}, - }) - - case 3: - // add both to segment 1 - segment1 = append(segment1, kv{ - key: rowKey, - values: pairs, - }) - - // update first key in the second segment - updated := pair2 - updated.Value = []byte("updated") - - segment2 = append(segment2, kv{ - key: rowKey, - values: []MapPair{updated}, - }) - - expected = append(expected, kv{ - key: rowKey, - values: []MapPair{pair1, updated}, - }) - - case 4: - // add both to segment 1 - segment1 = append(segment1, kv{ - key: rowKey, - values: pairs, - }) - - // delete first key in the second segment - updated := pair1 - updated.Value = nil - updated.Tombstone = true - - segment2 = append(segment2, kv{ - key: rowKey, - values: []MapPair{updated}, - }) - - expected = append(expected, kv{ - key: rowKey, - values: []MapPair{pair2}, - }) - - case 5: - // add both to segment 1 - segment1 = append(segment1, kv{ - key: rowKey, - values: pairs, - }) - - // delete second key in the second segment - updated := pair2 - updated.Value = nil - updated.Tombstone = true - - segment2 = append(segment2, kv{ - key: rowKey, - values: []MapPair{updated}, - }) - - expected = append(expected, kv{ - key: rowKey, - values: []MapPair{pair1}, - }) - - case 6: - // do not add to segment 2 - - // only add to segment 2 (first entry) - segment2 = append(segment2, kv{ - key: rowKey, - values: pairs, - }) - - expected = append(expected, kv{ - key: rowKey, - values: pairs, - }) - - case 7: - // only part of a previous segment, which is not part of the merge - previous1 = append(previous1, kv{ - key: rowKey, - values: pairs[:1], - }) - previous2 = append(previous2, kv{ - key: rowKey, - values: pairs[1:], - }) - - // delete in segment 1 - deleted1 := pair1 - deleted1.Value = nil - deleted1.Tombstone = true - - deleted2 := pair2 - deleted2.Value = nil - deleted2.Tombstone = true - - segment1 = append(segment1, kv{ - key: rowKey, - values: []MapPair{deleted1}, - }) - segment1 = append(segment1, kv{ - key: rowKey, - values: []MapPair{deleted2}, - }) - - // should not have any values in expected at all, not even a key - - case 8: - // only part of a previous segment, which is not part of the merge - previous1 = append(previous1, kv{ - key: rowKey, - values: pairs[:1], - }) - previous2 = append(previous2, kv{ - key: rowKey, - values: pairs[1:], - }) - - // delete in segment 1 - deleted1 := pair1 - deleted1.Value = nil - deleted1.Tombstone = true - - deleted2 := pair2 - deleted2.Value = nil - deleted2.Tombstone = true - - segment2 = append(segment2, kv{ - key: rowKey, - values: []MapPair{deleted1}, - }) - segment2 = append(segment2, kv{ - key: rowKey, - values: []MapPair{deleted2}, - }) - - // should not have any values in expected at all, not even a key - - case 9: - // only part of a previous segment - previous1 = append(previous1, kv{ - key: rowKey, - values: pairs[:1], - }) - previous2 = append(previous2, kv{ - key: rowKey, - values: pairs[1:], - }) - - expected = append(expected, kv{ - key: rowKey, - values: pairs, - }) - } - } - }) - - t.Run("shuffle the import order for each segment", func(t *testing.T) { - // this is to make sure we don't accidentally rely on the import order - rand.Shuffle(len(segment1), func(i, j int) { - segment1[i], segment1[j] = segment1[j], segment1[i] - }) - rand.Shuffle(len(segment2), func(i, j int) { - segment2[i], segment2[j] = segment2[j], segment2[i] - }) - }) - - t.Run("init bucket", func(t *testing.T) { - b, err := NewBucket(ctx, dirName, dirName, nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - bucket = b - }) - - t.Run("import and flush previous segments", func(t *testing.T) { - for _, kvs := range previous1 { - for _, pair := range kvs.values { - err := bucket.MapSet(kvs.key, pair) - require.Nil(t, err) - } - } - - require.Nil(t, bucket.FlushAndSwitch()) - - for _, kvs := range previous2 { - for _, pair := range kvs.values { - err := bucket.MapSet(kvs.key, pair) - require.Nil(t, err) - } - } - - require.Nil(t, bucket.FlushAndSwitch()) - }) - - t.Run("import segment 1", func(t *testing.T) { - for _, kvs := range segment1 { - for _, pair := range kvs.values { - err := bucket.MapSet(kvs.key, pair) - require.Nil(t, err) - } - } - }) - - t.Run("flush to disk", func(t *testing.T) { - require.Nil(t, bucket.FlushAndSwitch()) - }) - - t.Run("import segment 2", func(t *testing.T) { - for _, kvs := range segment2 { - for _, pair := range kvs.values { - err := bucket.MapSet(kvs.key, pair) - require.Nil(t, err) - } - } - }) - - t.Run("flush to disk", func(t *testing.T) { - require.Nil(t, bucket.FlushAndSwitch()) - }) - - t.Run("within control make sure map keys are sorted", func(t *testing.T) { - for i := range expected { - sort.Slice(expected[i].values, func(a, b int) bool { - return bytes.Compare(expected[i].values[a].Key, expected[i].values[b].Key) < 0 - }) - } - }) - - t.Run("verify control before compaction", func(t *testing.T) { - var retrieved []kv - - c := bucket.MapCursor() - defer c.Close() - - for k, v := c.First(); k != nil; k, v = c.Next() { - retrieved = append(retrieved, kv{ - key: k, - values: v, - }) - } - - assert.Equal(t, expected, retrieved) - }) - - t.Run("compact until no longer eligible", func(t *testing.T) { - i := 0 - var compacted bool - var err error - for compacted, err = bucket.disk.compactOnce(); err == nil && compacted; compacted, err = bucket.disk.compactOnce() { - if i == 1 { - // segment1 and segment2 merged - // none of them is root segment, so tombstones - // will not be removed regardless of keepTombstones setting - assertSecondSegmentOfSize(t, bucket, 11876, 11876) - } - i++ - } - require.Nil(t, err) - }) - - t.Run("verify control after compaction using a cursor", func(t *testing.T) { - var retrieved []kv - - c := bucket.MapCursor() - defer c.Close() - - for k, v := c.First(); k != nil; k, v = c.Next() { - retrieved = append(retrieved, kv{ - key: k, - values: v, - }) - } - - assert.Equal(t, expected, retrieved) - assertSingleSegmentOfSize(t, bucket, expectedMinSize, expectedMaxSize) - }) - - t.Run("verify control using individual get (MapList) operations", - func(t *testing.T) { - // Previously the only verification was done using the cursor. That - // guaranteed that all pairs are present in the payload, but it did not - // guarantee the integrity of the index (DiskTree) which is used to access - // _individual_ keys. Corrupting this index is exactly what happened in - // https://github.com/weaviate/weaviate/issues/3517 - for _, pair := range expected { - retrieved, err := bucket.MapList(pair.key) - require.NoError(t, err) - - assert.Equal(t, pair.values, retrieved) - } - }) -} - -func compactionMapStrategy_RemoveUnnecessary(ctx context.Context, t *testing.T, opts []BucketOption) { - // in this test each segment reverses the action of the previous segment so - // that in the end a lot of information is present in the individual segments - // which is no longer needed. We then verify that after all compaction this - // information is gone, thus freeing up disk space - size := 100 - - type kv struct { - key []byte - values []MapPair - } - - key := []byte("my-key") - - var bucket *Bucket - dirName := t.TempDir() - - t.Run("init bucket", func(t *testing.T) { - b, err := NewBucket(ctx, dirName, dirName, nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - bucket = b - }) - - t.Run("write segments", func(t *testing.T) { - for i := 0; i < size; i++ { - if i != 0 { - // we can only update an existing value if this isn't the first write - pair := MapPair{ - Key: []byte(fmt.Sprintf("value-%05d", i-1)), - Value: []byte(fmt.Sprintf("updated in round %d", i)), - } - err := bucket.MapSet(key, pair) - require.Nil(t, err) - } - - if i > 1 { - // we can only delete two back an existing value if this isn't the - // first or second write - pair := MapPair{ - Key: []byte(fmt.Sprintf("value-%05d", i-2)), - Tombstone: true, - } - err := bucket.MapSet(key, pair) - require.Nil(t, err) - } - - pair := MapPair{ - Key: []byte(fmt.Sprintf("value-%05d", i)), - Value: []byte("original value"), - } - err := bucket.MapSet(key, pair) - require.Nil(t, err) - - require.Nil(t, bucket.FlushAndSwitch()) - } - }) - - expected := []kv{ - { - key: key, - values: []MapPair{ - { - Key: []byte(fmt.Sprintf("value-%05d", size-2)), - Value: []byte(fmt.Sprintf("updated in round %d", size-1)), - }, - { - Key: []byte(fmt.Sprintf("value-%05d", size-1)), - Value: []byte("original value"), - }, - }, - }, - } - - t.Run("verify control before compaction", func(t *testing.T) { - var retrieved []kv - - c := bucket.MapCursor() - defer c.Close() - - for k, v := c.First(); k != nil; k, v = c.Next() { - retrieved = append(retrieved, kv{ - key: k, - values: v, - }) - } - - assert.Equal(t, expected, retrieved) - }) - - t.Run("compact until no longer eligible", func(t *testing.T) { - var compacted bool - var err error - for compacted, err = bucket.disk.compactOnce(); err == nil && compacted; compacted, err = bucket.disk.compactOnce() { - } - require.Nil(t, err) - }) - - t.Run("verify control before compaction", func(t *testing.T) { - var retrieved []kv - - c := bucket.MapCursor() - defer c.Close() - - for k, v := c.First(); k != nil; k, v = c.Next() { - retrieved = append(retrieved, kv{ - key: k, - values: v, - }) - } - - assert.Equal(t, expected, retrieved) - }) - - t.Run("verify control using individual get (MapList) operations", - func(t *testing.T) { - // Previously the only verification was done using the cursor. That - // guaranteed that all pairs are present in the payload, but it did not - // guarantee the integrity of the index (DiskTree) which is used to access - // _individual_ keys. Corrupting this index is exactly what happened in - // https://github.com/weaviate/weaviate/issues/3517 - for _, pair := range expected { - retrieved, err := bucket.MapList(pair.key) - require.NoError(t, err) - - assert.Equal(t, pair.values, retrieved) - } - }) -} - -func compactionMapStrategy_FrequentPutDeleteOperations(ctx context.Context, t *testing.T, opts []BucketOption) { - // In this test we are testing that the compaction works well for map collection - maxSize := 10 - - key := []byte("my-key") - mapKey := []byte("value-1") - - for size := 4; size < maxSize; size++ { - t.Run(fmt.Sprintf("compact %v segments", size), func(t *testing.T) { - var bucket *Bucket - dirName := t.TempDir() - - t.Run("init bucket", func(t *testing.T) { - b, err := NewBucket(ctx, dirName, dirName, nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - bucket = b - }) - - t.Run("write segments", func(t *testing.T) { - for i := 0; i < size; i++ { - value := []byte(fmt.Sprintf("updated in round %d", i)) - pair := MapPair{Key: mapKey, Value: value} - - err := bucket.MapSet(key, pair) - require.Nil(t, err) - - if size == 5 || size == 6 { - // delete all - err = bucket.MapDeleteKey(key, mapKey) - require.Nil(t, err) - } else if i != size-1 { - // don't delete at the end - err := bucket.MapDeleteKey(key, mapKey) - require.Nil(t, err) - } - - require.Nil(t, bucket.FlushAndSwitch()) - } - }) - - t.Run("check entries before compaction", func(t *testing.T) { - res, err := bucket.MapList(key) - assert.Nil(t, err) - if size == 5 || size == 6 { - assert.Empty(t, res) - } else { - assert.Len(t, res, 1) - assert.Equal(t, false, res[0].Tombstone) - } - }) - - t.Run("compact until no longer eligible", func(t *testing.T) { - var compacted bool - var err error - for compacted, err = bucket.disk.compactOnce(); err == nil && compacted; compacted, err = bucket.disk.compactOnce() { - } - require.Nil(t, err) - }) - - t.Run("check entries after compaction", func(t *testing.T) { - res, err := bucket.MapList(key) - assert.Nil(t, err) - if size == 5 || size == 6 { - assert.Empty(t, res) - } else { - assert.Len(t, res, 1) - assert.Equal(t, false, res[0].Tombstone) - } - }) - }) - } -} diff --git a/adapters/repos/db/lsmkv/compaction_replace_integration_test.go b/adapters/repos/db/lsmkv/compaction_replace_integration_test.go deleted file mode 100644 index 894bfbb6f3f49709d68d3b4f470c4a2003c683bc..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/compaction_replace_integration_test.go +++ /dev/null @@ -1,757 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package lsmkv - -import ( - "context" - "fmt" - "math/rand" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/cyclemanager" -) - -func compactionReplaceStrategy(ctx context.Context, t *testing.T, opts []BucketOption, - expectedMinSize, expectedMaxSize int64, -) { - size := 200 - - type kv struct { - key []byte - value []byte - delete bool - } - - var segment1 []kv - var segment2 []kv - var expected []kv - var bucket *Bucket - - dirName := t.TempDir() - - t.Run("create test data", func(t *testing.T) { - // The test data is split into 4 scenarios evenly: - // - // 1.) created in the first segment, never touched again - // 2.) created in the first segment, updated in the second - // 3.) created in the first segment, deleted in the second - // 4.) not present in the first segment, created in the second - for i := 0; i < size; i++ { - key := []byte(fmt.Sprintf("key-%3d", i)) - originalValue := []byte(fmt.Sprintf("value-%3d-original", i)) - - switch i % 4 { - case 0: - // add to segment 1 - segment1 = append(segment1, kv{ - key: key, - value: originalValue, - }) - - // leave this element untouched in the second segment - expected = append(expected, kv{ - key: key, - value: originalValue, - }) - case 1: - // add to segment 1 - segment1 = append(segment1, kv{ - key: key, - value: originalValue, - }) - - // update in the second segment - updatedValue := []byte(fmt.Sprintf("value-%3d-updated", i)) - segment2 = append(segment2, kv{ - key: key, - value: updatedValue, - }) - - expected = append(expected, kv{ - key: key, - value: updatedValue, - }) - case 2: - // add to segment 1 - segment1 = append(segment1, kv{ - key: key, - value: originalValue, - }) - - // delete in the second segment - segment2 = append(segment2, kv{ - key: key, - delete: true, - }) - - // do not add to expected at all - - case 3: - // do not add to segment 1 - - // only add to segment 2 (first entry) - segment2 = append(segment2, kv{ - key: key, - value: originalValue, - }) - - expected = append(expected, kv{ - key: key, - value: originalValue, - }) - } - } - }) - - t.Run("shuffle the import order for each segment", func(t *testing.T) { - // this is to make sure we don't accidentally rely on the import order - rand.Shuffle(len(segment1), func(i, j int) { - segment1[i], segment1[j] = segment1[j], segment1[i] - }) - rand.Shuffle(len(segment2), func(i, j int) { - segment2[i], segment2[j] = segment2[j], segment2[i] - }) - }) - - t.Run("init bucket", func(t *testing.T) { - b, err := NewBucket(ctx, dirName, dirName, nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - bucket = b - }) - - t.Run("import segment 1", func(t *testing.T) { - for _, pair := range segment1 { - if !pair.delete { - err := bucket.Put(pair.key, pair.value) - require.Nil(t, err) - } else { - err := bucket.Delete(pair.key) - require.Nil(t, err) - - } - } - }) - - t.Run("flush to disk", func(t *testing.T) { - require.Nil(t, bucket.FlushAndSwitch()) - }) - - t.Run("import segment 2", func(t *testing.T) { - for _, pair := range segment2 { - if !pair.delete { - err := bucket.Put(pair.key, pair.value) - require.Nil(t, err) - } else { - err := bucket.Delete(pair.key) - require.Nil(t, err) - - } - } - }) - - t.Run("flush to disk", func(t *testing.T) { - require.Nil(t, bucket.FlushAndSwitch()) - }) - - t.Run("verify control before compaction", func(t *testing.T) { - var retrieved []kv - - c := bucket.Cursor() - defer c.Close() - - for k, v := c.First(); k != nil; k, v = c.Next() { - keyCopy := copyByteSlice(k) - valueCopy := copyByteSlice(v) - retrieved = append(retrieved, kv{ - key: keyCopy, - value: valueCopy, - }) - } - - assert.Equal(t, expected, retrieved) - }) - - t.Run("verify count control before compaction", func(*testing.T) { - assert.Equal(t, len(expected), bucket.Count()) - }) - - t.Run("compact until no longer eligible", func(t *testing.T) { - var compacted bool - var err error - for compacted, err = bucket.disk.compactOnce(); err == nil && compacted; compacted, err = bucket.disk.compactOnce() { - } - require.Nil(t, err) - }) - - t.Run("verify control after compaction", func(t *testing.T) { - var retrieved []kv - - c := bucket.Cursor() - defer c.Close() - - for k, v := c.First(); k != nil; k, v = c.Next() { - keyCopy := copyByteSlice(k) - valueCopy := copyByteSlice(v) - retrieved = append(retrieved, kv{ - key: keyCopy, - value: valueCopy, - }) - } - - assert.Equal(t, expected, retrieved) - assertSingleSegmentOfSize(t, bucket, expectedMinSize, expectedMaxSize) - }) - - t.Run("verify control using individual get operations", - func(t *testing.T) { - for _, pair := range expected { - retrieved, err := bucket.Get(pair.key) - require.NoError(t, err) - - assert.Equal(t, pair.value, retrieved) - } - }) - - t.Run("verify count after compaction", func(*testing.T) { - assert.Equal(t, len(expected), bucket.Count()) - }) -} - -func compactionReplaceStrategy_WithSecondaryKeys(ctx context.Context, t *testing.T, opts []BucketOption) { - size := 4 - - type kv struct { - key []byte - value []byte - secondaryKeys [][]byte - delete bool - } - - var segment1 []kv - var segment2 []kv - var expected []kv - var expectedNotPresent []kv - var bucket *Bucket - - dirName := t.TempDir() - - t.Run("create test data", func(t *testing.T) { - // The test data is split into 4 scenarios evenly: - // - // 1.) created in the first segment, never touched again - // 2.) created in the first segment, updated in the second - // 3.) created in the first segment, deleted in the second - // 4.) not present in the first segment, created in the second - for i := 0; i < size; i++ { - key := []byte(fmt.Sprintf("key-%02d", i)) - secondaryKey := []byte(fmt.Sprintf("secondary-key-%02d", i)) - originalValue := []byte(fmt.Sprintf("value-%2d-original", i)) - - switch i % 4 { - case 0: - // add to segment 1 - segment1 = append(segment1, kv{ - key: key, - secondaryKeys: [][]byte{secondaryKey}, - value: originalValue, - }) - - // leave this element untouched in the second segment - expected = append(expected, kv{ - key: secondaryKey, - value: originalValue, - }) - case 1: - // add to segment 1 - segment1 = append(segment1, kv{ - key: key, - secondaryKeys: [][]byte{secondaryKey}, - value: originalValue, - }) - - // update in the second segment - updatedValue := []byte(fmt.Sprintf("value-%2d-updated", i)) - segment2 = append(segment2, kv{ - key: key, - secondaryKeys: [][]byte{secondaryKey}, - value: updatedValue, - }) - - expected = append(expected, kv{ - key: secondaryKey, - value: updatedValue, - }) - case 2: - // add to segment 1 - segment1 = append(segment1, kv{ - key: key, - secondaryKeys: [][]byte{secondaryKey}, - value: originalValue, - }) - - // delete in the second segment - segment2 = append(segment2, kv{ - key: key, - secondaryKeys: [][]byte{secondaryKey}, - delete: true, - }) - - expectedNotPresent = append(expectedNotPresent, kv{ - key: secondaryKey, - }) - - case 3: - // do not add to segment 1 - - // only add to segment 2 (first entry) - segment2 = append(segment2, kv{ - key: key, - secondaryKeys: [][]byte{secondaryKey}, - value: originalValue, - }) - - expected = append(expected, kv{ - key: secondaryKey, - value: originalValue, - }) - } - } - }) - - t.Run("shuffle the import order for each segment", func(t *testing.T) { - // this is to make sure we don't accidentally rely on the import order - rand.Shuffle(len(segment1), func(i, j int) { - segment1[i], segment1[j] = segment1[j], segment1[i] - }) - rand.Shuffle(len(segment2), func(i, j int) { - segment2[i], segment2[j] = segment2[j], segment2[i] - }) - }) - - t.Run("init bucket", func(t *testing.T) { - b, err := NewBucket(ctx, dirName, dirName, nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - bucket = b - }) - - t.Run("import segment 1", func(t *testing.T) { - for _, pair := range segment1 { - if !pair.delete { - err := bucket.Put(pair.key, pair.value, - WithSecondaryKey(0, pair.secondaryKeys[0])) - require.Nil(t, err) - } else { - err := bucket.Delete(pair.key, - WithSecondaryKey(0, pair.secondaryKeys[0])) - require.Nil(t, err) - - } - } - }) - - t.Run("flush to disk", func(t *testing.T) { - require.Nil(t, bucket.FlushAndSwitch()) - }) - - t.Run("import segment 2", func(t *testing.T) { - for _, pair := range segment2 { - if !pair.delete { - err := bucket.Put(pair.key, pair.value, - WithSecondaryKey(0, pair.secondaryKeys[0])) - require.Nil(t, err) - } else { - err := bucket.Delete(pair.key, - WithSecondaryKey(0, pair.secondaryKeys[0])) - require.Nil(t, err) - - } - } - }) - - t.Run("flush to disk", func(t *testing.T) { - require.Nil(t, bucket.FlushAndSwitch()) - }) - - t.Run("verify control before compaction", func(t *testing.T) { - t.Run("verify the ones that should exist", func(t *testing.T) { - for _, pair := range expected { - res, err := bucket.GetBySecondary(0, pair.key) - require.Nil(t, err) - - assert.Equal(t, pair.value, res) - } - }) - - t.Run("verify the ones that should NOT exist", func(t *testing.T) { - for _, pair := range expectedNotPresent { - res, err := bucket.GetBySecondary(0, pair.key) - require.Nil(t, err) - assert.Nil(t, res) - } - }) - }) - - t.Run("compact until no longer eligible", func(t *testing.T) { - var compacted bool - var err error - for compacted, err = bucket.disk.compactOnce(); err == nil && compacted; compacted, err = bucket.disk.compactOnce() { - } - require.Nil(t, err) - }) - - t.Run("verify control after compaction", func(t *testing.T) { - t.Run("verify the ones that should exist", func(t *testing.T) { - for _, pair := range expected { - res, err := bucket.GetBySecondary(0, pair.key) - require.Nil(t, err) - - assert.Equal(t, pair.value, res) - } - }) - - t.Run("verify the ones that should NOT exist", func(t *testing.T) { - for _, pair := range expectedNotPresent { - res, err := bucket.GetBySecondary(0, pair.key) - require.Nil(t, err) - assert.Nil(t, res) - } - }) - }) -} - -func compactionReplaceStrategy_RemoveUnnecessaryDeletes(ctx context.Context, t *testing.T, opts []BucketOption) { - // in this test each segment reverses the action of the previous segment so - // that in the end a lot of information is present in the individual segments - // which is no longer needed. We then verify that after all compaction this - // information is gone, thus freeing up disk space - size := 100 - - type kv struct { - key []byte - value []byte - } - - key := []byte("my-key") - - var bucket *Bucket - dirName := t.TempDir() - - t.Run("init bucket", func(t *testing.T) { - b, err := NewBucket(ctx, dirName, dirName, nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - bucket = b - }) - - t.Run("write segments", func(t *testing.T) { - for i := 0; i < size; i++ { - if i != 0 { - // we can only update an existing value if this isn't the first write - err := bucket.Delete(key) - require.Nil(t, err) - } - - err := bucket.Put(key, []byte(fmt.Sprintf("set in round %d", i))) - require.Nil(t, err) - - require.Nil(t, bucket.FlushAndSwitch()) - } - }) - - expected := []kv{ - { - key: key, - value: []byte(fmt.Sprintf("set in round %d", size-1)), - }, - } - - t.Run("verify control before compaction", func(t *testing.T) { - var retrieved []kv - - c := bucket.Cursor() - defer c.Close() - - for k, v := c.First(); k != nil; k, v = c.Next() { - retrieved = append(retrieved, kv{ - key: k, - value: v, - }) - } - - assert.Equal(t, expected, retrieved) - }) - - t.Run("compact until no longer eligible", func(t *testing.T) { - var compacted bool - var err error - for compacted, err = bucket.disk.compactOnce(); err == nil && compacted; compacted, err = bucket.disk.compactOnce() { - } - require.Nil(t, err) - }) - - t.Run("verify control before compaction", func(t *testing.T) { - var retrieved []kv - - c := bucket.Cursor() - defer c.Close() - - for k, v := c.First(); k != nil; k, v = c.Next() { - retrieved = append(retrieved, kv{ - key: k, - value: v, - }) - } - - assert.Equal(t, expected, retrieved) - }) -} - -func compactionReplaceStrategy_RemoveUnnecessaryUpdates(ctx context.Context, t *testing.T, opts []BucketOption) { - // in this test each segment reverses the action of the previous segment so - // that in the end a lot of information is present in the individual segments - // which is no longer needed. We then verify that after all compaction this - // information is gone, thus freeing up disk space - size := 100 - - type kv struct { - key []byte - value []byte - } - - key := []byte("my-key") - - var bucket *Bucket - dirName := t.TempDir() - - t.Run("init bucket", func(t *testing.T) { - b, err := NewBucket(ctx, dirName, dirName, nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - bucket = b - }) - - t.Run("write segments", func(t *testing.T) { - for i := 0; i < size; i++ { - err := bucket.Put(key, []byte(fmt.Sprintf("set in round %d", i))) - require.Nil(t, err) - - require.Nil(t, bucket.FlushAndSwitch()) - } - }) - - expected := []kv{ - { - key: key, - value: []byte(fmt.Sprintf("set in round %d", size-1)), - }, - } - - t.Run("verify control before compaction", func(t *testing.T) { - var retrieved []kv - - c := bucket.Cursor() - defer c.Close() - - for k, v := c.First(); k != nil; k, v = c.Next() { - retrieved = append(retrieved, kv{ - key: k, - value: v, - }) - } - - assert.Equal(t, expected, retrieved) - }) - - t.Run("compact until no longer eligible", func(t *testing.T) { - var compacted bool - var err error - for compacted, err = bucket.disk.compactOnce(); err == nil && compacted; compacted, err = bucket.disk.compactOnce() { - } - require.Nil(t, err) - }) - - t.Run("verify control after compaction", func(t *testing.T) { - var retrieved []kv - - c := bucket.Cursor() - defer c.Close() - - for k, v := c.First(); k != nil; k, v = c.Next() { - retrieved = append(retrieved, kv{ - key: k, - value: v, - }) - } - - assert.Equal(t, expected, retrieved) - }) -} - -func compactionReplaceStrategy_FrequentPutDeleteOperations(ctx context.Context, t *testing.T, opts []BucketOption) { - // In this test we are testing that the compaction doesn't make the object to disappear - // We are creating even number of segments in which first we create an object - // then we in the next segment with delete it and we do this operation in loop - // we make sure that the last operation done in the last segment is create object operation - // In this situation after the compaction the object has to exist - size := 100 - - key := []byte("my-key") - - var bucket *Bucket - dirName := t.TempDir() - - t.Run("init bucket", func(t *testing.T) { - b, err := NewBucket(ctx, dirName, dirName, nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - bucket = b - }) - - t.Run("write segments, leave the last segment with value", func(t *testing.T) { - for i := 0; i < size; i++ { - err := bucket.Put(key, []byte(fmt.Sprintf("set in round %d", i))) - require.Nil(t, err) - - if i != size-1 { - // don't delete from the last segment - err := bucket.Delete(key) - require.Nil(t, err) - } - - require.Nil(t, bucket.FlushAndSwitch()) - } - }) - - t.Run("verify that the object exists before compaction", func(t *testing.T) { - res, err := bucket.Get(key) - assert.Nil(t, err) - assert.NotNil(t, res) - }) - - t.Run("compact until no longer eligible", func(t *testing.T) { - var compacted bool - var err error - for compacted, err = bucket.disk.compactOnce(); err == nil && compacted; compacted, err = bucket.disk.compactOnce() { - } - require.Nil(t, err) - }) - - t.Run("verify that the object still exists after compaction", func(t *testing.T) { - res, err := bucket.Get(key) - assert.Nil(t, err) - assert.NotNil(t, res) - }) -} - -func compactionReplaceStrategy_FrequentPutDeleteOperations_WithSecondaryKeys(ctx context.Context, t *testing.T, opts []BucketOption) { - // In this test we are testing that the compaction doesn't make the object to disappear - // We are creating even number of segments in which first we create an object - // then we in the next segment with delete it and we do this operation in loop - // we make sure that the last operation done in the last segment is create object operation - // We are doing this for 4 to 10 segments scenarios, without the fix for firstWithAllKeys - // cursor method that now sets the nextOffset properly, we got discrepancies - // after compaction on 4 and 8 segments scenario. - maxSize := 10 - - for size := 4; size < maxSize; size++ { - t.Run(fmt.Sprintf("compact %v segments", size), func(t *testing.T) { - var bucket *Bucket - - key := []byte("key-original") - keySecondary := []byte(fmt.Sprintf("secondary-key-%02d", size-1)) - - dirName := t.TempDir() - - t.Run("init bucket", func(t *testing.T) { - b, err := NewBucket(ctx, dirName, dirName, nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - bucket = b - }) - - t.Run("write segments, leave the last segment with value", func(t *testing.T) { - for i := 0; i < size; i++ { - secondaryKey := []byte(fmt.Sprintf("secondary-key-%02d", i)) - originalValue := []byte(fmt.Sprintf("value-%2d-original", i)) - - err := bucket.Put(key, originalValue, WithSecondaryKey(0, secondaryKey)) - require.Nil(t, err) - - if i != size-1 { - // don't delete from the last segment - err := bucket.Delete(key, WithSecondaryKey(0, secondaryKey)) - require.Nil(t, err) - } - - require.Nil(t, bucket.FlushAndSwitch()) - } - }) - - t.Run("verify that the object exists before compaction", func(t *testing.T) { - res, err := bucket.GetBySecondary(0, keySecondary) - assert.Nil(t, err) - assert.NotNil(t, res) - res, err = bucket.Get(key) - assert.Nil(t, err) - assert.NotNil(t, res) - }) - - t.Run("compact until no longer eligible", func(t *testing.T) { - var compacted bool - var err error - for compacted, err = bucket.disk.compactOnce(); err == nil && compacted; compacted, err = bucket.disk.compactOnce() { - } - require.Nil(t, err) - }) - - t.Run("verify that the object still exists after compaction", func(t *testing.T) { - res, err := bucket.GetBySecondary(0, keySecondary) - assert.Nil(t, err) - assert.NotNil(t, res) - res, err = bucket.Get(key) - assert.Nil(t, err) - assert.NotNil(t, res) - }) - }) - } -} diff --git a/adapters/repos/db/lsmkv/compaction_roaring_set_integration_test.go b/adapters/repos/db/lsmkv/compaction_roaring_set_integration_test.go deleted file mode 100644 index 436446ccf270a1c3fd7f0c82c1403e3b434e5908..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/compaction_roaring_set_integration_test.go +++ /dev/null @@ -1,629 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package lsmkv - -import ( - "context" - "encoding/binary" - "fmt" - "math/rand" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/sroar" - "github.com/weaviate/weaviate/entities/cyclemanager" -) - -func compactionRoaringSetStrategy_Random(ctx context.Context, t *testing.T, opts []BucketOption) { - maxID := uint64(100) - maxElement := uint64(1e6) - iterations := uint64(100_000) - - deleteRatio := 0.2 // 20% of all operations will be deletes, 80% additions - flushChance := 0.001 // on average one flush per 1000 iterations - - r := getRandomSeed() - - instr := generateRandomInstructions(r, maxID, maxElement, iterations, deleteRatio) - control := controlFromInstructions(instr, maxID) - - b, err := NewBucket(ctx, t.TempDir(), "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - defer b.Shutdown(testCtx()) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - compactions := 0 - for _, inst := range instr { - key := make([]byte, 8) - binary.LittleEndian.PutUint64(key, inst.key) - if inst.addition { - b.RoaringSetAddOne(key, inst.element) - } else { - b.RoaringSetRemoveOne(key, inst.element) - } - - if r.Float64() < flushChance { - require.Nil(t, b.FlushAndSwitch()) - - for compacted, err := b.disk.compactOnce(); err == nil && compacted; compacted, err = b.disk.compactOnce() { - require.Nil(t, err) - compactions++ - } - } - - } - - // this is a sanity check to make sure the test setup actually does what we - // want. With the current setup, we expect on average to have ~100 - // compactions. It would be extremely unexpected to have fewer than 25. - assert.Greater(t, compactions, 25) - - verifyBucketAgainstControl(t, b, control) -} - -func verifyBucketAgainstControl(t *testing.T, b *Bucket, control []*sroar.Bitmap) { - // This test was built before the bucket had cursors, so we are retrieving - // each key individually, rather than cursing over the entire bucket. - // However, this is also good for isolation purposes, this test tests - // compactions, not cursors. - - for i, controlBM := range control { - key := make([]byte, 8) - binary.LittleEndian.PutUint64(key, uint64(i)) - - actual, err := b.RoaringSetGet(key) - require.Nil(t, err) - - assert.Equal(t, controlBM.ToArray(), actual.ToArray()) - - } -} - -type roaringSetInstruction struct { - // is a []byte in reality, but makes the test setup easier if we pretent - // its an int - key uint64 - element uint64 - - // true=addition, false=deletion - addition bool -} - -func generateRandomInstructions(r *rand.Rand, maxID, maxElement, iterations uint64, - deleteRatio float64, -) []roaringSetInstruction { - instr := make([]roaringSetInstruction, iterations) - - for i := range instr { - instr[i].key = uint64(r.Intn(int(maxID))) - instr[i].element = uint64(r.Intn(int(maxElement))) - - if r.Float64() > deleteRatio { - instr[i].addition = true - } else { - instr[i].addition = false - } - } - - return instr -} - -func controlFromInstructions(instr []roaringSetInstruction, maxID uint64) []*sroar.Bitmap { - out := make([]*sroar.Bitmap, maxID) - for i := range out { - out[i] = sroar.NewBitmap() - } - - for _, inst := range instr { - if inst.addition { - out[inst.key].Set(inst.element) - } else { - out[inst.key].Remove(inst.element) - } - } - - return out -} - -func compactionRoaringSetStrategy(ctx context.Context, t *testing.T, opts []BucketOption, - expectedMinSize, expectedMaxSize int64, -) { - size := 100 - - type kv struct { - key []byte - additions []uint64 - deletions []uint64 - } - // this segment is not part of the merge, but might still play a role in - // overall results. For example if one of the later segments has a tombstone - // for it - var previous1 []kv - var previous2 []kv - - var segment1 []kv - var segment2 []kv - var expected []kv - var bucket *Bucket - - dirName := t.TempDir() - - t.Run("create test data", func(t *testing.T) { - // The test data is split into 4 scenarios evenly: - // - // 0.) created in the first segment, never touched again - // 1.) created in the first segment, appended to it in the second - // 2.) created in the first segment, first element deleted in the second - // 3.) created in the first segment, second element deleted in the second - // 4.) not present in the first segment, created in the second - // 5.) present in an unrelated previous segment, deleted in the first - // 6.) present in an unrelated previous segment, deleted in the second - // 7.) present in an unrelated previous segment, never touched again - for i := 0; i < size; i++ { - key := []byte(fmt.Sprintf("key-%02d", i)) - value1 := uint64(i) + 1 - value2 := uint64(i) + 2 - values := []uint64{value1, value2} - - switch i % 8 { - case 0: - // add to segment 1 - segment1 = append(segment1, kv{ - key: key, - additions: values[:1], - }) - - // leave this element untouched in the second segment - expected = append(expected, kv{ - key: key, - additions: values[:1], - }) - - case 1: - // add to segment 1 - segment1 = append(segment1, kv{ - key: key, - additions: values[:1], - }) - - // update in the second segment - segment2 = append(segment2, kv{ - key: key, - additions: values[1:], - }) - - expected = append(expected, kv{ - key: key, - additions: values, - }) - - case 2: - // add both to segment 1, delete the first - segment1 = append(segment1, kv{ - key: key, - additions: values, - }) - - // delete first element in the second segment - segment2 = append(segment2, kv{ - key: key, - deletions: values[:1], - }) - - // only the 2nd element should be left in the expected - expected = append(expected, kv{ - key: key, - additions: values[1:], - }) - - case 3: - // add both to segment 1, delete the second - segment1 = append(segment1, kv{ - key: key, - additions: values, - }) - - // delete second element in the second segment - segment2 = append(segment2, kv{ - key: key, - deletions: values[1:], - }) - - // only the 1st element should be left in the expected - expected = append(expected, kv{ - key: key, - additions: values[:1], - }) - - case 4: - // do not add to segment 1 - - // only add to segment 2 (first entry) - segment2 = append(segment2, kv{ - key: key, - additions: values, - }) - - expected = append(expected, kv{ - key: key, - additions: values, - }) - - case 5: - // only part of a previous segment, which is not part of the merge - previous1 = append(previous1, kv{ - key: key, - additions: values[:1], - }) - previous2 = append(previous2, kv{ - key: key, - additions: values[1:], - }) - - // delete in segment 1 - segment1 = append(segment1, kv{ - key: key, - deletions: values, - }) - - // should not have any values in expected at all, not even a key - - case 6: - // only part of a previous segment, which is not part of the merge - previous1 = append(previous1, kv{ - key: key, - additions: values[:1], - }) - previous2 = append(previous2, kv{ - key: key, - additions: values[1:], - }) - - // delete in segment 2 - segment2 = append(segment2, kv{ - key: key, - deletions: values, - }) - - // should not have any values in expected at all, not even a key - - case 7: - // part of a previous segment - previous1 = append(previous1, kv{ - key: key, - additions: values[:1], - }) - previous2 = append(previous2, kv{ - key: key, - additions: values[1:], - }) - - expected = append(expected, kv{ - key: key, - additions: values, - }) - } - } - }) - - t.Run("shuffle the import order for each segment", func(t *testing.T) { - // this is to make sure we don't accidentally rely on the import order - rand.Shuffle(len(segment1), func(i, j int) { - segment1[i], segment1[j] = segment1[j], segment1[i] - }) - rand.Shuffle(len(segment2), func(i, j int) { - segment2[i], segment2[j] = segment2[j], segment2[i] - }) - }) - - t.Run("init bucket", func(t *testing.T) { - b, err := NewBucket(ctx, dirName, dirName, nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - bucket = b - }) - - t.Run("import and flush previous segments", func(t *testing.T) { - for _, kv := range previous1 { - err := bucket.RoaringSetAddList(kv.key, kv.additions) - require.NoError(t, err) - } - - require.NoError(t, bucket.FlushAndSwitch()) - - for _, kv := range previous2 { - err := bucket.RoaringSetAddList(kv.key, kv.additions) - require.NoError(t, err) - } - - require.NoError(t, bucket.FlushAndSwitch()) - }) - - t.Run("import segment 1", func(t *testing.T) { - for _, kv := range segment1 { - if len(kv.additions) > 0 { - err := bucket.RoaringSetAddList(kv.key, kv.additions) - require.NoError(t, err) - } - for i := range kv.deletions { - err := bucket.RoaringSetRemoveOne(kv.key, kv.deletions[i]) - require.NoError(t, err) - } - } - }) - - t.Run("flush to disk", func(t *testing.T) { - require.NoError(t, bucket.FlushAndSwitch()) - }) - - t.Run("import segment 2", func(t *testing.T) { - for _, kv := range segment2 { - if len(kv.additions) > 0 { - err := bucket.RoaringSetAddList(kv.key, kv.additions) - require.NoError(t, err) - } - for i := range kv.deletions { - err := bucket.RoaringSetRemoveOne(kv.key, kv.deletions[i]) - require.NoError(t, err) - } - } - }) - - t.Run("flush to disk", func(t *testing.T) { - require.NoError(t, bucket.FlushAndSwitch()) - }) - - t.Run("verify control before compaction", func(t *testing.T) { - var retrieved []kv - - c := bucket.CursorRoaringSet() - defer c.Close() - - for k, v := c.First(); k != nil; k, v = c.Next() { - retrieved = append(retrieved, kv{ - key: k, - additions: v.ToArray(), - }) - } - - assert.Equal(t, expected, retrieved) - }) - - t.Run("compact until no longer eligible", func(t *testing.T) { - i := 0 - var compacted bool - var err error - for compacted, err = bucket.disk.compactOnce(); err == nil && compacted; compacted, err = bucket.disk.compactOnce() { - if i == 1 { - // segment1 and segment2 merged - // none of them is root segment, so tombstones - // will not be removed regardless of keepTombstones setting - assertSecondSegmentOfSize(t, bucket, 26768, 26768) - } - i++ - } - require.Nil(t, err) - }) - - t.Run("verify control after compaction", func(t *testing.T) { - var retrieved []kv - - c := bucket.CursorRoaringSet() - defer c.Close() - - for k, v := c.First(); k != nil; k, v = c.Next() { - retrieved = append(retrieved, kv{ - key: k, - additions: v.ToArray(), - }) - } - - assert.Equal(t, expected, retrieved) - assertSingleSegmentOfSize(t, bucket, expectedMinSize, expectedMaxSize) - }) -} - -func compactionRoaringSetStrategy_RemoveUnnecessary(ctx context.Context, t *testing.T, opts []BucketOption) { - // in this test each segment reverses the action of the previous segment so - // that in the end a lot of information is present in the individual segments - // which is no longer needed. We then verify that after all compaction this - // information is gone, thus freeing up disk space - size := 100 - - type kv struct { - key []byte - values []uint64 - } - - key := []byte("my-key") - - var bucket *Bucket - dirName := t.TempDir() - - t.Run("init bucket", func(t *testing.T) { - b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - bucket = b - }) - - t.Run("write segments", func(t *testing.T) { - for i := 0; i < size; i++ { - if i != 0 { - // we can only delete an existing value if this isn't the first write - err := bucket.RoaringSetRemoveOne(key, uint64(i)-1) - require.NoError(t, err) - } - - err := bucket.RoaringSetAddOne(key, uint64(i)) - require.NoError(t, err) - - require.NoError(t, bucket.FlushAndSwitch()) - } - }) - - t.Run("verify control before compaction", func(t *testing.T) { - var retrieved []kv - expected := []kv{ - { - key: key, - values: []uint64{uint64(size) - 1}, - }, - } - - c := bucket.CursorRoaringSet() - defer c.Close() - - for k, v := c.First(); k != nil; k, v = c.Next() { - retrieved = append(retrieved, kv{ - key: k, - values: v.ToArray(), - }) - } - - assert.Equal(t, expected, retrieved) - }) - - t.Run("compact until no longer eligible", func(t *testing.T) { - var compacted bool - var err error - for compacted, err = bucket.disk.compactOnce(); err == nil && compacted; compacted, err = bucket.disk.compactOnce() { - } - require.Nil(t, err) - }) - - t.Run("verify control before compaction", func(t *testing.T) { - var retrieved []kv - expected := []kv{ - { - key: key, - values: []uint64{uint64(size) - 1}, - }, - } - - c := bucket.CursorRoaringSet() - defer c.Close() - - for k, v := c.First(); k != nil; k, v = c.Next() { - retrieved = append(retrieved, kv{ - key: k, - values: v.ToArray(), - }) - } - - assert.Equal(t, expected, retrieved) - }) -} - -func compactionRoaringSetStrategy_FrequentPutDeleteOperations(ctx context.Context, t *testing.T, opts []BucketOption) { - // In this test we are testing that the compaction works well for set collection - maxSize := 10 - - for size := 4; size < maxSize; size++ { - t.Run(fmt.Sprintf("compact %v segments", size), func(t *testing.T) { - var bucket *Bucket - - key := []byte("key-original") - value1 := uint64(1) - value2 := uint64(2) - values := []uint64{value1, value2} - - dirName := t.TempDir() - - t.Run("init bucket", func(t *testing.T) { - b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - bucket = b - }) - - t.Run("import and flush segments", func(t *testing.T) { - for i := 0; i < size; i++ { - err := bucket.RoaringSetAddList(key, values) - require.Nil(t, err) - - if size == 5 { - // delete all - err := bucket.RoaringSetRemoveOne(key, values[0]) - require.Nil(t, err) - err = bucket.RoaringSetRemoveOne(key, values[1]) - require.Nil(t, err) - } else if size == 6 { - // delete only one value - err := bucket.RoaringSetRemoveOne(key, values[0]) - require.Nil(t, err) - } else if i != size-1 { - // don't delete from the last segment - err := bucket.RoaringSetRemoveOne(key, values[0]) - require.Nil(t, err) - err = bucket.RoaringSetRemoveOne(key, values[1]) - require.Nil(t, err) - } - - require.Nil(t, bucket.FlushAndSwitch()) - } - }) - - t.Run("verify that objects exist before compaction", func(t *testing.T) { - res, err := bucket.RoaringSetGet(key) - require.NoError(t, err) - if size == 5 { - assert.Equal(t, 0, res.GetCardinality()) - } else if size == 6 { - assert.Equal(t, 1, res.GetCardinality()) - } else { - assert.Equal(t, 2, res.GetCardinality()) - } - }) - - t.Run("compact until no longer eligible", func(t *testing.T) { - var compacted bool - var err error - for compacted, err = bucket.disk.compactOnce(); err == nil && compacted; compacted, err = bucket.disk.compactOnce() { - } - require.Nil(t, err) - }) - - t.Run("verify that objects exist after compaction", func(t *testing.T) { - res, err := bucket.RoaringSetGet(key) - require.NoError(t, err) - if size == 5 { - assert.Equal(t, 0, res.GetCardinality()) - } else if size == 6 { - assert.Equal(t, 1, res.GetCardinality()) - } else { - assert.Equal(t, 2, res.GetCardinality()) - } - }) - }) - } -} diff --git a/adapters/repos/db/lsmkv/compaction_set_integration_test.go b/adapters/repos/db/lsmkv/compaction_set_integration_test.go deleted file mode 100644 index 38753148302747b6133c74c2d45120c2a23bda01..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/compaction_set_integration_test.go +++ /dev/null @@ -1,526 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package lsmkv - -import ( - "context" - "fmt" - "math/rand" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/cyclemanager" -) - -func compactionSetStrategy(ctx context.Context, t *testing.T, opts []BucketOption, - expectedMinSize, expectedMaxSize int64, -) { - size := 100 - - type kv struct { - key []byte - values [][]byte - delete bool - } - // this segment is not part of the merge, but might still play a role in - // overall results. For example if one of the later segments has a tombstone - // for it - var previous1 []kv - var previous2 []kv - - var segment1 []kv - var segment2 []kv - var expected []kv - var bucket *Bucket - - dirName := t.TempDir() - - t.Run("create test data", func(t *testing.T) { - // The test data is split into 4 scenarios evenly: - // - // 0.) created in the first segment, never touched again - // 1.) created in the first segment, appended to it in the second - // 2.) created in the first segment, first element deleted in the second - // 3.) created in the first segment, second element deleted in the second - // 4.) not present in the first segment, created in the second - // 5.) present in an unrelated previous segment, deleted in the first - // 6.) present in an unrelated previous segment, deleted in the second - // 7.) present in an unrelated previous segment, never touched again - for i := 0; i < size; i++ { - key := []byte(fmt.Sprintf("key-%02d", i)) - - value1 := []byte(fmt.Sprintf("value-%02d-01", i)) - value2 := []byte(fmt.Sprintf("value-%02d-02", i)) - values := [][]byte{value1, value2} - - switch i % 8 { - case 0: - // add to segment 1 - segment1 = append(segment1, kv{ - key: key, - values: values[:1], - }) - - // leave this element untouched in the second segment - expected = append(expected, kv{ - key: key, - values: values[:1], - }) - - case 1: - // add to segment 1 - segment1 = append(segment1, kv{ - key: key, - values: values[:1], - }) - - // update in the second segment - segment2 = append(segment2, kv{ - key: key, - values: values[1:2], - }) - - expected = append(expected, kv{ - key: key, - values: values, - }) - - case 2: - // add both to segment 1, delete the first - segment1 = append(segment1, kv{ - key: key, - values: values, - }) - - // delete first element in the second segment - segment2 = append(segment2, kv{ - key: key, - values: values[:1], - delete: true, - }) - - // only the 2nd element should be left in the expected - expected = append(expected, kv{ - key: key, - values: values[1:2], - }) - - case 3: - // add both to segment 1, delete the second - segment1 = append(segment1, kv{ - key: key, - values: values, - }) - - // delete second element in the second segment - segment2 = append(segment2, kv{ - key: key, - values: values[1:], - delete: true, - }) - - // only the 1st element should be left in the expected - expected = append(expected, kv{ - key: key, - values: values[:1], - }) - - case 4: - // do not add to segment 1 - - // only add to segment 2 (first entry) - segment2 = append(segment2, kv{ - key: key, - values: values, - }) - - expected = append(expected, kv{ - key: key, - values: values, - }) - - case 5: - // only part of a previous segment, which is not part of the merge - previous1 = append(previous1, kv{ - key: key, - values: values[:1], - }) - previous2 = append(previous2, kv{ - key: key, - values: values[1:], - }) - - // delete in segment 1 - segment1 = append(segment1, kv{ - key: key, - values: values[:1], - delete: true, - }) - segment1 = append(segment1, kv{ - key: key, - values: values[1:], - delete: true, - }) - - // should not have any values in expected at all, not even a key - - case 6: - // only part of a previous segment, which is not part of the merge - previous1 = append(previous1, kv{ - key: key, - values: values[:1], - }) - previous2 = append(previous2, kv{ - key: key, - values: values[1:], - }) - - // delete in segment 2 - segment2 = append(segment2, kv{ - key: key, - values: values[:1], - delete: true, - }) - segment2 = append(segment2, kv{ - key: key, - values: values[1:], - delete: true, - }) - - // should not have any values in expected at all, not even a key - - case 7: - // part of a previous segment - previous1 = append(previous1, kv{ - key: key, - values: values[:1], - }) - previous2 = append(previous2, kv{ - key: key, - values: values[1:], - }) - - expected = append(expected, kv{ - key: key, - values: values, - }) - } - } - }) - - t.Run("shuffle the import order for each segment", func(t *testing.T) { - // this is to make sure we don't accidentally rely on the import order - rand.Shuffle(len(segment1), func(i, j int) { - segment1[i], segment1[j] = segment1[j], segment1[i] - }) - rand.Shuffle(len(segment2), func(i, j int) { - segment2[i], segment2[j] = segment2[j], segment2[i] - }) - }) - - t.Run("init bucket", func(t *testing.T) { - b, err := NewBucket(ctx, dirName, dirName, nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - bucket = b - }) - - t.Run("import and flush previous segments", func(t *testing.T) { - for _, pair := range previous1 { - err := bucket.SetAdd(pair.key, pair.values) - require.Nil(t, err) - } - - require.Nil(t, bucket.FlushAndSwitch()) - - for _, pair := range previous2 { - err := bucket.SetAdd(pair.key, pair.values) - require.Nil(t, err) - } - - require.Nil(t, bucket.FlushAndSwitch()) - }) - - t.Run("import segment 1", func(t *testing.T) { - for _, pair := range segment1 { - if !pair.delete { - err := bucket.SetAdd(pair.key, pair.values) - require.Nil(t, err) - } else { - err := bucket.SetDeleteSingle(pair.key, pair.values[0]) - require.Nil(t, err) - } - } - }) - - t.Run("flush to disk", func(t *testing.T) { - require.Nil(t, bucket.FlushAndSwitch()) - }) - - t.Run("import segment 2", func(t *testing.T) { - for _, pair := range segment2 { - if !pair.delete { - err := bucket.SetAdd(pair.key, pair.values) - require.Nil(t, err) - } else { - err := bucket.SetDeleteSingle(pair.key, pair.values[0]) - require.Nil(t, err) - } - } - }) - - t.Run("flush to disk", func(t *testing.T) { - require.Nil(t, bucket.FlushAndSwitch()) - }) - - t.Run("verify control before compaction", func(t *testing.T) { - var retrieved []kv - - c := bucket.SetCursor() - defer c.Close() - - for k, v := c.First(); k != nil; k, v = c.Next() { - retrieved = append(retrieved, kv{ - key: k, - values: v, - }) - } - - assert.Equal(t, expected, retrieved) - }) - - t.Run("compact until no longer eligible", func(t *testing.T) { - i := 0 - var compacted bool - var err error - for compacted, err = bucket.disk.compactOnce(); err == nil && compacted; compacted, err = bucket.disk.compactOnce() { - if i == 1 { - // segment1 and segment2 merged - // none of them is root segment, so tombstones - // will not be removed regardless of keepTombstones setting - assertSecondSegmentOfSize(t, bucket, 8556, 8556) - } - i++ - } - require.Nil(t, err) - }) - - t.Run("verify control after compaction", func(t *testing.T) { - var retrieved []kv - - c := bucket.SetCursor() - defer c.Close() - - for k, v := c.First(); k != nil; k, v = c.Next() { - retrieved = append(retrieved, kv{ - key: k, - values: v, - }) - } - - assert.Equal(t, expected, retrieved) - assertSingleSegmentOfSize(t, bucket, expectedMinSize, expectedMaxSize) - }) -} - -func compactionSetStrategy_RemoveUnnecessary(ctx context.Context, t *testing.T, opts []BucketOption) { - // in this test each segment reverses the action of the previous segment so - // that in the end a lot of information is present in the individual segments - // which is no longer needed. We then verify that after all compaction this - // information is gone, thus freeing up disk space - size := 100 - - type kv struct { - key []byte - values [][]byte - } - - key := []byte("my-key") - - var bucket *Bucket - dirName := t.TempDir() - - t.Run("init bucket", func(t *testing.T) { - b, err := NewBucket(ctx, dirName, dirName, nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - bucket = b - }) - - t.Run("write segments", func(t *testing.T) { - for i := 0; i < size; i++ { - if i != 0 { - // we can only delete an existing value if this isn't the first write - value := []byte(fmt.Sprintf("value-%05d", i-1)) - err := bucket.SetDeleteSingle(key, value) - require.Nil(t, err) - } - - value := []byte(fmt.Sprintf("value-%05d", i)) - err := bucket.SetAdd(key, [][]byte{value}) - require.Nil(t, err) - - require.Nil(t, bucket.FlushAndSwitch()) - } - }) - - t.Run("verify control before compaction", func(t *testing.T) { - var retrieved []kv - expected := []kv{ - { - key: key, - values: [][]byte{[]byte(fmt.Sprintf("value-%05d", size-1))}, - }, - } - - c := bucket.SetCursor() - defer c.Close() - - for k, v := c.First(); k != nil; k, v = c.Next() { - retrieved = append(retrieved, kv{ - key: k, - values: v, - }) - } - - assert.Equal(t, expected, retrieved) - }) - t.Run("compact until no longer eligible", func(t *testing.T) { - var compacted bool - var err error - for compacted, err = bucket.disk.compactOnce(); err == nil && compacted; compacted, err = bucket.disk.compactOnce() { - } - require.Nil(t, err) - }) - - t.Run("verify control before compaction", func(t *testing.T) { - var retrieved []kv - expected := []kv{ - { - key: key, - values: [][]byte{[]byte(fmt.Sprintf("value-%05d", size-1))}, - }, - } - - c := bucket.SetCursor() - defer c.Close() - - for k, v := c.First(); k != nil; k, v = c.Next() { - retrieved = append(retrieved, kv{ - key: k, - values: v, - }) - } - - assert.Equal(t, expected, retrieved) - }) -} - -func compactionSetStrategy_FrequentPutDeleteOperations(ctx context.Context, t *testing.T, opts []BucketOption) { - // In this test we are testing that the compaction works well for set collection - maxSize := 10 - - for size := 4; size < maxSize; size++ { - t.Run(fmt.Sprintf("compact %v segments", size), func(t *testing.T) { - var bucket *Bucket - - key := []byte("key-original") - value1 := []byte("value-01") - value2 := []byte("value-02") - values := [][]byte{value1, value2} - - dirName := t.TempDir() - - t.Run("init bucket", func(t *testing.T) { - b, err := NewBucket(ctx, dirName, dirName, nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - bucket = b - }) - - t.Run("import and flush segments", func(t *testing.T) { - for i := 0; i < size; i++ { - err := bucket.SetAdd(key, values) - require.Nil(t, err) - - if size == 5 { - // delete all - err := bucket.SetDeleteSingle(key, values[0]) - require.Nil(t, err) - err = bucket.SetDeleteSingle(key, values[1]) - require.Nil(t, err) - } else if size == 6 { - // delete only one value - err := bucket.SetDeleteSingle(key, values[0]) - require.Nil(t, err) - } else if i != size-1 { - // don't delete from the last segment - err := bucket.SetDeleteSingle(key, values[0]) - require.Nil(t, err) - err = bucket.SetDeleteSingle(key, values[1]) - require.Nil(t, err) - } - - require.Nil(t, bucket.FlushAndSwitch()) - } - }) - - t.Run("verify that objects exist before compaction", func(t *testing.T) { - res, err := bucket.SetList(key) - assert.Nil(t, err) - if size == 5 { - assert.Len(t, res, 0) - } else if size == 6 { - assert.Len(t, res, 1) - } else { - assert.Len(t, res, 2) - } - }) - - t.Run("compact until no longer eligible", func(t *testing.T) { - var compacted bool - var err error - for compacted, err = bucket.disk.compactOnce(); err == nil && compacted; compacted, err = bucket.disk.compactOnce() { - } - require.Nil(t, err) - }) - - t.Run("verify that objects exist after compaction", func(t *testing.T) { - res, err := bucket.SetList(key) - assert.Nil(t, err) - if size == 5 { - assert.Len(t, res, 0) - } else if size == 6 { - assert.Len(t, res, 1) - } else { - assert.Len(t, res, 2) - } - }) - }) - } -} diff --git a/adapters/repos/db/lsmkv/compactor_map.go b/adapters/repos/db/lsmkv/compactor_map.go deleted file mode 100644 index 611a39940d9c0392b8e78c4420d85f0b3db9ea71..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/compactor_map.go +++ /dev/null @@ -1,298 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "bufio" - "bytes" - "io" - "sort" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv/segmentindex" -) - -type compactorMap struct { - // c1 is always the older segment, so when there is a conflict c2 wins - // (because of the replace strategy) - c1 *segmentCursorCollectionReusable - c2 *segmentCursorCollectionReusable - - // the level matching those of the cursors - currentLevel uint16 - secondaryIndexCount uint16 - // Tells if tombstones or keys without corresponding values - // can be removed from merged segment. - // (left segment is root (1st) one, keepTombstones is off for bucket) - cleanupTombstones bool - - w io.WriteSeeker - bufw *bufio.Writer - - scratchSpacePath string - - // for backward-compatibility with states where the disk state for maps was - // not guaranteed to be sorted yet - requiresSorting bool -} - -func newCompactorMapCollection(w io.WriteSeeker, - c1, c2 *segmentCursorCollectionReusable, level, secondaryIndexCount uint16, - scratchSpacePath string, requiresSorting bool, cleanupTombstones bool, -) *compactorMap { - return &compactorMap{ - c1: c1, - c2: c2, - w: w, - bufw: bufio.NewWriterSize(w, 256*1024), - currentLevel: level, - cleanupTombstones: cleanupTombstones, - secondaryIndexCount: secondaryIndexCount, - scratchSpacePath: scratchSpacePath, - requiresSorting: requiresSorting, - } -} - -func (c *compactorMap) do() error { - if err := c.init(); err != nil { - return errors.Wrap(err, "init") - } - - kis, err := c.writeKeys() - if err != nil { - return errors.Wrap(err, "write keys") - } - - if err := c.writeIndices(kis); err != nil { - return errors.Wrap(err, "write index") - } - - // flush buffered, so we can safely seek on underlying writer - if err := c.bufw.Flush(); err != nil { - return errors.Wrap(err, "flush buffered") - } - - var dataEnd uint64 = segmentindex.HeaderSize - if len(kis) > 0 { - dataEnd = uint64(kis[len(kis)-1].ValueEnd) - } - - if err := c.writeHeader(c.currentLevel, 0, c.secondaryIndexCount, - dataEnd); err != nil { - return errors.Wrap(err, "write header") - } - - return nil -} - -func (c *compactorMap) init() error { - // write a dummy header, we don't know the contents of the actual header yet, - // we will seek to the beginning and overwrite the actual header at the very - // end - - if _, err := c.bufw.Write(make([]byte, segmentindex.HeaderSize)); err != nil { - return errors.Wrap(err, "write empty header") - } - - return nil -} - -func (c *compactorMap) writeKeys() ([]segmentindex.Key, error) { - key1, value1, _ := c.c1.first() - key2, value2, _ := c.c2.first() - - // the (dummy) header was already written, this is our initial offset - offset := segmentindex.HeaderSize - - var kis []segmentindex.Key - pairs := newReusableMapPairs() - me := newMapEncoder() - ssm := newSortedMapMerger() - - for { - if key1 == nil && key2 == nil { - break - } - if bytes.Equal(key1, key2) { - pairs.ResizeLeft(len(value1)) - pairs.ResizeRight(len(value2)) - - for i, v := range value1 { - if err := pairs.left[i].FromBytes(v.value, false); err != nil { - return nil, err - } - pairs.left[i].Tombstone = v.tombstone - } - - for i, v := range value2 { - if err := pairs.right[i].FromBytes(v.value, false); err != nil { - return nil, err - } - pairs.right[i].Tombstone = v.tombstone - } - - if c.requiresSorting { - sort.Slice(pairs.left, func(a, b int) bool { - return bytes.Compare(pairs.left[a].Key, pairs.left[b].Key) < 0 - }) - sort.Slice(pairs.right, func(a, b int) bool { - return bytes.Compare(pairs.right[a].Key, pairs.right[b].Key) < 0 - }) - } - - ssm.reset([][]MapPair{pairs.left, pairs.right}) - mergedPairs, err := ssm. - doKeepTombstonesReusable() - if err != nil { - return nil, err - } - - mergedEncoded, err := me.DoMultiReusable(mergedPairs) - if err != nil { - return nil, err - } - - if values, skip := c.cleanupValues(mergedEncoded); !skip { - ki, err := c.writeIndividualNode(offset, key2, values) - if err != nil { - return nil, errors.Wrap(err, "write individual node (equal keys)") - } - - offset = ki.ValueEnd - kis = append(kis, ki) - } - // advance both! - key1, value1, _ = c.c1.next() - key2, value2, _ = c.c2.next() - continue - } - - if (key1 != nil && bytes.Compare(key1, key2) == -1) || key2 == nil { - // key 1 is smaller - if values, skip := c.cleanupValues(value1); !skip { - ki, err := c.writeIndividualNode(offset, key1, values) - if err != nil { - return nil, errors.Wrap(err, "write individual node (key1 smaller)") - } - - offset = ki.ValueEnd - kis = append(kis, ki) - } - key1, value1, _ = c.c1.next() - } else { - // key 2 is smaller - if values, skip := c.cleanupValues(value2); !skip { - ki, err := c.writeIndividualNode(offset, key2, values) - if err != nil { - return nil, errors.Wrap(err, "write individual node (key2 smaller)") - } - - offset = ki.ValueEnd - kis = append(kis, ki) - } - key2, value2, _ = c.c2.next() - } - } - - return kis, nil -} - -func (c *compactorMap) writeIndividualNode(offset int, key []byte, - values []value, -) (segmentindex.Key, error) { - // NOTE: There are no guarantees in the cursor logic that any memory is valid - // for more than a single iteration. Every time you call next() to advance - // the cursor, any memory might be reused. - // - // This includes the key buffer which was the cause of - // https://github.com/weaviate/weaviate/issues/3517 - // - // A previous logic created a new assignment in each iteration, but thatwas - // not an explicit guarantee. A change in v1.21 (for pread/mmap) added a - // reusable buffer for the key which surfaced this bug. - keyCopy := make([]byte, len(key)) - copy(keyCopy, key) - - return segmentCollectionNode{ - values: values, - primaryKey: keyCopy, - offset: offset, - }.KeyIndexAndWriteTo(c.bufw) -} - -func (c *compactorMap) writeIndices(keys []segmentindex.Key) error { - indices := segmentindex.Indexes{ - Keys: keys, - SecondaryIndexCount: c.secondaryIndexCount, - ScratchSpacePath: c.scratchSpacePath, - } - - _, err := indices.WriteTo(c.bufw) - return err -} - -// writeHeader assumes that everything has been written to the underlying -// writer and it is now safe to seek to the beginning and override the initial -// header -func (c *compactorMap) writeHeader(level, version, secondaryIndices uint16, - startOfIndex uint64, -) error { - if _, err := c.w.Seek(0, io.SeekStart); err != nil { - return errors.Wrap(err, "seek to beginning to write header") - } - - h := &segmentindex.Header{ - Level: level, - Version: version, - SecondaryIndices: secondaryIndices, - Strategy: segmentindex.StrategyMapCollection, - IndexStart: startOfIndex, - } - - if _, err := h.WriteTo(c.w); err != nil { - return err - } - - return nil -} - -// Removes values with tombstone set from input slice. Output slice may be smaller than input one. -// Returned skip of true means there are no values left (key can be omitted in segment) -// WARN: method can alter input slice by swapping its elements and reducing length (not capacity) -func (c *compactorMap) cleanupValues(values []value) (vals []value, skip bool) { - if !c.cleanupTombstones { - return values, false - } - - // Reuse input slice not to allocate new memory - // Rearrange slice in a way that tombstoned values are moved to the end - // and reduce slice's length. - last := 0 - for i := 0; i < len(values); i++ { - if !values[i].tombstone { - // Swap both elements instead overwritting `last` by `i`. - // Overwrite would result in `values[last].value` pointing to the same slice - // as `values[i].value`. - // If `values` slice is reused by multiple nodes (as it happens for map cursors - // `segmentCursorCollectionReusable` using `segmentCollectionNode` as buffer) - // populating slice `values[i].value` would overwrite slice `values[last].value`. - // Swaps makes sure `values[i].value` and `values[last].value` point to different slices. - values[last], values[i] = values[i], values[last] - last++ - } - } - - if last == 0 { - return nil, true - } - return values[:last], false -} diff --git a/adapters/repos/db/lsmkv/compactor_map_reusable_pairs.go b/adapters/repos/db/lsmkv/compactor_map_reusable_pairs.go deleted file mode 100644 index e604225c46fe6d479556b0062be40746f9b75105..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/compactor_map_reusable_pairs.go +++ /dev/null @@ -1,53 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -// reusableMapPairs is not thread-safe and intended for usage from a single -// thread. The caller is resoponsible for initializing each element themselves, -// the Resize functions will only set the size. If the size is reduced, this -// will only truncate elements, but will not reset values. -type reusableMapPairs struct { - left []MapPair - right []MapPair -} - -func newReusableMapPairs() *reusableMapPairs { - return &reusableMapPairs{} -} - -func (rmp *reusableMapPairs) ResizeLeft(size int) { - if cap(rmp.left) >= size { - rmp.left = rmp.left[:size] - } else { - // The 25% overhead for the capacity was chosen because we saw a lot - // re-allocations during testing with just a few elements more than before. - // This is something that really depends on the user's usage pattern, but - // in the test scenarios based on the - // weaviate-chaos-engineering/apps/importer-no-vector-index test script a - // simple 25% overhead reduced the resizing needs to almost zero. - rmp.left = make([]MapPair, size, int(float64(size)*1.25)) - } -} - -func (rmp *reusableMapPairs) ResizeRight(size int) { - if cap(rmp.right) >= size { - rmp.right = rmp.right[:size] - } else { - // The 25% overhead for the capacity was chosen because we saw a lot - // re-allocations during testing with just a few elements more than before. - // This is something that really depends on the user's usage pattern, but - // in the test scenarios based on the - // weaviate-chaos-engineering/apps/importer-no-vector-index test script a - // simple 25% overhead reduced the resizing needs to almost zero. - rmp.right = make([]MapPair, size, int(float64(size)*1.25)) - } -} diff --git a/adapters/repos/db/lsmkv/compactor_replace.go b/adapters/repos/db/lsmkv/compactor_replace.go deleted file mode 100644 index 1a9a58850a40ec3fe3d5599c2e211d5ee7b78b1e..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/compactor_replace.go +++ /dev/null @@ -1,214 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "bufio" - "bytes" - "errors" - "fmt" - "io" - - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv/segmentindex" - "github.com/weaviate/weaviate/entities/lsmkv" -) - -type compactorReplace struct { - // c1 is always the older segment, so when there is a conflict c2 wins - // (because of the replace strategy) - c1 *segmentCursorReplace - c2 *segmentCursorReplace - - // the level matching those of the cursors - currentLevel uint16 - // Tells if tombstones or keys without corresponding values - // can be removed from merged segment. - // (left segment is root (1st) one, keepTombstones is off for bucket) - cleanupTombstones bool - secondaryIndexCount uint16 - - w io.WriteSeeker - bufw *bufio.Writer - scratchSpacePath string -} - -func newCompactorReplace(w io.WriteSeeker, - c1, c2 *segmentCursorReplace, level, secondaryIndexCount uint16, - scratchSpacePath string, cleanupTombstones bool, -) *compactorReplace { - return &compactorReplace{ - c1: c1, - c2: c2, - w: w, - bufw: bufio.NewWriterSize(w, 256*1024), - currentLevel: level, - cleanupTombstones: cleanupTombstones, - secondaryIndexCount: secondaryIndexCount, - scratchSpacePath: scratchSpacePath, - } -} - -func (c *compactorReplace) do() error { - if err := c.init(); err != nil { - return fmt.Errorf("init: %w", err) - } - - kis, err := c.writeKeys() - if err != nil { - return fmt.Errorf("write keys: %w", err) - } - - if err := c.writeIndices(kis); err != nil { - return fmt.Errorf("write indices: %w", err) - } - - // flush buffered, so we can safely seek on underlying writer - if err := c.bufw.Flush(); err != nil { - return fmt.Errorf("flush buffered: %w", err) - } - - var dataEnd uint64 = segmentindex.HeaderSize - if len(kis) > 0 { - dataEnd = uint64(kis[len(kis)-1].ValueEnd) - } - - if err := c.writeHeader(c.currentLevel, 0, c.secondaryIndexCount, dataEnd); err != nil { - return fmt.Errorf("write header: %w", err) - } - - return nil -} - -func (c *compactorReplace) init() error { - // write a dummy header, we don't know the contents of the actual header yet, - // we will seek to the beginning and overwrite the actual header at the very - // end - - if _, err := c.bufw.Write(make([]byte, segmentindex.HeaderSize)); err != nil { - return fmt.Errorf("write empty header: %w", err) - } - - return nil -} - -func (c *compactorReplace) writeKeys() ([]segmentindex.Key, error) { - res1, err1 := c.c1.firstWithAllKeys() - res2, err2 := c.c2.firstWithAllKeys() - - // the (dummy) header was already written, this is our initial offset - offset := segmentindex.HeaderSize - - var kis []segmentindex.Key - - for { - if res1.primaryKey == nil && res2.primaryKey == nil { - break - } - if bytes.Equal(res1.primaryKey, res2.primaryKey) { - if !(c.cleanupTombstones && errors.Is(err2, lsmkv.Deleted)) { - ki, err := c.writeIndividualNode(offset, res2.primaryKey, res2.value, - res2.secondaryKeys, errors.Is(err2, lsmkv.Deleted)) - if err != nil { - return nil, fmt.Errorf("write individual node (equal keys): %w", err) - } - - offset = ki.ValueEnd - kis = append(kis, ki) - } - // advance both! - res1, err1 = c.c1.nextWithAllKeys() - res2, err2 = c.c2.nextWithAllKeys() - continue - } - - if (res1.primaryKey != nil && bytes.Compare(res1.primaryKey, res2.primaryKey) == -1) || res2.primaryKey == nil { - // key 1 is smaller - if !(c.cleanupTombstones && errors.Is(err1, lsmkv.Deleted)) { - ki, err := c.writeIndividualNode(offset, res1.primaryKey, res1.value, - res1.secondaryKeys, errors.Is(err1, lsmkv.Deleted)) - if err != nil { - return nil, fmt.Errorf("write individual node (res1.primaryKey smaller)") - } - - offset = ki.ValueEnd - kis = append(kis, ki) - } - res1, err1 = c.c1.nextWithAllKeys() - } else { - // key 2 is smaller - if !(c.cleanupTombstones && errors.Is(err2, lsmkv.Deleted)) { - ki, err := c.writeIndividualNode(offset, res2.primaryKey, res2.value, - res2.secondaryKeys, errors.Is(err2, lsmkv.Deleted)) - if err != nil { - return nil, fmt.Errorf("write individual node (res2.primaryKey smaller): %w", err) - } - - offset = ki.ValueEnd - kis = append(kis, ki) - } - res2, err2 = c.c2.nextWithAllKeys() - } - } - - return kis, nil -} - -func (c *compactorReplace) writeIndividualNode(offset int, key, value []byte, - secondaryKeys [][]byte, tombstone bool, -) (segmentindex.Key, error) { - segNode := segmentReplaceNode{ - offset: offset, - tombstone: tombstone, - value: value, - primaryKey: key, - secondaryIndexCount: c.secondaryIndexCount, - secondaryKeys: secondaryKeys, - } - - return segNode.KeyIndexAndWriteTo(c.bufw) -} - -func (c *compactorReplace) writeIndices(keys []segmentindex.Key) error { - indices := &segmentindex.Indexes{ - Keys: keys, - SecondaryIndexCount: c.secondaryIndexCount, - ScratchSpacePath: c.scratchSpacePath, - } - - _, err := indices.WriteTo(c.bufw) - return err -} - -// writeHeader assumes that everything has been written to the underlying -// writer and it is now safe to seek to the beginning and override the initial -// header -func (c *compactorReplace) writeHeader(level, version, secondaryIndices uint16, - startOfIndex uint64, -) error { - if _, err := c.w.Seek(0, io.SeekStart); err != nil { - return fmt.Errorf("seek to beginning to write header: %w", err) - } - - h := &segmentindex.Header{ - Level: level, - Version: version, - SecondaryIndices: secondaryIndices, - Strategy: segmentindex.StrategyReplace, - IndexStart: startOfIndex, - } - - if _, err := h.WriteTo(c.w); err != nil { - return err - } - - return nil -} diff --git a/adapters/repos/db/lsmkv/compactor_set.go b/adapters/repos/db/lsmkv/compactor_set.go deleted file mode 100644 index 2e3d870b20f54354fb8b20feefbae6dc4936dd7b..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/compactor_set.go +++ /dev/null @@ -1,240 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "bufio" - "bytes" - "io" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv/segmentindex" -) - -type compactorSet struct { - // c1 is always the older segment, so when there is a conflict c2 wins - // (because of the replace strategy) - c1 *segmentCursorCollection - c2 *segmentCursorCollection - - // the level matching those of the cursors - currentLevel uint16 - secondaryIndexCount uint16 - // Tells if tombstones or keys without corresponding values - // can be removed from merged segment. - // (left segment is root (1st) one, keepTombstones is off for bucket) - cleanupTombstones bool - - w io.WriteSeeker - bufw *bufio.Writer - - scratchSpacePath string -} - -func newCompactorSetCollection(w io.WriteSeeker, - c1, c2 *segmentCursorCollection, level, secondaryIndexCount uint16, - scratchSpacePath string, cleanupTombstones bool, -) *compactorSet { - return &compactorSet{ - c1: c1, - c2: c2, - w: w, - bufw: bufio.NewWriterSize(w, 256*1024), - currentLevel: level, - cleanupTombstones: cleanupTombstones, - secondaryIndexCount: secondaryIndexCount, - scratchSpacePath: scratchSpacePath, - } -} - -func (c *compactorSet) do() error { - if err := c.init(); err != nil { - return errors.Wrap(err, "init") - } - - kis, err := c.writeKeys() - if err != nil { - return errors.Wrap(err, "write keys") - } - - if err := c.writeIndices(kis); err != nil { - return errors.Wrap(err, "write index") - } - - // flush buffered, so we can safely seek on underlying writer - if err := c.bufw.Flush(); err != nil { - return errors.Wrap(err, "flush buffered") - } - - var dataEnd uint64 = segmentindex.HeaderSize - if len(kis) > 0 { - dataEnd = uint64(kis[len(kis)-1].ValueEnd) - } - - if err := c.writeHeader(c.currentLevel, 0, c.secondaryIndexCount, - dataEnd); err != nil { - return errors.Wrap(err, "write header") - } - - return nil -} - -func (c *compactorSet) init() error { - // write a dummy header, we don't know the contents of the actual header yet, - // we will seek to the beginning and overwrite the actual header at the very - // end - - if _, err := c.bufw.Write(make([]byte, segmentindex.HeaderSize)); err != nil { - return errors.Wrap(err, "write empty header") - } - - return nil -} - -func (c *compactorSet) writeKeys() ([]segmentindex.Key, error) { - key1, value1, _ := c.c1.first() - key2, value2, _ := c.c2.first() - - // the (dummy) header was already written, this is our initial offset - offset := segmentindex.HeaderSize - - var kis []segmentindex.Key - - for { - if key1 == nil && key2 == nil { - break - } - if bytes.Equal(key1, key2) { - values := append(value1, value2...) - valuesMerged := newSetDecoder().DoPartial(values) - if values, skip := c.cleanupValues(valuesMerged); !skip { - ki, err := c.writeIndividualNode(offset, key2, values) - if err != nil { - return nil, errors.Wrap(err, "write individual node (equal keys)") - } - - offset = ki.ValueEnd - kis = append(kis, ki) - } - // advance both! - key1, value1, _ = c.c1.next() - key2, value2, _ = c.c2.next() - continue - } - - if (key1 != nil && bytes.Compare(key1, key2) == -1) || key2 == nil { - // key 1 is smaller - if values, skip := c.cleanupValues(value1); !skip { - ki, err := c.writeIndividualNode(offset, key1, values) - if err != nil { - return nil, errors.Wrap(err, "write individual node (key1 smaller)") - } - - offset = ki.ValueEnd - kis = append(kis, ki) - } - key1, value1, _ = c.c1.next() - } else { - // key 2 is smaller - if values, skip := c.cleanupValues(value2); !skip { - ki, err := c.writeIndividualNode(offset, key2, values) - if err != nil { - return nil, errors.Wrap(err, "write individual node (key2 smaller)") - } - - offset = ki.ValueEnd - kis = append(kis, ki) - } - key2, value2, _ = c.c2.next() - } - } - - return kis, nil -} - -func (c *compactorSet) writeIndividualNode(offset int, key []byte, - values []value, -) (segmentindex.Key, error) { - return (&segmentCollectionNode{ - values: values, - primaryKey: key, - offset: offset, - }).KeyIndexAndWriteTo(c.bufw) -} - -func (c *compactorSet) writeIndices(keys []segmentindex.Key) error { - indices := &segmentindex.Indexes{ - Keys: keys, - SecondaryIndexCount: c.secondaryIndexCount, - ScratchSpacePath: c.scratchSpacePath, - } - - _, err := indices.WriteTo(c.bufw) - return err -} - -// writeHeader assumes that everything has been written to the underlying -// writer and it is now safe to seek to the beginning and override the initial -// header -func (c *compactorSet) writeHeader(level, version, secondaryIndices uint16, - startOfIndex uint64, -) error { - if _, err := c.w.Seek(0, io.SeekStart); err != nil { - return errors.Wrap(err, "seek to beginning to write header") - } - - h := &segmentindex.Header{ - Level: level, - Version: version, - SecondaryIndices: secondaryIndices, - Strategy: segmentindex.StrategySetCollection, - IndexStart: startOfIndex, - } - - if _, err := h.WriteTo(c.w); err != nil { - return err - } - - return nil -} - -// Removes values with tombstone set from input slice. Output slice may be smaller than input one. -// Returned skip of true means there are no values left (key can be omitted in segment) -// WARN: method can alter input slice by swapping its elements and reducing length (not capacity) -func (c *compactorSet) cleanupValues(values []value) (vals []value, skip bool) { - if !c.cleanupTombstones { - return values, false - } - - // Reuse input slice not to allocate new memory - // Rearrange slice in a way that tombstoned values are moved to the end - // and reduce slice's length. - last := 0 - for i := 0; i < len(values); i++ { - if !values[i].tombstone { - // Swap both elements instead overwritting `last` by `i`. - // Overwrite would result in `values[last].value` pointing to the same slice - // as `values[i].value`. - // If `values` slice is reused by multiple nodes (as it happens for map cursors - // `segmentCursorCollectionReusable` using `segmentCollectionNode` as buffer) - // populating values[i].value would overwrite values[last].value - // Swaps makes sure values[i].value and values[last].value point to different slices - values[last], values[i] = values[i], values[last] - last++ - } - } - - if last == 0 { - return nil, true - } - return values[:last], false -} diff --git a/adapters/repos/db/lsmkv/concurrent_reading_benchmark_test.go b/adapters/repos/db/lsmkv/concurrent_reading_benchmark_test.go deleted file mode 100644 index cbe6995a2ff678b243cf713fba1fe1520a459eb7..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/concurrent_reading_benchmark_test.go +++ /dev/null @@ -1,126 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "context" - "crypto/rand" - "fmt" - "os" - "sync" - "testing" - "time" - - "github.com/google/uuid" - "github.com/sirupsen/logrus" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/cyclemanager" -) - -func BenchmarkConcurrentReading(b *testing.B) { - bucket, cleanup := prepareBucket(b) - defer cleanup() - keys := populateBucket(b, bucket) - - b.ReportAllocs() - b.ResetTimer() - - routines := 500 - - for i := 0; i < b.N; i++ { - wg := sync.WaitGroup{} - for r := 0; r < routines; r++ { - wg.Add(1) - go func() { - defer wg.Done() - for _, key := range keys { - _, err := bucket.MapList(key) - assert.Nil(b, err) - } - }() - } - wg.Wait() - } -} - -func prepareBucket(b *testing.B) (bucket *Bucket, cleanup func()) { - dirName := fmt.Sprintf("./testdata/%d", mustRandIntn(10000000)) - os.MkdirAll(dirName, 0o777) - defer func() { - err := os.RemoveAll(dirName) - fmt.Println(err) - }() - - bucket, err := NewBucket(testCtxB(), dirName, "", nullLoggerB(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - WithStrategy(StrategyMapCollection), - WithMemtableThreshold(5000)) - require.Nil(b, err) - - return bucket, func() { - err := os.RemoveAll(dirName) - fmt.Println(err) - } -} - -func populateBucket(b *testing.B, bucket *Bucket) (keys [][]byte) { - amount := 2000 - valuesPerKey := 4 - sizePerKey := 8 - sizePerValue := 32 - - keys = make([][]byte, amount) - values := make([][]MapPair, amount) - - for i := range keys { - uuid, err := uuid.New().MarshalBinary() - require.Nil(b, err) - keys[i] = uuid - - values[i] = make([]MapPair, valuesPerKey) - for j := range values[i] { - values[i][j] = MapPair{ - Key: make([]byte, sizePerKey), - Value: make([]byte, sizePerValue), - } - rand.Read(values[i][j].Key) - rand.Read(values[i][j].Value) - } - } - - wg := sync.WaitGroup{} - for i := range keys { - for j := 0; j < valuesPerKey; j++ { - time.Sleep(50 * time.Microsecond) - wg.Add(1) - go func(rowIndex, valueIndex int) { - defer wg.Done() - err := bucket.MapSet(keys[rowIndex], values[rowIndex][valueIndex]) - assert.Nil(b, err) - }(i, j) - } - } - wg.Wait() - - return -} - -func testCtxB() context.Context { - return context.Background() -} - -func nullLoggerB() logrus.FieldLogger { - log, _ := test.NewNullLogger() - return log -} diff --git a/adapters/repos/db/lsmkv/concurrent_writing_integration_test.go b/adapters/repos/db/lsmkv/concurrent_writing_integration_test.go deleted file mode 100644 index 3eb46842a1253a9eae9eeb209e9ddb2d2c9acd96..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/concurrent_writing_integration_test.go +++ /dev/null @@ -1,328 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package lsmkv - -import ( - "bytes" - "context" - "crypto/rand" - "fmt" - "reflect" - "sync" - "testing" - "time" - - "github.com/google/uuid" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/cyclemanager" -) - -// This test continuously writes into a bucket with a small memtable threshold, -// so that a lot of flushing is happening while writing. This is to ensure that -// there will be no lost writes or other inconsistencies under load -func TestConcurrentWriting_Replace(t *testing.T) { - dirName := t.TempDir() - - amount := 2000 - sizePerValue := 128 - - keys := make([][]byte, amount) - values := make([][]byte, amount) - - bucket, err := NewBucket(testCtx(), dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - WithStrategy(StrategyReplace), - WithMemtableThreshold(10000)) - require.Nil(t, err) - - t.Run("generate random data", func(t *testing.T) { - for i := range keys { - uuid, err := uuid.New().MarshalBinary() - require.Nil(t, err) - keys[i] = uuid - - values[i] = make([]byte, sizePerValue) - rand.Read(values[i]) - } - }) - - t.Run("import", func(t *testing.T) { - wg := sync.WaitGroup{} - - for i := range keys { - time.Sleep(50 * time.Microsecond) - wg.Add(1) - go func(index int) { - defer wg.Done() - err := bucket.Put(keys[index], values[index]) - assert.Nil(t, err) - }(i) - } - wg.Wait() - }) - - t.Run("verify get", func(t *testing.T) { - correct := 0 - var missingKeys []int - - for i := range keys { - value, err := bucket.Get(keys[i]) - assert.Nil(t, err) - if bytes.Equal(values[i], value) { - correct++ - } else { - missingKeys = append(missingKeys, i) - } - } - - if len(missingKeys) > 0 { - fmt.Printf("missing keys: %v\n", missingKeys) - } - assert.Equal(t, amount, correct) - }) - - t.Run("verify cursor", func(t *testing.T) { - correct := 0 - // put all key value/pairs in a map so we can access them by key - targets := map[string][]byte{} - - for i := range keys { - targets[string(keys[i])] = values[i] - } - - c := bucket.Cursor() - defer c.Close() - for k, v := c.First(); k != nil; k, v = c.Next() { - control := targets[string(k)] - if bytes.Equal(control, v) { - correct++ - } - } - - assert.Equal(t, amount, correct) - }) - - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) - defer cancel() - - require.Nil(t, bucket.Shutdown(ctx)) -} - -// This test continuously writes into a bucket with a small memtable threshold, -// so that a lot of flushing is happening while writing. This is to ensure that -// there will be no lost writes or other inconsistencies under load -func TestConcurrentWriting_Set(t *testing.T) { - dirName := t.TempDir() - - amount := 2000 - valuesPerKey := 4 - sizePerValue := 32 - - keys := make([][]byte, amount) - values := make([][][]byte, amount) - - bucket, err := NewBucket(testCtx(), dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - WithStrategy(StrategySetCollection), - WithMemtableThreshold(10000)) - require.Nil(t, err) - - t.Run("generate random data", func(t *testing.T) { - for i := range keys { - uuid, err := uuid.New().MarshalBinary() - require.Nil(t, err) - keys[i] = uuid - - values[i] = make([][]byte, valuesPerKey) - for j := range values[i] { - values[i][j] = make([]byte, sizePerValue) - rand.Read(values[i][j]) - } - } - }) - - t.Run("import", func(t *testing.T) { - wg := sync.WaitGroup{} - - for i := range keys { - time.Sleep(50 * time.Microsecond) - wg.Add(1) - go func(index int) { - defer wg.Done() - err := bucket.SetAdd(keys[index], values[index]) - assert.Nil(t, err) - }(i) - } - wg.Wait() - }) - - t.Run("verify get", func(t *testing.T) { - correct := 0 - - for i := range keys { - value, err := bucket.SetList(keys[i]) - assert.Nil(t, err) - if reflect.DeepEqual(values[i], value) { - correct++ - } - } - - assert.Equal(t, amount, correct) - }) - - t.Run("verify cursor", func(t *testing.T) { - correct := 0 - // put all key value/pairs in a map so we can access them by key - targets := map[string][][]byte{} - - for i := range keys { - targets[string(keys[i])] = values[i] - } - - c := bucket.SetCursor() - defer c.Close() - for k, v := c.First(); k != nil; k, v = c.Next() { - control := targets[string(k)] - if reflect.DeepEqual(control, v) { - correct++ - } - } - - assert.Equal(t, amount, correct) - }) - - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) - defer cancel() - - require.Nil(t, bucket.Shutdown(ctx)) -} - -// This test continuously writes into a bucket with a small memtable threshold, -// so that a lot of flushing is happening while writing. This is to ensure that -// there will be no lost writes or other inconsistencies under load -func TestConcurrentWriting_Map(t *testing.T) { - dirName := t.TempDir() - - amount := 2000 - valuesPerKey := 4 - sizePerKey := 8 - sizePerValue := 32 - - keys := make([][]byte, amount) - values := make([][]MapPair, amount) - - bucket, err := NewBucket(testCtx(), dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - WithStrategy(StrategyMapCollection), - WithMemtableThreshold(5000)) - require.Nil(t, err) - - t.Run("generate random data", func(t *testing.T) { - for i := range keys { - uuid, err := uuid.New().MarshalBinary() - require.Nil(t, err) - keys[i] = uuid - - values[i] = make([]MapPair, valuesPerKey) - for j := range values[i] { - values[i][j] = MapPair{ - Key: make([]byte, sizePerKey), - Value: make([]byte, sizePerValue), - } - rand.Read(values[i][j].Key) - rand.Read(values[i][j].Value) - } - } - }) - - t.Run("import", func(t *testing.T) { - wg := sync.WaitGroup{} - - for i := range keys { - for j := 0; j < valuesPerKey; j++ { - time.Sleep(50 * time.Microsecond) - wg.Add(1) - go func(rowIndex, valueIndex int) { - defer wg.Done() - err := bucket.MapSet(keys[rowIndex], values[rowIndex][valueIndex]) - assert.Nil(t, err) - }(i, j) - } - } - wg.Wait() - }) - - t.Run("verify cursor", func(t *testing.T) { - correct := 0 - // put all key value/pairs in a map so we can access them by key - targets := map[string][]MapPair{} - - for i := range keys { - targets[string(keys[i])] = values[i] - } - - c := bucket.MapCursor() - defer c.Close() - for k, v := c.First(); k != nil; k, v = c.Next() { - control := targets[string(k)] - if mapElementsMatch(control, v) { - correct++ - } - } - - assert.Equal(t, amount, correct) - }) - - t.Run("verify get", func(t *testing.T) { - correct := 0 - - for i := range keys { - value, err := bucket.MapList(keys[i]) - assert.Nil(t, err) - if mapElementsMatch(values[i], value) { - correct++ - } - } - - assert.Equal(t, amount, correct) - }) - - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) - defer cancel() - - require.Nil(t, bucket.Shutdown(ctx)) -} - -func mapElementsMatch(a, b []MapPair) bool { - if len(a) != len(b) { - return false - } - - aMap := map[string][]byte{} - - for _, kv := range a { - aMap[string(kv.Key)] = kv.Value - } - - for _, kv := range b { - control := aMap[string(kv.Key)] - if !bytes.Equal(kv.Value, control) { - return false - } - } - - return true -} diff --git a/adapters/repos/db/lsmkv/cursor_bucket_map.go b/adapters/repos/db/lsmkv/cursor_bucket_map.go deleted file mode 100644 index 895ff20a070fb253b9b3f2b5b2a0a93e2fee9c67..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/cursor_bucket_map.go +++ /dev/null @@ -1,282 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "bytes" - "errors" - "fmt" - "sort" - - "github.com/weaviate/weaviate/entities/lsmkv" -) - -type CursorMap struct { - innerCursors []innerCursorMap - state []cursorStateMap - unlock func() - listCfg MapListOptionConfig - keyOnly bool -} - -type cursorStateMap struct { - key []byte - value []MapPair - err error -} - -type innerCursorMap interface { - first() ([]byte, []MapPair, error) - next() ([]byte, []MapPair, error) - seek([]byte) ([]byte, []MapPair, error) -} - -func (b *Bucket) MapCursor(cfgs ...MapListOption) *CursorMap { - b.flushLock.RLock() - - c := MapListOptionConfig{} - for _, cfg := range cfgs { - cfg(&c) - } - - innerCursors, unlockSegmentGroup := b.disk.newMapCursors() - - // we have a flush-RLock, so we have the guarantee that the flushing state - // will not change for the lifetime of the cursor, thus there can only be two - // states: either a flushing memtable currently exists - or it doesn't - if b.flushing != nil { - innerCursors = append(innerCursors, b.flushing.newMapCursor()) - } - - innerCursors = append(innerCursors, b.active.newMapCursor()) - - return &CursorMap{ - unlock: func() { - unlockSegmentGroup() - b.flushLock.RUnlock() - }, - // cursor are in order from oldest to newest, with the memtable cursor - // being at the very top - innerCursors: innerCursors, - listCfg: c, - } -} - -func (b *Bucket) MapCursorKeyOnly(cfgs ...MapListOption) *CursorMap { - c := b.MapCursor(cfgs...) - c.keyOnly = true - return c -} - -func (c *CursorMap) Seek(key []byte) ([]byte, []MapPair) { - c.seekAll(key) - return c.serveCurrentStateAndAdvance() -} - -func (c *CursorMap) Next() ([]byte, []MapPair) { - // before := time.Now() - // defer func() { - // fmt.Printf("-- total next took %s\n", time.Since(before)) - // }() - return c.serveCurrentStateAndAdvance() -} - -func (c *CursorMap) First() ([]byte, []MapPair) { - c.firstAll() - return c.serveCurrentStateAndAdvance() -} - -func (c *CursorMap) Close() { - c.unlock() -} - -func (c *CursorMap) seekAll(target []byte) { - state := make([]cursorStateMap, len(c.innerCursors)) - for i, cur := range c.innerCursors { - key, value, err := cur.seek(target) - if errors.Is(err, lsmkv.NotFound) { - state[i].err = err - continue - } - - if err != nil { - panic(fmt.Errorf("unexpected error in seek: %w", err)) - } - - state[i].key = key - if !c.keyOnly { - state[i].value = value - } - } - - c.state = state -} - -func (c *CursorMap) firstAll() { - state := make([]cursorStateMap, len(c.innerCursors)) - for i, cur := range c.innerCursors { - key, value, err := cur.first() - if errors.Is(err, lsmkv.NotFound) { - state[i].err = err - continue - } - - if err != nil { - panic(fmt.Errorf("unexpected error in seek: %w", err)) - } - - state[i].key = key - if !c.keyOnly { - state[i].value = value - } - } - - c.state = state -} - -func (c *CursorMap) serveCurrentStateAndAdvance() ([]byte, []MapPair) { - id, err := c.cursorWithLowestKey() - if err != nil { - if errors.Is(err, lsmkv.NotFound) { - return nil, nil - } - } - - // check if this is a duplicate key before checking for the remaining errors, - // as cases such as 'entities.Deleted' can be better handled inside - // mergeDuplicatesInCurrentStateAndAdvance where we can be sure to act on - // segments in the correct order - if ids, ok := c.haveDuplicatesInState(id); ok { - return c.mergeDuplicatesInCurrentStateAndAdvance(ids) - } else { - return c.mergeDuplicatesInCurrentStateAndAdvance([]int{id}) - } -} - -func (c *CursorMap) cursorWithLowestKey() (int, error) { - err := lsmkv.NotFound - pos := -1 - var lowest []byte - - for i, res := range c.state { - if errors.Is(res.err, lsmkv.NotFound) { - continue - } - - if lowest == nil || bytes.Compare(res.key, lowest) <= 0 { - pos = i - err = res.err - lowest = res.key - } - } - - if err != nil { - return pos, err - } - - return pos, nil -} - -func (c *CursorMap) haveDuplicatesInState(idWithLowestKey int) ([]int, bool) { - key := c.state[idWithLowestKey].key - - var idsFound []int - - for i, cur := range c.state { - if i == idWithLowestKey { - idsFound = append(idsFound, i) - continue - } - - if bytes.Equal(key, cur.key) { - idsFound = append(idsFound, i) - } - } - - return idsFound, len(idsFound) > 1 -} - -// if there are no duplicates present it will still work as returning the -// latest result is the same as returning the only result -func (c *CursorMap) mergeDuplicatesInCurrentStateAndAdvance(ids []int) ([]byte, []MapPair) { - // take the key from any of the results, we have the guarantee that they're - // all the same - key := c.state[ids[0]].key - - // appending := time.Duration(0) - // advancing := time.Duration(0) - - var perSegmentResults [][]MapPair - - for _, id := range ids { - candidates := c.state[id].value - perSegmentResults = append(perSegmentResults, candidates) - - // before = time.Now() - c.advanceInner(id) - // advancing += time.Since(before) - } - // fmt.Printf("--- extract values [appending] took %s\n", appending) - // fmt.Printf("--- extract values [advancing] took %s\n", advancing) - - if c.listCfg.legacyRequireManualSorting { - for i := range perSegmentResults { - sort.Slice(perSegmentResults[i], func(a, b int) bool { - return bytes.Compare(perSegmentResults[i][a].Key, - perSegmentResults[i][b].Key) == -1 - }) - } - } - - merged, err := newSortedMapMerger().do(perSegmentResults) - if err != nil { - panic(fmt.Errorf("unexpected error decoding map values: %w", err)) - } - if len(merged) == 0 { - // all values deleted, skip key - return c.Next() - } - - // TODO remove keyOnly option, not used anyway - if !c.keyOnly { - return key, merged - } else { - return key, nil - } -} - -func (c *CursorMap) advanceInner(id int) { - k, v, err := c.innerCursors[id].next() - if errors.Is(err, lsmkv.NotFound) { - c.state[id].err = err - c.state[id].key = nil - c.state[id].value = nil - return - } - - if errors.Is(err, lsmkv.Deleted) { - c.state[id].err = err - c.state[id].key = k - c.state[id].value = nil - return - } - - if err != nil { - panic(fmt.Errorf("unexpected error in advance: %w", err)) - } - - c.state[id].key = k - if !c.keyOnly { - c.state[id].value = v - } - c.state[id].err = nil -} diff --git a/adapters/repos/db/lsmkv/cursor_bucket_replace.go b/adapters/repos/db/lsmkv/cursor_bucket_replace.go deleted file mode 100644 index c6be253919fd157aae682e766e912d676f64e801..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/cursor_bucket_replace.go +++ /dev/null @@ -1,266 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "bytes" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/lsmkv" -) - -type CursorReplace struct { - innerCursors []innerCursorReplace - state []cursorStateReplace - unlock func() - serveCache cursorStateReplace - - reusableIDList []int -} - -type innerCursorReplace interface { - first() ([]byte, []byte, error) - next() ([]byte, []byte, error) - seek([]byte) ([]byte, []byte, error) -} - -type cursorStateReplace struct { - key []byte - value []byte - err error -} - -// Cursor holds a RLock for the flushing state. It needs to be closed using the -// .Close() methods or otherwise the lock will never be released -func (b *Bucket) Cursor() *CursorReplace { - b.flushLock.RLock() - - if b.strategy != StrategyReplace { - panic("Cursor() called on strategy other than 'replace'") - } - - innerCursors, unlockSegmentGroup := b.disk.newCursors() - - // we have a flush-RLock, so we have the guarantee that the flushing state - // will not change for the lifetime of the cursor, thus there can only be two - // states: either a flushing memtable currently exists - or it doesn't - if b.flushing != nil { - innerCursors = append(innerCursors, b.flushing.newCursor()) - } - - innerCursors = append(innerCursors, b.active.newCursor()) - - return &CursorReplace{ - // cursor are in order from oldest to newest, with the memtable cursor - // being at the very top - innerCursors: innerCursors, - unlock: func() { - unlockSegmentGroup() - b.flushLock.RUnlock() - }, - } -} - -func (c *CursorReplace) Close() { - c.unlock() -} - -func (c *CursorReplace) seekAll(target []byte) { - state := make([]cursorStateReplace, len(c.innerCursors)) - for i, cur := range c.innerCursors { - key, value, err := cur.seek(target) - if errors.Is(err, lsmkv.NotFound) { - state[i].err = err - continue - } - - if errors.Is(err, lsmkv.Deleted) { - state[i].err = err - state[i].key = key - continue - } - - if err != nil { - panic(errors.Wrap(err, "unexpected error in seek (cursor type 'replace')")) - } - - state[i].key = key - state[i].value = value - } - - c.state = state -} - -func (c *CursorReplace) serveCurrentStateAndAdvance() ([]byte, []byte) { - id, err := c.cursorWithLowestKey() - if err != nil { - if errors.Is(err, lsmkv.NotFound) { - return nil, nil - } - } - - // check if this is a duplicate key before checking for the remaining errors, - // as cases such as 'entities.Deleted' can be better handled inside - // mergeDuplicatesInCurrentStateAndAdvance where we can be sure to act on - // segments in the correct order - if ids, ok := c.haveDuplicatesInState(id); ok { - return c.mergeDuplicatesInCurrentStateAndAdvance(ids) - } else { - return c.mergeDuplicatesInCurrentStateAndAdvance([]int{id}) - } -} - -func (c *CursorReplace) haveDuplicatesInState(idWithLowestKey int) ([]int, bool) { - key := c.state[idWithLowestKey].key - - c.reusableIDList = c.reusableIDList[:0] - - for i, cur := range c.state { - if i == idWithLowestKey { - c.reusableIDList = append(c.reusableIDList, i) - continue - } - - if bytes.Equal(key, cur.key) { - c.reusableIDList = append(c.reusableIDList, i) - } - } - - return c.reusableIDList, len(c.reusableIDList) > 1 -} - -// if there are no duplicates present it will still work as returning the -// latest result is the same as returning the only result -func (c *CursorReplace) mergeDuplicatesInCurrentStateAndAdvance(ids []int) ([]byte, []byte) { - c.copyStateIntoServeCache(ids[len(ids)-1]) - - // with a replace strategy only the highest will be returned, but still all - // need to be advanced - or we would just encounter them again in the next - // round - for _, id := range ids { - c.advanceInner(id) - } - - if errors.Is(c.serveCache.err, lsmkv.Deleted) { - // element was deleted, proceed with next round - return c.Next() - } - - return c.serveCache.key, c.serveCache.value -} - -func (c *CursorReplace) copyStateIntoServeCache(pos int) { - resMut := c.state[pos] - if len(resMut.key) > cap(c.serveCache.key) { - c.serveCache.key = make([]byte, len(resMut.key)) - } else { - c.serveCache.key = c.serveCache.key[:len(resMut.key)] - } - - if len(resMut.value) > cap(c.serveCache.value) { - c.serveCache.value = make([]byte, len(resMut.value)) - } else { - c.serveCache.value = c.serveCache.value[:len(resMut.value)] - } - - copy(c.serveCache.key, resMut.key) - copy(c.serveCache.value, resMut.value) - c.serveCache.err = resMut.err -} - -func (c *CursorReplace) Seek(key []byte) ([]byte, []byte) { - c.seekAll(key) - return c.serveCurrentStateAndAdvance() -} - -func (c *CursorReplace) cursorWithLowestKey() (int, error) { - err := lsmkv.NotFound - pos := -1 - var lowest []byte - - for i, res := range c.state { - if errors.Is(res.err, lsmkv.NotFound) { - continue - } - - if lowest == nil || bytes.Compare(res.key, lowest) <= 0 { - pos = i - err = res.err - lowest = res.key - } - } - - if err != nil { - return pos, err - } - - return pos, nil -} - -func (c *CursorReplace) advanceInner(id int) { - k, v, err := c.innerCursors[id].next() - if errors.Is(err, lsmkv.NotFound) { - c.state[id].err = err - c.state[id].key = nil - c.state[id].value = nil - return - } - - if errors.Is(err, lsmkv.Deleted) { - c.state[id].err = err - c.state[id].key = k - c.state[id].value = nil - return - } - - if err != nil { - panic(errors.Wrap(err, "unexpected error in advance")) - } - - c.state[id].key = k - c.state[id].value = v - c.state[id].err = nil -} - -func (c *CursorReplace) Next() ([]byte, []byte) { - return c.serveCurrentStateAndAdvance() -} - -func (c *CursorReplace) firstAll() { - state := make([]cursorStateReplace, len(c.innerCursors)) - for i, cur := range c.innerCursors { - key, value, err := cur.first() - if errors.Is(err, lsmkv.NotFound) { - state[i].err = err - continue - } - if errors.Is(err, lsmkv.Deleted) { - state[i].err = err - state[i].key = key - continue - } - - if err != nil { - panic(errors.Wrap(err, "unexpected error in first (cursor type 'replace')")) - } - - state[i].key = key - state[i].value = value - } - - c.state = state -} - -func (c *CursorReplace) First() ([]byte, []byte) { - c.firstAll() - return c.serveCurrentStateAndAdvance() -} diff --git a/adapters/repos/db/lsmkv/cursor_bucket_roaring_set.go b/adapters/repos/db/lsmkv/cursor_bucket_roaring_set.go deleted file mode 100644 index f22afb43e5434c5ca6e86533b98a9111fd0e96aa..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/cursor_bucket_roaring_set.go +++ /dev/null @@ -1,84 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "fmt" - - "github.com/weaviate/sroar" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv/roaringset" -) - -type CursorRoaringSet interface { - First() ([]byte, *sroar.Bitmap) - Next() ([]byte, *sroar.Bitmap) - Seek([]byte) ([]byte, *sroar.Bitmap) - Close() -} - -type cursorRoaringSet struct { - combinedCursor *roaringset.CombinedCursor - unlock func() -} - -func (c *cursorRoaringSet) First() ([]byte, *sroar.Bitmap) { - return c.combinedCursor.First() -} - -func (c *cursorRoaringSet) Next() ([]byte, *sroar.Bitmap) { - return c.combinedCursor.Next() -} - -func (c *cursorRoaringSet) Seek(key []byte) ([]byte, *sroar.Bitmap) { - return c.combinedCursor.Seek(key) -} - -func (c *cursorRoaringSet) Close() { - c.unlock() -} - -func (b *Bucket) CursorRoaringSet() CursorRoaringSet { - return b.cursorRoaringSet(false) -} - -func (b *Bucket) CursorRoaringSetKeyOnly() CursorRoaringSet { - return b.cursorRoaringSet(true) -} - -func (b *Bucket) cursorRoaringSet(keyOnly bool) CursorRoaringSet { - b.flushLock.RLock() - - // TODO move to helper func - if err := checkStrategyRoaringSet(b.strategy); err != nil { - panic(fmt.Sprintf("CursorRoaringSet() called on strategy other than '%s'", StrategyRoaringSet)) - } - - innerCursors, unlockSegmentGroup := b.disk.newRoaringSetCursors() - - // we have a flush-RLock, so we have the guarantee that the flushing state - // will not change for the lifetime of the cursor, thus there can only be two - // states: either a flushing memtable currently exists - or it doesn't - if b.flushing != nil { - innerCursors = append(innerCursors, b.flushing.newRoaringSetCursor()) - } - innerCursors = append(innerCursors, b.active.newRoaringSetCursor()) - - // cursors are in order from oldest to newest, with the memtable cursor - // being at the very top - return &cursorRoaringSet{ - combinedCursor: roaringset.NewCombinedCursor(innerCursors, keyOnly), - unlock: func() { - unlockSegmentGroup() - b.flushLock.RUnlock() - }, - } -} diff --git a/adapters/repos/db/lsmkv/cursor_bucket_set.go b/adapters/repos/db/lsmkv/cursor_bucket_set.go deleted file mode 100644 index d80ed84139eaefd86833a8960531312e8c80aefa..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/cursor_bucket_set.go +++ /dev/null @@ -1,262 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "bytes" - "errors" - "fmt" - - "github.com/weaviate/weaviate/entities/lsmkv" -) - -type CursorSet struct { - innerCursors []innerCursorCollection - state []cursorStateCollection - unlock func() - keyOnly bool -} - -type innerCursorCollection interface { - first() ([]byte, []value, error) - next() ([]byte, []value, error) - seek([]byte) ([]byte, []value, error) -} - -type cursorStateCollection struct { - key []byte - value []value - err error -} - -// SetCursor holds a RLock for the flushing state. It needs to be closed using the -// .Close() methods or otherwise the lock will never be released -func (b *Bucket) SetCursor() *CursorSet { - b.flushLock.RLock() - - if b.strategy != StrategySetCollection { - panic("SetCursor() called on strategy other than 'set'") - } - - innerCursors, unlockSegmentGroup := b.disk.newCollectionCursors() - - // we have a flush-RLock, so we have the guarantee that the flushing state - // will not change for the lifetime of the cursor, thus there can only be two - // states: either a flushing memtable currently exists - or it doesn't - if b.flushing != nil { - innerCursors = append(innerCursors, b.flushing.newCollectionCursor()) - } - - innerCursors = append(innerCursors, b.active.newCollectionCursor()) - - return &CursorSet{ - unlock: func() { - unlockSegmentGroup() - b.flushLock.RUnlock() - }, - // cursor are in order from oldest to newest, with the memtable cursor - // being at the very top - innerCursors: innerCursors, - } -} - -// SetCursorKeyOnly returns nil for all values. It has no control over the -// underlying "inner" cursors which may still retrieve a value which is then -// discarded. It does however, omit any handling of values, such as decoding, -// making this considerably more efficient if only keys are required. -// -// The same locking rules as for SetCursor apply. -func (b *Bucket) SetCursorKeyOnly() *CursorSet { - c := b.SetCursor() - c.keyOnly = true - return c -} - -func (c *CursorSet) Seek(key []byte) ([]byte, [][]byte) { - c.seekAll(key) - return c.serveCurrentStateAndAdvance() -} - -func (c *CursorSet) Next() ([]byte, [][]byte) { - return c.serveCurrentStateAndAdvance() -} - -func (c *CursorSet) First() ([]byte, [][]byte) { - c.firstAll() - return c.serveCurrentStateAndAdvance() -} - -func (c *CursorSet) Close() { - c.unlock() -} - -func (c *CursorSet) seekAll(target []byte) { - state := make([]cursorStateCollection, len(c.innerCursors)) - for i, cur := range c.innerCursors { - key, value, err := cur.seek(target) - if errors.Is(err, lsmkv.NotFound) { - state[i].err = err - continue - } - - if err != nil { - panic(fmt.Errorf("unexpected error in seek: %w", err)) - } - - state[i].key = key - if !c.keyOnly { - state[i].value = value - } - } - - c.state = state -} - -func (c *CursorSet) firstAll() { - state := make([]cursorStateCollection, len(c.innerCursors)) - for i, cur := range c.innerCursors { - key, value, err := cur.first() - if errors.Is(err, lsmkv.NotFound) { - state[i].err = err - continue - } - - if err != nil { - panic(fmt.Errorf("unexpected error in seek: %w", err)) - } - - state[i].key = key - if !c.keyOnly { - state[i].value = value - } - } - - c.state = state -} - -func (c *CursorSet) serveCurrentStateAndAdvance() ([]byte, [][]byte) { - id, err := c.cursorWithLowestKey() - if err != nil { - if errors.Is(err, lsmkv.NotFound) { - return nil, nil - } - } - - // check if this is a duplicate key before checking for the remaining errors, - // as cases such as 'entities.Deleted' can be better handled inside - // mergeDuplicatesInCurrentStateAndAdvance where we can be sure to act on - // segments in the correct order - if ids, ok := c.haveDuplicatesInState(id); ok { - return c.mergeDuplicatesInCurrentStateAndAdvance(ids) - } else { - return c.mergeDuplicatesInCurrentStateAndAdvance([]int{id}) - } -} - -func (c *CursorSet) cursorWithLowestKey() (int, error) { - err := lsmkv.NotFound - pos := -1 - var lowest []byte - - for i, res := range c.state { - if errors.Is(res.err, lsmkv.NotFound) { - continue - } - - if lowest == nil || bytes.Compare(res.key, lowest) <= 0 { - pos = i - err = res.err - lowest = res.key - } - } - - if err != nil { - return pos, err - } - - return pos, nil -} - -func (c *CursorSet) haveDuplicatesInState(idWithLowestKey int) ([]int, bool) { - key := c.state[idWithLowestKey].key - - var idsFound []int - - for i, cur := range c.state { - if i == idWithLowestKey { - idsFound = append(idsFound, i) - continue - } - - if bytes.Equal(key, cur.key) { - idsFound = append(idsFound, i) - } - } - - return idsFound, len(idsFound) > 1 -} - -// if there are no duplicates present it will still work as returning the -// latest result is the same as returning the only result -func (c *CursorSet) mergeDuplicatesInCurrentStateAndAdvance(ids []int) ([]byte, [][]byte) { - // take the key from any of the results, we have the guarantee that they're - // all the same - key := c.state[ids[0]].key - - var raw []value - for _, id := range ids { - raw = append(raw, c.state[id].value...) - c.advanceInner(id) - } - - values := newSetDecoder().Do(raw) - if len(values) == 0 { - // all values deleted, skip key - return c.Next() - } - - // TODO remove keyOnly option, not used anyway - if !c.keyOnly { - return key, values - } else { - return key, nil - } -} - -func (c *CursorSet) advanceInner(id int) { - k, v, err := c.innerCursors[id].next() - if errors.Is(err, lsmkv.NotFound) { - c.state[id].err = err - c.state[id].key = nil - if !c.keyOnly { - c.state[id].value = nil - } - return - } - - if errors.Is(err, lsmkv.Deleted) { - c.state[id].err = err - c.state[id].key = k - c.state[id].value = nil - return - } - - if err != nil { - panic(fmt.Errorf("unexpected error in advance: %w", err)) - } - - c.state[id].key = k - if !c.keyOnly { - c.state[id].value = v - } - c.state[id].err = nil -} diff --git a/adapters/repos/db/lsmkv/cursor_memtable_collection.go b/adapters/repos/db/lsmkv/cursor_memtable_collection.go deleted file mode 100644 index 496858f7cd80857e5bab6198e89494276bc5a54a..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/cursor_memtable_collection.go +++ /dev/null @@ -1,99 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "bytes" - - "github.com/weaviate/weaviate/entities/lsmkv" -) - -type memtableCursorCollection struct { - data []*binarySearchNodeMulti - current int - lock func() - unlock func() -} - -func (m *Memtable) newCollectionCursor() innerCursorCollection { - // This cursor is a really primitive approach, it actually requires - // flattening the entire memtable - even if the cursor were to point to the - // very last element. However, given that the memtable will on average be - // only half it's max capacity and even that is relatively small, we might - // get away with the full-flattening and a linear search. Let's not optimize - // prematurely. - - m.RLock() - defer m.RUnlock() - - data := m.keyMulti.flattenInOrder() - - return &memtableCursorCollection{ - data: data, - lock: m.RLock, - unlock: m.RUnlock, - } -} - -func (c *memtableCursorCollection) first() ([]byte, []value, error) { - c.lock() - defer c.unlock() - - if len(c.data) == 0 { - return nil, nil, lsmkv.NotFound - } - - c.current = 0 - - // there is no key-level tombstone, only individual values can have - // tombstones - return c.data[c.current].key, c.data[c.current].values, nil -} - -func (c *memtableCursorCollection) seek(key []byte) ([]byte, []value, error) { - c.lock() - defer c.unlock() - - pos := c.posLargerThanEqual(key) - if pos == -1 { - return nil, nil, lsmkv.NotFound - } - - c.current = pos - // there is no key-level tombstone, only individual values can have - // tombstones - return c.data[pos].key, c.data[pos].values, nil -} - -func (c *memtableCursorCollection) posLargerThanEqual(key []byte) int { - for i, node := range c.data { - if bytes.Compare(node.key, key) >= 0 { - return i - } - } - - return -1 -} - -func (c *memtableCursorCollection) next() ([]byte, []value, error) { - c.lock() - defer c.unlock() - - c.current++ - if c.current >= len(c.data) { - return nil, nil, lsmkv.NotFound - } - - // there is no key-level tombstone, only individual values can have - // tombstones - return c.data[c.current].key, c.data[c.current].values, nil -} diff --git a/adapters/repos/db/lsmkv/cursor_memtable_map.go b/adapters/repos/db/lsmkv/cursor_memtable_map.go deleted file mode 100644 index 7cfc9ebf98b99d52b64c98151256ba080edec411..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/cursor_memtable_map.go +++ /dev/null @@ -1,99 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "bytes" - - "github.com/weaviate/weaviate/entities/lsmkv" -) - -type memtableCursorMap struct { - data []*binarySearchNodeMap - current int - lock func() - unlock func() -} - -func (m *Memtable) newMapCursor() innerCursorMap { - // This cursor is a really primitive approach, it actually requires - // flattening the entire memtable - even if the cursor were to point to the - // very last element. However, given that the memtable will on average be - // only half it's max capacity and even that is relatively small, we might - // get away with the full-flattening and a linear search. Let's not optimize - // prematurely. - - m.RLock() - defer m.RUnlock() - - data := m.keyMap.flattenInOrder() - - return &memtableCursorMap{ - data: data, - lock: m.RLock, - unlock: m.RUnlock, - } -} - -func (c *memtableCursorMap) first() ([]byte, []MapPair, error) { - c.lock() - defer c.unlock() - - if len(c.data) == 0 { - return nil, nil, lsmkv.NotFound - } - - c.current = 0 - - // there is no key-level tombstone, only individual values can have - // tombstones - return c.data[c.current].key, c.data[c.current].values, nil -} - -func (c *memtableCursorMap) seek(key []byte) ([]byte, []MapPair, error) { - c.lock() - defer c.unlock() - - pos := c.posLargerThanEqual(key) - if pos == -1 { - return nil, nil, lsmkv.NotFound - } - - c.current = pos - // there is no key-level tombstone, only individual values can have - // tombstones - return c.data[pos].key, c.data[pos].values, nil -} - -func (c *memtableCursorMap) posLargerThanEqual(key []byte) int { - for i, node := range c.data { - if bytes.Compare(node.key, key) >= 0 { - return i - } - } - - return -1 -} - -func (c *memtableCursorMap) next() ([]byte, []MapPair, error) { - c.lock() - defer c.unlock() - - c.current++ - if c.current >= len(c.data) { - return nil, nil, lsmkv.NotFound - } - - // there is no key-level tombstone, only individual values can have - // tombstones - return c.data[c.current].key, c.data[c.current].values, nil -} diff --git a/adapters/repos/db/lsmkv/cursor_memtable_replace.go b/adapters/repos/db/lsmkv/cursor_memtable_replace.go deleted file mode 100644 index a48700dc36cf226a328b593e63e52cdbd8191197..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/cursor_memtable_replace.go +++ /dev/null @@ -1,102 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "bytes" - - "github.com/weaviate/weaviate/entities/lsmkv" -) - -type memtableCursor struct { - data []*binarySearchNode - current int - lock func() - unlock func() -} - -func (m *Memtable) newCursor() innerCursorReplace { - // This cursor is a really primitive approach, it actually requires - // flattening the entire memtable - even if the cursor were to point to the - // very last element. However, given that the memtable will on average be - // only half it's max capacity and even that is relatively small, we might - // get away with the full-flattening and a linear search. Let's not optimize - // prematurely. - - m.RLock() - defer m.RUnlock() - - data := m.key.flattenInOrder() - - return &memtableCursor{ - data: data, - lock: m.RLock, - unlock: m.RUnlock, - } -} - -func (c *memtableCursor) first() ([]byte, []byte, error) { - c.lock() - defer c.unlock() - - if len(c.data) == 0 { - return nil, nil, lsmkv.NotFound - } - - c.current = 0 - - if c.data[c.current].tombstone { - return c.data[c.current].key, nil, lsmkv.Deleted - } - return c.data[c.current].key, c.data[c.current].value, nil -} - -func (c *memtableCursor) seek(key []byte) ([]byte, []byte, error) { - c.lock() - defer c.unlock() - - pos := c.posLargerThanEqual(key) - if pos == -1 { - return nil, nil, lsmkv.NotFound - } - - c.current = pos - if c.data[c.current].tombstone { - return c.data[c.current].key, nil, lsmkv.Deleted - } - return c.data[pos].key, c.data[pos].value, nil -} - -func (c *memtableCursor) posLargerThanEqual(key []byte) int { - for i, node := range c.data { - if bytes.Compare(node.key, key) >= 0 { - return i - } - } - - return -1 -} - -func (c *memtableCursor) next() ([]byte, []byte, error) { - c.lock() - defer c.unlock() - - c.current++ - if c.current >= len(c.data) { - return nil, nil, lsmkv.NotFound - } - - if c.data[c.current].tombstone { - return c.data[c.current].key, nil, lsmkv.Deleted - } - return c.data[c.current].key, c.data[c.current].value, nil -} diff --git a/adapters/repos/db/lsmkv/cursor_memtable_roaring_set.go b/adapters/repos/db/lsmkv/cursor_memtable_roaring_set.go deleted file mode 100644 index 7a1aba508894987c147edcb1ff037b40aed59d16..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/cursor_memtable_roaring_set.go +++ /dev/null @@ -1,23 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import "github.com/weaviate/weaviate/adapters/repos/db/lsmkv/roaringset" - -func (m *Memtable) newRoaringSetCursor() roaringset.InnerCursor { - m.RLock() - defer m.RUnlock() - - // Since FlattenInOrder makes deep copy of bst's nodes, - // no further memtable's locking in required on cursor's methods - return roaringset.NewBinarySearchTreeCursor(m.roaringSet) -} diff --git a/adapters/repos/db/lsmkv/cursor_segment_collection.go b/adapters/repos/db/lsmkv/cursor_segment_collection.go deleted file mode 100644 index 46173d22f5ea68cc08fd065b20ee05eb06a66aac..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/cursor_segment_collection.go +++ /dev/null @@ -1,97 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "github.com/weaviate/weaviate/entities/lsmkv" -) - -type segmentCursorCollection struct { - segment *segment - nextOffset uint64 -} - -func (s *segment) newCollectionCursor() *segmentCursorCollection { - return &segmentCursorCollection{ - segment: s, - } -} - -func (sg *SegmentGroup) newCollectionCursors() ([]innerCursorCollection, func()) { - sg.maintenanceLock.RLock() - out := make([]innerCursorCollection, len(sg.segments)) - - for i, segment := range sg.segments { - out[i] = segment.newCollectionCursor() - } - - return out, sg.maintenanceLock.RUnlock -} - -func (s *segmentCursorCollection) seek(key []byte) ([]byte, []value, error) { - node, err := s.segment.index.Seek(key) - if err != nil { - return nil, nil, err - } - - parsed, err := s.parseCollectionNode(nodeOffset{node.Start, node.End}) - // make sure to set the next offset before checking the error. The error - // could be 'entities.Deleted' which would require that the offset is still advanced - // for the next cycle - s.nextOffset = node.End - if err != nil { - return parsed.primaryKey, nil, err - } - - return parsed.primaryKey, parsed.values, nil -} - -func (s *segmentCursorCollection) next() ([]byte, []value, error) { - if s.nextOffset >= s.segment.dataEndPos { - return nil, nil, lsmkv.NotFound - } - - parsed, err := s.parseCollectionNode(nodeOffset{start: s.nextOffset}) - // make sure to set the next offset before checking the error. The error - // could be 'entities.Deleted' which would require that the offset is still advanced - // for the next cycle - s.nextOffset = s.nextOffset + uint64(parsed.offset) - if err != nil { - return parsed.primaryKey, nil, err - } - - return parsed.primaryKey, parsed.values, nil -} - -func (s *segmentCursorCollection) first() ([]byte, []value, error) { - s.nextOffset = s.segment.dataStartPos - - parsed, err := s.parseCollectionNode(nodeOffset{start: s.nextOffset}) - // make sure to set the next offset before checking the error. The error - // could be 'entities.Deleted' which would require that the offset is still advanced - // for the next cycle - s.nextOffset = s.nextOffset + uint64(parsed.offset) - if err != nil { - return parsed.primaryKey, nil, err - } - - return parsed.primaryKey, parsed.values, nil -} - -func (s *segmentCursorCollection) parseCollectionNode(offset nodeOffset) (segmentCollectionNode, error) { - r, err := s.segment.newNodeReader(offset) - if err != nil { - return segmentCollectionNode{}, err - } - - return ParseCollectionNode(r) -} diff --git a/adapters/repos/db/lsmkv/cursor_segment_collection_reusable.go b/adapters/repos/db/lsmkv/cursor_segment_collection_reusable.go deleted file mode 100644 index 026894265b67c7223e369c90b25de12d67672e2a..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/cursor_segment_collection_reusable.go +++ /dev/null @@ -1,83 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "github.com/weaviate/weaviate/entities/lsmkv" -) - -type segmentCursorCollectionReusable struct { - segment *segment - nextOffset uint64 - nodeBuf segmentCollectionNode -} - -func (s *segment) newCollectionCursorReusable() *segmentCursorCollectionReusable { - return &segmentCursorCollectionReusable{ - segment: s, - } -} - -func (s *segmentCursorCollectionReusable) seek(key []byte) ([]byte, []value, error) { - node, err := s.segment.index.Seek(key) - if err != nil { - return nil, nil, err - } - - err = s.parseCollectionNodeInto(nodeOffset{node.Start, node.End}) - if err != nil { - return s.nodeBuf.primaryKey, nil, err - } - - s.nextOffset = node.End - - return s.nodeBuf.primaryKey, s.nodeBuf.values, nil -} - -func (s *segmentCursorCollectionReusable) next() ([]byte, []value, error) { - if s.nextOffset >= s.segment.dataEndPos { - return nil, nil, lsmkv.NotFound - } - - err := s.parseCollectionNodeInto(nodeOffset{start: s.nextOffset}) - // make sure to set the next offset before checking the error. The error - // could be 'entities.Deleted' which would require that the offset is still advanced - // for the next cycle - s.nextOffset = s.nextOffset + uint64(s.nodeBuf.offset) - if err != nil { - return s.nodeBuf.primaryKey, nil, err - } - - return s.nodeBuf.primaryKey, s.nodeBuf.values, nil -} - -func (s *segmentCursorCollectionReusable) first() ([]byte, []value, error) { - s.nextOffset = s.segment.dataStartPos - - err := s.parseCollectionNodeInto(nodeOffset{start: s.nextOffset}) - if err != nil { - return s.nodeBuf.primaryKey, nil, err - } - - s.nextOffset = s.nextOffset + uint64(s.nodeBuf.offset) - - return s.nodeBuf.primaryKey, s.nodeBuf.values, nil -} - -func (s *segmentCursorCollectionReusable) parseCollectionNodeInto(offset nodeOffset) error { - r, err := s.segment.newNodeReader(offset) - if err != nil { - return err - } - - return ParseCollectionNodeInto(r, &s.nodeBuf) -} diff --git a/adapters/repos/db/lsmkv/cursor_segment_map.go b/adapters/repos/db/lsmkv/cursor_segment_map.go deleted file mode 100644 index ae667eb5496400ff709128f7c8194d532c32457e..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/cursor_segment_map.go +++ /dev/null @@ -1,118 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import "github.com/weaviate/weaviate/entities/lsmkv" - -type segmentCursorMap struct { - segment *segment - nextOffset uint64 -} - -func (s *segment) newMapCursor() *segmentCursorMap { - return &segmentCursorMap{ - segment: s, - } -} - -func (sg *SegmentGroup) newMapCursors() ([]innerCursorMap, func()) { - sg.maintenanceLock.RLock() - out := make([]innerCursorMap, len(sg.segments)) - - for i, segment := range sg.segments { - out[i] = segment.newMapCursor() - } - - return out, sg.maintenanceLock.RUnlock -} - -func (s *segmentCursorMap) seek(key []byte) ([]byte, []MapPair, error) { - node, err := s.segment.index.Seek(key) - if err != nil { - return nil, nil, err - } - - parsed, err := s.parseCollectionNode(nodeOffset{node.Start, node.End}) - // make sure to set the next offset before checking the error. The error - // could be 'Deleted' which would require that the offset is still advanced - // for the next cycle - s.nextOffset = node.End - if err != nil { - return parsed.primaryKey, nil, err - } - - pairs := make([]MapPair, len(parsed.values)) - for i := range pairs { - if err := pairs[i].FromBytes(parsed.values[i].value, false); err != nil { - return nil, nil, err - } - pairs[i].Tombstone = parsed.values[i].tombstone - } - - return parsed.primaryKey, pairs, nil -} - -func (s *segmentCursorMap) next() ([]byte, []MapPair, error) { - if s.nextOffset >= s.segment.dataEndPos { - return nil, nil, lsmkv.NotFound - } - - parsed, err := s.parseCollectionNode(nodeOffset{start: s.nextOffset}) - // make sure to set the next offset before checking the error. The error - // could be 'Deleted' which would require that the offset is still advanced - // for the next cycle - s.nextOffset = s.nextOffset + uint64(parsed.offset) - if err != nil { - return parsed.primaryKey, nil, err - } - - pairs := make([]MapPair, len(parsed.values)) - for i := range pairs { - if err := pairs[i].FromBytes(parsed.values[i].value, false); err != nil { - return nil, nil, err - } - pairs[i].Tombstone = parsed.values[i].tombstone - } - - return parsed.primaryKey, pairs, nil -} - -func (s *segmentCursorMap) first() ([]byte, []MapPair, error) { - s.nextOffset = s.segment.dataStartPos - - parsed, err := s.parseCollectionNode(nodeOffset{start: s.nextOffset}) - // make sure to set the next offset before checking the error. The error - // could be 'Deleted' which would require that the offset is still advanced - // for the next cycle - s.nextOffset = s.nextOffset + uint64(parsed.offset) - if err != nil { - return parsed.primaryKey, nil, err - } - - pairs := make([]MapPair, len(parsed.values)) - for i := range pairs { - if err := pairs[i].FromBytes(parsed.values[i].value, false); err != nil { - return nil, nil, err - } - pairs[i].Tombstone = parsed.values[i].tombstone - } - - return parsed.primaryKey, pairs, nil -} - -func (s *segmentCursorMap) parseCollectionNode(offset nodeOffset) (segmentCollectionNode, error) { - r, err := s.segment.newNodeReader(offset) - if err != nil { - return segmentCollectionNode{}, err - } - return ParseCollectionNode(r) -} diff --git a/adapters/repos/db/lsmkv/cursor_segment_replace.go b/adapters/repos/db/lsmkv/cursor_segment_replace.go deleted file mode 100644 index 015a33b8c4726181220d457be543beff83bc0e15..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/cursor_segment_replace.go +++ /dev/null @@ -1,182 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "github.com/weaviate/weaviate/entities/lsmkv" - "github.com/weaviate/weaviate/usecases/byteops" -) - -type segmentCursorReplace struct { - segment *segment - nextOffset uint64 - reusableNode *segmentReplaceNode - reusableBORW byteops.ReadWriter -} - -func (s *segment) newCursor() *segmentCursorReplace { - return &segmentCursorReplace{ - segment: s, - reusableNode: &segmentReplaceNode{}, - reusableBORW: byteops.NewReadWriter(nil), - } -} - -func (sg *SegmentGroup) newCursors() ([]innerCursorReplace, func()) { - sg.maintenanceLock.RLock() - out := make([]innerCursorReplace, len(sg.segments)) - - for i, segment := range sg.segments { - out[i] = segment.newCursor() - } - - return out, sg.maintenanceLock.RUnlock -} - -func (s *segmentCursorReplace) seek(key []byte) ([]byte, []byte, error) { - node, err := s.segment.index.Seek(key) - if err != nil { - return nil, nil, err - } - - err = s.parseReplaceNodeInto(nodeOffset{start: node.Start, end: node.End}, - s.segment.contents[node.Start:node.End]) - // make sure to set the next offset before checking the error. The error - // could be 'Deleted' which would require that the offset is still advanced - // for the next cycle - s.nextOffset = node.End - - if err != nil { - return s.reusableNode.primaryKey, nil, err - } - - return s.reusableNode.primaryKey, s.reusableNode.value, nil -} - -func (s *segmentCursorReplace) next() ([]byte, []byte, error) { - if s.nextOffset >= s.segment.dataEndPos { - return nil, nil, lsmkv.NotFound - } - - err := s.parseReplaceNodeInto(nodeOffset{start: s.nextOffset}, - s.segment.contents[s.nextOffset:]) - // make sure to set the next offset before checking the error. The error - // could be 'Deleted' which would require that the offset is still advanced - // for the next cycle - s.nextOffset = s.nextOffset + uint64(s.reusableNode.offset) - if err != nil { - return s.reusableNode.primaryKey, nil, err - } - - return s.reusableNode.primaryKey, s.reusableNode.value, nil -} - -func (s *segmentCursorReplace) first() ([]byte, []byte, error) { - s.nextOffset = s.segment.dataStartPos - err := s.parseReplaceNodeInto(nodeOffset{start: s.nextOffset}, - s.segment.contents[s.nextOffset:]) - // make sure to set the next offset before checking the error. The error - // could be 'Deleted' which would require that the offset is still advanced - // for the next cycle - s.nextOffset = s.nextOffset + uint64(s.reusableNode.offset) - if err != nil { - return s.reusableNode.primaryKey, nil, err - } - - return s.reusableNode.primaryKey, s.reusableNode.value, nil -} - -func (s *segmentCursorReplace) nextWithAllKeys() (segmentReplaceNode, error) { - out := segmentReplaceNode{} - if s.nextOffset >= s.segment.dataEndPos { - return out, lsmkv.NotFound - } - - parsed, err := s.parseReplaceNode(nodeOffset{start: s.nextOffset}) - // make sure to set the next offset before checking the error. The error - // could be 'Deleted' which would require that the offset is still advanced - // for the next cycle - s.nextOffset = s.nextOffset + uint64(parsed.offset) - if err != nil { - return parsed, err - } - - return parsed, nil -} - -func (s *segmentCursorReplace) firstWithAllKeys() (segmentReplaceNode, error) { - s.nextOffset = s.segment.dataStartPos - parsed, err := s.parseReplaceNode(nodeOffset{start: s.nextOffset}) - // make sure to set the next offset before checking the error. The error - // could be 'Deleted' which would require that the offset is still advanced - // for the next cycle - s.nextOffset = s.nextOffset + uint64(parsed.offset) - if err != nil { - return parsed, err - } - - return parsed, nil -} - -func (s *segmentCursorReplace) parseReplaceNode(offset nodeOffset) (segmentReplaceNode, error) { - r, err := s.segment.newNodeReader(offset) - if err != nil { - return segmentReplaceNode{}, err - } - out, err := ParseReplaceNode(r, s.segment.secondaryIndexCount) - if out.tombstone { - return out, lsmkv.Deleted - } - return out, err -} - -func (s *segmentCursorReplace) parseReplaceNodeInto(offset nodeOffset, buf []byte) error { - if s.segment.mmapContents { - return s.parse(buf) - } - - r, err := s.segment.newNodeReader(offset) - if err != nil { - return err - } - - err = ParseReplaceNodeIntoPread(r, s.segment.secondaryIndexCount, s.reusableNode) - if err != nil { - return err - } - - if s.reusableNode.tombstone { - return lsmkv.Deleted - } - - return nil -} - -func (s *segmentCursorReplace) parse(in []byte) error { - if len(in) == 0 { - return lsmkv.NotFound - } - - s.reusableBORW.ResetBuffer(in) - - err := ParseReplaceNodeIntoMMAP(&s.reusableBORW, s.segment.secondaryIndexCount, - s.reusableNode) - if err != nil { - return err - } - - if s.reusableNode.tombstone { - return lsmkv.Deleted - } - - return nil -} diff --git a/adapters/repos/db/lsmkv/cursor_segment_roaring_set.go b/adapters/repos/db/lsmkv/cursor_segment_roaring_set.go deleted file mode 100644 index 93b17d6231e3d726a57c9b099fc385a465715a02..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/cursor_segment_roaring_set.go +++ /dev/null @@ -1,53 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv/roaringset" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv/segmentindex" -) - -func (s *segment) newRoaringSetCursor() *roaringset.SegmentCursor { - return roaringset.NewSegmentCursor(s.contents[s.dataStartPos:s.dataEndPos], - &roaringSetSeeker{s.index}) -} - -func (sg *SegmentGroup) newRoaringSetCursors() ([]roaringset.InnerCursor, func()) { - sg.maintenanceLock.RLock() - out := make([]roaringset.InnerCursor, len(sg.segments)) - - for i, segment := range sg.segments { - out[i] = segment.newRoaringSetCursor() - } - - return out, sg.maintenanceLock.RUnlock -} - -// diskIndex returns node's Start and End offsets -// taking into account HeaderSize. SegmentCursor of RoaringSet -// accepts only payload part of underlying segment content, therefore -// offsets should be adjusted and reduced by HeaderSize -type roaringSetSeeker struct { - diskIndex diskIndex -} - -func (s *roaringSetSeeker) Seek(key []byte) (segmentindex.Node, error) { - node, err := s.diskIndex.Seek(key) - if err != nil { - return segmentindex.Node{}, err - } - return segmentindex.Node{ - Key: node.Key, - Start: node.Start - segmentindex.HeaderSize, - End: node.End - segmentindex.HeaderSize, - }, nil -} diff --git a/adapters/repos/db/lsmkv/doc.go b/adapters/repos/db/lsmkv/doc.go deleted file mode 100644 index 0b789bb3b0528bc0b4866f39c963d4b02eaf2bb5..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/doc.go +++ /dev/null @@ -1,75 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -/* -# LSMKV (= Log-structured Merge-Tree Key-Value Store) - -This package contains Weaviate's custom LSM store. While modeled after the -usecases that are required for Weaviate to be fast, reliable, and scalable, it -is technically completely independent. You could build your own database on top -of this key-value store. - -Covering the architecture of [LSM Stores] in general goes beyond the scope of -this documentation. Therefore things that are specific to this implementation -are highlighted. - -# Strategies - -To understand the different type of buckets in this store, you need to -familiarize yourself with the following strategies. A strategy defines a -different usecase for a [Bucket]. - - - "Replace" - - Replace resembles the classical key-value store. Each key has exactly one - value. A subsequent PUT on an an existing key, replaces the value (hence - the name "replace"). Once replaced a former value can no longer be - retrieved, and will eventually be removed in compactions. - - - "Set" (aka "SetCollection") - - A set behaves like an unordered collection of independent values. In other - words a single key has multiple values. For example, for key "foo", you - could have values "bar1", "bar2", "bazzinga". A bucket of this type is - optimized for cheap writes to add new set additions. For example adding - another set element has a fixed cost independent of the number of the - existing set length. This makes it very well suited for building an - inverted index. - - Retrieving a Set has a slight cost to it if a set is spread across multiple - segments. This cost will eventually reduce as more and more compactions - happen. In the ideal case (fully compacted DB), retrieving a Set requires - just a single disk read. - - - "Map" (aka "MapCollection") - - Maps are similar to Sets in the sense that for a single key there are - multiple values. However, each value is in itself a key-value pair. This - makes this type very similar to a dict or hashmap type. For example for - key "foo", you could have value pairs: "bar":17, "baz":19. - - This makes a map a great use case for an inverted index that needs to store - additional info beyond just the docid-pointer, such as in the case of a - BM25 index where the term frequency needs to be stored. - - The same performance-considerations as for sets apply. - -# Navigate around these docs - -Good entrypoints to learn more about how this package works include [Store] -with [New] and [Store.CreateOrLoadBucket], as well as [Bucket] with -[Bucket.Get], [Bucket.GetBySecondary], [Bucket.Put], etc. - -Each strategy also supports cursor types: [CursorReplace] can be created using [Bucket.Cursor], [CursorSet] can be created with [Bucket.SetCursor] , and [CursorMap] can be created with [Bucket.MapCursor]. - -[LSM Stores]: https://en.wikipedia.org/wiki/Log-structured_merge-tree -*/ -package lsmkv diff --git a/adapters/repos/db/lsmkv/entities/doc.go b/adapters/repos/db/lsmkv/entities/doc.go deleted file mode 100644 index 970fc6a7e59295eff0abc634f2d9d098d3d65c97..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/entities/doc.go +++ /dev/null @@ -1,13 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// ent contains common types used throughout various lsmkv (sub-)packages -package entities diff --git a/adapters/repos/db/lsmkv/entities/strategies.go b/adapters/repos/db/lsmkv/entities/strategies.go deleted file mode 100644 index ac3041d0b0608813147a156058c48e239c71c42a..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/entities/strategies.go +++ /dev/null @@ -1,44 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package entities - -const ( - // StrategyReplace allows for idem-potent PUT where the latest takes presence - StrategyReplace = "replace" - StrategySetCollection = "setcollection" - StrategyMapCollection = "mapcollection" - StrategyRoaringSet = "roaringset" -) - -type SegmentStrategy uint16 - -const ( - SegmentStrategyReplace SegmentStrategy = iota - SegmentStrategySetCollection - SegmentStrategyMapCollection - SegmentStrategyRoaringSet -) - -func SegmentStrategyFromString(in string) SegmentStrategy { - switch in { - case StrategyReplace: - return SegmentStrategyReplace - case StrategySetCollection: - return SegmentStrategySetCollection - case StrategyMapCollection: - return SegmentStrategyMapCollection - case StrategyRoaringSet: - return SegmentStrategyRoaringSet - default: - panic("unsupported strategy") - } -} diff --git a/adapters/repos/db/lsmkv/fake.wal b/adapters/repos/db/lsmkv/fake.wal deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/adapters/repos/db/lsmkv/helper_for_test.go b/adapters/repos/db/lsmkv/helper_for_test.go deleted file mode 100644 index f45aa7343482356ce95bba9235717e268430d7d4..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/helper_for_test.go +++ /dev/null @@ -1,24 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package lsmkv - -import ( - "math/rand" - "time" -) - -func getRandomSeed() *rand.Rand { - return rand.New(rand.NewSource(time.Now().UnixNano())) -} diff --git a/adapters/repos/db/lsmkv/memtable.go b/adapters/repos/db/lsmkv/memtable.go deleted file mode 100644 index 30b0b1d7963fb572759107f8a395e5b57bbfd36a..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/memtable.go +++ /dev/null @@ -1,351 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "path/filepath" - "sync" - "time" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv/roaringset" - "github.com/weaviate/weaviate/entities/lsmkv" -) - -type Memtable struct { - sync.RWMutex - key *binarySearchTree - keyMulti *binarySearchTreeMulti - keyMap *binarySearchTreeMap - primaryIndex *binarySearchTree - roaringSet *roaringset.BinarySearchTree - commitlog *commitLogger - size uint64 - path string - strategy string - secondaryIndices uint16 - secondaryToPrimary []map[string][]byte - lastWrite time.Time - createdAt time.Time - metrics *memtableMetrics -} - -func newMemtable(path string, strategy string, - secondaryIndices uint16, metrics *Metrics, -) (*Memtable, error) { - cl, err := newCommitLogger(path) - if err != nil { - return nil, errors.Wrap(err, "init commit logger") - } - - m := &Memtable{ - key: &binarySearchTree{}, - keyMulti: &binarySearchTreeMulti{}, - keyMap: &binarySearchTreeMap{}, - primaryIndex: &binarySearchTree{}, // todo, sort upfront - roaringSet: &roaringset.BinarySearchTree{}, - commitlog: cl, - path: path, - strategy: strategy, - secondaryIndices: secondaryIndices, - lastWrite: time.Now(), - createdAt: time.Now(), - metrics: newMemtableMetrics(metrics, filepath.Dir(path), strategy), - } - - if m.secondaryIndices > 0 { - m.secondaryToPrimary = make([]map[string][]byte, m.secondaryIndices) - for i := range m.secondaryToPrimary { - m.secondaryToPrimary[i] = map[string][]byte{} - } - } - - m.metrics.size(m.size) - - return m, nil -} - -func (m *Memtable) get(key []byte) ([]byte, error) { - start := time.Now() - defer m.metrics.get(start.UnixNano()) - - if m.strategy != StrategyReplace { - return nil, errors.Errorf("get only possible with strategy 'replace'") - } - - m.RLock() - defer m.RUnlock() - - v, err := m.key.get(key) - if err != nil { - return nil, err - } - - return v, nil -} - -func (m *Memtable) getBySecondary(pos int, key []byte) ([]byte, error) { - start := time.Now() - defer m.metrics.getBySecondary(start.UnixNano()) - - if m.strategy != StrategyReplace { - return nil, errors.Errorf("get only possible with strategy 'replace'") - } - - m.RLock() - defer m.RUnlock() - - primary := m.secondaryToPrimary[pos][string(key)] - if primary == nil { - return nil, lsmkv.NotFound - } - - v, err := m.key.get(primary) - if err != nil { - return nil, err - } - - return v, nil -} - -func (m *Memtable) put(key, value []byte, opts ...SecondaryKeyOption) error { - start := time.Now() - defer m.metrics.put(start.UnixNano()) - - if m.strategy != StrategyReplace { - return errors.Errorf("put only possible with strategy 'replace'") - } - - m.Lock() - defer m.Unlock() - - var secondaryKeys [][]byte - if m.secondaryIndices > 0 { - secondaryKeys = make([][]byte, m.secondaryIndices) - for _, opt := range opts { - if err := opt(secondaryKeys); err != nil { - return err - } - } - } - - if err := m.commitlog.put(segmentReplaceNode{ - primaryKey: key, - value: value, - secondaryIndexCount: m.secondaryIndices, - secondaryKeys: secondaryKeys, - tombstone: false, - }); err != nil { - return errors.Wrap(err, "write into commit log") - } - - netAdditions, previousKeys := m.key.insert(key, value, secondaryKeys) - m.size += uint64(netAdditions) - m.metrics.size(m.size) - - for i, sec := range previousKeys { - m.secondaryToPrimary[i][string(sec)] = nil - } - - for i, sec := range secondaryKeys { - m.secondaryToPrimary[i][string(sec)] = key - } - - m.lastWrite = time.Now() - - return nil -} - -func (m *Memtable) setTombstone(key []byte, opts ...SecondaryKeyOption) error { - start := time.Now() - defer m.metrics.setTombstone(start.UnixNano()) - - if m.strategy != "replace" { - return errors.Errorf("setTombstone only possible with strategy 'replace'") - } - - m.Lock() - defer m.Unlock() - - var secondaryKeys [][]byte - if m.secondaryIndices > 0 { - secondaryKeys = make([][]byte, m.secondaryIndices) - for _, opt := range opts { - if err := opt(secondaryKeys); err != nil { - return err - } - } - } - - if err := m.commitlog.put(segmentReplaceNode{ - primaryKey: key, - value: nil, - secondaryIndexCount: m.secondaryIndices, - secondaryKeys: secondaryKeys, - tombstone: true, - }); err != nil { - return errors.Wrap(err, "write into commit log") - } - - m.key.setTombstone(key, secondaryKeys) - m.size += uint64(len(key)) + 1 // 1 byte for tombstone - m.lastWrite = time.Now() - m.metrics.size(m.size) - - return nil -} - -func (m *Memtable) getCollection(key []byte) ([]value, error) { - start := time.Now() - defer m.metrics.getCollection(start.UnixNano()) - - if m.strategy != StrategySetCollection && m.strategy != StrategyMapCollection { - return nil, errors.Errorf("getCollection only possible with strategies %q, %q", - StrategySetCollection, StrategyMapCollection) - } - - m.RLock() - defer m.RUnlock() - - v, err := m.keyMulti.get(key) - if err != nil { - return nil, err - } - - return v, nil -} - -func (m *Memtable) getMap(key []byte) ([]MapPair, error) { - start := time.Now() - defer m.metrics.getMap(start.UnixNano()) - - if m.strategy != StrategyMapCollection { - return nil, errors.Errorf("getCollection only possible with strategy %q", - StrategyMapCollection) - } - - m.RLock() - defer m.RUnlock() - - v, err := m.keyMap.get(key) - if err != nil { - return nil, err - } - - return v, nil -} - -func (m *Memtable) append(key []byte, values []value) error { - start := time.Now() - defer m.metrics.append(start.UnixNano()) - - if m.strategy != StrategySetCollection && m.strategy != StrategyMapCollection { - return errors.Errorf("append only possible with strategies %q, %q", - StrategySetCollection, StrategyMapCollection) - } - - m.Lock() - defer m.Unlock() - if err := m.commitlog.append(segmentCollectionNode{ - primaryKey: key, - values: values, - }); err != nil { - return errors.Wrap(err, "write into commit log") - } - - m.keyMulti.insert(key, values) - m.size += uint64(len(key)) - for _, value := range values { - m.size += uint64(len(value.value)) - } - - m.metrics.size(m.size) - m.lastWrite = time.Now() - return nil -} - -func (m *Memtable) appendMapSorted(key []byte, pair MapPair) error { - start := time.Now() - defer m.metrics.appendMapSorted(start.UnixNano()) - - if m.strategy != StrategyMapCollection { - return errors.Errorf("append only possible with strategy %q", - StrategyMapCollection) - } - - m.Lock() - defer m.Unlock() - - valuesForCommitLog, err := pair.Bytes() - if err != nil { - return err - } - - if err := m.commitlog.append(segmentCollectionNode{ - primaryKey: key, - values: []value{ - { - value: valuesForCommitLog, - }, - }, - }); err != nil { - return errors.Wrap(err, "write into commit log") - } - - m.keyMap.insert(key, pair) - - m.size += uint64(len(key) + len(valuesForCommitLog)) - m.lastWrite = time.Now() - m.metrics.size(m.size) - - return nil -} - -func (m *Memtable) Size() uint64 { - m.RLock() - defer m.RUnlock() - - return m.size -} - -func (m *Memtable) ActiveDuration() time.Duration { - m.RLock() - defer m.RUnlock() - - return time.Since(m.createdAt) -} - -func (m *Memtable) IdleDuration() time.Duration { - m.RLock() - defer m.RUnlock() - - return time.Since(m.lastWrite) -} - -func (m *Memtable) countStats() *countStats { - m.RLock() - defer m.RUnlock() - return m.key.countStats() -} - -// the WAL uses a buffer and isn't written until the buffer size is crossed or -// this function explicitly called. This allows to safge unnecessary disk -// writes in larger operations, such as batches. It is sufficient to call write -// on the WAL just once. This does not make a batch atomic, but it guarantees -// that the WAL is written before a successful response is returned to the -// user. -func (m *Memtable) writeWAL() error { - m.Lock() - defer m.Unlock() - - return m.commitlog.flushBuffers() -} diff --git a/adapters/repos/db/lsmkv/memtable_flush.go b/adapters/repos/db/lsmkv/memtable_flush.go deleted file mode 100644 index 2c79aec59830c59aeedf98843038f610ae717953..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/memtable_flush.go +++ /dev/null @@ -1,246 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "bufio" - "fmt" - "io" - "os" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv/segmentindex" -) - -func (m *Memtable) flush() error { - // close the commit log first, this also forces it to be fsynced. If - // something fails there, don't proceed with flushing. The commit log will - // only be deleted at the very end, if the flush was successful - // (indicated by a successful close of the flush file - which indicates a - // successful fsync) - - if err := m.commitlog.close(); err != nil { - return errors.Wrap(err, "close commit log file") - } - - if m.Size() == 0 { - // this is an empty memtable, nothing to do - // however, we still have to cleanup the commit log, otherwise we will - // attempt to recover from it on the next cycle - if err := m.commitlog.delete(); err != nil { - return errors.Wrap(err, "delete commit log file") - } - return nil - } - - f, err := os.Create(m.path + ".db") - if err != nil { - return err - } - - w := bufio.NewWriterSize(f, int(float64(m.size)*1.3)) // calculate 30% overhead for disk representation - - var keys []segmentindex.Key - switch m.strategy { - case StrategyReplace: - if keys, err = m.flushDataReplace(w); err != nil { - return err - } - - case StrategySetCollection: - if keys, err = m.flushDataSet(w); err != nil { - return err - } - - case StrategyRoaringSet: - if keys, err = m.flushDataRoaringSet(w); err != nil { - return err - } - - case StrategyMapCollection: - if keys, err = m.flushDataMap(w); err != nil { - return err - } - - default: - return fmt.Errorf("cannot flush strategy %s", m.strategy) - } - - indices := &segmentindex.Indexes{ - Keys: keys, - SecondaryIndexCount: m.secondaryIndices, - ScratchSpacePath: m.path + ".scratch.d", - } - - if _, err := indices.WriteTo(w); err != nil { - return err - } - - if err := w.Flush(); err != nil { - return err - } - - if err := f.Close(); err != nil { - return err - } - - // only now that the file has been flushed is it safe to delete the commit log - // TODO: there might be an interest in keeping the commit logs around for - // longer as they might come in handy for replication - return m.commitlog.delete() -} - -func (m *Memtable) flushDataReplace(f io.Writer) ([]segmentindex.Key, error) { - flat := m.key.flattenInOrder() - - totalDataLength := totalKeyAndValueSize(flat) - perObjectAdditions := len(flat) * (1 + 8 + 4 + int(m.secondaryIndices)*4) // 1 byte for the tombstone, 8 bytes value length encoding, 4 bytes key length encoding, + 4 bytes key encoding for every secondary index - headerSize := segmentindex.HeaderSize - header := segmentindex.Header{ - IndexStart: uint64(totalDataLength + perObjectAdditions + headerSize), - Level: 0, // always level zero on a new one - Version: 0, // always version 0 for now - SecondaryIndices: m.secondaryIndices, - Strategy: SegmentStrategyFromString(m.strategy), - } - - n, err := header.WriteTo(f) - if err != nil { - return nil, err - } - headerSize = int(n) - keys := make([]segmentindex.Key, len(flat)) - - totalWritten := headerSize - for i, node := range flat { - segNode := &segmentReplaceNode{ - offset: totalWritten, - tombstone: node.tombstone, - value: node.value, - primaryKey: node.key, - secondaryKeys: node.secondaryKeys, - secondaryIndexCount: m.secondaryIndices, - } - - ki, err := segNode.KeyIndexAndWriteTo(f) - if err != nil { - return nil, errors.Wrapf(err, "write node %d", i) - } - - keys[i] = ki - totalWritten = ki.ValueEnd - } - - return keys, nil -} - -func (m *Memtable) flushDataSet(f io.Writer) ([]segmentindex.Key, error) { - flat := m.keyMulti.flattenInOrder() - return m.flushDataCollection(f, flat) -} - -func (m *Memtable) flushDataMap(f io.Writer) ([]segmentindex.Key, error) { - m.RLock() - flat := m.keyMap.flattenInOrder() - m.RUnlock() - - // by encoding each map pair we can force the same structure as for a - // collection, which means we can reuse the same flushing logic - asMulti := make([]*binarySearchNodeMulti, len(flat)) - for i, mapNode := range flat { - asMulti[i] = &binarySearchNodeMulti{ - key: mapNode.key, - values: make([]value, len(mapNode.values)), - } - - for j := range asMulti[i].values { - enc, err := mapNode.values[j].Bytes() - if err != nil { - return nil, err - } - - asMulti[i].values[j] = value{ - value: enc, - tombstone: mapNode.values[j].Tombstone, - } - } - - } - return m.flushDataCollection(f, asMulti) -} - -func (m *Memtable) flushDataCollection(f io.Writer, - flat []*binarySearchNodeMulti, -) ([]segmentindex.Key, error) { - totalDataLength := totalValueSizeCollection(flat) - header := segmentindex.Header{ - IndexStart: uint64(totalDataLength + segmentindex.HeaderSize), - Level: 0, // always level zero on a new one - Version: 0, // always version 0 for now - SecondaryIndices: m.secondaryIndices, - Strategy: SegmentStrategyFromString(m.strategy), - } - - n, err := header.WriteTo(f) - if err != nil { - return nil, err - } - headerSize := int(n) - keys := make([]segmentindex.Key, len(flat)) - - totalWritten := headerSize - for i, node := range flat { - ki, err := (&segmentCollectionNode{ - values: node.values, - primaryKey: node.key, - offset: totalWritten, - }).KeyIndexAndWriteTo(f) - if err != nil { - return nil, errors.Wrapf(err, "write node %d", i) - } - - keys[i] = ki - totalWritten = ki.ValueEnd - } - - return keys, nil -} - -func totalKeyAndValueSize(in []*binarySearchNode) int { - var sum int - for _, n := range in { - sum += len(n.value) - sum += len(n.key) - for _, sec := range n.secondaryKeys { - sum += len(sec) - } - } - - return sum -} - -func totalValueSizeCollection(in []*binarySearchNodeMulti) int { - var sum int - for _, n := range in { - sum += 8 // uint64 to indicate array length - for _, v := range n.values { - sum += 1 // bool to indicate value tombstone - sum += 8 // uint64 to indicate value length - sum += len(v.value) - } - - sum += 4 // uint32 to indicate key size - sum += len(n.key) - } - - return sum -} diff --git a/adapters/repos/db/lsmkv/memtable_flush_roaring_set.go b/adapters/repos/db/lsmkv/memtable_flush_roaring_set.go deleted file mode 100644 index a9f8500312b0d4ae7f7b01c391f32a317c8fe21b..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/memtable_flush_roaring_set.go +++ /dev/null @@ -1,74 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "fmt" - "io" - - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv/roaringset" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv/segmentindex" -) - -func (m *Memtable) flushDataRoaringSet(f io.Writer) ([]segmentindex.Key, error) { - flat := m.roaringSet.FlattenInOrder() - - totalDataLength := totalPayloadSizeRoaringSet(flat) - header := segmentindex.Header{ - IndexStart: uint64(totalDataLength + segmentindex.HeaderSize), - Level: 0, // always level zero on a new one - Version: 0, // always version 0 for now - SecondaryIndices: 0, - Strategy: segmentindex.StrategyRoaringSet, - } - - n, err := header.WriteTo(f) - if err != nil { - return nil, err - } - headerSize := int(n) - keys := make([]segmentindex.Key, len(flat)) - - totalWritten := headerSize - for i, node := range flat { - sn, err := roaringset.NewSegmentNode(node.Key, node.Value.Additions, - node.Value.Deletions) - if err != nil { - return nil, fmt.Errorf("create segment node: %w", err) - } - - ki, err := sn.KeyIndexAndWriteTo(f, totalWritten) - if err != nil { - return nil, fmt.Errorf("write node %d: %w", i, err) - } - - keys[i] = ki - totalWritten = ki.ValueEnd - } - - return keys, nil -} - -func totalPayloadSizeRoaringSet(in []*roaringset.BinarySearchNode) int { - var sum int - for _, n := range in { - sum += 8 // uint64 to segment length - sum += 8 // uint64 to indicate length of additions bitmap - sum += len(n.Value.Additions.ToBuffer()) - sum += 8 // uint64 to indicate length of deletions bitmap - sum += len(n.Value.Deletions.ToBuffer()) - sum += 4 // uint32 to indicate key size - sum += len(n.Key) - } - - return sum -} diff --git a/adapters/repos/db/lsmkv/memtable_metrics.go b/adapters/repos/db/lsmkv/memtable_metrics.go deleted file mode 100644 index d96f4d13731a36a725d823bc58b6444004805bc3..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/memtable_metrics.go +++ /dev/null @@ -1,41 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -type memtableMetrics struct { - put NsObserver - setTombstone NsObserver - append NsObserver - appendMapSorted NsObserver - get NsObserver - getBySecondary NsObserver - getMap NsObserver - getCollection NsObserver - size Setter -} - -// newMemtableMetrics curries the prometheus-functions just once to make sure -// they don't have to be curried on the hotpath where we this would lead to a -// lot of allocations. -func newMemtableMetrics(metrics *Metrics, path, strategy string) *memtableMetrics { - return &memtableMetrics{ - put: metrics.MemtableOpObserver(path, strategy, "put"), - setTombstone: metrics.MemtableOpObserver(path, strategy, "setTombstone"), - append: metrics.MemtableOpObserver(path, strategy, "append"), - appendMapSorted: metrics.MemtableOpObserver(path, strategy, "appendMapSorted"), - get: metrics.MemtableOpObserver(path, strategy, "get"), - getBySecondary: metrics.MemtableOpObserver(path, strategy, "getBySecondary"), - getMap: metrics.MemtableOpObserver(path, strategy, "getMap"), - getCollection: metrics.MemtableOpObserver(path, strategy, "getCollection"), - size: metrics.MemtableSizeSetter(path, strategy), - } -} diff --git a/adapters/repos/db/lsmkv/memtable_roaring_set.go b/adapters/repos/db/lsmkv/memtable_roaring_set.go deleted file mode 100644 index 04801a5b2aded6ad42aa83401918cc1c94a6281a..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/memtable_roaring_set.go +++ /dev/null @@ -1,150 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "time" - - "github.com/pkg/errors" - "github.com/weaviate/sroar" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv/roaringset" -) - -func (m *Memtable) roaringSetAddOne(key []byte, value uint64) error { - return m.roaringSetAddList(key, []uint64{value}) -} - -func (m *Memtable) roaringSetAddList(key []byte, values []uint64) error { - if err := checkStrategyRoaringSet(m.strategy); err != nil { - return err - } - - m.Lock() - defer m.Unlock() - - if err := m.roaringSetAddCommitLog(key, roaringset.NewBitmap(values...), roaringset.NewBitmap()); err != nil { - return err - } - - m.roaringSet.Insert(key, roaringset.Insert{Additions: values}) - - m.roaringSetAdjustMeta(len(values)) - return nil -} - -func (m *Memtable) roaringSetAddBitmap(key []byte, bm *sroar.Bitmap) error { - if err := checkStrategyRoaringSet(m.strategy); err != nil { - return err - } - - m.Lock() - defer m.Unlock() - - if err := m.roaringSetAddCommitLog(key, bm, roaringset.NewBitmap()); err != nil { - return err - } - - m.roaringSet.Insert(key, roaringset.Insert{Additions: bm.ToArray()}) - - m.roaringSetAdjustMeta(bm.GetCardinality()) - return nil -} - -func (m *Memtable) roaringSetRemoveOne(key []byte, value uint64) error { - return m.roaringSetRemoveList(key, []uint64{value}) -} - -func (m *Memtable) roaringSetRemoveList(key []byte, values []uint64) error { - if err := checkStrategyRoaringSet(m.strategy); err != nil { - return err - } - - m.Lock() - defer m.Unlock() - - if err := m.roaringSetAddCommitLog(key, roaringset.NewBitmap(), roaringset.NewBitmap(values...)); err != nil { - return err - } - - m.roaringSet.Insert(key, roaringset.Insert{Deletions: values}) - - m.roaringSetAdjustMeta(len(values)) - return nil -} - -func (m *Memtable) roaringSetRemoveBitmap(key []byte, bm *sroar.Bitmap) error { - if err := checkStrategyRoaringSet(m.strategy); err != nil { - return err - } - - m.Lock() - defer m.Unlock() - - if err := m.roaringSetAddCommitLog(key, roaringset.NewBitmap(), bm); err != nil { - return err - } - - m.roaringSet.Insert(key, roaringset.Insert{Deletions: bm.ToArray()}) - - m.roaringSetAdjustMeta(bm.GetCardinality()) - return nil -} - -func (m *Memtable) roaringSetAddRemoveBitmaps(key []byte, additions *sroar.Bitmap, deletions *sroar.Bitmap) error { - if err := checkStrategyRoaringSet(m.strategy); err != nil { - return err - } - - m.Lock() - defer m.Unlock() - - if err := m.roaringSetAddCommitLog(key, additions, deletions); err != nil { - return err - } - - m.roaringSet.Insert(key, roaringset.Insert{ - Additions: additions.ToArray(), - Deletions: deletions.ToArray(), - }) - - m.roaringSetAdjustMeta(additions.GetCardinality() + deletions.GetCardinality()) - return nil -} - -func (m *Memtable) roaringSetGet(key []byte) (roaringset.BitmapLayer, error) { - if err := checkStrategyRoaringSet(m.strategy); err != nil { - return roaringset.BitmapLayer{}, err - } - - m.RLock() - defer m.RUnlock() - - return m.roaringSet.Get(key) -} - -func (m *Memtable) roaringSetAdjustMeta(entriesChanged int) { - // in the worst case roaring bitmaps take 2 bytes per entry. A reasonable - // estimation is therefore to take the changed entries and multiply them by - // 2. - m.size += uint64(entriesChanged * 2) - m.metrics.size(m.size) - m.lastWrite = time.Now() -} - -func (m *Memtable) roaringSetAddCommitLog(key []byte, additions *sroar.Bitmap, deletions *sroar.Bitmap) error { - if node, err := roaringset.NewSegmentNode(key, additions, deletions); err != nil { - return errors.Wrap(err, "create node for commit log") - } else if err := m.commitlog.add(node); err != nil { - return errors.Wrap(err, "add node to commit log") - } - return nil -} diff --git a/adapters/repos/db/lsmkv/memtable_roaring_set_test.go b/adapters/repos/db/lsmkv/memtable_roaring_set_test.go deleted file mode 100644 index 4178b64b5925948d523e53f5a2e04edb976827ba..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/memtable_roaring_set_test.go +++ /dev/null @@ -1,222 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "path" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv/roaringset" -) - -func TestMemtableRoaringSet(t *testing.T) { - memPath := func() string { - return path.Join(t.TempDir(), "fake") - } - - t.Run("inserting individual entries", func(t *testing.T) { - m, err := newMemtable(memPath(), StrategyRoaringSet, 0, nil) - require.Nil(t, err) - - key1, key2 := []byte("key1"), []byte("key2") - - assert.Nil(t, m.roaringSetAddOne(key1, 1)) - assert.Nil(t, m.roaringSetAddOne(key1, 2)) - assert.Nil(t, m.roaringSetAddOne(key2, 3)) - assert.Nil(t, m.roaringSetAddOne(key2, 4)) - assert.Greater(t, m.Size(), uint64(0)) - - setKey1, err := m.roaringSetGet(key1) - require.Nil(t, err) - assert.True(t, setKey1.Additions.Contains(1)) - assert.True(t, setKey1.Additions.Contains(2)) - assert.False(t, setKey1.Additions.Contains(3)) - assert.False(t, setKey1.Additions.Contains(4)) - - setKey2, err := m.roaringSetGet(key2) - require.Nil(t, err) - assert.False(t, setKey2.Additions.Contains(1)) - assert.False(t, setKey2.Additions.Contains(2)) - assert.True(t, setKey2.Additions.Contains(3)) - assert.True(t, setKey2.Additions.Contains(4)) - - require.Nil(t, m.commitlog.close()) - }) - - t.Run("inserting lists", func(t *testing.T) { - m, err := newMemtable(memPath(), StrategyRoaringSet, 0, nil) - require.Nil(t, err) - - key1, key2 := []byte("key1"), []byte("key2") - - assert.Nil(t, m.roaringSetAddList(key1, []uint64{1, 2})) - assert.Nil(t, m.roaringSetAddList(key2, []uint64{3, 4})) - assert.Greater(t, m.Size(), uint64(0)) - - setKey1, err := m.roaringSetGet(key1) - require.Nil(t, err) - assert.True(t, setKey1.Additions.Contains(1)) - assert.True(t, setKey1.Additions.Contains(2)) - assert.False(t, setKey1.Additions.Contains(3)) - assert.False(t, setKey1.Additions.Contains(4)) - - setKey2, err := m.roaringSetGet(key2) - require.Nil(t, err) - assert.False(t, setKey2.Additions.Contains(1)) - assert.False(t, setKey2.Additions.Contains(2)) - assert.True(t, setKey2.Additions.Contains(3)) - assert.True(t, setKey2.Additions.Contains(4)) - - require.Nil(t, m.commitlog.close()) - }) - - t.Run("inserting bitmaps", func(t *testing.T) { - m, err := newMemtable(memPath(), StrategyRoaringSet, 0, nil) - require.Nil(t, err) - - key1, key2 := []byte("key1"), []byte("key2") - - bm1 := roaringset.NewBitmap(1, 2) - assert.Nil(t, m.roaringSetAddBitmap(key1, bm1)) - bm2 := roaringset.NewBitmap(3, 4) - assert.Nil(t, m.roaringSetAddBitmap(key2, bm2)) - assert.Greater(t, m.Size(), uint64(0)) - - setKey1, err := m.roaringSetGet(key1) - require.Nil(t, err) - assert.True(t, setKey1.Additions.Contains(1)) - assert.True(t, setKey1.Additions.Contains(2)) - assert.False(t, setKey1.Additions.Contains(3)) - assert.False(t, setKey1.Additions.Contains(4)) - - setKey2, err := m.roaringSetGet(key2) - require.Nil(t, err) - assert.False(t, setKey2.Additions.Contains(1)) - assert.False(t, setKey2.Additions.Contains(2)) - assert.True(t, setKey2.Additions.Contains(3)) - assert.True(t, setKey2.Additions.Contains(4)) - - require.Nil(t, m.commitlog.close()) - }) - - t.Run("removing individual entries", func(t *testing.T) { - m, err := newMemtable(memPath(), StrategyRoaringSet, 0, nil) - require.Nil(t, err) - - key1, key2 := []byte("key1"), []byte("key2") - - assert.Nil(t, m.roaringSetRemoveOne(key1, 7)) - assert.Nil(t, m.roaringSetRemoveOne(key2, 8)) - assert.Greater(t, m.Size(), uint64(0)) - - setKey1, err := m.roaringSetGet(key1) - require.Nil(t, err) - assert.False(t, setKey1.Additions.Contains(7)) - assert.True(t, setKey1.Deletions.Contains(7)) - - setKey2, err := m.roaringSetGet(key2) - require.Nil(t, err) - assert.False(t, setKey2.Additions.Contains(8)) - assert.True(t, setKey2.Deletions.Contains(8)) - - require.Nil(t, m.commitlog.close()) - }) - - t.Run("removing lists", func(t *testing.T) { - m, err := newMemtable(memPath(), StrategyRoaringSet, 0, nil) - require.Nil(t, err) - - key1, key2 := []byte("key1"), []byte("key2") - - assert.Nil(t, m.roaringSetRemoveList(key1, []uint64{7, 8})) - assert.Nil(t, m.roaringSetRemoveList(key2, []uint64{9, 10})) - assert.Greater(t, m.Size(), uint64(0)) - - setKey1, err := m.roaringSetGet(key1) - require.Nil(t, err) - assert.Equal(t, 0, setKey1.Additions.GetCardinality()) - assert.Equal(t, 2, setKey1.Deletions.GetCardinality()) - assert.True(t, setKey1.Deletions.Contains(7)) - assert.True(t, setKey1.Deletions.Contains(8)) - - setKey2, err := m.roaringSetGet(key2) - require.Nil(t, err) - assert.Equal(t, 0, setKey2.Additions.GetCardinality()) - assert.Equal(t, 2, setKey2.Deletions.GetCardinality()) - assert.True(t, setKey2.Deletions.Contains(9)) - assert.True(t, setKey2.Deletions.Contains(10)) - - require.Nil(t, m.commitlog.close()) - }) - - t.Run("removing bitmaps", func(t *testing.T) { - m, err := newMemtable(memPath(), StrategyRoaringSet, 0, nil) - require.Nil(t, err) - - key1, key2 := []byte("key1"), []byte("key2") - - assert.Nil(t, m.roaringSetRemoveBitmap(key1, roaringset.NewBitmap(7, 8))) - assert.Nil(t, m.roaringSetRemoveBitmap(key2, roaringset.NewBitmap(9, 10))) - assert.Greater(t, m.Size(), uint64(0)) - - setKey1, err := m.roaringSetGet(key1) - require.Nil(t, err) - assert.Equal(t, 0, setKey1.Additions.GetCardinality()) - assert.Equal(t, 2, setKey1.Deletions.GetCardinality()) - assert.True(t, setKey1.Deletions.Contains(7)) - assert.True(t, setKey1.Deletions.Contains(8)) - - setKey2, err := m.roaringSetGet(key2) - require.Nil(t, err) - assert.Equal(t, 0, setKey2.Additions.GetCardinality()) - assert.Equal(t, 2, setKey2.Deletions.GetCardinality()) - assert.True(t, setKey2.Deletions.Contains(9)) - assert.True(t, setKey2.Deletions.Contains(10)) - - require.Nil(t, m.commitlog.close()) - }) - - t.Run("adding/removing bitmaps", func(t *testing.T) { - m, err := newMemtable(memPath(), StrategyRoaringSet, 0, nil) - require.Nil(t, err) - - key1, key2 := []byte("key1"), []byte("key2") - - assert.Nil(t, m.roaringSetAddRemoveBitmaps(key1, - roaringset.NewBitmap(1, 2), roaringset.NewBitmap(7, 8))) - assert.Nil(t, m.roaringSetAddRemoveBitmaps(key2, - roaringset.NewBitmap(3, 4), roaringset.NewBitmap(9, 10))) - assert.Greater(t, m.Size(), uint64(0)) - - setKey1, err := m.roaringSetGet(key1) - require.Nil(t, err) - assert.Equal(t, 2, setKey1.Additions.GetCardinality()) - assert.True(t, setKey1.Additions.Contains(1)) - assert.True(t, setKey1.Additions.Contains(2)) - assert.Equal(t, 2, setKey1.Deletions.GetCardinality()) - assert.True(t, setKey1.Deletions.Contains(7)) - assert.True(t, setKey1.Deletions.Contains(8)) - - setKey2, err := m.roaringSetGet(key2) - require.Nil(t, err) - assert.Equal(t, 2, setKey2.Additions.GetCardinality()) - assert.True(t, setKey2.Additions.Contains(3)) - assert.True(t, setKey2.Additions.Contains(4)) - assert.Equal(t, 2, setKey2.Deletions.GetCardinality()) - assert.True(t, setKey2.Deletions.Contains(9)) - assert.True(t, setKey2.Deletions.Contains(10)) - - require.Nil(t, m.commitlog.close()) - }) -} diff --git a/adapters/repos/db/lsmkv/memtable_size_advisor.go b/adapters/repos/db/lsmkv/memtable_size_advisor.go deleted file mode 100644 index ea2d1871bbdcacc43663eed80382f1ac5905b956..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/memtable_size_advisor.go +++ /dev/null @@ -1,87 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import "time" - -// if not enough config is provided we can fall back to this reasonable default -// value -const reasonableMemtableDefault = 10 * 1024 * 1024 - -type memtableSizeAdvisorCfg struct { - initial int - stepSize int - maxSize int - minDuration time.Duration - maxDuration time.Duration -} - -type memtableSizeAdvisor struct { - cfg memtableSizeAdvisorCfg - active bool -} - -func newMemtableSizeAdvisor(cfg memtableSizeAdvisorCfg) *memtableSizeAdvisor { - a := &memtableSizeAdvisor{ - cfg: cfg, - } - - // only activate if initial size, step size, max size, and max duration are - // given - if a.cfg.maxSize > 0 && a.cfg.initial > 0 && a.cfg.stepSize > 0 && a.cfg.maxDuration > 0 { - a.active = true - } - - return a -} - -func (m memtableSizeAdvisor) Initial() int { - if m.active { - return m.cfg.initial - } else { - return reasonableMemtableDefault - } -} - -func (m memtableSizeAdvisor) NextTarget(previousTarget int, - timeSinceFlush time.Duration, -) (int, bool) { - if !m.active { - return reasonableMemtableDefault, false - } - - if timeSinceFlush < m.cfg.minDuration { - next := min(previousTarget+m.cfg.stepSize, m.cfg.maxSize) - return next, next != previousTarget - } - if timeSinceFlush > m.cfg.maxDuration { - next := max(previousTarget-m.cfg.stepSize, m.cfg.initial) - return next, next != previousTarget - } - return previousTarget, false -} - -func min(a, b int) int { - if a <= b { - return a - } - - return b -} - -func max(a, b int) int { - if a >= b { - return a - } - - return b -} diff --git a/adapters/repos/db/lsmkv/memtable_size_advisor_test.go b/adapters/repos/db/lsmkv/memtable_size_advisor_test.go deleted file mode 100644 index 7bf2ba88901dffcb442d2cf78fdea99340daeda3..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/memtable_size_advisor_test.go +++ /dev/null @@ -1,119 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -const Megabyte = 1024 * 1024 - -func TestMemtableSizeAdvisor_Initial(t *testing.T) { - a := newMemtableSizeAdvisor(memtableSizeAdvisorCfg{ - initial: 10 * Megabyte, - }) - - assert.Equal(t, 10485760, a.Initial()) -} - -func TestMemtableSizeAdvisor_NextTarget(t *testing.T) { - a := newMemtableSizeAdvisor(memtableSizeAdvisorCfg{ - initial: 10 * Megabyte, - minDuration: 10 * time.Second, - maxDuration: 30 * time.Second, - stepSize: 10 * Megabyte, - maxSize: 100 * Megabyte, - }) - - type test struct { - name string - current int - lastCycle time.Duration - expectedChanged bool - expectedTarget int - } - - tests := []test{ - { - name: "completely within range", - current: 10 * Megabyte, - lastCycle: 17 * time.Second, - expectedChanged: false, - expectedTarget: 10 * Megabyte, - }, - { - name: "cycle too short", - current: 10 * Megabyte, - lastCycle: 7 * time.Second, - expectedChanged: true, - expectedTarget: 20 * Megabyte, - }, - { - name: "cycle too long", - current: 100 * Megabyte, - lastCycle: 47 * time.Second, - expectedChanged: true, - expectedTarget: 90 * Megabyte, - }, - { - name: "cycle too short, but approaching limit", - current: 95 * Megabyte, - lastCycle: 7 * time.Second, - expectedChanged: true, - expectedTarget: 100 * Megabyte, // not 105 (!) - }, - { - name: "cycle too short, but already at limit", - current: 100 * Megabyte, - lastCycle: 7 * time.Second, - expectedChanged: false, - expectedTarget: 100 * Megabyte, - }, - { - name: "cycle too long, but barely above initial size", - current: 12 * Megabyte, - lastCycle: 47 * time.Second, - expectedChanged: true, - expectedTarget: 10 * Megabyte, // not 2 (1) - }, - { - name: "cycle too long, but already at initial size", - current: 10 * Megabyte, - lastCycle: 47 * time.Second, - expectedChanged: false, - expectedTarget: 10 * Megabyte, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - newTarget, changed := a.NextTarget(test.current, test.lastCycle) - assert.Equal(t, test.expectedTarget, newTarget, "expect new target") - assert.Equal(t, test.expectedChanged, changed, "expect changed") - }) - } - - target, changed := a.NextTarget(10*1024*1024, 17*time.Second) - assert.False(t, changed) - assert.Equal(t, 10*1024*1024, target) -} - -func TestMemtableSizeAdvisor_MissingConfig(t *testing.T) { - // even with an all-default value config the advisor should still return - // reasonable results, for example many integration tests might not provide a - // reasonable config to the advisor - a := newMemtableSizeAdvisor(memtableSizeAdvisorCfg{}) - assert.Equal(t, 10485760, a.Initial()) -} diff --git a/adapters/repos/db/lsmkv/memtable_test.go b/adapters/repos/db/lsmkv/memtable_test.go deleted file mode 100644 index ada1f507485fd93d6a7a8c1fcff4493636ac510d..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/memtable_test.go +++ /dev/null @@ -1,74 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "path" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/lsmkv" -) - -// This test prevents a regression on -// https://www.youtube.com/watch?v=OS8taasZl8k -func Test_MemtableSecondaryKeyBug(t *testing.T) { - dir := t.TempDir() - m, err := newMemtable(path.Join(dir, "will-never-flush"), StrategyReplace, 1, nil) - require.Nil(t, err) - t.Cleanup(func() { - require.Nil(t, m.commitlog.close()) - }) - - t.Run("add initial value", func(t *testing.T) { - err = m.put([]byte("my-key"), []byte("my-value"), - WithSecondaryKey(0, []byte("secondary-key-initial"))) - require.Nil(t, err) - }) - - t.Run("retrieve by primary", func(t *testing.T) { - val, err := m.get([]byte("my-key")) - require.Nil(t, err) - assert.Equal(t, []byte("my-value"), val) - }) - - t.Run("retrieve by initial secondary", func(t *testing.T) { - val, err := m.getBySecondary(0, []byte("secondary-key-initial")) - require.Nil(t, err) - assert.Equal(t, []byte("my-value"), val) - }) - - t.Run("update value with different secondary key", func(t *testing.T) { - err = m.put([]byte("my-key"), []byte("my-value-updated"), - WithSecondaryKey(0, []byte("different-secondary-key"))) - require.Nil(t, err) - }) - - t.Run("retrieve by primary again", func(t *testing.T) { - val, err := m.get([]byte("my-key")) - require.Nil(t, err) - assert.Equal(t, []byte("my-value-updated"), val) - }) - - t.Run("retrieve by updated secondary", func(t *testing.T) { - val, err := m.getBySecondary(0, []byte("different-secondary-key")) - require.Nil(t, err) - assert.Equal(t, []byte("my-value-updated"), val) - }) - - t.Run("retrieve by initial secondary - should not find anything", func(t *testing.T) { - val, err := m.getBySecondary(0, []byte("secondary-key-initial")) - assert.Equal(t, lsmkv.NotFound, err) - assert.Nil(t, val) - }) -} diff --git a/adapters/repos/db/lsmkv/metrics.go b/adapters/repos/db/lsmkv/metrics.go deleted file mode 100644 index f524a93ea1136fedab846fb1a4106842759014e3..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/metrics.go +++ /dev/null @@ -1,231 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "time" - - "github.com/prometheus/client_golang/prometheus" - "github.com/weaviate/weaviate/usecases/monitoring" -) - -type ( - NsObserver func(ns int64) - Setter func(val uint64) - TimeObserver func(start time.Time) -) - -type Metrics struct { - CompactionReplace *prometheus.GaugeVec - CompactionSet *prometheus.GaugeVec - CompactionMap *prometheus.GaugeVec - CompactionRoaringSet *prometheus.GaugeVec - ActiveSegments *prometheus.GaugeVec - bloomFilters prometheus.ObserverVec - SegmentObjects *prometheus.GaugeVec - SegmentSize *prometheus.GaugeVec - SegmentCount *prometheus.GaugeVec - startupDurations prometheus.ObserverVec - startupDiskIO prometheus.ObserverVec - objectCount prometheus.Gauge - memtableDurations prometheus.ObserverVec - memtableSize *prometheus.GaugeVec - DimensionSum *prometheus.GaugeVec - - groupClasses bool -} - -func NewMetrics(promMetrics *monitoring.PrometheusMetrics, className, - shardName string, -) *Metrics { - if promMetrics.Group { - className = "n/a" - shardName = "n/a" - } - - replace := promMetrics.AsyncOperations.MustCurryWith(prometheus.Labels{ - "operation": "compact_lsm_segments_stratreplace", - "class_name": className, - "shard_name": shardName, - }) - - set := promMetrics.AsyncOperations.MustCurryWith(prometheus.Labels{ - "operation": "compact_lsm_segments_stratset", - "class_name": className, - "shard_name": shardName, - }) - - roaringSet := promMetrics.AsyncOperations.MustCurryWith(prometheus.Labels{ - "operation": "compact_lsm_segments_stratroaringset", - "class_name": className, - "shard_name": shardName, - }) - - stratMap := promMetrics.AsyncOperations.MustCurryWith(prometheus.Labels{ - "operation": "compact_lsm_segments_stratmap", - "class_name": className, - "shard_name": shardName, - }) - - return &Metrics{ - groupClasses: promMetrics.Group, - CompactionReplace: replace, - CompactionSet: set, - CompactionMap: stratMap, - CompactionRoaringSet: roaringSet, - ActiveSegments: promMetrics.LSMSegmentCount.MustCurryWith(prometheus.Labels{ - "class_name": className, - "shard_name": shardName, - }), - bloomFilters: promMetrics.LSMBloomFilters.MustCurryWith(prometheus.Labels{ - "class_name": className, - "shard_name": shardName, - }), - SegmentObjects: promMetrics.LSMSegmentObjects.MustCurryWith(prometheus.Labels{ - "class_name": className, - "shard_name": shardName, - }), - SegmentSize: promMetrics.LSMSegmentSize.MustCurryWith(prometheus.Labels{ - "class_name": className, - "shard_name": shardName, - }), - SegmentCount: promMetrics.LSMSegmentCountByLevel.MustCurryWith(prometheus.Labels{ - "class_name": className, - "shard_name": shardName, - }), - startupDiskIO: promMetrics.StartupDiskIO.MustCurryWith(prometheus.Labels{ - "class_name": className, - "shard_name": shardName, - }), - startupDurations: promMetrics.StartupDurations.MustCurryWith(prometheus.Labels{ - "class_name": className, - "shard_name": shardName, - }), - objectCount: promMetrics.ObjectCount.With(prometheus.Labels{ - "class_name": className, - "shard_name": shardName, - }), - memtableDurations: promMetrics.LSMMemtableDurations.MustCurryWith(prometheus.Labels{ - "class_name": className, - "shard_name": shardName, - }), - memtableSize: promMetrics.LSMMemtableSize.MustCurryWith(prometheus.Labels{ - "class_name": className, - "shard_name": shardName, - }), - DimensionSum: promMetrics.VectorDimensionsSum.MustCurryWith(prometheus.Labels{ - "class_name": className, - "shard_name": shardName, - }), - } -} - -func noOpTimeObserver(start time.Time) { - // do nothing -} - -func noOpNsObserver(startNs int64) { - // do nothing -} - -func noOpSetter(val uint64) { - // do nothing -} - -func (m *Metrics) MemtableOpObserver(path, strategy, op string) NsObserver { - if m == nil { - return noOpNsObserver - } - - if m.groupClasses { - path = "n/a" - } - - curried := m.memtableDurations.With(prometheus.Labels{ - "operation": op, - "path": path, - "strategy": strategy, - }) - - return func(startNs int64) { - took := float64(time.Now().UnixNano()-startNs) / float64(time.Millisecond) - curried.Observe(took) - } -} - -func (m *Metrics) MemtableSizeSetter(path, strategy string) Setter { - if m == nil || m.groupClasses { - // this metric would set absolute values, that's not possible in - // grouped mode, each call would essentially overwrite the last - return noOpSetter - } - - curried := m.memtableSize.With(prometheus.Labels{ - "path": path, - "strategy": strategy, - }) - - return func(size uint64) { - curried.Set(float64(size)) - } -} - -func (m *Metrics) BloomFilterObserver(strategy, operation string) TimeObserver { - if m == nil { - return noOpTimeObserver - } - - curried := m.bloomFilters.With(prometheus.Labels{ - "strategy": strategy, - "operation": operation, - }) - - return func(before time.Time) { - curried.Observe(float64(time.Since(before)) / float64(time.Millisecond)) - } -} - -func (m *Metrics) TrackStartupReadWALDiskIO(read int64, nanoseconds int64) { - if m == nil { - return - } - - seconds := float64(nanoseconds) / float64(time.Second) - throughput := float64(read) / float64(seconds) - m.startupDiskIO.With(prometheus.Labels{"operation": "lsm_recover_wal"}).Observe(throughput) -} - -func (m *Metrics) TrackStartupBucket(start time.Time) { - if m == nil { - return - } - - took := float64(time.Since(start)) / float64(time.Millisecond) - m.startupDurations.With(prometheus.Labels{"operation": "lsm_startup_bucket"}).Observe(took) -} - -func (m *Metrics) TrackStartupBucketRecovery(start time.Time) { - if m == nil { - return - } - - took := float64(time.Since(start)) / float64(time.Millisecond) - m.startupDurations.With(prometheus.Labels{"operation": "lsm_startup_bucket_recovery"}).Observe(took) -} - -func (m *Metrics) ObjectCount(count int) { - if m == nil { - return - } - - m.objectCount.Set(float64(count)) -} diff --git a/adapters/repos/db/lsmkv/rbtree/rbtree.go b/adapters/repos/db/lsmkv/rbtree/rbtree.go deleted file mode 100644 index 513270c46e4b230a69a25730ce02fd8f89ef2a1c..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/rbtree/rbtree.go +++ /dev/null @@ -1,176 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package rbtree - -type Node interface { - Parent() Node - SetParent(Node) - Left() Node - SetLeft(Node) - Right() Node - SetRight(Node) - IsRed() bool - SetRed(bool) - IsNil() bool -} - -// This function rebalances and recolours trees to be valid RB trees. It needs to be called after each node that -// was added to the tree. -// -// Deletions are currently not supported as this is done through the tombstone flag and from the POV of the RB-tree -// tombstone-nodes are just normal nodes that get rebalanced the normal way. -// -// Throughout this file the following relationships between nodes are used: -// GP = grandparent, P = parent, U = uncle, S = sibling, N = node that was just added -// -// GP -// / \ -// U P -// / \ -// S N -func Rebalance(node Node) Node { - for { - parent := node.Parent() - - // if parent is black or the current node is the root node (== parent is nil) there is nothing to do - if !parent.IsRed() { - return nil - } - - grandparent := node.Parent().Parent() - var uncle Node - if parent == grandparent.Right() { - uncle = grandparent.Left() - } else { - uncle = grandparent.Right() - } - - if uncle.IsRed() { - // if uncle is red, recoloring the tree up to the grandparent results in a valid RBtree. - // The color of the grandfather changes to red, so there might be more fixes needed. Therefore - // go up the tree and repeat. - recolourNodes(parent, grandparent, uncle) - node = grandparent - } else { - // if uncle is black, there are four possible cases: - // parent is the right child grandparent: - // 1) node is right child of parent => left rotate around GP - // 2) node is left child of parent => right rotate around parent results in case 1 - // For cases 3 and 4 just replace left and right in the two cases above - // - // In all of these cases the grandfather stays black and there is no need for further fixes up the tree - var newRoot Node - if parent == grandparent.Right() { - if node == parent.Left() { - rightRotate(parent) - // node and parent switch places in the tree, update parent to recolour the current node - parent = node - } - newRoot = leftRotate(grandparent) - } else { // parent == grandparent.left - if node == parent.Right() { - leftRotate(parent) - parent = node - } - newRoot = rightRotate(grandparent) - } - recolourNodes(grandparent, parent) - return newRoot - } - } -} - -func recolourNodes(nodes ...Node) { - for _, n := range nodes { - if !n.IsNil() { - if n.IsRed() { - n.SetRed(false) - } else { - n.SetRed(true) - } - } - } -} - -// Rotate the tree left around the given node. -// -// After this rotation, the former right child (FC) will be the new parent and the former parent (FP) will -// be the left node of the new parent. The left child of the former child is transferred to the former parent. -// -// FP FC -// / \ left rotate / \ -// FP_R FC => FP FC_R -// / \ / \ -// FC_L FC_R FP_R FC_L -// -// In case FP was the root of the tree, FC will be the new root of the tree. -func leftRotate(rotationNode Node) Node { - formerChild := rotationNode.Right() - rootRotate := rotationNode.Parent().IsNil() - - // former child node becomes new parent unless the rotation is around the root node - if rootRotate { - formerChild.SetParent(nil) - } else { - if rotationNode.Parent().Left() == rotationNode { - rotationNode.Parent().SetLeft(formerChild) - } else { - rotationNode.Parent().SetRight(formerChild) - } - formerChild.SetParent(rotationNode.Parent()) - } - - rotationNode.SetParent(formerChild) - - // Switch left child from former_child to rotation node - rotationNode.SetRight(formerChild.Left()) - if formerChild.Left() != nil { - formerChild.Left().SetParent(rotationNode) - } - formerChild.SetLeft(rotationNode) - - if rootRotate { - return formerChild - } else { - return nil - } -} - -// Same as leftRotate, just switch left and right everywhere -func rightRotate(rotationNode Node) Node { - formerChild := rotationNode.Left() - rootRotate := rotationNode.Parent().IsNil() - - if rootRotate { - formerChild.SetParent(nil) - } else { - if rotationNode.Parent().Left() == rotationNode { - rotationNode.Parent().SetLeft(formerChild) - } else { - rotationNode.Parent().SetRight(formerChild) - } - formerChild.SetParent(rotationNode.Parent()) - } - rotationNode.SetParent(formerChild) - - rotationNode.SetLeft(formerChild.Right()) - if formerChild.Right() != nil { - formerChild.Right().SetParent(rotationNode) - } - formerChild.SetRight(rotationNode) - - if rootRotate { - return formerChild - } else { - return nil - } -} diff --git a/adapters/repos/db/lsmkv/recover_from_wal_integration_test.go b/adapters/repos/db/lsmkv/recover_from_wal_integration_test.go deleted file mode 100644 index 8ec42e51cd91530f0e90682d0b9996acc12b3122..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/recover_from_wal_integration_test.go +++ /dev/null @@ -1,664 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package lsmkv - -import ( - "bytes" - "context" - "fmt" - "io" - "os" - "os/exec" - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/cyclemanager" -) - -func TestReplaceStrategy_RecoverFromWAL(t *testing.T) { - dirNameOriginal := t.TempDir() - dirNameRecovered := t.TempDir() - - t.Run("with some previous state", func(t *testing.T) { - b, err := NewBucket(testCtx(), dirNameOriginal, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - WithStrategy(StrategyReplace)) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - t.Run("set one key that will be flushed orderly", func(t *testing.T) { - // the motivation behind flushing this initial segment is to check that - // deletion as part of the recovery also works correctly. If we would - // just delete something that was created as part of the same memtable, - // the tests would still pass, even with removing the logic that recovers - // tombstones. - // - // To make sure they fail in this case, this prior state was introduced. - // An entry with key "key-2" is introduced in a previous segment, so if - // the deletion fails as part of the recovery this key would still be - // present later on. With the deletion working correctly it will be gone. - // - // You can test this by commenting the "p.memtable.setTombstone()" line - // in p.doReplace(). This will fail the tests suite, but prior to this - // addition it would have passed. - key2 := []byte("key-2") - orig2 := []byte("delete me later - you should never find me again") - - err = b.Put(key2, orig2) - require.Nil(t, err) - }) - - t.Run("shutdown (orderly) bucket to create first segment", func(t *testing.T) { - b.Shutdown(context.Background()) - - // then recreate bucket - var err error - b, err = NewBucket(testCtx(), dirNameOriginal, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - WithStrategy(StrategyReplace)) - require.Nil(t, err) - }) - - t.Run("set original values", func(t *testing.T) { - key1 := []byte("key-1") - key2 := []byte("key-2") - key3 := []byte("key-3") - orig1 := []byte("original value for key1") - orig2 := []byte("original value for key2") - orig3 := []byte("original value for key3") - - err = b.Put(key1, orig1) - require.Nil(t, err) - err = b.Put(key2, orig2) - require.Nil(t, err) - err = b.Put(key3, orig3) - require.Nil(t, err) - }) - - t.Run("delete one, update one", func(t *testing.T) { - key2 := []byte("key-2") - key3 := []byte("key-3") - updated3 := []byte("updated value for key 3") - - err = b.Delete(key2) - require.Nil(t, err) - - err = b.Put(key3, updated3) - require.Nil(t, err) - }) - - t.Run("verify control", func(t *testing.T) { - key1 := []byte("key-1") - key2 := []byte("key-2") - key3 := []byte("key-3") - orig1 := []byte("original value for key1") - updated3 := []byte("updated value for key 3") - res, err := b.Get(key1) - require.Nil(t, err) - assert.Equal(t, res, orig1) - res, err = b.Get(key2) - require.Nil(t, err) - assert.Nil(t, res) - res, err = b.Get(key3) - require.Nil(t, err) - assert.Equal(t, res, updated3) - }) - - t.Run("make sure the WAL is flushed", func(t *testing.T) { - require.Nil(t, b.WriteWAL()) - }) - - t.Run("copy state into recovery folder and destroy original", func(t *testing.T) { - t.Run("copy over wals", func(t *testing.T) { - cmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("cp -r %s/*.wal %s", - dirNameOriginal, dirNameRecovered)) - var out bytes.Buffer - cmd.Stderr = &out - err := cmd.Run() - if err != nil { - fmt.Println(out.String()) - t.Fatal(err) - } - }) - - t.Run("copy over segments", func(t *testing.T) { - cmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("cp -r %s/*.db %s", - dirNameOriginal, dirNameRecovered)) - var out bytes.Buffer - cmd.Stderr = &out - err := cmd.Run() - if err != nil { - fmt.Println(out.String()) - t.Fatal(err) - } - }) - b = nil - require.Nil(t, os.RemoveAll(dirNameOriginal)) - }) - - var bRec *Bucket - - t.Run("create new bucket from existing state", func(t *testing.T) { - b, err := NewBucket(testCtx(), dirNameRecovered, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - WithStrategy(StrategyReplace)) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - bRec = b - }) - - t.Run("verify all data is present", func(t *testing.T) { - key1 := []byte("key-1") - key2 := []byte("key-2") - key3 := []byte("key-3") - orig1 := []byte("original value for key1") - updated3 := []byte("updated value for key 3") - res, err := bRec.Get(key1) - require.Nil(t, err) - assert.Equal(t, res, orig1) - res, err = bRec.Get(key2) - require.Nil(t, err) - assert.Nil(t, res) - res, err = bRec.Get(key3) - require.Nil(t, err) - assert.Equal(t, res, updated3) - }) - }) -} - -func TestReplaceStrategy_RecoverFromWALWithCorruptLastElement(t *testing.T) { - dirNameOriginal := t.TempDir() - dirNameRecovered := t.TempDir() - - t.Run("without previous state", func(t *testing.T) { - b, err := NewBucket(testCtx(), dirNameOriginal, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - WithStrategy(StrategyReplace)) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - t.Run("set original values", func(t *testing.T) { - key1 := []byte("key-1") - key2 := []byte("key-2") - key3 := []byte("key-3") - orig1 := []byte("original value for key1") - orig2 := []byte("original value for key2") - orig3 := []byte("original value for key3") - - err = b.Put(key1, orig1) - require.Nil(t, err) - err = b.Put(key2, orig2) - require.Nil(t, err) - err = b.Put(key3, orig3) - require.Nil(t, err) - }) - - t.Run("delete one, update one", func(t *testing.T) { - key2 := []byte("key-2") - key3 := []byte("key-3") - updated3 := []byte("updated value for key 3") - - err = b.Delete(key2) - require.Nil(t, err) - - err = b.Put(key3, updated3) - require.Nil(t, err) - }) - - t.Run("verify control", func(t *testing.T) { - key1 := []byte("key-1") - key2 := []byte("key-2") - key3 := []byte("key-3") - orig1 := []byte("original value for key1") - updated3 := []byte("updated value for key 3") - res, err := b.Get(key1) - require.Nil(t, err) - assert.Equal(t, res, orig1) - res, err = b.Get(key2) - require.Nil(t, err) - assert.Nil(t, res) - res, err = b.Get(key3) - require.Nil(t, err) - assert.Equal(t, res, updated3) - }) - - t.Run("make sure the WAL is flushed", func(t *testing.T) { - require.Nil(t, b.WriteWAL()) - }) - - t.Run("copy state into recovery folder and destroy original", func(t *testing.T) { - cmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("cp -r %s/*.wal %s", - dirNameOriginal, dirNameRecovered)) - var out bytes.Buffer - cmd.Stderr = &out - err := cmd.Run() - if err != nil { - fmt.Println(out.String()) - t.Fatal(err) - } - b = nil - require.Nil(t, os.RemoveAll(dirNameOriginal)) - }) - - t.Run("corrupt WAL by removing some bytes at the very end", func(t *testing.T) { - entries, err := os.ReadDir(dirNameRecovered) - require.Nil(t, err) - require.Len(t, entries, 1, "there should be exactly one .wal file") - - oldFileName := filepath.Join(dirNameRecovered, entries[0].Name()) - tmpFileName := oldFileName + ".tmp" - - err = os.Rename(oldFileName, tmpFileName) - require.Nil(t, err) - - orig, err := os.Open(tmpFileName) - require.Nil(t, err) - - correctLog, err := io.ReadAll(orig) - require.Nil(t, err) - err = orig.Close() - require.Nil(t, err) - - corruptLog := correctLog[:len(correctLog)-6] - - err = os.Remove(tmpFileName) - require.Nil(t, err) - - corrupt, err := os.Create(oldFileName) - require.Nil(t, err) - - _, err = corrupt.Write(corruptLog) - require.Nil(t, err) - - err = corrupt.Close() - require.Nil(t, err) - }) - - var bRec *Bucket - - t.Run("create new bucket from existing state", func(t *testing.T) { - b, err := NewBucket(testCtx(), dirNameRecovered, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - WithStrategy(StrategyReplace)) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - bRec = b - }) - - t.Run("verify all data prior to the corruption is present", func(t *testing.T) { - key1 := []byte("key-1") - key2 := []byte("key-2") - key3 := []byte("key-3") - orig1 := []byte("original value for key1") - notUpdated3 := []byte("original value for key3") - - // the last operation we performed (that now got corrupted) was an update - // on key3. So now that we're expecting all state prior to the corruption - // to be present, we would expect the original value for key3 - - res, err := bRec.Get(key1) - require.Nil(t, err) - assert.Equal(t, res, orig1) - res, err = bRec.Get(key2) - require.Nil(t, err) - assert.Nil(t, res) - res, err = bRec.Get(key3) - require.Nil(t, err) - assert.Equal(t, res, notUpdated3) - }) - }) -} - -func TestSetStrategy_RecoverFromWAL(t *testing.T) { - dirNameOriginal := t.TempDir() - dirNameRecovered := t.TempDir() - - t.Run("without prior state", func(t *testing.T) { - b, err := NewBucket(testCtx(), dirNameOriginal, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - WithStrategy(StrategySetCollection)) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - key1 := []byte("test1-key-1") - key2 := []byte("test1-key-2") - key3 := []byte("test1-key-3") - - t.Run("set original values and verify", func(t *testing.T) { - orig1 := [][]byte{[]byte("value 1.1"), []byte("value 1.2")} - orig2 := [][]byte{[]byte("value 2.1"), []byte("value 2.2")} - orig3 := [][]byte{[]byte("value 3.1"), []byte("value 3.2")} - - err = b.SetAdd(key1, orig1) - require.Nil(t, err) - err = b.SetAdd(key2, orig2) - require.Nil(t, err) - err = b.SetAdd(key3, orig3) - require.Nil(t, err) - - res, err := b.SetList(key1) - require.Nil(t, err) - assert.Equal(t, orig1, res) - res, err = b.SetList(key2) - require.Nil(t, err) - assert.Equal(t, orig2, res) - res, err = b.SetList(key3) - require.Nil(t, err) - assert.Equal(t, orig3, res) - }) - - t.Run("delete individual keys", func(t *testing.T) { - delete2 := []byte("value 2.1") - delete3 := []byte("value 3.2") - - err = b.SetDeleteSingle(key2, delete2) - require.Nil(t, err) - err = b.SetDeleteSingle(key3, delete3) - require.Nil(t, err) - }) - - t.Run("re-add keys which were previously deleted and new ones", func(t *testing.T) { - readd2 := [][]byte{[]byte("value 2.1"), []byte("value 2.3")} - readd3 := [][]byte{[]byte("value 3.2"), []byte("value 3.3")} - - err = b.SetAdd(key2, readd2) - require.Nil(t, err) - err = b.SetAdd(key3, readd3) - require.Nil(t, err) - }) - - t.Run("validate the results prior to recovery", func(t *testing.T) { - expected1 := [][]byte{[]byte("value 1.1"), []byte("value 1.2")} // unchanged - expected2 := [][]byte{ - []byte("value 2.2"), // from original import - []byte("value 2.1"), // added again after initial deletion - []byte("value 2.3"), // newly added - } - expected3 := [][]byte{ - []byte("value 3.1"), // form original import - []byte("value 3.2"), // added again after initial deletion - []byte("value 3.3"), // newly added - } // value2 deleted - - res, err := b.SetList(key1) - require.Nil(t, err) - assert.Equal(t, expected1, res) - res, err = b.SetList(key2) - require.Nil(t, err) - assert.Equal(t, expected2, res) - res, err = b.SetList(key3) - require.Nil(t, err) - assert.Equal(t, expected3, res) - }) - - t.Run("make sure the WAL is flushed", func(t *testing.T) { - require.Nil(t, b.WriteWAL()) - }) - - t.Run("copy state into recovery folder and destroy original", func(t *testing.T) { - cmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("cp -r %s/*.wal %s", - dirNameOriginal, dirNameRecovered)) - var out bytes.Buffer - cmd.Stderr = &out - err := cmd.Run() - if err != nil { - fmt.Println(out.String()) - t.Fatal(err) - } - b = nil - require.Nil(t, os.RemoveAll(dirNameOriginal)) - }) - - var bRec *Bucket - - t.Run("create new bucket from existing state", func(t *testing.T) { - b, err := NewBucket(testCtx(), dirNameRecovered, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - WithStrategy(StrategySetCollection)) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - bRec = b - }) - - t.Run("validate the results after recovery", func(t *testing.T) { - expected1 := [][]byte{[]byte("value 1.1"), []byte("value 1.2")} // unchanged - expected2 := [][]byte{ - []byte("value 2.2"), // from original import - []byte("value 2.1"), // added again after initial deletion - []byte("value 2.3"), // newly added - } - expected3 := [][]byte{ - []byte("value 3.1"), // form original import - []byte("value 3.2"), // added again after initial deletion - []byte("value 3.3"), // newly added - } // value2 deleted - - res, err := bRec.SetList(key1) - require.Nil(t, err) - assert.Equal(t, expected1, res) - res, err = bRec.SetList(key2) - require.Nil(t, err) - assert.Equal(t, expected2, res) - res, err = bRec.SetList(key3) - require.Nil(t, err) - assert.Equal(t, expected3, res) - }) - }) -} - -func TestMapStrategy_RecoverFromWAL(t *testing.T) { - dirNameOriginal := t.TempDir() - dirNameRecovered := t.TempDir() - - t.Run("without prior state", func(t *testing.T) { - b, err := NewBucket(testCtx(), dirNameOriginal, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - WithStrategy(StrategyMapCollection)) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - rowKey1 := []byte("test1-key-1") - rowKey2 := []byte("test1-key-2") - - t.Run("set original values and verify", func(t *testing.T) { - row1Map := []MapPair{ - { - Key: []byte("row1-key1"), - Value: []byte("row1-key1-value1"), - }, { - Key: []byte("row1-key2"), - Value: []byte("row1-key2-value1"), - }, - } - - row2Map := []MapPair{ - { - Key: []byte("row2-key1"), - Value: []byte("row2-key1-value1"), - }, { - Key: []byte("row2-key2"), - Value: []byte("row2-key2-value1"), - }, - } - - for _, pair := range row1Map { - err = b.MapSet(rowKey1, pair) - require.Nil(t, err) - } - - for _, pair := range row2Map { - err = b.MapSet(rowKey2, pair) - require.Nil(t, err) - } - - res, err := b.MapList(rowKey1) - require.Nil(t, err) - assert.Equal(t, row1Map, res) - res, err = b.MapList(rowKey2) - require.Nil(t, err) - assert.Equal(t, res, row2Map) - }) - - t.Run("replace an existing map key", func(t *testing.T) { - err = b.MapSet(rowKey1, MapPair{ - Key: []byte("row1-key1"), // existing key - Value: []byte("row1-key1-value2"), // updated value - }) - require.Nil(t, err) - - row1Updated := []MapPair{ - { - Key: []byte("row1-key1"), - Value: []byte("row1-key1-value2"), // <--- updated, rest unchanged - }, { - Key: []byte("row1-key2"), - Value: []byte("row1-key2-value1"), - }, - } - - row2Unchanged := []MapPair{ - { - Key: []byte("row2-key1"), - Value: []byte("row2-key1-value1"), - }, { - Key: []byte("row2-key2"), - Value: []byte("row2-key2-value1"), - }, - } - - res, err := b.MapList(rowKey1) - require.Nil(t, err) - assert.Equal(t, row1Updated, res) - res, err = b.MapList(rowKey2) - require.Nil(t, err) - assert.Equal(t, res, row2Unchanged) - }) - - t.Run("validate the results prior to recovery", func(t *testing.T) { - rowKey1 := []byte("test1-key-1") - rowKey2 := []byte("test1-key-2") - - expectedRow1 := []MapPair{ - { - Key: []byte("row1-key1"), - Value: []byte("row1-key1-value2"), - }, { - Key: []byte("row1-key2"), - Value: []byte("row1-key2-value1"), - }, - } - - expectedRow2 := []MapPair{ - { - Key: []byte("row2-key1"), - Value: []byte("row2-key1-value1"), - }, { - Key: []byte("row2-key2"), - Value: []byte("row2-key2-value1"), - }, - } - - res, err := b.MapList(rowKey1) - require.Nil(t, err) - assert.Equal(t, expectedRow1, res) - res, err = b.MapList(rowKey2) - require.Nil(t, err) - assert.Equal(t, expectedRow2, res) - }) - - t.Run("make sure the WAL is flushed", func(t *testing.T) { - require.Nil(t, b.WriteWAL()) - }) - - t.Run("copy state into recovery folder and destroy original", func(t *testing.T) { - cmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("cp -r %s/*.wal %s", - dirNameOriginal, dirNameRecovered)) - var out bytes.Buffer - cmd.Stderr = &out - err := cmd.Run() - if err != nil { - fmt.Println(out.String()) - t.Fatal(err) - } - b = nil - require.Nil(t, os.RemoveAll(dirNameOriginal)) - }) - - var bRec *Bucket - - t.Run("create new bucket from existing state", func(t *testing.T) { - b, err := NewBucket(testCtx(), dirNameRecovered, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - WithStrategy(StrategyMapCollection)) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - bRec = b - }) - - t.Run("validate the results after recovery", func(t *testing.T) { - rowKey1 := []byte("test1-key-1") - rowKey2 := []byte("test1-key-2") - - expectedRow1 := []MapPair{ - { - Key: []byte("row1-key1"), - Value: []byte("row1-key1-value2"), - }, { - Key: []byte("row1-key2"), - Value: []byte("row1-key2-value1"), - }, - } - - expectedRow2 := []MapPair{ - { - Key: []byte("row2-key1"), - Value: []byte("row2-key1-value1"), - }, { - Key: []byte("row2-key2"), - Value: []byte("row2-key2-value1"), - }, - } - - res, err := bRec.MapList(rowKey1) - require.Nil(t, err) - assert.Equal(t, expectedRow1, res) - res, err = bRec.MapList(rowKey2) - require.Nil(t, err) - assert.Equal(t, expectedRow2, res) - }) - }) -} diff --git a/adapters/repos/db/lsmkv/red_black_tree_test.go b/adapters/repos/db/lsmkv/red_black_tree_test.go deleted file mode 100644 index d5c357e593b1a6a876a52cdd771d0d3bf79d7052..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/red_black_tree_test.go +++ /dev/null @@ -1,424 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "crypto/rand" - "fmt" - "math" - "math/big" - "reflect" - "testing" - - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv/rbtree" -) - -const ( - R = true - B = false -) - -// This test adds keys to the RB tree. Afterwards the same nodes are added in the expected order, eg in the way -// the RB tree is expected to re-order the nodes -var rbTests = []struct { - name string - keys []uint - ReorderedKeys []uint - expectedColors []bool // with respect to the original keys -}{ - { - "Requires recoloring but no reordering", - []uint{61, 52, 83, 93}, - []uint{61, 52, 83, 93}, - []bool{B, B, B, R}, - }, - { - "Requires left rotate around root", - []uint{61, 83, 99}, - []uint{83, 61, 99}, - []bool{R, B, R}, - }, - { - "Requires left rotate with more nodes", - []uint{61, 52, 85, 93, 99}, - []uint{61, 52, 93, 85, 99}, - []bool{B, B, R, B, R}, - }, - { - "Requires right and then left rotate", - []uint{61, 52, 85, 93, 87}, - []uint{61, 52, 87, 85, 93}, - []bool{B, B, R, R, B}, - }, - { - "Requires right rotate around root", - []uint{61, 30, 10}, - []uint{30, 10, 61}, - []bool{R, B, R}, - }, - { - "Requires right rotate with more nodes", - []uint{61, 52, 85, 21, 10}, - []uint{61, 85, 21, 10, 52}, - []bool{B, R, B, B, R}, - }, - { - "Requires left and then right rotate", - []uint{61, 52, 85, 21, 36}, - []uint{61, 85, 36, 21, 52}, - []bool{B, R, B, R, B}, - }, - { - "Require reordering for two nodes", - []uint{61, 52, 40, 85, 105, 110}, - []uint{52, 40, 85, 61, 105, 110}, - []bool{B, B, B, R, B, R}, - }, - { - "Ordered nodes increasing", - []uint{1, 2, 3, 4, 5, 6, 7, 8}, - []uint{4, 2, 6, 1, 3, 5, 7, 8}, - []bool{B, R, B, B, B, R, B, R}, - }, - { - "Ordered nodes decreasing", - []uint{8, 7, 6, 5, 4, 3, 2, 1}, - []uint{5, 3, 7, 2, 4, 6, 8, 1}, - []bool{B, R, B, B, B, R, B, R}, - }, - { - "Multiple rotations along the tree and colour changes", - []uint{166, 92, 33, 133, 227, 236, 71, 183, 18, 139, 245, 161}, - []uint{166, 92, 227, 33, 139, 183, 236, 18, 71, 133, 161, 245}, - []bool{B, R, B, R, R, B, R, B, R, B, R, R}, - }, -} - -func TestRBTree(t *testing.T) { - for _, tt := range rbTests { - t.Run(tt.name, func(t *testing.T) { - tree := &binarySearchTree{} - for _, key := range tt.keys { - iByte := []byte{uint8(key)} - tree.insert(iByte, iByte, nil) - require.Empty(t, tree.root.parent) - } - validateRBTree(t, tree.root) - - flattenTree := tree.flattenInOrder() - require.Equal(t, len(tt.keys), len(flattenTree)) // no entries got lost - - // add tree with the same nodes in the "optimal" order to be able to compare their order afterwards - treeCorrectOrder := &binarySearchTree{} - for _, key := range tt.ReorderedKeys { - iByte := []byte{uint8(key)} - treeCorrectOrder.insert(iByte, iByte, nil) - } - - flattenTreeInput := treeCorrectOrder.flattenInOrder() - for i := range flattenTree { - byteKey := flattenTree[i].key - originalIndex := getIndexInSlice(tt.keys, byteKey) - require.Equal(t, byteKey, flattenTreeInput[i].key) - require.Equal(t, flattenTree[i].colourIsRed, tt.expectedColors[originalIndex]) - } - }) - } -} - -func TestRBTreeMap(t *testing.T) { - for _, tt := range rbTests { - t.Run(tt.name, func(t *testing.T) { - tree := &binarySearchTreeMap{} - for _, key := range tt.keys { - tree.insert([]byte{uint8(key)}, MapPair{ - Key: []byte("map-key-1"), - Value: []byte("map-value-1"), - }) - require.Empty(t, tree.root.parent) - } - validateRBTree(t, tree.root) - - flatten_tree := tree.flattenInOrder() - require.Equal(t, len(tt.keys), len(flatten_tree)) // no entries got lost - - // add tree with the same nodes in the "optimal" order to be able to compare their order afterwards - treeCorrectOrder := &binarySearchTreeMap{} - for _, key := range tt.ReorderedKeys { - treeCorrectOrder.insert([]byte{uint8(key)}, MapPair{ - Key: []byte("map-key-1"), - Value: []byte("map-value-1"), - }) - } - - flatten_tree_input := treeCorrectOrder.flattenInOrder() - for i := range flatten_tree { - byte_key := flatten_tree[i].key - originalIndex := getIndexInSlice(tt.keys, byte_key) - require.Equal(t, byte_key, flatten_tree_input[i].key) - require.Equal(t, flatten_tree[i].colourIsRed, tt.expectedColors[originalIndex]) - } - }) - } -} - -func TestRBTreeMulti(t *testing.T) { - for _, tt := range rbTests { - t.Run(tt.name, func(t *testing.T) { - tree := &binarySearchTreeMulti{} - for _, key := range tt.keys { - values := []value{} - for j := uint(0); j < 5; j++ { - values = append(values, value{value: []byte{uint8(key * j)}, tombstone: false}) - } - tree.insert([]byte{uint8(key)}, values) - require.Empty(t, tree.root.parent) - } - validateRBTree(t, tree.root) - - flatten_tree := tree.flattenInOrder() - require.Equal(t, len(tt.keys), len(flatten_tree)) // no entries got lost - - // add tree with the same nodes in the "optimal" order to be able to compare their order afterwards - treeCorrectOrder := &binarySearchTreeMulti{} - for _, key := range tt.ReorderedKeys { - values := []value{} - for j := uint(0); j < 5; j++ { - values = append(values, value{value: []byte{uint8(key * j)}, tombstone: false}) - } - treeCorrectOrder.insert([]byte{uint8(key)}, values) - } - - flatten_tree_input := treeCorrectOrder.flattenInOrder() - for i := range flatten_tree { - byte_key := flatten_tree[i].key - originalIndex := getIndexInSlice(tt.keys, byte_key) - require.Equal(t, byte_key, flatten_tree_input[i].key) - require.Equal(t, flatten_tree[i].colourIsRed, tt.expectedColors[originalIndex]) - } - }) - } -} - -// add keys as a) normal keys b) tombstone keys and c) half tombstone, half normal. -// The resulting (rebalanced) trees must have the same order and colors -var tombstoneTests = []struct { - name string - keys []uint -}{ - {"Rotate left around root", []uint{61, 83, 99}}, - {"Rotate right around root", []uint{61, 30, 10}}, - {"Multiple rotations along the tree and colour changes", []uint{166, 92, 33, 133, 227, 236, 71, 183, 18, 139, 245, 161}}, - {"Ordered nodes increasing", []uint{1, 2, 3, 4, 5, 6, 7, 8}}, - {"Ordered nodes decreasing", []uint{8, 7, 6, 5, 4, 3, 2, 1}}, -} - -func TestRBTrees_Tombstones(t *testing.T) { - for _, tt := range tombstoneTests { - t.Run(tt.name, func(t *testing.T) { - treeNormal := &binarySearchTree{} - treeTombstone := &binarySearchTree{} - treeHalfHalf := &binarySearchTree{} - for i, key := range tt.keys { - iByte := []byte{uint8(key)} - treeNormal.insert(iByte, iByte, nil) - treeTombstone.setTombstone(iByte, nil) - if i%2 == 0 { - treeHalfHalf.insert(iByte, iByte, nil) - } else { - treeHalfHalf.setTombstone(iByte, nil) - } - } - validateRBTree(t, treeNormal.root) - validateRBTree(t, treeTombstone.root) - validateRBTree(t, treeHalfHalf.root) - - treeNormalFlatten := treeNormal.flattenInOrder() - treeTombstoneFlatten := treeTombstone.flattenInOrder() - treeHalfHalfFlatten := treeHalfHalf.flattenInOrder() - require.Equal(t, len(tt.keys), len(treeNormalFlatten)) - require.Equal(t, len(tt.keys), len(treeTombstoneFlatten)) - require.Equal(t, len(tt.keys), len(treeHalfHalfFlatten)) - - for i := range treeNormalFlatten { - require.Equal(t, treeNormalFlatten[i].key, treeTombstoneFlatten[i].key) - require.Equal(t, treeNormalFlatten[i].key, treeHalfHalfFlatten[i].key) - require.Equal(t, treeNormalFlatten[i].colourIsRed, treeTombstoneFlatten[i].colourIsRed) - require.Equal(t, treeNormalFlatten[i].colourIsRed, treeHalfHalfFlatten[i].colourIsRed) - } - }) - } -} - -type void struct{} - -var member void - -func mustRandIntn(max int64) int { - randInt, err := rand.Int(rand.Reader, big.NewInt(max)) - if err != nil { - panic(fmt.Sprintf("mustRandIntn error: %v", err)) - } - return int(randInt.Int64()) -} - -func TestRBTrees_Random(t *testing.T) { - tree := &binarySearchTree{} - amount := mustRandIntn(100000) - keySize := mustRandIntn(100) - uniqueKeys := make(map[string]void) - for i := 0; i < amount; i++ { - key := make([]byte, keySize) - rand.Read(key) - uniqueKeys[fmt.Sprint(key)] = member - if mustRandIntn(5) == 1 { // add 20% of all entries as tombstone - tree.setTombstone(key, nil) - } else { - tree.insert(key, key, nil) - } - } - - // all added keys are still part of the tree - treeFlattened := tree.flattenInOrder() - require.Equal(t, len(uniqueKeys), len(treeFlattened)) - for _, entry := range treeFlattened { - _, ok := uniqueKeys[fmt.Sprint(entry.key)] - require.True(t, ok) - } - validateRBTree(t, tree.root) -} - -func TestRBTreesMap_Random(t *testing.T) { - tree := &binarySearchTreeMap{} - amount := mustRandIntn(100000) - keySize := mustRandIntn(100) - uniqueKeys := make(map[string]void) - for i := 0; i < amount; i++ { - key := make([]byte, keySize) - rand.Read(key) - uniqueKeys[fmt.Sprint(key)] = member - tree.insert(key, MapPair{ - Key: []byte("map-key-1"), - Value: []byte("map-value-1"), - }) - } - - // all added keys are still part of the tree - treeFlattened := tree.flattenInOrder() - require.Equal(t, len(uniqueKeys), len(treeFlattened)) - for _, entry := range treeFlattened { - _, ok := uniqueKeys[fmt.Sprint(entry.key)] - require.True(t, ok) - } - validateRBTree(t, tree.root) -} - -func TestRBTreesMulti_Random(t *testing.T) { - tree := &binarySearchTreeMulti{} - amount := mustRandIntn(100000) - keySize := mustRandIntn(100) - uniqueKeys := make(map[string]void) - for i := 0; i < amount; i++ { - key := make([]byte, keySize) - rand.Read(key) - uniqueKeys[fmt.Sprint(key)] = member - values := []value{} - for j := 0; j < 5; j++ { - values = append(values, value{value: []byte{uint8(i * j)}, tombstone: false}) - } - tree.insert(key, values) - } - - // all added keys are still part of the tree - treeFlattened := tree.flattenInOrder() - require.Equal(t, len(uniqueKeys), len(treeFlattened)) - for _, entry := range treeFlattened { - _, ok := uniqueKeys[fmt.Sprint(entry.key)] - require.True(t, ok) - } - validateRBTree(t, tree.root) -} - -func getIndexInSlice(reorderedKeys []uint, key []byte) int { - for i, v := range reorderedKeys { - if v == uint(key[0]) { - return i - } - } - return -1 -} - -// Checks if a tree is a RB tree -// -// There are several properties that valid RB trees follow: -// 1) The root node is always black -// 2) The max depth of a tree is 2* Log2(N+1), where N is the number of nodes -// 3) Every path from root to leave has the same number of _black_ nodes -// 4) Red nodes only have black (or nil) children -// -// In addition this also validates some general tree properties: -// - root has no parent -// - if node A is a child of B, B must be the parent of A) -func validateRBTree(t *testing.T, rootNode rbtree.Node) { - require.False(t, rootNode.IsRed()) - require.True(t, rootNode.Parent().IsNil()) - - treeDepth, nodeCount, _ := walkTree(t, rootNode) - maxDepth := 2 * math.Log2(float64(nodeCount)+1) - require.True(t, treeDepth <= int(maxDepth)) -} - -// Walks through the tree and counts the depth, number of nodes and number of black nodes -func walkTree(t *testing.T, node rbtree.Node) (int, int, int) { - if reflect.ValueOf(node).IsNil() { - return 0, 0, 0 - } - leftNode := node.Left() - leftNodeIsNil := reflect.ValueOf(leftNode).IsNil() - rightNode := node.Right() - rightNodeIsNil := reflect.ValueOf(rightNode).IsNil() - - // validate parent/child connections - if !rightNodeIsNil { - require.Equal(t, rightNode.Parent(), node) - } - if !leftNodeIsNil { - require.Equal(t, leftNode.Parent(), node) - } - - // red nodes need black (or nil) children - if node.IsRed() { - require.True(t, leftNodeIsNil || !node.Left().IsRed()) - require.True(t, rightNodeIsNil || !node.Left().IsRed()) - } - - blackNode := int(1) - if node.IsRed() { - blackNode = 0 - } - - if node.Right().IsNil() && node.Left().IsNil() { - return 1, 1, blackNode - } - - depthRight, nodeCountRight, blackNodesDepthRight := walkTree(t, node.Right()) - depthLeft, nodeCountLeft, blackNodesDepthLeft := walkTree(t, node.Left()) - require.Equal(t, blackNodesDepthRight, blackNodesDepthLeft) - - nodeCount := nodeCountLeft + nodeCountRight + 1 - if depthRight > depthLeft { - return depthRight + 1, nodeCount, blackNodesDepthRight + blackNode - } else { - return depthLeft + 1, nodeCount, blackNodesDepthRight + blackNode - } -} diff --git a/adapters/repos/db/lsmkv/roaringset/binary_search_tree.go b/adapters/repos/db/lsmkv/roaringset/binary_search_tree.go deleted file mode 100644 index 8b56e2f1c850b1db1216985df43d90d042898b00..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/roaringset/binary_search_tree.go +++ /dev/null @@ -1,268 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package roaringset - -import ( - "bytes" - - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv/rbtree" - "github.com/weaviate/weaviate/entities/lsmkv" -) - -type BinarySearchTree struct { - root *BinarySearchNode -} - -type Insert struct { - Additions []uint64 - Deletions []uint64 -} - -func (t *BinarySearchTree) Insert(key []byte, values Insert) { - if t.root == nil { - t.root = &BinarySearchNode{ - Key: key, - Value: BitmapLayer{ - Additions: NewBitmap(values.Additions...), - Deletions: NewBitmap(values.Deletions...), - }, - colourIsRed: false, // root node is always black - } - return - } - - if newRoot := t.root.insert(key, values); newRoot != nil { - t.root = newRoot - } - t.root.colourIsRed = false // Can be flipped in the process of balancing, but root is always black -} - -// Get creates copies of underlying bitmaps to prevent future (concurrent) -// read and writes after layer being returned -func (t *BinarySearchTree) Get(key []byte) (BitmapLayer, error) { - if t.root == nil { - return BitmapLayer{}, lsmkv.NotFound - } - - return t.root.get(key) -} - -// FlattenInOrder creates list of ordered copies of bst nodes -// Only Key and Value fields are populated -func (t *BinarySearchTree) FlattenInOrder() []*BinarySearchNode { - if t.root == nil { - return nil - } - - return t.root.flattenInOrder() -} - -type BinarySearchNode struct { - Key []byte - Value BitmapLayer - left *BinarySearchNode - right *BinarySearchNode - parent *BinarySearchNode - colourIsRed bool -} - -func (n *BinarySearchNode) Parent() rbtree.Node { - if n == nil { - return nil - } - return n.parent -} - -func (n *BinarySearchNode) SetParent(parent rbtree.Node) { - if n == nil { - addNewSearchNodeRoaringSetReceiver(&n) - } - - if parent == nil { - n.parent = nil - return - } - - n.parent = parent.(*BinarySearchNode) -} - -func (n *BinarySearchNode) Left() rbtree.Node { - if n == nil { - return nil - } - return n.left -} - -func (n *BinarySearchNode) SetLeft(left rbtree.Node) { - if n == nil { - addNewSearchNodeRoaringSetReceiver(&n) - } - - if left == nil { - n.left = nil - return - } - - n.left = left.(*BinarySearchNode) -} - -func (n *BinarySearchNode) Right() rbtree.Node { - if n == nil { - return nil - } - return n.right -} - -func (n *BinarySearchNode) SetRight(right rbtree.Node) { - if n == nil { - addNewSearchNodeRoaringSetReceiver(&n) - } - - if right == nil { - n.right = nil - return - } - - n.right = right.(*BinarySearchNode) -} - -func (n *BinarySearchNode) IsRed() bool { - if n == nil { - return false - } - return n.colourIsRed -} - -func (n *BinarySearchNode) SetRed(isRed bool) { - n.colourIsRed = isRed -} - -func (n *BinarySearchNode) IsNil() bool { - return n == nil -} - -func addNewSearchNodeRoaringSetReceiver(nodePtr **BinarySearchNode) { - *nodePtr = &BinarySearchNode{} -} - -func (n *BinarySearchNode) insert(key []byte, values Insert) *BinarySearchNode { - if bytes.Equal(key, n.Key) { - // Merging the new additions and deletions into the existing ones is a - // four-step process: - // - // 1. make sure anything that's added is not part of the deleted list, in - // case it was previously deleted - // 2. actually add the new entries to additions - // 3. make sure anything that's deleted is not part of the additions list, - // in case it was recently added - // 4. actually add the new entries to deletions (this step is vital in case - // a delete points to an entry of a previous segment that's not added in - // this memtable) - for _, x := range values.Additions { - n.Value.Deletions.Remove(x) - n.Value.Additions.Set(x) - } - - for _, x := range values.Deletions { - n.Value.Additions.Remove(x) - n.Value.Deletions.Set(x) - } - - return nil - } - - if bytes.Compare(key, n.Key) < 0 { - if n.left != nil { - return n.left.insert(key, values) - } else { - n.left = &BinarySearchNode{ - Key: key, - Value: BitmapLayer{ - Additions: NewBitmap(values.Additions...), - Deletions: NewBitmap(values.Deletions...), - }, - parent: n, - colourIsRed: true, - } - return BinarySearchNodeFromRB(rbtree.Rebalance(n.left)) - } - } else { - if n.right != nil { - return n.right.insert(key, values) - } else { - n.right = &BinarySearchNode{ - Key: key, - Value: BitmapLayer{ - Additions: NewBitmap(values.Additions...), - Deletions: NewBitmap(values.Deletions...), - }, - parent: n, - colourIsRed: true, - } - return BinarySearchNodeFromRB(rbtree.Rebalance(n.right)) - } - } -} - -func (n *BinarySearchNode) get(key []byte) (BitmapLayer, error) { - if bytes.Equal(n.Key, key) { - return n.Value.Clone(), nil - } - - if bytes.Compare(key, n.Key) < 0 { - if n.left == nil { - return BitmapLayer{}, lsmkv.NotFound - } - - return n.left.get(key) - } else { - if n.right == nil { - return BitmapLayer{}, lsmkv.NotFound - } - - return n.right.get(key) - } -} - -func BinarySearchNodeFromRB(rbNode rbtree.Node) (bsNode *BinarySearchNode) { - if rbNode == nil { - bsNode = nil - return - } - bsNode = rbNode.(*BinarySearchNode) - return -} - -func (n *BinarySearchNode) flattenInOrder() []*BinarySearchNode { - var left []*BinarySearchNode - var right []*BinarySearchNode - - if n.left != nil { - left = n.left.flattenInOrder() - } - - if n.right != nil { - right = n.right.flattenInOrder() - } - - key := make([]byte, len(n.Key)) - copy(key, n.Key) - - right = append([]*BinarySearchNode{{ - Key: key, - Value: BitmapLayer{ - Additions: Condense(n.Value.Additions), - Deletions: Condense(n.Value.Deletions), - }, - }}, right...) - return append(left, right...) -} diff --git a/adapters/repos/db/lsmkv/roaringset/binary_search_tree_cursor.go b/adapters/repos/db/lsmkv/roaringset/binary_search_tree_cursor.go deleted file mode 100644 index 86a0b5b58439f3ace013ba1f61e18ee2b1e6bc34..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/roaringset/binary_search_tree_cursor.go +++ /dev/null @@ -1,69 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package roaringset - -import ( - "bytes" - - "github.com/weaviate/weaviate/entities/lsmkv" -) - -type BinarySearchTreeCursor struct { - nodes []*BinarySearchNode - nextNodePos int -} - -func NewBinarySearchTreeCursor(bst *BinarySearchTree) *BinarySearchTreeCursor { - return &BinarySearchTreeCursor{nodes: bst.FlattenInOrder()} -} - -func (c *BinarySearchTreeCursor) First() ([]byte, BitmapLayer, error) { - c.nextNodePos = 0 - return c.Next() -} - -func (c *BinarySearchTreeCursor) Next() ([]byte, BitmapLayer, error) { - if c.nextNodePos >= len(c.nodes) { - return nil, BitmapLayer{}, nil - } - - pos := c.nextNodePos - c.nextNodePos++ - return c.nodes[pos].Key, c.nodes[pos].Value, nil -} - -func (c *BinarySearchTreeCursor) Seek(key []byte) ([]byte, BitmapLayer, error) { - pos := c.posKeyGreaterThanEqual(key) - if pos == -1 { - return nil, BitmapLayer{}, lsmkv.NotFound - } - c.nextNodePos = pos - return c.Next() -} - -func (c *BinarySearchTreeCursor) posKeyGreaterThanEqual(key []byte) int { - // seek from the end, return position of first (from the beginning) node with key >= given key - // if key > node_key return previous pos - // if key == node_key return current pos - // if key < node_key continue or return current pos if all nodes checked - pos := -1 - for i := len(c.nodes) - 1; i >= 0; i-- { - if cmp := bytes.Compare(key, c.nodes[i].Key); cmp > 0 { - break - } else if cmp == 0 { - pos = i - break - } - pos = i - } - return pos -} diff --git a/adapters/repos/db/lsmkv/roaringset/binary_search_tree_cursor_test.go b/adapters/repos/db/lsmkv/roaringset/binary_search_tree_cursor_test.go deleted file mode 100644 index 40c790a9cd99567cfb7a4587d27bf3a32c150d65..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/roaringset/binary_search_tree_cursor_test.go +++ /dev/null @@ -1,175 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package roaringset - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/entities/lsmkv" -) - -func TestBSTCursor(t *testing.T) { - bst := &BinarySearchTree{} - - in := []struct { - key string - addVal uint64 - delVal uint64 - }{ - {"aaa", 1, 11}, - {"bbb", 2, 22}, - {"ccc", 3, 33}, - {"ddd", 4, 44}, - } - - for _, v := range in { - bst.Insert([]byte(v.key), Insert{Additions: []uint64{v.addVal}, Deletions: []uint64{v.delVal}}) - } - - t.Run("start from beginning", func(t *testing.T) { - cursor := NewBinarySearchTreeCursor(bst) - - key, layer, err := cursor.First() - - assert.Equal(t, []byte(in[0].key), key) - assert.Equal(t, 1, layer.Additions.GetCardinality()) - assert.True(t, layer.Additions.Contains(in[0].addVal)) - assert.Equal(t, 1, layer.Deletions.GetCardinality()) - assert.True(t, layer.Deletions.Contains(in[0].delVal)) - assert.Nil(t, err) - }) - - t.Run("start from beginning and go through all", func(t *testing.T) { - cursor := NewBinarySearchTreeCursor(bst) - - i := 0 // 1st match is "aaa" - for key, layer, err := cursor.First(); key != nil; key, layer, err = cursor.Next() { - assert.Equal(t, []byte(in[i].key), key) - assert.Equal(t, 1, layer.Additions.GetCardinality()) - assert.True(t, layer.Additions.Contains(in[i].addVal)) - assert.Equal(t, 1, layer.Deletions.GetCardinality()) - assert.True(t, layer.Deletions.Contains(in[i].delVal)) - assert.Nil(t, err) - i++ - } - assert.Equal(t, i, len(in)) - }) - - t.Run("seek matching element and go through rest", func(t *testing.T) { - cursor := NewBinarySearchTreeCursor(bst) - - i := 1 // 1st match is "bbb" - matching := []byte("bbb") - for key, layer, err := cursor.Seek(matching); key != nil; key, layer, err = cursor.Next() { - assert.Equal(t, []byte(in[i].key), key) - assert.Equal(t, 1, layer.Additions.GetCardinality()) - assert.True(t, layer.Additions.Contains(in[i].addVal)) - assert.Equal(t, 1, layer.Deletions.GetCardinality()) - assert.True(t, layer.Deletions.Contains(in[i].delVal)) - assert.Nil(t, err) - i++ - } - assert.Equal(t, i, len(in)) - }) - - t.Run("seek non-matching element and go through rest", func(t *testing.T) { - cursor := NewBinarySearchTreeCursor(bst) - - i := 2 // 1st match is "ccc" - nonMatching := []byte("bcde") - for key, layer, err := cursor.Seek(nonMatching); key != nil; key, layer, err = cursor.Next() { - assert.Equal(t, []byte(in[i].key), key) - assert.Equal(t, 1, layer.Additions.GetCardinality()) - assert.True(t, layer.Additions.Contains(in[i].addVal)) - assert.Equal(t, 1, layer.Deletions.GetCardinality()) - assert.True(t, layer.Deletions.Contains(in[i].delVal)) - assert.Nil(t, err) - i++ - } - assert.Equal(t, i, len(in)) - }) - - t.Run("seek missing element", func(t *testing.T) { - cursor := NewBinarySearchTreeCursor(bst) - - missing := []byte("eee") - key, layer, err := cursor.Seek(missing) - - assert.Nil(t, key) - assert.True(t, layer.Additions.IsEmpty()) - assert.True(t, layer.Deletions.IsEmpty()) - assert.ErrorIs(t, err, lsmkv.NotFound) - }) - - t.Run("next after seek missing element does not change cursor's position", func(t *testing.T) { - cursor := NewBinarySearchTreeCursor(bst) - - key1, _, err1 := cursor.First() - - missing := []byte("eee") - cursor.Seek(missing) - - key2, _, err2 := cursor.Next() - - assert.Equal(t, []byte("aaa"), key1) - assert.Nil(t, err1) - assert.Equal(t, []byte("bbb"), key2) - assert.Nil(t, err2) - }) - - t.Run("next after last is nil/empty", func(t *testing.T) { - cursor := NewBinarySearchTreeCursor(bst) - - last := []byte("ddd") - cursor.Seek(last) - key, layer, err := cursor.Next() - - assert.Nil(t, key) - assert.True(t, layer.Additions.IsEmpty()) - assert.True(t, layer.Deletions.IsEmpty()) - assert.Nil(t, err) - }) - - t.Run("first after final/empty next", func(t *testing.T) { - cursor := NewBinarySearchTreeCursor(bst) - - last := []byte("ddd") - cursor.Seek(last) - cursor.Next() - key, layer, err := cursor.First() - - assert.Equal(t, []byte(in[0].key), key) - assert.Equal(t, 1, layer.Additions.GetCardinality()) - assert.True(t, layer.Additions.Contains(in[0].addVal)) - assert.Equal(t, 1, layer.Deletions.GetCardinality()) - assert.True(t, layer.Deletions.Contains(in[0].delVal)) - assert.Nil(t, err) - }) - - t.Run("seek after final/empty next", func(t *testing.T) { - cursor := NewBinarySearchTreeCursor(bst) - - last := []byte("ddd") - matching := []byte("bbb") - cursor.Seek(last) - cursor.Next() - key, layer, err := cursor.Seek(matching) - - assert.Equal(t, []byte(in[1].key), key) - assert.Equal(t, 1, layer.Additions.GetCardinality()) - assert.True(t, layer.Additions.Contains(in[1].addVal)) - assert.Equal(t, 1, layer.Deletions.GetCardinality()) - assert.True(t, layer.Deletions.Contains(in[1].delVal)) - assert.Nil(t, err) - }) -} diff --git a/adapters/repos/db/lsmkv/roaringset/binary_search_tree_test.go b/adapters/repos/db/lsmkv/roaringset/binary_search_tree_test.go deleted file mode 100644 index e5e0572a1b552ab4c03edba0fbe5f1a8a68b1fa5..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/roaringset/binary_search_tree_test.go +++ /dev/null @@ -1,206 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package roaringset - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestBSTRoaringSet(t *testing.T) { - t.Run("single key, single set entry", func(t *testing.T) { - bst := &BinarySearchTree{} - key := []byte("my-key") - - bst.Insert(key, Insert{Additions: []uint64{7}}) - - res, err := bst.Get(key) - require.Nil(t, err) - - assert.False(t, res.Additions.Contains(6)) - assert.True(t, res.Additions.Contains(7)) - }) - - t.Run("single key, set updated multiple times", func(t *testing.T) { - bst := &BinarySearchTree{} - key := []byte("my-key") - - for i := uint64(7); i < 14; i++ { - bst.Insert(key, Insert{Additions: []uint64{i}}) - } - - res, err := bst.Get(key) - require.Nil(t, err) - - assert.False(t, res.Additions.Contains(6)) - for i := uint64(7); i < 14; i++ { - assert.True(t, res.Additions.Contains(i)) - } - assert.False(t, res.Additions.Contains(15)) - }) - - t.Run("single key, entry added, then deleted", func(t *testing.T) { - bst := &BinarySearchTree{} - key := []byte("my-key") - - for i := uint64(7); i < 11; i++ { - bst.Insert(key, Insert{Additions: []uint64{i}}) - } - - bst.Insert(key, Insert{Deletions: []uint64{9}}) - - res, err := bst.Get(key) - require.Nil(t, err) - - // check Additions - assert.True(t, res.Additions.Contains(7)) - assert.True(t, res.Additions.Contains(8)) - assert.False(t, res.Additions.Contains(9)) - assert.True(t, res.Additions.Contains(10)) - - // check Deletions - assert.True(t, res.Deletions.Contains(9)) - }) - - t.Run("single key, entry added, then deleted, then re-added", func(t *testing.T) { - bst := &BinarySearchTree{} - key := []byte("my-key") - - for i := uint64(7); i < 11; i++ { - bst.Insert(key, Insert{Additions: []uint64{i}}) - } - - bst.Insert(key, Insert{Deletions: []uint64{9}}) - - bst.Insert(key, Insert{Additions: []uint64{9}}) - - res, err := bst.Get(key) - require.Nil(t, err) - - // check Additions - assert.True(t, res.Additions.Contains(7)) - assert.True(t, res.Additions.Contains(8)) - assert.True(t, res.Additions.Contains(9)) - assert.True(t, res.Additions.Contains(10)) - - // check Deletions - assert.False(t, res.Deletions.Contains(9)) - }) - - t.Run("get is snapshot of underlying bitmaps", func(t *testing.T) { - bst := &BinarySearchTree{} - key := []byte("my-key") - - for i := uint64(1); i <= 3; i++ { - bst.Insert(key, Insert{ - Additions: []uint64{10 + i}, - Deletions: []uint64{10 - i}, - }) - } - - getBeforeUpdate, err := bst.Get(key) - require.Nil(t, err) - - expectedAdditionsBeforeUpdate := []uint64{11, 12, 13} - expectedDeletionsBeforeUpdate := []uint64{7, 8, 9} - - assert.ElementsMatch(t, expectedAdditionsBeforeUpdate, getBeforeUpdate.Additions.ToArray()) - assert.ElementsMatch(t, expectedDeletionsBeforeUpdate, getBeforeUpdate.Deletions.ToArray()) - - t.Run("gotten layer does not change on bst update", func(t *testing.T) { - bst.Insert(key, Insert{Additions: []uint64{100}, Deletions: []uint64{1}}) - - getAfterUpdate, err := bst.Get(key) - require.Nil(t, err) - - expectedAdditionsAfterUpdate := []uint64{11, 12, 13, 100} - expectedDeletionsAfterUpdate := []uint64{1, 7, 8, 9} - - assert.ElementsMatch(t, expectedAdditionsBeforeUpdate, getBeforeUpdate.Additions.ToArray()) - assert.ElementsMatch(t, expectedDeletionsBeforeUpdate, getBeforeUpdate.Deletions.ToArray()) - - assert.ElementsMatch(t, expectedAdditionsAfterUpdate, getAfterUpdate.Additions.ToArray()) - assert.ElementsMatch(t, expectedDeletionsAfterUpdate, getAfterUpdate.Deletions.ToArray()) - }) - }) -} - -func TestBSTRoaringSet_Flatten(t *testing.T) { - t.Run("flattened bst is snapshot of current bst", func(t *testing.T) { - key1 := "key-1" - key2 := "key-2" - key3 := "key-3" - - bst := &BinarySearchTree{} - // mixed order - bst.Insert([]byte(key3), Insert{Additions: []uint64{7, 8, 9}, Deletions: []uint64{77, 88, 99}}) - bst.Insert([]byte(key1), Insert{Additions: []uint64{1, 2, 3}, Deletions: []uint64{11, 22, 33}}) - bst.Insert([]byte(key2), Insert{Additions: []uint64{4, 5, 6}, Deletions: []uint64{44, 55, 66}}) - - flatBeforeUpdate := bst.FlattenInOrder() - - expectedBeforeUpdate := []struct { - key string - additions []uint64 - deletions []uint64 - }{ - {key1, []uint64{1, 2, 3}, []uint64{11, 22, 33}}, - {key2, []uint64{4, 5, 6}, []uint64{44, 55, 66}}, - {key3, []uint64{7, 8, 9}, []uint64{77, 88, 99}}, - } - - assert.Len(t, flatBeforeUpdate, len(expectedBeforeUpdate)) - for i, exp := range expectedBeforeUpdate { - assert.Equal(t, []byte(exp.key), flatBeforeUpdate[i].Key) - assert.ElementsMatch(t, exp.additions, flatBeforeUpdate[i].Value.Additions.ToArray()) - assert.ElementsMatch(t, exp.deletions, flatBeforeUpdate[i].Value.Deletions.ToArray()) - } - - t.Run("flattened bst does not change on bst update", func(t *testing.T) { - key4 := "key-4" - - // mixed order - bst.Insert([]byte(key4), Insert{Additions: []uint64{111, 222, 333}, Deletions: []uint64{444, 555, 666}}) - bst.Insert([]byte(key3), Insert{Additions: []uint64{77, 88}, Deletions: []uint64{7, 8}}) - bst.Insert([]byte(key1), Insert{Additions: []uint64{11, 22}, Deletions: []uint64{1, 2}}) - - flatAfterUpdate := bst.FlattenInOrder() - - expectedAfterUpdate := []struct { - key string - additions []uint64 - deletions []uint64 - }{ - {key1, []uint64{3, 11, 22}, []uint64{1, 2, 33}}, - {key2, []uint64{4, 5, 6}, []uint64{44, 55, 66}}, - {key3, []uint64{9, 77, 88}, []uint64{7, 8, 99}}, - {key4, []uint64{111, 222, 333}, []uint64{444, 555, 666}}, - } - - assert.Len(t, flatBeforeUpdate, len(expectedBeforeUpdate)) - for i, exp := range expectedBeforeUpdate { - assert.Equal(t, []byte(exp.key), flatBeforeUpdate[i].Key) - assert.ElementsMatch(t, exp.additions, flatBeforeUpdate[i].Value.Additions.ToArray()) - assert.ElementsMatch(t, exp.deletions, flatBeforeUpdate[i].Value.Deletions.ToArray()) - } - - assert.Len(t, flatAfterUpdate, len(expectedAfterUpdate)) - for i, exp := range expectedAfterUpdate { - assert.Equal(t, []byte(exp.key), flatAfterUpdate[i].Key) - assert.ElementsMatch(t, exp.additions, flatAfterUpdate[i].Value.Additions.ToArray()) - assert.ElementsMatch(t, exp.deletions, flatAfterUpdate[i].Value.Deletions.ToArray()) - } - }) - }) -} diff --git a/adapters/repos/db/lsmkv/roaringset/compactor.go b/adapters/repos/db/lsmkv/roaringset/compactor.go deleted file mode 100644 index 6ffac0977b798ce640bbafebcb586cc2e547cfad..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/roaringset/compactor.go +++ /dev/null @@ -1,326 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package roaringset - -import ( - "bufio" - "bytes" - "fmt" - "io" - - "github.com/pkg/errors" - "github.com/weaviate/sroar" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv/segmentindex" -) - -// Compactor takes in a left and a right segment and merges them into a single -// segment. The input segments are represented by cursors without their -// respective segmentindexes. A new segmentindex is built from the merged nodes -// without taking the old indexes into account at all. -// -// The left segment must precede the right one in its creation time, as the -// compactor applies latest-takes-presence rules when there is a conflict. -// -// # Merging independent key/value pairs -// -// The new segment's nodes will be in sorted fashion (this is a requirement for -// the segment index and segment cursors to function). To achieve a sorted end -// result, the Compactor goes over both input cursors simultaneously and always -// works on the smaller of the two keys. After a key/value pair has been added -// to the output only the input cursor that provided the pair is advanced. -// -// # Merging key/value pairs with identical keys -// -// When both segment have a key/value pair with an overlapping key, the value -// has to be merged. The merge logic is not part of the compactor itself. -// Instead it makes use of [BitmapLayers.Merge]. -// -// # Exit Criterium -// -// When both cursors no longer return values, all key/value pairs are -// considered compacted. The compactor then deals with metadata. -// -// # Index and Header metadata -// -// Only once the key/value pairs have been compacted, will the compactor write -// the primary index based on the new key/value payload. Finally, the input -// writer is rewinded to be able to write the header metadata at the beginning -// of the file. Because of this, the input writer must be an [io.WriteSeeker], -// such as [*os.File]. -// -// The level of the resulting segment is the input level increased by one. -// Levels help the "eligible for compaction" cycle to find suitable compaction -// pairs. -type Compactor struct { - left, right *SegmentCursor - currentLevel uint16 - // Tells if deletions or keys without corresponding values - // can be removed from merged segment. - // (left segment is root (1st) one, keepTombstones is off for bucket) - cleanupDeletions bool - - w io.WriteSeeker - bufw *bufio.Writer - - scratchSpacePath string -} - -// NewCompactor from left (older) and right (newer) seeker. See [Compactor] for -// an explanation of what goes on under the hood, and why the input -// requirements are the way they are. -func NewCompactor(w io.WriteSeeker, - left, right *SegmentCursor, level uint16, - scratchSpacePath string, cleanupDeletions bool, -) *Compactor { - return &Compactor{ - left: left, - right: right, - w: w, - bufw: bufio.NewWriterSize(w, 256*1024), - currentLevel: level, - cleanupDeletions: cleanupDeletions, - scratchSpacePath: scratchSpacePath, - } -} - -// Do starts a compaction. See [Compactor] for an explanation of this process. -func (c *Compactor) Do() error { - if err := c.init(); err != nil { - return fmt.Errorf("init: %w", err) - } - - kis, err := c.writeNodes() - if err != nil { - return fmt.Errorf("write keys: %w", err) - } - - if err := c.writeIndexes(kis); err != nil { - return fmt.Errorf("write index: %w", err) - } - - // flush buffered, so we can safely seek on underlying writer - if err := c.bufw.Flush(); err != nil { - return fmt.Errorf("flush buffered: %w", err) - } - - var dataEnd uint64 = segmentindex.HeaderSize - if len(kis) > 0 { - dataEnd = uint64(kis[len(kis)-1].ValueEnd) - } - - if err := c.writeHeader(c.currentLevel, 0, 0, - dataEnd); err != nil { - return fmt.Errorf("write header: %w", err) - } - - return nil -} - -func (c *Compactor) init() error { - // write a dummy header, we don't know the contents of the actual header yet, - // we will seek to the beginning and overwrite the actual header at the very - // end - - if _, err := c.bufw.Write(make([]byte, segmentindex.HeaderSize)); err != nil { - return errors.Wrap(err, "write empty header") - } - - return nil -} - -// nodeCompactor is a helper type to improve the code structure of merging -// nodes in a compaction -type nodeCompactor struct { - left, right *SegmentCursor - keyLeft, keyRight []byte - valueLeft, valueRight BitmapLayer - output []segmentindex.Key - offset int - bufw *bufio.Writer - - cleanupDeletions bool - emptyBitmap *sroar.Bitmap -} - -func (c *Compactor) writeNodes() ([]segmentindex.Key, error) { - nc := &nodeCompactor{ - left: c.left, - right: c.right, - bufw: c.bufw, - cleanupDeletions: c.cleanupDeletions, - emptyBitmap: sroar.NewBitmap(), - } - - nc.init() - - if err := nc.loopThroughKeys(); err != nil { - return nil, err - } - - return nc.output, nil -} - -func (c *nodeCompactor) init() { - c.keyLeft, c.valueLeft, _ = c.left.First() - c.keyRight, c.valueRight, _ = c.right.First() - - // the (dummy) header was already written, this is our initial offset - c.offset = segmentindex.HeaderSize -} - -func (c *nodeCompactor) loopThroughKeys() error { - for { - if c.keyLeft == nil && c.keyRight == nil { - return nil - } - - if c.keysEqual() { - if err := c.mergeIdenticalKeys(); err != nil { - return err - } - } else if c.leftKeySmallerOrRightNotSet() { - if err := c.takeLeftKey(); err != nil { - return err - } - } else { - if err := c.takeRightKey(); err != nil { - return err - } - } - } -} - -func (c *nodeCompactor) keysEqual() bool { - return bytes.Equal(c.keyLeft, c.keyRight) -} - -func (c *nodeCompactor) leftKeySmallerOrRightNotSet() bool { - return (c.keyLeft != nil && bytes.Compare(c.keyLeft, c.keyRight) == -1) || c.keyRight == nil -} - -func (c *nodeCompactor) mergeIdenticalKeys() error { - layers := BitmapLayers{ - {Additions: c.valueLeft.Additions, Deletions: c.valueLeft.Deletions}, - {Additions: c.valueRight.Additions, Deletions: c.valueRight.Deletions}, - } - merged, err := layers.Merge() - if err != nil { - return fmt.Errorf("merge bitmap layers for identical keys: %w", err) - } - - if additions, deletions, skip := c.cleanupValues(merged.Additions, merged.Deletions); !skip { - sn, err := NewSegmentNode(c.keyRight, additions, deletions) - if err != nil { - return fmt.Errorf("new segment node for merged key: %w", err) - } - - ki, err := sn.KeyIndexAndWriteTo(c.bufw, c.offset) - if err != nil { - return fmt.Errorf("write individual node (merged key): %w", err) - } - - c.offset = ki.ValueEnd - c.output = append(c.output, ki) - } - - // advance both! - c.keyLeft, c.valueLeft, _ = c.left.Next() - c.keyRight, c.valueRight, _ = c.right.Next() - return nil -} - -func (c *nodeCompactor) takeLeftKey() error { - if additions, deletions, skip := c.cleanupValues(c.valueLeft.Additions, c.valueLeft.Deletions); !skip { - sn, err := NewSegmentNode(c.keyLeft, additions, deletions) - if err != nil { - return fmt.Errorf("new segment node for left key: %w", err) - } - - ki, err := sn.KeyIndexAndWriteTo(c.bufw, c.offset) - if err != nil { - return fmt.Errorf("write individual node (left key): %w", err) - } - - c.offset = ki.ValueEnd - c.output = append(c.output, ki) - } - - c.keyLeft, c.valueLeft, _ = c.left.Next() - return nil -} - -func (c *nodeCompactor) takeRightKey() error { - if additions, deletions, skip := c.cleanupValues(c.valueRight.Additions, c.valueRight.Deletions); !skip { - sn, err := NewSegmentNode(c.keyRight, additions, deletions) - if err != nil { - return fmt.Errorf("new segment node for right key: %w", err) - } - - ki, err := sn.KeyIndexAndWriteTo(c.bufw, c.offset) - if err != nil { - return fmt.Errorf("write individual node (right key): %w", err) - } - - c.offset = ki.ValueEnd - c.output = append(c.output, ki) - } - - c.keyRight, c.valueRight, _ = c.right.Next() - return nil -} - -func (c *nodeCompactor) cleanupValues(additions, deletions *sroar.Bitmap, -) (add, del *sroar.Bitmap, skip bool) { - if !c.cleanupDeletions { - return additions, deletions, false - } - if !additions.IsEmpty() { - return additions, c.emptyBitmap, false - } - return nil, nil, true -} - -func (c *Compactor) writeIndexes(keys []segmentindex.Key) error { - indexes := &segmentindex.Indexes{ - Keys: keys, - SecondaryIndexCount: 0, - ScratchSpacePath: c.scratchSpacePath, - } - - _, err := indexes.WriteTo(c.bufw) - return err -} - -// writeHeader assumes that everything has been written to the underlying -// writer and it is now safe to seek to the beginning and override the initial -// header -func (c *Compactor) writeHeader(level, version, secondaryIndices uint16, - startOfIndex uint64, -) error { - if _, err := c.w.Seek(0, io.SeekStart); err != nil { - return errors.Wrap(err, "seek to beginning to write header") - } - - h := &segmentindex.Header{ - Level: level, - Version: version, - SecondaryIndices: secondaryIndices, - Strategy: segmentindex.StrategyRoaringSet, - IndexStart: startOfIndex, - } - - if _, err := h.WriteTo(c.w); err != nil { - return err - } - - return nil -} diff --git a/adapters/repos/db/lsmkv/roaringset/compactor_test.go b/adapters/repos/db/lsmkv/roaringset/compactor_test.go deleted file mode 100644 index 0833536831a65d6ab24c82918036a7ce65f0323c..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/roaringset/compactor_test.go +++ /dev/null @@ -1,506 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package roaringset - -import ( - "io" - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv/segmentindex" -) - -func Test_Compactor(t *testing.T) { - type test struct { - name string - left []byte - right []byte - expected []keyWithBML - expectedRoot []keyWithBML - } - - tests := []test{ - { - name: "independent segments without overlap", - left: createSegmentsFromKeys(t, []keyWithBML{ - { - key: []byte("aaa"), - additions: []uint64{0}, - deletions: []uint64{1}, - }, - { - key: []byte("ccc"), - additions: []uint64{4}, - deletions: []uint64{5}, - }, - }), - right: createSegmentsFromKeys(t, []keyWithBML{ - { - key: []byte("bbb"), - additions: []uint64{2}, - deletions: []uint64{3}, - }, - { - key: []byte("ddd"), - additions: []uint64{6}, - deletions: []uint64{7}, - }, - }), - expected: []keyWithBML{ - { - key: []byte("aaa"), - additions: []uint64{0}, - deletions: []uint64{1}, - }, - { - key: []byte("bbb"), - additions: []uint64{2}, - deletions: []uint64{3}, - }, - { - key: []byte("ccc"), - additions: []uint64{4}, - deletions: []uint64{5}, - }, - { - key: []byte("ddd"), - additions: []uint64{6}, - deletions: []uint64{7}, - }, - }, - expectedRoot: []keyWithBML{ - { - key: []byte("aaa"), - additions: []uint64{0}, - }, - { - key: []byte("bbb"), - additions: []uint64{2}, - }, - { - key: []byte("ccc"), - additions: []uint64{4}, - }, - { - key: []byte("ddd"), - additions: []uint64{6}, - }, - }, - }, - { - name: "some segments overlap", - // note: there is no need to test every possible edge case for the - // overlapping segments in this place, as this logic is outsourced to - // BitmapLayer.Merge() which already has tests for edge cases - left: createSegmentsFromKeys(t, []keyWithBML{ - { - key: []byte("aaa"), - additions: []uint64{0}, - deletions: []uint64{1}, - }, - { - key: []byte("overlap"), - additions: []uint64{4, 5, 6}, - deletions: []uint64{1, 3, 7}, - }, - }), - right: createSegmentsFromKeys(t, []keyWithBML{ - { - key: []byte("overlap"), - additions: []uint64{3, 8}, - deletions: []uint64{5}, - }, - { - key: []byte("zzz"), - additions: []uint64{6}, - deletions: []uint64{7}, - }, - }), - expected: []keyWithBML{ - { - key: []byte("aaa"), - additions: []uint64{0}, - deletions: []uint64{1}, - }, - { - key: []byte("overlap"), - additions: []uint64{3, 4, 6, 8}, - deletions: []uint64{1, 5, 7}, - }, - { - key: []byte("zzz"), - additions: []uint64{6}, - deletions: []uint64{7}, - }, - }, - expectedRoot: []keyWithBML{ - { - key: []byte("aaa"), - additions: []uint64{0}, - }, - { - key: []byte("overlap"), - additions: []uint64{3, 4, 6, 8}, - }, - { - key: []byte("zzz"), - additions: []uint64{6}, - }, - }, - }, - { - name: "everything but one is deleted", - left: createSegmentsFromKeys(t, []keyWithBML{ - { - key: []byte("aaa"), - additions: []uint64{0}, - deletions: []uint64{}, - }, - { - key: []byte("bbb"), - additions: []uint64{4, 5, 6}, - deletions: []uint64{}, - }, - { - key: []byte("ddd"), - additions: []uint64{11, 12, 111}, - deletions: []uint64{}, - }, - }), - right: createSegmentsFromKeys(t, []keyWithBML{ - { - key: []byte("aaa"), - additions: []uint64{}, - deletions: []uint64{0}, - }, - { - key: []byte("bbb"), - additions: []uint64{}, - deletions: []uint64{4, 5, 6}, - }, - { - key: []byte("ccc"), - additions: []uint64{}, - deletions: []uint64{7, 8}, - }, - { - key: []byte("ddd"), - additions: []uint64{222}, - deletions: []uint64{11, 12, 13, 14}, - }, - }), - expected: []keyWithBML{ - { - key: []byte("aaa"), - additions: []uint64{}, - deletions: []uint64{0}, - }, - { - key: []byte("bbb"), - additions: []uint64{}, - deletions: []uint64{4, 5, 6}, - }, - { - key: []byte("ccc"), - additions: []uint64{}, - deletions: []uint64{7, 8}, - }, - { - key: []byte("ddd"), - additions: []uint64{111, 222}, - deletions: []uint64{11, 12, 13, 14}, - }, - }, - expectedRoot: []keyWithBML{ - { - key: []byte("ddd"), - additions: []uint64{111, 222}, - }, - }, - }, - - // the key loop is essentially a state machine. The next tests try to cover - // all possible states: - // - // 1. only the left key is set -> take left key - // 2. both left key and right key are set, but left is smaller -> take left - // key - // 3. only the right key is set -> take right key - // 4. both right and left keys are set, but right key is smaller -> take - // the right key - // 5. both keys are identical -> merge them - // - // Note: There is also an implicit 6th case: both keys are not set, this is - // the exit condition which is part of every test. - { - name: "state 1 - only left key is set", - left: createSegmentsFromKeys(t, []keyWithBML{ - { - key: []byte("aaa"), - additions: []uint64{0}, - deletions: []uint64{1}, - }, - }), - right: createSegmentsFromKeys(t, []keyWithBML{}), - expected: []keyWithBML{ - { - key: []byte("aaa"), - additions: []uint64{0}, - deletions: []uint64{1}, - }, - }, - expectedRoot: []keyWithBML{ - { - key: []byte("aaa"), - additions: []uint64{0}, - }, - }, - }, - { - name: "state 2 - left+right, left is smaller", - left: createSegmentsFromKeys(t, []keyWithBML{ - { - key: []byte("aaa"), - additions: []uint64{0}, - deletions: []uint64{1}, - }, - }), - right: createSegmentsFromKeys(t, []keyWithBML{ - { - key: []byte("bbb"), - additions: []uint64{2}, - deletions: []uint64{3}, - }, - }), - expected: []keyWithBML{ - { - key: []byte("aaa"), - additions: []uint64{0}, - deletions: []uint64{1}, - }, - { - key: []byte("bbb"), - additions: []uint64{2}, - deletions: []uint64{3}, - }, - }, - expectedRoot: []keyWithBML{ - { - key: []byte("aaa"), - additions: []uint64{0}, - }, - { - key: []byte("bbb"), - additions: []uint64{2}, - }, - }, - }, - { - name: "state 3 - only the right key is set", - left: createSegmentsFromKeys(t, []keyWithBML{}), - right: createSegmentsFromKeys(t, []keyWithBML{ - { - key: []byte("bbb"), - additions: []uint64{2}, - deletions: []uint64{3}, - }, - }), - expected: []keyWithBML{ - { - key: []byte("bbb"), - additions: []uint64{2}, - deletions: []uint64{3}, - }, - }, - expectedRoot: []keyWithBML{ - { - key: []byte("bbb"), - additions: []uint64{2}, - }, - }, - }, - { - name: "state 4 - left+right, right is smaller", - left: createSegmentsFromKeys(t, []keyWithBML{ - { - key: []byte("ccc"), - additions: []uint64{0}, - deletions: []uint64{1}, - }, - }), - right: createSegmentsFromKeys(t, []keyWithBML{ - { - key: []byte("bbb"), - additions: []uint64{2}, - deletions: []uint64{3}, - }, - }), - expected: []keyWithBML{ - { - key: []byte("bbb"), - additions: []uint64{2}, - deletions: []uint64{3}, - }, - { - key: []byte("ccc"), - additions: []uint64{0}, - deletions: []uint64{1}, - }, - }, - expectedRoot: []keyWithBML{ - { - key: []byte("bbb"), - additions: []uint64{2}, - }, - { - key: []byte("ccc"), - additions: []uint64{0}, - }, - }, - }, - { - name: "state 5 - left+right are identical", - left: createSegmentsFromKeys(t, []keyWithBML{ - { - key: []byte("aaa"), - additions: []uint64{0}, - deletions: []uint64{1}, - }, - }), - right: createSegmentsFromKeys(t, []keyWithBML{ - { - key: []byte("aaa"), - additions: []uint64{2}, - deletions: []uint64{3}, - }, - }), - expected: []keyWithBML{ - { - key: []byte("aaa"), - additions: []uint64{0, 2}, - deletions: []uint64{1, 3}, - }, - }, - expectedRoot: []keyWithBML{ - { - key: []byte("aaa"), - additions: []uint64{0, 2}, - }, - }, - }, - } - - for _, test := range tests { - t.Run("[keep]"+test.name, func(t *testing.T) { - dir := t.TempDir() - - leftCursor := NewSegmentCursor(test.left, nil) - rightCursor := NewSegmentCursor(test.right, nil) - - segmentFile := filepath.Join(dir, "result.db") - f, err := os.Create(segmentFile) - require.NoError(t, err) - - c := NewCompactor(f, leftCursor, rightCursor, 5, dir+"/scratch", false) - require.NoError(t, c.Do()) - - require.NoError(t, f.Close()) - - f, err = os.Open(segmentFile) - require.NoError(t, err) - - header, err := segmentindex.ParseHeader(f) - require.NoError(t, err) - - segmentBytes, err := io.ReadAll(f) - require.NoError(t, err) - - require.NoError(t, f.Close()) - - cu := NewSegmentCursor(segmentBytes[:header.IndexStart-segmentindex.HeaderSize], nil) - - i := 0 - for k, v, _ := cu.First(); k != nil; k, v, _ = cu.Next() { - assert.Equal(t, test.expected[i].key, k) - assert.Equal(t, test.expected[i].additions, v.Additions.ToArray()) - assert.Equal(t, test.expected[i].deletions, v.Deletions.ToArray()) - i++ - } - - assert.Equal(t, len(test.expected), i, "all expected keys must have been hit") - }) - } - - for _, test := range tests { - t.Run("[cleanup] "+test.name, func(t *testing.T) { - dir := t.TempDir() - - leftCursor := NewSegmentCursor(test.left, nil) - rightCursor := NewSegmentCursor(test.right, nil) - - segmentFile := filepath.Join(dir, "result.db") - f, err := os.Create(segmentFile) - require.NoError(t, err) - - c := NewCompactor(f, leftCursor, rightCursor, 5, dir+"/scratch", true) - require.NoError(t, c.Do()) - - require.NoError(t, f.Close()) - - f, err = os.Open(segmentFile) - require.NoError(t, err) - - header, err := segmentindex.ParseHeader(f) - require.NoError(t, err) - - segmentBytes, err := io.ReadAll(f) - require.NoError(t, err) - - require.NoError(t, f.Close()) - - cu := NewSegmentCursor(segmentBytes[:header.IndexStart-segmentindex.HeaderSize], nil) - - i := 0 - for k, v, _ := cu.First(); k != nil; k, v, _ = cu.Next() { - assert.Equal(t, test.expectedRoot[i].key, k) - assert.Equal(t, test.expectedRoot[i].additions, v.Additions.ToArray()) - assert.Empty(t, v.Deletions.ToArray()) - i++ - } - - assert.Equal(t, len(test.expectedRoot), i, "all expected keys must have been hit") - }) - } -} - -type keyWithBML struct { - key []byte - additions []uint64 - deletions []uint64 -} - -func createSegmentsFromKeys(t *testing.T, keys []keyWithBML) []byte { - out := []byte{} - - for _, k := range keys { - add := NewBitmap(k.additions...) - del := NewBitmap(k.deletions...) - sn, err := NewSegmentNode(k.key, add, del) - require.Nil(t, err) - out = append(out, sn.ToBuffer()...) - } - - return out -} diff --git a/adapters/repos/db/lsmkv/roaringset/cursor.go b/adapters/repos/db/lsmkv/roaringset/cursor.go deleted file mode 100644 index 5efc3002042a76d7086eb1a01a5daa2a7cd94a97..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/roaringset/cursor.go +++ /dev/null @@ -1,155 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package roaringset - -import ( - "bytes" - "errors" - "fmt" - - "github.com/weaviate/sroar" - "github.com/weaviate/weaviate/entities/lsmkv" -) - -type CombinedCursor struct { - cursors []InnerCursor - states []innerCursorState - keyOnly bool -} - -type InnerCursor interface { - First() ([]byte, BitmapLayer, error) - Next() ([]byte, BitmapLayer, error) - Seek(key []byte) ([]byte, BitmapLayer, error) -} - -type innerCursorState struct { - key []byte - layer BitmapLayer - err error -} - -// When keyOnly flag is set, only keys are returned by First/Next/Seek access methods, -// 2nd value returned is expected to be nil -// When keyOnly is not set, 2nd value is always bitmap. Returned bitmap can be empty (e.g. for Next call after last element was already returned) -func NewCombinedCursor(innerCursors []InnerCursor, keyOnly bool) *CombinedCursor { - return &CombinedCursor{cursors: innerCursors, keyOnly: keyOnly} -} - -func (c *CombinedCursor) First() ([]byte, *sroar.Bitmap) { - states := c.runAll(func(ic InnerCursor) ([]byte, BitmapLayer, error) { - return ic.First() - }) - return c.getResultFromStates(states) -} - -func (c *CombinedCursor) Next() ([]byte, *sroar.Bitmap) { - // fallback to First if no previous calls of First or Seek - if c.states == nil { - return c.First() - } - return c.getResultFromStates(c.states) -} - -func (c *CombinedCursor) Seek(key []byte) ([]byte, *sroar.Bitmap) { - states := c.runAll(func(ic InnerCursor) ([]byte, BitmapLayer, error) { - return ic.Seek(key) - }) - return c.getResultFromStates(states) -} - -type cursorRun func(ic InnerCursor) ([]byte, BitmapLayer, error) - -func (c *CombinedCursor) runAll(cursorRun cursorRun) []innerCursorState { - states := make([]innerCursorState, len(c.cursors)) - for id, ic := range c.cursors { - states[id] = c.createState(cursorRun(ic)) - } - return states -} - -func (c *CombinedCursor) createState(key []byte, layer BitmapLayer, err error) innerCursorState { - if errors.Is(err, lsmkv.NotFound) { - return innerCursorState{err: err} - } - if err != nil { - panic(fmt.Errorf("unexpected error: %w", err)) // TODO necessary? - } - state := innerCursorState{key: key} - state.layer = layer - - return state -} - -func (c *CombinedCursor) getResultFromStates(states []innerCursorState) ([]byte, *sroar.Bitmap) { - // NotFound is returned only by Seek call. - // If all cursors returned NotFound, combined Seek has no result, therefore inner cursors' states - // should not be updated to allow combined cursor to proceed with following Next calls - - key, ids, allNotFound := c.getCursorIdsWithLowestKey(states) - if !allNotFound { - c.states = states - } - layers := BitmapLayers{} - for _, id := range ids { - layers = append(layers, c.states[id].layer) - // forward cursors used in final result - c.states[id] = c.createState(c.cursors[id].Next()) - } - - if key == nil && c.keyOnly { - return nil, nil - } - - bm := layers.Flatten() - if key == nil { - return nil, bm - } - - if bm.IsEmpty() { - // all values deleted, skip key - return c.Next() - } - - // TODO remove keyOnly option, not used anyway - if !c.keyOnly { - return key, bm - } - return key, nil -} - -func (c *CombinedCursor) getCursorIdsWithLowestKey(states []innerCursorState) ([]byte, []int, bool) { - var lowestKey []byte - ids := []int{} - allNotFound := true - - for id, state := range states { - if errors.Is(state.err, lsmkv.NotFound) { - continue - } - allNotFound = false - if state.key == nil { - continue - } - if lowestKey == nil { - lowestKey = state.key - ids = []int{id} - } else if cmp := bytes.Compare(lowestKey, state.key); cmp > 0 { - lowestKey = state.key - ids = []int{id} - } else if cmp == 0 { - ids = append(ids, id) - } - } - - return lowestKey, ids, allNotFound -} diff --git a/adapters/repos/db/lsmkv/roaringset/cursor_test.go b/adapters/repos/db/lsmkv/roaringset/cursor_test.go deleted file mode 100644 index 5547ceb03ed916d1b3ef0b4af84bb6c42eb61904..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/roaringset/cursor_test.go +++ /dev/null @@ -1,408 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package roaringset - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestCombinedCursor(t *testing.T) { - bst1 := createBst(t, []bstIn{ - { - key: "aaa", - additions: []uint64{1}, - deletions: []uint64{}, - }, - { - key: "bbb", - additions: []uint64{22}, - deletions: []uint64{}, - }, - { - key: "ccc", - additions: []uint64{333}, - deletions: []uint64{}, - }, - { - key: "ddd", - additions: []uint64{4444}, - deletions: []uint64{}, - }, - }) - - bst2 := createBst(t, []bstIn{ - { - key: "aaa", - additions: []uint64{2, 3}, - deletions: []uint64{1}, - }, - { - key: "bbb", - additions: []uint64{33}, - deletions: []uint64{22}, - }, - { - key: "ggg", - additions: []uint64{7777777, 8888888}, - deletions: []uint64{}, - }, - }) - - bst3 := createBst(t, []bstIn{ - { - key: "bbb", - additions: []uint64{22}, - deletions: []uint64{}, - }, - { - key: "ccc", - additions: []uint64{}, - deletions: []uint64{333}, - }, - { - key: "eee", - additions: []uint64{55555, 66666}, - deletions: []uint64{}, - }, - { - key: "fff", - additions: []uint64{666666}, - deletions: []uint64{}, - }, - { - key: "hhh", - additions: []uint64{999999999}, - deletions: []uint64{111111111, 222222222, 333333333}, - }, - }) - - expected := []struct { - key string - values []uint64 - }{ - { // 0 - key: "aaa", - values: []uint64{2, 3}, - }, - { // 1 - key: "bbb", - values: []uint64{22, 33}, - }, - { // 2 - key: "ddd", - values: []uint64{4444}, - }, - { // 3 - key: "eee", - values: []uint64{55555, 66666}, - }, - { // 4 - key: "fff", - values: []uint64{666666}, - }, - { // 5 - key: "ggg", - values: []uint64{7777777, 8888888}, - }, - { // 6 - key: "hhh", - values: []uint64{999999999}, - }, - } - - t.Run("default cursor", func(t *testing.T) { - t.Run("start from beginning", func(t *testing.T) { - cursor := createCursor(t, bst1, bst2, bst3) - - key, bm := cursor.First() - - assert.Equal(t, []byte(expected[0].key), key) - assert.Equal(t, len(expected[0].values), bm.GetCardinality()) - for _, v := range expected[0].values { - assert.True(t, bm.Contains(v)) - } - }) - - t.Run("start from beginning and go through all", func(t *testing.T) { - cursor := createCursor(t, bst1, bst2, bst3) - - i := 0 // 1st match is "aaa" - for key, bm := cursor.First(); key != nil; key, bm = cursor.Next() { - assert.Equal(t, []byte(expected[i].key), key) - assert.Equal(t, len(expected[i].values), bm.GetCardinality()) - for _, v := range expected[i].values { - assert.True(t, bm.Contains(v)) - } - i++ - } - }) - - t.Run("start from beginning using Next and go through all", func(t *testing.T) { - cursor := createCursor(t, bst1, bst2, bst3) - - i := 0 // 1st match is "aaa" - for key, bm := cursor.Next(); key != nil; key, bm = cursor.Next() { - assert.Equal(t, []byte(expected[i].key), key) - assert.Equal(t, len(expected[i].values), bm.GetCardinality()) - for _, v := range expected[i].values { - assert.True(t, bm.Contains(v)) - } - i++ - } - }) - - t.Run("seek matching element and go through rest", func(t *testing.T) { - cursor := createCursor(t, bst1, bst2, bst3) - - i := 2 // 1st match is "ddd" - matching := []byte("ddd") - for key, bm := cursor.Seek(matching); key != nil; key, bm = cursor.Next() { - assert.Equal(t, []byte(expected[i].key), key) - assert.Equal(t, len(expected[i].values), bm.GetCardinality()) - for _, v := range expected[i].values { - assert.True(t, bm.Contains(v)) - } - i++ - } - }) - - t.Run("seek non-matching element and go through rest", func(t *testing.T) { - cursor := createCursor(t, bst1, bst2, bst3) - - i := 4 // 1st match is "fff" - nonMatching := []byte("efg") - for key, bm := cursor.Seek(nonMatching); key != nil; key, bm = cursor.Next() { - assert.Equal(t, []byte(expected[i].key), key) - assert.Equal(t, len(expected[i].values), bm.GetCardinality()) - for _, v := range expected[i].values { - assert.True(t, bm.Contains(v)) - } - i++ - } - }) - - t.Run("seek missing element", func(t *testing.T) { - cursor := createCursor(t, bst1, bst2, bst3) - - missing := []byte("lll") - key, bm := cursor.Seek(missing) - - assert.Nil(t, key) - assert.NotNil(t, bm) - assert.Equal(t, 0, bm.GetCardinality()) - }) - - t.Run("next after seek missing element does not change cursor's position", func(t *testing.T) { - cursor := createCursor(t, bst1, bst2, bst3) - - key1, _ := cursor.First() - - missing := []byte("lll") - cursor.Seek(missing) - - key2, _ := cursor.Next() - - assert.Equal(t, []byte("aaa"), key1) - assert.Equal(t, []byte("bbb"), key2) - }) - - t.Run("next after last is nil/empty", func(t *testing.T) { - cursor := createCursor(t, bst1, bst2, bst3) - - last := []byte("hhh") - cursor.Seek(last) - key, bm := cursor.Next() - - assert.Nil(t, key) - assert.NotNil(t, bm) - assert.Equal(t, 0, bm.GetCardinality()) - }) - - t.Run("first after final/empty next", func(t *testing.T) { - cursor := createCursor(t, bst1, bst2, bst3) - - last := []byte("hhh") - cursor.Seek(last) - cursor.Next() - key, bm := cursor.First() - - assert.Equal(t, []byte(expected[0].key), key) - assert.Equal(t, len(expected[0].values), bm.GetCardinality()) - for _, v := range expected[0].values { - assert.True(t, bm.Contains(v)) - } - }) - - t.Run("seek after final/empty next", func(t *testing.T) { - cursor := createCursor(t, bst1, bst2, bst3) - - last := []byte("hhh") - matching := []byte("eee") - cursor.Seek(last) - cursor.Next() - key, bm := cursor.Seek(matching) - - assert.Equal(t, []byte(expected[3].key), key) - assert.Equal(t, len(expected[3].values), bm.GetCardinality()) - for _, v := range expected[3].values { - assert.True(t, bm.Contains(v)) - } - }) - }) - - t.Run("cursor key only", func(t *testing.T) { - t.Run("start from beginning", func(t *testing.T) { - cursor := createCursorKeyOnly(t, bst1, bst2, bst3) - - key, bm := cursor.First() - - assert.Equal(t, []byte(expected[0].key), key) - assert.Nil(t, bm) - }) - - t.Run("start from beginning and go through all", func(t *testing.T) { - cursor := createCursorKeyOnly(t, bst1, bst2, bst3) - - i := 0 // 1st match is "aaa" - for key, bm := cursor.First(); key != nil; key, bm = cursor.Next() { - assert.Equal(t, []byte(expected[i].key), key) - assert.Nil(t, bm) - i++ - } - }) - - t.Run("start from beginning using Next and go through all", func(t *testing.T) { - cursor := createCursorKeyOnly(t, bst1, bst2, bst3) - - i := 0 // 1st match is "aaa" - for key, bm := cursor.Next(); key != nil; key, bm = cursor.Next() { - assert.Equal(t, []byte(expected[i].key), key) - assert.Nil(t, bm) - i++ - } - }) - - t.Run("seek matching element and go through rest", func(t *testing.T) { - cursor := createCursorKeyOnly(t, bst1, bst2, bst3) - - i := 2 // 1st match is "ddd" - matching := []byte("ddd") - for key, bm := cursor.Seek(matching); key != nil; key, bm = cursor.Next() { - assert.Equal(t, []byte(expected[i].key), key) - assert.Nil(t, bm) - i++ - } - }) - - t.Run("seek non-matching element and go through rest", func(t *testing.T) { - cursor := createCursorKeyOnly(t, bst1, bst2, bst3) - - i := 4 // 1st match is "fff" - nonMatching := []byte("efg") - for key, bm := cursor.Seek(nonMatching); key != nil; key, bm = cursor.Next() { - assert.Equal(t, []byte(expected[i].key), key) - assert.Nil(t, bm) - i++ - } - }) - - t.Run("seek missing element", func(t *testing.T) { - cursor := createCursorKeyOnly(t, bst1, bst2, bst3) - - missing := []byte("lll") - key, bm := cursor.Seek(missing) - - assert.Nil(t, key) - assert.Nil(t, bm) - }) - - t.Run("next after seek missing element does not change cursor's position", func(t *testing.T) { - cursor := createCursorKeyOnly(t, bst1, bst2, bst3) - - key1, _ := cursor.First() - - missing := []byte("lll") - cursor.Seek(missing) - - key2, _ := cursor.Next() - - assert.Equal(t, []byte("aaa"), key1) - assert.Equal(t, []byte("bbb"), key2) - }) - - t.Run("next after last is nil/empty", func(t *testing.T) { - cursor := createCursorKeyOnly(t, bst1, bst2, bst3) - - last := []byte("hhh") - cursor.Seek(last) - key, bm := cursor.Next() - - assert.Nil(t, key) - assert.Nil(t, bm) - }) - - t.Run("first after final/empty next", func(t *testing.T) { - cursor := createCursorKeyOnly(t, bst1, bst2, bst3) - - last := []byte("hhh") - cursor.Seek(last) - cursor.Next() - key, bm := cursor.First() - - assert.Equal(t, []byte(expected[0].key), key) - assert.Nil(t, bm) - }) - - t.Run("seek after final/empty next", func(t *testing.T) { - cursor := createCursorKeyOnly(t, bst1, bst2, bst3) - - last := []byte("hhh") - matching := []byte("eee") - cursor.Seek(last) - cursor.Next() - key, bm := cursor.Seek(matching) - - assert.Equal(t, []byte(expected[3].key), key) - assert.Nil(t, bm) - }) - }) -} - -type bstIn struct { - key string - additions []uint64 - deletions []uint64 -} - -func createBst(t *testing.T, in []bstIn) *BinarySearchTree { - bst := &BinarySearchTree{} - for i := range in { - bst.Insert([]byte(in[i].key), Insert{Additions: in[i].additions, Deletions: in[i].deletions}) - } - return bst -} - -func createCursor(t *testing.T, bsts ...*BinarySearchTree) *CombinedCursor { - innerCursors := []InnerCursor{} - for _, bst := range bsts { - innerCursors = append(innerCursors, NewBinarySearchTreeCursor(bst)) - } - return NewCombinedCursor(innerCursors, false) -} - -func createCursorKeyOnly(t *testing.T, bsts ...*BinarySearchTree) *CombinedCursor { - c := createCursor(t, bsts...) - c.keyOnly = true - return c -} diff --git a/adapters/repos/db/lsmkv/roaringset/doc.go b/adapters/repos/db/lsmkv/roaringset/doc.go deleted file mode 100644 index 843cad4bdca36b272a9d6abe1bc579ef70bee430..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/roaringset/doc.go +++ /dev/null @@ -1,85 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// The "roaringset" package contains all the LSM business logic that is unique -// to the "RoaringSet" strategy -// -// This package alone does not contain an entire LSM store. It's intended to be -// used as part of the [github.com/weaviate/weaviate/adapters/repos/db/lsmkv] package. -// -// # Motivation -// -// What makes the RoaringSet strategy unique is that it's essentially a fully -// persistent Roaring Bitmap that can be built up and updated incrementally -// (without write amplification) while being extremely fast to query. -// -// Without this specific strategy, it would not be efficient to use roaring -// bitmaps in an LSM store. For example: -// -// - Lucene uses posting lists in the inverted index on disk and supports -// converting them to a Roaring Bitmap at query time. This resulting bitmap -// can then be cached. However, the cost to initially convert a posting list -// to a roaring bitmap is quite huge. In our own tests, inserting 90M out of -// 100M possible ids into a [github.com/weaviate/sroar.Bitmap] takes about -// 3.5s. -// -// - You could store a regular roaring bitmap, such as -// [github.com/weaviate/sroar.Bitmap] in a regular LSM store, such as -// RocksDB. This would fix the retrieval issue and you should be able to -// retrieve and initialize a bitmap containing 90M objects in a few -// milliseconds. However, the cost to incrementally update this bitmap would -// be extreme. You would have to use a read-modify-write pattern which would -// lead to huge write-amplification on large setups. A 90M roaring bitmap -// is about 10.5MB, so to add a single entry (which would take up anywhere -// from 1 bit to 2 bytes), you would have to read 10.5MB and write 10.5MB -// again. That's not feasible except for bulk-loading. In Weaviate we cannot -// always assume bulk loading, as user behavior and insert orders are -// generally unpredictable. -// -// We solve this issue by making the LSM store roaring-bitmap-native. This way, -// we can keep the benefits of an LSM store (very fast writes) with the -// benefits of a serialized roaring bitmap (very fast reads/initializations). -// -// Essentially this means the RoaringSet strategy behaves like a fully -// persistent (and durable) Roaring Bitmap. See the next section to learn how -// it works under the hood. -// -// # Internals -// -// The public-facing methods make use of [github.com/weaviate/sroar.Bitmap]. -// This serialized bitmap already fulfills many of the criteria needed in -// Weaviate. It can be initialized at almost no cost (sharing memory) or very -// little cost (copying some memory). Furthermore, its set, remove, and -// intersection methods work well for the inverted index use cases in Weaviate. -// -// So, the novel part in the lsmkv.RoaringSet strategy does not sit in the -// roaring bitmap itself, but rather in the way it's persisted. It uses the -// standard principles of an LSM store where each new write is first cached in -// a memtable (and of course written into a Write-Ahead-Log to make it -// durable). The memtable is flushed into a disk segment when specific criteria -// are met (memtable size, WAL size, idle time, time since last flush, etc.). -// -// This means that each layer (represented by [BitmapLayer]) only contains the -// deltas that were written in a specific time interval. When reading, all -// layers must be combined into a single bitmap (see [BitmapLayers.Flatten]). -// -// Over time segments can be combined into fewer, larger segments using an LSM -// Compaction process. The logic for that can be found in [BitmapLayers.Merge]. -// -// To make sure access is efficient the entire RoaringSet strategy is built to -// avoid encoding/decoding steps. Instead we internally store data as simple -// byte slices. For example, see [SegmentNode]. You can access bitmaps without -// any meaningful allocations using [SegmentNode.Additions] and -// [SegmentNode.Deletions]. If you plan to hold on to the bitmap for a time -// window that is longer than holding a lock that prevents a compaction, you -// need to copy data (e.g. using [SegmentNode.AdditionsWithCopy]). Even with -// such a copy, reading a 90M-ids bitmap takes only single-digit milliseconds. -package roaringset diff --git a/adapters/repos/db/lsmkv/roaringset/helpers.go b/adapters/repos/db/lsmkv/roaringset/helpers.go deleted file mode 100644 index 37dd8ad65cc4d3648e42d2a7848da4dae796c371..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/roaringset/helpers.go +++ /dev/null @@ -1,41 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package roaringset - -import "github.com/weaviate/sroar" - -func NewBitmap(values ...uint64) *sroar.Bitmap { - bm := sroar.NewBitmap() - bm.SetMany(values) - return bm -} - -// Operations on bitmaps may result in oversized instances in relation to -// number of elements currently contained in bitmap -// Examples of such operations: -// - And-ing bitmaps may results in size being sum of both sizes -// (especially and-ing bitmap with itself) -// - Removing elements from bitmap results in size not being reduced -// (even if there is only few or no elements left) -// -// Method should be used before saving bitmap to file, to ensure -// minimal required size -// -// For most cases Or between empty bitmap and used bitmap -// works pretty well for reducing its final size, except for usecase, -// where used bitmap uses internally bitmap - it will not be converted -// to underlying array, even if there are single elements left -func Condense(bm *sroar.Bitmap) *sroar.Bitmap { - condensed := sroar.NewBitmap() - condensed.Or(bm) - return condensed -} diff --git a/adapters/repos/db/lsmkv/roaringset/helpers_test.go b/adapters/repos/db/lsmkv/roaringset/helpers_test.go deleted file mode 100644 index 7b8e6b35d5f04f3e11c3f8821178678e61e666bb..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/roaringset/helpers_test.go +++ /dev/null @@ -1,129 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package roaringset - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestCondense(t *testing.T) { - t.Run("And with itself (internal array)", func(t *testing.T) { - bm := NewBitmap(slice(0, 1000)...) - for i := 0; i < 10; i++ { - bm.And(bm) - } - bmLen := len(bm.ToBuffer()) - - condensed := Condense(bm) - condensedLen := len(condensed.ToBuffer()) - - assert.Greater(t, bmLen, condensedLen) - assert.ElementsMatch(t, bm.ToArray(), condensed.ToArray()) - }) - - t.Run("And with itself (internal bitmap)", func(t *testing.T) { - bm := NewBitmap(slice(0, 3000)...) - for i := 0; i < 10; i++ { - bm.And(bm) - } - bmLen := len(bm.ToBuffer()) - - condensed := Condense(bm) - condensedLen := len(condensed.ToBuffer()) - - assert.Greater(t, bmLen, condensedLen) - assert.ElementsMatch(t, bm.ToArray(), condensed.ToArray()) - }) - - t.Run("And (internal arrays)", func(t *testing.T) { - bm1 := NewBitmap(slice(0, 1000)...) - bm2 := NewBitmap(slice(500, 1500)...) - bm := bm1.Clone() - bm.And(bm2) - bmLen := len(bm.ToBuffer()) - - condensed := Condense(bm) - condensedLen := len(condensed.ToBuffer()) - - assert.Greater(t, bmLen, condensedLen) - assert.ElementsMatch(t, bm.ToArray(), condensed.ToArray()) - }) - - t.Run("And (internal bitmaps)", func(t *testing.T) { - bm1 := NewBitmap(slice(0, 4000)...) - bm2 := NewBitmap(slice(1000, 5000)...) - bm := bm1.Clone() - bm.And(bm2) - bmLen := len(bm.ToBuffer()) - - condensed := Condense(bm) - condensedLen := len(condensed.ToBuffer()) - - assert.Greater(t, bmLen, condensedLen) - assert.ElementsMatch(t, bm.ToArray(), condensed.ToArray()) - }) - - t.Run("And (internal bitmaps to bitmap with few elements)", func(t *testing.T) { - // this is not optimal. Internally elements will be stored in bitmap, - // though they would easily fit into array - bm1 := NewBitmap(slice(0, 4000)...) - bm2 := NewBitmap(slice(1000, 5000)...) - bm := bm1.Clone() - bm.And(bm2) - bmLen := len(bm.ToBuffer()) - - condensed := Condense(bm) - condensedLen := len(condensed.ToBuffer()) - - assert.Greater(t, bmLen, condensedLen) - assert.ElementsMatch(t, bm.ToArray(), condensed.ToArray()) - }) - - t.Run("Remove (array)", func(t *testing.T) { - bm := NewBitmap(slice(0, 1000)...) - for i := uint64(2); i < 1000; i++ { - bm.Remove(i) - } - bmLen := len(bm.ToBuffer()) - - condensed := Condense(bm) - condensedLen := len(condensed.ToBuffer()) - - assert.Greater(t, bmLen, condensedLen) - assert.ElementsMatch(t, bm.ToArray(), condensed.ToArray()) - }) - - t.Run("Remove (bitmap)", func(t *testing.T) { - bm := NewBitmap(slice(0, 100_000)...) - for i := uint64(10_000); i < 100_000; i++ { - bm.Remove(i) - } - bmLen := len(bm.ToBuffer()) - - condensed := Condense(bm) - condensedLen := len(condensed.ToBuffer()) - - assert.Greater(t, bmLen, condensedLen) - assert.ElementsMatch(t, bm.ToArray(), condensed.ToArray()) - }) -} - -func slice(from, to uint64) []uint64 { - len := to - from - s := make([]uint64, len) - for i := uint64(0); i < len; i++ { - s[i] = from + i - } - return s -} diff --git a/adapters/repos/db/lsmkv/roaringset/layers.go b/adapters/repos/db/lsmkv/roaringset/layers.go deleted file mode 100644 index 63eaf9b2b9c1f78823e2503495f0c070a21dbae2..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/roaringset/layers.go +++ /dev/null @@ -1,129 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package roaringset - -import ( - "fmt" - - "github.com/weaviate/sroar" -) - -// A BitmapLayer contains all the bitmap related delta-information stored for a -// specific key in one layer. A layer typically corresponds to one disk segment -// or a memtable layer -// -// A layer is essentially a snapshot in time and to get an accurate few of the -// set in its entirety multiple layers need to be combined using -// [BitmapLayers]. -// -// The contents of Additions and Deletions must be mutually exclusive. A layer -// cannot both add and delete an element. The only way to create new layers is -// through inserting into a Memtable. The memtable must make sure that: -// -// - When an element is added, any previous deletion of this element is -// removed -// - When an element is deleted, any previous addition of this element is -// removed. -// -// As a result, an element is either a net addition or a net deletion in a -// layer, but it can never be both. -type BitmapLayer struct { - Additions *sroar.Bitmap - Deletions *sroar.Bitmap -} - -func (l *BitmapLayer) Clone() BitmapLayer { - clone := BitmapLayer{} - if l.Additions != nil { - clone.Additions = l.Additions.Clone() - } - if l.Deletions != nil { - clone.Deletions = l.Deletions.Clone() - } - return clone -} - -// BitmapLayers are a helper type to perform operations on multiple layers, -// such as [BitmapLayers.Flatten] or [BitmapLayers.Merge]. -type BitmapLayers []BitmapLayer - -// Flatten reduces all snapshots into a single Bitmap. This bitmap no longer -// contains separate additions and deletions, but a single set where all -// additions and deletions have been applied in the correct order. -// -// If you do not wish to flatten all of history, but rather combine two layers, -// such as would happen in a Compaction, use [BitmapLayers.Merge] instead. -// -// Flatten is typically used when serving a specific key to the user: It -// flattens all disk segments, a currently flushing memtable if it exists, and -// the active memtable into a single bitmap. The final bitmap is returned to -// the user. -// -// # Flattening Logic -// -// - The first layer is seen as chronologically first. Deletions in the -// first layers are ignored, as there is nothing to be deleted. As a -// result, the additions of the first segment become the root state in the -// first iteration. -// - Any subsequent layer is merged into the root layer in the following way: -// Deletions remove any existing additions, Additions are added. -// - This process happens one layer at a time. This way delete-and-readd -// cycles are reflected correctly. For example, if layer 2 deletes an element -// X and layer 3 adds element X, then it is a net addition overall, and X -// should be represented in the final bitmap. If the order is reversed and -// layer 2 adds X, whereas layer 3 removes X, it is should not be contained -// in the final map. -func (bml BitmapLayers) Flatten() *sroar.Bitmap { - if len(bml) == 0 { - return sroar.NewBitmap() - } - - cur := bml[0] - // TODO: is this copy really needed? aren't we already operating on copied - // bms? - merged := cur.Additions.Clone() - - for i := 1; i < len(bml); i++ { - merged.AndNot(bml[i].Deletions) - merged.Or(bml[i].Additions) - } - - return merged -} - -// Merge turns two successive layers into one. It does not flatten the segment, -// but keeps additions and deletions separate. This is because there are no -// guarantees that the first segment was the root segment. A merge could run on -// segments 3+4 and they could contain deletions of elements that were added in -// segments 1 or 2. -// -// Merge is intended to be used as part of compactions. -func (bml BitmapLayers) Merge() (BitmapLayer, error) { - out := BitmapLayer{} - if len(bml) != 2 { - return out, fmt.Errorf("merge requires exactly two input segments") - } - - left, right := bml[0], bml[1] - - additions := left.Additions.Clone() - additions.Or(right.Additions) - additions.AndNot(right.Deletions) - - deletions := left.Deletions.Clone() - deletions.AndNot(right.Additions) - deletions.Or(right.Deletions) - - out.Additions = Condense(additions) - out.Deletions = Condense(deletions) - return out, nil -} diff --git a/adapters/repos/db/lsmkv/roaringset/layers_test.go b/adapters/repos/db/lsmkv/roaringset/layers_test.go deleted file mode 100644 index 261919fd705eeac8cbbc11cd018a99c3e6c20527..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/roaringset/layers_test.go +++ /dev/null @@ -1,328 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package roaringset - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/sroar" -) - -func Test_BitmapLayers_Flatten(t *testing.T) { - type inputSegment struct { - additions []uint64 - deletions []uint64 - } - - type test struct { - name string - inputs []inputSegment - expectedContained []uint64 - expectedNotContained []uint64 - } - - tests := []test{ - { - name: "no inputs", - inputs: nil, - expectedContained: nil, - expectedNotContained: nil, - }, - { - name: "single segment", - inputs: []inputSegment{ - { - additions: []uint64{4, 5}, - }, - }, - expectedContained: []uint64{4, 5}, - expectedNotContained: nil, - }, - { - name: "three segments, only additions", - inputs: []inputSegment{ - { - additions: []uint64{4, 5}, - }, - { - additions: []uint64{5, 6}, - }, - { - additions: []uint64{6, 7, 8}, - }, - }, - expectedContained: []uint64{4, 5, 6, 7, 8}, - expectedNotContained: nil, - }, - { - name: "two segments, including a delete", - inputs: []inputSegment{ - { - additions: []uint64{4, 5}, - }, - { - additions: []uint64{5, 6}, - deletions: []uint64{4}, - }, - }, - expectedContained: []uint64{5, 6}, - expectedNotContained: []uint64{4}, - }, - { - name: "three segments, including a delete, and a re-add", - inputs: []inputSegment{ - { - additions: []uint64{3, 4, 5}, - }, - { - additions: []uint64{6}, - deletions: []uint64{4, 5}, - }, - { - additions: []uint64{5}, - }, - }, - expectedContained: []uint64{3, 5, 6}, - expectedNotContained: []uint64{4}, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - input := make(BitmapLayers, len(test.inputs)) - for i, inp := range test.inputs { - input[i].Additions = NewBitmap(inp.additions...) - input[i].Deletions = NewBitmap(inp.deletions...) - } - - res := input.Flatten() - for _, x := range test.expectedContained { - assert.True(t, res.Contains(x)) - } - - for _, x := range test.expectedNotContained { - assert.False(t, res.Contains(x)) - } - }) - } -} - -func Test_BitmapLayers_Merge(t *testing.T) { - type inputSegment struct { - additions []uint64 - deletions []uint64 - } - - type test struct { - name string - inputs []inputSegment - expectedAdditions []uint64 - expectedDeletions []uint64 - expectErr bool - } - - tests := []test{ - { - name: "no inputs - should error", - inputs: nil, - expectedAdditions: nil, - expectedDeletions: nil, - expectErr: true, - }, - { - name: "single layer - should error", - inputs: []inputSegment{ - { - additions: []uint64{4, 5}, - }, - }, - expectedAdditions: nil, - expectedDeletions: nil, - expectErr: true, - }, - { - name: "three layers - should error", - inputs: []inputSegment{ - { - additions: []uint64{4, 5}, - }, - { - additions: []uint64{4, 5}, - }, - { - additions: []uint64{4, 5}, - }, - }, - expectedAdditions: nil, - expectedDeletions: nil, - expectErr: true, - }, - { - name: "two layers, only additions", - inputs: []inputSegment{ - { - additions: []uint64{4, 5}, - }, - { - additions: []uint64{5, 6, 7}, - }, - }, - expectedAdditions: []uint64{4, 5, 6, 7}, - expectedDeletions: nil, - }, - { - name: "additions and deletions without overlap", - inputs: []inputSegment{ - { - additions: []uint64{4, 5}, - deletions: []uint64{1, 2}, - }, - { - additions: []uint64{5, 6, 7}, - deletions: []uint64{2, 3}, - }, - }, - expectedAdditions: []uint64{4, 5, 6, 7}, - expectedDeletions: []uint64{1, 2, 3}, - }, - { - name: "previously deleted element, re-added", - inputs: []inputSegment{ - { - additions: []uint64{}, - deletions: []uint64{1, 2}, - }, - { - additions: []uint64{2}, - deletions: []uint64{}, - }, - }, - expectedAdditions: []uint64{2}, - expectedDeletions: []uint64{1}, - }, - { - name: "previously added element deleted later", - inputs: []inputSegment{ - { - additions: []uint64{3, 4}, - deletions: []uint64{}, - }, - { - additions: []uint64{}, - deletions: []uint64{3}, - }, - }, - expectedAdditions: []uint64{4}, - expectedDeletions: []uint64{3}, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - input := make(BitmapLayers, len(test.inputs)) - for i, inp := range test.inputs { - input[i].Additions = NewBitmap(inp.additions...) - input[i].Deletions = NewBitmap(inp.deletions...) - } - - res, err := input.Merge() - if test.expectErr { - require.NotNil(t, err) - return - } else { - require.Nil(t, err) - } - for _, x := range test.expectedAdditions { - assert.True(t, res.Additions.Contains(x)) - } - - for _, x := range test.expectedDeletions { - assert.True(t, res.Deletions.Contains(x)) - } - - intersect := sroar.And(res.Additions, res.Deletions) - assert.True(t, intersect.IsEmpty(), - "verify that additions and deletions never intersect") - }) - } -} - -func Test_BitmapLayer_Clone(t *testing.T) { - t.Run("cloning empty BitmapLayer", func(t *testing.T) { - layerEmpty := BitmapLayer{} - - cloned := layerEmpty.Clone() - - assert.Nil(t, cloned.Additions) - assert.Nil(t, cloned.Deletions) - }) - - t.Run("cloning partially inited BitmapLayer", func(t *testing.T) { - additions := NewBitmap(1) - deletions := NewBitmap(100) - - layerAdditions := BitmapLayer{Additions: additions} - layerDeletions := BitmapLayer{Deletions: deletions} - - clonedLayerAdditions := layerAdditions.Clone() - clonedLayerDeletions := layerDeletions.Clone() - additions.Remove(1) - deletions.Remove(100) - - assert.True(t, layerAdditions.Additions.IsEmpty()) - assert.ElementsMatch(t, []uint64{1}, clonedLayerAdditions.Additions.ToArray()) - assert.Nil(t, clonedLayerAdditions.Deletions) - - assert.True(t, layerDeletions.Deletions.IsEmpty()) - assert.Nil(t, clonedLayerDeletions.Additions) - assert.ElementsMatch(t, []uint64{100}, clonedLayerDeletions.Deletions.ToArray()) - }) - - t.Run("cloning fully inited BitmapLayer", func(t *testing.T) { - additions := NewBitmap(1) - deletions := NewBitmap(100) - - layer := BitmapLayer{Additions: additions, Deletions: deletions} - - clonedLayer := layer.Clone() - additions.Remove(1) - deletions.Remove(100) - - assert.True(t, layer.Additions.IsEmpty()) - assert.True(t, layer.Deletions.IsEmpty()) - assert.ElementsMatch(t, []uint64{1}, clonedLayer.Additions.ToArray()) - assert.ElementsMatch(t, []uint64{100}, clonedLayer.Deletions.ToArray()) - }) -} - -// This test aims to prevent a regression on -// https://github.com/weaviate/sroar/issues/1 -// found in Serialized Roaring Bitmaps library -func Test_BitmapLayers_Merge_PanicSliceBoundOutOfRange(t *testing.T) { - genSlice := func(fromInc, toExc uint64) []uint64 { - slice := []uint64{} - for i := fromInc; i < toExc; i++ { - slice = append(slice, i) - } - return slice - } - - leftLayer := BitmapLayer{Deletions: NewBitmap(genSlice(289_800, 290_100)...)} - rightLayer := BitmapLayer{Additions: NewBitmap(genSlice(290_000, 293_000)...)} - - failingDeletionsLayer, err := BitmapLayers{leftLayer, rightLayer}.Merge() - assert.Nil(t, err) - - assert.ElementsMatch(t, genSlice(289_800, 290_000), failingDeletionsLayer.Deletions.ToArray()) -} diff --git a/adapters/repos/db/lsmkv/roaringset/segment_cursor.go b/adapters/repos/db/lsmkv/roaringset/segment_cursor.go deleted file mode 100644 index cc93d979b25a88b2d788efcf64dd10ae7d4031a6..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/roaringset/segment_cursor.go +++ /dev/null @@ -1,69 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package roaringset - -import ( - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv/segmentindex" -) - -type Seeker interface { - Seek(key []byte) (segmentindex.Node, error) -} - -// A SegmentCursor iterates over all key-value pairs in a single disk segment. -// You can either start at the beginning using [*SegmentCursor.First] or start -// at an arbitrary key that you may find using [*SegmentCursor.Seek] -type SegmentCursor struct { - index Seeker - data []byte - nextOffset uint64 -} - -// NewSegmentCursor creates a cursor for a single disk segment. Make sure that -// the data buf is already sliced correctly to start at the payload, as calling -// [*SegmentCursor.First] will start reading at offset 0 relative to the passed -// in buffer. Similarly, the buffer may only contain payloads, as the buffer end -// is used to determine if more keys can be found. -// -// Therefore if the payload is part of a longer continuous buffer, the cursor -// should be initialized with data[payloadStartPos:payloadEndPos] -func NewSegmentCursor(data []byte, index Seeker) *SegmentCursor { - return &SegmentCursor{index: index, data: data, nextOffset: 0} -} - -func (c *SegmentCursor) Next() ([]byte, BitmapLayer, error) { - if c.nextOffset >= uint64(len(c.data)) { - return nil, BitmapLayer{}, nil - } - - sn := NewSegmentNodeFromBuffer(c.data[c.nextOffset:]) - c.nextOffset += sn.Len() - layer := BitmapLayer{ - Additions: sn.Additions(), - Deletions: sn.Deletions(), - } - return sn.PrimaryKey(), layer, nil -} - -func (c *SegmentCursor) First() ([]byte, BitmapLayer, error) { - c.nextOffset = 0 - return c.Next() -} - -func (c *SegmentCursor) Seek(key []byte) ([]byte, BitmapLayer, error) { - node, err := c.index.Seek(key) - if err != nil { - return nil, BitmapLayer{}, err - } - c.nextOffset = node.Start - return c.Next() -} diff --git a/adapters/repos/db/lsmkv/roaringset/segment_cursor_test.go b/adapters/repos/db/lsmkv/roaringset/segment_cursor_test.go deleted file mode 100644 index 4805c853a5950e452dec4a9d85329635ea0cd517..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/roaringset/segment_cursor_test.go +++ /dev/null @@ -1,117 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package roaringset - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv/segmentindex" -) - -func TestSegmentCursor(t *testing.T) { - seg, offsets := createDummySegment(t, 5) - - t.Run("starting from beginning", func(t *testing.T) { - c := NewSegmentCursor(seg, nil) - key, layer, err := c.First() - require.Nil(t, err) - assert.Equal(t, []byte("00000"), key) - assert.True(t, layer.Additions.Contains(0)) - assert.True(t, layer.Additions.Contains(1)) - assert.True(t, layer.Deletions.Contains(2)) - assert.True(t, layer.Deletions.Contains(3)) - }) - - t.Run("starting from beginning, page through all", func(t *testing.T) { - c := NewSegmentCursor(seg, nil) - it := uint64(0) - for key, layer, err := c.First(); key != nil; key, layer, err = c.Next() { - require.Nil(t, err) - assert.Equal(t, []byte(fmt.Sprintf("%05d", it)), key) - assert.True(t, layer.Additions.Contains(it*4)) - assert.True(t, layer.Additions.Contains(it*4+1)) - assert.True(t, layer.Deletions.Contains(it*4+2)) - assert.True(t, layer.Deletions.Contains(it*4+3)) - it++ - } - - assert.Equal(t, uint64(5), it) - }) - - t.Run("seek and iterate from there", func(t *testing.T) { - seeker := createDummySeeker(t, offsets, 3) - c := NewSegmentCursor(seg, seeker) - - // start on it 3 as this is where the seeker points us - it := uint64(3) - for key, layer, err := c.Seek([]byte("dummyseeker")); key != nil; key, layer, err = c.Next() { - require.Nil(t, err) - assert.Equal(t, []byte(fmt.Sprintf("%05d", it)), key) - assert.True(t, layer.Additions.Contains(it*4)) - assert.True(t, layer.Additions.Contains(it*4+1)) - assert.True(t, layer.Deletions.Contains(it*4+2)) - assert.True(t, layer.Deletions.Contains(it*4+3)) - it++ - } - - assert.Equal(t, uint64(5), it) - }) - - t.Run("seeker returns error", func(t *testing.T) { - seeker := createDummySeeker(t, offsets, 3) - seeker.err = fmt.Errorf("seek and fail") - c := NewSegmentCursor(seg, seeker) - - _, _, err := c.Seek([]byte("dummyseeker")) - require.NotNil(t, err) - assert.Contains(t, err.Error(), "seek and fail") - }) -} - -func createDummySegment(t *testing.T, count uint64) ([]byte, []uint64) { - out := []byte{} - offsets := []uint64{} - - for i := uint64(0); i < count; i++ { - key := []byte(fmt.Sprintf("%05d", i)) - add := NewBitmap(i*4, i*4+1) - del := NewBitmap(i*4+2, i*4+3) - sn, err := NewSegmentNode(key, add, del) - require.Nil(t, err) - offsets = append(offsets, uint64(len(out))) - out = append(out, sn.ToBuffer()...) - } - - return out, offsets -} - -func createDummySeeker(t *testing.T, offsets []uint64, pos int) *dummySeeker { - return &dummySeeker{offsets, pos, nil} -} - -type dummySeeker struct { - offsets []uint64 - pos int - err error -} - -// Seek returns the hard-coded pos that was set on init time, it ignores the -// key -func (s dummySeeker) Seek(key []byte) (segmentindex.Node, error) { - return segmentindex.Node{ - Start: s.offsets[s.pos], - End: s.offsets[s.pos+1], - }, s.err -} diff --git a/adapters/repos/db/lsmkv/roaringset/serialization.go b/adapters/repos/db/lsmkv/roaringset/serialization.go deleted file mode 100644 index 4ba52ab2176d284b1bde7162868cdd2ec13e2a37..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/roaringset/serialization.go +++ /dev/null @@ -1,202 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package roaringset - -import ( - "encoding/binary" - "fmt" - "io" - "math" - - "github.com/weaviate/sroar" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv/segmentindex" - "github.com/weaviate/weaviate/usecases/byteops" -) - -// SegmentNode stores one Key-Value pair (without its index) in -// the LSM Segment. It uses a single []byte internally. As a result there is -// no decode step required at runtime. Instead you can use -// -// - [*SegmentNode.Additions] -// - [*SegmentNode.AdditionsWithCopy] -// - [*SegmentNode.Deletions] -// - [*SegmentNode.DeletionsWithCopy] -// - [*SegmentNode.PrimaryKey] -// -// to access the contents. Those helpers in turn do not require a decoding -// step. The accessor methods that return Roaring Bitmaps only point to -// existing memory (methods without WithCopy suffix), or in the worst case copy -// one byte slice (methods with WithCopy suffix). -// -// This makes the SegmentNode very fast to access at query time, even when it -// contains a large amount of data. -// -// The internal structure of the data is: -// -// byte begin-start | description -// --------------------|----------------------------------------------------- -// 0-8 | uint64 indicating the total length of the node, -// | this is used in cursors to identify the next node. -// 8-16 | uint64 length indicator for additions sraor bm -> x -// 16-(x+16) | additions bitmap -// (x+16)-(x+24) | uint64 length indicator for deletions sroar bm -> y -// (x+24)-(x+y+24) | deletions bitmap -// (x+y+24)-(x+y+28) | uint32 length indicator for primary key length -> z -// (x+y+28)-(x+y+z+28) | primary key -type SegmentNode struct { - data []byte -} - -// Len indicates the total length of the [SegmentNode]. When reading multiple -// segments back-2-back, such as in a cursor situation, the offset of element -// (n+1) is the offset of element n + Len() -func (sn *SegmentNode) Len() uint64 { - return binary.LittleEndian.Uint64(sn.data[0:8]) -} - -// Additions returns the additions roaring bitmap with shared state. Only use -// this method if you can guarantee that you will only use it while holding a -// maintenance lock or can otherwise be sure that no compaction can occur. If -// you can't guarantee that, instead use [*SegmentNode.AdditionsWithCopy]. -func (sn *SegmentNode) Additions() *sroar.Bitmap { - rw := byteops.NewReadWriter(sn.data) - rw.MoveBufferToAbsolutePosition(8) - return sroar.FromBuffer(rw.ReadBytesFromBufferWithUint64LengthIndicator()) -} - -// AdditionsWithCopy returns the additions roaring bitmap without sharing state. It -// creates a copy of the underlying buffer. This is safe to use indefinitely, -// but much slower than [*SegmentNode.Additions] as it requires copying all the -// memory. If you know that you will only need the contents of the node for a -// duration of time where a lock is held that prevents compactions, it is more -// efficient to use [*SegmentNode.Additions]. -func (sn *SegmentNode) AdditionsWithCopy() *sroar.Bitmap { - rw := byteops.NewReadWriter(sn.data) - rw.MoveBufferToAbsolutePosition(8) - return sroar.FromBufferWithCopy(rw.ReadBytesFromBufferWithUint64LengthIndicator()) -} - -// Deletions returns the deletions roaring bitmap with shared state. Only use -// this method if you can guarantee that you will only use it while holding a -// maintenance lock or can otherwise be sure that no compaction can occur. If -// you can't guarantee that, instead use [*SegmentNode.DeletionsWithCopy]. -func (sn *SegmentNode) Deletions() *sroar.Bitmap { - rw := byteops.NewReadWriter(sn.data) - rw.MoveBufferToAbsolutePosition(8) - rw.DiscardBytesFromBufferWithUint64LengthIndicator() - return sroar.FromBuffer(rw.ReadBytesFromBufferWithUint64LengthIndicator()) -} - -// DeletionsWithCopy returns the deletions roaring bitmap without sharing state. It -// creates a copy of the underlying buffer. This is safe to use indefinitely, -// but much slower than [*SegmentNode.Deletions] as it requires copying all the -// memory. If you know that you will only need the contents of the node for a -// duration of time where a lock is held that prevents compactions, it is more -// efficient to use [*SegmentNode.Deletions]. -func (sn *SegmentNode) DeletionsWithCopy() *sroar.Bitmap { - rw := byteops.NewReadWriter(sn.data) - rw.MoveBufferToAbsolutePosition(8) - rw.DiscardBytesFromBufferWithUint64LengthIndicator() - return sroar.FromBufferWithCopy(rw.ReadBytesFromBufferWithUint64LengthIndicator()) -} - -func (sn *SegmentNode) PrimaryKey() []byte { - rw := byteops.NewReadWriter(sn.data) - rw.MoveBufferToAbsolutePosition(8) - rw.DiscardBytesFromBufferWithUint64LengthIndicator() - rw.DiscardBytesFromBufferWithUint64LengthIndicator() - return rw.ReadBytesFromBufferWithUint32LengthIndicator() -} - -func NewSegmentNode( - key []byte, additions, deletions *sroar.Bitmap, -) (*SegmentNode, error) { - if len(key) > math.MaxUint32 { - return nil, fmt.Errorf("key too long, max length is %d", math.MaxUint32) - } - - additionsBuf := additions.ToBuffer() - deletionsBuf := deletions.ToBuffer() - - // offset + 2*uint64 length indicators + uint32 length indicator + payloads - expectedSize := 8 + 8 + 8 + 4 + len(additionsBuf) + len(deletionsBuf) + len(key) - sn := SegmentNode{ - data: make([]byte, expectedSize), - } - - rw := byteops.NewReadWriter(sn.data) - - // reserve the first 8 bytes for the offset, which we will write at the very - // end - rw.MoveBufferPositionForward(8) - if err := rw.CopyBytesToBufferWithUint64LengthIndicator(additionsBuf); err != nil { - return nil, err - } - - if err := rw.CopyBytesToBufferWithUint64LengthIndicator(deletionsBuf); err != nil { - return nil, err - } - - if err := rw.CopyBytesToBufferWithUint32LengthIndicator(key); err != nil { - return nil, err - } - - offset := rw.Position - rw.MoveBufferToAbsolutePosition(0) - rw.WriteUint64(uint64(offset)) - - return &sn, nil -} - -// ToBuffer returns the internal buffer without copying data. Only use this, -// when you can be sure that it's safe to share the data, or create your own -// copy. -// -// It truncates the buffer at is own length, in case it was initialized with a -// long buffer that only had a beginning offset, but no end. Such a situation -// may occur with cursors. If we then returned the whole buffer and don't know -// what the caller plans on doing with the data, we risk passing around too -// much memory. Truncating at the length prevents this and has no other -// negative effects. -func (sn *SegmentNode) ToBuffer() []byte { - return sn.data[:sn.Len()] -} - -// NewSegmentNodeFromBuffer creates a new segment node by using the underlying -// buffer without copying data. Only use this when you can be sure that it's -// safe to share the data or create your own copy. -func NewSegmentNodeFromBuffer(buf []byte) *SegmentNode { - return &SegmentNode{data: buf} -} - -// KeyIndexAndWriteTo is a helper to flush a memtables full of SegmentNodes. It -// writes itself into the given writer and returns a [segmentindex.Key] with -// start and end indicators (respecting SegmentNode.Offset). Those keys can -// then be used to build an index for the nodes. The combination of index and -// node make up an LSM segment. -// -// RoaringSets do not support secondary keys, thus the segmentindex.Key will -// only ever contain a primary key. -func (sn *SegmentNode) KeyIndexAndWriteTo(w io.Writer, offset int) (segmentindex.Key, error) { - out := segmentindex.Key{} - - n, err := w.Write(sn.data) - if err != nil { - return out, err - } - - out.ValueStart = offset - out.ValueEnd = offset + n - out.Key = sn.PrimaryKey() - - return out, nil -} diff --git a/adapters/repos/db/lsmkv/roaringset/serialization_test.go b/adapters/repos/db/lsmkv/roaringset/serialization_test.go deleted file mode 100644 index 5c853971a4d71f22839b18a9758a233d77521677..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/roaringset/serialization_test.go +++ /dev/null @@ -1,115 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package roaringset - -import ( - "bytes" - "math" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestSerialization_HappyPath(t *testing.T) { - additions := NewBitmap(1, 2, 3, 4, 6) - deletions := NewBitmap(5, 7) - key := []byte("my-key") - - sn, err := NewSegmentNode(key, additions, deletions) - require.Nil(t, err) - - buf := sn.ToBuffer() - assert.Equal(t, sn.Len(), uint64(len(buf))) - - newSN := NewSegmentNodeFromBuffer(buf) - assert.Equal(t, newSN.Len(), uint64(len(buf))) - - // without copying - newAdditions := newSN.Additions() - assert.True(t, newAdditions.Contains(4)) - assert.False(t, newAdditions.Contains(5)) - newDeletions := newSN.Deletions() - assert.False(t, newDeletions.Contains(4)) - assert.True(t, newDeletions.Contains(5)) - assert.Equal(t, []byte("my-key"), newSN.PrimaryKey()) - - // with copying - newAdditions = newSN.AdditionsWithCopy() - assert.True(t, newAdditions.Contains(4)) - assert.False(t, newAdditions.Contains(5)) - newDeletions = newSN.DeletionsWithCopy() - assert.False(t, newDeletions.Contains(4)) - assert.True(t, newDeletions.Contains(5)) -} - -func TestSerialization_InitializingFromBufferTooLarge(t *testing.T) { - additions := NewBitmap(1, 2, 3, 4, 6) - deletions := NewBitmap(5, 7) - key := []byte("my-key") - - sn, err := NewSegmentNode(key, additions, deletions) - require.Nil(t, err) - - buf := sn.ToBuffer() - assert.Equal(t, sn.Len(), uint64(len(buf))) - - bufTooLarge := make([]byte, 3*len(buf)) - copy(bufTooLarge, buf) - - newSN := NewSegmentNodeFromBuffer(bufTooLarge) - // assert that the buffer self reports the useful length, not the length of - // the initialization buffer - assert.Equal(t, newSN.Len(), uint64(len(buf))) - // assert that ToBuffer() returns a buffer that is no longer than the useful - // length - assert.Equal(t, len(buf), len(newSN.ToBuffer())) -} - -func TestSerialization_UnhappyPath(t *testing.T) { - t.Run("with primary key that's too long", func(t *testing.T) { - key := make([]byte, math.MaxUint32+3) - _, err := NewSegmentNode(key, nil, nil) - - require.NotNil(t, err) - assert.Contains(t, err.Error(), "key too long") - }) -} - -func TestSerialization_KeyIndexAndWriteTo(t *testing.T) { - buf := &bytes.Buffer{} - offset := 7 - // write some dummy data, so we have an offset - buf.Write(make([]byte, offset)) - - additions := NewBitmap(1, 2, 3, 4, 6) - deletions := NewBitmap(5, 7) - key := []byte("my-key") - - sn, err := NewSegmentNode(key, additions, deletions) - require.Nil(t, err) - - keyIndex, err := sn.KeyIndexAndWriteTo(buf, offset) - require.Nil(t, err) - - res := buf.Bytes() - assert.Equal(t, keyIndex.ValueEnd, len(res)) - - newSN := NewSegmentNodeFromBuffer(res[keyIndex.ValueStart:keyIndex.ValueEnd]) - newAdditions := newSN.Additions() - assert.True(t, newAdditions.Contains(4)) - assert.False(t, newAdditions.Contains(5)) - newDeletions := newSN.Deletions() - assert.False(t, newDeletions.Contains(4)) - assert.True(t, newDeletions.Contains(5)) - assert.Equal(t, []byte("my-key"), newSN.PrimaryKey()) -} diff --git a/adapters/repos/db/lsmkv/segment.go b/adapters/repos/db/lsmkv/segment.go deleted file mode 100644 index 491b1e4a6ffc570923b78a9d6e13c6cd8b8e335f..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/segment.go +++ /dev/null @@ -1,278 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "bufio" - "bytes" - "fmt" - "io" - "os" - - "github.com/edsrzf/mmap-go" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv/segmentindex" - "github.com/weaviate/weaviate/entities/lsmkv" - "github.com/willf/bloom" -) - -type segment struct { - path string - level uint16 - secondaryIndexCount uint16 - version uint16 - segmentStartPos uint64 - segmentEndPos uint64 - dataStartPos uint64 - dataEndPos uint64 - contents []byte - contentFile *os.File - strategy segmentindex.Strategy - index diskIndex - secondaryIndices []diskIndex - logger logrus.FieldLogger - metrics *Metrics - size int64 - mmapContents bool - - useBloomFilter bool // see bucket for more datails - bloomFilter *bloom.BloomFilter - secondaryBloomFilters []*bloom.BloomFilter - bloomFilterMetrics *bloomFilterMetrics - - // the net addition this segment adds with respect to all previous segments - calcCountNetAdditions bool // see bucket for more datails - countNetAdditions int -} - -type diskIndex interface { - // Get return lsmkv.NotFound in case no node can be found - Get(key []byte) (segmentindex.Node, error) - - // Seek returns lsmkv.NotFound in case the seek value is larger than - // the highest value in the collection, otherwise it returns the next highest - // value (or the exact value if present) - Seek(key []byte) (segmentindex.Node, error) - - // AllKeys in no specific order, e.g. for building a bloom filter - AllKeys() ([][]byte, error) - - // Size of the index in bytes - Size() int -} - -func newSegment(path string, logger logrus.FieldLogger, metrics *Metrics, - existsLower existsOnLowerSegmentsFn, mmapContents bool, - useBloomFilter bool, calcCountNetAdditions bool, -) (*segment, error) { - file, err := os.Open(path) - if err != nil { - return nil, fmt.Errorf("open file: %w", err) - } - - fileInfo, err := file.Stat() - if err != nil { - return nil, fmt.Errorf("stat file: %w", err) - } - - contents, err := mmap.MapRegion(file, int(fileInfo.Size()), mmap.RDONLY, 0, 0) - if err != nil { - return nil, fmt.Errorf("mmap file: %w", err) - } - - header, err := segmentindex.ParseHeader(bytes.NewReader(contents[:segmentindex.HeaderSize])) - if err != nil { - return nil, fmt.Errorf("parse header: %w", err) - } - - switch header.Strategy { - case segmentindex.StrategyReplace, segmentindex.StrategySetCollection, - segmentindex.StrategyMapCollection, segmentindex.StrategyRoaringSet: - default: - return nil, fmt.Errorf("unsupported strategy in segment") - } - - primaryIndex, err := header.PrimaryIndex(contents) - if err != nil { - return nil, fmt.Errorf("extract primary index position: %w", err) - } - - primaryDiskIndex := segmentindex.NewDiskTree(primaryIndex) - - seg := &segment{ - level: header.Level, - path: path, - contents: contents, - version: header.Version, - secondaryIndexCount: header.SecondaryIndices, - segmentStartPos: header.IndexStart, - segmentEndPos: uint64(fileInfo.Size()), - strategy: header.Strategy, - dataStartPos: segmentindex.HeaderSize, // fixed value that's the same for all strategies - dataEndPos: header.IndexStart, - index: primaryDiskIndex, - logger: logger, - metrics: metrics, - size: fileInfo.Size(), - mmapContents: mmapContents, - useBloomFilter: useBloomFilter, - calcCountNetAdditions: calcCountNetAdditions, - } - - // Using pread strategy requires file to remain open for segment lifetime - if seg.mmapContents { - defer file.Close() - } else { - seg.contentFile = file - } - - if seg.secondaryIndexCount > 0 { - seg.secondaryIndices = make([]diskIndex, seg.secondaryIndexCount) - for i := range seg.secondaryIndices { - secondary, err := header.SecondaryIndex(contents, uint16(i)) - if err != nil { - return nil, fmt.Errorf("get position for secondary index at %d: %w", i, err) - } - seg.secondaryIndices[i] = segmentindex.NewDiskTree(secondary) - } - } - - if seg.useBloomFilter { - if err := seg.initBloomFilters(metrics); err != nil { - return nil, err - } - } - if seg.calcCountNetAdditions { - if err := seg.initCountNetAdditions(existsLower); err != nil { - return nil, err - } - } - - return seg, nil -} - -func (s *segment) close() error { - var munmapErr, fileCloseErr error - - m := mmap.MMap(s.contents) - munmapErr = m.Unmap() - if s.contentFile != nil { - fileCloseErr = s.contentFile.Close() - } - - if munmapErr != nil || fileCloseErr != nil { - return fmt.Errorf("close segment: munmap: %v, close contents file: %w", munmapErr, fileCloseErr) - } - - return nil -} - -func (s *segment) drop() error { - // support for persisting bloom filters and cnas was added in v1.17, - // therefore the files may not be present on segments created with previous - // versions. By using RemoveAll, which does not error on NotExists, these - // drop calls are backward-compatible: - if err := os.RemoveAll(s.bloomFilterPath()); err != nil { - return fmt.Errorf("drop bloom filter: %w", err) - } - - for i := 0; i < int(s.secondaryIndexCount); i++ { - if err := os.RemoveAll(s.bloomFilterSecondaryPath(i)); err != nil { - return fmt.Errorf("drop bloom filter: %w", err) - } - } - - if err := os.RemoveAll(s.countNetPath()); err != nil { - return fmt.Errorf("drop count net additions file: %w", err) - } - - // for the segment itself, we're not using RemoveAll, but Remove. If there - // was a NotExists error here, something would be seriously wrong, and we - // don't want to ignore it. - if err := os.Remove(s.path); err != nil { - return fmt.Errorf("drop segment: %w", err) - } - - return nil -} - -// Size returns the total size of the segment in bytes, including the header -// and index -func (s *segment) Size() int { - return int(s.size) -} - -// PayloadSize is only the payload of the index, excluding the index -func (s *segment) PayloadSize() int { - return int(s.dataEndPos) -} - -type nodeReader struct { - r io.Reader -} - -func (n *nodeReader) Read(b []byte) (int, error) { - return n.r.Read(b) -} - -type nodeOffset struct { - start, end uint64 -} - -func (s *segment) newNodeReader(offset nodeOffset) (*nodeReader, error) { - var ( - r io.Reader - err error - ) - if s.mmapContents { - contents := s.contents[offset.start:] - if offset.end != 0 { - contents = s.contents[offset.start:offset.end] - } - r, err = s.bytesReaderFrom(contents) - } else { - r, err = s.bufferedReaderAt(offset.start) - } - if err != nil { - return nil, fmt.Errorf("new nodeReader: %w", err) - } - return &nodeReader{r: r}, nil -} - -func (s *segment) copyNode(b []byte, offset nodeOffset) error { - if s.mmapContents { - copy(b, s.contents[offset.start:offset.end]) - return nil - } - n, err := s.newNodeReader(offset) - if err != nil { - return fmt.Errorf("copy node: %w", err) - } - _, err = n.Read(b) - return err -} - -func (s *segment) bytesReaderFrom(in []byte) (*bytes.Reader, error) { - if len(in) == 0 { - return nil, lsmkv.NotFound - } - return bytes.NewReader(in), nil -} - -func (s *segment) bufferedReaderAt(offset uint64) (*bufio.Reader, error) { - if s.contentFile == nil { - return nil, fmt.Errorf("nil contentFile for segment at %s", s.path) - } - - r := io.NewSectionReader(s.contentFile, int64(offset), s.size) - return bufio.NewReader(r), nil -} diff --git a/adapters/repos/db/lsmkv/segment_bloom_filters.go b/adapters/repos/db/lsmkv/segment_bloom_filters.go deleted file mode 100644 index f9778be8a8f5d0b6a45c7b3e33315529dfa58b1c..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/segment_bloom_filters.go +++ /dev/null @@ -1,351 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "bytes" - "encoding/binary" - "fmt" - "hash/crc32" - "os" - "path/filepath" - "strings" - "time" - - "github.com/willf/bloom" -) - -func (s *segment) bloomFilterPath() string { - extless := strings.TrimSuffix(s.path, filepath.Ext(s.path)) - return fmt.Sprintf("%s.bloom", extless) -} - -func (s *segment) bloomFilterSecondaryPath(pos int) string { - extless := strings.TrimSuffix(s.path, filepath.Ext(s.path)) - return fmt.Sprintf("%s.secondary.%d.bloom", extless, pos) -} - -func (s *segment) initBloomFilters(metrics *Metrics) error { - if err := s.initBloomFilter(); err != nil { - return fmt.Errorf("init bloom filter for primary index: %w", err) - } - if s.secondaryIndexCount > 0 { - s.secondaryBloomFilters = make([]*bloom.BloomFilter, s.secondaryIndexCount) - for i := range s.secondaryBloomFilters { - if err := s.initSecondaryBloomFilter(i); err != nil { - return fmt.Errorf("init bloom filter for secondary index at %d: %w", i, err) - } - } - } - s.bloomFilterMetrics = newBloomFilterMetrics(metrics) - return nil -} - -func (s *segment) initBloomFilter() error { - path := s.bloomFilterPath() - ok, err := fileExists(path) - if err != nil { - return err - } - - if ok { - err = s.loadBloomFilterFromDisk() - if err == nil { - return nil - } - - if err != ErrInvalidChecksum { - // not a recoverable error - return err - } - - // now continue re-calculating - } - - before := time.Now() - if err := s.computeAndStoreBloomFilter(path); err != nil { - return err - } - - took := time.Since(before) - s.logger.WithField("action", "lsm_init_disk_segment_build_bloom_filter_primary"). - WithField("path", s.path). - WithField("took", took). - Debugf("building bloom filter took %s\n", took) - return nil -} - -func (s *segment) computeAndStoreBloomFilter(path string) error { - keys, err := s.index.AllKeys() - if err != nil { - return err - } - - s.bloomFilter = bloom.NewWithEstimates(uint(len(keys)), 0.001) - for _, key := range keys { - s.bloomFilter.Add(key) - } - - if err := s.storeBloomFilterOnDisk(path); err != nil { - return fmt.Errorf("store bloom filter on disk: %w", err) - } - - return nil -} - -func (s *segment) precomputeBloomFilters() ([]string, error) { - out := []string{} - - if err := s.precomputeBloomFilter(); err != nil { - return nil, fmt.Errorf("precompute bloom filter for primary index: %w", err) - } - out = append(out, fmt.Sprintf("%s.tmp", s.bloomFilterPath())) - - if s.secondaryIndexCount > 0 { - s.secondaryBloomFilters = make([]*bloom.BloomFilter, s.secondaryIndexCount) - for i := range s.secondaryBloomFilters { - if err := s.precomputeSecondaryBloomFilter(i); err != nil { - return nil, fmt.Errorf("precompute bloom filter for secondary index at %d: %w", i, err) - } - out = append(out, fmt.Sprintf("%s.tmp", s.bloomFilterSecondaryPath(i))) - } - } - return out, nil -} - -func (s *segment) precomputeBloomFilter() error { - before := time.Now() - - path := fmt.Sprintf("%s.tmp", s.bloomFilterPath()) - ok, err := fileExists(path) - if err != nil { - return err - } - - if ok { - return fmt.Errorf("a bloom filter already exists with path %s", path) - } - - if err := s.computeAndStoreBloomFilter(path); err != nil { - return err - } - - took := time.Since(before) - s.logger.WithField("action", "lsm_precompute_disk_segment_build_bloom_filter_primary"). - WithField("path", s.path). - WithField("took", took). - Debugf("building bloom filter took %s\n", took) - - return nil -} - -func (s *segment) storeBloomFilterOnDisk(path string) error { - buf := new(bytes.Buffer) - - _, err := s.bloomFilter.WriteTo(buf) - if err != nil { - return fmt.Errorf("write bloom filter: %w", err) - } - - return writeWithChecksum(buf.Bytes(), path) -} - -func (s *segment) loadBloomFilterFromDisk() error { - data, err := loadWithChecksum(s.bloomFilterPath(), -1) - if err != nil { - return err - } - - s.bloomFilter = new(bloom.BloomFilter) - _, err = s.bloomFilter.ReadFrom(bytes.NewReader(data)) - if err != nil { - return fmt.Errorf("read bloom filter from disk: %w", err) - } - - return nil -} - -func (s *segment) initSecondaryBloomFilter(pos int) error { - before := time.Now() - - path := s.bloomFilterSecondaryPath(pos) - ok, err := fileExists(path) - if err != nil { - return err - } - - if ok { - err = s.loadBloomFilterSecondaryFromDisk(pos) - if err == nil { - return nil - } - - if err != ErrInvalidChecksum { - // not a recoverable error - return err - } - - // now continue re-calculating - } - - if err := s.computeAndStoreSecondaryBloomFilter(path, pos); err != nil { - return err - } - - took := time.Since(before) - - s.logger.WithField("action", "lsm_init_disk_segment_build_bloom_filter_secondary"). - WithField("secondary_index_position", pos). - WithField("path", s.path). - WithField("took", took). - Debugf("building bloom filter took %s\n", took) - return nil -} - -func (s *segment) computeAndStoreSecondaryBloomFilter(path string, pos int) error { - keys, err := s.secondaryIndices[pos].AllKeys() - if err != nil { - return err - } - - s.secondaryBloomFilters[pos] = bloom.NewWithEstimates(uint(len(keys)), 0.001) - for _, key := range keys { - s.secondaryBloomFilters[pos].Add(key) - } - - if err := s.storeBloomFilterSecondaryOnDisk(path, pos); err != nil { - return fmt.Errorf("store secondary bloom filter on disk: %w", err) - } - - return nil -} - -func (s *segment) precomputeSecondaryBloomFilter(pos int) error { - before := time.Now() - path := fmt.Sprintf("%s.tmp", s.bloomFilterSecondaryPath(pos)) - - ok, err := fileExists(path) - if err != nil { - return err - } - - if ok { - return fmt.Errorf("a secondary bloom filter already exists with path %s", path) - } - - if err := s.computeAndStoreSecondaryBloomFilter(path, pos); err != nil { - return err - } - - took := time.Since(before) - s.logger.WithField("action", "lsm_precompute_disk_segment_build_bloom_filter_secondary"). - WithField("secondary_index_position", pos). - WithField("path", s.path). - WithField("took", took). - Debugf("building bloom filter took %s\n", took) - return nil -} - -func (s *segment) storeBloomFilterSecondaryOnDisk(path string, pos int) error { - buf := new(bytes.Buffer) - _, err := s.secondaryBloomFilters[pos].WriteTo(buf) - if err != nil { - return fmt.Errorf("write bloom filter: %w", err) - } - - return writeWithChecksum(buf.Bytes(), path) -} - -func (s *segment) loadBloomFilterSecondaryFromDisk(pos int) error { - data, err := loadWithChecksum(s.bloomFilterSecondaryPath(pos), -1) - if err != nil { - return err - } - - s.secondaryBloomFilters[pos] = new(bloom.BloomFilter) - _, err = s.secondaryBloomFilters[pos].ReadFrom(bytes.NewReader(data)) - if err != nil { - return fmt.Errorf("read bloom filter from disk: %w", err) - } - - return nil -} - -func writeWithChecksum(data []byte, path string) error { - chksm := crc32.ChecksumIEEE(data) - - f, err := os.Create(path) - if err != nil { - return fmt.Errorf("open file for writing: %w", err) - } - - if err := binary.Write(f, binary.LittleEndian, chksm); err != nil { - // ignoring f.Close() error here, as we don't care about whether the file - // was flushed, the call is mainly intended to prevent a file descriptor - // leak. We still want to return the original error below. - f.Close() - return fmt.Errorf("write checkusm to file: %w", err) - } - - if _, err := f.Write(data); err != nil { - // ignoring f.Close() error here, as we don't care about whether the file - // was flushed, the call is mainly intended to prevent a file descriptor - // leak. We still want to return the original error below. - f.Close() - return fmt.Errorf("write bloom filter to disk: %w", err) - } - - if err := f.Close(); err != nil { - return fmt.Errorf("close bloom filter file: %w", err) - } - - return nil -} - -// use negative length check to indicate that no length check should be -// performed -func loadWithChecksum(path string, lengthCheck int) ([]byte, error) { - f, err := os.Open(path) - if err != nil { - return nil, fmt.Errorf("open file for reading: %w", err) - } - - buf := new(bytes.Buffer) - if _, err := buf.ReadFrom(f); err != nil { - // ignoring f.Close() error here, as we don't care about whether the file - // was flushed, the call is mainly intended to prevent a file descriptor - // leak. We still want to return the original error below. - f.Close() - return nil, fmt.Errorf("read bloom filter data from file: %w", err) - } - if err := f.Close(); err != nil { - return nil, fmt.Errorf("close bloom filter file: %w", err) - } - - data := buf.Bytes() - if lengthCheck > 0 && len(data) != lengthCheck { - return nil, ErrInvalidChecksum - } - - if len(data) < 4 { - // the file does not even contain the full checksum, we must consider it corrupt - return nil, ErrInvalidChecksum - } - - chcksm := binary.LittleEndian.Uint32(data[:4]) - actual := crc32.ChecksumIEEE(data[4:]) - if chcksm != actual { - return nil, ErrInvalidChecksum - } - - return data[4:], nil -} diff --git a/adapters/repos/db/lsmkv/segment_bloom_filters_test.go b/adapters/repos/db/lsmkv/segment_bloom_filters_test.go deleted file mode 100644 index cbc42bfd8c32bd7ba33e43673bb61bc36fcf01d7..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/segment_bloom_filters_test.go +++ /dev/null @@ -1,553 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "context" - "io" - "os" - "path" - "testing" - - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/cyclemanager" -) - -func TestCreateBloomOnFlush(t *testing.T) { - ctx := context.Background() - dirName := t.TempDir() - - logger, _ := test.NewNullLogger() - - b, err := NewBucket(ctx, dirName, "", logger, nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - WithStrategy(StrategyReplace), WithSecondaryIndices(1)) - require.Nil(t, err) - - require.Nil(t, b.Put([]byte("hello"), []byte("world"), - WithSecondaryKey(0, []byte("bonjour")))) - require.Nil(t, b.FlushMemtable()) - - files, err := os.ReadDir(dirName) - require.Nil(t, err) - - _, ok := findFileWithExt(files, ".bloom") - assert.True(t, ok) - - _, ok = findFileWithExt(files, "secondary.0.bloom") - assert.True(t, ok) - // on Windows we have to shutdown the bucket before opening it again - require.Nil(t, b.Shutdown(ctx)) - - b2, err := NewBucket(ctx, dirName, "", logger, nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - WithStrategy(StrategyReplace), WithSecondaryIndices(1)) - require.Nil(t, err) - defer b2.Shutdown(ctx) - - valuePrimary, err := b2.Get([]byte("hello")) - require.Nil(t, err) - valueSecondary, err := b2.GetBySecondary(0, []byte("bonjour")) - require.Nil(t, err) - - assert.Equal(t, []byte("world"), valuePrimary) - assert.Equal(t, []byte("world"), valueSecondary) -} - -func TestCreateBloomInit(t *testing.T) { - // this test deletes the initial bloom and makes sure it gets recreated after - // the bucket is initialized - ctx := context.Background() - dirName := t.TempDir() - - logger, _ := test.NewNullLogger() - - b, err := NewBucket(ctx, dirName, "", logger, nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - WithStrategy(StrategyReplace), WithSecondaryIndices(1)) - require.Nil(t, err) - defer b.Shutdown(ctx) - - require.Nil(t, b.Put([]byte("hello"), []byte("world"), - WithSecondaryKey(0, []byte("bonjour")))) - require.Nil(t, b.FlushMemtable()) - - for _, ext := range []string{".secondary.0.bloom", ".bloom"} { - files, err := os.ReadDir(dirName) - require.Nil(t, err) - fname, ok := findFileWithExt(files, ext) - require.True(t, ok) - - err = os.RemoveAll(path.Join(dirName, fname)) - require.Nil(t, err) - - files, err = os.ReadDir(dirName) - require.Nil(t, err) - _, ok = findFileWithExt(files, ext) - require.False(t, ok, "verify the file is really gone") - } - - require.Nil(t, b.Shutdown(ctx)) - - // now create a new bucket and assert that the file is re-created on init - b2, err := NewBucket(ctx, dirName, "", logger, nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - WithStrategy(StrategyReplace)) - require.Nil(t, err) - defer b2.Shutdown(ctx) - - files, err := os.ReadDir(dirName) - require.Nil(t, err) - _, ok := findFileWithExt(files, ".bloom") - require.True(t, ok) - _, ok = findFileWithExt(files, ".secondary.0.bloom") - require.True(t, ok) -} - -func TestRepairCorruptedBloomOnInit(t *testing.T) { - ctx := context.Background() - dirName := t.TempDir() - - logger, _ := test.NewNullLogger() - - b, err := NewBucket(ctx, dirName, "", logger, nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - WithStrategy(StrategyReplace)) - require.Nil(t, err) - - require.Nil(t, b.Put([]byte("hello"), []byte("world"))) - require.Nil(t, b.FlushMemtable()) - - files, err := os.ReadDir(dirName) - require.Nil(t, err) - fname, ok := findFileWithExt(files, ".bloom") - require.True(t, ok) - - // now corrupt the bloom filter by randomly overriding data - require.Nil(t, corruptBloomFile(path.Join(dirName, fname))) - // on Windows we have to shutdown the bucket before opening it again - require.Nil(t, b.Shutdown(ctx)) - - // now create a new bucket and assert that the file is ignored, re-created on - // init, and the count matches - b2, err := NewBucket(ctx, dirName, "", logger, nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - WithStrategy(StrategyReplace)) - require.Nil(t, err) - defer b2.Shutdown(ctx) - - value, err := b2.Get([]byte("hello")) - assert.Nil(t, err) - assert.Equal(t, []byte("world"), value) -} - -func TestRepairTooShortBloomOnInit(t *testing.T) { - ctx := context.Background() - dirName := t.TempDir() - - logger, _ := test.NewNullLogger() - - b, err := NewBucket(ctx, dirName, "", logger, nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - WithStrategy(StrategyReplace)) - require.Nil(t, err) - defer b.Shutdown(ctx) - - require.Nil(t, b.Put([]byte("hello"), []byte("world"))) - require.Nil(t, b.FlushMemtable()) - - files, err := os.ReadDir(dirName) - require.Nil(t, err) - fname, ok := findFileWithExt(files, ".bloom") - require.True(t, ok) - - // now corrupt the bloom filter by randomly overriding data - require.Nil(t, corruptBloomFileByTruncatingIt(path.Join(dirName, fname))) - - // now create a new bucket and assert that the file is ignored, re-created on - // init, and the count matches - b2, err := NewBucket(ctx, dirName, "", logger, nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - WithStrategy(StrategyReplace)) - require.Nil(t, err) - defer b2.Shutdown(ctx) - - value, err := b2.Get([]byte("hello")) - assert.Nil(t, err) - assert.Equal(t, []byte("world"), value) -} - -func TestRepairCorruptedBloomSecondaryOnInit(t *testing.T) { - ctx := context.Background() - dirName := t.TempDir() - - logger, _ := test.NewNullLogger() - - b, err := NewBucket(ctx, dirName, "", logger, nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - WithStrategy(StrategyReplace), WithSecondaryIndices(1)) - require.Nil(t, err) - - require.Nil(t, b.Put([]byte("hello"), []byte("world"), - WithSecondaryKey(0, []byte("bonjour")))) - require.Nil(t, b.FlushMemtable()) - - files, err := os.ReadDir(dirName) - require.Nil(t, err) - fname, ok := findFileWithExt(files, "secondary.0.bloom") - require.True(t, ok) - - // now corrupt the file by replacing the count value without adapting the checksum - require.Nil(t, corruptBloomFile(path.Join(dirName, fname))) - // on Windows we have to shutdown the bucket before opening it again - require.Nil(t, b.Shutdown(ctx)) - - // now create a new bucket and assert that the file is ignored, re-created on - // init, and the count matches - b2, err := NewBucket(ctx, dirName, "", logger, nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - WithStrategy(StrategyReplace), WithSecondaryIndices(1)) - require.Nil(t, err) - defer b2.Shutdown(ctx) - - value := make([]byte, 5) - value, _, err = b2.GetBySecondaryIntoMemory(0, []byte("bonjour"), value) - assert.Nil(t, err) - assert.Equal(t, []byte("world"), value) -} - -func TestRepairCorruptedBloomSecondaryOnInitIntoMemory(t *testing.T) { - ctx := context.Background() - dirName := t.TempDir() - - logger, _ := test.NewNullLogger() - - b, err := NewBucket(ctx, dirName, "", logger, nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - WithStrategy(StrategyReplace), WithSecondaryIndices(1)) - require.Nil(t, err) - defer b.Shutdown(ctx) - - require.Nil(t, b.Put([]byte("hello"), []byte("world"), - WithSecondaryKey(0, []byte("bonjour")))) - require.Nil(t, b.FlushMemtable()) - - files, err := os.ReadDir(dirName) - require.Nil(t, err) - fname, ok := findFileWithExt(files, "secondary.0.bloom") - require.True(t, ok) - - // now corrupt the file by replacing the count value without adapting the checksum - require.Nil(t, corruptBloomFile(path.Join(dirName, fname))) - - // now create a new bucket and assert that the file is ignored, re-created on - // init, and the count matches - b2, err := NewBucket(ctx, dirName, "", logger, nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - WithStrategy(StrategyReplace), WithSecondaryIndices(1)) - require.Nil(t, err) - defer b2.Shutdown(ctx) - - value, err := b2.GetBySecondary(0, []byte("bonjour")) - assert.Nil(t, err) - assert.Equal(t, []byte("world"), value) -} - -func TestRepairTooShortBloomSecondaryOnInit(t *testing.T) { - ctx := context.Background() - dirName := t.TempDir() - - logger, _ := test.NewNullLogger() - - b, err := NewBucket(ctx, dirName, "", logger, nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - WithStrategy(StrategyReplace), WithSecondaryIndices(1)) - require.Nil(t, err) - defer b.Shutdown(ctx) - - require.Nil(t, b.Put([]byte("hello"), []byte("world"), - WithSecondaryKey(0, []byte("bonjour")))) - require.Nil(t, b.FlushMemtable()) - - files, err := os.ReadDir(dirName) - require.Nil(t, err) - fname, ok := findFileWithExt(files, "secondary.0.bloom") - require.True(t, ok) - - // now corrupt the file by replacing the count value without adapting the checksum - require.Nil(t, corruptBloomFileByTruncatingIt(path.Join(dirName, fname))) - - // now create a new bucket and assert that the file is ignored, re-created on - // init, and the count matches - b2, err := NewBucket(ctx, dirName, "", logger, nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - WithStrategy(StrategyReplace), WithSecondaryIndices(1)) - require.Nil(t, err) - defer b2.Shutdown(ctx) - - value, err := b2.GetBySecondary(0, []byte("bonjour")) - assert.Nil(t, err) - assert.Equal(t, []byte("world"), value) -} - -func TestLoadWithChecksumErrorCases(t *testing.T) { - t.Run("file does not exist", func(t *testing.T) { - dirName := t.TempDir() - _, err := loadWithChecksum(path.Join(dirName, "my-file"), -1) - assert.NotNil(t, err) - }) - - t.Run("file has incorrect length", func(t *testing.T) { - dirName := t.TempDir() - fName := path.Join(dirName, "my-file") - f, err := os.Create(fName) - require.Nil(t, err) - - _, err = f.Write(make([]byte, 13)) - require.Nil(t, err) - - require.Nil(t, f.Close()) - - _, err = loadWithChecksum(path.Join(dirName, "my-file"), 17) - assert.NotNil(t, err) - }) -} - -func TestBloom_OFF(t *testing.T) { - ctx := context.Background() - tests := bucketTests{ - { - name: "dontCreateBloom", - f: dontCreateBloom, - opts: []BucketOption{ - WithStrategy(StrategyReplace), - WithSecondaryIndices(1), - WithUseBloomFilter(false), - }, - }, - { - name: "dontRecreateBloom", - f: dontRecreateBloom, - opts: []BucketOption{ - WithStrategy(StrategyReplace), - WithSecondaryIndices(1), - WithUseBloomFilter(false), - }, - }, - { - name: "dontPrecomputeBloom", - f: dontPrecomputeBloom, - opts: []BucketOption{ - WithStrategy(StrategyReplace), - WithSecondaryIndices(1), - WithUseBloomFilter(false), - }, - }, - } - tests.run(ctx, t) -} - -func dontCreateBloom(ctx context.Context, t *testing.T, opts []BucketOption) { - dirName := t.TempDir() - logger, _ := test.NewNullLogger() - - b, err := NewBucket(ctx, dirName, "", logger, nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - opts...) - require.NoError(t, err) - defer b.Shutdown(ctx) - - t.Run("populate", func(t *testing.T) { - require.NoError(t, b.Put([]byte("hello"), []byte("world"), - WithSecondaryKey(0, []byte("bonjour")))) - require.NoError(t, b.FlushMemtable()) - }) - - t.Run("check files", func(t *testing.T) { - files, err := os.ReadDir(dirName) - require.NoError(t, err) - - _, ok := findFileWithExt(files, ".bloom") - assert.False(t, ok) - _, ok = findFileWithExt(files, "secondary.0.bloom") - assert.False(t, ok) - }) - - t.Run("search", func(t *testing.T) { - valuePrimary, err := b.Get([]byte("hello")) - require.NoError(t, err) - valueSecondary, err := b.GetBySecondary(0, []byte("bonjour")) - require.NoError(t, err) - - assert.Equal(t, []byte("world"), valuePrimary) - assert.Equal(t, []byte("world"), valueSecondary) - }) -} - -func dontRecreateBloom(ctx context.Context, t *testing.T, opts []BucketOption) { - dirName := t.TempDir() - logger, _ := test.NewNullLogger() - - t.Run("create, populate, shutdown", func(t *testing.T) { - b, err := NewBucket(ctx, dirName, "", logger, nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - opts...) - require.NoError(t, err) - defer b.Shutdown(ctx) - - require.NoError(t, b.Put([]byte("hello"), []byte("world"), - WithSecondaryKey(0, []byte("bonjour")))) - require.NoError(t, b.FlushMemtable()) - }) - - b2, err := NewBucket(ctx, dirName, "", logger, nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - opts...) - require.NoError(t, err) - defer b2.Shutdown(ctx) - - t.Run("check files", func(t *testing.T) { - files, err := os.ReadDir(dirName) - require.NoError(t, err) - - _, ok := findFileWithExt(files, ".bloom") - assert.False(t, ok) - _, ok = findFileWithExt(files, "secondary.0.bloom") - assert.False(t, ok) - }) - - t.Run("search", func(t *testing.T) { - valuePrimary, err := b2.Get([]byte("hello")) - require.NoError(t, err) - valueSecondary, err := b2.GetBySecondary(0, []byte("bonjour")) - require.NoError(t, err) - - assert.Equal(t, []byte("world"), valuePrimary) - assert.Equal(t, []byte("world"), valueSecondary) - }) -} - -func dontPrecomputeBloom(ctx context.Context, t *testing.T, opts []BucketOption) { - dirName := t.TempDir() - logger, _ := test.NewNullLogger() - - b, err := NewBucket(ctx, dirName, "", logger, nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - opts...) - require.NoError(t, err) - defer b.Shutdown(ctx) - - t.Run("populate, compact", func(t *testing.T) { - require.NoError(t, b.Put([]byte("hello"), []byte("world"), - WithSecondaryKey(0, []byte("bonjour")))) - require.NoError(t, b.FlushMemtable()) - - require.NoError(t, b.Put([]byte("hello2"), []byte("world2"), - WithSecondaryKey(0, []byte("bonjour2")))) - require.NoError(t, b.FlushMemtable()) - - compacted, err := b.disk.compactOnce() - require.NoError(t, err) - require.True(t, compacted) - }) - - t.Run("check files", func(t *testing.T) { - files, err := os.ReadDir(dirName) - require.NoError(t, err) - - _, ok := findFileWithExt(files, ".bloom") - assert.False(t, ok) - _, ok = findFileWithExt(files, "secondary.0.bloom") - assert.False(t, ok) - }) - - t.Run("search", func(t *testing.T) { - valuePrimary, err := b.Get([]byte("hello")) - require.NoError(t, err) - valueSecondary, err := b.GetBySecondary(0, []byte("bonjour")) - require.NoError(t, err) - value2Primary, err := b.Get([]byte("hello2")) - require.NoError(t, err) - value2Secondary, err := b.GetBySecondary(0, []byte("bonjour2")) - require.NoError(t, err) - - assert.Equal(t, []byte("world"), valuePrimary) - assert.Equal(t, []byte("world"), valueSecondary) - assert.Equal(t, []byte("world2"), value2Primary) - assert.Equal(t, []byte("world2"), value2Secondary) - }) -} - -func corruptBloomFile(fname string) error { - f, err := os.Open(fname) - if err != nil { - return err - } - - data, err := io.ReadAll(f) - if err != nil { - return err - } - - if err := f.Close(); err != nil { - return err - } - - // corrupt it by setting all data bytes to 0x01 - for i := 5; i < len(data); i++ { - data[i] = 0x01 - } - - f, err = os.Create(fname) - if err != nil { - return err - } - - _, err = f.Write(data) - if err != nil { - return err - } - - return f.Close() -} - -func corruptBloomFileByTruncatingIt(fname string) error { - f, err := os.Open(fname) - if err != nil { - return err - } - - data, err := io.ReadAll(f) - if err != nil { - return err - } - - if err := f.Close(); err != nil { - return err - } - - data = data[:2] - - f, err = os.Create(fname) - if err != nil { - return err - } - - _, err = f.Write(data) - if err != nil { - return err - } - - return f.Close() -} diff --git a/adapters/repos/db/lsmkv/segment_collection_strategy.go b/adapters/repos/db/lsmkv/segment_collection_strategy.go deleted file mode 100644 index 7dd69b02c75b53b5f971ec951f48765f3d825d1c..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/segment_collection_strategy.go +++ /dev/null @@ -1,85 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "encoding/binary" - "fmt" - - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv/segmentindex" - "github.com/weaviate/weaviate/entities/lsmkv" -) - -func (s *segment) getCollection(key []byte) ([]value, error) { - if s.strategy != segmentindex.StrategySetCollection && - s.strategy != segmentindex.StrategyMapCollection { - return nil, fmt.Errorf("get only possible for strategies %q, %q", - StrategySetCollection, StrategyMapCollection) - } - - if s.useBloomFilter && !s.bloomFilter.Test(key) { - return nil, lsmkv.NotFound - } - - node, err := s.index.Get(key) - if err != nil { - return nil, err - } - - // We need to copy the data we read from the segment exactly once in this - // place. This means that future processing can share this memory as much as - // it wants to, as it can now be considered immutable. If we didn't copy in - // this place it would only be safe to hold this data while still under the - // protection of the segmentGroup.maintenanceLock. This lock makes sure that - // no compaction is started during an ongoing read. However, as we could show - // as part of https://github.com/weaviate/weaviate/issues/1837 - // further processing, such as map-decoding and eventually map-merging would - // happen inside the bucket.MapList() method. This scope has its own lock, - // but that lock can only protecting against flushing (i.e. changing the - // active/flushing memtable), not against removing the disk segment. If a - // compaction completes and the old segment is removed, we would be accessing - // invalid memory without the copy, thus leading to a SEGFAULT. - contentsCopy := make([]byte, node.End-node.Start) - if err = s.copyNode(contentsCopy, nodeOffset{node.Start, node.End}); err != nil { - return nil, err - } - - return s.collectionStratParseData(contentsCopy) -} - -func (s *segment) collectionStratParseData(in []byte) ([]value, error) { - if len(in) == 0 { - return nil, lsmkv.NotFound - } - - offset := 0 - - valuesLen := binary.LittleEndian.Uint64(in[offset : offset+8]) - offset += 8 - - values := make([]value, valuesLen) - valueIndex := 0 - for valueIndex < int(valuesLen) { - values[valueIndex].tombstone = in[offset] == 0x01 - offset += 1 - - valueLen := binary.LittleEndian.Uint64(in[offset : offset+8]) - offset += 8 - - values[valueIndex].value = in[offset : offset+int(valueLen)] - offset += int(valueLen) - - valueIndex++ - } - - return values, nil -} diff --git a/adapters/repos/db/lsmkv/segment_group.go b/adapters/repos/db/lsmkv/segment_group.go deleted file mode 100644 index ed7d7636ddf1e6e3f9386db7d01e5d8ebe8568b1..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/segment_group.go +++ /dev/null @@ -1,402 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "context" - "errors" - "fmt" - "io/fs" - "os" - "path/filepath" - "strings" - "sync" - - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv/roaringset" - "github.com/weaviate/weaviate/entities/cyclemanager" - "github.com/weaviate/weaviate/entities/lsmkv" - "github.com/weaviate/weaviate/entities/storagestate" -) - -type SegmentGroup struct { - segments []*segment - - // Lock() for changing the currently active segments, RLock() for normal - // operation - maintenanceLock sync.RWMutex - dir string - - strategy string - - compactionCallbackCtrl cyclemanager.CycleCallbackCtrl - - logger logrus.FieldLogger - - // for backward-compatibility with states where the disk state for maps was - // not guaranteed to be sorted yet - mapRequiresSorting bool - - status storagestate.Status - statusLock sync.Mutex - metrics *Metrics - - // all "replace" buckets support counting through net additions, but not all - // produce a meaningful count. Typically, the only count we're interested in - // is that of the bucket that holds objects - monitorCount bool - - mmapContents bool - keepTombstones bool // see bucket for more datails - useBloomFilter bool // see bucket for more datails - calcCountNetAdditions bool // see bucket for more datails - compactLeftOverSegments bool // see bucket for more datails -} - -type sgConfig struct { - dir string - strategy string - mapRequiresSorting bool - monitorCount bool - mmapContents bool - keepTombstones bool - useBloomFilter bool - calcCountNetAdditions bool - forceCompaction bool -} - -func newSegmentGroup(logger logrus.FieldLogger, metrics *Metrics, - compactionCallbacks cyclemanager.CycleCallbackGroup, cfg sgConfig, -) (*SegmentGroup, error) { - list, err := os.ReadDir(cfg.dir) - if err != nil { - return nil, err - } - - sg := &SegmentGroup{ - segments: make([]*segment, len(list)), - dir: cfg.dir, - logger: logger, - metrics: metrics, - monitorCount: cfg.monitorCount, - mapRequiresSorting: cfg.mapRequiresSorting, - strategy: cfg.strategy, - mmapContents: cfg.mmapContents, - keepTombstones: cfg.keepTombstones, - useBloomFilter: cfg.useBloomFilter, - calcCountNetAdditions: cfg.calcCountNetAdditions, - compactLeftOverSegments: cfg.forceCompaction, - } - - segmentIndex := 0 - for _, entry := range list { - if filepath.Ext(entry.Name()) != ".db" { - // skip, this could be commit log, etc. - continue - } - - // before we can mount this file, we need to check if a WAL exists for it. - // If yes, we must assume that the flush never finished, as otherwise the - // WAL would have been lsmkv.Deleted. Thus we must remove it. - potentialWALFileName := strings.TrimSuffix(entry.Name(), ".db") + ".wal" - ok, err := fileExists(filepath.Join(sg.dir, potentialWALFileName)) - if err != nil { - return nil, fmt.Errorf("check for presence of wals for segment %s: %w", - entry.Name(), err) - } - - if ok { - if err := os.Remove(filepath.Join(sg.dir, entry.Name())); err != nil { - return nil, fmt.Errorf("delete corrupt segment %s: %w", entry.Name(), err) - } - - logger.WithField("action", "lsm_segment_init"). - WithField("path", filepath.Join(sg.dir, entry.Name())). - WithField("wal_path", potentialWALFileName). - Info("Discarded (partially written) LSM segment, because an active WAL for " + - "the same segment was found. A recovery from the WAL will follow.") - - continue - } - - info, err := entry.Info() - if err != nil { - return nil, fmt.Errorf("obtain file info: %w", err) - } - - if info.Size() == 0 { - continue - } - - segment, err := newSegment(filepath.Join(sg.dir, entry.Name()), logger, - metrics, sg.makeExistsOnLower(segmentIndex), - sg.mmapContents, sg.useBloomFilter, sg.calcCountNetAdditions) - if err != nil { - return nil, fmt.Errorf("init segment %s: %w", entry.Name(), err) - } - - sg.segments[segmentIndex] = segment - segmentIndex++ - } - - sg.segments = sg.segments[:segmentIndex] - - if sg.monitorCount { - sg.metrics.ObjectCount(sg.count()) - } - - id := "segmentgroup/compaction/" + sg.dir - sg.compactionCallbackCtrl = compactionCallbacks.Register(id, sg.compactIfLevelsMatch) - - return sg, nil -} - -func (sg *SegmentGroup) makeExistsOnLower(nextSegmentIndex int) existsOnLowerSegmentsFn { - return func(key []byte) (bool, error) { - if nextSegmentIndex == 0 { - // this is already the lowest possible segment, we can guarantee that - // any key in this segment is previously unseen. - return false, nil - } - - v, err := sg.getWithUpperSegmentBoundary(key, nextSegmentIndex-1) - if err != nil { - return false, fmt.Errorf("check exists on segments lower than %d: %w", - nextSegmentIndex, err) - } - - return v != nil, nil - } -} - -func (sg *SegmentGroup) add(path string) error { - sg.maintenanceLock.Lock() - defer sg.maintenanceLock.Unlock() - - newSegmentIndex := len(sg.segments) - segment, err := newSegment(path, sg.logger, - sg.metrics, sg.makeExistsOnLower(newSegmentIndex), - sg.mmapContents, sg.useBloomFilter, sg.calcCountNetAdditions) - if err != nil { - return fmt.Errorf("init segment %s: %w", path, err) - } - - sg.segments = append(sg.segments, segment) - return nil -} - -func (sg *SegmentGroup) get(key []byte) ([]byte, error) { - sg.maintenanceLock.RLock() - defer sg.maintenanceLock.RUnlock() - - return sg.getWithUpperSegmentBoundary(key, len(sg.segments)-1) -} - -// not thread-safe on its own, as the assumption is that this is called from a -// lockholder, e.g. within .get() -func (sg *SegmentGroup) getWithUpperSegmentBoundary(key []byte, topMostSegment int) ([]byte, error) { - // assumes "replace" strategy - - // start with latest and exit as soon as something is found, thus making sure - // the latest takes presence - for i := topMostSegment; i >= 0; i-- { - v, err := sg.segments[i].get(key) - if err != nil { - if errors.Is(err, lsmkv.NotFound) { - continue - } - - if errors.Is(err, lsmkv.Deleted) { - return nil, nil - } - - panic(fmt.Sprintf("unsupported error in segmentGroup.get(): %v", err)) - } - - return v, nil - } - - return nil, nil -} - -func (sg *SegmentGroup) getBySecondaryIntoMemory(pos int, key []byte, buffer []byte) ([]byte, []byte, error) { - sg.maintenanceLock.RLock() - defer sg.maintenanceLock.RUnlock() - - // assumes "replace" strategy - - // start with latest and exit as soon as something is found, thus making sure - // the latest takes presence - for i := len(sg.segments) - 1; i >= 0; i-- { - v, err, allocatedBuff := sg.segments[i].getBySecondaryIntoMemory(pos, key, buffer) - if err != nil { - if errors.Is(err, lsmkv.NotFound) { - continue - } - - if errors.Is(err, lsmkv.Deleted) { - return nil, nil, nil - } - - panic(fmt.Sprintf("unsupported error in segmentGroup.get(): %v", err)) - } - - return v, allocatedBuff, nil - } - - return nil, nil, nil -} - -func (sg *SegmentGroup) getCollection(key []byte) ([]value, error) { - sg.maintenanceLock.RLock() - defer sg.maintenanceLock.RUnlock() - - var out []value - - // start with first and do not exit - for _, segment := range sg.segments { - v, err := segment.getCollection(key) - if err != nil { - if errors.Is(err, lsmkv.NotFound) { - continue - } - - return nil, err - } - - if len(out) == 0 { - out = v - } else { - out = append(out, v...) - } - } - - return out, nil -} - -func (sg *SegmentGroup) getCollectionBySegments(key []byte) ([][]value, error) { - sg.maintenanceLock.RLock() - defer sg.maintenanceLock.RUnlock() - - out := make([][]value, len(sg.segments)) - - i := 0 - // start with first and do not exit - for _, segment := range sg.segments { - v, err := segment.getCollection(key) - if err != nil { - if errors.Is(err, lsmkv.NotFound) { - continue - } - - return nil, err - } - - out[i] = v - i++ - } - - return out[:i], nil -} - -func (sg *SegmentGroup) roaringSetGet(key []byte) (roaringset.BitmapLayers, error) { - sg.maintenanceLock.RLock() - defer sg.maintenanceLock.RUnlock() - - var out roaringset.BitmapLayers - - // start with first and do not exit - for _, segment := range sg.segments { - rs, err := segment.roaringSetGet(key) - if err != nil { - if errors.Is(err, lsmkv.NotFound) { - continue - } - - return nil, err - } - - out = append(out, rs) - } - - return out, nil -} - -func (sg *SegmentGroup) count() int { - sg.maintenanceLock.RLock() - defer sg.maintenanceLock.RUnlock() - - count := 0 - for _, seg := range sg.segments { - count += seg.countNetAdditions - } - - return count -} - -func (sg *SegmentGroup) shutdown(ctx context.Context) error { - if err := sg.compactionCallbackCtrl.Unregister(ctx); err != nil { - return fmt.Errorf("long-running compaction in progress: %w", ctx.Err()) - } - - // Lock acquirement placed after compaction cycle stop request, due to occasional deadlock, - // because compaction logic used in cycle also requires maintenance lock. - // - // If lock is grabbed by shutdown method and compaction in cycle loop starts right after, - // it is blocked waiting for the same lock, eventually blocking entire cycle loop and preventing to read stop signal. - // If stop signal can not be read, shutdown will not receive stop result and will not proceed with further execution. - // Maintenance lock will then never be released. - sg.maintenanceLock.Lock() - defer sg.maintenanceLock.Unlock() - - for i, seg := range sg.segments { - if err := seg.close(); err != nil { - return err - } - - sg.segments[i] = nil - } - - // make sure the segment list itself is set to nil. In case a memtable will - // still flush after closing, it might try to read from a disk segment list - // otherwise and run into nil-pointer problems. - sg.segments = nil - - return nil -} - -func (sg *SegmentGroup) UpdateStatus(status storagestate.Status) { - sg.statusLock.Lock() - defer sg.statusLock.Unlock() - - sg.status = status -} - -func (sg *SegmentGroup) isReadyOnly() bool { - sg.statusLock.Lock() - defer sg.statusLock.Unlock() - - return sg.status == storagestate.StatusReadOnly -} - -func fileExists(path string) (bool, error) { - _, err := os.Stat(path) - if err == nil { - return true, nil - } - - if errors.Is(err, fs.ErrNotExist) { - return false, nil - } - - return false, err -} diff --git a/adapters/repos/db/lsmkv/segment_group_compaction.go b/adapters/repos/db/lsmkv/segment_group_compaction.go deleted file mode 100644 index c52990fa179b4cc03cb161c287feb19890962445..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/segment_group_compaction.go +++ /dev/null @@ -1,435 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "fmt" - "math" - "os" - "path/filepath" - - "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv/roaringset" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv/segmentindex" - "github.com/weaviate/weaviate/entities/cyclemanager" -) - -func (sg *SegmentGroup) bestCompactionCandidatePair() []int { - sg.maintenanceLock.RLock() - defer sg.maintenanceLock.RUnlock() - - // if true, the parent shard has indicated that it has - // entered an immutable state. During this time, the - // SegmentGroup should refrain from flushing until its - // shard indicates otherwise - if sg.isReadyOnly() { - return nil - } - - // Nothing to compact - if len(sg.segments) < 2 { - return nil - } - - // first determine the lowest level with candidates - levels := map[uint16]int{} - lowestPairLevel := uint16(math.MaxUint16) - lowestLevel := uint16(math.MaxUint16) - lowestIndex := -1 - secondLowestIndex := -1 - pairExists := false - - for ind, seg := range sg.segments { - levels[seg.level]++ - val := levels[seg.level] - if val > 1 { - if seg.level < lowestPairLevel { - lowestPairLevel = seg.level - pairExists = true - } - } - - if seg.level < lowestLevel { - secondLowestIndex = lowestIndex - lowestLevel = seg.level - lowestIndex = ind - } - } - - if pairExists { - // now pick any two segments which match the level - var res []int - - for i, segment := range sg.segments { - if len(res) >= 2 { - break - } - - if segment.level == lowestPairLevel { - res = append(res, i) - } - } - - return res - } else { - if sg.compactLeftOverSegments { - // Some segments exist, but none are of the same level - // Merge the two lowest segments - - return []int{secondLowestIndex, lowestIndex} - } else { - // No segments of the same level exist, and we are not allowed to merge the lowest segments - // This means we cannot compact. Set COMPACT_LEFTOVER_SEGMENTS to true to compact the remaining segments - return nil - } - } -} - -// segmentAtPos retrieves the segment for the given position using a read-lock -func (sg *SegmentGroup) segmentAtPos(pos int) *segment { - sg.maintenanceLock.RLock() - defer sg.maintenanceLock.RUnlock() - - return sg.segments[pos] -} - -func (sg *SegmentGroup) compactOnce() (bool, error) { - // Is it safe to only occasionally lock instead of the entire duration? Yes, - // because other than compaction the only change to the segments array could - // be an append because of a new flush cycle, so we do not need to guarantee - // that the array contents stay stable over the duration of an entire - // compaction. We do however need to protect against a read-while-write (race - // condition) on the array. Thus any read from sg.segments need to protected - pair := sg.bestCompactionCandidatePair() - if pair == nil { - // nothing to do - return false, nil - } - - path := fmt.Sprintf("%s.tmp", sg.segmentAtPos(pair[1]).path) - f, err := os.Create(path) - if err != nil { - return false, err - } - - scratchSpacePath := sg.segmentAtPos(pair[1]).path + "compaction.scratch.d" - - // the assumption is that the first element is older, and/or a higher level - level := sg.segmentAtPos(pair[0]).level - secondaryIndices := sg.segmentAtPos(pair[0]).secondaryIndexCount - - if level == sg.segmentAtPos(pair[1]).level { - level = level + 1 - } - - strategy := sg.segmentAtPos(pair[0]).strategy - cleanupTombstones := !sg.keepTombstones && pair[0] == 0 - - pathLabel := "n/a" - if sg.metrics != nil && !sg.metrics.groupClasses { - pathLabel = sg.dir - } - switch strategy { - - // TODO: call metrics just once with variable strategy label - - case segmentindex.StrategyReplace: - c := newCompactorReplace(f, sg.segmentAtPos(pair[0]).newCursor(), - sg.segmentAtPos(pair[1]).newCursor(), level, secondaryIndices, scratchSpacePath, cleanupTombstones) - - if sg.metrics != nil { - sg.metrics.CompactionReplace.With(prometheus.Labels{"path": pathLabel}).Inc() - defer sg.metrics.CompactionReplace.With(prometheus.Labels{"path": pathLabel}).Dec() - } - - if err := c.do(); err != nil { - return false, err - } - case segmentindex.StrategySetCollection: - c := newCompactorSetCollection(f, sg.segmentAtPos(pair[0]).newCollectionCursor(), - sg.segmentAtPos(pair[1]).newCollectionCursor(), level, secondaryIndices, - scratchSpacePath, cleanupTombstones) - - if sg.metrics != nil { - sg.metrics.CompactionSet.With(prometheus.Labels{"path": pathLabel}).Inc() - defer sg.metrics.CompactionSet.With(prometheus.Labels{"path": pathLabel}).Dec() - } - - if err := c.do(); err != nil { - return false, err - } - case segmentindex.StrategyMapCollection: - c := newCompactorMapCollection(f, - sg.segmentAtPos(pair[0]).newCollectionCursorReusable(), - sg.segmentAtPos(pair[1]).newCollectionCursorReusable(), - level, secondaryIndices, scratchSpacePath, sg.mapRequiresSorting, cleanupTombstones) - - if sg.metrics != nil { - sg.metrics.CompactionMap.With(prometheus.Labels{"path": pathLabel}).Inc() - defer sg.metrics.CompactionMap.With(prometheus.Labels{"path": pathLabel}).Dec() - } - - if err := c.do(); err != nil { - return false, err - } - case segmentindex.StrategyRoaringSet: - leftSegment := sg.segmentAtPos(pair[0]) - rightSegment := sg.segmentAtPos(pair[1]) - - leftCursor := leftSegment.newRoaringSetCursor() - rightCursor := rightSegment.newRoaringSetCursor() - - c := roaringset.NewCompactor(f, leftCursor, rightCursor, - level, scratchSpacePath, cleanupTombstones) - - if sg.metrics != nil { - sg.metrics.CompactionRoaringSet.With(prometheus.Labels{"path": pathLabel}).Set(1) - defer sg.metrics.CompactionRoaringSet.With(prometheus.Labels{"path": pathLabel}).Set(0) - } - - if err := c.Do(); err != nil { - return false, err - } - - default: - return false, errors.Errorf("unrecognized strategy %v", strategy) - } - - if err := f.Close(); err != nil { - return false, errors.Wrap(err, "close compacted segment file") - } - - if err := sg.replaceCompactedSegments(pair[0], pair[1], path); err != nil { - return false, errors.Wrap(err, "replace compacted segments") - } - - return true, nil -} - -func (sg *SegmentGroup) replaceCompactedSegments(old1, old2 int, - newPathTmp string, -) error { - sg.maintenanceLock.RLock() - updatedCountNetAdditions := sg.segments[old1].countNetAdditions + - sg.segments[old2].countNetAdditions - sg.maintenanceLock.RUnlock() - - precomputedFiles, err := preComputeSegmentMeta(newPathTmp, - updatedCountNetAdditions, sg.logger, - sg.useBloomFilter, sg.calcCountNetAdditions) - if err != nil { - return fmt.Errorf("precompute segment meta: %w", err) - } - - sg.maintenanceLock.Lock() - defer sg.maintenanceLock.Unlock() - - if err := sg.segments[old1].close(); err != nil { - return errors.Wrap(err, "close disk segment") - } - - if err := sg.segments[old2].close(); err != nil { - return errors.Wrap(err, "close disk segment") - } - - if err := sg.segments[old1].drop(); err != nil { - return errors.Wrap(err, "drop disk segment") - } - - if err := sg.segments[old2].drop(); err != nil { - return errors.Wrap(err, "drop disk segment") - } - - sg.segments[old1] = nil - sg.segments[old2] = nil - - var newPath string - // the old segments have been deleted, we can now safely remove the .tmp - // extension from the new segment itself and the pre-computed files which - // carried the name of the second old segment - for i, path := range precomputedFiles { - updated, err := sg.stripTmpExtension(path) - if err != nil { - return errors.Wrap(err, "strip .tmp extension of new segment") - } - - if i == 0 { - // the first element in the list is the segment itself - newPath = updated - } - } - - seg, err := newSegment(newPath, sg.logger, sg.metrics, nil, - sg.mmapContents, sg.useBloomFilter, sg.calcCountNetAdditions) - if err != nil { - return errors.Wrap(err, "create new segment") - } - - sg.segments[old2] = seg - - sg.segments = append(sg.segments[:old1], sg.segments[old1+1:]...) - - return nil -} - -func (sg *SegmentGroup) stripTmpExtension(oldPath string) (string, error) { - ext := filepath.Ext(oldPath) - if ext != ".tmp" { - return "", errors.Errorf("segment %q did not have .tmp extension", oldPath) - } - newPath := oldPath[:len(oldPath)-len(ext)] - - if err := os.Rename(oldPath, newPath); err != nil { - return "", errors.Wrapf(err, "rename %q -> %q", oldPath, newPath) - } - - return newPath, nil -} - -func (sg *SegmentGroup) compactIfLevelsMatch(shouldAbort cyclemanager.ShouldAbortCallback) bool { - sg.monitorSegments() - - compacted, err := sg.compactOnce() - if err != nil { - sg.logger.WithField("action", "lsm_compaction"). - WithField("path", sg.dir). - WithError(err). - Errorf("compaction failed") - } - - if compacted { - return true - } else { - sg.logger.WithField("action", "lsm_compaction"). - WithField("path", sg.dir). - Trace("no segment eligible for compaction") - return false - } -} - -func (sg *SegmentGroup) Len() int { - sg.maintenanceLock.RLock() - defer sg.maintenanceLock.RUnlock() - - return len(sg.segments) -} - -func (sg *SegmentGroup) monitorSegments() { - if sg.metrics == nil || sg.metrics.groupClasses { - return - } - - sg.metrics.ActiveSegments.With(prometheus.Labels{ - "strategy": sg.strategy, - "path": sg.dir, - }).Set(float64(sg.Len())) - - stats := sg.segmentLevelStats() - stats.fillMissingLevels() - stats.report(sg.metrics, sg.strategy, sg.dir) -} - -type segmentLevelStats struct { - indexes map[uint16]int - payloads map[uint16]int - count map[uint16]int -} - -func newSegmentLevelStats() segmentLevelStats { - return segmentLevelStats{ - indexes: map[uint16]int{}, - payloads: map[uint16]int{}, - count: map[uint16]int{}, - } -} - -func (sg *SegmentGroup) segmentLevelStats() segmentLevelStats { - sg.maintenanceLock.RLock() - defer sg.maintenanceLock.RUnlock() - - stats := newSegmentLevelStats() - - for _, seg := range sg.segments { - stats.count[seg.level]++ - - cur := stats.indexes[seg.level] - cur += seg.index.Size() - stats.indexes[seg.level] = cur - - cur = stats.payloads[seg.level] - cur += seg.PayloadSize() - stats.payloads[seg.level] = cur - } - - return stats -} - -// fill missing levels -// -// Imagine we had exactly two segments of level 4 before, and there were just -// compacted to single segment of level 5. As a result, there should be no -// more segments of level 4. However, our current logic only loops over -// existing segments. As a result, we need to check what the highest level -// is, then for every level lower than the highest check if we are missing -// data. If yes, we need to explicitly set the gauges to 0. -func (s *segmentLevelStats) fillMissingLevels() { - maxLevel := uint16(0) - for level := range s.count { - if level > maxLevel { - maxLevel = level - } - } - - if maxLevel > 0 { - for level := uint16(0); level < maxLevel; level++ { - if _, ok := s.count[level]; ok { - continue - } - - // there is no entry for this level, we must explicitly set it to 0 - s.count[level] = 0 - s.indexes[level] = 0 - s.payloads[level] = 0 - } - } -} - -func (s *segmentLevelStats) report(metrics *Metrics, - strategy, dir string, -) { - for level, size := range s.indexes { - metrics.SegmentSize.With(prometheus.Labels{ - "strategy": strategy, - "unit": "index", - "level": fmt.Sprint(level), - "path": dir, - }).Set(float64(size)) - } - - for level, size := range s.payloads { - metrics.SegmentSize.With(prometheus.Labels{ - "strategy": strategy, - "unit": "payload", - "level": fmt.Sprint(level), - "path": dir, - }).Set(float64(size)) - } - - for level, count := range s.count { - metrics.SegmentCount.With(prometheus.Labels{ - "strategy": strategy, - "level": fmt.Sprint(level), - "path": dir, - }).Set(float64(count)) - } -} diff --git a/adapters/repos/db/lsmkv/segment_key_and_tombstone_extractor.go b/adapters/repos/db/lsmkv/segment_key_and_tombstone_extractor.go deleted file mode 100644 index 01bd0ebf3c07d38b4e9ab3767c09752d2b8a6b46..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/segment_key_and_tombstone_extractor.go +++ /dev/null @@ -1,159 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "encoding/binary" -) - -// bufferedKeyAndTombstoneExtractor is a tool to build up the count stats for -// disk segments (i.e. all the keys in this segment as well as whether they -// contain a tombstone or not). It tries to be relatively memory-efficient -// while doing a whole-segment disk scan. It uses a primitive []byte buffer -// for its output which needs to be allocated just once. It can only read until -// the buffer is full, then it needs to call a callback fn which can do -// something with the data. After the callback function has been called on each -// key, the output buffer is reset. If the input segment it not at EOF yet, -// this cycle repeats -type bufferedKeyAndTombstoneExtractor struct { - outputBuffer []byte - outputBufferOffset uint64 - offset uint64 - end uint64 - rawSegment []byte - secondaryIndexCount uint16 - callback keyAndTombstoneCallbackFn - callbackCycle int -} - -type keyAndTombstoneCallbackFn func(key []byte, tombstone bool) - -func newBufferedKeyAndTombstoneExtractor(rawSegment []byte, initialOffset uint64, - end uint64, outputBufferSize uint64, secondaryIndexCount uint16, - callback keyAndTombstoneCallbackFn, -) *bufferedKeyAndTombstoneExtractor { - return &bufferedKeyAndTombstoneExtractor{ - rawSegment: rawSegment, - offset: initialOffset, - end: end, - outputBuffer: make([]byte, outputBufferSize), - outputBufferOffset: 0, - secondaryIndexCount: secondaryIndexCount, - callback: callback, - } -} - -func (e *bufferedKeyAndTombstoneExtractor) do() { - for { - if e.offset >= e.end { - break - } - - // returns false if the output buffer ran full - ok := e.readSingleEntry() - if !ok { - e.flushAndCallback() - } - } - - // one final callback - e.flushAndCallback() -} - -// returns true if the cycle completed, returns false if the cycle did not -// complete because the output buffer was full. In that case, the offsets have -// been reset to the values they had at the beginning of the cycle -func (e *bufferedKeyAndTombstoneExtractor) readSingleEntry() bool { - // if we discover during an iteration that the next entry can't fit in the - // buffer anymore, we must return to the start of this iteration, so that - // the this work can be picked up here once the buffer has been flushed - offsetAtLoopStart := e.offset - outputOffsetAtLoopStart := e.outputBufferOffset - - // the first output size check is static, as we will always read 5 bytes, - // no matter what. If they can't even fit, we can abort right away - if !e.outputBufferCanFit(5) { - e.offset = offsetAtLoopStart - e.outputBufferOffset = outputOffsetAtLoopStart - return false - } - - // copy tombstone value into output buffer - e.outputBuffer[e.outputBufferOffset] = e.rawSegment[e.offset] - e.offset++ - e.outputBufferOffset++ - - valueLen := binary.LittleEndian.Uint64(e.rawSegment[e.offset : e.offset+8]) - e.offset += 8 - - // we're not actually interested in the value, so we can skip it entirely - e.offset += valueLen - - primaryKeyLen := binary.LittleEndian.Uint32(e.rawSegment[e.offset : e.offset+4]) - if !e.outputBufferCanFit(uint64(primaryKeyLen) + 4) { - e.offset = offsetAtLoopStart - e.outputBufferOffset = outputOffsetAtLoopStart - return false - } - - // copy the primary key len indicator into the output buffer - copy(e.outputBuffer[e.outputBufferOffset:e.outputBufferOffset+4], - e.rawSegment[e.offset:e.offset+4]) - e.offset += 4 - e.outputBufferOffset += 4 - - // then copy the key itself - copy(e.outputBuffer[e.outputBufferOffset:e.outputBufferOffset+uint64(primaryKeyLen)], e.rawSegment[e.offset:e.offset+uint64(primaryKeyLen)]) - e.offset += uint64(primaryKeyLen) - e.outputBufferOffset += uint64(primaryKeyLen) - - for i := uint16(0); i < e.secondaryIndexCount; i++ { - secKeyLen := binary.LittleEndian.Uint32(e.rawSegment[e.offset : e.offset+4]) - e.offset += 4 - e.offset += uint64(secKeyLen) - } - - return true -} - -func (e *bufferedKeyAndTombstoneExtractor) outputBufferCanFit(size uint64) bool { - return (uint64(len(e.outputBuffer)) - e.outputBufferOffset) >= size -} - -// flushAndCallback calls the callback fn for each key/tombstone pair in the -// buffer, then resets the buffer offset, making it ready to be overwritten in -// the next cycle -func (e *bufferedKeyAndTombstoneExtractor) flushAndCallback() { - end := e.outputBufferOffset - e.outputBufferOffset = 0 - for e.outputBufferOffset < end { - var tombstone bool - if e.outputBuffer[e.outputBufferOffset] == 0x01 { - tombstone = true - } - - e.outputBufferOffset++ - - primaryKeyLen := binary.LittleEndian.Uint32(e.outputBuffer[e.outputBufferOffset : e.outputBufferOffset+4]) - - e.outputBufferOffset += 4 - - e.callback(e.outputBuffer[e.outputBufferOffset:e.outputBufferOffset+uint64(primaryKeyLen)], - tombstone) - e.outputBufferOffset += uint64(primaryKeyLen) - } - - // reset outputBufferOffset for next batch - e.outputBufferOffset = 0 - - e.callbackCycle++ -} diff --git a/adapters/repos/db/lsmkv/segment_net_count_additions.go b/adapters/repos/db/lsmkv/segment_net_count_additions.go deleted file mode 100644 index 5e6d46fe378da34903afeb791c29b1a148b4cf8c..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/segment_net_count_additions.go +++ /dev/null @@ -1,141 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "bytes" - "encoding/binary" - "errors" - "fmt" - "path/filepath" - "strings" - - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv/segmentindex" -) - -// ErrInvalidChecksum indicates that the read file should not be trusted. For -// any pre-computed data this is a recoverable issue, as the data can simply be -// re-computed at read-time. -var ErrInvalidChecksum = errors.New("invalid checksum") - -// existOnLowerSegments is a simple function that can be passed at segment -// initialization time to check if any of the keys are truly new or previously -// seen. This can in turn be used to build up the net count additions. The -// reason this is abstract: -type existsOnLowerSegmentsFn func(key []byte) (bool, error) - -func (s *segment) countNetPath() string { - return countNetPathFromSegmentPath(s.path) -} - -func countNetPathFromSegmentPath(segPath string) string { - extless := strings.TrimSuffix(segPath, filepath.Ext(segPath)) - return fmt.Sprintf("%s.cna", extless) -} - -func (s *segment) initCountNetAdditions(exists existsOnLowerSegmentsFn) error { - if s.strategy != segmentindex.StrategyReplace { - // replace is the only strategy that supports counting - return nil - } - - ok, err := fileExists(s.countNetPath()) - if err != nil { - return err - } - - if ok { - err = s.loadCountNetFromDisk() - if err == nil { - return nil - } - - if err != ErrInvalidChecksum { - // not a recoverable error - return err - } - - // now continue re-calculating - } - - var lastErr error - countNet := 0 - cb := func(key []byte, tombstone bool) { - existedOnPrior, err := exists(key) - if err != nil { - lastErr = err - } - - if tombstone && existedOnPrior { - countNet-- - } - - if !tombstone && !existedOnPrior { - countNet++ - } - } - - extr := newBufferedKeyAndTombstoneExtractor(s.contents, s.dataStartPos, - s.dataEndPos, 10e6, s.secondaryIndexCount, cb) - - extr.do() - - s.countNetAdditions = countNet - - if lastErr != nil { - return lastErr - } - - if err := s.storeCountNetOnDisk(); err != nil { - return fmt.Errorf("store count net additions on disk: %w", err) - } - - return nil -} - -func (s *segment) storeCountNetOnDisk() error { - return storeCountNetOnDisk(s.countNetPath(), s.countNetAdditions) -} - -func storeCountNetOnDisk(path string, value int) error { - buf := new(bytes.Buffer) - - if err := binary.Write(buf, binary.LittleEndian, uint64(value)); err != nil { - return fmt.Errorf("write cna to buf: %w", err) - } - - return writeWithChecksum(buf.Bytes(), path) -} - -func (s *segment) loadCountNetFromDisk() error { - data, err := loadWithChecksum(s.countNetPath(), 12) - if err != nil { - return err - } - - s.countNetAdditions = int(binary.LittleEndian.Uint64(data[0:8])) - - return nil -} - -func (s *segment) precomputeCountNetAdditions(updatedCountNetAdditions int) ([]string, error) { - if s.strategy != segmentindex.StrategyReplace { - // only "replace" has count net additions, so we are done - return []string{}, nil - } - - cnaPath := fmt.Sprintf("%s.tmp", s.countNetPath()) - if err := storeCountNetOnDisk(cnaPath, updatedCountNetAdditions); err != nil { - return nil, err - } - return []string{cnaPath}, nil -} diff --git a/adapters/repos/db/lsmkv/segment_net_count_additions_test.go b/adapters/repos/db/lsmkv/segment_net_count_additions_test.go deleted file mode 100644 index 58b8867a9f748e4ac6bcc56ac6ec7d066d12c070..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/segment_net_count_additions_test.go +++ /dev/null @@ -1,322 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "context" - "encoding/binary" - "io" - "os" - "path" - "strings" - "testing" - - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/cyclemanager" -) - -func TestCNA(t *testing.T) { - ctx := context.Background() - tests := bucketTests{ - { - name: "createCNAOnFlush", - f: createCNAOnFlush, - opts: []BucketOption{ - WithStrategy(StrategyReplace), - }, - }, - { - name: "createCNAInit", - f: createCNAInit, - opts: []BucketOption{ - WithStrategy(StrategyReplace), - }, - }, - { - name: "repairCorruptedCNAOnInit", - f: repairCorruptedCNAOnInit, - opts: []BucketOption{ - WithStrategy(StrategyReplace), - }, - }, - } - tests.run(ctx, t) -} - -func createCNAOnFlush(ctx context.Context, t *testing.T, opts []BucketOption) { - dirName := t.TempDir() - - logger, _ := test.NewNullLogger() - - b, err := NewBucket(ctx, dirName, "", logger, nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - defer b.Shutdown(ctx) - - require.Nil(t, b.Put([]byte("hello"), []byte("world"))) - require.Nil(t, b.FlushMemtable()) - - files, err := os.ReadDir(dirName) - require.Nil(t, err) - - _, ok := findFileWithExt(files, ".cna") - assert.True(t, ok) -} - -func createCNAInit(ctx context.Context, t *testing.T, opts []BucketOption) { - // this test deletes the initial cna and makes sure it gets recreated after - // the bucket is initialized - dirName := t.TempDir() - - logger, _ := test.NewNullLogger() - - b, err := NewBucket(ctx, dirName, "", logger, nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - defer b.Shutdown(ctx) - - require.Nil(t, b.Put([]byte("hello"), []byte("world"))) - require.Nil(t, b.FlushMemtable()) - - files, err := os.ReadDir(dirName) - require.Nil(t, err) - fname, ok := findFileWithExt(files, ".cna") - require.True(t, ok) - - err = os.RemoveAll(path.Join(dirName, fname)) - require.Nil(t, err) - - files, err = os.ReadDir(dirName) - require.Nil(t, err) - _, ok = findFileWithExt(files, ".cna") - require.False(t, ok, "verify the file is really gone") - - // on Windows we have to shutdown the bucket before opening it again - require.Nil(t, b.Shutdown(ctx)) - - // now create a new bucket and assert that the file is re-created on init - b2, err := NewBucket(ctx, dirName, "", logger, nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - defer b2.Shutdown(ctx) - - files, err = os.ReadDir(dirName) - require.Nil(t, err) - _, ok = findFileWithExt(files, ".cna") - require.True(t, ok) -} - -func repairCorruptedCNAOnInit(ctx context.Context, t *testing.T, opts []BucketOption) { - // this test deletes the initial cna and makes sure it gets recreated after - // the bucket is initialized - dirName := t.TempDir() - - logger, _ := test.NewNullLogger() - - b, err := NewBucket(ctx, dirName, "", logger, nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - defer b.Shutdown(ctx) - - require.Nil(t, b.Put([]byte("hello"), []byte("world"))) - require.Nil(t, b.FlushMemtable()) - - files, err := os.ReadDir(dirName) - require.Nil(t, err) - fname, ok := findFileWithExt(files, ".cna") - require.True(t, ok) - - // now corrupt the file by replacing the count value without adapting the checksum - require.Nil(t, corruptCNAFile(path.Join(dirName, fname), 12345)) - - // on Windows we have to shutdown the bucket before opening it again - require.Nil(t, b.Shutdown(ctx)) - // now create a new bucket and assert that the file is ignored, re-created on - // init, and the count matches - b2, err := NewBucket(ctx, dirName, "", logger, nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - defer b2.Shutdown(ctx) - - assert.Equal(t, 1, b2.Count()) -} - -func TestCNA_OFF(t *testing.T) { - ctx := context.Background() - tests := bucketTests{ - { - name: "dontCreateCNA", - f: dontCreateCNA, - opts: []BucketOption{ - WithStrategy(StrategyReplace), - WithCalcCountNetAdditions(false), - }, - }, - { - name: "dontRecreateCNA", - f: dontRecreateCNA, - opts: []BucketOption{ - WithStrategy(StrategyReplace), - WithCalcCountNetAdditions(false), - }, - }, - { - name: "dontPrecomputeCNA", - f: dontPrecomputeCNA, - opts: []BucketOption{ - WithStrategy(StrategyReplace), - WithCalcCountNetAdditions(false), - }, - }, - } - tests.run(ctx, t) -} - -func dontCreateCNA(ctx context.Context, t *testing.T, opts []BucketOption) { - dirName := t.TempDir() - logger, _ := test.NewNullLogger() - - b, err := NewBucket(ctx, dirName, "", logger, nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - opts...) - require.NoError(t, err) - defer b.Shutdown(ctx) - - t.Run("populate", func(t *testing.T) { - require.NoError(t, b.Put([]byte("hello"), []byte("world"))) - require.NoError(t, b.FlushMemtable()) - }) - - t.Run("check files", func(t *testing.T) { - files, err := os.ReadDir(dirName) - require.NoError(t, err) - - _, ok := findFileWithExt(files, ".cna") - assert.False(t, ok) - }) - - t.Run("count", func(t *testing.T) { - assert.Equal(t, 0, b.Count()) - }) -} - -func dontRecreateCNA(ctx context.Context, t *testing.T, opts []BucketOption) { - dirName := t.TempDir() - logger, _ := test.NewNullLogger() - - t.Run("create, populate, shutdown", func(t *testing.T) { - b, err := NewBucket(ctx, dirName, "", logger, nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - opts...) - require.NoError(t, err) - defer b.Shutdown(ctx) - - require.NoError(t, b.Put([]byte("hello"), []byte("world"))) - require.NoError(t, b.FlushMemtable()) - }) - - b2, err := NewBucket(ctx, dirName, "", logger, nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - opts...) - require.NoError(t, err) - defer b2.Shutdown(ctx) - - t.Run("check files", func(t *testing.T) { - files, err := os.ReadDir(dirName) - require.NoError(t, err) - - _, ok := findFileWithExt(files, ".cna") - assert.False(t, ok) - }) - - t.Run("count", func(t *testing.T) { - assert.Equal(t, 0, b2.Count()) - }) -} - -func dontPrecomputeCNA(ctx context.Context, t *testing.T, opts []BucketOption) { - dirName := t.TempDir() - logger, _ := test.NewNullLogger() - - b, err := NewBucket(ctx, dirName, "", logger, nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - opts...) - require.NoError(t, err) - defer b.Shutdown(ctx) - - t.Run("populate, compact", func(t *testing.T) { - require.NoError(t, b.Put([]byte("hello"), []byte("world"))) - require.NoError(t, b.FlushMemtable()) - - require.NoError(t, b.Put([]byte("hello2"), []byte("world2"))) - require.NoError(t, b.FlushMemtable()) - - compacted, err := b.disk.compactOnce() - require.NoError(t, err) - require.True(t, compacted) - }) - - t.Run("check files", func(t *testing.T) { - files, err := os.ReadDir(dirName) - require.NoError(t, err) - - _, ok := findFileWithExt(files, ".cna") - assert.False(t, ok) - }) - - t.Run("count", func(t *testing.T) { - assert.Equal(t, 0, b.Count()) - }) -} - -func findFileWithExt(files []os.DirEntry, ext string) (string, bool) { - for _, file := range files { - fname := file.Name() - if strings.HasSuffix(fname, ext) { - return fname, true - } - - } - return "", false -} - -func corruptCNAFile(fname string, corruptValue uint64) error { - f, err := os.Open(fname) - if err != nil { - return err - } - - data, err := io.ReadAll(f) - if err != nil { - return err - } - - if err := f.Close(); err != nil { - return err - } - - binary.LittleEndian.PutUint64(data[4:12], corruptValue) - - f, err = os.Create(fname) - if err != nil { - return err - } - - _, err = f.Write(data) - if err != nil { - return err - } - - return f.Close() -} diff --git a/adapters/repos/db/lsmkv/segment_precompute_for_compaction.go b/adapters/repos/db/lsmkv/segment_precompute_for_compaction.go deleted file mode 100644 index 271ce038553c0d5462f737ad6b68b8b52e34abf3..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/segment_precompute_for_compaction.go +++ /dev/null @@ -1,129 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "bytes" - "fmt" - "os" - "strings" - - "github.com/edsrzf/mmap-go" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv/segmentindex" -) - -// preComputeSegmentMeta has no side-effects for an already running store. As a -// result this can be run without the need to obtain any locks. All files -// created will have a .tmp suffix so they don't interfere with existing -// segments that might have a similar name. -func preComputeSegmentMeta(path string, updatedCountNetAdditions int, - logger logrus.FieldLogger, useBloomFilter bool, calcCountNetAdditions bool, -) ([]string, error) { - out := []string{path} - - // as a guardrail validate that the segment is considered a .tmp segment. - // This way we can be sure that we're not accidentally operating on a live - // segment as the segment group completely ignores .tmp segment files - if !strings.HasSuffix(path, ".tmp") { - return nil, fmt.Errorf("pre computing a segment expects a .tmp segment path") - } - - file, err := os.Open(path) - if err != nil { - return nil, fmt.Errorf("open file: %w", err) - } - defer file.Close() - - fileInfo, err := file.Stat() - if err != nil { - return nil, fmt.Errorf("stat file: %w", err) - } - - contents, err := mmap.MapRegion(file, int(fileInfo.Size()), mmap.RDONLY, 0, 0) - if err != nil { - return nil, fmt.Errorf("mmap file: %w", err) - } - - defer contents.Unmap() - - header, err := segmentindex.ParseHeader(bytes.NewReader(contents[:segmentindex.HeaderSize])) - if err != nil { - return nil, fmt.Errorf("parse header: %w", err) - } - - switch header.Strategy { - case segmentindex.StrategyReplace, segmentindex.StrategySetCollection, - segmentindex.StrategyMapCollection, segmentindex.StrategyRoaringSet: - default: - return nil, fmt.Errorf("unsupported strategy in segment") - } - - primaryIndex, err := header.PrimaryIndex(contents) - if err != nil { - return nil, fmt.Errorf("extract primary index position: %w", err) - } - - primaryDiskIndex := segmentindex.NewDiskTree(primaryIndex) - - seg := &segment{ - level: header.Level, - // trim the .tmp suffix to make sure the naming rules for the files we - // pre-compute later on still apply they will in turn be suffixed with - // .tmp, but that is supposed to be the end of the file. if we didn't trim - // the path here, we would end up with filenames like - // segment.tmp.bloom.tmp, whereas we want to end up with segment.bloom.tmp - path: strings.TrimSuffix(path, ".tmp"), - contents: contents, - contentFile: file, - version: header.Version, - secondaryIndexCount: header.SecondaryIndices, - segmentStartPos: header.IndexStart, - segmentEndPos: uint64(fileInfo.Size()), - strategy: header.Strategy, - dataStartPos: segmentindex.HeaderSize, // fixed value that's the same for all strategies - dataEndPos: header.IndexStart, - index: primaryDiskIndex, - logger: logger, - useBloomFilter: useBloomFilter, - calcCountNetAdditions: calcCountNetAdditions, - } - - if seg.secondaryIndexCount > 0 { - seg.secondaryIndices = make([]diskIndex, seg.secondaryIndexCount) - for i := range seg.secondaryIndices { - secondary, err := header.SecondaryIndex(contents, uint16(i)) - if err != nil { - return nil, errors.Wrapf(err, "get position for secondary index at %d", i) - } - seg.secondaryIndices[i] = segmentindex.NewDiskTree(secondary) - } - } - - if seg.useBloomFilter { - files, err := seg.precomputeBloomFilters() - if err != nil { - return nil, err - } - out = append(out, files...) - } - if seg.calcCountNetAdditions { - files, err := seg.precomputeCountNetAdditions(updatedCountNetAdditions) - if err != nil { - return nil, err - } - out = append(out, files...) - } - - return out, nil -} diff --git a/adapters/repos/db/lsmkv/segment_precompute_for_compation_test.go b/adapters/repos/db/lsmkv/segment_precompute_for_compation_test.go deleted file mode 100644 index 60c4a29832b55137ac9aa2230cab6db77d112bca..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/segment_precompute_for_compation_test.go +++ /dev/null @@ -1,226 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "context" - "fmt" - "os" - "path" - "strings" - "testing" - - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv/segmentindex" - "github.com/weaviate/weaviate/entities/cyclemanager" -) - -func TestPrecomputeForCompaction(t *testing.T) { - ctx := context.Background() - tests := bucketTests{ - { - name: "precomputeSegmentMeta_Replace", - f: precomputeSegmentMeta_Replace, - opts: []BucketOption{ - WithStrategy(StrategyReplace), - WithSecondaryIndices(1), - }, - }, - { - name: "precomputeSegmentMeta_Set", - f: precomputeSegmentMeta_Set, - opts: []BucketOption{ - WithStrategy(StrategySetCollection), - }, - }, - } - tests.run(ctx, t) -} - -func precomputeSegmentMeta_Replace(ctx context.Context, t *testing.T, opts []BucketOption) { - // first build a complete reference segment of which we can then strip its - // meta - dirName := t.TempDir() - - logger, _ := test.NewNullLogger() - - b, err := NewBucket(ctx, dirName, "", logger, nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - defer b.Shutdown(ctx) - - require.Nil(t, b.Put([]byte("hello"), []byte("world"), - WithSecondaryKey(0, []byte("bonjour")))) - require.Nil(t, b.FlushMemtable()) - - for _, ext := range []string{".secondary.0.bloom", ".bloom", ".cna"} { - files, err := os.ReadDir(dirName) - require.Nil(t, err) - fname, ok := findFileWithExt(files, ext) - require.True(t, ok) - - err = os.RemoveAll(path.Join(dirName, fname)) - require.Nil(t, err) - - files, err = os.ReadDir(dirName) - require.Nil(t, err) - _, ok = findFileWithExt(files, ext) - require.False(t, ok, "verify the file is really gone") - } - - require.Nil(t, b.Shutdown(ctx)) - - // now identify the segment file and rename it to be a tmp file - files, err := os.ReadDir(dirName) - require.Nil(t, err) - fname, ok := findFileWithExt(files, ".db") - require.True(t, ok) - - segmentTmp := path.Join(dirName, fmt.Sprintf("%s.tmp", fname)) - err = os.Rename(path.Join(dirName, fname), segmentTmp) - require.Nil(t, err) - - fileNames, err := preComputeSegmentMeta(segmentTmp, 1, logger, true, true) - require.Nil(t, err) - - // there should be 4 files and they should all have a .tmp suffix: - // segment.db.tmp - // segment.cna.tmp - // segment.bloom.tmp - // segment.secondary.0.bloom.tmp - assert.Len(t, fileNames, 4) - for _, fName := range fileNames { - assert.True(t, strings.HasSuffix(fName, ".tmp")) - } -} - -// Precomputing of segment is almost identical across segment types, however, -// only Replace supports CNA, so we should test at least one other segment type -// which does not support CNA, represented here by using the "Set" type -func precomputeSegmentMeta_Set(ctx context.Context, t *testing.T, opts []BucketOption) { - // first build a complete reference segment of which we can then strip its - // meta - dirName := t.TempDir() - - logger, _ := test.NewNullLogger() - - b, err := NewBucket(ctx, dirName, "", logger, nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - defer b.Shutdown(ctx) - - err = b.SetAdd([]byte("greetings"), [][]byte{[]byte("hello"), []byte("hola")}) - require.Nil(t, err) - require.Nil(t, b.FlushMemtable()) - - files, err := os.ReadDir(dirName) - require.Nil(t, err) - fname, ok := findFileWithExt(files, ".bloom") - require.True(t, ok) - - err = os.RemoveAll(path.Join(dirName, fname)) - require.Nil(t, err) - - // verify it's actually gone - files, err = os.ReadDir(dirName) - require.Nil(t, err) - _, ok = findFileWithExt(files, ".bloom") - require.False(t, ok) - - require.Nil(t, b.Shutdown(ctx)) - - // now identify the segment file and rename it to be a tmp file - fname, ok = findFileWithExt(files, ".db") - require.True(t, ok) - - segmentTmp := path.Join(dirName, fmt.Sprintf("%s.tmp", fname)) - err = os.Rename(path.Join(dirName, fname), segmentTmp) - require.Nil(t, err) - - fileNames, err := preComputeSegmentMeta(segmentTmp, 1, logger, true, true) - require.Nil(t, err) - - // there should be 2 files and they should all have a .tmp suffix: - // segment.db.tmp - // segment.bloom.tmp - assert.Len(t, fileNames, 2) - for _, fName := range fileNames { - assert.True(t, strings.HasSuffix(fName, ".tmp")) - } -} - -func TestPrecomputeSegmentMeta_UnhappyPaths(t *testing.T) { - t.Run("file without .tmp suffix", func(t *testing.T) { - logger, _ := test.NewNullLogger() - _, err := preComputeSegmentMeta("a-path-without-the-required-suffix", 7, logger, true, true) - require.NotNil(t, err) - assert.Contains(t, err.Error(), "expects a .tmp segment") - }) - - t.Run("file does not exist", func(t *testing.T) { - logger, _ := test.NewNullLogger() - _, err := preComputeSegmentMeta("i-dont-exist.tmp", 7, logger, true, true) - require.NotNil(t, err) - unixErr := "no such file or directory" - windowsErr := "The system cannot find the file specified." - assert.True(t, strings.Contains(err.Error(), unixErr) || strings.Contains(err.Error(), windowsErr)) - }) - - t.Run("segment header can't be parsed", func(t *testing.T) { - logger, _ := test.NewNullLogger() - dirName := t.TempDir() - segmentName := path.Join(dirName, "my-segment.tmp") - - header := &segmentindex.Header{ - Version: 100, // only supported version as of writing this test is 0 - } - - f, err := os.Create(segmentName) - require.Nil(t, err) - - _, err = header.WriteTo(f) - require.Nil(t, err) - - err = f.Close() - require.Nil(t, err) - - _, err = preComputeSegmentMeta(segmentName, 7, logger, true, true) - require.NotNil(t, err) - assert.Contains(t, err.Error(), "parse header") - }) - - t.Run("unsupported strategy", func(t *testing.T) { - logger, _ := test.NewNullLogger() - dirName := t.TempDir() - segmentName := path.Join(dirName, "my-segment.tmp") - - header := &segmentindex.Header{ - Version: 0, - Strategy: segmentindex.Strategy(100), // this strategy doesn't exist - } - - f, err := os.Create(segmentName) - require.Nil(t, err) - - _, err = header.WriteTo(f) - require.Nil(t, err) - - err = f.Close() - require.Nil(t, err) - - _, err = preComputeSegmentMeta(segmentName, 7, logger, true, true) - require.NotNil(t, err) - assert.Contains(t, err.Error(), "unsupported strategy") - }) -} diff --git a/adapters/repos/db/lsmkv/segment_replace_strategy.go b/adapters/repos/db/lsmkv/segment_replace_strategy.go deleted file mode 100644 index 9167e3d5b399eefce455bf90d6e32692a700ad56..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/segment_replace_strategy.go +++ /dev/null @@ -1,133 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "encoding/binary" - "errors" - "fmt" - "time" - - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv/segmentindex" - "github.com/weaviate/weaviate/entities/lsmkv" -) - -func (s *segment) get(key []byte) ([]byte, error) { - if s.strategy != segmentindex.StrategyReplace { - return nil, fmt.Errorf("get only possible for strategy %q", StrategyReplace) - } - - before := time.Now() - - if s.useBloomFilter && !s.bloomFilter.Test(key) { - s.bloomFilterMetrics.trueNegative(before) - return nil, lsmkv.NotFound - } - - node, err := s.index.Get(key) - if err != nil { - if errors.Is(err, lsmkv.NotFound) { - if s.useBloomFilter { - s.bloomFilterMetrics.falsePositive(before) - } - return nil, lsmkv.NotFound - } else { - return nil, err - } - } - - defer func() { - if s.useBloomFilter { - s.bloomFilterMetrics.truePositive(before) - } - }() - - // We need to copy the data we read from the segment exactly once in this - // place. This means that future processing can share this memory as much as - // it wants to, as it can now be considered immutable. If we didn't copy in - // this place it would only be safe to hold this data while still under the - // protection of the segmentGroup.maintenanceLock. This lock makes sure that - // no compaction is started during an ongoing read. However, once read, - // further processing is no longer protected by lock. - // If a compaction completes and the old segment is removed, we would be accessing - // invalid memory without the copy, thus leading to a SEGFAULT. - // Similar approach was used to fix SEGFAULT in collection strategy - // https://github.com/weaviate/weaviate/issues/1837 - contentsCopy := make([]byte, node.End-node.Start) - if err = s.copyNode(contentsCopy, nodeOffset{node.Start, node.End}); err != nil { - return nil, err - } - - return s.replaceStratParseData(contentsCopy) -} - -func (s *segment) getBySecondaryIntoMemory(pos int, key []byte, buffer []byte) ([]byte, error, []byte) { - if s.strategy != segmentindex.StrategyReplace { - return nil, fmt.Errorf("get only possible for strategy %q", StrategyReplace), nil - } - - if pos > len(s.secondaryIndices) || s.secondaryIndices[pos] == nil { - return nil, fmt.Errorf("no secondary index at pos %d", pos), nil - } - - if s.useBloomFilter && !s.secondaryBloomFilters[pos].Test(key) { - return nil, lsmkv.NotFound, nil - } - - node, err := s.secondaryIndices[pos].Get(key) - if err != nil { - return nil, err, nil - } - - // We need to copy the data we read from the segment exactly once in this - // place. This means that future processing can share this memory as much as - // it wants to, as it can now be considered immutable. If we didn't copy in - // this place it would only be safe to hold this data while still under the - // protection of the segmentGroup.maintenanceLock. This lock makes sure that - // no compaction is started during an ongoing read. However, once read, - // further processing is no longer protected by lock. - // If a compaction completes and the old segment is removed, we would be accessing - // invalid memory without the copy, thus leading to a SEGFAULT. - // Similar approach was used to fix SEGFAULT in collection strategy - // https://github.com/weaviate/weaviate/issues/1837 - var contentsCopy []byte - if uint64(cap(buffer)) >= node.End-node.Start { - contentsCopy = buffer[:node.End-node.Start] - } else { - contentsCopy = make([]byte, node.End-node.Start) - } - if err = s.copyNode(contentsCopy, nodeOffset{node.Start, node.End}); err != nil { - return nil, err, nil - } - currContent, err := s.replaceStratParseData(contentsCopy) - return currContent, err, contentsCopy -} - -func (s *segment) replaceStratParseData(in []byte) ([]byte, error) { - if len(in) == 0 { - return nil, lsmkv.NotFound - } - - // byte meaning - // 0 is tombstone - // 1-8 data length as Little Endian uint64 - // 9-length data - - // check the tombstone byte - if in[0] == 0x01 { - return nil, lsmkv.Deleted - } - - valueLength := binary.LittleEndian.Uint64(in[1:9]) - - return in[9 : 9+valueLength], nil -} diff --git a/adapters/repos/db/lsmkv/segment_roaring_set_strategy.go b/adapters/repos/db/lsmkv/segment_roaring_set_strategy.go deleted file mode 100644 index 6d642c8c2bb59446c22e5380bc328f715c4699b7..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/segment_roaring_set_strategy.go +++ /dev/null @@ -1,68 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "fmt" - - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv/roaringset" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv/segmentindex" - "github.com/weaviate/weaviate/entities/lsmkv" -) - -func (s *segment) roaringSetGet(key []byte) (roaringset.BitmapLayer, error) { - out := roaringset.BitmapLayer{} - - if s.strategy != segmentindex.StrategyRoaringSet { - return out, fmt.Errorf("need strategy %s", StrategyRoaringSet) - } - - if s.useBloomFilter && !s.bloomFilter.Test(key) { - return out, lsmkv.NotFound - } - - node, err := s.index.Get(key) - if err != nil { - return out, err - } - - sn, err := s.segmentNodeFromBuffer(nodeOffset{node.Start, node.End}) - if err != nil { - return out, err - } - - // make sure that any data is copied before exiting this method, otherwise we - // risk a SEGFAULT as described in - // https://github.com/weaviate/weaviate/issues/1837 - out.Additions = sn.AdditionsWithCopy() - out.Deletions = sn.DeletionsWithCopy() - return out, nil -} - -func (s *segment) segmentNodeFromBuffer(offset nodeOffset) (*roaringset.SegmentNode, error) { - var contents []byte - if s.mmapContents { - contents = s.contents[offset.start:offset.end] - } else { - contents = make([]byte, offset.end-offset.start) - r, err := s.bufferedReaderAt(offset.start) - if err != nil { - return nil, err - } - _, err = r.Read(contents) - if err != nil { - return nil, err - } - } - - return roaringset.NewSegmentNodeFromBuffer(contents), nil -} diff --git a/adapters/repos/db/lsmkv/segment_serialization.go b/adapters/repos/db/lsmkv/segment_serialization.go deleted file mode 100644 index 1c34f1014d323e3323b2189c63b551c306cd6712..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/segment_serialization.go +++ /dev/null @@ -1,495 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "encoding/binary" - "fmt" - "io" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv/segmentindex" - "github.com/weaviate/weaviate/usecases/byteops" -) - -// a single node of strategy "replace" -type segmentReplaceNode struct { - tombstone bool - value []byte - primaryKey []byte - secondaryIndexCount uint16 - secondaryKeys [][]byte - offset int -} - -func (s *segmentReplaceNode) KeyIndexAndWriteTo(w io.Writer) (segmentindex.Key, error) { - out := segmentindex.Key{} - written := 0 - - buf := make([]byte, 9) - if s.tombstone { - buf[0] = 1 - } else { - buf[0] = 0 - } - - valueLength := uint64(len(s.value)) - binary.LittleEndian.PutUint64(buf[1:9], valueLength) - if _, err := w.Write(buf); err != nil { - return out, err - } - - written += 9 - - n, err := w.Write(s.value) - if err != nil { - return out, errors.Wrapf(err, "write node value") - } - written += n - - keyLength := uint32(len(s.primaryKey)) - binary.LittleEndian.PutUint32(buf[0:4], keyLength) - if _, err := w.Write(buf[0:4]); err != nil { - return out, err - } - written += 4 - - n, err = w.Write(s.primaryKey) - if err != nil { - return out, errors.Wrapf(err, "write node key") - } - written += n - - for j := 0; j < int(s.secondaryIndexCount); j++ { - var secondaryKeyLength uint32 - if j < len(s.secondaryKeys) { - secondaryKeyLength = uint32(len(s.secondaryKeys[j])) - } - - // write the key length in any case - binary.LittleEndian.PutUint32(buf[0:4], secondaryKeyLength) - if _, err := w.Write(buf[0:4]); err != nil { - return out, err - } - written += 4 - - if secondaryKeyLength == 0 { - // we're done here - continue - } - - // only write the key if it exists - n, err = w.Write(s.secondaryKeys[j]) - if err != nil { - return out, errors.Wrapf(err, "write secondary key %d", j) - } - written += n - } - - return segmentindex.Key{ - ValueStart: s.offset, - ValueEnd: s.offset + written, - Key: s.primaryKey, - SecondaryKeys: s.secondaryKeys, - }, nil -} - -func ParseReplaceNode(r io.Reader, secondaryIndexCount uint16) (segmentReplaceNode, error) { - out := segmentReplaceNode{} - - // 9 bytes is the most we can ever read uninterrupted, i.e. without a dynamic - // read in between. - tmpBuf := make([]byte, 9) - if n, err := io.ReadFull(r, tmpBuf); err != nil { - return out, errors.Wrap(err, "read tombstone and value length") - } else { - out.offset += n - } - - out.tombstone = tmpBuf[0] == 0x1 - valueLength := binary.LittleEndian.Uint64(tmpBuf[1:9]) - out.value = make([]byte, valueLength) - if n, err := io.ReadFull(r, out.value); err != nil { - return out, errors.Wrap(err, "read value") - } else { - out.offset += n - } - - if n, err := io.ReadFull(r, tmpBuf[0:4]); err != nil { - return out, errors.Wrap(err, "read key length encoding") - } else { - out.offset += n - } - - keyLength := binary.LittleEndian.Uint32(tmpBuf[0:4]) - out.primaryKey = make([]byte, keyLength) - if n, err := io.ReadFull(r, out.primaryKey); err != nil { - return out, errors.Wrap(err, "read key") - } else { - out.offset += n - } - - if secondaryIndexCount > 0 { - out.secondaryKeys = make([][]byte, secondaryIndexCount) - } - - for j := 0; j < int(secondaryIndexCount); j++ { - if n, err := io.ReadFull(r, tmpBuf[0:4]); err != nil { - return out, errors.Wrap(err, "read secondary key length encoding") - } else { - out.offset += n - } - secKeyLen := binary.LittleEndian.Uint32(tmpBuf[0:4]) - if secKeyLen == 0 { - continue - } - - out.secondaryKeys[j] = make([]byte, secKeyLen) - if n, err := io.ReadFull(r, out.secondaryKeys[j]); err != nil { - return out, errors.Wrap(err, "read secondary key") - } else { - out.offset += n - } - } - - return out, nil -} - -func ParseReplaceNodeIntoPread(r io.Reader, secondaryIndexCount uint16, out *segmentReplaceNode) error { - out.offset = 0 - - if err := binary.Read(r, binary.LittleEndian, &out.tombstone); err != nil { - return errors.Wrap(err, "read tombstone") - } - out.offset += 1 - - var valueLength uint64 - if err := binary.Read(r, binary.LittleEndian, &valueLength); err != nil { - return errors.Wrap(err, "read value length encoding") - } - out.offset += 8 - - if int(valueLength) > cap(out.value) { - out.value = make([]byte, valueLength) - } else { - out.value = out.value[:valueLength] - } - - if n, err := r.Read(out.value); err != nil { - return errors.Wrap(err, "read value") - } else { - out.offset += n - } - - var keyLength uint32 - if err := binary.Read(r, binary.LittleEndian, &keyLength); err != nil { - return errors.Wrap(err, "read key length encoding") - } - out.offset += 4 - - out.primaryKey = make([]byte, keyLength) - if n, err := r.Read(out.primaryKey); err != nil { - return errors.Wrap(err, "read key") - } else { - out.offset += n - } - - if secondaryIndexCount > 0 { - out.secondaryKeys = make([][]byte, secondaryIndexCount) - } - - for j := 0; j < int(secondaryIndexCount); j++ { - var secKeyLen uint32 - if err := binary.Read(r, binary.LittleEndian, &secKeyLen); err != nil { - return errors.Wrap(err, "read secondary key length encoding") - } - out.offset += 4 - - if secKeyLen == 0 { - continue - } - - out.secondaryKeys[j] = make([]byte, secKeyLen) - if n, err := r.Read(out.secondaryKeys[j]); err != nil { - return errors.Wrap(err, "read secondary key") - } else { - out.offset += n - } - } - - return nil -} - -func ParseReplaceNodeIntoMMAP(r *byteops.ReadWriter, secondaryIndexCount uint16, out *segmentReplaceNode) error { - out.tombstone = r.ReadUint8() == 0x01 - valueLength := r.ReadUint64() - - if int(valueLength) > cap(out.value) { - out.value = make([]byte, valueLength) - } else { - out.value = out.value[:valueLength] - } - - if _, err := r.CopyBytesFromBuffer(valueLength, out.value); err != nil { - return err - } - - // Note: In a previous version (prior to - // https://github.com/weaviate/weaviate/pull/3660) this was a copy. The - // mentioned PR optimizes the Replace Cursor which led to this now being - // shared memory. After internal review, we believe this is safe to do. The - // cursor gives no guarantees about memory after calling .next(). Before - // .next() is called, this should be safe. Nevertheless, we are leaving this - // note in case a future bug appears, as this should make this spot easier to - // find. - out.primaryKey = r.ReadBytesFromBufferWithUint32LengthIndicator() - - if secondaryIndexCount > 0 { - out.secondaryKeys = make([][]byte, secondaryIndexCount) - } - - for j := 0; j < int(secondaryIndexCount); j++ { - // Note: In a previous version (prior to - // https://github.com/weaviate/weaviate/pull/3660) this was a copy. The - // mentioned PR optimizes the Replace Cursor which led to this now being - // shared memory. After internal review, we believe this is safe to do. The - // cursor gives no guarantees about memory after calling .next(). Before - // .next() is called, this should be safe. Nevertheless, we are leaving this - // note in case a future bug appears, as this should make this spot easier to - // find. - out.secondaryKeys[j] = r.ReadBytesFromBufferWithUint32LengthIndicator() - } - - out.offset = int(r.Position) - return nil -} - -// collection strategy does not support secondary keys at this time -type segmentCollectionNode struct { - values []value - primaryKey []byte - offset int -} - -func (s segmentCollectionNode) KeyIndexAndWriteTo(w io.Writer) (segmentindex.Key, error) { - out := segmentindex.Key{} - written := 0 - valueLen := uint64(len(s.values)) - buf := make([]byte, 9) - binary.LittleEndian.PutUint64(buf, valueLen) - if _, err := w.Write(buf[0:8]); err != nil { - return out, errors.Wrapf(err, "write values len for node") - } - written += 8 - - for i, value := range s.values { - if value.tombstone { - buf[0] = 0x01 - } else { - buf[0] = 0x00 - } - - valueLen := uint64(len(value.value)) - binary.LittleEndian.PutUint64(buf[1:9], valueLen) - if _, err := w.Write(buf[0:9]); err != nil { - return out, errors.Wrapf(err, "write len of value %d", i) - } - written += 9 - - n, err := w.Write(value.value) - if err != nil { - return out, errors.Wrapf(err, "write value %d", i) - } - written += n - } - - keyLength := uint32(len(s.primaryKey)) - binary.LittleEndian.PutUint32(buf[0:4], keyLength) - if _, err := w.Write(buf[0:4]); err != nil { - return out, errors.Wrapf(err, "write key length encoding for node") - } - written += 4 - - n, err := w.Write(s.primaryKey) - if err != nil { - return out, errors.Wrapf(err, "write node") - } - written += n - - out = segmentindex.Key{ - ValueStart: s.offset, - ValueEnd: s.offset + written, - Key: s.primaryKey, - } - - return out, nil -} - -// ParseCollectionNode reads from r and parses the collection values into a segmentCollectionNode -// -// When only given an offset, r is constructed as a *bufio.Reader to avoid first reading the -// entire segment (could be GBs). Each consecutive read will be buffered to avoid excessive -// syscalls. -// -// When we already have a finite and manageable []byte (i.e. when we have already seeked to an -// lsmkv node and have start+end offset), r should be constructed as a *bytes.Reader, since the -// contents have already been `pread` from the segment contentFile. -func ParseCollectionNode(r io.Reader) (segmentCollectionNode, error) { - out := segmentCollectionNode{} - // 9 bytes is the most we can ever read uninterrupted, i.e. without a dynamic - // read in between. - tmpBuf := make([]byte, 9) - - if n, err := io.ReadFull(r, tmpBuf[0:8]); err != nil { - return out, errors.Wrap(err, "read values len") - } else { - out.offset += n - } - - valuesLen := binary.LittleEndian.Uint64(tmpBuf[0:8]) - out.values = make([]value, valuesLen) - for i := range out.values { - if n, err := io.ReadFull(r, tmpBuf[0:9]); err != nil { - return out, errors.Wrap(err, "read value tombstone and len") - } else { - out.offset += n - } - out.values[i].tombstone = tmpBuf[0] == 0x1 - valueLen := binary.LittleEndian.Uint64(tmpBuf[1:9]) - out.values[i].value = make([]byte, valueLen) - n, err := io.ReadFull(r, out.values[i].value) - if err != nil { - return out, errors.Wrap(err, "read value") - } - out.offset += n - } - - if n, err := io.ReadFull(r, tmpBuf[0:4]); err != nil { - return out, errors.Wrap(err, "read key len") - } else { - out.offset += n - } - keyLen := binary.LittleEndian.Uint32(tmpBuf[0:4]) - out.primaryKey = make([]byte, keyLen) - n, err := io.ReadFull(r, out.primaryKey) - if err != nil { - return out, errors.Wrap(err, "read key") - } - out.offset += n - - return out, nil -} - -// ParseCollectionNodeInto takes the []byte slice and parses it into the -// specified node. It does not perform any copies and the caller must be aware -// that memory may be shared between the two. As a result, the caller must make -// sure that they do not modify "in" while "node" is still in use. A safer -// alternative is to use ParseCollectionNode. -// -// The primary intention of this function is to provide a way to reuse buffers -// when the lifetime is controlled tightly, for example in cursors used within -// compactions. Use at your own risk! -// -// If the buffers of the provided node have enough capacity they will be -// reused. Only if the capacity is not enough, will an allocation occur. This -// allocation uses 25% overhead to avoid future allocations for nodes of -// similar size. -// -// As a result calling this method only makes sense if you plan on calling it -// multiple times. Calling it just once on an uninitialized node does not have -// major advantages over calling ParseCollectionNode. -func ParseCollectionNodeInto(r io.Reader, node *segmentCollectionNode) error { - // offset is only the local offset relative to "in". In the end we need to - // update the global offset. - offset := 0 - - buf := make([]byte, 9) - _, err := io.ReadFull(r, buf[0:8]) - if err != nil { - return fmt.Errorf("read values len: %w", err) - } - - valuesLen := binary.LittleEndian.Uint64(buf[0:8]) - offset += 8 - - resizeValuesOfCollectionNode(node, valuesLen) - for i := range node.values { - _, err = io.ReadFull(r, buf) - if err != nil { - return fmt.Errorf("read values len: %w", err) - } - - node.values[i].tombstone = buf[0] == 0x1 - offset += 1 - - valueLen := binary.LittleEndian.Uint64(buf[1:9]) - offset += 8 - - resizeValueOfCollectionNodeAtPos(node, i, valueLen) - - _, err = io.ReadFull(r, node.values[i].value) - if err != nil { - return fmt.Errorf("read node value: %w", err) - } - - offset += int(valueLen) - } - - _, err = io.ReadFull(r, buf[0:4]) - if err != nil { - return fmt.Errorf("read values len: %w", err) - } - keyLen := binary.LittleEndian.Uint32(buf) - offset += 4 - - resizeKeyOfCollectionNode(node, keyLen) - _, err = io.ReadFull(r, node.primaryKey) - if err != nil { - return fmt.Errorf("read primary key: %w", err) - } - offset += int(keyLen) - - node.offset = offset - return nil -} - -func resizeValuesOfCollectionNode(node *segmentCollectionNode, size uint64) { - if cap(node.values) >= int(size) { - node.values = node.values[:size] - } else { - // Allocate with 25% overhead to reduce chance of having to do multiple - // allocations sequentially. - node.values = make([]value, size, int(float64(size)*1.25)) - } -} - -func resizeValueOfCollectionNodeAtPos(node *segmentCollectionNode, pos int, - size uint64, -) { - if cap(node.values[pos].value) >= int(size) { - node.values[pos].value = node.values[pos].value[:size] - } else { - // Allocate with 25% overhead to reduce chance of having to do multiple - // allocations sequentially. - node.values[pos].value = make([]byte, size, int(float64(size)*1.25)) - } -} - -func resizeKeyOfCollectionNode(node *segmentCollectionNode, size uint32) { - if cap(node.primaryKey) >= int(size) { - node.primaryKey = node.primaryKey[:size] - } else { - // Allocate with 25% overhead to reduce chance of having to do multiple - // allocations sequentially. - node.primaryKey = make([]byte, size, int(float64(size)*1.25)) - } -} diff --git a/adapters/repos/db/lsmkv/segment_serialization_benchmarks_test.go b/adapters/repos/db/lsmkv/segment_serialization_benchmarks_test.go deleted file mode 100644 index 12db8657f0cba7c51405c8adc9c9992f9e162169..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/segment_serialization_benchmarks_test.go +++ /dev/null @@ -1,65 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "bytes" - "testing" - - "github.com/stretchr/testify/require" -) - -func BenchmarkReplaceNodeKeyIndexAndWriteTo(b *testing.B) { - // targetBuf := bytes.NewBuffer(make([]byte, 32*1024*1024)) // large enough to avoid growths during running - targetBuf := bytes.NewBuffer(nil) // large enough to avoid growths during running - - node := segmentReplaceNode{ - tombstone: true, - value: []byte("foo bar"), - primaryKey: []byte("foo bar"), - secondaryIndexCount: 1, - secondaryKeys: [][]byte{[]byte("foo bar")}, - offset: 27, - } - - b.ReportAllocs() - - for i := 0; i < b.N; i++ { - _, err := node.KeyIndexAndWriteTo(targetBuf) - require.Nil(b, err) - } -} - -func BenchmarkCollectionNodeKeyIndexAndWriteTo(b *testing.B) { - targetBuf := bytes.NewBuffer(make([]byte, 32*1024*1024)) // large enough to avoid growths during running - - node := segmentCollectionNode{ - primaryKey: []byte("foo bar"), - offset: 27, - values: []value{ - { - value: []byte("my-value"), - tombstone: true, - }, - { - value: []byte("my-value"), - tombstone: true, - }, - }, - } - - b.ReportAllocs() - - for i := 0; i < b.N; i++ { - node.KeyIndexAndWriteTo(targetBuf) - } -} diff --git a/adapters/repos/db/lsmkv/segment_serialization_test.go b/adapters/repos/db/lsmkv/segment_serialization_test.go deleted file mode 100644 index 11f53c7eb846a8ab54579b12f091cb41316cf81f..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/segment_serialization_test.go +++ /dev/null @@ -1,61 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "bytes" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func Test_SerializeAndParseCollectionNode(t *testing.T) { - before := segmentCollectionNode{ - primaryKey: []byte("this-is-my-primary-key"), - values: []value{{ - value: []byte("the-first-value"), - }, { - value: []byte("the-second-value-with-a-tombstone"), - tombstone: true, - }}, - } - - buf := &bytes.Buffer{} - _, err := before.KeyIndexAndWriteTo(buf) - require.Nil(t, err) - encoded := buf.Bytes() - - expected := segmentCollectionNode{ - primaryKey: []byte("this-is-my-primary-key"), - values: []value{{ - value: []byte("the-first-value"), - }, { - value: []byte("the-second-value-with-a-tombstone"), - tombstone: true, - }}, - offset: buf.Len(), - } - - t.Run("parse using the _regular_ way", func(t *testing.T) { - after, err := ParseCollectionNode(bytes.NewReader(encoded)) - assert.Nil(t, err) - assert.Equal(t, expected, after) - }) - - t.Run("parse using the reusable way", func(t *testing.T) { - var node segmentCollectionNode - err := ParseCollectionNodeInto(bytes.NewReader(encoded), &node) - assert.Nil(t, err) - assert.Equal(t, expected, node) - }) -} diff --git a/adapters/repos/db/lsmkv/segmentindex/balanced_test.go b/adapters/repos/db/lsmkv/segmentindex/balanced_test.go deleted file mode 100644 index 8ab4bec1a705e1caea6356c11449943e8c4ec734..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/segmentindex/balanced_test.go +++ /dev/null @@ -1,67 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package segmentindex - -import ( - "crypto/rand" - "fmt" - "math" - "math/big" - "testing" - - "github.com/stretchr/testify/assert" -) - -func mustRandUint64() uint64 { - randInt, err := rand.Int(rand.Reader, big.NewInt(math.MaxInt64)) - if err != nil { - panic(fmt.Sprintf("mustRandUint64 error: %v", err)) - } - return randInt.Uint64() -} - -func TestBuildBalancedTree(t *testing.T) { - size := 2000 - idealHeight := int(math.Ceil(math.Log2(float64(size)))) - fmt.Printf("ideal height would be %d\n", idealHeight) - - nodes := make([]Node, size) - var tree Tree - - t.Run("generate random data", func(t *testing.T) { - for i := range nodes { - nodes[i].Key = make([]byte, 8) - rand.Read(nodes[i].Key) - - nodes[i].Start = mustRandUint64() - nodes[i].End = mustRandUint64() - } - }) - - t.Run("insert", func(t *testing.T) { - tree = NewBalanced(nodes) - }) - - t.Run("check height", func(t *testing.T) { - assert.Equal(t, idealHeight, tree.Height()) - }) - - t.Run("check values", func(t *testing.T) { - for _, control := range nodes { - k, s, e := tree.Get(control.Key) - - assert.Equal(t, control.Key, k) - assert.Equal(t, control.Start, s) - assert.Equal(t, control.End, e) - } - }) -} diff --git a/adapters/repos/db/lsmkv/segmentindex/disk_tree.go b/adapters/repos/db/lsmkv/segmentindex/disk_tree.go deleted file mode 100644 index b1c57182c1135648280089e8b3e769a411c776a5..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/segmentindex/disk_tree.go +++ /dev/null @@ -1,197 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package segmentindex - -import ( - "bytes" - "errors" - "fmt" - "io" - - "github.com/weaviate/weaviate/entities/lsmkv" - "github.com/weaviate/weaviate/usecases/byteops" -) - -// DiskTree is a read-only wrapper around a marshalled index search tree, which -// can be used for reading, but cannot change the underlying structure. It is -// thus perfectly suited as an index for an (immutable) LSM disk segment, but -// pretty much useless for anything else -type DiskTree struct { - data []byte -} - -type dtNode struct { - key []byte - startPos uint64 - endPos uint64 - leftChild int64 - rightChild int64 -} - -func NewDiskTree(data []byte) *DiskTree { - return &DiskTree{ - data: data, - } -} - -func (t *DiskTree) Get(key []byte) (Node, error) { - if len(t.data) == 0 { - return Node{}, lsmkv.NotFound - } - var out Node - rw := byteops.NewReadWriter(t.data) - - // jump to the buffer until the node with _key_ is found or return a NotFound error. - // This function avoids allocations by reusing the same buffer for all keys and avoids memory reads by only - // extracting the necessary pieces of information while skipping the rest - NodeKeyBuffer := make([]byte, len(key)) - for { - // detect if there is no node with the wanted key. - if rw.Position+4 > uint64(len(t.data)) || rw.Position+4 < 4 { - return out, lsmkv.NotFound - } - - keyLen := rw.ReadUint32() - if int(keyLen) > len(NodeKeyBuffer) { - NodeKeyBuffer = make([]byte, int(keyLen)) - } else if int(keyLen) < len(NodeKeyBuffer) { - NodeKeyBuffer = NodeKeyBuffer[:keyLen] - } - _, err := rw.CopyBytesFromBuffer(uint64(keyLen), NodeKeyBuffer) - if err != nil { - return out, fmt.Errorf("copy node key: %w", err) - } - - keyEqual := bytes.Compare(key, NodeKeyBuffer) - if keyEqual == 0 { - out.Key = NodeKeyBuffer - out.Start = rw.ReadUint64() - out.End = rw.ReadUint64() - return out, nil - } else if keyEqual < 0 { - rw.MoveBufferPositionForward(2 * 8) // jump over start+end position - rw.Position = rw.ReadUint64() // left child - } else { - rw.MoveBufferPositionForward(3 * 8) // jump over start+end position and left child - rw.Position = rw.ReadUint64() // right child - } - } -} - -func (t *DiskTree) readNodeAt(offset int64) (dtNode, error) { - retNode, _, err := t.readNode(t.data[offset:]) - return retNode, err -} - -func (t *DiskTree) readNode(in []byte) (dtNode, int, error) { - var out dtNode - // in buffer needs at least 36 bytes of data: - // 4bytes for key length, 32bytes for position and children - if len(in) < 36 { - return out, 0, io.EOF - } - - rw := byteops.NewReadWriter(in) - - keyLen := uint64(rw.ReadUint32()) - copiedBytes, err := rw.CopyBytesFromBuffer(keyLen, nil) - if err != nil { - return out, int(rw.Position), fmt.Errorf("copy node key: %w", err) - } - out.key = copiedBytes - - out.startPos = rw.ReadUint64() - out.endPos = rw.ReadUint64() - out.leftChild = int64(rw.ReadUint64()) - out.rightChild = int64(rw.ReadUint64()) - return out, int(rw.Position), nil -} - -func (t *DiskTree) Seek(key []byte) (Node, error) { - if len(t.data) == 0 { - return Node{}, lsmkv.NotFound - } - - return t.seekAt(0, key) -} - -func (t *DiskTree) seekAt(offset int64, key []byte) (Node, error) { - node, err := t.readNodeAt(offset) - if err != nil { - return Node{}, err - } - - self := Node{ - Key: node.key, - Start: node.startPos, - End: node.endPos, - } - - if bytes.Equal(key, node.key) { - return self, nil - } - - if bytes.Compare(key, node.key) < 0 { - if node.leftChild < 0 { - return self, nil - } - - left, err := t.seekAt(node.leftChild, key) - if err == nil { - return left, nil - } - - if errors.Is(err, lsmkv.NotFound) { - return self, nil - } - - return Node{}, err - } else { - if node.rightChild < 0 { - return Node{}, lsmkv.NotFound - } - - return t.seekAt(node.rightChild, key) - } -} - -// AllKeys is a relatively expensive operation as it basically does a full disk -// read of the index. It is meant for one of operations, such as initializing a -// segment where we need access to all keys, e.g. to build a bloom filter. This -// should not run at query time. -// -// The binary tree is traversed in Level-Order so keys have no meaningful -// order. Do not use this method if an In-Order traversal is required, but only -// for use cases who don't require a specific order, such as building a -// bloom filter. -func (t *DiskTree) AllKeys() ([][]byte, error) { - var out [][]byte - bufferPos := 0 - for { - node, readLength, err := t.readNode(t.data[bufferPos:]) - bufferPos += readLength - if err == io.EOF { - break - } - if err != nil { - return nil, err - } - - out = append(out, node.key) - } - - return out, nil -} - -func (t *DiskTree) Size() int { - return len(t.data) -} diff --git a/adapters/repos/db/lsmkv/segmentindex/header.go b/adapters/repos/db/lsmkv/segmentindex/header.go deleted file mode 100644 index e2a638d9c4a44f78cf85caf3842530b9fbee6efb..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/segmentindex/header.go +++ /dev/null @@ -1,137 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package segmentindex - -import ( - "bufio" - "bytes" - "encoding/binary" - "fmt" - "io" -) - -// HeaderSize describes the general offset in a segment until the data -// starts, it is composed of 2 bytes for level, 2 bytes for version, -// 2 bytes for secondary index count, 2 bytes for strategy, 8 bytes -// for the pointer to the index part -const HeaderSize = 16 - -type Header struct { - Level uint16 - Version uint16 - SecondaryIndices uint16 - Strategy Strategy - IndexStart uint64 -} - -func (h *Header) WriteTo(w io.Writer) (int64, error) { - if err := binary.Write(w, binary.LittleEndian, &h.Level); err != nil { - return -1, err - } - if err := binary.Write(w, binary.LittleEndian, &h.Version); err != nil { - return -1, err - } - if err := binary.Write(w, binary.LittleEndian, &h.SecondaryIndices); err != nil { - return -1, err - } - if err := binary.Write(w, binary.LittleEndian, h.Strategy); err != nil { - return -1, err - } - if err := binary.Write(w, binary.LittleEndian, &h.IndexStart); err != nil { - return -1, err - } - - return int64(HeaderSize), nil -} - -func (h *Header) PrimaryIndex(source []byte) ([]byte, error) { - if h.SecondaryIndices == 0 { - return source[h.IndexStart:], nil - } - - offsets, err := h.parseSecondaryIndexOffsets( - source[h.IndexStart:h.secondaryIndexOffsetsEnd()]) - if err != nil { - return nil, err - } - - // the beginning of the first secondary is also the end of the primary - end := offsets[0] - return source[h.secondaryIndexOffsetsEnd():end], nil -} - -func (h *Header) secondaryIndexOffsetsEnd() uint64 { - return h.IndexStart + (uint64(h.SecondaryIndices) * 8) -} - -func (h *Header) parseSecondaryIndexOffsets(source []byte) ([]uint64, error) { - r := bufio.NewReader(bytes.NewReader(source)) - - offsets := make([]uint64, h.SecondaryIndices) - if err := binary.Read(r, binary.LittleEndian, &offsets); err != nil { - return nil, err - } - - return offsets, nil -} - -func (h *Header) SecondaryIndex(source []byte, indexID uint16) ([]byte, error) { - if indexID >= h.SecondaryIndices { - return nil, fmt.Errorf("retrieve index %d with len %d", - indexID, h.SecondaryIndices) - } - - offsets, err := h.parseSecondaryIndexOffsets( - source[h.IndexStart:h.secondaryIndexOffsetsEnd()]) - if err != nil { - return nil, err - } - - start := offsets[indexID] - if indexID == h.SecondaryIndices-1 { - // this is the last index, return until EOF - return source[start:], nil - } - - end := offsets[indexID+1] - return source[start:end], nil -} - -func ParseHeader(r io.Reader) (*Header, error) { - out := &Header{} - - if err := binary.Read(r, binary.LittleEndian, &out.Level); err != nil { - return nil, err - } - - if err := binary.Read(r, binary.LittleEndian, &out.Version); err != nil { - return nil, err - } - - if err := binary.Read(r, binary.LittleEndian, &out.SecondaryIndices); err != nil { - return nil, err - } - - if out.Version != 0 { - return nil, fmt.Errorf("unsupported version %d", out.Version) - } - - if err := binary.Read(r, binary.LittleEndian, &out.Strategy); err != nil { - return nil, err - } - - if err := binary.Read(r, binary.LittleEndian, &out.IndexStart); err != nil { - return nil, err - } - - return out, nil -} diff --git a/adapters/repos/db/lsmkv/segmentindex/indexes.go b/adapters/repos/db/lsmkv/segmentindex/indexes.go deleted file mode 100644 index 8fe00d415ccb676efc45040a0ee76937b95bdcff..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/segmentindex/indexes.go +++ /dev/null @@ -1,200 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package segmentindex - -import ( - "bufio" - "bytes" - "encoding/binary" - "io" - "os" - "path/filepath" - "sort" - - "github.com/pkg/errors" -) - -type Indexes struct { - Keys []Key - SecondaryIndexCount uint16 - ScratchSpacePath string -} - -func (s Indexes) WriteTo(w io.Writer) (int64, error) { - var currentOffset uint64 = HeaderSize - if len(s.Keys) > 0 { - currentOffset = uint64(s.Keys[len(s.Keys)-1].ValueEnd) - } - var written int64 - - if _, err := os.Stat(s.ScratchSpacePath); err == nil { - // exists, we need to delete - // This could be the case if Weaviate shut down unexpectedly (i.e. crashed) - // while a compaction was running. We can safely discard the contents of - // the scratch space. - - if err := os.RemoveAll(s.ScratchSpacePath); err != nil { - return written, errors.Wrap(err, "clean up previous scratch space") - } - } else if os.IsNotExist(err) { - // does not exist yet, nothing to - will be created in the next step - } else { - return written, errors.Wrap(err, "check for scratch space directory") - } - - if err := os.Mkdir(s.ScratchSpacePath, 0o777); err != nil { - return written, errors.Wrap(err, "create scratch space") - } - - primaryFileName := filepath.Join(s.ScratchSpacePath, "primary") - primaryFD, err := os.Create(primaryFileName) - if err != nil { - return written, err - } - - primaryFDBuffered := bufio.NewWriter(primaryFD) - - n, err := s.buildAndMarshalPrimary(primaryFDBuffered, s.Keys) - if err != nil { - return written, err - } - - if err := primaryFDBuffered.Flush(); err != nil { - return written, err - } - - primaryFD.Seek(0, io.SeekStart) - - // pretend that primary index was already written, then also account for the - // additional offset pointers (one for each secondary index) - currentOffset = currentOffset + uint64(n) + - uint64(s.SecondaryIndexCount)*8 - - // secondaryIndicesBytes := bytes.NewBuffer(nil) - secondaryFileName := filepath.Join(s.ScratchSpacePath, "secondary") - secondaryFD, err := os.Create(secondaryFileName) - if err != nil { - return written, err - } - - secondaryFDBuffered := bufio.NewWriter(secondaryFD) - - if s.SecondaryIndexCount > 0 { - offsets := make([]uint64, s.SecondaryIndexCount) - for pos := range offsets { - n, err := s.buildAndMarshalSecondary(secondaryFDBuffered, pos, s.Keys) - if err != nil { - return written, err - } else { - written += int64(n) - } - - offsets[pos] = currentOffset - currentOffset = offsets[pos] + uint64(n) - } - - if err := binary.Write(w, binary.LittleEndian, &offsets); err != nil { - return written, err - } - - written += int64(len(offsets)) * 8 - } - - if err := secondaryFDBuffered.Flush(); err != nil { - return written, err - } - - secondaryFD.Seek(0, io.SeekStart) - - if n, err := io.Copy(w, primaryFD); err != nil { - return written, err - } else { - written += int64(n) - } - - if n, err := io.Copy(w, secondaryFD); err != nil { - return written, err - } else { - written += int64(n) - } - - if err := primaryFD.Close(); err != nil { - return written, err - } - - if err := secondaryFD.Close(); err != nil { - return written, err - } - - if err := os.RemoveAll(s.ScratchSpacePath); err != nil { - return written, err - } - - return written, nil -} - -// pos indicates the position of a secondary index, assumes unsorted keys and -// sorts them -func (s *Indexes) buildAndMarshalSecondary(w io.Writer, pos int, - keys []Key, -) (int64, error) { - keyNodes := make([]Node, len(keys)) - i := 0 - for _, key := range keys { - if pos >= len(key.SecondaryKeys) { - // a secondary key is not guaranteed to be present. For example, a delete - // operation could pe performed using only the primary key - continue - } - - keyNodes[i] = Node{ - Key: key.SecondaryKeys[pos], - Start: uint64(key.ValueStart), - End: uint64(key.ValueEnd), - } - i++ - } - - keyNodes = keyNodes[:i] - - sort.Slice(keyNodes, func(a, b int) bool { - return bytes.Compare(keyNodes[a].Key, keyNodes[b].Key) < 0 - }) - - index := NewBalanced(keyNodes) - n, err := index.MarshalBinaryInto(w) - if err != nil { - return 0, err - } - - return n, nil -} - -// assumes sorted keys and does NOT sort them again -func (s *Indexes) buildAndMarshalPrimary(w io.Writer, keys []Key) (int64, error) { - keyNodes := make([]Node, len(keys)) - for i, key := range keys { - keyNodes[i] = Node{ - Key: key.Key, - Start: uint64(key.ValueStart), - End: uint64(key.ValueEnd), - } - } - index := NewBalanced(keyNodes) - - n, err := index.MarshalBinaryInto(w) - if err != nil { - return -1, err - } - - return n, nil -} diff --git a/adapters/repos/db/lsmkv/segmentindex/key_index.go b/adapters/repos/db/lsmkv/segmentindex/key_index.go deleted file mode 100644 index 5da4f43a2ff3911db89e28af41328164a39a7f1b..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/segmentindex/key_index.go +++ /dev/null @@ -1,23 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package segmentindex - -// Key is a helper struct that can be used to build the key nodes for the -// segment index. It contains the primary key and an arbitrary number of -// secondary keys, as well as valueStart and valueEnd indicator. Those are used -// to find the correct payload for each key. -type Key struct { - Key []byte - SecondaryKeys [][]byte - ValueStart int - ValueEnd int -} diff --git a/adapters/repos/db/lsmkv/segmentindex/strategies.go b/adapters/repos/db/lsmkv/segmentindex/strategies.go deleted file mode 100644 index 251b77a565a3c760208556b9050721b4daa7b6b1..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/segmentindex/strategies.go +++ /dev/null @@ -1,21 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package segmentindex - -type Strategy uint16 - -const ( - StrategyReplace Strategy = iota - StrategySetCollection - StrategyMapCollection - StrategyRoaringSet -) diff --git a/adapters/repos/db/lsmkv/segmentindex/tree.go b/adapters/repos/db/lsmkv/segmentindex/tree.go deleted file mode 100644 index 34edfdc3ef121cd8d118602baafacd1c03a7f973..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/segmentindex/tree.go +++ /dev/null @@ -1,316 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package segmentindex - -import ( - "bytes" - "encoding/binary" - "fmt" - "io" - "math" - "sort" - - "github.com/pkg/errors" -) - -type Tree struct { - nodes []*Node -} - -type Node struct { - Key []byte - Start uint64 - End uint64 -} - -func NewTree(capacity int) Tree { - return Tree{ - nodes: make([]*Node, 0, capacity), - } -} - -func NewBalanced(nodes []Node) Tree { - t := Tree{nodes: make([]*Node, len(nodes))} - - if len(nodes) > 0 { - // sort the slice just once - sort.Slice(nodes, func(a, b int) bool { - return bytes.Compare(nodes[a].Key, nodes[b].Key) < 0 - }) - - t.buildBalanced(nodes, 0, 0, len(nodes)-1) - } - - return t -} - -func (t *Tree) buildBalanced(nodes []Node, targetPos, leftBound, rightBound int) { - t.grow(targetPos) - - if leftBound > rightBound { - return - } - - mid := (leftBound + rightBound) / 2 - t.nodes[targetPos] = &nodes[mid] - - t.buildBalanced(nodes, t.left(targetPos), leftBound, mid-1) - t.buildBalanced(nodes, t.right(targetPos), mid+1, rightBound) -} - -func (t *Tree) Insert(key []byte, start, end uint64) { - newNode := Node{ - Key: key, - Start: start, - End: end, - } - - if len(t.nodes) == 0 { - t.nodes = append(t.nodes, &newNode) - return - } - - t.insertAt(0, newNode) -} - -func (t *Tree) insertAt(nodeID int, newNode Node) { - if !t.exists(nodeID) { - // we are at the target and can insert now - t.grow(nodeID) - t.nodes[nodeID] = &newNode - return - } - - if bytes.Equal(newNode.Key, t.nodes[nodeID].Key) { - // this key already exists, which is an unexpected situation for an index - // key - panic(fmt.Sprintf("duplicate key %s", newNode.Key)) - } - - if bytes.Compare(newNode.Key, t.nodes[nodeID].Key) < 0 { - t.insertAt(t.left(nodeID), newNode) - } else { - t.insertAt(t.right(nodeID), newNode) - } -} - -func (t *Tree) Get(key []byte) ([]byte, uint64, uint64) { - if len(t.nodes) == 0 { - return nil, 0, 0 - } - - return t.getAt(0, key) -} - -func (t *Tree) getAt(nodeID int, key []byte) ([]byte, uint64, uint64) { - if !t.exists(nodeID) { - return nil, 0, 0 - } - - node := t.nodes[nodeID] - if bytes.Equal(node.Key, key) { - return node.Key, node.Start, node.End - } - - if bytes.Compare(key, node.Key) < 0 { - return t.getAt(t.left(nodeID), key) - } else { - return t.getAt(t.right(nodeID), key) - } -} - -func (t Tree) left(i int) int { - return 2*i + 1 -} - -func (t Tree) right(i int) int { - return 2*i + 2 -} - -func (t *Tree) exists(i int) bool { - if i >= len(t.nodes) { - return false - } - - return t.nodes[i] != nil -} - -// size calculates the exact size of this node on disk which is helpful to -// figure out the personal offset -func (n *Node) size() int { - if n == nil { - return 0 - } - size := 0 - size += 4 // uint32 for key length - size += len(n.Key) - size += 8 // uint64 startPos - size += 8 // uint64 endPos - size += 8 // int64 pointer left child - size += 8 // int64 pointer right child - return size -} - -func (t *Tree) grow(i int) { - if i < len(t.nodes) { - return - } - - oldSize := len(t.nodes) - newSize := oldSize - for newSize <= i { - newSize += oldSize - } - - newNodes := make([]*Node, newSize) - copy(newNodes, t.nodes) - for i := range t.nodes { - t.nodes[i] = nil - } - - t.nodes = newNodes -} - -func (t *Tree) MarshalBinary() ([]byte, error) { - offsets, size := t.calculateDiskOffsets() - - buf := bytes.NewBuffer(nil) - - for i, node := range t.nodes { - if node == nil { - continue - } - - var leftOffset int64 - var rightOffset int64 - - if t.exists(t.left(i)) { - leftOffset = int64(offsets[t.left(i)]) - } else { - leftOffset = -1 - } - - if t.exists(t.right(i)) { - rightOffset = int64(offsets[t.right(i)]) - } else { - rightOffset = -1 - } - - if len(node.Key) > math.MaxUint32 { - return nil, errors.Errorf("max key size is %d", math.MaxUint32) - } - - keyLen := uint32(len(node.Key)) - if err := binary.Write(buf, binary.LittleEndian, keyLen); err != nil { - return nil, err - } - if _, err := buf.Write(node.Key); err != nil { - return nil, err - } - if err := binary.Write(buf, binary.LittleEndian, node.Start); err != nil { - return nil, err - } - if err := binary.Write(buf, binary.LittleEndian, node.End); err != nil { - return nil, err - } - if err := binary.Write(buf, binary.LittleEndian, leftOffset); err != nil { - return nil, err - } - if err := binary.Write(buf, binary.LittleEndian, rightOffset); err != nil { - return nil, err - } - } - bytes := buf.Bytes() - if size != len(bytes) { - return nil, errors.Errorf("corrupt: wrote %d bytes with target %d", len(bytes), size) - } - - return bytes, nil -} - -func (t *Tree) MarshalBinaryInto(w io.Writer) (int64, error) { - offsets, size := t.calculateDiskOffsets() - - // create buf just once and reuse for each iteration, each iteration - // overwrites every single byte of the buffer, so no initializing or - // resetting after a round is required. - buf := make([]byte, 36) // 1x uint32 + 4x uint64 - - for i, node := range t.nodes { - if node == nil { - continue - } - - var leftOffset int64 - var rightOffset int64 - - if t.exists(t.left(i)) { - leftOffset = int64(offsets[t.left(i)]) - } else { - leftOffset = -1 - } - - if t.exists(t.right(i)) { - rightOffset = int64(offsets[t.right(i)]) - } else { - rightOffset = -1 - } - - if len(node.Key) > math.MaxUint32 { - return 0, errors.Errorf("max key size is %d", math.MaxUint32) - } - - keyLen := uint32(len(node.Key)) - binary.LittleEndian.PutUint32(buf[0:4], keyLen) - binary.LittleEndian.PutUint64(buf[4:12], node.Start) - binary.LittleEndian.PutUint64(buf[12:20], node.End) - binary.LittleEndian.PutUint64(buf[20:28], uint64(leftOffset)) - binary.LittleEndian.PutUint64(buf[28:36], uint64(rightOffset)) - - if _, err := w.Write(buf[:4]); err != nil { - return 0, err - } - if _, err := w.Write(node.Key); err != nil { - return 0, err - } - if _, err := w.Write(buf[4:36]); err != nil { - return 0, err - } - } - - return int64(size), nil -} - -// returns individual offsets and total size, nil nodes are skipped -func (t *Tree) calculateDiskOffsets() ([]int, int) { - current := 0 - out := make([]int, len(t.nodes)) - for i, node := range t.nodes { - out[i] = current - size := node.size() - current += size - } - - return out, current -} - -func (t *Tree) Height() int { - var highestElem int - for i := len(t.nodes) - 1; i >= 0; i-- { - if t.nodes[i] != nil { - highestElem = i - break - } - } - - return int(math.Ceil(math.Log2(float64(highestElem)))) -} diff --git a/adapters/repos/db/lsmkv/segmentindex/tree_test.go b/adapters/repos/db/lsmkv/segmentindex/tree_test.go deleted file mode 100644 index 3c23ea44302e60203434ce382bb9ca3f4ce37ea6..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/segmentindex/tree_test.go +++ /dev/null @@ -1,212 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package segmentindex - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/lsmkv" -) - -func TestTree(t *testing.T) { - type elem struct { - key []byte - start uint64 - end uint64 - } - - tree := NewTree(4) - - elements := []elem{ - { - key: []byte("foobar"), - start: 17, - end: 18, - }, - { - key: []byte("abc"), - start: 4, - end: 5, - }, - { - key: []byte("zzz"), - start: 34, - end: 35, - }, - { - key: []byte("aaa"), - start: 1, - end: 2, - }, - { - // makes the tree slightly imbalanced to the right, which in turn assures - // that we have a nil node in between - key: []byte("zzzz"), - start: 100, - end: 102, - }, - } - - t.Run("inserting", func(t *testing.T) { - for _, elem := range elements { - tree.Insert(elem.key, elem.start, elem.end) - } - }) - - t.Run("exact get", func(t *testing.T) { - key, start, end := tree.Get([]byte("foobar")) - assert.Equal(t, []byte("foobar"), key) - assert.Equal(t, uint64(17), start) - assert.Equal(t, uint64(18), end) - - key, start, end = tree.Get([]byte("abc")) - assert.Equal(t, []byte("abc"), key) - assert.Equal(t, uint64(4), start) - assert.Equal(t, uint64(5), end) - - key, start, end = tree.Get([]byte("zzz")) - assert.Equal(t, []byte("zzz"), key) - assert.Equal(t, uint64(34), start) - assert.Equal(t, uint64(35), end) - - key, start, end = tree.Get([]byte("aaa")) - assert.Equal(t, []byte("aaa"), key) - assert.Equal(t, uint64(1), start) - assert.Equal(t, uint64(2), end) - - key, start, end = tree.Get([]byte("zzzz")) - assert.Equal(t, []byte("zzzz"), key) - assert.Equal(t, uint64(100), start) - assert.Equal(t, uint64(102), end) - }) - - t.Run("marshalling and then reading the byte representation", func(t *testing.T) { - bytes, err := tree.MarshalBinary() - require.Nil(t, err) - - dTree := NewDiskTree(bytes) - - t.Run("get", func(t *testing.T) { - n, err := dTree.Get([]byte("foobar")) - assert.Nil(t, err) - assert.Equal(t, []byte("foobar"), n.Key) - assert.Equal(t, uint64(17), n.Start) - assert.Equal(t, uint64(18), n.End) - - n, err = dTree.Get([]byte("abc")) - assert.Nil(t, err) - assert.Equal(t, []byte("abc"), n.Key) - assert.Equal(t, uint64(4), n.Start) - assert.Equal(t, uint64(5), n.End) - - n, err = dTree.Get([]byte("zzz")) - assert.Nil(t, err) - assert.Equal(t, []byte("zzz"), n.Key) - assert.Equal(t, uint64(34), n.Start) - assert.Equal(t, uint64(35), n.End) - - n, err = dTree.Get([]byte("aaa")) - assert.Nil(t, err) - assert.Equal(t, []byte("aaa"), n.Key) - assert.Equal(t, uint64(1), n.Start) - assert.Equal(t, uint64(2), n.End) - - n, err = dTree.Get([]byte("zzzz")) - assert.Nil(t, err) - assert.Equal(t, []byte("zzzz"), n.Key) - assert.Equal(t, uint64(100), n.Start) - assert.Equal(t, uint64(102), n.End) - }) - - t.Run("seek", func(t *testing.T) { - n, err := dTree.Seek([]byte("foobar")) - assert.Nil(t, err) - assert.Equal(t, []byte("foobar"), n.Key) - assert.Equal(t, uint64(17), n.Start) - assert.Equal(t, uint64(18), n.End) - - n, err = dTree.Seek([]byte("f")) - assert.Nil(t, err) - assert.Equal(t, []byte("foobar"), n.Key) - assert.Equal(t, uint64(17), n.Start) - assert.Equal(t, uint64(18), n.End) - - n, err = dTree.Seek([]byte("abc")) - assert.Nil(t, err) - assert.Equal(t, []byte("abc"), n.Key) - assert.Equal(t, uint64(4), n.Start) - assert.Equal(t, uint64(5), n.End) - - n, err = dTree.Seek([]byte("ab")) - assert.Nil(t, err) - assert.Equal(t, []byte("abc"), n.Key) - assert.Equal(t, uint64(4), n.Start) - assert.Equal(t, uint64(5), n.End) - - n, err = dTree.Seek([]byte("zzz")) - assert.Nil(t, err) - assert.Equal(t, []byte("zzz"), n.Key) - assert.Equal(t, uint64(34), n.Start) - assert.Equal(t, uint64(35), n.End) - - n, err = dTree.Seek([]byte("z")) - assert.Nil(t, err) - assert.Equal(t, []byte("zzz"), n.Key) - assert.Equal(t, uint64(34), n.Start) - assert.Equal(t, uint64(35), n.End) - - n, err = dTree.Seek([]byte("aaa")) - assert.Nil(t, err) - assert.Equal(t, []byte("aaa"), n.Key) - assert.Equal(t, uint64(1), n.Start) - assert.Equal(t, uint64(2), n.End) - - n, err = dTree.Seek([]byte("a")) - assert.Nil(t, err) - assert.Equal(t, []byte("aaa"), n.Key) - assert.Equal(t, uint64(1), n.Start) - assert.Equal(t, uint64(2), n.End) - - n, err = dTree.Seek([]byte("zzzz")) - assert.Nil(t, err) - assert.Equal(t, []byte("zzzz"), n.Key) - assert.Equal(t, uint64(100), n.Start) - assert.Equal(t, uint64(102), n.End) - - n, err = dTree.Seek([]byte("zzza")) - assert.Nil(t, err) - assert.Equal(t, []byte("zzzz"), n.Key) - assert.Equal(t, uint64(100), n.Start) - assert.Equal(t, uint64(102), n.End) - - n, err = dTree.Seek([]byte("zzzzz")) - assert.Equal(t, lsmkv.NotFound, err) - }) - - t.Run("get all keys (for building bloom filters at segment init time)", func(t *testing.T) { - expected := [][]byte{ - []byte("aaa"), - []byte("abc"), - []byte("foobar"), - []byte("zzz"), - []byte("zzzz"), - } - - keys, err := dTree.AllKeys() - - require.Nil(t, err) - assert.ElementsMatch(t, expected, keys) - }) - }) -} diff --git a/adapters/repos/db/lsmkv/store.go b/adapters/repos/db/lsmkv/store.go deleted file mode 100644 index cfa43d4a6fed07fc4a97a6d4441724822dfec24a..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/store.go +++ /dev/null @@ -1,395 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "context" - "fmt" - "os" - "path" - "path/filepath" - "strings" - "sync" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/cyclemanager" - "github.com/weaviate/weaviate/entities/errorcompounder" - "github.com/weaviate/weaviate/entities/storagestate" -) - -// Store groups multiple buckets together, it "owns" one folder on the file -// system -type Store struct { - dir string - rootDir string - bucketsByName map[string]*Bucket - logger logrus.FieldLogger - metrics *Metrics - - cycleCallbacks *storeCycleCallbacks - - // Prevent concurrent manipulations to the bucketsByNameMap, most notably - // when initializing buckets in parallel - bucketAccessLock sync.RWMutex -} - -// New initializes a new [Store] based on the root dir. If state is present on -// disk, it is loaded, if the folder is empty a new store is initialized in -// there. -func New(dir, rootDir string, logger logrus.FieldLogger, metrics *Metrics, - shardCompactionCallbacks, shardFlushCallbacks cyclemanager.CycleCallbackGroup, -) (*Store, error) { - s := &Store{ - dir: dir, - rootDir: rootDir, - bucketsByName: map[string]*Bucket{}, - logger: logger, - metrics: metrics, - } - s.initCycleCallbacks(shardCompactionCallbacks, shardFlushCallbacks) - - return s, s.init() -} - -func (s *Store) Bucket(name string) *Bucket { - s.bucketAccessLock.RLock() - defer s.bucketAccessLock.RUnlock() - - return s.bucketsByName[name] -} - -func (s *Store) UpdateBucketsStatus(targetStatus storagestate.Status) { - // UpdateBucketsStatus is a write operation on the bucket itself, but from - // the perspective of our bucket access map this is a read-only operation, - // hence an RLock() - s.bucketAccessLock.RLock() - defer s.bucketAccessLock.RUnlock() - - for _, b := range s.bucketsByName { - if b == nil { - continue - } - - b.UpdateStatus(targetStatus) - } - - if targetStatus == storagestate.StatusReadOnly { - s.logger.WithField("action", "lsm_compaction"). - WithField("path", s.dir). - Warn("compaction halted due to shard READONLY status") - } -} - -func (s *Store) init() error { - if err := os.MkdirAll(s.dir, 0o700); err != nil { - return err - } - return nil -} - -func (s *Store) bucketDir(bucketName string) string { - return path.Join(s.dir, bucketName) -} - -// CreateOrLoadBucket registers a bucket with the given name. If state on disk -// exists for this bucket it is loaded, otherwise created. Pass [BucketOptions] -// to configure the strategy of a bucket. The strategy defaults to "replace". -// For example, to load or create a map-type bucket, do: -// -// ctx := context.Background() -// err := store.CreateOrLoadBucket(ctx, "my_bucket_name", WithStrategy(StrategyReplace)) -// if err != nil { /* handle error */ } -// -// // you can now access the bucket using store.Bucket() -// b := store.Bucket("my_bucket_name") -func (s *Store) CreateOrLoadBucket(ctx context.Context, bucketName string, - opts ...BucketOption, -) error { - if b := s.Bucket(bucketName); b != nil { - return nil - } - - b, err := NewBucket(ctx, s.bucketDir(bucketName), s.rootDir, s.logger, s.metrics, - s.cycleCallbacks.compactionCallbacks, s.cycleCallbacks.flushCallbacks, opts...) - if err != nil { - return err - } - - s.setBucket(bucketName, b) - return nil -} - -func (s *Store) setBucket(name string, b *Bucket) { - s.bucketAccessLock.Lock() - defer s.bucketAccessLock.Unlock() - - s.bucketsByName[name] = b -} - -func (s *Store) Shutdown(ctx context.Context) error { - s.bucketAccessLock.RLock() - defer s.bucketAccessLock.RUnlock() - - for name, bucket := range s.bucketsByName { - if err := bucket.Shutdown(ctx); err != nil { - return errors.Wrapf(err, "shutdown bucket %q", name) - } - } - - return nil -} - -func (s *Store) WriteWALs() error { - s.bucketAccessLock.RLock() - defer s.bucketAccessLock.RUnlock() - - for name, bucket := range s.bucketsByName { - if err := bucket.WriteWAL(); err != nil { - return errors.Wrapf(err, "bucket %q", name) - } - } - - return nil -} - -// bucketJobStatus is used to safely track the status of -// a job applied to each of a store's buckets when run -// in parallel -type bucketJobStatus struct { - sync.Mutex - buckets map[*Bucket]error -} - -func newBucketJobStatus() *bucketJobStatus { - return &bucketJobStatus{ - buckets: make(map[*Bucket]error), - } -} - -type jobFunc func(context.Context, *Bucket) (interface{}, error) - -type rollbackFunc func(context.Context, *Bucket) error - -func (s *Store) ListFiles(ctx context.Context, basePath string) ([]string, error) { - listFiles := func(ctx context.Context, b *Bucket) (interface{}, error) { - basePath, err := filepath.Rel(basePath, b.dir) - if err != nil { - return nil, fmt.Errorf("bucket relative path: %w", err) - } - return b.ListFiles(ctx, basePath) - } - - result, err := s.runJobOnBuckets(ctx, listFiles, nil) - if err != nil { - return nil, err - } - - var files []string - for _, res := range result { - files = append(files, res.([]string)...) - } - - return files, nil -} - -// runJobOnBuckets applies a jobFunc to each bucket in the store in parallel. -// The jobFunc allows for the job to return an arbitrary value. -// Additionally, a rollbackFunc may be provided which will be run on the target -// bucket in the event of an unsuccessful job. -func (s *Store) runJobOnBuckets(ctx context.Context, - jobFunc jobFunc, rollbackFunc rollbackFunc, -) ([]interface{}, error) { - var ( - status = newBucketJobStatus() - resultQueue = make(chan interface{}, len(s.bucketsByName)) - wg = sync.WaitGroup{} - ) - - for _, bucket := range s.bucketsByName { - wg.Add(1) - b := bucket - go func() { - status.Lock() - defer status.Unlock() - res, err := jobFunc(ctx, b) - resultQueue <- res - status.buckets[b] = err - wg.Done() - }() - } - - wg.Wait() - close(resultQueue) - - var errs errorcompounder.ErrorCompounder - for _, err := range status.buckets { - errs.Add(err) - } - - if errs.Len() != 0 { - // if any of the bucket jobs failed, and a - // rollbackFunc has been provided, attempt - // to roll back. if this fails, the err is - // added to the compounder - for b, jobErr := range status.buckets { - if jobErr != nil && rollbackFunc != nil { - if rollbackErr := rollbackFunc(ctx, b); rollbackErr != nil { - errs.AddWrap(rollbackErr, "bucket job rollback") - } - } - } - - return nil, errs.ToError() - } - - var finalResult []interface{} - for res := range resultQueue { - finalResult = append(finalResult, res) - } - - return finalResult, nil -} - -func (s *Store) GetBucketsByName() map[string]*Bucket { - s.bucketAccessLock.RLock() - defer s.bucketAccessLock.RUnlock() - - newMap := map[string]*Bucket{} - for name, bucket := range s.bucketsByName { - newMap[name] = bucket - } - - return newMap -} - -// Creates bucket, first removing any files if already exist -// Bucket can not be registered in bucketsByName before removal -func (s *Store) CreateBucket(ctx context.Context, bucketName string, - opts ...BucketOption, -) error { - if b := s.Bucket(bucketName); b != nil { - return fmt.Errorf("bucket %s exists and is already in use", bucketName) - } - - bucketDir := s.bucketDir(bucketName) - if err := os.RemoveAll(bucketDir); err != nil { - return errors.Wrapf(err, "failed removing bucket %s files", bucketName) - } - - b, err := NewBucket(ctx, bucketDir, s.rootDir, s.logger, s.metrics, - s.cycleCallbacks.compactionCallbacks, s.cycleCallbacks.flushCallbacks, opts...) - if err != nil { - return err - } - - s.setBucket(bucketName, b) - return nil -} - -// Replaces 1st bucket with 2nd one. Both buckets have to registered in bucketsByName. -// 2nd bucket swaps the 1st one in bucketsByName using 1st one's name, 2nd one's name is deleted. -// Dir path of 2nd bucket is changed to dir of 1st bucket as well as all other related paths of -// bucket resources (segment group, memtables, commit log). -// Dir path of 1st bucket is temporarily suffixed with "___del", later on bucket is shutdown and -// its files deleted. -// 2nd bucket becomes 1st bucket -func (s *Store) ReplaceBuckets(ctx context.Context, bucketName, replacementBucketName string) error { - s.bucketAccessLock.Lock() - defer s.bucketAccessLock.Unlock() - - bucket := s.bucketsByName[bucketName] - if bucket == nil { - return fmt.Errorf("bucket '%s' not found", bucketName) - } - replacementBucket := s.bucketsByName[replacementBucketName] - if replacementBucket == nil { - return fmt.Errorf("replacement bucket '%s' not found", replacementBucketName) - } - s.bucketsByName[bucketName] = replacementBucket - delete(s.bucketsByName, replacementBucketName) - - currBucketDir := bucket.dir - newBucketDir := bucket.dir + "___del" - currReplacementBucketDir := replacementBucket.dir - newReplacementBucketDir := currBucketDir - - if err := os.Rename(currBucketDir, newBucketDir); err != nil { - return errors.Wrapf(err, "failed moving orig bucket dir '%s'", currBucketDir) - } - if err := os.Rename(currReplacementBucketDir, newReplacementBucketDir); err != nil { - return errors.Wrapf(err, "failed moving replacement bucket dir '%s'", currReplacementBucketDir) - } - - s.updateBucketDir(bucket, currBucketDir, newBucketDir) - s.updateBucketDir(replacementBucket, currReplacementBucketDir, newReplacementBucketDir) - - if err := bucket.Shutdown(ctx); err != nil { - return errors.Wrapf(err, "failed shutting down bucket old '%s'", bucketName) - } - if err := os.RemoveAll(newBucketDir); err != nil { - return errors.Wrapf(err, "failed removing dir '%s'", newBucketDir) - } - - return nil -} - -func (s *Store) RenameBucket(ctx context.Context, bucketName, newBucketName string) error { - s.bucketAccessLock.Lock() - defer s.bucketAccessLock.Unlock() - - currBucket := s.bucketsByName[bucketName] - if currBucket == nil { - return fmt.Errorf("bucket '%s' not found", bucketName) - } - newBucket := s.bucketsByName[newBucketName] - if newBucket != nil { - return fmt.Errorf("bucket '%s' already exists", newBucketName) - } - s.bucketsByName[newBucketName] = currBucket - delete(s.bucketsByName, bucketName) - - currBucketDir := currBucket.dir - newBucketDir := s.bucketDir(newBucketName) - - if err := os.Rename(currBucketDir, newBucketDir); err != nil { - return errors.Wrapf(err, "failed renaming bucket dir '%s' to '%s'", currBucketDir, newBucketDir) - } - - s.updateBucketDir(currBucket, currBucketDir, newBucketDir) - return nil -} - -func (s *Store) updateBucketDir(bucket *Bucket, bucketDir, newBucketDir string) { - updatePath := func(src string) string { - return strings.Replace(src, bucketDir, newBucketDir, 1) - } - - bucket.flushLock.Lock() - bucket.dir = newBucketDir - if bucket.active != nil { - bucket.active.path = updatePath(bucket.active.path) - bucket.active.commitlog.path = updatePath(bucket.active.commitlog.path) - } - if bucket.flushing != nil { - bucket.flushing.path = updatePath(bucket.flushing.path) - bucket.flushing.commitlog.path = updatePath(bucket.flushing.commitlog.path) - } - bucket.flushLock.Unlock() - - bucket.disk.maintenanceLock.Lock() - bucket.disk.dir = newBucketDir - for _, segment := range bucket.disk.segments { - segment.path = updatePath(segment.path) - } - bucket.disk.maintenanceLock.Unlock() -} diff --git a/adapters/repos/db/lsmkv/store_backup.go b/adapters/repos/db/lsmkv/store_backup.go deleted file mode 100644 index 456e8402d066ac5c517197855a9d2d1b32879564..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/store_backup.go +++ /dev/null @@ -1,81 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "context" - - "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus" - "github.com/weaviate/weaviate/usecases/monitoring" -) - -// PauseCompaction waits for all ongoing compactions to finish, -// then makes sure that no new compaction can be started. -// -// This is a preparatory stage for creating backups. -// -// A timeout should be specified for the input context as some -// compactions are long-running, in which case it may be better -// to fail the backup attempt and retry later, than to block -// indefinitely. -func (s *Store) PauseCompaction(ctx context.Context) error { - if err := s.cycleCallbacks.compactionCallbacksCtrl.Deactivate(ctx); err != nil { - return errors.Wrap(err, "long-running compaction in progress") - } - - // TODO common_cycle_manager maybe not necessary, or to be replaced with store pause stats - for _, b := range s.bucketsByName { - if metric, err := monitoring.GetMetrics().BucketPauseDurations.GetMetricWithLabelValues(b.dir); err == nil { - b.pauseTimer = prometheus.NewTimer(metric) - } - } - - return nil -} - -// ResumeCompaction starts the compaction cycle again. -// It errors if compactions were not paused -func (s *Store) ResumeCompaction(ctx context.Context) error { - s.cycleCallbacks.compactionCallbacksCtrl.Activate() - - // TODO common_cycle_manager maybe not necessary, or to be replaced with store pause stats - for _, b := range s.bucketsByName { - if b.pauseTimer != nil { - b.pauseTimer.ObserveDuration() - } - } - - return nil -} - -// FlushMemtable flushes any active memtable and returns only once the memtable -// has been fully flushed and a stable state on disk has been reached. -// -// This is a preparatory stage for creating backups. -// -// A timeout should be specified for the input context as some -// flushes are long-running, in which case it may be better -// to fail the backup attempt and retry later, than to block -// indefinitely. -func (s *Store) FlushMemtables(ctx context.Context) error { - if err := s.cycleCallbacks.flushCallbacksCtrl.Deactivate(ctx); err != nil { - return errors.Wrap(err, "long-running memtable flush in progress") - } - defer s.cycleCallbacks.flushCallbacksCtrl.Activate() - - flushMemtable := func(ctx context.Context, b *Bucket) (interface{}, error) { - return nil, b.FlushMemtable() - } - _, err := s.runJobOnBuckets(ctx, flushMemtable, nil) - return err -} diff --git a/adapters/repos/db/lsmkv/store_backup_test.go b/adapters/repos/db/lsmkv/store_backup_test.go deleted file mode 100644 index 7b655cbe8aaa9c8aecd4599820334c03be747a01..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/store_backup_test.go +++ /dev/null @@ -1,300 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/cyclemanager" - "github.com/weaviate/weaviate/entities/errorcompounder" - "github.com/weaviate/weaviate/entities/storagestate" -) - -func TestStoreBackup(t *testing.T) { - ctx := context.Background() - tests := bucketTests{ - { - name: "pauseCompaction", - f: pauseCompaction, - }, - { - name: "resumeCompaction", - f: resumeCompaction, - }, - { - name: "flushMemtable", - f: flushMemtable, - }, - } - tests.run(ctx, t) -} - -func pauseCompaction(ctx context.Context, t *testing.T, opts []BucketOption) { - logger, _ := test.NewNullLogger() - - t.Run("assert that context timeout works for long compactions", func(t *testing.T) { - for _, buckets := range [][]string{ - {"test_bucket"}, - {"test_bucket1", "test_bucket2"}, - {"test_bucket1", "test_bucket2", "test_bucket3", "test_bucket4", "test_bucket5"}, - } { - t.Run(fmt.Sprintf("with %d buckets", len(buckets)), func(t *testing.T) { - dirName := t.TempDir() - - shardCompactionCallbacks := cyclemanager.NewCallbackGroup("classCompaction", logger, 1) - shardFlushCallbacks := cyclemanager.NewCallbackGroupNoop() - - store, err := New(dirName, dirName, logger, nil, shardCompactionCallbacks, shardFlushCallbacks) - require.Nil(t, err) - - for _, bucket := range buckets { - err = store.CreateOrLoadBucket(ctx, bucket, opts...) - require.Nil(t, err) - } - - expiredCtx, cancel := context.WithDeadline(ctx, time.Now()) - defer cancel() - - err = store.PauseCompaction(expiredCtx) - require.NotNil(t, err) - assert.Equal(t, "long-running compaction in progress:"+ - " deactivating callback 'store/compaction/.' of 'classCompaction' failed:"+ - " context deadline exceeded", err.Error()) - - err = store.Shutdown(ctx) - require.Nil(t, err) - }) - } - }) - - t.Run("assert compaction is successfully paused", func(t *testing.T) { - for _, buckets := range [][]string{ - {"test_bucket"}, - {"test_bucket1", "test_bucket2"}, - {"test_bucket1", "test_bucket2", "test_bucket3", "test_bucket4", "test_bucket5"}, - } { - t.Run(fmt.Sprintf("with %d buckets", len(buckets)), func(t *testing.T) { - dirName := t.TempDir() - - shardCompactionCallbacks := cyclemanager.NewCallbackGroup("classCompaction", logger, 1) - shardFlushCallbacks := cyclemanager.NewCallbackGroupNoop() - - store, err := New(dirName, dirName, logger, nil, shardCompactionCallbacks, shardFlushCallbacks) - require.Nil(t, err) - - for _, bucket := range buckets { - err = store.CreateOrLoadBucket(ctx, bucket, opts...) - require.Nil(t, err) - - t.Run("insert contents into bucket", func(t *testing.T) { - bucket := store.Bucket(bucket) - for i := 0; i < 10; i++ { - err := bucket.Put([]byte(fmt.Sprint(i)), []byte(fmt.Sprint(i))) - require.Nil(t, err) - } - }) - } - - expirableCtx, cancel := context.WithTimeout(ctx, 3*time.Second) - defer cancel() - - err = store.PauseCompaction(expirableCtx) - assert.Nil(t, err) - - err = store.Shutdown(context.Background()) - require.Nil(t, err) - }) - } - }) -} - -func resumeCompaction(ctx context.Context, t *testing.T, opts []BucketOption) { - logger, _ := test.NewNullLogger() - - t.Run("assert compaction restarts after pausing", func(t *testing.T) { - for _, buckets := range [][]string{ - {"test_bucket"}, - {"test_bucket1", "test_bucket2"}, - {"test_bucket1", "test_bucket2", "test_bucket3", "test_bucket4", "test_bucket5"}, - } { - t.Run(fmt.Sprintf("with %d buckets", len(buckets)), func(t *testing.T) { - dirName := t.TempDir() - - shardCompactionCallbacks := cyclemanager.NewCallbackGroup("classCompaction", logger, 1) - shardFlushCallbacks := cyclemanager.NewCallbackGroupNoop() - - store, err := New(dirName, dirName, logger, nil, shardCompactionCallbacks, shardFlushCallbacks) - require.Nil(t, err) - - for _, bucket := range buckets { - err = store.CreateOrLoadBucket(ctx, bucket, opts...) - require.Nil(t, err) - - t.Run("insert contents into bucket", func(t *testing.T) { - bucket := store.Bucket(bucket) - for i := 0; i < 10; i++ { - err := bucket.Put([]byte(fmt.Sprint(i)), []byte(fmt.Sprint(i))) - require.Nil(t, err) - } - }) - } - - expirableCtx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - - err = store.PauseCompaction(expirableCtx) - require.Nil(t, err) - - err = store.ResumeCompaction(expirableCtx) - require.Nil(t, err) - - assert.True(t, store.cycleCallbacks.compactionCallbacksCtrl.IsActive()) - - err = store.Shutdown(ctx) - require.Nil(t, err) - }) - } - }) -} - -func flushMemtable(ctx context.Context, t *testing.T, opts []BucketOption) { - logger, _ := test.NewNullLogger() - - t.Run("assert that context timeout works for long flushes", func(t *testing.T) { - for _, buckets := range [][]string{ - {"test_bucket"}, - {"test_bucket1", "test_bucket2"}, - {"test_bucket1", "test_bucket2", "test_bucket3", "test_bucket4", "test_bucket5"}, - } { - t.Run(fmt.Sprintf("with %d buckets", len(buckets)), func(t *testing.T) { - dirName := t.TempDir() - - shardCompactionCallbacks := cyclemanager.NewCallbackGroupNoop() - shardFlushCallbacks := cyclemanager.NewCallbackGroup("classFlush", logger, 1) - - store, err := New(dirName, dirName, logger, nil, shardCompactionCallbacks, shardFlushCallbacks) - require.Nil(t, err) - - for _, bucket := range buckets { - err = store.CreateOrLoadBucket(ctx, bucket, opts...) - require.Nil(t, err) - } - - expiredCtx, cancel := context.WithDeadline(ctx, time.Now()) - defer cancel() - - err = store.FlushMemtables(expiredCtx) - require.NotNil(t, err) - assert.Equal(t, "long-running memtable flush in progress:"+ - " deactivating callback 'store/flush/.' of 'classFlush' failed:"+ - " context deadline exceeded", err.Error()) - - err = store.Shutdown(ctx) - require.Nil(t, err) - }) - } - }) - - t.Run("assert that flushes run successfully", func(t *testing.T) { - for _, buckets := range [][]string{ - {"test_bucket"}, - {"test_bucket1", "test_bucket2"}, - {"test_bucket1", "test_bucket2", "test_bucket3", "test_bucket4", "test_bucket5"}, - } { - t.Run(fmt.Sprintf("with %d buckets", len(buckets)), func(t *testing.T) { - dirName := t.TempDir() - - shardCompactionCallbacks := cyclemanager.NewCallbackGroupNoop() - shardFlushCallbacks := cyclemanager.NewCallbackGroup("classFlush", logger, 1) - - store, err := New(dirName, dirName, logger, nil, shardCompactionCallbacks, shardFlushCallbacks) - require.Nil(t, err) - - err = store.CreateOrLoadBucket(ctx, "test_bucket", opts...) - require.Nil(t, err) - - for _, bucket := range buckets { - err = store.CreateOrLoadBucket(ctx, bucket, opts...) - require.Nil(t, err) - - t.Run("insert contents into bucket", func(t *testing.T) { - bucket := store.Bucket(bucket) - for i := 0; i < 10; i++ { - err := bucket.Put([]byte(fmt.Sprint(i)), []byte(fmt.Sprint(i))) - require.Nil(t, err) - } - }) - } - - expirableCtx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - - err = store.FlushMemtables(expirableCtx) - assert.Nil(t, err) - - err = store.Shutdown(ctx) - require.Nil(t, err) - }) - } - }) - - t.Run("assert that readonly bucket fails to flush", func(t *testing.T) { - singleErr := errors.Wrap(storagestate.ErrStatusReadOnly, "flush memtable") - expectedErr := func(bucketsCount int) error { - ec := &errorcompounder.ErrorCompounder{} - for i := 0; i < bucketsCount; i++ { - ec.Add(singleErr) - } - return ec.ToError() - } - - for _, buckets := range [][]string{ - {"test_bucket"}, - {"test_bucket1", "test_bucket2"}, - {"test_bucket1", "test_bucket2", "test_bucket3", "test_bucket4", "test_bucket5"}, - } { - t.Run(fmt.Sprintf("with %d buckets", len(buckets)), func(t *testing.T) { - dirName := t.TempDir() - - shardCompactionCallbacks := cyclemanager.NewCallbackGroupNoop() - shardFlushCallbacks := cyclemanager.NewCallbackGroup("classFlush", logger, 1) - - store, err := New(dirName, dirName, logger, nil, shardCompactionCallbacks, shardFlushCallbacks) - require.Nil(t, err) - - for _, bucket := range buckets { - err = store.CreateOrLoadBucket(ctx, bucket, opts...) - require.Nil(t, err) - } - - store.UpdateBucketsStatus(storagestate.StatusReadOnly) - - expirableCtx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - - err = store.FlushMemtables(expirableCtx) - require.NotNil(t, err) - assert.EqualError(t, expectedErr(len(buckets)), err.Error()) - - err = store.Shutdown(ctx) - require.Nil(t, err) - }) - } - }) -} diff --git a/adapters/repos/db/lsmkv/store_cyclecallbacks.go b/adapters/repos/db/lsmkv/store_cyclecallbacks.go deleted file mode 100644 index baac32572ecf1d259cf73983a24658cb73b7fc9b..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/store_cyclecallbacks.go +++ /dev/null @@ -1,57 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "path/filepath" - "strings" - - "github.com/weaviate/weaviate/entities/cyclemanager" -) - -type storeCycleCallbacks struct { - compactionCallbacks cyclemanager.CycleCallbackGroup - compactionCallbacksCtrl cyclemanager.CycleCallbackCtrl - - flushCallbacks cyclemanager.CycleCallbackGroup - flushCallbacksCtrl cyclemanager.CycleCallbackCtrl -} - -func (s *Store) initCycleCallbacks(shardCompactionCallbacks, shardFlushCallbacks cyclemanager.CycleCallbackGroup) { - id := func(elems ...string) string { - path, err := filepath.Rel(s.dir, s.rootDir) - if err != nil { - path = s.dir - } - elems = append([]string{"store"}, elems...) - elems = append(elems, path) - return strings.Join(elems, "/") - } - - compactionId := id("compaction") - compactionCallbacks := cyclemanager.NewCallbackGroup(compactionId, s.logger, 1) - compactionCallbacksCtrl := shardCompactionCallbacks.Register( - compactionId, compactionCallbacks.CycleCallback) - - flushId := id("flush") - flushCallbacks := cyclemanager.NewCallbackGroup(flushId, s.logger, 1) - flushCallbacksCtrl := shardFlushCallbacks.Register( - flushId, flushCallbacks.CycleCallback) - - s.cycleCallbacks = &storeCycleCallbacks{ - compactionCallbacks: compactionCallbacks, - compactionCallbacksCtrl: compactionCallbacksCtrl, - - flushCallbacks: flushCallbacks, - flushCallbacksCtrl: flushCallbacksCtrl, - } -} diff --git a/adapters/repos/db/lsmkv/store_integration_test.go b/adapters/repos/db/lsmkv/store_integration_test.go deleted file mode 100644 index 6d0e446d40cab5ff3b386a621892636b5e316114..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/store_integration_test.go +++ /dev/null @@ -1,99 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package lsmkv - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/cyclemanager" -) - -func TestStoreLifecycle(t *testing.T) { - ctx := testCtx() - tests := bucketIntegrationTests{ - { - name: "testStoreLifecycle", - f: testStoreLifecycle, - opts: []BucketOption{ - WithStrategy(StrategyReplace), - }, - }, - } - tests.run(ctx, t) -} - -func testStoreLifecycle(ctx context.Context, t *testing.T, opts []BucketOption) { - dirName := t.TempDir() - logger := nullLogger() - - t.Run("cycle 1", func(t *testing.T) { - store, err := New(dirName, dirName, logger, nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop()) - require.Nil(t, err) - - err = store.CreateOrLoadBucket(testCtx(), "bucket1", opts...) - require.Nil(t, err) - - b1 := store.Bucket("bucket1") - require.NotNil(t, b1) - - err = b1.Put([]byte("name"), []byte("Jane Doe")) - require.Nil(t, err) - - err = store.CreateOrLoadBucket(testCtx(), "bucket2", opts...) - require.Nil(t, err) - - b2 := store.Bucket("bucket2") - require.NotNil(t, b2) - - err = b2.Put([]byte("foo"), []byte("bar")) - require.Nil(t, err) - - err = store.Shutdown(context.Background()) - require.Nil(t, err) - }) - - t.Run("cycle 2", func(t *testing.T) { - store, err := New(dirName, dirName, logger, nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop()) - require.Nil(t, err) - - err = store.CreateOrLoadBucket(testCtx(), "bucket1", opts...) - require.Nil(t, err) - - b1 := store.Bucket("bucket1") - require.NotNil(t, b1) - - err = store.CreateOrLoadBucket(testCtx(), "bucket2", opts...) - require.Nil(t, err) - - b2 := store.Bucket("bucket2") - require.NotNil(t, b2) - - res, err := b1.Get([]byte("name")) - require.Nil(t, err) - assert.Equal(t, []byte("Jane Doe"), res) - - res, err = b2.Get([]byte("foo")) - require.Nil(t, err) - assert.Equal(t, []byte("bar"), res) - - err = store.Shutdown(context.Background()) - require.Nil(t, err) - }) -} diff --git a/adapters/repos/db/lsmkv/strategies.go b/adapters/repos/db/lsmkv/strategies.go deleted file mode 100644 index d0a0a2539e1329a985b36801b56bc8b9c0601d10..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/strategies.go +++ /dev/null @@ -1,60 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "fmt" - - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv/segmentindex" -) - -const ( - // StrategyReplace allows for idem-potent PUT where the latest takes presence - StrategyReplace = "replace" - StrategySetCollection = "setcollection" - StrategyMapCollection = "mapcollection" - StrategyRoaringSet = "roaringset" -) - -func SegmentStrategyFromString(in string) segmentindex.Strategy { - switch in { - case StrategyReplace: - return segmentindex.StrategyReplace - case StrategySetCollection: - return segmentindex.StrategySetCollection - case StrategyMapCollection: - return segmentindex.StrategyMapCollection - case StrategyRoaringSet: - return segmentindex.StrategyRoaringSet - default: - panic("unsupported strategy") - } -} - -func IsExpectedStrategy(strategy string, expectedStrategies ...string) bool { - if len(expectedStrategies) == 0 { - expectedStrategies = []string{StrategyReplace, StrategySetCollection, StrategyMapCollection, StrategyRoaringSet} - } - - for _, s := range expectedStrategies { - if s == strategy { - return true - } - } - return false -} - -func CheckExpectedStrategy(strategy string, expectedStrategies ...string) { - if !IsExpectedStrategy(strategy, expectedStrategies...) { - panic(fmt.Sprintf("one of strategies %v expected, strategy '%s' found", expectedStrategies, strategy)) - } -} diff --git a/adapters/repos/db/lsmkv/strategies_map.go b/adapters/repos/db/lsmkv/strategies_map.go deleted file mode 100644 index 648b2b12c4eeb122625912477678e5835632eab5..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/strategies_map.go +++ /dev/null @@ -1,404 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "bytes" - "encoding/binary" - "math" - - "github.com/pkg/errors" -) - -type mapDecoder struct{} - -func newMapDecoder() *mapDecoder { - return &mapDecoder{} -} - -func (m *mapDecoder) Do(in []value, acceptDuplicates bool) ([]MapPair, error) { - // if acceptDuplicates { - // return m.doSimplified(in) - // } - - seenKeys := map[string]uint{} - kvs := make([]MapPair, len(in)) - - // unmarshalling := time.Duration(0) - - // beforeFirst := time.Now() - for i, pair := range in { - kv := MapPair{} - // beforeUnmarshal := time.Now() - err := kv.FromBytes(pair.value, pair.tombstone) - if err != nil { - return nil, err - } - // unmarshalling += time.Since(beforeUnmarshal) - kv.Tombstone = pair.tombstone - kvs[i] = kv - count := seenKeys[string(kv.Key)] - seenKeys[string(kv.Key)] = count + 1 - } - // fmt.Printf("first decoder loop took %s\n", time.Since(beforeFirst)) - // fmt.Printf("unmarshalling in first loop took %s\n", unmarshalling) - - // beforeSecond := time.Now() - out := make([]MapPair, len(in)) - i := 0 - for _, pair := range kvs { - count := seenKeys[string(pair.Key)] - if count != 1 { - seenKeys[string(pair.Key)] = count - 1 - continue - - } - - if pair.Tombstone { - continue - } - - out[i] = pair - i++ - } - // fmt.Printf("second decoder loop took %s\n", time.Since(beforeSecond)) - - return out[:i], nil -} - -type tombstone struct { - pos int - key []byte -} - -func (m *mapDecoder) doSimplified(in []value) ([]MapPair, error) { - out := make([]MapPair, len(in)) - - var tombstones []tombstone - - i := 0 - for _, raw := range in { - if raw.tombstone { - mp := MapPair{} - mp.FromBytes(raw.value, true) - tombstones = append(tombstones, tombstone{pos: i, key: mp.Key}) - continue - } - - out[i].FromBytes(raw.value, raw.tombstone) - i++ - } - - out = out[:i] - - if len(tombstones) > 0 { - out = m.removeTombstonesFromResults(out, tombstones) - } - - return out, nil -} - -func (m *mapDecoder) removeTombstonesFromResults(candidates []MapPair, - tombstones []tombstone, -) []MapPair { - after := make([]MapPair, len(candidates)) - newPos := 0 - for origPos, candidate := range candidates { - - skip := false - for _, tombstone := range tombstones { - if tombstone.pos > origPos && bytes.Equal(tombstone.key, candidate.Key) { - skip = true - } - } - - if skip { - continue - } - - after[newPos] = candidate - newPos++ - } - - return after[:newPos] -} - -// DoPartial keeps "unused" tombstones -func (m *mapDecoder) DoPartial(in []value) ([]MapPair, error) { - seenKeys := map[string]uint{} - kvs := make([]MapPair, len(in)) - - for i, pair := range in { - kv := MapPair{} - err := kv.FromBytes(pair.value, pair.tombstone) - if err != nil { - return nil, err - } - kv.Tombstone = pair.tombstone - kvs[i] = kv - count := seenKeys[string(kv.Key)] - seenKeys[string(kv.Key)] = count + 1 - } - - out := make([]MapPair, len(in)) - i := 0 - for _, pair := range kvs { - count := seenKeys[string(pair.Key)] - if count != 1 { - seenKeys[string(pair.Key)] = count - 1 - continue - - } - - out[i] = pair - i++ - } - - return out[:i], nil -} - -type MapPair struct { - Key []byte - Value []byte - Tombstone bool -} - -// Size() returns the exact size in bytes that will be used when Bytes() is -// called -func (kv MapPair) Size() int { - // each field uses a uint16 (2 bytes) length indicator - return 2 + len(kv.Key) + 2 + len(kv.Value) -} - -func (kv MapPair) EncodeBytes(buf []byte) error { - if len(buf) != kv.Size() { - return errors.Errorf("buffer has size %d, but MapPair has size %d", - len(buf), kv.Size()) - } - - // make sure the 2 byte length indicators will never overflow: - if len(kv.Key) >= math.MaxUint16 { - return errors.Errorf("mapCollection key must be smaller than %d", - math.MaxUint16) - } - keyLen := uint16(len(kv.Key)) - - if len(kv.Value) >= math.MaxUint16 { - return errors.Errorf("mapCollection value must be smaller than %d", - math.MaxUint16) - } - valueLen := uint16(len(kv.Value)) - - offset := 0 - binary.LittleEndian.PutUint16(buf[offset:offset+2], keyLen) - offset += 2 - copy(buf[offset:], kv.Key) - offset += len(kv.Key) - - binary.LittleEndian.PutUint16(buf[offset:offset+2], valueLen) - offset += 2 - copy(buf[offset:], kv.Value) - - return nil -} - -func (kv MapPair) Bytes() ([]byte, error) { - // make sure the 2 byte length indicators will never overflow: - if len(kv.Key) >= math.MaxUint16 { - return nil, errors.Errorf("mapCollection key must be smaller than %d", - math.MaxUint16) - } - keyLen := uint16(len(kv.Key)) - - if len(kv.Value) >= math.MaxUint16 { - return nil, errors.Errorf("mapCollection value must be smaller than %d", - math.MaxUint16) - } - valueLen := uint16(len(kv.Value)) - - out := bytes.NewBuffer(nil) - - lenBuf := make([]byte, 2) // can be reused for both key and value len - binary.LittleEndian.PutUint16(lenBuf, keyLen) - if _, err := out.Write(lenBuf); err != nil { - return nil, errors.Wrap(err, "write map key length indicator") - } - - if _, err := out.Write(kv.Key); err != nil { - return nil, errors.Wrap(err, "write map key") - } - - binary.LittleEndian.PutUint16(lenBuf, valueLen) - if _, err := out.Write(lenBuf); err != nil { - return nil, errors.Wrap(err, "write map value length indicator") - } - - if _, err := out.Write(kv.Value); err != nil { - return nil, errors.Wrap(err, "write map value") - } - - return out.Bytes(), nil -} - -func (kv *MapPair) FromBytes(in []byte, keyOnly bool) error { - var read uint16 - - // NOTE: A previous implementation was using copy statements in here to avoid - // sharing the memory. The general idea of that is good (protect against the - // mmaped memory being removed from a completed compaction), however this is - // the wrong place. By the time we are in this method, we can no longer - // control the memory safety of the "in" argument. Thus, such a copy must - // happen at a much earlier scope when a lock is held that protects against - // removing the segment. Such an implementation can now be found in - // segment_collection_strategy.go as part of the *segment.getCollection - // method. As a result all memory used here can now be considered read-only - // and is safe to be used indefinitely. - - keyLen := binary.LittleEndian.Uint16(in[:2]) - read += 2 // uint16 -> 2 bytes - - kv.Key = in[read : read+keyLen] - read += keyLen - - if keyOnly { - return nil - } - - valueLen := binary.LittleEndian.Uint16(in[read : read+2]) - read += 2 - - kv.Value = in[read : read+valueLen] - read += valueLen - - if read != uint16(len(in)) { - return errors.Errorf("inconsistent map pair: read %d out of %d bytes", - read, len(in)) - } - - return nil -} - -func (kv *MapPair) FromBytesReusable(in []byte, keyOnly bool) error { - var read uint16 - - keyLen := binary.LittleEndian.Uint16(in[:2]) - read += 2 // uint16 -> 2 bytes - - if int(keyLen) > cap(kv.Key) { - kv.Key = make([]byte, keyLen) - } else { - kv.Key = kv.Key[:keyLen] - } - copy(kv.Key, in[read:read+keyLen]) - read += keyLen - - if keyOnly { - return nil - } - - valueLen := binary.LittleEndian.Uint16(in[read : read+2]) - read += 2 - - if int(valueLen) > cap(kv.Value) { - kv.Value = make([]byte, valueLen) - } else { - kv.Value = kv.Value[:valueLen] - } - copy(kv.Value, in[read:read+valueLen]) - read += valueLen - - if read != uint16(len(in)) { - return errors.Errorf("inconsistent map pair: read %d out of %d bytes", - read, len(in)) - } - - return nil -} - -type mapEncoder struct { - pairBuf []value -} - -func newMapEncoder() *mapEncoder { - return &mapEncoder{} -} - -func (m *mapEncoder) Do(kv MapPair) ([]value, error) { - v, err := kv.Bytes() - if err != nil { - return nil, err - } - - out := make([]value, 1) - out[0] = value{ - tombstone: kv.Tombstone, - value: v, - } - - return out, nil -} - -func (m *mapEncoder) DoMulti(kvs []MapPair) ([]value, error) { - out := make([]value, len(kvs)) - - for i, kv := range kvs { - v := make([]byte, kv.Size()) - err := kv.EncodeBytes(v) - if err != nil { - return nil, err - } - - out[i] = value{ - tombstone: kv.Tombstone, - value: v, - } - } - - return out, nil -} - -// DoMultiReusable reuses a MapPair buffer that it exposes to the caller on -// this request. Warning: The caller must make sure that they no longer access -// the return value once they call this method a second time, otherwise they -// risk overwriting a previous result. The intended usage for example in a loop -// where each loop copies the results, for example using a bufio.Writer. -func (m *mapEncoder) DoMultiReusable(kvs []MapPair) ([]value, error) { - m.resizeBuffer(len(kvs)) - - for i, kv := range kvs { - m.resizeValueAtBuffer(i, kv.Size()) - err := kv.EncodeBytes(m.pairBuf[i].value) - if err != nil { - return nil, err - } - - m.pairBuf[i].tombstone = kv.Tombstone - } - - return m.pairBuf, nil -} - -func (m *mapEncoder) resizeBuffer(size int) { - if cap(m.pairBuf) >= size { - m.pairBuf = m.pairBuf[:size] - } else { - m.pairBuf = make([]value, size, int(float64(size)*1.25)) - } -} - -func (m *mapEncoder) resizeValueAtBuffer(pos, size int) { - if cap(m.pairBuf[pos].value) >= size { - m.pairBuf[pos].value = m.pairBuf[pos].value[:size] - } else { - m.pairBuf[pos].value = make([]byte, size, int(float64(size)*1.25)) - } -} diff --git a/adapters/repos/db/lsmkv/strategies_map_benchmark_test.go b/adapters/repos/db/lsmkv/strategies_map_benchmark_test.go deleted file mode 100644 index 2f9890372a024631cc8828a46b6db1e70d18337d..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/strategies_map_benchmark_test.go +++ /dev/null @@ -1,139 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "crypto/rand" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func BenchmarkMapDecoderDoPartial_SingleKey(b *testing.B) { - before := []MapPair{{ - Key: []byte("my-key-1"), - Value: []byte("my-value-1"), - }} - - encoded, err := newMapEncoder().DoMulti(before) - require.Nil(b, err) - - md := newMapDecoder() - - b.ReportAllocs() - - for i := 0; i < b.N; i++ { - md.DoPartial(encoded) - } -} - -func BenchmarkMapPairFromBytes(b *testing.B) { - before := MapPair{ - Key: []byte("my-key-1"), - Value: make([]byte, 24*1024), - } - - rand.Read(before.Value) - - encoded, err := before.Bytes() - require.Nil(b, err) - - b.ReportAllocs() - - target := MapPair{} - - for i := 0; i < b.N; i++ { - target.FromBytes(encoded, false) - } -} - -func BenchmarkMapPairFromBytesReusable_Fits(b *testing.B) { - before := MapPair{ - Key: []byte("my-key-1"), - Value: make([]byte, 24*1024), - } - - rand.Read(before.Value) - - encoded, err := before.Bytes() - require.Nil(b, err) - - target := MapPair{ - Key: make([]byte, 8), - Value: make([]byte, 24*1024), - } - - b.ReportAllocs() - - for i := 0; i < b.N; i++ { - err := target.FromBytesReusable(encoded, false) - require.Nil(b, err) - } - - assert.Equal(b, before.Key, target.Key) - assert.Equal(b, before.Value, target.Value) -} - -func BenchmarkMapPairFromBytesReusable_BuffersTooLarge(b *testing.B) { - before := MapPair{ - Key: []byte("my-key-1"), - Value: make([]byte, 24*1024), - } - - rand.Read(before.Value) - - encoded, err := before.Bytes() - require.Nil(b, err) - - target := MapPair{ - Key: make([]byte, 100), - Value: make([]byte, 100*1024), - } - - b.ReportAllocs() - - for i := 0; i < b.N; i++ { - err := target.FromBytesReusable(encoded, false) - require.Nil(b, err) - } - - assert.Equal(b, before.Key, target.Key) - assert.Equal(b, before.Value, target.Value) -} - -func BenchmarkMapPairFromBytesReusable_BuffersTooSmall(b *testing.B) { - before := MapPair{ - Key: []byte("my-key-1"), - Value: make([]byte, 24*1024), - } - - rand.Read(before.Value) - - encoded, err := before.Bytes() - require.Nil(b, err) - - target := MapPair{ - Key: make([]byte, 1), - Value: make([]byte, 1*1024), - } - - b.ReportAllocs() - - for i := 0; i < b.N; i++ { - err := target.FromBytesReusable(encoded, false) - require.Nil(b, err) - } - - assert.Equal(b, before.Key, target.Key) - assert.Equal(b, before.Value, target.Value) -} diff --git a/adapters/repos/db/lsmkv/strategies_map_integration_test.go b/adapters/repos/db/lsmkv/strategies_map_integration_test.go deleted file mode 100644 index b2b63127daf86fd35a7d690b3f5a138483a678f1..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/strategies_map_integration_test.go +++ /dev/null @@ -1,1247 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package lsmkv - -import ( - "context" - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/cyclemanager" -) - -func TestMapCollectionStrategy(t *testing.T) { - ctx := testCtx() - tests := bucketIntegrationTests{ - { - name: "mapInsertAndAppend", - f: mapInsertAndAppend, - opts: []BucketOption{ - WithStrategy(StrategyMapCollection), - }, - }, - { - name: "mapInsertAndDelete", - f: mapInsertAndDelete, - opts: []BucketOption{ - WithStrategy(StrategyMapCollection), - }, - }, - { - name: "mapCursors", - f: mapCursors, - opts: []BucketOption{ - WithStrategy(StrategyMapCollection), - }, - }, - } - tests.run(ctx, t) -} - -func mapInsertAndAppend(ctx context.Context, t *testing.T, opts []BucketOption) { - dirName := t.TempDir() - - t.Run("memtable-only", func(t *testing.T) { - b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - rowKey1 := []byte("test1-key-1") - rowKey2 := []byte("test1-key-2") - - t.Run("set original values and verify", func(t *testing.T) { - row1Map := []MapPair{ - { - Key: []byte("row1-key1"), - Value: []byte("row1-key1-value1"), - }, { - Key: []byte("row1-key2"), - Value: []byte("row1-key2-value1"), - }, - } - - row2Map := []MapPair{ - { - Key: []byte("row2-key1"), - Value: []byte("row2-key1-value1"), - }, { - Key: []byte("row2-key2"), - Value: []byte("row2-key2-value1"), - }, - } - - for _, pair := range row1Map { - err = b.MapSet(rowKey1, pair) - require.Nil(t, err) - } - - for _, pair := range row2Map { - err = b.MapSet(rowKey2, pair) - require.Nil(t, err) - } - - res, err := b.MapList(rowKey1) - require.Nil(t, err) - assert.Equal(t, row1Map, res) - res, err = b.MapList(rowKey2) - require.Nil(t, err) - assert.Equal(t, res, row2Map) - }) - - t.Run("replace an existing map key", func(t *testing.T) { - err = b.MapSet(rowKey1, MapPair{ - Key: []byte("row1-key1"), // existing key - Value: []byte("row1-key1-value2"), // updated value - }) - require.Nil(t, err) - - row1Updated := []MapPair{ - { - Key: []byte("row1-key1"), - Value: []byte("row1-key1-value2"), // <--- updated, rest unchanged - }, { - Key: []byte("row1-key2"), - Value: []byte("row1-key2-value1"), - }, - } - - row2Unchanged := []MapPair{ - { - Key: []byte("row2-key1"), - Value: []byte("row2-key1-value1"), - }, { - Key: []byte("row2-key2"), - Value: []byte("row2-key2-value1"), - }, - } - - res, err := b.MapList(rowKey1) - require.Nil(t, err) - // NOTE: We are accepting that the order is changed here. Given the name - // "MapCollection" there should be no expectations regarding the order, - // but we have yet to validate if this fits with all of the intended use - // cases. - assert.ElementsMatch(t, row1Updated, res) - res, err = b.MapList(rowKey2) - require.Nil(t, err) - assert.Equal(t, res, row2Unchanged) - }) - }) - - t.Run("with a single flush between updates", func(t *testing.T) { - b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - rowKey1 := []byte("test2-key-1") - rowKey2 := []byte("test2-key-2") - - t.Run("set original values and verify", func(t *testing.T) { - row1Map := []MapPair{ - { - Key: []byte("row1-key1"), - Value: []byte("row1-key1-value1"), - }, { - Key: []byte("row1-key2"), - Value: []byte("row1-key2-value1"), - }, - } - - row2Map := []MapPair{ - { - Key: []byte("row2-key1"), - Value: []byte("row2-key1-value1"), - }, { - Key: []byte("row2-key2"), - Value: []byte("row2-key2-value1"), - }, - } - - for _, pair := range row1Map { - err = b.MapSet(rowKey1, pair) - require.Nil(t, err) - } - - for _, pair := range row2Map { - err = b.MapSet(rowKey2, pair) - require.Nil(t, err) - } - - res, err := b.MapList(rowKey1) - require.Nil(t, err) - assert.Equal(t, row1Map, res) - res, err = b.MapList(rowKey2) - require.Nil(t, err) - assert.Equal(t, res, row2Map) - }) - - t.Run("flush to disk", func(t *testing.T) { - require.Nil(t, b.FlushAndSwitch()) - }) - - t.Run("replace an existing map key", func(t *testing.T) { - err = b.MapSet(rowKey1, MapPair{ - Key: []byte("row1-key1"), // existing key - Value: []byte("row1-key1-value2"), // updated value - }) - require.Nil(t, err) - - row1Updated := []MapPair{ - { - Key: []byte("row1-key1"), - Value: []byte("row1-key1-value2"), // <--- updated, rest unchanged - }, { - Key: []byte("row1-key2"), - Value: []byte("row1-key2-value1"), - }, - } - - row2Unchanged := []MapPair{ - { - Key: []byte("row2-key1"), - Value: []byte("row2-key1-value1"), - }, { - Key: []byte("row2-key2"), - Value: []byte("row2-key2-value1"), - }, - } - - res, err := b.MapList(rowKey1) - require.Nil(t, err) - // NOTE: We are accepting that the order is changed here. Given the name - // "MapCollection" there should be no expectations regarding the order, - // but we have yet to validate if this fits with all of the intended use - // cases. - assert.ElementsMatch(t, row1Updated, res) - res, err = b.MapList(rowKey2) - require.Nil(t, err) - assert.Equal(t, res, row2Unchanged) - }) - }) - - t.Run("with flushes after initial and update", func(t *testing.T) { - b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - rowKey1 := []byte("test3-key-1") - rowKey2 := []byte("test3-key-2") - - t.Run("set original values and verify", func(t *testing.T) { - row1Map := []MapPair{ - { - Key: []byte("row1-key1"), - Value: []byte("row1-key1-value1"), - }, { - Key: []byte("row1-key2"), - Value: []byte("row1-key2-value1"), - }, - } - - row2Map := []MapPair{ - { - Key: []byte("row2-key1"), - Value: []byte("row2-key1-value1"), - }, { - Key: []byte("row2-key2"), - Value: []byte("row2-key2-value1"), - }, - } - - for _, pair := range row1Map { - err = b.MapSet(rowKey1, pair) - require.Nil(t, err) - } - - for _, pair := range row2Map { - err = b.MapSet(rowKey2, pair) - require.Nil(t, err) - } - - res, err := b.MapList(rowKey1) - require.Nil(t, err) - assert.Equal(t, row1Map, res) - res, err = b.MapList(rowKey2) - require.Nil(t, err) - assert.Equal(t, res, row2Map) - }) - - t.Run("flush to disk", func(t *testing.T) { - require.Nil(t, b.FlushAndSwitch()) - }) - - t.Run("replace an existing map key", func(t *testing.T) { - err = b.MapSet(rowKey1, MapPair{ - Key: []byte("row1-key1"), // existing key - Value: []byte("row1-key1-value2"), // updated value - }) - require.Nil(t, err) - - // Flush again! - require.Nil(t, b.FlushAndSwitch()) - - row1Updated := []MapPair{ - { - Key: []byte("row1-key1"), - Value: []byte("row1-key1-value2"), // <--- updated, rest unchanged - }, { - Key: []byte("row1-key2"), - Value: []byte("row1-key2-value1"), - }, - } - - row2Unchanged := []MapPair{ - { - Key: []byte("row2-key1"), - Value: []byte("row2-key1-value1"), - }, { - Key: []byte("row2-key2"), - Value: []byte("row2-key2-value1"), - }, - } - - res, err := b.MapList(rowKey1) - require.Nil(t, err) - // NOTE: We are accepting that the order is changed here. Given the name - // "MapCollection" there should be no expectations regarding the order, - // but we have yet to validate if this fits with all of the intended use - // cases. - assert.ElementsMatch(t, row1Updated, res) - res, err = b.MapList(rowKey2) - require.Nil(t, err) - assert.Equal(t, res, row2Unchanged) - }) - }) - - t.Run("update in memtable, then do an orderly shutdown, and re-init", func(t *testing.T) { - b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - rowKey1 := []byte("test4-key-1") - rowKey2 := []byte("test4-key-2") - - t.Run("set original values and verify", func(t *testing.T) { - row1Map := []MapPair{ - { - Key: []byte("row1-key1"), - Value: []byte("row1-key1-value1"), - }, { - Key: []byte("row1-key2"), - Value: []byte("row1-key2-value1"), - }, - } - - row2Map := []MapPair{ - { - Key: []byte("row2-key1"), - Value: []byte("row2-key1-value1"), - }, { - Key: []byte("row2-key2"), - Value: []byte("row2-key2-value1"), - }, - } - - for _, pair := range row1Map { - err = b.MapSet(rowKey1, pair) - require.Nil(t, err) - } - - for _, pair := range row2Map { - err = b.MapSet(rowKey2, pair) - require.Nil(t, err) - } - - res, err := b.MapList(rowKey1) - require.Nil(t, err) - assert.Equal(t, row1Map, res) - res, err = b.MapList(rowKey2) - require.Nil(t, err) - assert.Equal(t, res, row2Map) - }) - - t.Run("replace an existing map key", func(t *testing.T) { - err = b.MapSet(rowKey1, MapPair{ - Key: []byte("row1-key1"), // existing key - Value: []byte("row1-key1-value2"), // updated value - }) - require.Nil(t, err) - }) - - t.Run("orderly shutdown", func(t *testing.T) { - b.Shutdown(context.Background()) - }) - - t.Run("init another bucket on the same files", func(t *testing.T) { - b2, err := NewBucket(ctx, dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - row1Updated := []MapPair{ - { - Key: []byte("row1-key1"), - Value: []byte("row1-key1-value2"), // <--- updated, rest unchanged - }, { - Key: []byte("row1-key2"), - Value: []byte("row1-key2-value1"), - }, - } - - row2Unchanged := []MapPair{ - { - Key: []byte("row2-key1"), - Value: []byte("row2-key1-value1"), - }, { - Key: []byte("row2-key2"), - Value: []byte("row2-key2-value1"), - }, - } - - res, err := b2.MapList(rowKey1) - require.Nil(t, err) - // NOTE: We are accepting that the order is changed here. Given the name - // "MapCollection" there should be no expectations regarding the order, - // but we have yet to validate if this fits with all of the intended use - // cases. - assert.ElementsMatch(t, row1Updated, res) - res, err = b2.MapList(rowKey2) - require.Nil(t, err) - assert.Equal(t, res, row2Unchanged) - }) - }) -} - -func mapInsertAndDelete(ctx context.Context, t *testing.T, opts []BucketOption) { - dirName := t.TempDir() - - t.Run("memtable-only", func(t *testing.T) { - b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - rowKey1 := []byte("test1-key-1") - rowKey2 := []byte("test1-key-2") - - t.Run("set original values and verify", func(t *testing.T) { - row1Map := []MapPair{ - { - Key: []byte("row1-key1"), - Value: []byte("row1-key1-value1"), - }, { - Key: []byte("row1-key2"), - Value: []byte("row1-key2-value1"), - }, - } - - row2Map := []MapPair{ - { - Key: []byte("row2-key1"), - Value: []byte("row2-key1-value1"), - }, { - Key: []byte("row2-key2"), - Value: []byte("row2-key2-value1"), - }, - } - - for _, pair := range row1Map { - err = b.MapSet(rowKey1, pair) - require.Nil(t, err) - } - - for _, pair := range row2Map { - err = b.MapSet(rowKey2, pair) - require.Nil(t, err) - } - - res, err := b.MapList(rowKey1) - require.Nil(t, err) - assert.Equal(t, row1Map, res) - res, err = b.MapList(rowKey2) - require.Nil(t, err) - assert.Equal(t, res, row2Map) - }) - - t.Run("delete some keys, re-add one of them", func(t *testing.T) { - err := b.MapDeleteKey(rowKey1, []byte("row1-key1")) - require.Nil(t, err) - err = b.MapDeleteKey(rowKey2, []byte("row2-key2")) - require.Nil(t, err) - err = b.MapSet(rowKey2, MapPair{ - Key: []byte("row2-key2"), - Value: []byte("row2-key2-reinserted"), - }) - require.Nil(t, err) - }) - - t.Run("validate the results", func(t *testing.T) { - row1Updated := []MapPair{ - // key 1 was deleted - { - Key: []byte("row1-key2"), - Value: []byte("row1-key2-value1"), - }, - } - - row2Updated := []MapPair{ - { - Key: []byte("row2-key1"), - Value: []byte("row2-key1-value1"), - }, { - Key: []byte("row2-key2"), - Value: []byte("row2-key2-reinserted"), - }, - } - - // NOTE: We are accepting that the order is changed here. Given the name - // "MapCollection" there should be no expectations regarding the order, - // but we have yet to validate if this fits with all of the intended use - // cases. - res, err := b.MapList(rowKey1) - require.Nil(t, err) - assert.ElementsMatch(t, row1Updated, res) - res, err = b.MapList(rowKey2) - require.Nil(t, err) - assert.ElementsMatch(t, row2Updated, res) - }) - }) - - t.Run("with flushes between updates", func(t *testing.T) { - b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - rowKey1 := []byte("test1-key-1") - rowKey2 := []byte("test1-key-2") - - t.Run("set original values and verify", func(t *testing.T) { - row1Map := []MapPair{ - { - Key: []byte("row1-key1"), - Value: []byte("row1-key1-value1"), - }, { - Key: []byte("row1-key2"), - Value: []byte("row1-key2-value1"), - }, - } - - row2Map := []MapPair{ - { - Key: []byte("row2-key1"), - Value: []byte("row2-key1-value1"), - }, { - Key: []byte("row2-key2"), - Value: []byte("row2-key2-value1"), - }, - } - - for _, pair := range row1Map { - err = b.MapSet(rowKey1, pair) - require.Nil(t, err) - } - - for _, pair := range row2Map { - err = b.MapSet(rowKey2, pair) - require.Nil(t, err) - } - - res, err := b.MapList(rowKey1) - require.Nil(t, err) - assert.Equal(t, row1Map, res) - res, err = b.MapList(rowKey2) - require.Nil(t, err) - assert.Equal(t, res, row2Map) - }) - - t.Run("flush to disk", func(t *testing.T) { - require.Nil(t, b.FlushAndSwitch()) - }) - - t.Run("delete some keys, re-add one of them", func(t *testing.T) { - err := b.MapDeleteKey(rowKey1, []byte("row1-key1")) - require.Nil(t, err) - err = b.MapDeleteKey(rowKey2, []byte("row2-key2")) - require.Nil(t, err) - err = b.MapSet(rowKey2, MapPair{ - Key: []byte("row2-key2"), - Value: []byte("row2-key2-reinserted"), - }) - require.Nil(t, err) - }) - - t.Run("flush to disk", func(t *testing.T) { - require.Nil(t, b.FlushAndSwitch()) - }) - - t.Run("validate the results", func(t *testing.T) { - row1Updated := []MapPair{ - // key 1 was deleted - { - Key: []byte("row1-key2"), - Value: []byte("row1-key2-value1"), - }, - } - - row2Updated := []MapPair{ - { - Key: []byte("row2-key1"), - Value: []byte("row2-key1-value1"), - }, { - Key: []byte("row2-key2"), - Value: []byte("row2-key2-reinserted"), - }, - } - - // NOTE: We are accepting that the order is changed here. Given the name - // "MapCollection" there should be no expectations regarding the order, - // but we have yet to validate if this fits with all of the intended use - // cases. - res, err := b.MapList(rowKey1) - require.Nil(t, err) - assert.ElementsMatch(t, row1Updated, res) - res, err = b.MapList(rowKey2) - require.Nil(t, err) - assert.ElementsMatch(t, row2Updated, res) - }) - }) - - t.Run("with memtable only, then an orderly shutdown and restart", func(t *testing.T) { - b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - rowKey1 := []byte("test1-key-1") - rowKey2 := []byte("test1-key-2") - - t.Run("set original values and verify", func(t *testing.T) { - row1Map := []MapPair{ - { - Key: []byte("row1-key1"), - Value: []byte("row1-key1-value1"), - }, { - Key: []byte("row1-key2"), - Value: []byte("row1-key2-value1"), - }, - } - - row2Map := []MapPair{ - { - Key: []byte("row2-key1"), - Value: []byte("row2-key1-value1"), - }, { - Key: []byte("row2-key2"), - Value: []byte("row2-key2-value1"), - }, - } - - for _, pair := range row1Map { - err = b.MapSet(rowKey1, pair) - require.Nil(t, err) - } - - for _, pair := range row2Map { - err = b.MapSet(rowKey2, pair) - require.Nil(t, err) - } - - res, err := b.MapList(rowKey1) - require.Nil(t, err) - assert.Equal(t, row1Map, res) - res, err = b.MapList(rowKey2) - require.Nil(t, err) - assert.Equal(t, res, row2Map) - }) - - t.Run("delete some keys, re-add one of them", func(t *testing.T) { - err := b.MapDeleteKey(rowKey1, []byte("row1-key1")) - require.Nil(t, err) - err = b.MapDeleteKey(rowKey2, []byte("row2-key2")) - require.Nil(t, err) - err = b.MapSet(rowKey2, MapPair{ - Key: []byte("row2-key2"), - Value: []byte("row2-key2-reinserted"), - }) - require.Nil(t, err) - }) - - t.Run("orderly shutdown", func(t *testing.T) { - b.Shutdown(context.Background()) - }) - - t.Run("init another bucket on the same files", func(t *testing.T) { - b2, err := NewBucket(ctx, dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - row1Updated := []MapPair{ - // key 1 was deleted - { - Key: []byte("row1-key2"), - Value: []byte("row1-key2-value1"), - }, - } - - row2Updated := []MapPair{ - { - Key: []byte("row2-key1"), - Value: []byte("row2-key1-value1"), - }, { - Key: []byte("row2-key2"), - Value: []byte("row2-key2-reinserted"), - }, - } - - // NOTE: We are accepting that the order is changed here. Given the name - // "MapCollection" there should be no expectations regarding the order, - // but we have yet to validate if this fits with all of the intended use - // cases. - res, err := b2.MapList(rowKey1) - require.Nil(t, err) - assert.ElementsMatch(t, row1Updated, res) - res, err = b2.MapList(rowKey2) - require.Nil(t, err) - assert.ElementsMatch(t, row2Updated, res) - }) - }) -} - -func mapCursors(ctx context.Context, t *testing.T, opts []BucketOption) { - t.Run("memtable-only", func(t *testing.T) { - r := getRandomSeed() - dirName := t.TempDir() - - b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - t.Run("set original values", func(t *testing.T) { - pairs := 20 - valuesPerPair := 3 - keys := make([][]byte, pairs) - values := make([][]MapPair, pairs) - - for i := range keys { - keys[i] = []byte(fmt.Sprintf("row-%03d", i)) - values[i] = make([]MapPair, valuesPerPair) - for j := range values[i] { - values[i][j] = MapPair{ - Key: []byte(fmt.Sprintf("row-%03d-key-%d", i, j)), - Value: []byte(fmt.Sprintf("row-%03d-value-%d", i, j)), - } - } - } - - // shuffle to make sure the BST isn't accidentally in order - r.Shuffle(len(keys), func(i, j int) { - keys[i], keys[j] = keys[j], keys[i] - values[i], values[j] = values[j], values[i] - }) - - for i := range keys { - mapPairs := values[i] - for j := range mapPairs { - err = b.MapSet(keys[i], mapPairs[j]) - require.Nil(t, err) - } - } - }) - - t.Run("seek from somewhere in the middle", func(t *testing.T) { - expectedKeys := [][]byte{ - []byte("row-016"), - []byte("row-017"), - []byte("row-018"), - []byte("row-019"), - } - expectedValues := [][]MapPair{ - { - {Key: []byte("row-016-key-0"), Value: []byte("row-016-value-0")}, - {Key: []byte("row-016-key-1"), Value: []byte("row-016-value-1")}, - {Key: []byte("row-016-key-2"), Value: []byte("row-016-value-2")}, - }, - { - {Key: []byte("row-017-key-0"), Value: []byte("row-017-value-0")}, - {Key: []byte("row-017-key-1"), Value: []byte("row-017-value-1")}, - {Key: []byte("row-017-key-2"), Value: []byte("row-017-value-2")}, - }, - { - {Key: []byte("row-018-key-0"), Value: []byte("row-018-value-0")}, - {Key: []byte("row-018-key-1"), Value: []byte("row-018-value-1")}, - {Key: []byte("row-018-key-2"), Value: []byte("row-018-value-2")}, - }, - { - {Key: []byte("row-019-key-0"), Value: []byte("row-019-value-0")}, - {Key: []byte("row-019-key-1"), Value: []byte("row-019-value-1")}, - {Key: []byte("row-019-key-2"), Value: []byte("row-019-value-2")}, - }, - } - - var retrievedKeys [][]byte - var retrievedValues [][]MapPair - c := b.MapCursor() - defer c.Close() - for k, v := c.Seek([]byte("row-016")); k != nil; k, v = c.Next() { - retrievedKeys = append(retrievedKeys, k) - retrievedValues = append(retrievedValues, v) - } - - assert.Equal(t, expectedKeys, retrievedKeys) - - require.Equal(t, len(expectedValues), len(retrievedValues)) - for i := range expectedValues { - assert.ElementsMatch(t, expectedValues[i], retrievedValues[i]) - } - }) - - t.Run("start from beginning", func(t *testing.T) { - expectedKeys := [][]byte{ - []byte("row-000"), - []byte("row-001"), - []byte("row-002"), - } - expectedValues := [][]MapPair{ - { - {Key: []byte("row-000-key-0"), Value: []byte("row-000-value-0")}, - {Key: []byte("row-000-key-1"), Value: []byte("row-000-value-1")}, - {Key: []byte("row-000-key-2"), Value: []byte("row-000-value-2")}, - }, - { - {Key: []byte("row-001-key-0"), Value: []byte("row-001-value-0")}, - {Key: []byte("row-001-key-1"), Value: []byte("row-001-value-1")}, - {Key: []byte("row-001-key-2"), Value: []byte("row-001-value-2")}, - }, - { - {Key: []byte("row-002-key-0"), Value: []byte("row-002-value-0")}, - {Key: []byte("row-002-key-1"), Value: []byte("row-002-value-1")}, - {Key: []byte("row-002-key-2"), Value: []byte("row-002-value-2")}, - }, - } - - var retrievedKeys [][]byte - var retrievedValues [][]MapPair - c := b.MapCursor() - defer c.Close() - retrieved := 0 - for k, v := c.First(); k != nil && retrieved < 3; k, v = c.Next() { - retrieved++ - retrievedKeys = append(retrievedKeys, k) - retrievedValues = append(retrievedValues, v) - } - - assert.Equal(t, expectedKeys, retrievedKeys) - - require.Equal(t, len(expectedValues), len(retrievedValues)) - for i := range expectedValues { - assert.ElementsMatch(t, expectedValues[i], retrievedValues[i]) - } - }) - - t.Run("delete/replace an existing map key/value pair", func(t *testing.T) { - row := []byte("row-002") - pair := MapPair{ - Key: []byte("row-002-key-1"), // existing key - Value: []byte("row-002-value-1-updated"), // updated value - } - - require.Nil(t, b.MapSet(row, pair)) - - row = []byte("row-001") - key := []byte("row-001-key-1") - - require.Nil(t, b.MapDeleteKey(row, key)) - }) - - t.Run("verify update is contained", func(t *testing.T) { - expectedKeys := [][]byte{ - []byte("row-001"), - []byte("row-002"), - } - expectedValues := [][]MapPair{ - { - {Key: []byte("row-001-key-0"), Value: []byte("row-001-value-0")}, - // key-1 was deleted - {Key: []byte("row-001-key-2"), Value: []byte("row-001-value-2")}, - }, - { - {Key: []byte("row-002-key-0"), Value: []byte("row-002-value-0")}, - {Key: []byte("row-002-key-1"), Value: []byte("row-002-value-1-updated")}, - {Key: []byte("row-002-key-2"), Value: []byte("row-002-value-2")}, - }, - } - - var retrievedKeys [][]byte - var retrievedValues [][]MapPair - c := b.MapCursor() - defer c.Close() - retrieved := 0 - for k, v := c.Seek([]byte("row-001")); k != nil && retrieved < 2; k, v = c.Next() { - retrieved++ - retrievedKeys = append(retrievedKeys, k) - retrievedValues = append(retrievedValues, v) - } - - assert.Equal(t, expectedKeys, retrievedKeys) - - require.Equal(t, len(expectedValues), len(retrievedValues)) - for i := range expectedValues { - assert.ElementsMatch(t, expectedValues[i], retrievedValues[i]) - } - }) - }) - - t.Run("with flushes", func(t *testing.T) { - r := getRandomSeed() - dirName := t.TempDir() - - b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - t.Run("first third (%3==0)", func(t *testing.T) { - pairs := 20 - valuesPerPair := 3 - var keys [][]byte - var values [][]MapPair - - for i := 0; i < pairs; i++ { - if i%3 != 0 { - continue - } - - keys = append(keys, []byte(fmt.Sprintf("row-%03d", i))) - curValues := make([]MapPair, valuesPerPair) - for j := range curValues { - curValues[j] = MapPair{ - Key: []byte(fmt.Sprintf("row-%03d-key-%d", i, j)), - Value: []byte(fmt.Sprintf("row-%03d-value-%d", i, j)), - } - } - - values = append(values, curValues) - } - - // shuffle to make sure the BST isn't accidentally in order - r.Shuffle(len(keys), func(i, j int) { - keys[i], keys[j] = keys[j], keys[i] - values[i], values[j] = values[j], values[i] - }) - - for i := range keys { - mapPairs := values[i] - for j := range mapPairs { - err = b.MapSet(keys[i], mapPairs[j]) - require.Nil(t, err) - } - } - }) - - t.Run("flush to disk", func(t *testing.T) { - require.Nil(t, b.FlushAndSwitch()) - }) - - t.Run("second third (%3==1)", func(t *testing.T) { - pairs := 20 - valuesPerPair := 3 - var keys [][]byte - var values [][]MapPair - - for i := 0; i < pairs; i++ { - if i%3 != 1 { - continue - } - - keys = append(keys, []byte(fmt.Sprintf("row-%03d", i))) - curValues := make([]MapPair, valuesPerPair) - for j := range curValues { - curValues[j] = MapPair{ - Key: []byte(fmt.Sprintf("row-%03d-key-%d", i, j)), - Value: []byte(fmt.Sprintf("row-%03d-value-%d", i, j)), - } - } - - values = append(values, curValues) - } - - // shuffle to make sure the BST isn't accidentally in order - r.Shuffle(len(keys), func(i, j int) { - keys[i], keys[j] = keys[j], keys[i] - values[i], values[j] = values[j], values[i] - }) - - for i := range keys { - mapPairs := values[i] - for j := range mapPairs { - err = b.MapSet(keys[i], mapPairs[j]) - require.Nil(t, err) - } - } - }) - - t.Run("flush to disk", func(t *testing.T) { - require.Nil(t, b.FlushAndSwitch()) - }) - - t.Run("third third (%3==2) memtable only", func(t *testing.T) { - pairs := 20 - valuesPerPair := 3 - var keys [][]byte - var values [][]MapPair - - for i := 0; i < pairs; i++ { - if i%3 != 2 { - continue - } - - keys = append(keys, []byte(fmt.Sprintf("row-%03d", i))) - curValues := make([]MapPair, valuesPerPair) - for j := range curValues { - curValues[j] = MapPair{ - Key: []byte(fmt.Sprintf("row-%03d-key-%d", i, j)), - Value: []byte(fmt.Sprintf("row-%03d-value-%d", i, j)), - } - } - - values = append(values, curValues) - } - - // shuffle to make sure the BST isn't accidentally in order - r.Shuffle(len(keys), func(i, j int) { - keys[i], keys[j] = keys[j], keys[i] - values[i], values[j] = values[j], values[i] - }) - - for i := range keys { - mapPairs := values[i] - for j := range mapPairs { - err = b.MapSet(keys[i], mapPairs[j]) - require.Nil(t, err) - } - } - - // no flush for this one, so this segment stays in the memtable - }) - - t.Run("seek from somewhere in the middle", func(t *testing.T) { - expectedKeys := [][]byte{ - []byte("row-016"), - []byte("row-017"), - []byte("row-018"), - []byte("row-019"), - } - expectedValues := [][]MapPair{ - { - {Key: []byte("row-016-key-0"), Value: []byte("row-016-value-0")}, - {Key: []byte("row-016-key-1"), Value: []byte("row-016-value-1")}, - {Key: []byte("row-016-key-2"), Value: []byte("row-016-value-2")}, - }, - { - {Key: []byte("row-017-key-0"), Value: []byte("row-017-value-0")}, - {Key: []byte("row-017-key-1"), Value: []byte("row-017-value-1")}, - {Key: []byte("row-017-key-2"), Value: []byte("row-017-value-2")}, - }, - { - {Key: []byte("row-018-key-0"), Value: []byte("row-018-value-0")}, - {Key: []byte("row-018-key-1"), Value: []byte("row-018-value-1")}, - {Key: []byte("row-018-key-2"), Value: []byte("row-018-value-2")}, - }, - { - {Key: []byte("row-019-key-0"), Value: []byte("row-019-value-0")}, - {Key: []byte("row-019-key-1"), Value: []byte("row-019-value-1")}, - {Key: []byte("row-019-key-2"), Value: []byte("row-019-value-2")}, - }, - } - - var retrievedKeys [][]byte - var retrievedValues [][]MapPair - c := b.MapCursor() - defer c.Close() - for k, v := c.Seek([]byte("row-016")); k != nil; k, v = c.Next() { - retrievedKeys = append(retrievedKeys, k) - retrievedValues = append(retrievedValues, v) - } - - assert.Equal(t, expectedKeys, retrievedKeys) - - require.Equal(t, len(expectedValues), len(retrievedValues)) - for i := range expectedValues { - assert.ElementsMatch(t, expectedValues[i], retrievedValues[i]) - } - }) - - t.Run("start from beginning", func(t *testing.T) { - expectedKeys := [][]byte{ - []byte("row-000"), - []byte("row-001"), - []byte("row-002"), - } - expectedValues := [][]MapPair{ - { - {Key: []byte("row-000-key-0"), Value: []byte("row-000-value-0")}, - {Key: []byte("row-000-key-1"), Value: []byte("row-000-value-1")}, - {Key: []byte("row-000-key-2"), Value: []byte("row-000-value-2")}, - }, - { - {Key: []byte("row-001-key-0"), Value: []byte("row-001-value-0")}, - {Key: []byte("row-001-key-1"), Value: []byte("row-001-value-1")}, - {Key: []byte("row-001-key-2"), Value: []byte("row-001-value-2")}, - }, - { - {Key: []byte("row-002-key-0"), Value: []byte("row-002-value-0")}, - {Key: []byte("row-002-key-1"), Value: []byte("row-002-value-1")}, - {Key: []byte("row-002-key-2"), Value: []byte("row-002-value-2")}, - }, - } - - var retrievedKeys [][]byte - var retrievedValues [][]MapPair - c := b.MapCursor() - defer c.Close() - retrieved := 0 - for k, v := c.First(); k != nil && retrieved < 3; k, v = c.Next() { - retrieved++ - retrievedKeys = append(retrievedKeys, k) - retrievedValues = append(retrievedValues, v) - } - - assert.Equal(t, expectedKeys, retrievedKeys) - - require.Equal(t, len(expectedValues), len(retrievedValues)) - for i := range expectedValues { - assert.ElementsMatch(t, expectedValues[i], retrievedValues[i]) - } - }) - - t.Run("delete/replace an existing map key/value pair", func(t *testing.T) { - row := []byte("row-002") - pair := MapPair{ - Key: []byte("row-002-key-1"), // existing key - Value: []byte("row-002-value-1-updated"), // updated value - } - - require.Nil(t, b.MapSet(row, pair)) - - row = []byte("row-001") - key := []byte("row-001-key-1") - - require.Nil(t, b.MapDeleteKey(row, key)) - }) - - t.Run("verify update is contained", func(t *testing.T) { - expectedKeys := [][]byte{ - []byte("row-001"), - []byte("row-002"), - } - expectedValues := [][]MapPair{ - { - {Key: []byte("row-001-key-0"), Value: []byte("row-001-value-0")}, - // key-1 was deleted - {Key: []byte("row-001-key-2"), Value: []byte("row-001-value-2")}, - }, - { - {Key: []byte("row-002-key-0"), Value: []byte("row-002-value-0")}, - {Key: []byte("row-002-key-1"), Value: []byte("row-002-value-1-updated")}, - {Key: []byte("row-002-key-2"), Value: []byte("row-002-value-2")}, - }, - } - - var retrievedKeys [][]byte - var retrievedValues [][]MapPair - c := b.MapCursor() - defer c.Close() - retrieved := 0 - for k, v := c.Seek([]byte("row-001")); k != nil && retrieved < 2; k, v = c.Next() { - retrieved++ - retrievedKeys = append(retrievedKeys, k) - retrievedValues = append(retrievedValues, v) - } - - assert.Equal(t, expectedKeys, retrievedKeys) - - require.Equal(t, len(expectedValues), len(retrievedValues)) - for i := range expectedValues { - assert.ElementsMatch(t, expectedValues[i], retrievedValues[i]) - } - }) - - t.Run("one final flush to disk", func(t *testing.T) { - require.Nil(t, b.FlushAndSwitch()) - }) - - t.Run("verify update is contained - after flushing the update", func(t *testing.T) { - expectedKeys := [][]byte{ - []byte("row-001"), - []byte("row-002"), - } - expectedValues := [][]MapPair{ - { - {Key: []byte("row-001-key-0"), Value: []byte("row-001-value-0")}, - // key-1 was deleted - {Key: []byte("row-001-key-2"), Value: []byte("row-001-value-2")}, - }, - { - {Key: []byte("row-002-key-0"), Value: []byte("row-002-value-0")}, - {Key: []byte("row-002-key-1"), Value: []byte("row-002-value-1-updated")}, - {Key: []byte("row-002-key-2"), Value: []byte("row-002-value-2")}, - }, - } - - var retrievedKeys [][]byte - var retrievedValues [][]MapPair - c := b.MapCursor() - defer c.Close() - retrieved := 0 - for k, v := c.Seek([]byte("row-001")); k != nil && retrieved < 2; k, v = c.Next() { - retrieved++ - retrievedKeys = append(retrievedKeys, k) - retrievedValues = append(retrievedValues, v) - } - - assert.Equal(t, expectedKeys, retrievedKeys) - - require.Equal(t, len(expectedValues), len(retrievedValues)) - for i := range expectedValues { - assert.ElementsMatch(t, expectedValues[i], retrievedValues[i]) - } - }) - }) -} diff --git a/adapters/repos/db/lsmkv/strategies_map_sorted_merger.go b/adapters/repos/db/lsmkv/strategies_map_sorted_merger.go deleted file mode 100644 index a6da13726b8e986cfdb4f58417f6c35c8e347d2e..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/strategies_map_sorted_merger.go +++ /dev/null @@ -1,211 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "bytes" - - "github.com/pkg/errors" -) - -type sortedMapMerger struct { - input [][]MapPair - output []MapPair - offsets []int -} - -func newSortedMapMerger() *sortedMapMerger { - return &sortedMapMerger{} -} - -func (s *sortedMapMerger) do(segments [][]MapPair) ([]MapPair, error) { - if err := s.init(segments); err != nil { - return nil, errors.Wrap(err, "init sorted map decoder") - } - - i := 0 - for { - match, ok := s.findSegmentWithLowestKey() - if !ok { - break - } - - if match.Tombstone { - // the latest version of this key was a tombstone, so we can ignore it - continue - } - - s.output[i] = match - i++ - } - - return s.output[:i], nil -} - -// same as .do() but does not remove the tombstone if the most latest version -// of a key is a tombstone. It can thus also be used in compactions -func (s *sortedMapMerger) doKeepTombstones(segments [][]MapPair) ([]MapPair, error) { - if err := s.init(segments); err != nil { - return nil, errors.Wrap(err, "init sorted map decoder") - } - - i := 0 - for { - match, ok := s.findSegmentWithLowestKey() - if !ok { - break - } - - s.output[i] = match - i++ - } - - return s.output[:i], nil -} - -// same as .doKeepTombstone() but requires initialization from the outside and -// can thus reuse state from previous rounds without having to allocate again. -// must be pre-faced by a call of reset() -func (s *sortedMapMerger) doKeepTombstonesReusable() ([]MapPair, error) { - i := 0 - for { - match, ok := s.findSegmentWithLowestKey() - if !ok { - break - } - - s.output[i] = match - i++ - } - - return s.output[:i], nil -} - -// init is automatically called by .do() or .doKeepTombstones() -func (s *sortedMapMerger) init(segments [][]MapPair) error { - s.input = segments - - // all offset pointers initialized at 0 which is where we want to start - s.offsets = make([]int, len(segments)) - - // The maximum output is the sum of all the input segments if there are only - // unique keys and zero tombstones. If there are duplicate keys (i.e. - // updates) or tombstones, we will slice off some elements of the output - // later, but this way we can be sure each index will always be initialized - // correctly - maxOutput := 0 - for _, seg := range segments { - maxOutput += len(seg) - } - s.output = make([]MapPair, maxOutput) - - return nil -} - -// reset can be manually called if sharing allocated state is desired, such as -// with .doKeepTombstonesReusable() -func (s *sortedMapMerger) reset(segments [][]MapPair) error { - s.input = segments - - if cap(s.offsets) >= len(segments) { - s.offsets = s.offsets[:len(segments)] - - // it existed before so we need to reset all offsets to 0 - for i := range s.offsets { - s.offsets[i] = 0 - } - } else { - s.offsets = make([]int, len(segments), int(float64(len(segments))*1.25)) - } - - // The maximum output is the sum of all the input segments if there are only - // unique keys and zero tombstones. If there are duplicate keys (i.e. - // updates) or tombstones, we will slice off some elements of the output - // later, but this way we can be sure each index will always be initialized - // correctly - maxOutput := 0 - for _, seg := range segments { - maxOutput += len(seg) - } - - if cap(s.output) >= maxOutput { - s.output = s.output[:maxOutput] - // no need to reset any values as all of them will be overridden anyway - } else { - s.output = make([]MapPair, maxOutput, int(float64(maxOutput)*1.25)) - } - - return nil -} - -func (s *sortedMapMerger) findSegmentWithLowestKey() (MapPair, bool) { - bestSeg := -1 - bestKey := []byte(nil) - - for segmentID := 0; segmentID < len(s.input); segmentID++ { - // check if a segment is already exhausted, then skip - if s.offsets[segmentID] >= len(s.input[segmentID]) { - continue - } - - currKey := s.input[segmentID][s.offsets[segmentID]].Key - if bestSeg == -1 { - // first time we're running, no need to compare, just set to current - bestSeg = segmentID - bestKey = currKey - continue - } - - cmp := bytes.Compare(currKey, bestKey) - if cmp > 0 { - // the segment we are currently looking at has a higher key than our - // current best so we can completely ignore it - continue - } - - if cmp < 0 { - // the segment we are currently looking at is a better match than the - // previous, this means, we have found a new favorite, but the previous - // best will still be valid in a future round - bestSeg = segmentID - bestKey = currKey - } - - if cmp == 0 { - // this the most interesting case: we are looking at a duplicate key. In - // this case the rightmost ("latest") segment takes precedence, however, - // we must make sure that the previous match gets discarded, otherwise we - // will find it again in the next round. - // - // We can simply increase the offset before updating the bestSeg pointer, - // which means we will never look at this element again - s.offsets[bestSeg]++ - - // now that the old element is discarded, we can update our pointers - bestSeg = segmentID - bestKey = currKey - } - } - - if bestSeg == -1 { - // we didn't find anything, looks like we have exhausted all segments - return MapPair{}, false - } - - // we can now be sure that bestSeg,bestKey is the latest version of the - // lowest key, there is only one job left to do: increase the offset, so we - // never find this segment again - bestMatch := s.input[bestSeg][s.offsets[bestSeg]] - s.offsets[bestSeg]++ - - return bestMatch, true -} diff --git a/adapters/repos/db/lsmkv/strategies_map_sorted_merger_test.go b/adapters/repos/db/lsmkv/strategies_map_sorted_merger_test.go deleted file mode 100644 index 3e8589c302d64980cc3fc0d9db0a233e2c069b90..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/strategies_map_sorted_merger_test.go +++ /dev/null @@ -1,371 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func Test_SortedMapMerger_RemoveTombstones(t *testing.T) { - t.Run("single entry, no tombstones", func(t *testing.T) { - m := newSortedMapMerger() - input1 := []MapPair{ - { - Key: []byte("hello"), - Value: []byte("world"), - }, - } - - input := [][]MapPair{input1} - - actual, err := m.do(input) - require.Nil(t, err) - - expected := []MapPair{ - { - Key: []byte("hello"), - Value: []byte("world"), - }, - } - assert.Equal(t, expected, actual) - }) - - t.Run("single entry, single tombstone for unrelated key", func(t *testing.T) { - m := newSortedMapMerger() - input1 := []MapPair{ - { - Key: []byte("hello"), - Value: []byte("world"), - }, - { - Key: []byte("unrelated"), - Tombstone: true, - }, - } - - input := [][]MapPair{input1} - - actual, err := m.do(input) - require.Nil(t, err) - - expected := []MapPair{ - { - Key: []byte("hello"), - Value: []byte("world"), - }, - } - assert.Equal(t, expected, actual) - }) - - t.Run("single entry with tombstone over two segments", func(t *testing.T) { - m := newSortedMapMerger() - input := [][]MapPair{ - { - { - Key: []byte("hello"), - Value: []byte("world"), - }, - }, - { - { - Key: []byte("hello"), - Tombstone: true, - }, - }, - } - - actual, err := m.do(input) - require.Nil(t, err) - - expected := []MapPair{} - assert.Equal(t, expected, actual) - }) - - t.Run("multiple segments including updates", func(t *testing.T) { - m := newSortedMapMerger() - input := [][]MapPair{ - { - { - Key: []byte("a"), - Value: []byte("a1"), - }, - { - Key: []byte("c"), - Value: []byte("c1"), - }, - { - Key: []byte("e"), - Value: []byte("e1"), - }, - }, - { - { - Key: []byte("a"), - Value: []byte("a2"), - }, - { - Key: []byte("b"), - Value: []byte("b2"), - }, - { - Key: []byte("c"), - Value: []byte("c2"), - }, - }, - { - { - Key: []byte("b"), - Value: []byte("b3"), - }, - }, - } - - actual, err := m.do(input) - require.Nil(t, err) - - expected := []MapPair{ - { - Key: []byte("a"), - Value: []byte("a2"), - }, - { - Key: []byte("b"), - Value: []byte("b3"), - }, - { - Key: []byte("c"), - Value: []byte("c2"), - }, - { - Key: []byte("e"), - Value: []byte("e1"), - }, - } - assert.Equal(t, expected, actual) - }) - - t.Run("multiple segments including deletes and re-adds", func(t *testing.T) { - m := newSortedMapMerger() - input := [][]MapPair{ - { - { - Key: []byte("a"), - Value: []byte("a1"), - }, - { - Key: []byte("c"), - Value: []byte("c1"), - }, - { - Key: []byte("e"), - Value: []byte("e1"), - }, - }, - { - { - Key: []byte("a"), - Value: []byte("a2"), - }, - { - Key: []byte("b"), - Tombstone: true, - }, - { - Key: []byte("c"), - Value: []byte("c2"), - }, - }, - { - { - Key: []byte("b"), - Value: []byte("b3"), - }, - { - Key: []byte("e"), - Tombstone: true, - }, - }, - } - - actual, err := m.do(input) - require.Nil(t, err) - - expected := []MapPair{ - { - Key: []byte("a"), - Value: []byte("a2"), - }, - { - Key: []byte("b"), - Value: []byte("b3"), - }, - { - Key: []byte("c"), - Value: []byte("c2"), - }, - } - assert.Equal(t, expected, actual) - }) -} - -func Test_SortedMapMerger_KeepTombstones(t *testing.T) { - m := newSortedMapMerger() - - t.Run("multiple segments including updates, deletes in 2nd segment", func(t *testing.T) { - input := [][]MapPair{ - { - { - Key: []byte("a"), - Value: []byte("a1"), - }, - { - Key: []byte("c"), - Value: []byte("c1"), - }, - { - Key: []byte("e"), - Value: []byte("e1"), - }, - }, - { - { - Key: []byte("a"), - Value: []byte("a2"), - }, - { - Key: []byte("b"), - Value: []byte("b2"), - }, - { - Key: []byte("c"), - Tombstone: true, - }, - }, - { - { - Key: []byte("b"), - Value: []byte("b3"), - }, - }, - } - - expected := []MapPair{ - { - Key: []byte("a"), - Value: []byte("a2"), - }, - { - Key: []byte("b"), - Value: []byte("b3"), - }, - { - Key: []byte("c"), - Tombstone: true, - }, - { - Key: []byte("e"), - Value: []byte("e1"), - }, - } - - t.Run("without reusable functionality - fresh state", func(t *testing.T) { - actual, err := m.doKeepTombstones(input) - require.Nil(t, err) - - assert.Equal(t, expected, actual) - }) - - t.Run("with reusable functionality - fresh state", func(t *testing.T) { - m.reset(input) - actual, err := m.doKeepTombstonesReusable() - require.Nil(t, err) - - assert.Equal(t, expected, actual) - }) - }) - - t.Run("inverse order, deletes in 1st segment", func(t *testing.T) { - input := [][]MapPair{ - { - { - Key: []byte("b"), - Value: []byte("b3"), - }, - }, - { - { - Key: []byte("a"), - Value: []byte("a2"), - }, - { - Key: []byte("b"), - Value: []byte("b2"), - }, - { - Key: []byte("c"), - Tombstone: true, - }, - }, - { - { - Key: []byte("a"), - Value: []byte("a1"), - }, - { - Key: []byte("c"), - Value: []byte("c1"), - }, - { - Key: []byte("e"), - Value: []byte("e1"), - }, - }, - } - - expected := []MapPair{ - { - Key: []byte("a"), - Value: []byte("a1"), - }, - { - Key: []byte("b"), - Value: []byte("b2"), - }, - { - Key: []byte("c"), - Value: []byte("c1"), - }, - { - Key: []byte("e"), - Value: []byte("e1"), - }, - } - - t.Run("without reusable functionality - fresh state", func(t *testing.T) { - actual, err := m.doKeepTombstones(input) - require.Nil(t, err) - - assert.Equal(t, expected, actual) - }) - - t.Run("with reusable functionality - dirty state", func(t *testing.T) { - m.reset(input) - actual, err := m.doKeepTombstonesReusable() - require.Nil(t, err) - - assert.Equal(t, expected, actual) - }) - }) -} diff --git a/adapters/repos/db/lsmkv/strategies_map_test.go b/adapters/repos/db/lsmkv/strategies_map_test.go deleted file mode 100644 index 8aa6325b7aaeb2ac3f4b2fb2c71f31bee63fa202..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/strategies_map_test.go +++ /dev/null @@ -1,352 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestMapEncoderDecoderJourney(t *testing.T) { - // this test first encodes the map pairs, then decodes them and replace - // duplicates, remove tombstones, etc. - type test struct { - name string - in []MapPair - out []MapPair - } - - tests := []test{ - { - name: "single pair", - in: []MapPair{ - { - Key: []byte("foo"), - Value: []byte("bar"), - }, - }, - out: []MapPair{ - { - Key: []byte("foo"), - Value: []byte("bar"), - }, - }, - }, - { - name: "single pair, updated value", - in: []MapPair{ - { - Key: []byte("foo"), - Value: []byte("bar"), - }, - { - Key: []byte("foo"), - Value: []byte("bar2"), - }, - }, - out: []MapPair{ - { - Key: []byte("foo"), - Value: []byte("bar2"), - }, - }, - }, - { - name: "single pair, tombstone added", - in: []MapPair{ - { - Key: []byte("foo"), - Value: []byte("bar"), - }, - { - Key: []byte("foo"), - Tombstone: true, - }, - }, - out: []MapPair{}, - }, - { - name: "single pair, tombstone added, same value added again", - in: []MapPair{ - { - Key: []byte("foo"), - Value: []byte("bar"), - }, - { - Key: []byte("foo"), - Tombstone: true, - }, - { - Key: []byte("foo"), - Value: []byte("bar2"), - }, - }, - out: []MapPair{ - { - Key: []byte("foo"), - Value: []byte("bar2"), - }, - }, - }, - { - name: "multiple values, combination of updates and tombstones", - in: []MapPair{ - { - Key: []byte("foo"), - Value: []byte("never-updated"), - }, - { - Key: []byte("foo1"), - Value: []byte("bar1"), - }, - { - Key: []byte("foo2"), - Value: []byte("bar2"), - }, - { - Key: []byte("foo2"), - Value: []byte("bar2.2"), - }, - { - Key: []byte("foo1"), - Tombstone: true, - }, - { - Key: []byte("foo2"), - Value: []byte("bar2.3"), - }, - { - Key: []byte("foo1"), - Value: []byte("bar1.2"), - }, - }, - out: []MapPair{ - { - Key: []byte("foo"), - Value: []byte("never-updated"), - }, - { - Key: []byte("foo1"), - Value: []byte("bar1.2"), - }, - { - Key: []byte("foo2"), - Value: []byte("bar2.3"), - }, - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - encoded := make([]value, len(test.in)) - for i, kv := range test.in { - enc, err := newMapEncoder().Do(kv) - require.Nil(t, err) - encoded[i] = enc[0] - } - res, err := newMapDecoder().Do(encoded, false) - require.Nil(t, err) - // NOTE: we are accepting that the order can be lost on updates - assert.ElementsMatch(t, test.out, res) - }) - } -} - -func TestDecoderRemoveTombstones(t *testing.T) { - t.Run("single entry, no tombstones", func(t *testing.T) { - m := newMapDecoder() - input := mustEncode([]MapPair{ - { - Key: []byte("hello"), - Value: []byte("world"), - }, - }) - - actual, err := m.doSimplified(input) - require.Nil(t, err) - - expected := []MapPair{ - { - Key: []byte("hello"), - Value: []byte("world"), - }, - } - assert.Equal(t, expected, actual) - }) - - t.Run("single entry, single tombstone", func(t *testing.T) { - m := newMapDecoder() - input := mustEncode([]MapPair{ - { - Key: []byte("hello"), - Value: []byte("world"), - }, - { - Key: []byte("hello"), - Tombstone: true, - }, - }) - - actual, err := m.doSimplified(input) - require.Nil(t, err) - - expected := []MapPair{} - assert.Equal(t, expected, actual) - }) - - t.Run("single entry, single tombstone, then read", func(t *testing.T) { - m := newMapDecoder() - input := mustEncode([]MapPair{ - { - Key: []byte("hello"), - Value: []byte("world"), - }, - { - Key: []byte("hello"), - Tombstone: true, - }, - { - Key: []byte("hello"), - Value: []byte("world"), - }, - }) - - actual, err := m.doSimplified(input) - require.Nil(t, err) - - expected := []MapPair{ - { - Key: []byte("hello"), - Value: []byte("world"), - }, - } - assert.Equal(t, expected, actual) - }) - - t.Run("three entries, two tombstones at the end", func(t *testing.T) { - m := newMapDecoder() - input := mustEncode([]MapPair{ - { - Key: []byte("hello"), - Value: []byte("world"), - }, - { - Key: []byte("bonjour"), - Value: []byte("world"), - }, - { - Key: []byte("guten tag"), - Value: []byte("world"), - }, - { - Key: []byte("hello"), - Tombstone: true, - }, - { - Key: []byte("bonjour"), - Tombstone: true, - }, - }) - - actual, err := m.doSimplified(input) - require.Nil(t, err) - - expected := []MapPair{ - { - Key: []byte("guten tag"), - Value: []byte("world"), - }, - } - assert.Equal(t, expected, actual) - }) - - t.Run("three entries, two tombstones at the end, then recreate the first", func(t *testing.T) { - m := newMapDecoder() - input := mustEncode([]MapPair{ - { - Key: []byte("hello"), - Value: []byte("world"), - }, - { - Key: []byte("bonjour"), - Value: []byte("world"), - }, - { - Key: []byte("guten tag"), - Value: []byte("world"), - }, - { - Key: []byte("hello"), - Tombstone: true, - }, - { - Key: []byte("bonjour"), - Tombstone: true, - }, - { - Key: []byte("bonjour"), - Value: []byte("world"), - }, - { - Key: []byte("hello"), - Value: []byte("world"), - }, - }) - - actual, err := m.doSimplified(input) - require.Nil(t, err) - - expected := []MapPair{ - { - Key: []byte("guten tag"), - Value: []byte("world"), - }, - { - Key: []byte("bonjour"), - Value: []byte("world"), - }, - { - Key: []byte("hello"), - Value: []byte("world"), - }, - } - assert.Equal(t, expected, actual) - }) -} - -func mustEncode(kvs []MapPair) []value { - res, err := newMapEncoder().DoMulti(kvs) - if err != nil { - panic(err) - } - - return res -} - -func Test_MapPair_EncodingBytes(t *testing.T) { - kv := MapPair{ - Key: []byte("hello-world-key1"), - Value: []byte("this is the value ;-)"), - } - - control, err := kv.Bytes() - assert.Nil(t, err) - - encoded := make([]byte, kv.Size()) - err = kv.EncodeBytes(encoded) - assert.Nil(t, err) - - assert.Equal(t, control, encoded) -} diff --git a/adapters/repos/db/lsmkv/strategies_replace_integration_test.go b/adapters/repos/db/lsmkv/strategies_replace_integration_test.go deleted file mode 100644 index a7b95fc80e17d85936a0bdb568cb030512fc2de6..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/strategies_replace_integration_test.go +++ /dev/null @@ -1,1489 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package lsmkv - -import ( - "context" - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/cyclemanager" -) - -func TestReplaceStrategy(t *testing.T) { - ctx := testCtx() - tests := bucketIntegrationTests{ - { - name: "replaceInsertAndUpdate", - f: replaceInsertAndUpdate, - opts: []BucketOption{ - WithStrategy(StrategyReplace), - }, - }, - { - name: "replaceInsertAndUpdate_WithSecondaryKeys", - f: replaceInsertAndUpdate_WithSecondaryKeys, - opts: []BucketOption{ - WithStrategy(StrategyReplace), - WithSecondaryIndices(1), - }, - }, - { - name: "replaceInsertAndDelete", - f: replaceInsertAndDelete, - opts: []BucketOption{ - WithStrategy(StrategyReplace), - }, - }, - { - name: "replaceCursors", - f: replaceCursors, - opts: []BucketOption{ - WithStrategy(StrategyReplace), - }, - }, - } - tests.run(ctx, t) -} - -func replaceInsertAndUpdate(ctx context.Context, t *testing.T, opts []BucketOption) { - dirName := t.TempDir() - - t.Run("memtable-only", func(t *testing.T) { - b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - t.Run("set original values and verify", func(t *testing.T) { - key1 := []byte("key-1") - key2 := []byte("key-2") - key3 := []byte("key-3") - orig1 := []byte("original value for key1") - orig2 := []byte("original value for key2") - orig3 := []byte("original value for key3") - - err = b.Put(key1, orig1) - require.Nil(t, err) - err = b.Put(key2, orig2) - require.Nil(t, err) - err = b.Put(key3, orig3) - require.Nil(t, err) - - assert.Equal(t, 3, b.Count()) - - res, err := b.Get(key1) - require.Nil(t, err) - assert.Equal(t, res, orig1) - res, err = b.Get(key2) - require.Nil(t, err) - assert.Equal(t, res, orig2) - res, err = b.Get(key3) - require.Nil(t, err) - assert.Equal(t, res, orig3) - }) - - t.Run("replace some, keep one", func(t *testing.T) { - key1 := []byte("key-1") - key2 := []byte("key-2") - key3 := []byte("key-3") - orig1 := []byte("original value for key1") - replaced2 := []byte("updated value for key2") - replaced3 := []byte("updated value for key3") - - err = b.Put(key2, replaced2) - require.Nil(t, err) - err = b.Put(key3, replaced3) - require.Nil(t, err) - - assert.Equal(t, 3, b.Count()) - - res, err := b.Get(key1) - require.Nil(t, err) - assert.Equal(t, res, orig1) - res, err = b.Get(key2) - require.Nil(t, err) - assert.Equal(t, res, replaced2) - res, err = b.Get(key3) - require.Nil(t, err) - assert.Equal(t, res, replaced3) - }) - }) - - t.Run("with single flush in between updates", func(t *testing.T) { - b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - t.Run("set original values and verify", func(t *testing.T) { - key1 := []byte("key-1") - key2 := []byte("key-2") - key3 := []byte("key-3") - orig1 := []byte("original value for key1") - orig2 := []byte("original value for key2") - orig3 := []byte("original value for key3") - - err = b.Put(key1, orig1) - require.Nil(t, err) - err = b.Put(key2, orig2) - require.Nil(t, err) - err = b.Put(key3, orig3) - require.Nil(t, err) - - res, err := b.Get(key1) - require.Nil(t, err) - assert.Equal(t, res, orig1) - res, err = b.Get(key2) - require.Nil(t, err) - assert.Equal(t, res, orig2) - res, err = b.Get(key3) - require.Nil(t, err) - assert.Equal(t, res, orig3) - }) - - t.Run("flush memtable to disk", func(t *testing.T) { - require.Nil(t, b.FlushAndSwitch()) - }) - - t.Run("count only objects on disk segment", func(t *testing.T) { - assert.Equal(t, 3, b.Count()) - }) - - t.Run("replace some, keep one", func(t *testing.T) { - key1 := []byte("key-1") - key2 := []byte("key-2") - key3 := []byte("key-3") - orig1 := []byte("original value for key1") - replaced2 := []byte("updated value for key2") - replaced3 := []byte("updated value for key3") - - err = b.Put(key2, replaced2) - require.Nil(t, err) - err = b.Put(key3, replaced3) - require.Nil(t, err) - - // make sure that the updates aren't counted as additions - assert.Equal(t, 3, b.Count()) - - res, err := b.Get(key1) - require.Nil(t, err) - assert.Equal(t, orig1, res) - res, err = b.Get(key2) - require.Nil(t, err) - assert.Equal(t, replaced2, res) - res, err = b.Get(key3) - require.Nil(t, err) - assert.Equal(t, replaced3, res) - }) - }) - - t.Run("with a flush after the initial write and after the update", func(t *testing.T) { - b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - t.Run("set original values and verify", func(t *testing.T) { - key1 := []byte("key-1") - key2 := []byte("key-2") - key3 := []byte("key-3") - orig1 := []byte("original value for key1") - orig2 := []byte("original value for key2") - orig3 := []byte("original value for key3") - - err = b.Put(key1, orig1) - require.Nil(t, err) - err = b.Put(key2, orig2) - require.Nil(t, err) - err = b.Put(key3, orig3) - require.Nil(t, err) - - res, err := b.Get(key1) - require.Nil(t, err) - assert.Equal(t, res, orig1) - res, err = b.Get(key2) - require.Nil(t, err) - assert.Equal(t, res, orig2) - res, err = b.Get(key3) - require.Nil(t, err) - assert.Equal(t, res, orig3) - }) - - t.Run("flush memtable to disk", func(t *testing.T) { - require.Nil(t, b.FlushAndSwitch()) - }) - - t.Run("replace some, keep one", func(t *testing.T) { - key1 := []byte("key-1") - key2 := []byte("key-2") - key3 := []byte("key-3") - orig1 := []byte("original value for key1") - replaced2 := []byte("updated value for key2") - replaced3 := []byte("updated value for key3") - - err = b.Put(key2, replaced2) - require.Nil(t, err) - err = b.Put(key3, replaced3) - require.Nil(t, err) - - // Flush before verifying! - require.Nil(t, b.FlushAndSwitch()) - - res, err := b.Get(key1) - require.Nil(t, err) - assert.Equal(t, res, orig1) - res, err = b.Get(key2) - require.Nil(t, err) - assert.Equal(t, res, replaced2) - res, err = b.Get(key3) - require.Nil(t, err) - assert.Equal(t, res, replaced3) - }) - - t.Run("count objects over several segments", func(t *testing.T) { - assert.Equal(t, 3, b.Count()) - }) - }) - - t.Run("update in memtable, then do an orderly shutdown, and re-init", func(t *testing.T) { - b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - t.Run("set original values and verify", func(t *testing.T) { - key1 := []byte("key-1") - key2 := []byte("key-2") - key3 := []byte("key-3") - orig1 := []byte("original value for key1") - orig2 := []byte("original value for key2") - orig3 := []byte("original value for key3") - - err = b.Put(key1, orig1) - require.Nil(t, err) - err = b.Put(key2, orig2) - require.Nil(t, err) - err = b.Put(key3, orig3) - require.Nil(t, err) - }) - - t.Run("replace some, keep one", func(t *testing.T) { - key1 := []byte("key-1") - key2 := []byte("key-2") - key3 := []byte("key-3") - orig1 := []byte("original value for key1") - replaced2 := []byte("updated value for key2") - replaced3 := []byte("updated value for key3") - - err = b.Put(key2, replaced2) - require.Nil(t, err) - err = b.Put(key3, replaced3) - require.Nil(t, err) - - res, err := b.Get(key1) - require.Nil(t, err) - assert.Equal(t, res, orig1) - res, err = b.Get(key2) - require.Nil(t, err) - assert.Equal(t, res, replaced2) - res, err = b.Get(key3) - require.Nil(t, err) - assert.Equal(t, res, replaced3) - }) - - t.Run("orderly shutdown", func(t *testing.T) { - b.Shutdown(context.Background()) - }) - - t.Run("init another bucket on the same files", func(t *testing.T) { - b2, err := NewBucket(ctx, dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - key1 := []byte("key-1") - key2 := []byte("key-2") - key3 := []byte("key-3") - orig1 := []byte("original value for key1") - replaced2 := []byte("updated value for key2") - replaced3 := []byte("updated value for key3") - - res, err := b2.Get(key1) - require.Nil(t, err) - assert.Equal(t, res, orig1) - res, err = b2.Get(key2) - require.Nil(t, err) - assert.Equal(t, res, replaced2) - res, err = b2.Get(key3) - require.Nil(t, err) - assert.Equal(t, res, replaced3) - - // count objects over several segments after disk read - assert.Equal(t, 3, b2.Count()) - }) - }) -} - -func replaceInsertAndUpdate_WithSecondaryKeys(ctx context.Context, t *testing.T, opts []BucketOption) { - dirName := t.TempDir() - - t.Run("memtable-only", func(t *testing.T) { - b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - t.Run("set original values and verify", func(t *testing.T) { - key1 := []byte("key-1") - key2 := []byte("key-2") - key3 := []byte("key-3") - secondaryKey1 := []byte("secondary-key-1") - secondaryKey2 := []byte("secondary-key-2") - secondaryKey3 := []byte("secondary-key-3") - orig1 := []byte("original value for key1") - orig2 := []byte("original value for key2") - orig3 := []byte("original value for key3") - - err = b.Put(key1, orig1, WithSecondaryKey(0, secondaryKey1)) - require.Nil(t, err) - err = b.Put(key2, orig2, WithSecondaryKey(0, secondaryKey2)) - require.Nil(t, err) - err = b.Put(key3, orig3, WithSecondaryKey(0, secondaryKey3)) - require.Nil(t, err) - - res, err := b.GetBySecondary(0, secondaryKey1) - require.Nil(t, err) - assert.Equal(t, res, orig1) - res, err = b.GetBySecondary(0, secondaryKey2) - require.Nil(t, err) - assert.Equal(t, res, orig2) - res, err = b.GetBySecondary(0, secondaryKey3) - require.Nil(t, err) - assert.Equal(t, res, orig3) - }) - - t.Run("replace some values, keep one - secondary keys not changed", func(t *testing.T) { - key2 := []byte("key-2") - key3 := []byte("key-3") - secondaryKey1 := []byte("secondary-key-1") - secondaryKey2 := []byte("secondary-key-2") - secondaryKey3 := []byte("secondary-key-3") - orig1 := []byte("original value for key1") - replaced2 := []byte("updated value for key2") - replaced3 := []byte("updated value for key3") - - err = b.Put(key2, replaced2, WithSecondaryKey(0, secondaryKey2)) - require.Nil(t, err) - err = b.Put(key3, replaced3, WithSecondaryKey(0, secondaryKey3)) - require.Nil(t, err) - - res, err := b.GetBySecondary(0, secondaryKey1) - require.Nil(t, err) - assert.Equal(t, res, orig1) - res, err = b.GetBySecondary(0, secondaryKey2) - require.Nil(t, err) - assert.Equal(t, res, replaced2) - res, err = b.GetBySecondary(0, secondaryKey3) - require.Nil(t, err) - assert.Equal(t, res, replaced3) - }) - - t.Run("replace the secondary keys on an update", func(t *testing.T) { - key2 := []byte("key-2") - key3 := []byte("key-3") - secondaryKey1 := []byte("secondary-key-1") - secondaryKey2 := []byte("secondary-key-2-updated") - secondaryKey3 := []byte("secondary-key-3-updated") - orig1 := []byte("original value for key1") - replaced2 := []byte("twice updated value for key2") - replaced3 := []byte("twice updated value for key3") - - err = b.Put(key2, replaced2, WithSecondaryKey(0, secondaryKey2)) - require.Nil(t, err) - err = b.Put(key3, replaced3, WithSecondaryKey(0, secondaryKey3)) - require.Nil(t, err) - - // verify you can find by updated secondary keys - res, err := b.GetBySecondary(0, secondaryKey1) - require.Nil(t, err) - assert.Equal(t, res, orig1) - res, err = b.GetBySecondary(0, secondaryKey2) - require.Nil(t, err) - assert.Equal(t, res, replaced2) - res, err = b.GetBySecondary(0, secondaryKey3) - require.Nil(t, err) - assert.Equal(t, res, replaced3) - }) - }) - - t.Run("with single flush in between updates", func(t *testing.T) { - b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - t.Run("set original values", func(t *testing.T) { - key1 := []byte("key-1") - key2 := []byte("key-2") - key3 := []byte("key-3") - secondaryKey1 := []byte("secondary-key-1") - secondaryKey2 := []byte("secondary-key-2") - secondaryKey3 := []byte("secondary-key-3") - orig1 := []byte("original value for key1") - orig2 := []byte("original value for key2") - orig3 := []byte("original value for key3") - - err = b.Put(key1, orig1, WithSecondaryKey(0, secondaryKey1)) - require.Nil(t, err) - err = b.Put(key2, orig2, WithSecondaryKey(0, secondaryKey2)) - require.Nil(t, err) - err = b.Put(key3, orig3, WithSecondaryKey(0, secondaryKey3)) - require.Nil(t, err) - }) - - t.Run("flush memtable to disk", func(t *testing.T) { - require.Nil(t, b.FlushAndSwitch()) - }) - - t.Run("replace the secondary keys on an update", func(t *testing.T) { - key2 := []byte("key-2") - key3 := []byte("key-3") - secondaryKey1 := []byte("secondary-key-1") - secondaryKey2 := []byte("secondary-key-2-updated") - secondaryKey3 := []byte("secondary-key-3-updated") - orig1 := []byte("original value for key1") - replaced2 := []byte("twice updated value for key2") - replaced3 := []byte("twice updated value for key3") - - err = b.Put(key2, replaced2, WithSecondaryKey(0, secondaryKey2)) - require.Nil(t, err) - err = b.Put(key3, replaced3, WithSecondaryKey(0, secondaryKey3)) - require.Nil(t, err) - - // verify you can find by updated secondary keys - res, err := b.GetBySecondary(0, secondaryKey1) - require.Nil(t, err) - assert.Equal(t, res, orig1) - res, err = b.GetBySecondary(0, secondaryKey2) - require.Nil(t, err) - assert.Equal(t, res, replaced2) - res, err = b.GetBySecondary(0, secondaryKey3) - require.Nil(t, err) - assert.Equal(t, res, replaced3) - }) - }) - - t.Run("with a flush after initial write and update", func(t *testing.T) { - b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - t.Run("set original values", func(t *testing.T) { - key1 := []byte("key-1") - key2 := []byte("key-2") - key3 := []byte("key-3") - secondaryKey1 := []byte("secondary-key-1") - secondaryKey2 := []byte("secondary-key-2") - secondaryKey3 := []byte("secondary-key-3") - orig1 := []byte("original value for key1") - orig2 := []byte("original value for key2") - orig3 := []byte("original value for key3") - - err = b.Put(key1, orig1, WithSecondaryKey(0, secondaryKey1)) - require.Nil(t, err) - err = b.Put(key2, orig2, WithSecondaryKey(0, secondaryKey2)) - require.Nil(t, err) - err = b.Put(key3, orig3, WithSecondaryKey(0, secondaryKey3)) - require.Nil(t, err) - }) - - t.Run("flush memtable to disk", func(t *testing.T) { - require.Nil(t, b.FlushAndSwitch()) - }) - - t.Run("replace the secondary keys on an update", func(t *testing.T) { - key2 := []byte("key-2") - key3 := []byte("key-3") - secondaryKey2 := []byte("secondary-key-2-updated") - secondaryKey3 := []byte("secondary-key-3-updated") - replaced2 := []byte("twice updated value for key2") - replaced3 := []byte("twice updated value for key3") - - err = b.Put(key2, replaced2, WithSecondaryKey(0, secondaryKey2)) - require.Nil(t, err) - err = b.Put(key3, replaced3, WithSecondaryKey(0, secondaryKey3)) - require.Nil(t, err) - }) - - t.Run("flush memtable to disk", func(t *testing.T) { - require.Nil(t, b.FlushAndSwitch()) - }) - - t.Run("verify again", func(t *testing.T) { - secondaryKey1 := []byte("secondary-key-1") - secondaryKey2 := []byte("secondary-key-2-updated") - secondaryKey3 := []byte("secondary-key-3-updated") - orig1 := []byte("original value for key1") - replaced2 := []byte("twice updated value for key2") - replaced3 := []byte("twice updated value for key3") - - // verify you can find by updated secondary keys - res, err := b.GetBySecondary(0, secondaryKey1) - require.Nil(t, err) - assert.Equal(t, res, orig1) - res, err = b.GetBySecondary(0, secondaryKey2) - require.Nil(t, err) - assert.Equal(t, res, replaced2) - res, err = b.GetBySecondary(0, secondaryKey3) - require.Nil(t, err) - assert.Equal(t, res, replaced3) - }) - }) - - t.Run("update in memtable then do an orderly shutdown and reinit", func(t *testing.T) { - b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - t.Run("set original values", func(t *testing.T) { - key1 := []byte("key-1") - key2 := []byte("key-2") - key3 := []byte("key-3") - secondaryKey1 := []byte("secondary-key-1") - secondaryKey2 := []byte("secondary-key-2") - secondaryKey3 := []byte("secondary-key-3") - orig1 := []byte("original value for key1") - orig2 := []byte("original value for key2") - orig3 := []byte("original value for key3") - - err = b.Put(key1, orig1, WithSecondaryKey(0, secondaryKey1)) - require.Nil(t, err) - err = b.Put(key2, orig2, WithSecondaryKey(0, secondaryKey2)) - require.Nil(t, err) - err = b.Put(key3, orig3, WithSecondaryKey(0, secondaryKey3)) - require.Nil(t, err) - }) - - t.Run("replace the secondary keys on an update", func(t *testing.T) { - key2 := []byte("key-2") - key3 := []byte("key-3") - secondaryKey2 := []byte("secondary-key-2-updated") - secondaryKey3 := []byte("secondary-key-3-updated") - replaced2 := []byte("twice updated value for key2") - replaced3 := []byte("twice updated value for key3") - - err = b.Put(key2, replaced2, WithSecondaryKey(0, secondaryKey2)) - require.Nil(t, err) - err = b.Put(key3, replaced3, WithSecondaryKey(0, secondaryKey3)) - require.Nil(t, err) - }) - - t.Run("flush memtable to disk", func(t *testing.T) { - require.Nil(t, b.Shutdown(context.Background())) - }) - - t.Run("init a new one and verify", func(t *testing.T) { - b2, err := NewBucket(ctx, dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - secondaryKey1 := []byte("secondary-key-1") - secondaryKey2 := []byte("secondary-key-2-updated") - secondaryKey3 := []byte("secondary-key-3-updated") - orig1 := []byte("original value for key1") - replaced2 := []byte("twice updated value for key2") - replaced3 := []byte("twice updated value for key3") - - // verify you can find by updated secondary keys - res, err := b2.GetBySecondary(0, secondaryKey1) - require.Nil(t, err) - assert.Equal(t, res, orig1) - res, err = b2.GetBySecondary(0, secondaryKey2) - require.Nil(t, err) - assert.Equal(t, res, replaced2) - res, err = b2.GetBySecondary(0, secondaryKey3) - require.Nil(t, err) - assert.Equal(t, res, replaced3) - }) - }) -} - -func replaceInsertAndDelete(ctx context.Context, t *testing.T, opts []BucketOption) { - dirName := t.TempDir() - - t.Run("memtable-only", func(t *testing.T) { - b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - t.Run("set original values", func(t *testing.T) { - key1 := []byte("key-1") - key2 := []byte("key-2") - key3 := []byte("key-3") - orig1 := []byte("original value for key1") - orig2 := []byte("original value for key2") - orig3 := []byte("original value for key3") - - err = b.Put(key1, orig1) - require.Nil(t, err) - err = b.Put(key2, orig2) - require.Nil(t, err) - err = b.Put(key3, orig3) - require.Nil(t, err) - }) - - t.Run("delete some, keep one", func(t *testing.T) { - key1 := []byte("key-1") - key2 := []byte("key-2") - key3 := []byte("key-3") - orig1 := []byte("original value for key1") - - err = b.Delete(key2) - require.Nil(t, err) - err = b.Delete(key3) - require.Nil(t, err) - - res, err := b.Get(key1) - require.Nil(t, err) - assert.Equal(t, res, orig1) - res, err = b.Get(key2) - require.Nil(t, err) - assert.Nil(t, res) - res, err = b.Get(key3) - require.Nil(t, err) - assert.Nil(t, res) - }) - - t.Run("count objects", func(t *testing.T) { - assert.Equal(t, 1, b.Count()) - }) - }) - - t.Run("with single flush in between updates", func(t *testing.T) { - b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - t.Run("set original values", func(t *testing.T) { - key1 := []byte("key-1") - key2 := []byte("key-2") - key3 := []byte("key-3") - orig1 := []byte("original value for key1") - orig2 := []byte("original value for key2") - orig3 := []byte("original value for key3") - - err = b.Put(key1, orig1) - require.Nil(t, err) - err = b.Put(key2, orig2) - require.Nil(t, err) - err = b.Put(key3, orig3) - require.Nil(t, err) - }) - - t.Run("flush to disk", func(t *testing.T) { - require.Nil(t, b.FlushAndSwitch()) - }) - - t.Run("delete some, keep one", func(t *testing.T) { - key1 := []byte("key-1") - key2 := []byte("key-2") - key3 := []byte("key-3") - orig1 := []byte("original value for key1") - - err = b.Delete(key2) - require.Nil(t, err) - err = b.Delete(key3) - require.Nil(t, err) - - res, err := b.Get(key1) - require.Nil(t, err) - assert.Equal(t, res, orig1) - res, err = b.Get(key2) - require.Nil(t, err) - assert.Nil(t, res) - res, err = b.Get(key3) - require.Nil(t, err) - assert.Nil(t, res) - }) - - t.Run("count objects", func(t *testing.T) { - assert.Equal(t, 1, b.Count()) - }) - }) - - t.Run("with flushes after initial write and delete", func(t *testing.T) { - b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - t.Run("set original values", func(t *testing.T) { - key1 := []byte("key-1") - key2 := []byte("key-2") - key3 := []byte("key-3") - orig1 := []byte("original value for key1") - orig2 := []byte("original value for key2") - orig3 := []byte("original value for key3") - - err = b.Put(key1, orig1) - require.Nil(t, err) - err = b.Put(key2, orig2) - require.Nil(t, err) - err = b.Put(key3, orig3) - require.Nil(t, err) - }) - - t.Run("flush to disk", func(t *testing.T) { - require.Nil(t, b.FlushAndSwitch()) - }) - - t.Run("delete some, keep one", func(t *testing.T) { - key1 := []byte("key-1") - key2 := []byte("key-2") - key3 := []byte("key-3") - orig1 := []byte("original value for key1") - - err = b.Delete(key2) - require.Nil(t, err) - err = b.Delete(key3) - require.Nil(t, err) - - // Flush again! - require.Nil(t, b.FlushAndSwitch()) - - res, err := b.Get(key1) - require.Nil(t, err) - assert.Equal(t, res, orig1) - res, err = b.Get(key2) - require.Nil(t, err) - assert.Nil(t, res) - res, err = b.Get(key3) - require.Nil(t, err) - assert.Nil(t, res) - }) - - t.Run("count objects", func(t *testing.T) { - assert.Equal(t, 1, b.Count()) - }) - }) -} - -func replaceCursors(ctx context.Context, t *testing.T, opts []BucketOption) { - t.Run("memtable-only", func(t *testing.T) { - r := getRandomSeed() - dirName := t.TempDir() - - b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - t.Run("set original values", func(t *testing.T) { - pairs := 20 - keys := make([][]byte, pairs) - values := make([][]byte, pairs) - - for i := range keys { - keys[i] = []byte(fmt.Sprintf("key-%03d", i)) - values[i] = []byte(fmt.Sprintf("value-%03d", i)) - } - - // shuffle to make sure the BST isn't accidentally in order - r.Shuffle(len(keys), func(i, j int) { - keys[i], keys[j] = keys[j], keys[i] - values[i], values[j] = values[j], values[i] - }) - - for i := range keys { - err = b.Put(keys[i], values[i]) - require.Nil(t, err) - } - }) - - t.Run("seek from somewhere in the middle", func(t *testing.T) { - expectedKeys := [][]byte{ - []byte("key-016"), - []byte("key-017"), - []byte("key-018"), - []byte("key-019"), - } - expectedValues := [][]byte{ - []byte("value-016"), - []byte("value-017"), - []byte("value-018"), - []byte("value-019"), - } - - var retrievedKeys [][]byte - var retrievedValues [][]byte - c := b.Cursor() - defer c.Close() - for k, v := c.Seek([]byte("key-016")); k != nil; k, v = c.Next() { - retrievedKeys = copyAndAppend(retrievedKeys, k) - retrievedValues = copyAndAppend(retrievedValues, v) - } - - assert.Equal(t, expectedKeys, retrievedKeys) - assert.Equal(t, expectedValues, retrievedValues) - }) - - t.Run("start from the beginning", func(t *testing.T) { - expectedKeys := [][]byte{ - []byte("key-000"), - []byte("key-001"), - []byte("key-002"), - } - expectedValues := [][]byte{ - []byte("value-000"), - []byte("value-001"), - []byte("value-002"), - } - - var retrievedKeys [][]byte - var retrievedValues [][]byte - c := b.Cursor() - defer c.Close() - retrieved := 0 - for k, v := c.First(); k != nil && retrieved < 3; k, v = c.Next() { - retrieved++ - retrievedKeys = copyAndAppend(retrievedKeys, k) - retrievedValues = copyAndAppend(retrievedValues, v) - } - - assert.Equal(t, expectedKeys, retrievedKeys) - assert.Equal(t, expectedValues, retrievedValues) - }) - - t.Run("replace a key", func(t *testing.T) { - key := []byte("key-002") - value := []byte("value-002-updated") - - err = b.Put(key, value) - require.Nil(t, err) - - expectedKeys := [][]byte{ - []byte("key-001"), - []byte("key-002"), - } - expectedValues := [][]byte{ - []byte("value-001"), - []byte("value-002-updated"), - } - - var retrievedKeys [][]byte - var retrievedValues [][]byte - c := b.Cursor() - defer c.Close() - retrieved := 0 - for k, v := c.Seek([]byte("key-001")); k != nil && retrieved < 2; k, v = c.Next() { - retrieved++ - retrievedKeys = copyAndAppend(retrievedKeys, k) - retrievedValues = copyAndAppend(retrievedValues, v) - } - - assert.Equal(t, expectedKeys, retrievedKeys) - assert.Equal(t, expectedValues, retrievedValues) - }) - - t.Run("delete a key", func(t *testing.T) { - key := []byte("key-002") - - err = b.Delete(key) - require.Nil(t, err) - - t.Run("seek to a specific key", func(t *testing.T) { - expectedKeys := [][]byte{ - []byte("key-001"), - []byte("key-003"), - } - expectedValues := [][]byte{ - []byte("value-001"), - []byte("value-003"), - } - var retrievedKeys [][]byte - var retrievedValues [][]byte - c := b.Cursor() - defer c.Close() - retrieved := 0 - for k, v := c.Seek([]byte("key-001")); k != nil && retrieved < 2; k, v = c.Next() { - retrieved++ - retrievedKeys = copyAndAppend(retrievedKeys, k) - retrievedValues = copyAndAppend(retrievedValues, v) - } - - assert.Equal(t, expectedKeys, retrievedKeys) - assert.Equal(t, expectedValues, retrievedValues) - }) - - t.Run("seek to first key", func(t *testing.T) { - expectedKeys := [][]byte{ - []byte("key-000"), - []byte("key-001"), - []byte("key-003"), - } - expectedValues := [][]byte{ - []byte("value-000"), - []byte("value-001"), - []byte("value-003"), - } - - var retrievedKeys [][]byte - var retrievedValues [][]byte - c := b.Cursor() - defer c.Close() - retrieved := 0 - for k, v := c.First(); k != nil && retrieved < 3; k, v = c.Next() { - retrieved++ - retrievedKeys = copyAndAppend(retrievedKeys, k) - retrievedValues = copyAndAppend(retrievedValues, v) - } - - assert.Equal(t, expectedKeys, retrievedKeys) - assert.Equal(t, expectedValues, retrievedValues) - }) - }) - - t.Run("delete the first key", func(t *testing.T) { - key := []byte("key-000") - - err = b.Delete(key) - require.Nil(t, err) - - t.Run("seek to a specific key", func(t *testing.T) { - expectedKeys := [][]byte{ - []byte("key-001"), - []byte("key-003"), - } - expectedValues := [][]byte{ - []byte("value-001"), - []byte("value-003"), - } - var retrievedKeys [][]byte - var retrievedValues [][]byte - c := b.Cursor() - defer c.Close() - retrieved := 0 - for k, v := c.Seek([]byte("key-000")); k != nil && retrieved < 2; k, v = c.Next() { - retrieved++ - retrievedKeys = copyAndAppend(retrievedKeys, k) - retrievedValues = copyAndAppend(retrievedValues, v) - } - - assert.Equal(t, expectedKeys, retrievedKeys) - assert.Equal(t, expectedValues, retrievedValues) - }) - - t.Run("seek to first key", func(t *testing.T) { - expectedKeys := [][]byte{ - []byte("key-001"), - []byte("key-003"), - } - expectedValues := [][]byte{ - []byte("value-001"), - []byte("value-003"), - } - - var retrievedKeys [][]byte - var retrievedValues [][]byte - c := b.Cursor() - defer c.Close() - retrieved := 0 - for k, v := c.First(); k != nil && retrieved < 2; k, v = c.Next() { - retrieved++ - retrievedKeys = copyAndAppend(retrievedKeys, k) - retrievedValues = copyAndAppend(retrievedValues, v) - } - - assert.Equal(t, expectedKeys, retrievedKeys) - assert.Equal(t, expectedValues, retrievedValues) - }) - }) - }) - - t.Run("with a single flush", func(t *testing.T) { - r := getRandomSeed() - dirName := t.TempDir() - - b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - t.Run("set original values", func(t *testing.T) { - pairs := 20 - keys := make([][]byte, pairs) - values := make([][]byte, pairs) - - for i := range keys { - keys[i] = []byte(fmt.Sprintf("key-%03d", i)) - values[i] = []byte(fmt.Sprintf("value-%03d", i)) - } - - // shuffle to make sure the BST isn't accidentally in order - r.Shuffle(len(keys), func(i, j int) { - keys[i], keys[j] = keys[j], keys[i] - values[i], values[j] = values[j], values[i] - }) - - for i := range keys { - err = b.Put(keys[i], values[i]) - require.Nil(t, err) - } - }) - - t.Run("flush to disk", func(t *testing.T) { - require.Nil(t, b.FlushAndSwitch()) - }) - - t.Run("seek from somewhere in the middle", func(t *testing.T) { - expectedKeys := [][]byte{ - []byte("key-016"), - []byte("key-017"), - []byte("key-018"), - []byte("key-019"), - } - expectedValues := [][]byte{ - []byte("value-016"), - []byte("value-017"), - []byte("value-018"), - []byte("value-019"), - } - - var retrievedKeys [][]byte - var retrievedValues [][]byte - c := b.Cursor() - defer c.Close() - for k, v := c.Seek([]byte("key-016")); k != nil; k, v = c.Next() { - retrievedKeys = copyAndAppend(retrievedKeys, k) - retrievedValues = copyAndAppend(retrievedValues, v) - } - - assert.Equal(t, expectedKeys, retrievedKeys) - assert.Equal(t, expectedValues, retrievedValues) - }) - - t.Run("start from the beginning", func(t *testing.T) { - expectedKeys := [][]byte{ - []byte("key-000"), - []byte("key-001"), - []byte("key-002"), - } - expectedValues := [][]byte{ - []byte("value-000"), - []byte("value-001"), - []byte("value-002"), - } - - var retrievedKeys [][]byte - var retrievedValues [][]byte - c := b.Cursor() - defer c.Close() - retrieved := 0 - for k, v := c.First(); k != nil && retrieved < 3; k, v = c.Next() { - retrieved++ - retrievedKeys = copyAndAppend(retrievedKeys, k) - retrievedValues = copyAndAppend(retrievedValues, v) - } - - assert.Equal(t, expectedKeys, retrievedKeys) - assert.Equal(t, expectedValues, retrievedValues) - }) - }) - - t.Run("mixing several disk segments and memtable - with updates", func(t *testing.T) { - r := getRandomSeed() - dirName := t.TempDir() - - b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - t.Run("first third (%3==0)", func(t *testing.T) { - pairs := 20 - var keys [][]byte - var values [][]byte - - for i := 0; i < pairs; i++ { - if i%3 == 0 { - keys = copyAndAppend(keys, []byte(fmt.Sprintf("key-%03d", i))) - values = copyAndAppend(values, []byte(fmt.Sprintf("value-%03d", i))) - } - } - - // shuffle to make sure the BST isn't accidentally in order - r.Shuffle(len(keys), func(i, j int) { - keys[i], keys[j] = keys[j], keys[i] - values[i], values[j] = values[j], values[i] - }) - - for i := range keys { - err = b.Put(keys[i], values[i]) - require.Nil(t, err) - } - }) - - t.Run("flush to disk", func(t *testing.T) { - require.Nil(t, b.FlushAndSwitch()) - }) - - t.Run("second third (%3==1)", func(t *testing.T) { - pairs := 20 - var keys [][]byte - var values [][]byte - - for i := 0; i < pairs; i++ { - if i%3 == 1 { - keys = copyAndAppend(keys, []byte(fmt.Sprintf("key-%03d", i))) - values = copyAndAppend(values, []byte(fmt.Sprintf("value-%03d", i))) - } - } - - // shuffle to make sure the BST isn't accidentally in order - r.Shuffle(len(keys), func(i, j int) { - keys[i], keys[j] = keys[j], keys[i] - values[i], values[j] = values[j], values[i] - }) - - for i := range keys { - err = b.Put(keys[i], values[i]) - require.Nil(t, err) - } - }) - - t.Run("update something that was already written in segment 1", func(t *testing.T) { - require.Nil(t, b.Put([]byte("key-000"), []byte("updated-value-000"))) - require.Nil(t, b.Delete([]byte("key-003"))) - }) - - t.Run("flush to disk", func(t *testing.T) { - require.Nil(t, b.FlushAndSwitch()) - }) - - t.Run("third third (%3==2) memtable only", func(t *testing.T) { - pairs := 20 - var keys [][]byte - var values [][]byte - - for i := 0; i < pairs; i++ { - if i%3 == 2 { - keys = copyAndAppend(keys, []byte(fmt.Sprintf("key-%03d", i))) - values = copyAndAppend(values, []byte(fmt.Sprintf("value-%03d", i))) - } - } - - // shuffle to make sure the BST isn't accidentally in order - r.Shuffle(len(keys), func(i, j int) { - keys[i], keys[j] = keys[j], keys[i] - values[i], values[j] = values[j], values[i] - }) - - for i := range keys { - err = b.Put(keys[i], values[i]) - require.Nil(t, err) - } - - // no flush for this one, so this segment stays in the memtable - }) - - t.Run("update something that was already written previously", func(t *testing.T) { - require.Nil(t, b.Put([]byte("key-000"), []byte("twice-updated-value-000"))) - require.Nil(t, b.Put([]byte("key-001"), []byte("once-updated-value-001"))) - require.Nil(t, b.Put([]byte("key-019"), []byte("once-updated-value-019"))) - require.Nil(t, b.Delete([]byte("key-018"))) - }) - - t.Run("seek from somewhere in the middle", func(t *testing.T) { - expectedKeys := [][]byte{ - []byte("key-016"), - []byte("key-017"), - // key-018 deleted - []byte("key-019"), - } - expectedValues := [][]byte{ - []byte("value-016"), - []byte("value-017"), - []byte("once-updated-value-019"), - } - - var retrievedKeys [][]byte - var retrievedValues [][]byte - c := b.Cursor() - defer c.Close() - for k, v := c.Seek([]byte("key-016")); k != nil; k, v = c.Next() { - retrievedKeys = copyAndAppend(retrievedKeys, k) - retrievedValues = copyAndAppend(retrievedValues, v) - } - - assert.Equal(t, expectedKeys, retrievedKeys) - assert.Equal(t, expectedValues, retrievedValues) - }) - - t.Run("start from the beginning", func(t *testing.T) { - expectedKeys := [][]byte{ - []byte("key-000"), - []byte("key-001"), - []byte("key-002"), - // key-003 was deleted - []byte("key-004"), - } - expectedValues := [][]byte{ - []byte("twice-updated-value-000"), - []byte("once-updated-value-001"), - []byte("value-002"), - []byte("value-004"), - } - - var retrievedKeys [][]byte - var retrievedValues [][]byte - c := b.Cursor() - defer c.Close() - retrieved := 0 - for k, v := c.First(); k != nil && retrieved < 4; k, v = c.Next() { - retrieved++ - retrievedKeys = copyAndAppend(retrievedKeys, k) - retrievedValues = copyAndAppend(retrievedValues, v) - } - - assert.Equal(t, expectedKeys, retrievedKeys) - assert.Equal(t, expectedValues, retrievedValues) - }) - - t.Run("re-add the deleted keys", func(t *testing.T) { - require.Nil(t, b.Put([]byte("key-003"), []byte("readded-003"))) - require.Nil(t, b.Put([]byte("key-018"), []byte("readded-018"))) - // tombstones are now only in memtable - }) - - t.Run("seek from somewhere in the middle", func(t *testing.T) { - expectedKeys := [][]byte{ - []byte("key-016"), - []byte("key-017"), - []byte("key-018"), - []byte("key-019"), - } - expectedValues := [][]byte{ - []byte("value-016"), - []byte("value-017"), - []byte("readded-018"), - []byte("once-updated-value-019"), - } - - var retrievedKeys [][]byte - var retrievedValues [][]byte - c := b.Cursor() - defer c.Close() - for k, v := c.Seek([]byte("key-016")); k != nil; k, v = c.Next() { - retrievedKeys = copyAndAppend(retrievedKeys, k) - retrievedValues = copyAndAppend(retrievedValues, v) - } - - assert.Equal(t, expectedKeys, retrievedKeys) - assert.Equal(t, expectedValues, retrievedValues) - }) - - t.Run("start from the beginning", func(t *testing.T) { - expectedKeys := [][]byte{ - []byte("key-000"), - []byte("key-001"), - []byte("key-002"), - []byte("key-003"), - } - expectedValues := [][]byte{ - []byte("twice-updated-value-000"), - []byte("once-updated-value-001"), - []byte("value-002"), - []byte("readded-003"), - } - - var retrievedKeys [][]byte - var retrievedValues [][]byte - c := b.Cursor() - defer c.Close() - retrieved := 0 - for k, v := c.First(); k != nil && retrieved < 4; k, v = c.Next() { - retrieved++ - retrievedKeys = copyAndAppend(retrievedKeys, k) - retrievedValues = copyAndAppend(retrievedValues, v) - } - - assert.Equal(t, expectedKeys, retrievedKeys) - assert.Equal(t, expectedValues, retrievedValues) - }) - - t.Run("perform a final flush to disk", func(t *testing.T) { - require.Nil(t, b.FlushAndSwitch()) - }) - - t.Run("seek from somewhere in the middle", func(t *testing.T) { - expectedKeys := [][]byte{ - []byte("key-016"), - []byte("key-017"), - []byte("key-018"), - []byte("key-019"), - } - expectedValues := [][]byte{ - []byte("value-016"), - []byte("value-017"), - []byte("readded-018"), - []byte("once-updated-value-019"), - } - - var retrievedKeys [][]byte - var retrievedValues [][]byte - c := b.Cursor() - defer c.Close() - for k, v := c.Seek([]byte("key-016")); k != nil; k, v = c.Next() { - retrievedKeys = copyAndAppend(retrievedKeys, k) - retrievedValues = copyAndAppend(retrievedValues, v) - } - - assert.Equal(t, expectedKeys, retrievedKeys) - assert.Equal(t, expectedValues, retrievedValues) - }) - - t.Run("start from the beginning", func(t *testing.T) { - expectedKeys := [][]byte{ - []byte("key-000"), - []byte("key-001"), - []byte("key-002"), - []byte("key-003"), - } - expectedValues := [][]byte{ - []byte("twice-updated-value-000"), - []byte("once-updated-value-001"), - []byte("value-002"), - []byte("readded-003"), - } - - var retrievedKeys [][]byte - var retrievedValues [][]byte - c := b.Cursor() - defer c.Close() - retrieved := 0 - for k, v := c.First(); k != nil && retrieved < 4; k, v = c.Next() { - retrieved++ - retrievedKeys = copyAndAppend(retrievedKeys, k) - retrievedValues = copyAndAppend(retrievedValues, v) - } - - assert.Equal(t, expectedKeys, retrievedKeys) - assert.Equal(t, expectedValues, retrievedValues) - }) - }) - - // This test is inspired by unusual behavior encountered as part of the - // evaluation of gh-1569 where a delete could sometimes lead to no data after - // a restart which was caused by the disk segment cursor's .first() method - // not returning the correct key. Thus we'd have a null-key with a tombstone - // which would override whatever is the real "first" key, since null is - // always smaller - t.Run("with deletes as latest in some segments", func(t *testing.T) { - dirName := t.TempDir() - - b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - t.Run("add new datapoint", func(t *testing.T) { - err := b.Put([]byte("key-1"), []byte("value-1")) - require.Nil(t, err) - }) - - t.Run("add datapoint and flush", func(t *testing.T) { - err := b.Put([]byte("key-8"), []byte("value-8")) - require.Nil(t, err) - - require.Nil(t, b.FlushAndSwitch()) - }) - - t.Run("delete datapoint and flush", func(t *testing.T) { - err := b.Delete([]byte("key-8")) - // note that we are deleting the key with the 'higher' key, so a missing - // key on the delete would definitely be mismatched. If we had instead - // the deleted the first key, the incorrect tombstone would have been - // correct by coincidence - require.Nil(t, err) - - require.Nil(t, b.FlushAndSwitch()) - }) - - t.Run("verify", func(t *testing.T) { - expectedKeys := [][]byte{ - []byte("key-1"), - } - expectedValues := [][]byte{ - []byte("value-1"), - } - - var retrievedKeys [][]byte - var retrievedValues [][]byte - c := b.Cursor() - defer c.Close() - retrieved := 0 - for k, v := c.First(); k != nil && retrieved < 4; k, v = c.Next() { - retrieved++ - retrievedKeys = copyAndAppend(retrievedKeys, k) - retrievedValues = copyAndAppend(retrievedValues, v) - } - - assert.Equal(t, expectedKeys, retrievedKeys) - assert.Equal(t, expectedValues, retrievedValues) - }) - }) -} - -func copyAndAppend(list [][]byte, elem []byte) [][]byte { - elemCopy := make([]byte, len(elem)) - copy(elemCopy, elem) - return append(list, elemCopy) -} diff --git a/adapters/repos/db/lsmkv/strategies_roaring_set_integration_test.go b/adapters/repos/db/lsmkv/strategies_roaring_set_integration_test.go deleted file mode 100644 index 77dfb66db22b0b10be514cc70439eed1b343cf24..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/strategies_roaring_set_integration_test.go +++ /dev/null @@ -1,196 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package lsmkv - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/cyclemanager" -) - -func TestRoaringSetStrategy(t *testing.T) { - ctx := testCtx() - tests := bucketIntegrationTests{ - { - name: "roaringsetInsertAndSetAdd", - f: roaringsetInsertAndSetAdd, - opts: []BucketOption{ - WithStrategy(StrategyRoaringSet), - }, - }, - } - tests.run(ctx, t) -} - -func roaringsetInsertAndSetAdd(ctx context.Context, t *testing.T, opts []BucketOption) { - dirName := t.TempDir() - - t.Run("memtable-only", func(t *testing.T) { - b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - key1 := []byte("test1-key-1") - key2 := []byte("test1-key-2") - key3 := []byte("test1-key-3") - - t.Run("set original values and verify", func(t *testing.T) { - orig1 := []uint64{1, 2} - orig2 := []uint64{3, 4} - orig3 := []uint64{5, 6} - - err = b.RoaringSetAddList(key1, orig1) - require.Nil(t, err) - err = b.RoaringSetAddList(key2, orig2) - require.Nil(t, err) - err = b.RoaringSetAddList(key3, orig3) - require.Nil(t, err) - - res, err := b.RoaringSetGet(key1) - require.Nil(t, err) - for _, testVal := range orig1 { - assert.True(t, res.Contains(testVal)) - } - - res, err = b.RoaringSetGet(key2) - require.Nil(t, err) - for _, testVal := range orig2 { - assert.True(t, res.Contains(testVal)) - } - - res, err = b.RoaringSetGet(key3) - require.Nil(t, err) - for _, testVal := range orig3 { - assert.True(t, res.Contains(testVal)) - } - }) - - t.Run("extend some, delete some, keep some", func(t *testing.T) { - additions2 := []uint64{5} - removal3 := uint64(5) - - err = b.RoaringSetAddList(key2, additions2) - require.Nil(t, err) - err = b.RoaringSetRemoveOne(key3, removal3) - require.Nil(t, err) - - res, err := b.RoaringSetGet(key1) - require.Nil(t, err) - for _, testVal := range []uint64{1, 2} { // unchanged values - assert.True(t, res.Contains(testVal)) - } - - res, err = b.RoaringSetGet(key2) - require.Nil(t, err) - for _, testVal := range []uint64{3, 4, 5} { // extended with 5 - assert.True(t, res.Contains(testVal)) - } - - res, err = b.RoaringSetGet(key3) - require.Nil(t, err) - for _, testVal := range []uint64{6} { // fewer remain - assert.True(t, res.Contains(testVal)) - } - for _, testVal := range []uint64{5} { // no longer contained - assert.False(t, res.Contains(testVal)) - } - }) - }) - - t.Run("with a single flush in between updates", func(t *testing.T) { - b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - key1 := []byte("test1-key-1") - key2 := []byte("test1-key-2") - key3 := []byte("test1-key-3") - - t.Run("set original values and verify", func(t *testing.T) { - orig1 := []uint64{1, 2} - orig2 := []uint64{3, 4} - orig3 := []uint64{5, 6} - - err = b.RoaringSetAddList(key1, orig1) - require.Nil(t, err) - err = b.RoaringSetAddList(key2, orig2) - require.Nil(t, err) - err = b.RoaringSetAddList(key3, orig3) - require.Nil(t, err) - - res, err := b.RoaringSetGet(key1) - require.Nil(t, err) - for _, testVal := range orig1 { - assert.True(t, res.Contains(testVal)) - } - - res, err = b.RoaringSetGet(key2) - require.Nil(t, err) - for _, testVal := range orig2 { - assert.True(t, res.Contains(testVal)) - } - - res, err = b.RoaringSetGet(key3) - require.Nil(t, err) - for _, testVal := range orig3 { - assert.True(t, res.Contains(testVal)) - } - }) - - t.Run("flush to disk", func(t *testing.T) { - require.Nil(t, b.FlushAndSwitch()) - }) - - t.Run("extend some, delete some, keep some", func(t *testing.T) { - additions2 := []uint64{5} - removal3 := uint64(5) - - err = b.RoaringSetAddList(key2, additions2) - require.Nil(t, err) - err = b.RoaringSetRemoveOne(key3, removal3) - require.Nil(t, err) - - res, err := b.RoaringSetGet(key1) - require.Nil(t, err) - for _, testVal := range []uint64{1, 2} { // unchanged values - assert.True(t, res.Contains(testVal)) - } - - res, err = b.RoaringSetGet(key2) - require.Nil(t, err) - for _, testVal := range []uint64{3, 4, 5} { // extended with 5 - assert.True(t, res.Contains(testVal)) - } - - res, err = b.RoaringSetGet(key3) - require.Nil(t, err) - for _, testVal := range []uint64{6} { // fewer remain - assert.True(t, res.Contains(testVal)) - } - for _, testVal := range []uint64{5} { // no longer contained - assert.False(t, res.Contains(testVal)) - } - }) - }) -} diff --git a/adapters/repos/db/lsmkv/strategies_set.go b/adapters/repos/db/lsmkv/strategies_set.go deleted file mode 100644 index 9c4924c760427a3fb24faafb93bbf2c2f791663a..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/strategies_set.go +++ /dev/null @@ -1,143 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -type setDecoder struct{} - -func newSetDecoder() *setDecoder { - return &setDecoder{} -} - -func (s *setDecoder) Do(in []value) [][]byte { - // check if there are tombstones, if not, we can simply take the list without - // further processing - var tombstones int - for _, value := range in { - if value.tombstone { - tombstones++ - } - } - - if tombstones == 0 { - return s.doWithoutTombstones(in) - } - - // there are tombstones, we need to remove them - // TODO: The logic below can be improved since don't care about the "latest" - // write on a set, as all writes are per definition identical. Any write that - // is not followed by a tombstone is fine - count := make(map[string]uint, len(in)) - for _, value := range in { - count[string(value.value)] = count[string(value.value)] + 1 - } - out := make([][]byte, len(in)) - - i := 0 - for _, value := range in { - if count[string(value.value)] != 1 { - count[string(value.value)] = count[string(value.value)] - 1 - continue - } - - if value.tombstone { - continue - } - - out[i] = value.value - i++ - } - - return out[:i] -} - -func (s *setDecoder) doWithoutTombstones(in []value) [][]byte { - out := make([][]byte, len(in)) - for i := range in { - out[i] = in[i].value - } - - // take an arbitrary cutoff for when it is worth to remove duplicates. The - // assumption is that on larger lists, duplicates are more likely to be - // tolerated, for example, because the point is to build an allow list for a - // secondary index where a duplicate does not matter. If the amount is - // smaller than the cutoff this is more likely to be relevant to a user. - // - // As the list gets longer, removing duplicates gets a lot more expensive, - // hence it makes sense to skip the de-duplication, if we can be reasonably - // sure that it does not matter - if len(out) <= 1000 { - return s.deduplicateResults(out) - } - - return out -} - -func (s *setDecoder) deduplicateResults(in [][]byte) [][]byte { - out := make([][]byte, len(in)) - - seen := map[string]struct{}{} - - i := 0 - for _, elem := range in { - if _, ok := seen[string(elem)]; ok { - continue - } - - out[i] = elem - seen[string(elem)] = struct{}{} - i++ - } - - return out[:i] -} - -// DoPartial keeps any extra tombstones, but does not keep tombstones which -// were "consumed" -func (s *setDecoder) DoPartial(in []value) []value { - count := map[string]uint{} - for _, value := range in { - count[string(value.value)] = count[string(value.value)] + 1 - } - - out := make([]value, len(in)) - - i := 0 - for _, value := range in { - if count[string(value.value)] != 1 { - count[string(value.value)] = count[string(value.value)] - 1 - continue - } - - out[i] = value - i++ - } - - return out[:i] -} - -type setEncoder struct{} - -func newSetEncoder() *setEncoder { - return &setEncoder{} -} - -func (s *setEncoder) Do(in [][]byte) []value { - out := make([]value, len(in)) - for i, v := range in { - out[i] = value{ - tombstone: false, - value: v, - } - } - - return out -} diff --git a/adapters/repos/db/lsmkv/strategies_set_integration_test.go b/adapters/repos/db/lsmkv/strategies_set_integration_test.go deleted file mode 100644 index d26c62f8eac4bbf7d6f10b65efc0f4f7c757c4db..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/strategies_set_integration_test.go +++ /dev/null @@ -1,1082 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package lsmkv - -import ( - "context" - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/cyclemanager" -) - -func TestSetCollectionStrategy(t *testing.T) { - ctx := testCtx() - tests := bucketIntegrationTests{ - { - name: "collectionInsertAndSetAdd", - f: collectionInsertAndSetAdd, - opts: []BucketOption{ - WithStrategy(StrategySetCollection), - }, - }, - { - name: "collectionInsertAndSetAddInsertAndDelete", - f: collectionInsertAndSetAddInsertAndDelete, - opts: []BucketOption{ - WithStrategy(StrategySetCollection), - }, - }, - { - name: "collectionCursors", - f: collectionCursors, - opts: []BucketOption{ - WithStrategy(StrategySetCollection), - }, - }, - } - tests.run(ctx, t) -} - -func collectionInsertAndSetAdd(ctx context.Context, t *testing.T, opts []BucketOption) { - dirName := t.TempDir() - - t.Run("memtable-only", func(t *testing.T) { - b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - key1 := []byte("test1-key-1") - key2 := []byte("test1-key-2") - key3 := []byte("test1-key-3") - - t.Run("set original values and verify", func(t *testing.T) { - orig1 := [][]byte{[]byte("value 1.1"), []byte("value 1.2")} - orig2 := [][]byte{[]byte("value 2.1"), []byte("value 2.2")} - orig3 := [][]byte{[]byte("value 3.1"), []byte("value 3.2")} - - err = b.SetAdd(key1, orig1) - require.Nil(t, err) - err = b.SetAdd(key2, orig2) - require.Nil(t, err) - err = b.SetAdd(key3, orig3) - require.Nil(t, err) - - res, err := b.SetList(key1) - require.Nil(t, err) - assert.Equal(t, res, orig1) - res, err = b.SetList(key2) - require.Nil(t, err) - assert.Equal(t, res, orig2) - res, err = b.SetList(key3) - require.Nil(t, err) - assert.Equal(t, res, orig3) - }) - - t.Run("replace some, keep one", func(t *testing.T) { - orig1 := [][]byte{[]byte("value 1.1"), []byte("value 1.2")} - orig2 := [][]byte{[]byte("value 2.1"), []byte("value 2.2")} - orig3 := [][]byte{[]byte("value 3.1"), []byte("value 3.2")} - append2 := [][]byte{[]byte("value 2.3")} - append3 := [][]byte{[]byte("value 3.3")} - - err = b.SetAdd(key2, append2) - require.Nil(t, err) - err = b.SetAdd(key3, append3) - require.Nil(t, err) - - res, err := b.SetList(key1) - require.Nil(t, err) - assert.Equal(t, orig1, res) - res, err = b.SetList(key2) - require.Nil(t, err) - assert.Equal(t, append(orig2, append2...), res) - res, err = b.SetList(key3) - require.Nil(t, err) - assert.Equal(t, append(orig3, append3...), res) - }) - }) - - t.Run("with a single flush between updates", func(t *testing.T) { - b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - key1 := []byte("test2-key-1") - key2 := []byte("test2-key-2") - key3 := []byte("test2-key-3") - - t.Run("set original values and verify", func(t *testing.T) { - orig1 := [][]byte{[]byte("value 1.1"), []byte("value 1.2")} - orig2 := [][]byte{[]byte("value 2.1"), []byte("value 2.2")} - orig3 := [][]byte{[]byte("value 3.1"), []byte("value 3.2")} - - err = b.SetAdd(key1, orig1) - require.Nil(t, err) - err = b.SetAdd(key2, orig2) - require.Nil(t, err) - err = b.SetAdd(key3, orig3) - require.Nil(t, err) - - res, err := b.SetList(key1) - require.Nil(t, err) - assert.Equal(t, res, orig1) - res, err = b.SetList(key2) - require.Nil(t, err) - assert.Equal(t, res, orig2) - res, err = b.SetList(key3) - require.Nil(t, err) - assert.Equal(t, res, orig3) - }) - - t.Run("flush to disk", func(t *testing.T) { - require.Nil(t, b.FlushAndSwitch()) - }) - - t.Run("replace some, keep one", func(t *testing.T) { - orig1 := [][]byte{[]byte("value 1.1"), []byte("value 1.2")} - orig2 := [][]byte{[]byte("value 2.1"), []byte("value 2.2")} - orig3 := [][]byte{[]byte("value 3.1"), []byte("value 3.2")} - append2 := [][]byte{[]byte("value 2.3")} - append3 := [][]byte{[]byte("value 3.3")} - - err = b.SetAdd(key2, append2) - require.Nil(t, err) - err = b.SetAdd(key3, append3) - require.Nil(t, err) - - res, err := b.SetList(key1) - require.Nil(t, err) - assert.Equal(t, orig1, res) - res, err = b.SetList(key2) - require.Nil(t, err) - assert.Equal(t, append(orig2, append2...), res) - res, err = b.SetList(key3) - require.Nil(t, err) - assert.Equal(t, append(orig3, append3...), res) - }) - }) - - t.Run("with flushes after initial and update", func(t *testing.T) { - b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - key1 := []byte("test-3-key-1") - key2 := []byte("test-3-key-2") - key3 := []byte("test-3-key-3") - - t.Run("set original values and verify", func(t *testing.T) { - orig1 := [][]byte{[]byte("value 1.1"), []byte("value 1.2")} - orig2 := [][]byte{[]byte("value 2.1"), []byte("value 2.2")} - orig3 := [][]byte{[]byte("value 3.1"), []byte("value 3.2")} - - err = b.SetAdd(key1, orig1) - require.Nil(t, err) - err = b.SetAdd(key2, orig2) - require.Nil(t, err) - err = b.SetAdd(key3, orig3) - require.Nil(t, err) - - res, err := b.SetList(key1) - require.Nil(t, err) - assert.Equal(t, res, orig1) - res, err = b.SetList(key2) - require.Nil(t, err) - assert.Equal(t, res, orig2) - res, err = b.SetList(key3) - require.Nil(t, err) - assert.Equal(t, res, orig3) - }) - - t.Run("flush to disk", func(t *testing.T) { - require.Nil(t, b.FlushAndSwitch()) - }) - - t.Run("replace some, keep one", func(t *testing.T) { - orig1 := [][]byte{[]byte("value 1.1"), []byte("value 1.2")} - orig2 := [][]byte{[]byte("value 2.1"), []byte("value 2.2")} - orig3 := [][]byte{[]byte("value 3.1"), []byte("value 3.2")} - append2 := [][]byte{[]byte("value 2.3")} - append3 := [][]byte{[]byte("value 3.3")} - - err = b.SetAdd(key2, append2) - require.Nil(t, err) - err = b.SetAdd(key3, append3) - require.Nil(t, err) - - // Flush again! - require.Nil(t, b.FlushAndSwitch()) - - res, err := b.SetList(key1) - require.Nil(t, err) - assert.Equal(t, orig1, res) - res, err = b.SetList(key2) - require.Nil(t, err) - assert.Equal(t, append(orig2, append2...), res) - res, err = b.SetList(key3) - require.Nil(t, err) - assert.Equal(t, append(orig3, append3...), res) - }) - }) - - t.Run("update in memtable, then do an orderly shutdown, and re-init", func(t *testing.T) { - b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - key1 := []byte("test4-key-1") - key2 := []byte("test4-key-2") - key3 := []byte("test4-key-3") - - t.Run("set original values and verify", func(t *testing.T) { - orig1 := [][]byte{[]byte("value 1.1"), []byte("value 1.2")} - orig2 := [][]byte{[]byte("value 2.1"), []byte("value 2.2")} - orig3 := [][]byte{[]byte("value 3.1"), []byte("value 3.2")} - - err = b.SetAdd(key1, orig1) - require.Nil(t, err) - err = b.SetAdd(key2, orig2) - require.Nil(t, err) - err = b.SetAdd(key3, orig3) - require.Nil(t, err) - - res, err := b.SetList(key1) - require.Nil(t, err) - assert.Equal(t, res, orig1) - res, err = b.SetList(key2) - require.Nil(t, err) - assert.Equal(t, res, orig2) - res, err = b.SetList(key3) - require.Nil(t, err) - assert.Equal(t, res, orig3) - }) - - t.Run("replace some, keep one", func(t *testing.T) { - orig1 := [][]byte{[]byte("value 1.1"), []byte("value 1.2")} - orig2 := [][]byte{[]byte("value 2.1"), []byte("value 2.2")} - orig3 := [][]byte{[]byte("value 3.1"), []byte("value 3.2")} - append2 := [][]byte{[]byte("value 2.3")} - append3 := [][]byte{[]byte("value 3.3")} - - err = b.SetAdd(key2, append2) - require.Nil(t, err) - err = b.SetAdd(key3, append3) - require.Nil(t, err) - - res, err := b.SetList(key1) - require.Nil(t, err) - assert.Equal(t, orig1, res) - res, err = b.SetList(key2) - require.Nil(t, err) - assert.Equal(t, append(orig2, append2...), res) - res, err = b.SetList(key3) - require.Nil(t, err) - assert.Equal(t, append(orig3, append3...), res) - }) - - t.Run("orderly shutdown", func(t *testing.T) { - b.Shutdown(context.Background()) - }) - - t.Run("init another bucket on the same files", func(t *testing.T) { - b2, err := NewBucket(ctx, dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - orig1 := [][]byte{[]byte("value 1.1"), []byte("value 1.2")} - orig2 := [][]byte{[]byte("value 2.1"), []byte("value 2.2")} - orig3 := [][]byte{[]byte("value 3.1"), []byte("value 3.2")} - append2 := [][]byte{[]byte("value 2.3")} - append3 := [][]byte{[]byte("value 3.3")} - - res, err := b2.SetList(key1) - require.Nil(t, err) - assert.Equal(t, orig1, res) - res, err = b2.SetList(key2) - require.Nil(t, err) - assert.Equal(t, append(orig2, append2...), res) - res, err = b2.SetList(key3) - require.Nil(t, err) - assert.Equal(t, append(orig3, append3...), res) - }) - }) -} - -func collectionInsertAndSetAddInsertAndDelete(ctx context.Context, t *testing.T, opts []BucketOption) { - dirName := t.TempDir() - - t.Run("memtable-only", func(t *testing.T) { - b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - key1 := []byte("test1-key-1") - key2 := []byte("test1-key-2") - key3 := []byte("test1-key-3") - - t.Run("set original values and verify", func(t *testing.T) { - orig1 := [][]byte{[]byte("value 1.1"), []byte("value 1.2")} - orig2 := [][]byte{[]byte("value 2.1"), []byte("value 2.2")} - orig3 := [][]byte{[]byte("value 3.1"), []byte("value 3.2")} - - err = b.SetAdd(key1, orig1) - require.Nil(t, err) - err = b.SetAdd(key2, orig2) - require.Nil(t, err) - err = b.SetAdd(key3, orig3) - require.Nil(t, err) - - res, err := b.SetList(key1) - require.Nil(t, err) - assert.Equal(t, orig1, res) - res, err = b.SetList(key2) - require.Nil(t, err) - assert.Equal(t, orig2, res) - res, err = b.SetList(key3) - require.Nil(t, err) - assert.Equal(t, orig3, res) - }) - - t.Run("delete individual keys", func(t *testing.T) { - delete2 := []byte("value 2.1") - delete3 := []byte("value 3.2") - - err = b.SetDeleteSingle(key2, delete2) - require.Nil(t, err) - err = b.SetDeleteSingle(key3, delete3) - require.Nil(t, err) - }) - - t.Run("validate the results", func(t *testing.T) { - expected1 := [][]byte{[]byte("value 1.1"), []byte("value 1.2")} // unchanged - expected2 := [][]byte{[]byte("value 2.2")} // value1 deleted - expected3 := [][]byte{[]byte("value 3.1")} // value2 deleted - - res, err := b.SetList(key1) - require.Nil(t, err) - assert.Equal(t, expected1, res) - res, err = b.SetList(key2) - require.Nil(t, err) - assert.Equal(t, expected2, res) - res, err = b.SetList(key3) - require.Nil(t, err) - assert.Equal(t, expected3, res) - }) - - t.Run("re-add keys which were previously deleted and new ones", func(t *testing.T) { - readd2 := [][]byte{[]byte("value 2.1"), []byte("value 2.3")} - readd3 := [][]byte{[]byte("value 3.2"), []byte("value 3.3")} - - err = b.SetAdd(key2, readd2) - require.Nil(t, err) - err = b.SetAdd(key3, readd3) - require.Nil(t, err) - }) - - t.Run("validate the results again", func(t *testing.T) { - expected1 := [][]byte{[]byte("value 1.1"), []byte("value 1.2")} // unchanged - expected2 := [][]byte{ - []byte("value 2.2"), // from original import - []byte("value 2.1"), // added again after initial deletion - []byte("value 2.3"), // newly added - } - expected3 := [][]byte{ - []byte("value 3.1"), // form original import - []byte("value 3.2"), // added again after initial deletion - []byte("value 3.3"), // newly added - } // value2 deleted - - res, err := b.SetList(key1) - require.Nil(t, err) - assert.Equal(t, expected1, res) - res, err = b.SetList(key2) - require.Nil(t, err) - assert.Equal(t, expected2, res) - res, err = b.SetList(key3) - require.Nil(t, err) - assert.Equal(t, expected3, res) - }) - }) - - t.Run("with a single flush between updates", func(t *testing.T) { - b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - key1 := []byte("test2-key-1") - key2 := []byte("test2-key-2") - key3 := []byte("test2-key-3") - - t.Run("set original values and verify", func(t *testing.T) { - orig1 := [][]byte{[]byte("value 1.1"), []byte("value 1.2")} - orig2 := [][]byte{[]byte("value 2.1"), []byte("value 2.2")} - orig3 := [][]byte{[]byte("value 3.1"), []byte("value 3.2")} - - err = b.SetAdd(key1, orig1) - require.Nil(t, err) - err = b.SetAdd(key2, orig2) - require.Nil(t, err) - err = b.SetAdd(key3, orig3) - require.Nil(t, err) - - res, err := b.SetList(key1) - require.Nil(t, err) - assert.Equal(t, res, orig1) - res, err = b.SetList(key2) - require.Nil(t, err) - assert.Equal(t, res, orig2) - res, err = b.SetList(key3) - require.Nil(t, err) - assert.Equal(t, res, orig3) - }) - - t.Run("flush to disk", func(t *testing.T) { - require.Nil(t, b.FlushAndSwitch()) - }) - - t.Run("delete individual keys", func(t *testing.T) { - delete2 := []byte("value 2.1") - delete3 := []byte("value 3.2") - - err = b.SetDeleteSingle(key2, delete2) - require.Nil(t, err) - err = b.SetDeleteSingle(key3, delete3) - require.Nil(t, err) - }) - - t.Run("validate the results", func(t *testing.T) { - expected1 := [][]byte{[]byte("value 1.1"), []byte("value 1.2")} // unchanged - expected2 := [][]byte{[]byte("value 2.2")} // value1 deleted - expected3 := [][]byte{[]byte("value 3.1")} // value2 deleted - - res, err := b.SetList(key1) - require.Nil(t, err) - assert.Equal(t, expected1, res) - res, err = b.SetList(key2) - require.Nil(t, err) - assert.Equal(t, expected2, res) - res, err = b.SetList(key3) - require.Nil(t, err) - assert.Equal(t, expected3, res) - }) - - t.Run("re-add keys which were previously deleted and new ones", func(t *testing.T) { - readd2 := [][]byte{[]byte("value 2.1"), []byte("value 2.3")} - readd3 := [][]byte{[]byte("value 3.2"), []byte("value 3.3")} - - err = b.SetAdd(key2, readd2) - require.Nil(t, err) - err = b.SetAdd(key3, readd3) - require.Nil(t, err) - }) - - t.Run("validate the results again", func(t *testing.T) { - expected1 := [][]byte{[]byte("value 1.1"), []byte("value 1.2")} // unchanged - expected2 := [][]byte{ - []byte("value 2.2"), // from original import - []byte("value 2.1"), // added again after initial deletion - []byte("value 2.3"), // newly added - } - expected3 := [][]byte{ - []byte("value 3.1"), // form original import - []byte("value 3.2"), // added again after initial deletion - []byte("value 3.3"), // newly added - } // value2 deleted - - res, err := b.SetList(key1) - require.Nil(t, err) - assert.Equal(t, expected1, res) - res, err = b.SetList(key2) - require.Nil(t, err) - assert.Equal(t, expected2, res) - res, err = b.SetList(key3) - require.Nil(t, err) - assert.Equal(t, expected3, res) - }) - }) - - t.Run("with flushes in between and after the update", func(t *testing.T) { - b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - key1 := []byte("test3-key-1") - key2 := []byte("test3-key-2") - key3 := []byte("test3-key-3") - - t.Run("set original values and verify", func(t *testing.T) { - orig1 := [][]byte{[]byte("value 1.1"), []byte("value 1.2")} - orig2 := [][]byte{[]byte("value 2.1"), []byte("value 2.2")} - orig3 := [][]byte{[]byte("value 3.1"), []byte("value 3.2")} - - err = b.SetAdd(key1, orig1) - require.Nil(t, err) - err = b.SetAdd(key2, orig2) - require.Nil(t, err) - err = b.SetAdd(key3, orig3) - require.Nil(t, err) - - res, err := b.SetList(key1) - require.Nil(t, err) - assert.Equal(t, res, orig1) - res, err = b.SetList(key2) - require.Nil(t, err) - assert.Equal(t, res, orig2) - res, err = b.SetList(key3) - require.Nil(t, err) - assert.Equal(t, res, orig3) - }) - - t.Run("flush to disk", func(t *testing.T) { - require.Nil(t, b.FlushAndSwitch()) - }) - - t.Run("delete individual keys", func(t *testing.T) { - delete2 := []byte("value 2.1") - delete3 := []byte("value 3.2") - - err = b.SetDeleteSingle(key2, delete2) - require.Nil(t, err) - err = b.SetDeleteSingle(key3, delete3) - require.Nil(t, err) - }) - - t.Run("flush to disk - again!", func(t *testing.T) { - require.Nil(t, b.FlushAndSwitch()) - }) - - t.Run("validate", func(t *testing.T) { - expected1 := [][]byte{[]byte("value 1.1"), []byte("value 1.2")} // unchanged - expected2 := [][]byte{[]byte("value 2.2")} // value1 deleted - expected3 := [][]byte{[]byte("value 3.1")} // value2 deleted - - res, err := b.SetList(key1) - require.Nil(t, err) - assert.Equal(t, expected1, res) - res, err = b.SetList(key2) - require.Nil(t, err) - assert.Equal(t, expected2, res) - res, err = b.SetList(key3) - require.Nil(t, err) - assert.Equal(t, expected3, res) - }) - - t.Run("re-add keys which were previously deleted and new ones", func(t *testing.T) { - readd2 := [][]byte{[]byte("value 2.1"), []byte("value 2.3")} - readd3 := [][]byte{[]byte("value 3.2"), []byte("value 3.3")} - - err = b.SetAdd(key2, readd2) - require.Nil(t, err) - err = b.SetAdd(key3, readd3) - require.Nil(t, err) - }) - - t.Run("flush to disk - yet again!", func(t *testing.T) { - require.Nil(t, b.FlushAndSwitch()) - }) - - t.Run("validate the results again", func(t *testing.T) { - expected1 := [][]byte{[]byte("value 1.1"), []byte("value 1.2")} // unchanged - expected2 := [][]byte{ - []byte("value 2.2"), // from original import - []byte("value 2.1"), // added again after initial deletion - []byte("value 2.3"), // newly added - } - expected3 := [][]byte{ - []byte("value 3.1"), // form original import - []byte("value 3.2"), // added again after initial deletion - []byte("value 3.3"), // newly added - } // value2 deleted - - res, err := b.SetList(key1) - require.Nil(t, err) - assert.Equal(t, expected1, res) - res, err = b.SetList(key2) - require.Nil(t, err) - assert.Equal(t, expected2, res) - res, err = b.SetList(key3) - require.Nil(t, err) - assert.Equal(t, expected3, res) - }) - }) - - t.Run("update in memtable, make orderly shutdown, then create a new bucket from disk", - func(t *testing.T) { - b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - key1 := []byte("test4-key-1") - key2 := []byte("test4-key-2") - key3 := []byte("test4-key-3") - - t.Run("set original values and verify", func(t *testing.T) { - orig1 := [][]byte{[]byte("value 1.1"), []byte("value 1.2")} - orig2 := [][]byte{[]byte("value 2.1"), []byte("value 2.2")} - orig3 := [][]byte{[]byte("value 3.1"), []byte("value 3.2")} - - err = b.SetAdd(key1, orig1) - require.Nil(t, err) - err = b.SetAdd(key2, orig2) - require.Nil(t, err) - err = b.SetAdd(key3, orig3) - require.Nil(t, err) - - res, err := b.SetList(key1) - require.Nil(t, err) - assert.Equal(t, res, orig1) - res, err = b.SetList(key2) - require.Nil(t, err) - assert.Equal(t, res, orig2) - res, err = b.SetList(key3) - require.Nil(t, err) - assert.Equal(t, res, orig3) - }) - - t.Run("delete individual keys", func(t *testing.T) { - delete2 := []byte("value 2.1") - delete3 := []byte("value 3.2") - - err = b.SetDeleteSingle(key2, delete2) - require.Nil(t, err) - err = b.SetDeleteSingle(key3, delete3) - require.Nil(t, err) - }) - - t.Run("orderly shutdown", func(t *testing.T) { - b.Shutdown(context.Background()) - }) - - t.Run("init another bucket on the same files", func(t *testing.T) { - b2, err := NewBucket(ctx, dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - expected1 := [][]byte{[]byte("value 1.1"), []byte("value 1.2")} // unchanged - expected2 := [][]byte{[]byte("value 2.2")} // value1 deleted - expected3 := [][]byte{[]byte("value 3.1")} // value2 deleted - - res, err := b2.SetList(key1) - require.Nil(t, err) - assert.Equal(t, expected1, res) - res, err = b2.SetList(key2) - require.Nil(t, err) - assert.Equal(t, expected2, res) - res, err = b2.SetList(key3) - require.Nil(t, err) - assert.Equal(t, expected3, res) - }) - }) -} - -func collectionCursors(ctx context.Context, t *testing.T, opts []BucketOption) { - t.Run("memtable-only", func(t *testing.T) { - r := getRandomSeed() - dirName := t.TempDir() - - b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - t.Run("set original values", func(t *testing.T) { - pairs := 20 - valuesPerPair := 3 - keys := make([][]byte, pairs) - values := make([][][]byte, pairs) - - for i := range keys { - keys[i] = []byte(fmt.Sprintf("key-%03d", i)) - values[i] = make([][]byte, valuesPerPair) - for j := range values[i] { - values[i][j] = []byte(fmt.Sprintf("value-%03d.%d", i, j)) - } - } - - // shuffle to make sure the BST isn't accidentally in order - r.Shuffle(len(keys), func(i, j int) { - keys[i], keys[j] = keys[j], keys[i] - values[i], values[j] = values[j], values[i] - }) - - for i := range keys { - err = b.SetAdd(keys[i], values[i]) - require.Nil(t, err) - } - }) - - t.Run("seek from somewhere in the middle", func(t *testing.T) { - expectedKeys := [][]byte{ - []byte("key-016"), - []byte("key-017"), - []byte("key-018"), - []byte("key-019"), - } - expectedValues := [][][]byte{ - {[]byte("value-016.0"), []byte("value-016.1"), []byte("value-016.2")}, - {[]byte("value-017.0"), []byte("value-017.1"), []byte("value-017.2")}, - {[]byte("value-018.0"), []byte("value-018.1"), []byte("value-018.2")}, - {[]byte("value-019.0"), []byte("value-019.1"), []byte("value-019.2")}, - } - - var retrievedKeys [][]byte - var retrievedValues [][][]byte - c := b.SetCursor() - defer c.Close() - for k, v := c.Seek([]byte("key-016")); k != nil; k, v = c.Next() { - retrievedKeys = append(retrievedKeys, k) - retrievedValues = append(retrievedValues, v) - } - - assert.Equal(t, expectedKeys, retrievedKeys) - assert.Equal(t, expectedValues, retrievedValues) - }) - - t.Run("start from the beginning", func(t *testing.T) { - expectedKeys := [][]byte{ - []byte("key-000"), - []byte("key-001"), - []byte("key-002"), - } - expectedValues := [][][]byte{ - {[]byte("value-000.0"), []byte("value-000.1"), []byte("value-000.2")}, - {[]byte("value-001.0"), []byte("value-001.1"), []byte("value-001.2")}, - {[]byte("value-002.0"), []byte("value-002.1"), []byte("value-002.2")}, - } - - var retrievedKeys [][]byte - var retrievedValues [][][]byte - c := b.SetCursor() - defer c.Close() - retrieved := 0 - for k, v := c.First(); k != nil && retrieved < 3; k, v = c.Next() { - retrieved++ - retrievedKeys = append(retrievedKeys, k) - retrievedValues = append(retrievedValues, v) - } - - assert.Equal(t, expectedKeys, retrievedKeys) - assert.Equal(t, expectedValues, retrievedValues) - }) - - t.Run("extend an existing key", func(t *testing.T) { - key := []byte("key-002") - extend := [][]byte{[]byte("value-002.3")} - - require.Nil(t, b.SetAdd(key, extend)) - }) - - t.Run("verify the extension is contained", func(t *testing.T) { - expectedKeys := [][]byte{ - []byte("key-001"), - []byte("key-002"), - } - expectedValues := [][][]byte{ - {[]byte("value-001.0"), []byte("value-001.1"), []byte("value-001.2")}, - { - []byte("value-002.0"), []byte("value-002.1"), []byte("value-002.2"), - []byte("value-002.3"), - }, - } - - var retrievedKeys [][]byte - var retrievedValues [][][]byte - c := b.SetCursor() - defer c.Close() - retrieved := 0 - for k, v := c.Seek([]byte("key-001")); k != nil && retrieved < 2; k, v = c.Next() { - retrieved++ - retrievedKeys = append(retrievedKeys, k) - retrievedValues = append(retrievedValues, v) - } - - assert.Equal(t, expectedKeys, retrievedKeys) - assert.Equal(t, expectedValues, retrievedValues) - }) - }) - - t.Run("with flushes", func(t *testing.T) { - r := getRandomSeed() - dirName := t.TempDir() - - b, err := NewBucket(ctx, dirName, "", nullLogger(), nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), opts...) - require.Nil(t, err) - - // so big it effectively never triggers as part of this test - b.SetMemtableThreshold(1e9) - - t.Run("first third (%3==0)", func(t *testing.T) { - pairs := 20 - valuesPerPair := 3 - var keys [][]byte - var values [][][]byte - - for i := 0; i < pairs; i++ { - if i%3 != 0 { - continue - } - keys = append(keys, []byte(fmt.Sprintf("key-%03d", i))) - curValues := make([][]byte, valuesPerPair) - for j := range curValues { - curValues[j] = []byte(fmt.Sprintf("value-%03d.%d", i, j)) - } - values = append(values, curValues) - } - - // shuffle to make sure the BST isn't accidentally in order - r.Shuffle(len(keys), func(i, j int) { - keys[i], keys[j] = keys[j], keys[i] - values[i], values[j] = values[j], values[i] - }) - - for i := range keys { - err = b.SetAdd(keys[i], values[i]) - require.Nil(t, err) - } - }) - - t.Run("flush to disk", func(t *testing.T) { - require.Nil(t, b.FlushAndSwitch()) - }) - - t.Run("second third (%3==1)", func(t *testing.T) { - pairs := 20 - valuesPerPair := 3 - var keys [][]byte - var values [][][]byte - - for i := 0; i < pairs; i++ { - if i%3 != 1 { - continue - } - keys = append(keys, []byte(fmt.Sprintf("key-%03d", i))) - curValues := make([][]byte, valuesPerPair) - for j := range curValues { - curValues[j] = []byte(fmt.Sprintf("value-%03d.%d", i, j)) - } - values = append(values, curValues) - } - - // shuffle to make sure the BST isn't accidentally in order - r.Shuffle(len(keys), func(i, j int) { - keys[i], keys[j] = keys[j], keys[i] - values[i], values[j] = values[j], values[i] - }) - - for i := range keys { - err = b.SetAdd(keys[i], values[i]) - require.Nil(t, err) - } - }) - - t.Run("flush to disk", func(t *testing.T) { - require.Nil(t, b.FlushAndSwitch()) - }) - - t.Run("third (%3==2) memtable-only", func(t *testing.T) { - pairs := 20 - valuesPerPair := 3 - var keys [][]byte - var values [][][]byte - - for i := 0; i < pairs; i++ { - if i%3 != 2 { - continue - } - keys = append(keys, []byte(fmt.Sprintf("key-%03d", i))) - curValues := make([][]byte, valuesPerPair) - for j := range curValues { - curValues[j] = []byte(fmt.Sprintf("value-%03d.%d", i, j)) - } - values = append(values, curValues) - } - - // shuffle to make sure the BST isn't accidentally in order - r.Shuffle(len(keys), func(i, j int) { - keys[i], keys[j] = keys[j], keys[i] - values[i], values[j] = values[j], values[i] - }) - - for i := range keys { - err = b.SetAdd(keys[i], values[i]) - require.Nil(t, err) - } - - // no flush for this one, so this segment stays in the memtable - }) - - t.Run("seek from somewhere in the middle", func(t *testing.T) { - expectedKeys := [][]byte{ - []byte("key-016"), - []byte("key-017"), - []byte("key-018"), - []byte("key-019"), - } - expectedValues := [][][]byte{ - {[]byte("value-016.0"), []byte("value-016.1"), []byte("value-016.2")}, - {[]byte("value-017.0"), []byte("value-017.1"), []byte("value-017.2")}, - {[]byte("value-018.0"), []byte("value-018.1"), []byte("value-018.2")}, - {[]byte("value-019.0"), []byte("value-019.1"), []byte("value-019.2")}, - } - - var retrievedKeys [][]byte - var retrievedValues [][][]byte - c := b.SetCursor() - defer c.Close() - for k, v := c.Seek([]byte("key-016")); k != nil; k, v = c.Next() { - retrievedKeys = append(retrievedKeys, k) - retrievedValues = append(retrievedValues, v) - } - - assert.Equal(t, expectedKeys, retrievedKeys) - assert.Equal(t, expectedValues, retrievedValues) - }) - - t.Run("start from the beginning", func(t *testing.T) { - expectedKeys := [][]byte{ - []byte("key-000"), - []byte("key-001"), - []byte("key-002"), - } - expectedValues := [][][]byte{ - {[]byte("value-000.0"), []byte("value-000.1"), []byte("value-000.2")}, - {[]byte("value-001.0"), []byte("value-001.1"), []byte("value-001.2")}, - {[]byte("value-002.0"), []byte("value-002.1"), []byte("value-002.2")}, - } - - var retrievedKeys [][]byte - var retrievedValues [][][]byte - c := b.SetCursor() - defer c.Close() - retrieved := 0 - for k, v := c.First(); k != nil && retrieved < 3; k, v = c.Next() { - retrieved++ - retrievedKeys = append(retrievedKeys, k) - retrievedValues = append(retrievedValues, v) - } - - assert.Equal(t, expectedKeys, retrievedKeys) - assert.Equal(t, expectedValues, retrievedValues) - }) - - t.Run("delete & extend an existing key", func(t *testing.T) { - key := []byte("key-002") - extend := [][]byte{[]byte("value-002.3")} - - require.Nil(t, b.SetAdd(key, extend)) - - key = []byte("key-001") - deleteValue := []byte("value-001.1") - require.Nil(t, b.SetDeleteSingle(key, deleteValue)) - }) - - t.Run("verify the extension is contained", func(t *testing.T) { - expectedKeys := [][]byte{ - []byte("key-001"), - []byte("key-002"), - } - expectedValues := [][][]byte{ - { - []byte("value-001.0"), - // "value-001.1" deleted - []byte("value-001.2"), - }, - { - []byte("value-002.0"), []byte("value-002.1"), []byte("value-002.2"), - []byte("value-002.3"), - }, - } - - var retrievedKeys [][]byte - var retrievedValues [][][]byte - c := b.SetCursor() - defer c.Close() - retrieved := 0 - for k, v := c.Seek([]byte("key-001")); k != nil && retrieved < 2; k, v = c.Next() { - retrieved++ - retrievedKeys = append(retrievedKeys, k) - retrievedValues = append(retrievedValues, v) - } - - assert.Equal(t, expectedKeys, retrievedKeys) - assert.Equal(t, expectedValues, retrievedValues) - }) - - t.Run("flush to disk", func(t *testing.T) { - require.Nil(t, b.FlushAndSwitch()) - }) - - t.Run("verify again after flush", func(t *testing.T) { - expectedKeys := [][]byte{ - []byte("key-001"), - []byte("key-002"), - } - expectedValues := [][][]byte{ - { - []byte("value-001.0"), - // "value-001.1" deleted - []byte("value-001.2"), - }, - { - []byte("value-002.0"), []byte("value-002.1"), []byte("value-002.2"), - []byte("value-002.3"), - }, - } - - var retrievedKeys [][]byte - var retrievedValues [][][]byte - c := b.SetCursor() - defer c.Close() - retrieved := 0 - for k, v := c.Seek([]byte("key-001")); k != nil && retrieved < 2; k, v = c.Next() { - retrieved++ - retrievedKeys = append(retrievedKeys, k) - retrievedValues = append(retrievedValues, v) - } - - assert.Equal(t, expectedKeys, retrievedKeys) - assert.Equal(t, expectedValues, retrievedValues) - }) - }) -} diff --git a/adapters/repos/db/lsmkv/strategies_set_test.go b/adapters/repos/db/lsmkv/strategies_set_test.go deleted file mode 100644 index dd119d989bb079cb37f5aafcf83a0c45059dc858..0000000000000000000000000000000000000000 --- a/adapters/repos/db/lsmkv/strategies_set_test.go +++ /dev/null @@ -1,180 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestSetDecoder(t *testing.T) { - type test struct { - name string - in []value - out [][]byte - } - - tests := []test{ - { - name: "single value", - in: []value{ - { - value: []byte("foo"), - }, - }, - out: [][]byte{ - []byte("foo"), - }, - }, - - { - name: "single value with tombstone", - in: []value{ - { - value: []byte("foo"), - tombstone: true, - }, - }, - out: [][]byte{}, - }, - { - name: "single value, then a tombstone added", - in: []value{ - { - value: []byte("foo"), - }, - { - value: []byte("foo"), - tombstone: true, - }, - }, - out: [][]byte{}, - }, - { - name: "single value, then a tombstone added, then added again", - in: []value{ - { - value: []byte("foo"), - }, - { - value: []byte("foo"), - tombstone: true, - }, - { - value: []byte("foo"), - }, - }, - out: [][]byte{ - []byte("foo"), - }, - }, - { - name: "one value, repeating", - in: []value{ - { - value: []byte("foo"), - }, - { - value: []byte("foo"), - }, - }, - out: [][]byte{ - []byte("foo"), - }, - }, - { - name: "multiple values, some tombstones, ending in everything present", - in: []value{ - { - value: []byte("foo"), - }, - { - value: []byte("bar"), - }, - { - value: []byte("foo"), - tombstone: true, - }, - { - value: []byte("foo"), - tombstone: true, - }, - { - value: []byte("foo"), - tombstone: true, - }, - { - value: []byte("foo"), - }, - { - value: []byte("bar"), - }, - { - value: []byte("bar"), - tombstone: true, - }, - { - value: []byte("bar"), - }, - }, - out: [][]byte{ - []byte("foo"), - []byte("bar"), - }, - }, - { - name: "multiple values, some tombstones, ending in everything deleted", - in: []value{ - { - value: []byte("foo"), - }, - { - value: []byte("bar"), - }, - { - value: []byte("foo"), - tombstone: true, - }, - { - value: []byte("foo"), - tombstone: true, - }, - { - value: []byte("foo"), - tombstone: true, - }, - { - value: []byte("foo"), - }, - { - value: []byte("bar"), - }, - { - value: []byte("bar"), - tombstone: true, - }, - { - value: []byte("foo"), - tombstone: true, - }, - }, - out: [][]byte{}, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - assert.Equal(t, test.out, newSetDecoder().Do(test.in)) - }) - } -} diff --git a/adapters/repos/db/merge_integration_test.go b/adapters/repos/db/merge_integration_test.go deleted file mode 100644 index d757e5901b94c109b6884513da5bd09cc91e6d73..0000000000000000000000000000000000000000 --- a/adapters/repos/db/merge_integration_test.go +++ /dev/null @@ -1,933 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package db - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/go-openapi/strfmt" - "github.com/google/uuid" - "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/dto" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/schema/crossref" - enthnsw "github.com/weaviate/weaviate/entities/vectorindex/hnsw" - "github.com/weaviate/weaviate/usecases/objects" -) - -func Test_MergingObjects(t *testing.T) { - dirName := t.TempDir() - - logger := logrus.New() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - MaxImportGoroutinesFactor: 1, - TrackVectorDimensions: true, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - defer repo.Shutdown(context.Background()) - migrator := NewMigrator(repo, logger) - - sch := schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{ - { - Class: "MergeTestTarget", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - }, - { - Class: "MergeTestSource", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{ // tries to have "one of each property type" - { - Name: "string", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "text", - DataType: []string{"text"}, - }, - { - Name: "number", - DataType: []string{"number"}, - }, - { - Name: "int", - DataType: []string{"int"}, - }, - { - Name: "date", - DataType: []string{"date"}, - }, - { - Name: "geo", - DataType: []string{"geoCoordinates"}, - }, - { - Name: "toTarget", - DataType: []string{"MergeTestTarget"}, - }, - }, - }, - { - Class: "MergeTestNoVector", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{ - { - Name: "foo", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - }, - }, - }, - } - - t.Run("add required classes", func(t *testing.T) { - for _, class := range sch.Objects.Classes { - t.Run(fmt.Sprintf("add %s", class.Class), func(t *testing.T) { - err := migrator.AddClass(context.Background(), class, schemaGetter.shardState) - require.Nil(t, err) - }) - } - }) - - schemaGetter.schema = sch - - target1 := strfmt.UUID("897be7cc-1ae1-4b40-89d9-d3ea98037638") - target2 := strfmt.UUID("5cc94aba-93e4-408a-ab19-3d803216a04e") - target3 := strfmt.UUID("81982705-8b1e-4228-b84c-911818d7ee85") - target4 := strfmt.UUID("7f69c263-17f4-4529-a54d-891a7c008ca4") - sourceID := strfmt.UUID("8738ddd5-a0ed-408d-a5d6-6f818fd56be6") - noVecID := strfmt.UUID("b4933761-88b2-4666-856d-298eb1ad0a59") - - t.Run("add objects", func(t *testing.T) { - now := time.Now().UnixNano() / int64(time.Millisecond) - err := repo.PutObject(context.Background(), &models.Object{ - ID: sourceID, - Class: "MergeTestSource", - Properties: map[string]interface{}{ - "string": "only the string prop set", - }, - CreationTimeUnix: now, - LastUpdateTimeUnix: now, - }, []float32{0.5}, nil) - require.Nil(t, err) - - targetDimensionsBefore := GetDimensionsFromRepo(repo, "MergeTestTarget") - - targets := []strfmt.UUID{target1, target2, target3, target4} - - for i, target := range targets { - err = repo.PutObject(context.Background(), &models.Object{ - ID: target, - Class: "MergeTestTarget", - Properties: map[string]interface{}{ - "name": fmt.Sprintf("target item %d", i), - }, - }, []float32{0.5}, nil) - require.Nil(t, err) - } - - targetDimensionsAfter := GetDimensionsFromRepo(repo, "MergeTestTarget") - require.Equal(t, targetDimensionsBefore+4, targetDimensionsAfter) - - err = repo.PutObject(context.Background(), &models.Object{ - ID: noVecID, - Class: "MergeTestNoVector", - Properties: map[string]interface{}{ - "foo": "bar", - }, - CreationTimeUnix: now, - LastUpdateTimeUnix: now, - }, nil, nil) - require.Nil(t, err) - - targetDimensionsAfterNoVec := GetDimensionsFromRepo(repo, "MergeTestTarget") - require.Equal(t, targetDimensionsAfter, targetDimensionsAfterNoVec) - }) - - var lastUpdateTimeUnix int64 - - t.Run("fetch original object's update timestamp", func(t *testing.T) { - source, err := repo.ObjectByID(context.Background(), sourceID, nil, additional.Properties{ - LastUpdateTimeUnix: true, - }, "") - require.Nil(t, err) - - lastUpdateTimeUnix = source.Object().LastUpdateTimeUnix - require.NotEmpty(t, lastUpdateTimeUnix) - }) - - t.Run("merge other previously unset properties into it", func(t *testing.T) { - // give the lastUpdateTimeUnix time to be different. - // on some machines this may not be needed, but for - // faster processors, the difference is undetectable - time.Sleep(time.Millisecond) - - md := objects.MergeDocument{ - Class: "MergeTestSource", - ID: sourceID, - PrimitiveSchema: map[string]interface{}{ - "number": 7.0, - "int": int64(9), - "geo": &models.GeoCoordinates{ - Latitude: ptFloat32(30.2), - Longitude: ptFloat32(60.2), - }, - "text": "some text", - }, - UpdateTime: time.Now().UnixNano() / int64(time.Millisecond), - } - - err := repo.Merge(context.Background(), md, nil, "") - assert.Nil(t, err) - }) - - t.Run("compare merge object's update time with original", func(t *testing.T) { - source, err := repo.ObjectByID(context.Background(), sourceID, nil, additional.Properties{ - LastUpdateTimeUnix: true, - }, "") - require.Nil(t, err) - - assert.Greater(t, source.Object().LastUpdateTimeUnix, lastUpdateTimeUnix) - }) - - t.Run("check that the object was successfully merged", func(t *testing.T) { - source, err := repo.ObjectByID(context.Background(), sourceID, nil, additional.Properties{}, "") - require.Nil(t, err) - - sch := source.Object().Properties.(map[string]interface{}) - expectedSchema := map[string]interface{}{ - // from original - "string": "only the string prop set", - - // from merge - "number": 7.0, - "int": float64(9), - "geo": &models.GeoCoordinates{ - Latitude: ptFloat32(30.2), - Longitude: ptFloat32(60.2), - }, - "text": "some text", - } - - assert.Equal(t, expectedSchema, sch) - }) - - t.Run("trying to merge from non-existing index", func(t *testing.T) { - md := objects.MergeDocument{ - Class: "WrongClass", - ID: sourceID, - PrimitiveSchema: map[string]interface{}{ - "number": 7.0, - }, - } - - err := repo.Merge(context.Background(), md, nil, "") - assert.Equal(t, fmt.Errorf( - "merge from non-existing index for WrongClass"), err) - }) - t.Run("add a reference and replace one prop", func(t *testing.T) { - source, err := crossref.ParseSource(fmt.Sprintf( - "weaviate://localhost/MergeTestSource/%s/toTarget", sourceID)) - require.Nil(t, err) - targets := []strfmt.UUID{target1} - refs := make(objects.BatchReferences, len(targets)) - for i, target := range targets { - to, err := crossref.Parse(fmt.Sprintf("weaviate://localhost/%s", target)) - require.Nil(t, err) - refs[i] = objects.BatchReference{ - Err: nil, - From: source, - To: to, - } - } - md := objects.MergeDocument{ - Class: "MergeTestSource", - ID: sourceID, - PrimitiveSchema: map[string]interface{}{ - "string": "let's update the string prop", - }, - References: refs, - } - err = repo.Merge(context.Background(), md, nil, "") - assert.Nil(t, err) - }) - - t.Run("check that the object was successfully merged", func(t *testing.T) { - source, err := repo.ObjectByID(context.Background(), sourceID, nil, additional.Properties{}, "") - require.Nil(t, err) - - ref, err := crossref.Parse(fmt.Sprintf("weaviate://localhost/%s", target1)) - require.Nil(t, err) - - sch := source.Object().Properties.(map[string]interface{}) - expectedSchema := map[string]interface{}{ - "string": "let's update the string prop", - "number": 7.0, - "int": float64(9), - "geo": &models.GeoCoordinates{ - Latitude: ptFloat32(30.2), - Longitude: ptFloat32(60.2), - }, - "text": "some text", - "toTarget": models.MultipleRef{ - ref.SingleRef(), - }, - } - - assert.Equal(t, expectedSchema, sch) - }) - - t.Run("add more references in rapid succession", func(t *testing.T) { - // this test case prevents a regression on gh-1016 - source, err := crossref.ParseSource(fmt.Sprintf( - "weaviate://localhost/MergeTestSource/%s/toTarget", sourceID)) - require.Nil(t, err) - targets := []strfmt.UUID{target2, target3, target4} - refs := make(objects.BatchReferences, len(targets)) - for i, target := range targets { - to, err := crossref.Parse(fmt.Sprintf("weaviate://localhost/%s", target)) - require.Nil(t, err) - refs[i] = objects.BatchReference{ - Err: nil, - From: source, - To: to, - } - } - md := objects.MergeDocument{ - Class: "MergeTestSource", - ID: sourceID, - References: refs, - } - err = repo.Merge(context.Background(), md, nil, "") - assert.Nil(t, err) - }) - - t.Run("check all references are now present", func(t *testing.T) { - source, err := repo.ObjectByID(context.Background(), sourceID, nil, additional.Properties{}, "") - require.Nil(t, err) - - refs := source.Object().Properties.(map[string]interface{})["toTarget"] - refsSlice, ok := refs.(models.MultipleRef) - require.True(t, ok, fmt.Sprintf("toTarget must be models.MultipleRef, but got %#v", refs)) - - foundBeacons := []string{} - for _, ref := range refsSlice { - foundBeacons = append(foundBeacons, ref.Beacon.String()) - } - expectedBeacons := []string{ - fmt.Sprintf("weaviate://localhost/%s", target1), - fmt.Sprintf("weaviate://localhost/%s", target2), - fmt.Sprintf("weaviate://localhost/%s", target3), - fmt.Sprintf("weaviate://localhost/%s", target4), - } - - assert.ElementsMatch(t, foundBeacons, expectedBeacons) - }) - - t.Run("merge object with no vector", func(t *testing.T) { - err = repo.Merge(context.Background(), objects.MergeDocument{ - Class: "MergeTestNoVector", - ID: noVecID, - PrimitiveSchema: map[string]interface{}{"foo": "baz"}, - }, nil, "") - require.Nil(t, err) - - orig, err := repo.ObjectByID(context.Background(), noVecID, nil, additional.Properties{}, "") - require.Nil(t, err) - - expectedSchema := map[string]interface{}{ - "foo": "baz", - "id": noVecID, - } - - assert.Equal(t, expectedSchema, orig.Schema) - }) -} - -// This prevents a regression on -// https://github.com/weaviate/weaviate/issues/2193 -// -// Prior to the fix it was possible that a prop that was not touched during the -// merge (and therefore only loaded from disk) failed during the -// inverted-indexing for the new doc id. This was then hidden by the fact that -// error handling was broken inside the inverted.Analyzer. This test tries to -// make sure that every possible property type stays intact if untouched -// during a Merge operation -// -// To achieve this, every prop in this class exists twice, once with the prefix -// 'touched_' and once with 'untouched_'. In the initial insert both properties -// contain the same value, but then during the patch merge, the 'touched_' -// properties are updated to a different value while the 'untouched_' -// properties are left untouched. Then we try to retrieve the object through a -// filter matching each property. The 'untouched_' properties are matched with -// the original value, the 'touched_' props are matched with the updated ones -func Test_Merge_UntouchedPropsCorrectlyIndexed(t *testing.T) { - dirName := t.TempDir() - - logger := logrus.New() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - MaxImportGoroutinesFactor: 1, - QueryMaximumResults: 10000, - TrackVectorDimensions: true, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - defer repo.Shutdown(context.Background()) - migrator := NewMigrator(repo, logger) - hnswConfig := enthnsw.NewDefaultUserConfig() - hnswConfig.Skip = true - sch := schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{ - { - Class: "TestClass", - VectorIndexConfig: hnswConfig, - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{ // tries to have "one of each property type" - { - Name: "untouched_string", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "touched_string", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "untouched_string_array", - DataType: schema.DataTypeTextArray.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "touched_string_array", - DataType: schema.DataTypeTextArray.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "untouched_text", Tokenization: "word", - DataType: []string{"text"}, - }, - { - Name: "touched_text", Tokenization: "word", - DataType: []string{"text"}, - }, - { - Name: "untouched_text_array", Tokenization: "word", - DataType: []string{"text[]"}, - }, - { - Name: "touched_text_array", Tokenization: "word", - DataType: []string{"text[]"}, - }, - {Name: "untouched_number", DataType: []string{"number"}}, - {Name: "touched_number", DataType: []string{"number"}}, - {Name: "untouched_number_array", DataType: []string{"number[]"}}, - {Name: "touched_number_array", DataType: []string{"number[]"}}, - {Name: "untouched_int", DataType: []string{"int"}}, - {Name: "touched_int", DataType: []string{"int"}}, - {Name: "untouched_int_array", DataType: []string{"int[]"}}, - {Name: "touched_int_array", DataType: []string{"int[]"}}, - {Name: "untouched_date", DataType: []string{"date"}}, - {Name: "touched_date", DataType: []string{"date"}}, - {Name: "untouched_date_array", DataType: []string{"date[]"}}, - {Name: "touched_date_array", DataType: []string{"date[]"}}, - {Name: "untouched_geo", DataType: []string{"geoCoordinates"}}, - {Name: "touched_geo", DataType: []string{"geoCoordinates"}}, - }, - }, - }, - }, - } - - t.Run("add required classes", func(t *testing.T) { - for _, class := range sch.Objects.Classes { - t.Run(fmt.Sprintf("add %s", class.Class), func(t *testing.T) { - err := migrator.AddClass(context.Background(), class, schemaGetter.shardState) - require.Nil(t, err) - }) - } - }) - - schemaGetter.schema = sch - - t.Run("add initial object", func(t *testing.T) { - id := 0 - err := repo.PutObject(context.Background(), &models.Object{ - ID: uuidFromInt(id), - Class: "TestClass", - Properties: map[string]interface{}{ - "untouched_number": float64(id), - "untouched_number_array": []interface{}{float64(id)}, - "untouched_int": id, - "untouched_int_array": []interface{}{int64(id)}, - "untouched_string": fmt.Sprintf("%d", id), - "untouched_string_array": []string{fmt.Sprintf("%d", id)}, - "untouched_text": fmt.Sprintf("%d", id), - "untouched_text_array": []string{fmt.Sprintf("%d", id)}, - "untouched_date": time.Unix(0, 0).Add(time.Duration(id) * time.Hour), - "untouched_date_array": []time.Time{time.Unix(0, 0).Add(time.Duration(id) * time.Hour)}, - "untouched_geo": &models.GeoCoordinates{ - ptFloat32(float32(id)), ptFloat32(float32(id)), - }, - - "touched_number": float64(id), - "touched_number_array": []interface{}{float64(id)}, - "touched_int": id, - "touched_int_array": []interface{}{int64(id)}, - "touched_string": fmt.Sprintf("%d", id), - "touched_string_array": []string{fmt.Sprintf("%d", id)}, - "touched_text": fmt.Sprintf("%d", id), - "touched_text_array": []string{fmt.Sprintf("%d", id)}, - "touched_date": time.Unix(0, 0).Add(time.Duration(id) * time.Hour), - "touched_date_array": []time.Time{time.Unix(0, 0).Add(time.Duration(id) * time.Hour)}, - "touched_geo": &models.GeoCoordinates{ - ptFloat32(float32(id)), ptFloat32(float32(id)), - }, - }, - CreationTimeUnix: int64(id), - LastUpdateTimeUnix: int64(id), - }, []float32{0.5}, nil) - require.Nil(t, err) - }) - - t.Run("patch half the props (all that contain 'touched')", func(t *testing.T) { - updateID := 28 - md := objects.MergeDocument{ - Class: "TestClass", - ID: uuidFromInt(0), - PrimitiveSchema: map[string]interface{}{ - "touched_number": float64(updateID), - "touched_number_array": []interface{}{float64(updateID)}, - "touched_int": updateID, - "touched_int_array": []interface{}{int64(updateID)}, - "touched_string": fmt.Sprintf("%d", updateID), - "touched_string_array": []string{fmt.Sprintf("%d", updateID)}, - "touched_text": fmt.Sprintf("%d", updateID), - "touched_text_array": []string{fmt.Sprintf("%d", updateID)}, - "touched_date": time.Unix(0, 0).Add(time.Duration(updateID) * time.Hour), - "touched_date_array": []time.Time{time.Unix(0, 0).Add(time.Duration(updateID) * time.Hour)}, - "touched_geo": &models.GeoCoordinates{ - ptFloat32(float32(updateID)), ptFloat32(float32(updateID)), - }, - }, - References: nil, - } - err = repo.Merge(context.Background(), md, nil, "") - assert.Nil(t, err) - }) - - t.Run("retrieve by each individual prop", func(t *testing.T) { - retrieve := func(prefix string, id int) func(t *testing.T) { - return func(t *testing.T) { - type test struct { - name string - filter *filters.LocalFilter - } - - tests := []test{ - { - name: "string filter", - filter: buildFilter( - fmt.Sprintf("%s_string", prefix), - fmt.Sprintf("%d", id), - eq, - schema.DataTypeText), - }, - { - name: "string array filter", - filter: buildFilter( - fmt.Sprintf("%s_string_array", prefix), - fmt.Sprintf("%d", id), - eq, - schema.DataTypeText), - }, - { - name: "text filter", - filter: buildFilter( - fmt.Sprintf("%s_text", prefix), - fmt.Sprintf("%d", id), - eq, - dtText), - }, - { - name: "text array filter", - filter: buildFilter( - fmt.Sprintf("%s_text_array", prefix), - fmt.Sprintf("%d", id), - eq, - dtText), - }, - { - name: "int filter", - filter: buildFilter( - fmt.Sprintf("%s_int", prefix), id, eq, dtInt), - }, - { - name: "int array filter", - filter: buildFilter( - fmt.Sprintf("%s_int_array", prefix), id, eq, dtInt), - }, - { - name: "number filter", - filter: buildFilter( - fmt.Sprintf("%s_number", prefix), float64(id), eq, dtNumber), - }, - { - name: "number array filter", - filter: buildFilter( - fmt.Sprintf("%s_number_array", prefix), float64(id), eq, dtNumber), - }, - { - name: "date filter", - filter: buildFilter( - fmt.Sprintf("%s_date", prefix), - time.Unix(0, 0).Add(time.Duration(id)*time.Hour), - eq, dtDate), - }, - { - name: "date array filter", - filter: buildFilter( - fmt.Sprintf("%s_date_array", prefix), - time.Unix(0, 0).Add(time.Duration(id)*time.Hour), - eq, dtDate), - }, - { - name: "geoFilter filter", - filter: buildFilter( - fmt.Sprintf("%s_geo", prefix), - filters.GeoRange{ - GeoCoordinates: &models.GeoCoordinates{ - ptFloat32(float32(id)), ptFloat32(float32(id)), - }, - Distance: 2, - }, - wgr, dtGeoCoordinates), - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - params := dto.GetParams{ - ClassName: "TestClass", - Pagination: &filters.Pagination{Limit: 5}, - Filters: tc.filter, - } - res, err := repo.VectorSearch(context.Background(), params) - require.Nil(t, err) - require.Len(t, res, 1) - - // hard-code the only uuid - assert.Equal(t, uuidFromInt(0), res[0].ID) - }) - } - } - } - t.Run("using untouched", retrieve("untouched", 0)) - t.Run("using touched", retrieve("touched", 28)) - }) -} - -func Test_MergeDocIdPreserved_PropsCorrectlyIndexed(t *testing.T) { - dirName := t.TempDir() - - logger := logrus.New() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - MaxImportGoroutinesFactor: 1, - QueryMaximumResults: 10000, - TrackVectorDimensions: true, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - defer repo.Shutdown(context.Background()) - migrator := NewMigrator(repo, logger) - hnswConfig := enthnsw.NewDefaultUserConfig() - hnswConfig.Skip = true - sch := schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{ - { - Class: "TestClass", - VectorIndexConfig: hnswConfig, - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{ // tries to have "one of each property type" - { - Name: "untouched_string", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "touched_string", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "untouched_string_array", - DataType: schema.DataTypeTextArray.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "touched_string_array", - DataType: schema.DataTypeTextArray.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "untouched_text", Tokenization: "word", - DataType: []string{"text"}, - }, - { - Name: "touched_text", Tokenization: "word", - DataType: []string{"text"}, - }, - { - Name: "untouched_text_array", Tokenization: "word", - DataType: []string{"text[]"}, - }, - { - Name: "touched_text_array", Tokenization: "word", - DataType: []string{"text[]"}, - }, - {Name: "untouched_number", DataType: []string{"number"}}, - {Name: "touched_number", DataType: []string{"number"}}, - {Name: "untouched_number_array", DataType: []string{"number[]"}}, - {Name: "touched_number_array", DataType: []string{"number[]"}}, - {Name: "untouched_int", DataType: []string{"int"}}, - {Name: "touched_int", DataType: []string{"int"}}, - {Name: "untouched_int_array", DataType: []string{"int[]"}}, - {Name: "touched_int_array", DataType: []string{"int[]"}}, - {Name: "untouched_date", DataType: []string{"date"}}, - {Name: "touched_date", DataType: []string{"date"}}, - {Name: "untouched_date_array", DataType: []string{"date[]"}}, - {Name: "touched_date_array", DataType: []string{"date[]"}}, - }, - }, - }, - }, - } - - t.Run("add required classes", func(t *testing.T) { - for _, class := range sch.Objects.Classes { - t.Run(fmt.Sprintf("add %s", class.Class), func(t *testing.T) { - err := migrator.AddClass(context.Background(), class, schemaGetter.shardState) - require.Nil(t, err) - }) - } - }) - - schemaGetter.schema = sch - - t.Run("add initial object", func(t *testing.T) { - id := 0 - err := repo.PutObject(context.Background(), &models.Object{ - ID: uuidFromInt(id), - Class: "TestClass", - Properties: map[string]interface{}{ - "untouched_number": float64(id), - "untouched_number_array": []interface{}{float64(id)}, - "untouched_int": id, - "untouched_int_array": []interface{}{int64(id)}, - "untouched_string": fmt.Sprintf("%d", id), - "untouched_string_array": []string{fmt.Sprintf("%d", id)}, - "untouched_text": fmt.Sprintf("%d", id), - "untouched_text_array": []string{fmt.Sprintf("%d", id)}, - "untouched_date": time.Unix(0, 0).Add(time.Duration(id) * time.Hour), - "untouched_date_array": []time.Time{time.Unix(0, 0).Add(time.Duration(id) * time.Hour)}, - - "touched_number": float64(id), - "touched_number_array": []interface{}{float64(id)}, - "touched_int": id, - "touched_int_array": []interface{}{int64(id)}, - "touched_string": fmt.Sprintf("%d", id), - "touched_string_array": []string{fmt.Sprintf("%d", id)}, - "touched_text": fmt.Sprintf("%d", id), - "touched_text_array": []string{fmt.Sprintf("%d", id)}, - "touched_date": time.Unix(0, 0).Add(time.Duration(id) * time.Hour), - "touched_date_array": []time.Time{time.Unix(0, 0).Add(time.Duration(id) * time.Hour)}, - }, - CreationTimeUnix: int64(id), - LastUpdateTimeUnix: int64(id), - }, []float32{0.5}, nil) - require.Nil(t, err) - }) - - t.Run("patch half the props (all that contain 'touched')", func(t *testing.T) { - updateID := 28 - md := objects.MergeDocument{ - Class: "TestClass", - ID: uuidFromInt(0), - PrimitiveSchema: map[string]interface{}{ - "touched_number": float64(updateID), - "touched_number_array": []interface{}{float64(updateID)}, - "touched_int": updateID, - "touched_int_array": []interface{}{int64(updateID)}, - "touched_string": fmt.Sprintf("%d", updateID), - "touched_string_array": []string{fmt.Sprintf("%d", updateID)}, - "touched_text": fmt.Sprintf("%d", updateID), - "touched_text_array": []string{fmt.Sprintf("%d", updateID)}, - "touched_date": time.Unix(0, 0).Add(time.Duration(updateID) * time.Hour), - "touched_date_array": []time.Time{time.Unix(0, 0).Add(time.Duration(updateID) * time.Hour)}, - }, - References: nil, - } - err = repo.Merge(context.Background(), md, nil, "") - assert.Nil(t, err) - }) - - t.Run("retrieve by each individual prop", func(t *testing.T) { - retrieve := func(prefix string, id int) func(t *testing.T) { - return func(t *testing.T) { - type test struct { - name string - filter *filters.LocalFilter - } - - tests := []test{ - { - name: "string filter", - filter: buildFilter( - fmt.Sprintf("%s_string", prefix), - fmt.Sprintf("%d", id), - eq, - schema.DataTypeText), - }, - { - name: "string array filter", - filter: buildFilter( - fmt.Sprintf("%s_string_array", prefix), - fmt.Sprintf("%d", id), - eq, - schema.DataTypeText), - }, - { - name: "text filter", - filter: buildFilter( - fmt.Sprintf("%s_text", prefix), - fmt.Sprintf("%d", id), - eq, - dtText), - }, - { - name: "text array filter", - filter: buildFilter( - fmt.Sprintf("%s_text_array", prefix), - fmt.Sprintf("%d", id), - eq, - dtText), - }, - { - name: "int filter", - filter: buildFilter( - fmt.Sprintf("%s_int", prefix), id, eq, dtInt), - }, - { - name: "int array filter", - filter: buildFilter( - fmt.Sprintf("%s_int_array", prefix), id, eq, dtInt), - }, - { - name: "number filter", - filter: buildFilter( - fmt.Sprintf("%s_number", prefix), float64(id), eq, dtNumber), - }, - { - name: "number array filter", - filter: buildFilter( - fmt.Sprintf("%s_number_array", prefix), float64(id), eq, dtNumber), - }, - { - name: "date filter", - filter: buildFilter( - fmt.Sprintf("%s_date", prefix), - time.Unix(0, 0).Add(time.Duration(id)*time.Hour), - eq, dtDate), - }, - { - name: "date array filter", - filter: buildFilter( - fmt.Sprintf("%s_date_array", prefix), - time.Unix(0, 0).Add(time.Duration(id)*time.Hour), - eq, dtDate), - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - params := dto.GetParams{ - ClassName: "TestClass", - Pagination: &filters.Pagination{Limit: 5}, - Filters: tc.filter, - } - res, err := repo.VectorSearch(context.Background(), params) - require.Nil(t, err) - require.Len(t, res, 1) - - // hard-code the only uuid - assert.Equal(t, uuidFromInt(0), res[0].ID) - }) - } - } - } - t.Run("using untouched", retrieve("untouched", 0)) - t.Run("using touched", retrieve("touched", 28)) - }) -} - -func uuidFromInt(in int) strfmt.UUID { - return strfmt.UUID(uuid.MustParse(fmt.Sprintf("%032d", in)).String()) -} diff --git a/adapters/repos/db/metrics.go b/adapters/repos/db/metrics.go deleted file mode 100644 index b26969230e3a4ca589cd4f877fe25041f9bc28e6..0000000000000000000000000000000000000000 --- a/adapters/repos/db/metrics.go +++ /dev/null @@ -1,302 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "time" - - "github.com/prometheus/client_golang/prometheus" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/usecases/monitoring" -) - -type Metrics struct { - logger logrus.FieldLogger - monitoring bool - batchTime prometheus.ObserverVec - batchDeleteTime prometheus.ObserverVec - objectTime prometheus.ObserverVec - startupDurations prometheus.ObserverVec - filteredVectorFilter prometheus.Observer - filteredVectorVector prometheus.Observer - filteredVectorObjects prometheus.Observer - filteredVectorSort prometheus.Observer - grouped bool - baseMetrics *monitoring.PrometheusMetrics -} - -func NewMetrics( - logger logrus.FieldLogger, prom *monitoring.PrometheusMetrics, - className, shardName string, -) *Metrics { - m := &Metrics{ - logger: logger, - } - - if prom == nil { - return m - } - - m.baseMetrics = prom - - if prom.Group { - className = "n/a" - shardName = "n/a" - m.grouped = true - } - - m.monitoring = true - m.batchTime = prom.BatchTime.MustCurryWith(prometheus.Labels{ - "class_name": className, - "shard_name": shardName, - }) - m.batchDeleteTime = prom.BatchDeleteTime.MustCurryWith(prometheus.Labels{ - "class_name": className, - "shard_name": shardName, - }) - m.objectTime = prom.ObjectsTime.MustCurryWith(prometheus.Labels{ - "class_name": className, - "shard_name": shardName, - }) - m.startupDurations = prom.StartupDurations.MustCurryWith(prometheus.Labels{ - "class_name": className, - "shard_name": shardName, - }) - - m.filteredVectorFilter = prom.QueriesFilteredVectorDurations.With(prometheus.Labels{ - "class_name": className, - "shard_name": shardName, - "operation": "filter", - }) - - m.filteredVectorVector = prom.QueriesFilteredVectorDurations.With(prometheus.Labels{ - "class_name": className, - "shard_name": shardName, - "operation": "vector", - }) - - m.filteredVectorObjects = prom.QueriesFilteredVectorDurations.With(prometheus.Labels{ - "class_name": className, - "shard_name": shardName, - "operation": "objects", - }) - - m.filteredVectorSort = prom.QueriesFilteredVectorDurations.With(prometheus.Labels{ - "class_name": className, - "shard_name": shardName, - "operation": "sort", - }) - - return m -} - -func (m *Metrics) DeleteShardLabels(class, shard string) { - if m.grouped { - // never delete the shared label, only individual ones - return - } - - m.baseMetrics.DeleteShard(class, shard) -} - -func (m *Metrics) BatchObject(start time.Time, size int) { - took := time.Since(start) - m.logger.WithField("action", "batch_objects"). - WithField("batch_size", size). - WithField("took", took). - Tracef("object batch took %s", took) -} - -func (m *Metrics) ObjectStore(start time.Time) { - took := time.Since(start) - m.logger.WithField("action", "store_object_store"). - WithField("took", took). - Tracef("storing objects in KV/inverted store took %s", took) - - if !m.monitoring { - return - } - - m.batchTime.With(prometheus.Labels{"operation": "object_storage"}). - Observe(float64(took / time.Millisecond)) -} - -func (m *Metrics) VectorIndex(start time.Time) { - took := time.Since(start) - m.logger.WithField("action", "store_vector_index"). - WithField("took", took). - Tracef("storing objects vector index took %s", took) - - if !m.monitoring { - return - } - - m.batchTime.With(prometheus.Labels{"operation": "vector_storage"}). - Observe(float64(took / time.Millisecond)) -} - -func (m *Metrics) PutObject(start time.Time) { - took := time.Since(start) - m.logger.WithField("action", "store_object_store_single_object_in_tx"). - WithField("took", took). - Tracef("storing single object (complete) in KV/inverted took %s", took) - - if !m.monitoring { - return - } - - m.objectTime.With(prometheus.Labels{ - "operation": "put", - "step": "total", - }).Observe(float64(took) / float64(time.Millisecond)) -} - -func (m *Metrics) PutObjectDetermineStatus(start time.Time) { - took := time.Since(start) - m.logger.WithField("action", "store_object_store_determine_status"). - WithField("took", took). - Tracef("retrieving previous and determining status in KV took %s", took) - - if !m.monitoring { - return - } - - m.objectTime.With(prometheus.Labels{ - "operation": "put", - "step": "retrieve_previous_determine_status", - }).Observe(float64(took) / float64(time.Millisecond)) -} - -func (m *Metrics) PutObjectUpsertObject(start time.Time) { - took := time.Since(start) - m.logger.WithField("action", "store_object_store_upsert_object_data"). - WithField("took", took). - Tracef("storing object data in KV took %s", took) - - if !m.monitoring { - return - } - - m.objectTime.With(prometheus.Labels{ - "operation": "put", - "step": "upsert_object_store", - }).Observe(float64(took) / float64(time.Millisecond)) -} - -func (m *Metrics) PutObjectUpdateInverted(start time.Time) { - took := time.Since(start) - m.logger.WithField("action", "store_object_store_update_inverted"). - WithField("took", took). - Tracef("updating inverted index for single object took %s", took) - - if !m.monitoring { - return - } - - m.objectTime.With(prometheus.Labels{ - "operation": "put", - "step": "inverted_total", - }).Observe(float64(took) / float64(time.Millisecond)) -} - -func (m *Metrics) InvertedDeleteOld(start time.Time) { - took := time.Since(start) - m.logger.WithField("action", "inverted_delete_old"). - WithField("took", took). - Tracef("deleting old entries from inverted index %s", took) - if !m.monitoring { - return - } - - m.objectTime.With(prometheus.Labels{ - "operation": "put", - "step": "inverted_delete", - }).Observe(float64(took) / float64(time.Millisecond)) -} - -func (m *Metrics) InvertedDeleteDelta(start time.Time) { - took := time.Since(start) - m.logger.WithField("action", "inverted_delete_delta"). - WithField("took", took). - Tracef("deleting delta entries from inverted index %s", took) -} - -func (m *Metrics) InvertedExtend(start time.Time, propCount int) { - took := time.Since(start) - m.logger.WithField("action", "inverted_extend"). - WithField("took", took). - WithField("prop_count", propCount). - Tracef("extending inverted index took %s", took) - - if !m.monitoring { - return - } - - m.objectTime.With(prometheus.Labels{ - "operation": "put", - "step": "inverted_extend", - }).Observe(float64(took) / float64(time.Millisecond)) -} - -func (m *Metrics) ShardStartup(start time.Time) { - if !m.monitoring { - return - } - - took := time.Since(start) - m.startupDurations.With(prometheus.Labels{ - "operation": "shard_total_init", - }).Observe(float64(took) / float64(time.Millisecond)) -} - -func (m *Metrics) BatchDelete(start time.Time, op string) { - if !m.monitoring { - return - } - - took := time.Since(start) - m.batchDeleteTime.With(prometheus.Labels{ - "operation": op, - }).Observe(float64(took) / float64(time.Millisecond)) -} - -func (m *Metrics) FilteredVectorFilter(dur time.Duration) { - if !m.monitoring { - return - } - - m.filteredVectorFilter.Observe(float64(dur) / float64(time.Millisecond)) -} - -func (m *Metrics) FilteredVectorVector(dur time.Duration) { - if !m.monitoring { - return - } - - m.filteredVectorVector.Observe(float64(dur) / float64(time.Millisecond)) -} - -func (m *Metrics) FilteredVectorObjects(dur time.Duration) { - if !m.monitoring { - return - } - - m.filteredVectorObjects.Observe(float64(dur) / float64(time.Millisecond)) -} - -func (m *Metrics) FilteredVectorSort(dur time.Duration) { - if !m.monitoring { - return - } - - m.filteredVectorSort.Observe(float64(dur) / float64(time.Millisecond)) -} diff --git a/adapters/repos/db/migrator.go b/adapters/repos/db/migrator.go deleted file mode 100644 index 753cfaa2fa04037cc338f145539ead8f835f6a0d..0000000000000000000000000000000000000000 --- a/adapters/repos/db/migrator.go +++ /dev/null @@ -1,683 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "context" - "fmt" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/adapters/repos/db/inverted" - "github.com/weaviate/weaviate/adapters/repos/db/vector/flat" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw" - "github.com/weaviate/weaviate/entities/errorcompounder" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/storobj" - "github.com/weaviate/weaviate/usecases/replica" - "github.com/weaviate/weaviate/usecases/schema/migrate" - "github.com/weaviate/weaviate/usecases/sharding" - "golang.org/x/sync/errgroup" -) - -type Migrator struct { - db *DB - logger logrus.FieldLogger -} - -func (m *Migrator) AddClass(ctx context.Context, class *models.Class, - shardState *sharding.State, -) error { - if err := replica.ValidateConfig(class, m.db.config.Replication); err != nil { - return fmt.Errorf("replication config: %w", err) - } - - idx, err := NewIndex(ctx, - IndexConfig{ - ClassName: schema.ClassName(class.Class), - RootPath: m.db.config.RootPath, - ResourceUsage: m.db.config.ResourceUsage, - QueryMaximumResults: m.db.config.QueryMaximumResults, - QueryNestedRefLimit: m.db.config.QueryNestedRefLimit, - MemtablesFlushIdleAfter: m.db.config.MemtablesFlushIdleAfter, - MemtablesInitialSizeMB: m.db.config.MemtablesInitialSizeMB, - MemtablesMaxSizeMB: m.db.config.MemtablesMaxSizeMB, - MemtablesMinActiveSeconds: m.db.config.MemtablesMinActiveSeconds, - MemtablesMaxActiveSeconds: m.db.config.MemtablesMaxActiveSeconds, - TrackVectorDimensions: m.db.config.TrackVectorDimensions, - AvoidMMap: m.db.config.AvoidMMap, - DisableLazyLoadShards: m.db.config.DisableLazyLoadShards, - ReplicationFactor: class.ReplicationConfig.Factor, - }, - shardState, - // no backward-compatibility check required, since newly added classes will - // always have the field set - inverted.ConfigFromModel(class.InvertedIndexConfig), - class.VectorIndexConfig.(schema.VectorIndexConfig), - m.db.schemaGetter, m.db, m.logger, m.db.nodeResolver, m.db.remoteIndex, - m.db.replicaClient, m.db.promMetrics, class, m.db.jobQueueCh, m.db.indexCheckpoints) - if err != nil { - return errors.Wrap(err, "create index") - } - - err = idx.addUUIDProperty(ctx) - if err != nil { - return errors.Wrapf(err, "extend idx '%s' with uuid property", idx.ID()) - } - - if class.InvertedIndexConfig.IndexTimestamps { - err = idx.addTimestampProperties(ctx) - if err != nil { - return errors.Wrapf(err, "extend idx '%s' with timestamp properties", idx.ID()) - } - } - - if m.db.config.TrackVectorDimensions { - if err := idx.addDimensionsProperty(context.TODO()); err != nil { - return errors.Wrap(err, "init id property") - } - } - - m.db.indexLock.Lock() - m.db.indices[idx.ID()] = idx - idx.notifyReady() - m.db.indexLock.Unlock() - - return nil -} - -func (m *Migrator) DropClass(ctx context.Context, className string) error { - return m.db.DeleteIndex(schema.ClassName(className)) -} - -func (m *Migrator) UpdateClass(ctx context.Context, className string, newClassName *string) error { - if newClassName != nil { - return errors.New("weaviate does not support renaming of classes") - } - - return nil -} - -func (m *Migrator) AddProperty(ctx context.Context, className string, prop *models.Property) error { - idx := m.db.GetIndex(schema.ClassName(className)) - if idx == nil { - return errors.Errorf("cannot add property to a non-existing index for %s", className) - } - - return idx.addProperty(ctx, prop) -} - -// DropProperty is ignored, API compliant change -func (m *Migrator) DropProperty(ctx context.Context, className string, propertyName string) error { - // ignore but don't error - return nil -} - -func (m *Migrator) UpdateProperty(ctx context.Context, className string, propName string, newName *string) error { - if newName != nil { - return errors.New("weaviate does not support renaming of properties") - } - - return nil -} - -func (m *Migrator) GetShardsQueueSize(ctx context.Context, className, tenant string) (map[string]int64, error) { - idx := m.db.GetIndex(schema.ClassName(className)) - if idx == nil { - return nil, errors.Errorf("cannot get shards status for a non-existing index for %s", className) - } - - return idx.getShardsQueueSize(ctx, tenant) -} - -func (m *Migrator) GetShardsStatus(ctx context.Context, className, tenant string) (map[string]string, error) { - idx := m.db.GetIndex(schema.ClassName(className)) - if idx == nil { - return nil, errors.Errorf("cannot get shards status for a non-existing index for %s", className) - } - - return idx.getShardsStatus(ctx, tenant) -} - -func (m *Migrator) UpdateShardStatus(ctx context.Context, className, shardName, targetStatus string) error { - idx := m.db.GetIndex(schema.ClassName(className)) - if idx == nil { - return errors.Errorf("cannot update shard status to a non-existing index for %s", className) - } - - return idx.updateShardStatus(ctx, shardName, targetStatus) -} - -// NewTenants creates new partitions and returns a commit func -// that can be used to either commit or rollback the partitions -func (m *Migrator) NewTenants(ctx context.Context, class *models.Class, creates []*migrate.CreateTenantPayload) (commit func(success bool), err error) { - idx := m.db.GetIndex(schema.ClassName(class.Class)) - if idx == nil { - return nil, fmt.Errorf("cannot find index for %q", class.Class) - } - - shards := make(map[string]ShardLike, len(creates)) - rollback := func() { - for name, shard := range shards { - if err := shard.drop(); err != nil { - m.logger.WithField("action", "drop_shard"). - WithField("class", class.Class). - Errorf("cannot drop self created shard %s: %v", name, err) - } - } - } - commit = func(success bool) { - if success { - for name, shard := range shards { - idx.shards.Store(name, shard) - } - return - } - rollback() - } - defer func() { - if err != nil { - rollback() - } - }() - - for _, pl := range creates { - if shard := idx.shards.Load(pl.Name); shard != nil { - continue - } - if pl.Status != models.TenantActivityStatusHOT { - continue // skip creating inactive shards - } - - shard, err := idx.initShard(ctx, pl.Name, class, m.db.promMetrics) - if err != nil { - return nil, fmt.Errorf("cannot create partition %q: %w", pl, err) - } - shards[pl.Name] = shard - } - - return commit, nil -} - -// UpdateTenans activates or deactivates tenant partitions and returns a commit func -// that can be used to either commit or rollback the changes -func (m *Migrator) UpdateTenants(ctx context.Context, class *models.Class, updates []*migrate.UpdateTenantPayload) (commit func(success bool), err error) { - idx := m.db.GetIndex(schema.ClassName(class.Class)) - if idx == nil { - return nil, fmt.Errorf("cannot find index for %q", class.Class) - } - - shardsToHot := make([]string, 0, len(updates)) - shardsToCold := make([]string, 0, len(updates)) - shardsHotted := make(map[string]ShardLike) - shardsColded := make(map[string]ShardLike) - - rollbackHotted := func() { - eg := new(errgroup.Group) - eg.SetLimit(2 * _NUMCPU) - for name, shard := range shardsHotted { - name, shard := name, shard - eg.Go(func() error { - if err := shard.Shutdown(ctx); err != nil { - idx.logger.WithField("action", "rollback_shutdown_shard"). - WithField("shard", shard.ID()). - Errorf("cannot shutdown self activated shard %q: %s", name, err) - } - return nil - }) - } - eg.Wait() - } - rollbackColded := func() { - for name, shard := range shardsColded { - idx.shards.CompareAndSwap(name, nil, shard) - } - } - rollback := func() { - rollbackHotted() - rollbackColded() - } - - commitHotted := func() { - for name, shard := range shardsHotted { - idx.shards.Store(name, shard) - } - } - commitColded := func() { - for name := range shardsColded { - idx.shards.LoadAndDelete(name) - } - - eg := new(errgroup.Group) - eg.SetLimit(_NUMCPU * 2) - for name, shard := range shardsColded { - name, shard := name, shard - eg.Go(func() error { - if err := shard.Shutdown(ctx); err != nil { - idx.logger.WithField("action", "shutdown_shard"). - WithField("shard", shard.ID()). - Errorf("cannot shutdown shard %q: %s", name, err) - } - return nil - }) - } - eg.Wait() - } - commit = func(success bool) { - if !success { - rollback() - return - } - commitHotted() - commitColded() - } - - applyHot := func() error { - for _, name := range shardsToHot { - // shard already hot - if shard := idx.shards.Load(name); shard != nil { - continue - } - - shard, err := idx.initShard(ctx, name, class, m.db.promMetrics) - if err != nil { - return fmt.Errorf("cannot activate shard '%s': %w", name, err) - } - shardsHotted[name] = shard - } - return nil - } - applyCold := func() error { - idx.backupMutex.RLock() - defer idx.backupMutex.RUnlock() - - for _, name := range shardsToCold { - shard, ok := idx.shards.Swap(name, nil) // mark as deactivated - if !ok { // shard doesn't exit (already cold) - idx.shards.LoadAndDelete(name) // rollback nil value created by swap() - continue - } - if shard != nil { - shardsColded[name] = shard - } - } - return nil - } - - for _, tu := range updates { - switch tu.Status { - case models.TenantActivityStatusHOT: - shardsToHot = append(shardsToHot, tu.Name) - case models.TenantActivityStatusCOLD: - shardsToCold = append(shardsToCold, tu.Name) - } - } - - defer func() { - if err != nil { - rollback() - } - }() - - if err := applyHot(); err != nil { - return nil, err - } - if err := applyCold(); err != nil { - return nil, err - } - - return commit, nil -} - -// DeleteTenants deletes tenants and returns a commit func -// that can be used to either commit or rollback deletion -func (m *Migrator) DeleteTenants(ctx context.Context, class *models.Class, tenants []string) (commit func(success bool), err error) { - idx := m.db.GetIndex(schema.ClassName(class.Class)) - if idx == nil { - return func(bool) {}, nil - } - return idx.dropShards(tenants) -} - -func NewMigrator(db *DB, logger logrus.FieldLogger) *Migrator { - return &Migrator{db: db, logger: logger} -} - -func (m *Migrator) UpdateVectorIndexConfig(ctx context.Context, - className string, updated schema.VectorIndexConfig, -) error { - idx := m.db.GetIndex(schema.ClassName(className)) - if idx == nil { - return errors.Errorf("cannot update vector index config of non-existing index for %s", className) - } - - return idx.updateVectorIndexConfig(ctx, updated) -} - -func (m *Migrator) ValidateVectorIndexConfigUpdate(ctx context.Context, - old, updated schema.VectorIndexConfig, -) error { - // hnsw is the only supported vector index type at the moment, so no need - // to check, we can always use that an hnsw-specific validation should be - // used for now. - switch old.IndexType() { - case "hnsw": - return hnsw.ValidateUserConfigUpdate(old, updated) - case "flat": - return flat.ValidateUserConfigUpdate(old, updated) - } - return fmt.Errorf("Invalid index type: %s", old.IndexType()) -} - -func (m *Migrator) ValidateInvertedIndexConfigUpdate(ctx context.Context, - old, updated *models.InvertedIndexConfig, -) error { - return inverted.ValidateUserConfigUpdate(old, updated) -} - -func (m *Migrator) UpdateInvertedIndexConfig(ctx context.Context, className string, - updated *models.InvertedIndexConfig, -) error { - idx := m.db.GetIndex(schema.ClassName(className)) - if idx == nil { - return errors.Errorf("cannot update inverted index config of non-existing index for %s", className) - } - - conf := inverted.ConfigFromModel(updated) - - return idx.updateInvertedIndexConfig(ctx, conf) -} - -func (m *Migrator) RecalculateVectorDimensions(ctx context.Context) error { - count := 0 - m.logger. - WithField("action", "reindex"). - Info("Reindexing dimensions, this may take a while") - - // Iterate over all indexes - for _, index := range m.db.indices { - // Iterate over all shards - if err := index.IterateObjects(ctx, func(index *Index, shard ShardLike, object *storobj.Object) error { - count = count + 1 - err := shard.extendDimensionTrackerLSM(len(object.Vector), object.DocID()) - return err - }); err != nil { - return err - } - } - go func() { - for { - m.logger. - WithField("action", "reindex"). - Warnf("Reindexed %v objects. Reindexing dimensions complete. Please remove environment variable REINDEX_VECTOR_DIMENSIONS_AT_STARTUP before next startup", count) - time.Sleep(5 * time.Minute) - } - }() - - return nil -} - -func (m *Migrator) RecountProperties(ctx context.Context) error { - count := 0 - m.logger. - WithField("action", "recount"). - Info("Recounting properties, this may take a while") - - m.db.indexLock.Lock() - defer m.db.indexLock.Unlock() - // Iterate over all indexes - for _, index := range m.db.indices { - - // Clear the shards before counting - index.IterateShards(ctx, func(index *Index, shard ShardLike) error { - shard.GetPropertyLengthTracker().Clear() - return nil - }) - - // Iterate over all shards - index.IterateObjects(ctx, func(index *Index, shard ShardLike, object *storobj.Object) error { - count = count + 1 - props, _, err := shard.AnalyzeObject(object) - if err != nil { - m.logger.WithField("error", err).Error("could not analyze object") - return nil - } - - if err := shard.SetPropertyLengths(props); err != nil { - m.logger.WithField("error", err).Error("could not add prop lengths") - return nil - } - - shard.GetPropertyLengthTracker().Flush(false) - - return nil - }) - - // Flush the GetPropertyLengthTracker() to disk - err := index.IterateShards(ctx, func(index *Index, shard ShardLike) error { - return shard.GetPropertyLengthTracker().Flush(false) - }) - if err != nil { - m.logger.WithField("error", err).Error("could not flush prop lengths") - } - - } - go func() { - for { - m.logger. - WithField("action", "recount"). - Warnf("Recounted %v objects. Recounting properties complete. Please remove environment variable RECOUNT_PROPERTIES_AT_STARTUP before next startup", count) - time.Sleep(5 * time.Minute) - } - }() - - return nil -} - -func (m *Migrator) InvertedReindex(ctx context.Context, taskNames ...string) error { - var errs errorcompounder.ErrorCompounder - errs.Add(m.doInvertedReindex(ctx, taskNames...)) - errs.Add(m.doInvertedIndexMissingTextFilterable(ctx, taskNames...)) - return errs.ToError() -} - -func (m *Migrator) doInvertedReindex(ctx context.Context, taskNames ...string) error { - tasksProviders := map[string]func() ShardInvertedReindexTask{ - "ShardInvertedReindexTaskSetToRoaringSet": func() ShardInvertedReindexTask { - return &ShardInvertedReindexTaskSetToRoaringSet{} - }, - } - - tasks := map[string]ShardInvertedReindexTask{} - for _, taskName := range taskNames { - if taskProvider, ok := tasksProviders[taskName]; ok { - tasks[taskName] = taskProvider() - } - } - - if len(tasks) == 0 { - return nil - } - - eg := &errgroup.Group{} - eg.SetLimit(_NUMCPU) - for _, index := range m.db.indices { - index.ForEachShard(func(name string, shard ShardLike) error { - eg.Go(func() error { - reindexer := NewShardInvertedReindexer(shard, m.logger) - for taskName, task := range tasks { - reindexer.AddTask(task) - m.logInvertedReindexShard(shard). - WithField("task", taskName). - Info("About to start inverted reindexing, this may take a while") - } - if err := reindexer.Do(ctx); err != nil { - m.logInvertedReindexShard(shard). - WithError(err). - Error("failed reindexing") - return errors.Wrapf(err, "failed reindexing shard '%s'", shard.ID()) - } - m.logInvertedReindexShard(shard). - Info("Finished inverted reindexing") - return nil - }) - return nil - }) - } - return eg.Wait() -} - -func (m *Migrator) doInvertedIndexMissingTextFilterable(ctx context.Context, taskNames ...string) error { - taskName := "ShardInvertedReindexTaskMissingTextFilterable" - taskFound := false - for _, name := range taskNames { - if name == taskName { - taskFound = true - break - } - } - if !taskFound { - return nil - } - - task := newShardInvertedReindexTaskMissingTextFilterable(m) - if err := task.init(); err != nil { - m.logMissingFilterable().WithError(err).Error("failed init missing text filterable task") - return errors.Wrap(err, "failed init missing text filterable task") - } - - if len(task.migrationState.MissingFilterableClass2Props) == 0 { - m.logMissingFilterable().Info("no classes to create filterable index, skipping") - return nil - } - - m.logMissingFilterable().Info("staring missing text filterable task") - - eg := &errgroup.Group{} - eg.SetLimit(_NUMCPU * 2) - for _, index := range m.db.indices { - index := index - className := index.Config.ClassName.String() - - if _, ok := task.migrationState.MissingFilterableClass2Props[className]; !ok { - continue - } - - eg.Go(func() error { - errgrpShards := &errgroup.Group{} - index.ForEachShard(func(_ string, shard ShardLike) error { - errgrpShards.Go(func() error { - m.logMissingFilterableShard(shard). - Info("starting filterable indexing on shard, this may take a while") - - reindexer := NewShardInvertedReindexer(shard, m.logger) - reindexer.AddTask(task) - - if err := reindexer.Do(ctx); err != nil { - m.logMissingFilterableShard(shard). - WithError(err). - Error("failed filterable indexing on shard") - return errors.Wrapf(err, "failed filterable indexing for shard '%s' of index '%s'", - shard.ID(), index.ID()) - } - m.logMissingFilterableShard(shard). - Info("finished filterable indexing on shard") - return nil - }) - return nil - }) - - if err := errgrpShards.Wait(); err != nil { - m.logMissingFilterableIndex(index). - WithError(err). - Error("failed filterable indexing on index") - return errors.Wrapf(err, "failed filterable indexing of index '%s'", index.ID()) - } - - if err := task.updateMigrationStateAndSave(className); err != nil { - m.logMissingFilterableIndex(index). - WithError(err). - Error("failed updating migration state file") - return errors.Wrapf(err, "failed updating migration state file for class '%s'", className) - } - - m.logMissingFilterableIndex(index). - Info("finished filterable indexing on index") - - return nil - }) - } - - if err := eg.Wait(); err != nil { - m.logMissingFilterable(). - WithError(err). - Error("failed missing text filterable task") - return errors.Wrap(err, "failed missing text filterable task") - } - - m.logMissingFilterable().Info("finished missing text filterable task") - return nil -} - -func (m *Migrator) logInvertedReindex() *logrus.Entry { - return m.logger.WithField("action", "inverted_reindex") -} - -func (m *Migrator) logInvertedReindexShard(shard ShardLike) *logrus.Entry { - return m.logInvertedReindex(). - WithField("index", shard.Index().ID()). - WithField("shard", shard.ID()) -} - -func (m *Migrator) logMissingFilterable() *logrus.Entry { - return m.logger.WithField("action", "ii_missing_text_filterable") -} - -func (m *Migrator) logMissingFilterableIndex(index *Index) *logrus.Entry { - return m.logMissingFilterable().WithField("index", index.ID()) -} - -func (m *Migrator) logMissingFilterableShard(shard ShardLike) *logrus.Entry { - return m.logMissingFilterableIndex(shard.Index()).WithField("shard", shard.ID()) -} - -// As of v1.19 property's IndexInverted setting is replaced with IndexFilterable -// and IndexSearchable -// Filterable buckets use roaring set strategy and searchable ones use map strategy -// (therefore are applicable just for text/text[]) -// Since both type of buckets can coexist for text/text[] props they need to be -// distinguished by their name: searchable bucket has "searchable" suffix. -// Up until v1.19 default text/text[]/string/string[] (string/string[] deprecated since v1.19) -// strategy for buckets was map, migrating from pre v1.19 to v1.19 needs to properly -// handle existing text/text[] buckets of map strategy having filterable bucket name. -// -// Enabled InvertedIndex translates in v1.19 to both InvertedFilterable and InvertedSearchable -// enabled, but since only searchable bucket exist (with filterable name), it has to be renamed -// to searchable bucket. -// Though IndexFilterable setting is enabled filterable index does not exists, -// therefore shards are switched into fallback mode, to use searchable buckets instead of -// filterable ones whenever filtered are expected. -// Fallback mode effectively sets IndexFilterable to false, although it stays enabled according -// to schema. -// -// If filterable indexes will be created (that is up to user to decide whether missing indexes -// should be created later on), shards will not be working in fallback mode, and actual filterable index -// will be used when needed. -func (m *Migrator) AdjustFilterablePropSettings(ctx context.Context) error { - f2sm := newFilterableToSearchableMigrator(m) - if err := f2sm.migrate(ctx); err != nil { - return err - } - return f2sm.switchShardsToFallbackMode(ctx) -} diff --git a/adapters/repos/db/multi_shard_integration_test.go b/adapters/repos/db/multi_shard_integration_test.go deleted file mode 100644 index d7985a72d1912196e8296e0ee227f275925b19e0..0000000000000000000000000000000000000000 --- a/adapters/repos/db/multi_shard_integration_test.go +++ /dev/null @@ -1,919 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package db - -import ( - "context" - "fmt" - "math" - "math/rand" - "sort" - "testing" - - "github.com/go-openapi/strfmt" - "github.com/google/uuid" - "github.com/sirupsen/logrus" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/dto" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/schema/crossref" - "github.com/weaviate/weaviate/entities/search" - "github.com/weaviate/weaviate/entities/searchparams" - enthnsw "github.com/weaviate/weaviate/entities/vectorindex/hnsw" - "github.com/weaviate/weaviate/entities/verbosity" - "github.com/weaviate/weaviate/usecases/objects" - "github.com/weaviate/weaviate/usecases/sharding" -) - -func Test_MultiShardJourneys_IndividualImports(t *testing.T) { - r := getRandomSeed() - repo, logger := setupMultiShardTest(t) - defer func() { - repo.Shutdown(context.Background()) - }() - - t.Run("prepare", makeTestMultiShardSchema(repo, logger, false, testClassesForImporting()...)) - - data := multiShardTestData(r) - queryVec := exampleQueryVec(r) - groundTruth := bruteForceObjectsByQuery(data, queryVec) - refData := multiShardRefClassData(r, data) - - t.Run("import all individually", func(t *testing.T) { - for _, obj := range data { - require.Nil(t, repo.PutObject(context.Background(), obj, obj.Vector, nil)) - } - }) - - t.Run("nodes api", testNodesAPI(repo)) - - t.Run("sorting objects", makeTestSortingClass(repo)) - - t.Run("verify objects", makeTestRetrievingBaseClass(repo, data, queryVec, - groundTruth)) - - t.Run("import refs individually", func(t *testing.T) { - for _, obj := range refData { - require.Nil(t, repo.PutObject(context.Background(), obj, obj.Vector, nil)) - } - }) - - t.Run("verify refs", makeTestRetrieveRefClass(repo, data, refData)) - - t.Run("batch delete", makeTestBatchDeleteAllObjects(repo)) -} - -func Test_MultiShardJourneys_BatchedImports(t *testing.T) { - r := getRandomSeed() - repo, logger := setupMultiShardTest(t) - defer func() { - repo.Shutdown(context.Background()) - }() - - t.Run("prepare", makeTestMultiShardSchema(repo, logger, false, testClassesForImporting()...)) - - data := multiShardTestData(r) - queryVec := exampleQueryVec(r) - groundTruth := bruteForceObjectsByQuery(data, queryVec) - refData := multiShardRefClassData(r, data) - - t.Run("import in a batch", func(t *testing.T) { - batch := make(objects.BatchObjects, len(data)) - for i, obj := range data { - batch[i] = objects.BatchObject{ - OriginalIndex: i, - Object: obj, - Vector: obj.Vector, - UUID: obj.ID, - } - } - - _, err := repo.BatchPutObjects(context.Background(), batch, nil) - require.Nil(t, err) - }) - - t.Run("nodes api", testNodesAPI(repo)) - - t.Run("verify objects", makeTestRetrievingBaseClass(repo, data, queryVec, - groundTruth)) - - t.Run("import refs in large batch", func(t *testing.T) { - // first strip the refs from the objects, so we can import them in a second - // step as batch ref - - for _, obj := range refData { - withoutRef := &models.Object{ - ID: obj.ID, - Class: obj.Class, - Vector: obj.Vector, - Properties: map[string]interface{}{}, // empty so we remove the ref - } - - require.Nil(t, repo.PutObject(context.Background(), withoutRef, withoutRef.Vector, nil)) - } - - index := 0 - refBatch := make(objects.BatchReferences, len(refData)*len(data)) - for _, obj := range refData { - for _, ref := range obj.Properties.(map[string]interface{})["toOther"].(models.MultipleRef) { - to, _ := crossref.ParseSingleRef(ref) - refBatch[index] = objects.BatchReference{ - OriginalIndex: index, - To: to, - From: crossref.NewSource(schema.ClassName(obj.Class), "toOther", obj.ID), - } - index++ - } - } - - _, err := repo.AddBatchReferences(context.Background(), refBatch, nil) - require.Nil(t, err) - }) - - t.Run("verify refs", makeTestRetrieveRefClass(repo, data, refData)) - - t.Run("batch delete", makeTestBatchDeleteAllObjects(repo)) -} - -func Test_MultiShardJourneys_BM25_Search(t *testing.T) { - repo, logger := setupMultiShardTest(t) - defer func() { - repo.Shutdown(context.Background()) - }() - - className := "RacecarPosts" - - t.Run("prepare", func(t *testing.T) { - class := &models.Class{ - Class: className, - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: &models.InvertedIndexConfig{ - CleanupIntervalSeconds: 60, - }, - Properties: []*models.Property{ - { - Name: "contents", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWord, - }, - { - Name: "stringProp", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "textArrayProp", - DataType: []string{string(schema.DataTypeTextArray)}, - }, - }, - } - - t.Run("prepare", makeTestMultiShardSchema(repo, logger, true, class)) - }) - - t.Run("insert search data", func(t *testing.T) { - objs := objects.BatchObjects{ - { - UUID: "c39751ed-ddc2-4c9f-a45b-8b5732ddde56", - Object: &models.Object{ - ID: "c39751ed-ddc2-4c9f-a45b-8b5732ddde56", - Class: className, - Properties: map[string]interface{}{ - "contents": "Team Lotus was a domineering force in the early 90s", - }, - }, - }, - { - UUID: "5d034311-06e1-476e-b446-1306db91d906", - Object: &models.Object{ - ID: "5d034311-06e1-476e-b446-1306db91d906", - Class: className, - Properties: map[string]interface{}{ - "contents": "When a car becomes unserviceable, the driver must retire early from the race", - }, - }, - }, - { - UUID: "01989a8c-e37f-471d-89ca-9a787dbbf5f2", - Object: &models.Object{ - ID: "01989a8c-e37f-471d-89ca-9a787dbbf5f2", - Class: className, - Properties: map[string]interface{}{ - "contents": "A young driver is better than an old driver", - }, - }, - }, - { - UUID: "392614c5-4ca4-4630-a014-61fe868a20fd", - Object: &models.Object{ - ID: "392614c5-4ca4-4630-a014-61fe868a20fd", - Class: className, - Properties: map[string]interface{}{ - "contents": "an old driver doesn't retire early", - }, - }, - }, - } - - _, err := repo.BatchPutObjects(context.Background(), objs, nil) - require.Nil(t, err) - }) - - t.Run("ranked keyword search", func(t *testing.T) { - type testcase struct { - expectedResults []string - rankingParams *searchparams.KeywordRanking - } - - tests := []testcase{ - { - rankingParams: &searchparams.KeywordRanking{ - Query: "driver", - Properties: []string{"contents"}, - }, - expectedResults: []string{ - "01989a8c-e37f-471d-89ca-9a787dbbf5f2", - "392614c5-4ca4-4630-a014-61fe868a20fd", - "5d034311-06e1-476e-b446-1306db91d906", - }, - }, - } - - for _, test := range tests { - res, err := repo.Search(context.Background(), dto.GetParams{ - ClassName: className, - Pagination: &filters.Pagination{Limit: 10}, - KeywordRanking: test.rankingParams, - }) - require.Nil(t, err) - require.Equal(t, len(test.expectedResults), len(res)) - for i := range res { - assert.Equal(t, test.expectedResults[i], res[i].ID.String()) - } - t.Logf("res: %+v", res) - } - }) -} - -func setupMultiShardTest(t *testing.T) (*DB, *logrus.Logger) { - dirName := t.TempDir() - - logger, _ := test.NewNullLogger() - repo, err := New(logger, Config{ - ServerVersion: "server-version", - GitHash: "git-hash", - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - return repo, logger -} - -func makeTestMultiShardSchema(repo *DB, logger logrus.FieldLogger, fixedShardState bool, classes ...*models.Class) func(t *testing.T) { - return func(t *testing.T) { - var shardState *sharding.State - if fixedShardState { - shardState = fixedMultiShardState() - } else { - shardState = multiShardState() - } - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: shardState, - } - repo.SetSchemaGetter(schemaGetter) - err := repo.WaitForStartup(testCtx()) - require.Nil(t, err) - migrator := NewMigrator(repo, logger) - - t.Run("creating the class", func(t *testing.T) { - for _, class := range classes { - require.Nil(t, migrator.AddClass(context.Background(), class, schemaGetter.shardState)) - } - }) - - // update schema getter so it's in sync with class - schemaGetter.schema = schema.Schema{ - Objects: &models.Schema{ - Classes: classes, - }, - } - } -} - -func makeTestRetrievingBaseClass(repo *DB, data []*models.Object, - queryVec []float32, groundTruth []*models.Object, -) func(t *testing.T) { - return func(t *testing.T) { - t.Run("retrieve all individually", func(t *testing.T) { - for _, desired := range data { - res, err := repo.ObjectByID(context.Background(), desired.ID, search.SelectProperties{}, additional.Properties{}, "") - assert.Nil(t, err) - - require.NotNil(t, res) - assert.Equal(t, desired.Properties.(map[string]interface{})["boolProp"].(bool), - res.Object().Properties.(map[string]interface{})["boolProp"].(bool)) - assert.Equal(t, desired.ID, res.Object().ID) - } - }) - - t.Run("retrieve through filter (object search)", func(t *testing.T) { - do := func(limit, expected int) { - filters := &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorEqual, - Value: &filters.Value{ - Value: true, - Type: schema.DataTypeBoolean, - }, - On: &filters.Path{ - Property: "boolProp", - }, - }, - } - res, err := repo.ObjectSearch(context.Background(), 0, limit, filters, nil, - additional.Properties{}, "") - assert.Nil(t, err) - - assert.Len(t, res, expected) - for _, obj := range res { - assert.Equal(t, true, obj.Schema.(map[string]interface{})["boolProp"].(bool)) - } - } - - t.Run("with high limit", func(t *testing.T) { - do(100, 10) - }) - - t.Run("with low limit", func(t *testing.T) { - do(3, 3) - }) - }) - - t.Run("retrieve through filter (class search)", func(t *testing.T) { - do := func(limit, expected int) { - filter := &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorEqual, - Value: &filters.Value{ - Value: true, - Type: schema.DataTypeBoolean, - }, - On: &filters.Path{ - Property: "boolProp", - }, - }, - } - res, err := repo.Search(context.Background(), dto.GetParams{ - Filters: filter, - Pagination: &filters.Pagination{ - Limit: limit, - }, - ClassName: "TestClass", - }) - assert.Nil(t, err) - - assert.Len(t, res, expected) - for _, obj := range res { - assert.Equal(t, true, obj.Schema.(map[string]interface{})["boolProp"].(bool)) - } - } - - t.Run("with high limit", func(t *testing.T) { - do(100, 10) - }) - - t.Run("with low limit", func(t *testing.T) { - do(3, 3) - }) - }) - - t.Run("retrieve through class-level vector search", func(t *testing.T) { - do := func(t *testing.T, limit, expected int) { - res, err := repo.VectorSearch(context.Background(), dto.GetParams{ - SearchVector: queryVec, - Pagination: &filters.Pagination{ - Limit: limit, - }, - ClassName: "TestClass", - }) - assert.Nil(t, err) - assert.Len(t, res, expected) - for i, obj := range res { - assert.Equal(t, groundTruth[i].ID, obj.ID) - } - } - - t.Run("with high limit", func(t *testing.T) { - do(t, 100, 20) - }) - - t.Run("with low limit", func(t *testing.T) { - do(t, 3, 3) - }) - }) - - t.Run("retrieve through inter-class vector search", func(t *testing.T) { - do := func(t *testing.T, limit, expected int) { - res, err := repo.CrossClassVectorSearch(context.Background(), queryVec, 0, limit, nil) - assert.Nil(t, err) - assert.Len(t, res, expected) - for i, obj := range res { - assert.Equal(t, groundTruth[i].ID, obj.ID) - } - } - - t.Run("with high limit", func(t *testing.T) { - do(t, 100, 20) - }) - - t.Run("with low limit", func(t *testing.T) { - do(t, 3, 3) - }) - }) - } -} - -func makeTestRetrieveRefClass(repo *DB, data, refData []*models.Object) func(t *testing.T) { - return func(t *testing.T) { - t.Run("retrieve ref data individually with select props", func(t *testing.T) { - for _, desired := range refData { - res, err := repo.ObjectByID(context.Background(), desired.ID, search.SelectProperties{ - search.SelectProperty{ - IsPrimitive: false, - Name: "toOther", - Refs: []search.SelectClass{{ - ClassName: "TestClass", - RefProperties: search.SelectProperties{{ - Name: "index", - IsPrimitive: true, - }}, - }}, - }, - }, additional.Properties{}, "") - assert.Nil(t, err) - refs := res.Schema.(map[string]interface{})["toOther"].([]interface{}) - assert.Len(t, refs, len(data)) - for i, ref := range refs { - indexField := ref.(search.LocalRef).Fields["index"].(float64) - assert.Equal(t, i, int(indexField)) - } - } - }) - } -} - -func makeTestSortingClass(repo *DB) func(t *testing.T) { - return func(t *testing.T) { - t.Run("sort by property", func(t *testing.T) { - getIndex := func(res search.Result) float64 { - if prop := res.Object().Properties.(map[string]interface{})["index"]; prop != nil { - return prop.(float64) - } - return -1 - } - getBoolProp := func(res search.Result) bool { - if prop := res.Object().Properties.(map[string]interface{})["boolProp"]; prop != nil { - return prop.(bool) - } - return false - } - getStringProp := func(res search.Result) string { - if prop := res.Object().Properties.(map[string]interface{})["stringProp"]; prop != nil { - return prop.(string) - } - return "" - } - getTextArrayProp := func(res search.Result) []string { - if prop := res.Object().Properties.(map[string]interface{})["textArrayProp"]; prop != nil { - return prop.([]string) - } - return nil - } - type test struct { - name string - sort []filters.Sort - expectedIndexes []float64 - expectedBoolProps []bool - expectedStringProps []string - expectedTextArrayProps [][]string - constainsErrorMsgs []string - } - tests := []test{ - { - name: "indexProp desc", - sort: []filters.Sort{{Path: []string{"indexProp"}, Order: "desc"}}, - expectedIndexes: []float64{19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0}, - }, - { - name: "indexProp asc", - sort: []filters.Sort{{Path: []string{"indexProp"}, Order: "asc"}}, - expectedIndexes: []float64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19}, - }, - { - name: "stringProp desc", - sort: []filters.Sort{{Path: []string{"stringProp"}, Order: "desc"}}, - expectedStringProps: []string{"s19", "s18", "s17", "s16", "s15", "s14", "s13", "s12", "s11", "s10", "s09", "s08", "s07", "s06", "s05", "s04", "s03", "s02", "s01", "s00"}, - }, - { - name: "stringProp asc", - sort: []filters.Sort{{Path: []string{"stringProp"}, Order: "asc"}}, - expectedStringProps: []string{"s00", "s01", "s02", "s03", "s04", "s05", "s06", "s07", "s08", "s09", "s10", "s11", "s12", "s13", "s14", "s15", "s16", "s17", "s18", "s19"}, - }, - { - name: "textArrayProp desc", - sort: []filters.Sort{{Path: []string{"textArrayProp"}, Order: "desc"}}, - expectedTextArrayProps: [][]string{{"s19", "19"}, {"s18", "18"}, {"s17", "17"}, {"s16", "16"}, {"s15", "15"}, {"s14", "14"}, {"s13", "13"}, {"s12", "12"}, {"s11", "11"}, {"s10", "10"}, {"s09", "09"}, {"s08", "08"}, {"s07", "07"}, {"s06", "06"}, {"s05", "05"}, {"s04", "04"}, {"s03", "03"}, {"s02", "02"}, {"s01", "01"}, {"s00", "00"}}, - }, - { - name: "textArrayProp asc", - sort: []filters.Sort{{Path: []string{"textArrayProp"}, Order: "asc"}}, - expectedTextArrayProps: [][]string{{"s00", "00"}, {"s01", "01"}, {"s02", "02"}, {"s03", "03"}, {"s04", "04"}, {"s05", "05"}, {"s06", "06"}, {"s07", "07"}, {"s08", "08"}, {"s09", "09"}, {"s10", "10"}, {"s11", "11"}, {"s12", "12"}, {"s13", "13"}, {"s14", "14"}, {"s15", "15"}, {"s16", "16"}, {"s17", "17"}, {"s18", "18"}, {"s19", "19"}}, - }, - { - name: "boolProp desc", - sort: []filters.Sort{{Path: []string{"boolProp"}, Order: "desc"}}, - expectedBoolProps: []bool{true, true, true, true, true, true, true, true, true, true, false, false, false, false, false, false, false, false, false, false}, - }, - { - name: "boolProp asc", - sort: []filters.Sort{{Path: []string{"boolProp"}, Order: "asc"}}, - expectedBoolProps: []bool{false, false, false, false, false, false, false, false, false, false, true, true, true, true, true, true, true, true, true, true}, - }, - { - name: "boolProp asc stringProp asc", - sort: []filters.Sort{{Path: []string{"boolProp"}, Order: "asc"}, {Path: []string{"stringProp"}, Order: "asc"}}, - expectedBoolProps: []bool{false, false, false, false, false, false, false, false, false, false, true, true, true, true, true, true, true, true, true, true}, - expectedStringProps: []string{"s01", "s03", "s05", "s07", "s09", "s11", "s13", "s15", "s17", "s19", "s00", "s02", "s04", "s06", "s08", "s10", "s12", "s14", "s16", "s18"}, - }, - { - name: "boolProp desc stringProp asc", - sort: []filters.Sort{{Path: []string{"boolProp"}, Order: "desc"}, {Path: []string{"stringProp"}, Order: "asc"}}, - expectedBoolProps: []bool{true, true, true, true, true, true, true, true, true, true, false, false, false, false, false, false, false, false, false, false}, - expectedStringProps: []string{"s00", "s02", "s04", "s06", "s08", "s10", "s12", "s14", "s16", "s18", "s01", "s03", "s05", "s07", "s09", "s11", "s13", "s15", "s17", "s19"}, - }, - { - name: "boolProp asc indexProp asc", - sort: []filters.Sort{{Path: []string{"boolProp"}, Order: "asc"}, {Path: []string{"indexProp"}, Order: "asc"}}, - expectedBoolProps: []bool{false, false, false, false, false, false, false, false, false, false, true, true, true, true, true, true, true, true, true, true}, - expectedIndexes: []float64{1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18}, - }, - { - name: "boolProp asc indexProp desc", - sort: []filters.Sort{{Path: []string{"boolProp"}, Order: "asc"}, {Path: []string{"indexProp"}, Order: "desc"}}, - expectedBoolProps: []bool{false, false, false, false, false, false, false, false, false, false, true, true, true, true, true, true, true, true, true, true}, - expectedIndexes: []float64{19, 17, 15, 13, 11, 9, 7, 5, 3, 1, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0}, - }, - { - name: "index property doesn't exist in testrefclass", - sort: []filters.Sort{{Path: []string{"index"}, Order: "desc"}}, - expectedIndexes: nil, - constainsErrorMsgs: []string{ - "no such prop with name 'index' found in class 'TestRefClass' in the schema. " + - "Check your schema files for which properties in this class are available", - }, - }, - { - name: "non existent property in all classes", - sort: []filters.Sort{{Path: []string{"nonexistentproperty"}, Order: "desc"}}, - expectedIndexes: nil, - constainsErrorMsgs: []string{ - "no such prop with name 'nonexistentproperty' found in class 'TestClass' in the schema. " + - "Check your schema files for which properties in this class are available", - "no such prop with name 'nonexistentproperty' found in class 'TestRefClass' in the schema. " + - "Check your schema files for which properties in this class are available", - }, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - res, err := repo.ObjectSearch(context.Background(), 0, 1000, nil, test.sort, - additional.Properties{}, "") - if len(test.constainsErrorMsgs) > 0 { - require.NotNil(t, err) - for _, errorMsg := range test.constainsErrorMsgs { - assert.Contains(t, err.Error(), errorMsg) - } - } else { - require.Nil(t, err) - if len(test.expectedIndexes) > 0 { - for i := range res { - assert.Equal(t, test.expectedIndexes[i], getIndex(res[i])) - } - } - if len(test.expectedBoolProps) > 0 { - for i := range res { - assert.Equal(t, test.expectedBoolProps[i], getBoolProp(res[i])) - } - } - if len(test.expectedStringProps) > 0 { - for i := range res { - assert.Equal(t, test.expectedStringProps[i], getStringProp(res[i])) - } - } - if len(test.expectedTextArrayProps) > 0 { - for i := range res { - assert.EqualValues(t, test.expectedTextArrayProps[i], getTextArrayProp(res[i])) - } - } - } - }) - } - }) - } -} - -func testNodesAPI(repo *DB) func(t *testing.T) { - return func(t *testing.T) { - nodeStatues, err := repo.GetNodeStatus(context.Background(), "", verbosity.OutputVerbose) - require.Nil(t, err) - require.NotNil(t, nodeStatues) - - require.Len(t, nodeStatues, 1) - nodeStatus := nodeStatues[0] - assert.NotNil(t, nodeStatus) - assert.Equal(t, "node1", nodeStatus.Name) - assert.Equal(t, "server-version", nodeStatus.Version) - assert.Equal(t, "git-hash", nodeStatus.GitHash) - assert.Len(t, nodeStatus.Shards, 6) - var testClassShardsCount, testClassObjectsCount int64 - var testRefClassShardsCount, testRefClassObjectsCount int64 - for _, status := range nodeStatus.Shards { - if status.Class == "TestClass" { - testClassShardsCount += 1 - testClassObjectsCount += status.ObjectCount - } - if status.Class == "TestRefClass" { - testRefClassShardsCount += 1 - testRefClassObjectsCount += status.ObjectCount - } - } - assert.Equal(t, int64(3), testClassShardsCount) - assert.Equal(t, int64(20), testClassObjectsCount) - assert.Equal(t, int64(3), testRefClassShardsCount) - assert.Equal(t, int64(0), testRefClassObjectsCount) - assert.Equal(t, int64(20), nodeStatus.Stats.ObjectCount) - assert.Equal(t, int64(6), nodeStatus.Stats.ShardCount) - } -} - -func makeTestBatchDeleteAllObjects(repo *DB) func(t *testing.T) { - return func(t *testing.T) { - performDelete := func(t *testing.T, className string) { - getParams := func(className string, dryRun bool) objects.BatchDeleteParams { - return objects.BatchDeleteParams{ - ClassName: schema.ClassName(className), - Filters: &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorLike, - Value: &filters.Value{ - Value: "*", - Type: schema.DataTypeText, - }, - On: &filters.Path{ - Property: "id", - }, - }, - }, - DryRun: dryRun, - Output: "verbose", - } - } - performClassSearch := func(className string) ([]search.Result, error) { - return repo.Search(context.Background(), dto.GetParams{ - ClassName: className, - Pagination: &filters.Pagination{Limit: 10000}, - }) - } - // get the initial count of the objects - res, err := performClassSearch(className) - require.Nil(t, err) - beforeDelete := len(res) - require.True(t, beforeDelete > 0) - // dryRun == true - batchDeleteRes, err := repo.BatchDeleteObjects(context.Background(), getParams(className, true), nil, "") - require.Nil(t, err) - require.Equal(t, int64(beforeDelete), batchDeleteRes.Matches) - require.Equal(t, beforeDelete, len(batchDeleteRes.Objects)) - for _, batchRes := range batchDeleteRes.Objects { - require.Nil(t, batchRes.Err) - } - // check that every object is preserved (not deleted) - res, err = performClassSearch(className) - require.Nil(t, err) - require.Equal(t, beforeDelete, len(res)) - // dryRun == false, perform actual delete - batchDeleteRes, err = repo.BatchDeleteObjects(context.Background(), getParams(className, false), nil, "") - require.Nil(t, err) - require.Equal(t, int64(beforeDelete), batchDeleteRes.Matches) - require.Equal(t, beforeDelete, len(batchDeleteRes.Objects)) - for _, batchRes := range batchDeleteRes.Objects { - require.Nil(t, batchRes.Err) - } - // check that every object is deleted - res, err = performClassSearch(className) - require.Nil(t, err) - require.Equal(t, 0, len(res)) - } - t.Run("batch delete TestRefClass", func(t *testing.T) { - performDelete(t, "TestRefClass") - }) - t.Run("batch delete TestClass", func(t *testing.T) { - performDelete(t, "TestClass") - }) - } -} - -func exampleQueryVec(r *rand.Rand) []float32 { - dim := 10 - vec := make([]float32, dim) - for j := range vec { - vec[j] = r.Float32() - } - return vec -} - -func multiShardTestData(r *rand.Rand) []*models.Object { - size := 20 - dim := 10 - out := make([]*models.Object, size) - for i := range out { - vec := make([]float32, dim) - for j := range vec { - vec[j] = r.Float32() - } - - out[i] = &models.Object{ - ID: strfmt.UUID(uuid.New().String()), - Class: "TestClass", - Vector: vec, - Properties: map[string]interface{}{ - "boolProp": i%2 == 0, - "index": i, - "indexProp": i, - "stringProp": fmt.Sprintf("s%02d", i), - "textArrayProp": []string{fmt.Sprintf("s%02d", i), fmt.Sprintf("%02d", i)}, - }, - } - } - - return out -} - -func multiShardRefClassData(r *rand.Rand, targets []*models.Object) []*models.Object { - // each class will link to all possible targets, so that we can be sure that - // we hit cross-shard links - targetLinks := make(models.MultipleRef, len(targets)) - for i, obj := range targets { - targetLinks[i] = &models.SingleRef{ - Beacon: strfmt.URI(crossref.NewLocalhost("", obj.ID).String()), - } - } - - size := 20 - dim := 10 - out := make([]*models.Object, size) - for i := range out { - vec := make([]float32, dim) - for j := range vec { - vec[j] = r.Float32() - } - - out[i] = &models.Object{ - ID: strfmt.UUID(uuid.New().String()), - Class: "TestRefClass", - Vector: vec, - Properties: map[string]interface{}{ - "toOther": targetLinks, - }, - } - } - - return out -} - -func bruteForceObjectsByQuery(objs []*models.Object, - query []float32, -) []*models.Object { - type distanceAndObj struct { - distance float32 - obj *models.Object - } - - distProv := distancer.NewDotProductProvider() - distances := make([]distanceAndObj, len(objs)) - - for i := range objs { - dist, _, _ := distProv.SingleDist(normalize(query), normalize(objs[i].Vector)) - distances[i] = distanceAndObj{ - distance: dist, - obj: objs[i], - } - } - - sort.Slice(distances, func(a, b int) bool { - return distances[a].distance < distances[b].distance - }) - - out := make([]*models.Object, len(objs)) - for i := range out { - out[i] = distances[i].obj - } - - return out -} - -func testClassesForImporting() []*models.Class { - return []*models.Class{ - { - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Class: "TestClass", - Properties: []*models.Property{ - { - Name: "boolProp", - DataType: []string{string(schema.DataTypeBoolean)}, - }, - { - Name: "index", - DataType: []string{string(schema.DataTypeInt)}, - }, - { - Name: "indexProp", - DataType: []string{string(schema.DataTypeInt)}, - }, - { - Name: "stringProp", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "textArrayProp", - DataType: []string{string(schema.DataTypeTextArray)}, - }, - }, - }, - { - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Class: "TestRefClass", - Properties: []*models.Property{ - { - Name: "boolProp", - DataType: []string{string(schema.DataTypeBoolean)}, - }, - { - Name: "toOther", - DataType: []string{"TestClass"}, - }, - { - Name: "indexProp", - DataType: []string{string(schema.DataTypeInt)}, - }, - { - Name: "stringProp", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "textArrayProp", - DataType: []string{string(schema.DataTypeTextArray)}, - }, - }, - }, - } -} - -func normalize(v []float32) []float32 { - var norm float32 - for i := range v { - norm += v[i] * v[i] - } - - norm = float32(math.Sqrt(float64(norm))) - for i := range v { - v[i] = v[i] / norm - } - - return v -} diff --git a/adapters/repos/db/nodes.go b/adapters/repos/db/nodes.go deleted file mode 100644 index 59f679ed69a6b572625466bd47a8e63e40a8df06..0000000000000000000000000000000000000000 --- a/adapters/repos/db/nodes.go +++ /dev/null @@ -1,163 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "context" - "fmt" - "sort" - - enterrors "github.com/weaviate/weaviate/entities/errors" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/verbosity" -) - -// GetNodeStatus returns the status of all Weaviate nodes. -func (db *DB) GetNodeStatus(ctx context.Context, className string, verbosity string) ([]*models.NodeStatus, error) { - nodeStatuses := make([]*models.NodeStatus, len(db.schemaGetter.Nodes())) - for i, nodeName := range db.schemaGetter.Nodes() { - status, err := db.getNodeStatus(ctx, nodeName, className, verbosity) - if err != nil { - return nil, fmt.Errorf("node: %v: %w", nodeName, err) - } - if status.Status == nil { - return nil, enterrors.NewErrNotFound( - fmt.Errorf("class %q not found", className)) - } - nodeStatuses[i] = status - } - - sort.Slice(nodeStatuses, func(i, j int) bool { - return nodeStatuses[i].Name < nodeStatuses[j].Name - }) - return nodeStatuses, nil -} - -func (db *DB) getNodeStatus(ctx context.Context, nodeName string, className, output string) (*models.NodeStatus, error) { - if db.schemaGetter.NodeName() == nodeName { - return db.localNodeStatus(className, output), nil - } - status, err := db.remoteNode.GetNodeStatus(ctx, nodeName, className, output) - if err != nil { - switch err.(type) { - case enterrors.ErrOpenHttpRequest, enterrors.ErrSendHttpRequest: - nodeUnavailable := models.NodeStatusStatusUNAVAILABLE - return &models.NodeStatus{Name: nodeName, Status: &nodeUnavailable}, nil - default: - return nil, err - } - } - return status, nil -} - -// IncomingGetNodeStatus returns the index if it exists or nil if it doesn't -func (db *DB) IncomingGetNodeStatus(ctx context.Context, className, verbosity string) (*models.NodeStatus, error) { - return db.localNodeStatus(className, verbosity), nil -} - -func (db *DB) localNodeStatus(className, output string) *models.NodeStatus { - var ( - objectCount int64 - shardCount int64 - shards []*models.NodeShardStatus - ) - - if className != "" && db.GetIndex(schema.ClassName(className)) == nil { - // class not found - return &models.NodeStatus{} - } - - if className == "" { - objectCount, shardCount = db.localNodeStatusAll(&shards, output) - } else { - objectCount, shardCount = db.localNodeStatusForClass(&shards, className, output) - } - - clusterHealthStatus := models.NodeStatusStatusHEALTHY - if db.schemaGetter.ClusterHealthScore() > 0 { - clusterHealthStatus = models.NodeStatusStatusUNHEALTHY - } - db.batchMonitorLock.Lock() - rate := db.ratePerSecond - db.batchMonitorLock.Unlock() - - status := models.NodeStatus{ - Name: db.schemaGetter.NodeName(), - Version: db.config.ServerVersion, - GitHash: db.config.GitHash, - Status: &clusterHealthStatus, - Shards: shards, - Stats: &models.NodeStats{ - ShardCount: shardCount, - ObjectCount: objectCount, - }, - BatchStats: &models.BatchStats{ - RatePerSecond: int64(rate), - }, - } - - if !asyncEnabled() { - ql := int64(len(db.jobQueueCh)) - status.BatchStats.QueueLength = &ql - } - - return &status -} - -func (db *DB) localNodeStatusAll(status *[]*models.NodeShardStatus, output string) (totalCount, shardCount int64) { - db.indexLock.RLock() - defer db.indexLock.RUnlock() - for name, idx := range db.indices { - if idx == nil { - db.logger.WithField("action", "local_node_status_for_all"). - Warningf("no resource found for index %q", name) - continue - } - total, shard := idx.getShardsNodeStatus(status, output) - totalCount, shardCount = totalCount+total, shardCount+shard - } - return -} - -func (db *DB) localNodeStatusForClass(status *[]*models.NodeShardStatus, - className, output string, -) (totalCount, shardCount int64) { - idx := db.GetIndex(schema.ClassName(className)) - if idx == nil { - db.logger.WithField("action", "local_node_status_for_class"). - Warningf("no index found for class %q", className) - return 0, 0 - } - return idx.getShardsNodeStatus(status, output) -} - -func (i *Index) getShardsNodeStatus(status *[]*models.NodeShardStatus, output string) (totalCount, shardCount int64) { - i.ForEachShard(func(name string, shard ShardLike) error { - objectCount := int64(shard.ObjectCount()) - totalCount += objectCount - if output == verbosity.OutputVerbose { - shardStatus := &models.NodeShardStatus{ - Name: name, - Class: shard.Index().Config.ClassName.String(), - ObjectCount: objectCount, - VectorIndexingStatus: shard.GetStatus().String(), - VectorQueueLength: shard.Queue().Size(), - Compressed: shard.VectorIndex().Compressed(), - } - *status = append(*status, shardStatus) - } - shardCount++ - return nil - }) - return -} diff --git a/adapters/repos/db/nodes_integration_test.go b/adapters/repos/db/nodes_integration_test.go deleted file mode 100644 index 7269a3e1d531d364e8ea09510e0f7be31b7d4a71..0000000000000000000000000000000000000000 --- a/adapters/repos/db/nodes_integration_test.go +++ /dev/null @@ -1,142 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package db - -import ( - "context" - "testing" - - "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - enthnsw "github.com/weaviate/weaviate/entities/vectorindex/hnsw" - "github.com/weaviate/weaviate/entities/verbosity" - "github.com/weaviate/weaviate/usecases/objects" -) - -func TestNodesAPI_Journey(t *testing.T) { - dirName := t.TempDir() - - logger := logrus.New() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - ServerVersion: "server-version", - GitHash: "git-hash", - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - TrackVectorDimensions: true, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - - defer repo.Shutdown(context.Background()) - migrator := NewMigrator(repo, logger) - - // check nodes api response on empty DB - nodeStatues, err := repo.GetNodeStatus(context.Background(), "", verbosity.OutputVerbose) - require.Nil(t, err) - require.NotNil(t, nodeStatues) - - require.Len(t, nodeStatues, 1) - nodeStatus := nodeStatues[0] - assert.NotNil(t, nodeStatus) - assert.Equal(t, "node1", nodeStatus.Name) - assert.Equal(t, "server-version", nodeStatus.Version) - assert.Equal(t, "git-hash", nodeStatus.GitHash) - assert.Len(t, nodeStatus.Shards, 0) - assert.Equal(t, int64(0), nodeStatus.Stats.ObjectCount) - assert.Equal(t, int64(0), nodeStatus.Stats.ShardCount) - - // import 2 objects - class := &models.Class{ - Class: "ClassNodesAPI", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Properties: []*models.Property{ - { - Name: "stringProp", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - } - - require.Nil(t, - migrator.AddClass(context.Background(), class, schemaGetter.shardState)) - - schemaGetter.schema.Objects = &models.Schema{ - Classes: []*models.Class{class}, - } - - batch := objects.BatchObjects{ - objects.BatchObject{ - OriginalIndex: 0, - Err: nil, - Object: &models.Object{ - Class: "ClassNodesAPI", - Properties: map[string]interface{}{ - "stringProp": "first element", - }, - ID: "8d5a3aa2-3c8d-4589-9ae1-3f638f506970", - }, - UUID: "8d5a3aa2-3c8d-4589-9ae1-3f638f506970", - }, - objects.BatchObject{ - OriginalIndex: 1, - Err: nil, - Object: &models.Object{ - Class: "ClassNodesAPI", - Properties: map[string]interface{}{ - "stringProp": "second element", - }, - ID: "86a380e9-cb60-4b2a-bc48-51f52acd72d6", - }, - UUID: "86a380e9-cb60-4b2a-bc48-51f52acd72d6", - }, - } - batchRes, err := repo.BatchPutObjects(context.Background(), batch, nil) - require.Nil(t, err) - - assert.Nil(t, batchRes[0].Err) - assert.Nil(t, batchRes[1].Err) - - // check nodes api after importing 2 objects to DB - nodeStatues, err = repo.GetNodeStatus(context.Background(), "", verbosity.OutputVerbose) - require.Nil(t, err) - require.NotNil(t, nodeStatues) - - require.Len(t, nodeStatues, 1) - nodeStatus = nodeStatues[0] - assert.NotNil(t, nodeStatus) - assert.Equal(t, "node1", nodeStatus.Name) - assert.Equal(t, "server-version", nodeStatus.Version) - assert.Equal(t, "git-hash", nodeStatus.GitHash) - assert.Len(t, nodeStatus.Shards, 1) - assert.Equal(t, "ClassNodesAPI", nodeStatus.Shards[0].Class) - assert.True(t, len(nodeStatus.Shards[0].Name) > 0) - assert.Equal(t, int64(2), nodeStatus.Shards[0].ObjectCount) - assert.Equal(t, "READY", nodeStatus.Shards[0].VectorIndexingStatus) - assert.Equal(t, int64(0), nodeStatus.Shards[0].VectorQueueLength) - assert.Equal(t, int64(2), nodeStatus.Stats.ObjectCount) - assert.Equal(t, int64(1), nodeStatus.Stats.ShardCount) -} diff --git a/adapters/repos/db/priorityqueue/queue.go b/adapters/repos/db/priorityqueue/queue.go deleted file mode 100644 index 0af6286476b71ca74ad1f9cdaf5ba61d76599482..0000000000000000000000000000000000000000 --- a/adapters/repos/db/priorityqueue/queue.go +++ /dev/null @@ -1,147 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package priorityqueue - -type supportedValueType interface { - any | uint64 -} - -// Item represents a queue item supporting an optional additional Value -type Item[T supportedValueType] struct { - ID uint64 - Dist float32 - Rescored bool - Value T -} - -// Queue is a priority queue supporting generic item values -type Queue[T supportedValueType] struct { - items []Item[T] - less func(items []Item[T], i, j int) bool -} - -// NewMin constructs a priority queue which prioritizes items with smaller distance -func NewMin[T supportedValueType](capacity int) *Queue[T] { - return &Queue[T]{ - items: make([]Item[T], 0, capacity), - less: func(items []Item[T], i, j int) bool { - return items[i].Dist < items[j].Dist - }, - } -} - -// NewMax constructs a priority queue which prioritizes items with greater distance -func NewMax[T supportedValueType](capacity int) *Queue[T] { - return &Queue[T]{ - items: make([]Item[T], 0, capacity), - less: func(items []Item[T], i, j int) bool { - return items[i].Dist > items[j].Dist - }, - } -} - -// Pop removes the next item in the queue and returns it -func (q *Queue[T]) Pop() Item[T] { - out := q.items[0] - q.items[0] = q.items[len(q.items)-1] - q.items = q.items[:len(q.items)-1] - q.heapify(0) - return out -} - -// Top peeks at the next item in the queue -func (q *Queue[T]) Top() Item[T] { - return q.items[0] -} - -// Len returns the length of the queue -func (q *Queue[T]) Len() int { - return len(q.items) -} - -// Cap returns the remaining capacity of the queue -func (q *Queue[T]) Cap() int { - return cap(q.items) -} - -// Reset clears all items from the queue -func (q *Queue[T]) Reset() { - q.items = q.items[:0] -} - -// ResetCap drops existing queue items, and allocates a new queue with the given capacity -func (q *Queue[T]) ResetCap(capacity int) { - q.items = make([]Item[T], 0, capacity) -} - -// Insert creates a valueless item and adds it to the queue -func (q *Queue[T]) Insert(id uint64, distance float32) int { - item := Item[T]{ - ID: id, - Dist: distance, - } - return q.insert(item) -} - -// InsertWithValue creates an item with a T type value and adds it to the queue -func (q *Queue[T]) InsertWithValue(id uint64, distance float32, val T) int { - item := Item[T]{ - ID: id, - Dist: distance, - Value: val, - } - return q.insert(item) -} - -func (q *Queue[T]) insert(item Item[T]) int { - q.items = append(q.items, item) - i := len(q.items) - 1 - for i != 0 && q.less(q.items, i, q.parent(i)) { - q.swap(i, q.parent(i)) - i = q.parent(i) - } - return i -} - -func (q *Queue[T]) left(i int) int { - return 2*i + 1 -} - -func (q *Queue[T]) right(i int) int { - return 2*i + 2 -} - -func (q *Queue[T]) parent(i int) int { - return (i - 1) / 2 -} - -func (q *Queue[T]) swap(i, j int) { - q.items[i], q.items[j] = q.items[j], q.items[i] -} - -func (q *Queue[T]) heapify(i int) { - left := q.left(i) - right := q.right(i) - smallest := i - if left < len(q.items) && q.less(q.items, left, i) { - smallest = left - } - - if right < len(q.items) && q.less(q.items, right, smallest) { - smallest = right - } - - if smallest != i { - q.swap(i, smallest) - q.heapify(smallest) - } -} diff --git a/adapters/repos/db/priorityqueue/queue_test.go b/adapters/repos/db/priorityqueue/queue_test.go deleted file mode 100644 index a2875f523538ff942aacef14c9f96255ca1ac6f7..0000000000000000000000000000000000000000 --- a/adapters/repos/db/priorityqueue/queue_test.go +++ /dev/null @@ -1,82 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package priorityqueue - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestPriorityQueueMin(t *testing.T) { - values := map[uint64]float32{ - 0: 0.0, - 1: 0.23, - 2: 0.8, - 3: 0.222, - 4: 0.88, - 5: 1, - } - expectedResults := []Item[any]{ - {Dist: 0, ID: 0}, - {Dist: 0.222, ID: 3}, - {Dist: 0.23, ID: 1}, - {Dist: 0.8, ID: 2}, - {Dist: 0.88, ID: 4}, - {Dist: 1, ID: 5}, - } - - pq := NewMin[any](6) - - for id, dist := range values { - pq.Insert(id, dist) - } - - var results []Item[any] - for pq.Len() > 0 { - results = append(results, pq.Pop()) - } - - assert.Equal(t, expectedResults, results) -} - -func TestPriorityQueueMax(t *testing.T) { - values := map[uint64]float32{ - 0: 0.0, - 1: 0.23, - 2: 0.8, - 3: 0.222, - 4: 0.88, - 5: 1, - } - expectedResults := []Item[any]{ - {Dist: 1, ID: 5}, - {Dist: 0.88, ID: 4}, - {Dist: 0.8, ID: 2}, - {Dist: 0.23, ID: 1}, - {Dist: 0.222, ID: 3}, - {Dist: 0, ID: 0}, - } - - pq := NewMax[any](6) - - for id, dist := range values { - pq.Insert(id, dist) - } - - var results []Item[any] - for pq.Len() > 0 { - results = append(results, pq.Pop()) - } - - assert.Equal(t, expectedResults, results) -} diff --git a/adapters/repos/db/propertyspecific/index.go b/adapters/repos/db/propertyspecific/index.go deleted file mode 100644 index 52d8d28a6adcdd4598e4bbc84e83f14781fc17fc..0000000000000000000000000000000000000000 --- a/adapters/repos/db/propertyspecific/index.go +++ /dev/null @@ -1,58 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package propertyspecific - -import ( - "context" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/adapters/repos/db/vector/geo" - "github.com/weaviate/weaviate/entities/schema" -) - -// Index - for now - only supports a Geo index as a property-specific index. -// This could be extended in the future, for example to allow vectorization of -// single properties, as opposed to only allowing vectorization of the entire -// object. -type Index struct { - Name string - Type schema.DataType - GeoIndex *geo.Index -} - -// Indices is a collection of property-specific Indices by propname -type Indices map[string]Index - -// ByProp retrieves a property-specific index by prop name. Second argument is -// false, if the index doesn't exist. -func (i Indices) ByProp(propName string) (Index, bool) { - index, ok := i[propName] - return index, ok -} - -func (i Indices) DropAll(ctx context.Context) error { - for propName, index := range i { - if index.Type != schema.DataTypeGeoCoordinates { - return errors.Errorf("no implementation to delete property %s index of type %v", - propName, index.Type) - } - - if err := index.GeoIndex.Drop(ctx); err != nil { - return errors.Wrapf(err, "drop property %s", propName) - } - - index.GeoIndex = nil - delete(i, propName) - - } - return nil -} diff --git a/adapters/repos/db/ranked_fusion_test.go b/adapters/repos/db/ranked_fusion_test.go deleted file mode 100644 index 459272fecdb2b5114c09543e72808b19d74ca004..0000000000000000000000000000000000000000 --- a/adapters/repos/db/ranked_fusion_test.go +++ /dev/null @@ -1,985 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest - -package db - -import ( - "context" - "encoding/json" - "fmt" - "os" - "strconv" - "testing" - - "github.com/go-openapi/strfmt" - "github.com/google/uuid" - "github.com/sirupsen/logrus" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/dto" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/search" - "github.com/weaviate/weaviate/entities/searchparams" - "github.com/weaviate/weaviate/entities/storobj" - enthnsw "github.com/weaviate/weaviate/entities/vectorindex/hnsw" - "github.com/weaviate/weaviate/usecases/config" - "github.com/weaviate/weaviate/usecases/modules" - "github.com/weaviate/weaviate/usecases/traverser" - "github.com/weaviate/weaviate/usecases/traverser/hybrid" -) - -type TestDoc struct { - DocID string - Document string -} - -type TestQuery struct { - QueryID string - Query string - MatchingDocIDs []string -} - -var defaultConfig = config.Config{ - QueryDefaults: config.QueryDefaults{ - Limit: 100, - }, - QueryMaximumResults: 100, -} - -func SetupStandardTestData(t require.TestingT, repo *DB, schemaGetter *fakeSchemaGetter, logger logrus.FieldLogger, k1, b float32) { - class := &models.Class{ - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: BM25FinvertedConfig(k1, b, "none"), - Class: "StandardTest", - Properties: []*models.Property{ - { - Name: "document", - DataType: []string{string(schema.DataTypeText)}, - Tokenization: "word", - }, - }, - } - - schema := schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{class}, - }, - } - - schemaGetter.schema = schema - - migrator := NewMigrator(repo, logger) - migrator.AddClass(context.Background(), class, schemaGetter.shardState) - - // Load text from file standard_test_data.json - // This is a list of 1000 documents from the MEDLINE database - // Each document is a medical abstract - - data, _ := os.ReadFile("NFCorpus-Corpus.json") - var docs []TestDoc - json.Unmarshal(data, &docs) - - for i, doc := range docs { - id := strfmt.UUID(uuid.MustParse(fmt.Sprintf("%032d", i)).String()) - - data := map[string]interface{}{"document": doc.Document, "code": doc.DocID} - obj := &models.Object{Class: "StandardTest", ID: id, Properties: data, CreationTimeUnix: 1565612833955, LastUpdateTimeUnix: 10000020} - err := repo.PutObject(context.Background(), obj, nil, nil) - require.Nil(t, err) - } -} - -func TestHybrid(t *testing.T) { - dirName := t.TempDir() - logger := logrus.New() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, nil, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(context.TODO())) - defer repo.Shutdown(context.Background()) - - SetupStandardTestData(t, repo, schemaGetter, logger, 1.2, 0.75) - - idx := repo.GetIndex("StandardTest") - require.NotNil(t, idx) - - // Load queries from file standard_test_queries.json - // This is a list of 100 queries from the MEDLINE database - - data, _ := os.ReadFile("NFCorpus-Query.json") - var queries []TestQuery - json.Unmarshal(data, &queries) - for _, query := range queries { - kwr := &searchparams.KeywordRanking{Type: "bm25", Properties: []string{}, Query: query.Query} - addit := additional.Properties{} - res, _, _ := idx.objectSearch(context.TODO(), 1000, nil, kwr, nil, nil, addit, nil, "", 0) - - fmt.Printf("query for %s returned %d results\n", query.Query, len(res)) - - } -} - -func TestBIER(t *testing.T) { - dirName := t.TempDir() - - logger := logrus.New() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, nil, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(context.TODO())) - defer repo.Shutdown(context.Background()) - - SetupStandardTestData(t, repo, schemaGetter, logger, 1.2, 0.75) - - idx := repo.GetIndex("StandardTest") - require.NotNil(t, idx) - - // Load queries from file standard_test_queries.json - // This is a list of 100 queries from the MEDLINE database - - data, _ := os.ReadFile("NFCorpus-Query.json") - var queries []TestQuery - json.Unmarshal(data, &queries) - for _, query := range queries { - kwr := &searchparams.KeywordRanking{Type: "bm25", Properties: []string{}, Query: query.Query} - addit := additional.Properties{} - res, _, _ := idx.objectSearch(context.TODO(), 1000, nil, kwr, nil, nil, addit, nil, "", 0) - - fmt.Printf("query for %s returned %d results\n", query.Query, len(res)) - // fmt.Printf("Results: %v\n", res) - - //for j, doc := range res { - // fmt.Printf("res %v, %v\n", j, doc.Object.GetAdditionalProperty("code")) - //} - - //Check the docIDs are the same - //for j, doc := range res[0:10] { - // fmt.Printf("Result: rank %v, docID %v, score %v (%v)\n", j, doc.Object.GetAdditionalProperty("code"), doc.Score(), doc.Object.GetAdditionalProperty("document")) - // fmt.Printf("Expected: rank %v, docID %v\n", j, query.MatchingDocIDs[j].Object.GetAdditionalProperty("code")) - // require.Equal(t, query.MatchingDocIDs[j], doc.Object.GetAdditionalProperty("code").(string)) - //} - - } -} - -func addObj(repo *DB, i int, props map[string]interface{}, vec []float32) error { - id := strfmt.UUID(uuid.MustParse(fmt.Sprintf("%032d", i)).String()) - - obj := &models.Object{Class: "MyClass", ID: id, Properties: props, CreationTimeUnix: 1565612833955, LastUpdateTimeUnix: 10000020} - vector := vec - err := repo.PutObject(context.Background(), obj, vector, nil) - return err -} - -func SetupFusionClass(t require.TestingT, repo *DB, schemaGetter *fakeSchemaGetter, logger logrus.FieldLogger, k1, b float32) *models.Class { - class := &models.Class{ - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: BM25FinvertedConfig(k1, b, "none"), - Class: "MyClass", - Properties: []*models.Property{ - { - Name: "title", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWord, - }, - { - Name: "description", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWord, - }, - }, - } - - schema := schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{class}, - }, - } - - schemaGetter.schema = schema - - migrator := NewMigrator(repo, logger) - migrator.AddClass(context.Background(), class, schemaGetter.shardState) - - addObj(repo, 0, map[string]interface{}{"title": "Our journey to BM25F", "description": "This is how we get to BM25F"}, []float32{ - -0.04488207, -0.32971063, 0.23568298, 0.004971265, -0.12588727, 0.0036464946, -0.6209942, -0.23247992, -0.19338948, 0.3544481, 0.00050193403, 0.36604947, -0.13813384, -0.126298, 0.05640272, -0.36604303, -0.10976448, 0.196644, -0.02206351, -0.27649152, -0.0050981278, 0.020616557, 0.14605781, 0.093781, -0.25838777, -0.5038474, -0.46846977, 0.13360861, -0.15851927, 0.55075127, 0.34870508, 0.085248806, -0.02763206, -0.07068178, -0.26878145, 0.2814703, -0.33317414, 0.48958343, -0.39648432, 0.606744, 0.12882654, 0.07246548, 0.54059577, 0.19526751, 0.85892624, 0.1485534, 0.19790995, -0.34280643, 0.27512825, 0.105886005, 0.030610563, 0.3811836, 0.18384686, 0.29538825, -0.020791993, -0.31372088, -0.08811446, -0.12979206, 0.30209363, -0.14561261, 0.22077207, -0.40219122, -0.40567216, -0.08740993, -0.31625694, -0.18109407, 0.5411316, -0.09015073, 0.22272661, 0.13575949, -0.36186692, -0.02766613, -0.22463024, 0.15271285, 0.304631, -0.57313913, 0.31346974, -0.118818894, -0.36198893, 0.24609627, 0.47406524, -0.55662453, 0.37812573, -0.2959746, 0.6146945, -0.3934654, 0.30840993, -0.24944904, -0.2063059, -0.48078862, -0.08967737, -0.1273727, 0.1587198, -0.44776592, 0.0048942068, 0.18738478, -0.4544592, -0.4755225, 0.2156486, 0.39935833, 0.25160903, -0.35463294, 0.60699236, -0.12445828, -0.029569108, -0.4983043, 0.44752246, -0.2340386, -0.27559096, 0.67984164, 0.51470226, -0.5285723, 0.0024457057, -0.20425095, -0.4915065, -0.3221788, -0.15558766, 0.20102327, 0.23525643, 0.28365672, 0.58687097, 0.3190138, -0.31130886, 0.053733524, -0.10361888, -0.2598206, 1.5977107, 0.60224503, 0.0074084206, 0.17191416, 0.5619663, 0.41178456, 0.27006987, -0.5354418, -0.054428957, 0.6849038, -0.024342017, 0.43103293, -0.22892271, 0.036829337, -0.103084944, -0.2021301, -0.11352237, -0.17110321, 0.76075333, -0.1755375, 0.029183118, -0.34927735, 0.22040148, -0.18136469, 0.16048056, 0.34151044, -0.048658744, 0.03941434, 0.45190382, 0.103645615, 0.10437423, -0.086864054, 0.523172, -0.59672165, -0.1225319, -0.5800122, 0.2197229, 0.49325037, 0.30607533, 0.012414166, 0.1539727, -0.60095996, 0.05142522, 0.021675617, -0.54661363, -0.0050268047, -0.507448, -0.04522115, -0.77988946, 0.10536073, 0.099219516, 0.40711993, 0.27353838, 0.1728696, -0.4171313, -1.3076599, -0.19778727, -0.23201689, 0.40729725, -0.28640944, 0.06354561, -0.3877251, -0.7938625, 0.29908186, -0.24450836, -0.22622268, 0.32792783, 0.28376722, -0.3685573, 0.031423382, -0.012464195, 0.2254249, 0.26994115, -0.19821979, -0.24086252, 0.24454598, 0.30043048, -0.627896, -0.3355214, -0.14054148, 0.50488055, -0.073988594, -0.31053177, 0.36260405, -0.56093204, 0.12066587, -0.47301888, 0.88418764, 0.09010807, -0.10899238, 0.62317103, 0.27237964, -0.604178, 0.0067386073, 0.1370205, -0.094664395, 0.3479645, 0.25092986, 0.16948108, -0.20874223, -0.54980844, -0.100548536, 0.47177002, -0.4981452, 0.1815202, -0.80878633, -0.076736815, 0.43152434, -0.43210435, -0.28010413, 0.1249095, 0.385616, -0.2984289, -0.006841246, 0.3496464, -0.33298343, 0.06344994, 0.37393335, 0.18608452, -0.10631552, -0.40111285, -0.146849, -0.04161288, -0.31621853, -0.06889858, 0.13343252, -0.11599523, 0.5377954, 0.25938663, -0.43172404, -0.7476662, -0.54316807, 0.0029651164, 0.09958581, 0.0730254, 0.22785394, 0.3276773, 0.01816153, 0.094938636, 0.71604383, -0.09648144, -0.0035640472, -0.5383972, 0.28588042, 0.7625968, -0.22359839, 0.17167832, -0.06235203, -0.32480234, -0.18599075, 0.1570872, -0.06470149, -0.029198159, 0.23251827, 0.100047514, -0.06314679, 0.6390605, -0.06232509, 0.76272035, 0.2975126, 0.15871438, 0.18222457, -0.548036, 0.23633306, -0.17981203, 0.023965335, 0.24478278, -0.21601695, -0.108217336, 0.05834005, 0.3718355, 0.0970174, 0.04476983, -0.118143275, - }) - - addObj(repo, 1, map[string]interface{}{"title": "Our peanuts to BM25F", "description": "This is how we get to BM25F"}, []float32{0.11676752, -0.4837953, -0.06559026, 0.3242706, 0.08680799, -0.30777612, -0.22926088, 0.01667141, 0.31844103, 0.4666344, 0.417305, 0.06108997, -0.0740552, 0.14234918, 0.06823654, 0.16182217, -0.012199775, -0.17269811, -0.16104576, -0.09208117, 0.063624315, 0.3113634, -0.3830663, 0.05831715, -0.14125349, -0.26962206, -0.0696671, -0.013111545, 0.20097807, 0.033809602, -0.048573885, 0.46815604, 0.32582077, 0.32308698, 0.20355524, -0.08757271, 0.17099291, 0.31500003, -0.05445185, 0.7712824, -0.2096038, 0.28787872, 0.10871067, -0.3266944, -0.1633618, 0.34630018, -0.15387866, -0.45506623, -0.21508889, -0.19249445, -0.28801772, -0.2694916, -0.18476918, -0.12890251, -0.29947013, 0.0008435306, -0.06490287, -0.006560939, 0.24637267, -0.111215346, 0.3775517, -0.82433224, -0.3179537, 0.022306278, 0.19248968, -0.1701471, 0.052865, -0.044782564, -0.10222186, 0.09571932, -0.19251339, 0.241193, -0.13216764, -0.19301765, 0.46628228, -0.29973802, 0.0030274116, 0.01664786, 0.1216316, 0.12837356, -0.048461247, -0.56439394, 0.06110007, 0.102808535, 0.63137263, -0.13134736, 0.41365498, -0.113528065, -0.06924132, 0.1076709, -0.06833764, 0.31522226, 0.13445137, -0.16227263, -0.15102008, 0.23768687, -0.41108298, -0.473573, -0.35702798, 0.21465969, -0.30590045, -0.26616427, 0.7287231, -0.036261655, -0.34903425, -0.1396425, -0.022058574, -0.33956096, -0.3359471, -0.035496157, -0.1786069, -0.0857123, -0.0845917, 0.13232024, -0.02890402, -0.45281035, -0.026353035, -0.39124215, -0.15753527, -0.075793914, 0.35795033, 0.35925874, -0.1423145, -0.0969307, 0.08920737, 0.15772092, 1.3536518, 0.29779792, -0.05407743, -0.048793554, 0.12263066, -0.06248072, -0.49598575, -0.46484944, -0.31050035, 0.6283043, 0.5242193, 0.25987545, -0.2584134, 0.32898954, 0.014580286, 0.14016634, -0.010093123, -0.22610027, -0.029830063, 0.18112054, 0.020298548, -0.025797658, -0.40394786, -0.17097965, 0.11640611, 0.29304397, -0.27026933, -0.14832975, -0.099585906, 0.4554175, -0.0018298444, 0.23190805, -0.65866566, -0.09366216, -0.7000203, 0.004698127, -0.17523476, -0.34830904, -0.16284281, 0.15495956, 0.5772887, 0.048939474, -0.12923703, -0.236143, -0.03874896, 0.2960667, 0.029154046, 0.42814374, -0.4332385, -0.31293675, -0.10682973, -0.12069777, 0.071893886, 0.06644212, -0.46342105, -0.8599067, 0.017380634, -0.38347453, 0.14165273, -0.08906643, -0.06801824, 0.19660597, -0.06807183, 0.33882818, 0.044932134, 0.27550527, 0.2308957, -0.101730466, -0.19064885, -0.015364495, 0.0149245, 0.24177131, -0.15636654, -0.002376896, -0.6399841, 0.14845476, 0.46339074, 0.036926877, -0.067630276, 0.289784, 0.15529989, -0.5235124, 0.50196457, -0.004536148, -0.3716798, 0.047304608, -0.027990041, 0.15901157, 0.021176483, 0.35387334, 0.4457043, 0.094738215, 0.08722517, 0.0450516, 0.1739127, -0.2606226, 0.035999063, -0.12919275, -0.11809982, -0.20865, -0.6917279, 0.093973815, -0.38069052, -0.114874505, -0.3051481, -0.357749, 0.48254266, 0.31795567, 0.37491056, 0.0047062743, -0.1265727, 0.51655954, 0.1622121, 0.39811996, -0.002116253, -0.375531, 0.6347343, 0.14833164, 0.032251768, 0.021101426, -0.34346518, 0.22451165, 0.028649824, -0.04794777, 0.056036226, 0.14179966, 0.32724753, 0.17185552, 0.2504634, -0.05013007, -0.31430584, -0.22200464, -0.508279, -0.10017326, 0.16302426, -0.09568865, 0.05985463, -0.22916546, -0.084666654, -0.15271503, -0.24385636, -0.028514259, -0.33194387, -0.17132543, -0.1474212, -0.18526097, 0.2198915, -0.1689729, -0.19907063, 0.19941927, -0.47478884, 0.0695081, 0.3741401, 0.19423902, 0.085894205, -0.53214043, 0.33309302, 0.18701339, 0.23461546, -0.14038202, 0.07201847, 0.3462437, 0.1640635, 0.07200127, -0.09130982, 0.3868172, -0.09754013, 0.040958565, -0.18743117, 0.14117524, -0.18739408, 0.13669269, -0.09902989, -0.16762646}) - - addObj(repo, 2, map[string]interface{}{"title": "Elephant Parade", "description": "Elephants elephants elephant"}, []float32{-0.04488207, -0.32971063, 0.23568298, 0.004971265, -0.12588727, 0.0036464946, -0.6209942, -0.23247992, -0.19338948, 0.3544481, 0.00050193403, 0.36604947, -0.13813384, -0.126298, 0.05640272, -0.36604303, -0.10976448, 0.196644, -0.02206351, -0.27649152, -0.0050981278, 0.020616557, 0.14605781, 0.093781, -0.25838777, -0.5038474, -0.46846977, 0.13360861, -0.15851927, 0.55075127, 0.34870508, 0.085248806, -0.02763206, -0.07068178, -0.26878145, 0.2814703, -0.33317414, 0.48958343, -0.39648432, 0.606744, 0.12882654, 0.07246548, 0.54059577, 0.19526751, 0.85892624, 0.1485534, 0.19790995, -0.34280643, 0.27512825, 0.105886005, 0.030610563, 0.3811836, 0.18384686, 0.29538825, -0.020791993, -0.31372088, -0.08811446, -0.12979206, 0.30209363, -0.14561261, 0.22077207, -0.40219122, -0.40567216, -0.08740993, -0.31625694, -0.18109407, 0.5411316, -0.09015073, 0.22272661, 0.13575949, -0.36186692, -0.02766613, -0.22463024, 0.15271285, 0.304631, -0.57313913, 0.31346974, -0.118818894, -0.36198893, 0.24609627, 0.47406524, -0.55662453, 0.37812573, -0.2959746, 0.6146945, -0.3934654, 0.30840993, -0.24944904, -0.2063059, -0.48078862, -0.08967737, -0.1273727, 0.1587198, -0.44776592, 0.0048942068, 0.18738478, -0.4544592, -0.4755225, 0.2156486, 0.39935833, 0.25160903, -0.35463294, 0.60699236, -0.12445828, -0.029569108, -0.4983043, 0.44752246, -0.2340386, -0.27559096, 0.67984164, 0.51470226, -0.5285723, 0.0024457057, -0.20425095, -0.4915065, -0.3221788, -0.15558766, 0.20102327, 0.23525643, 0.28365672, 0.58687097, 0.3190138, -0.31130886, 0.053733524, -0.10361888, -0.2598206, 1.5977107, 0.60224503, 0.0074084206, 0.17191416, 0.5619663, 0.41178456, 0.27006987, -0.5354418, -0.054428957, 0.6849038, -0.024342017, 0.43103293, -0.22892271, 0.036829337, -0.103084944, -0.2021301, -0.11352237, -0.17110321, 0.76075333, -0.1755375, 0.029183118, -0.34927735, 0.22040148, -0.18136469, 0.16048056, 0.34151044, -0.048658744, 0.03941434, 0.45190382, 0.103645615, 0.10437423, -0.086864054, 0.523172, -0.59672165, -0.1225319, -0.5800122, 0.2197229, 0.49325037, 0.30607533, 0.012414166, 0.1539727, -0.60095996, 0.05142522, 0.021675617, -0.54661363, -0.0050268047, -0.507448, -0.04522115, -0.77988946, 0.10536073, 0.099219516, 0.40711993, 0.27353838, 0.1728696, -0.4171313, -1.3076599, -0.19778727, -0.23201689, 0.40729725, -0.28640944, 0.06354561, -0.3877251, -0.7938625, 0.29908186, -0.24450836, -0.22622268, 0.32792783, 0.28376722, -0.3685573, 0.031423382, -0.012464195, 0.2254249, 0.26994115, -0.19821979, -0.24086252, 0.24454598, 0.30043048, -0.627896, -0.3355214, -0.14054148, 0.50488055, -0.073988594, -0.31053177, 0.36260405, -0.56093204, 0.12066587, -0.47301888, 0.88418764, 0.09010807, -0.10899238, 0.62317103, 0.27237964, -0.604178, 0.0067386073, 0.1370205, -0.094664395, 0.3479645, 0.25092986, 0.16948108, -0.20874223, -0.54980844, -0.100548536, 0.47177002, -0.4981452, 0.1815202, -0.80878633, -0.076736815, 0.43152434, -0.43210435, -0.28010413, 0.1249095, 0.385616, -0.2984289, -0.006841246, 0.3496464, -0.33298343, 0.06344994, 0.37393335, 0.18608452, -0.10631552, -0.40111285, -0.146849, -0.04161288, -0.31621853, -0.06889858, 0.13343252, -0.11599523, 0.5377954, 0.25938663, -0.43172404, -0.7476662, -0.54316807, 0.0029651164, 0.09958581, 0.0730254, 0.22785394, 0.3276773, 0.01816153, 0.094938636, 0.71604383, -0.09648144, -0.0035640472, -0.5383972, 0.28588042, 0.7625968, -0.22359839, 0.17167832, -0.06235203, -0.32480234, -0.18599075, 0.1570872, -0.06470149, -0.029198159, 0.23251827, 0.100047514, -0.06314679, 0.6390605, -0.06232509, 0.76272035, 0.2975126, 0.15871438, 0.18222457, -0.548036, 0.23633306, -0.17981203, 0.023965335, 0.24478278, -0.21601695, -0.108217336, 0.05834005, 0.3718355, 0.0970174, 0.04476983, -0.118143275}) - - return class -} - -func TestRFJourney(t *testing.T) { - dirName := t.TempDir() - - logger := logrus.New() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - QueryLimit: 20, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, nil, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(context.TODO())) - defer repo.Shutdown(context.Background()) - - class := SetupFusionClass(t, repo, schemaGetter, logger, 1.2, 0.75) - idx := repo.GetIndex("MyClass") - require.NotNil(t, idx) - - doc1 := &hybrid.Result{ - Result: &search.Result{ - ID: strfmt.UUID("e6f7e8b1-ac53-48eb-b6e4-cbe67396bcfa"), - Schema: map[string]interface{}{ - "title": "peanuts", - }, - Vector: []float32{0.1, 0.2, 0.3, 0.4, 0.5}, - Score: 0.1, - }, - } - - doc2 := &hybrid.Result{ - Result: &search.Result{ - ID: strfmt.UUID("2b7a8bc9-29d9-4cc8-b145-a0baf5fc231d"), - Schema: map[string]interface{}{ - "title": "journey", - }, - Vector: []float32{0.5, 0.4, 0.3, 0.3, 0.1}, - Score: 0.2, - }, - } - - doc3 := &hybrid.Result{ - Result: &search.Result{ - ID: strfmt.UUID("dddddddd-29d9-4cc8-b145-a0baf5fc231d"), - Schema: map[string]interface{}{ - "title": "alalala", - }, - Vector: []float32{0.5, 0.4, 0.3, 0.3, 0.1}, - Score: 0.2, - }, - } - - resultSet1 := []*hybrid.Result{doc1, doc2, doc3} - resultSet2 := []*hybrid.Result{doc2, doc1, doc3} - - t.Run("Fusion Reciprocal", func(t *testing.T) { - results := hybrid.FusionRanked([]float64{0.4, 0.6}, - [][]*hybrid.Result{resultSet1, resultSet2}, []string{"set1", "set2"}) - fmt.Println("--- Start results for Fusion Reciprocal ---") - for _, result := range results { - schema := result.Schema.(map[string]interface{}) - fmt.Println(schema["title"], result.ID, result.Score) - } - require.Equal(t, 3, len(results)) - require.Equal(t, resultSet2[0].ID, results[0].ID) - require.Equal(t, resultSet2[1].ID, results[1].ID) - require.Equal(t, resultSet2[2].ID, results[2].ID) - require.Equal(t, float32(0.016287679), results[0].Score) - require.Equal(t, float32(0.016234796), results[1].Score) - }) - - t.Run("Fusion Reciprocal 2", func(t *testing.T) { - results := hybrid.FusionRanked([]float64{0.8, 0.2}, - [][]*hybrid.Result{resultSet1, resultSet2}, []string{"set1", "set2"}) - fmt.Println("--- Start results for Fusion Reciprocal ---") - for _, result := range results { - schema := result.Schema.(map[string]interface{}) - fmt.Println(schema["title"], result.ID, result.Score) - } - require.Equal(t, 3, len(results)) - require.Equal(t, resultSet2[0].ID, results[1].ID) - require.Equal(t, resultSet2[1].ID, results[0].ID) - require.Equal(t, resultSet2[2].ID, results[2].ID) - require.Equal(t, float32(0.016340561), results[0].Score) - require.Equal(t, float32(0.016181914), results[1].Score) - }) - - t.Run("Vector Only", func(t *testing.T) { - results := hybrid.FusionRanked([]float64{0.0, 1.0}, - [][]*hybrid.Result{resultSet1, resultSet2}, []string{"set1", "set2"}) - fmt.Println("--- Start results for Fusion Reciprocal ---") - for _, result := range results { - schema := result.Schema.(map[string]interface{}) - fmt.Println(schema["title"], result.ID, result.Score) - } - require.Equal(t, 3, len(results)) - require.Equal(t, resultSet2[0].ID, results[0].ID) - require.Equal(t, resultSet2[1].ID, results[1].ID) - require.Equal(t, resultSet2[2].ID, results[2].ID) - require.Equal(t, float32(0.016393442), results[0].Score) - require.Equal(t, float32(0.016129032), results[1].Score) - }) - - t.Run("BM25 only", func(t *testing.T) { - results := hybrid.FusionRanked([]float64{1.0, 0.0}, - [][]*hybrid.Result{resultSet1, resultSet2}, []string{"set1", "set2"}) - fmt.Println("--- Start results for Fusion Reciprocal ---") - for _, result := range results { - schema := result.Schema.(map[string]interface{}) - fmt.Println(schema["title"], result.ID, result.Score) - } - require.Equal(t, 3, len(results)) - require.Equal(t, resultSet1[0].ID, results[0].ID) - require.Equal(t, resultSet1[1].ID, results[1].ID) - require.Equal(t, resultSet1[2].ID, results[2].ID) - require.Equal(t, float32(0.016393442), results[0].Score) - require.Equal(t, float32(0.016129032), results[1].Score) - }) - - // Check basic search with one property - results_set_1, err := repo.VectorSearch(context.TODO(), dto.GetParams{ - ClassName: "MyClass", - SearchVector: peanutsVector(), - Pagination: &filters.Pagination{ - Offset: 0, - Limit: 6, - }, - }) - - require.Nil(t, err) - results_set_2, err := repo.VectorSearch(context.TODO(), dto.GetParams{ - ClassName: "MyClass", - SearchVector: journeyVector(), - Pagination: &filters.Pagination{ - Offset: 0, - Limit: 6, - }, - }) - require.Nil(t, err) - - // convert search.Result to hybrid.Result - var results_set_1_hybrid []*hybrid.Result - for _, r := range results_set_1 { - // parse the last 12 digits of the id to get the uint64 - id, err := strconv.Atoi(string(r.ID)[len(r.ID)-12:]) - if err != nil { - fmt.Println(err) - } - - results_set_1_hybrid = append(results_set_1_hybrid, &hybrid.Result{ - DocID: uint64(id), - Result: &r, - }) - } - - var results_set_2_hybrid []*hybrid.Result - for _, r := range results_set_2 { - // parse the last 12 digits of the id to get the uint64 - id, err := strconv.Atoi(string(r.ID)[len(r.ID)-12:]) - if err != nil { - fmt.Println(err) - } - - results_set_2_hybrid = append(results_set_2_hybrid, &hybrid.Result{ - DocID: uint64(id), - Result: &r, - }) - } - - res := hybrid.FusionRanked([]float64{0.2, 0.8}, [][]*hybrid.Result{results_set_1_hybrid, results_set_2_hybrid}, []string{"set1", "set2"}) - fmt.Println("--- Start results for Fusion Reciprocal (", len(res), ")---") - for _, r := range res { - - schema := r.Schema.(map[string]interface{}) - title := schema["title"].(string) - description := schema["description"].(string) - fmt.Printf("Result id: %v, score: %v, title: %v, description: %v, additional %+v\n", r.ID, r.Score, title, description, r.AdditionalProperties) - } - - require.Equal(t, "00000000-0000-0000-0000-000000000000", string(res[0].ID)) - - t.Run("Hybrid", func(t *testing.T) { - params := dto.GetParams{ - ClassName: "MyClass", - HybridSearch: &searchparams.HybridSearch{ - Query: "elephant", - Vector: elephantVector(), - Alpha: 0.5, - }, - Pagination: &filters.Pagination{ - Offset: 0, - Limit: 6, - }, - } - - prov := modules.NewProvider() - prov.SetClassDefaults(class) - - metrics := &fakeMetrics{} - log, _ := test.NewNullLogger() - explorer := traverser.NewExplorer(repo, log, prov, metrics, defaultConfig) - hybridResults, err := explorer.Hybrid(context.TODO(), params) - require.Nil(t, err) - - fmt.Println("--- Start results for hybrid ---") - for _, r := range hybridResults { - schema := r.Schema.(map[string]interface{}) - title := schema["title"].(string) - description := schema["description"].(string) - fmt.Printf("Result id: %v, score: %v, title: %v, description: %v, additional %+v\n", r.ID, r.Score, title, description, r.AdditionalProperties) - } - }) - - t.Run("Hybrid with negative limit", func(t *testing.T) { - params := dto.GetParams{ - ClassName: "MyClass", - HybridSearch: &searchparams.HybridSearch{ - Query: "Elephant Parade", - Vector: elephantVector(), - Alpha: 0.5, - }, - Pagination: &filters.Pagination{ - Offset: 0, - Limit: -1, - }, - } - - prov := modules.NewProvider() - prov.SetClassDefaults(class) - - metrics := &fakeMetrics{} - log, _ := test.NewNullLogger() - explorer := traverser.NewExplorer(repo, log, prov, metrics, defaultConfig) - hybridResults, err := explorer.Hybrid(context.TODO(), params) - - fmt.Println("--- Start results for hybrid with negative limit ---") - for _, r := range hybridResults { - schema := r.Schema.(map[string]interface{}) - title := schema["title"].(string) - description := schema["description"].(string) - fmt.Printf("Result id: %v, score: %v, title: %v, description: %v, additional %+v\n", r.ID, r.Score, title, description, r.AdditionalProperties) - } - require.Nil(t, err) - require.True(t, len(hybridResults) > 0) - }) - - t.Run("Hybrid with offset", func(t *testing.T) { - params := dto.GetParams{ - ClassName: "MyClass", - HybridSearch: &searchparams.HybridSearch{ - Query: "Elephant Parade", - Vector: elephantVector(), - Alpha: 0.5, - }, - Pagination: &filters.Pagination{ - Offset: 2, - Limit: 1, - }, - } - - prov := modules.NewProvider() - prov.SetClassDefaults(class) - - metrics := &fakeMetrics{} - log, _ := test.NewNullLogger() - explorer := traverser.NewExplorer(repo, log, prov, metrics, defaultConfig) - hybridResults, err := explorer.Hybrid(context.TODO(), params) - - fmt.Println("--- Start results for hybrid with offset ---") - for _, r := range hybridResults { - schema := r.Schema.(map[string]interface{}) - title := schema["title"].(string) - description := schema["description"].(string) - fmt.Printf("Result id: %v, score: %v, title: %v, description: %v, additional %+v\n", r.ID, r.Score, title, description, r.AdditionalProperties) - } - - require.Nil(t, err) - require.True(t, len(hybridResults) == 1) - require.True(t, hybridResults[0].ID == "00000000-0000-0000-0000-000000000001") - }) - - t.Run("Hybrid with offset", func(t *testing.T) { - params := dto.GetParams{ - ClassName: "MyClass", - HybridSearch: &searchparams.HybridSearch{ - Query: "Elephant Parade", - Vector: elephantVector(), - Alpha: 0.5, - }, - Pagination: &filters.Pagination{ - Offset: 4, - Limit: 1, - }, - } - - prov := modules.NewProvider() - prov.SetClassDefaults(class) - - metrics := &fakeMetrics{} - log, _ := test.NewNullLogger() - explorer := traverser.NewExplorer(repo, log, prov, metrics, defaultConfig) - hybridResults, err := explorer.Hybrid(context.TODO(), params) - - fmt.Println("--- Start results for hybrid with offset ---") - for _, r := range hybridResults { - schema := r.Schema.(map[string]interface{}) - title := schema["title"].(string) - description := schema["description"].(string) - fmt.Printf("Result id: %v, score: %v, title: %v, description: %v, additional %+v\n", r.ID, r.Score, title, description, r.AdditionalProperties) - } - - require.Nil(t, err) - require.True(t, len(hybridResults) == 0) - }) -} - -func TestRFJourneyWithFilters(t *testing.T) { - dirName := t.TempDir() - - logger := logrus.New() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - QueryLimit: 20, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, nil, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(context.TODO())) - defer repo.Shutdown(context.Background()) - - class := SetupFusionClass(t, repo, schemaGetter, logger, 1.2, 0.75) - idx := repo.GetIndex("MyClass") - require.NotNil(t, idx) - - filter := &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorOr, - Operands: []filters.Clause{ - { - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: schema.ClassName("MyClass"), - Property: schema.PropertyName("title"), - }, - Value: &filters.Value{ - Value: "elephant", - Type: schema.DataTypeText, - }, - }, - { - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: schema.ClassName("MyClass"), - Property: schema.PropertyName("title"), - }, - Value: &filters.Value{ - Value: "elephant", - Type: schema.DataTypeText, - }, - }, - }, - }, - } - - filter1 := &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorOr, - Operands: []filters.Clause{ - { - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: schema.ClassName("MyClass"), - Property: schema.PropertyName("title"), - }, - Value: &filters.Value{ - Value: "My", - Type: schema.DataTypeText, - }, - }, - { - Operator: filters.OperatorEqual, - On: &filters.Path{ - Class: schema.ClassName("MyClass"), - Property: schema.PropertyName("title"), - }, - Value: &filters.Value{ - Value: "journeys", - Type: schema.DataTypeText, - }, - }, - }, - }, - } - - t.Run("Hybrid with filter - no results expected", func(t *testing.T) { - params := dto.GetParams{ - ClassName: "MyClass", - HybridSearch: &searchparams.HybridSearch{ - Query: "elephant", - Vector: elephantVector(), - Alpha: 0.5, - }, - Pagination: &filters.Pagination{ - Offset: 0, - Limit: 100, - }, - Filters: filter1, - } - - prov := modules.NewProvider() - prov.SetClassDefaults(class) - - metrics := &fakeMetrics{} - log, _ := test.NewNullLogger() - explorer := traverser.NewExplorer(repo, log, prov, metrics, defaultConfig) - hybridResults, err := explorer.Hybrid(context.TODO(), params) - require.Nil(t, err) - require.Equal(t, 0, len(hybridResults)) - }) - - t.Run("Hybrid", func(t *testing.T) { - params := dto.GetParams{ - ClassName: "MyClass", - HybridSearch: &searchparams.HybridSearch{ - Query: "elephant", - Vector: elephantVector(), - Alpha: 0.5, - }, - Pagination: &filters.Pagination{ - Offset: 0, - Limit: -1, - }, - } - - prov := modules.NewProvider() - prov.SetClassDefaults(class) - - metrics := &fakeMetrics{} - log, _ := test.NewNullLogger() - explorer := traverser.NewExplorer(repo, log, prov, metrics, defaultConfig) - hybridResults, err := explorer.Hybrid(context.TODO(), params) - require.Nil(t, err) - require.Equal(t, 3, len(hybridResults)) - - fmt.Println("--- Start results for hybrid ---") - for _, r := range hybridResults { - schema := r.Schema.(map[string]interface{}) - title := schema["title"].(string) - description := schema["description"].(string) - fmt.Printf("Result id: %v, score: %v, title: %v, description: %v, additional %+v\n", r.ID, r.Score, title, description, r.AdditionalProperties) - } - require.Equal(t, strfmt.UUID("00000000-0000-0000-0000-000000000002"), hybridResults[0].ID) - }) - - t.Run("Hybrid with filter", func(t *testing.T) { - params := dto.GetParams{ - ClassName: "MyClass", - HybridSearch: &searchparams.HybridSearch{ - Query: "elephant", - Vector: elephantVector(), - Alpha: 0.5, - }, - Pagination: &filters.Pagination{ - Offset: 0, - Limit: -1, - }, - Filters: filter, - } - - prov := modules.NewProvider() - prov.SetClassDefaults(class) - - metrics := &fakeMetrics{} - log, _ := test.NewNullLogger() - explorer := traverser.NewExplorer(repo, log, prov, metrics, defaultConfig) - hybridResults, err := explorer.Hybrid(context.TODO(), params) - require.Nil(t, err) - require.Equal(t, 1, len(hybridResults)) - - fmt.Println("--- Start results for hybrid ---") - for _, r := range hybridResults { - schema := r.Schema.(map[string]interface{}) - title := schema["title"].(string) - description := schema["description"].(string) - fmt.Printf("Result id: %v, score: %v, title: %v, description: %v, additional %+v\n", r.ID, r.Score, title, description, r.AdditionalProperties) - } - require.Equal(t, strfmt.UUID("00000000-0000-0000-0000-000000000002"), hybridResults[0].ID) - }) -} - -func TestStability(t *testing.T) { - dirName := t.TempDir() - - logger := logrus.New() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - QueryLimit: 20, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, nil, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(context.TODO())) - defer repo.Shutdown(context.Background()) - - SetupFusionClass(t, repo, schemaGetter, logger, 1.2, 0.75) - idx := repo.GetIndex("MyClass") - require.NotNil(t, idx) - - doc1 := &hybrid.Result{ - Result: &search.Result{ - ID: strfmt.UUID("e6f7e8b1-ac53-48eb-b6e4-cbe67396bcfa"), - Schema: map[string]interface{}{ - "title": "peanuts", - }, - Vector: []float32{0.1, 0.2, 0.3, 0.4, 0.5}, - Score: 0.1, - }, - } - - doc2 := &hybrid.Result{ - Result: &search.Result{ - ID: strfmt.UUID("e6f7e8b1-ac53-48eb-b6e4-cbe67396bcfb"), - Schema: map[string]interface{}{ - "title": "peanuts", - }, - Vector: []float32{0.1, 0.2, 0.3, 0.4, 0.5}, - Score: 0.1, - }, - } - - doc3 := &hybrid.Result{ - Result: &search.Result{ - ID: strfmt.UUID("e6f7e8b1-ac53-48eb-b6e4-cbe67396bcfc"), - Schema: map[string]interface{}{ - "title": "peanuts", - }, - Vector: []float32{0.1, 0.2, 0.3, 0.4, 0.5}, - Score: 0.1, - }, - } - - resultSet1 := []*hybrid.Result{doc1, doc2, doc3} - resultSet2 := []*hybrid.Result{doc2, doc1, doc3} - - t.Run("Fusion Reciprocal", func(t *testing.T) { - results := hybrid.FusionRanked([]float64{0.4, 0.6}, - [][]*hybrid.Result{resultSet1, resultSet2}, []string{"set1", "set2"}) - fmt.Println("--- Start results for Fusion Reciprocal ---") - for _, result := range results { - schema := result.Schema.(map[string]interface{}) - fmt.Println(schema["title"], result.ID, result.Score) - } - require.Equal(t, 3, len(results)) - require.Equal(t, resultSet2[0].ID, results[0].ID) - require.Equal(t, resultSet2[1].ID, results[1].ID) - require.Equal(t, resultSet2[2].ID, results[2].ID) - }) -} - -func elephantVector() []float32 { - return []float32{ - -0.106136, -0.021716, 0.632442, 0.195315, -0.038854, -0.260533, -0.728847, -0.313725, -0.161967, 0.179243, -0.124185, 0.158839, 0.09563, -0.071267, 0.073928, -0.096735, 0.27266, -0.204127, -0.387028, -0.361406, -0.278027, 0.298766, 0.265405, 0.037477, -0.079904, -0.778953, -0.525643, -0.052346, -0.2174, 0.095746, 0.610937, 0.315672, -0.125526, 0.013475, -0.075578, -0.053183, -0.381475, 0.620278, -0.093857, 0.802608, -0.105773, -0.007902, 0.663528, 0.407708, 0.753832, 0.420718, 0.139289, -0.126864, 0.36345, -0.039222, 0.089002, 0.092151, 0.138025, 0.18881, 0.51416, -0.391045, -0.169528, -0.044023, 0.437196, -0.23917, 0.081247, -0.440846, -0.484764, 0.090495, 0.001852, -0.03441, 0.18548, -0.440182, 0.286827, -0.081451, 0.030155, -0.072746, -0.366531, 0.354118, 0.418432, -0.305682, 0.515893, -0.424999, -0.495273, 0.731375, 0.358407, -0.415989, 0.441337, -0.022167, 0.318837, -0.473018, 0.342046, -0.499794, -0.303161, -0.379234, -0.279082, -0.325648, 0.200613, -0.457396, 0.116745, 0.225836, -0.322175, -0.151425, 0.322014, 0.077097, 0.049998, -0.01005, 0.489028, -0.273297, 0.218896, -0.507729, 0.488891, -0.207774, -0.499136, 0.992803, 0.379556, -0.572352, -0.295821, -0.071392, -0.625823, -0.425159, 0.024593, 0.307965, 0.311686, 0.287844, 0.435028, 0.454474, -0.208158, -0.111947, -0.380334, -0.392014, 1.747561, 0.360315, 0.472088, 0.273835, 0.635424, 0.390057, -0.021349, -0.746944, 0.265353, 0.60709, -0.171053, 0.408823, -0.059646, 0.058306, -0.062817, -0.41064, -0.342016, -0.048077, 0.862758, -0.217101, -0.048961, -0.314094, 0.228395, -0.339353, 0.558551, 0.370054, -0.319855, 0.543137, 0.71334, 0.166296, 0.040412, -0.160482, 0.432088, -0.491292, 0.072819, -0.409627, 0.300197, 0.169077, 0.44379, 0.117131, 0.142459, -0.482226, -0.100245, 0.058273, -0.590567, -0.061971, -0.415718, -0.018105, -0.693528, -0.047609, -0.041873, 0.606186, 0.19767, -0.091001, -0.315381, -1.234111, 0.228805, -0.636861, 0.208757, -0.270024, -0.259684, -0.351592, -0.978549, 0.683986, -0.331669, -0.078729, 0.385676, 0.390955, -0.901898, -0.071451, -0.103991, 0.206379, 0.469656, 0.071528, -0.152589, 0.282268, 0.539651, -0.856463, -0.344053, -0.40572, 0.771483, -0.065611, -0.408832, 0.303948, -0.565157, 0.153293, -0.699892, 1.112725, 0.259508, 0.135771, 0.484552, 0.151274, -0.743235, 0.069811, 0.137583, 0.212661, 0.376839, 0.136164, 0.145626, -0.466645, -0.474334, -0.365033, 0.251158, -0.313904, 0.210487, -1.016155, 0.262768, 0.432895, -0.291339, -0.221825, 0.513278, 0.659038, -0.401398, -0.164522, 0.395279, -0.449811, 0.076142, 0.389243, 0.076184, 0.05539, -0.597094, -0.149824, 0.206724, -0.477001, -0.315719, 0.166689, -0.357187, 0.34429, 0.256624, -0.236781, -0.713059, -0.440255, 0.27353, -0.032257, 0.06925, 0.359134, -0.088975, 0.112507, -0.071103, 0.880417, 0.528587, 0.155656, -0.720531, 0.3068, 0.754715, 0.009366, 0.067487, -0.11898, -0.471064, -0.396507, 0.298669, 0.038283, 0.057218, -0.075818, -0.01513, -0.319236, 0.692123, -0.122985, 0.875938, 0.378184, 0.427029, 0.315545, -0.549573, 0.389602, -0.017071, 0.160122, 0.368208, 0.060474, -0.199651, 0.087829, 0.447339, 0.012265, -0.095388, -0.07034, - } -} - -// "journey" -func journeyVector() []float32 { - return []float32{ - -0.523002, 0.14169, 0.016461, -0.069062, 0.487908, -0.024193, -0.282436, 0.004778, -0.378135, 0.396011, 0.094045, -0.06584, 0.061162, -0.600018, -0.110189, 0.244562, 0.433501, 0.303775, -0.451004, -0.453709, 0.350324, 0.2047, -0.091615, -0.282805, -0.232953, -0.215143, 0.333113, -0.126952, -0.639225, 0.101498, 0.232343, 0.58831, 0.971, 0.494446, -0.483305, -0.873438, -0.483694, 0.406465, 0.342816, 1.253387, -0.24718, -0.046063, -0.660406, 0.103386, -0.06063, 0.3422, 0.322542, 0.026074, -0.623612, 0.489793, -0.632363, 0.448922, -0.370049, 0.212377, -0.315855, 0.364525, 0.056798, 0.805679, 0.145633, 0.850648, 0.432728, -1.431841, -0.226569, -0.315194, 0.560742, 0.261859, -0.001653, -0.068738, -0.662729, -0.049259, -0.380322, -0.374194, 0.363328, 0.341796, -0.077566, 0.503337, 0.353664, -0.045754, -0.499081, 0.198603, 0.038837, -0.460198, 0.00735, -0.270993, 0.950923, -0.085815, -0.52167, -0.10439, 0.31398, -0.560229, 0.411738, -0.129033, -0.009998, 0.443882, -0.045643, -0.078445, -0.259311, -0.08337, 0.232652, -0.015912, -0.229458, -0.474973, 1.265934, -0.204483, -0.293586, -0.619023, 0.158895, -0.730671, -0.163626, 0.411716, -0.000132, 0.069014, -0.682714, 0.303234, 0.299097, -0.484469, 0.608172, -0.163785, -0.419754, -0.160745, 0.278904, 0.550542, -0.008052, 0.160397, -0.211354, -0.19755, 1.182627, 0.705073, -0.461941, -0.235292, 0.534275, -0.096419, -0.405812, -0.157745, -0.335469, 0.200545, 0.406497, -0.05341, -0.009234, -0.029925, -0.394101, -0.060133, 0.182601, 0.615583, 0.212157, 0.363921, 0.41868, -0.652791, 0.657173, -0.131662, 0.269305, 0.381748, -0.827964, -0.452596, 0.201918, 0.0673, -0.020293, 0.486942, -0.72454, -0.435051, -0.615452, -0.218852, 0.090703, -0.471036, 0.032373, 0.569953, 0.098359, -0.570767, -0.21015, -0.53019, -0.227117, 0.327978, 0.087079, -0.115037, 0.09193, -0.922884, -0.165566, -0.353596, 0.535904, -0.328579, 0.029465, -1.508702, -0.320394, -0.596324, 0.290277, -0.272515, 0.104348, 0.062855, -0.236447, 0.388958, -0.186552, -0.156253, 0.355678, 0.53834, -0.321627, 0.486004, 0.301326, 0.786779, 0.430292, -0.012458, -0.164964, -0.072951, 0.746564, 0.19136, 0.003213, 0.53479, 0.511118, -0.559153, -0.088731, -0.436206, 0.421004, 0.193043, -0.656222, 0.133223, 0.00107, 0.037087, 0.263503, 0.378593, 0.158718, -0.401664, -0.10563, -0.111221, 0.018598, -0.036396, 0.189584, -0.347721, -0.544111, -0.018158, 0.134147, -0.362431, -0.702383, -0.375221, 0.365745, 0.118082, -0.19102, -0.150732, 0.638995, 0.070662, -0.054605, 0.221755, 0.23726, -0.274418, 0.294639, 0.221177, -0.012947, 0.08444, -0.486605, -0.225034, 0.774728, 0.167609, 0.766647, 0.381622, 0.241907, -0.196452, 0.245138, -0.203225, -0.701671, 0.236662, -0.627221, 0.143006, 0.055671, 0.564561, -0.114897, -0.542244, 0.464601, 0.201577, -0.177196, -0.795015, -0.580793, -0.134996, -0.579672, -0.399042, 0.008118, -0.458077, -0.43296, 0.074138, 0.328092, 0.02934, 0.406294, 0.330677, -0.138583, -0.676608, -0.099983, -0.137182, 0.713108, 0.248643, 0.153462, 0.56039, -0.109877, 0.260655, -0.529779, -0.13416, 0.067448, -0.139468, -0.179535, 0.372629, 0.287185, 0.100582, 0.093573, -0.208796, - } -} - -// "peanuts" -func peanutsVector() []float32 { - return []float32{0.563772, -0.779601, -0.18491, 0.509093, 0.080691, -0.621506, -0.127855, -0.165435, 0.57496, 0.006945, 0.452967, -0.285534, -0.129205, 0.193883, 0.092732, 0.083284, 0.714696, 0.107078, -0.398886, -0.117344, -0.387671, 0.026748, -0.562581, -0.007178, -0.354846, -0.431299, -0.788228, 0.175199, 0.914486, 0.441425, 0.089804, 0.284472, 0.106916, -0.133174, 0.399299, 0.002177, 0.551474, 0.389343, -0.016404, 0.770212, -0.219833, 0.303322, 0.127598, -0.378037, -0.172971, 0.394854, -0.424415, -0.71173, 0.080323, -0.406372, 0.398395, -0.594257, -0.418287, 0.055755, -0.352343, -0.393373, -0.732443, 0.333113, 0.420378, -0.50231, 0.261863, -0.061356, -0.180985, 0.311916, -0.180207, -0.154169, 0.371969, 0.454717, 0.320499, -0.182448, 0.087347, 0.585272, 0.136098, 0.288909, -0.229571, -0.140278, 0.229644, -0.557327, -0.110147, 0.034364, -0.021627, -0.598707, 0.221168, -0.059591, -0.203555, -0.434876, 0.209634, -0.460895, -0.345391, -0.18248, -0.24853, 0.730295, -0.295402, -0.562237, 0.255922, 0.076661, -0.713794, -0.354747, -1.109888, -0.066694, -0.195747, -0.282781, 0.459869, -0.309599, -0.002211, -0.274471, -0.003621, 0.008228, 0.011961, -0.258772, -0.210687, -0.664148, -0.257968, 0.231335, 0.530392, -0.205764, -0.621055, -0.440582, 0.080335, 0.017367, 0.880771, 0.656272, -0.713248, -0.208629, 0.095346, 0.336802, 0.888765, 0.251927, 0.066473, 0.182678, -0.220494, 0.288927, -0.602036, 0.057106, -0.594172, 0.848978, 0.751973, 0.090758, -0.732184, 0.683475, -0.075085, 0.381326, -0.076531, -0.253831, 0.10311, -0.02988, -0.043583, 0.005746, -0.460183, -0.189048, 0.25792, 0.477565, 0.391953, 0.08469, -0.10022, 0.454383, 0.170811, 0.196819, -0.760276, 0.045886, -0.743934, 0.190072, -0.216326, -0.624262, -0.22944, 0.066233, 1.024283, 0.044009, -0.373543, -0.243663, 0.204444, 0.402183, 0.043356, 0.31716, 0.302178, 0.369374, 0.36901, 0.02886, -0.26132, -0.234714, -0.791308, -0.433528, -0.098797, -0.447567, -0.124892, -0.119958, 0.31019, -0.096092, -0.259021, -0.078099, -0.178679, 0.14879, 0.106432, -0.450003, -0.294972, 0.044257, 0.402832, 0.263266, -0.309787, -0.17766, -0.399104, 0.577422, 0.30102, 0.05326, -0.271873, 0.204839, -0.019002, -0.743543, 0.739314, -0.115868, -0.504568, -0.115713, 0.042769, -0.123561, -0.057097, 0.407096, 0.770627, 0.372981, -0.321945, 0.349865, 0.437571, -0.77394, -0.090017, -0.011273, -0.468664, -0.735247, -0.745655, 0.018983, -0.248165, 0.215342, -0.136942, -0.458205, 0.4572, -0.032293, 0.654409, -0.024184, -0.392144, 0.634579, 0.222185, 0.471951, -0.063678, -0.473611, 0.796793, -0.295494, -0.157621, -0.103365, -0.564606, -0.092231, -0.517754, -0.369358, 0.137479, -0.214837, 0.11057, -0.095227, 0.726768, -0.079352, -0.065927, -0.846602, -0.317556, -0.344271, 0.201353, -0.367633, -0.004477, 0.157801, -0.249114, -0.549599, -0.147123, 0.308084, -0.175564, 0.306867, -0.071157, -0.588356, 0.450987, -0.184879, -0.096782, -0.006346, -0.017689, 0.005998, 0.200963, 0.225338, 0.189993, -1.105824, 0.520005, 0.129679, 0.198194, -0.254813, -0.127583, 0.326054, 0.009956, -0.016008, -0.483044, 0.801135, -0.517766, 0.067179, -0.372756, -0.511781, 0.058562, -0.082906, -0.28168, -0.285859} -} - -type fakeMetrics struct { - mock.Mock -} - -func (m *fakeMetrics) AddUsageDimensions(class, query, op string, dims int) { - m.Called(class, query, op, dims) -} - -type fakeObjectSearcher struct{} - -func (f *fakeObjectSearcher) Search(context.Context, dto.GetParams) ([]search.Result, error) { - return nil, nil -} - -func (f *fakeObjectSearcher) VectorSearch(context.Context, dto.GetParams) ([]search.Result, error) { - return nil, nil -} - -func (f *fakeObjectSearcher) CrossClassVectorSearch(context.Context, []float32, int, int, *filters.LocalFilter) ([]search.Result, error) { - return nil, nil -} - -func (f *fakeObjectSearcher) Object(ctx context.Context, className string, id strfmt.UUID, props search.SelectProperties, additional additional.Properties, properties *additional.ReplicationProperties, tenant string) (*search.Result, error) { - return nil, nil -} - -func (f *fakeObjectSearcher) ObjectsByID(ctx context.Context, id strfmt.UUID, props search.SelectProperties, additional additional.Properties, tenant string) (search.Results, error) { - return nil, nil -} - -func (f *fakeObjectSearcher) SparseObjectSearch(ctx context.Context, params dto.GetParams) ([]*storobj.Object, []float32, error) { - out := []*storobj.Object{ - { - Object: models.Object{ - ID: "9889a225-3b28-477d-b8fc-5f6071bb4731", - }, - - Vector: []float32{1, 2, 3}, - }, - { - Object: models.Object{ - ID: "0bcdef12-3314-442e-a4d1-e94d7c0afc3a", - }, - Vector: []float32{4, 5, 6}, - }, - } - lim := params.Pagination.Offset + params.Pagination.Limit - if lim > len(out) { - lim = len(out) - } - - return out[:lim], []float32{0.008, 0.001}[:lim], nil -} - -func (f *fakeObjectSearcher) DenseObjectSearch(ctx context.Context, class string, vector []float32, offset int, limit int, filters *filters.LocalFilter, additinal additional.Properties, tenant string) ([]*storobj.Object, []float32, error) { - out := []*storobj.Object{ - { - Object: models.Object{ - ID: "79a636c2-3314-442e-a4d1-e94d7c0afc3a", - }, - - Vector: []float32{4, 5, 6}, - }, - { - Object: models.Object{ - ID: "9889a225-3b28-477d-b8fc-5f6071bb4731", - }, - - Vector: []float32{1, 2, 3}, - }, - } - lim := offset + limit - if lim > len(out) { - lim = len(out) - } - - return out[:lim], []float32{0.009, 0.008}[:lim], nil -} - -func CopyElems[T any](list1, list2 []T, pos int) bool { - if len(list1) != len(list2) { - return false - } - if pos < 0 || pos >= len(list1) { - return true - } - list1[pos] = list2[pos] - return CopyElems(list1, list2, pos+1) -} - -func (f *fakeObjectSearcher) ResolveReferences(ctx context.Context, objs search.Results, props search.SelectProperties, groupBy *searchparams.GroupBy, additional additional.Properties, tenant string) (search.Results, error) { - // Convert res1 to search.Results - out := make(search.Results, len(objs)) - CopyElems(out, objs, 0) - - return out, nil -} - -func TestHybridOverSearch(t *testing.T) { - dirName := t.TempDir() - - logger := logrus.New() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: singleShardState(), - } - repo, err := New(logger, Config{ - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - QueryLimit: 20, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, nil, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(context.TODO())) - defer repo.Shutdown(context.Background()) - - fos := &fakeObjectSearcher{} - - class := SetupFusionClass(t, repo, schemaGetter, logger, 1.2, 0.75) - idx := repo.GetIndex("MyClass") - require.NotNil(t, idx) - - t.Run("Hybrid", func(t *testing.T) { - params := dto.GetParams{ - ClassName: "MyClass", - HybridSearch: &searchparams.HybridSearch{ - Query: "elephant", - Vector: elephantVector(), - Alpha: 0.5, - }, - Pagination: &filters.Pagination{ - Offset: 0, - Limit: 1, - }, - } - - prov := modules.NewProvider() - prov.SetClassDefaults(class) - - metrics := &fakeMetrics{} - log, _ := test.NewNullLogger() - explorer := traverser.NewExplorer(fos, log, prov, metrics, defaultConfig) - hybridResults, err := explorer.Hybrid(context.TODO(), params) - require.Nil(t, err) - require.Equal(t, 1, len(hybridResults)) - require.Equal(t, strfmt.UUID("9889a225-3b28-477d-b8fc-5f6071bb4731"), hybridResults[0].ID) - // require.Equal(t, "79a636c2-3314-442e-a4d1-e94d7c0afc3a", hybridResults[1].ID) - }) -} diff --git a/adapters/repos/db/refcache/cacher.go b/adapters/repos/db/refcache/cacher.go deleted file mode 100644 index 0ff216fb8bcfac0c23e90c9e81b3082fb053de67..0000000000000000000000000000000000000000 --- a/adapters/repos/db/refcache/cacher.go +++ /dev/null @@ -1,421 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package refcache - -import ( - "context" - "fmt" - "sync" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/multi" - "github.com/weaviate/weaviate/entities/schema/crossref" - "github.com/weaviate/weaviate/entities/search" -) - -type repo interface { - MultiGet(ctx context.Context, query []multi.Identifier, - additional additional.Properties, tenant string) ([]search.Result, error) -} - -func NewCacher(repo repo, logger logrus.FieldLogger, tenant string) *Cacher { - return &Cacher{ - logger: logger, - repo: repo, - store: map[multi.Identifier]search.Result{}, - withGroup: false, - tenant: tenant, - } -} - -func NewCacherWithGroup(repo repo, logger logrus.FieldLogger, tenant string) *Cacher { - return &Cacher{ - logger: logger, - repo: repo, - store: map[multi.Identifier]search.Result{}, - // for groupBy feature - withGroup: true, - getGroupSelectProperties: getGroupSelectProperties, - tenant: tenant, - } -} - -type cacherJob struct { - si multi.Identifier - props search.SelectProperties - complete bool -} - -type Cacher struct { - sync.Mutex - jobs []cacherJob - logger logrus.FieldLogger - repo repo - store map[multi.Identifier]search.Result - additional additional.Properties // meta is immutable for the lifetime of the request cacher, so we can safely store it - // for groupBy feature - withGroup bool - getGroupSelectProperties func(properties search.SelectProperties) search.SelectProperties - tenant string -} - -func (c *Cacher) Get(si multi.Identifier) (search.Result, bool) { - sr, ok := c.store[si] - return sr, ok -} - -// Build builds the lookup cache recursively and tries to be smart about it. This -// means that it aims to use only a single (multiget) transaction per layer. -// The recursion exit condition is jobs marked as done. At some point -// the cacher will realise that for every nested prop there is already a -// complete job, so it it stop the recursion. -// -// build is called on a "level" i.e. the search result. After working -// on the job list for the first time if the resolved items still contain -// references and the user set the SelectProperty to indicate they want to -// resolve them, build is called again on all the results (plural!) from the -// previous run. We thus end up with one request to the backend per level -// regardless of the amount of lookups per level. -// -// This keeps request times to a minimum even on deeply nested requests. -func (c *Cacher) Build(ctx context.Context, objects []search.Result, - properties search.SelectProperties, additional additional.Properties, -) error { - c.additional = additional - err := c.findJobsFromResponse(objects, properties) - if err != nil { - return fmt.Errorf("build request cache: %v", err) - } - - c.dedupJobList() - err = c.fetchJobs(ctx) - if err != nil { - return fmt.Errorf("build request cache: %v", err) - } - - return nil -} - -// A response is a []search.Result which has all primitive props parsed (and -// even ref-beacons parsed into their respective types, but not resolved!) -// findJobsFromResponse will traverse through it and check if there are -// references. In a recursive lookup this can both be done on the rootlevel to -// start the first lookup as well as recursively on the results of a lookup to -// further look if a next-level call is required. -func (c *Cacher) findJobsFromResponse(objects []search.Result, properties search.SelectProperties) error { - for _, obj := range objects { - var err error - - // we can only set SelectProperties on the rootlevel since this is the only - // place where we have a single root class. In nested lookups we need to - // first identify the correct path in the SelectProperties graph which - // correspends with the path we're currently traversing through. Thus we - // always cache the original SelectProps with the job. This call goes - // through the job history and looks up the correct SelectProperties - // subpath to use in this place. - // tl;dr: On root level (root=base) take props from the outside, on a - // nested level lookup the SelectProps matching the current base element - propertiesReplaced, err := c.ReplaceInitialPropertiesWithSpecific(obj, properties) - if err != nil { - return err - } - - if obj.Schema == nil { - return nil - } - - schemaMap, ok := obj.Schema.(map[string]interface{}) - if !ok { - return fmt.Errorf("object schema is present, but not a map: %T", obj) - } - - if err := c.parseSchemaMap(schemaMap, propertiesReplaced); err != nil { - return err - } - - if c.withGroup { - if err := c.parseAdditionalGroup(obj, properties); err != nil { - return err - } - } - } - - return nil -} - -func (c *Cacher) parseAdditionalGroup(obj search.Result, properties search.SelectProperties) error { - if obj.AdditionalProperties != nil && obj.AdditionalProperties["group"] != nil { - if group, ok := obj.AdditionalProperties["group"].(*additional.Group); ok { - for _, hitMap := range group.Hits { - if err := c.parseSchemaMap(hitMap, c.getGroupSelectProperties(properties)); err != nil { - return err - } - } - } - } - return nil -} - -func (c *Cacher) parseSchemaMap(schemaMap map[string]interface{}, propertiesReplaced search.SelectProperties) error { - for key, value := range schemaMap { - selectProp := propertiesReplaced.FindProperty(key) - skip, unresolved := c.skipProperty(key, value, selectProp) - if skip { - continue - } - - for _, selectPropRef := range selectProp.Refs { - innerProperties := selectPropRef.RefProperties - - for _, item := range unresolved { - ref, err := c.extractAndParseBeacon(item) - if err != nil { - return err - } - c.addJob(multi.Identifier{ - ID: ref.TargetID.String(), - ClassName: selectPropRef.ClassName, - }, innerProperties) - } - } - } - return nil -} - -func (c *Cacher) skipProperty(key string, value interface{}, selectProp *search.SelectProperty) (bool, models.MultipleRef) { - // the cacher runs at a point where primitive props have already been - // parsed, so we can simply look for parsed, but not resolved refenereces - parsed, ok := value.(models.MultipleRef) - if !ok { - // must be another kind of prop, not interesting for us - return true, nil - } - - if selectProp == nil { - // while we did hit a ref propr, the user is not interested in resolving - // this prop - return true, nil - } - - return false, parsed -} - -func (c *Cacher) extractAndParseBeacon(item *models.SingleRef) (*crossref.Ref, error) { - return crossref.Parse(item.Beacon.String()) -} - -func (c *Cacher) ReplaceInitialPropertiesWithSpecific(obj search.Result, - properties search.SelectProperties, -) (search.SelectProperties, error) { - if properties != nil { - // don't overwrite the properties if the caller has explicitly set them, - // this can only mean they're at the root level - return properties, nil - } - - // this is a nested level, we cannot rely on global initialSelectProperties - // anymore, instead we need to find the selectProperties for exactly this - // ID - job, ok := c.findJob(multi.Identifier{ - ID: obj.ID.String(), - ClassName: obj.ClassName, - }) - if ok { - return job.props, nil - } - - return properties, nil -} - -func (c *Cacher) addJob(si multi.Identifier, props search.SelectProperties) { - c.jobs = append(c.jobs, cacherJob{si, props, false}) -} - -func (c *Cacher) findJob(si multi.Identifier) (cacherJob, bool) { - for _, job := range c.jobs { - if job.si == si { - return job, true - } - } - - return cacherJob{}, false -} - -// finds incompleteJobs without altering the original job list -func (c *Cacher) incompleteJobs() []cacherJob { - out := make([]cacherJob, len(c.jobs)) - n := 0 - for _, job := range c.jobs { - if !job.complete { - out[n] = job - n++ - } - } - - return out[:n] -} - -// finds complete jobs without altering the original job list -func (c *Cacher) completeJobs() []cacherJob { - out := make([]cacherJob, len(c.jobs)) - n := 0 - for _, job := range c.jobs { - if job.complete { - out[n] = job - n++ - } - } - - return out[:n] -} - -// alters the list, removes duplicates. Ignores complete jobs, as a job could -// already marked as complete, but not yet stored since the completion is the -// exit condition for the recursion. However, the storage can only happen once -// the schema was parsed. If the schema contains more refs to an item that is -// already in the joblist we are in a catch-22. To resolve that, we allow -// duplicates with already complete jobs since retrieving the required item -// again (with different SelectProperties) comes at minimal cost and is the -// only way out of that deadlock situation. -func (c *Cacher) dedupJobList() { - incompleteJobs := c.incompleteJobs() - before := len(incompleteJobs) - if before == 0 { - // nothing to do - return - } - c.logger. - WithFields(logrus.Fields{ - "action": "request_cacher_dedup_joblist_start", - "jobs": before, - }). - Debug("starting job list deduplication") - deduped := make([]cacherJob, len(incompleteJobs)) - found := map[multi.Identifier]struct{}{} - - n := 0 - for _, job := range incompleteJobs { - if _, ok := found[job.si]; ok { - continue - } - - found[job.si] = struct{}{} - deduped[n] = job - n++ - } - - c.jobs = append(c.completeJobs(), deduped[:n]...) - - c.logger. - WithFields(logrus.Fields{ - "action": "request_cacher_dedup_joblist_complete", - "jobs": n, - "removedJobs": before - n, - }). - Debug("completed job list deduplication") -} - -func (c *Cacher) fetchJobs(ctx context.Context) error { - jobs := c.incompleteJobs() - if len(jobs) == 0 { - c.logSkipFetchJobs() - return nil - } - - query := jobListToMultiGetQuery(jobs) - res, err := c.repo.MultiGet(ctx, query, c.additional, c.tenant) - if err != nil { - return errors.Wrap(err, "fetch job list") - } - - return c.parseAndStore(ctx, res) -} - -func (c *Cacher) logSkipFetchJobs() { - c.logger. - WithFields( - logrus.Fields{ - "action": "request_cacher_fetch_jobs_skip", - }). - Trace("skip fetch jobs, have no incomplete jobs") -} - -// parseAndStore parses the results for nested refs. Since it is already a -// []search.Result no other parsing is required, as we can expect this type to -// have all primitive props parsed correctly -// -// If nested refs are found, the recursion is started. -// -// Once no more nested refs can be found, the recursion triggers its exit -// condition and all jobs are stored. -func (c *Cacher) parseAndStore(ctx context.Context, res []search.Result) error { - // mark all current jobs as done, as we use the amount of incomplete jobs as - // the exit condition for the recursion. Next up, we will start a nested - // Build() call. If the Build call returns no new jobs, we are done and the - // recursion stops. If it does return more jobs, we will enter a nested - // iteration which will eventually come to this place again - c.markAllJobsAsDone() - - err := c.Build(ctx, removeEmptyResults(res), nil, c.additional) - if err != nil { - return errors.Wrap(err, "build nested cache") - } - - err = c.storeResults(res) - if err != nil { - return err - } - - return nil -} - -func removeEmptyResults(in []search.Result) []search.Result { - out := make([]search.Result, len(in)) - n := 0 - for _, obj := range in { - if obj.ID != "" { - out[n] = obj - n++ - } - } - - return out[0:n] -} - -func (c *Cacher) storeResults(res search.Results) error { - for _, item := range res { - c.store[multi.Identifier{ - ID: item.ID.String(), - ClassName: item.ClassName, - }] = item - } - - return nil -} - -func (c *Cacher) markAllJobsAsDone() { - for i := range c.jobs { - c.jobs[i].complete = true - } -} - -func jobListToMultiGetQuery(jobs []cacherJob) []multi.Identifier { - query := make([]multi.Identifier, len(jobs)) - for i, job := range jobs { - query[i] = job.si - } - - return query -} diff --git a/adapters/repos/db/refcache/cacher_test.go b/adapters/repos/db/refcache/cacher_test.go deleted file mode 100644 index 2b371b8efcdcc9ac2e67affe5fd88b127378db92..0000000000000000000000000000000000000000 --- a/adapters/repos/db/refcache/cacher_test.go +++ /dev/null @@ -1,1060 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package refcache - -import ( - "context" - "fmt" - "testing" - - "github.com/go-openapi/strfmt" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/multi" - "github.com/weaviate/weaviate/entities/search" -) - -func TestCacher(t *testing.T) { - // some ids to be used in the tests, they carry no meaning outside each test - id1 := "132bdf92-ffec-4a52-9196-73ea7cbb5a5e" - id2 := "a60a26dc-791a-41fc-8dda-c0f21f90cc98" - id3 := "a60a26dc-791a-41fc-8dda-c0f21f90cc99" - id4 := "a60a26dc-791a-41fc-8dda-c0f21f90cc97" - - t.Run("with empty results", func(t *testing.T) { - repo := newFakeRepo() - logger, _ := test.NewNullLogger() - cr := NewCacher(repo, logger, "") - err := cr.Build(context.Background(), nil, nil, additional.Properties{}) - assert.Nil(t, err) - }) - - t.Run("with results with nil-schemas", func(t *testing.T) { - repo := newFakeRepo() - logger, _ := test.NewNullLogger() - cr := NewCacher(repo, logger, "") - input := []search.Result{ - { - ID: "foo", - ClassName: "BestClass", - }, - } - err := cr.Build(context.Background(), input, nil, additional.Properties{}) - assert.Nil(t, err) - }) - - t.Run("with results without refs in the schema", func(t *testing.T) { - repo := newFakeRepo() - logger, _ := test.NewNullLogger() - cr := NewCacher(repo, logger, "") - input := []search.Result{ - { - ID: "foo", - ClassName: "BestClass", - Schema: map[string]interface{}{ - "foo": "bar", - "baz": &models.PhoneNumber{}, - }, - }, - } - err := cr.Build(context.Background(), input, nil, additional.Properties{}) - assert.Nil(t, err) - }) - - t.Run("with a single ref, but no selectprops", func(t *testing.T) { - repo := newFakeRepo() - logger, _ := test.NewNullLogger() - cr := NewCacher(repo, logger, "") - input := []search.Result{ - { - ID: "foo", - ClassName: "BestClass", - Schema: map[string]interface{}{ - "refProp": models.MultipleRef{ - &models.SingleRef{ - Beacon: "weaviate://localhost/123", - }, - }, - }, - }, - } - err := cr.Build(context.Background(), input, nil, additional.Properties{}) - require.Nil(t, err) - _, ok := cr.Get(multi.Identifier{ID: "123", ClassName: "SomeClass"}) - assert.False(t, ok) - }) - - t.Run("with a single ref, and a matching select prop", func(t *testing.T) { - repo := newFakeRepo() - repo.lookup[multi.Identifier{ID: id1, ClassName: "SomeClass"}] = search.Result{ - ClassName: "SomeClass", - ID: strfmt.UUID(id1), - Schema: map[string]interface{}{ - "bar": "some string", - }, - } - logger, _ := test.NewNullLogger() - cr := NewCacher(repo, logger, "") - input := []search.Result{ - { - ID: "foo", - ClassName: "BestClass", - Schema: map[string]interface{}{ - "refProp": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", id1)), - }, - }, - }, - }, - } - selectProps := search.SelectProperties{ - search.SelectProperty{ - Name: "refProp", - Refs: []search.SelectClass{ - { - ClassName: "SomeClass", - RefProperties: search.SelectProperties{ - search.SelectProperty{ - Name: "bar", - IsPrimitive: true, - }, - }, - }, - }, - }, - } - - expected := search.Result{ - ID: strfmt.UUID(id1), - ClassName: "SomeClass", - Schema: map[string]interface{}{ - "bar": "some string", - }, - } - - err := cr.Build(context.Background(), input, selectProps, additional.Properties{}) - require.Nil(t, err) - res, ok := cr.Get(multi.Identifier{ID: id1, ClassName: "SomeClass"}) - require.True(t, ok) - assert.Equal(t, expected, res) - assert.Equal(t, 1, repo.counter, "required the expected amount of lookups") - }) - - t.Run("with a nested lookup, partially resolved", func(t *testing.T) { - repo := newFakeRepo() - repo.lookup[multi.Identifier{ID: id1, ClassName: "SomeClass"}] = search.Result{ - ClassName: "SomeClass", - ID: strfmt.UUID(id1), - Schema: map[string]interface{}{ - "primitive": "foobar", - "ignoredRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI("weaviate://localhost/ignoreMe"), - }, - }, - "nestedRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", id2)), - }, - }, - }, - } - repo.lookup[multi.Identifier{ID: id2, ClassName: "SomeNestedClass"}] = search.Result{ - ClassName: "SomeNestedClass", - ID: strfmt.UUID(id2), - Schema: map[string]interface{}{ - "name": "John Doe", - }, - } - logger, _ := test.NewNullLogger() - cr := NewCacher(repo, logger, "") - input := []search.Result{ - { - ID: "foo", - ClassName: "BestClass", - Schema: map[string]interface{}{ - "refProp": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", id1)), - }, - }, - }, - }, - } - selectProps := search.SelectProperties{ - search.SelectProperty{ - Name: "refProp", - Refs: []search.SelectClass{ - { - ClassName: "SomeClass", - RefProperties: search.SelectProperties{ - search.SelectProperty{ - Name: "primitive", - IsPrimitive: true, - }, - search.SelectProperty{ - Name: "nestedRef", - Refs: []search.SelectClass{ - { - ClassName: "SomeNestedClass", - RefProperties: []search.SelectProperty{ - { - Name: "name", - IsPrimitive: true, - }, - }, - }, - }, - }, - }, - }, - }, - }, - } - - expectedOuter := search.Result{ - ID: strfmt.UUID(id1), - ClassName: "SomeClass", - Schema: map[string]interface{}{ - "primitive": "foobar", - "ignoredRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI("weaviate://localhost/ignoreMe"), - }, - }, - "nestedRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", id2)), - }, - }, - }, - } - - expectedInner := search.Result{ - ClassName: "SomeNestedClass", - ID: strfmt.UUID(id2), - Schema: map[string]interface{}{ - "name": "John Doe", - }, - } - - err := cr.Build(context.Background(), input, selectProps, additional.Properties{}) - require.Nil(t, err) - res, ok := cr.Get(multi.Identifier{ID: id1, ClassName: "SomeClass"}) - require.True(t, ok) - assert.Equal(t, expectedOuter, res) - res, ok = cr.Get(multi.Identifier{ID: id2, ClassName: "SomeNestedClass"}) - require.True(t, ok) - assert.Equal(t, expectedInner, res) - assert.Equal(t, 2, repo.counter, "required the expected amount of lookups") - }) - - t.Run("with multiple items pointing to the same ref", func(t *testing.T) { - // this test asserts that we do not make unnecessary requests if an object - // is linked twice on the list. (This is very common if the reference is - // used for something like a product category, e.g. it would not be - // uncommon at all if all search results are of the same category) - repo := newFakeRepo() - repo.lookup[multi.Identifier{ID: id1, ClassName: "SomeClass"}] = search.Result{ - ClassName: "SomeClass", - ID: strfmt.UUID(id1), - Schema: map[string]interface{}{ - "primitive": "foobar", - "nestedRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", id2)), - }, - }, - }, - } - repo.lookup[multi.Identifier{ID: id2, ClassName: "SomeNestedClass"}] = search.Result{ - ClassName: "SomeNestedClass", - ID: strfmt.UUID(id2), - Schema: map[string]interface{}{ - "name": "John Doe", - }, - } - logger, _ := test.NewNullLogger() - cr := NewCacher(repo, logger, "") - - // contains three items, all pointing to the same inner class - input := []search.Result{ - { - ID: "foo", - ClassName: "BestClass", - Schema: map[string]interface{}{ - "refProp": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", id1)), - }, - }, - }, - }, - { - ID: "bar", - ClassName: "BestClass", - Schema: map[string]interface{}{ - "refProp": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", id1)), - }, - }, - }, - }, - { - ID: "baz", - ClassName: "BestClass", - Schema: map[string]interface{}{ - "refProp": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", id1)), - }, - }, - }, - }, - } - selectProps := search.SelectProperties{ - search.SelectProperty{ - Name: "refProp", - Refs: []search.SelectClass{ - { - ClassName: "SomeClass", - RefProperties: search.SelectProperties{ - search.SelectProperty{ - Name: "primitive", - IsPrimitive: true, - }, - search.SelectProperty{ - Name: "nestedRef", - Refs: []search.SelectClass{ - { - ClassName: "SomeNestedClass", - RefProperties: []search.SelectProperty{ - { - Name: "name", - IsPrimitive: true, - }, - }, - }, - }, - }, - }, - }, - }, - }, - } - - expectedOuter := search.Result{ - ID: strfmt.UUID(id1), - ClassName: "SomeClass", - Schema: map[string]interface{}{ - "primitive": "foobar", - "nestedRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", id2)), - }, - }, - }, - } - - expectedInner := search.Result{ - ClassName: "SomeNestedClass", - ID: strfmt.UUID(id2), - Schema: map[string]interface{}{ - "name": "John Doe", - }, - } - - err := cr.Build(context.Background(), input, selectProps, additional.Properties{}) - require.Nil(t, err) - res, ok := cr.Get(multi.Identifier{ID: id1, ClassName: "SomeClass"}) - require.True(t, ok) - assert.Equal(t, expectedOuter, res) - res, ok = cr.Get(multi.Identifier{ID: id2, ClassName: "SomeNestedClass"}) - require.True(t, ok) - assert.Equal(t, expectedInner, res) - assert.Equal(t, 2, repo.counter, "required the expected amount of lookup queries") - assert.Equal(t, 2, repo.counter, "required the expected amount of objects on the lookup queries") - }) - - t.Run("with a nested lookup, and nested refs in nested refs", func(t *testing.T) { - repo := newFakeRepo() - idNested2ID := "132bdf92-ffec-4a52-9196-73ea7cbb5a00" - idNestedInNestedID := "132bdf92-ffec-4a52-9196-73ea7cbb5a01" - repo.lookup[multi.Identifier{ID: id1, ClassName: "SomeClass"}] = search.Result{ - ClassName: "SomeClass", - ID: strfmt.UUID(id1), - Schema: map[string]interface{}{ - "primitive": "foobar", - "nestedRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", id2)), - }, - }, - "nestedRef2": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", idNested2ID)), - Schema: map[string]interface{}{ - "title": "nestedRef2Title", - "nestedRefInNestedRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", idNestedInNestedID)), - }, - }, - }, - }, - }, - }, - } - repo.lookup[multi.Identifier{ID: id2, ClassName: "SomeNestedClass"}] = search.Result{ - ClassName: "SomeNestedClass", - ID: strfmt.UUID(id2), - Schema: map[string]interface{}{ - "name": "John Doe", - }, - } - repo.lookup[multi.Identifier{ID: idNested2ID, ClassName: "SomeNestedClass2"}] = search.Result{ - ClassName: "SomeNestedClass2", - ID: strfmt.UUID(idNested2ID), - Schema: map[string]interface{}{ - "title": "nestedRef2Title", - "nestedRefInNestedRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", idNestedInNestedID)), - }, - }, - }, - } - repo.lookup[multi.Identifier{ID: idNestedInNestedID, ClassName: "SomeNestedClassNested2"}] = search.Result{ - ClassName: "SomeNestedClassNested2", - ID: strfmt.UUID(idNestedInNestedID), - Schema: map[string]interface{}{ - "titleNested": "Nested In Nested Title", - }, - } - logger, _ := test.NewNullLogger() - cr := NewCacher(repo, logger, "") - input := []search.Result{ - { - ID: "foo", - ClassName: "BestClass", - Schema: map[string]interface{}{ - "refProp": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", id1)), - }, - }, - }, - }, - } - selectProps := search.SelectProperties{ - search.SelectProperty{ - Name: "refProp", - Refs: []search.SelectClass{ - { - ClassName: "SomeClass", - RefProperties: search.SelectProperties{ - search.SelectProperty{ - Name: "primitive", - IsPrimitive: true, - }, - search.SelectProperty{ - Name: "nestedRef", - Refs: []search.SelectClass{ - { - ClassName: "SomeNestedClass", - RefProperties: []search.SelectProperty{ - { - Name: "name", - IsPrimitive: true, - }, - }, - }, - }, - }, - search.SelectProperty{ - Name: "nestedRef2", - Refs: []search.SelectClass{ - { - ClassName: "SomeNestedClass2", - RefProperties: []search.SelectProperty{ - { - Name: "title", - IsPrimitive: true, - }, - { - Name: "nestedRefInNestedRef", - Refs: []search.SelectClass{ - { - ClassName: "SomeNestedClassNested2", - RefProperties: []search.SelectProperty{ - { - Name: "titleNested", - IsPrimitive: true, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - } - - expectedOuter := search.Result{ - ID: strfmt.UUID(id1), - ClassName: "SomeClass", - Schema: map[string]interface{}{ - "primitive": "foobar", - "nestedRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", id2)), - }, - }, - "nestedRef2": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", idNested2ID)), - Schema: map[string]interface{}{ - "title": "nestedRef2Title", - "nestedRefInNestedRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", idNestedInNestedID)), - }, - }, - }, - }, - }, - }, - } - - expectedInner := search.Result{ - ClassName: "SomeNestedClass", - ID: strfmt.UUID(id2), - Schema: map[string]interface{}{ - "name": "John Doe", - }, - } - - expectedInner2 := search.Result{ - ClassName: "SomeNestedClass2", - ID: strfmt.UUID(idNested2ID), - Schema: map[string]interface{}{ - "title": "nestedRef2Title", - "nestedRefInNestedRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", idNestedInNestedID)), - }, - }, - }, - } - - expectedInnerInner := search.Result{ - ClassName: "SomeNestedClassNested2", - ID: strfmt.UUID(idNestedInNestedID), - Schema: map[string]interface{}{ - "titleNested": "Nested In Nested Title", - }, - } - - err := cr.Build(context.Background(), input, selectProps, additional.Properties{}) - require.Nil(t, err) - res, ok := cr.Get(multi.Identifier{ID: id1, ClassName: "SomeClass"}) - require.True(t, ok) - assert.Equal(t, expectedOuter, res) - input2 := []search.Result{expectedInner, expectedInner2} - err = cr.Build(context.Background(), input2, nil, additional.Properties{}) - require.Nil(t, err) - nested1, ok := cr.Get(multi.Identifier{ID: id2, ClassName: "SomeNestedClass"}) - require.True(t, ok) - assert.Equal(t, expectedInner, nested1) - nested2, ok := cr.Get(multi.Identifier{ID: idNested2ID, ClassName: "SomeNestedClass2"}) - require.True(t, ok) - assert.Equal(t, expectedInner2, nested2) - nestedSchema, ok := nested2.Schema.(map[string]interface{}) - require.True(t, ok) - nestedRefInNestedRef, ok := nestedSchema["nestedRefInNestedRef"] - require.True(t, ok) - require.NotNil(t, nestedRefInNestedRef) - nestedRefInNestedMultiRef, ok := nestedRefInNestedRef.(models.MultipleRef) - require.True(t, ok) - require.NotNil(t, nestedRefInNestedMultiRef) - require.Nil(t, err) - res, ok = cr.Get(multi.Identifier{ID: idNestedInNestedID, ClassName: "SomeNestedClassNested2"}) - require.True(t, ok) - assert.Equal(t, expectedInnerInner, res) - assert.Equal(t, 4, repo.counter, "required the expected amount of lookups") - }) - - t.Run("with group and with a additional group lookup", func(t *testing.T) { - repo := newFakeRepo() - repo.lookup[multi.Identifier{ID: id1, ClassName: "SomeClass"}] = search.Result{ - ClassName: "SomeClass", - ID: strfmt.UUID(id1), - Schema: map[string]interface{}{ - "primitive": "foobar", - }, - AdditionalProperties: models.AdditionalProperties{ - "group": &additional.Group{ - Hits: []map[string]interface{}{ - { - "primitive": "foobar", - "ignoredRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI("weaviate://localhost/ignoreMe"), - }, - }, - "nestedRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", id2)), - }, - }, - }, - }, - }, - }, - } - repo.lookup[multi.Identifier{ID: id2, ClassName: "SomeNestedClass"}] = search.Result{ - ClassName: "SomeNestedClass", - ID: strfmt.UUID(id2), - Schema: map[string]interface{}{ - "name": "John Doe", - }, - } - logger, _ := test.NewNullLogger() - cr := NewCacherWithGroup(repo, logger, "") - input := []search.Result{ - { - ID: "foo", - ClassName: "BestClass", - Schema: map[string]interface{}{ - "refProp": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", id1)), - }, - }, - }, - AdditionalProperties: models.AdditionalProperties{ - "group": &additional.Group{ - Hits: []map[string]interface{}{ - { - "primitive": "foobar", - "ignoredRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI("weaviate://localhost/ignoreMe"), - }, - }, - "nestedRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", id2)), - }, - }, - }, - }, - }, - }, - }, - } - selectProps := search.SelectProperties{ - search.SelectProperty{ - Name: "_additional:group:hits:nestedRef", - Refs: []search.SelectClass{ - { - ClassName: "SomeNestedClass", - RefProperties: []search.SelectProperty{ - { - Name: "name", - IsPrimitive: true, - }, - }, - }, - }, - }, - } - - expectedInner := search.Result{ - ClassName: "SomeNestedClass", - ID: strfmt.UUID(id2), - Schema: map[string]interface{}{ - "name": "John Doe", - }, - } - - err := cr.Build(context.Background(), input, selectProps, additional.Properties{}) - require.Nil(t, err) - res, ok := cr.Get(multi.Identifier{ID: id2, ClassName: "SomeNestedClass"}) - require.True(t, ok) - assert.Equal(t, expectedInner, res) - assert.Equal(t, 1, repo.counter, "required the expected amount of lookups") - }) - - t.Run("with group and with 2 additional group lookups", func(t *testing.T) { - repo := newFakeRepo() - repo.lookup[multi.Identifier{ID: id1, ClassName: "SomeClass"}] = search.Result{ - ClassName: "SomeClass", - ID: strfmt.UUID(id1), - Schema: map[string]interface{}{ - "primitive": "foobar", - }, - AdditionalProperties: models.AdditionalProperties{ - "group": &additional.Group{ - Hits: []map[string]interface{}{ - { - "primitive": "foobar", - "ignoredRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI("weaviate://localhost/ignoreMe"), - }, - }, - "nestedRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", id2)), - }, - }, - }, - }, - }, - }, - } - repo.lookup[multi.Identifier{ID: id2, ClassName: "SomeNestedClass"}] = search.Result{ - ClassName: "SomeNestedClass", - ID: strfmt.UUID(id2), - Schema: map[string]interface{}{ - "name": "John Doe", - }, - } - repo.lookup[multi.Identifier{ID: id3, ClassName: "OtherNestedClass"}] = search.Result{ - ClassName: "OtherNestedClass", - ID: strfmt.UUID(id3), - Schema: map[string]interface{}{ - "name": "John Doe", - }, - } - logger, _ := test.NewNullLogger() - cr := NewCacherWithGroup(repo, logger, "") - input := []search.Result{ - { - ID: "foo", - ClassName: "BestClass", - Schema: map[string]interface{}{ - "refProp": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", id1)), - }, - }, - }, - AdditionalProperties: models.AdditionalProperties{ - "group": &additional.Group{ - Hits: []map[string]interface{}{ - { - "primitive": "foobar", - "nestedRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", id2)), - }, - }, - "otherNestedRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", id3)), - }, - }, - }, - }, - }, - }, - }, - } - selectProps := search.SelectProperties{ - search.SelectProperty{ - Name: "_additional:group:hits:nestedRef", - Refs: []search.SelectClass{ - { - ClassName: "SomeNestedClass", - RefProperties: []search.SelectProperty{ - { - Name: "name", - IsPrimitive: true, - }, - }, - }, - }, - }, - search.SelectProperty{ - Name: "_additional:group:hits:otherNestedRef", - Refs: []search.SelectClass{ - { - ClassName: "OtherNestedClass", - RefProperties: []search.SelectProperty{ - { - Name: "name", - IsPrimitive: true, - }, - }, - }, - }, - }, - } - - expectedSomeNestedClass := search.Result{ - ClassName: "SomeNestedClass", - ID: strfmt.UUID(id2), - Schema: map[string]interface{}{ - "name": "John Doe", - }, - } - - expectedOtherNestedClass := search.Result{ - ClassName: "OtherNestedClass", - ID: strfmt.UUID(id3), - Schema: map[string]interface{}{ - "name": "John Doe", - }, - } - - err := cr.Build(context.Background(), input, selectProps, additional.Properties{}) - require.Nil(t, err) - res, ok := cr.Get(multi.Identifier{ID: id2, ClassName: "SomeNestedClass"}) - require.True(t, ok) - assert.Equal(t, expectedSomeNestedClass, res) - res, ok = cr.Get(multi.Identifier{ID: id3, ClassName: "OtherNestedClass"}) - require.True(t, ok) - assert.Equal(t, expectedOtherNestedClass, res) - assert.Equal(t, 1, repo.counter, "required the expected amount of lookups") - }) - - t.Run("with group with a nested lookup and with 2 additional group lookups", func(t *testing.T) { - repo := newFakeRepo() - repo.lookup[multi.Identifier{ID: id1, ClassName: "SomeClass"}] = search.Result{ - ClassName: "SomeClass", - ID: strfmt.UUID(id1), - Schema: map[string]interface{}{ - "primitive": "foobar", - "ignoredRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI("weaviate://localhost/ignoreMe"), - }, - }, - "nestedRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", id2)), - }, - }, - }, - } - repo.lookup[multi.Identifier{ID: id2, ClassName: "SomeNestedClass"}] = search.Result{ - ClassName: "SomeNestedClass", - ID: strfmt.UUID(id2), - Schema: map[string]interface{}{ - "name": "John Doe", - }, - } - repo.lookup[multi.Identifier{ID: id3, ClassName: "InnerNestedClass"}] = search.Result{ - ClassName: "InnerNestedClass", - ID: strfmt.UUID(id3), - Schema: map[string]interface{}{ - "name": "John Doe", - }, - } - repo.lookup[multi.Identifier{ID: id4, ClassName: "OtherNestedClass"}] = search.Result{ - ClassName: "OtherNestedClass", - ID: strfmt.UUID(id4), - Schema: map[string]interface{}{ - "name": "John Doe", - }, - } - logger, _ := test.NewNullLogger() - cr := NewCacherWithGroup(repo, logger, "") - input := []search.Result{ - { - ID: "foo", - ClassName: "BestClass", - Schema: map[string]interface{}{ - "refProp": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", id1)), - }, - }, - }, - AdditionalProperties: models.AdditionalProperties{ - "group": &additional.Group{ - Hits: []map[string]interface{}{ - { - "primitive": "foobar", - "innerNestedRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", id3)), - }, - }, - "otherNestedRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", id4)), - }, - }, - }, - }, - }, - }, - }, - } - selectProps := search.SelectProperties{ - search.SelectProperty{ - Name: "refProp", - Refs: []search.SelectClass{ - { - ClassName: "SomeClass", - RefProperties: search.SelectProperties{ - search.SelectProperty{ - Name: "primitive", - IsPrimitive: true, - }, - search.SelectProperty{ - Name: "nestedRef", - Refs: []search.SelectClass{ - { - ClassName: "SomeNestedClass", - RefProperties: []search.SelectProperty{ - { - Name: "name", - IsPrimitive: true, - }, - }, - }, - }, - }, - }, - }, - }, - }, - search.SelectProperty{ - Name: "_additional:group:hits:innerNestedRef", - Refs: []search.SelectClass{ - { - ClassName: "InnerNestedClass", - RefProperties: []search.SelectProperty{ - { - Name: "name", - IsPrimitive: true, - }, - }, - }, - }, - }, - search.SelectProperty{ - Name: "_additional:group:hits:otherNestedRef", - Refs: []search.SelectClass{ - { - ClassName: "OtherNestedClass", - RefProperties: []search.SelectProperty{ - { - Name: "name", - IsPrimitive: true, - }, - }, - }, - }, - }, - } - - expectedOuter := search.Result{ - ID: strfmt.UUID(id1), - ClassName: "SomeClass", - Schema: map[string]interface{}{ - "primitive": "foobar", - "ignoredRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI("weaviate://localhost/ignoreMe"), - }, - }, - "nestedRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", id2)), - }, - }, - }, - } - - expectedInner := search.Result{ - ClassName: "SomeNestedClass", - ID: strfmt.UUID(id2), - Schema: map[string]interface{}{ - "name": "John Doe", - }, - } - - expectedInnerNestedClass := search.Result{ - ClassName: "InnerNestedClass", - ID: strfmt.UUID(id3), - Schema: map[string]interface{}{ - "name": "John Doe", - }, - } - - expectedOtherNestedClass := search.Result{ - ClassName: "OtherNestedClass", - ID: strfmt.UUID(id4), - Schema: map[string]interface{}{ - "name": "John Doe", - }, - } - - err := cr.Build(context.Background(), input, selectProps, additional.Properties{}) - require.Nil(t, err) - res, ok := cr.Get(multi.Identifier{ID: id1, ClassName: "SomeClass"}) - require.True(t, ok) - assert.Equal(t, expectedOuter, res) - res, ok = cr.Get(multi.Identifier{ID: id2, ClassName: "SomeNestedClass"}) - require.True(t, ok) - assert.Equal(t, expectedInner, res) - res, ok = cr.Get(multi.Identifier{ID: id3, ClassName: "InnerNestedClass"}) - require.True(t, ok) - assert.Equal(t, expectedInnerNestedClass, res) - res, ok = cr.Get(multi.Identifier{ID: id4, ClassName: "OtherNestedClass"}) - require.True(t, ok) - assert.Equal(t, expectedOtherNestedClass, res) - assert.Equal(t, 2, repo.counter, "required the expected amount of lookups") - }) -} - -type fakeRepo struct { - lookup map[multi.Identifier]search.Result - counter int // count request - objectCounter int // count total objects on request(s) -} - -func newFakeRepo() *fakeRepo { - return &fakeRepo{ - lookup: map[multi.Identifier]search.Result{}, - } -} - -func (f *fakeRepo) MultiGet(ctx context.Context, query []multi.Identifier, additional additional.Properties, tenant string) ([]search.Result, error) { - f.counter++ - f.objectCounter += len(query) - out := make([]search.Result, len(query)) - for i, q := range query { - if res, ok := f.lookup[q]; ok { - out[i] = res - } else { - out[i] = search.Result{} - } - } - - return out, nil -} diff --git a/adapters/repos/db/refcache/group_properties_util.go b/adapters/repos/db/refcache/group_properties_util.go deleted file mode 100644 index 4166f1c1b8c2f2cc48d597f4e8d8aa6ded9f18ae..0000000000000000000000000000000000000000 --- a/adapters/repos/db/refcache/group_properties_util.go +++ /dev/null @@ -1,33 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package refcache - -import ( - "strings" - - "github.com/weaviate/weaviate/entities/search" -) - -func getGroupSelectProperties(properties search.SelectProperties) search.SelectProperties { - var selectGroupHitsProperties search.SelectProperties - for _, prop := range properties { - if strings.HasPrefix(prop.Name, "_additional:group:hits") { - selectGroupHitsProperties = append(selectGroupHitsProperties, search.SelectProperty{ - Name: strings.Replace(prop.Name, "_additional:group:hits:", "", 1), - IsPrimitive: prop.IsPrimitive, - IncludeTypeName: prop.IncludeTypeName, - Refs: prop.Refs, - }) - } - } - return selectGroupHitsProperties -} diff --git a/adapters/repos/db/refcache/resolver.go b/adapters/repos/db/refcache/resolver.go deleted file mode 100644 index 77259e0fe713e66e4ffe530deba830a1f314b214..0000000000000000000000000000000000000000 --- a/adapters/repos/db/refcache/resolver.go +++ /dev/null @@ -1,239 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package refcache - -import ( - "context" - "fmt" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/multi" - "github.com/weaviate/weaviate/entities/schema/crossref" - "github.com/weaviate/weaviate/entities/search" -) - -type Resolver struct { - cacher cacher - // for groupBy feature - withGroup bool - getGroupSelectProperties func(properties search.SelectProperties) search.SelectProperties -} - -type cacher interface { - Build(ctx context.Context, objects []search.Result, properties search.SelectProperties, additional additional.Properties) error - Get(si multi.Identifier) (search.Result, bool) -} - -func NewResolver(cacher cacher) *Resolver { - return &Resolver{cacher: cacher} -} - -func NewResolverWithGroup(cacher cacher) *Resolver { - return &Resolver{ - cacher: cacher, - // for groupBy feature - withGroup: true, - getGroupSelectProperties: getGroupSelectProperties, - } -} - -func (r *Resolver) Do(ctx context.Context, objects []search.Result, - properties search.SelectProperties, additional additional.Properties, -) ([]search.Result, error) { - if err := r.cacher.Build(ctx, objects, properties, additional); err != nil { - return nil, errors.Wrap(err, "build reference cache") - } - - return r.parseObjects(objects, properties, additional) -} - -func (r *Resolver) parseObjects(objects []search.Result, properties search.SelectProperties, - additional additional.Properties, -) ([]search.Result, error) { - for i, obj := range objects { - parsed, err := r.parseObject(obj, properties, additional) - if err != nil { - return nil, errors.Wrapf(err, "parse at position %d", i) - } - - objects[i] = parsed - } - - return objects, nil -} - -func (r *Resolver) parseObject(object search.Result, properties search.SelectProperties, - additional additional.Properties, -) (search.Result, error) { - if object.Schema == nil { - return object, nil - } - - schemaMap, ok := object.Schema.(map[string]interface{}) - if !ok { - return object, fmt.Errorf("schema is not a map: %T", object.Schema) - } - - schema, err := r.parseSchema(schemaMap, properties) - if err != nil { - return object, err - } - - object.Schema = schema - - if r.withGroup { - additionalProperties, err := r.parseAdditionalGroup(object.AdditionalProperties, properties) - if err != nil { - return object, err - } - object.AdditionalProperties = additionalProperties - } - return object, nil -} - -func (r *Resolver) parseAdditionalGroup( - additionalProperties models.AdditionalProperties, - properties search.SelectProperties, -) (models.AdditionalProperties, error) { - if additionalProperties != nil && additionalProperties["group"] != nil { - if group, ok := additionalProperties["group"].(*additional.Group); ok { - for j, hit := range group.Hits { - schema, err := r.parseSchema(hit, r.getGroupSelectProperties(properties)) - if err != nil { - return additionalProperties, fmt.Errorf("resolve group hit: %w", err) - } - group.Hits[j] = schema - } - } - } - return additionalProperties, nil -} - -func (r *Resolver) parseSchema(schema map[string]interface{}, - properties search.SelectProperties, -) (map[string]interface{}, error) { - for propName, value := range schema { - refs, ok := value.(models.MultipleRef) - if !ok { - // not a ref, not interesting for us - continue - } - - selectProp := properties.FindProperty(propName) - if selectProp == nil { - // user is not interested in this prop - continue - } - - parsed, err := r.parseRefs(refs, propName, *selectProp) - if err != nil { - return schema, errors.Wrapf(err, "parse refs for prop %q", propName) - } - - if parsed != nil { - schema[propName] = parsed - } - } - - return schema, nil -} - -func (r *Resolver) parseRefs(input models.MultipleRef, prop string, - selectProp search.SelectProperty, -) ([]interface{}, error) { - var refs []interface{} - for _, selectPropRef := range selectProp.Refs { - innerProperties := selectPropRef.RefProperties - additionalProperties := selectPropRef.AdditionalProperties - perClass, err := r.resolveRefs(input, selectPropRef.ClassName, innerProperties, additionalProperties) - if err != nil { - return nil, errors.Wrap(err, "resolve ref") - } - - refs = append(refs, perClass...) - } - return refs, nil -} - -func (r *Resolver) resolveRefs(input models.MultipleRef, desiredClass string, - innerProperties search.SelectProperties, - additionalProperties additional.Properties, -) ([]interface{}, error) { - var output []interface{} - for i, item := range input { - resolved, err := r.resolveRef(item, desiredClass, innerProperties, additionalProperties) - if err != nil { - return nil, errors.Wrapf(err, "at position %d", i) - } - - if resolved == nil { - continue - } - - output = append(output, *resolved) - } - - return output, nil -} - -func (r *Resolver) resolveRef(item *models.SingleRef, desiredClass string, - innerProperties search.SelectProperties, - additionalProperties additional.Properties, -) (*search.LocalRef, error) { - var out search.LocalRef - - ref, err := crossref.Parse(item.Beacon.String()) - if err != nil { - return nil, err - } - - si := multi.Identifier{ - ID: ref.TargetID.String(), - ClassName: desiredClass, - } - res, ok := r.cacher.Get(si) - if !ok { - // silently ignore, could have been deleted in the meantime, or we're - // asking for a non-matching selectProperty, for example if we ask for - // Article { published { ... on { Magazine { name } ... on { Journal { name } } - // we don't know at resolve time if this ID will point to a Magazine or a - // Journal, so we will get a few empty responses when trying both for any - // given ID. - // - // In turn this means we need to validate through automated and explorative - // tests, that we never skip results that should be contained, as we - // wouldn't throw an error, so the user would never notice - return nil, nil - } - - out.Class = res.ClassName - schema := res.Schema.(map[string]interface{}) - nested, err := r.parseSchema(schema, innerProperties) - if err != nil { - return nil, errors.Wrap(err, "resolve nested ref") - } - - if additionalProperties.Vector { - nested["vector"] = res.Vector - } - if additionalProperties.CreationTimeUnix { - nested["creationTimeUnix"] = res.Created - } - if additionalProperties.LastUpdateTimeUnix { - nested["lastUpdateTimeUnix"] = res.Updated - } - out.Fields = nested - - return &out, nil -} diff --git a/adapters/repos/db/refcache/resolver_test.go b/adapters/repos/db/refcache/resolver_test.go deleted file mode 100644 index bba385515a058b03e2108d0db6137c908a01fa4b..0000000000000000000000000000000000000000 --- a/adapters/repos/db/refcache/resolver_test.go +++ /dev/null @@ -1,551 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package refcache - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/go-openapi/strfmt" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/multi" - "github.com/weaviate/weaviate/entities/search" -) - -func TestResolver(t *testing.T) { - id1 := "df5d4e49-0c56-4b87-ade1-3d46cc9b425f" - id2 := "3a08d808-8eb5-49ee-86b2-68b6035e8b69" - - t.Run("with nil input", func(t *testing.T) { - r := NewResolver(newFakeCacher()) - res, err := r.Do(context.Background(), nil, nil, additional.Properties{}) - require.Nil(t, err) - assert.Nil(t, res) - }) - - t.Run("with nil-schemas", func(t *testing.T) { - r := NewResolver(newFakeCacher()) - input := []search.Result{ - { - ID: "foo", - ClassName: "BestClass", - }, - } - - expected := input - res, err := r.Do(context.Background(), input, nil, additional.Properties{}) - require.Nil(t, err) - assert.Equal(t, expected, res) - }) - - t.Run("with single ref but no select props", func(t *testing.T) { - r := NewResolver(newFakeCacher()) - input := []search.Result{ - { - ID: "foo", - ClassName: "BestClass", - Schema: map[string]interface{}{ - "refProp": models.MultipleRef{ - &models.SingleRef{ - Beacon: "weaviate://localhost/123", - }, - }, - }, - }, - } - - expected := input - res, err := r.Do(context.Background(), input, nil, additional.Properties{}) - require.Nil(t, err) - assert.Equal(t, expected, res) - }) - - t.Run("with single ref with vector and matching select prop", func(t *testing.T) { - getInput := func() []search.Result { - return []search.Result{ - { - ID: "foo", - ClassName: "BestClass", - Schema: map[string]interface{}{ - "refProp": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", id1)), - }, - }, - }, - }, - } - } - getResolver := func() *Resolver { - cacher := newFakeCacher() - r := NewResolver(cacher) - cacher.lookup[multi.Identifier{ID: id1, ClassName: "SomeClass"}] = search.Result{ - ClassName: "SomeClass", - ID: strfmt.UUID(id1), - Schema: map[string]interface{}{ - "bar": "some string", - }, - Vector: []float32{0.1, 0.2}, - } - return r - } - getSelectProps := func(withVector bool) search.SelectProperties { - return search.SelectProperties{ - search.SelectProperty{ - Name: "refProp", - Refs: []search.SelectClass{ - { - ClassName: "SomeClass", - RefProperties: search.SelectProperties{ - search.SelectProperty{ - Name: "bar", - IsPrimitive: true, - }, - }, - AdditionalProperties: additional.Properties{ - Vector: withVector, - }, - }, - }, - }, - } - } - getExpectedResult := func(withVector bool) []search.Result { - fields := map[string]interface{}{ - "bar": "some string", - } - if withVector { - fields["vector"] = []float32{0.1, 0.2} - } - return []search.Result{ - { - ID: "foo", - ClassName: "BestClass", - Schema: map[string]interface{}{ - "refProp": []interface{}{ - search.LocalRef{ - Class: "SomeClass", - Fields: fields, - }, - }, - }, - }, - } - } - // ask for vector in ref property - res, err := getResolver().Do(context.Background(), getInput(), getSelectProps(true), additional.Properties{}) - require.Nil(t, err) - assert.Equal(t, getExpectedResult(true), res) - // don't ask for vector in ref property - res, err = getResolver().Do(context.Background(), getInput(), getSelectProps(false), additional.Properties{}) - require.Nil(t, err) - assert.Equal(t, getExpectedResult(false), res) - }) - - t.Run("with single ref with creation/update timestamps and matching select prop", func(t *testing.T) { - now := time.Now().UnixMilli() - getInput := func() []search.Result { - return []search.Result{ - { - ID: "foo", - ClassName: "BestClass", - Schema: map[string]interface{}{ - "refProp": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", id1)), - }, - }, - }, - }, - } - } - getResolver := func() *Resolver { - cacher := newFakeCacher() - r := NewResolver(cacher) - cacher.lookup[multi.Identifier{ID: id1, ClassName: "SomeClass"}] = search.Result{ - ClassName: "SomeClass", - ID: strfmt.UUID(id1), - Schema: map[string]interface{}{ - "bar": "some string", - }, - Created: now, - Updated: now, - } - return r - } - selectProps := search.SelectProperties{ - search.SelectProperty{ - Name: "refProp", - Refs: []search.SelectClass{ - { - ClassName: "SomeClass", - RefProperties: search.SelectProperties{ - search.SelectProperty{ - Name: "bar", - IsPrimitive: true, - }, - }, - AdditionalProperties: additional.Properties{ - CreationTimeUnix: true, - LastUpdateTimeUnix: true, - }, - }, - }, - }, - } - expected := []search.Result{ - { - ID: "foo", - ClassName: "BestClass", - Schema: map[string]interface{}{ - "refProp": []interface{}{ - search.LocalRef{ - Class: "SomeClass", - Fields: map[string]interface{}{ - "bar": "some string", - "creationTimeUnix": now, - "lastUpdateTimeUnix": now, - }, - }, - }, - }, - }, - } - res, err := getResolver().Do(context.Background(), getInput(), selectProps, additional.Properties{}) - require.Nil(t, err) - assert.Equal(t, expected, res) - }) - - t.Run("with single ref and matching select prop", func(t *testing.T) { - cacher := newFakeCacher() - r := NewResolver(cacher) - cacher.lookup[multi.Identifier{ID: id1, ClassName: "SomeClass"}] = search.Result{ - ClassName: "SomeClass", - ID: strfmt.UUID(id1), - Schema: map[string]interface{}{ - "bar": "some string", - }, - } - input := []search.Result{ - { - ID: "foo", - ClassName: "BestClass", - Schema: map[string]interface{}{ - "refProp": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", id1)), - }, - }, - }, - }, - } - selectProps := search.SelectProperties{ - search.SelectProperty{ - Name: "refProp", - Refs: []search.SelectClass{ - { - ClassName: "SomeClass", - RefProperties: search.SelectProperties{ - search.SelectProperty{ - Name: "bar", - IsPrimitive: true, - }, - }, - }, - }, - }, - } - - expected := []search.Result{ - { - ID: "foo", - ClassName: "BestClass", - Schema: map[string]interface{}{ - "refProp": []interface{}{ - search.LocalRef{ - Class: "SomeClass", - Fields: map[string]interface{}{ - "bar": "some string", - }, - }, - }, - }, - }, - } - res, err := r.Do(context.Background(), input, selectProps, additional.Properties{}) - require.Nil(t, err) - assert.Equal(t, expected, res) - }) - - t.Run("with a nested lookup", func(t *testing.T) { - cacher := newFakeCacher() - r := NewResolver(cacher) - cacher.lookup[multi.Identifier{ID: id1, ClassName: "SomeClass"}] = search.Result{ - ClassName: "SomeClass", - ID: strfmt.UUID(id1), - Schema: map[string]interface{}{ - "primitive": "foobar", - "ignoredRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI("weaviate://localhost/ignoreMe"), - }, - }, - "nestedRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", id2)), - }, - }, - }, - } - cacher.lookup[multi.Identifier{ID: id2, ClassName: "SomeNestedClass"}] = search.Result{ - ClassName: "SomeNestedClass", - ID: strfmt.UUID(id2), - Schema: map[string]interface{}{ - "name": "John Doe", - }, - } - input := []search.Result{ - { - ID: "foo", - ClassName: "BestClass", - Schema: map[string]interface{}{ - "refProp": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", id1)), - }, - }, - }, - }, - } - selectProps := search.SelectProperties{ - search.SelectProperty{ - Name: "refProp", - Refs: []search.SelectClass{ - { - ClassName: "SomeClass", - RefProperties: search.SelectProperties{ - search.SelectProperty{ - Name: "primitive", - IsPrimitive: true, - }, - search.SelectProperty{ - Name: "nestedRef", - Refs: []search.SelectClass{ - { - ClassName: "SomeNestedClass", - RefProperties: []search.SelectProperty{ - { - Name: "name", - IsPrimitive: true, - }, - }, - }, - }, - }, - }, - }, - }, - }, - } - - expected := []search.Result{ - { - ID: "foo", - ClassName: "BestClass", - Schema: map[string]interface{}{ - "refProp": []interface{}{ - search.LocalRef{ - Class: "SomeClass", - Fields: map[string]interface{}{ - "primitive": "foobar", - "ignoredRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI("weaviate://localhost/ignoreMe"), - }, - }, - "nestedRef": []interface{}{ - search.LocalRef{ - Class: "SomeNestedClass", - Fields: map[string]interface{}{ - "name": "John Doe", - }, - }, - }, - }, - }, - }, - }, - }, - } - res, err := r.Do(context.Background(), input, selectProps, additional.Properties{}) - require.Nil(t, err) - assert.Equal(t, expected, res) - }) - - t.Run("with single ref with vector and matching select prop and group", func(t *testing.T) { - getInput := func() []search.Result { - return []search.Result{ - { - ID: "foo", - ClassName: "BestClass", - Schema: map[string]interface{}{ - "refProp": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", id1)), - }, - }, - }, - AdditionalProperties: models.AdditionalProperties{ - "group": &additional.Group{ - Hits: []map[string]interface{}{ - { - "nestedRef": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/SomeNestedClass/%s", id2)), - }, - }, - }, - }, - }, - }, - }, - } - } - getResolver := func() *Resolver { - cacher := newFakeCacher() - r := NewResolverWithGroup(cacher) - cacher.lookup[multi.Identifier{ID: id1, ClassName: "SomeClass"}] = search.Result{ - ClassName: "SomeClass", - ID: strfmt.UUID(id1), - Schema: map[string]interface{}{ - "bar": "some string", - }, - Vector: []float32{0.1, 0.2}, - } - cacher.lookup[multi.Identifier{ID: id2, ClassName: "SomeNestedClass"}] = search.Result{ - ClassName: "SomeNestedClass", - ID: strfmt.UUID(id2), - Schema: map[string]interface{}{ - "name": "John Doe", - }, - } - return r - } - getSelectProps := func(withVector bool) search.SelectProperties { - return search.SelectProperties{ - search.SelectProperty{ - Name: "refProp", - Refs: []search.SelectClass{ - { - ClassName: "SomeClass", - RefProperties: search.SelectProperties{ - search.SelectProperty{ - Name: "bar", - IsPrimitive: true, - }, - }, - AdditionalProperties: additional.Properties{ - Vector: withVector, - }, - }, - }, - }, - search.SelectProperty{ - Name: "_additional:group:hits:nestedRef", - Refs: []search.SelectClass{ - { - ClassName: "SomeNestedClass", - RefProperties: []search.SelectProperty{ - { - Name: "name", - IsPrimitive: true, - }, - }, - }, - }, - }, - } - } - getExpectedResult := func(withVector bool) []search.Result { - fields := map[string]interface{}{ - "bar": "some string", - } - if withVector { - fields["vector"] = []float32{0.1, 0.2} - } - return []search.Result{ - { - ID: "foo", - ClassName: "BestClass", - Schema: map[string]interface{}{ - "refProp": []interface{}{ - search.LocalRef{ - Class: "SomeClass", - Fields: fields, - }, - }, - }, - AdditionalProperties: models.AdditionalProperties{ - "group": &additional.Group{ - Hits: []map[string]interface{}{ - { - "nestedRef": []interface{}{ - search.LocalRef{ - Class: "SomeNestedClass", - Fields: map[string]interface{}{ - "name": "John Doe", - }, - }, - }, - }, - }, - }, - }, - }, - } - } - // ask for vector in ref property - res, err := getResolver().Do(context.Background(), getInput(), getSelectProps(true), additional.Properties{}) - require.Nil(t, err) - assert.Equal(t, getExpectedResult(true), res) - // don't ask for vector in ref property - res, err = getResolver().Do(context.Background(), getInput(), getSelectProps(false), additional.Properties{}) - require.Nil(t, err) - assert.Equal(t, getExpectedResult(false), res) - }) -} - -func newFakeCacher() *fakeCacher { - return &fakeCacher{ - lookup: map[multi.Identifier]search.Result{}, - } -} - -type fakeCacher struct { - lookup map[multi.Identifier]search.Result -} - -func (f *fakeCacher) Build(ctx context.Context, objects []search.Result, properties search.SelectProperties, - additional additional.Properties, -) error { - return nil -} - -func (f *fakeCacher) Get(si multi.Identifier) (search.Result, bool) { - res, ok := f.lookup[si] - return res, ok -} diff --git a/adapters/repos/db/replication.go b/adapters/repos/db/replication.go deleted file mode 100644 index 22511e23730e9e5002e6c1131fce2f53e647e910..0000000000000000000000000000000000000000 --- a/adapters/repos/db/replication.go +++ /dev/null @@ -1,495 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "context" - "fmt" - "io" - "os" - "path" - "path/filepath" - - "github.com/go-openapi/strfmt" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/multi" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/storobj" - "github.com/weaviate/weaviate/usecases/objects" - "github.com/weaviate/weaviate/usecases/replica" -) - -type Replicator interface { - ReplicateObject(ctx context.Context, shardName, requestID string, - object *storobj.Object) replica.SimpleResponse - ReplicateObjects(ctx context.Context, shardName, requestID string, - objects []*storobj.Object) replica.SimpleResponse - ReplicateUpdate(ctx context.Context, shard, requestID string, - doc *objects.MergeDocument) replica.SimpleResponse - ReplicateDeletion(ctx context.Context, shardName, requestID string, - uuid strfmt.UUID) replica.SimpleResponse - ReplicateDeletions(ctx context.Context, shardName, requestID string, - uuids []strfmt.UUID, dryRun bool) replica.SimpleResponse - ReplicateReferences(ctx context.Context, shard, requestID string, - refs []objects.BatchReference) replica.SimpleResponse - CommitReplication(shard, - requestID string) interface{} - AbortReplication(shardName, - requestID string) interface{} -} - -func (db *DB) ReplicateObject(ctx context.Context, class, - shard, requestID string, object *storobj.Object, -) replica.SimpleResponse { - index, pr := db.replicatedIndex(class) - if pr != nil { - return *pr - } - - return index.ReplicateObject(ctx, shard, requestID, object) -} - -func (db *DB) ReplicateObjects(ctx context.Context, class, - shard, requestID string, objects []*storobj.Object, -) replica.SimpleResponse { - index, pr := db.replicatedIndex(class) - if pr != nil { - return *pr - } - - return index.ReplicateObjects(ctx, shard, requestID, objects) -} - -func (db *DB) ReplicateUpdate(ctx context.Context, class, - shard, requestID string, mergeDoc *objects.MergeDocument, -) replica.SimpleResponse { - index, pr := db.replicatedIndex(class) - if pr != nil { - return *pr - } - - return index.ReplicateUpdate(ctx, shard, requestID, mergeDoc) -} - -func (db *DB) ReplicateDeletion(ctx context.Context, class, - shard, requestID string, uuid strfmt.UUID, -) replica.SimpleResponse { - index, pr := db.replicatedIndex(class) - if pr != nil { - return *pr - } - - return index.ReplicateDeletion(ctx, shard, requestID, uuid) -} - -func (db *DB) ReplicateDeletions(ctx context.Context, class, - shard, requestID string, uuids []strfmt.UUID, dryRun bool, -) replica.SimpleResponse { - index, pr := db.replicatedIndex(class) - if pr != nil { - return *pr - } - - return index.ReplicateDeletions(ctx, shard, requestID, uuids, dryRun) -} - -func (db *DB) ReplicateReferences(ctx context.Context, class, - shard, requestID string, refs []objects.BatchReference, -) replica.SimpleResponse { - index, pr := db.replicatedIndex(class) - if pr != nil { - return *pr - } - - return index.ReplicateReferences(ctx, shard, requestID, refs) -} - -func (db *DB) CommitReplication(class, - shard, requestID string, -) interface{} { - index, pr := db.replicatedIndex(class) - if pr != nil { - return nil - } - - return index.CommitReplication(shard, requestID) -} - -func (db *DB) AbortReplication(class, - shard, requestID string, -) interface{} { - index, pr := db.replicatedIndex(class) - if pr != nil { - return *pr - } - - return index.AbortReplication(shard, requestID) -} - -func (db *DB) replicatedIndex(name string) (idx *Index, resp *replica.SimpleResponse) { - if !db.StartupComplete() { - return nil, &replica.SimpleResponse{Errors: []replica.Error{ - *replica.NewError(replica.StatusNotReady, name), - }} - } - - if idx = db.GetIndex(schema.ClassName(name)); idx == nil { - return nil, &replica.SimpleResponse{Errors: []replica.Error{ - *replica.NewError(replica.StatusClassNotFound, name), - }} - } - return -} - -func (i *Index) writableShard(name string) (ShardLike, *replica.SimpleResponse) { - localShard := i.localShard(name) - if localShard == nil { - return nil, &replica.SimpleResponse{Errors: []replica.Error{ - {Code: replica.StatusShardNotFound, Msg: name}, - }} - } - if localShard.isReadOnly() { - return nil, &replica.SimpleResponse{Errors: []replica.Error{{ - Code: replica.StatusReadOnly, Msg: name, - }}} - } - return localShard, nil -} - -func (i *Index) ReplicateObject(ctx context.Context, shard, requestID string, object *storobj.Object) replica.SimpleResponse { - localShard, pr := i.writableShard(shard) - if pr != nil { - return *pr - } - return localShard.preparePutObject(ctx, requestID, object) -} - -func (i *Index) ReplicateUpdate(ctx context.Context, shard, requestID string, doc *objects.MergeDocument) replica.SimpleResponse { - localShard, pr := i.writableShard(shard) - if pr != nil { - return *pr - } - return localShard.prepareMergeObject(ctx, requestID, doc) -} - -func (i *Index) ReplicateDeletion(ctx context.Context, shard, requestID string, uuid strfmt.UUID) replica.SimpleResponse { - localShard, pr := i.writableShard(shard) - if pr != nil { - return *pr - } - return localShard.prepareDeleteObject(ctx, requestID, uuid) -} - -func (i *Index) ReplicateObjects(ctx context.Context, shard, requestID string, objects []*storobj.Object) replica.SimpleResponse { - localShard, pr := i.writableShard(shard) - if pr != nil { - return *pr - } - return localShard.preparePutObjects(ctx, requestID, objects) -} - -func (i *Index) ReplicateDeletions(ctx context.Context, shard, requestID string, uuids []strfmt.UUID, dryRun bool) replica.SimpleResponse { - localShard, pr := i.writableShard(shard) - if pr != nil { - return *pr - } - return localShard.prepareDeleteObjects(ctx, requestID, uuids, dryRun) -} - -func (i *Index) ReplicateReferences(ctx context.Context, shard, requestID string, refs []objects.BatchReference) replica.SimpleResponse { - localShard, pr := i.writableShard(shard) - if pr != nil { - return *pr - } - return localShard.prepareAddReferences(ctx, requestID, refs) -} - -func (i *Index) CommitReplication(shard, requestID string) interface{} { - localShard := i.localShard(shard) - if localShard == nil { - return nil - } - return localShard.commitReplication(context.Background(), requestID, &i.backupMutex) -} - -func (i *Index) AbortReplication(shard, requestID string) interface{} { - localShard := i.localShard(shard) - if localShard == nil { - return replica.SimpleResponse{Errors: []replica.Error{ - {Code: replica.StatusShardNotFound, Msg: shard}, - }} - } - return localShard.abortReplication(context.Background(), requestID) -} - -func (i *Index) IncomingFilePutter(ctx context.Context, shardName, - filePath string, -) (io.WriteCloser, error) { - localShard := i.localShard(shardName) - if localShard == nil { - return nil, fmt.Errorf("shard %q does not exist locally", shardName) - } - - return localShard.filePutter(ctx, filePath) -} - -func (i *Index) IncomingCreateShard(ctx context.Context, className string, shardName string) error { - sch := i.getSchema.GetSchemaSkipAuth() - class := sch.GetClass(schema.ClassName(className)) - if err := i.addNewShard(ctx, class, shardName); err != nil { - return fmt.Errorf("incoming create shard: %w", err) - } - return nil -} - -func (i *Index) IncomingReinitShard(ctx context.Context, - shardName string, -) error { - shard := i.localShard(shardName) - if shard == nil { - return fmt.Errorf("shard %q does not exist locally", shardName) - } - - return shard.reinit(ctx) -} - -func (s *Shard) filePutter(ctx context.Context, - filePath string, -) (io.WriteCloser, error) { - // TODO: validate file prefix to rule out that we're accidentally writing - // into another shard - finalPath := filepath.Join(s.Index().Config.RootPath, filePath) - dir := path.Dir(finalPath) - if err := os.MkdirAll(dir, os.ModePerm); err != nil { - return nil, fmt.Errorf("create parent folder for %s: %w", filePath, err) - } - - f, err := os.Create(finalPath) - if err != nil { - return nil, fmt.Errorf("open file %q for writing: %w", filePath, err) - } - - return f, nil -} - -func (s *Shard) reinit(ctx context.Context) error { - // This is a short term fix to deal with the problem that the shard is still running while we are changing its files on disk. The shard needs to be shut down before we can safely change the files. Otherwise the shutdown process will overwrite, or possibly even corrupt the changed files. - s.propLenTracker = nil - if err := s.Shutdown(ctx); err != nil { - return fmt.Errorf("shutdown shard: %w", err) - } - - s.initShard(ctx) - - return nil -} - -func (db *DB) OverwriteObjects(ctx context.Context, - class, shardName string, vobjects []*objects.VObject, -) ([]replica.RepairResponse, error) { - index := db.GetIndex(schema.ClassName(class)) - return index.overwriteObjects(ctx, shardName, vobjects) -} - -// overwrite objects if their state didn't change in the meantime -// It returns nil if all object have been successfully overwritten -// and otherwise a list of failed operations. -func (i *Index) overwriteObjects(ctx context.Context, - shard string, updates []*objects.VObject, -) ([]replica.RepairResponse, error) { - result := make([]replica.RepairResponse, 0, len(updates)/2) - s := i.localShard(shard) - if s == nil { - return nil, fmt.Errorf("shard %q not found locally", shard) - } - for i, u := range updates { - // Just in case but this should not happen - data := u.LatestObject - if data == nil || data.ID == "" { - msg := fmt.Sprintf("received nil object or empty uuid at position %d", i) - result = append(result, replica.RepairResponse{Err: msg}) - continue - } - // valid update - found, err := s.ObjectByID(ctx, data.ID, nil, additional.Properties{}) - var curUpdateTime int64 // 0 means object doesn't exist on this node - if found != nil { - curUpdateTime = found.LastUpdateTimeUnix() - } - r := replica.RepairResponse{ID: data.ID.String(), UpdateTime: curUpdateTime} - switch { - case err != nil: - r.Err = "not found: " + err.Error() - case curUpdateTime == u.StaleUpdateTime: - // the stored object is not the most recent version. in - // this case, we overwrite it with the more recent one. - err := s.PutObject(ctx, storobj.FromObject(data, u.Vector)) - if err != nil { - r.Err = fmt.Sprintf("overwrite stale object: %v", err) - } - case curUpdateTime != data.LastUpdateTimeUnix: - // object changed and its state differs from recent known state - r.Err = "conflict" - } - - if r.Err != "" { // include only unsuccessful responses - result = append(result, r) - } - } - if len(result) == 0 { - return nil, nil - } - return result, nil -} - -func (i *Index) IncomingOverwriteObjects(ctx context.Context, - shardName string, vobjects []*objects.VObject, -) ([]replica.RepairResponse, error) { - return i.overwriteObjects(ctx, shardName, vobjects) -} - -func (db *DB) DigestObjects(ctx context.Context, - class, shardName string, ids []strfmt.UUID, -) (result []replica.RepairResponse, err error) { - index := db.GetIndex(schema.ClassName(class)) - return index.digestObjects(ctx, shardName, ids) -} - -func (i *Index) digestObjects(ctx context.Context, - shardName string, ids []strfmt.UUID, -) (result []replica.RepairResponse, err error) { - result = make([]replica.RepairResponse, len(ids)) - s := i.localShard(shardName) - if s == nil { - return nil, fmt.Errorf("shard %q not found locally", shardName) - } - - multiIDs := make([]multi.Identifier, len(ids)) - for j := range multiIDs { - multiIDs[j] = multi.Identifier{ID: ids[j].String()} - } - - objs, err := s.MultiObjectByID(ctx, multiIDs) - if err != nil { - return nil, fmt.Errorf("shard objects digest: %w", err) - } - - for j := range objs { - if objs[j] == nil { - deleted, err := s.WasDeleted(ctx, ids[j]) - if err != nil { - return nil, err - } - result[j] = replica.RepairResponse{ - ID: ids[j].String(), - Deleted: deleted, - // TODO: use version when supported - Version: 0, - } - } else { - result[j] = replica.RepairResponse{ - ID: objs[j].ID().String(), - UpdateTime: objs[j].LastUpdateTimeUnix(), - // TODO: use version when supported - Version: 0, - } - } - } - - return -} - -func (i *Index) IncomingDigestObjects(ctx context.Context, - shardName string, ids []strfmt.UUID, -) (result []replica.RepairResponse, err error) { - return i.digestObjects(ctx, shardName, ids) -} - -func (db *DB) FetchObject(ctx context.Context, - class, shardName string, id strfmt.UUID, -) (objects.Replica, error) { - index := db.GetIndex(schema.ClassName(class)) - return index.readRepairGetObject(ctx, shardName, id) -} - -func (i *Index) readRepairGetObject(ctx context.Context, - shardName string, id strfmt.UUID, -) (objects.Replica, error) { - shard := i.localShard(shardName) - if shard == nil { - return objects.Replica{}, fmt.Errorf("shard %q does not exist locally", shardName) - } - - obj, err := shard.ObjectByID(ctx, id, nil, additional.Properties{}) - if err != nil { - return objects.Replica{}, fmt.Errorf("shard %q read repair get object: %w", shard.ID(), err) - } - - if obj == nil { - deleted, err := shard.WasDeleted(ctx, id) - if err != nil { - return objects.Replica{}, err - } - return objects.Replica{ - ID: id, - Deleted: deleted, - }, nil - } - - return objects.Replica{ - Object: obj, - ID: obj.ID(), - }, nil -} - -func (db *DB) FetchObjects(ctx context.Context, - class, shardName string, ids []strfmt.UUID, -) ([]objects.Replica, error) { - index := db.GetIndex(schema.ClassName(class)) - return index.fetchObjects(ctx, shardName, ids) -} - -func (i *Index) fetchObjects(ctx context.Context, - shardName string, ids []strfmt.UUID, -) ([]objects.Replica, error) { - shard := i.localShard(shardName) - if shard == nil { - return nil, fmt.Errorf("shard %q does not exist locally", shardName) - } - - objs, err := shard.MultiObjectByID(ctx, wrapIDsInMulti(ids)) - if err != nil { - return nil, fmt.Errorf("shard %q replication multi get objects: %w", shard.ID(), err) - } - - resp := make([]objects.Replica, len(ids)) - - for j, obj := range objs { - if obj == nil { - deleted, err := shard.WasDeleted(ctx, ids[j]) - if err != nil { - return nil, err - } - resp[j] = objects.Replica{ - ID: ids[j], - Deleted: deleted, - } - } else { - resp[j] = objects.Replica{ - Object: obj, - ID: obj.ID(), - } - } - } - - return resp, nil -} diff --git a/adapters/repos/db/repo.go b/adapters/repos/db/repo.go deleted file mode 100644 index 80731642bce10b8188fbb3950f5b86f5710602ed..0000000000000000000000000000000000000000 --- a/adapters/repos/db/repo.go +++ /dev/null @@ -1,390 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "context" - "math" - "runtime" - "runtime/debug" - "sync" - "sync/atomic" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/adapters/repos/db/indexcheckpoint" - "github.com/weaviate/weaviate/entities/replication" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/storobj" - "github.com/weaviate/weaviate/usecases/config" - "github.com/weaviate/weaviate/usecases/memwatch" - "github.com/weaviate/weaviate/usecases/monitoring" - "github.com/weaviate/weaviate/usecases/replica" - schemaUC "github.com/weaviate/weaviate/usecases/schema" - "github.com/weaviate/weaviate/usecases/sharding" -) - -type DB struct { - logger logrus.FieldLogger - schemaGetter schemaUC.SchemaGetter - config Config - indices map[string]*Index - remoteIndex sharding.RemoteIndexClient - replicaClient replica.Client - nodeResolver nodeResolver - remoteNode *sharding.RemoteNode - promMetrics *monitoring.PrometheusMetrics - indexCheckpoints *indexcheckpoint.Checkpoints - shutdown chan struct{} - startupComplete atomic.Bool - resourceScanState *resourceScanState - memMonitor *memwatch.Monitor - - // indexLock is an RWMutex which allows concurrent access to various indexes, - // but only one modification at a time. R/W can be a bit confusing here, - // because it does not refer to write or read requests from a user's - // perspective, but rather: - // - // - Read -> The array containing all indexes is read-only. In other words - // there will never be a race condition from doing something like index := - // indexes[0]. What you do with the Index after retrieving it from the array - // does not matter. Assuming that it is thread-safe (it is) you can - // read/write from the index itself. Therefore from a user's perspective - // something like a parallel import batch and a read-query can happen without - // any problems. - // - // - Write -> The index array is being modified, for example, because a new - // index is added. This is mutually exclusive with the other case (but - // hopefully very short). - // - // - // See also: https://github.com/weaviate/weaviate/issues/2351 - // - // This lock should be used to avoid that the indices-map is changed while iterating over it. To - // mark a given index in use, lock that index directly. - indexLock sync.RWMutex - - jobQueueCh chan job - asyncIndexRetryInterval time.Duration - shutDownWg sync.WaitGroup - maxNumberGoroutines int - batchMonitorLock sync.Mutex - ratePerSecond int -} - -func (db *DB) GetSchemaGetter() schemaUC.SchemaGetter { - return db.schemaGetter -} - -func (db *DB) GetSchema() schema.Schema { - return db.schemaGetter.GetSchemaSkipAuth() -} - -func (db *DB) GetConfig() Config { - return db.config -} - -func (db *DB) GetIndices() []*Index { - out := make([]*Index, 0, len(db.indices)) - for _, index := range db.indices { - out = append(out, index) - } - - return out -} - -func (db *DB) GetRemoteIndex() sharding.RemoteIndexClient { - return db.remoteIndex -} - -func (db *DB) SetSchemaGetter(sg schemaUC.SchemaGetter) { - db.schemaGetter = sg -} - -func (db *DB) WaitForStartup(ctx context.Context) error { - err := db.init(ctx) - if err != nil { - return err - } - - db.startupComplete.Store(true) - db.scanResourceUsage() - - return nil -} - -func (db *DB) StartupComplete() bool { return db.startupComplete.Load() } - -func New(logger logrus.FieldLogger, config Config, - remoteIndex sharding.RemoteIndexClient, nodeResolver nodeResolver, - remoteNodesClient sharding.RemoteNodeClient, replicaClient replica.Client, - promMetrics *monitoring.PrometheusMetrics, -) (*DB, error) { - db := &DB{ - logger: logger, - config: config, - indices: map[string]*Index{}, - remoteIndex: remoteIndex, - nodeResolver: nodeResolver, - remoteNode: sharding.NewRemoteNode(nodeResolver, remoteNodesClient), - replicaClient: replicaClient, - promMetrics: promMetrics, - shutdown: make(chan struct{}), - asyncIndexRetryInterval: 5 * time.Second, - maxNumberGoroutines: int(math.Round(config.MaxImportGoroutinesFactor * float64(runtime.GOMAXPROCS(0)))), - resourceScanState: newResourceScanState(), - memMonitor: memwatch.NewMonitor(memwatch.LiveHeapReader, debug.SetMemoryLimit, 0.97), - } - - // make sure memMonitor has an initial state - db.memMonitor.Refresh() - - if db.maxNumberGoroutines == 0 { - return db, errors.New("no workers to add batch-jobs configured.") - } - if !asyncEnabled() { - db.jobQueueCh = make(chan job, 100000) - db.shutDownWg.Add(db.maxNumberGoroutines) - for i := 0; i < db.maxNumberGoroutines; i++ { - go db.worker(i == 0) - } - } else { - logger.Info("async indexing enabled") - w := runtime.GOMAXPROCS(0) - 1 - db.shutDownWg.Add(w) - db.jobQueueCh = make(chan job, w) - for i := 0; i < w; i++ { - go func() { - defer db.shutDownWg.Done() - - asyncWorker(db.jobQueueCh, db.logger, db.asyncIndexRetryInterval) - }() - } - } - - return db, nil -} - -type Config struct { - RootPath string - QueryLimit int64 - QueryMaximumResults int64 - QueryNestedRefLimit int64 - ResourceUsage config.ResourceUsage - MaxImportGoroutinesFactor float64 - MemtablesFlushIdleAfter int - MemtablesInitialSizeMB int - MemtablesMaxSizeMB int - MemtablesMinActiveSeconds int - MemtablesMaxActiveSeconds int - TrackVectorDimensions bool - ServerVersion string - GitHash string - AvoidMMap bool - DisableLazyLoadShards bool - Replication replication.GlobalConfig -} - -// GetIndex returns the index if it exists or nil if it doesn't -func (db *DB) GetIndex(className schema.ClassName) *Index { - db.indexLock.RLock() - defer db.indexLock.RUnlock() - - id := indexID(className) - index, ok := db.indices[id] - if !ok { - return nil - } - - return index -} - -// IndexExists returns if an index exists -func (db *DB) IndexExists(className schema.ClassName) bool { - db.indexLock.RLock() - defer db.indexLock.RUnlock() - - id := indexID(className) - _, ok := db.indices[id] - return ok -} - -// GetIndexForIncoming returns the index if it exists or nil if it doesn't -func (db *DB) GetIndexForIncoming(className schema.ClassName) sharding.RemoteIndexIncomingRepo { - db.indexLock.RLock() - defer db.indexLock.RUnlock() - - id := indexID(className) - index, ok := db.indices[id] - if !ok { - return nil - } - - return index -} - -// DeleteIndex deletes the index -func (db *DB) DeleteIndex(className schema.ClassName) error { - db.indexLock.Lock() - defer db.indexLock.Unlock() - - // Get index - id := indexID(className) - index := db.indices[id] - if index == nil { - return nil - } - - // Drop index - index.dropIndex.Lock() - defer index.dropIndex.Unlock() - if err := index.drop(); err != nil { - db.logger.WithField("action", "delete_index").WithField("class", className).Error(err) - } - delete(db.indices, id) - - db.promMetrics.DeleteClass(className.String()) - return nil -} - -func (db *DB) Shutdown(ctx context.Context) error { - db.shutdown <- struct{}{} - - if !asyncEnabled() { - // shut down the workers that add objects to - for i := 0; i < db.maxNumberGoroutines; i++ { - db.jobQueueCh <- job{ - index: -1, - } - } - } - - db.indexLock.Lock() - defer db.indexLock.Unlock() - for id, index := range db.indices { - if err := index.Shutdown(ctx); err != nil { - return errors.Wrapf(err, "shutdown index %q", id) - } - } - - if asyncEnabled() { - // shut down the async workers - close(db.jobQueueCh) - } - - db.shutDownWg.Wait() // wait until job queue shutdown is completed - - if asyncEnabled() { - db.indexCheckpoints.Close() - } - - return nil -} - -func (db *DB) worker(first bool) { - objectCounter := 0 - checkTime := time.Now().Add(time.Second) - for jobToAdd := range db.jobQueueCh { - if jobToAdd.index < 0 { - db.shutDownWg.Done() - return - } - jobToAdd.batcher.storeSingleObjectInAdditionalStorage(jobToAdd.ctx, jobToAdd.object, jobToAdd.status, jobToAdd.index) - jobToAdd.batcher.wg.Done() - objectCounter += 1 - if first && time.Now().After(checkTime) { // only have one worker report the rate per second - db.batchMonitorLock.Lock() - db.ratePerSecond = objectCounter * db.maxNumberGoroutines - db.batchMonitorLock.Unlock() - - objectCounter = 0 - checkTime = time.Now().Add(time.Second) - } - } -} - -type job struct { - object *storobj.Object - status objectInsertStatus - index int - ctx context.Context - batcher *objectsBatcher - - // async only - chunk *chunk - indexer batchIndexer - queue *vectorQueue -} - -func asyncWorker(ch chan job, logger logrus.FieldLogger, retryInterval time.Duration) { - var ids []uint64 - var vectors [][]float32 - var deleted []uint64 - - for job := range ch { - c := job.chunk - for i := range c.data[:c.cursor] { - if job.queue.IsDeleted(c.data[i].id) { - deleted = append(deleted, c.data[i].id) - } else { - ids = append(ids, c.data[i].id) - vectors = append(vectors, c.data[i].vector) - } - } - - var err error - - if len(ids) > 0 { - LOOP: - for { - err = job.indexer.AddBatch(job.ctx, ids, vectors) - if err == nil { - break LOOP - } - - if errors.Is(err, context.Canceled) { - logger.WithError(err).Debugf("skipping indexing batch due to context cancellation") - break LOOP - } - - logger.WithError(err).Infof("failed to index vectors, retrying in %s", retryInterval.String()) - - t := time.NewTimer(retryInterval) - select { - case <-job.ctx.Done(): - // drain the timer - if !t.Stop() { - <-t.C - } - return - case <-t.C: - } - } - } - - // only persist checkpoint if we indexed a full batch - if err == nil { - job.queue.persistCheckpoint(ids) - } - - job.queue.releaseChunk(c) - - if len(deleted) > 0 { - job.queue.ResetDeleted(deleted...) - } - - ids = ids[:0] - vectors = vectors[:0] - deleted = deleted[:0] - } -} diff --git a/adapters/repos/db/resource_use.go b/adapters/repos/db/resource_use.go deleted file mode 100644 index e03d897dc7cf706833c88631edf49087e0bc34de..0000000000000000000000000000000000000000 --- a/adapters/repos/db/resource_use.go +++ /dev/null @@ -1,165 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "fmt" - "time" - - "github.com/weaviate/weaviate/entities/interval" - "github.com/weaviate/weaviate/entities/storagestate" - "github.com/weaviate/weaviate/usecases/memwatch" -) - -type diskUse struct { - total uint64 - free uint64 - avail uint64 -} - -func (d diskUse) percentUsed() float64 { - used := d.total - d.free - return (float64(used) / float64(d.total)) * 100 -} - -func (d diskUse) String() string { - GB := 1024 * 1024 * 1024 - - return fmt.Sprintf("total: %.2fGB, free: %.2fGB, used: %.2fGB (avail: %.2fGB)", - float64(d.total)/float64(GB), - float64(d.free)/float64(GB), - float64(d.total-d.free)/float64(GB), - float64(d.avail)/float64(GB)) -} - -func (d *DB) scanResourceUsage() { - go func() { - t := time.NewTicker(time.Millisecond * 500) - defer t.Stop() - for { - select { - case <-d.shutdown: - return - case <-t.C: - if !d.resourceScanState.isReadOnly { - du := d.getDiskUse(d.config.RootPath) - d.resourceUseWarn(d.memMonitor, du) - d.resourceUseReadonly(d.memMonitor, du) - } - } - } - }() -} - -type resourceScanState struct { - diskWarning *interval.BackoffTimer - memWarning *interval.BackoffTimer - isReadOnly bool -} - -func newResourceScanState() *resourceScanState { - return &resourceScanState{ - diskWarning: interval.NewBackoffTimer(), - memWarning: interval.NewBackoffTimer(), - } -} - -// logs a warning if user-set threshold is surpassed -func (db *DB) resourceUseWarn(mon *memwatch.Monitor, du diskUse) { - mon.Refresh() - db.diskUseWarn(du) - db.memUseWarn(mon) -} - -func (db *DB) diskUseWarn(du diskUse) { - diskWarnPercent := db.config.ResourceUsage.DiskUse.WarningPercentage - if diskWarnPercent > 0 { - if pu := du.percentUsed(); pu > float64(diskWarnPercent) { - if db.resourceScanState.diskWarning.IntervalElapsed() { - db.logger.WithField("action", "read_disk_use"). - WithField("path", db.config.RootPath). - Warnf("disk usage currently at %.2f%%, threshold set to %.2f%%", - pu, float64(diskWarnPercent)) - - db.logger.WithField("action", "disk_use_stats"). - WithField("path", db.config.RootPath). - Debugf("%s", du.String()) - db.resourceScanState.diskWarning.IncreaseInterval() - } - } - } -} - -func (db *DB) memUseWarn(mon *memwatch.Monitor) { - memWarnPercent := db.config.ResourceUsage.MemUse.WarningPercentage - if memWarnPercent > 0 { - if pu := mon.Ratio() * 100; pu > float64(memWarnPercent) { - if db.resourceScanState.memWarning.IntervalElapsed() { - db.logger.WithField("action", "read_memory_use"). - WithField("path", db.config.RootPath). - Warnf("memory usage currently at %.2f%%, threshold set to %.2f%%", - pu, float64(memWarnPercent)) - db.resourceScanState.memWarning.IncreaseInterval() - } - } - } -} - -// sets the shard to readonly if user-set threshold is surpassed -func (db *DB) resourceUseReadonly(mon *memwatch.Monitor, du diskUse) { - db.diskUseReadonly(du) - db.memUseReadonly(mon) -} - -func (db *DB) diskUseReadonly(du diskUse) { - diskROPercent := db.config.ResourceUsage.DiskUse.ReadOnlyPercentage - if diskROPercent > 0 { - if pu := du.percentUsed(); pu > float64(diskROPercent) { - db.setShardsReadOnly() - db.logger.WithField("action", "set_shard_read_only"). - WithField("path", db.config.RootPath). - Warnf("Set READONLY, disk usage currently at %.2f%%, threshold set to %.2f%%", - pu, float64(diskROPercent)) - } - } -} - -func (db *DB) memUseReadonly(mon *memwatch.Monitor) { - memROPercent := db.config.ResourceUsage.MemUse.ReadOnlyPercentage - if memROPercent > 0 { - if pu := mon.Ratio() * 100; pu > float64(memROPercent) { - db.setShardsReadOnly() - db.logger.WithField("action", "set_shard_read_only"). - WithField("path", db.config.RootPath). - Warnf("Set READONLY, memory usage currently at %.2f%%, threshold set to %.2f%%", - pu, float64(memROPercent)) - } - } -} - -func (db *DB) setShardsReadOnly() { - db.indexLock.Lock() - for _, index := range db.indices { - index.ForEachShard(func(name string, shard ShardLike) error { - err := shard.UpdateStatus(storagestate.StatusReadOnly.String()) - if err != nil { - db.logger.WithField("action", "set_shard_read_only"). - WithField("path", db.config.RootPath). - WithError(err). - Fatal("failed to set to READONLY") - } - return nil - }) - } - db.indexLock.Unlock() - db.resourceScanState.isReadOnly = true -} diff --git a/adapters/repos/db/restart_journey_integration_test.go b/adapters/repos/db/restart_journey_integration_test.go deleted file mode 100644 index 71873e3955417825a9fc0f9ae5328ba579f9ba46..0000000000000000000000000000000000000000 --- a/adapters/repos/db/restart_journey_integration_test.go +++ /dev/null @@ -1,250 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package db - -import ( - "context" - "testing" - - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/dto" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - enthnsw "github.com/weaviate/weaviate/entities/vectorindex/hnsw" -) - -func TestRestartJourney(t *testing.T) { - dirName := t.TempDir() - - logger, _ := test.NewNullLogger() - thingclass := &models.Class{ - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - Class: "Class", - Properties: []*models.Property{ - { - Name: "description", - DataType: []string{string(schema.DataTypeText)}, - Tokenization: "word", - }, - }, - } - shardState := singleShardState() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: shardState, - } - repo, err := New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - migrator := NewMigrator(repo, logger) - - t.Run("creating the thing class", func(t *testing.T) { - require.Nil(t, - migrator.AddClass(context.Background(), thingclass, - shardState)) - }) - - // update schema getter so it's in sync with class - schemaGetter.schema = schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{thingclass}, - }, - } - - t.Run("import some data", func(t *testing.T) { - err := repo.PutObject(context.Background(), &models.Object{ - Class: "Class", - ID: "9d64350e-5027-40ea-98db-e3b97e6f6f8f", - Properties: map[string]interface{}{ - "description": "the band is just fantastic that is really what I think", - }, - }, []float32{0.1, 0.2, 0.3}, nil) - require.Nil(t, err) - - err = repo.PutObject(context.Background(), &models.Object{ - Class: "Class", - ID: "46ebcce8-fb77-413b-ade6-26c427af3f33", - Properties: map[string]interface{}{ - "description": "oh by the way, which one's pink?", - }, - }, []float32{-0.1, 0.2, -0.3}, nil) - require.Nil(t, err) - }) - - t.Run("control", func(t *testing.T) { - t.Run("verify object by id", func(t *testing.T) { - res, err := repo.ObjectByID(context.Background(), "46ebcce8-fb77-413b-ade6-26c427af3f33", nil, additional.Properties{}, "") - require.Nil(t, err) - require.NotNil(t, res) - assert.Equal(t, "oh by the way, which one's pink?", - res.Schema.(map[string]interface{})["description"]) - }) - - t.Run("find object by id through filter", func(t *testing.T) { - res, err := repo.ObjectSearch(context.Background(), 0, 10, - &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorEqual, - Value: &filters.Value{ - Value: "9d64350e-5027-40ea-98db-e3b97e6f6f8f", - Type: schema.DataTypeText, - }, - On: &filters.Path{ - Class: "Class", - Property: "id", - }, - }, - }, nil, additional.Properties{}, "") - require.Nil(t, err) - require.Len(t, res, 1) - assert.Equal(t, "the band is just fantastic that is really what I think", - res[0].Schema.(map[string]interface{})["description"]) - }) - - t.Run("find object through regular inverted index", func(t *testing.T) { - res, err := repo.ObjectSearch(context.Background(), 0, 10, - &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorEqual, - Value: &filters.Value{ - Value: "pink", - Type: schema.DataTypeText, - }, - On: &filters.Path{ - Class: "Class", - Property: "description", - }, - }, - }, nil, additional.Properties{}, "") - require.Nil(t, err) - require.Len(t, res, 1) - assert.Equal(t, "oh by the way, which one's pink?", - res[0].Schema.(map[string]interface{})["description"]) - }) - - t.Run("find object through vector index", func(t *testing.T) { - res, err := repo.VectorSearch(context.Background(), - dto.GetParams{ - ClassName: "Class", - SearchVector: []float32{0.05, 0.1, 0.15}, - Pagination: &filters.Pagination{ - Limit: 1, - }, - }) - require.Nil(t, err) - require.Len(t, res, 1) - assert.Equal(t, "the band is just fantastic that is really what I think", - res[0].Schema.(map[string]interface{})["description"]) - }) - }) - - var newRepo *DB - t.Run("shutdown and recreate", func(t *testing.T) { - require.Nil(t, repo.Shutdown(context.Background())) - repo = nil - - newRepo, err = New(logger, Config{ - MemtablesFlushIdleAfter: 60, - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - newRepo.SetSchemaGetter(schemaGetter) - require.Nil(t, newRepo.WaitForStartup(testCtx())) - }) - - t.Run("verify after restart", func(t *testing.T) { - t.Run("verify object by id", func(t *testing.T) { - res, err := newRepo.ObjectByID(context.Background(), "46ebcce8-fb77-413b-ade6-26c427af3f33", nil, additional.Properties{}, "") - require.Nil(t, err) - require.NotNil(t, res) - assert.Equal(t, "oh by the way, which one's pink?", - res.Schema.(map[string]interface{})["description"]) - }) - - t.Run("find object by id through filter", func(t *testing.T) { - res, err := newRepo.ObjectSearch(context.Background(), 0, 10, - &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorEqual, - Value: &filters.Value{ - Value: "9d64350e-5027-40ea-98db-e3b97e6f6f8f", - Type: schema.DataTypeText, - }, - On: &filters.Path{ - Class: "Class", - Property: "id", - }, - }, - }, nil, additional.Properties{}, "") - require.Nil(t, err) - require.Len(t, res, 1) - assert.Equal(t, "the band is just fantastic that is really what I think", - res[0].Schema.(map[string]interface{})["description"]) - }) - - t.Run("find object through regular inverted index", func(t *testing.T) { - res, err := newRepo.ObjectSearch(context.Background(), 0, 10, - &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorEqual, - Value: &filters.Value{ - Value: "pink", - Type: schema.DataTypeText, - }, - On: &filters.Path{ - Class: "Class", - Property: "description", - }, - }, - }, nil, additional.Properties{}, "") - require.Nil(t, err) - require.Len(t, res, 1) - assert.Equal(t, "oh by the way, which one's pink?", - res[0].Schema.(map[string]interface{})["description"]) - }) - - t.Run("find object through vector index", func(t *testing.T) { - res, err := newRepo.VectorSearch(context.Background(), - dto.GetParams{ - ClassName: "Class", - SearchVector: []float32{0.05, 0.1, 0.15}, - Pagination: &filters.Pagination{ - Limit: 1, - }, - }) - require.Nil(t, err) - require.Len(t, res, 1) - assert.Equal(t, "the band is just fantastic that is really what I think", - res[0].Schema.(map[string]interface{})["description"]) - }) - }) - - t.Run("shutdown", func(t *testing.T) { - require.Nil(t, newRepo.Shutdown(context.Background())) - }) -} diff --git a/adapters/repos/db/search.go b/adapters/repos/db/search.go deleted file mode 100644 index 9500bdec52d62e3408f717f0074aad44399ecb37..0000000000000000000000000000000000000000 --- a/adapters/repos/db/search.go +++ /dev/null @@ -1,432 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "context" - "fmt" - "sort" - "strings" - "sync" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/adapters/repos/db/refcache" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/aggregation" - "github.com/weaviate/weaviate/entities/dto" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/search" - "github.com/weaviate/weaviate/entities/searchparams" - "github.com/weaviate/weaviate/entities/storobj" - "github.com/weaviate/weaviate/usecases/objects" - "github.com/weaviate/weaviate/usecases/traverser" -) - -func (db *DB) Aggregate(ctx context.Context, - params aggregation.Params, -) (*aggregation.Result, error) { - idx := db.GetIndex(params.ClassName) - if idx == nil { - return nil, fmt.Errorf("tried to browse non-existing index for %s", params.ClassName) - } - - return idx.aggregate(ctx, params) -} - -func (db *DB) GetQueryMaximumResults() int { - return int(db.config.QueryMaximumResults) -} - -// SparseObjectSearch is used to perform an inverted index search on the db -// -// Earlier use cases required only []search.Result as a return value from the db, and the -// Class ClassSearch method fit this need. Later on, other use cases presented the need -// for the raw storage objects, such as hybrid search. -func (db *DB) SparseObjectSearch(ctx context.Context, params dto.GetParams) ([]*storobj.Object, []float32, error) { - idx := db.GetIndex(schema.ClassName(params.ClassName)) - if idx == nil { - return nil, nil, fmt.Errorf("tried to browse non-existing index for %s", params.ClassName) - } - - if params.Pagination == nil { - return nil, nil, fmt.Errorf("invalid params, pagination object is nil") - } - - totalLimit, err := db.getTotalLimit(params.Pagination, params.AdditionalProperties) - if err != nil { - return nil, nil, errors.Wrapf(err, "invalid pagination params") - } - - // if this is reference search and tenant is given (as origin class is MT) - // but searched class is non-MT, then skip tenant to pass validation - tenant := params.Tenant - if !idx.partitioningEnabled && params.IsRefOrigin { - tenant = "" - } - - res, dist, err := idx.objectSearch(ctx, totalLimit, - params.Filters, params.KeywordRanking, params.Sort, params.Cursor, - params.AdditionalProperties, params.ReplicationProperties, tenant, params.Pagination.Autocut) - if err != nil { - return nil, nil, errors.Wrapf(err, "object search at index %s", idx.ID()) - } - - return res, dist, nil -} - -func (db *DB) Search(ctx context.Context, params dto.GetParams) ([]search.Result, error) { - if params.Pagination == nil { - return nil, fmt.Errorf("invalid params, pagination object is nil") - } - - res, _, err := db.SparseObjectSearch(ctx, params) - if err != nil { - return nil, err - } - - return db.ResolveReferences(ctx, - storobj.SearchResults(db.getStoreObjects(res, params.Pagination), params.AdditionalProperties, params.Tenant), - params.Properties, params.GroupBy, params.AdditionalProperties, params.Tenant) -} - -func (db *DB) VectorSearch(ctx context.Context, - params dto.GetParams, -) ([]search.Result, error) { - if params.SearchVector == nil { - return db.Search(ctx, params) - } - - totalLimit, err := db.getTotalLimit(params.Pagination, params.AdditionalProperties) - if err != nil { - return nil, fmt.Errorf("invalid pagination params: %w", err) - } - - idx := db.GetIndex(schema.ClassName(params.ClassName)) - if idx == nil { - return nil, fmt.Errorf("tried to browse non-existing index for %s", params.ClassName) - } - - targetDist := extractDistanceFromParams(params) - res, dists, err := idx.objectVectorSearch(ctx, params.SearchVector, - targetDist, totalLimit, params.Filters, params.Sort, params.GroupBy, - params.AdditionalProperties, params.ReplicationProperties, params.Tenant) - if err != nil { - return nil, errors.Wrapf(err, "object vector search at index %s", idx.ID()) - } - - if totalLimit < 0 { - params.Pagination.Limit = len(res) - } - - return db.ResolveReferences(ctx, - storobj.SearchResultsWithDists(db.getStoreObjects(res, params.Pagination), - params.AdditionalProperties, db.getDists(dists, params.Pagination)), - params.Properties, params.GroupBy, params.AdditionalProperties, params.Tenant) -} - -func extractDistanceFromParams(params dto.GetParams) float32 { - certainty := traverser.ExtractCertaintyFromParams(params) - if certainty != 0 { - return float32(additional.CertaintyToDist(certainty)) - } - - dist, _ := traverser.ExtractDistanceFromParams(params) - return float32(dist) -} - -// DenseObjectSearch is used to perform a vector search on the db -// -// Earlier use cases required only []search.Result as a return value from the db, and the -// Class VectorSearch method fit this need. Later on, other use cases presented the need -// for the raw storage objects, such as hybrid search. -func (db *DB) DenseObjectSearch(ctx context.Context, class string, vector []float32, - offset int, limit int, filters *filters.LocalFilter, addl additional.Properties, - tenant string, -) ([]*storobj.Object, []float32, error) { - totalLimit := offset + limit - - index := db.GetIndex(schema.ClassName(class)) - if index == nil { - return nil, nil, fmt.Errorf("tried to browse non-existing index for %s", class) - } - - // TODO: groupBy think of this - objs, dist, err := index.objectVectorSearch(ctx, vector, 0, - totalLimit, filters, nil, nil, addl, nil, tenant) - if err != nil { - return nil, nil, fmt.Errorf("search index %s: %w", index.ID(), err) - } - - return objs, dist, nil -} - -func (db *DB) CrossClassVectorSearch(ctx context.Context, vector []float32, offset, limit int, - filters *filters.LocalFilter, -) ([]search.Result, error) { - var found search.Results - - wg := &sync.WaitGroup{} - mutex := &sync.Mutex{} - var searchErrors []error - totalLimit := offset + limit - - db.indexLock.RLock() - for _, index := range db.indices { - wg.Add(1) - go func(index *Index, wg *sync.WaitGroup) { - defer wg.Done() - - objs, dist, err := index.objectVectorSearch(ctx, vector, - 0, totalLimit, filters, nil, nil, - additional.Properties{}, nil, "") - if err != nil { - mutex.Lock() - searchErrors = append(searchErrors, errors.Wrapf(err, "search index %s", index.ID())) - mutex.Unlock() - } - - mutex.Lock() - found = append(found, storobj.SearchResultsWithDists(objs, additional.Properties{}, dist)...) - mutex.Unlock() - }(index, wg) - } - db.indexLock.RUnlock() - - wg.Wait() - - if len(searchErrors) > 0 { - var msg strings.Builder - for i, err := range searchErrors { - if i != 0 { - msg.WriteString(", ") - } - msg.WriteString(err.Error()) - } - return nil, errors.New(msg.String()) - } - - sort.Slice(found, func(i, j int) bool { - return found[i].Dist < found[j].Dist - }) - - // not enriching by refs, as a vector search result cannot provide - // SelectProperties - return db.getSearchResults(found, offset, limit), nil -} - -// Query a specific class -func (db *DB) Query(ctx context.Context, q *objects.QueryInput) (search.Results, *objects.Error) { - totalLimit := q.Offset + q.Limit - if totalLimit == 0 { - return nil, nil - } - if len(q.Sort) > 0 { - scheme := db.schemaGetter.GetSchemaSkipAuth() - if err := filters.ValidateSort(scheme, schema.ClassName(q.Class), q.Sort); err != nil { - return nil, &objects.Error{Msg: "sorting", Code: objects.StatusBadRequest, Err: err} - } - } - idx := db.GetIndex(schema.ClassName(q.Class)) - if idx == nil { - return nil, &objects.Error{Msg: "class not found " + q.Class, Code: objects.StatusNotFound} - } - if q.Cursor != nil { - if err := filters.ValidateCursor(schema.ClassName(q.Class), q.Cursor, q.Offset, q.Filters, q.Sort); err != nil { - return nil, &objects.Error{Msg: "cursor api: invalid 'after' parameter", Code: objects.StatusBadRequest, Err: err} - } - } - res, _, err := idx.objectSearch(ctx, totalLimit, q.Filters, - nil, q.Sort, q.Cursor, q.Additional, nil, q.Tenant, 0) - if err != nil { - switch err.(type) { - case objects.ErrMultiTenancy: - return nil, &objects.Error{Msg: "search index " + idx.ID(), Code: objects.StatusUnprocessableEntity, Err: err} - default: - return nil, &objects.Error{Msg: "search index " + idx.ID(), Code: objects.StatusInternalServerError, Err: err} - } - } - return db.getSearchResults(storobj.SearchResults(res, q.Additional, ""), q.Offset, q.Limit), nil -} - -// ObjectSearch search each index. -// Deprecated by Query which searches a specific index -func (db *DB) ObjectSearch(ctx context.Context, offset, limit int, - filters *filters.LocalFilter, sort []filters.Sort, - additional additional.Properties, tenant string, -) (search.Results, error) { - return db.objectSearch(ctx, offset, limit, filters, sort, additional, tenant) -} - -func (db *DB) objectSearch(ctx context.Context, offset, limit int, - filters *filters.LocalFilter, sort []filters.Sort, - additional additional.Properties, tenant string, -) (search.Results, error) { - var found []*storobj.Object - - if err := db.validateSort(sort); err != nil { - return nil, errors.Wrap(err, "search") - } - - totalLimit := offset + limit - // TODO: Search in parallel, rather than sequentially or this will be - // painfully slow on large schemas - // wrapped in func to unlock mutex within defer - if err := func() error { - db.indexLock.RLock() - defer db.indexLock.RUnlock() - - for _, index := range db.indices { - // TODO support all additional props - res, _, err := index.objectSearch(ctx, totalLimit, - filters, nil, sort, nil, additional, nil, tenant, 0) - if err != nil { - // Multi tenancy specific errors - if errors.As(err, &objects.ErrMultiTenancy{}) { - // validation failed (either MT class without tenant or non-MT class with tenant) - if strings.Contains(err.Error(), "has multi-tenancy enabled, but request was without tenant") || - strings.Contains(err.Error(), "has multi-tenancy disabled, but request was with tenant") { - continue - } - // tenant not added to class - if strings.Contains(err.Error(), "no tenant found with key") { - continue - } - // tenant does belong to this class - if errors.As(err, &errTenantNotFound) { - continue // tenant does belong to this class - } - } - return errors.Wrapf(err, "search index %s", index.ID()) - } - - found = append(found, res...) - if len(found) >= totalLimit { - // we are done - break - } - } - return nil - }(); err != nil { - return nil, err - } - - return db.getSearchResults(storobj.SearchResults(found, additional, tenant), offset, limit), nil -} - -// ResolveReferences takes a list of search results and enriches them -// with any referenced objects -func (db *DB) ResolveReferences(ctx context.Context, objs search.Results, - props search.SelectProperties, groupBy *searchparams.GroupBy, - addl additional.Properties, tenant string, -) (search.Results, error) { - if addl.NoProps { - // If we have no props, there also can't be refs among them, so we can skip - // the refcache resolver - return objs, nil - } - - if groupBy != nil { - res, err := refcache.NewResolverWithGroup(refcache.NewCacherWithGroup(db, db.logger, tenant)). - Do(ctx, objs, props, addl) - if err != nil { - return nil, fmt.Errorf("resolve cross-refs: %w", err) - } - return res, nil - } - - res, err := refcache.NewResolver(refcache.NewCacher(db, db.logger, tenant)). - Do(ctx, objs, props, addl) - if err != nil { - return nil, fmt.Errorf("resolve cross-refs: %w", err) - } - - return res, nil -} - -func (db *DB) validateSort(sort []filters.Sort) error { - if len(sort) > 0 { - var errorMsgs []string - // needs to happen before the index lock as they might deadlock each other - schema := db.schemaGetter.GetSchemaSkipAuth() - db.indexLock.RLock() - for _, index := range db.indices { - err := filters.ValidateSort(schema, - index.Config.ClassName, sort) - if err != nil { - errorMsg := errors.Wrapf(err, "search index %s", index.ID()).Error() - errorMsgs = append(errorMsgs, errorMsg) - } - } - db.indexLock.RUnlock() - if len(errorMsgs) > 0 { - return errors.Errorf("%s", strings.Join(errorMsgs, ", ")) - } - } - return nil -} - -func (db *DB) getTotalLimit(pagination *filters.Pagination, addl additional.Properties) (int, error) { - if pagination.Limit == filters.LimitFlagSearchByDist { - return filters.LimitFlagSearchByDist, nil - } - - totalLimit := pagination.Offset + db.getLimit(pagination.Limit) - if totalLimit == 0 { - return 0, fmt.Errorf("invalid default limit: %v", db.getLimit(pagination.Limit)) - } - if !addl.ReferenceQuery && totalLimit > int(db.config.QueryMaximumResults) { - return 0, errors.New("query maximum results exceeded") - } - return totalLimit, nil -} - -func (db *DB) getSearchResults(found search.Results, paramOffset, paramLimit int) search.Results { - offset, limit := db.getOffsetLimit(len(found), paramOffset, paramLimit) - if offset == 0 && limit == 0 { - return nil - } - return found[offset:limit] -} - -func (db *DB) getStoreObjects(res []*storobj.Object, pagination *filters.Pagination) []*storobj.Object { - offset, limit := db.getOffsetLimit(len(res), pagination.Offset, pagination.Limit) - if offset == 0 && limit == 0 { - return nil - } - return res[offset:limit] -} - -func (db *DB) getDists(dists []float32, pagination *filters.Pagination) []float32 { - offset, limit := db.getOffsetLimit(len(dists), pagination.Offset, pagination.Limit) - if offset == 0 && limit == 0 { - return nil - } - return dists[offset:limit] -} - -func (db *DB) getOffsetLimit(arraySize int, offset, limit int) (int, int) { - totalLimit := offset + db.getLimit(limit) - if arraySize > totalLimit { - return offset, totalLimit - } else if arraySize > offset { - return offset, arraySize - } - return 0, 0 -} - -func (db *DB) getLimit(limit int) int { - if limit == filters.LimitFlagNotSet { - return int(db.config.QueryLimit) - } - return limit -} diff --git a/adapters/repos/db/shard.go b/adapters/repos/db/shard.go deleted file mode 100644 index 4896913867cf32568c5befaa8ec5f762637cc930..0000000000000000000000000000000000000000 --- a/adapters/repos/db/shard.go +++ /dev/null @@ -1,835 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "context" - "fmt" - "io" - "os" - "path" - "sync" - "time" - - "github.com/go-openapi/strfmt" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/adapters/repos/db/indexcheckpoint" - "github.com/weaviate/weaviate/adapters/repos/db/indexcounter" - "github.com/weaviate/weaviate/adapters/repos/db/inverted" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv" - "github.com/weaviate/weaviate/adapters/repos/db/propertyspecific" - "github.com/weaviate/weaviate/adapters/repos/db/vector/flat" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" - "github.com/weaviate/weaviate/adapters/repos/db/vector/noop" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/aggregation" - "github.com/weaviate/weaviate/entities/backup" - "github.com/weaviate/weaviate/entities/cyclemanager" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/multi" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/search" - "github.com/weaviate/weaviate/entities/searchparams" - "github.com/weaviate/weaviate/entities/storagestate" - "github.com/weaviate/weaviate/entities/storobj" - "github.com/weaviate/weaviate/entities/vectorindex" - "github.com/weaviate/weaviate/entities/vectorindex/common" - flatent "github.com/weaviate/weaviate/entities/vectorindex/flat" - hnswent "github.com/weaviate/weaviate/entities/vectorindex/hnsw" - "github.com/weaviate/weaviate/usecases/monitoring" - "github.com/weaviate/weaviate/usecases/objects" - "github.com/weaviate/weaviate/usecases/replica" - "golang.org/x/sync/errgroup" -) - -const IdLockPoolSize = 128 - -type ShardLike interface { - Index() *Index // Get the parent index - Name() string // Get the shard name - Store() *lsmkv.Store // Get the underlying store - NotifyReady() // Set shard status to ready - GetStatus() storagestate.Status // Return the shard status - UpdateStatus(status string) error // Set shard status - FindUUIDs(ctx context.Context, filters *filters.LocalFilter) ([]strfmt.UUID, error) // Search and return document ids - - Counter() *indexcounter.Counter - ObjectCount() int - GetPropertyLengthTracker() *inverted.JsonShardMetaData - - PutObject(context.Context, *storobj.Object) error - PutObjectBatch(context.Context, []*storobj.Object) []error - ObjectByID(ctx context.Context, id strfmt.UUID, props search.SelectProperties, additional additional.Properties) (*storobj.Object, error) - Exists(ctx context.Context, id strfmt.UUID) (bool, error) - ObjectSearch(ctx context.Context, limit int, filters *filters.LocalFilter, keywordRanking *searchparams.KeywordRanking, sort []filters.Sort, cursor *filters.Cursor, additional additional.Properties) ([]*storobj.Object, []float32, error) - ObjectVectorSearch(ctx context.Context, searchVector []float32, targetDist float32, limit int, filters *filters.LocalFilter, sort []filters.Sort, groupBy *searchparams.GroupBy, additional additional.Properties) ([]*storobj.Object, []float32, error) - UpdateVectorIndexConfig(ctx context.Context, updated schema.VectorIndexConfig) error - AddReferencesBatch(ctx context.Context, refs objects.BatchReferences) []error - DeleteObjectBatch(ctx context.Context, ids []strfmt.UUID, dryRun bool) objects.BatchSimpleObjects // Delete many objects by id - DeleteObject(ctx context.Context, id strfmt.UUID) error // Delete object by id - MultiObjectByID(ctx context.Context, query []multi.Identifier) ([]*storobj.Object, error) - ID() string // Get the shard id - drop() error - addIDProperty(ctx context.Context) error - addDimensionsProperty(ctx context.Context) error - addTimestampProperties(ctx context.Context) error - createPropertyIndex(ctx context.Context, prop *models.Property, eg *errgroup.Group) - BeginBackup(ctx context.Context) error - ListBackupFiles(ctx context.Context, ret *backup.ShardDescriptor) error // - resumeMaintenanceCycles(ctx context.Context) error - SetPropertyLengths(props []inverted.Property) error - AnalyzeObject(*storobj.Object) ([]inverted.Property, []inverted.NilProperty, error) // - - // TODO tests only - Dimensions() int // dim(vector)*number vectors - // TODO tests only - QuantizedDimensions(segments int) int - Aggregate(ctx context.Context, params aggregation.Params) (*aggregation.Result, error) // - MergeObject(ctx context.Context, object objects.MergeDocument) error // - Queue() *IndexQueue - Shutdown(context.Context) error // Shutdown the shard - // TODO tests only - ObjectList(ctx context.Context, limit int, sort []filters.Sort, cursor *filters.Cursor, additional additional.Properties, className schema.ClassName) ([]*storobj.Object, error) // Search and return objects - WasDeleted(ctx context.Context, id strfmt.UUID) (bool, error) // Check if an object was deleted - VectorIndex() VectorIndex // Get the vector index - // TODO tests only - Versioner() *shardVersioner // Get the shard versioner - - isReadOnly() bool - - preparePutObject(context.Context, string, *storobj.Object) replica.SimpleResponse - preparePutObjects(context.Context, string, []*storobj.Object) replica.SimpleResponse - prepareMergeObject(context.Context, string, *objects.MergeDocument) replica.SimpleResponse - prepareDeleteObject(context.Context, string, strfmt.UUID) replica.SimpleResponse - prepareDeleteObjects(context.Context, string, []strfmt.UUID, bool) replica.SimpleResponse - prepareAddReferences(context.Context, string, []objects.BatchReference) replica.SimpleResponse - - commitReplication(context.Context, string, *backupMutex) interface{} - abortReplication(context.Context, string) replica.SimpleResponse - reinit(context.Context) error - filePutter(context.Context, string) (io.WriteCloser, error) - - extendDimensionTrackerLSM(int, uint64) error - - addToPropertySetBucket(bucket *lsmkv.Bucket, docID uint64, key []byte) error - addToPropertyMapBucket(bucket *lsmkv.Bucket, pair lsmkv.MapPair, key []byte) error - pairPropertyWithFrequency(docID uint64, freq, propLen float32) lsmkv.MapPair - - setFallbackToSearchable(fallback bool) - addJobToQueue(job job) - uuidFromDocID(docID uint64) (strfmt.UUID, error) - batchDeleteObject(ctx context.Context, id strfmt.UUID) error - putObjectLSM(object *storobj.Object, idBytes []byte) (objectInsertStatus, error) - mutableMergeObjectLSM(merge objects.MergeDocument, idBytes []byte) (mutableMergeResult, error) - deleteFromPropertySetBucket(bucket *lsmkv.Bucket, docID uint64, key []byte) error - batchExtendInvertedIndexItemsLSMNoFrequency(b *lsmkv.Bucket, item inverted.MergeItem) error - updatePropertySpecificIndices(object *storobj.Object, status objectInsertStatus) error - updateVectorIndexIgnoreDelete(vector []float32, status objectInsertStatus) error - hasGeoIndex() bool - ChangeObjectCountBy(int) error - - Metrics() *Metrics -} - -// Shard is the smallest completely-contained index unit. A shard manages -// database files for all the objects it owns. How a shard is determined for a -// target object (e.g. Murmur hash, etc.) is still open at this point -type Shard struct { - class *models.Class - index *Index // a reference to the underlying index, which in turn contains schema information - queue *IndexQueue - name string - store *lsmkv.Store - counter *indexcounter.Counter - indexCheckpoints *indexcheckpoint.Checkpoints - vectorIndex VectorIndex - metrics *Metrics - promMetrics *monitoring.PrometheusMetrics - propertyIndices propertyspecific.Indices - propLenTracker *inverted.JsonShardMetaData - versioner *shardVersioner - - status storagestate.Status - statusLock sync.Mutex - propertyIndicesLock sync.RWMutex - stopMetrics chan struct{} - - centralJobQueue chan job // reference to queue used by all shards - - docIdLock []sync.Mutex - // replication - replicationMap pendingReplicaTasks - - // Indicates whether searchable buckets should be used - // when filterable buckets are missing for text/text[] properties - // This can happen for db created before v1.19, where - // only map (now called searchable) buckets were created as inverted - // indexes for text/text[] props. - // Now roaring set (filterable) and map (searchable) buckets can - // coexists for text/text[] props, and by default both are enabled. - // So despite property's IndexFilterable and IndexSearchable settings - // being enabled, only searchable bucket exists - fallbackToSearchable bool - - cycleCallbacks *shardCycleCallbacks -} - -func (s *Shard) initShard(ctx context.Context) (*Shard, error) { - before := time.Now() - var err error - s.initCycleCallbacks() - - s.docIdLock = make([]sync.Mutex, IdLockPoolSize) - - defer s.metrics.ShardStartup(before) - - _, err = os.Stat(s.path()) - exists := false - if err == nil { - exists = true - } - - if err := os.MkdirAll(s.path(), os.ModePerm); err != nil { - return nil, err - } - - if s.propLenTracker == nil { - plPath := path.Join(s.path(), "proplengths") - tracker, err := inverted.NewJsonShardMetaData(plPath, s.index.logger) - if err != nil { - return nil, errors.Wrapf(err, "init shard %q: prop length tracker", s.ID()) - } - - s.propLenTracker = tracker - } - - if err := s.initNonVector(ctx, s.class); err != nil { - return nil, errors.Wrapf(err, "init shard %q", s.ID()) - } - - if err := s.initVector(ctx); err != nil { - return nil, err - } - - s.queue, err = NewIndexQueue(s.ID(), s, s.VectorIndex(), s.centralJobQueue, s.indexCheckpoints, IndexQueueOptions{Logger: s.index.logger}) - if err != nil { - return nil, err - } - - if asyncEnabled() { - go func() { - // preload unindexed objects in the background - err = s.queue.PreloadShard(s) - if err != nil { - s.queue.Logger.WithError(err).Error("preload shard") - } - }() - } - s.NotifyReady() - - if exists { - s.index.logger.Printf("Completed loading shard %s in %s", s.ID(), time.Since(before)) - } else { - s.index.logger.Printf("Created shard %s in %s", s.ID(), time.Since(before)) - } - - return s, nil -} - -/* NewShard - create a new physical storage shard - * ctx - the timeout context - * promMetrics - prometheus metrics - * shardName - shard name - * index - The owning index - * class - The class this shard belongs to - * jobQueueCh - The central job queue - * indexCheckpoints - The index checkpoints - * propLengths - The property lengths tracker. There should only be one instance per shard, and it is stored in the shard. If a null pointer is passed, a new one will be created. Care must be taken to only create one, otherwise they will overwrite each other on disk. - */ -func NewShard(ctx context.Context, promMetrics *monitoring.PrometheusMetrics, - shardName string, index *Index, class *models.Class, jobQueueCh chan job, - indexCheckpoints *indexcheckpoint.Checkpoints, propLengths *inverted.JsonShardMetaData, -) (*Shard, error) { - s := &Shard{ - index: index, - name: shardName, - promMetrics: promMetrics, - metrics: NewMetrics(index.logger, promMetrics, - string(index.Config.ClassName), shardName), - stopMetrics: make(chan struct{}), - replicationMap: pendingReplicaTasks{Tasks: make(map[string]replicaTask, 32)}, - centralJobQueue: jobQueueCh, - indexCheckpoints: indexCheckpoints, - propLenTracker: propLengths, - class: class, - } - return s.initShard(ctx) -} - -func (s *Shard) initVector(ctx context.Context) error { - var distProv distancer.Provider - - switch s.index.vectorIndexUserConfig.DistanceName() { - case "", common.DistanceCosine: - distProv = distancer.NewCosineDistanceProvider() - case common.DistanceDot: - distProv = distancer.NewDotProductProvider() - case common.DistanceL2Squared: - distProv = distancer.NewL2SquaredProvider() - case common.DistanceManhattan: - distProv = distancer.NewManhattanProvider() - case common.DistanceHamming: - distProv = distancer.NewHammingProvider() - default: - return fmt.Errorf("init vector index: %w", - errors.Errorf("unrecognized distance metric %q,"+ - "choose one of [\"cosine\", \"dot\", \"l2-squared\", \"manhattan\",\"hamming\"]", s.index.vectorIndexUserConfig.DistanceName())) - } - - switch s.index.vectorIndexUserConfig.IndexType() { - case vectorindex.VectorIndexTypeHNSW: - hnswUserConfig, ok := s.index.vectorIndexUserConfig.(hnswent.UserConfig) - if !ok { - return errors.Errorf("hnsw vector index: config is not hnsw.UserConfig: %T", - s.index.vectorIndexUserConfig) - } - - if hnswUserConfig.Skip { - s.vectorIndex = noop.NewIndex() - } else { - // starts vector cycles if vector is configured - s.index.cycleCallbacks.vectorCommitLoggerCycle.Start() - s.index.cycleCallbacks.vectorTombstoneCleanupCycle.Start() - - // a shard can actually have multiple vector indexes: - // - the main index, which is used for all normal object vectors - // - a geo property index for each geo prop in the schema - // - // here we label the main vector index as such. - vecIdxID := "main" - - vi, err := hnsw.New(hnsw.Config{ - Logger: s.index.logger, - RootPath: s.path(), - ID: vecIdxID, - ShardName: s.name, - ClassName: s.index.Config.ClassName.String(), - PrometheusMetrics: s.promMetrics, - VectorForIDThunk: s.vectorByIndexID, - TempVectorForIDThunk: s.readVectorByIndexIDIntoSlice, - DistanceProvider: distProv, - MakeCommitLoggerThunk: func() (hnsw.CommitLogger, error) { - return hnsw.NewCommitLogger(s.path(), vecIdxID, - s.index.logger, s.cycleCallbacks.vectorCommitLoggerCallbacks) - }, - }, hnswUserConfig, s.cycleCallbacks.vectorTombstoneCleanupCallbacks, - s.cycleCallbacks.compactionCallbacks, s.cycleCallbacks.flushCallbacks, s.store) - if err != nil { - return errors.Wrapf(err, "init shard %q: hnsw index", s.ID()) - } - s.vectorIndex = vi - - defer s.vectorIndex.PostStartup() - } - case vectorindex.VectorIndexTypeFLAT: - flatUserConfig, ok := s.index.vectorIndexUserConfig.(flatent.UserConfig) - if !ok { - return errors.Errorf("flat vector index: config is not flat.UserConfig: %T", - s.index.vectorIndexUserConfig) - } - s.index.cycleCallbacks.vectorCommitLoggerCycle.Start() - - // a shard can actually have multiple vector indexes: - // - the main index, which is used for all normal object vectors - // - a geo property index for each geo prop in the schema - // - // here we label the main vector index as such. - vecIdxID := "main" - - vi, err := flat.New(flat.Config{ - ID: vecIdxID, - Logger: s.index.logger, - DistanceProvider: distProv, - }, flatUserConfig, s.store) - if err != nil { - return errors.Wrapf(err, "init shard %q: flat index", s.ID()) - } - s.vectorIndex = vi - default: - return fmt.Errorf("Unknown vector index type: %q. Choose one from [\"%s\", \"%s\"]", - s.index.vectorIndexUserConfig.IndexType(), vectorindex.VectorIndexTypeHNSW, vectorindex.VectorIndexTypeFLAT) - } - - return nil -} - -func (s *Shard) initNonVector(ctx context.Context, class *models.Class) error { - err := s.initLSMStore(ctx) - if err != nil { - return errors.Wrapf(err, "init shard %q: shard db", s.ID()) - } - - counter, err := indexcounter.New(s.path()) - if err != nil { - return errors.Wrapf(err, "init shard %q: index counter", s.ID()) - } - s.counter = counter - - dataPresent := s.counter.PreviewNext() != 0 - versionPath := path.Join(s.path(), "version") - versioner, err := newShardVersioner(versionPath, dataPresent) - if err != nil { - return errors.Wrapf(err, "init shard %q: check versions", s.ID()) - } - s.versioner = versioner - - if err := s.initProperties(class); err != nil { - return errors.Wrapf(err, "init shard %q: init per property indices", s.ID()) - } - - s.initDimensionTracking() - - return nil -} - -func (s *Shard) ID() string { - return shardId(s.index.ID(), s.name) -} - -func (s *Shard) path() string { - return shardPath(s.index.path(), s.name) -} - -func (s *Shard) pathLSM() string { - return path.Join(s.path(), "lsm") -} - -func (s *Shard) uuidToIdLockPoolId(idBytes []byte) uint8 { - // use the last byte of the uuid to determine which locking-pool a given object should use. The last byte is used - // as uuids probably often have some kind of order and the last byte will in general be the one that changes the most - return idBytes[15] % IdLockPoolSize -} - -func (s *Shard) initLSMStore(ctx context.Context) error { - annotatedLogger := s.index.logger.WithFields(logrus.Fields{ - "shard": s.name, - "index": s.index.ID(), - "class": s.index.Config.ClassName, - }) - var metrics *lsmkv.Metrics - if s.promMetrics != nil { - metrics = lsmkv.NewMetrics(s.promMetrics, string(s.index.Config.ClassName), s.name) - } - - store, err := lsmkv.New(s.pathLSM(), s.path(), annotatedLogger, metrics, - s.cycleCallbacks.compactionCallbacks, s.cycleCallbacks.flushCallbacks) - if err != nil { - return errors.Wrapf(err, "init lsmkv store at %s", s.pathLSM()) - } - - err = store.CreateOrLoadBucket(ctx, helpers.ObjectsBucketLSM, - lsmkv.WithStrategy(lsmkv.StrategyReplace), - lsmkv.WithSecondaryIndices(1), - lsmkv.WithMonitorCount(), - lsmkv.WithPread(s.index.Config.AvoidMMap), - lsmkv.WithKeepTombstones(true), - s.dynamicMemtableSizing(), - s.memtableIdleConfig(), - ) - if err != nil { - return errors.Wrap(err, "create objects bucket") - } - - s.store = store - - return nil -} - -// IMPORTANT: -// Be advised there exists LazyLoadShard::drop() implementation intended -// to drop shard that was not loaded (instantiated) yet. -// It deletes shard by performing required actions and removing entire shard directory. -// If there is any action that needs to be performed beside files/dirs being removed -// from shard directory, it needs to be reflected as well in LazyLoadShard::drop() -// method to keep drop behaviour consistent. -func (s *Shard) drop() error { - s.metrics.DeleteShardLabels(s.index.Config.ClassName.String(), s.name) - s.metrics.baseMetrics.StartUnloadingShard(s.index.Config.ClassName.String()) - s.replicationMap.clear() - - if s.index.Config.TrackVectorDimensions { - // tracking vector dimensions goroutine only works when tracking is enabled - // that's why we are trying to stop it only in this case - s.stopMetrics <- struct{}{} - if s.promMetrics != nil { - // send 0 in when index gets dropped - s.clearDimensionMetrics() - } - } - - ctx, cancel := context.WithTimeout(context.TODO(), 5*time.Second) - defer cancel() - - // unregister all callbacks at once, in parallel - if err := cyclemanager.NewCombinedCallbackCtrl(0, - s.cycleCallbacks.compactionCallbacksCtrl, - s.cycleCallbacks.flushCallbacksCtrl, - s.cycleCallbacks.vectorCombinedCallbacksCtrl, - s.cycleCallbacks.geoPropsCombinedCallbacksCtrl, - ).Unregister(ctx); err != nil { - return err - } - - if err := s.store.Shutdown(ctx); err != nil { - return errors.Wrap(err, "stop lsmkv store") - } - - if _, err := os.Stat(s.pathLSM()); err == nil { - err := os.RemoveAll(s.pathLSM()) - if err != nil { - return errors.Wrapf(err, "remove lsm store at %s", s.pathLSM()) - } - } - // delete indexcount - err := s.counter.Drop() - if err != nil { - return errors.Wrapf(err, "remove indexcount at %s", s.path()) - } - - // delete version - err = s.versioner.Drop() - if err != nil { - return errors.Wrapf(err, "remove version at %s", s.path()) - } - - // delete queue cursor - err = s.queue.Drop() - if err != nil { - return errors.Wrapf(err, "close queue at %s", s.path()) - } - // remove vector index - err = s.VectorIndex().Drop(ctx) - if err != nil { - return errors.Wrapf(err, "remove vector index at %s", s.path()) - } - - // delete property length tracker - err = s.GetPropertyLengthTracker().Drop() - if err != nil { - return errors.Wrapf(err, "remove prop length tracker at %s", s.path()) - } - - s.propertyIndicesLock.Lock() - err = s.propertyIndices.DropAll(ctx) - s.propertyIndicesLock.Unlock() - if err != nil { - return errors.Wrapf(err, "remove property specific indices at %s", s.path()) - } - - s.metrics.baseMetrics.FinishUnloadingShard(s.index.Config.ClassName.String()) - - return nil -} - -func (s *Shard) addIDProperty(ctx context.Context) error { - if s.isReadOnly() { - return storagestate.ErrStatusReadOnly - } - - return s.store.CreateOrLoadBucket(ctx, - helpers.BucketFromPropNameLSM(filters.InternalPropID), - lsmkv.WithIdleThreshold(time.Duration(s.index.Config.MemtablesFlushIdleAfter)*time.Second), - lsmkv.WithStrategy(lsmkv.StrategySetCollection), - lsmkv.WithPread(s.index.Config.AvoidMMap)) -} - -func (s *Shard) addDimensionsProperty(ctx context.Context) error { - if s.isReadOnly() { - return storagestate.ErrStatusReadOnly - } - - // Note: this data would fit the "Set" type better, but since the "Map" type - // is currently optimized better, it is more efficient to use a Map here. - err := s.store.CreateOrLoadBucket(ctx, - helpers.DimensionsBucketLSM, - lsmkv.WithStrategy(lsmkv.StrategyMapCollection), - lsmkv.WithPread(s.index.Config.AvoidMMap)) - if err != nil { - return err - } - - return nil -} - -func (s *Shard) addTimestampProperties(ctx context.Context) error { - if s.isReadOnly() { - return storagestate.ErrStatusReadOnly - } - - if err := s.addCreationTimeUnixProperty(ctx); err != nil { - return err - } - if err := s.addLastUpdateTimeUnixProperty(ctx); err != nil { - return err - } - - return nil -} - -func (s *Shard) addCreationTimeUnixProperty(ctx context.Context) error { - return s.store.CreateOrLoadBucket(ctx, - helpers.BucketFromPropNameLSM(filters.InternalPropCreationTimeUnix), - lsmkv.WithIdleThreshold(time.Duration(s.index.Config.MemtablesFlushIdleAfter)*time.Second), - lsmkv.WithStrategy(lsmkv.StrategyRoaringSet), - lsmkv.WithPread(s.index.Config.AvoidMMap)) -} - -func (s *Shard) addLastUpdateTimeUnixProperty(ctx context.Context) error { - return s.store.CreateOrLoadBucket(ctx, - helpers.BucketFromPropNameLSM(filters.InternalPropLastUpdateTimeUnix), - lsmkv.WithIdleThreshold(time.Duration(s.index.Config.MemtablesFlushIdleAfter)*time.Second), - lsmkv.WithStrategy(lsmkv.StrategyRoaringSet), - lsmkv.WithPread(s.index.Config.AvoidMMap)) -} - -func (s *Shard) memtableIdleConfig() lsmkv.BucketOption { - return lsmkv.WithIdleThreshold( - time.Duration(s.index.Config.MemtablesFlushIdleAfter) * time.Second) -} - -func (s *Shard) dynamicMemtableSizing() lsmkv.BucketOption { - return lsmkv.WithDynamicMemtableSizing( - s.index.Config.MemtablesInitialSizeMB, - s.index.Config.MemtablesMaxSizeMB, - s.index.Config.MemtablesMinActiveSeconds, - s.index.Config.MemtablesMaxActiveSeconds, - ) -} - -func (s *Shard) createPropertyIndex(ctx context.Context, prop *models.Property, eg *errgroup.Group) { - if !inverted.HasInvertedIndex(prop) { - return - } - - eg.Go(func() error { - if err := s.createPropertyValueIndex(ctx, prop); err != nil { - return errors.Wrapf(err, "create property '%s' value index on shard '%s'", prop.Name, s.ID()) - } - - if s.index.invertedIndexConfig.IndexNullState { - eg.Go(func() error { - if err := s.createPropertyNullIndex(ctx, prop); err != nil { - return errors.Wrapf(err, "create property '%s' null index on shard '%s'", prop.Name, s.ID()) - } - return nil - }) - } - - if s.index.invertedIndexConfig.IndexPropertyLength { - eg.Go(func() error { - if err := s.createPropertyLengthIndex(ctx, prop); err != nil { - return errors.Wrapf(err, "create property '%s' length index on shard '%s'", prop.Name, s.ID()) - } - return nil - }) - } - - return nil - }) -} - -func (s *Shard) createPropertyValueIndex(ctx context.Context, prop *models.Property) error { - if s.isReadOnly() { - return storagestate.ErrStatusReadOnly - } - - bucketOpts := []lsmkv.BucketOption{ - s.memtableIdleConfig(), - s.dynamicMemtableSizing(), - lsmkv.WithPread(s.index.Config.AvoidMMap), - } - - if inverted.HasFilterableIndex(prop) { - if dt, _ := schema.AsPrimitive(prop.DataType); dt == schema.DataTypeGeoCoordinates { - return s.initGeoProp(prop) - } - - if schema.IsRefDataType(prop.DataType) { - if err := s.store.CreateOrLoadBucket(ctx, - helpers.BucketFromPropNameMetaCountLSM(prop.Name), - append(bucketOpts, lsmkv.WithStrategy(lsmkv.StrategyRoaringSet))..., - ); err != nil { - return err - } - } - - if err := s.store.CreateOrLoadBucket(ctx, - helpers.BucketFromPropNameLSM(prop.Name), - append(bucketOpts, lsmkv.WithStrategy(lsmkv.StrategyRoaringSet))..., - ); err != nil { - return err - } - } - - if inverted.HasSearchableIndex(prop) { - searchableBucketOpts := append(bucketOpts, - lsmkv.WithStrategy(lsmkv.StrategyMapCollection), lsmkv.WithPread(s.index.Config.AvoidMMap)) - if s.versioner.Version() < 2 { - searchableBucketOpts = append(searchableBucketOpts, lsmkv.WithLegacyMapSorting()) - } - - if err := s.store.CreateOrLoadBucket(ctx, - helpers.BucketSearchableFromPropNameLSM(prop.Name), - searchableBucketOpts..., - ); err != nil { - return err - } - } - - return nil -} - -func (s *Shard) createPropertyLengthIndex(ctx context.Context, prop *models.Property) error { - if s.isReadOnly() { - return storagestate.ErrStatusReadOnly - } - - // some datatypes are not added to the inverted index, so we can skip them here - switch schema.DataType(prop.DataType[0]) { - case schema.DataTypeGeoCoordinates, schema.DataTypePhoneNumber, schema.DataTypeBlob, schema.DataTypeInt, - schema.DataTypeNumber, schema.DataTypeBoolean, schema.DataTypeDate: - return nil - default: - } - - return s.store.CreateOrLoadBucket(ctx, - helpers.BucketFromPropNameLengthLSM(prop.Name), - lsmkv.WithStrategy(lsmkv.StrategyRoaringSet), - lsmkv.WithPread(s.index.Config.AvoidMMap)) -} - -func (s *Shard) createPropertyNullIndex(ctx context.Context, prop *models.Property) error { - if s.isReadOnly() { - return storagestate.ErrStatusReadOnly - } - - return s.store.CreateOrLoadBucket(ctx, - helpers.BucketFromPropNameNullLSM(prop.Name), - lsmkv.WithStrategy(lsmkv.StrategyRoaringSet), - lsmkv.WithPread(s.index.Config.AvoidMMap)) -} - -func (s *Shard) UpdateVectorIndexConfig(ctx context.Context, updated schema.VectorIndexConfig) error { - if s.isReadOnly() { - return storagestate.ErrStatusReadOnly - } - - err := s.UpdateStatus(storagestate.StatusReadOnly.String()) - if err != nil { - return fmt.Errorf("attempt to mark read-only: %w", err) - } - - return s.VectorIndex().UpdateUserConfig(updated, func() { - s.UpdateStatus(storagestate.StatusReady.String()) - }) -} - -func (s *Shard) Shutdown(ctx context.Context) error { - if s.index.Config.TrackVectorDimensions { - // tracking vector dimensions goroutine only works when tracking is enabled - // that's why we are trying to stop it only in this case - s.stopMetrics <- struct{}{} - } - - if err := s.GetPropertyLengthTracker().Close(); err != nil { - return errors.Wrap(err, "close prop length tracker") - } - - if err := s.queue.Close(); err != nil { - return errors.Wrap(err, "shut down vector index queue") - } - - // to ensure that all commitlog entries are written to disk. - // otherwise in some cases the tombstone cleanup process' - // 'RemoveTombstone' entry is not picked up on restarts - // resulting in perpetually attempting to remove a tombstone - // which doesn't actually exist anymore - if err := s.VectorIndex().Flush(); err != nil { - return errors.Wrap(err, "flush vector index commitlog") - } - - if err := s.VectorIndex().Shutdown(ctx); err != nil { - return errors.Wrap(err, "shut down vector index") - } - - // unregister all callbacks at once, in parallel - if err := cyclemanager.NewCombinedCallbackCtrl(0, - s.cycleCallbacks.compactionCallbacksCtrl, - s.cycleCallbacks.flushCallbacksCtrl, - s.cycleCallbacks.vectorCombinedCallbacksCtrl, - s.cycleCallbacks.geoPropsCombinedCallbacksCtrl, - ).Unregister(ctx); err != nil { - return err - } - - if err := s.store.Shutdown(ctx); err != nil { - return errors.Wrap(err, "stop lsmkv store") - } - - return nil -} - -func (s *Shard) NotifyReady() { - s.initStatus() - s.index.logger. - WithField("action", "startup"). - Debugf("shard=%s is ready", s.name) -} - -func (s *Shard) ObjectCount() int { - return s.GetPropertyLengthTracker().ObjectTally() -} - -func (s *Shard) isFallbackToSearchable() bool { - return s.fallbackToSearchable -} - -func (s *Shard) tenant() string { - // TODO provide better impl - if s.index.partitioningEnabled { - return s.name - } - return "" -} - -func shardId(indexId, shardName string) string { - return fmt.Sprintf("%s_%s", indexId, shardName) -} - -func shardPath(indexPath, shardName string) string { - return path.Join(indexPath, shardName) -} - -func bucketKeyPropertyLength(length int) ([]byte, error) { - return inverted.LexicographicallySortableInt64(int64(length)) -} - -func bucketKeyPropertyNull(isNull bool) ([]byte, error) { - if isNull { - return []byte{uint8(filters.InternalNullState)}, nil - } - return []byte{uint8(filters.InternalNotNullState)}, nil -} diff --git a/adapters/repos/db/shard_accessors.go b/adapters/repos/db/shard_accessors.go deleted file mode 100644 index c682a853b5df5ed6eec129c805ac941f025cc08a..0000000000000000000000000000000000000000 --- a/adapters/repos/db/shard_accessors.go +++ /dev/null @@ -1,89 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Some standard accessors for the shard struct. -// It is important to NEVER access the shard struct directly, because we lazy load shards, so the information might not be there. -package db - -import ( - "github.com/weaviate/weaviate/adapters/repos/db/indexcounter" - "github.com/weaviate/weaviate/adapters/repos/db/inverted" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv" - "github.com/weaviate/weaviate/entities/schema" -) - -func (s *Shard) Queue() *IndexQueue { - return s.queue -} - -func (s *Shard) VectorIndex() VectorIndex { - return s.vectorIndex -} - -func (s *Shard) Versioner() *shardVersioner { - return s.versioner -} - -func (s *Shard) Index() *Index { - return s.index -} - -// Shard name(identifier?) -func (s *Shard) Name() string { - return s.name -} - -// The physical data store -func (s *Shard) Store() *lsmkv.Store { - return s.store -} - -func (s *Shard) Counter() *indexcounter.Counter { - return s.counter -} - -// Tracks the lengths of all properties. Must be updated on inserts/deletes. -func (s *Shard) GetPropertyLengthTracker() *inverted.JsonShardMetaData { - if s == nil { - return nil - } - return s.propLenTracker -} - -// Tracks the lengths of all properties. Must be updated on inserts/deletes. -func (s *Shard) SetPropertyLengthTracker(tracker *inverted.JsonShardMetaData) { - s.propLenTracker = tracker -} - -// Grafana metrics -func (s *Shard) Metrics() *Metrics { - return s.metrics -} - -func (s *Shard) setFallbackToSearchable(fallback bool) { - s.fallbackToSearchable = fallback -} - -func (s *Shard) addJobToQueue(job job) { - s.centralJobQueue <- job -} - -func (s *Shard) hasGeoIndex() bool { - s.propertyIndicesLock.RLock() - defer s.propertyIndicesLock.RUnlock() - - for _, idx := range s.propertyIndices { - if idx.Type == schema.DataTypeGeoCoordinates { - return true - } - } - return false -} diff --git a/adapters/repos/db/shard_aggregate.go b/adapters/repos/db/shard_aggregate.go deleted file mode 100644 index 0a0fa505ea0c6727ebfb65dde2476196be4347ee..0000000000000000000000000000000000000000 --- a/adapters/repos/db/shard_aggregate.go +++ /dev/null @@ -1,28 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "context" - - "github.com/weaviate/weaviate/adapters/repos/db/aggregator" - "github.com/weaviate/weaviate/entities/aggregation" -) - -func (s *Shard) Aggregate(ctx context.Context, - params aggregation.Params, -) (*aggregation.Result, error) { - return aggregator.New(s.store, params, s.index.getSchema, s.index.classSearcher, - s.index.stopwords, s.versioner.Version(), s.queue, s.index.logger, s.GetPropertyLengthTracker(), - s.isFallbackToSearchable, s.tenant(), s.index.Config.QueryNestedRefLimit). - Do(ctx) -} diff --git a/adapters/repos/db/shard_backup.go b/adapters/repos/db/shard_backup.go deleted file mode 100644 index cd91965af0fdacd0bb7c132189808a98021acb32..0000000000000000000000000000000000000000 --- a/adapters/repos/db/shard_backup.go +++ /dev/null @@ -1,125 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "context" - "fmt" - "os" - "path/filepath" - - "github.com/weaviate/weaviate/entities/backup" - "golang.org/x/sync/errgroup" -) - -// BeginBackup stops compaction, and flushing memtable and commit log to begin with the backup -func (s *Shard) BeginBackup(ctx context.Context) (err error) { - defer func() { - if err != nil { - err = fmt.Errorf("pause compaction: %w", err) - if err2 := s.resumeMaintenanceCycles(ctx); err2 != nil { - err = fmt.Errorf("%w: resume maintenance: %v", err, err2) - } - } - }() - if err = s.store.PauseCompaction(ctx); err != nil { - return fmt.Errorf("pause compaction: %w", err) - } - if err = s.store.FlushMemtables(ctx); err != nil { - return fmt.Errorf("flush memtables: %w", err) - } - if err = s.cycleCallbacks.vectorCombinedCallbacksCtrl.Deactivate(ctx); err != nil { - return fmt.Errorf("pause vector maintenance: %w", err) - } - if err = s.cycleCallbacks.geoPropsCombinedCallbacksCtrl.Deactivate(ctx); err != nil { - return fmt.Errorf("pause geo props maintenance: %w", err) - } - if err = s.VectorIndex().SwitchCommitLogs(ctx); err != nil { - return fmt.Errorf("switch commit logs: %w", err) - } - return nil -} - -// ListBackupFiles lists all files used to backup a shard -func (s *Shard) ListBackupFiles(ctx context.Context, ret *backup.ShardDescriptor) error { - var err error - if err := s.readBackupMetadata(ret); err != nil { - return err - } - - if ret.Files, err = s.store.ListFiles(ctx, s.index.Config.RootPath); err != nil { - return err - } - files, err := s.VectorIndex().ListFiles(ctx, s.index.Config.RootPath) - if err != nil { - return err - } - - ret.Files = append(ret.Files, files...) - return nil -} - -func (s *Shard) resumeMaintenanceCycles(ctx context.Context) error { - var g errgroup.Group - - g.Go(func() error { - return s.store.ResumeCompaction(ctx) - }) - g.Go(func() error { - return s.cycleCallbacks.vectorCombinedCallbacksCtrl.Activate() - }) - g.Go(func() error { - return s.cycleCallbacks.geoPropsCombinedCallbacksCtrl.Activate() - }) - - if err := g.Wait(); err != nil { - return fmt.Errorf("failed to resume maintenance cycles for shard '%s': %w", s.name, err) - } - - return nil -} - -func (s *Shard) readBackupMetadata(d *backup.ShardDescriptor) (err error) { - d.Name = s.name - d.Node = s.nodeName() - fpath := s.counter.FileName() - if d.DocIDCounter, err = os.ReadFile(fpath); err != nil { - return fmt.Errorf("read shard doc-id-counter %s: %w", fpath, err) - } - d.DocIDCounterPath, err = filepath.Rel(s.index.Config.RootPath, fpath) - if err != nil { - return fmt.Errorf("docid counter path: %w", err) - } - fpath = s.GetPropertyLengthTracker().FileName() - if d.PropLengthTracker, err = os.ReadFile(fpath); err != nil { - return fmt.Errorf("read shard prop-lengths %s: %w", fpath, err) - } - d.PropLengthTrackerPath, err = filepath.Rel(s.index.Config.RootPath, fpath) - if err != nil { - return fmt.Errorf("proplength tracker path: %w", err) - } - fpath = s.versioner.path - if d.Version, err = os.ReadFile(fpath); err != nil { - return fmt.Errorf("read shard version %s: %w", fpath, err) - } - d.ShardVersionPath, err = filepath.Rel(s.index.Config.RootPath, fpath) - if err != nil { - return fmt.Errorf("shard version path: %w", err) - } - return nil -} - -func (s *Shard) nodeName() string { - node, _ := s.index.getSchema.ShardOwner( - s.index.Config.ClassName.String(), s.name) - return node -} diff --git a/adapters/repos/db/shard_cyclecallbacks.go b/adapters/repos/db/shard_cyclecallbacks.go deleted file mode 100644 index 0324b7fce4eff29ee0410b700e7626fec5d78c04..0000000000000000000000000000000000000000 --- a/adapters/repos/db/shard_cyclecallbacks.go +++ /dev/null @@ -1,99 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "strings" - - "github.com/weaviate/weaviate/entities/cyclemanager" -) - -type shardCycleCallbacks struct { - compactionCallbacks cyclemanager.CycleCallbackGroup - compactionCallbacksCtrl cyclemanager.CycleCallbackCtrl - - flushCallbacks cyclemanager.CycleCallbackGroup - flushCallbacksCtrl cyclemanager.CycleCallbackCtrl - - vectorCommitLoggerCallbacks cyclemanager.CycleCallbackGroup - vectorTombstoneCleanupCallbacks cyclemanager.CycleCallbackGroup - vectorCombinedCallbacksCtrl cyclemanager.CycleCallbackCtrl - - geoPropsCommitLoggerCallbacks cyclemanager.CycleCallbackGroup - geoPropsTombstoneCleanupCallbacks cyclemanager.CycleCallbackGroup - geoPropsCombinedCallbacksCtrl cyclemanager.CycleCallbackCtrl -} - -func (s *Shard) initCycleCallbacks() { - id := func(elems ...string) string { - elems = append([]string{"shard", s.index.ID(), s.name}, elems...) - return strings.Join(elems, "/") - } - - compactionId := id("compaction") - compactionCallbacks := cyclemanager.NewCallbackGroup(compactionId, s.index.logger, 1) - compactionCallbacksCtrl := s.index.cycleCallbacks.compactionCallbacks.Register( - compactionId, compactionCallbacks.CycleCallback, - cyclemanager.WithIntervals(cyclemanager.CompactionCycleIntervals())) - - flushId := id("flush") - flushCallbacks := cyclemanager.NewCallbackGroup(flushId, s.index.logger, 1) - flushCallbacksCtrl := s.index.cycleCallbacks.flushCallbacks.Register( - flushId, flushCallbacks.CycleCallback, - cyclemanager.WithIntervals(cyclemanager.MemtableFlushCycleIntervals())) - - vectorCommitLoggerId := id("vector", "commit_logger") - vectorCommitLoggerCallbacks := cyclemanager.NewCallbackGroup(vectorCommitLoggerId, s.index.logger, 1) - vectorCommitLoggerCallbacksCtrl := s.index.cycleCallbacks.vectorCommitLoggerCallbacks.Register( - vectorCommitLoggerId, vectorCommitLoggerCallbacks.CycleCallback, - cyclemanager.WithIntervals(cyclemanager.HnswCommitLoggerCycleIntervals())) - - vectorTombstoneCleanupId := id("vector", "tombstone_cleanup") - vectorTombstoneCleanupCallbacks := cyclemanager.NewCallbackGroup(vectorTombstoneCleanupId, s.index.logger, 1) - // fixed interval on class level, no need to specify separate on shard level - vectorTombstoneCleanupCallbacksCtrl := s.index.cycleCallbacks.vectorTombstoneCleanupCallbacks.Register( - vectorTombstoneCleanupId, vectorTombstoneCleanupCallbacks.CycleCallback) - - vectorCombinedCallbacksCtrl := cyclemanager.NewCombinedCallbackCtrl(2, - vectorCommitLoggerCallbacksCtrl, vectorTombstoneCleanupCallbacksCtrl) - - geoPropsCommitLoggerId := id("geo_props", "commit_logger") - geoPropsCommitLoggerCallbacks := cyclemanager.NewCallbackGroup(geoPropsCommitLoggerId, s.index.logger, 1) - geoPropsCommitLoggerCallbacksCtrl := s.index.cycleCallbacks.geoPropsCommitLoggerCallbacks.Register( - geoPropsCommitLoggerId, geoPropsCommitLoggerCallbacks.CycleCallback, - cyclemanager.WithIntervals(cyclemanager.GeoCommitLoggerCycleIntervals())) - - geoPropsTombstoneCleanupId := id("geoProps", "tombstone_cleanup") - geoPropsTombstoneCleanupCallbacks := cyclemanager.NewCallbackGroup(geoPropsTombstoneCleanupId, s.index.logger, 1) - // fixed interval on class level, no need to specify separate on shard level - geoPropsTombstoneCleanupCallbacksCtrl := s.index.cycleCallbacks.geoPropsTombstoneCleanupCallbacks.Register( - geoPropsTombstoneCleanupId, geoPropsTombstoneCleanupCallbacks.CycleCallback) - - geoPropsCombinedCallbacksCtrl := cyclemanager.NewCombinedCallbackCtrl(2, - geoPropsCommitLoggerCallbacksCtrl, geoPropsTombstoneCleanupCallbacksCtrl) - - s.cycleCallbacks = &shardCycleCallbacks{ - compactionCallbacks: compactionCallbacks, - compactionCallbacksCtrl: compactionCallbacksCtrl, - - flushCallbacks: flushCallbacks, - flushCallbacksCtrl: flushCallbacksCtrl, - - vectorCommitLoggerCallbacks: vectorCommitLoggerCallbacks, - vectorTombstoneCleanupCallbacks: vectorTombstoneCleanupCallbacks, - vectorCombinedCallbacksCtrl: vectorCombinedCallbacksCtrl, - - geoPropsCommitLoggerCallbacks: geoPropsCommitLoggerCallbacks, - geoPropsTombstoneCleanupCallbacks: geoPropsTombstoneCleanupCallbacks, - geoPropsCombinedCallbacksCtrl: geoPropsCombinedCallbacksCtrl, - } -} diff --git a/adapters/repos/db/shard_dimension_tracking.go b/adapters/repos/db/shard_dimension_tracking.go deleted file mode 100644 index be006a325696a9fbf1dd660edccd0acb49694392..0000000000000000000000000000000000000000 --- a/adapters/repos/db/shard_dimension_tracking.go +++ /dev/null @@ -1,153 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "encoding/binary" - "time" - - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/entities/schema" - hnswent "github.com/weaviate/weaviate/entities/vectorindex/hnsw" - "github.com/weaviate/weaviate/usecases/monitoring" -) - -func (s *Shard) Dimensions() int { - b := s.store.Bucket(helpers.DimensionsBucketLSM) - if b == nil { - return 0 - } - - c := b.MapCursor() - defer c.Close() - sum := 0 - for k, v := c.First(); k != nil; k, v = c.Next() { - dimLength := binary.LittleEndian.Uint32(k) - sum += int(dimLength) * len(v) - } - - return sum -} - -func (s *Shard) QuantizedDimensions(segments int) int { - // Exit early if segments is 0 (unset), in this case PQ will use the same number of dimensions - // as the segment size - if segments <= 0 { - return s.Dimensions() - } - - b := s.store.Bucket(helpers.DimensionsBucketLSM) - if b == nil { - return 0 - } - - c := b.MapCursor() - defer c.Close() - sum := 0 - for k, v := c.First(); k != nil; k, v = c.Next() { - dimLength := binary.LittleEndian.Uint32(k) - if dimLength > 0 { - sum += segments * len(v) - } - } - - return sum -} - -func (s *Shard) clearDimensionMetrics() { - clearDimensionMetrics(s.promMetrics, s.index.Config.ClassName.String(), - s.name, s.index.vectorIndexUserConfig) -} - -func (s *Shard) publishDimensionMetrics() { - className := s.index.Config.ClassName.String() - - if pqEnabled, segments := getPQSegments(s.index.vectorIndexUserConfig); pqEnabled { - count := s.QuantizedDimensions(segments) - sendVectorSegmentsMetric(s.promMetrics, className, s.name, count) - sendVectorDimensionsMetric(s.promMetrics, className, s.name, 0) - } else { - count := s.Dimensions() - sendVectorDimensionsMetric(s.promMetrics, className, s.name, count) - } -} - -func (s *Shard) initDimensionTracking() { - if s.index.Config.TrackVectorDimensions { - // always send vector dimensions at startup if tracking is enabled - s.publishDimensionMetrics() - // start tracking vector dimensions goroutine only when tracking is enabled - go func() { - t := time.NewTicker(5 * time.Minute) - defer t.Stop() - for { - select { - case <-s.stopMetrics: - return - case <-t.C: - s.publishDimensionMetrics() - } - } - }() - } -} - -func getPQSegments(cfg schema.VectorIndexConfig) (bool, int) { - // Detect if vector index is HNSW - if hnswUserConfig, ok := cfg.(hnswent.UserConfig); ok && hnswUserConfig.PQ.Enabled { - return true, hnswUserConfig.PQ.Segments - } - return false, 0 -} - -func sendVectorSegmentsMetric(promMetrics *monitoring.PrometheusMetrics, - className, shardName string, count int, -) { - if promMetrics != nil { - metric, err := promMetrics.VectorSegmentsSum. - GetMetricWithLabelValues(className, shardName) - if err == nil { - metric.Set(float64(count)) - } - } -} - -func sendVectorDimensionsMetric(promMetrics *monitoring.PrometheusMetrics, - className, shardName string, count int, -) { - if promMetrics != nil { - // Important: Never group classes/shards for this metric. We need the - // granularity here as this tracks an absolute value per shard that changes - // independently over time. - // - // If we need to reduce metrics further, an alternative could be to not - // make dimension tracking shard-centric, but rather make it node-centric. - // Then have a single metric that aggregates all dimensions first, then - // observes only the sum - metric, err := promMetrics.VectorDimensionsSum. - GetMetricWithLabelValues(className, shardName) - if err == nil { - metric.Set(float64(count)) - } - } -} - -func clearDimensionMetrics(promMetrics *monitoring.PrometheusMetrics, - className, shardName string, cfg schema.VectorIndexConfig, -) { - if pqEnabled, _ := getPQSegments(cfg); pqEnabled { - sendVectorDimensionsMetric(promMetrics, className, shardName, 0) - sendVectorDimensionsMetric(promMetrics, className, shardName, 0) - } else { - sendVectorDimensionsMetric(promMetrics, className, shardName, 0) - } -} diff --git a/adapters/repos/db/shard_dimension_tracking_test.go b/adapters/repos/db/shard_dimension_tracking_test.go deleted file mode 100644 index 68bbe7045d950a77b681531caccbfe827516557b..0000000000000000000000000000000000000000 --- a/adapters/repos/db/shard_dimension_tracking_test.go +++ /dev/null @@ -1,375 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package db - -import ( - "context" - "fmt" - "math/rand" - "testing" - - "github.com/go-openapi/strfmt" - "github.com/google/uuid" - "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - enthnsw "github.com/weaviate/weaviate/entities/vectorindex/hnsw" -) - -func Benchmark_Migration(b *testing.B) { - fmt.Printf("Running benchmark %v times\n", b.N) - for i := 0; i < b.N; i++ { - func() { - r := getRandomSeed() - dirName := b.TempDir() - - shardState := singleShardState() - logger := logrus.New() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: shardState, - } - repo, err := New(logger, Config{ - RootPath: dirName, - QueryMaximumResults: 1000, - MaxImportGoroutinesFactor: 1, - TrackVectorDimensions: true, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(b, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(b, repo.WaitForStartup(testCtx())) - defer repo.Shutdown(context.Background()) - - migrator := NewMigrator(repo, logger) - - class := &models.Class{ - Class: "Test", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - } - schema := schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{class}, - }, - } - - migrator.AddClass(context.Background(), class, schemaGetter.shardState) - - schemaGetter.schema = schema - - repo.config.TrackVectorDimensions = false - - dim := 128 - for i := 0; i < 100; i++ { - vec := make([]float32, dim) - for j := range vec { - vec[j] = r.Float32() - } - - id := strfmt.UUID(uuid.MustParse(fmt.Sprintf("%032d", i)).String()) - obj := &models.Object{Class: "Test", ID: id} - err := repo.PutObject(context.Background(), obj, vec, nil) - if err != nil { - b.Fatal(err) - } - } - - fmt.Printf("Added vectors, now migrating\n") - - repo.config.TrackVectorDimensions = true - migrator.RecalculateVectorDimensions(context.TODO()) - fmt.Printf("Benchmark complete") - }() - } -} - -// Rebuild dimensions at startup -func Test_Migration(t *testing.T) { - r := getRandomSeed() - dirName := t.TempDir() - - shardState := singleShardState() - logger := logrus.New() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: shardState, - } - repo, err := New(logger, Config{ - RootPath: dirName, - QueryMaximumResults: 1000, - MaxImportGoroutinesFactor: 1, - TrackVectorDimensions: true, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - defer repo.Shutdown(context.Background()) - - migrator := NewMigrator(repo, logger) - - t.Run("set schema", func(t *testing.T) { - class := &models.Class{ - Class: "Test", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - } - schema := schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{class}, - }, - } - - require.Nil(t, - migrator.AddClass(context.Background(), class, schemaGetter.shardState)) - - schemaGetter.schema = schema - }) - - repo.config.TrackVectorDimensions = false - - t.Run("import objects with d=128", func(t *testing.T) { - dim := 128 - for i := 0; i < 100; i++ { - vec := make([]float32, dim) - for j := range vec { - vec[j] = r.Float32() - } - - id := strfmt.UUID(uuid.MustParse(fmt.Sprintf("%032d", i)).String()) - obj := &models.Object{Class: "Test", ID: id} - err := repo.PutObject(context.Background(), obj, vec, nil) - require.Nil(t, err) - } - dimAfter := GetDimensionsFromRepo(repo, "Test") - require.Equal(t, 0, dimAfter, "dimensions should not have been calculated") - }) - - dimBefore := GetDimensionsFromRepo(repo, "Test") - require.Equal(t, 0, dimBefore, "dimensions should not have been calculated") - repo.config.TrackVectorDimensions = true - migrator.RecalculateVectorDimensions(context.TODO()) - dimAfter := GetDimensionsFromRepo(repo, "Test") - require.Equal(t, 12800, dimAfter, "dimensions should be counted now") -} - -func Test_DimensionTracking(t *testing.T) { - r := getRandomSeed() - dirName := t.TempDir() - - shardState := singleShardState() - logger := logrus.New() - schemaGetter := &fakeSchemaGetter{ - schema: schema.Schema{Objects: &models.Schema{Classes: nil}}, - shardState: shardState, - } - repo, err := New(logger, Config{ - RootPath: dirName, - QueryMaximumResults: 10000, - MaxImportGoroutinesFactor: 1, - TrackVectorDimensions: true, - }, &fakeRemoteClient{}, &fakeNodeResolver{}, &fakeRemoteNodeClient{}, &fakeReplicationClient{}, nil) - require.Nil(t, err) - repo.SetSchemaGetter(schemaGetter) - require.Nil(t, repo.WaitForStartup(testCtx())) - defer repo.Shutdown(context.Background()) - - migrator := NewMigrator(repo, logger) - - t.Run("set schema", func(t *testing.T) { - class := &models.Class{ - Class: "Test", - VectorIndexConfig: enthnsw.NewDefaultUserConfig(), - InvertedIndexConfig: invertedConfig(), - } - schema := schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{class}, - }, - } - - require.Nil(t, - migrator.AddClass(context.Background(), class, schemaGetter.shardState)) - - schemaGetter.schema = schema - }) - - t.Run("import objects with d=128", func(t *testing.T) { - dim := 128 - for i := 0; i < 100; i++ { - vec := make([]float32, dim) - for j := range vec { - vec[j] = r.Float32() - } - - id := strfmt.UUID(uuid.MustParse(fmt.Sprintf("%032d", i)).String()) - obj := &models.Object{Class: "Test", ID: id} - err := repo.PutObject(context.Background(), obj, vec, nil) - require.Nil(t, err) - } - dimAfter := GetDimensionsFromRepo(repo, "Test") - require.Equal(t, 12800, dimAfter, "dimensions should not have changed") - quantDimAfter := GetQuantizedDimensionsFromRepo(repo, "Test", 64) - require.Equal(t, 6400, quantDimAfter, "quantized dimensions should not have changed") - }) - - t.Run("import objects with d=0", func(t *testing.T) { - dimBefore := GetDimensionsFromRepo(repo, "Test") - quantDimBefore := GetQuantizedDimensionsFromRepo(repo, "Test", 64) - for i := 100; i < 200; i++ { - id := strfmt.UUID(uuid.MustParse(fmt.Sprintf("%032d", i)).String()) - obj := &models.Object{Class: "Test", ID: id} - err := repo.PutObject(context.Background(), obj, nil, nil) - require.Nil(t, err) - } - dimAfter := GetDimensionsFromRepo(repo, "Test") - require.Equal(t, dimBefore, dimAfter, "dimensions should not have changed") - quantDimAfter := GetQuantizedDimensionsFromRepo(repo, "Test", 64) - require.Equal(t, quantDimBefore, quantDimAfter, "quantized dimensions should not have changed") - }) - - t.Run("verify dimensions after initial import", func(t *testing.T) { - idx := repo.GetIndex("Test") - idx.ForEachShard(func(name string, shard ShardLike) error { - assert.Equal(t, 12800, shard.Dimensions()) - assert.Equal(t, 6400, shard.QuantizedDimensions(64)) - return nil - }) - }) - - t.Run("delete 10 objects with d=128", func(t *testing.T) { - dimBefore := GetDimensionsFromRepo(repo, "Test") - quantDimBefore := GetQuantizedDimensionsFromRepo(repo, "Test", 64) - for i := 0; i < 10; i++ { - id := strfmt.UUID(uuid.MustParse(fmt.Sprintf("%032d", i)).String()) - err := repo.DeleteObject(context.Background(), "Test", id, nil, "") - require.Nil(t, err) - } - dimAfter := GetDimensionsFromRepo(repo, "Test") - require.Equal(t, dimBefore, dimAfter+10*128, "dimensions should have decreased") - quantDimAfter := GetQuantizedDimensionsFromRepo(repo, "Test", 64) - require.Equal(t, quantDimBefore, quantDimAfter+10*64, "dimensions should have decreased") - }) - - t.Run("verify dimensions after delete", func(t *testing.T) { - idx := repo.GetIndex("Test") - idx.ForEachShard(func(name string, shard ShardLike) error { - assert.Equal(t, 11520, shard.Dimensions()) - assert.Equal(t, 5760, shard.QuantizedDimensions(64)) - return nil - }) - }) - - t.Run("update some of the d=128 objects with a new vector", func(t *testing.T) { - dimBefore := GetDimensionsFromRepo(repo, "Test") - quantDimBefore := GetQuantizedDimensionsFromRepo(repo, "Test", 64) - dim := 128 - for i := 0; i < 50; i++ { - vec := make([]float32, dim) - for j := range vec { - vec[j] = rand.Float32() - } - - id := strfmt.UUID(uuid.MustParse(fmt.Sprintf("%032d", i)).String()) - obj := &models.Object{Class: "Test", ID: id} - // Put is idempotent, but since the IDs exist now, this is an update - // under the hood and a "reinstert" for the already deleted ones - err := repo.PutObject(context.Background(), obj, vec, nil) - require.Nil(t, err) - } - dimAfter := GetDimensionsFromRepo(repo, "Test") - quantDimAfter := GetQuantizedDimensionsFromRepo(repo, "Test", 64) - require.Equal(t, dimBefore+10*128, dimAfter, "dimensions should have been restored") - require.Equal(t, quantDimBefore+10*64, quantDimAfter, "dimensions should have been restored") - }) - - t.Run("update some of the d=128 objects with a nil vector", func(t *testing.T) { - dimBefore := GetDimensionsFromRepo(repo, "Test") - quantDimBefore := GetQuantizedDimensionsFromRepo(repo, "Test", 32) - for i := 50; i < 100; i++ { - id := strfmt.UUID(uuid.MustParse(fmt.Sprintf("%032d", i)).String()) - obj := &models.Object{Class: "Test", ID: id} - // Put is idempotent, but since the IDs exist now, this is an update - // under the hood and a "reinsert" for the already deleted ones - err := repo.PutObject(context.Background(), obj, nil, nil) - require.Nil(t, err) - } - dimAfter := GetDimensionsFromRepo(repo, "Test") - quantDimAfter := GetQuantizedDimensionsFromRepo(repo, "Test", 32) - require.Equal(t, dimBefore, dimAfter+50*128, "dimensions should decrease") - require.Equal(t, quantDimBefore, quantDimAfter+50*32, "dimensions should decrease") - }) - - t.Run("verify dimensions after first set of updates", func(t *testing.T) { - idx := repo.GetIndex("Test") - idx.ForEachShard(func(name string, shard ShardLike) error { - assert.Equal(t, 6400, shard.Dimensions()) - assert.Equal(t, 3200, shard.QuantizedDimensions(64)) - assert.Equal(t, 1600, shard.QuantizedDimensions(32)) - return nil - }) - }) - - t.Run("update some of the origin nil vector objects with a d=128 vector", func(t *testing.T) { - dimBefore := GetDimensionsFromRepo(repo, "Test") - quantDimBefore := GetQuantizedDimensionsFromRepo(repo, "Test", 64) - dim := 128 - for i := 100; i < 150; i++ { - vec := make([]float32, dim) - for j := range vec { - vec[j] = rand.Float32() - } - - id := strfmt.UUID(uuid.MustParse(fmt.Sprintf("%032d", i)).String()) - obj := &models.Object{Class: "Test", ID: id} - // Put is idempotent, but since the IDs exist now, this is an update - // under the hood and a "reinsert" for the already deleted ones - err := repo.PutObject(context.Background(), obj, vec, nil) - require.Nil(t, err) - } - dimAfter := GetDimensionsFromRepo(repo, "Test") - quantDimAfter := GetQuantizedDimensionsFromRepo(repo, "Test", 64) - require.Equal(t, dimBefore+50*128, dimAfter, "dimensions should increase") - require.Equal(t, quantDimBefore+50*64, quantDimAfter, "dimensions should increase") - }) - - t.Run("update some of the nil objects with another nil vector", func(t *testing.T) { - dimBefore := GetDimensionsFromRepo(repo, "Test") - quantDimBefore := GetQuantizedDimensionsFromRepo(repo, "Test", 64) - for i := 150; i < 200; i++ { - id := strfmt.UUID(uuid.MustParse(fmt.Sprintf("%032d", i)).String()) - obj := &models.Object{Class: "Test", ID: id} - // Put is idempotent, but since the IDs exist now, this is an update - // under the hood and a "reinstert" for the already deleted ones - err := repo.PutObject(context.Background(), obj, nil, nil) - require.Nil(t, err) - } - dimAfter := GetDimensionsFromRepo(repo, "Test") - quantDimAfter := GetQuantizedDimensionsFromRepo(repo, "Test", 64) - require.Equal(t, dimBefore, dimAfter, "dimensions should not have changed") - require.Equal(t, quantDimBefore, quantDimAfter, "dimensions should not have changed") - }) - - t.Run("verify dimensions after more updates", func(t *testing.T) { - idx := repo.GetIndex("Test") - idx.ForEachShard(func(name string, shard ShardLike) error { - assert.Equal(t, 12800, shard.Dimensions()) - assert.Equal(t, 6400, shard.QuantizedDimensions(64)) - assert.Equal(t, 12800, shard.QuantizedDimensions(0)) - return nil - }) - }) -} diff --git a/adapters/repos/db/shard_geo_props.go b/adapters/repos/db/shard_geo_props.go deleted file mode 100644 index fd33605089faf673e418aa6a2ec608fa7cbef7f7..0000000000000000000000000000000000000000 --- a/adapters/repos/db/shard_geo_props.go +++ /dev/null @@ -1,184 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "context" - "fmt" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/adapters/repos/db/propertyspecific" - "github.com/weaviate/weaviate/adapters/repos/db/vector/geo" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/storagestate" - "github.com/weaviate/weaviate/entities/storobj" -) - -func (s *Shard) initGeoProp(prop *models.Property) error { - // starts geo props cycles if actual geo property is present - // (safe to start multiple times) - s.index.cycleCallbacks.geoPropsCommitLoggerCycle.Start() - s.index.cycleCallbacks.geoPropsTombstoneCleanupCycle.Start() - - idx, err := geo.NewIndex(geo.Config{ - ID: geoPropID(prop.Name), - RootPath: s.path(), - CoordinatesForID: s.makeCoordinatesForID(prop.Name), - DisablePersistence: false, - Logger: s.index.logger, - }, - s.cycleCallbacks.geoPropsCommitLoggerCallbacks, - s.cycleCallbacks.geoPropsTombstoneCleanupCallbacks, - s.cycleCallbacks.compactionCallbacks, - s.cycleCallbacks.flushCallbacks, - ) - if err != nil { - return errors.Wrapf(err, "create geo index for prop %q", prop.Name) - } - - s.propertyIndicesLock.Lock() - s.propertyIndices[prop.Name] = propertyspecific.Index{ - Type: schema.DataTypeGeoCoordinates, - GeoIndex: idx, - Name: prop.Name, - } - s.propertyIndicesLock.Unlock() - - idx.PostStartup() - - return nil -} - -func (s *Shard) makeCoordinatesForID(propName string) geo.CoordinatesForID { - return func(ctx context.Context, id uint64) (*models.GeoCoordinates, error) { - obj, err := s.objectByIndexID(ctx, id, true) - if err != nil { - return nil, storobj.NewErrNotFoundf(id, "retrieve object") - } - - if obj.Properties() == nil { - return nil, storobj.NewErrNotFoundf(id, - "object has no properties") - } - - prop, ok := obj.Properties().(map[string]interface{})[propName] - if !ok { - return nil, storobj.NewErrNotFoundf(id, - "object has no property %q", propName) - } - - geoProp, ok := prop.(*models.GeoCoordinates) - if !ok { - return nil, fmt.Errorf("expected property to be of type %T, got: %T", - &models.GeoCoordinates{}, prop) - } - - return geoProp, nil - } -} - -func geoPropID(propName string) string { - return fmt.Sprintf("geo.%s", propName) -} - -func (s *Shard) updatePropertySpecificIndices(object *storobj.Object, - status objectInsertStatus, -) error { - if s.isReadOnly() { - return storagestate.ErrStatusReadOnly - } - s.propertyIndicesLock.RLock() - defer s.propertyIndicesLock.RUnlock() - - for propName, propIndex := range s.propertyIndices { - if err := s.updatePropertySpecificIndex(propName, propIndex, - object, status); err != nil { - return errors.Wrapf(err, "property %q", propName) - } - } - - return nil -} - -func (s *Shard) updatePropertySpecificIndex(propName string, - index propertyspecific.Index, obj *storobj.Object, - status objectInsertStatus, -) error { - if index.Type != schema.DataTypeGeoCoordinates { - return fmt.Errorf("unsupported per-property index type %q", index.Type) - } - - // currently the only property-specific index we support - return s.updateGeoIndex(propName, index, obj, status) -} - -func (s *Shard) updateGeoIndex(propName string, index propertyspecific.Index, - obj *storobj.Object, status objectInsertStatus, -) error { - if s.isReadOnly() { - return storagestate.ErrStatusReadOnly - } - - if status.docIDChanged { - if err := s.deleteFromGeoIndex(index, status.oldDocID); err != nil { - return errors.Wrap(err, "delete old doc id from geo index") - } - } - - return s.addToGeoIndex(propName, index, obj, status) -} - -func (s *Shard) addToGeoIndex(propName string, index propertyspecific.Index, - obj *storobj.Object, status objectInsertStatus, -) error { - if s.isReadOnly() { - return storagestate.ErrStatusReadOnly - } - - if obj.Properties() == nil { - return nil - } - - asMap := obj.Properties().(map[string]interface{}) - propValue, ok := asMap[propName] - if !ok { - return nil - } - - // geo coordinates is the only supported one at the moment - asGeo, ok := propValue.(*models.GeoCoordinates) - if !ok { - return fmt.Errorf("expected prop to be of type %T, but got: %T", - &models.GeoCoordinates{}, propValue) - } - - if err := index.GeoIndex.Add(status.docID, asGeo); err != nil { - return errors.Wrapf(err, "insert into geo index") - } - - return nil -} - -func (s *Shard) deleteFromGeoIndex(index propertyspecific.Index, - docID uint64, -) error { - if s.isReadOnly() { - return storagestate.ErrStatusReadOnly - } - - if err := index.GeoIndex.Delete(docID); err != nil { - return errors.Wrapf(err, "delete from geo index") - } - - return nil -} diff --git a/adapters/repos/db/shard_group_by.go b/adapters/repos/db/shard_group_by.go deleted file mode 100644 index a284cf76cddd6b50d57c58effce70f780b198afb..0000000000000000000000000000000000000000 --- a/adapters/repos/db/shard_group_by.go +++ /dev/null @@ -1,229 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "context" - "encoding/binary" - "encoding/json" - "fmt" - - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/searchparams" - "github.com/weaviate/weaviate/entities/storobj" -) - -func (s *Shard) groupResults(ctx context.Context, ids []uint64, - dists []float32, groupBy *searchparams.GroupBy, - additional additional.Properties, -) ([]*storobj.Object, []float32, error) { - objsBucket := s.store.Bucket(helpers.ObjectsBucketLSM) - className := s.index.Config.ClassName - sch := s.index.getSchema.GetSchemaSkipAuth() - prop, err := sch.GetProperty(className, schema.PropertyName(groupBy.Property)) - if err != nil { - return nil, nil, fmt.Errorf("%w: unrecognized property: %s", - err, groupBy.Property) - } - dt, err := sch.FindPropertyDataType(prop.DataType) - if err != nil { - return nil, nil, fmt.Errorf("%w: unrecognized data type for property: %s", - err, groupBy.Property) - } - - return newGrouper(ids, dists, groupBy, objsBucket, dt, additional).Do(ctx) -} - -type grouper struct { - ids []uint64 - dists []float32 - groupBy *searchparams.GroupBy - additional additional.Properties - propertyDataType schema.PropertyDataType - objBucket *lsmkv.Bucket -} - -func newGrouper(ids []uint64, dists []float32, - groupBy *searchparams.GroupBy, objBucket *lsmkv.Bucket, - propertyDataType schema.PropertyDataType, - additional additional.Properties, -) *grouper { - return &grouper{ - ids: ids, - dists: dists, - groupBy: groupBy, - objBucket: objBucket, - propertyDataType: propertyDataType, - additional: additional, - } -} - -func (g *grouper) Do(ctx context.Context) ([]*storobj.Object, []float32, error) { - docIDBytes := make([]byte, 8) - - groupsOrdered := []string{} - groups := map[string][]uint64{} - docIDObject := map[uint64]*storobj.Object{} - docIDDistance := map[uint64]float32{} - -DOCS_LOOP: - for i, docID := range g.ids { - binary.LittleEndian.PutUint64(docIDBytes, docID) - objData, err := g.objBucket.GetBySecondary(0, docIDBytes) - if err != nil { - return nil, nil, fmt.Errorf("%w: could not get obj by doc id %d", err, docID) - } - if objData == nil { - continue - } - value, ok, _ := storobj.ParseAndExtractProperty(objData, g.groupBy.Property) - if !ok { - continue - } - - values, err := g.getValues(value) - if err != nil { - return nil, nil, err - } - - for _, val := range values { - current, groupExists := groups[val] - if len(current) >= g.groupBy.ObjectsPerGroup { - continue - } - - if !groupExists && len(groups) >= g.groupBy.Groups { - continue DOCS_LOOP - } - - groups[val] = append(current, docID) - - if !groupExists { - // this group doesn't exist add it to the ordered list - groupsOrdered = append(groupsOrdered, val) - } - - if _, ok := docIDObject[docID]; !ok { - // whole object, might be that we only need value and ID to be extracted - unmarshalled, err := storobj.FromBinaryOptional(objData, g.additional) - if err != nil { - return nil, nil, fmt.Errorf("%w: unmarshal data object at position %d", err, i) - } - docIDObject[docID] = unmarshalled - docIDDistance[docID] = g.dists[i] - } - } - } - - objs := make([]*storobj.Object, len(groupsOrdered)) - dists := make([]float32, len(groupsOrdered)) - objIDs := []uint64{} - for i, val := range groupsOrdered { - docIDs := groups[val] - unmarshalled, err := g.getUnmarshalled(docIDs[0], docIDObject, objIDs) - if err != nil { - return nil, nil, err - } - dist := docIDDistance[docIDs[0]] - objIDs = append(objIDs, docIDs[0]) - hits := make([]map[string]interface{}, len(docIDs)) - for j, docID := range docIDs { - props := map[string]interface{}{} - for k, v := range docIDObject[docID].Properties().(map[string]interface{}) { - props[k] = v - } - props["_additional"] = &additional.GroupHitAdditional{ - ID: docIDObject[docID].ID(), - Distance: docIDDistance[docID], - Vector: docIDObject[docID].Vector, - } - hits[j] = props - } - group := &additional.Group{ - ID: i, - GroupedBy: &additional.GroupedBy{ - Value: val, - Path: []string{g.groupBy.Property}, - }, - Count: len(hits), - Hits: hits, - MinDistance: docIDDistance[docIDs[0]], - MaxDistance: docIDDistance[docIDs[len(docIDs)-1]], - } - - // add group - if unmarshalled.AdditionalProperties() == nil { - unmarshalled.Object.Additional = models.AdditionalProperties{} - } - unmarshalled.AdditionalProperties()["group"] = group - - objs[i] = unmarshalled - dists[i] = dist - } - - return objs, dists, nil -} - -func (g *grouper) getUnmarshalled(docID uint64, - docIDObject map[uint64]*storobj.Object, - objIDs []uint64, -) (*storobj.Object, error) { - containsDocID := false - for i := range objIDs { - if objIDs[i] == docID { - containsDocID = true - break - } - } - if containsDocID { - // we have already added this object containing a group to the result array - // and we need to unmarshall it again so that a group won't get overridden - docIDBytes := make([]byte, 8) - binary.LittleEndian.PutUint64(docIDBytes, docID) - objData, err := g.objBucket.GetBySecondary(0, docIDBytes) - if err != nil { - return nil, fmt.Errorf("%w: could not get obj by doc id %d", err, docID) - } - unmarshalled, err := storobj.FromBinaryOptional(objData, g.additional) - if err != nil { - return nil, fmt.Errorf("%w: unmarshal data object doc id %d", err, docID) - } - return unmarshalled, nil - } - return docIDObject[docID], nil -} - -func (g *grouper) getValues(values []string) ([]string, error) { - if len(values) == 0 { - return []string{""}, nil - } - if g.propertyDataType.IsReference() { - beacons := make([]string, len(values)) - for i := range values { - if values[i] != "" { - var ref models.SingleRef - err := json.Unmarshal([]byte(values[i]), &ref) - if err != nil { - return nil, fmt.Errorf("%w: unmarshal grouped by value %s at position %d", - err, values[i], i) - } - beacons[i] = ref.Beacon.String() - } - } - return beacons, nil - } - return values, nil -} diff --git a/adapters/repos/db/shard_init_properties.go b/adapters/repos/db/shard_init_properties.go deleted file mode 100644 index 29384ce3ad08cd26fe883bb5b9f3796c792d8cbc..0000000000000000000000000000000000000000 --- a/adapters/repos/db/shard_init_properties.go +++ /dev/null @@ -1,63 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "context" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/adapters/repos/db/propertyspecific" - "github.com/weaviate/weaviate/entities/models" - "golang.org/x/sync/errgroup" -) - -func (s *Shard) initProperties(class *models.Class) error { - s.propertyIndices = propertyspecific.Indices{} - if class == nil { - return nil - } - - eg := &errgroup.Group{} - for _, prop := range class.Properties { - s.createPropertyIndex(context.TODO(), prop, eg) - } - - eg.Go(func() error { - if err := s.addIDProperty(context.TODO()); err != nil { - return errors.Wrap(err, "create id property index") - } - return nil - }) - - if s.index.invertedIndexConfig.IndexTimestamps { - eg.Go(func() error { - if err := s.addTimestampProperties(context.TODO()); err != nil { - return errors.Wrap(err, "create timestamp properties indexes") - } - return nil - }) - } - - if s.index.Config.TrackVectorDimensions { - eg.Go(func() error { - if err := s.addDimensionsProperty(context.TODO()); err != nil { - return errors.Wrap(err, "crreate dimensions property index") - } - return nil - }) - } - - if err := eg.Wait(); err != nil { - return errors.Wrapf(err, "init properties on shard '%s'", s.ID()) - } - return nil -} diff --git a/adapters/repos/db/shard_lazyloader.go b/adapters/repos/db/shard_lazyloader.go deleted file mode 100644 index 2f6ae780d2c7feb0629812b6b7d09c94cdcff11c..0000000000000000000000000000000000000000 --- a/adapters/repos/db/shard_lazyloader.go +++ /dev/null @@ -1,580 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "context" - "errors" - "fmt" - "io" - "os" - "path" - "sync" - - "github.com/go-openapi/strfmt" - "github.com/weaviate/weaviate/adapters/repos/db/indexcheckpoint" - "github.com/weaviate/weaviate/adapters/repos/db/indexcounter" - "github.com/weaviate/weaviate/adapters/repos/db/inverted" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/aggregation" - "github.com/weaviate/weaviate/entities/backup" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/multi" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/search" - "github.com/weaviate/weaviate/entities/searchparams" - "github.com/weaviate/weaviate/entities/storagestate" - "github.com/weaviate/weaviate/entities/storobj" - "github.com/weaviate/weaviate/usecases/monitoring" - "github.com/weaviate/weaviate/usecases/objects" - "github.com/weaviate/weaviate/usecases/replica" - "golang.org/x/sync/errgroup" -) - -type LazyLoadShard struct { - shardOpts *deferredShardOpts - propLenTracker *inverted.JsonShardMetaData - shard *Shard - loaded bool - mutex sync.Mutex -} - -func NewLazyLoadShard(ctx context.Context, promMetrics *monitoring.PrometheusMetrics, - shardName string, index *Index, class *models.Class, jobQueueCh chan job, - indexCheckpoints *indexcheckpoint.Checkpoints, -) *LazyLoadShard { - promMetrics.NewUnloadedshard(class.Class) - return &LazyLoadShard{ - shardOpts: &deferredShardOpts{ - promMetrics: promMetrics, - - name: shardName, - index: index, - class: class, - jobQueueCh: jobQueueCh, - indexCheckpoints: indexCheckpoints, - }, - } -} - -type deferredShardOpts struct { - promMetrics *monitoring.PrometheusMetrics - name string - index *Index - class *models.Class - jobQueueCh chan job - indexCheckpoints *indexcheckpoint.Checkpoints -} - -func (l *LazyLoadShard) mustLoad() { - l.mustLoadCtx(context.Background()) -} - -func (l *LazyLoadShard) mustLoadCtx(ctx context.Context) { - if err := l.Load(ctx); err != nil { - panic(err.Error()) - } -} - -func (l *LazyLoadShard) Load(ctx context.Context) error { - l.mutex.Lock() - defer l.mutex.Unlock() - - l.propLenTracker = nil - - if l.loaded { - return nil - } - if l.shardOpts.class == nil { - l.shardOpts.promMetrics.StartLoadingShard("unknown class") - } else { - l.shardOpts.promMetrics.StartLoadingShard(l.shardOpts.class.Class) - } - shard, err := NewShard(ctx, l.shardOpts.promMetrics, l.shardOpts.name, l.shardOpts.index, - l.shardOpts.class, l.shardOpts.jobQueueCh, l.shardOpts.indexCheckpoints, l.propLenTracker) - if err != nil { - msg := fmt.Sprintf("Unable to load shard %s: %v", l.shardOpts.name, err) - l.shardOpts.index.logger.WithField("error", "shard_load").WithError(err).Error(msg) - return errors.New(msg) - } - l.shard = shard - l.loaded = true - if l.shardOpts.class == nil { - l.shardOpts.promMetrics.FinishLoadingShard("unknown class") - } else { - l.shardOpts.promMetrics.FinishLoadingShard(l.shardOpts.class.Class) - } - return nil -} - -func (l *LazyLoadShard) Index() *Index { - return l.shardOpts.index -} - -func (l *LazyLoadShard) Name() string { - return l.shardOpts.name -} - -func (l *LazyLoadShard) Store() *lsmkv.Store { - l.mustLoad() - return l.shard.Store() -} - -func (l *LazyLoadShard) NotifyReady() { - l.mustLoad() - l.shard.NotifyReady() -} - -func (l *LazyLoadShard) GetStatus() storagestate.Status { - l.mustLoad() - return l.shard.GetStatus() -} - -func (l *LazyLoadShard) UpdateStatus(status string) error { - l.mustLoad() - return l.shard.UpdateStatus(status) -} - -func (l *LazyLoadShard) ChangeObjectCountBy(delta int) error { - if err := l.Load(context.Background()); err != nil { - return err - } - return l.shard.ChangeObjectCountBy(delta) -} - -func (l *LazyLoadShard) FindUUIDs(ctx context.Context, filters *filters.LocalFilter) ([]strfmt.UUID, error) { - if err := l.Load(ctx); err != nil { - return []strfmt.UUID{}, err - } - return l.shard.FindUUIDs(ctx, filters) -} - -func (l *LazyLoadShard) Counter() *indexcounter.Counter { - l.mustLoad() - return l.shard.Counter() -} - -func (l *LazyLoadShard) ObjectCount() int { - return l.GetPropertyLengthTracker().ObjectTally() -} - -func (l *LazyLoadShard) GetPropertyLengthTracker() *inverted.JsonShardMetaData { - l.mutex.Lock() - defer l.mutex.Unlock() - - if l.loaded { - return l.shard.GetPropertyLengthTracker() - } - - if l.propLenTracker != nil { - return l.propLenTracker - } - - var tracker *inverted.JsonShardMetaData - - // FIXME add method for tracker path - plPath := path.Join(l.shardOpts.index.path(), "proplengths") - tracker, err := inverted.NewJsonShardMetaData(plPath, l.shardOpts.index.logger) - l.propLenTracker = tracker - if err != nil { - panic(fmt.Sprintf("could not create property length tracker at %v: %v", plPath, err)) - } - - return l.propLenTracker -} - -func (l *LazyLoadShard) PutObject(ctx context.Context, object *storobj.Object) error { - if err := l.Load(ctx); err != nil { - return err - } - return l.shard.PutObject(ctx, object) -} - -func (l *LazyLoadShard) PutObjectBatch(ctx context.Context, objects []*storobj.Object) []error { - if err := l.Load(ctx); err != nil { - return []error{err} - } // TODO check - return l.shard.PutObjectBatch(ctx, objects) -} - -func (l *LazyLoadShard) ObjectByID(ctx context.Context, id strfmt.UUID, props search.SelectProperties, additional additional.Properties) (*storobj.Object, error) { - if err := l.Load(ctx); err != nil { - return nil, err - } - return l.shard.ObjectByID(ctx, id, props, additional) -} - -func (l *LazyLoadShard) Exists(ctx context.Context, id strfmt.UUID) (bool, error) { - if err := l.Load(ctx); err != nil { - return false, err - } - return l.shard.Exists(ctx, id) -} - -func (l *LazyLoadShard) ObjectSearch(ctx context.Context, limit int, filters *filters.LocalFilter, keywordRanking *searchparams.KeywordRanking, sort []filters.Sort, cursor *filters.Cursor, additional additional.Properties) ([]*storobj.Object, []float32, error) { - if err := l.Load(ctx); err != nil { - return nil, nil, err - } - return l.shard.ObjectSearch(ctx, limit, filters, keywordRanking, sort, cursor, additional) -} - -func (l *LazyLoadShard) ObjectVectorSearch(ctx context.Context, searchVector []float32, targetDist float32, limit int, filters *filters.LocalFilter, sort []filters.Sort, groupBy *searchparams.GroupBy, additional additional.Properties) ([]*storobj.Object, []float32, error) { - if err := l.Load(ctx); err != nil { - return nil, nil, err - } - return l.shard.ObjectVectorSearch(ctx, searchVector, targetDist, limit, filters, sort, groupBy, additional) -} - -func (l *LazyLoadShard) UpdateVectorIndexConfig(ctx context.Context, updated schema.VectorIndexConfig) error { - if err := l.Load(ctx); err != nil { - return err - } - return l.shard.UpdateVectorIndexConfig(ctx, updated) -} - -func (l *LazyLoadShard) AddReferencesBatch(ctx context.Context, refs objects.BatchReferences) []error { - if err := l.Load(ctx); err != nil { - return []error{err} - } // TODO check - return l.shard.AddReferencesBatch(ctx, refs) -} - -func (l *LazyLoadShard) DeleteObjectBatch(ctx context.Context, ids []strfmt.UUID, dryRun bool) objects.BatchSimpleObjects { - l.mustLoadCtx(ctx) - return l.shard.DeleteObjectBatch(ctx, ids, dryRun) -} - -func (l *LazyLoadShard) DeleteObject(ctx context.Context, id strfmt.UUID) error { - if err := l.Load(ctx); err != nil { - return err - } - return l.shard.DeleteObject(ctx, id) -} - -func (l *LazyLoadShard) MultiObjectByID(ctx context.Context, query []multi.Identifier) ([]*storobj.Object, error) { - if err := l.Load(ctx); err != nil { - return nil, err - } - return l.shard.MultiObjectByID(ctx, query) -} - -func (l *LazyLoadShard) ID() string { - return shardId(l.shardOpts.index.ID(), l.shardOpts.name) -} - -func (l *LazyLoadShard) drop() error { - // if not loaded, execute simplified drop without loading shard: - // - perform required actions - // - remove entire shard directory - // use lock to prevent eventual concurrent droping and loading - l.mutex.Lock() - if !l.loaded { - defer l.mutex.Unlock() - - idx := l.shardOpts.index - className := idx.Config.ClassName.String() - shardName := l.shardOpts.name - - // cleanup metrics - NewMetrics(idx.logger, l.shardOpts.promMetrics, className, shardName). - DeleteShardLabels(className, shardName) - - // cleanup dimensions - if idx.Config.TrackVectorDimensions { - clearDimensionMetrics(l.shardOpts.promMetrics, className, shardName, idx.vectorIndexUserConfig) - } - - // cleanup queue - if l.shardOpts.indexCheckpoints != nil { - if err := l.shardOpts.indexCheckpoints.Delete(shardId(idx.ID(), shardName)); err != nil { - return fmt.Errorf("delete checkpoint: %w", err) - } - } - - // remove shard dir - if err := os.RemoveAll(shardPath(idx.path(), shardName)); err != nil { - return fmt.Errorf("delete shard dir: %w", err) - } - - return nil - } - l.mutex.Unlock() - - return l.shard.drop() -} - -func (l *LazyLoadShard) addIDProperty(ctx context.Context) error { - if err := l.Load(ctx); err != nil { - return err - } - return l.shard.addIDProperty(ctx) -} - -func (l *LazyLoadShard) addDimensionsProperty(ctx context.Context) error { - if err := l.Load(ctx); err != nil { - return err - } - return l.shard.addDimensionsProperty(ctx) -} - -func (l *LazyLoadShard) addTimestampProperties(ctx context.Context) error { - if err := l.Load(ctx); err != nil { - return err - } - return l.shard.addTimestampProperties(ctx) -} - -func (l *LazyLoadShard) createPropertyIndex(ctx context.Context, prop *models.Property, eg *errgroup.Group) { - l.mustLoad() - l.shard.createPropertyIndex(ctx, prop, eg) -} - -func (l *LazyLoadShard) BeginBackup(ctx context.Context) error { - if err := l.Load(ctx); err != nil { - return err - } - return l.shard.BeginBackup(ctx) -} - -func (l *LazyLoadShard) ListBackupFiles(ctx context.Context, ret *backup.ShardDescriptor) error { - if err := l.Load(ctx); err != nil { - return err - } - return l.shard.ListBackupFiles(ctx, ret) -} - -func (l *LazyLoadShard) resumeMaintenanceCycles(ctx context.Context) error { - if err := l.Load(ctx); err != nil { - return err - } - return l.shard.resumeMaintenanceCycles(ctx) -} - -func (l *LazyLoadShard) SetPropertyLengths(props []inverted.Property) error { - l.mustLoad() - return l.shard.SetPropertyLengths(props) -} - -func (l *LazyLoadShard) AnalyzeObject(object *storobj.Object) ([]inverted.Property, []inverted.NilProperty, error) { - l.mustLoad() - return l.shard.AnalyzeObject(object) -} - -func (l *LazyLoadShard) Dimensions() int { - l.mustLoad() - return l.shard.Dimensions() -} - -func (l *LazyLoadShard) QuantizedDimensions(segments int) int { - l.mustLoad() - return l.shard.QuantizedDimensions(segments) -} - -func (l *LazyLoadShard) Aggregate(ctx context.Context, params aggregation.Params) (*aggregation.Result, error) { - if err := l.Load(ctx); err != nil { - return nil, err - } - return l.shard.Aggregate(ctx, params) -} - -func (l *LazyLoadShard) MergeObject(ctx context.Context, object objects.MergeDocument) error { - if err := l.Load(ctx); err != nil { - return err - } - return l.shard.MergeObject(ctx, object) -} - -func (l *LazyLoadShard) Queue() *IndexQueue { - l.mustLoad() - return l.shard.Queue() -} - -func (l *LazyLoadShard) Shutdown(ctx context.Context) error { - if !l.isLoaded() { - return nil - } - return l.shard.Shutdown(ctx) -} - -func (l *LazyLoadShard) ObjectList(ctx context.Context, limit int, sort []filters.Sort, cursor *filters.Cursor, additional additional.Properties, className schema.ClassName) ([]*storobj.Object, error) { - if err := l.Load(ctx); err != nil { - return nil, err - } - return l.shard.ObjectList(ctx, limit, sort, cursor, additional, className) -} - -func (l *LazyLoadShard) WasDeleted(ctx context.Context, id strfmt.UUID) (bool, error) { - if err := l.Load(ctx); err != nil { - return false, err - } - return l.shard.WasDeleted(ctx, id) -} - -func (l *LazyLoadShard) VectorIndex() VectorIndex { - l.mustLoad() - return l.shard.VectorIndex() -} - -func (l *LazyLoadShard) Versioner() *shardVersioner { - l.mustLoad() - return l.shard.Versioner() -} - -func (l *LazyLoadShard) isReadOnly() bool { - l.mustLoad() - return l.shard.isReadOnly() -} - -func (l *LazyLoadShard) preparePutObject(ctx context.Context, shardID string, object *storobj.Object) replica.SimpleResponse { - l.mustLoadCtx(ctx) - return l.shard.preparePutObject(ctx, shardID, object) -} - -func (l *LazyLoadShard) preparePutObjects(ctx context.Context, shardID string, objects []*storobj.Object) replica.SimpleResponse { - l.mustLoadCtx(ctx) - return l.shard.preparePutObjects(ctx, shardID, objects) -} - -func (l *LazyLoadShard) prepareMergeObject(ctx context.Context, shardID string, object *objects.MergeDocument) replica.SimpleResponse { - l.mustLoadCtx(ctx) - return l.shard.prepareMergeObject(ctx, shardID, object) -} - -func (l *LazyLoadShard) prepareDeleteObject(ctx context.Context, shardID string, id strfmt.UUID) replica.SimpleResponse { - l.mustLoadCtx(ctx) - return l.shard.prepareDeleteObject(ctx, shardID, id) -} - -func (l *LazyLoadShard) prepareDeleteObjects(ctx context.Context, shardID string, ids []strfmt.UUID, dryRun bool) replica.SimpleResponse { - l.mustLoadCtx(ctx) - return l.shard.prepareDeleteObjects(ctx, shardID, ids, dryRun) -} - -func (l *LazyLoadShard) prepareAddReferences(ctx context.Context, shardID string, refs []objects.BatchReference) replica.SimpleResponse { - l.mustLoadCtx(ctx) - return l.shard.prepareAddReferences(ctx, shardID, refs) -} - -func (l *LazyLoadShard) commitReplication(ctx context.Context, shardID string, mutex *backupMutex) interface{} { - l.mustLoad() - return l.shard.commitReplication(ctx, shardID, mutex) -} - -func (l *LazyLoadShard) abortReplication(ctx context.Context, shardID string) replica.SimpleResponse { - l.mustLoad() - return l.shard.abortReplication(ctx, shardID) -} - -func (l *LazyLoadShard) reinit(ctx context.Context) error { - if err := l.Load(ctx); err != nil { - return err - } - return l.shard.reinit(ctx) -} - -func (l *LazyLoadShard) filePutter(ctx context.Context, shardID string) (io.WriteCloser, error) { - if err := l.Load(ctx); err != nil { - return nil, err - } - return l.shard.filePutter(ctx, shardID) -} - -func (l *LazyLoadShard) extendDimensionTrackerLSM(dimensions int, docID uint64) error { - l.mustLoad() - return l.shard.extendDimensionTrackerLSM(dimensions, docID) -} - -func (l *LazyLoadShard) addToPropertySetBucket(bucket *lsmkv.Bucket, docID uint64, key []byte) error { - l.mustLoad() - return l.shard.addToPropertySetBucket(bucket, docID, key) -} - -func (l *LazyLoadShard) addToPropertyMapBucket(bucket *lsmkv.Bucket, pair lsmkv.MapPair, key []byte) error { - l.mustLoad() - return l.shard.addToPropertyMapBucket(bucket, pair, key) -} - -func (l *LazyLoadShard) pairPropertyWithFrequency(docID uint64, freq, propLen float32) lsmkv.MapPair { - l.mustLoad() - return l.shard.pairPropertyWithFrequency(docID, freq, propLen) -} - -func (l *LazyLoadShard) setFallbackToSearchable(fallback bool) { - l.mustLoad() - l.shard.setFallbackToSearchable(fallback) -} - -func (l *LazyLoadShard) addJobToQueue(job job) { - l.mustLoad() - l.shard.addJobToQueue(job) -} - -func (l *LazyLoadShard) uuidFromDocID(docID uint64) (strfmt.UUID, error) { - l.mustLoad() - return l.shard.uuidFromDocID(docID) -} - -func (l *LazyLoadShard) batchDeleteObject(ctx context.Context, id strfmt.UUID) error { - if err := l.Load(ctx); err != nil { - return err - } - return l.shard.batchDeleteObject(ctx, id) -} - -func (l *LazyLoadShard) putObjectLSM(object *storobj.Object, idBytes []byte) (objectInsertStatus, error) { - l.mustLoad() - return l.shard.putObjectLSM(object, idBytes) -} - -func (l *LazyLoadShard) mutableMergeObjectLSM(merge objects.MergeDocument, idBytes []byte) (mutableMergeResult, error) { - l.mustLoad() - return l.shard.mutableMergeObjectLSM(merge, idBytes) -} - -func (l *LazyLoadShard) deleteFromPropertySetBucket(bucket *lsmkv.Bucket, docID uint64, key []byte) error { - l.mustLoad() - return l.shard.deleteFromPropertySetBucket(bucket, docID, key) -} - -func (l *LazyLoadShard) batchExtendInvertedIndexItemsLSMNoFrequency(b *lsmkv.Bucket, item inverted.MergeItem) error { - l.mustLoad() - return l.shard.batchExtendInvertedIndexItemsLSMNoFrequency(b, item) -} - -func (l *LazyLoadShard) updatePropertySpecificIndices(object *storobj.Object, status objectInsertStatus) error { - l.mustLoad() - return l.shard.updatePropertySpecificIndices(object, status) -} - -func (l *LazyLoadShard) updateVectorIndexIgnoreDelete(vector []float32, status objectInsertStatus) error { - l.mustLoad() - return l.shard.updateVectorIndexIgnoreDelete(vector, status) -} - -func (l *LazyLoadShard) hasGeoIndex() bool { - l.mustLoad() - return l.shard.hasGeoIndex() -} - -func (l *LazyLoadShard) Metrics() *Metrics { - l.mustLoad() - return l.shard.Metrics() -} - -func (l *LazyLoadShard) isLoaded() bool { - l.mutex.Lock() - defer l.mutex.Unlock() - - return l.loaded -} diff --git a/adapters/repos/db/shard_read.go b/adapters/repos/db/shard_read.go deleted file mode 100644 index 2b5bb09dc60decceb96d457451b5d19c0155472b..0000000000000000000000000000000000000000 --- a/adapters/repos/db/shard_read.go +++ /dev/null @@ -1,446 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "bytes" - "context" - "encoding/binary" - "time" - - "github.com/go-openapi/strfmt" - "github.com/google/uuid" - "github.com/pkg/errors" - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/adapters/repos/db/inverted" - "github.com/weaviate/weaviate/adapters/repos/db/sorter" - "github.com/weaviate/weaviate/adapters/repos/db/vector/common" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/multi" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/search" - "github.com/weaviate/weaviate/entities/searchparams" - "github.com/weaviate/weaviate/entities/storobj" -) - -func (s *Shard) ObjectByID(ctx context.Context, id strfmt.UUID, props search.SelectProperties, additional additional.Properties) (*storobj.Object, error) { - idBytes, err := uuid.MustParse(id.String()).MarshalBinary() - if err != nil { - return nil, err - } - - bytes, err := s.store.Bucket(helpers.ObjectsBucketLSM).Get(idBytes) - if err != nil { - return nil, err - } - - if bytes == nil { - return nil, nil - } - - obj, err := storobj.FromBinary(bytes) - if err != nil { - return nil, errors.Wrap(err, "unmarshal object") - } - - return obj, nil -} - -func (s *Shard) MultiObjectByID(ctx context.Context, query []multi.Identifier) ([]*storobj.Object, error) { - objects := make([]*storobj.Object, len(query)) - - ids := make([][]byte, len(query)) - for i, q := range query { - idBytes, err := uuid.MustParse(q.ID).MarshalBinary() - if err != nil { - return nil, err - } - - ids[i] = idBytes - } - - bucket := s.store.Bucket(helpers.ObjectsBucketLSM) - for i, id := range ids { - bytes, err := bucket.Get(id) - if err != nil { - return nil, err - } - - if bytes == nil { - continue - } - - obj, err := storobj.FromBinary(bytes) - if err != nil { - return nil, errors.Wrap(err, "unmarshal kind object") - } - objects[i] = obj - } - - return objects, nil -} - -// TODO: This does an actual read which is not really needed, if we see this -// come up in profiling, we could optimize this by adding an explicit Exists() -// on the LSMKV which only checks the bloom filters, which at least in the case -// of a true negative would be considerably faster. For a (false) positive, -// we'd still need to check, though. -func (s *Shard) Exists(ctx context.Context, id strfmt.UUID) (bool, error) { - idBytes, err := uuid.MustParse(id.String()).MarshalBinary() - if err != nil { - return false, err - } - - bytes, err := s.store.Bucket(helpers.ObjectsBucketLSM).Get(idBytes) - if err != nil { - return false, errors.Wrap(err, "read request") - } - - if bytes == nil { - return false, nil - } - - return true, nil -} - -func (s *Shard) objectByIndexID(ctx context.Context, indexID uint64, acceptDeleted bool) (*storobj.Object, error) { - keyBuf := make([]byte, 8) - binary.LittleEndian.PutUint64(keyBuf, indexID) - - bytes, err := s.store.Bucket(helpers.ObjectsBucketLSM). - GetBySecondary(0, keyBuf) - if err != nil { - return nil, err - } - - if bytes == nil { - return nil, storobj.NewErrNotFoundf(indexID, - "uuid found for docID, but object is nil") - } - - obj, err := storobj.FromBinary(bytes) - if err != nil { - return nil, errors.Wrap(err, "unmarshal kind object") - } - - return obj, nil -} - -func (s *Shard) vectorByIndexID(ctx context.Context, indexID uint64) ([]float32, error) { - keyBuf := make([]byte, 8) - return s.readVectorByIndexIDIntoSlice(ctx, indexID, &common.VectorSlice{Buff8: keyBuf}) -} - -func (s *Shard) readVectorByIndexIDIntoSlice(ctx context.Context, indexID uint64, container *common.VectorSlice) ([]float32, error) { - binary.LittleEndian.PutUint64(container.Buff8, indexID) - - bytes, newBuff, err := s.store.Bucket(helpers.ObjectsBucketLSM). - GetBySecondaryIntoMemory(0, container.Buff8, container.Buff) - if err != nil { - return nil, err - } - - if bytes == nil { - return nil, storobj.NewErrNotFoundf(indexID, - "no object for doc id, it could have been deleted") - } - - container.Buff = newBuff - return storobj.VectorFromBinary(bytes, container.Slice) -} - -func (s *Shard) ObjectSearch(ctx context.Context, limit int, filters *filters.LocalFilter, keywordRanking *searchparams.KeywordRanking, sort []filters.Sort, cursor *filters.Cursor, additional additional.Properties) ([]*storobj.Object, []float32, error) { - if keywordRanking != nil { - if v := s.versioner.Version(); v < 2 { - return nil, nil, errors.Errorf( - "shard was built with an older version of " + - "Weaviate which does not yet support BM25 search") - } - - var bm25objs []*storobj.Object - var bm25count []float32 - var err error - var objs helpers.AllowList - var filterDocIds helpers.AllowList - - if filters != nil { - objs, err = inverted.NewSearcher(s.index.logger, s.store, - s.index.getSchema.GetSchemaSkipAuth(), s.propertyIndices, - s.index.classSearcher, s.index.stopwords, s.versioner.Version(), - s.isFallbackToSearchable, s.tenant(), s.index.Config.QueryNestedRefLimit). - DocIDs(ctx, filters, additional, s.index.Config.ClassName) - if err != nil { - return nil, nil, err - } - - filterDocIds = objs - } - - className := s.index.Config.ClassName - bm25Config := s.index.getInvertedIndexConfig().BM25 - bm25searcher := inverted.NewBM25Searcher(bm25Config, s.store, - s.index.getSchema.GetSchemaSkipAuth(), s.propertyIndices, s.index.classSearcher, - s.GetPropertyLengthTracker(), s.index.logger, s.versioner.Version()) - bm25objs, bm25count, err = bm25searcher.BM25F(ctx, filterDocIds, className, limit, *keywordRanking) - if err != nil { - return nil, nil, err - } - - return bm25objs, bm25count, nil - } - - if filters == nil { - objs, err := s.ObjectList(ctx, limit, sort, - cursor, additional, s.index.Config.ClassName) - return objs, nil, err - } - objs, err := inverted.NewSearcher(s.index.logger, s.store, s.index.getSchema.GetSchemaSkipAuth(), - s.propertyIndices, s.index.classSearcher, s.index.stopwords, s.versioner.Version(), - s.isFallbackToSearchable, s.tenant(), s.index.Config.QueryNestedRefLimit). - Objects(ctx, limit, filters, sort, additional, s.index.Config.ClassName) - return objs, nil, err -} - -func (s *Shard) ObjectVectorSearch(ctx context.Context, searchVector []float32, targetDist float32, limit int, filters *filters.LocalFilter, sort []filters.Sort, groupBy *searchparams.GroupBy, additional additional.Properties) ([]*storobj.Object, []float32, error) { - var ( - ids []uint64 - dists []float32 - err error - allowList helpers.AllowList - ) - - if filters != nil { - beforeFilter := time.Now() - list, err := s.buildAllowList(ctx, filters, additional) - if err != nil { - return nil, nil, err - } - allowList = list - s.metrics.FilteredVectorFilter(time.Since(beforeFilter)) - } - - beforeVector := time.Now() - if limit < 0 { - ids, dists, err = s.queue.SearchByVectorDistance( - searchVector, targetDist, s.index.Config.QueryMaximumResults, allowList) - if err != nil { - return nil, nil, errors.Wrap(err, "vector search by distance") - } - } else { - ids, dists, err = s.queue.SearchByVector(searchVector, limit, allowList) - if err != nil { - return nil, nil, errors.Wrap(err, "vector search") - } - } - if len(ids) == 0 { - return nil, nil, nil - } - - if filters != nil { - s.metrics.FilteredVectorVector(time.Since(beforeVector)) - } - - if groupBy != nil { - return s.groupResults(ctx, ids, dists, groupBy, additional) - } - - if len(sort) > 0 { - beforeSort := time.Now() - ids, dists, err = s.sortDocIDsAndDists(ctx, limit, sort, - s.index.Config.ClassName, ids, dists) - if err != nil { - return nil, nil, errors.Wrap(err, "vector search sort") - } - if filters != nil { - s.metrics.FilteredVectorSort(time.Since(beforeSort)) - } - } - - beforeObjects := time.Now() - - bucket := s.store.Bucket(helpers.ObjectsBucketLSM) - objs, err := storobj.ObjectsByDocID(bucket, ids, additional) - if err != nil { - return nil, nil, err - } - - if filters != nil { - s.metrics.FilteredVectorObjects(time.Since(beforeObjects)) - } - - return objs, dists, nil -} - -func (s *Shard) ObjectList(ctx context.Context, limit int, sort []filters.Sort, cursor *filters.Cursor, additional additional.Properties, className schema.ClassName) ([]*storobj.Object, error) { - if len(sort) > 0 { - docIDs, err := s.sortedObjectList(ctx, limit, sort, className) - if err != nil { - return nil, err - } - bucket := s.store.Bucket(helpers.ObjectsBucketLSM) - return storobj.ObjectsByDocID(bucket, docIDs, additional) - } - - if cursor == nil { - cursor = &filters.Cursor{After: "", Limit: limit} - } - return s.cursorObjectList(ctx, cursor, additional, className) -} - -func (s *Shard) cursorObjectList(ctx context.Context, c *filters.Cursor, - additional additional.Properties, - className schema.ClassName, -) ([]*storobj.Object, error) { - cursor := s.store.Bucket(helpers.ObjectsBucketLSM).Cursor() - defer cursor.Close() - - var key, val []byte - if c.After == "" { - key, val = cursor.First() - } else { - uuidBytes, err := uuid.MustParse(c.After).MarshalBinary() - if err != nil { - return nil, errors.Wrap(err, "after argument is not a valid uuid") - } - key, val = cursor.Seek(uuidBytes) - if bytes.Equal(key, uuidBytes) { - // move cursor by one if it's the same ID - key, val = cursor.Next() - } - } - - i := 0 - out := make([]*storobj.Object, c.Limit) - - for ; key != nil && i < c.Limit; key, val = cursor.Next() { - obj, err := storobj.FromBinary(val) - if err != nil { - return nil, errors.Wrapf(err, "unmarhsal item %d", i) - } - - out[i] = obj - i++ - } - - return out[:i], nil -} - -func (s *Shard) sortedObjectList(ctx context.Context, limit int, sort []filters.Sort, className schema.ClassName) ([]uint64, error) { - lsmSorter, err := sorter.NewLSMSorter(s.store, s.index.getSchema.GetSchemaSkipAuth(), className) - if err != nil { - return nil, errors.Wrap(err, "sort object list") - } - docIDs, err := lsmSorter.Sort(ctx, limit, sort) - if err != nil { - return nil, errors.Wrap(err, "sort object list") - } - return docIDs, nil -} - -func (s *Shard) sortDocIDsAndDists(ctx context.Context, limit int, sort []filters.Sort, className schema.ClassName, docIDs []uint64, dists []float32) ([]uint64, []float32, error) { - lsmSorter, err := sorter.NewLSMSorter(s.store, s.index.getSchema.GetSchemaSkipAuth(), className) - if err != nil { - return nil, nil, errors.Wrap(err, "sort objects with distances") - } - sortedDocIDs, sortedDists, err := lsmSorter.SortDocIDsAndDists(ctx, limit, sort, docIDs, dists) - if err != nil { - return nil, nil, errors.Wrap(err, "sort objects with distances") - } - return sortedDocIDs, sortedDists, nil -} - -func (s *Shard) buildAllowList(ctx context.Context, filters *filters.LocalFilter, addl additional.Properties) (helpers.AllowList, error) { - list, err := inverted.NewSearcher(s.index.logger, s.store, s.index.getSchema.GetSchemaSkipAuth(), - s.propertyIndices, s.index.classSearcher, s.index.stopwords, s.versioner.Version(), - s.isFallbackToSearchable, s.tenant(), s.index.Config.QueryNestedRefLimit). - DocIDs(ctx, filters, addl, s.index.Config.ClassName) - if err != nil { - return nil, errors.Wrap(err, "build inverted filter allow list") - } - - return list, nil -} - -func (s *Shard) uuidFromDocID(docID uint64) (strfmt.UUID, error) { - bucket := s.store.Bucket(helpers.ObjectsBucketLSM) - if bucket == nil { - return "", errors.Errorf("objects bucket not found") - } - - keyBuf := bytes.NewBuffer(nil) - binary.Write(keyBuf, binary.LittleEndian, &docID) - docIDBytes := keyBuf.Bytes() - res, err := bucket.GetBySecondary(0, docIDBytes) - if err != nil { - return "", err - } - - prop, _, err := storobj.ParseAndExtractProperty(res, "id") - if err != nil { - return "", err - } - - return strfmt.UUID(prop[0]), nil -} - -func (s *Shard) batchDeleteObject(ctx context.Context, id strfmt.UUID) error { - idBytes, err := uuid.MustParse(id.String()).MarshalBinary() - if err != nil { - return err - } - - var docID uint64 - bucket := s.store.Bucket(helpers.ObjectsBucketLSM) - existing, err := bucket.Get(idBytes) - if err != nil { - return errors.Wrap(err, "unexpected error on previous lookup") - } - - if existing == nil { - // nothing to do - return nil - } - - // we need the doc ID so we can clean up inverted indices currently - // pointing to this object - docID, err = storobj.DocIDFromBinary(existing) - if err != nil { - return errors.Wrap(err, "get existing doc id from object binary") - } - - err = bucket.Delete(idBytes) - if err != nil { - return errors.Wrap(err, "delete object from bucket") - } - - err = s.cleanupInvertedIndexOnDelete(existing, docID) - if err != nil { - return errors.Wrap(err, "delete object from bucket") - } - - if err := s.queue.Delete(docID); err != nil { - return errors.Wrap(err, "delete from vector index") - } - - return nil -} - -func (s *Shard) WasDeleted(ctx context.Context, id strfmt.UUID) (bool, error) { - idBytes, err := uuid.MustParse(id.String()).MarshalBinary() - if err != nil { - return false, err - } - - bucket := s.store.Bucket(helpers.ObjectsBucketLSM) - return bucket.WasDeleted(idBytes) -} diff --git a/adapters/repos/db/shard_replication.go b/adapters/repos/db/shard_replication.go deleted file mode 100644 index 8e4d1dae01eb68f1c5b0c10524799dbe28145821..0000000000000000000000000000000000000000 --- a/adapters/repos/db/shard_replication.go +++ /dev/null @@ -1,194 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "context" - "fmt" - "sync" - - "github.com/go-openapi/strfmt" - "github.com/google/uuid" - "github.com/weaviate/weaviate/entities/storobj" - "github.com/weaviate/weaviate/usecases/objects" - "github.com/weaviate/weaviate/usecases/replica" -) - -type replicaTask func(context.Context) interface{} - -type pendingReplicaTasks struct { - sync.Mutex - Tasks map[string]replicaTask -} - -func (p *pendingReplicaTasks) clear() { - p.Lock() - // TODO: can we postpone deletion until all pending replications are done - p.Tasks = nil - p.Unlock() -} - -func (p *pendingReplicaTasks) get(requestID string) (replicaTask, bool) { - p.Lock() - defer p.Unlock() - t, ok := p.Tasks[requestID] - return t, ok -} - -func (p *pendingReplicaTasks) set(requestID string, task replicaTask) { - p.Lock() - p.Tasks[requestID] = task - p.Unlock() -} - -func (p *pendingReplicaTasks) delete(requestID string) { - p.Lock() - delete(p.Tasks, requestID) - p.Unlock() -} - -func (s *Shard) commitReplication(ctx context.Context, requestID string, backupReadLock *backupMutex) interface{} { - f, ok := s.replicationMap.get(requestID) - if !ok { - return nil - } - defer s.replicationMap.delete(requestID) - backupReadLock.RLock() - defer backupReadLock.RUnlock() - - return f(ctx) -} - -func (s *Shard) abortReplication(ctx context.Context, requestID string) replica.SimpleResponse { - s.replicationMap.delete(requestID) - return replica.SimpleResponse{} -} - -func (s *Shard) preparePutObject(ctx context.Context, requestID string, object *storobj.Object) replica.SimpleResponse { - uuid, err := parseBytesUUID(object.ID()) - if err != nil { - return replica.SimpleResponse{Errors: []replica.Error{{ - Code: replica.StatusPreconditionFailed, Msg: err.Error(), - }}} - } - task := func(ctx context.Context) interface{} { - resp := replica.SimpleResponse{} - if err := s.putOne(ctx, uuid, object); err != nil { - resp.Errors = []replica.Error{ - {Code: replica.StatusConflict, Msg: err.Error()}, - } - } - return resp - } - s.replicationMap.set(requestID, task) - return replica.SimpleResponse{} -} - -func (s *Shard) prepareMergeObject(ctx context.Context, requestID string, doc *objects.MergeDocument) replica.SimpleResponse { - uuid, err := parseBytesUUID(doc.ID) - if err != nil { - return replica.SimpleResponse{Errors: []replica.Error{ - {Code: replica.StatusPreconditionFailed, Msg: err.Error()}, - }} - } - task := func(ctx context.Context) interface{} { - resp := replica.SimpleResponse{} - if err := s.merge(ctx, uuid, *doc); err != nil { - resp.Errors = []replica.Error{ - {Code: replica.StatusConflict, Msg: err.Error()}, - } - } - return resp - } - s.replicationMap.set(requestID, task) - return replica.SimpleResponse{} -} - -func (s *Shard) prepareDeleteObject(ctx context.Context, requestID string, uuid strfmt.UUID) replica.SimpleResponse { - bucket, obj, idBytes, docID, err := s.canDeleteOne(ctx, uuid) - if err != nil { - return replica.SimpleResponse{ - Errors: []replica.Error{ - {Code: replica.StatusPreconditionFailed, Msg: err.Error()}, - }, - } - } - task := func(ctx context.Context) interface{} { - resp := replica.SimpleResponse{} - if err := s.deleteOne(ctx, bucket, obj, idBytes, docID); err != nil { - resp.Errors = []replica.Error{ - {Code: replica.StatusConflict, Msg: err.Error()}, - } - } - return resp - } - s.replicationMap.set(requestID, task) - return replica.SimpleResponse{} -} - -func (s *Shard) preparePutObjects(ctx context.Context, requestID string, objects []*storobj.Object) replica.SimpleResponse { - task := func(ctx context.Context) interface{} { - rawErrs := s.putBatch(ctx, objects) - resp := replica.SimpleResponse{Errors: make([]replica.Error, len(rawErrs))} - for i, err := range rawErrs { - if err != nil { - resp.Errors[i] = replica.Error{Code: replica.StatusConflict, Msg: err.Error()} - } - } - return resp - } - s.replicationMap.set(requestID, task) - return replica.SimpleResponse{} -} - -func (s *Shard) prepareDeleteObjects(ctx context.Context, requestID string, uuids []strfmt.UUID, dryRun bool) replica.SimpleResponse { - task := func(ctx context.Context) interface{} { - result := newDeleteObjectsBatcher(s).Delete(ctx, uuids, dryRun) - resp := replica.DeleteBatchResponse{ - Batch: make([]replica.UUID2Error, len(result)), - } - - for i, r := range result { - entry := replica.UUID2Error{UUID: string(r.UUID)} - if err := r.Err; err != nil { - entry.Error = replica.Error{Code: replica.StatusConflict, Msg: err.Error()} - } - resp.Batch[i] = entry - } - return resp - } - s.replicationMap.set(requestID, task) - return replica.SimpleResponse{} -} - -func (s *Shard) prepareAddReferences(ctx context.Context, requestID string, refs []objects.BatchReference) replica.SimpleResponse { - task := func(ctx context.Context) interface{} { - rawErrs := newReferencesBatcher(s).References(ctx, refs) - resp := replica.SimpleResponse{Errors: make([]replica.Error, len(rawErrs))} - for i, err := range rawErrs { - if err != nil { - resp.Errors[i] = replica.Error{Code: replica.StatusConflict, Msg: err.Error()} - } - } - return resp - } - s.replicationMap.set(requestID, task) - return replica.SimpleResponse{} -} - -func parseBytesUUID(id strfmt.UUID) ([]byte, error) { - uuid, err := uuid.Parse(string(id)) - if err != nil { - return nil, fmt.Errorf("parse uuid %q: %w", id, err) - } - return uuid[:], nil -} diff --git a/adapters/repos/db/shard_skip_vector_reindex_test.go b/adapters/repos/db/shard_skip_vector_reindex_test.go deleted file mode 100644 index ee359995456443d858437deed693114a6a72ff89..0000000000000000000000000000000000000000 --- a/adapters/repos/db/shard_skip_vector_reindex_test.go +++ /dev/null @@ -1,838 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package db - -import ( - "context" - "encoding/json" - "fmt" - "os" - "testing" - "time" - - "github.com/go-openapi/strfmt" - "github.com/google/uuid" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/storobj" - "github.com/weaviate/weaviate/entities/vectorindex/common" - "github.com/weaviate/weaviate/entities/vectorindex/hnsw" - "github.com/weaviate/weaviate/usecases/objects" -) - -func TestShard_SkipVectorReindex(t *testing.T) { - ctx := context.Background() - searchLimit := 10 - vectorSearchLimit := -1 // negative to limit results by distance - vectorSearchDist := float32(1) - - class := &models.Class{ - Class: "TestClass", - InvertedIndexConfig: &models.InvertedIndexConfig{ - IndexTimestamps: true, - IndexNullState: true, - IndexPropertyLength: true, - }, - Properties: []*models.Property{ - { - Name: "texts", - DataType: schema.DataTypeTextArray.PropString(), - Tokenization: models.PropertyTokenizationWord, - }, - { - Name: "numbers", - DataType: schema.DataTypeNumberArray.PropString(), - }, - { - Name: "ints", - DataType: schema.DataTypeIntArray.PropString(), - }, - { - Name: "booleans", - DataType: schema.DataTypeBooleanArray.PropString(), - }, - { - Name: "dates", - DataType: schema.DataTypeDateArray.PropString(), - }, - { - Name: "uuids", - DataType: schema.DataTypeUUIDArray.PropString(), - }, - { - Name: "text", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWord, - }, - { - Name: "number", - DataType: schema.DataTypeNumber.PropString(), - }, - { - Name: "int", - DataType: schema.DataTypeInt.PropString(), - }, - { - Name: "boolean", - DataType: schema.DataTypeBoolean.PropString(), - }, - { - Name: "date", - DataType: schema.DataTypeDate.PropString(), - }, - { - Name: "uuid", - DataType: schema.DataTypeUUID.PropString(), - }, - }, - } - vectorIndexConfig := hnsw.UserConfig{Distance: common.DefaultDistanceMetric} - - uuid := strfmt.UUID(uuid.NewString()) - vector := []float32{1, 2, 3} - altVector := []float32{10, 0, -20} - - origObj := &storobj.Object{ - MarshallerVersion: 1, - Object: models.Object{ - ID: uuid, - Class: class.Class, - Properties: map[string]interface{}{ - "texts": []interface{}{ - "aaa", - "bbb", - "ccc", - }, - "numbers": []interface{}{}, - "ints": []interface{}{ - asJsonNumber(101), asJsonNumber(101), asJsonNumber(101), asJsonNumber(101), asJsonNumber(101), asJsonNumber(101), - asJsonNumber(102), - asJsonNumber(103), - asJsonNumber(104), - }, - "booleans": []interface{}{ - true, true, true, - false, - }, - "dates": []interface{}{ - "2001-06-01T12:00:00.000000Z", - "2002-06-02T12:00:00.000000Z", - }, - // no uuids - "text": "ddd", - // no number - "int": int64(201), - "boolean": false, - "date": asTime("2003-06-01T12:00:00.000000Z"), - // no uuid - }, - }, - Vector: vector, - } - - updObj := &storobj.Object{ - MarshallerVersion: 1, - Object: models.Object{ - ID: uuid, - Class: class.Class, - Properties: map[string]interface{}{ - "texts": []interface{}{}, - // no numbers - "ints": []interface{}{ - asJsonNumber(101), asJsonNumber(101), asJsonNumber(101), asJsonNumber(101), - asJsonNumber(103), - asJsonNumber(104), - asJsonNumber(105), - }, - "booleans": []interface{}{ - true, true, true, - false, - }, - // no dates - "uuids": []interface{}{ - asUuid("d726c960-aede-411c-85d3-2c77e9290a6e"), - }, - "text": "", - // no number - "int": int64(202), - "boolean": true, - // no date - "uuid": asUuid("7fabaf01-9e10-458a-acea-cc627376c506"), - }, - }, - Vector: vector, - } - - mergeDoc := objects.MergeDocument{ - ID: uuid, - Class: class.Class, - PrimitiveSchema: map[string]interface{}{ - "texts": []interface{}{}, - "ints": []interface{}{ - asJsonNumber(101), asJsonNumber(101), asJsonNumber(101), asJsonNumber(101), - asJsonNumber(103), - asJsonNumber(104), - asJsonNumber(105), - }, - "uuids": []interface{}{ - asUuid("d726c960-aede-411c-85d3-2c77e9290a6e"), - }, - "text": "", - "int": int64(202), - "boolean": true, - "uuid": asUuid("7fabaf01-9e10-458a-acea-cc627376c506"), - }, - PropertiesToDelete: []string{"numbers", "dates", "date"}, - Vector: vector, - } - - filterTextsEqAAA := filterEqual[string]("aaa", schema.DataTypeText, class.Class, "texts") - filterTextsLen3 := filterEqual[int](3, schema.DataTypeInt, class.Class, "len(texts)") - filterTextsLen0 := filterEqual[int](0, schema.DataTypeInt, class.Class, "len(texts)") - filterTextsNotNil := filterNil(false, class.Class, "texts") - filterTextsNil := filterNil(true, class.Class, "texts") - - filterNumbersEq123 := filterEqual[float64](1.23, schema.DataTypeNumber, class.Class, "numbers") - filterNumbersLen1 := filterEqual[int](1, schema.DataTypeInt, class.Class, "len(numbers)") - filterNumbersLen0 := filterEqual[int](0, schema.DataTypeInt, class.Class, "len(numbers)") - filterNumbersNotNil := filterNil(false, class.Class, "numbers") - filterNumbersNil := filterNil(true, class.Class, "numbers") - - filterIntsEq102 := filterEqual[int](102, schema.DataTypeInt, class.Class, "ints") - filterIntsEq105 := filterEqual[int](105, schema.DataTypeInt, class.Class, "ints") - filterIntsLen9 := filterEqual[int](9, schema.DataTypeInt, class.Class, "len(ints)") - filterIntsLen7 := filterEqual[int](7, schema.DataTypeInt, class.Class, "len(ints)") - filterIntsNotNil := filterNil(false, class.Class, "ints") - filterIntsNil := filterNil(true, class.Class, "ints") - - filterBoolsEqTrue := filterEqual[bool](true, schema.DataTypeBoolean, class.Class, "booleans") - filterBoolsEqFalse := filterEqual[bool](false, schema.DataTypeBoolean, class.Class, "booleans") - filterBoolsLen4 := filterEqual[int](4, schema.DataTypeInt, class.Class, "len(booleans)") - filterBoolsLen0 := filterEqual[int](0, schema.DataTypeInt, class.Class, "len(booleans)") - filterBoolsNotNil := filterNil(false, class.Class, "booleans") - filterBoolsNil := filterNil(true, class.Class, "booleans") - - filterDatesEq2001 := filterEqual[string]("2001-06-01T12:00:00.000000Z", schema.DataTypeDate, class.Class, "dates") - filterDatesLen2 := filterEqual[int](2, schema.DataTypeInt, class.Class, "len(dates)") - filterDatesLen0 := filterEqual[int](0, schema.DataTypeInt, class.Class, "len(dates)") - filterDatesNotNil := filterNil(false, class.Class, "dates") - filterDatesNil := filterNil(true, class.Class, "dates") - - filterUuidsEqD726 := filterEqual[string]("d726c960-aede-411c-85d3-2c77e9290a6e", schema.DataTypeText, class.Class, "uuids") - filterUuidsLen1 := filterEqual[int](1, schema.DataTypeInt, class.Class, "len(uuids)") - filterUuidsLen0 := filterEqual[int](0, schema.DataTypeInt, class.Class, "len(uuids)") - filterUuidsNotNil := filterNil(false, class.Class, "uuids") - filterUuidsNil := filterNil(true, class.Class, "uuids") - - filterTextEqDDD := filterEqual[string]("ddd", schema.DataTypeText, class.Class, "text") - filterTextLen3 := filterEqual[int](3, schema.DataTypeInt, class.Class, "len(text)") - filterTextLen0 := filterEqual[int](0, schema.DataTypeInt, class.Class, "len(text)") - filterTextNotNil := filterNil(false, class.Class, "text") - filterTextNil := filterNil(true, class.Class, "text") - - filterNumberEq123 := filterEqual[float64](1.23, schema.DataTypeNumber, class.Class, "number") - filterNumberNotNil := filterNil(false, class.Class, "number") - filterNumberNil := filterNil(true, class.Class, "number") - - filterIntEq201 := filterEqual[int](201, schema.DataTypeInt, class.Class, "int") - filterIntEq202 := filterEqual[int](202, schema.DataTypeInt, class.Class, "int") - filterIntNotNil := filterNil(false, class.Class, "int") - filterIntNil := filterNil(true, class.Class, "int") - - filterBoolEqFalse := filterEqual[bool](false, schema.DataTypeBoolean, class.Class, "boolean") - filterBoolEqTrue := filterEqual[bool](true, schema.DataTypeBoolean, class.Class, "boolean") - filterBoolNotNil := filterNil(false, class.Class, "boolean") - filterBoolNil := filterNil(true, class.Class, "boolean") - - filterDateEq2003 := filterEqual[string]("2003-06-01T12:00:00.000000Z", schema.DataTypeDate, class.Class, "date") - filterDateNotNil := filterNil(false, class.Class, "date") - filterDateNil := filterNil(true, class.Class, "date") - - filterUuidEq7FAB := filterEqual[string]("7fabaf01-9e10-458a-acea-cc627376c506", schema.DataTypeText, class.Class, "uuid") - filterUuidNotNil := filterNil(false, class.Class, "uuid") - filterUuidNil := filterNil(true, class.Class, "uuid") - - verifySearchAfterAdd := func(shard ShardLike) func(t *testing.T) { - return func(t *testing.T) { - t.Run("to be found", func(t *testing.T) { - for name, filter := range map[string]*filters.LocalFilter{ - "textsEqAAA": filterTextsEqAAA, - "textsLen3": filterTextsLen3, - "textsNotNil": filterTextsNotNil, - - "numbersLen0": filterNumbersLen0, - "numbersNil": filterNumbersNil, - - "intsEq102": filterIntsEq102, - "intsLen9": filterIntsLen9, - "intsNotNil": filterIntsNotNil, - - "boolsEqTrue": filterBoolsEqTrue, - "boolsEqFalse": filterBoolsEqFalse, - "boolsLen4": filterBoolsLen4, - "boolsNotNil": filterBoolsNotNil, - - "datesEq2001": filterDatesEq2001, - "datesLen2": filterDatesLen2, - "datesNotNil": filterDatesNotNil, - - "uuidsLen0": filterUuidsLen0, - "uuidsNil": filterUuidsNil, - - "textEqDDD": filterTextEqDDD, - "textLen3": filterTextLen3, - "textNotNil": filterTextNotNil, - - "numberNil": filterNumberNil, - - "intEq201": filterIntEq201, - "intNotNil": filterIntNotNil, - - "boolEqFalse": filterBoolEqFalse, - "boolNotNil": filterBoolNotNil, - - "dateEq2003": filterDateEq2003, - "dateNotNil": filterDateNotNil, - - "uuidNil": filterUuidNil, - } { - t.Run(name, func(t *testing.T) { - found, _, err := shard.ObjectSearch(ctx, searchLimit, filter, - nil, nil, nil, additional.Properties{}) - require.NoError(t, err) - require.Len(t, found, 1) - require.Equal(t, uuid, found[0].Object.ID) - }) - } - }) - - t.Run("not to be found", func(t *testing.T) { - for name, filter := range map[string]*filters.LocalFilter{ - "textsLen0": filterTextsLen0, - "textsNil": filterTextsNil, - - "numbersEq123": filterNumbersEq123, - "numbersLen1": filterNumbersLen1, - "numbersNotNil": filterNumbersNotNil, - - "intsEq105": filterIntsEq105, - "intsLen7": filterIntsLen7, - "intsNil": filterIntsNil, - - "boolsLen0": filterBoolsLen0, - "boolsNil": filterBoolsNil, - - "datesLen0": filterDatesLen0, - "datesNil": filterDatesNil, - - "uuidsEqD726": filterUuidsEqD726, - "uuidsLen1": filterUuidsLen1, - "uuidsNotNil": filterUuidsNotNil, - - "textLen0": filterTextLen0, - "textNil": filterTextNil, - - "numberEq123": filterNumberEq123, - "numberNotNil": filterNumberNotNil, - - "intEq202": filterIntEq202, - "intNil": filterIntNil, - - "boolEqTrue": filterBoolEqTrue, - "boolNil": filterBoolNil, - - "dateNil": filterDateNil, - - "uuidEq7FAB": filterUuidEq7FAB, - "uuidNotNil": filterUuidNotNil, - } { - t.Run(name, func(t *testing.T) { - found, _, err := shard.ObjectSearch(ctx, searchLimit, filter, - nil, nil, nil, additional.Properties{}) - require.NoError(t, err) - require.Len(t, found, 0) - }) - } - }) - } - } - - verifySearchAfterUpdate := func(shard ShardLike) func(t *testing.T) { - return func(t *testing.T) { - t.Run("to be found", func(t *testing.T) { - for name, filter := range map[string]*filters.LocalFilter{ - "textsLen0": filterTextsLen0, - "textsNil": filterTextsNil, - - "numbersLen0": filterNumbersLen0, - "numbersNil": filterNumbersNil, - - "intsEq105": filterIntsEq105, - "intsLen7": filterIntsLen7, - "intsNotNil": filterIntsNotNil, - - "boolsEqTrue": filterBoolsEqTrue, - "boolsEqFalse": filterBoolsEqFalse, - "boolsLen4": filterBoolsLen4, - "boolsNotNil": filterBoolsNotNil, - - "datesLen0": filterDatesLen0, - "datesNil": filterDatesNil, - - "uuidsEqD726": filterUuidsEqD726, - "uuidsLen1": filterUuidsLen1, - "uuidsNotNil": filterUuidsNotNil, - - "textLen0": filterTextLen0, - "textNil": filterTextNil, - - "numberNil": filterNumberNil, - - "intEq202": filterIntEq202, - "intNotNil": filterIntNotNil, - - "boolEqTrue": filterBoolEqTrue, - "boolNotNil": filterBoolNotNil, - - "dateNil": filterDateNil, - - "uuidEq7FAB": filterUuidEq7FAB, - "uuidNotNil": filterUuidNotNil, - } { - t.Run(name, func(t *testing.T) { - found, _, err := shard.ObjectSearch(ctx, searchLimit, filter, - nil, nil, nil, additional.Properties{}) - require.NoError(t, err) - require.Len(t, found, 1) - require.Equal(t, uuid, found[0].Object.ID) - }) - } - }) - - t.Run("not to be found", func(t *testing.T) { - for name, filter := range map[string]*filters.LocalFilter{ - "textsEqAAA": filterTextsEqAAA, - "textsLen3": filterTextsLen3, - "textsNotNil": filterTextsNotNil, - - "numbersEq123": filterNumbersEq123, - "numbersLen1": filterNumbersLen1, - "numbersNotNil": filterNumbersNotNil, - - "intsEq102": filterIntsEq102, - "intsLen9": filterIntsLen9, - "intsNil": filterIntsNil, - - "boolsLen0": filterBoolsLen0, - "boolsNil": filterBoolsNil, - - "datesEq2001": filterDatesEq2001, - "datesLen2": filterDatesLen2, - "datesNotNil": filterDatesNotNil, - - "uuidsLen0": filterUuidsLen0, - "uuidsNil": filterUuidsNil, - - "textEqDDD": filterTextEqDDD, - "textLen3": filterTextLen3, - "textNotNil": filterTextNotNil, - - "numberEq123": filterNumberEq123, - "numberNotNil": filterNumberNotNil, - - "intEq201": filterIntEq201, - "intNil": filterIntNil, - - "boolEqFalse": filterBoolEqFalse, - "boolNil": filterBoolNil, - - "dateEq2003": filterDateEq2003, - "dateNotNil": filterDateNotNil, - - "uuidNil": filterUuidNil, - } { - t.Run(name, func(t *testing.T) { - found, _, err := shard.ObjectSearch(ctx, searchLimit, filter, - nil, nil, nil, additional.Properties{}) - require.NoError(t, err) - require.Len(t, found, 0) - }) - } - }) - } - } - - verifyVectorSearch := func(shard ShardLike, vectorToBeFound, vectorNotToBeFound []float32) func(t *testing.T) { - return func(t *testing.T) { - t.Run("to be found", func(t *testing.T) { - found, _, err := shard.ObjectVectorSearch(ctx, vectorToBeFound, - vectorSearchDist, vectorSearchLimit, nil, nil, nil, additional.Properties{}) - require.NoError(t, err) - require.Len(t, found, 1) - require.Equal(t, uuid, found[0].Object.ID) - }) - - t.Run("not to be found", func(t *testing.T) { - found, _, err := shard.ObjectVectorSearch(ctx, vectorNotToBeFound, - vectorSearchDist, vectorSearchLimit, nil, nil, nil, additional.Properties{}) - require.NoError(t, err) - require.Len(t, found, 0) - }) - } - } - - createShard := func(t *testing.T) ShardLike { - shard, _ := testShardWithSettings(t, ctx, class, vectorIndexConfig, true, true) - return shard - } - - t.Run("single object", func(t *testing.T) { - t.Run("sanity check - search after add", func(t *testing.T) { - shard := createShard(t) - - t.Run("add object", func(t *testing.T) { - expectedNextDocID := uint64(1) - - err := shard.PutObject(ctx, origObj) - require.NoError(t, err) - require.Equal(t, expectedNextDocID, shard.Counter().Get()) - }) - - t.Run("verify search after add", verifySearchAfterAdd(shard)) - t.Run("verify vector search after add", verifyVectorSearch(shard, vector, altVector)) - }) - - t.Run("replaces object, same vector", func(t *testing.T) { - shard := createShard(t) - - t.Run("add object", func(t *testing.T) { - expectedNextDocID := uint64(1) - - err := shard.PutObject(ctx, origObj) - require.NoError(t, err) - require.Equal(t, expectedNextDocID, shard.Counter().Get()) - }) - - t.Run("put object", func(t *testing.T) { - expectedNextDocID := uint64(1) // has not changed - - err := shard.PutObject(ctx, updObj) - require.NoError(t, err) - require.Equal(t, expectedNextDocID, shard.Counter().Get()) - }) - - t.Run("verify search after put", verifySearchAfterUpdate(shard)) - t.Run("verify vector search after put", verifyVectorSearch(shard, vector, altVector)) - }) - - t.Run("replaces object, different vector", func(t *testing.T) { - shard := createShard(t) - - t.Run("add object", func(t *testing.T) { - expectedNextDocID := uint64(1) - - err := shard.PutObject(ctx, origObj) - require.NoError(t, err) - require.Equal(t, expectedNextDocID, shard.Counter().Get()) - }) - - t.Run("put object", func(t *testing.T) { - altUpdObj := &storobj.Object{ - MarshallerVersion: updObj.MarshallerVersion, - Object: updObj.Object, - Vector: altVector, - } - expectedNextDocID := uint64(2) // has changed - - err := shard.PutObject(ctx, altUpdObj) - require.NoError(t, err) - require.Equal(t, expectedNextDocID, shard.Counter().Get()) - }) - - t.Run("verify search after put", verifySearchAfterUpdate(shard)) - t.Run("verify vector search after put", verifyVectorSearch(shard, altVector, vector)) - }) - - t.Run("merges object, same vector", func(t *testing.T) { - shard := createShard(t) - - t.Run("add object", func(t *testing.T) { - expectedNextDocID := uint64(1) - - err := shard.PutObject(ctx, origObj) - require.NoError(t, err) - require.Equal(t, expectedNextDocID, shard.Counter().Get()) - }) - - t.Run("merge object", func(t *testing.T) { - expectedNextDocID := uint64(1) // has not changed - - err := shard.MergeObject(ctx, mergeDoc) - require.NoError(t, err) - require.Equal(t, expectedNextDocID, shard.Counter().Get()) - }) - - t.Run("verify search after merge", verifySearchAfterUpdate(shard)) - t.Run("verify vector search after merge", verifyVectorSearch(shard, vector, altVector)) - }) - - t.Run("merges object, different vector", func(t *testing.T) { - shard := createShard(t) - - t.Run("add object", func(t *testing.T) { - expectedNextDocID := uint64(1) - - err := shard.PutObject(ctx, origObj) - require.NoError(t, err) - require.Equal(t, expectedNextDocID, shard.Counter().Get()) - }) - - t.Run("merge object", func(t *testing.T) { - altMergeDoc := objects.MergeDocument{ - ID: mergeDoc.ID, - Class: mergeDoc.Class, - PrimitiveSchema: mergeDoc.PrimitiveSchema, - PropertiesToDelete: mergeDoc.PropertiesToDelete, - Vector: altVector, - } - expectedNextDocID := uint64(2) // has changed - - err := shard.MergeObject(ctx, altMergeDoc) - require.NoError(t, err) - require.Equal(t, expectedNextDocID, shard.Counter().Get()) - }) - - t.Run("verify search after merge", verifySearchAfterUpdate(shard)) - t.Run("verify vector search after merge", verifyVectorSearch(shard, altVector, vector)) - }) - }) - - t.Run("batch", func(t *testing.T) { - runBatch := func(t *testing.T) { - t.Run("sanity check - search after add", func(t *testing.T) { - shard := createShard(t) - - t.Run("add batch", func(t *testing.T) { - expectedNextDocID := uint64(1) - - errs := shard.PutObjectBatch(ctx, []*storobj.Object{origObj}) - for i := range errs { - require.NoError(t, errs[i]) - } - require.Equal(t, expectedNextDocID, shard.Counter().Get()) - }) - - t.Run("verify search after batch", verifySearchAfterAdd(shard)) - t.Run("verify vector search after batch", verifyVectorSearch(shard, vector, altVector)) - }) - - t.Run("replaces object, same vector", func(t *testing.T) { - shard := createShard(t) - - t.Run("add batch", func(t *testing.T) { - expectedNextDocID := uint64(1) - - errs := shard.PutObjectBatch(ctx, []*storobj.Object{origObj}) - for i := range errs { - require.NoError(t, errs[i]) - } - require.Equal(t, expectedNextDocID, shard.Counter().Get()) - }) - - t.Run("add 2nd batch", func(t *testing.T) { - expectedNextDocID := uint64(1) // has not changed - - errs := shard.PutObjectBatch(ctx, []*storobj.Object{updObj}) - for i := range errs { - require.NoError(t, errs[i]) - } - require.Equal(t, expectedNextDocID, shard.Counter().Get()) - }) - - t.Run("verify search after 2nd batch", verifySearchAfterUpdate(shard)) - t.Run("verify vector search after 2nd batch", verifyVectorSearch(shard, vector, altVector)) - }) - - t.Run("replaces object, different vector", func(t *testing.T) { - shard := createShard(t) - - t.Run("add batch", func(t *testing.T) { - expectedNextDocID := uint64(1) - - errs := shard.PutObjectBatch(ctx, []*storobj.Object{origObj}) - for i := range errs { - require.NoError(t, errs[i]) - } - require.Equal(t, expectedNextDocID, shard.Counter().Get()) - }) - - t.Run("add 2nd batch", func(t *testing.T) { - altUpdObj := &storobj.Object{ - MarshallerVersion: updObj.MarshallerVersion, - Object: updObj.Object, - Vector: altVector, - } - expectedNextDocID := uint64(2) // has changed - - errs := shard.PutObjectBatch(ctx, []*storobj.Object{altUpdObj}) - for i := range errs { - require.NoError(t, errs[i]) - } - require.Equal(t, expectedNextDocID, shard.Counter().Get()) - }) - - t.Run("verify search after 2nd batch", verifySearchAfterUpdate(shard)) - t.Run("verify vector search after 2nd batch", verifyVectorSearch(shard, altVector, vector)) - }) - } - - t.Run("sync", func(t *testing.T) { - currentIndexing := os.Getenv("ASYNC_INDEXING") - t.Setenv("ASYNC_INDEXING", "") - defer t.Setenv("ASYNC_INDEXING", currentIndexing) - - runBatch(t) - }) - - // t.Run("async", func(t *testing.T) { - // asyncIndexing := os.Getenv("ASYNC_INDEXING") - // t.Setenv("ASYNC_INDEXING", "true") - // defer t.Setenv("ASYNC_INDEXING", asyncIndexing) - - // runBatch(t) - // }) - }) -} - -func filterEqual[T any](value T, dataType schema.DataType, className, propName string) *filters.LocalFilter { - return &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorEqual, - Value: &filters.Value{ - Value: value, - Type: dataType, - }, - On: &filters.Path{ - Class: schema.ClassName(className), - Property: schema.PropertyName(propName), - }, - }, - } -} - -func filterNil(value bool, className, propName string) *filters.LocalFilter { - return &filters.LocalFilter{ - Root: &filters.Clause{ - Operator: filters.OperatorIsNull, - Value: &filters.Value{ - Value: value, - Type: schema.DataTypeBoolean, - }, - On: &filters.Path{ - Class: schema.ClassName(className), - Property: schema.PropertyName(propName), - }, - }, - } -} - -func asJsonNumber(input int) json.Number { - return json.Number(fmt.Sprint(input)) -} - -func asTime(input string) time.Time { - asTime, err := time.Parse(time.RFC3339Nano, input) - if err != nil { - panic(err) - } - return asTime -} - -func asUuid(input string) uuid.UUID { - asUuid, err := uuid.Parse(input) - if err != nil { - panic(err) - } - return asUuid -} - -// "textsEqAAA": filterTextsEqAAA, -// "textsLen3": filterTextsLen3, -// "textsNotNil": filterTextsNotNil, -// "textsLen0": filterTextsLen0, -// "textsNil": filterTextsNil, - -// "numbersLen0": filterNumbersLen0, -// "numbersNil": filterNumbersNil, -// "numbersEq123": filterNumbersEq123, -// "numbersLen1": filterNumbersLen1, -// "numbersNotNil": filterNumbersNotNil, - -// "intsEq102": filterIntsEq102, -// "intsLen9": filterIntsLen9, -// "intsNotNil": filterIntsNotNil, -// "intsEq105": filterIntsEq105, -// "intsLen7": filterIntsLen7, -// "intsNil": filterIntsNil, - -// "boolsEqTrue": filterBoolsEqTrue, -// "boolsEqFalse": filterBoolsEqFalse, -// "boolsLen4": filterBoolsLen4, -// "boolsNotNil": filterBoolsNotNil, -// "boolsLen0": filterBoolsLen0, -// "boolsNil": filterBoolsNil, - -// "datesEq2001": filterDatesEq2001, -// "datesLen2": filterDatesLen2, -// "datesNotNil": filterDatesNotNil, -// "datesLen0": filterDatesLen0, -// "datesNil": filterDatesNil, - -// "uuidsLen0": filterUuidsLen0, -// "uuidsNil": filterUuidsNil, -// "uuidsEqD726": filterUuidsEqD726, -// "uuidsLen1": filterUuidsLen1, -// "uuidsNotNil": filterUuidsNotNil, - -// "textEqDDD": filterTextEqDDD, -// "textLen3": filterTextLen3, -// "textNotNil": filterTextNotNil, -// "textLen0": filterTextLen0, -// "textNil": filterTextNil, - -// "numberNil": filterNumberNil, -// "numberEq123": filterNumberEq123, -// "numberNotNil": filterNumberNotNil, - -// "intEq201": filterIntEq201, -// "intNotNil": filterIntNotNil, -// "intEq202": filterIntEq202, -// "intNil": filterIntNil, - -// "boolEqFalse": filterBoolEqFalse, -// "boolNotNil": filterBoolNotNil, -// "boolEqTrue": filterBoolEqTrue, -// "boolNil": filterBoolNil, - -// "dateEq2003": filterDateEq2003, -// "dateNotNil": filterDateNotNil, -// "dateNil": filterDateNil, - -// "uuidNil": filterUuidNil, -// "uuidEq7FAB": filterUuidEq7FAB, -// "uuidNotNil": filterUuidNotNil, diff --git a/adapters/repos/db/shard_status.go b/adapters/repos/db/shard_status.go deleted file mode 100644 index adadcac09e1f0790470082c9b29385ecd5c49f02..0000000000000000000000000000000000000000 --- a/adapters/repos/db/shard_status.go +++ /dev/null @@ -1,79 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "strings" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/storagestate" -) - -func (s *Shard) initStatus() { - s.statusLock.Lock() - defer s.statusLock.Unlock() - - s.status = storagestate.StatusReady -} - -func (s *Shard) GetStatus() storagestate.Status { - s.statusLock.Lock() - defer s.statusLock.Unlock() - - return s.status -} - -func (s *Shard) isReadOnly() bool { - return s.GetStatus() == storagestate.StatusReadOnly -} - -func (s *Shard) compareAndSwapStatus(old, new string) (storagestate.Status, error) { - s.statusLock.Lock() - defer s.statusLock.Unlock() - - if s.status.String() != old { - return s.status, nil - } - - return s.status, s.updateStatusUnlocked(new) -} - -func (s *Shard) UpdateStatus(in string) error { - s.statusLock.Lock() - defer s.statusLock.Unlock() - - return s.updateStatusUnlocked(in) -} - -// updateStatusUnlocked updates the status without locking the statusLock. -// Warning: Use UpdateStatus instead. -func (s *Shard) updateStatusUnlocked(in string) error { - targetStatus, err := storagestate.ValidateStatus(strings.ToUpper(in)) - if err != nil { - return errors.Wrap(err, in) - } - - s.status = targetStatus - s.updateStoreStatus(targetStatus) - - s.index.logger. - WithField("action", "update shard status"). - WithField("class", s.index.Config.ClassName). - WithField("shard", s.name). - WithField("status", in) - - return nil -} - -func (s *Shard) updateStoreStatus(targetStatus storagestate.Status) { - s.store.UpdateBucketsStatus(targetStatus) -} diff --git a/adapters/repos/db/shard_test.go b/adapters/repos/db/shard_test.go deleted file mode 100644 index 1071080ef91cfd50875cbc0bc7876a546ad159da..0000000000000000000000000000000000000000 --- a/adapters/repos/db/shard_test.go +++ /dev/null @@ -1,193 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package db - -import ( - "context" - "crypto/rand" - "encoding/json" - "fmt" - "os" - "path" - "sync" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/storagestate" - "github.com/weaviate/weaviate/entities/storobj" -) - -func TestShard_UpdateStatus(t *testing.T) { - ctx := testCtx() - className := "TestClass" - shd, idx := testShard(t, ctx, className) - - amount := 10 - - defer func(path string) { - err := os.RemoveAll(path) - if err != nil { - fmt.Println(err) - } - }(shd.Index().Config.RootPath) - - t.Run("insert data into shard", func(t *testing.T) { - for i := 0; i < amount; i++ { - obj := testObject(className) - - err := shd.PutObject(ctx, obj) - require.Nil(t, err) - } - - objs, err := shd.ObjectList(ctx, amount, nil, nil, additional.Properties{}, shd.Index().Config.ClassName) - require.Nil(t, err) - require.Equal(t, amount, len(objs)) - }) - - t.Run("mark shard readonly and fail to insert", func(t *testing.T) { - err := shd.UpdateStatus(storagestate.StatusReadOnly.String()) - require.Nil(t, err) - - err = shd.PutObject(ctx, testObject(className)) - require.EqualError(t, err, storagestate.ErrStatusReadOnly.Error()) - }) - - t.Run("mark shard ready and insert successfully", func(t *testing.T) { - err := shd.UpdateStatus(storagestate.StatusReady.String()) - require.Nil(t, err) - - err = shd.PutObject(ctx, testObject(className)) - require.Nil(t, err) - }) - - require.Nil(t, idx.drop()) - require.Nil(t, os.RemoveAll(idx.Config.RootPath)) -} - -func TestShard_ReadOnly_HaltCompaction(t *testing.T) { - amount := 10000 - sizePerValue := 8 - bucketName := "testbucket" - - keys := make([][]byte, amount) - values := make([][]byte, amount) - - shd, idx := testShard(t, context.Background(), "TestClass") - - defer func(path string) { - err := os.RemoveAll(path) - if err != nil { - fmt.Println(err) - } - }(shd.Index().Config.RootPath) - - err := shd.Store().CreateOrLoadBucket(context.Background(), bucketName, - lsmkv.WithMemtableThreshold(1024)) - require.Nil(t, err) - - bucket := shd.Store().Bucket(bucketName) - require.NotNil(t, bucket) - dirName := path.Join(shd.Index().path(), shd.Name(), "lsm", bucketName) - - t.Run("generate random data", func(t *testing.T) { - for i := range keys { - n, err := json.Marshal(i) - require.Nil(t, err) - - keys[i] = n - values[i] = make([]byte, sizePerValue) - rand.Read(values[i]) - } - }) - - t.Run("insert data into bucket", func(t *testing.T) { - for i := range keys { - err := bucket.Put(keys[i], values[i]) - assert.Nil(t, err) - time.Sleep(time.Microsecond) - } - - t.Logf("insertion complete!") - }) - - t.Run("halt compaction with readonly status", func(t *testing.T) { - err := shd.UpdateStatus(storagestate.StatusReadOnly.String()) - require.Nil(t, err) - - // give the status time to propagate - // before grabbing the baseline below - time.Sleep(time.Second) - - // once shard status is set to readonly, - // the number of segment files should - // not change - entries, err := os.ReadDir(dirName) - require.Nil(t, err) - numSegments := len(entries) - - // if the number of segments remain the - // same for 30 seconds, we can be - // reasonably sure that the compaction - // process was halted - for i := 0; i < 30; i++ { - entries, err := os.ReadDir(dirName) - require.Nil(t, err) - - require.Equal(t, numSegments, len(entries)) - t.Logf("iteration %d, sleeping", i) - time.Sleep(time.Second) - } - }) - - t.Run("update shard status to ready", func(t *testing.T) { - err := shd.UpdateStatus(storagestate.StatusReady.String()) - require.Nil(t, err) - - time.Sleep(time.Second) - }) - - require.Nil(t, idx.drop()) -} - -// tests adding multiple larger batches in parallel using different settings of the goroutine factor. -// In all cases all objects should be added -func TestShard_ParallelBatches(t *testing.T) { - r := getRandomSeed() - batches := make([][]*storobj.Object, 4) - for i := range batches { - batches[i] = createRandomObjects(r, "TestClass", 1000) - } - totalObjects := 1000 * len(batches) - ctx := testCtx() - shd, idx := testShard(t, context.Background(), "TestClass") - - // add batches in parallel - wg := sync.WaitGroup{} - wg.Add(len(batches)) - for _, batch := range batches { - go func(localBatch []*storobj.Object) { - shd.PutObjectBatch(ctx, localBatch) - wg.Done() - }(batch) - } - wg.Wait() - - require.Equal(t, totalObjects, int(shd.Counter().Get())) - require.Nil(t, idx.drop()) -} diff --git a/adapters/repos/db/shard_version.go b/adapters/repos/db/shard_version.go deleted file mode 100644 index 5db0af0496fa93f1d524080a5bc26fb8db91ad44..0000000000000000000000000000000000000000 --- a/adapters/repos/db/shard_version.go +++ /dev/null @@ -1,117 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "encoding/binary" - "os" - - "github.com/pkg/errors" -) - -// ShardCodeBaseVersion must be increased whenever there are breaking changes - -// including those that we can handle in a non-breaking way -// the version checker can then decide on init if it should prevent startup -// completely. If it does not prevent startup, but there is still a version -// mismatch, the version can be used to make specific decisions -// -// CHANGELOG -// - Version 1 - Everything up until Weaviate v1.10.1 inclusive -// - Version 2 - Inverted Index is now stored in an always sorted fashion and -// doc ids are stored as BigEndian. To make this backward-compatible with v1, -// doc ids need to be read and written as Little Endian. In addition, an -// additional sort step is required in three places: during a MapList call, -// during a Map Cursor and during Map Compactions. BM25 is entirely disabled -// prior to this version -const ( - ShardCodeBaseVersion = uint16(2) - ShardCodeBaseMinimumVersionForStartup = uint16(1) -) - -type shardVersioner struct { - version uint16 - - // we don't need the file after initialization, but still need to track its - // path so we can delete it on .Drop() - path string -} - -func newShardVersioner(baseDir string, dataPresent bool) (*shardVersioner, error) { - sv := &shardVersioner{} - - return sv, sv.init(baseDir, dataPresent) -} - -func (sv *shardVersioner) init(fileName string, dataPresent bool) error { - sv.path = fileName - - f, err := os.OpenFile(fileName, os.O_RDWR|os.O_CREATE, 0o666) - if err != nil { - return err - } - - stat, err := f.Stat() - if err != nil { - return err - } - - var version uint16 = 1 - if stat.Size() > 0 { - // the file has existed before, we need to initialize with its content - err := binary.Read(f, binary.LittleEndian, &version) - if err != nil { - return errors.Wrap(err, "read initial version from file") - } - } else { - // if the version file does not yet exist, there are two scenarios: - // 1) We are just creating this class, which means its version is - // ShardCodeBaseVersion. - // 2) There is data present, so we must assume it was built with a version - // that did not yet have this versioner present, so we assume it's v1 - if !dataPresent { - version = ShardCodeBaseVersion - } else { - version = 1 - } - - err := binary.Write(f, binary.LittleEndian, &version) - if err != nil { - return errors.Wrap(err, "write version back to file") - } - - if err := f.Close(); err != nil { - return errors.Wrap(err, "close version file") - } - } - - if version < ShardCodeBaseMinimumVersionForStartup { - return errors.Errorf("cannot start up shard: it was built with shard "+ - "version v%d, but this version of Weaviate requires at least shard version v%d", - version, ShardCodeBaseMinimumVersionForStartup) - } - - sv.version = version - - return nil -} - -func (sv *shardVersioner) Drop() error { - err := os.Remove(sv.path) - if err != nil { - return errors.Wrap(err, "drop versioner file") - } - return nil -} - -func (sv *shardVersioner) Version() uint16 { - return sv.version -} diff --git a/adapters/repos/db/shard_write_batch_delete.go b/adapters/repos/db/shard_write_batch_delete.go deleted file mode 100644 index 8660741fdc53c63cfba9faf1c7ebb370a7b320af..0000000000000000000000000000000000000000 --- a/adapters/repos/db/shard_write_batch_delete.go +++ /dev/null @@ -1,155 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "context" - "fmt" - "sync" - "time" - - "github.com/go-openapi/strfmt" - "github.com/pkg/errors" - "github.com/weaviate/weaviate/adapters/repos/db/inverted" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/storagestate" - "github.com/weaviate/weaviate/usecases/objects" -) - -// return value map[int]error gives the error for the index as it received it -func (s *Shard) DeleteObjectBatch(ctx context.Context, uuids []strfmt.UUID, dryRun bool) objects.BatchSimpleObjects { - if s.isReadOnly() { - return objects.BatchSimpleObjects{ - objects.BatchSimpleObject{Err: storagestate.ErrStatusReadOnly}, - } - } - return newDeleteObjectsBatcher(s).Delete(ctx, uuids, dryRun) -} - -type deleteObjectsBatcher struct { - sync.Mutex - shard ShardLike - objects objects.BatchSimpleObjects -} - -func newDeleteObjectsBatcher(shard ShardLike) *deleteObjectsBatcher { - return &deleteObjectsBatcher{shard: shard} -} - -func (b *deleteObjectsBatcher) Delete(ctx context.Context, uuids []strfmt.UUID, dryRun bool) objects.BatchSimpleObjects { - b.delete(ctx, uuids, dryRun) - b.flushWALs(ctx) - return b.objects -} - -func (b *deleteObjectsBatcher) delete(ctx context.Context, uuids []strfmt.UUID, dryRun bool) { - b.objects = b.deleteSingleBatchInLSM(ctx, uuids, dryRun) -} - -func (b *deleteObjectsBatcher) deleteSingleBatchInLSM(ctx context.Context, batch []strfmt.UUID, dryRun bool) objects.BatchSimpleObjects { - before := time.Now() - defer b.shard.Metrics().BatchDelete(before, "shard_delete_all") - - result := make(objects.BatchSimpleObjects, len(batch)) - objLock := &sync.Mutex{} - - // if the context is expired fail all - if err := ctx.Err(); err != nil { - for i := range result { - result[i] = objects.BatchSimpleObject{Err: errors.Wrap(err, "begin batch")} - } - return result - } - - wg := &sync.WaitGroup{} - for j, docID := range batch { - wg.Add(1) - go func(index int, uuid strfmt.UUID, dryRun bool) { - defer wg.Done() - // perform delete - obj := b.deleteObjectOfBatchInLSM(ctx, uuid, dryRun) - objLock.Lock() - result[index] = obj - objLock.Unlock() - }(j, docID, dryRun) - } - wg.Wait() - - return result -} - -func (b *deleteObjectsBatcher) deleteObjectOfBatchInLSM(ctx context.Context, uuid strfmt.UUID, dryRun bool) objects.BatchSimpleObject { - before := time.Now() - defer b.shard.Metrics().BatchDelete(before, "shard_delete_individual_total") - if !dryRun { - err := b.shard.batchDeleteObject(ctx, uuid) - return objects.BatchSimpleObject{UUID: uuid, Err: err} - } - - return objects.BatchSimpleObject{UUID: uuid, Err: nil} -} - -func (b *deleteObjectsBatcher) flushWALs(ctx context.Context) { - before := time.Now() - defer b.shard.Metrics().BatchDelete(before, "shard_flush_wals") - - if err := b.shard.Store().WriteWALs(); err != nil { - for i := range b.objects { - b.setErrorAtIndex(err, i) - } - } - - if err := b.shard.VectorIndex().Flush(); err != nil { - for i := range b.objects { - b.setErrorAtIndex(err, i) - } - } - - if err := b.shard.GetPropertyLengthTracker().Flush(false); err != nil { - for i := range b.objects { - b.setErrorAtIndex(err, i) - } - } -} - -func (b *deleteObjectsBatcher) setErrorAtIndex(err error, index int) { - b.Lock() - defer b.Unlock() - b.objects[index].Err = err -} - -func (s *Shard) findDocIDs(ctx context.Context, filters *filters.LocalFilter) ([]uint64, error) { - allowList, err := inverted.NewSearcher(s.index.logger, s.store, s.index.getSchema.GetSchemaSkipAuth(), - nil, s.index.classSearcher, s.index.stopwords, s.versioner.version, s.isFallbackToSearchable, - s.tenant(), s.index.Config.QueryNestedRefLimit). - DocIDs(ctx, filters, additional.Properties{}, s.index.Config.ClassName) - if err != nil { - return nil, err - } - return allowList.Slice(), nil -} - -func (s *Shard) FindUUIDs(ctx context.Context, filters *filters.LocalFilter) ([]strfmt.UUID, error) { - docs, err := s.findDocIDs(ctx, filters) - if err != nil { - return nil, err - } - uuids := make([]strfmt.UUID, len(docs)) - for i, doc := range docs { - uuids[i], err = s.uuidFromDocID(doc) - if err != nil { - return nil, fmt.Errorf("could not get uuid from doc_id=%v", doc) - } - } - return uuids, nil -} diff --git a/adapters/repos/db/shard_write_batch_objects.go b/adapters/repos/db/shard_write_batch_objects.go deleted file mode 100644 index 6f54e95135b40248c07532d588c65fbbc0d8cc4e..0000000000000000000000000000000000000000 --- a/adapters/repos/db/shard_write_batch_objects.go +++ /dev/null @@ -1,459 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "context" - "fmt" - "os" - "runtime/debug" - "sync" - "time" - - "github.com/go-openapi/strfmt" - "github.com/google/uuid" - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/storagestate" - "github.com/weaviate/weaviate/entities/storobj" - "github.com/weaviate/weaviate/usecases/config" -) - -// return value map[int]error gives the error for the index as it received it -func (s *Shard) PutObjectBatch(ctx context.Context, - objects []*storobj.Object, -) []error { - if s.isReadOnly() { - return []error{storagestate.ErrStatusReadOnly} - } - - return s.putBatch(ctx, objects) -} - -// asyncEnabled is a quick and dirty way to create a feature flag for async -// indexing. -func asyncEnabled() bool { - return config.Enabled(os.Getenv("ASYNC_INDEXING")) -} - -// Workers are started with the first batch and keep working as there are objects to add from any batch. Each batch -// adds its jobs (that contain the respective object) to a single queue that is then processed by the workers. -// When the last batch finishes, all workers receive a shutdown signal and exit -func (s *Shard) putBatch(ctx context.Context, - objects []*storobj.Object, -) []error { - if asyncEnabled() { - return s.putBatchAsync(ctx, objects) - } - // Workers are started with the first batch and keep working as there are objects to add from any batch. Each batch - // adds its jobs (that contain the respective object) to a single queue that is then processed by the workers. - // When the last batch finishes, all workers receive a shutdown signal and exit - batcher := newObjectsBatcher(s) - err := batcher.Objects(ctx, objects) - - // block until all objects of batch have been added - batcher.wg.Wait() - s.metrics.VectorIndex(batcher.batchStartTime) - - return err -} - -func (s *Shard) putBatchAsync(ctx context.Context, objects []*storobj.Object) []error { - beforeBatch := time.Now() - defer s.metrics.BatchObject(beforeBatch, len(objects)) - - batcher := newObjectsBatcher(s) - - batcher.init(objects) - batcher.storeInObjectStore(ctx) - batcher.markDeletedInVectorStorage(ctx) - batcher.storeAdditionalStorageWithAsyncQueue(ctx) - batcher.flushWALs(ctx) - - return batcher.errs -} - -// objectsBatcher is a helper type wrapping around an underlying shard that can -// execute objects batch operations on a shard (as opposed to references batch -// operations) -type objectsBatcher struct { - sync.Mutex - shard ShardLike - statuses map[strfmt.UUID]objectInsertStatus - errs []error - duplicates map[int]struct{} - objects []*storobj.Object - wg sync.WaitGroup - batchStartTime time.Time -} - -func newObjectsBatcher(s ShardLike) *objectsBatcher { - return &objectsBatcher{shard: s} -} - -// Objects imports the specified objects in parallel in a batch-fashion -func (ob *objectsBatcher) Objects(ctx context.Context, - objects []*storobj.Object, -) []error { - beforeBatch := time.Now() - defer ob.shard.Metrics().BatchObject(beforeBatch, len(objects)) - - ob.init(objects) - ob.storeInObjectStore(ctx) - ob.markDeletedInVectorStorage(ctx) - ob.storeAdditionalStorageWithWorkers(ctx) - ob.flushWALs(ctx) - return ob.errs -} - -func (ob *objectsBatcher) init(objects []*storobj.Object) { - ob.objects = objects - ob.statuses = map[strfmt.UUID]objectInsertStatus{} - ob.errs = make([]error, len(objects)) - ob.duplicates = findDuplicatesInBatchObjects(objects) -} - -// storeInObjectStore performs all storage operations on the underlying -// key/value store, this is they object-by-id store, the docID-lookup tables, -// as well as all inverted indices. -func (ob *objectsBatcher) storeInObjectStore(ctx context.Context) { - beforeObjectStore := time.Now() - - errs := ob.storeSingleBatchInLSM(ctx, ob.objects) - for i, err := range errs { - if err != nil { - ob.setErrorAtIndex(err, i) - } - } - - ob.shard.Metrics().ObjectStore(beforeObjectStore) -} - -func (ob *objectsBatcher) storeSingleBatchInLSM(ctx context.Context, - batch []*storobj.Object, -) []error { - errs := make([]error, len(batch)) - errLock := &sync.Mutex{} - - // if the context is expired fail all - if err := ctx.Err(); err != nil { - for i := range errs { - errs[i] = errors.Wrap(err, "begin batch") - } - return errs - } - - wg := &sync.WaitGroup{} - concurrencyLimit := make(chan struct{}, _NUMCPU) - - for j, object := range batch { - wg.Add(1) - go func(index int, object *storobj.Object) { - defer wg.Done() - - // Acquire a semaphore to control the concurrency. Otherwise we would - // spawn one routine per object here. With very large batch sizes (e.g. - // 1000 or 10000+), this isn't helpuful and just leads to more lock - // contention down the line – especially when there's lots of text to be - // indexed in the inverted index. - concurrencyLimit <- struct{}{} - defer func() { - // Release the semaphore when the goroutine is done. - <-concurrencyLimit - }() - - if err := ob.storeObjectOfBatchInLSM(ctx, index, object); err != nil { - errLock.Lock() - errs[index] = err - errLock.Unlock() - } - }(j, object) - } - wg.Wait() - - return errs -} - -func (ob *objectsBatcher) storeObjectOfBatchInLSM(ctx context.Context, - objectIndex int, object *storobj.Object, -) error { - if _, ok := ob.duplicates[objectIndex]; ok { - return nil - } - uuidParsed, err := uuid.Parse(object.ID().String()) - if err != nil { - return errors.Wrap(err, "invalid id") - } - - idBytes, err := uuidParsed.MarshalBinary() - if err != nil { - return err - } - - status, err := ob.shard.putObjectLSM(object, idBytes) - if err != nil { - return err - } - - ob.setStatusForID(status, object.ID()) - - if err := ctx.Err(); err != nil { - return errors.Wrapf(err, "end store object %d of batch", objectIndex) - } - return nil -} - -// setStatusForID is thread-safe as it uses the underlying mutex to lock the -// statuses map when writing into it -func (ob *objectsBatcher) setStatusForID(status objectInsertStatus, id strfmt.UUID) { - ob.Lock() - defer ob.Unlock() - ob.statuses[id] = status -} - -func (ob *objectsBatcher) markDeletedInVectorStorage(ctx context.Context) { - var docIDsToDelete []uint64 - var positions []int - for pos, object := range ob.objects { - status := ob.statuses[object.ID()] - if status.docIDChanged { - docIDsToDelete = append(docIDsToDelete, status.oldDocID) - positions = append(positions, pos) - } - } - - if len(docIDsToDelete) == 0 { - return - } - - if err := ob.shard.Queue().Delete(docIDsToDelete...); err != nil { - for _, pos := range positions { - ob.setErrorAtIndex(err, pos) - } - } -} - -// storeAdditionalStorageWithWorkers stores the object in all non-key-value -// stores, such as the main vector index as well as the property-specific -// indices, such as the geo-index. -func (ob *objectsBatcher) storeAdditionalStorageWithWorkers(ctx context.Context) { - if ok := ob.checkContext(ctx); !ok { - // if the context is no longer OK, there's no point in continuing - abort - // early - return - } - - ob.batchStartTime = time.Now() - - for i, object := range ob.objects { - if ob.shouldSkipInAdditionalStorage(i) { - continue - } - - ob.wg.Add(1) - status := ob.statuses[object.ID()] - ob.shard.addJobToQueue(job{ - object: object, - status: status, - index: i, - ctx: ctx, - batcher: ob, - }) - } -} - -func (ob *objectsBatcher) storeAdditionalStorageWithAsyncQueue(ctx context.Context) { - if ok := ob.checkContext(ctx); !ok { - // if the context is no longer OK, there's no point in continuing - abort - // early - return - } - - ob.batchStartTime = time.Now() - - shouldGeoIndex := ob.shard.hasGeoIndex() - vectors := make([]vectorDescriptor, 0, len(ob.objects)) - for i := range ob.objects { - object := ob.objects[i] - if ob.shouldSkipInAdditionalStorage(i) { - continue - } - - status := ob.statuses[object.ID()] - - if shouldGeoIndex { - if err := ob.shard.updatePropertySpecificIndices(object, status); err != nil { - ob.setErrorAtIndex(errors.Wrap(err, "update prop-specific indices"), i) - continue - } - } - - if status.docIDPreserved { - continue - } - - if len(object.Vector) == 0 { - continue - } - - desc := vectorDescriptor{ - id: status.docID, - vector: object.Vector, - } - - vectors = append(vectors, desc) - } - - err := ob.shard.Queue().Push(ctx, vectors...) - if err != nil { - ob.setErrorAtIndex(err, 0) - } -} - -func (ob *objectsBatcher) shouldSkipInAdditionalStorage(i int) bool { - if ok := ob.hasErrorAtIndex(i); ok { - // had an error prior, ignore - return true - } - - // no need to lock the mutex for a duplicate check, as we only ever write - // during init() in there - not concurrently - if _, ok := ob.duplicates[i]; ok { - // is a duplicate, ignore - return true - } - - return false -} - -func (ob *objectsBatcher) storeSingleObjectInAdditionalStorage(ctx context.Context, - object *storobj.Object, status objectInsertStatus, index int, -) { - defer func() { - err := recover() - if err != nil { - ob.setErrorAtIndex(fmt.Errorf("an unexpected error occurred: %s", err), index) - fmt.Fprintf(os.Stderr, "panic: %s\n", err) - debug.PrintStack() - } - }() - - if err := ctx.Err(); err != nil { - ob.setErrorAtIndex(errors.Wrap(err, "insert to vector index"), index) - return - } - - if object.Vector != nil { - // By this time all required deletes (e.g. because of DocID changes) have - // already been grouped and performed in bulk. Only the insertions are - // left. The motivation for this change is explained in - // https://github.com/weaviate/weaviate/pull/2697. - // - // Before this change, two identical batches in sequence would lead to - // massive lock contention in the hnsw index, as each individual delete - // requires a costly RW.Lock() operation which first drains all "readers" - // which represent the regular imports. See "deleteVsInsertLock" inside the - // hnsw store. - // - // With the improved logic, we group all batches up front in a single call, - // so this highly concurrent method no longer needs to compete for those - // expensive locks. - // - // Since this behavior is exclusive to batching, we can no longer call - // shard.updateVectorIndex which would also handle the delete as required - // for a non-batch update. Instead a new method has been introduced that - // ignores deletes. - if err := ob.shard.updateVectorIndexIgnoreDelete(object.Vector, status); err != nil { - ob.setErrorAtIndex(errors.Wrap(err, "insert to vector index"), index) - return - } - } - - if err := ob.shard.updatePropertySpecificIndices(object, status); err != nil { - ob.setErrorAtIndex(errors.Wrap(err, "update prop-specific indices"), index) - return - } -} - -// hasErrorAtIndex is thread-safe as it uses the underlying mutex to lock -// before reading from the errs map -func (ob *objectsBatcher) hasErrorAtIndex(i int) bool { - ob.Lock() - defer ob.Unlock() - return ob.errs[i] != nil -} - -// setErrorAtIndex is thread-safe as it uses the underlying mutex to lock -// writing into the errs map -func (ob *objectsBatcher) setErrorAtIndex(err error, index int) { - ob.Lock() - defer ob.Unlock() - ob.errs[index] = err -} - -// checkContext does nothing if the context is still active. But if the context -// has error'd, it marks all objects which have not previously error'd yet with -// the ctx error -func (ob *objectsBatcher) checkContext(ctx context.Context) bool { - if err := ctx.Err(); err != nil { - for i, err := range ob.errs { - if err == nil { - // already has an error, ignore - continue - } - - ob.errs[i] = errors.Wrapf(err, - "inverted indexing complete, about to start vector indexing") - } - - return false - } - - return true -} - -func (ob *objectsBatcher) flushWALs(ctx context.Context) { - if err := ob.shard.Store().WriteWALs(); err != nil { - for i := range ob.objects { - ob.setErrorAtIndex(err, i) - } - } - - if err := ob.shard.VectorIndex().Flush(); err != nil { - for i := range ob.objects { - ob.setErrorAtIndex(err, i) - } - } - - if err := ob.shard.GetPropertyLengthTracker().Flush(false); err != nil { - for i := range ob.objects { - ob.setErrorAtIndex(err, i) - } - } -} - -// returns the originalIndexIDs to be ignored -func findDuplicatesInBatchObjects(in []*storobj.Object) map[int]struct{} { - count := map[strfmt.UUID]int{} - for _, obj := range in { - count[obj.ID()] = count[obj.ID()] + 1 - } - - ignore := map[int]struct{}{} - for i, obj := range in { - if c := count[obj.ID()]; c > 1 { - count[obj.ID()] = c - 1 - ignore[i] = struct{}{} - } - } - - return ignore -} diff --git a/adapters/repos/db/shard_write_batch_references.go b/adapters/repos/db/shard_write_batch_references.go deleted file mode 100644 index 70a878672844c9d791d6ff0f3894e0a57a87c5b8..0000000000000000000000000000000000000000 --- a/adapters/repos/db/shard_write_batch_references.go +++ /dev/null @@ -1,347 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "context" - "fmt" - "sync" - "time" - - "github.com/google/uuid" - "github.com/pkg/errors" - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/adapters/repos/db/inverted" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/storobj" - "github.com/weaviate/weaviate/usecases/objects" -) - -// return value map[int]error gives the error for the index as it received it -func (s *Shard) AddReferencesBatch(ctx context.Context, refs objects.BatchReferences) []error { - if s.isReadOnly() { - return []error{errors.Errorf("shard is read-only")} - } - - return newReferencesBatcher(s).References(ctx, refs) -} - -// referencesBatcher is a helper type wrapping around an underlying shard that can -// execute references batch operations on a shard (as opposed to object batch -// operations) -type referencesBatcher struct { - sync.Mutex - shard ShardLike - errs []error - refs objects.BatchReferences -} - -func newReferencesBatcher(s ShardLike) *referencesBatcher { - return &referencesBatcher{ - shard: s, - } -} - -func (b *referencesBatcher) References(ctx context.Context, - refs objects.BatchReferences, -) []error { - b.init(refs) - b.storeInObjectStore(ctx) - b.flushWALs(ctx) - return b.errs -} - -func (b *referencesBatcher) init(refs objects.BatchReferences) { - b.refs = refs - b.errs = make([]error, len(refs)) -} - -func (b *referencesBatcher) storeInObjectStore( - ctx context.Context, -) { - errs := b.storeSingleBatchInLSM(ctx, b.refs) - for i, err := range errs { - if err != nil { - b.setErrorAtIndex(err, i) - } - } - - // adding references can not alter the vector position, so no need to alter - // the vector index -} - -func (b *referencesBatcher) storeSingleBatchInLSM(ctx context.Context, batch objects.BatchReferences) []error { - errs := make([]error, len(batch)) - errLock := &sync.Mutex{} - - // if the context is expired fail all - if err := ctx.Err(); err != nil { - for i := range errs { - errs[i] = errors.Wrap(err, "begin batch") - } - return errs - } - - invertedMerger := inverted.NewDeltaMerger() - propsByName, err := b.getSchemaPropsByName() - if err != nil { - for i := range errs { - errs[i] = errors.Wrap(err, "getting schema properties") - } - return errs - } - - // TODO: is there any benefit in having this parallelized? if so, don't forget to lock before assigning errors - // If we want them to run in parallel we need to look individual objects, - // otherwise we have a race inside the merge functions - // wg := &sync.WaitGroup{} - for i, ref := range batch { - // wg.Add(1) - // go func(index int, reference objects.BatchReference) { - // defer wg.Done() - uuidParsed, err := uuid.Parse(ref.From.TargetID.String()) - if err != nil { - errLock.Lock() - errs[i] = errors.Wrap(err, "invalid id") - errLock.Unlock() - continue - } - - idBytes, err := uuidParsed.MarshalBinary() - if err != nil { - errLock.Lock() - errs[i] = err - errLock.Unlock() - continue - } - - mergeDoc := mergeDocFromBatchReference(ref) - res, err := b.shard.mutableMergeObjectLSM(mergeDoc, idBytes) - if err != nil { - errLock.Lock() - errs[i] = err - errLock.Unlock() - continue - } - - prop, ok := propsByName[ref.From.Property.String()] - if !ok { - errLock.Lock() - errs[i] = fmt.Errorf("property '%s' not found in schema", ref.From.Property) - errLock.Unlock() - continue - } - - // generally the batch ref is an append only change which does not alter - // the vector position. There is however one inverted index link that needs - // to be cleanup: the ref count - if err := b.analyzeInverted(invertedMerger, res, ref, prop); err != nil { - errLock.Lock() - errs[i] = err - errLock.Unlock() - continue - } - } - - if err := b.writeInverted(invertedMerger.Merge()); err != nil { - for i := range errs { - errs[i] = errors.Wrap(err, "write inverted batch") - } - return errs - } - - return errs -} - -func (b *referencesBatcher) analyzeInverted(invertedMerger *inverted.DeltaMerger, mergeResult mutableMergeResult, ref objects.BatchReference, prop *models.Property) error { - prevProps, err := b.analyzeRef(mergeResult.previous, ref, prop) - if err != nil { - return err - } - - nextProps, err := b.analyzeRef(mergeResult.next, ref, prop) - if err != nil { - return err - } - - delta := inverted.Delta(prevProps, nextProps) - invertedMerger.AddAdditions(delta.ToAdd, mergeResult.status.docID) - invertedMerger.AddDeletions(delta.ToDelete, mergeResult.status.docID) - - return nil -} - -func (b *referencesBatcher) writeInverted(in inverted.DeltaMergeResult) error { - before := time.Now() - if err := b.writeInvertedAdditions(in.Additions); err != nil { - return errors.Wrap(err, "write additions") - } - b.shard.Metrics().InvertedExtend(before, len(in.Additions)) - - before = time.Now() - if err := b.writeInvertedDeletions(in.Deletions); err != nil { - return errors.Wrap(err, "write deletions") - } - b.shard.Metrics().InvertedDeleteDelta(before) - - return nil -} - -// TODO text_rbm_inverted_index unify bucket write -func (b *referencesBatcher) writeInvertedDeletions(in []inverted.MergeProperty) error { - for _, prop := range in { - // in the references batcher we can only ever write ref count entire which - // are guaranteed to be not have a frequency, meaning they will use the - // "Set" strategy in the lsmkv store - if prop.HasFilterableIndex { - bucket := b.shard.Store().Bucket(helpers.BucketFromPropNameLSM(prop.Name)) - if bucket == nil { - return errors.Errorf("no bucket for prop '%s' found", prop.Name) - } - - for _, item := range prop.MergeItems { - for _, id := range item.DocIDs { - err := b.shard.deleteFromPropertySetBucket(bucket, id.DocID, item.Data) - if err != nil { - return err - } - } - } - } - } - - return nil -} - -// TODO text_rbm_inverted_index unify bucket write -func (b *referencesBatcher) writeInvertedAdditions(in []inverted.MergeProperty) error { - for _, prop := range in { - // in the references batcher we can only ever write ref count entire which - // are guaranteed to be not have a frequency, meaning they will use the - // "Set" strategy in the lsmkv store - if prop.HasFilterableIndex { - bucket := b.shard.Store().Bucket(helpers.BucketFromPropNameLSM(prop.Name)) - if bucket == nil { - return errors.Errorf("no bucket for prop '%s' found", prop.Name) - } - - for _, item := range prop.MergeItems { - err := b.shard.batchExtendInvertedIndexItemsLSMNoFrequency(bucket, item) - if err != nil { - return err - } - } - } - } - - return nil -} - -func (b *referencesBatcher) analyzeRef(obj *storobj.Object, ref objects.BatchReference, prop *models.Property) ([]inverted.Property, error) { - if prop == nil { - return nil, fmt.Errorf("analyzeRef: property %q not found in schema", ref.From.Property) - } - - props := obj.Properties() - if props == nil { - return nil, nil - } - - propMap, ok := props.(map[string]interface{}) - if !ok { - return nil, nil - } - - var refs models.MultipleRef - refProp, ok := propMap[ref.From.Property.String()] - if !ok { - refs = make(models.MultipleRef, 0) // explicitly mark as length zero - } else { - parsed, ok := refProp.(models.MultipleRef) - if !ok { - return nil, errors.Errorf("prop %s is present, but not a ref, got: %T", - ref.From.Property.String(), refProp) - } - refs = parsed - } - - a := inverted.NewAnalyzer(nil) - - countItems, err := a.RefCount(refs) - if err != nil { - return nil, err - } - - valueItems, err := a.Ref(refs) - if err != nil { - return nil, err - } - - return []inverted.Property{{ - Name: helpers.MetaCountProp(ref.From.Property.String()), - Items: countItems, - HasFilterableIndex: inverted.HasFilterableIndexMetaCount && inverted.HasInvertedIndex(prop), - HasSearchableIndex: inverted.HasSearchableIndexMetaCount && inverted.HasInvertedIndex(prop), - }, { - Name: ref.From.Property.String(), - Items: valueItems, - HasFilterableIndex: inverted.HasFilterableIndex(prop), - HasSearchableIndex: inverted.HasSearchableIndex(prop), - }}, nil -} - -func (b *referencesBatcher) setErrorAtIndex(err error, i int) { - b.Lock() - defer b.Unlock() - - err = errors.Wrap(err, "ref batch") - b.errs[i] = err -} - -func mergeDocFromBatchReference(ref objects.BatchReference) objects.MergeDocument { - return objects.MergeDocument{ - Class: ref.From.Class.String(), - ID: ref.From.TargetID, - UpdateTime: time.Now().UnixMilli(), - References: objects.BatchReferences{ref}, - } -} - -func (b *referencesBatcher) flushWALs(ctx context.Context) { - if err := b.shard.Store().WriteWALs(); err != nil { - for i := range b.refs { - b.setErrorAtIndex(err, i) - } - } - - if err := b.shard.VectorIndex().Flush(); err != nil { - for i := range b.refs { - b.setErrorAtIndex(err, i) - } - } -} - -func (b *referencesBatcher) getSchemaPropsByName() (map[string]*models.Property, error) { - idx := b.shard.Index() - sch := idx.getSchema.GetSchemaSkipAuth().Objects - class, err := schema.GetClassByName(sch, idx.Config.ClassName.String()) - if err != nil { - return nil, err - } - - propsByName := map[string]*models.Property{} - for _, prop := range class.Properties { - propsByName[prop.Name] = prop - } - return propsByName, nil -} diff --git a/adapters/repos/db/shard_write_delete.go b/adapters/repos/db/shard_write_delete.go deleted file mode 100644 index c5d474fc2e508390274b2807f81ba80008e43e51..0000000000000000000000000000000000000000 --- a/adapters/repos/db/shard_write_delete.go +++ /dev/null @@ -1,165 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "context" - "fmt" - - "github.com/go-openapi/strfmt" - "github.com/google/uuid" - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv" - "github.com/weaviate/weaviate/entities/storagestate" - "github.com/weaviate/weaviate/entities/storobj" -) - -func (s *Shard) DeleteObject(ctx context.Context, id strfmt.UUID) error { - if s.isReadOnly() { - return storagestate.ErrStatusReadOnly - } - - idBytes, err := uuid.MustParse(id.String()).MarshalBinary() - if err != nil { - return err - } - - var docID uint64 - bucket := s.store.Bucket(helpers.ObjectsBucketLSM) - existing, err := bucket.Get([]byte(idBytes)) - if err != nil { - return fmt.Errorf("unexpected error on previous lookup: %w", err) - } - - if existing == nil { - // nothing to do - return nil - } - - // we need the doc ID so we can clean up inverted indices currently - // pointing to this object - docID, err = storobj.DocIDFromBinary(existing) - if err != nil { - return fmt.Errorf("get existing doc id from object binary: %w", err) - } - - err = bucket.Delete(idBytes) - if err != nil { - return fmt.Errorf("delete object from bucket: %w", err) - } - - err = s.cleanupInvertedIndexOnDelete(existing, docID) - if err != nil { - return fmt.Errorf("delete object from bucket: %w", err) - } - - if err = s.queue.Delete(docID); err != nil { - return fmt.Errorf("delete from vector index: %w", err) - } - - if err = s.store.WriteWALs(); err != nil { - return fmt.Errorf("flush all buffered WALs: %w", err) - } - - if err = s.VectorIndex().Flush(); err != nil { - return fmt.Errorf("flush all vector index buffered WALs: %w", err) - } - - if err = s.ChangeObjectCountBy(-1); err != nil { - return fmt.Errorf("subtract prop lengths: %w", err) - } - - return nil -} - -func (s *Shard) canDeleteOne(ctx context.Context, id strfmt.UUID) (bucket *lsmkv.Bucket, obj, uid []byte, docID uint64, err error) { - if uid, err = parseBytesUUID(id); err != nil { - return nil, nil, uid, 0, err - } - - bucket = s.store.Bucket(helpers.ObjectsBucketLSM) - existing, err := bucket.Get(uid) - if err != nil { - return nil, nil, uid, 0, fmt.Errorf("get previous object: %w", err) - } - - if existing == nil { - return bucket, nil, uid, 0, nil - } - - // we need the doc ID so we can clean up inverted indices currently - // pointing to this object - docID, err = storobj.DocIDFromBinary(existing) - if err != nil { - return bucket, nil, uid, 0, fmt.Errorf("get existing doc id from object binary: %w", err) - } - return bucket, existing, uid, docID, nil -} - -func (s *Shard) deleteOne(ctx context.Context, bucket *lsmkv.Bucket, obj, idBytes []byte, docID uint64) error { - if obj == nil || bucket == nil { - return nil - } - err := bucket.Delete(idBytes) - if err != nil { - return fmt.Errorf("delete object from bucket: %w", err) - } - - err = s.cleanupInvertedIndexOnDelete(obj, docID) - if err != nil { - return fmt.Errorf("delete object from bucket: %w", err) - } - - if err = s.queue.Delete(docID); err != nil { - return fmt.Errorf("delete from vector index: %w", err) - } - - if err = s.store.WriteWALs(); err != nil { - return fmt.Errorf("flush all buffered WALs: %w", err) - } - - if err = s.VectorIndex().Flush(); err != nil { - return fmt.Errorf("flush all vector index buffered WALs: %w", err) - } - - return nil -} - -func (s *Shard) cleanupInvertedIndexOnDelete(previous []byte, docID uint64) error { - previousObject, err := storobj.FromBinary(previous) - if err != nil { - return fmt.Errorf("unmarshal previous object: %w", err) - } - - previousProps, previousNilProps, err := s.AnalyzeObject(previousObject) - if err != nil { - return fmt.Errorf("analyze previous object: %w", err) - } - - if err = s.subtractPropLengths(previousProps); err != nil { - return fmt.Errorf("subtract prop lengths: %w", err) - } - - err = s.deleteFromInvertedIndicesLSM(previousProps, previousNilProps, docID) - if err != nil { - return fmt.Errorf("put inverted indices props: %w", err) - } - - if s.index.Config.TrackVectorDimensions { - err = s.removeDimensionsLSM(len(previousObject.Vector), docID) - if err != nil { - return fmt.Errorf("track dimensions (delete): %w", err) - } - } - - return nil -} diff --git a/adapters/repos/db/shard_write_inverted.go b/adapters/repos/db/shard_write_inverted.go deleted file mode 100644 index b40659d95460edd794d44efc679811e998b2b025..0000000000000000000000000000000000000000 --- a/adapters/repos/db/shard_write_inverted.go +++ /dev/null @@ -1,85 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "fmt" - - "github.com/weaviate/weaviate/adapters/repos/db/inverted" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/storobj" -) - -func isPropertyForLength(dt schema.DataType) bool { - switch dt { - case schema.DataTypeInt, schema.DataTypeNumber, schema.DataTypeBoolean, schema.DataTypeDate: - return false - default: - return true - } -} - -func (s *Shard) AnalyzeObject(object *storobj.Object) ([]inverted.Property, []inverted.NilProperty, error) { - schemaModel := s.index.getSchema.GetSchemaSkipAuth().Objects - c, err := schema.GetClassByName(schemaModel, object.Class().String()) - if err != nil { - return nil, nil, err - } - - var schemaMap map[string]interface{} - - if object.Properties() == nil { - schemaMap = make(map[string]interface{}) - } else { - maybeSchemaMap, ok := object.Properties().(map[string]interface{}) - if !ok { - return nil, nil, fmt.Errorf("expected schema to be map, but got %T", object.Properties()) - } - schemaMap = maybeSchemaMap - } - - // add nil for all properties that are not part of the object so that they can be added to the inverted index for - // the null state (if enabled) - var nilProps []inverted.NilProperty - if s.index.invertedIndexConfig.IndexNullState { - for _, prop := range c.Properties { - dt := schema.DataType(prop.DataType[0]) - // some datatypes are not added to the inverted index, so we can skip them here - if dt == schema.DataTypeGeoCoordinates || dt == schema.DataTypePhoneNumber || dt == schema.DataTypeBlob { - continue - } - - // Add props as nil props if - // 1. They are not in the schema map ( == nil) - // 2. Their inverted index is enabled - _, ok := schemaMap[prop.Name] - if !ok && inverted.HasInvertedIndex(prop) { - nilProps = append(nilProps, inverted.NilProperty{ - Name: prop.Name, - AddToPropertyLength: isPropertyForLength(dt), - }) - } - } - } - - if s.index.invertedIndexConfig.IndexTimestamps { - if schemaMap == nil { - schemaMap = make(map[string]interface{}) - } - schemaMap[filters.InternalPropCreationTimeUnix] = object.Object.CreationTimeUnix - schemaMap[filters.InternalPropLastUpdateTimeUnix] = object.Object.LastUpdateTimeUnix - } - - props, err := inverted.NewAnalyzer(s.isFallbackToSearchable).Object(schemaMap, c.Properties, object.ID()) - return props, nilProps, err -} diff --git a/adapters/repos/db/shard_write_inverted_lsm.go b/adapters/repos/db/shard_write_inverted_lsm.go deleted file mode 100644 index 329780c521242a588d0df23584f83202fa2f3e3e..0000000000000000000000000000000000000000 --- a/adapters/repos/db/shard_write_inverted_lsm.go +++ /dev/null @@ -1,295 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "encoding/binary" - "math" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/adapters/repos/db/inverted" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv" -) - -func (s *Shard) extendInvertedIndicesLSM(props []inverted.Property, nilProps []inverted.NilProperty, - docID uint64, -) error { - for _, prop := range props { - if err := s.addToPropertyValueIndex(docID, prop); err != nil { - return err - } - - // add non-nil properties to the null-state inverted index, but skip internal properties (__meta_count, _id etc) - if isMetaCountProperty(prop) || isInternalProperty(prop) { - continue - } - - // properties where defining a length does not make sense (floats etc.) have a negative entry as length - if s.index.invertedIndexConfig.IndexPropertyLength && prop.Length >= 0 { - if err := s.addToPropertyLengthIndex(prop.Name, docID, prop.Length); err != nil { - return errors.Wrap(err, "add indexed property length") - } - } - - if s.index.invertedIndexConfig.IndexNullState { - if err := s.addToPropertyNullIndex(prop.Name, docID, prop.Length == 0); err != nil { - return errors.Wrap(err, "add indexed null state") - } - } - } - - // add nil properties to the nullstate and property length inverted index - for _, nilProperty := range nilProps { - if s.index.invertedIndexConfig.IndexPropertyLength && nilProperty.AddToPropertyLength { - if err := s.addToPropertyLengthIndex(nilProperty.Name, docID, 0); err != nil { - return errors.Wrap(err, "add indexed property length") - } - } - - if s.index.invertedIndexConfig.IndexNullState { - if err := s.addToPropertyNullIndex(nilProperty.Name, docID, true); err != nil { - return errors.Wrap(err, "add indexed null state") - } - } - } - - return nil -} - -func (s *Shard) addToPropertyValueIndex(docID uint64, property inverted.Property) error { - if property.HasFilterableIndex { - bucketValue := s.store.Bucket(helpers.BucketFromPropNameLSM(property.Name)) - if bucketValue == nil { - return errors.Errorf("no bucket for prop '%s' found", property.Name) - } - - for _, item := range property.Items { - key := item.Data - if err := s.addToPropertySetBucket(bucketValue, docID, key); err != nil { - return errors.Wrapf(err, "failed adding to prop '%s' value bucket", property.Name) - } - } - } - - if property.HasSearchableIndex { - bucketValue := s.store.Bucket(helpers.BucketSearchableFromPropNameLSM(property.Name)) - if bucketValue == nil { - return errors.Errorf("no bucket searchable for prop '%s' found", property.Name) - } - - propLen := float32(len(property.Items)) - for _, item := range property.Items { - key := item.Data - pair := s.pairPropertyWithFrequency(docID, item.TermFrequency, propLen) - if err := s.addToPropertyMapBucket(bucketValue, pair, key); err != nil { - return errors.Wrapf(err, "failed adding to prop '%s' value bucket", property.Name) - } - } - } - - return nil -} - -func (s *Shard) addToPropertyLengthIndex(propName string, docID uint64, length int) error { - bucketLength := s.store.Bucket(helpers.BucketFromPropNameLengthLSM(propName)) - if bucketLength == nil { - return errors.Errorf("no bucket for prop '%s' length found", propName) - } - - key, err := bucketKeyPropertyLength(length) - if err != nil { - return errors.Wrapf(err, "failed creating key for prop '%s' length", propName) - } - if err := s.addToPropertySetBucket(bucketLength, docID, key); err != nil { - return errors.Wrapf(err, "failed adding to prop '%s' length bucket", propName) - } - return nil -} - -func (s *Shard) addToPropertyNullIndex(propName string, docID uint64, isNull bool) error { - bucketNull := s.store.Bucket(helpers.BucketFromPropNameNullLSM(propName)) - if bucketNull == nil { - return errors.Errorf("no bucket for prop '%s' null found", propName) - } - - key, err := bucketKeyPropertyNull(isNull) - if err != nil { - return errors.Wrapf(err, "failed creating key for prop '%s' null", propName) - } - if err := s.addToPropertySetBucket(bucketNull, docID, key); err != nil { - return errors.Wrapf(err, "failed adding to prop '%s' null bucket", propName) - } - return nil -} - -func (s *Shard) pairPropertyWithFrequency(docID uint64, freq, propLen float32) lsmkv.MapPair { - // 8 bytes for doc id, 4 bytes for frequency, 4 bytes for prop term length - buf := make([]byte, 16) - - // Shard Index version 2 requires BigEndian for sorting, if the shard was - // built prior assume it uses LittleEndian - if s.versioner.Version() < 2 { - binary.LittleEndian.PutUint64(buf[0:8], docID) - } else { - binary.BigEndian.PutUint64(buf[0:8], docID) - } - binary.LittleEndian.PutUint32(buf[8:12], math.Float32bits(freq)) - binary.LittleEndian.PutUint32(buf[12:16], math.Float32bits(propLen)) - - return lsmkv.MapPair{ - Key: buf[:8], - Value: buf[8:], - } -} - -func (s *Shard) addToPropertyMapBucket(bucket *lsmkv.Bucket, pair lsmkv.MapPair, key []byte) error { - lsmkv.CheckExpectedStrategy(bucket.Strategy(), lsmkv.StrategyMapCollection) - - return bucket.MapSet(key, pair) -} - -func (s *Shard) addToPropertySetBucket(bucket *lsmkv.Bucket, docID uint64, key []byte) error { - lsmkv.CheckExpectedStrategy(bucket.Strategy(), lsmkv.StrategySetCollection, lsmkv.StrategyRoaringSet) - - if bucket.Strategy() == lsmkv.StrategySetCollection { - docIDBytes := make([]byte, 8) - binary.LittleEndian.PutUint64(docIDBytes, docID) - - return bucket.SetAdd(key, [][]byte{docIDBytes}) - } - - return bucket.RoaringSetAddOne(key, docID) -} - -func (s *Shard) batchExtendInvertedIndexItemsLSMNoFrequency(b *lsmkv.Bucket, - item inverted.MergeItem, -) error { - if b.Strategy() != lsmkv.StrategySetCollection && b.Strategy() != lsmkv.StrategyRoaringSet { - panic("prop has no frequency, but bucket does not have 'Set' nor 'RoaringSet' strategy") - } - - if b.Strategy() == lsmkv.StrategyRoaringSet { - docIDs := make([]uint64, len(item.DocIDs)) - for i, idTuple := range item.DocIDs { - docIDs[i] = idTuple.DocID - } - return b.RoaringSetAddList(item.Data, docIDs) - } - - docIDs := make([][]byte, len(item.DocIDs)) - for i, idTuple := range item.DocIDs { - docIDs[i] = make([]byte, 8) - binary.LittleEndian.PutUint64(docIDs[i], idTuple.DocID) - } - - return b.SetAdd(item.Data, docIDs) -} - -func (s *Shard) SetPropertyLengths(props []inverted.Property) error { - for _, prop := range props { - if !prop.HasSearchableIndex { - continue - } - - if err := s.GetPropertyLengthTracker().TrackProperty(prop.Name, float32(len(prop.Items))); err != nil { - return err - } - - } - - s.GetPropertyLengthTracker().Flush(false) - return nil -} - -func (s *Shard) ChangeObjectCountBy(count int) error { - if err := s.GetPropertyLengthTracker().TrackObjects(count); err != nil { - return err - } - s.GetPropertyLengthTracker().Flush(false) - return nil -} - -func (s *Shard) subtractPropLengths(props []inverted.Property) error { - for _, prop := range props { - if !prop.HasSearchableIndex { - continue - } - - if err := s.GetPropertyLengthTracker().UnTrackProperty(prop.Name, float32(len(prop.Items))); err != nil { - return err - } - - } - - s.GetPropertyLengthTracker().Flush(false) - return nil -} - -func (s *Shard) extendDimensionTrackerLSM( - count int, docID uint64, -) error { - b := s.store.Bucket(helpers.DimensionsBucketLSM) - if b == nil { - return errors.Errorf("no bucket dimensions") - } - - // 4 bytes for dim count (row key), 8 bytes for doc id (map key), 0 bytes for - // map value - buf := make([]byte, 12) - - binary.LittleEndian.PutUint32(buf[0:4], uint32(count)) - binary.LittleEndian.PutUint64(buf[4:12], docID) - - pair := lsmkv.MapPair{ - Key: buf[4:12], - Value: buf[12:12], - } - - return b.MapSet(buf[0:4], pair) -} - -// Key (dimensionality) | Value Doc IDs -// 128 | 1,2,4,5,17 -// 128 | 1,2,4,5,17, Tombstone 4, - -func (s *Shard) removeDimensionsLSM( - count int, docID uint64, -) error { - b := s.store.Bucket(helpers.DimensionsBucketLSM) - if b == nil { - return errors.Errorf("no bucket dimensions") - } - - // 4 bytes for dim count (row key), 8 bytes for doc id (map key), 0 bytes for - // map value - buf := make([]byte, 12) - - binary.LittleEndian.PutUint32(buf[0:4], uint32(count)) - binary.LittleEndian.PutUint64(buf[4:12], docID) - - pair := lsmkv.MapPair{ - Key: buf[4:12], - Value: buf[12:12], - Tombstone: true, - } - - return b.MapSet(buf[0:4], pair) -} - -func isMetaCountProperty(property inverted.Property) bool { - return len(property.Name) > 12 && property.Name[len(property.Name)-12:] == "__meta_count" -} - -func isInternalProperty(property inverted.Property) bool { - return property.Name[0] == '_' -} diff --git a/adapters/repos/db/shard_write_inverted_lsm_delete.go b/adapters/repos/db/shard_write_inverted_lsm_delete.go deleted file mode 100644 index a0588285c46e60fbc168644c6255df7103ce1530..0000000000000000000000000000000000000000 --- a/adapters/repos/db/shard_write_inverted_lsm_delete.go +++ /dev/null @@ -1,154 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "encoding/binary" - "fmt" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/adapters/repos/db/inverted" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv" -) - -func (s *Shard) deleteFromInvertedIndicesLSM(props []inverted.Property, nilProps []inverted.NilProperty, - docID uint64, -) error { - for _, prop := range props { - if prop.HasFilterableIndex { - bucket := s.store.Bucket(helpers.BucketFromPropNameLSM(prop.Name)) - if bucket == nil { - return fmt.Errorf("no bucket for prop '%s' found", prop.Name) - } - - for _, item := range prop.Items { - if err := s.deleteFromPropertySetBucket(bucket, docID, item.Data); err != nil { - return errors.Wrapf(err, "delete item '%s' from index", - string(item.Data)) - } - } - } - - if prop.HasSearchableIndex { - bucket := s.store.Bucket(helpers.BucketSearchableFromPropNameLSM(prop.Name)) - if bucket == nil { - return fmt.Errorf("no bucket searchable for prop '%s' found", prop.Name) - } - - for _, item := range prop.Items { - if err := s.deleteInvertedIndexItemWithFrequencyLSM(bucket, item, - docID); err != nil { - return errors.Wrapf(err, "delete item '%s' from index", - string(item.Data)) - } - } - } - - // add non-nil properties to the null-state inverted index, but skip internal properties (__meta_count, _id etc) - if isMetaCountProperty(prop) || isInternalProperty(prop) { - continue - } - - // properties where defining a length does not make sense (floats etc.) have a negative entry as length - if s.index.invertedIndexConfig.IndexPropertyLength && prop.Length >= 0 { - if err := s.deleteFromPropertyLengthIndex(prop.Name, docID, prop.Length); err != nil { - return errors.Wrap(err, "add indexed property length") - } - } - - if s.index.invertedIndexConfig.IndexNullState { - if err := s.deleteFromPropertyNullIndex(prop.Name, docID, prop.Length == 0); err != nil { - return errors.Wrap(err, "add indexed null state") - } - } - } - - // remove nil properties from the nullstate and property length inverted index - for _, nilProperty := range nilProps { - if s.index.invertedIndexConfig.IndexPropertyLength && nilProperty.AddToPropertyLength { - if err := s.deleteFromPropertyLengthIndex(nilProperty.Name, docID, 0); err != nil { - return errors.Wrap(err, "add indexed property length") - } - } - - if s.index.invertedIndexConfig.IndexNullState { - if err := s.deleteFromPropertyNullIndex(nilProperty.Name, docID, true); err != nil { - return errors.Wrap(err, "add indexed null state") - } - } - } - - return nil -} - -func (s *Shard) deleteInvertedIndexItemWithFrequencyLSM(bucket *lsmkv.Bucket, - item inverted.Countable, docID uint64, -) error { - lsmkv.CheckExpectedStrategy(bucket.Strategy(), lsmkv.StrategyMapCollection) - - docIDBytes := make([]byte, 8) - // Shard Index version 2 requires BigEndian for sorting, if the shard was - // built prior assume it uses LittleEndian - if s.versioner.Version() < 2 { - binary.LittleEndian.PutUint64(docIDBytes, docID) - } else { - binary.BigEndian.PutUint64(docIDBytes, docID) - } - - return bucket.MapDeleteKey(item.Data, docIDBytes) -} - -func (s *Shard) deleteFromPropertyLengthIndex(propName string, docID uint64, length int) error { - bucketLength := s.store.Bucket(helpers.BucketFromPropNameLengthLSM(propName)) - if bucketLength == nil { - return errors.Errorf("no bucket for prop '%s' length found", propName) - } - - key, err := bucketKeyPropertyLength(length) - if err != nil { - return errors.Wrapf(err, "failed creating key for prop '%s' length", propName) - } - if err := s.deleteFromPropertySetBucket(bucketLength, docID, key); err != nil { - return errors.Wrapf(err, "failed adding to prop '%s' length bucket", propName) - } - return nil -} - -func (s *Shard) deleteFromPropertyNullIndex(propName string, docID uint64, isNull bool) error { - bucketNull := s.store.Bucket(helpers.BucketFromPropNameNullLSM(propName)) - if bucketNull == nil { - return errors.Errorf("no bucket for prop '%s' null found", propName) - } - - key, err := bucketKeyPropertyNull(isNull) - if err != nil { - return errors.Wrapf(err, "failed creating key for prop '%s' null", propName) - } - if err := s.deleteFromPropertySetBucket(bucketNull, docID, key); err != nil { - return errors.Wrapf(err, "failed adding to prop '%s' null bucket", propName) - } - return nil -} - -func (s *Shard) deleteFromPropertySetBucket(bucket *lsmkv.Bucket, docID uint64, key []byte) error { - lsmkv.CheckExpectedStrategy(bucket.Strategy(), lsmkv.StrategySetCollection, lsmkv.StrategyRoaringSet) - - if bucket.Strategy() == lsmkv.StrategySetCollection { - docIDBytes := make([]byte, 8) - binary.LittleEndian.PutUint64(docIDBytes, docID) - - return bucket.SetDeleteSingle(key, docIDBytes) - } - - return bucket.RoaringSetRemoveOne(key, docID) -} diff --git a/adapters/repos/db/shard_write_merge.go b/adapters/repos/db/shard_write_merge.go deleted file mode 100644 index bac3ca49fcd01ec9e9f78f01431d066860e4314c..0000000000000000000000000000000000000000 --- a/adapters/repos/db/shard_write_merge.go +++ /dev/null @@ -1,247 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "context" - - "github.com/google/uuid" - "github.com/pkg/errors" - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/storagestate" - "github.com/weaviate/weaviate/entities/storobj" - "github.com/weaviate/weaviate/usecases/objects" -) - -func (s *Shard) MergeObject(ctx context.Context, merge objects.MergeDocument) error { - if s.isReadOnly() { - return storagestate.ErrStatusReadOnly - } - - if merge.Vector != nil { - // validation needs to happen before any changes are done. Otherwise, insertion is aborted somewhere in-between. - err := s.VectorIndex().ValidateBeforeInsert(merge.Vector) - if err != nil { - return errors.Wrapf(err, "Validate vector index for update of %v", merge.ID) - } - } - - idBytes, err := uuid.MustParse(merge.ID.String()).MarshalBinary() - if err != nil { - return err - } - - return s.merge(ctx, idBytes, merge) -} - -func (s *Shard) merge(ctx context.Context, idBytes []byte, doc objects.MergeDocument) error { - next, status, err := s.mergeObjectInStorage(doc, idBytes) - if err != nil { - return err - } - - if err := s.updateVectorIndex(next.Vector, status); err != nil { - return errors.Wrap(err, "update vector index") - } - - if err := s.updatePropertySpecificIndices(next, status); err != nil { - return errors.Wrap(err, "update property-specific indices") - } - - if err := s.store.WriteWALs(); err != nil { - return errors.Wrap(err, "flush all buffered WALs") - } - - return nil -} - -func (s *Shard) mergeObjectInStorage(merge objects.MergeDocument, - idBytes []byte, -) (*storobj.Object, objectInsertStatus, error) { - bucket := s.store.Bucket(helpers.ObjectsBucketLSM) - - // see comment in shard_write_put.go::putObjectLSM - lock := &s.docIdLock[s.uuidToIdLockPoolId(idBytes)] - lock.Lock() - previous, err := bucket.Get(idBytes) - if err != nil { - lock.Unlock() - return nil, objectInsertStatus{}, errors.Wrap(err, "get bucket") - } - - nextObj, _, err := s.mergeObjectData(previous, merge) - if err != nil { - lock.Unlock() - return nil, objectInsertStatus{}, errors.Wrap(err, "merge object data") - } - - status, err := s.determineInsertStatus(previous, nextObj) - if err != nil { - lock.Unlock() - return nil, status, errors.Wrap(err, "check insert/update status") - } - - nextObj.SetDocID(status.docID) - nextBytes, err := nextObj.MarshalBinary() - if err != nil { - lock.Unlock() - return nil, status, errors.Wrapf(err, "marshal object %s to binary", nextObj.ID()) - } - - if err := s.upsertObjectDataLSM(bucket, idBytes, nextBytes, status.docID); err != nil { - lock.Unlock() - return nil, status, errors.Wrap(err, "upsert object data") - } - lock.Unlock() - - if err := s.updateInvertedIndexLSM(nextObj, status, previous); err != nil { - return nil, status, errors.Wrap(err, "update inverted indices") - } - - return nextObj, status, nil -} - -// mutableMergeObjectLSM is a special version of mergeObjectInTx where no doc -// id increases will be made, but instead the old doc ID will be re-used. This -// is only possible if the following two conditions are met: -// -// 1. We only add to the inverted index, but there is nothing which requires -// cleaning up. Example `name: "John"` is updated to `name: "John Doe"`, -// this is valid because we only add new entry for "Doe", but do not alter -// the existing entry for "John" -// An invalid update would be `name:"John"` is updated to `name:"Diane"`, -// this would require a cleanup for the existing link from "John" to this -// doc id, which is not possible. The only way to clean up is to increase -// the doc id and delete all entries for the old one -// -// 2. The vector position is not altered. Vector Indices cannot be mutated -// therefore a vector update would not be reflected -// -// The above makes this a perfect candidate for a batch reference update as -// this alters neither the vector position, nor does it remove anything from -// the inverted index -func (s *Shard) mutableMergeObjectLSM(merge objects.MergeDocument, - idBytes []byte, -) (mutableMergeResult, error) { - bucket := s.store.Bucket(helpers.ObjectsBucketLSM) - out := mutableMergeResult{} - - // see comment in shard_write_put.go::putObjectLSM - lock := &s.docIdLock[s.uuidToIdLockPoolId(idBytes)] - lock.Lock() - defer lock.Unlock() - - previous, err := bucket.Get(idBytes) - if err != nil { - return out, err - } - - nextObj, previousObj, err := s.mergeObjectData(previous, merge) - if err != nil { - return out, errors.Wrap(err, "merge object data") - } - - out.next = nextObj - out.previous = previousObj - - status, err := s.determineMutableInsertStatus(previous, nextObj) - if err != nil { - return out, errors.Wrap(err, "check insert/update status") - } - out.status = status - - nextObj.SetDocID(status.docID) // is not changed - nextBytes, err := nextObj.MarshalBinary() - if err != nil { - return out, errors.Wrapf(err, "marshal object %s to binary", nextObj.ID()) - } - - if err := s.upsertObjectDataLSM(bucket, idBytes, nextBytes, status.docID); err != nil { - return out, errors.Wrap(err, "upsert object data") - } - - // do not updated inverted index, since this requires delta analysis, which - // must be done by the caller! - - return out, nil -} - -type mutableMergeResult struct { - next *storobj.Object - previous *storobj.Object - status objectInsertStatus -} - -func (s *Shard) mergeObjectData(previous []byte, - merge objects.MergeDocument, -) (*storobj.Object, *storobj.Object, error) { - var previousObj *storobj.Object - if len(previous) == 0 { - // DocID must be overwritten after status check, simply set to initial - // value - previousObj = storobj.New(0) - previousObj.SetClass(merge.Class) - previousObj.SetID(merge.ID) - } else { - p, err := storobj.FromBinary(previous) - if err != nil { - return nil, nil, errors.Wrap(err, "unmarshal previous") - } - - previousObj = p - } - - return mergeProps(previousObj, merge), previousObj, nil -} - -func mergeProps(previous *storobj.Object, - merge objects.MergeDocument, -) *storobj.Object { - next := previous.DeepCopyDangerous() - properties, ok := next.Properties().(map[string]interface{}) - if !ok || properties == nil { - properties = map[string]interface{}{} - } - - // remove properties from object that have been set to nil - for _, propToDelete := range merge.PropertiesToDelete { - delete(properties, propToDelete) - } - - for propName, value := range merge.PrimitiveSchema { - // for primitive props, we simply need to overwrite - properties[propName] = value - } - - for _, ref := range merge.References { - propName := ref.From.Property.String() - prop := properties[propName] - propParsed, ok := prop.(models.MultipleRef) - if !ok { - propParsed = models.MultipleRef{} - } - propParsed = append(propParsed, ref.To.SingleRef()) - properties[propName] = propParsed - } - - if merge.Vector == nil { - next.Vector = previous.Vector - } else { - next.Vector = merge.Vector - } - - next.Object.LastUpdateTimeUnix = merge.UpdateTime - next.SetProperties(properties) - - return next -} diff --git a/adapters/repos/db/shard_write_put.go b/adapters/repos/db/shard_write_put.go deleted file mode 100644 index ee29db90b787a81382bec6b067f180fa39bc8725..0000000000000000000000000000000000000000 --- a/adapters/repos/db/shard_write_put.go +++ /dev/null @@ -1,376 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "bytes" - "context" - "encoding/binary" - "fmt" - "time" - - "github.com/google/uuid" - "github.com/pkg/errors" - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/adapters/repos/db/inverted" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv" - "github.com/weaviate/weaviate/adapters/repos/db/vector/common" - "github.com/weaviate/weaviate/entities/storagestate" - "github.com/weaviate/weaviate/entities/storobj" -) - -func (s *Shard) PutObject(ctx context.Context, object *storobj.Object) error { - if s.isReadOnly() { - return storagestate.ErrStatusReadOnly - } - uuid, err := uuid.MustParse(object.ID().String()).MarshalBinary() - if err != nil { - return err - } - return s.putOne(ctx, uuid, object) -} - -func (s *Shard) putOne(ctx context.Context, uuid []byte, object *storobj.Object) error { - if object.Vector != nil { - // validation needs to happen before any changes are done. Otherwise, insertion is aborted somewhere in-between. - err := s.VectorIndex().ValidateBeforeInsert(object.Vector) - if err != nil { - return errors.Wrapf(err, "Validate vector index for %s", object.ID()) - } - } - - status, err := s.putObjectLSM(object, uuid) - if err != nil { - return errors.Wrap(err, "store object in LSM store") - } - - if err := s.updateVectorIndex(object.Vector, status); err != nil { - return errors.Wrap(err, "update vector index") - } - - if err := s.updatePropertySpecificIndices(object, status); err != nil { - return errors.Wrap(err, "update property-specific indices") - } - - if err := s.store.WriteWALs(); err != nil { - return errors.Wrap(err, "flush all buffered WALs") - } - - if err := s.GetPropertyLengthTracker().Flush(false); err != nil { - return errors.Wrap(err, "flush prop length tracker to disk") - } - - return nil -} - -// as the name implies this method only performs the insertions, but completely -// ignores any deletes. It thus assumes that the caller has already taken care -// of all the deletes in another way -func (s *Shard) updateVectorIndexIgnoreDelete(vector []float32, - status objectInsertStatus, -) error { - // vector was not changed, object was updated without changing docID - // https://github.com/weaviate/weaviate/issues/3948 - if status.docIDPreserved { - return nil - } - - // vector is now optional as of - // https://github.com/weaviate/weaviate/issues/1800 - if len(vector) == 0 { - return nil - } - - if err := s.VectorIndex().Add(status.docID, vector); err != nil { - return errors.Wrapf(err, "insert doc id %d to vector index", status.docID) - } - - return nil -} - -func (s *Shard) updateVectorIndex(vector []float32, - status objectInsertStatus, -) error { - // even if no vector is provided in an update, we still need - // to delete the previous vector from the index, if it - // exists. otherwise, the associated doc id is left dangling, - // resulting in failed attempts to merge an object on restarts. - if status.docIDChanged { - if err := s.queue.Delete(status.oldDocID); err != nil { - return errors.Wrapf(err, "delete doc id %d from vector index", status.oldDocID) - } - } - - // vector was not changed, object was updated without changing docID - // https://github.com/weaviate/weaviate/issues/3948 - if status.docIDPreserved { - return nil - } - - // vector is now optional as of - // https://github.com/weaviate/weaviate/issues/1800 - if len(vector) == 0 { - return nil - } - - if err := s.VectorIndex().Add(status.docID, vector); err != nil { - return errors.Wrapf(err, "insert doc id %d to vector index", status.docID) - } - - if err := s.VectorIndex().Flush(); err != nil { - return errors.Wrap(err, "flush all vector index buffered WALs") - } - - return nil -} - -// TODO AL_skip_vector_reindex: adjust to batch? -func (s *Shard) putObjectLSM(object *storobj.Object, idBytes []byte, -) (objectInsertStatus, error) { - before := time.Now() - defer s.metrics.PutObject(before) - - bucket := s.store.Bucket(helpers.ObjectsBucketLSM) - - // First the object bucket is checked if already an object with the same uuid is present, to determine if it is new - // or an update. Afterwards the bucket is updates. To avoid races, only one goroutine can do this at once. - lock := &s.docIdLock[s.uuidToIdLockPoolId(idBytes)] - lock.Lock() - previous_object_bytes, err := bucket.Get(idBytes) - if err != nil { - lock.Unlock() - return objectInsertStatus{}, err - } - - status, err := s.determineInsertStatus(previous_object_bytes, object) - if err != nil { - lock.Unlock() - return status, errors.Wrap(err, "check insert/update status") - } - s.metrics.PutObjectDetermineStatus(before) - - object.SetDocID(status.docID) - data, err := object.MarshalBinary() - if err != nil { - lock.Unlock() - return status, errors.Wrapf(err, "marshal object %s to binary", object.ID()) - } - - before = time.Now() - if err := s.upsertObjectDataLSM(bucket, idBytes, data, status.docID); err != nil { - lock.Unlock() - return status, errors.Wrap(err, "upsert object data") - } - lock.Unlock() - s.metrics.PutObjectUpsertObject(before) - - before = time.Now() - if err := s.updateInvertedIndexLSM(object, status, previous_object_bytes); err != nil { - return status, errors.Wrap(err, "update inverted indices") - } - - s.metrics.PutObjectUpdateInverted(before) - - return status, nil -} - -type objectInsertStatus struct { - docID uint64 - docIDChanged bool - oldDocID uint64 - docIDPreserved bool // docID preserved (docID was not changed, although object did change) -} - -// to be called with the current contents of a row, if the row is empty (i.e. -// didn't exist before), we will get a new docID from the central counter. -// Otherwise, we will reuse the previous docID and mark this as an update -func (s *Shard) determineInsertStatus(previous []byte, - next *storobj.Object, -) (objectInsertStatus, error) { - var out objectInsertStatus - - if previous == nil { - docID, err := s.counter.GetAndInc() - if err != nil { - return out, errors.Wrap(err, "initial doc id: get new doc id from counter") - } - out.docID = docID - return out, nil - } - - docID, err := storobj.DocIDFromBinary(previous) - if err != nil { - return out, errors.Wrap(err, "get previous doc id from object binary") - } - out.oldDocID = docID - - // if vector was not changed, replace object and update indexed properties - // without changing docID - // https://github.com/weaviate/weaviate/issues/3948 - // - // Due to geo index does not support delete+insert of geo value for the same docID - // (as tombstones are used for hnsw geo index), - // preserving docID will not be supported for classes with geo properties for now. - // Feature can be improved in the future, by preserving docIDs for objects - // not updating values of geo properties. - if !s.hasGeoIndex() { - if l := len(next.Vector); l > 0 { - buffer := make([]float32, l) - prevVector, err := storobj.VectorFromBinary(previous, buffer) - if err != nil { - return out, fmt.Errorf("get previous vector from object binary: %w", err) - } - if common.VectorsEqual(prevVector, next.Vector) { - out.docID = docID - out.docIDPreserved = true - return out, nil - } - } - } - - // with docIDs now being immutable (see - // https://github.com/weaviate/weaviate/issues/1282) there is no - // more check if we need to increase a docID. Any update will mean a doc ID - // needs to be updated. - docID, err = s.counter.GetAndInc() - if err != nil { - return out, errors.Wrap(err, "doc id update: get new doc id from counter") - } - out.docID = docID - out.docIDChanged = true - - return out, nil -} - -// determineMutableInsertStatus is a special version of determineInsertStatus -// where it does not alter the doc id if one already exists. Calling this -// method only makes sense under very special conditions, such as those -// outlined in mutableMergeObjectInTx -func (s *Shard) determineMutableInsertStatus(previous []byte, - next *storobj.Object, -) (objectInsertStatus, error) { - var out objectInsertStatus - - if previous == nil { - docID, err := s.counter.GetAndInc() - if err != nil { - return out, errors.Wrap(err, "initial doc id: get new doc id from counter") - } - out.docID = docID - return out, nil - } - - docID, err := storobj.DocIDFromBinary(previous) - if err != nil { - return out, errors.Wrap(err, "get previous doc id from object binary") - } - out.docID = docID - - // we are planning on mutating and thus not altering the doc id - return out, nil -} - -func (s *Shard) upsertObjectDataLSM(bucket *lsmkv.Bucket, id []byte, data []byte, - docID uint64, -) error { - keyBuf := bytes.NewBuffer(nil) - binary.Write(keyBuf, binary.LittleEndian, &docID) - docIDBytes := keyBuf.Bytes() - - return bucket.Put(id, data, lsmkv.WithSecondaryKey(0, docIDBytes)) -} - -func (s *Shard) updateInvertedIndexLSM(object *storobj.Object, - status objectInsertStatus, previous []byte, -) error { - props, nilprops, err := s.AnalyzeObject(object) - if err != nil { - return errors.Wrap(err, "analyze next object") - } - - var prevObject *storobj.Object - var prevProps []inverted.Property - var prevNilprops []inverted.NilProperty - - if previous != nil { - prevObject, err = storobj.FromBinary(previous) - if err != nil { - return fmt.Errorf("unmarshal previous object: %w", err) - } - - prevProps, prevNilprops, err = s.AnalyzeObject(prevObject) - if err != nil { - return fmt.Errorf("analyze previous object: %w", err) - } - } - - // if object updated (with or without docID changed) - if status.docIDChanged || status.docIDPreserved { - if err := s.subtractPropLengths(prevProps); err != nil { - s.index.logger.WithField("action", "subtractPropLengths").WithError(err).Error("could not subtract prop lengths") - } - } else { - if err := s.ChangeObjectCountBy(1); err != nil { - return fmt.Errorf("increment object count: %w", err) - } - } - - if err := s.SetPropertyLengths(props); err != nil { - return errors.Wrap(err, "store field length values for props") - } - - var propsToAdd []inverted.Property - var propsToDel []inverted.Property - var nilpropsToAdd []inverted.NilProperty - var nilpropsToDel []inverted.NilProperty - - // determine only changed properties to avoid unnecessary updates of inverted indexes - if status.docIDPreserved { - delta := inverted.Delta(prevProps, props) - propsToAdd = delta.ToAdd - propsToDel = delta.ToDelete - deltaNil := inverted.DeltaNil(prevNilprops, nilprops) - nilpropsToAdd = deltaNil.ToAdd - nilpropsToDel = deltaNil.ToDelete - } else { - propsToAdd = inverted.DedupItems(props) - propsToDel = inverted.DedupItems(prevProps) - nilpropsToAdd = nilprops - nilpropsToDel = prevNilprops - } - - if previous != nil { - // TODO: metrics - if err := s.deleteFromInvertedIndicesLSM(propsToDel, nilpropsToDel, status.oldDocID); err != nil { - return fmt.Errorf("delete inverted indices props: %w", err) - } - if s.index.Config.TrackVectorDimensions { - if err := s.removeDimensionsLSM(len(prevObject.Vector), status.oldDocID); err != nil { - return fmt.Errorf("track dimensions (delete): %w", err) - } - } - } - - before := time.Now() - if err := s.extendInvertedIndicesLSM(propsToAdd, nilpropsToAdd, status.docID); err != nil { - return fmt.Errorf("put inverted indices props: %w", err) - } - s.metrics.InvertedExtend(before, len(propsToAdd)) - - if s.index.Config.TrackVectorDimensions { - if err := s.extendDimensionTrackerLSM(len(object.Vector), status.docID); err != nil { - return fmt.Errorf("track dimensions: %w", err) - } - } - - return nil -} diff --git a/adapters/repos/db/sortby_distances.go b/adapters/repos/db/sortby_distances.go deleted file mode 100644 index 363b188da482c2e9acfae4fadd2ff8d6e9bb04c1..0000000000000000000000000000000000000000 --- a/adapters/repos/db/sortby_distances.go +++ /dev/null @@ -1,48 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "sort" - - "github.com/weaviate/weaviate/entities/storobj" -) - -type sortByDistances struct { - objects []*storobj.Object - scores []float32 -} - -func (sbd *sortByDistances) Len() int { - return len(sbd.objects) -} - -func (sbd *sortByDistances) Less(i, j int) bool { - return sbd.scores[i] < sbd.scores[j] -} - -func (sbd *sortByDistances) Swap(i, j int) { - sbd.scores[i], sbd.scores[j] = sbd.scores[j], sbd.scores[i] - sbd.objects[i], sbd.objects[j] = sbd.objects[j], sbd.objects[i] -} - -type sortObjectsByDistance struct{} - -func newDistancesSorter() *sortObjectsByDistance { - return &sortObjectsByDistance{} -} - -func (s *sortObjectsByDistance) sort(objects []*storobj.Object, distances []float32) ([]*storobj.Object, []float32) { - sbd := &sortByDistances{objects, distances} - sort.Sort(sbd) - return sbd.objects, sbd.scores -} diff --git a/adapters/repos/db/sortby_id.go b/adapters/repos/db/sortby_id.go deleted file mode 100644 index f5b027d16e288c5cfd994a12712f2e215921b27b..0000000000000000000000000000000000000000 --- a/adapters/repos/db/sortby_id.go +++ /dev/null @@ -1,51 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "sort" - - "github.com/weaviate/weaviate/entities/storobj" -) - -type sortByID struct { - objects []*storobj.Object - scores []float32 -} - -func (s *sortByID) Swap(i, j int) { - if len(s.objects) == len(s.scores) { - s.scores[i], s.scores[j] = s.scores[j], s.scores[i] - } - s.objects[i], s.objects[j] = s.objects[j], s.objects[i] -} - -func (s *sortByID) Less(i, j int) bool { - return s.objects[i].ID() < s.objects[j].ID() -} - -func (s *sortByID) Len() int { - return len(s.objects) -} - -type sortObjectsByID struct{} - -func newIDSorter() *sortObjectsByID { - return &sortObjectsByID{} -} - -func (s *sortObjectsByID) sort(objects []*storobj.Object, scores []float32, -) ([]*storobj.Object, []float32) { - sbd := &sortByID{objects, scores} - sort.Sort(sbd) - return sbd.objects, sbd.scores -} diff --git a/adapters/repos/db/sortby_scores.go b/adapters/repos/db/sortby_scores.go deleted file mode 100644 index a4ac19e62692bcf0b5b03002b587fbe0a1b370da..0000000000000000000000000000000000000000 --- a/adapters/repos/db/sortby_scores.go +++ /dev/null @@ -1,51 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "sort" - - "github.com/weaviate/weaviate/entities/storobj" -) - -// sortByScores aka RankedResults implements sort.Interface, allowing -// results aggregated from multiple shards to be -// sorted according to their BM25 ranking -type sortByScores struct { - objects []*storobj.Object - scores []float32 -} - -func (r *sortByScores) Swap(i, j int) { - r.objects[i], r.objects[j] = r.objects[j], r.objects[i] - r.scores[i], r.scores[j] = r.scores[j], r.scores[i] -} - -func (r *sortByScores) Less(i, j int) bool { - return r.scores[i] > r.scores[j] -} - -func (r *sortByScores) Len() int { - return len(r.scores) -} - -type sortObjectsByScore struct{} - -func newScoresSorter() *sortObjectsByScore { - return &sortObjectsByScore{} -} - -func (s *sortObjectsByScore) sort(objects []*storobj.Object, scores []float32) ([]*storobj.Object, []float32) { - sbd := &sortByScores{objects, scores} - sort.Sort(sbd) - return sbd.objects, sbd.scores -} diff --git a/adapters/repos/db/sortby_scores_test.go b/adapters/repos/db/sortby_scores_test.go deleted file mode 100644 index 4fe9f3ad67f9332b5beff79653f9c663eb277146..0000000000000000000000000000000000000000 --- a/adapters/repos/db/sortby_scores_test.go +++ /dev/null @@ -1,76 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "testing" - - "github.com/go-openapi/strfmt" - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/storobj" -) - -func Test_SortBy_Scores(t *testing.T) { - type testcase struct { - testName string - givenObjects []*storobj.Object - givenScores []float32 - expectedOrder []string - } - - tests := []testcase{ - { - testName: "with multiple results", - givenObjects: []*storobj.Object{ - {Object: models.Object{ID: strfmt.UUID("40d3be3e-2ecc-49c8-b37c-d8983164848b")}}, - {Object: models.Object{ID: strfmt.UUID("31bdf9ef-d1c0-4b43-8331-1a89a48c1d2b")}}, - {Object: models.Object{ID: strfmt.UUID("4432797a-ef18-429f-83dc-d971dd9e4dd0")}}, - {Object: models.Object{ID: strfmt.UUID("8ef8c6fd-93b5-4452-b3c3-cef1cd0a18ed")}}, - {Object: models.Object{ID: strfmt.UUID("d79f0d2d-ebc5-4dad-b3df-323bc1e6f183")}}, - }, - givenScores: []float32{12, 34, 100, 43, 2}, - expectedOrder: []string{ - "4432797a-ef18-429f-83dc-d971dd9e4dd0", - "8ef8c6fd-93b5-4452-b3c3-cef1cd0a18ed", - "31bdf9ef-d1c0-4b43-8331-1a89a48c1d2b", - "40d3be3e-2ecc-49c8-b37c-d8983164848b", - "d79f0d2d-ebc5-4dad-b3df-323bc1e6f183", - }, - }, - { - testName: "with a single result", - givenObjects: []*storobj.Object{ - {Object: models.Object{ID: strfmt.UUID("4a483f11-7b2f-452b-be49-f7844dbc5693")}}, - }, - givenScores: []float32{1}, - expectedOrder: []string{ - "4a483f11-7b2f-452b-be49-f7844dbc5693", - }, - }, - { - testName: "with no results", - givenObjects: []*storobj.Object{}, - givenScores: []float32{}, - expectedOrder: []string{}, - }, - } - - for _, test := range tests { - t.Run(test.testName, func(t *testing.T) { - objects, _ := newScoresSorter().sort(test.givenObjects, test.givenScores) - for i := range objects { - assert.Equal(t, test.expectedOrder[i], objects[i].ID().String()) - } - }) - } -} diff --git a/adapters/repos/db/sorter/basic_comparators.go b/adapters/repos/db/sorter/basic_comparators.go deleted file mode 100644 index 965641bd0df7835ef064c644505b6544b9a05e4b..0000000000000000000000000000000000000000 --- a/adapters/repos/db/sorter/basic_comparators.go +++ /dev/null @@ -1,382 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package sorter - -import ( - "strings" - "time" - - "github.com/weaviate/weaviate/entities/schema" -) - -type basicComparatorProvider struct{} - -func (bcp *basicComparatorProvider) provide(dataType schema.DataType, order string) basicComparator { - switch dataType { - case schema.DataTypeBlob: - return newStringComparator(order) - case schema.DataTypeText: - return newStringComparator(order) - case schema.DataTypeTextArray: - return newStringArrayComparator(order) - case schema.DataTypeNumber, schema.DataTypeInt: - return newFloat64Comparator(order) - case schema.DataTypeNumberArray, schema.DataTypeIntArray: - return newFloat64ArrayComparator(order) - case schema.DataTypeDate: - return newDateComparator(order) - case schema.DataTypeDateArray: - return newDateArrayComparator(order) - case schema.DataTypeBoolean: - return newBoolComparator(order) - case schema.DataTypeBooleanArray: - return newBoolArrayComparator(order) - case schema.DataTypePhoneNumber: - return newFloat64ArrayComparator(order) - case schema.DataTypeGeoCoordinates: - return newFloat64ArrayComparator(order) - default: - return newAnyComparator(order) - } -} - -type basicComparator interface { - compare(a, b interface{}) int -} - -type stringComparator struct { - lessValue int -} - -func newStringComparator(order string) *stringComparator { - return &stringComparator{lessValue(order)} -} - -func (sc *stringComparator) compare(a, b interface{}) int { - a, b = sc.untypedNil(a), sc.untypedNil(b) - if a != nil && b != nil { - return sc.compareStrings(*(a.(*string)), *(b.(*string))) - } - return handleNils(a == nil, b == nil, sc.lessValue) -} - -func (sc *stringComparator) compareStrings(a, b string) int { - if strings.EqualFold(a, b) { - return 0 - } - if strings.ToLower(a) < strings.ToLower(b) { - return sc.lessValue - } - return -sc.lessValue -} - -func (sc *stringComparator) untypedNil(x interface{}) interface{} { - if x == (*string)(nil) { - return nil - } - return x -} - -type stringArrayComparator struct { - sc *stringComparator - ic *intComparator -} - -func newStringArrayComparator(order string) *stringArrayComparator { - return &stringArrayComparator{newStringComparator(order), newIntComparator(order)} -} - -func (sac *stringArrayComparator) compare(a, b interface{}) int { - a, b = sac.untypedNil(a), sac.untypedNil(b) - if a != nil && b != nil { - aArr, bArr := *(a.(*[]string)), *(b.(*[]string)) - aLen, bLen := len(aArr), len(bArr) - - for i := 0; i < aLen && i < bLen; i++ { - if res := sac.sc.compareStrings(aArr[i], bArr[i]); res != 0 { - return res - } - } - return sac.ic.compareInts(aLen, bLen) - } - return handleNils(a == nil, b == nil, sac.sc.lessValue) -} - -func (sac *stringArrayComparator) untypedNil(x interface{}) interface{} { - if x == (*[]string)(nil) { - return nil - } - return x -} - -type float64Comparator struct { - lessValue int -} - -func newFloat64Comparator(order string) *float64Comparator { - return &float64Comparator{lessValue(order)} -} - -func (fc *float64Comparator) compare(a, b interface{}) int { - a, b = fc.untypedNil(a), fc.untypedNil(b) - if a != nil && b != nil { - return fc.compareFloats64(*(a.(*float64)), *(b.(*float64))) - } - return handleNils(a == nil, b == nil, fc.lessValue) -} - -func (fc *float64Comparator) compareFloats64(a, b float64) int { - if a == b { - return 0 - } - if a < b { - return fc.lessValue - } - return -fc.lessValue -} - -func (fc *float64Comparator) untypedNil(x interface{}) interface{} { - if x == (*float64)(nil) { - return nil - } - return x -} - -type float64ArrayComparator struct { - fc *float64Comparator - ic *intComparator -} - -func newFloat64ArrayComparator(order string) *float64ArrayComparator { - return &float64ArrayComparator{newFloat64Comparator(order), newIntComparator(order)} -} - -func (fac *float64ArrayComparator) compare(a, b interface{}) int { - a, b = fac.untypedNil(a), fac.untypedNil(b) - if a != nil && b != nil { - aArr, bArr := *(a.(*[]float64)), *(b.(*[]float64)) - aLen, bLen := len(aArr), len(bArr) - - for i := 0; i < aLen && i < bLen; i++ { - if res := fac.fc.compareFloats64(aArr[i], bArr[i]); res != 0 { - return res - } - } - return fac.ic.compareInts(aLen, bLen) - } - return handleNils(a == nil, b == nil, fac.fc.lessValue) -} - -func (fac *float64ArrayComparator) untypedNil(x interface{}) interface{} { - if x == (*[]float64)(nil) { - return nil - } - return x -} - -type dateComparator struct { - lessValue int -} - -func newDateComparator(order string) *dateComparator { - return &dateComparator{lessValue(order)} -} - -func (dc *dateComparator) compare(a, b interface{}) int { - a, b = dc.untypedNil(a), dc.untypedNil(b) - if a != nil && b != nil { - return dc.compareDates(*(a.(*time.Time)), *(b.(*time.Time))) - } - return handleNils(a == nil, b == nil, dc.lessValue) -} - -func (dc *dateComparator) compareDates(a, b time.Time) int { - if a.Equal(b) { - return 0 - } - if a.Before(b) { - return dc.lessValue - } - return -dc.lessValue -} - -func (dc *dateComparator) untypedNil(x interface{}) interface{} { - if x == (*time.Time)(nil) { - return nil - } - return x -} - -type dateArrayComparator struct { - dc *dateComparator - ic *intComparator -} - -func newDateArrayComparator(order string) *dateArrayComparator { - return &dateArrayComparator{newDateComparator(order), newIntComparator(order)} -} - -func (dac *dateArrayComparator) compare(a, b interface{}) int { - a, b = dac.untypedNil(a), dac.untypedNil(b) - if a != nil && b != nil { - aArr, bArr := *(a.(*[]time.Time)), *(b.(*[]time.Time)) - aLen, bLen := len(aArr), len(bArr) - - for i := 0; i < aLen && i < bLen; i++ { - if res := dac.dc.compareDates(aArr[i], bArr[i]); res != 0 { - return res - } - } - return dac.ic.compareInts(aLen, bLen) - } - return handleNils(a == nil, b == nil, dac.dc.lessValue) -} - -func (dac *dateArrayComparator) untypedNil(x interface{}) interface{} { - if x == (*[]time.Time)(nil) { - return nil - } - return x -} - -type boolComparator struct { - lessValue int -} - -func newBoolComparator(order string) *boolComparator { - return &boolComparator{lessValue(order)} -} - -func (bc *boolComparator) compare(a, b interface{}) int { - a, b = bc.untypedNil(a), bc.untypedNil(b) - if a != nil && b != nil { - return bc.compareBools(*(a.(*bool)), *(b.(*bool))) - } - return handleNils(a == nil, b == nil, bc.lessValue) -} - -func (bc *boolComparator) compareBools(a, b bool) int { - if a && b { - return 0 - } - if !a && !b { - return 0 - } - if !a { - return bc.lessValue - } - return -bc.lessValue -} - -func (bc *boolComparator) untypedNil(x interface{}) interface{} { - if x == (*bool)(nil) { - return nil - } - return x -} - -type boolArrayComparator struct { - bc *boolComparator - ic *intComparator -} - -func newBoolArrayComparator(order string) *boolArrayComparator { - return &boolArrayComparator{newBoolComparator(order), newIntComparator(order)} -} - -func (bac *boolArrayComparator) compare(a, b interface{}) int { - a, b = bac.untypedNil(a), bac.untypedNil(b) - if a != nil && b != nil { - aArr, bArr := *(a.(*[]bool)), *(b.(*[]bool)) - aLen, bLen := len(aArr), len(bArr) - - for i := 0; i < aLen && i < bLen; i++ { - if res := bac.bc.compareBools(aArr[i], bArr[i]); res != 0 { - return res - } - } - return bac.ic.compareInts(aLen, bLen) - } - return handleNils(a == nil, b == nil, bac.bc.lessValue) -} - -func (bac *boolArrayComparator) untypedNil(x interface{}) interface{} { - if x == (*[]bool)(nil) { - return nil - } - return x -} - -type intComparator struct { - lessValue int -} - -func newIntComparator(order string) *intComparator { - return &intComparator{lessValue(order)} -} - -func (ic *intComparator) compare(a, b interface{}) int { - a, b = ic.untypedNil(a), ic.untypedNil(b) - if a != nil && b != nil { - return ic.compareInts(*(a.(*int)), *(b.(*int))) - } - return handleNils(a == nil, b == nil, ic.lessValue) -} - -func (ic *intComparator) compareInts(a, b int) int { - if a == b { - return 0 - } - if a < b { - return ic.lessValue - } - return -ic.lessValue -} - -func (ic *intComparator) untypedNil(x interface{}) interface{} { - if x == (*int)(nil) { - return nil - } - return x -} - -type anyComparator struct { - lessValue int -} - -func newAnyComparator(order string) *anyComparator { - return &anyComparator{lessValue(order)} -} - -func (ac *anyComparator) compare(a, b interface{}) int { - if a != nil && b != nil { - return 0 - } - return handleNils(a == nil, b == nil, ac.lessValue) -} - -func handleNils(aNil, bNil bool, lessValue int) int { - if aNil && bNil { - return 0 - } - if aNil { - return lessValue - } - return -lessValue -} - -func lessValue(order string) int { - if order == "desc" { - return 1 - } - return -1 -} diff --git a/adapters/repos/db/sorter/basic_comparators_test.go b/adapters/repos/db/sorter/basic_comparators_test.go deleted file mode 100644 index 4fbc1ab9da9a93505d0fab17c6a7c1bd3ce71bd5..0000000000000000000000000000000000000000 --- a/adapters/repos/db/sorter/basic_comparators_test.go +++ /dev/null @@ -1,625 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package sorter - -import ( - "fmt" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func TestBasicComparator_String(t *testing.T) { - Orange := "Orange" - orange := "orange" - apple := "apple" - - t.Run("strings asc", func(t *testing.T) { - comp := newStringComparator("asc") - - params := []struct { - a *string - b *string - expected int - }{ - {&Orange, &orange, 0}, - {&orange, &orange, 0}, - {&apple, &apple, 0}, - {&orange, &apple, 1}, - {&apple, &orange, -1}, - {nil, &apple, -1}, - {&orange, nil, 1}, - {nil, nil, 0}, - } - - for i, p := range params { - t.Run(fmt.Sprintf("data #%d", i), func(t *testing.T) { - assert.Equal(t, p.expected, comp.compare(p.a, p.b)) - }) - } - }) - - t.Run("strings desc", func(t *testing.T) { - comp := newStringComparator("desc") - - params := []struct { - a *string - b *string - expected int - }{ - {&Orange, &orange, 0}, - {&orange, &orange, 0}, - {&apple, &apple, 0}, - {&orange, &apple, -1}, - {&apple, &orange, 1}, - {nil, &apple, 1}, - {&orange, nil, -1}, - {nil, nil, 0}, - } - - for i, p := range params { - t.Run(fmt.Sprintf("data #%d", i), func(t *testing.T) { - assert.Equal(t, p.expected, comp.compare(p.a, p.b)) - }) - } - }) -} - -func TestBasicComparator_StringArray(t *testing.T) { - o_b_a := []string{"orange", "banana", "apple"} - p := []string{"pear"} - o_a := []string{"orange", "apple"} - o_b := []string{"orange", "banana"} - - t.Run("strings array asc", func(t *testing.T) { - comp := newStringArrayComparator("asc") - - params := []struct { - a *[]string - b *[]string - expected int - }{ - {&o_b_a, &o_b_a, 0}, - {&p, &p, 0}, - {&o_a, &o_a, 0}, - {&o_b, &o_b, 0}, - {&o_a, &o_b, -1}, - {&o_b, &o_b_a, -1}, - {&p, &o_b_a, 1}, - {nil, &o_a, -1}, - {&p, nil, 1}, - {nil, nil, 0}, - } - - for i, p := range params { - t.Run(fmt.Sprintf("data #%d", i), func(t *testing.T) { - assert.Equal(t, p.expected, comp.compare(p.a, p.b)) - }) - } - }) - - t.Run("strings array desc", func(t *testing.T) { - comp := newStringArrayComparator("desc") - - params := []struct { - a *[]string - b *[]string - expected int - }{ - {&o_b_a, &o_b_a, 0}, - {&p, &p, 0}, - {&o_a, &o_a, 0}, - {&o_b, &o_b, 0}, - {&o_a, &o_b, 1}, - {&o_b, &o_b_a, 1}, - {&p, &o_b_a, -1}, - {nil, &o_a, 1}, - {&p, nil, -1}, - {nil, nil, 0}, - } - - for i, p := range params { - t.Run(fmt.Sprintf("data #%d", i), func(t *testing.T) { - assert.Equal(t, p.expected, comp.compare(p.a, p.b)) - }) - } - }) -} - -func TestBasicComparator_Float64(t *testing.T) { - f_10 := -10.0 - f100 := 100.0 - f0 := 0.0 - - t.Run("floats asc", func(t *testing.T) { - comp := newFloat64Comparator("asc") - - params := []struct { - a *float64 - b *float64 - expected int - }{ - {&f_10, &f_10, 0}, - {&f100, &f100, 0}, - {&f0, &f0, 0}, - {&f100, &f_10, 1}, - {&f0, &f100, -1}, - {nil, &f_10, -1}, - {&f0, nil, 1}, - {nil, nil, 0}, - } - - for i, p := range params { - t.Run(fmt.Sprintf("data #%d", i), func(t *testing.T) { - assert.Equal(t, p.expected, comp.compare(p.a, p.b)) - }) - } - }) - - t.Run("floats desc", func(t *testing.T) { - comp := newFloat64Comparator("desc") - - params := []struct { - a *float64 - b *float64 - expected int - }{ - {&f_10, &f_10, 0}, - {&f100, &f100, 0}, - {&f0, &f0, 0}, - {&f100, &f_10, -1}, - {&f0, &f100, 1}, - {nil, &f_10, 1}, - {&f0, nil, -1}, - {nil, nil, 0}, - } - - for i, p := range params { - t.Run(fmt.Sprintf("data #%d", i), func(t *testing.T) { - assert.Equal(t, p.expected, comp.compare(p.a, p.b)) - }) - } - }) -} - -func TestBasicComparator_Float64Array(t *testing.T) { - f_3_2_1 := []float64{3, 2, 1} - f_4 := []float64{4} - f_3_1 := []float64{3, 1} - f_3_2 := []float64{3, 2} - - t.Run("floats array asc", func(t *testing.T) { - comp := newFloat64ArrayComparator("asc") - - params := []struct { - a *[]float64 - b *[]float64 - expected int - }{ - {&f_3_2_1, &f_3_2_1, 0}, - {&f_4, &f_4, 0}, - {&f_3_1, &f_3_1, 0}, - {&f_3_2, &f_3_2, 0}, - {&f_3_1, &f_3_2, -1}, - {&f_3_2, &f_3_2_1, -1}, - {&f_4, &f_3_2_1, 1}, - {nil, &f_3_1, -1}, - {&f_4, nil, 1}, - {nil, nil, 0}, - } - - for i, p := range params { - t.Run(fmt.Sprintf("data #%d", i), func(t *testing.T) { - assert.Equal(t, p.expected, comp.compare(p.a, p.b)) - }) - } - }) - - t.Run("floats array desc", func(t *testing.T) { - comp := newFloat64ArrayComparator("desc") - - params := []struct { - a *[]float64 - b *[]float64 - expected int - }{ - {&f_3_2_1, &f_3_2_1, 0}, - {&f_4, &f_4, 0}, - {&f_3_1, &f_3_1, 0}, - {&f_3_2, &f_3_2, 0}, - {&f_3_1, &f_3_2, 1}, - {&f_3_2, &f_3_2_1, 1}, - {&f_4, &f_3_2_1, -1}, - {nil, &f_3_1, 1}, - {&f_4, nil, -1}, - {nil, nil, 0}, - } - - for i, p := range params { - t.Run(fmt.Sprintf("data #%d", i), func(t *testing.T) { - assert.Equal(t, p.expected, comp.compare(p.a, p.b)) - }) - } - }) -} - -func TestBasicComparator_Date(t *testing.T) { - t1 := time.Now() - t2 := time.Now().Add(time.Second) - t3 := time.Now().Add(2 * time.Second) - - t.Run("dates asc", func(t *testing.T) { - comp := newDateComparator("asc") - - params := []struct { - a *time.Time - b *time.Time - expected int - }{ - {&t1, &t1, 0}, - {&t3, &t3, 0}, - {&t2, &t2, 0}, - {&t3, &t1, 1}, - {&t2, &t3, -1}, - {nil, &t1, -1}, - {&t2, nil, 1}, - {nil, nil, 0}, - } - - for i, p := range params { - t.Run(fmt.Sprintf("data #%d", i), func(t *testing.T) { - assert.Equal(t, p.expected, comp.compare(p.a, p.b)) - }) - } - }) - - t.Run("dates desc", func(t *testing.T) { - comp := newDateComparator("desc") - - params := []struct { - a *time.Time - b *time.Time - expected int - }{ - {&t1, &t1, 0}, - {&t3, &t3, 0}, - {&t2, &t2, 0}, - {&t3, &t1, -1}, - {&t2, &t3, 1}, - {nil, &t1, 1}, - {&t2, nil, -1}, - {nil, nil, 0}, - } - - for i, p := range params { - t.Run(fmt.Sprintf("data #%d", i), func(t *testing.T) { - assert.Equal(t, p.expected, comp.compare(p.a, p.b)) - }) - } - }) -} - -func TestBasicComparator_DateArray(t *testing.T) { - t1 := time.Now() - t2 := time.Now().Add(time.Second) - t3 := time.Now().Add(2 * time.Second) - t4 := time.Now().Add(3 * time.Second) - - t_3_2_1 := []time.Time{t3, t2, t1} - t_4 := []time.Time{t4} - t_3_1 := []time.Time{t3, t1} - t_3_2 := []time.Time{t3, t2} - - t.Run("dates array asc", func(t *testing.T) { - comp := newDateArrayComparator("asc") - - params := []struct { - a *[]time.Time - b *[]time.Time - expected int - }{ - {&t_3_2_1, &t_3_2_1, 0}, - {&t_4, &t_4, 0}, - {&t_3_1, &t_3_1, 0}, - {&t_3_2, &t_3_2, 0}, - {&t_3_1, &t_3_2, -1}, - {&t_3_2, &t_3_2_1, -1}, - {&t_4, &t_3_2_1, 1}, - {nil, &t_3_1, -1}, - {&t_4, nil, 1}, - {nil, nil, 0}, - } - - for i, p := range params { - t.Run(fmt.Sprintf("data #%d", i), func(t *testing.T) { - assert.Equal(t, p.expected, comp.compare(p.a, p.b)) - }) - } - }) - - t.Run("dates array desc", func(t *testing.T) { - comp := newDateArrayComparator("desc") - - params := []struct { - a *[]time.Time - b *[]time.Time - expected int - }{ - {&t_3_2_1, &t_3_2_1, 0}, - {&t_4, &t_4, 0}, - {&t_3_1, &t_3_1, 0}, - {&t_3_2, &t_3_2, 0}, - {&t_3_1, &t_3_2, 1}, - {&t_3_2, &t_3_2_1, 1}, - {&t_4, &t_3_2_1, -1}, - {nil, &t_3_1, 1}, - {&t_4, nil, -1}, - {nil, nil, 0}, - } - - for i, p := range params { - t.Run(fmt.Sprintf("data #%d", i), func(t *testing.T) { - assert.Equal(t, p.expected, comp.compare(p.a, p.b)) - }) - } - }) -} - -func TestBasicComparator_Bool(t *testing.T) { - fa := false - tr := true - - t.Run("bools asc", func(t *testing.T) { - comp := newBoolComparator("asc") - - params := []struct { - a *bool - b *bool - expected int - }{ - {&fa, &fa, 0}, - {&tr, &tr, 0}, - {&fa, &tr, -1}, - {nil, &fa, -1}, - {&tr, nil, 1}, - {nil, nil, 0}, - } - - for i, p := range params { - t.Run(fmt.Sprintf("data #%d", i), func(t *testing.T) { - assert.Equal(t, p.expected, comp.compare(p.a, p.b)) - }) - } - }) - - t.Run("bools desc", func(t *testing.T) { - comp := newBoolComparator("desc") - - params := []struct { - a *bool - b *bool - expected int - }{ - {&fa, &fa, 0}, - {&tr, &tr, 0}, - {&fa, &tr, 1}, - {nil, &fa, 1}, - {&tr, nil, -1}, - {nil, nil, 0}, - } - - for i, p := range params { - t.Run(fmt.Sprintf("data #%d", i), func(t *testing.T) { - assert.Equal(t, p.expected, comp.compare(p.a, p.b)) - }) - } - }) -} - -func TestBasicComparator_BoolArray(t *testing.T) { - fa_tr_fa := []bool{false, true, false} - tr := []bool{true} - fa_fa := []bool{false, false} - fa_tr := []bool{false, true} - - t.Run("bools array asc", func(t *testing.T) { - comp := newBoolArrayComparator("asc") - - params := []struct { - a *[]bool - b *[]bool - expected int - }{ - {&fa_tr_fa, &fa_tr_fa, 0}, - {&tr, &tr, 0}, - {&fa_fa, &fa_fa, 0}, - {&fa_tr, &fa_tr, 0}, - {&fa_fa, &fa_tr, -1}, - {&fa_tr, &fa_tr_fa, -1}, - {&tr, &fa_tr_fa, 1}, - {nil, &fa_fa, -1}, - {&tr, nil, 1}, - {nil, nil, 0}, - } - - for i, p := range params { - t.Run(fmt.Sprintf("data #%d", i), func(t *testing.T) { - assert.Equal(t, p.expected, comp.compare(p.a, p.b)) - }) - } - }) - - t.Run("bools array desc", func(t *testing.T) { - comp := newBoolArrayComparator("desc") - - params := []struct { - a *[]bool - b *[]bool - expected int - }{ - {&fa_tr_fa, &fa_tr_fa, 0}, - {&tr, &tr, 0}, - {&fa_fa, &fa_fa, 0}, - {&fa_tr, &fa_tr, 0}, - {&fa_fa, &fa_tr, 1}, - {&fa_tr, &fa_tr_fa, 1}, - {&tr, &fa_tr_fa, -1}, - {nil, &fa_fa, 1}, - {&tr, nil, -1}, - {nil, nil, 0}, - } - - for i, p := range params { - t.Run(fmt.Sprintf("data #%d", i), func(t *testing.T) { - assert.Equal(t, p.expected, comp.compare(p.a, p.b)) - }) - } - }) -} - -func TestBasicComparator_Int(t *testing.T) { - i_10 := -10 - i100 := 100 - i0 := 0 - - t.Run("ints asc", func(t *testing.T) { - comp := newIntComparator("asc") - - params := []struct { - a *int - b *int - expected int - }{ - {&i_10, &i_10, 0}, - {&i100, &i100, 0}, - {&i0, &i0, 0}, - {&i100, &i_10, 1}, - {&i0, &i100, -1}, - {nil, &i_10, -1}, - {&i0, nil, 1}, - {nil, nil, 0}, - } - - for i, p := range params { - t.Run(fmt.Sprintf("data #%d", i), func(t *testing.T) { - assert.Equal(t, p.expected, comp.compare(p.a, p.b)) - }) - } - }) - - t.Run("ints desc", func(t *testing.T) { - comp := newIntComparator("desc") - - params := []struct { - a *int - b *int - expected int - }{ - {&i_10, &i_10, 0}, - {&i100, &i100, 0}, - {&i0, &i0, 0}, - {&i100, &i_10, -1}, - {&i0, &i100, 1}, - {nil, &i_10, 1}, - {&i0, nil, -1}, - {nil, nil, 0}, - } - - for i, p := range params { - t.Run(fmt.Sprintf("data #%d", i), func(t *testing.T) { - assert.Equal(t, p.expected, comp.compare(p.a, p.b)) - }) - } - }) -} - -func TestBasicComparator_Any(t *testing.T) { - in := -10 - fl := 100.0 - st := "string" - ti := time.Now() - bo := true - an := struct{}{} - - t.Run("any asc", func(t *testing.T) { - comp := newAnyComparator("asc") - - params := []struct { - a interface{} - b interface{} - expected int - }{ - {&in, &in, 0}, - {&fl, &fl, 0}, - {&st, &st, 0}, - {&ti, &ti, 0}, - {&bo, &bo, 0}, - {&an, &an, 0}, - {&in, &fl, 0}, - {&fl, &st, 0}, - {&st, &ti, 0}, - {&ti, &bo, 0}, - {&bo, &an, 0}, - {&an, &in, 0}, - {nil, &in, -1}, - {nil, &fl, -1}, - {nil, &st, -1}, - {&ti, nil, 1}, - {&bo, nil, 1}, - {&an, nil, 1}, - {nil, nil, 0}, - } - - for i, p := range params { - t.Run(fmt.Sprintf("data #%d", i), func(t *testing.T) { - assert.Equal(t, p.expected, comp.compare(p.a, p.b)) - }) - } - }) - - t.Run("any desc", func(t *testing.T) { - comp := newAnyComparator("desc") - - params := []struct { - a interface{} - b interface{} - expected int - }{ - {&in, &in, 0}, - {&fl, &fl, 0}, - {&st, &st, 0}, - {&ti, &ti, 0}, - {&bo, &bo, 0}, - {&an, &an, 0}, - {&in, &fl, 0}, - {&fl, &st, 0}, - {&st, &ti, 0}, - {&ti, &bo, 0}, - {&bo, &an, 0}, - {&an, &in, 0}, - {nil, &in, 1}, - {nil, &fl, 1}, - {nil, &st, 1}, - {&ti, nil, -1}, - {&bo, nil, -1}, - {&an, nil, -1}, - {nil, nil, 0}, - } - - for i, p := range params { - t.Run(fmt.Sprintf("data #%d", i), func(t *testing.T) { - assert.Equal(t, p.expected, comp.compare(p.a, p.b)) - }) - } - }) -} diff --git a/adapters/repos/db/sorter/comparable_creator.go b/adapters/repos/db/sorter/comparable_creator.go deleted file mode 100644 index c57f220468bf6f9773c607f72b42ef3dc19967b7..0000000000000000000000000000000000000000 --- a/adapters/repos/db/sorter/comparable_creator.go +++ /dev/null @@ -1,74 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package sorter - -import "github.com/weaviate/weaviate/entities/storobj" - -type comparable struct { - docID uint64 - // all property values that will be used for comparison - // most important 1st, least important last - values []interface{} - // additional payload to hold data related to object, not used in sorting process - payload interface{} -} - -type comparableCreator struct { - extractor *comparableValueExtractor - propNames []string -} - -func newComparableCreator(extractor *comparableValueExtractor, propNames []string) *comparableCreator { - return &comparableCreator{extractor, propNames} -} - -func (c *comparableCreator) createFromBytes(docID uint64, objData []byte) *comparable { - return c.createFromBytesWithPayload(docID, objData, nil) -} - -func (c *comparableCreator) createFromBytesWithPayload(docID uint64, objData []byte, payload interface{}) *comparable { - values := make([]interface{}, len(c.propNames)) - for level, propName := range c.propNames { - values[level] = c.extractor.extractFromBytes(objData, propName) - } - return &comparable{docID, values, payload} -} - -// func (c *comparableCreator) createFromObject(object *storobj.Object) *comparable { -// return c.createFromObjectWithPayload(object, nil) -// } - -func (c *comparableCreator) createFromObjectWithPayload(object *storobj.Object, payload interface{}) *comparable { - values := make([]interface{}, len(c.propNames)) - for level, propName := range c.propNames { - values[level] = c.extractor.extractFromObject(object, propName) - } - return &comparable{object.DocID(), values, payload} -} - -func (c *comparableCreator) extractDocIDs(comparables []*comparable) []uint64 { - docIDs := make([]uint64, len(comparables)) - for i, comparable := range comparables { - docIDs[i] = comparable.docID - } - return docIDs -} - -func (c *comparableCreator) extractPayloads(comparables []*comparable, - consume func(i int, docID uint64, payload interface{}) (stop bool), -) { - for i, comparable := range comparables { - if consume(i, comparable.docID, comparable.payload) { - break - } - } -} diff --git a/adapters/repos/db/sorter/comparable_sorter.go b/adapters/repos/db/sorter/comparable_sorter.go deleted file mode 100644 index 5a75fbbd57b008adec41fb5834fc234a5c789e23..0000000000000000000000000000000000000000 --- a/adapters/repos/db/sorter/comparable_sorter.go +++ /dev/null @@ -1,111 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package sorter - -import ( - "sort" -) - -type comparabeSorter interface { - addComparable(el *comparable) (added bool) - getSorted() []*comparable -} - -// sort elements while adding new one, no following sorting is needed -// if limit applied (>0), only that many elements is in the result -type insertSorter struct { - comparator *comparator - limit int - comparables []*comparable -} - -func newInsertSorter(comparator *comparator, limit int) comparabeSorter { - comparables := make([]*comparable, 0, limit) - return &insertSorter{comparator, limit, comparables} -} - -func (is *insertSorter) addComparable(el *comparable) bool { - count := len(is.comparables) - // insert if there is no limit or limit not reached yet - if is.limit == 0 || count < is.limit { - is.insert(el) - return true - } - // limit reached - compare with last element and insert if "smaller" - // last element can be removed - if is.comparator.compare(el, is.comparables[count-1]) == -1 { - is.comparables = is.comparables[:count-1] - is.insert(el) - return true - } - return false -} - -func (is *insertSorter) insert(el *comparable) { - count := len(is.comparables) - pos := is.findPosition(el, 0, count) - if pos == count { - is.comparables = append(is.comparables, el) - return - } - is.comparables = append(is.comparables[:pos+1], is.comparables[pos:]...) - is.comparables[pos] = el -} - -func (is *insertSorter) findPosition(el *comparable, startInc, endExc int) int { - if startInc == endExc { - return startInc - } - - middle := startInc + (endExc-startInc)/2 - if is.comparator.compare(el, is.comparables[middle]) != -1 { - return is.findPosition(el, middle+1, endExc) - } - return is.findPosition(el, startInc, middle) -} - -func (is *insertSorter) getSorted() []*comparable { - return is.comparables -} - -// implementation of sort.Interface -// sorting is performed in getSorted() method -type defaultSorter struct { - comparator *comparator - comparables []*comparable -} - -func newDefaultSorter(comparator *comparator, cap int) comparabeSorter { - return &defaultSorter{comparator, make([]*comparable, 0, cap)} -} - -func (ds *defaultSorter) addComparable(el *comparable) bool { - ds.comparables = append(ds.comparables, el) - return true -} - -func (ds *defaultSorter) getSorted() []*comparable { - sort.Sort(ds) - return ds.comparables -} - -func (ds *defaultSorter) Len() int { - return len(ds.comparables) -} - -func (ds *defaultSorter) Swap(i, j int) { - ds.comparables[i], ds.comparables[j] = ds.comparables[j], ds.comparables[i] -} - -func (ds *defaultSorter) Less(i, j int) bool { - return ds.comparator.compare(ds.comparables[i], ds.comparables[j]) == -1 -} diff --git a/adapters/repos/db/sorter/comparable_value_extractor.go b/adapters/repos/db/sorter/comparable_value_extractor.go deleted file mode 100644 index 954542f092fafc8579171132bd2492ccd3fbdbae..0000000000000000000000000000000000000000 --- a/adapters/repos/db/sorter/comparable_value_extractor.go +++ /dev/null @@ -1,212 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package sorter - -import ( - "encoding/json" - "strconv" - "time" - - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/storobj" -) - -type comparableValueExtractor struct { - dataTypesHelper *dataTypesHelper -} - -func newComparableValueExtractor(dataTypesHelper *dataTypesHelper) *comparableValueExtractor { - return &comparableValueExtractor{dataTypesHelper} -} - -func (e *comparableValueExtractor) extractFromBytes(objData []byte, propName string) interface{} { - value, success, _ := storobj.ParseAndExtractProperty(objData, propName) - // in case the property does not exist for the object return nil - if len(value) == 0 { - return nil - } - if success { - switch e.dataTypesHelper.getType(propName) { - case schema.DataTypeBlob: - return &value[0] - case schema.DataTypeText: - return &value[0] - case schema.DataTypeTextArray: - return &value - case schema.DataTypeDate: - d := e.mustExtractDates(value[:1])[0] - return &d - case schema.DataTypeDateArray: - da := e.mustExtractDates(value) - return &da - case schema.DataTypeNumber, schema.DataTypeInt: - n := e.mustExtractNumbers(value[:1])[0] - return &n - case schema.DataTypeNumberArray, schema.DataTypeIntArray: - na := e.mustExtractNumbers(value) - return &na - case schema.DataTypeBoolean: - b := e.mustExtractBools(value[:1])[0] - return &b - case schema.DataTypeBooleanArray: - ba := e.mustExtractBools(value) - return &ba - case schema.DataTypePhoneNumber: - fa := e.toFloatArrayFromPhoneNumber(e.mustExtractPhoneNumber(value)) - return &fa - case schema.DataTypeGeoCoordinates: - fa := e.toFloatArrayFromGeoCoordinates(e.mustExtractGeoCoordinates(value)) - return &fa - default: - return nil - } - } - return nil -} - -func (e *comparableValueExtractor) extractFromObject(object *storobj.Object, propName string) interface{} { - if propName == filters.InternalPropID || propName == filters.InternalPropBackwardsCompatID { - id := object.ID().String() - return &id - } - if propName == filters.InternalPropCreationTimeUnix { - ts := float64(object.CreationTimeUnix()) - return &ts - } - if propName == filters.InternalPropLastUpdateTimeUnix { - ts := float64(object.LastUpdateTimeUnix()) - return &ts - } - - propertiesMap, ok := object.Properties().(map[string]interface{}) - if !ok { - return nil - } - value, ok := propertiesMap[propName] - if !ok { - return nil - } - - switch e.dataTypesHelper.getType(propName) { - case schema.DataTypeBlob: - s := value.(string) - return &s - case schema.DataTypeText: - s := value.(string) - return &s - case schema.DataTypeTextArray: - sa := value.([]string) - return &sa - case schema.DataTypeDate: - d := e.mustExtractDates([]string{value.(string)})[0] - return &d - case schema.DataTypeDateArray: - da := e.mustExtractDates(value.([]string)) - return &da - case schema.DataTypeNumber, schema.DataTypeInt: - n := value.(float64) - return &n - case schema.DataTypeNumberArray, schema.DataTypeIntArray: - na := value.([]float64) - return &na - case schema.DataTypeBoolean: - b := value.(bool) - return &b - case schema.DataTypeBooleanArray: - ba := value.([]bool) - return &ba - case schema.DataTypePhoneNumber: - fa := e.toFloatArrayFromPhoneNumber(value.(*models.PhoneNumber)) - return &fa - case schema.DataTypeGeoCoordinates: - fa := e.toFloatArrayFromGeoCoordinates(value.(*models.GeoCoordinates)) - return &fa - default: - return nil - } -} - -func (e *comparableValueExtractor) mustExtractNumbers(value []string) []float64 { - numbers := make([]float64, len(value)) - for i := range value { - number, err := strconv.ParseFloat(value[i], 64) - if err != nil { - panic("sorter: not a number") - } - numbers[i] = number - } - return numbers -} - -func (e *comparableValueExtractor) mustExtractBools(value []string) []bool { - bools := make([]bool, len(value)) - for i := range value { - switch value[i] { - case "true": - bools[i] = true - case "false": - bools[i] = false - default: - panic("sorter: not a bool") - } - } - return bools -} - -func (e *comparableValueExtractor) mustExtractDates(value []string) []time.Time { - dates := make([]time.Time, len(value)) - for i := range value { - date, err := time.Parse(time.RFC3339, value[i]) - if err != nil { - panic("sorter: not a date") - } - dates[i] = date - } - return dates -} - -func (e *comparableValueExtractor) mustExtractPhoneNumber(value []string) *models.PhoneNumber { - if len(value) == 1 { - var phoneNumber *models.PhoneNumber - if err := json.Unmarshal([]byte(value[0]), &phoneNumber); err == nil { - return phoneNumber - } - } - panic("sorter: not a phone number") -} - -func (e *comparableValueExtractor) mustExtractGeoCoordinates(value []string) *models.GeoCoordinates { - if len(value) == 1 { - var geoCoordinates *models.GeoCoordinates - if err := json.Unmarshal([]byte(value[0]), &geoCoordinates); err == nil { - return geoCoordinates - } - } - panic("sorter: not a geo coordinates") -} - -func (e *comparableValueExtractor) toFloatArrayFromPhoneNumber(value *models.PhoneNumber) []float64 { - return []float64{float64(value.CountryCode), float64(value.National)} -} - -func (e *comparableValueExtractor) toFloatArrayFromGeoCoordinates(value *models.GeoCoordinates) []float64 { - fa := make([]float64, 2) - if value.Longitude != nil { - fa[0] = float64(*value.Longitude) - } - if value.Latitude != nil { - fa[1] = float64(*value.Latitude) - } - return fa -} diff --git a/adapters/repos/db/sorter/comparable_value_extractor_test.go b/adapters/repos/db/sorter/comparable_value_extractor_test.go deleted file mode 100644 index 06c90e4fc67619f98e758035d66f7995f4084d07..0000000000000000000000000000000000000000 --- a/adapters/repos/db/sorter/comparable_value_extractor_test.go +++ /dev/null @@ -1,137 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package sorter - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestComparableValueExtractor(t *testing.T) { - schema := getMyFavoriteClassSchemaForTests() - class := schema.GetClass(testClassName) - helper := newDataTypesHelper(class) - extractor := newComparableValueExtractor(helper) - object := createMyFavoriteClassObject() - - params := []struct { - propName string - expected interface{} - }{ - { - "id", - ptrString("73f2eb5f-5abf-447a-81ca-74b1dd168247"), - }, - { - "_creationTimeUnix", - ptrFloat64(900000000001), - }, - { - "_lastUpdateTimeUnix", - ptrFloat64(900000000002), - }, - { - "textProp", - ptrString("text"), - }, - { - "textPropArray", - ptrStringArray("text", "text"), - }, - { - "intProp", - ptrFloat64(100), - }, - { - "numberProp", - ptrFloat64(17), - }, - { - "intPropArray", - ptrFloat64Array(10, 20, 30), - }, - { - "numberPropArray", - ptrFloat64Array(1, 2, 3), - }, - { - "boolProp", - ptrBool(true), - }, - { - "boolPropArray", - ptrBoolArray(true, false, true), - }, - { - "dateProp", - ptrTime("1980-01-01T00:00:00+02:00"), - }, - { - "datePropArray", - ptrTimeArray("1980-01-01T00:00:00+02:00"), - }, - { - "phoneProp", - ptrFloat64Array(49, 1000000), - }, - { - "geoProp", - ptrFloat64Array(1, 2), - }, - { - "emptyStringProp", - nil, - }, - { - "emptyBoolProp", - nil, - }, - { - "emptyNumberProp", - nil, - }, - { - "emptyIntProp", - nil, - }, - { - "crefProp", - nil, - }, - { - "nonExistentProp", - nil, - }, - } - - t.Run("extract comparable values from binary", func(t *testing.T) { - objData, err := object.MarshalBinary() - require.Nil(t, err) - - for _, p := range params { - t.Run(fmt.Sprintf("data %s", p.propName), func(t *testing.T) { - assert.Equal(t, p.expected, extractor.extractFromBytes(objData, p.propName)) - }) - } - }) - - t.Run("extract comparable values from object", func(t *testing.T) { - for _, p := range params { - t.Run(fmt.Sprintf("data %s", p.propName), func(t *testing.T) { - assert.Equal(t, p.expected, extractor.extractFromObject(object, p.propName)) - }) - } - }) -} diff --git a/adapters/repos/db/sorter/comparator.go b/adapters/repos/db/sorter/comparator.go deleted file mode 100644 index 78420b153cc25220cd0c899aef81e3845a0cb16c..0000000000000000000000000000000000000000 --- a/adapters/repos/db/sorter/comparator.go +++ /dev/null @@ -1,35 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package sorter - -type comparator struct { - comparators []basicComparator -} - -func newComparator(dataTypesHelper *dataTypesHelper, propNames []string, orders []string) *comparator { - provider := &basicComparatorProvider{} - comparators := make([]basicComparator, len(propNames)) - for level, propName := range propNames { - dataType := dataTypesHelper.getType(propName) - comparators[level] = provider.provide(dataType, orders[level]) - } - return &comparator{comparators} -} - -func (c *comparator) compare(a, b *comparable) int { - for level, comparator := range c.comparators { - if res := comparator.compare(a.values[level], b.values[level]); res != 0 { - return res - } - } - return 0 -} diff --git a/adapters/repos/db/sorter/datatypes_helper.go b/adapters/repos/db/sorter/datatypes_helper.go deleted file mode 100644 index 395a0191ad818206e3e198a6fb991187d5cd6346..0000000000000000000000000000000000000000 --- a/adapters/repos/db/sorter/datatypes_helper.go +++ /dev/null @@ -1,59 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package sorter - -import ( - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" -) - -type dataTypesHelper struct { - class *models.Class - dataTypes map[string][]string -} - -func newDataTypesHelper(class *models.Class) *dataTypesHelper { - return &dataTypesHelper{class, make(map[string][]string)} -} - -func (h *dataTypesHelper) getStrings(propName string) []string { - if dataType, ok := h.dataTypes[propName]; ok { - return dataType - } - - h.dataTypes[propName] = h.find(propName) - return h.dataTypes[propName] -} - -func (h *dataTypesHelper) find(propName string) []string { - if propName == filters.InternalPropID || propName == filters.InternalPropBackwardsCompatID { - return schema.DataTypeText.PropString() - } - if propName == filters.InternalPropCreationTimeUnix || propName == filters.InternalPropLastUpdateTimeUnix { - return []string{string(schema.DataTypeInt)} - } - for _, property := range h.class.Properties { - if property.Name == propName { - return property.DataType - } - } - return nil -} - -func (h *dataTypesHelper) getType(propName string) schema.DataType { - strings := h.getStrings(propName) - if len(strings) > 0 { - return schema.DataType(strings[0]) - } - return "" -} diff --git a/adapters/repos/db/sorter/datatypes_helper_test.go b/adapters/repos/db/sorter/datatypes_helper_test.go deleted file mode 100644 index 1f8d745a82d6437dda5f3820048c2245e29aae4a..0000000000000000000000000000000000000000 --- a/adapters/repos/db/sorter/datatypes_helper_test.go +++ /dev/null @@ -1,80 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package sorter - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/entities/schema" -) - -func TestDataTypesHelper(t *testing.T) { - sch := getMyFavoriteClassSchemaForTests() - class := sch.GetClass(testClassName) - helper := newDataTypesHelper(class) - - t.Run("get data types as strings", func(t *testing.T) { - params := []struct { - propName string - expected []string - }{ - {"textProp", []string{string(schema.DataTypeText)}}, - {"textPropArray", []string{string(schema.DataTypeTextArray)}}, - {"intProp", []string{string(schema.DataTypeInt)}}, - {"numberProp", []string{string(schema.DataTypeNumber)}}, - {"intPropArray", []string{string(schema.DataTypeIntArray)}}, - {"numberPropArray", []string{string(schema.DataTypeNumberArray)}}, - {"boolProp", []string{string(schema.DataTypeBoolean)}}, - {"boolPropArray", []string{string(schema.DataTypeBooleanArray)}}, - {"dateProp", []string{string(schema.DataTypeDate)}}, - {"datePropArray", []string{string(schema.DataTypeDateArray)}}, - {"phoneProp", []string{string(schema.DataTypePhoneNumber)}}, - {"geoProp", []string{string(schema.DataTypeGeoCoordinates)}}, - {"crefProp", []string{string(schema.DataTypeCRef)}}, - } - - for i, p := range params { - t.Run(fmt.Sprintf("data #%d", i), func(t *testing.T) { - assert.Equal(t, p.expected, helper.getStrings(p.propName)) - }) - } - }) - - t.Run("get data types as type", func(t *testing.T) { - params := []struct { - propName string - expected schema.DataType - }{ - {"textProp", schema.DataTypeText}, - {"textPropArray", schema.DataTypeTextArray}, - {"intProp", schema.DataTypeInt}, - {"numberProp", schema.DataTypeNumber}, - {"intPropArray", schema.DataTypeIntArray}, - {"numberPropArray", schema.DataTypeNumberArray}, - {"boolProp", schema.DataTypeBoolean}, - {"boolPropArray", schema.DataTypeBooleanArray}, - {"dateProp", schema.DataTypeDate}, - {"datePropArray", schema.DataTypeDateArray}, - {"phoneProp", schema.DataTypePhoneNumber}, - {"geoProp", schema.DataTypeGeoCoordinates}, - {"crefProp", schema.DataTypeCRef}, - } - - for i, p := range params { - t.Run(fmt.Sprintf("data #%d", i), func(t *testing.T) { - assert.Equal(t, p.expected, helper.getType(p.propName)) - }) - } - }) -} diff --git a/adapters/repos/db/sorter/fakes_for_test.go b/adapters/repos/db/sorter/fakes_for_test.go deleted file mode 100644 index c42156f5446eab4cbee651abd4c8ff0077bdccfe..0000000000000000000000000000000000000000 --- a/adapters/repos/db/sorter/fakes_for_test.go +++ /dev/null @@ -1,410 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package sorter - -import ( - "time" - - "github.com/go-openapi/strfmt" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/storobj" -) - -const testClassName = "MyFavoriteClass" - -func getMyFavoriteClassSchemaForTests() schema.Schema { - return schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{ - { - Class: testClassName, - Properties: []*models.Property{ - { - Name: "textProp", - DataType: []string{string(schema.DataTypeText)}, - }, - { - Name: "textPropArray", - DataType: []string{string(schema.DataTypeTextArray)}, - }, - { - Name: "intProp", - DataType: []string{string(schema.DataTypeInt)}, - }, - { - Name: "numberProp", - DataType: []string{string(schema.DataTypeNumber)}, - }, - { - Name: "intPropArray", - DataType: []string{string(schema.DataTypeIntArray)}, - }, - { - Name: "numberPropArray", - DataType: []string{string(schema.DataTypeNumberArray)}, - }, - { - Name: "boolProp", - DataType: []string{string(schema.DataTypeBoolean)}, - }, - { - Name: "boolPropArray", - DataType: []string{string(schema.DataTypeBooleanArray)}, - }, - { - Name: "dateProp", - DataType: []string{string(schema.DataTypeDate)}, - }, - { - Name: "datePropArray", - DataType: []string{string(schema.DataTypeDateArray)}, - }, - { - Name: "phoneProp", - DataType: []string{string(schema.DataTypePhoneNumber)}, - }, - { - Name: "geoProp", - DataType: []string{string(schema.DataTypeGeoCoordinates)}, - }, - { - Name: "emptyStringProp", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "emptyBoolProp", - DataType: []string{string(schema.DataTypeBoolean)}, - }, - { - Name: "emptyNumberProp", - DataType: []string{string(schema.DataTypeNumber)}, - }, - { - Name: "emptyIntProp", - DataType: []string{string(schema.DataTypeInt)}, - }, - { - Name: "crefProp", - DataType: []string{string(schema.DataTypeCRef)}, - }, - }, - }, - }, - }, - } -} - -func createMyFavoriteClassObject() *storobj.Object { - return storobj.FromObject( - &models.Object{ - Class: testClassName, - CreationTimeUnix: 900000000001, - LastUpdateTimeUnix: 900000000002, - ID: strfmt.UUID("73f2eb5f-5abf-447a-81ca-74b1dd168247"), - Properties: map[string]interface{}{ - "textProp": "text", - "textPropArray": []string{"text", "text"}, - "intProp": float64(100), - "numberProp": float64(17), - "intPropArray": []float64{10, 20, 30}, - "numberPropArray": []float64{1, 2, 3}, - "boolProp": true, - "boolPropArray": []bool{true, false, true}, - "dateProp": "1980-01-01T00:00:00+02:00", - "datePropArray": []string{"1980-01-01T00:00:00+02:00"}, - "phoneProp": &models.PhoneNumber{ - CountryCode: 49, - DefaultCountry: "DE", - Input: "0171 1000000", - Valid: true, - InternationalFormatted: "+49 171 1000000", - National: 1000000, - NationalFormatted: "0171 1000000", - }, - "geoProp": &models.GeoCoordinates{ - Longitude: ptrFloat32(1), - Latitude: ptrFloat32(2), - }, - }, - }, - []float32{1, 2, 0.7}, - ) -} - -func sorterCitySchema() schema.Schema { - return schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{ - { - Class: "City", - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "country", - DataType: []string{string(schema.DataTypeText)}, - }, - { - Name: "population", - DataType: []string{string(schema.DataTypeInt)}, - }, - { - Name: "cityArea", - DataType: []string{string(schema.DataTypeNumber)}, - }, - { - Name: "cityRights", - DataType: []string{string(schema.DataTypeDate)}, - }, - { - Name: "timezones", - DataType: schema.DataTypeTextArray.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "timezonesUTC", - DataType: []string{string(schema.DataTypeTextArray)}, - }, - { - Name: "isCapital", - DataType: []string{string(schema.DataTypeBoolean)}, - }, - { - Name: "isCapitalArray", - DataType: []string{string(schema.DataTypeBooleanArray)}, - }, - { - Name: "favoriteNumbers", - DataType: []string{string(schema.DataTypeNumberArray)}, - }, - { - Name: "favoriteInts", - DataType: []string{string(schema.DataTypeIntArray)}, - }, - { - Name: "favoriteDates", - DataType: []string{string(schema.DataTypeDateArray)}, - }, - { - Name: "phoneNumber", - DataType: []string{string(schema.DataTypePhoneNumber)}, - }, - { - Name: "location", - DataType: []string{string(schema.DataTypeGeoCoordinates)}, - }, - }, - }, - }, - }, - } -} - -func sorterCitySchemaDistances() []float32 { - return []float32{0.1, 0.0, 0.2, 0.3, 0.4, 0.0} -} - -func sorterCitySchemaObjects() []*storobj.Object { - return []*storobj.Object{cityWroclaw, cityNil2, cityBerlin, cityNewYork, cityAmsterdam, cityNil} -} - -var ( - cityWroclaw = &storobj.Object{ - Object: models.Object{ - Class: "City", - ID: strfmt.UUID("f10018a7-ad67-4774-a9ac-86a04df51cb6"), - CreationTimeUnix: 9000000006, - LastUpdateTimeUnix: 9100000006, - Properties: map[string]interface{}{ - "name": "Wroclaw", - "country": "Poland", - "population": float64(641928), - "cityArea": float64(292.23), - "cityRights": "1214-01-01T00:00:00+02:00", - "timezones": []string{"CET", "CEST"}, - "timezonesUTC": []string{"UTC+1", "UTC+2"}, - "isCapital": false, - "isCapitalArray": []bool{false, false}, - "favoriteNumbers": []float64{0, 0, 0}, - "favoriteInts": []float64{0, 0, 0}, - "favoriteDates": []string{"1214-01-01T00:00:00+02:00", "1214-01-01T00:00:00+02:00"}, - "phoneNumber": &models.PhoneNumber{ - CountryCode: 0, - National: 400500600, - }, - "location": &models.GeoCoordinates{ - Latitude: ptrFloat32(51.11), - Longitude: ptrFloat32(17.022222), - }, - }, - }, - } - cityBerlin = &storobj.Object{ - Object: models.Object{ - Class: "City", - ID: strfmt.UUID("b06bb8a7-ad67-4774-a9ac-86a04df51cb6"), - CreationTimeUnix: 9000000002, - LastUpdateTimeUnix: 9100000002, - Properties: map[string]interface{}{ - "name": "Berlin", - "country": "Germany", - "population": float64(3664088), - "cityArea": float64(891.95), - "cityRights": "1400-01-01T00:00:00+02:00", - "timezones": []string{"CET", "CEST"}, - "timezonesUTC": []string{"UTC+1", "UTC+2"}, - "isCapital": true, - "isCapitalArray": []bool{false, false, true}, - "favoriteNumbers": []float64{0, 10, 1}, - "favoriteInts": []float64{0, 10, 1}, - "favoriteDates": []string{"1400-01-01T00:00:00+02:00"}, - "phoneNumber": &models.PhoneNumber{ - CountryCode: 33, - National: 400500610, - }, - "location": &models.GeoCoordinates{ - Latitude: ptrFloat32(52.518611), - Longitude: ptrFloat32(13.408333), - }, - }, - }, - } - cityNewYork = &storobj.Object{ - Object: models.Object{ - Class: "City", - ID: strfmt.UUID("e06bb8a7-ad67-4774-a9ac-86a04df51cb6"), - CreationTimeUnix: 9000000003, - LastUpdateTimeUnix: 9100000003, - Properties: map[string]interface{}{ - "name": "New York", - "country": "USA", - "population": float64(8336817), - "cityArea": float64(1223.59), - "cityRights": "1653-01-01T00:00:00+02:00", - "timezones": []string{"EST", "EDT"}, - "timezonesUTC": []string{"UTC-5", "UTC-4"}, - "isCapital": false, - "isCapitalArray": []bool{true, true, true}, - "favoriteNumbers": []float64{-100000.23, -8.909}, - "favoriteInts": []float64{-100000, -8}, - "favoriteDates": []string{"1400-01-01T00:00:00+02:00", "1653-01-01T00:00:00+02:00"}, - "phoneNumber": &models.PhoneNumber{ - CountryCode: 33, - National: 400500609, - }, - "location": &models.GeoCoordinates{ - Latitude: ptrFloat32(40.716667), - Longitude: ptrFloat32(-74), - }, - }, - }, - } - cityAmsterdam = &storobj.Object{ - Object: models.Object{ - Class: "City", - ID: strfmt.UUID("a06bb8a7-ad67-4774-a9ac-86a04df51cb6"), - CreationTimeUnix: 9000000001, - LastUpdateTimeUnix: 9100000001, - Properties: map[string]interface{}{ - "name": "Amsterdam", - "country": "The Netherlands", - "population": float64(905234), - "cityArea": float64(219.32), - "cityRights": "1100-01-01T00:00:00+02:00", - "timezones": []string{"CET", "CEST"}, - "timezonesUTC": []string{"UTC+1", "UTC+2"}, - "isCapital": true, - "isCapitalArray": []bool{true}, - "favoriteNumbers": []float64{1, 2, 3, 4, 5, 6, 8.8, 9.9}, - "favoriteInts": []float64{1, 2, 3, 4, 5, 6, 8, 9}, - "favoriteDates": []string{"1100-01-01T00:00:00+02:00"}, - "phoneNumber": &models.PhoneNumber{ - CountryCode: 33, - National: 400500602, - }, - "location": &models.GeoCoordinates{ - Latitude: ptrFloat32(52.366667), - Longitude: ptrFloat32(4.9), - }, - }, - }, - } - cityNil = &storobj.Object{ - Object: models.Object{ - Class: "City", - ID: strfmt.UUID("f00018a7-ad67-4774-a9ac-86a04df51cb6"), - CreationTimeUnix: 9000000004, - LastUpdateTimeUnix: 9100000004, - Properties: map[string]interface{}{ - "name": "Nil", - }, - }, - } - cityNil2 = &storobj.Object{ - Object: models.Object{ - Class: "City", - ID: strfmt.UUID("f00028a7-ad67-4774-a9ac-86a04df51cb6"), - CreationTimeUnix: 9000000005, - LastUpdateTimeUnix: 9100000005, - Properties: map[string]interface{}{ - "name": "Nil2", - }, - }, - } -) - -func ptrString(s string) *string { - return &s -} - -func ptrStringArray(sa ...string) *[]string { - return &sa -} - -func ptrFloat32(f float32) *float32 { - return &f -} - -func ptrFloat64(f float64) *float64 { - return &f -} - -func ptrFloat64Array(fa ...float64) *[]float64 { - return &fa -} - -func ptrBool(b bool) *bool { - return &b -} - -func ptrBoolArray(ba ...bool) *[]bool { - return &ba -} - -func ptrTime(s string) *time.Time { - t, _ := time.Parse(time.RFC3339, s) - return &t -} - -func ptrTimeArray(sa ...string) *[]time.Time { - res := make([]time.Time, len(sa)) - for i := range sa { - t, _ := time.Parse(time.RFC3339, sa[i]) - res[i] = t - } - return &res -} diff --git a/adapters/repos/db/sorter/lsm_sorter.go b/adapters/repos/db/sorter/lsm_sorter.go deleted file mode 100644 index e6c875b2a1aa752d576da09c934b7e75f5ac890a..0000000000000000000000000000000000000000 --- a/adapters/repos/db/sorter/lsm_sorter.go +++ /dev/null @@ -1,174 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package sorter - -import ( - "context" - "encoding/binary" - "fmt" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/storobj" -) - -type LSMSorter interface { - Sort(ctx context.Context, limit int, sort []filters.Sort) ([]uint64, error) - SortDocIDs(ctx context.Context, limit int, sort []filters.Sort, ids helpers.AllowList) ([]uint64, error) - SortDocIDsAndDists(ctx context.Context, limit int, sort []filters.Sort, - ids []uint64, dists []float32) ([]uint64, []float32, error) -} - -type lsmSorter struct { - bucket *lsmkv.Bucket - dataTypesHelper *dataTypesHelper - valueExtractor *comparableValueExtractor -} - -func NewLSMSorter(store *lsmkv.Store, sch schema.Schema, className schema.ClassName) (LSMSorter, error) { - bucket := store.Bucket(helpers.ObjectsBucketLSM) - if bucket == nil { - return nil, fmt.Errorf("lsm sorter - bucket %s for class %s not found", helpers.ObjectsBucketLSM, className) - } - class := sch.GetClass(schema.ClassName(className)) - if class == nil { - return nil, fmt.Errorf("lsm sorter - class %s not found", className) - } - dataTypesHelper := newDataTypesHelper(class) - comparableValuesExtractor := newComparableValueExtractor(dataTypesHelper) - - return &lsmSorter{bucket, dataTypesHelper, comparableValuesExtractor}, nil -} - -func (s *lsmSorter) Sort(ctx context.Context, limit int, sort []filters.Sort) ([]uint64, error) { - helper, err := s.createHelper(sort, validateLimit(limit, s.bucket.Count())) - if err != nil { - return nil, err - } - return helper.getSorted(ctx) -} - -func (s *lsmSorter) SortDocIDs(ctx context.Context, limit int, sort []filters.Sort, ids helpers.AllowList) ([]uint64, error) { - helper, err := s.createHelper(sort, validateLimit(limit, ids.Len())) - if err != nil { - return nil, err - } - return helper.getSortedDocIDs(ctx, ids) -} - -func (s *lsmSorter) SortDocIDsAndDists(ctx context.Context, limit int, sort []filters.Sort, - ids []uint64, dists []float32, -) ([]uint64, []float32, error) { - helper, err := s.createHelper(sort, validateLimit(limit, len(ids))) - if err != nil { - return nil, nil, err - } - return helper.getSortedDocIDsAndDistances(ctx, ids, dists) -} - -func (s *lsmSorter) createHelper(sort []filters.Sort, limit int) (*lsmSorterHelper, error) { - propNames, orders, err := extractPropNamesAndOrders(sort) - if err != nil { - return nil, err - } - - comparator := newComparator(s.dataTypesHelper, propNames, orders) - creator := newComparableCreator(s.valueExtractor, propNames) - return newLsmSorterHelper(s.bucket, comparator, creator, limit), nil -} - -type lsmSorterHelper struct { - bucket *lsmkv.Bucket - comparator *comparator - creator *comparableCreator - limit int -} - -func newLsmSorterHelper(bucket *lsmkv.Bucket, comparator *comparator, - creator *comparableCreator, limit int, -) *lsmSorterHelper { - return &lsmSorterHelper{bucket, comparator, creator, limit} -} - -func (h *lsmSorterHelper) getSorted(ctx context.Context) ([]uint64, error) { - cursor := h.bucket.Cursor() - defer cursor.Close() - - sorter := newInsertSorter(h.comparator, h.limit) - - for k, objData := cursor.First(); k != nil; k, objData = cursor.Next() { - docID, err := storobj.DocIDFromBinary(objData) - if err != nil { - return nil, errors.Wrapf(err, "lsm sorter - could not get doc id") - } - comparable := h.creator.createFromBytes(docID, objData) - sorter.addComparable(comparable) - } - - return h.creator.extractDocIDs(sorter.getSorted()), nil -} - -func (h *lsmSorterHelper) getSortedDocIDs(ctx context.Context, docIDs helpers.AllowList) ([]uint64, error) { - sorter := newInsertSorter(h.comparator, h.limit) - docIDBytes := make([]byte, 8) - it := docIDs.Iterator() - - for docID, ok := it.Next(); ok; docID, ok = it.Next() { - binary.LittleEndian.PutUint64(docIDBytes, docID) - objData, err := h.bucket.GetBySecondary(0, docIDBytes) - if err != nil { - return nil, errors.Wrapf(err, "lsm sorter - could not get obj by doc id %d", docID) - } - if objData == nil { - continue - } - - comparable := h.creator.createFromBytes(docID, objData) - sorter.addComparable(comparable) - } - - return h.creator.extractDocIDs(sorter.getSorted()), nil -} - -func (h *lsmSorterHelper) getSortedDocIDsAndDistances(ctx context.Context, docIDs []uint64, - distances []float32, -) ([]uint64, []float32, error) { - sorter := newInsertSorter(h.comparator, h.limit) - docIDBytes := make([]byte, 8) - - for i, docID := range docIDs { - binary.LittleEndian.PutUint64(docIDBytes, docID) - objData, err := h.bucket.GetBySecondary(0, docIDBytes) - if err != nil { - return nil, nil, errors.Wrapf(err, "lsm sorter - could not get obj by doc id %d", docID) - } - if objData == nil { - continue - } - - comparable := h.creator.createFromBytesWithPayload(docID, objData, distances[i]) - sorter.addComparable(comparable) - } - - sorted := sorter.getSorted() - sortedDistances := make([]float32, len(sorted)) - consume := func(i int, _ uint64, payload interface{}) bool { - sortedDistances[i] = payload.(float32) - return false - } - h.creator.extractPayloads(sorted, consume) - - return h.creator.extractDocIDs(sorted), sortedDistances, nil -} diff --git a/adapters/repos/db/sorter/objects_sorter.go b/adapters/repos/db/sorter/objects_sorter.go deleted file mode 100644 index 4066d68d19e141a53ee5233fa884e51e1b445a28..0000000000000000000000000000000000000000 --- a/adapters/repos/db/sorter/objects_sorter.go +++ /dev/null @@ -1,109 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package sorter - -import ( - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/storobj" -) - -type Sorter interface { - Sort(objects []*storobj.Object, distances []float32, - limit int, sort []filters.Sort) ([]*storobj.Object, []float32, error) -} - -type objectsSorter struct { - schema schema.Schema -} - -func NewObjectsSorter(schema schema.Schema) *objectsSorter { - return &objectsSorter{schema} -} - -func (s objectsSorter) Sort(objects []*storobj.Object, - scores []float32, limit int, sort []filters.Sort, -) ([]*storobj.Object, []float32, error) { - count := len(objects) - if count == 0 { - return objects, scores, nil - } - - limit = validateLimit(limit, count) - propNames, orders, err := extractPropNamesAndOrders(sort) - if err != nil { - return nil, nil, err - } - - class := s.schema.GetClass(objects[0].Class()) - dataTypesHelper := newDataTypesHelper(class) - valueExtractor := newComparableValueExtractor(dataTypesHelper) - comparator := newComparator(dataTypesHelper, propNames, orders) - creator := newComparableCreator(valueExtractor, propNames) - - return newObjectsSorterHelper(comparator, creator, limit). - sort(objects, scores) -} - -type objectsSorterHelper struct { - comparator *comparator - creator *comparableCreator - limit int -} - -func newObjectsSorterHelper(comparator *comparator, creator *comparableCreator, limit int) *objectsSorterHelper { - return &objectsSorterHelper{comparator, creator, limit} -} - -func (h *objectsSorterHelper) sort(objects []*storobj.Object, distances []float32) ([]*storobj.Object, []float32, error) { - withDistances := len(distances) > 0 - count := len(objects) - sorter := newDefaultSorter(h.comparator, count) - - for i := range objects { - payload := objectDistancePayload{o: objects[i]} - if withDistances { - payload.d = distances[i] - } - comparable := h.creator.createFromObjectWithPayload(objects[i], payload) - sorter.addComparable(comparable) - } - - slice := h.limit - if slice == 0 { - slice = count - } - - sorted := sorter.getSorted() - consume := func(i int, _ uint64, payload interface{}) bool { - if i >= slice { - return true - } - p := payload.(objectDistancePayload) - objects[i] = p.o - if withDistances { - distances[i] = p.d - } - return false - } - h.creator.extractPayloads(sorted, consume) - - if withDistances { - return objects[:slice], distances[:slice], nil - } - return objects[:slice], distances, nil -} - -type objectDistancePayload struct { - o *storobj.Object - d float32 -} diff --git a/adapters/repos/db/sorter/objects_sorter_test.go b/adapters/repos/db/sorter/objects_sorter_test.go deleted file mode 100644 index 213ba9771dba0b0106655f6d4486496f22edc2d4..0000000000000000000000000000000000000000 --- a/adapters/repos/db/sorter/objects_sorter_test.go +++ /dev/null @@ -1,416 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package sorter - -import ( - "reflect" - "testing" - - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/storobj" -) - -func TestObjectsSorter(t *testing.T) { - tests := []struct { - name string - sort []filters.Sort - limit int - wantObjs []*storobj.Object - wantDists []float32 - }{ - { - name: "sort by string asc", - sort: sort1("name", "asc"), - limit: 3, - wantObjs: []*storobj.Object{cityAmsterdam, cityBerlin, cityNewYork, cityNil, cityNil2, cityWroclaw}, - wantDists: []float32{0.4, 0.2, 0.3, 0.0, 0.0, 0.1}, - }, - { - name: "sort by string desc", - sort: sort1("name", "desc"), - limit: 4, - wantObjs: []*storobj.Object{cityWroclaw, cityNil2, cityNil, cityNewYork, cityBerlin, cityAmsterdam}, - wantDists: []float32{0.1, 0.0, 0.0, 0.3, 0.2, 0.4}, - }, - { - name: "sort by text asc", - sort: sort1("country", "asc"), - limit: 5, - wantObjs: []*storobj.Object{cityNil2, cityNil, cityBerlin, cityWroclaw, cityAmsterdam, cityNewYork}, - wantDists: []float32{0.0, 0.0, 0.2, 0.1, 0.4, 0.3}, - }, - { - name: "sort by text desc", - sort: sort1("country", "desc"), - limit: 3, - wantObjs: []*storobj.Object{cityNewYork, cityAmsterdam, cityWroclaw, cityBerlin, cityNil2, cityNil}, - wantDists: []float32{0.3, 0.4, 0.1, 0.2, 0.0, 0.0}, - }, - { - name: "sort by int asc", - sort: sort1("population", "asc"), - limit: 4, - wantObjs: []*storobj.Object{cityNil2, cityNil, cityWroclaw, cityAmsterdam, cityBerlin, cityNewYork}, - wantDists: []float32{0.0, 0.0, 0.1, 0.4, 0.2, 0.3}, - }, - { - name: "sort by int desc", - sort: sort1("population", "desc"), - limit: 5, - wantObjs: []*storobj.Object{cityNewYork, cityBerlin, cityAmsterdam, cityWroclaw, cityNil2, cityNil}, - wantDists: []float32{0.3, 0.2, 0.4, 0.1, 0.0, 0.0}, - }, - { - name: "sort by number asc", - sort: sort1("cityArea", "asc"), - limit: 3, - wantObjs: []*storobj.Object{cityNil2, cityNil, cityAmsterdam, cityWroclaw, cityBerlin, cityNewYork}, - wantDists: []float32{0.0, 0.0, 0.4, 0.1, 0.2, 0.3}, - }, - { - name: "sort by number desc", - sort: sort1("cityArea", "desc"), - limit: 4, - wantObjs: []*storobj.Object{cityNewYork, cityBerlin, cityWroclaw, cityAmsterdam, cityNil2, cityNil}, - wantDists: []float32{0.3, 0.2, 0.1, 0.4, 0.0, 0.0}, - }, - { - name: "sort by date asc", - sort: sort1("cityRights", "asc"), - limit: 5, - wantObjs: []*storobj.Object{cityNil2, cityNil, cityAmsterdam, cityWroclaw, cityBerlin, cityNewYork}, - wantDists: []float32{0.0, 0.0, 0.4, 0.1, 0.2, 0.3}, - }, - { - name: "sort by date desc", - sort: sort1("cityRights", "desc"), - limit: 3, - wantObjs: []*storobj.Object{cityNewYork, cityBerlin, cityWroclaw, cityAmsterdam, cityNil2, cityNil}, - wantDists: []float32{0.3, 0.2, 0.1, 0.4, 0.0, 0.0}, - }, - { - name: "sort by string array asc", - sort: sort1("timezones", "asc"), - limit: 4, - wantObjs: []*storobj.Object{cityNil2, cityNil, cityWroclaw, cityBerlin, cityAmsterdam, cityNewYork}, - wantDists: []float32{0.0, 0.0, 0.1, 0.2, 0.4, 0.3}, - }, - { - name: "sort by string array desc", - sort: sort1("timezones", "desc"), - limit: 5, - wantObjs: []*storobj.Object{cityNewYork, cityWroclaw, cityBerlin, cityAmsterdam, cityNil2, cityNil}, - wantDists: []float32{0.3, 0.1, 0.2, 0.4, 0.0, 0.0}, - }, - { - name: "sort by text array asc", - sort: sort1("timezonesUTC", "asc"), - limit: 3, - wantObjs: []*storobj.Object{cityNil2, cityNil, cityWroclaw, cityBerlin, cityAmsterdam, cityNewYork}, - wantDists: []float32{0.0, 0.0, 0.1, 0.2, 0.4, 0.3}, - }, - { - name: "sort by text array desc", - sort: sort1("timezonesUTC", "desc"), - limit: 4, - wantObjs: []*storobj.Object{cityNewYork, cityWroclaw, cityBerlin, cityAmsterdam, cityNil2, cityNil}, - wantDists: []float32{0.3, 0.1, 0.2, 0.4, 0.0, 0.0}, - }, - { - name: "sort by bool asc", - sort: sort1("isCapital", "asc"), - limit: 5, - wantObjs: []*storobj.Object{cityNil2, cityNil, cityWroclaw, cityNewYork, cityBerlin, cityAmsterdam}, - wantDists: []float32{0.0, 0.0, 0.1, 0.3, 0.2, 0.4}, - }, - { - name: "sort by bool desc", - sort: sort1("isCapital", "desc"), - limit: 3, - wantObjs: []*storobj.Object{cityBerlin, cityAmsterdam, cityWroclaw, cityNewYork, cityNil2, cityNil}, - wantDists: []float32{0.2, 0.4, 0.1, 0.3, 0.0, 0.0}, - }, - { - name: "sort by bool array asc", - sort: sort1("isCapitalArray", "asc"), - limit: 4, - wantObjs: []*storobj.Object{cityNil2, cityNil, cityWroclaw, cityBerlin, cityAmsterdam, cityNewYork}, - wantDists: []float32{0.0, 0.0, 0.1, 0.2, 0.4, 0.3}, - }, - { - name: "sort by bool array desc", - sort: sort1("isCapitalArray", "desc"), - limit: 5, - wantObjs: []*storobj.Object{cityNewYork, cityAmsterdam, cityBerlin, cityWroclaw, cityNil2, cityNil}, - wantDists: []float32{0.3, 0.4, 0.2, 0.1, 0.0, 0.0}, - }, - { - name: "sort by number array asc", - sort: sort1("favoriteNumbers", "asc"), - limit: 3, - wantObjs: []*storobj.Object{cityNil2, cityNil, cityNewYork, cityWroclaw, cityBerlin, cityAmsterdam}, - wantDists: []float32{0.0, 0.0, 0.3, 0.1, 0.2, 0.4}, - }, - { - name: "sort by number array desc", - sort: sort1("favoriteNumbers", "desc"), - limit: 4, - wantObjs: []*storobj.Object{cityAmsterdam, cityBerlin, cityWroclaw, cityNewYork, cityNil2, cityNil}, - wantDists: []float32{0.4, 0.2, 0.1, 0.3, 0.0, 0.0}, - }, - { - name: "sort by int array asc", - sort: sort1("favoriteInts", "asc"), - limit: 5, - wantObjs: []*storobj.Object{cityNil2, cityNil, cityNewYork, cityWroclaw, cityBerlin, cityAmsterdam}, - wantDists: []float32{0.0, 0.0, 0.3, 0.1, 0.2, 0.4}, - }, - { - name: "sort by int array desc", - sort: sort1("favoriteInts", "desc"), - limit: 3, - wantObjs: []*storobj.Object{cityAmsterdam, cityBerlin, cityWroclaw, cityNewYork, cityNil2, cityNil}, - wantDists: []float32{0.4, 0.2, 0.1, 0.3, 0.0, 0.0}, - }, - { - name: "sort by date array asc", - sort: sort1("favoriteDates", "asc"), - limit: 4, - wantObjs: []*storobj.Object{cityNil2, cityNil, cityAmsterdam, cityWroclaw, cityBerlin, cityNewYork}, - wantDists: []float32{0.0, 0.0, 0.4, 0.1, 0.2, 0.3}, - }, - { - name: "sort by date array desc", - sort: sort1("favoriteDates", "desc"), - limit: 5, - wantObjs: []*storobj.Object{cityNewYork, cityBerlin, cityWroclaw, cityAmsterdam, cityNil2, cityNil}, - wantDists: []float32{0.3, 0.2, 0.1, 0.4, 0.0, 0.0}, - }, - { - name: "sort by phoneNumber asc", - sort: sort1("phoneNumber", "asc"), - limit: 3, - wantObjs: []*storobj.Object{cityNil2, cityNil, cityWroclaw, cityAmsterdam, cityNewYork, cityBerlin}, - wantDists: []float32{0.0, 0.0, 0.1, 0.4, 0.3, 0.2}, - }, - { - name: "sort by phoneNumber desc", - sort: sort1("phoneNumber", "desc"), - limit: 4, - wantObjs: []*storobj.Object{cityBerlin, cityNewYork, cityAmsterdam, cityWroclaw, cityNil2, cityNil}, - wantDists: []float32{0.2, 0.3, 0.4, 0.1, 0.0, 0.0}, - }, - { - name: "sort by location asc", - sort: sort1("location", "asc"), - limit: 5, - wantObjs: []*storobj.Object{cityNil2, cityNil, cityNewYork, cityAmsterdam, cityBerlin, cityWroclaw}, - wantDists: []float32{0.0, 0.0, 0.3, 0.4, 0.2, 0.1}, - }, - { - name: "sort by location desc", - sort: sort1("location", "desc"), - limit: 3, - wantObjs: []*storobj.Object{cityWroclaw, cityBerlin, cityAmsterdam, cityNewYork, cityNil2, cityNil}, - wantDists: []float32{0.1, 0.2, 0.4, 0.3, 0.0, 0.0}, - }, - { - name: "sort by special id property asc", - sort: sort1("id", "asc"), - limit: 4, - wantObjs: []*storobj.Object{cityAmsterdam, cityBerlin, cityNewYork, cityNil, cityNil2, cityWroclaw}, - wantDists: []float32{0.4, 0.2, 0.3, 0.0, 0.0, 0.1}, - }, - { - name: "sort by special id property desc", - sort: sort1("id", "desc"), - limit: 5, - wantObjs: []*storobj.Object{cityWroclaw, cityNil2, cityNil, cityNewYork, cityBerlin, cityAmsterdam}, - wantDists: []float32{0.1, 0.0, 0.0, 0.3, 0.2, 0.4}, - }, - { - name: "sort by special _id property asc", - sort: sort1("_id", "asc"), - limit: 3, - wantObjs: []*storobj.Object{cityAmsterdam, cityBerlin, cityNewYork, cityNil, cityNil2, cityWroclaw}, - wantDists: []float32{0.4, 0.2, 0.3, 0.0, 0.0, 0.1}, - }, - { - name: "sort by special _id property desc", - sort: sort1("_id", "desc"), - limit: 4, - wantObjs: []*storobj.Object{cityWroclaw, cityNil2, cityNil, cityNewYork, cityBerlin, cityAmsterdam}, - wantDists: []float32{0.1, 0.0, 0.0, 0.3, 0.2, 0.4}, - }, - { - name: "sort by special _creationTimeUnix property asc", - sort: sort1("_creationTimeUnix", "asc"), - limit: 5, - wantObjs: []*storobj.Object{cityAmsterdam, cityBerlin, cityNewYork, cityNil, cityNil2, cityWroclaw}, - wantDists: []float32{0.4, 0.2, 0.3, 0.0, 0.0, 0.1}, - }, - { - name: "sort by special _creationTimeUnix property desc", - sort: sort1("_creationTimeUnix", "desc"), - limit: 3, - wantObjs: []*storobj.Object{cityWroclaw, cityNil2, cityNil, cityNewYork, cityBerlin, cityAmsterdam}, - wantDists: []float32{0.1, 0.0, 0.0, 0.3, 0.2, 0.4}, - }, - { - name: "sort by special _lastUpdateTimeUnix property asc", - sort: sort1("_lastUpdateTimeUnix", "asc"), - limit: 4, - wantObjs: []*storobj.Object{cityAmsterdam, cityBerlin, cityNewYork, cityNil, cityNil2, cityWroclaw}, - wantDists: []float32{0.4, 0.2, 0.3, 0.0, 0.0, 0.1}, - }, - { - name: "sort by special _lastUpdateTimeUnix property desc", - sort: sort1("_lastUpdateTimeUnix", "desc"), - limit: 5, - wantObjs: []*storobj.Object{cityWroclaw, cityNil2, cityNil, cityNewYork, cityBerlin, cityAmsterdam}, - wantDists: []float32{0.1, 0.0, 0.0, 0.3, 0.2, 0.4}, - }, - { - name: "sort by isCapital asc & name asc", - sort: sort2("isCapital", "asc", "name", "asc"), - limit: 3, - wantObjs: []*storobj.Object{cityNil, cityNil2, cityNewYork, cityWroclaw, cityAmsterdam, cityBerlin}, - wantDists: []float32{0.0, 0.0, 0.3, 0.1, 0.4, 0.2}, - }, - { - name: "sort by isCapital desc & name asc", - sort: sort2("isCapital", "desc", "name", "asc"), - limit: 4, - wantObjs: []*storobj.Object{cityAmsterdam, cityBerlin, cityNewYork, cityWroclaw, cityNil, cityNil2}, - wantDists: []float32{0.4, 0.2, 0.3, 0.1, 0.0, 0.0}, - }, - { - name: "sort by timezones desc & name desc", - sort: sort2("timezones", "desc", "name", "desc"), - limit: 5, - wantObjs: []*storobj.Object{cityNewYork, cityWroclaw, cityBerlin, cityAmsterdam, cityNil2, cityNil}, - wantDists: []float32{0.3, 0.1, 0.2, 0.4, 0.0, 0.0}, - }, - { - name: "sort by timezones desc & name asc", - sort: sort2("timezones", "desc", "name", "asc"), - limit: 3, - wantObjs: []*storobj.Object{cityNewYork, cityAmsterdam, cityBerlin, cityWroclaw, cityNil, cityNil2}, - wantDists: []float32{0.3, 0.4, 0.2, 0.1, 0.0, 0.0}, - }, - { - name: "sort by timezonesUTC asc & timezones desc & isCapital asc & population asc", - sort: sort4("timezonesUTC", "asc", "timezones", "desc", "isCapital", "asc", "population", "asc"), - limit: 4, - wantObjs: []*storobj.Object{cityNil2, cityNil, cityWroclaw, cityAmsterdam, cityBerlin, cityNewYork}, - wantDists: []float32{0.0, 0.0, 0.1, 0.4, 0.2, 0.3}, - }, - { - name: "sort by timezonesUTC asc & timezones desc & isCapital desc & population asc", - sort: sort4("timezonesUTC", "asc", "timezones", "desc", "isCapital", "desc", "population", "asc"), - limit: 5, - wantObjs: []*storobj.Object{cityNil2, cityNil, cityAmsterdam, cityBerlin, cityWroclaw, cityNewYork}, - wantDists: []float32{0.0, 0.0, 0.4, 0.2, 0.1, 0.3}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Run("with distance", func(t *testing.T) { - sorter := NewObjectsSorter(sorterCitySchema()) - gotObjs, gotDists, err := sorter.Sort(sorterCitySchemaObjects(), sorterCitySchemaDistances(), 0, tt.sort) - - require.Nil(t, err) - - if !reflect.DeepEqual(gotObjs, tt.wantObjs) { - t.Fatalf("objects got = %v, want %v", - extractCityNames(gotObjs), extractCityNames(tt.wantObjs)) - } - if !reflect.DeepEqual(gotDists, tt.wantDists) { - t.Fatalf("distances got = %v, want %v", - gotDists, tt.wantDists) - } - }) - - t.Run("without distance", func(t *testing.T) { - sorter := NewObjectsSorter(sorterCitySchema()) - gotObjs, gotDists, err := sorter.Sort(sorterCitySchemaObjects(), nil, 0, tt.sort) - - require.Nil(t, err) - - if !reflect.DeepEqual(gotObjs, tt.wantObjs) { - t.Fatalf("objects got = %v, want %v", - extractCityNames(gotObjs), extractCityNames(tt.wantObjs)) - } - if gotDists != nil { - t.Fatalf("distances got = %v, want nil", - gotDists) - } - }) - - t.Run("with limit", func(t *testing.T) { - sorter := NewObjectsSorter(sorterCitySchema()) - gotObjs, gotDists, err := sorter.Sort(sorterCitySchemaObjects(), sorterCitySchemaDistances(), tt.limit, tt.sort) - - require.Nil(t, err) - - if !reflect.DeepEqual(gotObjs, tt.wantObjs[:tt.limit]) { - t.Fatalf("objects got = %v, want %v", - extractCityNames(gotObjs), extractCityNames(tt.wantObjs)) - } - if !reflect.DeepEqual(gotDists, tt.wantDists[:tt.limit]) { - t.Fatalf("distances got = %v, want %v", - gotDists, tt.wantDists) - } - }) - }) - } -} - -func createSort(property, order string) filters.Sort { - return filters.Sort{Path: []string{property}, Order: order} -} - -func sort1(property, order string) []filters.Sort { - return []filters.Sort{createSort(property, order)} -} - -func sort2(property1, order1, property2, order2 string) []filters.Sort { - return []filters.Sort{ - createSort(property1, order1), - createSort(property2, order2), - } -} - -func sort4(property1, order1, property2, order2, property3, order3, property4, order4 string) []filters.Sort { - return []filters.Sort{ - createSort(property1, order1), - createSort(property2, order2), - createSort(property3, order3), - createSort(property4, order4), - } -} - -func extractCityNames(in []*storobj.Object) []string { - out := make([]string, len(in)) - for i := range in { - if asMap, ok := in[i].Properties().(map[string]interface{}); ok { - for k, v := range asMap { - if k == "name" { - out[i] = v.(string) - } - } - } - } - return out -} diff --git a/adapters/repos/db/sorter/utils.go b/adapters/repos/db/sorter/utils.go deleted file mode 100644 index 154aa51d681a50cd8b00c26af6a4edf931d35b21..0000000000000000000000000000000000000000 --- a/adapters/repos/db/sorter/utils.go +++ /dev/null @@ -1,44 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package sorter - -import ( - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/filters" -) - -func extractPropNamesAndOrders(sort []filters.Sort) ([]string, []string, error) { - propNames := make([]string, len(sort)) - orders := make([]string, len(sort)) - - for i, srt := range sort { - if len(srt.Path) == 0 { - return nil, nil, errors.New("path parameter cannot be empty") - } - if len(srt.Path) > 1 { - return nil, nil, errors.New("sorting by reference not supported, path must have exactly one argument") - } - propNames[i] = srt.Path[0] - orders[i] = srt.Order - } - return propNames, orders, nil -} - -func validateLimit(limit, elementsCount int) int { - if limit > elementsCount { - return elementsCount - } - if limit < 0 { - return 0 - } - return limit -} diff --git a/adapters/repos/db/vector/cache/cache.go b/adapters/repos/db/vector/cache/cache.go deleted file mode 100644 index 18e149cfe562a0cd7960cbea67120282cbb72031..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/cache/cache.go +++ /dev/null @@ -1,34 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package cache - -import ( - "context" - "time" -) - -const DefaultDeletionInterval = 3 * time.Second - -type Cache[T any] interface { - Get(ctx context.Context, id uint64) ([]T, error) - MultiGet(ctx context.Context, ids []uint64) ([][]T, []error) - Len() int32 - CountVectors() int64 - Delete(ctx context.Context, id uint64) - Preload(id uint64, vec []T) - Prefetch(id uint64) - Grow(size uint64) - Drop() - UpdateMaxSize(size int64) - CopyMaxSize() int64 - All() [][]T -} diff --git a/adapters/repos/db/vector/cache/prefetch_amd64.go b/adapters/repos/db/vector/cache/prefetch_amd64.go deleted file mode 100644 index e4cac69d341900a88accb488a4c6c022bfc7301b..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/cache/prefetch_amd64.go +++ /dev/null @@ -1,18 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package cache - -import "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer/asm" - -func init() { - prefetchFunc = asm.Prefetch -} diff --git a/adapters/repos/db/vector/cache/sharded_lock_cache.go b/adapters/repos/db/vector/cache/sharded_lock_cache.go deleted file mode 100644 index 6005737c7b1c0b1238b672c78b5399e05d43ca82..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/cache/sharded_lock_cache.go +++ /dev/null @@ -1,300 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package cache - -import ( - "context" - "sync" - "sync/atomic" - "time" - "unsafe" - - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/adapters/repos/db/vector/common" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" -) - -type shardedLockCache[T float32 | byte | uint64] struct { - shardedLocks *common.ShardedLocks - cache [][]T - vectorForID common.VectorForID[T] - normalizeOnRead bool - maxSize int64 - count int64 - cancel chan bool - logger logrus.FieldLogger - deletionInterval time.Duration - - // The maintenanceLock makes sure that only one maintenance operation, such - // as growing the cache or clearing the cache happens at the same time. - maintenanceLock sync.RWMutex -} - -const ( - InitialSize = 1000 - MinimumIndexGrowthDelta = 2000 - indexGrowthRate = 1.25 -) - -func NewShardedFloat32LockCache(vecForID common.VectorForID[float32], maxSize int, - logger logrus.FieldLogger, normalizeOnRead bool, deletionInterval time.Duration, -) Cache[float32] { - vc := &shardedLockCache[float32]{ - vectorForID: func(ctx context.Context, id uint64) ([]float32, error) { - vec, err := vecForID(ctx, id) - if err != nil { - return nil, err - } - if normalizeOnRead { - vec = distancer.Normalize(vec) - } - return vec, nil - }, - cache: make([][]float32, InitialSize), - normalizeOnRead: normalizeOnRead, - count: 0, - maxSize: int64(maxSize), - cancel: make(chan bool), - logger: logger, - shardedLocks: common.NewDefaultShardedLocks(), - maintenanceLock: sync.RWMutex{}, - deletionInterval: deletionInterval, - } - - vc.watchForDeletion() - return vc -} - -func NewShardedByteLockCache(vecForID common.VectorForID[byte], maxSize int, - logger logrus.FieldLogger, deletionInterval time.Duration, -) Cache[byte] { - vc := &shardedLockCache[byte]{ - vectorForID: vecForID, - cache: make([][]byte, InitialSize), - normalizeOnRead: false, - count: 0, - maxSize: int64(maxSize), - cancel: make(chan bool), - logger: logger, - shardedLocks: common.NewDefaultShardedLocks(), - maintenanceLock: sync.RWMutex{}, - deletionInterval: deletionInterval, - } - - vc.watchForDeletion() - return vc -} - -func NewShardedUInt64LockCache(vecForID common.VectorForID[uint64], maxSize int, - logger logrus.FieldLogger, deletionInterval time.Duration, -) Cache[uint64] { - vc := &shardedLockCache[uint64]{ - vectorForID: vecForID, - cache: make([][]uint64, InitialSize), - normalizeOnRead: false, - count: 0, - maxSize: int64(maxSize), - cancel: make(chan bool), - logger: logger, - shardedLocks: common.NewDefaultShardedLocks(), - maintenanceLock: sync.RWMutex{}, - deletionInterval: deletionInterval, - } - - vc.watchForDeletion() - return vc -} - -func (s *shardedLockCache[T]) All() [][]T { - return s.cache -} - -func (s *shardedLockCache[T]) Get(ctx context.Context, id uint64) ([]T, error) { - s.shardedLocks.RLock(id) - vec := s.cache[id] - s.shardedLocks.RUnlock(id) - - if vec != nil { - return vec, nil - } - - return s.handleCacheMiss(ctx, id) -} - -func (s *shardedLockCache[T]) Delete(ctx context.Context, id uint64) { - s.shardedLocks.Lock(id) - defer s.shardedLocks.Unlock(id) - - if int(id) >= len(s.cache) || s.cache[id] == nil { - return - } - - s.cache[id] = nil - atomic.AddInt64(&s.count, -1) -} - -func (s *shardedLockCache[T]) handleCacheMiss(ctx context.Context, id uint64) ([]T, error) { - vec, err := s.vectorForID(ctx, id) - if err != nil { - return nil, err - } - - atomic.AddInt64(&s.count, 1) - s.shardedLocks.Lock(id) - s.cache[id] = vec - s.shardedLocks.Unlock(id) - - return vec, nil -} - -func (s *shardedLockCache[T]) MultiGet(ctx context.Context, ids []uint64) ([][]T, []error) { - out := make([][]T, len(ids)) - errs := make([]error, len(ids)) - - for i, id := range ids { - s.shardedLocks.RLock(id) - vec := s.cache[id] - s.shardedLocks.RUnlock(id) - - if vec == nil { - vecFromDisk, err := s.handleCacheMiss(ctx, id) - errs[i] = err - vec = vecFromDisk - } - - out[i] = vec - } - - return out, errs -} - -var prefetchFunc func(in uintptr) = func(in uintptr) { - // do nothing on default arch - // this function will be overridden for amd64 -} - -func (s *shardedLockCache[T]) Prefetch(id uint64) { - s.shardedLocks.RLock(id) - defer s.shardedLocks.RUnlock(id) - - prefetchFunc(uintptr(unsafe.Pointer(&s.cache[id]))) -} - -func (s *shardedLockCache[T]) Preload(id uint64, vec []T) { - s.shardedLocks.Lock(id) - defer s.shardedLocks.Unlock(id) - - atomic.AddInt64(&s.count, 1) - s.cache[id] = vec -} - -func (s *shardedLockCache[T]) Grow(node uint64) { - s.maintenanceLock.RLock() - if node < uint64(len(s.cache)) { - s.maintenanceLock.RUnlock() - return - } - s.maintenanceLock.RUnlock() - - s.maintenanceLock.Lock() - defer s.maintenanceLock.Unlock() - - // make sure cache still needs growing - // (it could have grown while waiting for maintenance lock) - if node < uint64(len(s.cache)) { - return - } - - s.shardedLocks.LockAll() - defer s.shardedLocks.UnlockAll() - - newSize := node + MinimumIndexGrowthDelta - newCache := make([][]T, newSize) - copy(newCache, s.cache) - s.cache = newCache -} - -func (s *shardedLockCache[T]) Len() int32 { - s.maintenanceLock.RLock() - defer s.maintenanceLock.RUnlock() - - return int32(len(s.cache)) -} - -func (s *shardedLockCache[T]) CountVectors() int64 { - return atomic.LoadInt64(&s.count) -} - -func (s *shardedLockCache[T]) Drop() { - s.deleteAllVectors() - if s.deletionInterval != 0 { - s.cancel <- true - } -} - -func (s *shardedLockCache[T]) deleteAllVectors() { - s.shardedLocks.LockAll() - defer s.shardedLocks.UnlockAll() - - s.logger.WithField("action", "hnsw_delete_vector_cache"). - Debug("deleting full vector cache") - for i := range s.cache { - s.cache[i] = nil - } - - atomic.StoreInt64(&s.count, 0) -} - -func (s *shardedLockCache[T]) watchForDeletion() { - if s.deletionInterval != 0 { - go func() { - t := time.NewTicker(s.deletionInterval) - defer t.Stop() - for { - select { - case <-s.cancel: - return - case <-t.C: - s.replaceIfFull() - } - } - }() - } -} - -func (s *shardedLockCache[T]) replaceIfFull() { - if atomic.LoadInt64(&s.count) >= atomic.LoadInt64(&s.maxSize) { - s.deleteAllVectors() - } -} - -func (s *shardedLockCache[T]) UpdateMaxSize(size int64) { - atomic.StoreInt64(&s.maxSize, size) -} - -func (s *shardedLockCache[T]) CopyMaxSize() int64 { - sizeCopy := atomic.LoadInt64(&s.maxSize) - return sizeCopy -} - -// noopCache can be helpful in debugging situations, where we want to -// explicitly pass through each vectorForID call to the underlying vectorForID -// function without caching in between. -type noopCache struct { - vectorForID common.VectorForID[float32] -} - -func NewNoopCache(vecForID common.VectorForID[float32], maxSize int, - logger logrus.FieldLogger, -) *noopCache { - return &noopCache{vectorForID: vecForID} -} diff --git a/adapters/repos/db/vector/cache/sharded_lock_cache_test.go b/adapters/repos/db/vector/cache/sharded_lock_cache_test.go deleted file mode 100644 index 80a00037385a5820cd6560bb1b3dc1e1a33be941..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/cache/sharded_lock_cache_test.go +++ /dev/null @@ -1,134 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package cache - -import ( - "context" - "math/rand" - "sync" - "testing" - "time" - - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/adapters/repos/db/vector/common" -) - -func TestVectorCacheGrowth(t *testing.T) { - logger, _ := test.NewNullLogger() - var vecForId common.VectorForID[float32] = nil - id := 100_000 - expectedCount := int64(0) - - vectorCache := NewShardedFloat32LockCache(vecForId, 1_000_000, logger, false, time.Duration(10_000)) - initialSize := vectorCache.Len() - assert.Less(t, int(initialSize), id) - assert.Equal(t, expectedCount, vectorCache.CountVectors()) - - vectorCache.Grow(uint64(id)) - size1stGrow := vectorCache.Len() - assert.Greater(t, int(size1stGrow), id) - assert.Equal(t, expectedCount, vectorCache.CountVectors()) - - vectorCache.Grow(uint64(id)) - size2ndGrow := vectorCache.Len() - assert.Equal(t, size1stGrow, size2ndGrow) - assert.Equal(t, expectedCount, vectorCache.CountVectors()) -} - -func TestCache_ParallelGrowth(t *testing.T) { - // no asserts - // ensures there is no "index out of range" panic on get - - logger, _ := test.NewNullLogger() - var vecForId common.VectorForID[float32] = func(context.Context, uint64) ([]float32, error) { return nil, nil } - vectorCache := NewShardedFloat32LockCache(vecForId, 1_000_000, logger, false, time.Second) - - r := rand.New(rand.NewSource(time.Now().UnixNano())) - count := 10_000 - maxNode := 100_000 - - wg := new(sync.WaitGroup) - wg.Add(count) - for i := 0; i < count; i++ { - node := uint64(r.Intn(maxNode)) - go func(node uint64) { - defer wg.Done() - - vectorCache.Grow(node) - vectorCache.Get(context.Background(), node) - }(node) - } - - wg.Wait() -} - -func TestCacheCleanup(t *testing.T) { - logger, _ := test.NewNullLogger() - var vecForId common.VectorForID[float32] = nil - - maxSize := 10 - batchSize := maxSize - 1 - deletionInterval := 200 * time.Millisecond // overwrite default deletionInterval of 3s - sleepMs := deletionInterval + 100*time.Millisecond - - t.Run("count is not reset on unnecessary deletion", func(t *testing.T) { - vectorCache := NewShardedFloat32LockCache(vecForId, maxSize, logger, false, deletionInterval) - shardedLockCache, ok := vectorCache.(*shardedLockCache[float32]) - assert.True(t, ok) - - for i := 0; i < batchSize; i++ { - shardedLockCache.Preload(uint64(i), []float32{float32(i), float32(i)}) - } - time.Sleep(sleepMs) // wait for deletion to fire - - assert.Equal(t, batchSize, int(shardedLockCache.CountVectors())) - assert.Equal(t, batchSize, countCached(shardedLockCache)) - - shardedLockCache.Drop() - - assert.Equal(t, 0, int(shardedLockCache.count)) - assert.Equal(t, 0, countCached(shardedLockCache)) - }) - - t.Run("deletion clears cache and counter when maxSize exceeded", func(t *testing.T) { - vectorCache := NewShardedFloat32LockCache(vecForId, maxSize, logger, false, deletionInterval) - shardedLockCache, ok := vectorCache.(*shardedLockCache[float32]) - assert.True(t, ok) - - for b := 0; b < 2; b++ { - for i := 0; i < batchSize; i++ { - id := b*batchSize + i - shardedLockCache.Preload(uint64(id), []float32{float32(id), float32(id)}) - } - time.Sleep(sleepMs) // wait for deletion to fire, 2nd should clean the cache - } - - assert.Equal(t, 0, int(shardedLockCache.CountVectors())) - assert.Equal(t, 0, countCached(shardedLockCache)) - - shardedLockCache.Drop() - }) -} - -func countCached(c *shardedLockCache[float32]) int { - c.shardedLocks.LockAll() - defer c.shardedLocks.UnlockAll() - - count := 0 - for _, vec := range c.cache { - if vec != nil { - count++ - } - } - return count -} diff --git a/adapters/repos/db/vector/common/search_by_dist_params.go b/adapters/repos/db/vector/common/search_by_dist_params.go deleted file mode 100644 index d5ad6171ea68a2163897e16a2fb934da388a036b..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/common/search_by_dist_params.go +++ /dev/null @@ -1,83 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package common - -const ( - // DefaultSearchByDistInitialLimit : - // the initial limit of 100 here is an - // arbitrary decision, and can be tuned - // as needed - DefaultSearchByDistInitialLimit = 100 - - // DefaultSearchByDistLimitMultiplier : - // the decision to increase the limit in - // multiples of 10 here is an arbitrary - // decision, and can be tuned as needed - DefaultSearchByDistLimitMultiplier = 10 -) - -type SearchByDistParams struct { - offset int - limit int - totalLimit int - maximumSearchLimit int64 -} - -func NewSearchByDistParams( - offset int, - limit int, - totalLimit int, - maximumSearchLimit int64, -) *SearchByDistParams { - return &SearchByDistParams{ - offset: offset, - limit: limit, - totalLimit: totalLimit, - maximumSearchLimit: maximumSearchLimit, - } -} - -func (params *SearchByDistParams) TotalLimit() int { - return params.totalLimit -} - -func (params *SearchByDistParams) MaximumSearchLimit() int64 { - return params.maximumSearchLimit -} - -func (params *SearchByDistParams) OffsetCapacity(ids []uint64) int { - if l := len(ids); l < params.offset { - return l - } - return params.offset -} - -func (params *SearchByDistParams) TotalLimitCapacity(ids []uint64) int { - if l := len(ids); l < params.totalLimit { - return l - } - return params.totalLimit -} - -func (params *SearchByDistParams) Iterate() { - params.offset = params.totalLimit - params.limit *= DefaultSearchByDistLimitMultiplier - params.totalLimit = params.offset + params.limit -} - -func (params *SearchByDistParams) MaxLimitReached() bool { - if params.maximumSearchLimit < 0 { - return false - } - - return int64(params.totalLimit) > params.maximumSearchLimit -} diff --git a/adapters/repos/db/vector/common/sharded_locks.go b/adapters/repos/db/vector/common/sharded_locks.go deleted file mode 100644 index db1f3a3ea35975d41846e6011ada083bfc87f0b3..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/common/sharded_locks.go +++ /dev/null @@ -1,154 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package common - -import "sync" - -const DefaultShardedLocksCount = 512 - -type ShardedLocks struct { - // number of locks - count int - // ensures single LockAll and multiple RLockAll, Lock and RLock - // LockAll is exclusive to RLockAll, Lock, RLock - writeAll *sync.RWMutex - // indicates whether write of any single shard is ongoing, exclusive with readAll - writeAny *sync.RWMutex - // indicates whether read of all shards is ongoing, exclusive with writeAny - readAll *sync.RWMutex - // allows safe transition between writeAny and readAll - change *sync.RWMutex - // sharded locks - shards []*sync.RWMutex -} - -func NewDefaultShardedLocks() *ShardedLocks { - return NewShardedLocks(DefaultShardedLocksCount) -} - -func NewShardedLocks(count int) *ShardedLocks { - if count < 2 { - count = 2 - } - - writeAll := new(sync.RWMutex) - writeAny := new(sync.RWMutex) - readAll := new(sync.RWMutex) - change := new(sync.RWMutex) - shards := make([]*sync.RWMutex, count) - for i := 0; i < count; i++ { - shards[i] = new(sync.RWMutex) - } - - return &ShardedLocks{ - count: count, - writeAll: writeAll, - readAll: readAll, - writeAny: writeAny, - change: change, - shards: shards, - } -} - -func (sl *ShardedLocks) LockAll() { - sl.writeAll.Lock() -} - -func (sl *ShardedLocks) UnlockAll() { - sl.writeAll.Unlock() -} - -func (sl *ShardedLocks) LockedAll(callback func()) { - sl.LockAll() - defer sl.UnlockAll() - - callback() -} - -func (sl *ShardedLocks) Lock(id uint64) { - sl.writeAll.RLock() - sl.markOngoingWriteAny() - sl.shards[sl.mid(id)].Lock() -} - -func (sl *ShardedLocks) Unlock(id uint64) { - sl.shards[sl.mid(id)].Unlock() - sl.writeAny.RUnlock() - sl.writeAll.RUnlock() -} - -func (sl *ShardedLocks) Locked(id uint64, callback func()) { - sl.Lock(id) - defer sl.Unlock(id) - - callback() -} - -func (sl *ShardedLocks) RLockAll() { - sl.writeAll.RLock() - sl.markOngoingReadAll() -} - -func (sl *ShardedLocks) RUnlockAll() { - sl.readAll.RUnlock() - sl.writeAll.RUnlock() -} - -func (sl *ShardedLocks) RLockedAll(callback func()) { - sl.RLockAll() - defer sl.RUnlockAll() - - callback() -} - -func (sl *ShardedLocks) RLock(id uint64) { - sl.writeAll.RLock() - sl.shards[sl.mid(id)].RLock() -} - -func (sl *ShardedLocks) RUnlock(id uint64) { - sl.shards[sl.mid(id)].RUnlock() - sl.writeAll.RUnlock() -} - -func (sl *ShardedLocks) RLocked(id uint64, callback func()) { - sl.RLock(id) - defer sl.RUnlock(id) - - callback() -} - -func (sl *ShardedLocks) mid(id uint64) uint64 { - return id % uint64(sl.count) -} - -func (sl *ShardedLocks) markOngoingWriteAny() { - sl.change.RLock() - defer sl.change.RUnlock() - - // wait until no ongoing readAll - sl.readAll.Lock() - // mark ongoing writeAny - sl.writeAny.RLock() - sl.readAll.Unlock() -} - -func (sl *ShardedLocks) markOngoingReadAll() { - sl.change.Lock() - defer sl.change.Unlock() - - // wait until no ongoing writeAny - sl.writeAny.Lock() - // mark ongoing readAll - sl.readAll.RLock() - sl.writeAny.Unlock() -} diff --git a/adapters/repos/db/vector/common/sharded_locks_test.go b/adapters/repos/db/vector/common/sharded_locks_test.go deleted file mode 100644 index fb573fc3d05bf9a0a6a6a5fffb95bc8635ead40f..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/common/sharded_locks_test.go +++ /dev/null @@ -1,110 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package common - -import ( - "sync" - "testing" -) - -func TestShardedLocks_ParallelLocksAll(t *testing.T) { - // no asserts - // ensures parallel LockAll does not fall into deadlock - count := 10 - sl := NewDefaultShardedLocks() - - wg := new(sync.WaitGroup) - wg.Add(count) - for i := 0; i < count; i++ { - go func() { - defer wg.Done() - sl.LockAll() - sl.UnlockAll() - }() - } - wg.Wait() -} - -func TestShardedLocks_ParallelRLocksAll(t *testing.T) { - // no asserts - // ensures parallel RLockAll does not fall into deadlock - count := 10 - sl := NewDefaultShardedLocks() - - wg := new(sync.WaitGroup) - wg.Add(count) - for i := 0; i < count; i++ { - go func() { - defer wg.Done() - sl.RLockAll() - sl.RUnlockAll() - }() - } - wg.Wait() -} - -func TestShardedLocks_ParallelLocksAllAndRLocksAll(t *testing.T) { - // no asserts - // ensures parallel LockAll + RLockAll does not fall into deadlock - count := 50 - sl := NewDefaultShardedLocks() - - wg := new(sync.WaitGroup) - wg.Add(count) - for i := 0; i < count; i++ { - go func(i int) { - defer wg.Done() - if i%2 == 0 { - sl.LockAll() - sl.UnlockAll() - } else { - sl.RLockAll() - sl.RUnlockAll() - } - }(i) - } - wg.Wait() -} - -func TestShardedLocks_MixedLocks(t *testing.T) { - // no asserts - // ensures parallel LockAll + RLockAll + Lock + RLock does not fall into deadlock - count := 1000 - sl := NewShardedLocks(10) - - wg := new(sync.WaitGroup) - wg.Add(count) - for i := 0; i < count; i++ { - go func(i int) { - defer wg.Done() - id := uint64(i) - if i%5 == 0 { - if i%2 == 0 { - sl.LockAll() - sl.UnlockAll() - } else { - sl.RLockAll() - sl.RUnlockAll() - } - } else { - if i%2 == 0 { - sl.Lock(id) - sl.Unlock(id) - } else { - sl.RLock(id) - sl.RUnlock(id) - } - } - }(i) - } - wg.Wait() -} diff --git a/adapters/repos/db/vector/common/vector_id.go b/adapters/repos/db/vector/common/vector_id.go deleted file mode 100644 index cd61b181a771df2d000cb4c931ff4e894a7960eb..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/common/vector_id.go +++ /dev/null @@ -1,95 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package common - -import ( - "context" - "sync" - - "github.com/weaviate/weaviate/adapters/repos/db/priorityqueue" -) - -type VectorSlice struct { - Slice []float32 - Mem []float32 - Buff8 []byte - Buff []byte -} - -type ( - VectorForID[T float32 | byte | uint64] func(ctx context.Context, id uint64) ([]T, error) - TempVectorForID func(ctx context.Context, id uint64, container *VectorSlice) ([]float32, error) - MultiVectorForID func(ctx context.Context, ids []uint64) ([][]float32, []error) -) - -type TempVectorsPool struct { - pool *sync.Pool -} - -func NewTempVectorsPool() *TempVectorsPool { - return &TempVectorsPool{ - pool: &sync.Pool{ - New: func() interface{} { - return &VectorSlice{ - Mem: nil, - Buff8: make([]byte, 8), - Buff: nil, - Slice: nil, - } - }, - }, - } -} - -func (pool *TempVectorsPool) Get(capacity int) *VectorSlice { - container := pool.pool.Get().(*VectorSlice) - if len(container.Slice) >= capacity { - container.Slice = container.Mem[:capacity] - } else { - container.Mem = make([]float32, capacity) - container.Slice = container.Mem[:capacity] - } - return container -} - -func (pool *TempVectorsPool) Put(container *VectorSlice) { - pool.pool.Put(container) -} - -type PqMaxPool struct { - pool *sync.Pool -} - -func NewPqMaxPool(defaultCap int) *PqMaxPool { - return &PqMaxPool{ - pool: &sync.Pool{ - New: func() interface{} { - return priorityqueue.NewMax[any](defaultCap) - }, - }, - } -} - -func (pqh *PqMaxPool) GetMax(capacity int) *priorityqueue.Queue[any] { - pq := pqh.pool.Get().(*priorityqueue.Queue[any]) - if pq.Cap() < capacity { - pq.ResetCap(capacity) - } else { - pq.Reset() - } - - return pq -} - -func (pqh *PqMaxPool) Put(pq *priorityqueue.Queue[any]) { - pqh.pool.Put(pq) -} diff --git a/adapters/repos/db/vector/common/vector_util.go b/adapters/repos/db/vector/common/vector_util.go deleted file mode 100644 index 45a53b4f06cfa6f899944d4dc63645df44791799..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/common/vector_util.go +++ /dev/null @@ -1,30 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package common - -func VectorsEqual(vecA, vecB []float32) bool { - if len(vecA) != len(vecB) { - return false - } - if vecA == nil && vecB != nil { - return false - } - if vecA != nil && vecB == nil { - return false - } - for i := range vecA { - if vecA[i] != vecB[i] { - return false - } - } - return true -} diff --git a/adapters/repos/db/vector/common/vector_util_test.go b/adapters/repos/db/vector/common/vector_util_test.go deleted file mode 100644 index d3a32aae77a25a9b43af80bd4601772f54630eaf..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/common/vector_util_test.go +++ /dev/null @@ -1,81 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package common - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestVectorUtil_Equal(t *testing.T) { - type testCase struct { - vecA []float32 - vecB []float32 - expectedEqual bool - } - - testCases := []testCase{ - { - vecA: nil, - vecB: nil, - expectedEqual: true, - }, - { - vecA: nil, - vecB: []float32{}, - expectedEqual: false, - }, - { - vecA: []float32{}, - vecB: nil, - expectedEqual: false, - }, - { - vecA: []float32{}, - vecB: []float32{}, - expectedEqual: true, - }, - { - vecA: []float32{1, 2, 3}, - vecB: []float32{1., 2., 3.}, - expectedEqual: true, - }, - { - vecA: []float32{1, 2, 3, 4}, - vecB: []float32{1., 2., 3.}, - expectedEqual: false, - }, - { - vecA: []float32{1, 2, 3}, - vecB: []float32{1., 2., 3., 4.}, - expectedEqual: false, - }, - { - vecA: []float32{}, - vecB: []float32{1., 2., 3.}, - expectedEqual: false, - }, - { - vecA: []float32{1, 2, 3}, - vecB: []float32{}, - expectedEqual: false, - }, - } - - for i, tc := range testCases { - t.Run(fmt.Sprintf("#%d", i+1), func(t *testing.T) { - assert.Equal(t, tc.expectedEqual, VectorsEqual(tc.vecA, tc.vecB)) - }) - } -} diff --git a/adapters/repos/db/vector/compressionhelpers/binary_quantization.go b/adapters/repos/db/vector/compressionhelpers/binary_quantization.go deleted file mode 100644 index 28143d5c461012e7c4761dcf93999a1990fc3ebf..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/compressionhelpers/binary_quantization.go +++ /dev/null @@ -1,56 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package compressionhelpers - -import ( - "errors" - "math" - "math/bits" - - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" -) - -type BinaryQuantizer struct { - distancer distancer.Provider -} - -func NewBinaryQuantizer(distancer distancer.Provider) BinaryQuantizer { - return BinaryQuantizer{ - distancer: distancer, - } -} - -func (bq BinaryQuantizer) Encode(vec []float32) []uint64 { - total := len(vec) / 64 - if len(vec)%64 != 0 { - total++ - } - code := make([]uint64, total) - for j := 0; j < len(vec); j++ { - if vec[j] < 0 { - segment := j / 64 - code[segment] += uint64(math.Pow(2, float64(j%64))) - } - } - return code -} - -func (bq BinaryQuantizer) DistanceBetweenCompressedVectors(x, y []uint64) (float32, error) { - if len(x) != len(y) { - return 0, errors.New("BinaryQuantizer.DistanceBetweenCompressedVectors: Both vectors should have the same len") - } - total := float32(0) - for segment := range x { - total += float32(bits.OnesCount64(x[segment] ^ y[segment])) - } - return total, nil -} diff --git a/adapters/repos/db/vector/compressionhelpers/binary_quantization_test.go b/adapters/repos/db/vector/compressionhelpers/binary_quantization_test.go deleted file mode 100644 index f916fb1959341c1eb083657109183157fdf43130..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/compressionhelpers/binary_quantization_test.go +++ /dev/null @@ -1,86 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package compressionhelpers_test - -import ( - "fmt" - "sync" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/adapters/repos/db/priorityqueue" - "github.com/weaviate/weaviate/adapters/repos/db/vector/compressionhelpers" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" - testinghelpers "github.com/weaviate/weaviate/adapters/repos/db/vector/testinghelpers" -) - -func TestBinaryQuantizerRecall(t *testing.T) { - k := 10 - distanceProvider := distancer.NewCosineDistanceProvider() - vectors, queryVecs := testinghelpers.RandomVecs(10_000, 100, 1536) - compressionhelpers.Concurrently(uint64(len(vectors)), func(i uint64) { - vectors[i] = distancer.Normalize(vectors[i]) - }) - compressionhelpers.Concurrently(uint64(len(queryVecs)), func(i uint64) { - queryVecs[i] = distancer.Normalize(queryVecs[i]) - }) - bq := compressionhelpers.NewBinaryQuantizer(nil) - - codes := make([][]uint64, len(vectors)) - compressionhelpers.Concurrently(uint64(len(vectors)), func(i uint64) { - codes[i] = bq.Encode(vectors[i]) - }) - neighbors := make([][]uint64, len(queryVecs)) - compressionhelpers.Concurrently(uint64(len(queryVecs)), func(i uint64) { - neighbors[i], _ = testinghelpers.BruteForce(vectors, queryVecs[i], k, func(f1, f2 []float32) float32 { - d, _, _ := distanceProvider.SingleDist(f1, f2) - return d - }) - }) - correctedK := 200 - hits := uint64(0) - mutex := sync.Mutex{} - duration := time.Duration(0) - compressionhelpers.Concurrently(uint64(len(queryVecs)), func(i uint64) { - before := time.Now() - query := bq.Encode(queryVecs[i]) - heap := priorityqueue.NewMax[any](correctedK) - for j := range codes { - d, _ := bq.DistanceBetweenCompressedVectors(codes[j], query) - if heap.Len() < correctedK || heap.Top().Dist > d { - if heap.Len() == correctedK { - heap.Pop() - } - heap.Insert(uint64(j), d) - } - } - ids := make([]uint64, correctedK) - for j := range ids { - ids[j] = heap.Pop().ID - } - mutex.Lock() - duration += time.Since(before) - hits += testinghelpers.MatchesInLists(neighbors[i][:k], ids) - mutex.Unlock() - }) - recall := float32(hits) / float32(k*len(queryVecs)) - latency := float32(duration.Microseconds()) / float32(len(queryVecs)) - fmt.Println(recall, latency) - assert.True(t, recall > 0.7) -} - -func TestBinaryQuantizerChecksSize(t *testing.T) { - bq := compressionhelpers.NewBinaryQuantizer(nil) - _, err := bq.DistanceBetweenCompressedVectors(make([]uint64, 3), make([]uint64, 4)) - assert.NotNil(t, err) -} diff --git a/adapters/repos/db/vector/compressionhelpers/compression.go b/adapters/repos/db/vector/compressionhelpers/compression.go deleted file mode 100644 index 5d8572eab8d25520ac4a8a4d87ea399729714b7a..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/compressionhelpers/compression.go +++ /dev/null @@ -1,292 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package compressionhelpers - -import ( - "context" - "encoding/binary" - "fmt" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv" - "github.com/weaviate/weaviate/adapters/repos/db/vector/cache" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" - "github.com/weaviate/weaviate/entities/storobj" - "github.com/weaviate/weaviate/entities/vectorindex/hnsw" -) - -type CompressorDistancer interface { - DistanceToNode(id uint64) (float32, bool, error) - DistanceToFloat(vec []float32) (float32, bool, error) -} - -type ReturnDistancerFn func() - -type VectorCompressor interface { - Drop() error - GrowCache(size uint64) - SetCacheMaxSize(size int64) - GetCacheMaxSize() int64 - Delete(ctx context.Context, id uint64) - Preload(id uint64, vector []float32) - Prefetch(id uint64) - PrefillCache() - - DistanceBetweenCompressedVectorsFromIDs(ctx context.Context, x, y uint64) (float32, error) - DistanceBetweenCompressedAndUncompressedVectorsFromID(ctx context.Context, x uint64, y []float32) (float32, error) - NewDistancer(vector []float32) (CompressorDistancer, ReturnDistancerFn) - NewDistancerFromID(id uint64) CompressorDistancer - NewBag() CompressionDistanceBag - - ExposeFields() PQData -} - -type quantizedVectorsCompressor[T byte | uint64] struct { - cache cache.Cache[T] - compressedStore *lsmkv.Store - quantizer quantizer[T] -} - -func (compressor *quantizedVectorsCompressor[T]) Drop() error { - compressor.cache.Drop() - return nil -} - -func (compressor *quantizedVectorsCompressor[T]) GrowCache(size uint64) { - compressor.cache.Grow(size) -} - -func (compressor *quantizedVectorsCompressor[T]) SetCacheMaxSize(size int64) { - compressor.cache.UpdateMaxSize(size) -} - -func (compressor *quantizedVectorsCompressor[T]) GetCacheMaxSize() int64 { - return compressor.cache.CopyMaxSize() -} - -func (compressor *quantizedVectorsCompressor[T]) Delete(ctx context.Context, id uint64) { - compressor.cache.Delete(ctx, id) - idBytes := make([]byte, 8) - binary.BigEndian.PutUint64(idBytes, id) - compressor.compressedStore.Bucket(helpers.VectorsCompressedBucketLSM).Delete(idBytes) -} - -func (compressor *quantizedVectorsCompressor[T]) Preload(id uint64, vector []float32) { - compressedVector := compressor.quantizer.Encode(vector) - idBytes := make([]byte, 8) - binary.BigEndian.PutUint64(idBytes, id) - compressor.compressedStore.Bucket(helpers.VectorsCompressedBucketLSM).Put(idBytes, compressor.quantizer.CompressedBytes(compressedVector)) - compressor.cache.Grow(id) - compressor.cache.Preload(id, compressedVector) -} - -func (compressor *quantizedVectorsCompressor[T]) Prefetch(id uint64) { - compressor.cache.Prefetch(id) -} - -func (compressor *quantizedVectorsCompressor[T]) DistanceBetweenCompressedVectors(x, y []T) (float32, error) { - return compressor.quantizer.DistanceBetweenCompressedVectors(x, y) -} - -func (compressor *quantizedVectorsCompressor[T]) DistanceBetweenCompressedAndUncompressedVectors(x []T, y []float32) (float32, error) { - return compressor.quantizer.DistanceBetweenCompressedAndUncompressedVectors(y, x) -} - -func (compressor *quantizedVectorsCompressor[T]) compressedVectorFromID(ctx context.Context, id uint64) ([]T, error) { - compressedVector, err := compressor.cache.Get(ctx, id) - if err != nil { - return nil, err - } - if len(compressedVector) == 0 { - return nil, fmt.Errorf("got a nil or zero-length vector at docID %d", id) - } - return compressedVector, nil -} - -func (compressor *quantizedVectorsCompressor[T]) DistanceBetweenCompressedVectorsFromIDs(ctx context.Context, id1, id2 uint64) (float32, error) { - compressedVector1, err := compressor.compressedVectorFromID(ctx, id1) - if err != nil { - return 0, err - } - - compressedVector2, err := compressor.compressedVectorFromID(ctx, id2) - if err != nil { - return 0, err - } - - dist, err := compressor.DistanceBetweenCompressedVectors(compressedVector1, compressedVector2) - return dist, err -} - -func (compressor *quantizedVectorsCompressor[T]) DistanceBetweenCompressedAndUncompressedVectorsFromID(ctx context.Context, id uint64, vector []float32) (float32, error) { - compressedVector, err := compressor.compressedVectorFromID(ctx, id) - if err != nil { - return 0, err - } - - dist, err := compressor.DistanceBetweenCompressedAndUncompressedVectors(compressedVector, vector) - return dist, err -} - -func (compressor *quantizedVectorsCompressor[T]) getCompressedVectorForID(ctx context.Context, id uint64) ([]T, error) { - idBytes := make([]byte, 8) - binary.BigEndian.PutUint64(idBytes, id) - compressedVector, err := compressor.compressedStore.Bucket(helpers.VectorsCompressedBucketLSM).Get(idBytes) - if err != nil { - return nil, errors.Wrap(err, "Getting vector for id") - } - if len(compressedVector) == 0 { - return nil, storobj.NewErrNotFoundf(id, "getCompressedVectorForID") - } - - return compressor.quantizer.FromCompressedBytes(compressedVector), nil -} - -func (compressor *quantizedVectorsCompressor[T]) NewDistancer(vector []float32) (CompressorDistancer, ReturnDistancerFn) { - d := &quantizedCompressorDistancer[T]{ - compressor: compressor, - distancer: compressor.quantizer.NewQuantizerDistancer(vector), - } - return d, func() { - compressor.returnDistancer(d) - } -} - -func (compressor *quantizedVectorsCompressor[T]) NewDistancerFromID(id uint64) CompressorDistancer { - compressedVector, _ := compressor.compressedVectorFromID(context.Background(), id) - d := &quantizedCompressorDistancer[T]{ - compressor: compressor, - distancer: compressor.quantizer.NewCompressedQuantizerDistancer(compressedVector), - } - return d -} - -func (compressor *quantizedVectorsCompressor[T]) returnDistancer(distancer CompressorDistancer) { - dst := distancer.(*quantizedCompressorDistancer[T]).distancer - if dst == nil { - return - } - compressor.quantizer.ReturnQuantizerDistancer(dst) -} - -func (compressor *quantizedVectorsCompressor[T]) NewBag() CompressionDistanceBag { - return &quantizedDistanceBag[T]{ - compressor: compressor, - elements: make(map[uint64][]T), - } -} - -func (compressor *quantizedVectorsCompressor[T]) initCompressedStore() error { - err := compressor.compressedStore.CreateOrLoadBucket(context.Background(), helpers.VectorsCompressedBucketLSM) - if err != nil { - return errors.Wrapf(err, "Create or load bucket (compressed vectors store)") - } - return nil -} - -func (compressor *quantizedVectorsCompressor[T]) PrefillCache() { - cursor := compressor.compressedStore.Bucket(helpers.VectorsCompressedBucketLSM).Cursor() - for k, v := cursor.First(); k != nil; k, v = cursor.Next() { - id := binary.BigEndian.Uint64(k) - compressor.cache.Grow(id) - - vc := make([]byte, len(v)) - copy(vc, v) - compressor.cache.Preload(id, compressor.quantizer.FromCompressedBytes(vc)) - } - cursor.Close() -} - -func (compressor *quantizedVectorsCompressor[T]) ExposeFields() PQData { - return compressor.quantizer.ExposeFields() -} - -func NewPQCompressor( - cfg hnsw.PQConfig, - distance distancer.Provider, - dimensions int, - vectorCacheMaxObjects int, - logger logrus.FieldLogger, - data [][]float32, - store *lsmkv.Store, -) (VectorCompressor, error) { - quantizer, err := NewProductQuantizer(cfg, distance, dimensions) - if err != nil { - return nil, err - } - pqVectorsCompressor := &quantizedVectorsCompressor[byte]{ - quantizer: quantizer, - compressedStore: store, - } - pqVectorsCompressor.initCompressedStore() - pqVectorsCompressor.cache = cache.NewShardedByteLockCache(pqVectorsCompressor.getCompressedVectorForID, vectorCacheMaxObjects, logger, 0) - pqVectorsCompressor.cache.Grow(uint64(len(data))) - quantizer.Fit(data) - return pqVectorsCompressor, nil -} - -func RestorePQCompressor( - cfg hnsw.PQConfig, - distance distancer.Provider, - dimensions int, - vectorCacheMaxObjects int, - logger logrus.FieldLogger, - encoders []PQEncoder, - store *lsmkv.Store, -) (VectorCompressor, error) { - quantizer, err := NewProductQuantizerWithEncoders(cfg, distance, dimensions, encoders) - if err != nil { - return nil, err - } - pqVectorsCompressor := &quantizedVectorsCompressor[byte]{ - quantizer: quantizer, - compressedStore: store, - } - pqVectorsCompressor.initCompressedStore() - pqVectorsCompressor.cache = cache.NewShardedByteLockCache(pqVectorsCompressor.getCompressedVectorForID, vectorCacheMaxObjects, logger, 0) - return pqVectorsCompressor, nil -} - -func NewBQCompressor( - distance distancer.Provider, - vectorCacheMaxObjects int, - logger logrus.FieldLogger, - store *lsmkv.Store, -) (VectorCompressor, error) { - quantizer := NewBinaryQuantizer(distance) - bqVectorsCompressor := &quantizedVectorsCompressor[uint64]{ - quantizer: &quantizer, - compressedStore: store, - } - bqVectorsCompressor.initCompressedStore() - bqVectorsCompressor.cache = cache.NewShardedUInt64LockCache(bqVectorsCompressor.getCompressedVectorForID, vectorCacheMaxObjects, logger, 0) - return bqVectorsCompressor, nil -} - -type quantizedCompressorDistancer[T byte | uint64] struct { - compressor *quantizedVectorsCompressor[T] - distancer quantizerDistancer[T] -} - -func (distancer *quantizedCompressorDistancer[T]) DistanceToNode(id uint64) (float32, bool, error) { - compressedVector, err := distancer.compressor.cache.Get(context.Background(), id) - if err != nil { - return 0, false, err - } - return distancer.distancer.Distance(compressedVector) -} - -func (distancer *quantizedCompressorDistancer[T]) DistanceToFloat(vector []float32) (float32, bool, error) { - return distancer.distancer.DistanceToFloat(vector) -} diff --git a/adapters/repos/db/vector/compressionhelpers/compression_distance_bag.go b/adapters/repos/db/vector/compressionhelpers/compression_distance_bag.go deleted file mode 100644 index 6df545732c793e655d70ed57edbaf79d88a8f7b7..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/compressionhelpers/compression_distance_bag.go +++ /dev/null @@ -1,45 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package compressionhelpers - -import ( - "context" - "fmt" -) - -type CompressionDistanceBag interface { - Load(ctx context.Context, id uint64) error - Distance(x, y uint64) (float32, error) -} - -type quantizedDistanceBag[T byte | uint64] struct { - elements map[uint64][]T - compressor *quantizedVectorsCompressor[T] -} - -func (bag *quantizedDistanceBag[T]) Load(ctx context.Context, id uint64) error { - var err error - bag.elements[id], err = bag.compressor.cache.Get(ctx, id) - return err -} - -func (bag *quantizedDistanceBag[T]) Distance(x, y uint64) (float32, error) { - v1, found := bag.elements[x] - if !found { - return 0, fmt.Errorf("missing id in bag: %d", x) - } - v2, found := bag.elements[y] - if !found { - return 0, fmt.Errorf("missing id in bag: %d", y) - } - return bag.compressor.DistanceBetweenCompressedVectors(v1, v2) -} diff --git a/adapters/repos/db/vector/compressionhelpers/compression_distance_bag_test.go b/adapters/repos/db/vector/compressionhelpers/compression_distance_bag_test.go deleted file mode 100644 index ad4e256855ceed8261331d638d90386bca5d7bdb..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/compressionhelpers/compression_distance_bag_test.go +++ /dev/null @@ -1,53 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build !race - -package compressionhelpers_test - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/adapters/repos/db/vector/compressionhelpers" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" - testinghelpers "github.com/weaviate/weaviate/adapters/repos/db/vector/testinghelpers" -) - -func Test_NoRaceQuantizedDistanceBag(t *testing.T) { - compressor, err := compressionhelpers.NewBQCompressor(distancer.NewCosineDistanceProvider(), 1e12, nil, testinghelpers.NewDummyStore(t)) - assert.Nil(t, err) - compressor.Preload(1, []float32{-0.5, 0.5}) - compressor.Preload(2, []float32{0.25, 0.7}) - compressor.Preload(3, []float32{0.5, 0.5}) - - t.Run("returns error when id has not been loaded", func(t *testing.T) { - bag := compressor.NewBag() - _, err = bag.Distance(1, 2) - assert.NotNil(t, err) - }) - - t.Run("returns error when id has not been loaded", func(t *testing.T) { - bag := compressor.NewBag() - bag.Load(context.Background(), 1) - bag.Load(context.Background(), 2) - bag.Load(context.Background(), 3) - - d, err := bag.Distance(1, 2) - assert.Nil(t, err) - assert.Equal(t, float32(1), d) - - d, err = bag.Distance(2, 3) - assert.Nil(t, err) - assert.Equal(t, float32(0), d) - }) -} diff --git a/adapters/repos/db/vector/compressionhelpers/compression_test.go b/adapters/repos/db/vector/compressionhelpers/compression_test.go deleted file mode 100644 index d51ab74b12a6fe900ba1197d6cf39aae52a3b963..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/compressionhelpers/compression_test.go +++ /dev/null @@ -1,113 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build !race - -package compressionhelpers_test - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/adapters/repos/db/vector/compressionhelpers" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" - testinghelpers "github.com/weaviate/weaviate/adapters/repos/db/vector/testinghelpers" -) - -func Test_NoRaceQuantizedVectorCompressor(t *testing.T) { - t.Run("loading and deleting data works", func(t *testing.T) { - compressor, err := compressionhelpers.NewBQCompressor(distancer.NewCosineDistanceProvider(), 1e12, nil, testinghelpers.NewDummyStore(t)) - assert.Nil(t, err) - compressor.Preload(1, []float32{-0.5, 0.5}) - vec, err := compressor.DistanceBetweenCompressedVectorsFromIDs(context.Background(), 1, 2) - assert.Equal(t, float32(0), vec) - assert.NotNil(t, err) - - compressor.Preload(2, []float32{0.25, 0.7}) - compressor.Preload(3, []float32{0.5, 0.5}) - compressor.Delete(context.Background(), 1) - - _, err = compressor.DistanceBetweenCompressedVectorsFromIDs(context.Background(), 1, 2) - assert.NotNil(t, err) - }) - - t.Run("distance are right when using BQ", func(t *testing.T) { - compressor, err := compressionhelpers.NewBQCompressor(distancer.NewCosineDistanceProvider(), 1e12, nil, testinghelpers.NewDummyStore(t)) - assert.Nil(t, err) - compressor.Preload(1, []float32{-0.5, 0.5}) - compressor.Preload(2, []float32{0.25, 0.7}) - compressor.Preload(3, []float32{0.5, 0.5}) - - d, err := compressor.DistanceBetweenCompressedVectorsFromIDs(context.Background(), 1, 2) - assert.Nil(t, err) - assert.Equal(t, float32(1), d) - - d, err = compressor.DistanceBetweenCompressedVectorsFromIDs(context.Background(), 1, 3) - assert.Nil(t, err) - assert.Equal(t, float32(1), d) - - d, err = compressor.DistanceBetweenCompressedVectorsFromIDs(context.Background(), 2, 3) - assert.Nil(t, err) - assert.Equal(t, float32(0), d) - }) - - t.Run("distance are right when using BQDistancer", func(t *testing.T) { - compressor, err := compressionhelpers.NewBQCompressor(distancer.NewCosineDistanceProvider(), 1e12, nil, testinghelpers.NewDummyStore(t)) - assert.Nil(t, err) - compressor.Preload(1, []float32{-0.5, 0.5}) - compressor.Preload(2, []float32{0.25, 0.7}) - compressor.Preload(3, []float32{0.5, 0.5}) - distancer, returnFn := compressor.NewDistancer([]float32{0.1, -0.2}) - defer returnFn() - - d, _, err := distancer.DistanceToNode(1) - assert.Nil(t, err) - assert.Equal(t, float32(2), d) - - d, _, err = distancer.DistanceToNode(2) - assert.Nil(t, err) - assert.Equal(t, float32(1), d) - - d, _, err = distancer.DistanceToNode(3) - assert.Nil(t, err) - assert.Equal(t, float32(1), d) - - d, _, err = distancer.DistanceToFloat([]float32{0.8, -0.2}) - assert.Nil(t, err) - assert.Equal(t, float32(0.88), d) - }) - - t.Run("distance are right when using BQDistancer to compressed node", func(t *testing.T) { - compressor, err := compressionhelpers.NewBQCompressor(distancer.NewCosineDistanceProvider(), 1e12, nil, testinghelpers.NewDummyStore(t)) - assert.Nil(t, err) - compressor.Preload(1, []float32{-0.5, 0.5}) - compressor.Preload(2, []float32{0.25, 0.7}) - compressor.Preload(3, []float32{0.5, 0.5}) - distancer := compressor.NewDistancerFromID(1) - - d, _, err := distancer.DistanceToNode(1) - assert.Nil(t, err) - assert.Equal(t, float32(0), d) - - d, _, err = distancer.DistanceToNode(2) - assert.Nil(t, err) - assert.Equal(t, float32(1), d) - - d, _, err = distancer.DistanceToNode(3) - assert.Nil(t, err) - assert.Equal(t, float32(1), d) - - d, _, err = distancer.DistanceToFloat([]float32{0.8, -0.2}) - assert.Nil(t, err) - assert.Equal(t, float32(2), d) - }) -} diff --git a/adapters/repos/db/vector/compressionhelpers/kmeans.go b/adapters/repos/db/vector/compressionhelpers/kmeans.go deleted file mode 100644 index 700fcbe097ef96ce2d210785c173dd70dab09fd1..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/compressionhelpers/kmeans.go +++ /dev/null @@ -1,263 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package compressionhelpers - -import ( - "encoding/binary" - "errors" - "fmt" - "math" - "math/rand" - - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" -) - -type FilterFunc func([]float32) []float32 - -type KMeans struct { - K int // How many centroids - DeltaThreshold float32 // Used to stop fitting if there are not too much changes in the centroids anymore - IterationThreshold int // Used to stop fitting after a certain amount of iterations - Distance distancer.Provider - centers [][]float32 // The current centroids - dimensions int // Dimensions of the data - segment int // Segment where it operates - - data KMeansPartitionData // Non-persistent data used only during the fitting process -} - -// String prints some minimal information about the encoder. This can be -// used for viability checks to see if the encoder was initialized -// correctly – for example after a restart. -func (k *KMeans) String() string { - maxElem := 5 - var firstCenters []float32 - i := 0 - for _, center := range k.centers { - for _, centerVal := range center { - if i == maxElem { - break - } - - firstCenters = append(firstCenters, centerVal) - i++ - } - if i == maxElem { - break - } - } - return fmt.Sprintf("KMeans Encoder: K=%d, dim=%d, segment=%d first_center_truncated=%v", k.K, k.dimensions, k.segment, firstCenters) -} - -type KMeansPartitionData struct { - changes int // How many vectors has jumped to a new cluster - points []uint64 // Cluster assigned to each point - cc [][]uint64 // Partition of the data into the clusters -} - -func NewKMeans(k int, dimensions int, segment int) *KMeans { - kMeans := &KMeans{ - K: k, - DeltaThreshold: 0.01, - IterationThreshold: 10, - Distance: distancer.NewL2SquaredProvider(), - dimensions: dimensions, - segment: segment, - } - return kMeans -} - -func NewKMeansWithCenters(k int, dimensions int, segment int, centers [][]float32) *KMeans { - kmeans := NewKMeans(k, dimensions, segment) - kmeans.centers = centers - return kmeans -} - -func (m *KMeans) ExposeDataForRestore() []byte { - ds := len(m.centers[0]) - len := 4 * m.K * ds - buffer := make([]byte, len) - for i := 0; i < len/4; i++ { - binary.LittleEndian.PutUint32(buffer[i*4:(i+1)*4], math.Float32bits(m.centers[i/ds][i%ds])) - } - return buffer -} - -func (m *KMeans) Add(x []float32) { - // nothing to do here -} - -func (m *KMeans) Centers() [][]float32 { - return m.centers -} - -func (m *KMeans) Encode(point []float32) byte { - return byte(m.Nearest(point)) -} - -func (m *KMeans) Nearest(point []float32) uint64 { - return m.NNearest(point, 1)[0] -} - -func (m *KMeans) nNearest(point []float32, n int) ([]uint64, []float32) { - mins := make([]uint64, n) - minD := make([]float32, n) - for i := range mins { - mins[i] = 0 - minD[i] = math.MaxFloat32 - } - filteredPoint := point[m.segment*m.dimensions : (m.segment+1)*m.dimensions] - for i, c := range m.centers { - distance, _, _ := m.Distance.SingleDist(filteredPoint, c) - j := 0 - for (j < n) && minD[j] < distance { - j++ - } - if j < n { - for l := n - 1; l >= j+1; l-- { - mins[l] = mins[l-1] - minD[l] = minD[l-1] - } - minD[j] = distance - mins[j] = uint64(i) - } - } - return mins, minD -} - -func (m *KMeans) NNearest(point []float32, n int) []uint64 { - nearest, _ := m.nNearest(point, n) - return nearest -} - -func (m *KMeans) initCenters(data [][]float32) { - if len(m.centers) == m.K { - return - } - m.centers = make([][]float32, 0, m.K) - for i := 0; i < m.K; i++ { - var vec []float32 - for vec == nil { - vec = data[rand.Intn(len(data))] - } - vecCopy := make([]float32, m.dimensions) - copy(vecCopy, vec[m.segment*m.dimensions:(m.segment+1)*m.dimensions]) - m.centers = append(m.centers, vecCopy) - } -} - -func (m *KMeans) recluster(data [][]float32) { - for p := 0; p < len(data); p++ { - point := data[p] - if point == nil { - continue - } - cis, _ := m.nNearest(point, 1) - ci := cis[0] - m.data.cc[ci] = append(m.data.cc[ci], uint64(p)) - if m.data.points[p] != ci { - m.data.points[p] = ci - m.data.changes++ - } - } -} - -func (m *KMeans) resortOnEmptySets(data [][]float32) { - k64 := uint64(m.K) - dataSize := len(data) - for ci := uint64(0); ci < k64; ci++ { - if len(m.data.cc[ci]) == 0 { - var ri int - for { - ri = rand.Intn(dataSize) - if data[ri] == nil { - continue - } - if len(m.data.cc[m.data.points[ri]]) > 1 { - break - } - } - m.data.cc[ci] = append(m.data.cc[ci], uint64(ri)) - m.data.points[ri] = ci - m.data.changes = dataSize - } - } -} - -func (m *KMeans) recalcCenters(data [][]float32) { - for index := 0; index < m.K; index++ { - for j := range m.centers[index] { - m.centers[index][j] = 0 - } - size := len(m.data.cc[index]) - for _, ci := range m.data.cc[index] { - vec := data[ci] - v := vec[m.segment*m.dimensions : (m.segment+1)*m.dimensions] - for j := 0; j < m.dimensions; j++ { - m.centers[index][j] += v[j] - } - } - for j := 0; j < m.dimensions; j++ { - m.centers[index][j] /= float32(size) - } - } -} - -func (m *KMeans) stopCondition(iterations int, dataSize int) bool { - return iterations >= m.IterationThreshold || - m.data.changes < int(float32(dataSize)*m.DeltaThreshold) -} - -func (m *KMeans) Fit(data [][]float32) error { // init centers using min/max per dimension - dataSize := len(data) - if dataSize < m.K { - return errors.New("not enough data to fit kmeans") - } - m.initCenters(data) - m.data.points = make([]uint64, dataSize) - m.data.changes = 1 - - for i := 0; m.data.changes > 0; i++ { - m.data.changes = 0 - m.data.cc = make([][]uint64, m.K) - for j := range m.data.cc { - m.data.cc[j] = make([]uint64, 0) - } - - m.recluster(data) - m.resortOnEmptySets(data) - if m.data.changes > 0 { - m.recalcCenters(data) - } - - if m.stopCondition(i, dataSize) { - break - } - - } - - m.clearData() - return nil -} - -func (m *KMeans) clearData() { - m.data.points = nil - m.data.cc = nil -} - -func (m *KMeans) Center(point []float32) []float32 { - return m.centers[m.Nearest(point)] -} - -func (m *KMeans) Centroid(i byte) []float32 { - return m.centers[i] -} diff --git a/adapters/repos/db/vector/compressionhelpers/kmeans_test.go b/adapters/repos/db/vector/compressionhelpers/kmeans_test.go deleted file mode 100644 index 8e0d4414bbe5b47b125dd955633470307cc80457..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/compressionhelpers/kmeans_test.go +++ /dev/null @@ -1,66 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build !race - -package compressionhelpers_test - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/adapters/repos/db/vector/compressionhelpers" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" - testinghelpers "github.com/weaviate/weaviate/adapters/repos/db/vector/testinghelpers" -) - -func Test_NoRaceKMeansNNearest(t *testing.T) { - distanceProvider := distancer.NewL2SquaredProvider() - vectors := [][]float32{ - {0, 5}, - {0.1, 4.9}, - {0.01, 5.1}, - {10.1, 7}, - {5.1, 2}, - {5.0, 2.1}, - } - kmeans := compressionhelpers.NewKMeans( - 3, - 2, - 0, - ) - kmeans.Fit(vectors) - centers := make([]byte, 6) - for i := range centers { - centers[i] = byte(kmeans.Nearest(vectors[i])) - } - for v := range vectors { - min, _, _ := distanceProvider.SingleDist(vectors[v], kmeans.Centroid(centers[v])) - for c := range centers { - dist, _, _ := distanceProvider.SingleDist(vectors[v], kmeans.Centroid(centers[c])) - assert.True(t, dist >= min) - } - } -} - -func Test_NoRaceRandomData(t *testing.T) { - vectorsSize := 10000 - vectors, _ := testinghelpers.RandomVecs(vectorsSize, 0, 128) - before := time.Now() - kmeans := compressionhelpers.NewKMeans( - 256, - 1, - 10, - ) - kmeans.Fit(vectors) - assert.True(t, time.Since(before).Seconds() < 50) -} diff --git a/adapters/repos/db/vector/compressionhelpers/product_quantization.go b/adapters/repos/db/vector/compressionhelpers/product_quantization.go deleted file mode 100644 index 10b7ffbc512b417dfb1929a1a453eee7650e20c1..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/compressionhelpers/product_quantization.go +++ /dev/null @@ -1,414 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package compressionhelpers - -import ( - "errors" - "fmt" - "math" - "sync" - - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" - ent "github.com/weaviate/weaviate/entities/vectorindex/hnsw" -) - -type Encoder byte - -const ( - UseTileEncoder Encoder = 0 - UseKMeansEncoder Encoder = 1 -) - -type DistanceLookUpTable struct { - calculated []bool - distances []float32 - center [][]float32 - segments int - centroids int - flatCenter []float32 -} - -func NewDistanceLookUpTable(segments int, centroids int, center []float32) *DistanceLookUpTable { - distances := make([]float32, segments*centroids) - calculated := make([]bool, segments*centroids) - parsedCenter := make([][]float32, segments) - ds := len(center) / segments - for c := 0; c < segments; c++ { - parsedCenter[c] = center[c*ds : (c+1)*ds] - } - - dlt := &DistanceLookUpTable{ - distances: distances, - calculated: calculated, - center: parsedCenter, - segments: segments, - centroids: centroids, - flatCenter: center, - } - return dlt -} - -func (lut *DistanceLookUpTable) Reset(segments int, centroids int, center []float32) { - elems := segments * centroids - lut.segments = segments - lut.centroids = centroids - if len(lut.distances) != elems || - len(lut.calculated) != elems || - len(lut.center) != segments { - lut.distances = make([]float32, segments*centroids) - lut.calculated = make([]bool, segments*centroids) - lut.center = make([][]float32, segments) - } else { - for i := range lut.calculated { - lut.calculated[i] = false - } - } - - ds := len(center) / segments - for c := 0; c < segments; c++ { - lut.center[c] = center[c*ds : (c+1)*ds] - } - lut.flatCenter = center -} - -func (lut *DistanceLookUpTable) LookUp( - encoded []byte, - pq *ProductQuantizer, -) float32 { - var sum float32 - - for i := range pq.kms { - c := ExtractCode8(encoded, i) - if lut.distCalculated(i, c) { - sum += lut.codeDist(i, c) - } else { - centroid := pq.kms[i].Centroid(c) - dist := pq.distance.Step(lut.center[i], centroid) - lut.setCodeDist(i, c, dist) - lut.setDistCalculated(i, c) - sum += dist - } - } - return pq.distance.Wrap(sum) -} - -// meant for better readability, rely on the fact that the compiler will inline this -func (lut *DistanceLookUpTable) posForSegmentAndCode(segment int, code byte) int { - return segment*lut.centroids + int(code) -} - -// meant for better readability, rely on the fact that the compiler will inline this -func (lut *DistanceLookUpTable) distCalculated(segment int, code byte) bool { - return lut.calculated[lut.posForSegmentAndCode(segment, code)] -} - -// meant for better readability, rely on the fact that the compiler will inline this -func (lut *DistanceLookUpTable) setDistCalculated(segment int, code byte) { - lut.calculated[lut.posForSegmentAndCode(segment, code)] = true -} - -// meant for better readability, rely on the fact that the compiler will inline this -func (lut *DistanceLookUpTable) codeDist(segment int, code byte) float32 { - return lut.distances[lut.posForSegmentAndCode(segment, code)] -} - -// meant for better readability, rely on the fact that the compiler will inline this -func (lut *DistanceLookUpTable) setCodeDist(segment int, code byte, dist float32) { - lut.distances[lut.posForSegmentAndCode(segment, code)] = dist -} - -type DLUTPool struct { - pool sync.Pool -} - -func NewDLUTPool() *DLUTPool { - return &DLUTPool{ - pool: sync.Pool{ - New: func() any { - return &DistanceLookUpTable{} - }, - }, - } -} - -func (p *DLUTPool) Get(segments, centroids int, centers []float32) *DistanceLookUpTable { - dlt := p.pool.Get().(*DistanceLookUpTable) - dlt.Reset(segments, centroids, centers) - return dlt -} - -func (p *DLUTPool) Return(dlt *DistanceLookUpTable) { - p.pool.Put(dlt) -} - -type ProductQuantizer struct { - ks int // centroids - m int // segments - ds int // dimensions per segment - distance distancer.Provider - dimensions int - kms []PQEncoder - encoderType Encoder - encoderDistribution EncoderDistribution - dlutPool *DLUTPool - trainingLimit int - globalDistances []float32 -} - -type PQData struct { - Ks uint16 - M uint16 - Dimensions uint16 - EncoderType Encoder - EncoderDistribution byte - Encoders []PQEncoder - UseBitsEncoding bool - TrainingLimit int -} - -type PQEncoder interface { - Encode(x []float32) byte - Centroid(b byte) []float32 - Add(x []float32) - Fit(data [][]float32) error - ExposeDataForRestore() []byte -} - -func NewProductQuantizer(cfg ent.PQConfig, distance distancer.Provider, dimensions int) (*ProductQuantizer, error) { - if cfg.Segments <= 0 { - return nil, errors.New("segments cannot be 0 nor negative") - } - if cfg.Centroids > 256 { - return nil, fmt.Errorf("centroids should not be higher than 256. Attempting to use %d", cfg.Centroids) - } - if dimensions%cfg.Segments != 0 { - return nil, errors.New("segments should be an integer divisor of dimensions") - } - encoderType, err := parseEncoder(cfg.Encoder.Type) - if err != nil { - return nil, errors.New("invalid encoder type") - } - - encoderDistribution, err := parseEncoderDistribution(cfg.Encoder.Distribution) - if err != nil { - return nil, errors.New("invalid encoder distribution") - } - pq := &ProductQuantizer{ - ks: cfg.Centroids, - m: cfg.Segments, - ds: int(dimensions / cfg.Segments), - distance: distance, - trainingLimit: cfg.TrainingLimit, - dimensions: dimensions, - encoderType: encoderType, - encoderDistribution: encoderDistribution, - dlutPool: NewDLUTPool(), - } - - return pq, nil -} - -func NewProductQuantizerWithEncoders(cfg ent.PQConfig, distance distancer.Provider, dimensions int, encoders []PQEncoder) (*ProductQuantizer, error) { - cfg.Segments = len(encoders) - pq, err := NewProductQuantizer(cfg, distance, dimensions) - if err != nil { - return nil, err - } - - pq.kms = encoders - pq.buildGlobalDistances() - return pq, nil -} - -func (pq *ProductQuantizer) buildGlobalDistances() { - // This hosts the partial distances between the centroids. This way we do not need - // to recalculate all the time when calculating full distances between compressed vecs - pq.globalDistances = make([]float32, pq.m*pq.ks*pq.ks) - for segment := 0; segment < pq.m; segment++ { - for i := 0; i < pq.ks; i++ { - cX := pq.kms[segment].Centroid(byte(i)) - for j := 0; j <= i; j++ { - cY := pq.kms[segment].Centroid(byte(j)) - pq.globalDistances[segment*pq.ks*pq.ks+i*pq.ks+j] = pq.distance.Step(cX, cY) - // Just copy from already calculated cell since step should be symmetric. - pq.globalDistances[segment*pq.ks*pq.ks+j*pq.ks+i] = pq.globalDistances[segment*pq.ks*pq.ks+i*pq.ks+j] - } - } - } -} - -// Only made public for testing purposes... Not sure we need it outside -func ExtractCode8(encoded []byte, index int) byte { - return encoded[index] -} - -func parseEncoder(encoder string) (Encoder, error) { - switch encoder { - case ent.PQEncoderTypeTile: - return UseTileEncoder, nil - case ent.PQEncoderTypeKMeans: - return UseKMeansEncoder, nil - default: - return 0, fmt.Errorf("invalid encoder type: %s", encoder) - } -} - -func parseEncoderDistribution(distribution string) (EncoderDistribution, error) { - switch distribution { - case ent.PQEncoderDistributionLogNormal: - return LogNormalEncoderDistribution, nil - case ent.PQEncoderDistributionNormal: - return NormalEncoderDistribution, nil - default: - return 0, fmt.Errorf("invalid encoder distribution: %s", distribution) - } -} - -// Only made public for testing purposes... Not sure we need it outside -func PutCode8(code byte, buffer []byte, index int) { - buffer[index] = code -} - -func (pq *ProductQuantizer) ExposeFields() PQData { - return PQData{ - Dimensions: uint16(pq.dimensions), - EncoderType: pq.encoderType, - Ks: uint16(pq.ks), - M: uint16(pq.m), - EncoderDistribution: byte(pq.encoderDistribution), - Encoders: pq.kms, - TrainingLimit: pq.trainingLimit, - } -} - -func (pq *ProductQuantizer) DistanceBetweenCompressedVectors(x, y []byte) (float32, error) { - dist := float32(0) - - for i := 0; i < pq.m; i++ { - cX := ExtractCode8(x, i) - cY := ExtractCode8(y, i) - dist += pq.globalDistances[i*pq.ks*pq.ks+int(cX)*pq.ks+int(cY)] - } - - return pq.distance.Wrap(dist), nil -} - -func (pq *ProductQuantizer) DistanceBetweenCompressedAndUncompressedVectors(x []float32, encoded []byte) (float32, error) { - dist := float32(0) - for i := 0; i < pq.m; i++ { - cY := pq.kms[i].Centroid(ExtractCode8(encoded, i)) - dist += pq.distance.Step(x[i*pq.ds:(i+1)*pq.ds], cY) - } - return pq.distance.Wrap(dist), nil -} - -type PQDistancer struct { - x []float32 - pq *ProductQuantizer - lut *DistanceLookUpTable - compressed []byte -} - -func (pq *ProductQuantizer) NewDistancer(a []float32) *PQDistancer { - lut := pq.CenterAt(a) - return &PQDistancer{ - x: a, - pq: pq, - lut: lut, - compressed: nil, - } -} - -func (pq *ProductQuantizer) NewCompressedQuantizerDistancer(a []byte) quantizerDistancer[byte] { - return &PQDistancer{ - x: nil, - pq: pq, - lut: nil, - compressed: a, - } -} - -func (pq *ProductQuantizer) ReturnDistancer(d *PQDistancer) { - pq.dlutPool.Return(d.lut) -} - -func (d *PQDistancer) Distance(x []byte) (float32, bool, error) { - if d.lut == nil { - dist, err := d.pq.DistanceBetweenCompressedVectors(d.compressed, x) - return dist, err == nil, err - } - return d.pq.Distance(x, d.lut), true, nil -} - -func (d *PQDistancer) DistanceToFloat(x []float32) (float32, bool, error) { - if d.lut != nil { - return d.pq.distance.SingleDist(x, d.lut.flatCenter) - } - xComp := d.pq.Encode(x) - dist, err := d.pq.DistanceBetweenCompressedVectors(d.compressed, xComp) - return dist, err == nil, err -} - -func (pq *ProductQuantizer) Fit(data [][]float32) { - if pq.trainingLimit > 0 && len(data) > pq.trainingLimit { - data = data[:pq.trainingLimit] - } - switch pq.encoderType { - case UseTileEncoder: - pq.kms = make([]PQEncoder, pq.m) - Concurrently(uint64(pq.m), func(i uint64) { - pq.kms[i] = NewTileEncoder(int(math.Log2(float64(pq.ks))), int(i), pq.encoderDistribution) - for j := 0; j < len(data); j++ { - pq.kms[i].Add(data[j]) - } - pq.kms[i].Fit(data) - }) - case UseKMeansEncoder: - pq.kms = make([]PQEncoder, pq.m) - Concurrently(uint64(pq.m), func(i uint64) { - pq.kms[i] = NewKMeans( - pq.ks, - pq.ds, - int(i), - ) - err := pq.kms[i].Fit(data) - if err != nil { - panic(err) - } - }) - } - pq.buildGlobalDistances() -} - -func (pq *ProductQuantizer) Encode(vec []float32) []byte { - codes := make([]byte, pq.m) - for i := 0; i < pq.m; i++ { - PutCode8(pq.kms[i].Encode(vec), codes, i) - } - return codes -} - -func (pq *ProductQuantizer) Decode(code []byte) []float32 { - vec := make([]float32, 0, pq.m) - for i := 0; i < pq.m; i++ { - vec = append(vec, pq.kms[i].Centroid(ExtractCode8(code, i))...) - } - return vec -} - -func (pq *ProductQuantizer) CenterAt(vec []float32) *DistanceLookUpTable { - return pq.dlutPool.Get(int(pq.m), int(pq.ks), vec) -} - -func (pq *ProductQuantizer) Distance(encoded []byte, lut *DistanceLookUpTable) float32 { - return lut.LookUp(encoded, pq) -} diff --git a/adapters/repos/db/vector/compressionhelpers/product_quantization_test.go b/adapters/repos/db/vector/compressionhelpers/product_quantization_test.go deleted file mode 100644 index 05b0bd86a90bf5ae16431d853d6ed0ef11411313..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/compressionhelpers/product_quantization_test.go +++ /dev/null @@ -1,240 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build !race - -package compressionhelpers_test - -import ( - "fmt" - "sort" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/adapters/repos/db/vector/compressionhelpers" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" - "github.com/weaviate/weaviate/adapters/repos/db/vector/testinghelpers" - "github.com/weaviate/weaviate/entities/vectorindex/hnsw" - ent "github.com/weaviate/weaviate/entities/vectorindex/hnsw" -) - -type IndexAndDistance struct { - index uint64 - distance float32 -} - -func distance(dp distancer.Provider) func(x, y []float32) float32 { - return func(x, y []float32) float32 { - dist, _, _ := dp.SingleDist(x, y) - return dist - } -} - -func Test_NoRacePQSettings(t *testing.T) { - distanceProvider := distancer.NewL2SquaredProvider() - - cfg := ent.PQConfig{ - Enabled: true, - Encoder: ent.PQEncoder{ - Type: ent.PQEncoderTypeKMeans, - Distribution: ent.PQEncoderDistributionLogNormal, - }, - Centroids: 512, - Segments: 128, - } - - _, err := compressionhelpers.NewProductQuantizer( - cfg, - distanceProvider, - 128, - ) - assert.NotNil(t, err) -} - -func Test_NoRacePQKMeans(t *testing.T) { - dimensions := 128 - vectors_size := 1000 - queries_size := 100 - k := 100 - vectors, queries := testinghelpers.RandomVecs(vectors_size, queries_size, int(dimensions)) - distanceProvider := distancer.NewDotProductProvider() - - cfg := ent.PQConfig{ - Enabled: true, - Encoder: ent.PQEncoder{ - Type: ent.PQEncoderTypeKMeans, - Distribution: ent.PQEncoderDistributionLogNormal, - }, - Centroids: 255, - Segments: dimensions, - } - pq, _ := compressionhelpers.NewProductQuantizer( - cfg, - distanceProvider, - dimensions, - ) - pq.Fit(vectors) - encoded := make([][]byte, vectors_size) - for i := 0; i < vectors_size; i++ { - encoded[i] = pq.Encode(vectors[i]) - } - - var relevant uint64 - queries_size = 100 - for _, query := range queries { - truth, _ := testinghelpers.BruteForce(vectors, query, k, distance(distanceProvider)) - distances := make([]IndexAndDistance, len(vectors)) - - distancer := pq.NewDistancer(query) - for v := range vectors { - d, _, _ := distancer.Distance(encoded[v]) - distances[v] = IndexAndDistance{index: uint64(v), distance: d} - } - sort.Slice(distances, func(a, b int) bool { - return distances[a].distance < distances[b].distance - }) - - results := make([]uint64, 0, k) - for i := 0; i < k; i++ { - results = append(results, distances[i].index) - } - relevant += testinghelpers.MatchesInLists(truth, results) - } - recall := float32(relevant) / float32(k*queries_size) - fmt.Println(recall) - assert.True(t, recall > 0.99) -} - -func Test_NoRacePQDecodeBytes(t *testing.T) { - t.Run("extracts correctly on one code per byte", func(t *testing.T) { - amount := 100 - values := make([]byte, 0, amount) - for i := byte(0); i < byte(amount); i++ { - values = append(values, i) - } - for i := 0; i < amount; i++ { - code := compressionhelpers.ExtractCode8(values, i) - assert.Equal(t, code, uint8(i)) - } - }) -} - -func Test_NoRacePQInvalidConfig(t *testing.T) { - t.Run("validate pq options", func(t *testing.T) { - amount := 100 - centroids := 256 - cfg := ent.PQConfig{ - Enabled: true, - Encoder: ent.PQEncoder{ - Type: "lmeans", - Distribution: ent.PQEncoderDistributionLogNormal, - }, - Centroids: centroids, - TrainingLimit: 75, - Segments: amount, - } - _, err := compressionhelpers.NewProductQuantizer( - cfg, - nil, - amount, - ) - assert.ErrorContains(t, err, "invalid encoder type") - cfg = ent.PQConfig{ - Enabled: true, - Encoder: ent.PQEncoder{ - Type: ent.DefaultPQEncoderType, - Distribution: "log", - }, - Centroids: centroids, - TrainingLimit: 75, - Segments: amount, - } - _, err = compressionhelpers.NewProductQuantizer( - cfg, - nil, - amount, - ) - assert.ErrorContains(t, err, "invalid encoder distribution") - cfg = ent.PQConfig{ - Enabled: true, - Encoder: ent.PQEncoder{ - Type: ent.DefaultPQEncoderType, - Distribution: ent.DefaultPQEncoderDistribution, - }, - Centroids: centroids, - TrainingLimit: 75, - Segments: 0, - } - _, err = compressionhelpers.NewProductQuantizer( - cfg, - nil, - amount, - ) - assert.ErrorContains(t, err, "segments cannot be 0 nor negative") - cfg = ent.PQConfig{ - Enabled: true, - Encoder: ent.PQEncoder{ - Type: ent.DefaultPQEncoderType, - Distribution: ent.DefaultPQEncoderDistribution, - }, - Centroids: centroids, - TrainingLimit: 75, - Segments: 3, - } - _, err = compressionhelpers.NewProductQuantizer( - cfg, - nil, - 4, - ) - assert.ErrorContains(t, err, "segments should be an integer divisor of dimensions") - }) - t.Run("validate training limit applied", func(t *testing.T) { - amount := 64 - centroids := 256 - vectors_size := 400 - vectors, _ := testinghelpers.RandomVecs(vectors_size, vectors_size, amount) - distanceProvider := distancer.NewL2SquaredProvider() - - cfg := ent.PQConfig{ - Enabled: true, - Encoder: ent.PQEncoder{ - Type: hnsw.PQEncoderTypeKMeans, - Distribution: ent.PQEncoderDistributionLogNormal, - }, - Centroids: centroids, - TrainingLimit: 260, - Segments: amount, - } - pq, err := compressionhelpers.NewProductQuantizer( - cfg, - distanceProvider, - amount, - ) - assert.NoError(t, err) - pq.Fit(vectors) - pqdata := pq.ExposeFields() - assert.Equal(t, pqdata.TrainingLimit, 260) - }) -} - -func Test_NoRacePQEncodeBytes(t *testing.T) { - t.Run("encodes correctly on one code per byte", func(t *testing.T) { - amount := 100 - values := make([]byte, amount) - for i := 0; i < amount; i++ { - compressionhelpers.PutCode8(uint8(i), values, i) - } - for i := 0; i < amount; i++ { - code := compressionhelpers.ExtractCode8(values, i) - assert.Equal(t, code, uint8(i)) - } - }) -} diff --git a/adapters/repos/db/vector/compressionhelpers/quantizer.go b/adapters/repos/db/vector/compressionhelpers/quantizer.go deleted file mode 100644 index 931f1148a5caf4046159e7e42031852342e4831e..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/compressionhelpers/quantizer.go +++ /dev/null @@ -1,123 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package compressionhelpers - -import "encoding/binary" - -type quantizerDistancer[T byte | uint64] interface { - Distance(x []T) (float32, bool, error) - DistanceToFloat(x []float32) (float32, bool, error) -} - -type quantizer[T byte | uint64] interface { - DistanceBetweenCompressedVectors(x, y []T) (float32, error) - DistanceBetweenCompressedAndUncompressedVectors(x []float32, encoded []T) (float32, error) - Encode(vec []float32) []T - NewQuantizerDistancer(a []float32) quantizerDistancer[T] - NewCompressedQuantizerDistancer(a []T) quantizerDistancer[T] - ReturnQuantizerDistancer(distancer quantizerDistancer[T]) - CompressedBytes(compressed []T) []byte - FromCompressedBytes(compressed []byte) []T - ExposeFields() PQData -} - -func (bq *BinaryQuantizer) ExposeFields() PQData { - return PQData{} -} - -func (bq *BinaryQuantizer) DistanceBetweenCompressedAndUncompressedVectors(x []float32, y []uint64) (float32, error) { - encoded := bq.Encode(x) - return bq.DistanceBetweenCompressedVectors(encoded, y) -} - -func (pq *ProductQuantizer) NewQuantizerDistancer(vec []float32) quantizerDistancer[byte] { - return pq.NewDistancer(vec) -} - -func (pq *ProductQuantizer) ReturnQuantizerDistancer(distancer quantizerDistancer[byte]) { - concreteDistancer := distancer.(*PQDistancer) - if concreteDistancer == nil { - return - } - pq.ReturnDistancer(concreteDistancer) -} - -func (bq *BinaryQuantizer) CompressedBytes(compressed []uint64) []byte { - slice := make([]byte, len(compressed)*8) - for i := range compressed { - binary.LittleEndian.PutUint64(slice[i*8:], compressed[i]) - } - return slice -} - -func (bq *BinaryQuantizer) FromCompressedBytes(compressed []byte) []uint64 { - l := len(compressed) / 8 - if len(compressed)%8 != 0 { - l++ - } - slice := make([]uint64, l) - - for i := range slice { - slice[i] = binary.LittleEndian.Uint64(compressed[i*8:]) - } - return slice -} - -func (pq *ProductQuantizer) CompressedBytes(compressed []byte) []byte { - return compressed -} - -func (pq *ProductQuantizer) FromCompressedBytes(compressed []byte) []byte { - return compressed -} - -type BQDistancer struct { - x []float32 - bq *BinaryQuantizer - compressed []uint64 -} - -func (bq *BinaryQuantizer) NewDistancer(a []float32) *BQDistancer { - return &BQDistancer{ - x: a, - bq: bq, - compressed: bq.Encode(a), - } -} - -func (bq *BinaryQuantizer) NewCompressedQuantizerDistancer(a []uint64) quantizerDistancer[uint64] { - return &BQDistancer{ - x: nil, - bq: bq, - compressed: a, - } -} - -func (d *BQDistancer) Distance(x []uint64) (float32, bool, error) { - dist, err := d.bq.DistanceBetweenCompressedVectors(d.compressed, x) - return dist, err == nil, err -} - -func (d *BQDistancer) DistanceToFloat(x []float32) (float32, bool, error) { - if len(d.x) > 0 { - return d.bq.distancer.SingleDist(d.x, x) - } - xComp := d.bq.Encode(x) - dist, err := d.bq.DistanceBetweenCompressedVectors(d.compressed, xComp) - return dist, err == nil, err -} - -func (bq *BinaryQuantizer) NewQuantizerDistancer(vec []float32) quantizerDistancer[uint64] { - return bq.NewDistancer(vec) -} - -func (bq *BinaryQuantizer) ReturnQuantizerDistancer(distancer quantizerDistancer[uint64]) {} diff --git a/adapters/repos/db/vector/compressionhelpers/tile_encoder.go b/adapters/repos/db/vector/compressionhelpers/tile_encoder.go deleted file mode 100644 index 35de733248af1056dd2f0bf3b6733e0876b160fa..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/compressionhelpers/tile_encoder.go +++ /dev/null @@ -1,204 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package compressionhelpers - -import ( - "encoding/binary" - "math" - "sync/atomic" - - "gonum.org/v1/gonum/stat/distuv" -) - -type distribution interface { - Transform(x float64) float64 - CDF(x float64) float64 - Quantile(x float64) float64 -} - -type logNormalDistribution struct { - dist *distuv.LogNormal -} - -func newLogNormalDistribution(mean float64, std float64) distribution { - return &logNormalDistribution{ - dist: &distuv.LogNormal{ - Mu: mean, - Sigma: std, - }, - } -} - -func (d *logNormalDistribution) Transform(x float64) float64 { - if x > 0 { - return math.Log(x) - } - return 0 -} - -func (d *logNormalDistribution) CDF(x float64) float64 { - return d.dist.CDF(x) -} - -func (d *logNormalDistribution) Quantile(x float64) float64 { - return d.dist.Quantile(x) -} - -type normalDistribution struct { - dist *distuv.Normal -} - -func newNormalDistribution(mean float64, std float64) distribution { - return &normalDistribution{ - dist: &distuv.Normal{ - Mu: mean, - Sigma: std, - }, - } -} - -func (d *normalDistribution) Transform(x float64) float64 { - return x -} - -func (d *normalDistribution) CDF(x float64) float64 { - return d.dist.CDF(x) -} - -func (d *normalDistribution) Quantile(x float64) float64 { - return d.dist.Quantile(x) -} - -type Centroid struct { - Center []float32 - Calculated atomic.Bool -} - -type EncoderDistribution byte - -const ( - NormalEncoderDistribution EncoderDistribution = 0 - LogNormalEncoderDistribution EncoderDistribution = 1 -) - -type TileEncoder struct { - bins float64 - mean float64 - stdDev float64 - size float64 - s1 float64 - s2 float64 - segment int - centroids []Centroid - encoderDistribution EncoderDistribution - distribution distribution -} - -func NewTileEncoder(bits int, segment int, encoderDistribution EncoderDistribution) *TileEncoder { - centroids := math.Pow(2, float64(bits)) - te := &TileEncoder{ - bins: centroids, - mean: 0, - stdDev: 0, - size: 0, - s1: 0, - s2: 0, - segment: segment, - centroids: make([]Centroid, int(centroids)), - encoderDistribution: encoderDistribution, - } - te.setEncoderDistribution() - return te -} - -func RestoreTileEncoder(bins float64, mean float64, stdDev float64, size float64, s1 float64, s2 float64, segment uint16, encoderDistribution byte) *TileEncoder { - te := &TileEncoder{ - bins: bins, - mean: mean, - stdDev: stdDev, - size: size, - s1: s1, - s2: s2, - segment: int(segment), - encoderDistribution: EncoderDistribution(encoderDistribution), - } - te.setEncoderDistribution() - return te -} - -func (te *TileEncoder) ExposeDataForRestore() []byte { - buffer := make([]byte, 51) - binary.LittleEndian.PutUint64(buffer[0:8], math.Float64bits(te.bins)) - binary.LittleEndian.PutUint64(buffer[8:16], math.Float64bits(te.mean)) - binary.LittleEndian.PutUint64(buffer[16:24], math.Float64bits(te.stdDev)) - binary.LittleEndian.PutUint64(buffer[24:32], math.Float64bits(te.size)) - binary.LittleEndian.PutUint64(buffer[32:40], math.Float64bits(te.s1)) - binary.LittleEndian.PutUint64(buffer[40:48], math.Float64bits(te.s2)) - binary.LittleEndian.PutUint16(buffer[48:50], uint16(te.segment)) - buffer[50] = byte(te.encoderDistribution) - return buffer -} - -func (te *TileEncoder) Fit(data [][]float32) error { - te.setEncoderDistribution() - return nil -} - -func (te *TileEncoder) setEncoderDistribution() { - switch te.encoderDistribution { - case LogNormalEncoderDistribution: - te.distribution = newLogNormalDistribution(te.mean, te.stdDev) - case NormalEncoderDistribution: - te.distribution = newNormalDistribution(te.mean, te.stdDev) - } -} - -func (te *TileEncoder) Add(x []float32) { - // calculate mean and stddev iteratively - x64 := te.distribution.Transform(float64(x[te.segment])) - te.s1 += x64 - te.s2 += x64 * x64 - te.size++ - te.mean = te.s1 / te.size - sum := te.s2 + te.size*te.mean*te.mean - prod := 2 * te.mean * te.s1 - te.stdDev = math.Sqrt((sum - prod) / te.size) -} - -func (te *TileEncoder) Encode(x []float32) byte { - cdf := te.distribution.CDF(float64(x[te.segment])) - intPart, _ := math.Modf(cdf * float64(te.bins)) - return byte(intPart) -} - -func (te *TileEncoder) centroid(b byte) []float32 { - res := make([]float32, 0, 1) - if b == 0 { - res = append(res, float32(te.distribution.Quantile(1/te.bins))) - } else if b == byte(te.bins) { - res = append(res, float32(te.distribution.Quantile((te.bins-1)/te.bins))) - } else { - b64 := float64(b) - mean := (b64/te.bins + (b64+1)/te.bins) / 2 - res = append(res, float32(te.distribution.Quantile(mean))) - } - return res -} - -func (te *TileEncoder) Centroid(b byte) []float32 { - if te.centroids[b].Calculated.Load() { - return te.centroids[b].Center - } - te.centroids[b].Center = te.centroid(b) - te.centroids[b].Calculated.Store(true) - return te.centroids[b].Center -} diff --git a/adapters/repos/db/vector/compressionhelpers/tile_encoder_test.go b/adapters/repos/db/vector/compressionhelpers/tile_encoder_test.go deleted file mode 100644 index 2c760c149c6aca9fb6abf578e85b54a0c9e2cde2..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/compressionhelpers/tile_encoder_test.go +++ /dev/null @@ -1,67 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build !race - -package compressionhelpers_test - -import ( - "math" - "math/rand" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/adapters/repos/db/vector/compressionhelpers" -) - -func Test_NoRaceTileEncoderEncode(t *testing.T) { - encoder := compressionhelpers.NewTileEncoder(4, 0, compressionhelpers.LogNormalEncoderDistribution) - for i := 0; i < 1000000; i++ { - encoder.Add([]float32{float32(rand.NormFloat64() + 100)}) - } - encoder.Fit([][]float32{}) - assert.Equal(t, encoder.Encode([]float32{0.1}), byte(0)) - assert.Equal(t, encoder.Encode([]float32{100}), byte(8)) - assert.Equal(t, encoder.Encode([]float32{1000}), byte(16)) -} - -func Test_NoRaceTileEncoderCentroids(t *testing.T) { - encoder := compressionhelpers.NewTileEncoder(4, 0, compressionhelpers.LogNormalEncoderDistribution) - for i := 0; i < 1000000; i++ { - encoder.Add([]float32{float32(rand.NormFloat64() + 100)}) - } - encoder.Fit([][]float32{}) - assert.Equal(t, math.Round(float64(encoder.Centroid(0)[0])), 98.0) - assert.Equal(t, math.Round(float64(encoder.Centroid(2)[0])), 99.0) - assert.Equal(t, math.Round(float64(encoder.Centroid(14)[0])), 101.0) -} - -func Test_NoRaceNormalTileEncoderEncode(t *testing.T) { - encoder := compressionhelpers.NewTileEncoder(4, 0, compressionhelpers.NormalEncoderDistribution) - for i := 0; i < 1000000; i++ { - encoder.Add([]float32{float32(rand.NormFloat64())}) - } - encoder.Fit([][]float32{}) - assert.Equal(t, encoder.Encode([]float32{0.1}), byte(8)) - assert.Equal(t, encoder.Encode([]float32{100}), byte(16)) - assert.Equal(t, encoder.Encode([]float32{1000}), byte(16)) -} - -func Test_NoRaceNormalTileEncoderCentroids(t *testing.T) { - encoder := compressionhelpers.NewTileEncoder(4, 0, compressionhelpers.NormalEncoderDistribution) - for i := 0; i < 1000000; i++ { - encoder.Add([]float32{float32(rand.NormFloat64())}) - } - encoder.Fit([][]float32{}) - assert.Equal(t, math.Round(float64(encoder.Centroid(0)[0])), -2.0) - assert.Equal(t, math.Round(float64(encoder.Centroid(8)[0])), 0.0) - assert.Equal(t, math.Round(float64(encoder.Centroid(15)[0])), 2.0) -} diff --git a/adapters/repos/db/vector/compressionhelpers/utils.go b/adapters/repos/db/vector/compressionhelpers/utils.go deleted file mode 100644 index 1ca3327b6bfcd20170bb34fc57f08d78fbd622c5..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/compressionhelpers/utils.go +++ /dev/null @@ -1,37 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package compressionhelpers - -import ( - "math" - "runtime" - "sync" -) - -type Action func(taskIndex uint64) - -func Concurrently(n uint64, action Action) { - n64 := float64(n) - workerCount := runtime.GOMAXPROCS(0) - wg := &sync.WaitGroup{} - split := uint64(math.Ceil(n64 / float64(workerCount))) - for worker := uint64(0); worker < uint64(workerCount); worker++ { - wg.Add(1) - go func(workerID uint64) { - defer wg.Done() - for i := workerID * split; i < uint64(math.Min(float64((workerID+1)*split), n64)); i++ { - action(i) - } - }(worker) - } - wg.Wait() -} diff --git a/adapters/repos/db/vector/flat/config.go b/adapters/repos/db/vector/flat/config.go deleted file mode 100644 index 013c67efa12228c29a9c66659a6a8cd49b1c703e..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/flat/config.go +++ /dev/null @@ -1,38 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package flat - -import ( - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" - "github.com/weaviate/weaviate/entities/errorcompounder" -) - -type Config struct { - ID string - Logger logrus.FieldLogger - DistanceProvider distancer.Provider -} - -func (c Config) Validate() error { - ec := &errorcompounder.ErrorCompounder{} - - if c.ID == "" { - ec.Addf("id cannot be empty") - } - - if c.DistanceProvider == nil { - ec.Addf("distancerProvider cannot be nil") - } - - return ec.ToError() -} diff --git a/adapters/repos/db/vector/flat/config_update_test.go b/adapters/repos/db/vector/flat/config_update_test.go deleted file mode 100644 index 98d98cad27970a7bfea8cb61bb36f43dd84bb1bf..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/flat/config_update_test.go +++ /dev/null @@ -1,78 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package flat - -import ( - "testing" - - "github.com/pkg/errors" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/schema" - ent "github.com/weaviate/weaviate/entities/vectorindex/flat" -) - -func TestFlatUserConfigUpdates(t *testing.T) { - t.Run("various immutable and mutable fields", func(t *testing.T) { - type test struct { - name string - initial schema.VectorIndexConfig - update schema.VectorIndexConfig - expectedError error - } - - tests := []test{ - { - name: "attempting to change pq enabled", - initial: ent.UserConfig{PQ: ent.CompressionUserConfig{Enabled: false}}, - update: ent.UserConfig{PQ: ent.CompressionUserConfig{Enabled: true}}, - expectedError: errors.Errorf( - "pq is immutable: " + - "attempted change from \"false\" to \"true\""), - }, - { - name: "attempting to change bq enabled", - initial: ent.UserConfig{BQ: ent.CompressionUserConfig{Enabled: true}}, - update: ent.UserConfig{BQ: ent.CompressionUserConfig{Enabled: false}}, - expectedError: errors.Errorf( - "bq is immutable: " + - "attempted change from \"true\" to \"false\""), - }, - { - name: "attempting to change distance", - initial: ent.UserConfig{Distance: "cosine"}, - update: ent.UserConfig{Distance: "l2-squared"}, - expectedError: errors.Errorf( - "distance is immutable: " + - "attempted change from \"cosine\" to \"l2-squared\""), - }, - { - name: "changing rescoreLimit", - initial: ent.UserConfig{BQ: ent.CompressionUserConfig{RescoreLimit: 10}}, - update: ent.UserConfig{BQ: ent.CompressionUserConfig{RescoreLimit: 100}}, - expectedError: nil, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - err := ValidateUserConfigUpdate(test.initial, test.update) - if test.expectedError == nil { - assert.Nil(t, err) - } else { - require.NotNil(t, err, "update validation must error") - assert.Equal(t, test.expectedError.Error(), err.Error()) - } - }) - } - }) -} diff --git a/adapters/repos/db/vector/flat/index.go b/adapters/repos/db/vector/flat/index.go deleted file mode 100644 index 4baf117eacf38b9d9dca16f0b2d15ebb8c59f75e..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/flat/index.go +++ /dev/null @@ -1,719 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package flat - -import ( - "context" - "encoding/binary" - "fmt" - "io" - "math" - "strings" - "sync" - "sync/atomic" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv" - "github.com/weaviate/weaviate/adapters/repos/db/priorityqueue" - "github.com/weaviate/weaviate/adapters/repos/db/vector/cache" - "github.com/weaviate/weaviate/adapters/repos/db/vector/common" - "github.com/weaviate/weaviate/adapters/repos/db/vector/compressionhelpers" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" - "github.com/weaviate/weaviate/entities/schema" - flatent "github.com/weaviate/weaviate/entities/vectorindex/flat" - "github.com/weaviate/weaviate/usecases/floatcomp" -) - -const ( - compressionBQ = "bq" - compressionPQ = "pq" - compressionNone = "none" -) - -type flat struct { - sync.Mutex - id string - dims int32 - store *lsmkv.Store - logger logrus.FieldLogger - distancerProvider distancer.Provider - trackDimensionsOnce sync.Once - rescore int64 - bq compressionhelpers.BinaryQuantizer - - pqResults *common.PqMaxPool - pool *pools - - compression string - bqCache cache.Cache[uint64] -} - -type distanceCalc func(vecAsBytes []byte) (float32, error) - -func New(cfg Config, uc flatent.UserConfig, store *lsmkv.Store) (*flat, error) { - if err := cfg.Validate(); err != nil { - return nil, errors.Wrap(err, "invalid config") - } - - logger := cfg.Logger - if logger == nil { - l := logrus.New() - l.Out = io.Discard - logger = l - } - - index := &flat{ - id: cfg.ID, - logger: logger, - distancerProvider: cfg.DistanceProvider, - rescore: extractCompressionRescore(uc), - pqResults: common.NewPqMaxPool(100), - compression: extractCompression(uc), - pool: newPools(), - store: store, - } - index.initBuckets(context.Background()) - if uc.BQ.Enabled && uc.BQ.Cache { - index.bqCache = cache.NewShardedUInt64LockCache(index.getBQVector, uc.VectorCacheMaxObjects, cfg.Logger, 0) - } - - return index, nil -} - -func (flat *flat) getBQVector(ctx context.Context, id uint64) ([]uint64, error) { - key := flat.pool.byteSlicePool.Get(8) - defer flat.pool.byteSlicePool.Put(key) - binary.BigEndian.PutUint64(key.slice, id) - bytes, err := flat.store.Bucket(helpers.VectorsCompressedBucketLSM).Get(key.slice) - if err != nil { - return nil, err - } - return uint64SliceFromByteSlice(bytes, make([]uint64, len(bytes)/8)), nil -} - -func extractCompression(uc flatent.UserConfig) string { - if uc.BQ.Enabled && uc.PQ.Enabled { - return compressionNone - } - - if uc.BQ.Enabled { - return compressionBQ - } - - if uc.PQ.Enabled { - return compressionPQ - } - - return compressionNone -} - -func extractCompressionRescore(uc flatent.UserConfig) int64 { - compression := extractCompression(uc) - switch compression { - case compressionPQ: - return int64(uc.PQ.RescoreLimit) - case compressionBQ: - return int64(uc.BQ.RescoreLimit) - default: - return 0 - } -} - -func (index *flat) storeCompressedVector(id uint64, vector []byte) { - index.storeGenericVector(id, vector, helpers.VectorsCompressedBucketLSM) -} - -func (index *flat) storeVector(id uint64, vector []byte) { - index.storeGenericVector(id, vector, helpers.VectorsBucketLSM) -} - -func (index *flat) storeGenericVector(id uint64, vector []byte, bucket string) { - idBytes := make([]byte, 8) - binary.BigEndian.PutUint64(idBytes, id) - index.store.Bucket(bucket).Put(idBytes, vector) -} - -func (index *flat) isBQ() bool { - return index.compression == compressionBQ -} - -func (index *flat) isBQCached() bool { - return index.bqCache != nil -} - -func (index *flat) Compressed() bool { - return index.compression != compressionNone -} - -func (index *flat) initBuckets(ctx context.Context) error { - if err := index.store.CreateOrLoadBucket(ctx, helpers.VectorsBucketLSM, - lsmkv.WithForceCompation(true), - lsmkv.WithUseBloomFilter(false), - lsmkv.WithCalcCountNetAdditions(false), - ); err != nil { - return fmt.Errorf("Create or load flat vectors bucket: %w", err) - } - if index.isBQ() { - if err := index.store.CreateOrLoadBucket(ctx, helpers.VectorsCompressedBucketLSM, - lsmkv.WithForceCompation(true), - lsmkv.WithUseBloomFilter(false), - lsmkv.WithCalcCountNetAdditions(false), - ); err != nil { - return fmt.Errorf("Create or load flat compressed vectors bucket: %w", err) - } - } - return nil -} - -func (index *flat) AddBatch(ctx context.Context, ids []uint64, vectors [][]float32) error { - if err := ctx.Err(); err != nil { - return err - } - if len(ids) != len(vectors) { - return errors.Errorf("ids and vectors sizes does not match") - } - if len(ids) == 0 { - return errors.Errorf("insertBatch called with empty lists") - } - for i := range ids { - if err := ctx.Err(); err != nil { - return err - } - if err := index.Add(ids[i], vectors[i]); err != nil { - return err - } - } - return nil -} - -func byteSliceFromUint64Slice(vector []uint64, slice []byte) []byte { - for i := range vector { - binary.LittleEndian.PutUint64(slice[i*8:], vector[i]) - } - return slice -} - -func byteSliceFromFloat32Slice(vector []float32, slice []byte) []byte { - for i := range vector { - binary.LittleEndian.PutUint32(slice[i*4:], math.Float32bits(vector[i])) - } - return slice -} - -func uint64SliceFromByteSlice(vector []byte, slice []uint64) []uint64 { - for i := range slice { - slice[i] = binary.LittleEndian.Uint64(vector[i*8:]) - } - return slice -} - -func float32SliceFromByteSlice(vector []byte, slice []float32) []float32 { - for i := range slice { - slice[i] = math.Float32frombits(binary.LittleEndian.Uint32(vector[i*4:])) - } - return slice -} - -func (index *flat) Add(id uint64, vector []float32) error { - index.trackDimensionsOnce.Do(func() { - atomic.StoreInt32(&index.dims, int32(len(vector))) - - if index.isBQ() { - index.bq = compressionhelpers.NewBinaryQuantizer(nil) - } - }) - if len(vector) != int(index.dims) { - return errors.Errorf("insert called with a vector of the wrong size") - } - vector = index.normalized(vector) - slice := make([]byte, len(vector)*4) - index.storeVector(id, byteSliceFromFloat32Slice(vector, slice)) - - if index.isBQ() { - vectorBQ := index.bq.Encode(vector) - if index.isBQCached() { - index.bqCache.Grow(id) - index.bqCache.Preload(id, vectorBQ) - } - slice = make([]byte, len(vectorBQ)*8) - index.storeCompressedVector(id, byteSliceFromUint64Slice(vectorBQ, slice)) - } - return nil -} - -func (index *flat) Delete(ids ...uint64) error { - for i := range ids { - if index.isBQCached() { - index.bqCache.Delete(context.Background(), ids[i]) - } - idBytes := make([]byte, 8) - binary.BigEndian.PutUint64(idBytes, ids[i]) - - if err := index.store.Bucket(helpers.VectorsBucketLSM).Delete(idBytes); err != nil { - return err - } - - if index.isBQ() { - if err := index.store.Bucket(helpers.VectorsCompressedBucketLSM).Delete(idBytes); err != nil { - return err - } - } - } - return nil -} - -func (index *flat) searchTimeRescore(k int) int { - // load atomically, so we can get away with concurrent updates of the - // userconfig without having to set a lock each time we try to read - which - // can be so common that it would cause considerable overhead - if rescore := int(atomic.LoadInt64(&index.rescore)); rescore > k { - return rescore - } - return k -} - -func (index *flat) SearchByVector(vector []float32, k int, allow helpers.AllowList) ([]uint64, []float32, error) { - switch index.compression { - case compressionBQ: - return index.searchByVectorBQ(vector, k, allow) - case compressionPQ: - // use uncompressed for now - fallthrough - default: - return index.searchByVector(vector, k, allow) - } -} - -func (index *flat) searchByVector(vector []float32, k int, allow helpers.AllowList) ([]uint64, []float32, error) { - heap := index.pqResults.GetMax(k) - defer index.pqResults.Put(heap) - - vector = index.normalized(vector) - - if err := index.findTopVectors(heap, allow, k, - index.store.Bucket(helpers.VectorsBucketLSM).Cursor, - index.createDistanceCalc(vector), - ); err != nil { - return nil, nil, err - } - - ids, dists := index.extractHeap(heap) - return ids, dists, nil -} - -func (index *flat) createDistanceCalc(vector []float32) distanceCalc { - return func(vecAsBytes []byte) (float32, error) { - vecSlice := index.pool.float32SlicePool.Get(len(vecAsBytes) / 4) - defer index.pool.float32SlicePool.Put(vecSlice) - - candidate := float32SliceFromByteSlice(vecAsBytes, vecSlice.slice) - distance, _, err := index.distancerProvider.SingleDist(vector, candidate) - return distance, err - } -} - -func (index *flat) searchByVectorBQ(vector []float32, k int, allow helpers.AllowList) ([]uint64, []float32, error) { - rescore := index.searchTimeRescore(k) - heap := index.pqResults.GetMax(rescore) - defer index.pqResults.Put(heap) - - vector = index.normalized(vector) - vectorBQ := index.bq.Encode(vector) - - if index.isBQCached() { - if err := index.findTopVectorsCached(heap, allow, rescore, vectorBQ); err != nil { - return nil, nil, err - } - } else { - if err := index.findTopVectors(heap, allow, rescore, - index.store.Bucket(helpers.VectorsCompressedBucketLSM).Cursor, - index.createDistanceCalcBQ(vectorBQ), - ); err != nil { - return nil, nil, err - } - } - - distanceCalc := index.createDistanceCalc(vector) - idsSlice := index.pool.uint64SlicePool.Get(heap.Len()) - defer index.pool.uint64SlicePool.Put(idsSlice) - - for i := range idsSlice.slice { - idsSlice.slice[i] = heap.Pop().ID - } - for _, id := range idsSlice.slice { - candidateAsBytes, err := index.vectorById(id) - if err != nil { - return nil, nil, err - } - distance, err := distanceCalc(candidateAsBytes) - if err != nil { - return nil, nil, err - } - index.insertToHeap(heap, k, id, distance) - } - - ids, dists := index.extractHeap(heap) - return ids, dists, nil -} - -func (index *flat) createDistanceCalcBQ(vectorBQ []uint64) distanceCalc { - return func(vecAsBytes []byte) (float32, error) { - vecSliceBQ := index.pool.uint64SlicePool.Get(len(vecAsBytes) / 8) - defer index.pool.uint64SlicePool.Put(vecSliceBQ) - - candidate := uint64SliceFromByteSlice(vecAsBytes, vecSliceBQ.slice) - return index.bq.DistanceBetweenCompressedVectors(candidate, vectorBQ) - } -} - -func (index *flat) vectorById(id uint64) ([]byte, error) { - idSlice := index.pool.byteSlicePool.Get(8) - defer index.pool.byteSlicePool.Put(idSlice) - - binary.BigEndian.PutUint64(idSlice.slice, id) - return index.store.Bucket(helpers.VectorsBucketLSM).Get(idSlice.slice) -} - -// populates given heap with smallest distances and corresponding ids calculated by -// distanceCalc -func (index *flat) findTopVectors(heap *priorityqueue.Queue[any], - allow helpers.AllowList, limit int, cursorFn func() *lsmkv.CursorReplace, - distanceCalc distanceCalc, -) error { - var key []byte - var v []byte - var id uint64 - allowMax := uint64(0) - - cursor := cursorFn() - defer cursor.Close() - - if allow != nil { - // nothing allowed, skip search - if allow.IsEmpty() { - return nil - } - - allowMax = allow.Max() - - idSlice := index.pool.byteSlicePool.Get(8) - binary.BigEndian.PutUint64(idSlice.slice, allow.Min()) - key, v = cursor.Seek(idSlice.slice) - index.pool.byteSlicePool.Put(idSlice) - } else { - key, v = cursor.First() - } - - // since keys are sorted, once key/id get greater than max allowed one - // further search can be stopped - for ; key != nil && (allow == nil || id <= allowMax); key, v = cursor.Next() { - id = binary.BigEndian.Uint64(key) - if allow == nil || allow.Contains(id) { - distance, err := distanceCalc(v) - if err != nil { - return err - } - index.insertToHeap(heap, limit, id, distance) - } - } - return nil -} - -// populates given heap with smallest distances and corresponding ids calculated by -// distanceCalc -func (index *flat) findTopVectorsCached(heap *priorityqueue.Queue[any], - allow helpers.AllowList, limit int, vectorBQ []uint64, -) error { - var id uint64 - allowMax := uint64(0) - - if allow != nil { - // nothing allowed, skip search - if allow.IsEmpty() { - return nil - } - - allowMax = allow.Max() - - id = allow.Min() - } else { - id = 0 - } - all := index.bqCache.Len() - - // since keys are sorted, once key/id get greater than max allowed one - // further search can be stopped - for ; id < uint64(all) && (allow == nil || id <= allowMax); id++ { - if allow == nil || allow.Contains(id) { - vec, err := index.bqCache.Get(context.Background(), id) - if err != nil { - return err - } - if len(vec) == 0 { - continue - } - distance, err := index.bq.DistanceBetweenCompressedVectors(vec, vectorBQ) - if err != nil { - return err - } - index.insertToHeap(heap, limit, id, distance) - } - } - return nil -} - -func (index *flat) insertToHeap(heap *priorityqueue.Queue[any], - limit int, id uint64, distance float32, -) { - if heap.Len() < limit { - heap.Insert(id, distance) - } else if heap.Top().Dist > distance { - heap.Pop() - heap.Insert(id, distance) - } -} - -func (index *flat) extractHeap(heap *priorityqueue.Queue[any], -) ([]uint64, []float32) { - len := heap.Len() - - ids := make([]uint64, len) - dists := make([]float32, len) - for i := len - 1; i >= 0; i-- { - item := heap.Pop() - ids[i] = item.ID - dists[i] = item.Dist - } - return ids, dists -} - -func (index *flat) normalized(vector []float32) []float32 { - if index.distancerProvider.Type() == "cosine-dot" { - // cosine-dot requires normalized vectors, as the dot product and cosine - // similarity are only identical if the vector is normalized - return distancer.Normalize(vector) - } - return vector -} - -func (index *flat) SearchByVectorDistance(vector []float32, targetDistance float32, maxLimit int64, allow helpers.AllowList) ([]uint64, []float32, error) { - var ( - searchParams = newSearchByDistParams(maxLimit) - - resultIDs []uint64 - resultDist []float32 - ) - - recursiveSearch := func() (bool, error) { - totalLimit := searchParams.TotalLimit() - ids, dist, err := index.SearchByVector(vector, totalLimit, allow) - if err != nil { - return false, errors.Wrap(err, "vector search") - } - - // if there is less results than given limit search can be stopped - shouldContinue := !(len(ids) < totalLimit) - - // ensures the indexes aren't out of range - offsetCap := searchParams.OffsetCapacity(ids) - totalLimitCap := searchParams.TotalLimitCapacity(ids) - - if offsetCap == totalLimitCap { - return false, nil - } - - ids, dist = ids[offsetCap:totalLimitCap], dist[offsetCap:totalLimitCap] - for i := range ids { - if aboveThresh := dist[i] <= targetDistance; aboveThresh || - floatcomp.InDelta(float64(dist[i]), float64(targetDistance), 1e-6) { - resultIDs = append(resultIDs, ids[i]) - resultDist = append(resultDist, dist[i]) - } else { - // as soon as we encounter a certainty which - // is below threshold, we can stop searching - shouldContinue = false - break - } - } - - return shouldContinue, nil - } - - var shouldContinue bool - var err error - for shouldContinue, err = recursiveSearch(); shouldContinue && err == nil; { - searchParams.Iterate() - if searchParams.MaxLimitReached() { - index.logger. - WithField("action", "unlimited_vector_search"). - Warnf("maximum search limit of %d results has been reached", - searchParams.MaximumSearchLimit()) - break - } - } - if err != nil { - return nil, nil, err - } - - return resultIDs, resultDist, nil -} - -func (index *flat) UpdateUserConfig(updated schema.VectorIndexConfig, callback func()) error { - parsed, ok := updated.(flatent.UserConfig) - if !ok { - callback() - return errors.Errorf("config is not UserConfig, but %T", updated) - } - - // Store automatically as a lock here would be very expensive, this value is - // read on every single user-facing search, which can be highly concurrent - atomic.StoreInt64(&index.rescore, extractCompressionRescore(parsed)) - - callback() - return nil -} - -func (index *flat) Drop(ctx context.Context) error { - // nothing to do here - // Shard::drop will take care of handling store's buckets - return nil -} - -func (index *flat) Flush() error { - // nothing to do here - // Shard will take care of handling store's buckets - return nil -} - -func (index *flat) Shutdown(ctx context.Context) error { - // nothing to do here - // Shard::shutdown will take care of handling store's buckets - return nil -} - -func (index *flat) SwitchCommitLogs(context.Context) error { - return nil -} - -func (index *flat) ListFiles(ctx context.Context, basePath string) ([]string, error) { - // nothing to do here - // Shard::ListBackupFiles will take care of handling store's buckets - return []string{}, nil -} - -func (i *flat) ValidateBeforeInsert(vector []float32) error { - return nil -} - -func (index *flat) PostStartup() { - if !index.isBQCached() { - return - } - cursor := index.store.Bucket(helpers.VectorsCompressedBucketLSM).Cursor() - defer cursor.Close() - - for key, v := cursor.First(); key != nil; key, v = cursor.Next() { - id := binary.BigEndian.Uint64(key) - index.bqCache.Preload(id, uint64SliceFromByteSlice(v, make([]uint64, len(v)/8))) - } -} - -func (index *flat) Dump(labels ...string) { - if len(labels) > 0 { - fmt.Printf("--------------------------------------------------\n") - fmt.Printf("-- %s\n", strings.Join(labels, ", ")) - } - fmt.Printf("--------------------------------------------------\n") - fmt.Printf("ID: %s\n", index.id) - fmt.Printf("--------------------------------------------------\n") -} - -func (index *flat) DistanceBetweenVectors(x, y []float32) (float32, bool, error) { - return index.distancerProvider.SingleDist(x, y) -} - -func (index *flat) ContainsNode(id uint64) bool { - return true -} - -func (index *flat) DistancerProvider() distancer.Provider { - return index.distancerProvider -} - -func newSearchByDistParams(maxLimit int64) *common.SearchByDistParams { - initialOffset := 0 - initialLimit := common.DefaultSearchByDistInitialLimit - - return common.NewSearchByDistParams(initialOffset, initialLimit, initialOffset+initialLimit, maxLimit) -} - -type immutableParameter struct { - accessor func(c flatent.UserConfig) interface{} - name string -} - -func validateImmutableField(u immutableParameter, - previous, next flatent.UserConfig, -) error { - oldField := u.accessor(previous) - newField := u.accessor(next) - if oldField != newField { - return errors.Errorf("%s is immutable: attempted change from \"%v\" to \"%v\"", - u.name, oldField, newField) - } - - return nil -} - -func ValidateUserConfigUpdate(initial, updated schema.VectorIndexConfig) error { - initialParsed, ok := initial.(flatent.UserConfig) - if !ok { - return errors.Errorf("initial is not UserConfig, but %T", initial) - } - - updatedParsed, ok := updated.(flatent.UserConfig) - if !ok { - return errors.Errorf("updated is not UserConfig, but %T", updated) - } - - immutableFields := []immutableParameter{ - { - name: "distance", - accessor: func(c flatent.UserConfig) interface{} { return c.Distance }, - }, - { - name: "bq.cache", - accessor: func(c flatent.UserConfig) interface{} { return c.BQ.Cache }, - }, - { - name: "pq.cache", - accessor: func(c flatent.UserConfig) interface{} { return c.PQ.Cache }, - }, - { - name: "pq", - accessor: func(c flatent.UserConfig) interface{} { return c.PQ.Enabled }, - }, - { - name: "bq", - accessor: func(c flatent.UserConfig) interface{} { return c.BQ.Enabled }, - }, - } - - for _, u := range immutableFields { - if err := validateImmutableField(u, initialParsed, updatedParsed); err != nil { - return err - } - } - return nil -} diff --git a/adapters/repos/db/vector/flat/index_test.go b/adapters/repos/db/vector/flat/index_test.go deleted file mode 100644 index e49cf3555869f3c34c00afb1fee11db1f637b28c..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/flat/index_test.go +++ /dev/null @@ -1,247 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build !race - -package flat - -import ( - "encoding/binary" - "errors" - "fmt" - "os" - "strconv" - "sync" - "testing" - "time" - - "github.com/google/uuid" - "github.com/sirupsen/logrus" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv" - - "github.com/weaviate/weaviate/adapters/repos/db/vector/compressionhelpers" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" - "github.com/weaviate/weaviate/adapters/repos/db/vector/testinghelpers" - "github.com/weaviate/weaviate/entities/cyclemanager" - flatent "github.com/weaviate/weaviate/entities/vectorindex/flat" -) - -func distanceWrapper(provider distancer.Provider) func(x, y []float32) float32 { - return func(x, y []float32) float32 { - dist, _, _ := provider.SingleDist(x, y) - return dist - } -} - -func run(dirName string, logger *logrus.Logger, compression string, vectorCache bool, - vectors [][]float32, queries [][]float32, k int, truths [][]uint64, - extraVectorsForDelete [][]float32, allowIds []uint64, - distancer distancer.Provider, -) (float32, float32, error) { - vectors_size := len(vectors) - queries_size := len(queries) - runId := uuid.New().String() - - store, err := lsmkv.New(dirName, dirName, logger, nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop()) - if err != nil { - return 0, 0, err - } - - pq := flatent.CompressionUserConfig{ - Enabled: false, - } - bq := flatent.CompressionUserConfig{ - Enabled: false, - } - switch compression { - case compressionPQ: - pq.Enabled = true - pq.RescoreLimit = 100 * k - pq.Cache = vectorCache - case compressionBQ: - bq.Enabled = true - bq.RescoreLimit = 100 * k - bq.Cache = vectorCache - } - index, err := New(Config{ - ID: runId, - DistanceProvider: distancer, - }, flatent.UserConfig{ - PQ: pq, - BQ: bq, - }, store) - if err != nil { - return 0, 0, err - } - - compressionhelpers.Concurrently(uint64(vectors_size), func(id uint64) { - index.Add(id, vectors[id]) - }) - - for i := range extraVectorsForDelete { - index.Add(uint64(vectors_size+i), extraVectorsForDelete[i]) - } - - for i := range extraVectorsForDelete { - Id := make([]byte, 16) - binary.BigEndian.PutUint64(Id[8:], uint64(vectors_size+i)) - err := index.Delete(uint64(vectors_size + i)) - if err != nil { - return 0, 0, err - } - } - - var relevant uint64 - var retrieved int - var querying time.Duration = 0 - mutex := new(sync.Mutex) - - var allowList helpers.AllowList = nil - if allowIds != nil { - allowList = helpers.NewAllowList(allowIds...) - } - err = nil - compressionhelpers.Concurrently(uint64(len(queries)), func(i uint64) { - before := time.Now() - results, _, _ := index.SearchByVector(queries[i], k, allowList) - - since := time.Since(before) - len := len(results) - matches := testinghelpers.MatchesInLists(truths[i], results) - - if hasDuplicates(results) { - err = errors.New("results have duplicates") - } - - mutex.Lock() - querying += since - retrieved += len - relevant += matches - mutex.Unlock() - }) - - return float32(relevant) / float32(retrieved), float32(querying.Microseconds()) / float32(queries_size), err -} - -func hasDuplicates(results []uint64) bool { - for i := 0; i < len(results)-1; i++ { - for j := i + 1; j < len(results); j++ { - if results[i] == results[j] { - return true - } - } - } - return false -} - -func Test_NoRaceFlatIndex(t *testing.T) { - dirName := t.TempDir() - - logger, _ := test.NewNullLogger() - - dimensions := 256 - vectors_size := 12000 - queries_size := 100 - k := 10 - vectors, queries := testinghelpers.RandomVecs(vectors_size, queries_size, dimensions) - testinghelpers.Normalize(vectors) - testinghelpers.Normalize(queries) - distancer := distancer.NewCosineDistanceProvider() - - truths := make([][]uint64, queries_size) - for i := range queries { - truths[i], _ = testinghelpers.BruteForce(vectors, queries[i], k, distanceWrapper(distancer)) - } - - extraVectorsForDelete, _ := testinghelpers.RandomVecs(5_000, 0, dimensions) - for _, compression := range []string{compressionNone, compressionBQ} { - t.Run("compression: "+compression, func(t *testing.T) { - for _, cache := range []bool{false, true} { - t.Run("cache: "+strconv.FormatBool(cache), func(t *testing.T) { - if compression == compressionNone && cache == true { - return - } - targetRecall := float32(0.99) - if compression == compressionBQ { - targetRecall = 0.8 - } - t.Run("recall", func(t *testing.T) { - recall, latency, err := run(dirName, logger, compression, cache, vectors, queries, k, truths, nil, nil, distancer) - require.Nil(t, err) - - fmt.Println(recall, latency) - assert.Greater(t, recall, targetRecall) - assert.Less(t, latency, float32(1_000_000)) - }) - - t.Run("recall with deletes", func(t *testing.T) { - recall, latency, err := run(dirName, logger, compression, cache, vectors, queries, k, truths, extraVectorsForDelete, nil, distancer) - require.Nil(t, err) - - fmt.Println(recall, latency) - assert.Greater(t, recall, targetRecall) - assert.Less(t, latency, float32(1_000_000)) - }) - }) - } - }) - } - for _, compression := range []string{compressionNone, compressionBQ} { - t.Run("compression: "+compression, func(t *testing.T) { - for _, cache := range []bool{false, true} { - t.Run("cache: "+strconv.FormatBool(cache), func(t *testing.T) { - from := 0 - to := 3_000 - for i := range queries { - truths[i], _ = testinghelpers.BruteForce(vectors[from:to], queries[i], k, distanceWrapper(distancer)) - } - - allowIds := make([]uint64, 0, to-from) - for i := uint64(from); i < uint64(to); i++ { - allowIds = append(allowIds, i) - } - targetRecall := float32(0.99) - if compression == compressionBQ { - targetRecall = 0.8 - } - - t.Run("recall on filtered", func(t *testing.T) { - recall, latency, err := run(dirName, logger, compression, cache, vectors, queries, k, truths, nil, allowIds, distancer) - require.Nil(t, err) - - fmt.Println(recall, latency) - assert.Greater(t, recall, targetRecall) - assert.Less(t, latency, float32(1_000_000)) - }) - - t.Run("recall on filtered with deletes", func(t *testing.T) { - recall, latency, err := run(dirName, logger, compression, cache, vectors, queries, k, truths, extraVectorsForDelete, allowIds, distancer) - require.Nil(t, err) - - fmt.Println(recall, latency) - assert.Greater(t, recall, targetRecall) - assert.Less(t, latency, float32(1_000_000)) - }) - }) - } - }) - } - - err := os.RemoveAll(dirName) - if err != nil { - fmt.Println(err) - } -} diff --git a/adapters/repos/db/vector/flat/pools.go b/adapters/repos/db/vector/flat/pools.go deleted file mode 100644 index 9266bdeb7ede659e93c3d3327454ed6bffce9d77..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/flat/pools.go +++ /dev/null @@ -1,65 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package flat - -import ( - "sync" -) - -const defaultSize = 200 - -type pools struct { - byteSlicePool *slicePool[byte] - uint64SlicePool *slicePool[uint64] - float32SlicePool *slicePool[float32] -} - -func newPools() *pools { - return &pools{ - byteSlicePool: newSlicePool[byte](), - uint64SlicePool: newSlicePool[uint64](), - float32SlicePool: newSlicePool[float32](), - } -} - -type slicePool[T any] struct { - pool *sync.Pool -} - -type SliceStruct[T any] struct { - slice []T -} - -func newSlicePool[T any]() *slicePool[T] { - return &slicePool[T]{ - pool: &sync.Pool{ - New: func() interface{} { - return &SliceStruct[T]{ - slice: make([]T, defaultSize), - } - }, - }, - } -} - -func (p *slicePool[T]) Get(capacity int) *SliceStruct[T] { - t := p.pool.Get().(*SliceStruct[T]) - if cap(t.slice) < capacity { - t.slice = make([]T, capacity) - } - t.slice = t.slice[:capacity] - return t -} - -func (p *slicePool[T]) Put(t *SliceStruct[T]) { - p.pool.Put(t) -} diff --git a/adapters/repos/db/vector/geo/coordinates_for_id.go b/adapters/repos/db/vector/geo/coordinates_for_id.go deleted file mode 100644 index e79bb60b5c870162257207caa97c5336434640c2..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/geo/coordinates_for_id.go +++ /dev/null @@ -1,47 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package geo - -import ( - "context" - "fmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// CoordinatesForID must provide the geo coordinates for the specified index -// id -type CoordinatesForID func(ctx context.Context, id uint64) (*models.GeoCoordinates, error) - -// VectorForID transforms the geo coordinates into a "vector" of fixed length -// two, where element 0 represents the latitude and element 1 represents the -// longitude. This way it is usable by a generic vector index such as HNSW -func (cfid CoordinatesForID) VectorForID(ctx context.Context, id uint64) ([]float32, error) { - coordinates, err := cfid(ctx, id) - if err != nil { - return nil, err - } - - return geoCoordiantesToVector(coordinates) -} - -func geoCoordiantesToVector(in *models.GeoCoordinates) ([]float32, error) { - if in.Latitude == nil { - return nil, fmt.Errorf("latitude must be set") - } - - if in.Longitude == nil { - return nil, fmt.Errorf("longitude must be set") - } - - return []float32{*in.Latitude, *in.Longitude}, nil -} diff --git a/adapters/repos/db/vector/geo/geo.go b/adapters/repos/db/vector/geo/geo.go deleted file mode 100644 index 415e3e700a04ea44fcc2ea5b2258a6853e38cc27..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/geo/geo.go +++ /dev/null @@ -1,141 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package geo - -import ( - "context" - "fmt" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" - "github.com/weaviate/weaviate/entities/cyclemanager" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - hnswent "github.com/weaviate/weaviate/entities/vectorindex/hnsw" -) - -// Index wraps another index to provide geo searches. This allows us to reuse -// the hnsw vector index, without making geo searches dependent on -// hnsw-specific features. -// -// In the future we could use this level of abstraction to provide a better -// suited geo-index if we deem it necessary -type Index struct { - config Config - vectorIndex vectorIndex -} - -// vectorIndex represents the underlying vector index, typically hnsw -type vectorIndex interface { - Add(id uint64, vector []float32) error - KnnSearchByVectorMaxDist(query []float32, dist float32, ef int, - allowList helpers.AllowList) ([]uint64, error) - Delete(id ...uint64) error - Dump(...string) - Drop(ctx context.Context) error - PostStartup() -} - -// Config is passed to the GeoIndex when its created -type Config struct { - ID string - CoordinatesForID CoordinatesForID - DisablePersistence bool - RootPath string - Logger logrus.FieldLogger -} - -func NewIndex(config Config, - commitLogMaintenanceCallbacks, tombstoneCleanupCallbacks, - compactionCallbacks, flushCallbacks cyclemanager.CycleCallbackGroup, -) (*Index, error) { - vi, err := hnsw.New(hnsw.Config{ - VectorForIDThunk: config.CoordinatesForID.VectorForID, - ID: config.ID, - RootPath: config.RootPath, - MakeCommitLoggerThunk: makeCommitLoggerFromConfig(config, commitLogMaintenanceCallbacks), - DistanceProvider: distancer.NewGeoProvider(), - }, hnswent.UserConfig{ - MaxConnections: 64, - EFConstruction: 128, - CleanupIntervalSeconds: hnswent.DefaultCleanupIntervalSeconds, - }, tombstoneCleanupCallbacks, compactionCallbacks, flushCallbacks, nil) - if err != nil { - return nil, errors.Wrap(err, "underlying hnsw index") - } - - i := &Index{ - config: config, - vectorIndex: vi, - } - - return i, nil -} - -func (i *Index) Drop(ctx context.Context) error { - if err := i.vectorIndex.Drop(ctx); err != nil { - return err - } - - i.vectorIndex = nil - return nil -} - -func (i *Index) PostStartup() { - i.vectorIndex.PostStartup() -} - -func makeCommitLoggerFromConfig(config Config, maintenanceCallbacks cyclemanager.CycleCallbackGroup, -) hnsw.MakeCommitLogger { - makeCL := hnsw.MakeNoopCommitLogger - if !config.DisablePersistence { - makeCL = func() (hnsw.CommitLogger, error) { - return hnsw.NewCommitLogger(config.RootPath, config.ID, config.Logger, maintenanceCallbacks) - } - } - return makeCL -} - -// Add extends the index with the specified GeoCoordinates. It is thread-safe -// and can be called concurrently. -func (i *Index) Add(id uint64, coordinates *models.GeoCoordinates) error { - v, err := geoCoordiantesToVector(coordinates) - if err != nil { - return errors.Wrap(err, "invalid arguments") - } - - return i.vectorIndex.Add(id, v) -} - -// WithinGeoRange searches the index by the specified range. It is thread-safe -// and can be called concurrently. -func (i *Index) WithinRange(ctx context.Context, - geoRange filters.GeoRange, -) ([]uint64, error) { - if geoRange.GeoCoordinates == nil { - return nil, fmt.Errorf("invalid arguments: GeoCoordinates in range must be set") - } - - query, err := geoCoordiantesToVector(geoRange.GeoCoordinates) - if err != nil { - return nil, errors.Wrap(err, "invalid arguments") - } - - return i.vectorIndex.KnnSearchByVectorMaxDist(query, geoRange.Distance, 800, nil) -} - -func (i *Index) Delete(id uint64) error { - return i.vectorIndex.Delete(id) -} diff --git a/adapters/repos/db/vector/geo/geo_test.go b/adapters/repos/db/vector/geo/geo_test.go deleted file mode 100644 index 6c226da70af1f602a1a44d144f43d4ce44fce91d..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/geo/geo_test.go +++ /dev/null @@ -1,117 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package geo - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/cyclemanager" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" -) - -func TestGeoJourney(t *testing.T) { - elements := []models.GeoCoordinates{ - { // coordinates of munich - Latitude: ptFloat32(48.13743), - Longitude: ptFloat32(11.57549), - }, - { // coordinates of stuttgart - Latitude: ptFloat32(48.78232), - Longitude: ptFloat32(9.17702), - }, - } - - getCoordinates := func(ctx context.Context, id uint64) (*models.GeoCoordinates, error) { - return &elements[id], nil - } - - geoIndex, err := NewIndex(Config{ - ID: "unit-test", - CoordinatesForID: getCoordinates, - DisablePersistence: true, - RootPath: "doesnt-matter-persistence-is-off", - }, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop()) - require.Nil(t, err) - - t.Run("importing all", func(t *testing.T) { - for id, coordinates := range elements { - err := geoIndex.Add(uint64(id), &coordinates) - require.Nil(t, err) - } - }) - - t.Run("importing an invalid object", func(t *testing.T) { - err := geoIndex.Add(9000, &models.GeoCoordinates{}) - assert.Equal(t, "invalid arguments: latitude must be set", err.Error()) - }) - - km := float32(1000) - t.Run("searching missing longitude", func(t *testing.T) { - _, err := geoIndex.WithinRange(context.Background(), filters.GeoRange{ - GeoCoordinates: &models.GeoCoordinates{ - Latitude: ptFloat32(48.13743), - }, - Distance: 300 * km, - }) - assert.Equal(t, "invalid arguments: longitude must be set", err.Error()) - }) - - t.Run("searching missing latitude", func(t *testing.T) { - _, err := geoIndex.WithinRange(context.Background(), filters.GeoRange{ - GeoCoordinates: &models.GeoCoordinates{ - Longitude: ptFloat32(11.57549), - }, - Distance: 300 * km, - }) - assert.Equal(t, "invalid arguments: latitude must be set", err.Error()) - }) - - t.Run("searching within 500km of munich", func(t *testing.T) { - // should return both cities, with munich first and stuttgart second - results, err := geoIndex.WithinRange(context.Background(), filters.GeoRange{ - GeoCoordinates: &models.GeoCoordinates{ - Latitude: ptFloat32(48.13743), - Longitude: ptFloat32(11.57549), - }, - Distance: 500 * km, - }) - require.Nil(t, err) - - expectedResults := []uint64{0, 1} - assert.Equal(t, expectedResults, results) - }) - - t.Run("searching within 10km of munich", func(t *testing.T) { - // should return both cities, with munich first and stuttgart second - results, err := geoIndex.WithinRange(context.Background(), filters.GeoRange{ - GeoCoordinates: &models.GeoCoordinates{ - Latitude: ptFloat32(48.13743), - Longitude: ptFloat32(11.57549), - }, - Distance: 10 * km, - }) - require.Nil(t, err) - - expectedResults := []uint64{0} - assert.Equal(t, expectedResults, results) - }) -} - -func ptFloat32(in float32) *float32 { - return &in -} diff --git a/adapters/repos/db/vector/hnsw/.gitignore b/adapters/repos/db/vector/hnsw/.gitignore deleted file mode 100644 index 62087ffd4ca8343753dc851234b35774b14b5f9a..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -recall_vectors.json -recall_queries.json -recall_truths.json -datasets/ann-benchmarks/ -datasets/big-ann-benchmarks/ \ No newline at end of file diff --git a/adapters/repos/db/vector/hnsw/backup.go b/adapters/repos/db/vector/hnsw/backup.go deleted file mode 100644 index 1572cfe21bd6556c309234ff1e6eba17f9059ae2..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/backup.go +++ /dev/null @@ -1,90 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "context" - "fmt" - "io/fs" - "os" - "path/filepath" -) - -// SwitchCommitLogs makes sure that the previously writeable commitlog is -// switched to a new one, thus making the existing file read-only. -func (h *hnsw) SwitchCommitLogs(ctx context.Context) error { - if err := h.commitLog.SwitchCommitLogs(true); err != nil { - return fmt.Errorf("switch commitlogs: %w", err) - } - - return nil -} - -// ListFiles lists all files that are part of the part of the HNSW -// except the last commit-log which is writable. This operation is typically -// called immediately after calling SwitchCommitlogs which means that the -// latest (writeable) log file is typically empty. -// ListFiles errors if maintenance is not paused, as a stable state -// cannot be guaranteed with maintenance going on in the background. -func (h *hnsw) ListFiles(ctx context.Context, basePath string) ([]string, error) { - var ( - logRoot = filepath.Join(h.commitLog.RootPath(), fmt.Sprintf("%s.hnsw.commitlog.d", h.commitLog.ID())) - found = make(map[string]struct{}) - files []string - ) - - err := filepath.WalkDir(logRoot, func(pth string, d fs.DirEntry, err error) error { - if d.IsDir() { - return nil - } - - st, statErr := os.Stat(pth) - if statErr != nil { - return statErr - } - - // only list non-empty files - if st.Size() > 0 { - rel, relErr := filepath.Rel(basePath, pth) - if relErr != nil { - return relErr - } - found[rel] = struct{}{} - } - - return nil - }) - if err != nil { - return nil, fmt.Errorf("failed to list files for hnsw commitlog: %w", err) - } - - curr, _, err := getCurrentCommitLogFileName(logRoot) - if err != nil { - return nil, fmt.Errorf("current commitlog file name: %w", err) - } - - // remove active log from list, as - // it is not part of the backup - path, err := filepath.Rel(basePath, filepath.Join(logRoot, curr)) - if err != nil { - return nil, fmt.Errorf("delete active log: %w", err) - } - delete(found, path) - - files, i := make([]string, len(found)), 0 - for file := range found { - files[i] = file - i++ - } - - return files, nil -} diff --git a/adapters/repos/db/vector/hnsw/backup_integration_test.go b/adapters/repos/db/vector/hnsw/backup_integration_test.go deleted file mode 100644 index 389757ba8d11a5a653f7d9c47d1d2f3e2182bc81..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/backup_integration_test.go +++ /dev/null @@ -1,142 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package hnsw - -import ( - "context" - "fmt" - "os" - "path" - "testing" - "time" - - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" - "github.com/weaviate/weaviate/entities/cyclemanager" - enthnsw "github.com/weaviate/weaviate/entities/vectorindex/hnsw" -) - -func TestBackup_Integration(t *testing.T) { - ctx := context.Background() - logger, _ := test.NewNullLogger() - - dirName := t.TempDir() - indexID := "backup-integration-test" - - parentCommitLoggerCallbacks := cyclemanager.NewCallbackGroup("parentCommitLogger", logger, 1) - parentCommitLoggerCycle := cyclemanager.NewManager( - cyclemanager.HnswCommitLoggerCycleTicker(), - parentCommitLoggerCallbacks.CycleCallback) - parentCommitLoggerCycle.Start() - defer parentCommitLoggerCycle.StopAndWait(ctx) - commitLoggerCallbacks := cyclemanager.NewCallbackGroup("childCommitLogger", logger, 1) - commitLoggerCallbacksCtrl := parentCommitLoggerCallbacks.Register("commitLogger", commitLoggerCallbacks.CycleCallback) - - parentTombstoneCleanupCallbacks := cyclemanager.NewCallbackGroup("parentTombstoneCleanup", logger, 1) - parentTombstoneCleanupCycle := cyclemanager.NewManager( - cyclemanager.NewFixedTicker(enthnsw.DefaultCleanupIntervalSeconds*time.Second), - parentTombstoneCleanupCallbacks.CycleCallback) - parentTombstoneCleanupCycle.Start() - defer parentTombstoneCleanupCycle.StopAndWait(ctx) - tombstoneCleanupCallbacks := cyclemanager.NewCallbackGroup("childTombstoneCleanup", logger, 1) - tombstoneCleanupCallbacksCtrl := parentTombstoneCleanupCallbacks.Register("tombstoneCleanup", tombstoneCleanupCallbacks.CycleCallback) - - combinedCtrl := cyclemanager.NewCombinedCallbackCtrl(2, commitLoggerCallbacksCtrl, tombstoneCleanupCallbacksCtrl) - - idx, err := New(Config{ - RootPath: dirName, - ID: indexID, - Logger: logger, - DistanceProvider: distancer.NewCosineDistanceProvider(), - VectorForIDThunk: testVectorForID, - MakeCommitLoggerThunk: func() (CommitLogger, error) { - return NewCommitLogger(dirName, indexID, logger, commitLoggerCallbacks) - }, - }, enthnsw.NewDefaultUserConfig(), tombstoneCleanupCallbacks, cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), nil) - require.Nil(t, err) - idx.PostStartup() - - t.Run("insert vector into index", func(t *testing.T) { - for i := 0; i < 10; i++ { - inc := float32(i) - err := idx.Add(uint64(i), []float32{inc, inc + 1, inc + 2}) - require.Nil(t, err) - } - }) - - // let the index age for a second so that - // the commitlogger filenames, which are - // based on current timestamp, can differ - time.Sleep(time.Second) - - t.Run("pause maintenance", func(t *testing.T) { - err = combinedCtrl.Deactivate(ctx) - require.Nil(t, err) - }) - - t.Run("switch commit logs", func(t *testing.T) { - err = idx.SwitchCommitLogs(ctx) - require.Nil(t, err) - }) - - t.Run("list files", func(t *testing.T) { - files, err := idx.ListFiles(ctx, dirName) - require.Nil(t, err) - - // by this point there should be two files in the commitlog directory. - // one is the active log file, and the other is the previous active - // log which was in use prior to `SwitchCommitLogs`. additionally, - // maintenance has been paused, so we shouldn't see any .condensed - // files either. - // - // because `ListFiles` is used within the context of backups, - // it excludes any currently active log files, which are not part - // of the backup. in this case, the only other file is the prev - // commitlog, so we should only have 1 result here. - assert.Len(t, files, 1) - - t.Run("verify commitlog dir contents", func(t *testing.T) { - // checking to ensure that indeed there are only 2 files in the - // commit log directory, and that one of them is the one result - // from `ListFiles`, and that the other is not a .condensed file - ls, err := os.ReadDir(path.Join(dirName, fmt.Sprintf("%s.hnsw.commitlog.d", indexID))) - require.Nil(t, err) - assert.Len(t, ls, 2) - - var prevLogFound bool - for _, info := range ls { - if path.Base(files[0]) == info.Name() { - prevLogFound = true - } - - assert.Empty(t, path.Ext(info.Name())) - } - assert.True(t, prevLogFound, "previous commitlog not found in commitlog root dir") - }) - }) - - t.Run("resume maintenance", func(t *testing.T) { - err = combinedCtrl.Activate() - require.Nil(t, err) - }) - - err = idx.Shutdown(ctx) - require.Nil(t, err) - - err = combinedCtrl.Unregister(ctx) - require.Nil(t, err) -} diff --git a/adapters/repos/db/vector/hnsw/backup_test.go b/adapters/repos/db/vector/hnsw/backup_test.go deleted file mode 100644 index cb51cb6705132f304fe81f4fd5e21c45ebd18b50..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/backup_test.go +++ /dev/null @@ -1,102 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "context" - "fmt" - "os" - "path" - "regexp" - "testing" - "time" - - "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" - "github.com/weaviate/weaviate/entities/cyclemanager" - enthnsw "github.com/weaviate/weaviate/entities/vectorindex/hnsw" -) - -func TestBackup_SwitchCommitLogs(t *testing.T) { - ctx := context.Background() - - dirName := t.TempDir() - indexID := "backup-switch-commitlogs-test" - - idx, err := New(Config{ - RootPath: dirName, - ID: indexID, - Logger: logrus.New(), - DistanceProvider: distancer.NewCosineDistanceProvider(), - VectorForIDThunk: testVectorForID, - MakeCommitLoggerThunk: func() (CommitLogger, error) { - return NewCommitLogger(dirName, indexID, logrus.New(), cyclemanager.NewCallbackGroupNoop()) - }, - }, enthnsw.NewDefaultUserConfig(), cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), nil) - require.Nil(t, err) - idx.PostStartup() - - ctx, cancel := context.WithTimeout(ctx, time.Second) - defer cancel() - - err = idx.SwitchCommitLogs(ctx) - assert.Nil(t, err) - - err = idx.Shutdown(ctx) - require.Nil(t, err) -} - -func TestBackup_ListFiles(t *testing.T) { - ctx := context.Background() - - dirName := t.TempDir() - indexID := "backup-list-files-test" - - idx, err := New(Config{ - RootPath: dirName, - ID: indexID, - Logger: logrus.New(), - DistanceProvider: distancer.NewCosineDistanceProvider(), - VectorForIDThunk: testVectorForID, - MakeCommitLoggerThunk: func() (CommitLogger, error) { - return NewCommitLogger(dirName, indexID, logrus.New(), cyclemanager.NewCallbackGroupNoop()) - }, - }, enthnsw.NewDefaultUserConfig(), cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), nil) - require.Nil(t, err) - idx.PostStartup() - - t.Run("assert expected index contents", func(t *testing.T) { - files, err := idx.ListFiles(ctx, dirName) - assert.Nil(t, err) - - // should return empty, because the only file which - // exists in the commitlog root is the current active - // log file. - assert.Len(t, files, 0) - - // checking to ensure that the commitlog root does - // contain a file. this is the one that was ignored - // in the check above. - ls, err := os.ReadDir(path.Join(dirName, fmt.Sprintf("%s.hnsw.commitlog.d", indexID))) - require.Nil(t, err) - require.Len(t, ls, 1) - // filename should just be a 10 digit int - matched, err := regexp.MatchString("[0-9]{10}", ls[0].Name()) - assert.Nil(t, err) - assert.True(t, matched, "regex does not match") - }) - - err = idx.Shutdown(ctx) - require.Nil(t, err) -} diff --git a/adapters/repos/db/vector/hnsw/benchmark_test.go b/adapters/repos/db/vector/hnsw/benchmark_test.go deleted file mode 100644 index 216fc8be95098d271281c4a7e631cf873345fa2b..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/benchmark_test.go +++ /dev/null @@ -1,345 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "flag" - "io" - "net/http" - "net/url" - "os" - "path/filepath" - "sort" - "strconv" - "strings" - "testing" - "time" - - "github.com/pkg/errors" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/adapters/repos/db/vector/compressionhelpers" - "gopkg.in/yaml.v2" -) - -var download = flag.Bool("download", false, "download datasets if not found locally") - -var datasets = map[string]string{ - "random-xs": "datasets/big-ann-benchmarks/random10000/data_10000_20", - "random-xs-clustered": "datasets/big-ann-benchmarks/random-clustered10000/clu-random.fbin.crop_nb_10000", - "msturing-1M": "datasets/big-ann-benchmarks/MSTuringANNS/base1b.fbin.crop_nb_1000000", - "msturing-10M": "datasets/big-ann-benchmarks/MSTuringANNS/base1b.fbin.crop_nb_10000000", - "msspacev-1M": "datasets/big-ann-benchmarks/MSSPACEV1B/spacev1b_base.i8bin.crop_nb_1000000", - "msspacev-10M": "datasets/big-ann-benchmarks/MSSPACEV1B/spacev1b_base.i8bin.crop_nb_10000000", - "msturing-10M-clustered": "datasets/big-ann-benchmarks/MSTuring-10M-clustered/msturing-10M-clustered.fbin", -} - -var queries = map[string]string{ - "random-xs": "datasets/big-ann-benchmarks/random10000/queries_1000_20", - "random-xs-clustered": "datasets/big-ann-benchmarks/random-clustered10000/queries_1000_20.fbin", - "msturing-1M": "datasets/big-ann-benchmarks/MSTuringANNS/query100K.fbin", - "msturing-10M": "datasets/big-ann-benchmarks/MSTuringANNS/query100K.fbin", - "msspacev-1M": "datasets/big-ann-benchmarks/MSSPACEV1B/query.i8bin", - "msspacev-10M": "datasets/big-ann-benchmarks/MSSPACEV1B/query.i8bin", - "msturing-10M-clustered": "datasets/big-ann-benchmarks/MSTuring-10M-clustered/testQuery10K.fbin", -} - -func BenchmarkHnswNeurips23(b *testing.B) { - runbooks := []string{ - "datasets/neurips23/simple_runbook.yaml", - "datasets/neurips23/clustered_runbook.yaml", - } - - type datasetPoints struct { - dataset string - points int - } - - readDatasets := make(map[datasetPoints][][]float32) - - for _, runbookFile := range runbooks { - b.Run(runbookFile, func(b *testing.B) { - runbook := readRunbook(b, runbookFile) - - for _, step := range runbook.Steps { - b.Run(step.Dataset, func(b *testing.B) { - // Read the dataset if we haven't already - vectors, ok := readDatasets[datasetPoints{step.Dataset, step.MaxPts}] - if !ok { - file, ok := datasets[step.Dataset] - if !ok { - b.Skipf("Neurips23 dataset %s not found", step.Dataset) - } - - if _, err := os.Stat(file); err != nil { - if !*download { - b.Skipf(`Neurips23 dataset %s not found. -Run test with -download to automatically download the dataset. -Ex: go test -v -benchmem -bench ^BenchmarkHnswNeurips23$ -download`, step.Dataset) - } - downloadDataset(b, step.Dataset) - } - - readDatasets[datasetPoints{step.Dataset, step.MaxPts}] = readBigAnnDataset(b, file, step.MaxPts) - vectors = readDatasets[datasetPoints{step.Dataset, step.MaxPts}] - } - - var queryVectors [][]float32 - - b.ResetTimer() - - for i := 0; i < b.N; i++ { - index := createEmptyHnswIndexForTests(b, idVectorSize(len(vectors[0]))) - - for _, op := range step.Operations { - switch op.Operation { - case "insert": - compressionhelpers.Concurrently(uint64(op.End-op.Start), func(i uint64) { - err := index.Add(uint64(op.Start+int(i)), vectors[op.Start+int(i)]) - require.NoError(b, err) - }) - case "delete": - compressionhelpers.Concurrently(uint64(op.End-op.Start), func(i uint64) { - err := index.Delete(uint64(op.Start + int(i))) - require.NoError(b, err) - }) - case "search": - if len(queryVectors) == 0 { - file, ok := queries[step.Dataset] - if !ok { - b.Errorf("query file: not found for %s dataset", step.Dataset) - } - - queryVectors = readBigAnnDataset(b, file, 0) - } - - compressionhelpers.Concurrently(uint64(len(queryVectors)), func(i uint64) { - _, _, err := index.SearchByVector(queryVectors[i], 0, nil) - require.NoError(b, err) - }) - default: - b.Errorf("Unknown operation %s", op.Operation) - } - } - } - }) - } - }) - } -} - -func downloadDataset(t testing.TB, name string) { - t.Helper() - - ds, ok := datasets[name] - if !ok { - t.Fatalf("Dataset %s not found", name) - } - - qs, ok := queries[name] - if !ok { - t.Fatalf("Query file not found for %s dataset", name) - } - - for _, f := range []string{ds, qs} { - downloadDatasetFile(t, f) - } -} - -func downloadDatasetFile(t testing.TB, file string) { - t.Helper() - - if _, err := os.Stat(file); err == nil { - return - } - - err := os.MkdirAll(filepath.Dir(file), 0o755) - require.NoError(t, err) - - path := strings.TrimPrefix(file, "datasets/") - - u, err := url.JoinPath("https://storage.googleapis.com/ann-datasets/", path) - require.NoError(t, err) - - t.Logf("Downloading dataset from %s", u) - - client := http.Client{ - Timeout: 60 * time.Second, - } - - resp, err := client.Get(u) - require.NoError(t, err) - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - t.Fatalf("Could not download dataset. Status code: %d", resp.StatusCode) - } - - f, err := os.Create(file) - require.NoError(t, err) - defer f.Close() - - _, err = io.Copy(f, resp.Body) - require.NoError(t, err) - - t.Logf("Downloaded dataset %s", file) -} - -func readBigAnnDataset(t testing.TB, file string, maxObjects int) [][]float32 { - t.Helper() - - var vectors [][]float32 - - f, err := os.Open(file) - if err != nil { - panic(errors.Wrap(err, "Could not open SIFT file")) - } - defer f.Close() - - fi, err := f.Stat() - if err != nil { - panic(errors.Wrap(err, "Could not get SIFT file properties")) - } - fileSize := fi.Size() - - b := make([]byte, 4) - - // The data is a binary file containing either floating point vectors or int8 vectors - // It starts with 8 bytes of header data - // The first 4 bytes are the number of vectors in the file - // The second 4 bytes are the dimensionality of the vectors in the file - // If the file is in fbin format, the vector data needs to be converted from bytes to float. - // If the file is in i8bin format, the vector data needs to be converted from bytes to int8 then to float. - - // The first 4 bytes are the number of vectors in the file - _, err = f.Read(b) - require.NoError(t, err) - n := int32FromBytes(b) - - // The second 4 bytes are the dimensionality of the vectors in the file - _, err = f.Read(b) - require.NoError(t, err) - d := int32FromBytes(b) - - var bytesPerVector int - switch { - case strings.Contains(file, "i8bin"): - bytesPerVector = 1 - case strings.Contains(file, "fbin"): - fallthrough - default: - bytesPerVector = 4 - } - - require.Equal(t, 8+n*d*bytesPerVector, int(fileSize)) - - vectorBytes := make([]byte, d*bytesPerVector) - if maxObjects > 0 && maxObjects < n { - n = maxObjects - } - - for i := 0; i < n; i++ { - _, err = f.Read(vectorBytes) - if err == io.EOF { - break - } - require.NoError(t, err) - - vectorFloat := make([]float32, 0, d) - for j := 0; j < d; j++ { - start := j * bytesPerVector - var f float32 - if bytesPerVector == 1 { - f = float32(vectorBytes[start]) - } else { - f = float32FromBytes(vectorBytes[start : start+bytesPerVector]) - } - - vectorFloat = append(vectorFloat, f) - } - - vectors = append(vectors, vectorFloat) - } - - if maxObjects > 0 { - require.Equal(t, maxObjects, len(vectors)) - } - - return vectors -} - -type runbook struct { - Steps []runbookStep -} -type runbookStep struct { - Dataset string - MaxPts int - Operations []runbookOperation -} - -type runbookOperation struct { - Operation string - Start int - End int -} - -func readRunbook(t testing.TB, file string) *runbook { - f, err := os.Open(file) - require.NoError(t, err, "Could not open runbook file") - defer f.Close() - - d := yaml.NewDecoder(f) - - var runbook runbook - - var m map[string]map[string]any - err = d.Decode(&m) - require.NoError(t, err) - - var datasets []string - for datasetName := range m { - datasets = append(datasets, datasetName) - } - - sort.Strings(datasets) - - for _, datasetName := range datasets { - stepInfo := m[datasetName] - var step runbookStep - - step.Dataset = datasetName - step.MaxPts = stepInfo["max_pts"].(int) - i := 1 - for { - s := strconv.Itoa(i) - if _, ok := stepInfo[s]; !ok { - break - } - - opInfo := stepInfo[s].(map[any]any) - - var op runbookOperation - op.Operation = opInfo["operation"].(string) - if op.Operation == "insert" || op.Operation == "delete" { - op.Start = opInfo["start"].(int) - op.End = opInfo["end"].(int) - } - - step.Operations = append(step.Operations, op) - - i++ - } - - runbook.Steps = append(runbook.Steps, step) - } - - return &runbook -} diff --git a/adapters/repos/db/vector/hnsw/bufiowriter.go b/adapters/repos/db/vector/hnsw/bufiowriter.go deleted file mode 100644 index 1b841a03f3252defb35c6fff93319a10a7575d87..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/bufiowriter.go +++ /dev/null @@ -1,186 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "io" - "os" - "unicode/utf8" -) - -const ( - defaultBufSize = 4096 -) - -// bufWriter implements buffering for an *os.File object. -// If an error occurs writing to a bufWriter, no more data will be -// accepted and all subsequent writes, and Flush, will return the error. -// After all data has been written, the client should call the -// Flush method to guarantee all data has been forwarded to -// the underlying *os.File. -type bufWriter struct { - err error - buf []byte - n int - wr *os.File -} - -// NewWriterSize returns a new Writer whose buffer has at least the specified -// size. If the argument *os.File is already a Writer with large enough -// size, it returns the underlying Writer. -func NewWriterSize(w *os.File, size int) *bufWriter { - if size <= 0 { - size = defaultBufSize - } - return &bufWriter{ - buf: make([]byte, size), - wr: w, - } -} - -// NewWriter returns a new Writer whose buffer has the default size. -func NewWriter(w *os.File) *bufWriter { - return NewWriterSize(w, defaultBufSize) -} - -// Size returns the size of the underlying buffer in bytes. -func (b *bufWriter) Size() int { return len(b.buf) } - -// Reset discards any unflushed buffered data, clears any error, and -// resets b to write its output to w. -func (b *bufWriter) Reset(w *os.File) { - b.err = nil - b.n = 0 - b.wr = w -} - -// Flush writes any buffered data to the underlying *os.File. -func (b *bufWriter) Flush() error { - if b.err != nil { - return b.err - } - if b.n == 0 { - return nil - } - n, err := b.wr.Write(b.buf[0:b.n]) - if n < b.n && err == nil { - err = io.ErrShortWrite - } - if err != nil { - if n > 0 && n < b.n { - copy(b.buf[0:b.n-n], b.buf[n:b.n]) - } - b.n -= n - b.err = err - return err - } - b.n = 0 - return nil -} - -// Available returns how many bytes are unused in the buffer. -func (b *bufWriter) Available() int { return len(b.buf) - b.n } - -// Buffered returns the number of bytes that have been written into the current buffer. -func (b *bufWriter) Buffered() int { return b.n } - -// Write writes the contents of p into the buffer. -// It returns the number of bytes written. -// If nn < len(p), it also returns an error explaining -// why the write is short. -func (b *bufWriter) Write(p []byte) (nn int, err error) { - for len(p) > b.Available() && b.err == nil { - var n int - if b.Buffered() == 0 { - // Large write, empty buffer. - // Write directly from p to avoid copy. - n, b.err = b.wr.Write(p) - } else { - n = copy(b.buf[b.n:], p) - b.n += n - b.Flush() - } - nn += n - p = p[n:] - } - if b.err != nil { - return nn, b.err - } - n := copy(b.buf[b.n:], p) - b.n += n - nn += n - return nn, nil -} - -// WriteByte writes a single byte. -func (b *bufWriter) WriteByte(c byte) error { - if b.err != nil { - return b.err - } - if b.Available() <= 0 && b.Flush() != nil { - return b.err - } - b.buf[b.n] = c - b.n++ - return nil -} - -// WriteRune writes a single Unicode code point, returning -// the number of bytes written and any error. -func (b *bufWriter) WriteRune(r rune) (size int, err error) { - if r < utf8.RuneSelf { - err = b.WriteByte(byte(r)) - if err != nil { - return 0, err - } - return 1, nil - } - if b.err != nil { - return 0, b.err - } - n := b.Available() - if n < utf8.UTFMax { - if b.Flush(); b.err != nil { - return 0, b.err - } - n = b.Available() - if n < utf8.UTFMax { - // Can only happen if buffer is silly small. - return b.WriteString(string(r)) - } - } - size = utf8.EncodeRune(b.buf[b.n:], r) - b.n += size - return size, nil -} - -// WriteString writes a string. -// It returns the number of bytes written. -// If the count is less than len(s), it also returns an error explaining -// why the write is short. -func (b *bufWriter) WriteString(s string) (int, error) { - nn := 0 - for len(s) > b.Available() && b.err == nil { - n := copy(b.buf[b.n:], s) - b.n += n - nn += n - s = s[n:] - b.Flush() - } - if b.err != nil { - return nn, b.err - } - n := copy(b.buf[b.n:], s) - b.n += n - nn += n - return nn, nil -} diff --git a/adapters/repos/db/vector/hnsw/commit_log_combiner.go b/adapters/repos/db/vector/hnsw/commit_log_combiner.go deleted file mode 100644 index 4665bfaf84f4b6a55cfb1cb7c91b178e90a80a33..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/commit_log_combiner.go +++ /dev/null @@ -1,203 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "io" - "os" - "strings" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" -) - -type CommitLogCombiner struct { - rootPath string - id string - threshold int64 - logger logrus.FieldLogger -} - -func NewCommitLogCombiner(rootPath, id string, threshold int64, - logger logrus.FieldLogger, -) *CommitLogCombiner { - return &CommitLogCombiner{ - rootPath: rootPath, - id: id, - threshold: threshold, - logger: logger, - } -} - -func (c *CommitLogCombiner) Do() (bool, error) { - executed := false - for { - // fileNames will already be in order - fileNames, err := getCommitFileNames(c.rootPath, c.id) - if err != nil { - return executed, errors.Wrap(err, "obtain files names") - } - - ok, err := c.combineFirstMatch(fileNames) - if err != nil { - return executed, err - } - - if ok { - executed = true - continue - } - - break - } - return executed, nil -} - -func (c *CommitLogCombiner) combineFirstMatch(fileNames []string) (bool, error) { - for i, fileName := range fileNames { - if !strings.HasSuffix(fileName, ".condensed") { - // not an already condensed file, so no candidate for combining - continue - } - - if i == len(fileNames)-1 { - // this is the last file, so there is nothing to combine it with - return false, nil - } - - if !strings.HasSuffix(fileNames[i+1], ".condensed") { - // the next file is not a condensed file, so this file is not candidate - // for merging with the next - continue - } - - currentStat, err := os.Stat(fileName) - if err != nil { - return false, errors.Wrapf(err, "stat file %q", fileName) - } - - if currentStat.Size() > c.threshold { - // already too big, can't combine further - continue - } - - nextStat, err := os.Stat(fileNames[i+1]) - if err != nil { - return false, errors.Wrapf(err, "stat file %q", fileNames[i+1]) - } - - if currentStat.Size()+nextStat.Size() > c.threshold { - // combining those two would exceed threshold - continue - } - - if err := c.combine(fileName, fileNames[i+1]); err != nil { - return false, errors.Wrapf(err, "combine %q and %q", fileName, fileNames[i+1]) - } - - return true, nil - } - - return false, nil -} - -func (c *CommitLogCombiner) combine(first, second string) error { - // all names are based on the first file, so that once file1 + file2 are - // combined it is as if file2 had never existed and file 1 was just always - // big enough to hold the contents of both - - // clearly indicate that the file is "in progress", in case we crash while - // combining and the after restart there are multiple alternatives - tmpName := strings.TrimSuffix(first, ".condensed") + (".combined.tmp") - - // finalName will look like an uncondensed original commit log, so the - // condensor will pick it up without even knowing that it's a combined file - finalName := strings.TrimSuffix(first, ".condensed") - - if err := c.mergeFiles(tmpName, first, second); err != nil { - return errors.Wrap(err, "merge files") - } - - if err := c.renameAndCleanUp(tmpName, finalName, first, second); err != nil { - return errors.Wrap(err, "rename and clean up files") - } - - c.logger.WithFields(logrus.Fields{ - "action": "hnsw_commit_logger_combine_condensed_logs", - "id": c.id, - "input_first": first, - "input_second": second, - "output": finalName, - }).Info("successfully combined previously condensed commit log files") - - return nil -} - -func (c *CommitLogCombiner) mergeFiles(outName, first, second string) error { - out, err := os.Create(outName) - if err != nil { - return errors.Wrapf(err, "open target file %q", outName) - } - - source1, err := os.Open(first) - if err != nil { - return errors.Wrapf(err, "open first source file %q", first) - } - defer source1.Close() - - source2, err := os.Open(second) - if err != nil { - return errors.Wrapf(err, "open second source file %q", second) - } - defer source2.Close() - - _, err = io.Copy(out, source1) - if err != nil { - return errors.Wrapf(err, "copy first source (%q) into target (%q)", first, - outName) - } - - _, err = io.Copy(out, source2) - if err != nil { - return errors.Wrapf(err, "copy second source (%q) into target (%q)", second, - outName) - } - - err = out.Close() - if err != nil { - return errors.Wrapf(err, "close target file %q", outName) - } - - return nil -} - -func (c *CommitLogCombiner) renameAndCleanUp(tmpName, finalName string, - toDeletes ...string, -) error { - // do the rename before the delete, because if we crash in between we end up - // with duplicate files both with and without the ".condensed" suffix. The - // new (and complete) merged file will not carry the suffix whereas the - // sources will. This will look to the corrupted file fixer as if a - // condensing had gone wrong and will delete the the source - - if err := os.Rename(tmpName, finalName); err != nil { - return errors.Wrapf(err, "rename tmp (%q) to final (%q)", tmpName, finalName) - } - - for _, toDelete := range toDeletes { - if err := os.Remove(toDelete); err != nil { - return errors.Wrapf(err, "clean up %q", toDelete) - } - } - - return nil -} diff --git a/adapters/repos/db/vector/hnsw/commit_log_combiner_integration_test.go b/adapters/repos/db/vector/hnsw/commit_log_combiner_integration_test.go deleted file mode 100644 index 1f94223c391966d04a6c7ae7c1f2764600f9773f..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/commit_log_combiner_integration_test.go +++ /dev/null @@ -1,118 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "os" - "testing" - - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func Test_CommitlogCombiner(t *testing.T) { - // For the combiner the contents of a commit log file don't actually matter - // so we can put arbitrary data in the files. It will only make decisions - // about what should be appended, the actual condensing will be taken care of - // by the condensor - - rootPath := t.TempDir() - logger, _ := test.NewNullLogger() - - threshold := int64(1000) - id := "combiner_test" - // create commit logger directory - require.Nil(t, os.MkdirAll(commitLogDirectory(rootPath, id), 0o777)) - - name := func(fileName string) string { - return commitLogFileName(rootPath, id, fileName) - } - - t.Run("create several condensed files below the threshold", func(t *testing.T) { - // 4 files of 300 bytes each, with 1000 byte threshold. This lets us verify - // that one and two will be combined, so will three and four. - require.Nil(t, createDummyFile(name("1000.condensed"), []byte("file1\n"), 300)) - require.Nil(t, createDummyFile(name("1001.condensed"), []byte("file2\n"), 300)) - require.Nil(t, createDummyFile(name("1002.condensed"), []byte("file3\n"), 300)) - require.Nil(t, createDummyFile(name("1003.condensed"), []byte("file4\n"), 300)) - require.Nil(t, createDummyFile(name("1004"), []byte("current\n"), 50)) - }) - - t.Run("run combiner", func(t *testing.T) { - _, err := NewCommitLogCombiner(rootPath, id, threshold, logger).Do() - require.Nil(t, err) - }) - - t.Run("we are now left with combined files", func(t *testing.T) { - dir, err := os.Open(commitLogDirectory(rootPath, id)) - require.Nil(t, err) - - fileNames, err := dir.Readdirnames(0) - require.Nil(t, err) - require.Len(t, fileNames, 3) - require.ElementsMatch(t, []string{"1000", "1002", "1004"}, fileNames) - - t.Run("the first file is correctly combined", func(t *testing.T) { - contents, err := os.ReadFile(commitLogFileName(rootPath, id, "1000")) - require.Nil(t, err) - require.Len(t, contents, 600) - assert.Equal(t, contents[0:6], []byte("file1\n")) - assert.Equal(t, contents[300:306], []byte("file2\n")) - }) - - t.Run("the second file is correctly combined", func(t *testing.T) { - contents, err := os.ReadFile(commitLogFileName(rootPath, id, "1002")) - require.Nil(t, err) - require.Len(t, contents, 600) - assert.Equal(t, contents[0:6], []byte("file3\n")) - assert.Equal(t, contents[300:306], []byte("file4\n")) - }) - - t.Run("latest file is unchanged", func(t *testing.T) { - contents, err := os.ReadFile(commitLogFileName(rootPath, id, "1004")) - require.Nil(t, err) - require.Len(t, contents, 50) - assert.Equal(t, contents[0:8], []byte("current\n")) - assert.Equal(t, contents[42:], []byte("rrent\ncu")) - }) - }) -} - -func createDummyFile(fileName string, content []byte, size int) error { - f, err := os.Create(fileName) - if err != nil { - return err - } - - defer f.Close() - - written := 0 - for { - if size == written { - break - } - - if size-written < len(content) { - content = content[:(size - written)] - } - - n, err := f.Write([]byte(content)) - written += n - - if err != nil { - return err - } - } - - return nil -} diff --git a/adapters/repos/db/vector/hnsw/commit_logger.go b/adapters/repos/db/vector/hnsw/commit_logger.go deleted file mode 100644 index 87369a22a4d963636f9465ac7c6e5df15a71ab32..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/commit_logger.go +++ /dev/null @@ -1,557 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "context" - "fmt" - "os" - "path/filepath" - "sort" - "strconv" - "strings" - "sync" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/adapters/repos/db/vector/compressionhelpers" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/commitlog" - "github.com/weaviate/weaviate/entities/cyclemanager" - "github.com/weaviate/weaviate/entities/errorcompounder" -) - -const defaultCommitLogSize = 500 * 1024 * 1024 - -func commitLogFileName(rootPath, indexName, fileName string) string { - return fmt.Sprintf("%s/%s", commitLogDirectory(rootPath, indexName), fileName) -} - -func commitLogDirectory(rootPath, name string) string { - return fmt.Sprintf("%s/%s.hnsw.commitlog.d", rootPath, name) -} - -func NewCommitLogger(rootPath, name string, logger logrus.FieldLogger, - maintenanceCallbacks cyclemanager.CycleCallbackGroup, opts ...CommitlogOption, -) (*hnswCommitLogger, error) { - l := &hnswCommitLogger{ - rootPath: rootPath, - id: name, - condensor: NewMemoryCondensor(logger), - logger: logger, - - // both can be overwritten using functional options - maxSizeIndividual: defaultCommitLogSize / 5, - maxSizeCombining: defaultCommitLogSize, - } - - for _, o := range opts { - if err := o(l); err != nil { - return nil, err - } - } - - fd, err := getLatestCommitFileOrCreate(rootPath, name) - if err != nil { - return nil, err - } - - id := func(elems ...string) string { - elems = append([]string{"commit_logger"}, elems...) - elems = append(elems, l.id) - return strings.Join(elems, "/") - } - l.commitLogger = commitlog.NewLoggerWithFile(fd) - l.switchLogsCallbackCtrl = maintenanceCallbacks.Register(id("switch_logs"), l.startSwitchLogs) - l.condenseLogsCallbackCtrl = maintenanceCallbacks.Register(id("condense_logs"), l.startCombineAndCondenseLogs) - - return l, nil -} - -func getLatestCommitFileOrCreate(rootPath, name string) (*os.File, error) { - dir := commitLogDirectory(rootPath, name) - err := os.MkdirAll(dir, os.ModePerm) - if err != nil { - return nil, errors.Wrap(err, "create commit logger directory") - } - - fileName, ok, err := getCurrentCommitLogFileName(dir) - if err != nil { - return nil, errors.Wrap(err, "find commit logger file in directory") - } - - if !ok { - // this is a new commit log, initialize with the current time stamp - fileName = fmt.Sprintf("%d", time.Now().Unix()) - } - - fd, err := os.OpenFile(commitLogFileName(rootPath, name, fileName), - os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0o666) - if err != nil { - return nil, errors.Wrap(err, "create commit log file") - } - - return fd, nil -} - -// getCommitFileNames in order, from old to new -func getCommitFileNames(rootPath, name string) ([]string, error) { - dir := commitLogDirectory(rootPath, name) - err := os.MkdirAll(dir, os.ModePerm) - if err != nil { - return nil, errors.Wrap(err, "create commit logger directory") - } - - files, err := os.ReadDir(dir) - if err != nil { - return nil, errors.Wrap(err, "browse commit logger directory") - } - - files = removeTmpScratchOrHiddenFiles(files) - files, err = removeTmpCombiningFiles(dir, files) - if err != nil { - return nil, errors.Wrap(err, "remove temporary files") - } - - if len(files) == 0 { - return nil, nil - } - - ec := &errorcompounder.ErrorCompounder{} - sort.Slice(files, func(a, b int) bool { - ts1, err := asTimeStamp(files[a].Name()) - if err != nil { - ec.Add(err) - } - - ts2, err := asTimeStamp(files[b].Name()) - if err != nil { - ec.Add(err) - } - return ts1 < ts2 - }) - if err := ec.ToError(); err != nil { - return nil, err - } - - out := make([]string, len(files)) - for i, file := range files { - out[i] = commitLogFileName(rootPath, name, file.Name()) - } - - return out, nil -} - -// getCurrentCommitLogFileName returns the fileName and true if a file was -// present. If no file was present, the second arg is false. -func getCurrentCommitLogFileName(dirPath string) (string, bool, error) { - files, err := os.ReadDir(dirPath) - if err != nil { - return "", false, errors.Wrap(err, "browse commit logger directory") - } - - if len(files) == 0 { - return "", false, nil - } - - files = removeTmpScratchOrHiddenFiles(files) - files, err = removeTmpCombiningFiles(dirPath, files) - if err != nil { - return "", false, errors.Wrap(err, "clean up tmp combining files") - } - - ec := &errorcompounder.ErrorCompounder{} - sort.Slice(files, func(a, b int) bool { - ts1, err := asTimeStamp(files[a].Name()) - if err != nil { - ec.Add(err) - } - - ts2, err := asTimeStamp(files[b].Name()) - if err != nil { - ec.Add(err) - } - return ts1 > ts2 - }) - if err := ec.ToError(); err != nil { - return "", false, err - } - - return files[0].Name(), true, nil -} - -func removeTmpScratchOrHiddenFiles(in []os.DirEntry) []os.DirEntry { - out := make([]os.DirEntry, len(in)) - i := 0 - for _, info := range in { - if strings.HasSuffix(info.Name(), ".scratch.tmp") { - continue - } - - if strings.HasPrefix(info.Name(), ".") { - continue - } - - out[i] = info - i++ - } - - return out[:i] -} - -func removeTmpCombiningFiles(dirPath string, - in []os.DirEntry, -) ([]os.DirEntry, error) { - out := make([]os.DirEntry, len(in)) - i := 0 - for _, info := range in { - if strings.HasSuffix(info.Name(), ".combined.tmp") { - // a temporary combining file was found which means that the combining - // process never completed, this file is thus considered corrupt (too - // short) and must be deleted. The original sources still exist (because - // the only get deleted after the .tmp file is removed), so it's safe to - // delete this without data loss. - - if err := os.Remove(filepath.Join(dirPath, info.Name())); err != nil { - return out, errors.Wrap(err, "remove tmp combining file") - } - continue - } - - out[i] = info - i++ - } - - return out[:i], nil -} - -func asTimeStamp(in string) (int64, error) { - return strconv.ParseInt(strings.TrimSuffix(in, ".condensed"), 10, 64) -} - -type condensor interface { - Do(filename string) error -} - -type hnswCommitLogger struct { - // protect against concurrent attempts to write in the underlying file or - // buffer - sync.Mutex - - rootPath string - id string - condensor condensor - logger logrus.FieldLogger - maxSizeIndividual int64 - maxSizeCombining int64 - commitLogger *commitlog.Logger - - switchLogsCallbackCtrl cyclemanager.CycleCallbackCtrl - condenseLogsCallbackCtrl cyclemanager.CycleCallbackCtrl -} - -type HnswCommitType uint8 // 256 options, plenty of room for future extensions - -const ( - AddNode HnswCommitType = iota - SetEntryPointMaxLevel - AddLinkAtLevel - ReplaceLinksAtLevel - AddTombstone - RemoveTombstone - ClearLinks - DeleteNode - ResetIndex - ClearLinksAtLevel // added in v1.8.0-rc.1, see https://github.com/weaviate/weaviate/issues/1701 - AddLinksAtLevel // added in v1.8.0-rc.1, see https://github.com/weaviate/weaviate/issues/1705 - AddPQ -) - -func (t HnswCommitType) String() string { - switch t { - case AddNode: - return "AddNode" - case SetEntryPointMaxLevel: - return "SetEntryPointWithMaxLayer" - case AddLinkAtLevel: - return "AddLinkAtLevel" - case AddLinksAtLevel: - return "AddLinksAtLevel" - case ReplaceLinksAtLevel: - return "ReplaceLinksAtLevel" - case AddTombstone: - return "AddTombstone" - case RemoveTombstone: - return "RemoveTombstone" - case ClearLinks: - return "ClearLinks" - case DeleteNode: - return "DeleteNode" - case ResetIndex: - return "ResetIndex" - case ClearLinksAtLevel: - return "ClearLinksAtLevel" - case AddPQ: - return "AddProductQuantizer" - } - return "unknown commit type" -} - -func (l *hnswCommitLogger) ID() string { - return l.id -} - -func (l *hnswCommitLogger) AddPQ(data compressionhelpers.PQData) error { - l.Lock() - defer l.Unlock() - - return l.commitLogger.AddPQ(data) -} - -// AddNode adds an empty node -func (l *hnswCommitLogger) AddNode(node *vertex) error { - l.Lock() - defer l.Unlock() - - return l.commitLogger.AddNode(node.id, node.level) -} - -func (l *hnswCommitLogger) SetEntryPointWithMaxLayer(id uint64, level int) error { - l.Lock() - defer l.Unlock() - - return l.commitLogger.SetEntryPointWithMaxLayer(id, level) -} - -func (l *hnswCommitLogger) ReplaceLinksAtLevel(nodeid uint64, level int, targets []uint64) error { - l.Lock() - defer l.Unlock() - - return l.commitLogger.ReplaceLinksAtLevel(nodeid, level, targets) -} - -func (l *hnswCommitLogger) AddLinkAtLevel(nodeid uint64, level int, - target uint64, -) error { - l.Lock() - defer l.Unlock() - - return l.commitLogger.AddLinkAtLevel(nodeid, level, target) -} - -func (l *hnswCommitLogger) AddTombstone(nodeid uint64) error { - l.Lock() - defer l.Unlock() - - return l.commitLogger.AddTombstone(nodeid) -} - -func (l *hnswCommitLogger) RemoveTombstone(nodeid uint64) error { - l.Lock() - defer l.Unlock() - - return l.commitLogger.RemoveTombstone(nodeid) -} - -func (l *hnswCommitLogger) ClearLinks(nodeid uint64) error { - l.Lock() - defer l.Unlock() - - return l.commitLogger.ClearLinks(nodeid) -} - -func (l *hnswCommitLogger) ClearLinksAtLevel(nodeid uint64, level uint16) error { - l.Lock() - defer l.Unlock() - - return l.commitLogger.ClearLinksAtLevel(nodeid, level) -} - -func (l *hnswCommitLogger) DeleteNode(nodeid uint64) error { - l.Lock() - defer l.Unlock() - - return l.commitLogger.DeleteNode(nodeid) -} - -func (l *hnswCommitLogger) Reset() error { - l.Lock() - defer l.Unlock() - - return l.commitLogger.Reset() -} - -// Shutdown waits for ongoing maintenance processes to stop, then cancels their -// scheduling. The caller can be sure that state on disk is immutable after -// calling Shutdown(). -func (l *hnswCommitLogger) Shutdown(ctx context.Context) error { - if err := l.switchLogsCallbackCtrl.Unregister(ctx); err != nil { - return errors.Wrap(err, "failed to unregister commitlog switch from maintenance cycle") - } - if err := l.condenseLogsCallbackCtrl.Unregister(ctx); err != nil { - return errors.Wrap(err, "failed to unregister commitlog condense from maintenance cycle") - } - return nil -} - -func (l *hnswCommitLogger) RootPath() string { - return l.rootPath -} - -func (l *hnswCommitLogger) startSwitchLogs(shouldAbort cyclemanager.ShouldAbortCallback) bool { - executed, err := l.switchCommitLogs(false) - if err != nil { - l.logger.WithError(err). - WithField("action", "hnsw_commit_log_maintenance"). - Error("hnsw commit log maintenance failed") - } - return executed -} - -func (l *hnswCommitLogger) startCombineAndCondenseLogs(shouldAbort cyclemanager.ShouldAbortCallback) bool { - executed1, err := l.combineLogs() - if err != nil { - l.logger.WithError(err). - WithField("action", "hnsw_commit_log_combining"). - Error("hnsw commit log maintenance (combining) failed") - } - - executed2, err := l.condenseOldLogs() - if err != nil { - l.logger.WithError(err). - WithField("action", "hnsw_commit_log_condensing"). - Error("hnsw commit log maintenance (condensing) failed") - } - return executed1 || executed2 -} - -func (l *hnswCommitLogger) SwitchCommitLogs(force bool) error { - _, err := l.switchCommitLogs(force) - return err -} - -func (l *hnswCommitLogger) switchCommitLogs(force bool) (bool, error) { - l.Lock() - defer l.Unlock() - - size, err := l.commitLogger.FileSize() - if err != nil { - return false, err - } - - if size <= l.maxSizeIndividual && !force { - return false, nil - } - - oldFileName, err := l.commitLogger.FileName() - if err != nil { - return false, err - } - - if err := l.commitLogger.Close(); err != nil { - return true, err - } - - // this is a new commit log, initialize with the current time stamp - fileName := fmt.Sprintf("%d", time.Now().Unix()) - - if force { - l.logger.WithField("action", "commit_log_file_switched"). - WithField("id", l.id). - WithField("old_file_name", oldFileName). - WithField("old_file_size", size). - WithField("new_file_name", fileName). - Debug("commit log switched forced") - } else { - l.logger.WithField("action", "commit_log_file_switched"). - WithField("id", l.id). - WithField("old_file_name", oldFileName). - WithField("old_file_size", size). - WithField("new_file_name", fileName). - Info("commit log size crossed threshold, switching to new file") - } - - fd, err := os.OpenFile(commitLogFileName(l.rootPath, l.id, fileName), - os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0o666) - if err != nil { - return true, errors.Wrap(err, "create commit log file") - } - - l.commitLogger = commitlog.NewLoggerWithFile(fd) - - return true, nil -} - -func (l *hnswCommitLogger) condenseOldLogs() (bool, error) { - files, err := getCommitFileNames(l.rootPath, l.id) - if err != nil { - return false, err - } - - if len(files) <= 1 { - // if there are no files there is nothing to do - // if there is only a single file, it must still be in use, we can't do - // anything yet - return false, nil - } - - // cut off last element, as that's never a candidate - candidates := files[:len(files)-1] - - for _, candidate := range candidates { - if strings.HasSuffix(candidate, ".condensed") { - // don't attempt to condense logs which are already condensed - continue - } - - return true, l.condensor.Do(candidate) - } - - return false, nil -} - -func (l *hnswCommitLogger) combineLogs() (bool, error) { - // maxSize is the desired final size, since we assume a lot of redundancy we - // can set the combining threshold higher than the final threshold under the - // assumption that the combined file will be considerably smaller than the - // sum of both input files - threshold := int64(float64(l.maxSizeCombining) * 1.75) - return NewCommitLogCombiner(l.rootPath, l.id, threshold, l.logger).Do() -} - -func (l *hnswCommitLogger) Drop(ctx context.Context) error { - if err := l.commitLogger.Close(); err != nil { - return errors.Wrap(err, "close hnsw commit logger prior to delete") - } - - // stop all goroutines - if err := l.Shutdown(ctx); err != nil { - return errors.Wrap(err, "drop commitlog") - } - - // remove commit log directory if exists - dir := commitLogDirectory(l.rootPath, l.id) - if _, err := os.Stat(dir); err == nil { - err := os.RemoveAll(dir) - if err != nil { - return errors.Wrap(err, "delete commit files directory") - } - } - return nil -} - -func (l *hnswCommitLogger) Flush() error { - l.Lock() - defer l.Unlock() - - return l.commitLogger.Flush() -} diff --git a/adapters/repos/db/vector/hnsw/commit_logger_buffered_links_logger.go b/adapters/repos/db/vector/hnsw/commit_logger_buffered_links_logger.go deleted file mode 100644 index ccf6e79a674f68a4448aa5365e0de2c1b70de071..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/commit_logger_buffered_links_logger.go +++ /dev/null @@ -1,66 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -// type bufferedLinksLogger struct { -// base *hnswCommitLogger -// buf *bytes.Buffer -// } - -// func (b *bufferedLinksLogger) ReplaceLinksAtLevel(nodeid uint64, -// level int, targets []uint64) error { -// b.base.writeCommitType(b.buf, ReplaceLinksAtLevel) -// b.base.writeUint64(b.buf, nodeid) -// b.base.writeUint16(b.buf, uint16(level)) -// targetLength := len(targets) -// if targetLength > math.MaxUint16 { -// // TODO: investigate why we get such massive connections -// targetLength = math.MaxUint16 -// b.base.logger.WithField("action", "hnsw_current_commit_log"). -// WithField("id", b.base.id). -// WithField("original_length", len(targets)). -// WithField("maximum_length", targetLength). -// Warning("condensor length of connections would overflow uint16, cutting off") -// } -// b.base.writeUint16(b.buf, uint16(targetLength)) -// b.base.writeUint64Slice(b.buf, targets[:targetLength]) - -// return nil -// } - -// func (b *bufferedLinksLogger) AddLinkAtLevel(nodeid uint64, level int, -// target uint64) error { -// ec := &errorCompounder{} -// ec.add(b.base.writeCommitType(b.buf, AddLinkAtLevel)) -// ec.add(b.base.writeUint64(b.buf, nodeid)) -// ec.add(b.base.writeUint16(b.buf, uint16(level))) -// ec.add(b.base.writeUint64(b.buf, target)) - -// if err := ec.toError(); err != nil { -// return errors.Wrapf(err, "write link at level %d->%d (%d) to commit log", -// nodeid, target, level) -// } - -// return nil -// } - -// func (b *bufferedLinksLogger) Close() error { -// b.base.Lock() -// defer b.base.Unlock() - -// _, err := b.base.logWriter.Write(b.buf.Bytes()) -// if err != nil { -// return errors.Wrap(err, "flush link buffer to commit logger") -// } - -// return nil -// } diff --git a/adapters/repos/db/vector/hnsw/commit_logger_functional_options.go b/adapters/repos/db/vector/hnsw/commit_logger_functional_options.go deleted file mode 100644 index 2080d0017e7096c6ce85b4f155f7a3ffac036c17..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/commit_logger_functional_options.go +++ /dev/null @@ -1,28 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -type CommitlogOption func(l *hnswCommitLogger) error - -func WithCommitlogThreshold(size int64) CommitlogOption { - return func(l *hnswCommitLogger) error { - l.maxSizeIndividual = size - return nil - } -} - -func WithCommitlogThresholdForCombining(size int64) CommitlogOption { - return func(l *hnswCommitLogger) error { - l.maxSizeCombining = size - return nil - } -} diff --git a/adapters/repos/db/vector/hnsw/commit_logger_noop.go b/adapters/repos/db/vector/hnsw/commit_logger_noop.go deleted file mode 100644 index 3d1efea2ee320d268e5be5dfab9e952b770f4140..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/commit_logger_noop.go +++ /dev/null @@ -1,106 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "context" - - "github.com/weaviate/weaviate/adapters/repos/db/vector/compressionhelpers" -) - -// NoopCommitLogger implements the CommitLogger interface, but does not -// actually write anything to disk -type NoopCommitLogger struct{} - -func (n *NoopCommitLogger) ID() string { - return "" -} - -func (n *NoopCommitLogger) AddPQ(data compressionhelpers.PQData) error { - return nil -} - -func (n *NoopCommitLogger) AddNode(node *vertex) error { - return nil -} - -func (n *NoopCommitLogger) Flush() error { - return nil -} - -func (n *NoopCommitLogger) SetEntryPointWithMaxLayer(id uint64, level int) error { - return nil -} - -func (n *NoopCommitLogger) AddLinkAtLevel(nodeid uint64, level int, target uint64) error { - return nil -} - -func (n *NoopCommitLogger) ReplaceLinksAtLevel(nodeid uint64, level int, targets []uint64) error { - return nil -} - -func (n *NoopCommitLogger) AddTombstone(nodeid uint64) error { - return nil -} - -func (n *NoopCommitLogger) RemoveTombstone(nodeid uint64) error { - return nil -} - -func (n *NoopCommitLogger) DeleteNode(nodeid uint64) error { - return nil -} - -func (n *NoopCommitLogger) ClearLinks(nodeid uint64) error { - return nil -} - -func (n *NoopCommitLogger) ClearLinksAtLevel(nodeid uint64, level uint16) error { - return nil -} - -func (n *NoopCommitLogger) Reset() error { - return nil -} - -func (n *NoopCommitLogger) Drop(ctx context.Context) error { - return nil -} - -func (n *NoopCommitLogger) Shutdown(context.Context) error { - return nil -} - -func MakeNoopCommitLogger() (CommitLogger, error) { - return &NoopCommitLogger{}, nil -} - -func (n *NoopCommitLogger) NewBufferedLinksLogger() BufferedLinksLogger { - return n // return self as it does not do anything anyway -} - -func (n *NoopCommitLogger) Close() error { - return nil -} - -func (n *NoopCommitLogger) StartSwitchLogs() chan struct{} { - return make(chan struct{}) -} - -func (n *NoopCommitLogger) RootPath() string { - return "" -} - -func (n *NoopCommitLogger) SwitchCommitLogs(force bool) error { - return nil -} diff --git a/adapters/repos/db/vector/hnsw/commit_logger_test.go b/adapters/repos/db/vector/hnsw/commit_logger_test.go deleted file mode 100644 index 7edfac66633ec741d34912124b7f3c2d7276335e..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/commit_logger_test.go +++ /dev/null @@ -1,66 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - _ "fmt" - "os" - "testing" -) - -type MockDirEntry struct { - name string - isDir bool -} - -func (d MockDirEntry) Name() string { - return d.name -} - -func (d MockDirEntry) IsDir() bool { - return d.isDir -} - -func (d MockDirEntry) Type() os.FileMode { - return os.ModePerm -} - -func (d MockDirEntry) Info() (os.FileInfo, error) { - return nil, nil -} - -func TestRemoveTmpScratchOrHiddenFiles(t *testing.T) { - entries := []os.DirEntry{ - MockDirEntry{name: "1682473161", isDir: false}, - MockDirEntry{name: ".nfs6b46801cd962afbc00000005", isDir: false}, - MockDirEntry{name: ".mystery-folder", isDir: false}, - MockDirEntry{name: "1682473161.condensed", isDir: false}, - MockDirEntry{name: "1682473161.scratch.tmp", isDir: false}, - } - - expected := []os.DirEntry{ - MockDirEntry{name: "1682473161", isDir: false}, - MockDirEntry{name: "1682473161.condensed", isDir: false}, - } - - result := removeTmpScratchOrHiddenFiles(entries) - - if len(result) != len(expected) { - t.Errorf("Expected %d entries, got %d", len(expected), len(result)) - } - - for i, entry := range result { - if entry.Name() != expected[i].Name() { - t.Errorf("Expected entry %d to be %s, got %s", i, expected[i].Name(), entry.Name()) - } - } -} diff --git a/adapters/repos/db/vector/hnsw/commitlog/bufiowriter.go b/adapters/repos/db/vector/hnsw/commitlog/bufiowriter.go deleted file mode 100644 index 279993ac00d8832996fa708ff1665e5396e548db..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/commitlog/bufiowriter.go +++ /dev/null @@ -1,186 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package commitlog - -import ( - "io" - "os" - "unicode/utf8" -) - -const ( - defaultBufSize = 4096 -) - -// bufWriter implements buffering for an *os.File object. -// If an error occurs writing to a bufWriter, no more data will be -// accepted and all subsequent writes, and Flush, will return the error. -// After all data has been written, the client should call the -// Flush method to guarantee all data has been forwarded to -// the underlying *os.File. -type bufWriter struct { - err error - buf []byte - n int - wr *os.File -} - -// NewWriterSize returns a new Writer whose buffer has at least the specified -// size. If the argument *os.File is already a Writer with large enough -// size, it returns the underlying Writer. -func NewWriterSize(w *os.File, size int) *bufWriter { - if size <= 0 { - size = defaultBufSize - } - return &bufWriter{ - buf: make([]byte, size), - wr: w, - } -} - -// NewWriter returns a new Writer whose buffer has the default size. -func NewWriter(w *os.File) *bufWriter { - return NewWriterSize(w, defaultBufSize) -} - -// Size returns the size of the underlying buffer in bytes. -func (b *bufWriter) Size() int { return len(b.buf) } - -// Reset discards any unflushed buffered data, clears any error, and -// resets b to write its output to w. -func (b *bufWriter) Reset(w *os.File) { - b.err = nil - b.n = 0 - b.wr = w -} - -// Flush writes any buffered data to the underlying *os.File. -func (b *bufWriter) Flush() error { - if b.err != nil { - return b.err - } - if b.n == 0 { - return nil - } - n, err := b.wr.Write(b.buf[0:b.n]) - if n < b.n && err == nil { - err = io.ErrShortWrite - } - if err != nil { - if n > 0 && n < b.n { - copy(b.buf[0:b.n-n], b.buf[n:b.n]) - } - b.n -= n - b.err = err - return err - } - b.n = 0 - return nil -} - -// Available returns how many bytes are unused in the buffer. -func (b *bufWriter) Available() int { return len(b.buf) - b.n } - -// Buffered returns the number of bytes that have been written into the current buffer. -func (b *bufWriter) Buffered() int { return b.n } - -// Write writes the contents of p into the buffer. -// It returns the number of bytes written. -// If nn < len(p), it also returns an error explaining -// why the write is short. -func (b *bufWriter) Write(p []byte) (nn int, err error) { - for len(p) > b.Available() && b.err == nil { - var n int - if b.Buffered() == 0 { - // Large write, empty buffer. - // Write directly from p to avoid copy. - n, b.err = b.wr.Write(p) - } else { - n = copy(b.buf[b.n:], p) - b.n += n - b.Flush() - } - nn += n - p = p[n:] - } - if b.err != nil { - return nn, b.err - } - n := copy(b.buf[b.n:], p) - b.n += n - nn += n - return nn, nil -} - -// WriteByte writes a single byte. -func (b *bufWriter) WriteByte(c byte) error { - if b.err != nil { - return b.err - } - if b.Available() <= 0 && b.Flush() != nil { - return b.err - } - b.buf[b.n] = c - b.n++ - return nil -} - -// WriteRune writes a single Unicode code point, returning -// the number of bytes written and any error. -func (b *bufWriter) WriteRune(r rune) (size int, err error) { - if r < utf8.RuneSelf { - err = b.WriteByte(byte(r)) - if err != nil { - return 0, err - } - return 1, nil - } - if b.err != nil { - return 0, b.err - } - n := b.Available() - if n < utf8.UTFMax { - if b.Flush(); b.err != nil { - return 0, b.err - } - n = b.Available() - if n < utf8.UTFMax { - // Can only happen if buffer is silly small. - return b.WriteString(string(r)) - } - } - size = utf8.EncodeRune(b.buf[b.n:], r) - b.n += size - return size, nil -} - -// WriteString writes a string. -// It returns the number of bytes written. -// If the count is less than len(s), it also returns an error explaining -// why the write is short. -func (b *bufWriter) WriteString(s string) (int, error) { - nn := 0 - for len(s) > b.Available() && b.err == nil { - n := copy(b.buf[b.n:], s) - b.n += n - nn += n - s = s[n:] - b.Flush() - } - if b.err != nil { - return nn, b.err - } - n := copy(b.buf[b.n:], s) - b.n += n - nn += n - return nn, nil -} diff --git a/adapters/repos/db/vector/hnsw/commitlog/logger.go b/adapters/repos/db/vector/hnsw/commitlog/logger.go deleted file mode 100644 index 81784f52ae47c3b8cec983168243b61d9e4f0ca7..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/commitlog/logger.go +++ /dev/null @@ -1,250 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package commitlog - -import ( - "encoding/binary" - "os" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/adapters/repos/db/vector/compressionhelpers" -) - -type Logger struct { - file *os.File - bufw *bufWriter -} - -// TODO: these are duplicates with the hnsw package, unify them -type HnswCommitType uint8 // 256 options, plenty of room for future extensions - -// TODO: these are duplicates with the hnsw package, unify them -const ( - AddNode HnswCommitType = iota - SetEntryPointMaxLevel - AddLinkAtLevel - ReplaceLinksAtLevel - AddTombstone - RemoveTombstone - ClearLinks - DeleteNode - ResetIndex - ClearLinksAtLevel // added in v1.8.0-rc.1, see https://github.com/weaviate/weaviate/issues/1701 - AddLinksAtLevel // added in v1.8.0-rc.1, see https://github.com/weaviate/weaviate/issues/1705 - AddPQ -) - -func NewLogger(fileName string) *Logger { - file, err := os.Create(fileName) - if err != nil { - panic(err) - } - - return &Logger{file: file, bufw: NewWriter(file)} -} - -func NewLoggerWithFile(file *os.File) *Logger { - return &Logger{file: file, bufw: NewWriterSize(file, 32*1024)} -} - -func (l *Logger) SetEntryPointWithMaxLayer(id uint64, level int) error { - toWrite := make([]byte, 11) - toWrite[0] = byte(SetEntryPointMaxLevel) - binary.LittleEndian.PutUint64(toWrite[1:9], id) - binary.LittleEndian.PutUint16(toWrite[9:11], uint16(level)) - _, err := l.bufw.Write(toWrite) - return err -} - -func (l *Logger) AddNode(id uint64, level int) error { - toWrite := make([]byte, 11) - toWrite[0] = byte(AddNode) - binary.LittleEndian.PutUint64(toWrite[1:9], id) - binary.LittleEndian.PutUint16(toWrite[9:11], uint16(level)) - _, err := l.bufw.Write(toWrite) - return err -} - -func (l *Logger) AddPQ(data compressionhelpers.PQData) error { - toWrite := make([]byte, 10) - toWrite[0] = byte(AddPQ) - binary.LittleEndian.PutUint16(toWrite[1:3], data.Dimensions) - toWrite[3] = byte(data.EncoderType) - binary.LittleEndian.PutUint16(toWrite[4:6], data.Ks) - binary.LittleEndian.PutUint16(toWrite[6:8], data.M) - toWrite[8] = data.EncoderDistribution - if data.UseBitsEncoding { - toWrite[9] = 1 - } else { - toWrite[9] = 0 - } - - for _, encoder := range data.Encoders { - toWrite = append(toWrite, encoder.ExposeDataForRestore()...) - } - _, err := l.bufw.Write(toWrite) - return err -} - -func (l *Logger) AddLinkAtLevel(id uint64, level int, target uint64) error { - toWrite := make([]byte, 19) - toWrite[0] = byte(AddLinkAtLevel) - binary.LittleEndian.PutUint64(toWrite[1:9], id) - binary.LittleEndian.PutUint16(toWrite[9:11], uint16(level)) - binary.LittleEndian.PutUint64(toWrite[11:19], target) - _, err := l.bufw.Write(toWrite) - return err -} - -func (l *Logger) AddLinksAtLevel(id uint64, level int, targets []uint64) error { - toWrite := make([]byte, 13+len(targets)*8) - toWrite[0] = byte(AddLinksAtLevel) - binary.LittleEndian.PutUint64(toWrite[1:9], id) - binary.LittleEndian.PutUint16(toWrite[9:11], uint16(level)) - binary.LittleEndian.PutUint16(toWrite[11:13], uint16(len(targets))) - for i, target := range targets { - offsetStart := 13 + i*8 - offsetEnd := offsetStart + 8 - binary.LittleEndian.PutUint64(toWrite[offsetStart:offsetEnd], target) - } - _, err := l.bufw.Write(toWrite) - return err -} - -// chunks links in increments of 8, so that we never have to allocate a dynamic -// []byte size which would be guaranteed to escape to the heap -func (l *Logger) ReplaceLinksAtLevel(id uint64, level int, targets []uint64) error { - headers := make([]byte, 13) - headers[0] = byte(ReplaceLinksAtLevel) - binary.LittleEndian.PutUint64(headers[1:9], id) - binary.LittleEndian.PutUint16(headers[9:11], uint16(level)) - binary.LittleEndian.PutUint16(headers[11:13], uint16(len(targets))) - _, err := l.bufw.Write(headers) - if err != nil { - return errors.Wrap(err, "write headers") - } - - i := 0 - // chunks of 8 - buf := make([]byte, 64) - for i < len(targets) { - if i != 0 && i%8 == 0 { - if _, err := l.bufw.Write(buf); err != nil { - return errors.Wrap(err, "write link chunk") - } - } - - pos := i % 8 - start := pos * 8 - end := start + 8 - binary.LittleEndian.PutUint64(buf[start:end], targets[i]) - - i++ - } - - // remainder - if i != 0 { - start := 0 - end := i % 8 * 8 - if end == 0 { - end = 64 - } - - if _, err := l.bufw.Write(buf[start:end]); err != nil { - return errors.Wrap(err, "write link remainder") - } - } - - return nil -} - -func (l *Logger) AddTombstone(id uint64) error { - toWrite := make([]byte, 9) - toWrite[0] = byte(AddTombstone) - binary.LittleEndian.PutUint64(toWrite[1:9], id) - _, err := l.bufw.Write(toWrite) - return err -} - -func (l *Logger) RemoveTombstone(id uint64) error { - toWrite := make([]byte, 9) - toWrite[0] = byte(RemoveTombstone) - binary.LittleEndian.PutUint64(toWrite[1:9], id) - _, err := l.bufw.Write(toWrite) - return err -} - -func (l *Logger) ClearLinks(id uint64) error { - toWrite := make([]byte, 9) - toWrite[0] = byte(ClearLinks) - binary.LittleEndian.PutUint64(toWrite[1:9], id) - _, err := l.bufw.Write(toWrite) - return err -} - -func (l *Logger) ClearLinksAtLevel(id uint64, level uint16) error { - toWrite := make([]byte, 11) - toWrite[0] = byte(ClearLinksAtLevel) - binary.LittleEndian.PutUint64(toWrite[1:9], id) - binary.LittleEndian.PutUint16(toWrite[9:11], level) - _, err := l.bufw.Write(toWrite) - return err -} - -func (l *Logger) DeleteNode(id uint64) error { - toWrite := make([]byte, 9) - toWrite[0] = byte(DeleteNode) - binary.LittleEndian.PutUint64(toWrite[1:9], id) - _, err := l.bufw.Write(toWrite) - return err -} - -func (l *Logger) Reset() error { - toWrite := make([]byte, 1) - toWrite[0] = byte(ResetIndex) - _, err := l.bufw.Write(toWrite) - return err -} - -func (l *Logger) FileSize() (int64, error) { - i, err := l.file.Stat() - if err != nil { - return -1, err - } - - return i.Size(), nil -} - -func (l *Logger) FileName() (string, error) { - i, err := l.file.Stat() - if err != nil { - return "", err - } - - return i.Name(), nil -} - -func (l *Logger) Flush() error { - return l.bufw.Flush() -} - -func (l *Logger) Close() error { - if err := l.bufw.Flush(); err != nil { - return err - } - - if err := l.file.Close(); err != nil { - return err - } - - return nil -} diff --git a/adapters/repos/db/vector/hnsw/commitlog/logger_test.go b/adapters/repos/db/vector/hnsw/commitlog/logger_test.go deleted file mode 100644 index 1d5d95241a31b7bd2aa9ce3f6624f45a716de888..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/commitlog/logger_test.go +++ /dev/null @@ -1,178 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package commitlog - -import ( - "os" - "testing" -) - -func BenchmarkSetEntryPoint(b *testing.B) { - defer os.Remove("./testfile") - ids := make([]uint64, 100) - levels := make([]int, 100) - - l := NewLogger("./testfile") - - b.ReportAllocs() - - for j := 0; j < b.N; j++ { - for i := 0; i < 100; i++ { - l.SetEntryPointWithMaxLayer(ids[i], levels[i]) - } - } -} - -func BenchmarkAddNode(b *testing.B) { - defer os.Remove("./testfile") - ids := make([]uint64, 100) - levels := make([]int, 100) - - l := NewLogger("./testfile") - - b.ReportAllocs() - - for j := 0; j < b.N; j++ { - for i := 0; i < 100; i++ { - l.AddNode(ids[i], levels[i]) - } - } -} - -func BenchmarkAddLinkAtLevel(b *testing.B) { - defer os.Remove("./testfile") - ids := make([]uint64, 100) - levels := make([]int, 100) - links := make([]uint64, 100) - - l := NewLogger("./testfile") - - b.ReportAllocs() - - for j := 0; j < b.N; j++ { - for i := 0; i < 100; i++ { - l.AddLinkAtLevel(ids[i], levels[i], links[i]) - } - } -} - -func BenchmarkReplaceLinksAtLevel32(b *testing.B) { - defer os.Remove("./testfile") - ids := make([]uint64, 100) - levels := make([]int, 100) - links := make([][]uint64, 100) - for i := range links { - links[i] = make([]uint64, 32) - } - - l := NewLogger("./testfile") - - b.ReportAllocs() - - for j := 0; j < b.N; j++ { - for i := 0; i < 100; i++ { - l.ReplaceLinksAtLevel(ids[i], levels[i], links[i]) - } - } -} - -func BenchmarkReplaceLinksAtLevel33(b *testing.B) { - defer os.Remove("./testfile") - ids := make([]uint64, 100) - levels := make([]int, 100) - links := make([][]uint64, 100) - for i := range links { - links[i] = make([]uint64, 33) - } - - l := NewLogger("./testfile") - - b.ReportAllocs() - - for j := 0; j < b.N; j++ { - for i := 0; i < 100; i++ { - l.ReplaceLinksAtLevel(ids[i], levels[i], links[i]) - } - } -} - -func BenchmarkAddTombstone(b *testing.B) { - defer os.Remove("./testfile") - ids := make([]uint64, 100) - - l := NewLogger("./testfile") - - b.ReportAllocs() - - for j := 0; j < b.N; j++ { - for i := 0; i < 100; i++ { - l.AddTombstone(ids[i]) - } - } -} - -func BenchmarkRemoveTombstone(b *testing.B) { - defer os.Remove("./testfile") - ids := make([]uint64, 100) - - l := NewLogger("./testfile") - - b.ReportAllocs() - - for j := 0; j < b.N; j++ { - for i := 0; i < 100; i++ { - l.AddTombstone(ids[i]) - } - } -} - -func BenchmarkClearLinks(b *testing.B) { - defer os.Remove("./testfile") - ids := make([]uint64, 100) - - l := NewLogger("./testfile") - - b.ReportAllocs() - - for j := 0; j < b.N; j++ { - for i := 0; i < 100; i++ { - l.ClearLinks(ids[i]) - } - } -} - -func BenchmarkDeleteNode(b *testing.B) { - ids := make([]uint64, 100) - - l := NewLogger("./testfile") - - b.ReportAllocs() - - for j := 0; j < b.N; j++ { - for i := 0; i < 100; i++ { - l.DeleteNode(ids[i]) - } - } -} - -func BenchmarkReset(b *testing.B) { - defer os.Remove("./testfile") - l := NewLogger("./testfile") - - b.ReportAllocs() - - for j := 0; j < b.N; j++ { - for i := 0; i < 100; i++ { - l.Reset() - } - } -} diff --git a/adapters/repos/db/vector/hnsw/compress.go b/adapters/repos/db/vector/hnsw/compress.go deleted file mode 100644 index 1b91e7cfa3dae67653d5e759dccab293da63bce8..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/compress.go +++ /dev/null @@ -1,106 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "context" - "errors" - "fmt" - - "github.com/weaviate/weaviate/adapters/repos/db/vector/compressionhelpers" - - "github.com/weaviate/weaviate/entities/storobj" - ent "github.com/weaviate/weaviate/entities/vectorindex/hnsw" -) - -func (h *hnsw) calculateOptimalSegments(dims int) int { - if dims >= 2048 && dims%8 == 0 { - return dims / 8 - } else if dims >= 768 && dims%6 == 0 { - return dims / 6 - } else if dims >= 256 && dims%4 == 0 { - return dims / 4 - } else if dims%2 == 0 { - return dims / 2 - } - return dims -} - -func (h *hnsw) compress(cfg ent.UserConfig) error { - if !cfg.PQ.Enabled && !cfg.BQ.Enabled { - return nil - } - - h.compressActionLock.Lock() - defer h.compressActionLock.Unlock() - data := h.cache.All() - if cfg.PQ.Enabled { - if h.isEmpty() { - return errors.New("Compress command cannot be executed before inserting some data. Please, insert your data first.") - } - dims := int(h.dims) - - if cfg.PQ.Segments <= 0 { - cfg.PQ.Segments = h.calculateOptimalSegments(dims) - h.pqConfig.Segments = cfg.PQ.Segments - } - - cleanData := make([][]float32, 0, len(data)) - for i := range data { - // Rather than just taking the cache dump at face value, let's explicitly - // request the vectors. Otherwise we would miss any vector that's currently - // not in the cache, for example because the cache is not hot yet after a - // restart. - p, err := h.cache.Get(context.Background(), uint64(i)) - if err != nil { - var e storobj.ErrNotFound - if errors.As(err, &e) { - // already deleted, ignore - continue - } else { - return fmt.Errorf("unexpected error obtaining vectors for fitting: %w", err) - } - } - - if p == nil { - // already deleted, ignore - continue - } - - cleanData = append(cleanData, p) - } - - var err error - h.compressor, err = compressionhelpers.NewPQCompressor(cfg.PQ, h.distancerProvider, dims, 1e12, h.logger, cleanData, h.store) - if err != nil { - return fmt.Errorf("Compressing vectors: %w", err) - } - h.commitLog.AddPQ(h.compressor.ExposeFields()) - } else { - var err error - h.compressor, err = compressionhelpers.NewBQCompressor(h.distancerProvider, 1e12, h.logger, h.store) - if err != nil { - return err - } - } - compressionhelpers.Concurrently(uint64(len(data)), - func(index uint64) { - if data[index] == nil { - return - } - h.compressor.Preload(index, data[index]) - }) - - h.compressed.Store(true) - h.cache.Drop() - return nil -} diff --git a/adapters/repos/db/vector/hnsw/compress_deletes_test.go b/adapters/repos/db/vector/hnsw/compress_deletes_test.go deleted file mode 100644 index 172efc66147af14983d56c999ccac8066ebe574d..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/compress_deletes_test.go +++ /dev/null @@ -1,183 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build !race - -package hnsw - -import ( - "context" - "fmt" - "os" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/adapters/repos/db/vector/common" - "github.com/weaviate/weaviate/adapters/repos/db/vector/compressionhelpers" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" - "github.com/weaviate/weaviate/adapters/repos/db/vector/testinghelpers" - "github.com/weaviate/weaviate/entities/cyclemanager" - "github.com/weaviate/weaviate/entities/storobj" - ent "github.com/weaviate/weaviate/entities/vectorindex/hnsw" -) - -func Test_NoRaceCompressDoesNotCrash(t *testing.T) { - efConstruction := 64 - ef := 32 - maxNeighbors := 32 - dimensions := 20 - vectors_size := 10000 - queries_size := 100 - k := 100 - delete_indices := make([]uint64, 0, 1000) - for i := 0; i < 1000; i++ { - delete_indices = append(delete_indices, uint64(i+10)) - } - delete_indices = append(delete_indices, uint64(1)) - - vectors, queries := testinghelpers.RandomVecs(vectors_size, queries_size, dimensions) - distancer := distancer.NewL2SquaredProvider() - - uc := ent.UserConfig{} - uc.MaxConnections = maxNeighbors - uc.EFConstruction = efConstruction - uc.EF = ef - uc.VectorCacheMaxObjects = 10e12 - uc.PQ = ent.PQConfig{Enabled: true, Encoder: ent.PQEncoder{Type: "title", Distribution: "normal"}} - - index, _ := New(Config{ - RootPath: t.TempDir(), - ID: "recallbenchmark", - MakeCommitLoggerThunk: MakeNoopCommitLogger, - DistanceProvider: distancer, - VectorForIDThunk: func(ctx context.Context, id uint64) ([]float32, error) { - if int(id) >= len(vectors) { - return nil, storobj.NewErrNotFoundf(id, "out of range") - } - return vectors[int(id)], nil - }, - TempVectorForIDThunk: func(ctx context.Context, id uint64, container *common.VectorSlice) ([]float32, error) { - copy(container.Slice, vectors[int(id)]) - return container.Slice, nil - }, - }, uc, cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - cyclemanager.NewCallbackGroupNoop(), testinghelpers.NewDummyStore(t)) - defer index.Shutdown(context.Background()) - compressionhelpers.Concurrently(uint64(len(vectors)), func(id uint64) { - index.Add(uint64(id), vectors[id]) - }) - index.Delete(delete_indices...) - - cfg := ent.PQConfig{ - Enabled: true, - Encoder: ent.PQEncoder{ - Type: ent.PQEncoderTypeKMeans, - Distribution: ent.PQEncoderDistributionLogNormal, - }, - Segments: dimensions, - Centroids: 256, - } - uc.PQ = cfg - index.compress(uc) - for _, v := range queries { - _, _, err := index.SearchByVector(v, k, nil) - assert.Nil(t, err) - } -} - -func TestHnswPqNilVectors(t *testing.T) { - dimensions := 20 - vectors_size := 10_000 - queries_size := 10 - - vectors, _ := testinghelpers.RandomVecs(vectors_size, queries_size, dimensions) - - // set some vectors to nil - for i := range vectors { - if i == 500 { - vectors[i] = nil - } - } - - userConfig := ent.UserConfig{ - MaxConnections: 30, - EFConstruction: 64, - EF: 32, - - // The actual size does not matter for this test, but if it defaults to - // zero it will constantly think it's full and needs to be deleted - even - // after just being deleted, so make sure to use a positive number here. - VectorCacheMaxObjects: 1000000, - } - - rootPath := "doesnt-matter-as-committlogger-is-mocked-out" - defer func(path string) { - err := os.RemoveAll(path) - if err != nil { - fmt.Println(err) - } - }(rootPath) - - index, err := New(Config{ - RootPath: rootPath, - ID: "nil-vector-test", - MakeCommitLoggerThunk: MakeNoopCommitLogger, - DistanceProvider: distancer.NewCosineDistanceProvider(), - VectorForIDThunk: func(ctx context.Context, id uint64) ([]float32, error) { - vec := vectors[int(id)] - if vec == nil { - return nil, storobj.NewErrNotFoundf(id, "nil vec") - } - return vec, nil - }, - TempVectorForIDThunk: TempVectorForIDThunk(vectors), - }, userConfig, cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), testinghelpers.NewDummyStore(t)) - - require.NoError(t, err) - - compressionhelpers.Concurrently(uint64(len(vectors)/2), func(id uint64) { - if vectors[id] == nil { - return - } - - err := index.Add(uint64(id), vectors[id]) - require.Nil(t, err) - }) - - userConfig.PQ = ent.PQConfig{ - Enabled: true, - Encoder: ent.PQEncoder{ - Type: ent.PQEncoderTypeTile, - Distribution: ent.PQEncoderDistributionLogNormal, - }, - BitCompression: false, - Segments: dimensions, - Centroids: 256, - } - - ch := make(chan error) - err = index.UpdateUserConfig(userConfig, func() { - close(ch) - }) - require.NoError(t, err) - - <-ch - start := uint64(len(vectors) / 2) - compressionhelpers.Concurrently(uint64(len(vectors)/2), func(id uint64) { - if vectors[id+start] == nil { - return - } - - err = index.Add(uint64(id)+start, vectors[id+start]) - require.Nil(t, err) - }) -} diff --git a/adapters/repos/db/vector/hnsw/compress_integration_test.go b/adapters/repos/db/vector/hnsw/compress_integration_test.go deleted file mode 100644 index 100209e3733fc4e36f20e0d60904acb63d3df3fe..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/compress_integration_test.go +++ /dev/null @@ -1,99 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package hnsw - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/adapters/repos/db/vector/common" - "github.com/weaviate/weaviate/adapters/repos/db/vector/compressionhelpers" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" - "github.com/weaviate/weaviate/adapters/repos/db/vector/testinghelpers" - "github.com/weaviate/weaviate/entities/cyclemanager" - ent "github.com/weaviate/weaviate/entities/vectorindex/hnsw" -) - -func Test_NoRaceCompressAdaptsSegments(t *testing.T) { - ctx := context.Background() - - efConstruction := 64 - ef := 32 - maxNeighbors := 32 - vectorsCount := 1000 - dimensions := 6 - expectedSegments := 3 - - vectors, _ := testinghelpers.RandomVecs(vectorsCount, 0, dimensions) - distancer := distancer.NewL2SquaredProvider() - - uc := ent.UserConfig{} - uc.MaxConnections = maxNeighbors - uc.EFConstruction = efConstruction - uc.EF = ef - uc.VectorCacheMaxObjects = 10e12 - uc.PQ = ent.PQConfig{ - Enabled: true, - Encoder: ent.PQEncoder{ - Type: ent.PQEncoderTypeKMeans, - Distribution: ent.PQEncoderDistributionNormal, - }, - } - - store := testinghelpers.NewDummyStore(t) - defer func() { - err := store.Shutdown(ctx) - require.NoError(t, err) - }() - - index, err := New( - Config{ - RootPath: t.TempDir(), - ID: "recallbenchmark", - MakeCommitLoggerThunk: MakeNoopCommitLogger, - DistanceProvider: distancer, - VectorForIDThunk: func(ctx context.Context, id uint64) ([]float32, error) { - return vectors[int(id)], nil - }, - TempVectorForIDThunk: func(ctx context.Context, id uint64, container *common.VectorSlice) ([]float32, error) { - copy(container.Slice, vectors[int(id)]) - return container.Slice, nil - }, - }, uc, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), store) - require.NoError(t, err) - defer func() { - err := index.Shutdown(ctx) - require.NoError(t, err) - }() - - compressionhelpers.Concurrently(uint64(len(vectors)), func(id uint64) { - index.Add(uint64(id), vectors[id]) - }) - uc.PQ = ent.PQConfig{ - Enabled: true, - Encoder: ent.PQEncoder{ - Type: ent.PQEncoderTypeKMeans, - Distribution: ent.PQEncoderDistributionNormal, - }, - Segments: 0, - Centroids: 256, - } - index.compress(uc) - assert.Equal(t, expectedSegments, int(index.compressor.ExposeFields().M)) - assert.Equal(t, expectedSegments, index.pqConfig.Segments) -} diff --git a/adapters/repos/db/vector/hnsw/compress_recall_test.go b/adapters/repos/db/vector/hnsw/compress_recall_test.go deleted file mode 100644 index 6a0d427fe4f24fcccf379c16e5e2c8da416ea396..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/compress_recall_test.go +++ /dev/null @@ -1,141 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build !race - -package hnsw_test - -import ( - "context" - "fmt" - "os" - "sync" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/adapters/repos/db/vector/common" - "github.com/weaviate/weaviate/adapters/repos/db/vector/compressionhelpers" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" - "github.com/weaviate/weaviate/adapters/repos/db/vector/testinghelpers" - "github.com/weaviate/weaviate/entities/cyclemanager" - "github.com/weaviate/weaviate/entities/storobj" - ent "github.com/weaviate/weaviate/entities/vectorindex/hnsw" -) - -func distanceWrapper(provider distancer.Provider) func(x, y []float32) float32 { - return func(x, y []float32) float32 { - dist, _, _ := provider.SingleDist(x, y) - return dist - } -} - -func Test_NoRaceCompressionRecall(t *testing.T) { - path := t.TempDir() - - efConstruction := 64 - ef := 64 - maxNeighbors := 32 - segments := 4 - dimensions := 64 - vectors_size := 10000 - queries_size := 100 - fmt.Println("Sift1M PQ") - before := time.Now() - vectors, queries := testinghelpers.RandomVecs(vectors_size, queries_size, dimensions) - testinghelpers.Normalize(vectors) - testinghelpers.Normalize(queries) - k := 100 - - distancers := []distancer.Provider{ - distancer.NewL2SquaredProvider(), - distancer.NewCosineDistanceProvider(), - distancer.NewDotProductProvider(), - } - - for _, distancer := range distancers { - truths := make([][]uint64, queries_size) - compressionhelpers.Concurrently(uint64(len(queries)), func(i uint64) { - truths[i], _ = testinghelpers.BruteForce(vectors, queries[i], k, distanceWrapper(distancer)) - }) - fmt.Printf("generating data took %s\n", time.Since(before)) - - uc := ent.UserConfig{ - MaxConnections: maxNeighbors, - EFConstruction: efConstruction, - EF: ef, - VectorCacheMaxObjects: 10e12, - } - index, _ := hnsw.New(hnsw.Config{ - RootPath: path, - ID: "recallbenchmark", - MakeCommitLoggerThunk: hnsw.MakeNoopCommitLogger, - ClassName: "clasRecallBenchmark", - ShardName: "shardRecallBenchmark", - DistanceProvider: distancer, - VectorForIDThunk: func(ctx context.Context, id uint64) ([]float32, error) { - if int(id) >= len(vectors) { - return nil, storobj.NewErrNotFoundf(id, "out of range") - } - return vectors[int(id)], nil - }, - TempVectorForIDThunk: func(ctx context.Context, id uint64, container *common.VectorSlice) ([]float32, error) { - copy(container.Slice, vectors[int(id)]) - return container.Slice, nil - }, - }, uc, cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - cyclemanager.NewCallbackGroupNoop(), testinghelpers.NewDummyStore(t)) - init := time.Now() - compressionhelpers.Concurrently(uint64(vectors_size), func(id uint64) { - index.Add(id, vectors[id]) - }) - before = time.Now() - fmt.Println("Start compressing...") - uc.PQ = ent.PQConfig{ - Enabled: true, - Segments: dimensions / segments, - Centroids: 256, - Encoder: ent.NewDefaultUserConfig().PQ.Encoder, - } - uc.EF = 256 - wg := sync.WaitGroup{} - wg.Add(1) - index.UpdateUserConfig(uc, func() { - fmt.Printf("Time to compress: %s\n", time.Since(before)) - fmt.Printf("Building the index took %s\n", time.Since(init)) - - var relevant uint64 - var retrieved int - - var querying time.Duration = 0 - compressionhelpers.Concurrently(uint64(len(queries)), func(i uint64) { - before = time.Now() - results, _, _ := index.SearchByVector(queries[i], k, nil) - querying += time.Since(before) - retrieved += k - relevant += testinghelpers.MatchesInLists(truths[i], results) - }) - - recall := float32(relevant) / float32(retrieved) - latency := float32(querying.Microseconds()) / float32(queries_size) - fmt.Println(recall, latency) - assert.True(t, recall > 0.9) - - err := os.RemoveAll(path) - if err != nil { - fmt.Println(err) - } - wg.Done() - }) - wg.Wait() - } -} diff --git a/adapters/repos/db/vector/hnsw/compress_sift_test.go b/adapters/repos/db/vector/hnsw/compress_sift_test.go deleted file mode 100644 index d0ab6b7885e1f64b8ddbc1e8b719d48a47ba3f39..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/compress_sift_test.go +++ /dev/null @@ -1,614 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build benchmarkSiftRecall -// +build benchmarkSiftRecall - -package hnsw_test - -import ( - "context" - "fmt" - "io/ioutil" - "math" - "os" - "strconv" - "strings" - "sync" - "testing" - "time" - - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv" - "github.com/weaviate/weaviate/adapters/repos/db/vector/compressionhelpers" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" - "github.com/weaviate/weaviate/adapters/repos/db/vector/testinghelpers" - "github.com/weaviate/weaviate/entities/cyclemanager" - ent "github.com/weaviate/weaviate/entities/vectorindex/hnsw" -) - -func distanceWrapper(provider distancer.Provider) func(x, y []float32) float32 { - return func(x, y []float32) float32 { - dist, _, _ := provider.SingleDist(x, y) - return dist - } -} - -const rootPath = "doesnt-matter-as-committlogger-is-mocked-out" - -func TestRecall(t *testing.T) { - defer func(path string) { - err := os.RemoveAll(path) - if err != nil { - fmt.Println(err) - } - }(rootPath) - fmt.Println("Sift1MPQKMeans 10K/1K") - efConstruction := 64 - ef := 32 - maxNeighbors := 32 - dimensions := 128 - vectors_size := 200000 - queries_size := 100 - switch_at := vectors_size - before := time.Now() - vectors, queries := testinghelpers.RandomVecs(vectors_size, queries_size, dimensions) - k := 10 - distancer := distancer.NewL2SquaredProvider() - fmt.Printf("generating data took %s\n", time.Since(before)) - - uc := ent.UserConfig{} - uc.MaxConnections = maxNeighbors - uc.EFConstruction = efConstruction - uc.EF = ef - uc.VectorCacheMaxObjects = 10e12 - - index, _ := hnsw.New(hnsw.Config{ - RootPath: rootPath, - ID: "recallbenchmark", - MakeCommitLoggerThunk: hnsw.MakeNoopCommitLogger, - DistanceProvider: distancer, - VectorForIDThunk: func(ctx context.Context, id uint64) ([]float32, error) { - return vectors[int(id)], nil - }, - }, uc, newDummyStore(t)) - init := time.Now() - compressionhelpers.Concurrently(uint64(switch_at), func(_, id uint64, _ *sync.Mutex) { - index.Add(uint64(id), vectors[id]) - if id%1000 == 0 { - fmt.Println(id, time.Since(before)) - } - }) - before = time.Now() - uc.PQ.Enabled = true - index.UpdateUserConfig(uc, func() {}) /*should have configuration.pr.enabled = true*/ - fmt.Printf("Time to compress: %s", time.Since(before)) - fmt.Println() - compressionhelpers.Concurrently(uint64(vectors_size-switch_at), func(_, id uint64, _ *sync.Mutex) { - idx := switch_at + int(id) - index.Add(uint64(idx), vectors[idx]) - if id%1000 == 0 { - fmt.Println(idx, time.Since(before)) - } - }) - fmt.Printf("Building the index took %s\n", time.Since(init)) - - lastRecall := float32(0.0) - for _, currentEF := range []int{32, 64, 128, 256, 512} { - uc.EF = currentEF - index.UpdateUserConfig(uc, func() {}) - fmt.Println(currentEF) - var relevant uint64 - var retrieved int - - var querying time.Duration = 0 - for i := 0; i < len(queries); i++ { - truth := testinghelpers.BruteForce(vectors, queries[i], k, distanceWrapper(distancer)) - before = time.Now() - results, _, _ := index.SearchByVector(queries[i], k, nil) - querying += time.Since(before) - retrieved += k - relevant += testinghelpers.MatchesInLists(truth, results) - } - - recall := float32(relevant) / float32(retrieved) - assert.True(t, recall > float32(lastRecall)) - lastRecall = recall - } - assert.True(t, lastRecall > 0.95) -} - -func TestHnswPqGist(t *testing.T) { - defer func(path string) { - err := os.RemoveAll(path) - if err != nil { - fmt.Println(err) - } - }(rootPath) - params := [][]int{ - //{64, 64, 32}, - {128, 128, 64}, - {256, 256, 128}, - {512, 512, 256}, - } - dimensions := 960 - vectors_size := 1000000 - queries_size := 1000 - switch_at := 200000 - - before := time.Now() - vectors, queries := testinghelpers.ReadVecs(vectors_size, queries_size, dimensions, "gist", "../diskAnn/testdata") - testinghelpers.Normalize(vectors) - testinghelpers.Normalize(queries) - for i, v := range vectors { - for j, x := range v { - if math.IsNaN(float64(x)) { - fmt.Println(i, j, v, x) - } - } - } - k := 100 - distancer := distancer.NewCosineDistanceProvider() - truths := testinghelpers.BuildTruths(queries_size, vectors_size, queries, vectors, k, distanceWrapper(distancer), "../diskAnn/testdata/gist/cosine") - fmt.Printf("generating data took %s\n", time.Since(before)) - for segmentRate := 3; segmentRate < 4; segmentRate++ { - fmt.Println(segmentRate) - fmt.Println() - for i := 0; i < len(params); i++ { - efConstruction := params[i][0] - ef := params[i][1] - maxNeighbors := params[i][2] - - uc := ent.UserConfig{ - MaxConnections: maxNeighbors, - EFConstruction: efConstruction, - EF: ef, - VectorCacheMaxObjects: 10e12, - PQ: ent.PQConfig{ - Enabled: false, - Segments: dimensions / int(math.Pow(2, float64(segmentRate))), - Encoder: ent.PQEncoder{ - Type: ent.PQEncoderTypeKMeans, - Distribution: ent.PQEncoderDistributionLogNormal, - }, - }, - } - index, _ := hnsw.New(hnsw.Config{ - RootPath: rootPath, - ID: "recallbenchmark", - MakeCommitLoggerThunk: hnsw.MakeNoopCommitLogger, - DistanceProvider: distancer, - VectorForIDThunk: func(ctx context.Context, id uint64) ([]float32, error) { - return vectors[int(id)], nil - }, - }, uc, newDummyStore(t)) - init := time.Now() - total := 200000 - compressionhelpers.Concurrently(uint64(switch_at), func(_, id uint64, _ *sync.Mutex) { - total++ - if total%100000 == 0 { - fmt.Println(total) - } - index.Add(uint64(id), vectors[id]) - }) - before = time.Now() - uc.PQ.Enabled = true - index.UpdateUserConfig(uc, func() {}) - fmt.Printf("Time to compress: %s", time.Since(before)) - fmt.Println() - compressionhelpers.Concurrently(uint64(vectors_size-switch_at), func(_, id uint64, _ *sync.Mutex) { - idx := switch_at + int(id) - index.Add(uint64(idx), vectors[idx]) - }) - fmt.Printf("Building the index took %s\n", time.Since(init)) - var relevant uint64 - var retrieved int - - var querying time.Duration = 0 - compressionhelpers.Concurrently(uint64(len(queries)), func(_, i uint64, _ *sync.Mutex) { - before = time.Now() - results, _, _ := index.SearchByVector(queries[i], k, nil) - querying += time.Since(before) - retrieved += k - relevant += testinghelpers.MatchesInLists(truths[i], results) - }) - - recall := float32(relevant) / float32(retrieved) - latency := float32(querying.Microseconds()) / float32(queries_size) - fmt.Println(recall, latency) - assert.True(t, recall > 0.9) - assert.True(t, latency < 100000) - } - } -} - -/* -10K -128 segments, 16 centroids -> 5.280255291s 0.90662 387.505 -64 segments, 256 centroids -> 6.585159916s 0.9326827 410.413 - -100K -128 segments, 16 centroids -> 1m17.634662125s 0.88258 692.081 -64 segments, 256 centroids -> 1m29.259369458s 0.92157 575.844 - -100000 -128 -Building the index took 47.7846745s -0.92627 664.66 - -{64, 64, 32, 256, 0}, - - {64, 64, 32, 1024, 1}, - {64, 64, 32, 4096, 1}, - {64, 64, 32, 16384, 1}, - {64, 64, 32, 65536, 1}, - -generating data took 2.072473792s -0 -Time to compress: 5m39.750884042s -Building the index took 16m30.632114542s -0.91401 747.82 -1 -Time to compress: 13m12.011102334s -Building the index took 28m12.879802125s -0.89564 1041.358 -2 -Time to compress: 58m15.252058416s -Building the index took 1h37m10.217039334s -0.90836 2299.629 -3 -Time to compress: 3h59m39.032524584s -Building the index took 5h54m55.046038916s -0.91295 4786.8 - -generating data took 2.119674416s -0 -Start compressing... -Time to compress: 1m3.037853584s -Building the index took 3m53.429316209s -0.40992 169.937 -1 -Start compressing... -Time to compress: 2m4.653952334s -Building the index took 5m44.454856667s -0.46251252 207.299 -2 -Start compressing... -Time to compress: 4m7.585857584s -Building the index took 8m36.2004675s -0.50494 293.362 -3 -Start compressing... -Time to compress: 8m16.4155035s -Building the index took 14m37.089003166s -0.54421 390.49 -4 -Start compressing... -Time to compress: 16m32.313318708s -Building the index took 26m46.66661125s -0.57827 442.589 -*/ -func TestHnswPqSift(t *testing.T) { - defer func(path string) { - err := os.RemoveAll(path) - if err != nil { - fmt.Println(err) - } - }(rootPath) - params := [][]int{ - {64, 64, 32, 256, 3}, - {64, 64, 32, 512, 3}, - {64, 64, 32, 1024, 3}, - {64, 64, 32, 2048, 3}, - {64, 64, 32, 4096, 3}, - {64, 64, 32, 65536, 3}, - } - dimensions := 128 - vectors_size := 1000000 - queries_size := 1000 - switch_at := 200000 - fmt.Println("Sift1M PQ") - before := time.Now() - vectors, queries := testinghelpers.ReadVecs(vectors_size, queries_size, dimensions, "sift", "../diskAnn/testdata") - k := 100 - distancer := distancer.NewL2SquaredProvider() - truths := testinghelpers.BuildTruths(queries_size, vectors_size, queries, vectors, k, distanceWrapper(distancer), "../diskAnn/testdata") - fmt.Printf("generating data took %s\n", time.Since(before)) - for i := 0; i < len(params); i++ { - fmt.Println(i) - efConstruction := params[i][0] - ef := params[i][1] - maxNeighbors := params[i][2] - centroids := params[i][3] - segmentRate := params[i][4] - if centroids > switch_at { - fmt.Println("Increasing switch at...") - switch_at = 650000 - } - - uc := ent.UserConfig{ - MaxConnections: maxNeighbors, - EFConstruction: efConstruction, - EF: ef, - PQ: ent.PQConfig{ - Enabled: false, - Segments: dimensions / int(math.Pow(2, float64(segmentRate))), - Encoder: ent.PQEncoder{ - Type: ent.PQEncoderTypeTile, - Distribution: ent.PQEncoderDistributionLogNormal, - }, - }, - VectorCacheMaxObjects: 10e12, - } - index, _ := hnsw.New(hnsw.Config{ - RootPath: rootPath, - ID: "recallbenchmark", - MakeCommitLoggerThunk: hnsw.MakeNoopCommitLogger, - DistanceProvider: distancer, - VectorForIDThunk: func(ctx context.Context, id uint64) ([]float32, error) { - return vectors[int(id)], nil - }, - }, uc, newDummyStore(t)) - init := time.Now() - compressionhelpers.Concurrently(uint64(switch_at), func(_, id uint64, _ *sync.Mutex) { - index.Add(uint64(id), vectors[id]) - }) - before = time.Now() - fmt.Println("Start compressing...") - - cfg := ent.PQConfig{ - Enabled: true, - Segments: dimensions / int(math.Pow(2, float64(segmentRate))), - Centroids: centroids, - BitCompression: false, - Encoder: ent.PQEncoder{ - Type: ent.PQEncoderTypeKMeans, - Distribution: ent.PQEncoderDistributionLogNormal, - }, - } - - index.Compress(cfg) /*should have configuration.compressed = true*/ - fmt.Printf("Time to compress: %s", time.Since(before)) - fmt.Println() - compressionhelpers.Concurrently(uint64(vectors_size-switch_at), func(_, id uint64, _ *sync.Mutex) { - idx := switch_at + int(id) - - index.Add(uint64(idx), vectors[idx]) - }) - fmt.Printf("Building the index took %s\n", time.Since(init)) - - var relevant uint64 - var retrieved int - - var querying time.Duration = 0 - compressionhelpers.Concurrently(uint64(len(queries)), func(_, i uint64, _ *sync.Mutex) { - before = time.Now() - results, _, _ := index.SearchByVector(queries[i], k, nil) - querying += time.Since(before) - retrieved += k - relevant += testinghelpers.MatchesInLists(truths[i], results) - }) - - recall := float32(relevant) / float32(retrieved) - latency := float32(querying.Microseconds()) / float32(queries_size) - fmt.Println(recall, latency) - assert.True(t, recall > 0.9) - assert.True(t, latency < 100000) - } -} - -func TestHnswPqSiftDeletes(t *testing.T) { - defer func(path string) { - err := os.RemoveAll(path) - if err != nil { - fmt.Println(err) - } - }(rootPath) - params := [][]int{ - {64, 64, 32}, - } - dimensions := 128 - vectors_size := 10000 - queries_size := 1000 - switch_at := 2000 - fmt.Println("Sift1M PQ Deletes") - before := time.Now() - vectors, queries := testinghelpers.ReadVecs(vectors_size, queries_size, dimensions, "sift", "../diskAnn/testdata") - k := 100 - distancer := distancer.NewL2SquaredProvider() - truths := testinghelpers.BuildTruths(queries_size, vectors_size, queries, vectors, k, distanceWrapper(distancer), "../diskAnn/testdata") - fmt.Printf("generating data took %s\n", time.Since(before)) - for segmentRate := 0; segmentRate < 1; segmentRate++ { - fmt.Println(segmentRate) - fmt.Println() - for i := 0; i < len(params); i++ { - efConstruction := params[i][0] - ef := params[i][1] - maxNeighbors := params[i][2] - - uc := ent.UserConfig{ - MaxConnections: maxNeighbors, - EFConstruction: efConstruction, - EF: ef, - PQ: ent.PQConfig{ - Enabled: false, - Segments: dimensions / int(math.Pow(2, float64(segmentRate))), - Encoder: ent.PQEncoder{ - Type: "tile", - Distribution: "log-normal", - }, - }, - VectorCacheMaxObjects: 10e12, - } - index, _ := hnsw.New(hnsw.Config{ - RootPath: rootPath, - ID: "recallbenchmark", - MakeCommitLoggerThunk: hnsw.MakeNoopCommitLogger, - DistanceProvider: distancer, - VectorForIDThunk: func(ctx context.Context, id uint64) ([]float32, error) { - return vectors[int(id)], nil - }, - }, uc, newDummyStore(t)) - init := time.Now() - compressionhelpers.Concurrently(uint64(switch_at), func(_, id uint64, _ *sync.Mutex) { - index.Add(uint64(id), vectors[id]) - }) - before = time.Now() - uc.PQ.Enabled = true - index.UpdateUserConfig(uc, func() {}) /*should have configuration.compressed = true*/ - fmt.Printf("Time to compress: %s", time.Since(before)) - fmt.Println() - compressionhelpers.Concurrently(uint64(vectors_size-switch_at), func(_, id uint64, _ *sync.Mutex) { - idx := switch_at + int(id) - index.Add(uint64(idx), vectors[idx]) - }) - fmt.Printf("Building the index took %s\n", time.Since(init)) - var relevant uint64 - var retrieved int - - var querying time.Duration = 0 - compressionhelpers.Concurrently(uint64(len(queries)), func(_, i uint64, _ *sync.Mutex) { - before = time.Now() - results, _, _ := index.SearchByVector(queries[i], k, nil) - querying += time.Since(before) - retrieved += k - relevant += testinghelpers.MatchesInLists(truths[i], results) - }) - - recall := float32(relevant) / float32(retrieved) - latency := float32(querying.Microseconds()) / float32(queries_size) - fmt.Println(recall, latency) - assert.True(t, recall > 0.9) - assert.True(t, latency < 100000) - } - } -} - -func TestHnswPqDeepImage(t *testing.T) { - defer func(path string) { - err := os.RemoveAll(path) - if err != nil { - fmt.Println(err) - } - }(rootPath) - vectors_size := 9990000 - queries_size := 1000 - vectors := parseFromTxt("../diskAnn/testdata/deep-image/train.txt", vectors_size) - queries := parseFromTxt("../diskAnn/testdata/deep-image/test.txt", queries_size) - dimensions := 96 - - params := [][]int{ - {64, 64, 32}, - {128, 128, 64}, - {256, 256, 128}, - {512, 512, 256}, - } - switch_at := 1000000 - - fmt.Println("Sift1MPQKMeans 10K/10K") - before := time.Now() - k := 100 - distancer := distancer.NewL2SquaredProvider() - truths := testinghelpers.BuildTruths(queries_size, vectors_size, queries, vectors, k, distanceWrapper(distancer), "../diskAnn/testdata/deep-image") - fmt.Printf("generating data took %s\n", time.Since(before)) - for segmentRate := 1; segmentRate < 4; segmentRate++ { - fmt.Println(segmentRate) - fmt.Println() - for i := 0; i < len(params); i++ { - efConstruction := params[i][0] - ef := params[i][1] - maxNeighbors := params[i][2] - - uc := ent.UserConfig{ - MaxConnections: maxNeighbors, - EFConstruction: efConstruction, - EF: ef, - PQ: ent.PQConfig{ - Enabled: false, - Segments: dimensions / int(math.Pow(2, float64(segmentRate))), - Encoder: ent.PQEncoder{ - Type: ent.PQEncoderTypeKMeans, - Distribution: ent.PQEncoderDistributionNormal, - }, - }, - VectorCacheMaxObjects: 10e12, - } - index, _ := hnsw.New(hnsw.Config{ - RootPath: rootPath, - ID: "recallbenchmark", - MakeCommitLoggerThunk: hnsw.MakeNoopCommitLogger, - DistanceProvider: distancer, - VectorForIDThunk: func(ctx context.Context, id uint64) ([]float32, error) { - return vectors[int(id)], nil - }, - }, uc, newDummyStore(t)) - init := time.Now() - compressionhelpers.Concurrently(uint64(switch_at), func(_, id uint64, _ *sync.Mutex) { - index.Add(uint64(id), vectors[id]) - }) - before = time.Now() - uc.PQ.Enabled = true - index.UpdateUserConfig(uc, func() {}) - fmt.Printf("Time to compress: %s", time.Since(before)) - fmt.Println() - compressionhelpers.Concurrently(uint64(vectors_size-switch_at), func(_, id uint64, _ *sync.Mutex) { - idx := switch_at + int(id) - index.Add(uint64(idx), vectors[idx]) - }) - fmt.Printf("Building the index took %s\n", time.Since(init)) - var relevant uint64 - var retrieved int - - var querying time.Duration = 0 - compressionhelpers.Concurrently(uint64(len(queries)), func(_, i uint64, _ *sync.Mutex) { - before = time.Now() - results, _, _ := index.SearchByVector(queries[i], k, nil) - querying += time.Since(before) - retrieved += k - relevant += testinghelpers.MatchesInLists(truths[i], results) - }) - - recall := float32(relevant) / float32(retrieved) - latency := float32(querying.Microseconds()) / float32(queries_size) - fmt.Println(recall, latency) - assert.True(t, recall > 0.9) - assert.True(t, latency < 100000) - } - } -} - -func parseFromTxt(file string, size int) [][]float32 { - content, _ := ioutil.ReadFile(file) - strContent := string(content) - testArray := strings.Split(strContent, "\n") - test := make([][]float32, 0, len(testArray)) - for j := 0; j < size; j++ { - elementArray := strings.Split(testArray[j], " ") - test = append(test, make([]float32, len(elementArray))) - for i := range elementArray { - f, _ := strconv.ParseFloat(elementArray[i], 16) - test[j][i] = float32(f) - } - } - return test -} - -func newDummyStore(t *testing.T) *lsmkv.Store { - logger, _ := test.NewNullLogger() - storeDir := t.TempDir() - store, err := lsmkv.New(storeDir, storeDir, logger, nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop()) - require.Nil(t, err) - return store -} diff --git a/adapters/repos/db/vector/hnsw/compress_test.go b/adapters/repos/db/vector/hnsw/compress_test.go deleted file mode 100644 index 21f1aec0536fe4d04a8f69a1a578138ea5f21217..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/compress_test.go +++ /dev/null @@ -1,73 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestCompression_CalculateOptimalSegments(t *testing.T) { - h := &hnsw{} - - type testCase struct { - dimensions int - expectedSegments int - } - - for _, tc := range []testCase{ - { - dimensions: 2048, - expectedSegments: 256, - }, - { - dimensions: 1536, - expectedSegments: 256, - }, - { - dimensions: 768, - expectedSegments: 128, - }, - { - dimensions: 512, - expectedSegments: 128, - }, - { - dimensions: 256, - expectedSegments: 64, - }, - { - dimensions: 125, - expectedSegments: 125, - }, - { - dimensions: 64, - expectedSegments: 32, - }, - { - dimensions: 27, - expectedSegments: 27, - }, - { - dimensions: 19, - expectedSegments: 19, - }, - { - dimensions: 2, - expectedSegments: 1, - }, - } { - segments := h.calculateOptimalSegments(tc.dimensions) - assert.Equal(t, tc.expectedSegments, segments) - } -} diff --git a/adapters/repos/db/vector/hnsw/compression_tests/fixtures/restart-from-zero-segments/1234567 b/adapters/repos/db/vector/hnsw/compression_tests/fixtures/restart-from-zero-segments/1234567 deleted file mode 100644 index 27b458b533ee643f0e50a0134cf66d63384f27f1..0000000000000000000000000000000000000000 Binary files a/adapters/repos/db/vector/hnsw/compression_tests/fixtures/restart-from-zero-segments/1234567 and /dev/null differ diff --git a/adapters/repos/db/vector/hnsw/condensor.go b/adapters/repos/db/vector/hnsw/condensor.go deleted file mode 100644 index 13b11d7c37b993f0bf746039d4a53876a69e9b8e..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/condensor.go +++ /dev/null @@ -1,261 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "bufio" - "encoding/binary" - "fmt" - "math" - "os" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/adapters/repos/db/vector/compressionhelpers" - "github.com/weaviate/weaviate/entities/errorcompounder" -) - -type MemoryCondensor struct { - newLogFile *os.File - newLog *bufWriter - logger logrus.FieldLogger -} - -func (c *MemoryCondensor) Do(fileName string) error { - fd, err := os.Open(fileName) - if err != nil { - return errors.Wrap(err, "open commit log to be condensed") - } - defer fd.Close() - fdBuf := bufio.NewReaderSize(fd, 256*1024) - - res, _, err := NewDeserializer(c.logger).Do(fdBuf, nil, true) - if err != nil { - return errors.Wrap(err, "read commit log to be condensed") - } - - newLogFile, err := os.OpenFile(fmt.Sprintf("%s.condensed", fileName), - os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0o666) - if err != nil { - return errors.Wrap(err, "open new commit log file for writing") - } - - c.newLogFile = newLogFile - - c.newLog = NewWriterSize(c.newLogFile, 1*1024*1024) - - if res.Compressed { - if err := c.AddPQ(res.PQData); err != nil { - return fmt.Errorf("write pq data: %w", err) - } - } - - for _, node := range res.Nodes { - if node == nil { - // nil nodes occur when we've grown, but not inserted anything yet - continue - } - - if node.level > 0 { - // nodes are implicitly added when they are first linked, if the level is - // not zero we know this node was new. If the level is zero it doesn't - // matter if it gets added explicitly or implicitly - if err := c.AddNode(node); err != nil { - return errors.Wrapf(err, "write node %d to commit log", node.id) - } - } - - for level, links := range node.connections { - if res.ReplaceLinks(node.id, uint16(level)) { - if err := c.SetLinksAtLevel(node.id, level, links); err != nil { - return errors.Wrapf(err, - "write links for node %d at level %d to commit log", node.id, level) - } - } else { - if err := c.AddLinksAtLevel(node.id, uint16(level), links); err != nil { - return errors.Wrapf(err, - "write links for node %d at level %d to commit log", node.id, level) - } - } - } - } - - if res.EntrypointChanged { - if err := c.SetEntryPointWithMaxLayer(res.Entrypoint, - int(res.Level)); err != nil { - return errors.Wrap(err, "write entrypoint to commit log") - } - } - - for ts := range res.Tombstones { - if err := c.AddTombstone(ts); err != nil { - return errors.Wrapf(err, - "write tombstone for node %d to commit log", ts) - } - } - - if err := c.newLog.Flush(); err != nil { - return errors.Wrap(err, "close new commit log") - } - - if err := c.newLogFile.Close(); err != nil { - return errors.Wrap(err, "close new commit log") - } - - if err := os.Remove(fileName); err != nil { - return errors.Wrap(err, "cleanup old (uncondensed) commit log") - } - - return nil -} - -func (c *MemoryCondensor) writeUint64(w *bufWriter, in uint64) error { - toWrite := make([]byte, 8) - binary.LittleEndian.PutUint64(toWrite[0:8], in) - _, err := w.Write(toWrite) - if err != nil { - return err - } - - return nil -} - -func (c *MemoryCondensor) writeUint16(w *bufWriter, in uint16) error { - toWrite := make([]byte, 2) - binary.LittleEndian.PutUint16(toWrite[0:2], in) - _, err := w.Write(toWrite) - if err != nil { - return err - } - - return nil -} - -func (c *MemoryCondensor) writeCommitType(w *bufWriter, in HnswCommitType) error { - toWrite := make([]byte, 1) - toWrite[0] = byte(in) - _, err := w.Write(toWrite) - if err != nil { - return err - } - - return nil -} - -func (c *MemoryCondensor) writeUint64Slice(w *bufWriter, in []uint64) error { - for _, v := range in { - err := c.writeUint64(w, v) - if err != nil { - return err - } - } - - return nil -} - -// AddNode adds an empty node -func (c *MemoryCondensor) AddNode(node *vertex) error { - ec := &errorcompounder.ErrorCompounder{} - ec.Add(c.writeCommitType(c.newLog, AddNode)) - ec.Add(c.writeUint64(c.newLog, node.id)) - ec.Add(c.writeUint16(c.newLog, uint16(node.level))) - - return ec.ToError() -} - -func (c *MemoryCondensor) SetLinksAtLevel(nodeid uint64, level int, targets []uint64) error { - ec := &errorcompounder.ErrorCompounder{} - ec.Add(c.writeCommitType(c.newLog, ReplaceLinksAtLevel)) - ec.Add(c.writeUint64(c.newLog, nodeid)) - ec.Add(c.writeUint16(c.newLog, uint16(level))) - - targetLength := len(targets) - if targetLength > math.MaxUint16 { - // TODO: investigate why we get such massive connections - targetLength = math.MaxUint16 - c.logger.WithField("action", "condense_commit_log"). - WithField("original_length", len(targets)). - WithField("maximum_length", targetLength). - Warning("condensor length of connections would overflow uint16, cutting off") - } - ec.Add(c.writeUint16(c.newLog, uint16(targetLength))) - ec.Add(c.writeUint64Slice(c.newLog, targets[:targetLength])) - - return ec.ToError() -} - -func (c *MemoryCondensor) AddLinksAtLevel(nodeid uint64, level uint16, targets []uint64) error { - toWrite := make([]byte, 13+len(targets)*8) - toWrite[0] = byte(AddLinksAtLevel) - binary.LittleEndian.PutUint64(toWrite[1:9], nodeid) - binary.LittleEndian.PutUint16(toWrite[9:11], uint16(level)) - binary.LittleEndian.PutUint16(toWrite[11:13], uint16(len(targets))) - for i, target := range targets { - offsetStart := 13 + i*8 - offsetEnd := offsetStart + 8 - binary.LittleEndian.PutUint64(toWrite[offsetStart:offsetEnd], target) - } - _, err := c.newLog.Write(toWrite) - return err -} - -func (c *MemoryCondensor) AddLinkAtLevel(nodeid uint64, level uint16, target uint64) error { - ec := &errorcompounder.ErrorCompounder{} - ec.Add(c.writeCommitType(c.newLog, AddLinkAtLevel)) - ec.Add(c.writeUint64(c.newLog, nodeid)) - ec.Add(c.writeUint16(c.newLog, uint16(level))) - ec.Add(c.writeUint64(c.newLog, target)) - - return ec.ToError() -} - -func (c *MemoryCondensor) SetEntryPointWithMaxLayer(id uint64, level int) error { - ec := &errorcompounder.ErrorCompounder{} - ec.Add(c.writeCommitType(c.newLog, SetEntryPointMaxLevel)) - ec.Add(c.writeUint64(c.newLog, id)) - ec.Add(c.writeUint16(c.newLog, uint16(level))) - - return ec.ToError() -} - -func (c *MemoryCondensor) AddTombstone(nodeid uint64) error { - ec := &errorcompounder.ErrorCompounder{} - ec.Add(c.writeCommitType(c.newLog, AddTombstone)) - ec.Add(c.writeUint64(c.newLog, nodeid)) - - return ec.ToError() -} - -func (c *MemoryCondensor) AddPQ(data compressionhelpers.PQData) error { - toWrite := make([]byte, 10) - toWrite[0] = byte(AddPQ) - binary.LittleEndian.PutUint16(toWrite[1:3], data.Dimensions) - toWrite[3] = byte(data.EncoderType) - binary.LittleEndian.PutUint16(toWrite[4:6], data.Ks) - binary.LittleEndian.PutUint16(toWrite[6:8], data.M) - toWrite[8] = data.EncoderDistribution - if data.UseBitsEncoding { - toWrite[9] = 1 - } else { - toWrite[9] = 0 - } - - for _, encoder := range data.Encoders { - toWrite = append(toWrite, encoder.ExposeDataForRestore()...) - } - _, err := c.newLog.Write(toWrite) - return err -} - -func NewMemoryCondensor(logger logrus.FieldLogger) *MemoryCondensor { - return &MemoryCondensor{logger: logger} -} diff --git a/adapters/repos/db/vector/hnsw/condensor_integration_test.go b/adapters/repos/db/vector/hnsw/condensor_integration_test.go deleted file mode 100644 index 7f0fc02793da8f1972b7b068e4b329fb5c32938a..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/condensor_integration_test.go +++ /dev/null @@ -1,613 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package hnsw - -import ( - "bufio" - "context" - "os" - "strings" - "testing" - - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/adapters/repos/db/vector/compressionhelpers" - "github.com/weaviate/weaviate/entities/cyclemanager" -) - -func TestCondensor(t *testing.T) { - rootPath := t.TempDir() - ctx := context.Background() - - logger, _ := test.NewNullLogger() - uncondensed, err := NewCommitLogger(rootPath, "uncondensed", logger, - cyclemanager.NewCallbackGroupNoop()) - require.Nil(t, err) - defer uncondensed.Shutdown(ctx) - - perfect, err := NewCommitLogger(rootPath, "perfect", logger, - cyclemanager.NewCallbackGroupNoop()) - require.Nil(t, err) - defer perfect.Shutdown(ctx) - - t.Run("add redundant data to the original log", func(t *testing.T) { - uncondensed.AddNode(&vertex{id: 0, level: 3}) - uncondensed.AddNode(&vertex{id: 1, level: 3}) - uncondensed.AddNode(&vertex{id: 2, level: 3}) - uncondensed.AddNode(&vertex{id: 3, level: 3}) - - // below are some pointless connection replacements, we expect that most of - // these will be gone after condensing, this gives us a good way of testing - // whether they're really gone - for level := 0; level <= 3; level++ { - uncondensed.ReplaceLinksAtLevel(0, level, []uint64{1, 2, 3}) - uncondensed.ReplaceLinksAtLevel(0, level, []uint64{1, 2}) - uncondensed.ReplaceLinksAtLevel(0, level, []uint64{1}) - uncondensed.ReplaceLinksAtLevel(0, level, []uint64{2}) - uncondensed.ReplaceLinksAtLevel(0, level, []uint64{3}) - uncondensed.ReplaceLinksAtLevel(0, level, []uint64{2, 3}) - uncondensed.ReplaceLinksAtLevel(0, level, []uint64{1, 2, 3}) - uncondensed.ReplaceLinksAtLevel(1, level, []uint64{0, 2, 3}) - uncondensed.ReplaceLinksAtLevel(1, level, []uint64{0, 2}) - uncondensed.ReplaceLinksAtLevel(1, level, []uint64{0}) - uncondensed.ReplaceLinksAtLevel(1, level, []uint64{2}) - uncondensed.ReplaceLinksAtLevel(1, level, []uint64{3}) - uncondensed.ReplaceLinksAtLevel(1, level, []uint64{2, 3}) - uncondensed.ReplaceLinksAtLevel(1, level, []uint64{0, 2, 3}) - uncondensed.ReplaceLinksAtLevel(2, level, []uint64{0, 1, 3}) - uncondensed.ReplaceLinksAtLevel(2, level, []uint64{0, 1}) - uncondensed.ReplaceLinksAtLevel(2, level, []uint64{0}) - uncondensed.ReplaceLinksAtLevel(2, level, []uint64{1}) - uncondensed.ReplaceLinksAtLevel(2, level, []uint64{3}) - uncondensed.ReplaceLinksAtLevel(2, level, []uint64{1, 3}) - uncondensed.ReplaceLinksAtLevel(2, level, []uint64{0, 1, 3}) - uncondensed.ReplaceLinksAtLevel(3, level, []uint64{0, 1, 2}) - uncondensed.ReplaceLinksAtLevel(3, level, []uint64{0, 1}) - uncondensed.ReplaceLinksAtLevel(3, level, []uint64{0}) - uncondensed.ReplaceLinksAtLevel(3, level, []uint64{1}) - uncondensed.ReplaceLinksAtLevel(3, level, []uint64{2}) - uncondensed.ReplaceLinksAtLevel(3, level, []uint64{1, 2}) - uncondensed.ReplaceLinksAtLevel(3, level, []uint64{0, 1, 2}) - } - uncondensed.SetEntryPointWithMaxLayer(3, 3) - uncondensed.AddTombstone(2) - - require.Nil(t, uncondensed.Flush()) - }) - - t.Run("create a hypothetical perfect log", func(t *testing.T) { - perfect.AddNode(&vertex{id: 0, level: 3}) - perfect.AddNode(&vertex{id: 1, level: 3}) - perfect.AddNode(&vertex{id: 2, level: 3}) - perfect.AddNode(&vertex{id: 3, level: 3}) - - // below are some pointless connection replacements, we expect that most of - // these will be gone after condensing, this gives us a good way of testing - // whether they're really gone - for level := 0; level <= 3; level++ { - perfect.ReplaceLinksAtLevel(0, level, []uint64{1, 2, 3}) - perfect.ReplaceLinksAtLevel(1, level, []uint64{0, 2, 3}) - perfect.ReplaceLinksAtLevel(2, level, []uint64{0, 1, 3}) - perfect.ReplaceLinksAtLevel(3, level, []uint64{0, 1, 2}) - } - perfect.SetEntryPointWithMaxLayer(3, 3) - perfect.AddTombstone(2) - - require.Nil(t, perfect.Flush()) - }) - - t.Run("condense the original and verify against the perfect one", func(t *testing.T) { - input, ok, err := getCurrentCommitLogFileName(commitLogDirectory(rootPath, "uncondensed")) - require.Nil(t, err) - require.True(t, ok) - - err = NewMemoryCondensor(logger).Do(commitLogFileName(rootPath, "uncondensed", input)) - require.Nil(t, err) - - control, ok, err := getCurrentCommitLogFileName( - commitLogDirectory(rootPath, "perfect")) - require.Nil(t, err) - require.True(t, ok) - - actual, ok, err := getCurrentCommitLogFileName( - commitLogDirectory(rootPath, "uncondensed")) - require.Nil(t, err) - require.True(t, ok) - - assert.True(t, strings.HasSuffix(actual, ".condensed"), - "commit log is now saved as condensed") - - controlStat, err := os.Stat(commitLogFileName(rootPath, "perfect", control)) - require.Nil(t, err) - - actualStat, err := os.Stat(commitLogFileName(rootPath, "uncondensed", actual)) - require.Nil(t, err) - - assert.Equal(t, controlStat.Size(), actualStat.Size()) - - // dumpIndexFromCommitLog(t, commitLogFileName(rootPath, "uncondensed", actual)) - // dumpIndexFromCommitLog(t, commitLogFileName(rootPath, "perfect", control)) - }) -} - -func TestCondensorAppendNodeLinks(t *testing.T) { - rootPath := t.TempDir() - ctx := context.Background() - - logger, _ := test.NewNullLogger() - uncondensed1, err := NewCommitLogger(rootPath, "uncondensed1", logger, - cyclemanager.NewCallbackGroupNoop()) - require.Nil(t, err) - defer uncondensed1.Shutdown(ctx) - - uncondensed2, err := NewCommitLogger(rootPath, "uncondensed2", logger, - cyclemanager.NewCallbackGroupNoop()) - require.Nil(t, err) - defer uncondensed2.Shutdown(ctx) - - control, err := NewCommitLogger(rootPath, "control", logger, - cyclemanager.NewCallbackGroupNoop()) - require.Nil(t, err) - defer control.Shutdown(ctx) - - t.Run("add data to the first log", func(t *testing.T) { - uncondensed1.AddLinkAtLevel(0, 0, 1) - uncondensed1.AddLinkAtLevel(0, 0, 2) - uncondensed1.AddLinkAtLevel(0, 0, 3) - - require.Nil(t, uncondensed1.Flush()) - }) - - t.Run("append data to the second log", func(t *testing.T) { - uncondensed2.AddLinkAtLevel(0, 0, 4) - uncondensed2.AddLinkAtLevel(0, 0, 5) - uncondensed2.AddLinkAtLevel(0, 0, 6) - - require.Nil(t, uncondensed2.Flush()) - }) - - t.Run("create a control log", func(t *testing.T) { - control.AddNode(&vertex{id: 0, level: 0}) - control.ReplaceLinksAtLevel(0, 0, []uint64{1, 2, 3, 4, 5, 6}) - - require.Nil(t, control.Flush()) - }) - - t.Run("condense both logs and verify the contents against the control", func(t *testing.T) { - input, ok, err := getCurrentCommitLogFileName(commitLogDirectory(rootPath, "uncondensed1")) - require.Nil(t, err) - require.True(t, ok) - - err = NewMemoryCondensor(logger).Do(commitLogFileName(rootPath, "uncondensed1", input)) - require.Nil(t, err) - - input, ok, err = getCurrentCommitLogFileName(commitLogDirectory(rootPath, "uncondensed2")) - require.Nil(t, err) - require.True(t, ok) - - err = NewMemoryCondensor(logger).Do(commitLogFileName(rootPath, "uncondensed2", input)) - require.Nil(t, err) - - control, ok, err := getCurrentCommitLogFileName( - commitLogDirectory(rootPath, "control")) - require.Nil(t, err) - require.True(t, ok) - - condensed1, ok, err := getCurrentCommitLogFileName( - commitLogDirectory(rootPath, "uncondensed1")) - require.Nil(t, err) - require.True(t, ok) - - condensed2, ok, err := getCurrentCommitLogFileName( - commitLogDirectory(rootPath, "uncondensed2")) - require.Nil(t, err) - require.True(t, ok) - - assert.True(t, strings.HasSuffix(condensed1, ".condensed"), - "commit log is now saved as condensed") - assert.True(t, strings.HasSuffix(condensed2, ".condensed"), - "commit log is now saved as condensed") - - assertIndicesFromCommitLogsMatch(t, commitLogFileName(rootPath, "control", control), - []string{ - commitLogFileName(rootPath, "uncondensed1", condensed1), - commitLogFileName(rootPath, "uncondensed2", condensed2), - }) - }) -} - -// This test was added as part of -// https://github.com/weaviate/weaviate/issues/1868 to rule out that -// replace links broken across two independent commit logs. It turned out that -// this was green and not the cause for the bug. The bug could be reproduced -// with the new test added in index_too_many_links_bug_integration_test.go. -// Nevertheless it makes sense to keep this test around as this might have been -// a potential cause as well and by having this test, we can prevent a -// regression. -func TestCondensorReplaceNodeLinks(t *testing.T) { - rootPath := t.TempDir() - ctx := context.Background() - - logger, _ := test.NewNullLogger() - uncondensed1, err := NewCommitLogger(rootPath, "uncondensed1", logger, - cyclemanager.NewCallbackGroupNoop()) - require.Nil(t, err) - defer uncondensed1.Shutdown(ctx) - - uncondensed2, err := NewCommitLogger(rootPath, "uncondensed2", logger, - cyclemanager.NewCallbackGroupNoop()) - require.Nil(t, err) - defer uncondensed2.Shutdown(ctx) - - control, err := NewCommitLogger(rootPath, "control", logger, - cyclemanager.NewCallbackGroupNoop()) - require.Nil(t, err) - defer control.Shutdown(ctx) - - t.Run("add data to the first log", func(t *testing.T) { - uncondensed1.AddNode(&vertex{id: 0, level: 1}) - uncondensed1.AddLinkAtLevel(0, 0, 1) - uncondensed1.AddLinkAtLevel(0, 0, 2) - uncondensed1.AddLinkAtLevel(0, 0, 3) - uncondensed1.AddLinkAtLevel(0, 1, 1) - uncondensed1.AddLinkAtLevel(0, 1, 2) - - require.Nil(t, uncondensed1.Flush()) - }) - - t.Run("replace all data from previous log", func(t *testing.T) { - uncondensed2.AddLinkAtLevel(0, 0, 10) - uncondensed2.ReplaceLinksAtLevel(0, 0, []uint64{4, 5, 6}) - uncondensed2.AddLinkAtLevel(0, 0, 7) - uncondensed2.ReplaceLinksAtLevel(0, 1, []uint64{8}) - - require.Nil(t, uncondensed2.Flush()) - }) - - t.Run("create a control log", func(t *testing.T) { - control.AddNode(&vertex{id: 0, level: 1}) - control.ReplaceLinksAtLevel(0, 0, []uint64{4, 5, 6, 7}) - control.ReplaceLinksAtLevel(0, 1, []uint64{8}) - - require.Nil(t, control.Flush()) - }) - - t.Run("condense both logs and verify the contents against the control", func(t *testing.T) { - input, ok, err := getCurrentCommitLogFileName(commitLogDirectory(rootPath, "uncondensed1")) - require.Nil(t, err) - require.True(t, ok) - - err = NewMemoryCondensor(logger).Do(commitLogFileName(rootPath, "uncondensed1", input)) - require.Nil(t, err) - - input, ok, err = getCurrentCommitLogFileName(commitLogDirectory(rootPath, "uncondensed2")) - require.Nil(t, err) - require.True(t, ok) - - err = NewMemoryCondensor(logger).Do(commitLogFileName(rootPath, "uncondensed2", input)) - require.Nil(t, err) - - control, ok, err := getCurrentCommitLogFileName( - commitLogDirectory(rootPath, "control")) - require.Nil(t, err) - require.True(t, ok) - - condensed1, ok, err := getCurrentCommitLogFileName( - commitLogDirectory(rootPath, "uncondensed1")) - require.Nil(t, err) - require.True(t, ok) - - condensed2, ok, err := getCurrentCommitLogFileName( - commitLogDirectory(rootPath, "uncondensed2")) - require.Nil(t, err) - require.True(t, ok) - - assert.True(t, strings.HasSuffix(condensed1, ".condensed"), - "commit log is now saved as condensed") - assert.True(t, strings.HasSuffix(condensed2, ".condensed"), - "commit log is now saved as condensed") - - assertIndicesFromCommitLogsMatch(t, commitLogFileName(rootPath, "control", control), - []string{ - commitLogFileName(rootPath, "uncondensed1", condensed1), - commitLogFileName(rootPath, "uncondensed2", condensed2), - }) - }) -} - -// This test was added as part of the investigation and fixing of -// https://github.com/weaviate/weaviate/issues/1868. We used the new -// (higher level) test in index_too_many_links_bug_integration_test.go to -// reproduce the problem without knowing what causes it. Eventually we came to -// the conclusion that "ClearLinksAtLevel" was not propagated correctly across -// two independently condensed commit logs. While the higher-level test already -// makes sure that the bug is gone and prevents regressions, this test was -// still added to test the broken (now fixed) behavior in relative isolation. -func TestCondensorClearLinksAtLevel(t *testing.T) { - rootPath := t.TempDir() - ctx := context.Background() - - logger, _ := test.NewNullLogger() - uncondensed1, err := NewCommitLogger(rootPath, "uncondensed1", logger, - cyclemanager.NewCallbackGroupNoop()) - require.Nil(t, err) - defer uncondensed1.Shutdown(ctx) - - uncondensed2, err := NewCommitLogger(rootPath, "uncondensed2", logger, - cyclemanager.NewCallbackGroupNoop()) - require.Nil(t, err) - defer uncondensed2.Shutdown(ctx) - - control, err := NewCommitLogger(rootPath, "control", logger, - cyclemanager.NewCallbackGroupNoop()) - require.Nil(t, err) - defer control.Shutdown(ctx) - - t.Run("add data to the first log", func(t *testing.T) { - uncondensed1.AddNode(&vertex{id: 0, level: 1}) - uncondensed1.AddLinkAtLevel(0, 0, 1) - uncondensed1.AddLinkAtLevel(0, 0, 2) - uncondensed1.AddLinkAtLevel(0, 0, 3) - uncondensed1.AddLinkAtLevel(0, 1, 1) - uncondensed1.AddLinkAtLevel(0, 1, 2) - - require.Nil(t, uncondensed1.Flush()) - }) - - t.Run("replace all data from previous log", func(t *testing.T) { - uncondensed2.AddLinkAtLevel(0, 0, 10) - uncondensed2.ClearLinksAtLevel(0, 0) - uncondensed2.AddLinkAtLevel(0, 0, 4) - uncondensed2.AddLinkAtLevel(0, 0, 5) - uncondensed2.AddLinkAtLevel(0, 0, 6) - uncondensed2.AddLinkAtLevel(0, 0, 7) - uncondensed2.ClearLinksAtLevel(0, 1) - uncondensed2.AddLinkAtLevel(0, 1, 8) - - require.Nil(t, uncondensed2.Flush()) - }) - - t.Run("create a control log", func(t *testing.T) { - control.AddNode(&vertex{id: 0, level: 1}) - control.ReplaceLinksAtLevel(0, 0, []uint64{4, 5, 6, 7}) - control.ReplaceLinksAtLevel(0, 1, []uint64{8}) - - require.Nil(t, control.Flush()) - }) - - t.Run("condense both logs and verify the contents against the control", func(t *testing.T) { - input, ok, err := getCurrentCommitLogFileName(commitLogDirectory(rootPath, "uncondensed1")) - require.Nil(t, err) - require.True(t, ok) - - err = NewMemoryCondensor(logger).Do(commitLogFileName(rootPath, "uncondensed1", input)) - require.Nil(t, err) - - input, ok, err = getCurrentCommitLogFileName(commitLogDirectory(rootPath, "uncondensed2")) - require.Nil(t, err) - require.True(t, ok) - - err = NewMemoryCondensor(logger).Do(commitLogFileName(rootPath, "uncondensed2", input)) - require.Nil(t, err) - - control, ok, err := getCurrentCommitLogFileName( - commitLogDirectory(rootPath, "control")) - require.Nil(t, err) - require.True(t, ok) - - condensed1, ok, err := getCurrentCommitLogFileName( - commitLogDirectory(rootPath, "uncondensed1")) - require.Nil(t, err) - require.True(t, ok) - - condensed2, ok, err := getCurrentCommitLogFileName( - commitLogDirectory(rootPath, "uncondensed2")) - require.Nil(t, err) - require.True(t, ok) - - assert.True(t, strings.HasSuffix(condensed1, ".condensed"), - "commit log is now saved as condensed") - assert.True(t, strings.HasSuffix(condensed2, ".condensed"), - "commit log is now saved as condensed") - - assertIndicesFromCommitLogsMatch(t, commitLogFileName(rootPath, "control", control), - []string{ - commitLogFileName(rootPath, "uncondensed1", condensed1), - commitLogFileName(rootPath, "uncondensed2", condensed2), - }) - }) -} - -func TestCondensorWithoutEntrypoint(t *testing.T) { - rootPath := t.TempDir() - ctx := context.Background() - - logger, _ := test.NewNullLogger() - uncondensed, err := NewCommitLogger(rootPath, "uncondensed", logger, - cyclemanager.NewCallbackGroupNoop()) - require.Nil(t, err) - defer uncondensed.Shutdown(ctx) - - t.Run("add data, but do not set an entrypoint", func(t *testing.T) { - uncondensed.AddNode(&vertex{id: 0, level: 3}) - - require.Nil(t, uncondensed.Flush()) - }) - - t.Run("condense the original and verify it doesn't overwrite the EP", func(t *testing.T) { - input, ok, err := getCurrentCommitLogFileName(commitLogDirectory(rootPath, "uncondensed")) - require.Nil(t, err) - require.True(t, ok) - - err = NewMemoryCondensor(logger).Do(commitLogFileName(rootPath, "uncondensed", input)) - require.Nil(t, err) - - actual, ok, err := getCurrentCommitLogFileName( - commitLogDirectory(rootPath, "uncondensed")) - require.Nil(t, err) - require.True(t, ok) - - assert.True(t, strings.HasSuffix(actual, ".condensed"), - "commit log is now saved as condensed") - - initialState := DeserializationResult{ - Nodes: nil, - Entrypoint: 17, - Level: 3, - } - fd, err := os.Open(commitLogFileName(rootPath, "uncondensed", actual)) - require.Nil(t, err) - - bufr := bufio.NewReader(fd) - res, _, err := NewDeserializer(logger).Do(bufr, &initialState, false) - require.Nil(t, err) - - assert.Contains(t, res.Nodes, &vertex{id: 0, level: 3, connections: make([][]uint64, 4)}) - assert.Equal(t, uint64(17), res.Entrypoint) - assert.Equal(t, uint16(3), res.Level) - }) -} - -func TestCondensorWithPQInformation(t *testing.T) { - rootPath := t.TempDir() - ctx := context.Background() - - logger, _ := test.NewNullLogger() - uncondensed, err := NewCommitLogger(rootPath, "uncondensed", logger, - cyclemanager.NewCallbackGroupNoop()) - require.Nil(t, err) - defer uncondensed.Shutdown(ctx) - - encoders := []compressionhelpers.PQEncoder{ - compressionhelpers.NewKMeansWithCenters( - 4, - 2, - 0, - [][]float32{{1, 2}, {3, 4}, {5, 6}, {7, 8}}, - ), - compressionhelpers.NewKMeansWithCenters( - 4, - 2, - 1, - [][]float32{{8, 7}, {6, 5}, {4, 3}, {2, 1}}, - ), - compressionhelpers.NewKMeansWithCenters( - 4, - 2, - 2, - [][]float32{{1, 2}, {3, 4}, {5, 6}, {7, 8}}, - ), - } - - t.Run("add pq info", func(t *testing.T) { - uncondensed.AddPQ(compressionhelpers.PQData{ - Ks: 4, - M: 3, - Dimensions: 6, - EncoderType: compressionhelpers.UseKMeansEncoder, - EncoderDistribution: uint8(0), - Encoders: encoders, - UseBitsEncoding: false, - }) - - require.Nil(t, uncondensed.Flush()) - }) - - t.Run("condense the original and verify the PQ info is present", func(t *testing.T) { - input, ok, err := getCurrentCommitLogFileName(commitLogDirectory(rootPath, "uncondensed")) - require.Nil(t, err) - require.True(t, ok) - - err = NewMemoryCondensor(logger).Do(commitLogFileName(rootPath, "uncondensed", input)) - require.Nil(t, err) - - actual, ok, err := getCurrentCommitLogFileName( - commitLogDirectory(rootPath, "uncondensed")) - require.Nil(t, err) - require.True(t, ok) - - assert.True(t, strings.HasSuffix(actual, ".condensed"), - "commit log is now saved as condensed") - - initialState := DeserializationResult{} - fd, err := os.Open(commitLogFileName(rootPath, "uncondensed", actual)) - require.Nil(t, err) - - bufr := bufio.NewReader(fd) - res, _, err := NewDeserializer(logger).Do(bufr, &initialState, false) - require.Nil(t, err) - - assert.True(t, res.Compressed) - expected := compressionhelpers.PQData{ - Ks: 4, - M: 3, - Dimensions: 6, - EncoderType: compressionhelpers.UseKMeansEncoder, - EncoderDistribution: uint8(0), - Encoders: encoders, - UseBitsEncoding: false, - } - - assert.Equal(t, expected, res.PQData) - }) -} - -func assertIndicesFromCommitLogsMatch(t *testing.T, fileNameControl string, - fileNames []string, -) { - control := readFromCommitLogs(t, fileNameControl) - actual := readFromCommitLogs(t, fileNames...) - - assert.Equal(t, control, actual) -} - -func readFromCommitLogs(t *testing.T, fileNames ...string) *hnsw { - var res *DeserializationResult - - for _, fileName := range fileNames { - fd, err := os.Open(fileName) - require.Nil(t, err) - - bufr := bufio.NewReader(fd) - logger, _ := test.NewNullLogger() - res, _, err = NewDeserializer(logger).Do(bufr, res, false) - require.Nil(t, err) - } - - return &hnsw{ - nodes: removeTrailingNilNodes(res.Nodes), - currentMaximumLayer: int(res.Level), - entryPointID: res.Entrypoint, - tombstones: res.Tombstones, - } -} - -// just a test helper to make the output easier to compare, remove all trailing -// nil nodes by starting from the last and stopping as soon as a node is not -// nil -func removeTrailingNilNodes(in []*vertex) []*vertex { - pos := len(in) - 1 - - for pos >= 0 { - if in[pos] != nil { - break - } - - pos-- - } - - return in[:pos+1] -} diff --git a/adapters/repos/db/vector/hnsw/condensor_mmap.go b/adapters/repos/db/vector/hnsw/condensor_mmap.go deleted file mode 100644 index 2c652bf9971e2f76c7f000811dd7cb945a4f0c4f..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/condensor_mmap.go +++ /dev/null @@ -1,89 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "io" - "os" - - "github.com/davecgh/go-spew/spew" - "github.com/pkg/errors" -) - -type MmapCondensor struct { - connectionsPerLevel int -} - -func NewMmapCondensor(connectionsPerLevel int) *MmapCondensor { - return &MmapCondensor{connectionsPerLevel: connectionsPerLevel} -} - -func (c *MmapCondensor) Do(fileName string) error { - fd, err := os.Open(fileName) - if err != nil { - return errors.Wrap(err, "open commit log to be condensed") - } - defer fd.Close() - - index, err := c.analyze(fd) - if err != nil { - return errors.Wrap(err, "analyze commit log and build index") - } - - index.calculateOffsets() - - // "rewind" file so we can read it again, this time into the mmap file - if _, err := fd.Seek(0, io.SeekStart); err != nil { - return errors.Wrap(err, "rewind uncondensed") - } - - if err := c.read(fd, index, fileName+".scratch.tmp"); err != nil { - return errors.Wrap(err, "read uncondensed into mmap file") - } - - spew.Dump(index) - spew.Dump(index.Size()) - return nil -} - -func (c *MmapCondensor) analyze(file *os.File) (mmapIndex, error) { - return newMmapCondensorAnalyzer(c.connectionsPerLevel).Do(file) -} - -func (c *MmapCondensor) read(source *os.File, index mmapIndex, - targetName string, -) error { - return newMmapCondensorReader().Do(source, index, targetName) -} - -func (mi *mmapIndex) calculateOffsets() { - for i := range mi.nodes { - if i == 0 { - // offset for the first element is 0, nothing to do - continue - } - - // we now have the guarantee that elem i-1 exists - mi.nodes[i].offset = mi.nodes[i-1].offset + uint64(mi.nodes[i-1].Size(mi.connectionsPerLevel)) - } -} - -// Size can only return a useful result if offsets have been calculated prior -// to calling Size() -func (mi *mmapIndex) Size() int { - if len(mi.nodes) == 0 { - return -1 - } - - return int(mi.nodes[len(mi.nodes)-1].offset) + - mi.nodes[len(mi.nodes)-1].Size(mi.connectionsPerLevel) -} diff --git a/adapters/repos/db/vector/hnsw/condensor_mmap_analyzer.go b/adapters/repos/db/vector/hnsw/condensor_mmap_analyzer.go deleted file mode 100644 index ccfbcc5206ebe7b7e591166d14f6a4d1f7e96175..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/condensor_mmap_analyzer.go +++ /dev/null @@ -1,261 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "bufio" - "encoding/binary" - "io" - "os" - "sort" - - "github.com/pkg/errors" -) - -type mmapIndex struct { - nodes []mmapIndexNode - connectionsPerLevel int -} - -func (mi *mmapIndex) UpsertNodeMaxLevel(node uint64, level uint16) { - n := sort.Search(len(mi.nodes), func(a int) bool { - return mi.nodes[a].id >= node - }) - - if n < len(mi.nodes) && mi.nodes[n].id == node { - // update - if mi.nodes[n].maxLevel < level { - mi.nodes[n].maxLevel = level - } - } else { - // insert - - // See https://github.com/golang/go/wiki/SliceTricks#insert - mi.nodes = append(mi.nodes, mmapIndexNode{}) - copy(mi.nodes[n+1:], mi.nodes[n:]) - mi.nodes[n].id = node - mi.nodes[n].maxLevel = level - } -} - -func (mi *mmapIndex) DeleteNode(node uint64) { -} - -type mmapIndexNode struct { - id uint64 - offset uint64 - maxLevel uint16 -} - -func (n mmapIndexNode) Size(connectionsPerLevel int) int { - return int(n.maxLevel)*2 + // overhead for uint16 length indicators - connectionsPerLevel*int(n.maxLevel+1) // level 0 has 2x connections -} - -type MmapCondensorAnalyzer struct { - reader *bufio.Reader - connectionsPerLevel int - index mmapIndex -} - -func newMmapCondensorAnalyzer(connectionsPerLevel int) *MmapCondensorAnalyzer { - return &MmapCondensorAnalyzer{connectionsPerLevel: connectionsPerLevel} -} - -func (a *MmapCondensorAnalyzer) Do(file *os.File) (mmapIndex, error) { - a.reader = bufio.NewReaderSize(file, 1024*1024) - - a.index = mmapIndex{ - connectionsPerLevel: a.connectionsPerLevel, - nodes: make([]mmapIndexNode, 0, 10000), - } - - if err := a.loop(); err != nil { - return a.index, err - } - - return a.index, nil -} - -func (a *MmapCondensorAnalyzer) loop() error { - for { - ct, err := a.ReadCommitType(a.reader) - if err != nil { - if errors.Is(err, io.EOF) { - break - } - - return err - } - - switch ct { - case AddNode: - err = a.ReadNode(a.reader) - case SetEntryPointMaxLevel: - err = a.ReadEP(a.reader) - case AddLinkAtLevel: - err = a.ReadLink(a.reader) - case ReplaceLinksAtLevel: - err = a.ReadLinks(a.reader) - case AddTombstone: - err = a.ReadAddTombstone(a.reader) - case RemoveTombstone: - err = a.ReadRemoveTombstone(a.reader) - case ClearLinks: - err = a.ReadClearLinks(a.reader) - case DeleteNode: - err = a.ReadDeleteNode(a.reader) - case ResetIndex: - a.index.nodes = make([]mmapIndexNode, 0, 10000) - default: - err = errors.Errorf("unrecognized commit type %d", ct) - } - if err != nil { - // do not return nil, err, because the err could be a recoverable one - return err - } - } - - return nil -} - -func (a *MmapCondensorAnalyzer) ReadNode(r io.Reader) error { - id, err := a.readUint64(r) - if err != nil { - return err - } - - level, err := a.readUint16(r) - if err != nil { - return err - } - - a.index.UpsertNodeMaxLevel(id, level) - return nil -} - -func (a *MmapCondensorAnalyzer) ReadEP(r io.Reader) error { - // TODO: is this an issue because of bufio Read vs ReadFull? - _, err := io.CopyN(io.Discard, r, 10) - return err -} - -func (a *MmapCondensorAnalyzer) ReadLink(r io.Reader) error { - source, err := a.readUint64(r) - if err != nil { - return err - } - - level, err := a.readUint16(r) - if err != nil { - return err - } - - // TODO: is this an issue because of bufio Read vs ReadFull? - _, err = io.CopyN(io.Discard, r, 8) - if err != nil { - return err - } - a.index.UpsertNodeMaxLevel(source, level) - - return nil -} - -func (a *MmapCondensorAnalyzer) ReadLinks(r io.Reader) error { - source, err := a.readUint64(r) - if err != nil { - return err - } - - level, err := a.readUint16(r) - if err != nil { - return err - } - - length, err := a.readUint16(r) - if err != nil { - return err - } - - a.index.UpsertNodeMaxLevel(source, level) - - // TODO: is this an issue because of bufio Read vs ReadFull? - _, err = io.CopyN(io.Discard, r, 8*int64(length)) - if err != nil { - return err - } - - return nil -} - -func (a *MmapCondensorAnalyzer) ReadAddTombstone(r io.Reader) error { - // TODO: is this an issue because of bufio Read vs ReadFull? - _, err := io.CopyN(io.Discard, r, 8) - return err -} - -func (a *MmapCondensorAnalyzer) ReadRemoveTombstone(r io.Reader) error { - // TODO: is this an issue because of bufio Read vs ReadFull? - _, err := io.CopyN(io.Discard, r, 8) - return err -} - -func (a *MmapCondensorAnalyzer) ReadClearLinks(r io.Reader) error { - // TODO: is this an issue because of bufio Read vs ReadFull? - _, err := io.CopyN(io.Discard, r, 8) - return err -} - -func (a *MmapCondensorAnalyzer) ReadDeleteNode(r io.Reader) error { - id, err := a.readUint64(r) - if err != nil { - return err - } - - a.index.DeleteNode(id) - return nil -} - -func (a *MmapCondensorAnalyzer) readUint64(r io.Reader) (uint64, error) { - var value uint64 - tmpBuf := make([]byte, 8) - _, err := io.ReadFull(r, tmpBuf) - if err != nil { - return 0, errors.Wrap(err, "failed to read uint64") - } - - value = binary.LittleEndian.Uint64(tmpBuf) - - return value, nil -} - -func (a *MmapCondensorAnalyzer) readUint16(r io.Reader) (uint16, error) { - var value uint16 - tmpBuf := make([]byte, 2) - _, err := io.ReadFull(r, tmpBuf) - if err != nil { - return 0, errors.Wrap(err, "failed to read uint16") - } - - value = binary.LittleEndian.Uint16(tmpBuf) - - return value, nil -} - -func (a *MmapCondensorAnalyzer) ReadCommitType(r io.Reader) (HnswCommitType, error) { - tmpBuf := make([]byte, 1) - if _, err := io.ReadFull(r, tmpBuf); err != nil { - return 0, errors.Wrap(err, "failed to read commit type") - } - - return HnswCommitType(tmpBuf[0]), nil -} diff --git a/adapters/repos/db/vector/hnsw/condensor_mmap_integration_test.go b/adapters/repos/db/vector/hnsw/condensor_mmap_integration_test.go deleted file mode 100644 index b6777c19479c81ca1cd6d7cb04f15b4f26dfe37f..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/condensor_mmap_integration_test.go +++ /dev/null @@ -1,187 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "os" - "strings" - "testing" - - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/cyclemanager" -) - -func TestMmapCondensor(t *testing.T) { - t.Skip() // TODO - - rootPath := t.TempDir() - - logger, _ := test.NewNullLogger() - uncondensed, err := NewCommitLogger(rootPath, "uncondensed", logger, - cyclemanager.NewCallbackGroupNoop()) - require.Nil(t, err) - - perfect, err := NewCommitLogger(rootPath, "perfect", logger, - cyclemanager.NewCallbackGroupNoop()) - require.Nil(t, err) - - t.Run("add redundant data to the original log", func(t *testing.T) { - uncondensed.AddNode(&vertex{id: 0, level: 3}) - uncondensed.AddNode(&vertex{id: 1, level: 3}) - uncondensed.AddNode(&vertex{id: 2, level: 3}) - uncondensed.AddNode(&vertex{id: 3, level: 3}) - - // below are some pointless connection replacements, we expect that most of - // these will be gone after condensing, this gives us a good way of testing - // whether they're really gone - for level := 0; level <= 3; level++ { - uncondensed.ReplaceLinksAtLevel(0, level, []uint64{1, 2, 3}) - uncondensed.ReplaceLinksAtLevel(0, level, []uint64{1, 2}) - uncondensed.ReplaceLinksAtLevel(0, level, []uint64{1}) - uncondensed.ReplaceLinksAtLevel(0, level, []uint64{2}) - uncondensed.ReplaceLinksAtLevel(0, level, []uint64{3}) - uncondensed.ReplaceLinksAtLevel(0, level, []uint64{2, 3}) - uncondensed.ReplaceLinksAtLevel(0, level, []uint64{1, 2, 3}) - uncondensed.ReplaceLinksAtLevel(1, level, []uint64{0, 2, 3}) - uncondensed.ReplaceLinksAtLevel(1, level, []uint64{0, 2}) - uncondensed.ReplaceLinksAtLevel(1, level, []uint64{0}) - uncondensed.ReplaceLinksAtLevel(1, level, []uint64{2}) - uncondensed.ReplaceLinksAtLevel(1, level, []uint64{3}) - uncondensed.ReplaceLinksAtLevel(1, level, []uint64{2, 3}) - uncondensed.ReplaceLinksAtLevel(1, level, []uint64{0, 2, 3}) - uncondensed.ReplaceLinksAtLevel(2, level, []uint64{0, 1, 3}) - uncondensed.ReplaceLinksAtLevel(2, level, []uint64{0, 1}) - uncondensed.ReplaceLinksAtLevel(2, level, []uint64{0}) - uncondensed.ReplaceLinksAtLevel(2, level, []uint64{1}) - uncondensed.ReplaceLinksAtLevel(2, level, []uint64{3}) - uncondensed.ReplaceLinksAtLevel(2, level, []uint64{1, 3}) - uncondensed.ReplaceLinksAtLevel(2, level, []uint64{0, 1, 3}) - uncondensed.ReplaceLinksAtLevel(3, level, []uint64{0, 1, 2}) - uncondensed.ReplaceLinksAtLevel(3, level, []uint64{0, 1}) - uncondensed.ReplaceLinksAtLevel(3, level, []uint64{0}) - uncondensed.ReplaceLinksAtLevel(3, level, []uint64{1}) - uncondensed.ReplaceLinksAtLevel(3, level, []uint64{2}) - uncondensed.ReplaceLinksAtLevel(3, level, []uint64{1, 2}) - uncondensed.ReplaceLinksAtLevel(3, level, []uint64{0, 1, 2}) - } - uncondensed.SetEntryPointWithMaxLayer(3, 3) - uncondensed.AddTombstone(2) - - require.Nil(t, uncondensed.Flush()) - }) - - t.Run("create a hypothetical perfect log", func(t *testing.T) { - perfect.AddNode(&vertex{id: 0, level: 3}) - perfect.AddNode(&vertex{id: 1, level: 3}) - perfect.AddNode(&vertex{id: 2, level: 3}) - perfect.AddNode(&vertex{id: 3, level: 3}) - - // below are some pointless connection replacements, we expect that most of - // these will be gone after condensing, this gives us a good way of testing - // whether they're really gone - for level := 0; level <= 3; level++ { - perfect.ReplaceLinksAtLevel(0, level, []uint64{1, 2, 3}) - perfect.ReplaceLinksAtLevel(1, level, []uint64{0, 2, 3}) - perfect.ReplaceLinksAtLevel(2, level, []uint64{0, 1, 3}) - perfect.ReplaceLinksAtLevel(3, level, []uint64{0, 1, 2}) - } - perfect.SetEntryPointWithMaxLayer(3, 3) - perfect.AddTombstone(2) - - require.Nil(t, perfect.Flush()) - }) - - t.Run("condense the original and verify against the perfect one", func(t *testing.T) { - input, ok, err := getCurrentCommitLogFileName(commitLogDirectory(rootPath, "uncondensed")) - require.Nil(t, err) - require.True(t, ok) - - err = NewMmapCondensor(3).Do(commitLogFileName(rootPath, "uncondensed", input)) - require.Nil(t, err) - - control, ok, err := getCurrentCommitLogFileName( - commitLogDirectory(rootPath, "perfect")) - require.Nil(t, err) - require.True(t, ok) - - actual, ok, err := getCurrentCommitLogFileName( - commitLogDirectory(rootPath, "uncondensed")) - require.Nil(t, err) - require.True(t, ok) - - assert.True(t, strings.HasSuffix(actual, ".condensed"), - "commit log is now saved as condensed") - - controlStat, err := os.Stat(commitLogFileName(rootPath, "perfect", control)) - require.Nil(t, err) - - actualStat, err := os.Stat(commitLogFileName(rootPath, "uncondensed", actual)) - require.Nil(t, err) - - assert.Equal(t, controlStat.Size(), actualStat.Size()) - - // dumpIndexFromCommitLog(t, commitLogFileName(rootPath, "uncondensed", actual)) - // dumpIndexFromCommitLog(t, commitLogFileName(rootPath, "perfect", control)) - }) -} - -// func TestCondensorWithoutEntrypoint(t *testing.T) { -// rand.Seed(time.Now().UnixNano()) -// rootPath := t.TempDir() - -// logger, _ := test.NewNullLogger() -// uncondensed, err := NewCommitLogger(rootPath, "uncondensed", logger, -// cyclemanager.NewCallbackGroupNoop()) -// require.Nil(t, err) - -// t.Run("add data, but do not set an entrypoint", func(t *testing.T) { -// uncondensed.AddNode(&vertex{id: 0, level: 3}) - -// require.Nil(t, uncondensed.Flush()) -// }) - -// t.Run("condense the original and verify it doesn't overwrite the EP", func(t *testing.T) { -// input, ok, err := getCurrentCommitLogFileName(commitLogDirectory(rootPath, "uncondensed")) -// require.Nil(t, err) -// require.True(t, ok) - -// err = NewMemoryCondensor2(logger).Do(commitLogFileName(rootPath, "uncondensed", input)) -// require.Nil(t, err) - -// actual, ok, err := getCurrentCommitLogFileName( -// commitLogDirectory(rootPath, "uncondensed")) -// require.Nil(t, err) -// require.True(t, ok) - -// assert.True(t, strings.HasSuffix(actual, ".condensed"), -// "commit log is now saved as condensed") - -// initialState := DeserializationResult{ -// Nodes: nil, -// Entrypoint: 17, -// Level: 3, -// } -// fd, err := os.Open(commitLogFileName(rootPath, "uncondensed", actual)) -// require.Nil(t, err) - -// bufr := bufio.NewReader(fd) -// res, err := NewDeserializer(logger).Do(bufr, &initialState) -// require.Nil(t, err) - -// assert.Contains(t, res.Nodes, &vertex{id: 0, level: 3, connections: map[int][]uint64{}}) -// assert.Equal(t, uint64(17), res.Entrypoint) -// assert.Equal(t, uint16(3), res.Level) - -// }) -// } diff --git a/adapters/repos/db/vector/hnsw/condensor_mmap_reader.go b/adapters/repos/db/vector/hnsw/condensor_mmap_reader.go deleted file mode 100644 index 77aadf99431ca04e90033add4ad4b30f160a1a1d..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/condensor_mmap_reader.go +++ /dev/null @@ -1,71 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "bufio" - "os" - - "github.com/edsrzf/mmap-go" - "github.com/pkg/errors" -) - -type MmapCondensorReader struct { - reader *bufio.Reader - target []byte -} - -func newMmapCondensorReader() *MmapCondensorReader { - return &MmapCondensorReader{} -} - -func (r *MmapCondensorReader) Do(source *os.File, index mmapIndex, targetName string) error { - r.reader = bufio.NewReaderSize(source, 1024*1024) - - scratchFile, err := os.Create(targetName) - if err != nil { - return err - } - - size := index.Size() - if err := scratchFile.Truncate(int64(index.Size())); err != nil { - return errors.Wrap(err, "truncate scratch file to size") - } - - mmapSpace, err := mmap.MapRegion(scratchFile, size, mmap.COPY, 0, 0) - if err != nil { - return errors.Wrap(err, "mmap scratch file") - } - - r.target = mmapSpace - - if err := r.loop(); err != nil { - return err - } - - if err := mmapSpace.Unmap(); err != nil { - return errors.Wrap(err, "munmap scratch file") - } - - if err := scratchFile.Close(); err != nil { - return errors.Wrap(err, "close scratch file") - } - - return nil -} - -func (r *MmapCondensorReader) loop() error { - // TODO: iterate through commit log - // TODO: get offset for specific part - // TODO: write into target at correct position - return nil -} diff --git a/adapters/repos/db/vector/hnsw/condensor_test.go b/adapters/repos/db/vector/hnsw/condensor_test.go deleted file mode 100644 index 1aac927f205626d8e581f1b0a569800238f357b2..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/condensor_test.go +++ /dev/null @@ -1,68 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - _ "fmt" - "math/rand" - "testing" - - "github.com/sirupsen/logrus/hooks/test" -) - -func BenchmarkCondensor2NewUint64Write(b *testing.B) { - b.StopTimer() - logger, _ := test.NewNullLogger() - c := NewMemoryCondensor(logger) - c.newLog = NewWriterSize(c.newLogFile, 1*1024*1024) - b.StartTimer() - for i := 0; i < b.N; i++ { - c.writeUint64(c.newLog, rand.Uint64()) - } -} - -func BenchmarkCondensor2NewUint16Write(b *testing.B) { - b.StopTimer() - logger, _ := test.NewNullLogger() - c := NewMemoryCondensor(logger) - c.newLog = NewWriterSize(c.newLogFile, 1*1024*1024) - b.StartTimer() - for i := 0; i < b.N; i++ { - c.writeUint16(c.newLog, uint16(rand.Uint32())) - } -} - -func BenchmarkCondensor2WriteCommitType(b *testing.B) { - b.StopTimer() - logger, _ := test.NewNullLogger() - c := NewMemoryCondensor(logger) - c.newLog = NewWriterSize(c.newLogFile, 1*1024*1024) - b.StartTimer() - for i := 0; i < b.N; i++ { - c.writeCommitType(c.newLog, HnswCommitType(1)) - } -} - -func BenchmarkCondensor2WriteUint64Slice(b *testing.B) { - b.StopTimer() - logger, _ := test.NewNullLogger() - c := NewMemoryCondensor(logger) - c.newLog = NewWriterSize(c.newLogFile, 1*1024*1024) - testInts := make([]uint64, 100) - for i := 0; i < 100; i++ { - testInts[i] = rand.Uint64() - } - b.StartTimer() - for i := 0; i < b.N; i++ { - c.writeUint64Slice(c.newLog, testInts) - } -} diff --git a/adapters/repos/db/vector/hnsw/config.go b/adapters/repos/db/vector/hnsw/config.go deleted file mode 100644 index f0146d9cb8ae39d858d3157fb1c157709bf1eaf2..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/config.go +++ /dev/null @@ -1,65 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/adapters/repos/db/vector/common" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" - "github.com/weaviate/weaviate/entities/errorcompounder" - "github.com/weaviate/weaviate/usecases/monitoring" -) - -// Config for a new HSNW index, this contains information that is derived -// internally, e.g. by the shard. All User-settable config is specified in -// Config.UserConfig -type Config struct { - // internal - RootPath string - ID string - MakeCommitLoggerThunk MakeCommitLogger - VectorForIDThunk common.VectorForID[float32] - TempVectorForIDThunk common.TempVectorForID - Logger logrus.FieldLogger - DistanceProvider distancer.Provider - PrometheusMetrics *monitoring.PrometheusMetrics - - // metadata for monitoring - ShardName string - ClassName string -} - -func (c Config) Validate() error { - ec := &errorcompounder.ErrorCompounder{} - - if c.ID == "" { - ec.Addf("id cannot be empty") - } - - if c.RootPath == "" { - ec.Addf("rootPath cannot be empty") - } - - if c.MakeCommitLoggerThunk == nil { - ec.Addf("makeCommitLoggerThunk cannot be nil") - } - - if c.VectorForIDThunk == nil { - ec.Addf("vectorForIDThunk cannot be nil") - } - - if c.DistanceProvider == nil { - ec.Addf("distancerProvider cannot be nil") - } - - return ec.ToError() -} diff --git a/adapters/repos/db/vector/hnsw/config_test.go b/adapters/repos/db/vector/hnsw/config_test.go deleted file mode 100644 index 87fb088bf4f09205924bca70b765da216c8f1ab5..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/config_test.go +++ /dev/null @@ -1,85 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "context" - "testing" - - "github.com/pkg/errors" - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" -) - -func Test_ValidConfig(t *testing.T) { - err := validConfig().Validate() - assert.Nil(t, err) -} - -func Test_InValidConfig(t *testing.T) { - type test struct { - config func() Config - expectedErr error - } - - tests := []test{ - { - config: func() Config { - v := validConfig() - v.ID = "" - return v - }, - expectedErr: errors.Errorf("id cannot be empty"), - }, - { - config: func() Config { - v := validConfig() - v.RootPath = "" - return v - }, - expectedErr: errors.Errorf("rootPath cannot be empty"), - }, - { - config: func() Config { - v := validConfig() - v.MakeCommitLoggerThunk = nil - return v - }, - expectedErr: errors.Errorf("makeCommitLoggerThunk cannot be nil"), - }, - { - config: func() Config { - v := validConfig() - v.VectorForIDThunk = nil - return v - }, - expectedErr: errors.Errorf("vectorForIDThunk cannot be nil"), - }, - } - - for _, test := range tests { - t.Run(test.expectedErr.Error(), func(t *testing.T) { - err := test.config().Validate() - assert.Equal(t, test.expectedErr.Error(), err.Error()) - }) - } -} - -func validConfig() Config { - return Config{ - RootPath: "some path", - ID: "someid", - MakeCommitLoggerThunk: func() (CommitLogger, error) { return nil, nil }, - VectorForIDThunk: func(context.Context, uint64) ([]float32, error) { return nil, nil }, - DistanceProvider: distancer.NewCosineDistanceProvider(), - } -} diff --git a/adapters/repos/db/vector/hnsw/config_update.go b/adapters/repos/db/vector/hnsw/config_update.go deleted file mode 100644 index 1da194333719ef81a1798b7380da5289c5f6e31d..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/config_update.go +++ /dev/null @@ -1,153 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "os" - "sync/atomic" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/schema" - ent "github.com/weaviate/weaviate/entities/vectorindex/hnsw" - "github.com/weaviate/weaviate/usecases/config" -) - -func ValidateUserConfigUpdate(initial, updated schema.VectorIndexConfig) error { - initialParsed, ok := initial.(ent.UserConfig) - if !ok { - return errors.Errorf("initial is not UserConfig, but %T", initial) - } - - updatedParsed, ok := updated.(ent.UserConfig) - if !ok { - return errors.Errorf("updated is not UserConfig, but %T", updated) - } - - immutableFields := []immutableParameter{ - { - name: "efConstruction", - accessor: func(c ent.UserConfig) interface{} { return c.EFConstruction }, - }, - { - name: "maxConnections", - accessor: func(c ent.UserConfig) interface{} { return c.MaxConnections }, - }, - { - // NOTE: There isn't a technical reason for this to be immutable, it - // simply hasn't been implemented yet. It would require to stop the - // current timer and start a new one. Certainly possible, but let's see - // if anyone actually needs this before implementing it. - name: "cleanupIntervalSeconds", - accessor: func(c ent.UserConfig) interface{} { return c.CleanupIntervalSeconds }, - }, - { - name: "distance", - accessor: func(c ent.UserConfig) interface{} { return c.Distance }, - }, - } - - for _, u := range immutableFields { - if err := validateImmutableField(u, initialParsed, updatedParsed); err != nil { - return err - } - } - - return nil -} - -type immutableParameter struct { - accessor func(c ent.UserConfig) interface{} - name string -} - -func validateImmutableField(u immutableParameter, - previous, next ent.UserConfig, -) error { - oldField := u.accessor(previous) - newField := u.accessor(next) - if oldField != newField { - return errors.Errorf("%s is immutable: attempted change from \"%v\" to \"%v\"", - u.name, oldField, newField) - } - - return nil -} - -func (h *hnsw) UpdateUserConfig(updated schema.VectorIndexConfig, callback func()) error { - parsed, ok := updated.(ent.UserConfig) - if !ok { - callback() - return errors.Errorf("config is not UserConfig, but %T", updated) - } - - // Store automatically as a lock here would be very expensive, this value is - // read on every single user-facing search, which can be highly concurrent - atomic.StoreInt64(&h.ef, int64(parsed.EF)) - atomic.StoreInt64(&h.efMin, int64(parsed.DynamicEFMin)) - atomic.StoreInt64(&h.efMax, int64(parsed.DynamicEFMax)) - atomic.StoreInt64(&h.efFactor, int64(parsed.DynamicEFFactor)) - atomic.StoreInt64(&h.flatSearchCutoff, int64(parsed.FlatSearchCutoff)) - - if !parsed.PQ.Enabled && !parsed.BQ.Enabled { - callback() - return nil - } - - h.pqConfig = parsed.PQ - if asyncEnabled() { - callback() - return nil - } - - if !h.compressed.Load() { - // the compression will fire the callback once it's complete - return h.TurnOnCompression(callback) - } else { - h.compressor.SetCacheMaxSize(int64(parsed.VectorCacheMaxObjects)) - callback() - return nil - } -} - -func asyncEnabled() bool { - return config.Enabled(os.Getenv("ASYNC_INDEXING")) -} - -func (h *hnsw) TurnOnCompression(callback func()) error { - h.logger.WithField("action", "compress").Info("switching to compressed vectors") - - err := ent.ValidatePQConfig(h.pqConfig) - if err != nil { - callback() - return err - } - - go h.compressThenCallback(callback) - - return nil -} - -func (h *hnsw) compressThenCallback(callback func()) { - defer callback() - - uc := ent.UserConfig{ - PQ: h.pqConfig, - BQ: ent.BQConfig{ - Enabled: !h.pqConfig.Enabled, - }, - } - if err := h.compress(uc); err != nil { - h.logger.Error(err) - return - } - h.logger.WithField("action", "compress").Info("vector compression complete") -} diff --git a/adapters/repos/db/vector/hnsw/config_update_test.go b/adapters/repos/db/vector/hnsw/config_update_test.go deleted file mode 100644 index 0b78e939696b5e042e1553e7f66f9c3b1b5bb72a..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/config_update_test.go +++ /dev/null @@ -1,126 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "testing" - - "github.com/pkg/errors" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/schema" - ent "github.com/weaviate/weaviate/entities/vectorindex/hnsw" -) - -func TestUserConfigUpdates(t *testing.T) { - t.Run("various immutable and mutable fields", func(t *testing.T) { - type test struct { - name string - initial schema.VectorIndexConfig - update schema.VectorIndexConfig - expectedError error - } - - tests := []test{ - { - name: "attempting to change ef construction", - initial: ent.UserConfig{EFConstruction: 64}, - update: ent.UserConfig{EFConstruction: 128}, - expectedError: errors.Errorf( - "efConstruction is immutable: " + - "attempted change from \"64\" to \"128\""), - }, - { - name: "attempting to change ef construction", - initial: ent.UserConfig{MaxConnections: 10}, - update: ent.UserConfig{MaxConnections: 15}, - expectedError: errors.Errorf( - "maxConnections is immutable: " + - "attempted change from \"10\" to \"15\""), - }, - { - name: "attempting to change cleanup interval seconds", - initial: ent.UserConfig{CleanupIntervalSeconds: 60}, - update: ent.UserConfig{CleanupIntervalSeconds: 90}, - expectedError: errors.Errorf( - "cleanupIntervalSeconds is immutable: " + - "attempted change from \"60\" to \"90\""), - }, - { - name: "attempting to change distance", - initial: ent.UserConfig{Distance: "cosine"}, - update: ent.UserConfig{Distance: "l2-squared"}, - expectedError: errors.Errorf( - "distance is immutable: " + - "attempted change from \"cosine\" to \"l2-squared\""), - }, - { - name: "changing ef", - initial: ent.UserConfig{EF: 100}, - update: ent.UserConfig{EF: -1}, - expectedError: nil, - }, - { - name: "changing other mutable settings", - initial: ent.UserConfig{ - VectorCacheMaxObjects: 700, - FlatSearchCutoff: 800, - }, - update: ent.UserConfig{ - VectorCacheMaxObjects: 730, - FlatSearchCutoff: 830, - }, - expectedError: nil, - }, - { - name: "attempting to change dynamic ef settings", - initial: ent.UserConfig{ - DynamicEFMin: 100, - DynamicEFMax: 200, - DynamicEFFactor: 5, - }, - update: ent.UserConfig{ - DynamicEFMin: 101, - DynamicEFMax: 201, - DynamicEFFactor: 6, - }, - expectedError: nil, - }, - { - name: "setting bq compression on", - initial: ent.UserConfig{ - BQ: ent.BQConfig{ - Enabled: false, - }, - }, - update: ent.UserConfig{ - BQ: ent.BQConfig{ - Enabled: true, - }, - }, - expectedError: nil, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - err := ValidateUserConfigUpdate(test.initial, test.update) - if test.expectedError == nil { - assert.Nil(t, err) - } else { - require.NotNil(t, err, "update validation must error") - assert.Equal(t, test.expectedError.Error(), err.Error()) - } - }) - } - }) -} diff --git a/adapters/repos/db/vector/hnsw/corrupt_commit_logs_fixer.go b/adapters/repos/db/vector/hnsw/corrupt_commit_logs_fixer.go deleted file mode 100644 index 01d2d0169485b733cb860d5033e7bea475309979..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/corrupt_commit_logs_fixer.go +++ /dev/null @@ -1,82 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "os" - "strings" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" -) - -// CorruptCommitLogFixer helps identify potentially corrupt commit logs and -// tries to mitigate the problem -type CorruptCommitLogFixer struct { - logger logrus.FieldLogger -} - -func NewCorruptedCommitLogFixer(logger logrus.FieldLogger) *CorruptCommitLogFixer { - return &CorruptCommitLogFixer{ - logger: logger, - } -} - -// Do tries to delete files that could be corrupt and removes them from the -// returned list, indicating that the index should no longer try to read them -// -// A file is considered corrupt if it has the .condensed suffix - yet there is -// a file with the same name without that suffix. This would indicate that -// trying to condense the file has somehow failed or been interrupted, as a -// successful condensing would have been succeeded by the removal of the -// original file. We thus assume the file must be corrupted, and delete it, so -// that the original will be used instead. -func (fixer *CorruptCommitLogFixer) Do(fileNames []string) ([]string, error) { - out := make([]string, len(fileNames)) - - i := 0 - for _, fileName := range fileNames { - if !strings.HasSuffix(fileName, ".condensed") { - // has no suffix, so it can never be considered corrupt - out[i] = fileName - i++ - continue - } - - // this file has a suffix, check if one without the suffix exists as well - if !fixer.listContains(fileNames, strings.TrimSuffix(fileName, ".condensed")) { - // does not seem corrupt, proceed - out[i] = fileName - i++ - continue - } - - // we have found a corrupt file, delete it and do not append it to the list - if err := os.Remove(fileName); err != nil { - return out, errors.Wrapf(err, "delete corrupt commit log file %q", fileName) - } - } - - return out[:i], nil -} - -func (fixer *CorruptCommitLogFixer) listContains(haystack []string, - needle string, -) bool { - for _, hay := range haystack { - if hay == needle { - return true - } - } - - return false -} diff --git a/adapters/repos/db/vector/hnsw/datasets/neurips23/clustered_runbook.yaml b/adapters/repos/db/vector/hnsw/datasets/neurips23/clustered_runbook.yaml deleted file mode 100644 index 74a89401f670b6b573df381c732e8fb446852751..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/datasets/neurips23/clustered_runbook.yaml +++ /dev/null @@ -1,392 +0,0 @@ -random-xs-clustered: - max_pts: 10000 - 1: - operation: "insert" - start: 0 - end: 326 - 2: - operation: "search" - 3: - operation: "insert" - start: 326 - end: 596 - 4: - operation: "search" - 5: - operation: "insert" - start: 596 - end: 945 - 6: - operation: "search" - 7: - operation: "insert" - start: 945 - end: 1323 - 8: - operation: "search" - 9: - operation: "insert" - start: 1323 - end: 1623 - 10: - operation: "search" - 11: - operation: "insert" - start: 1623 - end: 1986 - 12: - operation: "search" - 13: - operation: "insert" - start: 1986 - end: 2199 - 14: - operation: "search" - 15: - operation: "insert" - start: 2199 - end: 2576 - 16: - operation: "search" - 17: - operation: "insert" - start: 2576 - end: 2921 - 18: - operation: "search" - 19: - operation: "insert" - start: 2921 - end: 3252 - 20: - operation: "search" - 21: - operation: "insert" - start: 3252 - end: 3530 - 22: - operation: "search" - 23: - operation: "insert" - start: 3530 - end: 3866 - 24: - operation: "search" - 25: - operation: "insert" - start: 3866 - end: 4150 - 26: - operation: "search" - 27: - operation: "insert" - start: 4150 - end: 4434 - 28: - operation: "search" - 29: - operation: "insert" - start: 4434 - end: 4707 - 30: - operation: "search" - 31: - operation: "insert" - start: 4707 - end: 5073 - 32: - operation: "search" - 33: - operation: "insert" - start: 5073 - end: 5404 - 34: - operation: "search" - 35: - operation: "insert" - start: 5404 - end: 5718 - 36: - operation: "search" - 37: - operation: "insert" - start: 5718 - end: 6072 - 38: - operation: "search" - 39: - operation: "insert" - start: 6072 - end: 6338 - 40: - operation: "search" - 41: - operation: "insert" - start: 6338 - end: 6613 - 42: - operation: "search" - 43: - operation: "insert" - start: 6613 - end: 6908 - 44: - operation: "search" - 45: - operation: "insert" - start: 6908 - end: 7115 - 46: - operation: "search" - 47: - operation: "insert" - start: 7115 - end: 7452 - 48: - operation: "search" - 49: - operation: "insert" - start: 7452 - end: 7717 - 50: - operation: "search" - 51: - operation: "insert" - start: 7717 - end: 8065 - 52: - operation: "search" - 53: - operation: "insert" - start: 8065 - end: 8313 - 54: - operation: "search" - 55: - operation: "insert" - start: 8313 - end: 8698 - 56: - operation: "search" - 57: - operation: "insert" - start: 8698 - end: 9011 - 58: - operation: "search" - 59: - operation: "insert" - start: 9011 - end: 9307 - 60: - operation: "search" - 61: - operation: "insert" - start: 9307 - end: 9651 - 62: - operation: "search" - 63: - operation: "insert" - start: 9651 - end: 10000 - 64: - operation: "search" - gt_url: "https://comp21storage.blob.core.windows.net/publiccontainer/comp23/clustered_data/random-xs-clustered/clustered_runboook.yaml" -msturing-10M-clustered: - max_pts: 10000000 - # On Azure D8lds v5, runtime: 14 minutes - # recall@10: 0.9247 for "R":64, "L":50, "insert_threads":16, "consolidate_threads":16 "Ls":100, "T":16 - 1: - operation: "insert" - start: 0 - end: 255771 - 2: - operation: "search" - 3: - operation: "insert" - start: 255771 - end: 491965 - 4: - operation: "search" - 5: - operation: "insert" - start: 491965 - end: 824781 - 6: - operation: "search" - 7: - operation: "insert" - start: 824781 - end: 1081209 - 8: - operation: "search" - 9: - operation: "insert" - start: 1081209 - end: 1568760 - 10: - operation: "search" - 11: - operation: "insert" - start: 1568760 - end: 1959174 - 12: - operation: "search" - 13: - operation: "insert" - start: 1959174 - end: 2404186 - 14: - operation: "search" - 15: - operation: "insert" - start: 2404186 - end: 2798660 - 16: - operation: "search" - 17: - operation: "insert" - start: 2798660 - end: 3082959 - 18: - operation: "search" - 19: - operation: "insert" - start: 3082959 - end: 3480554 - 20: - operation: "search" - 21: - operation: "insert" - start: 3480554 - end: 3910930 - 22: - operation: "search" - 23: - operation: "insert" - start: 3910930 - end: 4194870 - 24: - operation: "search" - 25: - operation: "insert" - start: 4194870 - end: 4652840 - 26: - operation: "search" - 27: - operation: "insert" - start: 4652840 - end: 4872616 - 28: - operation: "search" - 29: - operation: "insert" - start: 4872616 - end: 5184725 - 30: - operation: "search" - 31: - operation: "insert" - start: 5184725 - end: 5629098 - 32: - operation: "search" - 33: - operation: "insert" - start: 5629098 - end: 6023119 - 34: - operation: "search" - 35: - operation: "insert" - start: 6023119 - end: 6292969 - 36: - operation: "search" - 37: - operation: "insert" - start: 6292969 - end: 6508987 - 38: - operation: "search" - 39: - operation: "insert" - start: 6508987 - end: 6767675 - 40: - operation: "search" - 41: - operation: "insert" - start: 6767675 - end: 7000498 - 42: - operation: "search" - 43: - operation: "insert" - start: 7000498 - end: 7263856 - 44: - operation: "search" - 45: - operation: "insert" - start: 7263856 - end: 7485517 - 46: - operation: "search" - 47: - operation: "insert" - start: 7485517 - end: 7739934 - 48: - operation: "search" - 49: - operation: "insert" - start: 7739934 - end: 8055691 - 50: - operation: "search" - 51: - operation: "insert" - start: 8055691 - end: 8381008 - 52: - operation: "search" - 53: - operation: "insert" - start: 8381008 - end: 8750107 - 54: - operation: "search" - 55: - operation: "insert" - start: 8750107 - end: 8942969 - 56: - operation: "search" - 57: - operation: "insert" - start: 8942969 - end: 9223315 - 58: - operation: "search" - 59: - operation: "insert" - start: 9223315 - end: 9508781 - 60: - operation: "search" - 61: - operation: "insert" - start: 9508781 - end: 9722747 - 62: - operation: "search" - 63: - operation: "insert" - start: 9722747 - end: 10000000 - 64: - operation: "search" - gt_url: "https://comp21storage.blob.core.windows.net/publiccontainer/comp23/clustered_data/msturing-10M-clustered/clustered_runboook.yaml" diff --git a/adapters/repos/db/vector/hnsw/datasets/neurips23/simple_runbook.yaml b/adapters/repos/db/vector/hnsw/datasets/neurips23/simple_runbook.yaml deleted file mode 100644 index 37d8dee920821fce9947920bb2eb2534fe667bfd..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/datasets/neurips23/simple_runbook.yaml +++ /dev/null @@ -1,110 +0,0 @@ -random-xs: - max_pts: 10000 - 1: - operation: "insert" - start: 0 - end: 10000 - 2: - operation: "search" - 3: - operation: "delete" - start: 0 - end: 5000 - 4: - operation: "search" - 5: - operation: "insert" - start: 0 - end: 5000 - 6: - operation: "search" - gt_url: "https://comp21storage.blob.core.windows.net/publiccontainer/comp23/str_gt/random10000/10000/simple_runbook.yaml" -msturing-10M: - max_pts: 10000000 - # On Azure D8lds v5 with "R":50, "L":50, "insert_threads":16, "consolidate_threads":16 - # ~28 mins run time "Ls":100, "search_threads":16, average recall@10: 0.892 - 1: - operation: "insert" - start: 0 - end: 10000000 - 2: - operation: "search" - 3: - operation: "delete" - start: 0 - end: 5000000 - 4: - operation: "search" - 5: - operation: "insert" - start: 0 - end: 5000000 - 6: - operation: "search" - gt_url: "https://comp21storage.blob.core.windows.net/publiccontainer/comp23/str_gt/MSTuringANNS/10000000/simple_runbook.yaml" -msturing-1M: - max_pts: 1000000 - # On Azure D8lds v5 with "R":50, "L":50, "insert_threads":16, "consolidate_threads":16 - # ~3.5 mins run time "Ls":300, "search_threads":16, average recall@10: 0.906 - # ~2 mins run time "Ls":100, "search_threads":16, average recall@10: 0.958 - 1: - operation: "insert" - start: 0 - end: 1000000 - 2: - operation: "search" - 3: - operation: "delete" - start: 0 - end: 500000 - 4: - operation: "search" - 5: - operation: "insert" - start: 0 - end: 500000 - 6: - operation: "search" - gt_url: "https://comp21storage.blob.core.windows.net/publiccontainer/comp23/str_gt/MSTuringANNS/1000000/simple_runbook.yaml" -msspacev-10M: - max_pts: 10000000 - 1: - operation: "insert" - start: 0 - end: 10000000 - 2: - operation: "search" - 3: - operation: "delete" - start: 0 - end: 5000000 - 4: - operation: "search" - 5: - operation: "insert" - start: 0 - end: 5000000 - 6: - operation: "search" - gt_url: "https://comp21storage.blob.core.windows.net/publiccontainer/comp23/str_gt/MSSPACEV1B/10000000/simple_runbook.yaml" -msspacev-1M: - max_pts: 1000000 - 1: - operation: "insert" - start: 0 - end: 1000000 - 2: - operation: "search" - 3: - operation: "delete" - start: 0 - end: 500000 - 4: - operation: "search" - 5: - operation: "insert" - start: 0 - end: 500000 - 6: - operation: "search" - gt_url: "https://comp21storage.blob.core.windows.net/publiccontainer/comp23/str_gt/MSSPACEV1B/1000000/simple_runbook.yaml" \ No newline at end of file diff --git a/adapters/repos/db/vector/hnsw/debug.go b/adapters/repos/db/vector/hnsw/debug.go deleted file mode 100644 index d22593226806db570c7de6d94c92a6ba707a693d..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/debug.go +++ /dev/null @@ -1,214 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "encoding/json" - "fmt" - "strings" - - "github.com/weaviate/weaviate/adapters/repos/db/vector/common" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" - "github.com/weaviate/weaviate/entities/cyclemanager" - ent "github.com/weaviate/weaviate/entities/vectorindex/hnsw" -) - -// Dump to stdout for debugging purposes -func (h *hnsw) Dump(labels ...string) { - if len(labels) > 0 { - fmt.Printf("--------------------------------------------------\n") - fmt.Printf("-- %s\n", strings.Join(labels, ", ")) - } - fmt.Printf("--------------------------------------------------\n") - fmt.Printf("ID: %s\n", h.id) - fmt.Printf("Entrypoint: %d\n", h.entryPointID) - fmt.Printf("Max Level: %d\n", h.currentMaximumLayer) - fmt.Printf("Tombstones %v\n", h.tombstones) - fmt.Printf("\nNodes and Connections:\n") - for _, node := range h.nodes { - if node == nil { - continue - } - - fmt.Printf(" Node %d (level %d)\n", node.id, node.level) - for level, conns := range node.connections { - fmt.Printf(" Level %d: Connections: %v\n", level, conns) - } - } - - fmt.Printf("--------------------------------------------------\n") -} - -// DumpJSON to stdout for debugging purposes -func (h *hnsw) DumpJSON(labels ...string) { - dump := JSONDump{ - Labels: labels, - ID: h.id, - Entrypoint: h.entryPointID, - CurrentMaximumLayer: h.currentMaximumLayer, - Tombstones: h.tombstones, - } - for _, node := range h.nodes { - if node == nil { - continue - } - - dumpNode := JSONDumpNode{ - ID: node.id, - Level: node.level, - Connections: node.connections, - } - dump.Nodes = append(dump.Nodes, dumpNode) - } - - out, err := json.Marshal(dump) - if err != nil { - fmt.Println(err) - } - fmt.Printf("%s\n", string(out)) -} - -type JSONDump struct { - Labels []string `json:"labels"` - ID string `json:"id"` - Entrypoint uint64 `json:"entrypoint"` - CurrentMaximumLayer int `json:"currentMaximumLayer"` - Tombstones map[uint64]struct{} `json:"tombstones"` - Nodes []JSONDumpNode `json:"nodes"` -} - -type JSONDumpNode struct { - ID uint64 `json:"id"` - Level int `json:"level"` - Connections [][]uint64 `json:"connections"` -} - -type JSONDumpMap struct { - Labels []string `json:"labels"` - ID string `json:"id"` - Entrypoint uint64 `json:"entrypoint"` - CurrentMaximumLayer int `json:"currentMaximumLayer"` - Tombstones map[uint64]struct{} `json:"tombstones"` - Nodes []JSONDumpNodeMap `json:"nodes"` -} - -type JSONDumpNodeMap struct { - ID uint64 `json:"id"` - Level int `json:"level"` - Connections map[int][]uint64 `json:"connections"` -} - -func NewFromJSONDump(dumpBytes []byte, vecForID common.VectorForID[float32]) (*hnsw, error) { - var dump JSONDump - err := json.Unmarshal(dumpBytes, &dump) - if err != nil { - return nil, err - } - - index, err := New(Config{ - RootPath: "doesnt-matter-as-committlogger-is-mocked-out", - ID: dump.ID, - MakeCommitLoggerThunk: MakeNoopCommitLogger, - DistanceProvider: distancer.NewCosineDistanceProvider(), - VectorForIDThunk: vecForID, - }, ent.UserConfig{ - MaxConnections: 30, - EFConstruction: 128, - }, cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), nil) - if err != nil { - return nil, err - } - - index.currentMaximumLayer = dump.CurrentMaximumLayer - index.entryPointID = dump.Entrypoint - index.tombstones = dump.Tombstones - - for _, n := range dump.Nodes { - index.nodes[n.ID] = &vertex{ - id: n.ID, - level: n.Level, - connections: n.Connections, - } - } - - return index, nil -} - -func NewFromJSONDumpMap(dumpBytes []byte, vecForID common.VectorForID[float32]) (*hnsw, error) { - var dump JSONDumpMap - err := json.Unmarshal(dumpBytes, &dump) - if err != nil { - return nil, err - } - - index, err := New(Config{ - RootPath: "doesnt-matter-as-committlogger-is-mocked-out", - ID: dump.ID, - MakeCommitLoggerThunk: MakeNoopCommitLogger, - DistanceProvider: distancer.NewCosineDistanceProvider(), - VectorForIDThunk: vecForID, - }, ent.UserConfig{ - MaxConnections: 30, - EFConstruction: 128, - }, cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), nil) - if err != nil { - return nil, err - } - - index.currentMaximumLayer = dump.CurrentMaximumLayer - index.entryPointID = dump.Entrypoint - index.tombstones = dump.Tombstones - - for _, n := range dump.Nodes { - index.nodes[n.ID] = &vertex{ - id: n.ID, - level: n.Level, - connections: make([][]uint64, len(n.Connections)), - } - for level, conns := range n.Connections { - index.nodes[n.ID].connections[level] = conns - } - } - - return index, nil -} - -// was added as part of -// https://github.com/weaviate/weaviate/issues/1868 for debugging. It -// is not currently in use anywhere as it is somewhat costly, it would lock the -// entire graph and iterate over every node which would lead to disruptions in -// production. However, keeping this method around may be valuable for future -// investigations where the amount of links may be a problem. -func (h *hnsw) ValidateLinkIntegrity() { - h.RLock() - defer h.RUnlock() - - for i, node := range h.nodes { - if node == nil { - continue - } - - for level, conns := range node.connections { - m := h.maximumConnections - if level == 0 { - m = h.maximumConnectionsLayerZero - } - - if len(conns) > m { - h.logger.Warnf("node %d at level %d has %d connections", i, level, len(conns)) - } - - } - } - - h.logger.Infof("completed link integrity check") -} diff --git a/adapters/repos/db/vector/hnsw/delete.go b/adapters/repos/db/vector/hnsw/delete.go deleted file mode 100644 index 26612a2858ed633598c295062d788c7c01151aa1..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/delete.go +++ /dev/null @@ -1,643 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "context" - "fmt" - "runtime/debug" - "sync" - "time" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/adapters/repos/db/vector/cache" - "github.com/weaviate/weaviate/adapters/repos/db/vector/compressionhelpers" - "github.com/weaviate/weaviate/entities/cyclemanager" - "github.com/weaviate/weaviate/entities/storobj" -) - -type breakCleanUpTombstonedNodesFunc func() bool - -// Delete attaches a tombstone to an item so it can be periodically cleaned up -// later and the edges reassigned -func (h *hnsw) Delete(ids ...uint64) error { - h.compressActionLock.RLock() - defer h.compressActionLock.RUnlock() - - h.deleteVsInsertLock.Lock() - defer h.deleteVsInsertLock.Unlock() - - h.deleteLock.Lock() - defer h.deleteLock.Unlock() - - before := time.Now() - defer h.metrics.TrackDelete(before, "total") - - if err := h.addTombstone(ids...); err != nil { - return err - } - - for _, id := range ids { - h.metrics.DeleteVector() - - // Adding a tombstone might not be enough in some cases, if the tombstoned - // entry was the entrypoint this might lead to issues for following inserts: - // On a nearly empty graph the entrypoint might be the only viable element to - // connect to, however, because the entrypoint itself is tombstones - // connections to it are impossible. So, unless we find a new entrypoint, - // subsequent inserts might end up isolated (without edges) in the graph. - // This is especially true if the tombstoned entrypoint is the only node in - // the graph. In this case we must reset the graph, so it acts like an empty - // one. Otherwise we'd insert the next id and have only one possible node to - // connect it to (the entrypoint). With that one being tombstoned, the new - // node would be guaranteed to have zero edges - - node := h.nodeByID(id) - if node == nil { - // node was already deleted/cleaned up - continue - } - - if h.getEntrypoint() == id { - beforeDeleteEP := time.Now() - defer h.metrics.TrackDelete(beforeDeleteEP, "delete_entrypoint") - - denyList := h.tombstonesAsDenyList() - if onlyNode, err := h.resetIfOnlyNode(node, denyList); err != nil { - return errors.Wrap(err, "reset index") - } else if !onlyNode { - if err := h.deleteEntrypoint(node, denyList); err != nil { - return errors.Wrap(err, "delete entrypoint") - } - } - } - } - - return nil -} - -func (h *hnsw) resetIfEmpty() (empty bool, err error) { - h.resetLock.Lock() - defer h.resetLock.Unlock() - h.Lock() - defer h.Unlock() - - empty = func() bool { - h.shardedNodeLocks.RLock(h.entryPointID) - defer h.shardedNodeLocks.RUnlock(h.entryPointID) - - return h.isEmptyUnlocked() - }() - // It can happen that between calls of isEmptyUnlocked and resetUnlocked - // values of h.nodes will change (due to locks being RUnlocked and Locked again) - // This is acceptable in order to avoid long Locking of all striped locks - if empty { - h.shardedNodeLocks.LockAll() - defer h.shardedNodeLocks.UnlockAll() - - return true, h.resetUnlocked() - } - return false, nil -} - -func (h *hnsw) resetIfOnlyNode(needle *vertex, denyList helpers.AllowList) (onlyNode bool, err error) { - h.resetLock.Lock() - defer h.resetLock.Unlock() - h.Lock() - defer h.Unlock() - - onlyNode = func() bool { - h.shardedNodeLocks.RLockAll() - defer h.shardedNodeLocks.RUnlockAll() - - return h.isOnlyNodeUnlocked(needle, denyList) - }() - // It can happen that between calls of isOnlyNodeUnlocked and resetUnlocked - // values of h.nodes will change (due to locks being RUnlocked and Locked again) - // This is acceptable in order to avoid long Locking of all striped locks - if onlyNode { - h.shardedNodeLocks.LockAll() - defer h.shardedNodeLocks.UnlockAll() - - return true, h.resetUnlocked() - } - return false, nil -} - -func (h *hnsw) resetUnlocked() error { - h.resetCtxCancel() - resetCtx, resetCtxCancel := context.WithCancel(context.Background()) - h.resetCtx = resetCtx - h.resetCtxCancel = resetCtxCancel - - h.entryPointID = 0 - h.currentMaximumLayer = 0 - h.initialInsertOnce = &sync.Once{} - h.nodes = make([]*vertex, cache.InitialSize) - - return h.commitLog.Reset() -} - -func (h *hnsw) tombstonesAsDenyList() helpers.AllowList { - deleteList := helpers.NewAllowList() - h.tombstoneLock.Lock() - defer h.tombstoneLock.Unlock() - - tombstones := h.tombstones - for id := range tombstones { - deleteList.Insert(id) - } - - return deleteList -} - -func (h *hnsw) getEntrypoint() uint64 { - h.RLock() - defer h.RUnlock() - - return h.entryPointID -} - -func (h *hnsw) copyTombstonesToAllowList(breakCleanUpTombstonedNodes breakCleanUpTombstonedNodesFunc) (ok bool, deleteList helpers.AllowList) { - h.resetLock.Lock() - defer h.resetLock.Unlock() - - if breakCleanUpTombstonedNodes() { - return false, nil - } - - h.RLock() - lenOfNodes := uint64(len(h.nodes)) - h.RUnlock() - - h.tombstoneLock.Lock() - defer h.tombstoneLock.Unlock() - - deleteList = helpers.NewAllowList() - for id := range h.tombstones { - if lenOfNodes <= id { - // we're trying to delete an id outside the possible range, nothing to do - continue - } - - deleteList.Insert(id) - } - - if deleteList.IsEmpty() { - return false, nil - } - - return true, deleteList -} - -// CleanUpTombstonedNodes removes nodes with a tombstone and reassigns -// edges that were previously pointing to the tombstoned nodes -func (h *hnsw) CleanUpTombstonedNodes(shouldAbort cyclemanager.ShouldAbortCallback) error { - _, err := h.cleanUpTombstonedNodes(shouldAbort) - return err -} - -func (h *hnsw) cleanUpTombstonedNodes(shouldAbort cyclemanager.ShouldAbortCallback) (bool, error) { - defer func() { - err := recover() - if err != nil { - h.logger.WithField("panic", err).Errorf("class %s: tombstone cleanup panicked", h.className) - debug.PrintStack() - } - }() - - h.metrics.StartCleanup(1) - defer h.metrics.EndCleanup(1) - - h.resetLock.Lock() - resetCtx := h.resetCtx - h.resetLock.Unlock() - - breakCleanUpTombstonedNodes := func() bool { - return resetCtx.Err() != nil || shouldAbort() - } - - executed := false - ok, deleteList := h.copyTombstonesToAllowList(breakCleanUpTombstonedNodes) - if !ok { - return executed, nil - } - - executed = true - if ok, err := h.reassignNeighborsOf(deleteList, breakCleanUpTombstonedNodes); err != nil { - return executed, err - } else if !ok { - return executed, nil - } - - if ok, err := h.replaceDeletedEntrypoint(deleteList, breakCleanUpTombstonedNodes); err != nil { - return executed, err - } else if !ok { - return executed, nil - } - - if ok, err := h.removeTombstonesAndNodes(deleteList, breakCleanUpTombstonedNodes); err != nil { - return executed, err - } else if !ok { - return executed, nil - } - - if _, err := h.resetIfEmpty(); err != nil { - return executed, err - } - - return executed, nil -} - -func (h *hnsw) replaceDeletedEntrypoint(deleteList helpers.AllowList, breakCleanUpTombstonedNodes breakCleanUpTombstonedNodesFunc) (ok bool, err error) { - h.resetLock.Lock() - defer h.resetLock.Unlock() - - if breakCleanUpTombstonedNodes() { - return false, nil - } - - it := deleteList.Iterator() - for id, ok := it.Next(); ok; id, ok = it.Next() { - if h.getEntrypoint() == id { - // this a special case because: - // - // 1. we need to find a new entrypoint, if this is the last point on this - // level, we need to find an entrypoint on a lower level - // 2. there is a risk that this is the only node in the entire graph. In - // this case we must reset the graph - h.shardedNodeLocks.RLock(id) - node := h.nodes[id] - h.shardedNodeLocks.RUnlock(id) - - if err := h.deleteEntrypoint(node, deleteList); err != nil { - return false, errors.Wrap(err, "delete entrypoint") - } - } - } - - return true, nil -} - -func (h *hnsw) reassignNeighborsOf(deleteList helpers.AllowList, breakCleanUpTombstonedNodes breakCleanUpTombstonedNodesFunc) (ok bool, err error) { - h.RLock() - size := len(h.nodes) - h.RUnlock() - - for n := 0; n < size; n++ { - if ok, err := h.reassignNeighbor(uint64(n), deleteList, breakCleanUpTombstonedNodes); err != nil { - return false, errors.Wrap(err, "reassign neighbor edges") - } else if !ok { - return false, nil - } - } - - return true, nil -} - -func (h *hnsw) reassignNeighbor(neighbor uint64, deleteList helpers.AllowList, breakCleanUpTombstonedNodes breakCleanUpTombstonedNodesFunc) (ok bool, err error) { - h.resetLock.Lock() - defer h.resetLock.Unlock() - - if breakCleanUpTombstonedNodes() { - return false, nil - } - - h.RLock() - h.shardedNodeLocks.RLock(neighbor) - neighborNode := h.nodes[neighbor] - h.shardedNodeLocks.RUnlock(neighbor) - currentEntrypoint := h.entryPointID - currentMaximumLayer := h.currentMaximumLayer - h.RUnlock() - - if neighborNode == nil || deleteList.Contains(neighborNode.id) { - return true, nil - } - - var neighborVec []float32 - var compressorDistancer compressionhelpers.CompressorDistancer - if h.compressed.Load() { - compressorDistancer = h.compressor.NewDistancerFromID(neighbor) - } else { - neighborVec, err = h.cache.Get(context.Background(), neighbor) - } - - if err != nil { - var e storobj.ErrNotFound - if errors.As(err, &e) { - h.handleDeletedNode(e.DocID) - return true, nil - } else { - // not a typed error, we can recover from, return with err - return false, errors.Wrap(err, "get neighbor vec") - } - } - neighborNode.Lock() - neighborLevel := neighborNode.level - if !connectionsPointTo(neighborNode.connections, deleteList) { - // nothing needs to be changed, skip - neighborNode.Unlock() - return true, nil - } - neighborNode.Unlock() - - entryPointID, err := h.findBestEntrypointForNode(currentMaximumLayer, - neighborLevel, currentEntrypoint, neighborVec, compressorDistancer) - if err != nil { - return false, errors.Wrap(err, "find best entrypoint") - } - - if entryPointID == neighbor { - // if we use ourselves as entrypoint and delete all connections in the - // next step, we won't find any neighbors, so we need to use an - // alternative entryPoint in this round - - if h.isOnlyNode(&vertex{id: neighbor}, deleteList) { - neighborNode.Lock() - // delete all existing connections before re-assigning - neighborLevel = neighborNode.level - neighborNode.connections = make([][]uint64, neighborLevel+1) - neighborNode.Unlock() - - if err := h.commitLog.ClearLinks(neighbor); err != nil { - return false, err - } - return true, nil - } - - tmpDenyList := deleteList.DeepCopy() - tmpDenyList.Insert(entryPointID) - - alternative, level := h.findNewLocalEntrypoint(tmpDenyList, currentMaximumLayer, - entryPointID) - if level > neighborLevel { - neighborNode.Lock() - // reset connections according to level - neighborNode.connections = make([][]uint64, level+1) - neighborNode.Unlock() - } - neighborLevel = level - entryPointID = alternative - } - - neighborNode.markAsMaintenance() - neighborNode.Lock() - // delete all existing connections before re-assigning - for level := range neighborNode.connections { - neighborNode.connections[level] = neighborNode.connections[level][:0] - } - neighborNode.Unlock() - if err := h.commitLog.ClearLinks(neighbor); err != nil { - return false, err - } - - if err := h.findAndConnectNeighbors(neighborNode, entryPointID, neighborVec, compressorDistancer, - neighborLevel, currentMaximumLayer, deleteList); err != nil { - return false, errors.Wrap(err, "find and connect neighbors") - } - neighborNode.unmarkAsMaintenance() - - h.metrics.CleanedUp() - return true, nil -} - -func connectionsPointTo(connections [][]uint64, needles helpers.AllowList) bool { - for _, atLevel := range connections { - for _, pointer := range atLevel { - if needles.Contains(pointer) { - return true - } - } - } - - return false -} - -// deleteEntrypoint deletes the current entrypoint and replaces it with a new -// one. It respects the attached denyList, so that it doesn't assign another -// node which also has a tombstone and is also in the process of being cleaned -// up -func (h *hnsw) deleteEntrypoint(node *vertex, denyList helpers.AllowList) error { - if h.isOnlyNode(node, denyList) { - // no point in finding another entrypoint if this is the only node - return nil - } - - node.Lock() - level := node.level - id := node.id - node.Unlock() - - newEntrypoint, level, ok := h.findNewGlobalEntrypoint(denyList, level, id) - if !ok { - return nil - } - - h.Lock() - h.entryPointID = newEntrypoint - h.currentMaximumLayer = level - h.Unlock() - if err := h.commitLog.SetEntryPointWithMaxLayer(newEntrypoint, level); err != nil { - return err - } - - return nil -} - -// returns entryPointID, level and whether a change occurred -func (h *hnsw) findNewGlobalEntrypoint(denyList helpers.AllowList, targetLevel int, - oldEntrypoint uint64, -) (uint64, int, bool) { - if h.getEntrypoint() != oldEntrypoint { - // entrypoint has already been changed (this could be due to a new import - // for example, nothing to do for us - return 0, 0, false - } - - for l := targetLevel; l >= 0; l-- { - // ideally we can find a new entrypoint at the same level of the - // to-be-deleted node. However, there is a chance it was the only node on - // that level, in that case we need to look at the next lower level for a - // better candidate - - h.RLock() - maxNodes := len(h.nodes) - h.RUnlock() - - for i := 0; i < maxNodes; i++ { - if h.getEntrypoint() != oldEntrypoint { - // entrypoint has already been changed (this could be due to a new import - // for example, nothing to do for us - return 0, 0, false - } - - if denyList.Contains(uint64(i)) { - continue - } - - h.shardedNodeLocks.RLock(uint64(i)) - candidate := h.nodes[i] - h.shardedNodeLocks.RUnlock(uint64(i)) - - if candidate == nil { - continue - } - - candidate.Lock() - candidateLevel := candidate.level - candidate.Unlock() - - if candidateLevel != l { - // not reaching up to the current level, skip in hope of finding another candidate - continue - } - - // we have a node that matches - return uint64(i), l, true - } - } - - // we made it through the entire graph and didn't find a new entrypoint all - // the way down to level 0. This can only mean the graph is empty, which is - // unexpected. This situation should have been prevented by the deleteLock. - panic(fmt.Sprintf( - "class %s: shard %s: findNewEntrypoint called on an empty hnsw graph", - h.className, h.shardName)) -} - -// returns entryPointID, level and whether a change occurred -func (h *hnsw) findNewLocalEntrypoint(denyList helpers.AllowList, targetLevel int, - oldEntrypoint uint64, -) (uint64, int) { - if h.getEntrypoint() != oldEntrypoint { - // the current global entrypoint is different from our local entrypoint, so - // we can just use the global one, as the global one is guaranteed to be - // present on every level, i.e. it is always chosen from the highest - // currently available level - return h.getEntrypoint(), h.currentMaximumLayer - } - - h.RLock() - maxNodes := len(h.nodes) - h.RUnlock() - - for l := targetLevel; l >= 0; l-- { - // ideally we can find a new entrypoint at the same level of the - // to-be-deleted node. However, there is a chance it was the only node on - // that level, in that case we need to look at the next lower level for a - // better candidate - for i := 0; i < maxNodes; i++ { - if denyList.Contains(uint64(i)) { - continue - } - - h.shardedNodeLocks.RLock(uint64(i)) - candidate := h.nodes[i] - h.shardedNodeLocks.RUnlock(uint64(i)) - - if candidate == nil { - continue - } - - candidate.Lock() - candidateLevel := candidate.level - candidate.Unlock() - - if candidateLevel != l { - // not reaching up to the current level, skip in hope of finding another candidate - continue - } - - // we have a node that matches - return uint64(i), l - } - } - - panic(fmt.Sprintf( - "class %s: shard %s: findNewLocalEntrypoint called on an empty hnsw graph", - h.className, h.shardName)) -} - -func (h *hnsw) isOnlyNode(needle *vertex, denyList helpers.AllowList) bool { - h.RLock() - h.shardedNodeLocks.RLockAll() - defer h.RUnlock() - defer h.shardedNodeLocks.RUnlockAll() - - return h.isOnlyNodeUnlocked(needle, denyList) -} - -func (h *hnsw) isOnlyNodeUnlocked(needle *vertex, denyList helpers.AllowList) bool { - for _, node := range h.nodes { - if node == nil || node.id == needle.id || denyList.Contains(node.id) { - continue - } - return false - } - return true -} - -func (h *hnsw) hasTombstone(id uint64) bool { - h.tombstoneLock.RLock() - defer h.tombstoneLock.RUnlock() - _, ok := h.tombstones[id] - return ok -} - -func (h *hnsw) addTombstone(ids ...uint64) error { - h.tombstoneLock.Lock() - defer h.tombstoneLock.Unlock() - - for _, id := range ids { - h.metrics.AddTombstone() - h.tombstones[id] = struct{}{} - if err := h.commitLog.AddTombstone(id); err != nil { - return err - } - } - return nil -} - -func (h *hnsw) removeTombstonesAndNodes(deleteList helpers.AllowList, breakCleanUpTombstonedNodes breakCleanUpTombstonedNodesFunc) (ok bool, err error) { - it := deleteList.Iterator() - for id, ok := it.Next(); ok; id, ok = it.Next() { - h.metrics.RemoveTombstone() - h.tombstoneLock.Lock() - delete(h.tombstones, id) - h.tombstoneLock.Unlock() - - h.resetLock.Lock() - if !breakCleanUpTombstonedNodes() { - h.shardedNodeLocks.Lock(id) - h.nodes[id] = nil - h.shardedNodeLocks.Unlock(id) - if h.compressed.Load() { - h.compressor.Delete(context.TODO(), id) - } else { - h.cache.Delete(context.TODO(), id) - } - if err := h.commitLog.DeleteNode(id); err != nil { - h.resetLock.Unlock() - return false, err - } - } - h.resetLock.Unlock() - - if err := h.commitLog.RemoveTombstone(id); err != nil { - return false, err - } - } - - return true, nil -} diff --git a/adapters/repos/db/vector/hnsw/delete_test.go b/adapters/repos/db/vector/hnsw/delete_test.go deleted file mode 100644 index 60a908ebf09a8530735ced625d5cdb8222ff8023..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/delete_test.go +++ /dev/null @@ -1,1322 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "context" - "fmt" - "sort" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv" - "github.com/weaviate/weaviate/adapters/repos/db/vector/common" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" - "github.com/weaviate/weaviate/adapters/repos/db/vector/testinghelpers" - "github.com/weaviate/weaviate/entities/cyclemanager" - "github.com/weaviate/weaviate/entities/storobj" - ent "github.com/weaviate/weaviate/entities/vectorindex/hnsw" -) - -func TempVectorForIDThunk(vectors [][]float32) func(context.Context, uint64, *common.VectorSlice) ([]float32, error) { - return func(ctx context.Context, id uint64, container *common.VectorSlice) ([]float32, error) { - copy(container.Slice, vectors[int(id)]) - return vectors[int(id)], nil - } -} - -func TestDelete_WithoutCleaningUpTombstones(t *testing.T) { - vectors := vectorsForDeleteTest() - var vectorIndex *hnsw - - store := testinghelpers.NewDummyStore(t) - t.Run("import the test vectors", func(t *testing.T) { - index, err := New(Config{ - RootPath: "doesnt-matter-as-committlogger-is-mocked-out", - ID: "delete-test", - MakeCommitLoggerThunk: MakeNoopCommitLogger, - DistanceProvider: distancer.NewCosineDistanceProvider(), - VectorForIDThunk: func(ctx context.Context, id uint64) ([]float32, error) { - return vectors[int(id)], nil - }, - TempVectorForIDThunk: TempVectorForIDThunk(vectors), - }, ent.UserConfig{ - MaxConnections: 30, - EFConstruction: 128, - - // The actual size does not matter for this test, but if it defaults to - // zero it will constantly think it's full and needs to be deleted - even - // after just being deleted, so make sure to use a positive number here. - VectorCacheMaxObjects: 100000, - }, cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - cyclemanager.NewCallbackGroupNoop(), store) - require.Nil(t, err) - vectorIndex = index - - for i, vec := range vectors { - err := vectorIndex.Add(uint64(i), vec) - require.Nil(t, err) - } - }) - - var control []uint64 - - t.Run("vectors are cached correctly", func(t *testing.T) { - assert.Equal(t, len(vectors), int(vectorIndex.cache.CountVectors())) - }) - - t.Run("doing a control search before delete with the respective allow list", func(t *testing.T) { - allowList := helpers.NewAllowList() - for i := range vectors { - if i%2 == 0 { - continue - } - - allowList.Insert(uint64(i)) - } - - res, _, err := vectorIndex.SearchByVector([]float32{0.1, 0.1, 0.1}, 20, allowList) - require.Nil(t, err) - require.True(t, len(res) > 0) - control = res - }) - - t.Run("deleting every even element", func(t *testing.T) { - for i := range vectors { - if i%2 != 0 { - continue - } - - err := vectorIndex.Delete(uint64(i)) - require.Nil(t, err) - } - }) - - t.Run("vector cache holds half the original vectors", func(t *testing.T) { - vectorIndex.CleanUpTombstonedNodes(neverStop) - assert.Equal(t, len(vectors)/2, int(vectorIndex.cache.CountVectors())) - }) - - t.Run("start a search that should only contain the remaining elements", func(t *testing.T) { - res, _, err := vectorIndex.SearchByVector([]float32{0.1, 0.1, 0.1}, 20, nil) - require.Nil(t, err) - require.True(t, len(res) > 0) - - for _, elem := range res { - if elem%2 == 0 { - t.Errorf("search result contained an even element: %d", elem) - } - } - - assert.Equal(t, control, res) - }) - - t.Run("destroy the index", func(t *testing.T) { - require.Nil(t, vectorIndex.Drop(context.Background())) - }) - - t.Run("vector cache holds no vectors", func(t *testing.T) { - assert.Equal(t, 0, int(vectorIndex.cache.CountVectors())) - }) -} - -func TestDelete_WithCleaningUpTombstonesOnce(t *testing.T) { - // there is a single bulk clean event after all the deletes - vectors := vectorsForDeleteTest() - var vectorIndex *hnsw - - store := testinghelpers.NewDummyStore(t) - - t.Run("import the test vectors", func(t *testing.T) { - index, err := New(Config{ - RootPath: "doesnt-matter-as-committlogger-is-mocked-out", - ID: "delete-test", - MakeCommitLoggerThunk: MakeNoopCommitLogger, - DistanceProvider: distancer.NewCosineDistanceProvider(), - VectorForIDThunk: func(ctx context.Context, id uint64) ([]float32, error) { - return vectors[int(id)], nil - }, - TempVectorForIDThunk: TempVectorForIDThunk(vectors), - }, ent.UserConfig{ - MaxConnections: 30, - EFConstruction: 128, - - // The actual size does not matter for this test, but if it defaults to - // zero it will constantly think it's full and needs to be deleted - even - // after just being deleted, so make sure to use a positive number here. - VectorCacheMaxObjects: 100000, - }, cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - cyclemanager.NewCallbackGroupNoop(), store) - require.Nil(t, err) - vectorIndex = index - - for i, vec := range vectors { - err := vectorIndex.Add(uint64(i), vec) - require.Nil(t, err) - } - }) - - var control []uint64 - var bfControl []uint64 - - t.Run("doing a control search before delete with the respective allow list", func(t *testing.T) { - allowList := helpers.NewAllowList() - for i := range vectors { - if i%2 == 0 { - continue - } - - allowList.Insert(uint64(i)) - } - - res, _, err := vectorIndex.SearchByVector([]float32{0.1, 0.1, 0.1}, 20, allowList) - require.Nil(t, err) - require.True(t, len(res) > 0) - require.Len(t, res, 20) - control = res - }) - - t.Run("brute force control", func(t *testing.T) { - bf := bruteForceCosine(vectors, []float32{0.1, 0.1, 0.1}, 100) - bfControl = make([]uint64, len(bf)) - i := 0 - for _, elem := range bf { - if elem%2 == 0 { - continue - } - - bfControl[i] = elem - i++ - } - - if i > 20 { - i = 20 - } - - bfControl = bfControl[:i] - assert.Equal(t, bfControl, control, "control should match bf control") - }) - - fmt.Printf("entrypoint before %d\n", vectorIndex.entryPointID) - t.Run("deleting every even element", func(t *testing.T) { - for i := range vectors { - if i%2 != 0 { - continue - } - - err := vectorIndex.Delete(uint64(i)) - require.Nil(t, err) - } - }) - - t.Run("running the cleanup", func(t *testing.T) { - err := vectorIndex.CleanUpTombstonedNodes(neverStop) - require.Nil(t, err) - }) - - t.Run("start a search that should only contain the remaining elements", func(t *testing.T) { - res, _, err := vectorIndex.SearchByVector([]float32{0.1, 0.1, 0.1}, 20, nil) - require.Nil(t, err) - require.True(t, len(res) > 0) - - for _, elem := range res { - if elem%2 == 0 { - t.Errorf("search result contained an even element: %d", elem) - } - } - - assert.Equal(t, control, res) - }) - - t.Run("verify the graph no longer has any tombstones", func(t *testing.T) { - assert.Len(t, vectorIndex.tombstones, 0) - }) - - t.Run("destroy the index", func(t *testing.T) { - require.Nil(t, vectorIndex.Drop(context.Background())) - }) -} - -func TestDelete_WithCleaningUpTombstonesInBetween(t *testing.T) { - // there is a single bulk clean event after all the deletes - vectors := vectorsForDeleteTest() - var vectorIndex *hnsw - store := testinghelpers.NewDummyStore(t) - - t.Run("import the test vectors", func(t *testing.T) { - index, err := New(Config{ - RootPath: "doesnt-matter-as-committlogger-is-mocked-out", - ID: "delete-test", - MakeCommitLoggerThunk: MakeNoopCommitLogger, - DistanceProvider: distancer.NewCosineDistanceProvider(), - VectorForIDThunk: func(ctx context.Context, id uint64) ([]float32, error) { - return vectors[int(id)], nil - }, - TempVectorForIDThunk: TempVectorForIDThunk(vectors), - }, ent.UserConfig{ - MaxConnections: 30, - EFConstruction: 128, - - // The actual size does not matter for this test, but if it defaults to - // zero it will constantly think it's full and needs to be deleted - even - // after just being deleted, so make sure to use a positive number here. - VectorCacheMaxObjects: 100000, - }, cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - cyclemanager.NewCallbackGroupNoop(), store) - // makes sure index is build only with level 0. To be removed after fixing WEAVIATE-179 - index.randFunc = func() float64 { return 0.1 } - - require.Nil(t, err) - vectorIndex = index - - for i, vec := range vectors { - err := vectorIndex.Add(uint64(i), vec) - require.Nil(t, err) - } - }) - - var control []uint64 - - t.Run("doing a control search before delete with the respective allow list", func(t *testing.T) { - allowList := helpers.NewAllowList() - for i := range vectors { - if i%2 == 0 { - continue - } - - allowList.Insert(uint64(i)) - } - - res, _, err := vectorIndex.SearchByVector([]float32{0.1, 0.1, 0.1}, 20, allowList) - require.Nil(t, err) - require.True(t, len(res) > 0) - - control = res - }) - - t.Run("deleting every even element", func(t *testing.T) { - for i := range vectors { - if i%10 == 0 { - // occasionally run clean up - err := vectorIndex.CleanUpTombstonedNodes(neverStop) - require.Nil(t, err) - } - - if i%2 != 0 { - continue - } - - err := vectorIndex.Delete(uint64(i)) - require.Nil(t, err) - } - - // finally run one final cleanup - err := vectorIndex.CleanUpTombstonedNodes(neverStop) - require.Nil(t, err) - }) - - t.Run("start a search that should only contain the remaining elements", func(t *testing.T) { - res, _, err := vectorIndex.SearchByVector([]float32{0.1, 0.1, 0.1}, 20, nil) - require.Nil(t, err) - require.True(t, len(res) > 0) - - for _, elem := range res { - if elem%2 == 0 { - t.Errorf("search result contained an even element: %d", elem) - } - } - - assert.Equal(t, control, res) - }) - - t.Run("verify the graph no longer has any tombstones", func(t *testing.T) { - assert.Len(t, vectorIndex.tombstones, 0) - }) - - t.Run("delete the remaining elements", func(t *testing.T) { - for i := range vectors { - if i%2 == 0 { - continue - } - - err := vectorIndex.Delete(uint64(i)) - require.Nil(t, err) - } - - err := vectorIndex.CleanUpTombstonedNodes(neverStop) - require.Nil(t, err) - }) - - t.Run("try to insert again and search", func(t *testing.T) { - for i := 0; i < 5; i++ { - err := vectorIndex.Add(uint64(i), vectors[i]) - require.Nil(t, err) - } - - res, _, err := vectorIndex.SearchByVector([]float32{0.1, 0.1, 0.1}, 20, nil) - require.Nil(t, err) - assert.ElementsMatch(t, []uint64{0, 1, 2, 3, 4}, res) - }) - - t.Run("destroy the index", func(t *testing.T) { - require.Nil(t, vectorIndex.Drop(context.Background())) - }) -} - -func createIndexImportAllVectorsAndDeleteEven(t *testing.T, vectors [][]float32, store *lsmkv.Store) (index *hnsw, remainingResult []uint64) { - index, err := New(Config{ - RootPath: "doesnt-matter-as-committlogger-is-mocked-out", - ID: "delete-test", - MakeCommitLoggerThunk: MakeNoopCommitLogger, - DistanceProvider: distancer.NewCosineDistanceProvider(), - VectorForIDThunk: func(ctx context.Context, id uint64) ([]float32, error) { - return vectors[int(id)], nil - }, - TempVectorForIDThunk: TempVectorForIDThunk(vectors), - }, ent.UserConfig{ - MaxConnections: 30, - EFConstruction: 128, - - // The actual size does not matter for this test, but if it defaults to - // zero it will constantly think it's full and needs to be deleted - even - // after just being deleted, so make sure to use a positive number here. - VectorCacheMaxObjects: 100000, - }, cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - cyclemanager.NewCallbackGroupNoop(), store) - require.Nil(t, err) - - // makes sure index is build only with level 0. To be removed after fixing WEAVIATE-179 - index.randFunc = func() float64 { return 0.1 } - - // to speed up test execution, size of nodes array is decreased - // from default 25k to little over number of vectors - index.nodes = make([]*vertex, int(1.2*float64(len(vectors)))) - - for i, vec := range vectors { - err := index.Add(uint64(i), vec) - require.Nil(t, err) - } - - for i := range vectors { - if i%2 != 0 { - continue - } - err := index.Delete(uint64(i)) - require.Nil(t, err) - } - - res, _, err := index.SearchByVector([]float32{0.1, 0.1, 0.1}, len(vectors), nil) - require.Nil(t, err) - require.True(t, len(res) > 0) - - for _, elem := range res { - if elem%2 == 0 { - t.Errorf("search result contained an even element: %d", elem) - } - } - - return index, res -} - -func genStopAtFunc(i int) func() bool { - counter := 0 - return func() bool { - if counter < i { - counter++ - return false - } - return true - } -} - -func TestDelete_WithCleaningUpTombstonesStopped(t *testing.T) { - vectors := vectorsForDeleteTest() - var index *hnsw - var possibleStopsCount int - // due to not yet resolved bug (https://semi-technology.atlassian.net/browse/WEAVIATE-179) - // db can return less vectors than are actually stored after tombstones cleanup - // controlRemainingResult contains all odd vectors (before cleanup was performed) - // controlRemainingResultAfterCleanup contains most of odd vectors (after cleanup was performed) - // - // this test verifies if partial cleanup will not change search output, therefore depending on - // where cleanup method was stopped, subset of controlRemainingResult is expected, though all - // vectors from controlRemainingResultAfterCleanup should be returned - // TODO to be simplified after fixing WEAVIATE-179, all results should be the same - var controlRemainingResult []uint64 - var controlRemainingResultAfterCleanup []uint64 - store := testinghelpers.NewDummyStore(t) - - t.Run("create control index", func(t *testing.T) { - index, controlRemainingResult = createIndexImportAllVectorsAndDeleteEven(t, vectors, store) - }) - - t.Run("count all cleanup tombstones stops", func(t *testing.T) { - counter := 0 - countingStopFunc := func() bool { - counter++ - return false - } - - err := index.CleanUpTombstonedNodes(countingStopFunc) - require.Nil(t, err) - - possibleStopsCount = counter - }) - - t.Run("search remaining elements after cleanup", func(t *testing.T) { - res, _, err := index.SearchByVector([]float32{0.1, 0.1, 0.1}, len(vectors), nil) - require.Nil(t, err) - require.True(t, len(res) > 0) - - for _, elem := range res { - if elem%2 == 0 { - t.Errorf("search result contained an even element: %d", elem) - } - } - controlRemainingResultAfterCleanup = res - }) - - t.Run("destroy the control index", func(t *testing.T) { - require.Nil(t, index.Drop(context.Background())) - }) - - for i := 0; i < possibleStopsCount; i++ { - index, _ = createIndexImportAllVectorsAndDeleteEven(t, vectors, store) - - t.Run("stop cleanup at place", func(t *testing.T) { - require.Nil(t, index.CleanUpTombstonedNodes(genStopAtFunc(i))) - }) - - t.Run("search remaining elements after partial cleanup", func(t *testing.T) { - res, _, err := index.SearchByVector([]float32{0.1, 0.1, 0.1}, len(vectors), nil) - require.Nil(t, err) - require.Subset(t, controlRemainingResult, res) - require.Subset(t, res, controlRemainingResultAfterCleanup) - }) - - t.Run("run complete cleanup", func(t *testing.T) { - require.Nil(t, index.CleanUpTombstonedNodes(neverStop)) - }) - - t.Run("search remaining elements after complete cleanup", func(t *testing.T) { - res, _, err := index.SearchByVector([]float32{0.1, 0.1, 0.1}, len(vectors), nil) - require.Nil(t, err) - require.Subset(t, controlRemainingResult, res) - require.Subset(t, res, controlRemainingResultAfterCleanup) - }) - - t.Run("destroy the index", func(t *testing.T) { - require.Nil(t, index.Drop(context.Background())) - }) - } -} - -func TestDelete_InCompressedIndex_WithCleaningUpTombstonesOnce(t *testing.T) { - var ( - vectorIndex *hnsw - // there is a single bulk clean event after all the deletes - vectors = vectorsForDeleteTest() - rootPath = t.TempDir() - userConfig = ent.UserConfig{ - MaxConnections: 30, - EFConstruction: 128, - - // The actual size does not matter for this test, but if it defaults to - // zero it will constantly think it's full and needs to be deleted - even - // after just being deleted, so make sure to use a positive number here. - VectorCacheMaxObjects: 100000, - PQ: ent.PQConfig{ - Enabled: true, - Encoder: ent.PQEncoder{ - Type: ent.PQEncoderTypeTile, - Distribution: ent.PQEncoderDistributionNormal, - }, - }, - } - ) - store := testinghelpers.NewDummyStore(t) - - t.Run("import the test vectors", func(t *testing.T) { - index, err := New(Config{ - RootPath: rootPath, - ID: "delete-test", - MakeCommitLoggerThunk: MakeNoopCommitLogger, - DistanceProvider: distancer.NewCosineDistanceProvider(), - VectorForIDThunk: func(ctx context.Context, id uint64) ([]float32, error) { - if int(id) >= len(vectors) { - return nil, storobj.NewErrNotFoundf(id, "out of range") - } - return vectors[int(id)], nil - }, - TempVectorForIDThunk: TempVectorForIDThunk(vectors), - }, userConfig, cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - cyclemanager.NewCallbackGroupNoop(), store) - require.Nil(t, err) - vectorIndex = index - - for i, vec := range vectors { - err := vectorIndex.Add(uint64(i), vec) - require.Nil(t, err) - } - cfg := ent.PQConfig{ - Enabled: true, - Encoder: ent.PQEncoder{ - Type: ent.PQEncoderTypeTile, - Distribution: ent.PQEncoderDistributionLogNormal, - }, - BitCompression: false, - Segments: 3, - Centroids: 256, - } - userConfig.PQ = cfg - index.compress(userConfig) - }) - - var control []uint64 - var bfControl []uint64 - - t.Run("doing a control search before delete with the respective allow list", func(t *testing.T) { - allowList := helpers.NewAllowList() - for i := range vectors { - if i%2 == 0 { - continue - } - - allowList.Insert(uint64(i)) - } - - res, _, err := vectorIndex.SearchByVector([]float32{0.1, 0.1, 0.1}, 20, allowList) - require.Nil(t, err) - require.True(t, len(res) > 0) - require.Len(t, res, 20) - control = res - }) - - t.Run("brute force control", func(t *testing.T) { - bf := bruteForceCosine(vectors, []float32{0.1, 0.1, 0.1}, 100) - bfControl = make([]uint64, len(bf)) - i := 0 - for _, elem := range bf { - if elem%2 == 0 { - continue - } - - bfControl[i] = elem - i++ - } - - if i > 20 { - i = 20 - } - - bfControl = bfControl[:i] - recall := float32(testinghelpers.MatchesInLists(bfControl, control)) / float32(len(bfControl)) - fmt.Println(recall) - assert.True(t, recall > 0.6, "control should match bf control") - }) - - fmt.Printf("entrypoint before %d\n", vectorIndex.entryPointID) - t.Run("deleting every even element", func(t *testing.T) { - for i := range vectors { - if i%2 != 0 { - continue - } - - err := vectorIndex.Delete(uint64(i)) - require.Nil(t, err) - } - }) - - t.Run("running the cleanup", func(t *testing.T) { - err := vectorIndex.CleanUpTombstonedNodes(neverStop) - require.Nil(t, err) - }) - - t.Run("start a search that should only contain the remaining elements", func(t *testing.T) { - res, _, err := vectorIndex.SearchByVector([]float32{0.1, 0.1, 0.1}, 20, nil) - require.Nil(t, err) - require.True(t, len(res) > 0) - - for _, elem := range res { - if elem%2 == 0 { - t.Errorf("search result contained an even element: %d", elem) - } - } - - recall := float32(testinghelpers.MatchesInLists(res, control)) / float32(len(control)) - assert.True(t, recall > 0.6) - }) - - t.Run("verify the graph no longer has any tombstones", func(t *testing.T) { - assert.Len(t, vectorIndex.tombstones, 0) - }) - - t.Run("destroy the index", func(t *testing.T) { - require.Nil(t, vectorIndex.Drop(context.Background())) - }) -} - -func TestDelete_InCompressedIndex_WithCleaningUpTombstonesOnce_DoesNotCrash(t *testing.T) { - var ( - vectorIndex *hnsw - // there is a single bulk clean event after all the deletes - vectors = vectorsForDeleteTest() - rootPath = t.TempDir() - userConfig = ent.UserConfig{ - MaxConnections: 30, - EFConstruction: 128, - - // The actual size does not matter for this test, but if it defaults to - // zero it will constantly think it's full and needs to be deleted - even - // after just being deleted, so make sure to use a positive number here. - VectorCacheMaxObjects: 100000, - PQ: ent.PQConfig{Enabled: true, Encoder: ent.PQEncoder{Type: "tile", Distribution: "normal"}}, - } - ) - - store := testinghelpers.NewDummyStore(t) - - t.Run("import the test vectors", func(t *testing.T) { - index, err := New(Config{ - RootPath: rootPath, - ID: "delete-test", - MakeCommitLoggerThunk: MakeNoopCommitLogger, - DistanceProvider: distancer.NewCosineDistanceProvider(), - VectorForIDThunk: func(ctx context.Context, id uint64) ([]float32, error) { - return vectors[int(id%uint64(len(vectors)))], nil - }, - TempVectorForIDThunk: TempVectorForIDThunk(vectors), - }, userConfig, cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - cyclemanager.NewCallbackGroupNoop(), store) - require.Nil(t, err) - vectorIndex = index - - for i, vec := range vectors { - err := vectorIndex.Add(uint64(i), vec) - require.Nil(t, err) - } - cfg := ent.PQConfig{ - Enabled: true, - Encoder: ent.PQEncoder{ - Type: ent.PQEncoderTypeTile, - Distribution: ent.PQEncoderDistributionLogNormal, - }, - BitCompression: false, - Segments: 3, - Centroids: 256, - } - userConfig.PQ = cfg - index.compress(userConfig) - for i := len(vectors); i < 1000; i++ { - err := vectorIndex.Add(uint64(i), vectors[i%len(vectors)]) - require.Nil(t, err) - } - }) - - t.Run("deleting every even element", func(t *testing.T) { - for i := range vectors { - if i%2 != 0 { - continue - } - - err := vectorIndex.Delete(uint64(i)) - require.Nil(t, err) - } - }) - - t.Run("running the cleanup", func(t *testing.T) { - err := vectorIndex.CleanUpTombstonedNodes(neverStop) - require.Nil(t, err) - }) - - t.Run("verify the graph no longer has any tombstones", func(t *testing.T) { - assert.Len(t, vectorIndex.tombstones, 0) - }) - - t.Run("destroy the index", func(t *testing.T) { - require.Nil(t, vectorIndex.Drop(context.Background())) - }) -} - -// we need a certain number of elements so that we can make sure that nodes -// from all layers will eventually be deleted, otherwise our test only tests -// edge cases which aren't very common in real life, but ignore the most common -// deletes -func vectorsForDeleteTest() [][]float32 { - return [][]float32{ - {0.27335858, 0.42670676, 0.12599982}, - {0.34369454, 0.78510034, 0.78000546}, - {0.2342731, 0.076864816, 0.6405078}, - {0.07597838, 0.7752282, 0.87022865}, - {0.78632426, 0.06902865, 0.7423889}, - {0.3055758, 0.3901508, 0.9399572}, - {0.48687622, 0.26338226, 0.06495104}, - {0.5384028, 0.35410047, 0.8821815}, - {0.25123185, 0.62722564, 0.86443096}, - {0.58484185, 0.13103616, 0.4034975}, - {0.0019696166, 0.46822622, 0.42492124}, - {0.42401955, 0.8278863, 0.5952888}, - {0.15367928, 0.70778894, 0.0070928824}, - {0.95760256, 0.45898128, 0.1541115}, - {0.9125976, 0.9021616, 0.21607016}, - {0.9876307, 0.5243228, 0.37294936}, - {0.8194746, 0.56142205, 0.5130103}, - {0.805065, 0.62250346, 0.63715476}, - {0.9969276, 0.5115748, 0.18916714}, - {0.16419733, 0.15029702, 0.36020836}, - {0.9660323, 0.35887036, 0.6072966}, - {0.72765416, 0.27891788, 0.9094314}, - {0.8626208, 0.3540126, 0.3100354}, - {0.7153876, 0.17094712, 0.7801294}, - {0.23180388, 0.107446484, 0.69542855}, - {0.54731685, 0.8949827, 0.68316746}, - {0.15049729, 0.1293767, 0.0574729}, - {0.89379513, 0.67022973, 0.57360715}, - {0.725353, 0.25326362, 0.44264215}, - {0.2568602, 0.4986094, 0.9759933}, - {0.7300015, 0.70019704, 0.49546525}, - {0.54314494, 0.2004176, 0.63803226}, - {0.6180191, 0.5260845, 0.9373999}, - {0.63356537, 0.81430644, 0.78373694}, - {0.69995105, 0.84198904, 0.17851257}, - {0.5197941, 0.11502675, 0.95129955}, - {0.15791401, 0.07516741, 0.113447875}, - {0.06811827, 0.4450082, 0.98595786}, - {0.7153448, 0.41833848, 0.06332495}, - {0.6704102, 0.28931814, 0.031580303}, - {0.47773632, 0.73334247, 0.6925025}, - {0.7976896, 0.9499536, 0.6394833}, - {0.3074854, 0.14025249, 0.35961738}, - {0.49956197, 0.093575336, 0.790093}, - {0.4641653, 0.21276893, 0.528895}, - {0.1021849, 0.9416305, 0.46738508}, - {0.3790398, 0.50099677, 0.98233247}, - {0.39650732, 0.020929832, 0.53968865}, - {0.77604437, 0.8554197, 0.24056046}, - {0.07174444, 0.28758526, 0.67587185}, - {0.22292718, 0.66624546, 0.6077909}, - {0.22090498, 0.36197436, 0.40415043}, - {0.04838009, 0.120789215, 0.17928012}, - {0.55166364, 0.3400502, 0.43698996}, - {0.7638108, 0.47014108, 0.23208627}, - {0.9239513, 0.8418566, 0.23518613}, - {0.289589, 0.85010827, 0.055741556}, - {0.32436147, 0.18756394, 0.4217864}, - {0.041671168, 0.37824047, 0.66486764}, - {0.5052222, 0.07982704, 0.64345413}, - {0.62675995, 0.20138603, 0.8231867}, - {0.86306876, 0.9698708, 0.11398846}, - {0.68566775, 0.22026269, 0.13525572}, - {0.57706076, 0.32325208, 0.6122228}, - {0.80035216, 0.18560356, 0.6328281}, - {0.87145543, 0.19380389, 0.8863942}, - {0.33777508, 0.6056442, 0.9110077}, - {0.3961719, 0.49714503, 0.14191929}, - {0.5344362, 0.8166916, 0.75880384}, - {0.015749464, 0.63223976, 0.5470922}, - {0.10512444, 0.2212036, 0.24995685}, - {0.10831311, 0.27044898, 0.8668174}, - {0.3272971, 0.6659298, 0.87119603}, - {0.42913893, 0.14528985, 0.69957525}, - {0.33012474, 0.81964344, 0.092787445}, - {0.093618214, 0.90637344, 0.94406706}, - {0.12161567, 0.75131124, 0.40563175}, - {0.9154454, 0.75925833, 0.8406739}, - {0.81649286, 0.9025715, 0.3105051}, - {0.2927649, 0.22649862, 0.9708593}, - {0.30813727, 0.0079439245, 0.39662006}, - {0.94943213, 0.36778906, 0.217876}, - {0.716794, 0.3811725, 0.18448676}, - {0.66879725, 0.29722908, 0.0031202603}, - {0.11104216, 0.13094379, 0.0787222}, - {0.8508966, 0.86416596, 0.15885831}, - {0.2303136, 0.56660503, 0.17114973}, - {0.8632685, 0.4229249, 0.1936724}, - {0.03060897, 0.35226125, 0.8115969}, - } -} - -func TestDelete_EntrypointIssues(t *testing.T) { - // This test is motivated by flakyness of other tests. We seemed to have - // experienced a failure with the following structure - // - // Entrypoint: 6 - // Max Level: 1 - // Tombstones map[] - - // Nodes and Connections: - // Node 0 - // Level 0: Connections: [1 2 3 4 5 6 7 8] - // Node 1 - // Level 0: Connections: [0 2 3 4 5 6 7 8] - // Node 2 - // Level 0: Connections: [1 0 3 4 5 6 7 8] - // Node 3 - // Level 0: Connections: [2 1 0 4 5 6 7 8] - // Node 4 - // Level 0: Connections: [3 2 1 0 5 6 7 8] - // Node 5 - // Level 0: Connections: [3 4 2 1 0 6 7 8] - // Node 6 - // Level 0: Connections: [4 2 1 3 5 0 7 8] - // Level 1: Connections: [7] - // Node 7 - // Level 1: Connections: [6] - // Level 0: Connections: [6 4 3 5 2 1 0 8] - // Node 8 - // Level 0: Connections: [7 6 4 3 5 2 1 0] - // - // This test aims to rebuild this tree exactly (manually) and verifies that - // deletion of the old entrypoint (element 6), works without issue - // - // The underlying test set can be found in vectors_for_test.go - - index, err := New(Config{ - RootPath: "doesnt-matter-as-committlogger-is-mocked-out", - ID: "delete-entrypoint-test", - MakeCommitLoggerThunk: MakeNoopCommitLogger, - DistanceProvider: distancer.NewCosineDistanceProvider(), - VectorForIDThunk: testVectorForID, - }, ent.UserConfig{ - MaxConnections: 30, - EFConstruction: 128, - - // The actual size does not matter for this test, but if it defaults to - // zero it will constantly think it's full and needs to be deleted - even - // after just being deleted, so make sure to use a positive number here. - VectorCacheMaxObjects: 100000, - }, cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - cyclemanager.NewCallbackGroupNoop(), testinghelpers.NewDummyStore(t)) - require.Nil(t, err) - - // manually build the index - index.entryPointID = 6 - index.currentMaximumLayer = 1 - index.nodes = make([]*vertex, 50) - index.nodes[0] = &vertex{ - id: 0, - connections: [][]uint64{ - {1, 2, 3, 4, 5, 6, 7, 8}, - }, - } - index.nodes[1] = &vertex{ - id: 1, - connections: [][]uint64{ - {0, 2, 3, 4, 5, 6, 7, 8}, - }, - } - index.nodes[2] = &vertex{ - id: 2, - connections: [][]uint64{ - {1, 0, 3, 4, 5, 6, 7, 8}, - }, - } - index.nodes[3] = &vertex{ - id: 3, - connections: [][]uint64{ - {2, 1, 0, 4, 5, 6, 7, 8}, - }, - } - index.nodes[4] = &vertex{ - id: 4, - connections: [][]uint64{ - {3, 2, 1, 0, 5, 6, 7, 8}, - }, - } - index.nodes[5] = &vertex{ - id: 5, - connections: [][]uint64{ - {3, 4, 2, 1, 0, 6, 7, 8}, - }, - } - index.nodes[6] = &vertex{ - id: 6, - connections: [][]uint64{ - {4, 3, 1, 3, 5, 0, 7, 8}, - {7}, - }, - level: 1, - } - index.nodes[7] = &vertex{ - id: 7, - connections: [][]uint64{ - {6, 4, 3, 5, 2, 1, 0, 8}, - {6}, - }, - level: 1, - } - index.nodes[8] = &vertex{ - id: 8, - connections: [][]uint64{ - 8: {7, 6, 4, 3, 5, 2, 1, 0}, - }, - } - - dumpIndex(index, "before delete") - - t.Run("delete some elements and permanently delete tombstoned elements", - func(t *testing.T) { - err := index.Delete(6) - require.Nil(t, err) - err = index.Delete(8) - require.Nil(t, err) - - err = index.CleanUpTombstonedNodes(neverStop) - require.Nil(t, err) - }) - - dumpIndex(index, "after delete") - - expectedResults := []uint64{ - 3, 5, 4, // cluster 2 - 7, // cluster 3 with element 6 and 8 deleted - 2, 1, 0, // cluster 1 - } - - t.Run("verify that the results are correct", func(t *testing.T) { - position := 3 - res, _, err := index.knnSearchByVector(testVectors[position], 50, 36, nil) - require.Nil(t, err) - assert.Equal(t, expectedResults, res) - }) - - // t.Fail() - t.Run("destroy the index", func(t *testing.T) { - require.Nil(t, index.Drop(context.Background())) - }) -} - -func TestDelete_MoreEntrypointIssues(t *testing.T) { - vectors := [][]float32{ - {7, 1}, - {8, 2}, - {23, 14}, - {6.5, -1}, - } - - vecForID := func(ctx context.Context, id uint64) ([]float32, error) { - return vectors[int(id)], nil - } - // This test is motivated by flakyness of other tests. We seemed to have - // experienced a failure with the following structure - // - // ID: thing_geoupdatetestclass_single_location - // Entrypoint: 2 - // Max Level: 1 - // Tombstones map[0:{} 1:{}] - // - // Nodes and Connections: - // Node 0 - // Level 0: Connections: [1] - // Node 1 - // Level 0: Connections: [0 2] - // Level 1: Connections: [2] - // Node 2 - // Level 1: Connections: [1] - // Level 0: Connections: [1] - - index, err := New(Config{ - RootPath: "doesnt-matter-as-committlogger-is-mocked-out", - ID: "more-delete-entrypoint-flakyness-test", - MakeCommitLoggerThunk: MakeNoopCommitLogger, - DistanceProvider: distancer.NewGeoProvider(), - VectorForIDThunk: vecForID, - TempVectorForIDThunk: TempVectorForIDThunk(vectors), - }, ent.UserConfig{ - MaxConnections: 30, - EFConstruction: 128, - - // The actual size does not matter for this test, but if it defaults to - // zero it will constantly think it's full and needs to be deleted - even - // after just being deleted, so make sure to use a positive number here. - VectorCacheMaxObjects: 100000, - }, cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - cyclemanager.NewCallbackGroupNoop(), testinghelpers.NewDummyStore(t)) - require.Nil(t, err) - - // manually build the index - index.entryPointID = 2 - index.currentMaximumLayer = 1 - index.tombstones = map[uint64]struct{}{ - 0: {}, - 1: {}, - } - index.nodes = make([]*vertex, 50) - index.nodes[0] = &vertex{ - id: 0, - connections: [][]uint64{ - 0: {1}, - }, - } - index.nodes[1] = &vertex{ - id: 1, - connections: [][]uint64{ - 0: {0, 2}, - 1: {2}, - }, - } - index.nodes[2] = &vertex{ - id: 2, - connections: [][]uint64{ - 0: {1}, - 1: {1}, - }, - } - - dumpIndex(index, "before adding another element") - t.Run("adding a third element", func(t *testing.T) { - vec, _ := testVectorForID(context.TODO(), 3) - index.Add(3, vec) - }) - - expectedResults := []uint64{ - 3, 2, - } - - t.Run("verify that the results are correct", func(t *testing.T) { - position := 3 - res, _, err := index.knnSearchByVector(testVectors[position], 50, 36, nil) - require.Nil(t, err) - assert.Equal(t, expectedResults, res) - }) - - t.Run("destroy the index", func(t *testing.T) { - require.Nil(t, index.Drop(context.Background())) - }) -} - -func TestDelete_TombstonedEntrypoint(t *testing.T) { - vecForID := func(ctx context.Context, id uint64) ([]float32, error) { - // always return same vec for all elements - return []float32{0.1, 0.2}, nil - } - index, err := New(Config{ - RootPath: "doesnt-matter-as-committlogger-is-mocked-out", - ID: "tombstoned-entrypoint-test", - MakeCommitLoggerThunk: MakeNoopCommitLogger, - DistanceProvider: distancer.NewCosineDistanceProvider(), - VectorForIDThunk: vecForID, - TempVectorForIDThunk: TempVectorForIDThunk([][]float32{{0.1, 0.2}}), - }, ent.UserConfig{ - MaxConnections: 30, - EFConstruction: 128, - // explicitly turn off, so we only focus on the tombstoned periods - CleanupIntervalSeconds: 0, - - // The actual size does not matter for this test, but if it defaults to - // zero it will constantly think it's full and needs to be deleted - even - // after just being deleted, so make sure to use a positive number here. - VectorCacheMaxObjects: 100000, - }, cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - cyclemanager.NewCallbackGroupNoop(), testinghelpers.NewDummyStore(t)) - require.Nil(t, err) - - objVec := []float32{0.1, 0.2} - searchVec := []float32{0.05, 0.05} - - require.Nil(t, index.Add(0, objVec)) - require.Nil(t, index.Delete(0)) - require.Nil(t, index.Add(1, objVec)) - - res, _, err := index.SearchByVector(searchVec, 100, nil) - require.Nil(t, err) - assert.Equal(t, []uint64{1}, res, "should contain the only result") - - t.Run("destroy the index", func(t *testing.T) { - require.Nil(t, index.Drop(context.Background())) - }) -} - -func TestDelete_Flakyness_gh_1369(t *testing.T) { - // parse a snapshot form a flaky test - snapshotBefore := []byte(`{"labels":["ran a cleanup cycle"],"id":"delete-test","entrypoint":3,"currentMaximumLayer":3,"tombstones":{},"nodes":[{"id":1,"level":0,"connections":{"0":[11,25,33,3,29,32,5,19,30,7,17,27,21,31,36,34,35,23,15,9,13]}},{"id":3,"level":3,"connections":{"0":[1,29,11,5,25,33,19,32,7,17,30,21,35,31,27,36,23,34,9,15,13],"1":[29,36,13],"2":[29,36],"3":[36]}},{"id":5,"level":0,"connections":{"0":[29,19,7,32,35,21,1,31,3,33,23,25,11,17,36,27,30,9,15,34,13]}},{"id":7,"level":0,"connections":{"0":[32,19,21,31,5,35,23,29,33,36,17,1,9,27,25,30,11,3,15,13,34]}},{"id":9,"level":0,"connections":{"0":[36,23,31,21,15,17,27,7,32,35,30,13,19,33,5,25,29,11,1,34,3]}},{"id":11,"level":0,"connections":{"0":[25,33,1,30,17,3,27,32,34,29,19,7,5,36,15,21,31,23,9,13,35]}},{"id":13,"level":1,"connections":{"0":[15,27,34,36,30,17,9,33,25,31,23,21,11,32,7,1,19,35,5,29,3],"1":[36,29,3]}},{"id":15,"level":0,"connections":{"0":[13,27,36,17,30,9,34,33,31,23,25,21,32,11,7,1,19,35,5,29,3]}},{"id":17,"level":0,"connections":{"0":[27,30,36,33,15,32,25,31,9,11,21,7,23,1,34,13,19,5,29,35,3]}},{"id":19,"level":0,"connections":{"0":[5,7,32,29,35,21,31,23,1,33,17,3,25,36,11,27,9,30,15,34,13]}},{"id":21,"level":0,"connections":{"0":[31,23,7,35,32,19,9,36,5,17,27,33,29,30,15,1,25,11,3,13,34]}},{"id":23,"level":0,"connections":{"0":[31,21,9,35,7,36,32,19,17,5,27,33,15,29,30,25,1,13,11,3,34]}},{"id":25,"level":0,"connections":{"0":[11,33,1,30,17,27,32,3,34,29,7,19,36,5,15,21,31,23,9,13,35]}},{"id":27,"level":0,"connections":{"0":[17,30,36,15,33,25,13,9,34,32,11,31,21,7,23,1,19,5,29,35,3]}},{"id":29,"level":2,"connections":{"0":[5,19,32,7,3,1,33,35,21,25,31,11,23,17,30,36,27,9,15,34,13],"1":[3,36,13],"2":[3,36]}},{"id":30,"level":0,"connections":{"0":[27,17,33,25,15,36,11,34,32,1,13,9,31,7,21,23,19,29,5,3,35]}},{"id":31,"level":0,"connections":{"0":[21,23,7,32,35,9,36,19,17,5,27,33,29,30,15,25,1,11,13,3,34]}},{"id":32,"level":0,"connections":{"0":[7,19,21,31,5,33,29,17,23,1,35,36,25,27,30,11,9,3,15,34,13]}},{"id":33,"level":0,"connections":{"0":[25,11,1,17,30,32,27,7,19,36,29,5,21,31,3,34,15,23,9,35,13]}},{"id":34,"level":0,"connections":{"0":[30,27,15,13,25,17,11,33,36,1,32,9,31,7,21,3,23,19,29,5,35]}},{"id":35,"level":0,"connections":{"0":[21,7,31,23,19,5,32,29,9,36,17,33,1,27,25,30,3,11,15,13,34]}},{"id":36,"level":3,"connections":{"0":[17,9,27,15,31,23,21,30,32,7,33,13,25,19,35,11,34,1,5,29,3],"1":[13,29,3],"2":[29,3],"3":[3]}}]} -`) - - vectors := vectorsForDeleteTest() - vecForID := func(ctx context.Context, id uint64) ([]float32, error) { - return vectors[int(id)], nil - } - - index, err := NewFromJSONDumpMap(snapshotBefore, vecForID) - require.Nil(t, err) - index.forbidFlat = true - - var control []uint64 - t.Run("control search before delete with the respective allow list", func(t *testing.T) { - allowList := helpers.NewAllowList() - for i := range vectors { - if i%2 == 0 { - continue - } - - allowList.Insert(uint64(i)) - } - - res, _, err := index.SearchByVector([]float32{0.1, 0.1, 0.1}, 20, allowList) - require.Nil(t, err) - require.True(t, len(res) > 0) - - control = res - }) - - t.Run("delete the remaining even entries", func(t *testing.T) { - require.Nil(t, index.Delete(30)) - require.Nil(t, index.Delete(32)) - require.Nil(t, index.Delete(34)) - require.Nil(t, index.Delete(36)) - }) - - t.Run("verify against control BEFORE Tombstone Cleanup", func(t *testing.T) { - res, _, err := index.SearchByVector([]float32{0.1, 0.1, 0.1}, 20, nil) - require.Nil(t, err) - require.True(t, len(res) > 0) - assert.Equal(t, control, res) - }) - - t.Run("clean up tombstoned nodes", func(t *testing.T) { - require.Nil(t, index.CleanUpTombstonedNodes(neverStop)) - }) - - t.Run("verify against control AFTER Tombstone Cleanup", func(t *testing.T) { - res, _, err := index.SearchByVector([]float32{0.1, 0.1, 0.1}, 20, nil) - require.Nil(t, err) - require.True(t, len(res) > 0) - assert.Equal(t, control, res) - }) - - t.Run("now delete the entrypoint", func(t *testing.T) { - require.Nil(t, index.Delete(index.entryPointID)) - }) - - t.Run("clean up tombstoned nodes", func(t *testing.T) { - require.Nil(t, index.CleanUpTombstonedNodes(neverStop)) - }) - - t.Run("now delete the entrypoint", func(t *testing.T) { - // this verifies that our findNewLocalEntrypoint also works when the global - // entrypoint is affected - require.Nil(t, index.Delete(index.entryPointID)) - }) - - t.Run("clean up tombstoned nodes", func(t *testing.T) { - require.Nil(t, index.CleanUpTombstonedNodes(neverStop)) - }) - - t.Run("destroy the index", func(t *testing.T) { - require.Nil(t, index.Drop(context.Background())) - }) -} - -func bruteForceCosine(vectors [][]float32, query []float32, k int) []uint64 { - type distanceAndIndex struct { - distance float32 - index uint64 - } - - distances := make([]distanceAndIndex, len(vectors)) - - d := distancer.NewCosineDistanceProvider().New(distancer.Normalize(query)) - for i, vec := range vectors { - dist, _, _ := d.Distance(distancer.Normalize(vec)) - distances[i] = distanceAndIndex{ - index: uint64(i), - distance: dist, - } - } - - sort.Slice(distances, func(a, b int) bool { - return distances[a].distance < distances[b].distance - }) - - if len(distances) < k { - k = len(distances) - } - - out := make([]uint64, k) - for i := 0; i < k; i++ { - out[i] = distances[i].index - } - - return out -} - -func neverStop() bool { - return false -} - -// This test simulates what happens when the EP is removed from the -// VectorForID-serving store -func Test_DeleteEPVecInUnderlyingObjectStore(t *testing.T) { - var vectorIndex *hnsw - - vectors := [][]float32{ - {1, 1}, - {2, 2}, - {3, 3}, - } - - vectorErrors := []error{ - nil, - nil, - nil, - } - store := testinghelpers.NewDummyStore(t) - - t.Run("import the test vectors", func(t *testing.T) { - index, err := New(Config{ - RootPath: "doesnt-matter-as-committlogger-is-mocked-out", - ID: "delete-ep-in-underlying-store-test", - MakeCommitLoggerThunk: MakeNoopCommitLogger, - DistanceProvider: distancer.NewL2SquaredProvider(), - VectorForIDThunk: func(ctx context.Context, id uint64) ([]float32, error) { - fmt.Printf("vec for pos=%d is %v\n", id, vectors[int(id)]) - return vectors[int(id)], vectorErrors[int(id)] - }, - TempVectorForIDThunk: TempVectorForIDThunk(vectors), - }, ent.UserConfig{ - MaxConnections: 30, - EFConstruction: 128, - - // The actual size does not matter for this test, but if it defaults to - // zero it will constantly think it's full and needs to be deleted - even - // after just being deleted, so make sure to use a positive number here. - VectorCacheMaxObjects: 100000, - }, cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - cyclemanager.NewCallbackGroupNoop(), store) - require.Nil(t, err) - vectorIndex = index - - for i, vec := range vectors { - err := vectorIndex.Add(uint64(i), vec) - require.Nil(t, err) - } - - fmt.Printf("ep is %d\n", vectorIndex.entryPointID) - }) - - t.Run("simulate ep vec deletion in object store", func(t *testing.T) { - vectors[0] = nil - vectorErrors[0] = storobj.NewErrNotFoundf(0, "deleted") - vectorIndex.cache.Delete(context.Background(), 0) - }) - - t.Run("try to insert a fourth vector", func(t *testing.T) { - vectors = append(vectors, []float32{4, 4}) - vectorErrors = append(vectorErrors, nil) - - pos := len(vectors) - 1 - err := vectorIndex.Add(uint64(pos), vectors[pos]) - require.Nil(t, err) - }) -} diff --git a/adapters/repos/db/vector/hnsw/deserializer.go b/adapters/repos/db/vector/hnsw/deserializer.go deleted file mode 100644 index b32d4c99b53e8c6997909ba8373fc8221e9de242..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/deserializer.go +++ /dev/null @@ -1,676 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "bufio" - "encoding/binary" - "io" - "math" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/adapters/repos/db/vector/cache" - "github.com/weaviate/weaviate/adapters/repos/db/vector/compressionhelpers" -) - -type Deserializer struct { - logger logrus.FieldLogger - reusableBuffer []byte - reusableConnectionsSlice []uint64 -} - -type DeserializationResult struct { - Nodes []*vertex - Entrypoint uint64 - Level uint16 - Tombstones map[uint64]struct{} - EntrypointChanged bool - PQData compressionhelpers.PQData - Compressed bool - - // If there is no entry for the links at a level to be replaced, we must - // assume that all links were appended and prior state must exist - // Similarly if we run into a "Clear" we need to explicitly set the replace - // flag, so that future appends aren't always appended and we run into a - // situation where reading multiple condensed logs in succession leads to too - // many connections as discovered in - // https://github.com/weaviate/weaviate/issues/1868 - LinksReplaced map[uint64]map[uint16]struct{} -} - -func (dr DeserializationResult) ReplaceLinks(node uint64, level uint16) bool { - levels, ok := dr.LinksReplaced[node] - if !ok { - return false - } - - _, ok = levels[level] - return ok -} - -func NewDeserializer(logger logrus.FieldLogger) *Deserializer { - return &Deserializer{logger: logger} -} - -func (d *Deserializer) resetResusableBuffer(size int) { - if size <= cap(d.reusableBuffer) { - d.reusableBuffer = d.reusableBuffer[:size] - } else { - d.reusableBuffer = make([]byte, size, size*2) - } -} - -func (d *Deserializer) resetReusableConnectionsSlice(size int) { - if size <= cap(d.reusableConnectionsSlice) { - d.reusableConnectionsSlice = d.reusableConnectionsSlice[:size] - } else { - d.reusableConnectionsSlice = make([]uint64, size, size*2) - } -} - -func (d *Deserializer) Do(fd *bufio.Reader, - initialState *DeserializationResult, keepLinkReplaceInformation bool, -) (*DeserializationResult, int, error) { - validLength := 0 - out := initialState - if out == nil { - out = &DeserializationResult{ - Nodes: make([]*vertex, cache.InitialSize), - Tombstones: make(map[uint64]struct{}), - LinksReplaced: make(map[uint64]map[uint16]struct{}), - } - } - - for { - ct, err := d.ReadCommitType(fd) - if err != nil { - if errors.Is(err, io.EOF) { - break - } - - return nil, validLength, err - } - - var readThisRound int - - switch ct { - case AddNode: - err = d.ReadNode(fd, out) - readThisRound = 10 - case SetEntryPointMaxLevel: - var entrypoint uint64 - var level uint16 - entrypoint, level, err = d.ReadEP(fd) - out.Entrypoint = entrypoint - out.Level = level - out.EntrypointChanged = true - readThisRound = 10 - case AddLinkAtLevel: - err = d.ReadLink(fd, out) - readThisRound = 18 - case AddLinksAtLevel: - readThisRound, err = d.ReadAddLinks(fd, out) - case ReplaceLinksAtLevel: - readThisRound, err = d.ReadLinks(fd, out, keepLinkReplaceInformation) - case AddTombstone: - err = d.ReadAddTombstone(fd, out.Tombstones) - readThisRound = 8 - case RemoveTombstone: - err = d.ReadRemoveTombstone(fd, out.Tombstones) - readThisRound = 8 - case ClearLinks: - err = d.ReadClearLinks(fd, out, keepLinkReplaceInformation) - readThisRound = 8 - case ClearLinksAtLevel: - err = d.ReadClearLinksAtLevel(fd, out, keepLinkReplaceInformation) - readThisRound = 10 - case DeleteNode: - err = d.ReadDeleteNode(fd, out) - readThisRound = 8 - case ResetIndex: - out.Entrypoint = 0 - out.Level = 0 - out.Nodes = make([]*vertex, cache.InitialSize) - case AddPQ: - err = d.ReadPQ(fd, out) - readThisRound = 9 - default: - err = errors.Errorf("unrecognized commit type %d", ct) - } - if err != nil { - // do not return nil, err, because the err could be a recoverable one - return out, validLength, err - } else { - validLength += 1 + readThisRound // 1 byte for commit type - } - } - - return out, validLength, nil -} - -func (d *Deserializer) ReadNode(r io.Reader, res *DeserializationResult) error { - id, err := d.readUint64(r) - if err != nil { - return err - } - - level, err := d.readUint16(r) - if err != nil { - return err - } - - newNodes, changed, err := growIndexToAccomodateNode(res.Nodes, id, d.logger) - if err != nil { - return err - } - - if changed { - res.Nodes = newNodes - } - - if res.Nodes[id] == nil { - res.Nodes[id] = &vertex{level: int(level), id: id, connections: make([][]uint64, level+1)} - } else { - maybeGrowConnectionsForLevel(&res.Nodes[id].connections, level) - res.Nodes[id].level = int(level) - } - return nil -} - -func (d *Deserializer) ReadEP(r io.Reader) (uint64, uint16, error) { - id, err := d.readUint64(r) - if err != nil { - return 0, 0, err - } - - level, err := d.readUint16(r) - if err != nil { - return 0, 0, err - } - - return id, level, nil -} - -func (d *Deserializer) ReadLink(r io.Reader, res *DeserializationResult) error { - source, err := d.readUint64(r) - if err != nil { - return err - } - - level, err := d.readUint16(r) - if err != nil { - return err - } - - target, err := d.readUint64(r) - if err != nil { - return err - } - - newNodes, changed, err := growIndexToAccomodateNode(res.Nodes, source, d.logger) - if err != nil { - return err - } - - if changed { - res.Nodes = newNodes - } - - if res.Nodes[int(source)] == nil { - res.Nodes[int(source)] = &vertex{id: source, connections: make([][]uint64, level+1)} - } - - maybeGrowConnectionsForLevel(&res.Nodes[int(source)].connections, level) - - res.Nodes[int(source)].connections[int(level)] = append(res.Nodes[int(source)].connections[int(level)], target) - return nil -} - -func (d *Deserializer) ReadLinks(r io.Reader, res *DeserializationResult, - keepReplaceInfo bool, -) (int, error) { - d.resetResusableBuffer(12) - _, err := io.ReadFull(r, d.reusableBuffer) - if err != nil { - return 0, err - } - - source := binary.LittleEndian.Uint64(d.reusableBuffer[0:8]) - level := binary.LittleEndian.Uint16(d.reusableBuffer[8:10]) - length := binary.LittleEndian.Uint16(d.reusableBuffer[10:12]) - - targets, err := d.readUint64Slice(r, int(length)) - if err != nil { - return 0, err - } - - newNodes, changed, err := growIndexToAccomodateNode(res.Nodes, source, d.logger) - if err != nil { - return 0, err - } - - if changed { - res.Nodes = newNodes - } - - if res.Nodes[int(source)] == nil { - res.Nodes[int(source)] = &vertex{id: source, connections: make([][]uint64, level+1)} - } - - maybeGrowConnectionsForLevel(&res.Nodes[int(source)].connections, level) - res.Nodes[int(source)].connections[int(level)] = make([]uint64, len(targets)) - copy(res.Nodes[int(source)].connections[int(level)], targets) - - if keepReplaceInfo { - // mark the replace flag for this node and level, so that new commit logs - // generated on this result (condensing) do not lose information - - if _, ok := res.LinksReplaced[source]; !ok { - res.LinksReplaced[source] = map[uint16]struct{}{} - } - - res.LinksReplaced[source][level] = struct{}{} - } - - return 12 + int(length)*8, nil -} - -func (d *Deserializer) ReadAddLinks(r io.Reader, - res *DeserializationResult, -) (int, error) { - d.resetResusableBuffer(12) - _, err := io.ReadFull(r, d.reusableBuffer) - if err != nil { - return 0, err - } - - source := binary.LittleEndian.Uint64(d.reusableBuffer[0:8]) - level := binary.LittleEndian.Uint16(d.reusableBuffer[8:10]) - length := binary.LittleEndian.Uint16(d.reusableBuffer[10:12]) - - targets, err := d.readUint64Slice(r, int(length)) - if err != nil { - return 0, err - } - - newNodes, changed, err := growIndexToAccomodateNode(res.Nodes, source, d.logger) - if err != nil { - return 0, err - } - - if changed { - res.Nodes = newNodes - } - - if res.Nodes[int(source)] == nil { - res.Nodes[int(source)] = &vertex{id: source, connections: make([][]uint64, level+1)} - } - - maybeGrowConnectionsForLevel(&res.Nodes[int(source)].connections, level) - - res.Nodes[int(source)].connections[int(level)] = append( - res.Nodes[int(source)].connections[int(level)], targets...) - - return 12 + int(length)*8, nil -} - -func (d *Deserializer) ReadAddTombstone(r io.Reader, tombstones map[uint64]struct{}) error { - id, err := d.readUint64(r) - if err != nil { - return err - } - - tombstones[id] = struct{}{} - - return nil -} - -func (d *Deserializer) ReadRemoveTombstone(r io.Reader, tombstones map[uint64]struct{}) error { - id, err := d.readUint64(r) - if err != nil { - return err - } - - delete(tombstones, id) - - return nil -} - -func (d *Deserializer) ReadClearLinks(r io.Reader, res *DeserializationResult, - keepReplaceInfo bool, -) error { - id, err := d.readUint64(r) - if err != nil { - return err - } - - newNodes, changed, err := growIndexToAccomodateNode(res.Nodes, id, d.logger) - if err != nil { - return err - } - - if changed { - res.Nodes = newNodes - } - - if res.Nodes[id] == nil { - // node has been deleted or never existed, nothing to do - return nil - } - - res.Nodes[id].connections = make([][]uint64, len(res.Nodes[id].connections)) - return nil -} - -func (d *Deserializer) ReadClearLinksAtLevel(r io.Reader, res *DeserializationResult, - keepReplaceInfo bool, -) error { - id, err := d.readUint64(r) - if err != nil { - return err - } - - level, err := d.readUint16(r) - if err != nil { - return err - } - - newNodes, changed, err := growIndexToAccomodateNode(res.Nodes, id, d.logger) - if err != nil { - return err - } - - if changed { - res.Nodes = newNodes - } - - if keepReplaceInfo { - // mark the replace flag for this node and level, so that new commit logs - // generated on this result (condensing) do not lose information - - if _, ok := res.LinksReplaced[id]; !ok { - res.LinksReplaced[id] = map[uint16]struct{}{} - } - - res.LinksReplaced[id][level] = struct{}{} - } - - if res.Nodes[id] == nil { - if !keepReplaceInfo { - // node has been deleted or never existed and we are not looking at a - // single log in isolation, nothing to do - return nil - } - - // we need to keep the replace info, meaning we have to explicitly create - // this node in order to be able to store the "clear links" information for - // it - res.Nodes[id] = &vertex{ - id: id, - connections: make([][]uint64, level+1), - } - } - - if res.Nodes[id].connections == nil { - res.Nodes[id].connections = make([][]uint64, level+1) - } else { - maybeGrowConnectionsForLevel(&res.Nodes[id].connections, level) - res.Nodes[id].connections[int(level)] = []uint64{} - } - - if keepReplaceInfo { - // mark the replace flag for this node and level, so that new commit logs - // generated on this result (condensing) do not lose information - - if _, ok := res.LinksReplaced[id]; !ok { - res.LinksReplaced[id] = map[uint16]struct{}{} - } - - res.LinksReplaced[id][level] = struct{}{} - } - - return nil -} - -func (d *Deserializer) ReadDeleteNode(r io.Reader, res *DeserializationResult) error { - id, err := d.readUint64(r) - if err != nil { - return err - } - - newNodes, changed, err := growIndexToAccomodateNode(res.Nodes, id, d.logger) - if err != nil { - return err - } - - if changed { - res.Nodes = newNodes - } - - res.Nodes[id] = nil - return nil -} - -func (d *Deserializer) ReadTileEncoder(r io.Reader, res *DeserializationResult, i uint16) (compressionhelpers.PQEncoder, error) { - bins, err := d.readFloat64(r) - if err != nil { - return nil, err - } - mean, err := d.readFloat64(r) - if err != nil { - return nil, err - } - stdDev, err := d.readFloat64(r) - if err != nil { - return nil, err - } - size, err := d.readFloat64(r) - if err != nil { - return nil, err - } - s1, err := d.readFloat64(r) - if err != nil { - return nil, err - } - s2, err := d.readFloat64(r) - if err != nil { - return nil, err - } - segment, err := d.readUint16(r) - if err != nil { - return nil, err - } - encDistribution, err := d.readByte(r) - if err != nil { - return nil, err - } - return compressionhelpers.RestoreTileEncoder(bins, mean, stdDev, size, s1, s2, segment, encDistribution), nil -} - -func (d *Deserializer) ReadKMeansEncoder(r io.Reader, res *DeserializationResult, i uint16) (compressionhelpers.PQEncoder, error) { - ds := int(res.PQData.Dimensions / res.PQData.M) - centers := make([][]float32, 0, res.PQData.Ks) - for k := uint16(0); k < res.PQData.Ks; k++ { - center := make([]float32, 0, ds) - for i := 0; i < ds; i++ { - c, err := d.readFloat32(r) - if err != nil { - return nil, err - } - center = append(center, c) - } - centers = append(centers, center) - } - kms := compressionhelpers.NewKMeansWithCenters( - int(res.PQData.Ks), - ds, - int(i), - centers, - ) - return kms, nil -} - -func (d *Deserializer) ReadPQ(r io.Reader, res *DeserializationResult) error { - dims, err := d.readUint16(r) - if err != nil { - return err - } - enc, err := d.readByte(r) - if err != nil { - return err - } - ks, err := d.readUint16(r) - if err != nil { - return err - } - m, err := d.readUint16(r) - if err != nil { - return err - } - dist, err := d.readByte(r) - if err != nil { - return err - } - useBitsEncoding, err := d.readByte(r) - if err != nil { - return err - } - encoder := compressionhelpers.Encoder(enc) - res.PQData = compressionhelpers.PQData{ - Dimensions: dims, - EncoderType: encoder, - Ks: ks, - M: m, - EncoderDistribution: byte(dist), - UseBitsEncoding: useBitsEncoding != 0, - } - var encoderReader func(io.Reader, *DeserializationResult, uint16) (compressionhelpers.PQEncoder, error) - switch encoder { - case compressionhelpers.UseTileEncoder: - encoderReader = d.ReadTileEncoder - case compressionhelpers.UseKMeansEncoder: - encoderReader = d.ReadKMeansEncoder - default: - return errors.New("Unsuported encoder type") - } - for i := uint16(0); i < m; i++ { - encoder, err := encoderReader(r, res, i) - if err != nil { - return err - } - res.PQData.Encoders = append(res.PQData.Encoders, encoder) - } - res.Compressed = true - - return nil -} - -func (d *Deserializer) readUint64(r io.Reader) (uint64, error) { - var value uint64 - d.resetResusableBuffer(8) - _, err := io.ReadFull(r, d.reusableBuffer) - if err != nil { - return 0, errors.Wrap(err, "failed to read uint64") - } - - value = binary.LittleEndian.Uint64(d.reusableBuffer) - - return value, nil -} - -func (d *Deserializer) readFloat64(r io.Reader) (float64, error) { - var value float64 - d.resetResusableBuffer(8) - _, err := io.ReadFull(r, d.reusableBuffer) - if err != nil { - return 0, errors.Wrap(err, "failed to read float64") - } - - bits := binary.LittleEndian.Uint64(d.reusableBuffer) - value = math.Float64frombits(bits) - - return value, nil -} - -func (d *Deserializer) readFloat32(r io.Reader) (float32, error) { - var value float32 - d.resetResusableBuffer(4) - _, err := io.ReadFull(r, d.reusableBuffer) - if err != nil { - return 0, errors.Wrap(err, "failed to read float32") - } - - bits := binary.LittleEndian.Uint32(d.reusableBuffer) - value = math.Float32frombits(bits) - - return value, nil -} - -func (d *Deserializer) readUint16(r io.Reader) (uint16, error) { - var value uint16 - d.resetResusableBuffer(2) - _, err := io.ReadFull(r, d.reusableBuffer) - if err != nil { - return 0, errors.Wrap(err, "failed to read uint16") - } - - value = binary.LittleEndian.Uint16(d.reusableBuffer) - - return value, nil -} - -func (d *Deserializer) readByte(r io.Reader) (byte, error) { - d.resetResusableBuffer(1) - _, err := io.ReadFull(r, d.reusableBuffer) - if err != nil { - return 0, errors.Wrap(err, "failed to read byte") - } - - return d.reusableBuffer[0], nil -} - -func (d *Deserializer) ReadCommitType(r io.Reader) (HnswCommitType, error) { - d.resetResusableBuffer(1) - if _, err := io.ReadFull(r, d.reusableBuffer); err != nil { - return 0, errors.Wrap(err, "failed to read commit type") - } - - return HnswCommitType(d.reusableBuffer[0]), nil -} - -func (d *Deserializer) readUint64Slice(r io.Reader, length int) ([]uint64, error) { - d.resetResusableBuffer(length * 8) - d.resetReusableConnectionsSlice(length) - _, err := io.ReadFull(r, d.reusableBuffer) - if err != nil { - return nil, errors.Wrap(err, "failed to read uint64 slice") - } - - for i := range d.reusableConnectionsSlice { - d.reusableConnectionsSlice[i] = binary.LittleEndian.Uint64(d.reusableBuffer[i*8 : (i+1)*8]) - } - - return d.reusableConnectionsSlice, nil -} - -// If the connections array is to small to contain the current target-levelit -// will be grown. Otherwise, nothing happens. -func maybeGrowConnectionsForLevel(connsPtr *[][]uint64, level uint16) { - conns := *connsPtr - if len(conns) <= int(level) { - // we need to grow the connections slice - newConns := make([][]uint64, level+1) - copy(newConns, conns) - *connsPtr = newConns - } -} diff --git a/adapters/repos/db/vector/hnsw/deserializer_test.go b/adapters/repos/db/vector/hnsw/deserializer_test.go deleted file mode 100644 index d8c017c766ec56b0487f7193ca095d1d1fbbdc7e..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/deserializer_test.go +++ /dev/null @@ -1,419 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "bufio" - "bytes" - "encoding/binary" - "math/rand" - "testing" - - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func BenchmarkDeserializer2ReadUint64(b *testing.B) { - b.StopTimer() - - randUint64 := rand.Uint64() - - val := make([]byte, 8) - binary.LittleEndian.PutUint64(val, uint64(randUint64)) - data := bytes.NewReader(val) - logger, _ := test.NewNullLogger() - d := NewDeserializer(logger) - reader := bufio.NewReader(data) - b.StartTimer() - - for i := 0; i < b.N; i++ { - d.readUint64(reader) - } -} - -func BenchmarkDeserializer2ReadUint16(b *testing.B) { - b.StopTimer() - - randUint16 := uint16(rand.Uint32()) - - val := make([]byte, 2) - binary.LittleEndian.PutUint16(val, randUint16) - data := bytes.NewReader(val) - logger, _ := test.NewNullLogger() - d := NewDeserializer(logger) - reader := bufio.NewReader(data) - b.StartTimer() - - for i := 0; i < b.N; i++ { - d.readUint16(reader) - } -} - -func BenchmarkDeserializer2ReadCommitType(b *testing.B) { - b.StopTimer() - - commitType := SetEntryPointMaxLevel - - val := make([]byte, 1) - val[0] = byte(commitType) - data := bytes.NewReader(val) - logger, _ := test.NewNullLogger() - d := NewDeserializer(logger) - reader := bufio.NewReader(data) - b.StartTimer() - - for i := 0; i < b.N; i++ { - d.ReadCommitType(reader) - } -} - -func BenchmarkDeserializer2ReadUint64Slice(b *testing.B) { - b.StopTimer() - - uint64Slice := []uint64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10} - - val := make([]byte, len(uint64Slice)*8) - for i, v := range uint64Slice { - binary.LittleEndian.PutUint64(val[i*8:], uint64(v)) - } - - data := bytes.NewReader(val) - logger, _ := test.NewNullLogger() - d := NewDeserializer(logger) - reader := bufio.NewReader(data) - b.StartTimer() - - for i := 0; i < b.N; i++ { - d.readUint64Slice(reader, len(uint64Slice)) - } -} - -func TestDeserializer2ReadCommitType(t *testing.T) { - commitTypes := []HnswCommitType{ - AddNode, - SetEntryPointMaxLevel, - AddLinkAtLevel, - ReplaceLinksAtLevel, - AddTombstone, - RemoveTombstone, - ClearLinks, - DeleteNode, - ResetIndex, - AddPQ, - } - for _, commitType := range commitTypes { - b := make([]byte, 1) - b[0] = byte(commitType) - data := bytes.NewReader(b) - logger, _ := test.NewNullLogger() - d := NewDeserializer(logger) - reader := bufio.NewReader(data) - res, err := d.ReadCommitType(reader) - if err != nil { - t.Errorf("Error reading commit type: %v", err) - } - if res != commitType { - t.Errorf("Commit type is not equal") - } - - } -} - -func TestDeserializerReadDeleteNode(t *testing.T) { - nodes := generateDummyVertices(4) - res := &DeserializationResult{ - Nodes: nodes, - } - ids := []uint64{2, 3, 4, 5, 6} - - for _, id := range ids { - val := make([]byte, 8) - binary.LittleEndian.PutUint64(val, id) - data := bytes.NewReader(val) - logger, _ := test.NewNullLogger() - d := NewDeserializer(logger) - reader := bufio.NewReader(data) - - err := d.ReadDeleteNode(reader, res) - if err != nil { - t.Errorf("Error reading commit type: %v", err) - } - } -} - -func TestDeserializerReadClearLinks(t *testing.T) { - nodes := generateDummyVertices(4) - res := &DeserializationResult{ - Nodes: nodes, - } - ids := []uint64{2, 3, 4, 5, 6} - - for _, id := range ids { - val := make([]byte, 8) - binary.LittleEndian.PutUint64(val, id) - data := bytes.NewReader(val) - logger, _ := test.NewNullLogger() - d := NewDeserializer(logger) - - reader := bufio.NewReader(data) - - err := d.ReadClearLinks(reader, res, true) - if err != nil { - t.Errorf("Error reading links: %v", err) - } - } -} - -func dummyInitialDeserializerState() *DeserializationResult { - return &DeserializationResult{ - LinksReplaced: make(map[uint64]map[uint16]struct{}), - Nodes: []*vertex{ - nil, - nil, - { - // This is a lower level than we will read, so this node will require - // growing - level: 1, - }, - { - // This is a lower level than we will read, so this node will require - // growing - level: 8, - connections: make([][]uint64, 16), - }, - }, - } -} - -func TestDeserializerReadNode(t *testing.T) { - res := dummyInitialDeserializerState() - ids := []uint64{2, 3, 4, 5, 6} - - for _, id := range ids { - val := make([]byte, 10) - level := uint16(id * 2) - binary.LittleEndian.PutUint64(val[:8], id) - binary.LittleEndian.PutUint16(val[8:10], level) - data := bytes.NewReader(val) - logger, _ := test.NewNullLogger() - d := NewDeserializer(logger) - - reader := bufio.NewReader(data) - - err := d.ReadNode(reader, res) - require.Nil(t, err) - require.NotNil(t, res.Nodes[id]) - assert.Equal(t, int(level), res.Nodes[id].level) - } -} - -func TestDeserializerReadEP(t *testing.T) { - ids := []uint64{2, 3, 4, 5, 6} - - for _, id := range ids { - val := make([]byte, 10) - level := uint16(id * 2) - binary.LittleEndian.PutUint64(val[:8], id) - binary.LittleEndian.PutUint16(val[8:10], level) - data := bytes.NewReader(val) - logger, _ := test.NewNullLogger() - d := NewDeserializer(logger) - - reader := bufio.NewReader(data) - - ep, l, err := d.ReadEP(reader) - require.Nil(t, err) - assert.Equal(t, id, ep) - assert.Equal(t, level, l) - } -} - -func TestDeserializerReadLink(t *testing.T) { - res := dummyInitialDeserializerState() - ids := []uint64{2, 3, 4, 5, 6} - - for _, id := range ids { - level := uint16(id * 2) - target := id * 3 - val := make([]byte, 18) - binary.LittleEndian.PutUint64(val[:8], id) - binary.LittleEndian.PutUint16(val[8:10], level) - binary.LittleEndian.PutUint64(val[10:18], target) - data := bytes.NewReader(val) - logger, _ := test.NewNullLogger() - d := NewDeserializer(logger) - - reader := bufio.NewReader(data) - - err := d.ReadLink(reader, res) - require.Nil(t, err) - require.NotNil(t, res.Nodes[id]) - lastAddedConnection := res.Nodes[id].connections[level][len(res.Nodes[id].connections[level])-1] - assert.Equal(t, target, lastAddedConnection) - } -} - -func TestDeserializerReadLinks(t *testing.T) { - res := dummyInitialDeserializerState() - ids := []uint64{2, 3, 4, 5, 6} - - for _, id := range ids { - level := uint16(id * 2) - connLen := uint16(id * 4) - val := make([]byte, 12+connLen*8) - binary.LittleEndian.PutUint64(val[:8], id) - binary.LittleEndian.PutUint16(val[8:10], level) - binary.LittleEndian.PutUint16(val[10:12], connLen) - for i := 0; i < int(connLen); i++ { - target := id + uint64(i) - binary.LittleEndian.PutUint64(val[12+(i*8):12+(i*8+8)], target) - } - data := bytes.NewReader(val) - logger, _ := test.NewNullLogger() - d := NewDeserializer(logger) - - reader := bufio.NewReader(data) - - _, err := d.ReadLinks(reader, res, true) - require.Nil(t, err) - require.NotNil(t, res.Nodes[id]) - lastAddedConnection := res.Nodes[id].connections[level][len(res.Nodes[id].connections[level])-1] - assert.Equal(t, id+uint64(connLen)-1, lastAddedConnection) - } -} - -func TestDeserializerReadAddLinks(t *testing.T) { - res := dummyInitialDeserializerState() - ids := []uint64{2, 3, 4, 5, 6} - - for _, id := range ids { - level := uint16(id * 2) - connLen := uint16(id * 4) - val := make([]byte, 12+connLen*8) - binary.LittleEndian.PutUint64(val[:8], id) - binary.LittleEndian.PutUint16(val[8:10], level) - binary.LittleEndian.PutUint16(val[10:12], connLen) - for i := 0; i < int(connLen); i++ { - target := id + uint64(i) - binary.LittleEndian.PutUint64(val[12+(i*8):12+(i*8+8)], target) - } - data := bytes.NewReader(val) - logger, _ := test.NewNullLogger() - d := NewDeserializer(logger) - - reader := bufio.NewReader(data) - - _, err := d.ReadAddLinks(reader, res) - require.Nil(t, err) - require.NotNil(t, res.Nodes[id]) - lastAddedConnection := res.Nodes[id].connections[level][len(res.Nodes[id].connections[level])-1] - assert.Equal(t, id+uint64(connLen)-1, lastAddedConnection) - } -} - -func TestDeserializerAddTombstone(t *testing.T) { - tombstones := map[uint64]struct{}{} - ids := []uint64{2, 3, 4, 5, 6} - - for _, id := range ids { - val := make([]byte, 8) - binary.LittleEndian.PutUint64(val[:8], id) - data := bytes.NewReader(val) - logger, _ := test.NewNullLogger() - d := NewDeserializer(logger) - - reader := bufio.NewReader(data) - - err := d.ReadAddTombstone(reader, tombstones) - require.Nil(t, err) - } - - expected := map[uint64]struct{}{ - 2: {}, - 3: {}, - 4: {}, - 5: {}, - 6: {}, - } - - assert.Equal(t, expected, tombstones) -} - -func TestDeserializerRemoveTombstone(t *testing.T) { - tombstones := map[uint64]struct{}{ - 1: {}, - 2: {}, - 3: {}, - 4: {}, - 5: {}, - } - ids := []uint64{2, 3, 4, 5, 6} - - for _, id := range ids { - val := make([]byte, 8) - binary.LittleEndian.PutUint64(val[:8], id) - data := bytes.NewReader(val) - logger, _ := test.NewNullLogger() - d := NewDeserializer(logger) - - reader := bufio.NewReader(data) - - err := d.ReadRemoveTombstone(reader, tombstones) - require.Nil(t, err) - } - - expected := map[uint64]struct{}{ - 1: {}, - } - - assert.Equal(t, expected, tombstones) -} - -func TestDeserializerClearLinksAtLevel(t *testing.T) { - res := &DeserializationResult{ - LinksReplaced: make(map[uint64]map[uint16]struct{}), - Nodes: []*vertex{ - nil, - nil, - { - // This is a lower level than we will read, so this node will require - // growing - level: 1, - }, - { - // This is a lower level than we will read, so this node will require - // growing - level: 4, - connections: make([][]uint64, 4), - }, - nil, - nil, - }, - } - ids := []uint64{2, 3, 4, 5, 6} - - for _, id := range ids { - level := uint16(id * 2) - val := make([]byte, 10) - binary.LittleEndian.PutUint64(val[:8], id) - binary.LittleEndian.PutUint16(val[8:10], level) - data := bytes.NewReader(val) - logger, _ := test.NewNullLogger() - d := NewDeserializer(logger) - - reader := bufio.NewReader(data) - - err := d.ReadClearLinksAtLevel(reader, res, true) - require.Nil(t, err) - } -} diff --git a/adapters/repos/db/vector/hnsw/distancer/asm/doc.go b/adapters/repos/db/vector/hnsw/distancer/asm/doc.go deleted file mode 100644 index 0a2e8623eb597c59ad219424fb82b251b0ba72e6..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/asm/doc.go +++ /dev/null @@ -1,13 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// asm only has amd64 specific implementations at the moment -package asm diff --git a/adapters/repos/db/vector/hnsw/distancer/asm/dot.go b/adapters/repos/db/vector/hnsw/distancer/asm/dot.go deleted file mode 100644 index 87ac214dcaa221b1fed8c982e4d5b81f876af8c5..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/asm/dot.go +++ /dev/null @@ -1,112 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build ignore -// +build ignore - -package main - -import ( - . "github.com/mmcloughlin/avo/build" - . "github.com/mmcloughlin/avo/operand" - . "github.com/mmcloughlin/avo/reg" -) - -var unroll = 4 - -// inspired by the avo example which is itself inspired by the PeachPy example. -// Adjusted unrolling that fits our use cases better. -func main() { - TEXT("Dot", NOSPLIT, "func(x, y []float32) float32") - x := Mem{Base: Load(Param("x").Base(), GP64())} - y := Mem{Base: Load(Param("y").Base(), GP64())} - n := Load(Param("x").Len(), GP64()) - - acc := make([]VecVirtual, unroll) - for i := 0; i < unroll; i++ { - acc[i] = YMM() - } - - for i := 0; i < unroll; i++ { - VXORPS(acc[i], acc[i], acc[i]) - } - - blockitems := 8 * unroll - blocksize := 4 * blockitems - Label("blockloop") - CMPQ(n, U32(blockitems)) - JL(LabelRef("tail")) - - // Load x. - xs := make([]VecVirtual, unroll) - for i := 0; i < unroll; i++ { - xs[i] = YMM() - } - - for i := 0; i < unroll; i++ { - VMOVUPS(x.Offset(32*i), xs[i]) - } - - // The actual FMA. - for i := 0; i < unroll; i++ { - VFMADD231PS(y.Offset(32*i), xs[i], acc[i]) - } - - ADDQ(U32(blocksize), x.Base) - ADDQ(U32(blocksize), y.Base) - SUBQ(U32(blockitems), n) - JMP(LabelRef("blockloop")) - - // Process any trailing entries. - Label("tail") - tail := XMM() - VXORPS(tail, tail, tail) - - Label("tailloop") - CMPQ(n, U32(0)) - JE(LabelRef("reduce")) - - xt := XMM() - VMOVSS(x, xt) - VFMADD231SS(y, xt, tail) - - ADDQ(U32(4), x.Base) - ADDQ(U32(4), y.Base) - DECQ(n) - JMP(LabelRef("tailloop")) - - // Reduce the lanes to one. - Label("reduce") - if unroll != 4 { - // we have hard-coded the reduction for this specific unrolling as it - // allows us to do 0+1 and 2+3 and only then have a multiplication which - // touches both. - panic("addition is hard-coded") - } - - // Manual reduction - VADDPS(acc[0], acc[1], acc[0]) - VADDPS(acc[2], acc[3], acc[2]) - VADDPS(acc[0], acc[2], acc[0]) - - result := acc[0].AsX() - top := XMM() - VEXTRACTF128(U8(1), acc[0], top) - VADDPS(result, top, result) - VADDPS(result, tail, result) - VHADDPS(result, result, result) - VHADDPS(result, result, result) - Store(result, ReturnIndex(0)) - - RET() - - Generate() -} diff --git a/adapters/repos/db/vector/hnsw/distancer/asm/dot_amd64.s b/adapters/repos/db/vector/hnsw/distancer/asm/dot_amd64.s deleted file mode 100644 index 41eb3a8915433a441e8b51fbff82433bcbfaeeeb..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/asm/dot_amd64.s +++ /dev/null @@ -1,55 +0,0 @@ -// Code generated by command: go run dot.go -out dot.s -stubs dot_stub.go. DO NOT EDIT. - -#include "textflag.h" - -// func Dot(x []float32, y []float32) float32 -// Requires: AVX, FMA3, SSE -TEXT ·Dot(SB), NOSPLIT, $0-52 - MOVQ x_base+0(FP), AX - MOVQ y_base+24(FP), CX - MOVQ x_len+8(FP), DX - VXORPS Y0, Y0, Y0 - VXORPS Y1, Y1, Y1 - VXORPS Y2, Y2, Y2 - VXORPS Y3, Y3, Y3 - -blockloop: - CMPQ DX, $0x00000020 - JL tail - VMOVUPS (AX), Y4 - VMOVUPS 32(AX), Y5 - VMOVUPS 64(AX), Y6 - VMOVUPS 96(AX), Y7 - VFMADD231PS (CX), Y4, Y0 - VFMADD231PS 32(CX), Y5, Y1 - VFMADD231PS 64(CX), Y6, Y2 - VFMADD231PS 96(CX), Y7, Y3 - ADDQ $0x00000080, AX - ADDQ $0x00000080, CX - SUBQ $0x00000020, DX - JMP blockloop - -tail: - VXORPS X4, X4, X4 - -tailloop: - CMPQ DX, $0x00000000 - JE reduce - VMOVSS (AX), X5 - VFMADD231SS (CX), X5, X4 - ADDQ $0x00000004, AX - ADDQ $0x00000004, CX - DECQ DX - JMP tailloop - -reduce: - VADDPS Y0, Y1, Y0 - VADDPS Y2, Y3, Y2 - VADDPS Y0, Y2, Y0 - VEXTRACTF128 $0x01, Y0, X1 - VADDPS X0, X1, X0 - VADDPS X0, X4, X0 - VHADDPS X0, X0, X0 - VHADDPS X0, X0, X0 - MOVSS X0, ret+48(FP) - RET diff --git a/adapters/repos/db/vector/hnsw/distancer/asm/dot_arm64.go b/adapters/repos/db/vector/hnsw/distancer/asm/dot_arm64.go deleted file mode 100644 index 9735aba3fe982acce8324dc914cb31f0318441c4..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/asm/dot_arm64.go +++ /dev/null @@ -1,21 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build !noasm && arm64 - -// AUTO-GENERATED BY GOAT -- DO NOT EDIT - -package asm - -import "unsafe" - -//go:noescape -func dot(a, b, res, len unsafe.Pointer) diff --git a/adapters/repos/db/vector/hnsw/distancer/asm/dot_arm64.s b/adapters/repos/db/vector/hnsw/distancer/asm/dot_arm64.s deleted file mode 100644 index 75825e278480dc5f91ae0ae20feddad3177406c8..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/asm/dot_arm64.s +++ /dev/null @@ -1,138 +0,0 @@ -//go:build !noasm && arm64 -// AUTO-GENERATED BY GOAT -- DO NOT EDIT - -TEXT ·dot(SB), $0-32 - MOVD a+0(FP), R0 - MOVD b+8(FP), R1 - MOVD res+16(FP), R2 - MOVD len+24(FP), R3 - WORD $0xa9bf7bfd // stp x29, x30, [sp, - WORD $0xf9400068 // ldr x8, [x3] - WORD $0x910003fd // mov x29, sp - WORD $0x6b0803e9 // negs w9, w8 - WORD $0x1200050a // and w10, w8, - WORD $0x12000529 // and w9, w9, - WORD $0x5a89454a // csneg w10, w10, w9, mi - WORD $0x4b0a0109 // sub w9, w8, w10 - WORD $0x7100413f // cmp w9, - WORD $0x540000ea // b.ge .LBB0_2 - WORD $0x6f00e400 // movi v0.2d, - WORD $0x2a1f03eb // mov w11, wzr - WORD $0x6f00e401 // movi v1.2d, - WORD $0x6f00e403 // movi v3.2d, - WORD $0x6f00e402 // movi v2.2d, - WORD $0x14000016 // b .LBB0_4 - -LBB0_2: - WORD $0x6f00e402 // movi v2.2d, - WORD $0xaa1f03eb // mov x11, xzr - WORD $0x6f00e403 // movi v3.2d, - WORD $0xaa0003ec // mov x12, x0 - WORD $0x6f00e401 // movi v1.2d, - WORD $0xaa0103ed // mov x13, x1 - WORD $0x6f00e400 // movi v0.2d, - -LBB0_3: - WORD $0x4cdf2984 // ld1 { v4.4s, v5.4s, v6.4s, v7.4s }, [x12], - WORD $0x9100816e // add x14, x11, - WORD $0x9100416b // add x11, x11, - WORD $0xeb0901df // cmp x14, x9 - WORD $0x4cdf29b0 // ld1 { v16.4s, v17.4s, v18.4s, v19.4s }, [x13], - WORD $0x6e30dc94 // fmul v20.4s, v4.4s, v16.4s - WORD $0x6e31dcb5 // fmul v21.4s, v5.4s, v17.4s - WORD $0x6e32dcd6 // fmul v22.4s, v6.4s, v18.4s - WORD $0x6e33dce4 // fmul v4.4s, v7.4s, v19.4s - WORD $0x4e34d442 // fadd v2.4s, v2.4s, v20.4s - WORD $0x4e35d463 // fadd v3.4s, v3.4s, v21.4s - WORD $0x4e36d421 // fadd v1.4s, v1.4s, v22.4s - WORD $0x4e24d400 // fadd v0.4s, v0.4s, v4.4s - WORD $0x54fffe69 // b.ls .LBB0_3 - -LBB0_4: - WORD $0x6b09017f // cmp w11, w9 - WORD $0x540001ca // b.ge .LBB0_7 - WORD $0x2a0b03eb // mov w11, w11 - WORD $0x2a0903ec // mov w12, w9 - WORD $0xd37ef56e // lsl x14, x11, - WORD $0x93407d8c // sxtw x12, w12 - WORD $0x8b0e002d // add x13, x1, x14 - WORD $0x8b0e000e // add x14, x0, x14 - -LBB0_6: - WORD $0x3cc105c4 // ldr q4, [x14], - WORD $0x3cc105a5 // ldr q5, [x13], - WORD $0x9100116b // add x11, x11, - WORD $0xeb0c017f // cmp x11, x12 - WORD $0x6e25dc84 // fmul v4.4s, v4.4s, v5.4s - WORD $0x4e24d442 // fadd v2.4s, v2.4s, v4.4s - WORD $0x54ffff4b // b.lt .LBB0_6 - -LBB0_7: - WORD $0x6e22d442 // faddp v2.4s, v2.4s, v2.4s - WORD $0x7100055f // cmp w10, - WORD $0x6e23d463 // faddp v3.4s, v3.4s, v3.4s - WORD $0x6e21d421 // faddp v1.4s, v1.4s, v1.4s - WORD $0x6e20d400 // faddp v0.4s, v0.4s, v0.4s - WORD $0x7e30d842 // faddp s2, v2.2s - WORD $0x7e30d863 // faddp s3, v3.2s - WORD $0x7e30d821 // faddp s1, v1.2s - WORD $0x7e30d800 // faddp s0, v0.2s - WORD $0x1e232842 // fadd s2, s2, s3 - WORD $0x1e212841 // fadd s1, s2, s1 - WORD $0x1e202820 // fadd s0, s1, s0 - WORD $0x540005eb // b.lt .LBB0_13 - WORD $0x93407d08 // sxtw x8, w8 - WORD $0x93407d29 // sxtw x9, w9 - WORD $0x9100052a // add x10, x9, - WORD $0xeb08015f // cmp x10, x8 - WORD $0x9a89d50a // csinc x10, x8, x9, le - WORD $0xcb09014a // sub x10, x10, x9 - WORD $0xf100215f // cmp x10, - WORD $0x54000403 // b.lo .LBB0_12 - WORD $0xd37ef52c // lsl x12, x9, - WORD $0x927df14b // and x11, x10, - WORD $0x9100418d // add x13, x12, - WORD $0x8b090169 // add x9, x11, x9 - WORD $0x8b0d000c // add x12, x0, x13 - WORD $0x8b0d002d // add x13, x1, x13 - WORD $0xaa0b03ee // mov x14, x11 - -LBB0_10: - WORD $0x3cdf0181 // ldur q1, [x12, - WORD $0xf10021ce // subs x14, x14, - WORD $0x3cdf01a2 // ldur q2, [x13, - WORD $0x6e22dc21 // fmul v1.4s, v1.4s, v2.4s - WORD $0x5e0c0422 // mov s2, v1.s[1] - WORD $0x1e212800 // fadd s0, s0, s1 - WORD $0x5e140423 // mov s3, v1.s[2] - WORD $0x5e1c0421 // mov s1, v1.s[3] - WORD $0x1e222800 // fadd s0, s0, s2 - WORD $0x3cc20582 // ldr q2, [x12], - WORD $0x1e232800 // fadd s0, s0, s3 - WORD $0x3cc205a3 // ldr q3, [x13], - WORD $0x6e23dc42 // fmul v2.4s, v2.4s, v3.4s - WORD $0x1e212800 // fadd s0, s0, s1 - WORD $0x5e0c0441 // mov s1, v2.s[1] - WORD $0x1e222800 // fadd s0, s0, s2 - WORD $0x5e140443 // mov s3, v2.s[2] - WORD $0x1e212800 // fadd s0, s0, s1 - WORD $0x5e1c0441 // mov s1, v2.s[3] - WORD $0x1e232800 // fadd s0, s0, s3 - WORD $0x1e212800 // fadd s0, s0, s1 - WORD $0x54fffd61 // b.ne .LBB0_10 - WORD $0xeb0b015f // cmp x10, x11 - WORD $0x54000100 // b.eq .LBB0_13 - -LBB0_12: - WORD $0xd37ef52a // lsl x10, x9, - WORD $0x91000529 // add x9, x9, - WORD $0xeb08013f // cmp x9, x8 - WORD $0xbc6a6801 // ldr s1, [x0, x10] - WORD $0xbc6a6822 // ldr s2, [x1, x10] - WORD $0x1f020020 // fmadd s0, s1, s2, s0 - WORD $0x54ffff4b // b.lt .LBB0_12 - -LBB0_13: - WORD $0xbd000040 // str s0, [x2] - WORD $0xa8c17bfd // ldp x29, x30, [sp], - WORD $0xd65f03c0 // ret diff --git a/adapters/repos/db/vector/hnsw/distancer/asm/dot_inline.go b/adapters/repos/db/vector/hnsw/distancer/asm/dot_inline.go deleted file mode 100644 index 7c24b158a0ec8a5070b4c7482e0cd42c4ff99bf3..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/asm/dot_inline.go +++ /dev/null @@ -1,55 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package asm - -// Experiment with inlining and flattening the L2Squared distancer. -// Theoretically, this should be faster than the loop version for small vectors -// - it avoids the loop overhead -// - it eliminates the bounds check by reversing the iteration -// - it allows dot2, dot4 and dot6 to be inlined (the other ones are too large) -// See go tool compile -d=ssa/check_bce/debug=1 -m dot_inline.go - -func dot2(x []float32, y []float32) float32 { - sum := x[1]*y[1] + x[0]*y[0] - - return sum -} - -func dot4(x []float32, y []float32) float32 { - sum := x[3]*y[3] + x[2]*y[2] - - return dot2(x, y) + sum -} - -func dot6(x []float32, y []float32) float32 { - sum := x[5]*y[5] + x[4]*y[4] - - return dot4(x, y) + sum -} - -func dot8(x []float32, y []float32) float32 { - sum := x[7]*y[7] + x[6]*y[6] - - return dot6(x, y) + sum -} - -func dot10(x []float32, y []float32) float32 { - sum := x[9]*y[9] + x[8]*y[8] + x[7]*y[7] + x[6]*y[6] - - return dot6(x, y) + sum -} - -func dot12(x []float32, y []float32) float32 { - sum := x[11]*y[11] + x[10]*y[10] + x[9]*y[9] + x[8]*y[8] + x[7]*y[7] + x[6]*y[6] - - return dot6(x, y) + sum -} diff --git a/adapters/repos/db/vector/hnsw/distancer/asm/dot_inline_test.go b/adapters/repos/db/vector/hnsw/distancer/asm/dot_inline_test.go deleted file mode 100644 index 8100db423a4de9718701df613b162aab8af075a6..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/asm/dot_inline_test.go +++ /dev/null @@ -1,87 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package asm - -import ( - "fmt" - "math/rand" - "testing" -) - -func dotLoop(x, y []float32) float32 { - var sum float32 - for i := range x { - sum += x[i] * y[i] - } - - return sum -} - -func BenchmarkDotInlineVsLoop(b *testing.B) { - lengths := []int{2, 4, 6, 8, 10, 12} - for _, length := range lengths { - x := make([]float32, length) - y := make([]float32, length) - - for i := range x { - x[i] = rand.Float32() - y[i] = rand.Float32() - } - - b.Run(fmt.Sprintf("vector dim=%d", length), func(b *testing.B) { - b.Run("loop", func(b *testing.B) { - for i := 0; i < b.N; i++ { - dotLoop(x, y) - } - }) - - b.Run("flat", func(b *testing.B) { - // written to ensure that the compiler - // inlines the function when possible - switch length { - case 2: - b.ResetTimer() - for i := 0; i < b.N; i++ { - dot2(x, y) - } - case 4: - b.ResetTimer() - for i := 0; i < b.N; i++ { - dot4(x, y) - } - case 6: - b.ResetTimer() - for i := 0; i < b.N; i++ { - dot6(x, y) - } - case 8: - b.ResetTimer() - for i := 0; i < b.N; i++ { - dot8(x, y) - } - case 10: - b.ResetTimer() - for i := 0; i < b.N; i++ { - dot10(x, y) - } - case 12: - b.ResetTimer() - for i := 0; i < b.N; i++ { - dot12(x, y) - } - default: - panic("unsupported length") - } - }) - }) - } -} diff --git a/adapters/repos/db/vector/hnsw/distancer/asm/dot_stub_amd64.go b/adapters/repos/db/vector/hnsw/distancer/asm/dot_stub_amd64.go deleted file mode 100644 index 73c4a9ee1345a7093fc65cb7e02721162d00d410..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/asm/dot_stub_amd64.go +++ /dev/null @@ -1,16 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by command: go run dot.go -out dot.s -stubs dot_stub.go. DO NOT EDIT. - -package asm - -func Dot(x []float32, y []float32) float32 diff --git a/adapters/repos/db/vector/hnsw/distancer/asm/dot_stub_arm64.go b/adapters/repos/db/vector/hnsw/distancer/asm/dot_stub_arm64.go deleted file mode 100644 index 2440c2fc294e594570c5e2079d9f875f156c0592..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/asm/dot_stub_arm64.go +++ /dev/null @@ -1,66 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package asm - -// To generate the asm code, run: -// go install github.com/gorse-io/goat@v0.1.0 -// go generate - -//go:generate goat ../c/dot_arm64.c -O3 -e="-mfpu=neon-fp-armv8" -e="-mfloat-abi=hard" -e="--target=arm64" -e="-march=armv8-a+simd+fp" - -import ( - "reflect" - "unsafe" -) - -// Dot calculates the dot product between two vectors -// using SIMD instructions. -func Dot(x []float32, y []float32) float32 { - switch len(x) { - case 2: - return dot2(x, y) - case 4: - return dot4(x, y) - case 6: - return dot6(x, y) - case 8: - // manually inlined dot8(x, y) - sum := x[7]*y[7] + x[6]*y[6] - return dot6(x, y) + sum - case 10: - // manually inlined dot10(x, y) - sum := x[9]*y[9] + x[8]*y[8] + x[7]*y[7] + x[6]*y[6] - return dot6(x, y) + sum - case 12: - // manually inlined dot12(x, y) - sum := x[11]*y[11] + x[10]*y[10] + x[9]*y[9] + x[8]*y[8] + x[7]*y[7] + x[6]*y[6] - return dot6(x, y) + sum - } - - var res float32 - - // The C function expects pointers to the underlying array, not slices. - hdrx := (*reflect.SliceHeader)(unsafe.Pointer(&x)) - hdry := (*reflect.SliceHeader)(unsafe.Pointer(&y)) - - l := len(x) - dot( - // The slice header contains the address of the underlying array. - // We only need to cast it to a pointer. - unsafe.Pointer(hdrx.Data), - unsafe.Pointer(hdry.Data), - // The C function expects pointers to the result and the length of the arrays. - unsafe.Pointer(&res), - unsafe.Pointer(&l)) - - return res -} diff --git a/adapters/repos/db/vector/hnsw/distancer/asm/l2.go b/adapters/repos/db/vector/hnsw/distancer/asm/l2.go deleted file mode 100644 index 7d24d02650966ad05f1223fc4ad8468408c0ebd0..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/asm/l2.go +++ /dev/null @@ -1,120 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build ignore -// +build ignore - -package main - -import ( - . "github.com/mmcloughlin/avo/build" - . "github.com/mmcloughlin/avo/operand" - . "github.com/mmcloughlin/avo/reg" -) - -var unroll = 4 - -func main() { - TEXT("L2", NOSPLIT, "func(x, y []float32) float32") - x := Mem{Base: Load(Param("x").Base(), GP64())} - y := Mem{Base: Load(Param("y").Base(), GP64())} - n := Load(Param("x").Len(), GP64()) - - acc := make([]VecVirtual, unroll) - diff := make([]VecVirtual, unroll) - for i := 0; i < unroll; i++ { - acc[i] = YMM() - diff[i] = YMM() - } - - for i := 0; i < unroll; i++ { - VXORPS(acc[i], acc[i], acc[i]) - VXORPS(diff[i], diff[i], diff[i]) - } - - blockitems := 8 * unroll - blocksize := 4 * blockitems - Label("blockloop") - CMPQ(n, U32(blockitems)) - JL(LabelRef("tail")) - - // Load x. - xs := make([]VecVirtual, unroll) - for i := 0; i < unroll; i++ { - xs[i] = YMM() - } - - for i := 0; i < unroll; i++ { - VMOVUPS(x.Offset(32*i), xs[i]) - } - - for i := 0; i < unroll; i++ { - VSUBPS(y.Offset(32*i), xs[i], diff[i]) - } - - for i := 0; i < unroll; i++ { - VFMADD231PS(diff[i], diff[i], acc[i]) - } - - ADDQ(U32(blocksize), x.Base) - ADDQ(U32(blocksize), y.Base) - SUBQ(U32(blockitems), n) - JMP(LabelRef("blockloop")) - - // Process any trailing entries. - Label("tail") - tail := XMM() - VXORPS(tail, tail, tail) - - Label("tailloop") - CMPQ(n, U32(0)) - JE(LabelRef("reduce")) - - xt := XMM() - VMOVSS(x, xt) - - difft := XMM() - VSUBSS(y, xt, difft) - - VFMADD231SS(difft, difft, tail) - - ADDQ(U32(4), x.Base) - ADDQ(U32(4), y.Base) - DECQ(n) - JMP(LabelRef("tailloop")) - - // Reduce the lanes to one. - Label("reduce") - if unroll != 4 { - // we have hard-coded the reduction for this specific unrolling as it - // allows us to do 0+1 and 2+3 and only then have a multiplication which - // touches both. - panic("addition is hard-coded") - } - - // Manual reduction - VADDPS(acc[0], acc[1], acc[0]) - VADDPS(acc[2], acc[3], acc[2]) - VADDPS(acc[0], acc[2], acc[0]) - - result := acc[0].AsX() - top := XMM() - VEXTRACTF128(U8(1), acc[0], top) - VADDPS(result, top, result) - VADDPS(result, tail, result) - VHADDPS(result, result, result) - VHADDPS(result, result, result) - Store(result, ReturnIndex(0)) - - RET() - - Generate() -} diff --git a/adapters/repos/db/vector/hnsw/distancer/asm/l2_amd64.s b/adapters/repos/db/vector/hnsw/distancer/asm/l2_amd64.s deleted file mode 100644 index 9b5b89a0ca970df9d523b57e258f703cb03defc1..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/asm/l2_amd64.s +++ /dev/null @@ -1,64 +0,0 @@ -// Code generated by command: go run l2.go -out l2_amd64.s -stubs l2_stub_amd64.go. DO NOT EDIT. - -#include "textflag.h" - -// func L2(x []float32, y []float32) float32 -// Requires: AVX, FMA3, SSE -TEXT ·L2(SB), NOSPLIT, $0-52 - MOVQ x_base+0(FP), AX - MOVQ y_base+24(FP), CX - MOVQ x_len+8(FP), DX - VXORPS Y0, Y0, Y0 - VXORPS Y1, Y1, Y1 - VXORPS Y2, Y2, Y2 - VXORPS Y3, Y3, Y3 - VXORPS Y4, Y4, Y4 - VXORPS Y5, Y5, Y5 - VXORPS Y6, Y6, Y6 - VXORPS Y7, Y7, Y7 - -blockloop: - CMPQ DX, $0x00000020 - JL tail - VMOVUPS (AX), Y1 - VMOVUPS 32(AX), Y3 - VMOVUPS 64(AX), Y5 - VMOVUPS 96(AX), Y7 - VSUBPS (CX), Y1, Y1 - VSUBPS 32(CX), Y3, Y3 - VSUBPS 64(CX), Y5, Y5 - VSUBPS 96(CX), Y7, Y7 - VFMADD231PS Y1, Y1, Y0 - VFMADD231PS Y3, Y3, Y2 - VFMADD231PS Y5, Y5, Y4 - VFMADD231PS Y7, Y7, Y6 - ADDQ $0x00000080, AX - ADDQ $0x00000080, CX - SUBQ $0x00000020, DX - JMP blockloop - -tail: - VXORPS X1, X1, X1 - -tailloop: - CMPQ DX, $0x00000000 - JE reduce - VMOVSS (AX), X3 - VSUBSS (CX), X3, X3 - VFMADD231SS X3, X3, X1 - ADDQ $0x00000004, AX - ADDQ $0x00000004, CX - DECQ DX - JMP tailloop - -reduce: - VADDPS Y0, Y2, Y0 - VADDPS Y4, Y6, Y4 - VADDPS Y0, Y4, Y0 - VEXTRACTF128 $0x01, Y0, X2 - VADDPS X0, X2, X0 - VADDPS X0, X1, X0 - VHADDPS X0, X0, X0 - VHADDPS X0, X0, X0 - MOVSS X0, ret+48(FP) - RET diff --git a/adapters/repos/db/vector/hnsw/distancer/asm/l2_arm64.go b/adapters/repos/db/vector/hnsw/distancer/asm/l2_arm64.go deleted file mode 100644 index 1532dacffa17fced4337519b270c6c99aeba3e95..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/asm/l2_arm64.go +++ /dev/null @@ -1,21 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build !noasm && arm64 - -// AUTO-GENERATED BY GOAT -- DO NOT EDIT - -package asm - -import "unsafe" - -//go:noescape -func l2(a, b, res, len unsafe.Pointer) diff --git a/adapters/repos/db/vector/hnsw/distancer/asm/l2_arm64.s b/adapters/repos/db/vector/hnsw/distancer/asm/l2_arm64.s deleted file mode 100644 index 3169439d5a6e6c6e176063eea3203513014a9511..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/asm/l2_arm64.s +++ /dev/null @@ -1,147 +0,0 @@ -//go:build !noasm && arm64 -// AUTO-GENERATED BY GOAT -- DO NOT EDIT - -TEXT ·l2(SB), $0-32 - MOVD a+0(FP), R0 - MOVD b+8(FP), R1 - MOVD res+16(FP), R2 - MOVD len+24(FP), R3 - WORD $0xa9bf7bfd // stp x29, x30, [sp, - WORD $0xf9400068 // ldr x8, [x3] - WORD $0x910003fd // mov x29, sp - WORD $0x6b0803e9 // negs w9, w8 - WORD $0x1200050a // and w10, w8, - WORD $0x12000529 // and w9, w9, - WORD $0x5a89454a // csneg w10, w10, w9, mi - WORD $0x4b0a0109 // sub w9, w8, w10 - WORD $0x7100413f // cmp w9, - WORD $0x540000ea // b.ge .LBB0_2 - WORD $0x6f00e400 // movi v0.2d, - WORD $0x2a1f03eb // mov w11, wzr - WORD $0x6f00e401 // movi v1.2d, - WORD $0x6f00e403 // movi v3.2d, - WORD $0x6f00e402 // movi v2.2d, - WORD $0x1400001a // b .LBB0_4 - -LBB0_2: - WORD $0x6f00e402 // movi v2.2d, - WORD $0xaa1f03eb // mov x11, xzr - WORD $0x6f00e403 // movi v3.2d, - WORD $0xaa0003ec // mov x12, x0 - WORD $0x6f00e401 // movi v1.2d, - WORD $0xaa0103ed // mov x13, x1 - WORD $0x6f00e400 // movi v0.2d, - -LBB0_3: - WORD $0x4cdf2984 // ld1 { v4.4s, v5.4s, v6.4s, v7.4s }, [x12], - WORD $0x9100816e // add x14, x11, - WORD $0x9100416b // add x11, x11, - WORD $0xeb0901df // cmp x14, x9 - WORD $0x4cdf29b0 // ld1 { v16.4s, v17.4s, v18.4s, v19.4s }, [x13], - WORD $0x4eb0d494 // fsub v20.4s, v4.4s, v16.4s - WORD $0x4eb1d4b5 // fsub v21.4s, v5.4s, v17.4s - WORD $0x4eb2d4d6 // fsub v22.4s, v6.4s, v18.4s - WORD $0x4eb3d4e4 // fsub v4.4s, v7.4s, v19.4s - WORD $0x6e34de85 // fmul v5.4s, v20.4s, v20.4s - WORD $0x6e35dea6 // fmul v6.4s, v21.4s, v21.4s - WORD $0x6e36dec7 // fmul v7.4s, v22.4s, v22.4s - WORD $0x6e24dc84 // fmul v4.4s, v4.4s, v4.4s - WORD $0x4e25d442 // fadd v2.4s, v2.4s, v5.4s - WORD $0x4e26d463 // fadd v3.4s, v3.4s, v6.4s - WORD $0x4e27d421 // fadd v1.4s, v1.4s, v7.4s - WORD $0x4e24d400 // fadd v0.4s, v0.4s, v4.4s - WORD $0x54fffde9 // b.ls .LBB0_3 - -LBB0_4: - WORD $0x6b09017f // cmp w11, w9 - WORD $0x540001ea // b.ge .LBB0_7 - WORD $0x2a0b03eb // mov w11, w11 - WORD $0x2a0903ec // mov w12, w9 - WORD $0xd37ef56e // lsl x14, x11, - WORD $0x93407d8c // sxtw x12, w12 - WORD $0x8b0e002d // add x13, x1, x14 - WORD $0x8b0e000e // add x14, x0, x14 - -LBB0_6: - WORD $0x3cc105c4 // ldr q4, [x14], - WORD $0x3cc105a5 // ldr q5, [x13], - WORD $0x9100116b // add x11, x11, - WORD $0xeb0c017f // cmp x11, x12 - WORD $0x4ea5d484 // fsub v4.4s, v4.4s, v5.4s - WORD $0x6e24dc84 // fmul v4.4s, v4.4s, v4.4s - WORD $0x4e24d442 // fadd v2.4s, v2.4s, v4.4s - WORD $0x54ffff2b // b.lt .LBB0_6 - -LBB0_7: - WORD $0x6e22d442 // faddp v2.4s, v2.4s, v2.4s - WORD $0x7100055f // cmp w10, - WORD $0x6e23d463 // faddp v3.4s, v3.4s, v3.4s - WORD $0x6e21d421 // faddp v1.4s, v1.4s, v1.4s - WORD $0x6e20d400 // faddp v0.4s, v0.4s, v0.4s - WORD $0x7e30d842 // faddp s2, v2.2s - WORD $0x7e30d863 // faddp s3, v3.2s - WORD $0x7e30d821 // faddp s1, v1.2s - WORD $0x7e30d800 // faddp s0, v0.2s - WORD $0x1e232842 // fadd s2, s2, s3 - WORD $0x1e212841 // fadd s1, s2, s1 - WORD $0x1e202820 // fadd s0, s1, s0 - WORD $0x5400066b // b.lt .LBB0_13 - WORD $0x93407d08 // sxtw x8, w8 - WORD $0x93407d29 // sxtw x9, w9 - WORD $0x9100052a // add x10, x9, - WORD $0xeb08015f // cmp x10, x8 - WORD $0x9a89d50a // csinc x10, x8, x9, le - WORD $0xcb09014a // sub x10, x10, x9 - WORD $0xf100215f // cmp x10, - WORD $0x54000443 // b.lo .LBB0_12 - WORD $0xd37ef52c // lsl x12, x9, - WORD $0x927df14b // and x11, x10, - WORD $0x9100418d // add x13, x12, - WORD $0x8b090169 // add x9, x11, x9 - WORD $0x8b0d000c // add x12, x0, x13 - WORD $0x8b0d002d // add x13, x1, x13 - WORD $0xaa0b03ee // mov x14, x11 - -LBB0_10: - WORD $0xad7f8d81 // ldp q1, q3, [x12, - WORD $0xf10021ce // subs x14, x14, - WORD $0x9100818c // add x12, x12, - WORD $0xad7f91a2 // ldp q2, q4, [x13, - WORD $0x910081ad // add x13, x13, - WORD $0x4ea2d421 // fsub v1.4s, v1.4s, v2.4s - WORD $0x6e21dc21 // fmul v1.4s, v1.4s, v1.4s - WORD $0x5e0c0422 // mov s2, v1.s[1] - WORD $0x1e212800 // fadd s0, s0, s1 - WORD $0x5e140425 // mov s5, v1.s[2] - WORD $0x5e1c0421 // mov s1, v1.s[3] - WORD $0x1e222800 // fadd s0, s0, s2 - WORD $0x4ea4d462 // fsub v2.4s, v3.4s, v4.4s - WORD $0x1e252800 // fadd s0, s0, s5 - WORD $0x6e22dc42 // fmul v2.4s, v2.4s, v2.4s - WORD $0x1e212800 // fadd s0, s0, s1 - WORD $0x5e0c0441 // mov s1, v2.s[1] - WORD $0x5e140443 // mov s3, v2.s[2] - WORD $0x1e222800 // fadd s0, s0, s2 - WORD $0x1e212800 // fadd s0, s0, s1 - WORD $0x5e1c0441 // mov s1, v2.s[3] - WORD $0x1e232800 // fadd s0, s0, s3 - WORD $0x1e212800 // fadd s0, s0, s1 - WORD $0x54fffd21 // b.ne .LBB0_10 - WORD $0xeb0b015f // cmp x10, x11 - WORD $0x54000140 // b.eq .LBB0_13 - -LBB0_12: - WORD $0xd37ef52a // lsl x10, x9, - WORD $0x91000529 // add x9, x9, - WORD $0xeb08013f // cmp x9, x8 - WORD $0xbc6a6801 // ldr s1, [x0, x10] - WORD $0xbc6a6822 // ldr s2, [x1, x10] - WORD $0x1e223821 // fsub s1, s1, s2 - WORD $0x1e210821 // fmul s1, s1, s1 - WORD $0x1e212800 // fadd s0, s0, s1 - WORD $0x54ffff0b // b.lt .LBB0_12 - -LBB0_13: - WORD $0xbd000040 // str s0, [x2] - WORD $0xa8c17bfd // ldp x29, x30, [sp], - WORD $0xd65f03c0 // ret diff --git a/adapters/repos/db/vector/hnsw/distancer/asm/l2_inline.go b/adapters/repos/db/vector/hnsw/distancer/asm/l2_inline.go deleted file mode 100644 index c5919f8303188e0f563ec3e138e23131a5ce7acb..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/asm/l2_inline.go +++ /dev/null @@ -1,115 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package asm - -// Experiment with inlining and flattening the L2Squared distancer. -// Theoretically, this should be faster than the loop version for small vectors -// - it avoids the loop overhead -// - it eliminates the bounds check by reversing the iteration -// - it allows l22 and l24 to be inlined (the other ones are too large) -// See go tool compile -d=ssa/check_bce/debug=1 -m l2_inline.go - -func l22(x []float32, y []float32) float32 { - diff := x[1] - y[1] - sum := diff * diff - - diff = x[0] - y[0] - sum += diff * diff - - return sum -} - -func l24(x []float32, y []float32) float32 { - diff := x[3] - y[3] - sum := diff * diff - - diff = x[2] - y[2] - sum += diff * diff - - return l22(x, y) + sum -} - -func l26(x []float32, y []float32) float32 { - diff := x[5] - y[5] - sum := diff * diff - - diff = x[4] - y[4] - sum += diff * diff - - return l24(x, y) + sum -} - -func l28(x []float32, y []float32) float32 { - diff := x[7] - y[7] - sum := diff * diff - - diff = x[6] - y[6] - sum += diff * diff - - diff = x[5] - y[5] - sum += diff * diff - - diff = x[4] - y[4] - sum += diff * diff - - return l24(x, y) + sum -} - -func l210(x []float32, y []float32) float32 { - diff := x[9] - y[9] - sum := diff * diff - - diff = x[8] - y[8] - sum += diff * diff - - diff = x[7] - y[7] - sum += diff * diff - - diff = x[6] - y[6] - sum += diff * diff - - diff = x[5] - y[5] - sum += diff * diff - - diff = x[4] - y[4] - sum += diff * diff - - return l24(x, y) + sum -} - -func l212(x []float32, y []float32) float32 { - diff := x[11] - y[11] - sum := diff * diff - - diff = x[10] - y[10] - sum += diff * diff - - diff = x[9] - y[9] - sum += diff * diff - - diff = x[8] - y[8] - sum += diff * diff - - diff = x[7] - y[7] - sum += diff * diff - - diff = x[6] - y[6] - sum += diff * diff - - diff = x[5] - y[5] - sum += diff * diff - - diff = x[4] - y[4] - sum += diff * diff - - return l24(x, y) + sum -} diff --git a/adapters/repos/db/vector/hnsw/distancer/asm/l2_inline_test.go b/adapters/repos/db/vector/hnsw/distancer/asm/l2_inline_test.go deleted file mode 100644 index 0a4c0faaac4882e766ed4d673a68f70b3c950119..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/asm/l2_inline_test.go +++ /dev/null @@ -1,90 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package asm - -import ( - "fmt" - "math/rand" - "testing" -) - -func l2Loop(a, b []float32) float32 { - var sum float32 - - for i := range a { - diff := a[i] - b[i] - sum += diff * diff - } - - return sum -} - -func BenchmarkL2InlineVsLoop(b *testing.B) { - lengths := []int{2, 4, 6, 8, 10, 12} - for _, length := range lengths { - x := make([]float32, length) - y := make([]float32, length) - - for i := range x { - x[i] = rand.Float32() - y[i] = rand.Float32() - } - - b.Run(fmt.Sprintf("vector dim=%d", length), func(b *testing.B) { - b.Run("loop", func(b *testing.B) { - for i := 0; i < b.N; i++ { - l2Loop(x, y) - } - }) - - b.Run("flat", func(b *testing.B) { - // written to ensure that the compiler - // inlines the function when possible - switch length { - case 2: - b.ResetTimer() - for i := 0; i < b.N; i++ { - l22(x, y) - } - case 4: - b.ResetTimer() - for i := 0; i < b.N; i++ { - l24(x, y) - } - case 6: - b.ResetTimer() - for i := 0; i < b.N; i++ { - l26(x, y) - } - case 8: - b.ResetTimer() - for i := 0; i < b.N; i++ { - l28(x, y) - } - case 10: - b.ResetTimer() - for i := 0; i < b.N; i++ { - l210(x, y) - } - case 12: - b.ResetTimer() - for i := 0; i < b.N; i++ { - l212(x, y) - } - default: - panic("unsupported length") - } - }) - }) - - } -} diff --git a/adapters/repos/db/vector/hnsw/distancer/asm/l2_stub_amd64.go b/adapters/repos/db/vector/hnsw/distancer/asm/l2_stub_amd64.go deleted file mode 100644 index b37136dd12c3a6d422949c4a7c7f1a384cb3c43f..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/asm/l2_stub_amd64.go +++ /dev/null @@ -1,16 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by command: go run l2.go -out l2_amd64.s -stubs l2_stub_amd64.go. DO NOT EDIT. - -package asm - -func L2(x []float32, y []float32) float32 diff --git a/adapters/repos/db/vector/hnsw/distancer/asm/l2_stub_arm64.go b/adapters/repos/db/vector/hnsw/distancer/asm/l2_stub_arm64.go deleted file mode 100644 index 08a8ca1b41e80f89aa3cb7f8041515e5537b0f68..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/asm/l2_stub_arm64.go +++ /dev/null @@ -1,94 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package asm - -import ( - "reflect" - "unsafe" -) - -// To generate the asm code, run: -// go install github.com/gorse-io/goat@v0.1.0 -// go generate - -//go:generate goat ../c/l2_arm64.c -O3 -e="-mfpu=neon-fp-armv8" -e="-mfloat-abi=hard" -e="--target=arm64" -e="-march=armv8-a+simd+fp" - -// L2 calculates the L2 distance between two vectors -// using SIMD instructions when possible. -// Vector lengths < 16 are handled by the Go implementation -// because the overhead of using reflection is too high. -func L2(x []float32, y []float32) float32 { - switch len(x) { - case 2: - return l22(x, y) - case 4: - return l24(x, y) - case 6: - // manually inlined l26(x, y) - diff := x[5] - y[5] - sum := diff * diff - - diff = x[4] - y[4] - sum += diff * diff - - return l24(x, y) + sum - case 8: - // manually inlined l28(x, y) - diff := x[7] - y[7] - sum := diff * diff - - diff = x[6] - y[6] - sum += diff * diff - - diff = x[5] - y[5] - sum += diff * diff - - diff = x[4] - y[4] - sum += diff * diff - - return l24(x, y) + sum - case 10: - return l210(x, y) - case 12: - return l212(x, y) - } - - // deal with odd lengths and lengths 13, 14, 15 - if len(x) < 16 { - var sum float32 - - for i := range x { - diff := x[i] - y[i] - sum += diff * diff - } - - return sum - } - - var res float32 - - // The C function expects pointers to the underlying array, not slices. - hdrx := (*reflect.SliceHeader)(unsafe.Pointer(&x)) - hdry := (*reflect.SliceHeader)(unsafe.Pointer(&y)) - - l := len(x) - l2( - // The slice header contains the address of the underlying array. - // We only need to cast it to a pointer. - unsafe.Pointer(hdrx.Data), - unsafe.Pointer(hdry.Data), - // The C function expects pointers to the result and the length of the arrays. - unsafe.Pointer(&res), - unsafe.Pointer(&l)) - - return res -} diff --git a/adapters/repos/db/vector/hnsw/distancer/asm/prefetch.go b/adapters/repos/db/vector/hnsw/distancer/asm/prefetch.go deleted file mode 100644 index 6886b970a6cb2d2880301b9064e188393f88029f..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/asm/prefetch.go +++ /dev/null @@ -1,33 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build ignore -// +build ignore - -package main - -import ( - . "github.com/mmcloughlin/avo/build" - . "github.com/mmcloughlin/avo/operand" - // . "github.com/mmcloughlin/avo/reg" -) - -func main() { - TEXT("Prefetch", NOSPLIT, "func(addr uintptr)") - addr := Mem{Base: Load(Param("addr"), GP64())} - _ = addr - - PREFETCHT0(addr) - - RET() - - Generate() -} diff --git a/adapters/repos/db/vector/hnsw/distancer/asm/prefetch_amd64.s b/adapters/repos/db/vector/hnsw/distancer/asm/prefetch_amd64.s deleted file mode 100644 index 0d6131bdfa20a1d7cc1ec6450d7e4cbd33a22acb..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/asm/prefetch_amd64.s +++ /dev/null @@ -1,10 +0,0 @@ -// Code generated by command: go run prefetch.go -out prefetch.s -stubs prefetch_stub.go. DO NOT EDIT. - -#include "textflag.h" - -// func Prefetch(addr uintptr) -// Requires: MMX+ -TEXT ·Prefetch(SB), NOSPLIT, $0-8 - MOVQ addr+0(FP), AX - PREFETCHT0 (AX) - RET diff --git a/adapters/repos/db/vector/hnsw/distancer/asm/prefetch_stub_amd64.go b/adapters/repos/db/vector/hnsw/distancer/asm/prefetch_stub_amd64.go deleted file mode 100644 index 4a127c04a1f43da0c88c2669d47f91ef3bfc8a06..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/asm/prefetch_stub_amd64.go +++ /dev/null @@ -1,16 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by command: go run prefetch.go -out prefetch.s -stubs prefetch_stub.go. DO NOT EDIT. - -package asm - -func Prefetch(addr uintptr) diff --git a/adapters/repos/db/vector/hnsw/distancer/bench_amd64_test.go b/adapters/repos/db/vector/hnsw/distancer/bench_amd64_test.go deleted file mode 100644 index 65361b8aa41c1d961384d9d5c5fcdc58e4078c89..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/bench_amd64_test.go +++ /dev/null @@ -1,61 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package distancer - -import ( - "fmt" - "testing" - - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer/asm" -) - -func benchmarkDotGo(b *testing.B, dims int) { - r := getRandomSeed() - - vec1 := make([]float32, dims) - vec2 := make([]float32, dims) - for i := range vec1 { - vec1[i] = r.Float32() - vec2[i] = r.Float32() - } - - b.ResetTimer() - for n := 0; n < b.N; n++ { - DotProductGo(vec1, vec2) - } -} - -func benchmarkDotAVX(b *testing.B, dims int) { - r := getRandomSeed() - - vec1 := make([]float32, dims) - vec2 := make([]float32, dims) - for i := range vec1 { - vec1[i] = r.Float32() - vec2[i] = r.Float32() - } - - b.ResetTimer() - for n := 0; n < b.N; n++ { - asm.Dot(vec1, vec2) - } -} - -func BenchmarkDot(b *testing.B) { - dims := []int{30, 32, 128, 256, 300, 384, 600, 768, 1024} - for _, dim := range dims { - b.Run(fmt.Sprintf("%d dimensions", dim), func(b *testing.B) { - b.Run("pure go", func(b *testing.B) { benchmarkDotGo(b, dim) }) - b.Run("avx", func(b *testing.B) { benchmarkDotAVX(b, dim) }) - }) - } -} diff --git a/adapters/repos/db/vector/hnsw/distancer/bench_arm64_test.go b/adapters/repos/db/vector/hnsw/distancer/bench_arm64_test.go deleted file mode 100644 index 1fa9f1e0e18c415ef3dfb30a012d227f760efed0..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/bench_arm64_test.go +++ /dev/null @@ -1,61 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package distancer - -import ( - "fmt" - "testing" - - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer/asm" -) - -func benchmarkDotGo(b *testing.B, dims int) { - r := getRandomSeed() - - vec1 := make([]float32, dims) - vec2 := make([]float32, dims) - for i := range vec1 { - vec1[i] = r.Float32() - vec2[i] = r.Float32() - } - - b.ResetTimer() - for n := 0; n < b.N; n++ { - DotProductGo(vec1, vec2) - } -} - -func benchmarkDotNeon(b *testing.B, dims int) { - r := getRandomSeed() - - vec1 := make([]float32, dims) - vec2 := make([]float32, dims) - for i := range vec1 { - vec1[i] = r.Float32() - vec2[i] = r.Float32() - } - - b.ResetTimer() - for n := 0; n < b.N; n++ { - asm.Dot(vec1, vec2) - } -} - -func BenchmarkDot(b *testing.B) { - dims := []int{30, 32, 128, 256, 300, 384, 600, 768, 1024} - for _, dim := range dims { - b.Run(fmt.Sprintf("%d dimensions", dim), func(b *testing.B) { - b.Run("pure go", func(b *testing.B) { benchmarkDotGo(b, dim) }) - b.Run("avx", func(b *testing.B) { benchmarkDotNeon(b, dim) }) - }) - } -} diff --git a/adapters/repos/db/vector/hnsw/distancer/c/.gitignore b/adapters/repos/db/vector/hnsw/distancer/c/.gitignore deleted file mode 100644 index 2d6b42a50e199b2741a778c4a4b120225d5d06b5..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/c/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.o -*.s \ No newline at end of file diff --git a/adapters/repos/db/vector/hnsw/distancer/c/dot_arm64.c b/adapters/repos/db/vector/hnsw/distancer/c/dot_arm64.c deleted file mode 100644 index 2877bd5ec7f666ea8de1155f8370127a0694e06e..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/c/dot_arm64.c +++ /dev/null @@ -1,66 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -#include - -// dot only works with length >= 16 -void dot(float *a, float *b, float *res, long *len) -{ - int size = *len; - - // use the vectorized version for the first n - (n % 4) elements - int l = size - (size % 4); - - // create 4*4 registers to store the result - float32x4_t res_vec0 = vdupq_n_f32(0); - float32x4_t res_vec1 = vdupq_n_f32(0); - float32x4_t res_vec2 = vdupq_n_f32(0); - float32x4_t res_vec3 = vdupq_n_f32(0); - - int i = 0; - - // load 4*4 floats at a time - while (i + 16 <= l) - { - float32x4x4_t a4 = vld1q_f32_x4(a + i); - float32x4x4_t b4 = vld1q_f32_x4(b + i); - - res_vec0 += vmulq_f32(a4.val[0], b4.val[0]); - res_vec1 += vmulq_f32(a4.val[1], b4.val[1]); - res_vec2 += vmulq_f32(a4.val[2], b4.val[2]); - res_vec3 += vmulq_f32(a4.val[3], b4.val[3]); - - i += 16; - } - - while (i < l) - { - float32x4_t a_vec = vld1q_f32(a + i); - float32x4_t b_vec = vld1q_f32(b + i); - res_vec0 += vmulq_f32(a_vec, b_vec); - - i += 4; - } - - // convert to scalar - float sum = vaddvq_f32(res_vec0); - sum += vaddvq_f32(res_vec1); - sum += vaddvq_f32(res_vec2); - sum += vaddvq_f32(res_vec3); - - // add the remaining vectors - for (int i = l; i < size; i++) - { - sum += a[i] * b[i]; - } - - res[0] = sum; -} diff --git a/adapters/repos/db/vector/hnsw/distancer/c/l2_arm64.c b/adapters/repos/db/vector/hnsw/distancer/c/l2_arm64.c deleted file mode 100644 index d837f5bc190212db0d3d8e5c10d744b62fa1be96..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/c/l2_arm64.c +++ /dev/null @@ -1,73 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -#include - -// l2 only works with length >= 16 -void l2(float *a, float *b, float *res, long *len) -{ - int size = *len; - - // use the vectorized version for the first n - (n % 4) elements - int l = size - (size % 4); - - // create 4*4 registers to store the result - float32x4_t res_vec0 = vdupq_n_f32(0); - float32x4_t res_vec1 = vdupq_n_f32(0); - float32x4_t res_vec2 = vdupq_n_f32(0); - float32x4_t res_vec3 = vdupq_n_f32(0); - - int i = 0; - - // load 4*4 floats at a time - while (i + 16 <= l) - { - float32x4x4_t a4 = vld1q_f32_x4(a + i); - float32x4x4_t b4 = vld1q_f32_x4(b + i); - - float32x4_t diff0 = vsubq_f32(a4.val[0], b4.val[0]); - float32x4_t diff1 = vsubq_f32(a4.val[1], b4.val[1]); - float32x4_t diff2 = vsubq_f32(a4.val[2], b4.val[2]); - float32x4_t diff3 = vsubq_f32(a4.val[3], b4.val[3]); - res_vec0 += vmulq_f32(diff0, diff0); - res_vec1 += vmulq_f32(diff1, diff1); - res_vec2 += vmulq_f32(diff2, diff2); - res_vec3 += vmulq_f32(diff3, diff3); - - i += 16; - } - - while (i < l) - { - float32x4_t a_vec = vld1q_f32(a + i); - float32x4_t b_vec = vld1q_f32(b + i); - float32x4_t diff = vsubq_f32(a_vec, b_vec); - res_vec0 += vmulq_f32(diff, diff); - - i += 4; - } - - // convert to scalar - float sum = vaddvq_f32(res_vec0); - sum += vaddvq_f32(res_vec1); - sum += vaddvq_f32(res_vec2); - sum += vaddvq_f32(res_vec3); - - // add the remaining vectors - for (int i = l; i < size; i++) - { - float diff = a[i] - b[i]; - float sq = diff * diff; - sum += sq; - } - - res[0] = sum; -} diff --git a/adapters/repos/db/vector/hnsw/distancer/cosine_dist.go b/adapters/repos/db/vector/hnsw/distancer/cosine_dist.go deleted file mode 100644 index 25fc65edbece5e4645241dd0f65cccf736e9273e..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/cosine_dist.go +++ /dev/null @@ -1,68 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package distancer - -import ( - "github.com/pkg/errors" -) - -type CosineDistance struct { - a []float32 -} - -func (d *CosineDistance) Distance(b []float32) (float32, bool, error) { - if len(d.a) != len(b) { - return 0, false, errors.Errorf("vector lengths don't match: %d vs %d", - len(d.a), len(b)) - } - - dist := 1 - dotProductImplementation(d.a, b) - return dist, true, nil -} - -type CosineDistanceProvider struct{} - -func NewCosineDistanceProvider() CosineDistanceProvider { - return CosineDistanceProvider{} -} - -func (d CosineDistanceProvider) SingleDist(a, b []float32) (float32, bool, error) { - if len(a) != len(b) { - return 0, false, errors.Errorf("vector lengths don't match: %d vs %d", - len(a), len(b)) - } - - prod := 1 - dotProductImplementation(a, b) - - return prod, true, nil -} - -func (d CosineDistanceProvider) Type() string { - return "cosine-dot" -} - -func (d CosineDistanceProvider) New(a []float32) Distancer { - return &CosineDistance{a: a} -} - -func (d CosineDistanceProvider) Step(x, y []float32) float32 { - var sum float32 - for i := range x { - sum += x[i] * y[i] - } - - return sum -} - -func (d CosineDistanceProvider) Wrap(x float32) float32 { - return 1 - x -} diff --git a/adapters/repos/db/vector/hnsw/distancer/cosine_dist_test.go b/adapters/repos/db/vector/hnsw/distancer/cosine_dist_test.go deleted file mode 100644 index fb984a8d23c53ad4b6e1d984c277d7ceb4afb003..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/cosine_dist_test.go +++ /dev/null @@ -1,102 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package distancer - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestCosineDistancer(t *testing.T) { - t.Run("identical vectors", func(t *testing.T) { - vec1 := Normalize([]float32{0.1, 0.3, 0.7}) - vec2 := Normalize([]float32{0.1, 0.3, 0.7}) - expectedDistance := float32(0.0) - - dist, ok, err := NewCosineDistanceProvider().New(vec1).Distance(vec2) - require.Nil(t, err) - require.True(t, ok) - control, ok, err := NewCosineDistanceProvider().SingleDist(vec1, vec2) - require.True(t, ok) - require.Nil(t, err) - assert.Equal(t, control, dist) - assert.Equal(t, expectedDistance, dist) - }) - - t.Run("different vectors, but identical angle", func(t *testing.T) { - vec1 := Normalize([]float32{0.1, 0.3, 0.7}) - vec2 := Normalize([]float32{0.2, 0.6, 1.4}) - expectedDistance := float32(0.0) - - dist, ok, err := NewCosineDistanceProvider().New(vec1).Distance(vec2) - require.Nil(t, err) - require.True(t, ok) - control, ok, err := NewCosineDistanceProvider().SingleDist(vec1, vec2) - require.True(t, ok) - require.Nil(t, err) - assert.Equal(t, control, dist) - assert.Equal(t, expectedDistance, dist) - }) - - t.Run("different vectors", func(t *testing.T) { - vec1 := Normalize([]float32{0.1, 0.3, 0.7}) - vec2 := Normalize([]float32{0.2, 0.2, 0.2}) - expectedDistance := float32(0.173) - - dist, ok, err := NewCosineDistanceProvider().New(vec1).Distance(vec2) - require.Nil(t, err) - require.True(t, ok) - control, ok, err := NewCosineDistanceProvider().SingleDist(vec1, vec2) - require.True(t, ok) - require.Nil(t, err) - assert.Equal(t, control, dist) - assert.InDelta(t, expectedDistance, dist, 0.01) - }) - - t.Run("opposite vectors", func(t *testing.T) { - // This is unique to cosine/angular distance. - vec1 := Normalize([]float32{0.1, 0.3, 0.7}) - vec2 := Normalize([]float32{-0.1, -0.3, -0.7}) - expectedDistance := float32(2) - - dist, ok, err := NewCosineDistanceProvider().New(vec1).Distance(vec2) - require.Nil(t, err) - require.True(t, ok) - control, ok, err := NewCosineDistanceProvider().SingleDist(vec1, vec2) - require.True(t, ok) - require.Nil(t, err) - assert.Equal(t, control, dist) - assert.InDelta(t, expectedDistance, dist, 0.01) - }) -} - -func TestCosineDistancerStepbyStep(t *testing.T) { - t.Run("step by step equals SingleDist", func(t *testing.T) { - vec1 := Normalize([]float32{3, 4, 5}) - vec2 := Normalize([]float32{-3, -4, -5}) - - expectedDistance, ok, err := NewCosineDistanceProvider().New(vec1).Distance(vec2) - require.Nil(t, err) - require.True(t, ok) - - distanceProvider := NewCosineDistanceProvider() - sum := float32(0.0) - for i := range vec1 { - sum += distanceProvider.Step([]float32{vec1[i]}, []float32{vec2[i]}) - } - control := distanceProvider.Wrap(sum) - - assert.Equal(t, control, expectedDistance) - }) -} diff --git a/adapters/repos/db/vector/hnsw/distancer/dot_product.go b/adapters/repos/db/vector/hnsw/distancer/dot_product.go deleted file mode 100644 index 5905ab82966069fb444217ed2d6abb75a4c7fb97..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/dot_product.go +++ /dev/null @@ -1,91 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package distancer - -import ( - "github.com/pkg/errors" -) - -// can be set depending on architecture, e.g. pure go, AVX-enabled assembly, etc. -// Warning: This is not the dot product distance, but the pure product. -// -// This default will always work, regardless of architecture. An init function -// will overwrite it on amd64 if AVX is present. -var dotProductImplementation func(a, b []float32) float32 = func(a, b []float32) float32 { - var sum float32 - for i := range a { - sum += a[i] * b[i] - } - - return sum -} - -type DotProduct struct { - a []float32 -} - -func (d *DotProduct) Distance(b []float32) (float32, bool, error) { - if len(d.a) != len(b) { - return 0, false, errors.Errorf("vector lengths don't match: %d vs %d", - len(d.a), len(b)) - } - - dist := -dotProductImplementation(d.a, b) - return dist, true, nil -} - -type DotProductProvider struct{} - -func NewDotProductProvider() DotProductProvider { - return DotProductProvider{} -} - -func DotProductGo(a, b []float32) float32 { - var sum float32 - for i := range a { - sum += a[i] * b[i] - } - - return -sum -} - -func (d DotProductProvider) SingleDist(a, b []float32) (float32, bool, error) { - if len(a) != len(b) { - return 0, false, errors.Errorf("vector lengths don't match: %d vs %d", - len(a), len(b)) - } - - prod := -dotProductImplementation(a, b) - - return prod, true, nil -} - -func (d DotProductProvider) Type() string { - return "dot" -} - -func (d DotProductProvider) New(a []float32) Distancer { - return &DotProduct{a: a} -} - -func (d DotProductProvider) Step(x, y []float32) float32 { - var sum float32 - for i := range x { - sum += x[i] * y[i] - } - - return sum -} - -func (d DotProductProvider) Wrap(x float32) float32 { - return -x -} diff --git a/adapters/repos/db/vector/hnsw/distancer/dot_product_amd64.go b/adapters/repos/db/vector/hnsw/distancer/dot_product_amd64.go deleted file mode 100644 index 84dcfa973271d4216967983a95eb84b10f6f7bae..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/dot_product_amd64.go +++ /dev/null @@ -1,23 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package distancer - -import ( - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer/asm" - "golang.org/x/sys/cpu" -) - -func init() { - if cpu.X86.HasAVX2 { - dotProductImplementation = asm.Dot - } -} diff --git a/adapters/repos/db/vector/hnsw/distancer/dot_product_amd64_test.go b/adapters/repos/db/vector/hnsw/distancer/dot_product_amd64_test.go deleted file mode 100644 index 097271918b7547d6275063d7a9c8889714f6497a..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/dot_product_amd64_test.go +++ /dev/null @@ -1,111 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package distancer - -import ( - "fmt" - "math" - "testing" - "unsafe" - - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer/asm" -) - -func testDotProductFixedValue(t *testing.T, size uint) { - count := 10000 - countFailed := 0 - for i := 0; i < count; i++ { - vec1 := make([]float32, size) - vec2 := make([]float32, size) - for i := range vec1 { - vec1[i] = 1 - vec2[i] = 1 - } - vec1 = Normalize(vec1) - vec2 = Normalize(vec2) - res := -asm.Dot(vec1, vec2) - if math.IsNaN(float64(res)) { - panic("NaN") - } - - resControl := DotProductGo(vec1, vec2) - delta := float64(0.01) - diff := float64(resControl) - float64(res) - if diff < -delta || diff > delta { - countFailed++ - - fmt.Printf("run %d: match: %f != %f\n", i, resControl, res) - - t.Fail() - } - } - - fmt.Printf("total failed: %d\n", countFailed) -} - -func testDotProductRandomValue(t *testing.T, size uint) { - r := getRandomSeed() - count := 10000 - countFailed := 0 - - vec1s := make([][]float32, count) - vec2s := make([][]float32, count) - - for i := 0; i < count; i++ { - vec1 := make([]float32, size) - vec2 := make([]float32, size) - for j := range vec1 { - vec1[j] = r.Float32() - vec2[j] = r.Float32() - } - vec1s[i] = Normalize(vec1) - vec2s[i] = Normalize(vec2) - } - - for i := 0; i < count; i++ { - res := -asm.Dot(vec1s[i], vec2s[i]) - if math.IsNaN(float64(res)) { - panic("NaN") - } - - resControl := DotProductGo(vec1s[i], vec2s[i]) - delta := float64(0.01) - diff := float64(resControl) - float64(res) - if diff < -delta || diff > delta { - countFailed++ - - fmt.Printf("run %d: match: %f != %f, %d\n", i, resControl, res, (unsafe.Pointer(&vec1s[i][0]))) - - t.Fail() - } - - } - fmt.Printf("total failed: %d\n", countFailed) -} - -func TestCompareDotProductImplementations(t *testing.T) { - sizes := []uint{ - 8, - 16, - 32, - 64, - 128, - 256, - } - - for _, size := range sizes { - t.Run(fmt.Sprintf("with size %d", size), func(t *testing.T) { - testDotProductFixedValue(t, size) - testDotProductRandomValue(t, size) - }) - } -} diff --git a/adapters/repos/db/vector/hnsw/distancer/dot_product_arm64.go b/adapters/repos/db/vector/hnsw/distancer/dot_product_arm64.go deleted file mode 100644 index 1adf28724b80a43ec5b66653dad2e55a2aa1396d..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/dot_product_arm64.go +++ /dev/null @@ -1,23 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package distancer - -import ( - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer/asm" - "golang.org/x/sys/cpu" -) - -func init() { - if cpu.ARM64.HasASIMD { - dotProductImplementation = asm.Dot - } -} diff --git a/adapters/repos/db/vector/hnsw/distancer/dot_product_arm64_test.go b/adapters/repos/db/vector/hnsw/distancer/dot_product_arm64_test.go deleted file mode 100644 index 0e423aaf5a777c0cc3d33a9515c769c87a24c831..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/dot_product_arm64_test.go +++ /dev/null @@ -1,122 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package distancer - -import ( - "fmt" - "math" - "testing" - "unsafe" - - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer/asm" -) - -func testDotProductFixedValue(t *testing.T, size uint) { - count := 10000 - countFailed := 0 - for i := 0; i < count; i++ { - vec1 := make([]float32, size) - vec2 := make([]float32, size) - for i := range vec1 { - vec1[i] = 1 - vec2[i] = 1 - } - vec1 = Normalize(vec1) - vec2 = Normalize(vec2) - res := -asm.Dot(vec1, vec2) - if math.IsNaN(float64(res)) { - panic("NaN") - } - - resControl := DotProductGo(vec1, vec2) - delta := float64(0.01) - diff := float64(resControl) - float64(res) - if diff < -delta || diff > delta { - countFailed++ - - fmt.Printf("run %d: match: %f != %f\n", i, resControl, res) - - t.Fail() - } - } - - fmt.Printf("total failed: %d\n", countFailed) -} - -func testDotProductRandomValue(t *testing.T, size uint) { - r := getRandomSeed() - count := 10000 - countFailed := 0 - - vec1s := make([][]float32, count) - vec2s := make([][]float32, count) - - for i := 0; i < count; i++ { - vec1 := make([]float32, size) - vec2 := make([]float32, size) - for j := range vec1 { - vec1[j] = r.Float32() - vec2[j] = r.Float32() - } - vec1s[i] = Normalize(vec1) - vec2s[i] = Normalize(vec2) - } - - for i := 0; i < count; i++ { - res := -asm.Dot(vec1s[i], vec2s[i]) - if math.IsNaN(float64(res)) { - panic("NaN") - } - - resControl := DotProductGo(vec1s[i], vec2s[i]) - delta := float64(0.01) - diff := float64(resControl) - float64(res) - if diff < -delta || diff > delta { - countFailed++ - - fmt.Printf("run %d: match: %f != %f, %d\n", i, resControl, res, (unsafe.Pointer(&vec1s[i][0]))) - - t.Fail() - } - - } - fmt.Printf("total failed: %d\n", countFailed) -} - -func TestCompareDotProductImplementations(t *testing.T) { - sizes := []uint{ - 1, - 4, - 8, - 16, - 31, - 32, - 35, - 64, - 67, - 128, - 130, - 256, - 260, - 384, - 390, - 768, - 777, - } - - for _, size := range sizes { - t.Run(fmt.Sprintf("with size %d", size), func(t *testing.T) { - testDotProductFixedValue(t, size) - testDotProductRandomValue(t, size) - }) - } -} diff --git a/adapters/repos/db/vector/hnsw/distancer/dot_product_test.go b/adapters/repos/db/vector/hnsw/distancer/dot_product_test.go deleted file mode 100644 index ae49f66b8e044bc0a56c85000fab32e292400ab9..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/dot_product_test.go +++ /dev/null @@ -1,86 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package distancer - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestDotDistancer(t *testing.T) { - t.Run("identical vectors", func(t *testing.T) { - vec1 := []float32{3, 4, 5} - vec2 := []float32{3, 4, 5} - expectedDistance := float32(-50) - - dist, ok, err := NewDotProductProvider().New(vec1).Distance(vec2) - require.Nil(t, err) - require.True(t, ok) - control, ok, err := NewDotProductProvider().SingleDist(vec1, vec2) - require.True(t, ok) - require.Nil(t, err) - assert.Equal(t, control, dist) - assert.Equal(t, expectedDistance, dist) - }) - - t.Run("without matching dimensions", func(t *testing.T) { - vec1 := []float32{0, 1, 0, 2, 0, 3} - vec2 := []float32{1, 0, 2, 0, 3, 0} - expectedDistance := float32(0) - - dist, ok, err := NewDotProductProvider().New(vec1).Distance(vec2) - require.Nil(t, err) - require.True(t, ok) - control, ok, err := NewDotProductProvider().SingleDist(vec1, vec2) - require.True(t, ok) - require.Nil(t, err) - assert.Equal(t, control, dist) - assert.Equal(t, expectedDistance, dist) - }) - - t.Run("very different vectors", func(t *testing.T) { - vec1 := []float32{3, 4, 5} - vec2 := []float32{-3, -4, -5} - expectedDistance := float32(+50) - - dist, ok, err := NewDotProductProvider().New(vec1).Distance(vec2) - require.Nil(t, err) - require.True(t, ok) - control, ok, err := NewDotProductProvider().SingleDist(vec1, vec2) - require.True(t, ok) - require.Nil(t, err) - assert.Equal(t, control, dist) - assert.Equal(t, expectedDistance, dist) - }) -} - -func TestDotDistancerStepbyStep(t *testing.T) { - t.Run("step by step equals SingleDist", func(t *testing.T) { - vec1 := []float32{3, 4, 5} - vec2 := []float32{-3, -4, -5} - - expectedDistance, ok, err := NewDotProductProvider().New(vec1).Distance(vec2) - require.Nil(t, err) - require.True(t, ok) - - distanceProvider := NewDotProductProvider() - sum := float32(0.0) - for i := range vec1 { - sum += distanceProvider.Step([]float32{vec1[i]}, []float32{vec2[i]}) - } - control := distanceProvider.Wrap(sum) - - assert.Equal(t, control, expectedDistance) - }) -} diff --git a/adapters/repos/db/vector/hnsw/distancer/geo_spatial.go b/adapters/repos/db/vector/hnsw/distancer/geo_spatial.go deleted file mode 100644 index 32b9ddaf3b2d2f394d6a28ab013c2b5996d07858..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/geo_spatial.go +++ /dev/null @@ -1,75 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package distancer - -import ( - "fmt" - "math" -) - -func geoDist(a, b []float32) (float32, bool, error) { - if len(a) != 2 || len(b) != 2 { - return 0, false, fmt.Errorf("distance vectors must have len 2") - } - - latA := a[0] - latB := b[0] - lonA := a[1] - lonB := b[1] - const R = float64(6371e3) - - latARadian := float64(latA * math.Pi / 180) - latBRadian := float64(latB * math.Pi / 180) - deltaLatRadian := float64(latB-latA) * math.Pi / 180 - deltaLonRadian := float64(lonB-lonA) * math.Pi / 180 - - A := math.Sin(deltaLatRadian/2)*math.Sin(deltaLatRadian/2) + - math.Cos(latARadian)*math.Cos(latBRadian)*math.Sin(deltaLonRadian/2)*math.Sin(deltaLonRadian/2) - - C := 2 * math.Atan2(math.Sqrt(A), math.Sqrt(1-A)) - - return float32(R * C), true, nil -} - -type GeoDistancer struct { - a []float32 -} - -func (g GeoDistancer) Distance(b []float32) (float32, bool, error) { - return geoDist(g.a, b) -} - -type GeoProvider struct{} - -func (gp GeoProvider) New(vec []float32) Distancer { - return GeoDistancer{a: vec} -} - -func (gp GeoProvider) SingleDist(vec1, vec2 []float32) (float32, bool, error) { - return geoDist(vec1, vec2) -} - -func (gp GeoProvider) Type() string { - return "geo" -} - -func (gp GeoProvider) Step(x, y []float32) float32 { - panic("Not implemented") -} - -func (gp GeoProvider) Wrap(x float32) float32 { - panic("Not implemented") -} - -func NewGeoProvider() Provider { - return GeoProvider{} -} diff --git a/adapters/repos/db/vector/hnsw/distancer/geo_spatial_test.go b/adapters/repos/db/vector/hnsw/distancer/geo_spatial_test.go deleted file mode 100644 index 1fd14ca098c83e08c6a31388d70d868bfe531e7a..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/geo_spatial_test.go +++ /dev/null @@ -1,31 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package distancer - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestGeoSpatialDistance(t *testing.T) { - t.Run("between Munich and Stuttgart", func(t *testing.T) { - munich := []float32{48.137154, 11.576124} - stuttgart := []float32{48.783333, 9.183333} - - dist, ok, err := NewGeoProvider().New(munich).Distance(stuttgart) - require.Nil(t, err) - require.True(t, ok) - assert.InDelta(t, 190000, dist, 1000) - }) -} diff --git a/adapters/repos/db/vector/hnsw/distancer/hamming.go b/adapters/repos/db/vector/hnsw/distancer/hamming.go deleted file mode 100644 index b411fa6816ce7492100df89c7cd1d493f75b7413..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/hamming.go +++ /dev/null @@ -1,80 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package distancer - -import ( - "github.com/pkg/errors" -) - -var hammingImpl func(a, b []float32) float32 = func(a, b []float32) float32 { - var sum float32 // default value of float in golang is 0 - - for i := range a { - if a[i] != b[i] { - sum += float32(1) - } - } - - return sum -} - -type Hamming struct { - a []float32 -} - -func (l Hamming) Distance(b []float32) (float32, bool, error) { - if len(l.a) != len(b) { - return 0, false, errors.Errorf("vector lengths don't match: %d vs %d", - len(l.a), len(b)) - } - - return hammingImpl(l.a, b), true, nil -} - -type HammingProvider struct{} - -func NewHammingProvider() HammingProvider { - return HammingProvider{} -} - -func (l HammingProvider) SingleDist(a, b []float32) (float32, bool, error) { - if len(a) != len(b) { - return 0, false, errors.Errorf("vector lengths don't match: %d vs %d", - len(a), len(b)) - } - - return hammingImpl(a, b), true, nil -} - -func (l HammingProvider) Type() string { - return "hamming" -} - -func (l HammingProvider) New(a []float32) Distancer { - return &Hamming{a: a} -} - -func (l HammingProvider) Step(x, y []float32) float32 { - var sum float32 // default value of float in golang is 0 - - for i := range x { - if x[i] != y[i] { - sum += float32(1) - } - } - - return sum -} - -func (l HammingProvider) Wrap(x float32) float32 { - return x -} diff --git a/adapters/repos/db/vector/hnsw/distancer/hamming_test.go b/adapters/repos/db/vector/hnsw/distancer/hamming_test.go deleted file mode 100644 index 1a0b6ed8049f0a7c26e85a18740189d3b53b78fb..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/hamming_test.go +++ /dev/null @@ -1,101 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package distancer - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestHammingDistancer(t *testing.T) { - t.Run("identical vectors", func(t *testing.T) { - vec1 := []float32{3, 4, 5} - vec2 := []float32{3, 4, 5} - expectedDistance := float32(0) - - dist, ok, err := NewHammingProvider().New(vec1).Distance(vec2) - require.Nil(t, err) - require.True(t, ok) - control, ok, err := NewHammingProvider().SingleDist(vec1, vec2) - require.True(t, ok) - require.Nil(t, err) - assert.Equal(t, control, dist) - assert.Equal(t, expectedDistance, dist) - }) - - t.Run("same angle, different euclidean position", func(t *testing.T) { - vec1 := []float32{3, 4, 5} - vec2 := []float32{1.5, 2, 2.5} - expectedDistance := float32(3) // all three positions are different - - dist, ok, err := NewHammingProvider().New(vec1).Distance(vec2) - require.Nil(t, err) - require.True(t, ok) - control, ok, err := NewHammingProvider().SingleDist(vec1, vec2) - require.True(t, ok) - require.Nil(t, err) - assert.Equal(t, control, dist) - assert.Equal(t, expectedDistance, dist) - }) - - t.Run("one position different", func(t *testing.T) { - vec1 := []float32{10, 11} - vec2 := []float32{10, 15} - expectedDistance := float32(1) - - dist, ok, err := NewHammingProvider().New(vec1).Distance(vec2) - require.Nil(t, err) - require.True(t, ok) - control, ok, err := NewHammingProvider().SingleDist(vec1, vec2) - require.True(t, ok) - require.Nil(t, err) - assert.Equal(t, control, dist) - assert.Equal(t, expectedDistance, dist) - }) - - t.Run("three positions different", func(t *testing.T) { - vec1 := []float32{10, 11, 15, 25, 31} - vec2 := []float32{10, 15, 16, 25, 30} - expectedDistance := float32(3) - - dist, ok, err := NewHammingProvider().New(vec1).Distance(vec2) - require.Nil(t, err) - require.True(t, ok) - control, ok, err := NewHammingProvider().SingleDist(vec1, vec2) - require.True(t, ok) - require.Nil(t, err) - assert.Equal(t, control, dist) - assert.Equal(t, expectedDistance, dist) - }) -} - -func TestHammingDistancerStepbyStep(t *testing.T) { - t.Run("step by step equals SingleDist", func(t *testing.T) { - vec1 := []float32{10, 11, 15, 25, 31} - vec2 := []float32{10, 15, 16, 25, 30} - - expectedDistance, ok, err := NewHammingProvider().New(vec1).Distance(vec2) - require.Nil(t, err) - require.True(t, ok) - - distanceProvider := NewHammingProvider() - sum := float32(0.0) - for i := range vec1 { - sum += distanceProvider.Step([]float32{vec1[i]}, []float32{vec2[i]}) - } - control := distanceProvider.Wrap(sum) - - assert.Equal(t, control, expectedDistance) - }) -} diff --git a/adapters/repos/db/vector/hnsw/distancer/helper_for_test.go b/adapters/repos/db/vector/hnsw/distancer/helper_for_test.go deleted file mode 100644 index 18cd79f8e93f05d7514da348d498e9f15915a561..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/helper_for_test.go +++ /dev/null @@ -1,21 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package distancer - -import ( - "math/rand" - "time" -) - -func getRandomSeed() *rand.Rand { - return rand.New(rand.NewSource(time.Now().UnixNano())) -} diff --git a/adapters/repos/db/vector/hnsw/distancer/l2.go b/adapters/repos/db/vector/hnsw/distancer/l2.go deleted file mode 100644 index 78d6659e439c444e2b8d8d10d2fb7423e7cb4be8..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/l2.go +++ /dev/null @@ -1,76 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package distancer - -import "github.com/pkg/errors" - -var l2SquaredImpl func(a, b []float32) float32 = func(a, b []float32) float32 { - var sum float32 - - for i := range a { - diff := a[i] - b[i] - sum += diff * diff - } - - return sum -} - -type L2Squared struct { - a []float32 -} - -func (l L2Squared) Distance(b []float32) (float32, bool, error) { - if len(l.a) != len(b) { - return 0, false, errors.Errorf("vector lengths don't match: %d vs %d", - len(l.a), len(b)) - } - - return l2SquaredImpl(l.a, b), true, nil -} - -type L2SquaredProvider struct{} - -func NewL2SquaredProvider() L2SquaredProvider { - return L2SquaredProvider{} -} - -func (l L2SquaredProvider) SingleDist(a, b []float32) (float32, bool, error) { - if len(a) != len(b) { - return 0, false, errors.Errorf("vector lengths don't match: %d vs %d", - len(a), len(b)) - } - - return l2SquaredImpl(a, b), true, nil -} - -func (l L2SquaredProvider) Type() string { - return "l2-squared" -} - -func (l L2SquaredProvider) New(a []float32) Distancer { - return &L2Squared{a: a} -} - -func (l L2SquaredProvider) Step(a, b []float32) float32 { - var sum float32 - - for i := range a { - diff := a[i] - b[i] - sum += diff * diff - } - - return sum -} - -func (l L2SquaredProvider) Wrap(x float32) float32 { - return x -} diff --git a/adapters/repos/db/vector/hnsw/distancer/l2_amd64.go b/adapters/repos/db/vector/hnsw/distancer/l2_amd64.go deleted file mode 100644 index bd91c0debc56e60fe5eed1226f7b9c101c4d6762..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/l2_amd64.go +++ /dev/null @@ -1,23 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package distancer - -import ( - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer/asm" - "golang.org/x/sys/cpu" -) - -func init() { - if cpu.X86.HasAVX2 { - l2SquaredImpl = asm.L2 - } -} diff --git a/adapters/repos/db/vector/hnsw/distancer/l2_amd64_test.go b/adapters/repos/db/vector/hnsw/distancer/l2_amd64_test.go deleted file mode 100644 index d01cf64653176b154294bfc624b656838af1579a..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/l2_amd64_test.go +++ /dev/null @@ -1,99 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package distancer - -import ( - "fmt" - "math/rand" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer/asm" -) - -func L2PureGo(a, b []float32) float32 { - var sum float32 - - for i := range a { - diff := a[i] - b[i] - sum += diff * diff - } - - return sum -} - -func Test_L2_DistanceImplementation(t *testing.T) { - lengths := []int{1, 4, 16, 31, 32, 35, 64, 67, 128, 130, 256, 260, 384, 390, 768, 777} - - for _, length := range lengths { - t.Run(fmt.Sprintf("with vector l=%d", length), func(t *testing.T) { - x := make([]float32, length) - y := make([]float32, length) - for i := range x { - x[i] = rand.Float32() - y[i] = rand.Float32() - } - - control := L2PureGo(x, y) - asmResult := asm.L2(x, y) - - assert.InEpsilon(t, control, asmResult, 0.01) - }) - } -} - -func Test_L2_DistanceImplementation_OneNegativeValue(t *testing.T) { - lengths := []int{1, 4, 16, 31, 32, 35, 64, 67, 128, 130, 256, 260, 384, 390, 768, 777} - - for _, length := range lengths { - t.Run(fmt.Sprintf("with vector l=%d", length), func(t *testing.T) { - x := make([]float32, length) - y := make([]float32, length) - for i := range x { - x[i] = -rand.Float32() - y[i] = rand.Float32() - } - - control := L2PureGo(x, y) - asmResult := asm.L2(x, y) - - assert.InEpsilon(t, control, asmResult, 0.01) - }) - } -} - -func Benchmark_L2_PureGo_VS_AVX(b *testing.B) { - r := getRandomSeed() - lengths := []int{30, 32, 128, 256, 300, 384, 600, 768, 1024} - for _, length := range lengths { - b.Run(fmt.Sprintf("vector dim=%d", length), func(b *testing.B) { - x := make([]float32, length) - y := make([]float32, length) - for i := range x { - x[i] = -r.Float32() - y[i] = r.Float32() - } - - b.Run("pure go", func(b *testing.B) { - for i := 0; i < b.N; i++ { - L2PureGo(x, y) - } - }) - - b.Run("asm AVX", func(b *testing.B) { - for i := 0; i < b.N; i++ { - asm.L2(x, y) - } - }) - }) - } -} diff --git a/adapters/repos/db/vector/hnsw/distancer/l2_arm64.go b/adapters/repos/db/vector/hnsw/distancer/l2_arm64.go deleted file mode 100644 index c685f1af7e85cdc77f55ff23ff106d5a5032e7e6..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/l2_arm64.go +++ /dev/null @@ -1,23 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package distancer - -import ( - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer/asm" - "golang.org/x/sys/cpu" -) - -func init() { - if cpu.ARM64.HasASIMD { - l2SquaredImpl = asm.L2 - } -} diff --git a/adapters/repos/db/vector/hnsw/distancer/l2_arm64_test.go b/adapters/repos/db/vector/hnsw/distancer/l2_arm64_test.go deleted file mode 100644 index beeaa488c42d8122175418fdac1f8e68ee4ea82f..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/l2_arm64_test.go +++ /dev/null @@ -1,101 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package distancer - -import ( - "fmt" - "math/rand" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer/asm" -) - -func L2PureGo(a, b []float32) float32 { - var sum float32 - - for i := range a { - diff := a[i] - b[i] - sum += diff * diff - } - - return sum -} - -func Test_L2_DistanceImplementation(t *testing.T) { - lengths := []int{1, 4, 16, 31, 32, 35, 64, 67, 128, 130, 256, 260, 384, 390, 768, 777} - - for _, length := range lengths { - t.Run(fmt.Sprintf("with vector l=%d", length), func(t *testing.T) { - x := make([]float32, length) - y := make([]float32, length) - for i := range x { - x[i] = rand.Float32() - y[i] = rand.Float32() - } - - control := L2PureGo(x, y) - asmResult := asm.L2(x, y) - - assert.InEpsilon(t, control, asmResult, 0.01) - }) - } -} - -func Test_L2_DistanceImplementation_OneNegativeValue(t *testing.T) { - lengths := []int{1, 4, 16, 31, 32, 35, 64, 67, 128, 130, 256, 260, 384, 390, 768, 777} - - for _, length := range lengths { - t.Run(fmt.Sprintf("with vector l=%d", length), func(t *testing.T) { - x := make([]float32, length) - y := make([]float32, length) - for i := range x { - x[i] = -rand.Float32() - y[i] = rand.Float32() - } - - control := L2PureGo(x, y) - asmResult := asm.L2(x, y) - - assert.InEpsilon(t, control, asmResult, 0.01) - }) - } -} - -func Benchmark_L2_PureGo_VS_Neon(b *testing.B) { - r := getRandomSeed() - lengths := []int{2, 4, 6, 8, 10, 12, 16, 24, 30, 32, 128, 256, 300, 384, 600, 768, 1024} - for _, length := range lengths { - b.Run(fmt.Sprintf("vector dim=%d", length), func(b *testing.B) { - x := make([]float32, length) - y := make([]float32, length) - for i := range x { - x[i] = -r.Float32() - y[i] = r.Float32() - } - - b.ResetTimer() - - b.Run("pure go", func(b *testing.B) { - for i := 0; i < b.N; i++ { - L2PureGo(x, y) - } - }) - - b.Run("asm Neon", func(b *testing.B) { - for i := 0; i < b.N; i++ { - asm.L2(x, y) - } - }) - }) - } -} diff --git a/adapters/repos/db/vector/hnsw/distancer/l2_test.go b/adapters/repos/db/vector/hnsw/distancer/l2_test.go deleted file mode 100644 index 3d4dbd143626f51dda84c1f493addb408bb9989d..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/l2_test.go +++ /dev/null @@ -1,86 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package distancer - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestL2Distancer(t *testing.T) { - t.Run("identical vectors", func(t *testing.T) { - vec1 := []float32{3, 4, 5} - vec2 := []float32{3, 4, 5} - expectedDistance := float32(0) - - dist, ok, err := NewL2SquaredProvider().New(vec1).Distance(vec2) - require.Nil(t, err) - require.True(t, ok) - control, ok, err := NewL2SquaredProvider().SingleDist(vec1, vec2) - require.True(t, ok) - require.Nil(t, err) - assert.Equal(t, control, dist) - assert.Equal(t, expectedDistance, dist) - }) - - t.Run("same angle, different euclidean position", func(t *testing.T) { - vec1 := []float32{3, 4, 5} - vec2 := []float32{1.5, 2, 2.5} - expectedDistance := float32(12.5) - - dist, ok, err := NewL2SquaredProvider().New(vec1).Distance(vec2) - require.Nil(t, err) - require.True(t, ok) - control, ok, err := NewL2SquaredProvider().SingleDist(vec1, vec2) - require.True(t, ok) - require.Nil(t, err) - assert.Equal(t, control, dist) - assert.Equal(t, expectedDistance, dist) - }) - - t.Run("different vectors", func(t *testing.T) { - vec1 := []float32{10, 11} - vec2 := []float32{13, 15} - expectedDistance := float32(25) - - dist, ok, err := NewL2SquaredProvider().New(vec1).Distance(vec2) - require.Nil(t, err) - require.True(t, ok) - control, ok, err := NewL2SquaredProvider().SingleDist(vec1, vec2) - require.True(t, ok) - require.Nil(t, err) - assert.Equal(t, control, dist) - assert.Equal(t, expectedDistance, dist) - }) -} - -func TestL2DistancerStepbyStep(t *testing.T) { - t.Run("step by step equals SingleDist", func(t *testing.T) { - vec1 := []float32{3, 4, 5} - vec2 := []float32{1.5, 2, 2.5} - - expectedDistance, ok, err := NewL2SquaredProvider().New(vec1).Distance(vec2) - require.Nil(t, err) - require.True(t, ok) - - distanceProvider := NewL2SquaredProvider() - sum := float32(0.0) - for i := range vec1 { - sum += distanceProvider.Step([]float32{vec1[i]}, []float32{vec2[i]}) - } - control := distanceProvider.Wrap(sum) - - assert.Equal(t, control, expectedDistance) - }) -} diff --git a/adapters/repos/db/vector/hnsw/distancer/manhattan.go b/adapters/repos/db/vector/hnsw/distancer/manhattan.go deleted file mode 100644 index 184250b178d5d723460b374a01e235b57748313e..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/manhattan.go +++ /dev/null @@ -1,82 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package distancer - -import ( - "math" - - "github.com/pkg/errors" -) - -var manhattanImpl func(a, b []float32) float32 = func(a, b []float32) float32 { - var sum float32 - - for i := range a { - // take absolute difference, converted to float64 because math.Abs needs that - // convert back to float32 as sum is float32 - sum += float32(math.Abs(float64(a[i] - b[i]))) - } - - return sum -} - -type Manhattan struct { - a []float32 -} - -func (l Manhattan) Distance(b []float32) (float32, bool, error) { - if len(l.a) != len(b) { - return 0, false, errors.Errorf("vector lengths don't match: %d vs %d", - len(l.a), len(b)) - } - - return manhattanImpl(l.a, b), true, nil -} - -type ManhattanProvider struct{} - -func NewManhattanProvider() ManhattanProvider { - return ManhattanProvider{} -} - -func (l ManhattanProvider) SingleDist(a, b []float32) (float32, bool, error) { - if len(a) != len(b) { - return 0, false, errors.Errorf("vector lengths don't match: %d vs %d", - len(a), len(b)) - } - - return manhattanImpl(a, b), true, nil -} - -func (l ManhattanProvider) Type() string { - return "manhattan" -} - -func (l ManhattanProvider) New(a []float32) Distancer { - return &Manhattan{a: a} -} - -func (l ManhattanProvider) Step(x, y []float32) float32 { - var sum float32 - - for i := range x { - // take absolute difference, converted to float64 because math.Abs needs that - // convert back to float32 as sum is float32 - sum += float32(math.Abs(float64(x[i] - y[i]))) - } - - return sum -} - -func (l ManhattanProvider) Wrap(x float32) float32 { - return x -} diff --git a/adapters/repos/db/vector/hnsw/distancer/manhattan_test.go b/adapters/repos/db/vector/hnsw/distancer/manhattan_test.go deleted file mode 100644 index e2481dba225a36fe110606ab2d45cbaa7253b119..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/manhattan_test.go +++ /dev/null @@ -1,88 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package distancer - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestManhattanDistancer(t *testing.T) { - t.Run("identical vectors", func(t *testing.T) { - vec1 := []float32{3, 4, 5} - vec2 := []float32{3, 4, 5} - expectedDistance := float32(0) - - dist, ok, err := NewManhattanProvider().New(vec1).Distance(vec2) - require.Nil(t, err) - require.True(t, ok) - control, ok, err := NewManhattanProvider().SingleDist(vec1, vec2) - require.True(t, ok) - require.Nil(t, err) - assert.Equal(t, control, dist) - assert.Equal(t, expectedDistance, dist) - }) - - t.Run("same angle, different euclidean position", func(t *testing.T) { - vec1 := []float32{3, 4, 5} - vec2 := []float32{1.5, 2, 2.5} - // distance will be abs(3-1.5) + abs(4-2) + abs(5-2.5) = 1.5 + 2 + 2.5 = 6 - expectedDistance := float32(6) - - dist, ok, err := NewManhattanProvider().New(vec1).Distance(vec2) - require.Nil(t, err) - require.True(t, ok) - control, ok, err := NewManhattanProvider().SingleDist(vec1, vec2) - require.True(t, ok) - require.Nil(t, err) - assert.Equal(t, control, dist) - assert.Equal(t, expectedDistance, dist) - }) - - t.Run("different vectors", func(t *testing.T) { - vec1 := []float32{10, 11} - vec2 := []float32{13, 15} - // distance will be calculated as abs(10-13) + abs(11-15) = 3 + 4 = 7 - expectedDistance := float32(7) - - dist, ok, err := NewManhattanProvider().New(vec1).Distance(vec2) - require.Nil(t, err) - require.True(t, ok) - control, ok, err := NewManhattanProvider().SingleDist(vec1, vec2) - require.True(t, ok) - require.Nil(t, err) - assert.Equal(t, control, dist) - assert.Equal(t, expectedDistance, dist) - }) -} - -func TestManhattanDistancerStepbyStep(t *testing.T) { - t.Run("step by step equals SingleDist", func(t *testing.T) { - vec1 := []float32{3, 4, 5} - vec2 := []float32{1.5, 2, 2.5} - - expectedDistance, ok, err := NewManhattanProvider().New(vec1).Distance(vec2) - require.Nil(t, err) - require.True(t, ok) - - distanceProvider := NewManhattanProvider() - sum := float32(0.0) - for i := range vec1 { - sum += distanceProvider.Step([]float32{vec1[i]}, []float32{vec2[i]}) - } - control := distanceProvider.Wrap(sum) - - assert.Equal(t, control, expectedDistance) - }) -} diff --git a/adapters/repos/db/vector/hnsw/distancer/normalize.go b/adapters/repos/db/vector/hnsw/distancer/normalize.go deleted file mode 100644 index a0791d4882961246e13070599d15c8fb20493037..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/normalize.go +++ /dev/null @@ -1,32 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package distancer - -import "math" - -func Normalize(v []float32) []float32 { - var norm float32 - out := make([]float32, len(v)) - for i := range v { - norm += v[i] * v[i] - } - if norm == 0 { - return out - } - - norm = float32(math.Sqrt(float64(norm))) - for i := range v { - out[i] = v[i] / norm - } - - return out -} diff --git a/adapters/repos/db/vector/hnsw/distancer/provider.go b/adapters/repos/db/vector/hnsw/distancer/provider.go deleted file mode 100644 index 67961c06656cf319a40c04af6cb96c95487c6830..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/distancer/provider.go +++ /dev/null @@ -1,24 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package distancer - -type Provider interface { - New(vec []float32) Distancer - SingleDist(vec1, vec2 []float32) (float32, bool, error) - Step(x, y []float32) float32 - Wrap(x float32) float32 - Type() string -} - -type Distancer interface { - Distance(vec []float32) (float32, bool, error) -} diff --git a/adapters/repos/db/vector/hnsw/dynamic_ef_test.go b/adapters/repos/db/vector/hnsw/dynamic_ef_test.go deleted file mode 100644 index 621cf46d41b570b8cc7b87485cec52f875f4fdad..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/dynamic_ef_test.go +++ /dev/null @@ -1,105 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "context" - "testing" - - "github.com/pkg/errors" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" - "github.com/weaviate/weaviate/adapters/repos/db/vector/testinghelpers" - "github.com/weaviate/weaviate/entities/cyclemanager" - ent "github.com/weaviate/weaviate/entities/vectorindex/hnsw" -) - -// To prevent a regression on -// https://github.com/weaviate/weaviate/issues/1878 -func Test_DynamicEF(t *testing.T) { - type test struct { - name string - config ent.UserConfig - limit int - expectedEf int - } - - tests := []test{ - { - name: "all defaults explicitly entered, limit: 100", - config: ent.UserConfig{ - VectorCacheMaxObjects: 10, - EF: -1, - DynamicEFMin: 100, - DynamicEFMax: 500, - DynamicEFFactor: 8, - }, - limit: 100, - expectedEf: 500, - }, - { - name: "limit lower than min", - config: ent.UserConfig{ - VectorCacheMaxObjects: 10, - EF: -1, - DynamicEFMin: 100, - DynamicEFMax: 500, - DynamicEFFactor: 8, - }, - limit: 10, - expectedEf: 100, - }, - { - name: "limit within the dynamic range", - config: ent.UserConfig{ - VectorCacheMaxObjects: 10, - EF: -1, - DynamicEFMin: 100, - DynamicEFMax: 500, - DynamicEFFactor: 8, - }, - limit: 23, - expectedEf: 184, - }, - { - name: "explicit ef", - config: ent.UserConfig{ - VectorCacheMaxObjects: 10, - EF: 78, - }, - limit: 5, - expectedEf: 78, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - index, err := New(Config{ - RootPath: "doesnt-matter-as-committlogger-is-mocked-out", - ID: "dynaimc-ef-test", - MakeCommitLoggerThunk: MakeNoopCommitLogger, - DistanceProvider: distancer.NewCosineDistanceProvider(), - VectorForIDThunk: func(ctx context.Context, id uint64) ([]float32, error) { - return nil, errors.Errorf("not implemented") - }, - }, test.config, cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - cyclemanager.NewCallbackGroupNoop(), testinghelpers.NewDummyStore(t)) - require.Nil(t, err) - - actualEF := index.searchTimeEF(test.limit) - assert.Equal(t, test.expectedEf, actualEF) - - require.Nil(t, index.Drop(context.Background())) - }) - } -} diff --git a/adapters/repos/db/vector/hnsw/flat_search.go b/adapters/repos/db/vector/hnsw/flat_search.go deleted file mode 100644 index 6e704f937b26b06251ae8675252c13ed42a513bd..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/flat_search.go +++ /dev/null @@ -1,81 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/adapters/repos/db/priorityqueue" -) - -func (h *hnsw) flatSearch(queryVector []float32, limit int, - allowList helpers.AllowList, -) ([]uint64, []float32, error) { - results := priorityqueue.NewMax[any](limit) - - it := allowList.Iterator() - for candidate, ok := it.Next(); ok; candidate, ok = it.Next() { - h.RLock() - // Hot fix for https://github.com/weaviate/weaviate/issues/1937 - // this if statement mitigates the problem but it doesn't resolve the issue - if candidate >= uint64(len(h.nodes)) { - h.logger.WithField("action", "flatSearch"). - Warnf("trying to get candidate: %v but we only have: %v elements.", - candidate, len(h.nodes)) - h.RUnlock() - continue - } - if len(h.nodes) <= int(candidate) { // if index hasn't grown yet for a newly inserted node - continue - } - - h.shardedNodeLocks.RLock(candidate) - c := h.nodes[candidate] - h.shardedNodeLocks.RUnlock(candidate) - - if c == nil || h.hasTombstone(candidate) { - h.RUnlock() - continue - } - h.RUnlock() - dist, ok, err := h.distBetweenNodeAndVec(candidate, queryVector) - if err != nil { - return nil, nil, err - } - - if !ok { - // deleted node, ignore - continue - } - - if results.Len() < limit { - results.Insert(candidate, dist) - } else if results.Top().Dist > dist { - results.Pop() - results.Insert(candidate, dist) - } - } - - ids := make([]uint64, results.Len()) - dists := make([]float32, results.Len()) - - // results is ordered in reverse, we need to flip the order before presenting - // to the user! - i := len(ids) - 1 - for results.Len() > 0 { - res := results.Pop() - ids[i] = res.ID - dists[i] = res.Dist - i-- - } - - return ids, dists, nil -} diff --git a/adapters/repos/db/vector/hnsw/generate_recall_datasets.go b/adapters/repos/db/vector/hnsw/generate_recall_datasets.go deleted file mode 100644 index 1340cc6beae58faeb04c18914866ea7b97a99bc1..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/generate_recall_datasets.go +++ /dev/null @@ -1,118 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build ignore -// +build ignore - -package main - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "math" - "math/rand" - "sort" - - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" -) - -func main() { - dimensions := 256 - size := 10000 - queries := 1000 - - vectors := make([][]float32, size) - queryVectors := make([][]float32, queries) - truths := make([][]uint64, queries) - - fmt.Printf("generating %d vectors", size) - for i := 0; i < size; i++ { - vector := make([]float32, dimensions) - for j := 0; j < dimensions; j++ { - vector[j] = rand.Float32() - } - vectors[i] = Normalize(vector) - - } - fmt.Printf("done\n") - - fmt.Printf("generating %d search queries", queries) - for i := 0; i < queries; i++ { - queryVector := make([]float32, dimensions) - for j := 0; j < dimensions; j++ { - queryVector[j] = rand.Float32() - } - queryVectors[i] = Normalize(queryVector) - } - fmt.Printf("done\n") - - fmt.Printf("defining truth through brute force") - - k := 10 - for i, query := range queryVectors { - truths[i] = bruteForce(vectors, query, k) - } - - vectorsJSON, _ := json.Marshal(vectors) - queriesJSON, _ := json.Marshal(queryVectors) - truthsJSON, _ := json.Marshal(truths) - - ioutil.WriteFile("recall_vectors.json", vectorsJSON, 0o644) - ioutil.WriteFile("recall_queries.json", queriesJSON, 0o644) - ioutil.WriteFile("recall_truths.json", truthsJSON, 0o644) -} - -func Normalize(v []float32) []float32 { - var norm float32 - for i := range v { - norm += v[i] * v[i] - } - - norm = float32(math.Sqrt(float64(norm))) - for i := range v { - v[i] = v[i] / norm - } - - return v -} - -func bruteForce(vectors [][]float32, query []float32, k int) []uint64 { - type distanceAndIndex struct { - distance float32 - index uint64 - } - - distances := make([]distanceAndIndex, len(vectors)) - - for i, vec := range vectors { - dist, _, _ := distancer.NewCosineDistanceProvider().SingleDist(query, vec) - distances[i] = distanceAndIndex{ - index: uint64(i), - distance: dist, - } - } - - sort.Slice(distances, func(a, b int) bool { - return distances[a].distance < distances[b].distance - }) - - if len(distances) < k { - k = len(distances) - } - - out := make([]uint64, k) - for i := 0; i < k; i++ { - out[i] = distances[i].index - } - - return out -} diff --git a/adapters/repos/db/vector/hnsw/graph_integrity_integration_test.go b/adapters/repos/db/vector/hnsw/graph_integrity_integration_test.go deleted file mode 100644 index 6a5bbafac6fa44a965de3062595ce1d874ea66d2..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/graph_integrity_integration_test.go +++ /dev/null @@ -1,109 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTestSlow || !race - -package hnsw - -import ( - "context" - "fmt" - "math/rand" - "runtime" - "sync" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" - "github.com/weaviate/weaviate/entities/cyclemanager" - ent "github.com/weaviate/weaviate/entities/vectorindex/hnsw" -) - -func TestGraphIntegrity(t *testing.T) { - dimensions := 300 - size := 1000 - efConstruction := 128 - maxNeighbors := 64 - - vectors := make([][]float32, size) - var vectorIndex *hnsw - - t.Run("generate random vectors", func(t *testing.T) { - fmt.Printf("generating %d vectors", size) - for i := 0; i < size; i++ { - vector := make([]float32, dimensions) - for j := 0; j < dimensions; j++ { - vector[j] = rand.Float32() - } - vectors[i] = vector - } - }) - - t.Run("importing into hnsw", func(t *testing.T) { - fmt.Printf("importing into hnsw\n") - index, err := New(Config{ - RootPath: "doesnt-matter-as-committlogger-is-mocked-out", - ID: "graphintegrity", - MakeCommitLoggerThunk: MakeNoopCommitLogger, - VectorForIDThunk: func(ctx context.Context, id uint64) ([]float32, error) { - return vectors[int(id)], nil - }, - DistanceProvider: distancer.NewDotProductProvider(), - }, ent.UserConfig{ - MaxConnections: maxNeighbors, - EFConstruction: efConstruction, - }, cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), nil) - require.Nil(t, err) - vectorIndex = index - - workerCount := runtime.GOMAXPROCS(0) - jobsForWorker := make([][][]float32, workerCount) - - for i, vec := range vectors { - workerID := i % workerCount - jobsForWorker[workerID] = append(jobsForWorker[workerID], vec) - } - - wg := &sync.WaitGroup{} - for workerID, jobs := range jobsForWorker { - wg.Add(1) - go func(workerID int, myJobs [][]float32) { - defer wg.Done() - for i, vec := range myJobs { - originalIndex := uint64(i*workerCount) + uint64(workerID) - err := vectorIndex.Add(originalIndex, vec) - require.Nil(t, err) - } - }(workerID, jobs) - } - - wg.Wait() - }) - - for _, node := range vectorIndex.nodes { - if node == nil { - continue - } - - conlen := len(node.connections[0]) - - // it is debatable how much value this test still adds. It used to check - // that a lot of connections are present before we had the heuristic. But - // with the heuristic it's not uncommon that a node's connections get - // reduced to a slow amount of key connections. We have thus set this value - // to 1 to make sure that no nodes are entirely unconnected, but it's - // questionable if this still adds any value at all - requiredMinimum := 1 - assert.True(t, conlen >= requiredMinimum, fmt.Sprintf( - "have %d connections, but want at least %d", conlen, requiredMinimum)) - } -} diff --git a/adapters/repos/db/vector/hnsw/helper_for_test.go b/adapters/repos/db/vector/hnsw/helper_for_test.go deleted file mode 100644 index c58ee6e3890e94742fbbf7229d56aa979bbe0dfa..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/helper_for_test.go +++ /dev/null @@ -1,48 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "fmt" - "math/rand" - "strings" - "time" -) - -func dumpIndex(index *hnsw, labels ...string) { - if len(labels) > 0 { - fmt.Printf("--------------------------------------------------\n") - fmt.Printf("-- %s\n", strings.Join(labels, ", ")) - } - fmt.Printf("--------------------------------------------------\n") - fmt.Printf("ID: %s\n", index.id) - fmt.Printf("Entrypoint: %d\n", index.entryPointID) - fmt.Printf("Max Level: %d\n", index.currentMaximumLayer) - fmt.Printf("Tombstones %v\n", index.tombstones) - fmt.Printf("\nNodes and Connections:\n") - for _, node := range index.nodes { - if node == nil { - continue - } - - fmt.Printf(" Node %d\n", node.id) - for level, conns := range node.connections { - fmt.Printf(" Level %d: Connections: %v\n", level, conns) - } - } - - fmt.Printf("--------------------------------------------------\n") -} - -func getRandomSeed() *rand.Rand { - return rand.New(rand.NewSource(time.Now().UnixNano())) -} diff --git a/adapters/repos/db/vector/hnsw/heuristic.go b/adapters/repos/db/vector/hnsw/heuristic.go deleted file mode 100644 index a800e97891d1d114bc324f4fa4a416d10861436f..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/heuristic.go +++ /dev/null @@ -1,135 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "context" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/adapters/repos/db/priorityqueue" - "github.com/weaviate/weaviate/entities/storobj" -) - -func (h *hnsw) selectNeighborsHeuristic(input *priorityqueue.Queue[any], - max int, denyList helpers.AllowList, -) error { - if input.Len() < max { - return nil - } - - // TODO, if this solution stays we might need something with fewer allocs - ids := make([]uint64, input.Len()) - - closestFirst := h.pools.pqHeuristic.GetMin(input.Len()) - i := uint64(0) - for input.Len() > 0 { - elem := input.Pop() - closestFirst.InsertWithValue(elem.ID, elem.Dist, i) - ids[i] = elem.ID - i++ - } - - var returnList []priorityqueue.Item[uint64] - - if h.compressed.Load() { - bag := h.compressor.NewBag() - for _, id := range ids { - err := bag.Load(context.Background(), id) - if err != nil { - return err - } - } - - returnList = h.pools.pqItemSlice.Get().([]priorityqueue.Item[uint64]) - for closestFirst.Len() > 0 && len(returnList) < max { - curr := closestFirst.Pop() - if denyList != nil && denyList.Contains(curr.ID) { - continue - } - distToQuery := curr.Dist - - good := true - for _, item := range returnList { - peerDist, err := bag.Distance(curr.ID, item.ID) - if err != nil { - return err - } - - if peerDist < distToQuery { - good = false - break - } - } - - if good { - returnList = append(returnList, curr) - } - - } - } else { - - vecs, errs := h.multiVectorForID(context.TODO(), ids) - - returnList = h.pools.pqItemSlice.Get().([]priorityqueue.Item[uint64]) - - for closestFirst.Len() > 0 && len(returnList) < max { - curr := closestFirst.Pop() - if denyList != nil && denyList.Contains(curr.ID) { - continue - } - distToQuery := curr.Dist - - currVec := vecs[curr.Value] - if err := errs[curr.Value]; err != nil { - var e storobj.ErrNotFound - if errors.As(err, &e) { - h.handleDeletedNode(e.DocID) - continue - } else { - // not a typed error, we can recover from, return with err - return errors.Wrapf(err, - "unrecoverable error for docID %d", curr.ID) - } - } - good := true - for _, item := range returnList { - peerDist, _, _ := h.distancerProvider.SingleDist(currVec, - vecs[item.Value]) - - if peerDist < distToQuery { - good = false - break - } - } - - if good { - returnList = append(returnList, curr) - } - - } - } - - h.pools.pqHeuristic.Put(closestFirst) - - for _, retElem := range returnList { - input.Insert(retElem.ID, retElem.Dist) - } - - // rewind and return to pool - returnList = returnList[:0] - - //nolint:staticcheck - h.pools.pqItemSlice.Put(returnList) - - return nil -} diff --git a/adapters/repos/db/vector/hnsw/hnsw_stress_test.go b/adapters/repos/db/vector/hnsw/hnsw_stress_test.go deleted file mode 100644 index 354c410cc0bf9c33034224fb9eff444a968621ce..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/hnsw_stress_test.go +++ /dev/null @@ -1,361 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "context" - "encoding/binary" - "fmt" - "io" - "log" - "math" - "math/rand" - "os" - "sync" - "testing" - "time" - - "github.com/pkg/errors" - "github.com/stretchr/testify/require" - "golang.org/x/sync/errgroup" -) - -const ( - vectorSize = 128 - vectorsPerGoroutine = 100 - parallelGoroutines = 100 -) - -func idVector(ctx context.Context, id uint64) ([]float32, error) { - vector := make([]float32, vectorSize) - for i := 0; i < vectorSize; i++ { - vector[i] = float32(id) - } - return vector, nil -} - -func idVectorSize(size int) func(ctx context.Context, id uint64) ([]float32, error) { - return func(ctx context.Context, id uint64) ([]float32, error) { - vector := make([]float32, size) - for i := 0; i < size; i++ { - vector[i] = float32(id) - } - return vector, nil - } -} - -func float32FromBytes(bytes []byte) float32 { - bits := binary.LittleEndian.Uint32(bytes) - float := math.Float32frombits(bits) - return float -} - -func int32FromBytes(bytes []byte) int { - return int(binary.LittleEndian.Uint32(bytes)) -} - -func TestHnswStress(t *testing.T) { - siftFile := "datasets/ann-benchmarks/siftsmall/siftsmall_base.fvecs" - if _, err := os.Stat(siftFile); err != nil { - if !*download { - t.Skip(`Sift data needs to be present. -Run test with -download to automatically download the dataset. -Ex: go test -v -run TestHnswStress . -download -`) - } - downloadDatasetFile(t, siftFile) - } - vectors := readSiftFloat(siftFile, parallelGoroutines*vectorsPerGoroutine) - - t.Run("Insert and search and maybe delete", func(t *testing.T) { - for n := 0; n < 1; n++ { // increase if you don't want to reread SIFT for every run - wg := sync.WaitGroup{} - index := createEmptyHnswIndexForTests(t, idVector) - for k := 0; k < parallelGoroutines; k++ { - wg.Add(2) - goroutineIndex := k * vectorsPerGoroutine - go func() { - for i := 0; i < vectorsPerGoroutine; i++ { - - err := index.Add(uint64(goroutineIndex+i), vectors[goroutineIndex+i]) - require.Nil(t, err) - } - wg.Done() - }() - - go func() { - for i := 0; i < vectorsPerGoroutine; i++ { - for j := 0; j < 5; j++ { // try a couple of times to delete if found - _, dists, err := index.SearchByVector(vectors[goroutineIndex+i], 0, nil) - require.Nil(t, err) - - if len(dists) > 0 && dists[0] == 0 { - err := index.Delete(uint64(goroutineIndex + i)) - require.Nil(t, err) - break - } else { - continue - } - } - } - wg.Done() - }() - } - wg.Wait() - } - }) - - t.Run("Insert and delete", func(t *testing.T) { - for i := 0; i < 1; i++ { // increase if you don't want to reread SIFT for every run - wg := sync.WaitGroup{} - index := createEmptyHnswIndexForTests(t, idVector) - for k := 0; k < parallelGoroutines; k++ { - wg.Add(1) - goroutineIndex := k * vectorsPerGoroutine - go func() { - for i := 0; i < vectorsPerGoroutine; i++ { - - err := index.Add(uint64(goroutineIndex+i), vectors[goroutineIndex+i]) - require.Nil(t, err) - err = index.Delete(uint64(goroutineIndex + i)) - require.Nil(t, err) - - } - wg.Done() - }() - - } - wg.Wait() - - } - }) - - t.Run("Concurrent deletes", func(t *testing.T) { - for i := 0; i < 10; i++ { // increase if you don't want to reread SIFT for every run - wg := sync.WaitGroup{} - - index := createEmptyHnswIndexForTests(t, idVector) - deleteIds := make([]uint64, 50) - for j := 0; j < len(deleteIds); j++ { - err := index.Add(uint64(j), vectors[j]) - require.Nil(t, err) - deleteIds[j] = uint64(j) - } - wg.Add(2) - - go func() { - err := index.Delete(deleteIds[25:]...) - require.Nil(t, err) - wg.Done() - }() - go func() { - err := index.Delete(deleteIds[:24]...) - require.Nil(t, err) - wg.Done() - }() - - wg.Wait() - - time.Sleep(time.Microsecond * 100) - index.Lock() - require.NotNil(t, index.nodes[24]) - index.Unlock() - - } - }) - - t.Run("Random operations", func(t *testing.T) { - for i := 0; i < 1; i++ { // increase if you don't want to reread SIFT for every run - index := createEmptyHnswIndexForTests(t, idVector) - - var inserted struct { - sync.Mutex - ids []uint64 - set map[uint64]struct{} - } - inserted.set = make(map[uint64]struct{}) - - claimUnusedID := func() (uint64, bool) { - inserted.Lock() - defer inserted.Unlock() - - if len(inserted.ids) == len(vectors) { - return 0, false - } - - try := 0 - for { - id := uint64(rand.Intn(len(vectors))) - if _, ok := inserted.set[id]; !ok { - inserted.ids = append(inserted.ids, id) - inserted.set[id] = struct{}{} - return id, true - } - - try++ - if try > 50 { - log.Printf("[WARN] tried %d times, retrying...\n", try) - } - } - } - - getInsertedIDs := func(n int) []uint64 { - inserted.Lock() - defer inserted.Unlock() - - if len(inserted.ids) < n { - return nil - } - - if n > len(inserted.ids) { - n = len(inserted.ids) - } - - ids := make([]uint64, n) - copy(ids, inserted.ids[:n]) - - return ids - } - - removeInsertedIDs := func(ids ...uint64) { - inserted.Lock() - defer inserted.Unlock() - - for _, id := range ids { - delete(inserted.set, id) - for i, insertedID := range inserted.ids { - if insertedID == id { - inserted.ids = append(inserted.ids[:i], inserted.ids[i+1:]...) - break - } - } - } - } - - ops := []func(){ - // Add - func() { - id, ok := claimUnusedID() - if !ok { - return - } - - err := index.Add(id, vectors[id]) - require.Nil(t, err) - }, - // Delete - func() { - // delete 5% of the time - if rand.Int31()%20 == 0 { - return - } - - ids := getInsertedIDs(rand.Intn(100) + 1) - - err := index.Delete(ids...) - require.Nil(t, err) - - removeInsertedIDs(ids...) - }, - // Search - func() { - // search 50% of the time - if rand.Int31()%2 == 0 { - return - } - - id := rand.Intn(len(vectors)) - - _, _, err := index.SearchByVector(vectors[id], 0, nil) - require.Nil(t, err) - }, - } - - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Second) - defer cancel() - - g, ctx := errgroup.WithContext(ctx) - - // run parallelGoroutines goroutines - for i := 0; i < parallelGoroutines; i++ { - g.Go(func() error { - for { - select { - case <-ctx.Done(): - return ctx.Err() - default: - ops[rand.Intn(len(ops))]() - } - } - }) - } - - g.Wait() - } - }) -} - -func readSiftFloat(file string, maxObjects int) [][]float32 { - var vectors [][]float32 - - f, err := os.Open(file) - if err != nil { - panic(errors.Wrap(err, "Could not open SIFT file")) - } - defer f.Close() - - fi, err := f.Stat() - if err != nil { - panic(errors.Wrap(err, "Could not get SIFT file properties")) - } - fileSize := fi.Size() - if fileSize < 1000000 { - panic("The file is only " + fmt.Sprint(fileSize) + " bytes long. Did you forgot to install git lfs?") - } - - // The sift data is a binary file containing floating point vectors - // For each entry, the first 4 bytes is the length of the vector (in number of floats, not in bytes) - // which is followed by the vector data with vector length * 4 bytes. - // |-length-vec1 (4bytes)-|-Vec1-data-(4*length-vector-1 bytes)-|-length-vec2 (4bytes)-|-Vec2-data-(4*length-vector-2 bytes)-| - // The vector length needs to be converted from bytes to int - // The vector data needs to be converted from bytes to float - // Note that the vector entries are of type float but are integer numbers eg 2.0 - bytesPerF := 4 - vectorBytes := make([]byte, bytesPerF+vectorSize*bytesPerF) - for i := 0; i >= 0; i++ { - _, err = f.Read(vectorBytes) - if err == io.EOF { - break - } else if err != nil { - panic(err) - } - if int32FromBytes(vectorBytes[0:bytesPerF]) != vectorSize { - panic("Each vector must have 128 entries.") - } - vectorFloat := make([]float32, 0, vectorSize) - for j := 0; j < vectorSize; j++ { - start := (j + 1) * bytesPerF // first bytesPerF are length of vector - vectorFloat = append(vectorFloat, float32FromBytes(vectorBytes[start:start+bytesPerF])) - } - - vectors = append(vectors, vectorFloat) - - if i >= maxObjects { - break - } - } - if len(vectors) < maxObjects { - panic("Could not load all elements.") - } - - return vectors -} diff --git a/adapters/repos/db/vector/hnsw/index.go b/adapters/repos/db/vector/hnsw/index.go deleted file mode 100644 index 79954737a59487533c8ccd9b0d2d0942c75f380b..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/index.go +++ /dev/null @@ -1,704 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "context" - "fmt" - "io" - "math" - "math/rand" - "strings" - "sync" - "sync/atomic" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv" - "github.com/weaviate/weaviate/adapters/repos/db/priorityqueue" - "github.com/weaviate/weaviate/adapters/repos/db/vector/cache" - "github.com/weaviate/weaviate/adapters/repos/db/vector/common" - "github.com/weaviate/weaviate/adapters/repos/db/vector/compressionhelpers" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" - "github.com/weaviate/weaviate/entities/cyclemanager" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/storobj" - ent "github.com/weaviate/weaviate/entities/vectorindex/hnsw" -) - -type hnsw struct { - // global lock to prevent concurrent map read/write, etc. - sync.RWMutex - - // certain operations related to deleting, such as finding a new entrypoint - // can only run sequentially, this separate lock helps assuring this without - // blocking the general usage of the hnsw index - deleteLock *sync.Mutex - - tombstoneLock *sync.RWMutex - - // prevents tombstones cleanup to be performed in parallel with index reset operation - resetLock *sync.Mutex - // indicates whether reset operation occurred or not - if so tombstones cleanup method - // is aborted as it makes no sense anymore - resetCtx context.Context - resetCtxCancel context.CancelFunc - - // make sure the very first insert happens just once, otherwise we - // accidentally overwrite previous entrypoints on parallel imports on an - // empty graph - initialInsertOnce *sync.Once - - // Each node should not have more edges than this number - maximumConnections int - - // Nodes in the lowest level have a separate (usually higher) max connection - // limit - maximumConnectionsLayerZero int - - // the current maximum can be smaller than the configured maximum because of - // the exponentially decaying layer function. The initial entry is started at - // layer 0, but this has the chance to grow with every subsequent entry - currentMaximumLayer int - - // this is a point on the highest level, if we insert a new point with a - // higher level it will become the new entry point. Note tat the level of - // this point is always currentMaximumLayer - entryPointID uint64 - - // ef parameter used in construction phases, should be higher than ef during querying - efConstruction int - - // ef at search time - ef int64 - - // only used if ef=-1 - efMin int64 - efMax int64 - efFactor int64 - - // on filtered searches with less than n elements, perform flat search - flatSearchCutoff int64 - - levelNormalizer float64 - - nodes []*vertex - - vectorForID common.VectorForID[float32] - TempVectorForIDThunk common.TempVectorForID - multiVectorForID common.MultiVectorForID - trackDimensionsOnce sync.Once - dims int32 - - cache cache.Cache[float32] - - commitLog CommitLogger - - // a lookup of current tombstones (i.e. nodes that have received a tombstone, - // but have not been cleaned up yet) Cleanup is the process of removal of all - // outgoing edges to the tombstone as well as deleting the tombstone itself. - // This process should happen periodically. - tombstones map[uint64]struct{} - - tombstoneCleanupCallbackCtrl cyclemanager.CycleCallbackCtrl - shardCompactionCallbacks cyclemanager.CycleCallbackGroup - shardFlushCallbacks cyclemanager.CycleCallbackGroup - - // // for distributed spike, can be used to call a insertExternal on a different graph - // insertHook func(node, targetLevel int, neighborsAtLevel map[int][]uint32) - - id string - rootPath string - - logger logrus.FieldLogger - distancerProvider distancer.Provider - - pools *pools - - forbidFlat bool // mostly used in testing scenarios where we want to use the index even in scenarios where we typically wouldn't - - metrics *Metrics - insertMetrics *insertMetrics - - randFunc func() float64 // added to temporarily get rid on flakiness in tombstones related tests. to be removed after fixing WEAVIATE-179 - - // The deleteVsInsertLock makes sure that there are no concurrent delete and - // insert operations happening. It uses an RW-Mutex with: - // - // RLock -> Insert operations, this means any number of import operations can - // happen concurrently. - // - // Lock -> Delete operation. This means only a single delete operation can - // occur at a time, no insert operation can occur simultaneously with a - // delete. Since the delete is cheap (just marking the node as deleted), the - // single-threadedness of deletes is not a big problem. - // - // This lock was introduced as part of - // https://github.com/weaviate/weaviate/issues/2194 - // - // See - // https://github.com/weaviate/weaviate/pull/2191#issuecomment-1242726787 - // where we ran performance tests to make sure introducing this lock has no - // negative impact on performance. - deleteVsInsertLock sync.RWMutex - - compressed atomic.Bool - doNotRescore bool - - compressor compressionhelpers.VectorCompressor - pqConfig ent.PQConfig - - compressActionLock *sync.RWMutex - className string - shardName string - VectorForIDThunk common.VectorForID[float32] - shardedNodeLocks *common.ShardedLocks - store *lsmkv.Store -} - -type CommitLogger interface { - ID() string - AddNode(node *vertex) error - SetEntryPointWithMaxLayer(id uint64, level int) error - AddLinkAtLevel(nodeid uint64, level int, target uint64) error - ReplaceLinksAtLevel(nodeid uint64, level int, targets []uint64) error - AddTombstone(nodeid uint64) error - RemoveTombstone(nodeid uint64) error - DeleteNode(nodeid uint64) error - ClearLinks(nodeid uint64) error - ClearLinksAtLevel(nodeid uint64, level uint16) error - Reset() error - Drop(ctx context.Context) error - Flush() error - Shutdown(ctx context.Context) error - RootPath() string - SwitchCommitLogs(bool) error - AddPQ(compressionhelpers.PQData) error -} - -type BufferedLinksLogger interface { - AddLinkAtLevel(nodeid uint64, level int, target uint64) error - ReplaceLinksAtLevel(nodeid uint64, level int, targets []uint64) error - Close() error // Close should Flush and Close -} - -type MakeCommitLogger func() (CommitLogger, error) - -// New creates a new HNSW index, the commit logger is provided through a thunk -// (a function which can be deferred). This is because creating a commit logger -// opens files for writing. However, checking whether a file is present, is a -// criterium for the index to see if it has to recover from disk or if its a -// truly new index. So instead the index is initialized, with un-biased disk -// checks first and only then is the commit logger created -func New(cfg Config, uc ent.UserConfig, tombstoneCallbacks, shardCompactionCallbacks, - shardFlushCallbacks cyclemanager.CycleCallbackGroup, store *lsmkv.Store, -) (*hnsw, error) { - if err := cfg.Validate(); err != nil { - return nil, errors.Wrap(err, "invalid config") - } - - if cfg.Logger == nil { - logger := logrus.New() - logger.Out = io.Discard - cfg.Logger = logger - } - - normalizeOnRead := false - if cfg.DistanceProvider.Type() == "cosine-dot" { - normalizeOnRead = true - } - - vectorCache := cache.NewShardedFloat32LockCache(cfg.VectorForIDThunk, uc.VectorCacheMaxObjects, - cfg.Logger, normalizeOnRead, cache.DefaultDeletionInterval) - - resetCtx, resetCtxCancel := context.WithCancel(context.Background()) - index := &hnsw{ - maximumConnections: uc.MaxConnections, - - // inspired by original paper and other implementations - maximumConnectionsLayerZero: 2 * uc.MaxConnections, - - // inspired by c++ implementation - levelNormalizer: 1 / math.Log(float64(uc.MaxConnections)), - efConstruction: uc.EFConstruction, - flatSearchCutoff: int64(uc.FlatSearchCutoff), - nodes: make([]*vertex, cache.InitialSize), - cache: vectorCache, - vectorForID: vectorCache.Get, - multiVectorForID: vectorCache.MultiGet, - id: cfg.ID, - rootPath: cfg.RootPath, - tombstones: map[uint64]struct{}{}, - logger: cfg.Logger, - distancerProvider: cfg.DistanceProvider, - deleteLock: &sync.Mutex{}, - tombstoneLock: &sync.RWMutex{}, - resetLock: &sync.Mutex{}, - resetCtx: resetCtx, - resetCtxCancel: resetCtxCancel, - initialInsertOnce: &sync.Once{}, - - ef: int64(uc.EF), - efMin: int64(uc.DynamicEFMin), - efMax: int64(uc.DynamicEFMax), - efFactor: int64(uc.DynamicEFFactor), - - metrics: NewMetrics(cfg.PrometheusMetrics, cfg.ClassName, cfg.ShardName), - shardName: cfg.ShardName, - - randFunc: rand.Float64, - compressActionLock: &sync.RWMutex{}, - className: cfg.ClassName, - VectorForIDThunk: cfg.VectorForIDThunk, - TempVectorForIDThunk: cfg.TempVectorForIDThunk, - pqConfig: uc.PQ, - shardedNodeLocks: common.NewDefaultShardedLocks(), - - shardCompactionCallbacks: shardCompactionCallbacks, - shardFlushCallbacks: shardFlushCallbacks, - store: store, - } - - if uc.BQ.Enabled { - var err error - index.compressor, err = compressionhelpers.NewBQCompressor(index.distancerProvider, uc.VectorCacheMaxObjects, cfg.Logger, store) - if err != nil { - return nil, err - } - index.compressed.Store(true) - index.cache.Drop() - index.cache = nil - } - - if err := index.init(cfg); err != nil { - return nil, errors.Wrapf(err, "init index %q", index.id) - } - - // TODO common_cycle_manager move to poststartup? - id := strings.Join([]string{ - "hnsw", "tombstone_cleanup", - index.className, index.shardName, index.id, - }, "/") - index.tombstoneCleanupCallbackCtrl = tombstoneCallbacks.Register(id, index.tombstoneCleanup) - index.insertMetrics = newInsertMetrics(index.metrics) - - return index, nil -} - -// TODO: use this for incoming replication -// func (h *hnsw) insertFromExternal(nodeId, targetLevel int, neighborsAtLevel map[int][]uint32) { -// defer m.addBuildingReplication(time.Now()) - -// // randomly introduce up to 50ms delay to account for network slowness -// time.Sleep(time.Duration(rand.Intn(500)) * time.Millisecond) - -// var node *hnswVertex -// h.RLock() -// total := len(h.nodes) -// if total > nodeId { -// node = h.nodes[nodeId] // it could be that we implicitly added this node already because it was referenced -// } -// h.RUnlock() - -// if node == nil { -// node = &hnswVertex{ -// id: nodeId, -// connections: make(map[int][]uint32), -// level: targetLevel, -// } -// } else { -// node.level = targetLevel -// } - -// if total == 0 { -// h.Lock() -// h.commitLog.SetEntryPointWithMaxLayer(node.id, 0) -// h.entryPointID = node.id -// h.currentMaximumLayer = 0 -// node.connections = map[int][]uint32{} -// node.level = 0 -// // h.nodes = make([]*hnswVertex, 100000) -// h.commitLog.AddNode(node) -// h.nodes[node.id] = node -// h.Unlock() -// return -// } - -// currentMaximumLayer := h.currentMaximumLayer -// h.Lock() -// h.nodes[nodeId] = node -// h.commitLog.AddNode(node) -// h.Unlock() - -// for level := min(targetLevel, currentMaximumLayer); level >= 0; level-- { -// neighbors := neighborsAtLevel[level] - -// for _, neighborID := range neighbors { -// h.RLock() -// neighbor := h.nodes[neighborID] -// if neighbor == nil { -// // due to everything being parallel it could be that the linked neighbor -// // doesn't exist yet -// h.nodes[neighborID] = &hnswVertex{ -// id: int(neighborID), -// connections: make(map[int][]uint32), -// } -// neighbor = h.nodes[neighborID] -// } -// h.RUnlock() - -// neighbor.linkAtLevel(level, uint32(nodeId), h.commitLog) -// node.linkAtLevel(level, uint32(neighbor.id), h.commitLog) - -// neighbor.RLock() -// currentConnections := neighbor.connections[level] -// neighbor.RUnlock() - -// maximumConnections := h.maximumConnections -// if level == 0 { -// maximumConnections = h.maximumConnectionsLayerZero -// } - -// if len(currentConnections) <= maximumConnections { -// // nothing to do, skip -// continue -// } - -// // TODO: support both neighbor selection algos -// updatedConnections := h.selectNeighborsSimpleFromId(nodeId, currentConnections, maximumConnections) - -// neighbor.Lock() -// h.commitLog.ReplaceLinksAtLevel(neighbor.id, level, updatedConnections) -// neighbor.connections[level] = updatedConnections -// neighbor.Unlock() -// } -// } - -// if targetLevel > h.currentMaximumLayer { -// h.Lock() -// h.commitLog.SetEntryPointWithMaxLayer(nodeId, targetLevel) -// h.entryPointID = nodeId -// h.currentMaximumLayer = targetLevel -// h.Unlock() -// } - -// } - -func (h *hnsw) findBestEntrypointForNode(currentMaxLevel, targetLevel int, - entryPointID uint64, nodeVec []float32, distancer compressionhelpers.CompressorDistancer, -) (uint64, error) { - // in case the new target is lower than the current max, we need to search - // each layer for a better candidate and update the candidate - for level := currentMaxLevel; level > targetLevel; level-- { - eps := priorityqueue.NewMin[any](1) - var dist float32 - var ok bool - var err error - if h.compressed.Load() { - dist, ok, err = distancer.DistanceToNode(entryPointID) - } else { - dist, ok, err = h.distBetweenNodeAndVec(entryPointID, nodeVec) - } - if err != nil { - return 0, errors.Wrapf(err, - "calculate distance between insert node and entry point at level %d", level) - } - if !ok { - continue - } - - eps.Insert(entryPointID, dist) - res, err := h.searchLayerByVectorWithDistancer(nodeVec, eps, 1, level, nil, distancer) - if err != nil { - return 0, - errors.Wrapf(err, "update candidate: search layer at level %d", level) - } - if res.Len() > 0 { - // if we could find a new entrypoint, use it - // in case everything was tombstoned, stick with the existing one - elem := res.Pop() - n := h.nodeByID(elem.ID) - if n != nil && !n.isUnderMaintenance() { - // but not if the entrypoint is under maintenance - entryPointID = elem.ID - } - } - - h.pools.pqResults.Put(res) - } - - return entryPointID, nil -} - -func min(a, b int) int { - if a < b { - return a - } - return b -} - -func (h *hnsw) distBetweenNodes(a, b uint64) (float32, bool, error) { - if h.compressed.Load() { - dist, err := h.compressor.DistanceBetweenCompressedVectorsFromIDs(context.Background(), a, b) - if err != nil { - var e storobj.ErrNotFound - if errors.As(err, &e) { - h.handleDeletedNode(e.DocID) - return 0, false, nil - } else { - return 0, false, err - } - } - - return dist, true, nil - } - - // TODO: introduce single search/transaction context instead of spawning new - // ones - vecA, err := h.vectorForID(context.Background(), a) - if err != nil { - var e storobj.ErrNotFound - if errors.As(err, &e) { - h.handleDeletedNode(e.DocID) - return 0, false, nil - } else { - // not a typed error, we can recover from, return with err - return 0, false, errors.Wrapf(err, - "could not get vector of object at docID %d", a) - } - } - - if len(vecA) == 0 { - return 0, false, fmt.Errorf("got a nil or zero-length vector at docID %d", a) - } - - vecB, err := h.vectorForID(context.Background(), b) - if err != nil { - var e storobj.ErrNotFound - if errors.As(err, &e) { - h.handleDeletedNode(e.DocID) - return 0, false, nil - } else { - // not a typed error, we can recover from, return with err - return 0, false, errors.Wrapf(err, - "could not get vector of object at docID %d", b) - } - } - - if len(vecB) == 0 { - return 0, false, fmt.Errorf("got a nil or zero-length vector at docID %d", b) - } - - return h.distancerProvider.SingleDist(vecA, vecB) -} - -func (h *hnsw) distBetweenNodeAndVec(node uint64, vecB []float32) (float32, bool, error) { - if h.compressed.Load() { - dist, err := h.compressor.DistanceBetweenCompressedAndUncompressedVectorsFromID(context.Background(), node, vecB) - if err != nil { - var e storobj.ErrNotFound - if errors.As(err, &e) { - h.handleDeletedNode(e.DocID) - return 0, false, nil - } else { - return 0, false, err - } - } - - return dist, true, nil - } - - // TODO: introduce single search/transaction context instead of spawning new - // ones - vecA, err := h.vectorForID(context.Background(), node) - if err != nil { - var e storobj.ErrNotFound - if errors.As(err, &e) { - h.handleDeletedNode(e.DocID) - return 0, false, nil - } else { - // not a typed error, we can recover from, return with err - return 0, false, errors.Wrapf(err, - "could not get vector of object at docID %d", node) - } - } - - if len(vecA) == 0 { - return 0, false, fmt.Errorf( - "got a nil or zero-length vector at docID %d", node) - } - - if len(vecB) == 0 { - return 0, false, fmt.Errorf( - "got a nil or zero-length vector as search vector") - } - - return h.distancerProvider.SingleDist(vecA, vecB) -} - -func (h *hnsw) Stats() { - fmt.Printf("levels: %d\n", h.currentMaximumLayer) - - perLevelCount := map[int]uint{} - - for _, node := range h.nodes { - if node == nil { - continue - } - l := node.level - if l == 0 && len(node.connections) == 0 { - // filter out allocated space without nodes - continue - } - c, ok := perLevelCount[l] - if !ok { - perLevelCount[l] = 0 - } - - perLevelCount[l] = c + 1 - } - - for level, count := range perLevelCount { - fmt.Printf("unique count on level %d: %d\n", level, count) - } -} - -func (h *hnsw) isEmpty() bool { - h.RLock() - defer h.RUnlock() - h.shardedNodeLocks.RLock(h.entryPointID) - defer h.shardedNodeLocks.RUnlock(h.entryPointID) - - return h.isEmptyUnlocked() -} - -func (h *hnsw) isEmptyUnlocked() bool { - return h.nodes[h.entryPointID] == nil -} - -func (h *hnsw) nodeByID(id uint64) *vertex { - h.RLock() - defer h.RUnlock() - - if id >= uint64(len(h.nodes)) { - // See https://github.com/weaviate/weaviate/issues/1838 for details. - // This could be after a crash recovery when the object store is "further - // ahead" than the hnsw index and we receive a delete request - return nil - } - - h.shardedNodeLocks.RLock(id) - defer h.shardedNodeLocks.RUnlock(id) - - return h.nodes[id] -} - -func (h *hnsw) Drop(ctx context.Context) error { - // cancel tombstone cleanup goroutine - if err := h.tombstoneCleanupCallbackCtrl.Unregister(ctx); err != nil { - return errors.Wrap(err, "hnsw drop") - } - - if h.compressed.Load() { - err := h.compressor.Drop() - if err != nil { - return fmt.Errorf("failed to shutdown compressed store") - } - } else { - // cancel vector cache goroutine - h.cache.Drop() - } - - // cancel commit logger last, as the tombstone cleanup cycle might still - // write while it's still running - err := h.commitLog.Drop(ctx) - if err != nil { - return errors.Wrap(err, "commit log drop") - } - - return nil -} - -func (h *hnsw) Shutdown(ctx context.Context) error { - if err := h.commitLog.Shutdown(ctx); err != nil { - return errors.Wrap(err, "hnsw shutdown") - } - - if err := h.tombstoneCleanupCallbackCtrl.Unregister(ctx); err != nil { - return errors.Wrap(err, "hnsw shutdown") - } - - if h.compressed.Load() { - err := h.compressor.Drop() - if err != nil { - return errors.Wrap(err, "hnsw shutdown") - } - } else { - h.cache.Drop() - } - - return nil -} - -func (h *hnsw) Flush() error { - return h.commitLog.Flush() -} - -func (h *hnsw) Entrypoint() uint64 { - h.RLock() - defer h.RUnlock() - - return h.entryPointID -} - -func (h *hnsw) DistanceBetweenVectors(x, y []float32) (float32, bool, error) { - return h.distancerProvider.SingleDist(x, y) -} - -func (h *hnsw) ContainsNode(id uint64) bool { - h.RLock() - defer h.RUnlock() - h.shardedNodeLocks.RLock(id) - defer h.shardedNodeLocks.RUnlock(id) - - return len(h.nodes) > int(id) && h.nodes[id] != nil -} - -func (h *hnsw) DistancerProvider() distancer.Provider { - return h.distancerProvider -} - -func (h *hnsw) ShouldCompress() (bool, int) { - return h.pqConfig.Enabled, h.pqConfig.TrainingLimit -} - -func (h *hnsw) ShouldCompressFromConfig(config schema.VectorIndexConfig) (bool, int) { - hnswConfig := config.(ent.UserConfig) - return hnswConfig.PQ.Enabled, hnswConfig.PQ.TrainingLimit -} - -func (h *hnsw) Compressed() bool { - return h.compressed.Load() -} - -func (h *hnsw) AlreadyIndexed() uint64 { - return uint64(h.cache.CountVectors()) -} - -func (h *hnsw) normalizeVec(vec []float32) []float32 { - if h.distancerProvider.Type() == "cosine-dot" { - // cosine-dot requires normalized vectors, as the dot product and cosine - // similarity are only identical if the vector is normalized - return distancer.Normalize(vec) - } - return vec -} diff --git a/adapters/repos/db/vector/hnsw/index_corrupt_commitlogs_integration_test.go b/adapters/repos/db/vector/hnsw/index_corrupt_commitlogs_integration_test.go deleted file mode 100644 index 7c18ce3b305b1f699d778011aa250cc13c4b7e3d..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/index_corrupt_commitlogs_integration_test.go +++ /dev/null @@ -1,133 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package hnsw - -import ( - "context" - "fmt" - "os" - "path" - "testing" - - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" - "github.com/weaviate/weaviate/adapters/repos/db/vector/testinghelpers" - "github.com/weaviate/weaviate/entities/cyclemanager" - hnswent "github.com/weaviate/weaviate/entities/vectorindex/hnsw" -) - -func TestStartupWithCorruptCondenseFiles(t *testing.T) { - rootPath := t.TempDir() - - logger, _ := test.NewNullLogger() - original, err := NewCommitLogger(rootPath, "corrupt_test", logger, - cyclemanager.NewCallbackGroupNoop()) - require.Nil(t, err) - - data := [][]float32{ - {0.1, 0.2}, - {0.12, 0.2}, - {0.13, 0.2}, - {0.14, 0.2}, - {0.15, 0.2}, - {0.16, 0.2}, - {0.16, 0.2}, - {0.17, 0.2}, - } - - var index *hnsw - - t.Run("set up an index with the specified commit logger", func(t *testing.T) { - idx, err := New(Config{ - MakeCommitLoggerThunk: func() (CommitLogger, error) { - return original, nil - }, - ID: "corrupt_test", - RootPath: rootPath, - DistanceProvider: distancer.NewCosineDistanceProvider(), - Logger: logger, - VectorForIDThunk: func(ctx context.Context, id uint64) ([]float32, error) { - return data[int(id)], nil - }, - }, hnswent.UserConfig{ - MaxConnections: 100, - EFConstruction: 100, - CleanupIntervalSeconds: 0, - }, cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - cyclemanager.NewCallbackGroupNoop(), testinghelpers.NewDummyStore(t)) - require.Nil(t, err) - index = idx - }) - - t.Run("add data", func(t *testing.T) { - for i, vec := range data { - err := index.Add(uint64(i), vec) - require.Nil(t, err) - } - }) - - index.Flush() - - t.Run("create a corrupt commit log file without deleting the original", - func(t *testing.T) { - input, ok, err := getCurrentCommitLogFileName(commitLogDirectory(rootPath, - "corrupt_test")) - require.Nil(t, err) - require.True(t, ok) - - f, err := os.Create(path.Join(commitLogDirectory(rootPath, "corrupt_test"), - fmt.Sprintf("%s.condensed", input))) - require.Nil(t, err) - - // write random non-sense to make sure the file is corrupt - _, err = f.Write([]uint8{0xa8, 0x07, 0x34, 0x77, 0xf8, 0xff}) - require.Nil(t, err) - f.Close() - }) - - t.Run("destroy the old index", func(t *testing.T) { - // kill the index - index = nil - original = nil - }) - - t.Run("create a new one from the disk files", func(t *testing.T) { - idx, err := New(Config{ - MakeCommitLoggerThunk: MakeNoopCommitLogger, // no longer need a real one - ID: "corrupt_test", - RootPath: rootPath, - DistanceProvider: distancer.NewCosineDistanceProvider(), - Logger: logger, - VectorForIDThunk: func(ctx context.Context, id uint64) ([]float32, error) { - return data[int(id)], nil - }, - }, hnswent.UserConfig{ - MaxConnections: 100, - EFConstruction: 100, - CleanupIntervalSeconds: 0, - }, cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - cyclemanager.NewCallbackGroupNoop(), testinghelpers.NewDummyStore(t)) - require.Nil(t, err) - index = idx - }) - - t.Run("verify querying works", func(t *testing.T) { - res, _, err := index.SearchByVector([]float32{0.08, 0.08}, 100, nil) - require.Nil(t, err) - assert.Len(t, res, 8) - }) -} diff --git a/adapters/repos/db/vector/hnsw/index_slowdown_bug_intergration_test.go b/adapters/repos/db/vector/hnsw/index_slowdown_bug_intergration_test.go deleted file mode 100644 index 5d98c6d0ef66d8f8b7661a86f4fee6058d52ec11..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/index_slowdown_bug_intergration_test.go +++ /dev/null @@ -1,118 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTestBug -// +build integrationTestBug - -package hnsw - -import ( - "context" - "fmt" - "math" - "math/rand" - "runtime" - "sync" - "testing" - "time" - - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" - "github.com/weaviate/weaviate/adapters/repos/db/vector/testinghelpers" -) - -func Normalize(v []float32) []float32 { - var norm float32 - for i := range v { - norm += v[i] * v[i] - } - - norm = float32(math.Sqrt(float64(norm))) - for i := range v { - v[i] = v[i] / norm - } - - return v -} - -func TestSlowDownBugAtHighEF(t *testing.T) { - dimensions := 256 - size := 25000 - efConstruction := 2000 - maxNeighbors := 100 - - vectors := make([][]float32, size) - var vectorIndex *hnsw - - t.Run("generate random vectors", func(t *testing.T) { - fmt.Printf("generating %d vectors", size) - for i := 0; i < size; i++ { - vector := make([]float32, dimensions) - for j := 0; j < dimensions; j++ { - vector[j] = rand.Float32() - } - vectors[i] = Normalize(vector) - } - fmt.Printf("done\n") - }) - - t.Run("importing into hnsw", func(t *testing.T) { - fmt.Printf("importing into hnsw\n") - index, err := New(Config{ - RootPath: "doesnt-matter-as-committlogger-is-mocked-out", - ID: "recallbenchmark", - MakeCommitLoggerThunk: MakeNoopCommitLogger, - DistanceProvider: distancer.NewDotProductProvider(), - // DistanceProvider: distancer.NewCosineDistanceProvider(), - VectorForIDThunk: func(ctx context.Context, id uint64) ([]float32, error) { - return nil, nil - }, - }, UserConfig{ - MaxConnections: maxNeighbors, - EFConstruction: efConstruction, - }, testinghelpers.NewDummyStore(t)) - - require.Nil(t, err) - vectorIndex = index - - workerCount := runtime.GOMAXPROCS(0) - // workerCount := 1 - jobsForWorker := make([][][]float32, workerCount) - - for i, vec := range vectors { - workerID := i % workerCount - jobsForWorker[workerID] = append(jobsForWorker[workerID], vec) - } - - beforeImport := time.Now() - wg := &sync.WaitGroup{} - for workerID, jobs := range jobsForWorker { - wg.Add(1) - go func(workerID int, myJobs [][]float32) { - defer wg.Done() - for i, vec := range myJobs { - originalIndex := (i * workerCount) + workerID - err := vectorIndex.Add(uint64(originalIndex), vec) - require.Nil(t, err) - } - }(workerID, jobs) - } - - wg.Wait() - // neighbor := bruteForceCosine(vectors, vectors[0], 2) - // dist, _, _ := distancer.NewCosineDistanceProvider().SingleDist(vectors[0], vectors[neighbor[1]]) - // fmt.Printf("distance between 0 and %d is %f\n", neighbor[1], dist) - fmt.Printf("import took %s\n", time.Since(beforeImport)) - // vectorIndex.Dump() - - t.Fail() - }) -} diff --git a/adapters/repos/db/vector/hnsw/index_test.go b/adapters/repos/db/vector/hnsw/index_test.go deleted file mode 100644 index 533cf8629c3326742c56e7d64859cf09d443eaaa..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/index_test.go +++ /dev/null @@ -1,154 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/adapters/repos/db/vector/cache" - "github.com/weaviate/weaviate/adapters/repos/db/vector/common" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" - "github.com/weaviate/weaviate/adapters/repos/db/vector/testinghelpers" - "github.com/weaviate/weaviate/entities/cyclemanager" - ent "github.com/weaviate/weaviate/entities/vectorindex/hnsw" -) - -func TestHnswIndex(t *testing.T) { - index := createEmptyHnswIndexForTests(t, testVectorForID) - - for i, vec := range testVectors { - err := index.Add(uint64(i), vec) - require.Nil(t, err) - } - - t.Run("searching within cluster 1", func(t *testing.T) { - position := 0 - res, _, err := index.knnSearchByVector(testVectors[position], 3, 36, nil) - require.Nil(t, err) - assert.ElementsMatch(t, []uint64{0, 1, 2}, res) - }) - - t.Run("searching within cluster 2", func(t *testing.T) { - position := 3 - res, _, err := index.knnSearchByVector(testVectors[position], 3, 36, nil) - require.Nil(t, err) - assert.ElementsMatch(t, []uint64{3, 4, 5}, res) - }) - - t.Run("searching within cluster 3", func(t *testing.T) { - position := 6 - res, _, err := index.knnSearchByVector(testVectors[position], 3, 36, nil) - require.Nil(t, err) - assert.ElementsMatch(t, []uint64{6, 7, 8}, res) - }) - - t.Run("searching within cluster 2 with a scope larger than the cluster", func(t *testing.T) { - position := 3 - res, _, err := index.knnSearchByVector(testVectors[position], 50, 36, nil) - require.Nil(t, err) - assert.Equal(t, []uint64{ - 3, 5, 4, // cluster 2 - 7, 8, 6, // cluster 3 - 2, 1, 0, // cluster 1 - }, res) - }) - - t.Run("searching with negative value of k", func(t *testing.T) { - position := 0 - _, _, err := index.knnSearchByVector(testVectors[position], -1, 36, nil) - require.Error(t, err) - }) -} - -func TestHnswIndexGrow(t *testing.T) { - vector := []float32{0.1, 0.2} - vecForIDFn := func(ctx context.Context, id uint64) ([]float32, error) { - return vector, nil - } - index := createEmptyHnswIndexForTests(t, vecForIDFn) - - t.Run("should grow initial empty index", func(t *testing.T) { - // when we invoke Add method suggesting a size bigger then the default - // initial size, then if we don't grow an index at initial state - // we get: panic: runtime error: index out of range [25001] with length 25000 - // in order to avoid this, insertInitialElement method is now able - // to grow it's size at initial state - err := index.Add(uint64(cache.InitialSize+1), vector) - require.Nil(t, err) - }) - - t.Run("should grow index without panic", func(t *testing.T) { - // This test shows that we had an edge case that was not covered - // in growIndexToAccomodateNode method which was leading to panic: - // panic: runtime error: index out of range [170001] with length 170001 - vector := []float32{0.11, 0.22} - id := uint64(5*cache.InitialSize + 1) - err := index.Add(id, vector) - require.Nil(t, err) - // index should grow to 5001 - assert.Equal(t, int(id)+cache.MinimumIndexGrowthDelta, len(index.nodes)) - assert.Equal(t, int32(id+2*cache.MinimumIndexGrowthDelta), index.cache.Len()) - // try to add a vector with id: 8001 - id = uint64(6*cache.InitialSize + cache.MinimumIndexGrowthDelta + 1) - err = index.Add(id, vector) - require.Nil(t, err) - // index should grow to at least 8001 - assert.GreaterOrEqual(t, len(index.nodes), 8001) - assert.GreaterOrEqual(t, index.cache.Len(), int32(8001)) - }) - - t.Run("should grow index", func(t *testing.T) { - // should not increase the nodes size - sizeBefore := len(index.nodes) - cacheBefore := index.cache.Len() - idDontGrowIndex := uint64(6*cache.InitialSize - 1) - err := index.Add(idDontGrowIndex, vector) - require.Nil(t, err) - assert.Equal(t, sizeBefore, len(index.nodes)) - assert.Equal(t, cacheBefore, index.cache.Len()) - // should increase nodes - id := uint64(8*cache.InitialSize + 1) - err = index.Add(id, vector) - require.Nil(t, err) - assert.GreaterOrEqual(t, len(index.nodes), int(id)) - assert.GreaterOrEqual(t, index.cache.Len(), int32(id)) - // should increase nodes when a much greater id is passed - id = uint64(20*cache.InitialSize + 22) - err = index.Add(id, vector) - require.Nil(t, err) - assert.Equal(t, int(id)+cache.MinimumIndexGrowthDelta, len(index.nodes)) - assert.Equal(t, int32(id+2*cache.MinimumIndexGrowthDelta), index.cache.Len()) - }) -} - -func createEmptyHnswIndexForTests(t testing.TB, vecForIDFn common.VectorForID[float32]) *hnsw { - // mock out commit logger before adding data so we don't leave a disk - // footprint. Commit logging and deserializing from a (condensed) commit log - // is tested in a separate integration test that takes care of providing and - // cleaning up the correct place on disk to write test files - index, err := New(Config{ - RootPath: "doesnt-matter-as-committlogger-is-mocked-out", - ID: "unittest", - MakeCommitLoggerThunk: MakeNoopCommitLogger, - DistanceProvider: distancer.NewCosineDistanceProvider(), - VectorForIDThunk: vecForIDFn, - }, ent.UserConfig{ - MaxConnections: 30, - EFConstruction: 60, - }, cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - cyclemanager.NewCallbackGroupNoop(), testinghelpers.NewDummyStore(t)) - require.Nil(t, err) - return index -} diff --git a/adapters/repos/db/vector/hnsw/index_too_many_links_bug_integration_test.go b/adapters/repos/db/vector/hnsw/index_too_many_links_bug_integration_test.go deleted file mode 100644 index 2807c9c27815c174c2bab73925ec959a1f56147d..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/index_too_many_links_bug_integration_test.go +++ /dev/null @@ -1,271 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest && !race -// +build integrationTest,!race - -package hnsw - -import ( - "context" - "runtime" - "sync" - "testing" - - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" - "github.com/weaviate/weaviate/adapters/repos/db/vector/testinghelpers" - "github.com/weaviate/weaviate/entities/cyclemanager" - ent "github.com/weaviate/weaviate/entities/vectorindex/hnsw" -) - -// The !race build tag makes sure that this test is EXCLUDED from running with -// the race detector on, but now we also need to make sure that it runs in the -// separate no-race test run. To INCLUDE it there we use the Test_NoRace_ -// prefix. -// This test imports 10,000 objects concurrently which is extremely expensive -// with the race detector on. -// It prevents a regression on -// https://github.com/weaviate/weaviate/issues/1868 -func Test_NoRace_ManySmallCommitlogs(t *testing.T) { - n := 10000 - dim := 16 - m := 8 - - r := getRandomSeed() - rootPath := t.TempDir() - - logger, _ := test.NewNullLogger() - ctx := context.Background() - - parentCommitLoggerCallbacks := cyclemanager.NewCallbackGroup("parentCommitLogger", logger, 1) - parentCommitLoggerCycle := cyclemanager.NewManager( - cyclemanager.HnswCommitLoggerCycleTicker(), - parentCommitLoggerCallbacks.CycleCallback) - parentCommitLoggerCycle.Start() - defer parentCommitLoggerCycle.StopAndWait(ctx) - commitLoggerCallbacks := cyclemanager.NewCallbackGroup("childCommitLogger", logger, 1) - commitLoggerCallbacksCtrl := parentCommitLoggerCallbacks.Register("commitLogger", commitLoggerCallbacks.CycleCallback) - - parentTombstoneCleanupCallbacks := cyclemanager.NewCallbackGroup("parentTombstoneCleanup", logger, 1) - parentTombstoneCleanupCycle := cyclemanager.NewManager( - cyclemanager.NewFixedTicker(1), - parentTombstoneCleanupCallbacks.CycleCallback) - parentTombstoneCleanupCycle.Start() - defer parentTombstoneCleanupCycle.StopAndWait(ctx) - tombstoneCleanupCallbacks := cyclemanager.NewCallbackGroup("childTombstoneCleanup", logger, 1) - tombstoneCleanupCallbacksCtrl := parentTombstoneCleanupCallbacks.Register("tombstoneCleanup", tombstoneCleanupCallbacks.CycleCallback) - - original, err := NewCommitLogger(rootPath, "too_many_links_test", logger, commitLoggerCallbacks, - WithCommitlogThreshold(1e5), - WithCommitlogThresholdForCombining(5e5)) - require.Nil(t, err) - - data := make([][]float32, n) - for i := range data { - data[i] = make([]float32, dim) - for j := range data[i] { - data[i][j] = r.Float32() - } - - } - - var index *hnsw - - t.Run("set up an index with the specified commit logger", func(t *testing.T) { - idx, err := New(Config{ - MakeCommitLoggerThunk: func() (CommitLogger, error) { - return original, nil - }, - ID: "too_many_links_test", - RootPath: rootPath, - DistanceProvider: distancer.NewCosineDistanceProvider(), - Logger: logger, - VectorForIDThunk: func(ctx context.Context, id uint64) ([]float32, error) { - return data[int(id)], nil - }, - }, ent.UserConfig{ - MaxConnections: m, - EFConstruction: 128, - CleanupIntervalSeconds: 0, - - // The actual size does not matter for this test, but if it defaults to - // zero it will constantly think it's full and needs to be deleted - even - // after just being deleted, so make sure to use a positive number here. - VectorCacheMaxObjects: 2 * n, - }, tombstoneCleanupCallbacks, cyclemanager.NewCallbackGroupNoop(), - cyclemanager.NewCallbackGroupNoop(), testinghelpers.NewDummyStore(t)) - require.Nil(t, err) - idx.PostStartup() - index = idx - }) - - t.Run("add data", func(t *testing.T) { - type tuple struct { - vec []float32 - id uint64 - } - - jobs := make(chan tuple, n) - - wg := sync.WaitGroup{} - worker := func(jobs chan tuple) { - for job := range jobs { - index.Add(job.id, job.vec) - } - - wg.Done() - } - - for i := 0; i < runtime.GOMAXPROCS(0); i++ { - wg.Add(1) - go worker(jobs) - } - - for i, vec := range data { - jobs <- tuple{id: uint64(i), vec: vec} - } - - close(jobs) - - wg.Wait() - }) - - index.Flush() - - t.Run("verify there are no nodes with too many links - control", func(t *testing.T) { - for i, node := range index.nodes { - if node == nil { - continue - } - - for level, conns := range node.connections { - m := index.maximumConnections - if level == 0 { - m = index.maximumConnectionsLayerZero - } - - assert.LessOrEqualf(t, len(conns), m, "node %d at level %d with %d conns", - i, level, len(conns)) - } - } - }) - - t.Run("delete 10 percent of data", func(t *testing.T) { - type tuple struct { - vec []float32 - id uint64 - } - - jobs := make(chan tuple, n) - - wg := sync.WaitGroup{} - worker := func(jobs chan tuple) { - for job := range jobs { - index.Delete(job.id) - } - - wg.Done() - } - - for i := 0; i < runtime.GOMAXPROCS(0); i++ { - wg.Add(1) - go worker(jobs) - } - - for i, vec := range data[:n/10] { - jobs <- tuple{id: uint64(i), vec: vec} - } - - close(jobs) - - wg.Wait() - }) - - index.Flush() - - t.Run("verify there are no nodes with too many links - post deletion", func(t *testing.T) { - for i, node := range index.nodes { - if node == nil { - continue - } - - for level, conns := range node.connections { - m := index.maximumConnections - if level == 0 { - m = index.maximumConnectionsLayerZero - } - - assert.LessOrEqualf(t, len(conns), m, "node %d at level %d with %d conns", - i, level, len(conns)) - } - } - }) - - t.Run("destroy the old index", func(t *testing.T) { - // kill the commit loger and index - require.Nil(t, original.Shutdown(context.Background())) - index = nil - original = nil - }) - - t.Run("create a new one from the disk files", func(t *testing.T) { - idx, err := New(Config{ - MakeCommitLoggerThunk: MakeNoopCommitLogger, // no longer need a real one - ID: "too_many_links_test", - RootPath: rootPath, - DistanceProvider: distancer.NewCosineDistanceProvider(), - Logger: logger, - VectorForIDThunk: func(ctx context.Context, id uint64) ([]float32, error) { - return data[int(id)], nil - }, - }, ent.UserConfig{ - MaxConnections: m, - EFConstruction: 128, - CleanupIntervalSeconds: 1, - - // The actual size does not matter for this test, but if it defaults to - // zero it will constantly think it's full and needs to be deleted - even - // after just being deleted, so make sure to use a positive number here. - VectorCacheMaxObjects: 2 * n, - }, tombstoneCleanupCallbacks, cyclemanager.NewCallbackGroupNoop(), - cyclemanager.NewCallbackGroupNoop(), testinghelpers.NewDummyStore(t)) - require.Nil(t, err) - idx.PostStartup() - index = idx - }) - - t.Run("verify there are no nodes with too many links - after restart", func(t *testing.T) { - for i, node := range index.nodes { - if node == nil { - continue - } - - for level, conns := range node.connections { - m := index.maximumConnections - if level == 0 { - m = index.maximumConnectionsLayerZero - } - - require.LessOrEqualf(t, len(conns), m, "node %d at level %d with %d conns", - i, level, len(conns)) - } - } - }) - - t.Run("destroy the index", func(t *testing.T) { - require.Nil(t, index.Drop(context.Background())) - require.Nil(t, commitLoggerCallbacksCtrl.Unregister(ctx)) - require.Nil(t, tombstoneCleanupCallbacksCtrl.Unregister(ctx)) - }) -} diff --git a/adapters/repos/db/vector/hnsw/insert.go b/adapters/repos/db/vector/hnsw/insert.go deleted file mode 100644 index ab1ef2b095cb2d258d397d19add8761235f384bb..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/insert.go +++ /dev/null @@ -1,267 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "context" - "fmt" - "math" - "sync/atomic" - "time" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/adapters/repos/db/vector/compressionhelpers" -) - -func (h *hnsw) ValidateBeforeInsert(vector []float32) error { - dims := int(atomic.LoadInt32(&h.dims)) - - // no vectors exist - if dims == 0 { - return nil - } - - // check if vector length is the same as existing nodes - if dims != len(vector) { - return fmt.Errorf("new node has a vector with length %v. "+ - "Existing nodes have vectors with length %v", len(vector), dims) - } - - return nil -} - -func (h *hnsw) AddBatch(ctx context.Context, ids []uint64, vectors [][]float32) error { - if err := ctx.Err(); err != nil { - return err - } - if len(ids) != len(vectors) { - return errors.Errorf("ids and vectors sizes does not match") - } - if len(ids) == 0 { - return errors.Errorf("insertBatch called with empty lists") - } - h.trackDimensionsOnce.Do(func() { - atomic.StoreInt32(&h.dims, int32(len(vectors[0]))) - }) - levels := make([]int, len(ids)) - maxId := uint64(0) - for i, id := range ids { - if maxId < id { - maxId = id - } - levels[i] = int(math.Floor(-math.Log(h.randFunc()) * h.levelNormalizer)) - } - h.RLock() - if maxId >= uint64(len(h.nodes)) { - h.RUnlock() - h.Lock() - if maxId >= uint64(len(h.nodes)) { - err := h.growIndexToAccomodateNode(maxId, h.logger) - if err != nil { - h.Unlock() - return errors.Wrapf(err, "grow HNSW index to accommodate node %d", maxId) - } - } - h.Unlock() - } else { - h.RUnlock() - } - - for i := range ids { - if err := ctx.Err(); err != nil { - return err - } - - vector := vectors[i] - node := &vertex{ - id: ids[i], - level: levels[i], - } - globalBefore := time.Now() - if len(vector) == 0 { - return errors.Errorf("insert called with nil-vector") - } - - h.metrics.InsertVector() - - vector = h.normalizeVec(vector) - err := h.addOne(vector, node) - if err != nil { - return err - } - - h.insertMetrics.total(globalBefore) - } - return nil -} - -func (h *hnsw) addOne(vector []float32, node *vertex) error { - h.compressActionLock.RLock() - h.deleteVsInsertLock.RLock() - - before := time.Now() - - defer func() { - h.deleteVsInsertLock.RUnlock() - h.compressActionLock.RUnlock() - h.insertMetrics.updateGlobalEntrypoint(before) - }() - - wasFirst := false - var firstInsertError error - h.initialInsertOnce.Do(func() { - if h.isEmpty() { - wasFirst = true - firstInsertError = h.insertInitialElement(node, vector) - } - }) - if wasFirst { - if firstInsertError != nil { - return firstInsertError - } - return nil - } - - node.markAsMaintenance() - - h.RLock() - // initially use the "global" entrypoint which is guaranteed to be on the - // currently highest layer - entryPointID := h.entryPointID - // initially use the level of the entrypoint which is the highest level of - // the h-graph in the first iteration - currentMaximumLayer := h.currentMaximumLayer - h.RUnlock() - - targetLevel := node.level - node.connections = make([][]uint64, targetLevel+1) - - for i := targetLevel; i >= 0; i-- { - capacity := h.maximumConnections - if i == 0 { - capacity = h.maximumConnectionsLayerZero - } - - node.connections[i] = make([]uint64, 0, capacity) - } - - if err := h.commitLog.AddNode(node); err != nil { - return err - } - - nodeId := node.id - - h.shardedNodeLocks.Lock(nodeId) - h.nodes[nodeId] = node - h.shardedNodeLocks.Unlock(nodeId) - - if h.compressed.Load() { - h.compressor.Preload(node.id, vector) - } else { - h.cache.Preload(node.id, vector) - } - - h.insertMetrics.prepareAndInsertNode(before) - before = time.Now() - - var err error - var distancer compressionhelpers.CompressorDistancer - var returnFn compressionhelpers.ReturnDistancerFn - if h.compressed.Load() { - distancer, returnFn = h.compressor.NewDistancer(vector) - } - entryPointID, err = h.findBestEntrypointForNode(currentMaximumLayer, targetLevel, - entryPointID, vector, distancer) - if h.compressed.Load() { - returnFn() - } - if err != nil { - return errors.Wrap(err, "find best entrypoint") - } - - h.insertMetrics.findEntrypoint(before) - before = time.Now() - - // TODO: check findAndConnectNeighbors... - if err := h.findAndConnectNeighbors(node, entryPointID, vector, distancer, - targetLevel, currentMaximumLayer, helpers.NewAllowList()); err != nil { - return errors.Wrap(err, "find and connect neighbors") - } - - h.insertMetrics.findAndConnectTotal(before) - before = time.Now() - - node.unmarkAsMaintenance() - - h.RLock() - if targetLevel > h.currentMaximumLayer { - h.RUnlock() - h.Lock() - // check again to avoid changes from RUnlock to Lock again - if targetLevel > h.currentMaximumLayer { - if err := h.commitLog.SetEntryPointWithMaxLayer(nodeId, targetLevel); err != nil { - h.Unlock() - return err - } - - h.entryPointID = nodeId - h.currentMaximumLayer = targetLevel - } - h.Unlock() - } else { - h.RUnlock() - } - - return nil -} - -func (h *hnsw) Add(id uint64, vector []float32) error { - return h.AddBatch(context.TODO(), []uint64{id}, [][]float32{vector}) -} - -func (h *hnsw) insertInitialElement(node *vertex, nodeVec []float32) error { - h.Lock() - defer h.Unlock() - - if err := h.commitLog.SetEntryPointWithMaxLayer(node.id, 0); err != nil { - return err - } - - h.entryPointID = node.id - h.currentMaximumLayer = 0 - node.connections = [][]uint64{ - make([]uint64, 0, h.maximumConnectionsLayerZero), - } - node.level = 0 - if err := h.commitLog.AddNode(node); err != nil { - return err - } - - err := h.growIndexToAccomodateNode(node.id, h.logger) - if err != nil { - return errors.Wrapf(err, "grow HNSW index to accommodate node %d", node.id) - } - - h.shardedNodeLocks.Lock(node.id) - h.nodes[node.id] = node - h.shardedNodeLocks.Unlock(node.id) - - if h.compressed.Load() { - h.compressor.Preload(node.id, nodeVec) - } else { - h.cache.Preload(node.id, nodeVec) - } - - // go h.insertHook(node.id, 0, node.connections) - return nil -} diff --git a/adapters/repos/db/vector/hnsw/insert_metrics.go b/adapters/repos/db/vector/hnsw/insert_metrics.go deleted file mode 100644 index eb835b172e7a19b15f48a00294c04f5bd28330f6..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/insert_metrics.go +++ /dev/null @@ -1,38 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -type insertMetrics struct { - total Observer - prepareAndInsertNode Observer - findEntrypoint Observer - updateGlobalEntrypoint Observer - findAndConnectTotal Observer - findAndConnectSearch Observer - findAndConnectHeuristic Observer - findAndConnectUpdateConnections Observer -} - -// newInsertMetrics curries the prometheus observers just once at creation time -// and therefore avoids having to make a lot of allocations on the hot path -func newInsertMetrics(metrics *Metrics) *insertMetrics { - return &insertMetrics{ - total: metrics.TrackInsertObserver("total"), - prepareAndInsertNode: metrics.TrackInsertObserver("prepare_and_insert_node"), - findEntrypoint: metrics.TrackInsertObserver("find_entrypoint"), - updateGlobalEntrypoint: metrics.TrackInsertObserver("update_global_entrypoint"), - findAndConnectTotal: metrics.TrackInsertObserver("find_and_connect_total"), - findAndConnectSearch: metrics.TrackInsertObserver("find_and_connect_search"), - findAndConnectHeuristic: metrics.TrackInsertObserver("find_and_connect_heuristic"), - findAndConnectUpdateConnections: metrics.TrackInsertObserver("find_and_connect_update_connections"), - } -} diff --git a/adapters/repos/db/vector/hnsw/maintenance.go b/adapters/repos/db/vector/hnsw/maintenance.go deleted file mode 100644 index 47c7af53b6e321851f72c76fffb2980863313e0a..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/maintenance.go +++ /dev/null @@ -1,121 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "time" - - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/adapters/repos/db/vector/cache" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/visited" -) - -const ( - indexGrowthRate = 1.25 -) - -// growIndexToAccomodateNode is a wrapper around the growIndexToAccomodateNode -// function growing the index of the hnsw struct. It does not do any locking on -// its own, make sure that this function is called from a single-thread or -// locked situation -func (h *hnsw) growIndexToAccomodateNode(id uint64, logger logrus.FieldLogger) error { - defer func() { - h.metrics.SetSize(len(h.nodes)) - }() - - before := time.Now() - - // check whether h.nodes slice needs growing - // not to unnecessarily lock h.shardedNodeLocks - if id < uint64(len(h.nodes)) { - return nil - } - - // lock h.nodes' individual elements to avoid race between writing to elements - // and copying entire slice in growIndexToAccomodateNode method - newIndex, err := func() ([]*vertex, error) { - h.shardedNodeLocks.RLockAll() - defer h.shardedNodeLocks.RUnlockAll() - - newIndex, _, err := growIndexToAccomodateNode(h.nodes, id, logger) - return newIndex, err - }() - if err != nil { - return err - } - - defer h.metrics.GrowDuration(before) - - if h.compressed.Load() { - h.compressor.GrowCache(uint64(len(newIndex))) - } else { - h.cache.Grow(uint64(len(newIndex))) - } - - h.pools.visitedListsLock.Lock() - h.pools.visitedLists.Destroy() - h.pools.visitedLists = nil - h.pools.visitedLists = visited.NewPool(1, len(newIndex)+512) - h.pools.visitedListsLock.Unlock() - - h.shardedNodeLocks.LockAll() - h.nodes = newIndex - h.shardedNodeLocks.UnlockAll() - - return nil -} - -// growIndexToAccomodateNode does not lock the graph for writes as the -// assumption is that it is called as part of an operation that is already -// wrapped inside a lock, such as inserting a node into the graph. If -// growIndexToAccomodateNode is ever called outside of such an operation, the -// caller must make sure to lock the graph as concurrent reads/write would -// otherwise be possible -func growIndexToAccomodateNode(index []*vertex, id uint64, - logger logrus.FieldLogger, -) ([]*vertex, bool, error) { - previousSize := uint64(len(index)) - if id < previousSize { - // node will fit, nothing to do - return nil, false, nil - } - before := time.Now() - - var newSize uint64 - - if (indexGrowthRate-1)*float64(previousSize) < float64(cache.MinimumIndexGrowthDelta) { - // typically grow the index by the delta - newSize = previousSize + cache.MinimumIndexGrowthDelta - } else { - newSize = uint64(float64(previousSize) * indexGrowthRate) - } - - if newSize <= id { - // There are situations were docIDs are not in order. For example, if the - // default size is 10k and the default delta is 10k. Imagine the user - // imports 21 objects, then deletes the first 20,500. When rebuilding the - // index from disk the first id to be imported would be 20,501, however the - // index default size and default delta would only reach up to 20,000. - newSize = id + cache.MinimumIndexGrowthDelta - } - - newIndex := make([]*vertex, newSize) - copy(newIndex, index) - - took := time.Since(before) - logger.WithField("action", "hnsw_grow_index"). - WithField("took", took). - WithField("previous_size", previousSize). - WithField("new_size", newSize). - Debugf("index grown from %d to %d, took %s\n", previousSize, newSize, took) - return newIndex, true, nil -} diff --git a/adapters/repos/db/vector/hnsw/maintenance_test.go b/adapters/repos/db/vector/hnsw/maintenance_test.go deleted file mode 100644 index 56cbae40f60f61d4bda55b7d904d327375b20407..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/maintenance_test.go +++ /dev/null @@ -1,133 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "testing" - - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/adapters/repos/db/vector/cache" -) - -func Test_growIndexToAccomodateNode(t *testing.T) { - createVertexSlice := func(size int) []*vertex { - index := make([]*vertex, size) - for i := 0; i < len(index); i++ { - index[i] = &vertex{id: uint64(i)} - } - return index - } - type args struct { - index []*vertex - id uint64 - } - tests := []struct { - name string - args args - wantIndexSize int - changed bool - err error - }{ - { - name: "is one before the initial size", - args: args{ - id: cache.InitialSize - 1, - index: createVertexSlice(cache.InitialSize), - }, - wantIndexSize: 0, - changed: false, - }, - { - name: "exactly equals the initial size", - args: args{ - id: cache.InitialSize, - index: createVertexSlice(cache.InitialSize), - }, - wantIndexSize: cache.InitialSize + cache.MinimumIndexGrowthDelta, - changed: true, - }, - { - name: "is one after the initial size", - args: args{ - id: cache.InitialSize + 1, - index: createVertexSlice(cache.InitialSize), - }, - wantIndexSize: cache.InitialSize + cache.MinimumIndexGrowthDelta, - changed: true, - }, - { - name: "4 times the initial size minus 1", - args: args{ - id: 4*cache.InitialSize - 1, - index: createVertexSlice(cache.InitialSize), - }, - wantIndexSize: 4*cache.InitialSize - 1 + cache.MinimumIndexGrowthDelta, - changed: true, - }, - { - name: "4 times the initial size", - args: args{ - id: 4 * cache.InitialSize, - index: createVertexSlice(cache.InitialSize), - }, - wantIndexSize: 4*cache.InitialSize + cache.MinimumIndexGrowthDelta, - changed: true, - }, - { - name: "4 times the initial size plus 1", - args: args{ - id: 4*cache.InitialSize + 1, - index: createVertexSlice(cache.InitialSize), - }, - wantIndexSize: 4*cache.InitialSize + 1 + cache.MinimumIndexGrowthDelta, - changed: true, - }, - { - name: "14160016 case", - args: args{ - id: uint64(14160016), - index: createVertexSlice(14160016), - }, - wantIndexSize: int(14160016 * indexGrowthRate), - changed: true, - }, - { - name: "panic case", - args: args{ - id: uint64(cache.InitialSize + cache.MinimumIndexGrowthDelta + 1), - index: createVertexSlice(cache.InitialSize + 1), - }, - wantIndexSize: cache.InitialSize + 1 + 2*cache.MinimumIndexGrowthDelta, - changed: true, - }, - } - for _, tt := range tests { - logger, _ := test.NewNullLogger() - t.Run(tt.name, func(t *testing.T) { - newNodes, changed, err := growIndexToAccomodateNode(tt.args.index, tt.args.id, logger) - assert.Len(t, newNodes, tt.wantIndexSize) - assert.Equal(t, tt.changed, changed) - if err != nil { - require.NotNil(t, tt.err) - assert.EqualError(t, err, tt.err.Error()) - } - // check the newly grown index - index := tt.args.index - if changed { - index = newNodes - } - assert.Greater(t, len(index), int(tt.args.id)) - }) - } -} diff --git a/adapters/repos/db/vector/hnsw/metrics.go b/adapters/repos/db/vector/hnsw/metrics.go deleted file mode 100644 index c902aa09e9471615ffa2cadbeb3987eb3e3790bf..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/metrics.go +++ /dev/null @@ -1,267 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "time" - - "github.com/prometheus/client_golang/prometheus" - "github.com/weaviate/weaviate/usecases/monitoring" -) - -type Metrics struct { - enabled bool - tombstones prometheus.Gauge - threads prometheus.Gauge - insert prometheus.Gauge - insertTime prometheus.ObserverVec - delete prometheus.Gauge - deleteTime prometheus.ObserverVec - cleaned prometheus.Counter - size prometheus.Gauge - grow prometheus.Observer - startupProgress prometheus.Gauge - startupDurations prometheus.ObserverVec - startupDiskIO prometheus.ObserverVec -} - -func NewMetrics(prom *monitoring.PrometheusMetrics, - className, shardName string, -) *Metrics { - if prom == nil { - return &Metrics{enabled: false} - } - - if prom.Group { - className = "n/a" - shardName = "n/a" - } - - tombstones := prom.VectorIndexTombstones.With(prometheus.Labels{ - "class_name": className, - "shard_name": shardName, - }) - - threads := prom.VectorIndexTombstoneCleanupThreads.With(prometheus.Labels{ - "class_name": className, - "shard_name": shardName, - }) - - cleaned := prom.VectorIndexTombstoneCleanedCount.With(prometheus.Labels{ - "class_name": className, - "shard_name": shardName, - }) - - insert := prom.VectorIndexOperations.With(prometheus.Labels{ - "class_name": className, - "shard_name": shardName, - "operation": "create", - }) - - insertTime := prom.VectorIndexDurations.MustCurryWith(prometheus.Labels{ - "class_name": className, - "shard_name": shardName, - "operation": "create", - }) - - del := prom.VectorIndexOperations.With(prometheus.Labels{ - "class_name": className, - "shard_name": shardName, - "operation": "delete", - }) - - deleteTime := prom.VectorIndexDurations.MustCurryWith(prometheus.Labels{ - "class_name": className, - "shard_name": shardName, - "operation": "delete", - }) - - size := prom.VectorIndexSize.With(prometheus.Labels{ - "class_name": className, - "shard_name": shardName, - }) - - grow := prom.VectorIndexMaintenanceDurations.With(prometheus.Labels{ - "class_name": className, - "shard_name": shardName, - "operation": "grow", - }) - - startupProgress := prom.StartupProgress.With(prometheus.Labels{ - "class_name": className, - "shard_name": shardName, - "operation": "hnsw_read_commitlogs", - }) - - startupDurations := prom.StartupDurations.MustCurryWith(prometheus.Labels{ - "class_name": className, - "shard_name": shardName, - }) - - startupDiskIO := prom.StartupDiskIO.MustCurryWith(prometheus.Labels{ - "class_name": className, - "shard_name": shardName, - }) - - return &Metrics{ - enabled: true, - tombstones: tombstones, - threads: threads, - cleaned: cleaned, - insert: insert, - insertTime: insertTime, - delete: del, - deleteTime: deleteTime, - size: size, - grow: grow, - startupProgress: startupProgress, - startupDurations: startupDurations, - startupDiskIO: startupDiskIO, - } -} - -func (m *Metrics) AddTombstone() { - if !m.enabled { - return - } - - m.tombstones.Inc() -} - -func (m *Metrics) RemoveTombstone() { - if !m.enabled { - return - } - - m.tombstones.Dec() -} - -func (m *Metrics) StartCleanup(threads int) { - if !m.enabled { - return - } - - m.threads.Add(float64(threads)) -} - -func (m *Metrics) EndCleanup(threads int) { - if !m.enabled { - return - } - - m.threads.Sub(float64(threads)) -} - -func (m *Metrics) CleanedUp() { - if !m.enabled { - return - } - - m.cleaned.Inc() -} - -func (m *Metrics) InsertVector() { - if !m.enabled { - return - } - - m.insert.Inc() -} - -func (m *Metrics) DeleteVector() { - if !m.enabled { - return - } - - m.delete.Inc() -} - -func (m *Metrics) SetSize(size int) { - if !m.enabled { - return - } - - m.size.Set(float64(size)) -} - -func (m *Metrics) GrowDuration(start time.Time) { - if !m.enabled { - return - } - - took := float64(time.Since(start)) / float64(time.Millisecond) - m.grow.Observe(took) -} - -type Observer func(start time.Time) - -func noOpObserver(start time.Time) { - // do nothing -} - -func (m *Metrics) TrackInsertObserver(step string) Observer { - if !m.enabled { - return noOpObserver - } - - curried := m.insertTime.With(prometheus.Labels{"step": step}) - - return func(start time.Time) { - took := float64(time.Since(start)) / float64(time.Millisecond) - curried.Observe(took) - } -} - -func (m *Metrics) TrackDelete(start time.Time, step string) { - if !m.enabled { - return - } - - took := float64(time.Since(start)) / float64(time.Millisecond) - m.deleteTime.With(prometheus.Labels{"step": step}).Observe(took) -} - -func (m *Metrics) StartupProgress(ratio float64) { - if !m.enabled { - return - } - - m.startupProgress.Set(ratio) -} - -func (m *Metrics) TrackStartupTotal(start time.Time) { - if !m.enabled { - return - } - - took := float64(time.Since(start)) / float64(time.Millisecond) - m.startupDurations.With(prometheus.Labels{"operation": "hnsw_read_all_commitlogs"}).Observe(took) -} - -func (m *Metrics) TrackStartupIndividual(start time.Time) { - if !m.enabled { - return - } - - took := float64(time.Since(start)) / float64(time.Millisecond) - m.startupDurations.With(prometheus.Labels{"operation": "hnsw_read_single_commitlog"}).Observe(took) -} - -func (m *Metrics) TrackStartupReadCommitlogDiskIO(read int64, nanoseconds int64) { - if !m.enabled { - return - } - - seconds := float64(nanoseconds) / float64(time.Second) - throughput := float64(read) / float64(seconds) - m.startupDiskIO.With(prometheus.Labels{"operation": "hnsw_read_commitlog"}).Observe(throughput) -} diff --git a/adapters/repos/db/vector/hnsw/neighbor_connections.go b/adapters/repos/db/vector/hnsw/neighbor_connections.go deleted file mode 100644 index 538dbb45fe62b543ee11e888c337a98a84ad0632..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/neighbor_connections.go +++ /dev/null @@ -1,317 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "context" - "time" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/adapters/repos/db/priorityqueue" - "github.com/weaviate/weaviate/adapters/repos/db/vector/compressionhelpers" -) - -func (h *hnsw) findAndConnectNeighbors(node *vertex, - entryPointID uint64, nodeVec []float32, distancer compressionhelpers.CompressorDistancer, targetLevel, currentMaxLevel int, - denyList helpers.AllowList, -) error { - nfc := newNeighborFinderConnector(h, node, entryPointID, nodeVec, distancer, targetLevel, - currentMaxLevel, denyList) - - return nfc.Do() -} - -type neighborFinderConnector struct { - graph *hnsw - node *vertex - entryPointID uint64 - entryPointDist float32 - nodeVec []float32 - distancer compressionhelpers.CompressorDistancer - targetLevel int - currentMaxLevel int - denyList helpers.AllowList - // bufLinksLog BufferedLinksLogger -} - -func newNeighborFinderConnector(graph *hnsw, node *vertex, entryPointID uint64, - nodeVec []float32, distancer compressionhelpers.CompressorDistancer, targetLevel, currentMaxLevel int, - denyList helpers.AllowList, -) *neighborFinderConnector { - return &neighborFinderConnector{ - graph: graph, - node: node, - entryPointID: entryPointID, - nodeVec: nodeVec, - distancer: distancer, - targetLevel: targetLevel, - currentMaxLevel: currentMaxLevel, - denyList: denyList, - } -} - -func (n *neighborFinderConnector) Do() error { - for level := min(n.targetLevel, n.currentMaxLevel); level >= 0; level-- { - err := n.doAtLevel(level) - if err != nil { - return errors.Wrapf(err, "at level %d", level) - } - } - - return nil -} - -func (n *neighborFinderConnector) doAtLevel(level int) error { - before := time.Now() - if err := n.pickEntrypoint(); err != nil { - return errors.Wrap(err, "pick entrypoint at level beginning") - } - - eps := priorityqueue.NewMin[any](1) - eps.Insert(n.entryPointID, n.entryPointDist) - - results, err := n.graph.searchLayerByVectorWithDistancer(n.nodeVec, eps, n.graph.efConstruction, - level, nil, n.distancer) - if err != nil { - return errors.Wrapf(err, "search layer at level %d", level) - } - - n.graph.insertMetrics.findAndConnectSearch(before) - before = time.Now() - - // max := n.maximumConnections(level) - max := n.graph.maximumConnections - if err := n.graph.selectNeighborsHeuristic(results, max, n.denyList); err != nil { - return errors.Wrap(err, "heuristic") - } - - n.graph.insertMetrics.findAndConnectHeuristic(before) - before = time.Now() - - // // for distributed spike - // neighborsAtLevel[level] = neighbors - - neighbors := make([]uint64, 0, results.Len()) - for results.Len() > 0 { - id := results.Pop().ID - neighbors = append(neighbors, id) - } - - n.graph.pools.pqResults.Put(results) - - // set all outgoing in one go - n.node.setConnectionsAtLevel(level, neighbors) - n.graph.commitLog.ReplaceLinksAtLevel(n.node.id, level, neighbors) - - for _, neighborID := range neighbors { - if err := n.connectNeighborAtLevel(neighborID, level); err != nil { - return errors.Wrapf(err, "connect neighbor %d", neighborID) - } - } - - if len(neighbors) > 0 { - // there could be no neighbors left, if all are marked deleted, in this - // case, don't change the entrypoint - nextEntryPointID := neighbors[len(neighbors)-1] - if nextEntryPointID == n.node.id { - return nil - } - - n.entryPointID = nextEntryPointID - } - - n.graph.insertMetrics.findAndConnectUpdateConnections(before) - return nil -} - -func (n *neighborFinderConnector) connectNeighborAtLevel(neighborID uint64, - level int, -) error { - neighbor := n.graph.nodeByID(neighborID) - if skip := n.skipNeighbor(neighbor); skip { - return nil - } - - neighbor.Lock() - defer neighbor.Unlock() - if level > neighbor.level { - // upgrade neighbor level if the level is out of sync due to a delete re-assign - neighbor.upgradeToLevelNoLock(level) - } - currentConnections := neighbor.connectionsAtLevelNoLock(level) - - maximumConnections := n.maximumConnections(level) - if len(currentConnections) < maximumConnections { - // we can simply append - // updatedConnections = append(currentConnections, n.node.id) - neighbor.appendConnectionAtLevelNoLock(level, n.node.id, maximumConnections) - if err := n.graph.commitLog.AddLinkAtLevel(neighbor.id, level, n.node.id); err != nil { - return err - } - } else { - // we need to run the heuristic - - dist, ok, err := n.graph.distBetweenNodes(n.node.id, neighborID) - if err != nil { - return errors.Wrapf(err, "dist between %d and %d", n.node.id, neighborID) - } - - if !ok { - // it seems either the node or the neighbor were deleted in the meantime, - // there is nothing we can do now - return nil - } - - candidates := priorityqueue.NewMax[any](len(currentConnections) + 1) - candidates.Insert(n.node.id, dist) - - for _, existingConnection := range currentConnections { - dist, ok, err := n.graph.distBetweenNodes(existingConnection, neighborID) - if err != nil { - return errors.Wrapf(err, "dist between %d and %d", existingConnection, neighborID) - } - - if !ok { - // was deleted in the meantime - continue - } - - candidates.Insert(existingConnection, dist) - } - - err = n.graph.selectNeighborsHeuristic(candidates, maximumConnections, n.denyList) - if err != nil { - return errors.Wrap(err, "connect neighbors") - } - - neighbor.resetConnectionsAtLevelNoLock(level) - if err := n.graph.commitLog.ClearLinksAtLevel(neighbor.id, uint16(level)); err != nil { - return err - } - - for candidates.Len() > 0 { - id := candidates.Pop().ID - neighbor.appendConnectionAtLevelNoLock(level, id, maximumConnections) - if err := n.graph.commitLog.AddLinkAtLevel(neighbor.id, level, id); err != nil { - return err - } - } - } - - return nil -} - -func (n *neighborFinderConnector) skipNeighbor(neighbor *vertex) bool { - if neighbor == n.node { - // don't connect to self - return true - } - - if neighbor == nil || n.graph.hasTombstone(neighbor.id) { - // don't connect to tombstoned nodes. This would only increase the - // cleanup that needs to be done. Even worse: A tombstoned node can be - // cleaned up at any time, also while we are connecting to it. So, - // while the node still exists right now, it might already be nil in - // the next line, which would lead to a nil-pointer panic. - return true - } - - return false -} - -func (n *neighborFinderConnector) maximumConnections(level int) int { - if level == 0 { - return n.graph.maximumConnectionsLayerZero - } - - return n.graph.maximumConnections -} - -func (n *neighborFinderConnector) pickEntrypoint() error { - // the neighborFinderConnector always has a suggestion for an entrypoint that - // it got from the outside, most of the times we can use this, but in some - // cases we can't. To see if we can use it, three conditions need to be met: - // - // 1. it needs to exist in the graph, i.e. be not nil - // - // 2. it can't be under maintenance - // - // 3. we need to be able to obtain a vector for it - - localDeny := n.denyList.DeepCopy() - candidate := n.entryPointID - - // make sure the loop cannot block forever. In most cases, results should be - // found within micro to milliseconds, this is just a last resort to handle - // the unknown somewhat gracefully, for example if there is a bug in the - // underlying object store and we cannot retrieve the vector in time, etc. - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) - defer cancel() - - for { - if err := ctx.Err(); err != nil { - return err - } - - success, err := n.tryEpCandidate(candidate) - if err != nil { - return err - } - - if success { - return nil - } - - // no success so far, we need to keep going and find a better candidate - // make sure we never visit this candidate again - localDeny.Insert(candidate) - // now find a new one - - alternative, _ := n.graph.findNewLocalEntrypoint(localDeny, - n.graph.currentMaximumLayer, candidate) - candidate = alternative - } -} - -func (n *neighborFinderConnector) tryEpCandidate(candidate uint64) (bool, error) { - node := n.graph.nodeByID(candidate) - if node == nil { - return false, nil - } - - if node.isUnderMaintenance() { - return false, nil - } - - var dist float32 - var ok bool - var err error - if n.distancer == nil { - dist, ok, err = n.graph.distBetweenNodeAndVec(candidate, n.nodeVec) - } else { - dist, ok, err = n.distancer.DistanceToNode(candidate) - } - if err != nil { - // not an error we could recover from - fail! - return false, errors.Wrapf(err, - "calculate distance between insert node and entrypoint") - } - if !ok { - return false, nil - } - - // we were able to calculate a distance, we're good - n.entryPointDist = dist - n.entryPointID = candidate - return true, nil -} diff --git a/adapters/repos/db/vector/hnsw/periodic_tombstone_removal_test.go b/adapters/repos/db/vector/hnsw/periodic_tombstone_removal_test.go deleted file mode 100644 index 0e866ffbad2ac03cd852c30d994a681c6a2ed1f2..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/periodic_tombstone_removal_test.go +++ /dev/null @@ -1,92 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "context" - "testing" - "time" - - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" - "github.com/weaviate/weaviate/adapters/repos/db/vector/testinghelpers" - "github.com/weaviate/weaviate/entities/cyclemanager" - ent "github.com/weaviate/weaviate/entities/vectorindex/hnsw" - testhelper "github.com/weaviate/weaviate/test/helper" -) - -func TestPeriodicTombstoneRemoval(t *testing.T) { - logger, _ := test.NewNullLogger() - cleanupIntervalSeconds := 1 - tombstoneCallbacks := cyclemanager.NewCallbackGroup("tombstone", logger, 1) - tombstoneCleanupCycle := cyclemanager.NewManager( - cyclemanager.NewFixedTicker(time.Duration(cleanupIntervalSeconds)*time.Second), - tombstoneCallbacks.CycleCallback) - tombstoneCleanupCycle.Start() - - index, err := New(Config{ - RootPath: "doesnt-matter-as-committlogger-is-mocked-out", - ID: "automatic-tombstone-removal", - MakeCommitLoggerThunk: MakeNoopCommitLogger, - DistanceProvider: distancer.NewCosineDistanceProvider(), - VectorForIDThunk: testVectorForID, - }, ent.UserConfig{ - CleanupIntervalSeconds: cleanupIntervalSeconds, - MaxConnections: 30, - EFConstruction: 128, - }, tombstoneCallbacks, cyclemanager.NewCallbackGroupNoop(), - cyclemanager.NewCallbackGroupNoop(), testinghelpers.NewDummyStore(t)) - index.PostStartup() - - require.Nil(t, err) - - for i, vec := range testVectors { - err := index.Add(uint64(i), vec) - require.Nil(t, err) - } - - t.Run("delete an entry and verify there is a tombstone", func(t *testing.T) { - for i := range testVectors { - if i%2 != 0 { - continue - } - - err := index.Delete(uint64(i)) - require.Nil(t, err) - } - }) - - t.Run("verify there are now tombstones", func(t *testing.T) { - index.tombstoneLock.RLock() - ts := len(index.tombstones) - index.tombstoneLock.RUnlock() - assert.True(t, ts > 0) - }) - - t.Run("wait for tombstones to disappear", func(t *testing.T) { - testhelper.AssertEventuallyEqual(t, true, func() interface{} { - index.tombstoneLock.RLock() - ts := len(index.tombstones) - index.tombstoneLock.RUnlock() - return ts == 0 - }, "wait until tombstones have been cleaned up") - }) - - if err := index.Shutdown(context.Background()); err != nil { - t.Fatal(err) - } - if err := tombstoneCleanupCycle.StopAndWait(context.Background()); err != nil { - t.Fatal(err) - } -} diff --git a/adapters/repos/db/vector/hnsw/persistence_integration_test.go b/adapters/repos/db/vector/hnsw/persistence_integration_test.go deleted file mode 100644 index d2048f0350b1934ada6e9832c9924ef86a42e3b7..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/persistence_integration_test.go +++ /dev/null @@ -1,473 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package hnsw - -import ( - "context" - "fmt" - "io" - "os" - "path/filepath" - "testing" - "time" - - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" - "github.com/weaviate/weaviate/adapters/repos/db/vector/testinghelpers" - "github.com/weaviate/weaviate/entities/cyclemanager" - ent "github.com/weaviate/weaviate/entities/vectorindex/hnsw" -) - -func TestHnswPersistence(t *testing.T) { - dirName := t.TempDir() - indexID := "integrationtest" - - logger, _ := test.NewNullLogger() - cl, clErr := NewCommitLogger(dirName, indexID, logger, - cyclemanager.NewCallbackGroupNoop()) - makeCL := func() (CommitLogger, error) { - return cl, clErr - } - index, err := New(Config{ - RootPath: dirName, - ID: indexID, - MakeCommitLoggerThunk: makeCL, - DistanceProvider: distancer.NewCosineDistanceProvider(), - VectorForIDThunk: testVectorForID, - }, ent.UserConfig{ - MaxConnections: 30, - EFConstruction: 60, - }, cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - cyclemanager.NewCallbackGroupNoop(), testinghelpers.NewDummyStore(t)) - require.Nil(t, err) - - for i, vec := range testVectors { - err := index.Add(uint64(i), vec) - require.Nil(t, err) - } - - require.Nil(t, index.Flush()) - - // see index_test.go for more context - expectedResults := []uint64{ - 3, 5, 4, // cluster 2 - 7, 8, 6, // cluster 3 - 2, 1, 0, // cluster 1 - } - - t.Run("verify that the results match originally", func(t *testing.T) { - position := 3 - res, _, err := index.knnSearchByVector(testVectors[position], 50, 36, nil) - require.Nil(t, err) - assert.Equal(t, expectedResults, res) - }) - - // destroy the index - index = nil - - // build a new index from the (uncondensed) commit log - secondIndex, err := New(Config{ - RootPath: dirName, - ID: indexID, - MakeCommitLoggerThunk: makeCL, - DistanceProvider: distancer.NewCosineDistanceProvider(), - VectorForIDThunk: testVectorForID, - }, ent.UserConfig{ - MaxConnections: 30, - EFConstruction: 60, - }, cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - cyclemanager.NewCallbackGroupNoop(), testinghelpers.NewDummyStore(t)) - require.Nil(t, err) - - t.Run("verify that the results match after rebuilding from disk", - func(t *testing.T) { - position := 3 - res, _, err := secondIndex.knnSearchByVector(testVectors[position], 50, 36, nil) - require.Nil(t, err) - assert.Equal(t, expectedResults, res) - }) -} - -func TestHnswPersistence_CorruptWAL(t *testing.T) { - dirName := t.TempDir() - indexID := "integrationtest_corrupt" - - logger, _ := test.NewNullLogger() - cl, clErr := NewCommitLogger(dirName, indexID, logger, - cyclemanager.NewCallbackGroupNoop()) - makeCL := func() (CommitLogger, error) { - return cl, clErr - } - index, err := New(Config{ - RootPath: dirName, - ID: indexID, - MakeCommitLoggerThunk: makeCL, - DistanceProvider: distancer.NewCosineDistanceProvider(), - VectorForIDThunk: testVectorForID, - }, ent.UserConfig{ - MaxConnections: 30, - EFConstruction: 60, - }, cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - cyclemanager.NewCallbackGroupNoop(), testinghelpers.NewDummyStore(t)) - require.Nil(t, err) - - for i, vec := range testVectors { - err := index.Add(uint64(i), vec) - require.Nil(t, err) - } - - require.Nil(t, index.Flush()) - - // see index_test.go for more context - expectedResults := []uint64{ - 3, 5, 4, // cluster 2 - 7, 8, 6, // cluster 3 - 2, 1, 0, // cluster 1 - } - - t.Run("verify that the results match originally", func(t *testing.T) { - position := 3 - res, _, err := index.knnSearchByVector(testVectors[position], 50, 36, nil) - require.Nil(t, err) - assert.Equal(t, expectedResults, res) - }) - - // destroy the index - index.Shutdown(context.Background()) - index = nil - indexDir := filepath.Join(dirName, "integrationtest_corrupt.hnsw.commitlog.d") - - t.Run("corrupt the commit log on purpose", func(t *testing.T) { - res, err := os.ReadDir(indexDir) - require.Nil(t, err) - require.Len(t, res, 1) - fName := filepath.Join(indexDir, res[0].Name()) - newFName := filepath.Join(indexDir, fmt.Sprintf("%d", time.Now().Unix())) - - orig, err := os.Open(fName) - require.Nil(t, err) - - correctLog, err := io.ReadAll(orig) - require.Nil(t, err) - err = orig.Close() - require.Nil(t, err) - - os.Remove(fName) - - corruptLog := correctLog[:len(correctLog)-6] - corrupt, err := os.Create(newFName) - require.Nil(t, err) - - _, err = corrupt.Write(corruptLog) - require.Nil(t, err) - - err = corrupt.Close() - require.Nil(t, err) - - // double check that we only have one file left (the corrupted one) - res, err = os.ReadDir(indexDir) - require.Nil(t, err) - require.Len(t, res, 1) - }) - - // build a new index from the (uncondensed, corrupted) commit log - secondIndex, err := New(Config{ - RootPath: dirName, - ID: indexID, - MakeCommitLoggerThunk: makeCL, - DistanceProvider: distancer.NewCosineDistanceProvider(), - VectorForIDThunk: testVectorForID, - }, ent.UserConfig{ - MaxConnections: 30, - EFConstruction: 60, - }, cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - cyclemanager.NewCallbackGroupNoop(), testinghelpers.NewDummyStore(t)) - require.Nil(t, err) - - // the minor corruption (just one missing link) will most likely not render - // the index unusable, so we should still expect to retrieve results as - // normal - t.Run("verify that the results match after rebuilding from disk", - func(t *testing.T) { - position := 3 - res, _, err := secondIndex.knnSearchByVector(testVectors[position], 50, 36, nil) - require.Nil(t, err) - assert.Equal(t, expectedResults, res) - }) -} - -func TestHnswPersistence_WithDeletion_WithoutTombstoneCleanup(t *testing.T) { - dirName := t.TempDir() - indexID := "integrationtest_deletion" - logger, _ := test.NewNullLogger() - cl, clErr := NewCommitLogger(dirName, indexID, logger, - cyclemanager.NewCallbackGroupNoop()) - makeCL := func() (CommitLogger, error) { - return cl, clErr - } - index, err := New(Config{ - RootPath: dirName, - ID: indexID, - MakeCommitLoggerThunk: makeCL, - DistanceProvider: distancer.NewCosineDistanceProvider(), - VectorForIDThunk: testVectorForID, - }, ent.UserConfig{ - MaxConnections: 30, - EFConstruction: 60, - }, cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - cyclemanager.NewCallbackGroupNoop(), testinghelpers.NewDummyStore(t)) - require.Nil(t, err) - - for i, vec := range testVectors { - err := index.Add(uint64(i), vec) - require.Nil(t, err) - } - - t.Run("delete some elements", func(t *testing.T) { - err := index.Delete(6) - require.Nil(t, err) - err = index.Delete(8) - require.Nil(t, err) - }) - - // see index_test.go for more context - expectedResults := []uint64{ - 3, 5, 4, // cluster 2 - 7, // cluster 3 with element 6 and 8 deleted - 2, 1, 0, // cluster 1 - } - - require.Nil(t, index.Flush()) - - t.Run("verify that the results match originally", func(t *testing.T) { - position := 3 - res, _, err := index.knnSearchByVector(testVectors[position], 50, 36, nil) - require.Nil(t, err) - assert.Equal(t, expectedResults, res) - }) - - dumpIndex(index, "without_cleanup_original_index_before_storage") - - // destroy the index - index = nil - - // build a new index from the (uncondensed) commit log - secondIndex, err := New(Config{ - RootPath: dirName, - ID: indexID, - MakeCommitLoggerThunk: makeCL, - DistanceProvider: distancer.NewCosineDistanceProvider(), - VectorForIDThunk: testVectorForID, - }, ent.UserConfig{ - MaxConnections: 30, - EFConstruction: 60, - }, cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - cyclemanager.NewCallbackGroupNoop(), testinghelpers.NewDummyStore(t)) - require.Nil(t, err) - - dumpIndex(secondIndex, "without_cleanup_after_rebuild") - t.Run("verify that the results match after rebuilding from disk", - func(t *testing.T) { - position := 3 - res, _, err := secondIndex.knnSearchByVector(testVectors[position], 50, 36, nil) - require.Nil(t, err) - assert.Equal(t, expectedResults, res) - }) -} - -func TestHnswPersistence_WithDeletion_WithTombstoneCleanup(t *testing.T) { - dirName := t.TempDir() - indexID := "integrationtest_tombstonecleanup" - - logger, _ := test.NewNullLogger() - makeCL := func() (CommitLogger, error) { - return NewCommitLogger(dirName, indexID, logger, - cyclemanager.NewCallbackGroupNoop()) - } - index, err := New(Config{ - RootPath: dirName, - ID: indexID, - MakeCommitLoggerThunk: makeCL, - DistanceProvider: distancer.NewCosineDistanceProvider(), - VectorForIDThunk: testVectorForID, - }, ent.UserConfig{ - MaxConnections: 30, - EFConstruction: 60, - }, cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - cyclemanager.NewCallbackGroupNoop(), testinghelpers.NewDummyStore(t)) - require.Nil(t, err) - - for i, vec := range testVectors { - err := index.Add(uint64(i), vec) - require.Nil(t, err) - } - dumpIndex(index, "with cleanup after import") - require.Nil(t, index.Flush()) - - t.Run("delete some elements and permanently delete tombstoned elements", - func(t *testing.T) { - err := index.Delete(6) - require.Nil(t, err) - err = index.Delete(8) - require.Nil(t, err) - - err = index.CleanUpTombstonedNodes(neverStop) - require.Nil(t, err) - }) - - dumpIndex(index, "with cleanup after delete") - - require.Nil(t, index.Flush()) - - // see index_test.go for more context - expectedResults := []uint64{ - 3, 5, 4, // cluster 2 - 7, // cluster 3 with element 6 and 8 deleted - 2, 1, 0, // cluster 1 - } - - t.Run("verify that the results match originally", func(t *testing.T) { - position := 3 - res, _, err := index.knnSearchByVector(testVectors[position], 50, 36, nil) - require.Nil(t, err) - assert.Equal(t, expectedResults, res) - }) - - // destroy the index - index.Shutdown(context.Background()) - index = nil - - // build a new index from the (uncondensed) commit log - secondIndex, err := New(Config{ - RootPath: dirName, - ID: indexID, - MakeCommitLoggerThunk: makeCL, - DistanceProvider: distancer.NewCosineDistanceProvider(), - VectorForIDThunk: testVectorForID, - }, ent.UserConfig{ - MaxConnections: 30, - EFConstruction: 60, - }, cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - cyclemanager.NewCallbackGroupNoop(), testinghelpers.NewDummyStore(t)) - require.Nil(t, err) - dumpIndex(secondIndex, "with cleanup second index") - - t.Run("verify that the results match after rebuilding from disk", - func(t *testing.T) { - position := 3 - res, _, err := secondIndex.knnSearchByVector(testVectors[position], 50, 36, nil) - require.Nil(t, err) - assert.Equal(t, expectedResults, res) - }) - - t.Run("further deleting all elements and reimporting one", func(t *testing.T) { - toDelete := []uint64{0, 1, 2, 3, 4, 5, 7} - - for _, id := range toDelete { - err := secondIndex.Delete(id) - require.Nil(t, err) - } - - err = secondIndex.CleanUpTombstonedNodes(neverStop) - require.Nil(t, err) - - err := secondIndex.Add(3, testVectors[3]) - require.Nil(t, err) - }) - - require.Nil(t, secondIndex.Flush()) - - dumpIndex(secondIndex) - - secondIndex.Shutdown(context.Background()) - secondIndex = nil - - // build a new index from the (uncondensed) commit log - thirdIndex, err := New(Config{ - RootPath: dirName, - ID: indexID, - MakeCommitLoggerThunk: makeCL, - DistanceProvider: distancer.NewCosineDistanceProvider(), - VectorForIDThunk: testVectorForID, - }, ent.UserConfig{ - MaxConnections: 30, - EFConstruction: 60, - }, cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - cyclemanager.NewCallbackGroupNoop(), testinghelpers.NewDummyStore(t)) - require.Nil(t, err) - - dumpIndex(thirdIndex) - - t.Run("verify that the results match after rebuilding from disk", - func(t *testing.T) { - position := 3 - res, _, err := thirdIndex.knnSearchByVector(testVectors[position], 50, 36, nil) - require.Nil(t, err) - assert.Equal(t, []uint64{3}, res) - }) - - t.Run("delete all elements so the commitlog ends with an empty graph", func(t *testing.T) { - toDelete := []uint64{3} - - for _, id := range toDelete { - err := thirdIndex.Delete(id) - require.Nil(t, err) - } - - err = thirdIndex.CleanUpTombstonedNodes(neverStop) - require.Nil(t, err) - }) - - require.Nil(t, thirdIndex.Flush()) - - thirdIndex.Shutdown(context.Background()) - thirdIndex = nil - // build a new index from the (uncondensed) commit log - fourthIndex, err := New(Config{ - RootPath: dirName, - ID: indexID, - MakeCommitLoggerThunk: makeCL, - DistanceProvider: distancer.NewCosineDistanceProvider(), - VectorForIDThunk: testVectorForID, - }, ent.UserConfig{ - MaxConnections: 30, - EFConstruction: 60, - }, cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - cyclemanager.NewCallbackGroupNoop(), testinghelpers.NewDummyStore(t)) - require.Nil(t, err) - - t.Run("load from disk and try to insert again", func(t *testing.T) { - for i, vec := range testVectors { - err := fourthIndex.Add(uint64(i), vec) - require.Nil(t, err) - } - }) - - t.Run("verify that searching works normally", func(t *testing.T) { - expectedResults := []uint64{ - 3, 5, 4, // cluster 2 - 7, 8, 6, // cluster 3 with element 6 and 8 deleted - 2, 1, 0, // cluster 1 - } - position := 3 - res, _, err := fourthIndex.knnSearchByVector(testVectors[position], 50, 36, nil) - require.Nil(t, err) - assert.Equal(t, expectedResults, res) - }) - - fourthIndex.Shutdown(context.Background()) -} diff --git a/adapters/repos/db/vector/hnsw/pools.go b/adapters/repos/db/vector/hnsw/pools.go deleted file mode 100644 index d24354040a07dfc95aed3f5212582495b8fd0918..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/pools.go +++ /dev/null @@ -1,107 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "sync" - - "github.com/weaviate/weaviate/adapters/repos/db/priorityqueue" - "github.com/weaviate/weaviate/adapters/repos/db/vector/cache" - "github.com/weaviate/weaviate/adapters/repos/db/vector/common" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/visited" -) - -type pools struct { - visitedLists *visited.Pool - visitedListsLock *sync.Mutex - - pqItemSlice *sync.Pool - pqHeuristic *pqMinWithIndexPool - pqResults *common.PqMaxPool - pqCandidates *pqMinPool - - tempVectors *common.TempVectorsPool -} - -func newPools(maxConnectionsLayerZero int) *pools { - return &pools{ - visitedLists: visited.NewPool(1, cache.InitialSize+500), - visitedListsLock: &sync.Mutex{}, - pqItemSlice: &sync.Pool{ - New: func() interface{} { - return make([]priorityqueue.Item[uint64], 0, maxConnectionsLayerZero) - }, - }, - pqHeuristic: newPqMinWithIndexPool(maxConnectionsLayerZero), - pqResults: common.NewPqMaxPool(maxConnectionsLayerZero), - pqCandidates: newPqMinPool(maxConnectionsLayerZero), - tempVectors: common.NewTempVectorsPool(), - } -} - -type pqMinPool struct { - pool *sync.Pool -} - -func newPqMinPool(defaultCap int) *pqMinPool { - return &pqMinPool{ - pool: &sync.Pool{ - New: func() interface{} { - return priorityqueue.NewMin[any](defaultCap) - }, - }, - } -} - -func (pqh *pqMinPool) GetMin(capacity int) *priorityqueue.Queue[any] { - pq := pqh.pool.Get().(*priorityqueue.Queue[any]) - if pq.Cap() < capacity { - pq.ResetCap(capacity) - } else { - pq.Reset() - } - - return pq -} - -func (pqh *pqMinPool) Put(pq *priorityqueue.Queue[any]) { - pqh.pool.Put(pq) -} - -type pqMinWithIndexPool struct { - pool *sync.Pool -} - -func newPqMinWithIndexPool(defaultCap int) *pqMinWithIndexPool { - return &pqMinWithIndexPool{ - pool: &sync.Pool{ - New: func() interface{} { - return priorityqueue.NewMin[uint64](defaultCap) - }, - }, - } -} - -func (pqh *pqMinWithIndexPool) GetMin(capacity int) *priorityqueue.Queue[uint64] { - pq := pqh.pool.Get().(*priorityqueue.Queue[uint64]) - if pq.Cap() < capacity { - pq.ResetCap(capacity) - } else { - pq.Reset() - } - - return pq -} - -func (pqh *pqMinWithIndexPool) Put(pq *priorityqueue.Queue[uint64]) { - pqh.pool.Put(pq) -} diff --git a/adapters/repos/db/vector/hnsw/recall_geo_spatial_test.go b/adapters/repos/db/vector/hnsw/recall_geo_spatial_test.go deleted file mode 100644 index 189472985311cc87e6ba278763be49919da3d2b5..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/recall_geo_spatial_test.go +++ /dev/null @@ -1,276 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTestSlow && !race -// +build integrationTestSlow,!race - -package hnsw - -import ( - "context" - "fmt" - "math/rand" - "runtime" - "sort" - "sync" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" - "github.com/weaviate/weaviate/adapters/repos/db/vector/testinghelpers" - "github.com/weaviate/weaviate/entities/cyclemanager" - ent "github.com/weaviate/weaviate/entities/vectorindex/hnsw" -) - -func TestRecallGeo(t *testing.T) { - size := 10000 - queries := 100 - efConstruction := 128 - maxNeighbors := 64 - - vectors := make([][]float32, size) - queryVectors := make([][]float32, queries) - var vectorIndex *hnsw - - t.Run("generate random vectors", func(t *testing.T) { - fmt.Printf("generating %d vectors", size) - for i := 0; i < size; i++ { - lat, lon := randLatLon() - vectors[i] = []float32{lat, lon} - } - fmt.Printf("done\n") - - fmt.Printf("generating %d search queries", queries) - for i := 0; i < queries; i++ { - lat, lon := randLatLon() - queryVectors[i] = []float32{lat, lon} - } - fmt.Printf("done\n") - }) - - t.Run("importing into hnsw", func(t *testing.T) { - fmt.Printf("importing into hnsw\n") - index, err := New(Config{ - RootPath: "doesnt-matter-as-committlogger-is-mocked-out", - ID: "recallbenchmark", - MakeCommitLoggerThunk: MakeNoopCommitLogger, - DistanceProvider: distancer.NewGeoProvider(), - VectorForIDThunk: func(ctx context.Context, id uint64) ([]float32, error) { - return vectors[int(id)], nil - }, - }, ent.UserConfig{ - MaxConnections: maxNeighbors, - EFConstruction: efConstruction, - }, cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - cyclemanager.NewCallbackGroupNoop(), testinghelpers.NewDummyStore(t)) - - require.Nil(t, err) - vectorIndex = index - - workerCount := runtime.GOMAXPROCS(0) - jobsForWorker := make([][][]float32, workerCount) - - for i, vec := range vectors { - workerID := i % workerCount - jobsForWorker[workerID] = append(jobsForWorker[workerID], vec) - } - - beforeImport := time.Now() - wg := &sync.WaitGroup{} - for workerID, jobs := range jobsForWorker { - wg.Add(1) - go func(workerID int, myJobs [][]float32) { - defer wg.Done() - for i, vec := range myJobs { - originalIndex := (i * workerCount) + workerID - err := vectorIndex.Add(uint64(originalIndex), vec) - require.Nil(t, err) - } - }(workerID, jobs) - } - - wg.Wait() - fmt.Printf("import took %s\n", time.Since(beforeImport)) - }) - - t.Run("with k=10", func(t *testing.T) { - k := 10 - - var relevant int - var retrieved int - - var times time.Duration - - for i := 0; i < queries; i++ { - controlList := bruteForce(vectors, queryVectors[i], k) - before := time.Now() - results, _, err := vectorIndex.knnSearchByVector(queryVectors[i], k, 800, nil) - times += time.Since(before) - - require.Nil(t, err) - - retrieved += k - relevant += matchesInLists(controlList, results) - } - - recall := float32(relevant) / float32(retrieved) - fmt.Printf("recall is %f\n", recall) - fmt.Printf("avg search time for k=%d is %s\n", k, times/time.Duration(queries)) - assert.True(t, recall >= 0.99) - }) - - t.Run("with max dist set", func(t *testing.T) { - distances := []float32{ - 0.1, - 1, - 10, - 100, - 1000, - 2000, - 5000, - 7500, - 10000, - 12500, - 15000, - 20000, - 35000, - 100000, // larger than the circumference of the earth, should contain all - } - - for _, maxDist := range distances { - t.Run(fmt.Sprintf("with maxDist=%f", maxDist), func(t *testing.T) { - var relevant int - var retrieved int - - var times time.Duration - - for i := 0; i < queries; i++ { - controlList := bruteForceMaxDist(vectors, queryVectors[i], maxDist) - before := time.Now() - results, err := vectorIndex.KnnSearchByVectorMaxDist(queryVectors[i], maxDist, 800, nil) - times += time.Since(before) - require.Nil(t, err) - - retrieved += len(results) - relevant += matchesInLists(controlList, results) - } - - if relevant == 0 { - // skip, as we risk dividing by zero, if both relevant and retrieved - // are zero, however, we want to fail with a divide-by-zero if only - // retrieved is 0 and relevant was more than 0 - return - } - recall := float32(relevant) / float32(retrieved) - fmt.Printf("recall is %f\n", recall) - fmt.Printf("avg search time for maxDist=%f is %s\n", maxDist, times/time.Duration(queries)) - assert.True(t, recall >= 0.99) - }) - } - }) -} - -func matchesInLists(control []uint64, results []uint64) int { - desired := map[uint64]struct{}{} - for _, relevant := range control { - desired[relevant] = struct{}{} - } - - var matches int - for _, candidate := range results { - _, ok := desired[candidate] - if ok { - matches++ - } - } - - return matches -} - -func bruteForce(vectors [][]float32, query []float32, k int) []uint64 { - type distanceAndIndex struct { - distance float32 - index uint64 - } - - distances := make([]distanceAndIndex, len(vectors)) - - distancer := distancer.NewGeoProvider().New(query) - for i, vec := range vectors { - dist, _, _ := distancer.Distance(vec) - distances[i] = distanceAndIndex{ - index: uint64(i), - distance: dist, - } - } - - sort.Slice(distances, func(a, b int) bool { - return distances[a].distance < distances[b].distance - }) - - if len(distances) < k { - k = len(distances) - } - - out := make([]uint64, k) - for i := 0; i < k; i++ { - out[i] = distances[i].index - } - - return out -} - -func bruteForceMaxDist(vectors [][]float32, query []float32, maxDist float32) []uint64 { - type distanceAndIndex struct { - distance float32 - index uint64 - } - - distances := make([]distanceAndIndex, len(vectors)) - - distancer := distancer.NewGeoProvider().New(query) - for i, vec := range vectors { - dist, _, _ := distancer.Distance(vec) - distances[i] = distanceAndIndex{ - index: uint64(i), - distance: dist, - } - } - - sort.Slice(distances, func(a, b int) bool { - return distances[a].distance < distances[b].distance - }) - - out := make([]uint64, len(distances)) - i := 0 - for _, elem := range distances { - if elem.distance > maxDist { - break - } - out[i] = distances[i].index - i++ - } - - return out[:i] -} - -func randLatLon() (float32, float32) { - maxLat := float32(90.0) - minLat := float32(-90.0) - maxLon := float32(180) - minLon := float32(-180) - - lat := minLat + (maxLat-minLat)*rand.Float32() - lon := minLon + (maxLon-minLon)*rand.Float32() - return lat, lon -} diff --git a/adapters/repos/db/vector/hnsw/recall_test.go b/adapters/repos/db/vector/hnsw/recall_test.go deleted file mode 100644 index 90244972b024997ae450282e66e78d07063c7b0e..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/recall_test.go +++ /dev/null @@ -1,170 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build benchmarkRecall -// +build benchmarkRecall - -package hnsw - -import ( - "context" - "encoding/json" - "fmt" - "io/ioutil" - "runtime" - "sync" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" - "github.com/weaviate/weaviate/adapters/repos/db/vector/testinghelpers" -) - -func TestRecall(t *testing.T) { - efConstruction := 256 - ef := 256 - maxNeighbors := 64 - - var vectors [][]float32 - var queries [][]float32 - var truths [][]uint64 - var vectorIndex *hnsw - - t.Run("generate random vectors", func(t *testing.T) { - vectorsJSON, err := ioutil.ReadFile("recall_vectors.json") - require.Nil(t, err) - err = json.Unmarshal(vectorsJSON, &vectors) - require.Nil(t, err) - - queriesJSON, err := ioutil.ReadFile("recall_queries.json") - require.Nil(t, err) - err = json.Unmarshal(queriesJSON, &queries) - require.Nil(t, err) - - truthsJSON, err := ioutil.ReadFile("recall_truths.json") - require.Nil(t, err) - err = json.Unmarshal(truthsJSON, &truths) - require.Nil(t, err) - }) - - t.Run("importing into hnsw", func(t *testing.T) { - fmt.Printf("importing into hnsw\n") - - index, err := New(Config{ - RootPath: "doesnt-matter-as-committlogger-is-mocked-out", - ID: "recallbenchmark", - MakeCommitLoggerThunk: MakeNoopCommitLogger, - DistanceProvider: distancer.NewCosineDistanceProvider(), - VectorForIDThunk: func(ctx context.Context, id uint64) ([]float32, error) { - return vectors[int(id)], nil - }, - }, UserConfig{ - MaxConnections: maxNeighbors, - EFConstruction: efConstruction, - EF: ef, - }, testinghelpers.NewDummyStore(t)) - require.Nil(t, err) - vectorIndex = index - - workerCount := runtime.GOMAXPROCS(0) - jobsForWorker := make([][][]float32, workerCount) - - before := time.Now() - for i, vec := range vectors { - workerID := i % workerCount - jobsForWorker[workerID] = append(jobsForWorker[workerID], vec) - } - - wg := &sync.WaitGroup{} - for workerID, jobs := range jobsForWorker { - wg.Add(1) - go func(workerID int, myJobs [][]float32) { - defer wg.Done() - for i, vec := range myJobs { - originalIndex := (i * workerCount) + workerID - err := vectorIndex.Add(uint64(originalIndex), vec) - require.Nil(t, err) - } - }(workerID, jobs) - } - - wg.Wait() - fmt.Printf("importing took %s\n", time.Since(before)) - }) - - t.Run("inspect a query", func(t *testing.T) { - k := 20 - - hasDuplicates := 0 - - for _, vec := range queries { - results, _, err := vectorIndex.SearchByVector(vec, k, nil) - require.Nil(t, err) - if containsDuplicates(results) { - hasDuplicates++ - panic("stop") - } - } - - fmt.Printf("%d out of %d searches contained duplicates", hasDuplicates, len(queries)) - }) - - t.Run("with k=10", func(t *testing.T) { - k := 10 - - var relevant int - var retrieved int - - for i := 0; i < len(queries); i++ { - results, _, err := vectorIndex.SearchByVector(queries[i], k, nil) - require.Nil(t, err) - - retrieved += k - relevant += matchesInLists(truths[i], results) - } - - recall := float32(relevant) / float32(retrieved) - fmt.Printf("recall is %f\n", recall) - assert.True(t, recall >= 0.99) - }) -} - -func matchesInLists(control []uint64, results []uint64) int { - desired := map[uint64]struct{}{} - for _, relevant := range control { - desired[relevant] = struct{}{} - } - - var matches int - for _, candidate := range results { - _, ok := desired[candidate] - if ok { - matches++ - } - } - - return matches -} - -func containsDuplicates(in []uint64) bool { - seen := map[uint64]struct{}{} - - for _, value := range in { - if _, ok := seen[value]; ok { - return true - } - seen[value] = struct{}{} - } - - return false -} diff --git a/adapters/repos/db/vector/hnsw/restore_test.go b/adapters/repos/db/vector/hnsw/restore_test.go deleted file mode 100644 index c18c66f35f622c8056a55c6e813dabcbd930d2ef..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/restore_test.go +++ /dev/null @@ -1,76 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "context" - "io" - "os" - "path" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/adapters/repos/db/vector/common" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" - "github.com/weaviate/weaviate/adapters/repos/db/vector/testinghelpers" - "github.com/weaviate/weaviate/entities/cyclemanager" - ent "github.com/weaviate/weaviate/entities/vectorindex/hnsw" -) - -func Test_RestartFromZeroSegments(t *testing.T) { - testPath := t.TempDir() - src := path.Join(".", "compression_tests", "fixtures", "restart-from-zero-segments", "1234567") - source, err := os.Open(src) - assert.Nil(t, err) - dstPath := path.Join(testPath, "main.hnsw.commitlog.d") - assert.Nil(t, os.Mkdir(dstPath, 0o777)) - destination, err := os.Create(path.Join(dstPath, "1234567")) - assert.Nil(t, err) - _, err = io.Copy(destination, source) - assert.Nil(t, err) - source.Close() - destination.Close() - - efConstruction := 64 - ef := 32 - maxNeighbors := 32 - dimensions := 20 - vectors_size := 10000 - queries_size := 1 - vectors, _ := testinghelpers.RandomVecs(vectors_size, queries_size, dimensions) - distancer := distancer.NewL2SquaredProvider() - uc := ent.UserConfig{} - uc.MaxConnections = maxNeighbors - uc.EFConstruction = efConstruction - uc.EF = ef - uc.VectorCacheMaxObjects = 10e12 - uc.PQ = ent.PQConfig{Enabled: true, Encoder: ent.PQEncoder{Type: ent.PQEncoderTypeKMeans, Distribution: ent.PQEncoderDistributionNormal}} - config := Config{ - RootPath: testPath, - ID: "main", - MakeCommitLoggerThunk: MakeNoopCommitLogger, - DistanceProvider: distancer, - VectorForIDThunk: func(ctx context.Context, id uint64) ([]float32, error) { - return vectors[int(id)], nil - }, - TempVectorForIDThunk: func(ctx context.Context, id uint64, container *common.VectorSlice) ([]float32, error) { - copy(container.Slice, vectors[int(id)]) - return container.Slice, nil - }, - } - - _, err = New( - config, uc, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), testinghelpers.NewDummyStore(t)) - - assert.Nil(t, err) -} diff --git a/adapters/repos/db/vector/hnsw/search.go b/adapters/repos/db/vector/hnsw/search.go deleted file mode 100644 index d2e22144329499676affc4cea35ce1345d717a76..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/search.go +++ /dev/null @@ -1,670 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "context" - "fmt" - "math" - "sync/atomic" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/adapters/repos/db/priorityqueue" - "github.com/weaviate/weaviate/adapters/repos/db/vector/compressionhelpers" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/visited" - "github.com/weaviate/weaviate/entities/storobj" - "github.com/weaviate/weaviate/usecases/floatcomp" -) - -func (h *hnsw) searchTimeEF(k int) int { - // load atomically, so we can get away with concurrent updates of the - // userconfig without having to set a lock each time we try to read - which - // can be so common that it would cause considerable overhead - ef := int(atomic.LoadInt64(&h.ef)) - if ef < 1 { - return h.autoEfFromK(k) - } - - if ef < k { - ef = k - } - - return ef -} - -func (h *hnsw) autoEfFromK(k int) int { - factor := int(atomic.LoadInt64(&h.efFactor)) - min := int(atomic.LoadInt64(&h.efMin)) - max := int(atomic.LoadInt64(&h.efMax)) - - ef := k * factor - if ef > max { - ef = max - } else if ef < min { - ef = min - } - if k > ef { - ef = k // otherwise results will get cut off early - } - - return ef -} - -func (h *hnsw) SearchByVector(vector []float32, k int, allowList helpers.AllowList) ([]uint64, []float32, error) { - h.compressActionLock.RLock() - defer h.compressActionLock.RUnlock() - - vector = h.normalizeVec(vector) - flatSearchCutoff := int(atomic.LoadInt64(&h.flatSearchCutoff)) - if allowList != nil && !h.forbidFlat && allowList.Len() < flatSearchCutoff { - return h.flatSearch(vector, k, allowList) - } - return h.knnSearchByVector(vector, k, h.searchTimeEF(k), allowList) -} - -// SearchByVectorDistance wraps SearchByVector, and calls it recursively until -// the search results contain all vector within the threshold specified by the -// target distance. -// -// The maxLimit param will place an upper bound on the number of search results -// returned. This is used in situations where the results of the method are all -// eventually turned into objects, for example, a Get query. If the caller just -// needs ids for sake of something like aggregation, a maxLimit of -1 can be -// passed in to truly obtain all results from the vector index. -func (h *hnsw) SearchByVectorDistance(vector []float32, targetDistance float32, maxLimit int64, - allowList helpers.AllowList, -) ([]uint64, []float32, error) { - var ( - searchParams = newSearchByDistParams(maxLimit) - - resultIDs []uint64 - resultDist []float32 - ) - - recursiveSearch := func() (bool, error) { - shouldContinue := false - - ids, dist, err := h.SearchByVector(vector, searchParams.totalLimit, allowList) - if err != nil { - return false, errors.Wrap(err, "vector search") - } - - // ensures the indexers aren't out of range - offsetCap := searchParams.offsetCapacity(ids) - totalLimitCap := searchParams.totalLimitCapacity(ids) - - ids, dist = ids[offsetCap:totalLimitCap], dist[offsetCap:totalLimitCap] - - if len(ids) == 0 { - return false, nil - } - - lastFound := dist[len(dist)-1] - shouldContinue = lastFound <= targetDistance - - for i := range ids { - if aboveThresh := dist[i] <= targetDistance; aboveThresh || - floatcomp.InDelta(float64(dist[i]), float64(targetDistance), 1e-6) { - resultIDs = append(resultIDs, ids[i]) - resultDist = append(resultDist, dist[i]) - } else { - // as soon as we encounter a certainty which - // is below threshold, we can stop searching - break - } - } - - return shouldContinue, nil - } - - shouldContinue, err := recursiveSearch() - if err != nil { - return nil, nil, err - } - - for shouldContinue { - searchParams.iterate() - if searchParams.maxLimitReached() { - h.logger. - WithField("action", "unlimited_vector_search"). - Warnf("maximum search limit of %d results has been reached", - searchParams.maximumSearchLimit) - break - } - - shouldContinue, err = recursiveSearch() - if err != nil { - return nil, nil, err - } - } - - return resultIDs, resultDist, nil -} - -func (h *hnsw) shouldRescore() bool { - return h.compressed.Load() && !h.doNotRescore -} - -func (h *hnsw) searchLayerByVector(queryVector []float32, - entrypoints *priorityqueue.Queue[any], ef int, level int, - allowList helpers.AllowList, -) (*priorityqueue.Queue[any], error, -) { - var compressorDistancer compressionhelpers.CompressorDistancer - if h.compressed.Load() { - var returnFn compressionhelpers.ReturnDistancerFn - compressorDistancer, returnFn = h.compressor.NewDistancer(queryVector) - defer returnFn() - } - return h.searchLayerByVectorWithDistancer(queryVector, entrypoints, ef, level, allowList, compressorDistancer) -} - -func (h *hnsw) searchLayerByVectorWithDistancer(queryVector []float32, - entrypoints *priorityqueue.Queue[any], ef int, level int, - allowList helpers.AllowList, compressorDistancer compressionhelpers.CompressorDistancer) (*priorityqueue.Queue[any], error, -) { - h.pools.visitedListsLock.Lock() - visited := h.pools.visitedLists.Borrow() - h.pools.visitedListsLock.Unlock() - - candidates := h.pools.pqCandidates.GetMin(ef) - results := h.pools.pqResults.GetMax(ef) - var floatDistancer distancer.Distancer - if h.compressed.Load() { - if compressorDistancer == nil { - var returnFn compressionhelpers.ReturnDistancerFn - compressorDistancer, returnFn = h.compressor.NewDistancer(queryVector) - defer returnFn() - } - } else { - floatDistancer = h.distancerProvider.New(queryVector) - } - - h.insertViableEntrypointsAsCandidatesAndResults(entrypoints, candidates, - results, level, visited, allowList) - - var worstResultDistance float32 - var err error - if h.compressed.Load() { - worstResultDistance, err = h.currentWorstResultDistanceToByte(results, compressorDistancer) - } else { - worstResultDistance, err = h.currentWorstResultDistanceToFloat(results, floatDistancer) - } - if err != nil { - return nil, errors.Wrapf(err, "calculate distance of current last result") - } - connectionsReusable := make([]uint64, h.maximumConnectionsLayerZero) - - for candidates.Len() > 0 { - var dist float32 - candidate := candidates.Pop() - dist = candidate.Dist - - if dist > worstResultDistance && results.Len() >= ef { - break - } - - h.shardedNodeLocks.RLock(candidate.ID) - candidateNode := h.nodes[candidate.ID] - h.shardedNodeLocks.RUnlock(candidate.ID) - - if candidateNode == nil { - // could have been a node that already had a tombstone attached and was - // just cleaned up while we were waiting for a read lock - continue - } - - candidateNode.Lock() - if candidateNode.level < level { - // a node level could have been downgraded as part of a delete-reassign, - // but the connections pointing to it not yet cleaned up. In this case - // the node doesn't have any outgoing connections at this level and we - // must discard it. - candidateNode.Unlock() - continue - } - - if len(candidateNode.connections[level]) > h.maximumConnectionsLayerZero { - // How is it possible that we could ever have more connections than the - // allowed maximum? It is not anymore, but there was a bug that allowed - // this to happen in versions prior to v1.12.0: - // https://github.com/weaviate/weaviate/issues/1868 - // - // As a result the length of this slice is entirely unpredictable and we - // can no longer retrieve it from the pool. Instead we need to fallback - // to allocating a new slice. - // - // This was discovered as part of - // https://github.com/weaviate/weaviate/issues/1897 - connectionsReusable = make([]uint64, len(candidateNode.connections[level])) - } else { - connectionsReusable = connectionsReusable[:len(candidateNode.connections[level])] - } - - copy(connectionsReusable, candidateNode.connections[level]) - candidateNode.Unlock() - - for _, neighborID := range connectionsReusable { - - if ok := visited.Visited(neighborID); ok { - // skip if we've already visited this neighbor - continue - } - - // make sure we never visit this neighbor again - visited.Visit(neighborID) - var distance float32 - var ok bool - var err error - if h.compressed.Load() { - distance, ok, err = compressorDistancer.DistanceToNode(neighborID) - } else { - distance, ok, err = h.distanceToFloatNode(floatDistancer, neighborID) - } - if err != nil { - var e storobj.ErrNotFound - if errors.As(err, &e) { - h.handleDeletedNode(e.DocID) - } else { - if err != nil { - return nil, errors.Wrap(err, "calculate distance between candidate and query") - } - } - } - - if !ok { - // node was deleted in the underlying object store - continue - } - - if distance < worstResultDistance || results.Len() < ef { - candidates.Insert(neighborID, distance) - if level == 0 && allowList != nil { - // we are on the lowest level containing the actual candidates and we - // have an allow list (i.e. the user has probably set some sort of a - // filter restricting this search further. As a result we have to - // ignore items not on the list - if !allowList.Contains(neighborID) { - continue - } - } - - if h.hasTombstone(neighborID) { - continue - } - - results.Insert(neighborID, distance) - - if h.compressed.Load() { - h.compressor.Prefetch(candidates.Top().ID) - } else { - h.cache.Prefetch(candidates.Top().ID) - } - - // +1 because we have added one node size calculating the len - if results.Len() > ef { - results.Pop() - } - - if results.Len() > 0 { - worstResultDistance = results.Top().Dist - } - } - } - } - - h.pools.pqCandidates.Put(candidates) - - h.pools.visitedListsLock.Lock() - h.pools.visitedLists.Return(visited) - h.pools.visitedListsLock.Unlock() - - return results, nil -} - -func (h *hnsw) insertViableEntrypointsAsCandidatesAndResults( - entrypoints, candidates, results *priorityqueue.Queue[any], level int, - visitedList visited.ListSet, allowList helpers.AllowList, -) { - for entrypoints.Len() > 0 { - ep := entrypoints.Pop() - visitedList.Visit(ep.ID) - candidates.Insert(ep.ID, ep.Dist) - if level == 0 && allowList != nil { - // we are on the lowest level containing the actual candidates and we - // have an allow list (i.e. the user has probably set some sort of a - // filter restricting this search further. As a result we have to - // ignore items not on the list - if !allowList.Contains(ep.ID) { - continue - } - } - - if h.hasTombstone(ep.ID) { - continue - } - - results.Insert(ep.ID, ep.Dist) - } -} - -func (h *hnsw) currentWorstResultDistanceToFloat(results *priorityqueue.Queue[any], - distancer distancer.Distancer, -) (float32, error) { - if results.Len() > 0 { - id := results.Top().ID - - d, ok, err := h.distanceToFloatNode(distancer, id) - if err != nil { - var e storobj.ErrNotFound - if errors.As(err, &e) { - h.handleDeletedNode(e.DocID) - } else { - if err != nil { - return 0, errors.Wrap(err, "calculated distance between worst result and query") - } - } - } - - if !ok { - return math.MaxFloat32, nil - } - return d, nil - } else { - // if the entrypoint (which we received from a higher layer doesn't match - // the allow List the result list is empty. In this case we can just set - // the worstDistance to an arbitrarily large number, so that any - // (allowed) candidate will have a lower distance in comparison - return math.MaxFloat32, nil - } -} - -func (h *hnsw) currentWorstResultDistanceToByte(results *priorityqueue.Queue[any], - distancer compressionhelpers.CompressorDistancer, -) (float32, error) { - if results.Len() > 0 { - item := results.Top() - if item.Dist != 0 { - return item.Dist, nil - } - id := item.ID - d, ok, err := distancer.DistanceToNode(id) - if err != nil { - return 0, errors.Wrap(err, - "calculated distance between worst result and query") - } - - if !ok { - return math.MaxFloat32, nil - } - return d, nil - } else { - // if the entrypoint (which we received from a higher layer doesn't match - // the allow List the result list is empty. In this case we can just set - // the worstDistance to an arbitrarily large number, so that any - // (allowed) candidate will have a lower distance in comparison - return math.MaxFloat32, nil - } -} - -func (h *hnsw) distanceFromBytesToFloatNode(concreteDistancer compressionhelpers.CompressorDistancer, nodeID uint64) (float32, bool, error) { - slice := h.pools.tempVectors.Get(int(h.dims)) - defer h.pools.tempVectors.Put(slice) - vec, err := h.TempVectorForIDThunk(context.Background(), nodeID, slice) - if err != nil { - var e storobj.ErrNotFound - if errors.As(err, &e) { - h.handleDeletedNode(e.DocID) - return 0, false, nil - } else { - // not a typed error, we can recover from, return with err - return 0, false, errors.Wrapf(err, "get vector of docID %d", nodeID) - } - } - vec = h.normalizeVec(vec) - return concreteDistancer.DistanceToFloat(vec) -} - -func (h *hnsw) distanceToFloatNode(distancer distancer.Distancer, - nodeID uint64, -) (float32, bool, error) { - candidateVec, err := h.vectorForID(context.Background(), nodeID) - if err != nil { - var e storobj.ErrNotFound - if errors.As(err, &e) { - h.handleDeletedNode(e.DocID) - return 0, false, nil - } else { - // not a typed error, we can recover from, return with err - return 0, false, errors.Wrapf(err, "get vector of docID %d", nodeID) - } - } - - dist, _, err := distancer.Distance(candidateVec) - if err != nil { - return 0, false, errors.Wrap(err, "calculate distance between candidate and query") - } - - return dist, true, nil -} - -// the underlying object seems to have been deleted, to recover from -// this situation let's add a tombstone to the deleted object, so it -// will be cleaned up and skip this candidate in the current search -func (h *hnsw) handleDeletedNode(docID uint64) { - if h.hasTombstone(docID) { - // nothing to do, this node already has a tombstone, it will be cleaned up - // in the next deletion cycle - return - } - - h.addTombstone(docID) - h.logger.WithField("action", "attach_tombstone_to_deleted_node"). - WithField("node_id", docID). - Infof("found a deleted node (%d) without a tombstone, "+ - "tombstone was added", docID) -} - -func (h *hnsw) knnSearchByVector(searchVec []float32, k int, - ef int, allowList helpers.AllowList, -) ([]uint64, []float32, error) { - if h.isEmpty() { - return nil, nil, nil - } - - if k < 0 { - return nil, nil, fmt.Errorf("k must be greater than zero") - } - - h.RLock() - entryPointID := h.entryPointID - maxLayer := h.currentMaximumLayer - h.RUnlock() - - entryPointDistance, ok, err := h.distBetweenNodeAndVec(entryPointID, searchVec) - if err != nil { - return nil, nil, errors.Wrap(err, "knn search: distance between entrypoint and query node") - } - - if !ok { - return nil, nil, fmt.Errorf("entrypoint was deleted in the object store, " + - "it has been flagged for cleanup and should be fixed in the next cleanup cycle") - } - - var compressorDistancer compressionhelpers.CompressorDistancer - if h.compressed.Load() { - var returnFn compressionhelpers.ReturnDistancerFn - compressorDistancer, returnFn = h.compressor.NewDistancer(searchVec) - defer returnFn() - } - // stop at layer 1, not 0! - for level := maxLayer; level >= 1; level-- { - eps := priorityqueue.NewMin[any](10) - eps.Insert(entryPointID, entryPointDistance) - - res, err := h.searchLayerByVectorWithDistancer(searchVec, eps, 1, level, nil, compressorDistancer) - if err != nil { - return nil, nil, errors.Wrapf(err, "knn search: search layer at level %d", level) - } - - // There might be situations where we did not find a better entrypoint at - // that particular level, so instead we're keeping whatever entrypoint we - // had before (i.e. either from a previous level or even the main - // entrypoint) - // - // If we do, however, have results, any candidate that's not nil (not - // deleted), and not under maintenance is a viable candidate - for res.Len() > 0 { - cand := res.Pop() - n := h.nodeByID(cand.ID) - if n == nil { - // we have found a node in results that is nil. This means it was - // deleted, but not cleaned up properly. Make sure to add a tombstone to - // this node, so it can be cleaned up in the next cycle. - if err := h.addTombstone(cand.ID); err != nil { - return nil, nil, err - } - - // skip the nil node, as it does not make a valid entrypoint - continue - } - - if !n.isUnderMaintenance() { - entryPointID = cand.ID - entryPointDistance = cand.Dist - break - } - - // if we managed to go through the loop without finding a single - // suitable node, we simply stick with the original, i.e. the global - // entrypoint - } - - h.pools.pqResults.Put(res) - } - - eps := priorityqueue.NewMin[any](10) - eps.Insert(entryPointID, entryPointDistance) - res, err := h.searchLayerByVectorWithDistancer(searchVec, eps, ef, 0, allowList, compressorDistancer) - if err != nil { - return nil, nil, errors.Wrapf(err, "knn search: search layer at level %d", 0) - } - - if h.shouldRescore() { - ids := make([]uint64, res.Len()) - i := len(ids) - 1 - for res.Len() > 0 { - res := res.Pop() - ids[i] = res.ID - i-- - } - res.Reset() - for _, id := range ids { - dist, _, _ := h.distanceFromBytesToFloatNode(compressorDistancer, id) - res.Insert(id, dist) - if res.Len() > ef { - res.Pop() - } - } - - } - - for res.Len() > k { - res.Pop() - } - - ids := make([]uint64, res.Len()) - dists := make([]float32, res.Len()) - - // results is ordered in reverse, we need to flip the order before presenting - // to the user! - i := len(ids) - 1 - for res.Len() > 0 { - res := res.Pop() - ids[i] = res.ID - dists[i] = res.Dist - i-- - } - h.pools.pqResults.Put(res) - return ids, dists, nil -} - -func newSearchByDistParams(maxLimit int64) *searchByDistParams { - initialOffset := 0 - initialLimit := DefaultSearchByDistInitialLimit - - return &searchByDistParams{ - offset: initialOffset, - limit: initialLimit, - totalLimit: initialOffset + initialLimit, - maximumSearchLimit: maxLimit, - } -} - -const ( - // DefaultSearchByDistInitialLimit : - // the initial limit of 100 here is an - // arbitrary decision, and can be tuned - // as needed - DefaultSearchByDistInitialLimit = 100 - - // DefaultSearchByDistLimitMultiplier : - // the decision to increase the limit in - // multiples of 10 here is an arbitrary - // decision, and can be tuned as needed - DefaultSearchByDistLimitMultiplier = 10 -) - -type searchByDistParams struct { - offset int - limit int - totalLimit int - maximumSearchLimit int64 -} - -func (params *searchByDistParams) offsetCapacity(ids []uint64) int { - var offsetCap int - if params.offset < len(ids) { - offsetCap = params.offset - } else { - offsetCap = len(ids) - } - - return offsetCap -} - -func (params *searchByDistParams) totalLimitCapacity(ids []uint64) int { - var totalLimitCap int - if params.totalLimit < len(ids) { - totalLimitCap = params.totalLimit - } else { - totalLimitCap = len(ids) - } - - return totalLimitCap -} - -func (params *searchByDistParams) iterate() { - params.offset = params.totalLimit - params.limit *= DefaultSearchByDistLimitMultiplier - params.totalLimit = params.offset + params.limit -} - -func (params *searchByDistParams) maxLimitReached() bool { - if params.maximumSearchLimit < 0 { - return false - } - - return int64(params.totalLimit) > params.maximumSearchLimit -} diff --git a/adapters/repos/db/vector/hnsw/search_by_dist_test.go b/adapters/repos/db/vector/hnsw/search_by_dist_test.go deleted file mode 100644 index cffd7d6fb0addb2cf34e03e3f3d48894f6aa535e..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/search_by_dist_test.go +++ /dev/null @@ -1,32 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestSearchByDistParams(t *testing.T) { - t.Run("param iteration", func(t *testing.T) { - params := newSearchByDistParams(100) - assert.Equal(t, 0, params.offset) - assert.Equal(t, DefaultSearchByDistInitialLimit, params.limit) - assert.Equal(t, 100, params.totalLimit) - - params.iterate() - assert.Equal(t, 100, params.offset) - assert.Equal(t, 1000, params.limit) - assert.Equal(t, 1100, params.totalLimit) - }) -} diff --git a/adapters/repos/db/vector/hnsw/search_test.go b/adapters/repos/db/vector/hnsw/search_test.go deleted file mode 100644 index 90a8194fd47f03ff8e0fd29d3aaf4efe693ba72a..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/search_test.go +++ /dev/null @@ -1,92 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" - "github.com/weaviate/weaviate/adapters/repos/db/vector/testinghelpers" - "github.com/weaviate/weaviate/entities/cyclemanager" - ent "github.com/weaviate/weaviate/entities/vectorindex/hnsw" -) - -// prevents a regression of -// https://github.com/weaviate/weaviate/issues/2155 -func TestNilCheckOnPartiallyCleanedNode(t *testing.T) { - vectors := [][]float32{ - {100, 100}, // first to import makes this the EP, it is far from any query which means it will be replaced. - {2, 2}, // a good potential entrypoint, but we will corrupt it later on - {1, 1}, // the perfect search result - } - - var vectorIndex *hnsw - - t.Run("import", func(*testing.T) { - index, err := New(Config{ - RootPath: "doesnt-matter-as-committlogger-is-mocked-out", - ID: "bug-2155", - MakeCommitLoggerThunk: MakeNoopCommitLogger, - DistanceProvider: distancer.NewL2SquaredProvider(), - VectorForIDThunk: func(ctx context.Context, id uint64) ([]float32, error) { - return vectors[int(id)], nil - }, - }, ent.UserConfig{ - MaxConnections: 30, - EFConstruction: 128, - - // The actual size does not matter for this test, but if it defaults to - // zero it will constantly think it's full and needs to be deleted - even - // after just being deleted, so make sure to use a positive number here. - VectorCacheMaxObjects: 100000, - }, cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop(), - cyclemanager.NewCallbackGroupNoop(), testinghelpers.NewDummyStore(t)) - require.Nil(t, err) - vectorIndex = index - }) - - t.Run("manually add the nodes", func(t *testing.T) { - vectorIndex.entryPointID = 0 - vectorIndex.currentMaximumLayer = 1 - vectorIndex.nodes = []*vertex{ - { - // must be on a non-zero layer for this bug to occur - level: 1, - connections: [][]uint64{ - {1, 2}, - {1}, - }, - }, - nil, // corrupt node - { - level: 0, - connections: [][]uint64{ - {0, 1, 2}, - }, - }, - } - }) - - t.Run("run a search that would typically find the new ep", func(t *testing.T) { - res, _, err := vectorIndex.SearchByVector([]float32{1.7, 1.7}, 20, nil) - require.Nil(t, err) - assert.Equal(t, []uint64{2, 0}, res, "right results are found") - }) - - t.Run("the corrupt node is now marked deleted", func(t *testing.T) { - _, ok := vectorIndex.tombstones[1] - assert.True(t, ok) - }) -} diff --git a/adapters/repos/db/vector/hnsw/search_with_max_dist.go b/adapters/repos/db/vector/hnsw/search_with_max_dist.go deleted file mode 100644 index 1aa82c6548719706344def87c4de239d7835403b..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/search_with_max_dist.go +++ /dev/null @@ -1,80 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "fmt" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/adapters/repos/db/priorityqueue" -) - -func (h *hnsw) KnnSearchByVectorMaxDist(searchVec []float32, dist float32, - ef int, allowList helpers.AllowList, -) ([]uint64, error) { - entryPointID := h.entryPointID - entryPointDistance, ok, err := h.distBetweenNodeAndVec(entryPointID, searchVec) - if err != nil { - return nil, errors.Wrap(err, "knn search: distance between entrypoint and query node") - } - - if !ok { - return nil, fmt.Errorf("entrypoint was deleted in the object store, " + - "it has been flagged for cleanup and should be fixed in the next cleanup cycle") - } - - // stop at layer 1, not 0! - for level := h.currentMaximumLayer; level >= 1; level-- { - eps := priorityqueue.NewMin[any](1) - eps.Insert(entryPointID, entryPointDistance) - // ignore allowList on layers > 0 - res, err := h.searchLayerByVector(searchVec, eps, 1, level, nil) - if err != nil { - return nil, errors.Wrapf(err, "knn search: search layer at level %d", level) - } - if res.Len() > 0 { - best := res.Pop() - entryPointID = best.ID - entryPointDistance = best.Dist - } - - h.pools.pqResults.Put(res) - } - - eps := priorityqueue.NewMin[any](1) - eps.Insert(entryPointID, entryPointDistance) - res, err := h.searchLayerByVector(searchVec, eps, ef, 0, allowList) - if err != nil { - return nil, errors.Wrapf(err, "knn search: search layer at level %d", 0) - } - - all := make([]priorityqueue.Item[any], res.Len()) - i := res.Len() - 1 - for res.Len() > 0 { - all[i] = res.Pop() - i-- - } - - out := make([]uint64, len(all)) - i = 0 - for _, elem := range all { - if elem.Dist > dist { - break - } - out[i] = elem.ID - i++ - } - - h.pools.pqResults.Put(res) - return out[:i], nil -} diff --git a/adapters/repos/db/vector/hnsw/startup.go b/adapters/repos/db/vector/hnsw/startup.go deleted file mode 100644 index 3d7c153fd72d7024eff0d83f55e7d745970c5447..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/startup.go +++ /dev/null @@ -1,216 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "bufio" - "context" - "io" - "os" - "time" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/adapters/repos/db/vector/compressionhelpers" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/visited" - "github.com/weaviate/weaviate/entities/cyclemanager" - "github.com/weaviate/weaviate/entities/diskio" -) - -func (h *hnsw) init(cfg Config) error { - h.pools = newPools(h.maximumConnectionsLayerZero) - - if err := h.restoreFromDisk(); err != nil { - return errors.Wrapf(err, "restore hnsw index %q", cfg.ID) - } - - // init commit logger for future writes - cl, err := cfg.MakeCommitLoggerThunk() - if err != nil { - return errors.Wrap(err, "create commit logger") - } - - h.commitLog = cl - - // report the vector_index_size at server startup. - // otherwise on server restart, prometheus reports - // a vector_index_size of 0 until more vectors are - // added. - h.metrics.SetSize(len(h.nodes)) - - return nil -} - -// if a commit log is already present it will be read into memory, if not we -// start with an empty model -func (h *hnsw) restoreFromDisk() error { - beforeAll := time.Now() - defer h.metrics.TrackStartupTotal(beforeAll) - - fileNames, err := getCommitFileNames(h.rootPath, h.id) - if err != nil { - return err - } - - if len(fileNames) == 0 { - // nothing to do - return nil - } - - fileNames, err = NewCorruptedCommitLogFixer(h.logger).Do(fileNames) - if err != nil { - return errors.Wrap(err, "corrupted commit log fixer") - } - - var state *DeserializationResult - for i, fileName := range fileNames { - beforeIndividual := time.Now() - - fd, err := os.Open(fileName) - if err != nil { - return errors.Wrapf(err, "open commit log %q for reading", fileName) - } - - defer fd.Close() - - metered := diskio.NewMeteredReader(fd, - h.metrics.TrackStartupReadCommitlogDiskIO) - fdBuf := bufio.NewReaderSize(metered, 256*1024) - - var valid int - state, valid, err = NewDeserializer(h.logger).Do(fdBuf, state, false) - if err != nil { - if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { - // we need to check for both EOF or UnexpectedEOF, as we don't know where - // the commit log got corrupted, a field ending that weset a longer - // encoding for would return EOF, whereas a field read with binary.Read - // with a fixed size would return UnexpectedEOF. From our perspective both - // are unexpected. - - h.logger.WithField("action", "hnsw_load_commit_log_corruption"). - WithField("path", fileName). - Error("write-ahead-log ended abruptly, some elements may not have been recovered") - - // we need to truncate the file to its valid length! - if err := os.Truncate(fileName, int64(valid)); err != nil { - return errors.Wrapf(err, "truncate corrupt commit log %q", fileName) - } - } else { - // only return an actual error on non-EOF errors, otherwise we'll end - // up in a startup crashloop - return errors.Wrapf(err, "deserialize commit log %q", fileName) - } - } - - h.metrics.StartupProgress(float64(i+1) / float64(len(fileNames))) - h.metrics.TrackStartupIndividual(beforeIndividual) - } - - h.Lock() - h.shardedNodeLocks.LockAll() - h.nodes = state.Nodes - h.shardedNodeLocks.UnlockAll() - - h.currentMaximumLayer = int(state.Level) - h.entryPointID = state.Entrypoint - h.Unlock() - - h.tombstoneLock.Lock() - h.tombstones = state.Tombstones - h.tombstoneLock.Unlock() - - if state.Compressed { - h.compressed.Store(state.Compressed) - h.dims = int32(state.PQData.Dimensions) - h.cache.Drop() - - if len(state.PQData.Encoders) > 0 { - // 0 means it was created using the default value. The user did not set the value, we calculated for him/her - if h.pqConfig.Segments == 0 { - h.pqConfig.Segments = int(state.PQData.Dimensions) - } - h.compressor, err = compressionhelpers.RestorePQCompressor( - h.pqConfig, - h.distancerProvider, - int(state.PQData.Dimensions), - // ToDo: we need to read this value from somewhere - 1e12, - h.logger, - state.PQData.Encoders, - h.store, - ) - if err != nil { - return errors.Wrap(err, "Restoring compressed data.") - } - } - // make sure the compressed cache fits the current size - h.compressor.GrowCache(uint64(len(h.nodes))) - } else if !h.compressed.Load() { - // make sure the cache fits the current size - h.cache.Grow(uint64(len(h.nodes))) - - if len(h.nodes) > 0 { - if vec, err := h.vectorForID(context.Background(), h.entryPointID); err == nil { - h.dims = int32(len(vec)) - } - } - } - - // make sure the visited list pool fits the current size - h.pools.visitedLists.Destroy() - h.pools.visitedLists = nil - h.pools.visitedLists = visited.NewPool(1, len(h.nodes)+512) - - return nil -} - -func (h *hnsw) tombstoneCleanup(shouldAbort cyclemanager.ShouldAbortCallback) bool { - executed, err := h.cleanUpTombstonedNodes(shouldAbort) - if err != nil { - h.logger.WithField("action", "hnsw_tombstone_cleanup"). - WithError(err).Error("tombstone cleanup errord") - } - return executed -} - -// PostStartup triggers routines that should happen after startup. The startup -// process is triggered during the creation which in turn happens as part of -// the shard creation. Some post-startup routines, such as prefilling the -// vector cache, however, depend on the shard being ready as they will call -// getVectorForID. -func (h *hnsw) PostStartup() { - h.prefillCache() -} - -func (h *hnsw) prefillCache() { - limit := 0 - if h.compressed.Load() { - limit = int(h.compressor.GetCacheMaxSize()) - } else { - limit = int(h.cache.CopyMaxSize()) - } - - go func() { - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute) - defer cancel() - - var err error - if h.compressed.Load() { - h.compressor.PrefillCache() - } else { - err = newVectorCachePrefiller(h.cache, h, h.logger).Prefill(ctx, limit) - } - - if err != nil { - h.logger.WithError(err).Error("prefill vector cache") - } - }() -} diff --git a/adapters/repos/db/vector/hnsw/test_recall_hnswlib.py b/adapters/repos/db/vector/hnsw/test_recall_hnswlib.py deleted file mode 100644 index 5717f2604af73c702685f27b1eb350c41e73e2be..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/test_recall_hnswlib.py +++ /dev/null @@ -1,50 +0,0 @@ -import hnswlib -import numpy as np -import time -import json - -data=None -queries=None -truths=None - -with open("recall_vectors.json", 'r') as f: - data = json.load(f) - -with open("recall_queries.json", 'r') as f: - queries = json.load(f) - -with open("recall_truths.json", 'r') as f: - truths = json.load(f) - -num_elements = len(data) -dim = len(data[0]) -data_labels = np.arange(num_elements) - -# Declaring index -p = hnswlib.Index(space = 'cosine', dim = dim) # possible options are l2, cosine or ip - -# Initializing index - the maximum number of elements should be known beforehand -p.init_index(max_elements = num_elements, ef_construction = 2000, M = 100) - -before = time.time() -# Element insertion (can be called several times): -p.add_items(data, data_labels) -print("import took {}".format(time.time() - before)) - -# Controlling the recall by setting ef: -p.set_ef(100) # ef should always be > k - -# Query dataset, k - number of closest elements (returns 2 numpy arrays) -results, distances = p.knn_query(queries, k = 1) - -relevant=0 -retrieved=0 - -for i, res in enumerate(results): - retrieved+=1 - - # take elem 0 because k==1 - if res[0] == truths[i][0]: - relevant+=1 - -print("Recall: {}".format(relevant/retrieved)) diff --git a/adapters/repos/db/vector/hnsw/vector_cache_prefiller.go b/adapters/repos/db/vector/hnsw/vector_cache_prefiller.go deleted file mode 100644 index 031958c887e4247ab66e43db98764bfb223b487b..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/vector_cache_prefiller.go +++ /dev/null @@ -1,134 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "context" - "time" - - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/adapters/repos/db/vector/cache" -) - -type vectorCachePrefiller[T any] struct { - cache cache.Cache[T] - index *hnsw - logger logrus.FieldLogger -} - -func newVectorCachePrefiller[T any](cache cache.Cache[T], index *hnsw, - logger logrus.FieldLogger, -) *vectorCachePrefiller[T] { - return &vectorCachePrefiller[T]{ - cache: cache, - index: index, - logger: logger, - } -} - -func (pf *vectorCachePrefiller[T]) Prefill(ctx context.Context, limit int) error { - before := time.Now() - for level := pf.maxLevel(); level >= 0; level-- { - ok, err := pf.prefillLevel(ctx, level, limit) - if err != nil { - return err - } - - if !ok { - break - } - } - - pf.logTotal(int(pf.cache.Len()), limit, before) - return nil -} - -// returns false if the max has been reached, true otherwise -func (pf *vectorCachePrefiller[T]) prefillLevel(ctx context.Context, - level, limit int, -) (bool, error) { - // TODO: this makes zero sense, just copy the lists, don't actually block - // !!!! - - before := time.Now() - layerCount := 0 - - pf.index.Lock() - nodesLen := len(pf.index.nodes) - pf.index.Unlock() - - for i := 0; i < nodesLen; i++ { - if int(pf.cache.Len()) >= limit { - break - } - - if err := ctx.Err(); err != nil { - return false, err - } - - pf.index.shardedNodeLocks.RLock(uint64(i)) - node := pf.index.nodes[i] - pf.index.shardedNodeLocks.RUnlock(uint64(i)) - - if node == nil { - continue - } - - if levelOfNode(node) != level { - continue - } - - // we are not really interested in the result, we just want to populate the - // cache - pf.index.Lock() - pf.cache.Get(ctx, uint64(i)) - layerCount++ - pf.index.Unlock() - } - - pf.logLevel(level, layerCount, before) - return true, nil -} - -func (pf *vectorCachePrefiller[T]) logLevel(level, count int, before time.Time) { - pf.logger.WithFields(logrus.Fields{ - "action": "hnsw_vector_cache_prefill_level", - "hnsw_level": level, - "count": count, - "took": time.Since(before), - "index_id": pf.index.id, - }).Debug("prefilled level in vector cache") -} - -func (pf *vectorCachePrefiller[T]) logTotal(count, limit int, before time.Time) { - pf.logger.WithFields(logrus.Fields{ - "action": "hnsw_vector_cache_prefill", - "limit": limit, - "count": count, - "took": time.Since(before), - "index_id": pf.index.id, - }).Info("prefilled vector cache") -} - -func levelOfNode(node *vertex) int { - node.Lock() - defer node.Unlock() - - return node.level -} - -func (pf *vectorCachePrefiller[T]) maxLevel() int { - pf.index.Lock() - defer pf.index.Unlock() - - return pf.index.currentMaximumLayer -} diff --git a/adapters/repos/db/vector/hnsw/vector_cache_prefiller_test.go b/adapters/repos/db/vector/hnsw/vector_cache_prefiller_test.go deleted file mode 100644 index ed65a934bbdb21292e495d1f68e2b5a4750828b9..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/vector_cache_prefiller_test.go +++ /dev/null @@ -1,179 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "context" - "testing" - - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/adapters/repos/db/vector/common" -) - -func TestVectorCachePrefilling(t *testing.T) { - cache := newFakeCache() - index := &hnsw{ - nodes: generateDummyVertices(100), - currentMaximumLayer: 3, - shardedNodeLocks: common.NewDefaultShardedLocks(), - } - - logger, _ := test.NewNullLogger() - - pf := newVectorCachePrefiller[float32](cache, index, logger) - - t.Run("prefill with limit >= graph size", func(t *testing.T) { - cache.Reset() - pf.Prefill(context.Background(), 100) - assert.Equal(t, allNumbersUpTo(100), cache.store) - }) - - t.Run("prefill with small limit so only the upper layer fits", func(t *testing.T) { - cache.Reset() - pf.Prefill(context.Background(), 7) - assert.Equal(t, map[uint64]struct{}{ - 0: {}, - 15: {}, - 30: {}, - 45: {}, - 60: {}, - 75: {}, - 90: {}, - }, cache.store) - }) - - t.Run("limit where a layer partially fits", func(t *testing.T) { - cache.Reset() - pf.Prefill(context.Background(), 10) - assert.Equal(t, map[uint64]struct{}{ - // layer 3 - 0: {}, - 15: {}, - 30: {}, - 45: {}, - 60: {}, - 75: {}, - 90: {}, - - // additional layer 2 - 5: {}, - 10: {}, - 20: {}, - }, cache.store) - }) -} - -func newFakeCache() *fakeCache { - return &fakeCache{ - store: map[uint64]struct{}{}, - } -} - -type fakeCache struct { - store map[uint64]struct{} -} - -func (f *fakeCache) MultiGet(ctx context.Context, id []uint64) ([][]float32, []error) { - panic("not implemented") -} - -func (f *fakeCache) Get(ctx context.Context, id uint64) ([]float32, error) { - f.store[id] = struct{}{} - return nil, nil -} - -func (f *fakeCache) Delete(ctx context.Context, id uint64) { - panic("not implemented") -} - -func (f *fakeCache) Preload(id uint64, vec []float32) { - panic("not implemented") -} - -func (f *fakeCache) Prefetch(id uint64) { - panic("not implemented") -} - -func (f *fakeCache) Grow(id uint64) { - panic("not implemented") -} - -func (f *fakeCache) UpdateMaxSize(size int64) { - panic("not implemented") -} - -func (f *fakeCache) All() [][]float32 { - panic("not implemented") -} - -func (f *fakeCache) Drop() { - panic("not implemented") -} - -func (f *fakeCache) CopyMaxSize() int64 { - return 1e6 -} - -func (f *fakeCache) Reset() { - f.store = map[uint64]struct{}{} -} - -func (f *fakeCache) Len() int32 { - return int32(len(f.store)) -} - -func (f *fakeCache) CountVectors() int64 { - panic("not implemented") -} - -func generateDummyVertices(amount int) []*vertex { - out := make([]*vertex, amount) - for i := range out { - out[i] = &vertex{ - id: uint64(i), - level: levelForDummyVertex(i), - } - } - - return out -} - -// maximum of 3 layers -// if id % 15 == 0 -> layer 3 -// if id % 5 == 0 -> layer 2 -// if id % 3 == 0 -> layer 1 -// remainder -> layer 0 -func levelForDummyVertex(id int) int { - if id%15 == 0 { - return 3 - } - - if id%5 == 0 { - return 2 - } - - if id%3 == 0 { - return 1 - } - - return 0 -} - -func allNumbersUpTo(size int) map[uint64]struct{} { - out := map[uint64]struct{}{} - for i := 0; i < size; i++ { - out[uint64(i)] = struct{}{} - } - - return out -} diff --git a/adapters/repos/db/vector/hnsw/vectors_for_test.go b/adapters/repos/db/vector/hnsw/vectors_for_test.go deleted file mode 100644 index 48314c896a91525a41da81ae11193f57363a8b7d..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/vectors_for_test.go +++ /dev/null @@ -1,33 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import "context" - -// roughly grouped into three clusters of three -var testVectors = [][]float32{ - {0.1, 0.9}, - {0.15, 0.8}, - {0.13, 0.65}, - - {0.6, 0.1}, - {0.63, 0.2}, - {0.65, 0.08}, - - {0.8, 0.8}, - {0.9, 0.75}, - {0.8, 0.7}, -} - -func testVectorForID(ctx context.Context, id uint64) ([]float32, error) { - return testVectors[int(id)], nil -} diff --git a/adapters/repos/db/vector/hnsw/vertex.go b/adapters/repos/db/vector/hnsw/vertex.go deleted file mode 100644 index 2bceac2d2e7c9b8ddbf093de69a8e5d64ad55ea3..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/vertex.go +++ /dev/null @@ -1,120 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "sync" -) - -type vertex struct { - id uint64 - sync.Mutex - level int - connections [][]uint64 - maintenance bool -} - -func (v *vertex) markAsMaintenance() { - v.Lock() - v.maintenance = true - v.Unlock() -} - -func (v *vertex) unmarkAsMaintenance() { - v.Lock() - v.maintenance = false - v.Unlock() -} - -func (v *vertex) isUnderMaintenance() bool { - v.Lock() - m := v.maintenance - v.Unlock() - return m -} - -func (v *vertex) connectionsAtLevelNoLock(level int) []uint64 { - return v.connections[level] -} - -func (v *vertex) upgradeToLevelNoLock(level int) { - newConnections := make([][]uint64, level+1) - copy(newConnections, v.connections) - v.level = level - v.connections = newConnections -} - -func (v *vertex) setConnectionsAtLevel(level int, connections []uint64) { - v.Lock() - defer v.Unlock() - - // before we simply copy the connections let's check how much smaller the new - // list is. If it's considerably smaller, we might want to downsize the - // current allocation - oldCap := cap(v.connections[level]) - newLen := len(connections) - ratio := float64(1) - float64(newLen)/float64(oldCap) - if ratio > 0.33 || oldCap < newLen { - // the replaced slice is over 33% smaller than the last one, this makes it - // worth to replace it entirely. This has a small performance cost, it - // means that if we need append to this node again, we need to re-allocate, - // but we gain at least a 33% memory saving on this particular node right - // away. - v.connections[level] = connections - return - } - - v.connections[level] = v.connections[level][:newLen] - copy(v.connections[level], connections) -} - -func (v *vertex) appendConnectionAtLevelNoLock(level int, connection uint64, maxConns int) { - if len(v.connections[level]) == cap(v.connections[level]) { - // if the len is the capacity, this means a new array needs to be - // allocated to back this slice. The go runtime would do this - // automatically, if we just use 'append', but it wouldn't do it very - // efficiently. It would always double the existing capacity. Since we have - // a hard limit (maxConns), we don't ever want to grow it beyond that. In - // the worst case, the current len & cap could be maxConns-1, which would - // mean we would double to 2*(maxConns-1) which would be way too large. - // - // Instead let's grow in 4 steps: 25%, 50%, 75% or full capacity - ratio := float64(len(v.connections[level])) / float64(maxConns) - - target := 0 - switch { - case ratio < 0.25: - target = int(float64(0.25) * float64(maxConns)) - case ratio < 0.50: - target = int(float64(0.50) * float64(maxConns)) - case ratio < 0.75: - target = int(float64(0.75) * float64(maxConns)) - default: - target = maxConns - } - - // handle rounding errors on maxConns not cleanly divisible by 4 - if target < len(v.connections[level])+1 { - target = len(v.connections[level]) + 1 - } - - newConns := make([]uint64, len(v.connections[level]), target) - copy(newConns, v.connections[level]) - v.connections[level] = newConns - } - - v.connections[level] = append(v.connections[level], connection) -} - -func (v *vertex) resetConnectionsAtLevelNoLock(level int) { - v.connections[level] = v.connections[level][:0] -} diff --git a/adapters/repos/db/vector/hnsw/vertex_test.go b/adapters/repos/db/vector/hnsw/vertex_test.go deleted file mode 100644 index 5357294d54e32e20ddbeb511f4e3abb463c54df8..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/vertex_test.go +++ /dev/null @@ -1,224 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestVertex_SetConnections(t *testing.T) { - type test struct { - name string - initial []uint64 - updated []uint64 - expectedCap int - } - - tests := []test{ - { - name: "no connections set before", - initial: nil, - updated: makeConnections(7, 7), - expectedCap: 7, - }, - { - name: "connections had a slightly higher cap before", - initial: makeConnections(24, 24), - updated: makeConnections(22, 22), - // we don't expect any downsizing, since it's a small diff - expectedCap: 24, - }, - { - name: "connections had a considerably higher cap before", - initial: makeConnections(24, 24), - updated: makeConnections(10, 10), - // large diff, we expect downsizing - expectedCap: 10, - }, - { - name: "connections had a lower cap before", - initial: makeConnections(10, 10), - updated: makeConnections(24, 24), - expectedCap: 24, - }, - { - name: "connections had the same length and cap", - initial: makeConnections(13, 13), - updated: makeConnections(13, 13), - expectedCap: 13, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - v := &vertex{ - connections: make([][]uint64, 1), - } - v.connections[0] = tc.initial - - v.setConnectionsAtLevel(0, tc.updated) - - assert.Equal(t, tc.updated, v.connections[0]) - assert.Equal(t, tc.expectedCap, cap(v.connections[0])) - }) - } -} - -func TestVertex_AppendConnection(t *testing.T) { - type test struct { - name string - initial []uint64 - expectedCap int - } - - tests := []test{ - { - name: "no connections set before, expect 1/4 of max", - initial: nil, - expectedCap: 16, - }, - { - name: "less than 1/4, expect 1/4 of max", - initial: makeConnections(15, 15), - expectedCap: 16, - }, - { - name: "less than 1/2, expect 1/2 of max", - initial: makeConnections(31, 31), - expectedCap: 32, - }, - { - name: "less than 3/4, expect 3/4 of max", - initial: makeConnections(42, 42), - expectedCap: 48, - }, - { - name: "more than 3/4, expect full size", - initial: makeConnections(53, 53), - expectedCap: 64, - }, - { - name: "enough capacity to not require growing", - initial: makeConnections(17, 53), - expectedCap: 53, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - v := &vertex{ - connections: make([][]uint64, 1), - } - v.connections[0] = tc.initial - - v.appendConnectionAtLevelNoLock(0, 18, 64) - - newConns := make([]uint64, len(tc.initial)+1) - copy(newConns, tc.initial) - newConns[len(newConns)-1] = 18 - - assert.Equal(t, newConns, v.connectionsAtLevelNoLock(0)) - assert.Equal(t, tc.expectedCap, cap(v.connections[0])) - }) - } -} - -func TestVertex_AppendConnection_NotCleanlyDivisible(t *testing.T) { - type test struct { - name string - initial []uint64 - expectedCap int - } - - tests := []test{ - { - name: "no connections set before, expect 1/4 of max", - initial: nil, - expectedCap: 15, - }, - { - name: "less than 1/4, expect 1/4 of max", - initial: makeConnections(15, 15), - // provoke rounding error - expectedCap: 16, - }, - { - name: "less than 1/2, expect 1/2 of max", - initial: makeConnections(31, 31), - expectedCap: 32, - }, - { - name: "less than 3/4, expect 3/4 of max", - initial: makeConnections(42, 42), - expectedCap: 47, - }, - { - name: "more than 3/4, expect full size", - initial: makeConnections(53, 53), - expectedCap: 63, - }, - { - name: "enough capacity to not require growing", - initial: makeConnections(17, 53), - expectedCap: 53, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - v := &vertex{ - connections: make([][]uint64, 1), - } - v.connections[0] = tc.initial - - v.appendConnectionAtLevelNoLock(0, 18, 63) - - newConns := make([]uint64, len(tc.initial)+1) - copy(newConns, tc.initial) - newConns[len(newConns)-1] = 18 - - assert.Equal(t, newConns, v.connectionsAtLevelNoLock(0)) - assert.Equal(t, tc.expectedCap, cap(v.connections[0])) - }) - } -} - -func TestVertex_ResetConnections(t *testing.T) { - v := &vertex{ - connections: make([][]uint64, 1), - } - v.connections[0] = makeConnections(4, 4) - - v.resetConnectionsAtLevelNoLock(0) - assert.Equal(t, 0, len(v.connections[0])) - assert.Equal(t, 4, cap(v.connections[0])) -} - -func makeConnections(length, capacity int) []uint64 { - out := make([]uint64, length, capacity) - for i := 0; i < length; i++ { - out[i] = uint64(i) - } - return out -} - -func TestVertex_Maintenance(t *testing.T) { - v := &vertex{} - - assert.False(t, v.isUnderMaintenance()) - v.markAsMaintenance() - assert.True(t, v.isUnderMaintenance()) - v.unmarkAsMaintenance() - assert.False(t, v.isUnderMaintenance()) -} diff --git a/adapters/repos/db/vector/hnsw/visited/list_set.go b/adapters/repos/db/vector/hnsw/visited/list_set.go deleted file mode 100644 index eebce73380a677d7116e2aa2fd7247beb9b40823..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/visited/list_set.go +++ /dev/null @@ -1,93 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package visited - -// ListSet is a reusable list with very efficient resets. Inspired by the C++ -// implementation in hnswlib it can be reset with zero memory writes in the -// array by moving the match target instead of altering the list. Only after a -// version overflow do we need to actually reset -// -// The new implementation uses a slice where the first element is reserved for the marker. -// This allow us to use ListSet as a value (i.e. no pointer is required) -// The marker (i.e. set[0]) allows for reusing the same list without having to zero all elements on each list reset. -// Resetting the list takes place once the marker (i.e. set[0]) overflows -type ListSet struct { - set []uint8 // set[0] is reserved for the marker (version) -} - -// Len returns the number of elements in the list. -func (l ListSet) Len() int { return len(l.set) - 1 } - -// free allocated slice. This list should not be reusable after this call. -func (l *ListSet) free() { l.set = nil } - -// NewList creates a new list. It allocates memory for elements and marker -func NewList(size int) ListSet { - set := make([]uint8, size+1) - set[0] = 1 // the marker starts always by 1 since on reset all element are set to 0 - return ListSet{set: set} -} - -// Visit sets element at node to the marker value -func (l *ListSet) Visit(node uint64) { - if int(node) >= l.Len() { // resize - newset := make([]uint8, growth(len(l.set), int(node)+1024)) - copy(newset, l.set) - l.set = newset - } - l.set[node+1] = l.set[0] -} - -// Visited checks if l contains the specified node -func (l *ListSet) Visited(node uint64) bool { - return int(node) < l.Len() && l.set[node+1] == l.set[0] -} - -// Reset list only in case of an overflow. -func (l *ListSet) Reset() { - l.set[0]++ - if l.set[0] == 0 { // if overflowed - for i := range l.set { - l.set[i] = 0 - } - l.set[0] = 1 // restart counting - } -} - -// threshold let us double the size if the old size is below it -const threshold = 2048 - -// growth calculates the amount a list should grow in a smooth way. -// -// Inspired by the go standard implementation -func growth(oldsize, size int) int { - doublesize := oldsize << 1 - if size > doublesize { - return size - } - if oldsize < threshold { - return doublesize // grow by 2x for small slices - } - // detect overflow newsize > 0 - // and prevent an infinite loop. - newsize := oldsize - for newsize > 0 && newsize < size { - // grow by 1.25x for large slices - // This formula allows for smothly growing - newsize += (newsize + threshold) / 4 - } - // return requested size in case of overflow - if newsize <= 0 { - newsize = size - } - return newsize -} diff --git a/adapters/repos/db/vector/hnsw/visited/list_set_test.go b/adapters/repos/db/vector/hnsw/visited/list_set_test.go deleted file mode 100644 index 195baf52639e33ddfcc54f4930d563015ecb187e..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/visited/list_set_test.go +++ /dev/null @@ -1,152 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package visited - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestVisitedList(t *testing.T) { - t.Run("creating a new list, filling it and checking against it", func(t *testing.T) { - l := NewList(1000) - - l.Visit(7) - l.Visit(38) - l.Visit(999) - - assert.True(t, l.Visited(7), "visited node should be marked visited") - assert.True(t, l.Visited(38), "visited node should be marked visited") - assert.True(t, l.Visited(999), "visited node should be marked visited") - assert.False(t, l.Visited(6), "unvisited node should NOT be marked visited") - assert.False(t, l.Visited(37), "unvisited node should NOT be marked visited") - assert.False(t, l.Visited(998), "unvisited node should NOT be marked visited") - }) - - t.Run("reusing a list it is not affected by past entries", func(t *testing.T) { - l := NewList(1000) - - l.Visit(7) - l.Visit(38) - l.Visit(999) - - l.Reset() - - l.Visit(6) - l.Visit(37) - l.Visit(998) - - assert.False(t, l.Visited(7), "an entry before the reset has no influence") - assert.False(t, l.Visited(38), "an entry before the reset has no influence") - assert.False(t, l.Visited(999), "an entry before the reset has no influence") - assert.False(t, l.Visited(20), "an entry never visited is not visited") - assert.True(t, l.Visited(6), "a node visited in this round is marked as such") - assert.True(t, l.Visited(37), "a node visited in this round is marked as such") - assert.True(t, l.Visited(998), "a node visited in this round is marked as such") - }) - - t.Run("it creates no false positives after a version overflow (v=1)", func(t *testing.T) { - l := NewList(1000) - - for i := 0; i < 255; i++ { - l.Reset() - } - - // verify the test is correct and we are indeed at the version we think we are - assert.Equal(t, uint8(1), l.set[0]) - - // verify there are zero visited nodes - for i := uint64(0); i < 1000; i++ { - assert.False(t, l.Visited(i), "node should not be visited") - } - }) - - t.Run("it creates no false positives after a version overflow (v=1)", func(t *testing.T) { - l := NewList(1000) - - // mark every node as visited in version==1 - for i := uint64(0); i < 1000; i++ { - l.Visit(i) - } - - // v==0 does not exist, so we only need 255 runs to be at version==1 again - for i := 0; i < 255; i++ { - l.Reset() - } - - // verify the test is correct and we are indeed at the version we think we are - assert.Equal(t, l.set[0], uint8(1)) - - // verify there are zero visited nodes - for i := uint64(0); i < 1000; i++ { - assert.False(t, l.Visited(i), "node should not be visited") - } - }) -} - -func TestListSetResize(t *testing.T) { - l := NewList(2) - assert.Equal(t, []uint8{1, 0, 0}, l.set) - assert.Equal(t, l.Len(), 2) - l.Visit(1) - assert.Equal(t, []uint8{1, 0, 1}, l.set) - assert.Equal(t, l.Len(), 2) - l.Reset() - assert.Equal(t, []uint8{2, 0, 1}, l.set) - assert.Equal(t, l.Len(), 2) - l.Visit(1) - assert.Equal(t, []uint8{2, 0, 2}, l.set) - assert.Equal(t, l.Len(), (2)) - l.Visit(3) - assert.Equal(t, []uint8{2, 0, 2, 0, 2}, l.set[0:5]) - assert.Equal(t, (2 + 1024), l.Len()) - l.free() - assert.Equal(t, []uint8(nil), l.set) -} - -func TestGrowth(t *testing.T) { - MaxInt := 1<<63 - 1 // math.MaxInt needs go >= 1.17 - tests := []struct { - old int - new int - want int - }{ - {512, 1000, 1024}, - {1024, 1048, 2048}, - {2000, 3500, 4000}, - {3500, 4500, 4887}, - {threshold, threshold + 32, threshold + threshold/2}, - {2097152, 4194304, 5122952}, - {threshold, MaxInt, MaxInt}, - {MaxInt / 2, MaxInt - 1, MaxInt - 1}, - } - for _, tc := range tests { - got := growth(tc.old, tc.new) - if got != tc.want { - t.Errorf("growth(%d,%d) got:%d want:%d", tc.old, tc.new, got, tc.want) - } - } -} - -func insertItems() { - list := NewList(1024) - for i := uint64(1); i < 8000000; i++ { - list.Visit(i) - } -} - -func BenchmarkListInsert(b *testing.B) { - for i := 0; i < b.N; i++ { - insertItems() - } -} diff --git a/adapters/repos/db/vector/hnsw/visited/pool.go b/adapters/repos/db/vector/hnsw/visited/pool.go deleted file mode 100644 index 6bec0f63f0150036d0343b590c27f4490838ecbd..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/visited/pool.go +++ /dev/null @@ -1,77 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package visited - -import ( - "sync" -) - -type Pool struct { - sync.Mutex - listSetSize int - listSets []ListSet -} - -// NewPool creates a new pool with specified size. -// listSetSize specifies the size of a list at creation time point -func NewPool(size int, listSetSize int) *Pool { - p := &Pool{ - listSetSize: listSetSize, - listSets: make([]ListSet, size, size+32), // make enough room - } - - for i := 0; i < size; i++ { - p.listSets[i] = NewList(listSetSize) - } - - return p -} - -// Borrow return a free list -func (p *Pool) Borrow() ListSet { - p.Lock() - defer p.Unlock() - - if n := len(p.listSets); n > 0 { - l := p.listSets[n-1] - p.listSets[n-1].free() // prevent memory leak - p.listSets = p.listSets[:n-1] - return l - } - - return NewList(p.listSetSize) -} - -// Return list l to the pool -// The list l might be thrown if l.Len() > listSetSize*1.10 -func (p *Pool) Return(l ListSet) { - p.Lock() - defer p.Unlock() - - if n := l.Len(); n < p.listSetSize || n > p.listSetSize*11/10 { // 11/10 could be tuned - return // discard this list, it does not match our current criteria - } - - l.Reset() - p.listSets = append(p.listSets, l) -} - -// Destroy and empty pool -func (p *Pool) Destroy() { - p.Lock() - defer p.Unlock() - for i := range p.listSets { - p.listSets[i].free() - } - - p.listSets = nil -} diff --git a/adapters/repos/db/vector/hnsw/visited/pool_test.go b/adapters/repos/db/vector/hnsw/visited/pool_test.go deleted file mode 100644 index 934cb3158fe110b307cc91de945effdbc97819c5..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/hnsw/visited/pool_test.go +++ /dev/null @@ -1,53 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package visited - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestPool(t *testing.T) { - // pool with two liss - pool := NewPool(2, 2) - assert.Equal(t, 2, len(pool.listSets)) - - // get first list - l1 := pool.Borrow() - assert.Equal(t, 2, l1.Len()) - assert.Equal(t, 1, len(pool.listSets)) - - // get second list - l2 := pool.Borrow() - assert.Equal(t, 0, len(pool.listSets)) - - // get third list from empty pool and return it back - l3 := pool.Borrow() - assert.Equal(t, 0, len(pool.listSets)) - pool.Return(l3) - assert.Equal(t, 1, len(pool.listSets)) - - // get same list again and modify its size - // so that it is not accepted when returned to the pool - l3 = pool.Borrow() - l3.Visit(2) - pool.Return(l3) - assert.Equal(t, 0, len(pool.listSets)) - - // add two list and destroy the pool - pool.Return(l1) - pool.Return(l2) - assert.Equal(t, 2, len(pool.listSets)) - pool.Destroy() - assert.Equal(t, 0, len(pool.listSets)) -} diff --git a/adapters/repos/db/vector/noop/index.go b/adapters/repos/db/vector/noop/index.go deleted file mode 100644 index 54bcc4433b7f174c07a48c00e9d9d57591fac14b..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/noop/index.go +++ /dev/null @@ -1,133 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package noop - -import ( - "context" - "fmt" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/vectorindex/hnsw" -) - -type Index struct{} - -func NewIndex() *Index { - return &Index{} -} - -func (i *Index) AddBatch(ctx context.Context, id []uint64, vector [][]float32) error { - // silently ignore - return nil -} - -func (i *Index) Add(id uint64, vector []float32) error { - // silently ignore - return nil -} - -func (i *Index) Delete(id ...uint64) error { - // silently ignore - return nil -} - -func (i *Index) SearchByVector(vector []float32, k int, allow helpers.AllowList) ([]uint64, []float32, error) { - return nil, nil, errors.Errorf("cannot vector-search on a class not vector-indexed") -} - -func (i *Index) SearchByVectorDistance(vector []float32, dist float32, maxLimit int64, allow helpers.AllowList) ([]uint64, []float32, error) { - return nil, nil, errors.Errorf("cannot vector-search on a class not vector-indexed") -} - -func (i *Index) UpdateUserConfig(updated schema.VectorIndexConfig, callback func()) error { - callback() - switch t := updated.(type) { - case hnsw.UserConfig: - // the fact that we are in the noop index means that 'skip' must have been - // set to true before, so changing it now is not possible. But if it - // stays, we don't mind. - if t.Skip { - return nil - } - return errors.Errorf("cannot update vector index config on a non-indexed class. Delete and re-create without skip property") - - default: - return fmt.Errorf("unrecognized vector index config: %T", updated) - - } -} - -func (i *Index) Drop(context.Context) error { - // silently ignore - return nil -} - -func (i *Index) Flush() error { - return nil -} - -func (i *Index) Shutdown(context.Context) error { - return nil -} - -func (i *Index) SwitchCommitLogs(context.Context) error { - return nil -} - -func (i *Index) ListFiles(context.Context, string) ([]string, error) { - return nil, nil -} - -func (i *Index) ValidateBeforeInsert(vector []float32) error { - return nil -} - -func (i *Index) PostStartup() { -} - -func (i *Index) Dump(labels ...string) { -} - -func (i *Index) DistanceBetweenVectors(x, y []float32) (float32, bool, error) { - return 0, true, nil -} - -func (i *Index) ContainsNode(id uint64) bool { - return false -} - -func (i *Index) DistancerProvider() distancer.Provider { - return nil -} - -func (i *Index) ShouldCompress() (bool, int) { - return false, 0 -} - -func (i *Index) ShouldCompressFromConfig(config schema.VectorIndexConfig) (bool, int) { - return false, 0 -} - -func (i *Index) Compressed() bool { - return false -} - -func (i *Index) AlreadyIndexed() uint64 { - return 0 -} - -func (i *Index) TurnOnCompression(callback func()) error { - return nil -} diff --git a/adapters/repos/db/vector/noop/index_test.go b/adapters/repos/db/vector/noop/index_test.go deleted file mode 100644 index a44e2d57e05ca1e4b516392f4f95e885b444952f..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/noop/index_test.go +++ /dev/null @@ -1,51 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package noop - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/vectorindex/hnsw" -) - -func Test_UpdateConfig(t *testing.T) { - t.Run("hnsw: with skip==true", func(t *testing.T) { - // the param we care about was not changed, do not error - - ind := NewIndex() - err := ind.UpdateUserConfig(hnsw.UserConfig{ - Skip: true, - }, func() {}) - - assert.Nil(t, err) - }) - - t.Run("hnsw: with skip==false", func(t *testing.T) { - ind := NewIndex() - err := ind.UpdateUserConfig(hnsw.UserConfig{ - Skip: false, - }, func() {}) - - require.NotNil(t, err) - assert.Contains(t, err.Error(), "Delete and re-create") - }) - - t.Run("with unrecognized vector index config", func(t *testing.T) { - ind := NewIndex() - err := ind.UpdateUserConfig(nil, func() {}) - - require.NotNil(t, err) - assert.Contains(t, err.Error(), "unrecognized vector index config") - }) -} diff --git a/adapters/repos/db/vector/testinghelpers/helpers.go b/adapters/repos/db/vector/testinghelpers/helpers.go deleted file mode 100644 index 126f46d79b6973c543a17c1944d3dafb384fec64..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector/testinghelpers/helpers.go +++ /dev/null @@ -1,263 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package testinghelpers - -import ( - "encoding/binary" - "encoding/gob" - "fmt" - "io" - "math" - "math/rand" - "os" - "sort" - "testing" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv" - "github.com/weaviate/weaviate/adapters/repos/db/vector/compressionhelpers" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" - "github.com/weaviate/weaviate/entities/cyclemanager" -) - -type DistanceFunction func([]float32, []float32) float32 - -func getRandomSeed() *rand.Rand { - return rand.New(rand.NewSource(time.Now().UnixNano())) -} - -func int32FromBytes(bytes []byte) int { - return int(binary.LittleEndian.Uint32(bytes)) -} - -func float32FromBytes(bytes []byte) float32 { - bits := binary.LittleEndian.Uint32(bytes) - float := math.Float32frombits(bits) - return float -} - -func readSiftFloat(file string, maxObjects int, vectorLengthFloat int) [][]float32 { - f, err := os.Open(file) - if err != nil { - panic(errors.Wrap(err, "Could not open SIFT file")) - } - defer f.Close() - - fi, err := f.Stat() - if err != nil { - panic(errors.Wrap(err, "Could not get SIFT file properties")) - } - fileSize := fi.Size() - if fileSize < 1000000 { - panic("The file is only " + fmt.Sprint(fileSize) + " bytes long. Did you forgot to install git lfs?") - } - - // The sift data is a binary file containing floating point vectors - // For each entry, the first 4 bytes is the length of the vector (in number of floats, not in bytes) - // which is followed by the vector data with vector length * 4 bytes. - // |-length-vec1 (4bytes)-|-Vec1-data-(4*length-vector-1 bytes)-|-length-vec2 (4bytes)-|-Vec2-data-(4*length-vector-2 bytes)-| - // The vector length needs to be converted from bytes to int - // The vector data needs to be converted from bytes to float - // Note that the vector entries are of type float but are integer numbers eg 2.0 - bytesPerF := 4 - objects := make([][]float32, maxObjects) - vectorBytes := make([]byte, bytesPerF+vectorLengthFloat*bytesPerF) - for i := 0; i >= 0; i++ { - _, err = f.Read(vectorBytes) - if err == io.EOF { - break - } else if err != nil { - panic(err) - } - if int32FromBytes(vectorBytes[0:bytesPerF]) != vectorLengthFloat { - panic("Each vector must have 128 entries.") - } - vectorFloat := []float32{} - for j := 0; j < vectorLengthFloat; j++ { - start := (j + 1) * bytesPerF // first bytesPerF are length of vector - vectorFloat = append(vectorFloat, float32FromBytes(vectorBytes[start:start+bytesPerF])) - } - objects[i] = vectorFloat - - if i >= maxObjects-1 { - break - } - } - - return objects -} - -func ReadSiftVecsFrom(path string, size int, dimensions int) [][]float32 { - fmt.Printf("generating %d sift vectors...", size) - vectors := readSiftFloat(path, size, dimensions) - fmt.Printf(" done\n") - return vectors -} - -func RandomVecs(size int, queriesSize int, dimensions int) ([][]float32, [][]float32) { - fmt.Printf("generating %d vectors...\n", size+queriesSize) - r := getRandomSeed() - vectors := make([][]float32, 0, size) - queries := make([][]float32, 0, queriesSize) - for i := 0; i < size; i++ { - vectors = append(vectors, genVector(r, dimensions)) - } - for i := 0; i < queriesSize; i++ { - queries = append(queries, genVector(r, dimensions)) - } - return vectors, queries -} - -func genVector(r *rand.Rand, dimensions int) []float32 { - vector := make([]float32, 0, dimensions) - for i := 0; i < dimensions; i++ { - // Some distances like dot could produce negative values when the vectors have negative values - // This change will not affect anything when using a distance like l2, but will cover some bugs - // when using distances like dot - vector = append(vector, r.Float32()*2-1) - } - return vector -} - -func Normalize(vectors [][]float32) { - for i := range vectors { - vectors[i] = distancer.Normalize(vectors[i]) - } -} - -func ReadVecs(size int, queriesSize int, dimensions int, db string, path ...string) ([][]float32, [][]float32) { - fmt.Printf("generating %d read vectors...", size+queriesSize) - uri := db - if len(path) > 0 { - uri = fmt.Sprintf("%s/%s", path[0], uri) - } - vectors := readSiftFloat(fmt.Sprintf("%s/%s_base.fvecs", uri, db), size, dimensions) - queries := readSiftFloat(fmt.Sprintf("%s/%s_query.fvecs", uri, db), queriesSize, dimensions) - fmt.Printf(" done\n") - return vectors, queries -} - -func ReadQueries(queriesSize int) [][]float32 { - fmt.Printf("generating %d read queries vectors...", queriesSize) - queries := readSiftFloat("sift/sift_query.fvecs", queriesSize, 128) - fmt.Printf(" done\n") - return queries -} - -func BruteForce(vectors [][]float32, query []float32, k int, distance DistanceFunction) ([]uint64, []float32) { - type distanceAndIndex struct { - distance float32 - index uint64 - } - - distances := make([]distanceAndIndex, len(vectors)) - - compressionhelpers.Concurrently(uint64(len(vectors)), func(i uint64) { - dist := distance(query, vectors[i]) - distances[i] = distanceAndIndex{ - index: uint64(i), - distance: dist, - } - }) - - sort.Slice(distances, func(a, b int) bool { - return distances[a].distance < distances[b].distance - }) - - if len(distances) < k { - k = len(distances) - } - - out := make([]uint64, k) - dists := make([]float32, k) - for i := 0; i < k; i++ { - out[i] = distances[i].index - dists[i] = distances[i].distance - } - - return out, dists -} - -func BuildTruths(queriesSize int, vectorsSize int, queries [][]float32, vectors [][]float32, k int, distance DistanceFunction, path ...string) [][]uint64 { - uri := "sift/sift_truths%d.%d.gob" - if len(path) > 0 { - uri = fmt.Sprintf("%s/%s", path[0], uri) - } - fileName := fmt.Sprintf(uri, k, vectorsSize) - truths := make([][]uint64, queriesSize) - - if _, err := os.Stat(fileName); err == nil { - return loadTruths(fileName, queriesSize, k) - } - - compressionhelpers.Concurrently(uint64(len(queries)), func(i uint64) { - truths[i], _ = BruteForce(vectors, queries[i], k, distance) - }) - - f, err := os.Create(fileName) - if err != nil { - panic(errors.Wrap(err, "Could not open file")) - } - - defer f.Close() - enc := gob.NewEncoder(f) - err = enc.Encode(truths) - if err != nil { - panic(errors.Wrap(err, "Could not encode truths")) - } - return truths -} - -func loadTruths(fileName string, queriesSize int, k int) [][]uint64 { - f, err := os.Open(fileName) - if err != nil { - panic(errors.Wrap(err, "Could not open truths file")) - } - defer f.Close() - - truths := make([][]uint64, queriesSize) - cDec := gob.NewDecoder(f) - err = cDec.Decode(&truths) - if err != nil { - panic(errors.Wrap(err, "Could not decode truths")) - } - return truths -} - -func MatchesInLists(control []uint64, results []uint64) uint64 { - desired := map[uint64]struct{}{} - for _, relevant := range control { - desired[relevant] = struct{}{} - } - - var matches uint64 - for _, candidate := range results { - _, ok := desired[candidate] - if ok { - matches++ - } - } - - return matches -} - -func NewDummyStore(t testing.TB) *lsmkv.Store { - logger, _ := test.NewNullLogger() - storeDir := t.TempDir() - store, err := lsmkv.New(storeDir, storeDir, logger, nil, - cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop()) - require.Nil(t, err) - return store -} diff --git a/adapters/repos/db/vector_index.go b/adapters/repos/db/vector_index.go deleted file mode 100644 index 726d8b804b078a3ed366146f05384538964d3d61..0000000000000000000000000000000000000000 --- a/adapters/repos/db/vector_index.go +++ /dev/null @@ -1,44 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package db - -import ( - "context" - - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/adapters/repos/db/vector/hnsw/distancer" - "github.com/weaviate/weaviate/entities/schema" -) - -// VectorIndex is anything that indexes vectors efficiently. For an example -// look at ./vector/hnsw/index.go -type VectorIndex interface { - Dump(labels ...string) - Add(id uint64, vector []float32) error - AddBatch(ctx context.Context, id []uint64, vector [][]float32) error - Delete(id ...uint64) error - SearchByVector(vector []float32, k int, allow helpers.AllowList) ([]uint64, []float32, error) - SearchByVectorDistance(vector []float32, dist float32, - maxLimit int64, allow helpers.AllowList) ([]uint64, []float32, error) - UpdateUserConfig(updated schema.VectorIndexConfig, callback func()) error - Drop(ctx context.Context) error - Shutdown(ctx context.Context) error - Flush() error - SwitchCommitLogs(ctx context.Context) error - ListFiles(ctx context.Context, basePath string) ([]string, error) - PostStartup() - Compressed() bool - ValidateBeforeInsert(vector []float32) error - DistanceBetweenVectors(x, y []float32) (float32, bool, error) - ContainsNode(id uint64) bool - DistancerProvider() distancer.Provider -} diff --git a/adapters/repos/modules/modules.go b/adapters/repos/modules/modules.go deleted file mode 100644 index 512022547dc2c011d372b0af6fd2d600fa36f005..0000000000000000000000000000000000000000 --- a/adapters/repos/modules/modules.go +++ /dev/null @@ -1,139 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modulestorage - -import ( - "fmt" - "os" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/moduletools" - bolt "go.etcd.io/bbolt" -) - -type Repo struct { - logger logrus.FieldLogger - baseDir string - db *bolt.DB -} - -func NewRepo(baseDir string, logger logrus.FieldLogger) (*Repo, error) { - r := &Repo{ - baseDir: baseDir, - logger: logger, - } - - err := r.init() - return r, err -} - -func (r *Repo) DBPath() string { - return fmt.Sprintf("%s/modules.db", r.baseDir) -} - -func (r *Repo) DataPath() string { - return r.baseDir -} - -func (r *Repo) init() error { - if err := os.MkdirAll(r.baseDir, 0o777); err != nil { - return errors.Wrapf(err, "create root path directory at %s", r.baseDir) - } - - boltdb, err := bolt.Open(r.DBPath(), 0o600, nil) - if err != nil { - return errors.Wrapf(err, "open bolt at %s", r.DBPath()) - } - - r.db = boltdb - - return nil -} - -type storageBucket struct { - bucketKey []byte - repo *Repo -} - -func (r *Repo) Storage(bucketName string) (moduletools.Storage, error) { - storage := &storageBucket{ - bucketKey: []byte(bucketName), - repo: r, - } - - err := storage.init() - return storage, err -} - -func (s *storageBucket) init() error { - return s.repo.db.Update(func(tx *bolt.Tx) error { - if _, err := tx.CreateBucketIfNotExists(s.bucketKey); err != nil { - return errors.Wrapf(err, "create module storage bucket '%s'", - string(s.bucketKey)) - } - - return nil - }) -} - -func (s *storageBucket) Put(key, value []byte) error { - return s.repo.db.Batch(func(tx *bolt.Tx) error { - b := tx.Bucket(s.bucketKey) - if b == nil { - return errors.Errorf("no bucket for key %s found", string(s.bucketKey)) - } - - if err := b.Put(key, value); err != nil { - return errors.Wrapf(err, "put value for key %s", string(key)) - } - - return nil - }) -} - -func (s *storageBucket) Get(key []byte) ([]byte, error) { - var out []byte - err := s.repo.db.View(func(tx *bolt.Tx) error { - b := tx.Bucket(s.bucketKey) - if b == nil { - return errors.Errorf("no bucket for key %s found", string(s.bucketKey)) - } - - out = b.Get(key) - return nil - }) - - return out, err -} - -func (s *storageBucket) Scan(scan moduletools.ScanFn) error { - err := s.repo.db.View(func(tx *bolt.Tx) error { - b := tx.Bucket(s.bucketKey) - - c := b.Cursor() - for k, v := c.First(); k != nil; k, v = c.Next() { - ok, err := scan(k, v) - if err != nil { - return errors.Wrapf(err, "read item %q", string(k)) - } - - if !ok { - break - } - } - - return nil - }) - - return err -} diff --git a/adapters/repos/modules/modules_integration_test.go b/adapters/repos/modules/modules_integration_test.go deleted file mode 100644 index 1a269e755718c24e9742bfe88387137c4bb7ee43..0000000000000000000000000000000000000000 --- a/adapters/repos/modules/modules_integration_test.go +++ /dev/null @@ -1,190 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build integrationTest -// +build integrationTest - -package modulestorage - -import ( - "crypto/rand" - "fmt" - "math/big" - "os" - "testing" - - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func mustRandIntn(max int64) int { - randInt, err := rand.Int(rand.Reader, big.NewInt(max)) - if err != nil { - panic(fmt.Sprintf("mustRandIntn error: %v", err)) - } - return int(randInt.Int64()) -} - -func Test_ModuleStorage(t *testing.T) { - dirName := fmt.Sprintf("./testdata/%d", mustRandIntn(10000000)) - os.MkdirAll(dirName, 0o777) - defer func() { - err := os.RemoveAll(dirName) - fmt.Println(err) - }() - - logger, _ := test.NewNullLogger() - - r, err := NewRepo(dirName, logger) - require.Nil(t, err) - - module1, err := r.Storage("my-module") - require.Nil(t, err) - module2, err := r.Storage("my-other-module") - require.Nil(t, err) - - t.Run("storing two k/v pairs for each bucket", func(t *testing.T) { - err := module1.Put([]byte("module1-key1"), []byte("module1-value1")) - require.Nil(t, err) - err = module1.Put([]byte("module1-key2"), []byte("module1-value2")) - require.Nil(t, err) - err = module2.Put([]byte("module2-key1"), []byte("module2-value1")) - require.Nil(t, err) - err = module2.Put([]byte("module2-key2"), []byte("module2-value2")) - require.Nil(t, err) - }) - - t.Run("retrieving values across buckets and keys", func(t *testing.T) { - var v []byte - var err error - - // on module 1 bucket - v, err = module1.Get([]byte("module1-key1")) - require.Nil(t, err) - assert.Equal(t, []byte("module1-value1"), v) - - v, err = module1.Get([]byte("module1-key2")) - require.Nil(t, err) - assert.Equal(t, []byte("module1-value2"), v) - - v, err = module1.Get([]byte("module2-key1")) - require.Nil(t, err) - assert.Equal(t, []byte(nil), v) - - v, err = module1.Get([]byte("module2-key2")) - require.Nil(t, err) - assert.Equal(t, []byte(nil), v) - - // on module 2 bucket - v, err = module2.Get([]byte("module1-key1")) - require.Nil(t, err) - assert.Equal(t, []byte(nil), v) - - v, err = module2.Get([]byte("module1-key2")) - require.Nil(t, err) - assert.Equal(t, []byte(nil), v) - - v, err = module2.Get([]byte("module2-key1")) - require.Nil(t, err) - assert.Equal(t, []byte("module2-value1"), v) - - v, err = module2.Get([]byte("module2-key2")) - require.Nil(t, err) - assert.Equal(t, []byte("module2-value2"), v) - }) - - t.Run("scanning all k/v for a bucket", func(t *testing.T) { - t.Run("module1 - full range", func(t *testing.T) { - var ( - keys [][]byte - values [][]byte - ) - expectedKeys := [][]byte{ - []byte("module1-key1"), - []byte("module1-key2"), - } - expectedValues := [][]byte{ - []byte("module1-value1"), - []byte("module1-value2"), - } - - err := module1.Scan(func(k, v []byte) (bool, error) { - keys = append(keys, k) - values = append(values, v) - return true, nil - }) - - require.Nil(t, err) - - assert.Equal(t, expectedKeys, keys) - assert.Equal(t, expectedValues, values) - }) - - t.Run("module2 - full range", func(t *testing.T) { - var ( - keys [][]byte - values [][]byte - ) - expectedKeys := [][]byte{ - []byte("module2-key1"), - []byte("module2-key2"), - } - expectedValues := [][]byte{ - []byte("module2-value1"), - []byte("module2-value2"), - } - - err := module2.Scan(func(k, v []byte) (bool, error) { - keys = append(keys, k) - values = append(values, v) - return true, nil - }) - - require.Nil(t, err) - - assert.Equal(t, expectedKeys, keys) - assert.Equal(t, expectedValues, values) - }) - - t.Run("module2 - stop after single row", func(t *testing.T) { - var ( - keys [][]byte - values [][]byte - ) - expectedKeys := [][]byte{ - []byte("module2-key1"), - } - expectedValues := [][]byte{ - []byte("module2-value1"), - } - - err := module2.Scan(func(k, v []byte) (bool, error) { - keys = append(keys, k) - values = append(values, v) - return false, nil - }) - - require.Nil(t, err) - - assert.Equal(t, expectedKeys, keys) - assert.Equal(t, expectedValues, values) - }) - - t.Run("module2 - with scan error", func(t *testing.T) { - err := module2.Scan(func(k, v []byte) (bool, error) { - return false, fmt.Errorf("oops") - }) - - assert.Equal(t, "read item \"module2-key1\": oops", err.Error()) - }) - }) -} diff --git a/adapters/repos/schema/store.go b/adapters/repos/schema/store.go deleted file mode 100644 index dfef87e64927607274b227c929f7d22e7613c277..0000000000000000000000000000000000000000 --- a/adapters/repos/schema/store.go +++ /dev/null @@ -1,536 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package schema - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "os" - "path" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/models" - ucs "github.com/weaviate/weaviate/usecases/schema" - "github.com/weaviate/weaviate/usecases/sharding" - bolt "go.etcd.io/bbolt" -) - -var ( - // old keys are still needed for migration - schemaBucket = []byte("schema") - schemaKey = []byte("schema") - // static keys - keyMetaClass = []byte{eTypeMeta, 0} - keyShardingState = []byte{eTypeSharingState, 0} - keyConfig = []byte{eTypeConfig, 0} - _Version int = 2 -) - -// constant to encode the type of entry in the DB -const ( - eTypeConfig byte = 1 - eTypeClass byte = 2 - eTypeShard byte = 4 - eTypeMeta byte = 5 - eTypeSharingState byte = 15 -) - -// config configuration specific the stored schema -type config struct { - Version int - // add more fields -} - -/* -Store is responsible for storing and persisting the schema in a structured manner. -It ensures that each class has a dedicated bucket, which includes metadata, and sharding state. - -Schema Structure: - - Config: contains metadata related to parsing the schema - - Nested buckets for each class - -Schema Structure for a class Bucket: - - Metadata contains models.Class - - Sharding state without shards - - Class shards: individual shard associated with the sharding state - -By organizing the schema in this manner, it facilitates efficient management of class specific data during runtime. -In addition, old schema are backed up and migrated to the new structure for a seamless transitions -*/ -type store struct { - version int // schema version - homeDir string // home directory of schema files - log logrus.FieldLogger - db *bolt.DB -} - -// NewStore returns a new schema repository. Call the Open() method to open the underlying DB. -// To free the resources, call the Close() method. -func NewStore(homeDir string, logger logrus.FieldLogger) *store { - return &store{ - version: _Version, - homeDir: homeDir, - log: logger, - } -} - -func initBoltDB(filePath string, version int, cfg *config) (*bolt.DB, error) { - db, err := bolt.Open(filePath, 0o600, nil) - if err != nil { - return nil, fmt.Errorf("open %q: %w", filePath, err) - } - root := func(tx *bolt.Tx) error { - b, err := tx.CreateBucket(schemaBucket) - // A new bucket has been created - if err == nil { - *cfg = config{Version: version} - return saveConfig(b, *cfg) - } - // load existing bucket - b = tx.Bucket(schemaBucket) - if b == nil { - return fmt.Errorf("retrieve existing bucket %q", schemaBucket) - } - // read config: config exists since version 2 - data := b.Get(keyConfig) - if len(data) > 0 { - if err := json.Unmarshal(data, &cfg); err != nil { - return fmt.Errorf("cannot read config: %w", err) - } - } - return nil - } - - return db, db.Update(root) -} - -// Open the underlying DB -func (r *store) Open() (err error) { - if err := os.MkdirAll(r.homeDir, 0o777); err != nil { - return fmt.Errorf("create root directory %q: %w", r.homeDir, err) - } - cfg := config{} - path := path.Join(r.homeDir, "schema.db") - boltDB, err := initBoltDB(path, r.version, &cfg) - if err != nil { - return fmt.Errorf("init bolt_db: %w", err) - } - defer func() { - if err != nil { - boltDB.Close() - } - }() - r.db = boltDB - if cfg.Version < r.version { - if err := r.migrate(path, cfg.Version, r.version); err != nil { - return fmt.Errorf("migrate: %w", err) - } - } - if cfg.Version > r.version { - return fmt.Errorf("schema version %d higher than %d", cfg.Version, r.version) - } - return err -} - -// Close the underlying DB -func (r *store) Close() { - r.db.Close() -} - -// migrate from old to new schema -// It will back up the old schema file if it exists -func (r *store) migrate(filePath string, from, to int) (err error) { - r.log.Infof("schema migration from v%d to v%d process has started", from, to) - defer func() { - if err == nil { - r.log.Infof("successfully completed schema migration from v%d to v%d", from, to) - } - }() - state, err := r.loadSchemaV1() - if err != nil { - return fmt.Errorf("load old schema: %w", err) - } - if state != nil { - // create backupPath by copying file - backupPath := fmt.Sprintf("%s_v%d.bak", filePath, from) - if err := copyFile(backupPath, filePath); err != nil { - return fmt.Errorf("schema backup: %w", err) - } - - // write new schema - f := func(tx *bolt.Tx) error { - b := tx.Bucket(schemaBucket) - if err := saveConfig(b, config{Version: to}); err != nil { - return err - } - b.Delete(schemaKey) // remove old schema - return r.saveAllTx(context.Background(), b, *state)(tx) - } - if err := r.db.Update(f); err != nil { - os.Remove(backupPath) - return fmt.Errorf("convert to new schema: %w", err) - } - } - return nil -} - -// saveSchemaV1 might be needed to migrate from v2 to v0 -func (r *store) saveSchemaV1(schema ucs.State) error { - schemaJSON, err := json.Marshal(schema) - if err != nil { - return errors.Wrapf(err, "marshal schema state to json") - } - - return r.db.Update(func(tx *bolt.Tx) error { - b := tx.Bucket(schemaBucket) - return b.Put(schemaKey, schemaJSON) - }) -} - -// loadSchemaV1 is needed to migrate from v0 to v2 -func (r *store) loadSchemaV1() (*ucs.State, error) { - var schemaJSON []byte - r.db.View(func(tx *bolt.Tx) error { - b := tx.Bucket(schemaBucket) - schemaJSON = b.Get(schemaKey) - return nil - }) - - if len(schemaJSON) == 0 { - return nil, nil - } - - var state ucs.State - err := json.Unmarshal(schemaJSON, &state) - if err != nil { - return nil, errors.Wrapf(err, "parse schema state from JSON") - } - - return &state, nil -} - -// UpdateClass if it exists, otherwise return an error. -func (r *store) UpdateClass(_ context.Context, data ucs.ClassPayload) error { - classKey := encodeClassName(data.Name) - f := func(tx *bolt.Tx) error { - b := tx.Bucket(schemaBucket).Bucket(classKey) - if b == nil { - return fmt.Errorf("class not found") - } - return r.updateClass(b, data) - } - return r.db.Update(f) -} - -// NewClass creates a new class if it doesn't exists, otherwise return an error -func (r *store) NewClass(_ context.Context, data ucs.ClassPayload) error { - classKey := encodeClassName(data.Name) - f := func(tx *bolt.Tx) error { - b, err := tx.Bucket(schemaBucket).CreateBucket(classKey) - if err != nil { - return err - } - return r.updateClass(b, data) - } - return r.db.Update(f) -} - -func (r *store) updateClass(b *bolt.Bucket, data ucs.ClassPayload) error { - // remove old shards - if data.ReplaceShards { - cursor := b.Cursor() // b.Put before - for key, _ := cursor.First(); key != nil; { - if key[0] == eTypeShard { - b.Delete(key) - } - key, _ = cursor.Next() - } - } - if data.Metadata != nil { - if err := b.Put(keyMetaClass, data.Metadata); err != nil { - return err - } - } - - if data.ShardingState != nil { - if err := b.Put(keyShardingState, data.ShardingState); err != nil { - return err - } - } - - return appendShards(b, data.Shards, make([]byte, 1, 68)) -} - -// DeleteClass class -func (r *store) DeleteClass(_ context.Context, class string) error { - classKey := encodeClassName(class) - f := func(tx *bolt.Tx) error { - err := tx.Bucket(schemaBucket).DeleteBucket(classKey) - if err != nil && !errors.Is(err, bolt.ErrBucketNotFound) { - return err - } - return nil - } - return r.db.Update(f) -} - -// NewShards add new shards to an existing class -func (r *store) NewShards(_ context.Context, class string, shards []ucs.KeyValuePair) error { - classKey := encodeClassName(class) - f := func(tx *bolt.Tx) error { - b := tx.Bucket(schemaBucket).Bucket(classKey) - if b == nil { - return fmt.Errorf("class not found") - } - return appendShards(b, shards, make([]byte, 1, 68)) - } - return r.db.Update(f) -} - -// Update shards updates (replaces) shards of existing class -// Error is returned if class or shard does not exist -func (r *store) UpdateShards(_ context.Context, class string, shards []ucs.KeyValuePair) error { - classKey := encodeClassName(class) - f := func(tx *bolt.Tx) error { - b := tx.Bucket(schemaBucket).Bucket(classKey) - if b == nil { - return fmt.Errorf("class not found") - } - keyBuf := make([]byte, 1, 68) - if !existShards(b, shards, keyBuf) { - return fmt.Errorf("shard not found") - } - return appendShards(b, shards, keyBuf) - } - return r.db.Update(f) -} - -// DeleteShards of a specific class -// -// If the class or a shard does not exist then nothing is done and a nil error is returned -func (r *store) DeleteShards(_ context.Context, class string, shards []string) error { - classKey := encodeClassName(class) - f := func(tx *bolt.Tx) error { - b := tx.Bucket(schemaBucket).Bucket(classKey) - if b == nil { - return nil - } - return deleteShards(b, shards, make([]byte, 1, 68)) - } - return r.db.Update(f) -} - -// Load loads the complete schema from the persistent storage -func (r *store) Load(ctx context.Context) (ucs.State, error) { - state := ucs.NewState(32) - for data := range r.load(ctx) { - if data.Error != nil { - return state, data.Error - } - cls := models.Class{Class: string(data.Name)} - ss := sharding.State{} - - if err := json.Unmarshal(data.Metadata, &cls); err != nil { - return state, fmt.Errorf("unmarshal class %q", cls.Class) - } - if err := json.Unmarshal(data.ShardingState, &ss); err != nil { - return state, fmt.Errorf("unmarshal sharding state for class %q size %d", - cls.Class, len(data.ShardingState)) - } - if n := len(data.Shards); n > 0 { - ss.Physical = make(map[string]sharding.Physical, n) - } - for _, shard := range data.Shards { - phy := sharding.Physical{} - name := string(shard.Key) - if err := json.Unmarshal(shard.Value, &phy); err != nil { - return state, fmt.Errorf("unmarshal shard %q for class %q", name, cls.Class) - } - ss.Physical[name] = phy - } - state.ObjectSchema.Classes = append(state.ObjectSchema.Classes, &cls) - state.ShardingState[cls.Class] = &ss - } - return state, nil -} - -func (r *store) load(ctx context.Context) <-chan ucs.ClassPayload { - ch := make(chan ucs.ClassPayload, 1) - f := func(tx *bolt.Tx) (err error) { - root := tx.Bucket(schemaBucket) - rootCursor := root.Cursor() - for cls, _ := rootCursor.First(); cls != nil; { - if cls[0] != eTypeClass { - cls, _ = rootCursor.Next() - continue - } - if err := ctx.Err(); err != nil { - ch <- ucs.ClassPayload{Error: err} - return err - } - b := root.Bucket(cls) - if b == nil { - err := fmt.Errorf("class not found") - ch <- ucs.ClassPayload{Error: err} - return err - } - x := ucs.ClassPayload{ - Name: string(cls[1:]), - Shards: make([]ucs.KeyValuePair, 0, 32), - } - cursor := b.Cursor() - for key, value := cursor.First(); key != nil; { - if bytes.Equal(key, keyMetaClass) { - x.Metadata = value - } else if bytes.Equal(key, keyShardingState) { - x.ShardingState = value - } else { - x.Shards = append(x.Shards, ucs.KeyValuePair{Key: string(key[1:]), Value: value}) - } - key, value = cursor.Next() - } - ch <- x - cls, _ = rootCursor.Next() - } - return nil - } - go func() { - defer close(ch) - r.db.View(f) - }() - return ch -} - -// Save saves the complete schema to the persistent storage -func (r *store) Save(ctx context.Context, ss ucs.State) error { - if (ss.ObjectSchema == nil || len(ss.ObjectSchema.Classes) == 0) && - len(ss.ShardingState) == 0 { - return nil // empty schema nothing to store - } - - if ss.ObjectSchema == nil || - len(ss.ObjectSchema.Classes) == 0 || - len(ss.ShardingState) == 0 { - return fmt.Errorf("inconsistent schema: missing required fields") - } - - f := func(tx *bolt.Tx) error { - root := tx.Bucket(schemaBucket) - return r.saveAllTx(ctx, root, ss)(tx) - } - return r.db.Update(f) -} - -func (r *store) saveAllTx(ctx context.Context, root *bolt.Bucket, ss ucs.State) func(tx *bolt.Tx) error { - return func(tx *bolt.Tx) error { - rootCursor := root.Cursor() - for cls, _ := rootCursor.First(); cls != nil; { - if cls[0] == eTypeClass { - err := root.DeleteBucket(cls) - if err != nil && !errors.Is(err, bolt.ErrBucketNotFound) { - return err - } - } - cls, _ = rootCursor.Next() - } - for _, cls := range ss.ObjectSchema.Classes { - if err := ctx.Err(); err != nil { - return fmt.Errorf("context for class %q: %w", cls.Class, err) - } - sharding := ss.ShardingState[cls.Class] - payload, err := ucs.CreateClassPayload(cls, sharding) - if err != nil { - return fmt.Errorf("create payload for class %q: %w", cls.Class, err) - } - b, err := root.CreateBucket(encodeClassName(cls.Class)) - if err != nil { - return fmt.Errorf("create bucket for class %q: %w", cls.Class, err) - } - if err := r.updateClass(b, payload); err != nil { - return fmt.Errorf("update bucket %q: %w", cls.Class, err) - } - } - - return nil - } -} - -func saveConfig(root *bolt.Bucket, cfg config) error { - data, err := json.Marshal(&cfg) - if err != nil { - return fmt.Errorf("marshal config: %w", err) - } - if err := root.Put(keyConfig, data); err != nil { - return fmt.Errorf("write config: %w", err) - } - return nil -} - -func existShards(b *bolt.Bucket, shards []ucs.KeyValuePair, keyBuf []byte) bool { - keyBuf[0] = eTypeShard - for _, pair := range shards { - kLen := len(pair.Key) + 1 - keyBuf = append(keyBuf, pair.Key...) - if val := b.Get(keyBuf[:kLen]); val == nil { - return false - } - keyBuf = keyBuf[:1] - } - return true -} - -func appendShards(b *bolt.Bucket, shards []ucs.KeyValuePair, key []byte) error { - key[0] = eTypeShard - for _, pair := range shards { - kLen := len(pair.Key) + 1 - key = append(key, pair.Key...) - if err := b.Put(key[:kLen], pair.Value); err != nil { - return err - } - key = key[:1] - } - return nil -} - -func deleteShards(b *bolt.Bucket, shards []string, keyBuf []byte) error { - keyBuf[0] = eTypeShard - for _, name := range shards { - kLen := len(name) + 1 - keyBuf = append(keyBuf, name...) - if err := b.Delete(keyBuf[:kLen]); err != nil { - return err - } - keyBuf = keyBuf[:1] - } - return nil -} - -func encodeClassName(name string) []byte { - len := len(name) + 1 - buf := make([]byte, 1, len) - buf[0] = eTypeClass - buf = append(buf, name...) - return buf[:len] -} - -func copyFile(dst, src string) error { - data, err := os.ReadFile(src) - if err != nil { - return err - } - return os.WriteFile(dst, data, 0o644) -} - -// var _ = schemauc.Repo(&Repo{}) diff --git a/adapters/repos/schema/store_test.go b/adapters/repos/schema/store_test.go deleted file mode 100644 index 0e2199f38821f82f5ab565b756a9a9fce101d4d6..0000000000000000000000000000000000000000 --- a/adapters/repos/schema/store_test.go +++ /dev/null @@ -1,360 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package schema - -import ( - "context" - "encoding/json" - "fmt" - "testing" - - "github.com/sirupsen/logrus" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - - ucs "github.com/weaviate/weaviate/usecases/schema" - "github.com/weaviate/weaviate/usecases/sharding" -) - -func TestRepositoryMigrate(t *testing.T) { - var ( - ctx = context.Background() - logger, _ = test.NewNullLogger() - dirName = t.TempDir() - canceledCtx, cancel = context.WithCancel(ctx) - ) - cancel() - schema := ucs.NewState(3) - addClass(&schema, "C1", 0, 1, 0) - addClass(&schema, "C2", 0, 3, 3) - t.Run("SaveOldSchema", func(t *testing.T) { - repo, _ := newRepo(dirName, 0, logger) - defer repo.Close() - if err := repo.saveSchemaV1(schema); err != nil { - t.Fatalf("save all schema: %v", err) - } - }) - t.Run("LoadOldchema", func(t *testing.T) { - repo, err := newRepo(dirName, -1, logger) - if err != nil { - t.Fatalf("create new repo %v", err) - } - defer repo.Close() - - _, err = repo.Load(canceledCtx) - assert.ErrorIs(t, err, context.Canceled) - - state, err := repo.Load(ctx) - assert.Nil(t, err) - assert.Equal(t, schema, state) - }) - t.Run("LoadSchema", func(t *testing.T) { - repo, err := newRepo(dirName, -1, logger) - if err != nil { - t.Fatalf("create new repo %v", err) - } - defer repo.Close() - - state, err := repo.Load(ctx) - assert.Nil(t, err) - assert.Equal(t, schema, state) - }) - - t.Run("LoadSchemaWithHigherVersion", func(t *testing.T) { - _, err := newRepo(dirName, 1, logger) - assert.NotNil(t, err) - }) -} - -func TestRepositorySaveLoad(t *testing.T) { - var ( - ctx = context.Background() - canceledCtx, cancel = context.WithCancel(ctx) - logger, _ = test.NewNullLogger() - dirName = t.TempDir() - ) - cancel() - repo, err := newRepo(dirName, -1, logger) - if err != nil { - t.Fatalf("create new repo: %v", err) - } - // load empty schema - res, err := repo.Load(ctx) - if err != nil { - t.Fatalf("loading schema from empty file: %v", err) - } - if len(res.ShardingState) != 0 || len(res.ObjectSchema.Classes) != 0 { - t.Fatalf("expected empty schema got %v", res) - } - - // save and load non empty schema - schema := ucs.NewState(3) - addClass(&schema, "C1", 0, 1, 0) - addClass(&schema, "C2", 0, 3, 3) - err = repo.Save(canceledCtx, schema) - assert.ErrorIs(t, err, context.Canceled) - - if err = repo.Save(ctx, schema); err != nil { - t.Fatalf("save schema: %v", err) - } - if err = repo.Save(ctx, schema); err != nil { - t.Fatalf("save schema: %v", err) - } - - res, err = repo.Load(context.Background()) - if err != nil { - t.Fatalf("load schema: %v", err) - } - assert.Equal(t, schema, res) - - // delete class - deleteClass(&schema, "C2") - repo.DeleteClass(ctx, "C2") // second call to test impotency - if err := repo.DeleteClass(ctx, "C2"); err != nil { - t.Errorf("delete bucket: %v", err) - } - repo.asserEqualSchema(t, schema, "delete class") -} - -func TestRepositoryUpdateClass(t *testing.T) { - var ( - ctx = context.Background() - logger, _ = test.NewNullLogger() - dirName = t.TempDir() - ) - repo, err := newRepo(dirName, -1, logger) - if err != nil { - t.Fatalf("create new repo: %v", err) - } - - // save and load non empty schema - schema := ucs.NewState(3) - cls, ss := addClass(&schema, "C1", 0, 1, 0) - payload, err := ucs.CreateClassPayload(cls, ss) - assert.Nil(t, err) - if err := repo.NewClass(ctx, payload); err != nil { - t.Fatalf("create new class: %v", err) - } - if err := repo.NewClass(ctx, payload); err == nil { - t.Fatal("create new class: must fail since class already exits") - } - repo.asserEqualSchema(t, schema, "create class") - - // update class - deleteClass(&schema, "C1") - cls, ss = addClass(&schema, "C1", 0, 2, 1) - - payload, err = ucs.CreateClassPayload(cls, ss) - assert.Nil(t, err) - payload.Name = "C3" - if err := repo.UpdateClass(ctx, payload); err == nil { - t.Fatal("updating class by adding shards to non existing class must fail") - } - payload.Name = "C1" - if err := repo.UpdateClass(ctx, payload); err != nil { - t.Errorf("update class: %v", err) - } - repo.asserEqualSchema(t, schema, "update class") - - // overwrite class - deleteClass(&schema, "C1") - cls, ss = addClass(&schema, "C1", 2, 2, 3) - payload, err = ucs.CreateClassPayload(cls, ss) - assert.Nil(t, err) - payload.ReplaceShards = true - if err := repo.UpdateClass(ctx, payload); err != nil { - t.Errorf("update class: %v", err) - } - repo.asserEqualSchema(t, schema, "overwrite class") - - // delete class - deleteClass(&schema, "C1") - repo.DeleteClass(ctx, "C1") // second call to test impotency - if err := repo.DeleteClass(ctx, "C1"); err != nil { - t.Errorf("delete bucket: %v", err) - } - repo.asserEqualSchema(t, schema, "delete class") -} - -func TestRepositoryUpdateShards(t *testing.T) { - var ( - ctx = context.Background() - logger, _ = test.NewNullLogger() - dirName = t.TempDir() - ) - repo, err := newRepo(dirName, -1, logger) - if err != nil { - t.Fatalf("create new repo: %v", err) - } - - schema := ucs.NewState(2) - cls, ss := addClass(&schema, "C1", 0, 2, 1) - payload, err := ucs.CreateClassPayload(cls, ss) - assert.Nil(t, err) - if err := repo.NewClass(ctx, payload); err != nil { - t.Errorf("update class: %v", err) - } - repo.asserEqualSchema(t, schema, "update class") - - // add two shards - deleteClass(&schema, "C1") - _, ss = addClass(&schema, "C1", 0, 2, 5) - shards := serializeShards(ss.Physical) - if err := repo.NewShards(ctx, "C1", shards); err != nil { - t.Fatalf("add new shards: %v", err) - } - if err := repo.NewShards(ctx, "C3", shards); err == nil { - t.Fatal("add new shards to a non existing class must fail") - } - repo.asserEqualSchema(t, schema, "adding new shards") - - t.Run("fails updating non existent shards", func(t *testing.T) { - nonExistentShards := createShards(4, 2, models.TenantActivityStatusCOLD) - nonExistentShardPairs := serializeShards(nonExistentShards) - err := repo.UpdateShards(ctx, "C1", nonExistentShardPairs) - require.NotNil(t, err) - assert.ErrorContains(t, err, "shard not found") - }) - - existentShards := createShards(3, 2, models.TenantActivityStatusCOLD) - existentShardPairs := serializeShards(existentShards) - - t.Run("fails updating shards of non existent class", func(t *testing.T) { - err := repo.UpdateShards(ctx, "ClassNonExistent", existentShardPairs) - require.NotNil(t, err) - assert.ErrorContains(t, err, "class not found") - }) - t.Run("succeeds updating shards", func(t *testing.T) { - err := repo.UpdateShards(ctx, "C1", existentShardPairs) - require.Nil(t, err) - - replaceShards(ss, existentShards) - repo.asserEqualSchema(t, schema, "update shards") - }) - - xset := removeShards(ss, []int{0, 3, 4}) - if err := repo.DeleteShards(ctx, "C1", xset); err != nil { - t.Fatalf("delete shards: %v", err) - } - repo.asserEqualSchema(t, schema, "remove shards") - - if err := repo.DeleteShards(ctx, "C3", xset); err != nil { - t.Fatalf("delete shards from unknown class: %v", err) - } -} - -func createClass(name string, start, nProps, nShards int) (models.Class, sharding.State) { - cls := models.Class{Class: name} - for i := start; i < start+nProps; i++ { - prop := models.Property{ - Name: fmt.Sprintf("property-%d", i), - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - } - cls.Properties = append(cls.Properties, &prop) - } - ss := sharding.State{IndexID: name} - ss.Physical = createShards(start, nShards, models.TenantActivityStatusHOT) - - return cls, ss -} - -func createShards(start, nShards int, activityStatus string) map[string]sharding.Physical { - if nShards < 1 { - return nil - } - - shards := make(map[string]sharding.Physical, nShards) - for i := start; i < start+nShards; i++ { - name := fmt.Sprintf("shard-%d", i) - node := fmt.Sprintf("node-%d", i) - shards[name] = sharding.Physical{ - Name: name, - BelongsToNodes: []string{node}, - Status: activityStatus, - } - } - return shards -} - -func replaceShards(ss *sharding.State, shards map[string]sharding.Physical) { - for name, shard := range shards { - ss.Physical[name] = shard - } -} - -func removeShards(ss *sharding.State, shards []int) []string { - res := make([]string, len(shards)) - for i, j := range shards { - name := fmt.Sprintf("shard-%d", j) - delete(ss.Physical, name) - res[i] = name - } - return res -} - -func addClass(schema *ucs.State, name string, start, nProps, nShards int) (*models.Class, *sharding.State) { - cls, ss := createClass(name, start, nProps, nShards) - if schema.ObjectSchema == nil { - schema.ObjectSchema = &models.Schema{} - } - if schema.ShardingState == nil { - schema.ShardingState = make(map[string]*sharding.State) - } - schema.ObjectSchema.Classes = append(schema.ObjectSchema.Classes, &cls) - schema.ShardingState[name] = &ss - return &cls, &ss -} - -func deleteClass(schema *ucs.State, name string) { - idx := -1 - for i, cls := range schema.ObjectSchema.Classes { - if cls.Class == name { - idx = i - break - } - } - if idx == -1 { - return - } - schema.ObjectSchema.Classes = append(schema.ObjectSchema.Classes[:idx], schema.ObjectSchema.Classes[idx+1:]...) - delete(schema.ShardingState, name) -} - -func (r *store) asserEqualSchema(t *testing.T, expected ucs.State, msg string) { - t.Helper() - actual, err := r.Load(context.Background()) - if err != nil { - t.Fatalf("load schema: %s: %v", msg, err) - } - assert.Equal(t, expected, actual) -} - -func serializeShards(shards map[string]sharding.Physical) []ucs.KeyValuePair { - xs := make([]ucs.KeyValuePair, 0, len(shards)) - for k, v := range shards { - val, _ := json.Marshal(&v) - xs = append(xs, ucs.KeyValuePair{Key: k, Value: val}) - } - return xs -} - -func newRepo(homeDir string, version int, logger logrus.FieldLogger) (*store, error) { - r := NewStore(homeDir, logger) - if version > -1 { - r.version = version - } - return r, r.Open() -} diff --git a/adapters/repos/transactions/store.go b/adapters/repos/transactions/store.go deleted file mode 100644 index 313ef9fbf5d2b46773f03c403b05a9002be25665..0000000000000000000000000000000000000000 --- a/adapters/repos/transactions/store.go +++ /dev/null @@ -1,148 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package txstore - -import ( - "context" - "encoding/json" - "fmt" - "os" - "path" - - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/usecases/cluster" - "go.etcd.io/bbolt" -) - -var txBucket = []byte("transactions") - -type Store struct { - db *bbolt.DB - log logrus.FieldLogger - homeDir string - unmarshaller unmarshalFn -} - -func NewStore(homeDir string, logger logrus.FieldLogger) *Store { - return &Store{ - homeDir: homeDir, - log: logger, - } -} - -func (s *Store) SetUmarshalFn(fn unmarshalFn) { - s.unmarshaller = fn -} - -func (s *Store) Open() error { - if err := os.MkdirAll(s.homeDir, 0o777); err != nil { - return fmt.Errorf("create root directory %q: %w", s.homeDir, err) - } - - path := path.Join(s.homeDir, "tx.db") - boltDB, err := initBoltDB(path) - if err != nil { - return fmt.Errorf("init bolt_db: %w", err) - } - - s.db = boltDB - - return nil -} - -func (s *Store) StoreTx(ctx context.Context, tx *cluster.Transaction) error { - data, err := json.Marshal(txWrapper{ - ID: tx.ID, - Payload: tx.Payload, - Type: tx.Type, - }) - if err != nil { - return fmt.Errorf("marshal tx: %w", err) - } - - return s.db.Update(func(boltTx *bbolt.Tx) error { - b := boltTx.Bucket(txBucket) - return b.Put([]byte(tx.ID), data) - }) -} - -func (s *Store) DeleteTx(ctx context.Context, txId string) error { - return s.db.Update(func(boltTx *bbolt.Tx) error { - b := boltTx.Bucket(txBucket) - return b.Delete([]byte(txId)) - }) -} - -func (s *Store) IterateAll(ctx context.Context, - cb func(tx *cluster.Transaction), -) error { - return s.db.View(func(boltTx *bbolt.Tx) error { - b := boltTx.Bucket(txBucket) - c := b.Cursor() - for k, v := c.First(); k != nil; k, v = c.Next() { - var txWrap txWrapperRead - if err := json.Unmarshal(v, &txWrap); err != nil { - return err - } - - tx := cluster.Transaction{ - ID: txWrap.ID, - Type: txWrap.Type, - } - - pl, err := s.unmarshaller(tx.Type, txWrap.Payload) - if err != nil { - return err - } - - tx.Payload = pl - - cb(&tx) - - } - return nil - }) -} - -func (s *Store) Close() error { - return nil -} - -func initBoltDB(filePath string) (*bbolt.DB, error) { - db, err := bbolt.Open(filePath, 0o600, nil) - if err != nil { - return nil, fmt.Errorf("open %q: %w", filePath, err) - } - - root := func(tx *bbolt.Tx) error { - _, err := tx.CreateBucketIfNotExists(txBucket) - return err - } - - return db, db.Update(root) -} - -type txWrapper struct { - ID string `json:"id"` - Payload any `json:"payload"` - Type cluster.TransactionType `json:"type"` -} - -// delayed unmarshalling of the payload, so we can inject a specific -// marshaller -type txWrapperRead struct { - ID string `json:"id"` - Payload json.RawMessage `json:"payload"` - Type cluster.TransactionType `json:"type"` -} - -type unmarshalFn func(txType cluster.TransactionType, payload json.RawMessage) (any, error) diff --git a/ci/docker_report.md.tpl b/ci/docker_report.md.tpl deleted file mode 100644 index 0ff412f494e2dfa5aa6c63473a92a95d0b1a547e..0000000000000000000000000000000000000000 --- a/ci/docker_report.md.tpl +++ /dev/null @@ -1,41 +0,0 @@ -## Docker Preview Image :whale: - -A preview docker image for this branch is available with the following tag: - -``` -$PREVIEW_TAG -``` - -## Use at your own risk :warning: - -Preview builds make no promises about stability or feature completeness. Use them at your own risk. A preview build is not generated if tests failed, so they have at least passed the common test suite. They may or may not have been subjected to the asynchronous stress test and chaos pipelines. - -## Usage :newspaper: - -### Docker-compose - -You can obtain a [docker-compose.yaml here](https://weaviate.io/developers/weaviate/current/installation/docker-compose.html#configurator) and configure it to your liking. Then make sure to set `services.weaviate.image` to `$PREVIEW_TAG`. For example, like so: - -```yaml -services: - weaviate: - image: $PREVIEW_TAG -``` - -### Helm / Kubernetes - -To use this preview image with Helm/Kubernetes, set `image.tag` to `$TAG_ONLY` in your `values.yaml`. For example, like so: - -```yaml -image: - tag: $TAG_ONLY -``` - -### Chaos-Pipeline - -To use this build in a chaos pipeline, change the `WEAVIATE_VERSION` in `.github/workflows/tests.yaml` to `$TAG_ONLY`, for example, like so: - -```yaml -env: - WEAVIATE_VERSION: $TAG_ONLY -``` diff --git a/ci/generate_docker_report.sh b/ci/generate_docker_report.sh deleted file mode 100644 index 282d9ddc2b6d6e707fb806c8e3ec19fb34556d11..0000000000000000000000000000000000000000 --- a/ci/generate_docker_report.sh +++ /dev/null @@ -1,15 +0,0 @@ -set -e - -cd "${0%/*}" - -function generate_report() { - echo "$PREVIEW_TAG" - if [ -z "$PREVIEW_TAG" ]; then - return - fi - - export TAG_ONLY="$(echo "$PREVIEW_TAG" | cut -d ':' -f 2)" - envsubst < docker_report.md.tpl >> "$GITHUB_STEP_SUMMARY" -} - -generate_report diff --git a/ci/push_docker.sh b/ci/push_docker.sh deleted file mode 100644 index be4c694bc76994032edeb090b89cadc24f781e8f..0000000000000000000000000000000000000000 --- a/ci/push_docker.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash - -DOCKER_REPO="semitechnologies/weaviate" - -function release() { - # for multi-platform build - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - docker buildx create --use - - tag_latest="${DOCKER_REPO}:latest" - tag_exact= - tag_preview= - - git_hash=$(echo "$GITHUB_SHA" | cut -c1-7) - - weaviate_version="$(jq -r '.info.version' < openapi-specs/schema.json)" - if [ "$GITHUB_REF_NAME" = "master" ]; then - tag_exact="${DOCKER_REPO}:${weaviate_version}-${git_hash}" - elif [ "$GITHUB_REF_TYPE" == "tag" ]; then - if [ "$GITHUB_REF_NAME" != "v$weaviate_version" ]; then - echo "The release tag ($GITHUB_REF_NAME) and Weaviate version (v$weaviate_version) are not equal! Can't release." - return 1 - fi - tag_exact="${DOCKER_REPO}:${weaviate_version}" - else - pr_title="$(echo -n "$PR_TITLE" | tr '[:upper:]' '[:lower:]' | tr -c -s '[:alnum:]' '-' | sed 's/-$//g')" - tag_preview="${DOCKER_REPO}:preview-${pr_title}-${git_hash}" - fi - - args=("--build-arg=GITHASH=$git_hash" "--platform=linux/amd64,linux/arm64" "--target=weaviate" "-t=$tag_latest" "--push") - if [ -n "$tag_exact" ]; then - # exact tag on master - args+=("-t=$tag_exact") - fi - if [ -n "$tag_preview" ]; then - # preview tag on PR builds - args+=("-t=$tag_preview") - fi - - docker buildx build "${args[@]}" . - - if [ -n "$tag_preview" ]; then - echo "PREVIEW_TAG=$tag_preview" >> "$GITHUB_OUTPUT" - elif [ -n "$tag_exact" ]; then - echo "PREVIEW_TAG=$tag_exact" >> "$GITHUB_OUTPUT" - fi -} - -release diff --git a/client/backups/backups_client.go b/client/backups/backups_client.go deleted file mode 100644 index 4ae0916fc5e1f387cbdacadfd707b04718a91fd8..0000000000000000000000000000000000000000 --- a/client/backups/backups_client.go +++ /dev/null @@ -1,214 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package backups - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" -) - -// New creates a new backups API client. -func New(transport runtime.ClientTransport, formats strfmt.Registry) ClientService { - return &Client{transport: transport, formats: formats} -} - -/* -Client for backups API -*/ -type Client struct { - transport runtime.ClientTransport - formats strfmt.Registry -} - -// ClientOption is the option for Client methods -type ClientOption func(*runtime.ClientOperation) - -// ClientService is the interface for Client methods -type ClientService interface { - BackupsCreate(params *BackupsCreateParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*BackupsCreateOK, error) - - BackupsCreateStatus(params *BackupsCreateStatusParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*BackupsCreateStatusOK, error) - - BackupsRestore(params *BackupsRestoreParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*BackupsRestoreOK, error) - - BackupsRestoreStatus(params *BackupsRestoreStatusParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*BackupsRestoreStatusOK, error) - - SetTransport(transport runtime.ClientTransport) -} - -/* -BackupsCreate Starts a process of creating a backup for a set of classes -*/ -func (a *Client) BackupsCreate(params *BackupsCreateParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*BackupsCreateOK, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewBackupsCreateParams() - } - op := &runtime.ClientOperation{ - ID: "backups.create", - Method: "POST", - PathPattern: "/backups/{backend}", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &BackupsCreateReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*BackupsCreateOK) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for backups.create: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -/* -BackupsCreateStatus Returns status of backup creation attempt for a set of classes -*/ -func (a *Client) BackupsCreateStatus(params *BackupsCreateStatusParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*BackupsCreateStatusOK, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewBackupsCreateStatusParams() - } - op := &runtime.ClientOperation{ - ID: "backups.create.status", - Method: "GET", - PathPattern: "/backups/{backend}/{id}", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &BackupsCreateStatusReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*BackupsCreateStatusOK) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for backups.create.status: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -/* -BackupsRestore Starts a process of restoring a backup for a set of classes -*/ -func (a *Client) BackupsRestore(params *BackupsRestoreParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*BackupsRestoreOK, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewBackupsRestoreParams() - } - op := &runtime.ClientOperation{ - ID: "backups.restore", - Method: "POST", - PathPattern: "/backups/{backend}/{id}/restore", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &BackupsRestoreReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*BackupsRestoreOK) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for backups.restore: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -/* -BackupsRestoreStatus Returns status of a backup restoration attempt for a set of classes -*/ -func (a *Client) BackupsRestoreStatus(params *BackupsRestoreStatusParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*BackupsRestoreStatusOK, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewBackupsRestoreStatusParams() - } - op := &runtime.ClientOperation{ - ID: "backups.restore.status", - Method: "GET", - PathPattern: "/backups/{backend}/{id}/restore", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &BackupsRestoreStatusReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*BackupsRestoreStatusOK) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for backups.restore.status: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -// SetTransport changes the transport on the client -func (a *Client) SetTransport(transport runtime.ClientTransport) { - a.transport = transport -} diff --git a/client/backups/backups_create_parameters.go b/client/backups/backups_create_parameters.go deleted file mode 100644 index 6b13028d763119a1a452cccafc3f32a38521d534..0000000000000000000000000000000000000000 --- a/client/backups/backups_create_parameters.go +++ /dev/null @@ -1,183 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package backups - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewBackupsCreateParams creates a new BackupsCreateParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewBackupsCreateParams() *BackupsCreateParams { - return &BackupsCreateParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewBackupsCreateParamsWithTimeout creates a new BackupsCreateParams object -// with the ability to set a timeout on a request. -func NewBackupsCreateParamsWithTimeout(timeout time.Duration) *BackupsCreateParams { - return &BackupsCreateParams{ - timeout: timeout, - } -} - -// NewBackupsCreateParamsWithContext creates a new BackupsCreateParams object -// with the ability to set a context for a request. -func NewBackupsCreateParamsWithContext(ctx context.Context) *BackupsCreateParams { - return &BackupsCreateParams{ - Context: ctx, - } -} - -// NewBackupsCreateParamsWithHTTPClient creates a new BackupsCreateParams object -// with the ability to set a custom HTTPClient for a request. -func NewBackupsCreateParamsWithHTTPClient(client *http.Client) *BackupsCreateParams { - return &BackupsCreateParams{ - HTTPClient: client, - } -} - -/* -BackupsCreateParams contains all the parameters to send to the API endpoint - - for the backups create operation. - - Typically these are written to a http.Request. -*/ -type BackupsCreateParams struct { - - /* Backend. - - Backup backend name e.g. filesystem, gcs, s3. - */ - Backend string - - // Body. - Body *models.BackupCreateRequest - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the backups create params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *BackupsCreateParams) WithDefaults() *BackupsCreateParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the backups create params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *BackupsCreateParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the backups create params -func (o *BackupsCreateParams) WithTimeout(timeout time.Duration) *BackupsCreateParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the backups create params -func (o *BackupsCreateParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the backups create params -func (o *BackupsCreateParams) WithContext(ctx context.Context) *BackupsCreateParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the backups create params -func (o *BackupsCreateParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the backups create params -func (o *BackupsCreateParams) WithHTTPClient(client *http.Client) *BackupsCreateParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the backups create params -func (o *BackupsCreateParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithBackend adds the backend to the backups create params -func (o *BackupsCreateParams) WithBackend(backend string) *BackupsCreateParams { - o.SetBackend(backend) - return o -} - -// SetBackend adds the backend to the backups create params -func (o *BackupsCreateParams) SetBackend(backend string) { - o.Backend = backend -} - -// WithBody adds the body to the backups create params -func (o *BackupsCreateParams) WithBody(body *models.BackupCreateRequest) *BackupsCreateParams { - o.SetBody(body) - return o -} - -// SetBody adds the body to the backups create params -func (o *BackupsCreateParams) SetBody(body *models.BackupCreateRequest) { - o.Body = body -} - -// WriteToRequest writes these params to a swagger request -func (o *BackupsCreateParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - - // path param backend - if err := r.SetPathParam("backend", o.Backend); err != nil { - return err - } - if o.Body != nil { - if err := r.SetBodyParam(o.Body); err != nil { - return err - } - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/backups/backups_create_responses.go b/client/backups/backups_create_responses.go deleted file mode 100644 index 1e4e9ec4a7b7bcff41a817a3a0d3afa839466758..0000000000000000000000000000000000000000 --- a/client/backups/backups_create_responses.go +++ /dev/null @@ -1,398 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package backups - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// BackupsCreateReader is a Reader for the BackupsCreate structure. -type BackupsCreateReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *BackupsCreateReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 200: - result := NewBackupsCreateOK() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 401: - result := NewBackupsCreateUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewBackupsCreateForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 422: - result := NewBackupsCreateUnprocessableEntity() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewBackupsCreateInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewBackupsCreateOK creates a BackupsCreateOK with default headers values -func NewBackupsCreateOK() *BackupsCreateOK { - return &BackupsCreateOK{} -} - -/* -BackupsCreateOK describes a response with status code 200, with default header values. - -Backup create process successfully started. -*/ -type BackupsCreateOK struct { - Payload *models.BackupCreateResponse -} - -// IsSuccess returns true when this backups create o k response has a 2xx status code -func (o *BackupsCreateOK) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this backups create o k response has a 3xx status code -func (o *BackupsCreateOK) IsRedirect() bool { - return false -} - -// IsClientError returns true when this backups create o k response has a 4xx status code -func (o *BackupsCreateOK) IsClientError() bool { - return false -} - -// IsServerError returns true when this backups create o k response has a 5xx status code -func (o *BackupsCreateOK) IsServerError() bool { - return false -} - -// IsCode returns true when this backups create o k response a status code equal to that given -func (o *BackupsCreateOK) IsCode(code int) bool { - return code == 200 -} - -// Code gets the status code for the backups create o k response -func (o *BackupsCreateOK) Code() int { - return 200 -} - -func (o *BackupsCreateOK) Error() string { - return fmt.Sprintf("[POST /backups/{backend}][%d] backupsCreateOK %+v", 200, o.Payload) -} - -func (o *BackupsCreateOK) String() string { - return fmt.Sprintf("[POST /backups/{backend}][%d] backupsCreateOK %+v", 200, o.Payload) -} - -func (o *BackupsCreateOK) GetPayload() *models.BackupCreateResponse { - return o.Payload -} - -func (o *BackupsCreateOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.BackupCreateResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewBackupsCreateUnauthorized creates a BackupsCreateUnauthorized with default headers values -func NewBackupsCreateUnauthorized() *BackupsCreateUnauthorized { - return &BackupsCreateUnauthorized{} -} - -/* -BackupsCreateUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type BackupsCreateUnauthorized struct { -} - -// IsSuccess returns true when this backups create unauthorized response has a 2xx status code -func (o *BackupsCreateUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this backups create unauthorized response has a 3xx status code -func (o *BackupsCreateUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this backups create unauthorized response has a 4xx status code -func (o *BackupsCreateUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this backups create unauthorized response has a 5xx status code -func (o *BackupsCreateUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this backups create unauthorized response a status code equal to that given -func (o *BackupsCreateUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the backups create unauthorized response -func (o *BackupsCreateUnauthorized) Code() int { - return 401 -} - -func (o *BackupsCreateUnauthorized) Error() string { - return fmt.Sprintf("[POST /backups/{backend}][%d] backupsCreateUnauthorized ", 401) -} - -func (o *BackupsCreateUnauthorized) String() string { - return fmt.Sprintf("[POST /backups/{backend}][%d] backupsCreateUnauthorized ", 401) -} - -func (o *BackupsCreateUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewBackupsCreateForbidden creates a BackupsCreateForbidden with default headers values -func NewBackupsCreateForbidden() *BackupsCreateForbidden { - return &BackupsCreateForbidden{} -} - -/* -BackupsCreateForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type BackupsCreateForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this backups create forbidden response has a 2xx status code -func (o *BackupsCreateForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this backups create forbidden response has a 3xx status code -func (o *BackupsCreateForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this backups create forbidden response has a 4xx status code -func (o *BackupsCreateForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this backups create forbidden response has a 5xx status code -func (o *BackupsCreateForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this backups create forbidden response a status code equal to that given -func (o *BackupsCreateForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the backups create forbidden response -func (o *BackupsCreateForbidden) Code() int { - return 403 -} - -func (o *BackupsCreateForbidden) Error() string { - return fmt.Sprintf("[POST /backups/{backend}][%d] backupsCreateForbidden %+v", 403, o.Payload) -} - -func (o *BackupsCreateForbidden) String() string { - return fmt.Sprintf("[POST /backups/{backend}][%d] backupsCreateForbidden %+v", 403, o.Payload) -} - -func (o *BackupsCreateForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *BackupsCreateForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewBackupsCreateUnprocessableEntity creates a BackupsCreateUnprocessableEntity with default headers values -func NewBackupsCreateUnprocessableEntity() *BackupsCreateUnprocessableEntity { - return &BackupsCreateUnprocessableEntity{} -} - -/* -BackupsCreateUnprocessableEntity describes a response with status code 422, with default header values. - -Invalid backup creation attempt. -*/ -type BackupsCreateUnprocessableEntity struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this backups create unprocessable entity response has a 2xx status code -func (o *BackupsCreateUnprocessableEntity) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this backups create unprocessable entity response has a 3xx status code -func (o *BackupsCreateUnprocessableEntity) IsRedirect() bool { - return false -} - -// IsClientError returns true when this backups create unprocessable entity response has a 4xx status code -func (o *BackupsCreateUnprocessableEntity) IsClientError() bool { - return true -} - -// IsServerError returns true when this backups create unprocessable entity response has a 5xx status code -func (o *BackupsCreateUnprocessableEntity) IsServerError() bool { - return false -} - -// IsCode returns true when this backups create unprocessable entity response a status code equal to that given -func (o *BackupsCreateUnprocessableEntity) IsCode(code int) bool { - return code == 422 -} - -// Code gets the status code for the backups create unprocessable entity response -func (o *BackupsCreateUnprocessableEntity) Code() int { - return 422 -} - -func (o *BackupsCreateUnprocessableEntity) Error() string { - return fmt.Sprintf("[POST /backups/{backend}][%d] backupsCreateUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *BackupsCreateUnprocessableEntity) String() string { - return fmt.Sprintf("[POST /backups/{backend}][%d] backupsCreateUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *BackupsCreateUnprocessableEntity) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *BackupsCreateUnprocessableEntity) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewBackupsCreateInternalServerError creates a BackupsCreateInternalServerError with default headers values -func NewBackupsCreateInternalServerError() *BackupsCreateInternalServerError { - return &BackupsCreateInternalServerError{} -} - -/* -BackupsCreateInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type BackupsCreateInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this backups create internal server error response has a 2xx status code -func (o *BackupsCreateInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this backups create internal server error response has a 3xx status code -func (o *BackupsCreateInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this backups create internal server error response has a 4xx status code -func (o *BackupsCreateInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this backups create internal server error response has a 5xx status code -func (o *BackupsCreateInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this backups create internal server error response a status code equal to that given -func (o *BackupsCreateInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the backups create internal server error response -func (o *BackupsCreateInternalServerError) Code() int { - return 500 -} - -func (o *BackupsCreateInternalServerError) Error() string { - return fmt.Sprintf("[POST /backups/{backend}][%d] backupsCreateInternalServerError %+v", 500, o.Payload) -} - -func (o *BackupsCreateInternalServerError) String() string { - return fmt.Sprintf("[POST /backups/{backend}][%d] backupsCreateInternalServerError %+v", 500, o.Payload) -} - -func (o *BackupsCreateInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *BackupsCreateInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/backups/backups_create_status_parameters.go b/client/backups/backups_create_status_parameters.go deleted file mode 100644 index 5ef1ffbd511f30925026d22c58b3ac6c3c124fd8..0000000000000000000000000000000000000000 --- a/client/backups/backups_create_status_parameters.go +++ /dev/null @@ -1,184 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package backups - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" -) - -// NewBackupsCreateStatusParams creates a new BackupsCreateStatusParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewBackupsCreateStatusParams() *BackupsCreateStatusParams { - return &BackupsCreateStatusParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewBackupsCreateStatusParamsWithTimeout creates a new BackupsCreateStatusParams object -// with the ability to set a timeout on a request. -func NewBackupsCreateStatusParamsWithTimeout(timeout time.Duration) *BackupsCreateStatusParams { - return &BackupsCreateStatusParams{ - timeout: timeout, - } -} - -// NewBackupsCreateStatusParamsWithContext creates a new BackupsCreateStatusParams object -// with the ability to set a context for a request. -func NewBackupsCreateStatusParamsWithContext(ctx context.Context) *BackupsCreateStatusParams { - return &BackupsCreateStatusParams{ - Context: ctx, - } -} - -// NewBackupsCreateStatusParamsWithHTTPClient creates a new BackupsCreateStatusParams object -// with the ability to set a custom HTTPClient for a request. -func NewBackupsCreateStatusParamsWithHTTPClient(client *http.Client) *BackupsCreateStatusParams { - return &BackupsCreateStatusParams{ - HTTPClient: client, - } -} - -/* -BackupsCreateStatusParams contains all the parameters to send to the API endpoint - - for the backups create status operation. - - Typically these are written to a http.Request. -*/ -type BackupsCreateStatusParams struct { - - /* Backend. - - Backup backend name e.g. filesystem, gcs, s3. - */ - Backend string - - /* ID. - - The ID of a backup. Must be URL-safe and work as a filesystem path, only lowercase, numbers, underscore, minus characters allowed. - */ - ID string - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the backups create status params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *BackupsCreateStatusParams) WithDefaults() *BackupsCreateStatusParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the backups create status params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *BackupsCreateStatusParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the backups create status params -func (o *BackupsCreateStatusParams) WithTimeout(timeout time.Duration) *BackupsCreateStatusParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the backups create status params -func (o *BackupsCreateStatusParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the backups create status params -func (o *BackupsCreateStatusParams) WithContext(ctx context.Context) *BackupsCreateStatusParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the backups create status params -func (o *BackupsCreateStatusParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the backups create status params -func (o *BackupsCreateStatusParams) WithHTTPClient(client *http.Client) *BackupsCreateStatusParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the backups create status params -func (o *BackupsCreateStatusParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithBackend adds the backend to the backups create status params -func (o *BackupsCreateStatusParams) WithBackend(backend string) *BackupsCreateStatusParams { - o.SetBackend(backend) - return o -} - -// SetBackend adds the backend to the backups create status params -func (o *BackupsCreateStatusParams) SetBackend(backend string) { - o.Backend = backend -} - -// WithID adds the id to the backups create status params -func (o *BackupsCreateStatusParams) WithID(id string) *BackupsCreateStatusParams { - o.SetID(id) - return o -} - -// SetID adds the id to the backups create status params -func (o *BackupsCreateStatusParams) SetID(id string) { - o.ID = id -} - -// WriteToRequest writes these params to a swagger request -func (o *BackupsCreateStatusParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - - // path param backend - if err := r.SetPathParam("backend", o.Backend); err != nil { - return err - } - - // path param id - if err := r.SetPathParam("id", o.ID); err != nil { - return err - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/backups/backups_create_status_responses.go b/client/backups/backups_create_status_responses.go deleted file mode 100644 index 43bc0f473aa70eea3aa9eba76f6578c1886ff3cc..0000000000000000000000000000000000000000 --- a/client/backups/backups_create_status_responses.go +++ /dev/null @@ -1,472 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package backups - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// BackupsCreateStatusReader is a Reader for the BackupsCreateStatus structure. -type BackupsCreateStatusReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *BackupsCreateStatusReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 200: - result := NewBackupsCreateStatusOK() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 401: - result := NewBackupsCreateStatusUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewBackupsCreateStatusForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 404: - result := NewBackupsCreateStatusNotFound() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 422: - result := NewBackupsCreateStatusUnprocessableEntity() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewBackupsCreateStatusInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewBackupsCreateStatusOK creates a BackupsCreateStatusOK with default headers values -func NewBackupsCreateStatusOK() *BackupsCreateStatusOK { - return &BackupsCreateStatusOK{} -} - -/* -BackupsCreateStatusOK describes a response with status code 200, with default header values. - -Backup creation status successfully returned -*/ -type BackupsCreateStatusOK struct { - Payload *models.BackupCreateStatusResponse -} - -// IsSuccess returns true when this backups create status o k response has a 2xx status code -func (o *BackupsCreateStatusOK) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this backups create status o k response has a 3xx status code -func (o *BackupsCreateStatusOK) IsRedirect() bool { - return false -} - -// IsClientError returns true when this backups create status o k response has a 4xx status code -func (o *BackupsCreateStatusOK) IsClientError() bool { - return false -} - -// IsServerError returns true when this backups create status o k response has a 5xx status code -func (o *BackupsCreateStatusOK) IsServerError() bool { - return false -} - -// IsCode returns true when this backups create status o k response a status code equal to that given -func (o *BackupsCreateStatusOK) IsCode(code int) bool { - return code == 200 -} - -// Code gets the status code for the backups create status o k response -func (o *BackupsCreateStatusOK) Code() int { - return 200 -} - -func (o *BackupsCreateStatusOK) Error() string { - return fmt.Sprintf("[GET /backups/{backend}/{id}][%d] backupsCreateStatusOK %+v", 200, o.Payload) -} - -func (o *BackupsCreateStatusOK) String() string { - return fmt.Sprintf("[GET /backups/{backend}/{id}][%d] backupsCreateStatusOK %+v", 200, o.Payload) -} - -func (o *BackupsCreateStatusOK) GetPayload() *models.BackupCreateStatusResponse { - return o.Payload -} - -func (o *BackupsCreateStatusOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.BackupCreateStatusResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewBackupsCreateStatusUnauthorized creates a BackupsCreateStatusUnauthorized with default headers values -func NewBackupsCreateStatusUnauthorized() *BackupsCreateStatusUnauthorized { - return &BackupsCreateStatusUnauthorized{} -} - -/* -BackupsCreateStatusUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type BackupsCreateStatusUnauthorized struct { -} - -// IsSuccess returns true when this backups create status unauthorized response has a 2xx status code -func (o *BackupsCreateStatusUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this backups create status unauthorized response has a 3xx status code -func (o *BackupsCreateStatusUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this backups create status unauthorized response has a 4xx status code -func (o *BackupsCreateStatusUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this backups create status unauthorized response has a 5xx status code -func (o *BackupsCreateStatusUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this backups create status unauthorized response a status code equal to that given -func (o *BackupsCreateStatusUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the backups create status unauthorized response -func (o *BackupsCreateStatusUnauthorized) Code() int { - return 401 -} - -func (o *BackupsCreateStatusUnauthorized) Error() string { - return fmt.Sprintf("[GET /backups/{backend}/{id}][%d] backupsCreateStatusUnauthorized ", 401) -} - -func (o *BackupsCreateStatusUnauthorized) String() string { - return fmt.Sprintf("[GET /backups/{backend}/{id}][%d] backupsCreateStatusUnauthorized ", 401) -} - -func (o *BackupsCreateStatusUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewBackupsCreateStatusForbidden creates a BackupsCreateStatusForbidden with default headers values -func NewBackupsCreateStatusForbidden() *BackupsCreateStatusForbidden { - return &BackupsCreateStatusForbidden{} -} - -/* -BackupsCreateStatusForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type BackupsCreateStatusForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this backups create status forbidden response has a 2xx status code -func (o *BackupsCreateStatusForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this backups create status forbidden response has a 3xx status code -func (o *BackupsCreateStatusForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this backups create status forbidden response has a 4xx status code -func (o *BackupsCreateStatusForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this backups create status forbidden response has a 5xx status code -func (o *BackupsCreateStatusForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this backups create status forbidden response a status code equal to that given -func (o *BackupsCreateStatusForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the backups create status forbidden response -func (o *BackupsCreateStatusForbidden) Code() int { - return 403 -} - -func (o *BackupsCreateStatusForbidden) Error() string { - return fmt.Sprintf("[GET /backups/{backend}/{id}][%d] backupsCreateStatusForbidden %+v", 403, o.Payload) -} - -func (o *BackupsCreateStatusForbidden) String() string { - return fmt.Sprintf("[GET /backups/{backend}/{id}][%d] backupsCreateStatusForbidden %+v", 403, o.Payload) -} - -func (o *BackupsCreateStatusForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *BackupsCreateStatusForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewBackupsCreateStatusNotFound creates a BackupsCreateStatusNotFound with default headers values -func NewBackupsCreateStatusNotFound() *BackupsCreateStatusNotFound { - return &BackupsCreateStatusNotFound{} -} - -/* -BackupsCreateStatusNotFound describes a response with status code 404, with default header values. - -Not Found - Backup does not exist -*/ -type BackupsCreateStatusNotFound struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this backups create status not found response has a 2xx status code -func (o *BackupsCreateStatusNotFound) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this backups create status not found response has a 3xx status code -func (o *BackupsCreateStatusNotFound) IsRedirect() bool { - return false -} - -// IsClientError returns true when this backups create status not found response has a 4xx status code -func (o *BackupsCreateStatusNotFound) IsClientError() bool { - return true -} - -// IsServerError returns true when this backups create status not found response has a 5xx status code -func (o *BackupsCreateStatusNotFound) IsServerError() bool { - return false -} - -// IsCode returns true when this backups create status not found response a status code equal to that given -func (o *BackupsCreateStatusNotFound) IsCode(code int) bool { - return code == 404 -} - -// Code gets the status code for the backups create status not found response -func (o *BackupsCreateStatusNotFound) Code() int { - return 404 -} - -func (o *BackupsCreateStatusNotFound) Error() string { - return fmt.Sprintf("[GET /backups/{backend}/{id}][%d] backupsCreateStatusNotFound %+v", 404, o.Payload) -} - -func (o *BackupsCreateStatusNotFound) String() string { - return fmt.Sprintf("[GET /backups/{backend}/{id}][%d] backupsCreateStatusNotFound %+v", 404, o.Payload) -} - -func (o *BackupsCreateStatusNotFound) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *BackupsCreateStatusNotFound) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewBackupsCreateStatusUnprocessableEntity creates a BackupsCreateStatusUnprocessableEntity with default headers values -func NewBackupsCreateStatusUnprocessableEntity() *BackupsCreateStatusUnprocessableEntity { - return &BackupsCreateStatusUnprocessableEntity{} -} - -/* -BackupsCreateStatusUnprocessableEntity describes a response with status code 422, with default header values. - -Invalid backup restoration status attempt. -*/ -type BackupsCreateStatusUnprocessableEntity struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this backups create status unprocessable entity response has a 2xx status code -func (o *BackupsCreateStatusUnprocessableEntity) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this backups create status unprocessable entity response has a 3xx status code -func (o *BackupsCreateStatusUnprocessableEntity) IsRedirect() bool { - return false -} - -// IsClientError returns true when this backups create status unprocessable entity response has a 4xx status code -func (o *BackupsCreateStatusUnprocessableEntity) IsClientError() bool { - return true -} - -// IsServerError returns true when this backups create status unprocessable entity response has a 5xx status code -func (o *BackupsCreateStatusUnprocessableEntity) IsServerError() bool { - return false -} - -// IsCode returns true when this backups create status unprocessable entity response a status code equal to that given -func (o *BackupsCreateStatusUnprocessableEntity) IsCode(code int) bool { - return code == 422 -} - -// Code gets the status code for the backups create status unprocessable entity response -func (o *BackupsCreateStatusUnprocessableEntity) Code() int { - return 422 -} - -func (o *BackupsCreateStatusUnprocessableEntity) Error() string { - return fmt.Sprintf("[GET /backups/{backend}/{id}][%d] backupsCreateStatusUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *BackupsCreateStatusUnprocessableEntity) String() string { - return fmt.Sprintf("[GET /backups/{backend}/{id}][%d] backupsCreateStatusUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *BackupsCreateStatusUnprocessableEntity) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *BackupsCreateStatusUnprocessableEntity) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewBackupsCreateStatusInternalServerError creates a BackupsCreateStatusInternalServerError with default headers values -func NewBackupsCreateStatusInternalServerError() *BackupsCreateStatusInternalServerError { - return &BackupsCreateStatusInternalServerError{} -} - -/* -BackupsCreateStatusInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type BackupsCreateStatusInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this backups create status internal server error response has a 2xx status code -func (o *BackupsCreateStatusInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this backups create status internal server error response has a 3xx status code -func (o *BackupsCreateStatusInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this backups create status internal server error response has a 4xx status code -func (o *BackupsCreateStatusInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this backups create status internal server error response has a 5xx status code -func (o *BackupsCreateStatusInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this backups create status internal server error response a status code equal to that given -func (o *BackupsCreateStatusInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the backups create status internal server error response -func (o *BackupsCreateStatusInternalServerError) Code() int { - return 500 -} - -func (o *BackupsCreateStatusInternalServerError) Error() string { - return fmt.Sprintf("[GET /backups/{backend}/{id}][%d] backupsCreateStatusInternalServerError %+v", 500, o.Payload) -} - -func (o *BackupsCreateStatusInternalServerError) String() string { - return fmt.Sprintf("[GET /backups/{backend}/{id}][%d] backupsCreateStatusInternalServerError %+v", 500, o.Payload) -} - -func (o *BackupsCreateStatusInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *BackupsCreateStatusInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/backups/backups_restore_parameters.go b/client/backups/backups_restore_parameters.go deleted file mode 100644 index 7704c22fa03563eb53ab9f9c79d54850e4971c0e..0000000000000000000000000000000000000000 --- a/client/backups/backups_restore_parameters.go +++ /dev/null @@ -1,205 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package backups - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewBackupsRestoreParams creates a new BackupsRestoreParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewBackupsRestoreParams() *BackupsRestoreParams { - return &BackupsRestoreParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewBackupsRestoreParamsWithTimeout creates a new BackupsRestoreParams object -// with the ability to set a timeout on a request. -func NewBackupsRestoreParamsWithTimeout(timeout time.Duration) *BackupsRestoreParams { - return &BackupsRestoreParams{ - timeout: timeout, - } -} - -// NewBackupsRestoreParamsWithContext creates a new BackupsRestoreParams object -// with the ability to set a context for a request. -func NewBackupsRestoreParamsWithContext(ctx context.Context) *BackupsRestoreParams { - return &BackupsRestoreParams{ - Context: ctx, - } -} - -// NewBackupsRestoreParamsWithHTTPClient creates a new BackupsRestoreParams object -// with the ability to set a custom HTTPClient for a request. -func NewBackupsRestoreParamsWithHTTPClient(client *http.Client) *BackupsRestoreParams { - return &BackupsRestoreParams{ - HTTPClient: client, - } -} - -/* -BackupsRestoreParams contains all the parameters to send to the API endpoint - - for the backups restore operation. - - Typically these are written to a http.Request. -*/ -type BackupsRestoreParams struct { - - /* Backend. - - Backup backend name e.g. filesystem, gcs, s3. - */ - Backend string - - // Body. - Body *models.BackupRestoreRequest - - /* ID. - - The ID of a backup. Must be URL-safe and work as a filesystem path, only lowercase, numbers, underscore, minus characters allowed. - */ - ID string - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the backups restore params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *BackupsRestoreParams) WithDefaults() *BackupsRestoreParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the backups restore params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *BackupsRestoreParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the backups restore params -func (o *BackupsRestoreParams) WithTimeout(timeout time.Duration) *BackupsRestoreParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the backups restore params -func (o *BackupsRestoreParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the backups restore params -func (o *BackupsRestoreParams) WithContext(ctx context.Context) *BackupsRestoreParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the backups restore params -func (o *BackupsRestoreParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the backups restore params -func (o *BackupsRestoreParams) WithHTTPClient(client *http.Client) *BackupsRestoreParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the backups restore params -func (o *BackupsRestoreParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithBackend adds the backend to the backups restore params -func (o *BackupsRestoreParams) WithBackend(backend string) *BackupsRestoreParams { - o.SetBackend(backend) - return o -} - -// SetBackend adds the backend to the backups restore params -func (o *BackupsRestoreParams) SetBackend(backend string) { - o.Backend = backend -} - -// WithBody adds the body to the backups restore params -func (o *BackupsRestoreParams) WithBody(body *models.BackupRestoreRequest) *BackupsRestoreParams { - o.SetBody(body) - return o -} - -// SetBody adds the body to the backups restore params -func (o *BackupsRestoreParams) SetBody(body *models.BackupRestoreRequest) { - o.Body = body -} - -// WithID adds the id to the backups restore params -func (o *BackupsRestoreParams) WithID(id string) *BackupsRestoreParams { - o.SetID(id) - return o -} - -// SetID adds the id to the backups restore params -func (o *BackupsRestoreParams) SetID(id string) { - o.ID = id -} - -// WriteToRequest writes these params to a swagger request -func (o *BackupsRestoreParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - - // path param backend - if err := r.SetPathParam("backend", o.Backend); err != nil { - return err - } - if o.Body != nil { - if err := r.SetBodyParam(o.Body); err != nil { - return err - } - } - - // path param id - if err := r.SetPathParam("id", o.ID); err != nil { - return err - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/backups/backups_restore_responses.go b/client/backups/backups_restore_responses.go deleted file mode 100644 index 6477062e3736c0dce44710311502f38de7c7ed07..0000000000000000000000000000000000000000 --- a/client/backups/backups_restore_responses.go +++ /dev/null @@ -1,472 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package backups - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// BackupsRestoreReader is a Reader for the BackupsRestore structure. -type BackupsRestoreReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *BackupsRestoreReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 200: - result := NewBackupsRestoreOK() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 401: - result := NewBackupsRestoreUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewBackupsRestoreForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 404: - result := NewBackupsRestoreNotFound() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 422: - result := NewBackupsRestoreUnprocessableEntity() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewBackupsRestoreInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewBackupsRestoreOK creates a BackupsRestoreOK with default headers values -func NewBackupsRestoreOK() *BackupsRestoreOK { - return &BackupsRestoreOK{} -} - -/* -BackupsRestoreOK describes a response with status code 200, with default header values. - -Backup restoration process successfully started. -*/ -type BackupsRestoreOK struct { - Payload *models.BackupRestoreResponse -} - -// IsSuccess returns true when this backups restore o k response has a 2xx status code -func (o *BackupsRestoreOK) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this backups restore o k response has a 3xx status code -func (o *BackupsRestoreOK) IsRedirect() bool { - return false -} - -// IsClientError returns true when this backups restore o k response has a 4xx status code -func (o *BackupsRestoreOK) IsClientError() bool { - return false -} - -// IsServerError returns true when this backups restore o k response has a 5xx status code -func (o *BackupsRestoreOK) IsServerError() bool { - return false -} - -// IsCode returns true when this backups restore o k response a status code equal to that given -func (o *BackupsRestoreOK) IsCode(code int) bool { - return code == 200 -} - -// Code gets the status code for the backups restore o k response -func (o *BackupsRestoreOK) Code() int { - return 200 -} - -func (o *BackupsRestoreOK) Error() string { - return fmt.Sprintf("[POST /backups/{backend}/{id}/restore][%d] backupsRestoreOK %+v", 200, o.Payload) -} - -func (o *BackupsRestoreOK) String() string { - return fmt.Sprintf("[POST /backups/{backend}/{id}/restore][%d] backupsRestoreOK %+v", 200, o.Payload) -} - -func (o *BackupsRestoreOK) GetPayload() *models.BackupRestoreResponse { - return o.Payload -} - -func (o *BackupsRestoreOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.BackupRestoreResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewBackupsRestoreUnauthorized creates a BackupsRestoreUnauthorized with default headers values -func NewBackupsRestoreUnauthorized() *BackupsRestoreUnauthorized { - return &BackupsRestoreUnauthorized{} -} - -/* -BackupsRestoreUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type BackupsRestoreUnauthorized struct { -} - -// IsSuccess returns true when this backups restore unauthorized response has a 2xx status code -func (o *BackupsRestoreUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this backups restore unauthorized response has a 3xx status code -func (o *BackupsRestoreUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this backups restore unauthorized response has a 4xx status code -func (o *BackupsRestoreUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this backups restore unauthorized response has a 5xx status code -func (o *BackupsRestoreUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this backups restore unauthorized response a status code equal to that given -func (o *BackupsRestoreUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the backups restore unauthorized response -func (o *BackupsRestoreUnauthorized) Code() int { - return 401 -} - -func (o *BackupsRestoreUnauthorized) Error() string { - return fmt.Sprintf("[POST /backups/{backend}/{id}/restore][%d] backupsRestoreUnauthorized ", 401) -} - -func (o *BackupsRestoreUnauthorized) String() string { - return fmt.Sprintf("[POST /backups/{backend}/{id}/restore][%d] backupsRestoreUnauthorized ", 401) -} - -func (o *BackupsRestoreUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewBackupsRestoreForbidden creates a BackupsRestoreForbidden with default headers values -func NewBackupsRestoreForbidden() *BackupsRestoreForbidden { - return &BackupsRestoreForbidden{} -} - -/* -BackupsRestoreForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type BackupsRestoreForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this backups restore forbidden response has a 2xx status code -func (o *BackupsRestoreForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this backups restore forbidden response has a 3xx status code -func (o *BackupsRestoreForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this backups restore forbidden response has a 4xx status code -func (o *BackupsRestoreForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this backups restore forbidden response has a 5xx status code -func (o *BackupsRestoreForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this backups restore forbidden response a status code equal to that given -func (o *BackupsRestoreForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the backups restore forbidden response -func (o *BackupsRestoreForbidden) Code() int { - return 403 -} - -func (o *BackupsRestoreForbidden) Error() string { - return fmt.Sprintf("[POST /backups/{backend}/{id}/restore][%d] backupsRestoreForbidden %+v", 403, o.Payload) -} - -func (o *BackupsRestoreForbidden) String() string { - return fmt.Sprintf("[POST /backups/{backend}/{id}/restore][%d] backupsRestoreForbidden %+v", 403, o.Payload) -} - -func (o *BackupsRestoreForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *BackupsRestoreForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewBackupsRestoreNotFound creates a BackupsRestoreNotFound with default headers values -func NewBackupsRestoreNotFound() *BackupsRestoreNotFound { - return &BackupsRestoreNotFound{} -} - -/* -BackupsRestoreNotFound describes a response with status code 404, with default header values. - -Not Found - Backup does not exist -*/ -type BackupsRestoreNotFound struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this backups restore not found response has a 2xx status code -func (o *BackupsRestoreNotFound) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this backups restore not found response has a 3xx status code -func (o *BackupsRestoreNotFound) IsRedirect() bool { - return false -} - -// IsClientError returns true when this backups restore not found response has a 4xx status code -func (o *BackupsRestoreNotFound) IsClientError() bool { - return true -} - -// IsServerError returns true when this backups restore not found response has a 5xx status code -func (o *BackupsRestoreNotFound) IsServerError() bool { - return false -} - -// IsCode returns true when this backups restore not found response a status code equal to that given -func (o *BackupsRestoreNotFound) IsCode(code int) bool { - return code == 404 -} - -// Code gets the status code for the backups restore not found response -func (o *BackupsRestoreNotFound) Code() int { - return 404 -} - -func (o *BackupsRestoreNotFound) Error() string { - return fmt.Sprintf("[POST /backups/{backend}/{id}/restore][%d] backupsRestoreNotFound %+v", 404, o.Payload) -} - -func (o *BackupsRestoreNotFound) String() string { - return fmt.Sprintf("[POST /backups/{backend}/{id}/restore][%d] backupsRestoreNotFound %+v", 404, o.Payload) -} - -func (o *BackupsRestoreNotFound) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *BackupsRestoreNotFound) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewBackupsRestoreUnprocessableEntity creates a BackupsRestoreUnprocessableEntity with default headers values -func NewBackupsRestoreUnprocessableEntity() *BackupsRestoreUnprocessableEntity { - return &BackupsRestoreUnprocessableEntity{} -} - -/* -BackupsRestoreUnprocessableEntity describes a response with status code 422, with default header values. - -Invalid backup restoration attempt. -*/ -type BackupsRestoreUnprocessableEntity struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this backups restore unprocessable entity response has a 2xx status code -func (o *BackupsRestoreUnprocessableEntity) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this backups restore unprocessable entity response has a 3xx status code -func (o *BackupsRestoreUnprocessableEntity) IsRedirect() bool { - return false -} - -// IsClientError returns true when this backups restore unprocessable entity response has a 4xx status code -func (o *BackupsRestoreUnprocessableEntity) IsClientError() bool { - return true -} - -// IsServerError returns true when this backups restore unprocessable entity response has a 5xx status code -func (o *BackupsRestoreUnprocessableEntity) IsServerError() bool { - return false -} - -// IsCode returns true when this backups restore unprocessable entity response a status code equal to that given -func (o *BackupsRestoreUnprocessableEntity) IsCode(code int) bool { - return code == 422 -} - -// Code gets the status code for the backups restore unprocessable entity response -func (o *BackupsRestoreUnprocessableEntity) Code() int { - return 422 -} - -func (o *BackupsRestoreUnprocessableEntity) Error() string { - return fmt.Sprintf("[POST /backups/{backend}/{id}/restore][%d] backupsRestoreUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *BackupsRestoreUnprocessableEntity) String() string { - return fmt.Sprintf("[POST /backups/{backend}/{id}/restore][%d] backupsRestoreUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *BackupsRestoreUnprocessableEntity) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *BackupsRestoreUnprocessableEntity) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewBackupsRestoreInternalServerError creates a BackupsRestoreInternalServerError with default headers values -func NewBackupsRestoreInternalServerError() *BackupsRestoreInternalServerError { - return &BackupsRestoreInternalServerError{} -} - -/* -BackupsRestoreInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type BackupsRestoreInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this backups restore internal server error response has a 2xx status code -func (o *BackupsRestoreInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this backups restore internal server error response has a 3xx status code -func (o *BackupsRestoreInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this backups restore internal server error response has a 4xx status code -func (o *BackupsRestoreInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this backups restore internal server error response has a 5xx status code -func (o *BackupsRestoreInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this backups restore internal server error response a status code equal to that given -func (o *BackupsRestoreInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the backups restore internal server error response -func (o *BackupsRestoreInternalServerError) Code() int { - return 500 -} - -func (o *BackupsRestoreInternalServerError) Error() string { - return fmt.Sprintf("[POST /backups/{backend}/{id}/restore][%d] backupsRestoreInternalServerError %+v", 500, o.Payload) -} - -func (o *BackupsRestoreInternalServerError) String() string { - return fmt.Sprintf("[POST /backups/{backend}/{id}/restore][%d] backupsRestoreInternalServerError %+v", 500, o.Payload) -} - -func (o *BackupsRestoreInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *BackupsRestoreInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/backups/backups_restore_status_parameters.go b/client/backups/backups_restore_status_parameters.go deleted file mode 100644 index cb676a578ee45e1bc5f434482c78d60a1cd851b6..0000000000000000000000000000000000000000 --- a/client/backups/backups_restore_status_parameters.go +++ /dev/null @@ -1,184 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package backups - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" -) - -// NewBackupsRestoreStatusParams creates a new BackupsRestoreStatusParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewBackupsRestoreStatusParams() *BackupsRestoreStatusParams { - return &BackupsRestoreStatusParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewBackupsRestoreStatusParamsWithTimeout creates a new BackupsRestoreStatusParams object -// with the ability to set a timeout on a request. -func NewBackupsRestoreStatusParamsWithTimeout(timeout time.Duration) *BackupsRestoreStatusParams { - return &BackupsRestoreStatusParams{ - timeout: timeout, - } -} - -// NewBackupsRestoreStatusParamsWithContext creates a new BackupsRestoreStatusParams object -// with the ability to set a context for a request. -func NewBackupsRestoreStatusParamsWithContext(ctx context.Context) *BackupsRestoreStatusParams { - return &BackupsRestoreStatusParams{ - Context: ctx, - } -} - -// NewBackupsRestoreStatusParamsWithHTTPClient creates a new BackupsRestoreStatusParams object -// with the ability to set a custom HTTPClient for a request. -func NewBackupsRestoreStatusParamsWithHTTPClient(client *http.Client) *BackupsRestoreStatusParams { - return &BackupsRestoreStatusParams{ - HTTPClient: client, - } -} - -/* -BackupsRestoreStatusParams contains all the parameters to send to the API endpoint - - for the backups restore status operation. - - Typically these are written to a http.Request. -*/ -type BackupsRestoreStatusParams struct { - - /* Backend. - - Backup backend name e.g. filesystem, gcs, s3. - */ - Backend string - - /* ID. - - The ID of a backup. Must be URL-safe and work as a filesystem path, only lowercase, numbers, underscore, minus characters allowed. - */ - ID string - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the backups restore status params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *BackupsRestoreStatusParams) WithDefaults() *BackupsRestoreStatusParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the backups restore status params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *BackupsRestoreStatusParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the backups restore status params -func (o *BackupsRestoreStatusParams) WithTimeout(timeout time.Duration) *BackupsRestoreStatusParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the backups restore status params -func (o *BackupsRestoreStatusParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the backups restore status params -func (o *BackupsRestoreStatusParams) WithContext(ctx context.Context) *BackupsRestoreStatusParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the backups restore status params -func (o *BackupsRestoreStatusParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the backups restore status params -func (o *BackupsRestoreStatusParams) WithHTTPClient(client *http.Client) *BackupsRestoreStatusParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the backups restore status params -func (o *BackupsRestoreStatusParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithBackend adds the backend to the backups restore status params -func (o *BackupsRestoreStatusParams) WithBackend(backend string) *BackupsRestoreStatusParams { - o.SetBackend(backend) - return o -} - -// SetBackend adds the backend to the backups restore status params -func (o *BackupsRestoreStatusParams) SetBackend(backend string) { - o.Backend = backend -} - -// WithID adds the id to the backups restore status params -func (o *BackupsRestoreStatusParams) WithID(id string) *BackupsRestoreStatusParams { - o.SetID(id) - return o -} - -// SetID adds the id to the backups restore status params -func (o *BackupsRestoreStatusParams) SetID(id string) { - o.ID = id -} - -// WriteToRequest writes these params to a swagger request -func (o *BackupsRestoreStatusParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - - // path param backend - if err := r.SetPathParam("backend", o.Backend); err != nil { - return err - } - - // path param id - if err := r.SetPathParam("id", o.ID); err != nil { - return err - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/backups/backups_restore_status_responses.go b/client/backups/backups_restore_status_responses.go deleted file mode 100644 index 9d3737582647db105b859cae45e98a5e163a3241..0000000000000000000000000000000000000000 --- a/client/backups/backups_restore_status_responses.go +++ /dev/null @@ -1,398 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package backups - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// BackupsRestoreStatusReader is a Reader for the BackupsRestoreStatus structure. -type BackupsRestoreStatusReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *BackupsRestoreStatusReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 200: - result := NewBackupsRestoreStatusOK() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 401: - result := NewBackupsRestoreStatusUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewBackupsRestoreStatusForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 404: - result := NewBackupsRestoreStatusNotFound() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewBackupsRestoreStatusInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewBackupsRestoreStatusOK creates a BackupsRestoreStatusOK with default headers values -func NewBackupsRestoreStatusOK() *BackupsRestoreStatusOK { - return &BackupsRestoreStatusOK{} -} - -/* -BackupsRestoreStatusOK describes a response with status code 200, with default header values. - -Backup restoration status successfully returned -*/ -type BackupsRestoreStatusOK struct { - Payload *models.BackupRestoreStatusResponse -} - -// IsSuccess returns true when this backups restore status o k response has a 2xx status code -func (o *BackupsRestoreStatusOK) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this backups restore status o k response has a 3xx status code -func (o *BackupsRestoreStatusOK) IsRedirect() bool { - return false -} - -// IsClientError returns true when this backups restore status o k response has a 4xx status code -func (o *BackupsRestoreStatusOK) IsClientError() bool { - return false -} - -// IsServerError returns true when this backups restore status o k response has a 5xx status code -func (o *BackupsRestoreStatusOK) IsServerError() bool { - return false -} - -// IsCode returns true when this backups restore status o k response a status code equal to that given -func (o *BackupsRestoreStatusOK) IsCode(code int) bool { - return code == 200 -} - -// Code gets the status code for the backups restore status o k response -func (o *BackupsRestoreStatusOK) Code() int { - return 200 -} - -func (o *BackupsRestoreStatusOK) Error() string { - return fmt.Sprintf("[GET /backups/{backend}/{id}/restore][%d] backupsRestoreStatusOK %+v", 200, o.Payload) -} - -func (o *BackupsRestoreStatusOK) String() string { - return fmt.Sprintf("[GET /backups/{backend}/{id}/restore][%d] backupsRestoreStatusOK %+v", 200, o.Payload) -} - -func (o *BackupsRestoreStatusOK) GetPayload() *models.BackupRestoreStatusResponse { - return o.Payload -} - -func (o *BackupsRestoreStatusOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.BackupRestoreStatusResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewBackupsRestoreStatusUnauthorized creates a BackupsRestoreStatusUnauthorized with default headers values -func NewBackupsRestoreStatusUnauthorized() *BackupsRestoreStatusUnauthorized { - return &BackupsRestoreStatusUnauthorized{} -} - -/* -BackupsRestoreStatusUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type BackupsRestoreStatusUnauthorized struct { -} - -// IsSuccess returns true when this backups restore status unauthorized response has a 2xx status code -func (o *BackupsRestoreStatusUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this backups restore status unauthorized response has a 3xx status code -func (o *BackupsRestoreStatusUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this backups restore status unauthorized response has a 4xx status code -func (o *BackupsRestoreStatusUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this backups restore status unauthorized response has a 5xx status code -func (o *BackupsRestoreStatusUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this backups restore status unauthorized response a status code equal to that given -func (o *BackupsRestoreStatusUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the backups restore status unauthorized response -func (o *BackupsRestoreStatusUnauthorized) Code() int { - return 401 -} - -func (o *BackupsRestoreStatusUnauthorized) Error() string { - return fmt.Sprintf("[GET /backups/{backend}/{id}/restore][%d] backupsRestoreStatusUnauthorized ", 401) -} - -func (o *BackupsRestoreStatusUnauthorized) String() string { - return fmt.Sprintf("[GET /backups/{backend}/{id}/restore][%d] backupsRestoreStatusUnauthorized ", 401) -} - -func (o *BackupsRestoreStatusUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewBackupsRestoreStatusForbidden creates a BackupsRestoreStatusForbidden with default headers values -func NewBackupsRestoreStatusForbidden() *BackupsRestoreStatusForbidden { - return &BackupsRestoreStatusForbidden{} -} - -/* -BackupsRestoreStatusForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type BackupsRestoreStatusForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this backups restore status forbidden response has a 2xx status code -func (o *BackupsRestoreStatusForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this backups restore status forbidden response has a 3xx status code -func (o *BackupsRestoreStatusForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this backups restore status forbidden response has a 4xx status code -func (o *BackupsRestoreStatusForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this backups restore status forbidden response has a 5xx status code -func (o *BackupsRestoreStatusForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this backups restore status forbidden response a status code equal to that given -func (o *BackupsRestoreStatusForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the backups restore status forbidden response -func (o *BackupsRestoreStatusForbidden) Code() int { - return 403 -} - -func (o *BackupsRestoreStatusForbidden) Error() string { - return fmt.Sprintf("[GET /backups/{backend}/{id}/restore][%d] backupsRestoreStatusForbidden %+v", 403, o.Payload) -} - -func (o *BackupsRestoreStatusForbidden) String() string { - return fmt.Sprintf("[GET /backups/{backend}/{id}/restore][%d] backupsRestoreStatusForbidden %+v", 403, o.Payload) -} - -func (o *BackupsRestoreStatusForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *BackupsRestoreStatusForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewBackupsRestoreStatusNotFound creates a BackupsRestoreStatusNotFound with default headers values -func NewBackupsRestoreStatusNotFound() *BackupsRestoreStatusNotFound { - return &BackupsRestoreStatusNotFound{} -} - -/* -BackupsRestoreStatusNotFound describes a response with status code 404, with default header values. - -Not Found - Backup does not exist -*/ -type BackupsRestoreStatusNotFound struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this backups restore status not found response has a 2xx status code -func (o *BackupsRestoreStatusNotFound) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this backups restore status not found response has a 3xx status code -func (o *BackupsRestoreStatusNotFound) IsRedirect() bool { - return false -} - -// IsClientError returns true when this backups restore status not found response has a 4xx status code -func (o *BackupsRestoreStatusNotFound) IsClientError() bool { - return true -} - -// IsServerError returns true when this backups restore status not found response has a 5xx status code -func (o *BackupsRestoreStatusNotFound) IsServerError() bool { - return false -} - -// IsCode returns true when this backups restore status not found response a status code equal to that given -func (o *BackupsRestoreStatusNotFound) IsCode(code int) bool { - return code == 404 -} - -// Code gets the status code for the backups restore status not found response -func (o *BackupsRestoreStatusNotFound) Code() int { - return 404 -} - -func (o *BackupsRestoreStatusNotFound) Error() string { - return fmt.Sprintf("[GET /backups/{backend}/{id}/restore][%d] backupsRestoreStatusNotFound %+v", 404, o.Payload) -} - -func (o *BackupsRestoreStatusNotFound) String() string { - return fmt.Sprintf("[GET /backups/{backend}/{id}/restore][%d] backupsRestoreStatusNotFound %+v", 404, o.Payload) -} - -func (o *BackupsRestoreStatusNotFound) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *BackupsRestoreStatusNotFound) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewBackupsRestoreStatusInternalServerError creates a BackupsRestoreStatusInternalServerError with default headers values -func NewBackupsRestoreStatusInternalServerError() *BackupsRestoreStatusInternalServerError { - return &BackupsRestoreStatusInternalServerError{} -} - -/* -BackupsRestoreStatusInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type BackupsRestoreStatusInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this backups restore status internal server error response has a 2xx status code -func (o *BackupsRestoreStatusInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this backups restore status internal server error response has a 3xx status code -func (o *BackupsRestoreStatusInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this backups restore status internal server error response has a 4xx status code -func (o *BackupsRestoreStatusInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this backups restore status internal server error response has a 5xx status code -func (o *BackupsRestoreStatusInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this backups restore status internal server error response a status code equal to that given -func (o *BackupsRestoreStatusInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the backups restore status internal server error response -func (o *BackupsRestoreStatusInternalServerError) Code() int { - return 500 -} - -func (o *BackupsRestoreStatusInternalServerError) Error() string { - return fmt.Sprintf("[GET /backups/{backend}/{id}/restore][%d] backupsRestoreStatusInternalServerError %+v", 500, o.Payload) -} - -func (o *BackupsRestoreStatusInternalServerError) String() string { - return fmt.Sprintf("[GET /backups/{backend}/{id}/restore][%d] backupsRestoreStatusInternalServerError %+v", 500, o.Payload) -} - -func (o *BackupsRestoreStatusInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *BackupsRestoreStatusInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/batch/batch_client.go b/client/batch/batch_client.go deleted file mode 100644 index 1107539a1d924ae830156e51fa9dde25082fcb9a..0000000000000000000000000000000000000000 --- a/client/batch/batch_client.go +++ /dev/null @@ -1,179 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package batch - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" -) - -// New creates a new batch API client. -func New(transport runtime.ClientTransport, formats strfmt.Registry) ClientService { - return &Client{transport: transport, formats: formats} -} - -/* -Client for batch API -*/ -type Client struct { - transport runtime.ClientTransport - formats strfmt.Registry -} - -// ClientOption is the option for Client methods -type ClientOption func(*runtime.ClientOperation) - -// ClientService is the interface for Client methods -type ClientService interface { - BatchObjectsCreate(params *BatchObjectsCreateParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*BatchObjectsCreateOK, error) - - BatchObjectsDelete(params *BatchObjectsDeleteParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*BatchObjectsDeleteOK, error) - - BatchReferencesCreate(params *BatchReferencesCreateParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*BatchReferencesCreateOK, error) - - SetTransport(transport runtime.ClientTransport) -} - -/* -BatchObjectsCreate creates new objects based on a object template as a batch - -Register new Objects in bulk. Provided meta-data and schema values are validated. -*/ -func (a *Client) BatchObjectsCreate(params *BatchObjectsCreateParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*BatchObjectsCreateOK, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewBatchObjectsCreateParams() - } - op := &runtime.ClientOperation{ - ID: "batch.objects.create", - Method: "POST", - PathPattern: "/batch/objects", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &BatchObjectsCreateReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*BatchObjectsCreateOK) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for batch.objects.create: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -/* -BatchObjectsDelete deletes objects based on a match filter as a batch - -Delete Objects in bulk that match a certain filter. -*/ -func (a *Client) BatchObjectsDelete(params *BatchObjectsDeleteParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*BatchObjectsDeleteOK, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewBatchObjectsDeleteParams() - } - op := &runtime.ClientOperation{ - ID: "batch.objects.delete", - Method: "DELETE", - PathPattern: "/batch/objects", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &BatchObjectsDeleteReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*BatchObjectsDeleteOK) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for batch.objects.delete: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -/* -BatchReferencesCreate creates new cross references between arbitrary classes in bulk - -Register cross-references between any class items (objects or objects) in bulk. -*/ -func (a *Client) BatchReferencesCreate(params *BatchReferencesCreateParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*BatchReferencesCreateOK, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewBatchReferencesCreateParams() - } - op := &runtime.ClientOperation{ - ID: "batch.references.create", - Method: "POST", - PathPattern: "/batch/references", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &BatchReferencesCreateReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*BatchReferencesCreateOK) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for batch.references.create: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -// SetTransport changes the transport on the client -func (a *Client) SetTransport(transport runtime.ClientTransport) { - a.transport = transport -} diff --git a/client/batch/batch_objects_create_parameters.go b/client/batch/batch_objects_create_parameters.go deleted file mode 100644 index 727570310fc4a18cac3891dc97dcdbead68f39b1..0000000000000000000000000000000000000000 --- a/client/batch/batch_objects_create_parameters.go +++ /dev/null @@ -1,191 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package batch - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" -) - -// NewBatchObjectsCreateParams creates a new BatchObjectsCreateParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewBatchObjectsCreateParams() *BatchObjectsCreateParams { - return &BatchObjectsCreateParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewBatchObjectsCreateParamsWithTimeout creates a new BatchObjectsCreateParams object -// with the ability to set a timeout on a request. -func NewBatchObjectsCreateParamsWithTimeout(timeout time.Duration) *BatchObjectsCreateParams { - return &BatchObjectsCreateParams{ - timeout: timeout, - } -} - -// NewBatchObjectsCreateParamsWithContext creates a new BatchObjectsCreateParams object -// with the ability to set a context for a request. -func NewBatchObjectsCreateParamsWithContext(ctx context.Context) *BatchObjectsCreateParams { - return &BatchObjectsCreateParams{ - Context: ctx, - } -} - -// NewBatchObjectsCreateParamsWithHTTPClient creates a new BatchObjectsCreateParams object -// with the ability to set a custom HTTPClient for a request. -func NewBatchObjectsCreateParamsWithHTTPClient(client *http.Client) *BatchObjectsCreateParams { - return &BatchObjectsCreateParams{ - HTTPClient: client, - } -} - -/* -BatchObjectsCreateParams contains all the parameters to send to the API endpoint - - for the batch objects create operation. - - Typically these are written to a http.Request. -*/ -type BatchObjectsCreateParams struct { - - // Body. - Body BatchObjectsCreateBody - - /* ConsistencyLevel. - - Determines how many replicas must acknowledge a request before it is considered successful - */ - ConsistencyLevel *string - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the batch objects create params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *BatchObjectsCreateParams) WithDefaults() *BatchObjectsCreateParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the batch objects create params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *BatchObjectsCreateParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the batch objects create params -func (o *BatchObjectsCreateParams) WithTimeout(timeout time.Duration) *BatchObjectsCreateParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the batch objects create params -func (o *BatchObjectsCreateParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the batch objects create params -func (o *BatchObjectsCreateParams) WithContext(ctx context.Context) *BatchObjectsCreateParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the batch objects create params -func (o *BatchObjectsCreateParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the batch objects create params -func (o *BatchObjectsCreateParams) WithHTTPClient(client *http.Client) *BatchObjectsCreateParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the batch objects create params -func (o *BatchObjectsCreateParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithBody adds the body to the batch objects create params -func (o *BatchObjectsCreateParams) WithBody(body BatchObjectsCreateBody) *BatchObjectsCreateParams { - o.SetBody(body) - return o -} - -// SetBody adds the body to the batch objects create params -func (o *BatchObjectsCreateParams) SetBody(body BatchObjectsCreateBody) { - o.Body = body -} - -// WithConsistencyLevel adds the consistencyLevel to the batch objects create params -func (o *BatchObjectsCreateParams) WithConsistencyLevel(consistencyLevel *string) *BatchObjectsCreateParams { - o.SetConsistencyLevel(consistencyLevel) - return o -} - -// SetConsistencyLevel adds the consistencyLevel to the batch objects create params -func (o *BatchObjectsCreateParams) SetConsistencyLevel(consistencyLevel *string) { - o.ConsistencyLevel = consistencyLevel -} - -// WriteToRequest writes these params to a swagger request -func (o *BatchObjectsCreateParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - if err := r.SetBodyParam(o.Body); err != nil { - return err - } - - if o.ConsistencyLevel != nil { - - // query param consistency_level - var qrConsistencyLevel string - - if o.ConsistencyLevel != nil { - qrConsistencyLevel = *o.ConsistencyLevel - } - qConsistencyLevel := qrConsistencyLevel - if qConsistencyLevel != "" { - - if err := r.SetQueryParam("consistency_level", qConsistencyLevel); err != nil { - return err - } - } - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/batch/batch_objects_create_responses.go b/client/batch/batch_objects_create_responses.go deleted file mode 100644 index 9b43699cba9afe3d4aa17f7dc6fe21797cf5d0d7..0000000000000000000000000000000000000000 --- a/client/batch/batch_objects_create_responses.go +++ /dev/null @@ -1,624 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package batch - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "encoding/json" - "fmt" - "io" - "strconv" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" - "github.com/go-openapi/validate" - - "github.com/weaviate/weaviate/entities/models" -) - -// BatchObjectsCreateReader is a Reader for the BatchObjectsCreate structure. -type BatchObjectsCreateReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *BatchObjectsCreateReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 200: - result := NewBatchObjectsCreateOK() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 400: - result := NewBatchObjectsCreateBadRequest() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 401: - result := NewBatchObjectsCreateUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewBatchObjectsCreateForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 422: - result := NewBatchObjectsCreateUnprocessableEntity() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewBatchObjectsCreateInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewBatchObjectsCreateOK creates a BatchObjectsCreateOK with default headers values -func NewBatchObjectsCreateOK() *BatchObjectsCreateOK { - return &BatchObjectsCreateOK{} -} - -/* -BatchObjectsCreateOK describes a response with status code 200, with default header values. - -Request succeeded, see response body to get detailed information about each batched item. -*/ -type BatchObjectsCreateOK struct { - Payload []*models.ObjectsGetResponse -} - -// IsSuccess returns true when this batch objects create o k response has a 2xx status code -func (o *BatchObjectsCreateOK) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this batch objects create o k response has a 3xx status code -func (o *BatchObjectsCreateOK) IsRedirect() bool { - return false -} - -// IsClientError returns true when this batch objects create o k response has a 4xx status code -func (o *BatchObjectsCreateOK) IsClientError() bool { - return false -} - -// IsServerError returns true when this batch objects create o k response has a 5xx status code -func (o *BatchObjectsCreateOK) IsServerError() bool { - return false -} - -// IsCode returns true when this batch objects create o k response a status code equal to that given -func (o *BatchObjectsCreateOK) IsCode(code int) bool { - return code == 200 -} - -// Code gets the status code for the batch objects create o k response -func (o *BatchObjectsCreateOK) Code() int { - return 200 -} - -func (o *BatchObjectsCreateOK) Error() string { - return fmt.Sprintf("[POST /batch/objects][%d] batchObjectsCreateOK %+v", 200, o.Payload) -} - -func (o *BatchObjectsCreateOK) String() string { - return fmt.Sprintf("[POST /batch/objects][%d] batchObjectsCreateOK %+v", 200, o.Payload) -} - -func (o *BatchObjectsCreateOK) GetPayload() []*models.ObjectsGetResponse { - return o.Payload -} - -func (o *BatchObjectsCreateOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - // response payload - if err := consumer.Consume(response.Body(), &o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewBatchObjectsCreateBadRequest creates a BatchObjectsCreateBadRequest with default headers values -func NewBatchObjectsCreateBadRequest() *BatchObjectsCreateBadRequest { - return &BatchObjectsCreateBadRequest{} -} - -/* -BatchObjectsCreateBadRequest describes a response with status code 400, with default header values. - -Malformed request. -*/ -type BatchObjectsCreateBadRequest struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this batch objects create bad request response has a 2xx status code -func (o *BatchObjectsCreateBadRequest) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this batch objects create bad request response has a 3xx status code -func (o *BatchObjectsCreateBadRequest) IsRedirect() bool { - return false -} - -// IsClientError returns true when this batch objects create bad request response has a 4xx status code -func (o *BatchObjectsCreateBadRequest) IsClientError() bool { - return true -} - -// IsServerError returns true when this batch objects create bad request response has a 5xx status code -func (o *BatchObjectsCreateBadRequest) IsServerError() bool { - return false -} - -// IsCode returns true when this batch objects create bad request response a status code equal to that given -func (o *BatchObjectsCreateBadRequest) IsCode(code int) bool { - return code == 400 -} - -// Code gets the status code for the batch objects create bad request response -func (o *BatchObjectsCreateBadRequest) Code() int { - return 400 -} - -func (o *BatchObjectsCreateBadRequest) Error() string { - return fmt.Sprintf("[POST /batch/objects][%d] batchObjectsCreateBadRequest %+v", 400, o.Payload) -} - -func (o *BatchObjectsCreateBadRequest) String() string { - return fmt.Sprintf("[POST /batch/objects][%d] batchObjectsCreateBadRequest %+v", 400, o.Payload) -} - -func (o *BatchObjectsCreateBadRequest) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *BatchObjectsCreateBadRequest) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewBatchObjectsCreateUnauthorized creates a BatchObjectsCreateUnauthorized with default headers values -func NewBatchObjectsCreateUnauthorized() *BatchObjectsCreateUnauthorized { - return &BatchObjectsCreateUnauthorized{} -} - -/* -BatchObjectsCreateUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type BatchObjectsCreateUnauthorized struct { -} - -// IsSuccess returns true when this batch objects create unauthorized response has a 2xx status code -func (o *BatchObjectsCreateUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this batch objects create unauthorized response has a 3xx status code -func (o *BatchObjectsCreateUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this batch objects create unauthorized response has a 4xx status code -func (o *BatchObjectsCreateUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this batch objects create unauthorized response has a 5xx status code -func (o *BatchObjectsCreateUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this batch objects create unauthorized response a status code equal to that given -func (o *BatchObjectsCreateUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the batch objects create unauthorized response -func (o *BatchObjectsCreateUnauthorized) Code() int { - return 401 -} - -func (o *BatchObjectsCreateUnauthorized) Error() string { - return fmt.Sprintf("[POST /batch/objects][%d] batchObjectsCreateUnauthorized ", 401) -} - -func (o *BatchObjectsCreateUnauthorized) String() string { - return fmt.Sprintf("[POST /batch/objects][%d] batchObjectsCreateUnauthorized ", 401) -} - -func (o *BatchObjectsCreateUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewBatchObjectsCreateForbidden creates a BatchObjectsCreateForbidden with default headers values -func NewBatchObjectsCreateForbidden() *BatchObjectsCreateForbidden { - return &BatchObjectsCreateForbidden{} -} - -/* -BatchObjectsCreateForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type BatchObjectsCreateForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this batch objects create forbidden response has a 2xx status code -func (o *BatchObjectsCreateForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this batch objects create forbidden response has a 3xx status code -func (o *BatchObjectsCreateForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this batch objects create forbidden response has a 4xx status code -func (o *BatchObjectsCreateForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this batch objects create forbidden response has a 5xx status code -func (o *BatchObjectsCreateForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this batch objects create forbidden response a status code equal to that given -func (o *BatchObjectsCreateForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the batch objects create forbidden response -func (o *BatchObjectsCreateForbidden) Code() int { - return 403 -} - -func (o *BatchObjectsCreateForbidden) Error() string { - return fmt.Sprintf("[POST /batch/objects][%d] batchObjectsCreateForbidden %+v", 403, o.Payload) -} - -func (o *BatchObjectsCreateForbidden) String() string { - return fmt.Sprintf("[POST /batch/objects][%d] batchObjectsCreateForbidden %+v", 403, o.Payload) -} - -func (o *BatchObjectsCreateForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *BatchObjectsCreateForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewBatchObjectsCreateUnprocessableEntity creates a BatchObjectsCreateUnprocessableEntity with default headers values -func NewBatchObjectsCreateUnprocessableEntity() *BatchObjectsCreateUnprocessableEntity { - return &BatchObjectsCreateUnprocessableEntity{} -} - -/* -BatchObjectsCreateUnprocessableEntity describes a response with status code 422, with default header values. - -Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file? -*/ -type BatchObjectsCreateUnprocessableEntity struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this batch objects create unprocessable entity response has a 2xx status code -func (o *BatchObjectsCreateUnprocessableEntity) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this batch objects create unprocessable entity response has a 3xx status code -func (o *BatchObjectsCreateUnprocessableEntity) IsRedirect() bool { - return false -} - -// IsClientError returns true when this batch objects create unprocessable entity response has a 4xx status code -func (o *BatchObjectsCreateUnprocessableEntity) IsClientError() bool { - return true -} - -// IsServerError returns true when this batch objects create unprocessable entity response has a 5xx status code -func (o *BatchObjectsCreateUnprocessableEntity) IsServerError() bool { - return false -} - -// IsCode returns true when this batch objects create unprocessable entity response a status code equal to that given -func (o *BatchObjectsCreateUnprocessableEntity) IsCode(code int) bool { - return code == 422 -} - -// Code gets the status code for the batch objects create unprocessable entity response -func (o *BatchObjectsCreateUnprocessableEntity) Code() int { - return 422 -} - -func (o *BatchObjectsCreateUnprocessableEntity) Error() string { - return fmt.Sprintf("[POST /batch/objects][%d] batchObjectsCreateUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *BatchObjectsCreateUnprocessableEntity) String() string { - return fmt.Sprintf("[POST /batch/objects][%d] batchObjectsCreateUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *BatchObjectsCreateUnprocessableEntity) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *BatchObjectsCreateUnprocessableEntity) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewBatchObjectsCreateInternalServerError creates a BatchObjectsCreateInternalServerError with default headers values -func NewBatchObjectsCreateInternalServerError() *BatchObjectsCreateInternalServerError { - return &BatchObjectsCreateInternalServerError{} -} - -/* -BatchObjectsCreateInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type BatchObjectsCreateInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this batch objects create internal server error response has a 2xx status code -func (o *BatchObjectsCreateInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this batch objects create internal server error response has a 3xx status code -func (o *BatchObjectsCreateInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this batch objects create internal server error response has a 4xx status code -func (o *BatchObjectsCreateInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this batch objects create internal server error response has a 5xx status code -func (o *BatchObjectsCreateInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this batch objects create internal server error response a status code equal to that given -func (o *BatchObjectsCreateInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the batch objects create internal server error response -func (o *BatchObjectsCreateInternalServerError) Code() int { - return 500 -} - -func (o *BatchObjectsCreateInternalServerError) Error() string { - return fmt.Sprintf("[POST /batch/objects][%d] batchObjectsCreateInternalServerError %+v", 500, o.Payload) -} - -func (o *BatchObjectsCreateInternalServerError) String() string { - return fmt.Sprintf("[POST /batch/objects][%d] batchObjectsCreateInternalServerError %+v", 500, o.Payload) -} - -func (o *BatchObjectsCreateInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *BatchObjectsCreateInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -/* -BatchObjectsCreateBody batch objects create body -swagger:model BatchObjectsCreateBody -*/ -type BatchObjectsCreateBody struct { - - // Define which fields need to be returned. Default value is ALL - Fields []*string `json:"fields"` - - // objects - Objects []*models.Object `json:"objects"` -} - -// Validate validates this batch objects create body -func (o *BatchObjectsCreateBody) Validate(formats strfmt.Registry) error { - var res []error - - if err := o.validateFields(formats); err != nil { - res = append(res, err) - } - - if err := o.validateObjects(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -var batchObjectsCreateBodyFieldsItemsEnum []interface{} - -func init() { - var res []string - if err := json.Unmarshal([]byte(`["ALL","class","schema","id","creationTimeUnix"]`), &res); err != nil { - panic(err) - } - for _, v := range res { - batchObjectsCreateBodyFieldsItemsEnum = append(batchObjectsCreateBodyFieldsItemsEnum, v) - } -} - -func (o *BatchObjectsCreateBody) validateFieldsItemsEnum(path, location string, value string) error { - if err := validate.EnumCase(path, location, value, batchObjectsCreateBodyFieldsItemsEnum, true); err != nil { - return err - } - return nil -} - -func (o *BatchObjectsCreateBody) validateFields(formats strfmt.Registry) error { - if swag.IsZero(o.Fields) { // not required - return nil - } - - for i := 0; i < len(o.Fields); i++ { - if swag.IsZero(o.Fields[i]) { // not required - continue - } - - // value enum - if err := o.validateFieldsItemsEnum("body"+"."+"fields"+"."+strconv.Itoa(i), "body", *o.Fields[i]); err != nil { - return err - } - - } - - return nil -} - -func (o *BatchObjectsCreateBody) validateObjects(formats strfmt.Registry) error { - if swag.IsZero(o.Objects) { // not required - return nil - } - - for i := 0; i < len(o.Objects); i++ { - if swag.IsZero(o.Objects[i]) { // not required - continue - } - - if o.Objects[i] != nil { - if err := o.Objects[i].Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("body" + "." + "objects" + "." + strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("body" + "." + "objects" + "." + strconv.Itoa(i)) - } - return err - } - } - - } - - return nil -} - -// ContextValidate validate this batch objects create body based on the context it is used -func (o *BatchObjectsCreateBody) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - if err := o.contextValidateObjects(ctx, formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (o *BatchObjectsCreateBody) contextValidateObjects(ctx context.Context, formats strfmt.Registry) error { - - for i := 0; i < len(o.Objects); i++ { - - if o.Objects[i] != nil { - if err := o.Objects[i].ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("body" + "." + "objects" + "." + strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("body" + "." + "objects" + "." + strconv.Itoa(i)) - } - return err - } - } - - } - - return nil -} - -// MarshalBinary interface implementation -func (o *BatchObjectsCreateBody) MarshalBinary() ([]byte, error) { - if o == nil { - return nil, nil - } - return swag.WriteJSON(o) -} - -// UnmarshalBinary interface implementation -func (o *BatchObjectsCreateBody) UnmarshalBinary(b []byte) error { - var res BatchObjectsCreateBody - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *o = res - return nil -} diff --git a/client/batch/batch_objects_delete_parameters.go b/client/batch/batch_objects_delete_parameters.go deleted file mode 100644 index e1f47def38379392367cbaae6216fc5d0ce46014..0000000000000000000000000000000000000000 --- a/client/batch/batch_objects_delete_parameters.go +++ /dev/null @@ -1,229 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package batch - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewBatchObjectsDeleteParams creates a new BatchObjectsDeleteParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewBatchObjectsDeleteParams() *BatchObjectsDeleteParams { - return &BatchObjectsDeleteParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewBatchObjectsDeleteParamsWithTimeout creates a new BatchObjectsDeleteParams object -// with the ability to set a timeout on a request. -func NewBatchObjectsDeleteParamsWithTimeout(timeout time.Duration) *BatchObjectsDeleteParams { - return &BatchObjectsDeleteParams{ - timeout: timeout, - } -} - -// NewBatchObjectsDeleteParamsWithContext creates a new BatchObjectsDeleteParams object -// with the ability to set a context for a request. -func NewBatchObjectsDeleteParamsWithContext(ctx context.Context) *BatchObjectsDeleteParams { - return &BatchObjectsDeleteParams{ - Context: ctx, - } -} - -// NewBatchObjectsDeleteParamsWithHTTPClient creates a new BatchObjectsDeleteParams object -// with the ability to set a custom HTTPClient for a request. -func NewBatchObjectsDeleteParamsWithHTTPClient(client *http.Client) *BatchObjectsDeleteParams { - return &BatchObjectsDeleteParams{ - HTTPClient: client, - } -} - -/* -BatchObjectsDeleteParams contains all the parameters to send to the API endpoint - - for the batch objects delete operation. - - Typically these are written to a http.Request. -*/ -type BatchObjectsDeleteParams struct { - - // Body. - Body *models.BatchDelete - - /* ConsistencyLevel. - - Determines how many replicas must acknowledge a request before it is considered successful - */ - ConsistencyLevel *string - - /* Tenant. - - Specifies the tenant in a request targeting a multi-tenant class - */ - Tenant *string - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the batch objects delete params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *BatchObjectsDeleteParams) WithDefaults() *BatchObjectsDeleteParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the batch objects delete params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *BatchObjectsDeleteParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the batch objects delete params -func (o *BatchObjectsDeleteParams) WithTimeout(timeout time.Duration) *BatchObjectsDeleteParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the batch objects delete params -func (o *BatchObjectsDeleteParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the batch objects delete params -func (o *BatchObjectsDeleteParams) WithContext(ctx context.Context) *BatchObjectsDeleteParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the batch objects delete params -func (o *BatchObjectsDeleteParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the batch objects delete params -func (o *BatchObjectsDeleteParams) WithHTTPClient(client *http.Client) *BatchObjectsDeleteParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the batch objects delete params -func (o *BatchObjectsDeleteParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithBody adds the body to the batch objects delete params -func (o *BatchObjectsDeleteParams) WithBody(body *models.BatchDelete) *BatchObjectsDeleteParams { - o.SetBody(body) - return o -} - -// SetBody adds the body to the batch objects delete params -func (o *BatchObjectsDeleteParams) SetBody(body *models.BatchDelete) { - o.Body = body -} - -// WithConsistencyLevel adds the consistencyLevel to the batch objects delete params -func (o *BatchObjectsDeleteParams) WithConsistencyLevel(consistencyLevel *string) *BatchObjectsDeleteParams { - o.SetConsistencyLevel(consistencyLevel) - return o -} - -// SetConsistencyLevel adds the consistencyLevel to the batch objects delete params -func (o *BatchObjectsDeleteParams) SetConsistencyLevel(consistencyLevel *string) { - o.ConsistencyLevel = consistencyLevel -} - -// WithTenant adds the tenant to the batch objects delete params -func (o *BatchObjectsDeleteParams) WithTenant(tenant *string) *BatchObjectsDeleteParams { - o.SetTenant(tenant) - return o -} - -// SetTenant adds the tenant to the batch objects delete params -func (o *BatchObjectsDeleteParams) SetTenant(tenant *string) { - o.Tenant = tenant -} - -// WriteToRequest writes these params to a swagger request -func (o *BatchObjectsDeleteParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - if o.Body != nil { - if err := r.SetBodyParam(o.Body); err != nil { - return err - } - } - - if o.ConsistencyLevel != nil { - - // query param consistency_level - var qrConsistencyLevel string - - if o.ConsistencyLevel != nil { - qrConsistencyLevel = *o.ConsistencyLevel - } - qConsistencyLevel := qrConsistencyLevel - if qConsistencyLevel != "" { - - if err := r.SetQueryParam("consistency_level", qConsistencyLevel); err != nil { - return err - } - } - } - - if o.Tenant != nil { - - // query param tenant - var qrTenant string - - if o.Tenant != nil { - qrTenant = *o.Tenant - } - qTenant := qrTenant - if qTenant != "" { - - if err := r.SetQueryParam("tenant", qTenant); err != nil { - return err - } - } - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/batch/batch_objects_delete_responses.go b/client/batch/batch_objects_delete_responses.go deleted file mode 100644 index 42a48d07449d94aa7637f05f88b0ca83baed2891..0000000000000000000000000000000000000000 --- a/client/batch/batch_objects_delete_responses.go +++ /dev/null @@ -1,472 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package batch - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// BatchObjectsDeleteReader is a Reader for the BatchObjectsDelete structure. -type BatchObjectsDeleteReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *BatchObjectsDeleteReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 200: - result := NewBatchObjectsDeleteOK() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 400: - result := NewBatchObjectsDeleteBadRequest() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 401: - result := NewBatchObjectsDeleteUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewBatchObjectsDeleteForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 422: - result := NewBatchObjectsDeleteUnprocessableEntity() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewBatchObjectsDeleteInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewBatchObjectsDeleteOK creates a BatchObjectsDeleteOK with default headers values -func NewBatchObjectsDeleteOK() *BatchObjectsDeleteOK { - return &BatchObjectsDeleteOK{} -} - -/* -BatchObjectsDeleteOK describes a response with status code 200, with default header values. - -Request succeeded, see response body to get detailed information about each batched item. -*/ -type BatchObjectsDeleteOK struct { - Payload *models.BatchDeleteResponse -} - -// IsSuccess returns true when this batch objects delete o k response has a 2xx status code -func (o *BatchObjectsDeleteOK) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this batch objects delete o k response has a 3xx status code -func (o *BatchObjectsDeleteOK) IsRedirect() bool { - return false -} - -// IsClientError returns true when this batch objects delete o k response has a 4xx status code -func (o *BatchObjectsDeleteOK) IsClientError() bool { - return false -} - -// IsServerError returns true when this batch objects delete o k response has a 5xx status code -func (o *BatchObjectsDeleteOK) IsServerError() bool { - return false -} - -// IsCode returns true when this batch objects delete o k response a status code equal to that given -func (o *BatchObjectsDeleteOK) IsCode(code int) bool { - return code == 200 -} - -// Code gets the status code for the batch objects delete o k response -func (o *BatchObjectsDeleteOK) Code() int { - return 200 -} - -func (o *BatchObjectsDeleteOK) Error() string { - return fmt.Sprintf("[DELETE /batch/objects][%d] batchObjectsDeleteOK %+v", 200, o.Payload) -} - -func (o *BatchObjectsDeleteOK) String() string { - return fmt.Sprintf("[DELETE /batch/objects][%d] batchObjectsDeleteOK %+v", 200, o.Payload) -} - -func (o *BatchObjectsDeleteOK) GetPayload() *models.BatchDeleteResponse { - return o.Payload -} - -func (o *BatchObjectsDeleteOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.BatchDeleteResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewBatchObjectsDeleteBadRequest creates a BatchObjectsDeleteBadRequest with default headers values -func NewBatchObjectsDeleteBadRequest() *BatchObjectsDeleteBadRequest { - return &BatchObjectsDeleteBadRequest{} -} - -/* -BatchObjectsDeleteBadRequest describes a response with status code 400, with default header values. - -Malformed request. -*/ -type BatchObjectsDeleteBadRequest struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this batch objects delete bad request response has a 2xx status code -func (o *BatchObjectsDeleteBadRequest) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this batch objects delete bad request response has a 3xx status code -func (o *BatchObjectsDeleteBadRequest) IsRedirect() bool { - return false -} - -// IsClientError returns true when this batch objects delete bad request response has a 4xx status code -func (o *BatchObjectsDeleteBadRequest) IsClientError() bool { - return true -} - -// IsServerError returns true when this batch objects delete bad request response has a 5xx status code -func (o *BatchObjectsDeleteBadRequest) IsServerError() bool { - return false -} - -// IsCode returns true when this batch objects delete bad request response a status code equal to that given -func (o *BatchObjectsDeleteBadRequest) IsCode(code int) bool { - return code == 400 -} - -// Code gets the status code for the batch objects delete bad request response -func (o *BatchObjectsDeleteBadRequest) Code() int { - return 400 -} - -func (o *BatchObjectsDeleteBadRequest) Error() string { - return fmt.Sprintf("[DELETE /batch/objects][%d] batchObjectsDeleteBadRequest %+v", 400, o.Payload) -} - -func (o *BatchObjectsDeleteBadRequest) String() string { - return fmt.Sprintf("[DELETE /batch/objects][%d] batchObjectsDeleteBadRequest %+v", 400, o.Payload) -} - -func (o *BatchObjectsDeleteBadRequest) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *BatchObjectsDeleteBadRequest) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewBatchObjectsDeleteUnauthorized creates a BatchObjectsDeleteUnauthorized with default headers values -func NewBatchObjectsDeleteUnauthorized() *BatchObjectsDeleteUnauthorized { - return &BatchObjectsDeleteUnauthorized{} -} - -/* -BatchObjectsDeleteUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type BatchObjectsDeleteUnauthorized struct { -} - -// IsSuccess returns true when this batch objects delete unauthorized response has a 2xx status code -func (o *BatchObjectsDeleteUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this batch objects delete unauthorized response has a 3xx status code -func (o *BatchObjectsDeleteUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this batch objects delete unauthorized response has a 4xx status code -func (o *BatchObjectsDeleteUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this batch objects delete unauthorized response has a 5xx status code -func (o *BatchObjectsDeleteUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this batch objects delete unauthorized response a status code equal to that given -func (o *BatchObjectsDeleteUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the batch objects delete unauthorized response -func (o *BatchObjectsDeleteUnauthorized) Code() int { - return 401 -} - -func (o *BatchObjectsDeleteUnauthorized) Error() string { - return fmt.Sprintf("[DELETE /batch/objects][%d] batchObjectsDeleteUnauthorized ", 401) -} - -func (o *BatchObjectsDeleteUnauthorized) String() string { - return fmt.Sprintf("[DELETE /batch/objects][%d] batchObjectsDeleteUnauthorized ", 401) -} - -func (o *BatchObjectsDeleteUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewBatchObjectsDeleteForbidden creates a BatchObjectsDeleteForbidden with default headers values -func NewBatchObjectsDeleteForbidden() *BatchObjectsDeleteForbidden { - return &BatchObjectsDeleteForbidden{} -} - -/* -BatchObjectsDeleteForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type BatchObjectsDeleteForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this batch objects delete forbidden response has a 2xx status code -func (o *BatchObjectsDeleteForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this batch objects delete forbidden response has a 3xx status code -func (o *BatchObjectsDeleteForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this batch objects delete forbidden response has a 4xx status code -func (o *BatchObjectsDeleteForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this batch objects delete forbidden response has a 5xx status code -func (o *BatchObjectsDeleteForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this batch objects delete forbidden response a status code equal to that given -func (o *BatchObjectsDeleteForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the batch objects delete forbidden response -func (o *BatchObjectsDeleteForbidden) Code() int { - return 403 -} - -func (o *BatchObjectsDeleteForbidden) Error() string { - return fmt.Sprintf("[DELETE /batch/objects][%d] batchObjectsDeleteForbidden %+v", 403, o.Payload) -} - -func (o *BatchObjectsDeleteForbidden) String() string { - return fmt.Sprintf("[DELETE /batch/objects][%d] batchObjectsDeleteForbidden %+v", 403, o.Payload) -} - -func (o *BatchObjectsDeleteForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *BatchObjectsDeleteForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewBatchObjectsDeleteUnprocessableEntity creates a BatchObjectsDeleteUnprocessableEntity with default headers values -func NewBatchObjectsDeleteUnprocessableEntity() *BatchObjectsDeleteUnprocessableEntity { - return &BatchObjectsDeleteUnprocessableEntity{} -} - -/* -BatchObjectsDeleteUnprocessableEntity describes a response with status code 422, with default header values. - -Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file? -*/ -type BatchObjectsDeleteUnprocessableEntity struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this batch objects delete unprocessable entity response has a 2xx status code -func (o *BatchObjectsDeleteUnprocessableEntity) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this batch objects delete unprocessable entity response has a 3xx status code -func (o *BatchObjectsDeleteUnprocessableEntity) IsRedirect() bool { - return false -} - -// IsClientError returns true when this batch objects delete unprocessable entity response has a 4xx status code -func (o *BatchObjectsDeleteUnprocessableEntity) IsClientError() bool { - return true -} - -// IsServerError returns true when this batch objects delete unprocessable entity response has a 5xx status code -func (o *BatchObjectsDeleteUnprocessableEntity) IsServerError() bool { - return false -} - -// IsCode returns true when this batch objects delete unprocessable entity response a status code equal to that given -func (o *BatchObjectsDeleteUnprocessableEntity) IsCode(code int) bool { - return code == 422 -} - -// Code gets the status code for the batch objects delete unprocessable entity response -func (o *BatchObjectsDeleteUnprocessableEntity) Code() int { - return 422 -} - -func (o *BatchObjectsDeleteUnprocessableEntity) Error() string { - return fmt.Sprintf("[DELETE /batch/objects][%d] batchObjectsDeleteUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *BatchObjectsDeleteUnprocessableEntity) String() string { - return fmt.Sprintf("[DELETE /batch/objects][%d] batchObjectsDeleteUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *BatchObjectsDeleteUnprocessableEntity) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *BatchObjectsDeleteUnprocessableEntity) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewBatchObjectsDeleteInternalServerError creates a BatchObjectsDeleteInternalServerError with default headers values -func NewBatchObjectsDeleteInternalServerError() *BatchObjectsDeleteInternalServerError { - return &BatchObjectsDeleteInternalServerError{} -} - -/* -BatchObjectsDeleteInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type BatchObjectsDeleteInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this batch objects delete internal server error response has a 2xx status code -func (o *BatchObjectsDeleteInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this batch objects delete internal server error response has a 3xx status code -func (o *BatchObjectsDeleteInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this batch objects delete internal server error response has a 4xx status code -func (o *BatchObjectsDeleteInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this batch objects delete internal server error response has a 5xx status code -func (o *BatchObjectsDeleteInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this batch objects delete internal server error response a status code equal to that given -func (o *BatchObjectsDeleteInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the batch objects delete internal server error response -func (o *BatchObjectsDeleteInternalServerError) Code() int { - return 500 -} - -func (o *BatchObjectsDeleteInternalServerError) Error() string { - return fmt.Sprintf("[DELETE /batch/objects][%d] batchObjectsDeleteInternalServerError %+v", 500, o.Payload) -} - -func (o *BatchObjectsDeleteInternalServerError) String() string { - return fmt.Sprintf("[DELETE /batch/objects][%d] batchObjectsDeleteInternalServerError %+v", 500, o.Payload) -} - -func (o *BatchObjectsDeleteInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *BatchObjectsDeleteInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/batch/batch_references_create_parameters.go b/client/batch/batch_references_create_parameters.go deleted file mode 100644 index 7e2fba87949f013dce620e7de25df90f65f792ee..0000000000000000000000000000000000000000 --- a/client/batch/batch_references_create_parameters.go +++ /dev/null @@ -1,198 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package batch - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewBatchReferencesCreateParams creates a new BatchReferencesCreateParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewBatchReferencesCreateParams() *BatchReferencesCreateParams { - return &BatchReferencesCreateParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewBatchReferencesCreateParamsWithTimeout creates a new BatchReferencesCreateParams object -// with the ability to set a timeout on a request. -func NewBatchReferencesCreateParamsWithTimeout(timeout time.Duration) *BatchReferencesCreateParams { - return &BatchReferencesCreateParams{ - timeout: timeout, - } -} - -// NewBatchReferencesCreateParamsWithContext creates a new BatchReferencesCreateParams object -// with the ability to set a context for a request. -func NewBatchReferencesCreateParamsWithContext(ctx context.Context) *BatchReferencesCreateParams { - return &BatchReferencesCreateParams{ - Context: ctx, - } -} - -// NewBatchReferencesCreateParamsWithHTTPClient creates a new BatchReferencesCreateParams object -// with the ability to set a custom HTTPClient for a request. -func NewBatchReferencesCreateParamsWithHTTPClient(client *http.Client) *BatchReferencesCreateParams { - return &BatchReferencesCreateParams{ - HTTPClient: client, - } -} - -/* -BatchReferencesCreateParams contains all the parameters to send to the API endpoint - - for the batch references create operation. - - Typically these are written to a http.Request. -*/ -type BatchReferencesCreateParams struct { - - /* Body. - - A list of references to be batched. The ideal size depends on the used database connector. Please see the documentation of the used connector for help - */ - Body []*models.BatchReference - - /* ConsistencyLevel. - - Determines how many replicas must acknowledge a request before it is considered successful - */ - ConsistencyLevel *string - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the batch references create params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *BatchReferencesCreateParams) WithDefaults() *BatchReferencesCreateParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the batch references create params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *BatchReferencesCreateParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the batch references create params -func (o *BatchReferencesCreateParams) WithTimeout(timeout time.Duration) *BatchReferencesCreateParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the batch references create params -func (o *BatchReferencesCreateParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the batch references create params -func (o *BatchReferencesCreateParams) WithContext(ctx context.Context) *BatchReferencesCreateParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the batch references create params -func (o *BatchReferencesCreateParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the batch references create params -func (o *BatchReferencesCreateParams) WithHTTPClient(client *http.Client) *BatchReferencesCreateParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the batch references create params -func (o *BatchReferencesCreateParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithBody adds the body to the batch references create params -func (o *BatchReferencesCreateParams) WithBody(body []*models.BatchReference) *BatchReferencesCreateParams { - o.SetBody(body) - return o -} - -// SetBody adds the body to the batch references create params -func (o *BatchReferencesCreateParams) SetBody(body []*models.BatchReference) { - o.Body = body -} - -// WithConsistencyLevel adds the consistencyLevel to the batch references create params -func (o *BatchReferencesCreateParams) WithConsistencyLevel(consistencyLevel *string) *BatchReferencesCreateParams { - o.SetConsistencyLevel(consistencyLevel) - return o -} - -// SetConsistencyLevel adds the consistencyLevel to the batch references create params -func (o *BatchReferencesCreateParams) SetConsistencyLevel(consistencyLevel *string) { - o.ConsistencyLevel = consistencyLevel -} - -// WriteToRequest writes these params to a swagger request -func (o *BatchReferencesCreateParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - if o.Body != nil { - if err := r.SetBodyParam(o.Body); err != nil { - return err - } - } - - if o.ConsistencyLevel != nil { - - // query param consistency_level - var qrConsistencyLevel string - - if o.ConsistencyLevel != nil { - qrConsistencyLevel = *o.ConsistencyLevel - } - qConsistencyLevel := qrConsistencyLevel - if qConsistencyLevel != "" { - - if err := r.SetQueryParam("consistency_level", qConsistencyLevel); err != nil { - return err - } - } - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/batch/batch_references_create_responses.go b/client/batch/batch_references_create_responses.go deleted file mode 100644 index d53caab6440d0850d063c7ae26f775d2fc8291ac..0000000000000000000000000000000000000000 --- a/client/batch/batch_references_create_responses.go +++ /dev/null @@ -1,470 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package batch - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// BatchReferencesCreateReader is a Reader for the BatchReferencesCreate structure. -type BatchReferencesCreateReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *BatchReferencesCreateReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 200: - result := NewBatchReferencesCreateOK() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 400: - result := NewBatchReferencesCreateBadRequest() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 401: - result := NewBatchReferencesCreateUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewBatchReferencesCreateForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 422: - result := NewBatchReferencesCreateUnprocessableEntity() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewBatchReferencesCreateInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewBatchReferencesCreateOK creates a BatchReferencesCreateOK with default headers values -func NewBatchReferencesCreateOK() *BatchReferencesCreateOK { - return &BatchReferencesCreateOK{} -} - -/* -BatchReferencesCreateOK describes a response with status code 200, with default header values. - -Request Successful. Warning: A successful request does not guarantee that every batched reference was successfully created. Inspect the response body to see which references succeeded and which failed. -*/ -type BatchReferencesCreateOK struct { - Payload []*models.BatchReferenceResponse -} - -// IsSuccess returns true when this batch references create o k response has a 2xx status code -func (o *BatchReferencesCreateOK) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this batch references create o k response has a 3xx status code -func (o *BatchReferencesCreateOK) IsRedirect() bool { - return false -} - -// IsClientError returns true when this batch references create o k response has a 4xx status code -func (o *BatchReferencesCreateOK) IsClientError() bool { - return false -} - -// IsServerError returns true when this batch references create o k response has a 5xx status code -func (o *BatchReferencesCreateOK) IsServerError() bool { - return false -} - -// IsCode returns true when this batch references create o k response a status code equal to that given -func (o *BatchReferencesCreateOK) IsCode(code int) bool { - return code == 200 -} - -// Code gets the status code for the batch references create o k response -func (o *BatchReferencesCreateOK) Code() int { - return 200 -} - -func (o *BatchReferencesCreateOK) Error() string { - return fmt.Sprintf("[POST /batch/references][%d] batchReferencesCreateOK %+v", 200, o.Payload) -} - -func (o *BatchReferencesCreateOK) String() string { - return fmt.Sprintf("[POST /batch/references][%d] batchReferencesCreateOK %+v", 200, o.Payload) -} - -func (o *BatchReferencesCreateOK) GetPayload() []*models.BatchReferenceResponse { - return o.Payload -} - -func (o *BatchReferencesCreateOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - // response payload - if err := consumer.Consume(response.Body(), &o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewBatchReferencesCreateBadRequest creates a BatchReferencesCreateBadRequest with default headers values -func NewBatchReferencesCreateBadRequest() *BatchReferencesCreateBadRequest { - return &BatchReferencesCreateBadRequest{} -} - -/* -BatchReferencesCreateBadRequest describes a response with status code 400, with default header values. - -Malformed request. -*/ -type BatchReferencesCreateBadRequest struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this batch references create bad request response has a 2xx status code -func (o *BatchReferencesCreateBadRequest) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this batch references create bad request response has a 3xx status code -func (o *BatchReferencesCreateBadRequest) IsRedirect() bool { - return false -} - -// IsClientError returns true when this batch references create bad request response has a 4xx status code -func (o *BatchReferencesCreateBadRequest) IsClientError() bool { - return true -} - -// IsServerError returns true when this batch references create bad request response has a 5xx status code -func (o *BatchReferencesCreateBadRequest) IsServerError() bool { - return false -} - -// IsCode returns true when this batch references create bad request response a status code equal to that given -func (o *BatchReferencesCreateBadRequest) IsCode(code int) bool { - return code == 400 -} - -// Code gets the status code for the batch references create bad request response -func (o *BatchReferencesCreateBadRequest) Code() int { - return 400 -} - -func (o *BatchReferencesCreateBadRequest) Error() string { - return fmt.Sprintf("[POST /batch/references][%d] batchReferencesCreateBadRequest %+v", 400, o.Payload) -} - -func (o *BatchReferencesCreateBadRequest) String() string { - return fmt.Sprintf("[POST /batch/references][%d] batchReferencesCreateBadRequest %+v", 400, o.Payload) -} - -func (o *BatchReferencesCreateBadRequest) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *BatchReferencesCreateBadRequest) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewBatchReferencesCreateUnauthorized creates a BatchReferencesCreateUnauthorized with default headers values -func NewBatchReferencesCreateUnauthorized() *BatchReferencesCreateUnauthorized { - return &BatchReferencesCreateUnauthorized{} -} - -/* -BatchReferencesCreateUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type BatchReferencesCreateUnauthorized struct { -} - -// IsSuccess returns true when this batch references create unauthorized response has a 2xx status code -func (o *BatchReferencesCreateUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this batch references create unauthorized response has a 3xx status code -func (o *BatchReferencesCreateUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this batch references create unauthorized response has a 4xx status code -func (o *BatchReferencesCreateUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this batch references create unauthorized response has a 5xx status code -func (o *BatchReferencesCreateUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this batch references create unauthorized response a status code equal to that given -func (o *BatchReferencesCreateUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the batch references create unauthorized response -func (o *BatchReferencesCreateUnauthorized) Code() int { - return 401 -} - -func (o *BatchReferencesCreateUnauthorized) Error() string { - return fmt.Sprintf("[POST /batch/references][%d] batchReferencesCreateUnauthorized ", 401) -} - -func (o *BatchReferencesCreateUnauthorized) String() string { - return fmt.Sprintf("[POST /batch/references][%d] batchReferencesCreateUnauthorized ", 401) -} - -func (o *BatchReferencesCreateUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewBatchReferencesCreateForbidden creates a BatchReferencesCreateForbidden with default headers values -func NewBatchReferencesCreateForbidden() *BatchReferencesCreateForbidden { - return &BatchReferencesCreateForbidden{} -} - -/* -BatchReferencesCreateForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type BatchReferencesCreateForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this batch references create forbidden response has a 2xx status code -func (o *BatchReferencesCreateForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this batch references create forbidden response has a 3xx status code -func (o *BatchReferencesCreateForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this batch references create forbidden response has a 4xx status code -func (o *BatchReferencesCreateForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this batch references create forbidden response has a 5xx status code -func (o *BatchReferencesCreateForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this batch references create forbidden response a status code equal to that given -func (o *BatchReferencesCreateForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the batch references create forbidden response -func (o *BatchReferencesCreateForbidden) Code() int { - return 403 -} - -func (o *BatchReferencesCreateForbidden) Error() string { - return fmt.Sprintf("[POST /batch/references][%d] batchReferencesCreateForbidden %+v", 403, o.Payload) -} - -func (o *BatchReferencesCreateForbidden) String() string { - return fmt.Sprintf("[POST /batch/references][%d] batchReferencesCreateForbidden %+v", 403, o.Payload) -} - -func (o *BatchReferencesCreateForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *BatchReferencesCreateForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewBatchReferencesCreateUnprocessableEntity creates a BatchReferencesCreateUnprocessableEntity with default headers values -func NewBatchReferencesCreateUnprocessableEntity() *BatchReferencesCreateUnprocessableEntity { - return &BatchReferencesCreateUnprocessableEntity{} -} - -/* -BatchReferencesCreateUnprocessableEntity describes a response with status code 422, with default header values. - -Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file? -*/ -type BatchReferencesCreateUnprocessableEntity struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this batch references create unprocessable entity response has a 2xx status code -func (o *BatchReferencesCreateUnprocessableEntity) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this batch references create unprocessable entity response has a 3xx status code -func (o *BatchReferencesCreateUnprocessableEntity) IsRedirect() bool { - return false -} - -// IsClientError returns true when this batch references create unprocessable entity response has a 4xx status code -func (o *BatchReferencesCreateUnprocessableEntity) IsClientError() bool { - return true -} - -// IsServerError returns true when this batch references create unprocessable entity response has a 5xx status code -func (o *BatchReferencesCreateUnprocessableEntity) IsServerError() bool { - return false -} - -// IsCode returns true when this batch references create unprocessable entity response a status code equal to that given -func (o *BatchReferencesCreateUnprocessableEntity) IsCode(code int) bool { - return code == 422 -} - -// Code gets the status code for the batch references create unprocessable entity response -func (o *BatchReferencesCreateUnprocessableEntity) Code() int { - return 422 -} - -func (o *BatchReferencesCreateUnprocessableEntity) Error() string { - return fmt.Sprintf("[POST /batch/references][%d] batchReferencesCreateUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *BatchReferencesCreateUnprocessableEntity) String() string { - return fmt.Sprintf("[POST /batch/references][%d] batchReferencesCreateUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *BatchReferencesCreateUnprocessableEntity) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *BatchReferencesCreateUnprocessableEntity) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewBatchReferencesCreateInternalServerError creates a BatchReferencesCreateInternalServerError with default headers values -func NewBatchReferencesCreateInternalServerError() *BatchReferencesCreateInternalServerError { - return &BatchReferencesCreateInternalServerError{} -} - -/* -BatchReferencesCreateInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type BatchReferencesCreateInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this batch references create internal server error response has a 2xx status code -func (o *BatchReferencesCreateInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this batch references create internal server error response has a 3xx status code -func (o *BatchReferencesCreateInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this batch references create internal server error response has a 4xx status code -func (o *BatchReferencesCreateInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this batch references create internal server error response has a 5xx status code -func (o *BatchReferencesCreateInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this batch references create internal server error response a status code equal to that given -func (o *BatchReferencesCreateInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the batch references create internal server error response -func (o *BatchReferencesCreateInternalServerError) Code() int { - return 500 -} - -func (o *BatchReferencesCreateInternalServerError) Error() string { - return fmt.Sprintf("[POST /batch/references][%d] batchReferencesCreateInternalServerError %+v", 500, o.Payload) -} - -func (o *BatchReferencesCreateInternalServerError) String() string { - return fmt.Sprintf("[POST /batch/references][%d] batchReferencesCreateInternalServerError %+v", 500, o.Payload) -} - -func (o *BatchReferencesCreateInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *BatchReferencesCreateInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/classifications/classifications_client.go b/client/classifications/classifications_client.go deleted file mode 100644 index e7957baa25bb67e8a75dda6cb74c10d3026bc77f..0000000000000000000000000000000000000000 --- a/client/classifications/classifications_client.go +++ /dev/null @@ -1,136 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package classifications - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" -) - -// New creates a new classifications API client. -func New(transport runtime.ClientTransport, formats strfmt.Registry) ClientService { - return &Client{transport: transport, formats: formats} -} - -/* -Client for classifications API -*/ -type Client struct { - transport runtime.ClientTransport - formats strfmt.Registry -} - -// ClientOption is the option for Client methods -type ClientOption func(*runtime.ClientOperation) - -// ClientService is the interface for Client methods -type ClientService interface { - ClassificationsGet(params *ClassificationsGetParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*ClassificationsGetOK, error) - - ClassificationsPost(params *ClassificationsPostParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*ClassificationsPostCreated, error) - - SetTransport(transport runtime.ClientTransport) -} - -/* -ClassificationsGet views previously created classification - -Get status, results and metadata of a previously created classification -*/ -func (a *Client) ClassificationsGet(params *ClassificationsGetParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*ClassificationsGetOK, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewClassificationsGetParams() - } - op := &runtime.ClientOperation{ - ID: "classifications.get", - Method: "GET", - PathPattern: "/classifications/{id}", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &ClassificationsGetReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*ClassificationsGetOK) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for classifications.get: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -/* -ClassificationsPost starts a classification - -Trigger a classification based on the specified params. Classifications will run in the background, use GET /classifications/ to retrieve the status of your classification. -*/ -func (a *Client) ClassificationsPost(params *ClassificationsPostParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*ClassificationsPostCreated, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewClassificationsPostParams() - } - op := &runtime.ClientOperation{ - ID: "classifications.post", - Method: "POST", - PathPattern: "/classifications/", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &ClassificationsPostReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*ClassificationsPostCreated) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for classifications.post: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -// SetTransport changes the transport on the client -func (a *Client) SetTransport(transport runtime.ClientTransport) { - a.transport = transport -} diff --git a/client/classifications/classifications_get_parameters.go b/client/classifications/classifications_get_parameters.go deleted file mode 100644 index 9d77fcc1d61e7d92969dcfc12ad216d917568ba3..0000000000000000000000000000000000000000 --- a/client/classifications/classifications_get_parameters.go +++ /dev/null @@ -1,162 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package classifications - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" -) - -// NewClassificationsGetParams creates a new ClassificationsGetParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewClassificationsGetParams() *ClassificationsGetParams { - return &ClassificationsGetParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewClassificationsGetParamsWithTimeout creates a new ClassificationsGetParams object -// with the ability to set a timeout on a request. -func NewClassificationsGetParamsWithTimeout(timeout time.Duration) *ClassificationsGetParams { - return &ClassificationsGetParams{ - timeout: timeout, - } -} - -// NewClassificationsGetParamsWithContext creates a new ClassificationsGetParams object -// with the ability to set a context for a request. -func NewClassificationsGetParamsWithContext(ctx context.Context) *ClassificationsGetParams { - return &ClassificationsGetParams{ - Context: ctx, - } -} - -// NewClassificationsGetParamsWithHTTPClient creates a new ClassificationsGetParams object -// with the ability to set a custom HTTPClient for a request. -func NewClassificationsGetParamsWithHTTPClient(client *http.Client) *ClassificationsGetParams { - return &ClassificationsGetParams{ - HTTPClient: client, - } -} - -/* -ClassificationsGetParams contains all the parameters to send to the API endpoint - - for the classifications get operation. - - Typically these are written to a http.Request. -*/ -type ClassificationsGetParams struct { - - /* ID. - - classification id - */ - ID string - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the classifications get params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *ClassificationsGetParams) WithDefaults() *ClassificationsGetParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the classifications get params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *ClassificationsGetParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the classifications get params -func (o *ClassificationsGetParams) WithTimeout(timeout time.Duration) *ClassificationsGetParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the classifications get params -func (o *ClassificationsGetParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the classifications get params -func (o *ClassificationsGetParams) WithContext(ctx context.Context) *ClassificationsGetParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the classifications get params -func (o *ClassificationsGetParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the classifications get params -func (o *ClassificationsGetParams) WithHTTPClient(client *http.Client) *ClassificationsGetParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the classifications get params -func (o *ClassificationsGetParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithID adds the id to the classifications get params -func (o *ClassificationsGetParams) WithID(id string) *ClassificationsGetParams { - o.SetID(id) - return o -} - -// SetID adds the id to the classifications get params -func (o *ClassificationsGetParams) SetID(id string) { - o.ID = id -} - -// WriteToRequest writes these params to a swagger request -func (o *ClassificationsGetParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - - // path param id - if err := r.SetPathParam("id", o.ID); err != nil { - return err - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/classifications/classifications_get_responses.go b/client/classifications/classifications_get_responses.go deleted file mode 100644 index 3703061f9876749c0c5b91a24a2ae9fb2aa58973..0000000000000000000000000000000000000000 --- a/client/classifications/classifications_get_responses.go +++ /dev/null @@ -1,386 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package classifications - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// ClassificationsGetReader is a Reader for the ClassificationsGet structure. -type ClassificationsGetReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *ClassificationsGetReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 200: - result := NewClassificationsGetOK() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 401: - result := NewClassificationsGetUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewClassificationsGetForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 404: - result := NewClassificationsGetNotFound() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewClassificationsGetInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewClassificationsGetOK creates a ClassificationsGetOK with default headers values -func NewClassificationsGetOK() *ClassificationsGetOK { - return &ClassificationsGetOK{} -} - -/* -ClassificationsGetOK describes a response with status code 200, with default header values. - -Found the classification, returned as body -*/ -type ClassificationsGetOK struct { - Payload *models.Classification -} - -// IsSuccess returns true when this classifications get o k response has a 2xx status code -func (o *ClassificationsGetOK) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this classifications get o k response has a 3xx status code -func (o *ClassificationsGetOK) IsRedirect() bool { - return false -} - -// IsClientError returns true when this classifications get o k response has a 4xx status code -func (o *ClassificationsGetOK) IsClientError() bool { - return false -} - -// IsServerError returns true when this classifications get o k response has a 5xx status code -func (o *ClassificationsGetOK) IsServerError() bool { - return false -} - -// IsCode returns true when this classifications get o k response a status code equal to that given -func (o *ClassificationsGetOK) IsCode(code int) bool { - return code == 200 -} - -// Code gets the status code for the classifications get o k response -func (o *ClassificationsGetOK) Code() int { - return 200 -} - -func (o *ClassificationsGetOK) Error() string { - return fmt.Sprintf("[GET /classifications/{id}][%d] classificationsGetOK %+v", 200, o.Payload) -} - -func (o *ClassificationsGetOK) String() string { - return fmt.Sprintf("[GET /classifications/{id}][%d] classificationsGetOK %+v", 200, o.Payload) -} - -func (o *ClassificationsGetOK) GetPayload() *models.Classification { - return o.Payload -} - -func (o *ClassificationsGetOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.Classification) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewClassificationsGetUnauthorized creates a ClassificationsGetUnauthorized with default headers values -func NewClassificationsGetUnauthorized() *ClassificationsGetUnauthorized { - return &ClassificationsGetUnauthorized{} -} - -/* -ClassificationsGetUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type ClassificationsGetUnauthorized struct { -} - -// IsSuccess returns true when this classifications get unauthorized response has a 2xx status code -func (o *ClassificationsGetUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this classifications get unauthorized response has a 3xx status code -func (o *ClassificationsGetUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this classifications get unauthorized response has a 4xx status code -func (o *ClassificationsGetUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this classifications get unauthorized response has a 5xx status code -func (o *ClassificationsGetUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this classifications get unauthorized response a status code equal to that given -func (o *ClassificationsGetUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the classifications get unauthorized response -func (o *ClassificationsGetUnauthorized) Code() int { - return 401 -} - -func (o *ClassificationsGetUnauthorized) Error() string { - return fmt.Sprintf("[GET /classifications/{id}][%d] classificationsGetUnauthorized ", 401) -} - -func (o *ClassificationsGetUnauthorized) String() string { - return fmt.Sprintf("[GET /classifications/{id}][%d] classificationsGetUnauthorized ", 401) -} - -func (o *ClassificationsGetUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewClassificationsGetForbidden creates a ClassificationsGetForbidden with default headers values -func NewClassificationsGetForbidden() *ClassificationsGetForbidden { - return &ClassificationsGetForbidden{} -} - -/* -ClassificationsGetForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type ClassificationsGetForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this classifications get forbidden response has a 2xx status code -func (o *ClassificationsGetForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this classifications get forbidden response has a 3xx status code -func (o *ClassificationsGetForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this classifications get forbidden response has a 4xx status code -func (o *ClassificationsGetForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this classifications get forbidden response has a 5xx status code -func (o *ClassificationsGetForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this classifications get forbidden response a status code equal to that given -func (o *ClassificationsGetForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the classifications get forbidden response -func (o *ClassificationsGetForbidden) Code() int { - return 403 -} - -func (o *ClassificationsGetForbidden) Error() string { - return fmt.Sprintf("[GET /classifications/{id}][%d] classificationsGetForbidden %+v", 403, o.Payload) -} - -func (o *ClassificationsGetForbidden) String() string { - return fmt.Sprintf("[GET /classifications/{id}][%d] classificationsGetForbidden %+v", 403, o.Payload) -} - -func (o *ClassificationsGetForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ClassificationsGetForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewClassificationsGetNotFound creates a ClassificationsGetNotFound with default headers values -func NewClassificationsGetNotFound() *ClassificationsGetNotFound { - return &ClassificationsGetNotFound{} -} - -/* -ClassificationsGetNotFound describes a response with status code 404, with default header values. - -Not Found - Classification does not exist -*/ -type ClassificationsGetNotFound struct { -} - -// IsSuccess returns true when this classifications get not found response has a 2xx status code -func (o *ClassificationsGetNotFound) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this classifications get not found response has a 3xx status code -func (o *ClassificationsGetNotFound) IsRedirect() bool { - return false -} - -// IsClientError returns true when this classifications get not found response has a 4xx status code -func (o *ClassificationsGetNotFound) IsClientError() bool { - return true -} - -// IsServerError returns true when this classifications get not found response has a 5xx status code -func (o *ClassificationsGetNotFound) IsServerError() bool { - return false -} - -// IsCode returns true when this classifications get not found response a status code equal to that given -func (o *ClassificationsGetNotFound) IsCode(code int) bool { - return code == 404 -} - -// Code gets the status code for the classifications get not found response -func (o *ClassificationsGetNotFound) Code() int { - return 404 -} - -func (o *ClassificationsGetNotFound) Error() string { - return fmt.Sprintf("[GET /classifications/{id}][%d] classificationsGetNotFound ", 404) -} - -func (o *ClassificationsGetNotFound) String() string { - return fmt.Sprintf("[GET /classifications/{id}][%d] classificationsGetNotFound ", 404) -} - -func (o *ClassificationsGetNotFound) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewClassificationsGetInternalServerError creates a ClassificationsGetInternalServerError with default headers values -func NewClassificationsGetInternalServerError() *ClassificationsGetInternalServerError { - return &ClassificationsGetInternalServerError{} -} - -/* -ClassificationsGetInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type ClassificationsGetInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this classifications get internal server error response has a 2xx status code -func (o *ClassificationsGetInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this classifications get internal server error response has a 3xx status code -func (o *ClassificationsGetInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this classifications get internal server error response has a 4xx status code -func (o *ClassificationsGetInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this classifications get internal server error response has a 5xx status code -func (o *ClassificationsGetInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this classifications get internal server error response a status code equal to that given -func (o *ClassificationsGetInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the classifications get internal server error response -func (o *ClassificationsGetInternalServerError) Code() int { - return 500 -} - -func (o *ClassificationsGetInternalServerError) Error() string { - return fmt.Sprintf("[GET /classifications/{id}][%d] classificationsGetInternalServerError %+v", 500, o.Payload) -} - -func (o *ClassificationsGetInternalServerError) String() string { - return fmt.Sprintf("[GET /classifications/{id}][%d] classificationsGetInternalServerError %+v", 500, o.Payload) -} - -func (o *ClassificationsGetInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ClassificationsGetInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/classifications/classifications_post_parameters.go b/client/classifications/classifications_post_parameters.go deleted file mode 100644 index 4bdf74c21f1a50fb5e05c58b09b28a9647e20745..0000000000000000000000000000000000000000 --- a/client/classifications/classifications_post_parameters.go +++ /dev/null @@ -1,164 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package classifications - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewClassificationsPostParams creates a new ClassificationsPostParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewClassificationsPostParams() *ClassificationsPostParams { - return &ClassificationsPostParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewClassificationsPostParamsWithTimeout creates a new ClassificationsPostParams object -// with the ability to set a timeout on a request. -func NewClassificationsPostParamsWithTimeout(timeout time.Duration) *ClassificationsPostParams { - return &ClassificationsPostParams{ - timeout: timeout, - } -} - -// NewClassificationsPostParamsWithContext creates a new ClassificationsPostParams object -// with the ability to set a context for a request. -func NewClassificationsPostParamsWithContext(ctx context.Context) *ClassificationsPostParams { - return &ClassificationsPostParams{ - Context: ctx, - } -} - -// NewClassificationsPostParamsWithHTTPClient creates a new ClassificationsPostParams object -// with the ability to set a custom HTTPClient for a request. -func NewClassificationsPostParamsWithHTTPClient(client *http.Client) *ClassificationsPostParams { - return &ClassificationsPostParams{ - HTTPClient: client, - } -} - -/* -ClassificationsPostParams contains all the parameters to send to the API endpoint - - for the classifications post operation. - - Typically these are written to a http.Request. -*/ -type ClassificationsPostParams struct { - - /* Params. - - parameters to start a classification - */ - Params *models.Classification - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the classifications post params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *ClassificationsPostParams) WithDefaults() *ClassificationsPostParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the classifications post params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *ClassificationsPostParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the classifications post params -func (o *ClassificationsPostParams) WithTimeout(timeout time.Duration) *ClassificationsPostParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the classifications post params -func (o *ClassificationsPostParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the classifications post params -func (o *ClassificationsPostParams) WithContext(ctx context.Context) *ClassificationsPostParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the classifications post params -func (o *ClassificationsPostParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the classifications post params -func (o *ClassificationsPostParams) WithHTTPClient(client *http.Client) *ClassificationsPostParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the classifications post params -func (o *ClassificationsPostParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithParams adds the params to the classifications post params -func (o *ClassificationsPostParams) WithParams(params *models.Classification) *ClassificationsPostParams { - o.SetParams(params) - return o -} - -// SetParams adds the params to the classifications post params -func (o *ClassificationsPostParams) SetParams(params *models.Classification) { - o.Params = params -} - -// WriteToRequest writes these params to a swagger request -func (o *ClassificationsPostParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - if o.Params != nil { - if err := r.SetBodyParam(o.Params); err != nil { - return err - } - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/classifications/classifications_post_responses.go b/client/classifications/classifications_post_responses.go deleted file mode 100644 index 6c426c858b62601d51eb0876b64a54f070560c14..0000000000000000000000000000000000000000 --- a/client/classifications/classifications_post_responses.go +++ /dev/null @@ -1,398 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package classifications - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// ClassificationsPostReader is a Reader for the ClassificationsPost structure. -type ClassificationsPostReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *ClassificationsPostReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 201: - result := NewClassificationsPostCreated() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 400: - result := NewClassificationsPostBadRequest() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 401: - result := NewClassificationsPostUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewClassificationsPostForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewClassificationsPostInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewClassificationsPostCreated creates a ClassificationsPostCreated with default headers values -func NewClassificationsPostCreated() *ClassificationsPostCreated { - return &ClassificationsPostCreated{} -} - -/* -ClassificationsPostCreated describes a response with status code 201, with default header values. - -Successfully started classification. -*/ -type ClassificationsPostCreated struct { - Payload *models.Classification -} - -// IsSuccess returns true when this classifications post created response has a 2xx status code -func (o *ClassificationsPostCreated) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this classifications post created response has a 3xx status code -func (o *ClassificationsPostCreated) IsRedirect() bool { - return false -} - -// IsClientError returns true when this classifications post created response has a 4xx status code -func (o *ClassificationsPostCreated) IsClientError() bool { - return false -} - -// IsServerError returns true when this classifications post created response has a 5xx status code -func (o *ClassificationsPostCreated) IsServerError() bool { - return false -} - -// IsCode returns true when this classifications post created response a status code equal to that given -func (o *ClassificationsPostCreated) IsCode(code int) bool { - return code == 201 -} - -// Code gets the status code for the classifications post created response -func (o *ClassificationsPostCreated) Code() int { - return 201 -} - -func (o *ClassificationsPostCreated) Error() string { - return fmt.Sprintf("[POST /classifications/][%d] classificationsPostCreated %+v", 201, o.Payload) -} - -func (o *ClassificationsPostCreated) String() string { - return fmt.Sprintf("[POST /classifications/][%d] classificationsPostCreated %+v", 201, o.Payload) -} - -func (o *ClassificationsPostCreated) GetPayload() *models.Classification { - return o.Payload -} - -func (o *ClassificationsPostCreated) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.Classification) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewClassificationsPostBadRequest creates a ClassificationsPostBadRequest with default headers values -func NewClassificationsPostBadRequest() *ClassificationsPostBadRequest { - return &ClassificationsPostBadRequest{} -} - -/* -ClassificationsPostBadRequest describes a response with status code 400, with default header values. - -Incorrect request -*/ -type ClassificationsPostBadRequest struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this classifications post bad request response has a 2xx status code -func (o *ClassificationsPostBadRequest) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this classifications post bad request response has a 3xx status code -func (o *ClassificationsPostBadRequest) IsRedirect() bool { - return false -} - -// IsClientError returns true when this classifications post bad request response has a 4xx status code -func (o *ClassificationsPostBadRequest) IsClientError() bool { - return true -} - -// IsServerError returns true when this classifications post bad request response has a 5xx status code -func (o *ClassificationsPostBadRequest) IsServerError() bool { - return false -} - -// IsCode returns true when this classifications post bad request response a status code equal to that given -func (o *ClassificationsPostBadRequest) IsCode(code int) bool { - return code == 400 -} - -// Code gets the status code for the classifications post bad request response -func (o *ClassificationsPostBadRequest) Code() int { - return 400 -} - -func (o *ClassificationsPostBadRequest) Error() string { - return fmt.Sprintf("[POST /classifications/][%d] classificationsPostBadRequest %+v", 400, o.Payload) -} - -func (o *ClassificationsPostBadRequest) String() string { - return fmt.Sprintf("[POST /classifications/][%d] classificationsPostBadRequest %+v", 400, o.Payload) -} - -func (o *ClassificationsPostBadRequest) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ClassificationsPostBadRequest) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewClassificationsPostUnauthorized creates a ClassificationsPostUnauthorized with default headers values -func NewClassificationsPostUnauthorized() *ClassificationsPostUnauthorized { - return &ClassificationsPostUnauthorized{} -} - -/* -ClassificationsPostUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type ClassificationsPostUnauthorized struct { -} - -// IsSuccess returns true when this classifications post unauthorized response has a 2xx status code -func (o *ClassificationsPostUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this classifications post unauthorized response has a 3xx status code -func (o *ClassificationsPostUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this classifications post unauthorized response has a 4xx status code -func (o *ClassificationsPostUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this classifications post unauthorized response has a 5xx status code -func (o *ClassificationsPostUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this classifications post unauthorized response a status code equal to that given -func (o *ClassificationsPostUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the classifications post unauthorized response -func (o *ClassificationsPostUnauthorized) Code() int { - return 401 -} - -func (o *ClassificationsPostUnauthorized) Error() string { - return fmt.Sprintf("[POST /classifications/][%d] classificationsPostUnauthorized ", 401) -} - -func (o *ClassificationsPostUnauthorized) String() string { - return fmt.Sprintf("[POST /classifications/][%d] classificationsPostUnauthorized ", 401) -} - -func (o *ClassificationsPostUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewClassificationsPostForbidden creates a ClassificationsPostForbidden with default headers values -func NewClassificationsPostForbidden() *ClassificationsPostForbidden { - return &ClassificationsPostForbidden{} -} - -/* -ClassificationsPostForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type ClassificationsPostForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this classifications post forbidden response has a 2xx status code -func (o *ClassificationsPostForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this classifications post forbidden response has a 3xx status code -func (o *ClassificationsPostForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this classifications post forbidden response has a 4xx status code -func (o *ClassificationsPostForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this classifications post forbidden response has a 5xx status code -func (o *ClassificationsPostForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this classifications post forbidden response a status code equal to that given -func (o *ClassificationsPostForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the classifications post forbidden response -func (o *ClassificationsPostForbidden) Code() int { - return 403 -} - -func (o *ClassificationsPostForbidden) Error() string { - return fmt.Sprintf("[POST /classifications/][%d] classificationsPostForbidden %+v", 403, o.Payload) -} - -func (o *ClassificationsPostForbidden) String() string { - return fmt.Sprintf("[POST /classifications/][%d] classificationsPostForbidden %+v", 403, o.Payload) -} - -func (o *ClassificationsPostForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ClassificationsPostForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewClassificationsPostInternalServerError creates a ClassificationsPostInternalServerError with default headers values -func NewClassificationsPostInternalServerError() *ClassificationsPostInternalServerError { - return &ClassificationsPostInternalServerError{} -} - -/* -ClassificationsPostInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type ClassificationsPostInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this classifications post internal server error response has a 2xx status code -func (o *ClassificationsPostInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this classifications post internal server error response has a 3xx status code -func (o *ClassificationsPostInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this classifications post internal server error response has a 4xx status code -func (o *ClassificationsPostInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this classifications post internal server error response has a 5xx status code -func (o *ClassificationsPostInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this classifications post internal server error response a status code equal to that given -func (o *ClassificationsPostInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the classifications post internal server error response -func (o *ClassificationsPostInternalServerError) Code() int { - return 500 -} - -func (o *ClassificationsPostInternalServerError) Error() string { - return fmt.Sprintf("[POST /classifications/][%d] classificationsPostInternalServerError %+v", 500, o.Payload) -} - -func (o *ClassificationsPostInternalServerError) String() string { - return fmt.Sprintf("[POST /classifications/][%d] classificationsPostInternalServerError %+v", 500, o.Payload) -} - -func (o *ClassificationsPostInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ClassificationsPostInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/graphql/graphql_batch_parameters.go b/client/graphql/graphql_batch_parameters.go deleted file mode 100644 index 041d4152a737a526d6e340bd514a82f8e0787d4c..0000000000000000000000000000000000000000 --- a/client/graphql/graphql_batch_parameters.go +++ /dev/null @@ -1,164 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package graphql - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewGraphqlBatchParams creates a new GraphqlBatchParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewGraphqlBatchParams() *GraphqlBatchParams { - return &GraphqlBatchParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewGraphqlBatchParamsWithTimeout creates a new GraphqlBatchParams object -// with the ability to set a timeout on a request. -func NewGraphqlBatchParamsWithTimeout(timeout time.Duration) *GraphqlBatchParams { - return &GraphqlBatchParams{ - timeout: timeout, - } -} - -// NewGraphqlBatchParamsWithContext creates a new GraphqlBatchParams object -// with the ability to set a context for a request. -func NewGraphqlBatchParamsWithContext(ctx context.Context) *GraphqlBatchParams { - return &GraphqlBatchParams{ - Context: ctx, - } -} - -// NewGraphqlBatchParamsWithHTTPClient creates a new GraphqlBatchParams object -// with the ability to set a custom HTTPClient for a request. -func NewGraphqlBatchParamsWithHTTPClient(client *http.Client) *GraphqlBatchParams { - return &GraphqlBatchParams{ - HTTPClient: client, - } -} - -/* -GraphqlBatchParams contains all the parameters to send to the API endpoint - - for the graphql batch operation. - - Typically these are written to a http.Request. -*/ -type GraphqlBatchParams struct { - - /* Body. - - The GraphQL queries. - */ - Body models.GraphQLQueries - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the graphql batch params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *GraphqlBatchParams) WithDefaults() *GraphqlBatchParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the graphql batch params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *GraphqlBatchParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the graphql batch params -func (o *GraphqlBatchParams) WithTimeout(timeout time.Duration) *GraphqlBatchParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the graphql batch params -func (o *GraphqlBatchParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the graphql batch params -func (o *GraphqlBatchParams) WithContext(ctx context.Context) *GraphqlBatchParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the graphql batch params -func (o *GraphqlBatchParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the graphql batch params -func (o *GraphqlBatchParams) WithHTTPClient(client *http.Client) *GraphqlBatchParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the graphql batch params -func (o *GraphqlBatchParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithBody adds the body to the graphql batch params -func (o *GraphqlBatchParams) WithBody(body models.GraphQLQueries) *GraphqlBatchParams { - o.SetBody(body) - return o -} - -// SetBody adds the body to the graphql batch params -func (o *GraphqlBatchParams) SetBody(body models.GraphQLQueries) { - o.Body = body -} - -// WriteToRequest writes these params to a swagger request -func (o *GraphqlBatchParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - if o.Body != nil { - if err := r.SetBodyParam(o.Body); err != nil { - return err - } - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/graphql/graphql_batch_responses.go b/client/graphql/graphql_batch_responses.go deleted file mode 100644 index a2fa16ebb517bd9c0faea672e0101b967d5fb641..0000000000000000000000000000000000000000 --- a/client/graphql/graphql_batch_responses.go +++ /dev/null @@ -1,396 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package graphql - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// GraphqlBatchReader is a Reader for the GraphqlBatch structure. -type GraphqlBatchReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *GraphqlBatchReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 200: - result := NewGraphqlBatchOK() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 401: - result := NewGraphqlBatchUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewGraphqlBatchForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 422: - result := NewGraphqlBatchUnprocessableEntity() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewGraphqlBatchInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewGraphqlBatchOK creates a GraphqlBatchOK with default headers values -func NewGraphqlBatchOK() *GraphqlBatchOK { - return &GraphqlBatchOK{} -} - -/* -GraphqlBatchOK describes a response with status code 200, with default header values. - -Successful query (with select). -*/ -type GraphqlBatchOK struct { - Payload models.GraphQLResponses -} - -// IsSuccess returns true when this graphql batch o k response has a 2xx status code -func (o *GraphqlBatchOK) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this graphql batch o k response has a 3xx status code -func (o *GraphqlBatchOK) IsRedirect() bool { - return false -} - -// IsClientError returns true when this graphql batch o k response has a 4xx status code -func (o *GraphqlBatchOK) IsClientError() bool { - return false -} - -// IsServerError returns true when this graphql batch o k response has a 5xx status code -func (o *GraphqlBatchOK) IsServerError() bool { - return false -} - -// IsCode returns true when this graphql batch o k response a status code equal to that given -func (o *GraphqlBatchOK) IsCode(code int) bool { - return code == 200 -} - -// Code gets the status code for the graphql batch o k response -func (o *GraphqlBatchOK) Code() int { - return 200 -} - -func (o *GraphqlBatchOK) Error() string { - return fmt.Sprintf("[POST /graphql/batch][%d] graphqlBatchOK %+v", 200, o.Payload) -} - -func (o *GraphqlBatchOK) String() string { - return fmt.Sprintf("[POST /graphql/batch][%d] graphqlBatchOK %+v", 200, o.Payload) -} - -func (o *GraphqlBatchOK) GetPayload() models.GraphQLResponses { - return o.Payload -} - -func (o *GraphqlBatchOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - // response payload - if err := consumer.Consume(response.Body(), &o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewGraphqlBatchUnauthorized creates a GraphqlBatchUnauthorized with default headers values -func NewGraphqlBatchUnauthorized() *GraphqlBatchUnauthorized { - return &GraphqlBatchUnauthorized{} -} - -/* -GraphqlBatchUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type GraphqlBatchUnauthorized struct { -} - -// IsSuccess returns true when this graphql batch unauthorized response has a 2xx status code -func (o *GraphqlBatchUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this graphql batch unauthorized response has a 3xx status code -func (o *GraphqlBatchUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this graphql batch unauthorized response has a 4xx status code -func (o *GraphqlBatchUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this graphql batch unauthorized response has a 5xx status code -func (o *GraphqlBatchUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this graphql batch unauthorized response a status code equal to that given -func (o *GraphqlBatchUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the graphql batch unauthorized response -func (o *GraphqlBatchUnauthorized) Code() int { - return 401 -} - -func (o *GraphqlBatchUnauthorized) Error() string { - return fmt.Sprintf("[POST /graphql/batch][%d] graphqlBatchUnauthorized ", 401) -} - -func (o *GraphqlBatchUnauthorized) String() string { - return fmt.Sprintf("[POST /graphql/batch][%d] graphqlBatchUnauthorized ", 401) -} - -func (o *GraphqlBatchUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewGraphqlBatchForbidden creates a GraphqlBatchForbidden with default headers values -func NewGraphqlBatchForbidden() *GraphqlBatchForbidden { - return &GraphqlBatchForbidden{} -} - -/* -GraphqlBatchForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type GraphqlBatchForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this graphql batch forbidden response has a 2xx status code -func (o *GraphqlBatchForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this graphql batch forbidden response has a 3xx status code -func (o *GraphqlBatchForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this graphql batch forbidden response has a 4xx status code -func (o *GraphqlBatchForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this graphql batch forbidden response has a 5xx status code -func (o *GraphqlBatchForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this graphql batch forbidden response a status code equal to that given -func (o *GraphqlBatchForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the graphql batch forbidden response -func (o *GraphqlBatchForbidden) Code() int { - return 403 -} - -func (o *GraphqlBatchForbidden) Error() string { - return fmt.Sprintf("[POST /graphql/batch][%d] graphqlBatchForbidden %+v", 403, o.Payload) -} - -func (o *GraphqlBatchForbidden) String() string { - return fmt.Sprintf("[POST /graphql/batch][%d] graphqlBatchForbidden %+v", 403, o.Payload) -} - -func (o *GraphqlBatchForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *GraphqlBatchForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewGraphqlBatchUnprocessableEntity creates a GraphqlBatchUnprocessableEntity with default headers values -func NewGraphqlBatchUnprocessableEntity() *GraphqlBatchUnprocessableEntity { - return &GraphqlBatchUnprocessableEntity{} -} - -/* -GraphqlBatchUnprocessableEntity describes a response with status code 422, with default header values. - -Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file? -*/ -type GraphqlBatchUnprocessableEntity struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this graphql batch unprocessable entity response has a 2xx status code -func (o *GraphqlBatchUnprocessableEntity) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this graphql batch unprocessable entity response has a 3xx status code -func (o *GraphqlBatchUnprocessableEntity) IsRedirect() bool { - return false -} - -// IsClientError returns true when this graphql batch unprocessable entity response has a 4xx status code -func (o *GraphqlBatchUnprocessableEntity) IsClientError() bool { - return true -} - -// IsServerError returns true when this graphql batch unprocessable entity response has a 5xx status code -func (o *GraphqlBatchUnprocessableEntity) IsServerError() bool { - return false -} - -// IsCode returns true when this graphql batch unprocessable entity response a status code equal to that given -func (o *GraphqlBatchUnprocessableEntity) IsCode(code int) bool { - return code == 422 -} - -// Code gets the status code for the graphql batch unprocessable entity response -func (o *GraphqlBatchUnprocessableEntity) Code() int { - return 422 -} - -func (o *GraphqlBatchUnprocessableEntity) Error() string { - return fmt.Sprintf("[POST /graphql/batch][%d] graphqlBatchUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *GraphqlBatchUnprocessableEntity) String() string { - return fmt.Sprintf("[POST /graphql/batch][%d] graphqlBatchUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *GraphqlBatchUnprocessableEntity) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *GraphqlBatchUnprocessableEntity) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewGraphqlBatchInternalServerError creates a GraphqlBatchInternalServerError with default headers values -func NewGraphqlBatchInternalServerError() *GraphqlBatchInternalServerError { - return &GraphqlBatchInternalServerError{} -} - -/* -GraphqlBatchInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type GraphqlBatchInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this graphql batch internal server error response has a 2xx status code -func (o *GraphqlBatchInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this graphql batch internal server error response has a 3xx status code -func (o *GraphqlBatchInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this graphql batch internal server error response has a 4xx status code -func (o *GraphqlBatchInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this graphql batch internal server error response has a 5xx status code -func (o *GraphqlBatchInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this graphql batch internal server error response a status code equal to that given -func (o *GraphqlBatchInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the graphql batch internal server error response -func (o *GraphqlBatchInternalServerError) Code() int { - return 500 -} - -func (o *GraphqlBatchInternalServerError) Error() string { - return fmt.Sprintf("[POST /graphql/batch][%d] graphqlBatchInternalServerError %+v", 500, o.Payload) -} - -func (o *GraphqlBatchInternalServerError) String() string { - return fmt.Sprintf("[POST /graphql/batch][%d] graphqlBatchInternalServerError %+v", 500, o.Payload) -} - -func (o *GraphqlBatchInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *GraphqlBatchInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/graphql/graphql_client.go b/client/graphql/graphql_client.go deleted file mode 100644 index a332a12b3203ee7c6a6624c779b6b1e52959aa39..0000000000000000000000000000000000000000 --- a/client/graphql/graphql_client.go +++ /dev/null @@ -1,136 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package graphql - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" -) - -// New creates a new graphql API client. -func New(transport runtime.ClientTransport, formats strfmt.Registry) ClientService { - return &Client{transport: transport, formats: formats} -} - -/* -Client for graphql API -*/ -type Client struct { - transport runtime.ClientTransport - formats strfmt.Registry -} - -// ClientOption is the option for Client methods -type ClientOption func(*runtime.ClientOperation) - -// ClientService is the interface for Client methods -type ClientService interface { - GraphqlBatch(params *GraphqlBatchParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*GraphqlBatchOK, error) - - GraphqlPost(params *GraphqlPostParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*GraphqlPostOK, error) - - SetTransport(transport runtime.ClientTransport) -} - -/* -GraphqlBatch gets a response based on graph q l - -Perform a batched GraphQL query -*/ -func (a *Client) GraphqlBatch(params *GraphqlBatchParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*GraphqlBatchOK, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewGraphqlBatchParams() - } - op := &runtime.ClientOperation{ - ID: "graphql.batch", - Method: "POST", - PathPattern: "/graphql/batch", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &GraphqlBatchReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*GraphqlBatchOK) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for graphql.batch: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -/* -GraphqlPost gets a response based on graph q l - -Get an object based on GraphQL -*/ -func (a *Client) GraphqlPost(params *GraphqlPostParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*GraphqlPostOK, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewGraphqlPostParams() - } - op := &runtime.ClientOperation{ - ID: "graphql.post", - Method: "POST", - PathPattern: "/graphql", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &GraphqlPostReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*GraphqlPostOK) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for graphql.post: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -// SetTransport changes the transport on the client -func (a *Client) SetTransport(transport runtime.ClientTransport) { - a.transport = transport -} diff --git a/client/graphql/graphql_post_parameters.go b/client/graphql/graphql_post_parameters.go deleted file mode 100644 index d0824e88c05ffee3196d324297ecf04452f6ce80..0000000000000000000000000000000000000000 --- a/client/graphql/graphql_post_parameters.go +++ /dev/null @@ -1,164 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package graphql - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewGraphqlPostParams creates a new GraphqlPostParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewGraphqlPostParams() *GraphqlPostParams { - return &GraphqlPostParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewGraphqlPostParamsWithTimeout creates a new GraphqlPostParams object -// with the ability to set a timeout on a request. -func NewGraphqlPostParamsWithTimeout(timeout time.Duration) *GraphqlPostParams { - return &GraphqlPostParams{ - timeout: timeout, - } -} - -// NewGraphqlPostParamsWithContext creates a new GraphqlPostParams object -// with the ability to set a context for a request. -func NewGraphqlPostParamsWithContext(ctx context.Context) *GraphqlPostParams { - return &GraphqlPostParams{ - Context: ctx, - } -} - -// NewGraphqlPostParamsWithHTTPClient creates a new GraphqlPostParams object -// with the ability to set a custom HTTPClient for a request. -func NewGraphqlPostParamsWithHTTPClient(client *http.Client) *GraphqlPostParams { - return &GraphqlPostParams{ - HTTPClient: client, - } -} - -/* -GraphqlPostParams contains all the parameters to send to the API endpoint - - for the graphql post operation. - - Typically these are written to a http.Request. -*/ -type GraphqlPostParams struct { - - /* Body. - - The GraphQL query request parameters. - */ - Body *models.GraphQLQuery - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the graphql post params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *GraphqlPostParams) WithDefaults() *GraphqlPostParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the graphql post params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *GraphqlPostParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the graphql post params -func (o *GraphqlPostParams) WithTimeout(timeout time.Duration) *GraphqlPostParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the graphql post params -func (o *GraphqlPostParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the graphql post params -func (o *GraphqlPostParams) WithContext(ctx context.Context) *GraphqlPostParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the graphql post params -func (o *GraphqlPostParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the graphql post params -func (o *GraphqlPostParams) WithHTTPClient(client *http.Client) *GraphqlPostParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the graphql post params -func (o *GraphqlPostParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithBody adds the body to the graphql post params -func (o *GraphqlPostParams) WithBody(body *models.GraphQLQuery) *GraphqlPostParams { - o.SetBody(body) - return o -} - -// SetBody adds the body to the graphql post params -func (o *GraphqlPostParams) SetBody(body *models.GraphQLQuery) { - o.Body = body -} - -// WriteToRequest writes these params to a swagger request -func (o *GraphqlPostParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - if o.Body != nil { - if err := r.SetBodyParam(o.Body); err != nil { - return err - } - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/graphql/graphql_post_responses.go b/client/graphql/graphql_post_responses.go deleted file mode 100644 index b2f8860356b25a5ddf1d9c94361e13d5338ff71c..0000000000000000000000000000000000000000 --- a/client/graphql/graphql_post_responses.go +++ /dev/null @@ -1,398 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package graphql - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// GraphqlPostReader is a Reader for the GraphqlPost structure. -type GraphqlPostReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *GraphqlPostReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 200: - result := NewGraphqlPostOK() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 401: - result := NewGraphqlPostUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewGraphqlPostForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 422: - result := NewGraphqlPostUnprocessableEntity() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewGraphqlPostInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewGraphqlPostOK creates a GraphqlPostOK with default headers values -func NewGraphqlPostOK() *GraphqlPostOK { - return &GraphqlPostOK{} -} - -/* -GraphqlPostOK describes a response with status code 200, with default header values. - -Successful query (with select). -*/ -type GraphqlPostOK struct { - Payload *models.GraphQLResponse -} - -// IsSuccess returns true when this graphql post o k response has a 2xx status code -func (o *GraphqlPostOK) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this graphql post o k response has a 3xx status code -func (o *GraphqlPostOK) IsRedirect() bool { - return false -} - -// IsClientError returns true when this graphql post o k response has a 4xx status code -func (o *GraphqlPostOK) IsClientError() bool { - return false -} - -// IsServerError returns true when this graphql post o k response has a 5xx status code -func (o *GraphqlPostOK) IsServerError() bool { - return false -} - -// IsCode returns true when this graphql post o k response a status code equal to that given -func (o *GraphqlPostOK) IsCode(code int) bool { - return code == 200 -} - -// Code gets the status code for the graphql post o k response -func (o *GraphqlPostOK) Code() int { - return 200 -} - -func (o *GraphqlPostOK) Error() string { - return fmt.Sprintf("[POST /graphql][%d] graphqlPostOK %+v", 200, o.Payload) -} - -func (o *GraphqlPostOK) String() string { - return fmt.Sprintf("[POST /graphql][%d] graphqlPostOK %+v", 200, o.Payload) -} - -func (o *GraphqlPostOK) GetPayload() *models.GraphQLResponse { - return o.Payload -} - -func (o *GraphqlPostOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.GraphQLResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewGraphqlPostUnauthorized creates a GraphqlPostUnauthorized with default headers values -func NewGraphqlPostUnauthorized() *GraphqlPostUnauthorized { - return &GraphqlPostUnauthorized{} -} - -/* -GraphqlPostUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type GraphqlPostUnauthorized struct { -} - -// IsSuccess returns true when this graphql post unauthorized response has a 2xx status code -func (o *GraphqlPostUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this graphql post unauthorized response has a 3xx status code -func (o *GraphqlPostUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this graphql post unauthorized response has a 4xx status code -func (o *GraphqlPostUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this graphql post unauthorized response has a 5xx status code -func (o *GraphqlPostUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this graphql post unauthorized response a status code equal to that given -func (o *GraphqlPostUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the graphql post unauthorized response -func (o *GraphqlPostUnauthorized) Code() int { - return 401 -} - -func (o *GraphqlPostUnauthorized) Error() string { - return fmt.Sprintf("[POST /graphql][%d] graphqlPostUnauthorized ", 401) -} - -func (o *GraphqlPostUnauthorized) String() string { - return fmt.Sprintf("[POST /graphql][%d] graphqlPostUnauthorized ", 401) -} - -func (o *GraphqlPostUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewGraphqlPostForbidden creates a GraphqlPostForbidden with default headers values -func NewGraphqlPostForbidden() *GraphqlPostForbidden { - return &GraphqlPostForbidden{} -} - -/* -GraphqlPostForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type GraphqlPostForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this graphql post forbidden response has a 2xx status code -func (o *GraphqlPostForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this graphql post forbidden response has a 3xx status code -func (o *GraphqlPostForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this graphql post forbidden response has a 4xx status code -func (o *GraphqlPostForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this graphql post forbidden response has a 5xx status code -func (o *GraphqlPostForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this graphql post forbidden response a status code equal to that given -func (o *GraphqlPostForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the graphql post forbidden response -func (o *GraphqlPostForbidden) Code() int { - return 403 -} - -func (o *GraphqlPostForbidden) Error() string { - return fmt.Sprintf("[POST /graphql][%d] graphqlPostForbidden %+v", 403, o.Payload) -} - -func (o *GraphqlPostForbidden) String() string { - return fmt.Sprintf("[POST /graphql][%d] graphqlPostForbidden %+v", 403, o.Payload) -} - -func (o *GraphqlPostForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *GraphqlPostForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewGraphqlPostUnprocessableEntity creates a GraphqlPostUnprocessableEntity with default headers values -func NewGraphqlPostUnprocessableEntity() *GraphqlPostUnprocessableEntity { - return &GraphqlPostUnprocessableEntity{} -} - -/* -GraphqlPostUnprocessableEntity describes a response with status code 422, with default header values. - -Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file? -*/ -type GraphqlPostUnprocessableEntity struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this graphql post unprocessable entity response has a 2xx status code -func (o *GraphqlPostUnprocessableEntity) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this graphql post unprocessable entity response has a 3xx status code -func (o *GraphqlPostUnprocessableEntity) IsRedirect() bool { - return false -} - -// IsClientError returns true when this graphql post unprocessable entity response has a 4xx status code -func (o *GraphqlPostUnprocessableEntity) IsClientError() bool { - return true -} - -// IsServerError returns true when this graphql post unprocessable entity response has a 5xx status code -func (o *GraphqlPostUnprocessableEntity) IsServerError() bool { - return false -} - -// IsCode returns true when this graphql post unprocessable entity response a status code equal to that given -func (o *GraphqlPostUnprocessableEntity) IsCode(code int) bool { - return code == 422 -} - -// Code gets the status code for the graphql post unprocessable entity response -func (o *GraphqlPostUnprocessableEntity) Code() int { - return 422 -} - -func (o *GraphqlPostUnprocessableEntity) Error() string { - return fmt.Sprintf("[POST /graphql][%d] graphqlPostUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *GraphqlPostUnprocessableEntity) String() string { - return fmt.Sprintf("[POST /graphql][%d] graphqlPostUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *GraphqlPostUnprocessableEntity) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *GraphqlPostUnprocessableEntity) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewGraphqlPostInternalServerError creates a GraphqlPostInternalServerError with default headers values -func NewGraphqlPostInternalServerError() *GraphqlPostInternalServerError { - return &GraphqlPostInternalServerError{} -} - -/* -GraphqlPostInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type GraphqlPostInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this graphql post internal server error response has a 2xx status code -func (o *GraphqlPostInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this graphql post internal server error response has a 3xx status code -func (o *GraphqlPostInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this graphql post internal server error response has a 4xx status code -func (o *GraphqlPostInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this graphql post internal server error response has a 5xx status code -func (o *GraphqlPostInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this graphql post internal server error response a status code equal to that given -func (o *GraphqlPostInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the graphql post internal server error response -func (o *GraphqlPostInternalServerError) Code() int { - return 500 -} - -func (o *GraphqlPostInternalServerError) Error() string { - return fmt.Sprintf("[POST /graphql][%d] graphqlPostInternalServerError %+v", 500, o.Payload) -} - -func (o *GraphqlPostInternalServerError) String() string { - return fmt.Sprintf("[POST /graphql][%d] graphqlPostInternalServerError %+v", 500, o.Payload) -} - -func (o *GraphqlPostInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *GraphqlPostInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/meta/meta_client.go b/client/meta/meta_client.go deleted file mode 100644 index 088f88962312328a991185884c78fd851d3d3d3b..0000000000000000000000000000000000000000 --- a/client/meta/meta_client.go +++ /dev/null @@ -1,93 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package meta - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" -) - -// New creates a new meta API client. -func New(transport runtime.ClientTransport, formats strfmt.Registry) ClientService { - return &Client{transport: transport, formats: formats} -} - -/* -Client for meta API -*/ -type Client struct { - transport runtime.ClientTransport - formats strfmt.Registry -} - -// ClientOption is the option for Client methods -type ClientOption func(*runtime.ClientOperation) - -// ClientService is the interface for Client methods -type ClientService interface { - MetaGet(params *MetaGetParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*MetaGetOK, error) - - SetTransport(transport runtime.ClientTransport) -} - -/* -MetaGet returns meta information of the current weaviate instance - -Gives meta information about the server and can be used to provide information to another Weaviate instance that wants to interact with the current instance. -*/ -func (a *Client) MetaGet(params *MetaGetParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*MetaGetOK, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewMetaGetParams() - } - op := &runtime.ClientOperation{ - ID: "meta.get", - Method: "GET", - PathPattern: "/meta", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &MetaGetReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*MetaGetOK) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for meta.get: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -// SetTransport changes the transport on the client -func (a *Client) SetTransport(transport runtime.ClientTransport) { - a.transport = transport -} diff --git a/client/meta/meta_get_parameters.go b/client/meta/meta_get_parameters.go deleted file mode 100644 index 238d0f11df8209aa7ba5e246f69c377a3af99eb1..0000000000000000000000000000000000000000 --- a/client/meta/meta_get_parameters.go +++ /dev/null @@ -1,139 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package meta - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" -) - -// NewMetaGetParams creates a new MetaGetParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewMetaGetParams() *MetaGetParams { - return &MetaGetParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewMetaGetParamsWithTimeout creates a new MetaGetParams object -// with the ability to set a timeout on a request. -func NewMetaGetParamsWithTimeout(timeout time.Duration) *MetaGetParams { - return &MetaGetParams{ - timeout: timeout, - } -} - -// NewMetaGetParamsWithContext creates a new MetaGetParams object -// with the ability to set a context for a request. -func NewMetaGetParamsWithContext(ctx context.Context) *MetaGetParams { - return &MetaGetParams{ - Context: ctx, - } -} - -// NewMetaGetParamsWithHTTPClient creates a new MetaGetParams object -// with the ability to set a custom HTTPClient for a request. -func NewMetaGetParamsWithHTTPClient(client *http.Client) *MetaGetParams { - return &MetaGetParams{ - HTTPClient: client, - } -} - -/* -MetaGetParams contains all the parameters to send to the API endpoint - - for the meta get operation. - - Typically these are written to a http.Request. -*/ -type MetaGetParams struct { - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the meta get params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *MetaGetParams) WithDefaults() *MetaGetParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the meta get params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *MetaGetParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the meta get params -func (o *MetaGetParams) WithTimeout(timeout time.Duration) *MetaGetParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the meta get params -func (o *MetaGetParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the meta get params -func (o *MetaGetParams) WithContext(ctx context.Context) *MetaGetParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the meta get params -func (o *MetaGetParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the meta get params -func (o *MetaGetParams) WithHTTPClient(client *http.Client) *MetaGetParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the meta get params -func (o *MetaGetParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WriteToRequest writes these params to a swagger request -func (o *MetaGetParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/meta/meta_get_responses.go b/client/meta/meta_get_responses.go deleted file mode 100644 index 443fabc7b2c529975c934d6c7f95741400707cab..0000000000000000000000000000000000000000 --- a/client/meta/meta_get_responses.go +++ /dev/null @@ -1,324 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package meta - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// MetaGetReader is a Reader for the MetaGet structure. -type MetaGetReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *MetaGetReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 200: - result := NewMetaGetOK() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 401: - result := NewMetaGetUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewMetaGetForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewMetaGetInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewMetaGetOK creates a MetaGetOK with default headers values -func NewMetaGetOK() *MetaGetOK { - return &MetaGetOK{} -} - -/* -MetaGetOK describes a response with status code 200, with default header values. - -Successful response. -*/ -type MetaGetOK struct { - Payload *models.Meta -} - -// IsSuccess returns true when this meta get o k response has a 2xx status code -func (o *MetaGetOK) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this meta get o k response has a 3xx status code -func (o *MetaGetOK) IsRedirect() bool { - return false -} - -// IsClientError returns true when this meta get o k response has a 4xx status code -func (o *MetaGetOK) IsClientError() bool { - return false -} - -// IsServerError returns true when this meta get o k response has a 5xx status code -func (o *MetaGetOK) IsServerError() bool { - return false -} - -// IsCode returns true when this meta get o k response a status code equal to that given -func (o *MetaGetOK) IsCode(code int) bool { - return code == 200 -} - -// Code gets the status code for the meta get o k response -func (o *MetaGetOK) Code() int { - return 200 -} - -func (o *MetaGetOK) Error() string { - return fmt.Sprintf("[GET /meta][%d] metaGetOK %+v", 200, o.Payload) -} - -func (o *MetaGetOK) String() string { - return fmt.Sprintf("[GET /meta][%d] metaGetOK %+v", 200, o.Payload) -} - -func (o *MetaGetOK) GetPayload() *models.Meta { - return o.Payload -} - -func (o *MetaGetOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.Meta) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewMetaGetUnauthorized creates a MetaGetUnauthorized with default headers values -func NewMetaGetUnauthorized() *MetaGetUnauthorized { - return &MetaGetUnauthorized{} -} - -/* -MetaGetUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type MetaGetUnauthorized struct { -} - -// IsSuccess returns true when this meta get unauthorized response has a 2xx status code -func (o *MetaGetUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this meta get unauthorized response has a 3xx status code -func (o *MetaGetUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this meta get unauthorized response has a 4xx status code -func (o *MetaGetUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this meta get unauthorized response has a 5xx status code -func (o *MetaGetUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this meta get unauthorized response a status code equal to that given -func (o *MetaGetUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the meta get unauthorized response -func (o *MetaGetUnauthorized) Code() int { - return 401 -} - -func (o *MetaGetUnauthorized) Error() string { - return fmt.Sprintf("[GET /meta][%d] metaGetUnauthorized ", 401) -} - -func (o *MetaGetUnauthorized) String() string { - return fmt.Sprintf("[GET /meta][%d] metaGetUnauthorized ", 401) -} - -func (o *MetaGetUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewMetaGetForbidden creates a MetaGetForbidden with default headers values -func NewMetaGetForbidden() *MetaGetForbidden { - return &MetaGetForbidden{} -} - -/* -MetaGetForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type MetaGetForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this meta get forbidden response has a 2xx status code -func (o *MetaGetForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this meta get forbidden response has a 3xx status code -func (o *MetaGetForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this meta get forbidden response has a 4xx status code -func (o *MetaGetForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this meta get forbidden response has a 5xx status code -func (o *MetaGetForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this meta get forbidden response a status code equal to that given -func (o *MetaGetForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the meta get forbidden response -func (o *MetaGetForbidden) Code() int { - return 403 -} - -func (o *MetaGetForbidden) Error() string { - return fmt.Sprintf("[GET /meta][%d] metaGetForbidden %+v", 403, o.Payload) -} - -func (o *MetaGetForbidden) String() string { - return fmt.Sprintf("[GET /meta][%d] metaGetForbidden %+v", 403, o.Payload) -} - -func (o *MetaGetForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *MetaGetForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewMetaGetInternalServerError creates a MetaGetInternalServerError with default headers values -func NewMetaGetInternalServerError() *MetaGetInternalServerError { - return &MetaGetInternalServerError{} -} - -/* -MetaGetInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type MetaGetInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this meta get internal server error response has a 2xx status code -func (o *MetaGetInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this meta get internal server error response has a 3xx status code -func (o *MetaGetInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this meta get internal server error response has a 4xx status code -func (o *MetaGetInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this meta get internal server error response has a 5xx status code -func (o *MetaGetInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this meta get internal server error response a status code equal to that given -func (o *MetaGetInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the meta get internal server error response -func (o *MetaGetInternalServerError) Code() int { - return 500 -} - -func (o *MetaGetInternalServerError) Error() string { - return fmt.Sprintf("[GET /meta][%d] metaGetInternalServerError %+v", 500, o.Payload) -} - -func (o *MetaGetInternalServerError) String() string { - return fmt.Sprintf("[GET /meta][%d] metaGetInternalServerError %+v", 500, o.Payload) -} - -func (o *MetaGetInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *MetaGetInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/nodes/nodes_client.go b/client/nodes/nodes_client.go deleted file mode 100644 index 81fe59ff441f32c44283d3fcc63661d73ed80c91..0000000000000000000000000000000000000000 --- a/client/nodes/nodes_client.go +++ /dev/null @@ -1,132 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package nodes - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" -) - -// New creates a new nodes API client. -func New(transport runtime.ClientTransport, formats strfmt.Registry) ClientService { - return &Client{transport: transport, formats: formats} -} - -/* -Client for nodes API -*/ -type Client struct { - transport runtime.ClientTransport - formats strfmt.Registry -} - -// ClientOption is the option for Client methods -type ClientOption func(*runtime.ClientOperation) - -// ClientService is the interface for Client methods -type ClientService interface { - NodesGet(params *NodesGetParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*NodesGetOK, error) - - NodesGetClass(params *NodesGetClassParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*NodesGetClassOK, error) - - SetTransport(transport runtime.ClientTransport) -} - -/* -NodesGet Returns status of Weaviate DB. -*/ -func (a *Client) NodesGet(params *NodesGetParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*NodesGetOK, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewNodesGetParams() - } - op := &runtime.ClientOperation{ - ID: "nodes.get", - Method: "GET", - PathPattern: "/nodes", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &NodesGetReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*NodesGetOK) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for nodes.get: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -/* -NodesGetClass Returns status of Weaviate DB. -*/ -func (a *Client) NodesGetClass(params *NodesGetClassParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*NodesGetClassOK, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewNodesGetClassParams() - } - op := &runtime.ClientOperation{ - ID: "nodes.get.class", - Method: "GET", - PathPattern: "/nodes/{className}", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &NodesGetClassReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*NodesGetClassOK) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for nodes.get.class: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -// SetTransport changes the transport on the client -func (a *Client) SetTransport(transport runtime.ClientTransport) { - a.transport = transport -} diff --git a/client/nodes/nodes_get_class_parameters.go b/client/nodes/nodes_get_class_parameters.go deleted file mode 100644 index d3af80f2124eed888ae2e04f04b72fd828c7f027..0000000000000000000000000000000000000000 --- a/client/nodes/nodes_get_class_parameters.go +++ /dev/null @@ -1,206 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package nodes - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" -) - -// NewNodesGetClassParams creates a new NodesGetClassParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewNodesGetClassParams() *NodesGetClassParams { - return &NodesGetClassParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewNodesGetClassParamsWithTimeout creates a new NodesGetClassParams object -// with the ability to set a timeout on a request. -func NewNodesGetClassParamsWithTimeout(timeout time.Duration) *NodesGetClassParams { - return &NodesGetClassParams{ - timeout: timeout, - } -} - -// NewNodesGetClassParamsWithContext creates a new NodesGetClassParams object -// with the ability to set a context for a request. -func NewNodesGetClassParamsWithContext(ctx context.Context) *NodesGetClassParams { - return &NodesGetClassParams{ - Context: ctx, - } -} - -// NewNodesGetClassParamsWithHTTPClient creates a new NodesGetClassParams object -// with the ability to set a custom HTTPClient for a request. -func NewNodesGetClassParamsWithHTTPClient(client *http.Client) *NodesGetClassParams { - return &NodesGetClassParams{ - HTTPClient: client, - } -} - -/* -NodesGetClassParams contains all the parameters to send to the API endpoint - - for the nodes get class operation. - - Typically these are written to a http.Request. -*/ -type NodesGetClassParams struct { - - // ClassName. - ClassName string - - /* Output. - - Controls the verbosity of the output, possible values are: "minimal", "verbose". Defaults to "minimal". - - Default: "minimal" - */ - Output *string - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the nodes get class params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *NodesGetClassParams) WithDefaults() *NodesGetClassParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the nodes get class params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *NodesGetClassParams) SetDefaults() { - var ( - outputDefault = string("minimal") - ) - - val := NodesGetClassParams{ - Output: &outputDefault, - } - - val.timeout = o.timeout - val.Context = o.Context - val.HTTPClient = o.HTTPClient - *o = val -} - -// WithTimeout adds the timeout to the nodes get class params -func (o *NodesGetClassParams) WithTimeout(timeout time.Duration) *NodesGetClassParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the nodes get class params -func (o *NodesGetClassParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the nodes get class params -func (o *NodesGetClassParams) WithContext(ctx context.Context) *NodesGetClassParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the nodes get class params -func (o *NodesGetClassParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the nodes get class params -func (o *NodesGetClassParams) WithHTTPClient(client *http.Client) *NodesGetClassParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the nodes get class params -func (o *NodesGetClassParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithClassName adds the className to the nodes get class params -func (o *NodesGetClassParams) WithClassName(className string) *NodesGetClassParams { - o.SetClassName(className) - return o -} - -// SetClassName adds the className to the nodes get class params -func (o *NodesGetClassParams) SetClassName(className string) { - o.ClassName = className -} - -// WithOutput adds the output to the nodes get class params -func (o *NodesGetClassParams) WithOutput(output *string) *NodesGetClassParams { - o.SetOutput(output) - return o -} - -// SetOutput adds the output to the nodes get class params -func (o *NodesGetClassParams) SetOutput(output *string) { - o.Output = output -} - -// WriteToRequest writes these params to a swagger request -func (o *NodesGetClassParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - - // path param className - if err := r.SetPathParam("className", o.ClassName); err != nil { - return err - } - - if o.Output != nil { - - // query param output - var qrOutput string - - if o.Output != nil { - qrOutput = *o.Output - } - qOutput := qrOutput - if qOutput != "" { - - if err := r.SetQueryParam("output", qOutput); err != nil { - return err - } - } - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/nodes/nodes_get_class_responses.go b/client/nodes/nodes_get_class_responses.go deleted file mode 100644 index eb0f9b71ab1894388ee0e4df8bc9c329397f0d0a..0000000000000000000000000000000000000000 --- a/client/nodes/nodes_get_class_responses.go +++ /dev/null @@ -1,472 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package nodes - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// NodesGetClassReader is a Reader for the NodesGetClass structure. -type NodesGetClassReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *NodesGetClassReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 200: - result := NewNodesGetClassOK() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 401: - result := NewNodesGetClassUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewNodesGetClassForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 404: - result := NewNodesGetClassNotFound() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 422: - result := NewNodesGetClassUnprocessableEntity() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewNodesGetClassInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewNodesGetClassOK creates a NodesGetClassOK with default headers values -func NewNodesGetClassOK() *NodesGetClassOK { - return &NodesGetClassOK{} -} - -/* -NodesGetClassOK describes a response with status code 200, with default header values. - -Nodes status successfully returned -*/ -type NodesGetClassOK struct { - Payload *models.NodesStatusResponse -} - -// IsSuccess returns true when this nodes get class o k response has a 2xx status code -func (o *NodesGetClassOK) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this nodes get class o k response has a 3xx status code -func (o *NodesGetClassOK) IsRedirect() bool { - return false -} - -// IsClientError returns true when this nodes get class o k response has a 4xx status code -func (o *NodesGetClassOK) IsClientError() bool { - return false -} - -// IsServerError returns true when this nodes get class o k response has a 5xx status code -func (o *NodesGetClassOK) IsServerError() bool { - return false -} - -// IsCode returns true when this nodes get class o k response a status code equal to that given -func (o *NodesGetClassOK) IsCode(code int) bool { - return code == 200 -} - -// Code gets the status code for the nodes get class o k response -func (o *NodesGetClassOK) Code() int { - return 200 -} - -func (o *NodesGetClassOK) Error() string { - return fmt.Sprintf("[GET /nodes/{className}][%d] nodesGetClassOK %+v", 200, o.Payload) -} - -func (o *NodesGetClassOK) String() string { - return fmt.Sprintf("[GET /nodes/{className}][%d] nodesGetClassOK %+v", 200, o.Payload) -} - -func (o *NodesGetClassOK) GetPayload() *models.NodesStatusResponse { - return o.Payload -} - -func (o *NodesGetClassOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.NodesStatusResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewNodesGetClassUnauthorized creates a NodesGetClassUnauthorized with default headers values -func NewNodesGetClassUnauthorized() *NodesGetClassUnauthorized { - return &NodesGetClassUnauthorized{} -} - -/* -NodesGetClassUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type NodesGetClassUnauthorized struct { -} - -// IsSuccess returns true when this nodes get class unauthorized response has a 2xx status code -func (o *NodesGetClassUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this nodes get class unauthorized response has a 3xx status code -func (o *NodesGetClassUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this nodes get class unauthorized response has a 4xx status code -func (o *NodesGetClassUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this nodes get class unauthorized response has a 5xx status code -func (o *NodesGetClassUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this nodes get class unauthorized response a status code equal to that given -func (o *NodesGetClassUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the nodes get class unauthorized response -func (o *NodesGetClassUnauthorized) Code() int { - return 401 -} - -func (o *NodesGetClassUnauthorized) Error() string { - return fmt.Sprintf("[GET /nodes/{className}][%d] nodesGetClassUnauthorized ", 401) -} - -func (o *NodesGetClassUnauthorized) String() string { - return fmt.Sprintf("[GET /nodes/{className}][%d] nodesGetClassUnauthorized ", 401) -} - -func (o *NodesGetClassUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewNodesGetClassForbidden creates a NodesGetClassForbidden with default headers values -func NewNodesGetClassForbidden() *NodesGetClassForbidden { - return &NodesGetClassForbidden{} -} - -/* -NodesGetClassForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type NodesGetClassForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this nodes get class forbidden response has a 2xx status code -func (o *NodesGetClassForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this nodes get class forbidden response has a 3xx status code -func (o *NodesGetClassForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this nodes get class forbidden response has a 4xx status code -func (o *NodesGetClassForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this nodes get class forbidden response has a 5xx status code -func (o *NodesGetClassForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this nodes get class forbidden response a status code equal to that given -func (o *NodesGetClassForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the nodes get class forbidden response -func (o *NodesGetClassForbidden) Code() int { - return 403 -} - -func (o *NodesGetClassForbidden) Error() string { - return fmt.Sprintf("[GET /nodes/{className}][%d] nodesGetClassForbidden %+v", 403, o.Payload) -} - -func (o *NodesGetClassForbidden) String() string { - return fmt.Sprintf("[GET /nodes/{className}][%d] nodesGetClassForbidden %+v", 403, o.Payload) -} - -func (o *NodesGetClassForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *NodesGetClassForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewNodesGetClassNotFound creates a NodesGetClassNotFound with default headers values -func NewNodesGetClassNotFound() *NodesGetClassNotFound { - return &NodesGetClassNotFound{} -} - -/* -NodesGetClassNotFound describes a response with status code 404, with default header values. - -Not Found - Backup does not exist -*/ -type NodesGetClassNotFound struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this nodes get class not found response has a 2xx status code -func (o *NodesGetClassNotFound) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this nodes get class not found response has a 3xx status code -func (o *NodesGetClassNotFound) IsRedirect() bool { - return false -} - -// IsClientError returns true when this nodes get class not found response has a 4xx status code -func (o *NodesGetClassNotFound) IsClientError() bool { - return true -} - -// IsServerError returns true when this nodes get class not found response has a 5xx status code -func (o *NodesGetClassNotFound) IsServerError() bool { - return false -} - -// IsCode returns true when this nodes get class not found response a status code equal to that given -func (o *NodesGetClassNotFound) IsCode(code int) bool { - return code == 404 -} - -// Code gets the status code for the nodes get class not found response -func (o *NodesGetClassNotFound) Code() int { - return 404 -} - -func (o *NodesGetClassNotFound) Error() string { - return fmt.Sprintf("[GET /nodes/{className}][%d] nodesGetClassNotFound %+v", 404, o.Payload) -} - -func (o *NodesGetClassNotFound) String() string { - return fmt.Sprintf("[GET /nodes/{className}][%d] nodesGetClassNotFound %+v", 404, o.Payload) -} - -func (o *NodesGetClassNotFound) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *NodesGetClassNotFound) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewNodesGetClassUnprocessableEntity creates a NodesGetClassUnprocessableEntity with default headers values -func NewNodesGetClassUnprocessableEntity() *NodesGetClassUnprocessableEntity { - return &NodesGetClassUnprocessableEntity{} -} - -/* -NodesGetClassUnprocessableEntity describes a response with status code 422, with default header values. - -Invalid backup restoration status attempt. -*/ -type NodesGetClassUnprocessableEntity struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this nodes get class unprocessable entity response has a 2xx status code -func (o *NodesGetClassUnprocessableEntity) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this nodes get class unprocessable entity response has a 3xx status code -func (o *NodesGetClassUnprocessableEntity) IsRedirect() bool { - return false -} - -// IsClientError returns true when this nodes get class unprocessable entity response has a 4xx status code -func (o *NodesGetClassUnprocessableEntity) IsClientError() bool { - return true -} - -// IsServerError returns true when this nodes get class unprocessable entity response has a 5xx status code -func (o *NodesGetClassUnprocessableEntity) IsServerError() bool { - return false -} - -// IsCode returns true when this nodes get class unprocessable entity response a status code equal to that given -func (o *NodesGetClassUnprocessableEntity) IsCode(code int) bool { - return code == 422 -} - -// Code gets the status code for the nodes get class unprocessable entity response -func (o *NodesGetClassUnprocessableEntity) Code() int { - return 422 -} - -func (o *NodesGetClassUnprocessableEntity) Error() string { - return fmt.Sprintf("[GET /nodes/{className}][%d] nodesGetClassUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *NodesGetClassUnprocessableEntity) String() string { - return fmt.Sprintf("[GET /nodes/{className}][%d] nodesGetClassUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *NodesGetClassUnprocessableEntity) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *NodesGetClassUnprocessableEntity) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewNodesGetClassInternalServerError creates a NodesGetClassInternalServerError with default headers values -func NewNodesGetClassInternalServerError() *NodesGetClassInternalServerError { - return &NodesGetClassInternalServerError{} -} - -/* -NodesGetClassInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type NodesGetClassInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this nodes get class internal server error response has a 2xx status code -func (o *NodesGetClassInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this nodes get class internal server error response has a 3xx status code -func (o *NodesGetClassInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this nodes get class internal server error response has a 4xx status code -func (o *NodesGetClassInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this nodes get class internal server error response has a 5xx status code -func (o *NodesGetClassInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this nodes get class internal server error response a status code equal to that given -func (o *NodesGetClassInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the nodes get class internal server error response -func (o *NodesGetClassInternalServerError) Code() int { - return 500 -} - -func (o *NodesGetClassInternalServerError) Error() string { - return fmt.Sprintf("[GET /nodes/{className}][%d] nodesGetClassInternalServerError %+v", 500, o.Payload) -} - -func (o *NodesGetClassInternalServerError) String() string { - return fmt.Sprintf("[GET /nodes/{className}][%d] nodesGetClassInternalServerError %+v", 500, o.Payload) -} - -func (o *NodesGetClassInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *NodesGetClassInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/nodes/nodes_get_parameters.go b/client/nodes/nodes_get_parameters.go deleted file mode 100644 index 8633c1e9dabe6d7a9b39b0d489b051403fee3063..0000000000000000000000000000000000000000 --- a/client/nodes/nodes_get_parameters.go +++ /dev/null @@ -1,187 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package nodes - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" -) - -// NewNodesGetParams creates a new NodesGetParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewNodesGetParams() *NodesGetParams { - return &NodesGetParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewNodesGetParamsWithTimeout creates a new NodesGetParams object -// with the ability to set a timeout on a request. -func NewNodesGetParamsWithTimeout(timeout time.Duration) *NodesGetParams { - return &NodesGetParams{ - timeout: timeout, - } -} - -// NewNodesGetParamsWithContext creates a new NodesGetParams object -// with the ability to set a context for a request. -func NewNodesGetParamsWithContext(ctx context.Context) *NodesGetParams { - return &NodesGetParams{ - Context: ctx, - } -} - -// NewNodesGetParamsWithHTTPClient creates a new NodesGetParams object -// with the ability to set a custom HTTPClient for a request. -func NewNodesGetParamsWithHTTPClient(client *http.Client) *NodesGetParams { - return &NodesGetParams{ - HTTPClient: client, - } -} - -/* -NodesGetParams contains all the parameters to send to the API endpoint - - for the nodes get operation. - - Typically these are written to a http.Request. -*/ -type NodesGetParams struct { - - /* Output. - - Controls the verbosity of the output, possible values are: "minimal", "verbose". Defaults to "minimal". - - Default: "minimal" - */ - Output *string - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the nodes get params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *NodesGetParams) WithDefaults() *NodesGetParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the nodes get params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *NodesGetParams) SetDefaults() { - var ( - outputDefault = string("minimal") - ) - - val := NodesGetParams{ - Output: &outputDefault, - } - - val.timeout = o.timeout - val.Context = o.Context - val.HTTPClient = o.HTTPClient - *o = val -} - -// WithTimeout adds the timeout to the nodes get params -func (o *NodesGetParams) WithTimeout(timeout time.Duration) *NodesGetParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the nodes get params -func (o *NodesGetParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the nodes get params -func (o *NodesGetParams) WithContext(ctx context.Context) *NodesGetParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the nodes get params -func (o *NodesGetParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the nodes get params -func (o *NodesGetParams) WithHTTPClient(client *http.Client) *NodesGetParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the nodes get params -func (o *NodesGetParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithOutput adds the output to the nodes get params -func (o *NodesGetParams) WithOutput(output *string) *NodesGetParams { - o.SetOutput(output) - return o -} - -// SetOutput adds the output to the nodes get params -func (o *NodesGetParams) SetOutput(output *string) { - o.Output = output -} - -// WriteToRequest writes these params to a swagger request -func (o *NodesGetParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - - if o.Output != nil { - - // query param output - var qrOutput string - - if o.Output != nil { - qrOutput = *o.Output - } - qOutput := qrOutput - if qOutput != "" { - - if err := r.SetQueryParam("output", qOutput); err != nil { - return err - } - } - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/nodes/nodes_get_responses.go b/client/nodes/nodes_get_responses.go deleted file mode 100644 index ca00f02e13b6d3c532bed1a5dec4fdc0d1eb47f6..0000000000000000000000000000000000000000 --- a/client/nodes/nodes_get_responses.go +++ /dev/null @@ -1,472 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package nodes - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// NodesGetReader is a Reader for the NodesGet structure. -type NodesGetReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *NodesGetReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 200: - result := NewNodesGetOK() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 401: - result := NewNodesGetUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewNodesGetForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 404: - result := NewNodesGetNotFound() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 422: - result := NewNodesGetUnprocessableEntity() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewNodesGetInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewNodesGetOK creates a NodesGetOK with default headers values -func NewNodesGetOK() *NodesGetOK { - return &NodesGetOK{} -} - -/* -NodesGetOK describes a response with status code 200, with default header values. - -Nodes status successfully returned -*/ -type NodesGetOK struct { - Payload *models.NodesStatusResponse -} - -// IsSuccess returns true when this nodes get o k response has a 2xx status code -func (o *NodesGetOK) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this nodes get o k response has a 3xx status code -func (o *NodesGetOK) IsRedirect() bool { - return false -} - -// IsClientError returns true when this nodes get o k response has a 4xx status code -func (o *NodesGetOK) IsClientError() bool { - return false -} - -// IsServerError returns true when this nodes get o k response has a 5xx status code -func (o *NodesGetOK) IsServerError() bool { - return false -} - -// IsCode returns true when this nodes get o k response a status code equal to that given -func (o *NodesGetOK) IsCode(code int) bool { - return code == 200 -} - -// Code gets the status code for the nodes get o k response -func (o *NodesGetOK) Code() int { - return 200 -} - -func (o *NodesGetOK) Error() string { - return fmt.Sprintf("[GET /nodes][%d] nodesGetOK %+v", 200, o.Payload) -} - -func (o *NodesGetOK) String() string { - return fmt.Sprintf("[GET /nodes][%d] nodesGetOK %+v", 200, o.Payload) -} - -func (o *NodesGetOK) GetPayload() *models.NodesStatusResponse { - return o.Payload -} - -func (o *NodesGetOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.NodesStatusResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewNodesGetUnauthorized creates a NodesGetUnauthorized with default headers values -func NewNodesGetUnauthorized() *NodesGetUnauthorized { - return &NodesGetUnauthorized{} -} - -/* -NodesGetUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type NodesGetUnauthorized struct { -} - -// IsSuccess returns true when this nodes get unauthorized response has a 2xx status code -func (o *NodesGetUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this nodes get unauthorized response has a 3xx status code -func (o *NodesGetUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this nodes get unauthorized response has a 4xx status code -func (o *NodesGetUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this nodes get unauthorized response has a 5xx status code -func (o *NodesGetUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this nodes get unauthorized response a status code equal to that given -func (o *NodesGetUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the nodes get unauthorized response -func (o *NodesGetUnauthorized) Code() int { - return 401 -} - -func (o *NodesGetUnauthorized) Error() string { - return fmt.Sprintf("[GET /nodes][%d] nodesGetUnauthorized ", 401) -} - -func (o *NodesGetUnauthorized) String() string { - return fmt.Sprintf("[GET /nodes][%d] nodesGetUnauthorized ", 401) -} - -func (o *NodesGetUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewNodesGetForbidden creates a NodesGetForbidden with default headers values -func NewNodesGetForbidden() *NodesGetForbidden { - return &NodesGetForbidden{} -} - -/* -NodesGetForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type NodesGetForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this nodes get forbidden response has a 2xx status code -func (o *NodesGetForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this nodes get forbidden response has a 3xx status code -func (o *NodesGetForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this nodes get forbidden response has a 4xx status code -func (o *NodesGetForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this nodes get forbidden response has a 5xx status code -func (o *NodesGetForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this nodes get forbidden response a status code equal to that given -func (o *NodesGetForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the nodes get forbidden response -func (o *NodesGetForbidden) Code() int { - return 403 -} - -func (o *NodesGetForbidden) Error() string { - return fmt.Sprintf("[GET /nodes][%d] nodesGetForbidden %+v", 403, o.Payload) -} - -func (o *NodesGetForbidden) String() string { - return fmt.Sprintf("[GET /nodes][%d] nodesGetForbidden %+v", 403, o.Payload) -} - -func (o *NodesGetForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *NodesGetForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewNodesGetNotFound creates a NodesGetNotFound with default headers values -func NewNodesGetNotFound() *NodesGetNotFound { - return &NodesGetNotFound{} -} - -/* -NodesGetNotFound describes a response with status code 404, with default header values. - -Not Found - Backup does not exist -*/ -type NodesGetNotFound struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this nodes get not found response has a 2xx status code -func (o *NodesGetNotFound) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this nodes get not found response has a 3xx status code -func (o *NodesGetNotFound) IsRedirect() bool { - return false -} - -// IsClientError returns true when this nodes get not found response has a 4xx status code -func (o *NodesGetNotFound) IsClientError() bool { - return true -} - -// IsServerError returns true when this nodes get not found response has a 5xx status code -func (o *NodesGetNotFound) IsServerError() bool { - return false -} - -// IsCode returns true when this nodes get not found response a status code equal to that given -func (o *NodesGetNotFound) IsCode(code int) bool { - return code == 404 -} - -// Code gets the status code for the nodes get not found response -func (o *NodesGetNotFound) Code() int { - return 404 -} - -func (o *NodesGetNotFound) Error() string { - return fmt.Sprintf("[GET /nodes][%d] nodesGetNotFound %+v", 404, o.Payload) -} - -func (o *NodesGetNotFound) String() string { - return fmt.Sprintf("[GET /nodes][%d] nodesGetNotFound %+v", 404, o.Payload) -} - -func (o *NodesGetNotFound) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *NodesGetNotFound) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewNodesGetUnprocessableEntity creates a NodesGetUnprocessableEntity with default headers values -func NewNodesGetUnprocessableEntity() *NodesGetUnprocessableEntity { - return &NodesGetUnprocessableEntity{} -} - -/* -NodesGetUnprocessableEntity describes a response with status code 422, with default header values. - -Invalid backup restoration status attempt. -*/ -type NodesGetUnprocessableEntity struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this nodes get unprocessable entity response has a 2xx status code -func (o *NodesGetUnprocessableEntity) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this nodes get unprocessable entity response has a 3xx status code -func (o *NodesGetUnprocessableEntity) IsRedirect() bool { - return false -} - -// IsClientError returns true when this nodes get unprocessable entity response has a 4xx status code -func (o *NodesGetUnprocessableEntity) IsClientError() bool { - return true -} - -// IsServerError returns true when this nodes get unprocessable entity response has a 5xx status code -func (o *NodesGetUnprocessableEntity) IsServerError() bool { - return false -} - -// IsCode returns true when this nodes get unprocessable entity response a status code equal to that given -func (o *NodesGetUnprocessableEntity) IsCode(code int) bool { - return code == 422 -} - -// Code gets the status code for the nodes get unprocessable entity response -func (o *NodesGetUnprocessableEntity) Code() int { - return 422 -} - -func (o *NodesGetUnprocessableEntity) Error() string { - return fmt.Sprintf("[GET /nodes][%d] nodesGetUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *NodesGetUnprocessableEntity) String() string { - return fmt.Sprintf("[GET /nodes][%d] nodesGetUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *NodesGetUnprocessableEntity) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *NodesGetUnprocessableEntity) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewNodesGetInternalServerError creates a NodesGetInternalServerError with default headers values -func NewNodesGetInternalServerError() *NodesGetInternalServerError { - return &NodesGetInternalServerError{} -} - -/* -NodesGetInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type NodesGetInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this nodes get internal server error response has a 2xx status code -func (o *NodesGetInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this nodes get internal server error response has a 3xx status code -func (o *NodesGetInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this nodes get internal server error response has a 4xx status code -func (o *NodesGetInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this nodes get internal server error response has a 5xx status code -func (o *NodesGetInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this nodes get internal server error response a status code equal to that given -func (o *NodesGetInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the nodes get internal server error response -func (o *NodesGetInternalServerError) Code() int { - return 500 -} - -func (o *NodesGetInternalServerError) Error() string { - return fmt.Sprintf("[GET /nodes][%d] nodesGetInternalServerError %+v", 500, o.Payload) -} - -func (o *NodesGetInternalServerError) String() string { - return fmt.Sprintf("[GET /nodes][%d] nodesGetInternalServerError %+v", 500, o.Payload) -} - -func (o *NodesGetInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *NodesGetInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/objects/objects_class_delete_parameters.go b/client/objects/objects_class_delete_parameters.go deleted file mode 100644 index 208466ec6f6cf09c6f9ece2766efab9f70496364..0000000000000000000000000000000000000000 --- a/client/objects/objects_class_delete_parameters.go +++ /dev/null @@ -1,251 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" -) - -// NewObjectsClassDeleteParams creates a new ObjectsClassDeleteParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewObjectsClassDeleteParams() *ObjectsClassDeleteParams { - return &ObjectsClassDeleteParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewObjectsClassDeleteParamsWithTimeout creates a new ObjectsClassDeleteParams object -// with the ability to set a timeout on a request. -func NewObjectsClassDeleteParamsWithTimeout(timeout time.Duration) *ObjectsClassDeleteParams { - return &ObjectsClassDeleteParams{ - timeout: timeout, - } -} - -// NewObjectsClassDeleteParamsWithContext creates a new ObjectsClassDeleteParams object -// with the ability to set a context for a request. -func NewObjectsClassDeleteParamsWithContext(ctx context.Context) *ObjectsClassDeleteParams { - return &ObjectsClassDeleteParams{ - Context: ctx, - } -} - -// NewObjectsClassDeleteParamsWithHTTPClient creates a new ObjectsClassDeleteParams object -// with the ability to set a custom HTTPClient for a request. -func NewObjectsClassDeleteParamsWithHTTPClient(client *http.Client) *ObjectsClassDeleteParams { - return &ObjectsClassDeleteParams{ - HTTPClient: client, - } -} - -/* -ObjectsClassDeleteParams contains all the parameters to send to the API endpoint - - for the objects class delete operation. - - Typically these are written to a http.Request. -*/ -type ObjectsClassDeleteParams struct { - - // ClassName. - ClassName string - - /* ConsistencyLevel. - - Determines how many replicas must acknowledge a request before it is considered successful - */ - ConsistencyLevel *string - - /* ID. - - Unique ID of the Object. - - Format: uuid - */ - ID strfmt.UUID - - /* Tenant. - - Specifies the tenant in a request targeting a multi-tenant class - */ - Tenant *string - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the objects class delete params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *ObjectsClassDeleteParams) WithDefaults() *ObjectsClassDeleteParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the objects class delete params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *ObjectsClassDeleteParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the objects class delete params -func (o *ObjectsClassDeleteParams) WithTimeout(timeout time.Duration) *ObjectsClassDeleteParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the objects class delete params -func (o *ObjectsClassDeleteParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the objects class delete params -func (o *ObjectsClassDeleteParams) WithContext(ctx context.Context) *ObjectsClassDeleteParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the objects class delete params -func (o *ObjectsClassDeleteParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the objects class delete params -func (o *ObjectsClassDeleteParams) WithHTTPClient(client *http.Client) *ObjectsClassDeleteParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the objects class delete params -func (o *ObjectsClassDeleteParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithClassName adds the className to the objects class delete params -func (o *ObjectsClassDeleteParams) WithClassName(className string) *ObjectsClassDeleteParams { - o.SetClassName(className) - return o -} - -// SetClassName adds the className to the objects class delete params -func (o *ObjectsClassDeleteParams) SetClassName(className string) { - o.ClassName = className -} - -// WithConsistencyLevel adds the consistencyLevel to the objects class delete params -func (o *ObjectsClassDeleteParams) WithConsistencyLevel(consistencyLevel *string) *ObjectsClassDeleteParams { - o.SetConsistencyLevel(consistencyLevel) - return o -} - -// SetConsistencyLevel adds the consistencyLevel to the objects class delete params -func (o *ObjectsClassDeleteParams) SetConsistencyLevel(consistencyLevel *string) { - o.ConsistencyLevel = consistencyLevel -} - -// WithID adds the id to the objects class delete params -func (o *ObjectsClassDeleteParams) WithID(id strfmt.UUID) *ObjectsClassDeleteParams { - o.SetID(id) - return o -} - -// SetID adds the id to the objects class delete params -func (o *ObjectsClassDeleteParams) SetID(id strfmt.UUID) { - o.ID = id -} - -// WithTenant adds the tenant to the objects class delete params -func (o *ObjectsClassDeleteParams) WithTenant(tenant *string) *ObjectsClassDeleteParams { - o.SetTenant(tenant) - return o -} - -// SetTenant adds the tenant to the objects class delete params -func (o *ObjectsClassDeleteParams) SetTenant(tenant *string) { - o.Tenant = tenant -} - -// WriteToRequest writes these params to a swagger request -func (o *ObjectsClassDeleteParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - - // path param className - if err := r.SetPathParam("className", o.ClassName); err != nil { - return err - } - - if o.ConsistencyLevel != nil { - - // query param consistency_level - var qrConsistencyLevel string - - if o.ConsistencyLevel != nil { - qrConsistencyLevel = *o.ConsistencyLevel - } - qConsistencyLevel := qrConsistencyLevel - if qConsistencyLevel != "" { - - if err := r.SetQueryParam("consistency_level", qConsistencyLevel); err != nil { - return err - } - } - } - - // path param id - if err := r.SetPathParam("id", o.ID.String()); err != nil { - return err - } - - if o.Tenant != nil { - - // query param tenant - var qrTenant string - - if o.Tenant != nil { - qrTenant = *o.Tenant - } - qTenant := qrTenant - if qTenant != "" { - - if err := r.SetQueryParam("tenant", qTenant); err != nil { - return err - } - } - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/objects/objects_class_delete_responses.go b/client/objects/objects_class_delete_responses.go deleted file mode 100644 index 90a036ce4fb55a9dd0b3c1e77971bc9673144e6c..0000000000000000000000000000000000000000 --- a/client/objects/objects_class_delete_responses.go +++ /dev/null @@ -1,522 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsClassDeleteReader is a Reader for the ObjectsClassDelete structure. -type ObjectsClassDeleteReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *ObjectsClassDeleteReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 204: - result := NewObjectsClassDeleteNoContent() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 400: - result := NewObjectsClassDeleteBadRequest() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 401: - result := NewObjectsClassDeleteUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewObjectsClassDeleteForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 404: - result := NewObjectsClassDeleteNotFound() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 422: - result := NewObjectsClassDeleteUnprocessableEntity() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewObjectsClassDeleteInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewObjectsClassDeleteNoContent creates a ObjectsClassDeleteNoContent with default headers values -func NewObjectsClassDeleteNoContent() *ObjectsClassDeleteNoContent { - return &ObjectsClassDeleteNoContent{} -} - -/* -ObjectsClassDeleteNoContent describes a response with status code 204, with default header values. - -Successfully deleted. -*/ -type ObjectsClassDeleteNoContent struct { -} - -// IsSuccess returns true when this objects class delete no content response has a 2xx status code -func (o *ObjectsClassDeleteNoContent) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this objects class delete no content response has a 3xx status code -func (o *ObjectsClassDeleteNoContent) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class delete no content response has a 4xx status code -func (o *ObjectsClassDeleteNoContent) IsClientError() bool { - return false -} - -// IsServerError returns true when this objects class delete no content response has a 5xx status code -func (o *ObjectsClassDeleteNoContent) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class delete no content response a status code equal to that given -func (o *ObjectsClassDeleteNoContent) IsCode(code int) bool { - return code == 204 -} - -// Code gets the status code for the objects class delete no content response -func (o *ObjectsClassDeleteNoContent) Code() int { - return 204 -} - -func (o *ObjectsClassDeleteNoContent) Error() string { - return fmt.Sprintf("[DELETE /objects/{className}/{id}][%d] objectsClassDeleteNoContent ", 204) -} - -func (o *ObjectsClassDeleteNoContent) String() string { - return fmt.Sprintf("[DELETE /objects/{className}/{id}][%d] objectsClassDeleteNoContent ", 204) -} - -func (o *ObjectsClassDeleteNoContent) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsClassDeleteBadRequest creates a ObjectsClassDeleteBadRequest with default headers values -func NewObjectsClassDeleteBadRequest() *ObjectsClassDeleteBadRequest { - return &ObjectsClassDeleteBadRequest{} -} - -/* -ObjectsClassDeleteBadRequest describes a response with status code 400, with default header values. - -Malformed request. -*/ -type ObjectsClassDeleteBadRequest struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects class delete bad request response has a 2xx status code -func (o *ObjectsClassDeleteBadRequest) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class delete bad request response has a 3xx status code -func (o *ObjectsClassDeleteBadRequest) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class delete bad request response has a 4xx status code -func (o *ObjectsClassDeleteBadRequest) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects class delete bad request response has a 5xx status code -func (o *ObjectsClassDeleteBadRequest) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class delete bad request response a status code equal to that given -func (o *ObjectsClassDeleteBadRequest) IsCode(code int) bool { - return code == 400 -} - -// Code gets the status code for the objects class delete bad request response -func (o *ObjectsClassDeleteBadRequest) Code() int { - return 400 -} - -func (o *ObjectsClassDeleteBadRequest) Error() string { - return fmt.Sprintf("[DELETE /objects/{className}/{id}][%d] objectsClassDeleteBadRequest %+v", 400, o.Payload) -} - -func (o *ObjectsClassDeleteBadRequest) String() string { - return fmt.Sprintf("[DELETE /objects/{className}/{id}][%d] objectsClassDeleteBadRequest %+v", 400, o.Payload) -} - -func (o *ObjectsClassDeleteBadRequest) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsClassDeleteBadRequest) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsClassDeleteUnauthorized creates a ObjectsClassDeleteUnauthorized with default headers values -func NewObjectsClassDeleteUnauthorized() *ObjectsClassDeleteUnauthorized { - return &ObjectsClassDeleteUnauthorized{} -} - -/* -ObjectsClassDeleteUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type ObjectsClassDeleteUnauthorized struct { -} - -// IsSuccess returns true when this objects class delete unauthorized response has a 2xx status code -func (o *ObjectsClassDeleteUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class delete unauthorized response has a 3xx status code -func (o *ObjectsClassDeleteUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class delete unauthorized response has a 4xx status code -func (o *ObjectsClassDeleteUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects class delete unauthorized response has a 5xx status code -func (o *ObjectsClassDeleteUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class delete unauthorized response a status code equal to that given -func (o *ObjectsClassDeleteUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the objects class delete unauthorized response -func (o *ObjectsClassDeleteUnauthorized) Code() int { - return 401 -} - -func (o *ObjectsClassDeleteUnauthorized) Error() string { - return fmt.Sprintf("[DELETE /objects/{className}/{id}][%d] objectsClassDeleteUnauthorized ", 401) -} - -func (o *ObjectsClassDeleteUnauthorized) String() string { - return fmt.Sprintf("[DELETE /objects/{className}/{id}][%d] objectsClassDeleteUnauthorized ", 401) -} - -func (o *ObjectsClassDeleteUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsClassDeleteForbidden creates a ObjectsClassDeleteForbidden with default headers values -func NewObjectsClassDeleteForbidden() *ObjectsClassDeleteForbidden { - return &ObjectsClassDeleteForbidden{} -} - -/* -ObjectsClassDeleteForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type ObjectsClassDeleteForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects class delete forbidden response has a 2xx status code -func (o *ObjectsClassDeleteForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class delete forbidden response has a 3xx status code -func (o *ObjectsClassDeleteForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class delete forbidden response has a 4xx status code -func (o *ObjectsClassDeleteForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects class delete forbidden response has a 5xx status code -func (o *ObjectsClassDeleteForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class delete forbidden response a status code equal to that given -func (o *ObjectsClassDeleteForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the objects class delete forbidden response -func (o *ObjectsClassDeleteForbidden) Code() int { - return 403 -} - -func (o *ObjectsClassDeleteForbidden) Error() string { - return fmt.Sprintf("[DELETE /objects/{className}/{id}][%d] objectsClassDeleteForbidden %+v", 403, o.Payload) -} - -func (o *ObjectsClassDeleteForbidden) String() string { - return fmt.Sprintf("[DELETE /objects/{className}/{id}][%d] objectsClassDeleteForbidden %+v", 403, o.Payload) -} - -func (o *ObjectsClassDeleteForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsClassDeleteForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsClassDeleteNotFound creates a ObjectsClassDeleteNotFound with default headers values -func NewObjectsClassDeleteNotFound() *ObjectsClassDeleteNotFound { - return &ObjectsClassDeleteNotFound{} -} - -/* -ObjectsClassDeleteNotFound describes a response with status code 404, with default header values. - -Successful query result but no resource was found. -*/ -type ObjectsClassDeleteNotFound struct { -} - -// IsSuccess returns true when this objects class delete not found response has a 2xx status code -func (o *ObjectsClassDeleteNotFound) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class delete not found response has a 3xx status code -func (o *ObjectsClassDeleteNotFound) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class delete not found response has a 4xx status code -func (o *ObjectsClassDeleteNotFound) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects class delete not found response has a 5xx status code -func (o *ObjectsClassDeleteNotFound) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class delete not found response a status code equal to that given -func (o *ObjectsClassDeleteNotFound) IsCode(code int) bool { - return code == 404 -} - -// Code gets the status code for the objects class delete not found response -func (o *ObjectsClassDeleteNotFound) Code() int { - return 404 -} - -func (o *ObjectsClassDeleteNotFound) Error() string { - return fmt.Sprintf("[DELETE /objects/{className}/{id}][%d] objectsClassDeleteNotFound ", 404) -} - -func (o *ObjectsClassDeleteNotFound) String() string { - return fmt.Sprintf("[DELETE /objects/{className}/{id}][%d] objectsClassDeleteNotFound ", 404) -} - -func (o *ObjectsClassDeleteNotFound) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsClassDeleteUnprocessableEntity creates a ObjectsClassDeleteUnprocessableEntity with default headers values -func NewObjectsClassDeleteUnprocessableEntity() *ObjectsClassDeleteUnprocessableEntity { - return &ObjectsClassDeleteUnprocessableEntity{} -} - -/* -ObjectsClassDeleteUnprocessableEntity describes a response with status code 422, with default header values. - -Request is well-formed (i.e., syntactically correct), but erroneous. -*/ -type ObjectsClassDeleteUnprocessableEntity struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects class delete unprocessable entity response has a 2xx status code -func (o *ObjectsClassDeleteUnprocessableEntity) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class delete unprocessable entity response has a 3xx status code -func (o *ObjectsClassDeleteUnprocessableEntity) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class delete unprocessable entity response has a 4xx status code -func (o *ObjectsClassDeleteUnprocessableEntity) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects class delete unprocessable entity response has a 5xx status code -func (o *ObjectsClassDeleteUnprocessableEntity) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class delete unprocessable entity response a status code equal to that given -func (o *ObjectsClassDeleteUnprocessableEntity) IsCode(code int) bool { - return code == 422 -} - -// Code gets the status code for the objects class delete unprocessable entity response -func (o *ObjectsClassDeleteUnprocessableEntity) Code() int { - return 422 -} - -func (o *ObjectsClassDeleteUnprocessableEntity) Error() string { - return fmt.Sprintf("[DELETE /objects/{className}/{id}][%d] objectsClassDeleteUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *ObjectsClassDeleteUnprocessableEntity) String() string { - return fmt.Sprintf("[DELETE /objects/{className}/{id}][%d] objectsClassDeleteUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *ObjectsClassDeleteUnprocessableEntity) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsClassDeleteUnprocessableEntity) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsClassDeleteInternalServerError creates a ObjectsClassDeleteInternalServerError with default headers values -func NewObjectsClassDeleteInternalServerError() *ObjectsClassDeleteInternalServerError { - return &ObjectsClassDeleteInternalServerError{} -} - -/* -ObjectsClassDeleteInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type ObjectsClassDeleteInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects class delete internal server error response has a 2xx status code -func (o *ObjectsClassDeleteInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class delete internal server error response has a 3xx status code -func (o *ObjectsClassDeleteInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class delete internal server error response has a 4xx status code -func (o *ObjectsClassDeleteInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this objects class delete internal server error response has a 5xx status code -func (o *ObjectsClassDeleteInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this objects class delete internal server error response a status code equal to that given -func (o *ObjectsClassDeleteInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the objects class delete internal server error response -func (o *ObjectsClassDeleteInternalServerError) Code() int { - return 500 -} - -func (o *ObjectsClassDeleteInternalServerError) Error() string { - return fmt.Sprintf("[DELETE /objects/{className}/{id}][%d] objectsClassDeleteInternalServerError %+v", 500, o.Payload) -} - -func (o *ObjectsClassDeleteInternalServerError) String() string { - return fmt.Sprintf("[DELETE /objects/{className}/{id}][%d] objectsClassDeleteInternalServerError %+v", 500, o.Payload) -} - -func (o *ObjectsClassDeleteInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsClassDeleteInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/objects/objects_class_get_parameters.go b/client/objects/objects_class_get_parameters.go deleted file mode 100644 index 07eafec75d032a528beb1e01ba6c75c52891df85..0000000000000000000000000000000000000000 --- a/client/objects/objects_class_get_parameters.go +++ /dev/null @@ -1,319 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" -) - -// NewObjectsClassGetParams creates a new ObjectsClassGetParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewObjectsClassGetParams() *ObjectsClassGetParams { - return &ObjectsClassGetParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewObjectsClassGetParamsWithTimeout creates a new ObjectsClassGetParams object -// with the ability to set a timeout on a request. -func NewObjectsClassGetParamsWithTimeout(timeout time.Duration) *ObjectsClassGetParams { - return &ObjectsClassGetParams{ - timeout: timeout, - } -} - -// NewObjectsClassGetParamsWithContext creates a new ObjectsClassGetParams object -// with the ability to set a context for a request. -func NewObjectsClassGetParamsWithContext(ctx context.Context) *ObjectsClassGetParams { - return &ObjectsClassGetParams{ - Context: ctx, - } -} - -// NewObjectsClassGetParamsWithHTTPClient creates a new ObjectsClassGetParams object -// with the ability to set a custom HTTPClient for a request. -func NewObjectsClassGetParamsWithHTTPClient(client *http.Client) *ObjectsClassGetParams { - return &ObjectsClassGetParams{ - HTTPClient: client, - } -} - -/* -ObjectsClassGetParams contains all the parameters to send to the API endpoint - - for the objects class get operation. - - Typically these are written to a http.Request. -*/ -type ObjectsClassGetParams struct { - - // ClassName. - ClassName string - - /* ConsistencyLevel. - - Determines how many replicas must acknowledge a request before it is considered successful - */ - ConsistencyLevel *string - - /* ID. - - Unique ID of the Object. - - Format: uuid - */ - ID strfmt.UUID - - /* Include. - - Include additional information, such as classification infos. Allowed values include: classification, vector, interpretation - */ - Include *string - - /* NodeName. - - The target node which should fulfill the request - */ - NodeName *string - - /* Tenant. - - Specifies the tenant in a request targeting a multi-tenant class - */ - Tenant *string - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the objects class get params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *ObjectsClassGetParams) WithDefaults() *ObjectsClassGetParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the objects class get params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *ObjectsClassGetParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the objects class get params -func (o *ObjectsClassGetParams) WithTimeout(timeout time.Duration) *ObjectsClassGetParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the objects class get params -func (o *ObjectsClassGetParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the objects class get params -func (o *ObjectsClassGetParams) WithContext(ctx context.Context) *ObjectsClassGetParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the objects class get params -func (o *ObjectsClassGetParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the objects class get params -func (o *ObjectsClassGetParams) WithHTTPClient(client *http.Client) *ObjectsClassGetParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the objects class get params -func (o *ObjectsClassGetParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithClassName adds the className to the objects class get params -func (o *ObjectsClassGetParams) WithClassName(className string) *ObjectsClassGetParams { - o.SetClassName(className) - return o -} - -// SetClassName adds the className to the objects class get params -func (o *ObjectsClassGetParams) SetClassName(className string) { - o.ClassName = className -} - -// WithConsistencyLevel adds the consistencyLevel to the objects class get params -func (o *ObjectsClassGetParams) WithConsistencyLevel(consistencyLevel *string) *ObjectsClassGetParams { - o.SetConsistencyLevel(consistencyLevel) - return o -} - -// SetConsistencyLevel adds the consistencyLevel to the objects class get params -func (o *ObjectsClassGetParams) SetConsistencyLevel(consistencyLevel *string) { - o.ConsistencyLevel = consistencyLevel -} - -// WithID adds the id to the objects class get params -func (o *ObjectsClassGetParams) WithID(id strfmt.UUID) *ObjectsClassGetParams { - o.SetID(id) - return o -} - -// SetID adds the id to the objects class get params -func (o *ObjectsClassGetParams) SetID(id strfmt.UUID) { - o.ID = id -} - -// WithInclude adds the include to the objects class get params -func (o *ObjectsClassGetParams) WithInclude(include *string) *ObjectsClassGetParams { - o.SetInclude(include) - return o -} - -// SetInclude adds the include to the objects class get params -func (o *ObjectsClassGetParams) SetInclude(include *string) { - o.Include = include -} - -// WithNodeName adds the nodeName to the objects class get params -func (o *ObjectsClassGetParams) WithNodeName(nodeName *string) *ObjectsClassGetParams { - o.SetNodeName(nodeName) - return o -} - -// SetNodeName adds the nodeName to the objects class get params -func (o *ObjectsClassGetParams) SetNodeName(nodeName *string) { - o.NodeName = nodeName -} - -// WithTenant adds the tenant to the objects class get params -func (o *ObjectsClassGetParams) WithTenant(tenant *string) *ObjectsClassGetParams { - o.SetTenant(tenant) - return o -} - -// SetTenant adds the tenant to the objects class get params -func (o *ObjectsClassGetParams) SetTenant(tenant *string) { - o.Tenant = tenant -} - -// WriteToRequest writes these params to a swagger request -func (o *ObjectsClassGetParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - - // path param className - if err := r.SetPathParam("className", o.ClassName); err != nil { - return err - } - - if o.ConsistencyLevel != nil { - - // query param consistency_level - var qrConsistencyLevel string - - if o.ConsistencyLevel != nil { - qrConsistencyLevel = *o.ConsistencyLevel - } - qConsistencyLevel := qrConsistencyLevel - if qConsistencyLevel != "" { - - if err := r.SetQueryParam("consistency_level", qConsistencyLevel); err != nil { - return err - } - } - } - - // path param id - if err := r.SetPathParam("id", o.ID.String()); err != nil { - return err - } - - if o.Include != nil { - - // query param include - var qrInclude string - - if o.Include != nil { - qrInclude = *o.Include - } - qInclude := qrInclude - if qInclude != "" { - - if err := r.SetQueryParam("include", qInclude); err != nil { - return err - } - } - } - - if o.NodeName != nil { - - // query param node_name - var qrNodeName string - - if o.NodeName != nil { - qrNodeName = *o.NodeName - } - qNodeName := qrNodeName - if qNodeName != "" { - - if err := r.SetQueryParam("node_name", qNodeName); err != nil { - return err - } - } - } - - if o.Tenant != nil { - - // query param tenant - var qrTenant string - - if o.Tenant != nil { - qrTenant = *o.Tenant - } - qTenant := qrTenant - if qTenant != "" { - - if err := r.SetQueryParam("tenant", qTenant); err != nil { - return err - } - } - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/objects/objects_class_get_responses.go b/client/objects/objects_class_get_responses.go deleted file mode 100644 index 4c440995f49243b9ee5ee0c80053dc9d010e8112..0000000000000000000000000000000000000000 --- a/client/objects/objects_class_get_responses.go +++ /dev/null @@ -1,534 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsClassGetReader is a Reader for the ObjectsClassGet structure. -type ObjectsClassGetReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *ObjectsClassGetReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 200: - result := NewObjectsClassGetOK() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 400: - result := NewObjectsClassGetBadRequest() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 401: - result := NewObjectsClassGetUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewObjectsClassGetForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 404: - result := NewObjectsClassGetNotFound() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 422: - result := NewObjectsClassGetUnprocessableEntity() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewObjectsClassGetInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewObjectsClassGetOK creates a ObjectsClassGetOK with default headers values -func NewObjectsClassGetOK() *ObjectsClassGetOK { - return &ObjectsClassGetOK{} -} - -/* -ObjectsClassGetOK describes a response with status code 200, with default header values. - -Successful response. -*/ -type ObjectsClassGetOK struct { - Payload *models.Object -} - -// IsSuccess returns true when this objects class get o k response has a 2xx status code -func (o *ObjectsClassGetOK) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this objects class get o k response has a 3xx status code -func (o *ObjectsClassGetOK) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class get o k response has a 4xx status code -func (o *ObjectsClassGetOK) IsClientError() bool { - return false -} - -// IsServerError returns true when this objects class get o k response has a 5xx status code -func (o *ObjectsClassGetOK) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class get o k response a status code equal to that given -func (o *ObjectsClassGetOK) IsCode(code int) bool { - return code == 200 -} - -// Code gets the status code for the objects class get o k response -func (o *ObjectsClassGetOK) Code() int { - return 200 -} - -func (o *ObjectsClassGetOK) Error() string { - return fmt.Sprintf("[GET /objects/{className}/{id}][%d] objectsClassGetOK %+v", 200, o.Payload) -} - -func (o *ObjectsClassGetOK) String() string { - return fmt.Sprintf("[GET /objects/{className}/{id}][%d] objectsClassGetOK %+v", 200, o.Payload) -} - -func (o *ObjectsClassGetOK) GetPayload() *models.Object { - return o.Payload -} - -func (o *ObjectsClassGetOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.Object) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsClassGetBadRequest creates a ObjectsClassGetBadRequest with default headers values -func NewObjectsClassGetBadRequest() *ObjectsClassGetBadRequest { - return &ObjectsClassGetBadRequest{} -} - -/* -ObjectsClassGetBadRequest describes a response with status code 400, with default header values. - -Malformed request. -*/ -type ObjectsClassGetBadRequest struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects class get bad request response has a 2xx status code -func (o *ObjectsClassGetBadRequest) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class get bad request response has a 3xx status code -func (o *ObjectsClassGetBadRequest) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class get bad request response has a 4xx status code -func (o *ObjectsClassGetBadRequest) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects class get bad request response has a 5xx status code -func (o *ObjectsClassGetBadRequest) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class get bad request response a status code equal to that given -func (o *ObjectsClassGetBadRequest) IsCode(code int) bool { - return code == 400 -} - -// Code gets the status code for the objects class get bad request response -func (o *ObjectsClassGetBadRequest) Code() int { - return 400 -} - -func (o *ObjectsClassGetBadRequest) Error() string { - return fmt.Sprintf("[GET /objects/{className}/{id}][%d] objectsClassGetBadRequest %+v", 400, o.Payload) -} - -func (o *ObjectsClassGetBadRequest) String() string { - return fmt.Sprintf("[GET /objects/{className}/{id}][%d] objectsClassGetBadRequest %+v", 400, o.Payload) -} - -func (o *ObjectsClassGetBadRequest) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsClassGetBadRequest) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsClassGetUnauthorized creates a ObjectsClassGetUnauthorized with default headers values -func NewObjectsClassGetUnauthorized() *ObjectsClassGetUnauthorized { - return &ObjectsClassGetUnauthorized{} -} - -/* -ObjectsClassGetUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type ObjectsClassGetUnauthorized struct { -} - -// IsSuccess returns true when this objects class get unauthorized response has a 2xx status code -func (o *ObjectsClassGetUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class get unauthorized response has a 3xx status code -func (o *ObjectsClassGetUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class get unauthorized response has a 4xx status code -func (o *ObjectsClassGetUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects class get unauthorized response has a 5xx status code -func (o *ObjectsClassGetUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class get unauthorized response a status code equal to that given -func (o *ObjectsClassGetUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the objects class get unauthorized response -func (o *ObjectsClassGetUnauthorized) Code() int { - return 401 -} - -func (o *ObjectsClassGetUnauthorized) Error() string { - return fmt.Sprintf("[GET /objects/{className}/{id}][%d] objectsClassGetUnauthorized ", 401) -} - -func (o *ObjectsClassGetUnauthorized) String() string { - return fmt.Sprintf("[GET /objects/{className}/{id}][%d] objectsClassGetUnauthorized ", 401) -} - -func (o *ObjectsClassGetUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsClassGetForbidden creates a ObjectsClassGetForbidden with default headers values -func NewObjectsClassGetForbidden() *ObjectsClassGetForbidden { - return &ObjectsClassGetForbidden{} -} - -/* -ObjectsClassGetForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type ObjectsClassGetForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects class get forbidden response has a 2xx status code -func (o *ObjectsClassGetForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class get forbidden response has a 3xx status code -func (o *ObjectsClassGetForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class get forbidden response has a 4xx status code -func (o *ObjectsClassGetForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects class get forbidden response has a 5xx status code -func (o *ObjectsClassGetForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class get forbidden response a status code equal to that given -func (o *ObjectsClassGetForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the objects class get forbidden response -func (o *ObjectsClassGetForbidden) Code() int { - return 403 -} - -func (o *ObjectsClassGetForbidden) Error() string { - return fmt.Sprintf("[GET /objects/{className}/{id}][%d] objectsClassGetForbidden %+v", 403, o.Payload) -} - -func (o *ObjectsClassGetForbidden) String() string { - return fmt.Sprintf("[GET /objects/{className}/{id}][%d] objectsClassGetForbidden %+v", 403, o.Payload) -} - -func (o *ObjectsClassGetForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsClassGetForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsClassGetNotFound creates a ObjectsClassGetNotFound with default headers values -func NewObjectsClassGetNotFound() *ObjectsClassGetNotFound { - return &ObjectsClassGetNotFound{} -} - -/* -ObjectsClassGetNotFound describes a response with status code 404, with default header values. - -Successful query result but no resource was found. -*/ -type ObjectsClassGetNotFound struct { -} - -// IsSuccess returns true when this objects class get not found response has a 2xx status code -func (o *ObjectsClassGetNotFound) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class get not found response has a 3xx status code -func (o *ObjectsClassGetNotFound) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class get not found response has a 4xx status code -func (o *ObjectsClassGetNotFound) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects class get not found response has a 5xx status code -func (o *ObjectsClassGetNotFound) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class get not found response a status code equal to that given -func (o *ObjectsClassGetNotFound) IsCode(code int) bool { - return code == 404 -} - -// Code gets the status code for the objects class get not found response -func (o *ObjectsClassGetNotFound) Code() int { - return 404 -} - -func (o *ObjectsClassGetNotFound) Error() string { - return fmt.Sprintf("[GET /objects/{className}/{id}][%d] objectsClassGetNotFound ", 404) -} - -func (o *ObjectsClassGetNotFound) String() string { - return fmt.Sprintf("[GET /objects/{className}/{id}][%d] objectsClassGetNotFound ", 404) -} - -func (o *ObjectsClassGetNotFound) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsClassGetUnprocessableEntity creates a ObjectsClassGetUnprocessableEntity with default headers values -func NewObjectsClassGetUnprocessableEntity() *ObjectsClassGetUnprocessableEntity { - return &ObjectsClassGetUnprocessableEntity{} -} - -/* -ObjectsClassGetUnprocessableEntity describes a response with status code 422, with default header values. - -Request is well-formed (i.e., syntactically correct), but erroneous. -*/ -type ObjectsClassGetUnprocessableEntity struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects class get unprocessable entity response has a 2xx status code -func (o *ObjectsClassGetUnprocessableEntity) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class get unprocessable entity response has a 3xx status code -func (o *ObjectsClassGetUnprocessableEntity) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class get unprocessable entity response has a 4xx status code -func (o *ObjectsClassGetUnprocessableEntity) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects class get unprocessable entity response has a 5xx status code -func (o *ObjectsClassGetUnprocessableEntity) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class get unprocessable entity response a status code equal to that given -func (o *ObjectsClassGetUnprocessableEntity) IsCode(code int) bool { - return code == 422 -} - -// Code gets the status code for the objects class get unprocessable entity response -func (o *ObjectsClassGetUnprocessableEntity) Code() int { - return 422 -} - -func (o *ObjectsClassGetUnprocessableEntity) Error() string { - return fmt.Sprintf("[GET /objects/{className}/{id}][%d] objectsClassGetUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *ObjectsClassGetUnprocessableEntity) String() string { - return fmt.Sprintf("[GET /objects/{className}/{id}][%d] objectsClassGetUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *ObjectsClassGetUnprocessableEntity) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsClassGetUnprocessableEntity) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsClassGetInternalServerError creates a ObjectsClassGetInternalServerError with default headers values -func NewObjectsClassGetInternalServerError() *ObjectsClassGetInternalServerError { - return &ObjectsClassGetInternalServerError{} -} - -/* -ObjectsClassGetInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type ObjectsClassGetInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects class get internal server error response has a 2xx status code -func (o *ObjectsClassGetInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class get internal server error response has a 3xx status code -func (o *ObjectsClassGetInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class get internal server error response has a 4xx status code -func (o *ObjectsClassGetInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this objects class get internal server error response has a 5xx status code -func (o *ObjectsClassGetInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this objects class get internal server error response a status code equal to that given -func (o *ObjectsClassGetInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the objects class get internal server error response -func (o *ObjectsClassGetInternalServerError) Code() int { - return 500 -} - -func (o *ObjectsClassGetInternalServerError) Error() string { - return fmt.Sprintf("[GET /objects/{className}/{id}][%d] objectsClassGetInternalServerError %+v", 500, o.Payload) -} - -func (o *ObjectsClassGetInternalServerError) String() string { - return fmt.Sprintf("[GET /objects/{className}/{id}][%d] objectsClassGetInternalServerError %+v", 500, o.Payload) -} - -func (o *ObjectsClassGetInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsClassGetInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/objects/objects_class_head_parameters.go b/client/objects/objects_class_head_parameters.go deleted file mode 100644 index 70ff59cd65efb3bcab748282f439e914b30412d2..0000000000000000000000000000000000000000 --- a/client/objects/objects_class_head_parameters.go +++ /dev/null @@ -1,254 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" -) - -// NewObjectsClassHeadParams creates a new ObjectsClassHeadParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewObjectsClassHeadParams() *ObjectsClassHeadParams { - return &ObjectsClassHeadParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewObjectsClassHeadParamsWithTimeout creates a new ObjectsClassHeadParams object -// with the ability to set a timeout on a request. -func NewObjectsClassHeadParamsWithTimeout(timeout time.Duration) *ObjectsClassHeadParams { - return &ObjectsClassHeadParams{ - timeout: timeout, - } -} - -// NewObjectsClassHeadParamsWithContext creates a new ObjectsClassHeadParams object -// with the ability to set a context for a request. -func NewObjectsClassHeadParamsWithContext(ctx context.Context) *ObjectsClassHeadParams { - return &ObjectsClassHeadParams{ - Context: ctx, - } -} - -// NewObjectsClassHeadParamsWithHTTPClient creates a new ObjectsClassHeadParams object -// with the ability to set a custom HTTPClient for a request. -func NewObjectsClassHeadParamsWithHTTPClient(client *http.Client) *ObjectsClassHeadParams { - return &ObjectsClassHeadParams{ - HTTPClient: client, - } -} - -/* -ObjectsClassHeadParams contains all the parameters to send to the API endpoint - - for the objects class head operation. - - Typically these are written to a http.Request. -*/ -type ObjectsClassHeadParams struct { - - /* ClassName. - - The class name as defined in the schema - */ - ClassName string - - /* ConsistencyLevel. - - Determines how many replicas must acknowledge a request before it is considered successful - */ - ConsistencyLevel *string - - /* ID. - - The uuid of the data object - - Format: uuid - */ - ID strfmt.UUID - - /* Tenant. - - Specifies the tenant in a request targeting a multi-tenant class - */ - Tenant *string - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the objects class head params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *ObjectsClassHeadParams) WithDefaults() *ObjectsClassHeadParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the objects class head params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *ObjectsClassHeadParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the objects class head params -func (o *ObjectsClassHeadParams) WithTimeout(timeout time.Duration) *ObjectsClassHeadParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the objects class head params -func (o *ObjectsClassHeadParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the objects class head params -func (o *ObjectsClassHeadParams) WithContext(ctx context.Context) *ObjectsClassHeadParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the objects class head params -func (o *ObjectsClassHeadParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the objects class head params -func (o *ObjectsClassHeadParams) WithHTTPClient(client *http.Client) *ObjectsClassHeadParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the objects class head params -func (o *ObjectsClassHeadParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithClassName adds the className to the objects class head params -func (o *ObjectsClassHeadParams) WithClassName(className string) *ObjectsClassHeadParams { - o.SetClassName(className) - return o -} - -// SetClassName adds the className to the objects class head params -func (o *ObjectsClassHeadParams) SetClassName(className string) { - o.ClassName = className -} - -// WithConsistencyLevel adds the consistencyLevel to the objects class head params -func (o *ObjectsClassHeadParams) WithConsistencyLevel(consistencyLevel *string) *ObjectsClassHeadParams { - o.SetConsistencyLevel(consistencyLevel) - return o -} - -// SetConsistencyLevel adds the consistencyLevel to the objects class head params -func (o *ObjectsClassHeadParams) SetConsistencyLevel(consistencyLevel *string) { - o.ConsistencyLevel = consistencyLevel -} - -// WithID adds the id to the objects class head params -func (o *ObjectsClassHeadParams) WithID(id strfmt.UUID) *ObjectsClassHeadParams { - o.SetID(id) - return o -} - -// SetID adds the id to the objects class head params -func (o *ObjectsClassHeadParams) SetID(id strfmt.UUID) { - o.ID = id -} - -// WithTenant adds the tenant to the objects class head params -func (o *ObjectsClassHeadParams) WithTenant(tenant *string) *ObjectsClassHeadParams { - o.SetTenant(tenant) - return o -} - -// SetTenant adds the tenant to the objects class head params -func (o *ObjectsClassHeadParams) SetTenant(tenant *string) { - o.Tenant = tenant -} - -// WriteToRequest writes these params to a swagger request -func (o *ObjectsClassHeadParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - - // path param className - if err := r.SetPathParam("className", o.ClassName); err != nil { - return err - } - - if o.ConsistencyLevel != nil { - - // query param consistency_level - var qrConsistencyLevel string - - if o.ConsistencyLevel != nil { - qrConsistencyLevel = *o.ConsistencyLevel - } - qConsistencyLevel := qrConsistencyLevel - if qConsistencyLevel != "" { - - if err := r.SetQueryParam("consistency_level", qConsistencyLevel); err != nil { - return err - } - } - } - - // path param id - if err := r.SetPathParam("id", o.ID.String()); err != nil { - return err - } - - if o.Tenant != nil { - - // query param tenant - var qrTenant string - - if o.Tenant != nil { - qrTenant = *o.Tenant - } - qTenant := qrTenant - if qTenant != "" { - - if err := r.SetQueryParam("tenant", qTenant); err != nil { - return err - } - } - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/objects/objects_class_head_responses.go b/client/objects/objects_class_head_responses.go deleted file mode 100644 index 0955de354149bee753ac9b33d35152ec1ec2942c..0000000000000000000000000000000000000000 --- a/client/objects/objects_class_head_responses.go +++ /dev/null @@ -1,448 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsClassHeadReader is a Reader for the ObjectsClassHead structure. -type ObjectsClassHeadReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *ObjectsClassHeadReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 204: - result := NewObjectsClassHeadNoContent() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 401: - result := NewObjectsClassHeadUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewObjectsClassHeadForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 404: - result := NewObjectsClassHeadNotFound() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 422: - result := NewObjectsClassHeadUnprocessableEntity() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewObjectsClassHeadInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewObjectsClassHeadNoContent creates a ObjectsClassHeadNoContent with default headers values -func NewObjectsClassHeadNoContent() *ObjectsClassHeadNoContent { - return &ObjectsClassHeadNoContent{} -} - -/* -ObjectsClassHeadNoContent describes a response with status code 204, with default header values. - -Object exists. -*/ -type ObjectsClassHeadNoContent struct { -} - -// IsSuccess returns true when this objects class head no content response has a 2xx status code -func (o *ObjectsClassHeadNoContent) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this objects class head no content response has a 3xx status code -func (o *ObjectsClassHeadNoContent) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class head no content response has a 4xx status code -func (o *ObjectsClassHeadNoContent) IsClientError() bool { - return false -} - -// IsServerError returns true when this objects class head no content response has a 5xx status code -func (o *ObjectsClassHeadNoContent) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class head no content response a status code equal to that given -func (o *ObjectsClassHeadNoContent) IsCode(code int) bool { - return code == 204 -} - -// Code gets the status code for the objects class head no content response -func (o *ObjectsClassHeadNoContent) Code() int { - return 204 -} - -func (o *ObjectsClassHeadNoContent) Error() string { - return fmt.Sprintf("[HEAD /objects/{className}/{id}][%d] objectsClassHeadNoContent ", 204) -} - -func (o *ObjectsClassHeadNoContent) String() string { - return fmt.Sprintf("[HEAD /objects/{className}/{id}][%d] objectsClassHeadNoContent ", 204) -} - -func (o *ObjectsClassHeadNoContent) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsClassHeadUnauthorized creates a ObjectsClassHeadUnauthorized with default headers values -func NewObjectsClassHeadUnauthorized() *ObjectsClassHeadUnauthorized { - return &ObjectsClassHeadUnauthorized{} -} - -/* -ObjectsClassHeadUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type ObjectsClassHeadUnauthorized struct { -} - -// IsSuccess returns true when this objects class head unauthorized response has a 2xx status code -func (o *ObjectsClassHeadUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class head unauthorized response has a 3xx status code -func (o *ObjectsClassHeadUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class head unauthorized response has a 4xx status code -func (o *ObjectsClassHeadUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects class head unauthorized response has a 5xx status code -func (o *ObjectsClassHeadUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class head unauthorized response a status code equal to that given -func (o *ObjectsClassHeadUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the objects class head unauthorized response -func (o *ObjectsClassHeadUnauthorized) Code() int { - return 401 -} - -func (o *ObjectsClassHeadUnauthorized) Error() string { - return fmt.Sprintf("[HEAD /objects/{className}/{id}][%d] objectsClassHeadUnauthorized ", 401) -} - -func (o *ObjectsClassHeadUnauthorized) String() string { - return fmt.Sprintf("[HEAD /objects/{className}/{id}][%d] objectsClassHeadUnauthorized ", 401) -} - -func (o *ObjectsClassHeadUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsClassHeadForbidden creates a ObjectsClassHeadForbidden with default headers values -func NewObjectsClassHeadForbidden() *ObjectsClassHeadForbidden { - return &ObjectsClassHeadForbidden{} -} - -/* -ObjectsClassHeadForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type ObjectsClassHeadForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects class head forbidden response has a 2xx status code -func (o *ObjectsClassHeadForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class head forbidden response has a 3xx status code -func (o *ObjectsClassHeadForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class head forbidden response has a 4xx status code -func (o *ObjectsClassHeadForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects class head forbidden response has a 5xx status code -func (o *ObjectsClassHeadForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class head forbidden response a status code equal to that given -func (o *ObjectsClassHeadForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the objects class head forbidden response -func (o *ObjectsClassHeadForbidden) Code() int { - return 403 -} - -func (o *ObjectsClassHeadForbidden) Error() string { - return fmt.Sprintf("[HEAD /objects/{className}/{id}][%d] objectsClassHeadForbidden %+v", 403, o.Payload) -} - -func (o *ObjectsClassHeadForbidden) String() string { - return fmt.Sprintf("[HEAD /objects/{className}/{id}][%d] objectsClassHeadForbidden %+v", 403, o.Payload) -} - -func (o *ObjectsClassHeadForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsClassHeadForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsClassHeadNotFound creates a ObjectsClassHeadNotFound with default headers values -func NewObjectsClassHeadNotFound() *ObjectsClassHeadNotFound { - return &ObjectsClassHeadNotFound{} -} - -/* -ObjectsClassHeadNotFound describes a response with status code 404, with default header values. - -Object doesn't exist. -*/ -type ObjectsClassHeadNotFound struct { -} - -// IsSuccess returns true when this objects class head not found response has a 2xx status code -func (o *ObjectsClassHeadNotFound) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class head not found response has a 3xx status code -func (o *ObjectsClassHeadNotFound) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class head not found response has a 4xx status code -func (o *ObjectsClassHeadNotFound) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects class head not found response has a 5xx status code -func (o *ObjectsClassHeadNotFound) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class head not found response a status code equal to that given -func (o *ObjectsClassHeadNotFound) IsCode(code int) bool { - return code == 404 -} - -// Code gets the status code for the objects class head not found response -func (o *ObjectsClassHeadNotFound) Code() int { - return 404 -} - -func (o *ObjectsClassHeadNotFound) Error() string { - return fmt.Sprintf("[HEAD /objects/{className}/{id}][%d] objectsClassHeadNotFound ", 404) -} - -func (o *ObjectsClassHeadNotFound) String() string { - return fmt.Sprintf("[HEAD /objects/{className}/{id}][%d] objectsClassHeadNotFound ", 404) -} - -func (o *ObjectsClassHeadNotFound) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsClassHeadUnprocessableEntity creates a ObjectsClassHeadUnprocessableEntity with default headers values -func NewObjectsClassHeadUnprocessableEntity() *ObjectsClassHeadUnprocessableEntity { - return &ObjectsClassHeadUnprocessableEntity{} -} - -/* -ObjectsClassHeadUnprocessableEntity describes a response with status code 422, with default header values. - -Request is well-formed (i.e., syntactically correct), but erroneous. -*/ -type ObjectsClassHeadUnprocessableEntity struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects class head unprocessable entity response has a 2xx status code -func (o *ObjectsClassHeadUnprocessableEntity) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class head unprocessable entity response has a 3xx status code -func (o *ObjectsClassHeadUnprocessableEntity) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class head unprocessable entity response has a 4xx status code -func (o *ObjectsClassHeadUnprocessableEntity) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects class head unprocessable entity response has a 5xx status code -func (o *ObjectsClassHeadUnprocessableEntity) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class head unprocessable entity response a status code equal to that given -func (o *ObjectsClassHeadUnprocessableEntity) IsCode(code int) bool { - return code == 422 -} - -// Code gets the status code for the objects class head unprocessable entity response -func (o *ObjectsClassHeadUnprocessableEntity) Code() int { - return 422 -} - -func (o *ObjectsClassHeadUnprocessableEntity) Error() string { - return fmt.Sprintf("[HEAD /objects/{className}/{id}][%d] objectsClassHeadUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *ObjectsClassHeadUnprocessableEntity) String() string { - return fmt.Sprintf("[HEAD /objects/{className}/{id}][%d] objectsClassHeadUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *ObjectsClassHeadUnprocessableEntity) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsClassHeadUnprocessableEntity) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsClassHeadInternalServerError creates a ObjectsClassHeadInternalServerError with default headers values -func NewObjectsClassHeadInternalServerError() *ObjectsClassHeadInternalServerError { - return &ObjectsClassHeadInternalServerError{} -} - -/* -ObjectsClassHeadInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type ObjectsClassHeadInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects class head internal server error response has a 2xx status code -func (o *ObjectsClassHeadInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class head internal server error response has a 3xx status code -func (o *ObjectsClassHeadInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class head internal server error response has a 4xx status code -func (o *ObjectsClassHeadInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this objects class head internal server error response has a 5xx status code -func (o *ObjectsClassHeadInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this objects class head internal server error response a status code equal to that given -func (o *ObjectsClassHeadInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the objects class head internal server error response -func (o *ObjectsClassHeadInternalServerError) Code() int { - return 500 -} - -func (o *ObjectsClassHeadInternalServerError) Error() string { - return fmt.Sprintf("[HEAD /objects/{className}/{id}][%d] objectsClassHeadInternalServerError %+v", 500, o.Payload) -} - -func (o *ObjectsClassHeadInternalServerError) String() string { - return fmt.Sprintf("[HEAD /objects/{className}/{id}][%d] objectsClassHeadInternalServerError %+v", 500, o.Payload) -} - -func (o *ObjectsClassHeadInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsClassHeadInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/objects/objects_class_patch_parameters.go b/client/objects/objects_class_patch_parameters.go deleted file mode 100644 index 83949f1819370df801d724d859f9fabb6b57be8b..0000000000000000000000000000000000000000 --- a/client/objects/objects_class_patch_parameters.go +++ /dev/null @@ -1,244 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewObjectsClassPatchParams creates a new ObjectsClassPatchParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewObjectsClassPatchParams() *ObjectsClassPatchParams { - return &ObjectsClassPatchParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewObjectsClassPatchParamsWithTimeout creates a new ObjectsClassPatchParams object -// with the ability to set a timeout on a request. -func NewObjectsClassPatchParamsWithTimeout(timeout time.Duration) *ObjectsClassPatchParams { - return &ObjectsClassPatchParams{ - timeout: timeout, - } -} - -// NewObjectsClassPatchParamsWithContext creates a new ObjectsClassPatchParams object -// with the ability to set a context for a request. -func NewObjectsClassPatchParamsWithContext(ctx context.Context) *ObjectsClassPatchParams { - return &ObjectsClassPatchParams{ - Context: ctx, - } -} - -// NewObjectsClassPatchParamsWithHTTPClient creates a new ObjectsClassPatchParams object -// with the ability to set a custom HTTPClient for a request. -func NewObjectsClassPatchParamsWithHTTPClient(client *http.Client) *ObjectsClassPatchParams { - return &ObjectsClassPatchParams{ - HTTPClient: client, - } -} - -/* -ObjectsClassPatchParams contains all the parameters to send to the API endpoint - - for the objects class patch operation. - - Typically these are written to a http.Request. -*/ -type ObjectsClassPatchParams struct { - - /* Body. - - RFC 7396-style patch, the body contains the object to merge into the existing object. - */ - Body *models.Object - - /* ClassName. - - The class name as defined in the schema - */ - ClassName string - - /* ConsistencyLevel. - - Determines how many replicas must acknowledge a request before it is considered successful - */ - ConsistencyLevel *string - - /* ID. - - The uuid of the data object to update. - - Format: uuid - */ - ID strfmt.UUID - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the objects class patch params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *ObjectsClassPatchParams) WithDefaults() *ObjectsClassPatchParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the objects class patch params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *ObjectsClassPatchParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the objects class patch params -func (o *ObjectsClassPatchParams) WithTimeout(timeout time.Duration) *ObjectsClassPatchParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the objects class patch params -func (o *ObjectsClassPatchParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the objects class patch params -func (o *ObjectsClassPatchParams) WithContext(ctx context.Context) *ObjectsClassPatchParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the objects class patch params -func (o *ObjectsClassPatchParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the objects class patch params -func (o *ObjectsClassPatchParams) WithHTTPClient(client *http.Client) *ObjectsClassPatchParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the objects class patch params -func (o *ObjectsClassPatchParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithBody adds the body to the objects class patch params -func (o *ObjectsClassPatchParams) WithBody(body *models.Object) *ObjectsClassPatchParams { - o.SetBody(body) - return o -} - -// SetBody adds the body to the objects class patch params -func (o *ObjectsClassPatchParams) SetBody(body *models.Object) { - o.Body = body -} - -// WithClassName adds the className to the objects class patch params -func (o *ObjectsClassPatchParams) WithClassName(className string) *ObjectsClassPatchParams { - o.SetClassName(className) - return o -} - -// SetClassName adds the className to the objects class patch params -func (o *ObjectsClassPatchParams) SetClassName(className string) { - o.ClassName = className -} - -// WithConsistencyLevel adds the consistencyLevel to the objects class patch params -func (o *ObjectsClassPatchParams) WithConsistencyLevel(consistencyLevel *string) *ObjectsClassPatchParams { - o.SetConsistencyLevel(consistencyLevel) - return o -} - -// SetConsistencyLevel adds the consistencyLevel to the objects class patch params -func (o *ObjectsClassPatchParams) SetConsistencyLevel(consistencyLevel *string) { - o.ConsistencyLevel = consistencyLevel -} - -// WithID adds the id to the objects class patch params -func (o *ObjectsClassPatchParams) WithID(id strfmt.UUID) *ObjectsClassPatchParams { - o.SetID(id) - return o -} - -// SetID adds the id to the objects class patch params -func (o *ObjectsClassPatchParams) SetID(id strfmt.UUID) { - o.ID = id -} - -// WriteToRequest writes these params to a swagger request -func (o *ObjectsClassPatchParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - if o.Body != nil { - if err := r.SetBodyParam(o.Body); err != nil { - return err - } - } - - // path param className - if err := r.SetPathParam("className", o.ClassName); err != nil { - return err - } - - if o.ConsistencyLevel != nil { - - // query param consistency_level - var qrConsistencyLevel string - - if o.ConsistencyLevel != nil { - qrConsistencyLevel = *o.ConsistencyLevel - } - qConsistencyLevel := qrConsistencyLevel - if qConsistencyLevel != "" { - - if err := r.SetQueryParam("consistency_level", qConsistencyLevel); err != nil { - return err - } - } - } - - // path param id - if err := r.SetPathParam("id", o.ID.String()); err != nil { - return err - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/objects/objects_class_patch_responses.go b/client/objects/objects_class_patch_responses.go deleted file mode 100644 index e2d904ddfc50495c56bb2d7c2e17cc8c0196c799..0000000000000000000000000000000000000000 --- a/client/objects/objects_class_patch_responses.go +++ /dev/null @@ -1,522 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsClassPatchReader is a Reader for the ObjectsClassPatch structure. -type ObjectsClassPatchReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *ObjectsClassPatchReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 204: - result := NewObjectsClassPatchNoContent() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 400: - result := NewObjectsClassPatchBadRequest() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 401: - result := NewObjectsClassPatchUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewObjectsClassPatchForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 404: - result := NewObjectsClassPatchNotFound() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 422: - result := NewObjectsClassPatchUnprocessableEntity() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewObjectsClassPatchInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewObjectsClassPatchNoContent creates a ObjectsClassPatchNoContent with default headers values -func NewObjectsClassPatchNoContent() *ObjectsClassPatchNoContent { - return &ObjectsClassPatchNoContent{} -} - -/* -ObjectsClassPatchNoContent describes a response with status code 204, with default header values. - -Successfully applied. No content provided. -*/ -type ObjectsClassPatchNoContent struct { -} - -// IsSuccess returns true when this objects class patch no content response has a 2xx status code -func (o *ObjectsClassPatchNoContent) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this objects class patch no content response has a 3xx status code -func (o *ObjectsClassPatchNoContent) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class patch no content response has a 4xx status code -func (o *ObjectsClassPatchNoContent) IsClientError() bool { - return false -} - -// IsServerError returns true when this objects class patch no content response has a 5xx status code -func (o *ObjectsClassPatchNoContent) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class patch no content response a status code equal to that given -func (o *ObjectsClassPatchNoContent) IsCode(code int) bool { - return code == 204 -} - -// Code gets the status code for the objects class patch no content response -func (o *ObjectsClassPatchNoContent) Code() int { - return 204 -} - -func (o *ObjectsClassPatchNoContent) Error() string { - return fmt.Sprintf("[PATCH /objects/{className}/{id}][%d] objectsClassPatchNoContent ", 204) -} - -func (o *ObjectsClassPatchNoContent) String() string { - return fmt.Sprintf("[PATCH /objects/{className}/{id}][%d] objectsClassPatchNoContent ", 204) -} - -func (o *ObjectsClassPatchNoContent) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsClassPatchBadRequest creates a ObjectsClassPatchBadRequest with default headers values -func NewObjectsClassPatchBadRequest() *ObjectsClassPatchBadRequest { - return &ObjectsClassPatchBadRequest{} -} - -/* -ObjectsClassPatchBadRequest describes a response with status code 400, with default header values. - -The patch-JSON is malformed. -*/ -type ObjectsClassPatchBadRequest struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects class patch bad request response has a 2xx status code -func (o *ObjectsClassPatchBadRequest) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class patch bad request response has a 3xx status code -func (o *ObjectsClassPatchBadRequest) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class patch bad request response has a 4xx status code -func (o *ObjectsClassPatchBadRequest) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects class patch bad request response has a 5xx status code -func (o *ObjectsClassPatchBadRequest) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class patch bad request response a status code equal to that given -func (o *ObjectsClassPatchBadRequest) IsCode(code int) bool { - return code == 400 -} - -// Code gets the status code for the objects class patch bad request response -func (o *ObjectsClassPatchBadRequest) Code() int { - return 400 -} - -func (o *ObjectsClassPatchBadRequest) Error() string { - return fmt.Sprintf("[PATCH /objects/{className}/{id}][%d] objectsClassPatchBadRequest %+v", 400, o.Payload) -} - -func (o *ObjectsClassPatchBadRequest) String() string { - return fmt.Sprintf("[PATCH /objects/{className}/{id}][%d] objectsClassPatchBadRequest %+v", 400, o.Payload) -} - -func (o *ObjectsClassPatchBadRequest) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsClassPatchBadRequest) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsClassPatchUnauthorized creates a ObjectsClassPatchUnauthorized with default headers values -func NewObjectsClassPatchUnauthorized() *ObjectsClassPatchUnauthorized { - return &ObjectsClassPatchUnauthorized{} -} - -/* -ObjectsClassPatchUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type ObjectsClassPatchUnauthorized struct { -} - -// IsSuccess returns true when this objects class patch unauthorized response has a 2xx status code -func (o *ObjectsClassPatchUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class patch unauthorized response has a 3xx status code -func (o *ObjectsClassPatchUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class patch unauthorized response has a 4xx status code -func (o *ObjectsClassPatchUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects class patch unauthorized response has a 5xx status code -func (o *ObjectsClassPatchUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class patch unauthorized response a status code equal to that given -func (o *ObjectsClassPatchUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the objects class patch unauthorized response -func (o *ObjectsClassPatchUnauthorized) Code() int { - return 401 -} - -func (o *ObjectsClassPatchUnauthorized) Error() string { - return fmt.Sprintf("[PATCH /objects/{className}/{id}][%d] objectsClassPatchUnauthorized ", 401) -} - -func (o *ObjectsClassPatchUnauthorized) String() string { - return fmt.Sprintf("[PATCH /objects/{className}/{id}][%d] objectsClassPatchUnauthorized ", 401) -} - -func (o *ObjectsClassPatchUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsClassPatchForbidden creates a ObjectsClassPatchForbidden with default headers values -func NewObjectsClassPatchForbidden() *ObjectsClassPatchForbidden { - return &ObjectsClassPatchForbidden{} -} - -/* -ObjectsClassPatchForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type ObjectsClassPatchForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects class patch forbidden response has a 2xx status code -func (o *ObjectsClassPatchForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class patch forbidden response has a 3xx status code -func (o *ObjectsClassPatchForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class patch forbidden response has a 4xx status code -func (o *ObjectsClassPatchForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects class patch forbidden response has a 5xx status code -func (o *ObjectsClassPatchForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class patch forbidden response a status code equal to that given -func (o *ObjectsClassPatchForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the objects class patch forbidden response -func (o *ObjectsClassPatchForbidden) Code() int { - return 403 -} - -func (o *ObjectsClassPatchForbidden) Error() string { - return fmt.Sprintf("[PATCH /objects/{className}/{id}][%d] objectsClassPatchForbidden %+v", 403, o.Payload) -} - -func (o *ObjectsClassPatchForbidden) String() string { - return fmt.Sprintf("[PATCH /objects/{className}/{id}][%d] objectsClassPatchForbidden %+v", 403, o.Payload) -} - -func (o *ObjectsClassPatchForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsClassPatchForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsClassPatchNotFound creates a ObjectsClassPatchNotFound with default headers values -func NewObjectsClassPatchNotFound() *ObjectsClassPatchNotFound { - return &ObjectsClassPatchNotFound{} -} - -/* -ObjectsClassPatchNotFound describes a response with status code 404, with default header values. - -Successful query result but no resource was found. -*/ -type ObjectsClassPatchNotFound struct { -} - -// IsSuccess returns true when this objects class patch not found response has a 2xx status code -func (o *ObjectsClassPatchNotFound) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class patch not found response has a 3xx status code -func (o *ObjectsClassPatchNotFound) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class patch not found response has a 4xx status code -func (o *ObjectsClassPatchNotFound) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects class patch not found response has a 5xx status code -func (o *ObjectsClassPatchNotFound) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class patch not found response a status code equal to that given -func (o *ObjectsClassPatchNotFound) IsCode(code int) bool { - return code == 404 -} - -// Code gets the status code for the objects class patch not found response -func (o *ObjectsClassPatchNotFound) Code() int { - return 404 -} - -func (o *ObjectsClassPatchNotFound) Error() string { - return fmt.Sprintf("[PATCH /objects/{className}/{id}][%d] objectsClassPatchNotFound ", 404) -} - -func (o *ObjectsClassPatchNotFound) String() string { - return fmt.Sprintf("[PATCH /objects/{className}/{id}][%d] objectsClassPatchNotFound ", 404) -} - -func (o *ObjectsClassPatchNotFound) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsClassPatchUnprocessableEntity creates a ObjectsClassPatchUnprocessableEntity with default headers values -func NewObjectsClassPatchUnprocessableEntity() *ObjectsClassPatchUnprocessableEntity { - return &ObjectsClassPatchUnprocessableEntity{} -} - -/* -ObjectsClassPatchUnprocessableEntity describes a response with status code 422, with default header values. - -The patch-JSON is valid but unprocessable. -*/ -type ObjectsClassPatchUnprocessableEntity struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects class patch unprocessable entity response has a 2xx status code -func (o *ObjectsClassPatchUnprocessableEntity) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class patch unprocessable entity response has a 3xx status code -func (o *ObjectsClassPatchUnprocessableEntity) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class patch unprocessable entity response has a 4xx status code -func (o *ObjectsClassPatchUnprocessableEntity) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects class patch unprocessable entity response has a 5xx status code -func (o *ObjectsClassPatchUnprocessableEntity) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class patch unprocessable entity response a status code equal to that given -func (o *ObjectsClassPatchUnprocessableEntity) IsCode(code int) bool { - return code == 422 -} - -// Code gets the status code for the objects class patch unprocessable entity response -func (o *ObjectsClassPatchUnprocessableEntity) Code() int { - return 422 -} - -func (o *ObjectsClassPatchUnprocessableEntity) Error() string { - return fmt.Sprintf("[PATCH /objects/{className}/{id}][%d] objectsClassPatchUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *ObjectsClassPatchUnprocessableEntity) String() string { - return fmt.Sprintf("[PATCH /objects/{className}/{id}][%d] objectsClassPatchUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *ObjectsClassPatchUnprocessableEntity) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsClassPatchUnprocessableEntity) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsClassPatchInternalServerError creates a ObjectsClassPatchInternalServerError with default headers values -func NewObjectsClassPatchInternalServerError() *ObjectsClassPatchInternalServerError { - return &ObjectsClassPatchInternalServerError{} -} - -/* -ObjectsClassPatchInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type ObjectsClassPatchInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects class patch internal server error response has a 2xx status code -func (o *ObjectsClassPatchInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class patch internal server error response has a 3xx status code -func (o *ObjectsClassPatchInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class patch internal server error response has a 4xx status code -func (o *ObjectsClassPatchInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this objects class patch internal server error response has a 5xx status code -func (o *ObjectsClassPatchInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this objects class patch internal server error response a status code equal to that given -func (o *ObjectsClassPatchInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the objects class patch internal server error response -func (o *ObjectsClassPatchInternalServerError) Code() int { - return 500 -} - -func (o *ObjectsClassPatchInternalServerError) Error() string { - return fmt.Sprintf("[PATCH /objects/{className}/{id}][%d] objectsClassPatchInternalServerError %+v", 500, o.Payload) -} - -func (o *ObjectsClassPatchInternalServerError) String() string { - return fmt.Sprintf("[PATCH /objects/{className}/{id}][%d] objectsClassPatchInternalServerError %+v", 500, o.Payload) -} - -func (o *ObjectsClassPatchInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsClassPatchInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/objects/objects_class_put_parameters.go b/client/objects/objects_class_put_parameters.go deleted file mode 100644 index 4e3632afc11999daf5b9b261377a51fd76614f1f..0000000000000000000000000000000000000000 --- a/client/objects/objects_class_put_parameters.go +++ /dev/null @@ -1,238 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewObjectsClassPutParams creates a new ObjectsClassPutParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewObjectsClassPutParams() *ObjectsClassPutParams { - return &ObjectsClassPutParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewObjectsClassPutParamsWithTimeout creates a new ObjectsClassPutParams object -// with the ability to set a timeout on a request. -func NewObjectsClassPutParamsWithTimeout(timeout time.Duration) *ObjectsClassPutParams { - return &ObjectsClassPutParams{ - timeout: timeout, - } -} - -// NewObjectsClassPutParamsWithContext creates a new ObjectsClassPutParams object -// with the ability to set a context for a request. -func NewObjectsClassPutParamsWithContext(ctx context.Context) *ObjectsClassPutParams { - return &ObjectsClassPutParams{ - Context: ctx, - } -} - -// NewObjectsClassPutParamsWithHTTPClient creates a new ObjectsClassPutParams object -// with the ability to set a custom HTTPClient for a request. -func NewObjectsClassPutParamsWithHTTPClient(client *http.Client) *ObjectsClassPutParams { - return &ObjectsClassPutParams{ - HTTPClient: client, - } -} - -/* -ObjectsClassPutParams contains all the parameters to send to the API endpoint - - for the objects class put operation. - - Typically these are written to a http.Request. -*/ -type ObjectsClassPutParams struct { - - // Body. - Body *models.Object - - // ClassName. - ClassName string - - /* ConsistencyLevel. - - Determines how many replicas must acknowledge a request before it is considered successful - */ - ConsistencyLevel *string - - /* ID. - - The uuid of the data object to update. - - Format: uuid - */ - ID strfmt.UUID - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the objects class put params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *ObjectsClassPutParams) WithDefaults() *ObjectsClassPutParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the objects class put params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *ObjectsClassPutParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the objects class put params -func (o *ObjectsClassPutParams) WithTimeout(timeout time.Duration) *ObjectsClassPutParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the objects class put params -func (o *ObjectsClassPutParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the objects class put params -func (o *ObjectsClassPutParams) WithContext(ctx context.Context) *ObjectsClassPutParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the objects class put params -func (o *ObjectsClassPutParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the objects class put params -func (o *ObjectsClassPutParams) WithHTTPClient(client *http.Client) *ObjectsClassPutParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the objects class put params -func (o *ObjectsClassPutParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithBody adds the body to the objects class put params -func (o *ObjectsClassPutParams) WithBody(body *models.Object) *ObjectsClassPutParams { - o.SetBody(body) - return o -} - -// SetBody adds the body to the objects class put params -func (o *ObjectsClassPutParams) SetBody(body *models.Object) { - o.Body = body -} - -// WithClassName adds the className to the objects class put params -func (o *ObjectsClassPutParams) WithClassName(className string) *ObjectsClassPutParams { - o.SetClassName(className) - return o -} - -// SetClassName adds the className to the objects class put params -func (o *ObjectsClassPutParams) SetClassName(className string) { - o.ClassName = className -} - -// WithConsistencyLevel adds the consistencyLevel to the objects class put params -func (o *ObjectsClassPutParams) WithConsistencyLevel(consistencyLevel *string) *ObjectsClassPutParams { - o.SetConsistencyLevel(consistencyLevel) - return o -} - -// SetConsistencyLevel adds the consistencyLevel to the objects class put params -func (o *ObjectsClassPutParams) SetConsistencyLevel(consistencyLevel *string) { - o.ConsistencyLevel = consistencyLevel -} - -// WithID adds the id to the objects class put params -func (o *ObjectsClassPutParams) WithID(id strfmt.UUID) *ObjectsClassPutParams { - o.SetID(id) - return o -} - -// SetID adds the id to the objects class put params -func (o *ObjectsClassPutParams) SetID(id strfmt.UUID) { - o.ID = id -} - -// WriteToRequest writes these params to a swagger request -func (o *ObjectsClassPutParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - if o.Body != nil { - if err := r.SetBodyParam(o.Body); err != nil { - return err - } - } - - // path param className - if err := r.SetPathParam("className", o.ClassName); err != nil { - return err - } - - if o.ConsistencyLevel != nil { - - // query param consistency_level - var qrConsistencyLevel string - - if o.ConsistencyLevel != nil { - qrConsistencyLevel = *o.ConsistencyLevel - } - qConsistencyLevel := qrConsistencyLevel - if qConsistencyLevel != "" { - - if err := r.SetQueryParam("consistency_level", qConsistencyLevel); err != nil { - return err - } - } - } - - // path param id - if err := r.SetPathParam("id", o.ID.String()); err != nil { - return err - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/objects/objects_class_put_responses.go b/client/objects/objects_class_put_responses.go deleted file mode 100644 index 4e01cf94d122ade633f1bd41e544da153a1a233e..0000000000000000000000000000000000000000 --- a/client/objects/objects_class_put_responses.go +++ /dev/null @@ -1,460 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsClassPutReader is a Reader for the ObjectsClassPut structure. -type ObjectsClassPutReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *ObjectsClassPutReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 200: - result := NewObjectsClassPutOK() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 401: - result := NewObjectsClassPutUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewObjectsClassPutForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 404: - result := NewObjectsClassPutNotFound() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 422: - result := NewObjectsClassPutUnprocessableEntity() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewObjectsClassPutInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewObjectsClassPutOK creates a ObjectsClassPutOK with default headers values -func NewObjectsClassPutOK() *ObjectsClassPutOK { - return &ObjectsClassPutOK{} -} - -/* -ObjectsClassPutOK describes a response with status code 200, with default header values. - -Successfully received. -*/ -type ObjectsClassPutOK struct { - Payload *models.Object -} - -// IsSuccess returns true when this objects class put o k response has a 2xx status code -func (o *ObjectsClassPutOK) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this objects class put o k response has a 3xx status code -func (o *ObjectsClassPutOK) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class put o k response has a 4xx status code -func (o *ObjectsClassPutOK) IsClientError() bool { - return false -} - -// IsServerError returns true when this objects class put o k response has a 5xx status code -func (o *ObjectsClassPutOK) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class put o k response a status code equal to that given -func (o *ObjectsClassPutOK) IsCode(code int) bool { - return code == 200 -} - -// Code gets the status code for the objects class put o k response -func (o *ObjectsClassPutOK) Code() int { - return 200 -} - -func (o *ObjectsClassPutOK) Error() string { - return fmt.Sprintf("[PUT /objects/{className}/{id}][%d] objectsClassPutOK %+v", 200, o.Payload) -} - -func (o *ObjectsClassPutOK) String() string { - return fmt.Sprintf("[PUT /objects/{className}/{id}][%d] objectsClassPutOK %+v", 200, o.Payload) -} - -func (o *ObjectsClassPutOK) GetPayload() *models.Object { - return o.Payload -} - -func (o *ObjectsClassPutOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.Object) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsClassPutUnauthorized creates a ObjectsClassPutUnauthorized with default headers values -func NewObjectsClassPutUnauthorized() *ObjectsClassPutUnauthorized { - return &ObjectsClassPutUnauthorized{} -} - -/* -ObjectsClassPutUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type ObjectsClassPutUnauthorized struct { -} - -// IsSuccess returns true when this objects class put unauthorized response has a 2xx status code -func (o *ObjectsClassPutUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class put unauthorized response has a 3xx status code -func (o *ObjectsClassPutUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class put unauthorized response has a 4xx status code -func (o *ObjectsClassPutUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects class put unauthorized response has a 5xx status code -func (o *ObjectsClassPutUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class put unauthorized response a status code equal to that given -func (o *ObjectsClassPutUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the objects class put unauthorized response -func (o *ObjectsClassPutUnauthorized) Code() int { - return 401 -} - -func (o *ObjectsClassPutUnauthorized) Error() string { - return fmt.Sprintf("[PUT /objects/{className}/{id}][%d] objectsClassPutUnauthorized ", 401) -} - -func (o *ObjectsClassPutUnauthorized) String() string { - return fmt.Sprintf("[PUT /objects/{className}/{id}][%d] objectsClassPutUnauthorized ", 401) -} - -func (o *ObjectsClassPutUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsClassPutForbidden creates a ObjectsClassPutForbidden with default headers values -func NewObjectsClassPutForbidden() *ObjectsClassPutForbidden { - return &ObjectsClassPutForbidden{} -} - -/* -ObjectsClassPutForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type ObjectsClassPutForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects class put forbidden response has a 2xx status code -func (o *ObjectsClassPutForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class put forbidden response has a 3xx status code -func (o *ObjectsClassPutForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class put forbidden response has a 4xx status code -func (o *ObjectsClassPutForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects class put forbidden response has a 5xx status code -func (o *ObjectsClassPutForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class put forbidden response a status code equal to that given -func (o *ObjectsClassPutForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the objects class put forbidden response -func (o *ObjectsClassPutForbidden) Code() int { - return 403 -} - -func (o *ObjectsClassPutForbidden) Error() string { - return fmt.Sprintf("[PUT /objects/{className}/{id}][%d] objectsClassPutForbidden %+v", 403, o.Payload) -} - -func (o *ObjectsClassPutForbidden) String() string { - return fmt.Sprintf("[PUT /objects/{className}/{id}][%d] objectsClassPutForbidden %+v", 403, o.Payload) -} - -func (o *ObjectsClassPutForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsClassPutForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsClassPutNotFound creates a ObjectsClassPutNotFound with default headers values -func NewObjectsClassPutNotFound() *ObjectsClassPutNotFound { - return &ObjectsClassPutNotFound{} -} - -/* -ObjectsClassPutNotFound describes a response with status code 404, with default header values. - -Successful query result but no resource was found. -*/ -type ObjectsClassPutNotFound struct { -} - -// IsSuccess returns true when this objects class put not found response has a 2xx status code -func (o *ObjectsClassPutNotFound) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class put not found response has a 3xx status code -func (o *ObjectsClassPutNotFound) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class put not found response has a 4xx status code -func (o *ObjectsClassPutNotFound) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects class put not found response has a 5xx status code -func (o *ObjectsClassPutNotFound) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class put not found response a status code equal to that given -func (o *ObjectsClassPutNotFound) IsCode(code int) bool { - return code == 404 -} - -// Code gets the status code for the objects class put not found response -func (o *ObjectsClassPutNotFound) Code() int { - return 404 -} - -func (o *ObjectsClassPutNotFound) Error() string { - return fmt.Sprintf("[PUT /objects/{className}/{id}][%d] objectsClassPutNotFound ", 404) -} - -func (o *ObjectsClassPutNotFound) String() string { - return fmt.Sprintf("[PUT /objects/{className}/{id}][%d] objectsClassPutNotFound ", 404) -} - -func (o *ObjectsClassPutNotFound) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsClassPutUnprocessableEntity creates a ObjectsClassPutUnprocessableEntity with default headers values -func NewObjectsClassPutUnprocessableEntity() *ObjectsClassPutUnprocessableEntity { - return &ObjectsClassPutUnprocessableEntity{} -} - -/* -ObjectsClassPutUnprocessableEntity describes a response with status code 422, with default header values. - -Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file? -*/ -type ObjectsClassPutUnprocessableEntity struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects class put unprocessable entity response has a 2xx status code -func (o *ObjectsClassPutUnprocessableEntity) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class put unprocessable entity response has a 3xx status code -func (o *ObjectsClassPutUnprocessableEntity) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class put unprocessable entity response has a 4xx status code -func (o *ObjectsClassPutUnprocessableEntity) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects class put unprocessable entity response has a 5xx status code -func (o *ObjectsClassPutUnprocessableEntity) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class put unprocessable entity response a status code equal to that given -func (o *ObjectsClassPutUnprocessableEntity) IsCode(code int) bool { - return code == 422 -} - -// Code gets the status code for the objects class put unprocessable entity response -func (o *ObjectsClassPutUnprocessableEntity) Code() int { - return 422 -} - -func (o *ObjectsClassPutUnprocessableEntity) Error() string { - return fmt.Sprintf("[PUT /objects/{className}/{id}][%d] objectsClassPutUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *ObjectsClassPutUnprocessableEntity) String() string { - return fmt.Sprintf("[PUT /objects/{className}/{id}][%d] objectsClassPutUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *ObjectsClassPutUnprocessableEntity) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsClassPutUnprocessableEntity) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsClassPutInternalServerError creates a ObjectsClassPutInternalServerError with default headers values -func NewObjectsClassPutInternalServerError() *ObjectsClassPutInternalServerError { - return &ObjectsClassPutInternalServerError{} -} - -/* -ObjectsClassPutInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type ObjectsClassPutInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects class put internal server error response has a 2xx status code -func (o *ObjectsClassPutInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class put internal server error response has a 3xx status code -func (o *ObjectsClassPutInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class put internal server error response has a 4xx status code -func (o *ObjectsClassPutInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this objects class put internal server error response has a 5xx status code -func (o *ObjectsClassPutInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this objects class put internal server error response a status code equal to that given -func (o *ObjectsClassPutInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the objects class put internal server error response -func (o *ObjectsClassPutInternalServerError) Code() int { - return 500 -} - -func (o *ObjectsClassPutInternalServerError) Error() string { - return fmt.Sprintf("[PUT /objects/{className}/{id}][%d] objectsClassPutInternalServerError %+v", 500, o.Payload) -} - -func (o *ObjectsClassPutInternalServerError) String() string { - return fmt.Sprintf("[PUT /objects/{className}/{id}][%d] objectsClassPutInternalServerError %+v", 500, o.Payload) -} - -func (o *ObjectsClassPutInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsClassPutInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/objects/objects_class_references_create_parameters.go b/client/objects/objects_class_references_create_parameters.go deleted file mode 100644 index a418261b889093280b059817ea0fb023a1a9b24e..0000000000000000000000000000000000000000 --- a/client/objects/objects_class_references_create_parameters.go +++ /dev/null @@ -1,297 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewObjectsClassReferencesCreateParams creates a new ObjectsClassReferencesCreateParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewObjectsClassReferencesCreateParams() *ObjectsClassReferencesCreateParams { - return &ObjectsClassReferencesCreateParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewObjectsClassReferencesCreateParamsWithTimeout creates a new ObjectsClassReferencesCreateParams object -// with the ability to set a timeout on a request. -func NewObjectsClassReferencesCreateParamsWithTimeout(timeout time.Duration) *ObjectsClassReferencesCreateParams { - return &ObjectsClassReferencesCreateParams{ - timeout: timeout, - } -} - -// NewObjectsClassReferencesCreateParamsWithContext creates a new ObjectsClassReferencesCreateParams object -// with the ability to set a context for a request. -func NewObjectsClassReferencesCreateParamsWithContext(ctx context.Context) *ObjectsClassReferencesCreateParams { - return &ObjectsClassReferencesCreateParams{ - Context: ctx, - } -} - -// NewObjectsClassReferencesCreateParamsWithHTTPClient creates a new ObjectsClassReferencesCreateParams object -// with the ability to set a custom HTTPClient for a request. -func NewObjectsClassReferencesCreateParamsWithHTTPClient(client *http.Client) *ObjectsClassReferencesCreateParams { - return &ObjectsClassReferencesCreateParams{ - HTTPClient: client, - } -} - -/* -ObjectsClassReferencesCreateParams contains all the parameters to send to the API endpoint - - for the objects class references create operation. - - Typically these are written to a http.Request. -*/ -type ObjectsClassReferencesCreateParams struct { - - // Body. - Body *models.SingleRef - - /* ClassName. - - The class name as defined in the schema - */ - ClassName string - - /* ConsistencyLevel. - - Determines how many replicas must acknowledge a request before it is considered successful - */ - ConsistencyLevel *string - - /* ID. - - Unique ID of the Object. - - Format: uuid - */ - ID strfmt.UUID - - /* PropertyName. - - Unique name of the property related to the Object. - */ - PropertyName string - - /* Tenant. - - Specifies the tenant in a request targeting a multi-tenant class - */ - Tenant *string - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the objects class references create params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *ObjectsClassReferencesCreateParams) WithDefaults() *ObjectsClassReferencesCreateParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the objects class references create params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *ObjectsClassReferencesCreateParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the objects class references create params -func (o *ObjectsClassReferencesCreateParams) WithTimeout(timeout time.Duration) *ObjectsClassReferencesCreateParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the objects class references create params -func (o *ObjectsClassReferencesCreateParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the objects class references create params -func (o *ObjectsClassReferencesCreateParams) WithContext(ctx context.Context) *ObjectsClassReferencesCreateParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the objects class references create params -func (o *ObjectsClassReferencesCreateParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the objects class references create params -func (o *ObjectsClassReferencesCreateParams) WithHTTPClient(client *http.Client) *ObjectsClassReferencesCreateParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the objects class references create params -func (o *ObjectsClassReferencesCreateParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithBody adds the body to the objects class references create params -func (o *ObjectsClassReferencesCreateParams) WithBody(body *models.SingleRef) *ObjectsClassReferencesCreateParams { - o.SetBody(body) - return o -} - -// SetBody adds the body to the objects class references create params -func (o *ObjectsClassReferencesCreateParams) SetBody(body *models.SingleRef) { - o.Body = body -} - -// WithClassName adds the className to the objects class references create params -func (o *ObjectsClassReferencesCreateParams) WithClassName(className string) *ObjectsClassReferencesCreateParams { - o.SetClassName(className) - return o -} - -// SetClassName adds the className to the objects class references create params -func (o *ObjectsClassReferencesCreateParams) SetClassName(className string) { - o.ClassName = className -} - -// WithConsistencyLevel adds the consistencyLevel to the objects class references create params -func (o *ObjectsClassReferencesCreateParams) WithConsistencyLevel(consistencyLevel *string) *ObjectsClassReferencesCreateParams { - o.SetConsistencyLevel(consistencyLevel) - return o -} - -// SetConsistencyLevel adds the consistencyLevel to the objects class references create params -func (o *ObjectsClassReferencesCreateParams) SetConsistencyLevel(consistencyLevel *string) { - o.ConsistencyLevel = consistencyLevel -} - -// WithID adds the id to the objects class references create params -func (o *ObjectsClassReferencesCreateParams) WithID(id strfmt.UUID) *ObjectsClassReferencesCreateParams { - o.SetID(id) - return o -} - -// SetID adds the id to the objects class references create params -func (o *ObjectsClassReferencesCreateParams) SetID(id strfmt.UUID) { - o.ID = id -} - -// WithPropertyName adds the propertyName to the objects class references create params -func (o *ObjectsClassReferencesCreateParams) WithPropertyName(propertyName string) *ObjectsClassReferencesCreateParams { - o.SetPropertyName(propertyName) - return o -} - -// SetPropertyName adds the propertyName to the objects class references create params -func (o *ObjectsClassReferencesCreateParams) SetPropertyName(propertyName string) { - o.PropertyName = propertyName -} - -// WithTenant adds the tenant to the objects class references create params -func (o *ObjectsClassReferencesCreateParams) WithTenant(tenant *string) *ObjectsClassReferencesCreateParams { - o.SetTenant(tenant) - return o -} - -// SetTenant adds the tenant to the objects class references create params -func (o *ObjectsClassReferencesCreateParams) SetTenant(tenant *string) { - o.Tenant = tenant -} - -// WriteToRequest writes these params to a swagger request -func (o *ObjectsClassReferencesCreateParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - if o.Body != nil { - if err := r.SetBodyParam(o.Body); err != nil { - return err - } - } - - // path param className - if err := r.SetPathParam("className", o.ClassName); err != nil { - return err - } - - if o.ConsistencyLevel != nil { - - // query param consistency_level - var qrConsistencyLevel string - - if o.ConsistencyLevel != nil { - qrConsistencyLevel = *o.ConsistencyLevel - } - qConsistencyLevel := qrConsistencyLevel - if qConsistencyLevel != "" { - - if err := r.SetQueryParam("consistency_level", qConsistencyLevel); err != nil { - return err - } - } - } - - // path param id - if err := r.SetPathParam("id", o.ID.String()); err != nil { - return err - } - - // path param propertyName - if err := r.SetPathParam("propertyName", o.PropertyName); err != nil { - return err - } - - if o.Tenant != nil { - - // query param tenant - var qrTenant string - - if o.Tenant != nil { - qrTenant = *o.Tenant - } - qTenant := qrTenant - if qTenant != "" { - - if err := r.SetQueryParam("tenant", qTenant); err != nil { - return err - } - } - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/objects/objects_class_references_create_responses.go b/client/objects/objects_class_references_create_responses.go deleted file mode 100644 index 59fc53560cd8a17bade94566eaf47ea67c795f55..0000000000000000000000000000000000000000 --- a/client/objects/objects_class_references_create_responses.go +++ /dev/null @@ -1,522 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsClassReferencesCreateReader is a Reader for the ObjectsClassReferencesCreate structure. -type ObjectsClassReferencesCreateReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *ObjectsClassReferencesCreateReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 200: - result := NewObjectsClassReferencesCreateOK() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 400: - result := NewObjectsClassReferencesCreateBadRequest() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 401: - result := NewObjectsClassReferencesCreateUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewObjectsClassReferencesCreateForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 404: - result := NewObjectsClassReferencesCreateNotFound() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 422: - result := NewObjectsClassReferencesCreateUnprocessableEntity() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewObjectsClassReferencesCreateInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewObjectsClassReferencesCreateOK creates a ObjectsClassReferencesCreateOK with default headers values -func NewObjectsClassReferencesCreateOK() *ObjectsClassReferencesCreateOK { - return &ObjectsClassReferencesCreateOK{} -} - -/* -ObjectsClassReferencesCreateOK describes a response with status code 200, with default header values. - -Successfully added the reference. -*/ -type ObjectsClassReferencesCreateOK struct { -} - -// IsSuccess returns true when this objects class references create o k response has a 2xx status code -func (o *ObjectsClassReferencesCreateOK) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this objects class references create o k response has a 3xx status code -func (o *ObjectsClassReferencesCreateOK) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class references create o k response has a 4xx status code -func (o *ObjectsClassReferencesCreateOK) IsClientError() bool { - return false -} - -// IsServerError returns true when this objects class references create o k response has a 5xx status code -func (o *ObjectsClassReferencesCreateOK) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class references create o k response a status code equal to that given -func (o *ObjectsClassReferencesCreateOK) IsCode(code int) bool { - return code == 200 -} - -// Code gets the status code for the objects class references create o k response -func (o *ObjectsClassReferencesCreateOK) Code() int { - return 200 -} - -func (o *ObjectsClassReferencesCreateOK) Error() string { - return fmt.Sprintf("[POST /objects/{className}/{id}/references/{propertyName}][%d] objectsClassReferencesCreateOK ", 200) -} - -func (o *ObjectsClassReferencesCreateOK) String() string { - return fmt.Sprintf("[POST /objects/{className}/{id}/references/{propertyName}][%d] objectsClassReferencesCreateOK ", 200) -} - -func (o *ObjectsClassReferencesCreateOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsClassReferencesCreateBadRequest creates a ObjectsClassReferencesCreateBadRequest with default headers values -func NewObjectsClassReferencesCreateBadRequest() *ObjectsClassReferencesCreateBadRequest { - return &ObjectsClassReferencesCreateBadRequest{} -} - -/* -ObjectsClassReferencesCreateBadRequest describes a response with status code 400, with default header values. - -Malformed request. -*/ -type ObjectsClassReferencesCreateBadRequest struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects class references create bad request response has a 2xx status code -func (o *ObjectsClassReferencesCreateBadRequest) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class references create bad request response has a 3xx status code -func (o *ObjectsClassReferencesCreateBadRequest) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class references create bad request response has a 4xx status code -func (o *ObjectsClassReferencesCreateBadRequest) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects class references create bad request response has a 5xx status code -func (o *ObjectsClassReferencesCreateBadRequest) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class references create bad request response a status code equal to that given -func (o *ObjectsClassReferencesCreateBadRequest) IsCode(code int) bool { - return code == 400 -} - -// Code gets the status code for the objects class references create bad request response -func (o *ObjectsClassReferencesCreateBadRequest) Code() int { - return 400 -} - -func (o *ObjectsClassReferencesCreateBadRequest) Error() string { - return fmt.Sprintf("[POST /objects/{className}/{id}/references/{propertyName}][%d] objectsClassReferencesCreateBadRequest %+v", 400, o.Payload) -} - -func (o *ObjectsClassReferencesCreateBadRequest) String() string { - return fmt.Sprintf("[POST /objects/{className}/{id}/references/{propertyName}][%d] objectsClassReferencesCreateBadRequest %+v", 400, o.Payload) -} - -func (o *ObjectsClassReferencesCreateBadRequest) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsClassReferencesCreateBadRequest) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsClassReferencesCreateUnauthorized creates a ObjectsClassReferencesCreateUnauthorized with default headers values -func NewObjectsClassReferencesCreateUnauthorized() *ObjectsClassReferencesCreateUnauthorized { - return &ObjectsClassReferencesCreateUnauthorized{} -} - -/* -ObjectsClassReferencesCreateUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type ObjectsClassReferencesCreateUnauthorized struct { -} - -// IsSuccess returns true when this objects class references create unauthorized response has a 2xx status code -func (o *ObjectsClassReferencesCreateUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class references create unauthorized response has a 3xx status code -func (o *ObjectsClassReferencesCreateUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class references create unauthorized response has a 4xx status code -func (o *ObjectsClassReferencesCreateUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects class references create unauthorized response has a 5xx status code -func (o *ObjectsClassReferencesCreateUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class references create unauthorized response a status code equal to that given -func (o *ObjectsClassReferencesCreateUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the objects class references create unauthorized response -func (o *ObjectsClassReferencesCreateUnauthorized) Code() int { - return 401 -} - -func (o *ObjectsClassReferencesCreateUnauthorized) Error() string { - return fmt.Sprintf("[POST /objects/{className}/{id}/references/{propertyName}][%d] objectsClassReferencesCreateUnauthorized ", 401) -} - -func (o *ObjectsClassReferencesCreateUnauthorized) String() string { - return fmt.Sprintf("[POST /objects/{className}/{id}/references/{propertyName}][%d] objectsClassReferencesCreateUnauthorized ", 401) -} - -func (o *ObjectsClassReferencesCreateUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsClassReferencesCreateForbidden creates a ObjectsClassReferencesCreateForbidden with default headers values -func NewObjectsClassReferencesCreateForbidden() *ObjectsClassReferencesCreateForbidden { - return &ObjectsClassReferencesCreateForbidden{} -} - -/* -ObjectsClassReferencesCreateForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type ObjectsClassReferencesCreateForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects class references create forbidden response has a 2xx status code -func (o *ObjectsClassReferencesCreateForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class references create forbidden response has a 3xx status code -func (o *ObjectsClassReferencesCreateForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class references create forbidden response has a 4xx status code -func (o *ObjectsClassReferencesCreateForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects class references create forbidden response has a 5xx status code -func (o *ObjectsClassReferencesCreateForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class references create forbidden response a status code equal to that given -func (o *ObjectsClassReferencesCreateForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the objects class references create forbidden response -func (o *ObjectsClassReferencesCreateForbidden) Code() int { - return 403 -} - -func (o *ObjectsClassReferencesCreateForbidden) Error() string { - return fmt.Sprintf("[POST /objects/{className}/{id}/references/{propertyName}][%d] objectsClassReferencesCreateForbidden %+v", 403, o.Payload) -} - -func (o *ObjectsClassReferencesCreateForbidden) String() string { - return fmt.Sprintf("[POST /objects/{className}/{id}/references/{propertyName}][%d] objectsClassReferencesCreateForbidden %+v", 403, o.Payload) -} - -func (o *ObjectsClassReferencesCreateForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsClassReferencesCreateForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsClassReferencesCreateNotFound creates a ObjectsClassReferencesCreateNotFound with default headers values -func NewObjectsClassReferencesCreateNotFound() *ObjectsClassReferencesCreateNotFound { - return &ObjectsClassReferencesCreateNotFound{} -} - -/* -ObjectsClassReferencesCreateNotFound describes a response with status code 404, with default header values. - -Source object doesn't exist. -*/ -type ObjectsClassReferencesCreateNotFound struct { -} - -// IsSuccess returns true when this objects class references create not found response has a 2xx status code -func (o *ObjectsClassReferencesCreateNotFound) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class references create not found response has a 3xx status code -func (o *ObjectsClassReferencesCreateNotFound) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class references create not found response has a 4xx status code -func (o *ObjectsClassReferencesCreateNotFound) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects class references create not found response has a 5xx status code -func (o *ObjectsClassReferencesCreateNotFound) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class references create not found response a status code equal to that given -func (o *ObjectsClassReferencesCreateNotFound) IsCode(code int) bool { - return code == 404 -} - -// Code gets the status code for the objects class references create not found response -func (o *ObjectsClassReferencesCreateNotFound) Code() int { - return 404 -} - -func (o *ObjectsClassReferencesCreateNotFound) Error() string { - return fmt.Sprintf("[POST /objects/{className}/{id}/references/{propertyName}][%d] objectsClassReferencesCreateNotFound ", 404) -} - -func (o *ObjectsClassReferencesCreateNotFound) String() string { - return fmt.Sprintf("[POST /objects/{className}/{id}/references/{propertyName}][%d] objectsClassReferencesCreateNotFound ", 404) -} - -func (o *ObjectsClassReferencesCreateNotFound) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsClassReferencesCreateUnprocessableEntity creates a ObjectsClassReferencesCreateUnprocessableEntity with default headers values -func NewObjectsClassReferencesCreateUnprocessableEntity() *ObjectsClassReferencesCreateUnprocessableEntity { - return &ObjectsClassReferencesCreateUnprocessableEntity{} -} - -/* -ObjectsClassReferencesCreateUnprocessableEntity describes a response with status code 422, with default header values. - -Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the property exists or that it is a class? -*/ -type ObjectsClassReferencesCreateUnprocessableEntity struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects class references create unprocessable entity response has a 2xx status code -func (o *ObjectsClassReferencesCreateUnprocessableEntity) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class references create unprocessable entity response has a 3xx status code -func (o *ObjectsClassReferencesCreateUnprocessableEntity) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class references create unprocessable entity response has a 4xx status code -func (o *ObjectsClassReferencesCreateUnprocessableEntity) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects class references create unprocessable entity response has a 5xx status code -func (o *ObjectsClassReferencesCreateUnprocessableEntity) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class references create unprocessable entity response a status code equal to that given -func (o *ObjectsClassReferencesCreateUnprocessableEntity) IsCode(code int) bool { - return code == 422 -} - -// Code gets the status code for the objects class references create unprocessable entity response -func (o *ObjectsClassReferencesCreateUnprocessableEntity) Code() int { - return 422 -} - -func (o *ObjectsClassReferencesCreateUnprocessableEntity) Error() string { - return fmt.Sprintf("[POST /objects/{className}/{id}/references/{propertyName}][%d] objectsClassReferencesCreateUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *ObjectsClassReferencesCreateUnprocessableEntity) String() string { - return fmt.Sprintf("[POST /objects/{className}/{id}/references/{propertyName}][%d] objectsClassReferencesCreateUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *ObjectsClassReferencesCreateUnprocessableEntity) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsClassReferencesCreateUnprocessableEntity) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsClassReferencesCreateInternalServerError creates a ObjectsClassReferencesCreateInternalServerError with default headers values -func NewObjectsClassReferencesCreateInternalServerError() *ObjectsClassReferencesCreateInternalServerError { - return &ObjectsClassReferencesCreateInternalServerError{} -} - -/* -ObjectsClassReferencesCreateInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type ObjectsClassReferencesCreateInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects class references create internal server error response has a 2xx status code -func (o *ObjectsClassReferencesCreateInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class references create internal server error response has a 3xx status code -func (o *ObjectsClassReferencesCreateInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class references create internal server error response has a 4xx status code -func (o *ObjectsClassReferencesCreateInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this objects class references create internal server error response has a 5xx status code -func (o *ObjectsClassReferencesCreateInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this objects class references create internal server error response a status code equal to that given -func (o *ObjectsClassReferencesCreateInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the objects class references create internal server error response -func (o *ObjectsClassReferencesCreateInternalServerError) Code() int { - return 500 -} - -func (o *ObjectsClassReferencesCreateInternalServerError) Error() string { - return fmt.Sprintf("[POST /objects/{className}/{id}/references/{propertyName}][%d] objectsClassReferencesCreateInternalServerError %+v", 500, o.Payload) -} - -func (o *ObjectsClassReferencesCreateInternalServerError) String() string { - return fmt.Sprintf("[POST /objects/{className}/{id}/references/{propertyName}][%d] objectsClassReferencesCreateInternalServerError %+v", 500, o.Payload) -} - -func (o *ObjectsClassReferencesCreateInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsClassReferencesCreateInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/objects/objects_class_references_delete_parameters.go b/client/objects/objects_class_references_delete_parameters.go deleted file mode 100644 index f15abafc15e2a55a45f76190bc0045c975981db3..0000000000000000000000000000000000000000 --- a/client/objects/objects_class_references_delete_parameters.go +++ /dev/null @@ -1,297 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewObjectsClassReferencesDeleteParams creates a new ObjectsClassReferencesDeleteParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewObjectsClassReferencesDeleteParams() *ObjectsClassReferencesDeleteParams { - return &ObjectsClassReferencesDeleteParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewObjectsClassReferencesDeleteParamsWithTimeout creates a new ObjectsClassReferencesDeleteParams object -// with the ability to set a timeout on a request. -func NewObjectsClassReferencesDeleteParamsWithTimeout(timeout time.Duration) *ObjectsClassReferencesDeleteParams { - return &ObjectsClassReferencesDeleteParams{ - timeout: timeout, - } -} - -// NewObjectsClassReferencesDeleteParamsWithContext creates a new ObjectsClassReferencesDeleteParams object -// with the ability to set a context for a request. -func NewObjectsClassReferencesDeleteParamsWithContext(ctx context.Context) *ObjectsClassReferencesDeleteParams { - return &ObjectsClassReferencesDeleteParams{ - Context: ctx, - } -} - -// NewObjectsClassReferencesDeleteParamsWithHTTPClient creates a new ObjectsClassReferencesDeleteParams object -// with the ability to set a custom HTTPClient for a request. -func NewObjectsClassReferencesDeleteParamsWithHTTPClient(client *http.Client) *ObjectsClassReferencesDeleteParams { - return &ObjectsClassReferencesDeleteParams{ - HTTPClient: client, - } -} - -/* -ObjectsClassReferencesDeleteParams contains all the parameters to send to the API endpoint - - for the objects class references delete operation. - - Typically these are written to a http.Request. -*/ -type ObjectsClassReferencesDeleteParams struct { - - // Body. - Body *models.SingleRef - - /* ClassName. - - The class name as defined in the schema - */ - ClassName string - - /* ConsistencyLevel. - - Determines how many replicas must acknowledge a request before it is considered successful - */ - ConsistencyLevel *string - - /* ID. - - Unique ID of the Object. - - Format: uuid - */ - ID strfmt.UUID - - /* PropertyName. - - Unique name of the property related to the Object. - */ - PropertyName string - - /* Tenant. - - Specifies the tenant in a request targeting a multi-tenant class - */ - Tenant *string - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the objects class references delete params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *ObjectsClassReferencesDeleteParams) WithDefaults() *ObjectsClassReferencesDeleteParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the objects class references delete params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *ObjectsClassReferencesDeleteParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the objects class references delete params -func (o *ObjectsClassReferencesDeleteParams) WithTimeout(timeout time.Duration) *ObjectsClassReferencesDeleteParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the objects class references delete params -func (o *ObjectsClassReferencesDeleteParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the objects class references delete params -func (o *ObjectsClassReferencesDeleteParams) WithContext(ctx context.Context) *ObjectsClassReferencesDeleteParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the objects class references delete params -func (o *ObjectsClassReferencesDeleteParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the objects class references delete params -func (o *ObjectsClassReferencesDeleteParams) WithHTTPClient(client *http.Client) *ObjectsClassReferencesDeleteParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the objects class references delete params -func (o *ObjectsClassReferencesDeleteParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithBody adds the body to the objects class references delete params -func (o *ObjectsClassReferencesDeleteParams) WithBody(body *models.SingleRef) *ObjectsClassReferencesDeleteParams { - o.SetBody(body) - return o -} - -// SetBody adds the body to the objects class references delete params -func (o *ObjectsClassReferencesDeleteParams) SetBody(body *models.SingleRef) { - o.Body = body -} - -// WithClassName adds the className to the objects class references delete params -func (o *ObjectsClassReferencesDeleteParams) WithClassName(className string) *ObjectsClassReferencesDeleteParams { - o.SetClassName(className) - return o -} - -// SetClassName adds the className to the objects class references delete params -func (o *ObjectsClassReferencesDeleteParams) SetClassName(className string) { - o.ClassName = className -} - -// WithConsistencyLevel adds the consistencyLevel to the objects class references delete params -func (o *ObjectsClassReferencesDeleteParams) WithConsistencyLevel(consistencyLevel *string) *ObjectsClassReferencesDeleteParams { - o.SetConsistencyLevel(consistencyLevel) - return o -} - -// SetConsistencyLevel adds the consistencyLevel to the objects class references delete params -func (o *ObjectsClassReferencesDeleteParams) SetConsistencyLevel(consistencyLevel *string) { - o.ConsistencyLevel = consistencyLevel -} - -// WithID adds the id to the objects class references delete params -func (o *ObjectsClassReferencesDeleteParams) WithID(id strfmt.UUID) *ObjectsClassReferencesDeleteParams { - o.SetID(id) - return o -} - -// SetID adds the id to the objects class references delete params -func (o *ObjectsClassReferencesDeleteParams) SetID(id strfmt.UUID) { - o.ID = id -} - -// WithPropertyName adds the propertyName to the objects class references delete params -func (o *ObjectsClassReferencesDeleteParams) WithPropertyName(propertyName string) *ObjectsClassReferencesDeleteParams { - o.SetPropertyName(propertyName) - return o -} - -// SetPropertyName adds the propertyName to the objects class references delete params -func (o *ObjectsClassReferencesDeleteParams) SetPropertyName(propertyName string) { - o.PropertyName = propertyName -} - -// WithTenant adds the tenant to the objects class references delete params -func (o *ObjectsClassReferencesDeleteParams) WithTenant(tenant *string) *ObjectsClassReferencesDeleteParams { - o.SetTenant(tenant) - return o -} - -// SetTenant adds the tenant to the objects class references delete params -func (o *ObjectsClassReferencesDeleteParams) SetTenant(tenant *string) { - o.Tenant = tenant -} - -// WriteToRequest writes these params to a swagger request -func (o *ObjectsClassReferencesDeleteParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - if o.Body != nil { - if err := r.SetBodyParam(o.Body); err != nil { - return err - } - } - - // path param className - if err := r.SetPathParam("className", o.ClassName); err != nil { - return err - } - - if o.ConsistencyLevel != nil { - - // query param consistency_level - var qrConsistencyLevel string - - if o.ConsistencyLevel != nil { - qrConsistencyLevel = *o.ConsistencyLevel - } - qConsistencyLevel := qrConsistencyLevel - if qConsistencyLevel != "" { - - if err := r.SetQueryParam("consistency_level", qConsistencyLevel); err != nil { - return err - } - } - } - - // path param id - if err := r.SetPathParam("id", o.ID.String()); err != nil { - return err - } - - // path param propertyName - if err := r.SetPathParam("propertyName", o.PropertyName); err != nil { - return err - } - - if o.Tenant != nil { - - // query param tenant - var qrTenant string - - if o.Tenant != nil { - qrTenant = *o.Tenant - } - qTenant := qrTenant - if qTenant != "" { - - if err := r.SetQueryParam("tenant", qTenant); err != nil { - return err - } - } - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/objects/objects_class_references_delete_responses.go b/client/objects/objects_class_references_delete_responses.go deleted file mode 100644 index 0c1d91b07ed3dde3b078f54b9cd5b26ca8e57aea..0000000000000000000000000000000000000000 --- a/client/objects/objects_class_references_delete_responses.go +++ /dev/null @@ -1,534 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsClassReferencesDeleteReader is a Reader for the ObjectsClassReferencesDelete structure. -type ObjectsClassReferencesDeleteReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *ObjectsClassReferencesDeleteReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 204: - result := NewObjectsClassReferencesDeleteNoContent() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 400: - result := NewObjectsClassReferencesDeleteBadRequest() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 401: - result := NewObjectsClassReferencesDeleteUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewObjectsClassReferencesDeleteForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 404: - result := NewObjectsClassReferencesDeleteNotFound() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 422: - result := NewObjectsClassReferencesDeleteUnprocessableEntity() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewObjectsClassReferencesDeleteInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewObjectsClassReferencesDeleteNoContent creates a ObjectsClassReferencesDeleteNoContent with default headers values -func NewObjectsClassReferencesDeleteNoContent() *ObjectsClassReferencesDeleteNoContent { - return &ObjectsClassReferencesDeleteNoContent{} -} - -/* -ObjectsClassReferencesDeleteNoContent describes a response with status code 204, with default header values. - -Successfully deleted. -*/ -type ObjectsClassReferencesDeleteNoContent struct { -} - -// IsSuccess returns true when this objects class references delete no content response has a 2xx status code -func (o *ObjectsClassReferencesDeleteNoContent) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this objects class references delete no content response has a 3xx status code -func (o *ObjectsClassReferencesDeleteNoContent) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class references delete no content response has a 4xx status code -func (o *ObjectsClassReferencesDeleteNoContent) IsClientError() bool { - return false -} - -// IsServerError returns true when this objects class references delete no content response has a 5xx status code -func (o *ObjectsClassReferencesDeleteNoContent) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class references delete no content response a status code equal to that given -func (o *ObjectsClassReferencesDeleteNoContent) IsCode(code int) bool { - return code == 204 -} - -// Code gets the status code for the objects class references delete no content response -func (o *ObjectsClassReferencesDeleteNoContent) Code() int { - return 204 -} - -func (o *ObjectsClassReferencesDeleteNoContent) Error() string { - return fmt.Sprintf("[DELETE /objects/{className}/{id}/references/{propertyName}][%d] objectsClassReferencesDeleteNoContent ", 204) -} - -func (o *ObjectsClassReferencesDeleteNoContent) String() string { - return fmt.Sprintf("[DELETE /objects/{className}/{id}/references/{propertyName}][%d] objectsClassReferencesDeleteNoContent ", 204) -} - -func (o *ObjectsClassReferencesDeleteNoContent) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsClassReferencesDeleteBadRequest creates a ObjectsClassReferencesDeleteBadRequest with default headers values -func NewObjectsClassReferencesDeleteBadRequest() *ObjectsClassReferencesDeleteBadRequest { - return &ObjectsClassReferencesDeleteBadRequest{} -} - -/* -ObjectsClassReferencesDeleteBadRequest describes a response with status code 400, with default header values. - -Malformed request. -*/ -type ObjectsClassReferencesDeleteBadRequest struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects class references delete bad request response has a 2xx status code -func (o *ObjectsClassReferencesDeleteBadRequest) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class references delete bad request response has a 3xx status code -func (o *ObjectsClassReferencesDeleteBadRequest) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class references delete bad request response has a 4xx status code -func (o *ObjectsClassReferencesDeleteBadRequest) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects class references delete bad request response has a 5xx status code -func (o *ObjectsClassReferencesDeleteBadRequest) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class references delete bad request response a status code equal to that given -func (o *ObjectsClassReferencesDeleteBadRequest) IsCode(code int) bool { - return code == 400 -} - -// Code gets the status code for the objects class references delete bad request response -func (o *ObjectsClassReferencesDeleteBadRequest) Code() int { - return 400 -} - -func (o *ObjectsClassReferencesDeleteBadRequest) Error() string { - return fmt.Sprintf("[DELETE /objects/{className}/{id}/references/{propertyName}][%d] objectsClassReferencesDeleteBadRequest %+v", 400, o.Payload) -} - -func (o *ObjectsClassReferencesDeleteBadRequest) String() string { - return fmt.Sprintf("[DELETE /objects/{className}/{id}/references/{propertyName}][%d] objectsClassReferencesDeleteBadRequest %+v", 400, o.Payload) -} - -func (o *ObjectsClassReferencesDeleteBadRequest) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsClassReferencesDeleteBadRequest) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsClassReferencesDeleteUnauthorized creates a ObjectsClassReferencesDeleteUnauthorized with default headers values -func NewObjectsClassReferencesDeleteUnauthorized() *ObjectsClassReferencesDeleteUnauthorized { - return &ObjectsClassReferencesDeleteUnauthorized{} -} - -/* -ObjectsClassReferencesDeleteUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type ObjectsClassReferencesDeleteUnauthorized struct { -} - -// IsSuccess returns true when this objects class references delete unauthorized response has a 2xx status code -func (o *ObjectsClassReferencesDeleteUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class references delete unauthorized response has a 3xx status code -func (o *ObjectsClassReferencesDeleteUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class references delete unauthorized response has a 4xx status code -func (o *ObjectsClassReferencesDeleteUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects class references delete unauthorized response has a 5xx status code -func (o *ObjectsClassReferencesDeleteUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class references delete unauthorized response a status code equal to that given -func (o *ObjectsClassReferencesDeleteUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the objects class references delete unauthorized response -func (o *ObjectsClassReferencesDeleteUnauthorized) Code() int { - return 401 -} - -func (o *ObjectsClassReferencesDeleteUnauthorized) Error() string { - return fmt.Sprintf("[DELETE /objects/{className}/{id}/references/{propertyName}][%d] objectsClassReferencesDeleteUnauthorized ", 401) -} - -func (o *ObjectsClassReferencesDeleteUnauthorized) String() string { - return fmt.Sprintf("[DELETE /objects/{className}/{id}/references/{propertyName}][%d] objectsClassReferencesDeleteUnauthorized ", 401) -} - -func (o *ObjectsClassReferencesDeleteUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsClassReferencesDeleteForbidden creates a ObjectsClassReferencesDeleteForbidden with default headers values -func NewObjectsClassReferencesDeleteForbidden() *ObjectsClassReferencesDeleteForbidden { - return &ObjectsClassReferencesDeleteForbidden{} -} - -/* -ObjectsClassReferencesDeleteForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type ObjectsClassReferencesDeleteForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects class references delete forbidden response has a 2xx status code -func (o *ObjectsClassReferencesDeleteForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class references delete forbidden response has a 3xx status code -func (o *ObjectsClassReferencesDeleteForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class references delete forbidden response has a 4xx status code -func (o *ObjectsClassReferencesDeleteForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects class references delete forbidden response has a 5xx status code -func (o *ObjectsClassReferencesDeleteForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class references delete forbidden response a status code equal to that given -func (o *ObjectsClassReferencesDeleteForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the objects class references delete forbidden response -func (o *ObjectsClassReferencesDeleteForbidden) Code() int { - return 403 -} - -func (o *ObjectsClassReferencesDeleteForbidden) Error() string { - return fmt.Sprintf("[DELETE /objects/{className}/{id}/references/{propertyName}][%d] objectsClassReferencesDeleteForbidden %+v", 403, o.Payload) -} - -func (o *ObjectsClassReferencesDeleteForbidden) String() string { - return fmt.Sprintf("[DELETE /objects/{className}/{id}/references/{propertyName}][%d] objectsClassReferencesDeleteForbidden %+v", 403, o.Payload) -} - -func (o *ObjectsClassReferencesDeleteForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsClassReferencesDeleteForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsClassReferencesDeleteNotFound creates a ObjectsClassReferencesDeleteNotFound with default headers values -func NewObjectsClassReferencesDeleteNotFound() *ObjectsClassReferencesDeleteNotFound { - return &ObjectsClassReferencesDeleteNotFound{} -} - -/* -ObjectsClassReferencesDeleteNotFound describes a response with status code 404, with default header values. - -Successful query result but no resource was found. -*/ -type ObjectsClassReferencesDeleteNotFound struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects class references delete not found response has a 2xx status code -func (o *ObjectsClassReferencesDeleteNotFound) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class references delete not found response has a 3xx status code -func (o *ObjectsClassReferencesDeleteNotFound) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class references delete not found response has a 4xx status code -func (o *ObjectsClassReferencesDeleteNotFound) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects class references delete not found response has a 5xx status code -func (o *ObjectsClassReferencesDeleteNotFound) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class references delete not found response a status code equal to that given -func (o *ObjectsClassReferencesDeleteNotFound) IsCode(code int) bool { - return code == 404 -} - -// Code gets the status code for the objects class references delete not found response -func (o *ObjectsClassReferencesDeleteNotFound) Code() int { - return 404 -} - -func (o *ObjectsClassReferencesDeleteNotFound) Error() string { - return fmt.Sprintf("[DELETE /objects/{className}/{id}/references/{propertyName}][%d] objectsClassReferencesDeleteNotFound %+v", 404, o.Payload) -} - -func (o *ObjectsClassReferencesDeleteNotFound) String() string { - return fmt.Sprintf("[DELETE /objects/{className}/{id}/references/{propertyName}][%d] objectsClassReferencesDeleteNotFound %+v", 404, o.Payload) -} - -func (o *ObjectsClassReferencesDeleteNotFound) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsClassReferencesDeleteNotFound) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsClassReferencesDeleteUnprocessableEntity creates a ObjectsClassReferencesDeleteUnprocessableEntity with default headers values -func NewObjectsClassReferencesDeleteUnprocessableEntity() *ObjectsClassReferencesDeleteUnprocessableEntity { - return &ObjectsClassReferencesDeleteUnprocessableEntity{} -} - -/* -ObjectsClassReferencesDeleteUnprocessableEntity describes a response with status code 422, with default header values. - -Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the property exists or that it is a class? -*/ -type ObjectsClassReferencesDeleteUnprocessableEntity struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects class references delete unprocessable entity response has a 2xx status code -func (o *ObjectsClassReferencesDeleteUnprocessableEntity) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class references delete unprocessable entity response has a 3xx status code -func (o *ObjectsClassReferencesDeleteUnprocessableEntity) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class references delete unprocessable entity response has a 4xx status code -func (o *ObjectsClassReferencesDeleteUnprocessableEntity) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects class references delete unprocessable entity response has a 5xx status code -func (o *ObjectsClassReferencesDeleteUnprocessableEntity) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class references delete unprocessable entity response a status code equal to that given -func (o *ObjectsClassReferencesDeleteUnprocessableEntity) IsCode(code int) bool { - return code == 422 -} - -// Code gets the status code for the objects class references delete unprocessable entity response -func (o *ObjectsClassReferencesDeleteUnprocessableEntity) Code() int { - return 422 -} - -func (o *ObjectsClassReferencesDeleteUnprocessableEntity) Error() string { - return fmt.Sprintf("[DELETE /objects/{className}/{id}/references/{propertyName}][%d] objectsClassReferencesDeleteUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *ObjectsClassReferencesDeleteUnprocessableEntity) String() string { - return fmt.Sprintf("[DELETE /objects/{className}/{id}/references/{propertyName}][%d] objectsClassReferencesDeleteUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *ObjectsClassReferencesDeleteUnprocessableEntity) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsClassReferencesDeleteUnprocessableEntity) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsClassReferencesDeleteInternalServerError creates a ObjectsClassReferencesDeleteInternalServerError with default headers values -func NewObjectsClassReferencesDeleteInternalServerError() *ObjectsClassReferencesDeleteInternalServerError { - return &ObjectsClassReferencesDeleteInternalServerError{} -} - -/* -ObjectsClassReferencesDeleteInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type ObjectsClassReferencesDeleteInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects class references delete internal server error response has a 2xx status code -func (o *ObjectsClassReferencesDeleteInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class references delete internal server error response has a 3xx status code -func (o *ObjectsClassReferencesDeleteInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class references delete internal server error response has a 4xx status code -func (o *ObjectsClassReferencesDeleteInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this objects class references delete internal server error response has a 5xx status code -func (o *ObjectsClassReferencesDeleteInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this objects class references delete internal server error response a status code equal to that given -func (o *ObjectsClassReferencesDeleteInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the objects class references delete internal server error response -func (o *ObjectsClassReferencesDeleteInternalServerError) Code() int { - return 500 -} - -func (o *ObjectsClassReferencesDeleteInternalServerError) Error() string { - return fmt.Sprintf("[DELETE /objects/{className}/{id}/references/{propertyName}][%d] objectsClassReferencesDeleteInternalServerError %+v", 500, o.Payload) -} - -func (o *ObjectsClassReferencesDeleteInternalServerError) String() string { - return fmt.Sprintf("[DELETE /objects/{className}/{id}/references/{propertyName}][%d] objectsClassReferencesDeleteInternalServerError %+v", 500, o.Payload) -} - -func (o *ObjectsClassReferencesDeleteInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsClassReferencesDeleteInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/objects/objects_class_references_put_parameters.go b/client/objects/objects_class_references_put_parameters.go deleted file mode 100644 index a93ef2d2149cd183129f11a9a6839a2957aea330..0000000000000000000000000000000000000000 --- a/client/objects/objects_class_references_put_parameters.go +++ /dev/null @@ -1,297 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewObjectsClassReferencesPutParams creates a new ObjectsClassReferencesPutParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewObjectsClassReferencesPutParams() *ObjectsClassReferencesPutParams { - return &ObjectsClassReferencesPutParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewObjectsClassReferencesPutParamsWithTimeout creates a new ObjectsClassReferencesPutParams object -// with the ability to set a timeout on a request. -func NewObjectsClassReferencesPutParamsWithTimeout(timeout time.Duration) *ObjectsClassReferencesPutParams { - return &ObjectsClassReferencesPutParams{ - timeout: timeout, - } -} - -// NewObjectsClassReferencesPutParamsWithContext creates a new ObjectsClassReferencesPutParams object -// with the ability to set a context for a request. -func NewObjectsClassReferencesPutParamsWithContext(ctx context.Context) *ObjectsClassReferencesPutParams { - return &ObjectsClassReferencesPutParams{ - Context: ctx, - } -} - -// NewObjectsClassReferencesPutParamsWithHTTPClient creates a new ObjectsClassReferencesPutParams object -// with the ability to set a custom HTTPClient for a request. -func NewObjectsClassReferencesPutParamsWithHTTPClient(client *http.Client) *ObjectsClassReferencesPutParams { - return &ObjectsClassReferencesPutParams{ - HTTPClient: client, - } -} - -/* -ObjectsClassReferencesPutParams contains all the parameters to send to the API endpoint - - for the objects class references put operation. - - Typically these are written to a http.Request. -*/ -type ObjectsClassReferencesPutParams struct { - - // Body. - Body models.MultipleRef - - /* ClassName. - - The class name as defined in the schema - */ - ClassName string - - /* ConsistencyLevel. - - Determines how many replicas must acknowledge a request before it is considered successful - */ - ConsistencyLevel *string - - /* ID. - - Unique ID of the Object. - - Format: uuid - */ - ID strfmt.UUID - - /* PropertyName. - - Unique name of the property related to the Object. - */ - PropertyName string - - /* Tenant. - - Specifies the tenant in a request targeting a multi-tenant class - */ - Tenant *string - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the objects class references put params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *ObjectsClassReferencesPutParams) WithDefaults() *ObjectsClassReferencesPutParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the objects class references put params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *ObjectsClassReferencesPutParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the objects class references put params -func (o *ObjectsClassReferencesPutParams) WithTimeout(timeout time.Duration) *ObjectsClassReferencesPutParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the objects class references put params -func (o *ObjectsClassReferencesPutParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the objects class references put params -func (o *ObjectsClassReferencesPutParams) WithContext(ctx context.Context) *ObjectsClassReferencesPutParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the objects class references put params -func (o *ObjectsClassReferencesPutParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the objects class references put params -func (o *ObjectsClassReferencesPutParams) WithHTTPClient(client *http.Client) *ObjectsClassReferencesPutParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the objects class references put params -func (o *ObjectsClassReferencesPutParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithBody adds the body to the objects class references put params -func (o *ObjectsClassReferencesPutParams) WithBody(body models.MultipleRef) *ObjectsClassReferencesPutParams { - o.SetBody(body) - return o -} - -// SetBody adds the body to the objects class references put params -func (o *ObjectsClassReferencesPutParams) SetBody(body models.MultipleRef) { - o.Body = body -} - -// WithClassName adds the className to the objects class references put params -func (o *ObjectsClassReferencesPutParams) WithClassName(className string) *ObjectsClassReferencesPutParams { - o.SetClassName(className) - return o -} - -// SetClassName adds the className to the objects class references put params -func (o *ObjectsClassReferencesPutParams) SetClassName(className string) { - o.ClassName = className -} - -// WithConsistencyLevel adds the consistencyLevel to the objects class references put params -func (o *ObjectsClassReferencesPutParams) WithConsistencyLevel(consistencyLevel *string) *ObjectsClassReferencesPutParams { - o.SetConsistencyLevel(consistencyLevel) - return o -} - -// SetConsistencyLevel adds the consistencyLevel to the objects class references put params -func (o *ObjectsClassReferencesPutParams) SetConsistencyLevel(consistencyLevel *string) { - o.ConsistencyLevel = consistencyLevel -} - -// WithID adds the id to the objects class references put params -func (o *ObjectsClassReferencesPutParams) WithID(id strfmt.UUID) *ObjectsClassReferencesPutParams { - o.SetID(id) - return o -} - -// SetID adds the id to the objects class references put params -func (o *ObjectsClassReferencesPutParams) SetID(id strfmt.UUID) { - o.ID = id -} - -// WithPropertyName adds the propertyName to the objects class references put params -func (o *ObjectsClassReferencesPutParams) WithPropertyName(propertyName string) *ObjectsClassReferencesPutParams { - o.SetPropertyName(propertyName) - return o -} - -// SetPropertyName adds the propertyName to the objects class references put params -func (o *ObjectsClassReferencesPutParams) SetPropertyName(propertyName string) { - o.PropertyName = propertyName -} - -// WithTenant adds the tenant to the objects class references put params -func (o *ObjectsClassReferencesPutParams) WithTenant(tenant *string) *ObjectsClassReferencesPutParams { - o.SetTenant(tenant) - return o -} - -// SetTenant adds the tenant to the objects class references put params -func (o *ObjectsClassReferencesPutParams) SetTenant(tenant *string) { - o.Tenant = tenant -} - -// WriteToRequest writes these params to a swagger request -func (o *ObjectsClassReferencesPutParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - if o.Body != nil { - if err := r.SetBodyParam(o.Body); err != nil { - return err - } - } - - // path param className - if err := r.SetPathParam("className", o.ClassName); err != nil { - return err - } - - if o.ConsistencyLevel != nil { - - // query param consistency_level - var qrConsistencyLevel string - - if o.ConsistencyLevel != nil { - qrConsistencyLevel = *o.ConsistencyLevel - } - qConsistencyLevel := qrConsistencyLevel - if qConsistencyLevel != "" { - - if err := r.SetQueryParam("consistency_level", qConsistencyLevel); err != nil { - return err - } - } - } - - // path param id - if err := r.SetPathParam("id", o.ID.String()); err != nil { - return err - } - - // path param propertyName - if err := r.SetPathParam("propertyName", o.PropertyName); err != nil { - return err - } - - if o.Tenant != nil { - - // query param tenant - var qrTenant string - - if o.Tenant != nil { - qrTenant = *o.Tenant - } - qTenant := qrTenant - if qTenant != "" { - - if err := r.SetQueryParam("tenant", qTenant); err != nil { - return err - } - } - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/objects/objects_class_references_put_responses.go b/client/objects/objects_class_references_put_responses.go deleted file mode 100644 index e453dc157053bb713f8f1373c4d0bef68393fc03..0000000000000000000000000000000000000000 --- a/client/objects/objects_class_references_put_responses.go +++ /dev/null @@ -1,522 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsClassReferencesPutReader is a Reader for the ObjectsClassReferencesPut structure. -type ObjectsClassReferencesPutReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *ObjectsClassReferencesPutReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 200: - result := NewObjectsClassReferencesPutOK() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 400: - result := NewObjectsClassReferencesPutBadRequest() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 401: - result := NewObjectsClassReferencesPutUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewObjectsClassReferencesPutForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 404: - result := NewObjectsClassReferencesPutNotFound() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 422: - result := NewObjectsClassReferencesPutUnprocessableEntity() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewObjectsClassReferencesPutInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewObjectsClassReferencesPutOK creates a ObjectsClassReferencesPutOK with default headers values -func NewObjectsClassReferencesPutOK() *ObjectsClassReferencesPutOK { - return &ObjectsClassReferencesPutOK{} -} - -/* -ObjectsClassReferencesPutOK describes a response with status code 200, with default header values. - -Successfully replaced all the references. -*/ -type ObjectsClassReferencesPutOK struct { -} - -// IsSuccess returns true when this objects class references put o k response has a 2xx status code -func (o *ObjectsClassReferencesPutOK) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this objects class references put o k response has a 3xx status code -func (o *ObjectsClassReferencesPutOK) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class references put o k response has a 4xx status code -func (o *ObjectsClassReferencesPutOK) IsClientError() bool { - return false -} - -// IsServerError returns true when this objects class references put o k response has a 5xx status code -func (o *ObjectsClassReferencesPutOK) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class references put o k response a status code equal to that given -func (o *ObjectsClassReferencesPutOK) IsCode(code int) bool { - return code == 200 -} - -// Code gets the status code for the objects class references put o k response -func (o *ObjectsClassReferencesPutOK) Code() int { - return 200 -} - -func (o *ObjectsClassReferencesPutOK) Error() string { - return fmt.Sprintf("[PUT /objects/{className}/{id}/references/{propertyName}][%d] objectsClassReferencesPutOK ", 200) -} - -func (o *ObjectsClassReferencesPutOK) String() string { - return fmt.Sprintf("[PUT /objects/{className}/{id}/references/{propertyName}][%d] objectsClassReferencesPutOK ", 200) -} - -func (o *ObjectsClassReferencesPutOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsClassReferencesPutBadRequest creates a ObjectsClassReferencesPutBadRequest with default headers values -func NewObjectsClassReferencesPutBadRequest() *ObjectsClassReferencesPutBadRequest { - return &ObjectsClassReferencesPutBadRequest{} -} - -/* -ObjectsClassReferencesPutBadRequest describes a response with status code 400, with default header values. - -Malformed request. -*/ -type ObjectsClassReferencesPutBadRequest struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects class references put bad request response has a 2xx status code -func (o *ObjectsClassReferencesPutBadRequest) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class references put bad request response has a 3xx status code -func (o *ObjectsClassReferencesPutBadRequest) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class references put bad request response has a 4xx status code -func (o *ObjectsClassReferencesPutBadRequest) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects class references put bad request response has a 5xx status code -func (o *ObjectsClassReferencesPutBadRequest) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class references put bad request response a status code equal to that given -func (o *ObjectsClassReferencesPutBadRequest) IsCode(code int) bool { - return code == 400 -} - -// Code gets the status code for the objects class references put bad request response -func (o *ObjectsClassReferencesPutBadRequest) Code() int { - return 400 -} - -func (o *ObjectsClassReferencesPutBadRequest) Error() string { - return fmt.Sprintf("[PUT /objects/{className}/{id}/references/{propertyName}][%d] objectsClassReferencesPutBadRequest %+v", 400, o.Payload) -} - -func (o *ObjectsClassReferencesPutBadRequest) String() string { - return fmt.Sprintf("[PUT /objects/{className}/{id}/references/{propertyName}][%d] objectsClassReferencesPutBadRequest %+v", 400, o.Payload) -} - -func (o *ObjectsClassReferencesPutBadRequest) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsClassReferencesPutBadRequest) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsClassReferencesPutUnauthorized creates a ObjectsClassReferencesPutUnauthorized with default headers values -func NewObjectsClassReferencesPutUnauthorized() *ObjectsClassReferencesPutUnauthorized { - return &ObjectsClassReferencesPutUnauthorized{} -} - -/* -ObjectsClassReferencesPutUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type ObjectsClassReferencesPutUnauthorized struct { -} - -// IsSuccess returns true when this objects class references put unauthorized response has a 2xx status code -func (o *ObjectsClassReferencesPutUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class references put unauthorized response has a 3xx status code -func (o *ObjectsClassReferencesPutUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class references put unauthorized response has a 4xx status code -func (o *ObjectsClassReferencesPutUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects class references put unauthorized response has a 5xx status code -func (o *ObjectsClassReferencesPutUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class references put unauthorized response a status code equal to that given -func (o *ObjectsClassReferencesPutUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the objects class references put unauthorized response -func (o *ObjectsClassReferencesPutUnauthorized) Code() int { - return 401 -} - -func (o *ObjectsClassReferencesPutUnauthorized) Error() string { - return fmt.Sprintf("[PUT /objects/{className}/{id}/references/{propertyName}][%d] objectsClassReferencesPutUnauthorized ", 401) -} - -func (o *ObjectsClassReferencesPutUnauthorized) String() string { - return fmt.Sprintf("[PUT /objects/{className}/{id}/references/{propertyName}][%d] objectsClassReferencesPutUnauthorized ", 401) -} - -func (o *ObjectsClassReferencesPutUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsClassReferencesPutForbidden creates a ObjectsClassReferencesPutForbidden with default headers values -func NewObjectsClassReferencesPutForbidden() *ObjectsClassReferencesPutForbidden { - return &ObjectsClassReferencesPutForbidden{} -} - -/* -ObjectsClassReferencesPutForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type ObjectsClassReferencesPutForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects class references put forbidden response has a 2xx status code -func (o *ObjectsClassReferencesPutForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class references put forbidden response has a 3xx status code -func (o *ObjectsClassReferencesPutForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class references put forbidden response has a 4xx status code -func (o *ObjectsClassReferencesPutForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects class references put forbidden response has a 5xx status code -func (o *ObjectsClassReferencesPutForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class references put forbidden response a status code equal to that given -func (o *ObjectsClassReferencesPutForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the objects class references put forbidden response -func (o *ObjectsClassReferencesPutForbidden) Code() int { - return 403 -} - -func (o *ObjectsClassReferencesPutForbidden) Error() string { - return fmt.Sprintf("[PUT /objects/{className}/{id}/references/{propertyName}][%d] objectsClassReferencesPutForbidden %+v", 403, o.Payload) -} - -func (o *ObjectsClassReferencesPutForbidden) String() string { - return fmt.Sprintf("[PUT /objects/{className}/{id}/references/{propertyName}][%d] objectsClassReferencesPutForbidden %+v", 403, o.Payload) -} - -func (o *ObjectsClassReferencesPutForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsClassReferencesPutForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsClassReferencesPutNotFound creates a ObjectsClassReferencesPutNotFound with default headers values -func NewObjectsClassReferencesPutNotFound() *ObjectsClassReferencesPutNotFound { - return &ObjectsClassReferencesPutNotFound{} -} - -/* -ObjectsClassReferencesPutNotFound describes a response with status code 404, with default header values. - -Source object doesn't exist. -*/ -type ObjectsClassReferencesPutNotFound struct { -} - -// IsSuccess returns true when this objects class references put not found response has a 2xx status code -func (o *ObjectsClassReferencesPutNotFound) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class references put not found response has a 3xx status code -func (o *ObjectsClassReferencesPutNotFound) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class references put not found response has a 4xx status code -func (o *ObjectsClassReferencesPutNotFound) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects class references put not found response has a 5xx status code -func (o *ObjectsClassReferencesPutNotFound) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class references put not found response a status code equal to that given -func (o *ObjectsClassReferencesPutNotFound) IsCode(code int) bool { - return code == 404 -} - -// Code gets the status code for the objects class references put not found response -func (o *ObjectsClassReferencesPutNotFound) Code() int { - return 404 -} - -func (o *ObjectsClassReferencesPutNotFound) Error() string { - return fmt.Sprintf("[PUT /objects/{className}/{id}/references/{propertyName}][%d] objectsClassReferencesPutNotFound ", 404) -} - -func (o *ObjectsClassReferencesPutNotFound) String() string { - return fmt.Sprintf("[PUT /objects/{className}/{id}/references/{propertyName}][%d] objectsClassReferencesPutNotFound ", 404) -} - -func (o *ObjectsClassReferencesPutNotFound) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsClassReferencesPutUnprocessableEntity creates a ObjectsClassReferencesPutUnprocessableEntity with default headers values -func NewObjectsClassReferencesPutUnprocessableEntity() *ObjectsClassReferencesPutUnprocessableEntity { - return &ObjectsClassReferencesPutUnprocessableEntity{} -} - -/* -ObjectsClassReferencesPutUnprocessableEntity describes a response with status code 422, with default header values. - -Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the property exists or that it is a class? -*/ -type ObjectsClassReferencesPutUnprocessableEntity struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects class references put unprocessable entity response has a 2xx status code -func (o *ObjectsClassReferencesPutUnprocessableEntity) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class references put unprocessable entity response has a 3xx status code -func (o *ObjectsClassReferencesPutUnprocessableEntity) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class references put unprocessable entity response has a 4xx status code -func (o *ObjectsClassReferencesPutUnprocessableEntity) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects class references put unprocessable entity response has a 5xx status code -func (o *ObjectsClassReferencesPutUnprocessableEntity) IsServerError() bool { - return false -} - -// IsCode returns true when this objects class references put unprocessable entity response a status code equal to that given -func (o *ObjectsClassReferencesPutUnprocessableEntity) IsCode(code int) bool { - return code == 422 -} - -// Code gets the status code for the objects class references put unprocessable entity response -func (o *ObjectsClassReferencesPutUnprocessableEntity) Code() int { - return 422 -} - -func (o *ObjectsClassReferencesPutUnprocessableEntity) Error() string { - return fmt.Sprintf("[PUT /objects/{className}/{id}/references/{propertyName}][%d] objectsClassReferencesPutUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *ObjectsClassReferencesPutUnprocessableEntity) String() string { - return fmt.Sprintf("[PUT /objects/{className}/{id}/references/{propertyName}][%d] objectsClassReferencesPutUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *ObjectsClassReferencesPutUnprocessableEntity) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsClassReferencesPutUnprocessableEntity) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsClassReferencesPutInternalServerError creates a ObjectsClassReferencesPutInternalServerError with default headers values -func NewObjectsClassReferencesPutInternalServerError() *ObjectsClassReferencesPutInternalServerError { - return &ObjectsClassReferencesPutInternalServerError{} -} - -/* -ObjectsClassReferencesPutInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type ObjectsClassReferencesPutInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects class references put internal server error response has a 2xx status code -func (o *ObjectsClassReferencesPutInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects class references put internal server error response has a 3xx status code -func (o *ObjectsClassReferencesPutInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects class references put internal server error response has a 4xx status code -func (o *ObjectsClassReferencesPutInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this objects class references put internal server error response has a 5xx status code -func (o *ObjectsClassReferencesPutInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this objects class references put internal server error response a status code equal to that given -func (o *ObjectsClassReferencesPutInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the objects class references put internal server error response -func (o *ObjectsClassReferencesPutInternalServerError) Code() int { - return 500 -} - -func (o *ObjectsClassReferencesPutInternalServerError) Error() string { - return fmt.Sprintf("[PUT /objects/{className}/{id}/references/{propertyName}][%d] objectsClassReferencesPutInternalServerError %+v", 500, o.Payload) -} - -func (o *ObjectsClassReferencesPutInternalServerError) String() string { - return fmt.Sprintf("[PUT /objects/{className}/{id}/references/{propertyName}][%d] objectsClassReferencesPutInternalServerError %+v", 500, o.Payload) -} - -func (o *ObjectsClassReferencesPutInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsClassReferencesPutInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/objects/objects_client.go b/client/objects/objects_client.go deleted file mode 100644 index 95b96c8eb4d2a73364e842c3aa3bb31ffd39f9a1..0000000000000000000000000000000000000000 --- a/client/objects/objects_client.go +++ /dev/null @@ -1,867 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" -) - -// New creates a new objects API client. -func New(transport runtime.ClientTransport, formats strfmt.Registry) ClientService { - return &Client{transport: transport, formats: formats} -} - -/* -Client for objects API -*/ -type Client struct { - transport runtime.ClientTransport - formats strfmt.Registry -} - -// ClientOption is the option for Client methods -type ClientOption func(*runtime.ClientOperation) - -// ClientService is the interface for Client methods -type ClientService interface { - ObjectsClassDelete(params *ObjectsClassDeleteParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*ObjectsClassDeleteNoContent, error) - - ObjectsClassGet(params *ObjectsClassGetParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*ObjectsClassGetOK, error) - - ObjectsClassHead(params *ObjectsClassHeadParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*ObjectsClassHeadNoContent, error) - - ObjectsClassPatch(params *ObjectsClassPatchParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*ObjectsClassPatchNoContent, error) - - ObjectsClassPut(params *ObjectsClassPutParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*ObjectsClassPutOK, error) - - ObjectsClassReferencesCreate(params *ObjectsClassReferencesCreateParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*ObjectsClassReferencesCreateOK, error) - - ObjectsClassReferencesDelete(params *ObjectsClassReferencesDeleteParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*ObjectsClassReferencesDeleteNoContent, error) - - ObjectsClassReferencesPut(params *ObjectsClassReferencesPutParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*ObjectsClassReferencesPutOK, error) - - ObjectsCreate(params *ObjectsCreateParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*ObjectsCreateOK, error) - - ObjectsDelete(params *ObjectsDeleteParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*ObjectsDeleteNoContent, error) - - ObjectsGet(params *ObjectsGetParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*ObjectsGetOK, error) - - ObjectsHead(params *ObjectsHeadParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*ObjectsHeadNoContent, error) - - ObjectsList(params *ObjectsListParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*ObjectsListOK, error) - - ObjectsPatch(params *ObjectsPatchParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*ObjectsPatchNoContent, error) - - ObjectsReferencesCreate(params *ObjectsReferencesCreateParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*ObjectsReferencesCreateOK, error) - - ObjectsReferencesDelete(params *ObjectsReferencesDeleteParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*ObjectsReferencesDeleteNoContent, error) - - ObjectsReferencesUpdate(params *ObjectsReferencesUpdateParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*ObjectsReferencesUpdateOK, error) - - ObjectsUpdate(params *ObjectsUpdateParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*ObjectsUpdateOK, error) - - ObjectsValidate(params *ObjectsValidateParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*ObjectsValidateOK, error) - - SetTransport(transport runtime.ClientTransport) -} - -/* -ObjectsClassDelete deletes object based on its class and UUID - -Delete a single data object. -*/ -func (a *Client) ObjectsClassDelete(params *ObjectsClassDeleteParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*ObjectsClassDeleteNoContent, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewObjectsClassDeleteParams() - } - op := &runtime.ClientOperation{ - ID: "objects.class.delete", - Method: "DELETE", - PathPattern: "/objects/{className}/{id}", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &ObjectsClassDeleteReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*ObjectsClassDeleteNoContent) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for objects.class.delete: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -/* -ObjectsClassGet gets a specific object based on its class and UUID also available as websocket bus - -Get a single data object -*/ -func (a *Client) ObjectsClassGet(params *ObjectsClassGetParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*ObjectsClassGetOK, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewObjectsClassGetParams() - } - op := &runtime.ClientOperation{ - ID: "objects.class.get", - Method: "GET", - PathPattern: "/objects/{className}/{id}", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &ObjectsClassGetReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*ObjectsClassGetOK) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for objects.class.get: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -/* -ObjectsClassHead checks object s existence based on its class and uuid - -Checks if a data object exists without retrieving it. -*/ -func (a *Client) ObjectsClassHead(params *ObjectsClassHeadParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*ObjectsClassHeadNoContent, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewObjectsClassHeadParams() - } - op := &runtime.ClientOperation{ - ID: "objects.class.head", - Method: "HEAD", - PathPattern: "/objects/{className}/{id}", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &ObjectsClassHeadReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*ObjectsClassHeadNoContent) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for objects.class.head: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -/* -ObjectsClassPatch updates an object based on its UUID using patch semantics - -Update an individual data object based on its class and uuid. This method supports json-merge style patch semantics (RFC 7396). Provided meta-data and schema values are validated. LastUpdateTime is set to the time this function is called. -*/ -func (a *Client) ObjectsClassPatch(params *ObjectsClassPatchParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*ObjectsClassPatchNoContent, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewObjectsClassPatchParams() - } - op := &runtime.ClientOperation{ - ID: "objects.class.patch", - Method: "PATCH", - PathPattern: "/objects/{className}/{id}", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &ObjectsClassPatchReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*ObjectsClassPatchNoContent) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for objects.class.patch: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -/* -ObjectsClassPut updates a class object based on its uuid - -Update an individual data object based on its class and uuid. -*/ -func (a *Client) ObjectsClassPut(params *ObjectsClassPutParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*ObjectsClassPutOK, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewObjectsClassPutParams() - } - op := &runtime.ClientOperation{ - ID: "objects.class.put", - Method: "PUT", - PathPattern: "/objects/{className}/{id}", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &ObjectsClassPutReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*ObjectsClassPutOK) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for objects.class.put: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -/* -ObjectsClassReferencesCreate adds a single reference to a class property - -Add a single reference to a class-property. -*/ -func (a *Client) ObjectsClassReferencesCreate(params *ObjectsClassReferencesCreateParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*ObjectsClassReferencesCreateOK, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewObjectsClassReferencesCreateParams() - } - op := &runtime.ClientOperation{ - ID: "objects.class.references.create", - Method: "POST", - PathPattern: "/objects/{className}/{id}/references/{propertyName}", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &ObjectsClassReferencesCreateReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*ObjectsClassReferencesCreateOK) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for objects.class.references.create: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -/* -ObjectsClassReferencesDelete deletes the single reference that is given in the body from the list of references that this property has - -Delete the single reference that is given in the body from the list of references that this property of a data object has -*/ -func (a *Client) ObjectsClassReferencesDelete(params *ObjectsClassReferencesDeleteParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*ObjectsClassReferencesDeleteNoContent, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewObjectsClassReferencesDeleteParams() - } - op := &runtime.ClientOperation{ - ID: "objects.class.references.delete", - Method: "DELETE", - PathPattern: "/objects/{className}/{id}/references/{propertyName}", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &ObjectsClassReferencesDeleteReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*ObjectsClassReferencesDeleteNoContent) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for objects.class.references.delete: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -/* -ObjectsClassReferencesPut replaces all references to a class property - -Update all references of a property of a data object. -*/ -func (a *Client) ObjectsClassReferencesPut(params *ObjectsClassReferencesPutParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*ObjectsClassReferencesPutOK, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewObjectsClassReferencesPutParams() - } - op := &runtime.ClientOperation{ - ID: "objects.class.references.put", - Method: "PUT", - PathPattern: "/objects/{className}/{id}/references/{propertyName}", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &ObjectsClassReferencesPutReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*ObjectsClassReferencesPutOK) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for objects.class.references.put: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -/* -ObjectsCreate creates objects between two objects object and subject - -Registers a new Object. Provided meta-data and schema values are validated. -*/ -func (a *Client) ObjectsCreate(params *ObjectsCreateParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*ObjectsCreateOK, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewObjectsCreateParams() - } - op := &runtime.ClientOperation{ - ID: "objects.create", - Method: "POST", - PathPattern: "/objects", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &ObjectsCreateReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*ObjectsCreateOK) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for objects.create: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -/* -ObjectsDelete deletes an object based on its UUID - -Deletes an Object from the system. -*/ -func (a *Client) ObjectsDelete(params *ObjectsDeleteParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*ObjectsDeleteNoContent, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewObjectsDeleteParams() - } - op := &runtime.ClientOperation{ - ID: "objects.delete", - Method: "DELETE", - PathPattern: "/objects/{id}", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &ObjectsDeleteReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*ObjectsDeleteNoContent) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for objects.delete: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -/* -ObjectsGet gets a specific object based on its UUID and a object UUID also available as websocket bus - -Lists Objects. -*/ -func (a *Client) ObjectsGet(params *ObjectsGetParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*ObjectsGetOK, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewObjectsGetParams() - } - op := &runtime.ClientOperation{ - ID: "objects.get", - Method: "GET", - PathPattern: "/objects/{id}", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &ObjectsGetReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*ObjectsGetOK) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for objects.get: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -/* -ObjectsHead checks object s existence based on its UUID - -Checks if an Object exists in the system. -*/ -func (a *Client) ObjectsHead(params *ObjectsHeadParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*ObjectsHeadNoContent, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewObjectsHeadParams() - } - op := &runtime.ClientOperation{ - ID: "objects.head", - Method: "HEAD", - PathPattern: "/objects/{id}", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &ObjectsHeadReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*ObjectsHeadNoContent) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for objects.head: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -/* -ObjectsList gets a list of objects - -Lists all Objects in reverse order of creation, owned by the user that belongs to the used token. -*/ -func (a *Client) ObjectsList(params *ObjectsListParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*ObjectsListOK, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewObjectsListParams() - } - op := &runtime.ClientOperation{ - ID: "objects.list", - Method: "GET", - PathPattern: "/objects", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &ObjectsListReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*ObjectsListOK) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for objects.list: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -/* -ObjectsPatch updates an object based on its UUID using patch semantics - -Updates an Object. This method supports json-merge style patch semantics (RFC 7396). Provided meta-data and schema values are validated. LastUpdateTime is set to the time this function is called. -*/ -func (a *Client) ObjectsPatch(params *ObjectsPatchParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*ObjectsPatchNoContent, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewObjectsPatchParams() - } - op := &runtime.ClientOperation{ - ID: "objects.patch", - Method: "PATCH", - PathPattern: "/objects/{id}", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &ObjectsPatchReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*ObjectsPatchNoContent) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for objects.patch: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -/* -ObjectsReferencesCreate adds a single reference to a class property - -Add a single reference to a class-property. -*/ -func (a *Client) ObjectsReferencesCreate(params *ObjectsReferencesCreateParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*ObjectsReferencesCreateOK, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewObjectsReferencesCreateParams() - } - op := &runtime.ClientOperation{ - ID: "objects.references.create", - Method: "POST", - PathPattern: "/objects/{id}/references/{propertyName}", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &ObjectsReferencesCreateReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*ObjectsReferencesCreateOK) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for objects.references.create: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -/* -ObjectsReferencesDelete deletes the single reference that is given in the body from the list of references that this property has - -Delete the single reference that is given in the body from the list of references that this property has. -*/ -func (a *Client) ObjectsReferencesDelete(params *ObjectsReferencesDeleteParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*ObjectsReferencesDeleteNoContent, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewObjectsReferencesDeleteParams() - } - op := &runtime.ClientOperation{ - ID: "objects.references.delete", - Method: "DELETE", - PathPattern: "/objects/{id}/references/{propertyName}", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &ObjectsReferencesDeleteReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*ObjectsReferencesDeleteNoContent) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for objects.references.delete: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -/* -ObjectsReferencesUpdate replaces all references to a class property - -Replace all references to a class-property. -*/ -func (a *Client) ObjectsReferencesUpdate(params *ObjectsReferencesUpdateParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*ObjectsReferencesUpdateOK, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewObjectsReferencesUpdateParams() - } - op := &runtime.ClientOperation{ - ID: "objects.references.update", - Method: "PUT", - PathPattern: "/objects/{id}/references/{propertyName}", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &ObjectsReferencesUpdateReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*ObjectsReferencesUpdateOK) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for objects.references.update: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -/* -ObjectsUpdate updates an object based on its UUID - -Updates an Object's data. Given meta-data and schema values are validated. LastUpdateTime is set to the time this function is called. -*/ -func (a *Client) ObjectsUpdate(params *ObjectsUpdateParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*ObjectsUpdateOK, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewObjectsUpdateParams() - } - op := &runtime.ClientOperation{ - ID: "objects.update", - Method: "PUT", - PathPattern: "/objects/{id}", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &ObjectsUpdateReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*ObjectsUpdateOK) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for objects.update: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -/* -ObjectsValidate validates an object based on a schema - -Validate an Object's schema and meta-data. It has to be based on a schema, which is related to the given Object to be accepted by this validation. -*/ -func (a *Client) ObjectsValidate(params *ObjectsValidateParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*ObjectsValidateOK, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewObjectsValidateParams() - } - op := &runtime.ClientOperation{ - ID: "objects.validate", - Method: "POST", - PathPattern: "/objects/validate", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &ObjectsValidateReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*ObjectsValidateOK) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for objects.validate: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -// SetTransport changes the transport on the client -func (a *Client) SetTransport(transport runtime.ClientTransport) { - a.transport = transport -} diff --git a/client/objects/objects_create_parameters.go b/client/objects/objects_create_parameters.go deleted file mode 100644 index a171fba648d9dbd9bf5b0f950059c5f7bca122ed..0000000000000000000000000000000000000000 --- a/client/objects/objects_create_parameters.go +++ /dev/null @@ -1,195 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewObjectsCreateParams creates a new ObjectsCreateParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewObjectsCreateParams() *ObjectsCreateParams { - return &ObjectsCreateParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewObjectsCreateParamsWithTimeout creates a new ObjectsCreateParams object -// with the ability to set a timeout on a request. -func NewObjectsCreateParamsWithTimeout(timeout time.Duration) *ObjectsCreateParams { - return &ObjectsCreateParams{ - timeout: timeout, - } -} - -// NewObjectsCreateParamsWithContext creates a new ObjectsCreateParams object -// with the ability to set a context for a request. -func NewObjectsCreateParamsWithContext(ctx context.Context) *ObjectsCreateParams { - return &ObjectsCreateParams{ - Context: ctx, - } -} - -// NewObjectsCreateParamsWithHTTPClient creates a new ObjectsCreateParams object -// with the ability to set a custom HTTPClient for a request. -func NewObjectsCreateParamsWithHTTPClient(client *http.Client) *ObjectsCreateParams { - return &ObjectsCreateParams{ - HTTPClient: client, - } -} - -/* -ObjectsCreateParams contains all the parameters to send to the API endpoint - - for the objects create operation. - - Typically these are written to a http.Request. -*/ -type ObjectsCreateParams struct { - - // Body. - Body *models.Object - - /* ConsistencyLevel. - - Determines how many replicas must acknowledge a request before it is considered successful - */ - ConsistencyLevel *string - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the objects create params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *ObjectsCreateParams) WithDefaults() *ObjectsCreateParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the objects create params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *ObjectsCreateParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the objects create params -func (o *ObjectsCreateParams) WithTimeout(timeout time.Duration) *ObjectsCreateParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the objects create params -func (o *ObjectsCreateParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the objects create params -func (o *ObjectsCreateParams) WithContext(ctx context.Context) *ObjectsCreateParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the objects create params -func (o *ObjectsCreateParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the objects create params -func (o *ObjectsCreateParams) WithHTTPClient(client *http.Client) *ObjectsCreateParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the objects create params -func (o *ObjectsCreateParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithBody adds the body to the objects create params -func (o *ObjectsCreateParams) WithBody(body *models.Object) *ObjectsCreateParams { - o.SetBody(body) - return o -} - -// SetBody adds the body to the objects create params -func (o *ObjectsCreateParams) SetBody(body *models.Object) { - o.Body = body -} - -// WithConsistencyLevel adds the consistencyLevel to the objects create params -func (o *ObjectsCreateParams) WithConsistencyLevel(consistencyLevel *string) *ObjectsCreateParams { - o.SetConsistencyLevel(consistencyLevel) - return o -} - -// SetConsistencyLevel adds the consistencyLevel to the objects create params -func (o *ObjectsCreateParams) SetConsistencyLevel(consistencyLevel *string) { - o.ConsistencyLevel = consistencyLevel -} - -// WriteToRequest writes these params to a swagger request -func (o *ObjectsCreateParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - if o.Body != nil { - if err := r.SetBodyParam(o.Body); err != nil { - return err - } - } - - if o.ConsistencyLevel != nil { - - // query param consistency_level - var qrConsistencyLevel string - - if o.ConsistencyLevel != nil { - qrConsistencyLevel = *o.ConsistencyLevel - } - qConsistencyLevel := qrConsistencyLevel - if qConsistencyLevel != "" { - - if err := r.SetQueryParam("consistency_level", qConsistencyLevel); err != nil { - return err - } - } - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/objects/objects_create_responses.go b/client/objects/objects_create_responses.go deleted file mode 100644 index a87a6e50c58b9245359c8a346ab4ce31c9cfefda..0000000000000000000000000000000000000000 --- a/client/objects/objects_create_responses.go +++ /dev/null @@ -1,472 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsCreateReader is a Reader for the ObjectsCreate structure. -type ObjectsCreateReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *ObjectsCreateReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 200: - result := NewObjectsCreateOK() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 400: - result := NewObjectsCreateBadRequest() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 401: - result := NewObjectsCreateUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewObjectsCreateForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 422: - result := NewObjectsCreateUnprocessableEntity() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewObjectsCreateInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewObjectsCreateOK creates a ObjectsCreateOK with default headers values -func NewObjectsCreateOK() *ObjectsCreateOK { - return &ObjectsCreateOK{} -} - -/* -ObjectsCreateOK describes a response with status code 200, with default header values. - -Object created. -*/ -type ObjectsCreateOK struct { - Payload *models.Object -} - -// IsSuccess returns true when this objects create o k response has a 2xx status code -func (o *ObjectsCreateOK) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this objects create o k response has a 3xx status code -func (o *ObjectsCreateOK) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects create o k response has a 4xx status code -func (o *ObjectsCreateOK) IsClientError() bool { - return false -} - -// IsServerError returns true when this objects create o k response has a 5xx status code -func (o *ObjectsCreateOK) IsServerError() bool { - return false -} - -// IsCode returns true when this objects create o k response a status code equal to that given -func (o *ObjectsCreateOK) IsCode(code int) bool { - return code == 200 -} - -// Code gets the status code for the objects create o k response -func (o *ObjectsCreateOK) Code() int { - return 200 -} - -func (o *ObjectsCreateOK) Error() string { - return fmt.Sprintf("[POST /objects][%d] objectsCreateOK %+v", 200, o.Payload) -} - -func (o *ObjectsCreateOK) String() string { - return fmt.Sprintf("[POST /objects][%d] objectsCreateOK %+v", 200, o.Payload) -} - -func (o *ObjectsCreateOK) GetPayload() *models.Object { - return o.Payload -} - -func (o *ObjectsCreateOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.Object) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsCreateBadRequest creates a ObjectsCreateBadRequest with default headers values -func NewObjectsCreateBadRequest() *ObjectsCreateBadRequest { - return &ObjectsCreateBadRequest{} -} - -/* -ObjectsCreateBadRequest describes a response with status code 400, with default header values. - -Malformed request. -*/ -type ObjectsCreateBadRequest struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects create bad request response has a 2xx status code -func (o *ObjectsCreateBadRequest) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects create bad request response has a 3xx status code -func (o *ObjectsCreateBadRequest) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects create bad request response has a 4xx status code -func (o *ObjectsCreateBadRequest) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects create bad request response has a 5xx status code -func (o *ObjectsCreateBadRequest) IsServerError() bool { - return false -} - -// IsCode returns true when this objects create bad request response a status code equal to that given -func (o *ObjectsCreateBadRequest) IsCode(code int) bool { - return code == 400 -} - -// Code gets the status code for the objects create bad request response -func (o *ObjectsCreateBadRequest) Code() int { - return 400 -} - -func (o *ObjectsCreateBadRequest) Error() string { - return fmt.Sprintf("[POST /objects][%d] objectsCreateBadRequest %+v", 400, o.Payload) -} - -func (o *ObjectsCreateBadRequest) String() string { - return fmt.Sprintf("[POST /objects][%d] objectsCreateBadRequest %+v", 400, o.Payload) -} - -func (o *ObjectsCreateBadRequest) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsCreateBadRequest) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsCreateUnauthorized creates a ObjectsCreateUnauthorized with default headers values -func NewObjectsCreateUnauthorized() *ObjectsCreateUnauthorized { - return &ObjectsCreateUnauthorized{} -} - -/* -ObjectsCreateUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type ObjectsCreateUnauthorized struct { -} - -// IsSuccess returns true when this objects create unauthorized response has a 2xx status code -func (o *ObjectsCreateUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects create unauthorized response has a 3xx status code -func (o *ObjectsCreateUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects create unauthorized response has a 4xx status code -func (o *ObjectsCreateUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects create unauthorized response has a 5xx status code -func (o *ObjectsCreateUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this objects create unauthorized response a status code equal to that given -func (o *ObjectsCreateUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the objects create unauthorized response -func (o *ObjectsCreateUnauthorized) Code() int { - return 401 -} - -func (o *ObjectsCreateUnauthorized) Error() string { - return fmt.Sprintf("[POST /objects][%d] objectsCreateUnauthorized ", 401) -} - -func (o *ObjectsCreateUnauthorized) String() string { - return fmt.Sprintf("[POST /objects][%d] objectsCreateUnauthorized ", 401) -} - -func (o *ObjectsCreateUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsCreateForbidden creates a ObjectsCreateForbidden with default headers values -func NewObjectsCreateForbidden() *ObjectsCreateForbidden { - return &ObjectsCreateForbidden{} -} - -/* -ObjectsCreateForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type ObjectsCreateForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects create forbidden response has a 2xx status code -func (o *ObjectsCreateForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects create forbidden response has a 3xx status code -func (o *ObjectsCreateForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects create forbidden response has a 4xx status code -func (o *ObjectsCreateForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects create forbidden response has a 5xx status code -func (o *ObjectsCreateForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this objects create forbidden response a status code equal to that given -func (o *ObjectsCreateForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the objects create forbidden response -func (o *ObjectsCreateForbidden) Code() int { - return 403 -} - -func (o *ObjectsCreateForbidden) Error() string { - return fmt.Sprintf("[POST /objects][%d] objectsCreateForbidden %+v", 403, o.Payload) -} - -func (o *ObjectsCreateForbidden) String() string { - return fmt.Sprintf("[POST /objects][%d] objectsCreateForbidden %+v", 403, o.Payload) -} - -func (o *ObjectsCreateForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsCreateForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsCreateUnprocessableEntity creates a ObjectsCreateUnprocessableEntity with default headers values -func NewObjectsCreateUnprocessableEntity() *ObjectsCreateUnprocessableEntity { - return &ObjectsCreateUnprocessableEntity{} -} - -/* -ObjectsCreateUnprocessableEntity describes a response with status code 422, with default header values. - -Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file? -*/ -type ObjectsCreateUnprocessableEntity struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects create unprocessable entity response has a 2xx status code -func (o *ObjectsCreateUnprocessableEntity) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects create unprocessable entity response has a 3xx status code -func (o *ObjectsCreateUnprocessableEntity) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects create unprocessable entity response has a 4xx status code -func (o *ObjectsCreateUnprocessableEntity) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects create unprocessable entity response has a 5xx status code -func (o *ObjectsCreateUnprocessableEntity) IsServerError() bool { - return false -} - -// IsCode returns true when this objects create unprocessable entity response a status code equal to that given -func (o *ObjectsCreateUnprocessableEntity) IsCode(code int) bool { - return code == 422 -} - -// Code gets the status code for the objects create unprocessable entity response -func (o *ObjectsCreateUnprocessableEntity) Code() int { - return 422 -} - -func (o *ObjectsCreateUnprocessableEntity) Error() string { - return fmt.Sprintf("[POST /objects][%d] objectsCreateUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *ObjectsCreateUnprocessableEntity) String() string { - return fmt.Sprintf("[POST /objects][%d] objectsCreateUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *ObjectsCreateUnprocessableEntity) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsCreateUnprocessableEntity) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsCreateInternalServerError creates a ObjectsCreateInternalServerError with default headers values -func NewObjectsCreateInternalServerError() *ObjectsCreateInternalServerError { - return &ObjectsCreateInternalServerError{} -} - -/* -ObjectsCreateInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type ObjectsCreateInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects create internal server error response has a 2xx status code -func (o *ObjectsCreateInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects create internal server error response has a 3xx status code -func (o *ObjectsCreateInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects create internal server error response has a 4xx status code -func (o *ObjectsCreateInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this objects create internal server error response has a 5xx status code -func (o *ObjectsCreateInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this objects create internal server error response a status code equal to that given -func (o *ObjectsCreateInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the objects create internal server error response -func (o *ObjectsCreateInternalServerError) Code() int { - return 500 -} - -func (o *ObjectsCreateInternalServerError) Error() string { - return fmt.Sprintf("[POST /objects][%d] objectsCreateInternalServerError %+v", 500, o.Payload) -} - -func (o *ObjectsCreateInternalServerError) String() string { - return fmt.Sprintf("[POST /objects][%d] objectsCreateInternalServerError %+v", 500, o.Payload) -} - -func (o *ObjectsCreateInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsCreateInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/objects/objects_delete_parameters.go b/client/objects/objects_delete_parameters.go deleted file mode 100644 index 26cc321779c288e434e5277f7333b4e8b22eb858..0000000000000000000000000000000000000000 --- a/client/objects/objects_delete_parameters.go +++ /dev/null @@ -1,232 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" -) - -// NewObjectsDeleteParams creates a new ObjectsDeleteParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewObjectsDeleteParams() *ObjectsDeleteParams { - return &ObjectsDeleteParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewObjectsDeleteParamsWithTimeout creates a new ObjectsDeleteParams object -// with the ability to set a timeout on a request. -func NewObjectsDeleteParamsWithTimeout(timeout time.Duration) *ObjectsDeleteParams { - return &ObjectsDeleteParams{ - timeout: timeout, - } -} - -// NewObjectsDeleteParamsWithContext creates a new ObjectsDeleteParams object -// with the ability to set a context for a request. -func NewObjectsDeleteParamsWithContext(ctx context.Context) *ObjectsDeleteParams { - return &ObjectsDeleteParams{ - Context: ctx, - } -} - -// NewObjectsDeleteParamsWithHTTPClient creates a new ObjectsDeleteParams object -// with the ability to set a custom HTTPClient for a request. -func NewObjectsDeleteParamsWithHTTPClient(client *http.Client) *ObjectsDeleteParams { - return &ObjectsDeleteParams{ - HTTPClient: client, - } -} - -/* -ObjectsDeleteParams contains all the parameters to send to the API endpoint - - for the objects delete operation. - - Typically these are written to a http.Request. -*/ -type ObjectsDeleteParams struct { - - /* ConsistencyLevel. - - Determines how many replicas must acknowledge a request before it is considered successful - */ - ConsistencyLevel *string - - /* ID. - - Unique ID of the Object. - - Format: uuid - */ - ID strfmt.UUID - - /* Tenant. - - Specifies the tenant in a request targeting a multi-tenant class - */ - Tenant *string - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the objects delete params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *ObjectsDeleteParams) WithDefaults() *ObjectsDeleteParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the objects delete params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *ObjectsDeleteParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the objects delete params -func (o *ObjectsDeleteParams) WithTimeout(timeout time.Duration) *ObjectsDeleteParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the objects delete params -func (o *ObjectsDeleteParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the objects delete params -func (o *ObjectsDeleteParams) WithContext(ctx context.Context) *ObjectsDeleteParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the objects delete params -func (o *ObjectsDeleteParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the objects delete params -func (o *ObjectsDeleteParams) WithHTTPClient(client *http.Client) *ObjectsDeleteParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the objects delete params -func (o *ObjectsDeleteParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithConsistencyLevel adds the consistencyLevel to the objects delete params -func (o *ObjectsDeleteParams) WithConsistencyLevel(consistencyLevel *string) *ObjectsDeleteParams { - o.SetConsistencyLevel(consistencyLevel) - return o -} - -// SetConsistencyLevel adds the consistencyLevel to the objects delete params -func (o *ObjectsDeleteParams) SetConsistencyLevel(consistencyLevel *string) { - o.ConsistencyLevel = consistencyLevel -} - -// WithID adds the id to the objects delete params -func (o *ObjectsDeleteParams) WithID(id strfmt.UUID) *ObjectsDeleteParams { - o.SetID(id) - return o -} - -// SetID adds the id to the objects delete params -func (o *ObjectsDeleteParams) SetID(id strfmt.UUID) { - o.ID = id -} - -// WithTenant adds the tenant to the objects delete params -func (o *ObjectsDeleteParams) WithTenant(tenant *string) *ObjectsDeleteParams { - o.SetTenant(tenant) - return o -} - -// SetTenant adds the tenant to the objects delete params -func (o *ObjectsDeleteParams) SetTenant(tenant *string) { - o.Tenant = tenant -} - -// WriteToRequest writes these params to a swagger request -func (o *ObjectsDeleteParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - - if o.ConsistencyLevel != nil { - - // query param consistency_level - var qrConsistencyLevel string - - if o.ConsistencyLevel != nil { - qrConsistencyLevel = *o.ConsistencyLevel - } - qConsistencyLevel := qrConsistencyLevel - if qConsistencyLevel != "" { - - if err := r.SetQueryParam("consistency_level", qConsistencyLevel); err != nil { - return err - } - } - } - - // path param id - if err := r.SetPathParam("id", o.ID.String()); err != nil { - return err - } - - if o.Tenant != nil { - - // query param tenant - var qrTenant string - - if o.Tenant != nil { - qrTenant = *o.Tenant - } - qTenant := qrTenant - if qTenant != "" { - - if err := r.SetQueryParam("tenant", qTenant); err != nil { - return err - } - } - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/objects/objects_delete_responses.go b/client/objects/objects_delete_responses.go deleted file mode 100644 index 9c346703901ab32ede7c4c2e9b8ea03a344b08e7..0000000000000000000000000000000000000000 --- a/client/objects/objects_delete_responses.go +++ /dev/null @@ -1,374 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsDeleteReader is a Reader for the ObjectsDelete structure. -type ObjectsDeleteReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *ObjectsDeleteReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 204: - result := NewObjectsDeleteNoContent() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 401: - result := NewObjectsDeleteUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewObjectsDeleteForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 404: - result := NewObjectsDeleteNotFound() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewObjectsDeleteInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewObjectsDeleteNoContent creates a ObjectsDeleteNoContent with default headers values -func NewObjectsDeleteNoContent() *ObjectsDeleteNoContent { - return &ObjectsDeleteNoContent{} -} - -/* -ObjectsDeleteNoContent describes a response with status code 204, with default header values. - -Successfully deleted. -*/ -type ObjectsDeleteNoContent struct { -} - -// IsSuccess returns true when this objects delete no content response has a 2xx status code -func (o *ObjectsDeleteNoContent) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this objects delete no content response has a 3xx status code -func (o *ObjectsDeleteNoContent) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects delete no content response has a 4xx status code -func (o *ObjectsDeleteNoContent) IsClientError() bool { - return false -} - -// IsServerError returns true when this objects delete no content response has a 5xx status code -func (o *ObjectsDeleteNoContent) IsServerError() bool { - return false -} - -// IsCode returns true when this objects delete no content response a status code equal to that given -func (o *ObjectsDeleteNoContent) IsCode(code int) bool { - return code == 204 -} - -// Code gets the status code for the objects delete no content response -func (o *ObjectsDeleteNoContent) Code() int { - return 204 -} - -func (o *ObjectsDeleteNoContent) Error() string { - return fmt.Sprintf("[DELETE /objects/{id}][%d] objectsDeleteNoContent ", 204) -} - -func (o *ObjectsDeleteNoContent) String() string { - return fmt.Sprintf("[DELETE /objects/{id}][%d] objectsDeleteNoContent ", 204) -} - -func (o *ObjectsDeleteNoContent) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsDeleteUnauthorized creates a ObjectsDeleteUnauthorized with default headers values -func NewObjectsDeleteUnauthorized() *ObjectsDeleteUnauthorized { - return &ObjectsDeleteUnauthorized{} -} - -/* -ObjectsDeleteUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type ObjectsDeleteUnauthorized struct { -} - -// IsSuccess returns true when this objects delete unauthorized response has a 2xx status code -func (o *ObjectsDeleteUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects delete unauthorized response has a 3xx status code -func (o *ObjectsDeleteUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects delete unauthorized response has a 4xx status code -func (o *ObjectsDeleteUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects delete unauthorized response has a 5xx status code -func (o *ObjectsDeleteUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this objects delete unauthorized response a status code equal to that given -func (o *ObjectsDeleteUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the objects delete unauthorized response -func (o *ObjectsDeleteUnauthorized) Code() int { - return 401 -} - -func (o *ObjectsDeleteUnauthorized) Error() string { - return fmt.Sprintf("[DELETE /objects/{id}][%d] objectsDeleteUnauthorized ", 401) -} - -func (o *ObjectsDeleteUnauthorized) String() string { - return fmt.Sprintf("[DELETE /objects/{id}][%d] objectsDeleteUnauthorized ", 401) -} - -func (o *ObjectsDeleteUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsDeleteForbidden creates a ObjectsDeleteForbidden with default headers values -func NewObjectsDeleteForbidden() *ObjectsDeleteForbidden { - return &ObjectsDeleteForbidden{} -} - -/* -ObjectsDeleteForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type ObjectsDeleteForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects delete forbidden response has a 2xx status code -func (o *ObjectsDeleteForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects delete forbidden response has a 3xx status code -func (o *ObjectsDeleteForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects delete forbidden response has a 4xx status code -func (o *ObjectsDeleteForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects delete forbidden response has a 5xx status code -func (o *ObjectsDeleteForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this objects delete forbidden response a status code equal to that given -func (o *ObjectsDeleteForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the objects delete forbidden response -func (o *ObjectsDeleteForbidden) Code() int { - return 403 -} - -func (o *ObjectsDeleteForbidden) Error() string { - return fmt.Sprintf("[DELETE /objects/{id}][%d] objectsDeleteForbidden %+v", 403, o.Payload) -} - -func (o *ObjectsDeleteForbidden) String() string { - return fmt.Sprintf("[DELETE /objects/{id}][%d] objectsDeleteForbidden %+v", 403, o.Payload) -} - -func (o *ObjectsDeleteForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsDeleteForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsDeleteNotFound creates a ObjectsDeleteNotFound with default headers values -func NewObjectsDeleteNotFound() *ObjectsDeleteNotFound { - return &ObjectsDeleteNotFound{} -} - -/* -ObjectsDeleteNotFound describes a response with status code 404, with default header values. - -Successful query result but no resource was found. -*/ -type ObjectsDeleteNotFound struct { -} - -// IsSuccess returns true when this objects delete not found response has a 2xx status code -func (o *ObjectsDeleteNotFound) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects delete not found response has a 3xx status code -func (o *ObjectsDeleteNotFound) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects delete not found response has a 4xx status code -func (o *ObjectsDeleteNotFound) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects delete not found response has a 5xx status code -func (o *ObjectsDeleteNotFound) IsServerError() bool { - return false -} - -// IsCode returns true when this objects delete not found response a status code equal to that given -func (o *ObjectsDeleteNotFound) IsCode(code int) bool { - return code == 404 -} - -// Code gets the status code for the objects delete not found response -func (o *ObjectsDeleteNotFound) Code() int { - return 404 -} - -func (o *ObjectsDeleteNotFound) Error() string { - return fmt.Sprintf("[DELETE /objects/{id}][%d] objectsDeleteNotFound ", 404) -} - -func (o *ObjectsDeleteNotFound) String() string { - return fmt.Sprintf("[DELETE /objects/{id}][%d] objectsDeleteNotFound ", 404) -} - -func (o *ObjectsDeleteNotFound) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsDeleteInternalServerError creates a ObjectsDeleteInternalServerError with default headers values -func NewObjectsDeleteInternalServerError() *ObjectsDeleteInternalServerError { - return &ObjectsDeleteInternalServerError{} -} - -/* -ObjectsDeleteInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type ObjectsDeleteInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects delete internal server error response has a 2xx status code -func (o *ObjectsDeleteInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects delete internal server error response has a 3xx status code -func (o *ObjectsDeleteInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects delete internal server error response has a 4xx status code -func (o *ObjectsDeleteInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this objects delete internal server error response has a 5xx status code -func (o *ObjectsDeleteInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this objects delete internal server error response a status code equal to that given -func (o *ObjectsDeleteInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the objects delete internal server error response -func (o *ObjectsDeleteInternalServerError) Code() int { - return 500 -} - -func (o *ObjectsDeleteInternalServerError) Error() string { - return fmt.Sprintf("[DELETE /objects/{id}][%d] objectsDeleteInternalServerError %+v", 500, o.Payload) -} - -func (o *ObjectsDeleteInternalServerError) String() string { - return fmt.Sprintf("[DELETE /objects/{id}][%d] objectsDeleteInternalServerError %+v", 500, o.Payload) -} - -func (o *ObjectsDeleteInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsDeleteInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/objects/objects_get_parameters.go b/client/objects/objects_get_parameters.go deleted file mode 100644 index 2b17dd095ceda325033f4ebaeadb7f013b0914af..0000000000000000000000000000000000000000 --- a/client/objects/objects_get_parameters.go +++ /dev/null @@ -1,198 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" -) - -// NewObjectsGetParams creates a new ObjectsGetParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewObjectsGetParams() *ObjectsGetParams { - return &ObjectsGetParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewObjectsGetParamsWithTimeout creates a new ObjectsGetParams object -// with the ability to set a timeout on a request. -func NewObjectsGetParamsWithTimeout(timeout time.Duration) *ObjectsGetParams { - return &ObjectsGetParams{ - timeout: timeout, - } -} - -// NewObjectsGetParamsWithContext creates a new ObjectsGetParams object -// with the ability to set a context for a request. -func NewObjectsGetParamsWithContext(ctx context.Context) *ObjectsGetParams { - return &ObjectsGetParams{ - Context: ctx, - } -} - -// NewObjectsGetParamsWithHTTPClient creates a new ObjectsGetParams object -// with the ability to set a custom HTTPClient for a request. -func NewObjectsGetParamsWithHTTPClient(client *http.Client) *ObjectsGetParams { - return &ObjectsGetParams{ - HTTPClient: client, - } -} - -/* -ObjectsGetParams contains all the parameters to send to the API endpoint - - for the objects get operation. - - Typically these are written to a http.Request. -*/ -type ObjectsGetParams struct { - - /* ID. - - Unique ID of the Object. - - Format: uuid - */ - ID strfmt.UUID - - /* Include. - - Include additional information, such as classification infos. Allowed values include: classification, vector, interpretation - */ - Include *string - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the objects get params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *ObjectsGetParams) WithDefaults() *ObjectsGetParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the objects get params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *ObjectsGetParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the objects get params -func (o *ObjectsGetParams) WithTimeout(timeout time.Duration) *ObjectsGetParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the objects get params -func (o *ObjectsGetParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the objects get params -func (o *ObjectsGetParams) WithContext(ctx context.Context) *ObjectsGetParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the objects get params -func (o *ObjectsGetParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the objects get params -func (o *ObjectsGetParams) WithHTTPClient(client *http.Client) *ObjectsGetParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the objects get params -func (o *ObjectsGetParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithID adds the id to the objects get params -func (o *ObjectsGetParams) WithID(id strfmt.UUID) *ObjectsGetParams { - o.SetID(id) - return o -} - -// SetID adds the id to the objects get params -func (o *ObjectsGetParams) SetID(id strfmt.UUID) { - o.ID = id -} - -// WithInclude adds the include to the objects get params -func (o *ObjectsGetParams) WithInclude(include *string) *ObjectsGetParams { - o.SetInclude(include) - return o -} - -// SetInclude adds the include to the objects get params -func (o *ObjectsGetParams) SetInclude(include *string) { - o.Include = include -} - -// WriteToRequest writes these params to a swagger request -func (o *ObjectsGetParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - - // path param id - if err := r.SetPathParam("id", o.ID.String()); err != nil { - return err - } - - if o.Include != nil { - - // query param include - var qrInclude string - - if o.Include != nil { - qrInclude = *o.Include - } - qInclude := qrInclude - if qInclude != "" { - - if err := r.SetQueryParam("include", qInclude); err != nil { - return err - } - } - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/objects/objects_get_responses.go b/client/objects/objects_get_responses.go deleted file mode 100644 index dbd2620ff9e29650e21cb5782a9331a21e0a1b94..0000000000000000000000000000000000000000 --- a/client/objects/objects_get_responses.go +++ /dev/null @@ -1,460 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsGetReader is a Reader for the ObjectsGet structure. -type ObjectsGetReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *ObjectsGetReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 200: - result := NewObjectsGetOK() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 400: - result := NewObjectsGetBadRequest() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 401: - result := NewObjectsGetUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewObjectsGetForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 404: - result := NewObjectsGetNotFound() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewObjectsGetInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewObjectsGetOK creates a ObjectsGetOK with default headers values -func NewObjectsGetOK() *ObjectsGetOK { - return &ObjectsGetOK{} -} - -/* -ObjectsGetOK describes a response with status code 200, with default header values. - -Successful response. -*/ -type ObjectsGetOK struct { - Payload *models.Object -} - -// IsSuccess returns true when this objects get o k response has a 2xx status code -func (o *ObjectsGetOK) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this objects get o k response has a 3xx status code -func (o *ObjectsGetOK) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects get o k response has a 4xx status code -func (o *ObjectsGetOK) IsClientError() bool { - return false -} - -// IsServerError returns true when this objects get o k response has a 5xx status code -func (o *ObjectsGetOK) IsServerError() bool { - return false -} - -// IsCode returns true when this objects get o k response a status code equal to that given -func (o *ObjectsGetOK) IsCode(code int) bool { - return code == 200 -} - -// Code gets the status code for the objects get o k response -func (o *ObjectsGetOK) Code() int { - return 200 -} - -func (o *ObjectsGetOK) Error() string { - return fmt.Sprintf("[GET /objects/{id}][%d] objectsGetOK %+v", 200, o.Payload) -} - -func (o *ObjectsGetOK) String() string { - return fmt.Sprintf("[GET /objects/{id}][%d] objectsGetOK %+v", 200, o.Payload) -} - -func (o *ObjectsGetOK) GetPayload() *models.Object { - return o.Payload -} - -func (o *ObjectsGetOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.Object) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsGetBadRequest creates a ObjectsGetBadRequest with default headers values -func NewObjectsGetBadRequest() *ObjectsGetBadRequest { - return &ObjectsGetBadRequest{} -} - -/* -ObjectsGetBadRequest describes a response with status code 400, with default header values. - -Malformed request. -*/ -type ObjectsGetBadRequest struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects get bad request response has a 2xx status code -func (o *ObjectsGetBadRequest) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects get bad request response has a 3xx status code -func (o *ObjectsGetBadRequest) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects get bad request response has a 4xx status code -func (o *ObjectsGetBadRequest) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects get bad request response has a 5xx status code -func (o *ObjectsGetBadRequest) IsServerError() bool { - return false -} - -// IsCode returns true when this objects get bad request response a status code equal to that given -func (o *ObjectsGetBadRequest) IsCode(code int) bool { - return code == 400 -} - -// Code gets the status code for the objects get bad request response -func (o *ObjectsGetBadRequest) Code() int { - return 400 -} - -func (o *ObjectsGetBadRequest) Error() string { - return fmt.Sprintf("[GET /objects/{id}][%d] objectsGetBadRequest %+v", 400, o.Payload) -} - -func (o *ObjectsGetBadRequest) String() string { - return fmt.Sprintf("[GET /objects/{id}][%d] objectsGetBadRequest %+v", 400, o.Payload) -} - -func (o *ObjectsGetBadRequest) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsGetBadRequest) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsGetUnauthorized creates a ObjectsGetUnauthorized with default headers values -func NewObjectsGetUnauthorized() *ObjectsGetUnauthorized { - return &ObjectsGetUnauthorized{} -} - -/* -ObjectsGetUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type ObjectsGetUnauthorized struct { -} - -// IsSuccess returns true when this objects get unauthorized response has a 2xx status code -func (o *ObjectsGetUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects get unauthorized response has a 3xx status code -func (o *ObjectsGetUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects get unauthorized response has a 4xx status code -func (o *ObjectsGetUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects get unauthorized response has a 5xx status code -func (o *ObjectsGetUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this objects get unauthorized response a status code equal to that given -func (o *ObjectsGetUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the objects get unauthorized response -func (o *ObjectsGetUnauthorized) Code() int { - return 401 -} - -func (o *ObjectsGetUnauthorized) Error() string { - return fmt.Sprintf("[GET /objects/{id}][%d] objectsGetUnauthorized ", 401) -} - -func (o *ObjectsGetUnauthorized) String() string { - return fmt.Sprintf("[GET /objects/{id}][%d] objectsGetUnauthorized ", 401) -} - -func (o *ObjectsGetUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsGetForbidden creates a ObjectsGetForbidden with default headers values -func NewObjectsGetForbidden() *ObjectsGetForbidden { - return &ObjectsGetForbidden{} -} - -/* -ObjectsGetForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type ObjectsGetForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects get forbidden response has a 2xx status code -func (o *ObjectsGetForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects get forbidden response has a 3xx status code -func (o *ObjectsGetForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects get forbidden response has a 4xx status code -func (o *ObjectsGetForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects get forbidden response has a 5xx status code -func (o *ObjectsGetForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this objects get forbidden response a status code equal to that given -func (o *ObjectsGetForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the objects get forbidden response -func (o *ObjectsGetForbidden) Code() int { - return 403 -} - -func (o *ObjectsGetForbidden) Error() string { - return fmt.Sprintf("[GET /objects/{id}][%d] objectsGetForbidden %+v", 403, o.Payload) -} - -func (o *ObjectsGetForbidden) String() string { - return fmt.Sprintf("[GET /objects/{id}][%d] objectsGetForbidden %+v", 403, o.Payload) -} - -func (o *ObjectsGetForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsGetForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsGetNotFound creates a ObjectsGetNotFound with default headers values -func NewObjectsGetNotFound() *ObjectsGetNotFound { - return &ObjectsGetNotFound{} -} - -/* -ObjectsGetNotFound describes a response with status code 404, with default header values. - -Successful query result but no resource was found. -*/ -type ObjectsGetNotFound struct { -} - -// IsSuccess returns true when this objects get not found response has a 2xx status code -func (o *ObjectsGetNotFound) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects get not found response has a 3xx status code -func (o *ObjectsGetNotFound) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects get not found response has a 4xx status code -func (o *ObjectsGetNotFound) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects get not found response has a 5xx status code -func (o *ObjectsGetNotFound) IsServerError() bool { - return false -} - -// IsCode returns true when this objects get not found response a status code equal to that given -func (o *ObjectsGetNotFound) IsCode(code int) bool { - return code == 404 -} - -// Code gets the status code for the objects get not found response -func (o *ObjectsGetNotFound) Code() int { - return 404 -} - -func (o *ObjectsGetNotFound) Error() string { - return fmt.Sprintf("[GET /objects/{id}][%d] objectsGetNotFound ", 404) -} - -func (o *ObjectsGetNotFound) String() string { - return fmt.Sprintf("[GET /objects/{id}][%d] objectsGetNotFound ", 404) -} - -func (o *ObjectsGetNotFound) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsGetInternalServerError creates a ObjectsGetInternalServerError with default headers values -func NewObjectsGetInternalServerError() *ObjectsGetInternalServerError { - return &ObjectsGetInternalServerError{} -} - -/* -ObjectsGetInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type ObjectsGetInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects get internal server error response has a 2xx status code -func (o *ObjectsGetInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects get internal server error response has a 3xx status code -func (o *ObjectsGetInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects get internal server error response has a 4xx status code -func (o *ObjectsGetInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this objects get internal server error response has a 5xx status code -func (o *ObjectsGetInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this objects get internal server error response a status code equal to that given -func (o *ObjectsGetInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the objects get internal server error response -func (o *ObjectsGetInternalServerError) Code() int { - return 500 -} - -func (o *ObjectsGetInternalServerError) Error() string { - return fmt.Sprintf("[GET /objects/{id}][%d] objectsGetInternalServerError %+v", 500, o.Payload) -} - -func (o *ObjectsGetInternalServerError) String() string { - return fmt.Sprintf("[GET /objects/{id}][%d] objectsGetInternalServerError %+v", 500, o.Payload) -} - -func (o *ObjectsGetInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsGetInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/objects/objects_head_parameters.go b/client/objects/objects_head_parameters.go deleted file mode 100644 index 5d053176337506b296e9687268d9fa878b9835c7..0000000000000000000000000000000000000000 --- a/client/objects/objects_head_parameters.go +++ /dev/null @@ -1,164 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" -) - -// NewObjectsHeadParams creates a new ObjectsHeadParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewObjectsHeadParams() *ObjectsHeadParams { - return &ObjectsHeadParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewObjectsHeadParamsWithTimeout creates a new ObjectsHeadParams object -// with the ability to set a timeout on a request. -func NewObjectsHeadParamsWithTimeout(timeout time.Duration) *ObjectsHeadParams { - return &ObjectsHeadParams{ - timeout: timeout, - } -} - -// NewObjectsHeadParamsWithContext creates a new ObjectsHeadParams object -// with the ability to set a context for a request. -func NewObjectsHeadParamsWithContext(ctx context.Context) *ObjectsHeadParams { - return &ObjectsHeadParams{ - Context: ctx, - } -} - -// NewObjectsHeadParamsWithHTTPClient creates a new ObjectsHeadParams object -// with the ability to set a custom HTTPClient for a request. -func NewObjectsHeadParamsWithHTTPClient(client *http.Client) *ObjectsHeadParams { - return &ObjectsHeadParams{ - HTTPClient: client, - } -} - -/* -ObjectsHeadParams contains all the parameters to send to the API endpoint - - for the objects head operation. - - Typically these are written to a http.Request. -*/ -type ObjectsHeadParams struct { - - /* ID. - - Unique ID of the Object. - - Format: uuid - */ - ID strfmt.UUID - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the objects head params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *ObjectsHeadParams) WithDefaults() *ObjectsHeadParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the objects head params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *ObjectsHeadParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the objects head params -func (o *ObjectsHeadParams) WithTimeout(timeout time.Duration) *ObjectsHeadParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the objects head params -func (o *ObjectsHeadParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the objects head params -func (o *ObjectsHeadParams) WithContext(ctx context.Context) *ObjectsHeadParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the objects head params -func (o *ObjectsHeadParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the objects head params -func (o *ObjectsHeadParams) WithHTTPClient(client *http.Client) *ObjectsHeadParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the objects head params -func (o *ObjectsHeadParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithID adds the id to the objects head params -func (o *ObjectsHeadParams) WithID(id strfmt.UUID) *ObjectsHeadParams { - o.SetID(id) - return o -} - -// SetID adds the id to the objects head params -func (o *ObjectsHeadParams) SetID(id strfmt.UUID) { - o.ID = id -} - -// WriteToRequest writes these params to a swagger request -func (o *ObjectsHeadParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - - // path param id - if err := r.SetPathParam("id", o.ID.String()); err != nil { - return err - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/objects/objects_head_responses.go b/client/objects/objects_head_responses.go deleted file mode 100644 index 3d70922a5a41fd5800a11d22292ca47fb2f49c08..0000000000000000000000000000000000000000 --- a/client/objects/objects_head_responses.go +++ /dev/null @@ -1,374 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsHeadReader is a Reader for the ObjectsHead structure. -type ObjectsHeadReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *ObjectsHeadReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 204: - result := NewObjectsHeadNoContent() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 401: - result := NewObjectsHeadUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewObjectsHeadForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 404: - result := NewObjectsHeadNotFound() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewObjectsHeadInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewObjectsHeadNoContent creates a ObjectsHeadNoContent with default headers values -func NewObjectsHeadNoContent() *ObjectsHeadNoContent { - return &ObjectsHeadNoContent{} -} - -/* -ObjectsHeadNoContent describes a response with status code 204, with default header values. - -Object exists. -*/ -type ObjectsHeadNoContent struct { -} - -// IsSuccess returns true when this objects head no content response has a 2xx status code -func (o *ObjectsHeadNoContent) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this objects head no content response has a 3xx status code -func (o *ObjectsHeadNoContent) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects head no content response has a 4xx status code -func (o *ObjectsHeadNoContent) IsClientError() bool { - return false -} - -// IsServerError returns true when this objects head no content response has a 5xx status code -func (o *ObjectsHeadNoContent) IsServerError() bool { - return false -} - -// IsCode returns true when this objects head no content response a status code equal to that given -func (o *ObjectsHeadNoContent) IsCode(code int) bool { - return code == 204 -} - -// Code gets the status code for the objects head no content response -func (o *ObjectsHeadNoContent) Code() int { - return 204 -} - -func (o *ObjectsHeadNoContent) Error() string { - return fmt.Sprintf("[HEAD /objects/{id}][%d] objectsHeadNoContent ", 204) -} - -func (o *ObjectsHeadNoContent) String() string { - return fmt.Sprintf("[HEAD /objects/{id}][%d] objectsHeadNoContent ", 204) -} - -func (o *ObjectsHeadNoContent) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsHeadUnauthorized creates a ObjectsHeadUnauthorized with default headers values -func NewObjectsHeadUnauthorized() *ObjectsHeadUnauthorized { - return &ObjectsHeadUnauthorized{} -} - -/* -ObjectsHeadUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type ObjectsHeadUnauthorized struct { -} - -// IsSuccess returns true when this objects head unauthorized response has a 2xx status code -func (o *ObjectsHeadUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects head unauthorized response has a 3xx status code -func (o *ObjectsHeadUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects head unauthorized response has a 4xx status code -func (o *ObjectsHeadUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects head unauthorized response has a 5xx status code -func (o *ObjectsHeadUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this objects head unauthorized response a status code equal to that given -func (o *ObjectsHeadUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the objects head unauthorized response -func (o *ObjectsHeadUnauthorized) Code() int { - return 401 -} - -func (o *ObjectsHeadUnauthorized) Error() string { - return fmt.Sprintf("[HEAD /objects/{id}][%d] objectsHeadUnauthorized ", 401) -} - -func (o *ObjectsHeadUnauthorized) String() string { - return fmt.Sprintf("[HEAD /objects/{id}][%d] objectsHeadUnauthorized ", 401) -} - -func (o *ObjectsHeadUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsHeadForbidden creates a ObjectsHeadForbidden with default headers values -func NewObjectsHeadForbidden() *ObjectsHeadForbidden { - return &ObjectsHeadForbidden{} -} - -/* -ObjectsHeadForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type ObjectsHeadForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects head forbidden response has a 2xx status code -func (o *ObjectsHeadForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects head forbidden response has a 3xx status code -func (o *ObjectsHeadForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects head forbidden response has a 4xx status code -func (o *ObjectsHeadForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects head forbidden response has a 5xx status code -func (o *ObjectsHeadForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this objects head forbidden response a status code equal to that given -func (o *ObjectsHeadForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the objects head forbidden response -func (o *ObjectsHeadForbidden) Code() int { - return 403 -} - -func (o *ObjectsHeadForbidden) Error() string { - return fmt.Sprintf("[HEAD /objects/{id}][%d] objectsHeadForbidden %+v", 403, o.Payload) -} - -func (o *ObjectsHeadForbidden) String() string { - return fmt.Sprintf("[HEAD /objects/{id}][%d] objectsHeadForbidden %+v", 403, o.Payload) -} - -func (o *ObjectsHeadForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsHeadForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsHeadNotFound creates a ObjectsHeadNotFound with default headers values -func NewObjectsHeadNotFound() *ObjectsHeadNotFound { - return &ObjectsHeadNotFound{} -} - -/* -ObjectsHeadNotFound describes a response with status code 404, with default header values. - -Object doesn't exist. -*/ -type ObjectsHeadNotFound struct { -} - -// IsSuccess returns true when this objects head not found response has a 2xx status code -func (o *ObjectsHeadNotFound) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects head not found response has a 3xx status code -func (o *ObjectsHeadNotFound) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects head not found response has a 4xx status code -func (o *ObjectsHeadNotFound) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects head not found response has a 5xx status code -func (o *ObjectsHeadNotFound) IsServerError() bool { - return false -} - -// IsCode returns true when this objects head not found response a status code equal to that given -func (o *ObjectsHeadNotFound) IsCode(code int) bool { - return code == 404 -} - -// Code gets the status code for the objects head not found response -func (o *ObjectsHeadNotFound) Code() int { - return 404 -} - -func (o *ObjectsHeadNotFound) Error() string { - return fmt.Sprintf("[HEAD /objects/{id}][%d] objectsHeadNotFound ", 404) -} - -func (o *ObjectsHeadNotFound) String() string { - return fmt.Sprintf("[HEAD /objects/{id}][%d] objectsHeadNotFound ", 404) -} - -func (o *ObjectsHeadNotFound) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsHeadInternalServerError creates a ObjectsHeadInternalServerError with default headers values -func NewObjectsHeadInternalServerError() *ObjectsHeadInternalServerError { - return &ObjectsHeadInternalServerError{} -} - -/* -ObjectsHeadInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type ObjectsHeadInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects head internal server error response has a 2xx status code -func (o *ObjectsHeadInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects head internal server error response has a 3xx status code -func (o *ObjectsHeadInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects head internal server error response has a 4xx status code -func (o *ObjectsHeadInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this objects head internal server error response has a 5xx status code -func (o *ObjectsHeadInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this objects head internal server error response a status code equal to that given -func (o *ObjectsHeadInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the objects head internal server error response -func (o *ObjectsHeadInternalServerError) Code() int { - return 500 -} - -func (o *ObjectsHeadInternalServerError) Error() string { - return fmt.Sprintf("[HEAD /objects/{id}][%d] objectsHeadInternalServerError %+v", 500, o.Payload) -} - -func (o *ObjectsHeadInternalServerError) String() string { - return fmt.Sprintf("[HEAD /objects/{id}][%d] objectsHeadInternalServerError %+v", 500, o.Payload) -} - -func (o *ObjectsHeadInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsHeadInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/objects/objects_list_parameters.go b/client/objects/objects_list_parameters.go deleted file mode 100644 index 7688ceed461c11a05fdfef6a8767a2874ae433b9..0000000000000000000000000000000000000000 --- a/client/objects/objects_list_parameters.go +++ /dev/null @@ -1,428 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" -) - -// NewObjectsListParams creates a new ObjectsListParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewObjectsListParams() *ObjectsListParams { - return &ObjectsListParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewObjectsListParamsWithTimeout creates a new ObjectsListParams object -// with the ability to set a timeout on a request. -func NewObjectsListParamsWithTimeout(timeout time.Duration) *ObjectsListParams { - return &ObjectsListParams{ - timeout: timeout, - } -} - -// NewObjectsListParamsWithContext creates a new ObjectsListParams object -// with the ability to set a context for a request. -func NewObjectsListParamsWithContext(ctx context.Context) *ObjectsListParams { - return &ObjectsListParams{ - Context: ctx, - } -} - -// NewObjectsListParamsWithHTTPClient creates a new ObjectsListParams object -// with the ability to set a custom HTTPClient for a request. -func NewObjectsListParamsWithHTTPClient(client *http.Client) *ObjectsListParams { - return &ObjectsListParams{ - HTTPClient: client, - } -} - -/* -ObjectsListParams contains all the parameters to send to the API endpoint - - for the objects list operation. - - Typically these are written to a http.Request. -*/ -type ObjectsListParams struct { - - /* After. - - The starting ID of the result window. - */ - After *string - - /* Class. - - Class parameter specifies the class from which to query objects - */ - Class *string - - /* Include. - - Include additional information, such as classification infos. Allowed values include: classification, vector, interpretation - */ - Include *string - - /* Limit. - - The maximum number of items to be returned per page. Default value is set in Weaviate config. - - Format: int64 - */ - Limit *int64 - - /* Offset. - - The starting index of the result window. Default value is 0. - - Format: int64 - */ - Offset *int64 - - /* Order. - - Order parameter to tell how to order (asc or desc) data within given field - */ - Order *string - - /* Sort. - - Sort parameter to pass an information about the names of the sort fields - */ - Sort *string - - /* Tenant. - - Specifies the tenant in a request targeting a multi-tenant class - */ - Tenant *string - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the objects list params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *ObjectsListParams) WithDefaults() *ObjectsListParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the objects list params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *ObjectsListParams) SetDefaults() { - var ( - offsetDefault = int64(0) - ) - - val := ObjectsListParams{ - Offset: &offsetDefault, - } - - val.timeout = o.timeout - val.Context = o.Context - val.HTTPClient = o.HTTPClient - *o = val -} - -// WithTimeout adds the timeout to the objects list params -func (o *ObjectsListParams) WithTimeout(timeout time.Duration) *ObjectsListParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the objects list params -func (o *ObjectsListParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the objects list params -func (o *ObjectsListParams) WithContext(ctx context.Context) *ObjectsListParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the objects list params -func (o *ObjectsListParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the objects list params -func (o *ObjectsListParams) WithHTTPClient(client *http.Client) *ObjectsListParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the objects list params -func (o *ObjectsListParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithAfter adds the after to the objects list params -func (o *ObjectsListParams) WithAfter(after *string) *ObjectsListParams { - o.SetAfter(after) - return o -} - -// SetAfter adds the after to the objects list params -func (o *ObjectsListParams) SetAfter(after *string) { - o.After = after -} - -// WithClass adds the class to the objects list params -func (o *ObjectsListParams) WithClass(class *string) *ObjectsListParams { - o.SetClass(class) - return o -} - -// SetClass adds the class to the objects list params -func (o *ObjectsListParams) SetClass(class *string) { - o.Class = class -} - -// WithInclude adds the include to the objects list params -func (o *ObjectsListParams) WithInclude(include *string) *ObjectsListParams { - o.SetInclude(include) - return o -} - -// SetInclude adds the include to the objects list params -func (o *ObjectsListParams) SetInclude(include *string) { - o.Include = include -} - -// WithLimit adds the limit to the objects list params -func (o *ObjectsListParams) WithLimit(limit *int64) *ObjectsListParams { - o.SetLimit(limit) - return o -} - -// SetLimit adds the limit to the objects list params -func (o *ObjectsListParams) SetLimit(limit *int64) { - o.Limit = limit -} - -// WithOffset adds the offset to the objects list params -func (o *ObjectsListParams) WithOffset(offset *int64) *ObjectsListParams { - o.SetOffset(offset) - return o -} - -// SetOffset adds the offset to the objects list params -func (o *ObjectsListParams) SetOffset(offset *int64) { - o.Offset = offset -} - -// WithOrder adds the order to the objects list params -func (o *ObjectsListParams) WithOrder(order *string) *ObjectsListParams { - o.SetOrder(order) - return o -} - -// SetOrder adds the order to the objects list params -func (o *ObjectsListParams) SetOrder(order *string) { - o.Order = order -} - -// WithSort adds the sort to the objects list params -func (o *ObjectsListParams) WithSort(sort *string) *ObjectsListParams { - o.SetSort(sort) - return o -} - -// SetSort adds the sort to the objects list params -func (o *ObjectsListParams) SetSort(sort *string) { - o.Sort = sort -} - -// WithTenant adds the tenant to the objects list params -func (o *ObjectsListParams) WithTenant(tenant *string) *ObjectsListParams { - o.SetTenant(tenant) - return o -} - -// SetTenant adds the tenant to the objects list params -func (o *ObjectsListParams) SetTenant(tenant *string) { - o.Tenant = tenant -} - -// WriteToRequest writes these params to a swagger request -func (o *ObjectsListParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - - if o.After != nil { - - // query param after - var qrAfter string - - if o.After != nil { - qrAfter = *o.After - } - qAfter := qrAfter - if qAfter != "" { - - if err := r.SetQueryParam("after", qAfter); err != nil { - return err - } - } - } - - if o.Class != nil { - - // query param class - var qrClass string - - if o.Class != nil { - qrClass = *o.Class - } - qClass := qrClass - if qClass != "" { - - if err := r.SetQueryParam("class", qClass); err != nil { - return err - } - } - } - - if o.Include != nil { - - // query param include - var qrInclude string - - if o.Include != nil { - qrInclude = *o.Include - } - qInclude := qrInclude - if qInclude != "" { - - if err := r.SetQueryParam("include", qInclude); err != nil { - return err - } - } - } - - if o.Limit != nil { - - // query param limit - var qrLimit int64 - - if o.Limit != nil { - qrLimit = *o.Limit - } - qLimit := swag.FormatInt64(qrLimit) - if qLimit != "" { - - if err := r.SetQueryParam("limit", qLimit); err != nil { - return err - } - } - } - - if o.Offset != nil { - - // query param offset - var qrOffset int64 - - if o.Offset != nil { - qrOffset = *o.Offset - } - qOffset := swag.FormatInt64(qrOffset) - if qOffset != "" { - - if err := r.SetQueryParam("offset", qOffset); err != nil { - return err - } - } - } - - if o.Order != nil { - - // query param order - var qrOrder string - - if o.Order != nil { - qrOrder = *o.Order - } - qOrder := qrOrder - if qOrder != "" { - - if err := r.SetQueryParam("order", qOrder); err != nil { - return err - } - } - } - - if o.Sort != nil { - - // query param sort - var qrSort string - - if o.Sort != nil { - qrSort = *o.Sort - } - qSort := qrSort - if qSort != "" { - - if err := r.SetQueryParam("sort", qSort); err != nil { - return err - } - } - } - - if o.Tenant != nil { - - // query param tenant - var qrTenant string - - if o.Tenant != nil { - qrTenant = *o.Tenant - } - qTenant := qrTenant - if qTenant != "" { - - if err := r.SetQueryParam("tenant", qTenant); err != nil { - return err - } - } - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/objects/objects_list_responses.go b/client/objects/objects_list_responses.go deleted file mode 100644 index 1bdcfe1cdccf8d29963ac8c16f5d1d40491b756b..0000000000000000000000000000000000000000 --- a/client/objects/objects_list_responses.go +++ /dev/null @@ -1,534 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsListReader is a Reader for the ObjectsList structure. -type ObjectsListReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *ObjectsListReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 200: - result := NewObjectsListOK() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 400: - result := NewObjectsListBadRequest() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 401: - result := NewObjectsListUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewObjectsListForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 404: - result := NewObjectsListNotFound() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 422: - result := NewObjectsListUnprocessableEntity() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewObjectsListInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewObjectsListOK creates a ObjectsListOK with default headers values -func NewObjectsListOK() *ObjectsListOK { - return &ObjectsListOK{} -} - -/* -ObjectsListOK describes a response with status code 200, with default header values. - -Successful response. -*/ -type ObjectsListOK struct { - Payload *models.ObjectsListResponse -} - -// IsSuccess returns true when this objects list o k response has a 2xx status code -func (o *ObjectsListOK) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this objects list o k response has a 3xx status code -func (o *ObjectsListOK) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects list o k response has a 4xx status code -func (o *ObjectsListOK) IsClientError() bool { - return false -} - -// IsServerError returns true when this objects list o k response has a 5xx status code -func (o *ObjectsListOK) IsServerError() bool { - return false -} - -// IsCode returns true when this objects list o k response a status code equal to that given -func (o *ObjectsListOK) IsCode(code int) bool { - return code == 200 -} - -// Code gets the status code for the objects list o k response -func (o *ObjectsListOK) Code() int { - return 200 -} - -func (o *ObjectsListOK) Error() string { - return fmt.Sprintf("[GET /objects][%d] objectsListOK %+v", 200, o.Payload) -} - -func (o *ObjectsListOK) String() string { - return fmt.Sprintf("[GET /objects][%d] objectsListOK %+v", 200, o.Payload) -} - -func (o *ObjectsListOK) GetPayload() *models.ObjectsListResponse { - return o.Payload -} - -func (o *ObjectsListOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ObjectsListResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsListBadRequest creates a ObjectsListBadRequest with default headers values -func NewObjectsListBadRequest() *ObjectsListBadRequest { - return &ObjectsListBadRequest{} -} - -/* -ObjectsListBadRequest describes a response with status code 400, with default header values. - -Malformed request. -*/ -type ObjectsListBadRequest struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects list bad request response has a 2xx status code -func (o *ObjectsListBadRequest) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects list bad request response has a 3xx status code -func (o *ObjectsListBadRequest) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects list bad request response has a 4xx status code -func (o *ObjectsListBadRequest) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects list bad request response has a 5xx status code -func (o *ObjectsListBadRequest) IsServerError() bool { - return false -} - -// IsCode returns true when this objects list bad request response a status code equal to that given -func (o *ObjectsListBadRequest) IsCode(code int) bool { - return code == 400 -} - -// Code gets the status code for the objects list bad request response -func (o *ObjectsListBadRequest) Code() int { - return 400 -} - -func (o *ObjectsListBadRequest) Error() string { - return fmt.Sprintf("[GET /objects][%d] objectsListBadRequest %+v", 400, o.Payload) -} - -func (o *ObjectsListBadRequest) String() string { - return fmt.Sprintf("[GET /objects][%d] objectsListBadRequest %+v", 400, o.Payload) -} - -func (o *ObjectsListBadRequest) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsListBadRequest) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsListUnauthorized creates a ObjectsListUnauthorized with default headers values -func NewObjectsListUnauthorized() *ObjectsListUnauthorized { - return &ObjectsListUnauthorized{} -} - -/* -ObjectsListUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type ObjectsListUnauthorized struct { -} - -// IsSuccess returns true when this objects list unauthorized response has a 2xx status code -func (o *ObjectsListUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects list unauthorized response has a 3xx status code -func (o *ObjectsListUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects list unauthorized response has a 4xx status code -func (o *ObjectsListUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects list unauthorized response has a 5xx status code -func (o *ObjectsListUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this objects list unauthorized response a status code equal to that given -func (o *ObjectsListUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the objects list unauthorized response -func (o *ObjectsListUnauthorized) Code() int { - return 401 -} - -func (o *ObjectsListUnauthorized) Error() string { - return fmt.Sprintf("[GET /objects][%d] objectsListUnauthorized ", 401) -} - -func (o *ObjectsListUnauthorized) String() string { - return fmt.Sprintf("[GET /objects][%d] objectsListUnauthorized ", 401) -} - -func (o *ObjectsListUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsListForbidden creates a ObjectsListForbidden with default headers values -func NewObjectsListForbidden() *ObjectsListForbidden { - return &ObjectsListForbidden{} -} - -/* -ObjectsListForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type ObjectsListForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects list forbidden response has a 2xx status code -func (o *ObjectsListForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects list forbidden response has a 3xx status code -func (o *ObjectsListForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects list forbidden response has a 4xx status code -func (o *ObjectsListForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects list forbidden response has a 5xx status code -func (o *ObjectsListForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this objects list forbidden response a status code equal to that given -func (o *ObjectsListForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the objects list forbidden response -func (o *ObjectsListForbidden) Code() int { - return 403 -} - -func (o *ObjectsListForbidden) Error() string { - return fmt.Sprintf("[GET /objects][%d] objectsListForbidden %+v", 403, o.Payload) -} - -func (o *ObjectsListForbidden) String() string { - return fmt.Sprintf("[GET /objects][%d] objectsListForbidden %+v", 403, o.Payload) -} - -func (o *ObjectsListForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsListForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsListNotFound creates a ObjectsListNotFound with default headers values -func NewObjectsListNotFound() *ObjectsListNotFound { - return &ObjectsListNotFound{} -} - -/* -ObjectsListNotFound describes a response with status code 404, with default header values. - -Successful query result but no resource was found. -*/ -type ObjectsListNotFound struct { -} - -// IsSuccess returns true when this objects list not found response has a 2xx status code -func (o *ObjectsListNotFound) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects list not found response has a 3xx status code -func (o *ObjectsListNotFound) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects list not found response has a 4xx status code -func (o *ObjectsListNotFound) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects list not found response has a 5xx status code -func (o *ObjectsListNotFound) IsServerError() bool { - return false -} - -// IsCode returns true when this objects list not found response a status code equal to that given -func (o *ObjectsListNotFound) IsCode(code int) bool { - return code == 404 -} - -// Code gets the status code for the objects list not found response -func (o *ObjectsListNotFound) Code() int { - return 404 -} - -func (o *ObjectsListNotFound) Error() string { - return fmt.Sprintf("[GET /objects][%d] objectsListNotFound ", 404) -} - -func (o *ObjectsListNotFound) String() string { - return fmt.Sprintf("[GET /objects][%d] objectsListNotFound ", 404) -} - -func (o *ObjectsListNotFound) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsListUnprocessableEntity creates a ObjectsListUnprocessableEntity with default headers values -func NewObjectsListUnprocessableEntity() *ObjectsListUnprocessableEntity { - return &ObjectsListUnprocessableEntity{} -} - -/* -ObjectsListUnprocessableEntity describes a response with status code 422, with default header values. - -Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file? -*/ -type ObjectsListUnprocessableEntity struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects list unprocessable entity response has a 2xx status code -func (o *ObjectsListUnprocessableEntity) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects list unprocessable entity response has a 3xx status code -func (o *ObjectsListUnprocessableEntity) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects list unprocessable entity response has a 4xx status code -func (o *ObjectsListUnprocessableEntity) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects list unprocessable entity response has a 5xx status code -func (o *ObjectsListUnprocessableEntity) IsServerError() bool { - return false -} - -// IsCode returns true when this objects list unprocessable entity response a status code equal to that given -func (o *ObjectsListUnprocessableEntity) IsCode(code int) bool { - return code == 422 -} - -// Code gets the status code for the objects list unprocessable entity response -func (o *ObjectsListUnprocessableEntity) Code() int { - return 422 -} - -func (o *ObjectsListUnprocessableEntity) Error() string { - return fmt.Sprintf("[GET /objects][%d] objectsListUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *ObjectsListUnprocessableEntity) String() string { - return fmt.Sprintf("[GET /objects][%d] objectsListUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *ObjectsListUnprocessableEntity) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsListUnprocessableEntity) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsListInternalServerError creates a ObjectsListInternalServerError with default headers values -func NewObjectsListInternalServerError() *ObjectsListInternalServerError { - return &ObjectsListInternalServerError{} -} - -/* -ObjectsListInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type ObjectsListInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects list internal server error response has a 2xx status code -func (o *ObjectsListInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects list internal server error response has a 3xx status code -func (o *ObjectsListInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects list internal server error response has a 4xx status code -func (o *ObjectsListInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this objects list internal server error response has a 5xx status code -func (o *ObjectsListInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this objects list internal server error response a status code equal to that given -func (o *ObjectsListInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the objects list internal server error response -func (o *ObjectsListInternalServerError) Code() int { - return 500 -} - -func (o *ObjectsListInternalServerError) Error() string { - return fmt.Sprintf("[GET /objects][%d] objectsListInternalServerError %+v", 500, o.Payload) -} - -func (o *ObjectsListInternalServerError) String() string { - return fmt.Sprintf("[GET /objects][%d] objectsListInternalServerError %+v", 500, o.Payload) -} - -func (o *ObjectsListInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsListInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/objects/objects_patch_parameters.go b/client/objects/objects_patch_parameters.go deleted file mode 100644 index 0fb43f7c01072baad79ee6e8d4d70255d6da6896..0000000000000000000000000000000000000000 --- a/client/objects/objects_patch_parameters.go +++ /dev/null @@ -1,222 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewObjectsPatchParams creates a new ObjectsPatchParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewObjectsPatchParams() *ObjectsPatchParams { - return &ObjectsPatchParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewObjectsPatchParamsWithTimeout creates a new ObjectsPatchParams object -// with the ability to set a timeout on a request. -func NewObjectsPatchParamsWithTimeout(timeout time.Duration) *ObjectsPatchParams { - return &ObjectsPatchParams{ - timeout: timeout, - } -} - -// NewObjectsPatchParamsWithContext creates a new ObjectsPatchParams object -// with the ability to set a context for a request. -func NewObjectsPatchParamsWithContext(ctx context.Context) *ObjectsPatchParams { - return &ObjectsPatchParams{ - Context: ctx, - } -} - -// NewObjectsPatchParamsWithHTTPClient creates a new ObjectsPatchParams object -// with the ability to set a custom HTTPClient for a request. -func NewObjectsPatchParamsWithHTTPClient(client *http.Client) *ObjectsPatchParams { - return &ObjectsPatchParams{ - HTTPClient: client, - } -} - -/* -ObjectsPatchParams contains all the parameters to send to the API endpoint - - for the objects patch operation. - - Typically these are written to a http.Request. -*/ -type ObjectsPatchParams struct { - - /* Body. - - RFC 7396-style patch, the body contains the object to merge into the existing object. - */ - Body *models.Object - - /* ConsistencyLevel. - - Determines how many replicas must acknowledge a request before it is considered successful - */ - ConsistencyLevel *string - - /* ID. - - Unique ID of the Object. - - Format: uuid - */ - ID strfmt.UUID - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the objects patch params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *ObjectsPatchParams) WithDefaults() *ObjectsPatchParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the objects patch params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *ObjectsPatchParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the objects patch params -func (o *ObjectsPatchParams) WithTimeout(timeout time.Duration) *ObjectsPatchParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the objects patch params -func (o *ObjectsPatchParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the objects patch params -func (o *ObjectsPatchParams) WithContext(ctx context.Context) *ObjectsPatchParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the objects patch params -func (o *ObjectsPatchParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the objects patch params -func (o *ObjectsPatchParams) WithHTTPClient(client *http.Client) *ObjectsPatchParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the objects patch params -func (o *ObjectsPatchParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithBody adds the body to the objects patch params -func (o *ObjectsPatchParams) WithBody(body *models.Object) *ObjectsPatchParams { - o.SetBody(body) - return o -} - -// SetBody adds the body to the objects patch params -func (o *ObjectsPatchParams) SetBody(body *models.Object) { - o.Body = body -} - -// WithConsistencyLevel adds the consistencyLevel to the objects patch params -func (o *ObjectsPatchParams) WithConsistencyLevel(consistencyLevel *string) *ObjectsPatchParams { - o.SetConsistencyLevel(consistencyLevel) - return o -} - -// SetConsistencyLevel adds the consistencyLevel to the objects patch params -func (o *ObjectsPatchParams) SetConsistencyLevel(consistencyLevel *string) { - o.ConsistencyLevel = consistencyLevel -} - -// WithID adds the id to the objects patch params -func (o *ObjectsPatchParams) WithID(id strfmt.UUID) *ObjectsPatchParams { - o.SetID(id) - return o -} - -// SetID adds the id to the objects patch params -func (o *ObjectsPatchParams) SetID(id strfmt.UUID) { - o.ID = id -} - -// WriteToRequest writes these params to a swagger request -func (o *ObjectsPatchParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - if o.Body != nil { - if err := r.SetBodyParam(o.Body); err != nil { - return err - } - } - - if o.ConsistencyLevel != nil { - - // query param consistency_level - var qrConsistencyLevel string - - if o.ConsistencyLevel != nil { - qrConsistencyLevel = *o.ConsistencyLevel - } - qConsistencyLevel := qrConsistencyLevel - if qConsistencyLevel != "" { - - if err := r.SetQueryParam("consistency_level", qConsistencyLevel); err != nil { - return err - } - } - } - - // path param id - if err := r.SetPathParam("id", o.ID.String()); err != nil { - return err - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/objects/objects_patch_responses.go b/client/objects/objects_patch_responses.go deleted file mode 100644 index 598c8fcf972d98f0cf9208fd45f8c9f65468f0a3..0000000000000000000000000000000000000000 --- a/client/objects/objects_patch_responses.go +++ /dev/null @@ -1,510 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsPatchReader is a Reader for the ObjectsPatch structure. -type ObjectsPatchReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *ObjectsPatchReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 204: - result := NewObjectsPatchNoContent() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 400: - result := NewObjectsPatchBadRequest() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 401: - result := NewObjectsPatchUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewObjectsPatchForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 404: - result := NewObjectsPatchNotFound() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 422: - result := NewObjectsPatchUnprocessableEntity() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewObjectsPatchInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewObjectsPatchNoContent creates a ObjectsPatchNoContent with default headers values -func NewObjectsPatchNoContent() *ObjectsPatchNoContent { - return &ObjectsPatchNoContent{} -} - -/* -ObjectsPatchNoContent describes a response with status code 204, with default header values. - -Successfully applied. No content provided. -*/ -type ObjectsPatchNoContent struct { -} - -// IsSuccess returns true when this objects patch no content response has a 2xx status code -func (o *ObjectsPatchNoContent) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this objects patch no content response has a 3xx status code -func (o *ObjectsPatchNoContent) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects patch no content response has a 4xx status code -func (o *ObjectsPatchNoContent) IsClientError() bool { - return false -} - -// IsServerError returns true when this objects patch no content response has a 5xx status code -func (o *ObjectsPatchNoContent) IsServerError() bool { - return false -} - -// IsCode returns true when this objects patch no content response a status code equal to that given -func (o *ObjectsPatchNoContent) IsCode(code int) bool { - return code == 204 -} - -// Code gets the status code for the objects patch no content response -func (o *ObjectsPatchNoContent) Code() int { - return 204 -} - -func (o *ObjectsPatchNoContent) Error() string { - return fmt.Sprintf("[PATCH /objects/{id}][%d] objectsPatchNoContent ", 204) -} - -func (o *ObjectsPatchNoContent) String() string { - return fmt.Sprintf("[PATCH /objects/{id}][%d] objectsPatchNoContent ", 204) -} - -func (o *ObjectsPatchNoContent) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsPatchBadRequest creates a ObjectsPatchBadRequest with default headers values -func NewObjectsPatchBadRequest() *ObjectsPatchBadRequest { - return &ObjectsPatchBadRequest{} -} - -/* -ObjectsPatchBadRequest describes a response with status code 400, with default header values. - -The patch-JSON is malformed. -*/ -type ObjectsPatchBadRequest struct { -} - -// IsSuccess returns true when this objects patch bad request response has a 2xx status code -func (o *ObjectsPatchBadRequest) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects patch bad request response has a 3xx status code -func (o *ObjectsPatchBadRequest) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects patch bad request response has a 4xx status code -func (o *ObjectsPatchBadRequest) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects patch bad request response has a 5xx status code -func (o *ObjectsPatchBadRequest) IsServerError() bool { - return false -} - -// IsCode returns true when this objects patch bad request response a status code equal to that given -func (o *ObjectsPatchBadRequest) IsCode(code int) bool { - return code == 400 -} - -// Code gets the status code for the objects patch bad request response -func (o *ObjectsPatchBadRequest) Code() int { - return 400 -} - -func (o *ObjectsPatchBadRequest) Error() string { - return fmt.Sprintf("[PATCH /objects/{id}][%d] objectsPatchBadRequest ", 400) -} - -func (o *ObjectsPatchBadRequest) String() string { - return fmt.Sprintf("[PATCH /objects/{id}][%d] objectsPatchBadRequest ", 400) -} - -func (o *ObjectsPatchBadRequest) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsPatchUnauthorized creates a ObjectsPatchUnauthorized with default headers values -func NewObjectsPatchUnauthorized() *ObjectsPatchUnauthorized { - return &ObjectsPatchUnauthorized{} -} - -/* -ObjectsPatchUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type ObjectsPatchUnauthorized struct { -} - -// IsSuccess returns true when this objects patch unauthorized response has a 2xx status code -func (o *ObjectsPatchUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects patch unauthorized response has a 3xx status code -func (o *ObjectsPatchUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects patch unauthorized response has a 4xx status code -func (o *ObjectsPatchUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects patch unauthorized response has a 5xx status code -func (o *ObjectsPatchUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this objects patch unauthorized response a status code equal to that given -func (o *ObjectsPatchUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the objects patch unauthorized response -func (o *ObjectsPatchUnauthorized) Code() int { - return 401 -} - -func (o *ObjectsPatchUnauthorized) Error() string { - return fmt.Sprintf("[PATCH /objects/{id}][%d] objectsPatchUnauthorized ", 401) -} - -func (o *ObjectsPatchUnauthorized) String() string { - return fmt.Sprintf("[PATCH /objects/{id}][%d] objectsPatchUnauthorized ", 401) -} - -func (o *ObjectsPatchUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsPatchForbidden creates a ObjectsPatchForbidden with default headers values -func NewObjectsPatchForbidden() *ObjectsPatchForbidden { - return &ObjectsPatchForbidden{} -} - -/* -ObjectsPatchForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type ObjectsPatchForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects patch forbidden response has a 2xx status code -func (o *ObjectsPatchForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects patch forbidden response has a 3xx status code -func (o *ObjectsPatchForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects patch forbidden response has a 4xx status code -func (o *ObjectsPatchForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects patch forbidden response has a 5xx status code -func (o *ObjectsPatchForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this objects patch forbidden response a status code equal to that given -func (o *ObjectsPatchForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the objects patch forbidden response -func (o *ObjectsPatchForbidden) Code() int { - return 403 -} - -func (o *ObjectsPatchForbidden) Error() string { - return fmt.Sprintf("[PATCH /objects/{id}][%d] objectsPatchForbidden %+v", 403, o.Payload) -} - -func (o *ObjectsPatchForbidden) String() string { - return fmt.Sprintf("[PATCH /objects/{id}][%d] objectsPatchForbidden %+v", 403, o.Payload) -} - -func (o *ObjectsPatchForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsPatchForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsPatchNotFound creates a ObjectsPatchNotFound with default headers values -func NewObjectsPatchNotFound() *ObjectsPatchNotFound { - return &ObjectsPatchNotFound{} -} - -/* -ObjectsPatchNotFound describes a response with status code 404, with default header values. - -Successful query result but no resource was found. -*/ -type ObjectsPatchNotFound struct { -} - -// IsSuccess returns true when this objects patch not found response has a 2xx status code -func (o *ObjectsPatchNotFound) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects patch not found response has a 3xx status code -func (o *ObjectsPatchNotFound) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects patch not found response has a 4xx status code -func (o *ObjectsPatchNotFound) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects patch not found response has a 5xx status code -func (o *ObjectsPatchNotFound) IsServerError() bool { - return false -} - -// IsCode returns true when this objects patch not found response a status code equal to that given -func (o *ObjectsPatchNotFound) IsCode(code int) bool { - return code == 404 -} - -// Code gets the status code for the objects patch not found response -func (o *ObjectsPatchNotFound) Code() int { - return 404 -} - -func (o *ObjectsPatchNotFound) Error() string { - return fmt.Sprintf("[PATCH /objects/{id}][%d] objectsPatchNotFound ", 404) -} - -func (o *ObjectsPatchNotFound) String() string { - return fmt.Sprintf("[PATCH /objects/{id}][%d] objectsPatchNotFound ", 404) -} - -func (o *ObjectsPatchNotFound) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsPatchUnprocessableEntity creates a ObjectsPatchUnprocessableEntity with default headers values -func NewObjectsPatchUnprocessableEntity() *ObjectsPatchUnprocessableEntity { - return &ObjectsPatchUnprocessableEntity{} -} - -/* -ObjectsPatchUnprocessableEntity describes a response with status code 422, with default header values. - -The patch-JSON is valid but unprocessable. -*/ -type ObjectsPatchUnprocessableEntity struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects patch unprocessable entity response has a 2xx status code -func (o *ObjectsPatchUnprocessableEntity) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects patch unprocessable entity response has a 3xx status code -func (o *ObjectsPatchUnprocessableEntity) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects patch unprocessable entity response has a 4xx status code -func (o *ObjectsPatchUnprocessableEntity) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects patch unprocessable entity response has a 5xx status code -func (o *ObjectsPatchUnprocessableEntity) IsServerError() bool { - return false -} - -// IsCode returns true when this objects patch unprocessable entity response a status code equal to that given -func (o *ObjectsPatchUnprocessableEntity) IsCode(code int) bool { - return code == 422 -} - -// Code gets the status code for the objects patch unprocessable entity response -func (o *ObjectsPatchUnprocessableEntity) Code() int { - return 422 -} - -func (o *ObjectsPatchUnprocessableEntity) Error() string { - return fmt.Sprintf("[PATCH /objects/{id}][%d] objectsPatchUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *ObjectsPatchUnprocessableEntity) String() string { - return fmt.Sprintf("[PATCH /objects/{id}][%d] objectsPatchUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *ObjectsPatchUnprocessableEntity) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsPatchUnprocessableEntity) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsPatchInternalServerError creates a ObjectsPatchInternalServerError with default headers values -func NewObjectsPatchInternalServerError() *ObjectsPatchInternalServerError { - return &ObjectsPatchInternalServerError{} -} - -/* -ObjectsPatchInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type ObjectsPatchInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects patch internal server error response has a 2xx status code -func (o *ObjectsPatchInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects patch internal server error response has a 3xx status code -func (o *ObjectsPatchInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects patch internal server error response has a 4xx status code -func (o *ObjectsPatchInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this objects patch internal server error response has a 5xx status code -func (o *ObjectsPatchInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this objects patch internal server error response a status code equal to that given -func (o *ObjectsPatchInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the objects patch internal server error response -func (o *ObjectsPatchInternalServerError) Code() int { - return 500 -} - -func (o *ObjectsPatchInternalServerError) Error() string { - return fmt.Sprintf("[PATCH /objects/{id}][%d] objectsPatchInternalServerError %+v", 500, o.Payload) -} - -func (o *ObjectsPatchInternalServerError) String() string { - return fmt.Sprintf("[PATCH /objects/{id}][%d] objectsPatchInternalServerError %+v", 500, o.Payload) -} - -func (o *ObjectsPatchInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsPatchInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/objects/objects_references_create_parameters.go b/client/objects/objects_references_create_parameters.go deleted file mode 100644 index e94c8dae70dd695494362764e80ddb121cebd322..0000000000000000000000000000000000000000 --- a/client/objects/objects_references_create_parameters.go +++ /dev/null @@ -1,241 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewObjectsReferencesCreateParams creates a new ObjectsReferencesCreateParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewObjectsReferencesCreateParams() *ObjectsReferencesCreateParams { - return &ObjectsReferencesCreateParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewObjectsReferencesCreateParamsWithTimeout creates a new ObjectsReferencesCreateParams object -// with the ability to set a timeout on a request. -func NewObjectsReferencesCreateParamsWithTimeout(timeout time.Duration) *ObjectsReferencesCreateParams { - return &ObjectsReferencesCreateParams{ - timeout: timeout, - } -} - -// NewObjectsReferencesCreateParamsWithContext creates a new ObjectsReferencesCreateParams object -// with the ability to set a context for a request. -func NewObjectsReferencesCreateParamsWithContext(ctx context.Context) *ObjectsReferencesCreateParams { - return &ObjectsReferencesCreateParams{ - Context: ctx, - } -} - -// NewObjectsReferencesCreateParamsWithHTTPClient creates a new ObjectsReferencesCreateParams object -// with the ability to set a custom HTTPClient for a request. -func NewObjectsReferencesCreateParamsWithHTTPClient(client *http.Client) *ObjectsReferencesCreateParams { - return &ObjectsReferencesCreateParams{ - HTTPClient: client, - } -} - -/* -ObjectsReferencesCreateParams contains all the parameters to send to the API endpoint - - for the objects references create operation. - - Typically these are written to a http.Request. -*/ -type ObjectsReferencesCreateParams struct { - - // Body. - Body *models.SingleRef - - /* ID. - - Unique ID of the Object. - - Format: uuid - */ - ID strfmt.UUID - - /* PropertyName. - - Unique name of the property related to the Object. - */ - PropertyName string - - /* Tenant. - - Specifies the tenant in a request targeting a multi-tenant class - */ - Tenant *string - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the objects references create params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *ObjectsReferencesCreateParams) WithDefaults() *ObjectsReferencesCreateParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the objects references create params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *ObjectsReferencesCreateParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the objects references create params -func (o *ObjectsReferencesCreateParams) WithTimeout(timeout time.Duration) *ObjectsReferencesCreateParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the objects references create params -func (o *ObjectsReferencesCreateParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the objects references create params -func (o *ObjectsReferencesCreateParams) WithContext(ctx context.Context) *ObjectsReferencesCreateParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the objects references create params -func (o *ObjectsReferencesCreateParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the objects references create params -func (o *ObjectsReferencesCreateParams) WithHTTPClient(client *http.Client) *ObjectsReferencesCreateParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the objects references create params -func (o *ObjectsReferencesCreateParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithBody adds the body to the objects references create params -func (o *ObjectsReferencesCreateParams) WithBody(body *models.SingleRef) *ObjectsReferencesCreateParams { - o.SetBody(body) - return o -} - -// SetBody adds the body to the objects references create params -func (o *ObjectsReferencesCreateParams) SetBody(body *models.SingleRef) { - o.Body = body -} - -// WithID adds the id to the objects references create params -func (o *ObjectsReferencesCreateParams) WithID(id strfmt.UUID) *ObjectsReferencesCreateParams { - o.SetID(id) - return o -} - -// SetID adds the id to the objects references create params -func (o *ObjectsReferencesCreateParams) SetID(id strfmt.UUID) { - o.ID = id -} - -// WithPropertyName adds the propertyName to the objects references create params -func (o *ObjectsReferencesCreateParams) WithPropertyName(propertyName string) *ObjectsReferencesCreateParams { - o.SetPropertyName(propertyName) - return o -} - -// SetPropertyName adds the propertyName to the objects references create params -func (o *ObjectsReferencesCreateParams) SetPropertyName(propertyName string) { - o.PropertyName = propertyName -} - -// WithTenant adds the tenant to the objects references create params -func (o *ObjectsReferencesCreateParams) WithTenant(tenant *string) *ObjectsReferencesCreateParams { - o.SetTenant(tenant) - return o -} - -// SetTenant adds the tenant to the objects references create params -func (o *ObjectsReferencesCreateParams) SetTenant(tenant *string) { - o.Tenant = tenant -} - -// WriteToRequest writes these params to a swagger request -func (o *ObjectsReferencesCreateParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - if o.Body != nil { - if err := r.SetBodyParam(o.Body); err != nil { - return err - } - } - - // path param id - if err := r.SetPathParam("id", o.ID.String()); err != nil { - return err - } - - // path param propertyName - if err := r.SetPathParam("propertyName", o.PropertyName); err != nil { - return err - } - - if o.Tenant != nil { - - // query param tenant - var qrTenant string - - if o.Tenant != nil { - qrTenant = *o.Tenant - } - qTenant := qrTenant - if qTenant != "" { - - if err := r.SetQueryParam("tenant", qTenant); err != nil { - return err - } - } - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/objects/objects_references_create_responses.go b/client/objects/objects_references_create_responses.go deleted file mode 100644 index 45a5015714c4ec87dce8572cbe39ee80a131783e..0000000000000000000000000000000000000000 --- a/client/objects/objects_references_create_responses.go +++ /dev/null @@ -1,386 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsReferencesCreateReader is a Reader for the ObjectsReferencesCreate structure. -type ObjectsReferencesCreateReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *ObjectsReferencesCreateReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 200: - result := NewObjectsReferencesCreateOK() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 401: - result := NewObjectsReferencesCreateUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewObjectsReferencesCreateForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 422: - result := NewObjectsReferencesCreateUnprocessableEntity() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewObjectsReferencesCreateInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewObjectsReferencesCreateOK creates a ObjectsReferencesCreateOK with default headers values -func NewObjectsReferencesCreateOK() *ObjectsReferencesCreateOK { - return &ObjectsReferencesCreateOK{} -} - -/* -ObjectsReferencesCreateOK describes a response with status code 200, with default header values. - -Successfully added the reference. -*/ -type ObjectsReferencesCreateOK struct { -} - -// IsSuccess returns true when this objects references create o k response has a 2xx status code -func (o *ObjectsReferencesCreateOK) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this objects references create o k response has a 3xx status code -func (o *ObjectsReferencesCreateOK) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects references create o k response has a 4xx status code -func (o *ObjectsReferencesCreateOK) IsClientError() bool { - return false -} - -// IsServerError returns true when this objects references create o k response has a 5xx status code -func (o *ObjectsReferencesCreateOK) IsServerError() bool { - return false -} - -// IsCode returns true when this objects references create o k response a status code equal to that given -func (o *ObjectsReferencesCreateOK) IsCode(code int) bool { - return code == 200 -} - -// Code gets the status code for the objects references create o k response -func (o *ObjectsReferencesCreateOK) Code() int { - return 200 -} - -func (o *ObjectsReferencesCreateOK) Error() string { - return fmt.Sprintf("[POST /objects/{id}/references/{propertyName}][%d] objectsReferencesCreateOK ", 200) -} - -func (o *ObjectsReferencesCreateOK) String() string { - return fmt.Sprintf("[POST /objects/{id}/references/{propertyName}][%d] objectsReferencesCreateOK ", 200) -} - -func (o *ObjectsReferencesCreateOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsReferencesCreateUnauthorized creates a ObjectsReferencesCreateUnauthorized with default headers values -func NewObjectsReferencesCreateUnauthorized() *ObjectsReferencesCreateUnauthorized { - return &ObjectsReferencesCreateUnauthorized{} -} - -/* -ObjectsReferencesCreateUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type ObjectsReferencesCreateUnauthorized struct { -} - -// IsSuccess returns true when this objects references create unauthorized response has a 2xx status code -func (o *ObjectsReferencesCreateUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects references create unauthorized response has a 3xx status code -func (o *ObjectsReferencesCreateUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects references create unauthorized response has a 4xx status code -func (o *ObjectsReferencesCreateUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects references create unauthorized response has a 5xx status code -func (o *ObjectsReferencesCreateUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this objects references create unauthorized response a status code equal to that given -func (o *ObjectsReferencesCreateUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the objects references create unauthorized response -func (o *ObjectsReferencesCreateUnauthorized) Code() int { - return 401 -} - -func (o *ObjectsReferencesCreateUnauthorized) Error() string { - return fmt.Sprintf("[POST /objects/{id}/references/{propertyName}][%d] objectsReferencesCreateUnauthorized ", 401) -} - -func (o *ObjectsReferencesCreateUnauthorized) String() string { - return fmt.Sprintf("[POST /objects/{id}/references/{propertyName}][%d] objectsReferencesCreateUnauthorized ", 401) -} - -func (o *ObjectsReferencesCreateUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsReferencesCreateForbidden creates a ObjectsReferencesCreateForbidden with default headers values -func NewObjectsReferencesCreateForbidden() *ObjectsReferencesCreateForbidden { - return &ObjectsReferencesCreateForbidden{} -} - -/* -ObjectsReferencesCreateForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type ObjectsReferencesCreateForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects references create forbidden response has a 2xx status code -func (o *ObjectsReferencesCreateForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects references create forbidden response has a 3xx status code -func (o *ObjectsReferencesCreateForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects references create forbidden response has a 4xx status code -func (o *ObjectsReferencesCreateForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects references create forbidden response has a 5xx status code -func (o *ObjectsReferencesCreateForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this objects references create forbidden response a status code equal to that given -func (o *ObjectsReferencesCreateForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the objects references create forbidden response -func (o *ObjectsReferencesCreateForbidden) Code() int { - return 403 -} - -func (o *ObjectsReferencesCreateForbidden) Error() string { - return fmt.Sprintf("[POST /objects/{id}/references/{propertyName}][%d] objectsReferencesCreateForbidden %+v", 403, o.Payload) -} - -func (o *ObjectsReferencesCreateForbidden) String() string { - return fmt.Sprintf("[POST /objects/{id}/references/{propertyName}][%d] objectsReferencesCreateForbidden %+v", 403, o.Payload) -} - -func (o *ObjectsReferencesCreateForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsReferencesCreateForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsReferencesCreateUnprocessableEntity creates a ObjectsReferencesCreateUnprocessableEntity with default headers values -func NewObjectsReferencesCreateUnprocessableEntity() *ObjectsReferencesCreateUnprocessableEntity { - return &ObjectsReferencesCreateUnprocessableEntity{} -} - -/* -ObjectsReferencesCreateUnprocessableEntity describes a response with status code 422, with default header values. - -Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the property exists or that it is a class? -*/ -type ObjectsReferencesCreateUnprocessableEntity struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects references create unprocessable entity response has a 2xx status code -func (o *ObjectsReferencesCreateUnprocessableEntity) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects references create unprocessable entity response has a 3xx status code -func (o *ObjectsReferencesCreateUnprocessableEntity) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects references create unprocessable entity response has a 4xx status code -func (o *ObjectsReferencesCreateUnprocessableEntity) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects references create unprocessable entity response has a 5xx status code -func (o *ObjectsReferencesCreateUnprocessableEntity) IsServerError() bool { - return false -} - -// IsCode returns true when this objects references create unprocessable entity response a status code equal to that given -func (o *ObjectsReferencesCreateUnprocessableEntity) IsCode(code int) bool { - return code == 422 -} - -// Code gets the status code for the objects references create unprocessable entity response -func (o *ObjectsReferencesCreateUnprocessableEntity) Code() int { - return 422 -} - -func (o *ObjectsReferencesCreateUnprocessableEntity) Error() string { - return fmt.Sprintf("[POST /objects/{id}/references/{propertyName}][%d] objectsReferencesCreateUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *ObjectsReferencesCreateUnprocessableEntity) String() string { - return fmt.Sprintf("[POST /objects/{id}/references/{propertyName}][%d] objectsReferencesCreateUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *ObjectsReferencesCreateUnprocessableEntity) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsReferencesCreateUnprocessableEntity) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsReferencesCreateInternalServerError creates a ObjectsReferencesCreateInternalServerError with default headers values -func NewObjectsReferencesCreateInternalServerError() *ObjectsReferencesCreateInternalServerError { - return &ObjectsReferencesCreateInternalServerError{} -} - -/* -ObjectsReferencesCreateInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type ObjectsReferencesCreateInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects references create internal server error response has a 2xx status code -func (o *ObjectsReferencesCreateInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects references create internal server error response has a 3xx status code -func (o *ObjectsReferencesCreateInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects references create internal server error response has a 4xx status code -func (o *ObjectsReferencesCreateInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this objects references create internal server error response has a 5xx status code -func (o *ObjectsReferencesCreateInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this objects references create internal server error response a status code equal to that given -func (o *ObjectsReferencesCreateInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the objects references create internal server error response -func (o *ObjectsReferencesCreateInternalServerError) Code() int { - return 500 -} - -func (o *ObjectsReferencesCreateInternalServerError) Error() string { - return fmt.Sprintf("[POST /objects/{id}/references/{propertyName}][%d] objectsReferencesCreateInternalServerError %+v", 500, o.Payload) -} - -func (o *ObjectsReferencesCreateInternalServerError) String() string { - return fmt.Sprintf("[POST /objects/{id}/references/{propertyName}][%d] objectsReferencesCreateInternalServerError %+v", 500, o.Payload) -} - -func (o *ObjectsReferencesCreateInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsReferencesCreateInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/objects/objects_references_delete_parameters.go b/client/objects/objects_references_delete_parameters.go deleted file mode 100644 index 19b683e9320d44e21b54b9c964837384fc8e3138..0000000000000000000000000000000000000000 --- a/client/objects/objects_references_delete_parameters.go +++ /dev/null @@ -1,241 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewObjectsReferencesDeleteParams creates a new ObjectsReferencesDeleteParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewObjectsReferencesDeleteParams() *ObjectsReferencesDeleteParams { - return &ObjectsReferencesDeleteParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewObjectsReferencesDeleteParamsWithTimeout creates a new ObjectsReferencesDeleteParams object -// with the ability to set a timeout on a request. -func NewObjectsReferencesDeleteParamsWithTimeout(timeout time.Duration) *ObjectsReferencesDeleteParams { - return &ObjectsReferencesDeleteParams{ - timeout: timeout, - } -} - -// NewObjectsReferencesDeleteParamsWithContext creates a new ObjectsReferencesDeleteParams object -// with the ability to set a context for a request. -func NewObjectsReferencesDeleteParamsWithContext(ctx context.Context) *ObjectsReferencesDeleteParams { - return &ObjectsReferencesDeleteParams{ - Context: ctx, - } -} - -// NewObjectsReferencesDeleteParamsWithHTTPClient creates a new ObjectsReferencesDeleteParams object -// with the ability to set a custom HTTPClient for a request. -func NewObjectsReferencesDeleteParamsWithHTTPClient(client *http.Client) *ObjectsReferencesDeleteParams { - return &ObjectsReferencesDeleteParams{ - HTTPClient: client, - } -} - -/* -ObjectsReferencesDeleteParams contains all the parameters to send to the API endpoint - - for the objects references delete operation. - - Typically these are written to a http.Request. -*/ -type ObjectsReferencesDeleteParams struct { - - // Body. - Body *models.SingleRef - - /* ID. - - Unique ID of the Object. - - Format: uuid - */ - ID strfmt.UUID - - /* PropertyName. - - Unique name of the property related to the Object. - */ - PropertyName string - - /* Tenant. - - Specifies the tenant in a request targeting a multi-tenant class - */ - Tenant *string - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the objects references delete params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *ObjectsReferencesDeleteParams) WithDefaults() *ObjectsReferencesDeleteParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the objects references delete params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *ObjectsReferencesDeleteParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the objects references delete params -func (o *ObjectsReferencesDeleteParams) WithTimeout(timeout time.Duration) *ObjectsReferencesDeleteParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the objects references delete params -func (o *ObjectsReferencesDeleteParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the objects references delete params -func (o *ObjectsReferencesDeleteParams) WithContext(ctx context.Context) *ObjectsReferencesDeleteParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the objects references delete params -func (o *ObjectsReferencesDeleteParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the objects references delete params -func (o *ObjectsReferencesDeleteParams) WithHTTPClient(client *http.Client) *ObjectsReferencesDeleteParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the objects references delete params -func (o *ObjectsReferencesDeleteParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithBody adds the body to the objects references delete params -func (o *ObjectsReferencesDeleteParams) WithBody(body *models.SingleRef) *ObjectsReferencesDeleteParams { - o.SetBody(body) - return o -} - -// SetBody adds the body to the objects references delete params -func (o *ObjectsReferencesDeleteParams) SetBody(body *models.SingleRef) { - o.Body = body -} - -// WithID adds the id to the objects references delete params -func (o *ObjectsReferencesDeleteParams) WithID(id strfmt.UUID) *ObjectsReferencesDeleteParams { - o.SetID(id) - return o -} - -// SetID adds the id to the objects references delete params -func (o *ObjectsReferencesDeleteParams) SetID(id strfmt.UUID) { - o.ID = id -} - -// WithPropertyName adds the propertyName to the objects references delete params -func (o *ObjectsReferencesDeleteParams) WithPropertyName(propertyName string) *ObjectsReferencesDeleteParams { - o.SetPropertyName(propertyName) - return o -} - -// SetPropertyName adds the propertyName to the objects references delete params -func (o *ObjectsReferencesDeleteParams) SetPropertyName(propertyName string) { - o.PropertyName = propertyName -} - -// WithTenant adds the tenant to the objects references delete params -func (o *ObjectsReferencesDeleteParams) WithTenant(tenant *string) *ObjectsReferencesDeleteParams { - o.SetTenant(tenant) - return o -} - -// SetTenant adds the tenant to the objects references delete params -func (o *ObjectsReferencesDeleteParams) SetTenant(tenant *string) { - o.Tenant = tenant -} - -// WriteToRequest writes these params to a swagger request -func (o *ObjectsReferencesDeleteParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - if o.Body != nil { - if err := r.SetBodyParam(o.Body); err != nil { - return err - } - } - - // path param id - if err := r.SetPathParam("id", o.ID.String()); err != nil { - return err - } - - // path param propertyName - if err := r.SetPathParam("propertyName", o.PropertyName); err != nil { - return err - } - - if o.Tenant != nil { - - // query param tenant - var qrTenant string - - if o.Tenant != nil { - qrTenant = *o.Tenant - } - qTenant := qrTenant - if qTenant != "" { - - if err := r.SetQueryParam("tenant", qTenant); err != nil { - return err - } - } - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/objects/objects_references_delete_responses.go b/client/objects/objects_references_delete_responses.go deleted file mode 100644 index bfeb59e60e9725596c3dc750fbac9ed24d117ee4..0000000000000000000000000000000000000000 --- a/client/objects/objects_references_delete_responses.go +++ /dev/null @@ -1,386 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsReferencesDeleteReader is a Reader for the ObjectsReferencesDelete structure. -type ObjectsReferencesDeleteReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *ObjectsReferencesDeleteReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 204: - result := NewObjectsReferencesDeleteNoContent() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 401: - result := NewObjectsReferencesDeleteUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewObjectsReferencesDeleteForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 404: - result := NewObjectsReferencesDeleteNotFound() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewObjectsReferencesDeleteInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewObjectsReferencesDeleteNoContent creates a ObjectsReferencesDeleteNoContent with default headers values -func NewObjectsReferencesDeleteNoContent() *ObjectsReferencesDeleteNoContent { - return &ObjectsReferencesDeleteNoContent{} -} - -/* -ObjectsReferencesDeleteNoContent describes a response with status code 204, with default header values. - -Successfully deleted. -*/ -type ObjectsReferencesDeleteNoContent struct { -} - -// IsSuccess returns true when this objects references delete no content response has a 2xx status code -func (o *ObjectsReferencesDeleteNoContent) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this objects references delete no content response has a 3xx status code -func (o *ObjectsReferencesDeleteNoContent) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects references delete no content response has a 4xx status code -func (o *ObjectsReferencesDeleteNoContent) IsClientError() bool { - return false -} - -// IsServerError returns true when this objects references delete no content response has a 5xx status code -func (o *ObjectsReferencesDeleteNoContent) IsServerError() bool { - return false -} - -// IsCode returns true when this objects references delete no content response a status code equal to that given -func (o *ObjectsReferencesDeleteNoContent) IsCode(code int) bool { - return code == 204 -} - -// Code gets the status code for the objects references delete no content response -func (o *ObjectsReferencesDeleteNoContent) Code() int { - return 204 -} - -func (o *ObjectsReferencesDeleteNoContent) Error() string { - return fmt.Sprintf("[DELETE /objects/{id}/references/{propertyName}][%d] objectsReferencesDeleteNoContent ", 204) -} - -func (o *ObjectsReferencesDeleteNoContent) String() string { - return fmt.Sprintf("[DELETE /objects/{id}/references/{propertyName}][%d] objectsReferencesDeleteNoContent ", 204) -} - -func (o *ObjectsReferencesDeleteNoContent) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsReferencesDeleteUnauthorized creates a ObjectsReferencesDeleteUnauthorized with default headers values -func NewObjectsReferencesDeleteUnauthorized() *ObjectsReferencesDeleteUnauthorized { - return &ObjectsReferencesDeleteUnauthorized{} -} - -/* -ObjectsReferencesDeleteUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type ObjectsReferencesDeleteUnauthorized struct { -} - -// IsSuccess returns true when this objects references delete unauthorized response has a 2xx status code -func (o *ObjectsReferencesDeleteUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects references delete unauthorized response has a 3xx status code -func (o *ObjectsReferencesDeleteUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects references delete unauthorized response has a 4xx status code -func (o *ObjectsReferencesDeleteUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects references delete unauthorized response has a 5xx status code -func (o *ObjectsReferencesDeleteUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this objects references delete unauthorized response a status code equal to that given -func (o *ObjectsReferencesDeleteUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the objects references delete unauthorized response -func (o *ObjectsReferencesDeleteUnauthorized) Code() int { - return 401 -} - -func (o *ObjectsReferencesDeleteUnauthorized) Error() string { - return fmt.Sprintf("[DELETE /objects/{id}/references/{propertyName}][%d] objectsReferencesDeleteUnauthorized ", 401) -} - -func (o *ObjectsReferencesDeleteUnauthorized) String() string { - return fmt.Sprintf("[DELETE /objects/{id}/references/{propertyName}][%d] objectsReferencesDeleteUnauthorized ", 401) -} - -func (o *ObjectsReferencesDeleteUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsReferencesDeleteForbidden creates a ObjectsReferencesDeleteForbidden with default headers values -func NewObjectsReferencesDeleteForbidden() *ObjectsReferencesDeleteForbidden { - return &ObjectsReferencesDeleteForbidden{} -} - -/* -ObjectsReferencesDeleteForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type ObjectsReferencesDeleteForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects references delete forbidden response has a 2xx status code -func (o *ObjectsReferencesDeleteForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects references delete forbidden response has a 3xx status code -func (o *ObjectsReferencesDeleteForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects references delete forbidden response has a 4xx status code -func (o *ObjectsReferencesDeleteForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects references delete forbidden response has a 5xx status code -func (o *ObjectsReferencesDeleteForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this objects references delete forbidden response a status code equal to that given -func (o *ObjectsReferencesDeleteForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the objects references delete forbidden response -func (o *ObjectsReferencesDeleteForbidden) Code() int { - return 403 -} - -func (o *ObjectsReferencesDeleteForbidden) Error() string { - return fmt.Sprintf("[DELETE /objects/{id}/references/{propertyName}][%d] objectsReferencesDeleteForbidden %+v", 403, o.Payload) -} - -func (o *ObjectsReferencesDeleteForbidden) String() string { - return fmt.Sprintf("[DELETE /objects/{id}/references/{propertyName}][%d] objectsReferencesDeleteForbidden %+v", 403, o.Payload) -} - -func (o *ObjectsReferencesDeleteForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsReferencesDeleteForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsReferencesDeleteNotFound creates a ObjectsReferencesDeleteNotFound with default headers values -func NewObjectsReferencesDeleteNotFound() *ObjectsReferencesDeleteNotFound { - return &ObjectsReferencesDeleteNotFound{} -} - -/* -ObjectsReferencesDeleteNotFound describes a response with status code 404, with default header values. - -Successful query result but no resource was found. -*/ -type ObjectsReferencesDeleteNotFound struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects references delete not found response has a 2xx status code -func (o *ObjectsReferencesDeleteNotFound) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects references delete not found response has a 3xx status code -func (o *ObjectsReferencesDeleteNotFound) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects references delete not found response has a 4xx status code -func (o *ObjectsReferencesDeleteNotFound) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects references delete not found response has a 5xx status code -func (o *ObjectsReferencesDeleteNotFound) IsServerError() bool { - return false -} - -// IsCode returns true when this objects references delete not found response a status code equal to that given -func (o *ObjectsReferencesDeleteNotFound) IsCode(code int) bool { - return code == 404 -} - -// Code gets the status code for the objects references delete not found response -func (o *ObjectsReferencesDeleteNotFound) Code() int { - return 404 -} - -func (o *ObjectsReferencesDeleteNotFound) Error() string { - return fmt.Sprintf("[DELETE /objects/{id}/references/{propertyName}][%d] objectsReferencesDeleteNotFound %+v", 404, o.Payload) -} - -func (o *ObjectsReferencesDeleteNotFound) String() string { - return fmt.Sprintf("[DELETE /objects/{id}/references/{propertyName}][%d] objectsReferencesDeleteNotFound %+v", 404, o.Payload) -} - -func (o *ObjectsReferencesDeleteNotFound) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsReferencesDeleteNotFound) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsReferencesDeleteInternalServerError creates a ObjectsReferencesDeleteInternalServerError with default headers values -func NewObjectsReferencesDeleteInternalServerError() *ObjectsReferencesDeleteInternalServerError { - return &ObjectsReferencesDeleteInternalServerError{} -} - -/* -ObjectsReferencesDeleteInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type ObjectsReferencesDeleteInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects references delete internal server error response has a 2xx status code -func (o *ObjectsReferencesDeleteInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects references delete internal server error response has a 3xx status code -func (o *ObjectsReferencesDeleteInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects references delete internal server error response has a 4xx status code -func (o *ObjectsReferencesDeleteInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this objects references delete internal server error response has a 5xx status code -func (o *ObjectsReferencesDeleteInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this objects references delete internal server error response a status code equal to that given -func (o *ObjectsReferencesDeleteInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the objects references delete internal server error response -func (o *ObjectsReferencesDeleteInternalServerError) Code() int { - return 500 -} - -func (o *ObjectsReferencesDeleteInternalServerError) Error() string { - return fmt.Sprintf("[DELETE /objects/{id}/references/{propertyName}][%d] objectsReferencesDeleteInternalServerError %+v", 500, o.Payload) -} - -func (o *ObjectsReferencesDeleteInternalServerError) String() string { - return fmt.Sprintf("[DELETE /objects/{id}/references/{propertyName}][%d] objectsReferencesDeleteInternalServerError %+v", 500, o.Payload) -} - -func (o *ObjectsReferencesDeleteInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsReferencesDeleteInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/objects/objects_references_update_parameters.go b/client/objects/objects_references_update_parameters.go deleted file mode 100644 index f69db9f5d02c417d4b6913c466704b2eca689f12..0000000000000000000000000000000000000000 --- a/client/objects/objects_references_update_parameters.go +++ /dev/null @@ -1,241 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewObjectsReferencesUpdateParams creates a new ObjectsReferencesUpdateParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewObjectsReferencesUpdateParams() *ObjectsReferencesUpdateParams { - return &ObjectsReferencesUpdateParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewObjectsReferencesUpdateParamsWithTimeout creates a new ObjectsReferencesUpdateParams object -// with the ability to set a timeout on a request. -func NewObjectsReferencesUpdateParamsWithTimeout(timeout time.Duration) *ObjectsReferencesUpdateParams { - return &ObjectsReferencesUpdateParams{ - timeout: timeout, - } -} - -// NewObjectsReferencesUpdateParamsWithContext creates a new ObjectsReferencesUpdateParams object -// with the ability to set a context for a request. -func NewObjectsReferencesUpdateParamsWithContext(ctx context.Context) *ObjectsReferencesUpdateParams { - return &ObjectsReferencesUpdateParams{ - Context: ctx, - } -} - -// NewObjectsReferencesUpdateParamsWithHTTPClient creates a new ObjectsReferencesUpdateParams object -// with the ability to set a custom HTTPClient for a request. -func NewObjectsReferencesUpdateParamsWithHTTPClient(client *http.Client) *ObjectsReferencesUpdateParams { - return &ObjectsReferencesUpdateParams{ - HTTPClient: client, - } -} - -/* -ObjectsReferencesUpdateParams contains all the parameters to send to the API endpoint - - for the objects references update operation. - - Typically these are written to a http.Request. -*/ -type ObjectsReferencesUpdateParams struct { - - // Body. - Body models.MultipleRef - - /* ID. - - Unique ID of the Object. - - Format: uuid - */ - ID strfmt.UUID - - /* PropertyName. - - Unique name of the property related to the Object. - */ - PropertyName string - - /* Tenant. - - Specifies the tenant in a request targeting a multi-tenant class - */ - Tenant *string - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the objects references update params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *ObjectsReferencesUpdateParams) WithDefaults() *ObjectsReferencesUpdateParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the objects references update params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *ObjectsReferencesUpdateParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the objects references update params -func (o *ObjectsReferencesUpdateParams) WithTimeout(timeout time.Duration) *ObjectsReferencesUpdateParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the objects references update params -func (o *ObjectsReferencesUpdateParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the objects references update params -func (o *ObjectsReferencesUpdateParams) WithContext(ctx context.Context) *ObjectsReferencesUpdateParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the objects references update params -func (o *ObjectsReferencesUpdateParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the objects references update params -func (o *ObjectsReferencesUpdateParams) WithHTTPClient(client *http.Client) *ObjectsReferencesUpdateParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the objects references update params -func (o *ObjectsReferencesUpdateParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithBody adds the body to the objects references update params -func (o *ObjectsReferencesUpdateParams) WithBody(body models.MultipleRef) *ObjectsReferencesUpdateParams { - o.SetBody(body) - return o -} - -// SetBody adds the body to the objects references update params -func (o *ObjectsReferencesUpdateParams) SetBody(body models.MultipleRef) { - o.Body = body -} - -// WithID adds the id to the objects references update params -func (o *ObjectsReferencesUpdateParams) WithID(id strfmt.UUID) *ObjectsReferencesUpdateParams { - o.SetID(id) - return o -} - -// SetID adds the id to the objects references update params -func (o *ObjectsReferencesUpdateParams) SetID(id strfmt.UUID) { - o.ID = id -} - -// WithPropertyName adds the propertyName to the objects references update params -func (o *ObjectsReferencesUpdateParams) WithPropertyName(propertyName string) *ObjectsReferencesUpdateParams { - o.SetPropertyName(propertyName) - return o -} - -// SetPropertyName adds the propertyName to the objects references update params -func (o *ObjectsReferencesUpdateParams) SetPropertyName(propertyName string) { - o.PropertyName = propertyName -} - -// WithTenant adds the tenant to the objects references update params -func (o *ObjectsReferencesUpdateParams) WithTenant(tenant *string) *ObjectsReferencesUpdateParams { - o.SetTenant(tenant) - return o -} - -// SetTenant adds the tenant to the objects references update params -func (o *ObjectsReferencesUpdateParams) SetTenant(tenant *string) { - o.Tenant = tenant -} - -// WriteToRequest writes these params to a swagger request -func (o *ObjectsReferencesUpdateParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - if o.Body != nil { - if err := r.SetBodyParam(o.Body); err != nil { - return err - } - } - - // path param id - if err := r.SetPathParam("id", o.ID.String()); err != nil { - return err - } - - // path param propertyName - if err := r.SetPathParam("propertyName", o.PropertyName); err != nil { - return err - } - - if o.Tenant != nil { - - // query param tenant - var qrTenant string - - if o.Tenant != nil { - qrTenant = *o.Tenant - } - qTenant := qrTenant - if qTenant != "" { - - if err := r.SetQueryParam("tenant", qTenant); err != nil { - return err - } - } - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/objects/objects_references_update_responses.go b/client/objects/objects_references_update_responses.go deleted file mode 100644 index 9f4c5ed4371c20319633b2c1a1ae9d3460ebc5df..0000000000000000000000000000000000000000 --- a/client/objects/objects_references_update_responses.go +++ /dev/null @@ -1,386 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsReferencesUpdateReader is a Reader for the ObjectsReferencesUpdate structure. -type ObjectsReferencesUpdateReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *ObjectsReferencesUpdateReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 200: - result := NewObjectsReferencesUpdateOK() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 401: - result := NewObjectsReferencesUpdateUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewObjectsReferencesUpdateForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 422: - result := NewObjectsReferencesUpdateUnprocessableEntity() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewObjectsReferencesUpdateInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewObjectsReferencesUpdateOK creates a ObjectsReferencesUpdateOK with default headers values -func NewObjectsReferencesUpdateOK() *ObjectsReferencesUpdateOK { - return &ObjectsReferencesUpdateOK{} -} - -/* -ObjectsReferencesUpdateOK describes a response with status code 200, with default header values. - -Successfully replaced all the references. -*/ -type ObjectsReferencesUpdateOK struct { -} - -// IsSuccess returns true when this objects references update o k response has a 2xx status code -func (o *ObjectsReferencesUpdateOK) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this objects references update o k response has a 3xx status code -func (o *ObjectsReferencesUpdateOK) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects references update o k response has a 4xx status code -func (o *ObjectsReferencesUpdateOK) IsClientError() bool { - return false -} - -// IsServerError returns true when this objects references update o k response has a 5xx status code -func (o *ObjectsReferencesUpdateOK) IsServerError() bool { - return false -} - -// IsCode returns true when this objects references update o k response a status code equal to that given -func (o *ObjectsReferencesUpdateOK) IsCode(code int) bool { - return code == 200 -} - -// Code gets the status code for the objects references update o k response -func (o *ObjectsReferencesUpdateOK) Code() int { - return 200 -} - -func (o *ObjectsReferencesUpdateOK) Error() string { - return fmt.Sprintf("[PUT /objects/{id}/references/{propertyName}][%d] objectsReferencesUpdateOK ", 200) -} - -func (o *ObjectsReferencesUpdateOK) String() string { - return fmt.Sprintf("[PUT /objects/{id}/references/{propertyName}][%d] objectsReferencesUpdateOK ", 200) -} - -func (o *ObjectsReferencesUpdateOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsReferencesUpdateUnauthorized creates a ObjectsReferencesUpdateUnauthorized with default headers values -func NewObjectsReferencesUpdateUnauthorized() *ObjectsReferencesUpdateUnauthorized { - return &ObjectsReferencesUpdateUnauthorized{} -} - -/* -ObjectsReferencesUpdateUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type ObjectsReferencesUpdateUnauthorized struct { -} - -// IsSuccess returns true when this objects references update unauthorized response has a 2xx status code -func (o *ObjectsReferencesUpdateUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects references update unauthorized response has a 3xx status code -func (o *ObjectsReferencesUpdateUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects references update unauthorized response has a 4xx status code -func (o *ObjectsReferencesUpdateUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects references update unauthorized response has a 5xx status code -func (o *ObjectsReferencesUpdateUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this objects references update unauthorized response a status code equal to that given -func (o *ObjectsReferencesUpdateUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the objects references update unauthorized response -func (o *ObjectsReferencesUpdateUnauthorized) Code() int { - return 401 -} - -func (o *ObjectsReferencesUpdateUnauthorized) Error() string { - return fmt.Sprintf("[PUT /objects/{id}/references/{propertyName}][%d] objectsReferencesUpdateUnauthorized ", 401) -} - -func (o *ObjectsReferencesUpdateUnauthorized) String() string { - return fmt.Sprintf("[PUT /objects/{id}/references/{propertyName}][%d] objectsReferencesUpdateUnauthorized ", 401) -} - -func (o *ObjectsReferencesUpdateUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsReferencesUpdateForbidden creates a ObjectsReferencesUpdateForbidden with default headers values -func NewObjectsReferencesUpdateForbidden() *ObjectsReferencesUpdateForbidden { - return &ObjectsReferencesUpdateForbidden{} -} - -/* -ObjectsReferencesUpdateForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type ObjectsReferencesUpdateForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects references update forbidden response has a 2xx status code -func (o *ObjectsReferencesUpdateForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects references update forbidden response has a 3xx status code -func (o *ObjectsReferencesUpdateForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects references update forbidden response has a 4xx status code -func (o *ObjectsReferencesUpdateForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects references update forbidden response has a 5xx status code -func (o *ObjectsReferencesUpdateForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this objects references update forbidden response a status code equal to that given -func (o *ObjectsReferencesUpdateForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the objects references update forbidden response -func (o *ObjectsReferencesUpdateForbidden) Code() int { - return 403 -} - -func (o *ObjectsReferencesUpdateForbidden) Error() string { - return fmt.Sprintf("[PUT /objects/{id}/references/{propertyName}][%d] objectsReferencesUpdateForbidden %+v", 403, o.Payload) -} - -func (o *ObjectsReferencesUpdateForbidden) String() string { - return fmt.Sprintf("[PUT /objects/{id}/references/{propertyName}][%d] objectsReferencesUpdateForbidden %+v", 403, o.Payload) -} - -func (o *ObjectsReferencesUpdateForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsReferencesUpdateForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsReferencesUpdateUnprocessableEntity creates a ObjectsReferencesUpdateUnprocessableEntity with default headers values -func NewObjectsReferencesUpdateUnprocessableEntity() *ObjectsReferencesUpdateUnprocessableEntity { - return &ObjectsReferencesUpdateUnprocessableEntity{} -} - -/* -ObjectsReferencesUpdateUnprocessableEntity describes a response with status code 422, with default header values. - -Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the property exists or that it is a class? -*/ -type ObjectsReferencesUpdateUnprocessableEntity struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects references update unprocessable entity response has a 2xx status code -func (o *ObjectsReferencesUpdateUnprocessableEntity) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects references update unprocessable entity response has a 3xx status code -func (o *ObjectsReferencesUpdateUnprocessableEntity) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects references update unprocessable entity response has a 4xx status code -func (o *ObjectsReferencesUpdateUnprocessableEntity) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects references update unprocessable entity response has a 5xx status code -func (o *ObjectsReferencesUpdateUnprocessableEntity) IsServerError() bool { - return false -} - -// IsCode returns true when this objects references update unprocessable entity response a status code equal to that given -func (o *ObjectsReferencesUpdateUnprocessableEntity) IsCode(code int) bool { - return code == 422 -} - -// Code gets the status code for the objects references update unprocessable entity response -func (o *ObjectsReferencesUpdateUnprocessableEntity) Code() int { - return 422 -} - -func (o *ObjectsReferencesUpdateUnprocessableEntity) Error() string { - return fmt.Sprintf("[PUT /objects/{id}/references/{propertyName}][%d] objectsReferencesUpdateUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *ObjectsReferencesUpdateUnprocessableEntity) String() string { - return fmt.Sprintf("[PUT /objects/{id}/references/{propertyName}][%d] objectsReferencesUpdateUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *ObjectsReferencesUpdateUnprocessableEntity) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsReferencesUpdateUnprocessableEntity) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsReferencesUpdateInternalServerError creates a ObjectsReferencesUpdateInternalServerError with default headers values -func NewObjectsReferencesUpdateInternalServerError() *ObjectsReferencesUpdateInternalServerError { - return &ObjectsReferencesUpdateInternalServerError{} -} - -/* -ObjectsReferencesUpdateInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type ObjectsReferencesUpdateInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects references update internal server error response has a 2xx status code -func (o *ObjectsReferencesUpdateInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects references update internal server error response has a 3xx status code -func (o *ObjectsReferencesUpdateInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects references update internal server error response has a 4xx status code -func (o *ObjectsReferencesUpdateInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this objects references update internal server error response has a 5xx status code -func (o *ObjectsReferencesUpdateInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this objects references update internal server error response a status code equal to that given -func (o *ObjectsReferencesUpdateInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the objects references update internal server error response -func (o *ObjectsReferencesUpdateInternalServerError) Code() int { - return 500 -} - -func (o *ObjectsReferencesUpdateInternalServerError) Error() string { - return fmt.Sprintf("[PUT /objects/{id}/references/{propertyName}][%d] objectsReferencesUpdateInternalServerError %+v", 500, o.Payload) -} - -func (o *ObjectsReferencesUpdateInternalServerError) String() string { - return fmt.Sprintf("[PUT /objects/{id}/references/{propertyName}][%d] objectsReferencesUpdateInternalServerError %+v", 500, o.Payload) -} - -func (o *ObjectsReferencesUpdateInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsReferencesUpdateInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/objects/objects_update_parameters.go b/client/objects/objects_update_parameters.go deleted file mode 100644 index fd9d11393a9e13c46b767d2c55573c55b6b97686..0000000000000000000000000000000000000000 --- a/client/objects/objects_update_parameters.go +++ /dev/null @@ -1,219 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewObjectsUpdateParams creates a new ObjectsUpdateParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewObjectsUpdateParams() *ObjectsUpdateParams { - return &ObjectsUpdateParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewObjectsUpdateParamsWithTimeout creates a new ObjectsUpdateParams object -// with the ability to set a timeout on a request. -func NewObjectsUpdateParamsWithTimeout(timeout time.Duration) *ObjectsUpdateParams { - return &ObjectsUpdateParams{ - timeout: timeout, - } -} - -// NewObjectsUpdateParamsWithContext creates a new ObjectsUpdateParams object -// with the ability to set a context for a request. -func NewObjectsUpdateParamsWithContext(ctx context.Context) *ObjectsUpdateParams { - return &ObjectsUpdateParams{ - Context: ctx, - } -} - -// NewObjectsUpdateParamsWithHTTPClient creates a new ObjectsUpdateParams object -// with the ability to set a custom HTTPClient for a request. -func NewObjectsUpdateParamsWithHTTPClient(client *http.Client) *ObjectsUpdateParams { - return &ObjectsUpdateParams{ - HTTPClient: client, - } -} - -/* -ObjectsUpdateParams contains all the parameters to send to the API endpoint - - for the objects update operation. - - Typically these are written to a http.Request. -*/ -type ObjectsUpdateParams struct { - - // Body. - Body *models.Object - - /* ConsistencyLevel. - - Determines how many replicas must acknowledge a request before it is considered successful - */ - ConsistencyLevel *string - - /* ID. - - Unique ID of the Object. - - Format: uuid - */ - ID strfmt.UUID - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the objects update params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *ObjectsUpdateParams) WithDefaults() *ObjectsUpdateParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the objects update params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *ObjectsUpdateParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the objects update params -func (o *ObjectsUpdateParams) WithTimeout(timeout time.Duration) *ObjectsUpdateParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the objects update params -func (o *ObjectsUpdateParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the objects update params -func (o *ObjectsUpdateParams) WithContext(ctx context.Context) *ObjectsUpdateParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the objects update params -func (o *ObjectsUpdateParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the objects update params -func (o *ObjectsUpdateParams) WithHTTPClient(client *http.Client) *ObjectsUpdateParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the objects update params -func (o *ObjectsUpdateParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithBody adds the body to the objects update params -func (o *ObjectsUpdateParams) WithBody(body *models.Object) *ObjectsUpdateParams { - o.SetBody(body) - return o -} - -// SetBody adds the body to the objects update params -func (o *ObjectsUpdateParams) SetBody(body *models.Object) { - o.Body = body -} - -// WithConsistencyLevel adds the consistencyLevel to the objects update params -func (o *ObjectsUpdateParams) WithConsistencyLevel(consistencyLevel *string) *ObjectsUpdateParams { - o.SetConsistencyLevel(consistencyLevel) - return o -} - -// SetConsistencyLevel adds the consistencyLevel to the objects update params -func (o *ObjectsUpdateParams) SetConsistencyLevel(consistencyLevel *string) { - o.ConsistencyLevel = consistencyLevel -} - -// WithID adds the id to the objects update params -func (o *ObjectsUpdateParams) WithID(id strfmt.UUID) *ObjectsUpdateParams { - o.SetID(id) - return o -} - -// SetID adds the id to the objects update params -func (o *ObjectsUpdateParams) SetID(id strfmt.UUID) { - o.ID = id -} - -// WriteToRequest writes these params to a swagger request -func (o *ObjectsUpdateParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - if o.Body != nil { - if err := r.SetBodyParam(o.Body); err != nil { - return err - } - } - - if o.ConsistencyLevel != nil { - - // query param consistency_level - var qrConsistencyLevel string - - if o.ConsistencyLevel != nil { - qrConsistencyLevel = *o.ConsistencyLevel - } - qConsistencyLevel := qrConsistencyLevel - if qConsistencyLevel != "" { - - if err := r.SetQueryParam("consistency_level", qConsistencyLevel); err != nil { - return err - } - } - } - - // path param id - if err := r.SetPathParam("id", o.ID.String()); err != nil { - return err - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/objects/objects_update_responses.go b/client/objects/objects_update_responses.go deleted file mode 100644 index c9375d3f4753bf621d75894eb62f0f83e688e1ae..0000000000000000000000000000000000000000 --- a/client/objects/objects_update_responses.go +++ /dev/null @@ -1,460 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsUpdateReader is a Reader for the ObjectsUpdate structure. -type ObjectsUpdateReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *ObjectsUpdateReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 200: - result := NewObjectsUpdateOK() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 401: - result := NewObjectsUpdateUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewObjectsUpdateForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 404: - result := NewObjectsUpdateNotFound() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 422: - result := NewObjectsUpdateUnprocessableEntity() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewObjectsUpdateInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewObjectsUpdateOK creates a ObjectsUpdateOK with default headers values -func NewObjectsUpdateOK() *ObjectsUpdateOK { - return &ObjectsUpdateOK{} -} - -/* -ObjectsUpdateOK describes a response with status code 200, with default header values. - -Successfully received. -*/ -type ObjectsUpdateOK struct { - Payload *models.Object -} - -// IsSuccess returns true when this objects update o k response has a 2xx status code -func (o *ObjectsUpdateOK) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this objects update o k response has a 3xx status code -func (o *ObjectsUpdateOK) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects update o k response has a 4xx status code -func (o *ObjectsUpdateOK) IsClientError() bool { - return false -} - -// IsServerError returns true when this objects update o k response has a 5xx status code -func (o *ObjectsUpdateOK) IsServerError() bool { - return false -} - -// IsCode returns true when this objects update o k response a status code equal to that given -func (o *ObjectsUpdateOK) IsCode(code int) bool { - return code == 200 -} - -// Code gets the status code for the objects update o k response -func (o *ObjectsUpdateOK) Code() int { - return 200 -} - -func (o *ObjectsUpdateOK) Error() string { - return fmt.Sprintf("[PUT /objects/{id}][%d] objectsUpdateOK %+v", 200, o.Payload) -} - -func (o *ObjectsUpdateOK) String() string { - return fmt.Sprintf("[PUT /objects/{id}][%d] objectsUpdateOK %+v", 200, o.Payload) -} - -func (o *ObjectsUpdateOK) GetPayload() *models.Object { - return o.Payload -} - -func (o *ObjectsUpdateOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.Object) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsUpdateUnauthorized creates a ObjectsUpdateUnauthorized with default headers values -func NewObjectsUpdateUnauthorized() *ObjectsUpdateUnauthorized { - return &ObjectsUpdateUnauthorized{} -} - -/* -ObjectsUpdateUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type ObjectsUpdateUnauthorized struct { -} - -// IsSuccess returns true when this objects update unauthorized response has a 2xx status code -func (o *ObjectsUpdateUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects update unauthorized response has a 3xx status code -func (o *ObjectsUpdateUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects update unauthorized response has a 4xx status code -func (o *ObjectsUpdateUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects update unauthorized response has a 5xx status code -func (o *ObjectsUpdateUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this objects update unauthorized response a status code equal to that given -func (o *ObjectsUpdateUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the objects update unauthorized response -func (o *ObjectsUpdateUnauthorized) Code() int { - return 401 -} - -func (o *ObjectsUpdateUnauthorized) Error() string { - return fmt.Sprintf("[PUT /objects/{id}][%d] objectsUpdateUnauthorized ", 401) -} - -func (o *ObjectsUpdateUnauthorized) String() string { - return fmt.Sprintf("[PUT /objects/{id}][%d] objectsUpdateUnauthorized ", 401) -} - -func (o *ObjectsUpdateUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsUpdateForbidden creates a ObjectsUpdateForbidden with default headers values -func NewObjectsUpdateForbidden() *ObjectsUpdateForbidden { - return &ObjectsUpdateForbidden{} -} - -/* -ObjectsUpdateForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type ObjectsUpdateForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects update forbidden response has a 2xx status code -func (o *ObjectsUpdateForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects update forbidden response has a 3xx status code -func (o *ObjectsUpdateForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects update forbidden response has a 4xx status code -func (o *ObjectsUpdateForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects update forbidden response has a 5xx status code -func (o *ObjectsUpdateForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this objects update forbidden response a status code equal to that given -func (o *ObjectsUpdateForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the objects update forbidden response -func (o *ObjectsUpdateForbidden) Code() int { - return 403 -} - -func (o *ObjectsUpdateForbidden) Error() string { - return fmt.Sprintf("[PUT /objects/{id}][%d] objectsUpdateForbidden %+v", 403, o.Payload) -} - -func (o *ObjectsUpdateForbidden) String() string { - return fmt.Sprintf("[PUT /objects/{id}][%d] objectsUpdateForbidden %+v", 403, o.Payload) -} - -func (o *ObjectsUpdateForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsUpdateForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsUpdateNotFound creates a ObjectsUpdateNotFound with default headers values -func NewObjectsUpdateNotFound() *ObjectsUpdateNotFound { - return &ObjectsUpdateNotFound{} -} - -/* -ObjectsUpdateNotFound describes a response with status code 404, with default header values. - -Successful query result but no resource was found. -*/ -type ObjectsUpdateNotFound struct { -} - -// IsSuccess returns true when this objects update not found response has a 2xx status code -func (o *ObjectsUpdateNotFound) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects update not found response has a 3xx status code -func (o *ObjectsUpdateNotFound) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects update not found response has a 4xx status code -func (o *ObjectsUpdateNotFound) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects update not found response has a 5xx status code -func (o *ObjectsUpdateNotFound) IsServerError() bool { - return false -} - -// IsCode returns true when this objects update not found response a status code equal to that given -func (o *ObjectsUpdateNotFound) IsCode(code int) bool { - return code == 404 -} - -// Code gets the status code for the objects update not found response -func (o *ObjectsUpdateNotFound) Code() int { - return 404 -} - -func (o *ObjectsUpdateNotFound) Error() string { - return fmt.Sprintf("[PUT /objects/{id}][%d] objectsUpdateNotFound ", 404) -} - -func (o *ObjectsUpdateNotFound) String() string { - return fmt.Sprintf("[PUT /objects/{id}][%d] objectsUpdateNotFound ", 404) -} - -func (o *ObjectsUpdateNotFound) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsUpdateUnprocessableEntity creates a ObjectsUpdateUnprocessableEntity with default headers values -func NewObjectsUpdateUnprocessableEntity() *ObjectsUpdateUnprocessableEntity { - return &ObjectsUpdateUnprocessableEntity{} -} - -/* -ObjectsUpdateUnprocessableEntity describes a response with status code 422, with default header values. - -Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file? -*/ -type ObjectsUpdateUnprocessableEntity struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects update unprocessable entity response has a 2xx status code -func (o *ObjectsUpdateUnprocessableEntity) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects update unprocessable entity response has a 3xx status code -func (o *ObjectsUpdateUnprocessableEntity) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects update unprocessable entity response has a 4xx status code -func (o *ObjectsUpdateUnprocessableEntity) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects update unprocessable entity response has a 5xx status code -func (o *ObjectsUpdateUnprocessableEntity) IsServerError() bool { - return false -} - -// IsCode returns true when this objects update unprocessable entity response a status code equal to that given -func (o *ObjectsUpdateUnprocessableEntity) IsCode(code int) bool { - return code == 422 -} - -// Code gets the status code for the objects update unprocessable entity response -func (o *ObjectsUpdateUnprocessableEntity) Code() int { - return 422 -} - -func (o *ObjectsUpdateUnprocessableEntity) Error() string { - return fmt.Sprintf("[PUT /objects/{id}][%d] objectsUpdateUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *ObjectsUpdateUnprocessableEntity) String() string { - return fmt.Sprintf("[PUT /objects/{id}][%d] objectsUpdateUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *ObjectsUpdateUnprocessableEntity) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsUpdateUnprocessableEntity) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsUpdateInternalServerError creates a ObjectsUpdateInternalServerError with default headers values -func NewObjectsUpdateInternalServerError() *ObjectsUpdateInternalServerError { - return &ObjectsUpdateInternalServerError{} -} - -/* -ObjectsUpdateInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type ObjectsUpdateInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects update internal server error response has a 2xx status code -func (o *ObjectsUpdateInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects update internal server error response has a 3xx status code -func (o *ObjectsUpdateInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects update internal server error response has a 4xx status code -func (o *ObjectsUpdateInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this objects update internal server error response has a 5xx status code -func (o *ObjectsUpdateInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this objects update internal server error response a status code equal to that given -func (o *ObjectsUpdateInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the objects update internal server error response -func (o *ObjectsUpdateInternalServerError) Code() int { - return 500 -} - -func (o *ObjectsUpdateInternalServerError) Error() string { - return fmt.Sprintf("[PUT /objects/{id}][%d] objectsUpdateInternalServerError %+v", 500, o.Payload) -} - -func (o *ObjectsUpdateInternalServerError) String() string { - return fmt.Sprintf("[PUT /objects/{id}][%d] objectsUpdateInternalServerError %+v", 500, o.Payload) -} - -func (o *ObjectsUpdateInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsUpdateInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/objects/objects_validate_parameters.go b/client/objects/objects_validate_parameters.go deleted file mode 100644 index 59b874da5763eb40ed25999d6813d9ca8ebe71d0..0000000000000000000000000000000000000000 --- a/client/objects/objects_validate_parameters.go +++ /dev/null @@ -1,161 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewObjectsValidateParams creates a new ObjectsValidateParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewObjectsValidateParams() *ObjectsValidateParams { - return &ObjectsValidateParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewObjectsValidateParamsWithTimeout creates a new ObjectsValidateParams object -// with the ability to set a timeout on a request. -func NewObjectsValidateParamsWithTimeout(timeout time.Duration) *ObjectsValidateParams { - return &ObjectsValidateParams{ - timeout: timeout, - } -} - -// NewObjectsValidateParamsWithContext creates a new ObjectsValidateParams object -// with the ability to set a context for a request. -func NewObjectsValidateParamsWithContext(ctx context.Context) *ObjectsValidateParams { - return &ObjectsValidateParams{ - Context: ctx, - } -} - -// NewObjectsValidateParamsWithHTTPClient creates a new ObjectsValidateParams object -// with the ability to set a custom HTTPClient for a request. -func NewObjectsValidateParamsWithHTTPClient(client *http.Client) *ObjectsValidateParams { - return &ObjectsValidateParams{ - HTTPClient: client, - } -} - -/* -ObjectsValidateParams contains all the parameters to send to the API endpoint - - for the objects validate operation. - - Typically these are written to a http.Request. -*/ -type ObjectsValidateParams struct { - - // Body. - Body *models.Object - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the objects validate params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *ObjectsValidateParams) WithDefaults() *ObjectsValidateParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the objects validate params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *ObjectsValidateParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the objects validate params -func (o *ObjectsValidateParams) WithTimeout(timeout time.Duration) *ObjectsValidateParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the objects validate params -func (o *ObjectsValidateParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the objects validate params -func (o *ObjectsValidateParams) WithContext(ctx context.Context) *ObjectsValidateParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the objects validate params -func (o *ObjectsValidateParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the objects validate params -func (o *ObjectsValidateParams) WithHTTPClient(client *http.Client) *ObjectsValidateParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the objects validate params -func (o *ObjectsValidateParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithBody adds the body to the objects validate params -func (o *ObjectsValidateParams) WithBody(body *models.Object) *ObjectsValidateParams { - o.SetBody(body) - return o -} - -// SetBody adds the body to the objects validate params -func (o *ObjectsValidateParams) SetBody(body *models.Object) { - o.Body = body -} - -// WriteToRequest writes these params to a swagger request -func (o *ObjectsValidateParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - if o.Body != nil { - if err := r.SetBodyParam(o.Body); err != nil { - return err - } - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/objects/objects_validate_responses.go b/client/objects/objects_validate_responses.go deleted file mode 100644 index 0d0802a15a0d55fa4eb4d43a32c1e5ac0ebfd226..0000000000000000000000000000000000000000 --- a/client/objects/objects_validate_responses.go +++ /dev/null @@ -1,386 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package objects - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// ObjectsValidateReader is a Reader for the ObjectsValidate structure. -type ObjectsValidateReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *ObjectsValidateReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 200: - result := NewObjectsValidateOK() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 401: - result := NewObjectsValidateUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewObjectsValidateForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 422: - result := NewObjectsValidateUnprocessableEntity() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewObjectsValidateInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewObjectsValidateOK creates a ObjectsValidateOK with default headers values -func NewObjectsValidateOK() *ObjectsValidateOK { - return &ObjectsValidateOK{} -} - -/* -ObjectsValidateOK describes a response with status code 200, with default header values. - -Successfully validated. -*/ -type ObjectsValidateOK struct { -} - -// IsSuccess returns true when this objects validate o k response has a 2xx status code -func (o *ObjectsValidateOK) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this objects validate o k response has a 3xx status code -func (o *ObjectsValidateOK) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects validate o k response has a 4xx status code -func (o *ObjectsValidateOK) IsClientError() bool { - return false -} - -// IsServerError returns true when this objects validate o k response has a 5xx status code -func (o *ObjectsValidateOK) IsServerError() bool { - return false -} - -// IsCode returns true when this objects validate o k response a status code equal to that given -func (o *ObjectsValidateOK) IsCode(code int) bool { - return code == 200 -} - -// Code gets the status code for the objects validate o k response -func (o *ObjectsValidateOK) Code() int { - return 200 -} - -func (o *ObjectsValidateOK) Error() string { - return fmt.Sprintf("[POST /objects/validate][%d] objectsValidateOK ", 200) -} - -func (o *ObjectsValidateOK) String() string { - return fmt.Sprintf("[POST /objects/validate][%d] objectsValidateOK ", 200) -} - -func (o *ObjectsValidateOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsValidateUnauthorized creates a ObjectsValidateUnauthorized with default headers values -func NewObjectsValidateUnauthorized() *ObjectsValidateUnauthorized { - return &ObjectsValidateUnauthorized{} -} - -/* -ObjectsValidateUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type ObjectsValidateUnauthorized struct { -} - -// IsSuccess returns true when this objects validate unauthorized response has a 2xx status code -func (o *ObjectsValidateUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects validate unauthorized response has a 3xx status code -func (o *ObjectsValidateUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects validate unauthorized response has a 4xx status code -func (o *ObjectsValidateUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects validate unauthorized response has a 5xx status code -func (o *ObjectsValidateUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this objects validate unauthorized response a status code equal to that given -func (o *ObjectsValidateUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the objects validate unauthorized response -func (o *ObjectsValidateUnauthorized) Code() int { - return 401 -} - -func (o *ObjectsValidateUnauthorized) Error() string { - return fmt.Sprintf("[POST /objects/validate][%d] objectsValidateUnauthorized ", 401) -} - -func (o *ObjectsValidateUnauthorized) String() string { - return fmt.Sprintf("[POST /objects/validate][%d] objectsValidateUnauthorized ", 401) -} - -func (o *ObjectsValidateUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewObjectsValidateForbidden creates a ObjectsValidateForbidden with default headers values -func NewObjectsValidateForbidden() *ObjectsValidateForbidden { - return &ObjectsValidateForbidden{} -} - -/* -ObjectsValidateForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type ObjectsValidateForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects validate forbidden response has a 2xx status code -func (o *ObjectsValidateForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects validate forbidden response has a 3xx status code -func (o *ObjectsValidateForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects validate forbidden response has a 4xx status code -func (o *ObjectsValidateForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects validate forbidden response has a 5xx status code -func (o *ObjectsValidateForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this objects validate forbidden response a status code equal to that given -func (o *ObjectsValidateForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the objects validate forbidden response -func (o *ObjectsValidateForbidden) Code() int { - return 403 -} - -func (o *ObjectsValidateForbidden) Error() string { - return fmt.Sprintf("[POST /objects/validate][%d] objectsValidateForbidden %+v", 403, o.Payload) -} - -func (o *ObjectsValidateForbidden) String() string { - return fmt.Sprintf("[POST /objects/validate][%d] objectsValidateForbidden %+v", 403, o.Payload) -} - -func (o *ObjectsValidateForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsValidateForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsValidateUnprocessableEntity creates a ObjectsValidateUnprocessableEntity with default headers values -func NewObjectsValidateUnprocessableEntity() *ObjectsValidateUnprocessableEntity { - return &ObjectsValidateUnprocessableEntity{} -} - -/* -ObjectsValidateUnprocessableEntity describes a response with status code 422, with default header values. - -Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file? -*/ -type ObjectsValidateUnprocessableEntity struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects validate unprocessable entity response has a 2xx status code -func (o *ObjectsValidateUnprocessableEntity) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects validate unprocessable entity response has a 3xx status code -func (o *ObjectsValidateUnprocessableEntity) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects validate unprocessable entity response has a 4xx status code -func (o *ObjectsValidateUnprocessableEntity) IsClientError() bool { - return true -} - -// IsServerError returns true when this objects validate unprocessable entity response has a 5xx status code -func (o *ObjectsValidateUnprocessableEntity) IsServerError() bool { - return false -} - -// IsCode returns true when this objects validate unprocessable entity response a status code equal to that given -func (o *ObjectsValidateUnprocessableEntity) IsCode(code int) bool { - return code == 422 -} - -// Code gets the status code for the objects validate unprocessable entity response -func (o *ObjectsValidateUnprocessableEntity) Code() int { - return 422 -} - -func (o *ObjectsValidateUnprocessableEntity) Error() string { - return fmt.Sprintf("[POST /objects/validate][%d] objectsValidateUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *ObjectsValidateUnprocessableEntity) String() string { - return fmt.Sprintf("[POST /objects/validate][%d] objectsValidateUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *ObjectsValidateUnprocessableEntity) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsValidateUnprocessableEntity) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewObjectsValidateInternalServerError creates a ObjectsValidateInternalServerError with default headers values -func NewObjectsValidateInternalServerError() *ObjectsValidateInternalServerError { - return &ObjectsValidateInternalServerError{} -} - -/* -ObjectsValidateInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type ObjectsValidateInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this objects validate internal server error response has a 2xx status code -func (o *ObjectsValidateInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this objects validate internal server error response has a 3xx status code -func (o *ObjectsValidateInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this objects validate internal server error response has a 4xx status code -func (o *ObjectsValidateInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this objects validate internal server error response has a 5xx status code -func (o *ObjectsValidateInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this objects validate internal server error response a status code equal to that given -func (o *ObjectsValidateInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the objects validate internal server error response -func (o *ObjectsValidateInternalServerError) Code() int { - return 500 -} - -func (o *ObjectsValidateInternalServerError) Error() string { - return fmt.Sprintf("[POST /objects/validate][%d] objectsValidateInternalServerError %+v", 500, o.Payload) -} - -func (o *ObjectsValidateInternalServerError) String() string { - return fmt.Sprintf("[POST /objects/validate][%d] objectsValidateInternalServerError %+v", 500, o.Payload) -} - -func (o *ObjectsValidateInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *ObjectsValidateInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/operations/operations_client.go b/client/operations/operations_client.go deleted file mode 100644 index 2e6997e11b4fb88db3a713e6311be4887d943a9b..0000000000000000000000000000000000000000 --- a/client/operations/operations_client.go +++ /dev/null @@ -1,173 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package operations - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" -) - -// New creates a new operations API client. -func New(transport runtime.ClientTransport, formats strfmt.Registry) ClientService { - return &Client{transport: transport, formats: formats} -} - -/* -Client for operations API -*/ -type Client struct { - transport runtime.ClientTransport - formats strfmt.Registry -} - -// ClientOption is the option for Client methods -type ClientOption func(*runtime.ClientOperation) - -// ClientService is the interface for Client methods -type ClientService interface { - WeaviateRoot(params *WeaviateRootParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*WeaviateRootOK, error) - - WeaviateWellknownLiveness(params *WeaviateWellknownLivenessParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*WeaviateWellknownLivenessOK, error) - - WeaviateWellknownReadiness(params *WeaviateWellknownReadinessParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*WeaviateWellknownReadinessOK, error) - - SetTransport(transport runtime.ClientTransport) -} - -/* -WeaviateRoot Home. Discover the REST API -*/ -func (a *Client) WeaviateRoot(params *WeaviateRootParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*WeaviateRootOK, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewWeaviateRootParams() - } - op := &runtime.ClientOperation{ - ID: "weaviate.root", - Method: "GET", - PathPattern: "/", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &WeaviateRootReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*WeaviateRootOK) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for weaviate.root: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -/* -WeaviateWellknownLiveness Determines whether the application is alive. Can be used for kubernetes liveness probe -*/ -func (a *Client) WeaviateWellknownLiveness(params *WeaviateWellknownLivenessParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*WeaviateWellknownLivenessOK, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewWeaviateWellknownLivenessParams() - } - op := &runtime.ClientOperation{ - ID: "weaviate.wellknown.liveness", - Method: "GET", - PathPattern: "/.well-known/live", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &WeaviateWellknownLivenessReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*WeaviateWellknownLivenessOK) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for weaviate.wellknown.liveness: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -/* -WeaviateWellknownReadiness Determines whether the application is ready to receive traffic. Can be used for kubernetes readiness probe. -*/ -func (a *Client) WeaviateWellknownReadiness(params *WeaviateWellknownReadinessParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*WeaviateWellknownReadinessOK, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewWeaviateWellknownReadinessParams() - } - op := &runtime.ClientOperation{ - ID: "weaviate.wellknown.readiness", - Method: "GET", - PathPattern: "/.well-known/ready", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &WeaviateWellknownReadinessReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*WeaviateWellknownReadinessOK) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for weaviate.wellknown.readiness: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -// SetTransport changes the transport on the client -func (a *Client) SetTransport(transport runtime.ClientTransport) { - a.transport = transport -} diff --git a/client/operations/weaviate_root_parameters.go b/client/operations/weaviate_root_parameters.go deleted file mode 100644 index 4613cb16cc3a821c2e1d6011c40d9eaae30e1015..0000000000000000000000000000000000000000 --- a/client/operations/weaviate_root_parameters.go +++ /dev/null @@ -1,139 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package operations - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" -) - -// NewWeaviateRootParams creates a new WeaviateRootParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewWeaviateRootParams() *WeaviateRootParams { - return &WeaviateRootParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewWeaviateRootParamsWithTimeout creates a new WeaviateRootParams object -// with the ability to set a timeout on a request. -func NewWeaviateRootParamsWithTimeout(timeout time.Duration) *WeaviateRootParams { - return &WeaviateRootParams{ - timeout: timeout, - } -} - -// NewWeaviateRootParamsWithContext creates a new WeaviateRootParams object -// with the ability to set a context for a request. -func NewWeaviateRootParamsWithContext(ctx context.Context) *WeaviateRootParams { - return &WeaviateRootParams{ - Context: ctx, - } -} - -// NewWeaviateRootParamsWithHTTPClient creates a new WeaviateRootParams object -// with the ability to set a custom HTTPClient for a request. -func NewWeaviateRootParamsWithHTTPClient(client *http.Client) *WeaviateRootParams { - return &WeaviateRootParams{ - HTTPClient: client, - } -} - -/* -WeaviateRootParams contains all the parameters to send to the API endpoint - - for the weaviate root operation. - - Typically these are written to a http.Request. -*/ -type WeaviateRootParams struct { - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the weaviate root params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *WeaviateRootParams) WithDefaults() *WeaviateRootParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the weaviate root params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *WeaviateRootParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the weaviate root params -func (o *WeaviateRootParams) WithTimeout(timeout time.Duration) *WeaviateRootParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the weaviate root params -func (o *WeaviateRootParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the weaviate root params -func (o *WeaviateRootParams) WithContext(ctx context.Context) *WeaviateRootParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the weaviate root params -func (o *WeaviateRootParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the weaviate root params -func (o *WeaviateRootParams) WithHTTPClient(client *http.Client) *WeaviateRootParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the weaviate root params -func (o *WeaviateRootParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WriteToRequest writes these params to a swagger request -func (o *WeaviateRootParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/operations/weaviate_root_responses.go b/client/operations/weaviate_root_responses.go deleted file mode 100644 index fafdaaa5fd2191a854ee6a2b7dbcf642aca77920..0000000000000000000000000000000000000000 --- a/client/operations/weaviate_root_responses.go +++ /dev/null @@ -1,220 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package operations - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "fmt" - "io" - "strconv" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" - - "github.com/weaviate/weaviate/entities/models" -) - -// WeaviateRootReader is a Reader for the WeaviateRoot structure. -type WeaviateRootReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *WeaviateRootReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 200: - result := NewWeaviateRootOK() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewWeaviateRootOK creates a WeaviateRootOK with default headers values -func NewWeaviateRootOK() *WeaviateRootOK { - return &WeaviateRootOK{} -} - -/* -WeaviateRootOK describes a response with status code 200, with default header values. - -Weaviate is alive and ready to serve content -*/ -type WeaviateRootOK struct { - Payload *WeaviateRootOKBody -} - -// IsSuccess returns true when this weaviate root o k response has a 2xx status code -func (o *WeaviateRootOK) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this weaviate root o k response has a 3xx status code -func (o *WeaviateRootOK) IsRedirect() bool { - return false -} - -// IsClientError returns true when this weaviate root o k response has a 4xx status code -func (o *WeaviateRootOK) IsClientError() bool { - return false -} - -// IsServerError returns true when this weaviate root o k response has a 5xx status code -func (o *WeaviateRootOK) IsServerError() bool { - return false -} - -// IsCode returns true when this weaviate root o k response a status code equal to that given -func (o *WeaviateRootOK) IsCode(code int) bool { - return code == 200 -} - -// Code gets the status code for the weaviate root o k response -func (o *WeaviateRootOK) Code() int { - return 200 -} - -func (o *WeaviateRootOK) Error() string { - return fmt.Sprintf("[GET /][%d] weaviateRootOK %+v", 200, o.Payload) -} - -func (o *WeaviateRootOK) String() string { - return fmt.Sprintf("[GET /][%d] weaviateRootOK %+v", 200, o.Payload) -} - -func (o *WeaviateRootOK) GetPayload() *WeaviateRootOKBody { - return o.Payload -} - -func (o *WeaviateRootOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(WeaviateRootOKBody) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -/* -WeaviateRootOKBody weaviate root o k body -swagger:model WeaviateRootOKBody -*/ -type WeaviateRootOKBody struct { - - // links - Links []*models.Link `json:"links"` -} - -// Validate validates this weaviate root o k body -func (o *WeaviateRootOKBody) Validate(formats strfmt.Registry) error { - var res []error - - if err := o.validateLinks(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (o *WeaviateRootOKBody) validateLinks(formats strfmt.Registry) error { - if swag.IsZero(o.Links) { // not required - return nil - } - - for i := 0; i < len(o.Links); i++ { - if swag.IsZero(o.Links[i]) { // not required - continue - } - - if o.Links[i] != nil { - if err := o.Links[i].Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("weaviateRootOK" + "." + "links" + "." + strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("weaviateRootOK" + "." + "links" + "." + strconv.Itoa(i)) - } - return err - } - } - - } - - return nil -} - -// ContextValidate validate this weaviate root o k body based on the context it is used -func (o *WeaviateRootOKBody) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - if err := o.contextValidateLinks(ctx, formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (o *WeaviateRootOKBody) contextValidateLinks(ctx context.Context, formats strfmt.Registry) error { - - for i := 0; i < len(o.Links); i++ { - - if o.Links[i] != nil { - if err := o.Links[i].ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("weaviateRootOK" + "." + "links" + "." + strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("weaviateRootOK" + "." + "links" + "." + strconv.Itoa(i)) - } - return err - } - } - - } - - return nil -} - -// MarshalBinary interface implementation -func (o *WeaviateRootOKBody) MarshalBinary() ([]byte, error) { - if o == nil { - return nil, nil - } - return swag.WriteJSON(o) -} - -// UnmarshalBinary interface implementation -func (o *WeaviateRootOKBody) UnmarshalBinary(b []byte) error { - var res WeaviateRootOKBody - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *o = res - return nil -} diff --git a/client/operations/weaviate_wellknown_liveness_parameters.go b/client/operations/weaviate_wellknown_liveness_parameters.go deleted file mode 100644 index e3d32e6ae04ca19aadb7adc589f7699a840e1abe..0000000000000000000000000000000000000000 --- a/client/operations/weaviate_wellknown_liveness_parameters.go +++ /dev/null @@ -1,139 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package operations - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" -) - -// NewWeaviateWellknownLivenessParams creates a new WeaviateWellknownLivenessParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewWeaviateWellknownLivenessParams() *WeaviateWellknownLivenessParams { - return &WeaviateWellknownLivenessParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewWeaviateWellknownLivenessParamsWithTimeout creates a new WeaviateWellknownLivenessParams object -// with the ability to set a timeout on a request. -func NewWeaviateWellknownLivenessParamsWithTimeout(timeout time.Duration) *WeaviateWellknownLivenessParams { - return &WeaviateWellknownLivenessParams{ - timeout: timeout, - } -} - -// NewWeaviateWellknownLivenessParamsWithContext creates a new WeaviateWellknownLivenessParams object -// with the ability to set a context for a request. -func NewWeaviateWellknownLivenessParamsWithContext(ctx context.Context) *WeaviateWellknownLivenessParams { - return &WeaviateWellknownLivenessParams{ - Context: ctx, - } -} - -// NewWeaviateWellknownLivenessParamsWithHTTPClient creates a new WeaviateWellknownLivenessParams object -// with the ability to set a custom HTTPClient for a request. -func NewWeaviateWellknownLivenessParamsWithHTTPClient(client *http.Client) *WeaviateWellknownLivenessParams { - return &WeaviateWellknownLivenessParams{ - HTTPClient: client, - } -} - -/* -WeaviateWellknownLivenessParams contains all the parameters to send to the API endpoint - - for the weaviate wellknown liveness operation. - - Typically these are written to a http.Request. -*/ -type WeaviateWellknownLivenessParams struct { - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the weaviate wellknown liveness params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *WeaviateWellknownLivenessParams) WithDefaults() *WeaviateWellknownLivenessParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the weaviate wellknown liveness params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *WeaviateWellknownLivenessParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the weaviate wellknown liveness params -func (o *WeaviateWellknownLivenessParams) WithTimeout(timeout time.Duration) *WeaviateWellknownLivenessParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the weaviate wellknown liveness params -func (o *WeaviateWellknownLivenessParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the weaviate wellknown liveness params -func (o *WeaviateWellknownLivenessParams) WithContext(ctx context.Context) *WeaviateWellknownLivenessParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the weaviate wellknown liveness params -func (o *WeaviateWellknownLivenessParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the weaviate wellknown liveness params -func (o *WeaviateWellknownLivenessParams) WithHTTPClient(client *http.Client) *WeaviateWellknownLivenessParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the weaviate wellknown liveness params -func (o *WeaviateWellknownLivenessParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WriteToRequest writes these params to a swagger request -func (o *WeaviateWellknownLivenessParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/operations/weaviate_wellknown_liveness_responses.go b/client/operations/weaviate_wellknown_liveness_responses.go deleted file mode 100644 index d8621a26ee26f88a87eee9f5838878cf4264e34a..0000000000000000000000000000000000000000 --- a/client/operations/weaviate_wellknown_liveness_responses.go +++ /dev/null @@ -1,99 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package operations - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" -) - -// WeaviateWellknownLivenessReader is a Reader for the WeaviateWellknownLiveness structure. -type WeaviateWellknownLivenessReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *WeaviateWellknownLivenessReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 200: - result := NewWeaviateWellknownLivenessOK() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewWeaviateWellknownLivenessOK creates a WeaviateWellknownLivenessOK with default headers values -func NewWeaviateWellknownLivenessOK() *WeaviateWellknownLivenessOK { - return &WeaviateWellknownLivenessOK{} -} - -/* -WeaviateWellknownLivenessOK describes a response with status code 200, with default header values. - -The application is able to respond to HTTP requests -*/ -type WeaviateWellknownLivenessOK struct { -} - -// IsSuccess returns true when this weaviate wellknown liveness o k response has a 2xx status code -func (o *WeaviateWellknownLivenessOK) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this weaviate wellknown liveness o k response has a 3xx status code -func (o *WeaviateWellknownLivenessOK) IsRedirect() bool { - return false -} - -// IsClientError returns true when this weaviate wellknown liveness o k response has a 4xx status code -func (o *WeaviateWellknownLivenessOK) IsClientError() bool { - return false -} - -// IsServerError returns true when this weaviate wellknown liveness o k response has a 5xx status code -func (o *WeaviateWellknownLivenessOK) IsServerError() bool { - return false -} - -// IsCode returns true when this weaviate wellknown liveness o k response a status code equal to that given -func (o *WeaviateWellknownLivenessOK) IsCode(code int) bool { - return code == 200 -} - -// Code gets the status code for the weaviate wellknown liveness o k response -func (o *WeaviateWellknownLivenessOK) Code() int { - return 200 -} - -func (o *WeaviateWellknownLivenessOK) Error() string { - return fmt.Sprintf("[GET /.well-known/live][%d] weaviateWellknownLivenessOK ", 200) -} - -func (o *WeaviateWellknownLivenessOK) String() string { - return fmt.Sprintf("[GET /.well-known/live][%d] weaviateWellknownLivenessOK ", 200) -} - -func (o *WeaviateWellknownLivenessOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} diff --git a/client/operations/weaviate_wellknown_readiness_parameters.go b/client/operations/weaviate_wellknown_readiness_parameters.go deleted file mode 100644 index e3768efe48c46fc74b66071a0b71a3a85328a132..0000000000000000000000000000000000000000 --- a/client/operations/weaviate_wellknown_readiness_parameters.go +++ /dev/null @@ -1,139 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package operations - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" -) - -// NewWeaviateWellknownReadinessParams creates a new WeaviateWellknownReadinessParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewWeaviateWellknownReadinessParams() *WeaviateWellknownReadinessParams { - return &WeaviateWellknownReadinessParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewWeaviateWellknownReadinessParamsWithTimeout creates a new WeaviateWellknownReadinessParams object -// with the ability to set a timeout on a request. -func NewWeaviateWellknownReadinessParamsWithTimeout(timeout time.Duration) *WeaviateWellknownReadinessParams { - return &WeaviateWellknownReadinessParams{ - timeout: timeout, - } -} - -// NewWeaviateWellknownReadinessParamsWithContext creates a new WeaviateWellknownReadinessParams object -// with the ability to set a context for a request. -func NewWeaviateWellknownReadinessParamsWithContext(ctx context.Context) *WeaviateWellknownReadinessParams { - return &WeaviateWellknownReadinessParams{ - Context: ctx, - } -} - -// NewWeaviateWellknownReadinessParamsWithHTTPClient creates a new WeaviateWellknownReadinessParams object -// with the ability to set a custom HTTPClient for a request. -func NewWeaviateWellknownReadinessParamsWithHTTPClient(client *http.Client) *WeaviateWellknownReadinessParams { - return &WeaviateWellknownReadinessParams{ - HTTPClient: client, - } -} - -/* -WeaviateWellknownReadinessParams contains all the parameters to send to the API endpoint - - for the weaviate wellknown readiness operation. - - Typically these are written to a http.Request. -*/ -type WeaviateWellknownReadinessParams struct { - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the weaviate wellknown readiness params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *WeaviateWellknownReadinessParams) WithDefaults() *WeaviateWellknownReadinessParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the weaviate wellknown readiness params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *WeaviateWellknownReadinessParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the weaviate wellknown readiness params -func (o *WeaviateWellknownReadinessParams) WithTimeout(timeout time.Duration) *WeaviateWellknownReadinessParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the weaviate wellknown readiness params -func (o *WeaviateWellknownReadinessParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the weaviate wellknown readiness params -func (o *WeaviateWellknownReadinessParams) WithContext(ctx context.Context) *WeaviateWellknownReadinessParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the weaviate wellknown readiness params -func (o *WeaviateWellknownReadinessParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the weaviate wellknown readiness params -func (o *WeaviateWellknownReadinessParams) WithHTTPClient(client *http.Client) *WeaviateWellknownReadinessParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the weaviate wellknown readiness params -func (o *WeaviateWellknownReadinessParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WriteToRequest writes these params to a swagger request -func (o *WeaviateWellknownReadinessParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/operations/weaviate_wellknown_readiness_responses.go b/client/operations/weaviate_wellknown_readiness_responses.go deleted file mode 100644 index 1b85a43fd262d57a7d354e9bb63959c30a9007ef..0000000000000000000000000000000000000000 --- a/client/operations/weaviate_wellknown_readiness_responses.go +++ /dev/null @@ -1,161 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package operations - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" -) - -// WeaviateWellknownReadinessReader is a Reader for the WeaviateWellknownReadiness structure. -type WeaviateWellknownReadinessReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *WeaviateWellknownReadinessReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 200: - result := NewWeaviateWellknownReadinessOK() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 503: - result := NewWeaviateWellknownReadinessServiceUnavailable() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewWeaviateWellknownReadinessOK creates a WeaviateWellknownReadinessOK with default headers values -func NewWeaviateWellknownReadinessOK() *WeaviateWellknownReadinessOK { - return &WeaviateWellknownReadinessOK{} -} - -/* -WeaviateWellknownReadinessOK describes a response with status code 200, with default header values. - -The application has completed its start-up routine and is ready to accept traffic. -*/ -type WeaviateWellknownReadinessOK struct { -} - -// IsSuccess returns true when this weaviate wellknown readiness o k response has a 2xx status code -func (o *WeaviateWellknownReadinessOK) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this weaviate wellknown readiness o k response has a 3xx status code -func (o *WeaviateWellknownReadinessOK) IsRedirect() bool { - return false -} - -// IsClientError returns true when this weaviate wellknown readiness o k response has a 4xx status code -func (o *WeaviateWellknownReadinessOK) IsClientError() bool { - return false -} - -// IsServerError returns true when this weaviate wellknown readiness o k response has a 5xx status code -func (o *WeaviateWellknownReadinessOK) IsServerError() bool { - return false -} - -// IsCode returns true when this weaviate wellknown readiness o k response a status code equal to that given -func (o *WeaviateWellknownReadinessOK) IsCode(code int) bool { - return code == 200 -} - -// Code gets the status code for the weaviate wellknown readiness o k response -func (o *WeaviateWellknownReadinessOK) Code() int { - return 200 -} - -func (o *WeaviateWellknownReadinessOK) Error() string { - return fmt.Sprintf("[GET /.well-known/ready][%d] weaviateWellknownReadinessOK ", 200) -} - -func (o *WeaviateWellknownReadinessOK) String() string { - return fmt.Sprintf("[GET /.well-known/ready][%d] weaviateWellknownReadinessOK ", 200) -} - -func (o *WeaviateWellknownReadinessOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewWeaviateWellknownReadinessServiceUnavailable creates a WeaviateWellknownReadinessServiceUnavailable with default headers values -func NewWeaviateWellknownReadinessServiceUnavailable() *WeaviateWellknownReadinessServiceUnavailable { - return &WeaviateWellknownReadinessServiceUnavailable{} -} - -/* -WeaviateWellknownReadinessServiceUnavailable describes a response with status code 503, with default header values. - -The application is currently not able to serve traffic. If other horizontal replicas of weaviate are available and they are capable of receiving traffic, all traffic should be redirected there instead. -*/ -type WeaviateWellknownReadinessServiceUnavailable struct { -} - -// IsSuccess returns true when this weaviate wellknown readiness service unavailable response has a 2xx status code -func (o *WeaviateWellknownReadinessServiceUnavailable) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this weaviate wellknown readiness service unavailable response has a 3xx status code -func (o *WeaviateWellknownReadinessServiceUnavailable) IsRedirect() bool { - return false -} - -// IsClientError returns true when this weaviate wellknown readiness service unavailable response has a 4xx status code -func (o *WeaviateWellknownReadinessServiceUnavailable) IsClientError() bool { - return false -} - -// IsServerError returns true when this weaviate wellknown readiness service unavailable response has a 5xx status code -func (o *WeaviateWellknownReadinessServiceUnavailable) IsServerError() bool { - return true -} - -// IsCode returns true when this weaviate wellknown readiness service unavailable response a status code equal to that given -func (o *WeaviateWellknownReadinessServiceUnavailable) IsCode(code int) bool { - return code == 503 -} - -// Code gets the status code for the weaviate wellknown readiness service unavailable response -func (o *WeaviateWellknownReadinessServiceUnavailable) Code() int { - return 503 -} - -func (o *WeaviateWellknownReadinessServiceUnavailable) Error() string { - return fmt.Sprintf("[GET /.well-known/ready][%d] weaviateWellknownReadinessServiceUnavailable ", 503) -} - -func (o *WeaviateWellknownReadinessServiceUnavailable) String() string { - return fmt.Sprintf("[GET /.well-known/ready][%d] weaviateWellknownReadinessServiceUnavailable ", 503) -} - -func (o *WeaviateWellknownReadinessServiceUnavailable) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} diff --git a/client/schema/schema_client.go b/client/schema/schema_client.go deleted file mode 100644 index 71467ce98854a73361bc2df8b144026675b3553b..0000000000000000000000000000000000000000 --- a/client/schema/schema_client.go +++ /dev/null @@ -1,585 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" -) - -// New creates a new schema API client. -func New(transport runtime.ClientTransport, formats strfmt.Registry) ClientService { - return &Client{transport: transport, formats: formats} -} - -/* -Client for schema API -*/ -type Client struct { - transport runtime.ClientTransport - formats strfmt.Registry -} - -// ClientOption is the option for Client methods -type ClientOption func(*runtime.ClientOperation) - -// ClientService is the interface for Client methods -type ClientService interface { - SchemaClusterStatus(params *SchemaClusterStatusParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*SchemaClusterStatusOK, error) - - SchemaDump(params *SchemaDumpParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*SchemaDumpOK, error) - - SchemaObjectsCreate(params *SchemaObjectsCreateParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*SchemaObjectsCreateOK, error) - - SchemaObjectsDelete(params *SchemaObjectsDeleteParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*SchemaObjectsDeleteOK, error) - - SchemaObjectsGet(params *SchemaObjectsGetParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*SchemaObjectsGetOK, error) - - SchemaObjectsPropertiesAdd(params *SchemaObjectsPropertiesAddParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*SchemaObjectsPropertiesAddOK, error) - - SchemaObjectsShardsGet(params *SchemaObjectsShardsGetParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*SchemaObjectsShardsGetOK, error) - - SchemaObjectsShardsUpdate(params *SchemaObjectsShardsUpdateParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*SchemaObjectsShardsUpdateOK, error) - - SchemaObjectsUpdate(params *SchemaObjectsUpdateParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*SchemaObjectsUpdateOK, error) - - TenantsCreate(params *TenantsCreateParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*TenantsCreateOK, error) - - TenantsDelete(params *TenantsDeleteParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*TenantsDeleteOK, error) - - TenantsGet(params *TenantsGetParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*TenantsGetOK, error) - - TenantsUpdate(params *TenantsUpdateParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*TenantsUpdateOK, error) - - SetTransport(transport runtime.ClientTransport) -} - -/* -SchemaClusterStatus schema cluster status API -*/ -func (a *Client) SchemaClusterStatus(params *SchemaClusterStatusParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*SchemaClusterStatusOK, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewSchemaClusterStatusParams() - } - op := &runtime.ClientOperation{ - ID: "schema.cluster.status", - Method: "GET", - PathPattern: "/schema/cluster-status", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &SchemaClusterStatusReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*SchemaClusterStatusOK) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for schema.cluster.status: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -/* -SchemaDump dumps the current the database schema -*/ -func (a *Client) SchemaDump(params *SchemaDumpParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*SchemaDumpOK, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewSchemaDumpParams() - } - op := &runtime.ClientOperation{ - ID: "schema.dump", - Method: "GET", - PathPattern: "/schema", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &SchemaDumpReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*SchemaDumpOK) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for schema.dump: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -/* -SchemaObjectsCreate creates a new object class in the schema -*/ -func (a *Client) SchemaObjectsCreate(params *SchemaObjectsCreateParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*SchemaObjectsCreateOK, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewSchemaObjectsCreateParams() - } - op := &runtime.ClientOperation{ - ID: "schema.objects.create", - Method: "POST", - PathPattern: "/schema", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &SchemaObjectsCreateReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*SchemaObjectsCreateOK) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for schema.objects.create: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -/* -SchemaObjectsDelete removes an object class and all data in the instances from the schema -*/ -func (a *Client) SchemaObjectsDelete(params *SchemaObjectsDeleteParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*SchemaObjectsDeleteOK, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewSchemaObjectsDeleteParams() - } - op := &runtime.ClientOperation{ - ID: "schema.objects.delete", - Method: "DELETE", - PathPattern: "/schema/{className}", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &SchemaObjectsDeleteReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*SchemaObjectsDeleteOK) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for schema.objects.delete: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -/* -SchemaObjectsGet gets a single class from the schema -*/ -func (a *Client) SchemaObjectsGet(params *SchemaObjectsGetParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*SchemaObjectsGetOK, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewSchemaObjectsGetParams() - } - op := &runtime.ClientOperation{ - ID: "schema.objects.get", - Method: "GET", - PathPattern: "/schema/{className}", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &SchemaObjectsGetReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*SchemaObjectsGetOK) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for schema.objects.get: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -/* -SchemaObjectsPropertiesAdd adds a property to an object class -*/ -func (a *Client) SchemaObjectsPropertiesAdd(params *SchemaObjectsPropertiesAddParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*SchemaObjectsPropertiesAddOK, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewSchemaObjectsPropertiesAddParams() - } - op := &runtime.ClientOperation{ - ID: "schema.objects.properties.add", - Method: "POST", - PathPattern: "/schema/{className}/properties", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &SchemaObjectsPropertiesAddReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*SchemaObjectsPropertiesAddOK) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for schema.objects.properties.add: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -/* -SchemaObjectsShardsGet gets the shards status of an object class -*/ -func (a *Client) SchemaObjectsShardsGet(params *SchemaObjectsShardsGetParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*SchemaObjectsShardsGetOK, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewSchemaObjectsShardsGetParams() - } - op := &runtime.ClientOperation{ - ID: "schema.objects.shards.get", - Method: "GET", - PathPattern: "/schema/{className}/shards", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &SchemaObjectsShardsGetReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*SchemaObjectsShardsGetOK) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for schema.objects.shards.get: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -/* -SchemaObjectsShardsUpdate Update shard status of an Object Class -*/ -func (a *Client) SchemaObjectsShardsUpdate(params *SchemaObjectsShardsUpdateParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*SchemaObjectsShardsUpdateOK, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewSchemaObjectsShardsUpdateParams() - } - op := &runtime.ClientOperation{ - ID: "schema.objects.shards.update", - Method: "PUT", - PathPattern: "/schema/{className}/shards/{shardName}", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &SchemaObjectsShardsUpdateReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*SchemaObjectsShardsUpdateOK) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for schema.objects.shards.update: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -/* -SchemaObjectsUpdate updates settings of an existing schema class - -Use this endpoint to alter an existing class in the schema. Note that not all settings are mutable. If an error about immutable fields is returned and you still need to update this particular setting, you will have to delete the class (and the underlying data) and recreate. This endpoint cannot be used to modify properties. Instead use POST /v1/schema/{className}/properties. A typical use case for this endpoint is to update configuration, such as the vectorIndexConfig. Note that even in mutable sections, such as vectorIndexConfig, some fields may be immutable. -*/ -func (a *Client) SchemaObjectsUpdate(params *SchemaObjectsUpdateParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*SchemaObjectsUpdateOK, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewSchemaObjectsUpdateParams() - } - op := &runtime.ClientOperation{ - ID: "schema.objects.update", - Method: "PUT", - PathPattern: "/schema/{className}", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &SchemaObjectsUpdateReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*SchemaObjectsUpdateOK) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for schema.objects.update: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -/* -TenantsCreate Create a new tenant for a specific class -*/ -func (a *Client) TenantsCreate(params *TenantsCreateParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*TenantsCreateOK, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewTenantsCreateParams() - } - op := &runtime.ClientOperation{ - ID: "tenants.create", - Method: "POST", - PathPattern: "/schema/{className}/tenants", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &TenantsCreateReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*TenantsCreateOK) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for tenants.create: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -/* -TenantsDelete delete tenants from a specific class -*/ -func (a *Client) TenantsDelete(params *TenantsDeleteParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*TenantsDeleteOK, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewTenantsDeleteParams() - } - op := &runtime.ClientOperation{ - ID: "tenants.delete", - Method: "DELETE", - PathPattern: "/schema/{className}/tenants", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &TenantsDeleteReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*TenantsDeleteOK) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for tenants.delete: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -/* -TenantsGet get all tenants from a specific class -*/ -func (a *Client) TenantsGet(params *TenantsGetParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*TenantsGetOK, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewTenantsGetParams() - } - op := &runtime.ClientOperation{ - ID: "tenants.get", - Method: "GET", - PathPattern: "/schema/{className}/tenants", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &TenantsGetReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*TenantsGetOK) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for tenants.get: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -/* -TenantsUpdate Update tenant of a specific class -*/ -func (a *Client) TenantsUpdate(params *TenantsUpdateParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*TenantsUpdateOK, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewTenantsUpdateParams() - } - op := &runtime.ClientOperation{ - ID: "tenants.update", - Method: "PUT", - PathPattern: "/schema/{className}/tenants", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &TenantsUpdateReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*TenantsUpdateOK) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for tenants.update: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -// SetTransport changes the transport on the client -func (a *Client) SetTransport(transport runtime.ClientTransport) { - a.transport = transport -} diff --git a/client/schema/schema_cluster_status_parameters.go b/client/schema/schema_cluster_status_parameters.go deleted file mode 100644 index 010e99c4dde8a6bad5aaee7269bc9e771e38d84f..0000000000000000000000000000000000000000 --- a/client/schema/schema_cluster_status_parameters.go +++ /dev/null @@ -1,139 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" -) - -// NewSchemaClusterStatusParams creates a new SchemaClusterStatusParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewSchemaClusterStatusParams() *SchemaClusterStatusParams { - return &SchemaClusterStatusParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewSchemaClusterStatusParamsWithTimeout creates a new SchemaClusterStatusParams object -// with the ability to set a timeout on a request. -func NewSchemaClusterStatusParamsWithTimeout(timeout time.Duration) *SchemaClusterStatusParams { - return &SchemaClusterStatusParams{ - timeout: timeout, - } -} - -// NewSchemaClusterStatusParamsWithContext creates a new SchemaClusterStatusParams object -// with the ability to set a context for a request. -func NewSchemaClusterStatusParamsWithContext(ctx context.Context) *SchemaClusterStatusParams { - return &SchemaClusterStatusParams{ - Context: ctx, - } -} - -// NewSchemaClusterStatusParamsWithHTTPClient creates a new SchemaClusterStatusParams object -// with the ability to set a custom HTTPClient for a request. -func NewSchemaClusterStatusParamsWithHTTPClient(client *http.Client) *SchemaClusterStatusParams { - return &SchemaClusterStatusParams{ - HTTPClient: client, - } -} - -/* -SchemaClusterStatusParams contains all the parameters to send to the API endpoint - - for the schema cluster status operation. - - Typically these are written to a http.Request. -*/ -type SchemaClusterStatusParams struct { - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the schema cluster status params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *SchemaClusterStatusParams) WithDefaults() *SchemaClusterStatusParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the schema cluster status params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *SchemaClusterStatusParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the schema cluster status params -func (o *SchemaClusterStatusParams) WithTimeout(timeout time.Duration) *SchemaClusterStatusParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the schema cluster status params -func (o *SchemaClusterStatusParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the schema cluster status params -func (o *SchemaClusterStatusParams) WithContext(ctx context.Context) *SchemaClusterStatusParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the schema cluster status params -func (o *SchemaClusterStatusParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the schema cluster status params -func (o *SchemaClusterStatusParams) WithHTTPClient(client *http.Client) *SchemaClusterStatusParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the schema cluster status params -func (o *SchemaClusterStatusParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WriteToRequest writes these params to a swagger request -func (o *SchemaClusterStatusParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/schema/schema_cluster_status_responses.go b/client/schema/schema_cluster_status_responses.go deleted file mode 100644 index 9fb036323d07d5ce3775a86b8638fc06b60d1629..0000000000000000000000000000000000000000 --- a/client/schema/schema_cluster_status_responses.go +++ /dev/null @@ -1,188 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// SchemaClusterStatusReader is a Reader for the SchemaClusterStatus structure. -type SchemaClusterStatusReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *SchemaClusterStatusReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 200: - result := NewSchemaClusterStatusOK() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 500: - result := NewSchemaClusterStatusInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewSchemaClusterStatusOK creates a SchemaClusterStatusOK with default headers values -func NewSchemaClusterStatusOK() *SchemaClusterStatusOK { - return &SchemaClusterStatusOK{} -} - -/* -SchemaClusterStatusOK describes a response with status code 200, with default header values. - -The schema in the cluster is in sync. -*/ -type SchemaClusterStatusOK struct { - Payload *models.SchemaClusterStatus -} - -// IsSuccess returns true when this schema cluster status o k response has a 2xx status code -func (o *SchemaClusterStatusOK) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this schema cluster status o k response has a 3xx status code -func (o *SchemaClusterStatusOK) IsRedirect() bool { - return false -} - -// IsClientError returns true when this schema cluster status o k response has a 4xx status code -func (o *SchemaClusterStatusOK) IsClientError() bool { - return false -} - -// IsServerError returns true when this schema cluster status o k response has a 5xx status code -func (o *SchemaClusterStatusOK) IsServerError() bool { - return false -} - -// IsCode returns true when this schema cluster status o k response a status code equal to that given -func (o *SchemaClusterStatusOK) IsCode(code int) bool { - return code == 200 -} - -// Code gets the status code for the schema cluster status o k response -func (o *SchemaClusterStatusOK) Code() int { - return 200 -} - -func (o *SchemaClusterStatusOK) Error() string { - return fmt.Sprintf("[GET /schema/cluster-status][%d] schemaClusterStatusOK %+v", 200, o.Payload) -} - -func (o *SchemaClusterStatusOK) String() string { - return fmt.Sprintf("[GET /schema/cluster-status][%d] schemaClusterStatusOK %+v", 200, o.Payload) -} - -func (o *SchemaClusterStatusOK) GetPayload() *models.SchemaClusterStatus { - return o.Payload -} - -func (o *SchemaClusterStatusOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.SchemaClusterStatus) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewSchemaClusterStatusInternalServerError creates a SchemaClusterStatusInternalServerError with default headers values -func NewSchemaClusterStatusInternalServerError() *SchemaClusterStatusInternalServerError { - return &SchemaClusterStatusInternalServerError{} -} - -/* -SchemaClusterStatusInternalServerError describes a response with status code 500, with default header values. - -The schema is either out of sync (see response body) or the sync check could not be completed. -*/ -type SchemaClusterStatusInternalServerError struct { - Payload *models.SchemaClusterStatus -} - -// IsSuccess returns true when this schema cluster status internal server error response has a 2xx status code -func (o *SchemaClusterStatusInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this schema cluster status internal server error response has a 3xx status code -func (o *SchemaClusterStatusInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this schema cluster status internal server error response has a 4xx status code -func (o *SchemaClusterStatusInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this schema cluster status internal server error response has a 5xx status code -func (o *SchemaClusterStatusInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this schema cluster status internal server error response a status code equal to that given -func (o *SchemaClusterStatusInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the schema cluster status internal server error response -func (o *SchemaClusterStatusInternalServerError) Code() int { - return 500 -} - -func (o *SchemaClusterStatusInternalServerError) Error() string { - return fmt.Sprintf("[GET /schema/cluster-status][%d] schemaClusterStatusInternalServerError %+v", 500, o.Payload) -} - -func (o *SchemaClusterStatusInternalServerError) String() string { - return fmt.Sprintf("[GET /schema/cluster-status][%d] schemaClusterStatusInternalServerError %+v", 500, o.Payload) -} - -func (o *SchemaClusterStatusInternalServerError) GetPayload() *models.SchemaClusterStatus { - return o.Payload -} - -func (o *SchemaClusterStatusInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.SchemaClusterStatus) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/schema/schema_dump_parameters.go b/client/schema/schema_dump_parameters.go deleted file mode 100644 index 66be4275bf2a813aa15f63436c0c409d210e6018..0000000000000000000000000000000000000000 --- a/client/schema/schema_dump_parameters.go +++ /dev/null @@ -1,139 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" -) - -// NewSchemaDumpParams creates a new SchemaDumpParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewSchemaDumpParams() *SchemaDumpParams { - return &SchemaDumpParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewSchemaDumpParamsWithTimeout creates a new SchemaDumpParams object -// with the ability to set a timeout on a request. -func NewSchemaDumpParamsWithTimeout(timeout time.Duration) *SchemaDumpParams { - return &SchemaDumpParams{ - timeout: timeout, - } -} - -// NewSchemaDumpParamsWithContext creates a new SchemaDumpParams object -// with the ability to set a context for a request. -func NewSchemaDumpParamsWithContext(ctx context.Context) *SchemaDumpParams { - return &SchemaDumpParams{ - Context: ctx, - } -} - -// NewSchemaDumpParamsWithHTTPClient creates a new SchemaDumpParams object -// with the ability to set a custom HTTPClient for a request. -func NewSchemaDumpParamsWithHTTPClient(client *http.Client) *SchemaDumpParams { - return &SchemaDumpParams{ - HTTPClient: client, - } -} - -/* -SchemaDumpParams contains all the parameters to send to the API endpoint - - for the schema dump operation. - - Typically these are written to a http.Request. -*/ -type SchemaDumpParams struct { - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the schema dump params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *SchemaDumpParams) WithDefaults() *SchemaDumpParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the schema dump params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *SchemaDumpParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the schema dump params -func (o *SchemaDumpParams) WithTimeout(timeout time.Duration) *SchemaDumpParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the schema dump params -func (o *SchemaDumpParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the schema dump params -func (o *SchemaDumpParams) WithContext(ctx context.Context) *SchemaDumpParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the schema dump params -func (o *SchemaDumpParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the schema dump params -func (o *SchemaDumpParams) WithHTTPClient(client *http.Client) *SchemaDumpParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the schema dump params -func (o *SchemaDumpParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WriteToRequest writes these params to a swagger request -func (o *SchemaDumpParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/schema/schema_dump_responses.go b/client/schema/schema_dump_responses.go deleted file mode 100644 index 974155929c1608290595a3bd1942074f136290e0..0000000000000000000000000000000000000000 --- a/client/schema/schema_dump_responses.go +++ /dev/null @@ -1,324 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// SchemaDumpReader is a Reader for the SchemaDump structure. -type SchemaDumpReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *SchemaDumpReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 200: - result := NewSchemaDumpOK() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 401: - result := NewSchemaDumpUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewSchemaDumpForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewSchemaDumpInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewSchemaDumpOK creates a SchemaDumpOK with default headers values -func NewSchemaDumpOK() *SchemaDumpOK { - return &SchemaDumpOK{} -} - -/* -SchemaDumpOK describes a response with status code 200, with default header values. - -Successfully dumped the database schema. -*/ -type SchemaDumpOK struct { - Payload *models.Schema -} - -// IsSuccess returns true when this schema dump o k response has a 2xx status code -func (o *SchemaDumpOK) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this schema dump o k response has a 3xx status code -func (o *SchemaDumpOK) IsRedirect() bool { - return false -} - -// IsClientError returns true when this schema dump o k response has a 4xx status code -func (o *SchemaDumpOK) IsClientError() bool { - return false -} - -// IsServerError returns true when this schema dump o k response has a 5xx status code -func (o *SchemaDumpOK) IsServerError() bool { - return false -} - -// IsCode returns true when this schema dump o k response a status code equal to that given -func (o *SchemaDumpOK) IsCode(code int) bool { - return code == 200 -} - -// Code gets the status code for the schema dump o k response -func (o *SchemaDumpOK) Code() int { - return 200 -} - -func (o *SchemaDumpOK) Error() string { - return fmt.Sprintf("[GET /schema][%d] schemaDumpOK %+v", 200, o.Payload) -} - -func (o *SchemaDumpOK) String() string { - return fmt.Sprintf("[GET /schema][%d] schemaDumpOK %+v", 200, o.Payload) -} - -func (o *SchemaDumpOK) GetPayload() *models.Schema { - return o.Payload -} - -func (o *SchemaDumpOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.Schema) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewSchemaDumpUnauthorized creates a SchemaDumpUnauthorized with default headers values -func NewSchemaDumpUnauthorized() *SchemaDumpUnauthorized { - return &SchemaDumpUnauthorized{} -} - -/* -SchemaDumpUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type SchemaDumpUnauthorized struct { -} - -// IsSuccess returns true when this schema dump unauthorized response has a 2xx status code -func (o *SchemaDumpUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this schema dump unauthorized response has a 3xx status code -func (o *SchemaDumpUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this schema dump unauthorized response has a 4xx status code -func (o *SchemaDumpUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this schema dump unauthorized response has a 5xx status code -func (o *SchemaDumpUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this schema dump unauthorized response a status code equal to that given -func (o *SchemaDumpUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the schema dump unauthorized response -func (o *SchemaDumpUnauthorized) Code() int { - return 401 -} - -func (o *SchemaDumpUnauthorized) Error() string { - return fmt.Sprintf("[GET /schema][%d] schemaDumpUnauthorized ", 401) -} - -func (o *SchemaDumpUnauthorized) String() string { - return fmt.Sprintf("[GET /schema][%d] schemaDumpUnauthorized ", 401) -} - -func (o *SchemaDumpUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewSchemaDumpForbidden creates a SchemaDumpForbidden with default headers values -func NewSchemaDumpForbidden() *SchemaDumpForbidden { - return &SchemaDumpForbidden{} -} - -/* -SchemaDumpForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type SchemaDumpForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this schema dump forbidden response has a 2xx status code -func (o *SchemaDumpForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this schema dump forbidden response has a 3xx status code -func (o *SchemaDumpForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this schema dump forbidden response has a 4xx status code -func (o *SchemaDumpForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this schema dump forbidden response has a 5xx status code -func (o *SchemaDumpForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this schema dump forbidden response a status code equal to that given -func (o *SchemaDumpForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the schema dump forbidden response -func (o *SchemaDumpForbidden) Code() int { - return 403 -} - -func (o *SchemaDumpForbidden) Error() string { - return fmt.Sprintf("[GET /schema][%d] schemaDumpForbidden %+v", 403, o.Payload) -} - -func (o *SchemaDumpForbidden) String() string { - return fmt.Sprintf("[GET /schema][%d] schemaDumpForbidden %+v", 403, o.Payload) -} - -func (o *SchemaDumpForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *SchemaDumpForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewSchemaDumpInternalServerError creates a SchemaDumpInternalServerError with default headers values -func NewSchemaDumpInternalServerError() *SchemaDumpInternalServerError { - return &SchemaDumpInternalServerError{} -} - -/* -SchemaDumpInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type SchemaDumpInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this schema dump internal server error response has a 2xx status code -func (o *SchemaDumpInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this schema dump internal server error response has a 3xx status code -func (o *SchemaDumpInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this schema dump internal server error response has a 4xx status code -func (o *SchemaDumpInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this schema dump internal server error response has a 5xx status code -func (o *SchemaDumpInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this schema dump internal server error response a status code equal to that given -func (o *SchemaDumpInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the schema dump internal server error response -func (o *SchemaDumpInternalServerError) Code() int { - return 500 -} - -func (o *SchemaDumpInternalServerError) Error() string { - return fmt.Sprintf("[GET /schema][%d] schemaDumpInternalServerError %+v", 500, o.Payload) -} - -func (o *SchemaDumpInternalServerError) String() string { - return fmt.Sprintf("[GET /schema][%d] schemaDumpInternalServerError %+v", 500, o.Payload) -} - -func (o *SchemaDumpInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *SchemaDumpInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/schema/schema_objects_create_parameters.go b/client/schema/schema_objects_create_parameters.go deleted file mode 100644 index 6a02574cb1a84c68fd0352eb94b7cbbce0dd96e5..0000000000000000000000000000000000000000 --- a/client/schema/schema_objects_create_parameters.go +++ /dev/null @@ -1,161 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewSchemaObjectsCreateParams creates a new SchemaObjectsCreateParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewSchemaObjectsCreateParams() *SchemaObjectsCreateParams { - return &SchemaObjectsCreateParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewSchemaObjectsCreateParamsWithTimeout creates a new SchemaObjectsCreateParams object -// with the ability to set a timeout on a request. -func NewSchemaObjectsCreateParamsWithTimeout(timeout time.Duration) *SchemaObjectsCreateParams { - return &SchemaObjectsCreateParams{ - timeout: timeout, - } -} - -// NewSchemaObjectsCreateParamsWithContext creates a new SchemaObjectsCreateParams object -// with the ability to set a context for a request. -func NewSchemaObjectsCreateParamsWithContext(ctx context.Context) *SchemaObjectsCreateParams { - return &SchemaObjectsCreateParams{ - Context: ctx, - } -} - -// NewSchemaObjectsCreateParamsWithHTTPClient creates a new SchemaObjectsCreateParams object -// with the ability to set a custom HTTPClient for a request. -func NewSchemaObjectsCreateParamsWithHTTPClient(client *http.Client) *SchemaObjectsCreateParams { - return &SchemaObjectsCreateParams{ - HTTPClient: client, - } -} - -/* -SchemaObjectsCreateParams contains all the parameters to send to the API endpoint - - for the schema objects create operation. - - Typically these are written to a http.Request. -*/ -type SchemaObjectsCreateParams struct { - - // ObjectClass. - ObjectClass *models.Class - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the schema objects create params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *SchemaObjectsCreateParams) WithDefaults() *SchemaObjectsCreateParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the schema objects create params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *SchemaObjectsCreateParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the schema objects create params -func (o *SchemaObjectsCreateParams) WithTimeout(timeout time.Duration) *SchemaObjectsCreateParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the schema objects create params -func (o *SchemaObjectsCreateParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the schema objects create params -func (o *SchemaObjectsCreateParams) WithContext(ctx context.Context) *SchemaObjectsCreateParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the schema objects create params -func (o *SchemaObjectsCreateParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the schema objects create params -func (o *SchemaObjectsCreateParams) WithHTTPClient(client *http.Client) *SchemaObjectsCreateParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the schema objects create params -func (o *SchemaObjectsCreateParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithObjectClass adds the objectClass to the schema objects create params -func (o *SchemaObjectsCreateParams) WithObjectClass(objectClass *models.Class) *SchemaObjectsCreateParams { - o.SetObjectClass(objectClass) - return o -} - -// SetObjectClass adds the objectClass to the schema objects create params -func (o *SchemaObjectsCreateParams) SetObjectClass(objectClass *models.Class) { - o.ObjectClass = objectClass -} - -// WriteToRequest writes these params to a swagger request -func (o *SchemaObjectsCreateParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - if o.ObjectClass != nil { - if err := r.SetBodyParam(o.ObjectClass); err != nil { - return err - } - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/schema/schema_objects_create_responses.go b/client/schema/schema_objects_create_responses.go deleted file mode 100644 index 7d88c74e4592cab07e11a24e0c7aa1bf34cb0c9b..0000000000000000000000000000000000000000 --- a/client/schema/schema_objects_create_responses.go +++ /dev/null @@ -1,398 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// SchemaObjectsCreateReader is a Reader for the SchemaObjectsCreate structure. -type SchemaObjectsCreateReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *SchemaObjectsCreateReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 200: - result := NewSchemaObjectsCreateOK() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 401: - result := NewSchemaObjectsCreateUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewSchemaObjectsCreateForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 422: - result := NewSchemaObjectsCreateUnprocessableEntity() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewSchemaObjectsCreateInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewSchemaObjectsCreateOK creates a SchemaObjectsCreateOK with default headers values -func NewSchemaObjectsCreateOK() *SchemaObjectsCreateOK { - return &SchemaObjectsCreateOK{} -} - -/* -SchemaObjectsCreateOK describes a response with status code 200, with default header values. - -Added the new Object class to the schema. -*/ -type SchemaObjectsCreateOK struct { - Payload *models.Class -} - -// IsSuccess returns true when this schema objects create o k response has a 2xx status code -func (o *SchemaObjectsCreateOK) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this schema objects create o k response has a 3xx status code -func (o *SchemaObjectsCreateOK) IsRedirect() bool { - return false -} - -// IsClientError returns true when this schema objects create o k response has a 4xx status code -func (o *SchemaObjectsCreateOK) IsClientError() bool { - return false -} - -// IsServerError returns true when this schema objects create o k response has a 5xx status code -func (o *SchemaObjectsCreateOK) IsServerError() bool { - return false -} - -// IsCode returns true when this schema objects create o k response a status code equal to that given -func (o *SchemaObjectsCreateOK) IsCode(code int) bool { - return code == 200 -} - -// Code gets the status code for the schema objects create o k response -func (o *SchemaObjectsCreateOK) Code() int { - return 200 -} - -func (o *SchemaObjectsCreateOK) Error() string { - return fmt.Sprintf("[POST /schema][%d] schemaObjectsCreateOK %+v", 200, o.Payload) -} - -func (o *SchemaObjectsCreateOK) String() string { - return fmt.Sprintf("[POST /schema][%d] schemaObjectsCreateOK %+v", 200, o.Payload) -} - -func (o *SchemaObjectsCreateOK) GetPayload() *models.Class { - return o.Payload -} - -func (o *SchemaObjectsCreateOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.Class) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewSchemaObjectsCreateUnauthorized creates a SchemaObjectsCreateUnauthorized with default headers values -func NewSchemaObjectsCreateUnauthorized() *SchemaObjectsCreateUnauthorized { - return &SchemaObjectsCreateUnauthorized{} -} - -/* -SchemaObjectsCreateUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type SchemaObjectsCreateUnauthorized struct { -} - -// IsSuccess returns true when this schema objects create unauthorized response has a 2xx status code -func (o *SchemaObjectsCreateUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this schema objects create unauthorized response has a 3xx status code -func (o *SchemaObjectsCreateUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this schema objects create unauthorized response has a 4xx status code -func (o *SchemaObjectsCreateUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this schema objects create unauthorized response has a 5xx status code -func (o *SchemaObjectsCreateUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this schema objects create unauthorized response a status code equal to that given -func (o *SchemaObjectsCreateUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the schema objects create unauthorized response -func (o *SchemaObjectsCreateUnauthorized) Code() int { - return 401 -} - -func (o *SchemaObjectsCreateUnauthorized) Error() string { - return fmt.Sprintf("[POST /schema][%d] schemaObjectsCreateUnauthorized ", 401) -} - -func (o *SchemaObjectsCreateUnauthorized) String() string { - return fmt.Sprintf("[POST /schema][%d] schemaObjectsCreateUnauthorized ", 401) -} - -func (o *SchemaObjectsCreateUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewSchemaObjectsCreateForbidden creates a SchemaObjectsCreateForbidden with default headers values -func NewSchemaObjectsCreateForbidden() *SchemaObjectsCreateForbidden { - return &SchemaObjectsCreateForbidden{} -} - -/* -SchemaObjectsCreateForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type SchemaObjectsCreateForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this schema objects create forbidden response has a 2xx status code -func (o *SchemaObjectsCreateForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this schema objects create forbidden response has a 3xx status code -func (o *SchemaObjectsCreateForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this schema objects create forbidden response has a 4xx status code -func (o *SchemaObjectsCreateForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this schema objects create forbidden response has a 5xx status code -func (o *SchemaObjectsCreateForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this schema objects create forbidden response a status code equal to that given -func (o *SchemaObjectsCreateForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the schema objects create forbidden response -func (o *SchemaObjectsCreateForbidden) Code() int { - return 403 -} - -func (o *SchemaObjectsCreateForbidden) Error() string { - return fmt.Sprintf("[POST /schema][%d] schemaObjectsCreateForbidden %+v", 403, o.Payload) -} - -func (o *SchemaObjectsCreateForbidden) String() string { - return fmt.Sprintf("[POST /schema][%d] schemaObjectsCreateForbidden %+v", 403, o.Payload) -} - -func (o *SchemaObjectsCreateForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *SchemaObjectsCreateForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewSchemaObjectsCreateUnprocessableEntity creates a SchemaObjectsCreateUnprocessableEntity with default headers values -func NewSchemaObjectsCreateUnprocessableEntity() *SchemaObjectsCreateUnprocessableEntity { - return &SchemaObjectsCreateUnprocessableEntity{} -} - -/* -SchemaObjectsCreateUnprocessableEntity describes a response with status code 422, with default header values. - -Invalid Object class -*/ -type SchemaObjectsCreateUnprocessableEntity struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this schema objects create unprocessable entity response has a 2xx status code -func (o *SchemaObjectsCreateUnprocessableEntity) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this schema objects create unprocessable entity response has a 3xx status code -func (o *SchemaObjectsCreateUnprocessableEntity) IsRedirect() bool { - return false -} - -// IsClientError returns true when this schema objects create unprocessable entity response has a 4xx status code -func (o *SchemaObjectsCreateUnprocessableEntity) IsClientError() bool { - return true -} - -// IsServerError returns true when this schema objects create unprocessable entity response has a 5xx status code -func (o *SchemaObjectsCreateUnprocessableEntity) IsServerError() bool { - return false -} - -// IsCode returns true when this schema objects create unprocessable entity response a status code equal to that given -func (o *SchemaObjectsCreateUnprocessableEntity) IsCode(code int) bool { - return code == 422 -} - -// Code gets the status code for the schema objects create unprocessable entity response -func (o *SchemaObjectsCreateUnprocessableEntity) Code() int { - return 422 -} - -func (o *SchemaObjectsCreateUnprocessableEntity) Error() string { - return fmt.Sprintf("[POST /schema][%d] schemaObjectsCreateUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *SchemaObjectsCreateUnprocessableEntity) String() string { - return fmt.Sprintf("[POST /schema][%d] schemaObjectsCreateUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *SchemaObjectsCreateUnprocessableEntity) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *SchemaObjectsCreateUnprocessableEntity) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewSchemaObjectsCreateInternalServerError creates a SchemaObjectsCreateInternalServerError with default headers values -func NewSchemaObjectsCreateInternalServerError() *SchemaObjectsCreateInternalServerError { - return &SchemaObjectsCreateInternalServerError{} -} - -/* -SchemaObjectsCreateInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type SchemaObjectsCreateInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this schema objects create internal server error response has a 2xx status code -func (o *SchemaObjectsCreateInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this schema objects create internal server error response has a 3xx status code -func (o *SchemaObjectsCreateInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this schema objects create internal server error response has a 4xx status code -func (o *SchemaObjectsCreateInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this schema objects create internal server error response has a 5xx status code -func (o *SchemaObjectsCreateInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this schema objects create internal server error response a status code equal to that given -func (o *SchemaObjectsCreateInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the schema objects create internal server error response -func (o *SchemaObjectsCreateInternalServerError) Code() int { - return 500 -} - -func (o *SchemaObjectsCreateInternalServerError) Error() string { - return fmt.Sprintf("[POST /schema][%d] schemaObjectsCreateInternalServerError %+v", 500, o.Payload) -} - -func (o *SchemaObjectsCreateInternalServerError) String() string { - return fmt.Sprintf("[POST /schema][%d] schemaObjectsCreateInternalServerError %+v", 500, o.Payload) -} - -func (o *SchemaObjectsCreateInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *SchemaObjectsCreateInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/schema/schema_objects_delete_parameters.go b/client/schema/schema_objects_delete_parameters.go deleted file mode 100644 index 80bdaf882a9bf3bd60f63a794dc79849db8a5fe3..0000000000000000000000000000000000000000 --- a/client/schema/schema_objects_delete_parameters.go +++ /dev/null @@ -1,159 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" -) - -// NewSchemaObjectsDeleteParams creates a new SchemaObjectsDeleteParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewSchemaObjectsDeleteParams() *SchemaObjectsDeleteParams { - return &SchemaObjectsDeleteParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewSchemaObjectsDeleteParamsWithTimeout creates a new SchemaObjectsDeleteParams object -// with the ability to set a timeout on a request. -func NewSchemaObjectsDeleteParamsWithTimeout(timeout time.Duration) *SchemaObjectsDeleteParams { - return &SchemaObjectsDeleteParams{ - timeout: timeout, - } -} - -// NewSchemaObjectsDeleteParamsWithContext creates a new SchemaObjectsDeleteParams object -// with the ability to set a context for a request. -func NewSchemaObjectsDeleteParamsWithContext(ctx context.Context) *SchemaObjectsDeleteParams { - return &SchemaObjectsDeleteParams{ - Context: ctx, - } -} - -// NewSchemaObjectsDeleteParamsWithHTTPClient creates a new SchemaObjectsDeleteParams object -// with the ability to set a custom HTTPClient for a request. -func NewSchemaObjectsDeleteParamsWithHTTPClient(client *http.Client) *SchemaObjectsDeleteParams { - return &SchemaObjectsDeleteParams{ - HTTPClient: client, - } -} - -/* -SchemaObjectsDeleteParams contains all the parameters to send to the API endpoint - - for the schema objects delete operation. - - Typically these are written to a http.Request. -*/ -type SchemaObjectsDeleteParams struct { - - // ClassName. - ClassName string - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the schema objects delete params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *SchemaObjectsDeleteParams) WithDefaults() *SchemaObjectsDeleteParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the schema objects delete params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *SchemaObjectsDeleteParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the schema objects delete params -func (o *SchemaObjectsDeleteParams) WithTimeout(timeout time.Duration) *SchemaObjectsDeleteParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the schema objects delete params -func (o *SchemaObjectsDeleteParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the schema objects delete params -func (o *SchemaObjectsDeleteParams) WithContext(ctx context.Context) *SchemaObjectsDeleteParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the schema objects delete params -func (o *SchemaObjectsDeleteParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the schema objects delete params -func (o *SchemaObjectsDeleteParams) WithHTTPClient(client *http.Client) *SchemaObjectsDeleteParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the schema objects delete params -func (o *SchemaObjectsDeleteParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithClassName adds the className to the schema objects delete params -func (o *SchemaObjectsDeleteParams) WithClassName(className string) *SchemaObjectsDeleteParams { - o.SetClassName(className) - return o -} - -// SetClassName adds the className to the schema objects delete params -func (o *SchemaObjectsDeleteParams) SetClassName(className string) { - o.ClassName = className -} - -// WriteToRequest writes these params to a swagger request -func (o *SchemaObjectsDeleteParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - - // path param className - if err := r.SetPathParam("className", o.ClassName); err != nil { - return err - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/schema/schema_objects_delete_responses.go b/client/schema/schema_objects_delete_responses.go deleted file mode 100644 index ca08fb6fcd829743b0413adfe244df6c8602b2ab..0000000000000000000000000000000000000000 --- a/client/schema/schema_objects_delete_responses.go +++ /dev/null @@ -1,386 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// SchemaObjectsDeleteReader is a Reader for the SchemaObjectsDelete structure. -type SchemaObjectsDeleteReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *SchemaObjectsDeleteReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 200: - result := NewSchemaObjectsDeleteOK() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 400: - result := NewSchemaObjectsDeleteBadRequest() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 401: - result := NewSchemaObjectsDeleteUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewSchemaObjectsDeleteForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewSchemaObjectsDeleteInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewSchemaObjectsDeleteOK creates a SchemaObjectsDeleteOK with default headers values -func NewSchemaObjectsDeleteOK() *SchemaObjectsDeleteOK { - return &SchemaObjectsDeleteOK{} -} - -/* -SchemaObjectsDeleteOK describes a response with status code 200, with default header values. - -Removed the Object class from the schema. -*/ -type SchemaObjectsDeleteOK struct { -} - -// IsSuccess returns true when this schema objects delete o k response has a 2xx status code -func (o *SchemaObjectsDeleteOK) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this schema objects delete o k response has a 3xx status code -func (o *SchemaObjectsDeleteOK) IsRedirect() bool { - return false -} - -// IsClientError returns true when this schema objects delete o k response has a 4xx status code -func (o *SchemaObjectsDeleteOK) IsClientError() bool { - return false -} - -// IsServerError returns true when this schema objects delete o k response has a 5xx status code -func (o *SchemaObjectsDeleteOK) IsServerError() bool { - return false -} - -// IsCode returns true when this schema objects delete o k response a status code equal to that given -func (o *SchemaObjectsDeleteOK) IsCode(code int) bool { - return code == 200 -} - -// Code gets the status code for the schema objects delete o k response -func (o *SchemaObjectsDeleteOK) Code() int { - return 200 -} - -func (o *SchemaObjectsDeleteOK) Error() string { - return fmt.Sprintf("[DELETE /schema/{className}][%d] schemaObjectsDeleteOK ", 200) -} - -func (o *SchemaObjectsDeleteOK) String() string { - return fmt.Sprintf("[DELETE /schema/{className}][%d] schemaObjectsDeleteOK ", 200) -} - -func (o *SchemaObjectsDeleteOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewSchemaObjectsDeleteBadRequest creates a SchemaObjectsDeleteBadRequest with default headers values -func NewSchemaObjectsDeleteBadRequest() *SchemaObjectsDeleteBadRequest { - return &SchemaObjectsDeleteBadRequest{} -} - -/* -SchemaObjectsDeleteBadRequest describes a response with status code 400, with default header values. - -Could not delete the Object class. -*/ -type SchemaObjectsDeleteBadRequest struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this schema objects delete bad request response has a 2xx status code -func (o *SchemaObjectsDeleteBadRequest) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this schema objects delete bad request response has a 3xx status code -func (o *SchemaObjectsDeleteBadRequest) IsRedirect() bool { - return false -} - -// IsClientError returns true when this schema objects delete bad request response has a 4xx status code -func (o *SchemaObjectsDeleteBadRequest) IsClientError() bool { - return true -} - -// IsServerError returns true when this schema objects delete bad request response has a 5xx status code -func (o *SchemaObjectsDeleteBadRequest) IsServerError() bool { - return false -} - -// IsCode returns true when this schema objects delete bad request response a status code equal to that given -func (o *SchemaObjectsDeleteBadRequest) IsCode(code int) bool { - return code == 400 -} - -// Code gets the status code for the schema objects delete bad request response -func (o *SchemaObjectsDeleteBadRequest) Code() int { - return 400 -} - -func (o *SchemaObjectsDeleteBadRequest) Error() string { - return fmt.Sprintf("[DELETE /schema/{className}][%d] schemaObjectsDeleteBadRequest %+v", 400, o.Payload) -} - -func (o *SchemaObjectsDeleteBadRequest) String() string { - return fmt.Sprintf("[DELETE /schema/{className}][%d] schemaObjectsDeleteBadRequest %+v", 400, o.Payload) -} - -func (o *SchemaObjectsDeleteBadRequest) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *SchemaObjectsDeleteBadRequest) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewSchemaObjectsDeleteUnauthorized creates a SchemaObjectsDeleteUnauthorized with default headers values -func NewSchemaObjectsDeleteUnauthorized() *SchemaObjectsDeleteUnauthorized { - return &SchemaObjectsDeleteUnauthorized{} -} - -/* -SchemaObjectsDeleteUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type SchemaObjectsDeleteUnauthorized struct { -} - -// IsSuccess returns true when this schema objects delete unauthorized response has a 2xx status code -func (o *SchemaObjectsDeleteUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this schema objects delete unauthorized response has a 3xx status code -func (o *SchemaObjectsDeleteUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this schema objects delete unauthorized response has a 4xx status code -func (o *SchemaObjectsDeleteUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this schema objects delete unauthorized response has a 5xx status code -func (o *SchemaObjectsDeleteUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this schema objects delete unauthorized response a status code equal to that given -func (o *SchemaObjectsDeleteUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the schema objects delete unauthorized response -func (o *SchemaObjectsDeleteUnauthorized) Code() int { - return 401 -} - -func (o *SchemaObjectsDeleteUnauthorized) Error() string { - return fmt.Sprintf("[DELETE /schema/{className}][%d] schemaObjectsDeleteUnauthorized ", 401) -} - -func (o *SchemaObjectsDeleteUnauthorized) String() string { - return fmt.Sprintf("[DELETE /schema/{className}][%d] schemaObjectsDeleteUnauthorized ", 401) -} - -func (o *SchemaObjectsDeleteUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewSchemaObjectsDeleteForbidden creates a SchemaObjectsDeleteForbidden with default headers values -func NewSchemaObjectsDeleteForbidden() *SchemaObjectsDeleteForbidden { - return &SchemaObjectsDeleteForbidden{} -} - -/* -SchemaObjectsDeleteForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type SchemaObjectsDeleteForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this schema objects delete forbidden response has a 2xx status code -func (o *SchemaObjectsDeleteForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this schema objects delete forbidden response has a 3xx status code -func (o *SchemaObjectsDeleteForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this schema objects delete forbidden response has a 4xx status code -func (o *SchemaObjectsDeleteForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this schema objects delete forbidden response has a 5xx status code -func (o *SchemaObjectsDeleteForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this schema objects delete forbidden response a status code equal to that given -func (o *SchemaObjectsDeleteForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the schema objects delete forbidden response -func (o *SchemaObjectsDeleteForbidden) Code() int { - return 403 -} - -func (o *SchemaObjectsDeleteForbidden) Error() string { - return fmt.Sprintf("[DELETE /schema/{className}][%d] schemaObjectsDeleteForbidden %+v", 403, o.Payload) -} - -func (o *SchemaObjectsDeleteForbidden) String() string { - return fmt.Sprintf("[DELETE /schema/{className}][%d] schemaObjectsDeleteForbidden %+v", 403, o.Payload) -} - -func (o *SchemaObjectsDeleteForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *SchemaObjectsDeleteForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewSchemaObjectsDeleteInternalServerError creates a SchemaObjectsDeleteInternalServerError with default headers values -func NewSchemaObjectsDeleteInternalServerError() *SchemaObjectsDeleteInternalServerError { - return &SchemaObjectsDeleteInternalServerError{} -} - -/* -SchemaObjectsDeleteInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type SchemaObjectsDeleteInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this schema objects delete internal server error response has a 2xx status code -func (o *SchemaObjectsDeleteInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this schema objects delete internal server error response has a 3xx status code -func (o *SchemaObjectsDeleteInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this schema objects delete internal server error response has a 4xx status code -func (o *SchemaObjectsDeleteInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this schema objects delete internal server error response has a 5xx status code -func (o *SchemaObjectsDeleteInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this schema objects delete internal server error response a status code equal to that given -func (o *SchemaObjectsDeleteInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the schema objects delete internal server error response -func (o *SchemaObjectsDeleteInternalServerError) Code() int { - return 500 -} - -func (o *SchemaObjectsDeleteInternalServerError) Error() string { - return fmt.Sprintf("[DELETE /schema/{className}][%d] schemaObjectsDeleteInternalServerError %+v", 500, o.Payload) -} - -func (o *SchemaObjectsDeleteInternalServerError) String() string { - return fmt.Sprintf("[DELETE /schema/{className}][%d] schemaObjectsDeleteInternalServerError %+v", 500, o.Payload) -} - -func (o *SchemaObjectsDeleteInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *SchemaObjectsDeleteInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/schema/schema_objects_get_parameters.go b/client/schema/schema_objects_get_parameters.go deleted file mode 100644 index 0d8a48e355908a24c9bfb9992f737d691f1490ec..0000000000000000000000000000000000000000 --- a/client/schema/schema_objects_get_parameters.go +++ /dev/null @@ -1,159 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" -) - -// NewSchemaObjectsGetParams creates a new SchemaObjectsGetParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewSchemaObjectsGetParams() *SchemaObjectsGetParams { - return &SchemaObjectsGetParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewSchemaObjectsGetParamsWithTimeout creates a new SchemaObjectsGetParams object -// with the ability to set a timeout on a request. -func NewSchemaObjectsGetParamsWithTimeout(timeout time.Duration) *SchemaObjectsGetParams { - return &SchemaObjectsGetParams{ - timeout: timeout, - } -} - -// NewSchemaObjectsGetParamsWithContext creates a new SchemaObjectsGetParams object -// with the ability to set a context for a request. -func NewSchemaObjectsGetParamsWithContext(ctx context.Context) *SchemaObjectsGetParams { - return &SchemaObjectsGetParams{ - Context: ctx, - } -} - -// NewSchemaObjectsGetParamsWithHTTPClient creates a new SchemaObjectsGetParams object -// with the ability to set a custom HTTPClient for a request. -func NewSchemaObjectsGetParamsWithHTTPClient(client *http.Client) *SchemaObjectsGetParams { - return &SchemaObjectsGetParams{ - HTTPClient: client, - } -} - -/* -SchemaObjectsGetParams contains all the parameters to send to the API endpoint - - for the schema objects get operation. - - Typically these are written to a http.Request. -*/ -type SchemaObjectsGetParams struct { - - // ClassName. - ClassName string - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the schema objects get params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *SchemaObjectsGetParams) WithDefaults() *SchemaObjectsGetParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the schema objects get params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *SchemaObjectsGetParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the schema objects get params -func (o *SchemaObjectsGetParams) WithTimeout(timeout time.Duration) *SchemaObjectsGetParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the schema objects get params -func (o *SchemaObjectsGetParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the schema objects get params -func (o *SchemaObjectsGetParams) WithContext(ctx context.Context) *SchemaObjectsGetParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the schema objects get params -func (o *SchemaObjectsGetParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the schema objects get params -func (o *SchemaObjectsGetParams) WithHTTPClient(client *http.Client) *SchemaObjectsGetParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the schema objects get params -func (o *SchemaObjectsGetParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithClassName adds the className to the schema objects get params -func (o *SchemaObjectsGetParams) WithClassName(className string) *SchemaObjectsGetParams { - o.SetClassName(className) - return o -} - -// SetClassName adds the className to the schema objects get params -func (o *SchemaObjectsGetParams) SetClassName(className string) { - o.ClassName = className -} - -// WriteToRequest writes these params to a swagger request -func (o *SchemaObjectsGetParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - - // path param className - if err := r.SetPathParam("className", o.ClassName); err != nil { - return err - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/schema/schema_objects_get_responses.go b/client/schema/schema_objects_get_responses.go deleted file mode 100644 index b2c267018a72517fbbe6d22f1ffef276305125ec..0000000000000000000000000000000000000000 --- a/client/schema/schema_objects_get_responses.go +++ /dev/null @@ -1,386 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// SchemaObjectsGetReader is a Reader for the SchemaObjectsGet structure. -type SchemaObjectsGetReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *SchemaObjectsGetReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 200: - result := NewSchemaObjectsGetOK() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 401: - result := NewSchemaObjectsGetUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewSchemaObjectsGetForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 404: - result := NewSchemaObjectsGetNotFound() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewSchemaObjectsGetInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewSchemaObjectsGetOK creates a SchemaObjectsGetOK with default headers values -func NewSchemaObjectsGetOK() *SchemaObjectsGetOK { - return &SchemaObjectsGetOK{} -} - -/* -SchemaObjectsGetOK describes a response with status code 200, with default header values. - -Found the Class, returned as body -*/ -type SchemaObjectsGetOK struct { - Payload *models.Class -} - -// IsSuccess returns true when this schema objects get o k response has a 2xx status code -func (o *SchemaObjectsGetOK) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this schema objects get o k response has a 3xx status code -func (o *SchemaObjectsGetOK) IsRedirect() bool { - return false -} - -// IsClientError returns true when this schema objects get o k response has a 4xx status code -func (o *SchemaObjectsGetOK) IsClientError() bool { - return false -} - -// IsServerError returns true when this schema objects get o k response has a 5xx status code -func (o *SchemaObjectsGetOK) IsServerError() bool { - return false -} - -// IsCode returns true when this schema objects get o k response a status code equal to that given -func (o *SchemaObjectsGetOK) IsCode(code int) bool { - return code == 200 -} - -// Code gets the status code for the schema objects get o k response -func (o *SchemaObjectsGetOK) Code() int { - return 200 -} - -func (o *SchemaObjectsGetOK) Error() string { - return fmt.Sprintf("[GET /schema/{className}][%d] schemaObjectsGetOK %+v", 200, o.Payload) -} - -func (o *SchemaObjectsGetOK) String() string { - return fmt.Sprintf("[GET /schema/{className}][%d] schemaObjectsGetOK %+v", 200, o.Payload) -} - -func (o *SchemaObjectsGetOK) GetPayload() *models.Class { - return o.Payload -} - -func (o *SchemaObjectsGetOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.Class) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewSchemaObjectsGetUnauthorized creates a SchemaObjectsGetUnauthorized with default headers values -func NewSchemaObjectsGetUnauthorized() *SchemaObjectsGetUnauthorized { - return &SchemaObjectsGetUnauthorized{} -} - -/* -SchemaObjectsGetUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type SchemaObjectsGetUnauthorized struct { -} - -// IsSuccess returns true when this schema objects get unauthorized response has a 2xx status code -func (o *SchemaObjectsGetUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this schema objects get unauthorized response has a 3xx status code -func (o *SchemaObjectsGetUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this schema objects get unauthorized response has a 4xx status code -func (o *SchemaObjectsGetUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this schema objects get unauthorized response has a 5xx status code -func (o *SchemaObjectsGetUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this schema objects get unauthorized response a status code equal to that given -func (o *SchemaObjectsGetUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the schema objects get unauthorized response -func (o *SchemaObjectsGetUnauthorized) Code() int { - return 401 -} - -func (o *SchemaObjectsGetUnauthorized) Error() string { - return fmt.Sprintf("[GET /schema/{className}][%d] schemaObjectsGetUnauthorized ", 401) -} - -func (o *SchemaObjectsGetUnauthorized) String() string { - return fmt.Sprintf("[GET /schema/{className}][%d] schemaObjectsGetUnauthorized ", 401) -} - -func (o *SchemaObjectsGetUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewSchemaObjectsGetForbidden creates a SchemaObjectsGetForbidden with default headers values -func NewSchemaObjectsGetForbidden() *SchemaObjectsGetForbidden { - return &SchemaObjectsGetForbidden{} -} - -/* -SchemaObjectsGetForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type SchemaObjectsGetForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this schema objects get forbidden response has a 2xx status code -func (o *SchemaObjectsGetForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this schema objects get forbidden response has a 3xx status code -func (o *SchemaObjectsGetForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this schema objects get forbidden response has a 4xx status code -func (o *SchemaObjectsGetForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this schema objects get forbidden response has a 5xx status code -func (o *SchemaObjectsGetForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this schema objects get forbidden response a status code equal to that given -func (o *SchemaObjectsGetForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the schema objects get forbidden response -func (o *SchemaObjectsGetForbidden) Code() int { - return 403 -} - -func (o *SchemaObjectsGetForbidden) Error() string { - return fmt.Sprintf("[GET /schema/{className}][%d] schemaObjectsGetForbidden %+v", 403, o.Payload) -} - -func (o *SchemaObjectsGetForbidden) String() string { - return fmt.Sprintf("[GET /schema/{className}][%d] schemaObjectsGetForbidden %+v", 403, o.Payload) -} - -func (o *SchemaObjectsGetForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *SchemaObjectsGetForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewSchemaObjectsGetNotFound creates a SchemaObjectsGetNotFound with default headers values -func NewSchemaObjectsGetNotFound() *SchemaObjectsGetNotFound { - return &SchemaObjectsGetNotFound{} -} - -/* -SchemaObjectsGetNotFound describes a response with status code 404, with default header values. - -This class does not exist -*/ -type SchemaObjectsGetNotFound struct { -} - -// IsSuccess returns true when this schema objects get not found response has a 2xx status code -func (o *SchemaObjectsGetNotFound) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this schema objects get not found response has a 3xx status code -func (o *SchemaObjectsGetNotFound) IsRedirect() bool { - return false -} - -// IsClientError returns true when this schema objects get not found response has a 4xx status code -func (o *SchemaObjectsGetNotFound) IsClientError() bool { - return true -} - -// IsServerError returns true when this schema objects get not found response has a 5xx status code -func (o *SchemaObjectsGetNotFound) IsServerError() bool { - return false -} - -// IsCode returns true when this schema objects get not found response a status code equal to that given -func (o *SchemaObjectsGetNotFound) IsCode(code int) bool { - return code == 404 -} - -// Code gets the status code for the schema objects get not found response -func (o *SchemaObjectsGetNotFound) Code() int { - return 404 -} - -func (o *SchemaObjectsGetNotFound) Error() string { - return fmt.Sprintf("[GET /schema/{className}][%d] schemaObjectsGetNotFound ", 404) -} - -func (o *SchemaObjectsGetNotFound) String() string { - return fmt.Sprintf("[GET /schema/{className}][%d] schemaObjectsGetNotFound ", 404) -} - -func (o *SchemaObjectsGetNotFound) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewSchemaObjectsGetInternalServerError creates a SchemaObjectsGetInternalServerError with default headers values -func NewSchemaObjectsGetInternalServerError() *SchemaObjectsGetInternalServerError { - return &SchemaObjectsGetInternalServerError{} -} - -/* -SchemaObjectsGetInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type SchemaObjectsGetInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this schema objects get internal server error response has a 2xx status code -func (o *SchemaObjectsGetInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this schema objects get internal server error response has a 3xx status code -func (o *SchemaObjectsGetInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this schema objects get internal server error response has a 4xx status code -func (o *SchemaObjectsGetInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this schema objects get internal server error response has a 5xx status code -func (o *SchemaObjectsGetInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this schema objects get internal server error response a status code equal to that given -func (o *SchemaObjectsGetInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the schema objects get internal server error response -func (o *SchemaObjectsGetInternalServerError) Code() int { - return 500 -} - -func (o *SchemaObjectsGetInternalServerError) Error() string { - return fmt.Sprintf("[GET /schema/{className}][%d] schemaObjectsGetInternalServerError %+v", 500, o.Payload) -} - -func (o *SchemaObjectsGetInternalServerError) String() string { - return fmt.Sprintf("[GET /schema/{className}][%d] schemaObjectsGetInternalServerError %+v", 500, o.Payload) -} - -func (o *SchemaObjectsGetInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *SchemaObjectsGetInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/schema/schema_objects_properties_add_parameters.go b/client/schema/schema_objects_properties_add_parameters.go deleted file mode 100644 index cc3e046428928e7b241c6d7c89083b0c5c851432..0000000000000000000000000000000000000000 --- a/client/schema/schema_objects_properties_add_parameters.go +++ /dev/null @@ -1,180 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewSchemaObjectsPropertiesAddParams creates a new SchemaObjectsPropertiesAddParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewSchemaObjectsPropertiesAddParams() *SchemaObjectsPropertiesAddParams { - return &SchemaObjectsPropertiesAddParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewSchemaObjectsPropertiesAddParamsWithTimeout creates a new SchemaObjectsPropertiesAddParams object -// with the ability to set a timeout on a request. -func NewSchemaObjectsPropertiesAddParamsWithTimeout(timeout time.Duration) *SchemaObjectsPropertiesAddParams { - return &SchemaObjectsPropertiesAddParams{ - timeout: timeout, - } -} - -// NewSchemaObjectsPropertiesAddParamsWithContext creates a new SchemaObjectsPropertiesAddParams object -// with the ability to set a context for a request. -func NewSchemaObjectsPropertiesAddParamsWithContext(ctx context.Context) *SchemaObjectsPropertiesAddParams { - return &SchemaObjectsPropertiesAddParams{ - Context: ctx, - } -} - -// NewSchemaObjectsPropertiesAddParamsWithHTTPClient creates a new SchemaObjectsPropertiesAddParams object -// with the ability to set a custom HTTPClient for a request. -func NewSchemaObjectsPropertiesAddParamsWithHTTPClient(client *http.Client) *SchemaObjectsPropertiesAddParams { - return &SchemaObjectsPropertiesAddParams{ - HTTPClient: client, - } -} - -/* -SchemaObjectsPropertiesAddParams contains all the parameters to send to the API endpoint - - for the schema objects properties add operation. - - Typically these are written to a http.Request. -*/ -type SchemaObjectsPropertiesAddParams struct { - - // Body. - Body *models.Property - - // ClassName. - ClassName string - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the schema objects properties add params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *SchemaObjectsPropertiesAddParams) WithDefaults() *SchemaObjectsPropertiesAddParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the schema objects properties add params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *SchemaObjectsPropertiesAddParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the schema objects properties add params -func (o *SchemaObjectsPropertiesAddParams) WithTimeout(timeout time.Duration) *SchemaObjectsPropertiesAddParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the schema objects properties add params -func (o *SchemaObjectsPropertiesAddParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the schema objects properties add params -func (o *SchemaObjectsPropertiesAddParams) WithContext(ctx context.Context) *SchemaObjectsPropertiesAddParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the schema objects properties add params -func (o *SchemaObjectsPropertiesAddParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the schema objects properties add params -func (o *SchemaObjectsPropertiesAddParams) WithHTTPClient(client *http.Client) *SchemaObjectsPropertiesAddParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the schema objects properties add params -func (o *SchemaObjectsPropertiesAddParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithBody adds the body to the schema objects properties add params -func (o *SchemaObjectsPropertiesAddParams) WithBody(body *models.Property) *SchemaObjectsPropertiesAddParams { - o.SetBody(body) - return o -} - -// SetBody adds the body to the schema objects properties add params -func (o *SchemaObjectsPropertiesAddParams) SetBody(body *models.Property) { - o.Body = body -} - -// WithClassName adds the className to the schema objects properties add params -func (o *SchemaObjectsPropertiesAddParams) WithClassName(className string) *SchemaObjectsPropertiesAddParams { - o.SetClassName(className) - return o -} - -// SetClassName adds the className to the schema objects properties add params -func (o *SchemaObjectsPropertiesAddParams) SetClassName(className string) { - o.ClassName = className -} - -// WriteToRequest writes these params to a swagger request -func (o *SchemaObjectsPropertiesAddParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - if o.Body != nil { - if err := r.SetBodyParam(o.Body); err != nil { - return err - } - } - - // path param className - if err := r.SetPathParam("className", o.ClassName); err != nil { - return err - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/schema/schema_objects_properties_add_responses.go b/client/schema/schema_objects_properties_add_responses.go deleted file mode 100644 index 29b12589db74dfd021dab6cb5c8385feabc876fd..0000000000000000000000000000000000000000 --- a/client/schema/schema_objects_properties_add_responses.go +++ /dev/null @@ -1,398 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// SchemaObjectsPropertiesAddReader is a Reader for the SchemaObjectsPropertiesAdd structure. -type SchemaObjectsPropertiesAddReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *SchemaObjectsPropertiesAddReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 200: - result := NewSchemaObjectsPropertiesAddOK() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 401: - result := NewSchemaObjectsPropertiesAddUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewSchemaObjectsPropertiesAddForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 422: - result := NewSchemaObjectsPropertiesAddUnprocessableEntity() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewSchemaObjectsPropertiesAddInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewSchemaObjectsPropertiesAddOK creates a SchemaObjectsPropertiesAddOK with default headers values -func NewSchemaObjectsPropertiesAddOK() *SchemaObjectsPropertiesAddOK { - return &SchemaObjectsPropertiesAddOK{} -} - -/* -SchemaObjectsPropertiesAddOK describes a response with status code 200, with default header values. - -Added the property. -*/ -type SchemaObjectsPropertiesAddOK struct { - Payload *models.Property -} - -// IsSuccess returns true when this schema objects properties add o k response has a 2xx status code -func (o *SchemaObjectsPropertiesAddOK) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this schema objects properties add o k response has a 3xx status code -func (o *SchemaObjectsPropertiesAddOK) IsRedirect() bool { - return false -} - -// IsClientError returns true when this schema objects properties add o k response has a 4xx status code -func (o *SchemaObjectsPropertiesAddOK) IsClientError() bool { - return false -} - -// IsServerError returns true when this schema objects properties add o k response has a 5xx status code -func (o *SchemaObjectsPropertiesAddOK) IsServerError() bool { - return false -} - -// IsCode returns true when this schema objects properties add o k response a status code equal to that given -func (o *SchemaObjectsPropertiesAddOK) IsCode(code int) bool { - return code == 200 -} - -// Code gets the status code for the schema objects properties add o k response -func (o *SchemaObjectsPropertiesAddOK) Code() int { - return 200 -} - -func (o *SchemaObjectsPropertiesAddOK) Error() string { - return fmt.Sprintf("[POST /schema/{className}/properties][%d] schemaObjectsPropertiesAddOK %+v", 200, o.Payload) -} - -func (o *SchemaObjectsPropertiesAddOK) String() string { - return fmt.Sprintf("[POST /schema/{className}/properties][%d] schemaObjectsPropertiesAddOK %+v", 200, o.Payload) -} - -func (o *SchemaObjectsPropertiesAddOK) GetPayload() *models.Property { - return o.Payload -} - -func (o *SchemaObjectsPropertiesAddOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.Property) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewSchemaObjectsPropertiesAddUnauthorized creates a SchemaObjectsPropertiesAddUnauthorized with default headers values -func NewSchemaObjectsPropertiesAddUnauthorized() *SchemaObjectsPropertiesAddUnauthorized { - return &SchemaObjectsPropertiesAddUnauthorized{} -} - -/* -SchemaObjectsPropertiesAddUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type SchemaObjectsPropertiesAddUnauthorized struct { -} - -// IsSuccess returns true when this schema objects properties add unauthorized response has a 2xx status code -func (o *SchemaObjectsPropertiesAddUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this schema objects properties add unauthorized response has a 3xx status code -func (o *SchemaObjectsPropertiesAddUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this schema objects properties add unauthorized response has a 4xx status code -func (o *SchemaObjectsPropertiesAddUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this schema objects properties add unauthorized response has a 5xx status code -func (o *SchemaObjectsPropertiesAddUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this schema objects properties add unauthorized response a status code equal to that given -func (o *SchemaObjectsPropertiesAddUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the schema objects properties add unauthorized response -func (o *SchemaObjectsPropertiesAddUnauthorized) Code() int { - return 401 -} - -func (o *SchemaObjectsPropertiesAddUnauthorized) Error() string { - return fmt.Sprintf("[POST /schema/{className}/properties][%d] schemaObjectsPropertiesAddUnauthorized ", 401) -} - -func (o *SchemaObjectsPropertiesAddUnauthorized) String() string { - return fmt.Sprintf("[POST /schema/{className}/properties][%d] schemaObjectsPropertiesAddUnauthorized ", 401) -} - -func (o *SchemaObjectsPropertiesAddUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewSchemaObjectsPropertiesAddForbidden creates a SchemaObjectsPropertiesAddForbidden with default headers values -func NewSchemaObjectsPropertiesAddForbidden() *SchemaObjectsPropertiesAddForbidden { - return &SchemaObjectsPropertiesAddForbidden{} -} - -/* -SchemaObjectsPropertiesAddForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type SchemaObjectsPropertiesAddForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this schema objects properties add forbidden response has a 2xx status code -func (o *SchemaObjectsPropertiesAddForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this schema objects properties add forbidden response has a 3xx status code -func (o *SchemaObjectsPropertiesAddForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this schema objects properties add forbidden response has a 4xx status code -func (o *SchemaObjectsPropertiesAddForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this schema objects properties add forbidden response has a 5xx status code -func (o *SchemaObjectsPropertiesAddForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this schema objects properties add forbidden response a status code equal to that given -func (o *SchemaObjectsPropertiesAddForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the schema objects properties add forbidden response -func (o *SchemaObjectsPropertiesAddForbidden) Code() int { - return 403 -} - -func (o *SchemaObjectsPropertiesAddForbidden) Error() string { - return fmt.Sprintf("[POST /schema/{className}/properties][%d] schemaObjectsPropertiesAddForbidden %+v", 403, o.Payload) -} - -func (o *SchemaObjectsPropertiesAddForbidden) String() string { - return fmt.Sprintf("[POST /schema/{className}/properties][%d] schemaObjectsPropertiesAddForbidden %+v", 403, o.Payload) -} - -func (o *SchemaObjectsPropertiesAddForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *SchemaObjectsPropertiesAddForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewSchemaObjectsPropertiesAddUnprocessableEntity creates a SchemaObjectsPropertiesAddUnprocessableEntity with default headers values -func NewSchemaObjectsPropertiesAddUnprocessableEntity() *SchemaObjectsPropertiesAddUnprocessableEntity { - return &SchemaObjectsPropertiesAddUnprocessableEntity{} -} - -/* -SchemaObjectsPropertiesAddUnprocessableEntity describes a response with status code 422, with default header values. - -Invalid property. -*/ -type SchemaObjectsPropertiesAddUnprocessableEntity struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this schema objects properties add unprocessable entity response has a 2xx status code -func (o *SchemaObjectsPropertiesAddUnprocessableEntity) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this schema objects properties add unprocessable entity response has a 3xx status code -func (o *SchemaObjectsPropertiesAddUnprocessableEntity) IsRedirect() bool { - return false -} - -// IsClientError returns true when this schema objects properties add unprocessable entity response has a 4xx status code -func (o *SchemaObjectsPropertiesAddUnprocessableEntity) IsClientError() bool { - return true -} - -// IsServerError returns true when this schema objects properties add unprocessable entity response has a 5xx status code -func (o *SchemaObjectsPropertiesAddUnprocessableEntity) IsServerError() bool { - return false -} - -// IsCode returns true when this schema objects properties add unprocessable entity response a status code equal to that given -func (o *SchemaObjectsPropertiesAddUnprocessableEntity) IsCode(code int) bool { - return code == 422 -} - -// Code gets the status code for the schema objects properties add unprocessable entity response -func (o *SchemaObjectsPropertiesAddUnprocessableEntity) Code() int { - return 422 -} - -func (o *SchemaObjectsPropertiesAddUnprocessableEntity) Error() string { - return fmt.Sprintf("[POST /schema/{className}/properties][%d] schemaObjectsPropertiesAddUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *SchemaObjectsPropertiesAddUnprocessableEntity) String() string { - return fmt.Sprintf("[POST /schema/{className}/properties][%d] schemaObjectsPropertiesAddUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *SchemaObjectsPropertiesAddUnprocessableEntity) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *SchemaObjectsPropertiesAddUnprocessableEntity) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewSchemaObjectsPropertiesAddInternalServerError creates a SchemaObjectsPropertiesAddInternalServerError with default headers values -func NewSchemaObjectsPropertiesAddInternalServerError() *SchemaObjectsPropertiesAddInternalServerError { - return &SchemaObjectsPropertiesAddInternalServerError{} -} - -/* -SchemaObjectsPropertiesAddInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type SchemaObjectsPropertiesAddInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this schema objects properties add internal server error response has a 2xx status code -func (o *SchemaObjectsPropertiesAddInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this schema objects properties add internal server error response has a 3xx status code -func (o *SchemaObjectsPropertiesAddInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this schema objects properties add internal server error response has a 4xx status code -func (o *SchemaObjectsPropertiesAddInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this schema objects properties add internal server error response has a 5xx status code -func (o *SchemaObjectsPropertiesAddInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this schema objects properties add internal server error response a status code equal to that given -func (o *SchemaObjectsPropertiesAddInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the schema objects properties add internal server error response -func (o *SchemaObjectsPropertiesAddInternalServerError) Code() int { - return 500 -} - -func (o *SchemaObjectsPropertiesAddInternalServerError) Error() string { - return fmt.Sprintf("[POST /schema/{className}/properties][%d] schemaObjectsPropertiesAddInternalServerError %+v", 500, o.Payload) -} - -func (o *SchemaObjectsPropertiesAddInternalServerError) String() string { - return fmt.Sprintf("[POST /schema/{className}/properties][%d] schemaObjectsPropertiesAddInternalServerError %+v", 500, o.Payload) -} - -func (o *SchemaObjectsPropertiesAddInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *SchemaObjectsPropertiesAddInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/schema/schema_objects_shards_get_parameters.go b/client/schema/schema_objects_shards_get_parameters.go deleted file mode 100644 index 33c26baf366e2da444d9b3150c81500c3b4adae0..0000000000000000000000000000000000000000 --- a/client/schema/schema_objects_shards_get_parameters.go +++ /dev/null @@ -1,190 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" -) - -// NewSchemaObjectsShardsGetParams creates a new SchemaObjectsShardsGetParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewSchemaObjectsShardsGetParams() *SchemaObjectsShardsGetParams { - return &SchemaObjectsShardsGetParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewSchemaObjectsShardsGetParamsWithTimeout creates a new SchemaObjectsShardsGetParams object -// with the ability to set a timeout on a request. -func NewSchemaObjectsShardsGetParamsWithTimeout(timeout time.Duration) *SchemaObjectsShardsGetParams { - return &SchemaObjectsShardsGetParams{ - timeout: timeout, - } -} - -// NewSchemaObjectsShardsGetParamsWithContext creates a new SchemaObjectsShardsGetParams object -// with the ability to set a context for a request. -func NewSchemaObjectsShardsGetParamsWithContext(ctx context.Context) *SchemaObjectsShardsGetParams { - return &SchemaObjectsShardsGetParams{ - Context: ctx, - } -} - -// NewSchemaObjectsShardsGetParamsWithHTTPClient creates a new SchemaObjectsShardsGetParams object -// with the ability to set a custom HTTPClient for a request. -func NewSchemaObjectsShardsGetParamsWithHTTPClient(client *http.Client) *SchemaObjectsShardsGetParams { - return &SchemaObjectsShardsGetParams{ - HTTPClient: client, - } -} - -/* -SchemaObjectsShardsGetParams contains all the parameters to send to the API endpoint - - for the schema objects shards get operation. - - Typically these are written to a http.Request. -*/ -type SchemaObjectsShardsGetParams struct { - - // ClassName. - ClassName string - - // Tenant. - Tenant *string - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the schema objects shards get params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *SchemaObjectsShardsGetParams) WithDefaults() *SchemaObjectsShardsGetParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the schema objects shards get params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *SchemaObjectsShardsGetParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the schema objects shards get params -func (o *SchemaObjectsShardsGetParams) WithTimeout(timeout time.Duration) *SchemaObjectsShardsGetParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the schema objects shards get params -func (o *SchemaObjectsShardsGetParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the schema objects shards get params -func (o *SchemaObjectsShardsGetParams) WithContext(ctx context.Context) *SchemaObjectsShardsGetParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the schema objects shards get params -func (o *SchemaObjectsShardsGetParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the schema objects shards get params -func (o *SchemaObjectsShardsGetParams) WithHTTPClient(client *http.Client) *SchemaObjectsShardsGetParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the schema objects shards get params -func (o *SchemaObjectsShardsGetParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithClassName adds the className to the schema objects shards get params -func (o *SchemaObjectsShardsGetParams) WithClassName(className string) *SchemaObjectsShardsGetParams { - o.SetClassName(className) - return o -} - -// SetClassName adds the className to the schema objects shards get params -func (o *SchemaObjectsShardsGetParams) SetClassName(className string) { - o.ClassName = className -} - -// WithTenant adds the tenant to the schema objects shards get params -func (o *SchemaObjectsShardsGetParams) WithTenant(tenant *string) *SchemaObjectsShardsGetParams { - o.SetTenant(tenant) - return o -} - -// SetTenant adds the tenant to the schema objects shards get params -func (o *SchemaObjectsShardsGetParams) SetTenant(tenant *string) { - o.Tenant = tenant -} - -// WriteToRequest writes these params to a swagger request -func (o *SchemaObjectsShardsGetParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - - // path param className - if err := r.SetPathParam("className", o.ClassName); err != nil { - return err - } - - if o.Tenant != nil { - - // query param tenant - var qrTenant string - - if o.Tenant != nil { - qrTenant = *o.Tenant - } - qTenant := qrTenant - if qTenant != "" { - - if err := r.SetQueryParam("tenant", qTenant); err != nil { - return err - } - } - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/schema/schema_objects_shards_get_responses.go b/client/schema/schema_objects_shards_get_responses.go deleted file mode 100644 index b9395e6e1507eb434bf0cab6b4edda9a6a5c0cc7..0000000000000000000000000000000000000000 --- a/client/schema/schema_objects_shards_get_responses.go +++ /dev/null @@ -1,396 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// SchemaObjectsShardsGetReader is a Reader for the SchemaObjectsShardsGet structure. -type SchemaObjectsShardsGetReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *SchemaObjectsShardsGetReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 200: - result := NewSchemaObjectsShardsGetOK() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 401: - result := NewSchemaObjectsShardsGetUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewSchemaObjectsShardsGetForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 404: - result := NewSchemaObjectsShardsGetNotFound() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewSchemaObjectsShardsGetInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewSchemaObjectsShardsGetOK creates a SchemaObjectsShardsGetOK with default headers values -func NewSchemaObjectsShardsGetOK() *SchemaObjectsShardsGetOK { - return &SchemaObjectsShardsGetOK{} -} - -/* -SchemaObjectsShardsGetOK describes a response with status code 200, with default header values. - -Found the status of the shards, returned as body -*/ -type SchemaObjectsShardsGetOK struct { - Payload models.ShardStatusList -} - -// IsSuccess returns true when this schema objects shards get o k response has a 2xx status code -func (o *SchemaObjectsShardsGetOK) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this schema objects shards get o k response has a 3xx status code -func (o *SchemaObjectsShardsGetOK) IsRedirect() bool { - return false -} - -// IsClientError returns true when this schema objects shards get o k response has a 4xx status code -func (o *SchemaObjectsShardsGetOK) IsClientError() bool { - return false -} - -// IsServerError returns true when this schema objects shards get o k response has a 5xx status code -func (o *SchemaObjectsShardsGetOK) IsServerError() bool { - return false -} - -// IsCode returns true when this schema objects shards get o k response a status code equal to that given -func (o *SchemaObjectsShardsGetOK) IsCode(code int) bool { - return code == 200 -} - -// Code gets the status code for the schema objects shards get o k response -func (o *SchemaObjectsShardsGetOK) Code() int { - return 200 -} - -func (o *SchemaObjectsShardsGetOK) Error() string { - return fmt.Sprintf("[GET /schema/{className}/shards][%d] schemaObjectsShardsGetOK %+v", 200, o.Payload) -} - -func (o *SchemaObjectsShardsGetOK) String() string { - return fmt.Sprintf("[GET /schema/{className}/shards][%d] schemaObjectsShardsGetOK %+v", 200, o.Payload) -} - -func (o *SchemaObjectsShardsGetOK) GetPayload() models.ShardStatusList { - return o.Payload -} - -func (o *SchemaObjectsShardsGetOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - // response payload - if err := consumer.Consume(response.Body(), &o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewSchemaObjectsShardsGetUnauthorized creates a SchemaObjectsShardsGetUnauthorized with default headers values -func NewSchemaObjectsShardsGetUnauthorized() *SchemaObjectsShardsGetUnauthorized { - return &SchemaObjectsShardsGetUnauthorized{} -} - -/* -SchemaObjectsShardsGetUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type SchemaObjectsShardsGetUnauthorized struct { -} - -// IsSuccess returns true when this schema objects shards get unauthorized response has a 2xx status code -func (o *SchemaObjectsShardsGetUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this schema objects shards get unauthorized response has a 3xx status code -func (o *SchemaObjectsShardsGetUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this schema objects shards get unauthorized response has a 4xx status code -func (o *SchemaObjectsShardsGetUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this schema objects shards get unauthorized response has a 5xx status code -func (o *SchemaObjectsShardsGetUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this schema objects shards get unauthorized response a status code equal to that given -func (o *SchemaObjectsShardsGetUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the schema objects shards get unauthorized response -func (o *SchemaObjectsShardsGetUnauthorized) Code() int { - return 401 -} - -func (o *SchemaObjectsShardsGetUnauthorized) Error() string { - return fmt.Sprintf("[GET /schema/{className}/shards][%d] schemaObjectsShardsGetUnauthorized ", 401) -} - -func (o *SchemaObjectsShardsGetUnauthorized) String() string { - return fmt.Sprintf("[GET /schema/{className}/shards][%d] schemaObjectsShardsGetUnauthorized ", 401) -} - -func (o *SchemaObjectsShardsGetUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewSchemaObjectsShardsGetForbidden creates a SchemaObjectsShardsGetForbidden with default headers values -func NewSchemaObjectsShardsGetForbidden() *SchemaObjectsShardsGetForbidden { - return &SchemaObjectsShardsGetForbidden{} -} - -/* -SchemaObjectsShardsGetForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type SchemaObjectsShardsGetForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this schema objects shards get forbidden response has a 2xx status code -func (o *SchemaObjectsShardsGetForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this schema objects shards get forbidden response has a 3xx status code -func (o *SchemaObjectsShardsGetForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this schema objects shards get forbidden response has a 4xx status code -func (o *SchemaObjectsShardsGetForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this schema objects shards get forbidden response has a 5xx status code -func (o *SchemaObjectsShardsGetForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this schema objects shards get forbidden response a status code equal to that given -func (o *SchemaObjectsShardsGetForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the schema objects shards get forbidden response -func (o *SchemaObjectsShardsGetForbidden) Code() int { - return 403 -} - -func (o *SchemaObjectsShardsGetForbidden) Error() string { - return fmt.Sprintf("[GET /schema/{className}/shards][%d] schemaObjectsShardsGetForbidden %+v", 403, o.Payload) -} - -func (o *SchemaObjectsShardsGetForbidden) String() string { - return fmt.Sprintf("[GET /schema/{className}/shards][%d] schemaObjectsShardsGetForbidden %+v", 403, o.Payload) -} - -func (o *SchemaObjectsShardsGetForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *SchemaObjectsShardsGetForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewSchemaObjectsShardsGetNotFound creates a SchemaObjectsShardsGetNotFound with default headers values -func NewSchemaObjectsShardsGetNotFound() *SchemaObjectsShardsGetNotFound { - return &SchemaObjectsShardsGetNotFound{} -} - -/* -SchemaObjectsShardsGetNotFound describes a response with status code 404, with default header values. - -This class does not exist -*/ -type SchemaObjectsShardsGetNotFound struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this schema objects shards get not found response has a 2xx status code -func (o *SchemaObjectsShardsGetNotFound) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this schema objects shards get not found response has a 3xx status code -func (o *SchemaObjectsShardsGetNotFound) IsRedirect() bool { - return false -} - -// IsClientError returns true when this schema objects shards get not found response has a 4xx status code -func (o *SchemaObjectsShardsGetNotFound) IsClientError() bool { - return true -} - -// IsServerError returns true when this schema objects shards get not found response has a 5xx status code -func (o *SchemaObjectsShardsGetNotFound) IsServerError() bool { - return false -} - -// IsCode returns true when this schema objects shards get not found response a status code equal to that given -func (o *SchemaObjectsShardsGetNotFound) IsCode(code int) bool { - return code == 404 -} - -// Code gets the status code for the schema objects shards get not found response -func (o *SchemaObjectsShardsGetNotFound) Code() int { - return 404 -} - -func (o *SchemaObjectsShardsGetNotFound) Error() string { - return fmt.Sprintf("[GET /schema/{className}/shards][%d] schemaObjectsShardsGetNotFound %+v", 404, o.Payload) -} - -func (o *SchemaObjectsShardsGetNotFound) String() string { - return fmt.Sprintf("[GET /schema/{className}/shards][%d] schemaObjectsShardsGetNotFound %+v", 404, o.Payload) -} - -func (o *SchemaObjectsShardsGetNotFound) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *SchemaObjectsShardsGetNotFound) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewSchemaObjectsShardsGetInternalServerError creates a SchemaObjectsShardsGetInternalServerError with default headers values -func NewSchemaObjectsShardsGetInternalServerError() *SchemaObjectsShardsGetInternalServerError { - return &SchemaObjectsShardsGetInternalServerError{} -} - -/* -SchemaObjectsShardsGetInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type SchemaObjectsShardsGetInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this schema objects shards get internal server error response has a 2xx status code -func (o *SchemaObjectsShardsGetInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this schema objects shards get internal server error response has a 3xx status code -func (o *SchemaObjectsShardsGetInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this schema objects shards get internal server error response has a 4xx status code -func (o *SchemaObjectsShardsGetInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this schema objects shards get internal server error response has a 5xx status code -func (o *SchemaObjectsShardsGetInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this schema objects shards get internal server error response a status code equal to that given -func (o *SchemaObjectsShardsGetInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the schema objects shards get internal server error response -func (o *SchemaObjectsShardsGetInternalServerError) Code() int { - return 500 -} - -func (o *SchemaObjectsShardsGetInternalServerError) Error() string { - return fmt.Sprintf("[GET /schema/{className}/shards][%d] schemaObjectsShardsGetInternalServerError %+v", 500, o.Payload) -} - -func (o *SchemaObjectsShardsGetInternalServerError) String() string { - return fmt.Sprintf("[GET /schema/{className}/shards][%d] schemaObjectsShardsGetInternalServerError %+v", 500, o.Payload) -} - -func (o *SchemaObjectsShardsGetInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *SchemaObjectsShardsGetInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/schema/schema_objects_shards_update_parameters.go b/client/schema/schema_objects_shards_update_parameters.go deleted file mode 100644 index 21583fdfef2d0bba8019e0e4a3bd090ee630e2ed..0000000000000000000000000000000000000000 --- a/client/schema/schema_objects_shards_update_parameters.go +++ /dev/null @@ -1,199 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewSchemaObjectsShardsUpdateParams creates a new SchemaObjectsShardsUpdateParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewSchemaObjectsShardsUpdateParams() *SchemaObjectsShardsUpdateParams { - return &SchemaObjectsShardsUpdateParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewSchemaObjectsShardsUpdateParamsWithTimeout creates a new SchemaObjectsShardsUpdateParams object -// with the ability to set a timeout on a request. -func NewSchemaObjectsShardsUpdateParamsWithTimeout(timeout time.Duration) *SchemaObjectsShardsUpdateParams { - return &SchemaObjectsShardsUpdateParams{ - timeout: timeout, - } -} - -// NewSchemaObjectsShardsUpdateParamsWithContext creates a new SchemaObjectsShardsUpdateParams object -// with the ability to set a context for a request. -func NewSchemaObjectsShardsUpdateParamsWithContext(ctx context.Context) *SchemaObjectsShardsUpdateParams { - return &SchemaObjectsShardsUpdateParams{ - Context: ctx, - } -} - -// NewSchemaObjectsShardsUpdateParamsWithHTTPClient creates a new SchemaObjectsShardsUpdateParams object -// with the ability to set a custom HTTPClient for a request. -func NewSchemaObjectsShardsUpdateParamsWithHTTPClient(client *http.Client) *SchemaObjectsShardsUpdateParams { - return &SchemaObjectsShardsUpdateParams{ - HTTPClient: client, - } -} - -/* -SchemaObjectsShardsUpdateParams contains all the parameters to send to the API endpoint - - for the schema objects shards update operation. - - Typically these are written to a http.Request. -*/ -type SchemaObjectsShardsUpdateParams struct { - - // Body. - Body *models.ShardStatus - - // ClassName. - ClassName string - - // ShardName. - ShardName string - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the schema objects shards update params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *SchemaObjectsShardsUpdateParams) WithDefaults() *SchemaObjectsShardsUpdateParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the schema objects shards update params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *SchemaObjectsShardsUpdateParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the schema objects shards update params -func (o *SchemaObjectsShardsUpdateParams) WithTimeout(timeout time.Duration) *SchemaObjectsShardsUpdateParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the schema objects shards update params -func (o *SchemaObjectsShardsUpdateParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the schema objects shards update params -func (o *SchemaObjectsShardsUpdateParams) WithContext(ctx context.Context) *SchemaObjectsShardsUpdateParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the schema objects shards update params -func (o *SchemaObjectsShardsUpdateParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the schema objects shards update params -func (o *SchemaObjectsShardsUpdateParams) WithHTTPClient(client *http.Client) *SchemaObjectsShardsUpdateParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the schema objects shards update params -func (o *SchemaObjectsShardsUpdateParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithBody adds the body to the schema objects shards update params -func (o *SchemaObjectsShardsUpdateParams) WithBody(body *models.ShardStatus) *SchemaObjectsShardsUpdateParams { - o.SetBody(body) - return o -} - -// SetBody adds the body to the schema objects shards update params -func (o *SchemaObjectsShardsUpdateParams) SetBody(body *models.ShardStatus) { - o.Body = body -} - -// WithClassName adds the className to the schema objects shards update params -func (o *SchemaObjectsShardsUpdateParams) WithClassName(className string) *SchemaObjectsShardsUpdateParams { - o.SetClassName(className) - return o -} - -// SetClassName adds the className to the schema objects shards update params -func (o *SchemaObjectsShardsUpdateParams) SetClassName(className string) { - o.ClassName = className -} - -// WithShardName adds the shardName to the schema objects shards update params -func (o *SchemaObjectsShardsUpdateParams) WithShardName(shardName string) *SchemaObjectsShardsUpdateParams { - o.SetShardName(shardName) - return o -} - -// SetShardName adds the shardName to the schema objects shards update params -func (o *SchemaObjectsShardsUpdateParams) SetShardName(shardName string) { - o.ShardName = shardName -} - -// WriteToRequest writes these params to a swagger request -func (o *SchemaObjectsShardsUpdateParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - if o.Body != nil { - if err := r.SetBodyParam(o.Body); err != nil { - return err - } - } - - // path param className - if err := r.SetPathParam("className", o.ClassName); err != nil { - return err - } - - // path param shardName - if err := r.SetPathParam("shardName", o.ShardName); err != nil { - return err - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/schema/schema_objects_shards_update_responses.go b/client/schema/schema_objects_shards_update_responses.go deleted file mode 100644 index 1c49428dcbf1a84bc713999791c6adc12000246c..0000000000000000000000000000000000000000 --- a/client/schema/schema_objects_shards_update_responses.go +++ /dev/null @@ -1,472 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// SchemaObjectsShardsUpdateReader is a Reader for the SchemaObjectsShardsUpdate structure. -type SchemaObjectsShardsUpdateReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *SchemaObjectsShardsUpdateReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 200: - result := NewSchemaObjectsShardsUpdateOK() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 401: - result := NewSchemaObjectsShardsUpdateUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewSchemaObjectsShardsUpdateForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 404: - result := NewSchemaObjectsShardsUpdateNotFound() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 422: - result := NewSchemaObjectsShardsUpdateUnprocessableEntity() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewSchemaObjectsShardsUpdateInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewSchemaObjectsShardsUpdateOK creates a SchemaObjectsShardsUpdateOK with default headers values -func NewSchemaObjectsShardsUpdateOK() *SchemaObjectsShardsUpdateOK { - return &SchemaObjectsShardsUpdateOK{} -} - -/* -SchemaObjectsShardsUpdateOK describes a response with status code 200, with default header values. - -Shard status was updated successfully -*/ -type SchemaObjectsShardsUpdateOK struct { - Payload *models.ShardStatus -} - -// IsSuccess returns true when this schema objects shards update o k response has a 2xx status code -func (o *SchemaObjectsShardsUpdateOK) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this schema objects shards update o k response has a 3xx status code -func (o *SchemaObjectsShardsUpdateOK) IsRedirect() bool { - return false -} - -// IsClientError returns true when this schema objects shards update o k response has a 4xx status code -func (o *SchemaObjectsShardsUpdateOK) IsClientError() bool { - return false -} - -// IsServerError returns true when this schema objects shards update o k response has a 5xx status code -func (o *SchemaObjectsShardsUpdateOK) IsServerError() bool { - return false -} - -// IsCode returns true when this schema objects shards update o k response a status code equal to that given -func (o *SchemaObjectsShardsUpdateOK) IsCode(code int) bool { - return code == 200 -} - -// Code gets the status code for the schema objects shards update o k response -func (o *SchemaObjectsShardsUpdateOK) Code() int { - return 200 -} - -func (o *SchemaObjectsShardsUpdateOK) Error() string { - return fmt.Sprintf("[PUT /schema/{className}/shards/{shardName}][%d] schemaObjectsShardsUpdateOK %+v", 200, o.Payload) -} - -func (o *SchemaObjectsShardsUpdateOK) String() string { - return fmt.Sprintf("[PUT /schema/{className}/shards/{shardName}][%d] schemaObjectsShardsUpdateOK %+v", 200, o.Payload) -} - -func (o *SchemaObjectsShardsUpdateOK) GetPayload() *models.ShardStatus { - return o.Payload -} - -func (o *SchemaObjectsShardsUpdateOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ShardStatus) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewSchemaObjectsShardsUpdateUnauthorized creates a SchemaObjectsShardsUpdateUnauthorized with default headers values -func NewSchemaObjectsShardsUpdateUnauthorized() *SchemaObjectsShardsUpdateUnauthorized { - return &SchemaObjectsShardsUpdateUnauthorized{} -} - -/* -SchemaObjectsShardsUpdateUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type SchemaObjectsShardsUpdateUnauthorized struct { -} - -// IsSuccess returns true when this schema objects shards update unauthorized response has a 2xx status code -func (o *SchemaObjectsShardsUpdateUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this schema objects shards update unauthorized response has a 3xx status code -func (o *SchemaObjectsShardsUpdateUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this schema objects shards update unauthorized response has a 4xx status code -func (o *SchemaObjectsShardsUpdateUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this schema objects shards update unauthorized response has a 5xx status code -func (o *SchemaObjectsShardsUpdateUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this schema objects shards update unauthorized response a status code equal to that given -func (o *SchemaObjectsShardsUpdateUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the schema objects shards update unauthorized response -func (o *SchemaObjectsShardsUpdateUnauthorized) Code() int { - return 401 -} - -func (o *SchemaObjectsShardsUpdateUnauthorized) Error() string { - return fmt.Sprintf("[PUT /schema/{className}/shards/{shardName}][%d] schemaObjectsShardsUpdateUnauthorized ", 401) -} - -func (o *SchemaObjectsShardsUpdateUnauthorized) String() string { - return fmt.Sprintf("[PUT /schema/{className}/shards/{shardName}][%d] schemaObjectsShardsUpdateUnauthorized ", 401) -} - -func (o *SchemaObjectsShardsUpdateUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewSchemaObjectsShardsUpdateForbidden creates a SchemaObjectsShardsUpdateForbidden with default headers values -func NewSchemaObjectsShardsUpdateForbidden() *SchemaObjectsShardsUpdateForbidden { - return &SchemaObjectsShardsUpdateForbidden{} -} - -/* -SchemaObjectsShardsUpdateForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type SchemaObjectsShardsUpdateForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this schema objects shards update forbidden response has a 2xx status code -func (o *SchemaObjectsShardsUpdateForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this schema objects shards update forbidden response has a 3xx status code -func (o *SchemaObjectsShardsUpdateForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this schema objects shards update forbidden response has a 4xx status code -func (o *SchemaObjectsShardsUpdateForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this schema objects shards update forbidden response has a 5xx status code -func (o *SchemaObjectsShardsUpdateForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this schema objects shards update forbidden response a status code equal to that given -func (o *SchemaObjectsShardsUpdateForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the schema objects shards update forbidden response -func (o *SchemaObjectsShardsUpdateForbidden) Code() int { - return 403 -} - -func (o *SchemaObjectsShardsUpdateForbidden) Error() string { - return fmt.Sprintf("[PUT /schema/{className}/shards/{shardName}][%d] schemaObjectsShardsUpdateForbidden %+v", 403, o.Payload) -} - -func (o *SchemaObjectsShardsUpdateForbidden) String() string { - return fmt.Sprintf("[PUT /schema/{className}/shards/{shardName}][%d] schemaObjectsShardsUpdateForbidden %+v", 403, o.Payload) -} - -func (o *SchemaObjectsShardsUpdateForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *SchemaObjectsShardsUpdateForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewSchemaObjectsShardsUpdateNotFound creates a SchemaObjectsShardsUpdateNotFound with default headers values -func NewSchemaObjectsShardsUpdateNotFound() *SchemaObjectsShardsUpdateNotFound { - return &SchemaObjectsShardsUpdateNotFound{} -} - -/* -SchemaObjectsShardsUpdateNotFound describes a response with status code 404, with default header values. - -Shard to be updated does not exist -*/ -type SchemaObjectsShardsUpdateNotFound struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this schema objects shards update not found response has a 2xx status code -func (o *SchemaObjectsShardsUpdateNotFound) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this schema objects shards update not found response has a 3xx status code -func (o *SchemaObjectsShardsUpdateNotFound) IsRedirect() bool { - return false -} - -// IsClientError returns true when this schema objects shards update not found response has a 4xx status code -func (o *SchemaObjectsShardsUpdateNotFound) IsClientError() bool { - return true -} - -// IsServerError returns true when this schema objects shards update not found response has a 5xx status code -func (o *SchemaObjectsShardsUpdateNotFound) IsServerError() bool { - return false -} - -// IsCode returns true when this schema objects shards update not found response a status code equal to that given -func (o *SchemaObjectsShardsUpdateNotFound) IsCode(code int) bool { - return code == 404 -} - -// Code gets the status code for the schema objects shards update not found response -func (o *SchemaObjectsShardsUpdateNotFound) Code() int { - return 404 -} - -func (o *SchemaObjectsShardsUpdateNotFound) Error() string { - return fmt.Sprintf("[PUT /schema/{className}/shards/{shardName}][%d] schemaObjectsShardsUpdateNotFound %+v", 404, o.Payload) -} - -func (o *SchemaObjectsShardsUpdateNotFound) String() string { - return fmt.Sprintf("[PUT /schema/{className}/shards/{shardName}][%d] schemaObjectsShardsUpdateNotFound %+v", 404, o.Payload) -} - -func (o *SchemaObjectsShardsUpdateNotFound) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *SchemaObjectsShardsUpdateNotFound) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewSchemaObjectsShardsUpdateUnprocessableEntity creates a SchemaObjectsShardsUpdateUnprocessableEntity with default headers values -func NewSchemaObjectsShardsUpdateUnprocessableEntity() *SchemaObjectsShardsUpdateUnprocessableEntity { - return &SchemaObjectsShardsUpdateUnprocessableEntity{} -} - -/* -SchemaObjectsShardsUpdateUnprocessableEntity describes a response with status code 422, with default header values. - -Invalid update attempt -*/ -type SchemaObjectsShardsUpdateUnprocessableEntity struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this schema objects shards update unprocessable entity response has a 2xx status code -func (o *SchemaObjectsShardsUpdateUnprocessableEntity) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this schema objects shards update unprocessable entity response has a 3xx status code -func (o *SchemaObjectsShardsUpdateUnprocessableEntity) IsRedirect() bool { - return false -} - -// IsClientError returns true when this schema objects shards update unprocessable entity response has a 4xx status code -func (o *SchemaObjectsShardsUpdateUnprocessableEntity) IsClientError() bool { - return true -} - -// IsServerError returns true when this schema objects shards update unprocessable entity response has a 5xx status code -func (o *SchemaObjectsShardsUpdateUnprocessableEntity) IsServerError() bool { - return false -} - -// IsCode returns true when this schema objects shards update unprocessable entity response a status code equal to that given -func (o *SchemaObjectsShardsUpdateUnprocessableEntity) IsCode(code int) bool { - return code == 422 -} - -// Code gets the status code for the schema objects shards update unprocessable entity response -func (o *SchemaObjectsShardsUpdateUnprocessableEntity) Code() int { - return 422 -} - -func (o *SchemaObjectsShardsUpdateUnprocessableEntity) Error() string { - return fmt.Sprintf("[PUT /schema/{className}/shards/{shardName}][%d] schemaObjectsShardsUpdateUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *SchemaObjectsShardsUpdateUnprocessableEntity) String() string { - return fmt.Sprintf("[PUT /schema/{className}/shards/{shardName}][%d] schemaObjectsShardsUpdateUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *SchemaObjectsShardsUpdateUnprocessableEntity) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *SchemaObjectsShardsUpdateUnprocessableEntity) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewSchemaObjectsShardsUpdateInternalServerError creates a SchemaObjectsShardsUpdateInternalServerError with default headers values -func NewSchemaObjectsShardsUpdateInternalServerError() *SchemaObjectsShardsUpdateInternalServerError { - return &SchemaObjectsShardsUpdateInternalServerError{} -} - -/* -SchemaObjectsShardsUpdateInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type SchemaObjectsShardsUpdateInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this schema objects shards update internal server error response has a 2xx status code -func (o *SchemaObjectsShardsUpdateInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this schema objects shards update internal server error response has a 3xx status code -func (o *SchemaObjectsShardsUpdateInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this schema objects shards update internal server error response has a 4xx status code -func (o *SchemaObjectsShardsUpdateInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this schema objects shards update internal server error response has a 5xx status code -func (o *SchemaObjectsShardsUpdateInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this schema objects shards update internal server error response a status code equal to that given -func (o *SchemaObjectsShardsUpdateInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the schema objects shards update internal server error response -func (o *SchemaObjectsShardsUpdateInternalServerError) Code() int { - return 500 -} - -func (o *SchemaObjectsShardsUpdateInternalServerError) Error() string { - return fmt.Sprintf("[PUT /schema/{className}/shards/{shardName}][%d] schemaObjectsShardsUpdateInternalServerError %+v", 500, o.Payload) -} - -func (o *SchemaObjectsShardsUpdateInternalServerError) String() string { - return fmt.Sprintf("[PUT /schema/{className}/shards/{shardName}][%d] schemaObjectsShardsUpdateInternalServerError %+v", 500, o.Payload) -} - -func (o *SchemaObjectsShardsUpdateInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *SchemaObjectsShardsUpdateInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/schema/schema_objects_update_parameters.go b/client/schema/schema_objects_update_parameters.go deleted file mode 100644 index ff261bcc715567dd3ba3fc34d256610b323781cd..0000000000000000000000000000000000000000 --- a/client/schema/schema_objects_update_parameters.go +++ /dev/null @@ -1,180 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewSchemaObjectsUpdateParams creates a new SchemaObjectsUpdateParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewSchemaObjectsUpdateParams() *SchemaObjectsUpdateParams { - return &SchemaObjectsUpdateParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewSchemaObjectsUpdateParamsWithTimeout creates a new SchemaObjectsUpdateParams object -// with the ability to set a timeout on a request. -func NewSchemaObjectsUpdateParamsWithTimeout(timeout time.Duration) *SchemaObjectsUpdateParams { - return &SchemaObjectsUpdateParams{ - timeout: timeout, - } -} - -// NewSchemaObjectsUpdateParamsWithContext creates a new SchemaObjectsUpdateParams object -// with the ability to set a context for a request. -func NewSchemaObjectsUpdateParamsWithContext(ctx context.Context) *SchemaObjectsUpdateParams { - return &SchemaObjectsUpdateParams{ - Context: ctx, - } -} - -// NewSchemaObjectsUpdateParamsWithHTTPClient creates a new SchemaObjectsUpdateParams object -// with the ability to set a custom HTTPClient for a request. -func NewSchemaObjectsUpdateParamsWithHTTPClient(client *http.Client) *SchemaObjectsUpdateParams { - return &SchemaObjectsUpdateParams{ - HTTPClient: client, - } -} - -/* -SchemaObjectsUpdateParams contains all the parameters to send to the API endpoint - - for the schema objects update operation. - - Typically these are written to a http.Request. -*/ -type SchemaObjectsUpdateParams struct { - - // ClassName. - ClassName string - - // ObjectClass. - ObjectClass *models.Class - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the schema objects update params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *SchemaObjectsUpdateParams) WithDefaults() *SchemaObjectsUpdateParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the schema objects update params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *SchemaObjectsUpdateParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the schema objects update params -func (o *SchemaObjectsUpdateParams) WithTimeout(timeout time.Duration) *SchemaObjectsUpdateParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the schema objects update params -func (o *SchemaObjectsUpdateParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the schema objects update params -func (o *SchemaObjectsUpdateParams) WithContext(ctx context.Context) *SchemaObjectsUpdateParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the schema objects update params -func (o *SchemaObjectsUpdateParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the schema objects update params -func (o *SchemaObjectsUpdateParams) WithHTTPClient(client *http.Client) *SchemaObjectsUpdateParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the schema objects update params -func (o *SchemaObjectsUpdateParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithClassName adds the className to the schema objects update params -func (o *SchemaObjectsUpdateParams) WithClassName(className string) *SchemaObjectsUpdateParams { - o.SetClassName(className) - return o -} - -// SetClassName adds the className to the schema objects update params -func (o *SchemaObjectsUpdateParams) SetClassName(className string) { - o.ClassName = className -} - -// WithObjectClass adds the objectClass to the schema objects update params -func (o *SchemaObjectsUpdateParams) WithObjectClass(objectClass *models.Class) *SchemaObjectsUpdateParams { - o.SetObjectClass(objectClass) - return o -} - -// SetObjectClass adds the objectClass to the schema objects update params -func (o *SchemaObjectsUpdateParams) SetObjectClass(objectClass *models.Class) { - o.ObjectClass = objectClass -} - -// WriteToRequest writes these params to a swagger request -func (o *SchemaObjectsUpdateParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - - // path param className - if err := r.SetPathParam("className", o.ClassName); err != nil { - return err - } - if o.ObjectClass != nil { - if err := r.SetBodyParam(o.ObjectClass); err != nil { - return err - } - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/schema/schema_objects_update_responses.go b/client/schema/schema_objects_update_responses.go deleted file mode 100644 index 4b8b05bd6727b5a628514aee9e55b1e06cfc0dc1..0000000000000000000000000000000000000000 --- a/client/schema/schema_objects_update_responses.go +++ /dev/null @@ -1,472 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// SchemaObjectsUpdateReader is a Reader for the SchemaObjectsUpdate structure. -type SchemaObjectsUpdateReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *SchemaObjectsUpdateReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 200: - result := NewSchemaObjectsUpdateOK() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 401: - result := NewSchemaObjectsUpdateUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewSchemaObjectsUpdateForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 404: - result := NewSchemaObjectsUpdateNotFound() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 422: - result := NewSchemaObjectsUpdateUnprocessableEntity() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewSchemaObjectsUpdateInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewSchemaObjectsUpdateOK creates a SchemaObjectsUpdateOK with default headers values -func NewSchemaObjectsUpdateOK() *SchemaObjectsUpdateOK { - return &SchemaObjectsUpdateOK{} -} - -/* -SchemaObjectsUpdateOK describes a response with status code 200, with default header values. - -Class was updated successfully -*/ -type SchemaObjectsUpdateOK struct { - Payload *models.Class -} - -// IsSuccess returns true when this schema objects update o k response has a 2xx status code -func (o *SchemaObjectsUpdateOK) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this schema objects update o k response has a 3xx status code -func (o *SchemaObjectsUpdateOK) IsRedirect() bool { - return false -} - -// IsClientError returns true when this schema objects update o k response has a 4xx status code -func (o *SchemaObjectsUpdateOK) IsClientError() bool { - return false -} - -// IsServerError returns true when this schema objects update o k response has a 5xx status code -func (o *SchemaObjectsUpdateOK) IsServerError() bool { - return false -} - -// IsCode returns true when this schema objects update o k response a status code equal to that given -func (o *SchemaObjectsUpdateOK) IsCode(code int) bool { - return code == 200 -} - -// Code gets the status code for the schema objects update o k response -func (o *SchemaObjectsUpdateOK) Code() int { - return 200 -} - -func (o *SchemaObjectsUpdateOK) Error() string { - return fmt.Sprintf("[PUT /schema/{className}][%d] schemaObjectsUpdateOK %+v", 200, o.Payload) -} - -func (o *SchemaObjectsUpdateOK) String() string { - return fmt.Sprintf("[PUT /schema/{className}][%d] schemaObjectsUpdateOK %+v", 200, o.Payload) -} - -func (o *SchemaObjectsUpdateOK) GetPayload() *models.Class { - return o.Payload -} - -func (o *SchemaObjectsUpdateOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.Class) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewSchemaObjectsUpdateUnauthorized creates a SchemaObjectsUpdateUnauthorized with default headers values -func NewSchemaObjectsUpdateUnauthorized() *SchemaObjectsUpdateUnauthorized { - return &SchemaObjectsUpdateUnauthorized{} -} - -/* -SchemaObjectsUpdateUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type SchemaObjectsUpdateUnauthorized struct { -} - -// IsSuccess returns true when this schema objects update unauthorized response has a 2xx status code -func (o *SchemaObjectsUpdateUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this schema objects update unauthorized response has a 3xx status code -func (o *SchemaObjectsUpdateUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this schema objects update unauthorized response has a 4xx status code -func (o *SchemaObjectsUpdateUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this schema objects update unauthorized response has a 5xx status code -func (o *SchemaObjectsUpdateUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this schema objects update unauthorized response a status code equal to that given -func (o *SchemaObjectsUpdateUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the schema objects update unauthorized response -func (o *SchemaObjectsUpdateUnauthorized) Code() int { - return 401 -} - -func (o *SchemaObjectsUpdateUnauthorized) Error() string { - return fmt.Sprintf("[PUT /schema/{className}][%d] schemaObjectsUpdateUnauthorized ", 401) -} - -func (o *SchemaObjectsUpdateUnauthorized) String() string { - return fmt.Sprintf("[PUT /schema/{className}][%d] schemaObjectsUpdateUnauthorized ", 401) -} - -func (o *SchemaObjectsUpdateUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewSchemaObjectsUpdateForbidden creates a SchemaObjectsUpdateForbidden with default headers values -func NewSchemaObjectsUpdateForbidden() *SchemaObjectsUpdateForbidden { - return &SchemaObjectsUpdateForbidden{} -} - -/* -SchemaObjectsUpdateForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type SchemaObjectsUpdateForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this schema objects update forbidden response has a 2xx status code -func (o *SchemaObjectsUpdateForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this schema objects update forbidden response has a 3xx status code -func (o *SchemaObjectsUpdateForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this schema objects update forbidden response has a 4xx status code -func (o *SchemaObjectsUpdateForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this schema objects update forbidden response has a 5xx status code -func (o *SchemaObjectsUpdateForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this schema objects update forbidden response a status code equal to that given -func (o *SchemaObjectsUpdateForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the schema objects update forbidden response -func (o *SchemaObjectsUpdateForbidden) Code() int { - return 403 -} - -func (o *SchemaObjectsUpdateForbidden) Error() string { - return fmt.Sprintf("[PUT /schema/{className}][%d] schemaObjectsUpdateForbidden %+v", 403, o.Payload) -} - -func (o *SchemaObjectsUpdateForbidden) String() string { - return fmt.Sprintf("[PUT /schema/{className}][%d] schemaObjectsUpdateForbidden %+v", 403, o.Payload) -} - -func (o *SchemaObjectsUpdateForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *SchemaObjectsUpdateForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewSchemaObjectsUpdateNotFound creates a SchemaObjectsUpdateNotFound with default headers values -func NewSchemaObjectsUpdateNotFound() *SchemaObjectsUpdateNotFound { - return &SchemaObjectsUpdateNotFound{} -} - -/* -SchemaObjectsUpdateNotFound describes a response with status code 404, with default header values. - -Class to be updated does not exist -*/ -type SchemaObjectsUpdateNotFound struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this schema objects update not found response has a 2xx status code -func (o *SchemaObjectsUpdateNotFound) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this schema objects update not found response has a 3xx status code -func (o *SchemaObjectsUpdateNotFound) IsRedirect() bool { - return false -} - -// IsClientError returns true when this schema objects update not found response has a 4xx status code -func (o *SchemaObjectsUpdateNotFound) IsClientError() bool { - return true -} - -// IsServerError returns true when this schema objects update not found response has a 5xx status code -func (o *SchemaObjectsUpdateNotFound) IsServerError() bool { - return false -} - -// IsCode returns true when this schema objects update not found response a status code equal to that given -func (o *SchemaObjectsUpdateNotFound) IsCode(code int) bool { - return code == 404 -} - -// Code gets the status code for the schema objects update not found response -func (o *SchemaObjectsUpdateNotFound) Code() int { - return 404 -} - -func (o *SchemaObjectsUpdateNotFound) Error() string { - return fmt.Sprintf("[PUT /schema/{className}][%d] schemaObjectsUpdateNotFound %+v", 404, o.Payload) -} - -func (o *SchemaObjectsUpdateNotFound) String() string { - return fmt.Sprintf("[PUT /schema/{className}][%d] schemaObjectsUpdateNotFound %+v", 404, o.Payload) -} - -func (o *SchemaObjectsUpdateNotFound) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *SchemaObjectsUpdateNotFound) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewSchemaObjectsUpdateUnprocessableEntity creates a SchemaObjectsUpdateUnprocessableEntity with default headers values -func NewSchemaObjectsUpdateUnprocessableEntity() *SchemaObjectsUpdateUnprocessableEntity { - return &SchemaObjectsUpdateUnprocessableEntity{} -} - -/* -SchemaObjectsUpdateUnprocessableEntity describes a response with status code 422, with default header values. - -Invalid update attempt -*/ -type SchemaObjectsUpdateUnprocessableEntity struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this schema objects update unprocessable entity response has a 2xx status code -func (o *SchemaObjectsUpdateUnprocessableEntity) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this schema objects update unprocessable entity response has a 3xx status code -func (o *SchemaObjectsUpdateUnprocessableEntity) IsRedirect() bool { - return false -} - -// IsClientError returns true when this schema objects update unprocessable entity response has a 4xx status code -func (o *SchemaObjectsUpdateUnprocessableEntity) IsClientError() bool { - return true -} - -// IsServerError returns true when this schema objects update unprocessable entity response has a 5xx status code -func (o *SchemaObjectsUpdateUnprocessableEntity) IsServerError() bool { - return false -} - -// IsCode returns true when this schema objects update unprocessable entity response a status code equal to that given -func (o *SchemaObjectsUpdateUnprocessableEntity) IsCode(code int) bool { - return code == 422 -} - -// Code gets the status code for the schema objects update unprocessable entity response -func (o *SchemaObjectsUpdateUnprocessableEntity) Code() int { - return 422 -} - -func (o *SchemaObjectsUpdateUnprocessableEntity) Error() string { - return fmt.Sprintf("[PUT /schema/{className}][%d] schemaObjectsUpdateUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *SchemaObjectsUpdateUnprocessableEntity) String() string { - return fmt.Sprintf("[PUT /schema/{className}][%d] schemaObjectsUpdateUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *SchemaObjectsUpdateUnprocessableEntity) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *SchemaObjectsUpdateUnprocessableEntity) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewSchemaObjectsUpdateInternalServerError creates a SchemaObjectsUpdateInternalServerError with default headers values -func NewSchemaObjectsUpdateInternalServerError() *SchemaObjectsUpdateInternalServerError { - return &SchemaObjectsUpdateInternalServerError{} -} - -/* -SchemaObjectsUpdateInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type SchemaObjectsUpdateInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this schema objects update internal server error response has a 2xx status code -func (o *SchemaObjectsUpdateInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this schema objects update internal server error response has a 3xx status code -func (o *SchemaObjectsUpdateInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this schema objects update internal server error response has a 4xx status code -func (o *SchemaObjectsUpdateInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this schema objects update internal server error response has a 5xx status code -func (o *SchemaObjectsUpdateInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this schema objects update internal server error response a status code equal to that given -func (o *SchemaObjectsUpdateInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the schema objects update internal server error response -func (o *SchemaObjectsUpdateInternalServerError) Code() int { - return 500 -} - -func (o *SchemaObjectsUpdateInternalServerError) Error() string { - return fmt.Sprintf("[PUT /schema/{className}][%d] schemaObjectsUpdateInternalServerError %+v", 500, o.Payload) -} - -func (o *SchemaObjectsUpdateInternalServerError) String() string { - return fmt.Sprintf("[PUT /schema/{className}][%d] schemaObjectsUpdateInternalServerError %+v", 500, o.Payload) -} - -func (o *SchemaObjectsUpdateInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *SchemaObjectsUpdateInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/schema/tenants_create_parameters.go b/client/schema/tenants_create_parameters.go deleted file mode 100644 index c7d0f016709ff9e0b01875eeafa973f849396060..0000000000000000000000000000000000000000 --- a/client/schema/tenants_create_parameters.go +++ /dev/null @@ -1,180 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewTenantsCreateParams creates a new TenantsCreateParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewTenantsCreateParams() *TenantsCreateParams { - return &TenantsCreateParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewTenantsCreateParamsWithTimeout creates a new TenantsCreateParams object -// with the ability to set a timeout on a request. -func NewTenantsCreateParamsWithTimeout(timeout time.Duration) *TenantsCreateParams { - return &TenantsCreateParams{ - timeout: timeout, - } -} - -// NewTenantsCreateParamsWithContext creates a new TenantsCreateParams object -// with the ability to set a context for a request. -func NewTenantsCreateParamsWithContext(ctx context.Context) *TenantsCreateParams { - return &TenantsCreateParams{ - Context: ctx, - } -} - -// NewTenantsCreateParamsWithHTTPClient creates a new TenantsCreateParams object -// with the ability to set a custom HTTPClient for a request. -func NewTenantsCreateParamsWithHTTPClient(client *http.Client) *TenantsCreateParams { - return &TenantsCreateParams{ - HTTPClient: client, - } -} - -/* -TenantsCreateParams contains all the parameters to send to the API endpoint - - for the tenants create operation. - - Typically these are written to a http.Request. -*/ -type TenantsCreateParams struct { - - // Body. - Body []*models.Tenant - - // ClassName. - ClassName string - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the tenants create params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *TenantsCreateParams) WithDefaults() *TenantsCreateParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the tenants create params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *TenantsCreateParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the tenants create params -func (o *TenantsCreateParams) WithTimeout(timeout time.Duration) *TenantsCreateParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the tenants create params -func (o *TenantsCreateParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the tenants create params -func (o *TenantsCreateParams) WithContext(ctx context.Context) *TenantsCreateParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the tenants create params -func (o *TenantsCreateParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the tenants create params -func (o *TenantsCreateParams) WithHTTPClient(client *http.Client) *TenantsCreateParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the tenants create params -func (o *TenantsCreateParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithBody adds the body to the tenants create params -func (o *TenantsCreateParams) WithBody(body []*models.Tenant) *TenantsCreateParams { - o.SetBody(body) - return o -} - -// SetBody adds the body to the tenants create params -func (o *TenantsCreateParams) SetBody(body []*models.Tenant) { - o.Body = body -} - -// WithClassName adds the className to the tenants create params -func (o *TenantsCreateParams) WithClassName(className string) *TenantsCreateParams { - o.SetClassName(className) - return o -} - -// SetClassName adds the className to the tenants create params -func (o *TenantsCreateParams) SetClassName(className string) { - o.ClassName = className -} - -// WriteToRequest writes these params to a swagger request -func (o *TenantsCreateParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - if o.Body != nil { - if err := r.SetBodyParam(o.Body); err != nil { - return err - } - } - - // path param className - if err := r.SetPathParam("className", o.ClassName); err != nil { - return err - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/schema/tenants_create_responses.go b/client/schema/tenants_create_responses.go deleted file mode 100644 index dc588e31a524e7c18116f6813315bd9bd17003b5..0000000000000000000000000000000000000000 --- a/client/schema/tenants_create_responses.go +++ /dev/null @@ -1,396 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// TenantsCreateReader is a Reader for the TenantsCreate structure. -type TenantsCreateReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *TenantsCreateReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 200: - result := NewTenantsCreateOK() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 401: - result := NewTenantsCreateUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewTenantsCreateForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 422: - result := NewTenantsCreateUnprocessableEntity() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewTenantsCreateInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewTenantsCreateOK creates a TenantsCreateOK with default headers values -func NewTenantsCreateOK() *TenantsCreateOK { - return &TenantsCreateOK{} -} - -/* -TenantsCreateOK describes a response with status code 200, with default header values. - -Added new tenants to the specified class -*/ -type TenantsCreateOK struct { - Payload []*models.Tenant -} - -// IsSuccess returns true when this tenants create o k response has a 2xx status code -func (o *TenantsCreateOK) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this tenants create o k response has a 3xx status code -func (o *TenantsCreateOK) IsRedirect() bool { - return false -} - -// IsClientError returns true when this tenants create o k response has a 4xx status code -func (o *TenantsCreateOK) IsClientError() bool { - return false -} - -// IsServerError returns true when this tenants create o k response has a 5xx status code -func (o *TenantsCreateOK) IsServerError() bool { - return false -} - -// IsCode returns true when this tenants create o k response a status code equal to that given -func (o *TenantsCreateOK) IsCode(code int) bool { - return code == 200 -} - -// Code gets the status code for the tenants create o k response -func (o *TenantsCreateOK) Code() int { - return 200 -} - -func (o *TenantsCreateOK) Error() string { - return fmt.Sprintf("[POST /schema/{className}/tenants][%d] tenantsCreateOK %+v", 200, o.Payload) -} - -func (o *TenantsCreateOK) String() string { - return fmt.Sprintf("[POST /schema/{className}/tenants][%d] tenantsCreateOK %+v", 200, o.Payload) -} - -func (o *TenantsCreateOK) GetPayload() []*models.Tenant { - return o.Payload -} - -func (o *TenantsCreateOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - // response payload - if err := consumer.Consume(response.Body(), &o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewTenantsCreateUnauthorized creates a TenantsCreateUnauthorized with default headers values -func NewTenantsCreateUnauthorized() *TenantsCreateUnauthorized { - return &TenantsCreateUnauthorized{} -} - -/* -TenantsCreateUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type TenantsCreateUnauthorized struct { -} - -// IsSuccess returns true when this tenants create unauthorized response has a 2xx status code -func (o *TenantsCreateUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this tenants create unauthorized response has a 3xx status code -func (o *TenantsCreateUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this tenants create unauthorized response has a 4xx status code -func (o *TenantsCreateUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this tenants create unauthorized response has a 5xx status code -func (o *TenantsCreateUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this tenants create unauthorized response a status code equal to that given -func (o *TenantsCreateUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the tenants create unauthorized response -func (o *TenantsCreateUnauthorized) Code() int { - return 401 -} - -func (o *TenantsCreateUnauthorized) Error() string { - return fmt.Sprintf("[POST /schema/{className}/tenants][%d] tenantsCreateUnauthorized ", 401) -} - -func (o *TenantsCreateUnauthorized) String() string { - return fmt.Sprintf("[POST /schema/{className}/tenants][%d] tenantsCreateUnauthorized ", 401) -} - -func (o *TenantsCreateUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewTenantsCreateForbidden creates a TenantsCreateForbidden with default headers values -func NewTenantsCreateForbidden() *TenantsCreateForbidden { - return &TenantsCreateForbidden{} -} - -/* -TenantsCreateForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type TenantsCreateForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this tenants create forbidden response has a 2xx status code -func (o *TenantsCreateForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this tenants create forbidden response has a 3xx status code -func (o *TenantsCreateForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this tenants create forbidden response has a 4xx status code -func (o *TenantsCreateForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this tenants create forbidden response has a 5xx status code -func (o *TenantsCreateForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this tenants create forbidden response a status code equal to that given -func (o *TenantsCreateForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the tenants create forbidden response -func (o *TenantsCreateForbidden) Code() int { - return 403 -} - -func (o *TenantsCreateForbidden) Error() string { - return fmt.Sprintf("[POST /schema/{className}/tenants][%d] tenantsCreateForbidden %+v", 403, o.Payload) -} - -func (o *TenantsCreateForbidden) String() string { - return fmt.Sprintf("[POST /schema/{className}/tenants][%d] tenantsCreateForbidden %+v", 403, o.Payload) -} - -func (o *TenantsCreateForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *TenantsCreateForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewTenantsCreateUnprocessableEntity creates a TenantsCreateUnprocessableEntity with default headers values -func NewTenantsCreateUnprocessableEntity() *TenantsCreateUnprocessableEntity { - return &TenantsCreateUnprocessableEntity{} -} - -/* -TenantsCreateUnprocessableEntity describes a response with status code 422, with default header values. - -Invalid Tenant class -*/ -type TenantsCreateUnprocessableEntity struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this tenants create unprocessable entity response has a 2xx status code -func (o *TenantsCreateUnprocessableEntity) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this tenants create unprocessable entity response has a 3xx status code -func (o *TenantsCreateUnprocessableEntity) IsRedirect() bool { - return false -} - -// IsClientError returns true when this tenants create unprocessable entity response has a 4xx status code -func (o *TenantsCreateUnprocessableEntity) IsClientError() bool { - return true -} - -// IsServerError returns true when this tenants create unprocessable entity response has a 5xx status code -func (o *TenantsCreateUnprocessableEntity) IsServerError() bool { - return false -} - -// IsCode returns true when this tenants create unprocessable entity response a status code equal to that given -func (o *TenantsCreateUnprocessableEntity) IsCode(code int) bool { - return code == 422 -} - -// Code gets the status code for the tenants create unprocessable entity response -func (o *TenantsCreateUnprocessableEntity) Code() int { - return 422 -} - -func (o *TenantsCreateUnprocessableEntity) Error() string { - return fmt.Sprintf("[POST /schema/{className}/tenants][%d] tenantsCreateUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *TenantsCreateUnprocessableEntity) String() string { - return fmt.Sprintf("[POST /schema/{className}/tenants][%d] tenantsCreateUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *TenantsCreateUnprocessableEntity) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *TenantsCreateUnprocessableEntity) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewTenantsCreateInternalServerError creates a TenantsCreateInternalServerError with default headers values -func NewTenantsCreateInternalServerError() *TenantsCreateInternalServerError { - return &TenantsCreateInternalServerError{} -} - -/* -TenantsCreateInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type TenantsCreateInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this tenants create internal server error response has a 2xx status code -func (o *TenantsCreateInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this tenants create internal server error response has a 3xx status code -func (o *TenantsCreateInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this tenants create internal server error response has a 4xx status code -func (o *TenantsCreateInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this tenants create internal server error response has a 5xx status code -func (o *TenantsCreateInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this tenants create internal server error response a status code equal to that given -func (o *TenantsCreateInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the tenants create internal server error response -func (o *TenantsCreateInternalServerError) Code() int { - return 500 -} - -func (o *TenantsCreateInternalServerError) Error() string { - return fmt.Sprintf("[POST /schema/{className}/tenants][%d] tenantsCreateInternalServerError %+v", 500, o.Payload) -} - -func (o *TenantsCreateInternalServerError) String() string { - return fmt.Sprintf("[POST /schema/{className}/tenants][%d] tenantsCreateInternalServerError %+v", 500, o.Payload) -} - -func (o *TenantsCreateInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *TenantsCreateInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/schema/tenants_delete_parameters.go b/client/schema/tenants_delete_parameters.go deleted file mode 100644 index fac34398367b99ee1b206db193834ee21120975c..0000000000000000000000000000000000000000 --- a/client/schema/tenants_delete_parameters.go +++ /dev/null @@ -1,178 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" -) - -// NewTenantsDeleteParams creates a new TenantsDeleteParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewTenantsDeleteParams() *TenantsDeleteParams { - return &TenantsDeleteParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewTenantsDeleteParamsWithTimeout creates a new TenantsDeleteParams object -// with the ability to set a timeout on a request. -func NewTenantsDeleteParamsWithTimeout(timeout time.Duration) *TenantsDeleteParams { - return &TenantsDeleteParams{ - timeout: timeout, - } -} - -// NewTenantsDeleteParamsWithContext creates a new TenantsDeleteParams object -// with the ability to set a context for a request. -func NewTenantsDeleteParamsWithContext(ctx context.Context) *TenantsDeleteParams { - return &TenantsDeleteParams{ - Context: ctx, - } -} - -// NewTenantsDeleteParamsWithHTTPClient creates a new TenantsDeleteParams object -// with the ability to set a custom HTTPClient for a request. -func NewTenantsDeleteParamsWithHTTPClient(client *http.Client) *TenantsDeleteParams { - return &TenantsDeleteParams{ - HTTPClient: client, - } -} - -/* -TenantsDeleteParams contains all the parameters to send to the API endpoint - - for the tenants delete operation. - - Typically these are written to a http.Request. -*/ -type TenantsDeleteParams struct { - - // ClassName. - ClassName string - - // Tenants. - Tenants []string - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the tenants delete params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *TenantsDeleteParams) WithDefaults() *TenantsDeleteParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the tenants delete params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *TenantsDeleteParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the tenants delete params -func (o *TenantsDeleteParams) WithTimeout(timeout time.Duration) *TenantsDeleteParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the tenants delete params -func (o *TenantsDeleteParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the tenants delete params -func (o *TenantsDeleteParams) WithContext(ctx context.Context) *TenantsDeleteParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the tenants delete params -func (o *TenantsDeleteParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the tenants delete params -func (o *TenantsDeleteParams) WithHTTPClient(client *http.Client) *TenantsDeleteParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the tenants delete params -func (o *TenantsDeleteParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithClassName adds the className to the tenants delete params -func (o *TenantsDeleteParams) WithClassName(className string) *TenantsDeleteParams { - o.SetClassName(className) - return o -} - -// SetClassName adds the className to the tenants delete params -func (o *TenantsDeleteParams) SetClassName(className string) { - o.ClassName = className -} - -// WithTenants adds the tenants to the tenants delete params -func (o *TenantsDeleteParams) WithTenants(tenants []string) *TenantsDeleteParams { - o.SetTenants(tenants) - return o -} - -// SetTenants adds the tenants to the tenants delete params -func (o *TenantsDeleteParams) SetTenants(tenants []string) { - o.Tenants = tenants -} - -// WriteToRequest writes these params to a swagger request -func (o *TenantsDeleteParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - - // path param className - if err := r.SetPathParam("className", o.ClassName); err != nil { - return err - } - if o.Tenants != nil { - if err := r.SetBodyParam(o.Tenants); err != nil { - return err - } - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/schema/tenants_delete_responses.go b/client/schema/tenants_delete_responses.go deleted file mode 100644 index 8a1f937acd81b7d0cb43c6d83e3406f952edcf31..0000000000000000000000000000000000000000 --- a/client/schema/tenants_delete_responses.go +++ /dev/null @@ -1,386 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// TenantsDeleteReader is a Reader for the TenantsDelete structure. -type TenantsDeleteReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *TenantsDeleteReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 200: - result := NewTenantsDeleteOK() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 401: - result := NewTenantsDeleteUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewTenantsDeleteForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 422: - result := NewTenantsDeleteUnprocessableEntity() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewTenantsDeleteInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewTenantsDeleteOK creates a TenantsDeleteOK with default headers values -func NewTenantsDeleteOK() *TenantsDeleteOK { - return &TenantsDeleteOK{} -} - -/* -TenantsDeleteOK describes a response with status code 200, with default header values. - -Deleted tenants from specified class. -*/ -type TenantsDeleteOK struct { -} - -// IsSuccess returns true when this tenants delete o k response has a 2xx status code -func (o *TenantsDeleteOK) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this tenants delete o k response has a 3xx status code -func (o *TenantsDeleteOK) IsRedirect() bool { - return false -} - -// IsClientError returns true when this tenants delete o k response has a 4xx status code -func (o *TenantsDeleteOK) IsClientError() bool { - return false -} - -// IsServerError returns true when this tenants delete o k response has a 5xx status code -func (o *TenantsDeleteOK) IsServerError() bool { - return false -} - -// IsCode returns true when this tenants delete o k response a status code equal to that given -func (o *TenantsDeleteOK) IsCode(code int) bool { - return code == 200 -} - -// Code gets the status code for the tenants delete o k response -func (o *TenantsDeleteOK) Code() int { - return 200 -} - -func (o *TenantsDeleteOK) Error() string { - return fmt.Sprintf("[DELETE /schema/{className}/tenants][%d] tenantsDeleteOK ", 200) -} - -func (o *TenantsDeleteOK) String() string { - return fmt.Sprintf("[DELETE /schema/{className}/tenants][%d] tenantsDeleteOK ", 200) -} - -func (o *TenantsDeleteOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewTenantsDeleteUnauthorized creates a TenantsDeleteUnauthorized with default headers values -func NewTenantsDeleteUnauthorized() *TenantsDeleteUnauthorized { - return &TenantsDeleteUnauthorized{} -} - -/* -TenantsDeleteUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type TenantsDeleteUnauthorized struct { -} - -// IsSuccess returns true when this tenants delete unauthorized response has a 2xx status code -func (o *TenantsDeleteUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this tenants delete unauthorized response has a 3xx status code -func (o *TenantsDeleteUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this tenants delete unauthorized response has a 4xx status code -func (o *TenantsDeleteUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this tenants delete unauthorized response has a 5xx status code -func (o *TenantsDeleteUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this tenants delete unauthorized response a status code equal to that given -func (o *TenantsDeleteUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the tenants delete unauthorized response -func (o *TenantsDeleteUnauthorized) Code() int { - return 401 -} - -func (o *TenantsDeleteUnauthorized) Error() string { - return fmt.Sprintf("[DELETE /schema/{className}/tenants][%d] tenantsDeleteUnauthorized ", 401) -} - -func (o *TenantsDeleteUnauthorized) String() string { - return fmt.Sprintf("[DELETE /schema/{className}/tenants][%d] tenantsDeleteUnauthorized ", 401) -} - -func (o *TenantsDeleteUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewTenantsDeleteForbidden creates a TenantsDeleteForbidden with default headers values -func NewTenantsDeleteForbidden() *TenantsDeleteForbidden { - return &TenantsDeleteForbidden{} -} - -/* -TenantsDeleteForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type TenantsDeleteForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this tenants delete forbidden response has a 2xx status code -func (o *TenantsDeleteForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this tenants delete forbidden response has a 3xx status code -func (o *TenantsDeleteForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this tenants delete forbidden response has a 4xx status code -func (o *TenantsDeleteForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this tenants delete forbidden response has a 5xx status code -func (o *TenantsDeleteForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this tenants delete forbidden response a status code equal to that given -func (o *TenantsDeleteForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the tenants delete forbidden response -func (o *TenantsDeleteForbidden) Code() int { - return 403 -} - -func (o *TenantsDeleteForbidden) Error() string { - return fmt.Sprintf("[DELETE /schema/{className}/tenants][%d] tenantsDeleteForbidden %+v", 403, o.Payload) -} - -func (o *TenantsDeleteForbidden) String() string { - return fmt.Sprintf("[DELETE /schema/{className}/tenants][%d] tenantsDeleteForbidden %+v", 403, o.Payload) -} - -func (o *TenantsDeleteForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *TenantsDeleteForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewTenantsDeleteUnprocessableEntity creates a TenantsDeleteUnprocessableEntity with default headers values -func NewTenantsDeleteUnprocessableEntity() *TenantsDeleteUnprocessableEntity { - return &TenantsDeleteUnprocessableEntity{} -} - -/* -TenantsDeleteUnprocessableEntity describes a response with status code 422, with default header values. - -Invalid Tenant class -*/ -type TenantsDeleteUnprocessableEntity struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this tenants delete unprocessable entity response has a 2xx status code -func (o *TenantsDeleteUnprocessableEntity) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this tenants delete unprocessable entity response has a 3xx status code -func (o *TenantsDeleteUnprocessableEntity) IsRedirect() bool { - return false -} - -// IsClientError returns true when this tenants delete unprocessable entity response has a 4xx status code -func (o *TenantsDeleteUnprocessableEntity) IsClientError() bool { - return true -} - -// IsServerError returns true when this tenants delete unprocessable entity response has a 5xx status code -func (o *TenantsDeleteUnprocessableEntity) IsServerError() bool { - return false -} - -// IsCode returns true when this tenants delete unprocessable entity response a status code equal to that given -func (o *TenantsDeleteUnprocessableEntity) IsCode(code int) bool { - return code == 422 -} - -// Code gets the status code for the tenants delete unprocessable entity response -func (o *TenantsDeleteUnprocessableEntity) Code() int { - return 422 -} - -func (o *TenantsDeleteUnprocessableEntity) Error() string { - return fmt.Sprintf("[DELETE /schema/{className}/tenants][%d] tenantsDeleteUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *TenantsDeleteUnprocessableEntity) String() string { - return fmt.Sprintf("[DELETE /schema/{className}/tenants][%d] tenantsDeleteUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *TenantsDeleteUnprocessableEntity) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *TenantsDeleteUnprocessableEntity) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewTenantsDeleteInternalServerError creates a TenantsDeleteInternalServerError with default headers values -func NewTenantsDeleteInternalServerError() *TenantsDeleteInternalServerError { - return &TenantsDeleteInternalServerError{} -} - -/* -TenantsDeleteInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type TenantsDeleteInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this tenants delete internal server error response has a 2xx status code -func (o *TenantsDeleteInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this tenants delete internal server error response has a 3xx status code -func (o *TenantsDeleteInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this tenants delete internal server error response has a 4xx status code -func (o *TenantsDeleteInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this tenants delete internal server error response has a 5xx status code -func (o *TenantsDeleteInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this tenants delete internal server error response a status code equal to that given -func (o *TenantsDeleteInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the tenants delete internal server error response -func (o *TenantsDeleteInternalServerError) Code() int { - return 500 -} - -func (o *TenantsDeleteInternalServerError) Error() string { - return fmt.Sprintf("[DELETE /schema/{className}/tenants][%d] tenantsDeleteInternalServerError %+v", 500, o.Payload) -} - -func (o *TenantsDeleteInternalServerError) String() string { - return fmt.Sprintf("[DELETE /schema/{className}/tenants][%d] tenantsDeleteInternalServerError %+v", 500, o.Payload) -} - -func (o *TenantsDeleteInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *TenantsDeleteInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/schema/tenants_get_parameters.go b/client/schema/tenants_get_parameters.go deleted file mode 100644 index c60dee355ac31f8778e4c13d503dfdd6b374ad22..0000000000000000000000000000000000000000 --- a/client/schema/tenants_get_parameters.go +++ /dev/null @@ -1,159 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" -) - -// NewTenantsGetParams creates a new TenantsGetParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewTenantsGetParams() *TenantsGetParams { - return &TenantsGetParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewTenantsGetParamsWithTimeout creates a new TenantsGetParams object -// with the ability to set a timeout on a request. -func NewTenantsGetParamsWithTimeout(timeout time.Duration) *TenantsGetParams { - return &TenantsGetParams{ - timeout: timeout, - } -} - -// NewTenantsGetParamsWithContext creates a new TenantsGetParams object -// with the ability to set a context for a request. -func NewTenantsGetParamsWithContext(ctx context.Context) *TenantsGetParams { - return &TenantsGetParams{ - Context: ctx, - } -} - -// NewTenantsGetParamsWithHTTPClient creates a new TenantsGetParams object -// with the ability to set a custom HTTPClient for a request. -func NewTenantsGetParamsWithHTTPClient(client *http.Client) *TenantsGetParams { - return &TenantsGetParams{ - HTTPClient: client, - } -} - -/* -TenantsGetParams contains all the parameters to send to the API endpoint - - for the tenants get operation. - - Typically these are written to a http.Request. -*/ -type TenantsGetParams struct { - - // ClassName. - ClassName string - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the tenants get params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *TenantsGetParams) WithDefaults() *TenantsGetParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the tenants get params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *TenantsGetParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the tenants get params -func (o *TenantsGetParams) WithTimeout(timeout time.Duration) *TenantsGetParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the tenants get params -func (o *TenantsGetParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the tenants get params -func (o *TenantsGetParams) WithContext(ctx context.Context) *TenantsGetParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the tenants get params -func (o *TenantsGetParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the tenants get params -func (o *TenantsGetParams) WithHTTPClient(client *http.Client) *TenantsGetParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the tenants get params -func (o *TenantsGetParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithClassName adds the className to the tenants get params -func (o *TenantsGetParams) WithClassName(className string) *TenantsGetParams { - o.SetClassName(className) - return o -} - -// SetClassName adds the className to the tenants get params -func (o *TenantsGetParams) SetClassName(className string) { - o.ClassName = className -} - -// WriteToRequest writes these params to a swagger request -func (o *TenantsGetParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - - // path param className - if err := r.SetPathParam("className", o.ClassName); err != nil { - return err - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/schema/tenants_get_responses.go b/client/schema/tenants_get_responses.go deleted file mode 100644 index 978458186c5dd348ec49cb5ec6da3bed4c4a970d..0000000000000000000000000000000000000000 --- a/client/schema/tenants_get_responses.go +++ /dev/null @@ -1,396 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// TenantsGetReader is a Reader for the TenantsGet structure. -type TenantsGetReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *TenantsGetReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 200: - result := NewTenantsGetOK() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 401: - result := NewTenantsGetUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewTenantsGetForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 422: - result := NewTenantsGetUnprocessableEntity() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewTenantsGetInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewTenantsGetOK creates a TenantsGetOK with default headers values -func NewTenantsGetOK() *TenantsGetOK { - return &TenantsGetOK{} -} - -/* -TenantsGetOK describes a response with status code 200, with default header values. - -tenants from specified class. -*/ -type TenantsGetOK struct { - Payload []*models.Tenant -} - -// IsSuccess returns true when this tenants get o k response has a 2xx status code -func (o *TenantsGetOK) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this tenants get o k response has a 3xx status code -func (o *TenantsGetOK) IsRedirect() bool { - return false -} - -// IsClientError returns true when this tenants get o k response has a 4xx status code -func (o *TenantsGetOK) IsClientError() bool { - return false -} - -// IsServerError returns true when this tenants get o k response has a 5xx status code -func (o *TenantsGetOK) IsServerError() bool { - return false -} - -// IsCode returns true when this tenants get o k response a status code equal to that given -func (o *TenantsGetOK) IsCode(code int) bool { - return code == 200 -} - -// Code gets the status code for the tenants get o k response -func (o *TenantsGetOK) Code() int { - return 200 -} - -func (o *TenantsGetOK) Error() string { - return fmt.Sprintf("[GET /schema/{className}/tenants][%d] tenantsGetOK %+v", 200, o.Payload) -} - -func (o *TenantsGetOK) String() string { - return fmt.Sprintf("[GET /schema/{className}/tenants][%d] tenantsGetOK %+v", 200, o.Payload) -} - -func (o *TenantsGetOK) GetPayload() []*models.Tenant { - return o.Payload -} - -func (o *TenantsGetOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - // response payload - if err := consumer.Consume(response.Body(), &o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewTenantsGetUnauthorized creates a TenantsGetUnauthorized with default headers values -func NewTenantsGetUnauthorized() *TenantsGetUnauthorized { - return &TenantsGetUnauthorized{} -} - -/* -TenantsGetUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type TenantsGetUnauthorized struct { -} - -// IsSuccess returns true when this tenants get unauthorized response has a 2xx status code -func (o *TenantsGetUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this tenants get unauthorized response has a 3xx status code -func (o *TenantsGetUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this tenants get unauthorized response has a 4xx status code -func (o *TenantsGetUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this tenants get unauthorized response has a 5xx status code -func (o *TenantsGetUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this tenants get unauthorized response a status code equal to that given -func (o *TenantsGetUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the tenants get unauthorized response -func (o *TenantsGetUnauthorized) Code() int { - return 401 -} - -func (o *TenantsGetUnauthorized) Error() string { - return fmt.Sprintf("[GET /schema/{className}/tenants][%d] tenantsGetUnauthorized ", 401) -} - -func (o *TenantsGetUnauthorized) String() string { - return fmt.Sprintf("[GET /schema/{className}/tenants][%d] tenantsGetUnauthorized ", 401) -} - -func (o *TenantsGetUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewTenantsGetForbidden creates a TenantsGetForbidden with default headers values -func NewTenantsGetForbidden() *TenantsGetForbidden { - return &TenantsGetForbidden{} -} - -/* -TenantsGetForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type TenantsGetForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this tenants get forbidden response has a 2xx status code -func (o *TenantsGetForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this tenants get forbidden response has a 3xx status code -func (o *TenantsGetForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this tenants get forbidden response has a 4xx status code -func (o *TenantsGetForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this tenants get forbidden response has a 5xx status code -func (o *TenantsGetForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this tenants get forbidden response a status code equal to that given -func (o *TenantsGetForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the tenants get forbidden response -func (o *TenantsGetForbidden) Code() int { - return 403 -} - -func (o *TenantsGetForbidden) Error() string { - return fmt.Sprintf("[GET /schema/{className}/tenants][%d] tenantsGetForbidden %+v", 403, o.Payload) -} - -func (o *TenantsGetForbidden) String() string { - return fmt.Sprintf("[GET /schema/{className}/tenants][%d] tenantsGetForbidden %+v", 403, o.Payload) -} - -func (o *TenantsGetForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *TenantsGetForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewTenantsGetUnprocessableEntity creates a TenantsGetUnprocessableEntity with default headers values -func NewTenantsGetUnprocessableEntity() *TenantsGetUnprocessableEntity { - return &TenantsGetUnprocessableEntity{} -} - -/* -TenantsGetUnprocessableEntity describes a response with status code 422, with default header values. - -Invalid Tenant class -*/ -type TenantsGetUnprocessableEntity struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this tenants get unprocessable entity response has a 2xx status code -func (o *TenantsGetUnprocessableEntity) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this tenants get unprocessable entity response has a 3xx status code -func (o *TenantsGetUnprocessableEntity) IsRedirect() bool { - return false -} - -// IsClientError returns true when this tenants get unprocessable entity response has a 4xx status code -func (o *TenantsGetUnprocessableEntity) IsClientError() bool { - return true -} - -// IsServerError returns true when this tenants get unprocessable entity response has a 5xx status code -func (o *TenantsGetUnprocessableEntity) IsServerError() bool { - return false -} - -// IsCode returns true when this tenants get unprocessable entity response a status code equal to that given -func (o *TenantsGetUnprocessableEntity) IsCode(code int) bool { - return code == 422 -} - -// Code gets the status code for the tenants get unprocessable entity response -func (o *TenantsGetUnprocessableEntity) Code() int { - return 422 -} - -func (o *TenantsGetUnprocessableEntity) Error() string { - return fmt.Sprintf("[GET /schema/{className}/tenants][%d] tenantsGetUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *TenantsGetUnprocessableEntity) String() string { - return fmt.Sprintf("[GET /schema/{className}/tenants][%d] tenantsGetUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *TenantsGetUnprocessableEntity) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *TenantsGetUnprocessableEntity) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewTenantsGetInternalServerError creates a TenantsGetInternalServerError with default headers values -func NewTenantsGetInternalServerError() *TenantsGetInternalServerError { - return &TenantsGetInternalServerError{} -} - -/* -TenantsGetInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type TenantsGetInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this tenants get internal server error response has a 2xx status code -func (o *TenantsGetInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this tenants get internal server error response has a 3xx status code -func (o *TenantsGetInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this tenants get internal server error response has a 4xx status code -func (o *TenantsGetInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this tenants get internal server error response has a 5xx status code -func (o *TenantsGetInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this tenants get internal server error response a status code equal to that given -func (o *TenantsGetInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the tenants get internal server error response -func (o *TenantsGetInternalServerError) Code() int { - return 500 -} - -func (o *TenantsGetInternalServerError) Error() string { - return fmt.Sprintf("[GET /schema/{className}/tenants][%d] tenantsGetInternalServerError %+v", 500, o.Payload) -} - -func (o *TenantsGetInternalServerError) String() string { - return fmt.Sprintf("[GET /schema/{className}/tenants][%d] tenantsGetInternalServerError %+v", 500, o.Payload) -} - -func (o *TenantsGetInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *TenantsGetInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/schema/tenants_update_parameters.go b/client/schema/tenants_update_parameters.go deleted file mode 100644 index 1a945e0f2529906693578ae665ee303278af2cf1..0000000000000000000000000000000000000000 --- a/client/schema/tenants_update_parameters.go +++ /dev/null @@ -1,180 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// NewTenantsUpdateParams creates a new TenantsUpdateParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewTenantsUpdateParams() *TenantsUpdateParams { - return &TenantsUpdateParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewTenantsUpdateParamsWithTimeout creates a new TenantsUpdateParams object -// with the ability to set a timeout on a request. -func NewTenantsUpdateParamsWithTimeout(timeout time.Duration) *TenantsUpdateParams { - return &TenantsUpdateParams{ - timeout: timeout, - } -} - -// NewTenantsUpdateParamsWithContext creates a new TenantsUpdateParams object -// with the ability to set a context for a request. -func NewTenantsUpdateParamsWithContext(ctx context.Context) *TenantsUpdateParams { - return &TenantsUpdateParams{ - Context: ctx, - } -} - -// NewTenantsUpdateParamsWithHTTPClient creates a new TenantsUpdateParams object -// with the ability to set a custom HTTPClient for a request. -func NewTenantsUpdateParamsWithHTTPClient(client *http.Client) *TenantsUpdateParams { - return &TenantsUpdateParams{ - HTTPClient: client, - } -} - -/* -TenantsUpdateParams contains all the parameters to send to the API endpoint - - for the tenants update operation. - - Typically these are written to a http.Request. -*/ -type TenantsUpdateParams struct { - - // Body. - Body []*models.Tenant - - // ClassName. - ClassName string - - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the tenants update params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *TenantsUpdateParams) WithDefaults() *TenantsUpdateParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the tenants update params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *TenantsUpdateParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the tenants update params -func (o *TenantsUpdateParams) WithTimeout(timeout time.Duration) *TenantsUpdateParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the tenants update params -func (o *TenantsUpdateParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the tenants update params -func (o *TenantsUpdateParams) WithContext(ctx context.Context) *TenantsUpdateParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the tenants update params -func (o *TenantsUpdateParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the tenants update params -func (o *TenantsUpdateParams) WithHTTPClient(client *http.Client) *TenantsUpdateParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the tenants update params -func (o *TenantsUpdateParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WithBody adds the body to the tenants update params -func (o *TenantsUpdateParams) WithBody(body []*models.Tenant) *TenantsUpdateParams { - o.SetBody(body) - return o -} - -// SetBody adds the body to the tenants update params -func (o *TenantsUpdateParams) SetBody(body []*models.Tenant) { - o.Body = body -} - -// WithClassName adds the className to the tenants update params -func (o *TenantsUpdateParams) WithClassName(className string) *TenantsUpdateParams { - o.SetClassName(className) - return o -} - -// SetClassName adds the className to the tenants update params -func (o *TenantsUpdateParams) SetClassName(className string) { - o.ClassName = className -} - -// WriteToRequest writes these params to a swagger request -func (o *TenantsUpdateParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - if o.Body != nil { - if err := r.SetBodyParam(o.Body); err != nil { - return err - } - } - - // path param className - if err := r.SetPathParam("className", o.ClassName); err != nil { - return err - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/schema/tenants_update_responses.go b/client/schema/tenants_update_responses.go deleted file mode 100644 index 3c0ba53f4907685b6f0a59cc512b9e2062961f6b..0000000000000000000000000000000000000000 --- a/client/schema/tenants_update_responses.go +++ /dev/null @@ -1,396 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package schema - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/entities/models" -) - -// TenantsUpdateReader is a Reader for the TenantsUpdate structure. -type TenantsUpdateReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *TenantsUpdateReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 200: - result := NewTenantsUpdateOK() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 401: - result := NewTenantsUpdateUnauthorized() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 403: - result := NewTenantsUpdateForbidden() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 422: - result := NewTenantsUpdateUnprocessableEntity() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewTenantsUpdateInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewTenantsUpdateOK creates a TenantsUpdateOK with default headers values -func NewTenantsUpdateOK() *TenantsUpdateOK { - return &TenantsUpdateOK{} -} - -/* -TenantsUpdateOK describes a response with status code 200, with default header values. - -Updated tenants of the specified class -*/ -type TenantsUpdateOK struct { - Payload []*models.Tenant -} - -// IsSuccess returns true when this tenants update o k response has a 2xx status code -func (o *TenantsUpdateOK) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this tenants update o k response has a 3xx status code -func (o *TenantsUpdateOK) IsRedirect() bool { - return false -} - -// IsClientError returns true when this tenants update o k response has a 4xx status code -func (o *TenantsUpdateOK) IsClientError() bool { - return false -} - -// IsServerError returns true when this tenants update o k response has a 5xx status code -func (o *TenantsUpdateOK) IsServerError() bool { - return false -} - -// IsCode returns true when this tenants update o k response a status code equal to that given -func (o *TenantsUpdateOK) IsCode(code int) bool { - return code == 200 -} - -// Code gets the status code for the tenants update o k response -func (o *TenantsUpdateOK) Code() int { - return 200 -} - -func (o *TenantsUpdateOK) Error() string { - return fmt.Sprintf("[PUT /schema/{className}/tenants][%d] tenantsUpdateOK %+v", 200, o.Payload) -} - -func (o *TenantsUpdateOK) String() string { - return fmt.Sprintf("[PUT /schema/{className}/tenants][%d] tenantsUpdateOK %+v", 200, o.Payload) -} - -func (o *TenantsUpdateOK) GetPayload() []*models.Tenant { - return o.Payload -} - -func (o *TenantsUpdateOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - // response payload - if err := consumer.Consume(response.Body(), &o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewTenantsUpdateUnauthorized creates a TenantsUpdateUnauthorized with default headers values -func NewTenantsUpdateUnauthorized() *TenantsUpdateUnauthorized { - return &TenantsUpdateUnauthorized{} -} - -/* -TenantsUpdateUnauthorized describes a response with status code 401, with default header values. - -Unauthorized or invalid credentials. -*/ -type TenantsUpdateUnauthorized struct { -} - -// IsSuccess returns true when this tenants update unauthorized response has a 2xx status code -func (o *TenantsUpdateUnauthorized) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this tenants update unauthorized response has a 3xx status code -func (o *TenantsUpdateUnauthorized) IsRedirect() bool { - return false -} - -// IsClientError returns true when this tenants update unauthorized response has a 4xx status code -func (o *TenantsUpdateUnauthorized) IsClientError() bool { - return true -} - -// IsServerError returns true when this tenants update unauthorized response has a 5xx status code -func (o *TenantsUpdateUnauthorized) IsServerError() bool { - return false -} - -// IsCode returns true when this tenants update unauthorized response a status code equal to that given -func (o *TenantsUpdateUnauthorized) IsCode(code int) bool { - return code == 401 -} - -// Code gets the status code for the tenants update unauthorized response -func (o *TenantsUpdateUnauthorized) Code() int { - return 401 -} - -func (o *TenantsUpdateUnauthorized) Error() string { - return fmt.Sprintf("[PUT /schema/{className}/tenants][%d] tenantsUpdateUnauthorized ", 401) -} - -func (o *TenantsUpdateUnauthorized) String() string { - return fmt.Sprintf("[PUT /schema/{className}/tenants][%d] tenantsUpdateUnauthorized ", 401) -} - -func (o *TenantsUpdateUnauthorized) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewTenantsUpdateForbidden creates a TenantsUpdateForbidden with default headers values -func NewTenantsUpdateForbidden() *TenantsUpdateForbidden { - return &TenantsUpdateForbidden{} -} - -/* -TenantsUpdateForbidden describes a response with status code 403, with default header values. - -Forbidden -*/ -type TenantsUpdateForbidden struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this tenants update forbidden response has a 2xx status code -func (o *TenantsUpdateForbidden) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this tenants update forbidden response has a 3xx status code -func (o *TenantsUpdateForbidden) IsRedirect() bool { - return false -} - -// IsClientError returns true when this tenants update forbidden response has a 4xx status code -func (o *TenantsUpdateForbidden) IsClientError() bool { - return true -} - -// IsServerError returns true when this tenants update forbidden response has a 5xx status code -func (o *TenantsUpdateForbidden) IsServerError() bool { - return false -} - -// IsCode returns true when this tenants update forbidden response a status code equal to that given -func (o *TenantsUpdateForbidden) IsCode(code int) bool { - return code == 403 -} - -// Code gets the status code for the tenants update forbidden response -func (o *TenantsUpdateForbidden) Code() int { - return 403 -} - -func (o *TenantsUpdateForbidden) Error() string { - return fmt.Sprintf("[PUT /schema/{className}/tenants][%d] tenantsUpdateForbidden %+v", 403, o.Payload) -} - -func (o *TenantsUpdateForbidden) String() string { - return fmt.Sprintf("[PUT /schema/{className}/tenants][%d] tenantsUpdateForbidden %+v", 403, o.Payload) -} - -func (o *TenantsUpdateForbidden) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *TenantsUpdateForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewTenantsUpdateUnprocessableEntity creates a TenantsUpdateUnprocessableEntity with default headers values -func NewTenantsUpdateUnprocessableEntity() *TenantsUpdateUnprocessableEntity { - return &TenantsUpdateUnprocessableEntity{} -} - -/* -TenantsUpdateUnprocessableEntity describes a response with status code 422, with default header values. - -Invalid Tenant class -*/ -type TenantsUpdateUnprocessableEntity struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this tenants update unprocessable entity response has a 2xx status code -func (o *TenantsUpdateUnprocessableEntity) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this tenants update unprocessable entity response has a 3xx status code -func (o *TenantsUpdateUnprocessableEntity) IsRedirect() bool { - return false -} - -// IsClientError returns true when this tenants update unprocessable entity response has a 4xx status code -func (o *TenantsUpdateUnprocessableEntity) IsClientError() bool { - return true -} - -// IsServerError returns true when this tenants update unprocessable entity response has a 5xx status code -func (o *TenantsUpdateUnprocessableEntity) IsServerError() bool { - return false -} - -// IsCode returns true when this tenants update unprocessable entity response a status code equal to that given -func (o *TenantsUpdateUnprocessableEntity) IsCode(code int) bool { - return code == 422 -} - -// Code gets the status code for the tenants update unprocessable entity response -func (o *TenantsUpdateUnprocessableEntity) Code() int { - return 422 -} - -func (o *TenantsUpdateUnprocessableEntity) Error() string { - return fmt.Sprintf("[PUT /schema/{className}/tenants][%d] tenantsUpdateUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *TenantsUpdateUnprocessableEntity) String() string { - return fmt.Sprintf("[PUT /schema/{className}/tenants][%d] tenantsUpdateUnprocessableEntity %+v", 422, o.Payload) -} - -func (o *TenantsUpdateUnprocessableEntity) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *TenantsUpdateUnprocessableEntity) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewTenantsUpdateInternalServerError creates a TenantsUpdateInternalServerError with default headers values -func NewTenantsUpdateInternalServerError() *TenantsUpdateInternalServerError { - return &TenantsUpdateInternalServerError{} -} - -/* -TenantsUpdateInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type TenantsUpdateInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this tenants update internal server error response has a 2xx status code -func (o *TenantsUpdateInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this tenants update internal server error response has a 3xx status code -func (o *TenantsUpdateInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this tenants update internal server error response has a 4xx status code -func (o *TenantsUpdateInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this tenants update internal server error response has a 5xx status code -func (o *TenantsUpdateInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this tenants update internal server error response a status code equal to that given -func (o *TenantsUpdateInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the tenants update internal server error response -func (o *TenantsUpdateInternalServerError) Code() int { - return 500 -} - -func (o *TenantsUpdateInternalServerError) Error() string { - return fmt.Sprintf("[PUT /schema/{className}/tenants][%d] tenantsUpdateInternalServerError %+v", 500, o.Payload) -} - -func (o *TenantsUpdateInternalServerError) String() string { - return fmt.Sprintf("[PUT /schema/{className}/tenants][%d] tenantsUpdateInternalServerError %+v", 500, o.Payload) -} - -func (o *TenantsUpdateInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *TenantsUpdateInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} diff --git a/client/weaviate_client.go b/client/weaviate_client.go deleted file mode 100644 index b85ccdb833a042de6c018cda58dab330ed298274..0000000000000000000000000000000000000000 --- a/client/weaviate_client.go +++ /dev/null @@ -1,168 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package client - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "github.com/go-openapi/runtime" - httptransport "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" - - "github.com/weaviate/weaviate/client/backups" - "github.com/weaviate/weaviate/client/batch" - "github.com/weaviate/weaviate/client/classifications" - "github.com/weaviate/weaviate/client/graphql" - "github.com/weaviate/weaviate/client/meta" - "github.com/weaviate/weaviate/client/nodes" - "github.com/weaviate/weaviate/client/objects" - "github.com/weaviate/weaviate/client/operations" - "github.com/weaviate/weaviate/client/schema" - "github.com/weaviate/weaviate/client/well_known" -) - -// Default weaviate HTTP client. -var Default = NewHTTPClient(nil) - -const ( - // DefaultHost is the default Host - // found in Meta (info) section of spec file - DefaultHost string = "localhost" - // DefaultBasePath is the default BasePath - // found in Meta (info) section of spec file - DefaultBasePath string = "/v1" -) - -// DefaultSchemes are the default schemes found in Meta (info) section of spec file -var DefaultSchemes = []string{"https"} - -// NewHTTPClient creates a new weaviate HTTP client. -func NewHTTPClient(formats strfmt.Registry) *Weaviate { - return NewHTTPClientWithConfig(formats, nil) -} - -// NewHTTPClientWithConfig creates a new weaviate HTTP client, -// using a customizable transport config. -func NewHTTPClientWithConfig(formats strfmt.Registry, cfg *TransportConfig) *Weaviate { - // ensure nullable parameters have default - if cfg == nil { - cfg = DefaultTransportConfig() - } - - // create transport and client - transport := httptransport.New(cfg.Host, cfg.BasePath, cfg.Schemes) - return New(transport, formats) -} - -// New creates a new weaviate client -func New(transport runtime.ClientTransport, formats strfmt.Registry) *Weaviate { - // ensure nullable parameters have default - if formats == nil { - formats = strfmt.Default - } - - cli := new(Weaviate) - cli.Transport = transport - cli.Backups = backups.New(transport, formats) - cli.Batch = batch.New(transport, formats) - cli.Classifications = classifications.New(transport, formats) - cli.Graphql = graphql.New(transport, formats) - cli.Meta = meta.New(transport, formats) - cli.Nodes = nodes.New(transport, formats) - cli.Objects = objects.New(transport, formats) - cli.Operations = operations.New(transport, formats) - cli.Schema = schema.New(transport, formats) - cli.WellKnown = well_known.New(transport, formats) - return cli -} - -// DefaultTransportConfig creates a TransportConfig with the -// default settings taken from the meta section of the spec file. -func DefaultTransportConfig() *TransportConfig { - return &TransportConfig{ - Host: DefaultHost, - BasePath: DefaultBasePath, - Schemes: DefaultSchemes, - } -} - -// TransportConfig contains the transport related info, -// found in the meta section of the spec file. -type TransportConfig struct { - Host string - BasePath string - Schemes []string -} - -// WithHost overrides the default host, -// provided by the meta section of the spec file. -func (cfg *TransportConfig) WithHost(host string) *TransportConfig { - cfg.Host = host - return cfg -} - -// WithBasePath overrides the default basePath, -// provided by the meta section of the spec file. -func (cfg *TransportConfig) WithBasePath(basePath string) *TransportConfig { - cfg.BasePath = basePath - return cfg -} - -// WithSchemes overrides the default schemes, -// provided by the meta section of the spec file. -func (cfg *TransportConfig) WithSchemes(schemes []string) *TransportConfig { - cfg.Schemes = schemes - return cfg -} - -// Weaviate is a client for weaviate -type Weaviate struct { - Backups backups.ClientService - - Batch batch.ClientService - - Classifications classifications.ClientService - - Graphql graphql.ClientService - - Meta meta.ClientService - - Nodes nodes.ClientService - - Objects objects.ClientService - - Operations operations.ClientService - - Schema schema.ClientService - - WellKnown well_known.ClientService - - Transport runtime.ClientTransport -} - -// SetTransport changes the transport on the client and all its subresources -func (c *Weaviate) SetTransport(transport runtime.ClientTransport) { - c.Transport = transport - c.Backups.SetTransport(transport) - c.Batch.SetTransport(transport) - c.Classifications.SetTransport(transport) - c.Graphql.SetTransport(transport) - c.Meta.SetTransport(transport) - c.Nodes.SetTransport(transport) - c.Objects.SetTransport(transport) - c.Operations.SetTransport(transport) - c.Schema.SetTransport(transport) - c.WellKnown.SetTransport(transport) -} diff --git a/client/well_known/get_well_known_openid_configuration_parameters.go b/client/well_known/get_well_known_openid_configuration_parameters.go deleted file mode 100644 index d14889d0aaa766d0f8770236a05fd1185723e2e2..0000000000000000000000000000000000000000 --- a/client/well_known/get_well_known_openid_configuration_parameters.go +++ /dev/null @@ -1,139 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package well_known - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "net/http" - "time" - - "github.com/go-openapi/errors" - "github.com/go-openapi/runtime" - cr "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" -) - -// NewGetWellKnownOpenidConfigurationParams creates a new GetWellKnownOpenidConfigurationParams object, -// with the default timeout for this client. -// -// Default values are not hydrated, since defaults are normally applied by the API server side. -// -// To enforce default values in parameter, use SetDefaults or WithDefaults. -func NewGetWellKnownOpenidConfigurationParams() *GetWellKnownOpenidConfigurationParams { - return &GetWellKnownOpenidConfigurationParams{ - timeout: cr.DefaultTimeout, - } -} - -// NewGetWellKnownOpenidConfigurationParamsWithTimeout creates a new GetWellKnownOpenidConfigurationParams object -// with the ability to set a timeout on a request. -func NewGetWellKnownOpenidConfigurationParamsWithTimeout(timeout time.Duration) *GetWellKnownOpenidConfigurationParams { - return &GetWellKnownOpenidConfigurationParams{ - timeout: timeout, - } -} - -// NewGetWellKnownOpenidConfigurationParamsWithContext creates a new GetWellKnownOpenidConfigurationParams object -// with the ability to set a context for a request. -func NewGetWellKnownOpenidConfigurationParamsWithContext(ctx context.Context) *GetWellKnownOpenidConfigurationParams { - return &GetWellKnownOpenidConfigurationParams{ - Context: ctx, - } -} - -// NewGetWellKnownOpenidConfigurationParamsWithHTTPClient creates a new GetWellKnownOpenidConfigurationParams object -// with the ability to set a custom HTTPClient for a request. -func NewGetWellKnownOpenidConfigurationParamsWithHTTPClient(client *http.Client) *GetWellKnownOpenidConfigurationParams { - return &GetWellKnownOpenidConfigurationParams{ - HTTPClient: client, - } -} - -/* -GetWellKnownOpenidConfigurationParams contains all the parameters to send to the API endpoint - - for the get well known openid configuration operation. - - Typically these are written to a http.Request. -*/ -type GetWellKnownOpenidConfigurationParams struct { - timeout time.Duration - Context context.Context - HTTPClient *http.Client -} - -// WithDefaults hydrates default values in the get well known openid configuration params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *GetWellKnownOpenidConfigurationParams) WithDefaults() *GetWellKnownOpenidConfigurationParams { - o.SetDefaults() - return o -} - -// SetDefaults hydrates default values in the get well known openid configuration params (not the query body). -// -// All values with no default are reset to their zero value. -func (o *GetWellKnownOpenidConfigurationParams) SetDefaults() { - // no default values defined for this parameter -} - -// WithTimeout adds the timeout to the get well known openid configuration params -func (o *GetWellKnownOpenidConfigurationParams) WithTimeout(timeout time.Duration) *GetWellKnownOpenidConfigurationParams { - o.SetTimeout(timeout) - return o -} - -// SetTimeout adds the timeout to the get well known openid configuration params -func (o *GetWellKnownOpenidConfigurationParams) SetTimeout(timeout time.Duration) { - o.timeout = timeout -} - -// WithContext adds the context to the get well known openid configuration params -func (o *GetWellKnownOpenidConfigurationParams) WithContext(ctx context.Context) *GetWellKnownOpenidConfigurationParams { - o.SetContext(ctx) - return o -} - -// SetContext adds the context to the get well known openid configuration params -func (o *GetWellKnownOpenidConfigurationParams) SetContext(ctx context.Context) { - o.Context = ctx -} - -// WithHTTPClient adds the HTTPClient to the get well known openid configuration params -func (o *GetWellKnownOpenidConfigurationParams) WithHTTPClient(client *http.Client) *GetWellKnownOpenidConfigurationParams { - o.SetHTTPClient(client) - return o -} - -// SetHTTPClient adds the HTTPClient to the get well known openid configuration params -func (o *GetWellKnownOpenidConfigurationParams) SetHTTPClient(client *http.Client) { - o.HTTPClient = client -} - -// WriteToRequest writes these params to a swagger request -func (o *GetWellKnownOpenidConfigurationParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { - - if err := r.SetTimeout(o.timeout); err != nil { - return err - } - var res []error - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/client/well_known/get_well_known_openid_configuration_responses.go b/client/well_known/get_well_known_openid_configuration_responses.go deleted file mode 100644 index 3fad4d81c6646aef7e163a2952c693d76e77c3b0..0000000000000000000000000000000000000000 --- a/client/well_known/get_well_known_openid_configuration_responses.go +++ /dev/null @@ -1,296 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package well_known - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "fmt" - "io" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" - - "github.com/weaviate/weaviate/entities/models" -) - -// GetWellKnownOpenidConfigurationReader is a Reader for the GetWellKnownOpenidConfiguration structure. -type GetWellKnownOpenidConfigurationReader struct { - formats strfmt.Registry -} - -// ReadResponse reads a server response into the received o. -func (o *GetWellKnownOpenidConfigurationReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { - switch response.Code() { - case 200: - result := NewGetWellKnownOpenidConfigurationOK() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return result, nil - case 404: - result := NewGetWellKnownOpenidConfigurationNotFound() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - case 500: - result := NewGetWellKnownOpenidConfigurationInternalServerError() - if err := result.readResponse(response, consumer, o.formats); err != nil { - return nil, err - } - return nil, result - default: - return nil, runtime.NewAPIError("response status code does not match any response statuses defined for this endpoint in the swagger spec", response, response.Code()) - } -} - -// NewGetWellKnownOpenidConfigurationOK creates a GetWellKnownOpenidConfigurationOK with default headers values -func NewGetWellKnownOpenidConfigurationOK() *GetWellKnownOpenidConfigurationOK { - return &GetWellKnownOpenidConfigurationOK{} -} - -/* -GetWellKnownOpenidConfigurationOK describes a response with status code 200, with default header values. - -Successful response, inspect body -*/ -type GetWellKnownOpenidConfigurationOK struct { - Payload *GetWellKnownOpenidConfigurationOKBody -} - -// IsSuccess returns true when this get well known openid configuration o k response has a 2xx status code -func (o *GetWellKnownOpenidConfigurationOK) IsSuccess() bool { - return true -} - -// IsRedirect returns true when this get well known openid configuration o k response has a 3xx status code -func (o *GetWellKnownOpenidConfigurationOK) IsRedirect() bool { - return false -} - -// IsClientError returns true when this get well known openid configuration o k response has a 4xx status code -func (o *GetWellKnownOpenidConfigurationOK) IsClientError() bool { - return false -} - -// IsServerError returns true when this get well known openid configuration o k response has a 5xx status code -func (o *GetWellKnownOpenidConfigurationOK) IsServerError() bool { - return false -} - -// IsCode returns true when this get well known openid configuration o k response a status code equal to that given -func (o *GetWellKnownOpenidConfigurationOK) IsCode(code int) bool { - return code == 200 -} - -// Code gets the status code for the get well known openid configuration o k response -func (o *GetWellKnownOpenidConfigurationOK) Code() int { - return 200 -} - -func (o *GetWellKnownOpenidConfigurationOK) Error() string { - return fmt.Sprintf("[GET /.well-known/openid-configuration][%d] getWellKnownOpenidConfigurationOK %+v", 200, o.Payload) -} - -func (o *GetWellKnownOpenidConfigurationOK) String() string { - return fmt.Sprintf("[GET /.well-known/openid-configuration][%d] getWellKnownOpenidConfigurationOK %+v", 200, o.Payload) -} - -func (o *GetWellKnownOpenidConfigurationOK) GetPayload() *GetWellKnownOpenidConfigurationOKBody { - return o.Payload -} - -func (o *GetWellKnownOpenidConfigurationOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(GetWellKnownOpenidConfigurationOKBody) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -// NewGetWellKnownOpenidConfigurationNotFound creates a GetWellKnownOpenidConfigurationNotFound with default headers values -func NewGetWellKnownOpenidConfigurationNotFound() *GetWellKnownOpenidConfigurationNotFound { - return &GetWellKnownOpenidConfigurationNotFound{} -} - -/* -GetWellKnownOpenidConfigurationNotFound describes a response with status code 404, with default header values. - -Not found, no oidc provider present -*/ -type GetWellKnownOpenidConfigurationNotFound struct { -} - -// IsSuccess returns true when this get well known openid configuration not found response has a 2xx status code -func (o *GetWellKnownOpenidConfigurationNotFound) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this get well known openid configuration not found response has a 3xx status code -func (o *GetWellKnownOpenidConfigurationNotFound) IsRedirect() bool { - return false -} - -// IsClientError returns true when this get well known openid configuration not found response has a 4xx status code -func (o *GetWellKnownOpenidConfigurationNotFound) IsClientError() bool { - return true -} - -// IsServerError returns true when this get well known openid configuration not found response has a 5xx status code -func (o *GetWellKnownOpenidConfigurationNotFound) IsServerError() bool { - return false -} - -// IsCode returns true when this get well known openid configuration not found response a status code equal to that given -func (o *GetWellKnownOpenidConfigurationNotFound) IsCode(code int) bool { - return code == 404 -} - -// Code gets the status code for the get well known openid configuration not found response -func (o *GetWellKnownOpenidConfigurationNotFound) Code() int { - return 404 -} - -func (o *GetWellKnownOpenidConfigurationNotFound) Error() string { - return fmt.Sprintf("[GET /.well-known/openid-configuration][%d] getWellKnownOpenidConfigurationNotFound ", 404) -} - -func (o *GetWellKnownOpenidConfigurationNotFound) String() string { - return fmt.Sprintf("[GET /.well-known/openid-configuration][%d] getWellKnownOpenidConfigurationNotFound ", 404) -} - -func (o *GetWellKnownOpenidConfigurationNotFound) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - return nil -} - -// NewGetWellKnownOpenidConfigurationInternalServerError creates a GetWellKnownOpenidConfigurationInternalServerError with default headers values -func NewGetWellKnownOpenidConfigurationInternalServerError() *GetWellKnownOpenidConfigurationInternalServerError { - return &GetWellKnownOpenidConfigurationInternalServerError{} -} - -/* -GetWellKnownOpenidConfigurationInternalServerError describes a response with status code 500, with default header values. - -An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error. -*/ -type GetWellKnownOpenidConfigurationInternalServerError struct { - Payload *models.ErrorResponse -} - -// IsSuccess returns true when this get well known openid configuration internal server error response has a 2xx status code -func (o *GetWellKnownOpenidConfigurationInternalServerError) IsSuccess() bool { - return false -} - -// IsRedirect returns true when this get well known openid configuration internal server error response has a 3xx status code -func (o *GetWellKnownOpenidConfigurationInternalServerError) IsRedirect() bool { - return false -} - -// IsClientError returns true when this get well known openid configuration internal server error response has a 4xx status code -func (o *GetWellKnownOpenidConfigurationInternalServerError) IsClientError() bool { - return false -} - -// IsServerError returns true when this get well known openid configuration internal server error response has a 5xx status code -func (o *GetWellKnownOpenidConfigurationInternalServerError) IsServerError() bool { - return true -} - -// IsCode returns true when this get well known openid configuration internal server error response a status code equal to that given -func (o *GetWellKnownOpenidConfigurationInternalServerError) IsCode(code int) bool { - return code == 500 -} - -// Code gets the status code for the get well known openid configuration internal server error response -func (o *GetWellKnownOpenidConfigurationInternalServerError) Code() int { - return 500 -} - -func (o *GetWellKnownOpenidConfigurationInternalServerError) Error() string { - return fmt.Sprintf("[GET /.well-known/openid-configuration][%d] getWellKnownOpenidConfigurationInternalServerError %+v", 500, o.Payload) -} - -func (o *GetWellKnownOpenidConfigurationInternalServerError) String() string { - return fmt.Sprintf("[GET /.well-known/openid-configuration][%d] getWellKnownOpenidConfigurationInternalServerError %+v", 500, o.Payload) -} - -func (o *GetWellKnownOpenidConfigurationInternalServerError) GetPayload() *models.ErrorResponse { - return o.Payload -} - -func (o *GetWellKnownOpenidConfigurationInternalServerError) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { - - o.Payload = new(models.ErrorResponse) - - // response payload - if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF { - return err - } - - return nil -} - -/* -GetWellKnownOpenidConfigurationOKBody get well known openid configuration o k body -swagger:model GetWellKnownOpenidConfigurationOKBody -*/ -type GetWellKnownOpenidConfigurationOKBody struct { - - // OAuth Client ID - ClientID string `json:"clientId,omitempty"` - - // The Location to redirect to - Href string `json:"href,omitempty"` - - // OAuth Scopes - Scopes []string `json:"scopes,omitempty"` -} - -// Validate validates this get well known openid configuration o k body -func (o *GetWellKnownOpenidConfigurationOKBody) Validate(formats strfmt.Registry) error { - return nil -} - -// ContextValidate validates this get well known openid configuration o k body based on context it is used -func (o *GetWellKnownOpenidConfigurationOKBody) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - return nil -} - -// MarshalBinary interface implementation -func (o *GetWellKnownOpenidConfigurationOKBody) MarshalBinary() ([]byte, error) { - if o == nil { - return nil, nil - } - return swag.WriteJSON(o) -} - -// UnmarshalBinary interface implementation -func (o *GetWellKnownOpenidConfigurationOKBody) UnmarshalBinary(b []byte) error { - var res GetWellKnownOpenidConfigurationOKBody - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *o = res - return nil -} diff --git a/client/well_known/well_known_client.go b/client/well_known/well_known_client.go deleted file mode 100644 index 1e7263709fa89abc5c92949bebb2deb75f8aa5c5..0000000000000000000000000000000000000000 --- a/client/well_known/well_known_client.go +++ /dev/null @@ -1,93 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package well_known - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "fmt" - - "github.com/go-openapi/runtime" - "github.com/go-openapi/strfmt" -) - -// New creates a new well known API client. -func New(transport runtime.ClientTransport, formats strfmt.Registry) ClientService { - return &Client{transport: transport, formats: formats} -} - -/* -Client for well known API -*/ -type Client struct { - transport runtime.ClientTransport - formats strfmt.Registry -} - -// ClientOption is the option for Client methods -type ClientOption func(*runtime.ClientOperation) - -// ClientService is the interface for Client methods -type ClientService interface { - GetWellKnownOpenidConfiguration(params *GetWellKnownOpenidConfigurationParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*GetWellKnownOpenidConfigurationOK, error) - - SetTransport(transport runtime.ClientTransport) -} - -/* -GetWellKnownOpenidConfiguration os ID c discovery information if o ID c auth is enabled - -OIDC Discovery page, redirects to the token issuer if one is configured -*/ -func (a *Client) GetWellKnownOpenidConfiguration(params *GetWellKnownOpenidConfigurationParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*GetWellKnownOpenidConfigurationOK, error) { - // TODO: Validate the params before sending - if params == nil { - params = NewGetWellKnownOpenidConfigurationParams() - } - op := &runtime.ClientOperation{ - ID: "GetWellKnownOpenidConfiguration", - Method: "GET", - PathPattern: "/.well-known/openid-configuration", - ProducesMediaTypes: []string{"application/json"}, - ConsumesMediaTypes: []string{"application/json", "application/yaml"}, - Schemes: []string{"https"}, - Params: params, - Reader: &GetWellKnownOpenidConfigurationReader{formats: a.formats}, - AuthInfo: authInfo, - Context: params.Context, - Client: params.HTTPClient, - } - for _, opt := range opts { - opt(op) - } - - result, err := a.transport.Submit(op) - if err != nil { - return nil, err - } - success, ok := result.(*GetWellKnownOpenidConfigurationOK) - if ok { - return success, nil - } - // unexpected success response - // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue - msg := fmt.Sprintf("unexpected success response for GetWellKnownOpenidConfiguration: API contract not enforced by server. Client expected to get an error, but got: %T", result) - panic(msg) -} - -// SetTransport changes the transport on the client -func (a *Client) SetTransport(transport runtime.ClientTransport) { - a.transport = transport -} diff --git a/cmd/libweaviate/bindings/tcl.c b/cmd/libweaviate/bindings/tcl.c deleted file mode 100644 index 37139f55b977dda1fa8e3b2abe9c1f4f23f9393e..0000000000000000000000000000000000000000 --- a/cmd/libweaviate/bindings/tcl.c +++ /dev/null @@ -1,28 +0,0 @@ -#include -#include -#include - - -static int dumpBucket_tcl(ClientData clientData, Tcl_Interp *interp, int objc, Tcl_Obj *const objv[]) { - printf("called with %d arguments\n", objc); - return TCL_OK; -} - -static int startWeaviate_tcl(ClientData clientData, Tcl_Interp *interp, int objc, Tcl_Obj *const objv[]) { - printf("called with %d arguments\n", objc); - return TCL_OK; -} - - -int DLLEXPORT Weaviatetcl_Init(Tcl_Interp *interp) { - printf("Initialising TCL library libweaviateTCL\n"); - if (Tcl_InitStubs(interp, "8.1", 0) == NULL) { - printf("Incorrect TCL version\n"); - return TCL_ERROR; - } - printf("creating startWeaviate command\n"); - Tcl_CreateObjCommand(interp, "startWeaviate", startWeaviate_tcl, NULL, NULL); - printf("creating dumpBucket command\n"); - Tcl_CreateObjCommand(interp, "dumpBucket", dumpBucket_tcl, NULL, NULL); - return TCL_OK; -} diff --git a/cmd/libweaviate/build.sh b/cmd/libweaviate/build.sh deleted file mode 100644 index 2e222fbb9fa469ada95dc544116d89eab4163383..0000000000000000000000000000000000000000 --- a/cmd/libweaviate/build.sh +++ /dev/null @@ -1,7 +0,0 @@ -go build -buildmode c-shared -o libweaviate.so . -go build -buildmode c-shared -o libweaviate.dylib . -gcc -dynamiclib -DUSE_TCL_STUBS bindings/tcl.c -L/Library/Developer/CommandLineTools/SDKs/MacOSX12.1.sdk/System/Library/Frameworks/Tcl.framework -L/Library/Developer/CommandLineTools/SDKs/MacOSX13.3.sdk/System/Library/Frameworks/Tcl.framework/Versions/8.5/ -L/Library/Developer/CommandLineTools/SDKs/MacOSX12.1.sdk/System/Library/Frameworks/Tcl.framework/Versions/8.5/ -L/Library/Developer/CommandLineTools/SDKs/MacOSX12.3.sdk/System/Library/Frameworks/Tcl.framework/Versions/8.5/ -L. -I. -lweaviate -ltclstub -o libweaviateTCL.dylib - -#Now load tclsh and run load [file join [pwd] libweaviateTCL[info sharedlibextension]] -echo start tclsh and run -echo load [file join [pwd] libweaviateTCL[info sharedlibextension]] diff --git a/cmd/libweaviate/libweaviate.go b/cmd/libweaviate/libweaviate.go deleted file mode 100644 index 33accfe3ab324d81a9a1997e6b5f05fd7ba23039..0000000000000000000000000000000000000000 --- a/cmd/libweaviate/libweaviate.go +++ /dev/null @@ -1,79 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package main - -// build with: go build -buildmode c-shared -o libweaviate.so . - -// The function signature must not include neither Go struct nor Go interface nor Go array nor variadic argument. - -import ( - "C" - "context" - "fmt" - "os" - - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/adapters/handlers/rest" - "github.com/weaviate/weaviate/adapters/repos/db/helpers" - "github.com/weaviate/weaviate/adapters/repos/db/lsmkv" - "github.com/weaviate/weaviate/entities/cyclemanager" - "github.com/weaviate/weaviate/usecases/config" -) - -var started = false - -//export dumpBucket -func dumpBucket(storeDir, propName string) { - if !started { - fmt.Println("Weaviate has not been started! Call startWeaviate() before any other function") - os.Exit(1) - } - - kvstore, err := lsmkv.New(storeDir, storeDir, &logrus.Logger{}, nil, cyclemanager.NewCallbackGroupNoop(), cyclemanager.NewCallbackGroupNoop()) - if err != nil { - panic(err) - } - - fmt.Printf("propName: %s\n", propName) - bucketName := helpers.BucketFromPropNameLSM(propName) - fmt.Printf("bucketName: %s\n", bucketName) - err = kvstore.CreateOrLoadBucket(context.Background(), bucketName, lsmkv.WithStrategy(lsmkv.StrategyMapCollection)) - if err != nil { - panic(err) - } - - bucket := kvstore.Bucket(bucketName) - fmt.Printf("Dir: %v, bucket %v\n", storeDir, bucketName) - bucket.IterateMapObjects(context.Background(), func(k1, k2, v []byte, tombstone bool) error { - fmt.Printf("k1: %s\n", k1) - fmt.Printf("k2: %x\n", k2) - fmt.Printf("v: %x\n", v) - fmt.Printf("tombstone: %v\n", tombstone) - fmt.Println("-----") - return nil - }) - fmt.Printf("Dir: %v, bucket %v\n", storeDir, bucketName) -} - -//export startWeaviate -func startWeaviate() { - config := config.GetConfigOptionGroup() - rest.MakeAppState(context.Background(), config) - if len(os.Args) != 3 { - fmt.Printf("Usage: %s \n", os.Args[0]) - os.Exit(1) - } - started = true -} - -func main() { -} diff --git a/cmd/weaviate-server/main.go b/cmd/weaviate-server/main.go deleted file mode 100644 index b8b14c1b02e1e3c53c25b732f46bde0f0e4f254e..0000000000000000000000000000000000000000 --- a/cmd/weaviate-server/main.go +++ /dev/null @@ -1,68 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package main - -import ( - "log" - "os" - - "github.com/go-openapi/loads" - flags "github.com/jessevdk/go-flags" - - "github.com/weaviate/weaviate/adapters/handlers/rest" - "github.com/weaviate/weaviate/adapters/handlers/rest/operations" -) - -// This file was generated by the swagger tool. -// Make sure not to overwrite this file after you generated it because all your edits would be lost! - -func main() { - - swaggerSpec, err := loads.Embedded(rest.SwaggerJSON, rest.FlatSwaggerJSON) - if err != nil { - log.Fatalln(err) - } - - api := operations.NewWeaviateAPI(swaggerSpec) - server := rest.NewServer(api) - defer server.Shutdown() - - parser := flags.NewParser(server, flags.Default) - parser.ShortDescription = "Weaviate" - parser.LongDescription = "Cloud-native, modular vector database" - server.ConfigureFlags() - for _, optsGroup := range api.CommandLineOptionsGroups { - _, err := parser.AddGroup(optsGroup.ShortDescription, optsGroup.LongDescription, optsGroup.Options) - if err != nil { - log.Fatalln(err) - } - } - - if _, err := parser.Parse(); err != nil { - code := 1 - if fe, ok := err.(*flags.Error); ok { - if fe.Type == flags.ErrHelp { - code = 0 - } - } - os.Exit(code) - } - - server.ConfigureAPI() - - if err := server.Serve(); err != nil { - log.Fatalln(err) - } - -} diff --git a/codecov.yml b/codecov.yml deleted file mode 100644 index 232da4b2730f9b5a8444ab30ade88c152ecce2ab..0000000000000000000000000000000000000000 --- a/codecov.yml +++ /dev/null @@ -1,9 +0,0 @@ -coverage: - status: - project: - default: - informational: true - patch: - default: - informational: true -comment: false diff --git a/deprecations/data.go b/deprecations/data.go deleted file mode 100644 index 92d12b80310a18c2d0519762d0f14265eec176f7..0000000000000000000000000000000000000000 --- a/deprecations/data.go +++ /dev/null @@ -1,109 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go generate; DO NOT EDIT. -// This file was generated by go generate ./deprecations -package deprecations - -import ( - "time" - - "github.com/go-openapi/strfmt" - "github.com/weaviate/weaviate/entities/models" -) - -func timeMust(t time.Time, err error) strfmt.DateTime { - if err != nil { - panic(err) - } - - return strfmt.DateTime(t) -} - -func timeMustPtr(t time.Time, err error) *strfmt.DateTime { - if err != nil { - panic(err) - } - - parsed := strfmt.DateTime(t) - return &parsed -} - -func ptString(in string) *string { - return &in -} - -var ByID = map[string]models.Deprecation{ - "rest-meta-prop": { - ID: "rest-meta-prop", - Locations: []string{ - "GET /v1/thing/{id}", - "GET /v1/things", - "GET /v1/action/{id}", - "GET /v1/actions", - }, - Status: "deprecated", - APIType: "REST", - Mitigation: "Use ?include=, e.g. ?include=_classification for classification meta or ?include=_vector to show the vector position or ?include=_classification,_vector for both. When consuming the response use the additional fields such as _vector, as the meta object in the response, such as meta.vector will be removed.", - Msg: "use of deprecated property ?meta=true/false", - SinceVersion: "0.22.8", - SinceTime: timeMust(time.Parse(time.RFC3339, "2020-06-15T16:18:06.000Z")), - RemovedIn: ptString("0.23.0"), - RemovedTime: timeMustPtr(time.Parse(time.RFC3339, "2020-06-15T16:18:06.000Z")), - }, - "config-files": { - ID: "config-files", - Locations: []string{ - "--config-file=\"\"", - }, - Status: "deprecated", - APIType: "Configuration", - Mitigation: "Configure Weaviate using environment variables.", - Msg: "use of deprecated command line argument --config-file", - SinceVersion: "0.22.16", - SinceTime: timeMust(time.Parse(time.RFC3339, "2020-09-08T09:46:00.000Z")), - }, - "cardinality": { - ID: "cardinality", - Locations: []string{ - "GET /v1/schema", - "POST /v1/schema/things", - "POST /v1/schema/actions", - "POST /v1/schema/things/{className}/properties", - "POST /v1/schema/actions/{className}/properties", - }, - Status: "deprecated", - APIType: "REST", - Mitigation: "Omit this field. Starting in 0.22.7 it no longer has any effect.", - Msg: "use of deprecated property option 'cardinality'", - SinceVersion: "0.22.17", - SinceTime: timeMust(time.Parse(time.RFC3339, "2020-09-16T09:06:00.000Z")), - RemovedIn: ptString("0.23.0"), - RemovedTime: timeMustPtr(time.Parse(time.RFC3339, "2020-09-16T09:06:00.000Z")), - }, - "ref-meta-deprecated-fields": { - ID: "ref-meta-deprecated-fields", - Locations: []string{ - "GET /v1/thing/{id}", - "GET /v1/things", - "GET /v1/action/{id}", - "GET /v1/actions", - }, - Status: "deprecated", - APIType: "REST", - Mitigation: "when using _classification the reference meta after a successful\nclassification contains various counts and distances. Starting in 0.22.20\nthe fields winningDistance and losingDistance are considered deprecated.\nNew fields were added and they have more descriptive names. User\nmeanWinningDistance instead of winningDistance and use meanLosingDistance\ninstead of losingDistance", - Msg: "response contains deprecated fields winningDistance and losingDistance", - SinceVersion: "0.22.20", - SinceTime: timeMust(time.Parse(time.RFC3339, "2020-11-26T14:58:00.000Z")), - RemovedIn: ptString("0.23.0"), - RemovedTime: timeMustPtr(time.Parse(time.RFC3339, "2020-11-26T14:58:00.000Z")), - }, -} diff --git a/deprecations/deprecations.yml b/deprecations/deprecations.yml deleted file mode 100644 index be406aec12dd79ed2745ae219b379a5892fbb693..0000000000000000000000000000000000000000 --- a/deprecations/deprecations.yml +++ /dev/null @@ -1,65 +0,0 @@ -deprecations: - - id: rest-meta-prop - status: deprecated # switch to removed once feature is completely removed - apiType: REST - locations: - - GET /v1/thing/{id} - - GET /v1/things - - GET /v1/action/{id} - - GET /v1/actions - msg: "use of deprecated property ?meta=true/false" - mitigation: "Use ?include=, e.g. ?include=_classification for classification meta or ?include=_vector to show the vector position or ?include=_classification,_vector for both. When consuming the response use the additional fields such as _vector, as the meta object in the response, such as meta.vector will be removed." - sinceVersion: "0.22.8" - sinceTime: "2020-06-15T16:18:06+00:00" - plannedRemovalVersion: "0.23.0" - removedIn: "0.23.0" - removedTime: "2020-12-18T18:00:00+00:00" - - id: config-files - status: deprecated # switch to removed once feature is completely removed - apiType: Configuration - locations: - - --config-file="" - msg: "use of deprecated command line argument --config-file" - mitigation: "Configure Weaviate using environment variables." - sinceVersion: "0.22.16" - sinceTime: "2020-09-08T09:46:00+00:00" - plannedRemovalVersion: "0.23.0" - removedIn: null - removedTime: null - - id: cardinality - status: deprecated # switch to removed once feature is completely removed - apiType: REST - locations: - - GET /v1/schema - - POST /v1/schema/things - - POST /v1/schema/actions - - POST /v1/schema/things/{className}/properties - - POST /v1/schema/actions/{className}/properties - msg: "use of deprecated property option 'cardinality'" - mitigation: "Omit this field. Starting in 0.22.7 it no longer has any effect." - sinceVersion: "0.22.17" - sinceTime: "2020-09-16T09:06:00+00:00" - plannedRemovalVersion: "0.23.0" - removedIn: "0.23.0" - removedTime: "2020-12-18T18:00:00+00:00" - - id: ref-meta-deprecated-fields - status: deprecated # switch to removed once feature is completely removed - apiType: REST - locations: - - GET /v1/thing/{id} - - GET /v1/things - - GET /v1/action/{id} - - GET /v1/actions - msg: "response contains deprecated fields winningDistance and losingDistance" - mitigation: |- - when using _classification the reference meta after a successful - classification contains various counts and distances. Starting in 0.22.20 - the fields winningDistance and losingDistance are considered deprecated. - New fields were added and they have more descriptive names. User - meanWinningDistance instead of winningDistance and use meanLosingDistance - instead of losingDistance - sinceVersion: "0.22.20" - sinceTime: "2020-11-26T14:58:00+00:00" - plannedRemovalVersion: "0.23.0" - removedIn: "0.23.0" - removedTime: "2020-12-18T18:00:00+00:00" diff --git a/deprecations/gen.go b/deprecations/gen.go deleted file mode 100644 index f47f608f2fa3d44fe039edef23be6161e9b6dc45..0000000000000000000000000000000000000000 --- a/deprecations/gen.go +++ /dev/null @@ -1,176 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -//go:build ignore -// +build ignore - -// The following directive is necessary to make the package coherent: -// This program generates data.go. -// -// - -package main - -import ( - "fmt" - "log" - "os" - "text/template" - "time" - - "github.com/go-openapi/strfmt" - "github.com/weaviate/weaviate/entities/models" - "gopkg.in/yaml.v2" -) - -func main() { - fmt.Println("Generating deprecations code") - - fd, err := os.Open("deprecations.yml") - fatal(err) - defer fd.Close() - - var deprecations struct { - // yaml tags not working on the go-swagger model, so we need to do the - // map[string]interface{} workaround - Deprecations []map[string]interface{} `yaml:"deprecations"` - } - err = yaml.NewDecoder(fd).Decode(&deprecations) - fatal(err) - - parsed, err := parseDeprecations(deprecations.Deprecations) - fatal(err) - - f, err := os.Create("data.go") - fatal(err) - defer f.Close() - err = packageTemplate.Execute(f, struct { - Deprecations []models.Deprecation - }{ - Deprecations: parsed, - }) - fatal(err) -} - -func parseDeprecations(in []map[string]interface{}) ([]models.Deprecation, error) { - out := make([]models.Deprecation, len(in)) - - for i, d := range in { - out[i] = models.Deprecation{ - ID: d["id"].(string), - Status: d["status"].(string), - APIType: d["apiType"].(string), - Msg: d["msg"].(string), - Mitigation: d["mitigation"].(string), - SinceVersion: d["sinceVersion"].(string), - PlannedRemovalVersion: d["plannedRemovalVersion"].(string), - Locations: parseStringSlice(d["locations"].([]interface{})), - SinceTime: timeMust(time.Parse(time.RFC3339, d["sinceTime"].(string))), - } - - if t, ok := d["removedTime"]; ok && t != nil { - parsed := timeMust(time.Parse(time.RFC3339, t.(string))) - out[i].RemovedTime = &parsed - } - - if v, ok := d["removedIn"]; ok && v != nil { - parsed := v.(string) - out[i].RemovedIn = &parsed - } - } - - return out, nil -} - -func timeMust(t time.Time, err error) strfmt.DateTime { - if err != nil { - panic(err) - } - - return strfmt.DateTime(t) -} - -func parseStringSlice(in []interface{}) []string { - out := make([]string, len(in)) - for i, elem := range in { - out[i] = elem.(string) - } - - return out -} - -func fatal(err error) { - if err != nil { - log.Fatal(err) - } -} - -var packageTemplate = template.Must(template.New("").Funcs( - template.FuncMap{ - "DerefString": func(i *string) string { return *i }, - }, -).Parse(`// Code generated by go generate; DO NOT EDIT. -// This file was generated by go generate ./deprecations -package deprecations - -import ( - "time" - - "github.com/go-openapi/strfmt" - "github.com/weaviate/weaviate/entities/models" -) - -func timeMust(t time.Time, err error) strfmt.DateTime { - if err != nil { - panic(err) - } - - return strfmt.DateTime(t) -} - -func timeMustPtr(t time.Time, err error) *strfmt.DateTime { - if err != nil { - panic(err) - } - - parsed := strfmt.DateTime(t) - return &parsed -} - -func ptString(in string) *string { - return &in -} - -var ByID = map[string]models.Deprecation{ -{{- range .Deprecations }} - {{ printf "%q" .ID }}: { - ID: {{ printf "%q" .ID }}, - Locations: []string{ - {{- range $index, $element := .Locations }} - {{ printf "%q," $element }} - {{- end }} - }, - Status: {{ printf "%q" .Status }}, - APIType: {{ printf "%q" .APIType }}, - Mitigation: {{ printf "%q" .Mitigation }}, - Msg: {{ printf "%q" .Msg }}, - SinceVersion: {{ printf "%q" .SinceVersion }}, - SinceTime: timeMust(time.Parse(time.RFC3339, {{ printf "%q" .SinceTime }})), - {{ if .RemovedIn -}} - RemovedIn: ptString({{ printf "%q" (DerefString .RemovedIn) }}), - {{- end }} - {{ if .RemovedTime -}} - RemovedTime: timeMustPtr(time.Parse(time.RFC3339, {{ printf "%q" .SinceTime }})), - {{- end }} - }, -{{- end }} -} -`)) diff --git a/deprecations/main.go b/deprecations/main.go deleted file mode 100644 index 0faf4435f37067460fa98f7d62a97a13cd4803eb..0000000000000000000000000000000000000000 --- a/deprecations/main.go +++ /dev/null @@ -1,21 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package deprecations - -import "github.com/sirupsen/logrus" - -//go:generate go run gen.go -//go:generate goimports -w data.go - -func Log(logger logrus.FieldLogger, id string) { - logger.WithField("deprecation", ByID[id]).Warning(ByID[id].Msg) -} diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index cf361c7d631697d7a1f2f7d94fe57ba415632315..0000000000000000000000000000000000000000 --- a/docker-compose.yml +++ /dev/null @@ -1,36 +0,0 @@ ---- -version: '3.4' -services: - weaviate: - command: - - --host - - 0.0.0.0 - - --port - - '8080' - - --scheme - - http - image: semitechnologies/weaviate:1.23.8 - ports: - - 8080:8080 - - 50051:50051 - volumes: - - weaviate_data:/var/lib/weaviate - restart: on-failure:0 - environment: - QUERY_DEFAULTS_LIMIT: 25 - AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true' - PERSISTENCE_DATA_PATH: '/var/lib/weaviate' - DEFAULT_VECTORIZER_MODULE: 'text2vec-transformers' - TRANSFORMERS_INFERENCE_API: http://t2v-transformers:8080 - ENABLE_MODULES: 'text2vec-transformers' - #ENABLE_MODULES: 'text2vec-cohere,text2vec-huggingface,text2vec-palm,text2vec-openai,generative-openai,generative-cohere,generative-palm,ref2vec-centroid,reranker-cohere,qna-openai' - #ENABLE_MODULES: 'text2vec-gpt4all' - #GPT4ALL_INFERENCE_API: "http://localhost:4891" - CLUSTER_HOSTNAME: 'node1' - t2v-transformers: - image: semitechnologies/transformers-inference:sentence-transformers-multi-qa-MiniLM-L6-cos-v1 - environment: - ENABLE_CUDA: 0 # set to 1 to enable -volumes: - weaviate_data: -... \ No newline at end of file diff --git a/docker-compose/Readme.md b/docker-compose/Readme.md deleted file mode 100644 index 0515e701b1cc31539803cce00950047ae76524f7..0000000000000000000000000000000000000000 --- a/docker-compose/Readme.md +++ /dev/null @@ -1,8 +0,0 @@ -## Docker-compose runtime files - -This folder used to contain example `docker-compose.yml` files for various -languages. Instead you can now configure your own docker-compose file - on the -fly - using Weaviate's documentation. - -Click here to [generate a `docker-compose.yml` file with your desired -configuration](https://weaviate.io/developers/weaviate/installation/docker-compose). diff --git a/docker-compose_Orig.yml b/docker-compose_Orig.yml deleted file mode 100644 index 01ee708f83c1dc16b03db9ace4c20ed959cddce0..0000000000000000000000000000000000000000 --- a/docker-compose_Orig.yml +++ /dev/null @@ -1,121 +0,0 @@ -## -# This docker-compose is used for developing Weaviate, i.e. for contributors. -# THIS IS NOT INTENDED FOR USERS !! -# -# To get a docker-compose file to run Weaviate, please follow the instructions at -# https://weaviate.io/developers/weaviate/installation/docker-compose -## -version: '3.4' -services: - contextionary: - image: semitechnologies/contextionary:en0.16.0-v1.2.1 - ports: - - "9999:9999" - environment: - EXTENSIONS_STORAGE_MODE: weaviate - EXTENSIONS_STORAGE_ORIGIN: http://host.docker.internal:8080 - OCCURRENCE_WEIGHT_LINEAR_FACTOR: 0.75 - LOG_LEVEL: debug - prometheus: - image: prom/prometheus:v2.46.0 - volumes: - - ./tools/dev/prometheus_config/:/etc/prometheus/ - - ./data/prometheus:/prometheus - command: - - '--config.file=/etc/prometheus/prometheus.yml' - - '--storage.tsdb.path=/prometheus' - - '--web.console.libraries=/usr/share/prometheus/console_libraries' - - '--web.console.templates=/usr/share/prometheus/consoles' - ports: - - "9090:9090" - grafana: - image: grafana/grafana-oss - ports: - - "3000:3000" - volumes: - - ./tools/dev/grafana/grafana.ini:/etc/grafana/grafana.ini - - ./tools/dev/grafana/datasource.yml:/etc/grafana/provisioning/datasources/prometheus.yml - - ./tools/dev/grafana/dashboard_provider.yml:/etc/grafana/provisioning/dashboards/dashboards.yml - - ./tools/dev/grafana/dashboards:/var/lib/grafana/dashboards - keycloak: - image: jboss/keycloak:5.0.0 - environment: - KEYCLOAK_USER: admin - KEYCLOAK_PASSWORD: admin - KEYCLOAK_IMPORT: /tmp/weaviate-realm.json - volumes: - - ./tools/dev/keycloak/weaviate-realm.json:/tmp/weaviate-realm.json - ports: - - "9090:8080" - t2v-transformers: - image: semitechnologies/transformers-inference:distilbert-base-uncased - ports: - - "8000:8080" - qna-transformers: - image: semitechnologies/qna-transformers:bert-large-uncased-whole-word-masking-finetuned-squad - ports: - - "8001:8080" - i2v-neural: - image: semitechnologies/img2vec-pytorch:resnet50 - ports: - - "8002:8080" - ner-transformers: - image: semitechnologies/ner-transformers:latest - ports: - - "8003:8080" - text-spellcheck: - image: semitechnologies/text-spellcheck-model:pyspellchecker-en - ports: - - "8004:8080" - multi2vec-clip: - image: semitechnologies/multi2vec-clip:sentence-transformers-clip-ViT-B-32-multilingual-v1 - ports: - - "8005:8080" - t2v-transformers-passage: - image: semitechnologies/transformers-inference:facebook-dpr-ctx_encoder-single-nq-base - ports: - - "8006:8080" - t2v-transformers-query: - image: semitechnologies/transformers-inference:facebook-dpr-question_encoder-single-nq-base - ports: - - "8007:8080" - sum-transformers: - image: semitechnologies/sum-transformers:facebook-bart-large-cnn - ports: - - "8008:8080" - reranker-transformers: - image: semitechnologies/reranker-transformers:cross-encoder-ms-marco-MiniLM-L-6-v2 - ports: - - "8009:8080" - t2v-gpt4all: - image: semitechnologies/gpt4all-inference:all-MiniLM-L6-v2 - ports: - - "8010:8080" - multi2vec-bind: - image: semitechnologies/multi2vec-bind:imagebind - ports: - - "8011:8080" - backup-s3: - image: minio/minio - ports: - - "9000:9000" - volumes: - - ./backups-s3:/data - environment: - MINIO_ROOT_USER: aws_access_key - MINIO_ROOT_PASSWORD: aws_secret_key - entrypoint: sh - command: -c 'mkdir -p /data/weaviate-backups && minio server /data' - backup-gcs: - image: oittaa/gcp-storage-emulator - ports: - - "9090:8080" - volumes: - - ./backups-gcs:/storage - backup-azure: - image: mcr.microsoft.com/azure-storage/azurite - ports: - - "10000:10000" - volumes: - - ./backups-azure:/data - command: "azurite --blobHost 0.0.0.0 --blobPort 10000" diff --git a/entities/additional/classification.go b/entities/additional/classification.go deleted file mode 100644 index e8640af573b1b6722eb6a5e5a243796fc682a281..0000000000000000000000000000000000000000 --- a/entities/additional/classification.go +++ /dev/null @@ -1,60 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package additional - -import "github.com/go-openapi/strfmt" - -type Classification struct { - BasedOn []string `json:"basedOn"` - ClassifiedFields []string `json:"classifiedFields"` - Completed strfmt.DateTime `json:"completed,omitempty"` - ID strfmt.UUID `json:"id,omitempty"` - Scope []string `json:"scope"` -} - -type Properties struct { - Classification bool `json:"classification"` - RefMeta bool `json:"refMeta"` - Vector bool `json:"vector"` - Certainty bool `json:"certainty"` - ID bool `json:"id"` - CreationTimeUnix bool `json:"creationTimeUnix"` - LastUpdateTimeUnix bool `json:"lastUpdateTimeUnix"` - ModuleParams map[string]interface{} `json:"moduleParams"` - Distance bool `json:"distance"` - Score bool `json:"score"` - ExplainScore bool `json:"explainScore"` - IsConsistent bool `json:"isConsistent"` - Group bool `json:"group"` - - // The User is not interested in returning props, we can skip any costly - // operation that isn't required. - NoProps bool `json:"noProps"` - - // ReferenceQuery is used to indicate that a search - // is being conducted on behalf of a referenced - // property. for example: this is relevant when a - // where filter operand is passed in with a path to - // a referenced class, rather than a path to one of - // its own props. - // - // The reason we need this indication is that - // without it, the sub-Search which is - // conducted to extract the reference propValuePair - // is conducted with the pagination set to whatever - // the QueryMaximumResults. if this value is set low - // relative to the number of objects being searched, - // weaviate will be unable to find enough results to - // make any comparisons, and erroneously returns - // empty, or with fewer results than expected. - ReferenceQuery bool `json:"-"` -} diff --git a/entities/additional/distance.go b/entities/additional/distance.go deleted file mode 100644 index 345df2887321a63a9e0c014fc4b56282f30ac4de..0000000000000000000000000000000000000000 --- a/entities/additional/distance.go +++ /dev/null @@ -1,30 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package additional - -func CertaintyToDistPtr(maybeCertainty *float64) (distPtr *float64) { - if maybeCertainty != nil { - dist := (1 - *maybeCertainty) * 2 - distPtr = &dist - } - return -} - -func CertaintyToDist(certainty float64) (dist float64) { - dist = (1 - certainty) * 2 - return -} - -func DistToCertainty(dist float64) (certainty float64) { - certainty = 1 - (dist / 2) - return -} diff --git a/entities/additional/group.go b/entities/additional/group.go deleted file mode 100644 index 0f2abb9af2c8d94c3fca60f30653b44c052dab9b..0000000000000000000000000000000000000000 --- a/entities/additional/group.go +++ /dev/null @@ -1,34 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package additional - -import "github.com/go-openapi/strfmt" - -type Group struct { - ID int `json:"id"` - GroupedBy *GroupedBy `json:"groupedBy"` - MinDistance float32 `json:"minDistance"` - MaxDistance float32 `json:"maxDistance"` - Count int `json:"count"` - Hits []map[string]interface{} `json:"hits"` -} - -type GroupedBy struct { - Value string `json:"value"` - Path []string `json:"path"` -} - -type GroupHitAdditional struct { - ID strfmt.UUID `json:"id"` - Vector []float32 `json:"vector"` - Distance float32 `json:"distance"` -} diff --git a/entities/additional/replication.go b/entities/additional/replication.go deleted file mode 100644 index 9a6eb665880a4ed721d0e57dd60eecacc860634f..0000000000000000000000000000000000000000 --- a/entities/additional/replication.go +++ /dev/null @@ -1,29 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package additional - -// ReplicationProperties are replication-related handles and configurations which -// allow replication context to pass through different layers of -// abstraction, usually initiated via client requests -type ReplicationProperties struct { - // ConsistencyLevel indicates how many nodes should - // respond to a request before it is considered - // successful. Can be "ONE", "QUORUM", or "ALL" - // - // This is only relevant for a replicated - // class - ConsistencyLevel string - - // NodeName is the node which is expected to - // fulfill the request - NodeName string -} diff --git a/entities/aggregation/params.go b/entities/aggregation/params.go deleted file mode 100644 index 18ffcedb67d908837efbefde70855a6a2d143ea2..0000000000000000000000000000000000000000 --- a/entities/aggregation/params.go +++ /dev/null @@ -1,137 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package aggregation - -import ( - "fmt" - - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/searchparams" -) - -type Params struct { - Filters *filters.LocalFilter `json:"filters"` - ClassName schema.ClassName `json:"className"` - Properties []ParamProperty `json:"properties"` - GroupBy *filters.Path `json:"groupBy"` - IncludeMetaCount bool `json:"includeMetaCount"` - Limit *int `json:"limit"` - ObjectLimit *int `json:"objectLimit"` - SearchVector []float32 `json:"searchVector"` - Certainty float64 `json:"certainty"` - Tenant string `json:"tenant"` - ModuleParams map[string]interface{} `json:"moduleParams"` - NearVector *searchparams.NearVector `json:"nearVector"` - NearObject *searchparams.NearObject `json:"nearObject"` - Hybrid *searchparams.HybridSearch `json:"hybrid"` -} - -type ParamProperty struct { - Name schema.PropertyName `json:"name"` - Aggregators []Aggregator `json:"aggregators"` -} - -type Aggregator struct { - Type string `json:"type"` - Limit *int `json:"limit"` // used on TopOccurrence Agg -} - -func (a Aggregator) String() string { - return a.Type -} - -// Aggregators used in every prop -var ( - CountAggregator = Aggregator{Type: "count"} - TypeAggregator = Aggregator{Type: "type"} -) - -// Aggregators used in numerical props -var ( - SumAggregator = Aggregator{Type: "sum"} - MeanAggregator = Aggregator{Type: "mean"} - ModeAggregator = Aggregator{Type: "mode"} - MedianAggregator = Aggregator{Type: "median"} - MaximumAggregator = Aggregator{Type: "maximum"} - MinimumAggregator = Aggregator{Type: "minimum"} -) - -// Aggregators used in boolean props -var ( - TotalTrueAggregator = Aggregator{Type: "totalTrue"} - PercentageTrueAggregator = Aggregator{Type: "percentageTrue"} - TotalFalseAggregator = Aggregator{Type: "totalFalse"} - PercentageFalseAggregator = Aggregator{Type: "percentageFalse"} -) - -const TopOccurrencesType = "topOccurrences" - -// NewTopOccurrencesAggregator creates a TopOccurrencesAggregator, we cannot -// use a singleton for this as the desired limit can be different each time -func NewTopOccurrencesAggregator(limit *int) Aggregator { - return Aggregator{Type: TopOccurrencesType, Limit: limit} -} - -// Aggregators used in ref props -var ( - PointingToAggregator = Aggregator{Type: "pointingTo"} -) - -func ParseAggregatorProp(name string) (Aggregator, error) { - switch name { - // common - case CountAggregator.String(): - return CountAggregator, nil - case TypeAggregator.String(): - return TypeAggregator, nil - - // numerical - case MeanAggregator.String(): - return MeanAggregator, nil - case MedianAggregator.String(): - return MedianAggregator, nil - case ModeAggregator.String(): - return ModeAggregator, nil - case MaximumAggregator.String(): - return MaximumAggregator, nil - case MinimumAggregator.String(): - return MinimumAggregator, nil - case SumAggregator.String(): - return SumAggregator, nil - - // boolean - case TotalTrueAggregator.String(): - return TotalTrueAggregator, nil - case TotalFalseAggregator.String(): - return TotalFalseAggregator, nil - case PercentageTrueAggregator.String(): - return PercentageTrueAggregator, nil - case PercentageFalseAggregator.String(): - return PercentageFalseAggregator, nil - - // string/text - case TopOccurrencesType: - return NewTopOccurrencesAggregator(ptInt(5)), nil // default to limit 5, can be overwritten - - // ref - case PointingToAggregator.String(): - return PointingToAggregator, nil - - default: - return Aggregator{}, fmt.Errorf("unrecognized aggregator prop '%s'", name) - } -} - -func ptInt(in int) *int { - return &in -} diff --git a/entities/aggregation/result.go b/entities/aggregation/result.go deleted file mode 100644 index 24d18b4ab8601bebdaa0663203a2d3a55291d40e..0000000000000000000000000000000000000000 --- a/entities/aggregation/result.go +++ /dev/null @@ -1,69 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package aggregation - -type Result struct { - Groups []Group `json:"groups"` -} - -type Group struct { - Properties map[string]Property `json:"properties"` - GroupedBy *GroupedBy `json:"groupedBy"` // optional to support ungrouped aggregations (formerly meta) - Count int `json:"count"` -} - -type Property struct { - Type PropertyType `json:"type"` - NumericalAggregations map[string]interface{} `json:"numericalAggregations"` - TextAggregation Text `json:"textAggregation"` - BooleanAggregation Boolean `json:"booleanAggregation"` - SchemaType string `json:"schemaType"` - ReferenceAggregation Reference `json:"referenceAggregation"` - DateAggregations map[string]interface{} `json:"dateAggregation"` -} - -type Text struct { - Items []TextOccurrence `json:"items"` - Count int `json:"count"` -} - -type PropertyType string - -const ( - PropertyTypeNumerical PropertyType = "numerical" - PropertyTypeBoolean PropertyType = "boolean" - PropertyTypeText PropertyType = "text" - PropertyTypeDate PropertyType = "date" - PropertyTypeReference PropertyType = "cref" -) - -type GroupedBy struct { - Value interface{} `json:"value"` - Path []string `json:"path"` -} - -type TextOccurrence struct { - Value string `json:"value"` - Occurs int `json:"occurs"` -} - -type Boolean struct { - Count int `json:"count"` - TotalTrue int `json:"totalTrue"` - TotalFalse int `json:"totalFalse"` - PercentageTrue float64 `json:"percentageTrue"` - PercentageFalse float64 `json:"percentageFalse"` -} - -type Reference struct { - PointingTo []string `json:"pointingTo"` -} diff --git a/entities/autocut/autocut.go b/entities/autocut/autocut.go deleted file mode 100644 index e310a89a7187a58e02d397212bfa102dcfbcc4cb..0000000000000000000000000000000000000000 --- a/entities/autocut/autocut.go +++ /dev/null @@ -1,51 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package autocut - -func Autocut(yValues []float32, cutOff int) int { - if len(yValues) <= 1 { - return len(yValues) - } - - diff := make([]float32, len(yValues)) - step := 1. / (float32(len(yValues)) - 1.) - - for i := range yValues { - xValue := 0. + float32(i)*step - yValueNorm := (yValues[i] - yValues[0]) / (yValues[len(yValues)-1] - yValues[0]) - diff[i] = yValueNorm - xValue - } - - extremaCount := 0 - for i := range diff { - if i == 0 { - continue // we want the index _before_ the extrema - } - - if i == len(diff)-1 && len(diff) > 1 { // for last element there is no "next" point - if diff[i] > diff[i-1] && diff[i] > diff[i-2] { - extremaCount += 1 - if extremaCount >= cutOff { - return i - } - } - } else { - if diff[i] > diff[i-1] && diff[i] > diff[i+1] { - extremaCount += 1 - if extremaCount >= cutOff { - return i - } - } - } - } - return len(yValues) -} diff --git a/entities/autocut/autocut_test.go b/entities/autocut/autocut_test.go deleted file mode 100644 index dcbf7ecc9a46a65daebca6c4d7126981efd2ca1c..0000000000000000000000000000000000000000 --- a/entities/autocut/autocut_test.go +++ /dev/null @@ -1,46 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package autocut - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestAutoCut(t *testing.T) { - cases := []struct { - values []float32 - cutOff int - expectedResults int - }{ - {values: []float32{}, cutOff: 1, expectedResults: 0}, - {values: []float32{2}, cutOff: 1, expectedResults: 1}, - {values: []float32{2, 1.95, 1.9, 0.2, 0.1, 0.1, -1}, cutOff: 1, expectedResults: 3}, - {values: []float32{2, 1.95, 1.9, 0.2, 0.1, 0.1, -2}, cutOff: 2, expectedResults: 6}, - {values: []float32{5, 1, 1, 1, 1, 0, 0}, cutOff: 1, expectedResults: 1}, - {values: []float32{5, 1, 1, 1, 1, 0, 0}, cutOff: 2, expectedResults: 5}, - {values: []float32{0.298, 0.260, 0.169, 0.108, 0.108, 0.104, 0.093}, cutOff: 1, expectedResults: 3}, - {values: []float32{0.5, 0.32, 0.31, 0.30, 0.29, 0.15}, cutOff: 1, expectedResults: 1}, - {values: []float32{0.5, 0.32, 0.31, 0.30, 0.29, 0.15, 0.15, 0.15}, cutOff: 2, expectedResults: 5}, - {values: []float32{1.0, 0.98, 0.95, 0.9, 0.88, 0.87, 0.80, 0.79}, cutOff: 1, expectedResults: 3}, - {values: []float32{1.0, 0.98, 0.95, 0.9, 0.88, 0.87, 0.80, 0.79}, cutOff: 2, expectedResults: 6}, - {values: []float32{1.0, 0.98, 0.95, 0.9, 0.88, 0.87, 0.80, 0.79}, cutOff: 3, expectedResults: 8}, // all values - {values: []float32{0.586835, 0.5450372, 0.34137487, 0.30482167, 0.2753393}, cutOff: 1, expectedResults: 2}, - {values: []float32{0.36663342, 0.33818772, 0.045160502, 0.045160501}, cutOff: 1, expectedResults: 2}, - } - for _, tt := range cases { - t.Run("", func(t *testing.T) { - assert.Equal(t, tt.expectedResults, Autocut(tt.values, tt.cutOff)) - }) - } -} diff --git a/entities/backup/descriptor.go b/entities/backup/descriptor.go deleted file mode 100644 index 7655a7a1db52f505c2fb187cded5aff96d071b3f..0000000000000000000000000000000000000000 --- a/entities/backup/descriptor.go +++ /dev/null @@ -1,403 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package backup - -import ( - "fmt" - "time" -) - -// NodeDescriptor contains data related to one participant in DBRO -type NodeDescriptor struct { - Classes []string `json:"classes"` - Status Status `json:"status"` - Error string `json:"error"` -} - -// DistributedBAckupDescriptor contains everything need to completely restore a distributed backup -type DistributedBackupDescriptor struct { - StartedAt time.Time `json:"startedAt"` - CompletedAt time.Time `json:"completedAt"` - ID string `json:"id"` // User created backup id - Nodes map[string]*NodeDescriptor `json:"nodes"` - NodeMapping map[string]string `json:"node_mapping"` - Status Status `json:"status"` // - Version string `json:"version"` // - ServerVersion string `json:"serverVersion"` - Error string `json:"error"` -} - -// Len returns how many nodes exist in d -func (d *DistributedBackupDescriptor) Len() int { - return len(d.Nodes) -} - -// Count number of classes -func (d *DistributedBackupDescriptor) Count() int { - count := 0 - for _, desc := range d.Nodes { - count += len(desc.Classes) - } - return count -} - -// RemoveEmpty removes any nodes with an empty class list -func (d *DistributedBackupDescriptor) RemoveEmpty() *DistributedBackupDescriptor { - for node, desc := range d.Nodes { - if len(desc.Classes) == 0 { - delete(d.Nodes, node) - } - } - return d -} - -// Classes returns all classes contained in d -func (d *DistributedBackupDescriptor) Classes() []string { - set := make(map[string]struct{}, 32) - for _, desc := range d.Nodes { - for _, cls := range desc.Classes { - set[cls] = struct{}{} - } - } - lst := make([]string, len(set)) - i := 0 - for cls := range set { - lst[i] = cls - i++ - } - return lst -} - -// Filter classes based on predicate -func (d *DistributedBackupDescriptor) Filter(pred func(s string) bool) { - for _, desc := range d.Nodes { - cs := make([]string, 0, len(desc.Classes)) - for _, cls := range desc.Classes { - if pred(cls) { - cs = append(cs, cls) - } - } - if len(cs) != len(desc.Classes) { - desc.Classes = cs - } - } -} - -// Include only these classes and remove everything else -func (d *DistributedBackupDescriptor) Include(classes []string) { - if len(classes) == 0 { - return - } - set := make(map[string]struct{}, len(classes)) - for _, cls := range classes { - set[cls] = struct{}{} - } - pred := func(s string) bool { - _, ok := set[s] - return ok - } - d.Filter(pred) -} - -// Exclude removes classes from d -func (d *DistributedBackupDescriptor) Exclude(classes []string) { - if len(classes) == 0 { - return - } - set := make(map[string]struct{}, len(classes)) - for _, cls := range classes { - set[cls] = struct{}{} - } - pred := func(s string) bool { - _, ok := set[s] - return !ok - } - d.Filter(pred) -} - -// ToMappedNodeName will return nodeName after applying d.NodeMapping translation on it. -// If nodeName is not contained in d.nodeMapping, returns nodeName unmodified -func (d *DistributedBackupDescriptor) ToMappedNodeName(nodeName string) string { - if newNodeName, ok := d.NodeMapping[nodeName]; ok { - return newNodeName - } - return nodeName -} - -// ToOriginalNodeName will return nodeName after trying to find an original node name from d.NodeMapping values. -// If nodeName is not contained in d.nodeMapping values, returns nodeName unmodified -func (d *DistributedBackupDescriptor) ToOriginalNodeName(nodeName string) string { - for oldNodeName, newNodeName := range d.NodeMapping { - if newNodeName == nodeName { - return oldNodeName - } - } - return nodeName -} - -// ApplyNodeMapping applies d.NodeMapping translation to d.Nodes. If a node in d.Nodes is not translated by d.NodeMapping, it will remain -// unchanged. -func (d *DistributedBackupDescriptor) ApplyNodeMapping() { - if len(d.NodeMapping) == 0 { - return - } - - for k, v := range d.NodeMapping { - if nodeDescriptor, ok := d.Nodes[k]; !ok { - d.Nodes[v] = nodeDescriptor - delete(d.Nodes, k) - } - } -} - -// AllExist checks if all classes exist in d. -// It returns either "" or the first class which it could not find -func (d *DistributedBackupDescriptor) AllExist(classes []string) string { - if len(classes) == 0 { - return "" - } - set := make(map[string]struct{}, len(classes)) - for _, cls := range classes { - set[cls] = struct{}{} - } - for _, dest := range d.Nodes { - for _, cls := range dest.Classes { - delete(set, cls) - if len(set) == 0 { - return "" - } - } - } - first := "" - for k := range set { - first = k - break - } - return first -} - -func (d *DistributedBackupDescriptor) Validate() error { - if d.StartedAt.IsZero() || d.ID == "" || - d.Version == "" || d.ServerVersion == "" || d.Error != "" { - return fmt.Errorf("attribute mismatch: [id versions time error]") - } - if len(d.Nodes) == 0 { - return fmt.Errorf("empty list of node descriptors") - } - return nil -} - -// resetStatus sets status and sub-statuses to Started -// It also empties error and sub-errors -func (d *DistributedBackupDescriptor) ResetStatus() *DistributedBackupDescriptor { - d.Status = Started - d.Error = "" - d.StartedAt = time.Now() - d.CompletedAt = time.Time{} - for _, node := range d.Nodes { - node.Status = Started - node.Error = "" - } - return d -} - -// ShardDescriptor contains everything needed to completely restore a partition of a specific class -type ShardDescriptor struct { - Name string `json:"name"` - Node string `json:"node"` - Files []string `json:"files,omitempty"` - - DocIDCounterPath string `json:"docIdCounterPath,omitempty"` - DocIDCounter []byte `json:"docIdCounter,omitempty"` - PropLengthTrackerPath string `json:"propLengthTrackerPath,omitempty"` - PropLengthTracker []byte `json:"propLengthTracker,omitempty"` - ShardVersionPath string `json:"shardVersionPath,omitempty"` - Version []byte `json:"version,omitempty"` - Chunk int32 `json:"chunk"` -} - -// ClearTemporary clears fields that are no longer needed once compression is done. -// These fields are not required in versions > 1 because they are stored in the tarball. -func (s *ShardDescriptor) ClearTemporary() { - s.ShardVersionPath = "" - s.Version = nil - - s.DocIDCounterPath = "" - s.DocIDCounter = nil - - s.PropLengthTrackerPath = "" - s.PropLengthTracker = nil -} - -// ClassDescriptor contains everything needed to completely restore a class -type ClassDescriptor struct { - Name string `json:"name"` // DB class name, also selected by user - Shards []*ShardDescriptor `json:"shards"` - ShardingState []byte `json:"shardingState"` - Schema []byte `json:"schema"` - Chunks map[int32][]string `json:"chunks,omitempty"` - Error error `json:"-"` -} - -// BackupDescriptor contains everything needed to completely restore a list of classes -type BackupDescriptor struct { - StartedAt time.Time `json:"startedAt"` - CompletedAt time.Time `json:"completedAt"` - ID string `json:"id"` // User created backup id - Classes []ClassDescriptor `json:"classes"` - Status string `json:"status"` // "STARTED|TRANSFERRING|TRANSFERRED|SUCCESS|FAILED" - Version string `json:"version"` // - ServerVersion string `json:"serverVersion"` - Error string `json:"error"` -} - -// List all existing classes in d -func (d *BackupDescriptor) List() []string { - lst := make([]string, len(d.Classes)) - for i, cls := range d.Classes { - lst[i] = cls.Name - } - return lst -} - -// AllExist checks if all classes exist in d. -// It returns either "" or the first class which it could not find -func (d *BackupDescriptor) AllExist(classes []string) string { - if len(classes) == 0 { - return "" - } - set := make(map[string]struct{}, len(classes)) - for _, cls := range classes { - set[cls] = struct{}{} - } - for _, dest := range d.Classes { - delete(set, dest.Name) - } - first := "" - for k := range set { - first = k - break - } - return first -} - -// Include only these classes and remove everything else -func (d *BackupDescriptor) Include(classes []string) { - if len(classes) == 0 { - return - } - set := make(map[string]struct{}, len(classes)) - for _, cls := range classes { - set[cls] = struct{}{} - } - pred := func(s string) bool { - _, ok := set[s] - return ok - } - d.Filter(pred) -} - -// Exclude removes classes from d -func (d *BackupDescriptor) Exclude(classes []string) { - if len(classes) == 0 { - return - } - set := make(map[string]struct{}, len(classes)) - for _, cls := range classes { - set[cls] = struct{}{} - } - pred := func(s string) bool { - _, ok := set[s] - return !ok - } - d.Filter(pred) -} - -// Filter classes based on predicate -func (d *BackupDescriptor) Filter(pred func(s string) bool) { - cs := make([]ClassDescriptor, 0, len(d.Classes)) - for _, dest := range d.Classes { - if pred(dest.Name) { - cs = append(cs, dest) - } - } - d.Classes = cs -} - -// ValidateV1 validates d -func (d *BackupDescriptor) validateV1() error { - for _, c := range d.Classes { - if c.Name == "" || len(c.Schema) == 0 || len(c.ShardingState) == 0 { - return fmt.Errorf("invalid class %q: [name schema sharding]", c.Name) - } - for _, s := range c.Shards { - n := len(s.Files) - if s.Name == "" || s.Node == "" || s.DocIDCounterPath == "" || - s.ShardVersionPath == "" || s.PropLengthTrackerPath == "" || - (n > 0 && (len(s.DocIDCounter) == 0 || - len(s.PropLengthTracker) == 0 || - len(s.Version) == 0)) { - return fmt.Errorf("invalid shard %q.%q", c.Name, s.Name) - } - for i, fpath := range s.Files { - if fpath == "" { - return fmt.Errorf("invalid shard %q.%q: file number %d", c.Name, s.Name, i) - } - } - } - } - return nil -} - -func (d *BackupDescriptor) Validate(newSchema bool) error { - if d.StartedAt.IsZero() || d.ID == "" || - d.Version == "" || d.ServerVersion == "" || d.Error != "" { - return fmt.Errorf("attribute mismatch: [id versions time error]") - } - if !newSchema { - return d.validateV1() - } - for _, c := range d.Classes { - if c.Name == "" || len(c.Schema) == 0 || len(c.ShardingState) == 0 { - return fmt.Errorf("class=%q: invalid attributes [name schema sharding]", c.Name) - } - for _, s := range c.Shards { - if s.Name == "" || s.Node == "" { - return fmt.Errorf("class=%q: invalid shard %q node=%q", c.Name, s.Name, s.Node) - } - } - } - return nil -} - -// ToDistributed is used just for backward compatibility with the old version. -func (d *BackupDescriptor) ToDistributed() *DistributedBackupDescriptor { - node, cs := "", d.List() - for _, xs := range d.Classes { - for _, s := range xs.Shards { - node = s.Node - } - } - result := &DistributedBackupDescriptor{ - StartedAt: d.StartedAt, - CompletedAt: d.CompletedAt, - ID: d.ID, - Status: Status(d.Status), - Version: d.Version, - ServerVersion: d.ServerVersion, - Error: d.Error, - } - if node != "" && len(cs) > 0 { - result.Nodes = map[string]*NodeDescriptor{node: {Classes: cs}} - } - return result -} diff --git a/entities/backup/descriptor_test.go b/entities/backup/descriptor_test.go deleted file mode 100644 index d9743e8690dac76df6c52ca7b851988e57ef5847..0000000000000000000000000000000000000000 --- a/entities/backup/descriptor_test.go +++ /dev/null @@ -1,475 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package backup - -import ( - "sort" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func TestExcludeClasses(t *testing.T) { - tests := []struct { - in BackupDescriptor - xs []string - out []string - }{ - {in: BackupDescriptor{}, xs: []string{}, out: []string{}}, - {in: BackupDescriptor{Classes: []ClassDescriptor{{Name: "a"}}}, xs: []string{}, out: []string{"a"}}, - {in: BackupDescriptor{Classes: []ClassDescriptor{{Name: "a"}}}, xs: []string{"a"}, out: []string{}}, - {in: BackupDescriptor{Classes: []ClassDescriptor{{Name: "1"}, {Name: "2"}, {Name: "3"}, {Name: "4"}}}, xs: []string{"2", "3"}, out: []string{"1", "4"}}, - {in: BackupDescriptor{Classes: []ClassDescriptor{{Name: "1"}, {Name: "2"}, {Name: "3"}}}, xs: []string{"1", "3"}, out: []string{"2"}}, - - // {in: []BackupDescriptor{"1", "2", "3", "4"}, xs: []string{"2", "3"}, out: []string{"1", "4"}}, - // {in: []BackupDescriptor{"1", "2", "3"}, xs: []string{"1", "3"}, out: []string{"2"}}, - } - for _, tc := range tests { - tc.in.Exclude(tc.xs) - lst := tc.in.List() - assert.Equal(t, tc.out, lst) - } -} - -func TestIncludeClasses(t *testing.T) { - tests := []struct { - in BackupDescriptor - xs []string - out []string - }{ - {in: BackupDescriptor{}, xs: []string{}, out: []string{}}, - {in: BackupDescriptor{Classes: []ClassDescriptor{{Name: "a"}}}, xs: []string{}, out: []string{"a"}}, - {in: BackupDescriptor{Classes: []ClassDescriptor{{Name: "a"}}}, xs: []string{"a"}, out: []string{"a"}}, - {in: BackupDescriptor{Classes: []ClassDescriptor{{Name: "1"}, {Name: "2"}, {Name: "3"}, {Name: "4"}}}, xs: []string{"2", "3"}, out: []string{"2", "3"}}, - {in: BackupDescriptor{Classes: []ClassDescriptor{{Name: "1"}, {Name: "2"}, {Name: "3"}}}, xs: []string{"1", "3"}, out: []string{"1", "3"}}, - } - for _, tc := range tests { - tc.in.Include(tc.xs) - lst := tc.in.List() - assert.Equal(t, tc.out, lst) - } -} - -func TestAllExist(t *testing.T) { - x := BackupDescriptor{Classes: []ClassDescriptor{{Name: "a"}}} - if y := x.AllExist(nil); y != "" { - t.Errorf("x.AllExists(nil) got=%v want=%v", y, "") - } - if y := x.AllExist([]string{"a"}); y != "" { - t.Errorf("x.AllExists(['a']) got=%v want=%v", y, "") - } - if y := x.AllExist([]string{"b"}); y != "b" { - t.Errorf("x.AllExists(['a']) got=%v want=%v", y, "b") - } -} - -func TestValidateBackup(t *testing.T) { - timept := time.Now().UTC() - bytes := []byte("hello") - tests := []struct { - desc BackupDescriptor - successV1 bool - successV2 bool - }{ - // first level check - {desc: BackupDescriptor{}}, - {desc: BackupDescriptor{ID: "1"}}, - {desc: BackupDescriptor{ID: "1", Version: "1"}}, - {desc: BackupDescriptor{ID: "1", Version: "1", ServerVersion: "1"}}, - { - desc: BackupDescriptor{ID: "1", Version: "1", ServerVersion: "1", StartedAt: timept}, - successV1: true, successV2: true, - }, - {desc: BackupDescriptor{ID: "1", Version: "1", ServerVersion: "1", StartedAt: timept, Error: "err"}}, - {desc: BackupDescriptor{ - ID: "1", Version: "1", ServerVersion: "1", StartedAt: timept, - Classes: []ClassDescriptor{{}}, - }}, - {desc: BackupDescriptor{ - ID: "1", Version: "1", ServerVersion: "1", StartedAt: timept, - Classes: []ClassDescriptor{{Name: "n"}}, - }}, - {desc: BackupDescriptor{ - ID: "1", Version: "1", ServerVersion: "1", StartedAt: timept, - Classes: []ClassDescriptor{{Name: "n", Schema: bytes}}, - }}, - {desc: BackupDescriptor{ - ID: "1", Version: "1", ServerVersion: "1", StartedAt: timept, - Classes: []ClassDescriptor{{Name: "n", Schema: bytes, ShardingState: bytes}}, - }, successV1: true, successV2: true}, - {desc: BackupDescriptor{ - ID: "1", Version: "1", ServerVersion: "1", StartedAt: timept, - Classes: []ClassDescriptor{{ - Name: "n", Schema: bytes, ShardingState: bytes, - Shards: []*ShardDescriptor{{Name: ""}}, - }}, - }}, - {desc: BackupDescriptor{ - ID: "1", Version: "1", ServerVersion: "1", StartedAt: timept, - Classes: []ClassDescriptor{{ - Name: "n", Schema: bytes, ShardingState: bytes, - Shards: []*ShardDescriptor{{Name: "n", Node: ""}}, - }}, - }}, - {desc: BackupDescriptor{ - ID: "1", Version: "1", ServerVersion: "1", StartedAt: timept, - Classes: []ClassDescriptor{{ - Name: "n", Schema: bytes, ShardingState: bytes, - Shards: []*ShardDescriptor{{Name: "n", Node: "n"}}, - }}, - }, successV2: true}, - {desc: BackupDescriptor{ - ID: "1", Version: "1", ServerVersion: "1", StartedAt: timept, - Classes: []ClassDescriptor{{ - Name: "n", Schema: bytes, ShardingState: bytes, - Shards: []*ShardDescriptor{{ - Name: "n", Node: "n", - PropLengthTrackerPath: "n", DocIDCounterPath: "n", ShardVersionPath: "n", - }}, - }}, - }, successV1: true, successV2: true}, - {desc: BackupDescriptor{ - ID: "1", Version: "1", ServerVersion: "1", StartedAt: timept, - Classes: []ClassDescriptor{{ - Name: "n", Schema: bytes, ShardingState: bytes, - Shards: []*ShardDescriptor{{ - Name: "n", Node: "n", - PropLengthTrackerPath: "n", DocIDCounterPath: "n", ShardVersionPath: "n", - Files: []string{"file"}, - }}, - }}, - }, successV2: true}, - {desc: BackupDescriptor{ - ID: "1", Version: "1", ServerVersion: "1", StartedAt: timept, - Classes: []ClassDescriptor{{ - Name: "n", Schema: bytes, ShardingState: bytes, - Shards: []*ShardDescriptor{{ - Name: "n", Node: "n", - PropLengthTrackerPath: "n", DocIDCounterPath: "n", ShardVersionPath: "n", - DocIDCounter: bytes, Files: []string{"file"}, - }}, - }}, - }, successV2: true}, - {desc: BackupDescriptor{ - ID: "1", Version: "1", ServerVersion: "1", StartedAt: timept, - Classes: []ClassDescriptor{{ - Name: "n", Schema: bytes, ShardingState: bytes, - Shards: []*ShardDescriptor{{ - Name: "n", Node: "n", - PropLengthTrackerPath: "n", DocIDCounterPath: "n", ShardVersionPath: "n", - DocIDCounter: bytes, Version: bytes, PropLengthTracker: bytes, Files: []string{""}, - }}, - }}, - }, successV2: true}, - {desc: BackupDescriptor{ - ID: "1", Version: "1", ServerVersion: "1", StartedAt: timept, - Classes: []ClassDescriptor{{ - Name: "n", Schema: bytes, ShardingState: bytes, - Shards: []*ShardDescriptor{{ - Name: "n", Node: "n", - PropLengthTrackerPath: "n", DocIDCounterPath: "n", ShardVersionPath: "n", - DocIDCounter: bytes, Version: bytes, PropLengthTracker: bytes, Files: []string{"file"}, - }}, - }}, - }, successV1: true, successV2: true}, - } - for i, tc := range tests { - err := tc.desc.Validate(false) - if got := err == nil; got != tc.successV1 { - t.Errorf("%d. validate(%+v): want=%v got=%v err=%v", i, tc.desc, tc.successV1, got, err) - } - err = tc.desc.Validate(true) - if got := err == nil; got != tc.successV2 { - t.Errorf("%d. validate(%+v): want=%v got=%v err=%v", i, tc.desc, tc.successV1, got, err) - } - } -} - -func TestBackwardCompatibility(t *testing.T) { - timept := time.Now().UTC() - tests := []struct { - desc BackupDescriptor - success bool - }{ - // first level check - {desc: BackupDescriptor{}}, - {desc: BackupDescriptor{ID: "1"}}, - {desc: BackupDescriptor{ID: "1", Version: "1"}}, - {desc: BackupDescriptor{ID: "1", Version: "1", ServerVersion: "1"}}, - {desc: BackupDescriptor{ID: "1", Version: "1", ServerVersion: "1", StartedAt: timept}}, - {desc: BackupDescriptor{ID: "1", Version: "1", ServerVersion: "1", StartedAt: timept, Error: "err"}}, - {desc: BackupDescriptor{ - ID: "1", Version: "1", ServerVersion: "1", StartedAt: timept, - Classes: []ClassDescriptor{{ - Name: "n", - Shards: []*ShardDescriptor{{Name: "n", Node: ""}}, - }}, - }}, - {desc: BackupDescriptor{ - ID: "1", Version: "1", ServerVersion: "1", StartedAt: timept, - Classes: []ClassDescriptor{{ - Name: "n", - Shards: []*ShardDescriptor{{ - Name: "n", Node: "n", - }}, - }}, - }, success: true}, - } - for i, tc := range tests { - desc := tc.desc.ToDistributed() - err := desc.Validate() - if got := err == nil; got != tc.success { - t.Errorf("%d. validate(%+v): want=%v got=%v err=%v", i, tc.desc, tc.success, got, err) - } - } -} - -func TestDistributedBackup(t *testing.T) { - d := DistributedBackupDescriptor{ - Nodes: map[string]*NodeDescriptor{ - "N1": {Classes: []string{"1", "2"}}, - "N2": {Classes: []string{"3", "4"}}, - }, - } - if n := d.Len(); n != 2 { - t.Errorf("#nodes got:%v want:%v", n, 2) - } - if n := d.Count(); n != 4 { - t.Errorf("#classes got:%v want:%v", n, 4) - } - d.Exclude([]string{"3", "4"}) - d.RemoveEmpty() - if n := d.Len(); n != 1 { - t.Errorf("#nodes got:%v want:%v", n, 2) - } - if n := d.Count(); n != 2 { - t.Errorf("#classes got:%v want:%v", n, 4) - } -} - -func TestDistributedBackupExcludeClasses(t *testing.T) { - tests := []struct { - in DistributedBackupDescriptor - xs []string - out []string - }{ - { - in: DistributedBackupDescriptor{}, - xs: []string{}, - out: []string{}, - }, - { - in: DistributedBackupDescriptor{ - Nodes: map[string]*NodeDescriptor{ - "N1": {Classes: []string{"a"}}, - }, - }, - xs: []string{}, - out: []string{"a"}, - }, - { - in: DistributedBackupDescriptor{ - Nodes: map[string]*NodeDescriptor{ - "N1": {Classes: []string{"a"}}, - }, - }, - xs: []string{"a"}, - out: []string{}, - }, - { - in: DistributedBackupDescriptor{ - Nodes: map[string]*NodeDescriptor{ - "N1": {Classes: []string{"1", "2"}}, - "N2": {Classes: []string{"3", "4"}}, - }, - }, - xs: []string{"2", "3"}, - out: []string{"1", "4"}, - }, - - { - in: DistributedBackupDescriptor{ - Nodes: map[string]*NodeDescriptor{ - "N1": {Classes: []string{"1", "2"}}, - "N2": {Classes: []string{"3"}}, - }, - }, - xs: []string{"1", "3"}, - out: []string{"2"}, - }, - } - - for _, tc := range tests { - tc.in.Exclude(tc.xs) - lst := tc.in.Classes() - sort.Strings(lst) - assert.Equal(t, tc.out, lst) - } -} - -func TestDistributedBackupIncludeClasses(t *testing.T) { - tests := []struct { - in DistributedBackupDescriptor - xs []string - out []string - }{ - { - in: DistributedBackupDescriptor{}, - xs: []string{}, - out: []string{}, - }, - { - in: DistributedBackupDescriptor{ - Nodes: map[string]*NodeDescriptor{ - "N1": {Classes: []string{"a"}}, - }, - }, - xs: []string{}, - out: []string{"a"}, - }, - { - in: DistributedBackupDescriptor{ - Nodes: map[string]*NodeDescriptor{ - "N1": {Classes: []string{"a"}}, - }, - }, - xs: []string{"a"}, - out: []string{"a"}, - }, - { - in: DistributedBackupDescriptor{ - Nodes: map[string]*NodeDescriptor{ - "N1": {Classes: []string{"1", "2"}}, - "N2": {Classes: []string{"3", "4"}}, - }, - }, - xs: []string{"2", "3"}, - out: []string{"2", "3"}, - }, - - { - in: DistributedBackupDescriptor{ - Nodes: map[string]*NodeDescriptor{ - "N1": {Classes: []string{"1", "2"}}, - "N2": {Classes: []string{"3"}}, - }, - }, - xs: []string{"1", "3"}, - out: []string{"1", "3"}, - }, - } - for _, tc := range tests { - tc.in.Include(tc.xs) - lst := tc.in.Classes() - sort.Strings(lst) - assert.Equal(t, tc.out, lst) - } -} - -func TestDistributedBackupAllExist(t *testing.T) { - x := DistributedBackupDescriptor{Nodes: map[string]*NodeDescriptor{"N1": {Classes: []string{"a"}}}} - if y := x.AllExist(nil); y != "" { - t.Errorf("x.AllExists(nil) got=%v want=%v", y, "") - } - if y := x.AllExist([]string{"a"}); y != "" { - t.Errorf("x.AllExists(['a']) got=%v want=%v", y, "") - } - if y := x.AllExist([]string{"b"}); y != "b" { - t.Errorf("x.AllExists(['a']) got=%v want=%v", y, "b") - } -} - -func TestDistributedBackupValidate(t *testing.T) { - timept := time.Now().UTC() - tests := []struct { - desc DistributedBackupDescriptor - success bool - }{ - // first level check - {desc: DistributedBackupDescriptor{}}, - {desc: DistributedBackupDescriptor{ID: "1"}}, - {desc: DistributedBackupDescriptor{ID: "1", Version: "1"}}, - {desc: DistributedBackupDescriptor{ID: "1", Version: "1", ServerVersion: "1"}}, - {desc: DistributedBackupDescriptor{ID: "1", Version: "1", ServerVersion: "1", StartedAt: timept, Error: "err"}}, - {desc: DistributedBackupDescriptor{ID: "1", Version: "1", ServerVersion: "1", StartedAt: timept}}, - {desc: DistributedBackupDescriptor{ - ID: "1", Version: "1", ServerVersion: "1", StartedAt: timept, - Nodes: map[string]*NodeDescriptor{"N": {}}, - }, success: true}, - } - for i, tc := range tests { - err := tc.desc.Validate() - if got := err == nil; got != tc.success { - t.Errorf("%d. validate(%+v): want=%v got=%v err=%v", i, tc.desc, tc.success, got, err) - } - } -} - -func TestTestDistributedBackupResetStatus(t *testing.T) { - begin := time.Now().UTC().Add(-2) - desc := DistributedBackupDescriptor{ - StartedAt: begin, - CompletedAt: begin.Add(2), - ID: "1", - Version: "1", - ServerVersion: "1", - Nodes: map[string]*NodeDescriptor{ - "1": {}, - "2": {Status: Success}, - "3": {Error: "error"}, - }, - Error: "error", - } - - desc.ResetStatus() - if !desc.StartedAt.After(begin) { - t.Fatalf("!desc.StartedAt.After(begin)") - } - want := DistributedBackupDescriptor{ - StartedAt: desc.StartedAt, - ID: "1", - Version: "1", - ServerVersion: "1", - Nodes: map[string]*NodeDescriptor{ - "1": {Status: Started}, - "2": {Status: Started}, - "3": {Status: Started, Error: ""}, - }, - Status: Started, - } - assert.Equal(t, want, desc) -} - -func TestShardDescriptorClear(t *testing.T) { - s := ShardDescriptor{ - Name: "name", - Node: "node", - PropLengthTrackerPath: "a/b", - PropLengthTracker: []byte{1}, - DocIDCounterPath: "a/c", - DocIDCounter: []byte{2}, - ShardVersionPath: "a/d", - Version: []byte{3}, - Files: []string{"file"}, - Chunk: 1, - } - - want := ShardDescriptor{ - Name: "name", - Node: "node", - Files: []string{"file"}, - Chunk: 1, - } - s.ClearTemporary() - assert.Equal(t, want, s) -} diff --git a/entities/backup/errors.go b/entities/backup/errors.go deleted file mode 100644 index a6e61aa8c3059ede03bfcf94a8db7bc68c1d59b6..0000000000000000000000000000000000000000 --- a/entities/backup/errors.go +++ /dev/null @@ -1,63 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package backup - -type ErrUnprocessable struct { - err error -} - -func (e ErrUnprocessable) Error() string { - return e.err.Error() -} - -func NewErrUnprocessable(err error) ErrUnprocessable { - return ErrUnprocessable{err} -} - -type ErrNotFound struct { - err error -} - -func (e ErrNotFound) Error() string { - if e.err != nil { - return e.err.Error() - } - return "" -} - -func NewErrNotFound(err error) ErrNotFound { - return ErrNotFound{err} -} - -type ErrContextExpired struct { - err error -} - -func (e ErrContextExpired) Error() string { - return e.err.Error() -} - -func NewErrContextExpired(err error) ErrContextExpired { - return ErrContextExpired{err} -} - -type ErrInternal struct { - err error -} - -func (e ErrInternal) Error() string { - return e.err.Error() -} - -func NewErrInternal(err error) ErrInternal { - return ErrInternal{err} -} diff --git a/entities/backup/status.go b/entities/backup/status.go deleted file mode 100644 index bac308504ae605c427a05d25bb5a5cbedf22d130..0000000000000000000000000000000000000000 --- a/entities/backup/status.go +++ /dev/null @@ -1,32 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package backup - -type Status string - -const ( - Started Status = "STARTED" - Transferring Status = "TRANSFERRING" - Transferred Status = "TRANSFERRED" - Success Status = "SUCCESS" - Failed Status = "FAILED" -) - -type CreateMeta struct { - Path string - Status Status -} - -type RestoreMeta struct { - Path string - Status Status -} diff --git a/entities/cyclemanager/cyclecallbackctrl.go b/entities/cyclemanager/cyclecallbackctrl.go deleted file mode 100644 index f10bfd06ca473e2ee3aeda9ba23c41c9e7c18642..0000000000000000000000000000000000000000 --- a/entities/cyclemanager/cyclecallbackctrl.go +++ /dev/null @@ -1,210 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package cyclemanager - -import ( - "context" - "sync" - - "github.com/weaviate/weaviate/entities/errorcompounder" - "golang.org/x/sync/errgroup" -) - -// Used to control of registered in CycleCallbacks container callback -// Allows deactivating and activating registered callback or unregistering it -type CycleCallbackCtrl interface { - IsActive() bool - Activate() error - Deactivate(ctx context.Context) error - Unregister(ctx context.Context) error -} - -type cycleCallbackCtrl struct { - callbackId uint32 - callbackCustomId string - - isActive func(callbackId uint32, callbackCustomId string) bool - activate func(callbackId uint32, callbackCustomId string) error - deactivate func(ctx context.Context, callbackId uint32, callbackCustomId string) error - unregister func(ctx context.Context, callbackId uint32, callbackCustomId string) error -} - -func (c *cycleCallbackCtrl) IsActive() bool { - return c.isActive(c.callbackId, c.callbackCustomId) -} - -func (c *cycleCallbackCtrl) Activate() error { - return c.activate(c.callbackId, c.callbackCustomId) -} - -func (c *cycleCallbackCtrl) Deactivate(ctx context.Context) error { - return c.deactivate(ctx, c.callbackId, c.callbackCustomId) -} - -func (c *cycleCallbackCtrl) Unregister(ctx context.Context) error { - return c.unregister(ctx, c.callbackId, c.callbackCustomId) -} - -type cycleCombinedCallbackCtrl struct { - routinesLimit int - ctrls []CycleCallbackCtrl -} - -// Creates combined controller to manage all provided controllers at once as it was single instance. -// Methods (activate, deactivate, unregister) calls nested controllers' methods in parallel by number of -// goroutines given as argument. If < 1 value given, NumCPU is used. -func NewCombinedCallbackCtrl(routinesLimit int, ctrls ...CycleCallbackCtrl) CycleCallbackCtrl { - if routinesLimit <= 0 { - routinesLimit = _NUMCPU - } - - return &cycleCombinedCallbackCtrl{routinesLimit: routinesLimit, ctrls: ctrls} -} - -func (c *cycleCombinedCallbackCtrl) IsActive() bool { - for _, ctrl := range c.ctrls { - if !ctrl.IsActive() { - return false - } - } - return true -} - -func (c *cycleCombinedCallbackCtrl) Activate() error { - return c.combineErrors(c.activate()...) -} - -func (c *cycleCombinedCallbackCtrl) activate() []error { - eg := &errgroup.Group{} - eg.SetLimit(c.routinesLimit) - lock := new(sync.Mutex) - - errs := make([]error, 0, len(c.ctrls)) - for _, ctrl := range c.ctrls { - ctrl := ctrl - eg.Go(func() error { - if err := ctrl.Activate(); err != nil { - c.locked(lock, func() { errs = append(errs, err) }) - return err - } - return nil - }) - } - - eg.Wait() - return errs -} - -func (c *cycleCombinedCallbackCtrl) Deactivate(ctx context.Context) error { - errs, deactivated := c.deactivate(ctx) - if len(errs) == 0 { - return nil - } - - // try activating back deactivated - eg := &errgroup.Group{} - eg.SetLimit(c.routinesLimit) - for _, id := range deactivated { - id := id - eg.Go(func() error { - return c.ctrls[id].Activate() - }) - } - - eg.Wait() - return c.combineErrors(errs...) -} - -func (c *cycleCombinedCallbackCtrl) deactivate(ctx context.Context) ([]error, []int) { - eg := &errgroup.Group{} - eg.SetLimit(c.routinesLimit) - lock := new(sync.Mutex) - - errs := make([]error, 0, len(c.ctrls)) - deactivated := make([]int, 0, len(c.ctrls)) - for id, ctrl := range c.ctrls { - id, ctrl := id, ctrl - eg.Go(func() error { - if err := ctrl.Deactivate(ctx); err != nil { - c.locked(lock, func() { errs = append(errs, err) }) - return err - } - c.locked(lock, func() { deactivated = append(deactivated, id) }) - return nil - }) - } - - eg.Wait() - return errs, deactivated -} - -func (c *cycleCombinedCallbackCtrl) Unregister(ctx context.Context) error { - return c.combineErrors(c.unregister(ctx)...) -} - -func (c *cycleCombinedCallbackCtrl) unregister(ctx context.Context) []error { - eg := &errgroup.Group{} - eg.SetLimit(c.routinesLimit) - lock := new(sync.Mutex) - - errs := make([]error, 0, len(c.ctrls)) - for _, ctrl := range c.ctrls { - ctrl := ctrl - eg.Go(func() error { - if err := ctrl.Unregister(ctx); err != nil { - c.locked(lock, func() { errs = append(errs, err) }) - return err - } - return nil - }) - } - - eg.Wait() - return errs -} - -func (c *cycleCombinedCallbackCtrl) locked(lock *sync.Mutex, mutate func()) { - lock.Lock() - defer lock.Unlock() - - mutate() -} - -func (c *cycleCombinedCallbackCtrl) combineErrors(errors ...error) error { - ec := &errorcompounder.ErrorCompounder{} - for _, err := range errors { - ec.Add(err) - } - return ec.ToError() -} - -type cycleCallbackCtrlNoop struct{} - -func NewCallbackCtrlNoop() CycleCallbackCtrl { - return &cycleCallbackCtrlNoop{} -} - -func (c *cycleCallbackCtrlNoop) IsActive() bool { - return false -} - -func (c *cycleCallbackCtrlNoop) Activate() error { - return nil -} - -func (c *cycleCallbackCtrlNoop) Deactivate(ctx context.Context) error { - return ctx.Err() -} - -func (c *cycleCallbackCtrlNoop) Unregister(ctx context.Context) error { - return ctx.Err() -} diff --git a/entities/cyclemanager/cyclecallbackctrl_test.go b/entities/cyclemanager/cyclecallbackctrl_test.go deleted file mode 100644 index 559e2c0bb4c2fce23031ac575e9d976dfa92b4ba..0000000000000000000000000000000000000000 --- a/entities/cyclemanager/cyclecallbackctrl_test.go +++ /dev/null @@ -1,254 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package cyclemanager - -import ( - "context" - "testing" - "time" - - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestCycleCombineCallbackCtrl_Unregister(t *testing.T) { - logger, _ := test.NewNullLogger() - ctx := context.Background() - - t.Run("unregisters both", func(t *testing.T) { - callback1 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(100 * time.Millisecond) - return true - } - callback2 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(100 * time.Millisecond) - return true - } - - callbacks := NewCallbackGroup("id", logger, 2) - ctrl1 := callbacks.Register("c1", callback1) - ctrl2 := callbacks.Register("c2", callback2) - combinedCtrl := NewCombinedCallbackCtrl(2, ctrl1, ctrl2) - - cycle := NewManager(NewFixedTicker(100*time.Millisecond), callbacks.CycleCallback) - cycle.Start() - defer cycle.StopAndWait(ctx) - - err := combinedCtrl.Unregister(ctx) - require.Nil(t, err) - - assert.False(t, combinedCtrl.IsActive()) - assert.False(t, ctrl1.IsActive()) - assert.False(t, ctrl2.IsActive()) - }) - - t.Run("does not unregister on expired context", func(t *testing.T) { - expiredCtx, cancel := context.WithDeadline(ctx, time.Now()) - defer cancel() - - callback1 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(100 * time.Millisecond) - return true - } - callback2 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(100 * time.Millisecond) - return true - } - - callbacks := NewCallbackGroup("id", logger, 2) - ctrl1 := callbacks.Register("c1", callback1) - ctrl2 := callbacks.Register("c2", callback2) - combinedCtrl := NewCombinedCallbackCtrl(2, ctrl1, ctrl2) - - cycle := NewManager(NewFixedTicker(100*time.Millisecond), callbacks.CycleCallback) - cycle.Start() - defer cycle.StopAndWait(ctx) - - err := combinedCtrl.Unregister(expiredCtx) - require.NotNil(t, err) - assert.Contains(t, err.Error(), "unregistering callback 'c1' of 'id' failed: context deadline exceeded") - assert.Contains(t, err.Error(), "unregistering callback 'c2' of 'id' failed: context deadline exceeded") - - assert.True(t, combinedCtrl.IsActive()) - assert.True(t, ctrl1.IsActive()) - assert.True(t, ctrl2.IsActive()) - }) - - t.Run("fails unregistering one", func(t *testing.T) { - callbackShort := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(100 * time.Millisecond) - return true - } - callbackLong := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(500 * time.Millisecond) - return true - } - - callbacks := NewCallbackGroup("id", logger, 2) - ctrlShort := callbacks.Register("short", callbackShort) - ctrlLong := callbacks.Register("long", callbackLong) - combinedCtrl := NewCombinedCallbackCtrl(2, ctrlShort, ctrlLong) - - cycle := NewManager(NewFixedTicker(100*time.Millisecond), callbacks.CycleCallback) - cycle.Start() - defer cycle.StopAndWait(ctx) - - // wait long enough to call Unregister while 2nd callback is still processed. - // set timeout short enough to expire before 2nd callback finishes - time.Sleep(300 * time.Millisecond) - expirableCtx, cancel := context.WithTimeout(ctx, 100*time.Millisecond) - defer cancel() - - err := combinedCtrl.Unregister(expirableCtx) - require.NotNil(t, err) - assert.EqualError(t, err, "unregistering callback 'long' of 'id' failed: context deadline exceeded") - - assert.False(t, combinedCtrl.IsActive()) - assert.False(t, ctrlShort.IsActive()) - assert.True(t, ctrlLong.IsActive()) - }) -} - -func TestCycleCombineCallbackCtrl_Deactivate(t *testing.T) { - logger, _ := test.NewNullLogger() - ctx := context.Background() - - t.Run("deactivates both", func(t *testing.T) { - callback1 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(100 * time.Millisecond) - return true - } - callback2 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(100 * time.Millisecond) - return true - } - - callbacks := NewCallbackGroup("id", logger, 2) - ctrl1 := callbacks.Register("c1", callback1) - ctrl2 := callbacks.Register("c2", callback2) - combinedCtrl := NewCombinedCallbackCtrl(2, ctrl1, ctrl2) - - cycle := NewManager(NewFixedTicker(100*time.Millisecond), callbacks.CycleCallback) - cycle.Start() - defer cycle.StopAndWait(ctx) - - err := combinedCtrl.Deactivate(ctx) - require.Nil(t, err) - - assert.False(t, combinedCtrl.IsActive()) - assert.False(t, ctrl1.IsActive()) - assert.False(t, ctrl2.IsActive()) - }) - - t.Run("does not deactivate on expired context", func(t *testing.T) { - expiredCtx, cancel := context.WithDeadline(ctx, time.Now()) - defer cancel() - - callback1 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(100 * time.Millisecond) - return true - } - callback2 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(100 * time.Millisecond) - return true - } - - callbacks := NewCallbackGroup("id", logger, 2) - ctrl1 := callbacks.Register("c1", callback1) - ctrl2 := callbacks.Register("c2", callback2) - combinedCtrl := NewCombinedCallbackCtrl(2, ctrl1, ctrl2) - - cycle := NewManager(NewFixedTicker(100*time.Millisecond), callbacks.CycleCallback) - cycle.Start() - defer cycle.StopAndWait(ctx) - - err := combinedCtrl.Deactivate(expiredCtx) - require.NotNil(t, err) - assert.Contains(t, err.Error(), "deactivating callback 'c1' of 'id' failed: context deadline exceeded") - assert.Contains(t, err.Error(), "deactivating callback 'c1' of 'id' failed: context deadline exceeded") - - assert.True(t, combinedCtrl.IsActive()) - assert.True(t, ctrl1.IsActive()) - assert.True(t, ctrl2.IsActive()) - }) - - t.Run("fails deactivating one, activates other again", func(t *testing.T) { - callbackShort := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(100 * time.Millisecond) - return true - } - callbackLong := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(500 * time.Millisecond) - return true - } - - callbacks := NewCallbackGroup("id", logger, 2) - ctrlShort := callbacks.Register("short", callbackShort) - ctrlLong := callbacks.Register("long", callbackLong) - combinedCtrl := NewCombinedCallbackCtrl(2, ctrlShort, ctrlLong) - - cycle := NewManager(NewFixedTicker(100*time.Millisecond), callbacks.CycleCallback) - cycle.Start() - defer cycle.StopAndWait(ctx) - - // wait long enough to call Deactivate while 2nd callback is still processed. - // set timeout short enough to expire before 2nd callback finishes - time.Sleep(300 * time.Millisecond) - expirableCtx, cancel := context.WithTimeout(ctx, 100*time.Millisecond) - defer cancel() - - err := combinedCtrl.Deactivate(expirableCtx) - require.NotNil(t, err) - assert.EqualError(t, err, "deactivating callback 'long' of 'id' failed: context deadline exceeded") - - assert.True(t, combinedCtrl.IsActive()) - assert.True(t, ctrlShort.IsActive()) - assert.True(t, ctrlLong.IsActive()) - }) -} - -func TestCycleCombineCallbackCtrl_Activate(t *testing.T) { - logger, _ := test.NewNullLogger() - ctx := context.Background() - - t.Run("activates both", func(t *testing.T) { - callback1 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(100 * time.Millisecond) - return true - } - callback2 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(100 * time.Millisecond) - return true - } - - callbacks := NewCallbackGroup("id", logger, 2) - ctrl1 := callbacks.Register("c1", callback1, AsInactive()) - ctrl2 := callbacks.Register("c2", callback2, AsInactive()) - combinedCtrl := NewCombinedCallbackCtrl(2, ctrl1, ctrl2) - - cycle := NewManager(NewFixedTicker(100*time.Millisecond), callbacks.CycleCallback) - cycle.Start() - defer cycle.StopAndWait(ctx) - - assert.False(t, combinedCtrl.IsActive()) - assert.False(t, ctrl1.IsActive()) - assert.False(t, ctrl2.IsActive()) - - err := combinedCtrl.Activate() - require.Nil(t, err) - - assert.True(t, combinedCtrl.IsActive()) - assert.True(t, ctrl1.IsActive()) - assert.True(t, ctrl2.IsActive()) - }) -} diff --git a/entities/cyclemanager/cyclecallbackgroup.go b/entities/cyclemanager/cyclecallbackgroup.go deleted file mode 100644 index 36bf746c943cc77f7cc53cbe3b2df200f62d3f72..0000000000000000000000000000000000000000 --- a/entities/cyclemanager/cyclecallbackgroup.go +++ /dev/null @@ -1,411 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package cyclemanager - -import ( - "context" - "sync" - "time" - - "github.com/sirupsen/logrus" -) - -// Container for multiple callbacks exposing CycleCallback method acting as single callback. -// Can be provided to CycleManager. -type CycleCallbackGroup interface { - // Adds CycleCallback method to container - Register(id string, cycleCallback CycleCallback, options ...RegisterOption) CycleCallbackCtrl - // Method of CycleCallback acting as single callback for all callbacks added to the container - CycleCallback(shouldAbort ShouldAbortCallback) bool -} - -type cycleCallbackGroup struct { - sync.Mutex - - logger logrus.FieldLogger - customId string - routinesLimit int - nextId uint32 - callbackIds []uint32 - callbacks map[uint32]*cycleCallbackMeta -} - -func NewCallbackGroup(id string, logger logrus.FieldLogger, routinesLimit int) CycleCallbackGroup { - return &cycleCallbackGroup{ - logger: logger, - customId: id, - routinesLimit: routinesLimit, - nextId: 0, - callbackIds: []uint32{}, - callbacks: map[uint32]*cycleCallbackMeta{}, - } -} - -func (c *cycleCallbackGroup) Register(id string, cycleCallback CycleCallback, options ...RegisterOption) CycleCallbackCtrl { - c.Lock() - defer c.Unlock() - - meta := &cycleCallbackMeta{ - customId: id, - cycleCallback: cycleCallback, - active: true, - runningCtx: nil, - started: time.Now(), - intervals: nil, - } - for _, option := range options { - if option != nil { - option(meta) - } - } - - callbackId := c.nextId - c.callbackIds = append(c.callbackIds, callbackId) - c.callbacks[callbackId] = meta - c.nextId++ - - return &cycleCallbackCtrl{ - callbackId: callbackId, - callbackCustomId: id, - - isActive: c.isActive, - activate: c.activate, - deactivate: c.deactivate, - unregister: c.unregister, - } -} - -func (c *cycleCallbackGroup) CycleCallback(shouldAbort ShouldAbortCallback) bool { - if c.routinesLimit <= 1 { - return c.cycleCallbackSequential(shouldAbort) - } - return c.cycleCallbackParallel(shouldAbort, c.routinesLimit) -} - -func (c *cycleCallbackGroup) cycleCallbackSequential(shouldAbort ShouldAbortCallback) bool { - anyExecuted := false - i := 0 - for { - if shouldAbort() { - break - } - - c.Lock() - // no more callbacks left, exit the loop - if i >= len(c.callbackIds) { - c.Unlock() - break - } - - callbackId := c.callbackIds[i] - meta, ok := c.callbacks[callbackId] - // callback deleted in the meantime, remove its id - // and proceed to the next one (no "i" increment required) - if !ok { - c.callbackIds = append(c.callbackIds[:i], c.callbackIds[i+1:]...) - c.Unlock() - continue - } - i++ - // callback deactivated, proceed to the next one - if !meta.active { - c.Unlock() - continue - } - now := time.Now() - // not enough time passed since previous execution - if meta.intervals != nil && now.Sub(meta.started) < meta.intervals.Get() { - c.Unlock() - continue - } - // callback active, mark as running - runningCtx, cancel := context.WithCancel(context.Background()) - meta.runningCtx = runningCtx - meta.started = now - c.Unlock() - - func() { - // cancel called in recover, regardless of panic occurred or not - defer c.recover(meta.customId, cancel) - executed := meta.cycleCallback(func() bool { - if shouldAbort() { - return true - } - - // abort if callback deactivated or being unregistered - c.Lock() - defer c.Unlock() - - meta, ok := c.callbacks[callbackId] - if !ok || !meta.active || meta.unregisterRequested { - return true - } - - return false - }) - anyExecuted = executed || anyExecuted - - if meta.intervals != nil { - if executed { - meta.intervals.Reset() - } else { - meta.intervals.Advance() - } - } - }() - } - - return anyExecuted -} - -func (c *cycleCallbackGroup) cycleCallbackParallel(shouldAbort ShouldAbortCallback, routinesLimit int) bool { - anyExecuted := false - ch := make(chan uint32) - lock := new(sync.Mutex) - wg := new(sync.WaitGroup) - wg.Add(routinesLimit) - - i := 0 - for r := 0; r < routinesLimit; r++ { - go func() { - for callbackId := range ch { - if shouldAbort() { - // keep reading from channel until it is closed - continue - } - - c.Lock() - meta, ok := c.callbacks[callbackId] - // callback missing or deactivated, proceed to the next one - if !ok || !meta.active { - c.Unlock() - continue - } - now := time.Now() - // not enough time passed since previous execution - if meta.intervals != nil && now.Sub(meta.started) < meta.intervals.Get() { - c.Unlock() - continue - } - // callback active, mark as running - runningCtx, cancel := context.WithCancel(context.Background()) - meta.runningCtx = runningCtx - meta.started = now - c.Unlock() - - func() { - // cancel called in recover, regardless of panic occurred or not - defer c.recover(meta.customId, cancel) - executed := meta.cycleCallback(shouldAbort) - if executed { - lock.Lock() - anyExecuted = true - lock.Unlock() - } - if meta.intervals != nil { - if executed { - meta.intervals.Reset() - } else { - meta.intervals.Advance() - } - } - }() - } - wg.Done() - }() - } - - for { - if shouldAbort() { - close(ch) - break - } - - c.Lock() - // no more callbacks left, exit the loop - if i >= len(c.callbackIds) { - c.Unlock() - close(ch) - break - } - - callbackId := c.callbackIds[i] - _, ok := c.callbacks[callbackId] - // callback deleted in the meantime, remove its id - // and proceed to the next one (no "i" increment required) - if !ok { - c.callbackIds = append(c.callbackIds[:i], c.callbackIds[i+1:]...) - c.Unlock() - continue - } - c.Unlock() - ch <- callbackId - i++ - } - - wg.Wait() - return anyExecuted -} - -func (c *cycleCallbackGroup) recover(callbackCustomId string, cancel context.CancelFunc) { - if r := recover(); r != nil { - c.logger.WithFields(logrus.Fields{ - "action": "cyclemanager", - "callback_id": callbackCustomId, - "callbacks_id": c.customId, - }).Errorf("callback panic: %v", r) - } - cancel() -} - -func (c *cycleCallbackGroup) mutateCallback(ctx context.Context, callbackId uint32, - onNotFound func(callbackId uint32) error, - onFound func(callbackId uint32, meta *cycleCallbackMeta) error, -) error { - if ctx.Err() != nil { - return ctx.Err() - } - - for { - // mutate callback in collection only if not running (not yet started of finished) - c.Lock() - meta, ok := c.callbacks[callbackId] - if !ok { - err := onNotFound(callbackId) - c.Unlock() - return err - } - meta.unregisterRequested = true - runningCtx := meta.runningCtx - if runningCtx == nil || runningCtx.Err() != nil { - err := onFound(callbackId, meta) - c.Unlock() - return err - } - c.Unlock() - - // wait for callback to finish - select { - case <-runningCtx.Done(): - // get back to the beginning of the loop to make sure state.runningCtx - // was not changed. If not, loop will finish on runningCtx.Err() != nil check - continue - case <-ctx.Done(): - // in case both contexts are ready, but input ctx was selected - // check again running ctx as priority one - if runningCtx.Err() != nil { - // get back to the beginning of the loop to make sure state.runningCtx - // was not changed. If not, loop will finish on runningCtx.Err() != nil check - continue - } - // input ctx expired - return ctx.Err() - } - } -} - -func (c *cycleCallbackGroup) unregister(ctx context.Context, callbackId uint32, callbackCustomId string) error { - err := c.mutateCallback(ctx, callbackId, - func(callbackId uint32) error { - return nil - }, - func(callbackId uint32, meta *cycleCallbackMeta) error { - delete(c.callbacks, callbackId) - return nil - }, - ) - return errorUnregisterCallback(callbackCustomId, c.customId, err) -} - -func (c *cycleCallbackGroup) deactivate(ctx context.Context, callbackId uint32, callbackCustomId string) error { - err := c.mutateCallback(ctx, callbackId, - func(callbackId uint32) error { - return ErrorCallbackNotFound - }, - func(callbackId uint32, meta *cycleCallbackMeta) error { - meta.active = false - return nil - }, - ) - return errorDeactivateCallback(callbackCustomId, c.customId, err) -} - -func (c *cycleCallbackGroup) activate(callbackId uint32, callbackCustomId string) error { - c.Lock() - defer c.Unlock() - - meta, ok := c.callbacks[callbackId] - if !ok { - return errorActivateCallback(callbackCustomId, c.customId, ErrorCallbackNotFound) - } - - meta.active = true - return nil -} - -func (c *cycleCallbackGroup) isActive(callbackId uint32, callbackCustomId string) bool { - c.Lock() - defer c.Unlock() - - if meta, ok := c.callbacks[callbackId]; ok { - return meta.active - } - return false -} - -type cycleCallbackMeta struct { - customId string - cycleCallback CycleCallback - active bool - // indicates whether callback is already running - context active - // or not running (already finished) - context expired - // or not running (not yet started) - context nil - runningCtx context.Context - started time.Time - intervals CycleIntervals - // set to true if callback is being unregistered, but still running - unregisterRequested bool -} - -type cycleCallbackGroupNoop struct{} - -func NewCallbackGroupNoop() CycleCallbackGroup { - return &cycleCallbackGroupNoop{} -} - -func (c *cycleCallbackGroupNoop) Register(id string, cycleCallback CycleCallback, options ...RegisterOption) CycleCallbackCtrl { - return NewCallbackCtrlNoop() -} - -func (c *cycleCallbackGroupNoop) CycleCallback(shouldAbort ShouldAbortCallback) bool { - return false -} - -type RegisterOption func(meta *cycleCallbackMeta) - -func AsInactive() RegisterOption { - return func(meta *cycleCallbackMeta) { - meta.active = false - } -} - -func WithIntervals(intervals CycleIntervals) RegisterOption { - if intervals == nil { - return nil - } - return func(meta *cycleCallbackMeta) { - meta.intervals = intervals - // adjusts start time to allow for immediate callback execution without - // having to wait for interval duration to pass - meta.started = time.Now().Add(-intervals.Get()) - } -} diff --git a/entities/cyclemanager/cyclecallbackgroup_test.go b/entities/cyclemanager/cyclecallbackgroup_test.go deleted file mode 100644 index c38c1c278a0ffa92391250da1fb151fa2e558bb3..0000000000000000000000000000000000000000 --- a/entities/cyclemanager/cyclecallbackgroup_test.go +++ /dev/null @@ -1,1771 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package cyclemanager - -import ( - "context" - "sync/atomic" - "testing" - "time" - - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestCycleCallback_Parallel(t *testing.T) { - logger, _ := test.NewNullLogger() - shouldNotAbort := func() bool { return false } - - t.Run("no callbacks", func(t *testing.T) { - var executed bool - - callbacks := NewCallbackGroup("id", logger, 2) - - executed = callbacks.CycleCallback(shouldNotAbort) - - assert.False(t, executed) - }) - - t.Run("2 executable callbacks", func(t *testing.T) { - executedCounter1 := 0 - callback1 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(50 * time.Millisecond) - executedCounter1++ - return true - } - executedCounter2 := 0 - callback2 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(25 * time.Millisecond) - executedCounter2++ - return true - } - var executed bool - var d time.Duration - - callbacks := NewCallbackGroup("id", logger, 2) - callbacks.Register("c1", callback1) - callbacks.Register("c2", callback2) - - start := time.Now() - executed = callbacks.CycleCallback(shouldNotAbort) - d = time.Since(start) - - assert.True(t, executed) - assert.Equal(t, 1, executedCounter1) - assert.Equal(t, 1, executedCounter2) - assert.GreaterOrEqual(t, d, 50*time.Millisecond) - }) - - t.Run("2 non-executable callbacks", func(t *testing.T) { - executedCounter1 := 0 - callback1 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(10 * time.Millisecond) - executedCounter1++ - return false - } - executedCounter2 := 0 - callback2 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(10 * time.Millisecond) - executedCounter2++ - return false - } - var executed bool - var d time.Duration - - callbacks := NewCallbackGroup("id", logger, 2) - callbacks.Register("c1", callback1) - callbacks.Register("c2", callback2) - - start := time.Now() - executed = callbacks.CycleCallback(shouldNotAbort) - d = time.Since(start) - - assert.False(t, executed) - assert.Equal(t, 1, executedCounter1) - assert.Equal(t, 1, executedCounter2) - assert.GreaterOrEqual(t, d, 10*time.Millisecond) - }) - - t.Run("3 executable callbacks, not all executed due to should abort", func(t *testing.T) { - executedCounter1 := 0 - callback1 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(25 * time.Millisecond) - executedCounter1++ - return true - } - executedCounter2 := 0 - callback2 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(25 * time.Millisecond) - executedCounter2++ - return true - } - executedCounter3 := 0 - callback3 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(25 * time.Millisecond) - executedCounter3++ - return true - } - // due to async calls of shouldAbort callback by main for loop - // and goroutines reading from shared channel it is hard to - // establish order of calls. - // with 3 callbacks and shouldAbort returning true on 6th call - // 1 or 2 callbacks should be executed, but not all 3. - shouldAbortCounter := uint32(0) - shouldAbort := func() bool { - return atomic.AddUint32(&shouldAbortCounter, 1) > 5 - } - var executed bool - var d time.Duration - - callbacks := NewCallbackGroup("id", logger, 2) - callbacks.Register("c1", callback1) - callbacks.Register("c2", callback2) - callbacks.Register("c3", callback3) - - start := time.Now() - executed = callbacks.CycleCallback(shouldAbort) - d = time.Since(start) - - assert.True(t, executed) - totalExecuted := executedCounter1 + executedCounter2 + executedCounter3 - assert.Greater(t, totalExecuted, 0) - assert.Less(t, totalExecuted, 3) - assert.GreaterOrEqual(t, d, 25*time.Millisecond) - }) - - t.Run("register new while executing", func(t *testing.T) { - executedCounter1 := 0 - callback1 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(50 * time.Millisecond) - executedCounter1++ - return true - } - executedCounter2 := 0 - callback2 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(50 * time.Millisecond) - executedCounter2++ - return true - } - executedCounter3 := 0 - callback3 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(50 * time.Millisecond) - executedCounter3++ - return true - } - executedCounter4 := 0 - callback4 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(50 * time.Millisecond) - executedCounter4++ - return true - } - chStarted := make(chan struct{}, 1) - chFinished := make(chan struct{}, 1) - var executed bool - var d time.Duration - - callbacks := NewCallbackGroup("id", logger, 2) - callbacks.Register("c1", callback1) - callbacks.Register("c2", callback2) - callbacks.Register("c3", callback3) - - // register 4th callback while other are executed, - // - // while 1st and 2nd are being processed (50ms), - // 3rd is waiting for available routine (without 3rd callback loop would be finished) - // 4th is registered (25ms) to be called next along with 3rd - go func() { - chStarted <- struct{}{} - start := time.Now() - executed = callbacks.CycleCallback(shouldNotAbort) - d = time.Since(start) - chFinished <- struct{}{} - }() - <-chStarted - time.Sleep(25 * time.Millisecond) - callbacks.Register("c4", callback4) - <-chFinished - - assert.True(t, executed) - assert.Equal(t, 1, executedCounter1) - assert.Equal(t, 1, executedCounter2) - assert.Equal(t, 1, executedCounter3) - assert.Equal(t, 1, executedCounter4) - assert.GreaterOrEqual(t, d, 100*time.Millisecond) - }) - - t.Run("run with intervals", func(T *testing.T) { - ticker := NewFixedTicker(10 * time.Millisecond) - intervals2 := NewSeriesIntervals([]time.Duration{ - 10 * time.Millisecond, 30 * time.Millisecond, 50 * time.Millisecond, - }) - intervals3 := NewFixedIntervals(60 * time.Millisecond) - now := time.Now() - - executionTimes1 := []time.Duration{} - callback1 := func(shouldAbort ShouldAbortCallback) bool { - executionTimes1 = append(executionTimes1, time.Since(now)) - return true - } - executionCounter2 := 0 - executionTimes2 := []time.Duration{} - callback2 := func(shouldAbort ShouldAbortCallback) bool { - executionCounter2++ - executionTimes2 = append(executionTimes2, time.Since(now)) - // reports executed every 3 calls, should result in 10, 30, 50, 50, 10, 30, 50, 50, ... intervals - return executionCounter2%4 == 0 - } - executionTimes3 := []time.Duration{} - callback3 := func(shouldAbort ShouldAbortCallback) bool { - executionTimes3 = append(executionTimes3, time.Since(now)) - return true - } - - callbacks := NewCallbackGroup("id", logger, 2) - // should be called on every tick, with 10 intervals - callbacks.Register("c1", callback1) - // should be called with 10, 30, 50, 50, 10, 30, 50, 50, ... intervals - callbacks.Register("c2", callback2, WithIntervals(intervals2)) - // should be called with 60, 60, ... intervals - callbacks.Register("c3", callback3, WithIntervals(intervals3)) - - cm := NewManager(ticker, callbacks.CycleCallback) - cm.Start() - time.Sleep(400 * time.Millisecond) - cm.StopAndWait(context.Background()) - - // within 400 ms c1 should be called at least 30x - require.GreaterOrEqual(t, len(executionTimes1), 30) - // 1st call on 1st tick after 10ms - sumDuration := time.Duration(10) - for i := 0; i < 30; i++ { - assert.GreaterOrEqual(t, executionTimes1[i], sumDuration) - sumDuration += 10 * time.Millisecond - } - - // within 400 ms c2 should be called at least 8x - require.GreaterOrEqual(t, len(executionTimes2), 8) - // 1st call on 1st tick after 10ms - sumDuration = time.Duration(0) - for i := 0; i < 8; i++ { - assert.GreaterOrEqual(t, executionTimes2[i], sumDuration) - switch (i + 1) % 4 { - case 0: - sumDuration += 10 * time.Millisecond - case 1: - sumDuration += 30 * time.Millisecond - case 2, 3: - sumDuration += 50 * time.Millisecond - } - } - - // within 400 ms c3 should be called at least 6x - require.GreaterOrEqual(t, len(executionTimes3), 6) - // 1st call on 1st tick after 10ms - sumDuration = time.Duration(0) - for i := 0; i < 6; i++ { - assert.GreaterOrEqual(t, executionTimes3[i], sumDuration) - sumDuration += 60 * time.Millisecond - } - }) - - t.Run("unregister while running", func(t *testing.T) { - counter := 0 - callback := func(shouldAbort ShouldAbortCallback) bool { - for { - if shouldAbort() { - return true - } - - time.Sleep(10 * time.Millisecond) - counter++ - - // 10ms * 100 = 1s - if counter > 100 { - return false - } - } - } - chStarted := make(chan struct{}, 1) - chFinished := make(chan struct{}, 1) - var executed bool - var d time.Duration - - callbacks := NewCallbackGroup("id", logger, 1) - ctrl := callbacks.Register("c", callback) - - go func() { - chStarted <- struct{}{} - start := time.Now() - executed = callbacks.CycleCallback(shouldNotAbort) - d = time.Since(start) - chFinished <- struct{}{} - }() - <-chStarted - time.Sleep(50 * time.Millisecond) - err := ctrl.Unregister(context.Background()) - assert.NoError(t, err) - <-chFinished - - assert.True(t, executed) - assert.GreaterOrEqual(t, d, 50*time.Millisecond) - }) -} - -func TestCycleCallback_Parallel_Unregister(t *testing.T) { - ctx := context.Background() - logger, _ := test.NewNullLogger() - shouldNotAbort := func() bool { return false } - - t.Run("1 executable callback, 1 unregistered", func(t *testing.T) { - executedCounter := 0 - callback := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(50 * time.Millisecond) - executedCounter++ - return true - } - var executed bool - var d time.Duration - - callbacks := NewCallbackGroup("id", logger, 2) - ctrl := callbacks.Register("c1", callback) - require.Nil(t, ctrl.Unregister(ctx)) - - start := time.Now() - executed = callbacks.CycleCallback(shouldNotAbort) - d = time.Since(start) - - assert.False(t, executed) - assert.Equal(t, 0, executedCounter) - assert.GreaterOrEqual(t, d, 0*time.Millisecond) - }) - - t.Run("2 executable callbacks, 2 unregistered", func(t *testing.T) { - executedCounter1 := 0 - callback1 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(50 * time.Millisecond) - executedCounter1++ - return true - } - executedCounter2 := 0 - callback2 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(25 * time.Millisecond) - executedCounter2++ - return true - } - var executed bool - var d time.Duration - - callbacks := NewCallbackGroup("id", logger, 2) - ctrl1 := callbacks.Register("c1", callback1) - ctrl2 := callbacks.Register("c2", callback2) - require.Nil(t, ctrl1.Unregister(ctx)) - require.Nil(t, ctrl2.Unregister(ctx)) - - start := time.Now() - executed = callbacks.CycleCallback(shouldNotAbort) - d = time.Since(start) - - assert.False(t, executed) - assert.Equal(t, 0, executedCounter1) - assert.Equal(t, 0, executedCounter2) - assert.GreaterOrEqual(t, d, 0*time.Millisecond) - }) - - t.Run("2 executable callbacks, 1 unregistered", func(t *testing.T) { - executedCounter1 := 0 - callback1 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(50 * time.Millisecond) - executedCounter1++ - return true - } - executedCounter2 := 0 - callback2 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(25 * time.Millisecond) - executedCounter2++ - return true - } - var executed bool - var d time.Duration - - callbacks := NewCallbackGroup("id", logger, 2) - ctrl1 := callbacks.Register("c1", callback1) - callbacks.Register("c2", callback2) - require.Nil(t, ctrl1.Unregister(ctx)) - - start := time.Now() - executed = callbacks.CycleCallback(shouldNotAbort) - d = time.Since(start) - - assert.True(t, executed) - assert.Equal(t, 0, executedCounter1) - assert.Equal(t, 1, executedCounter2) - assert.GreaterOrEqual(t, d, 25*time.Millisecond) - }) - - t.Run("4 executable callbacks, all unregistered at different time", func(t *testing.T) { - executedCounter1 := 0 - callback1 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(25 * time.Millisecond) - executedCounter1++ - return true - } - executedCounter2 := 0 - callback2 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(25 * time.Millisecond) - executedCounter2++ - return true - } - executedCounter3 := 0 - callback3 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(25 * time.Millisecond) - executedCounter3++ - return true - } - executedCounter4 := 0 - callback4 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(25 * time.Millisecond) - executedCounter4++ - return true - } - var executed1 bool - var executed2 bool - var executed3 bool - var executed4 bool - var d1 time.Duration - var d2 time.Duration - var d3 time.Duration - var d4 time.Duration - - callbacks := NewCallbackGroup("id", logger, 2) - ctrl1 := callbacks.Register("c1", callback1) - ctrl2 := callbacks.Register("c2", callback2) - ctrl3 := callbacks.Register("c3", callback3) - ctrl4 := callbacks.Register("c4", callback4) - require.Nil(t, ctrl3.Unregister(ctx)) - - start := time.Now() - executed1 = callbacks.CycleCallback(shouldNotAbort) - d1 = time.Since(start) - - require.Nil(t, ctrl1.Unregister(ctx)) - - start = time.Now() - executed2 = callbacks.CycleCallback(shouldNotAbort) - d2 = time.Since(start) - - require.Nil(t, ctrl4.Unregister(ctx)) - - start = time.Now() - executed3 = callbacks.CycleCallback(shouldNotAbort) - d3 = time.Since(start) - - require.Nil(t, ctrl2.Unregister(ctx)) - - start = time.Now() - executed4 = callbacks.CycleCallback(shouldNotAbort) - d4 = time.Since(start) - - assert.True(t, executed1) - assert.True(t, executed2) - assert.True(t, executed3) - assert.False(t, executed4) - assert.Equal(t, 1, executedCounter1) - assert.Equal(t, 3, executedCounter2) - assert.Equal(t, 0, executedCounter3) - assert.Equal(t, 2, executedCounter4) - assert.GreaterOrEqual(t, d1, 50*time.Millisecond) - assert.GreaterOrEqual(t, d2, 25*time.Millisecond) - assert.GreaterOrEqual(t, d3, 25*time.Millisecond) - assert.GreaterOrEqual(t, d4, 0*time.Millisecond) - }) - - t.Run("unregister is waiting till the end of execution", func(t *testing.T) { - executedCounter := 0 - callback := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(50 * time.Millisecond) - executedCounter++ - return true - } - chStarted := make(chan struct{}, 1) - chFinished := make(chan struct{}, 1) - var executed bool - var d time.Duration - - callbacks := NewCallbackGroup("id", logger, 2) - ctrl := callbacks.Register("c", callback) - - go func() { - chStarted <- struct{}{} - start := time.Now() - executed = callbacks.CycleCallback(shouldNotAbort) - d = time.Since(start) - chFinished <- struct{}{} - }() - <-chStarted - start := time.Now() - time.Sleep(25 * time.Millisecond) - require.Nil(t, ctrl.Unregister(ctx)) - du := time.Since(start) - <-chFinished - - assert.True(t, executed) - assert.Equal(t, 1, executedCounter) - assert.GreaterOrEqual(t, d, 50*time.Millisecond) - assert.GreaterOrEqual(t, du, 40*time.Millisecond) - }) - - t.Run("unregister fails due to context timeout", func(t *testing.T) { - executedCounter := 0 - callback := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(50 * time.Millisecond) - executedCounter++ - return true - } - chStarted := make(chan struct{}, 1) - chFinished := make(chan struct{}, 1) - var executed1 bool - var executed2 bool - var d1 time.Duration - var d2 time.Duration - - callbacks := NewCallbackGroup("id", logger, 2) - ctrl := callbacks.Register("c", callback) - - go func() { - chStarted <- struct{}{} - start := time.Now() - executed1 = callbacks.CycleCallback(shouldNotAbort) - d1 = time.Since(start) - chFinished <- struct{}{} - }() - <-chStarted - start := time.Now() - time.Sleep(25 * time.Millisecond) - ctxTimeout, cancel := context.WithTimeout(ctx, 5*time.Millisecond) - defer cancel() - require.NotNil(t, ctrl.Unregister(ctxTimeout)) - du := time.Since(start) - <-chFinished - - go func() { - start := time.Now() - executed2 = callbacks.CycleCallback(shouldNotAbort) - d2 = time.Since(start) - chFinished <- struct{}{} - }() - <-chFinished - - assert.True(t, executed1) - assert.True(t, executed2) - assert.Equal(t, 2, executedCounter) - assert.GreaterOrEqual(t, d1, 50*time.Millisecond) - assert.GreaterOrEqual(t, d2, 50*time.Millisecond) - assert.GreaterOrEqual(t, du, 30*time.Millisecond) - }) - - t.Run("unregister 3rd and 4th while executing", func(t *testing.T) { - executedCounter1 := 0 - callback1 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(50 * time.Millisecond) - executedCounter1++ - return true - } - executedCounter2 := 0 - callback2 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(50 * time.Millisecond) - executedCounter2++ - return true - } - executedCounter3 := 0 - callback3 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(50 * time.Millisecond) - executedCounter3++ - return true - } - executedCounter4 := 0 - callback4 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(50 * time.Millisecond) - executedCounter4++ - return true - } - chStarted := make(chan struct{}, 1) - chFinished := make(chan struct{}, 1) - var executed bool - var d time.Duration - - callbacks := NewCallbackGroup("id", logger, 2) - callbacks.Register("c1", callback1) - callbacks.Register("c2", callback2) - ctrl3 := callbacks.Register("c3", callback3) - ctrl4 := callbacks.Register("c4", callback4) - - go func() { - chStarted <- struct{}{} - start := time.Now() - executed = callbacks.CycleCallback(shouldNotAbort) - d = time.Since(start) - chFinished <- struct{}{} - }() - <-chStarted - time.Sleep(25 * time.Millisecond) - require.Nil(t, ctrl3.Unregister(ctx)) - require.Nil(t, ctrl4.Unregister(ctx)) - <-chFinished - - assert.True(t, executed) - assert.Equal(t, 1, executedCounter1) - assert.Equal(t, 1, executedCounter2) - assert.Equal(t, 0, executedCounter3) - assert.Equal(t, 0, executedCounter3) - assert.GreaterOrEqual(t, d, 50*time.Millisecond) - }) -} - -func TestCycleCallback_Parallel_Deactivate(t *testing.T) { - ctx := context.Background() - logger, _ := test.NewNullLogger() - shouldNotAbort := func() bool { return false } - - t.Run("1 executable callback, 1 deactivated", func(t *testing.T) { - executedCounter := 0 - callback := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(50 * time.Millisecond) - executedCounter++ - return true - } - var executed bool - var d time.Duration - - callbacks := NewCallbackGroup("id", logger, 2) - ctrl := callbacks.Register("c1", callback) - require.Nil(t, ctrl.Deactivate(ctx)) - - start := time.Now() - executed = callbacks.CycleCallback(shouldNotAbort) - d = time.Since(start) - - assert.False(t, executed) - assert.Equal(t, 0, executedCounter) - assert.GreaterOrEqual(t, d, 0*time.Millisecond) - }) - - t.Run("2 executable callbacks, 2 deactivated", func(t *testing.T) { - executedCounter1 := 0 - callback1 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(50 * time.Millisecond) - executedCounter1++ - return true - } - executedCounter2 := 0 - callback2 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(25 * time.Millisecond) - executedCounter2++ - return true - } - var executed bool - var d time.Duration - - callbacks := NewCallbackGroup("id", logger, 2) - ctrl1 := callbacks.Register("c1", callback1) - ctrl2 := callbacks.Register("c2", callback2) - require.Nil(t, ctrl1.Deactivate(ctx)) - require.Nil(t, ctrl2.Deactivate(ctx)) - - start := time.Now() - executed = callbacks.CycleCallback(shouldNotAbort) - d = time.Since(start) - - assert.False(t, executed) - assert.Equal(t, 0, executedCounter1) - assert.Equal(t, 0, executedCounter2) - assert.GreaterOrEqual(t, d, 0*time.Millisecond) - }) - - t.Run("2 executable callbacks, 1 deactivated", func(t *testing.T) { - executedCounter1 := 0 - callback1 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(50 * time.Millisecond) - executedCounter1++ - return true - } - executedCounter2 := 0 - callback2 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(25 * time.Millisecond) - executedCounter2++ - return true - } - var executed bool - var d time.Duration - - callbacks := NewCallbackGroup("id", logger, 2) - ctrl1 := callbacks.Register("c1", callback1) - callbacks.Register("c2", callback2) - require.Nil(t, ctrl1.Deactivate(ctx)) - - start := time.Now() - executed = callbacks.CycleCallback(shouldNotAbort) - d = time.Since(start) - - assert.True(t, executed) - assert.Equal(t, 0, executedCounter1) - assert.Equal(t, 1, executedCounter2) - assert.GreaterOrEqual(t, d, 25*time.Millisecond) - }) - - t.Run("4 executable callbacks, all deactivated at different time", func(t *testing.T) { - executedCounter1 := 0 - callback1 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(25 * time.Millisecond) - executedCounter1++ - return true - } - executedCounter2 := 0 - callback2 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(25 * time.Millisecond) - executedCounter2++ - return true - } - executedCounter3 := 0 - callback3 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(25 * time.Millisecond) - executedCounter3++ - return true - } - executedCounter4 := 0 - callback4 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(25 * time.Millisecond) - executedCounter4++ - return true - } - var executed1 bool - var executed2 bool - var executed3 bool - var executed4 bool - var d1 time.Duration - var d2 time.Duration - var d3 time.Duration - var d4 time.Duration - - callbacks := NewCallbackGroup("id", logger, 2) - ctrl1 := callbacks.Register("c1", callback1) - ctrl2 := callbacks.Register("c2", callback2) - ctrl3 := callbacks.Register("c3", callback3) - ctrl4 := callbacks.Register("c4", callback4) - require.Nil(t, ctrl3.Deactivate(ctx)) - - start := time.Now() - executed1 = callbacks.CycleCallback(shouldNotAbort) - d1 = time.Since(start) - - require.Nil(t, ctrl1.Deactivate(ctx)) - - start = time.Now() - executed2 = callbacks.CycleCallback(shouldNotAbort) - d2 = time.Since(start) - - require.Nil(t, ctrl4.Deactivate(ctx)) - - start = time.Now() - executed3 = callbacks.CycleCallback(shouldNotAbort) - d3 = time.Since(start) - - require.Nil(t, ctrl2.Deactivate(ctx)) - - start = time.Now() - executed4 = callbacks.CycleCallback(shouldNotAbort) - d4 = time.Since(start) - - assert.True(t, executed1) - assert.True(t, executed2) - assert.True(t, executed3) - assert.False(t, executed4) - assert.Equal(t, 1, executedCounter1) - assert.Equal(t, 3, executedCounter2) - assert.Equal(t, 0, executedCounter3) - assert.Equal(t, 2, executedCounter4) - assert.GreaterOrEqual(t, d1, 50*time.Millisecond) - assert.GreaterOrEqual(t, d2, 25*time.Millisecond) - assert.GreaterOrEqual(t, d3, 25*time.Millisecond) - assert.GreaterOrEqual(t, d4, 0*time.Millisecond) - }) - - t.Run("deactivate is waiting till the end of execution", func(t *testing.T) { - executedCounter := 0 - callback := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(50 * time.Millisecond) - executedCounter++ - return true - } - chStarted := make(chan struct{}, 1) - chFinished := make(chan struct{}, 1) - var executed bool - var d time.Duration - - callbacks := NewCallbackGroup("id", logger, 2) - ctrl := callbacks.Register("c", callback) - - go func() { - chStarted <- struct{}{} - start := time.Now() - executed = callbacks.CycleCallback(shouldNotAbort) - d = time.Since(start) - chFinished <- struct{}{} - }() - <-chStarted - start := time.Now() - time.Sleep(25 * time.Millisecond) - require.Nil(t, ctrl.Deactivate(ctx)) - du := time.Since(start) - <-chFinished - - assert.True(t, executed) - assert.Equal(t, 1, executedCounter) - assert.GreaterOrEqual(t, d, 50*time.Millisecond) - assert.GreaterOrEqual(t, du, 40*time.Millisecond) - }) - - t.Run("deactivate fails due to context timeout", func(t *testing.T) { - executedCounter := 0 - callback := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(50 * time.Millisecond) - executedCounter++ - return true - } - chStarted := make(chan struct{}, 1) - chFinished := make(chan struct{}, 1) - var executed1 bool - var executed2 bool - var d1 time.Duration - var d2 time.Duration - - callbacks := NewCallbackGroup("id", logger, 2) - ctrl := callbacks.Register("c", callback) - - go func() { - chStarted <- struct{}{} - start := time.Now() - executed1 = callbacks.CycleCallback(shouldNotAbort) - d1 = time.Since(start) - chFinished <- struct{}{} - }() - <-chStarted - start := time.Now() - time.Sleep(25 * time.Millisecond) - ctxTimeout, cancel := context.WithTimeout(ctx, 5*time.Millisecond) - defer cancel() - require.NotNil(t, ctrl.Deactivate(ctxTimeout)) - du := time.Since(start) - <-chFinished - - go func() { - start := time.Now() - executed2 = callbacks.CycleCallback(shouldNotAbort) - d2 = time.Since(start) - chFinished <- struct{}{} - }() - <-chFinished - - assert.True(t, executed1) - assert.True(t, executed2) - assert.Equal(t, 2, executedCounter) - assert.GreaterOrEqual(t, d1, 50*time.Millisecond) - assert.GreaterOrEqual(t, d2, 50*time.Millisecond) - assert.GreaterOrEqual(t, du, 30*time.Millisecond) - }) - - t.Run("deactivate 3rd and 4th while executing", func(t *testing.T) { - executedCounter1 := 0 - callback1 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(50 * time.Millisecond) - executedCounter1++ - return true - } - executedCounter2 := 0 - callback2 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(50 * time.Millisecond) - executedCounter2++ - return true - } - executedCounter3 := 0 - callback3 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(50 * time.Millisecond) - executedCounter3++ - return true - } - executedCounter4 := 0 - callback4 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(50 * time.Millisecond) - executedCounter4++ - return true - } - chStarted := make(chan struct{}, 1) - chFinished := make(chan struct{}, 1) - var executed bool - var d time.Duration - - callbacks := NewCallbackGroup("id", logger, 2) - callbacks.Register("c1", callback1) - callbacks.Register("c2", callback2) - ctrl3 := callbacks.Register("c3", callback3) - ctrl4 := callbacks.Register("c4", callback4) - - go func() { - chStarted <- struct{}{} - start := time.Now() - executed = callbacks.CycleCallback(shouldNotAbort) - d = time.Since(start) - chFinished <- struct{}{} - }() - <-chStarted - time.Sleep(25 * time.Millisecond) - require.Nil(t, ctrl3.Deactivate(ctx)) - require.Nil(t, ctrl4.Deactivate(ctx)) - <-chFinished - - assert.True(t, executed) - assert.Equal(t, 1, executedCounter1) - assert.Equal(t, 1, executedCounter2) - assert.Equal(t, 0, executedCounter3) - assert.Equal(t, 0, executedCounter3) - assert.GreaterOrEqual(t, d, 50*time.Millisecond) - }) -} - -func TestCycleCallback_Sequential(t *testing.T) { - logger, _ := test.NewNullLogger() - shouldNotAbort := func() bool { return false } - - t.Run("no callbacks", func(t *testing.T) { - var executed bool - - callbacks := NewCallbackGroup("id", logger, 1) - - executed = callbacks.CycleCallback(shouldNotAbort) - - assert.False(t, executed) - }) - - t.Run("2 executable callbacks", func(t *testing.T) { - executedCounter1 := 0 - callback1 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(50 * time.Millisecond) - executedCounter1++ - return true - } - executedCounter2 := 0 - callback2 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(25 * time.Millisecond) - executedCounter2++ - return true - } - var executed bool - var d time.Duration - - callbacks := NewCallbackGroup("id", logger, 1) - callbacks.Register("c1", callback1) - callbacks.Register("c2", callback2) - - start := time.Now() - executed = callbacks.CycleCallback(shouldNotAbort) - d = time.Since(start) - - assert.True(t, executed) - assert.Equal(t, 1, executedCounter1) - assert.Equal(t, 1, executedCounter2) - assert.GreaterOrEqual(t, d, 75*time.Millisecond) - }) - - t.Run("2 non-executable callbacks", func(t *testing.T) { - executedCounter1 := 0 - callback1 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(10 * time.Millisecond) - executedCounter1++ - return false - } - executedCounter2 := 0 - callback2 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(10 * time.Millisecond) - executedCounter2++ - return false - } - var executed bool - var d time.Duration - - callbacks := NewCallbackGroup("id", logger, 1) - callbacks.Register("c1", callback1) - callbacks.Register("c2", callback2) - - start := time.Now() - executed = callbacks.CycleCallback(shouldNotAbort) - d = time.Since(start) - - assert.False(t, executed) - assert.Equal(t, 1, executedCounter1) - assert.Equal(t, 1, executedCounter2) - assert.GreaterOrEqual(t, d, 10*time.Millisecond) - }) - - t.Run("2 executable callbacks, not executed due to should abort", func(t *testing.T) { - executedCounter1 := 0 - callback1 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(25 * time.Millisecond) - executedCounter1++ - return true - } - executedCounter2 := 0 - callback2 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(25 * time.Millisecond) - executedCounter2++ - return true - } - shouldAbortCounter := 0 - shouldAbort := func() bool { - shouldAbortCounter++ - return shouldAbortCounter > 1 - } - - var executed bool - var d time.Duration - - callbacks := NewCallbackGroup("id", logger, 1) - callbacks.Register("c1", callback1) - callbacks.Register("c2", callback2) - - start := time.Now() - executed = callbacks.CycleCallback(shouldAbort) - d = time.Since(start) - - assert.True(t, executed) - assert.Equal(t, 1, executedCounter1) - assert.Equal(t, 0, executedCounter2) - assert.GreaterOrEqual(t, d, 25*time.Millisecond) - }) - - t.Run("register new while executing", func(t *testing.T) { - executedCounter1 := 0 - callback1 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(50 * time.Millisecond) - executedCounter1++ - return true - } - executedCounter2 := 0 - callback2 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(50 * time.Millisecond) - executedCounter2++ - return true - } - chStarted := make(chan struct{}, 1) - chFinished := make(chan struct{}, 1) - var executed bool - var d time.Duration - - callbacks := NewCallbackGroup("id", logger, 1) - callbacks.Register("c1", callback1) - - go func() { - chStarted <- struct{}{} - start := time.Now() - executed = callbacks.CycleCallback(shouldNotAbort) - d = time.Since(start) - chFinished <- struct{}{} - }() - <-chStarted - time.Sleep(25 * time.Millisecond) - callbacks.Register("c2", callback2) - <-chFinished - - assert.True(t, executed) - assert.Equal(t, 1, executedCounter1) - assert.Equal(t, 1, executedCounter2) - assert.GreaterOrEqual(t, d, 100*time.Millisecond) - }) - - t.Run("run with intervals", func(T *testing.T) { - ticker := NewFixedTicker(10 * time.Millisecond) - intervals2 := NewSeriesIntervals([]time.Duration{ - 10 * time.Millisecond, 30 * time.Millisecond, 50 * time.Millisecond, - }) - intervals3 := NewFixedIntervals(60 * time.Millisecond) - now := time.Now() - - executionTimes1 := []time.Duration{} - callback1 := func(shouldAbort ShouldAbortCallback) bool { - executionTimes1 = append(executionTimes1, time.Since(now)) - return true - } - executionCounter2 := 0 - executionTimes2 := []time.Duration{} - callback2 := func(shouldAbort ShouldAbortCallback) bool { - executionCounter2++ - executionTimes2 = append(executionTimes2, time.Since(now)) - // reports executed every 3 calls, should result in 10, 30, 50, 50, 10, 30, 50, 50, ... intervals - return executionCounter2%4 == 0 - } - executionTimes3 := []time.Duration{} - callback3 := func(shouldAbort ShouldAbortCallback) bool { - executionTimes3 = append(executionTimes3, time.Since(now)) - return true - } - - callbacks := NewCallbackGroup("id", logger, 1) - // should be called on every tick, with 10 intervals - callbacks.Register("c1", callback1) - // should be called with 10, 30, 50, 50, 10, 30, 50, 50, ... intervals - callbacks.Register("c2", callback2, WithIntervals(intervals2)) - // should be called with 60, 60, ... intervals - callbacks.Register("c3", callback3, WithIntervals(intervals3)) - - cm := NewManager(ticker, callbacks.CycleCallback) - cm.Start() - time.Sleep(400 * time.Millisecond) - cm.StopAndWait(context.Background()) - - // within 400 ms c1 should be called at least 30x - require.GreaterOrEqual(t, len(executionTimes1), 30) - // 1st call on 1st tick after 10ms - sumDuration := time.Duration(10) - for i := 0; i < 30; i++ { - assert.GreaterOrEqual(t, executionTimes1[i], sumDuration) - sumDuration += 10 * time.Millisecond - } - - // within 400 ms c2 should be called at least 8x - require.GreaterOrEqual(t, len(executionTimes2), 8) - // 1st call on 1st tick after 10ms - sumDuration = time.Duration(0) - for i := 0; i < 8; i++ { - assert.GreaterOrEqual(t, executionTimes2[i], sumDuration) - switch (i + 1) % 4 { - case 0: - sumDuration += 10 * time.Millisecond - case 1: - sumDuration += 30 * time.Millisecond - case 2, 3: - sumDuration += 50 * time.Millisecond - } - } - - // within 400 ms c3 should be called at least 6x - require.GreaterOrEqual(t, len(executionTimes3), 6) - // 1st call on 1st tick after 10ms - sumDuration = time.Duration(0) - for i := 0; i < 6; i++ { - assert.GreaterOrEqual(t, executionTimes3[i], sumDuration) - sumDuration += 60 * time.Millisecond - } - }) -} - -func TestCycleCallback_Sequential_Unregister(t *testing.T) { - ctx := context.Background() - logger, _ := test.NewNullLogger() - shouldNotAbort := func() bool { return false } - - t.Run("1 executable callback, 1 unregistered", func(t *testing.T) { - executedCounter := 0 - callback := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(50 * time.Millisecond) - executedCounter++ - return true - } - var executed bool - var d time.Duration - - callbacks := NewCallbackGroup("id", logger, 1) - ctrl := callbacks.Register("c1", callback) - require.Nil(t, ctrl.Unregister(ctx)) - - start := time.Now() - executed = callbacks.CycleCallback(shouldNotAbort) - d = time.Since(start) - - assert.False(t, executed) - assert.Equal(t, 0, executedCounter) - assert.GreaterOrEqual(t, d, 0*time.Millisecond) - }) - - t.Run("2 executable callbacks, 2 unregistered", func(t *testing.T) { - executedCounter1 := 0 - callback1 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(50 * time.Millisecond) - executedCounter1++ - return true - } - executedCounter2 := 0 - callback2 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(25 * time.Millisecond) - executedCounter2++ - return true - } - var executed bool - var d time.Duration - - callbacks := NewCallbackGroup("id", logger, 1) - ctrl1 := callbacks.Register("c1", callback1) - ctrl2 := callbacks.Register("c2", callback2) - require.Nil(t, ctrl1.Unregister(ctx)) - require.Nil(t, ctrl2.Unregister(ctx)) - - start := time.Now() - executed = callbacks.CycleCallback(shouldNotAbort) - d = time.Since(start) - - assert.False(t, executed) - assert.Equal(t, 0, executedCounter1) - assert.Equal(t, 0, executedCounter2) - assert.GreaterOrEqual(t, d, 0*time.Millisecond) - }) - - t.Run("2 executable callbacks, 1 unregistered", func(t *testing.T) { - executedCounter1 := 0 - callback1 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(50 * time.Millisecond) - executedCounter1++ - return true - } - executedCounter2 := 0 - callback2 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(25 * time.Millisecond) - executedCounter2++ - return true - } - var executed bool - var d time.Duration - - callbacks := NewCallbackGroup("id", logger, 1) - ctrl1 := callbacks.Register("c1", callback1) - callbacks.Register("c2", callback2) - require.Nil(t, ctrl1.Unregister(ctx)) - - start := time.Now() - executed = callbacks.CycleCallback(shouldNotAbort) - d = time.Since(start) - - assert.True(t, executed) - assert.Equal(t, 0, executedCounter1) - assert.Equal(t, 1, executedCounter2) - assert.GreaterOrEqual(t, d, 25*time.Millisecond) - }) - - t.Run("4 executable callbacks, all unregistered at different time", func(t *testing.T) { - executedCounter1 := 0 - callback1 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(25 * time.Millisecond) - executedCounter1++ - return true - } - executedCounter2 := 0 - callback2 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(25 * time.Millisecond) - executedCounter2++ - return true - } - executedCounter3 := 0 - callback3 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(25 * time.Millisecond) - executedCounter3++ - return true - } - executedCounter4 := 0 - callback4 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(25 * time.Millisecond) - executedCounter4++ - return true - } - var executed1 bool - var executed2 bool - var executed3 bool - var executed4 bool - var d1 time.Duration - var d2 time.Duration - var d3 time.Duration - var d4 time.Duration - - callbacks := NewCallbackGroup("id", logger, 1) - ctrl1 := callbacks.Register("c1", callback1) - ctrl2 := callbacks.Register("c2", callback2) - ctrl3 := callbacks.Register("c3", callback3) - ctrl4 := callbacks.Register("c4", callback4) - require.Nil(t, ctrl3.Unregister(ctx)) - - start := time.Now() - executed1 = callbacks.CycleCallback(shouldNotAbort) - d1 = time.Since(start) - - require.Nil(t, ctrl1.Unregister(ctx)) - - start = time.Now() - executed2 = callbacks.CycleCallback(shouldNotAbort) - d2 = time.Since(start) - - require.Nil(t, ctrl4.Unregister(ctx)) - - start = time.Now() - executed3 = callbacks.CycleCallback(shouldNotAbort) - d3 = time.Since(start) - - require.Nil(t, ctrl2.Unregister(ctx)) - - start = time.Now() - executed4 = callbacks.CycleCallback(shouldNotAbort) - d4 = time.Since(start) - - assert.True(t, executed1) - assert.True(t, executed2) - assert.True(t, executed3) - assert.False(t, executed4) - assert.Equal(t, 1, executedCounter1) - assert.Equal(t, 3, executedCounter2) - assert.Equal(t, 0, executedCounter3) - assert.Equal(t, 2, executedCounter4) - assert.GreaterOrEqual(t, d1, 75*time.Millisecond) - assert.GreaterOrEqual(t, d2, 50*time.Millisecond) - assert.GreaterOrEqual(t, d3, 25*time.Millisecond) - assert.GreaterOrEqual(t, d4, 0*time.Millisecond) - }) - - t.Run("unregister is waiting till the end of execution", func(t *testing.T) { - executedCounter := 0 - callback := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(50 * time.Millisecond) - executedCounter++ - return true - } - chStarted := make(chan struct{}, 1) - chFinished := make(chan struct{}, 1) - var executed bool - var d time.Duration - - callbacks := NewCallbackGroup("id", logger, 1) - ctrl := callbacks.Register("c", callback) - - go func() { - chStarted <- struct{}{} - start := time.Now() - executed = callbacks.CycleCallback(shouldNotAbort) - d = time.Since(start) - chFinished <- struct{}{} - }() - <-chStarted - start := time.Now() - time.Sleep(25 * time.Millisecond) - require.Nil(t, ctrl.Unregister(ctx)) - du := time.Since(start) - <-chFinished - - assert.True(t, executed) - assert.Equal(t, 1, executedCounter) - assert.GreaterOrEqual(t, d, 50*time.Millisecond) - assert.GreaterOrEqual(t, du, 40*time.Millisecond) - }) - - t.Run("unregister fails due to context timeout", func(t *testing.T) { - executedCounter := 0 - callback := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(50 * time.Millisecond) - executedCounter++ - return true - } - chStarted := make(chan struct{}, 1) - chFinished := make(chan struct{}, 1) - var executed1 bool - var executed2 bool - var d1 time.Duration - var d2 time.Duration - - callbacks := NewCallbackGroup("id", logger, 1) - ctrl := callbacks.Register("c", callback) - - go func() { - chStarted <- struct{}{} - start := time.Now() - executed1 = callbacks.CycleCallback(shouldNotAbort) - d1 = time.Since(start) - chFinished <- struct{}{} - }() - <-chStarted - start := time.Now() - time.Sleep(25 * time.Millisecond) - ctxTimeout, cancel := context.WithTimeout(ctx, 5*time.Millisecond) - defer cancel() - require.NotNil(t, ctrl.Unregister(ctxTimeout)) - du := time.Since(start) - <-chFinished - - go func() { - start := time.Now() - executed2 = callbacks.CycleCallback(shouldNotAbort) - d2 = time.Since(start) - chFinished <- struct{}{} - }() - <-chFinished - - assert.True(t, executed1) - assert.True(t, executed2) - assert.Equal(t, 2, executedCounter) - assert.GreaterOrEqual(t, d1, 50*time.Millisecond) - assert.GreaterOrEqual(t, d2, 50*time.Millisecond) - assert.GreaterOrEqual(t, du, 30*time.Millisecond) - }) - - t.Run("unregister 2nd and 3rd while executing", func(t *testing.T) { - executedCounter1 := 0 - callback1 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(50 * time.Millisecond) - executedCounter1++ - return true - } - executedCounter2 := 0 - callback2 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(50 * time.Millisecond) - executedCounter2++ - return true - } - executedCounter3 := 0 - callback3 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(50 * time.Millisecond) - executedCounter3++ - return true - } - chStarted := make(chan struct{}, 1) - chFinished := make(chan struct{}, 1) - var executed bool - var d time.Duration - - callbacks := NewCallbackGroup("id", logger, 1) - callbacks.Register("c1", callback1) - ctrl2 := callbacks.Register("c2", callback2) - ctrl3 := callbacks.Register("c3", callback3) - - go func() { - chStarted <- struct{}{} - start := time.Now() - executed = callbacks.CycleCallback(shouldNotAbort) - d = time.Since(start) - chFinished <- struct{}{} - }() - <-chStarted - time.Sleep(25 * time.Millisecond) - require.Nil(t, ctrl2.Unregister(ctx)) - require.Nil(t, ctrl3.Unregister(ctx)) - <-chFinished - - assert.True(t, executed) - assert.Equal(t, 1, executedCounter1) - assert.Equal(t, 0, executedCounter2) - assert.Equal(t, 0, executedCounter3) - assert.GreaterOrEqual(t, d, 50*time.Millisecond) - }) -} - -func TestCycleCallback_Sequential_Deactivate(t *testing.T) { - ctx := context.Background() - logger, _ := test.NewNullLogger() - shouldNotAbort := func() bool { return false } - - t.Run("1 executable callback, 1 deactivated", func(t *testing.T) { - executedCounter := 0 - callback := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(50 * time.Millisecond) - executedCounter++ - return true - } - var executed bool - var d time.Duration - - callbacks := NewCallbackGroup("id", logger, 1) - ctrl := callbacks.Register("c1", callback) - require.Nil(t, ctrl.Deactivate(ctx)) - - start := time.Now() - executed = callbacks.CycleCallback(shouldNotAbort) - d = time.Since(start) - - assert.False(t, executed) - assert.Equal(t, 0, executedCounter) - assert.GreaterOrEqual(t, d, 0*time.Millisecond) - }) - - t.Run("2 executable callbacks, 2 deactivated", func(t *testing.T) { - executedCounter1 := 0 - callback1 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(50 * time.Millisecond) - executedCounter1++ - return true - } - executedCounter2 := 0 - callback2 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(25 * time.Millisecond) - executedCounter2++ - return true - } - var executed bool - var d time.Duration - - callbacks := NewCallbackGroup("id", logger, 1) - ctrl1 := callbacks.Register("c1", callback1) - ctrl2 := callbacks.Register("c2", callback2) - require.Nil(t, ctrl1.Deactivate(ctx)) - require.Nil(t, ctrl2.Deactivate(ctx)) - - start := time.Now() - executed = callbacks.CycleCallback(shouldNotAbort) - d = time.Since(start) - - assert.False(t, executed) - assert.Equal(t, 0, executedCounter1) - assert.Equal(t, 0, executedCounter2) - assert.GreaterOrEqual(t, d, 0*time.Millisecond) - }) - - t.Run("2 executable callbacks, 1 deactivated", func(t *testing.T) { - executedCounter1 := 0 - callback1 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(50 * time.Millisecond) - executedCounter1++ - return true - } - executedCounter2 := 0 - callback2 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(25 * time.Millisecond) - executedCounter2++ - return true - } - var executed bool - var d time.Duration - - callbacks := NewCallbackGroup("id", logger, 1) - ctrl1 := callbacks.Register("c1", callback1) - callbacks.Register("c2", callback2) - require.Nil(t, ctrl1.Deactivate(ctx)) - - start := time.Now() - executed = callbacks.CycleCallback(shouldNotAbort) - d = time.Since(start) - - assert.True(t, executed) - assert.Equal(t, 0, executedCounter1) - assert.Equal(t, 1, executedCounter2) - assert.GreaterOrEqual(t, d, 25*time.Millisecond) - }) - - t.Run("4 executable callbacks, all deactivated at different time", func(t *testing.T) { - executedCounter1 := 0 - callback1 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(25 * time.Millisecond) - executedCounter1++ - return true - } - executedCounter2 := 0 - callback2 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(25 * time.Millisecond) - executedCounter2++ - return true - } - executedCounter3 := 0 - callback3 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(25 * time.Millisecond) - executedCounter3++ - return true - } - executedCounter4 := 0 - callback4 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(25 * time.Millisecond) - executedCounter4++ - return true - } - var executed1 bool - var executed2 bool - var executed3 bool - var executed4 bool - var d1 time.Duration - var d2 time.Duration - var d3 time.Duration - var d4 time.Duration - - callbacks := NewCallbackGroup("id", logger, 1) - ctrl1 := callbacks.Register("c1", callback1) - ctrl2 := callbacks.Register("c2", callback2) - ctrl3 := callbacks.Register("c3", callback3) - ctrl4 := callbacks.Register("c4", callback4) - require.Nil(t, ctrl3.Deactivate(ctx)) - - start := time.Now() - executed1 = callbacks.CycleCallback(shouldNotAbort) - d1 = time.Since(start) - - require.Nil(t, ctrl1.Deactivate(ctx)) - - start = time.Now() - executed2 = callbacks.CycleCallback(shouldNotAbort) - d2 = time.Since(start) - - require.Nil(t, ctrl4.Deactivate(ctx)) - - start = time.Now() - executed3 = callbacks.CycleCallback(shouldNotAbort) - d3 = time.Since(start) - - require.Nil(t, ctrl2.Deactivate(ctx)) - - start = time.Now() - executed4 = callbacks.CycleCallback(shouldNotAbort) - d4 = time.Since(start) - - assert.True(t, executed1) - assert.True(t, executed2) - assert.True(t, executed3) - assert.False(t, executed4) - assert.Equal(t, 1, executedCounter1) - assert.Equal(t, 3, executedCounter2) - assert.Equal(t, 0, executedCounter3) - assert.Equal(t, 2, executedCounter4) - assert.GreaterOrEqual(t, d1, 75*time.Millisecond) - assert.GreaterOrEqual(t, d2, 50*time.Millisecond) - assert.GreaterOrEqual(t, d3, 25*time.Millisecond) - assert.GreaterOrEqual(t, d4, 0*time.Millisecond) - }) - - t.Run("deactivate is waiting till the end of execution", func(t *testing.T) { - executedCounter := 0 - callback := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(50 * time.Millisecond) - executedCounter++ - return true - } - chStarted := make(chan struct{}, 1) - chFinished := make(chan struct{}, 1) - var executed bool - var d time.Duration - - callbacks := NewCallbackGroup("id", logger, 1) - ctrl := callbacks.Register("c", callback) - - go func() { - chStarted <- struct{}{} - start := time.Now() - executed = callbacks.CycleCallback(shouldNotAbort) - d = time.Since(start) - chFinished <- struct{}{} - }() - <-chStarted - start := time.Now() - time.Sleep(25 * time.Millisecond) - require.Nil(t, ctrl.Deactivate(ctx)) - du := time.Since(start) - <-chFinished - - assert.True(t, executed) - assert.Equal(t, 1, executedCounter) - assert.GreaterOrEqual(t, d, 50*time.Millisecond) - assert.GreaterOrEqual(t, du, 40*time.Millisecond) - }) - - t.Run("deactivate fails due to context timeout", func(t *testing.T) { - executedCounter := 0 - callback := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(50 * time.Millisecond) - executedCounter++ - return true - } - chStarted := make(chan struct{}, 1) - chFinished := make(chan struct{}, 1) - var executed1 bool - var executed2 bool - var d1 time.Duration - var d2 time.Duration - - callbacks := NewCallbackGroup("id", logger, 1) - ctrl := callbacks.Register("c", callback) - - go func() { - chStarted <- struct{}{} - start := time.Now() - executed1 = callbacks.CycleCallback(shouldNotAbort) - d1 = time.Since(start) - chFinished <- struct{}{} - }() - <-chStarted - start := time.Now() - time.Sleep(25 * time.Millisecond) - ctxTimeout, cancel := context.WithTimeout(ctx, 5*time.Millisecond) - defer cancel() - require.NotNil(t, ctrl.Deactivate(ctxTimeout)) - du := time.Since(start) - <-chFinished - - go func() { - start := time.Now() - executed2 = callbacks.CycleCallback(shouldNotAbort) - d2 = time.Since(start) - chFinished <- struct{}{} - }() - <-chFinished - - assert.True(t, executed1) - assert.True(t, executed2) - assert.Equal(t, 2, executedCounter) - assert.GreaterOrEqual(t, d1, 50*time.Millisecond) - assert.GreaterOrEqual(t, d2, 50*time.Millisecond) - assert.GreaterOrEqual(t, du, 30*time.Millisecond) - }) - - t.Run("deactivate 2nd and 3rd while executing", func(t *testing.T) { - executedCounter1 := 0 - callback1 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(50 * time.Millisecond) - executedCounter1++ - return true - } - executedCounter2 := 0 - callback2 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(50 * time.Millisecond) - executedCounter2++ - return true - } - executedCounter3 := 0 - callback3 := func(shouldAbort ShouldAbortCallback) bool { - time.Sleep(50 * time.Millisecond) - executedCounter3++ - return true - } - chStarted := make(chan struct{}, 1) - chFinished := make(chan struct{}, 1) - var executed bool - var d time.Duration - - callbacks := NewCallbackGroup("id", logger, 1) - callbacks.Register("c1", callback1) - ctrl2 := callbacks.Register("c2", callback2) - ctrl3 := callbacks.Register("c3", callback3) - - go func() { - chStarted <- struct{}{} - start := time.Now() - executed = callbacks.CycleCallback(shouldNotAbort) - d = time.Since(start) - chFinished <- struct{}{} - }() - <-chStarted - time.Sleep(25 * time.Millisecond) - require.Nil(t, ctrl2.Deactivate(ctx)) - require.Nil(t, ctrl3.Deactivate(ctx)) - <-chFinished - - assert.True(t, executed) - assert.Equal(t, 1, executedCounter1) - assert.Equal(t, 0, executedCounter2) - assert.Equal(t, 0, executedCounter3) - assert.GreaterOrEqual(t, d, 50*time.Millisecond) - }) -} diff --git a/entities/cyclemanager/cyclemanager.go b/entities/cyclemanager/cyclemanager.go deleted file mode 100644 index b260232cf197487771ee64b6feb45911cc74c434..0000000000000000000000000000000000000000 --- a/entities/cyclemanager/cyclemanager.go +++ /dev/null @@ -1,238 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package cyclemanager - -import ( - "context" - "fmt" - "runtime" - "sync" -) - -var _NUMCPU = runtime.NumCPU() - -type ( - // indicates whether cyclemanager's stop was requested to allow safely - // abort execution of CycleCallback and stop cyclemanager earlier - ShouldAbortCallback func() bool - // return value indicates whether actual work was done in the cycle - CycleCallback func(shouldAbort ShouldAbortCallback) bool -) - -type CycleManager interface { - Start() - Stop(ctx context.Context) chan bool - StopAndWait(ctx context.Context) error - Running() bool -} - -type cycleManager struct { - sync.RWMutex - - cycleCallback CycleCallback - cycleTicker CycleTicker - running bool - stopSignal chan struct{} - - stopContexts []context.Context - stopResults []chan bool -} - -func NewManager(cycleTicker CycleTicker, cycleCallback CycleCallback) CycleManager { - return &cycleManager{ - cycleCallback: cycleCallback, - cycleTicker: cycleTicker, - running: false, - stopSignal: make(chan struct{}, 1), - } -} - -// Starts instance, does not block -// Does nothing if instance is already started -func (c *cycleManager) Start() { - c.Lock() - defer c.Unlock() - - if c.running { - return - } - - go func() { - c.cycleTicker.Start() - defer c.cycleTicker.Stop() - - for { - if c.isStopRequested() { - c.Lock() - if c.shouldStop() { - c.handleStopRequest(true) - c.Unlock() - break - } - c.handleStopRequest(false) - c.Unlock() - continue - } - c.cycleTicker.CycleExecuted(c.cycleCallback(c.shouldAbortCycleCallback)) - } - }() - - c.running = true -} - -// Stops running instance, does not block -// Returns channel with final stop result - true / false -// -// If given context is cancelled before it is handled by stop logic, instance is not stopped -// If called multiple times, all contexts have to be cancelled to cancel stop -// (any valid will result in stopping instance) -// stopResult is the same (consistent) for multiple calls -func (c *cycleManager) Stop(ctx context.Context) (stopResult chan bool) { - c.Lock() - defer c.Unlock() - - stopResult = make(chan bool, 1) - if !c.running { - stopResult <- true - close(stopResult) - return stopResult - } - - if len(c.stopContexts) == 0 { - defer func() { - c.stopSignal <- struct{}{} - }() - } - c.stopContexts = append(c.stopContexts, ctx) - c.stopResults = append(c.stopResults, stopResult) - - return stopResult -} - -// Stops running instance, waits for stop to occur or context to expire (which comes first) -// Returns error if instance was not stopped -func (c *cycleManager) StopAndWait(ctx context.Context) error { - // if both channels are ready, chan is selected randomly, therefore regardless of - // channel selected first, second one is also checked - stop := c.Stop(ctx) - done := ctx.Done() - - select { - case <-done: - select { - case stopped := <-stop: - if !stopped { - return ctx.Err() - } - default: - return ctx.Err() - } - case stopped := <-stop: - if !stopped { - if ctx.Err() != nil { - return ctx.Err() - } - return fmt.Errorf("failed to stop cycle") - } - } - return nil -} - -func (c *cycleManager) Running() bool { - c.RLock() - defer c.RUnlock() - - return c.running -} - -func (c *cycleManager) shouldStop() bool { - for _, ctx := range c.stopContexts { - if ctx.Err() == nil { - return true - } - } - return false -} - -func (c *cycleManager) shouldAbortCycleCallback() bool { - c.RLock() - defer c.RUnlock() - - return c.shouldStop() -} - -func (c *cycleManager) isStopRequested() bool { - select { - case <-c.stopSignal: - case <-c.cycleTicker.C(): - // as stop chan has higher priority, - // it is checked again in case of ticker was selected over stop if both were ready - select { - case <-c.stopSignal: - default: - return false - } - } - return true -} - -func (c *cycleManager) handleStopRequest(stopped bool) { - for _, stopResult := range c.stopResults { - stopResult <- stopped - close(stopResult) - } - c.running = !stopped - c.stopContexts = nil - c.stopResults = nil -} - -func NewManagerNoop() CycleManager { - return &cycleManagerNoop{running: false} -} - -type cycleManagerNoop struct { - running bool -} - -func (c *cycleManagerNoop) Start() { - c.running = true -} - -func (c *cycleManagerNoop) Stop(ctx context.Context) chan bool { - if !c.running { - return c.closedChan(true) - } - if ctx.Err() != nil { - return c.closedChan(false) - } - - c.running = false - return c.closedChan(true) -} - -func (c *cycleManagerNoop) StopAndWait(ctx context.Context) error { - if <-c.Stop(ctx) { - return nil - } - return ctx.Err() -} - -func (c *cycleManagerNoop) Running() bool { - return c.running -} - -func (c *cycleManagerNoop) closedChan(val bool) chan bool { - ch := make(chan bool, 1) - ch <- val - close(ch) - return ch -} diff --git a/entities/cyclemanager/cyclemanager_test.go b/entities/cyclemanager/cyclemanager_test.go deleted file mode 100644 index d6f26fb251f0c9e7ecc049e0e0768c0e6e44fb4d..0000000000000000000000000000000000000000 --- a/entities/cyclemanager/cyclemanager_test.go +++ /dev/null @@ -1,405 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package cyclemanager - -import ( - "context" - "sync" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -type cycleCallbackProvider struct { - sync.Mutex - - firstCycleStarted chan struct{} - cycleCallback CycleCallback - results chan string -} - -func newProvider(cycleDuration time.Duration, resultsSize uint) *cycleCallbackProvider { - return newProviderAbortable(cycleDuration, resultsSize, 1) -} - -func newProviderAbortable(cycleDuration time.Duration, resultsSize uint, aborts int) *cycleCallbackProvider { - fs := false - p := &cycleCallbackProvider{} - p.results = make(chan string, resultsSize) - p.firstCycleStarted = make(chan struct{}, 1) - p.cycleCallback = func(shouldAbort ShouldAbortCallback) bool { - p.Lock() - if !fs { - p.firstCycleStarted <- struct{}{} - fs = true - } - p.Unlock() - - if aborts > 1 { - for i := 0; i < aborts; i++ { - time.Sleep(cycleDuration / time.Duration(aborts)) - if shouldAbort() { - return true - } - } - } else { - time.Sleep(cycleDuration) - } - p.results <- "something wonderful..." - return true - } - return p -} - -func TestCycleManager_beforeTimeout(t *testing.T) { - cycleInterval := 5 * time.Millisecond - cycleDuration := 1 * time.Millisecond - stopTimeout := 12 * time.Millisecond - - p := newProvider(cycleDuration, 1) - var cm CycleManager - - t.Run("create new", func(t *testing.T) { - cm = NewManager(NewFixedTicker(cycleInterval), p.cycleCallback) - - assert.False(t, cm.Running()) - }) - - t.Run("start", func(t *testing.T) { - cm.Start() - <-p.firstCycleStarted - - assert.True(t, cm.Running()) - }) - - t.Run("stop", func(t *testing.T) { - timeoutCtx, cancel := context.WithTimeout(context.Background(), stopTimeout) - defer cancel() - - stopResult := cm.Stop(timeoutCtx) - - select { - case <-timeoutCtx.Done(): - t.Fatal(timeoutCtx.Err().Error(), "failed to stop") - case stopped := <-stopResult: - assert.True(t, stopped) - assert.False(t, cm.Running()) - assert.Equal(t, "something wonderful...", <-p.results) - } - }) -} - -func TestCycleManager_beforeTimeoutWithWait(t *testing.T) { - cycleInterval := 5 * time.Millisecond - cycleDuration := 1 * time.Millisecond - stopTimeout := 12 * time.Millisecond - - p := newProvider(cycleDuration, 1) - var cm CycleManager - - t.Run("create new", func(t *testing.T) { - cm = NewManager(NewFixedTicker(cycleInterval), p.cycleCallback) - - assert.False(t, cm.Running()) - }) - - t.Run("start", func(t *testing.T) { - cm.Start() - <-p.firstCycleStarted - - assert.True(t, cm.Running()) - }) - - t.Run("stop", func(t *testing.T) { - timeoutCtx, cancel := context.WithTimeout(context.Background(), stopTimeout) - defer cancel() - - err := cm.StopAndWait(timeoutCtx) - - assert.Nil(t, err) - assert.False(t, cm.Running()) - assert.Equal(t, "something wonderful...", <-p.results) - }) -} - -func TestCycleManager_timeout(t *testing.T) { - cycleInterval := 5 * time.Millisecond - cycleDuration := 20 * time.Millisecond - stopTimeout := 12 * time.Millisecond - - p := newProvider(cycleDuration, 1) - cm := NewManager(NewFixedTicker(cycleInterval), p.cycleCallback) - - t.Run("timeout is reached", func(t *testing.T) { - timeoutCtx, cancel := context.WithTimeout(context.Background(), stopTimeout) - defer cancel() - - cm.Start() - <-p.firstCycleStarted - - stopResult := cm.Stop(timeoutCtx) - - select { - case <-timeoutCtx.Done(): - assert.True(t, cm.Running()) - case <-stopResult: - t.Fatal("stopped before timeout") - } - - // make sure it is still running - assert.False(t, <-stopResult) - assert.True(t, cm.Running()) - assert.Equal(t, "something wonderful...", <-p.results) - }) - - t.Run("stop", func(t *testing.T) { - stopResult := cm.Stop(context.Background()) - assert.True(t, <-stopResult) - assert.False(t, cm.Running()) - }) -} - -func TestCycleManager_timeoutWithWait(t *testing.T) { - cycleInterval := 5 * time.Millisecond - cycleDuration := 20 * time.Millisecond - stopTimeout := 12 * time.Millisecond - - p := newProvider(cycleDuration, 1) - cm := NewManager(NewFixedTicker(cycleInterval), p.cycleCallback) - - t.Run("timeout is reached", func(t *testing.T) { - timeoutCtx, cancel := context.WithTimeout(context.Background(), stopTimeout) - defer cancel() - - cm.Start() - <-p.firstCycleStarted - - err := cm.StopAndWait(timeoutCtx) - - assert.NotNil(t, err) - assert.Equal(t, "context deadline exceeded", err.Error()) - assert.True(t, cm.Running()) - assert.Equal(t, "something wonderful...", <-p.results) - }) - - t.Run("stop", func(t *testing.T) { - stopResult := cm.Stop(context.Background()) - assert.True(t, <-stopResult) - assert.False(t, cm.Running()) - }) -} - -func TestCycleManager_doesNotStartMultipleTimes(t *testing.T) { - cycleInterval := 5 * time.Millisecond - cycleDuration := 1 * time.Millisecond - - startCount := 5 - - p := newProvider(cycleDuration, uint(startCount)) - cm := NewManager(NewFixedTicker(cycleInterval), p.cycleCallback) - - t.Run("multiple starts", func(t *testing.T) { - for i := 0; i < startCount; i++ { - cm.Start() - } - <-p.firstCycleStarted - - stopResult := cm.Stop(context.Background()) - - assert.True(t, <-stopResult) - assert.False(t, cm.Running()) - // just one result produced - assert.Equal(t, 1, len(p.results)) - }) -} - -func TestCycleManager_doesNotStartMultipleTimesWithWait(t *testing.T) { - cycleInterval := 5 * time.Millisecond - cycleDuration := 1 * time.Millisecond - - startCount := 5 - - p := newProvider(cycleDuration, uint(startCount)) - cm := NewManager(NewFixedTicker(cycleInterval), p.cycleCallback) - - t.Run("multiple starts", func(t *testing.T) { - for i := 0; i < startCount; i++ { - cm.Start() - } - <-p.firstCycleStarted - - err := cm.StopAndWait(context.Background()) - - assert.Nil(t, err) - assert.False(t, cm.Running()) - // just one result produced - assert.Equal(t, 1, len(p.results)) - }) -} - -func TestCycleManager_handlesMultipleStops(t *testing.T) { - cycleInterval := 5 * time.Millisecond - cycleDuration := 1 * time.Millisecond - - stopCount := 5 - - p := newProvider(cycleDuration, 1) - cm := NewManager(NewFixedTicker(cycleInterval), p.cycleCallback) - - t.Run("multiple stops", func(t *testing.T) { - cm.Start() - <-p.firstCycleStarted - - stopResult := make([]chan bool, stopCount) - for i := 0; i < stopCount; i++ { - stopResult[i] = cm.Stop(context.Background()) - } - - for i := 0; i < stopCount; i++ { - assert.True(t, <-stopResult[i]) - } - assert.False(t, cm.Running()) - assert.Equal(t, "something wonderful...", <-p.results) - }) -} - -func TestCycleManager_stopsIfNotAllContextsAreCancelled(t *testing.T) { - cycleInterval := 5 * time.Millisecond - cycleDuration := 1 * time.Millisecond - stopTimeout := 5 * time.Millisecond - - p := newProvider(cycleDuration, 1) - cm := NewManager(NewFixedTicker(cycleInterval), p.cycleCallback) - - t.Run("multiple stops, few cancelled", func(t *testing.T) { - timeout1Ctx, cancel1 := context.WithTimeout(context.Background(), stopTimeout) - timeout2Ctx, cancel2 := context.WithTimeout(context.Background(), stopTimeout) - defer cancel1() - defer cancel2() - - cm.Start() - <-p.firstCycleStarted - - stopResult1 := cm.Stop(timeout1Ctx) - stopResult2 := cm.Stop(timeout2Ctx) - stopResult3 := cm.Stop(context.Background()) - - // all produce the same result: cycle was stopped - assert.True(t, <-stopResult1) - assert.True(t, <-stopResult2) - assert.True(t, <-stopResult3) - - assert.False(t, cm.Running()) - assert.Equal(t, "something wonderful...", <-p.results) - }) -} - -func TestCycleManager_doesNotStopIfAllContextsAreCancelled(t *testing.T) { - cycleInterval := 50 * time.Millisecond - cycleDuration := 10 * time.Millisecond - stopTimeout := 50 * time.Millisecond - - p := newProvider(cycleDuration, 1) - cm := NewManager(NewFixedTicker(cycleInterval), p.cycleCallback) - - t.Run("multiple stops, few cancelled", func(t *testing.T) { - timeout1Ctx, cancel1 := context.WithTimeout(context.Background(), stopTimeout) - timeout2Ctx, cancel2 := context.WithTimeout(context.Background(), stopTimeout) - timeout3Ctx, cancel3 := context.WithTimeout(context.Background(), stopTimeout) - defer cancel1() - defer cancel2() - defer cancel3() - - cm.Start() - <-p.firstCycleStarted - - stopResult1 := cm.Stop(timeout1Ctx) - stopResult2 := cm.Stop(timeout2Ctx) - stopResult3 := cm.Stop(timeout3Ctx) - - // all produce the same result: cycle was stopped - assert.False(t, <-stopResult1) - assert.False(t, <-stopResult2) - assert.False(t, <-stopResult3) - - assert.True(t, cm.Running()) - assert.Equal(t, "something wonderful...", <-p.results) - }) - - t.Run("stop", func(t *testing.T) { - stopResult := cm.Stop(context.Background()) - assert.True(t, <-stopResult) - assert.False(t, cm.Running()) - }) -} - -func TestCycleManager_cycleCallbackStoppedDueToFrequentStopChecks(t *testing.T) { - cycleInterval := 50 * time.Millisecond - cycleDuration := 300 * time.Millisecond - stopTimeout := 100 * time.Millisecond - - // despite cycleDuration is 30ms, cycle callback checks every 20ms (300/15) if it needs to be stopped - p := newProviderAbortable(cycleDuration, 1, 15) - cm := NewManager(NewFixedTicker(cycleInterval), p.cycleCallback) - - t.Run("cycle function stopped before timeout reached", func(t *testing.T) { - timeoutCtx, cancel := context.WithTimeout(context.Background(), stopTimeout) - defer cancel() - - cm.Start() - <-p.firstCycleStarted - - err := cm.StopAndWait(timeoutCtx) - - assert.Nil(t, err) - assert.False(t, cm.Running()) - assert.Equal(t, 0, len(p.results)) - }) - - t.Run("stop", func(t *testing.T) { - stopResult := cm.Stop(context.Background()) - assert.True(t, <-stopResult) - assert.False(t, cm.Running()) - }) -} - -func TestCycleManager_cycleCallbackNotStoppedDueToRareStopChecks(t *testing.T) { - cycleInterval := 50 * time.Millisecond - cycleDuration := 300 * time.Millisecond - stopTimeout := 100 * time.Millisecond - - // despite cycleDuration is 30ms, cycle callback checks every 150ms (300/2) if it needs to be stopped - p := newProviderAbortable(cycleDuration, 1, 2) - cm := NewManager(NewFixedTicker(cycleInterval), p.cycleCallback) - - t.Run("timeout reached", func(t *testing.T) { - timeoutCtx, cancel := context.WithTimeout(context.Background(), stopTimeout) - defer cancel() - - cm.Start() - <-p.firstCycleStarted - - err := cm.StopAndWait(timeoutCtx) - - assert.NotNil(t, err) - assert.Equal(t, "context deadline exceeded", err.Error()) - assert.True(t, cm.Running()) - assert.Equal(t, "something wonderful...", <-p.results) - }) - - t.Run("stop", func(t *testing.T) { - stopResult := cm.Stop(context.Background()) - assert.True(t, <-stopResult) - assert.False(t, cm.Running()) - }) -} diff --git a/entities/cyclemanager/errors.go b/entities/cyclemanager/errors.go deleted file mode 100644 index aeb409bb43dd60362fb7d8ceda5064b3611a4e99..0000000000000000000000000000000000000000 --- a/entities/cyclemanager/errors.go +++ /dev/null @@ -1,45 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package cyclemanager - -import ( - "errors" - "fmt" -) - -var ErrorCallbackNotFound = errors.New("callback not found") -var ( - formatActivateCallback = "activating callback '%s' of '%s' failed: %w" - formatDeactivateCallback = "deactivating callback '%s' of '%s' failed: %w" - formatUnregisterCallback = "unregistering callback '%s' of '%s' failed: %w" -) - -func errorActivateCallback(callbackCustomId, callbacksCustomId string, err error) error { - if err == nil { - return nil - } - return fmt.Errorf(formatActivateCallback, callbackCustomId, callbacksCustomId, err) -} - -func errorDeactivateCallback(callbackCustomId, callbacksCustomId string, err error) error { - if err == nil { - return nil - } - return fmt.Errorf(formatDeactivateCallback, callbackCustomId, callbacksCustomId, err) -} - -func errorUnregisterCallback(callbackCustomId, callbacksCustomId string, err error) error { - if err == nil { - return nil - } - return fmt.Errorf(formatUnregisterCallback, callbackCustomId, callbacksCustomId, err) -} diff --git a/entities/cyclemanager/interval.go b/entities/cyclemanager/interval.go deleted file mode 100644 index 2dcc02f86a17cdbcd4f462c05bcb7477eaaeba41..0000000000000000000000000000000000000000 --- a/entities/cyclemanager/interval.go +++ /dev/null @@ -1,90 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package cyclemanager - -import "time" - -const ( - compactionMinInterval = 3 * time.Second - compactionMaxInterval = time.Minute - compactionBase = uint(2) - compactionSteps = uint(4) -) - -// 3s . 6.8s .. 14.4s .... 29.6s ........ 60s -func CompactionCycleIntervals() CycleIntervals { - return NewExpIntervals(compactionMinInterval, compactionMaxInterval, - compactionBase, compactionSteps) -} - -// run cycle ticker with fixed minimal interval and let each shard -// take care of its intervals -func CompactionCycleTicker() CycleTicker { - return NewFixedTicker(compactionMinInterval) -} - -const ( - memtableFlushMinInterval = 100 * time.Millisecond - memtableFlushMaxInterval = 5 * time.Second - memtableFlushBase = uint(2) - memtableFlushSteps = uint(5) -) - -// 100ms . 258ms .. 574ms .... 1.206s ........ 2.471s ................ 5s -func MemtableFlushCycleIntervals() CycleIntervals { - return NewExpIntervals(memtableFlushMinInterval, memtableFlushMaxInterval, - memtableFlushBase, memtableFlushSteps) -} - -// run cycle ticker with fixed minimal interval and let each shard -// take care of its intervals -func MemtableFlushCycleTicker() CycleTicker { - return NewFixedTicker(memtableFlushMinInterval) -} - -const ( - geoCommitLoggerMinInterval = 10 * time.Second - geoCommitLoggerMaxInterval = 60 * time.Second - geoCommitLoggerBase = uint(2) - geoCommitLoggerSteps = uint(4) -) - -// 10s . 13.3s .. 20s .... 33.3s ........ 60s -func GeoCommitLoggerCycleIntervals() CycleIntervals { - return NewExpIntervals(geoCommitLoggerMinInterval, geoCommitLoggerMaxInterval, - geoCommitLoggerBase, geoCommitLoggerSteps) -} - -// run cycle ticker with fixed minimal interval and let each shard -// take care of its intervals -func GeoCommitLoggerCycleTicker() CycleTicker { - return NewFixedTicker(geoCommitLoggerMinInterval) -} - -const ( - hnswCommitLoggerMinInterval = 500 * time.Millisecond - hnswCommitLoggerMaxInterval = 10 * time.Second - hnswCommitLoggerBase = uint(2) - hnswCommitLoggerSteps = uint(5) -) - -// 500ms . 806ms .. 1.42s .... 2.65s ........ 5.1s ................10s -func HnswCommitLoggerCycleIntervals() CycleIntervals { - return NewExpIntervals(hnswCommitLoggerMinInterval, hnswCommitLoggerMaxInterval, - hnswCommitLoggerBase, hnswCommitLoggerSteps) -} - -// run cycle ticker with fixed minimal interval and let each shard -// take care of its intervals -func HnswCommitLoggerCycleTicker() CycleTicker { - return NewFixedTicker(hnswCommitLoggerMinInterval) -} diff --git a/entities/cyclemanager/ticker.go b/entities/cyclemanager/ticker.go deleted file mode 100644 index daf78238539bfb458874c9074b3ad9da16a371d1..0000000000000000000000000000000000000000 --- a/entities/cyclemanager/ticker.go +++ /dev/null @@ -1,258 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package cyclemanager - -import ( - "time" -) - -// ===== Tickers ===== - -type CycleTicker interface { - Start() - Stop() - C() <-chan time.Time - // called with bool value whenever cycle function finished execution - // true - indicates cycle function actually did some processing - // false - cycle function returned without doing anything - CycleExecuted(executed bool) -} - -type cycleTicker struct { - intervals CycleIntervals - ticker *time.Ticker -} - -func newCycleTicker(intervals CycleIntervals) CycleTicker { - if intervals == nil { - return NewNoopTicker() - } - ticker := time.NewTicker(time.Second) - ticker.Stop() - return &cycleTicker{ticker: ticker, intervals: intervals} -} - -func (t *cycleTicker) Start() { - t.ticker.Reset(t.intervals.Get()) -} - -func (t *cycleTicker) Stop() { - t.ticker.Stop() -} - -func (t *cycleTicker) C() <-chan time.Time { - return t.ticker.C -} - -func (t *cycleTicker) CycleExecuted(executed bool) { - if executed { - t.intervals.Reset() - } else { - t.intervals.Advance() - } - t.ticker.Reset(t.intervals.Get()) -} - -// Creates ticker with fixed interval. Interval is not changed regardless -// of execution results reported by cycle function -// -// If interval <= 0 given, ticker will not fire -func NewFixedTicker(interval time.Duration) CycleTicker { - return newCycleTicker(NewFixedIntervals(interval)) -} - -// Creates ticker with set of interval values. -// Ticker starts with intervals[0] value and with every report of executed "false" -// changes interval value to next one in given array up until last one. -// Report of executed "true" resets interval to interval[0] -// -// If any of intervals given is <= 0 given, ticker will not fire -func NewSeriesTicker(intervals []time.Duration) CycleTicker { - return newCycleTicker(NewSeriesIntervals(intervals)) -} - -// Creates ticker with intervals between minInterval and maxInterval values. -// Number of intervals in-between is determined by steps value. -// Ticker starts with minInterval value and with every report of executed "false" -// changes interval value to next one, up until maxInterval. -// Report of executed "true" resets interval to minInterval -// Example: for minInterval = 100ms, maxInterval = 5s, steps = 4, intervals are -// 100ms . 1325ms . 2550ms . 3775ms . 5000ms -// -// If min- or maxInterval is <= 0 or steps = 0 or min > maxInterval, ticker will not fire -func NewLinearTicker(minInterval, maxInterval time.Duration, steps uint) CycleTicker { - return newCycleTicker(NewLinearIntervals(minInterval, maxInterval, steps)) -} - -// Creates ticker with intervals between minInterval and maxInterval values. -// Number of intervals in-between is determined by steps value. -// Ticker starts with minInterval value and with every report of executed "false" -// changes interval value to next one, up until maxInterval. -// Report of executed "true" resets interval to minInterval -// Example: for minInterval = 100ms, maxInterval = 5s, base = 2, steps = 4, intervals are -// 100ms . 427ms .. 1080ms .... 2387ms ........ 5000ms -// -// If min- or maxInterval is <= 0 or base = 0 or steps = 0 or min > maxInterval, ticker will not fire -func NewExpTicker(minInterval, maxInterval time.Duration, base, steps uint) CycleTicker { - return newCycleTicker(NewExpIntervals(minInterval, maxInterval, base, steps)) -} - -type noopTicker struct { - ch chan time.Time -} - -func NewNoopTicker() CycleTicker { - return &noopTicker{ - ch: make(chan time.Time), - } -} - -func (t *noopTicker) Start() { -} - -func (t *noopTicker) Stop() { -} - -func (t *noopTicker) C() <-chan time.Time { - return t.ch -} - -func (t *noopTicker) CycleExecuted(executed bool) { -} - -// ===== Intervals ===== - -type CycleIntervals interface { - Get() time.Duration - Reset() - Advance() -} - -type fixedIntervals struct { - interval time.Duration -} - -func (i *fixedIntervals) Get() time.Duration { - return i.interval -} - -func (i *fixedIntervals) Reset() { -} - -func (i *fixedIntervals) Advance() { -} - -type seriesIntervals struct { - intervals []time.Duration - pos int -} - -func (i *seriesIntervals) Get() time.Duration { - return i.intervals[i.pos] -} - -func (i *seriesIntervals) Reset() { - i.pos = 0 -} - -func (i *seriesIntervals) Advance() { - if i.pos < len(i.intervals)-1 { - i.pos++ - } -} - -func NewFixedIntervals(interval time.Duration) CycleIntervals { - if interval <= 0 { - return nil - } - return &fixedIntervals{interval: interval} -} - -func NewSeriesIntervals(intervals []time.Duration) CycleIntervals { - if len(intervals) == 0 { - return nil - } - allSame := true - for i := range intervals { - if intervals[i] <= 0 { - return nil - } - if intervals[i] != intervals[0] { - allSame = false - } - } - if allSame { - return &fixedIntervals{interval: intervals[0]} - } - return &seriesIntervals{intervals: intervals, pos: 0} -} - -func NewLinearIntervals(minInterval, maxInterval time.Duration, steps uint) CycleIntervals { - if minInterval <= 0 || maxInterval <= 0 || steps == 0 || minInterval > maxInterval { - return nil - } - if minInterval == maxInterval { - return &fixedIntervals{interval: minInterval} - } - return &seriesIntervals{intervals: linearToIntervals(minInterval, maxInterval, steps), pos: 0} -} - -func NewExpIntervals(minInterval, maxInterval time.Duration, base, steps uint) CycleIntervals { - if minInterval <= 0 || maxInterval <= 0 || base == 0 || steps == 0 || minInterval > maxInterval { - return nil - } - if minInterval == maxInterval { - return &fixedIntervals{interval: minInterval} - } - if base == 1 { - return &seriesIntervals{intervals: linearToIntervals(minInterval, maxInterval, steps), pos: 0} - } - return &seriesIntervals{intervals: expToIntervals(minInterval, maxInterval, base, steps), pos: 0} -} - -// ===== Helper funcs ===== - -func linearToIntervals(minInterval, maxInterval time.Duration, steps uint) []time.Duration { - delta := float64(maxInterval-minInterval) / float64(steps) - floatInterval := float64(minInterval) - - intervals := make([]time.Duration, steps+1) - intervals[0] = minInterval - for i := uint(1); i <= steps; i++ { - floatInterval += delta - intervals[i] = time.Duration(floatInterval) - } - return intervals -} - -func expToIntervals(minInterval, maxInterval time.Duration, base, steps uint) []time.Duration { - sum := uint(1) - power := uint(1) - for i := uint(1); i < steps; i++ { - power *= base - sum += power - } - delta := float64(maxInterval-minInterval) / float64(sum) - floatInterval := float64(minInterval) - floatBase := float64(base) - - intervals := make([]time.Duration, steps+1) - intervals[0] = minInterval - for i := uint(1); i <= steps; i++ { - floatInterval += delta - intervals[i] = time.Duration(floatInterval) - if i < steps { - delta *= floatBase - } - } - return intervals -} diff --git a/entities/cyclemanager/ticker_test.go b/entities/cyclemanager/ticker_test.go deleted file mode 100644 index 3dfaa49c8dc80183524fe36938320c0497f53b5d..0000000000000000000000000000000000000000 --- a/entities/cyclemanager/ticker_test.go +++ /dev/null @@ -1,940 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package cyclemanager - -import ( - "context" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func Test_FixedIntervalTicker(t *testing.T) { - t.Run("channel is empty before started", func(t *testing.T) { - interval := 10 * time.Millisecond - ticker := NewFixedTicker(10 * time.Millisecond) - - assert.Len(t, ticker.C(), 0) - - ticker.Start() - time.Sleep(2 * interval) - - assert.Len(t, ticker.C(), 1) - }) - - t.Run("interval is fixed", func(t *testing.T) { - interval := 50 * time.Millisecond - tolerance := 25 * time.Millisecond - - ticker := NewFixedTicker(interval) - ticker.Start() - - t0 := time.Now() - val1 := <-ticker.C() - t1 := time.Now() - val2 := <-ticker.C() - t2 := time.Now() - val3 := <-ticker.C() - t3 := time.Now() - val4 := <-ticker.C() - t4 := time.Now() - - ticker.Stop() - - assertTimeDiffEquals(t, val1, val2, interval, tolerance) - assertTimeDiffEquals(t, val2, val3, interval, tolerance) - assertTimeDiffEquals(t, val3, val4, interval, tolerance) - assertTimeDiffEquals(t, t0, t1, interval, tolerance) - assertTimeDiffEquals(t, t1, t2, interval, tolerance) - assertTimeDiffEquals(t, t2, t3, interval, tolerance) - assertTimeDiffEquals(t, t3, t4, interval, tolerance) - }) - - t.Run("interval does not change on CycleExecuted call", func(t *testing.T) { - interval := 50 * time.Millisecond - tolerance := 25 * time.Millisecond - - ticker := NewFixedTicker(interval) - ticker.Start() - - t0 := time.Now() - val1 := <-ticker.C() - t1 := time.Now() - val2 := <-ticker.C() - t2 := time.Now() - - ticker.CycleExecuted(false) - - val3 := <-ticker.C() - t3 := time.Now() - val4 := <-ticker.C() - t4 := time.Now() - - ticker.CycleExecuted(true) - - val5 := <-ticker.C() - t5 := time.Now() - val6 := <-ticker.C() - t6 := time.Now() - - ticker.Stop() - - assertTimeDiffEquals(t, val1, val2, interval, tolerance) - assertTimeDiffEquals(t, val2, val3, interval, tolerance) - assertTimeDiffEquals(t, val3, val4, interval, tolerance) - assertTimeDiffEquals(t, val4, val5, interval, tolerance) - assertTimeDiffEquals(t, val5, val6, interval, tolerance) - assertTimeDiffEquals(t, t0, t1, interval, tolerance) - assertTimeDiffEquals(t, t1, t2, interval, tolerance) - assertTimeDiffEquals(t, t2, t3, interval, tolerance) - assertTimeDiffEquals(t, t3, t4, interval, tolerance) - assertTimeDiffEquals(t, t4, t5, interval, tolerance) - assertTimeDiffEquals(t, t5, t6, interval, tolerance) - }) - - t.Run("no ticks after stop", func(t *testing.T) { - interval := 50 * time.Millisecond - tolerance := 25 * time.Millisecond - - ticker := NewFixedTicker(interval) - ticker.Start() - - t0 := time.Now() - val1 := <-ticker.C() - t1 := time.Now() - val2 := <-ticker.C() - t2 := time.Now() - - ticker.Stop() - - tickOccurred := false - ctx, cancel := context.WithTimeout(context.Background(), 2*interval) - defer cancel() - - select { - case <-ticker.C(): - tickOccurred = true - case <-ctx.Done(): - tickOccurred = false - } - - assert.False(t, tickOccurred) - - assertTimeDiffEquals(t, val1, val2, interval, tolerance) - assertTimeDiffEquals(t, t0, t1, interval, tolerance) - assertTimeDiffEquals(t, t1, t2, interval, tolerance) - }) - - t.Run("ticker starts again", func(t *testing.T) { - interval := 50 * time.Millisecond - tolerance := 25 * time.Millisecond - - ticker := NewFixedTicker(interval) - ticker.Start() - - t01 := time.Now() - val1 := <-ticker.C() - t1 := time.Now() - val2 := <-ticker.C() - t2 := time.Now() - - ticker.Stop() - ticker.Start() - - t02 := time.Now() - val3 := <-ticker.C() - t3 := time.Now() - val4 := <-ticker.C() - t4 := time.Now() - - ticker.Stop() - - assertTimeDiffEquals(t, val1, val2, interval, tolerance) - assertTimeDiffEquals(t, val3, val4, interval, tolerance) - assertTimeDiffEquals(t, t01, t1, interval, tolerance) - assertTimeDiffEquals(t, t1, t2, interval, tolerance) - assertTimeDiffEquals(t, t02, t3, interval, tolerance) - assertTimeDiffEquals(t, t3, t4, interval, tolerance) - }) - - t.Run("ticker does not run with <= 0 interval", func(t *testing.T) { - interval := time.Duration(0) - - ticker := NewFixedTicker(interval) - ticker.Start() - - tickOccurred := false - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) - defer cancel() - - select { - case <-ticker.C(): - tickOccurred = true - case <-ctx.Done(): - tickOccurred = false - } - - assert.False(t, tickOccurred) - - ticker.Stop() - }) -} - -func Test_SeriesTicker(t *testing.T) { - t.Run("channel is empty before started", func(t *testing.T) { - intervals := []time.Duration{10 * time.Millisecond, 20 * time.Millisecond} - ticker := NewSeriesTicker(intervals) - - assert.Len(t, ticker.C(), 0) - - ticker.Start() - time.Sleep(2 * intervals[0]) - - assert.Len(t, ticker.C(), 1) - }) - - t.Run("interval is fixed between CycleExecuted calls, advances on false, resets on true", func(t *testing.T) { - intervals := []time.Duration{50 * time.Millisecond, 100 * time.Millisecond, 150 * time.Millisecond} - tolerance := 25 * time.Millisecond - - ticker := NewSeriesTicker(intervals) - ticker.Start() - - t0 := time.Now() - val1 := <-ticker.C() - t1 := time.Now() - val2 := <-ticker.C() - t2 := time.Now() - - ticker.CycleExecuted(false) - - val3 := <-ticker.C() - t3 := time.Now() - val4 := <-ticker.C() - t4 := time.Now() - - ticker.CycleExecuted(false) - - val5 := <-ticker.C() - t5 := time.Now() - val6 := <-ticker.C() - t6 := time.Now() - - ticker.CycleExecuted(false) - - val7 := <-ticker.C() - t7 := time.Now() - val8 := <-ticker.C() - t8 := time.Now() - - ticker.CycleExecuted(true) - - val9 := <-ticker.C() - t9 := time.Now() - val10 := <-ticker.C() - t10 := time.Now() - - ticker.Stop() - - assertTimeDiffEquals(t, val1, val2, intervals[0], tolerance) - assertTimeDiffEquals(t, val2, val3, intervals[1], tolerance) - assertTimeDiffEquals(t, val3, val4, intervals[1], tolerance) - assertTimeDiffEquals(t, val4, val5, intervals[2], tolerance) - assertTimeDiffEquals(t, val5, val6, intervals[2], tolerance) - assertTimeDiffEquals(t, val6, val7, intervals[2], tolerance) - assertTimeDiffEquals(t, val7, val8, intervals[2], tolerance) - assertTimeDiffEquals(t, val8, val9, intervals[0], tolerance) - assertTimeDiffEquals(t, val9, val10, intervals[0], tolerance) - assertTimeDiffEquals(t, t0, t1, intervals[0], tolerance) - assertTimeDiffEquals(t, t1, t2, intervals[0], tolerance) - assertTimeDiffEquals(t, t2, t3, intervals[1], tolerance) - assertTimeDiffEquals(t, t3, t4, intervals[1], tolerance) - assertTimeDiffEquals(t, t4, t5, intervals[2], tolerance) - assertTimeDiffEquals(t, t5, t6, intervals[2], tolerance) - assertTimeDiffEquals(t, t6, t7, intervals[2], tolerance) - assertTimeDiffEquals(t, t7, t8, intervals[2], tolerance) - assertTimeDiffEquals(t, t8, t9, intervals[0], tolerance) - assertTimeDiffEquals(t, t9, t10, intervals[0], tolerance) - }) - - t.Run("no ticks after stop", func(t *testing.T) { - intervals := []time.Duration{50 * time.Millisecond} - tolerance := 25 * time.Millisecond - - ticker := NewSeriesTicker(intervals) - ticker.Start() - - t0 := time.Now() - val1 := <-ticker.C() - t1 := time.Now() - val2 := <-ticker.C() - t2 := time.Now() - - ticker.Stop() - - tickOccurred := false - ctx, cancel := context.WithTimeout(context.Background(), 2*intervals[0]) - defer cancel() - - select { - case <-ticker.C(): - tickOccurred = true - case <-ctx.Done(): - tickOccurred = false - } - - assert.False(t, tickOccurred) - - assertTimeDiffEquals(t, val1, val2, intervals[0], tolerance) - assertTimeDiffEquals(t, t0, t1, intervals[0], tolerance) - assertTimeDiffEquals(t, t1, t2, intervals[0], tolerance) - }) - - t.Run("ticker starts again", func(t *testing.T) { - intervals := []time.Duration{50 * time.Millisecond} - tolerance := 25 * time.Millisecond - - ticker := NewSeriesTicker(intervals) - ticker.Start() - - t01 := time.Now() - val1 := <-ticker.C() - t1 := time.Now() - val2 := <-ticker.C() - t2 := time.Now() - - ticker.Stop() - ticker.Start() - - t02 := time.Now() - val3 := <-ticker.C() - t3 := time.Now() - val4 := <-ticker.C() - t4 := time.Now() - - ticker.Stop() - - assertTimeDiffEquals(t, val1, val2, intervals[0], tolerance) - assertTimeDiffEquals(t, val3, val4, intervals[0], tolerance) - assertTimeDiffEquals(t, t01, t1, intervals[0], tolerance) - assertTimeDiffEquals(t, t1, t2, intervals[0], tolerance) - assertTimeDiffEquals(t, t02, t3, intervals[0], tolerance) - assertTimeDiffEquals(t, t3, t4, intervals[0], tolerance) - }) - - t.Run("ticker does not run with invalid params", func(t *testing.T) { - run := func(t *testing.T, ticker CycleTicker) { - ticker.Start() - - tickOccurred := false - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) - defer cancel() - - select { - case <-ticker.C(): - tickOccurred = true - case <-ctx.Done(): - tickOccurred = false - } - - assert.False(t, tickOccurred) - - ticker.Stop() - } - - t.Run("any interval <= 0", func(t *testing.T) { - ticker := NewSeriesTicker([]time.Duration{50 * time.Millisecond, 0}) - - run(t, ticker) - }) - - t.Run("no intervals", func(t *testing.T) { - ticker := NewSeriesTicker([]time.Duration{}) - - run(t, ticker) - }) - }) -} - -func Test_LinearTicker(t *testing.T) { - t.Run("channel is empty before started", func(t *testing.T) { - minInterval := 10 * time.Millisecond - maxInterval := 50 * time.Millisecond - steps := uint(2) - ticker := NewLinearTicker(minInterval, maxInterval, steps) - - assert.Len(t, ticker.C(), 0) - - ticker.Start() - time.Sleep(2 * minInterval) - - assert.Len(t, ticker.C(), 1) - }) - - t.Run("interval is fixed between CycleExecuted calls, advances on false, resets on true", func(t *testing.T) { - ms50 := 50 * time.Millisecond - ms75 := 75 * time.Millisecond - ms100 := 100 * time.Millisecond - tolerance := 25 * time.Millisecond - - minInterval := ms50 - maxInterval := ms100 - steps := uint(2) - - ticker := NewLinearTicker(minInterval, maxInterval, steps) - ticker.Start() - - t0 := time.Now() - val1 := <-ticker.C() - t1 := time.Now() - val2 := <-ticker.C() - t2 := time.Now() - - ticker.CycleExecuted(false) - - val3 := <-ticker.C() - t3 := time.Now() - val4 := <-ticker.C() - t4 := time.Now() - - ticker.CycleExecuted(false) - - val5 := <-ticker.C() - t5 := time.Now() - val6 := <-ticker.C() - t6 := time.Now() - - ticker.CycleExecuted(false) - - val7 := <-ticker.C() - t7 := time.Now() - val8 := <-ticker.C() - t8 := time.Now() - - ticker.CycleExecuted(true) - - val9 := <-ticker.C() - t9 := time.Now() - val10 := <-ticker.C() - t10 := time.Now() - - ticker.Stop() - - assertTimeDiffEquals(t, val1, val2, ms50, tolerance) - assertTimeDiffEquals(t, val2, val3, ms75, tolerance) - assertTimeDiffEquals(t, val3, val4, ms75, tolerance) - assertTimeDiffEquals(t, val4, val5, ms100, tolerance) - assertTimeDiffEquals(t, val5, val6, ms100, tolerance) - assertTimeDiffEquals(t, val6, val7, ms100, tolerance) - assertTimeDiffEquals(t, val7, val8, ms100, tolerance) - assertTimeDiffEquals(t, val8, val9, ms50, tolerance) - assertTimeDiffEquals(t, val9, val10, ms50, tolerance) - assertTimeDiffEquals(t, t0, t1, ms50, tolerance) - assertTimeDiffEquals(t, t1, t2, ms50, tolerance) - assertTimeDiffEquals(t, t2, t3, ms75, tolerance) - assertTimeDiffEquals(t, t3, t4, ms75, tolerance) - assertTimeDiffEquals(t, t4, t5, ms100, tolerance) - assertTimeDiffEquals(t, t5, t6, ms100, tolerance) - assertTimeDiffEquals(t, t6, t7, ms100, tolerance) - assertTimeDiffEquals(t, t7, t8, ms100, tolerance) - assertTimeDiffEquals(t, t8, t9, ms50, tolerance) - assertTimeDiffEquals(t, t9, t10, ms50, tolerance) - }) - - t.Run("no ticks after stop", func(t *testing.T) { - minInterval := 50 * time.Millisecond - maxInterval := 100 * time.Millisecond - steps := uint(2) - tolerance := 10 * time.Millisecond - - ticker := NewLinearTicker(minInterval, maxInterval, steps) - ticker.Start() - - t0 := time.Now() - val1 := <-ticker.C() - t1 := time.Now() - val2 := <-ticker.C() - t2 := time.Now() - - ticker.Stop() - - tickOccurred := false - ctx, cancel := context.WithTimeout(context.Background(), 2*minInterval) - defer cancel() - - select { - case <-ticker.C(): - tickOccurred = true - case <-ctx.Done(): - tickOccurred = false - } - - assert.False(t, tickOccurred) - - assertTimeDiffEquals(t, val1, val2, minInterval, tolerance) - assertTimeDiffEquals(t, t0, t1, minInterval, tolerance) - assertTimeDiffEquals(t, t1, t2, minInterval, tolerance) - }) - - t.Run("ticker starts again", func(t *testing.T) { - minInterval := 50 * time.Millisecond - maxInterval := 100 * time.Millisecond - steps := uint(2) - tolerance := 25 * time.Millisecond - - ticker := NewLinearTicker(minInterval, maxInterval, steps) - ticker.Start() - - t01 := time.Now() - val1 := <-ticker.C() - t1 := time.Now() - val2 := <-ticker.C() - t2 := time.Now() - - ticker.Stop() - ticker.Start() - - t02 := time.Now() - val3 := <-ticker.C() - t3 := time.Now() - val4 := <-ticker.C() - t4 := time.Now() - - ticker.Stop() - - assertTimeDiffEquals(t, val1, val2, minInterval, tolerance) - assertTimeDiffEquals(t, val3, val4, minInterval, tolerance) - assertTimeDiffEquals(t, t01, t1, minInterval, tolerance) - assertTimeDiffEquals(t, t1, t2, minInterval, tolerance) - assertTimeDiffEquals(t, t02, t3, minInterval, tolerance) - assertTimeDiffEquals(t, t3, t4, minInterval, tolerance) - }) - - t.Run("ticker does not run with invalid params", func(t *testing.T) { - run := func(t *testing.T, ticker CycleTicker) { - ticker.Start() - - tickOccurred := false - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) - defer cancel() - - select { - case <-ticker.C(): - tickOccurred = true - case <-ctx.Done(): - tickOccurred = false - } - - assert.False(t, tickOccurred) - - ticker.Stop() - } - - t.Run("minInterval <= 0", func(t *testing.T) { - ticker := NewLinearTicker(0, 100*time.Millisecond, 1) - - run(t, ticker) - }) - - t.Run("maxInterval <= 0", func(t *testing.T) { - ticker := NewLinearTicker(50*time.Millisecond, 0, 1) - - run(t, ticker) - }) - - t.Run("steps = 0", func(t *testing.T) { - ticker := NewLinearTicker(50*time.Millisecond, 100*time.Millisecond, 0) - - run(t, ticker) - }) - - t.Run("minInterval > maxInterval", func(t *testing.T) { - ticker := NewLinearTicker(100*time.Millisecond, 50*time.Millisecond, 0) - - run(t, ticker) - }) - }) -} - -func Test_ExpTicker(t *testing.T) { - t.Run("channel is empty before started", func(t *testing.T) { - minInterval := 10 * time.Millisecond - maxInterval := 20 * time.Millisecond - base := uint(2) - steps := uint(2) - ticker := NewExpTicker(minInterval, maxInterval, base, steps) - - assert.Len(t, ticker.C(), 0) - - ticker.Start() - time.Sleep(2 * minInterval) - - assert.Len(t, ticker.C(), 1) - }) - - t.Run("interval is fixed between CycleExecuted calls, advances on false, resets on true", func(t *testing.T) { - ms25 := 25 * time.Millisecond - ms50 := 50 * time.Millisecond - ms100 := 100 * time.Millisecond - tolerance := 25 * time.Millisecond - - minInterval := ms25 - maxInterval := ms100 - base := uint(2) - steps := uint(2) - - ticker := NewExpTicker(minInterval, maxInterval, base, steps) - ticker.Start() - - t0 := time.Now() - val1 := <-ticker.C() - t1 := time.Now() - val2 := <-ticker.C() - t2 := time.Now() - - ticker.CycleExecuted(false) - - val3 := <-ticker.C() - t3 := time.Now() - val4 := <-ticker.C() - t4 := time.Now() - - ticker.CycleExecuted(false) - - val5 := <-ticker.C() - t5 := time.Now() - val6 := <-ticker.C() - t6 := time.Now() - - ticker.CycleExecuted(false) - - val7 := <-ticker.C() - t7 := time.Now() - val8 := <-ticker.C() - t8 := time.Now() - - ticker.CycleExecuted(true) - - val9 := <-ticker.C() - t9 := time.Now() - val10 := <-ticker.C() - t10 := time.Now() - - ticker.Stop() - - assertTimeDiffEquals(t, val1, val2, ms25, tolerance) - assertTimeDiffEquals(t, val2, val3, ms50, tolerance) - assertTimeDiffEquals(t, val3, val4, ms50, tolerance) - assertTimeDiffEquals(t, val4, val5, ms100, tolerance) - assertTimeDiffEquals(t, val5, val6, ms100, tolerance) - assertTimeDiffEquals(t, val6, val7, ms100, tolerance) - assertTimeDiffEquals(t, val7, val8, ms100, tolerance) - assertTimeDiffEquals(t, val8, val9, ms25, tolerance) - assertTimeDiffEquals(t, val9, val10, ms25, tolerance) - assertTimeDiffEquals(t, t0, t1, ms25, tolerance) - assertTimeDiffEquals(t, t1, t2, ms25, tolerance) - assertTimeDiffEquals(t, t2, t3, ms50, tolerance) - assertTimeDiffEquals(t, t3, t4, ms50, tolerance) - assertTimeDiffEquals(t, t4, t5, ms100, tolerance) - assertTimeDiffEquals(t, t5, t6, ms100, tolerance) - assertTimeDiffEquals(t, t6, t7, ms100, tolerance) - assertTimeDiffEquals(t, t7, t8, ms100, tolerance) - assertTimeDiffEquals(t, t8, t9, ms25, tolerance) - assertTimeDiffEquals(t, t9, t10, ms25, tolerance) - }) - - t.Run("no ticks after stop", func(t *testing.T) { - minInterval := 25 * time.Millisecond - maxInterval := 100 * time.Millisecond - base := uint(2) - steps := uint(2) - tolerance := 25 * time.Millisecond - - ticker := NewExpTicker(minInterval, maxInterval, base, steps) - ticker.Start() - - t0 := time.Now() - val1 := <-ticker.C() - t1 := time.Now() - val2 := <-ticker.C() - t2 := time.Now() - - ticker.Stop() - - tickOccurred := false - ctx, cancel := context.WithTimeout(context.Background(), 2*minInterval) - defer cancel() - - select { - case <-ticker.C(): - tickOccurred = true - case <-ctx.Done(): - tickOccurred = false - } - - assert.False(t, tickOccurred) - - assertTimeDiffEquals(t, val1, val2, minInterval, tolerance) - assertTimeDiffEquals(t, t0, t1, minInterval, tolerance) - assertTimeDiffEquals(t, t1, t2, minInterval, tolerance) - }) - - t.Run("ticker starts again", func(t *testing.T) { - minInterval := 25 * time.Millisecond - maxInterval := 100 * time.Millisecond - base := uint(2) - steps := uint(2) - tolerance := 25 * time.Millisecond - - ticker := NewExpTicker(minInterval, maxInterval, base, steps) - ticker.Start() - - t01 := time.Now() - val1 := <-ticker.C() - t1 := time.Now() - val2 := <-ticker.C() - t2 := time.Now() - - ticker.Stop() - ticker.Start() - - t02 := time.Now() - val3 := <-ticker.C() - t3 := time.Now() - val4 := <-ticker.C() - t4 := time.Now() - - ticker.Stop() - - assertTimeDiffEquals(t, val1, val2, minInterval, tolerance) - assertTimeDiffEquals(t, val3, val4, minInterval, tolerance) - assertTimeDiffEquals(t, t01, t1, minInterval, tolerance) - assertTimeDiffEquals(t, t1, t2, minInterval, tolerance) - assertTimeDiffEquals(t, t02, t3, minInterval, tolerance) - assertTimeDiffEquals(t, t3, t4, minInterval, tolerance) - }) - - t.Run("ticker does not run with invalid params", func(t *testing.T) { - run := func(t *testing.T, ticker CycleTicker) { - ticker.Start() - - tickOccurred := false - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) - defer cancel() - - select { - case <-ticker.C(): - tickOccurred = true - case <-ctx.Done(): - tickOccurred = false - } - - assert.False(t, tickOccurred) - - ticker.Stop() - } - - t.Run("minInterval <= 0", func(t *testing.T) { - ticker := NewExpTicker(0, 100*time.Millisecond, 2, 2) - - run(t, ticker) - }) - - t.Run("maxInterval <= 0", func(t *testing.T) { - ticker := NewExpTicker(100*time.Millisecond, 0, 2, 2) - - run(t, ticker) - }) - - t.Run("base == 0", func(t *testing.T) { - ticker := NewExpTicker(25*time.Millisecond, 100*time.Millisecond, 0, 2) - - run(t, ticker) - }) - - t.Run("steps = 0", func(t *testing.T) { - ticker := NewExpTicker(25*time.Millisecond, 100*time.Millisecond, 2, 0) - - run(t, ticker) - }) - - t.Run("minInterval > maxInterval", func(t *testing.T) { - ticker := NewExpTicker(100*time.Millisecond, 25*time.Millisecond, 2, 2) - - run(t, ticker) - }) - }) -} - -func Test_LinearToIntervals(t *testing.T) { - type testCase struct { - name string - minInterval time.Duration - maxInterval time.Duration - steps uint - expected []time.Duration - } - - testCases := []testCase{ - { - name: "100 => 5000; steps 2", - minInterval: 100 * time.Millisecond, - maxInterval: 5 * time.Second, - steps: 2, - expected: []time.Duration{ - 100_000_000, - 2_550_000_000, - 5_000_000_000, - }, - }, - { - name: "100 => 5000; steps 3", - minInterval: 100 * time.Millisecond, - maxInterval: 5 * time.Second, - steps: 3, - expected: []time.Duration{ - 100_000_000, - 1_733_333_333, - 3_366_666_666, - 5_000_000_000, - }, - }, - { - name: "100 => 5000; steps 4", - minInterval: 100 * time.Millisecond, - maxInterval: 5 * time.Second, - steps: 4, - expected: []time.Duration{ - 100_000_000, - 1_325_000_000, - 2_550_000_000, - 3_775_000_000, - 5_000_000_000, - }, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - res := linearToIntervals(tc.minInterval, tc.maxInterval, tc.steps) - - assert.ElementsMatch(t, res, tc.expected) - }) - } -} - -func Test_ExpToIntervals(t *testing.T) { - type testCase struct { - name string - minInterval time.Duration - maxInterval time.Duration - base uint - steps uint - expected []time.Duration - } - - testCases := []testCase{ - { - name: "100 => 5000; base 2; steps 2", - minInterval: 100 * time.Millisecond, - maxInterval: 5 * time.Second, - base: 2, - steps: 2, - expected: []time.Duration{ - 100_000_000, - 1_733_333_333, - 5_000_000_000, - }, - }, - { - name: "100 => 5000; base 2; steps 3", - minInterval: 100 * time.Millisecond, - maxInterval: 5 * time.Second, - base: 2, - steps: 3, - expected: []time.Duration{ - 100_000_000, - 800_000_000, - 2_200_000_000, - 5_000_000_000, - }, - }, - { - name: "100 => 5000; base 2; steps 4", - minInterval: 100 * time.Millisecond, - maxInterval: 5 * time.Second, - base: 2, - steps: 4, - expected: []time.Duration{ - 100_000_000, - 426_666_666, - 1_080_000_000, - 2_386_666_666, - 5_000_000_000, - }, - }, - { - name: "100 => 5000; base 3; steps 2", - minInterval: 100 * time.Millisecond, - maxInterval: 5 * time.Second, - base: 3, - steps: 2, - expected: []time.Duration{ - 100_000_000, - 1_325_000_000, - 5_000_000_000, - }, - }, - { - name: "100 => 5000; base 3; steps 3", - minInterval: 100 * time.Millisecond, - maxInterval: 5 * time.Second, - base: 3, - steps: 3, - expected: []time.Duration{ - 100_000_000, - 476_923_076, - 1_607_692_307, - 5_000_000_000, - }, - }, - { - name: "100 => 5000; base 3; steps 4", - minInterval: 100 * time.Millisecond, - maxInterval: 5 * time.Second, - base: 3, - steps: 4, - expected: []time.Duration{ - 100_000_000, - 222_500_000, - 590_000_000, - 1_692_500_000, - 5_000_000_000, - }, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - res := expToIntervals(tc.minInterval, tc.maxInterval, tc.base, tc.steps) - - assert.ElementsMatch(t, res, tc.expected) - }) - } -} - -func assertTimeDiffEquals(t *testing.T, time1, time2 time.Time, expected time.Duration, tolerance time.Duration) { - diff := time2.Sub(time1) - assert.GreaterOrEqual(t, diff, expected-tolerance) - assert.LessOrEqual(t, diff, expected+tolerance) -} diff --git a/entities/deepcopy/models_deepcopy.go b/entities/deepcopy/models_deepcopy.go deleted file mode 100644 index b6e9a57aab7bcc154e8cf4555103b45df8db2a3b..0000000000000000000000000000000000000000 --- a/entities/deepcopy/models_deepcopy.go +++ /dev/null @@ -1,99 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package deepcopy - -import "github.com/weaviate/weaviate/entities/models" - -func Schema(s *models.Schema) *models.Schema { - classes := make([]*models.Class, len(s.Classes)) - for i, class := range s.Classes { - classes[i] = Class(class) - } - - return &models.Schema{Name: s.Name, Maintainer: s.Maintainer, Classes: classes} -} - -func Class(c *models.Class) *models.Class { - if c == nil { - return nil - } - - var properties []*models.Property = nil - if c.Properties != nil { - properties = make([]*models.Property, len(c.Properties)) - for i, prop := range c.Properties { - properties[i] = Prop(prop) - } - } - var replicationConf *models.ReplicationConfig = nil - if c.ReplicationConfig != nil { - replicationConf = &models.ReplicationConfig{Factor: c.ReplicationConfig.Factor} - } - - return &models.Class{ - Class: c.Class, - Description: c.Description, - ModuleConfig: c.ModuleConfig, - ShardingConfig: c.ShardingConfig, - VectorIndexConfig: c.VectorIndexConfig, - VectorIndexType: c.VectorIndexType, - ReplicationConfig: replicationConf, - Vectorizer: c.Vectorizer, - InvertedIndexConfig: InvertedIndexConfig(c.InvertedIndexConfig), - Properties: properties, - } -} - -func Prop(p *models.Property) *models.Property { - return &models.Property{ - DataType: p.DataType, - Description: p.Description, - ModuleConfig: p.ModuleConfig, - Name: p.Name, - Tokenization: p.Tokenization, - IndexFilterable: ptrBoolCopy(p.IndexFilterable), - IndexSearchable: ptrBoolCopy(p.IndexSearchable), - } -} - -func ptrBoolCopy(ptrBool *bool) *bool { - if ptrBool != nil { - b := *ptrBool - return &b - } - return nil -} - -func InvertedIndexConfig(i *models.InvertedIndexConfig) *models.InvertedIndexConfig { - if i == nil { - return nil - } - - var bm25 *models.BM25Config = nil - if i.Bm25 != nil { - bm25 = &models.BM25Config{B: i.Bm25.B, K1: i.Bm25.K1} - } - - var stopwords *models.StopwordConfig = nil - if i.Stopwords != nil { - stopwords = &models.StopwordConfig{Additions: i.Stopwords.Additions, Preset: i.Stopwords.Preset, Removals: i.Stopwords.Removals} - } - - return &models.InvertedIndexConfig{ - Bm25: bm25, - CleanupIntervalSeconds: i.CleanupIntervalSeconds, - IndexNullState: i.IndexNullState, - IndexPropertyLength: i.IndexPropertyLength, - IndexTimestamps: i.IndexTimestamps, - Stopwords: stopwords, - } -} diff --git a/entities/diskio/metered_reader.go b/entities/diskio/metered_reader.go deleted file mode 100644 index 44af2a723aa21937674a84d02e622ff309555c93..0000000000000000000000000000000000000000 --- a/entities/diskio/metered_reader.go +++ /dev/null @@ -1,46 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package diskio - -import ( - "io" - "time" -) - -type MeteredReaderCallback func(read int64, nanoseconds int64) - -type MeteredReader struct { - r io.Reader - cb MeteredReaderCallback -} - -// Read passes the read through to the underlying reader. On a successful read, -// it will trigger the attached callback and provide it with metrics. If no -// callback is set, it will ignore it. -func (m *MeteredReader) Read(p []byte) (n int, err error) { - start := time.Now() - n, err = m.r.Read(p) - took := time.Since(start).Nanoseconds() - if err != nil { - return - } - - if m.cb != nil { - m.cb(int64(n), took) - } - - return -} - -func NewMeteredReader(r io.Reader, cb MeteredReaderCallback) *MeteredReader { - return &MeteredReader{r: r, cb: cb} -} diff --git a/entities/diskio/metered_reader_test.go b/entities/diskio/metered_reader_test.go deleted file mode 100644 index 0fa56b11f27afe7e8ac01773b8b565d4e9dc627c..0000000000000000000000000000000000000000 --- a/entities/diskio/metered_reader_test.go +++ /dev/null @@ -1,81 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package diskio - -import ( - "bytes" - "io" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestMeteredReader(t *testing.T) { - data := make([]byte, 128) - - t.Run("happy path - with callback", func(t *testing.T) { - var ( - read int64 - took int64 - ) - - cb := func(r int64, n int64) { - read = r - took = n - } - - mr := NewMeteredReader(bytes.NewReader(data), cb) - - target := make([]byte, 128) - n, err := mr.Read(target) - - require.Nil(t, err) - assert.Equal(t, int64(n), read) - assert.Greater(t, took, int64(0)) - }) - - t.Run("happy path - without callback", func(t *testing.T) { - mr := NewMeteredReader(bytes.NewReader(data), nil) - - target := make([]byte, 128) - _, err := mr.Read(target) - require.Nil(t, err) - }) - - t.Run("with an error", func(t *testing.T) { - var ( - read int64 - took int64 - ) - - cb := func(r int64, n int64) { - read = r - took = n - } - - underlying := bytes.NewReader(data) - // provoke EOF error by seeking to end of data - underlying.Seek(128, 0) - mr := NewMeteredReader(underlying, cb) - - target := make([]byte, 128) - _, err := mr.Read(target) - - assert.Equal(t, io.EOF, err) - - // callback should not have been called in error cases, so we expect to - // read initial values - assert.Equal(t, int64(0), read) - assert.Equal(t, int64(0), took) - }) -} diff --git a/entities/dto/dto.go b/entities/dto/dto.go deleted file mode 100644 index ee328015251282b4e12bf577e5d9c65272e80392..0000000000000000000000000000000000000000 --- a/entities/dto/dto.go +++ /dev/null @@ -1,45 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package dto - -import ( - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/search" - "github.com/weaviate/weaviate/entities/searchparams" -) - -type GroupParams struct { - Strategy string - Force float32 -} - -type GetParams struct { - Filters *filters.LocalFilter - ClassName string - Pagination *filters.Pagination - Cursor *filters.Cursor - Sort []filters.Sort - Properties search.SelectProperties - NearVector *searchparams.NearVector - NearObject *searchparams.NearObject - KeywordRanking *searchparams.KeywordRanking - HybridSearch *searchparams.HybridSearch - GroupBy *searchparams.GroupBy - SearchVector []float32 - Group *GroupParams - ModuleParams map[string]interface{} - AdditionalProperties additional.Properties - ReplicationProperties *additional.ReplicationProperties - Tenant string - IsRefOrigin bool // is created by ref filter -} diff --git a/entities/errorcompounder/compounder.go b/entities/errorcompounder/compounder.go deleted file mode 100644 index acdefa286b5e97838e876333f951e28116c23cd4..0000000000000000000000000000000000000000 --- a/entities/errorcompounder/compounder.go +++ /dev/null @@ -1,60 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package errorcompounder - -import ( - "fmt" - "strings" - - "github.com/pkg/errors" -) - -type ErrorCompounder struct { - errors []error -} - -func (ec *ErrorCompounder) Add(err error) { - if err != nil { - ec.errors = append(ec.errors, err) - } -} - -func (ec *ErrorCompounder) Addf(msg string, args ...interface{}) { - ec.errors = append(ec.errors, fmt.Errorf(msg, args...)) -} - -func (ec *ErrorCompounder) AddWrap(err error, wrapMsg ...string) { - if err != nil { - ec.errors = append(ec.errors, errors.Wrap(err, wrapMsg[0])) - } -} - -func (ec *ErrorCompounder) ToError() error { - if len(ec.errors) == 0 { - return nil - } - - var msg strings.Builder - for i, err := range ec.errors { - if i != 0 { - msg.WriteString(", ") - } - - msg.WriteString(err.Error()) - } - - return errors.New(msg.String()) -} - -func (ec *ErrorCompounder) Len() int { - return len(ec.errors) -} diff --git a/entities/errorcompounder/compounder_thread_safe.go b/entities/errorcompounder/compounder_thread_safe.go deleted file mode 100644 index eceeb36ba7b0796521f1b8fe6d4a4aae15dd036c..0000000000000000000000000000000000000000 --- a/entities/errorcompounder/compounder_thread_safe.go +++ /dev/null @@ -1,58 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package errorcompounder - -import ( - "fmt" - "strings" - "sync" - - "github.com/pkg/errors" -) - -type SafeErrorCompounder struct { - sync.Mutex - errors []error -} - -func (ec *SafeErrorCompounder) Add(err error) { - ec.Lock() - defer ec.Unlock() - if err != nil { - ec.errors = append(ec.errors, err) - } -} - -func (ec *SafeErrorCompounder) Addf(msg string, args ...interface{}) { - ec.Lock() - defer ec.Unlock() - ec.errors = append(ec.errors, fmt.Errorf(msg, args...)) -} - -func (ec *SafeErrorCompounder) ToError() error { - ec.Lock() - defer ec.Unlock() - if len(ec.errors) == 0 { - return nil - } - - var msg strings.Builder - for i, err := range ec.errors { - if i != 0 { - msg.WriteString(", ") - } - - msg.WriteString(err.Error()) - } - - return errors.New(msg.String()) -} diff --git a/entities/errors/errors_graphql.go b/entities/errors/errors_graphql.go deleted file mode 100644 index afdab054e253fcdd582c6ddef01c674174470830..0000000000000000000000000000000000000000 --- a/entities/errors/errors_graphql.go +++ /dev/null @@ -1,66 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package errors - -import ( - "errors" - "fmt" -) - -type ErrGraphQLUser struct { - err error - queryType, className string -} - -func (e ErrGraphQLUser) Error() string { - return e.err.Error() -} - -func (e ErrGraphQLUser) OriginalError() error { - return e.err -} - -func (e ErrGraphQLUser) QueryType() string { - return e.queryType -} - -func (e ErrGraphQLUser) ClassName() string { - return e.className -} - -func NewErrGraphQLUser(err error, operation, className string) ErrGraphQLUser { - return ErrGraphQLUser{err, operation, className} -} - -type ErrRateLimit struct { - err error -} - -func (e ErrRateLimit) Error() string { - return e.err.Error() -} - -func NewErrRateLimit() ErrRateLimit { - return ErrRateLimit{errors.New("429 Too many requests")} -} - -type ErrLockConnector struct { - err error -} - -func (e ErrLockConnector) Error() string { - return e.err.Error() -} - -func NewErrLockConnector(err error) ErrLockConnector { - return ErrLockConnector{fmt.Errorf("could not acquire lock: %w", err)} -} diff --git a/entities/errors/errors_http.go b/entities/errors/errors_http.go deleted file mode 100644 index 7db4d023a7a689148d7ae73c6644ce36cef0fabf..0000000000000000000000000000000000000000 --- a/entities/errors/errors_http.go +++ /dev/null @@ -1,63 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package errors - -type ErrUnprocessable struct { - err error -} - -func (e ErrUnprocessable) Error() string { - return e.err.Error() -} - -func NewErrUnprocessable(err error) ErrUnprocessable { - return ErrUnprocessable{err} -} - -type ErrNotFound struct { - err error -} - -func (e ErrNotFound) Error() string { - if e.err != nil { - return e.err.Error() - } - return "" -} - -func NewErrNotFound(err error) ErrNotFound { - return ErrNotFound{err} -} - -type ErrContextExpired struct { - err error -} - -func (e ErrContextExpired) Error() string { - return e.err.Error() -} - -func NewErrContextExpired(err error) ErrContextExpired { - return ErrContextExpired{err} -} - -type ErrInternal struct { - err error -} - -func (e ErrInternal) Error() string { - return e.err.Error() -} - -func NewErrInternal(err error) ErrInternal { - return ErrInternal{err} -} diff --git a/entities/errors/errors_remote_client.go b/entities/errors/errors_remote_client.go deleted file mode 100644 index 60ebeea94ab6535fbeca88d937209a9c6803b754..0000000000000000000000000000000000000000 --- a/entities/errors/errors_remote_client.go +++ /dev/null @@ -1,64 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package errors - -import "fmt" - -type ErrOpenHttpRequest struct { - err error -} - -func (e ErrOpenHttpRequest) Error() string { - return e.err.Error() -} - -func NewErrOpenHttpRequest(err error) ErrOpenHttpRequest { - return ErrOpenHttpRequest{fmt.Errorf("open http request: %w", err)} -} - -type ErrSendHttpRequest struct { - err error -} - -func (e ErrSendHttpRequest) Error() string { - return e.err.Error() -} - -func NewErrSendHttpRequest(err error) ErrSendHttpRequest { - return ErrSendHttpRequest{fmt.Errorf("send http request: %w", err)} -} - -type ErrUnexpectedStatusCode struct { - err error -} - -func (e ErrUnexpectedStatusCode) Error() string { - return e.err.Error() -} - -func NewErrUnexpectedStatusCode(statusCode int, body []byte) ErrUnexpectedStatusCode { - return ErrUnexpectedStatusCode{ - err: fmt.Errorf("unexpected status code %d (%s)", statusCode, body), - } -} - -type ErrUnmarshalBody struct { - err error -} - -func (e ErrUnmarshalBody) Error() string { - return e.err.Error() -} - -func NewErrUnmarshalBody(err error) ErrUnmarshalBody { - return ErrUnmarshalBody{fmt.Errorf("unmarshal body: %w", err)} -} diff --git a/entities/filters/analtyics_props.go b/entities/filters/analtyics_props.go deleted file mode 100644 index 410b5dc317885e06f1b3981ef15bea0534348cb9..0000000000000000000000000000000000000000 --- a/entities/filters/analtyics_props.go +++ /dev/null @@ -1,20 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package filters - -// AnalyticsProps will be extracted from the graphql args of analytics -// functions (such as Meta and Aggregate). They tell the connectors whether -// to use an external analytics engine if such an engine is configured. -type AnalyticsProps struct { - UseAnalyticsEngine bool - ForceRecalculate bool -} diff --git a/entities/filters/consts.go b/entities/filters/consts.go deleted file mode 100644 index 62d7cc60ced27b2e8a817a80626fa6012f5c05b1..0000000000000000000000000000000000000000 --- a/entities/filters/consts.go +++ /dev/null @@ -1,27 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package filters - -const ( - InternalPropBackwardsCompatID = "id" - InternalPropID = "_id" - InternalNullIndex = "_nullState" - InternalPropertyLength = "_propertyLength" - InternalPropCreationTimeUnix = "_creationTimeUnix" - InternalPropLastUpdateTimeUnix = "_lastUpdateTimeUnix" -) - -// NotNullState is encoded as 0, so it can be read with the IsNull operator and value false. -const ( - InternalNotNullState = iota - InternalNullState -) diff --git a/entities/filters/cursor.go b/entities/filters/cursor.go deleted file mode 100644 index c01ddeedb1187889b71df3a68032fc325b63ba69..0000000000000000000000000000000000000000 --- a/entities/filters/cursor.go +++ /dev/null @@ -1,37 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package filters - -type Cursor struct { - After string `json:"after"` - Limit int `json:"limit"` -} - -// ExtractCursorFromArgs gets the limit key out of a map. Not specific to -// GQL, but can be used from GQL -func ExtractCursorFromArgs(args map[string]interface{}) (*Cursor, error) { - after, afterOk := args["after"] - - limit, limitOk := args["limit"] - if !limitOk || limit.(int) < 0 { - limit = LimitFlagNotSet - } - - if !afterOk && !limitOk || after == nil { - return nil, nil - } - - return &Cursor{ - After: after.(string), - Limit: limit.(int), - }, nil -} diff --git a/entities/filters/cursor_validator.go b/entities/filters/cursor_validator.go deleted file mode 100644 index 564e69fec55ed5cc38a87728f19a2f054d4763d7..0000000000000000000000000000000000000000 --- a/entities/filters/cursor_validator.go +++ /dev/null @@ -1,49 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package filters - -import ( - "fmt" - "strings" - - "github.com/google/uuid" - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/schema" -) - -func ValidateCursor(className schema.ClassName, cursor *Cursor, offset int, filters *LocalFilter, sort []Sort) error { - if className == "" { - return fmt.Errorf("class parameter cannot be empty") - } - if offset > 0 || filters != nil || sort != nil { - var params []string - if offset > 0 { - params = append(params, "offset") - } - if filters != nil { - params = append(params, "where") - } - if sort != nil { - params = append(params, "sort") - } - return fmt.Errorf("%s cannot be set with after and limit parameters", strings.Join(params, ",")) - } - if cursor.After != "" { - if _, err := uuid.Parse(cursor.After); err != nil { - return errors.Wrapf(err, "after parameter '%s' is not a valid uuid", cursor.After) - } - } - if cursor.Limit < 0 { - return fmt.Errorf("limit parameter must be set") - } - return nil -} diff --git a/entities/filters/filters.go b/entities/filters/filters.go deleted file mode 100644 index 4b489633e9876c07c9bf37f2a5be31fdcf2d9519..0000000000000000000000000000000000000000 --- a/entities/filters/filters.go +++ /dev/null @@ -1,144 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package filters - -import ( - "encoding/json" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" -) - -type Operator int - -const ( - OperatorEqual Operator = iota + 1 - OperatorNotEqual - OperatorGreaterThan - OperatorGreaterThanEqual - OperatorLessThan - OperatorLessThanEqual - OperatorAnd - OperatorOr - OperatorWithinGeoRange - OperatorLike - OperatorIsNull - ContainsAny - ContainsAll -) - -func (o Operator) OnValue() bool { - switch o { - case OperatorEqual, - OperatorNotEqual, - OperatorGreaterThan, - OperatorGreaterThanEqual, - OperatorLessThan, - OperatorLessThanEqual, - OperatorWithinGeoRange, - OperatorLike, - OperatorIsNull, - ContainsAny, - ContainsAll: - return true - default: - return false - } -} - -func (o Operator) Name() string { - switch o { - case OperatorEqual: - return "Equal" - case OperatorNotEqual: - return "NotEqual" - case OperatorGreaterThan: - return "GreaterThan" - case OperatorGreaterThanEqual: - return "GreaterThanEqual" - case OperatorLessThan: - return "LessThan" - case OperatorLessThanEqual: - return "LessThanEqual" - case OperatorAnd: - return "And" - case OperatorOr: - return "Or" - case OperatorWithinGeoRange: - return "WithinGeoRange" - case OperatorLike: - return "Like" - case OperatorIsNull: - return "IsNull" - case ContainsAny: - return "ContainsAny" - case ContainsAll: - return "ContainsAll" - default: - panic("Unknown operator") - } -} - -type LocalFilter struct { - Root *Clause `json:"root"` -} - -type Value struct { - Value interface{} `json:"value"` - Type schema.DataType `json:"type"` -} - -func (v *Value) UnmarshalJSON(data []byte) error { - type Alias Value - aux := struct { - *Alias - }{ - Alias: (*Alias)(v), - } - - err := json.Unmarshal(data, &aux) - if err != nil { - return err - } - - asFloat, ok := v.Value.(float64) - if v.Type == schema.DataTypeInt && ok { - v.Value = int(asFloat) - } - - if v.Type == schema.DataTypeGeoCoordinates { - temp := struct { - Value GeoRange `json:"value"` - }{} - - if err := json.Unmarshal(data, &temp); err != nil { - return err - } - v.Value = temp.Value - } - - return nil -} - -type Clause struct { - Operator Operator `json:"operator"` - On *Path `json:"on"` - Value *Value `json:"value"` - Operands []Clause `json:"operands"` -} - -// GeoRange to be used with fields of type GeoCoordinates. Identifies a point -// and a maximum distance from that point. -type GeoRange struct { - *models.GeoCoordinates - Distance float32 `json:"distance"` -} diff --git a/entities/filters/filters_serialization_test.go b/entities/filters/filters_serialization_test.go deleted file mode 100644 index afd3d3e7f1c22ee89f6063d6e6538d79bbff5baa..0000000000000000000000000000000000000000 --- a/entities/filters/filters_serialization_test.go +++ /dev/null @@ -1,82 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package filters - -import ( - "encoding/json" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" -) - -func TestSerializeValue(t *testing.T) { - t.Run("with a float value", func(t *testing.T) { - before := Value{ - Value: float64(3), - Type: schema.DataTypeNumber, - } - - bytes, err := json.Marshal(before) - require.Nil(t, err) - - var after Value - err = json.Unmarshal(bytes, &after) - require.Nil(t, err) - - assert.Equal(t, before, after) - }) - - t.Run("with an int value", func(t *testing.T) { - before := Value{ - Value: int(3), - Type: schema.DataTypeInt, - } - - bytes, err := json.Marshal(before) - require.Nil(t, err) - - var after Value - err = json.Unmarshal(bytes, &after) - require.Nil(t, err) - - assert.Equal(t, before, after) - }) - - t.Run("with a geo value", func(t *testing.T) { - before := Value{ - Value: GeoRange{ - GeoCoordinates: &models.GeoCoordinates{ - Latitude: ptFloat32(51.51), - Longitude: ptFloat32(-0.09), - }, - Distance: 2000, - }, - Type: schema.DataTypeGeoCoordinates, - } - - bytes, err := json.Marshal(before) - require.Nil(t, err) - - var after Value - err = json.Unmarshal(bytes, &after) - require.Nil(t, err) - - assert.Equal(t, before, after) - }) -} - -func ptFloat32(v float32) *float32 { - return &v -} diff --git a/entities/filters/filters_test.go b/entities/filters/filters_test.go deleted file mode 100644 index dc83590821ca5351d10a7e2ec3dce6113e5847c9..0000000000000000000000000000000000000000 --- a/entities/filters/filters_test.go +++ /dev/null @@ -1,46 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package filters - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestOperators(t *testing.T) { - type test struct { - op Operator - expectedName string - expectedOnValue bool - } - - tests := []test{ - {op: OperatorEqual, expectedName: "Equal", expectedOnValue: true}, - {op: OperatorNotEqual, expectedName: "NotEqual", expectedOnValue: true}, - {op: OperatorGreaterThan, expectedName: "GreaterThan", expectedOnValue: true}, - {op: OperatorGreaterThanEqual, expectedName: "GreaterThanEqual", expectedOnValue: true}, - {op: OperatorLessThanEqual, expectedName: "LessThanEqual", expectedOnValue: true}, - {op: OperatorLessThan, expectedName: "LessThan", expectedOnValue: true}, - {op: OperatorWithinGeoRange, expectedName: "WithinGeoRange", expectedOnValue: true}, - {op: OperatorLike, expectedName: "Like", expectedOnValue: true}, - {op: OperatorAnd, expectedName: "And", expectedOnValue: false}, - {op: OperatorOr, expectedName: "Or", expectedOnValue: false}, - } - - for _, test := range tests { - t.Run(test.expectedName, func(t *testing.T) { - assert.Equal(t, test.expectedName, test.op.Name(), "name must match") - assert.Equal(t, test.expectedOnValue, test.op.OnValue(), "onValue must match") - }) - } -} diff --git a/entities/filters/filters_validator.go b/entities/filters/filters_validator.go deleted file mode 100644 index fede954e98ea65efc48cb52252e9d6abc4978dfa..0000000000000000000000000000000000000000 --- a/entities/filters/filters_validator.go +++ /dev/null @@ -1,283 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package filters - -import ( - "fmt" - "strings" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/schema" -) - -// string and stringArray are deprecated as of v1.19 -// however they are allowed in filters and considered aliases -// for text and textArray -var deprecatedDataTypeAliases map[schema.DataType]schema.DataType = map[schema.DataType]schema.DataType{ - schema.DataTypeString: schema.DataTypeText, - schema.DataTypeStringArray: schema.DataTypeTextArray, -} - -func ValidateFilters(sch schema.Schema, filters *LocalFilter) error { - if filters == nil { - return errors.New("empty where") - } - cw := newClauseWrapper(filters.Root) - if err := validateClause(sch, cw); err != nil { - return err - } - cw.updateClause() - return nil -} - -func validateClause(sch schema.Schema, cw *clauseWrapper) error { - // check if nested - if cw.getOperands() != nil { - var errs []error - - for i, child := range cw.getOperands() { - if err := validateClause(sch, child); err != nil { - errs = append(errs, errors.Wrapf(err, "child operand at position %d", i)) - } - } - - if len(errs) > 0 { - return mergeErrs(errs) - } - return nil - } - - // validate current - - className := cw.getClassName() - propName := cw.getPropertyName() - - if IsInternalProperty(propName) { - return validateInternalPropertyClause(propName, cw) - } - - class := sch.FindClassByName(className) - if class == nil { - return errors.Errorf("class %q does not exist in schema", - className) - } - - propNameTyped := string(propName) - lengthPropName, isPropLengthFilter := schema.IsPropertyLength(propNameTyped, 0) - if isPropLengthFilter { - propName = schema.PropertyName(lengthPropName) - } - - prop, err := sch.GetProperty(className, propName) - if err != nil { - return err - } - - if cw.getOperator() == OperatorIsNull { - if !cw.isType(schema.DataTypeBoolean) { - return errors.Errorf("operator IsNull requires a booleanValue, got %q instead", - cw.getValueNameFromType()) - } - return nil - } - - if isPropLengthFilter { - if !cw.isType(schema.DataTypeInt) { - return errors.Errorf("Filtering for property length requires IntValue, got %q instead", - cw.getValueNameFromType()) - } - switch op := cw.getOperator(); op { - case OperatorEqual, OperatorNotEqual, OperatorGreaterThan, OperatorGreaterThanEqual, - OperatorLessThan, OperatorLessThanEqual: - // ok - default: - return errors.Errorf("Filtering for property length supports operators (not) equal and greater/less than (equal), got %q instead", - op) - } - if val := cw.getValue(); val.(int) < 0 { - return errors.Errorf("Can only filter for positive property length got %v instead", val) - } - return nil - } - - if isUUIDType(prop.DataType[0]) { - return validateUUIDType(propName, cw) - } - - if schema.IsRefDataType(prop.DataType) { - // bit of an edge case, directly on refs (i.e. not on a primitive prop of a - // ref) we only allow valueInt which is what's used to count references - if cw.isType(schema.DataTypeInt) { - return nil - } - return errors.Errorf("Property %q is a ref prop to the class %q. Only "+ - "\"valueInt\" can be used on a ref prop directly to count the number of refs. "+ - "Or did you mean to filter on a primitive prop of the referenced class? "+ - "In this case make sure your path contains 3 elements in the form of "+ - "[, , ]", - propName, prop.DataType[0]) - } else if baseType, ok := schema.IsArrayType(schema.DataType(prop.DataType[0])); ok { - if !cw.isType(baseType) { - return errors.Errorf("data type filter cannot use %q on type %q, use %q instead", - cw.getValueNameFromType(), - schema.DataType(prop.DataType[0]), - valueNameFromDataType(baseType)) - } - } else if !cw.isType(schema.DataType(prop.DataType[0])) { - return errors.Errorf("data type filter cannot use %q on type %q, use %q instead", - cw.getValueNameFromType(), - schema.DataType(prop.DataType[0]), - valueNameFromDataType(schema.DataType(prop.DataType[0]))) - } - - return nil -} - -func valueNameFromDataType(dt schema.DataType) string { - return "value" + strings.ToUpper(string(dt[0])) + string(dt[1:]) -} - -func mergeErrs(errs []error) error { - msgs := make([]string, len(errs)) - for i, err := range errs { - msgs[i] = err.Error() - } - - return errors.Errorf("%s", strings.Join(msgs, ", ")) -} - -func IsInternalProperty(propName schema.PropertyName) bool { - switch propName { - case InternalPropBackwardsCompatID, - InternalPropID, - InternalPropCreationTimeUnix, - InternalPropLastUpdateTimeUnix: - return true - default: - return false - } -} - -func validateInternalPropertyClause(propName schema.PropertyName, cw *clauseWrapper) error { - switch propName { - case InternalPropBackwardsCompatID, InternalPropID: - if cw.isType(schema.DataTypeText) { - return nil - } - return errors.Errorf( - `using ["_id"] to filter by uuid: must use "valueText" to specify the id`) - case InternalPropCreationTimeUnix, InternalPropLastUpdateTimeUnix: - if cw.isType(schema.DataTypeDate) || cw.isType(schema.DataTypeText) { - return nil - } - return errors.Errorf( - `using ["%s"] to filter by timestamp: must use "valueText" or "valueDate"`, propName) - default: - return errors.Errorf("unsupported internal property: %s", propName) - } -} - -func isUUIDType(dtString string) bool { - dt := schema.DataType(dtString) - return dt == schema.DataTypeUUID || dt == schema.DataTypeUUIDArray -} - -func validateUUIDType(propName schema.PropertyName, cw *clauseWrapper) error { - if cw.isType(schema.DataTypeText) { - return validateUUIDOperators(propName, cw) - } - - return fmt.Errorf("property %q is of type \"uuid\" or \"uuid[]\": "+ - "specify uuid as string using \"valueText\"", propName) -} - -func validateUUIDOperators(propName schema.PropertyName, cw *clauseWrapper) error { - op := cw.getOperator() - - switch op { - case OperatorEqual, OperatorNotEqual, OperatorLessThan, OperatorLessThanEqual, - OperatorGreaterThan, OperatorGreaterThanEqual, ContainsAll, ContainsAny: - return nil - default: - return fmt.Errorf("operator %q cannot be used on uuid/uuid[] props", op.Name()) - } -} - -type clauseWrapper struct { - clause *Clause - origType schema.DataType - aliasType schema.DataType - operands []*clauseWrapper -} - -func newClauseWrapper(clause *Clause) *clauseWrapper { - w := &clauseWrapper{clause: clause} - if clause.Operands != nil { - w.operands = make([]*clauseWrapper, len(clause.Operands)) - for i := range clause.Operands { - w.operands[i] = newClauseWrapper(&clause.Operands[i]) - } - } else { - w.origType = clause.Value.Type - w.aliasType = deprecatedDataTypeAliases[clause.Value.Type] - } - return w -} - -func (w *clauseWrapper) isType(dt schema.DataType) bool { - if w.operands != nil { - return false - } - return dt == w.origType || (dt == w.aliasType && w.aliasType != "") -} - -func (w *clauseWrapper) getValueNameFromType() string { - return valueNameFromDataType(w.origType) -} - -func (w *clauseWrapper) getOperands() []*clauseWrapper { - return w.operands -} - -func (w *clauseWrapper) getOperator() Operator { - return w.clause.Operator -} - -func (w *clauseWrapper) getValue() interface{} { - return w.clause.Value.Value -} - -func (w *clauseWrapper) getClassName() schema.ClassName { - if w.operands != nil { - return "" - } - return w.clause.On.GetInnerMost().Class -} - -func (w *clauseWrapper) getPropertyName() schema.PropertyName { - if w.operands != nil { - return "" - } - return w.clause.On.GetInnerMost().Property -} - -func (w *clauseWrapper) updateClause() { - if w.operands != nil { - for i := range w.operands { - w.operands[i].updateClause() - } - } else { - if w.aliasType != "" { - w.clause.Value.Type = w.aliasType - } - } -} diff --git a/entities/filters/filters_validator_test.go b/entities/filters/filters_validator_test.go deleted file mode 100644 index 9b3811593f7d0c7ccc22ecf6384acac2373e13af..0000000000000000000000000000000000000000 --- a/entities/filters/filters_validator_test.go +++ /dev/null @@ -1,308 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package filters - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" -) - -func TestValidateIsNullOperator(t *testing.T) { - tests := []struct { - name string - schemaType schema.DataType - valid bool - }{ - { - name: "Valid datatype", - schemaType: schema.DataTypeBoolean, - valid: true, - }, - { - name: "Invalid datatype (array)", - schemaType: schema.DataTypeBooleanArray, - valid: false, - }, - { - name: "Invalid datatype (text)", - schemaType: schema.DataTypeText, - valid: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - sch := schema.Schema{Objects: &models.Schema{ - Classes: []*models.Class{ - { - Class: "Car", - Properties: []*models.Property{ - {Name: "modelName", DataType: schema.DataTypeText.PropString(), Tokenization: models.PropertyTokenizationWhitespace}, - {Name: "manufacturerName", DataType: schema.DataTypeText.PropString(), Tokenization: models.PropertyTokenizationWhitespace}, - {Name: "horsepower", DataType: []string{"int"}}, - }, - }, - }, - }} - cl := Clause{ - Operator: OperatorIsNull, - Value: &Value{Value: true, Type: tt.schemaType}, - On: &Path{Class: "Car", Property: "horsepower"}, - } - err := validateClause(sch, newClauseWrapper(&cl)) - if tt.valid { - require.Nil(t, err) - } else { - require.NotNil(t, err) - } - }) - } -} - -func TestValidatePropertyLength(t *testing.T) { - tests := []struct { - name string - schemaType schema.DataType - valid bool - operator Operator - value int - }{ - { - name: "Valid datatype and operator", - schemaType: schema.DataTypeInt, - valid: true, - operator: OperatorEqual, - value: 0, - }, - { - name: "Invalid datatype (array)", - schemaType: schema.DataTypeBooleanArray, - valid: false, - operator: OperatorEqual, - value: 1, - }, - { - name: "Invalid datatype (text)", - schemaType: schema.DataTypeText, - valid: false, - operator: OperatorEqual, - value: 2, - }, - { - name: "Invalid operator (Or)", - schemaType: schema.DataTypeText, - valid: false, - operator: OperatorOr, - value: 10, - }, - { - name: "Invalid value (negative)", - schemaType: schema.DataTypeText, - valid: false, - operator: OperatorEqual, - value: -5, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - sch := schema.Schema{Objects: &models.Schema{ - Classes: []*models.Class{ - { - Class: "Car", - Properties: []*models.Property{ - {Name: "horsepower", DataType: []string{"int"}}, - }, - }, - }, - }} - cl := Clause{ - Operator: OperatorEqual, - Value: &Value{Value: tt.value, Type: tt.schemaType}, - On: &Path{Class: "Car", Property: "len(horsepower)"}, - } - err := validateClause(sch, newClauseWrapper(&cl)) - if tt.valid { - require.Nil(t, err) - } else { - require.NotNil(t, err) - } - }) - } -} - -func TestValidateUUIDFilter(t *testing.T) { - tests := []struct { - name string - schemaType schema.DataType - valid bool - operator Operator - value int - }{ - { - name: "Valid datatype and operator", - schemaType: schema.DataTypeText, - valid: true, - operator: OperatorEqual, - value: 0, - }, - { - name: "Wrong data type (int)", - schemaType: schema.DataTypeInt, - valid: false, - operator: OperatorEqual, - value: 0, - }, - { - name: "Wrong operator (Like)", - schemaType: schema.DataTypeText, - valid: false, - operator: OperatorLike, - value: 0, - }, - - { - name: "[deprecated string] Valid datatype and operator", - schemaType: schema.DataTypeString, - valid: true, - operator: OperatorEqual, - value: 0, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - sch := schema.Schema{Objects: &models.Schema{ - Classes: []*models.Class{ - { - Class: "Car", - Properties: []*models.Property{ - {Name: "my_id", DataType: []string{string(schema.DataTypeUUID)}}, - {Name: "my_idz", DataType: []string{string(schema.DataTypeUUIDArray)}}, - }, - }, - }, - }} - for _, prop := range []schema.PropertyName{"my_id", "my_idz"} { - cl := Clause{ - Operator: tt.operator, - Value: &Value{Value: tt.value, Type: tt.schemaType}, - On: &Path{Class: "Car", Property: prop}, - } - err := validateClause(sch, newClauseWrapper(&cl)) - if tt.valid { - require.Nil(t, err) - } else { - require.NotNil(t, err) - } - } - }) - } -} - -func TestClauseWrapper(t *testing.T) { - type testCase struct { - name string - valueType schema.DataType - requiredType schema.DataType - - expectedValid bool - expectedValueName string - } - - testCases := []testCase{ - { - name: "string accepted where text is required", - valueType: schema.DataTypeString, - requiredType: schema.DataTypeText, - expectedValid: true, - expectedValueName: "valueString", - }, - { - name: "text accepted where text is required", - valueType: schema.DataTypeText, - requiredType: schema.DataTypeText, - expectedValid: true, - expectedValueName: "valueText", - }, - { - name: "string[] accepted where text[] is required", - valueType: schema.DataTypeStringArray, - requiredType: schema.DataTypeTextArray, - expectedValid: true, - expectedValueName: "valueString[]", - }, - { - name: "text[] accepted where text[] is required", - valueType: schema.DataTypeTextArray, - requiredType: schema.DataTypeTextArray, - expectedValid: true, - expectedValueName: "valueText[]", - }, - { - name: "text not accepted where string is required", - valueType: schema.DataTypeText, - requiredType: schema.DataTypeString, - expectedValid: false, - expectedValueName: "valueText", - }, - { - name: "text[] not accepted where string[] is required", - valueType: schema.DataTypeTextArray, - requiredType: schema.DataTypeStringArray, - expectedValid: false, - expectedValueName: "valueText[]", - }, - { - name: "int not accepted where boolean is required", - valueType: schema.DataTypeInt, - requiredType: schema.DataTypeBoolean, - expectedValid: false, - expectedValueName: "valueInt", - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - clause := Clause{ - Operator: OperatorEqual, - Value: &Value{Value: "someValue", Type: tc.valueType}, - On: &Path{Class: "SomeClass", Property: "someProperty"}, - } - - cw := newClauseWrapper(&clause) - - assert.Equal(t, tc.expectedValid, cw.isType(tc.requiredType)) - assert.Equal(t, tc.expectedValueName, cw.getValueNameFromType()) - - assert.Equal(t, "someValue", cw.getValue()) - assert.Equal(t, schema.ClassName("SomeClass"), cw.getClassName()) - assert.Equal(t, schema.PropertyName("someProperty"), cw.getPropertyName()) - assert.Equal(t, OperatorEqual, cw.getOperator()) - assert.Nil(t, cw.getOperands()) - - t.Run("clause is updated to required type if valid", func(t *testing.T) { - cw.updateClause() - - if tc.expectedValid { - assert.Equal(t, tc.requiredType, clause.Value.Type) - } else { - assert.Equal(t, tc.valueType, clause.Value.Type) - } - }) - }) - } -} diff --git a/entities/filters/pagination.go b/entities/filters/pagination.go deleted file mode 100644 index bca599a2d1dd4a37fb6efbef7e6fa64d314d47cb..0000000000000000000000000000000000000000 --- a/entities/filters/pagination.go +++ /dev/null @@ -1,58 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package filters - -const ( - // LimitFlagSearchByDist indicates that the - // vector search should be conducted by - // distance, without limit - LimitFlagSearchByDist int = iota - 2 - - // LimitFlagNotSet indicates that no limit - // was provided by the client - LimitFlagNotSet -) - -type Pagination struct { - Offset int - Limit int - Autocut int -} - -// ExtractPaginationFromArgs gets the limit key out of a map. Not specific to -// GQL, but can be used from GQL -func ExtractPaginationFromArgs(args map[string]interface{}) (*Pagination, error) { - offset, offsetOk := args["offset"] - if !offsetOk { - offset = 0 - } - - limit, limitOk := args["limit"] - if !limitOk || limit.(int) < 0 { - limit = LimitFlagNotSet - } - - autocut, autocutOk := args["autocut"] - if !autocutOk { - autocut = 0 // disabled - } - - if !offsetOk && !limitOk && !autocutOk { - return nil, nil - } - - return &Pagination{ - Offset: offset.(int), - Limit: limit.(int), - Autocut: autocut.(int), - }, nil -} diff --git a/entities/filters/pagination_test.go b/entities/filters/pagination_test.go deleted file mode 100644 index 9d05cac6dae5f10abfa656631d8ba3aa472f2a34..0000000000000000000000000000000000000000 --- a/entities/filters/pagination_test.go +++ /dev/null @@ -1,58 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package filters - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestExtractPagination(t *testing.T) { - t.Run("without a limit present", func(t *testing.T) { - p, err := ExtractPaginationFromArgs(map[string]interface{}{}) - require.Nil(t, err) - assert.Nil(t, p) - }) - - t.Run("with a limit present", func(t *testing.T) { - p, err := ExtractPaginationFromArgs(map[string]interface{}{ - "limit": 25, - }) - require.Nil(t, err) - require.NotNil(t, p) - assert.Equal(t, 0, p.Offset) - assert.Equal(t, 25, p.Limit) - }) - - t.Run("with a offset present", func(t *testing.T) { - p, err := ExtractPaginationFromArgs(map[string]interface{}{ - "offset": 11, - }) - require.Nil(t, err) - require.NotNil(t, p) - assert.Equal(t, 11, p.Offset) - assert.Equal(t, -1, p.Limit) - }) - - t.Run("with offset and limit present", func(t *testing.T) { - p, err := ExtractPaginationFromArgs(map[string]interface{}{ - "offset": 11, - "limit": 25, - }) - require.Nil(t, err) - require.NotNil(t, p) - assert.Equal(t, 11, p.Offset) - assert.Equal(t, 25, p.Limit) - }) -} diff --git a/entities/filters/path.go b/entities/filters/path.go deleted file mode 100644 index a50ef11f7f49f1e63d0a849d4417b1e807d891f5..0000000000000000000000000000000000000000 --- a/entities/filters/path.go +++ /dev/null @@ -1,151 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package filters - -import ( - "fmt" - "strings" - - "github.com/weaviate/weaviate/entities/schema" -) - -// Represents the path in a filter. -// Either RelationProperty or PrimitiveProperty must be empty (e.g. ""). -type Path struct { - Class schema.ClassName `json:"class"` - Property schema.PropertyName `json:"property"` - - // If nil, then this is the property we're interested in. - // If a pointer to another Path, the constraint applies to that one. - Child *Path `json:"child"` -} - -// GetInnerMost recursively searches for child paths, only when no more -// children can be found will the path be returned -func (p *Path) GetInnerMost() *Path { - if p.Child == nil { - return p - } - - return p.Child.GetInnerMost() -} - -// Slice flattens the nested path into a slice of segments -func (p *Path) Slice() []string { - return appendNestedPath(p, true) -} - -func (p *Path) SliceInterface() []interface{} { - path := appendNestedPath(p, true) - out := make([]interface{}, len(path)) - for i, element := range path { - out[i] = element - } - return out -} - -// TODO: This is now identical with Slice(), so it can be removed once all -// callers have been adopted -func (p *Path) SliceNonTitleized() []string { - return appendNestedPath(p, true) -} - -func appendNestedPath(p *Path, omitClass bool) []string { - result := []string{} - if !omitClass { - result = append(result, string(p.Class)) - } - - if p.Child != nil { - property := string(p.Property) - result = append(result, property) - result = append(result, appendNestedPath(p.Child, false)...) - } else { - result = append(result, string(p.Property)) - } - - return result -} - -// ParsePath Parses the path -// It parses an array of strings in this format -// [0] ClassName -> The root class name we're drilling down from -// [1] propertyName -> The property name we're interested in. -func ParsePath(pathElements []interface{}, rootClass string) (*Path, error) { - // we need to manually insert the root class, as that is omitted from the user - pathElements = append([]interface{}{rootClass}, pathElements...) - - // The sentinel is used to bootstrap the inlined recursion. - // we return sentinel.Child at the end. - var sentinel Path - - // Keep track of where we are in the path (e.g. always points to latest Path segment) - current := &sentinel - - // Now go through the path elements, step over it in increments of two. - // Simple case: ClassName -> property - // Nested path case: ClassName -> HasRef -> ClassOfRef -> Property - for i := 0; i < len(pathElements); i += 2 { - lengthRemaining := len(pathElements) - i - if lengthRemaining < 2 { - return nil, fmt.Errorf("missing an argument after '%s'", pathElements[i]) - } - - rawClassName, ok := pathElements[i].(string) - if !ok { - return nil, fmt.Errorf("element %v is not a string", i+1) - } - - rawPropertyName, ok := pathElements[i+1].(string) - if !ok { - return nil, fmt.Errorf("element %v is not a string", i+2) - } - - className, err := schema.ValidateClassName(rawClassName) - if err != nil { - return nil, fmt.Errorf("Expected a valid class name in 'path' field for the filter but got '%s'", rawClassName) - } - - var propertyName schema.PropertyName - lengthPropName, isPropLengthFilter := schema.IsPropertyLength(rawPropertyName, 0) - if isPropLengthFilter { - // check if property in len(PROPERTY) is valid - _, err = schema.ValidatePropertyName(lengthPropName) - if err != nil { - return nil, fmt.Errorf("Expected a valid property name in 'path' field for the filter, but got '%s'", lengthPropName) - } - propertyName = schema.PropertyName(rawPropertyName) - } else { - propertyName, err = schema.ValidatePropertyName(rawPropertyName) - // Invalid property name? - // Try to parse it as as a reference or a length. - if err != nil { - untitlizedPropertyName := strings.ToLower(rawPropertyName[0:1]) + rawPropertyName[1:] - propertyName, err = schema.ValidatePropertyName(untitlizedPropertyName) - if err != nil { - return nil, fmt.Errorf("Expected a valid property name in 'path' field for the filter, but got '%s'", rawPropertyName) - } - } - - } - - current.Child = &Path{ - Class: className, - Property: propertyName, - } - - // And down we go. - current = current.Child - } - - return sentinel.Child, nil -} diff --git a/entities/filters/path_test.go b/entities/filters/path_test.go deleted file mode 100644 index a2e13cb33486f7890486b613e04f89eb3351b22a..0000000000000000000000000000000000000000 --- a/entities/filters/path_test.go +++ /dev/null @@ -1,146 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package filters - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func Test_ParsePath(t *testing.T) { - t.Run("with a primitive prop", func(t *testing.T) { - rootClass := "City" - segments := []interface{}{"population"} - expectedPath := &Path{ - Class: "City", - Property: "population", - } - - path, err := ParsePath(segments, rootClass) - - require.Nil(t, err, "should not error") - assert.Equal(t, expectedPath, path, "should parse the path correctly") - }) - - t.Run("with len prop", func(t *testing.T) { - rootClass := "City" - segments := []interface{}{"len(population)"} - expectedPath := &Path{ - Class: "City", - Property: "len(population)", - } - - path, err := ParsePath(segments, rootClass) - - require.Nil(t, err, "should not error") - assert.Equal(t, expectedPath, path, "should parse the path correctly") - }) - - t.Run("with nested refs", func(t *testing.T) { - rootClass := "City" - segments := []interface{}{"inCountry", "Country", "inContinent", "Continent", "onPlanet", "Planet", "name"} - expectedPath := &Path{ - Class: "City", - Property: "inCountry", - Child: &Path{ - Class: "Country", - Property: "inContinent", - Child: &Path{ - Class: "Continent", - Property: "onPlanet", - Child: &Path{ - Class: "Planet", - Property: "name", - }, - }, - }, - } - - path, err := ParsePath(segments, rootClass) - - require.Nil(t, err, "should not error") - assert.Equal(t, expectedPath, path, "should parse the path correctly") - - // Extract innermost path element - innerMost := path.GetInnerMost() - assert.Equal(t, innerMost, &Path{Class: "Planet", Property: "name"}) - - // Print Slice - }) - - t.Run("with non-valid prop", func(t *testing.T) { - rootClass := "City" - segments := []interface{}{"populatS356()ion"} - _, err := ParsePath(segments, rootClass) - require.NotNil(t, err, "should error") - }) - - t.Run("with non-valid len prop", func(t *testing.T) { - rootClass := "City" - segments := []interface{}{"len(populatS356()ion)"} - _, err := ParsePath(segments, rootClass) - require.NotNil(t, err, "should error") - }) -} - -func Test_SlicePath(t *testing.T) { - t.Run("with a primitive prop", func(t *testing.T) { - path := &Path{ - Class: "City", - Property: "population", - } - expectedSegments := []interface{}{"population"} - - segments := path.SliceInterface() - - assert.Equal(t, expectedSegments, segments, "should slice the path correctly") - }) - - t.Run("with nested refs", func(t *testing.T) { - path := &Path{ - Class: "City", - Property: "inCountry", - Child: &Path{ - Class: "Country", - Property: "inContinent", - Child: &Path{ - Class: "Continent", - Property: "onPlanet", - Child: &Path{ - Class: "Planet", - Property: "name", - }, - }, - }, - } - - t.Run("as []interface{}", func(t *testing.T) { - expectedSegments := []interface{}{"inCountry", "Country", "inContinent", "Continent", "onPlanet", "Planet", "name"} - segments := path.SliceInterface() - assert.Equal(t, expectedSegments, segments, "should slice the path correctly") - }) - - t.Run("as []string titleized", func(t *testing.T) { - expectedSegments := []string{"inCountry", "Country", "inContinent", "Continent", "onPlanet", "Planet", "name"} - segments := path.Slice() - assert.Equal(t, expectedSegments, segments, "should slice the path correctly") - }) - - t.Run("as []string non-titleized", func(t *testing.T) { - expectedSegments := []string{"inCountry", "Country", "inContinent", "Continent", "onPlanet", "Planet", "name"} - segments := path.SliceNonTitleized() - assert.Equal(t, expectedSegments, segments, "should slice the path correctly") - }) - }) -} diff --git a/entities/filters/sort.go b/entities/filters/sort.go deleted file mode 100644 index 0b4bdca8b6ceef585d0ee1c89081a4e2f552aed8..0000000000000000000000000000000000000000 --- a/entities/filters/sort.go +++ /dev/null @@ -1,45 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package filters - -// Sort contains path and order (asc, desc) information -type Sort struct { - Path []string `json:"path"` - Order string `json:"order"` -} - -// ExtractSortFromArgs gets the sort parameters -func ExtractSortFromArgs(in []interface{}) []Sort { - var args []Sort - - for i := range in { - sortFilter, ok := in[i].(map[string]interface{}) - if ok { - var path []string - pathParam, ok := sortFilter["path"].([]interface{}) - if ok { - path = make([]string, len(pathParam)) - for i, value := range pathParam { - path[i] = value.(string) - } - } - var order string - orderParam, ok := sortFilter["order"] - if ok { - order = orderParam.(string) - } - args = append(args, Sort{path, order}) - } - } - - return args -} diff --git a/entities/filters/sort_validator.go b/entities/filters/sort_validator.go deleted file mode 100644 index 218913142d046ba6675a4c6a91bad7e8f1403da8..0000000000000000000000000000000000000000 --- a/entities/filters/sort_validator.go +++ /dev/null @@ -1,83 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package filters - -import ( - "fmt" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/schema" -) - -func ValidateSort(sch schema.Schema, className schema.ClassName, sort []Sort) error { - if len(sort) == 0 { - return errors.New("empty sort") - } - - var errs []error - for i := range sort { - if err := validateSortClause(sch, className, sort[i]); err != nil { - errs = append(errs, errors.Wrapf(err, "sort parameter at position %d", i)) - } - } - - if len(errs) > 0 { - return mergeErrs(errs) - } else { - return nil - } -} - -func validateSortClause(sch schema.Schema, className schema.ClassName, sort Sort) error { - // validate current - path, order := sort.Path, sort.Order - - if len(order) > 0 && order != "asc" && order != "desc" { - return errors.Errorf(`invalid order parameter, `+ - `possible values are: ["asc", "desc"] not: "%s"`, order) - } - - switch len(path) { - case 0: - return errors.New("path parameter cannot be empty") - case 1: - class := sch.FindClassByName(className) - if class == nil { - return errors.Errorf("class %q does not exist in schema", - className) - } - propName := schema.PropertyName(path[0]) - if IsInternalProperty(propName) { - // handle internal properties - return nil - } - prop, err := sch.GetProperty(className, propName) - if err != nil { - return err - } - - if isUUIDType(prop.DataType[0]) { - return fmt.Errorf("prop %q is of type uuid/uuid[]: "+ - "sorting by uuid is currently not supported - if you believe it should be, "+ - "please open a feature request on github.com/weaviate/weaviate", prop.Name) - } - - if schema.IsRefDataType(prop.DataType) { - return errors.Errorf("sorting by reference not supported, "+ - "property %q is a ref prop to the class %q", propName, prop.DataType[0]) - } - return nil - default: - return errors.New("sorting by reference not supported, " + - "path must have exactly one argument") - } -} diff --git a/entities/filters/sort_validator_test.go b/entities/filters/sort_validator_test.go deleted file mode 100644 index 6de0ca58a01e27a11dfee591920f2690b7f36dce..0000000000000000000000000000000000000000 --- a/entities/filters/sort_validator_test.go +++ /dev/null @@ -1,85 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package filters - -import ( - "testing" - - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" -) - -func TestSortValidation(t *testing.T) { - tests := []struct { - name string - prop string - valid bool - }{ - { - name: "existing prop - string", - valid: true, - prop: "modelName", - }, - { - name: "existing prop - int", - valid: true, - prop: "horsepower", - }, - { - name: "invalid prop", - valid: false, - prop: "idontexist", - }, - { - name: "uuid prop", - valid: false, - prop: "my_id", - }, - { - name: "uuid[] prop", - valid: false, - prop: "my_idz", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - sch := schema.Schema{Objects: &models.Schema{ - Classes: []*models.Class{ - { - Class: "Car", - Properties: []*models.Property{ - {Name: "modelName", DataType: schema.DataTypeText.PropString(), Tokenization: models.PropertyTokenizationWhitespace}, - {Name: "manufacturerName", DataType: schema.DataTypeText.PropString(), Tokenization: models.PropertyTokenizationWhitespace}, - {Name: "horsepower", DataType: []string{"int"}}, - {Name: "my_id", DataType: []string{"uuid"}}, - {Name: "my_idz", DataType: []string{"uuid[]"}}, - }, - }, - }, - }} - - sort := []Sort{{ - Path: []string{tt.prop}, - Order: "asc", - }} - - err := ValidateSort(sch, schema.ClassName("Car"), sort) - if tt.valid { - require.Nil(t, err) - } else { - require.NotNil(t, err) - } - }) - } -} diff --git a/entities/interval/backoff.go b/entities/interval/backoff.go deleted file mode 100644 index 654dc977b9b1a5dcbca4ef6ca554add22c4c9584..0000000000000000000000000000000000000000 --- a/entities/interval/backoff.go +++ /dev/null @@ -1,78 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package interval - -import ( - "sort" - "time" -) - -var defaultBackoffs = []time.Duration{ - time.Duration(0), - 30 * time.Second, - 2 * time.Minute, - 10 * time.Minute, - 1 * time.Hour, - 12 * time.Hour, -} - -// BackoffTimer tracks a given range of intervals with increasing duration -type BackoffTimer struct { - backoffLevel int - backoffs []time.Duration - lastInterval time.Time -} - -// NewBackoffTimer constructs and returns a *BackoffTimer instance -// If no backoffs are provided, defaultBackoffs is used. When the -// last backoff duration has elapsed, the timer will use the final -// duration for the remainder of the BackoffTimer's lifetime -func NewBackoffTimer(backoffs ...time.Duration) *BackoffTimer { - boff := &BackoffTimer{backoffs: backoffs} - if len(backoffs) == 0 { - boff.backoffs = defaultBackoffs - } else { - sort.Slice(backoffs, func(i, j int) bool { - return backoffs[i] < backoffs[j] - }) - } - return boff -} - -// IncreaseInterval bumps the duration of the interval up to the next given value -func (b *BackoffTimer) IncreaseInterval() { - b.lastInterval = time.Now() - if b.backoffLevel < len(b.backoffs) { - b.backoffLevel += 1 - } -} - -// IntervalElapsed returns if the current interval has elapsed -func (b *BackoffTimer) IntervalElapsed() bool { - return time.Since(b.lastInterval) > b.calculateInterval() -} - -// Reset returns BackoffTimer to its original empty state -func (b *BackoffTimer) Reset() { - b.lastInterval = time.Time{} - b.backoffLevel = 0 -} - -func (b *BackoffTimer) calculateInterval() time.Duration { - if b.backoffLevel >= len(b.backoffs) { - return b.backoffs[len(b.backoffs)-1] - } - - interval := b.backoffs[b.backoffLevel] - - return interval -} diff --git a/entities/interval/backoff_test.go b/entities/interval/backoff_test.go deleted file mode 100644 index df6c5d2c2f5a8a663c18eeb161cf6065bb81874a..0000000000000000000000000000000000000000 --- a/entities/interval/backoff_test.go +++ /dev/null @@ -1,78 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package interval - -import ( - "sort" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func TestBackoffInterval(t *testing.T) { - t.Run("with default backoffs", func(t *testing.T) { - boff := NewBackoffTimer() - - assert.Equal(t, boff.backoffs, defaultBackoffs) - assert.Zero(t, boff.backoffLevel) - assert.Zero(t, boff.lastInterval) - assert.Equal(t, time.Duration(0), boff.calculateInterval()) - assert.True(t, boff.IntervalElapsed()) - - i := 1 - for ; i < len(defaultBackoffs); i++ { - boff.IncreaseInterval() - assert.False(t, boff.IntervalElapsed()) - assert.Equal(t, i, boff.backoffLevel) - assert.Equal(t, defaultBackoffs[i], boff.calculateInterval()) - } - - boff.IncreaseInterval() - assert.False(t, boff.IntervalElapsed()) - assert.Equal(t, i, boff.backoffLevel) - assert.Equal(t, defaultBackoffs[len(defaultBackoffs)-1], boff.calculateInterval()) - }) - - t.Run("with custom backoffs", func(t *testing.T) { - var ( - durations = []time.Duration{time.Second, time.Nanosecond, time.Millisecond} - sorted = make([]time.Duration, len(durations)) - ) - - copy(sorted, durations) - sort.Slice(sorted, func(i, j int) bool { - return sorted[i] < sorted[j] - }) - - boff := NewBackoffTimer(durations...) - assert.Equal(t, boff.backoffs, sorted) - assert.True(t, boff.IntervalElapsed()) - assert.Equal(t, sorted[0], boff.calculateInterval()) - - boff.IncreaseInterval() - time.Sleep(time.Millisecond) - assert.True(t, boff.IntervalElapsed()) - assert.Equal(t, sorted[1], boff.calculateInterval()) - - boff.IncreaseInterval() - assert.False(t, boff.IntervalElapsed()) - time.Sleep(time.Second) - assert.True(t, boff.IntervalElapsed()) - assert.Equal(t, sorted[2], boff.calculateInterval()) - - boff.IncreaseInterval() - assert.False(t, boff.IntervalElapsed()) - assert.False(t, boff.IntervalElapsed()) - assert.Equal(t, sorted[len(sorted)-1], boff.calculateInterval()) - }) -} diff --git a/entities/inverted/errors.go b/entities/inverted/errors.go deleted file mode 100644 index bd483bed02f1aa1823a0a7f7bb0d338b74352fe8..0000000000000000000000000000000000000000 --- a/entities/inverted/errors.go +++ /dev/null @@ -1,47 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package inverted - -import "fmt" - -type MissingIndexError struct { - format string - args []any -} - -func NewMissingFilterableIndexError(propName string) error { - return MissingIndexError{missingFilterableFormat, []any{propName, propName}} -} - -func NewMissingSearchableIndexError(propName string) error { - return MissingIndexError{missingSearchableFormat, []any{propName, propName}} -} - -func NewMissingFilterableMetaCountIndexError(propName string) error { - return MissingIndexError{missingFilterableMetaCountFormat, []any{propName, propName}} -} - -func (e MissingIndexError) Error() string { - return fmt.Sprintf(e.format, e.args...) -} - -const ( - missingFilterableFormat = "Filtering by property '%s' requires inverted index. " + - "Is `indexFilterable` option of property '%s' enabled? " + - "Set it to `true` or leave empty" - missingSearchableFormat = "Searching by property '%s' requires inverted index. " + - "Is `indexSearchable` option of property '%s' enabled? " + - "Set it to `true` or leave empty" - missingFilterableMetaCountFormat = "Searching by property '%s' count requires inverted index. " + - "Is `indexFilterable` option of property '%s' enabled? " + - "Set it to `true` or leave empty" -) diff --git a/entities/lsmkv/errors.go b/entities/lsmkv/errors.go deleted file mode 100644 index 403e6fe87040d4aff2ba773f78b2a94fce1b5a6a..0000000000000000000000000000000000000000 --- a/entities/lsmkv/errors.go +++ /dev/null @@ -1,21 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lsmkv - -import ( - "errors" -) - -var ( - NotFound = errors.New("not found") - Deleted = errors.New("deleted") -) diff --git a/entities/models/additional_properties.go b/entities/models/additional_properties.go deleted file mode 100644 index 27b17fd35f5aa323f763c67be1c966b04633ab2d..0000000000000000000000000000000000000000 --- a/entities/models/additional_properties.go +++ /dev/null @@ -1,38 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - - "github.com/go-openapi/strfmt" -) - -// AdditionalProperties Additional Meta information about a single object object. -// -// swagger:model AdditionalProperties -type AdditionalProperties map[string]interface{} - -// Validate validates this additional properties -func (m AdditionalProperties) Validate(formats strfmt.Registry) error { - return nil -} - -// ContextValidate validates this additional properties based on context it is used -func (m AdditionalProperties) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - return nil -} diff --git a/entities/models/b_m25_config.go b/entities/models/b_m25_config.go deleted file mode 100644 index 8e8ab658354401399bc94a6d9d641082f03b27d0..0000000000000000000000000000000000000000 --- a/entities/models/b_m25_config.go +++ /dev/null @@ -1,64 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" -) - -// BM25Config tuning parameters for the BM25 algorithm -// -// swagger:model BM25Config -type BM25Config struct { - - // calibrates term-weight scaling based on the document length - B float32 `json:"b,omitempty"` - - // calibrates term-weight scaling based on the term frequency within a document - K1 float32 `json:"k1,omitempty"` -} - -// Validate validates this b m25 config -func (m *BM25Config) Validate(formats strfmt.Registry) error { - return nil -} - -// ContextValidate validates this b m25 config based on context it is used -func (m *BM25Config) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - return nil -} - -// MarshalBinary interface implementation -func (m *BM25Config) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *BM25Config) UnmarshalBinary(b []byte) error { - var res BM25Config - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/backup_config.go b/entities/models/backup_config.go deleted file mode 100644 index aebb046868e06a6433e336474499b2c1e5a31b80..0000000000000000000000000000000000000000 --- a/entities/models/backup_config.go +++ /dev/null @@ -1,169 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "encoding/json" - - "github.com/go-openapi/errors" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" - "github.com/go-openapi/validate" -) - -// BackupConfig Backup custom configuration -// -// swagger:model BackupConfig -type BackupConfig struct { - - // Desired CPU core utilization ranging from 1%-80% - // Maximum: 80 - // Minimum: 1 - CPUPercentage int64 `json:"CPUPercentage,omitempty"` - - // Weaviate will attempt to come close the specified size, with a minimum of 2MB, default of 128MB, and a maximum of 512MB - // Maximum: 512 - // Minimum: 2 - ChunkSize int64 `json:"ChunkSize,omitempty"` - - // compression level used by compression algorithm - // Enum: [DefaultCompression BestSpeed BestCompression] - CompressionLevel string `json:"CompressionLevel,omitempty"` -} - -// Validate validates this backup config -func (m *BackupConfig) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateCPUPercentage(formats); err != nil { - res = append(res, err) - } - - if err := m.validateChunkSize(formats); err != nil { - res = append(res, err) - } - - if err := m.validateCompressionLevel(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *BackupConfig) validateCPUPercentage(formats strfmt.Registry) error { - if swag.IsZero(m.CPUPercentage) { // not required - return nil - } - - if err := validate.MinimumInt("CPUPercentage", "body", m.CPUPercentage, 1, false); err != nil { - return err - } - - if err := validate.MaximumInt("CPUPercentage", "body", m.CPUPercentage, 80, false); err != nil { - return err - } - - return nil -} - -func (m *BackupConfig) validateChunkSize(formats strfmt.Registry) error { - if swag.IsZero(m.ChunkSize) { // not required - return nil - } - - if err := validate.MinimumInt("ChunkSize", "body", m.ChunkSize, 2, false); err != nil { - return err - } - - if err := validate.MaximumInt("ChunkSize", "body", m.ChunkSize, 512, false); err != nil { - return err - } - - return nil -} - -var backupConfigTypeCompressionLevelPropEnum []interface{} - -func init() { - var res []string - if err := json.Unmarshal([]byte(`["DefaultCompression","BestSpeed","BestCompression"]`), &res); err != nil { - panic(err) - } - for _, v := range res { - backupConfigTypeCompressionLevelPropEnum = append(backupConfigTypeCompressionLevelPropEnum, v) - } -} - -const ( - - // BackupConfigCompressionLevelDefaultCompression captures enum value "DefaultCompression" - BackupConfigCompressionLevelDefaultCompression string = "DefaultCompression" - - // BackupConfigCompressionLevelBestSpeed captures enum value "BestSpeed" - BackupConfigCompressionLevelBestSpeed string = "BestSpeed" - - // BackupConfigCompressionLevelBestCompression captures enum value "BestCompression" - BackupConfigCompressionLevelBestCompression string = "BestCompression" -) - -// prop value enum -func (m *BackupConfig) validateCompressionLevelEnum(path, location string, value string) error { - if err := validate.EnumCase(path, location, value, backupConfigTypeCompressionLevelPropEnum, true); err != nil { - return err - } - return nil -} - -func (m *BackupConfig) validateCompressionLevel(formats strfmt.Registry) error { - if swag.IsZero(m.CompressionLevel) { // not required - return nil - } - - // value enum - if err := m.validateCompressionLevelEnum("CompressionLevel", "body", m.CompressionLevel); err != nil { - return err - } - - return nil -} - -// ContextValidate validates this backup config based on context it is used -func (m *BackupConfig) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - return nil -} - -// MarshalBinary interface implementation -func (m *BackupConfig) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *BackupConfig) UnmarshalBinary(b []byte) error { - var res BackupConfig - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/backup_create_request.go b/entities/models/backup_create_request.go deleted file mode 100644 index d8482de9a301f6fe73858145c2bc8698a60b9d27..0000000000000000000000000000000000000000 --- a/entities/models/backup_create_request.go +++ /dev/null @@ -1,124 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - - "github.com/go-openapi/errors" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" -) - -// BackupCreateRequest Request body for creating a backup of a set of classes -// -// swagger:model BackupCreateRequest -type BackupCreateRequest struct { - - // Custom configuration for the backup creation process - Config *BackupConfig `json:"config,omitempty"` - - // List of classes to exclude from the backup creation process - Exclude []string `json:"exclude"` - - // The ID of the backup. Must be URL-safe and work as a filesystem path, only lowercase, numbers, underscore, minus characters allowed. - ID string `json:"id,omitempty"` - - // List of classes to include in the backup creation process - Include []string `json:"include"` -} - -// Validate validates this backup create request -func (m *BackupCreateRequest) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateConfig(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *BackupCreateRequest) validateConfig(formats strfmt.Registry) error { - if swag.IsZero(m.Config) { // not required - return nil - } - - if m.Config != nil { - if err := m.Config.Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("config") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("config") - } - return err - } - } - - return nil -} - -// ContextValidate validate this backup create request based on the context it is used -func (m *BackupCreateRequest) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - if err := m.contextValidateConfig(ctx, formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *BackupCreateRequest) contextValidateConfig(ctx context.Context, formats strfmt.Registry) error { - - if m.Config != nil { - if err := m.Config.ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("config") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("config") - } - return err - } - } - - return nil -} - -// MarshalBinary interface implementation -func (m *BackupCreateRequest) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *BackupCreateRequest) UnmarshalBinary(b []byte) error { - var res BackupCreateRequest - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/backup_create_response.go b/entities/models/backup_create_response.go deleted file mode 100644 index 415d57d7a9203ae7f5e01da7c7a00a5897cf812c..0000000000000000000000000000000000000000 --- a/entities/models/backup_create_response.go +++ /dev/null @@ -1,140 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "encoding/json" - - "github.com/go-openapi/errors" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" - "github.com/go-openapi/validate" -) - -// BackupCreateResponse The definition of a backup create response body -// -// swagger:model BackupCreateResponse -type BackupCreateResponse struct { - - // Backup backend name e.g. filesystem, gcs, s3. - Backend string `json:"backend,omitempty"` - - // The list of classes for which the backup creation process was started - Classes []string `json:"classes"` - - // error message if creation failed - Error string `json:"error,omitempty"` - - // The ID of the backup. Must be URL-safe and work as a filesystem path, only lowercase, numbers, underscore, minus characters allowed. - ID string `json:"id,omitempty"` - - // destination path of backup files proper to selected backend - Path string `json:"path,omitempty"` - - // phase of backup creation process - // Enum: [STARTED TRANSFERRING TRANSFERRED SUCCESS FAILED] - Status *string `json:"status,omitempty"` -} - -// Validate validates this backup create response -func (m *BackupCreateResponse) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateStatus(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -var backupCreateResponseTypeStatusPropEnum []interface{} - -func init() { - var res []string - if err := json.Unmarshal([]byte(`["STARTED","TRANSFERRING","TRANSFERRED","SUCCESS","FAILED"]`), &res); err != nil { - panic(err) - } - for _, v := range res { - backupCreateResponseTypeStatusPropEnum = append(backupCreateResponseTypeStatusPropEnum, v) - } -} - -const ( - - // BackupCreateResponseStatusSTARTED captures enum value "STARTED" - BackupCreateResponseStatusSTARTED string = "STARTED" - - // BackupCreateResponseStatusTRANSFERRING captures enum value "TRANSFERRING" - BackupCreateResponseStatusTRANSFERRING string = "TRANSFERRING" - - // BackupCreateResponseStatusTRANSFERRED captures enum value "TRANSFERRED" - BackupCreateResponseStatusTRANSFERRED string = "TRANSFERRED" - - // BackupCreateResponseStatusSUCCESS captures enum value "SUCCESS" - BackupCreateResponseStatusSUCCESS string = "SUCCESS" - - // BackupCreateResponseStatusFAILED captures enum value "FAILED" - BackupCreateResponseStatusFAILED string = "FAILED" -) - -// prop value enum -func (m *BackupCreateResponse) validateStatusEnum(path, location string, value string) error { - if err := validate.EnumCase(path, location, value, backupCreateResponseTypeStatusPropEnum, true); err != nil { - return err - } - return nil -} - -func (m *BackupCreateResponse) validateStatus(formats strfmt.Registry) error { - if swag.IsZero(m.Status) { // not required - return nil - } - - // value enum - if err := m.validateStatusEnum("status", "body", *m.Status); err != nil { - return err - } - - return nil -} - -// ContextValidate validates this backup create response based on context it is used -func (m *BackupCreateResponse) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - return nil -} - -// MarshalBinary interface implementation -func (m *BackupCreateResponse) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *BackupCreateResponse) UnmarshalBinary(b []byte) error { - var res BackupCreateResponse - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/backup_create_status_response.go b/entities/models/backup_create_status_response.go deleted file mode 100644 index e1a77a7bd9d091ebc2c675f49f83ed75cf7ae51c..0000000000000000000000000000000000000000 --- a/entities/models/backup_create_status_response.go +++ /dev/null @@ -1,137 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "encoding/json" - - "github.com/go-openapi/errors" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" - "github.com/go-openapi/validate" -) - -// BackupCreateStatusResponse The definition of a backup create metadata -// -// swagger:model BackupCreateStatusResponse -type BackupCreateStatusResponse struct { - - // Backup backend name e.g. filesystem, gcs, s3. - Backend string `json:"backend,omitempty"` - - // error message if creation failed - Error string `json:"error,omitempty"` - - // The ID of the backup. Must be URL-safe and work as a filesystem path, only lowercase, numbers, underscore, minus characters allowed. - ID string `json:"id,omitempty"` - - // destination path of backup files proper to selected backend - Path string `json:"path,omitempty"` - - // phase of backup creation process - // Enum: [STARTED TRANSFERRING TRANSFERRED SUCCESS FAILED] - Status *string `json:"status,omitempty"` -} - -// Validate validates this backup create status response -func (m *BackupCreateStatusResponse) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateStatus(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -var backupCreateStatusResponseTypeStatusPropEnum []interface{} - -func init() { - var res []string - if err := json.Unmarshal([]byte(`["STARTED","TRANSFERRING","TRANSFERRED","SUCCESS","FAILED"]`), &res); err != nil { - panic(err) - } - for _, v := range res { - backupCreateStatusResponseTypeStatusPropEnum = append(backupCreateStatusResponseTypeStatusPropEnum, v) - } -} - -const ( - - // BackupCreateStatusResponseStatusSTARTED captures enum value "STARTED" - BackupCreateStatusResponseStatusSTARTED string = "STARTED" - - // BackupCreateStatusResponseStatusTRANSFERRING captures enum value "TRANSFERRING" - BackupCreateStatusResponseStatusTRANSFERRING string = "TRANSFERRING" - - // BackupCreateStatusResponseStatusTRANSFERRED captures enum value "TRANSFERRED" - BackupCreateStatusResponseStatusTRANSFERRED string = "TRANSFERRED" - - // BackupCreateStatusResponseStatusSUCCESS captures enum value "SUCCESS" - BackupCreateStatusResponseStatusSUCCESS string = "SUCCESS" - - // BackupCreateStatusResponseStatusFAILED captures enum value "FAILED" - BackupCreateStatusResponseStatusFAILED string = "FAILED" -) - -// prop value enum -func (m *BackupCreateStatusResponse) validateStatusEnum(path, location string, value string) error { - if err := validate.EnumCase(path, location, value, backupCreateStatusResponseTypeStatusPropEnum, true); err != nil { - return err - } - return nil -} - -func (m *BackupCreateStatusResponse) validateStatus(formats strfmt.Registry) error { - if swag.IsZero(m.Status) { // not required - return nil - } - - // value enum - if err := m.validateStatusEnum("status", "body", *m.Status); err != nil { - return err - } - - return nil -} - -// ContextValidate validates this backup create status response based on context it is used -func (m *BackupCreateStatusResponse) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - return nil -} - -// MarshalBinary interface implementation -func (m *BackupCreateStatusResponse) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *BackupCreateStatusResponse) UnmarshalBinary(b []byte) error { - var res BackupCreateStatusResponse - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/backup_restore_request.go b/entities/models/backup_restore_request.go deleted file mode 100644 index a6fa98dd308541a656a472249fa0342a7031a38d..0000000000000000000000000000000000000000 --- a/entities/models/backup_restore_request.go +++ /dev/null @@ -1,124 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - - "github.com/go-openapi/errors" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" -) - -// BackupRestoreRequest Request body for restoring a backup for a set of classes -// -// swagger:model BackupRestoreRequest -type BackupRestoreRequest struct { - - // Custom configuration for the backup restoration process - Config *RestoreConfig `json:"config,omitempty"` - - // List of classes to exclude from the backup restoration process - Exclude []string `json:"exclude"` - - // List of classes to include in the backup restoration process - Include []string `json:"include"` - - // Allows overriding the node names stored in the backup with different ones. Useful when restoring backups to a different environment. - NodeMapping map[string]string `json:"node_mapping,omitempty"` -} - -// Validate validates this backup restore request -func (m *BackupRestoreRequest) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateConfig(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *BackupRestoreRequest) validateConfig(formats strfmt.Registry) error { - if swag.IsZero(m.Config) { // not required - return nil - } - - if m.Config != nil { - if err := m.Config.Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("config") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("config") - } - return err - } - } - - return nil -} - -// ContextValidate validate this backup restore request based on the context it is used -func (m *BackupRestoreRequest) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - if err := m.contextValidateConfig(ctx, formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *BackupRestoreRequest) contextValidateConfig(ctx context.Context, formats strfmt.Registry) error { - - if m.Config != nil { - if err := m.Config.ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("config") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("config") - } - return err - } - } - - return nil -} - -// MarshalBinary interface implementation -func (m *BackupRestoreRequest) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *BackupRestoreRequest) UnmarshalBinary(b []byte) error { - var res BackupRestoreRequest - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/backup_restore_response.go b/entities/models/backup_restore_response.go deleted file mode 100644 index 0800eb7565511f7ed78c530c178f9aa4c0929d51..0000000000000000000000000000000000000000 --- a/entities/models/backup_restore_response.go +++ /dev/null @@ -1,140 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "encoding/json" - - "github.com/go-openapi/errors" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" - "github.com/go-openapi/validate" -) - -// BackupRestoreResponse The definition of a backup restore response body -// -// swagger:model BackupRestoreResponse -type BackupRestoreResponse struct { - - // Backup backend name e.g. filesystem, gcs, s3. - Backend string `json:"backend,omitempty"` - - // The list of classes for which the backup restoration process was started - Classes []string `json:"classes"` - - // error message if restoration failed - Error string `json:"error,omitempty"` - - // The ID of the backup. Must be URL-safe and work as a filesystem path, only lowercase, numbers, underscore, minus characters allowed. - ID string `json:"id,omitempty"` - - // destination path of backup files proper to selected backend - Path string `json:"path,omitempty"` - - // phase of backup restoration process - // Enum: [STARTED TRANSFERRING TRANSFERRED SUCCESS FAILED] - Status *string `json:"status,omitempty"` -} - -// Validate validates this backup restore response -func (m *BackupRestoreResponse) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateStatus(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -var backupRestoreResponseTypeStatusPropEnum []interface{} - -func init() { - var res []string - if err := json.Unmarshal([]byte(`["STARTED","TRANSFERRING","TRANSFERRED","SUCCESS","FAILED"]`), &res); err != nil { - panic(err) - } - for _, v := range res { - backupRestoreResponseTypeStatusPropEnum = append(backupRestoreResponseTypeStatusPropEnum, v) - } -} - -const ( - - // BackupRestoreResponseStatusSTARTED captures enum value "STARTED" - BackupRestoreResponseStatusSTARTED string = "STARTED" - - // BackupRestoreResponseStatusTRANSFERRING captures enum value "TRANSFERRING" - BackupRestoreResponseStatusTRANSFERRING string = "TRANSFERRING" - - // BackupRestoreResponseStatusTRANSFERRED captures enum value "TRANSFERRED" - BackupRestoreResponseStatusTRANSFERRED string = "TRANSFERRED" - - // BackupRestoreResponseStatusSUCCESS captures enum value "SUCCESS" - BackupRestoreResponseStatusSUCCESS string = "SUCCESS" - - // BackupRestoreResponseStatusFAILED captures enum value "FAILED" - BackupRestoreResponseStatusFAILED string = "FAILED" -) - -// prop value enum -func (m *BackupRestoreResponse) validateStatusEnum(path, location string, value string) error { - if err := validate.EnumCase(path, location, value, backupRestoreResponseTypeStatusPropEnum, true); err != nil { - return err - } - return nil -} - -func (m *BackupRestoreResponse) validateStatus(formats strfmt.Registry) error { - if swag.IsZero(m.Status) { // not required - return nil - } - - // value enum - if err := m.validateStatusEnum("status", "body", *m.Status); err != nil { - return err - } - - return nil -} - -// ContextValidate validates this backup restore response based on context it is used -func (m *BackupRestoreResponse) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - return nil -} - -// MarshalBinary interface implementation -func (m *BackupRestoreResponse) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *BackupRestoreResponse) UnmarshalBinary(b []byte) error { - var res BackupRestoreResponse - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/backup_restore_status_response.go b/entities/models/backup_restore_status_response.go deleted file mode 100644 index 0776b1b491fde851ae3feca311c1fff40207c23b..0000000000000000000000000000000000000000 --- a/entities/models/backup_restore_status_response.go +++ /dev/null @@ -1,137 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "encoding/json" - - "github.com/go-openapi/errors" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" - "github.com/go-openapi/validate" -) - -// BackupRestoreStatusResponse The definition of a backup restore metadata -// -// swagger:model BackupRestoreStatusResponse -type BackupRestoreStatusResponse struct { - - // Backup backend name e.g. filesystem, gcs, s3. - Backend string `json:"backend,omitempty"` - - // error message if restoration failed - Error string `json:"error,omitempty"` - - // The ID of the backup. Must be URL-safe and work as a filesystem path, only lowercase, numbers, underscore, minus characters allowed. - ID string `json:"id,omitempty"` - - // destination path of backup files proper to selected backup backend - Path string `json:"path,omitempty"` - - // phase of backup restoration process - // Enum: [STARTED TRANSFERRING TRANSFERRED SUCCESS FAILED] - Status *string `json:"status,omitempty"` -} - -// Validate validates this backup restore status response -func (m *BackupRestoreStatusResponse) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateStatus(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -var backupRestoreStatusResponseTypeStatusPropEnum []interface{} - -func init() { - var res []string - if err := json.Unmarshal([]byte(`["STARTED","TRANSFERRING","TRANSFERRED","SUCCESS","FAILED"]`), &res); err != nil { - panic(err) - } - for _, v := range res { - backupRestoreStatusResponseTypeStatusPropEnum = append(backupRestoreStatusResponseTypeStatusPropEnum, v) - } -} - -const ( - - // BackupRestoreStatusResponseStatusSTARTED captures enum value "STARTED" - BackupRestoreStatusResponseStatusSTARTED string = "STARTED" - - // BackupRestoreStatusResponseStatusTRANSFERRING captures enum value "TRANSFERRING" - BackupRestoreStatusResponseStatusTRANSFERRING string = "TRANSFERRING" - - // BackupRestoreStatusResponseStatusTRANSFERRED captures enum value "TRANSFERRED" - BackupRestoreStatusResponseStatusTRANSFERRED string = "TRANSFERRED" - - // BackupRestoreStatusResponseStatusSUCCESS captures enum value "SUCCESS" - BackupRestoreStatusResponseStatusSUCCESS string = "SUCCESS" - - // BackupRestoreStatusResponseStatusFAILED captures enum value "FAILED" - BackupRestoreStatusResponseStatusFAILED string = "FAILED" -) - -// prop value enum -func (m *BackupRestoreStatusResponse) validateStatusEnum(path, location string, value string) error { - if err := validate.EnumCase(path, location, value, backupRestoreStatusResponseTypeStatusPropEnum, true); err != nil { - return err - } - return nil -} - -func (m *BackupRestoreStatusResponse) validateStatus(formats strfmt.Registry) error { - if swag.IsZero(m.Status) { // not required - return nil - } - - // value enum - if err := m.validateStatusEnum("status", "body", *m.Status); err != nil { - return err - } - - return nil -} - -// ContextValidate validates this backup restore status response based on context it is used -func (m *BackupRestoreStatusResponse) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - return nil -} - -// MarshalBinary interface implementation -func (m *BackupRestoreStatusResponse) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *BackupRestoreStatusResponse) UnmarshalBinary(b []byte) error { - var res BackupRestoreStatusResponse - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/batch_delete.go b/entities/models/batch_delete.go deleted file mode 100644 index 1c244b87cc94a41cc5410bdb87e747d4fdd5ccf5..0000000000000000000000000000000000000000 --- a/entities/models/batch_delete.go +++ /dev/null @@ -1,215 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - - "github.com/go-openapi/errors" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" -) - -// BatchDelete batch delete -// -// swagger:model BatchDelete -type BatchDelete struct { - - // If true, objects will not be deleted yet, but merely listed. Defaults to false. - DryRun *bool `json:"dryRun,omitempty"` - - // match - Match *BatchDeleteMatch `json:"match,omitempty"` - - // Controls the verbosity of the output, possible values are: "minimal", "verbose". Defaults to "minimal". - Output *string `json:"output,omitempty"` -} - -// Validate validates this batch delete -func (m *BatchDelete) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateMatch(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *BatchDelete) validateMatch(formats strfmt.Registry) error { - if swag.IsZero(m.Match) { // not required - return nil - } - - if m.Match != nil { - if err := m.Match.Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("match") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("match") - } - return err - } - } - - return nil -} - -// ContextValidate validate this batch delete based on the context it is used -func (m *BatchDelete) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - if err := m.contextValidateMatch(ctx, formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *BatchDelete) contextValidateMatch(ctx context.Context, formats strfmt.Registry) error { - - if m.Match != nil { - if err := m.Match.ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("match") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("match") - } - return err - } - } - - return nil -} - -// MarshalBinary interface implementation -func (m *BatchDelete) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *BatchDelete) UnmarshalBinary(b []byte) error { - var res BatchDelete - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} - -// BatchDeleteMatch Outlines how to find the objects to be deleted. -// -// swagger:model BatchDeleteMatch -type BatchDeleteMatch struct { - - // Class (name) which objects will be deleted. - // Example: City - Class string `json:"class,omitempty"` - - // Filter to limit the objects to be deleted. - Where *WhereFilter `json:"where,omitempty"` -} - -// Validate validates this batch delete match -func (m *BatchDeleteMatch) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateWhere(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *BatchDeleteMatch) validateWhere(formats strfmt.Registry) error { - if swag.IsZero(m.Where) { // not required - return nil - } - - if m.Where != nil { - if err := m.Where.Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("match" + "." + "where") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("match" + "." + "where") - } - return err - } - } - - return nil -} - -// ContextValidate validate this batch delete match based on the context it is used -func (m *BatchDeleteMatch) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - if err := m.contextValidateWhere(ctx, formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *BatchDeleteMatch) contextValidateWhere(ctx context.Context, formats strfmt.Registry) error { - - if m.Where != nil { - if err := m.Where.ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("match" + "." + "where") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("match" + "." + "where") - } - return err - } - } - - return nil -} - -// MarshalBinary interface implementation -func (m *BatchDeleteMatch) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *BatchDeleteMatch) UnmarshalBinary(b []byte) error { - var res BatchDeleteMatch - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/batch_delete_response.go b/entities/models/batch_delete_response.go deleted file mode 100644 index 9162952cd911e6e7591dbcd3056d12cb6524ec27..0000000000000000000000000000000000000000 --- a/entities/models/batch_delete_response.go +++ /dev/null @@ -1,540 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "encoding/json" - "strconv" - - "github.com/go-openapi/errors" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" - "github.com/go-openapi/validate" -) - -// BatchDeleteResponse Delete Objects response. -// -// swagger:model BatchDeleteResponse -type BatchDeleteResponse struct { - - // If true, objects will not be deleted yet, but merely listed. Defaults to false. - DryRun *bool `json:"dryRun,omitempty"` - - // match - Match *BatchDeleteResponseMatch `json:"match,omitempty"` - - // Controls the verbosity of the output, possible values are: "minimal", "verbose". Defaults to "minimal". - Output *string `json:"output,omitempty"` - - // results - Results *BatchDeleteResponseResults `json:"results,omitempty"` -} - -// Validate validates this batch delete response -func (m *BatchDeleteResponse) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateMatch(formats); err != nil { - res = append(res, err) - } - - if err := m.validateResults(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *BatchDeleteResponse) validateMatch(formats strfmt.Registry) error { - if swag.IsZero(m.Match) { // not required - return nil - } - - if m.Match != nil { - if err := m.Match.Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("match") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("match") - } - return err - } - } - - return nil -} - -func (m *BatchDeleteResponse) validateResults(formats strfmt.Registry) error { - if swag.IsZero(m.Results) { // not required - return nil - } - - if m.Results != nil { - if err := m.Results.Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("results") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("results") - } - return err - } - } - - return nil -} - -// ContextValidate validate this batch delete response based on the context it is used -func (m *BatchDeleteResponse) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - if err := m.contextValidateMatch(ctx, formats); err != nil { - res = append(res, err) - } - - if err := m.contextValidateResults(ctx, formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *BatchDeleteResponse) contextValidateMatch(ctx context.Context, formats strfmt.Registry) error { - - if m.Match != nil { - if err := m.Match.ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("match") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("match") - } - return err - } - } - - return nil -} - -func (m *BatchDeleteResponse) contextValidateResults(ctx context.Context, formats strfmt.Registry) error { - - if m.Results != nil { - if err := m.Results.ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("results") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("results") - } - return err - } - } - - return nil -} - -// MarshalBinary interface implementation -func (m *BatchDeleteResponse) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *BatchDeleteResponse) UnmarshalBinary(b []byte) error { - var res BatchDeleteResponse - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} - -// BatchDeleteResponseMatch Outlines how to find the objects to be deleted. -// -// swagger:model BatchDeleteResponseMatch -type BatchDeleteResponseMatch struct { - - // Class (name) which objects will be deleted. - // Example: City - Class string `json:"class,omitempty"` - - // Filter to limit the objects to be deleted. - Where *WhereFilter `json:"where,omitempty"` -} - -// Validate validates this batch delete response match -func (m *BatchDeleteResponseMatch) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateWhere(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *BatchDeleteResponseMatch) validateWhere(formats strfmt.Registry) error { - if swag.IsZero(m.Where) { // not required - return nil - } - - if m.Where != nil { - if err := m.Where.Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("match" + "." + "where") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("match" + "." + "where") - } - return err - } - } - - return nil -} - -// ContextValidate validate this batch delete response match based on the context it is used -func (m *BatchDeleteResponseMatch) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - if err := m.contextValidateWhere(ctx, formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *BatchDeleteResponseMatch) contextValidateWhere(ctx context.Context, formats strfmt.Registry) error { - - if m.Where != nil { - if err := m.Where.ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("match" + "." + "where") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("match" + "." + "where") - } - return err - } - } - - return nil -} - -// MarshalBinary interface implementation -func (m *BatchDeleteResponseMatch) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *BatchDeleteResponseMatch) UnmarshalBinary(b []byte) error { - var res BatchDeleteResponseMatch - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} - -// BatchDeleteResponseResults batch delete response results -// -// swagger:model BatchDeleteResponseResults -type BatchDeleteResponseResults struct { - - // How many objects should have been deleted but could not be deleted. - Failed int64 `json:"failed"` - - // The most amount of objects that can be deleted in a single query, equals QUERY_MAXIMUM_RESULTS. - Limit int64 `json:"limit"` - - // How many objects were matched by the filter. - Matches int64 `json:"matches"` - - // With output set to "minimal" only objects with error occurred will the be described. Successfully deleted objects would be omitted. Output set to "verbose" will list all of the objets with their respective statuses. - Objects []*BatchDeleteResponseResultsObjectsItems0 `json:"objects"` - - // How many objects were successfully deleted in this round. - Successful int64 `json:"successful"` -} - -// Validate validates this batch delete response results -func (m *BatchDeleteResponseResults) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateObjects(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *BatchDeleteResponseResults) validateObjects(formats strfmt.Registry) error { - if swag.IsZero(m.Objects) { // not required - return nil - } - - for i := 0; i < len(m.Objects); i++ { - if swag.IsZero(m.Objects[i]) { // not required - continue - } - - if m.Objects[i] != nil { - if err := m.Objects[i].Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("results" + "." + "objects" + "." + strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("results" + "." + "objects" + "." + strconv.Itoa(i)) - } - return err - } - } - - } - - return nil -} - -// ContextValidate validate this batch delete response results based on the context it is used -func (m *BatchDeleteResponseResults) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - if err := m.contextValidateObjects(ctx, formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *BatchDeleteResponseResults) contextValidateObjects(ctx context.Context, formats strfmt.Registry) error { - - for i := 0; i < len(m.Objects); i++ { - - if m.Objects[i] != nil { - if err := m.Objects[i].ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("results" + "." + "objects" + "." + strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("results" + "." + "objects" + "." + strconv.Itoa(i)) - } - return err - } - } - - } - - return nil -} - -// MarshalBinary interface implementation -func (m *BatchDeleteResponseResults) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *BatchDeleteResponseResults) UnmarshalBinary(b []byte) error { - var res BatchDeleteResponseResults - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} - -// BatchDeleteResponseResultsObjectsItems0 Results for this specific Object. -// -// swagger:model BatchDeleteResponseResultsObjectsItems0 -type BatchDeleteResponseResultsObjectsItems0 struct { - - // errors - Errors *ErrorResponse `json:"errors,omitempty"` - - // ID of the Object. - // Format: uuid - ID strfmt.UUID `json:"id,omitempty"` - - // status - // Enum: [SUCCESS DRYRUN FAILED] - Status *string `json:"status,omitempty"` -} - -// Validate validates this batch delete response results objects items0 -func (m *BatchDeleteResponseResultsObjectsItems0) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateErrors(formats); err != nil { - res = append(res, err) - } - - if err := m.validateID(formats); err != nil { - res = append(res, err) - } - - if err := m.validateStatus(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *BatchDeleteResponseResultsObjectsItems0) validateErrors(formats strfmt.Registry) error { - if swag.IsZero(m.Errors) { // not required - return nil - } - - if m.Errors != nil { - if err := m.Errors.Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("errors") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("errors") - } - return err - } - } - - return nil -} - -func (m *BatchDeleteResponseResultsObjectsItems0) validateID(formats strfmt.Registry) error { - if swag.IsZero(m.ID) { // not required - return nil - } - - if err := validate.FormatOf("id", "body", "uuid", m.ID.String(), formats); err != nil { - return err - } - - return nil -} - -var batchDeleteResponseResultsObjectsItems0TypeStatusPropEnum []interface{} - -func init() { - var res []string - if err := json.Unmarshal([]byte(`["SUCCESS","DRYRUN","FAILED"]`), &res); err != nil { - panic(err) - } - for _, v := range res { - batchDeleteResponseResultsObjectsItems0TypeStatusPropEnum = append(batchDeleteResponseResultsObjectsItems0TypeStatusPropEnum, v) - } -} - -const ( - - // BatchDeleteResponseResultsObjectsItems0StatusSUCCESS captures enum value "SUCCESS" - BatchDeleteResponseResultsObjectsItems0StatusSUCCESS string = "SUCCESS" - - // BatchDeleteResponseResultsObjectsItems0StatusDRYRUN captures enum value "DRYRUN" - BatchDeleteResponseResultsObjectsItems0StatusDRYRUN string = "DRYRUN" - - // BatchDeleteResponseResultsObjectsItems0StatusFAILED captures enum value "FAILED" - BatchDeleteResponseResultsObjectsItems0StatusFAILED string = "FAILED" -) - -// prop value enum -func (m *BatchDeleteResponseResultsObjectsItems0) validateStatusEnum(path, location string, value string) error { - if err := validate.EnumCase(path, location, value, batchDeleteResponseResultsObjectsItems0TypeStatusPropEnum, true); err != nil { - return err - } - return nil -} - -func (m *BatchDeleteResponseResultsObjectsItems0) validateStatus(formats strfmt.Registry) error { - if swag.IsZero(m.Status) { // not required - return nil - } - - // value enum - if err := m.validateStatusEnum("status", "body", *m.Status); err != nil { - return err - } - - return nil -} - -// ContextValidate validate this batch delete response results objects items0 based on the context it is used -func (m *BatchDeleteResponseResultsObjectsItems0) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - if err := m.contextValidateErrors(ctx, formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *BatchDeleteResponseResultsObjectsItems0) contextValidateErrors(ctx context.Context, formats strfmt.Registry) error { - - if m.Errors != nil { - if err := m.Errors.ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("errors") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("errors") - } - return err - } - } - - return nil -} - -// MarshalBinary interface implementation -func (m *BatchDeleteResponseResultsObjectsItems0) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *BatchDeleteResponseResultsObjectsItems0) UnmarshalBinary(b []byte) error { - var res BatchDeleteResponseResultsObjectsItems0 - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/batch_reference.go b/entities/models/batch_reference.go deleted file mode 100644 index 140ad02f99ab7d38320113b363ed21f757558627..0000000000000000000000000000000000000000 --- a/entities/models/batch_reference.go +++ /dev/null @@ -1,110 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - - "github.com/go-openapi/errors" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" - "github.com/go-openapi/validate" -) - -// BatchReference batch reference -// -// swagger:model BatchReference -type BatchReference struct { - - // Long-form beacon-style URI to identify the source of the cross-ref including the property name. Should be in the form of weaviate://localhost////, where must be one of 'objects', 'objects' and and must represent the cross-ref property of source class to be used. - // Example: weaviate://localhost/Zoo/a5d09582-4239-4702-81c9-92a6e0122bb4/hasAnimals - // Format: uri - From strfmt.URI `json:"from,omitempty"` - - // Name of the reference tenant. - Tenant string `json:"tenant,omitempty"` - - // Short-form URI to point to the cross-ref. Should be in the form of weaviate://localhost/ for the example of a local cross-ref to an object - // Example: weaviate://localhost/97525810-a9a5-4eb0-858a-71449aeb007f - // Format: uri - To strfmt.URI `json:"to,omitempty"` -} - -// Validate validates this batch reference -func (m *BatchReference) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateFrom(formats); err != nil { - res = append(res, err) - } - - if err := m.validateTo(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *BatchReference) validateFrom(formats strfmt.Registry) error { - if swag.IsZero(m.From) { // not required - return nil - } - - if err := validate.FormatOf("from", "body", "uri", m.From.String(), formats); err != nil { - return err - } - - return nil -} - -func (m *BatchReference) validateTo(formats strfmt.Registry) error { - if swag.IsZero(m.To) { // not required - return nil - } - - if err := validate.FormatOf("to", "body", "uri", m.To.String(), formats); err != nil { - return err - } - - return nil -} - -// ContextValidate validates this batch reference based on context it is used -func (m *BatchReference) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - return nil -} - -// MarshalBinary interface implementation -func (m *BatchReference) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *BatchReference) UnmarshalBinary(b []byte) error { - var res BatchReference - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/batch_reference_response.go b/entities/models/batch_reference_response.go deleted file mode 100644 index 3331cd16328361d3a59b1e1fbe26371b6e4127e9..0000000000000000000000000000000000000000 --- a/entities/models/batch_reference_response.go +++ /dev/null @@ -1,317 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "encoding/json" - - "github.com/go-openapi/errors" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" - "github.com/go-openapi/validate" -) - -// BatchReferenceResponse batch reference response -// -// swagger:model BatchReferenceResponse -type BatchReferenceResponse struct { - BatchReference - - // result - Result *BatchReferenceResponseAO1Result `json:"result,omitempty"` -} - -// UnmarshalJSON unmarshals this object from a JSON structure -func (m *BatchReferenceResponse) UnmarshalJSON(raw []byte) error { - // AO0 - var aO0 BatchReference - if err := swag.ReadJSON(raw, &aO0); err != nil { - return err - } - m.BatchReference = aO0 - - // AO1 - var dataAO1 struct { - Result *BatchReferenceResponseAO1Result `json:"result,omitempty"` - } - if err := swag.ReadJSON(raw, &dataAO1); err != nil { - return err - } - - m.Result = dataAO1.Result - - return nil -} - -// MarshalJSON marshals this object to a JSON structure -func (m BatchReferenceResponse) MarshalJSON() ([]byte, error) { - _parts := make([][]byte, 0, 2) - - aO0, err := swag.WriteJSON(m.BatchReference) - if err != nil { - return nil, err - } - _parts = append(_parts, aO0) - var dataAO1 struct { - Result *BatchReferenceResponseAO1Result `json:"result,omitempty"` - } - - dataAO1.Result = m.Result - - jsonDataAO1, errAO1 := swag.WriteJSON(dataAO1) - if errAO1 != nil { - return nil, errAO1 - } - _parts = append(_parts, jsonDataAO1) - return swag.ConcatJSON(_parts...), nil -} - -// Validate validates this batch reference response -func (m *BatchReferenceResponse) Validate(formats strfmt.Registry) error { - var res []error - - // validation for a type composition with BatchReference - if err := m.BatchReference.Validate(formats); err != nil { - res = append(res, err) - } - - if err := m.validateResult(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *BatchReferenceResponse) validateResult(formats strfmt.Registry) error { - - if swag.IsZero(m.Result) { // not required - return nil - } - - if m.Result != nil { - if err := m.Result.Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("result") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("result") - } - return err - } - } - - return nil -} - -// ContextValidate validate this batch reference response based on the context it is used -func (m *BatchReferenceResponse) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - // validation for a type composition with BatchReference - if err := m.BatchReference.ContextValidate(ctx, formats); err != nil { - res = append(res, err) - } - - if err := m.contextValidateResult(ctx, formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *BatchReferenceResponse) contextValidateResult(ctx context.Context, formats strfmt.Registry) error { - - if m.Result != nil { - if err := m.Result.ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("result") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("result") - } - return err - } - } - - return nil -} - -// MarshalBinary interface implementation -func (m *BatchReferenceResponse) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *BatchReferenceResponse) UnmarshalBinary(b []byte) error { - var res BatchReferenceResponse - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} - -// BatchReferenceResponseAO1Result Results for this specific reference. -// -// swagger:model BatchReferenceResponseAO1Result -type BatchReferenceResponseAO1Result struct { - - // errors - Errors *ErrorResponse `json:"errors,omitempty"` - - // status - // Enum: [SUCCESS PENDING FAILED] - Status *string `json:"status,omitempty"` -} - -// Validate validates this batch reference response a o1 result -func (m *BatchReferenceResponseAO1Result) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateErrors(formats); err != nil { - res = append(res, err) - } - - if err := m.validateStatus(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *BatchReferenceResponseAO1Result) validateErrors(formats strfmt.Registry) error { - if swag.IsZero(m.Errors) { // not required - return nil - } - - if m.Errors != nil { - if err := m.Errors.Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("result" + "." + "errors") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("result" + "." + "errors") - } - return err - } - } - - return nil -} - -var batchReferenceResponseAO1ResultTypeStatusPropEnum []interface{} - -func init() { - var res []string - if err := json.Unmarshal([]byte(`["SUCCESS","PENDING","FAILED"]`), &res); err != nil { - panic(err) - } - for _, v := range res { - batchReferenceResponseAO1ResultTypeStatusPropEnum = append(batchReferenceResponseAO1ResultTypeStatusPropEnum, v) - } -} - -const ( - - // BatchReferenceResponseAO1ResultStatusSUCCESS captures enum value "SUCCESS" - BatchReferenceResponseAO1ResultStatusSUCCESS string = "SUCCESS" - - // BatchReferenceResponseAO1ResultStatusPENDING captures enum value "PENDING" - BatchReferenceResponseAO1ResultStatusPENDING string = "PENDING" - - // BatchReferenceResponseAO1ResultStatusFAILED captures enum value "FAILED" - BatchReferenceResponseAO1ResultStatusFAILED string = "FAILED" -) - -// prop value enum -func (m *BatchReferenceResponseAO1Result) validateStatusEnum(path, location string, value string) error { - if err := validate.EnumCase(path, location, value, batchReferenceResponseAO1ResultTypeStatusPropEnum, true); err != nil { - return err - } - return nil -} - -func (m *BatchReferenceResponseAO1Result) validateStatus(formats strfmt.Registry) error { - if swag.IsZero(m.Status) { // not required - return nil - } - - // value enum - if err := m.validateStatusEnum("result"+"."+"status", "body", *m.Status); err != nil { - return err - } - - return nil -} - -// ContextValidate validate this batch reference response a o1 result based on the context it is used -func (m *BatchReferenceResponseAO1Result) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - if err := m.contextValidateErrors(ctx, formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *BatchReferenceResponseAO1Result) contextValidateErrors(ctx context.Context, formats strfmt.Registry) error { - - if m.Errors != nil { - if err := m.Errors.ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("result" + "." + "errors") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("result" + "." + "errors") - } - return err - } - } - - return nil -} - -// MarshalBinary interface implementation -func (m *BatchReferenceResponseAO1Result) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *BatchReferenceResponseAO1Result) UnmarshalBinary(b []byte) error { - var res BatchReferenceResponseAO1Result - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/batch_stats.go b/entities/models/batch_stats.go deleted file mode 100644 index 3e64da06b1c60454ff664f4f1a44c610f3c24511..0000000000000000000000000000000000000000 --- a/entities/models/batch_stats.go +++ /dev/null @@ -1,64 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" -) - -// BatchStats The summary of a nodes batch queue congestion status. -// -// swagger:model BatchStats -type BatchStats struct { - - // How many objects are currently in the batch queue. - QueueLength *int64 `json:"queueLength,omitempty"` - - // How many objects are approximately processed from the batch queue per second. - RatePerSecond int64 `json:"ratePerSecond"` -} - -// Validate validates this batch stats -func (m *BatchStats) Validate(formats strfmt.Registry) error { - return nil -} - -// ContextValidate validates this batch stats based on context it is used -func (m *BatchStats) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - return nil -} - -// MarshalBinary interface implementation -func (m *BatchStats) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *BatchStats) UnmarshalBinary(b []byte) error { - var res BatchStats - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/c11y_extension.go b/entities/models/c11y_extension.go deleted file mode 100644 index 3e1ef3ca37f04fca54d933678e0b3bfaedf1d812..0000000000000000000000000000000000000000 --- a/entities/models/c11y_extension.go +++ /dev/null @@ -1,68 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" -) - -// C11yExtension A resource describing an extension to the contextinoary, containing both the identifier and the definition of the extension -// -// swagger:model C11yExtension -type C11yExtension struct { - - // The new concept you want to extend. Must be an all-lowercase single word, or a space delimited compound word. Examples: 'foobarium', 'my custom concept' - // Example: foobarium - Concept string `json:"concept,omitempty"` - - // A list of space-delimited words or a sentence describing what the custom concept is about. Avoid using the custom concept itself. An Example definition for the custom concept 'foobarium': would be 'a naturally occurring element which can only be seen by programmers' - Definition string `json:"definition,omitempty"` - - // Weight of the definition of the new concept where 1='override existing definition entirely' and 0='ignore custom definition'. Note that if the custom concept is not present in the contextionary yet, the weight cannot be less than 1. - Weight float32 `json:"weight,omitempty"` -} - -// Validate validates this c11y extension -func (m *C11yExtension) Validate(formats strfmt.Registry) error { - return nil -} - -// ContextValidate validates this c11y extension based on context it is used -func (m *C11yExtension) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - return nil -} - -// MarshalBinary interface implementation -func (m *C11yExtension) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *C11yExtension) UnmarshalBinary(b []byte) error { - var res C11yExtension - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/c11y_nearest_neighbors.go b/entities/models/c11y_nearest_neighbors.go deleted file mode 100644 index 53ccae254b1d02ec570fad5e8c769996ff82b631..0000000000000000000000000000000000000000 --- a/entities/models/c11y_nearest_neighbors.go +++ /dev/null @@ -1,124 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "strconv" - - "github.com/go-openapi/errors" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" -) - -// C11yNearestNeighbors C11y function to show the nearest neighbors to a word. -// -// swagger:model C11yNearestNeighbors -type C11yNearestNeighbors []*C11yNearestNeighborsItems0 - -// Validate validates this c11y nearest neighbors -func (m C11yNearestNeighbors) Validate(formats strfmt.Registry) error { - var res []error - - for i := 0; i < len(m); i++ { - if swag.IsZero(m[i]) { // not required - continue - } - - if m[i] != nil { - if err := m[i].Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName(strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName(strconv.Itoa(i)) - } - return err - } - } - - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// ContextValidate validate this c11y nearest neighbors based on the context it is used -func (m C11yNearestNeighbors) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - for i := 0; i < len(m); i++ { - - if m[i] != nil { - if err := m[i].ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName(strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName(strconv.Itoa(i)) - } - return err - } - } - - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// C11yNearestNeighborsItems0 c11y nearest neighbors items0 -// -// swagger:model C11yNearestNeighborsItems0 -type C11yNearestNeighborsItems0 struct { - - // distance - Distance float32 `json:"distance,omitempty"` - - // word - Word string `json:"word,omitempty"` -} - -// Validate validates this c11y nearest neighbors items0 -func (m *C11yNearestNeighborsItems0) Validate(formats strfmt.Registry) error { - return nil -} - -// ContextValidate validates this c11y nearest neighbors items0 based on context it is used -func (m *C11yNearestNeighborsItems0) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - return nil -} - -// MarshalBinary interface implementation -func (m *C11yNearestNeighborsItems0) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *C11yNearestNeighborsItems0) UnmarshalBinary(b []byte) error { - var res C11yNearestNeighborsItems0 - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/c11y_vector.go b/entities/models/c11y_vector.go deleted file mode 100644 index ec9d221d471a83ca87f659b7ea8a5c068e6dd213..0000000000000000000000000000000000000000 --- a/entities/models/c11y_vector.go +++ /dev/null @@ -1,38 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - - "github.com/go-openapi/strfmt" -) - -// C11yVector A Vector in the Contextionary -// -// swagger:model C11yVector -type C11yVector []float32 - -// Validate validates this c11y vector -func (m C11yVector) Validate(formats strfmt.Registry) error { - return nil -} - -// ContextValidate validates this c11y vector based on context it is used -func (m C11yVector) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - return nil -} diff --git a/entities/models/c11y_vector_based_question.go b/entities/models/c11y_vector_based_question.go deleted file mode 100644 index 9291d431c226cd1821fa35b00752dde2739d89cc..0000000000000000000000000000000000000000 --- a/entities/models/c11y_vector_based_question.go +++ /dev/null @@ -1,265 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "strconv" - - "github.com/go-openapi/errors" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" - "github.com/go-openapi/validate" -) - -// C11yVectorBasedQuestion Receive question based on array of classes, properties and values. -// -// swagger:model C11yVectorBasedQuestion -type C11yVectorBasedQuestion []*C11yVectorBasedQuestionItems0 - -// Validate validates this c11y vector based question -func (m C11yVectorBasedQuestion) Validate(formats strfmt.Registry) error { - var res []error - - for i := 0; i < len(m); i++ { - if swag.IsZero(m[i]) { // not required - continue - } - - if m[i] != nil { - if err := m[i].Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName(strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName(strconv.Itoa(i)) - } - return err - } - } - - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// ContextValidate validate this c11y vector based question based on the context it is used -func (m C11yVectorBasedQuestion) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - for i := 0; i < len(m); i++ { - - if m[i] != nil { - if err := m[i].ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName(strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName(strconv.Itoa(i)) - } - return err - } - } - - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// C11yVectorBasedQuestionItems0 c11y vector based question items0 -// -// swagger:model C11yVectorBasedQuestionItems0 -type C11yVectorBasedQuestionItems0 struct { - - // Vectorized properties. - // Max Items: 300 - // Min Items: 300 - ClassProps []*C11yVectorBasedQuestionItems0ClassPropsItems0 `json:"classProps"` - - // Vectorized classname. - // Max Items: 300 - // Min Items: 300 - ClassVectors []float32 `json:"classVectors"` -} - -// Validate validates this c11y vector based question items0 -func (m *C11yVectorBasedQuestionItems0) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateClassProps(formats); err != nil { - res = append(res, err) - } - - if err := m.validateClassVectors(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *C11yVectorBasedQuestionItems0) validateClassProps(formats strfmt.Registry) error { - if swag.IsZero(m.ClassProps) { // not required - return nil - } - - iClassPropsSize := int64(len(m.ClassProps)) - - if err := validate.MinItems("classProps", "body", iClassPropsSize, 300); err != nil { - return err - } - - if err := validate.MaxItems("classProps", "body", iClassPropsSize, 300); err != nil { - return err - } - - for i := 0; i < len(m.ClassProps); i++ { - if swag.IsZero(m.ClassProps[i]) { // not required - continue - } - - if m.ClassProps[i] != nil { - if err := m.ClassProps[i].Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("classProps" + "." + strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("classProps" + "." + strconv.Itoa(i)) - } - return err - } - } - - } - - return nil -} - -func (m *C11yVectorBasedQuestionItems0) validateClassVectors(formats strfmt.Registry) error { - if swag.IsZero(m.ClassVectors) { // not required - return nil - } - - iClassVectorsSize := int64(len(m.ClassVectors)) - - if err := validate.MinItems("classVectors", "body", iClassVectorsSize, 300); err != nil { - return err - } - - if err := validate.MaxItems("classVectors", "body", iClassVectorsSize, 300); err != nil { - return err - } - - return nil -} - -// ContextValidate validate this c11y vector based question items0 based on the context it is used -func (m *C11yVectorBasedQuestionItems0) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - if err := m.contextValidateClassProps(ctx, formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *C11yVectorBasedQuestionItems0) contextValidateClassProps(ctx context.Context, formats strfmt.Registry) error { - - for i := 0; i < len(m.ClassProps); i++ { - - if m.ClassProps[i] != nil { - if err := m.ClassProps[i].ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("classProps" + "." + strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("classProps" + "." + strconv.Itoa(i)) - } - return err - } - } - - } - - return nil -} - -// MarshalBinary interface implementation -func (m *C11yVectorBasedQuestionItems0) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *C11yVectorBasedQuestionItems0) UnmarshalBinary(b []byte) error { - var res C11yVectorBasedQuestionItems0 - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} - -// C11yVectorBasedQuestionItems0ClassPropsItems0 c11y vector based question items0 class props items0 -// -// swagger:model C11yVectorBasedQuestionItems0ClassPropsItems0 -type C11yVectorBasedQuestionItems0ClassPropsItems0 struct { - - // props vectors - PropsVectors []float32 `json:"propsVectors"` - - // String with valuename. - Value string `json:"value,omitempty"` -} - -// Validate validates this c11y vector based question items0 class props items0 -func (m *C11yVectorBasedQuestionItems0ClassPropsItems0) Validate(formats strfmt.Registry) error { - return nil -} - -// ContextValidate validates this c11y vector based question items0 class props items0 based on context it is used -func (m *C11yVectorBasedQuestionItems0ClassPropsItems0) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - return nil -} - -// MarshalBinary interface implementation -func (m *C11yVectorBasedQuestionItems0ClassPropsItems0) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *C11yVectorBasedQuestionItems0ClassPropsItems0) UnmarshalBinary(b []byte) error { - var res C11yVectorBasedQuestionItems0ClassPropsItems0 - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/c11y_words_response.go b/entities/models/c11y_words_response.go deleted file mode 100644 index dd88c55f3428100334b1a3a859d44bc2bdbcab34..0000000000000000000000000000000000000000 --- a/entities/models/c11y_words_response.go +++ /dev/null @@ -1,531 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "strconv" - - "github.com/go-openapi/errors" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" -) - -// C11yWordsResponse An array of available words and contexts. -// -// swagger:model C11yWordsResponse -type C11yWordsResponse struct { - - // concatenated word - ConcatenatedWord *C11yWordsResponseConcatenatedWord `json:"concatenatedWord,omitempty"` - - // Weighted results for per individual word - IndividualWords []*C11yWordsResponseIndividualWordsItems0 `json:"individualWords"` -} - -// Validate validates this c11y words response -func (m *C11yWordsResponse) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateConcatenatedWord(formats); err != nil { - res = append(res, err) - } - - if err := m.validateIndividualWords(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *C11yWordsResponse) validateConcatenatedWord(formats strfmt.Registry) error { - if swag.IsZero(m.ConcatenatedWord) { // not required - return nil - } - - if m.ConcatenatedWord != nil { - if err := m.ConcatenatedWord.Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("concatenatedWord") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("concatenatedWord") - } - return err - } - } - - return nil -} - -func (m *C11yWordsResponse) validateIndividualWords(formats strfmt.Registry) error { - if swag.IsZero(m.IndividualWords) { // not required - return nil - } - - for i := 0; i < len(m.IndividualWords); i++ { - if swag.IsZero(m.IndividualWords[i]) { // not required - continue - } - - if m.IndividualWords[i] != nil { - if err := m.IndividualWords[i].Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("individualWords" + "." + strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("individualWords" + "." + strconv.Itoa(i)) - } - return err - } - } - - } - - return nil -} - -// ContextValidate validate this c11y words response based on the context it is used -func (m *C11yWordsResponse) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - if err := m.contextValidateConcatenatedWord(ctx, formats); err != nil { - res = append(res, err) - } - - if err := m.contextValidateIndividualWords(ctx, formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *C11yWordsResponse) contextValidateConcatenatedWord(ctx context.Context, formats strfmt.Registry) error { - - if m.ConcatenatedWord != nil { - if err := m.ConcatenatedWord.ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("concatenatedWord") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("concatenatedWord") - } - return err - } - } - - return nil -} - -func (m *C11yWordsResponse) contextValidateIndividualWords(ctx context.Context, formats strfmt.Registry) error { - - for i := 0; i < len(m.IndividualWords); i++ { - - if m.IndividualWords[i] != nil { - if err := m.IndividualWords[i].ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("individualWords" + "." + strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("individualWords" + "." + strconv.Itoa(i)) - } - return err - } - } - - } - - return nil -} - -// MarshalBinary interface implementation -func (m *C11yWordsResponse) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *C11yWordsResponse) UnmarshalBinary(b []byte) error { - var res C11yWordsResponse - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} - -// C11yWordsResponseConcatenatedWord Weighted results for all words -// -// swagger:model C11yWordsResponseConcatenatedWord -type C11yWordsResponseConcatenatedWord struct { - - // concatenated nearest neighbors - ConcatenatedNearestNeighbors C11yNearestNeighbors `json:"concatenatedNearestNeighbors,omitempty"` - - // concatenated vector - ConcatenatedVector C11yVector `json:"concatenatedVector,omitempty"` - - // concatenated word - ConcatenatedWord string `json:"concatenatedWord,omitempty"` - - // single words - SingleWords []string `json:"singleWords"` -} - -// Validate validates this c11y words response concatenated word -func (m *C11yWordsResponseConcatenatedWord) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateConcatenatedNearestNeighbors(formats); err != nil { - res = append(res, err) - } - - if err := m.validateConcatenatedVector(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *C11yWordsResponseConcatenatedWord) validateConcatenatedNearestNeighbors(formats strfmt.Registry) error { - if swag.IsZero(m.ConcatenatedNearestNeighbors) { // not required - return nil - } - - if err := m.ConcatenatedNearestNeighbors.Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("concatenatedWord" + "." + "concatenatedNearestNeighbors") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("concatenatedWord" + "." + "concatenatedNearestNeighbors") - } - return err - } - - return nil -} - -func (m *C11yWordsResponseConcatenatedWord) validateConcatenatedVector(formats strfmt.Registry) error { - if swag.IsZero(m.ConcatenatedVector) { // not required - return nil - } - - if err := m.ConcatenatedVector.Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("concatenatedWord" + "." + "concatenatedVector") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("concatenatedWord" + "." + "concatenatedVector") - } - return err - } - - return nil -} - -// ContextValidate validate this c11y words response concatenated word based on the context it is used -func (m *C11yWordsResponseConcatenatedWord) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - if err := m.contextValidateConcatenatedNearestNeighbors(ctx, formats); err != nil { - res = append(res, err) - } - - if err := m.contextValidateConcatenatedVector(ctx, formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *C11yWordsResponseConcatenatedWord) contextValidateConcatenatedNearestNeighbors(ctx context.Context, formats strfmt.Registry) error { - - if err := m.ConcatenatedNearestNeighbors.ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("concatenatedWord" + "." + "concatenatedNearestNeighbors") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("concatenatedWord" + "." + "concatenatedNearestNeighbors") - } - return err - } - - return nil -} - -func (m *C11yWordsResponseConcatenatedWord) contextValidateConcatenatedVector(ctx context.Context, formats strfmt.Registry) error { - - if err := m.ConcatenatedVector.ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("concatenatedWord" + "." + "concatenatedVector") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("concatenatedWord" + "." + "concatenatedVector") - } - return err - } - - return nil -} - -// MarshalBinary interface implementation -func (m *C11yWordsResponseConcatenatedWord) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *C11yWordsResponseConcatenatedWord) UnmarshalBinary(b []byte) error { - var res C11yWordsResponseConcatenatedWord - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} - -// C11yWordsResponseIndividualWordsItems0 c11y words response individual words items0 -// -// swagger:model C11yWordsResponseIndividualWordsItems0 -type C11yWordsResponseIndividualWordsItems0 struct { - - // info - Info *C11yWordsResponseIndividualWordsItems0Info `json:"info,omitempty"` - - // present - Present bool `json:"present,omitempty"` - - // word - Word string `json:"word,omitempty"` -} - -// Validate validates this c11y words response individual words items0 -func (m *C11yWordsResponseIndividualWordsItems0) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateInfo(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *C11yWordsResponseIndividualWordsItems0) validateInfo(formats strfmt.Registry) error { - if swag.IsZero(m.Info) { // not required - return nil - } - - if m.Info != nil { - if err := m.Info.Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("info") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("info") - } - return err - } - } - - return nil -} - -// ContextValidate validate this c11y words response individual words items0 based on the context it is used -func (m *C11yWordsResponseIndividualWordsItems0) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - if err := m.contextValidateInfo(ctx, formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *C11yWordsResponseIndividualWordsItems0) contextValidateInfo(ctx context.Context, formats strfmt.Registry) error { - - if m.Info != nil { - if err := m.Info.ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("info") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("info") - } - return err - } - } - - return nil -} - -// MarshalBinary interface implementation -func (m *C11yWordsResponseIndividualWordsItems0) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *C11yWordsResponseIndividualWordsItems0) UnmarshalBinary(b []byte) error { - var res C11yWordsResponseIndividualWordsItems0 - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} - -// C11yWordsResponseIndividualWordsItems0Info c11y words response individual words items0 info -// -// swagger:model C11yWordsResponseIndividualWordsItems0Info -type C11yWordsResponseIndividualWordsItems0Info struct { - - // nearest neighbors - NearestNeighbors C11yNearestNeighbors `json:"nearestNeighbors,omitempty"` - - // vector - Vector C11yVector `json:"vector,omitempty"` -} - -// Validate validates this c11y words response individual words items0 info -func (m *C11yWordsResponseIndividualWordsItems0Info) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateNearestNeighbors(formats); err != nil { - res = append(res, err) - } - - if err := m.validateVector(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *C11yWordsResponseIndividualWordsItems0Info) validateNearestNeighbors(formats strfmt.Registry) error { - if swag.IsZero(m.NearestNeighbors) { // not required - return nil - } - - if err := m.NearestNeighbors.Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("info" + "." + "nearestNeighbors") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("info" + "." + "nearestNeighbors") - } - return err - } - - return nil -} - -func (m *C11yWordsResponseIndividualWordsItems0Info) validateVector(formats strfmt.Registry) error { - if swag.IsZero(m.Vector) { // not required - return nil - } - - if err := m.Vector.Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("info" + "." + "vector") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("info" + "." + "vector") - } - return err - } - - return nil -} - -// ContextValidate validate this c11y words response individual words items0 info based on the context it is used -func (m *C11yWordsResponseIndividualWordsItems0Info) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - if err := m.contextValidateNearestNeighbors(ctx, formats); err != nil { - res = append(res, err) - } - - if err := m.contextValidateVector(ctx, formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *C11yWordsResponseIndividualWordsItems0Info) contextValidateNearestNeighbors(ctx context.Context, formats strfmt.Registry) error { - - if err := m.NearestNeighbors.ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("info" + "." + "nearestNeighbors") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("info" + "." + "nearestNeighbors") - } - return err - } - - return nil -} - -func (m *C11yWordsResponseIndividualWordsItems0Info) contextValidateVector(ctx context.Context, formats strfmt.Registry) error { - - if err := m.Vector.ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("info" + "." + "vector") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("info" + "." + "vector") - } - return err - } - - return nil -} - -// MarshalBinary interface implementation -func (m *C11yWordsResponseIndividualWordsItems0Info) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *C11yWordsResponseIndividualWordsItems0Info) UnmarshalBinary(b []byte) error { - var res C11yWordsResponseIndividualWordsItems0Info - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/class.go b/entities/models/class.go deleted file mode 100644 index 9c6247e300fa9b2de0560cf770d0a559bcb7de06..0000000000000000000000000000000000000000 --- a/entities/models/class.go +++ /dev/null @@ -1,286 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "strconv" - - "github.com/go-openapi/errors" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" -) - -// Class class -// -// swagger:model Class -type Class struct { - - // Name of the class as URI relative to the schema URL. - Class string `json:"class,omitempty"` - - // Description of the class. - Description string `json:"description,omitempty"` - - // inverted index config - InvertedIndexConfig *InvertedIndexConfig `json:"invertedIndexConfig,omitempty"` - - // Configuration specific to modules this Weaviate instance has installed - ModuleConfig interface{} `json:"moduleConfig,omitempty"` - - // multi tenancy config - MultiTenancyConfig *MultiTenancyConfig `json:"multiTenancyConfig,omitempty"` - - // The properties of the class. - Properties []*Property `json:"properties"` - - // replication config - ReplicationConfig *ReplicationConfig `json:"replicationConfig,omitempty"` - - // Manage how the index should be sharded and distributed in the cluster - ShardingConfig interface{} `json:"shardingConfig,omitempty"` - - // Vector-index config, that is specific to the type of index selected in vectorIndexType - VectorIndexConfig interface{} `json:"vectorIndexConfig,omitempty"` - - // Name of the vector index to use, eg. (HNSW) - VectorIndexType string `json:"vectorIndexType,omitempty"` - - // Specify how the vectors for this class should be determined. The options are either 'none' - this means you have to import a vector with each object yourself - or the name of a module that provides vectorization capabilities, such as 'text2vec-contextionary'. If left empty, it will use the globally configured default which can itself either be 'none' or a specific module. - Vectorizer string `json:"vectorizer,omitempty"` -} - -// Validate validates this class -func (m *Class) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateInvertedIndexConfig(formats); err != nil { - res = append(res, err) - } - - if err := m.validateMultiTenancyConfig(formats); err != nil { - res = append(res, err) - } - - if err := m.validateProperties(formats); err != nil { - res = append(res, err) - } - - if err := m.validateReplicationConfig(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *Class) validateInvertedIndexConfig(formats strfmt.Registry) error { - if swag.IsZero(m.InvertedIndexConfig) { // not required - return nil - } - - if m.InvertedIndexConfig != nil { - if err := m.InvertedIndexConfig.Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("invertedIndexConfig") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("invertedIndexConfig") - } - return err - } - } - - return nil -} - -func (m *Class) validateMultiTenancyConfig(formats strfmt.Registry) error { - if swag.IsZero(m.MultiTenancyConfig) { // not required - return nil - } - - if m.MultiTenancyConfig != nil { - if err := m.MultiTenancyConfig.Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("multiTenancyConfig") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("multiTenancyConfig") - } - return err - } - } - - return nil -} - -func (m *Class) validateProperties(formats strfmt.Registry) error { - if swag.IsZero(m.Properties) { // not required - return nil - } - - for i := 0; i < len(m.Properties); i++ { - if swag.IsZero(m.Properties[i]) { // not required - continue - } - - if m.Properties[i] != nil { - if err := m.Properties[i].Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("properties" + "." + strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("properties" + "." + strconv.Itoa(i)) - } - return err - } - } - - } - - return nil -} - -func (m *Class) validateReplicationConfig(formats strfmt.Registry) error { - if swag.IsZero(m.ReplicationConfig) { // not required - return nil - } - - if m.ReplicationConfig != nil { - if err := m.ReplicationConfig.Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("replicationConfig") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("replicationConfig") - } - return err - } - } - - return nil -} - -// ContextValidate validate this class based on the context it is used -func (m *Class) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - if err := m.contextValidateInvertedIndexConfig(ctx, formats); err != nil { - res = append(res, err) - } - - if err := m.contextValidateMultiTenancyConfig(ctx, formats); err != nil { - res = append(res, err) - } - - if err := m.contextValidateProperties(ctx, formats); err != nil { - res = append(res, err) - } - - if err := m.contextValidateReplicationConfig(ctx, formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *Class) contextValidateInvertedIndexConfig(ctx context.Context, formats strfmt.Registry) error { - - if m.InvertedIndexConfig != nil { - if err := m.InvertedIndexConfig.ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("invertedIndexConfig") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("invertedIndexConfig") - } - return err - } - } - - return nil -} - -func (m *Class) contextValidateMultiTenancyConfig(ctx context.Context, formats strfmt.Registry) error { - - if m.MultiTenancyConfig != nil { - if err := m.MultiTenancyConfig.ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("multiTenancyConfig") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("multiTenancyConfig") - } - return err - } - } - - return nil -} - -func (m *Class) contextValidateProperties(ctx context.Context, formats strfmt.Registry) error { - - for i := 0; i < len(m.Properties); i++ { - - if m.Properties[i] != nil { - if err := m.Properties[i].ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("properties" + "." + strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("properties" + "." + strconv.Itoa(i)) - } - return err - } - } - - } - - return nil -} - -func (m *Class) contextValidateReplicationConfig(ctx context.Context, formats strfmt.Registry) error { - - if m.ReplicationConfig != nil { - if err := m.ReplicationConfig.ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("replicationConfig") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("replicationConfig") - } - return err - } - } - - return nil -} - -// MarshalBinary interface implementation -func (m *Class) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *Class) UnmarshalBinary(b []byte) error { - var res Class - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/classification.go b/entities/models/classification.go deleted file mode 100644 index f7926122130173be115062c2807842016af94b8f..0000000000000000000000000000000000000000 --- a/entities/models/classification.go +++ /dev/null @@ -1,442 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "encoding/json" - - "github.com/go-openapi/errors" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" - "github.com/go-openapi/validate" -) - -// Classification Manage classifications, trigger them and view status of past classifications. -// -// swagger:model Classification -type Classification struct { - - // base the text-based classification on these fields (of type text) - // Example: ["description"] - BasedOnProperties []string `json:"basedOnProperties"` - - // class (name) which is used in this classification - // Example: City - Class string `json:"class,omitempty"` - - // which ref-property to set as part of the classification - // Example: ["inCountry"] - ClassifyProperties []string `json:"classifyProperties"` - - // error message if status == failed - // Example: classify xzy: something went wrong - Error string `json:"error,omitempty"` - - // filters - Filters *ClassificationFilters `json:"filters,omitempty"` - - // ID to uniquely identify this classification run - // Example: ee722219-b8ec-4db1-8f8d-5150bb1a9e0c - // Format: uuid - ID strfmt.UUID `json:"id,omitempty"` - - // additional meta information about the classification - Meta *ClassificationMeta `json:"meta,omitempty"` - - // classification-type specific settings - Settings interface{} `json:"settings,omitempty"` - - // status of this classification - // Example: running - // Enum: [running completed failed] - Status string `json:"status,omitempty"` - - // which algorithm to use for classifications - Type string `json:"type,omitempty"` -} - -// Validate validates this classification -func (m *Classification) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateFilters(formats); err != nil { - res = append(res, err) - } - - if err := m.validateID(formats); err != nil { - res = append(res, err) - } - - if err := m.validateMeta(formats); err != nil { - res = append(res, err) - } - - if err := m.validateStatus(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *Classification) validateFilters(formats strfmt.Registry) error { - if swag.IsZero(m.Filters) { // not required - return nil - } - - if m.Filters != nil { - if err := m.Filters.Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("filters") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("filters") - } - return err - } - } - - return nil -} - -func (m *Classification) validateID(formats strfmt.Registry) error { - if swag.IsZero(m.ID) { // not required - return nil - } - - if err := validate.FormatOf("id", "body", "uuid", m.ID.String(), formats); err != nil { - return err - } - - return nil -} - -func (m *Classification) validateMeta(formats strfmt.Registry) error { - if swag.IsZero(m.Meta) { // not required - return nil - } - - if m.Meta != nil { - if err := m.Meta.Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("meta") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("meta") - } - return err - } - } - - return nil -} - -var classificationTypeStatusPropEnum []interface{} - -func init() { - var res []string - if err := json.Unmarshal([]byte(`["running","completed","failed"]`), &res); err != nil { - panic(err) - } - for _, v := range res { - classificationTypeStatusPropEnum = append(classificationTypeStatusPropEnum, v) - } -} - -const ( - - // ClassificationStatusRunning captures enum value "running" - ClassificationStatusRunning string = "running" - - // ClassificationStatusCompleted captures enum value "completed" - ClassificationStatusCompleted string = "completed" - - // ClassificationStatusFailed captures enum value "failed" - ClassificationStatusFailed string = "failed" -) - -// prop value enum -func (m *Classification) validateStatusEnum(path, location string, value string) error { - if err := validate.EnumCase(path, location, value, classificationTypeStatusPropEnum, true); err != nil { - return err - } - return nil -} - -func (m *Classification) validateStatus(formats strfmt.Registry) error { - if swag.IsZero(m.Status) { // not required - return nil - } - - // value enum - if err := m.validateStatusEnum("status", "body", m.Status); err != nil { - return err - } - - return nil -} - -// ContextValidate validate this classification based on the context it is used -func (m *Classification) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - if err := m.contextValidateFilters(ctx, formats); err != nil { - res = append(res, err) - } - - if err := m.contextValidateMeta(ctx, formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *Classification) contextValidateFilters(ctx context.Context, formats strfmt.Registry) error { - - if m.Filters != nil { - if err := m.Filters.ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("filters") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("filters") - } - return err - } - } - - return nil -} - -func (m *Classification) contextValidateMeta(ctx context.Context, formats strfmt.Registry) error { - - if m.Meta != nil { - if err := m.Meta.ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("meta") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("meta") - } - return err - } - } - - return nil -} - -// MarshalBinary interface implementation -func (m *Classification) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *Classification) UnmarshalBinary(b []byte) error { - var res Classification - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} - -// ClassificationFilters classification filters -// -// swagger:model ClassificationFilters -type ClassificationFilters struct { - - // limit the objects to be classified - SourceWhere *WhereFilter `json:"sourceWhere,omitempty"` - - // Limit the possible sources when using an algorithm which doesn't really on training data, e.g. 'contextual'. When using an algorithm with a training set, such as 'knn', limit the training set instead - TargetWhere *WhereFilter `json:"targetWhere,omitempty"` - - // Limit the training objects to be considered during the classification. Can only be used on types with explicit training sets, such as 'knn' - TrainingSetWhere *WhereFilter `json:"trainingSetWhere,omitempty"` -} - -// Validate validates this classification filters -func (m *ClassificationFilters) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateSourceWhere(formats); err != nil { - res = append(res, err) - } - - if err := m.validateTargetWhere(formats); err != nil { - res = append(res, err) - } - - if err := m.validateTrainingSetWhere(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *ClassificationFilters) validateSourceWhere(formats strfmt.Registry) error { - if swag.IsZero(m.SourceWhere) { // not required - return nil - } - - if m.SourceWhere != nil { - if err := m.SourceWhere.Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("filters" + "." + "sourceWhere") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("filters" + "." + "sourceWhere") - } - return err - } - } - - return nil -} - -func (m *ClassificationFilters) validateTargetWhere(formats strfmt.Registry) error { - if swag.IsZero(m.TargetWhere) { // not required - return nil - } - - if m.TargetWhere != nil { - if err := m.TargetWhere.Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("filters" + "." + "targetWhere") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("filters" + "." + "targetWhere") - } - return err - } - } - - return nil -} - -func (m *ClassificationFilters) validateTrainingSetWhere(formats strfmt.Registry) error { - if swag.IsZero(m.TrainingSetWhere) { // not required - return nil - } - - if m.TrainingSetWhere != nil { - if err := m.TrainingSetWhere.Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("filters" + "." + "trainingSetWhere") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("filters" + "." + "trainingSetWhere") - } - return err - } - } - - return nil -} - -// ContextValidate validate this classification filters based on the context it is used -func (m *ClassificationFilters) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - if err := m.contextValidateSourceWhere(ctx, formats); err != nil { - res = append(res, err) - } - - if err := m.contextValidateTargetWhere(ctx, formats); err != nil { - res = append(res, err) - } - - if err := m.contextValidateTrainingSetWhere(ctx, formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *ClassificationFilters) contextValidateSourceWhere(ctx context.Context, formats strfmt.Registry) error { - - if m.SourceWhere != nil { - if err := m.SourceWhere.ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("filters" + "." + "sourceWhere") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("filters" + "." + "sourceWhere") - } - return err - } - } - - return nil -} - -func (m *ClassificationFilters) contextValidateTargetWhere(ctx context.Context, formats strfmt.Registry) error { - - if m.TargetWhere != nil { - if err := m.TargetWhere.ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("filters" + "." + "targetWhere") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("filters" + "." + "targetWhere") - } - return err - } - } - - return nil -} - -func (m *ClassificationFilters) contextValidateTrainingSetWhere(ctx context.Context, formats strfmt.Registry) error { - - if m.TrainingSetWhere != nil { - if err := m.TrainingSetWhere.ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("filters" + "." + "trainingSetWhere") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("filters" + "." + "trainingSetWhere") - } - return err - } - } - - return nil -} - -// MarshalBinary interface implementation -func (m *ClassificationFilters) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *ClassificationFilters) UnmarshalBinary(b []byte) error { - var res ClassificationFilters - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/classification_meta.go b/entities/models/classification_meta.go deleted file mode 100644 index 82083dbf8df605909e13cf2cb97e1c9ec17fb9ff..0000000000000000000000000000000000000000 --- a/entities/models/classification_meta.go +++ /dev/null @@ -1,119 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - - "github.com/go-openapi/errors" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" - "github.com/go-openapi/validate" -) - -// ClassificationMeta Additional information to a specific classification -// -// swagger:model ClassificationMeta -type ClassificationMeta struct { - - // time when this classification finished - // Example: 2017-07-21T17:32:28Z - // Format: date-time - Completed strfmt.DateTime `json:"completed,omitempty"` - - // number of objects which were taken into consideration for classification - // Example: 147 - Count int64 `json:"count,omitempty"` - - // number of objects which could not be classified - see error message for details - // Example: 7 - CountFailed int64 `json:"countFailed,omitempty"` - - // number of objects successfully classified - // Example: 140 - CountSucceeded int64 `json:"countSucceeded,omitempty"` - - // time when this classification was started - // Example: 2017-07-21T17:32:28Z - // Format: date-time - Started strfmt.DateTime `json:"started,omitempty"` -} - -// Validate validates this classification meta -func (m *ClassificationMeta) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateCompleted(formats); err != nil { - res = append(res, err) - } - - if err := m.validateStarted(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *ClassificationMeta) validateCompleted(formats strfmt.Registry) error { - if swag.IsZero(m.Completed) { // not required - return nil - } - - if err := validate.FormatOf("completed", "body", "date-time", m.Completed.String(), formats); err != nil { - return err - } - - return nil -} - -func (m *ClassificationMeta) validateStarted(formats strfmt.Registry) error { - if swag.IsZero(m.Started) { // not required - return nil - } - - if err := validate.FormatOf("started", "body", "date-time", m.Started.String(), formats); err != nil { - return err - } - - return nil -} - -// ContextValidate validates this classification meta based on context it is used -func (m *ClassificationMeta) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - return nil -} - -// MarshalBinary interface implementation -func (m *ClassificationMeta) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *ClassificationMeta) UnmarshalBinary(b []byte) error { - var res ClassificationMeta - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/deprecation.go b/entities/models/deprecation.go deleted file mode 100644 index 14628b945c021ad2e142aa2508e8fe59f96512c2..0000000000000000000000000000000000000000 --- a/entities/models/deprecation.go +++ /dev/null @@ -1,132 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - - "github.com/go-openapi/errors" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" - "github.com/go-openapi/validate" -) - -// Deprecation deprecation -// -// swagger:model Deprecation -type Deprecation struct { - - // Describes which API is effected, usually one of: REST, GraphQL - APIType string `json:"apiType,omitempty"` - - // The id that uniquely identifies this particular deprecations (mostly used internally) - ID string `json:"id,omitempty"` - - // The locations within the specified API affected by this deprecation - Locations []string `json:"locations"` - - // User-required object to not be affected by the (planned) removal - Mitigation string `json:"mitigation,omitempty"` - - // What this deprecation is about - Msg string `json:"msg,omitempty"` - - // A best-effort guess of which upcoming version will remove the feature entirely - PlannedRemovalVersion string `json:"plannedRemovalVersion,omitempty"` - - // If the feature has already been removed, it was removed in this version - RemovedIn *string `json:"removedIn,omitempty"` - - // If the feature has already been removed, it was removed at this timestamp - // Format: date-time - RemovedTime *strfmt.DateTime `json:"removedTime,omitempty"` - - // The deprecation was introduced in this version - // Format: date-time - SinceTime strfmt.DateTime `json:"sinceTime,omitempty"` - - // The deprecation was introduced in this version - SinceVersion string `json:"sinceVersion,omitempty"` - - // Whether the problematic API functionality is deprecated (planned to be removed) or already removed - Status string `json:"status,omitempty"` -} - -// Validate validates this deprecation -func (m *Deprecation) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateRemovedTime(formats); err != nil { - res = append(res, err) - } - - if err := m.validateSinceTime(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *Deprecation) validateRemovedTime(formats strfmt.Registry) error { - if swag.IsZero(m.RemovedTime) { // not required - return nil - } - - if err := validate.FormatOf("removedTime", "body", "date-time", m.RemovedTime.String(), formats); err != nil { - return err - } - - return nil -} - -func (m *Deprecation) validateSinceTime(formats strfmt.Registry) error { - if swag.IsZero(m.SinceTime) { // not required - return nil - } - - if err := validate.FormatOf("sinceTime", "body", "date-time", m.SinceTime.String(), formats); err != nil { - return err - } - - return nil -} - -// ContextValidate validates this deprecation based on context it is used -func (m *Deprecation) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - return nil -} - -// MarshalBinary interface implementation -func (m *Deprecation) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *Deprecation) UnmarshalBinary(b []byte) error { - var res Deprecation - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/error_response.go b/entities/models/error_response.go deleted file mode 100644 index aef73f2f59243b671f25748edc9678a060aad83f..0000000000000000000000000000000000000000 --- a/entities/models/error_response.go +++ /dev/null @@ -1,164 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "strconv" - - "github.com/go-openapi/errors" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" -) - -// ErrorResponse An error response given by Weaviate end-points. -// -// swagger:model ErrorResponse -type ErrorResponse struct { - - // error - Error []*ErrorResponseErrorItems0 `json:"error"` -} - -// Validate validates this error response -func (m *ErrorResponse) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateError(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *ErrorResponse) validateError(formats strfmt.Registry) error { - if swag.IsZero(m.Error) { // not required - return nil - } - - for i := 0; i < len(m.Error); i++ { - if swag.IsZero(m.Error[i]) { // not required - continue - } - - if m.Error[i] != nil { - if err := m.Error[i].Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("error" + "." + strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("error" + "." + strconv.Itoa(i)) - } - return err - } - } - - } - - return nil -} - -// ContextValidate validate this error response based on the context it is used -func (m *ErrorResponse) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - if err := m.contextValidateError(ctx, formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *ErrorResponse) contextValidateError(ctx context.Context, formats strfmt.Registry) error { - - for i := 0; i < len(m.Error); i++ { - - if m.Error[i] != nil { - if err := m.Error[i].ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("error" + "." + strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("error" + "." + strconv.Itoa(i)) - } - return err - } - } - - } - - return nil -} - -// MarshalBinary interface implementation -func (m *ErrorResponse) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *ErrorResponse) UnmarshalBinary(b []byte) error { - var res ErrorResponse - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} - -// ErrorResponseErrorItems0 error response error items0 -// -// swagger:model ErrorResponseErrorItems0 -type ErrorResponseErrorItems0 struct { - - // message - Message string `json:"message,omitempty"` -} - -// Validate validates this error response error items0 -func (m *ErrorResponseErrorItems0) Validate(formats strfmt.Registry) error { - return nil -} - -// ContextValidate validates this error response error items0 based on context it is used -func (m *ErrorResponseErrorItems0) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - return nil -} - -// MarshalBinary interface implementation -func (m *ErrorResponseErrorItems0) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *ErrorResponseErrorItems0) UnmarshalBinary(b []byte) error { - var res ErrorResponseErrorItems0 - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/geo_coordinates.go b/entities/models/geo_coordinates.go deleted file mode 100644 index 841fa30abd0a2f336dd4ef91d954315d80c44a7e..0000000000000000000000000000000000000000 --- a/entities/models/geo_coordinates.go +++ /dev/null @@ -1,64 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" -) - -// GeoCoordinates geo coordinates -// -// swagger:model GeoCoordinates -type GeoCoordinates struct { - - // The latitude of the point on earth in decimal form - Latitude *float32 `json:"latitude,omitempty"` - - // The longitude of the point on earth in decimal form - Longitude *float32 `json:"longitude,omitempty"` -} - -// Validate validates this geo coordinates -func (m *GeoCoordinates) Validate(formats strfmt.Registry) error { - return nil -} - -// ContextValidate validates this geo coordinates based on context it is used -func (m *GeoCoordinates) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - return nil -} - -// MarshalBinary interface implementation -func (m *GeoCoordinates) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *GeoCoordinates) UnmarshalBinary(b []byte) error { - var res GeoCoordinates - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/graph_q_l_error.go b/entities/models/graph_q_l_error.go deleted file mode 100644 index d403ca9809d01ba0ca07dd1ec8f35db2de5d24ad..0000000000000000000000000000000000000000 --- a/entities/models/graph_q_l_error.go +++ /dev/null @@ -1,173 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "strconv" - - "github.com/go-openapi/errors" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" -) - -// GraphQLError An error response caused by a GraphQL query. -// -// swagger:model GraphQLError -type GraphQLError struct { - - // locations - Locations []*GraphQLErrorLocationsItems0 `json:"locations"` - - // message - Message string `json:"message,omitempty"` - - // path - Path []string `json:"path"` -} - -// Validate validates this graph q l error -func (m *GraphQLError) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateLocations(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *GraphQLError) validateLocations(formats strfmt.Registry) error { - if swag.IsZero(m.Locations) { // not required - return nil - } - - for i := 0; i < len(m.Locations); i++ { - if swag.IsZero(m.Locations[i]) { // not required - continue - } - - if m.Locations[i] != nil { - if err := m.Locations[i].Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("locations" + "." + strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("locations" + "." + strconv.Itoa(i)) - } - return err - } - } - - } - - return nil -} - -// ContextValidate validate this graph q l error based on the context it is used -func (m *GraphQLError) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - if err := m.contextValidateLocations(ctx, formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *GraphQLError) contextValidateLocations(ctx context.Context, formats strfmt.Registry) error { - - for i := 0; i < len(m.Locations); i++ { - - if m.Locations[i] != nil { - if err := m.Locations[i].ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("locations" + "." + strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("locations" + "." + strconv.Itoa(i)) - } - return err - } - } - - } - - return nil -} - -// MarshalBinary interface implementation -func (m *GraphQLError) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *GraphQLError) UnmarshalBinary(b []byte) error { - var res GraphQLError - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} - -// GraphQLErrorLocationsItems0 graph q l error locations items0 -// -// swagger:model GraphQLErrorLocationsItems0 -type GraphQLErrorLocationsItems0 struct { - - // column - Column int64 `json:"column,omitempty"` - - // line - Line int64 `json:"line,omitempty"` -} - -// Validate validates this graph q l error locations items0 -func (m *GraphQLErrorLocationsItems0) Validate(formats strfmt.Registry) error { - return nil -} - -// ContextValidate validates this graph q l error locations items0 based on context it is used -func (m *GraphQLErrorLocationsItems0) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - return nil -} - -// MarshalBinary interface implementation -func (m *GraphQLErrorLocationsItems0) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *GraphQLErrorLocationsItems0) UnmarshalBinary(b []byte) error { - var res GraphQLErrorLocationsItems0 - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/graph_q_l_queries.go b/entities/models/graph_q_l_queries.go deleted file mode 100644 index 00fa306e2e5410f5ffb0762140349020d675a38b..0000000000000000000000000000000000000000 --- a/entities/models/graph_q_l_queries.go +++ /dev/null @@ -1,84 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "strconv" - - "github.com/go-openapi/errors" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" -) - -// GraphQLQueries A list of GraphQL queries. -// -// swagger:model GraphQLQueries -type GraphQLQueries []*GraphQLQuery - -// Validate validates this graph q l queries -func (m GraphQLQueries) Validate(formats strfmt.Registry) error { - var res []error - - for i := 0; i < len(m); i++ { - if swag.IsZero(m[i]) { // not required - continue - } - - if m[i] != nil { - if err := m[i].Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName(strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName(strconv.Itoa(i)) - } - return err - } - } - - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// ContextValidate validate this graph q l queries based on the context it is used -func (m GraphQLQueries) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - for i := 0; i < len(m); i++ { - - if m[i] != nil { - if err := m[i].ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName(strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName(strconv.Itoa(i)) - } - return err - } - } - - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/entities/models/graph_q_l_query.go b/entities/models/graph_q_l_query.go deleted file mode 100644 index 906151df6a69f5778949eeade9c51ac060841586..0000000000000000000000000000000000000000 --- a/entities/models/graph_q_l_query.go +++ /dev/null @@ -1,67 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" -) - -// GraphQLQuery GraphQL query based on: http://facebook.github.io/graphql/. -// -// swagger:model GraphQLQuery -type GraphQLQuery struct { - - // The name of the operation if multiple exist in the query. - OperationName string `json:"operationName,omitempty"` - - // Query based on GraphQL syntax. - Query string `json:"query,omitempty"` - - // Additional variables for the query. - Variables interface{} `json:"variables,omitempty"` -} - -// Validate validates this graph q l query -func (m *GraphQLQuery) Validate(formats strfmt.Registry) error { - return nil -} - -// ContextValidate validates this graph q l query based on context it is used -func (m *GraphQLQuery) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - return nil -} - -// MarshalBinary interface implementation -func (m *GraphQLQuery) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *GraphQLQuery) UnmarshalBinary(b []byte) error { - var res GraphQLQuery - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/graph_q_l_response.go b/entities/models/graph_q_l_response.go deleted file mode 100644 index eba69285775466c5c945d6537b934264c771ef43..0000000000000000000000000000000000000000 --- a/entities/models/graph_q_l_response.go +++ /dev/null @@ -1,130 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "strconv" - - "github.com/go-openapi/errors" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" -) - -// GraphQLResponse GraphQL based response: http://facebook.github.io/graphql/. -// -// swagger:model GraphQLResponse -type GraphQLResponse struct { - - // GraphQL data object. - Data map[string]JSONObject `json:"data,omitempty"` - - // Array with errors. - Errors []*GraphQLError `json:"errors,omitempty"` -} - -// Validate validates this graph q l response -func (m *GraphQLResponse) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateErrors(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *GraphQLResponse) validateErrors(formats strfmt.Registry) error { - if swag.IsZero(m.Errors) { // not required - return nil - } - - for i := 0; i < len(m.Errors); i++ { - if swag.IsZero(m.Errors[i]) { // not required - continue - } - - if m.Errors[i] != nil { - if err := m.Errors[i].Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("errors" + "." + strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("errors" + "." + strconv.Itoa(i)) - } - return err - } - } - - } - - return nil -} - -// ContextValidate validate this graph q l response based on the context it is used -func (m *GraphQLResponse) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - if err := m.contextValidateErrors(ctx, formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *GraphQLResponse) contextValidateErrors(ctx context.Context, formats strfmt.Registry) error { - - for i := 0; i < len(m.Errors); i++ { - - if m.Errors[i] != nil { - if err := m.Errors[i].ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("errors" + "." + strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("errors" + "." + strconv.Itoa(i)) - } - return err - } - } - - } - - return nil -} - -// MarshalBinary interface implementation -func (m *GraphQLResponse) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *GraphQLResponse) UnmarshalBinary(b []byte) error { - var res GraphQLResponse - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/graph_q_l_responses.go b/entities/models/graph_q_l_responses.go deleted file mode 100644 index 48199b5ad949e2e654a058a63d4b38616ee9f8bb..0000000000000000000000000000000000000000 --- a/entities/models/graph_q_l_responses.go +++ /dev/null @@ -1,84 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "strconv" - - "github.com/go-openapi/errors" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" -) - -// GraphQLResponses A list of GraphQL responses. -// -// swagger:model GraphQLResponses -type GraphQLResponses []*GraphQLResponse - -// Validate validates this graph q l responses -func (m GraphQLResponses) Validate(formats strfmt.Registry) error { - var res []error - - for i := 0; i < len(m); i++ { - if swag.IsZero(m[i]) { // not required - continue - } - - if m[i] != nil { - if err := m[i].Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName(strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName(strconv.Itoa(i)) - } - return err - } - } - - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// ContextValidate validate this graph q l responses based on the context it is used -func (m GraphQLResponses) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - for i := 0; i < len(m); i++ { - - if m[i] != nil { - if err := m[i].ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName(strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName(strconv.Itoa(i)) - } - return err - } - } - - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/entities/models/inverted_index_config.go b/entities/models/inverted_index_config.go deleted file mode 100644 index 83ba95da04ffdb25672100a8148e1033457c7414..0000000000000000000000000000000000000000 --- a/entities/models/inverted_index_config.go +++ /dev/null @@ -1,173 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - - "github.com/go-openapi/errors" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" -) - -// InvertedIndexConfig Configure the inverted index built into Weaviate -// -// swagger:model InvertedIndexConfig -type InvertedIndexConfig struct { - - // bm25 - Bm25 *BM25Config `json:"bm25,omitempty"` - - // Asynchronous index clean up happens every n seconds - CleanupIntervalSeconds int64 `json:"cleanupIntervalSeconds,omitempty"` - - // Index each object with the null state - IndexNullState bool `json:"indexNullState,omitempty"` - - // Index length of properties - IndexPropertyLength bool `json:"indexPropertyLength,omitempty"` - - // Index each object by its internal timestamps - IndexTimestamps bool `json:"indexTimestamps,omitempty"` - - // stopwords - Stopwords *StopwordConfig `json:"stopwords,omitempty"` -} - -// Validate validates this inverted index config -func (m *InvertedIndexConfig) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateBm25(formats); err != nil { - res = append(res, err) - } - - if err := m.validateStopwords(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *InvertedIndexConfig) validateBm25(formats strfmt.Registry) error { - if swag.IsZero(m.Bm25) { // not required - return nil - } - - if m.Bm25 != nil { - if err := m.Bm25.Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("bm25") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("bm25") - } - return err - } - } - - return nil -} - -func (m *InvertedIndexConfig) validateStopwords(formats strfmt.Registry) error { - if swag.IsZero(m.Stopwords) { // not required - return nil - } - - if m.Stopwords != nil { - if err := m.Stopwords.Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("stopwords") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("stopwords") - } - return err - } - } - - return nil -} - -// ContextValidate validate this inverted index config based on the context it is used -func (m *InvertedIndexConfig) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - if err := m.contextValidateBm25(ctx, formats); err != nil { - res = append(res, err) - } - - if err := m.contextValidateStopwords(ctx, formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *InvertedIndexConfig) contextValidateBm25(ctx context.Context, formats strfmt.Registry) error { - - if m.Bm25 != nil { - if err := m.Bm25.ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("bm25") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("bm25") - } - return err - } - } - - return nil -} - -func (m *InvertedIndexConfig) contextValidateStopwords(ctx context.Context, formats strfmt.Registry) error { - - if m.Stopwords != nil { - if err := m.Stopwords.ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("stopwords") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("stopwords") - } - return err - } - } - - return nil -} - -// MarshalBinary interface implementation -func (m *InvertedIndexConfig) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *InvertedIndexConfig) UnmarshalBinary(b []byte) error { - var res InvertedIndexConfig - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/json_object.go b/entities/models/json_object.go deleted file mode 100644 index 41bc53bf96079527aa5f264af8124c7a31edddd4..0000000000000000000000000000000000000000 --- a/entities/models/json_object.go +++ /dev/null @@ -1,22 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -// JSONObject JSON object value. -// -// swagger:model JsonObject -type JSONObject interface{} diff --git a/entities/models/link.go b/entities/models/link.go deleted file mode 100644 index 83d5145258fdbf0352b303a5dd0c361a7bc3e593..0000000000000000000000000000000000000000 --- a/entities/models/link.go +++ /dev/null @@ -1,70 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" -) - -// Link link -// -// swagger:model Link -type Link struct { - - // weaviate documentation about this resource group - DocumentationHref string `json:"documentationHref,omitempty"` - - // target of the link - Href string `json:"href,omitempty"` - - // human readable name of the resource group - Name string `json:"name,omitempty"` - - // relationship if both resources are related, e.g. 'next', 'previous', 'parent', etc. - Rel string `json:"rel,omitempty"` -} - -// Validate validates this link -func (m *Link) Validate(formats strfmt.Registry) error { - return nil -} - -// ContextValidate validates this link based on context it is used -func (m *Link) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - return nil -} - -// MarshalBinary interface implementation -func (m *Link) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *Link) UnmarshalBinary(b []byte) error { - var res Link - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/meta.go b/entities/models/meta.go deleted file mode 100644 index 6ec6828f5eceff720995cbfbfbad5a0ea54a8bd4..0000000000000000000000000000000000000000 --- a/entities/models/meta.go +++ /dev/null @@ -1,67 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" -) - -// Meta Contains meta information of the current Weaviate instance. -// -// swagger:model Meta -type Meta struct { - - // The url of the host. - Hostname string `json:"hostname,omitempty"` - - // Module-specific meta information - Modules interface{} `json:"modules,omitempty"` - - // Version of weaviate you are currently running - Version string `json:"version,omitempty"` -} - -// Validate validates this meta -func (m *Meta) Validate(formats strfmt.Registry) error { - return nil -} - -// ContextValidate validates this meta based on context it is used -func (m *Meta) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - return nil -} - -// MarshalBinary interface implementation -func (m *Meta) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *Meta) UnmarshalBinary(b []byte) error { - var res Meta - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/multi_tenancy_config.go b/entities/models/multi_tenancy_config.go deleted file mode 100644 index 38fcf0f7791c82c0c8b5e7bc92398afb18309406..0000000000000000000000000000000000000000 --- a/entities/models/multi_tenancy_config.go +++ /dev/null @@ -1,61 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" -) - -// MultiTenancyConfig Configuration related to multi-tenancy within a class -// -// swagger:model MultiTenancyConfig -type MultiTenancyConfig struct { - - // Whether or not multi-tenancy is enabled for this class - Enabled bool `json:"enabled"` -} - -// Validate validates this multi tenancy config -func (m *MultiTenancyConfig) Validate(formats strfmt.Registry) error { - return nil -} - -// ContextValidate validates this multi tenancy config based on context it is used -func (m *MultiTenancyConfig) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - return nil -} - -// MarshalBinary interface implementation -func (m *MultiTenancyConfig) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *MultiTenancyConfig) UnmarshalBinary(b []byte) error { - var res MultiTenancyConfig - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/multiple_ref.go b/entities/models/multiple_ref.go deleted file mode 100644 index 6065e888fdcd3726fda9252a3d26393ec1c30009..0000000000000000000000000000000000000000 --- a/entities/models/multiple_ref.go +++ /dev/null @@ -1,84 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "strconv" - - "github.com/go-openapi/errors" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" -) - -// MultipleRef Multiple instances of references to other objects. -// -// swagger:model MultipleRef -type MultipleRef []*SingleRef - -// Validate validates this multiple ref -func (m MultipleRef) Validate(formats strfmt.Registry) error { - var res []error - - for i := 0; i < len(m); i++ { - if swag.IsZero(m[i]) { // not required - continue - } - - if m[i] != nil { - if err := m[i].Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName(strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName(strconv.Itoa(i)) - } - return err - } - } - - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// ContextValidate validate this multiple ref based on the context it is used -func (m MultipleRef) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - for i := 0; i < len(m); i++ { - - if m[i] != nil { - if err := m[i].ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName(strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName(strconv.Itoa(i)) - } - return err - } - } - - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/entities/models/nested_property.go b/entities/models/nested_property.go deleted file mode 100644 index 60ed9837959ffed00ebac57e2fe2bb24b62d1acb..0000000000000000000000000000000000000000 --- a/entities/models/nested_property.go +++ /dev/null @@ -1,200 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "encoding/json" - "strconv" - - "github.com/go-openapi/errors" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" - "github.com/go-openapi/validate" -) - -// NestedProperty nested property -// -// swagger:model NestedProperty -type NestedProperty struct { - - // data type - DataType []string `json:"dataType"` - - // description - Description string `json:"description,omitempty"` - - // index filterable - IndexFilterable *bool `json:"indexFilterable,omitempty"` - - // index searchable - IndexSearchable *bool `json:"indexSearchable,omitempty"` - - // name - Name string `json:"name,omitempty"` - - // nested properties - NestedProperties []*NestedProperty `json:"nestedProperties,omitempty"` - - // tokenization - // Enum: [word lowercase whitespace field] - Tokenization string `json:"tokenization,omitempty"` -} - -// Validate validates this nested property -func (m *NestedProperty) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateNestedProperties(formats); err != nil { - res = append(res, err) - } - - if err := m.validateTokenization(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *NestedProperty) validateNestedProperties(formats strfmt.Registry) error { - if swag.IsZero(m.NestedProperties) { // not required - return nil - } - - for i := 0; i < len(m.NestedProperties); i++ { - if swag.IsZero(m.NestedProperties[i]) { // not required - continue - } - - if m.NestedProperties[i] != nil { - if err := m.NestedProperties[i].Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("nestedProperties" + "." + strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("nestedProperties" + "." + strconv.Itoa(i)) - } - return err - } - } - - } - - return nil -} - -var nestedPropertyTypeTokenizationPropEnum []interface{} - -func init() { - var res []string - if err := json.Unmarshal([]byte(`["word","lowercase","whitespace","field"]`), &res); err != nil { - panic(err) - } - for _, v := range res { - nestedPropertyTypeTokenizationPropEnum = append(nestedPropertyTypeTokenizationPropEnum, v) - } -} - -const ( - - // NestedPropertyTokenizationWord captures enum value "word" - NestedPropertyTokenizationWord string = "word" - - // NestedPropertyTokenizationLowercase captures enum value "lowercase" - NestedPropertyTokenizationLowercase string = "lowercase" - - // NestedPropertyTokenizationWhitespace captures enum value "whitespace" - NestedPropertyTokenizationWhitespace string = "whitespace" - - // NestedPropertyTokenizationField captures enum value "field" - NestedPropertyTokenizationField string = "field" -) - -// prop value enum -func (m *NestedProperty) validateTokenizationEnum(path, location string, value string) error { - if err := validate.EnumCase(path, location, value, nestedPropertyTypeTokenizationPropEnum, true); err != nil { - return err - } - return nil -} - -func (m *NestedProperty) validateTokenization(formats strfmt.Registry) error { - if swag.IsZero(m.Tokenization) { // not required - return nil - } - - // value enum - if err := m.validateTokenizationEnum("tokenization", "body", m.Tokenization); err != nil { - return err - } - - return nil -} - -// ContextValidate validate this nested property based on the context it is used -func (m *NestedProperty) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - if err := m.contextValidateNestedProperties(ctx, formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *NestedProperty) contextValidateNestedProperties(ctx context.Context, formats strfmt.Registry) error { - - for i := 0; i < len(m.NestedProperties); i++ { - - if m.NestedProperties[i] != nil { - if err := m.NestedProperties[i].ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("nestedProperties" + "." + strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("nestedProperties" + "." + strconv.Itoa(i)) - } - return err - } - } - - } - - return nil -} - -// MarshalBinary interface implementation -func (m *NestedProperty) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *NestedProperty) UnmarshalBinary(b []byte) error { - var res NestedProperty - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/node_shard_status.go b/entities/models/node_shard_status.go deleted file mode 100644 index 142578c0a7a9cfd95a5a939706c6841adda2b721..0000000000000000000000000000000000000000 --- a/entities/models/node_shard_status.go +++ /dev/null @@ -1,76 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" -) - -// NodeShardStatus The definition of a node shard status response body -// -// swagger:model NodeShardStatus -type NodeShardStatus struct { - - // The name of shard's class. - Class string `json:"class"` - - // The status of vector compression/quantization. - Compressed bool `json:"compressed"` - - // The name of the shard. - Name string `json:"name"` - - // The number of objects in shard. - ObjectCount int64 `json:"objectCount"` - - // The status of the vector indexing process. - VectorIndexingStatus string `json:"vectorIndexingStatus"` - - // The length of the vector indexing queue. - VectorQueueLength int64 `json:"vectorQueueLength"` -} - -// Validate validates this node shard status -func (m *NodeShardStatus) Validate(formats strfmt.Registry) error { - return nil -} - -// ContextValidate validates this node shard status based on context it is used -func (m *NodeShardStatus) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - return nil -} - -// MarshalBinary interface implementation -func (m *NodeShardStatus) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *NodeShardStatus) UnmarshalBinary(b []byte) error { - var res NodeShardStatus - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/node_stats.go b/entities/models/node_stats.go deleted file mode 100644 index 379f9e7122c1bc9e079accdbfb31463607ee144d..0000000000000000000000000000000000000000 --- a/entities/models/node_stats.go +++ /dev/null @@ -1,64 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" -) - -// NodeStats The summary of Weaviate's statistics. -// -// swagger:model NodeStats -type NodeStats struct { - - // The total number of objects in DB. - ObjectCount int64 `json:"objectCount"` - - // The count of Weaviate's shards. - ShardCount int64 `json:"shardCount"` -} - -// Validate validates this node stats -func (m *NodeStats) Validate(formats strfmt.Registry) error { - return nil -} - -// ContextValidate validates this node stats based on context it is used -func (m *NodeStats) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - return nil -} - -// MarshalBinary interface implementation -func (m *NodeStats) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *NodeStats) UnmarshalBinary(b []byte) error { - var res NodeStats - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/node_status.go b/entities/models/node_status.go deleted file mode 100644 index 1c04140f2913170377edaf7cf7400ddcb45f694b..0000000000000000000000000000000000000000 --- a/entities/models/node_status.go +++ /dev/null @@ -1,283 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "encoding/json" - "strconv" - - "github.com/go-openapi/errors" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" - "github.com/go-openapi/validate" -) - -// NodeStatus The definition of a backup node status response body -// -// swagger:model NodeStatus -type NodeStatus struct { - - // Weaviate batch statistics. - BatchStats *BatchStats `json:"batchStats,omitempty"` - - // The gitHash of Weaviate. - GitHash string `json:"gitHash,omitempty"` - - // The name of the node. - Name string `json:"name,omitempty"` - - // The list of the shards with it's statistics. - Shards []*NodeShardStatus `json:"shards"` - - // Weaviate overall statistics. - Stats *NodeStats `json:"stats,omitempty"` - - // Node's status. - // Enum: [HEALTHY UNHEALTHY UNAVAILABLE] - Status *string `json:"status,omitempty"` - - // The version of Weaviate. - Version string `json:"version,omitempty"` -} - -// Validate validates this node status -func (m *NodeStatus) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateBatchStats(formats); err != nil { - res = append(res, err) - } - - if err := m.validateShards(formats); err != nil { - res = append(res, err) - } - - if err := m.validateStats(formats); err != nil { - res = append(res, err) - } - - if err := m.validateStatus(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *NodeStatus) validateBatchStats(formats strfmt.Registry) error { - if swag.IsZero(m.BatchStats) { // not required - return nil - } - - if m.BatchStats != nil { - if err := m.BatchStats.Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("batchStats") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("batchStats") - } - return err - } - } - - return nil -} - -func (m *NodeStatus) validateShards(formats strfmt.Registry) error { - if swag.IsZero(m.Shards) { // not required - return nil - } - - for i := 0; i < len(m.Shards); i++ { - if swag.IsZero(m.Shards[i]) { // not required - continue - } - - if m.Shards[i] != nil { - if err := m.Shards[i].Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("shards" + "." + strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("shards" + "." + strconv.Itoa(i)) - } - return err - } - } - - } - - return nil -} - -func (m *NodeStatus) validateStats(formats strfmt.Registry) error { - if swag.IsZero(m.Stats) { // not required - return nil - } - - if m.Stats != nil { - if err := m.Stats.Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("stats") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("stats") - } - return err - } - } - - return nil -} - -var nodeStatusTypeStatusPropEnum []interface{} - -func init() { - var res []string - if err := json.Unmarshal([]byte(`["HEALTHY","UNHEALTHY","UNAVAILABLE"]`), &res); err != nil { - panic(err) - } - for _, v := range res { - nodeStatusTypeStatusPropEnum = append(nodeStatusTypeStatusPropEnum, v) - } -} - -const ( - - // NodeStatusStatusHEALTHY captures enum value "HEALTHY" - NodeStatusStatusHEALTHY string = "HEALTHY" - - // NodeStatusStatusUNHEALTHY captures enum value "UNHEALTHY" - NodeStatusStatusUNHEALTHY string = "UNHEALTHY" - - // NodeStatusStatusUNAVAILABLE captures enum value "UNAVAILABLE" - NodeStatusStatusUNAVAILABLE string = "UNAVAILABLE" -) - -// prop value enum -func (m *NodeStatus) validateStatusEnum(path, location string, value string) error { - if err := validate.EnumCase(path, location, value, nodeStatusTypeStatusPropEnum, true); err != nil { - return err - } - return nil -} - -func (m *NodeStatus) validateStatus(formats strfmt.Registry) error { - if swag.IsZero(m.Status) { // not required - return nil - } - - // value enum - if err := m.validateStatusEnum("status", "body", *m.Status); err != nil { - return err - } - - return nil -} - -// ContextValidate validate this node status based on the context it is used -func (m *NodeStatus) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - if err := m.contextValidateBatchStats(ctx, formats); err != nil { - res = append(res, err) - } - - if err := m.contextValidateShards(ctx, formats); err != nil { - res = append(res, err) - } - - if err := m.contextValidateStats(ctx, formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *NodeStatus) contextValidateBatchStats(ctx context.Context, formats strfmt.Registry) error { - - if m.BatchStats != nil { - if err := m.BatchStats.ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("batchStats") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("batchStats") - } - return err - } - } - - return nil -} - -func (m *NodeStatus) contextValidateShards(ctx context.Context, formats strfmt.Registry) error { - - for i := 0; i < len(m.Shards); i++ { - - if m.Shards[i] != nil { - if err := m.Shards[i].ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("shards" + "." + strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("shards" + "." + strconv.Itoa(i)) - } - return err - } - } - - } - - return nil -} - -func (m *NodeStatus) contextValidateStats(ctx context.Context, formats strfmt.Registry) error { - - if m.Stats != nil { - if err := m.Stats.ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("stats") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("stats") - } - return err - } - } - - return nil -} - -// MarshalBinary interface implementation -func (m *NodeStatus) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *NodeStatus) UnmarshalBinary(b []byte) error { - var res NodeStatus - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/nodes_status_response.go b/entities/models/nodes_status_response.go deleted file mode 100644 index 5e1c773cf64efe42c3cf0d81db9a31dca415166e..0000000000000000000000000000000000000000 --- a/entities/models/nodes_status_response.go +++ /dev/null @@ -1,127 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "strconv" - - "github.com/go-openapi/errors" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" -) - -// NodesStatusResponse The status of all of the Weaviate nodes -// -// swagger:model NodesStatusResponse -type NodesStatusResponse struct { - - // nodes - Nodes []*NodeStatus `json:"nodes"` -} - -// Validate validates this nodes status response -func (m *NodesStatusResponse) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateNodes(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *NodesStatusResponse) validateNodes(formats strfmt.Registry) error { - if swag.IsZero(m.Nodes) { // not required - return nil - } - - for i := 0; i < len(m.Nodes); i++ { - if swag.IsZero(m.Nodes[i]) { // not required - continue - } - - if m.Nodes[i] != nil { - if err := m.Nodes[i].Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("nodes" + "." + strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("nodes" + "." + strconv.Itoa(i)) - } - return err - } - } - - } - - return nil -} - -// ContextValidate validate this nodes status response based on the context it is used -func (m *NodesStatusResponse) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - if err := m.contextValidateNodes(ctx, formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *NodesStatusResponse) contextValidateNodes(ctx context.Context, formats strfmt.Registry) error { - - for i := 0; i < len(m.Nodes); i++ { - - if m.Nodes[i] != nil { - if err := m.Nodes[i].ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("nodes" + "." + strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("nodes" + "." + strconv.Itoa(i)) - } - return err - } - } - - } - - return nil -} - -// MarshalBinary interface implementation -func (m *NodesStatusResponse) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *NodesStatusResponse) UnmarshalBinary(b []byte) error { - var res NodesStatusResponse - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/object.go b/entities/models/object.go deleted file mode 100644 index 5e8d90fcee87d85dc68159363dc8d759b3bb98be..0000000000000000000000000000000000000000 --- a/entities/models/object.go +++ /dev/null @@ -1,194 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - - "github.com/go-openapi/errors" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" - "github.com/go-openapi/validate" -) - -// Object object -// -// swagger:model Object -type Object struct { - - // additional - Additional AdditionalProperties `json:"additional,omitempty"` - - // Class of the Object, defined in the schema. - Class string `json:"class,omitempty"` - - // Timestamp of creation of this Object in milliseconds since epoch UTC. - CreationTimeUnix int64 `json:"creationTimeUnix,omitempty"` - - // ID of the Object. - // Format: uuid - ID strfmt.UUID `json:"id,omitempty"` - - // Timestamp of the last Object update in milliseconds since epoch UTC. - LastUpdateTimeUnix int64 `json:"lastUpdateTimeUnix,omitempty"` - - // properties - Properties PropertySchema `json:"properties,omitempty"` - - // Name of the Objects tenant. - Tenant string `json:"tenant,omitempty"` - - // This object's position in the Contextionary vector space. Read-only if using a vectorizer other than 'none'. Writable and required if using 'none' as vectorizer. - Vector C11yVector `json:"vector,omitempty"` - - // vector weights - VectorWeights VectorWeights `json:"vectorWeights,omitempty"` -} - -// Validate validates this object -func (m *Object) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateAdditional(formats); err != nil { - res = append(res, err) - } - - if err := m.validateID(formats); err != nil { - res = append(res, err) - } - - if err := m.validateVector(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *Object) validateAdditional(formats strfmt.Registry) error { - if swag.IsZero(m.Additional) { // not required - return nil - } - - if m.Additional != nil { - if err := m.Additional.Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("additional") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("additional") - } - return err - } - } - - return nil -} - -func (m *Object) validateID(formats strfmt.Registry) error { - if swag.IsZero(m.ID) { // not required - return nil - } - - if err := validate.FormatOf("id", "body", "uuid", m.ID.String(), formats); err != nil { - return err - } - - return nil -} - -func (m *Object) validateVector(formats strfmt.Registry) error { - if swag.IsZero(m.Vector) { // not required - return nil - } - - if err := m.Vector.Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("vector") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("vector") - } - return err - } - - return nil -} - -// ContextValidate validate this object based on the context it is used -func (m *Object) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - if err := m.contextValidateAdditional(ctx, formats); err != nil { - res = append(res, err) - } - - if err := m.contextValidateVector(ctx, formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *Object) contextValidateAdditional(ctx context.Context, formats strfmt.Registry) error { - - if err := m.Additional.ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("additional") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("additional") - } - return err - } - - return nil -} - -func (m *Object) contextValidateVector(ctx context.Context, formats strfmt.Registry) error { - - if err := m.Vector.ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("vector") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("vector") - } - return err - } - - return nil -} - -// MarshalBinary interface implementation -func (m *Object) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *Object) UnmarshalBinary(b []byte) error { - var res Object - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/objects_get_response.go b/entities/models/objects_get_response.go deleted file mode 100644 index 8f9ac32efc21287a18c12d2b7295fd368b8f6557..0000000000000000000000000000000000000000 --- a/entities/models/objects_get_response.go +++ /dev/null @@ -1,397 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "encoding/json" - "strconv" - - "github.com/go-openapi/errors" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" - "github.com/go-openapi/validate" -) - -// ObjectsGetResponse objects get response -// -// swagger:model ObjectsGetResponse -type ObjectsGetResponse struct { - Object - - // deprecations - Deprecations []*Deprecation `json:"deprecations"` - - // result - Result *ObjectsGetResponseAO2Result `json:"result,omitempty"` -} - -// UnmarshalJSON unmarshals this object from a JSON structure -func (m *ObjectsGetResponse) UnmarshalJSON(raw []byte) error { - // AO0 - var aO0 Object - if err := swag.ReadJSON(raw, &aO0); err != nil { - return err - } - m.Object = aO0 - - // AO1 - var dataAO1 struct { - Deprecations []*Deprecation `json:"deprecations"` - } - if err := swag.ReadJSON(raw, &dataAO1); err != nil { - return err - } - - m.Deprecations = dataAO1.Deprecations - - // AO2 - var dataAO2 struct { - Result *ObjectsGetResponseAO2Result `json:"result,omitempty"` - } - if err := swag.ReadJSON(raw, &dataAO2); err != nil { - return err - } - - m.Result = dataAO2.Result - - return nil -} - -// MarshalJSON marshals this object to a JSON structure -func (m ObjectsGetResponse) MarshalJSON() ([]byte, error) { - _parts := make([][]byte, 0, 3) - - aO0, err := swag.WriteJSON(m.Object) - if err != nil { - return nil, err - } - _parts = append(_parts, aO0) - var dataAO1 struct { - Deprecations []*Deprecation `json:"deprecations"` - } - - dataAO1.Deprecations = m.Deprecations - - jsonDataAO1, errAO1 := swag.WriteJSON(dataAO1) - if errAO1 != nil { - return nil, errAO1 - } - _parts = append(_parts, jsonDataAO1) - var dataAO2 struct { - Result *ObjectsGetResponseAO2Result `json:"result,omitempty"` - } - - dataAO2.Result = m.Result - - jsonDataAO2, errAO2 := swag.WriteJSON(dataAO2) - if errAO2 != nil { - return nil, errAO2 - } - _parts = append(_parts, jsonDataAO2) - return swag.ConcatJSON(_parts...), nil -} - -// Validate validates this objects get response -func (m *ObjectsGetResponse) Validate(formats strfmt.Registry) error { - var res []error - - // validation for a type composition with Object - if err := m.Object.Validate(formats); err != nil { - res = append(res, err) - } - - if err := m.validateDeprecations(formats); err != nil { - res = append(res, err) - } - - if err := m.validateResult(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *ObjectsGetResponse) validateDeprecations(formats strfmt.Registry) error { - - if swag.IsZero(m.Deprecations) { // not required - return nil - } - - for i := 0; i < len(m.Deprecations); i++ { - if swag.IsZero(m.Deprecations[i]) { // not required - continue - } - - if m.Deprecations[i] != nil { - if err := m.Deprecations[i].Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("deprecations" + "." + strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("deprecations" + "." + strconv.Itoa(i)) - } - return err - } - } - - } - - return nil -} - -func (m *ObjectsGetResponse) validateResult(formats strfmt.Registry) error { - - if swag.IsZero(m.Result) { // not required - return nil - } - - if m.Result != nil { - if err := m.Result.Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("result") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("result") - } - return err - } - } - - return nil -} - -// ContextValidate validate this objects get response based on the context it is used -func (m *ObjectsGetResponse) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - // validation for a type composition with Object - if err := m.Object.ContextValidate(ctx, formats); err != nil { - res = append(res, err) - } - - if err := m.contextValidateDeprecations(ctx, formats); err != nil { - res = append(res, err) - } - - if err := m.contextValidateResult(ctx, formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *ObjectsGetResponse) contextValidateDeprecations(ctx context.Context, formats strfmt.Registry) error { - - for i := 0; i < len(m.Deprecations); i++ { - - if m.Deprecations[i] != nil { - if err := m.Deprecations[i].ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("deprecations" + "." + strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("deprecations" + "." + strconv.Itoa(i)) - } - return err - } - } - - } - - return nil -} - -func (m *ObjectsGetResponse) contextValidateResult(ctx context.Context, formats strfmt.Registry) error { - - if m.Result != nil { - if err := m.Result.ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("result") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("result") - } - return err - } - } - - return nil -} - -// MarshalBinary interface implementation -func (m *ObjectsGetResponse) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *ObjectsGetResponse) UnmarshalBinary(b []byte) error { - var res ObjectsGetResponse - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} - -// ObjectsGetResponseAO2Result Results for this specific Object. -// -// swagger:model ObjectsGetResponseAO2Result -type ObjectsGetResponseAO2Result struct { - - // errors - Errors *ErrorResponse `json:"errors,omitempty"` - - // status - // Enum: [SUCCESS PENDING FAILED] - Status *string `json:"status,omitempty"` -} - -// Validate validates this objects get response a o2 result -func (m *ObjectsGetResponseAO2Result) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateErrors(formats); err != nil { - res = append(res, err) - } - - if err := m.validateStatus(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *ObjectsGetResponseAO2Result) validateErrors(formats strfmt.Registry) error { - if swag.IsZero(m.Errors) { // not required - return nil - } - - if m.Errors != nil { - if err := m.Errors.Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("result" + "." + "errors") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("result" + "." + "errors") - } - return err - } - } - - return nil -} - -var objectsGetResponseAO2ResultTypeStatusPropEnum []interface{} - -func init() { - var res []string - if err := json.Unmarshal([]byte(`["SUCCESS","PENDING","FAILED"]`), &res); err != nil { - panic(err) - } - for _, v := range res { - objectsGetResponseAO2ResultTypeStatusPropEnum = append(objectsGetResponseAO2ResultTypeStatusPropEnum, v) - } -} - -const ( - - // ObjectsGetResponseAO2ResultStatusSUCCESS captures enum value "SUCCESS" - ObjectsGetResponseAO2ResultStatusSUCCESS string = "SUCCESS" - - // ObjectsGetResponseAO2ResultStatusPENDING captures enum value "PENDING" - ObjectsGetResponseAO2ResultStatusPENDING string = "PENDING" - - // ObjectsGetResponseAO2ResultStatusFAILED captures enum value "FAILED" - ObjectsGetResponseAO2ResultStatusFAILED string = "FAILED" -) - -// prop value enum -func (m *ObjectsGetResponseAO2Result) validateStatusEnum(path, location string, value string) error { - if err := validate.EnumCase(path, location, value, objectsGetResponseAO2ResultTypeStatusPropEnum, true); err != nil { - return err - } - return nil -} - -func (m *ObjectsGetResponseAO2Result) validateStatus(formats strfmt.Registry) error { - if swag.IsZero(m.Status) { // not required - return nil - } - - // value enum - if err := m.validateStatusEnum("result"+"."+"status", "body", *m.Status); err != nil { - return err - } - - return nil -} - -// ContextValidate validate this objects get response a o2 result based on the context it is used -func (m *ObjectsGetResponseAO2Result) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - if err := m.contextValidateErrors(ctx, formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *ObjectsGetResponseAO2Result) contextValidateErrors(ctx context.Context, formats strfmt.Registry) error { - - if m.Errors != nil { - if err := m.Errors.ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("result" + "." + "errors") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("result" + "." + "errors") - } - return err - } - } - - return nil -} - -// MarshalBinary interface implementation -func (m *ObjectsGetResponseAO2Result) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *ObjectsGetResponseAO2Result) UnmarshalBinary(b []byte) error { - var res ObjectsGetResponseAO2Result - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/objects_list_response.go b/entities/models/objects_list_response.go deleted file mode 100644 index 02372b48e7327f19c7cea193dd072c4453294a49..0000000000000000000000000000000000000000 --- a/entities/models/objects_list_response.go +++ /dev/null @@ -1,187 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "strconv" - - "github.com/go-openapi/errors" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" -) - -// ObjectsListResponse List of Objects. -// -// swagger:model ObjectsListResponse -type ObjectsListResponse struct { - - // deprecations - Deprecations []*Deprecation `json:"deprecations"` - - // The actual list of Objects. - Objects []*Object `json:"objects"` - - // The total number of Objects for the query. The number of items in a response may be smaller due to paging. - TotalResults int64 `json:"totalResults,omitempty"` -} - -// Validate validates this objects list response -func (m *ObjectsListResponse) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateDeprecations(formats); err != nil { - res = append(res, err) - } - - if err := m.validateObjects(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *ObjectsListResponse) validateDeprecations(formats strfmt.Registry) error { - if swag.IsZero(m.Deprecations) { // not required - return nil - } - - for i := 0; i < len(m.Deprecations); i++ { - if swag.IsZero(m.Deprecations[i]) { // not required - continue - } - - if m.Deprecations[i] != nil { - if err := m.Deprecations[i].Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("deprecations" + "." + strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("deprecations" + "." + strconv.Itoa(i)) - } - return err - } - } - - } - - return nil -} - -func (m *ObjectsListResponse) validateObjects(formats strfmt.Registry) error { - if swag.IsZero(m.Objects) { // not required - return nil - } - - for i := 0; i < len(m.Objects); i++ { - if swag.IsZero(m.Objects[i]) { // not required - continue - } - - if m.Objects[i] != nil { - if err := m.Objects[i].Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("objects" + "." + strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("objects" + "." + strconv.Itoa(i)) - } - return err - } - } - - } - - return nil -} - -// ContextValidate validate this objects list response based on the context it is used -func (m *ObjectsListResponse) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - if err := m.contextValidateDeprecations(ctx, formats); err != nil { - res = append(res, err) - } - - if err := m.contextValidateObjects(ctx, formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *ObjectsListResponse) contextValidateDeprecations(ctx context.Context, formats strfmt.Registry) error { - - for i := 0; i < len(m.Deprecations); i++ { - - if m.Deprecations[i] != nil { - if err := m.Deprecations[i].ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("deprecations" + "." + strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("deprecations" + "." + strconv.Itoa(i)) - } - return err - } - } - - } - - return nil -} - -func (m *ObjectsListResponse) contextValidateObjects(ctx context.Context, formats strfmt.Registry) error { - - for i := 0; i < len(m.Objects); i++ { - - if m.Objects[i] != nil { - if err := m.Objects[i].ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("objects" + "." + strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("objects" + "." + strconv.Itoa(i)) - } - return err - } - } - - } - - return nil -} - -// MarshalBinary interface implementation -func (m *ObjectsListResponse) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *ObjectsListResponse) UnmarshalBinary(b []byte) error { - var res ObjectsListResponse - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/patch_document_action.go b/entities/models/patch_document_action.go deleted file mode 100644 index 69821a88ad0753c7a58fdc81e3d4e37d66be8947..0000000000000000000000000000000000000000 --- a/entities/models/patch_document_action.go +++ /dev/null @@ -1,204 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "encoding/json" - - "github.com/go-openapi/errors" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" - "github.com/go-openapi/validate" -) - -// PatchDocumentAction Either a JSONPatch document as defined by RFC 6902 (from, op, path, value), or a merge document (RFC 7396). -// -// swagger:model PatchDocumentAction -type PatchDocumentAction struct { - - // A string containing a JSON Pointer value. - From string `json:"from,omitempty"` - - // merge - Merge *Object `json:"merge,omitempty"` - - // The operation to be performed. - // Required: true - // Enum: [add remove replace move copy test] - Op *string `json:"op"` - - // A JSON-Pointer. - // Required: true - Path *string `json:"path"` - - // The value to be used within the operations. - Value interface{} `json:"value,omitempty"` -} - -// Validate validates this patch document action -func (m *PatchDocumentAction) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateMerge(formats); err != nil { - res = append(res, err) - } - - if err := m.validateOp(formats); err != nil { - res = append(res, err) - } - - if err := m.validatePath(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *PatchDocumentAction) validateMerge(formats strfmt.Registry) error { - if swag.IsZero(m.Merge) { // not required - return nil - } - - if m.Merge != nil { - if err := m.Merge.Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("merge") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("merge") - } - return err - } - } - - return nil -} - -var patchDocumentActionTypeOpPropEnum []interface{} - -func init() { - var res []string - if err := json.Unmarshal([]byte(`["add","remove","replace","move","copy","test"]`), &res); err != nil { - panic(err) - } - for _, v := range res { - patchDocumentActionTypeOpPropEnum = append(patchDocumentActionTypeOpPropEnum, v) - } -} - -const ( - - // PatchDocumentActionOpAdd captures enum value "add" - PatchDocumentActionOpAdd string = "add" - - // PatchDocumentActionOpRemove captures enum value "remove" - PatchDocumentActionOpRemove string = "remove" - - // PatchDocumentActionOpReplace captures enum value "replace" - PatchDocumentActionOpReplace string = "replace" - - // PatchDocumentActionOpMove captures enum value "move" - PatchDocumentActionOpMove string = "move" - - // PatchDocumentActionOpCopy captures enum value "copy" - PatchDocumentActionOpCopy string = "copy" - - // PatchDocumentActionOpTest captures enum value "test" - PatchDocumentActionOpTest string = "test" -) - -// prop value enum -func (m *PatchDocumentAction) validateOpEnum(path, location string, value string) error { - if err := validate.EnumCase(path, location, value, patchDocumentActionTypeOpPropEnum, true); err != nil { - return err - } - return nil -} - -func (m *PatchDocumentAction) validateOp(formats strfmt.Registry) error { - - if err := validate.Required("op", "body", m.Op); err != nil { - return err - } - - // value enum - if err := m.validateOpEnum("op", "body", *m.Op); err != nil { - return err - } - - return nil -} - -func (m *PatchDocumentAction) validatePath(formats strfmt.Registry) error { - - if err := validate.Required("path", "body", m.Path); err != nil { - return err - } - - return nil -} - -// ContextValidate validate this patch document action based on the context it is used -func (m *PatchDocumentAction) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - if err := m.contextValidateMerge(ctx, formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *PatchDocumentAction) contextValidateMerge(ctx context.Context, formats strfmt.Registry) error { - - if m.Merge != nil { - if err := m.Merge.ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("merge") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("merge") - } - return err - } - } - - return nil -} - -// MarshalBinary interface implementation -func (m *PatchDocumentAction) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *PatchDocumentAction) UnmarshalBinary(b []byte) error { - var res PatchDocumentAction - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/patch_document_object.go b/entities/models/patch_document_object.go deleted file mode 100644 index f97a45b9455d72a292d0952baafb4dce73bacc69..0000000000000000000000000000000000000000 --- a/entities/models/patch_document_object.go +++ /dev/null @@ -1,204 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "encoding/json" - - "github.com/go-openapi/errors" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" - "github.com/go-openapi/validate" -) - -// PatchDocumentObject Either a JSONPatch document as defined by RFC 6902 (from, op, path, value), or a merge document (RFC 7396). -// -// swagger:model PatchDocumentObject -type PatchDocumentObject struct { - - // A string containing a JSON Pointer value. - From string `json:"from,omitempty"` - - // merge - Merge *Object `json:"merge,omitempty"` - - // The operation to be performed. - // Required: true - // Enum: [add remove replace move copy test] - Op *string `json:"op"` - - // A JSON-Pointer. - // Required: true - Path *string `json:"path"` - - // The value to be used within the operations. - Value interface{} `json:"value,omitempty"` -} - -// Validate validates this patch document object -func (m *PatchDocumentObject) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateMerge(formats); err != nil { - res = append(res, err) - } - - if err := m.validateOp(formats); err != nil { - res = append(res, err) - } - - if err := m.validatePath(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *PatchDocumentObject) validateMerge(formats strfmt.Registry) error { - if swag.IsZero(m.Merge) { // not required - return nil - } - - if m.Merge != nil { - if err := m.Merge.Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("merge") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("merge") - } - return err - } - } - - return nil -} - -var patchDocumentObjectTypeOpPropEnum []interface{} - -func init() { - var res []string - if err := json.Unmarshal([]byte(`["add","remove","replace","move","copy","test"]`), &res); err != nil { - panic(err) - } - for _, v := range res { - patchDocumentObjectTypeOpPropEnum = append(patchDocumentObjectTypeOpPropEnum, v) - } -} - -const ( - - // PatchDocumentObjectOpAdd captures enum value "add" - PatchDocumentObjectOpAdd string = "add" - - // PatchDocumentObjectOpRemove captures enum value "remove" - PatchDocumentObjectOpRemove string = "remove" - - // PatchDocumentObjectOpReplace captures enum value "replace" - PatchDocumentObjectOpReplace string = "replace" - - // PatchDocumentObjectOpMove captures enum value "move" - PatchDocumentObjectOpMove string = "move" - - // PatchDocumentObjectOpCopy captures enum value "copy" - PatchDocumentObjectOpCopy string = "copy" - - // PatchDocumentObjectOpTest captures enum value "test" - PatchDocumentObjectOpTest string = "test" -) - -// prop value enum -func (m *PatchDocumentObject) validateOpEnum(path, location string, value string) error { - if err := validate.EnumCase(path, location, value, patchDocumentObjectTypeOpPropEnum, true); err != nil { - return err - } - return nil -} - -func (m *PatchDocumentObject) validateOp(formats strfmt.Registry) error { - - if err := validate.Required("op", "body", m.Op); err != nil { - return err - } - - // value enum - if err := m.validateOpEnum("op", "body", *m.Op); err != nil { - return err - } - - return nil -} - -func (m *PatchDocumentObject) validatePath(formats strfmt.Registry) error { - - if err := validate.Required("path", "body", m.Path); err != nil { - return err - } - - return nil -} - -// ContextValidate validate this patch document object based on the context it is used -func (m *PatchDocumentObject) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - if err := m.contextValidateMerge(ctx, formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *PatchDocumentObject) contextValidateMerge(ctx context.Context, formats strfmt.Registry) error { - - if m.Merge != nil { - if err := m.Merge.ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("merge") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("merge") - } - return err - } - } - - return nil -} - -// MarshalBinary interface implementation -func (m *PatchDocumentObject) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *PatchDocumentObject) UnmarshalBinary(b []byte) error { - var res PatchDocumentObject - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/peer_update.go b/entities/models/peer_update.go deleted file mode 100644 index 1475a066f2b2af0f090bbd967743cba4cc2a9326..0000000000000000000000000000000000000000 --- a/entities/models/peer_update.go +++ /dev/null @@ -1,111 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - - "github.com/go-openapi/errors" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" - "github.com/go-openapi/validate" -) - -// PeerUpdate A single peer in the network. -// -// swagger:model PeerUpdate -type PeerUpdate struct { - - // The session ID of the peer. - // Format: uuid - ID strfmt.UUID `json:"id,omitempty"` - - // Human readable name. - Name string `json:"name,omitempty"` - - // The latest known hash of the peer's schema. - SchemaHash string `json:"schemaHash,omitempty"` - - // The location where the peer is exposed to the internet. - // Format: uri - URI strfmt.URI `json:"uri,omitempty"` -} - -// Validate validates this peer update -func (m *PeerUpdate) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateID(formats); err != nil { - res = append(res, err) - } - - if err := m.validateURI(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *PeerUpdate) validateID(formats strfmt.Registry) error { - if swag.IsZero(m.ID) { // not required - return nil - } - - if err := validate.FormatOf("id", "body", "uuid", m.ID.String(), formats); err != nil { - return err - } - - return nil -} - -func (m *PeerUpdate) validateURI(formats strfmt.Registry) error { - if swag.IsZero(m.URI) { // not required - return nil - } - - if err := validate.FormatOf("uri", "body", "uri", m.URI.String(), formats); err != nil { - return err - } - - return nil -} - -// ContextValidate validates this peer update based on context it is used -func (m *PeerUpdate) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - return nil -} - -// MarshalBinary interface implementation -func (m *PeerUpdate) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *PeerUpdate) UnmarshalBinary(b []byte) error { - var res PeerUpdate - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/peer_update_list.go b/entities/models/peer_update_list.go deleted file mode 100644 index 39b034c32ef2280d3e21fbfd7651fc211f98d5f3..0000000000000000000000000000000000000000 --- a/entities/models/peer_update_list.go +++ /dev/null @@ -1,84 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "strconv" - - "github.com/go-openapi/errors" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" -) - -// PeerUpdateList List of known peers. -// -// swagger:model PeerUpdateList -type PeerUpdateList []*PeerUpdate - -// Validate validates this peer update list -func (m PeerUpdateList) Validate(formats strfmt.Registry) error { - var res []error - - for i := 0; i < len(m); i++ { - if swag.IsZero(m[i]) { // not required - continue - } - - if m[i] != nil { - if err := m[i].Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName(strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName(strconv.Itoa(i)) - } - return err - } - } - - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// ContextValidate validate this peer update list based on the context it is used -func (m PeerUpdateList) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - for i := 0; i < len(m); i++ { - - if m[i] != nil { - if err := m[i].ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName(strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName(strconv.Itoa(i)) - } - return err - } - } - - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/entities/models/phone_number.go b/entities/models/phone_number.go deleted file mode 100644 index 4db651e197e646dad4a7777137ce304d1197e088..0000000000000000000000000000000000000000 --- a/entities/models/phone_number.go +++ /dev/null @@ -1,79 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" -) - -// PhoneNumber phone number -// -// swagger:model PhoneNumber -type PhoneNumber struct { - - // Read-only. The numerical country code (e.g. 49) - CountryCode uint64 `json:"countryCode,omitempty"` - - // Optional. The ISO 3166-1 alpha-2 country code. This is used to figure out the correct countryCode and international format if only a national number (e.g. 0123 4567) is provided - DefaultCountry string `json:"defaultCountry,omitempty"` - - // The raw input as the phone number is present in your raw data set. It will be parsed into the standardized formats if valid. - Input string `json:"input,omitempty"` - - // Read-only. Parsed result in the international format (e.g. +49 123 ...) - InternationalFormatted string `json:"internationalFormatted,omitempty"` - - // Read-only. The numerical representation of the national part - National uint64 `json:"national,omitempty"` - - // Read-only. Parsed result in the national format (e.g. 0123 456789) - NationalFormatted string `json:"nationalFormatted,omitempty"` - - // Read-only. Indicates whether the parsed number is a valid phone number - Valid bool `json:"valid,omitempty"` -} - -// Validate validates this phone number -func (m *PhoneNumber) Validate(formats strfmt.Registry) error { - return nil -} - -// ContextValidate validates this phone number based on context it is used -func (m *PhoneNumber) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - return nil -} - -// MarshalBinary interface implementation -func (m *PhoneNumber) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *PhoneNumber) UnmarshalBinary(b []byte) error { - var res PhoneNumber - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/principal.go b/entities/models/principal.go deleted file mode 100644 index 1310e26fdbc87bea2a31782add8c342aaa03ffde..0000000000000000000000000000000000000000 --- a/entities/models/principal.go +++ /dev/null @@ -1,64 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" -) - -// Principal principal -// -// swagger:model Principal -type Principal struct { - - // groups - Groups []string `json:"groups"` - - // The username that was extracted either from the authentication information - Username string `json:"username,omitempty"` -} - -// Validate validates this principal -func (m *Principal) Validate(formats strfmt.Registry) error { - return nil -} - -// ContextValidate validates this principal based on context it is used -func (m *Principal) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - return nil -} - -// MarshalBinary interface implementation -func (m *Principal) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *Principal) UnmarshalBinary(b []byte) error { - var res Principal - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/property.go b/entities/models/property.go deleted file mode 100644 index d2c3b2aeb571d02e741d89b94eb32bffee002a47..0000000000000000000000000000000000000000 --- a/entities/models/property.go +++ /dev/null @@ -1,206 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "encoding/json" - "strconv" - - "github.com/go-openapi/errors" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" - "github.com/go-openapi/validate" -) - -// Property property -// -// swagger:model Property -type Property struct { - - // Can be a reference to another type when it starts with a capital (for example Person), otherwise "string" or "int". - DataType []string `json:"dataType"` - - // Description of the property. - Description string `json:"description,omitempty"` - - // Optional. Should this property be indexed in the inverted index. Defaults to true. If you choose false, you will not be able to use this property in where filters. This property has no affect on vectorization decisions done by modules - IndexFilterable *bool `json:"indexFilterable,omitempty"` - - // Optional. Should this property be indexed in the inverted index. Defaults to true. If you choose false, you will not be able to use this property in where filters, bm25 or hybrid search. This property has no affect on vectorization decisions done by modules (deprecated as of v1.19; use indexFilterable or/and indexSearchable instead) - IndexInverted *bool `json:"indexInverted,omitempty"` - - // Optional. Should this property be indexed in the inverted index. Defaults to true. Applicable only to properties of data type text and text[]. If you choose false, you will not be able to use this property in bm25 or hybrid search. This property has no affect on vectorization decisions done by modules - IndexSearchable *bool `json:"indexSearchable,omitempty"` - - // Configuration specific to modules this Weaviate instance has installed - ModuleConfig interface{} `json:"moduleConfig,omitempty"` - - // Name of the property as URI relative to the schema URL. - Name string `json:"name,omitempty"` - - // The properties of the nested object(s). Applies to object and object[] data types. - NestedProperties []*NestedProperty `json:"nestedProperties,omitempty"` - - // Determines tokenization of the property as separate words or whole field. Optional. Applies to text and text[] data types. Allowed values are `word` (default; splits on any non-alphanumerical, lowercases), `lowercase` (splits on white spaces, lowercases), `whitespace` (splits on white spaces), `field` (trims). Not supported for remaining data types - // Enum: [word lowercase whitespace field] - Tokenization string `json:"tokenization,omitempty"` -} - -// Validate validates this property -func (m *Property) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateNestedProperties(formats); err != nil { - res = append(res, err) - } - - if err := m.validateTokenization(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *Property) validateNestedProperties(formats strfmt.Registry) error { - if swag.IsZero(m.NestedProperties) { // not required - return nil - } - - for i := 0; i < len(m.NestedProperties); i++ { - if swag.IsZero(m.NestedProperties[i]) { // not required - continue - } - - if m.NestedProperties[i] != nil { - if err := m.NestedProperties[i].Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("nestedProperties" + "." + strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("nestedProperties" + "." + strconv.Itoa(i)) - } - return err - } - } - - } - - return nil -} - -var propertyTypeTokenizationPropEnum []interface{} - -func init() { - var res []string - if err := json.Unmarshal([]byte(`["word","lowercase","whitespace","field"]`), &res); err != nil { - panic(err) - } - for _, v := range res { - propertyTypeTokenizationPropEnum = append(propertyTypeTokenizationPropEnum, v) - } -} - -const ( - - // PropertyTokenizationWord captures enum value "word" - PropertyTokenizationWord string = "word" - - // PropertyTokenizationLowercase captures enum value "lowercase" - PropertyTokenizationLowercase string = "lowercase" - - // PropertyTokenizationWhitespace captures enum value "whitespace" - PropertyTokenizationWhitespace string = "whitespace" - - // PropertyTokenizationField captures enum value "field" - PropertyTokenizationField string = "field" -) - -// prop value enum -func (m *Property) validateTokenizationEnum(path, location string, value string) error { - if err := validate.EnumCase(path, location, value, propertyTypeTokenizationPropEnum, true); err != nil { - return err - } - return nil -} - -func (m *Property) validateTokenization(formats strfmt.Registry) error { - if swag.IsZero(m.Tokenization) { // not required - return nil - } - - // value enum - if err := m.validateTokenizationEnum("tokenization", "body", m.Tokenization); err != nil { - return err - } - - return nil -} - -// ContextValidate validate this property based on the context it is used -func (m *Property) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - if err := m.contextValidateNestedProperties(ctx, formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *Property) contextValidateNestedProperties(ctx context.Context, formats strfmt.Registry) error { - - for i := 0; i < len(m.NestedProperties); i++ { - - if m.NestedProperties[i] != nil { - if err := m.NestedProperties[i].ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("nestedProperties" + "." + strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("nestedProperties" + "." + strconv.Itoa(i)) - } - return err - } - } - - } - - return nil -} - -// MarshalBinary interface implementation -func (m *Property) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *Property) UnmarshalBinary(b []byte) error { - var res Property - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/property_schema.go b/entities/models/property_schema.go deleted file mode 100644 index bb1a7d363b8758f65e59bb3a6f9d78a8023ffbe3..0000000000000000000000000000000000000000 --- a/entities/models/property_schema.go +++ /dev/null @@ -1,22 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -// PropertySchema This is an open object, with OpenAPI Specification 3.0 this will be more detailed. See Weaviate docs for more info. In the future this will become a key/value OR a SingleRef definition. -// -// swagger:model PropertySchema -type PropertySchema interface{} diff --git a/entities/models/reference_meta_classification.go b/entities/models/reference_meta_classification.go deleted file mode 100644 index 7ab4308951457cb499ae9f72c978f2dfeb98b15e..0000000000000000000000000000000000000000 --- a/entities/models/reference_meta_classification.go +++ /dev/null @@ -1,88 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" -) - -// ReferenceMetaClassification This meta field contains additional info about the classified reference property -// -// swagger:model ReferenceMetaClassification -type ReferenceMetaClassification struct { - - // The lowest distance of a neighbor in the losing group. Optional. If k equals the size of the winning group, there is no losing group - ClosestLosingDistance *float64 `json:"closestLosingDistance,omitempty"` - - // The lowest distance of any neighbor, regardless of whether they were in the winning or losing group - ClosestOverallDistance float64 `json:"closestOverallDistance,omitempty"` - - // Closest distance of a neighbor from the winning group - ClosestWinningDistance float64 `json:"closestWinningDistance,omitempty"` - - // size of the losing group, can be 0 if the winning group size equals k - LosingCount int64 `json:"losingCount,omitempty"` - - // deprecated - do not use, to be removed in 0.23.0 - LosingDistance *float64 `json:"losingDistance,omitempty"` - - // Mean distance of all neighbors from the losing group. Optional. If k equals the size of the winning group, there is no losing group. - MeanLosingDistance *float64 `json:"meanLosingDistance,omitempty"` - - // Mean distance of all neighbors from the winning group - MeanWinningDistance float64 `json:"meanWinningDistance,omitempty"` - - // overall neighbors checked as part of the classification. In most cases this will equal k, but could be lower than k - for example if not enough data was present - OverallCount int64 `json:"overallCount,omitempty"` - - // size of the winning group, a number between 1..k - WinningCount int64 `json:"winningCount,omitempty"` - - // deprecated - do not use, to be removed in 0.23.0 - WinningDistance float64 `json:"winningDistance,omitempty"` -} - -// Validate validates this reference meta classification -func (m *ReferenceMetaClassification) Validate(formats strfmt.Registry) error { - return nil -} - -// ContextValidate validates this reference meta classification based on context it is used -func (m *ReferenceMetaClassification) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - return nil -} - -// MarshalBinary interface implementation -func (m *ReferenceMetaClassification) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *ReferenceMetaClassification) UnmarshalBinary(b []byte) error { - var res ReferenceMetaClassification - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/replication_config.go b/entities/models/replication_config.go deleted file mode 100644 index acfd55c6f3f52c39d39d8b7bcd1257888aa355d9..0000000000000000000000000000000000000000 --- a/entities/models/replication_config.go +++ /dev/null @@ -1,61 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" -) - -// ReplicationConfig Configure how replication is executed in a cluster -// -// swagger:model ReplicationConfig -type ReplicationConfig struct { - - // Number of times a class is replicated - Factor int64 `json:"factor,omitempty"` -} - -// Validate validates this replication config -func (m *ReplicationConfig) Validate(formats strfmt.Registry) error { - return nil -} - -// ContextValidate validates this replication config based on context it is used -func (m *ReplicationConfig) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - return nil -} - -// MarshalBinary interface implementation -func (m *ReplicationConfig) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *ReplicationConfig) UnmarshalBinary(b []byte) error { - var res ReplicationConfig - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/restore_config.go b/entities/models/restore_config.go deleted file mode 100644 index e0875e847976767a3f2d78d0bad0b7f3375c3e83..0000000000000000000000000000000000000000 --- a/entities/models/restore_config.go +++ /dev/null @@ -1,90 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - - "github.com/go-openapi/errors" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" - "github.com/go-openapi/validate" -) - -// RestoreConfig Backup custom configuration -// -// swagger:model RestoreConfig -type RestoreConfig struct { - - // Desired CPU core utilization ranging from 1%-80% - // Maximum: 80 - // Minimum: 1 - CPUPercentage int64 `json:"CPUPercentage,omitempty"` -} - -// Validate validates this restore config -func (m *RestoreConfig) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateCPUPercentage(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *RestoreConfig) validateCPUPercentage(formats strfmt.Registry) error { - if swag.IsZero(m.CPUPercentage) { // not required - return nil - } - - if err := validate.MinimumInt("CPUPercentage", "body", m.CPUPercentage, 1, false); err != nil { - return err - } - - if err := validate.MaximumInt("CPUPercentage", "body", m.CPUPercentage, 80, false); err != nil { - return err - } - - return nil -} - -// ContextValidate validates this restore config based on context it is used -func (m *RestoreConfig) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - return nil -} - -// MarshalBinary interface implementation -func (m *RestoreConfig) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *RestoreConfig) UnmarshalBinary(b []byte) error { - var res RestoreConfig - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/schema.go b/entities/models/schema.go deleted file mode 100644 index 99d178222043da4836fff84c0b237f1dc1e81969..0000000000000000000000000000000000000000 --- a/entities/models/schema.go +++ /dev/null @@ -1,151 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "strconv" - - "github.com/go-openapi/errors" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" - "github.com/go-openapi/validate" -) - -// Schema Definitions of semantic schemas (also see: https://github.com/weaviate/weaviate-semantic-schemas). -// -// swagger:model Schema -type Schema struct { - - // Semantic classes that are available. - Classes []*Class `json:"classes"` - - // Email of the maintainer. - // Format: email - Maintainer strfmt.Email `json:"maintainer,omitempty"` - - // Name of the schema. - Name string `json:"name,omitempty"` -} - -// Validate validates this schema -func (m *Schema) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateClasses(formats); err != nil { - res = append(res, err) - } - - if err := m.validateMaintainer(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *Schema) validateClasses(formats strfmt.Registry) error { - if swag.IsZero(m.Classes) { // not required - return nil - } - - for i := 0; i < len(m.Classes); i++ { - if swag.IsZero(m.Classes[i]) { // not required - continue - } - - if m.Classes[i] != nil { - if err := m.Classes[i].Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("classes" + "." + strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("classes" + "." + strconv.Itoa(i)) - } - return err - } - } - - } - - return nil -} - -func (m *Schema) validateMaintainer(formats strfmt.Registry) error { - if swag.IsZero(m.Maintainer) { // not required - return nil - } - - if err := validate.FormatOf("maintainer", "body", "email", m.Maintainer.String(), formats); err != nil { - return err - } - - return nil -} - -// ContextValidate validate this schema based on the context it is used -func (m *Schema) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - if err := m.contextValidateClasses(ctx, formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *Schema) contextValidateClasses(ctx context.Context, formats strfmt.Registry) error { - - for i := 0; i < len(m.Classes); i++ { - - if m.Classes[i] != nil { - if err := m.Classes[i].ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("classes" + "." + strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("classes" + "." + strconv.Itoa(i)) - } - return err - } - } - - } - - return nil -} - -// MarshalBinary interface implementation -func (m *Schema) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *Schema) UnmarshalBinary(b []byte) error { - var res Schema - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/schema_cluster_status.go b/entities/models/schema_cluster_status.go deleted file mode 100644 index f24577bcfc8825338cdb508351ce46a908cb57f6..0000000000000000000000000000000000000000 --- a/entities/models/schema_cluster_status.go +++ /dev/null @@ -1,73 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" -) - -// SchemaClusterStatus Indicates the health of the schema in a cluster. -// -// swagger:model SchemaClusterStatus -type SchemaClusterStatus struct { - - // Contains the sync check error if one occurred - Error string `json:"error,omitempty"` - - // True if the cluster is in sync, false if there is an issue (see error). - Healthy bool `json:"healthy"` - - // Hostname of the coordinating node, i.e. the one that received the cluster. This can be useful information if the error message contains phrases such as 'other nodes agree, but local does not', etc. - Hostname string `json:"hostname,omitempty"` - - // The cluster check at startup can be ignored (to recover from an out-of-sync situation). - IgnoreSchemaSync bool `json:"ignoreSchemaSync"` - - // Number of nodes that participated in the sync check - NodeCount int64 `json:"nodeCount,omitempty"` -} - -// Validate validates this schema cluster status -func (m *SchemaClusterStatus) Validate(formats strfmt.Registry) error { - return nil -} - -// ContextValidate validates this schema cluster status based on context it is used -func (m *SchemaClusterStatus) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - return nil -} - -// MarshalBinary interface implementation -func (m *SchemaClusterStatus) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *SchemaClusterStatus) UnmarshalBinary(b []byte) error { - var res SchemaClusterStatus - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/schema_history.go b/entities/models/schema_history.go deleted file mode 100644 index cb7618bf6ba6a43bd507ee04246e6a82a606c7c7..0000000000000000000000000000000000000000 --- a/entities/models/schema_history.go +++ /dev/null @@ -1,22 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -// SchemaHistory This is an open object, with OpenAPI Specification 3.0 this will be more detailed. See Weaviate docs for more info. In the future this will become a key/value OR a SingleRef definition. -// -// swagger:model SchemaHistory -type SchemaHistory interface{} diff --git a/entities/models/shard_status.go b/entities/models/shard_status.go deleted file mode 100644 index ef2efe9cb779fae45d95dc11d489a0ef76625f59..0000000000000000000000000000000000000000 --- a/entities/models/shard_status.go +++ /dev/null @@ -1,61 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" -) - -// ShardStatus The status of a single shard -// -// swagger:model ShardStatus -type ShardStatus struct { - - // Status of the shard - Status string `json:"status,omitempty"` -} - -// Validate validates this shard status -func (m *ShardStatus) Validate(formats strfmt.Registry) error { - return nil -} - -// ContextValidate validates this shard status based on context it is used -func (m *ShardStatus) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - return nil -} - -// MarshalBinary interface implementation -func (m *ShardStatus) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *ShardStatus) UnmarshalBinary(b []byte) error { - var res ShardStatus - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/shard_status_get_response.go b/entities/models/shard_status_get_response.go deleted file mode 100644 index af5a70cff99107e82851490485d6b30b7a34a009..0000000000000000000000000000000000000000 --- a/entities/models/shard_status_get_response.go +++ /dev/null @@ -1,67 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" -) - -// ShardStatusGetResponse Response body of shard status get request -// -// swagger:model ShardStatusGetResponse -type ShardStatusGetResponse struct { - - // Name of the shard - Name string `json:"name,omitempty"` - - // Status of the shard - Status string `json:"status,omitempty"` - - // Size of the vector queue of the shard - VectorQueueSize int64 `json:"vectorQueueSize"` -} - -// Validate validates this shard status get response -func (m *ShardStatusGetResponse) Validate(formats strfmt.Registry) error { - return nil -} - -// ContextValidate validates this shard status get response based on context it is used -func (m *ShardStatusGetResponse) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - return nil -} - -// MarshalBinary interface implementation -func (m *ShardStatusGetResponse) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *ShardStatusGetResponse) UnmarshalBinary(b []byte) error { - var res ShardStatusGetResponse - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/shard_status_list.go b/entities/models/shard_status_list.go deleted file mode 100644 index 9091d78523cdac0f6cd8ea102f0117f6a1fcb721..0000000000000000000000000000000000000000 --- a/entities/models/shard_status_list.go +++ /dev/null @@ -1,84 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "strconv" - - "github.com/go-openapi/errors" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" -) - -// ShardStatusList The status of all the shards of a Class -// -// swagger:model ShardStatusList -type ShardStatusList []*ShardStatusGetResponse - -// Validate validates this shard status list -func (m ShardStatusList) Validate(formats strfmt.Registry) error { - var res []error - - for i := 0; i < len(m); i++ { - if swag.IsZero(m[i]) { // not required - continue - } - - if m[i] != nil { - if err := m[i].Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName(strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName(strconv.Itoa(i)) - } - return err - } - } - - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -// ContextValidate validate this shard status list based on the context it is used -func (m ShardStatusList) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - for i := 0; i < len(m); i++ { - - if m[i] != nil { - if err := m[i].ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName(strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName(strconv.Itoa(i)) - } - return err - } - } - - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} diff --git a/entities/models/single_ref.go b/entities/models/single_ref.go deleted file mode 100644 index 85157eee76d08c465bda2c3783effe7cecbfc03a..0000000000000000000000000000000000000000 --- a/entities/models/single_ref.go +++ /dev/null @@ -1,179 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - - "github.com/go-openapi/errors" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" - "github.com/go-openapi/validate" -) - -// SingleRef Either set beacon (direct reference) or set class and schema (concept reference) -// -// swagger:model SingleRef -type SingleRef struct { - - // If using a direct reference, specify the URI to point to the cross-ref here. Should be in the form of weaviate://localhost/ for the example of a local cross-ref to an object - // Format: uri - Beacon strfmt.URI `json:"beacon,omitempty"` - - // If using a concept reference (rather than a direct reference), specify the desired class name here - // Format: uri - Class strfmt.URI `json:"class,omitempty"` - - // Additional Meta information about classifications if the item was part of one - Classification *ReferenceMetaClassification `json:"classification,omitempty"` - - // If using a direct reference, this read-only fields provides a link to the referenced resource. If 'origin' is globally configured, an absolute URI is shown - a relative URI otherwise. - // Format: uri - Href strfmt.URI `json:"href,omitempty"` - - // If using a concept reference (rather than a direct reference), specify the desired properties here - Schema PropertySchema `json:"schema,omitempty"` -} - -// Validate validates this single ref -func (m *SingleRef) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateBeacon(formats); err != nil { - res = append(res, err) - } - - if err := m.validateClass(formats); err != nil { - res = append(res, err) - } - - if err := m.validateClassification(formats); err != nil { - res = append(res, err) - } - - if err := m.validateHref(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *SingleRef) validateBeacon(formats strfmt.Registry) error { - if swag.IsZero(m.Beacon) { // not required - return nil - } - - if err := validate.FormatOf("beacon", "body", "uri", m.Beacon.String(), formats); err != nil { - return err - } - - return nil -} - -func (m *SingleRef) validateClass(formats strfmt.Registry) error { - if swag.IsZero(m.Class) { // not required - return nil - } - - if err := validate.FormatOf("class", "body", "uri", m.Class.String(), formats); err != nil { - return err - } - - return nil -} - -func (m *SingleRef) validateClassification(formats strfmt.Registry) error { - if swag.IsZero(m.Classification) { // not required - return nil - } - - if m.Classification != nil { - if err := m.Classification.Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("classification") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("classification") - } - return err - } - } - - return nil -} - -func (m *SingleRef) validateHref(formats strfmt.Registry) error { - if swag.IsZero(m.Href) { // not required - return nil - } - - if err := validate.FormatOf("href", "body", "uri", m.Href.String(), formats); err != nil { - return err - } - - return nil -} - -// ContextValidate validate this single ref based on the context it is used -func (m *SingleRef) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - if err := m.contextValidateClassification(ctx, formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *SingleRef) contextValidateClassification(ctx context.Context, formats strfmt.Registry) error { - - if m.Classification != nil { - if err := m.Classification.ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("classification") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("classification") - } - return err - } - } - - return nil -} - -// MarshalBinary interface implementation -func (m *SingleRef) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *SingleRef) UnmarshalBinary(b []byte) error { - var res SingleRef - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/stopword_config.go b/entities/models/stopword_config.go deleted file mode 100644 index 0c6a1c423bd353d761109a2cdc9654c9a16c8f29..0000000000000000000000000000000000000000 --- a/entities/models/stopword_config.go +++ /dev/null @@ -1,67 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" -) - -// StopwordConfig fine-grained control over stopword list usage -// -// swagger:model StopwordConfig -type StopwordConfig struct { - - // stopwords to be considered additionally - Additions []string `json:"additions"` - - // pre-existing list of common words by language - Preset string `json:"preset,omitempty"` - - // stopwords to be removed from consideration - Removals []string `json:"removals"` -} - -// Validate validates this stopword config -func (m *StopwordConfig) Validate(formats strfmt.Registry) error { - return nil -} - -// ContextValidate validates this stopword config based on context it is used -func (m *StopwordConfig) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - return nil -} - -// MarshalBinary interface implementation -func (m *StopwordConfig) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *StopwordConfig) UnmarshalBinary(b []byte) error { - var res StopwordConfig - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/tenant.go b/entities/models/tenant.go deleted file mode 100644 index e8318ff9dccde0adbc4ec1f104afa3237c765050..0000000000000000000000000000000000000000 --- a/entities/models/tenant.go +++ /dev/null @@ -1,125 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "encoding/json" - - "github.com/go-openapi/errors" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" - "github.com/go-openapi/validate" -) - -// Tenant attributes representing a single tenant within weaviate -// -// swagger:model Tenant -type Tenant struct { - - // activity status of the tenant's shard. Optional for creating tenant (implicit `HOT`) and required for updating tenant. Allowed values are `HOT` - tenant is fully active, `WARM` - tenant is active, some restrictions are imposed (TBD; not supported yet), `COLD` - tenant is inactive; no actions can be performed on tenant, tenant's files are stored locally, `FROZEN` - as COLD, but files are stored on cloud storage (not supported yet) - // Enum: [HOT WARM COLD FROZEN] - ActivityStatus string `json:"activityStatus,omitempty"` - - // name of the tenant - Name string `json:"name,omitempty"` -} - -// Validate validates this tenant -func (m *Tenant) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateActivityStatus(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -var tenantTypeActivityStatusPropEnum []interface{} - -func init() { - var res []string - if err := json.Unmarshal([]byte(`["HOT","WARM","COLD","FROZEN"]`), &res); err != nil { - panic(err) - } - for _, v := range res { - tenantTypeActivityStatusPropEnum = append(tenantTypeActivityStatusPropEnum, v) - } -} - -const ( - - // TenantActivityStatusHOT captures enum value "HOT" - TenantActivityStatusHOT string = "HOT" - - // TenantActivityStatusWARM captures enum value "WARM" - TenantActivityStatusWARM string = "WARM" - - // TenantActivityStatusCOLD captures enum value "COLD" - TenantActivityStatusCOLD string = "COLD" - - // TenantActivityStatusFROZEN captures enum value "FROZEN" - TenantActivityStatusFROZEN string = "FROZEN" -) - -// prop value enum -func (m *Tenant) validateActivityStatusEnum(path, location string, value string) error { - if err := validate.EnumCase(path, location, value, tenantTypeActivityStatusPropEnum, true); err != nil { - return err - } - return nil -} - -func (m *Tenant) validateActivityStatus(formats strfmt.Registry) error { - if swag.IsZero(m.ActivityStatus) { // not required - return nil - } - - // value enum - if err := m.validateActivityStatusEnum("activityStatus", "body", m.ActivityStatus); err != nil { - return err - } - - return nil -} - -// ContextValidate validates this tenant based on context it is used -func (m *Tenant) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - return nil -} - -// MarshalBinary interface implementation -func (m *Tenant) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *Tenant) UnmarshalBinary(b []byte) error { - var res Tenant - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/vector_weights.go b/entities/models/vector_weights.go deleted file mode 100644 index dc1e90af4aafec757177b0ae80a6a3c886cae0b0..0000000000000000000000000000000000000000 --- a/entities/models/vector_weights.go +++ /dev/null @@ -1,22 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -// VectorWeights Allow custom overrides of vector weights as math expressions. E.g. "pancake": "7" will set the weight for the word pancake to 7 in the vectorization, whereas "w * 3" would triple the originally calculated word. This is an open object, with OpenAPI Specification 3.0 this will be more detailed. See Weaviate docs for more info. In the future this will become a key/value (string/string) object. -// -// swagger:model VectorWeights -type VectorWeights interface{} diff --git a/entities/models/where_filter.go b/entities/models/where_filter.go deleted file mode 100644 index 71da77ee37c39a1cd97b754199f5d3d5fc00720a..0000000000000000000000000000000000000000 --- a/entities/models/where_filter.go +++ /dev/null @@ -1,311 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - "encoding/json" - "strconv" - - "github.com/go-openapi/errors" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" - "github.com/go-openapi/validate" -) - -// WhereFilter Filter search results using a where filter -// -// swagger:model WhereFilter -type WhereFilter struct { - - // combine multiple where filters, requires 'And' or 'Or' operator - Operands []*WhereFilter `json:"operands"` - - // operator to use - // Example: GreaterThanEqual - // Enum: [And Or Equal Like NotEqual GreaterThan GreaterThanEqual LessThan LessThanEqual WithinGeoRange IsNull ContainsAny ContainsAll] - Operator string `json:"operator,omitempty"` - - // path to the property currently being filtered - // Example: ["inCity","City","name"] - Path []string `json:"path"` - - // value as boolean - // Example: false - ValueBoolean *bool `json:"valueBoolean,omitempty"` - - // value as boolean - // Example: [true,false] - ValueBooleanArray []bool `json:"valueBooleanArray,omitempty"` - - // value as date (as string) - // Example: TODO - ValueDate *string `json:"valueDate,omitempty"` - - // value as date (as string) - // Example: TODO - ValueDateArray []string `json:"valueDateArray,omitempty"` - - // value as geo coordinates and distance - ValueGeoRange *WhereFilterGeoRange `json:"valueGeoRange,omitempty"` - - // value as integer - // Example: 2000 - ValueInt *int64 `json:"valueInt,omitempty"` - - // value as integer - // Example: [100, 200] - ValueIntArray []int64 `json:"valueIntArray,omitempty"` - - // value as number/float - // Example: 3.14 - ValueNumber *float64 `json:"valueNumber,omitempty"` - - // value as number/float - // Example: [3.14] - ValueNumberArray []float64 `json:"valueNumberArray,omitempty"` - - // value as text (deprecated as of v1.19; alias for valueText) - // Example: my search term - ValueString *string `json:"valueString,omitempty"` - - // value as text (deprecated as of v1.19; alias for valueText) - // Example: ["my search term"] - ValueStringArray []string `json:"valueStringArray,omitempty"` - - // value as text - // Example: my search term - ValueText *string `json:"valueText,omitempty"` - - // value as text - // Example: ["my search term"] - ValueTextArray []string `json:"valueTextArray,omitempty"` -} - -// Validate validates this where filter -func (m *WhereFilter) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateOperands(formats); err != nil { - res = append(res, err) - } - - if err := m.validateOperator(formats); err != nil { - res = append(res, err) - } - - if err := m.validateValueGeoRange(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *WhereFilter) validateOperands(formats strfmt.Registry) error { - if swag.IsZero(m.Operands) { // not required - return nil - } - - for i := 0; i < len(m.Operands); i++ { - if swag.IsZero(m.Operands[i]) { // not required - continue - } - - if m.Operands[i] != nil { - if err := m.Operands[i].Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("operands" + "." + strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("operands" + "." + strconv.Itoa(i)) - } - return err - } - } - - } - - return nil -} - -var whereFilterTypeOperatorPropEnum []interface{} - -func init() { - var res []string - if err := json.Unmarshal([]byte(`["And","Or","Equal","Like","NotEqual","GreaterThan","GreaterThanEqual","LessThan","LessThanEqual","WithinGeoRange","IsNull","ContainsAny","ContainsAll"]`), &res); err != nil { - panic(err) - } - for _, v := range res { - whereFilterTypeOperatorPropEnum = append(whereFilterTypeOperatorPropEnum, v) - } -} - -const ( - - // WhereFilterOperatorAnd captures enum value "And" - WhereFilterOperatorAnd string = "And" - - // WhereFilterOperatorOr captures enum value "Or" - WhereFilterOperatorOr string = "Or" - - // WhereFilterOperatorEqual captures enum value "Equal" - WhereFilterOperatorEqual string = "Equal" - - // WhereFilterOperatorLike captures enum value "Like" - WhereFilterOperatorLike string = "Like" - - // WhereFilterOperatorNotEqual captures enum value "NotEqual" - WhereFilterOperatorNotEqual string = "NotEqual" - - // WhereFilterOperatorGreaterThan captures enum value "GreaterThan" - WhereFilterOperatorGreaterThan string = "GreaterThan" - - // WhereFilterOperatorGreaterThanEqual captures enum value "GreaterThanEqual" - WhereFilterOperatorGreaterThanEqual string = "GreaterThanEqual" - - // WhereFilterOperatorLessThan captures enum value "LessThan" - WhereFilterOperatorLessThan string = "LessThan" - - // WhereFilterOperatorLessThanEqual captures enum value "LessThanEqual" - WhereFilterOperatorLessThanEqual string = "LessThanEqual" - - // WhereFilterOperatorWithinGeoRange captures enum value "WithinGeoRange" - WhereFilterOperatorWithinGeoRange string = "WithinGeoRange" - - // WhereFilterOperatorIsNull captures enum value "IsNull" - WhereFilterOperatorIsNull string = "IsNull" - - // WhereFilterOperatorContainsAny captures enum value "ContainsAny" - WhereFilterOperatorContainsAny string = "ContainsAny" - - // WhereFilterOperatorContainsAll captures enum value "ContainsAll" - WhereFilterOperatorContainsAll string = "ContainsAll" -) - -// prop value enum -func (m *WhereFilter) validateOperatorEnum(path, location string, value string) error { - if err := validate.EnumCase(path, location, value, whereFilterTypeOperatorPropEnum, true); err != nil { - return err - } - return nil -} - -func (m *WhereFilter) validateOperator(formats strfmt.Registry) error { - if swag.IsZero(m.Operator) { // not required - return nil - } - - // value enum - if err := m.validateOperatorEnum("operator", "body", m.Operator); err != nil { - return err - } - - return nil -} - -func (m *WhereFilter) validateValueGeoRange(formats strfmt.Registry) error { - if swag.IsZero(m.ValueGeoRange) { // not required - return nil - } - - if m.ValueGeoRange != nil { - if err := m.ValueGeoRange.Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("valueGeoRange") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("valueGeoRange") - } - return err - } - } - - return nil -} - -// ContextValidate validate this where filter based on the context it is used -func (m *WhereFilter) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - if err := m.contextValidateOperands(ctx, formats); err != nil { - res = append(res, err) - } - - if err := m.contextValidateValueGeoRange(ctx, formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *WhereFilter) contextValidateOperands(ctx context.Context, formats strfmt.Registry) error { - - for i := 0; i < len(m.Operands); i++ { - - if m.Operands[i] != nil { - if err := m.Operands[i].ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("operands" + "." + strconv.Itoa(i)) - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("operands" + "." + strconv.Itoa(i)) - } - return err - } - } - - } - - return nil -} - -func (m *WhereFilter) contextValidateValueGeoRange(ctx context.Context, formats strfmt.Registry) error { - - if m.ValueGeoRange != nil { - if err := m.ValueGeoRange.ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("valueGeoRange") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("valueGeoRange") - } - return err - } - } - - return nil -} - -// MarshalBinary interface implementation -func (m *WhereFilter) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *WhereFilter) UnmarshalBinary(b []byte) error { - var res WhereFilter - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/models/where_filter_geo_range.go b/entities/models/where_filter_geo_range.go deleted file mode 100644 index a93499beaadf376338304ad418444136d590e0ee..0000000000000000000000000000000000000000 --- a/entities/models/where_filter_geo_range.go +++ /dev/null @@ -1,198 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Code generated by go-swagger; DO NOT EDIT. - -package models - -// This file was generated by the swagger tool. -// Editing this file might prove futile when you re-run the swagger generate command - -import ( - "context" - - "github.com/go-openapi/errors" - "github.com/go-openapi/strfmt" - "github.com/go-openapi/swag" -) - -// WhereFilterGeoRange filter within a distance of a georange -// -// swagger:model WhereFilterGeoRange -type WhereFilterGeoRange struct { - - // distance - Distance *WhereFilterGeoRangeDistance `json:"distance,omitempty"` - - // geo coordinates - GeoCoordinates *GeoCoordinates `json:"geoCoordinates,omitempty"` -} - -// Validate validates this where filter geo range -func (m *WhereFilterGeoRange) Validate(formats strfmt.Registry) error { - var res []error - - if err := m.validateDistance(formats); err != nil { - res = append(res, err) - } - - if err := m.validateGeoCoordinates(formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *WhereFilterGeoRange) validateDistance(formats strfmt.Registry) error { - if swag.IsZero(m.Distance) { // not required - return nil - } - - if m.Distance != nil { - if err := m.Distance.Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("distance") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("distance") - } - return err - } - } - - return nil -} - -func (m *WhereFilterGeoRange) validateGeoCoordinates(formats strfmt.Registry) error { - if swag.IsZero(m.GeoCoordinates) { // not required - return nil - } - - if m.GeoCoordinates != nil { - if err := m.GeoCoordinates.Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("geoCoordinates") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("geoCoordinates") - } - return err - } - } - - return nil -} - -// ContextValidate validate this where filter geo range based on the context it is used -func (m *WhereFilterGeoRange) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - var res []error - - if err := m.contextValidateDistance(ctx, formats); err != nil { - res = append(res, err) - } - - if err := m.contextValidateGeoCoordinates(ctx, formats); err != nil { - res = append(res, err) - } - - if len(res) > 0 { - return errors.CompositeValidationError(res...) - } - return nil -} - -func (m *WhereFilterGeoRange) contextValidateDistance(ctx context.Context, formats strfmt.Registry) error { - - if m.Distance != nil { - if err := m.Distance.ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("distance") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("distance") - } - return err - } - } - - return nil -} - -func (m *WhereFilterGeoRange) contextValidateGeoCoordinates(ctx context.Context, formats strfmt.Registry) error { - - if m.GeoCoordinates != nil { - if err := m.GeoCoordinates.ContextValidate(ctx, formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("geoCoordinates") - } else if ce, ok := err.(*errors.CompositeError); ok { - return ce.ValidateName("geoCoordinates") - } - return err - } - } - - return nil -} - -// MarshalBinary interface implementation -func (m *WhereFilterGeoRange) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *WhereFilterGeoRange) UnmarshalBinary(b []byte) error { - var res WhereFilterGeoRange - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} - -// WhereFilterGeoRangeDistance where filter geo range distance -// -// swagger:model WhereFilterGeoRangeDistance -type WhereFilterGeoRangeDistance struct { - - // max - Max float64 `json:"max,omitempty"` -} - -// Validate validates this where filter geo range distance -func (m *WhereFilterGeoRangeDistance) Validate(formats strfmt.Registry) error { - return nil -} - -// ContextValidate validates this where filter geo range distance based on context it is used -func (m *WhereFilterGeoRangeDistance) ContextValidate(ctx context.Context, formats strfmt.Registry) error { - return nil -} - -// MarshalBinary interface implementation -func (m *WhereFilterGeoRangeDistance) MarshalBinary() ([]byte, error) { - if m == nil { - return nil, nil - } - return swag.WriteJSON(m) -} - -// UnmarshalBinary interface implementation -func (m *WhereFilterGeoRangeDistance) UnmarshalBinary(b []byte) error { - var res WhereFilterGeoRangeDistance - if err := swag.ReadJSON(b, &res); err != nil { - return err - } - *m = res - return nil -} diff --git a/entities/modulecapabilities/additional.go b/entities/modulecapabilities/additional.go deleted file mode 100644 index 3cfe03ec07f5f4580e402cc386b8b8d56ef2151c..0000000000000000000000000000000000000000 --- a/entities/modulecapabilities/additional.go +++ /dev/null @@ -1,65 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modulecapabilities - -import ( - "context" - - "github.com/tailor-inc/graphql" - "github.com/tailor-inc/graphql/language/ast" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/search" -) - -// GraphQLFieldFn generates graphql field based on classname -type GraphQLFieldFn = func(classname string) *graphql.Field - -// ExtractAdditionalFn extracts parameters from graphql queries -type ExtractAdditionalFn = func(param []*ast.Argument) interface{} - -// AdditionalPropertyWithSearchVector defines additional property params -// with the ability to pass search vector -type AdditionalPropertyWithSearchVector interface { - SetSearchVector(vector []float32) -} - -// AdditionalPropertyFn defines interface for additional property -// functions performing given logic -type AdditionalPropertyFn = func(ctx context.Context, - in []search.Result, params interface{}, limit *int, - argumentModuleParams map[string]interface{}, cfg moduletools.ClassConfig) ([]search.Result, error) - -// AdditionalSearch defines on which type of query a given -// additional logic can be performed -type AdditionalSearch struct { - ObjectGet AdditionalPropertyFn - ObjectList AdditionalPropertyFn - ExploreGet AdditionalPropertyFn - ExploreList AdditionalPropertyFn -} - -// AdditionalProperty defines all the needed settings / methods -// to be set in order to add the additional property to Weaviate -type AdditionalProperty struct { - RestNames []string - DefaultValue interface{} - GraphQLNames []string - GraphQLFieldFunction GraphQLFieldFn - GraphQLExtractFunction ExtractAdditionalFn - SearchFunctions AdditionalSearch -} - -// AdditionalProperties groups whole interface methods needed -// for adding the capability of additional properties -type AdditionalProperties interface { - AdditionalProperties() map[string]AdditionalProperty -} diff --git a/entities/modulecapabilities/backup.go b/entities/modulecapabilities/backup.go deleted file mode 100644 index 626bab9b4670f6e4f4b989a6a0cf27fd1321e9df..0000000000000000000000000000000000000000 --- a/entities/modulecapabilities/backup.go +++ /dev/null @@ -1,47 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modulecapabilities - -import ( - "context" - "io" -) - -type BackupBackend interface { - // IsExternal returns whether the storage is an external storage (e.g. gcs, s3) - IsExternal() bool - // Name returns backend's name - Name() string - // HomeDir is the home directory of all backup files - HomeDir(backupID string) string - - // GetObject giving backupID and key - GetObject(ctx context.Context, backupID, key string) ([]byte, error) - - // WriteToFile writes an object in the specified file with path destPath - // The file will be created if it doesn't exist - // The file will be overwritten if it exists - WriteToFile(ctx context.Context, backupID, key, destPath string) error - - // SourceDataPath is data path of all source files - SourceDataPath() string - - // PutFile reads a file from srcPath and uploads it to the destination folder - PutFile(ctx context.Context, backupID, key, srcPath string) error - // PutObject writes bytes to the object with key `key` - PutObject(ctx context.Context, backupID, key string, byes []byte) error - // Initialize initializes backup provider and make sure that app have access rights to write into the object store. - Initialize(ctx context.Context, backupID string) error - - Write(ctx context.Context, backupID, key string, r io.ReadCloser) (int64, error) - Read(ctx context.Context, backupID, key string, w io.WriteCloser) (int64, error) -} diff --git a/entities/modulecapabilities/classification.go b/entities/modulecapabilities/classification.go deleted file mode 100644 index 0b2a278887c720944411d658f5d559a6e4315342..0000000000000000000000000000000000000000 --- a/entities/modulecapabilities/classification.go +++ /dev/null @@ -1,71 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modulecapabilities - -import ( - "context" - - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/search" -) - -type VectorClassSearchParams struct { - Filters *filters.LocalFilter - Pagination *filters.Pagination - ClassName string - Properties []string -} - -type VectorClassSearchRepo interface { - VectorClassSearch(ctx context.Context, params VectorClassSearchParams) ([]search.Result, error) -} - -type ClassifyParams struct { - Schema schema.Schema - Params models.Classification - Filters Filters - UnclassifiedItems []search.Result - VectorRepo VectorClassSearchRepo -} - -type Filters interface { - Source() *filters.LocalFilter - Target() *filters.LocalFilter - TrainingSet() *filters.LocalFilter -} - -type Writer interface { - Start() - Store(item search.Result) error - Stop() WriterResults -} - -type WriterResults interface { - SuccessCount() int64 - ErrorCount() int64 - Err() error -} - -type ClassifyItemFn func(item search.Result, itemIndex int, - params models.Classification, filters Filters, writer Writer) error - -type Classifier interface { - Name() string - ClassifyFn(params ClassifyParams) (ClassifyItemFn, error) - ParseClassifierSettings(params *models.Classification) error -} - -type ClassificationProvider interface { - Classifiers() []Classifier -} diff --git a/entities/modulecapabilities/client.go b/entities/modulecapabilities/client.go deleted file mode 100644 index cb43c15bbaaf5b529948f6469b3cb32c916cac6b..0000000000000000000000000000000000000000 --- a/entities/modulecapabilities/client.go +++ /dev/null @@ -1,29 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modulecapabilities - -import "context" - -type VectorizerClient interface { - MultiVectorForWord(ctx context.Context, - words []string) ([][]float32, error) - VectorOnlyForCorpi(ctx context.Context, corpi []string, - overrides map[string]string) ([]float32, error) -} - -type MetaProvider interface { - MetaInfo() (map[string]interface{}, error) -} - -type Client interface { - Vectorizers() map[string]VectorizerClient -} diff --git a/entities/modulecapabilities/config.go b/entities/modulecapabilities/config.go deleted file mode 100644 index bfdd07bc67624973793cf8802f449607a169f84c..0000000000000000000000000000000000000000 --- a/entities/modulecapabilities/config.go +++ /dev/null @@ -1,48 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modulecapabilities - -import ( - "context" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/schema" -) - -// ClassConfigurator is an optional capability interface which a module MAY -// implement. If it is implemented, all methods will be called when the user -// adds or updates a class which has the module set as the vectorizer -type ClassConfigurator interface { - // ClassDefaults provides the defaults for a per-class module config. The - // module provider will merge the props into the user-specified config with - // the user-provided values taking precedence - ClassConfigDefaults() map[string]interface{} - - // PropertyConfigDefaults provides the defaults for a per-property module - // config. The module provider will merge the props into the user-specified - // config with the user-provided values taking precedence. The property's - // dataType MAY be taken into consideration when deciding defaults. - // dataType is not guaranteed to be non-nil, it might be nil in the case a - // user specified an invalid dataType, as some validation only occurs after - // defaults are set. - PropertyConfigDefaults(dataType *schema.DataType) map[string]interface{} - - // ValidateClass MAY validate anything about the class, except the config of - // another module. The specified ClassConfig can be used to easily retrieve - // the config specific for the module. For example, a module could iterate - // over class.Properties and call classConfig.Property(prop.Name) to validate - // the per-property config. A module MUST NOT extract another module's config - // from class.ModuleConfig["other-modules-name"]. - ValidateClass(ctx context.Context, class *models.Class, - classConfig moduletools.ClassConfig) error -} diff --git a/entities/modulecapabilities/graphql.go b/entities/modulecapabilities/graphql.go deleted file mode 100644 index 5837f711fa1d6570146c3ee181ea80c1437a2ebc..0000000000000000000000000000000000000000 --- a/entities/modulecapabilities/graphql.go +++ /dev/null @@ -1,54 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modulecapabilities - -import ( - "github.com/tailor-inc/graphql" -) - -// GetArgumentsFn generates get graphql config for a given classname -type GetArgumentsFn = func(classname string) *graphql.ArgumentConfig - -// AggregateArgumentsFn generates aggregate graphql config for a given classname -type AggregateArgumentsFn = func(classname string) *graphql.ArgumentConfig - -// ExploreArgumentsFn generates explore graphql config -type ExploreArgumentsFn = func() *graphql.ArgumentConfig - -// ExtractFn extracts graphql params to given struct implementation -type ExtractFn = func(param map[string]interface{}) interface{} - -// NearParam defines params with certainty information -type NearParam interface { - GetCertainty() float64 - GetDistance() float64 - SimilarityMetricProvided() bool -} - -// ValidateFn validates a given module param -type ValidateFn = func(param interface{}) error - -// GraphQLArgument defines all the needed settings / methods -// to add a module specific graphql argument -type GraphQLArgument struct { - GetArgumentsFunction GetArgumentsFn - AggregateArgumentsFunction AggregateArgumentsFn - ExploreArgumentsFunction ExploreArgumentsFn - ExtractFunction ExtractFn - ValidateFunction ValidateFn -} - -// GraphQLArguments defines the capabilities of modules to add their -// arguments to graphql API -type GraphQLArguments interface { - Arguments() map[string]GraphQLArgument -} diff --git a/entities/modulecapabilities/module.go b/entities/modulecapabilities/module.go deleted file mode 100644 index c8aa2639a96dc240a519b8d53ed92d4b0bd27d79..0000000000000000000000000000000000000000 --- a/entities/modulecapabilities/module.go +++ /dev/null @@ -1,64 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modulecapabilities - -import ( - "context" - "net/http" - - "github.com/weaviate/weaviate/entities/moduletools" -) - -type ModuleType string - -const ( - Backup ModuleType = "Backup" - Extension ModuleType = "Extension" - Img2Vec ModuleType = "Img2Vec" - Multi2Vec ModuleType = "Multi2Vec" - Ref2Vec ModuleType = "Ref2Vec" - Text2MultiVec ModuleType = "Text2MultiVec" - Text2TextGenerative ModuleType = "Text2TextGenerative" - Text2TextSummarize ModuleType = "Text2TextSummarize" - Text2TextReranker ModuleType = "Text2TextReranker" - Text2TextNER ModuleType = "Text2TextNER" - Text2TextQnA ModuleType = "Text2TextQnA" - Text2Vec ModuleType = "Text2Vec" -) - -type Module interface { - Name() string - Init(ctx context.Context, params moduletools.ModuleInitParams) error - RootHandler() http.Handler // TODO: remove from overall module, this is a capability - Type() ModuleType -} - -type ModuleExtension interface { - Module - InitExtension(modules []Module) error -} - -type ModuleDependency interface { - Module - InitDependency(modules []Module) error -} - -type Dependency interface { - ModuleName() string - Argument() string - GraphQLArgument() GraphQLArgument - VectorSearch() VectorForParams -} - -type ModuleHasAltNames interface { - AltNames() []string -} diff --git a/entities/modulecapabilities/searcher.go b/entities/modulecapabilities/searcher.go deleted file mode 100644 index 2d0818d6825b3c89fa96ed3a8ace664bf538a9bd..0000000000000000000000000000000000000000 --- a/entities/modulecapabilities/searcher.go +++ /dev/null @@ -1,50 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modulecapabilities - -import ( - "context" - - "github.com/go-openapi/strfmt" - "github.com/weaviate/weaviate/entities/moduletools" -) - -// FindVectorFn method for getting a vector of given object by its ID -type FindVectorFn = func(ctx context.Context, className string, id strfmt.UUID, tenant string) ([]float32, error) - -// VectorForParams defines method for passing a raw searcher content to the module -// and exchanging it for a vector. Warning: Argument "cfg" -// (moduletools.ClassConfig) is not guaranteed to be non-nil. Implementations -// have to provide a nil check before using it. It is generally present on -// class-based action, but is not present on Cross-Class requests, such as -// Explore {} -type VectorForParams = func(ctx context.Context, params interface{}, - className string, findVectorFn FindVectorFn, cfg moduletools.ClassConfig) ([]float32, error) - -// ArgumentVectorForParams contains argument definitions and it's respective -// vector for params search method -type ArgumentVectorForParams = map[string]VectorForParams - -// Searcher defines all methods for all searchers -// for getting a vector from a given raw searcher content -type Searcher interface { - VectorSearches() ArgumentVectorForParams -} - -// ModuleArgumentVectorForParams contains module's argument definitions and it's -// vector for params search method -type ModuleArgumentVectorForParams = map[string]ArgumentVectorForParams - -// DependencySearcher defines all of the available searches loaded as a dependency -type DependencySearcher interface { - VectorSearches() ModuleArgumentVectorForParams -} diff --git a/entities/modulecapabilities/texttransformer.go b/entities/modulecapabilities/texttransformer.go deleted file mode 100644 index 69ef5246d4f7185150b91d9fa0ad4b0aa7c3bc97..0000000000000000000000000000000000000000 --- a/entities/modulecapabilities/texttransformer.go +++ /dev/null @@ -1,23 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modulecapabilities - -// TextTransform performs text transformation operation -type TextTransform interface { - Transform(in []string) ([]string, error) -} - -// TextTransformers defines all text transformers -// for given arguments -type TextTransformers interface { - TextTransformers() map[string]TextTransform -} diff --git a/entities/modulecapabilities/vectorizer.go b/entities/modulecapabilities/vectorizer.go deleted file mode 100644 index 5e03e377af534d02ab02674b81c68d56442b0f53..0000000000000000000000000000000000000000 --- a/entities/modulecapabilities/vectorizer.go +++ /dev/null @@ -1,48 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modulecapabilities - -import ( - "context" - - "github.com/go-openapi/strfmt" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/search" -) - -type Vectorizer interface { - // VectorizeObject should mutate the object which is passed in as a pointer-type - // by extending it with the desired vector and - if applicable - any meta - // information as part of _additional properties - VectorizeObject(ctx context.Context, obj *models.Object, objDiff *moduletools.ObjectDiff, - cfg moduletools.ClassConfig) error -} - -type FindObjectFn = func(ctx context.Context, class string, id strfmt.UUID, - props search.SelectProperties, adds additional.Properties, tenant string) (*search.Result, error) - -// ReferenceVectorizer is implemented by ref2vec modules, which calculate a target -// object's vector based only on the vectors of its references. If the object has -// no references, the object will have a nil vector -type ReferenceVectorizer interface { - // VectorizeObject should mutate the object which is passed in as a pointer-type - // by extending it with the desired vector, which is calculated by the module - VectorizeObject(ctx context.Context, object *models.Object, - cfg moduletools.ClassConfig, findObjectFn FindObjectFn) error -} - -type InputVectorizer interface { - VectorizeInput(ctx context.Context, input string, - cfg moduletools.ClassConfig) ([]float32, error) -} diff --git a/entities/moduletools/config.go b/entities/moduletools/config.go deleted file mode 100644 index f1516810ba3b714a526163d9216f0cb7bb813c9f..0000000000000000000000000000000000000000 --- a/entities/moduletools/config.go +++ /dev/null @@ -1,22 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package moduletools - -// ClassConfig is a helper type which is passed to the module to read it's -// per-class config. This is - among other places - used when vectorizing and -// when validation schema config -type ClassConfig interface { - Tenant() string - Class() map[string]interface{} - ClassByModuleName(moduleName string) map[string]interface{} - Property(propName string) map[string]interface{} -} diff --git a/entities/moduletools/doc.go b/entities/moduletools/doc.go deleted file mode 100644 index 59069a38995effdc83dcc2367ec0c0627acd9c13..0000000000000000000000000000000000000000 --- a/entities/moduletools/doc.go +++ /dev/null @@ -1,14 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// moduletools contains helpers that are passed to modules as part of their -// capability methods -package moduletools diff --git a/entities/moduletools/init_params.go b/entities/moduletools/init_params.go deleted file mode 100644 index f5bf5317800b4c99e7b81cba1eca4f8c72a9622b..0000000000000000000000000000000000000000 --- a/entities/moduletools/init_params.go +++ /dev/null @@ -1,53 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package moduletools - -import ( - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/usecases/config" -) - -type ModuleInitParams interface { - GetStorageProvider() StorageProvider - GetAppState() interface{} - GetLogger() logrus.FieldLogger - GetConfig() config.Config -} - -type InitParams struct { - storageProvider StorageProvider - appState interface{} - config config.Config - logger logrus.FieldLogger -} - -func NewInitParams(storageProvider StorageProvider, appState interface{}, - config config.Config, logger logrus.FieldLogger, -) ModuleInitParams { - return &InitParams{storageProvider, appState, config, logger} -} - -func (p *InitParams) GetStorageProvider() StorageProvider { - return p.storageProvider -} - -func (p *InitParams) GetAppState() interface{} { - return p.appState -} - -func (p *InitParams) GetLogger() logrus.FieldLogger { - return p.logger -} - -func (p *InitParams) GetConfig() config.Config { - return p.config -} diff --git a/entities/moduletools/object_diff.go b/entities/moduletools/object_diff.go deleted file mode 100644 index a68f646493b57e3af1091a24f3f51513f1018eb4..0000000000000000000000000000000000000000 --- a/entities/moduletools/object_diff.go +++ /dev/null @@ -1,103 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package moduletools - -type ObjectDiff struct { - oldVec []float32 - oldPropValues map[string]interface{} - newPropValues map[string]interface{} -} - -func NewObjectDiff(oldVec []float32) *ObjectDiff { - return &ObjectDiff{ - oldVec: oldVec, - oldPropValues: map[string]interface{}{}, - newPropValues: map[string]interface{}{}, - } -} - -func (od *ObjectDiff) WithProp(propName string, oldValue, newValue interface{}) *ObjectDiff { - od.oldPropValues[propName] = oldValue - od.newPropValues[propName] = newValue - return od -} - -func (od *ObjectDiff) GetVec() []float32 { - return od.oldVec -} - -func (od *ObjectDiff) IsChangedProp(propName string) bool { - oldVal, oldExists := od.oldPropValues[propName] - newVal, newExists := od.newPropValues[propName] - - if !oldExists && !newExists { - return false - } - if !(oldExists && newExists) { - return true - } - - // only strings are vectorized, therefore property changes are determined - // on values of types string, []string and []interface{} which are in fact strings - switch o := oldVal.(type) { - case string: - if n, ok := newVal.(string); ok { - return n != o - } - case []string: - if ns, ok := newVal.([]string); ok { - if len(ns) != len(o) { - return true - } - for i := range o { - if ns[i] != o[i] { - return true - } - } - return false - } else if ni, ok := newVal.([]interface{}); ok { - if len(ni) != len(o) { - return true - } - for i := range o { - if ni[i] != o[i] { - return true - } - } - return false - } - case []interface{}: - if ns, ok := newVal.([]string); ok { - if len(ns) != len(o) { - return true - } - for i := range o { - if ns[i] != o[i] { - return true - } - } - return false - } else if ni, ok := newVal.([]interface{}); ok { - if len(ni) != len(o) { - return true - } - for i := range o { - if ni[i] != o[i] { - return true - } - } - return false - } - } - - return true -} diff --git a/entities/moduletools/object_diff_test.go b/entities/moduletools/object_diff_test.go deleted file mode 100644 index c2beed6f51879a4aae11b139ff1295a82ecb0895..0000000000000000000000000000000000000000 --- a/entities/moduletools/object_diff_test.go +++ /dev/null @@ -1,115 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package moduletools - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestObjectDiff(t *testing.T) { - t.Run("strings are the same", func(t *testing.T) { - objDiff := NewObjectDiff(nil). - WithProp("sameStrings", "Some string", "Some string") - - assert.False(t, objDiff.IsChangedProp("sameStrings")) - }) - - t.Run("strings are different", func(t *testing.T) { - objDiff := NewObjectDiff(nil). - WithProp("differentStrings1", "Some string", "some string"). - WithProp("differentStrings2", "Some string", "Some string, but different") - - assert.True(t, objDiff.IsChangedProp("differentStrings1")) - assert.True(t, objDiff.IsChangedProp("differentStrings2")) - }) - - t.Run("string slices are the same", func(t *testing.T) { - objDiff := NewObjectDiff(nil). - WithProp("sameStringSlices1", []string{"aa", "bb", "cc"}, []string{"aa", "bb", "cc"}). - WithProp("sameStringSlices2", []string{"aa", "bb", "cc"}, []interface{}{"aa", "bb", "cc"}). - WithProp("sameStringSlices3", []interface{}{"aa", "bb", "cc"}, []string{"aa", "bb", "cc"}). - WithProp("sameStringSlices4", []interface{}{"aa", "bb", "cc"}, []interface{}{"aa", "bb", "cc"}). - WithProp("sameStringSlices5", []string{}, []string{}). - WithProp("sameStringSlices6", []string{}, []interface{}{}). - WithProp("sameStringSlices7", []interface{}{}, []string{}). - WithProp("sameStringSlices8", []interface{}{}, []interface{}{}) - - for _, prop := range []string{ - "sameStringSlices1", "sameStringSlices2", "sameStringSlices3", "sameStringSlices4", - "sameStringSlices5", "sameStringSlices6", "sameStringSlices7", "sameStringSlices8", - } { - assert.False(t, objDiff.IsChangedProp(prop)) - } - }) - - t.Run("string slices are different", func(t *testing.T) { - objDiff := NewObjectDiff(nil). - WithProp("differentStringSlices1", []string{"aa", "bb", "cc"}, []string{"aa", "bb", "cc", "dd"}). - WithProp("differentStringSlices2", []string{"aa", "bb", "cc"}, []interface{}{"aa", "bb", "cc", "dd"}). - WithProp("differentStringSlices3", []interface{}{"aa", "bb", "cc"}, []string{"aa", "bb", "cc", "dd"}). - WithProp("differentStringSlices4", []interface{}{"aa", "bb", "cc"}, []interface{}{"aa", "bb", "cc", "dd"}). - WithProp("differentStringSlices5", []string{"aa", "bb", "cc"}, []string{"cc", "bb", "aa"}). - WithProp("differentStringSlices6", []string{"aa", "bb", "cc"}, []interface{}{"cc", "bb", "aa"}). - WithProp("differentStringSlices7", []interface{}{"aa", "bb", "cc"}, []string{"cc", "bb", "aa"}). - WithProp("differentStringSlices8", []interface{}{"aa", "bb", "cc"}, []interface{}{"cc", "bb", "aa"}) - - for _, prop := range []string{ - "differentStringSlices1", "differentStringSlices2", "differentStringSlices3", "differentStringSlices4", - "differentStringSlices5", "differentStringSlices6", "differentStringSlices7", "differentStringSlices8", - } { - assert.True(t, objDiff.IsChangedProp(prop)) - } - }) - - t.Run("nils are different", func(t *testing.T) { - objDiff := NewObjectDiff(nil). - WithProp("nil1", "some value", nil). - WithProp("nil2", nil, "some value"). - WithProp("nil3", []string{"some value"}, nil). - WithProp("nil4", nil, []string{"some value"}) - - for _, prop := range []string{"nil1", "nil2", "nil3", "nil4"} { - assert.True(t, objDiff.IsChangedProp(prop)) - } - }) - - t.Run("not set is the same", func(t *testing.T) { - objDiff := NewObjectDiff(nil) - - assert.False(t, objDiff.IsChangedProp("notSet")) - }) - - t.Run("non strings are different", func(t *testing.T) { - objDiff := NewObjectDiff(nil). - WithProp("float1", 1.23, 1.23). - WithProp("float2", 1.23, 1.234). - WithProp("int1", 1, 1). - WithProp("int2", 1, 2). - WithProp("bool1", true, true). - WithProp("bool2", true, false). - WithProp("floatSlice1", []float64{1.23}, []float64{1.23}). - WithProp("floatSlice2", []float64{1.23}, []float64{1.23, 1.234}). - WithProp("intSlice1", []int64{1}, []int64{1}). - WithProp("intSlice2", []int64{1}, []int64{1, 2}). - WithProp("boolSlice1", []bool{false}, []bool{false}). - WithProp("boolSlice2", []bool{false}, []bool{false, true}) - - for _, prop := range []string{ - "float1", "float2", "int1", "int2", "bool1", "bool2", - "floatSlice1", "floatSlice2", "intSlice1", "intSlice2", "boolSlice1", "boolSlice2", - } { - assert.True(t, objDiff.IsChangedProp(prop)) - } - }) -} diff --git a/entities/moduletools/storage.go b/entities/moduletools/storage.go deleted file mode 100644 index 8b3e54a64e81bc5a51d90c235c5daf9d03b86fed..0000000000000000000000000000000000000000 --- a/entities/moduletools/storage.go +++ /dev/null @@ -1,25 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package moduletools - -type StorageProvider interface { - Storage(name string) (Storage, error) - DataPath() string -} - -type ScanFn func(k, v []byte) (bool, error) - -type Storage interface { - Get(key []byte) ([]byte, error) - Scan(scan ScanFn) error - Put(key, value []byte) error -} diff --git a/entities/multi/get.go b/entities/multi/get.go deleted file mode 100644 index 27477953fa4ddd8ccdbe1edf2e4560df2ab6e6be..0000000000000000000000000000000000000000 --- a/entities/multi/get.go +++ /dev/null @@ -1,18 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package multi - -type Identifier struct { - ID string - ClassName string - OriginalPosition int -} diff --git a/entities/replication/config.go b/entities/replication/config.go deleted file mode 100644 index 3b7f534229754029060094d52ae0ab77464a3ca7..0000000000000000000000000000000000000000 --- a/entities/replication/config.go +++ /dev/null @@ -1,21 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package replication - -// GlobalConfig represents system-wide config that may restrict settings of an -// individual class -type GlobalConfig struct { - // MinimumFactor can enforce replication. For example, with MinimumFactor set - // to 2, users can no longer create classes with a factor of 1, therefore - // forcing them to have replicated classes. - MinimumFactor int `json:"minimum_factor" yaml:"minimum_factor"` -} diff --git a/entities/schema/accessors.go b/entities/schema/accessors.go deleted file mode 100644 index 8d9557a8679d67fc15be1b03ea895c1e1b77a9ae..0000000000000000000000000000000000000000 --- a/entities/schema/accessors.go +++ /dev/null @@ -1,78 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package schema - -import ( - "github.com/weaviate/weaviate/entities/models" -) - -func (s *Schema) GetClass(className ClassName) *models.Class { - class, err := GetClassByName(s.Objects, string(className)) - if err != nil { - return nil - } - - return class -} - -// FindClassByName will find either a Thing or Class by name. -func (s *Schema) FindClassByName(className ClassName) *models.Class { - semSchemaClass, err := GetClassByName(s.Objects, string(className)) - if err == nil { - return semSchemaClass - } - - return nil -} - -// func (s *Schema) GetKindOfClass(className ClassName) (kind.Kind, bool) { -// _, err := GetClassByName(s.Objects, string(className)) -// if err == nil { -// return kind.Object, true -// } - -// return "", false -// } - -func (s *Schema) GetProperty(className ClassName, propName PropertyName) (*models.Property, error) { - semSchemaClass, err := GetClassByName(s.Objects, string(className)) - if err != nil { - return nil, err - } - - semProp, err := GetPropertyByName(semSchemaClass, string(propName)) - if err != nil { - return nil, err - } - - return semProp, nil -} - -func (s *Schema) GetPropsOfType(propType string) []ClassAndProperty { - return extractAllOfPropType(s.Objects.Classes, propType) -} - -func extractAllOfPropType(classes []*models.Class, propType string) []ClassAndProperty { - var result []ClassAndProperty - for _, class := range classes { - for _, prop := range class.Properties { - if prop.DataType[0] == propType { - result = append(result, ClassAndProperty{ - ClassName: ClassName(class.Class), - PropertyName: PropertyName(prop.Name), - }) - } - } - } - - return result -} diff --git a/entities/schema/accessors_test.go b/entities/schema/accessors_test.go deleted file mode 100644 index 4b13673bec4307acbc3829db680e9d48a3e58288..0000000000000000000000000000000000000000 --- a/entities/schema/accessors_test.go +++ /dev/null @@ -1,110 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package schema - -import ( - "errors" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/entities/models" -) - -func Test_Accessors(t *testing.T) { - car := &models.Class{ - Class: "Car", - Properties: []*models.Property{ - {Name: "modelName", DataType: DataTypeText.PropString(), Tokenization: models.PropertyTokenizationWhitespace}, - {Name: "manufacturerName", DataType: DataTypeText.PropString(), Tokenization: models.PropertyTokenizationWhitespace}, - {Name: "horsepower", DataType: []string{"int"}}, - }, - } - - train := &models.Class{ - Class: "Train", - Properties: []*models.Property{ - {Name: "capacity", DataType: []string{"int"}}, - {Name: "trainCompany", DataType: DataTypeText.PropString(), Tokenization: models.PropertyTokenizationWhitespace}, - }, - } - - action := &models.Class{ - Class: "SomeAction", - Properties: []*models.Property{}, - } - - sch := Empty() - sch.Objects.Classes = []*models.Class{car, train, action} - - t.Run("GetClass by kind and name", func(t *testing.T) { - class := sch.GetClass("Car") - assert.Equal(t, car, class) - - class = sch.GetClass("Invalid") - assert.Equal(t, (*models.Class)(nil), class) - }) - - t.Run("FindClass by name (without providing the kind)", func(t *testing.T) { - class := sch.FindClassByName("Car") - assert.Equal(t, car, class) - - class = sch.FindClassByName("SomeAction") - assert.Equal(t, action, class) - - class = sch.FindClassByName("Invalid") - assert.Equal(t, (*models.Class)(nil), class) - }) - - t.Run("GetPropsOfType", func(t *testing.T) { - props := sch.GetPropsOfType(DataTypeText.String()) - - expectedProps := []ClassAndProperty{ - { - ClassName: "Car", - PropertyName: "modelName", - }, - { - ClassName: "Car", - PropertyName: "manufacturerName", - }, - { - ClassName: "Train", - PropertyName: "trainCompany", - }, - } - - assert.ElementsMatch(t, expectedProps, props) - }) - - t.Run("GetProperty by kind, classname, name", func(t *testing.T) { - prop, err := sch.GetProperty("Car", "modelName") - assert.Nil(t, err) - - expectedProp := &models.Property{ - Name: "modelName", - DataType: DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - } - - assert.Equal(t, expectedProp, prop) - }) - - t.Run("GetProperty for invalid class", func(t *testing.T) { - _, err := sch.GetProperty("WrongClass", "modelName") - assert.Equal(t, errors.New("no such class with name 'WrongClass' found in the schema. Check your schema files for which classes are available"), err) - }) - - t.Run("GetProperty for invalid prop", func(t *testing.T) { - _, err := sch.GetProperty("Car", "wrongProperty") - assert.Equal(t, errors.New("no such prop with name 'wrongProperty' found in class 'Car' in the schema. Check your schema files for which properties in this class are available"), err) - }) -} diff --git a/entities/schema/backward_compat.go b/entities/schema/backward_compat.go deleted file mode 100644 index 7b77e24b30c6bd51f99eb6e4834075828bf2be59..0000000000000000000000000000000000000000 --- a/entities/schema/backward_compat.go +++ /dev/null @@ -1,198 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package schema - -import ( - errors_ "errors" - "fmt" - "strings" - - "github.com/weaviate/weaviate/entities/models" -) - -type PropertyInterface interface { - GetName() string - GetNestedProperties() []*models.NestedProperty -} - -// GetClassByName returns the class by its name -func GetClassByName(s *models.Schema, className string) (*models.Class, error) { - if s == nil { - return nil, fmt.Errorf(ErrorNoSuchClass, className) - } - // For each class - for _, class := range s.Classes { - // Check if the name of the class is the given name, that's the class we need - if class.Class == className { - return class, nil - } - } - - return nil, fmt.Errorf(ErrorNoSuchClass, className) -} - -// GetPropertyByName returns the class by its name -func GetPropertyByName(c *models.Class, propName string) (*models.Property, error) { - // For each class-property - for _, prop := range c.Properties { - // Check if the name of the property is the given name, that's the property we need - if prop.Name == strings.Split(propName, ".")[0] { - return prop, nil - } - } - - return nil, fmt.Errorf(ErrorNoSuchProperty, propName, c.Class) -} - -// GetPropertyDataType checks whether the given string is a valid data type -func GetPropertyDataType(class *models.Class, propertyName string) (*DataType, error) { - // Get the class-property - prop, err := GetPropertyByName(class, propertyName) - if err != nil { - return nil, err - } - - // Init the return value - var returnDataType DataType - - // For each data type - for _, dataType := range prop.DataType { - if len(dataType) == 0 { - return nil, fmt.Errorf("invalid-dataType") - } - // Get the first letter to see if it is a capital - firstLetter := string(dataType[0]) - if strings.ToUpper(firstLetter) == firstLetter { - returnDataType = DataTypeCRef - } else { - // Get the value-data type (non-cref), return error if there is one, otherwise assign it to return data type - valueDataType, err := GetValueDataTypeFromString(dataType) - if err != nil { - return nil, err - } - returnDataType = *valueDataType - } - } - return &returnDataType, nil -} - -func GetNestedPropertyByName[P PropertyInterface](p P, propName string) (*models.NestedProperty, error) { - // For each nested-property - for _, prop := range p.GetNestedProperties() { - // Check if the name of the property is the given name, that's the property we need - if prop.Name == strings.Split(propName, ".")[0] { - return prop, nil - } - } - - return nil, fmt.Errorf(ErrorNoSuchProperty, propName, p.GetName()) -} - -func GetNestedPropertyDataType[P PropertyInterface](p P, propertyName string) (*DataType, error) { - // Get the class-property - prop, err := GetNestedPropertyByName(p, propertyName) - if err != nil { - return nil, err - } - - // Init the return value - var returnDataType DataType - - // For each data type - for _, dataType := range prop.DataType { - if len(dataType) == 0 { - return nil, fmt.Errorf("invalid-dataType") - } - // Get the first letter to see if it is a capital - firstLetter := string(dataType[0]) - if strings.ToUpper(firstLetter) == firstLetter { - returnDataType = DataTypeCRef - } else { - // Get the value-data type (non-cref), return error if there is one, otherwise assign it to return data type - valueDataType, err := GetValueDataTypeFromString(dataType) - if err != nil { - return nil, err - } - returnDataType = *valueDataType - } - } - return &returnDataType, nil -} - -// GetValueDataTypeFromString checks whether the given string is a valid data type -func GetValueDataTypeFromString(dt string) (*DataType, error) { - var returnDataType DataType - - if IsValidValueDataType(dt) { - returnDataType = DataType(dt) - } else { - return nil, errors_.New(ErrorNoSuchDatatype) - } - - return &returnDataType, nil -} - -// IsValidValueDataType checks whether the given string is a valid data type -func IsValidValueDataType(dt string) bool { - switch dt { - case - string(DataTypeString), - string(DataTypeText), - string(DataTypeInt), - string(DataTypeNumber), - string(DataTypeBoolean), - string(DataTypeDate), - string(DataTypeGeoCoordinates), - string(DataTypePhoneNumber), - string(DataTypeBlob), - string(DataTypeUUID), - string(DataTypeUUIDArray), - string(DataTypeStringArray), - string(DataTypeTextArray), - string(DataTypeIntArray), - string(DataTypeNumberArray), - string(DataTypeBooleanArray), - string(DataTypeDateArray), - string(DataTypeObject), - string(DataTypeObjectArray): - return true - } - return false -} - -func IsRefDataType(dt []string) bool { - firstLetter := string(dt[0][0]) - return strings.ToUpper(firstLetter) == firstLetter -} - -func IsBlobDataType(dt []string) bool { - for i := range dt { - if dt[i] == string(DataTypeBlob) { - return true - } - } - return false -} - -func IsArrayDataType(dt []string) bool { - for i := range dt { - switch DataType(dt[i]) { - case DataTypeStringArray, DataTypeTextArray, DataTypeIntArray, - DataTypeNumberArray, DataTypeBooleanArray, DataTypeDateArray, - DataTypeUUIDArray: - return true - default: - // move to the next loop - } - } - return false -} diff --git a/entities/schema/backward_compat_test.go b/entities/schema/backward_compat_test.go deleted file mode 100644 index 70b3c1b762d8eb82bfd44cea5ef7f506b3ee55eb..0000000000000000000000000000000000000000 --- a/entities/schema/backward_compat_test.go +++ /dev/null @@ -1,105 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package schema - -import ( - "testing" -) - -func TestIsArrayDataType(t *testing.T) { - type args struct { - dt []string - } - tests := []struct { - name string - args args - want bool - }{ - { - name: "is string array", - args: args{ - dt: DataTypeTextArray.PropString(), - }, - want: true, - }, - { - name: "is not string array", - args: args{ - dt: DataTypeText.PropString(), - }, - want: false, - }, - { - name: "is text array", - args: args{ - dt: []string{"text[]"}, - }, - want: true, - }, - { - name: "is not text array", - args: args{ - dt: []string{"text"}, - }, - want: false, - }, - { - name: "is number array", - args: args{ - dt: []string{"number[]"}, - }, - want: true, - }, - { - name: "is not number array", - args: args{ - dt: []string{"number"}, - }, - want: false, - }, - { - name: "is int array", - args: args{ - dt: []string{"int[]"}, - }, - want: true, - }, - { - name: "is not int array", - args: args{ - dt: []string{"int"}, - }, - want: false, - }, - { - name: "is not uuid array", - args: args{ - dt: []string{"uuid"}, - }, - want: false, - }, - { - name: "is uuid array", - args: args{ - dt: []string{"uuid[]"}, - }, - want: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := IsArrayDataType(tt.args.dt); got != tt.want { - t.Errorf("IsArrayDataType() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/entities/schema/crossref/bulk_builder.go b/entities/schema/crossref/bulk_builder.go deleted file mode 100644 index 00869c88d060bc5e6c0d1efa088a33937c6d0d1b..0000000000000000000000000000000000000000 --- a/entities/schema/crossref/bulk_builder.go +++ /dev/null @@ -1,107 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package crossref - -import ( - "fmt" - - "github.com/go-openapi/strfmt" - "github.com/weaviate/weaviate/usecases/byteops" -) - -// BulkBuilder is a low-alloc tool to build many beacon strings (as []byte). It -// is optimized to allocate just once as opposed to once per ID. This makes it -// considerably faster when generating 100s of thousand of beacons strings. The -// main intended use case for this is building propValuePairs in ref-filters. -// -// The BulkBuilder makes some estimations for how much memory will be necessary -// based on expected input params. If those requirements get exceeded, it will -// still be safe to use, but will fallback to allocating dynamically. -type BulkBuilder struct { - byteops.ReadWriter - prefix []byte -} - -func NewBulkBuilderWithEstimates(expectedCount int, exampleClassName string, - overheadRatio float64, -) *BulkBuilder { - prefix := []byte("weaviate://localhost/") - - lenOfTypicalClassName := int(float64(len(exampleClassName)) * overheadRatio) - predictedSize := expectedCount * (len(prefix) + 1 + lenOfTypicalClassName + 36) - - bb := &BulkBuilder{ - prefix: prefix, - ReadWriter: byteops.NewReadWriter(make([]byte, predictedSize)), - } - - return bb -} - -func (bb *BulkBuilder) ClassAndID(className string, - id strfmt.UUID, -) []byte { - requiredSpace := len(bb.prefix) + len(id) - if int(bb.Position)+requiredSpace >= len(bb.Buffer) { - return bb.fallbackWithClassName(className, id) - } - - // copy the start pos, we will need this at the end to know what to return to - // the caller - start := bb.Position - bb.CopyBytesToBuffer(bb.prefix) - - // This is a safe way, in case a class-name ever contains non-ASCII - // characters. If we could be 100% sure that a class is ASCII-only, we could - // remove this allocation and instead use the same copy-by-rune approach that - // we use later on for the ID. - bb.CopyBytesToBuffer([]byte(className)) - bb.WriteByte('/') // The separating slash between class and ID - for _, runeValue := range id { - // We know that the UUID-string never contains non-ASCII characters. This - // means it safe to convert the uint32-rune into a uint8. This allows us to - // copy char by char without any additional allocs - bb.WriteByte(uint8(runeValue)) - } - - return bb.Buffer[start:bb.Position] -} - -func (bb *BulkBuilder) LegacyIDOnly(id strfmt.UUID) []byte { - requiredSpace := len(bb.prefix) + len(id) - if int(bb.Position)+requiredSpace >= len(bb.Buffer) { - return bb.fallbackWithoutClassName(id) - } - - // copy the start pos, we will need this at the end to know what to return to - // the caller - start := bb.Position - bb.CopyBytesToBuffer(bb.prefix) - for _, runeValue := range id { - // We know that the UUID-string never contains non-ASCII characters. This - // means it safe to convert the uint32-rune into a uint8. This allows us to - // copy char by char without any additional allocs - bb.WriteByte(uint8(runeValue)) - } - - return bb.Buffer[start:bb.Position] -} - -func (bb *BulkBuilder) fallbackWithClassName( - className string, id strfmt.UUID, -) []byte { - return []byte(fmt.Sprintf("%s%s/%s", bb.prefix, className, id)) -} - -func (bb *BulkBuilder) fallbackWithoutClassName(id strfmt.UUID) []byte { - return []byte(fmt.Sprintf("%s%s", bb.prefix, id)) -} diff --git a/entities/schema/crossref/bulk_builder_test.go b/entities/schema/crossref/bulk_builder_test.go deleted file mode 100644 index 9a6a5a875bd70afed36c1f91ee7c23d7777b6da6..0000000000000000000000000000000000000000 --- a/entities/schema/crossref/bulk_builder_test.go +++ /dev/null @@ -1,109 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package crossref - -import ( - "fmt" - "testing" - - "github.com/go-openapi/strfmt" - "github.com/google/uuid" - "github.com/stretchr/testify/assert" -) - -func TestBulkBuilder(t *testing.T) { - tests := []struct { - name string - expectedFn func(id string) string - estimatedSize int - iterations int - className string - withClassName bool - }{ - { - name: "with class name - enough-prealloc", - withClassName: true, - className: "MyClass", - expectedFn: func(id string) string { - return fmt.Sprintf("weaviate://localhost/MyClass/%s", id) - }, - estimatedSize: 25, - iterations: 25, - }, - { - name: "with class name with non-ASCII- enough-prealloc", - withClassName: true, - className: "My國Class", - expectedFn: func(id string) string { - return fmt.Sprintf("weaviate://localhost/My國Class/%s", id) - }, - estimatedSize: 25, - iterations: 25, - }, - { - name: "with class name - not enough-prealloc", - withClassName: true, - className: "MyClass", - expectedFn: func(id string) string { - return fmt.Sprintf("weaviate://localhost/MyClass/%s", id) - }, - estimatedSize: 10, - iterations: 25, - }, - { - name: "with class name with non-ASCII - not enough-prealloc", - withClassName: true, - className: "My國Class", - expectedFn: func(id string) string { - return fmt.Sprintf("weaviate://localhost/My國Class/%s", id) - }, - estimatedSize: 10, - iterations: 25, - }, - { - name: "without class name - enough-prealloc", - withClassName: false, - className: "MyClass", - expectedFn: func(id string) string { - return fmt.Sprintf("weaviate://localhost/%s", id) - }, - estimatedSize: 25, - iterations: 25, - }, - { - name: "without class name - not enough-prealloc", - withClassName: false, - className: "MyClass", - expectedFn: func(id string) string { - return fmt.Sprintf("weaviate://localhost/%s", id) - }, - estimatedSize: 10, - iterations: 25, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - bb := NewBulkBuilderWithEstimates(tt.estimatedSize, tt.className, 1.00) - for i := 0; i < tt.iterations; i++ { - id := uuid.New().String() - if tt.withClassName { - res := bb.ClassAndID(tt.className, strfmt.UUID(id)) - assert.Equal(t, tt.expectedFn(id), string(res)) - } else { - res := bb.LegacyIDOnly(strfmt.UUID(id)) - assert.Equal(t, tt.expectedFn(id), string(res)) - } - } - }) - } -} diff --git a/entities/schema/crossref/crossref.go b/entities/schema/crossref/crossref.go deleted file mode 100644 index 877773520bfbca32e82abca9c6424405a1421e03..0000000000000000000000000000000000000000 --- a/entities/schema/crossref/crossref.go +++ /dev/null @@ -1,117 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package crossref - -import ( - "fmt" - "net/url" - "strings" - - "github.com/go-openapi/strfmt" - "github.com/weaviate/weaviate/entities/models" -) - -const ( - _LocalHost = "localhost" - _Schema = "weaviate" -) - -// Ref is an abstraction of the cross-refs which are specified in a URI format -// in the API. When this type is used it is safe to assume that a Ref is -// semantically valid. This guarantee would not be possible on the URI format, -// as the URI can be well-formed, but not contain the data we expect in it. -// Do not use directly, such as crossref.Ref{}, as you won't have any -// guarantees in this case. Always use one of the parsing options or New() -type Ref struct { - Local bool `json:"local"` - PeerName string `json:"peerName"` - TargetID strfmt.UUID `json:"targetID"` - Class string `json:"className"` -} - -// Parse is a safe way to generate a Ref, as it will error if any of the input -// parameters are not as expected. -func Parse(uriString string) (*Ref, error) { - uri, err := url.Parse(uriString) - if err != nil || uri.Path == "" { - return nil, fmt.Errorf("invalid cref URI: %s", err) - } - - segments := strings.Split(uri.Path, "/") - class, id, idx := "", "", 1 - switch len(segments) { - case 3: - class = segments[1] - id = segments[2] - idx = 2 - case 2: - id = segments[1] - default: - return nil, fmt.Errorf( - "invalid cref URI: path must be of format '/', but got '%s'", uri.Path) - } - if ok := strfmt.IsUUID(id); !ok { - return nil, fmt.Errorf("invalid cref URI: %dnd path segment must be uuid, but got '%s'", - idx, id) - } - - return &Ref{ - Local: uri.Host == _LocalHost, - PeerName: uri.Host, - TargetID: strfmt.UUID(id), - Class: class, - }, nil -} - -// ParseSingleRef is a safe way to generate a Ref from a models.SingleRef, a -// helper construct that represents the API structure. It will error if any of -// the input parameters are not as expected. -func ParseSingleRef(singleRef *models.SingleRef) (*Ref, error) { - return Parse(string(singleRef.Beacon)) -} - -// New is a safe way to generate a Reference, as all required arguments must be -// set in the constructor fn -func New(peerName string, class string, target strfmt.UUID) *Ref { - return &Ref{ - Local: peerName == _LocalHost, - PeerName: peerName, - TargetID: target, - Class: class, - } -} - -func NewLocalhost(class string, target strfmt.UUID) *Ref { - return New(_LocalHost, class, target) -} - -func (r *Ref) String() string { - path := fmt.Sprintf("%s/%s", r.Class, r.TargetID) - if r.Class == "" { - path = fmt.Sprintf("/%s", r.TargetID) - } - uri := url.URL{ - Host: r.PeerName, - Scheme: _Schema, - Path: path, - } - - return uri.String() -} - -// SingleRef converts the parsed Ref back into the API helper construct -// containing a stringified representation (URI format) of the Ref -func (r *Ref) SingleRef() *models.SingleRef { - return &models.SingleRef{ - Beacon: strfmt.URI(r.String()), - } -} diff --git a/entities/schema/crossref/crossref_source.go b/entities/schema/crossref/crossref_source.go deleted file mode 100644 index f42dafceacdb227c62ab4e729c4ffbcb7686ea6c..0000000000000000000000000000000000000000 --- a/entities/schema/crossref/crossref_source.go +++ /dev/null @@ -1,102 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package crossref - -import ( - "fmt" - "net/url" - "strings" - "unicode" - - "github.com/go-openapi/strfmt" - "github.com/weaviate/weaviate/entities/schema" -) - -// RefSource is an abstraction of the source of a cross-ref. The opposite would -// be Ref which represents the target instead. A RefSource is specified in a URI -// format in the API. When this type is used it is safe to assume that a Ref is -// semantically valid. This guarantee would not be possible on the URI format, -// as the URI can be well-formed, but not contain the data we expect in it. Do -// not use directly, such as crossref.RefSource{}, as you won't have any -// guarantees in this case. Always use one of the parsing options or New() -type RefSource struct { - Local bool `json:"local"` - PeerName string `json:"peerName"` - Property schema.PropertyName `json:"property"` - Class schema.ClassName `json:"class"` - TargetID strfmt.UUID `json:"targetID"` -} - -func NewSource(className schema.ClassName, - property schema.PropertyName, id strfmt.UUID, -) *RefSource { - return &RefSource{ - Local: true, - PeerName: "localhost", - Class: className, - TargetID: id, - Property: property, - } -} - -// ParseSource is a safe way to generate a RefSource, as it will error if any -// of the input parameters are not as expected. -func ParseSource(uriString string) (*RefSource, error) { - uri, err := url.Parse(uriString) - if err != nil { - return nil, fmt.Errorf("invalid cref URI: %s", err) - } - - pathSegments := strings.Split(uri.Path, "/") - if len(pathSegments) != 4 { - return nil, fmt.Errorf( - "invalid cref URI: must use long-form: path must be of format '///', but got '%s'", - uri.Path) - } - - if ok := strfmt.IsUUID(pathSegments[2]); !ok { - return nil, fmt.Errorf("invalid cref URI: 2nd path segment must be uuid, but got '%s'", - pathSegments[3]) - } - - class := pathSegments[1] - if class == "" { - return nil, fmt.Errorf("className cannot be empty") - } - - if unicode.IsLower(rune(class[0])) { - return nil, fmt.Errorf("className must start with an uppercase letter, but got %s", class) - } - - property := pathSegments[3] - if property == "" { - return nil, fmt.Errorf("property cannot be empty") - } - - return &RefSource{ - Local: (uri.Host == "localhost"), - PeerName: uri.Host, - TargetID: strfmt.UUID(pathSegments[2]), - Class: schema.ClassName(class), - Property: schema.PropertyName(property), - }, nil -} - -func (r *RefSource) String() string { - uri := url.URL{ - Host: r.PeerName, - Scheme: "weaviate", - Path: fmt.Sprintf("/%s/%s/%s", r.Class, r.TargetID, r.Property), - } - - return uri.String() -} diff --git a/entities/schema/crossref/crossref_source_test.go b/entities/schema/crossref/crossref_source_test.go deleted file mode 100644 index bebd096f6d39c566bc75ed37dc429251c08098ff..0000000000000000000000000000000000000000 --- a/entities/schema/crossref/crossref_source_test.go +++ /dev/null @@ -1,169 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package crossref - -import ( - "testing" - - "github.com/go-openapi/strfmt" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/schema" -) - -func Test_Source_ParsingFromString(t *testing.T) { - t.Run("from a local object ref that is well-formed", func(t *testing.T) { - uri := "weaviate://localhost/MyClassName/c2cd3f91-0160-477e-869a-8da8829e0a4d/myRefProp" - ref, err := ParseSource(uri) - - require.Nil(t, err, "should not error") - - t.Run("is a local ref", func(t *testing.T) { - assert.Equal(t, ref.Local, true) - }) - - t.Run("peerName points to localhost", func(t *testing.T) { - assert.Equal(t, ref.PeerName, "localhost") - }) - - t.Run("id points correctly", func(t *testing.T) { - assert.Equal(t, ref.TargetID, strfmt.UUID("c2cd3f91-0160-477e-869a-8da8829e0a4d")) - }) - - t.Run("the class name is correct", func(t *testing.T) { - assert.Equal(t, ref.Class, schema.ClassName("MyClassName")) - }) - - t.Run("the property name is correct", func(t *testing.T) { - assert.Equal(t, ref.Property, schema.PropertyName("myRefProp")) - }) - - t.Run("assembling a new source and comparing if the match", func(t *testing.T) { - alt := NewSource("MyClassName", "myRefProp", - "c2cd3f91-0160-477e-869a-8da8829e0a4d") - assert.Equal(t, ref, alt) - }) - }) - - t.Run("from a local action ref that is well-formed", func(t *testing.T) { - uri := "weaviate://localhost/MyActionClass/c2cd3f91-0160-477e-869a-8da8829e0a4d/myRefProp" - ref, err := ParseSource(uri) - - require.Nil(t, err, "should not error") - - t.Run("is a local ref", func(t *testing.T) { - assert.Equal(t, ref.Local, true) - }) - - t.Run("peerName points to localhost", func(t *testing.T) { - assert.Equal(t, ref.PeerName, "localhost") - }) - - t.Run("id points correctly", func(t *testing.T) { - assert.Equal(t, ref.TargetID, strfmt.UUID("c2cd3f91-0160-477e-869a-8da8829e0a4d")) - }) - - t.Run("the class name is correct", func(t *testing.T) { - assert.Equal(t, ref.Class, schema.ClassName("MyActionClass")) - }) - - t.Run("the property name is correct", func(t *testing.T) { - assert.Equal(t, ref.Property, schema.PropertyName("myRefProp")) - }) - - t.Run("assembling a new source and comparing if the match", func(t *testing.T) { - alt := NewSource("MyActionClass", "myRefProp", - "c2cd3f91-0160-477e-869a-8da8829e0a4d") - assert.Equal(t, ref, alt) - }) - }) - - t.Run("from a network action ref that is well-formed", func(t *testing.T) { - uri := "weaviate://another-weaviate/SomeActionClass/c2cd3f91-0160-477e-869a-8da8829e0a4d/myRefProp" - ref, err := ParseSource(uri) - - require.Nil(t, err, "should not error") - - t.Run("is a local ref", func(t *testing.T) { - assert.Equal(t, ref.Local, false) - }) - - t.Run("peerName points to localhost", func(t *testing.T) { - assert.Equal(t, ref.PeerName, "another-weaviate") - }) - - t.Run("id points correctly", func(t *testing.T) { - assert.Equal(t, ref.TargetID, strfmt.UUID("c2cd3f91-0160-477e-869a-8da8829e0a4d")) - }) - - t.Run("the class name is correct", func(t *testing.T) { - assert.Equal(t, ref.Class, schema.ClassName("SomeActionClass")) - }) - - t.Run("the property name is correct", func(t *testing.T) { - assert.Equal(t, ref.Property, schema.PropertyName("myRefProp")) - }) - }) - - t.Run("with formatting errors", func(t *testing.T) { - type testCaseError struct { - name string - uri string - } - - tests := []testCaseError{ - { - name: "with an invalid URL", - uri: "i:am:not:a:url", - }, - { - name: "with too few path segments", - uri: "weaviate://localhost/SomeClass", - }, - { - name: "with too many path segments", - uri: "weaviate://localhost/SomeClass/c2cd3f91-0160-477e-869a-8da8829e0a4d/myRefProp/somethingElse", - }, - { - name: "without a property", - uri: "weaviate://localhost/SomeClass/c2cd3f91-0160-477e-869a-8da8829e0a4d/", - }, - { - name: "with an invalid uuid", - uri: "weaviate://localhost/SomeClass/c2cd3f91-iSneakedInHere-477e-869a-8da8829e0a4d", - }, - { - name: "with an invalid kind", // was /humans/SomeClass - uri: "weaviate://localhost/SomeClass/c2cd3f91-0160-477e-869a-8da8829e0a4d", - }, - { - name: "with a lowercased class name", - uri: "weaviate://localhost/someClass/c2cd3f91-0160-477e-869a-8da8829e0a4d/myRefProp", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - _, err := ParseSource(test.uri) - assert.NotNil(t, err, test.name) - }) - } - }) -} - -func Test_Source_GenerateString(t *testing.T) { - uri := "weaviate://localhost/MyClass/c2cd3f91-0160-477e-869a-8da8829e0a4d/myRefProp" - ref, err := ParseSource(uri) - - require.Nil(t, err, "should not error") - assert.Equal(t, uri, ref.String(), "should be the same as the input string") -} diff --git a/entities/schema/crossref/crossref_test.go b/entities/schema/crossref/crossref_test.go deleted file mode 100644 index 716f476050291bde37b5570d2adac9147efcc960..0000000000000000000000000000000000000000 --- a/entities/schema/crossref/crossref_test.go +++ /dev/null @@ -1,233 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package crossref - -import ( - "testing" - - "github.com/go-openapi/strfmt" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" -) - -func TestParseCrossReference(t *testing.T) { - ref := Ref{ - Local: true, - PeerName: _LocalHost, - TargetID: "c2cd3f91-0160-477e-869a-8da8829e0a4d", - Class: "class", - } - tests := []struct { - beacon string - ref Ref - ok bool - }{ - { - beacon: "weaviate://localhost/class/c2cd3f91-0160-477e-869a-8da8829e0a4d", - ref: ref, - ok: true, - }, - { - beacon: "weaviate://remote/class/c2cd3f91-0160-477e-869a-8da8829e0a4d", - ref: Ref{false, "remote", ref.TargetID, "class"}, - ok: true, - }, - { - beacon: "weaviate://localhost/c2cd3f91-0160-477e-869a-8da8829e0a4d", - ref: Ref{true, _LocalHost, ref.TargetID, ""}, - ok: true, - }, - { - beacon: "weaviate://remote/c2cd3f91-0160-477e-869a-8da8829e0a4d", - ref: Ref{false, "remote", ref.TargetID, ""}, - ok: true, - }, - { - beacon: "weaviate://localhost/class/c2cd3f91-0160-477e-869a-8da8829e0a4d/i-shouldnt-be-here", - }, - { - beacon: "weaviate://localhost/class/invalid-id", - }, - { - beacon: "weaviate://localhost/class", - }, - { - beacon: "weaviate://localhost", - }, - { - beacon: "i:am:not:a:url", - }, - } - for i, tc := range tests { - got, err := Parse(tc.beacon) - if (err == nil) != tc.ok { - t.Errorf("%d - Parse(%s) error %v error expected: %t", i, tc.beacon, err, tc.ok) - continue - } - if err != nil { - continue - } - if *got != tc.ref { - t.Errorf("%d - Parse(%s) got %v want %v", i, tc.beacon, *got, tc.ref) - } - if beacon := got.String(); beacon != tc.beacon { - t.Errorf("beacon expected: %v want %v", tc.beacon, beacon) - } - } -} - -func TestSingleRef(t *testing.T) { - ref := NewLocalhost("class", "c2cd3f91-0160-477e-869a-8da8829e0a4d") - expected := &models.SingleRef{ - Beacon: strfmt.URI("weaviate://localhost/class/c2cd3f91-0160-477e-869a-8da8829e0a4d"), - } - sref := ref.SingleRef() - assert.Equal(t, expected, sref, "should create a singleRef") - xref, _ := ParseSingleRef(sref) - assert.Equal(t, ref, xref) -} - -func Test_ParsingFromStringDeprecated(t *testing.T) { - t.Run("from a local object ref that is well-formed", func(t *testing.T) { - uri := "weaviate://localhost/c2cd3f91-0160-477e-869a-8da8829e0a4d" - ref, err := Parse(uri) - - require.Nil(t, err, "should not error") - - t.Run("is a local ref", func(t *testing.T) { - assert.Equal(t, ref.Local, true) - }) - - t.Run("peerName points to localhost", func(t *testing.T) { - assert.Equal(t, ref.PeerName, "localhost") - }) - - t.Run("id points correctly", func(t *testing.T) { - assert.Equal(t, ref.TargetID, strfmt.UUID("c2cd3f91-0160-477e-869a-8da8829e0a4d")) - }) - }) - - t.Run("from a local action ref that is well-formed", func(t *testing.T) { - uri := "weaviate://localhost/c2cd3f91-0160-477e-869a-8da8829e0a4d" - ref, err := Parse(uri) - - require.Nil(t, err, "should not error") - - t.Run("is a local ref", func(t *testing.T) { - assert.Equal(t, ref.Local, true) - }) - - t.Run("peerName points to localhost", func(t *testing.T) { - assert.Equal(t, ref.PeerName, "localhost") - }) - - t.Run("id points correctly", func(t *testing.T) { - assert.Equal(t, ref.TargetID, strfmt.UUID("c2cd3f91-0160-477e-869a-8da8829e0a4d")) - }) - }) - - t.Run("from a network action ref that is well-formed", func(t *testing.T) { - uri := "weaviate://another-weaviate/c2cd3f91-0160-477e-869a-8da8829e0a4d" - ref, err := Parse(uri) - - require.Nil(t, err, "should not error") - - t.Run("is a local ref", func(t *testing.T) { - assert.Equal(t, ref.Local, false) - }) - - t.Run("peerName points to localhost", func(t *testing.T) { - assert.Equal(t, ref.PeerName, "another-weaviate") - }) - - t.Run("id points correctly", func(t *testing.T) { - assert.Equal(t, ref.TargetID, strfmt.UUID("c2cd3f91-0160-477e-869a-8da8829e0a4d")) - }) - }) - - t.Run("with formatting errors", func(t *testing.T) { - type testCaseError struct { - name string - uri string - } - - tests := []testCaseError{ - { - name: "with an invalid URL", - uri: "i:am:not:a:url", - }, - { - name: "with too few path segments", - uri: "weaviate://localhost", - }, - { - name: "with too many path segments", - uri: "weaviate://localhost/c2cd3f91-0160-477e-869a-8da8829e0a4d/i-shouldnt-be-here", - }, - { - name: "with an invalid uuid", - uri: "weaviate://localhost/c2cd3f91-iSneakedInHere-477e-869a-8da8829e0a4d", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - _, err := Parse(test.uri) - assert.NotNil(t, err, test.name) - }) - } - }) -} - -func Test_ParsingFromSingleRefDeprecated(t *testing.T) { - t.Run("from a local object ref that is well-formed", func(t *testing.T) { - uri := strfmt.URI("weaviate://localhost/c2cd3f91-0160-477e-869a-8da8829e0a4d") - singleRef := &models.SingleRef{ - Beacon: uri, - } - ref, err := ParseSingleRef(singleRef) - - require.Nil(t, err, "should not error") - - t.Run("is a local ref", func(t *testing.T) { - assert.Equal(t, ref.Local, true) - }) - - t.Run("peerName points to localhost", func(t *testing.T) { - assert.Equal(t, ref.PeerName, "localhost") - }) - - t.Run("id points correctly", func(t *testing.T) { - assert.Equal(t, ref.TargetID, strfmt.UUID("c2cd3f91-0160-477e-869a-8da8829e0a4d")) - }) - }) -} - -func Test_GenerateStringDeprecated(t *testing.T) { - uri := "weaviate://localhost/c2cd3f91-0160-477e-869a-8da8829e0a4d" - ref, err := Parse(uri) - - require.Nil(t, err, "should not error") - assert.Equal(t, uri, ref.String(), "should be the same as the input string") -} - -func Test_DeprecatedSingleRef(t *testing.T) { - uri := "weaviate://localhost/c2cd3f91-0160-477e-869a-8da8829e0a4d" - ref, err := Parse(uri) - expectedResult := &models.SingleRef{ - Beacon: strfmt.URI(uri), - } - - require.Nil(t, err, "should not error") - assert.Equal(t, expectedResult, ref.SingleRef(), "should create a singleRef (api construct)") -} diff --git a/entities/schema/data_types.go b/entities/schema/data_types.go deleted file mode 100644 index 5c1b184eccad923ccc986b81b0da6710173158c2..0000000000000000000000000000000000000000 --- a/entities/schema/data_types.go +++ /dev/null @@ -1,315 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package schema - -import ( - "errors" - "fmt" - "strings" - "unicode" -) - -type DataType string - -const ( - // DataTypeCRef The data type is a cross-reference, it is starting with a capital letter - DataTypeCRef DataType = "cref" - // DataTypeText The data type is a value of type string - DataTypeText DataType = "text" - // DataTypeInt The data type is a value of type int - DataTypeInt DataType = "int" - // DataTypeNumber The data type is a value of type number/float - DataTypeNumber DataType = "number" - // DataTypeBoolean The data type is a value of type boolean - DataTypeBoolean DataType = "boolean" - // DataTypeDate The data type is a value of type date - DataTypeDate DataType = "date" - // DataTypeGeoCoordinates is used to represent geo coordinates, i.e. latitude - // and longitude pairs of locations on earth - DataTypeGeoCoordinates DataType = "geoCoordinates" - // DataTypePhoneNumber represents a parsed/to-be-parsed phone number - DataTypePhoneNumber DataType = "phoneNumber" - // DataTypeBlob represents a base64 encoded data - DataTypeBlob DataType = "blob" - // DataTypeTextArray The data type is a value of type string array - DataTypeTextArray DataType = "text[]" - // DataTypeIntArray The data type is a value of type int array - DataTypeIntArray DataType = "int[]" - // DataTypeNumberArray The data type is a value of type number/float array - DataTypeNumberArray DataType = "number[]" - // DataTypeBooleanArray The data type is a value of type boolean array - DataTypeBooleanArray DataType = "boolean[]" - // DataTypeDateArray The data type is a value of type date array - DataTypeDateArray DataType = "date[]" - // DataTypeUUID is a native UUID data type. It is stored in it's raw byte - // representation and therefore takes up less space than storing a UUID as a - // string - DataTypeUUID DataType = "uuid" - // DataTypeUUIDArray is the array version of DataTypeUUID - DataTypeUUIDArray DataType = "uuid[]" - - DataTypeObject DataType = "object" - DataTypeObjectArray DataType = "object[]" - - // deprecated as of v1.19, replaced by DataTypeText + relevant tokenization setting - // DataTypeString The data type is a value of type string - DataTypeString DataType = "string" - // deprecated as of v1.19, replaced by DataTypeTextArray + relevant tokenization setting - // DataTypeArrayString The data type is a value of type string array - DataTypeStringArray DataType = "string[]" -) - -func (dt DataType) String() string { - return string(dt) -} - -func (dt DataType) PropString() []string { - return []string{dt.String()} -} - -func (dt DataType) AsName() string { - return strings.ReplaceAll(dt.String(), "[]", "Array") -} - -var PrimitiveDataTypes []DataType = []DataType{ - DataTypeText, DataTypeInt, DataTypeNumber, DataTypeBoolean, DataTypeDate, - DataTypeGeoCoordinates, DataTypePhoneNumber, DataTypeBlob, DataTypeTextArray, - DataTypeIntArray, DataTypeNumberArray, DataTypeBooleanArray, DataTypeDateArray, - DataTypeUUID, DataTypeUUIDArray, -} - -var NestedDataTypes []DataType = []DataType{ - DataTypeObject, DataTypeObjectArray, -} - -var DeprecatedPrimitiveDataTypes []DataType = []DataType{ - // deprecated as of v1.19 - DataTypeString, DataTypeStringArray, -} - -type PropertyKind int - -const ( - PropertyKindPrimitive PropertyKind = 1 - PropertyKindRef PropertyKind = 2 - PropertyKindNested PropertyKind = 3 -) - -type PropertyDataType interface { - Kind() PropertyKind - IsPrimitive() bool - AsPrimitive() DataType - IsReference() bool - Classes() []ClassName - ContainsClass(name ClassName) bool - IsNested() bool - AsNested() DataType -} - -type propertyDataType struct { - kind PropertyKind - primitiveType DataType - classes []ClassName - nestedType DataType -} - -// IsPropertyLength returns if a string is a filters for property length. They have the form len(*PROPNAME*) -func IsPropertyLength(propName string, offset int) (string, bool) { - isPropLengthFilter := len(propName) > 4+offset && propName[offset:offset+4] == "len(" && propName[len(propName)-1:] == ")" - - if isPropLengthFilter { - return propName[offset+4 : len(propName)-1], isPropLengthFilter - } - return "", false -} - -func IsArrayType(dt DataType) (DataType, bool) { - switch dt { - case DataTypeStringArray: - return DataTypeString, true - case DataTypeTextArray: - return DataTypeText, true - case DataTypeNumberArray: - return DataTypeNumber, true - case DataTypeIntArray: - return DataTypeInt, true - case DataTypeBooleanArray: - return DataTypeBoolean, true - case DataTypeDateArray: - return DataTypeDate, true - case DataTypeUUIDArray: - return DataTypeUUID, true - case DataTypeObjectArray: - return DataTypeObject, true - default: - return "", false - } -} - -func (p *propertyDataType) Kind() PropertyKind { - return p.kind -} - -func (p *propertyDataType) IsPrimitive() bool { - return p.kind == PropertyKindPrimitive -} - -func (p *propertyDataType) AsPrimitive() DataType { - if !p.IsPrimitive() { - panic("not primitive type") - } - - return p.primitiveType -} - -func (p *propertyDataType) IsReference() bool { - return p.kind == PropertyKindRef -} - -func (p *propertyDataType) Classes() []ClassName { - if !p.IsReference() { - panic("not MultipleRef type") - } - - return p.classes -} - -func (p *propertyDataType) ContainsClass(needle ClassName) bool { - if !p.IsReference() { - panic("not MultipleRef type") - } - - for _, class := range p.classes { - if class == needle { - return true - } - } - - return false -} - -func (p *propertyDataType) IsNested() bool { - return p.kind == PropertyKindNested -} - -func (p *propertyDataType) AsNested() DataType { - if !p.IsNested() { - panic("not nested type") - } - return p.nestedType -} - -// Based on the schema, return a valid description of the defined datatype -// -// Note that this function will error if referenced classes do not exist. If -// you don't want such validation, use [Schema.FindPropertyDataTypeRelaxedRefs] -// instead and set relax to true -func (s *Schema) FindPropertyDataType(dataType []string) (PropertyDataType, error) { - return s.FindPropertyDataTypeWithRefs(dataType, false, "") -} - -// Based on the schema, return a valid description of the defined datatype -// If relaxCrossRefValidation is set, there is no check if the referenced class -// exists in the schema. This can be helpful in scenarios, such as restoring -// from a backup where we have no guarantee over the order of class creation. -// If belongingToClass is set and equal to referenced class, check whether class -// exists in the schema is skipped. This is done to allow creating class schema with -// properties referencing to itself. Previously such properties had to be created separately -// only after creation of class schema -func (s *Schema) FindPropertyDataTypeWithRefs( - dataType []string, relaxCrossRefValidation bool, beloningToClass ClassName, -) (PropertyDataType, error) { - if len(dataType) < 1 { - return nil, errors.New("dataType must have at least one element") - } - if len(dataType) == 1 { - for _, dt := range append(PrimitiveDataTypes, DeprecatedPrimitiveDataTypes...) { - if dataType[0] == dt.String() { - return &propertyDataType{ - kind: PropertyKindPrimitive, - primitiveType: dt, - }, nil - } - } - for _, dt := range NestedDataTypes { - if dataType[0] == dt.String() { - return &propertyDataType{ - kind: PropertyKindNested, - nestedType: dt, - }, nil - } - } - if len(dataType[0]) == 0 { - return nil, fmt.Errorf("dataType cannot be an empty string") - } - firstLetter := rune(dataType[0][0]) - if unicode.IsLower(firstLetter) { - return nil, fmt.Errorf("Unknown primitive data type '%s'", dataType[0]) - } - } - /* implies len(dataType) > 1, or first element is a class already */ - var classes []ClassName - - for _, someDataType := range dataType { - className, err := ValidateClassName(someDataType) - if err != nil { - return nil, err - } - - if beloningToClass != className && !relaxCrossRefValidation { - if s.FindClassByName(className) == nil { - return nil, ErrRefToNonexistentClass - } - } - - classes = append(classes, className) - } - - return &propertyDataType{ - kind: PropertyKindRef, - classes: classes, - }, nil -} - -func AsPrimitive(dataType []string) (DataType, bool) { - if len(dataType) == 1 { - for _, dt := range append(PrimitiveDataTypes, DeprecatedPrimitiveDataTypes...) { - if dataType[0] == dt.String() { - return dt, true - } - } - if len(dataType[0]) == 0 { - return "", true - } - } - return "", false -} - -func AsNested(dataType []string) (DataType, bool) { - if len(dataType) == 1 { - for _, dt := range NestedDataTypes { - if dataType[0] == dt.String() { - return dt, true - } - } - } - return "", false -} - -func IsNested(dataType DataType) bool { - for _, dt := range NestedDataTypes { - if dt == dataType { - return true - } - } - return false -} diff --git a/entities/schema/data_types_test.go b/entities/schema/data_types_test.go deleted file mode 100644 index b1b338fc087775e2b69b118d823ddda1f093d0a5..0000000000000000000000000000000000000000 --- a/entities/schema/data_types_test.go +++ /dev/null @@ -1,417 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package schema - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" -) - -func TestDetectPrimitiveTypes(t *testing.T) { - s := Empty() - - for _, dt := range append(PrimitiveDataTypes, DeprecatedPrimitiveDataTypes...) { - pdt, err := s.FindPropertyDataType(dt.PropString()) - - assert.Nil(t, err) - assert.True(t, pdt.IsPrimitive()) - assert.Equal(t, dt, pdt.AsPrimitive()) - - assert.False(t, pdt.IsNested()) - assert.False(t, pdt.IsReference()) - } -} - -func TestDetectNestedTypes(t *testing.T) { - s := Empty() - - for _, dt := range NestedDataTypes { - ndt, err := s.FindPropertyDataType(dt.PropString()) - - assert.Nil(t, err) - assert.True(t, ndt.IsNested()) - assert.Equal(t, dt, ndt.AsNested()) - - assert.False(t, ndt.IsPrimitive()) - assert.False(t, ndt.IsReference()) - } -} - -func TestExistingClassSingleRef(t *testing.T) { - className := "ExistingClass" - s := Empty() - s.Objects.Classes = []*models.Class{{Class: className}} - - pdt, err := s.FindPropertyDataType([]string{className}) - - assert.Nil(t, err) - assert.True(t, pdt.IsReference()) - assert.True(t, pdt.ContainsClass(ClassName(className))) -} - -func TestNonExistingClassSingleRef(t *testing.T) { - className := "NonExistingClass" - s := Empty() - - pdt, err := s.FindPropertyDataType([]string{className}) - - assert.EqualError(t, err, ErrRefToNonexistentClass.Error()) - assert.Nil(t, pdt) -} - -func TestNonExistingClassRelaxedCrossValidation(t *testing.T) { - className := "NonExistingClass" - s := Empty() - - pdt, err := s.FindPropertyDataTypeWithRefs([]string{className}, true, ClassName("AnotherNonExistingClass")) - - assert.Nil(t, err) - assert.True(t, pdt.IsReference()) - assert.True(t, pdt.ContainsClass(ClassName(className))) -} - -func TestNonExistingClassPropertyBelongsTo(t *testing.T) { - className := "NonExistingClass" - s := Empty() - - pdt, err := s.FindPropertyDataTypeWithRefs([]string{className}, false, ClassName(className)) - - assert.Nil(t, err) - assert.True(t, pdt.IsReference()) - assert.True(t, pdt.ContainsClass(ClassName(className))) -} - -func TestGetPropertyDataType(t *testing.T) { - class := &models.Class{Class: "TestClass"} - dataTypes := []string{ - "string", "text", "int", "number", "boolean", - "date", "geoCoordinates", "phoneNumber", "blob", "Ref", "invalid", - "string[]", "text[]", "int[]", "number[]", "boolean[]", "date[]", - "uuid", "uuid[]", - - "object", "object[]", - } - class.Properties = make([]*models.Property, len(dataTypes)) - for i, dtString := range dataTypes { - class.Properties[i] = &models.Property{ - Name: dtString + "Prop", - DataType: []string{dtString}, - } - } - - type test struct { - propName string - expectedDataType *DataType - expectedErr error - } - - tests := []test{ - { - propName: "stringProp", - expectedDataType: ptDataType(DataTypeString), - }, - { - propName: "textProp", - expectedDataType: ptDataType(DataTypeText), - }, - { - propName: "numberProp", - expectedDataType: ptDataType(DataTypeNumber), - }, - { - propName: "intProp", - expectedDataType: ptDataType(DataTypeInt), - }, - { - propName: "booleanProp", - expectedDataType: ptDataType(DataTypeBoolean), - }, - { - propName: "dateProp", - expectedDataType: ptDataType(DataTypeDate), - }, - { - propName: "phoneNumberProp", - expectedDataType: ptDataType(DataTypePhoneNumber), - }, - { - propName: "geoCoordinatesProp", - expectedDataType: ptDataType(DataTypeGeoCoordinates), - }, - { - propName: "blobProp", - expectedDataType: ptDataType(DataTypeBlob), - }, - { - propName: "string[]Prop", - expectedDataType: ptDataType(DataTypeStringArray), - }, - { - propName: "text[]Prop", - expectedDataType: ptDataType(DataTypeTextArray), - }, - { - propName: "int[]Prop", - expectedDataType: ptDataType(DataTypeIntArray), - }, - { - propName: "number[]Prop", - expectedDataType: ptDataType(DataTypeNumberArray), - }, - { - propName: "boolean[]Prop", - expectedDataType: ptDataType(DataTypeBooleanArray), - }, - { - propName: "date[]Prop", - expectedDataType: ptDataType(DataTypeDateArray), - }, - { - propName: "uuidProp", - expectedDataType: ptDataType(DataTypeUUID), - }, - { - propName: "uuid[]Prop", - expectedDataType: ptDataType(DataTypeUUIDArray), - }, - { - propName: "objectProp", - expectedDataType: ptDataType(DataTypeObject), - }, - { - propName: "object[]Prop", - expectedDataType: ptDataType(DataTypeObjectArray), - }, - { - propName: "RefProp", - expectedDataType: ptDataType(DataTypeCRef), - }, - { - propName: "wrongProp", - expectedDataType: nil, - expectedErr: fmt.Errorf("no such prop with name 'wrongProp' found in class 'TestClass' in the schema. Check your schema files for which properties in this class are available"), - }, - { - propName: "invalidProp", - expectedDataType: nil, - expectedErr: fmt.Errorf("given value-DataType does not exist."), - }, - } - - for _, test := range tests { - t.Run(test.propName, func(t *testing.T) { - dt, err := GetPropertyDataType(class, test.propName) - require.Equal(t, test.expectedErr, err) - assert.Equal(t, test.expectedDataType, dt) - }) - } -} - -func Test_DataType_AsPrimitive(t *testing.T) { - type testCase struct { - name string - inputDataType []string - expectedDataType DataType - expectedIsPrimitive bool - } - - runTestCases := func(t *testing.T, testCases []testCase) { - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - dataType, ok := AsPrimitive(tc.inputDataType) - assert.Equal(t, tc.expectedDataType, dataType) - assert.Equal(t, tc.expectedIsPrimitive, ok) - }) - } - } - - t.Run("is primitive data type", func(t *testing.T) { - testCases := []testCase{} - for _, dt := range append(PrimitiveDataTypes, DeprecatedPrimitiveDataTypes...) { - inputDataType := dt.PropString() - testCases = append(testCases, testCase{ - name: fmt.Sprintf("%v", inputDataType), - inputDataType: inputDataType, - expectedDataType: dt, - expectedIsPrimitive: true, - }) - } - - runTestCases(t, testCases) - }) - - t.Run("is empty data type", func(t *testing.T) { - testCases := []testCase{} - for _, dtStr := range []string{""} { - inputDataType := []string{dtStr} - testCases = append(testCases, testCase{ - name: fmt.Sprintf("%v", inputDataType), - inputDataType: inputDataType, - expectedDataType: "", - expectedIsPrimitive: true, - }) - } - - runTestCases(t, testCases) - }) - - t.Run("is non existent data type", func(t *testing.T) { - testCases := []testCase{} - for _, dtStr := range []string{"non-existent"} { - inputDataType := []string{dtStr} - testCases = append(testCases, testCase{ - name: fmt.Sprintf("%v", inputDataType), - inputDataType: inputDataType, - expectedDataType: "", - expectedIsPrimitive: false, - }) - } - - runTestCases(t, testCases) - }) - - t.Run("is nested data type", func(t *testing.T) { - testCases := []testCase{} - for _, dt := range NestedDataTypes { - inputDataType := dt.PropString() - testCases = append(testCases, testCase{ - name: fmt.Sprintf("%v", inputDataType), - inputDataType: inputDataType, - expectedDataType: "", - expectedIsPrimitive: false, - }) - } - - runTestCases(t, testCases) - }) - - t.Run("is reference data type", func(t *testing.T) { - testCases := []testCase{} - for _, inputDataType := range [][]string{ - {"SomeClass"}, - {"SomeOtherClass", "AndAnotherOne"}, - } { - testCases = append(testCases, testCase{ - name: fmt.Sprintf("%v", inputDataType), - inputDataType: inputDataType, - expectedDataType: "", - expectedIsPrimitive: false, - }) - } - - runTestCases(t, testCases) - }) -} - -func Test_DataType_AsNested(t *testing.T) { - type testCase struct { - name string - inputDataType []string - expectedDataType DataType - expectedIsNested bool - } - - runTestCases := func(t *testing.T, testCases []testCase) { - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - dataType, ok := AsNested(tc.inputDataType) - assert.Equal(t, tc.expectedDataType, dataType) - assert.Equal(t, tc.expectedIsNested, ok) - }) - } - } - - t.Run("is nested data type", func(t *testing.T) { - testCases := []testCase{} - for _, dt := range NestedDataTypes { - inputDataType := dt.PropString() - testCases = append(testCases, testCase{ - name: fmt.Sprintf("%v", inputDataType), - inputDataType: inputDataType, - expectedDataType: dt, - expectedIsNested: true, - }) - } - - runTestCases(t, testCases) - }) - - t.Run("is empty data type", func(t *testing.T) { - testCases := []testCase{} - for _, dtStr := range []string{""} { - inputDataType := []string{dtStr} - testCases = append(testCases, testCase{ - name: fmt.Sprintf("%v", inputDataType), - inputDataType: inputDataType, - expectedDataType: "", - expectedIsNested: false, - }) - } - - runTestCases(t, testCases) - }) - - t.Run("is non existent data type", func(t *testing.T) { - testCases := []testCase{} - for _, dtStr := range []string{"non-existent"} { - inputDataType := []string{dtStr} - testCases = append(testCases, testCase{ - name: fmt.Sprintf("%v", inputDataType), - inputDataType: inputDataType, - expectedDataType: "", - expectedIsNested: false, - }) - } - - runTestCases(t, testCases) - }) - - t.Run("is primitive data type", func(t *testing.T) { - testCases := []testCase{} - for _, dt := range append(PrimitiveDataTypes, DeprecatedPrimitiveDataTypes...) { - inputDataType := dt.PropString() - testCases = append(testCases, testCase{ - name: fmt.Sprintf("%v", inputDataType), - inputDataType: inputDataType, - expectedDataType: "", - expectedIsNested: false, - }) - } - - runTestCases(t, testCases) - }) - - t.Run("is reference data type", func(t *testing.T) { - testCases := []testCase{} - for _, inputDataType := range [][]string{ - {"SomeClass"}, - {"SomeOtherClass", "AndAnotherOne"}, - } { - testCases = append(testCases, testCase{ - name: fmt.Sprintf("%v", inputDataType), - inputDataType: inputDataType, - expectedDataType: "", - expectedIsNested: false, - }) - } - - runTestCases(t, testCases) - }) -} - -func ptDataType(dt DataType) *DataType { - return &dt -} diff --git a/entities/schema/errors.go b/entities/schema/errors.go deleted file mode 100644 index a5db9c1049f7afd97b451b29e87d093848b96c6d..0000000000000000000000000000000000000000 --- a/entities/schema/errors.go +++ /dev/null @@ -1,22 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package schema - -import "errors" - -const ( - ErrorNoSuchClass string = "no such class with name '%s' found in the schema. Check your schema files for which classes are available" - ErrorNoSuchProperty string = "no such prop with name '%s' found in class '%s' in the schema. Check your schema files for which properties in this class are available" - ErrorNoSuchDatatype string = "given value-DataType does not exist." -) - -var ErrRefToNonexistentClass = errors.New("reference property to nonexistent class") diff --git a/entities/schema/inverted_index_config.go b/entities/schema/inverted_index_config.go deleted file mode 100644 index 29b9a91f447ced285f6725668cd8432b5d92c438..0000000000000000000000000000000000000000 --- a/entities/schema/inverted_index_config.go +++ /dev/null @@ -1,27 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package schema - -import "github.com/weaviate/weaviate/entities/models" - -type InvertedIndexConfig struct { - BM25 BM25Config - Stopwords models.StopwordConfig - IndexTimestamps bool - IndexNullState bool - IndexPropertyLength bool -} - -type BM25Config struct { - K1 float64 - B float64 -} diff --git a/entities/schema/multi_tenancy.go b/entities/schema/multi_tenancy.go deleted file mode 100644 index dbe3617c885587ae28a8a7a295096fb8b4c17fb2..0000000000000000000000000000000000000000 --- a/entities/schema/multi_tenancy.go +++ /dev/null @@ -1,28 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package schema - -import "github.com/weaviate/weaviate/entities/models" - -func MultiTenancyEnabled(class *models.Class) bool { - if class.MultiTenancyConfig != nil { - return class.MultiTenancyConfig.Enabled - } - return false -} - -func ActivityStatus(status string) string { - if status == "" { - return models.TenantActivityStatusHOT - } - return status -} diff --git a/entities/schema/nested_properties.go b/entities/schema/nested_properties.go deleted file mode 100644 index c73111e4526ab04265115253bd31bcfd843b7fcf..0000000000000000000000000000000000000000 --- a/entities/schema/nested_properties.go +++ /dev/null @@ -1,49 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package schema - -import "github.com/weaviate/weaviate/entities/models" - -// merges nPropsExt with nPropsBase -// returns new slice without changing input ones -// and bool indicating whether there was changes done comparing to base slice -func MergeRecursivelyNestedProperties(nPropsBase, nPropsExt []*models.NestedProperty, -) ([]*models.NestedProperty, bool) { - merged := false - nProps := make([]*models.NestedProperty, len(nPropsBase), len(nPropsBase)+len(nPropsExt)) - copy(nProps, nPropsBase) - - existingIndexMap := map[string]int{} - for index := range nProps { - existingIndexMap[nProps[index].Name] = index - } - - for _, nProp := range nPropsExt { - index, exists := existingIndexMap[nProp.Name] - if !exists { - existingIndexMap[nProp.Name] = len(nProps) - nProps = append(nProps, nProp) - merged = true - } else if _, isNested := AsNested(nProps[index].DataType); isNested { - if mergedProps, mergedNested := MergeRecursivelyNestedProperties(nProps[index].NestedProperties, - nProp.NestedProperties, - ); mergedNested { - nPropCopy := *nProps[index] - nProps[index] = &nPropCopy - nProps[index].NestedProperties = mergedProps - merged = true - } - } - } - - return nProps, merged -} diff --git a/entities/schema/nested_properties_test.go b/entities/schema/nested_properties_test.go deleted file mode 100644 index 762b595d36404cb17eb2b356dcfefaa89b594ab4..0000000000000000000000000000000000000000 --- a/entities/schema/nested_properties_test.go +++ /dev/null @@ -1,253 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package schema_test - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/schema/test_utils" -) - -func Test_MergeRecursivelyNestedProperties(t *testing.T) { - vFalse := false - vTrue := true - - emptyProps := []*models.NestedProperty{} - nestedProps1 := []*models.NestedProperty{ - { - Name: "nested_int", - DataType: schema.DataTypeInt.PropString(), - IndexFilterable: &vTrue, - IndexSearchable: &vFalse, - Tokenization: "", - }, - { - Name: "nested_text", - DataType: schema.DataTypeText.PropString(), - IndexFilterable: &vTrue, - IndexSearchable: &vTrue, - Tokenization: models.PropertyTokenizationWord, - }, - { - Name: "nested_objects", - DataType: schema.DataTypeObjectArray.PropString(), - IndexFilterable: &vTrue, - IndexSearchable: &vFalse, - Tokenization: "", - NestedProperties: []*models.NestedProperty{ - { - Name: "nested_bool_lvl2", - DataType: schema.DataTypeBoolean.PropString(), - IndexFilterable: &vTrue, - IndexSearchable: &vFalse, - Tokenization: "", - }, - { - Name: "nested_numbers_lvl2", - DataType: schema.DataTypeNumberArray.PropString(), - IndexFilterable: &vTrue, - IndexSearchable: &vFalse, - Tokenization: "", - }, - }, - }, - } - nestedProps2 := []*models.NestedProperty{ - { - Name: "nested_number", - DataType: schema.DataTypeNumber.PropString(), - IndexFilterable: &vTrue, - IndexSearchable: &vFalse, - Tokenization: "", - }, - { - Name: "nested_text", - DataType: schema.DataTypeText.PropString(), - IndexFilterable: &vTrue, - IndexSearchable: &vTrue, - Tokenization: models.PropertyTokenizationField, // different setting than (1) - }, - { - Name: "nested_objects", - DataType: schema.DataTypeObjectArray.PropString(), - IndexFilterable: &vTrue, - IndexSearchable: &vFalse, - Tokenization: "", - NestedProperties: []*models.NestedProperty{ - { - Name: "nested_date_lvl2", - DataType: schema.DataTypeDate.PropString(), - IndexFilterable: &vTrue, - IndexSearchable: &vFalse, - Tokenization: "", - }, - { - Name: "nested_numbers_lvl2", - DataType: schema.DataTypeNumberArray.PropString(), - IndexFilterable: &vFalse, // different setting than (1) - IndexSearchable: &vFalse, - Tokenization: "", - }, - }, - }, - } - - mergedProps_1_2 := []*models.NestedProperty{ - { - Name: "nested_int", - DataType: schema.DataTypeInt.PropString(), - IndexFilterable: &vTrue, - IndexSearchable: &vFalse, - Tokenization: "", - }, - { - Name: "nested_number", - DataType: schema.DataTypeNumber.PropString(), - IndexFilterable: &vTrue, - IndexSearchable: &vFalse, - Tokenization: "", - }, - { - Name: "nested_text", - DataType: schema.DataTypeText.PropString(), - IndexFilterable: &vTrue, - IndexSearchable: &vTrue, - Tokenization: models.PropertyTokenizationWord, // from (1) - }, - { - Name: "nested_objects", - DataType: schema.DataTypeObjectArray.PropString(), - IndexFilterable: &vTrue, - IndexSearchable: &vFalse, - Tokenization: "", - NestedProperties: []*models.NestedProperty{ - { - Name: "nested_bool_lvl2", - DataType: schema.DataTypeBoolean.PropString(), - IndexFilterable: &vTrue, - IndexSearchable: &vFalse, - Tokenization: "", - }, - { - Name: "nested_date_lvl2", - DataType: schema.DataTypeDate.PropString(), - IndexFilterable: &vTrue, - IndexSearchable: &vFalse, - Tokenization: "", - }, - { - Name: "nested_numbers_lvl2", - DataType: schema.DataTypeNumberArray.PropString(), - IndexFilterable: &vTrue, // from (1) - IndexSearchable: &vFalse, - Tokenization: "", - }, - }, - }, - } - - mergedProps_2_1 := []*models.NestedProperty{ - { - Name: "nested_int", - DataType: schema.DataTypeInt.PropString(), - IndexFilterable: &vTrue, - IndexSearchable: &vFalse, - Tokenization: "", - }, - { - Name: "nested_number", - DataType: schema.DataTypeNumber.PropString(), - IndexFilterable: &vTrue, - IndexSearchable: &vFalse, - Tokenization: "", - }, - { - Name: "nested_text", - DataType: schema.DataTypeText.PropString(), - IndexFilterable: &vTrue, - IndexSearchable: &vTrue, - Tokenization: models.PropertyTokenizationField, // from (2) - }, - { - Name: "nested_objects", - DataType: schema.DataTypeObjectArray.PropString(), - IndexFilterable: &vTrue, - IndexSearchable: &vFalse, - Tokenization: "", - NestedProperties: []*models.NestedProperty{ - { - Name: "nested_bool_lvl2", - DataType: schema.DataTypeBoolean.PropString(), - IndexFilterable: &vTrue, - IndexSearchable: &vFalse, - Tokenization: "", - }, - { - Name: "nested_date_lvl2", - DataType: schema.DataTypeDate.PropString(), - IndexFilterable: &vTrue, - IndexSearchable: &vFalse, - Tokenization: "", - }, - { - Name: "nested_numbers_lvl2", - DataType: schema.DataTypeNumberArray.PropString(), - IndexFilterable: &vFalse, // from (2) - IndexSearchable: &vFalse, - Tokenization: "", - }, - }, - }, - } - - t.Run("empty + nested", func(t *testing.T) { - nestedProps, merged := schema.MergeRecursivelyNestedProperties(emptyProps, nestedProps1) - - assert.True(t, merged) - assert.Equal(t, nestedProps1, nestedProps) - }) - - t.Run("nested + empty", func(t *testing.T) { - nestedProps, merged := schema.MergeRecursivelyNestedProperties(nestedProps1, emptyProps) - - assert.False(t, merged) - assert.Equal(t, nestedProps1, nestedProps) - }) - - t.Run("2 x nested", func(t *testing.T) { - nestedProps, merged := schema.MergeRecursivelyNestedProperties(nestedProps1, nestedProps1) - - assert.False(t, merged) - assert.Equal(t, nestedProps1, nestedProps) - }) - - t.Run("nested1 + nested2", func(t *testing.T) { - nestedProps, merged := schema.MergeRecursivelyNestedProperties(nestedProps1, nestedProps2) - - assert.True(t, merged) - assert.NotEqual(t, nestedProps1, nestedProps) - assert.NotEqual(t, nestedProps2, nestedProps) - test_utils.AssertNestedPropsMatch(t, mergedProps_1_2, nestedProps) - }) - - t.Run("nested2 + nested1", func(t *testing.T) { - nestedProps, merged := schema.MergeRecursivelyNestedProperties(nestedProps2, nestedProps1) - - assert.True(t, merged) - assert.NotEqual(t, nestedProps1, nestedProps) - assert.NotEqual(t, nestedProps2, nestedProps) - test_utils.AssertNestedPropsMatch(t, mergedProps_2_1, nestedProps) - }) -} diff --git a/entities/schema/schema.go b/entities/schema/schema.go deleted file mode 100644 index 1b2dab5ab0aa0beb4d71c9611fa0763797abe3e3..0000000000000000000000000000000000000000 --- a/entities/schema/schema.go +++ /dev/null @@ -1,99 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package schema - -import ( - "strings" - - "github.com/weaviate/weaviate/entities/models" -) - -// Newtype to denote that this string is used as a Class name -type ClassName string - -func (c ClassName) String() string { - return string(c) -} - -// Newtype to denote that this string is used as a Property name -type PropertyName string - -func (p PropertyName) String() string { - return string(p) -} - -type ClassAndProperty struct { - ClassName ClassName - PropertyName PropertyName -} - -// Describes the schema that is used in Weaviate. -type Schema struct { - Objects *models.Schema -} - -func Empty() Schema { - return Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{}, - }, - } -} - -// Return one of the semantic schema's -func (s *Schema) SemanticSchemaFor() *models.Schema { - return s.Objects -} - -func UppercaseClassName(name string) string { - if len(name) < 1 { - return name - } - - if len(name) == 1 { - return strings.ToUpper(name) - } - - return strings.ToUpper(string(name[0])) + name[1:] -} - -func LowercaseAllPropertyNames(props []*models.Property) []*models.Property { - for i, prop := range props { - props[i].Name = LowercaseFirstLetter(prop.Name) - } - - return props -} - -func LowercaseFirstLetter(name string) string { - if len(name) < 1 { - return name - } - - if len(name) == 1 { - return strings.ToLower(name) - } - - return strings.ToLower(string(name[0])) + name[1:] -} - -func LowercaseFirstLetterOfStrings(in []string) []string { - if len(in) < 1 { - return in - } - out := make([]string, len(in)) - for i, str := range in { - out[i] = LowercaseFirstLetter(str) - } - - return out -} diff --git a/entities/schema/test_utils/nested_properties.go b/entities/schema/test_utils/nested_properties.go deleted file mode 100644 index 217179b0e1580e0ef640a03253d2b11bd2223603..0000000000000000000000000000000000000000 --- a/entities/schema/test_utils/nested_properties.go +++ /dev/null @@ -1,44 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test_utils - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" -) - -func AssertNestedPropsMatch(t *testing.T, nestedPropsA, nestedPropsB []*models.NestedProperty) { - require.Len(t, nestedPropsB, len(nestedPropsA), "nestedProps: different length") - - npMap := map[string]int{} - for index, np := range nestedPropsA { - npMap[np.Name] = index - } - - for _, npB := range nestedPropsB { - require.Contains(t, npMap, npB.Name) - npA := nestedPropsA[npMap[npB.Name]] - - assert.Equal(t, npA.DataType, npB.DataType) - assert.Equal(t, npA.IndexFilterable, npB.IndexFilterable) - assert.Equal(t, npA.IndexSearchable, npB.IndexSearchable) - assert.Equal(t, npA.Tokenization, npB.Tokenization) - - if _, isNested := schema.AsNested(npA.DataType); isNested { - AssertNestedPropsMatch(t, npA.NestedProperties, npB.NestedProperties) - } - } -} diff --git a/entities/schema/validation.go b/entities/schema/validation.go deleted file mode 100644 index 2c9d182caf17da5a90b701bc106eee5fd9f90668..0000000000000000000000000000000000000000 --- a/entities/schema/validation.go +++ /dev/null @@ -1,96 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package schema - -import ( - "fmt" - "regexp" -) - -var ( - validateClassNameRegex *regexp.Regexp - validatePropertyNameRegex *regexp.Regexp - validateNestedPropertyNameRegex *regexp.Regexp - reservedPropertyNames []string -) - -const ( - ClassNameRegexCore = `[A-Z][_0-9A-Za-z]*` - ShardNameRegexCore = `[A-Za-z0-9\-\_]{1,64}` - PropertyNameRegex = `[_A-Za-z][_0-9A-Za-z]*` - NestedPropertyNameRegex = `[_A-Za-z][_0-9A-Za-z]*` -) - -func init() { - validateClassNameRegex = regexp.MustCompile(`^` + ClassNameRegexCore + `$`) - validatePropertyNameRegex = regexp.MustCompile(`^` + PropertyNameRegex + `$`) - validateNestedPropertyNameRegex = regexp.MustCompile(`^` + NestedPropertyNameRegex + `$`) - reservedPropertyNames = []string{"_additional", "_id", "id"} -} - -// ValidateClassName validates that this string is a valid class name (format wise) -func ValidateClassName(name string) (ClassName, error) { - if validateClassNameRegex.MatchString(name) { - return ClassName(name), nil - } - return "", fmt.Errorf("'%s' is not a valid class name", name) -} - -// ValidatePropertyName validates that this string is a valid property name -func ValidatePropertyName(name string) (PropertyName, error) { - if !validatePropertyNameRegex.MatchString(name) { - return "", fmt.Errorf("'%s' is not a valid property name. "+ - "Property names in Weaviate are restricted to valid GraphQL names, "+ - "which must be “/%s/”.", name, PropertyNameRegex) - } - return PropertyName(name), nil -} - -// ValidateNestedPropertyName validates that this string is a valid nested property name -func ValidateNestedPropertyName(name, prefix string) error { - if !validateNestedPropertyNameRegex.MatchString(name) { - return fmt.Errorf("'%s' is not a valid nested property name of '%s'. "+ - "NestedProperty names in Weaviate are restricted to valid GraphQL names, "+ - "which must be “/%s/”.", name, prefix, NestedPropertyNameRegex) - } - return nil -} - -// ValidateReservedPropertyName validates that a string is not a reserved property name -func ValidateReservedPropertyName(name string) error { - for i := range reservedPropertyNames { - if name == reservedPropertyNames[i] { - return fmt.Errorf("'%s' is a reserved property name", name) - } - } - return nil -} - -// AssertValidClassName assert that this string is a valid class name or -// panics and should therefore most likely not be used -func AssertValidClassName(name string) ClassName { - n, err := ValidateClassName(name) - if err != nil { - panic(fmt.Sprintf("Did not expect to be handled '%s', an invalid class name", name)) - } - return n -} - -// AssertValidPropertyName asserts that this string is a valid property name or -// panics and should therefore most likely never be used. -func AssertValidPropertyName(name string) PropertyName { - n, err := ValidatePropertyName(name) - if err != nil { - panic(fmt.Sprintf("Did not expect to be handled '%s', an invalid property name", name)) - } - return n -} diff --git a/entities/schema/validation_test.go b/entities/schema/validation_test.go deleted file mode 100644 index 51d17a584afc84d0c7e04b08ef3a00ac186b4391..0000000000000000000000000000000000000000 --- a/entities/schema/validation_test.go +++ /dev/null @@ -1,190 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package schema - -import ( - "testing" -) - -func TestValidateOKClassName(t *testing.T) { - _, err := ValidateClassName("FooBar") - if err != nil { - t.Fail() - } - - _, err = ValidateClassName("FooBar2") - if err != nil { - t.Fail() - } - - _, err = ValidateClassName("Foo_______bar__with_numbers___1234567890_and_2") - if err != nil { - t.Fail() - } - - _, err = ValidateClassName("C_123456___foo_bar_2") - if err != nil { - t.Fail() - } - - _, err = ValidateClassName("NormalClassNameWithNumber1") - if err != nil { - t.Fail() - } - - _, err = ValidateClassName("Normal__Class__Name__With__Number__1") - if err != nil { - t.Fail() - } - - _, err = ValidateClassName("CClassName") - if err != nil { - t.Fail() - } -} - -func TestFailValidateBadClassName(t *testing.T) { - _, err := ValidateClassName("Foo Bar") - if err == nil { - t.Fail() - } - - _, err = ValidateClassName("foo") - if err == nil { - t.Fail() - } - - _, err = ValidateClassName("fooBar") - if err == nil { - t.Fail() - } - - _, err = ValidateClassName("_foo") - if err == nil { - t.Fail() - } -} - -func TestValidateOKPropertyName(t *testing.T) { - // valid proper names - _, err := ValidatePropertyName("fooBar") - if err != nil { - t.Fail() - } - - _, err = ValidatePropertyName("fooBar2") - if err != nil { - t.Fail() - } - - _, err = ValidatePropertyName("_fooBar2") - if err != nil { - t.Fail() - } - - _, err = ValidatePropertyName("intField") - if err != nil { - t.Fail() - } - - _, err = ValidatePropertyName("hasAction") - if err != nil { - t.Fail() - } - - _, err = ValidatePropertyName("_foo_bar_2") - if err != nil { - t.Fail() - } - - _, err = ValidatePropertyName("______foo_bar_2") - if err != nil { - t.Fail() - } - - _, err = ValidatePropertyName("___123456___foo_bar_2") - if err != nil { - t.Fail() - } - - _, err = ValidatePropertyName("a_very_Long_property_Name__22_with_numbers_9") - if err != nil { - t.Fail() - } - - _, err = ValidatePropertyName("a_very_Long_property_Name__22_with_numbers_9880888800888800008") - if err != nil { - t.Fail() - } - - _, err = ValidatePropertyName("FooBar") - if err != nil { - t.Fail() - } -} - -func TestFailValidateBadPropertyName(t *testing.T) { - _, err := ValidatePropertyName("foo Bar") - if err == nil { - t.Fail() - } - - _, err = ValidatePropertyName("a_very_Long_property_Name__22_with-dash_9") - if err == nil { - t.Fail() - } - - _, err = ValidatePropertyName("1_FooBar") - if err == nil { - t.Fail() - } -} - -func TestValidateReservedPropertyName(t *testing.T) { - type args struct { - name string - } - tests := []struct { - name string - args args - wantErr bool - }{ - { - name: "Reserved name: _additional", - args: args{ - name: "_additional", - }, - wantErr: true, - }, - { - name: "Reserved name: id", - args: args{ - name: "id", - }, - wantErr: true, - }, - { - name: "Reserved name: _id", - args: args{ - name: "_id", - }, - wantErr: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := ValidateReservedPropertyName(tt.args.name); (err != nil) != tt.wantErr { - t.Errorf("ValidateReservedPropertyName() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} diff --git a/entities/schema/vector_index_config.go b/entities/schema/vector_index_config.go deleted file mode 100644 index 9ff2e06665bd9fae2ee896fd303ed22c2c92c605..0000000000000000000000000000000000000000 --- a/entities/schema/vector_index_config.go +++ /dev/null @@ -1,32 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package schema - -import ( - "fmt" - - "github.com/weaviate/weaviate/entities/models" -) - -type VectorIndexConfig interface { - IndexType() string - DistanceName() string -} - -func TypeAssertVectorIndex(class *models.Class) (VectorIndexConfig, error) { - if config, ok := class.VectorIndexConfig.(VectorIndexConfig); ok { - return config, nil - } - - return nil, fmt.Errorf("class '%s' vector index: config is not schema.VectorIndexConfig: %T", - class.Class, class.VectorIndexConfig) -} diff --git a/entities/search/ref.go b/entities/search/ref.go deleted file mode 100644 index 4ed471120b2432a607084506df39c163eabf3581..0000000000000000000000000000000000000000 --- a/entities/search/ref.go +++ /dev/null @@ -1,20 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package search - -// LocalRef to be filled by the search backend to indicate that the -// particular reference field is a local ref and does not require further -// resolving, as opposed to a NetworkRef. -type LocalRef struct { - Class string - Fields map[string]interface{} -} diff --git a/entities/search/result.go b/entities/search/result.go deleted file mode 100644 index e9e7a356499eccf86e023b90fed5be51c3cc662f..0000000000000000000000000000000000000000 --- a/entities/search/result.go +++ /dev/null @@ -1,88 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package search - -import ( - "github.com/go-openapi/strfmt" - "github.com/weaviate/weaviate/entities/models" -) - -// Result contains some info of a concept (kind), but not all. For -// additional info the ID can be used to retrieve the full concept from the -// connector storage -type Result struct { - ID strfmt.UUID - ClassName string - Score float32 - SecondarySortValue float32 - ExplainScore string - Dist float32 - Vector []float32 - Beacon string - Certainty float32 - Schema models.PropertySchema - Created int64 - Updated int64 - AdditionalProperties models.AdditionalProperties - VectorWeights map[string]string - IsConsistent bool - Tenant string - - // Dimensions in case search was vector-based, 0 otherwise - Dims int -} - -type Results []Result - -func (r Result) Object() *models.Object { - return r.ObjectWithVector(true) -} - -func (r Result) ObjectWithVector(includeVector bool) *models.Object { - schema, ok := r.Schema.(map[string]interface{}) - if ok { - delete(schema, "id") - } - - t := &models.Object{ - Class: r.ClassName, - ID: r.ID, - Properties: schema, - CreationTimeUnix: r.Created, - LastUpdateTimeUnix: r.Updated, - VectorWeights: r.VectorWeights, - Tenant: r.Tenant, - } - - if r.AdditionalProperties != nil { - t.Additional = r.AdditionalProperties - } - - if includeVector { - t.Vector = r.Vector - } - - return t -} - -func (rs Results) Objects() []*models.Object { - return rs.ObjectsWithVector(true) -} - -func (rs Results) ObjectsWithVector(includeVector bool) []*models.Object { - objects := make([]*models.Object, len(rs)) - for i, res := range rs { - objects[i] = res.ObjectWithVector(includeVector) - } - - return objects -} diff --git a/entities/search/select_property.go b/entities/search/select_property.go deleted file mode 100644 index 9054d8142b24124cd312d4f727923defcd2b238d..0000000000000000000000000000000000000000 --- a/entities/search/select_property.go +++ /dev/null @@ -1,144 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package search - -import ( - "fmt" - "regexp" - - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/schema" -) - -type SelectProperty struct { - Name string `json:"name"` - - IsPrimitive bool `json:"isPrimitive"` - - IsObject bool `json:"isObject"` - - // Include the __typename in all the Refs below. - IncludeTypeName bool `json:"includeTypeName"` - - // Not a primitive nor nested type? Then select these properties. - Refs []SelectClass `json:"refs"` - - // Nested type? Then select these properties. - Props []SelectProperty `json:"objs"` -} - -type SelectClass struct { - ClassName string `json:"className"` - RefProperties SelectProperties `json:"refProperties"` - AdditionalProperties additional.Properties `json:"additionalProperties"` -} - -// FindSelectClass by specifying the exact class name -func (sp SelectProperty) FindSelectClass(className schema.ClassName) *SelectClass { - for _, selectClass := range sp.Refs { - if selectClass.ClassName == string(className) { - return &selectClass - } - } - - return nil -} - -// FindSelectObject by specifying the exact object name -func (sp SelectProperty) FindSelectProperty(name string) *SelectProperty { - for _, selectProp := range sp.Props { - if selectProp.Name == name { - return &selectProp - } - } - - return nil -} - -// HasPeer returns true if any of the referenced classes are from the specified -// peer -func (sp SelectProperty) HasPeer(peerName string) bool { - r := regexp.MustCompile(fmt.Sprintf("^%s__", peerName)) - for _, selectClass := range sp.Refs { - if r.MatchString(selectClass.ClassName) { - return true - } - } - - return false -} - -type SelectProperties []SelectProperty - -func (sp SelectProperties) HasRefs() bool { - for _, p := range sp { - if len(p.Refs) > 0 { - return true - } - } - return false -} - -func (sp SelectProperties) HasProps() bool { - for _, p := range sp { - if len(p.Props) > 0 { - return true - } - } - return false -} - -func (sp SelectProperties) ShouldResolve(path []string) (bool, error) { - if len(path)%2 != 0 || len(path) == 0 { - return false, fmt.Errorf("used incorrectly: path must have even number of segments in the form of " + - "refProp, className, refProp, className, etc.") - } - - // the above gives us the guarantee that path contains at least two elements - property := path[0] - class := schema.ClassName(path[1]) - - for _, p := range sp { - if p.IsPrimitive { - continue - } - - if p.Name != property { - continue - } - - selectClass := p.FindSelectClass(class) - if selectClass == nil { - continue - } - - if len(path) > 2 { - // we're not done yet, this one's nested - return selectClass.RefProperties.ShouldResolve(path[2:]) - } - - // we are done and found the path - return true, nil - } - - return false, nil -} - -func (sp SelectProperties) FindProperty(propName string) *SelectProperty { - for _, prop := range sp { - if prop.Name == propName { - return &prop - } - } - - return nil -} diff --git a/entities/searchparams/retrieval.go b/entities/searchparams/retrieval.go deleted file mode 100644 index 2072537a487015673bfec756c09083e900a31e0a..0000000000000000000000000000000000000000 --- a/entities/searchparams/retrieval.go +++ /dev/null @@ -1,80 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package searchparams - -type NearVector struct { - Vector []float32 `json:"vector"` - Certainty float64 `json:"certainty"` - Distance float64 `json:"distance"` - WithDistance bool `json:"-"` -} - -type KeywordRanking struct { - Type string `json:"type"` - Properties []string `json:"properties"` - Query string `json:"query"` - AdditionalExplanations bool `json:"additionalExplanations"` -} - -type WeightedSearchResult struct { - SearchParams interface{} `json:"searchParams"` - Weight float64 `json:"weight"` - Type string `json:"type"` -} - -type HybridSearch struct { - SubSearches interface{} `json:"subSearches"` - Type string `json:"type"` - Alpha float64 `json:"alpha"` - Query string `json:"query"` - Vector []float32 `json:"vector"` - Properties []string `json:"properties"` - FusionAlgorithm int `json:"fusionalgorithm"` -} - -type NearObject struct { - ID string `json:"id"` - Beacon string `json:"beacon"` - Certainty float64 `json:"certainty"` - Distance float64 `json:"distance"` - WithDistance bool `json:"-"` -} - -type ObjectMove struct { - ID string - Beacon string -} - -// ExploreMove moves an existing Search Vector closer (or further away from) a specific other search term -type ExploreMove struct { - Values []string - Force float32 - Objects []ObjectMove -} - -type NearTextParams struct { - Values []string - Limit int - MoveTo ExploreMove - MoveAwayFrom ExploreMove - Certainty float64 - Distance float64 - WithDistance bool - Network bool - Autocorrect bool -} - -type GroupBy struct { - Property string - Groups int - ObjectsPerGroup int -} diff --git a/entities/storagestate/status.go b/entities/storagestate/status.go deleted file mode 100644 index de1a0ff7d2a1ff81bddbc0ce300accc7e0e7bbfb..0000000000000000000000000000000000000000 --- a/entities/storagestate/status.go +++ /dev/null @@ -1,46 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package storagestate - -import "errors" - -const ( - StatusReadOnly Status = "READONLY" - StatusIndexing Status = "INDEXING" - StatusReady Status = "READY" -) - -var ( - ErrStatusReadOnly = errors.New("store is read-only") - ErrInvalidStatus = errors.New("invalid storage status") -) - -type Status string - -func (s Status) String() string { - return string(s) -} - -func ValidateStatus(in string) (status Status, err error) { - switch in { - case string(StatusReadOnly): - status = StatusReadOnly - case string(StatusIndexing): - status = StatusIndexing - case string(StatusReady): - status = StatusReady - default: - err = ErrInvalidStatus - } - - return -} diff --git a/entities/storagestate/status_test.go b/entities/storagestate/status_test.go deleted file mode 100644 index 852c35a6265eb22f33748748d8b49a430456c972..0000000000000000000000000000000000000000 --- a/entities/storagestate/status_test.go +++ /dev/null @@ -1,53 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package storagestate - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestStatusValidation(t *testing.T) { - t.Run("with invalid status", func(t *testing.T) { - tests := []string{ - "READ_ONLY", - "read only", - "ok", - "WRITEONLY", - "INDESKING", - "", - } - - for _, test := range tests { - _, err := ValidateStatus(test) - require.EqualError(t, ErrInvalidStatus, err.Error()) - } - }) - - t.Run("with valid status", func(t *testing.T) { - tests := []struct { - in string - expected Status - }{ - {"READONLY", StatusReadOnly}, - {"READY", StatusReady}, - {"INDEXING", StatusIndexing}, - } - - for _, test := range tests { - status, err := ValidateStatus(test.in) - require.Nil(t, err) - require.Equal(t, test.expected, status) - } - }) -} diff --git a/entities/storobj/buffer_pool.go b/entities/storobj/buffer_pool.go deleted file mode 100644 index f9151fefba44b2b409f1bd1463920134107cd71c..0000000000000000000000000000000000000000 --- a/entities/storobj/buffer_pool.go +++ /dev/null @@ -1,50 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package storobj - -import "sync" - -func newBufferPool(initialSize int) *bufferPool { - return &bufferPool{ - pool: sync.Pool{ - New: func() any { - // initialize with len=0 to make sure we get a consistent result - // whether it's a new or used buffer. Every buffer will always have - // len=0 and cap>=initialSize - return make([]byte, 0, initialSize) - }, - }, - } -} - -type bufferPool struct { - pool sync.Pool -} - -func (b *bufferPool) Get() []byte { - buf := b.pool.Get().([]byte) - - return buf -} - -func (b *bufferPool) Put(buf []byte) { - // make sure the length is reset before putting it back into the pool. This - // way all buffers will always have len=0, either because they are brand-new - // or because they have been reset. - buf = buf[:0] - - //nolint:staticcheck // I disagree with the linter, this doesn't need to be a - //pointer. Even if we copy the slicestruct header, the backing array is - //what's allocation-heavy and that will be reused. The profiles in the PR - //description prove this. - b.pool.Put(buf) -} diff --git a/entities/storobj/enrich_schema_datatypes.go b/entities/storobj/enrich_schema_datatypes.go deleted file mode 100644 index f3b939d6e7d2a04aced9b827a21f508a4cd510c5..0000000000000000000000000000000000000000 --- a/entities/storobj/enrich_schema_datatypes.go +++ /dev/null @@ -1,409 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package storobj - -import ( - "fmt" - - "github.com/go-openapi/strfmt" - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/models" -) - -func enrichSchemaTypes(schema map[string]interface{}, ofNestedProp bool) error { - if schema == nil { - return nil - } - - for propName, value := range schema { - switch typed := value.(type) { - case []interface{}: - if isArrayValue(typed) { - switch typed[0].(type) { - case float64: - parsed, err := parseNumberArrayValue(typed) - if err != nil { - return errors.Wrapf(err, "property %q of type string array", propName) - } - - schema[propName] = parsed - case bool: - parsed, err := parseBoolArrayValue(typed) - if err != nil { - return errors.Wrapf(err, "property %q of type boolean array", propName) - } - - schema[propName] = parsed - default: - parsed, err := parseStringArrayValue(typed) - if err != nil { - return errors.Wrapf(err, "property %q of type string array", propName) - } - - schema[propName] = parsed - } - } else if len(typed) == 0 { - // empty arrays. Here we use []interface{} as a placeholder - // type for an empty array, since we cannot determine its - // actual type. in the future, we should persist the schema - // property type information alongside the value to avoid - // this situation - schema[propName] = typed - } else { - // nested properties does not support refs - if !ofNestedProp { - parsed, err := parseCrossRef(typed) - if err == nil { - schema[propName] = parsed - continue - } - } - // apperently object[] - for i := range typed { - t2, ok := typed[i].(map[string]interface{}) - if !ok { - return fmt.Errorf("expected element [%d] of '%s' to be map, %T found", i, propName, typed[i]) - } - enrichSchemaTypes(t2, true) - } - schema[propName] = typed - } - case map[string]interface{}: - parsed, err := parseMapProp(typed, ofNestedProp) - if err != nil { - return errors.Wrapf(err, "property %q of type map", propName) - } - - schema[propName] = parsed - default: - continue - } - } - - return nil -} - -// nested properties does not support phone or geo data types -func parseMapProp(input map[string]interface{}, ofNestedProp bool) (interface{}, error) { - if !ofNestedProp && isGeoProp(input) { - return parseGeoProp(input) - } - if !ofNestedProp && isPhoneProp(input) { - return parsePhoneNumber(input) - } - // apparently object - err := enrichSchemaTypes(input, true) - return input, err -} - -func isGeoProp(input map[string]interface{}) bool { - expectedProps := []string{"latitude", "longitude"} - - if len(input) != len(expectedProps) { - return false - } - for _, prop := range expectedProps { - if _, ok := input[prop]; !ok { - return false - } - } - return true -} - -func isPhoneProp(input map[string]interface{}) bool { - validExpectedProps := [][]string{ - {"input", "internationalFormatted", "nationalFormatted", "national", "countryCode", "valid"}, - {"input", "internationalFormatted", "nationalFormatted", "national", "countryCode", "valid", "defaultCountry"}, - {"input", "internationalFormatted", "nationalFormatted", "national", "countryCode"}, - {"input", "internationalFormatted", "nationalFormatted", "national", "countryCode", "defaultCountry"}, - } - - for _, expectedProps := range validExpectedProps { - match := true - if len(expectedProps) != len(input) { - match = false - } else { - for _, prop := range expectedProps { - if _, ok := input[prop]; !ok { - match = false - break - } - } - } - if match { - return true - } - } - return false -} - -func parseGeoProp(input map[string]interface{}) (*models.GeoCoordinates, error) { - latFloat, ok := input["latitude"].(float64) - if !ok { - return nil, fmt.Errorf("explected lat to be float64, but is %T", input["latitude"]) - } - - lonFloat, ok := input["longitude"].(float64) - if !ok { - return nil, fmt.Errorf("explected lon to be float64, but is %T", input["longitude"]) - } - - return &models.GeoCoordinates{ - Latitude: ptFloat32(float32(latFloat)), - Longitude: ptFloat32(float32(lonFloat)), - }, nil -} - -func ptFloat32(in float32) *float32 { - return &in -} - -func parsePhoneNumber(input map[string]interface{}) (*models.PhoneNumber, error) { - out := &models.PhoneNumber{} - - phoneInput, err := extractStringFromMap(input, "input") - if err != nil { - return nil, err - } - out.Input = phoneInput - - international, err := extractStringFromMap(input, "internationalFormatted") - if err != nil { - return nil, err - } - out.InternationalFormatted = international - - nationalFormatted, err := extractStringFromMap(input, "nationalFormatted") - if err != nil { - return nil, err - } - out.NationalFormatted = nationalFormatted - - national, err := extractNumberFromMap(input, "national") - if err != nil { - return nil, err - } - out.National = uint64(national) - - countryCode, err := extractNumberFromMap(input, "countryCode") - if err != nil { - return nil, err - } - out.CountryCode = uint64(countryCode) - - defaultCountry, err := extractStringFromMap(input, "defaultCountry") - if err != nil { - return nil, err - } - out.DefaultCountry = defaultCountry - - valid, err := extractBoolFromMap(input, "valid") - if err != nil { - return nil, err - } - out.Valid = valid - - return out, nil -} - -func extractNumberFromMap(input map[string]interface{}, key string) (float64, error) { - field, ok := input[key] - if ok { - asFloat, ok := field.(float64) - if !ok { - return 0, fmt.Errorf("expected '%s' to be float64, but is %T", key, field) - } - - return asFloat, nil - } - return 0, nil -} - -func extractStringFromMap(input map[string]interface{}, key string) (string, error) { - field, ok := input[key] - if ok { - asString, ok := field.(string) - if !ok { - return "", fmt.Errorf("expected '%s' to be string, but is %T", key, field) - } - - return asString, nil - } - return "", nil -} - -func extractBoolFromMap(input map[string]interface{}, key string) (bool, error) { - field, ok := input[key] - if ok { - asBool, ok := field.(bool) - if !ok { - return false, fmt.Errorf("expected '%s' to be bool, but is %T", key, field) - } - - return asBool, nil - } - return false, nil -} - -func isArrayValue(value []interface{}) bool { - if len(value) > 0 { - _, ok := value[0].(map[string]interface{}) - return !ok - } - return false -} - -func parseStringArrayValue(value []interface{}) ([]string, error) { - parsed := make([]string, len(value)) - for i := range value { - asString, ok := value[i].(string) - if !ok { - return nil, fmt.Errorf("string array: expected element %d to be string - got %T", i, value[i]) - } - parsed[i] = asString - } - return parsed, nil -} - -func parseNumberArrayValue(value []interface{}) ([]float64, error) { - parsed := make([]float64, len(value)) - for i := range value { - asFloat, ok := value[i].(float64) - if !ok { - return nil, fmt.Errorf("number array: expected element %d to be float - got %T", i, value[i]) - } - parsed[i] = asFloat - } - return parsed, nil -} - -func parseBoolArrayValue(value []interface{}) ([]bool, error) { - parsed := make([]bool, len(value)) - for i := range value { - asBool, ok := value[i].(bool) - if !ok { - return nil, fmt.Errorf("boolean array: expected element %d to be bool - got %T", i, value[i]) - } - parsed[i] = asBool - } - return parsed, nil -} - -func parseCrossRef(value []interface{}) (models.MultipleRef, error) { - parsed := make(models.MultipleRef, len(value)) - for i, elem := range value { - asMap, ok := elem.(map[string]interface{}) - if !ok { - return nil, fmt.Errorf("crossref: expected element %d to be map - got %T", i, elem) - } - - beacon, ok := asMap["beacon"] - if !ok { - return nil, fmt.Errorf("crossref: expected element %d to have key %q - got %v", i, "beacon", elem) - } - - beaconStr, ok := beacon.(string) - if !ok { - return nil, fmt.Errorf("crossref: expected element %d.beacon to be string - got %T", i, beacon) - } - - parsed[i] = &models.SingleRef{ - Beacon: strfmt.URI(beaconStr), - } - - c, ok := asMap["classification"] - if ok { - classification, err := parseRefClassificationMeta(c) - if err != nil { - return nil, errors.Wrap(err, "crossref: parse classifiation meta") - } - - parsed[i].Classification = classification - } - } - - return parsed, nil -} - -func parseRefClassificationMeta(in interface{}) (*models.ReferenceMetaClassification, error) { - out := &models.ReferenceMetaClassification{} - asMap, ok := in.(map[string]interface{}) - if !ok { - return nil, fmt.Errorf("expected classification to be map - got %T", in) - } - - if cod, err := extractFloat64(asMap, "closestOverallDistance"); err != nil { - return nil, err - } else { - out.ClosestOverallDistance = cod - } - - if mwd, err := extractFloat64(asMap, "meanWinningDistance"); err != nil { - return nil, err - } else { - out.WinningDistance = mwd // deprecated remove in 0.23.0 - out.MeanWinningDistance = mwd - } - - if cwd, err := extractFloat64(asMap, "closestWinningDistance"); err != nil { - return nil, err - } else { - out.ClosestWinningDistance = cwd - } - - if mcd, err := extractFloat64(asMap, "meanLosingDistance"); err != nil { - return nil, err - } else { - out.LosingDistance = &mcd // deprecated remove in 0.23.0 - out.MeanLosingDistance = &mcd - } - - if ccd, err := extractFloat64(asMap, "closestLosingDistance"); err != nil { - return nil, err - } else { - out.ClosestLosingDistance = &ccd - } - - if oc, err := extractFloat64(asMap, "overallCount"); err != nil { - return nil, err - } else { - out.OverallCount = int64(oc) - } - - if wc, err := extractFloat64(asMap, "winningCount"); err != nil { - return nil, err - } else { - out.WinningCount = int64(wc) - } - - if lc, err := extractFloat64(asMap, "losingCount"); err != nil { - return nil, err - } else { - out.LosingCount = int64(lc) - } - - return out, nil -} - -func extractFloat64(source map[string]interface{}, key string) (float64, error) { - value, ok := source[key] - if !ok { - return 0, nil - } - - asFloat, ok := value.(float64) - if !ok { - return 0, fmt.Errorf("expected %s to be float64 - got %T", key, value) - } - - return asFloat, nil -} diff --git a/entities/storobj/errors.go b/entities/storobj/errors.go deleted file mode 100644 index 3f9b357581405a15db5931d60f3fe8104c37e27b..0000000000000000000000000000000000000000 --- a/entities/storobj/errors.go +++ /dev/null @@ -1,30 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package storobj - -import "fmt" - -type ErrNotFound struct { - DocID uint64 - OriginalMsg string -} - -func NewErrNotFoundf(docID uint64, msg string, args ...interface{}) error { - return ErrNotFound{ - DocID: docID, - OriginalMsg: fmt.Sprintf(msg, args...), - } -} - -func (err ErrNotFound) Error() string { - return fmt.Sprintf("no object found for doc id %d: %s", err.DocID, err.OriginalMsg) -} diff --git a/entities/storobj/parse_single_object.go b/entities/storobj/parse_single_object.go deleted file mode 100644 index 267155506fc61f765383fa213d1c1ba59687d8e7..0000000000000000000000000000000000000000 --- a/entities/storobj/parse_single_object.go +++ /dev/null @@ -1,173 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package storobj - -import ( - "bytes" - "encoding/binary" - "strconv" - - "github.com/buger/jsonparser" - "github.com/google/uuid" - "github.com/pkg/errors" -) - -func ParseAndExtractProperty(data []byte, propName string) ([]string, bool, error) { - if propName == "id" || propName == "_id" { - return extractID(data) - } - if propName == "_creationTimeUnix" { - return extractCreationTimeUnix(data) - } - if propName == "_lastUpdateTimeUnix" { - return extractLastUpdateTimeUnix(data) - } - return ParseAndExtractTextProp(data, propName) -} - -func ParseAndExtractTextProp(data []byte, propName string) ([]string, bool, error) { - vals := []string{} - err := parseAndExtractValueProp(data, propName, func(value []byte) { - vals = append(vals, string(value)) - }) - if err != nil { - return nil, false, err - } - return vals, true, nil -} - -func ParseAndExtractNumberArrayProp(data []byte, propName string) ([]float64, bool, error) { - vals := []float64{} - err := parseAndExtractValueProp(data, propName, func(value []byte) { - vals = append(vals, mustExtractNumber(value)) - }) - if err != nil { - return nil, false, err - } - return vals, true, nil -} - -func ParseAndExtractBoolArrayProp(data []byte, propName string) ([]bool, bool, error) { - vals := []bool{} - err := parseAndExtractValueProp(data, propName, func(value []byte) { - vals = append(vals, mustExtractBool(value)) - }) - if err != nil { - return nil, false, err - } - return vals, true, nil -} - -func parseAndExtractValueProp(data []byte, propName string, valueFn func(value []byte)) error { - propsBytes, err := extractPropsBytes(data) - if err != nil { - return err - } - - val, t, _, err := jsonparser.Get(propsBytes, propName) - // Some objects can have nil as value for the property, in this case skip the object - if err != nil { - if err.Error() == "Key path not found" { - return nil - } - return err - } - - if t == jsonparser.Array { - jsonparser.ArrayEach(val, func(value []byte, dataType jsonparser.ValueType, offset int, err error) { - valueFn(value) - }) - } else { - valueFn(val) - } - - return nil -} - -func mustExtractNumber(value []byte) float64 { - number, err := strconv.ParseFloat(string(value), 64) - if err != nil { - panic("not a float64") - } - return number -} - -func mustExtractBool(value []byte) bool { - boolVal, err := strconv.ParseBool(string(value)) - if err != nil { - panic("not a bool") - } - return boolVal -} - -func extractID(data []byte) ([]string, bool, error) { - start := 1 + 8 + 1 - end := start + 16 - if len(data) > end { - uuidParsed, err := uuid.FromBytes(data[start:end]) - if err != nil { - return nil, false, errors.New("cannot parse id property") - } - return []string{uuidParsed.String()}, true, nil - } - return nil, false, errors.New("id property not found") -} - -func extractCreationTimeUnix(data []byte) ([]string, bool, error) { - start := 1 + 8 + 1 + 16 - end := start + 8 - if len(data) > end { - return extractTimeUnix(data[start:end], "_creationTimeUnix") - } - return nil, false, errors.New("_creationTimeUnix property not found") -} - -func extractLastUpdateTimeUnix(data []byte) ([]string, bool, error) { - start := 1 + 8 + 1 + 16 + 8 - end := start + 8 - if len(data) > end { - return extractTimeUnix(data[start:end], "_lastUpdateTimeUnix") - } - return nil, false, errors.New("_lastUpdateTimeUnix property not found") -} - -func extractTimeUnix(data []byte, propertyName string) ([]string, bool, error) { - var timeUnix int64 - r := bytes.NewReader(data) - if err := binary.Read(r, binary.LittleEndian, &timeUnix); err != nil { - return nil, false, errors.Errorf("cannot parse %s property", propertyName) - } - return []string{strconv.FormatInt(timeUnix, 10)}, true, nil -} - -func extractPropsBytes(data []byte) ([]byte, error) { - version := uint8(data[0]) - if version != 1 { - return nil, errors.Errorf("unsupported binary marshaller version %d", version) - } - - vecLen := binary.LittleEndian.Uint16(data[discardBytesPreVector : discardBytesPreVector+2]) - - classNameStart := int64(discardBytesPreVector) + 2 + int64(vecLen)*4 - - classNameLen := binary.LittleEndian.Uint16(data[classNameStart : classNameStart+2]) - - propsLenStart := classNameStart + 2 + int64(classNameLen) - propsLen := binary.LittleEndian.Uint32(data[propsLenStart : propsLenStart+4]) - - start := int64(propsLenStart + 4) - end := start + int64(propsLen) - - return data[start:end], nil -} - -const discardBytesPreVector = 1 + 8 + 1 + 16 + 8 + 8 diff --git a/entities/storobj/storage_object.go b/entities/storobj/storage_object.go deleted file mode 100644 index 6b6c8ff55a57585d3bd12ea3423d304947371082..0000000000000000000000000000000000000000 --- a/entities/storobj/storage_object.go +++ /dev/null @@ -1,912 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package storobj - -import ( - "bytes" - "encoding/binary" - "encoding/json" - "fmt" - "math" - - "github.com/buger/jsonparser" - - "github.com/go-openapi/strfmt" - "github.com/google/uuid" - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/search" - "github.com/weaviate/weaviate/usecases/byteops" -) - -var bufPool *bufferPool - -func init() { - // a 10kB buffer should be large enough for typical cases, it can fit a - // 1536d uncompressed vector and about 3kB of object payload. If the - // initial size is not large enoug, the caller can always allocate a larger - // buffer and return that to the pool instead. - bufPool = newBufferPool(10 * 1024) -} - -type Object struct { - MarshallerVersion uint8 - Object models.Object `json:"object"` - Vector []float32 `json:"vector"` - VectorLen int `json:"-"` - BelongsToNode string `json:"-"` - BelongsToShard string `json:"-"` - IsConsistent bool `json:"-"` - - docID uint64 -} - -func New(docID uint64) *Object { - return &Object{ - MarshallerVersion: 1, - docID: docID, - } -} - -func FromObject(object *models.Object, vector []float32) *Object { - // clear out nil entries of properties to make sure leaving a property out and setting it nil is identical - properties, ok := object.Properties.(map[string]interface{}) - if ok { - for key, prop := range properties { - if prop == nil { - delete(properties, key) - } - } - object.Properties = properties - } - - return &Object{ - Object: *object, - Vector: vector, - MarshallerVersion: 1, - VectorLen: len(vector), - } -} - -func FromBinary(data []byte) (*Object, error) { - ko := &Object{} - if err := ko.UnmarshalBinary(data); err != nil { - return nil, err - } - - return ko, nil -} - -func FromBinaryUUIDOnly(data []byte) (*Object, error) { - ko := &Object{} - - rw := byteops.NewReadWriter(data) - version := rw.ReadUint8() - if version != 1 { - return nil, errors.Errorf("unsupported binary marshaller version %d", version) - } - - ko.MarshallerVersion = version - - ko.docID = rw.ReadUint64() - rw.MoveBufferPositionForward(1) // ignore kind-byte - uuidObj, err := uuid.FromBytes(rw.ReadBytesFromBuffer(16)) - if err != nil { - return nil, fmt.Errorf("parse uuid: %w", err) - } - ko.Object.ID = strfmt.UUID(uuidObj.String()) - - rw.MoveBufferPositionForward(16) - - vecLen := rw.ReadUint16() - rw.MoveBufferPositionForward(uint64(vecLen * 4)) - classNameLen := rw.ReadUint16() - - ko.Object.Class = string(rw.ReadBytesFromBuffer(uint64(classNameLen))) - - return ko, nil -} - -func FromBinaryOptional(data []byte, - addProp additional.Properties, -) (*Object, error) { - ko := &Object{} - - rw := byteops.NewReadWriter(data) - ko.MarshallerVersion = rw.ReadUint8() - if ko.MarshallerVersion != 1 { - return nil, errors.Errorf("unsupported binary marshaller version %d", ko.MarshallerVersion) - } - ko.docID = rw.ReadUint64() - rw.MoveBufferPositionForward(1) // ignore kind-byte - uuidObj, err := uuid.FromBytes(rw.ReadBytesFromBuffer(16)) - if err != nil { - return nil, fmt.Errorf("parse uuid: %w", err) - } - uuidParsed := strfmt.UUID(uuidObj.String()) - - createTime := int64(rw.ReadUint64()) - updateTime := int64(rw.ReadUint64()) - vectorLength := rw.ReadUint16() - // The vector length should always be returned (for usage metrics purposes) even if the vector itself is skipped - ko.VectorLen = int(vectorLength) - if addProp.Vector { - ko.Object.Vector = make([]float32, vectorLength) - vectorBytes := rw.ReadBytesFromBuffer(uint64(vectorLength) * 4) - for i := 0; i < int(vectorLength); i++ { - bits := binary.LittleEndian.Uint32(vectorBytes[i*4 : (i+1)*4]) - ko.Object.Vector[i] = math.Float32frombits(bits) - } - } else { - rw.MoveBufferPositionForward(uint64(vectorLength) * 4) - ko.Object.Vector = nil - } - ko.Vector = ko.Object.Vector - - classNameLen := rw.ReadUint16() - className := string(rw.ReadBytesFromBuffer(uint64(classNameLen))) - - propLength := rw.ReadUint32() - var props []byte - if addProp.NoProps { - rw.MoveBufferPositionForward(uint64(propLength)) - } else { - props = rw.ReadBytesFromBuffer(uint64(propLength)) - } - - var meta []byte - metaLength := rw.ReadUint32() - if addProp.Classification || len(addProp.ModuleParams) > 0 { - meta = rw.ReadBytesFromBuffer(uint64(metaLength)) - } else { - rw.MoveBufferPositionForward(uint64(metaLength)) - } - - vectorWeightsLength := rw.ReadUint32() - vectorWeights := rw.ReadBytesFromBuffer(uint64(vectorWeightsLength)) - - // some object members need additional "enrichment". Only do this if necessary, ie if they are actually present - if len(props) > 0 || - len(meta) > 0 || - vectorWeightsLength > 0 && - !( // if the length is 4 and the encoded value is "null" (in ascii), vectorweights are not actually present - vectorWeightsLength == 4 && - vectorWeights[0] == 110 && // n - vectorWeights[1] == 117 && // u - vectorWeights[2] == 108 && // l - vectorWeights[3] == 108) { // l - - if err := ko.parseObject( - uuidParsed, - createTime, - updateTime, - className, - props, - meta, - vectorWeights, - ); err != nil { - return nil, errors.Wrap(err, "parse") - } - } else { - ko.Object.ID = uuidParsed - ko.Object.CreationTimeUnix = createTime - ko.Object.LastUpdateTimeUnix = updateTime - ko.Object.Class = className - } - - return ko, nil -} - -type bucket interface { - GetBySecondary(int, []byte) ([]byte, error) - GetBySecondaryWithBuffer(int, []byte, []byte) ([]byte, []byte, error) -} - -func ObjectsByDocID(bucket bucket, ids []uint64, - additional additional.Properties, -) ([]*Object, error) { - if bucket == nil { - return nil, fmt.Errorf("objects bucket not found") - } - - var ( - docIDBuf = make([]byte, 8) - out = make([]*Object, len(ids)) - i = 0 - lsmBuf = bufPool.Get() - ) - - defer func() { - bufPool.Put(lsmBuf) - }() - - for _, id := range ids { - binary.LittleEndian.PutUint64(docIDBuf, id) - res, newBuf, err := bucket.GetBySecondaryWithBuffer(0, docIDBuf, lsmBuf) - if err != nil { - return nil, err - } - - lsmBuf = newBuf // may have changed, e.g. because it was grown - - if res == nil { - continue - } - - unmarshalled, err := FromBinaryOptional(res, additional) - if err != nil { - return nil, errors.Wrapf(err, "unmarshal data object at position %d", i) - } - - out[i] = unmarshalled - i++ - } - - return out[:i], nil -} - -func (ko *Object) Class() schema.ClassName { - return schema.ClassName(ko.Object.Class) -} - -func (ko *Object) SetDocID(id uint64) { - ko.docID = id -} - -func (ko *Object) DocID() uint64 { - return ko.docID -} - -func (ko *Object) CreationTimeUnix() int64 { - return ko.Object.CreationTimeUnix -} - -func (ko *Object) Score() float32 { - props := ko.AdditionalProperties() - if props != nil { - iface := props["score"] - if iface != nil { - return iface.(float32) - } - } - return 0 -} - -func (ko *Object) ExplainScore() string { - props := ko.AdditionalProperties() - if props != nil { - iface := props["explainScore"] - if iface != nil { - return iface.(string) - } - } - return "" -} - -func (ko *Object) ID() strfmt.UUID { - return ko.Object.ID -} - -func (ko *Object) SetID(id strfmt.UUID) { - ko.Object.ID = id -} - -func (ko *Object) SetClass(class string) { - ko.Object.Class = class -} - -func (ko *Object) LastUpdateTimeUnix() int64 { - return ko.Object.LastUpdateTimeUnix -} - -// AdditionalProperties groups all properties which are stored with the -// object and not generated at runtime -func (ko *Object) AdditionalProperties() models.AdditionalProperties { - return ko.Object.Additional -} - -func (ko *Object) Properties() models.PropertySchema { - return ko.Object.Properties -} - -func (ko *Object) PropertiesWithAdditional( - additional additional.Properties, -) models.PropertySchema { - properties := ko.Properties() - - if additional.RefMeta { - // nothing to remove - return properties - } - - asMap, ok := properties.(map[string]interface{}) - if !ok || asMap == nil { - return properties - } - - for propName, value := range asMap { - asRefs, ok := value.(models.MultipleRef) - if !ok { - // not a ref, we can skip - continue - } - - for i := range asRefs { - asRefs[i].Classification = nil - } - - asMap[propName] = asRefs - } - - return asMap -} - -func (ko *Object) SetProperties(schema models.PropertySchema) { - ko.Object.Properties = schema -} - -func (ko *Object) VectorWeights() models.VectorWeights { - return ko.Object.VectorWeights -} - -func (ko *Object) SearchResult(additional additional.Properties, tenant string) *search.Result { - propertiesMap, ok := ko.PropertiesWithAdditional(additional).(map[string]interface{}) - if !ok || propertiesMap == nil { - propertiesMap = map[string]interface{}{} - } - propertiesMap["id"] = ko.ID() - ko.SetProperties(propertiesMap) - - additionalProperties := models.AdditionalProperties{} - if ko.AdditionalProperties() != nil { - if interpretation, ok := additional.ModuleParams["interpretation"]; ok { - if interpretationValue, ok := interpretation.(bool); ok && interpretationValue { - additionalProperties["interpretation"] = ko.AdditionalProperties()["interpretation"] - } - } - if additional.Classification { - additionalProperties["classification"] = ko.AdditionalProperties()["classification"] - } - if additional.Group { - additionalProperties["group"] = ko.AdditionalProperties()["group"] - } - } - if ko.ExplainScore() != "" { - additionalProperties["explainScore"] = ko.ExplainScore() - } - - return &search.Result{ - ID: ko.ID(), - ClassName: ko.Class().String(), - Schema: ko.Properties(), - Vector: ko.Vector, - Dims: ko.VectorLen, - // VectorWeights: ko.VectorWeights(), // TODO: add vector weights - Created: ko.CreationTimeUnix(), - Updated: ko.LastUpdateTimeUnix(), - AdditionalProperties: additionalProperties, - Score: ko.Score(), - ExplainScore: ko.ExplainScore(), - IsConsistent: ko.IsConsistent, - Tenant: tenant, // not part of the binary - // TODO: Beacon? - } -} - -func (ko *Object) SearchResultWithDist(addl additional.Properties, dist float32) search.Result { - res := ko.SearchResult(addl, "") - res.Dist = dist - res.Certainty = float32(additional.DistToCertainty(float64(dist))) - return *res -} - -func (ko *Object) Valid() bool { - return ko.ID() != "" && - ko.Class().String() != "" -} - -func SearchResults(in []*Object, additional additional.Properties, tenant string) search.Results { - out := make(search.Results, len(in)) - - for i, elem := range in { - out[i] = *(elem.SearchResult(additional, tenant)) - } - - return out -} - -func SearchResultsWithDists(in []*Object, addl additional.Properties, - dists []float32, -) search.Results { - out := make(search.Results, len(in)) - - for i, elem := range in { - out[i] = elem.SearchResultWithDist(addl, dists[i]) - } - - return out -} - -func DocIDFromBinary(in []byte) (uint64, error) { - var version uint8 - r := bytes.NewReader(in) - le := binary.LittleEndian - if err := binary.Read(r, le, &version); err != nil { - return 0, err - } - - if version != 1 { - return 0, errors.Errorf("unsupported binary marshaller version %d", version) - } - - var docID uint64 - err := binary.Read(r, le, &docID) - return docID, err -} - -// MarshalBinary creates the binary representation of a kind object. Regardless -// of the marshaller version the first byte is a uint8 indicating the version -// followed by the payload which depends on the specific version -// -// Version 1 -// No. of B | Type | Content -// ------------------------------------------------ -// 1 | uint8 | MarshallerVersion = 1 -// 8 | uint64 | index id, keep early so id-only lookups are maximum efficient -// 1 | uint8 | kind, 0=action, 1=thing - deprecated -// 16 | uint128 | uuid -// 8 | int64 | create time -// 8 | int64 | update time -// 2 | uint16 | VectorLength -// n*4 | []float32 | vector of length n -// 2 | uint16 | length of class name -// n | []byte | className -// 4 | uint32 | length of schema json -// n | []byte | schema as json -// 2 | uint32 | length of meta json -// n | []byte | meta as json -// 2 | uint32 | length of vectorweights json -// n | []byte | vectorweights as json -func (ko *Object) MarshalBinary() ([]byte, error) { - if ko.MarshallerVersion != 1 { - return nil, errors.Errorf("unsupported marshaller version %d", ko.MarshallerVersion) - } - - kindByte := uint8(0) - // Deprecated Kind field - kindByte = 1 - - idParsed, err := uuid.Parse(ko.ID().String()) - if err != nil { - return nil, err - } - idBytes, err := idParsed.MarshalBinary() - if err != nil { - return nil, err - } - vectorLength := uint32(len(ko.Vector)) - className := []byte(ko.Class()) - classNameLength := uint32(len(className)) - schema, err := json.Marshal(ko.Properties()) - if err != nil { - return nil, err - } - schemaLength := uint32(len(schema)) - meta, err := json.Marshal(ko.AdditionalProperties()) - if err != nil { - return nil, err - } - metaLength := uint32(len(meta)) - vectorWeights, err := json.Marshal(ko.VectorWeights()) - if err != nil { - return nil, err - } - vectorWeightsLength := uint32(len(vectorWeights)) - - totalBufferLength := 1 + 8 + 1 + 16 + 8 + 8 + 2 + vectorLength*4 + 2 + classNameLength + 4 + schemaLength + 4 + metaLength + 4 + vectorWeightsLength - byteBuffer := make([]byte, totalBufferLength) - rw := byteops.NewReadWriter(byteBuffer) - rw.WriteByte(ko.MarshallerVersion) - rw.WriteUint64(ko.docID) - rw.WriteByte(kindByte) - - rw.CopyBytesToBuffer(idBytes) - - rw.WriteUint64(uint64(ko.CreationTimeUnix())) - rw.WriteUint64(uint64(ko.LastUpdateTimeUnix())) - rw.WriteUint16(uint16(vectorLength)) - - for j := uint32(0); j < vectorLength; j++ { - rw.WriteUint32(math.Float32bits(ko.Vector[j])) - } - - rw.WriteUint16(uint16(classNameLength)) - err = rw.CopyBytesToBuffer(className) - if err != nil { - return byteBuffer, errors.Wrap(err, "Could not copy className") - } - - rw.WriteUint32(schemaLength) - err = rw.CopyBytesToBuffer(schema) - if err != nil { - return byteBuffer, errors.Wrap(err, "Could not copy schema") - } - - rw.WriteUint32(metaLength) - err = rw.CopyBytesToBuffer(meta) - if err != nil { - return byteBuffer, errors.Wrap(err, "Could not copy meta") - } - rw.WriteUint32(vectorWeightsLength) - err = rw.CopyBytesToBuffer(vectorWeights) - if err != nil { - return byteBuffer, errors.Wrap(err, "Could not copy vectorWeights") - } - - return byteBuffer, nil -} - -// UnmarshalPropertiesFromObject only unmarshals and returns the properties part of the object -// -// Check MarshalBinary for the order of elements in the input array -func UnmarshalPropertiesFromObject(data []byte, properties *map[string]interface{}, aggregationProperties []string, propStrings [][]string) error { - if data[0] != uint8(1) { - return errors.Errorf("unsupported binary marshaller version %d", data[0]) - } - - // clear out old values in case an object misses values. This should NOT shrink the capacity of the map, eg there - // are no allocations when adding the properties of the next object again - for k := range *properties { - delete(*properties, k) - } - - startPos := uint64(1 + 8 + 1 + 16 + 8 + 8) // elements at the start - rw := byteops.NewReadWriter(data, byteops.WithPosition(startPos)) - // get the length of the vector, each element is a float32 (4 bytes) - vectorLength := uint64(rw.ReadUint16()) - rw.MoveBufferPositionForward(vectorLength * 4) - - classnameLength := uint64(rw.ReadUint16()) - rw.MoveBufferPositionForward(classnameLength) - propertyLength := uint64(rw.ReadUint32()) - - jsonparser.EachKey(data[rw.Position:rw.Position+propertyLength], func(idx int, value []byte, dataType jsonparser.ValueType, err error) { - var errParse error - switch dataType { - case jsonparser.Number, jsonparser.String, jsonparser.Boolean: - val, err := parseValues(dataType, value) - errParse = err - (*properties)[aggregationProperties[idx]] = val - case jsonparser.Array: // can be a beacon or an actual array - arrayEntries := value[1 : len(value)-1] // without leading and trailing [] - beaconVal, errBeacon := jsonparser.GetUnsafeString(arrayEntries, "beacon") - if errBeacon == nil { - (*properties)[aggregationProperties[idx]] = []interface{}{map[string]interface{}{"beacon": beaconVal}} - } else { - // check how many entries there are in the array by counting the ",". This allows us to allocate an - // array with the right size without extending it with every append. - // The size can be too large for string arrays, when they contain "," as part of their content. - entryCount := 0 - for _, b := range arrayEntries { - if b == uint8(44) { // ',' as byte - entryCount++ - } - } - - array := make([]interface{}, 0, entryCount) - jsonparser.ArrayEach(value, func(innerValue []byte, innerDataType jsonparser.ValueType, offset int, innerErr error) { - var val interface{} - - switch innerDataType { - case jsonparser.Number, jsonparser.String, jsonparser.Boolean: - val, errParse = parseValues(innerDataType, innerValue) - default: - panic("Unknown data type ArrayEach") // returning an error would be better - } - array = append(array, val) - }) - (*properties)[aggregationProperties[idx]] = array - - } - default: - panic("Unknown data type EachKey") // returning an error would be better - } - if errParse != nil { - panic(errParse) - } - }, propStrings...) - - return nil -} - -func parseValues(dt jsonparser.ValueType, value []byte) (interface{}, error) { - switch dt { - case jsonparser.Number: - return jsonparser.ParseFloat(value) - case jsonparser.String: - return jsonparser.ParseString(value) - case jsonparser.Boolean: - return jsonparser.ParseBoolean(value) - default: - panic("Unknown data type") // returning an error would be better - } -} - -// UnmarshalBinary is the versioned way to unmarshal a kind object from binary, -// see MarshalBinary for the exact contents of each version -func (ko *Object) UnmarshalBinary(data []byte) error { - version := data[0] - if version != 1 { - return errors.Errorf("unsupported binary marshaller version %d", version) - } - ko.MarshallerVersion = version - - rw := byteops.NewReadWriter(data, byteops.WithPosition(1)) - ko.docID = rw.ReadUint64() - rw.MoveBufferPositionForward(1) // kind-byte - - uuidParsed, err := uuid.FromBytes(data[rw.Position : rw.Position+16]) - if err != nil { - return err - } - rw.MoveBufferPositionForward(16) - - createTime := int64(rw.ReadUint64()) - updateTime := int64(rw.ReadUint64()) - - vectorLength := rw.ReadUint16() - ko.VectorLen = int(vectorLength) - ko.Vector = make([]float32, vectorLength) - for j := 0; j < int(vectorLength); j++ { - ko.Vector[j] = math.Float32frombits(rw.ReadUint32()) - } - - classNameLength := uint64(rw.ReadUint16()) - className, err := rw.CopyBytesFromBuffer(classNameLength, nil) - if err != nil { - return errors.Wrap(err, "Could not copy class name") - } - - schemaLength := uint64(rw.ReadUint32()) - schema, err := rw.CopyBytesFromBuffer(schemaLength, nil) - if err != nil { - return errors.Wrap(err, "Could not copy schema") - } - - metaLength := uint64(rw.ReadUint32()) - meta, err := rw.CopyBytesFromBuffer(metaLength, nil) - if err != nil { - return errors.Wrap(err, "Could not copy meta") - } - - vectorWeightsLength := uint64(rw.ReadUint32()) - vectorWeights, err := rw.CopyBytesFromBuffer(vectorWeightsLength, nil) - if err != nil { - return errors.Wrap(err, "Could not copy vectorWeights") - } - - return ko.parseObject( - strfmt.UUID(uuidParsed.String()), - createTime, - updateTime, - string(className), - schema, - meta, - vectorWeights, - ) -} - -func VectorFromBinary(in []byte, buffer []float32) ([]float32, error) { - if len(in) == 0 { - return nil, nil - } - - version := in[0] - if version != 1 { - return nil, errors.Errorf("unsupported marshaller version %d", version) - } - - // since we know the version and know that the blob is not len(0), we can - // assume that we can directly access the vector length field. The only - // situation where this is not accessible would be on corrupted data - where - // it would be acceptable to panic - vecLen := binary.LittleEndian.Uint16(in[42:44]) - - var out []float32 - if cap(buffer) >= int(vecLen) { - out = buffer[:vecLen] - } else { - out = make([]float32, vecLen) - } - vecStart := 44 - vecEnd := vecStart + int(vecLen*4) - - i := 0 - for start := vecStart; start < vecEnd; start += 4 { - asUint := binary.LittleEndian.Uint32(in[start : start+4]) - out[i] = math.Float32frombits(asUint) - i++ - } - - return out, nil -} - -func (ko *Object) parseObject(uuid strfmt.UUID, create, update int64, className string, - propsB []byte, additionalB []byte, vectorWeightsB []byte, -) error { - var props map[string]interface{} - if err := json.Unmarshal(propsB, &props); err != nil { - return err - } - - if err := enrichSchemaTypes(props, false); err != nil { - return errors.Wrap(err, "enrich schema datatypes") - } - - var additionalProperties models.AdditionalProperties - if len(additionalB) > 0 { - if err := json.Unmarshal(additionalB, &additionalProperties); err != nil { - return err - } - - if prop, ok := additionalProperties["classification"]; ok { - if classificationMap, ok := prop.(map[string]interface{}); ok { - marshalled, err := json.Marshal(classificationMap) - if err != nil { - return err - } - var classification additional.Classification - err = json.Unmarshal(marshalled, &classification) - if err != nil { - return err - } - additionalProperties["classification"] = &classification - } - } - - if prop, ok := additionalProperties["group"]; ok { - if groupMap, ok := prop.(map[string]interface{}); ok { - marshalled, err := json.Marshal(groupMap) - if err != nil { - return err - } - var group additional.Group - err = json.Unmarshal(marshalled, &group) - if err != nil { - return err - } - - for i, hit := range group.Hits { - if groupHitAdditionalMap, ok := hit["_additional"].(map[string]interface{}); ok { - marshalled, err := json.Marshal(groupHitAdditionalMap) - if err != nil { - return err - } - var groupHitsAdditional additional.GroupHitAdditional - err = json.Unmarshal(marshalled, &groupHitsAdditional) - if err != nil { - return err - } - group.Hits[i]["_additional"] = &groupHitsAdditional - } - } - - additionalProperties["group"] = &group - } - } - } - - var vectorWeights interface{} - if err := json.Unmarshal(vectorWeightsB, &vectorWeights); err != nil { - return err - } - - ko.Object = models.Object{ - Class: className, - CreationTimeUnix: create, - LastUpdateTimeUnix: update, - ID: uuid, - Properties: props, - VectorWeights: vectorWeights, - Additional: additionalProperties, - } - - return nil -} - -// DeepCopyDangerous creates a deep copy of the underlying Object -// WARNING: This was purpose built for the batch ref usecase and only covers -// the situations that are required there. This means that cases which aren't -// reflected in that usecase may still contain references. Thus the suffix -// "Dangerous". If needed, make sure everything is copied and remove the -// suffix. -func (ko *Object) DeepCopyDangerous() *Object { - return &Object{ - MarshallerVersion: ko.MarshallerVersion, - docID: ko.docID, - Object: deepCopyObject(ko.Object), - Vector: deepCopyVector(ko.Vector), - } -} - -func AddOwnership(objs []*Object, node, shard string) { - for i := range objs { - objs[i].BelongsToNode = node - objs[i].BelongsToShard = shard - } -} - -func deepCopyVector(orig []float32) []float32 { - out := make([]float32, len(orig)) - copy(out, orig) - return out -} - -func deepCopyObject(orig models.Object) models.Object { - return models.Object{ - Class: orig.Class, - ID: orig.ID, - CreationTimeUnix: orig.CreationTimeUnix, - LastUpdateTimeUnix: orig.LastUpdateTimeUnix, - Vector: deepCopyVector(orig.Vector), - VectorWeights: orig.VectorWeights, - Additional: orig.Additional, // WARNING: not a deep copy!! - Properties: deepCopyProperties(orig.Properties), - } -} - -func deepCopyProperties(orig models.PropertySchema) models.PropertySchema { - if orig == nil { - return nil - } - - asMap, ok := orig.(map[string]interface{}) - if !ok { - // not a map, don't know what to do with this - return nil - } - - out := map[string]interface{}{} - - for key, value := range asMap { - if mref, ok := value.(models.MultipleRef); ok { - out[key] = deepCopyMRef(mref) - continue - } - - // Note: This is not a true deep copy, value could still be a pointer type, - // such as *models.GeoCoordinates, thus leading to passing a reference - // instead of actually making a copy. However, for the purposes we need - // this method for this is acceptable based on our current knowledge - out[key] = value - } - - return out -} - -func deepCopyMRef(orig models.MultipleRef) models.MultipleRef { - if orig == nil { - return nil - } - - out := make(models.MultipleRef, len(orig)) - for i, ref := range orig { - // models.SingleRef contains only pass-by-value props, so a simple deref as - // the struct creates a copy - copiedRef := *ref - out[i] = &copiedRef - } - - return out -} diff --git a/entities/storobj/storage_object_test.go b/entities/storobj/storage_object_test.go deleted file mode 100644 index fc5795402960bef167ef5e530f97ceb67d57d468..0000000000000000000000000000000000000000 --- a/entities/storobj/storage_object_test.go +++ /dev/null @@ -1,602 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package storobj - -import ( - "fmt" - "testing" - "time" - - "github.com/go-openapi/strfmt" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" -) - -func TestStorageObjectMarshalling(t *testing.T) { - before := FromObject( - &models.Object{ - Class: "MyFavoriteClass", - CreationTimeUnix: 123456, - LastUpdateTimeUnix: 56789, - ID: strfmt.UUID("73f2eb5f-5abf-447a-81ca-74b1dd168247"), - Additional: models.AdditionalProperties{ - "classification": &additional.Classification{ - BasedOn: []string{"some", "fields"}, - }, - "interpretation": map[string]interface{}{ - "Source": []interface{}{ - map[string]interface{}{ - "concept": "foo", - "occurrence": float64(7), - "weight": float64(3), - }, - }, - }, - }, - Properties: map[string]interface{}{ - "name": "MyName", - "foo": float64(17), - }, - }, - []float32{1, 2, 0.7}, - ) - before.SetDocID(7) - - asBinary, err := before.MarshalBinary() - require.Nil(t, err) - - after, err := FromBinary(asBinary) - require.Nil(t, err) - - t.Run("compare", func(t *testing.T) { - assert.Equal(t, before, after) - }) - - t.Run("extract only doc id and compare", func(t *testing.T) { - id, err := DocIDFromBinary(asBinary) - require.Nil(t, err) - assert.Equal(t, uint64(7), id) - }) - - t.Run("extract single text prop", func(t *testing.T) { - prop, ok, err := ParseAndExtractTextProp(asBinary, "name") - require.Nil(t, err) - require.True(t, ok) - require.NotEmpty(t, prop) - assert.Equal(t, "MyName", prop[0]) - }) - - t.Run("extract non-existing text prop", func(t *testing.T) { - prop, ok, err := ParseAndExtractTextProp(asBinary, "IDoNotExist") - require.Nil(t, err) - require.True(t, ok) - require.Empty(t, prop) - }) -} - -func TestFilteringNilProperty(t *testing.T) { - object := FromObject( - &models.Object{ - Class: "MyFavoriteClass", - ID: "73f2eb5f-5abf-447a-81ca-74b1dd168247", - Properties: map[string]interface{}{ - "IWillBeRemoved": nil, - "IWillStay": float64(17), - }, - }, - []float32{1, 2, 0.7}, - ) - props := object.Properties() - propsTyped, ok := props.(map[string]interface{}) - require.True(t, ok) - assert.Equal(t, propsTyped["IWillStay"], float64(17)) - - elem, ok := propsTyped["IWillBeRemoved"] - require.False(t, ok) - require.Nil(t, elem) -} - -func TestStorageObjectUnmarshallingSpecificProps(t *testing.T) { - before := FromObject( - &models.Object{ - Class: "MyFavoriteClass", - CreationTimeUnix: 123456, - LastUpdateTimeUnix: 56789, - ID: strfmt.UUID("73f2eb5f-5abf-447a-81ca-74b1dd168247"), - Additional: models.AdditionalProperties{ - "classification": &additional.Classification{ - BasedOn: []string{"some", "fields"}, - }, - "interpretation": map[string]interface{}{ - "Source": []interface{}{ - map[string]interface{}{ - "concept": "foo", - "occurrence": float64(7), - "weight": float64(3), - }, - }, - }, - }, - Properties: map[string]interface{}{ - "name": "MyName", - "foo": float64(17), - }, - }, - []float32{1, 2, 0.7}, - ) - before.SetDocID(7) - - asBinary, err := before.MarshalBinary() - require.Nil(t, err) - - t.Run("without any optional", func(t *testing.T) { - after, err := FromBinaryOptional(asBinary, additional.Properties{}) - require.Nil(t, err) - - t.Run("compare", func(t *testing.T) { - // modify before to match expectations of after - before.Object.Additional = nil - before.Vector = nil - before.VectorLen = 3 - assert.Equal(t, before, after) - - assert.Equal(t, before.docID, after.docID) - - // The vector length should always be returned (for usage metrics - // purposes) even if the vector itself is skipped - assert.Equal(t, after.VectorLen, 3) - }) - }) -} - -func TestNewStorageObject(t *testing.T) { - t.Run("objects", func(t *testing.T) { - so := New(12) - - t.Run("check index id", func(t *testing.T) { - assert.Equal(t, uint64(12), so.docID) - }) - - t.Run("is invalid without required params", func(t *testing.T) { - assert.False(t, so.Valid()) - }) - - t.Run("reassign index id", func(t *testing.T) { - so.SetDocID(13) - assert.Equal(t, uint64(13), so.docID) - }) - - t.Run("assign class", func(t *testing.T) { - so.SetClass("MyClass") - assert.Equal(t, schema.ClassName("MyClass"), so.Class()) - }) - - t.Run("assign uuid", func(t *testing.T) { - id := strfmt.UUID("bf706904-8618-463f-899c-4a2aafd48d56") - so.SetID(id) - assert.Equal(t, id, so.ID()) - }) - - t.Run("assign uuid", func(t *testing.T) { - schema := map[string]interface{}{ - "foo": "bar", - } - so.SetProperties(schema) - assert.Equal(t, schema, so.Properties()) - }) - - t.Run("must now be valid", func(t *testing.T) { - assert.True(t, so.Valid()) - }) - - t.Run("make sure it's identical with an object created from an existing object", - func(t *testing.T) { - alt := FromObject(&models.Object{ - Class: "MyClass", - ID: "bf706904-8618-463f-899c-4a2aafd48d56", - Properties: map[string]interface{}{ - "foo": "bar", - }, - }, nil) - alt.SetDocID(13) - - assert.Equal(t, so, alt) - }) - }) - - t.Run("objects", func(t *testing.T) { - so := New(12) - - t.Run("check index id", func(t *testing.T) { - assert.Equal(t, uint64(12), so.docID) - }) - - t.Run("is invalid without required params", func(t *testing.T) { - assert.False(t, so.Valid()) - }) - - t.Run("reassign index id", func(t *testing.T) { - so.SetDocID(13) - assert.Equal(t, uint64(13), so.docID) - }) - - t.Run("assign class", func(t *testing.T) { - so.SetClass("MyClass") - assert.Equal(t, schema.ClassName("MyClass"), so.Class()) - }) - - t.Run("assign uuid", func(t *testing.T) { - id := strfmt.UUID("bf706904-8618-463f-899c-4a2aafd48d56") - so.SetID(id) - assert.Equal(t, id, so.ID()) - }) - - t.Run("assign uuid", func(t *testing.T) { - schema := map[string]interface{}{ - "foo": "bar", - } - so.SetProperties(schema) - assert.Equal(t, schema, so.Properties()) - }) - - t.Run("must now be valid", func(t *testing.T) { - assert.True(t, so.Valid()) - }) - - t.Run("make sure it's identical with an object created from an existing action", - func(t *testing.T) { - alt := FromObject(&models.Object{ - Class: "MyClass", - ID: "bf706904-8618-463f-899c-4a2aafd48d56", - Properties: map[string]interface{}{ - "foo": "bar", - }, - }, nil) - alt.SetDocID(13) - - assert.Equal(t, so, alt) - }) - }) -} - -func TestStorageArrayObjectMarshalling(t *testing.T) { - before := FromObject( - &models.Object{ - Class: "MyFavoriteClass", - CreationTimeUnix: 123456, - LastUpdateTimeUnix: 56789, - ID: strfmt.UUID("73f2eb5f-5abf-447a-81ca-74b1dd168247"), - Additional: models.AdditionalProperties{ - "classification": &additional.Classification{ - BasedOn: []string{"some", "fields"}, - }, - "interpretation": map[string]interface{}{ - "Source": []interface{}{ - map[string]interface{}{ - "concept": "foo", - "occurrence": float64(7), - "weight": float64(3), - }, - }, - }, - }, - Properties: map[string]interface{}{ - "textArray": []string{"c", "d"}, - "numberArray": []float64{1.1, 2.1}, - "foo": float64(17), - }, - }, - []float32{1, 2, 0.7}, - ) - before.SetDocID(7) - - asBinary, err := before.MarshalBinary() - require.Nil(t, err) - - after, err := FromBinary(asBinary) - require.Nil(t, err) - - t.Run("compare", func(t *testing.T) { - assert.Equal(t, before, after) - }) - - t.Run("extract only doc id and compare", func(t *testing.T) { - id, err := DocIDFromBinary(asBinary) - require.Nil(t, err) - assert.Equal(t, uint64(7), id) - }) - - t.Run("extract text array prop", func(t *testing.T) { - prop, ok, err := ParseAndExtractTextProp(asBinary, "textArray") - require.Nil(t, err) - require.True(t, ok) - assert.Equal(t, []string{"c", "d"}, prop) - }) - - t.Run("extract number array prop", func(t *testing.T) { - prop, ok, err := ParseAndExtractNumberArrayProp(asBinary, "numberArray") - require.Nil(t, err) - require.True(t, ok) - assert.Equal(t, []float64{1.1, 2.1}, prop) - }) -} - -func TestExtractionOfSingleProperties(t *testing.T) { - expected := map[string]interface{}{ - "numberArray": []interface{}{1.1, 2.1}, - "intArray": []interface{}{1., 2., 5000.}, - "textArrayUTF": []interface{}{"語", "b"}, - "textArray": []interface{}{"hello", ",", "I", "am", "a", "veeery", "long", "Array", "with some text."}, - "foo": float64(17), - "text": "single string", - "bool": true, - "time": "2011-11-23T01:52:23.000004234Z", - "boolArray": []interface{}{true, false, true}, - "beacon": []interface{}{map[string]interface{}{"beacon": "weaviate://localhost/SomeClass/3453/73f4eb5f-5abf-447a-81ca-74b1dd168247"}}, - "ref": []interface{}{map[string]interface{}{"beacon": "weaviate://localhost/SomeClass/3453/73f4eb5f-5abf-447a-81ca-74b1dd168247"}}, - } - properties := map[string]interface{}{ - "numberArray": []float64{1.1, 2.1}, - "intArray": []int32{1, 2, 5000}, - "textArrayUTF": []string{"語", "b"}, - "textArray": []string{"hello", ",", "I", "am", "a", "veeery", "long", "Array", "with some text."}, - "foo": float64(17), - "text": "single string", - "bool": true, - "time": time.Date(2011, 11, 23, 1, 52, 23, 4234, time.UTC), - "boolArray": []bool{true, false, true}, - "beacon": []map[string]interface{}{{"beacon": "weaviate://localhost/SomeClass/3453/73f4eb5f-5abf-447a-81ca-74b1dd168247"}}, - "ref": []models.SingleRef{{Beacon: "weaviate://localhost/SomeClass/3453/73f4eb5f-5abf-447a-81ca-74b1dd168247", Class: "OtherClass", Href: "/v1/f81bfe5e-16ba-4615-a516-46c2ae2e5a80"}}, - } - before := FromObject( - &models.Object{ - Class: "MyFavoriteClass", - CreationTimeUnix: 123456, - LastUpdateTimeUnix: 56789, - ID: strfmt.UUID("73f2eb5f-5abf-447a-81ca-74b1dd168247"), - Properties: properties, - }, - []float32{1, 2, 0.7}, - ) - - before.SetDocID(7) - byteObject, err := before.MarshalBinary() - require.Nil(t, err) - - var propertyNames []string - var propStrings [][]string - for key := range properties { - propertyNames = append(propertyNames, key) - propStrings = append(propStrings, []string{key}) - } - - extractedProperties := map[string]interface{}{} - - // test with reused property map - for i := 0; i < 2; i++ { - require.Nil(t, UnmarshalPropertiesFromObject(byteObject, &extractedProperties, propertyNames, propStrings)) - for key := range expected { - require.Equal(t, expected[key], extractedProperties[key]) - } - - } -} - -func TestStorageObjectMarshallingWithGroup(t *testing.T) { - before := FromObject( - &models.Object{ - Class: "MyFavoriteClass", - CreationTimeUnix: 123456, - LastUpdateTimeUnix: 56789, - ID: strfmt.UUID("73f2eb5f-5abf-447a-81ca-74b1dd168247"), - Additional: models.AdditionalProperties{ - "classification": &additional.Classification{ - BasedOn: []string{"some", "fields"}, - }, - "interpretation": map[string]interface{}{ - "Source": []interface{}{ - map[string]interface{}{ - "concept": "foo", - "occurrence": float64(7), - "weight": float64(3), - }, - }, - }, - "group": &additional.Group{ - ID: 100, - GroupedBy: &additional.GroupedBy{ - Value: "group-by-some-property", - Path: []string{"property-path"}, - }, - MaxDistance: 0.1, - MinDistance: 0.2, - Count: 200, - Hits: []map[string]interface{}{ - { - "property1": "value1", - "_additional": &additional.GroupHitAdditional{ - ID: "2c76ca18-2073-4c48-aa52-7f444d2f5b80", - Distance: 0.24, - }, - }, - { - "property1": "value2", - }, - }, - }, - }, - Properties: map[string]interface{}{ - "name": "MyName", - "foo": float64(17), - }, - }, - []float32{1, 2, 0.7}, - ) - before.SetDocID(7) - - asBinary, err := before.MarshalBinary() - require.Nil(t, err) - - after, err := FromBinary(asBinary) - require.Nil(t, err) - - t.Run("compare", func(t *testing.T) { - assert.Equal(t, before, after) - }) - - t.Run("extract only doc id and compare", func(t *testing.T) { - id, err := DocIDFromBinary(asBinary) - require.Nil(t, err) - assert.Equal(t, uint64(7), id) - }) - - t.Run("extract single text prop", func(t *testing.T) { - prop, ok, err := ParseAndExtractTextProp(asBinary, "name") - require.Nil(t, err) - require.True(t, ok) - require.NotEmpty(t, prop) - assert.Equal(t, "MyName", prop[0]) - }) - - t.Run("extract non-existing text prop", func(t *testing.T) { - prop, ok, err := ParseAndExtractTextProp(asBinary, "IDoNotExist") - require.Nil(t, err) - require.True(t, ok) - require.Empty(t, prop) - }) - - t.Run("extract group additional property", func(t *testing.T) { - require.NotNil(t, after.AdditionalProperties()) - require.NotNil(t, after.AdditionalProperties()["group"]) - group, ok := after.AdditionalProperties()["group"].(*additional.Group) - require.True(t, ok) - assert.Equal(t, 100, group.ID) - assert.NotNil(t, group.GroupedBy) - assert.Equal(t, "group-by-some-property", group.GroupedBy.Value) - assert.Equal(t, []string{"property-path"}, group.GroupedBy.Path) - assert.Equal(t, 200, group.Count) - assert.Equal(t, float32(0.1), group.MaxDistance) - assert.Equal(t, float32(0.2), group.MinDistance) - require.Len(t, group.Hits, 2) - require.NotNil(t, group.Hits[0]["_additional"]) - groupHitAdditional, ok := group.Hits[0]["_additional"].(*additional.GroupHitAdditional) - require.True(t, ok) - assert.Equal(t, strfmt.UUID("2c76ca18-2073-4c48-aa52-7f444d2f5b80"), groupHitAdditional.ID) - assert.Equal(t, float32(0.24), groupHitAdditional.Distance) - assert.Equal(t, "value1", group.Hits[0]["property1"]) - require.Nil(t, group.Hits[1]["_additional"]) - assert.Equal(t, "value2", group.Hits[1]["property1"]) - }) -} - -func TestStorageMaxVectorDimensionsObjectMarshalling(t *testing.T) { - generateVector := func(dims uint16) []float32 { - vector := make([]float32, dims) - for i := range vector { - vector[i] = 0.1 - } - return vector - } - // 65535 is max uint16 number - edgeVectorLengths := []uint16{0, 1, 768, 50000, 65535} - for _, vectorLength := range edgeVectorLengths { - t.Run(fmt.Sprintf("%v vector dimensions", vectorLength), func(t *testing.T) { - t.Run("marshal binary", func(t *testing.T) { - vector := generateVector(vectorLength) - before := FromObject( - &models.Object{ - Class: "MyFavoriteClass", - CreationTimeUnix: 123456, - ID: strfmt.UUID("73f2eb5f-5abf-447a-81ca-74b1dd168247"), - Properties: map[string]interface{}{ - "name": "myName", - }, - }, - vector, - ) - before.SetDocID(7) - - asBinary, err := before.MarshalBinary() - require.Nil(t, err) - - after, err := FromBinary(asBinary) - require.Nil(t, err) - - t.Run("compare", func(t *testing.T) { - assert.Equal(t, before, after) - }) - - t.Run("try to extract a property", func(t *testing.T) { - prop, ok, err := ParseAndExtractTextProp(asBinary, "name") - require.Nil(t, err) - require.True(t, ok) - assert.Equal(t, []string{"myName"}, prop) - }) - }) - - t.Run("marshal optional binary", func(t *testing.T) { - vector := generateVector(vectorLength) - before := FromObject( - &models.Object{ - Class: "MyFavoriteClass", - CreationTimeUnix: 123456, - ID: strfmt.UUID("73f2eb5f-5abf-447a-81ca-74b1dd168247"), - Properties: map[string]interface{}{ - "name": "myName", - }, - }, - vector, - ) - before.SetDocID(7) - - asBinary, err := before.MarshalBinary() - require.Nil(t, err) - - t.Run("get without additional properties", func(t *testing.T) { - after, err := FromBinaryOptional(asBinary, additional.Properties{}) - require.Nil(t, err) - // modify before to match expectations of after - before.Object.Additional = nil - before.Vector = nil - before.VectorLen = int(vectorLength) - assert.Equal(t, before, after) - - assert.Equal(t, before.docID, after.docID) - - // The vector length should always be returned (for usage metrics - // purposes) even if the vector itself is skipped - assert.Equal(t, after.VectorLen, int(vectorLength)) - }) - - t.Run("get with additional property vector", func(t *testing.T) { - after, err := FromBinaryOptional(asBinary, additional.Properties{Vector: true}) - require.Nil(t, err) - // modify before to match expectations of after - before.Object.Additional = nil - before.Vector = vector - before.VectorLen = int(vectorLength) - assert.Equal(t, before, after) - - assert.Equal(t, before.docID, after.docID) - - // The vector length should always be returned (for usage metrics - // purposes) even if the vector itself is skipped - assert.Equal(t, after.VectorLen, int(vectorLength)) - assert.Equal(t, vector, after.Vector) - }) - }) - }) - } -} diff --git a/entities/vectorindex/common/config.go b/entities/vectorindex/common/config.go deleted file mode 100644 index 4fbf7cf9c02a0446d4c9802be06e917ffb8ac79d..0000000000000000000000000000000000000000 --- a/entities/vectorindex/common/config.go +++ /dev/null @@ -1,102 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package common - -import ( - "encoding/json" - "math" - "strconv" - - "github.com/pkg/errors" -) - -const ( - DistanceCosine = "cosine" - DistanceDot = "dot" - DistanceL2Squared = "l2-squared" - DistanceManhattan = "manhattan" - DistanceHamming = "hamming" - - // Set these defaults if the user leaves them blank - DefaultVectorCacheMaxObjects = 1e12 - DefaultDistanceMetric = DistanceCosine -) - -// Tries to parse the int value from the map, if it overflows math.MaxInt64, it -// uses math.MaxInt64 instead. This is to protect from rounding errors from -// json marshalling where the type may be assumed as float64 -func OptionalIntFromMap(in map[string]interface{}, name string, - setFn func(v int), -) error { - value, ok := in[name] - if !ok { - return nil - } - - var asInt64 int64 - var err error - - // depending on whether we get the results from disk or from the REST API, - // numbers may be represented slightly differently - switch typed := value.(type) { - case json.Number: - asInt64, err = typed.Int64() - case float64: - asInt64 = int64(typed) - } - if err != nil { - // try to recover from error - if errors.Is(err, strconv.ErrRange) { - setFn(int(math.MaxInt64)) - return nil - } - - return errors.Wrapf(err, "json.Number to int64 for %q", name) - } - - setFn(int(asInt64)) - return nil -} - -func OptionalBoolFromMap(in map[string]interface{}, name string, - setFn func(v bool), -) error { - value, ok := in[name] - if !ok { - return nil - } - - asBool, ok := value.(bool) - if !ok { - return nil - } - - setFn(asBool) - return nil -} - -func OptionalStringFromMap(in map[string]interface{}, name string, - setFn func(v string), -) error { - value, ok := in[name] - if !ok { - return nil - } - - asString, ok := value.(string) - if !ok { - return nil - } - - setFn(asString) - return nil -} diff --git a/entities/vectorindex/config.go b/entities/vectorindex/config.go deleted file mode 100644 index c17ca1933e9307b5cefd046ac8221e0c85cd4af8..0000000000000000000000000000000000000000 --- a/entities/vectorindex/config.go +++ /dev/null @@ -1,43 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorindex - -import ( - "fmt" - - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/vectorindex/flat" - "github.com/weaviate/weaviate/entities/vectorindex/hnsw" -) - -const ( - DefaultVectorIndexType = VectorIndexTypeHNSW - VectorIndexTypeHNSW = "hnsw" - VectorIndexTypeFLAT = "flat" -) - -// ParseAndValidateConfig from an unknown input value, as this is not further -// specified in the API to allow of exchanging the index type -func ParseAndValidateConfig(input interface{}, vectorIndexType string) (schema.VectorIndexConfig, error) { - if len(vectorIndexType) == 0 { - vectorIndexType = DefaultVectorIndexType - } - - switch vectorIndexType { - case VectorIndexTypeHNSW: - return hnsw.ParseAndValidateConfig(input) - case VectorIndexTypeFLAT: - return flat.ParseAndValidateConfig(input) - default: - return nil, fmt.Errorf("Invalid vectorIndexType (­%s). Supported types are hnsw and flat", vectorIndexType) - } -} diff --git a/entities/vectorindex/flat/config.go b/entities/vectorindex/flat/config.go deleted file mode 100644 index 017fb6d845d87002e93c9f7a990de648e363f149..0000000000000000000000000000000000000000 --- a/entities/vectorindex/flat/config.go +++ /dev/null @@ -1,173 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package flat - -import ( - "errors" - "fmt" - - "github.com/weaviate/weaviate/entities/schema" - vectorindexcommon "github.com/weaviate/weaviate/entities/vectorindex/common" -) - -const ( - DefaultVectorCache = false - DefaultVectorCacheMaxObjects = 1e12 - DefaultCompressionEnabled = false - DefaultCompressionRescore = -1 // indicates "let Weaviate pick" -) - -type CompressionUserConfig struct { - Enabled bool `json:"enabled"` - RescoreLimit int `json:"rescoreLimit"` - Cache bool `json:"cache"` -} - -type UserConfig struct { - Distance string `json:"distance"` - VectorCacheMaxObjects int `json:"vectorCacheMaxObjects"` - PQ CompressionUserConfig `json:"pq"` - BQ CompressionUserConfig `json:"bq"` -} - -// IndexType returns the type of the underlying vector index, thus making sure -// the schema.VectorIndexConfig interface is implemented -func (u UserConfig) IndexType() string { - return "flat" -} - -func (u UserConfig) DistanceName() string { - return u.Distance -} - -// SetDefaults in the user-specifyable part of the config -func (u *UserConfig) SetDefaults() { - u.PQ.Cache = DefaultVectorCache - u.BQ.Cache = DefaultVectorCache - u.VectorCacheMaxObjects = DefaultVectorCacheMaxObjects - u.Distance = vectorindexcommon.DefaultDistanceMetric - u.PQ.Enabled = DefaultCompressionEnabled - u.PQ.RescoreLimit = DefaultCompressionRescore - u.BQ.Enabled = DefaultCompressionEnabled - u.BQ.RescoreLimit = DefaultCompressionRescore -} - -// ParseAndValidateConfig from an unknown input value, as this is not further -// specified in the API to allow of exchanging the index type -func ParseAndValidateConfig(input interface{}) (schema.VectorIndexConfig, error) { - uc := UserConfig{} - uc.SetDefaults() - - if input == nil { - return uc, nil - } - - asMap, ok := input.(map[string]interface{}) - if !ok || asMap == nil { - return uc, fmt.Errorf("input must be a non-nil map") - } - - if err := vectorindexcommon.OptionalStringFromMap(asMap, "distance", func(v string) { - uc.Distance = v - }); err != nil { - return uc, err - } - - if err := vectorindexcommon.OptionalIntFromMap(asMap, "vectorCacheMaxObjects", func(v int) { - uc.VectorCacheMaxObjects = v - }); err != nil { - return uc, err - } - - if err := parseCompressionMap(asMap, &uc); err != nil { - return uc, err - } - - return uc, nil -} - -func parseCompressionMap(in map[string]interface{}, uc *UserConfig) error { - pqConfigValue, pqOk := in["pq"] - bqConfigValue, bqOk := in["bq"] - if !pqOk && !bqOk { - return nil - } - - if pqOk { - pqConfigMap, ok := pqConfigValue.(map[string]interface{}) - if ok { - if err := vectorindexcommon.OptionalBoolFromMap(pqConfigMap, "enabled", func(v bool) { - uc.PQ.Enabled = v - }); err != nil { - return err - } - - if err := vectorindexcommon.OptionalBoolFromMap(pqConfigMap, "cache", func(v bool) { - uc.PQ.Cache = v - }); err != nil { - return err - } - - if err := vectorindexcommon.OptionalIntFromMap(pqConfigMap, "rescoreLimit", func(v int) { - uc.PQ.RescoreLimit = v - }); err != nil { - return err - } - } - } - - if bqOk { - bqConfigMap, ok := bqConfigValue.(map[string]interface{}) - if !ok { - return nil - } - - if err := vectorindexcommon.OptionalBoolFromMap(bqConfigMap, "enabled", func(v bool) { - uc.BQ.Enabled = v - }); err != nil { - return err - } - - if err := vectorindexcommon.OptionalBoolFromMap(bqConfigMap, "cache", func(v bool) { - uc.BQ.Cache = v - }); err != nil { - return err - } - - if err := vectorindexcommon.OptionalIntFromMap(bqConfigMap, "rescoreLimit", func(v int) { - uc.BQ.RescoreLimit = v - }); err != nil { - return err - } - - } - // TODO: remove once PQ is supported - if uc.PQ.Enabled { - return errors.New("PQ is not currently supported for flat indices") - } - if uc.PQ.Cache && !uc.PQ.Enabled { - return errors.New("not possible to use the cache without compression") - } - if uc.BQ.Cache && !uc.BQ.Enabled { - return errors.New("not possible to use the cache without compression") - } - if uc.PQ.Enabled && uc.BQ.Enabled { - return errors.New("cannot activate dual compression. Select either PQ or BQ please") - } - return nil -} - -func NewDefaultUserConfig() UserConfig { - uc := UserConfig{} - uc.SetDefaults() - return uc -} diff --git a/entities/vectorindex/flat/config_test.go b/entities/vectorindex/flat/config_test.go deleted file mode 100644 index a1666b51ec148dd56ca1931f9649112d5ff9dee9..0000000000000000000000000000000000000000 --- a/entities/vectorindex/flat/config_test.go +++ /dev/null @@ -1,119 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package flat - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/vectorindex/common" -) - -func Test_FlatUserConfig(t *testing.T) { - type test struct { - name string - input interface{} - expected UserConfig - expectErr bool - expectErrMsg string - } - - tests := []test{ - { - name: "nothing specified, all defaults", - input: nil, - expected: UserConfig{ - VectorCacheMaxObjects: common.DefaultVectorCacheMaxObjects, - Distance: common.DefaultDistanceMetric, - PQ: CompressionUserConfig{ - Enabled: DefaultCompressionEnabled, - RescoreLimit: DefaultCompressionRescore, - Cache: DefaultVectorCache, - }, - BQ: CompressionUserConfig{ - Enabled: DefaultCompressionEnabled, - RescoreLimit: DefaultCompressionRescore, - Cache: DefaultVectorCache, - }, - }, - }, - { - name: "bq enabled", - input: map[string]interface{}{ - "vectorCacheMaxObjects": float64(100), - "distance": "cosine", - "bq": map[string]interface{}{ - "enabled": true, - "rescoreLimit": float64(100), - "cache": true, - }, - }, - expected: UserConfig{ - VectorCacheMaxObjects: 100, - Distance: common.DefaultDistanceMetric, - PQ: CompressionUserConfig{ - Enabled: false, - RescoreLimit: DefaultCompressionRescore, - Cache: DefaultVectorCache, - }, - BQ: CompressionUserConfig{ - Enabled: true, - RescoreLimit: 100, - Cache: true, - }, - }, - }, - { - name: "pq enabled", - input: map[string]interface{}{ - "vectorCacheMaxObjects": float64(100), - "distance": "cosine", - "pq": map[string]interface{}{ - "enabled": true, - "rescoreLimit": float64(100), - "cache": true, - }, - }, - expectErr: true, - expectErrMsg: "PQ is not currently supported for flat indices", - // expected: UserConfig{ - // VectorCacheMaxObjects: 100, - // Distance: common.DefaultDistanceMetric, - // PQ: CompressionUserConfig{ - // Enabled: true, - // RescoreLimit: 100, - // Cache: true, - // }, - // BQ: CompressionUserConfig{ - // Enabled: false, - // RescoreLimit: DefaultCompressionRescore, - // Cache: DefaultVectorCache, - // }, - // }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - cfg, err := ParseAndValidateConfig(test.input) - if test.expectErr { - require.NotNil(t, err) - assert.Contains(t, err.Error(), test.expectErrMsg) - return - } else { - assert.Nil(t, err) - assert.Equal(t, test.expected, cfg) - } - }) - } -} diff --git a/entities/vectorindex/hnsw/bq_config.go b/entities/vectorindex/hnsw/bq_config.go deleted file mode 100644 index 4ab20ce48612d9cafbb4e98d36d1792ba280fd61..0000000000000000000000000000000000000000 --- a/entities/vectorindex/hnsw/bq_config.go +++ /dev/null @@ -1,42 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import "github.com/weaviate/weaviate/entities/vectorindex/common" - -const ( - DefaultBQEnabled = false -) - -type BQConfig struct { - Enabled bool `json:"enabled"` -} - -func parseBQMap(in map[string]interface{}, bq *BQConfig) error { - bqConfigValue, ok := in["bq"] - if !ok { - return nil - } - - bqConfigMap, ok := bqConfigValue.(map[string]interface{}) - if !ok { - return nil - } - - if err := common.OptionalBoolFromMap(bqConfigMap, "enabled", func(v bool) { - bq.Enabled = v - }); err != nil { - return err - } - - return nil -} diff --git a/entities/vectorindex/hnsw/config.go b/entities/vectorindex/hnsw/config.go deleted file mode 100644 index b1b93ff7604b3d7ee45d8514b348912f0f837f10..0000000000000000000000000000000000000000 --- a/entities/vectorindex/hnsw/config.go +++ /dev/null @@ -1,220 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "fmt" - "strings" - - vectorIndexCommon "github.com/weaviate/weaviate/entities/vectorindex/common" - - "github.com/weaviate/weaviate/entities/schema" -) - -const ( - // Set these defaults if the user leaves them blank - DefaultCleanupIntervalSeconds = 5 * 60 - DefaultMaxConnections = 64 - DefaultEFConstruction = 128 - DefaultEF = -1 // indicates "let Weaviate pick" - DefaultDynamicEFMin = 100 - DefaultDynamicEFMax = 500 - DefaultDynamicEFFactor = 8 - DefaultSkip = false - DefaultFlatSearchCutoff = 40000 - - // Fail validation if those criteria are not met - MinmumMaxConnections = 4 - MinmumEFConstruction = 4 -) - -// UserConfig bundles all values settable by a user in the per-class settings -type UserConfig struct { - Skip bool `json:"skip"` - CleanupIntervalSeconds int `json:"cleanupIntervalSeconds"` - MaxConnections int `json:"maxConnections"` - EFConstruction int `json:"efConstruction"` - EF int `json:"ef"` - DynamicEFMin int `json:"dynamicEfMin"` - DynamicEFMax int `json:"dynamicEfMax"` - DynamicEFFactor int `json:"dynamicEfFactor"` - VectorCacheMaxObjects int `json:"vectorCacheMaxObjects"` - FlatSearchCutoff int `json:"flatSearchCutoff"` - Distance string `json:"distance"` - PQ PQConfig `json:"pq"` - BQ BQConfig `json:"bq"` -} - -// IndexType returns the type of the underlying vector index, thus making sure -// the schema.VectorIndexConfig interface is implemented -func (u UserConfig) IndexType() string { - return "hnsw" -} - -func (u UserConfig) DistanceName() string { - return u.Distance -} - -// SetDefaults in the user-specifyable part of the config -func (u *UserConfig) SetDefaults() { - u.MaxConnections = DefaultMaxConnections - u.EFConstruction = DefaultEFConstruction - u.CleanupIntervalSeconds = DefaultCleanupIntervalSeconds - u.VectorCacheMaxObjects = vectorIndexCommon.DefaultVectorCacheMaxObjects - u.EF = DefaultEF - u.DynamicEFFactor = DefaultDynamicEFFactor - u.DynamicEFMax = DefaultDynamicEFMax - u.DynamicEFMin = DefaultDynamicEFMin - u.Skip = DefaultSkip - u.FlatSearchCutoff = DefaultFlatSearchCutoff - u.Distance = vectorIndexCommon.DefaultDistanceMetric - u.PQ = PQConfig{ - Enabled: DefaultPQEnabled, - BitCompression: DefaultPQBitCompression, - Segments: DefaultPQSegments, - Centroids: DefaultPQCentroids, - TrainingLimit: DefaultPQTrainingLimit, - Encoder: PQEncoder{ - Type: DefaultPQEncoderType, - Distribution: DefaultPQEncoderDistribution, - }, - } - u.BQ = BQConfig{ - Enabled: DefaultBQEnabled, - } -} - -// ParseAndValidateConfig from an unknown input value, as this is not further -// specified in the API to allow of exchanging the index type -func ParseAndValidateConfig(input interface{}) (schema.VectorIndexConfig, error) { - uc := UserConfig{} - uc.SetDefaults() - - if input == nil { - return uc, nil - } - - asMap, ok := input.(map[string]interface{}) - if !ok || asMap == nil { - return uc, fmt.Errorf("input must be a non-nil map") - } - - if err := vectorIndexCommon.OptionalIntFromMap(asMap, "maxConnections", func(v int) { - uc.MaxConnections = v - }); err != nil { - return uc, err - } - - if err := vectorIndexCommon.OptionalIntFromMap(asMap, "cleanupIntervalSeconds", func(v int) { - uc.CleanupIntervalSeconds = v - }); err != nil { - return uc, err - } - - if err := vectorIndexCommon.OptionalIntFromMap(asMap, "efConstruction", func(v int) { - uc.EFConstruction = v - }); err != nil { - return uc, err - } - - if err := vectorIndexCommon.OptionalIntFromMap(asMap, "ef", func(v int) { - uc.EF = v - }); err != nil { - return uc, err - } - - if err := vectorIndexCommon.OptionalIntFromMap(asMap, "dynamicEfFactor", func(v int) { - uc.DynamicEFFactor = v - }); err != nil { - return uc, err - } - - if err := vectorIndexCommon.OptionalIntFromMap(asMap, "dynamicEfMax", func(v int) { - uc.DynamicEFMax = v - }); err != nil { - return uc, err - } - - if err := vectorIndexCommon.OptionalIntFromMap(asMap, "dynamicEfMin", func(v int) { - uc.DynamicEFMin = v - }); err != nil { - return uc, err - } - - if err := vectorIndexCommon.OptionalIntFromMap(asMap, "vectorCacheMaxObjects", func(v int) { - uc.VectorCacheMaxObjects = v - }); err != nil { - return uc, err - } - - if err := vectorIndexCommon.OptionalIntFromMap(asMap, "flatSearchCutoff", func(v int) { - uc.FlatSearchCutoff = v - }); err != nil { - return uc, err - } - - if err := vectorIndexCommon.OptionalBoolFromMap(asMap, "skip", func(v bool) { - uc.Skip = v - }); err != nil { - return uc, err - } - - if err := vectorIndexCommon.OptionalStringFromMap(asMap, "distance", func(v string) { - uc.Distance = v - }); err != nil { - return uc, err - } - - if err := parsePQMap(asMap, &uc.PQ); err != nil { - return uc, err - } - - if err := parseBQMap(asMap, &uc.BQ); err != nil { - return uc, err - } - - return uc, uc.validate() -} - -func (u *UserConfig) validate() error { - var errMsgs []string - if u.MaxConnections < MinmumMaxConnections { - errMsgs = append(errMsgs, fmt.Sprintf( - "maxConnections must be a positive integer with a minimum of %d", - MinmumMaxConnections, - )) - } - - if u.EFConstruction < MinmumEFConstruction { - errMsgs = append(errMsgs, fmt.Sprintf( - "efConstruction must be a positive integer with a minimum of %d", - MinmumMaxConnections, - )) - } - - if len(errMsgs) > 0 { - return fmt.Errorf("invalid hnsw config: %s", - strings.Join(errMsgs, ", ")) - } - - if u.PQ.Enabled && u.BQ.Enabled { - return fmt.Errorf("invalid hnsw config: two compression methods enabled: PQ and BQ") - } - - return nil -} - -func NewDefaultUserConfig() UserConfig { - uc := UserConfig{} - uc.SetDefaults() - return uc -} diff --git a/entities/vectorindex/hnsw/config_test.go b/entities/vectorindex/hnsw/config_test.go deleted file mode 100644 index 472d6d27a718bc6b79de1d2b37e2689c6e10ee44..0000000000000000000000000000000000000000 --- a/entities/vectorindex/hnsw/config_test.go +++ /dev/null @@ -1,525 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "encoding/json" - "math" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/vectorindex/common" -) - -func Test_UserConfig(t *testing.T) { - type test struct { - name string - input interface{} - expected UserConfig - expectErr bool - expectErrMsg string - } - - tests := []test{ - { - name: "nothing specified, all defaults", - input: nil, - expected: UserConfig{ - CleanupIntervalSeconds: DefaultCleanupIntervalSeconds, - MaxConnections: DefaultMaxConnections, - EFConstruction: DefaultEFConstruction, - VectorCacheMaxObjects: common.DefaultVectorCacheMaxObjects, - EF: DefaultEF, - Skip: DefaultSkip, - FlatSearchCutoff: DefaultFlatSearchCutoff, - DynamicEFMin: DefaultDynamicEFMin, - DynamicEFMax: DefaultDynamicEFMax, - DynamicEFFactor: DefaultDynamicEFFactor, - Distance: common.DefaultDistanceMetric, - PQ: PQConfig{ - Enabled: DefaultPQEnabled, - BitCompression: DefaultPQBitCompression, - Segments: DefaultPQSegments, - Centroids: DefaultPQCentroids, - TrainingLimit: DefaultPQTrainingLimit, - Encoder: PQEncoder{ - Type: DefaultPQEncoderType, - Distribution: DefaultPQEncoderDistribution, - }, - }, - }, - }, - - { - name: "with maximum connections", - input: map[string]interface{}{ - "maxConnections": json.Number("100"), - }, - expected: UserConfig{ - CleanupIntervalSeconds: DefaultCleanupIntervalSeconds, - MaxConnections: 100, - EFConstruction: DefaultEFConstruction, - VectorCacheMaxObjects: common.DefaultVectorCacheMaxObjects, - EF: DefaultEF, - FlatSearchCutoff: DefaultFlatSearchCutoff, - DynamicEFMin: DefaultDynamicEFMin, - DynamicEFMax: DefaultDynamicEFMax, - DynamicEFFactor: DefaultDynamicEFFactor, - Distance: common.DefaultDistanceMetric, - PQ: PQConfig{ - Enabled: DefaultPQEnabled, - BitCompression: DefaultPQBitCompression, - Segments: DefaultPQSegments, - Centroids: DefaultPQCentroids, - TrainingLimit: DefaultPQTrainingLimit, - Encoder: PQEncoder{ - Type: DefaultPQEncoderType, - Distribution: DefaultPQEncoderDistribution, - }, - }, - }, - }, - - { - name: "with all optional fields", - input: map[string]interface{}{ - "cleanupIntervalSeconds": json.Number("11"), - "maxConnections": json.Number("12"), - "efConstruction": json.Number("13"), - "vectorCacheMaxObjects": json.Number("14"), - "ef": json.Number("15"), - "flatSearchCutoff": json.Number("16"), - "dynamicEfMin": json.Number("17"), - "dynamicEfMax": json.Number("18"), - "dynamicEfFactor": json.Number("19"), - "skip": true, - "distance": "l2-squared", - }, - expected: UserConfig{ - CleanupIntervalSeconds: 11, - MaxConnections: 12, - EFConstruction: 13, - VectorCacheMaxObjects: 14, - EF: 15, - FlatSearchCutoff: 16, - DynamicEFMin: 17, - DynamicEFMax: 18, - DynamicEFFactor: 19, - Skip: true, - Distance: "l2-squared", - PQ: PQConfig{ - Enabled: DefaultPQEnabled, - BitCompression: DefaultPQBitCompression, - Segments: DefaultPQSegments, - Centroids: DefaultPQCentroids, - TrainingLimit: DefaultPQTrainingLimit, - Encoder: PQEncoder{ - Type: DefaultPQEncoderType, - Distribution: DefaultPQEncoderDistribution, - }, - }, - }, - }, - - { - name: "with all optional fields", - input: map[string]interface{}{ - "cleanupIntervalSeconds": json.Number("11"), - "maxConnections": json.Number("12"), - "efConstruction": json.Number("13"), - "vectorCacheMaxObjects": json.Number("14"), - "ef": json.Number("15"), - "flatSearchCutoff": json.Number("16"), - "dynamicEfMin": json.Number("17"), - "dynamicEfMax": json.Number("18"), - "dynamicEfFactor": json.Number("19"), - "skip": true, - "distance": "manhattan", - }, - expected: UserConfig{ - CleanupIntervalSeconds: 11, - MaxConnections: 12, - EFConstruction: 13, - VectorCacheMaxObjects: 14, - EF: 15, - FlatSearchCutoff: 16, - DynamicEFMin: 17, - DynamicEFMax: 18, - DynamicEFFactor: 19, - Skip: true, - Distance: "manhattan", - PQ: PQConfig{ - Enabled: DefaultPQEnabled, - BitCompression: DefaultPQBitCompression, - Segments: DefaultPQSegments, - Centroids: DefaultPQCentroids, - TrainingLimit: DefaultPQTrainingLimit, - Encoder: PQEncoder{ - Type: DefaultPQEncoderType, - Distribution: DefaultPQEncoderDistribution, - }, - }, - }, - }, - - { - name: "with all optional fields", - input: map[string]interface{}{ - "cleanupIntervalSeconds": json.Number("11"), - "maxConnections": json.Number("12"), - "efConstruction": json.Number("13"), - "vectorCacheMaxObjects": json.Number("14"), - "ef": json.Number("15"), - "flatSearchCutoff": json.Number("16"), - "dynamicEfMin": json.Number("17"), - "dynamicEfMax": json.Number("18"), - "dynamicEfFactor": json.Number("19"), - "skip": true, - "distance": "hamming", - }, - expected: UserConfig{ - CleanupIntervalSeconds: 11, - MaxConnections: 12, - EFConstruction: 13, - VectorCacheMaxObjects: 14, - EF: 15, - FlatSearchCutoff: 16, - DynamicEFMin: 17, - DynamicEFMax: 18, - DynamicEFFactor: 19, - Skip: true, - Distance: "hamming", - PQ: PQConfig{ - Enabled: DefaultPQEnabled, - BitCompression: DefaultPQBitCompression, - Segments: DefaultPQSegments, - Centroids: DefaultPQCentroids, - TrainingLimit: DefaultPQTrainingLimit, - Encoder: PQEncoder{ - Type: DefaultPQEncoderType, - Distribution: DefaultPQEncoderDistribution, - }, - }, - }, - }, - - { - // opposed to from the API - name: "with raw data as floats", - input: map[string]interface{}{ - "cleanupIntervalSeconds": float64(11), - "maxConnections": float64(12), - "efConstruction": float64(13), - "vectorCacheMaxObjects": float64(14), - "ef": float64(15), - "flatSearchCutoff": float64(16), - "dynamicEfMin": float64(17), - "dynamicEfMax": float64(18), - "dynamicEfFactor": float64(19), - }, - expected: UserConfig{ - CleanupIntervalSeconds: 11, - MaxConnections: 12, - EFConstruction: 13, - VectorCacheMaxObjects: 14, - EF: 15, - FlatSearchCutoff: 16, - DynamicEFMin: 17, - DynamicEFMax: 18, - DynamicEFFactor: 19, - Distance: common.DefaultDistanceMetric, - PQ: PQConfig{ - Enabled: DefaultPQEnabled, - BitCompression: DefaultPQBitCompression, - Segments: DefaultPQSegments, - Centroids: DefaultPQCentroids, - TrainingLimit: DefaultPQTrainingLimit, - Encoder: PQEncoder{ - Type: DefaultPQEncoderType, - Distribution: DefaultPQEncoderDistribution, - }, - }, - }, - }, - - { - name: "with pq tile normal encoder", - input: map[string]interface{}{ - "cleanupIntervalSeconds": float64(11), - "maxConnections": float64(12), - "efConstruction": float64(13), - "vectorCacheMaxObjects": float64(14), - "ef": float64(15), - "flatSearchCutoff": float64(16), - "dynamicEfMin": float64(17), - "dynamicEfMax": float64(18), - "dynamicEfFactor": float64(19), - "pq": map[string]interface{}{ - "enabled": true, - "bitCompression": false, - "segments": float64(64), - "centroids": float64(DefaultPQCentroids), - "trainingLimit": float64(DefaultPQTrainingLimit), - "encoder": map[string]interface{}{ - "type": "tile", - "distribution": "normal", - }, - }, - }, - expected: UserConfig{ - CleanupIntervalSeconds: 11, - MaxConnections: 12, - EFConstruction: 13, - VectorCacheMaxObjects: 14, - EF: 15, - FlatSearchCutoff: 16, - DynamicEFMin: 17, - DynamicEFMax: 18, - DynamicEFFactor: 19, - Distance: common.DefaultDistanceMetric, - PQ: PQConfig{ - Enabled: true, - Segments: 64, - Centroids: DefaultPQCentroids, - TrainingLimit: DefaultPQTrainingLimit, - Encoder: PQEncoder{ - Type: "tile", - Distribution: "normal", - }, - }, - }, - }, - - { - name: "with pq kmeans normal encoder", - input: map[string]interface{}{ - "cleanupIntervalSeconds": float64(11), - "maxConnections": float64(12), - "efConstruction": float64(13), - "vectorCacheMaxObjects": float64(14), - "ef": float64(15), - "flatSearchCutoff": float64(16), - "dynamicEfMin": float64(17), - "dynamicEfMax": float64(18), - "dynamicEfFactor": float64(19), - "pq": map[string]interface{}{ - "enabled": true, - "bitCompression": false, - "segments": float64(64), - "centroids": float64(DefaultPQCentroids), - "trainingLimit": float64(DefaultPQTrainingLimit), - "encoder": map[string]interface{}{ - "type": PQEncoderTypeKMeans, - }, - }, - }, - expected: UserConfig{ - CleanupIntervalSeconds: 11, - MaxConnections: 12, - EFConstruction: 13, - VectorCacheMaxObjects: 14, - EF: 15, - FlatSearchCutoff: 16, - DynamicEFMin: 17, - DynamicEFMax: 18, - DynamicEFFactor: 19, - Distance: common.DefaultDistanceMetric, - PQ: PQConfig{ - Enabled: true, - Segments: 64, - Centroids: DefaultPQCentroids, - TrainingLimit: DefaultPQTrainingLimit, - Encoder: PQEncoder{ - Type: DefaultPQEncoderType, - Distribution: DefaultPQEncoderDistribution, - }, - }, - }, - }, - - { - name: "with invalid encoder", - input: map[string]interface{}{ - "pq": map[string]interface{}{ - "enabled": true, - "encoder": map[string]interface{}{ - "type": "bernoulli", - }, - }, - }, - expectErr: true, - expectErrMsg: "invalid encoder type bernoulli", - }, - - { - name: "with invalid distribution", - input: map[string]interface{}{ - "pq": map[string]interface{}{ - "enabled": true, - "encoder": map[string]interface{}{ - "distribution": "lognormal", - }, - }, - }, - expectErr: true, - expectErrMsg: "invalid encoder distribution lognormal", - }, - - { - // opposed to from the API - name: "with rounded vectorCacheMaxObjects that would otherwise overflow", - input: map[string]interface{}{ - "cleanupIntervalSeconds": json.Number("11"), - "maxConnections": json.Number("12"), - "efConstruction": json.Number("13"), - "vectorCacheMaxObjects": json.Number("9223372036854776000"), - "ef": json.Number("15"), - "flatSearchCutoff": json.Number("16"), - "dynamicEfMin": json.Number("17"), - "dynamicEfMax": json.Number("18"), - "dynamicEfFactor": json.Number("19"), - }, - expected: UserConfig{ - CleanupIntervalSeconds: 11, - MaxConnections: 12, - EFConstruction: 13, - VectorCacheMaxObjects: math.MaxInt64, - EF: 15, - FlatSearchCutoff: 16, - DynamicEFMin: 17, - DynamicEFMax: 18, - DynamicEFFactor: 19, - Distance: common.DefaultDistanceMetric, - PQ: PQConfig{ - Enabled: DefaultPQEnabled, - BitCompression: DefaultPQBitCompression, - Segments: DefaultPQSegments, - Centroids: DefaultPQCentroids, - TrainingLimit: DefaultPQTrainingLimit, - Encoder: PQEncoder{ - Type: DefaultPQEncoderType, - Distribution: DefaultPQEncoderDistribution, - }, - }, - }, - }, - { - name: "invalid max connections (json)", - input: map[string]interface{}{ - "maxConnections": json.Number("0"), - }, - expectErr: true, - expectErrMsg: "maxConnections must be a positive integer " + - "with a minimum of 4", - }, - { - name: "invalid max connections (float)", - input: map[string]interface{}{ - "maxConnections": float64(3), - }, - expectErr: true, - expectErrMsg: "maxConnections must be a positive integer " + - "with a minimum of 4", - }, - { - name: "invalid efConstruction (json)", - input: map[string]interface{}{ - "efConstruction": json.Number("0"), - }, - expectErr: true, - expectErrMsg: "efConstruction must be a positive integer " + - "with a minimum of 4", - }, - { - name: "invalid efConstruction (float)", - input: map[string]interface{}{ - "efConstruction": float64(3), - }, - expectErr: true, - expectErrMsg: "efConstruction must be a positive integer " + - "with a minimum of 4", - }, - { - name: "with bq", - input: map[string]interface{}{ - "cleanupIntervalSeconds": float64(11), - "maxConnections": float64(12), - "efConstruction": float64(13), - "vectorCacheMaxObjects": float64(14), - "ef": float64(15), - "flatSearchCutoff": float64(16), - "dynamicEfMin": float64(17), - "dynamicEfMax": float64(18), - "dynamicEfFactor": float64(19), - "bq": map[string]interface{}{ - "enabled": true, - }, - }, - expected: UserConfig{ - CleanupIntervalSeconds: 11, - MaxConnections: 12, - EFConstruction: 13, - VectorCacheMaxObjects: 14, - EF: 15, - FlatSearchCutoff: 16, - DynamicEFMin: 17, - DynamicEFMax: 18, - DynamicEFFactor: 19, - Distance: common.DefaultDistanceMetric, - PQ: PQConfig{ - Enabled: false, - Segments: 0, - Centroids: DefaultPQCentroids, - TrainingLimit: DefaultPQTrainingLimit, - Encoder: PQEncoder{ - Type: DefaultPQEncoderType, - Distribution: DefaultPQEncoderDistribution, - }, - }, - BQ: BQConfig{ - Enabled: true, - }, - }, - }, - { - name: "with invalid compression", - input: map[string]interface{}{ - "pq": map[string]interface{}{ - "enabled": true, - "encoder": map[string]interface{}{ - "type": "kmeans", - }, - }, - "bq": map[string]interface{}{ - "enabled": true, - }, - }, - expectErr: true, - expectErrMsg: "invalid hnsw config: two compression methods enabled: PQ and BQ", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - cfg, err := ParseAndValidateConfig(test.input) - if test.expectErr { - require.NotNil(t, err) - assert.Contains(t, err.Error(), test.expectErrMsg) - return - } else { - assert.Nil(t, err) - assert.Equal(t, test.expected, cfg) - } - }) - } -} diff --git a/entities/vectorindex/hnsw/pq_config.go b/entities/vectorindex/hnsw/pq_config.go deleted file mode 100644 index e20714855042cf2d10af2bf2a0bc23be158e7513..0000000000000000000000000000000000000000 --- a/entities/vectorindex/hnsw/pq_config.go +++ /dev/null @@ -1,196 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package hnsw - -import ( - "fmt" - - "github.com/weaviate/weaviate/entities/vectorindex/common" -) - -const ( - PQEncoderTypeKMeans = "kmeans" - PQEncoderTypeTile = "tile" - PQEncoderDistributionLogNormal = "log-normal" - PQEncoderDistributionNormal = "normal" -) - -const ( - DefaultPQEnabled = false - DefaultPQBitCompression = false - DefaultPQSegments = 0 - DefaultPQEncoderType = PQEncoderTypeKMeans - DefaultPQEncoderDistribution = PQEncoderDistributionLogNormal - DefaultPQCentroids = 256 - DefaultPQTrainingLimit = 100000 -) - -// Product Quantization encoder configuration -type PQEncoder struct { - Type string `json:"type"` - Distribution string `json:"distribution,omitempty"` -} - -// Product Quantization configuration -type PQConfig struct { - Enabled bool `json:"enabled"` - BitCompression bool `json:"bitCompression"` - Segments int `json:"segments"` - Centroids int `json:"centroids"` - TrainingLimit int `json:"trainingLimit"` - Encoder PQEncoder `json:"encoder"` -} - -func validEncoder(v string) error { - switch v { - case PQEncoderTypeKMeans: - case PQEncoderTypeTile: - default: - return fmt.Errorf("invalid encoder type %s", v) - } - - return nil -} - -func validEncoderDistribution(v string) error { - switch v { - case PQEncoderDistributionLogNormal: - case PQEncoderDistributionNormal: - default: - return fmt.Errorf("invalid encoder distribution %s", v) - } - - return nil -} - -func ValidatePQConfig(cfg PQConfig) error { - if !cfg.Enabled { - return nil - } - err := validEncoder(cfg.Encoder.Type) - if err != nil { - return err - } - - err = validEncoderDistribution(cfg.Encoder.Distribution) - if err != nil { - return err - } - - return nil -} - -func encoderFromMap(in map[string]interface{}, setFn func(v string)) error { - value, ok := in["type"] - if !ok { - return nil - } - - asString, ok := value.(string) - if !ok { - return nil - } - - err := validEncoder(asString) - if err != nil { - return err - } - - setFn(asString) - return nil -} - -func encoderDistributionFromMap(in map[string]interface{}, setFn func(v string)) error { - value, ok := in["distribution"] - if !ok { - return nil - } - - asString, ok := value.(string) - if !ok { - return nil - } - - err := validEncoderDistribution(asString) - if err != nil { - return err - } - - setFn(asString) - return nil -} - -func parsePQMap(in map[string]interface{}, pq *PQConfig) error { - pqConfigValue, ok := in["pq"] - if !ok { - return nil - } - - pqConfigMap, ok := pqConfigValue.(map[string]interface{}) - if !ok { - return nil - } - - if err := common.OptionalBoolFromMap(pqConfigMap, "enabled", func(v bool) { - pq.Enabled = v - }); err != nil { - return err - } - - if err := common.OptionalBoolFromMap(pqConfigMap, "bitCompression", func(v bool) { - pq.BitCompression = v - }); err != nil { - return err - } - - if err := common.OptionalIntFromMap(pqConfigMap, "segments", func(v int) { - pq.Segments = v - }); err != nil { - return err - } - - if err := common.OptionalIntFromMap(pqConfigMap, "centroids", func(v int) { - pq.Centroids = v - }); err != nil { - return err - } - - if err := common.OptionalIntFromMap(pqConfigMap, "trainingLimit", func(v int) { - pq.TrainingLimit = v - }); err != nil { - return err - } - - pqEncoderValue, ok := pqConfigMap["encoder"] - if !ok { - return nil - } - - pqEncoderMap, ok := pqEncoderValue.(map[string]interface{}) - if !ok { - return nil - } - - if err := encoderFromMap(pqEncoderMap, func(v string) { - pq.Encoder.Type = v - }); err != nil { - return err - } - - if err := encoderDistributionFromMap(pqEncoderMap, func(v string) { - pq.Encoder.Distribution = v - }); err != nil { - return err - } - - return nil -} diff --git a/entities/verbosity/verbosity.go b/entities/verbosity/verbosity.go deleted file mode 100644 index 3aa3ce13f450872cf1c35f418f62226e9f7c0c6d..0000000000000000000000000000000000000000 --- a/entities/verbosity/verbosity.go +++ /dev/null @@ -1,34 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package verbosity - -import "fmt" - -const ( - OutputMinimal = "minimal" - OutputVerbose = "verbose" -) - -// ParseOutput extracts the verbosity value from the provided nullable string -// If `output` is nil, the default selection is "minimal" -func ParseOutput(output *string) (string, error) { - if output != nil { - switch *output { - case OutputMinimal, OutputVerbose: - return *output, nil - default: - return "", fmt.Errorf(`invalid output: "%s", possible values are: "%s", "%s"`, - *output, OutputMinimal, OutputVerbose) - } - } - return OutputMinimal, nil -} diff --git a/go.mod b/go.mod deleted file mode 100644 index 6944898cc26d8f7f32cc8d9166f2399dd18ecd25..0000000000000000000000000000000000000000 --- a/go.mod +++ /dev/null @@ -1,159 +0,0 @@ -module github.com/weaviate/weaviate - -require ( - cloud.google.com/go/storage v1.33.0 - github.com/bmatcuk/doublestar v1.1.3 - github.com/buger/jsonparser v1.1.1 - github.com/danaugrs/go-tsne v0.0.0-20200708172100-6b7d1d577fd3 - github.com/davecgh/go-spew v1.1.1 - github.com/docker/go-connections v0.4.0 - github.com/fatih/camelcase v1.0.0 - github.com/go-openapi/errors v0.20.3 - github.com/go-openapi/loads v0.21.1 - github.com/go-openapi/runtime v0.24.2 - github.com/go-openapi/spec v0.20.4 - github.com/go-openapi/strfmt v0.21.3 - github.com/go-openapi/swag v0.22.3 - github.com/go-openapi/validate v0.21.0 - github.com/golang-jwt/jwt/v4 v4.5.0 - github.com/google/uuid v1.3.1 - github.com/hashicorp/memberlist v0.5.0 - github.com/jessevdk/go-flags v1.4.0 - github.com/minio/minio-go/v7 v7.0.63 - github.com/nyaruka/phonenumbers v1.0.54 - github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.17.0 - github.com/rs/cors v1.5.0 - github.com/sirupsen/logrus v1.9.3 - github.com/spaolacci/murmur3 v1.1.0 - github.com/square/go-jose v2.3.0+incompatible - github.com/stretchr/testify v1.8.4 - github.com/testcontainers/testcontainers-go v0.26.0 - github.com/weaviate/contextionary v1.2.1 - github.com/willf/bloom v2.0.3+incompatible - go.etcd.io/bbolt v1.3.8 - golang.org/x/net v0.17.0 - golang.org/x/oauth2 v0.11.0 - golang.org/x/sync v0.3.0 - golang.org/x/sys v0.15.0 - gonum.org/v1/gonum v0.12.0 - google.golang.org/api v0.132.0 - google.golang.org/grpc v1.59.0 - gopkg.in/yaml.v2 v2.4.0 -) - -require ( - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0 - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0 - github.com/KimMachineGun/automemlimit v0.3.0 - github.com/coreos/go-oidc/v3 v3.4.0 - github.com/edsrzf/mmap-go v1.1.0 - github.com/googleapis/gax-go/v2 v2.12.0 - github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 - github.com/pkoukk/tiktoken-go v0.1.6 - github.com/tailor-inc/graphql v0.2.1 - github.com/weaviate/sroar v0.0.0-20230210105426-26108af5465d - golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea - golang.org/x/text v0.14.0 - google.golang.org/protobuf v1.31.0 -) - -require ( - cloud.google.com/go v0.110.7 // indirect - cloud.google.com/go/compute v1.23.0 // indirect - cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v1.1.1 // indirect - dario.cat/mergo v1.0.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect - github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect - github.com/Microsoft/go-winio v0.6.1 // indirect - github.com/Microsoft/hcsshim v0.11.4 // indirect - github.com/PuerkitoBio/purell v1.1.1 // indirect - github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect - github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da // indirect - github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect - github.com/beorn7/perks v1.0.1 // indirect - github.com/cenkalti/backoff/v4 v4.2.1 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/cilium/ebpf v0.11.0 // indirect - github.com/containerd/cgroups/v3 v3.0.2 // indirect - github.com/containerd/containerd v1.7.11 // indirect - github.com/containerd/log v0.1.0 // indirect - github.com/coreos/go-systemd/v22 v22.5.0 // indirect - github.com/cpuguy83/dockercfg v0.3.1 // indirect - github.com/dlclark/regexp2 v1.10.0 // indirect - github.com/docker/distribution v2.8.2+incompatible // indirect - github.com/docker/docker v24.0.7+incompatible // indirect - github.com/docker/go-units v0.5.0 // indirect - github.com/dustin/go-humanize v1.0.1 // indirect - github.com/go-ole/go-ole v1.2.6 // indirect - github.com/go-openapi/analysis v0.21.2 // indirect - github.com/go-openapi/jsonpointer v0.19.5 // indirect - github.com/go-openapi/jsonreference v0.19.6 // indirect - github.com/godbus/dbus/v5 v5.1.0 // indirect - github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.3 // indirect - github.com/google/btree v1.0.0 // indirect - github.com/google/s2a-go v0.1.4 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect - github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/go-immutable-radix v1.0.0 // indirect - github.com/hashicorp/go-msgpack v0.5.3 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect - github.com/hashicorp/go-sockaddr v1.0.0 // indirect - github.com/hashicorp/golang-lru v0.5.1 // indirect - github.com/josharian/intern v1.0.0 // indirect - github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.16.7 // indirect - github.com/klauspost/cpuid/v2 v2.2.5 // indirect - github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect - github.com/magiconair/properties v1.8.7 // indirect - github.com/mailru/easyjson v0.7.7 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect - github.com/miekg/dns v1.1.26 // indirect - github.com/minio/md5-simd v1.1.2 // indirect - github.com/minio/sha256-simd v1.0.1 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/moby/patternmatcher v0.6.0 // indirect - github.com/moby/sys/sequential v0.5.0 // indirect - github.com/moby/term v0.5.0 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/morikuni/aec v1.0.0 // indirect - github.com/oklog/ulid v1.3.1 // indirect - github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.0-rc5 // indirect - github.com/opencontainers/runc v1.1.5 // indirect - github.com/opencontainers/runtime-spec v1.1.0-rc.1 // indirect - github.com/opentracing/opentracing-go v1.2.0 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect - github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect - github.com/prometheus/common v0.44.0 // indirect - github.com/prometheus/procfs v0.11.1 // indirect - github.com/rs/xid v1.5.0 // indirect - github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect - github.com/shirou/gopsutil/v3 v3.23.9 // indirect - github.com/shoenig/go-m1cpu v0.1.6 // indirect - github.com/stretchr/objx v0.5.0 // indirect - github.com/tklauser/go-sysconf v0.3.12 // indirect - github.com/tklauser/numcpus v0.6.1 // indirect - github.com/willf/bitset v1.1.11 // indirect - github.com/yusufpapurcu/wmi v1.2.3 // indirect - go.mongodb.org/mongo-driver v1.11.0 // indirect - go.opencensus.io v0.24.0 // indirect - golang.org/x/crypto v0.17.0 // indirect - golang.org/x/mod v0.11.0 // indirect - golang.org/x/tools v0.10.0 // indirect - golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect - gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/square/go-jose.v2 v2.6.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect -) - -go 1.21 diff --git a/go.sum b/go.sum deleted file mode 100644 index ae2802b9bef088565ba5f7d4f61dd158d3dae21b..0000000000000000000000000000000000000000 --- a/go.sum +++ /dev/null @@ -1,1176 +0,0 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= -cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= -cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= -cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= -cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= -cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= -cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= -cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= -cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= -cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= -cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= -cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= -cloud.google.com/go v0.110.7 h1:rJyC7nWRg2jWGZ4wSJ5nY65GTdYJkg0cd/uXb+ACI6o= -cloud.google.com/go v0.110.7/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= -cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= -cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= -cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= -cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= -cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= -cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= -cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= -cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= -cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= -cloud.google.com/go/iam v1.1.1 h1:lW7fzj15aVIXYHREOqjRBV9PsH0Z6u8Y46a1YGvQP4Y= -cloud.google.com/go/iam v1.1.1/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= -cloud.google.com/go/storage v1.33.0 h1:PVrDOkIC8qQVa1P3SXGpQvfuJhN2LHOoyZvWs8D2X5M= -cloud.google.com/go/storage v1.33.0/go.mod h1:Hhh/dogNRGca7IWv1RC2YqEn0c0G77ctA/OxflYkiD8= -dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= -dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= -github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0 h1:9kDVnTz3vbfweTqAUmk/a/pH5pWFCHtvRpHYC0G/dcA= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0/go.mod h1:3Ug6Qzto9anB6mGlEdgYMDF5zHQ+wwhEaYR4s17PHMw= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 h1:vcYCAze6p19qBW7MhZybIsqD8sMV8js0NyQM8JDnVtg= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0/go.mod h1:OQeznEEkTZ9OrhHJoDD8ZDq51FHgXjqtP9z6bEwBq9U= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0 h1:Ma67P/GGprNwsslzEH6+Kb8nybI8jpDTm4Wmzu2ReK8= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0/go.mod h1:c+Lifp3EDEamAkPVzMooRNOK6CZjNSdEnf1A7jsI9u4= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0 h1:gggzg0SUMs6SQbEw+3LoSsYf9YMjkupeAnHMX8O9mmY= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0/go.mod h1:+6KLcKIVgxoBDMqMO/Nvy7bZ9a0nbU3I1DtFQK3YvB4= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 h1:OBhqkivkhkMqLPymWEppkm7vgPQY2XsHoEkaMQ0AdZY= -github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0/go.mod h1:kgDmCTgBzIEPFElEF+FK0SdjAor06dRq2Go927dnQ6o= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/KimMachineGun/automemlimit v0.3.0 h1:khgwM5ESVN85cE6Bq2ozMAAWDfrOEwQ51D/YlmThE04= -github.com/KimMachineGun/automemlimit v0.3.0/go.mod h1:pJhTW/nWJMj6SnWSU2TEKSlCaM+1N5Mej+IfS/5/Ol0= -github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= -github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= -github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7fz8= -github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYxPRqGcpAD9w= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= -github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/RoaringBitmap/roaring v0.6.1 h1:O36Tdaj1Fi/zyr25shTHwlQPGdq53+u4WkM08AOEjiE= -github.com/RoaringBitmap/roaring v0.6.1/go.mod h1:WZ83fjBF/7uBHi6QoFyfGL4+xuV4Qn+xFkm4+vSzrhE= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= -github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= -github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= -github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bmatcuk/doublestar v1.1.3 h1:S4Ka/fLvUtm+5TqKuByWyuGenBjTP8w+Z/GpQIWB9Yg= -github.com/bmatcuk/doublestar v1.1.3/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE= -github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= -github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= -github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= -github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= -github.com/cilium/ebpf v0.11.0 h1:V8gS/bTCCjX9uUnkUFUpPsksM8n1lXBAvHcpiFk1X2Y= -github.com/cilium/ebpf v0.11.0/go.mod h1:WE7CZAnqOL2RouJ4f1uyNhqr2P4CCvXFIqdRDUgWsVs= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/containerd/cgroups/v3 v3.0.2 h1:f5WFqIVSgo5IZmtTT3qVBo6TzI1ON6sycSBKkymb9L0= -github.com/containerd/cgroups/v3 v3.0.2/go.mod h1:JUgITrzdFqp42uI2ryGA+ge0ap/nxzYgkGmIcetmErE= -github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= -github.com/containerd/containerd v1.7.11 h1:lfGKw3eU35sjV0aG2eYZTiwFEY1pCzxdzicHP3SZILw= -github.com/containerd/containerd v1.7.11/go.mod h1:5UluHxHTX2rdvYuZ5OJTC5m/KJNs0Zs9wVoJm9zf5ZE= -github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= -github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= -github.com/coreos/go-oidc/v3 v3.4.0 h1:xz7elHb/LDwm/ERpwHd+5nb7wFHL32rsr6bBOgaeu6g= -github.com/coreos/go-oidc/v3 v3.4.0/go.mod h1:eHUXhZtXPQLgEaDrOVTgwbgmz1xGOkJNye6h3zkD2Pw= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= -github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/cpuguy83/dockercfg v0.3.1 h1:/FpZ+JaygUR/lZP2NlFI2DVfrOEMAIKP5wWEJdoYe9E= -github.com/cpuguy83/dockercfg v0.3.1/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= -github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= -github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= -github.com/danaugrs/go-tsne v0.0.0-20200708172100-6b7d1d577fd3 h1:4V3w6LD+GOVbkF0jtjAzMRczS18+Gx0/nSZ3Pub3h00= -github.com/danaugrs/go-tsne v0.0.0-20200708172100-6b7d1d577fd3/go.mod h1:tcVxJUGCaPp/YynlqJTfJtGc/LF9vn4WUZSSmaGu3dA= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0= -github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= -github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= -github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= -github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= -github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM= -github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= -github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= -github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= -github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ= -github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= -github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8= -github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= -github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= -github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA= -github.com/frankban/quicktest v1.14.5/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= -github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= -github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/go-openapi/analysis v0.21.2 h1:hXFrOYFHUAMQdu6zwAiKKJHJQ8kqZs1ux/ru1P1wLJU= -github.com/go-openapi/analysis v0.21.2/go.mod h1:HZwRk4RRisyG8vx2Oe6aqeSQcoxRp47Xkp3+K6q+LdY= -github.com/go-openapi/errors v0.19.8/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= -github.com/go-openapi/errors v0.19.9/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= -github.com/go-openapi/errors v0.20.2/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= -github.com/go-openapi/errors v0.20.3 h1:rz6kiC84sqNQoqrtulzaL/VERgkoCyB6WdEkc2ujzUc= -github.com/go-openapi/errors v0.20.3/go.mod h1:Z3FlZ4I8jEGxjUK+bugx3on2mIAk4txuAOhlsB1FSgk= -github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= -github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.19.6 h1:UBIxjkht+AWIgYzCDSv2GN+E/togfwXUJFRTWhl2Jjs= -github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= -github.com/go-openapi/loads v0.21.1 h1:Wb3nVZpdEzDTcly8S4HMkey6fjARRzb7iEaySimlDW0= -github.com/go-openapi/loads v0.21.1/go.mod h1:/DtAMXXneXFjbQMGEtbamCZb+4x7eGwkvZCvBmwUG+g= -github.com/go-openapi/runtime v0.24.2 h1:yX9HMGQbz32M87ECaAhGpJjBmErO3QLcgdZj9BzGx7c= -github.com/go-openapi/runtime v0.24.2/go.mod h1:AKurw9fNre+h3ELZfk6ILsfvPN+bvvlaU/M9q/r9hpk= -github.com/go-openapi/spec v0.20.4 h1:O8hJrt0UMnhHcluhIdUgCLRWyM2x7QkBXRvOs7m+O1M= -github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I= -github.com/go-openapi/strfmt v0.21.0/go.mod h1:ZRQ409bWMj+SOgXofQAGTIo2Ebu72Gs+WaRADcS5iNg= -github.com/go-openapi/strfmt v0.21.1/go.mod h1:I/XVKeLc5+MM5oPNN7P6urMOpuLXEcNrCX/rPGuWb0k= -github.com/go-openapi/strfmt v0.21.2/go.mod h1:I/XVKeLc5+MM5oPNN7P6urMOpuLXEcNrCX/rPGuWb0k= -github.com/go-openapi/strfmt v0.21.3 h1:xwhj5X6CjXEZZHMWy1zKJxvW9AfHC9pkyUjLvHtKG7o= -github.com/go-openapi/strfmt v0.21.3/go.mod h1:k+RzNO0Da+k3FrrynSNN8F7n/peCmQQqbbXjtDfvmGg= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= -github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-openapi/validate v0.21.0 h1:+Wqk39yKOhfpLqNLEC0/eViCkzM5FVXVqrvt526+wcI= -github.com/go-openapi/validate v0.21.0/go.mod h1:rjnrwK57VJ7A8xqfpAOEKRH8yQSGUriMu5/zuPSQ1hg= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= -github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= -github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= -github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= -github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= -github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= -github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= -github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= -github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= -github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= -github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= -github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= -github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= -github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= -github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= -github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= -github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= -github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= -github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= -github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= -github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= -github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= -github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= -github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= -github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= -github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= -github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= -github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= -github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= -github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= -github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= -github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= -github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvkiqTYKBCKLNmlge2eVjoZfySzM= -github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= -github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= -github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= -github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= -github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= -github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= -github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= -github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= -github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= -github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= -github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/memberlist v0.5.0 h1:EtYPN8DpAURiapus508I4n9CzHs2W+8NZGbmmR/prTM= -github.com/hashicorp/memberlist v0.5.0/go.mod h1:yvyXLpo0QaGE59Y7hDTsTzDD25JYBZ4mHgHUZ8lrOI0= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= -github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= -github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= -github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= -github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= -github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= -github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= -github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= -github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/miekg/dns v1.1.26 h1:gPxPSwALAeHJSjarOs00QjVdV9QoBvc1D2ujQUr5BzU= -github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= -github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= -github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.63 h1:GbZ2oCvaUdgT5640WJOpyDhhDxvknAJU2/T3yurwcbQ= -github.com/minio/minio-go/v7 v7.0.63/go.mod h1:Q6X7Qjb7WMhvG65qKf4gUgA5XaiSox74kR1uAEjxRS4= -github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= -github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= -github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= -github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= -github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= -github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= -github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= -github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= -github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= -github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= -github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae h1:VeRdUYdCw49yizlSbMEn2SZ+gT+3IUKx8BqxyQdz+BY= -github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/nyaruka/phonenumbers v1.0.54 h1:vU9IUfiHrpu+lZcCkjEzDsCIdurQV8lxjrAdqW2osAU= -github.com/nyaruka/phonenumbers v1.0.54/go.mod h1:sDaTZ/KPX5f8qyV9qN+hIm+4ZBARJrupC6LuhshJq1U= -github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.15.2/go.mod h1:Dd6YFfwBW84ETqqtL0CPyPXillHgY6XhQH3uuCCTr/o= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.11.0/go.mod h1:azGKhqFUon9Vuj0YmTfLSmx0FUwqXYSTl5re8lQLTUg= -github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= -github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.0-rc5 h1:Ygwkfw9bpDvs+c9E34SdgGOj41dX/cbdlwvlWt0pnFI= -github.com/opencontainers/image-spec v1.1.0-rc5/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= -github.com/opencontainers/runc v1.1.5 h1:L44KXEpKmfWDcS02aeGm8QNTFXTo2D+8MYGDIJ/GDEs= -github.com/opencontainers/runc v1.1.5/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg= -github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.1.0-rc.1 h1:wHa9jroFfKGQqFHj0I1fMRKLl0pfj+ynAqBxo3v6u9w= -github.com/opencontainers/runtime-spec v1.1.0-rc.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= -github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= -github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= -github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= -github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= -github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= -github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= -github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkoukk/tiktoken-go v0.1.6 h1:JF0TlJzhTbrI30wCvFuiw6FzP2+/bR+FIxUdgEAcUsw= -github.com/pkoukk/tiktoken-go v0.1.6/go.mod h1:9NiV+i9mJKGj1rYOT+njbv+ZwA/zJxYdewGl6qVatpg= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= -github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= -github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= -github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM= -github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= -github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= -github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= -github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= -github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/rs/cors v1.5.0 h1:dgSHE6+ia18arGOTIYQKKGWLvEbGvmbNE6NfxhoNHUY= -github.com/rs/cors v1.5.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= -github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= -github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= -github.com/shirou/gopsutil/v3 v3.23.9 h1:ZI5bWVeu2ep4/DIxB4U9okeYJ7zp/QLTO4auRb/ty/E= -github.com/shirou/gopsutil/v3 v3.23.9/go.mod h1:x/NWSb71eMcjFIO0vhyGW5nZ7oSIgVjrCnADckb85GA= -github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= -github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= -github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= -github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= -github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/square/go-jose v2.3.0+incompatible h1:PYzqfNGdv4dwk11sF556SzL3oKQ1oNfysu6S7CxmMK0= -github.com/square/go-jose v2.3.0+incompatible/go.mod h1:7MxpAF/1WTVUu8Am+T5kNy+t0902CaLWM4Z745MkOa8= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= -github.com/syndtr/goleveldb v0.0.0-20180708030551-c4c61651e9e3/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= -github.com/tailor-inc/graphql v0.2.1 h1:l0zILC0GiSH02DjeJVvGPoDCMWhqQa+fSvQDyCg3zYk= -github.com/tailor-inc/graphql v0.2.1/go.mod h1:Rl0/u8OoidpQkaoKFph1ElyMc3EI6GYdC30rI6fQHak= -github.com/testcontainers/testcontainers-go v0.26.0 h1:uqcYdoOHBy1ca7gKODfBd9uTHVK3a7UL848z09MVZ0c= -github.com/testcontainers/testcontainers-go v0.26.0/go.mod h1:ICriE9bLX5CLxL9OFQ2N+2N+f+803LNJ1utJb1+Inx0= -github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= -github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= -github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= -github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= -github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= -github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= -github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= -github.com/weaviate/contextionary v1.2.1 h1:mmxHVc1mWpqivLHEA/ITHUiAOZziIDluYfKysIgEmnM= -github.com/weaviate/contextionary v1.2.1/go.mod h1:nIEM3Gq1BzTZLuY+Pl7t8hD3eR6VAU43fRdZTEZ9LRY= -github.com/weaviate/sroar v0.0.0-20230210105426-26108af5465d h1:bULMGmIS786YSmm/SssAmwu86y4saMoHhvuL0u7pWLc= -github.com/weaviate/sroar v0.0.0-20230210105426-26108af5465d/go.mod h1:bJUcu8a/7XKOeaCWZtSjuBogUGReUiwJTyGSvcAjDzQ= -github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= -github.com/willf/bitset v1.1.11 h1:N7Z7E9UvjW+sGsEl7k/SJrvY2reP1A07MrGuCjIOjRE= -github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= -github.com/willf/bloom v2.0.3+incompatible h1:QDacWdqcAUI1MPOwIQZRy9kOR7yxfyEmxX8Wdm2/JPA= -github.com/willf/bloom v2.0.3+incompatible/go.mod h1:MmAltL9pDMNTrvUkxdg0k0q5I0suxmuwp3KbyrZLOZ8= -github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= -github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= -github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= -github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= -github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= -github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= -github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA= -go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= -go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg= -go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng= -go.mongodb.org/mongo-driver v1.8.3/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= -go.mongodb.org/mongo-driver v1.10.0/go.mod h1:wsihk0Kdgv8Kqu1Anit4sfK+22vSFbUrAVEYRhCXrA8= -go.mongodb.org/mongo-driver v1.11.0 h1:FZKhBSTydeuffHj9CBjXlR8vQLee1cQyTWYPA6/tqiE= -go.mongodb.org/mongo-driver v1.11.0/go.mod h1:s7p5vEtfbeR1gYi6pnj3c3/urpbLv2T5Sfd6Rp2HBB8= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= -go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= -go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea h1:vLCWI/yYrdEHyN2JzIzPO3aaQJHQdp89IZBA/+azVC4= -golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= -golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= -golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= -golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= -golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= -golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20200928182047-19e03678916f/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg= -golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -gonum.org/v1/gonum v0.12.0 h1:xKuo6hzt+gMav00meVPUlXwSdoEJP46BR+wdxQEFK2o= -gonum.org/v1/gonum v0.12.0/go.mod h1:73TDxJfAAHeA8Mk9mf8NlIppyhQNo5GLTcYeqgo2lvY= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= -google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= -google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= -google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= -google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= -google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= -google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= -google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= -google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= -google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= -google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= -google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= -google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= -google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= -google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= -google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg= -google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o= -google.golang.org/api v0.132.0 h1:8t2/+qZ26kAOGSmOiHwVycqVaDg7q3JDILrNi/Z6rvc= -google.golang.org/api v0.132.0/go.mod h1:AeTBC6GpJnJSRJjktDcPX0QwtS8pGYZOV6MSuSCusw0= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210329143202-679c6ae281ee/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= -google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= -google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= -google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY= -google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= -google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d h1:DoPTO70H+bcDXcd39vOqb2viZxgqeBeSGtZ55yZU4/Q= -google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= -google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= -google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= -google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= -gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI= -gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.5.0 h1:Ljk6PdHdOhAb5aDMWXjDLMMhph+BpztA4v1QdqEW2eY= -gotest.tools/v3 v3.5.0/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/grpc/generated/protocol/v0/batch.pb.go b/grpc/generated/protocol/v0/batch.pb.go deleted file mode 100644 index aebf701f6f2a05c9a93549c8e6b18da354308565..0000000000000000000000000000000000000000 --- a/grpc/generated/protocol/v0/batch.pb.go +++ /dev/null @@ -1,188 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. - -package protocol - -import ( - reflect "reflect" - sync "sync" - - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -type BatchObjectsRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields -} - -func (x *BatchObjectsRequest) Reset() { - *x = BatchObjectsRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_v0_batch_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *BatchObjectsRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*BatchObjectsRequest) ProtoMessage() {} - -func (x *BatchObjectsRequest) ProtoReflect() protoreflect.Message { - mi := &file_v0_batch_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use BatchObjectsRequest.ProtoReflect.Descriptor instead. -func (*BatchObjectsRequest) Descriptor() ([]byte, []int) { - return file_v0_batch_proto_rawDescGZIP(), []int{0} -} - -type BatchObjectsReply struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields -} - -func (x *BatchObjectsReply) Reset() { - *x = BatchObjectsReply{} - if protoimpl.UnsafeEnabled { - mi := &file_v0_batch_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *BatchObjectsReply) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*BatchObjectsReply) ProtoMessage() {} - -func (x *BatchObjectsReply) ProtoReflect() protoreflect.Message { - mi := &file_v0_batch_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use BatchObjectsReply.ProtoReflect.Descriptor instead. -func (*BatchObjectsReply) Descriptor() ([]byte, []int) { - return file_v0_batch_proto_rawDescGZIP(), []int{1} -} - -var File_v0_batch_proto protoreflect.FileDescriptor - -var file_v0_batch_proto_rawDesc = []byte{ - 0x0a, 0x0e, 0x76, 0x30, 0x2f, 0x62, 0x61, 0x74, 0x63, 0x68, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x12, 0x0c, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x67, 0x72, 0x70, 0x63, 0x22, 0x15, - 0x0a, 0x13, 0x42, 0x61, 0x74, 0x63, 0x68, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x13, 0x0a, 0x11, 0x42, 0x61, 0x74, 0x63, 0x68, 0x4f, 0x62, - 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x42, 0x6f, 0x0a, 0x23, 0x69, 0x6f, - 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, - 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x76, - 0x30, 0x42, 0x12, 0x57, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, - 0x42, 0x61, 0x74, 0x63, 0x68, 0x5a, 0x34, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, - 0x6d, 0x2f, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2f, 0x77, 0x65, 0x61, 0x76, 0x69, - 0x61, 0x74, 0x65, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, - 0x65, 0x64, 0x3b, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x33, -} - -var ( - file_v0_batch_proto_rawDescOnce sync.Once - file_v0_batch_proto_rawDescData = file_v0_batch_proto_rawDesc -) - -func file_v0_batch_proto_rawDescGZIP() []byte { - file_v0_batch_proto_rawDescOnce.Do(func() { - file_v0_batch_proto_rawDescData = protoimpl.X.CompressGZIP(file_v0_batch_proto_rawDescData) - }) - return file_v0_batch_proto_rawDescData -} - -var file_v0_batch_proto_msgTypes = make([]protoimpl.MessageInfo, 2) -var file_v0_batch_proto_goTypes = []interface{}{ - (*BatchObjectsRequest)(nil), // 0: weaviategrpc.BatchObjectsRequest - (*BatchObjectsReply)(nil), // 1: weaviategrpc.BatchObjectsReply -} -var file_v0_batch_proto_depIdxs = []int32{ - 0, // [0:0] is the sub-list for method output_type - 0, // [0:0] is the sub-list for method input_type - 0, // [0:0] is the sub-list for extension type_name - 0, // [0:0] is the sub-list for extension extendee - 0, // [0:0] is the sub-list for field type_name -} - -func init() { file_v0_batch_proto_init() } -func file_v0_batch_proto_init() { - if File_v0_batch_proto != nil { - return - } - if !protoimpl.UnsafeEnabled { - file_v0_batch_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*BatchObjectsRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v0_batch_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*BatchObjectsReply); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_v0_batch_proto_rawDesc, - NumEnums: 0, - NumMessages: 2, - NumExtensions: 0, - NumServices: 0, - }, - GoTypes: file_v0_batch_proto_goTypes, - DependencyIndexes: file_v0_batch_proto_depIdxs, - MessageInfos: file_v0_batch_proto_msgTypes, - }.Build() - File_v0_batch_proto = out.File - file_v0_batch_proto_rawDesc = nil - file_v0_batch_proto_goTypes = nil - file_v0_batch_proto_depIdxs = nil -} diff --git a/grpc/generated/protocol/v0/search_get.pb.go b/grpc/generated/protocol/v0/search_get.pb.go deleted file mode 100644 index 0bb19e84811a131b944cc67a205bbe3f44ba1895..0000000000000000000000000000000000000000 --- a/grpc/generated/protocol/v0/search_get.pb.go +++ /dev/null @@ -1,187 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. - -package protocol - -import ( - reflect "reflect" - sync "sync" - - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -type SearchRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields -} - -func (x *SearchRequest) Reset() { - *x = SearchRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_v0_search_get_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *SearchRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*SearchRequest) ProtoMessage() {} - -func (x *SearchRequest) ProtoReflect() protoreflect.Message { - mi := &file_v0_search_get_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use SearchRequest.ProtoReflect.Descriptor instead. -func (*SearchRequest) Descriptor() ([]byte, []int) { - return file_v0_search_get_proto_rawDescGZIP(), []int{0} -} - -type SearchReply struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields -} - -func (x *SearchReply) Reset() { - *x = SearchReply{} - if protoimpl.UnsafeEnabled { - mi := &file_v0_search_get_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *SearchReply) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*SearchReply) ProtoMessage() {} - -func (x *SearchReply) ProtoReflect() protoreflect.Message { - mi := &file_v0_search_get_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use SearchReply.ProtoReflect.Descriptor instead. -func (*SearchReply) Descriptor() ([]byte, []int) { - return file_v0_search_get_proto_rawDescGZIP(), []int{1} -} - -var File_v0_search_get_proto protoreflect.FileDescriptor - -var file_v0_search_get_proto_rawDesc = []byte{ - 0x0a, 0x13, 0x76, 0x30, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x5f, 0x67, 0x65, 0x74, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x67, - 0x72, 0x70, 0x63, 0x22, 0x0f, 0x0a, 0x0d, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x22, 0x0d, 0x0a, 0x0b, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, - 0x70, 0x6c, 0x79, 0x42, 0x73, 0x0a, 0x23, 0x69, 0x6f, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, - 0x74, 0x65, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x76, 0x30, 0x42, 0x16, 0x57, 0x65, 0x61, 0x76, - 0x69, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x47, - 0x65, 0x74, 0x5a, 0x34, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x77, - 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2f, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, - 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x3b, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -} - -var ( - file_v0_search_get_proto_rawDescOnce sync.Once - file_v0_search_get_proto_rawDescData = file_v0_search_get_proto_rawDesc -) - -func file_v0_search_get_proto_rawDescGZIP() []byte { - file_v0_search_get_proto_rawDescOnce.Do(func() { - file_v0_search_get_proto_rawDescData = protoimpl.X.CompressGZIP(file_v0_search_get_proto_rawDescData) - }) - return file_v0_search_get_proto_rawDescData -} - -var file_v0_search_get_proto_msgTypes = make([]protoimpl.MessageInfo, 2) -var file_v0_search_get_proto_goTypes = []interface{}{ - (*SearchRequest)(nil), // 0: weaviategrpc.SearchRequest - (*SearchReply)(nil), // 1: weaviategrpc.SearchReply -} -var file_v0_search_get_proto_depIdxs = []int32{ - 0, // [0:0] is the sub-list for method output_type - 0, // [0:0] is the sub-list for method input_type - 0, // [0:0] is the sub-list for extension type_name - 0, // [0:0] is the sub-list for extension extendee - 0, // [0:0] is the sub-list for field type_name -} - -func init() { file_v0_search_get_proto_init() } -func file_v0_search_get_proto_init() { - if File_v0_search_get_proto != nil { - return - } - if !protoimpl.UnsafeEnabled { - file_v0_search_get_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SearchRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v0_search_get_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SearchReply); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_v0_search_get_proto_rawDesc, - NumEnums: 0, - NumMessages: 2, - NumExtensions: 0, - NumServices: 0, - }, - GoTypes: file_v0_search_get_proto_goTypes, - DependencyIndexes: file_v0_search_get_proto_depIdxs, - MessageInfos: file_v0_search_get_proto_msgTypes, - }.Build() - File_v0_search_get_proto = out.File - file_v0_search_get_proto_rawDesc = nil - file_v0_search_get_proto_goTypes = nil - file_v0_search_get_proto_depIdxs = nil -} diff --git a/grpc/generated/protocol/v0/weaviate.pb.go b/grpc/generated/protocol/v0/weaviate.pb.go deleted file mode 100644 index 616e9605f9abc63bad536415ba01e55d08f37d5f..0000000000000000000000000000000000000000 --- a/grpc/generated/protocol/v0/weaviate.pb.go +++ /dev/null @@ -1,89 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. - -package protocol - -import ( - reflect "reflect" - - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -var File_v0_weaviate_proto protoreflect.FileDescriptor - -var file_v0_weaviate_proto_rawDesc = []byte{ - 0x0a, 0x11, 0x76, 0x30, 0x2f, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x67, 0x72, 0x70, - 0x63, 0x1a, 0x0e, 0x76, 0x30, 0x2f, 0x62, 0x61, 0x74, 0x63, 0x68, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x1a, 0x13, 0x76, 0x30, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x5f, 0x67, 0x65, 0x74, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x32, 0xa4, 0x01, 0x0a, 0x08, 0x57, 0x65, 0x61, 0x76, 0x69, - 0x61, 0x74, 0x65, 0x12, 0x42, 0x0a, 0x06, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x12, 0x1b, 0x2e, - 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x65, 0x61, - 0x72, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x77, 0x65, 0x61, - 0x76, 0x69, 0x61, 0x74, 0x65, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, - 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x54, 0x0a, 0x0c, 0x42, 0x61, 0x74, 0x63, 0x68, - 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x21, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, - 0x74, 0x65, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x42, 0x61, 0x74, 0x63, 0x68, 0x4f, 0x62, 0x6a, 0x65, - 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x77, 0x65, 0x61, - 0x76, 0x69, 0x61, 0x74, 0x65, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x42, 0x61, 0x74, 0x63, 0x68, 0x4f, - 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x42, 0x6a, 0x0a, - 0x23, 0x69, 0x6f, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x63, 0x6c, 0x69, - 0x65, 0x6e, 0x74, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, - 0x6c, 0x2e, 0x76, 0x30, 0x42, 0x0d, 0x57, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x50, 0x72, - 0x6f, 0x74, 0x6f, 0x5a, 0x34, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, - 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2f, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, - 0x65, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, - 0x3b, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x33, -} - -var file_v0_weaviate_proto_goTypes = []interface{}{ - (*SearchRequest)(nil), // 0: weaviategrpc.SearchRequest - (*BatchObjectsRequest)(nil), // 1: weaviategrpc.BatchObjectsRequest - (*SearchReply)(nil), // 2: weaviategrpc.SearchReply - (*BatchObjectsReply)(nil), // 3: weaviategrpc.BatchObjectsReply -} -var file_v0_weaviate_proto_depIdxs = []int32{ - 0, // 0: weaviategrpc.Weaviate.Search:input_type -> weaviategrpc.SearchRequest - 1, // 1: weaviategrpc.Weaviate.BatchObjects:input_type -> weaviategrpc.BatchObjectsRequest - 2, // 2: weaviategrpc.Weaviate.Search:output_type -> weaviategrpc.SearchReply - 3, // 3: weaviategrpc.Weaviate.BatchObjects:output_type -> weaviategrpc.BatchObjectsReply - 2, // [2:4] is the sub-list for method output_type - 0, // [0:2] is the sub-list for method input_type - 0, // [0:0] is the sub-list for extension type_name - 0, // [0:0] is the sub-list for extension extendee - 0, // [0:0] is the sub-list for field type_name -} - -func init() { file_v0_weaviate_proto_init() } -func file_v0_weaviate_proto_init() { - if File_v0_weaviate_proto != nil { - return - } - file_v0_batch_proto_init() - file_v0_search_get_proto_init() - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_v0_weaviate_proto_rawDesc, - NumEnums: 0, - NumMessages: 0, - NumExtensions: 0, - NumServices: 1, - }, - GoTypes: file_v0_weaviate_proto_goTypes, - DependencyIndexes: file_v0_weaviate_proto_depIdxs, - }.Build() - File_v0_weaviate_proto = out.File - file_v0_weaviate_proto_rawDesc = nil - file_v0_weaviate_proto_goTypes = nil - file_v0_weaviate_proto_depIdxs = nil -} diff --git a/grpc/generated/protocol/v0/weaviate_grpc.pb.go b/grpc/generated/protocol/v0/weaviate_grpc.pb.go deleted file mode 100644 index 06736aa023c414abb7f436cb3b2d6bf4a173b4fa..0000000000000000000000000000000000000000 --- a/grpc/generated/protocol/v0/weaviate_grpc.pb.go +++ /dev/null @@ -1,138 +0,0 @@ -// Code generated by protoc-gen-go-grpc. DO NOT EDIT. - -package protocol - -import ( - context "context" - - grpc "google.golang.org/grpc" - codes "google.golang.org/grpc/codes" - status "google.golang.org/grpc/status" -) - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.32.0 or later. -const _ = grpc.SupportPackageIsVersion7 - -// WeaviateClient is the client API for Weaviate service. -// -// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. -type WeaviateClient interface { - Search(ctx context.Context, in *SearchRequest, opts ...grpc.CallOption) (*SearchReply, error) - BatchObjects(ctx context.Context, in *BatchObjectsRequest, opts ...grpc.CallOption) (*BatchObjectsReply, error) -} - -type weaviateClient struct { - cc grpc.ClientConnInterface -} - -func NewWeaviateClient(cc grpc.ClientConnInterface) WeaviateClient { - return &weaviateClient{cc} -} - -func (c *weaviateClient) Search(ctx context.Context, in *SearchRequest, opts ...grpc.CallOption) (*SearchReply, error) { - out := new(SearchReply) - err := c.cc.Invoke(ctx, "/weaviategrpc.Weaviate/Search", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *weaviateClient) BatchObjects(ctx context.Context, in *BatchObjectsRequest, opts ...grpc.CallOption) (*BatchObjectsReply, error) { - out := new(BatchObjectsReply) - err := c.cc.Invoke(ctx, "/weaviategrpc.Weaviate/BatchObjects", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -// WeaviateServer is the server API for Weaviate service. -// All implementations must embed UnimplementedWeaviateServer -// for forward compatibility -type WeaviateServer interface { - Search(context.Context, *SearchRequest) (*SearchReply, error) - BatchObjects(context.Context, *BatchObjectsRequest) (*BatchObjectsReply, error) - mustEmbedUnimplementedWeaviateServer() -} - -// UnimplementedWeaviateServer must be embedded to have forward compatible implementations. -type UnimplementedWeaviateServer struct { -} - -func (UnimplementedWeaviateServer) Search(context.Context, *SearchRequest) (*SearchReply, error) { - return nil, status.Errorf(codes.Unimplemented, "method Search not implemented") -} -func (UnimplementedWeaviateServer) BatchObjects(context.Context, *BatchObjectsRequest) (*BatchObjectsReply, error) { - return nil, status.Errorf(codes.Unimplemented, "method BatchObjects not implemented") -} -func (UnimplementedWeaviateServer) mustEmbedUnimplementedWeaviateServer() {} - -// UnsafeWeaviateServer may be embedded to opt out of forward compatibility for this service. -// Use of this interface is not recommended, as added methods to WeaviateServer will -// result in compilation errors. -type UnsafeWeaviateServer interface { - mustEmbedUnimplementedWeaviateServer() -} - -func RegisterWeaviateServer(s grpc.ServiceRegistrar, srv WeaviateServer) { - s.RegisterService(&Weaviate_ServiceDesc, srv) -} - -func _Weaviate_Search_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(SearchRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(WeaviateServer).Search(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/weaviategrpc.Weaviate/Search", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(WeaviateServer).Search(ctx, req.(*SearchRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _Weaviate_BatchObjects_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(BatchObjectsRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(WeaviateServer).BatchObjects(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/weaviategrpc.Weaviate/BatchObjects", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(WeaviateServer).BatchObjects(ctx, req.(*BatchObjectsRequest)) - } - return interceptor(ctx, in, info, handler) -} - -// Weaviate_ServiceDesc is the grpc.ServiceDesc for Weaviate service. -// It's only intended for direct use with grpc.RegisterService, -// and not to be introspected or modified (even as a copy) -var Weaviate_ServiceDesc = grpc.ServiceDesc{ - ServiceName: "weaviategrpc.Weaviate", - HandlerType: (*WeaviateServer)(nil), - Methods: []grpc.MethodDesc{ - { - MethodName: "Search", - Handler: _Weaviate_Search_Handler, - }, - { - MethodName: "BatchObjects", - Handler: _Weaviate_BatchObjects_Handler, - }, - }, - Streams: []grpc.StreamDesc{}, - Metadata: "v0/weaviate.proto", -} diff --git a/grpc/generated/protocol/v1/base.pb.go b/grpc/generated/protocol/v1/base.pb.go deleted file mode 100644 index 865ba01e9f88e212c65f70a0c1479ecee2643488..0000000000000000000000000000000000000000 --- a/grpc/generated/protocol/v1/base.pb.go +++ /dev/null @@ -1,1777 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. - -package protocol - -import ( - reflect "reflect" - sync "sync" - - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" - structpb "google.golang.org/protobuf/types/known/structpb" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -type ConsistencyLevel int32 - -const ( - ConsistencyLevel_CONSISTENCY_LEVEL_UNSPECIFIED ConsistencyLevel = 0 - ConsistencyLevel_CONSISTENCY_LEVEL_ONE ConsistencyLevel = 1 - ConsistencyLevel_CONSISTENCY_LEVEL_QUORUM ConsistencyLevel = 2 - ConsistencyLevel_CONSISTENCY_LEVEL_ALL ConsistencyLevel = 3 -) - -// Enum value maps for ConsistencyLevel. -var ( - ConsistencyLevel_name = map[int32]string{ - 0: "CONSISTENCY_LEVEL_UNSPECIFIED", - 1: "CONSISTENCY_LEVEL_ONE", - 2: "CONSISTENCY_LEVEL_QUORUM", - 3: "CONSISTENCY_LEVEL_ALL", - } - ConsistencyLevel_value = map[string]int32{ - "CONSISTENCY_LEVEL_UNSPECIFIED": 0, - "CONSISTENCY_LEVEL_ONE": 1, - "CONSISTENCY_LEVEL_QUORUM": 2, - "CONSISTENCY_LEVEL_ALL": 3, - } -) - -func (x ConsistencyLevel) Enum() *ConsistencyLevel { - p := new(ConsistencyLevel) - *p = x - return p -} - -func (x ConsistencyLevel) String() string { - return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) -} - -func (ConsistencyLevel) Descriptor() protoreflect.EnumDescriptor { - return file_v1_base_proto_enumTypes[0].Descriptor() -} - -func (ConsistencyLevel) Type() protoreflect.EnumType { - return &file_v1_base_proto_enumTypes[0] -} - -func (x ConsistencyLevel) Number() protoreflect.EnumNumber { - return protoreflect.EnumNumber(x) -} - -// Deprecated: Use ConsistencyLevel.Descriptor instead. -func (ConsistencyLevel) EnumDescriptor() ([]byte, []int) { - return file_v1_base_proto_rawDescGZIP(), []int{0} -} - -type Filters_Operator int32 - -const ( - Filters_OPERATOR_UNSPECIFIED Filters_Operator = 0 - Filters_OPERATOR_EQUAL Filters_Operator = 1 - Filters_OPERATOR_NOT_EQUAL Filters_Operator = 2 - Filters_OPERATOR_GREATER_THAN Filters_Operator = 3 - Filters_OPERATOR_GREATER_THAN_EQUAL Filters_Operator = 4 - Filters_OPERATOR_LESS_THAN Filters_Operator = 5 - Filters_OPERATOR_LESS_THAN_EQUAL Filters_Operator = 6 - Filters_OPERATOR_AND Filters_Operator = 7 - Filters_OPERATOR_OR Filters_Operator = 8 - Filters_OPERATOR_WITHIN_GEO_RANGE Filters_Operator = 9 - Filters_OPERATOR_LIKE Filters_Operator = 10 - Filters_OPERATOR_IS_NULL Filters_Operator = 11 - Filters_OPERATOR_CONTAINS_ANY Filters_Operator = 12 - Filters_OPERATOR_CONTAINS_ALL Filters_Operator = 13 -) - -// Enum value maps for Filters_Operator. -var ( - Filters_Operator_name = map[int32]string{ - 0: "OPERATOR_UNSPECIFIED", - 1: "OPERATOR_EQUAL", - 2: "OPERATOR_NOT_EQUAL", - 3: "OPERATOR_GREATER_THAN", - 4: "OPERATOR_GREATER_THAN_EQUAL", - 5: "OPERATOR_LESS_THAN", - 6: "OPERATOR_LESS_THAN_EQUAL", - 7: "OPERATOR_AND", - 8: "OPERATOR_OR", - 9: "OPERATOR_WITHIN_GEO_RANGE", - 10: "OPERATOR_LIKE", - 11: "OPERATOR_IS_NULL", - 12: "OPERATOR_CONTAINS_ANY", - 13: "OPERATOR_CONTAINS_ALL", - } - Filters_Operator_value = map[string]int32{ - "OPERATOR_UNSPECIFIED": 0, - "OPERATOR_EQUAL": 1, - "OPERATOR_NOT_EQUAL": 2, - "OPERATOR_GREATER_THAN": 3, - "OPERATOR_GREATER_THAN_EQUAL": 4, - "OPERATOR_LESS_THAN": 5, - "OPERATOR_LESS_THAN_EQUAL": 6, - "OPERATOR_AND": 7, - "OPERATOR_OR": 8, - "OPERATOR_WITHIN_GEO_RANGE": 9, - "OPERATOR_LIKE": 10, - "OPERATOR_IS_NULL": 11, - "OPERATOR_CONTAINS_ANY": 12, - "OPERATOR_CONTAINS_ALL": 13, - } -) - -func (x Filters_Operator) Enum() *Filters_Operator { - p := new(Filters_Operator) - *p = x - return p -} - -func (x Filters_Operator) String() string { - return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) -} - -func (Filters_Operator) Descriptor() protoreflect.EnumDescriptor { - return file_v1_base_proto_enumTypes[1].Descriptor() -} - -func (Filters_Operator) Type() protoreflect.EnumType { - return &file_v1_base_proto_enumTypes[1] -} - -func (x Filters_Operator) Number() protoreflect.EnumNumber { - return protoreflect.EnumNumber(x) -} - -// Deprecated: Use Filters_Operator.Descriptor instead. -func (Filters_Operator) EnumDescriptor() ([]byte, []int) { - return file_v1_base_proto_rawDescGZIP(), []int{11, 0} -} - -type NumberArrayProperties struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Deprecated: Marked as deprecated in v1/base.proto. - Values []float64 `protobuf:"fixed64,1,rep,packed,name=values,proto3" json:"values,omitempty"` // will be removed in the future, use vector_bytes - PropName string `protobuf:"bytes,2,opt,name=prop_name,json=propName,proto3" json:"prop_name,omitempty"` - ValuesBytes []byte `protobuf:"bytes,3,opt,name=values_bytes,json=valuesBytes,proto3" json:"values_bytes,omitempty"` -} - -func (x *NumberArrayProperties) Reset() { - *x = NumberArrayProperties{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_base_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *NumberArrayProperties) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*NumberArrayProperties) ProtoMessage() {} - -func (x *NumberArrayProperties) ProtoReflect() protoreflect.Message { - mi := &file_v1_base_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use NumberArrayProperties.ProtoReflect.Descriptor instead. -func (*NumberArrayProperties) Descriptor() ([]byte, []int) { - return file_v1_base_proto_rawDescGZIP(), []int{0} -} - -// Deprecated: Marked as deprecated in v1/base.proto. -func (x *NumberArrayProperties) GetValues() []float64 { - if x != nil { - return x.Values - } - return nil -} - -func (x *NumberArrayProperties) GetPropName() string { - if x != nil { - return x.PropName - } - return "" -} - -func (x *NumberArrayProperties) GetValuesBytes() []byte { - if x != nil { - return x.ValuesBytes - } - return nil -} - -type IntArrayProperties struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Values []int64 `protobuf:"varint,1,rep,packed,name=values,proto3" json:"values,omitempty"` - PropName string `protobuf:"bytes,2,opt,name=prop_name,json=propName,proto3" json:"prop_name,omitempty"` -} - -func (x *IntArrayProperties) Reset() { - *x = IntArrayProperties{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_base_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *IntArrayProperties) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*IntArrayProperties) ProtoMessage() {} - -func (x *IntArrayProperties) ProtoReflect() protoreflect.Message { - mi := &file_v1_base_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use IntArrayProperties.ProtoReflect.Descriptor instead. -func (*IntArrayProperties) Descriptor() ([]byte, []int) { - return file_v1_base_proto_rawDescGZIP(), []int{1} -} - -func (x *IntArrayProperties) GetValues() []int64 { - if x != nil { - return x.Values - } - return nil -} - -func (x *IntArrayProperties) GetPropName() string { - if x != nil { - return x.PropName - } - return "" -} - -type TextArrayProperties struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Values []string `protobuf:"bytes,1,rep,name=values,proto3" json:"values,omitempty"` - PropName string `protobuf:"bytes,2,opt,name=prop_name,json=propName,proto3" json:"prop_name,omitempty"` -} - -func (x *TextArrayProperties) Reset() { - *x = TextArrayProperties{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_base_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TextArrayProperties) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TextArrayProperties) ProtoMessage() {} - -func (x *TextArrayProperties) ProtoReflect() protoreflect.Message { - mi := &file_v1_base_proto_msgTypes[2] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TextArrayProperties.ProtoReflect.Descriptor instead. -func (*TextArrayProperties) Descriptor() ([]byte, []int) { - return file_v1_base_proto_rawDescGZIP(), []int{2} -} - -func (x *TextArrayProperties) GetValues() []string { - if x != nil { - return x.Values - } - return nil -} - -func (x *TextArrayProperties) GetPropName() string { - if x != nil { - return x.PropName - } - return "" -} - -type BooleanArrayProperties struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Values []bool `protobuf:"varint,1,rep,packed,name=values,proto3" json:"values,omitempty"` - PropName string `protobuf:"bytes,2,opt,name=prop_name,json=propName,proto3" json:"prop_name,omitempty"` -} - -func (x *BooleanArrayProperties) Reset() { - *x = BooleanArrayProperties{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_base_proto_msgTypes[3] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *BooleanArrayProperties) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*BooleanArrayProperties) ProtoMessage() {} - -func (x *BooleanArrayProperties) ProtoReflect() protoreflect.Message { - mi := &file_v1_base_proto_msgTypes[3] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use BooleanArrayProperties.ProtoReflect.Descriptor instead. -func (*BooleanArrayProperties) Descriptor() ([]byte, []int) { - return file_v1_base_proto_rawDescGZIP(), []int{3} -} - -func (x *BooleanArrayProperties) GetValues() []bool { - if x != nil { - return x.Values - } - return nil -} - -func (x *BooleanArrayProperties) GetPropName() string { - if x != nil { - return x.PropName - } - return "" -} - -type ObjectPropertiesValue struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - NonRefProperties *structpb.Struct `protobuf:"bytes,1,opt,name=non_ref_properties,json=nonRefProperties,proto3" json:"non_ref_properties,omitempty"` - NumberArrayProperties []*NumberArrayProperties `protobuf:"bytes,2,rep,name=number_array_properties,json=numberArrayProperties,proto3" json:"number_array_properties,omitempty"` - IntArrayProperties []*IntArrayProperties `protobuf:"bytes,3,rep,name=int_array_properties,json=intArrayProperties,proto3" json:"int_array_properties,omitempty"` - TextArrayProperties []*TextArrayProperties `protobuf:"bytes,4,rep,name=text_array_properties,json=textArrayProperties,proto3" json:"text_array_properties,omitempty"` - BooleanArrayProperties []*BooleanArrayProperties `protobuf:"bytes,5,rep,name=boolean_array_properties,json=booleanArrayProperties,proto3" json:"boolean_array_properties,omitempty"` - ObjectProperties []*ObjectProperties `protobuf:"bytes,6,rep,name=object_properties,json=objectProperties,proto3" json:"object_properties,omitempty"` - ObjectArrayProperties []*ObjectArrayProperties `protobuf:"bytes,7,rep,name=object_array_properties,json=objectArrayProperties,proto3" json:"object_array_properties,omitempty"` -} - -func (x *ObjectPropertiesValue) Reset() { - *x = ObjectPropertiesValue{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_base_proto_msgTypes[4] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ObjectPropertiesValue) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ObjectPropertiesValue) ProtoMessage() {} - -func (x *ObjectPropertiesValue) ProtoReflect() protoreflect.Message { - mi := &file_v1_base_proto_msgTypes[4] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ObjectPropertiesValue.ProtoReflect.Descriptor instead. -func (*ObjectPropertiesValue) Descriptor() ([]byte, []int) { - return file_v1_base_proto_rawDescGZIP(), []int{4} -} - -func (x *ObjectPropertiesValue) GetNonRefProperties() *structpb.Struct { - if x != nil { - return x.NonRefProperties - } - return nil -} - -func (x *ObjectPropertiesValue) GetNumberArrayProperties() []*NumberArrayProperties { - if x != nil { - return x.NumberArrayProperties - } - return nil -} - -func (x *ObjectPropertiesValue) GetIntArrayProperties() []*IntArrayProperties { - if x != nil { - return x.IntArrayProperties - } - return nil -} - -func (x *ObjectPropertiesValue) GetTextArrayProperties() []*TextArrayProperties { - if x != nil { - return x.TextArrayProperties - } - return nil -} - -func (x *ObjectPropertiesValue) GetBooleanArrayProperties() []*BooleanArrayProperties { - if x != nil { - return x.BooleanArrayProperties - } - return nil -} - -func (x *ObjectPropertiesValue) GetObjectProperties() []*ObjectProperties { - if x != nil { - return x.ObjectProperties - } - return nil -} - -func (x *ObjectPropertiesValue) GetObjectArrayProperties() []*ObjectArrayProperties { - if x != nil { - return x.ObjectArrayProperties - } - return nil -} - -type ObjectArrayProperties struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Values []*ObjectPropertiesValue `protobuf:"bytes,1,rep,name=values,proto3" json:"values,omitempty"` - PropName string `protobuf:"bytes,2,opt,name=prop_name,json=propName,proto3" json:"prop_name,omitempty"` -} - -func (x *ObjectArrayProperties) Reset() { - *x = ObjectArrayProperties{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_base_proto_msgTypes[5] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ObjectArrayProperties) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ObjectArrayProperties) ProtoMessage() {} - -func (x *ObjectArrayProperties) ProtoReflect() protoreflect.Message { - mi := &file_v1_base_proto_msgTypes[5] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ObjectArrayProperties.ProtoReflect.Descriptor instead. -func (*ObjectArrayProperties) Descriptor() ([]byte, []int) { - return file_v1_base_proto_rawDescGZIP(), []int{5} -} - -func (x *ObjectArrayProperties) GetValues() []*ObjectPropertiesValue { - if x != nil { - return x.Values - } - return nil -} - -func (x *ObjectArrayProperties) GetPropName() string { - if x != nil { - return x.PropName - } - return "" -} - -type ObjectProperties struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Value *ObjectPropertiesValue `protobuf:"bytes,1,opt,name=value,proto3" json:"value,omitempty"` - PropName string `protobuf:"bytes,2,opt,name=prop_name,json=propName,proto3" json:"prop_name,omitempty"` -} - -func (x *ObjectProperties) Reset() { - *x = ObjectProperties{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_base_proto_msgTypes[6] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ObjectProperties) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ObjectProperties) ProtoMessage() {} - -func (x *ObjectProperties) ProtoReflect() protoreflect.Message { - mi := &file_v1_base_proto_msgTypes[6] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ObjectProperties.ProtoReflect.Descriptor instead. -func (*ObjectProperties) Descriptor() ([]byte, []int) { - return file_v1_base_proto_rawDescGZIP(), []int{6} -} - -func (x *ObjectProperties) GetValue() *ObjectPropertiesValue { - if x != nil { - return x.Value - } - return nil -} - -func (x *ObjectProperties) GetPropName() string { - if x != nil { - return x.PropName - } - return "" -} - -type TextArray struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Values []string `protobuf:"bytes,1,rep,name=values,proto3" json:"values,omitempty"` -} - -func (x *TextArray) Reset() { - *x = TextArray{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_base_proto_msgTypes[7] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *TextArray) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*TextArray) ProtoMessage() {} - -func (x *TextArray) ProtoReflect() protoreflect.Message { - mi := &file_v1_base_proto_msgTypes[7] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use TextArray.ProtoReflect.Descriptor instead. -func (*TextArray) Descriptor() ([]byte, []int) { - return file_v1_base_proto_rawDescGZIP(), []int{7} -} - -func (x *TextArray) GetValues() []string { - if x != nil { - return x.Values - } - return nil -} - -type IntArray struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Values []int64 `protobuf:"varint,1,rep,packed,name=values,proto3" json:"values,omitempty"` -} - -func (x *IntArray) Reset() { - *x = IntArray{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_base_proto_msgTypes[8] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *IntArray) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*IntArray) ProtoMessage() {} - -func (x *IntArray) ProtoReflect() protoreflect.Message { - mi := &file_v1_base_proto_msgTypes[8] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use IntArray.ProtoReflect.Descriptor instead. -func (*IntArray) Descriptor() ([]byte, []int) { - return file_v1_base_proto_rawDescGZIP(), []int{8} -} - -func (x *IntArray) GetValues() []int64 { - if x != nil { - return x.Values - } - return nil -} - -type NumberArray struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Values []float64 `protobuf:"fixed64,1,rep,packed,name=values,proto3" json:"values,omitempty"` -} - -func (x *NumberArray) Reset() { - *x = NumberArray{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_base_proto_msgTypes[9] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *NumberArray) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*NumberArray) ProtoMessage() {} - -func (x *NumberArray) ProtoReflect() protoreflect.Message { - mi := &file_v1_base_proto_msgTypes[9] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use NumberArray.ProtoReflect.Descriptor instead. -func (*NumberArray) Descriptor() ([]byte, []int) { - return file_v1_base_proto_rawDescGZIP(), []int{9} -} - -func (x *NumberArray) GetValues() []float64 { - if x != nil { - return x.Values - } - return nil -} - -type BooleanArray struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Values []bool `protobuf:"varint,1,rep,packed,name=values,proto3" json:"values,omitempty"` -} - -func (x *BooleanArray) Reset() { - *x = BooleanArray{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_base_proto_msgTypes[10] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *BooleanArray) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*BooleanArray) ProtoMessage() {} - -func (x *BooleanArray) ProtoReflect() protoreflect.Message { - mi := &file_v1_base_proto_msgTypes[10] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use BooleanArray.ProtoReflect.Descriptor instead. -func (*BooleanArray) Descriptor() ([]byte, []int) { - return file_v1_base_proto_rawDescGZIP(), []int{10} -} - -func (x *BooleanArray) GetValues() []bool { - if x != nil { - return x.Values - } - return nil -} - -type Filters struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Operator Filters_Operator `protobuf:"varint,1,opt,name=operator,proto3,enum=weaviate.v1.Filters_Operator" json:"operator,omitempty"` - // protolint:disable:next REPEATED_FIELD_NAMES_PLURALIZED - // - // Deprecated: Marked as deprecated in v1/base.proto. - On []string `protobuf:"bytes,2,rep,name=on,proto3" json:"on,omitempty"` // will be removed in the future, use path - Filters []*Filters `protobuf:"bytes,3,rep,name=filters,proto3" json:"filters,omitempty"` - // Types that are assignable to TestValue: - // - // *Filters_ValueText - // *Filters_ValueInt - // *Filters_ValueBoolean - // *Filters_ValueNumber - // *Filters_ValueTextArray - // *Filters_ValueIntArray - // *Filters_ValueBooleanArray - // *Filters_ValueNumberArray - // *Filters_ValueGeo - TestValue isFilters_TestValue `protobuf_oneof:"test_value"` - Target *FilterTarget `protobuf:"bytes,20,opt,name=target,proto3" json:"target,omitempty"` // leave space for more filter values -} - -func (x *Filters) Reset() { - *x = Filters{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_base_proto_msgTypes[11] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *Filters) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*Filters) ProtoMessage() {} - -func (x *Filters) ProtoReflect() protoreflect.Message { - mi := &file_v1_base_proto_msgTypes[11] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use Filters.ProtoReflect.Descriptor instead. -func (*Filters) Descriptor() ([]byte, []int) { - return file_v1_base_proto_rawDescGZIP(), []int{11} -} - -func (x *Filters) GetOperator() Filters_Operator { - if x != nil { - return x.Operator - } - return Filters_OPERATOR_UNSPECIFIED -} - -// Deprecated: Marked as deprecated in v1/base.proto. -func (x *Filters) GetOn() []string { - if x != nil { - return x.On - } - return nil -} - -func (x *Filters) GetFilters() []*Filters { - if x != nil { - return x.Filters - } - return nil -} - -func (m *Filters) GetTestValue() isFilters_TestValue { - if m != nil { - return m.TestValue - } - return nil -} - -func (x *Filters) GetValueText() string { - if x, ok := x.GetTestValue().(*Filters_ValueText); ok { - return x.ValueText - } - return "" -} - -func (x *Filters) GetValueInt() int64 { - if x, ok := x.GetTestValue().(*Filters_ValueInt); ok { - return x.ValueInt - } - return 0 -} - -func (x *Filters) GetValueBoolean() bool { - if x, ok := x.GetTestValue().(*Filters_ValueBoolean); ok { - return x.ValueBoolean - } - return false -} - -func (x *Filters) GetValueNumber() float64 { - if x, ok := x.GetTestValue().(*Filters_ValueNumber); ok { - return x.ValueNumber - } - return 0 -} - -func (x *Filters) GetValueTextArray() *TextArray { - if x, ok := x.GetTestValue().(*Filters_ValueTextArray); ok { - return x.ValueTextArray - } - return nil -} - -func (x *Filters) GetValueIntArray() *IntArray { - if x, ok := x.GetTestValue().(*Filters_ValueIntArray); ok { - return x.ValueIntArray - } - return nil -} - -func (x *Filters) GetValueBooleanArray() *BooleanArray { - if x, ok := x.GetTestValue().(*Filters_ValueBooleanArray); ok { - return x.ValueBooleanArray - } - return nil -} - -func (x *Filters) GetValueNumberArray() *NumberArray { - if x, ok := x.GetTestValue().(*Filters_ValueNumberArray); ok { - return x.ValueNumberArray - } - return nil -} - -func (x *Filters) GetValueGeo() *GeoCoordinatesFilter { - if x, ok := x.GetTestValue().(*Filters_ValueGeo); ok { - return x.ValueGeo - } - return nil -} - -func (x *Filters) GetTarget() *FilterTarget { - if x != nil { - return x.Target - } - return nil -} - -type isFilters_TestValue interface { - isFilters_TestValue() -} - -type Filters_ValueText struct { - ValueText string `protobuf:"bytes,4,opt,name=value_text,json=valueText,proto3,oneof"` -} - -type Filters_ValueInt struct { - ValueInt int64 `protobuf:"varint,5,opt,name=value_int,json=valueInt,proto3,oneof"` -} - -type Filters_ValueBoolean struct { - ValueBoolean bool `protobuf:"varint,6,opt,name=value_boolean,json=valueBoolean,proto3,oneof"` -} - -type Filters_ValueNumber struct { - ValueNumber float64 `protobuf:"fixed64,7,opt,name=value_number,json=valueNumber,proto3,oneof"` -} - -type Filters_ValueTextArray struct { - ValueTextArray *TextArray `protobuf:"bytes,9,opt,name=value_text_array,json=valueTextArray,proto3,oneof"` -} - -type Filters_ValueIntArray struct { - ValueIntArray *IntArray `protobuf:"bytes,10,opt,name=value_int_array,json=valueIntArray,proto3,oneof"` -} - -type Filters_ValueBooleanArray struct { - ValueBooleanArray *BooleanArray `protobuf:"bytes,11,opt,name=value_boolean_array,json=valueBooleanArray,proto3,oneof"` -} - -type Filters_ValueNumberArray struct { - ValueNumberArray *NumberArray `protobuf:"bytes,12,opt,name=value_number_array,json=valueNumberArray,proto3,oneof"` -} - -type Filters_ValueGeo struct { - ValueGeo *GeoCoordinatesFilter `protobuf:"bytes,13,opt,name=value_geo,json=valueGeo,proto3,oneof"` -} - -func (*Filters_ValueText) isFilters_TestValue() {} - -func (*Filters_ValueInt) isFilters_TestValue() {} - -func (*Filters_ValueBoolean) isFilters_TestValue() {} - -func (*Filters_ValueNumber) isFilters_TestValue() {} - -func (*Filters_ValueTextArray) isFilters_TestValue() {} - -func (*Filters_ValueIntArray) isFilters_TestValue() {} - -func (*Filters_ValueBooleanArray) isFilters_TestValue() {} - -func (*Filters_ValueNumberArray) isFilters_TestValue() {} - -func (*Filters_ValueGeo) isFilters_TestValue() {} - -type FilterReferenceSingleTarget struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - On string `protobuf:"bytes,1,opt,name=on,proto3" json:"on,omitempty"` - Target *FilterTarget `protobuf:"bytes,2,opt,name=target,proto3" json:"target,omitempty"` -} - -func (x *FilterReferenceSingleTarget) Reset() { - *x = FilterReferenceSingleTarget{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_base_proto_msgTypes[12] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *FilterReferenceSingleTarget) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*FilterReferenceSingleTarget) ProtoMessage() {} - -func (x *FilterReferenceSingleTarget) ProtoReflect() protoreflect.Message { - mi := &file_v1_base_proto_msgTypes[12] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use FilterReferenceSingleTarget.ProtoReflect.Descriptor instead. -func (*FilterReferenceSingleTarget) Descriptor() ([]byte, []int) { - return file_v1_base_proto_rawDescGZIP(), []int{12} -} - -func (x *FilterReferenceSingleTarget) GetOn() string { - if x != nil { - return x.On - } - return "" -} - -func (x *FilterReferenceSingleTarget) GetTarget() *FilterTarget { - if x != nil { - return x.Target - } - return nil -} - -type FilterReferenceMultiTarget struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - On string `protobuf:"bytes,1,opt,name=on,proto3" json:"on,omitempty"` - Target *FilterTarget `protobuf:"bytes,2,opt,name=target,proto3" json:"target,omitempty"` - TargetCollection string `protobuf:"bytes,3,opt,name=target_collection,json=targetCollection,proto3" json:"target_collection,omitempty"` -} - -func (x *FilterReferenceMultiTarget) Reset() { - *x = FilterReferenceMultiTarget{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_base_proto_msgTypes[13] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *FilterReferenceMultiTarget) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*FilterReferenceMultiTarget) ProtoMessage() {} - -func (x *FilterReferenceMultiTarget) ProtoReflect() protoreflect.Message { - mi := &file_v1_base_proto_msgTypes[13] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use FilterReferenceMultiTarget.ProtoReflect.Descriptor instead. -func (*FilterReferenceMultiTarget) Descriptor() ([]byte, []int) { - return file_v1_base_proto_rawDescGZIP(), []int{13} -} - -func (x *FilterReferenceMultiTarget) GetOn() string { - if x != nil { - return x.On - } - return "" -} - -func (x *FilterReferenceMultiTarget) GetTarget() *FilterTarget { - if x != nil { - return x.Target - } - return nil -} - -func (x *FilterReferenceMultiTarget) GetTargetCollection() string { - if x != nil { - return x.TargetCollection - } - return "" -} - -type FilterTarget struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Types that are assignable to Target: - // - // *FilterTarget_Property - // *FilterTarget_SingleTarget - // *FilterTarget_MultiTarget - Target isFilterTarget_Target `protobuf_oneof:"target"` -} - -func (x *FilterTarget) Reset() { - *x = FilterTarget{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_base_proto_msgTypes[14] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *FilterTarget) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*FilterTarget) ProtoMessage() {} - -func (x *FilterTarget) ProtoReflect() protoreflect.Message { - mi := &file_v1_base_proto_msgTypes[14] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use FilterTarget.ProtoReflect.Descriptor instead. -func (*FilterTarget) Descriptor() ([]byte, []int) { - return file_v1_base_proto_rawDescGZIP(), []int{14} -} - -func (m *FilterTarget) GetTarget() isFilterTarget_Target { - if m != nil { - return m.Target - } - return nil -} - -func (x *FilterTarget) GetProperty() string { - if x, ok := x.GetTarget().(*FilterTarget_Property); ok { - return x.Property - } - return "" -} - -func (x *FilterTarget) GetSingleTarget() *FilterReferenceSingleTarget { - if x, ok := x.GetTarget().(*FilterTarget_SingleTarget); ok { - return x.SingleTarget - } - return nil -} - -func (x *FilterTarget) GetMultiTarget() *FilterReferenceMultiTarget { - if x, ok := x.GetTarget().(*FilterTarget_MultiTarget); ok { - return x.MultiTarget - } - return nil -} - -type isFilterTarget_Target interface { - isFilterTarget_Target() -} - -type FilterTarget_Property struct { - Property string `protobuf:"bytes,1,opt,name=property,proto3,oneof"` -} - -type FilterTarget_SingleTarget struct { - SingleTarget *FilterReferenceSingleTarget `protobuf:"bytes,2,opt,name=single_target,json=singleTarget,proto3,oneof"` -} - -type FilterTarget_MultiTarget struct { - MultiTarget *FilterReferenceMultiTarget `protobuf:"bytes,3,opt,name=multi_target,json=multiTarget,proto3,oneof"` -} - -func (*FilterTarget_Property) isFilterTarget_Target() {} - -func (*FilterTarget_SingleTarget) isFilterTarget_Target() {} - -func (*FilterTarget_MultiTarget) isFilterTarget_Target() {} - -type GeoCoordinatesFilter struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Latitude float32 `protobuf:"fixed32,1,opt,name=latitude,proto3" json:"latitude,omitempty"` - Longitude float32 `protobuf:"fixed32,2,opt,name=longitude,proto3" json:"longitude,omitempty"` - Distance float32 `protobuf:"fixed32,3,opt,name=distance,proto3" json:"distance,omitempty"` -} - -func (x *GeoCoordinatesFilter) Reset() { - *x = GeoCoordinatesFilter{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_base_proto_msgTypes[15] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GeoCoordinatesFilter) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GeoCoordinatesFilter) ProtoMessage() {} - -func (x *GeoCoordinatesFilter) ProtoReflect() protoreflect.Message { - mi := &file_v1_base_proto_msgTypes[15] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GeoCoordinatesFilter.ProtoReflect.Descriptor instead. -func (*GeoCoordinatesFilter) Descriptor() ([]byte, []int) { - return file_v1_base_proto_rawDescGZIP(), []int{15} -} - -func (x *GeoCoordinatesFilter) GetLatitude() float32 { - if x != nil { - return x.Latitude - } - return 0 -} - -func (x *GeoCoordinatesFilter) GetLongitude() float32 { - if x != nil { - return x.Longitude - } - return 0 -} - -func (x *GeoCoordinatesFilter) GetDistance() float32 { - if x != nil { - return x.Distance - } - return 0 -} - -var File_v1_base_proto protoreflect.FileDescriptor - -var file_v1_base_proto_rawDesc = []byte{ - 0x0a, 0x0d, 0x76, 0x31, 0x2f, 0x62, 0x61, 0x73, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, - 0x0b, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x1a, 0x1c, 0x67, 0x6f, - 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x73, 0x74, - 0x72, 0x75, 0x63, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x73, 0x0a, 0x15, 0x4e, 0x75, - 0x6d, 0x62, 0x65, 0x72, 0x41, 0x72, 0x72, 0x61, 0x79, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, - 0x69, 0x65, 0x73, 0x12, 0x1a, 0x0a, 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x01, 0x20, - 0x03, 0x28, 0x01, 0x42, 0x02, 0x18, 0x01, 0x52, 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x12, - 0x1b, 0x0a, 0x09, 0x70, 0x72, 0x6f, 0x70, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x21, 0x0a, 0x0c, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x0c, 0x52, 0x0b, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x42, 0x79, 0x74, 0x65, 0x73, 0x22, - 0x49, 0x0a, 0x12, 0x49, 0x6e, 0x74, 0x41, 0x72, 0x72, 0x61, 0x79, 0x50, 0x72, 0x6f, 0x70, 0x65, - 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, - 0x01, 0x20, 0x03, 0x28, 0x03, 0x52, 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x12, 0x1b, 0x0a, - 0x09, 0x70, 0x72, 0x6f, 0x70, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x08, 0x70, 0x72, 0x6f, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x4a, 0x0a, 0x13, 0x54, 0x65, - 0x78, 0x74, 0x41, 0x72, 0x72, 0x61, 0x79, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, - 0x73, 0x12, 0x16, 0x0a, 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, - 0x09, 0x52, 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x70, 0x72, 0x6f, - 0x70, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, - 0x6f, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x4d, 0x0a, 0x16, 0x42, 0x6f, 0x6f, 0x6c, 0x65, 0x61, - 0x6e, 0x41, 0x72, 0x72, 0x61, 0x79, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, - 0x12, 0x16, 0x0a, 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x08, - 0x52, 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x70, 0x72, 0x6f, 0x70, - 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, - 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0xea, 0x04, 0x0a, 0x15, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, - 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, - 0x45, 0x0a, 0x12, 0x6e, 0x6f, 0x6e, 0x5f, 0x72, 0x65, 0x66, 0x5f, 0x70, 0x72, 0x6f, 0x70, 0x65, - 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x67, 0x6f, - 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, - 0x72, 0x75, 0x63, 0x74, 0x52, 0x10, 0x6e, 0x6f, 0x6e, 0x52, 0x65, 0x66, 0x50, 0x72, 0x6f, 0x70, - 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x5a, 0x0a, 0x17, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, - 0x5f, 0x61, 0x72, 0x72, 0x61, 0x79, 0x5f, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, - 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, - 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x41, 0x72, 0x72, 0x61, - 0x79, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x52, 0x15, 0x6e, 0x75, 0x6d, - 0x62, 0x65, 0x72, 0x41, 0x72, 0x72, 0x61, 0x79, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, - 0x65, 0x73, 0x12, 0x51, 0x0a, 0x14, 0x69, 0x6e, 0x74, 0x5f, 0x61, 0x72, 0x72, 0x61, 0x79, 0x5f, - 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x1f, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x49, - 0x6e, 0x74, 0x41, 0x72, 0x72, 0x61, 0x79, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, - 0x73, 0x52, 0x12, 0x69, 0x6e, 0x74, 0x41, 0x72, 0x72, 0x61, 0x79, 0x50, 0x72, 0x6f, 0x70, 0x65, - 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x54, 0x0a, 0x15, 0x74, 0x65, 0x78, 0x74, 0x5f, 0x61, 0x72, - 0x72, 0x61, 0x79, 0x5f, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x04, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, - 0x76, 0x31, 0x2e, 0x54, 0x65, 0x78, 0x74, 0x41, 0x72, 0x72, 0x61, 0x79, 0x50, 0x72, 0x6f, 0x70, - 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x52, 0x13, 0x74, 0x65, 0x78, 0x74, 0x41, 0x72, 0x72, 0x61, - 0x79, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x5d, 0x0a, 0x18, 0x62, - 0x6f, 0x6f, 0x6c, 0x65, 0x61, 0x6e, 0x5f, 0x61, 0x72, 0x72, 0x61, 0x79, 0x5f, 0x70, 0x72, 0x6f, - 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, - 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x6f, 0x6f, 0x6c, - 0x65, 0x61, 0x6e, 0x41, 0x72, 0x72, 0x61, 0x79, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, - 0x65, 0x73, 0x52, 0x16, 0x62, 0x6f, 0x6f, 0x6c, 0x65, 0x61, 0x6e, 0x41, 0x72, 0x72, 0x61, 0x79, - 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x4a, 0x0a, 0x11, 0x6f, 0x62, - 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, - 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, - 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, - 0x74, 0x69, 0x65, 0x73, 0x52, 0x10, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x50, 0x72, 0x6f, 0x70, - 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x5a, 0x0a, 0x17, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, - 0x5f, 0x61, 0x72, 0x72, 0x61, 0x79, 0x5f, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, - 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, - 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x41, 0x72, 0x72, 0x61, - 0x79, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x52, 0x15, 0x6f, 0x62, 0x6a, - 0x65, 0x63, 0x74, 0x41, 0x72, 0x72, 0x61, 0x79, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, - 0x65, 0x73, 0x22, 0x70, 0x0a, 0x15, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x41, 0x72, 0x72, 0x61, - 0x79, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x3a, 0x0a, 0x06, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x77, 0x65, - 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, - 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, - 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x70, 0x72, 0x6f, 0x70, 0x5f, - 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x70, - 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x69, 0x0a, 0x10, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x50, 0x72, - 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x38, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, - 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x50, 0x72, 0x6f, 0x70, - 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x70, 0x72, 0x6f, 0x70, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x22, - 0x23, 0x0a, 0x09, 0x54, 0x65, 0x78, 0x74, 0x41, 0x72, 0x72, 0x61, 0x79, 0x12, 0x16, 0x0a, 0x06, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x73, 0x22, 0x22, 0x0a, 0x08, 0x49, 0x6e, 0x74, 0x41, 0x72, 0x72, 0x61, 0x79, - 0x12, 0x16, 0x0a, 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x03, - 0x52, 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x22, 0x25, 0x0a, 0x0b, 0x4e, 0x75, 0x6d, 0x62, - 0x65, 0x72, 0x41, 0x72, 0x72, 0x61, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x01, 0x52, 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x22, - 0x26, 0x0a, 0x0c, 0x42, 0x6f, 0x6f, 0x6c, 0x65, 0x61, 0x6e, 0x41, 0x72, 0x72, 0x61, 0x79, 0x12, - 0x16, 0x0a, 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x08, 0x52, - 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x22, 0x99, 0x08, 0x0a, 0x07, 0x46, 0x69, 0x6c, 0x74, - 0x65, 0x72, 0x73, 0x12, 0x39, 0x0a, 0x08, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1d, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, - 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x2e, 0x4f, 0x70, 0x65, 0x72, - 0x61, 0x74, 0x6f, 0x72, 0x52, 0x08, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x12, 0x12, - 0x0a, 0x02, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x42, 0x02, 0x18, 0x01, 0x52, 0x02, - 0x6f, 0x6e, 0x12, 0x2e, 0x0a, 0x07, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x18, 0x03, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, - 0x31, 0x2e, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x52, 0x07, 0x66, 0x69, 0x6c, 0x74, 0x65, - 0x72, 0x73, 0x12, 0x1f, 0x0a, 0x0a, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x5f, 0x74, 0x65, 0x78, 0x74, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x09, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x54, - 0x65, 0x78, 0x74, 0x12, 0x1d, 0x0a, 0x09, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x5f, 0x69, 0x6e, 0x74, - 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x48, 0x00, 0x52, 0x08, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x49, - 0x6e, 0x74, 0x12, 0x25, 0x0a, 0x0d, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x5f, 0x62, 0x6f, 0x6f, 0x6c, - 0x65, 0x61, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, 0x0c, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x42, 0x6f, 0x6f, 0x6c, 0x65, 0x61, 0x6e, 0x12, 0x23, 0x0a, 0x0c, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x5f, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x07, 0x20, 0x01, 0x28, 0x01, 0x48, - 0x00, 0x52, 0x0b, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, 0x42, - 0x0a, 0x10, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x5f, 0x74, 0x65, 0x78, 0x74, 0x5f, 0x61, 0x72, 0x72, - 0x61, 0x79, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, - 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x65, 0x78, 0x74, 0x41, 0x72, 0x72, 0x61, 0x79, - 0x48, 0x00, 0x52, 0x0e, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x54, 0x65, 0x78, 0x74, 0x41, 0x72, 0x72, - 0x61, 0x79, 0x12, 0x3f, 0x0a, 0x0f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x5f, 0x69, 0x6e, 0x74, 0x5f, - 0x61, 0x72, 0x72, 0x61, 0x79, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x77, 0x65, - 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x6e, 0x74, 0x41, 0x72, 0x72, - 0x61, 0x79, 0x48, 0x00, 0x52, 0x0d, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x49, 0x6e, 0x74, 0x41, 0x72, - 0x72, 0x61, 0x79, 0x12, 0x4b, 0x0a, 0x13, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x5f, 0x62, 0x6f, 0x6f, - 0x6c, 0x65, 0x61, 0x6e, 0x5f, 0x61, 0x72, 0x72, 0x61, 0x79, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x19, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x42, - 0x6f, 0x6f, 0x6c, 0x65, 0x61, 0x6e, 0x41, 0x72, 0x72, 0x61, 0x79, 0x48, 0x00, 0x52, 0x11, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x42, 0x6f, 0x6f, 0x6c, 0x65, 0x61, 0x6e, 0x41, 0x72, 0x72, 0x61, 0x79, - 0x12, 0x48, 0x0a, 0x12, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x5f, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, - 0x5f, 0x61, 0x72, 0x72, 0x61, 0x79, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x77, - 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x75, 0x6d, 0x62, 0x65, - 0x72, 0x41, 0x72, 0x72, 0x61, 0x79, 0x48, 0x00, 0x52, 0x10, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x4e, - 0x75, 0x6d, 0x62, 0x65, 0x72, 0x41, 0x72, 0x72, 0x61, 0x79, 0x12, 0x40, 0x0a, 0x09, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x5f, 0x67, 0x65, 0x6f, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, - 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x6f, 0x43, - 0x6f, 0x6f, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x73, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, - 0x48, 0x00, 0x52, 0x08, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x47, 0x65, 0x6f, 0x12, 0x31, 0x0a, 0x06, - 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x14, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x77, - 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x69, 0x6c, 0x74, 0x65, - 0x72, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x22, - 0xe3, 0x02, 0x0a, 0x08, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x12, 0x18, 0x0a, 0x14, - 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x4f, 0x52, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, - 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x12, 0x0a, 0x0e, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, - 0x4f, 0x52, 0x5f, 0x45, 0x51, 0x55, 0x41, 0x4c, 0x10, 0x01, 0x12, 0x16, 0x0a, 0x12, 0x4f, 0x50, - 0x45, 0x52, 0x41, 0x54, 0x4f, 0x52, 0x5f, 0x4e, 0x4f, 0x54, 0x5f, 0x45, 0x51, 0x55, 0x41, 0x4c, - 0x10, 0x02, 0x12, 0x19, 0x0a, 0x15, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x4f, 0x52, 0x5f, 0x47, - 0x52, 0x45, 0x41, 0x54, 0x45, 0x52, 0x5f, 0x54, 0x48, 0x41, 0x4e, 0x10, 0x03, 0x12, 0x1f, 0x0a, - 0x1b, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x4f, 0x52, 0x5f, 0x47, 0x52, 0x45, 0x41, 0x54, 0x45, - 0x52, 0x5f, 0x54, 0x48, 0x41, 0x4e, 0x5f, 0x45, 0x51, 0x55, 0x41, 0x4c, 0x10, 0x04, 0x12, 0x16, - 0x0a, 0x12, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x4f, 0x52, 0x5f, 0x4c, 0x45, 0x53, 0x53, 0x5f, - 0x54, 0x48, 0x41, 0x4e, 0x10, 0x05, 0x12, 0x1c, 0x0a, 0x18, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, - 0x4f, 0x52, 0x5f, 0x4c, 0x45, 0x53, 0x53, 0x5f, 0x54, 0x48, 0x41, 0x4e, 0x5f, 0x45, 0x51, 0x55, - 0x41, 0x4c, 0x10, 0x06, 0x12, 0x10, 0x0a, 0x0c, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x4f, 0x52, - 0x5f, 0x41, 0x4e, 0x44, 0x10, 0x07, 0x12, 0x0f, 0x0a, 0x0b, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, - 0x4f, 0x52, 0x5f, 0x4f, 0x52, 0x10, 0x08, 0x12, 0x1d, 0x0a, 0x19, 0x4f, 0x50, 0x45, 0x52, 0x41, - 0x54, 0x4f, 0x52, 0x5f, 0x57, 0x49, 0x54, 0x48, 0x49, 0x4e, 0x5f, 0x47, 0x45, 0x4f, 0x5f, 0x52, - 0x41, 0x4e, 0x47, 0x45, 0x10, 0x09, 0x12, 0x11, 0x0a, 0x0d, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, - 0x4f, 0x52, 0x5f, 0x4c, 0x49, 0x4b, 0x45, 0x10, 0x0a, 0x12, 0x14, 0x0a, 0x10, 0x4f, 0x50, 0x45, - 0x52, 0x41, 0x54, 0x4f, 0x52, 0x5f, 0x49, 0x53, 0x5f, 0x4e, 0x55, 0x4c, 0x4c, 0x10, 0x0b, 0x12, - 0x19, 0x0a, 0x15, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x4f, 0x52, 0x5f, 0x43, 0x4f, 0x4e, 0x54, - 0x41, 0x49, 0x4e, 0x53, 0x5f, 0x41, 0x4e, 0x59, 0x10, 0x0c, 0x12, 0x19, 0x0a, 0x15, 0x4f, 0x50, - 0x45, 0x52, 0x41, 0x54, 0x4f, 0x52, 0x5f, 0x43, 0x4f, 0x4e, 0x54, 0x41, 0x49, 0x4e, 0x53, 0x5f, - 0x41, 0x4c, 0x4c, 0x10, 0x0d, 0x42, 0x0c, 0x0a, 0x0a, 0x74, 0x65, 0x73, 0x74, 0x5f, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x22, 0x60, 0x0a, 0x1b, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x65, 0x66, - 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x53, 0x69, 0x6e, 0x67, 0x6c, 0x65, 0x54, 0x61, 0x72, 0x67, - 0x65, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, - 0x6f, 0x6e, 0x12, 0x31, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, - 0x2e, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, - 0x61, 0x72, 0x67, 0x65, 0x74, 0x22, 0x8c, 0x01, 0x0a, 0x1a, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, - 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x54, 0x61, - 0x72, 0x67, 0x65, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x02, 0x6f, 0x6e, 0x12, 0x31, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, - 0x76, 0x31, 0x2e, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, - 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x2b, 0x0a, 0x11, 0x74, 0x61, 0x72, 0x67, 0x65, - 0x74, 0x5f, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x10, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xd5, 0x01, 0x0a, 0x0c, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x54, - 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x1c, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, - 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x70, 0x65, - 0x72, 0x74, 0x79, 0x12, 0x4f, 0x0a, 0x0d, 0x73, 0x69, 0x6e, 0x67, 0x6c, 0x65, 0x5f, 0x74, 0x61, - 0x72, 0x67, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x77, 0x65, 0x61, - 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, - 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x53, 0x69, 0x6e, 0x67, 0x6c, 0x65, 0x54, 0x61, - 0x72, 0x67, 0x65, 0x74, 0x48, 0x00, 0x52, 0x0c, 0x73, 0x69, 0x6e, 0x67, 0x6c, 0x65, 0x54, 0x61, - 0x72, 0x67, 0x65, 0x74, 0x12, 0x4c, 0x0a, 0x0c, 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x5f, 0x74, 0x61, - 0x72, 0x67, 0x65, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x77, 0x65, 0x61, - 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, - 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x54, 0x61, 0x72, - 0x67, 0x65, 0x74, 0x48, 0x00, 0x52, 0x0b, 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x54, 0x61, 0x72, 0x67, - 0x65, 0x74, 0x42, 0x08, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x22, 0x6c, 0x0a, 0x14, - 0x47, 0x65, 0x6f, 0x43, 0x6f, 0x6f, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x73, 0x46, 0x69, - 0x6c, 0x74, 0x65, 0x72, 0x12, 0x1a, 0x0a, 0x08, 0x6c, 0x61, 0x74, 0x69, 0x74, 0x75, 0x64, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x02, 0x52, 0x08, 0x6c, 0x61, 0x74, 0x69, 0x74, 0x75, 0x64, 0x65, - 0x12, 0x1c, 0x0a, 0x09, 0x6c, 0x6f, 0x6e, 0x67, 0x69, 0x74, 0x75, 0x64, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x02, 0x52, 0x09, 0x6c, 0x6f, 0x6e, 0x67, 0x69, 0x74, 0x75, 0x64, 0x65, 0x12, 0x1a, - 0x0a, 0x08, 0x64, 0x69, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x02, - 0x52, 0x08, 0x64, 0x69, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x2a, 0x89, 0x01, 0x0a, 0x10, 0x43, - 0x6f, 0x6e, 0x73, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, - 0x21, 0x0a, 0x1d, 0x43, 0x4f, 0x4e, 0x53, 0x49, 0x53, 0x54, 0x45, 0x4e, 0x43, 0x59, 0x5f, 0x4c, - 0x45, 0x56, 0x45, 0x4c, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, - 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x43, 0x4f, 0x4e, 0x53, 0x49, 0x53, 0x54, 0x45, 0x4e, 0x43, - 0x59, 0x5f, 0x4c, 0x45, 0x56, 0x45, 0x4c, 0x5f, 0x4f, 0x4e, 0x45, 0x10, 0x01, 0x12, 0x1c, 0x0a, - 0x18, 0x43, 0x4f, 0x4e, 0x53, 0x49, 0x53, 0x54, 0x45, 0x4e, 0x43, 0x59, 0x5f, 0x4c, 0x45, 0x56, - 0x45, 0x4c, 0x5f, 0x51, 0x55, 0x4f, 0x52, 0x55, 0x4d, 0x10, 0x02, 0x12, 0x19, 0x0a, 0x15, 0x43, - 0x4f, 0x4e, 0x53, 0x49, 0x53, 0x54, 0x45, 0x4e, 0x43, 0x59, 0x5f, 0x4c, 0x45, 0x56, 0x45, 0x4c, - 0x5f, 0x41, 0x4c, 0x4c, 0x10, 0x03, 0x42, 0x6e, 0x0a, 0x23, 0x69, 0x6f, 0x2e, 0x77, 0x65, 0x61, - 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x2e, 0x67, 0x72, 0x70, - 0x63, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x76, 0x31, 0x42, 0x11, 0x57, - 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x42, 0x61, 0x73, 0x65, - 0x5a, 0x34, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x77, 0x65, 0x61, - 0x76, 0x69, 0x61, 0x74, 0x65, 0x2f, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2f, 0x67, - 0x72, 0x70, 0x63, 0x2f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x3b, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -} - -var ( - file_v1_base_proto_rawDescOnce sync.Once - file_v1_base_proto_rawDescData = file_v1_base_proto_rawDesc -) - -func file_v1_base_proto_rawDescGZIP() []byte { - file_v1_base_proto_rawDescOnce.Do(func() { - file_v1_base_proto_rawDescData = protoimpl.X.CompressGZIP(file_v1_base_proto_rawDescData) - }) - return file_v1_base_proto_rawDescData -} - -var file_v1_base_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_v1_base_proto_msgTypes = make([]protoimpl.MessageInfo, 16) -var file_v1_base_proto_goTypes = []interface{}{ - (ConsistencyLevel)(0), // 0: weaviate.v1.ConsistencyLevel - (Filters_Operator)(0), // 1: weaviate.v1.Filters.Operator - (*NumberArrayProperties)(nil), // 2: weaviate.v1.NumberArrayProperties - (*IntArrayProperties)(nil), // 3: weaviate.v1.IntArrayProperties - (*TextArrayProperties)(nil), // 4: weaviate.v1.TextArrayProperties - (*BooleanArrayProperties)(nil), // 5: weaviate.v1.BooleanArrayProperties - (*ObjectPropertiesValue)(nil), // 6: weaviate.v1.ObjectPropertiesValue - (*ObjectArrayProperties)(nil), // 7: weaviate.v1.ObjectArrayProperties - (*ObjectProperties)(nil), // 8: weaviate.v1.ObjectProperties - (*TextArray)(nil), // 9: weaviate.v1.TextArray - (*IntArray)(nil), // 10: weaviate.v1.IntArray - (*NumberArray)(nil), // 11: weaviate.v1.NumberArray - (*BooleanArray)(nil), // 12: weaviate.v1.BooleanArray - (*Filters)(nil), // 13: weaviate.v1.Filters - (*FilterReferenceSingleTarget)(nil), // 14: weaviate.v1.FilterReferenceSingleTarget - (*FilterReferenceMultiTarget)(nil), // 15: weaviate.v1.FilterReferenceMultiTarget - (*FilterTarget)(nil), // 16: weaviate.v1.FilterTarget - (*GeoCoordinatesFilter)(nil), // 17: weaviate.v1.GeoCoordinatesFilter - (*structpb.Struct)(nil), // 18: google.protobuf.Struct -} -var file_v1_base_proto_depIdxs = []int32{ - 18, // 0: weaviate.v1.ObjectPropertiesValue.non_ref_properties:type_name -> google.protobuf.Struct - 2, // 1: weaviate.v1.ObjectPropertiesValue.number_array_properties:type_name -> weaviate.v1.NumberArrayProperties - 3, // 2: weaviate.v1.ObjectPropertiesValue.int_array_properties:type_name -> weaviate.v1.IntArrayProperties - 4, // 3: weaviate.v1.ObjectPropertiesValue.text_array_properties:type_name -> weaviate.v1.TextArrayProperties - 5, // 4: weaviate.v1.ObjectPropertiesValue.boolean_array_properties:type_name -> weaviate.v1.BooleanArrayProperties - 8, // 5: weaviate.v1.ObjectPropertiesValue.object_properties:type_name -> weaviate.v1.ObjectProperties - 7, // 6: weaviate.v1.ObjectPropertiesValue.object_array_properties:type_name -> weaviate.v1.ObjectArrayProperties - 6, // 7: weaviate.v1.ObjectArrayProperties.values:type_name -> weaviate.v1.ObjectPropertiesValue - 6, // 8: weaviate.v1.ObjectProperties.value:type_name -> weaviate.v1.ObjectPropertiesValue - 1, // 9: weaviate.v1.Filters.operator:type_name -> weaviate.v1.Filters.Operator - 13, // 10: weaviate.v1.Filters.filters:type_name -> weaviate.v1.Filters - 9, // 11: weaviate.v1.Filters.value_text_array:type_name -> weaviate.v1.TextArray - 10, // 12: weaviate.v1.Filters.value_int_array:type_name -> weaviate.v1.IntArray - 12, // 13: weaviate.v1.Filters.value_boolean_array:type_name -> weaviate.v1.BooleanArray - 11, // 14: weaviate.v1.Filters.value_number_array:type_name -> weaviate.v1.NumberArray - 17, // 15: weaviate.v1.Filters.value_geo:type_name -> weaviate.v1.GeoCoordinatesFilter - 16, // 16: weaviate.v1.Filters.target:type_name -> weaviate.v1.FilterTarget - 16, // 17: weaviate.v1.FilterReferenceSingleTarget.target:type_name -> weaviate.v1.FilterTarget - 16, // 18: weaviate.v1.FilterReferenceMultiTarget.target:type_name -> weaviate.v1.FilterTarget - 14, // 19: weaviate.v1.FilterTarget.single_target:type_name -> weaviate.v1.FilterReferenceSingleTarget - 15, // 20: weaviate.v1.FilterTarget.multi_target:type_name -> weaviate.v1.FilterReferenceMultiTarget - 21, // [21:21] is the sub-list for method output_type - 21, // [21:21] is the sub-list for method input_type - 21, // [21:21] is the sub-list for extension type_name - 21, // [21:21] is the sub-list for extension extendee - 0, // [0:21] is the sub-list for field type_name -} - -func init() { file_v1_base_proto_init() } -func file_v1_base_proto_init() { - if File_v1_base_proto != nil { - return - } - if !protoimpl.UnsafeEnabled { - file_v1_base_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NumberArrayProperties); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_base_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*IntArrayProperties); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_base_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TextArrayProperties); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_base_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*BooleanArrayProperties); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_base_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ObjectPropertiesValue); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_base_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ObjectArrayProperties); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_base_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ObjectProperties); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_base_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TextArray); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_base_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*IntArray); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_base_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NumberArray); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_base_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*BooleanArray); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_base_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Filters); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_base_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FilterReferenceSingleTarget); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_base_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FilterReferenceMultiTarget); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_base_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FilterTarget); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_base_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GeoCoordinatesFilter); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } - file_v1_base_proto_msgTypes[11].OneofWrappers = []interface{}{ - (*Filters_ValueText)(nil), - (*Filters_ValueInt)(nil), - (*Filters_ValueBoolean)(nil), - (*Filters_ValueNumber)(nil), - (*Filters_ValueTextArray)(nil), - (*Filters_ValueIntArray)(nil), - (*Filters_ValueBooleanArray)(nil), - (*Filters_ValueNumberArray)(nil), - (*Filters_ValueGeo)(nil), - } - file_v1_base_proto_msgTypes[14].OneofWrappers = []interface{}{ - (*FilterTarget_Property)(nil), - (*FilterTarget_SingleTarget)(nil), - (*FilterTarget_MultiTarget)(nil), - } - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_v1_base_proto_rawDesc, - NumEnums: 2, - NumMessages: 16, - NumExtensions: 0, - NumServices: 0, - }, - GoTypes: file_v1_base_proto_goTypes, - DependencyIndexes: file_v1_base_proto_depIdxs, - EnumInfos: file_v1_base_proto_enumTypes, - MessageInfos: file_v1_base_proto_msgTypes, - }.Build() - File_v1_base_proto = out.File - file_v1_base_proto_rawDesc = nil - file_v1_base_proto_goTypes = nil - file_v1_base_proto_depIdxs = nil -} diff --git a/grpc/generated/protocol/v1/batch.pb.go b/grpc/generated/protocol/v1/batch.pb.go deleted file mode 100644 index 6a450f04893dd477f4666e919160de6a6b423d2f..0000000000000000000000000000000000000000 --- a/grpc/generated/protocol/v1/batch.pb.go +++ /dev/null @@ -1,784 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. - -package protocol - -import ( - reflect "reflect" - sync "sync" - - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" - structpb "google.golang.org/protobuf/types/known/structpb" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -type BatchObjectsRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Objects []*BatchObject `protobuf:"bytes,1,rep,name=objects,proto3" json:"objects,omitempty"` - ConsistencyLevel *ConsistencyLevel `protobuf:"varint,2,opt,name=consistency_level,json=consistencyLevel,proto3,enum=weaviate.v1.ConsistencyLevel,oneof" json:"consistency_level,omitempty"` -} - -func (x *BatchObjectsRequest) Reset() { - *x = BatchObjectsRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_batch_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *BatchObjectsRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*BatchObjectsRequest) ProtoMessage() {} - -func (x *BatchObjectsRequest) ProtoReflect() protoreflect.Message { - mi := &file_v1_batch_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use BatchObjectsRequest.ProtoReflect.Descriptor instead. -func (*BatchObjectsRequest) Descriptor() ([]byte, []int) { - return file_v1_batch_proto_rawDescGZIP(), []int{0} -} - -func (x *BatchObjectsRequest) GetObjects() []*BatchObject { - if x != nil { - return x.Objects - } - return nil -} - -func (x *BatchObjectsRequest) GetConsistencyLevel() ConsistencyLevel { - if x != nil && x.ConsistencyLevel != nil { - return *x.ConsistencyLevel - } - return ConsistencyLevel_CONSISTENCY_LEVEL_UNSPECIFIED -} - -type BatchObject struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Uuid string `protobuf:"bytes,1,opt,name=uuid,proto3" json:"uuid,omitempty"` - // protolint:disable:next REPEATED_FIELD_NAMES_PLURALIZED - // - // Deprecated: Marked as deprecated in v1/batch.proto. - Vector []float32 `protobuf:"fixed32,2,rep,packed,name=vector,proto3" json:"vector,omitempty"` // deprecated, will be removed - Properties *BatchObject_Properties `protobuf:"bytes,3,opt,name=properties,proto3" json:"properties,omitempty"` - Collection string `protobuf:"bytes,4,opt,name=collection,proto3" json:"collection,omitempty"` - Tenant string `protobuf:"bytes,5,opt,name=tenant,proto3" json:"tenant,omitempty"` - VectorBytes []byte `protobuf:"bytes,6,opt,name=vector_bytes,json=vectorBytes,proto3" json:"vector_bytes,omitempty"` -} - -func (x *BatchObject) Reset() { - *x = BatchObject{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_batch_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *BatchObject) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*BatchObject) ProtoMessage() {} - -func (x *BatchObject) ProtoReflect() protoreflect.Message { - mi := &file_v1_batch_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use BatchObject.ProtoReflect.Descriptor instead. -func (*BatchObject) Descriptor() ([]byte, []int) { - return file_v1_batch_proto_rawDescGZIP(), []int{1} -} - -func (x *BatchObject) GetUuid() string { - if x != nil { - return x.Uuid - } - return "" -} - -// Deprecated: Marked as deprecated in v1/batch.proto. -func (x *BatchObject) GetVector() []float32 { - if x != nil { - return x.Vector - } - return nil -} - -func (x *BatchObject) GetProperties() *BatchObject_Properties { - if x != nil { - return x.Properties - } - return nil -} - -func (x *BatchObject) GetCollection() string { - if x != nil { - return x.Collection - } - return "" -} - -func (x *BatchObject) GetTenant() string { - if x != nil { - return x.Tenant - } - return "" -} - -func (x *BatchObject) GetVectorBytes() []byte { - if x != nil { - return x.VectorBytes - } - return nil -} - -type BatchObjectsReply struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Took float32 `protobuf:"fixed32,1,opt,name=took,proto3" json:"took,omitempty"` - Errors []*BatchObjectsReply_BatchError `protobuf:"bytes,2,rep,name=errors,proto3" json:"errors,omitempty"` -} - -func (x *BatchObjectsReply) Reset() { - *x = BatchObjectsReply{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_batch_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *BatchObjectsReply) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*BatchObjectsReply) ProtoMessage() {} - -func (x *BatchObjectsReply) ProtoReflect() protoreflect.Message { - mi := &file_v1_batch_proto_msgTypes[2] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use BatchObjectsReply.ProtoReflect.Descriptor instead. -func (*BatchObjectsReply) Descriptor() ([]byte, []int) { - return file_v1_batch_proto_rawDescGZIP(), []int{2} -} - -func (x *BatchObjectsReply) GetTook() float32 { - if x != nil { - return x.Took - } - return 0 -} - -func (x *BatchObjectsReply) GetErrors() []*BatchObjectsReply_BatchError { - if x != nil { - return x.Errors - } - return nil -} - -type BatchObject_Properties struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - NonRefProperties *structpb.Struct `protobuf:"bytes,1,opt,name=non_ref_properties,json=nonRefProperties,proto3" json:"non_ref_properties,omitempty"` - SingleTargetRefProps []*BatchObject_SingleTargetRefProps `protobuf:"bytes,2,rep,name=single_target_ref_props,json=singleTargetRefProps,proto3" json:"single_target_ref_props,omitempty"` - MultiTargetRefProps []*BatchObject_MultiTargetRefProps `protobuf:"bytes,3,rep,name=multi_target_ref_props,json=multiTargetRefProps,proto3" json:"multi_target_ref_props,omitempty"` - NumberArrayProperties []*NumberArrayProperties `protobuf:"bytes,4,rep,name=number_array_properties,json=numberArrayProperties,proto3" json:"number_array_properties,omitempty"` - IntArrayProperties []*IntArrayProperties `protobuf:"bytes,5,rep,name=int_array_properties,json=intArrayProperties,proto3" json:"int_array_properties,omitempty"` - TextArrayProperties []*TextArrayProperties `protobuf:"bytes,6,rep,name=text_array_properties,json=textArrayProperties,proto3" json:"text_array_properties,omitempty"` - BooleanArrayProperties []*BooleanArrayProperties `protobuf:"bytes,7,rep,name=boolean_array_properties,json=booleanArrayProperties,proto3" json:"boolean_array_properties,omitempty"` - ObjectProperties []*ObjectProperties `protobuf:"bytes,8,rep,name=object_properties,json=objectProperties,proto3" json:"object_properties,omitempty"` - ObjectArrayProperties []*ObjectArrayProperties `protobuf:"bytes,9,rep,name=object_array_properties,json=objectArrayProperties,proto3" json:"object_array_properties,omitempty"` -} - -func (x *BatchObject_Properties) Reset() { - *x = BatchObject_Properties{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_batch_proto_msgTypes[3] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *BatchObject_Properties) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*BatchObject_Properties) ProtoMessage() {} - -func (x *BatchObject_Properties) ProtoReflect() protoreflect.Message { - mi := &file_v1_batch_proto_msgTypes[3] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use BatchObject_Properties.ProtoReflect.Descriptor instead. -func (*BatchObject_Properties) Descriptor() ([]byte, []int) { - return file_v1_batch_proto_rawDescGZIP(), []int{1, 0} -} - -func (x *BatchObject_Properties) GetNonRefProperties() *structpb.Struct { - if x != nil { - return x.NonRefProperties - } - return nil -} - -func (x *BatchObject_Properties) GetSingleTargetRefProps() []*BatchObject_SingleTargetRefProps { - if x != nil { - return x.SingleTargetRefProps - } - return nil -} - -func (x *BatchObject_Properties) GetMultiTargetRefProps() []*BatchObject_MultiTargetRefProps { - if x != nil { - return x.MultiTargetRefProps - } - return nil -} - -func (x *BatchObject_Properties) GetNumberArrayProperties() []*NumberArrayProperties { - if x != nil { - return x.NumberArrayProperties - } - return nil -} - -func (x *BatchObject_Properties) GetIntArrayProperties() []*IntArrayProperties { - if x != nil { - return x.IntArrayProperties - } - return nil -} - -func (x *BatchObject_Properties) GetTextArrayProperties() []*TextArrayProperties { - if x != nil { - return x.TextArrayProperties - } - return nil -} - -func (x *BatchObject_Properties) GetBooleanArrayProperties() []*BooleanArrayProperties { - if x != nil { - return x.BooleanArrayProperties - } - return nil -} - -func (x *BatchObject_Properties) GetObjectProperties() []*ObjectProperties { - if x != nil { - return x.ObjectProperties - } - return nil -} - -func (x *BatchObject_Properties) GetObjectArrayProperties() []*ObjectArrayProperties { - if x != nil { - return x.ObjectArrayProperties - } - return nil -} - -type BatchObject_SingleTargetRefProps struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Uuids []string `protobuf:"bytes,1,rep,name=uuids,proto3" json:"uuids,omitempty"` - PropName string `protobuf:"bytes,2,opt,name=prop_name,json=propName,proto3" json:"prop_name,omitempty"` -} - -func (x *BatchObject_SingleTargetRefProps) Reset() { - *x = BatchObject_SingleTargetRefProps{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_batch_proto_msgTypes[4] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *BatchObject_SingleTargetRefProps) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*BatchObject_SingleTargetRefProps) ProtoMessage() {} - -func (x *BatchObject_SingleTargetRefProps) ProtoReflect() protoreflect.Message { - mi := &file_v1_batch_proto_msgTypes[4] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use BatchObject_SingleTargetRefProps.ProtoReflect.Descriptor instead. -func (*BatchObject_SingleTargetRefProps) Descriptor() ([]byte, []int) { - return file_v1_batch_proto_rawDescGZIP(), []int{1, 1} -} - -func (x *BatchObject_SingleTargetRefProps) GetUuids() []string { - if x != nil { - return x.Uuids - } - return nil -} - -func (x *BatchObject_SingleTargetRefProps) GetPropName() string { - if x != nil { - return x.PropName - } - return "" -} - -type BatchObject_MultiTargetRefProps struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Uuids []string `protobuf:"bytes,1,rep,name=uuids,proto3" json:"uuids,omitempty"` - PropName string `protobuf:"bytes,2,opt,name=prop_name,json=propName,proto3" json:"prop_name,omitempty"` - TargetCollection string `protobuf:"bytes,3,opt,name=target_collection,json=targetCollection,proto3" json:"target_collection,omitempty"` -} - -func (x *BatchObject_MultiTargetRefProps) Reset() { - *x = BatchObject_MultiTargetRefProps{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_batch_proto_msgTypes[5] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *BatchObject_MultiTargetRefProps) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*BatchObject_MultiTargetRefProps) ProtoMessage() {} - -func (x *BatchObject_MultiTargetRefProps) ProtoReflect() protoreflect.Message { - mi := &file_v1_batch_proto_msgTypes[5] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use BatchObject_MultiTargetRefProps.ProtoReflect.Descriptor instead. -func (*BatchObject_MultiTargetRefProps) Descriptor() ([]byte, []int) { - return file_v1_batch_proto_rawDescGZIP(), []int{1, 2} -} - -func (x *BatchObject_MultiTargetRefProps) GetUuids() []string { - if x != nil { - return x.Uuids - } - return nil -} - -func (x *BatchObject_MultiTargetRefProps) GetPropName() string { - if x != nil { - return x.PropName - } - return "" -} - -func (x *BatchObject_MultiTargetRefProps) GetTargetCollection() string { - if x != nil { - return x.TargetCollection - } - return "" -} - -type BatchObjectsReply_BatchError struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Index int32 `protobuf:"varint,1,opt,name=index,proto3" json:"index,omitempty"` - Error string `protobuf:"bytes,2,opt,name=error,proto3" json:"error,omitempty"` -} - -func (x *BatchObjectsReply_BatchError) Reset() { - *x = BatchObjectsReply_BatchError{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_batch_proto_msgTypes[6] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *BatchObjectsReply_BatchError) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*BatchObjectsReply_BatchError) ProtoMessage() {} - -func (x *BatchObjectsReply_BatchError) ProtoReflect() protoreflect.Message { - mi := &file_v1_batch_proto_msgTypes[6] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use BatchObjectsReply_BatchError.ProtoReflect.Descriptor instead. -func (*BatchObjectsReply_BatchError) Descriptor() ([]byte, []int) { - return file_v1_batch_proto_rawDescGZIP(), []int{2, 0} -} - -func (x *BatchObjectsReply_BatchError) GetIndex() int32 { - if x != nil { - return x.Index - } - return 0 -} - -func (x *BatchObjectsReply_BatchError) GetError() string { - if x != nil { - return x.Error - } - return "" -} - -var File_v1_batch_proto protoreflect.FileDescriptor - -var file_v1_batch_proto_rawDesc = []byte{ - 0x0a, 0x0e, 0x76, 0x31, 0x2f, 0x62, 0x61, 0x74, 0x63, 0x68, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x12, 0x0b, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x1a, 0x1c, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x73, - 0x74, 0x72, 0x75, 0x63, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x0d, 0x76, 0x31, 0x2f, - 0x62, 0x61, 0x73, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xb0, 0x01, 0x0a, 0x13, 0x42, - 0x61, 0x74, 0x63, 0x68, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x32, 0x0a, 0x07, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x01, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, - 0x31, 0x2e, 0x42, 0x61, 0x74, 0x63, 0x68, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x07, 0x6f, - 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x4f, 0x0a, 0x11, 0x63, 0x6f, 0x6e, 0x73, 0x69, 0x73, - 0x74, 0x65, 0x6e, 0x63, 0x79, 0x5f, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0e, 0x32, 0x1d, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, - 0x43, 0x6f, 0x6e, 0x73, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x4c, 0x65, 0x76, 0x65, 0x6c, - 0x48, 0x00, 0x52, 0x10, 0x63, 0x6f, 0x6e, 0x73, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x4c, - 0x65, 0x76, 0x65, 0x6c, 0x88, 0x01, 0x01, 0x42, 0x14, 0x0a, 0x12, 0x5f, 0x63, 0x6f, 0x6e, 0x73, - 0x69, 0x73, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x5f, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x22, 0xca, 0x09, - 0x0a, 0x0b, 0x42, 0x61, 0x74, 0x63, 0x68, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x12, 0x0a, - 0x04, 0x75, 0x75, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x75, 0x75, 0x69, - 0x64, 0x12, 0x1a, 0x0a, 0x06, 0x76, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x03, 0x28, - 0x02, 0x42, 0x02, 0x18, 0x01, 0x52, 0x06, 0x76, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x43, 0x0a, - 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x23, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, - 0x42, 0x61, 0x74, 0x63, 0x68, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x2e, 0x50, 0x72, 0x6f, 0x70, - 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x52, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, - 0x65, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x06, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x76, 0x65, - 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, - 0x52, 0x0b, 0x76, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x42, 0x79, 0x74, 0x65, 0x73, 0x1a, 0xa8, 0x06, - 0x0a, 0x0a, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x45, 0x0a, 0x12, - 0x6e, 0x6f, 0x6e, 0x5f, 0x72, 0x65, 0x66, 0x5f, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, - 0x65, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x75, 0x63, - 0x74, 0x52, 0x10, 0x6e, 0x6f, 0x6e, 0x52, 0x65, 0x66, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, - 0x69, 0x65, 0x73, 0x12, 0x64, 0x0a, 0x17, 0x73, 0x69, 0x6e, 0x67, 0x6c, 0x65, 0x5f, 0x74, 0x61, - 0x72, 0x67, 0x65, 0x74, 0x5f, 0x72, 0x65, 0x66, 0x5f, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x18, 0x02, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, - 0x76, 0x31, 0x2e, 0x42, 0x61, 0x74, 0x63, 0x68, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x2e, 0x53, - 0x69, 0x6e, 0x67, 0x6c, 0x65, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, 0x66, 0x50, 0x72, - 0x6f, 0x70, 0x73, 0x52, 0x14, 0x73, 0x69, 0x6e, 0x67, 0x6c, 0x65, 0x54, 0x61, 0x72, 0x67, 0x65, - 0x74, 0x52, 0x65, 0x66, 0x50, 0x72, 0x6f, 0x70, 0x73, 0x12, 0x61, 0x0a, 0x16, 0x6d, 0x75, 0x6c, - 0x74, 0x69, 0x5f, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x5f, 0x72, 0x65, 0x66, 0x5f, 0x70, 0x72, - 0x6f, 0x70, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x77, 0x65, 0x61, 0x76, - 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x61, 0x74, 0x63, 0x68, 0x4f, 0x62, 0x6a, - 0x65, 0x63, 0x74, 0x2e, 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, - 0x65, 0x66, 0x50, 0x72, 0x6f, 0x70, 0x73, 0x52, 0x13, 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x54, 0x61, - 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, 0x66, 0x50, 0x72, 0x6f, 0x70, 0x73, 0x12, 0x5a, 0x0a, 0x17, - 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x5f, 0x61, 0x72, 0x72, 0x61, 0x79, 0x5f, 0x70, 0x72, 0x6f, - 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x22, 0x2e, - 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x75, 0x6d, 0x62, - 0x65, 0x72, 0x41, 0x72, 0x72, 0x61, 0x79, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, - 0x73, 0x52, 0x15, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x41, 0x72, 0x72, 0x61, 0x79, 0x50, 0x72, - 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x51, 0x0a, 0x14, 0x69, 0x6e, 0x74, 0x5f, - 0x61, 0x72, 0x72, 0x61, 0x79, 0x5f, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, - 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, - 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x6e, 0x74, 0x41, 0x72, 0x72, 0x61, 0x79, 0x50, 0x72, 0x6f, - 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x52, 0x12, 0x69, 0x6e, 0x74, 0x41, 0x72, 0x72, 0x61, - 0x79, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x54, 0x0a, 0x15, 0x74, - 0x65, 0x78, 0x74, 0x5f, 0x61, 0x72, 0x72, 0x61, 0x79, 0x5f, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, - 0x74, 0x69, 0x65, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x77, 0x65, 0x61, - 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x65, 0x78, 0x74, 0x41, 0x72, 0x72, - 0x61, 0x79, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x52, 0x13, 0x74, 0x65, - 0x78, 0x74, 0x41, 0x72, 0x72, 0x61, 0x79, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, - 0x73, 0x12, 0x5d, 0x0a, 0x18, 0x62, 0x6f, 0x6f, 0x6c, 0x65, 0x61, 0x6e, 0x5f, 0x61, 0x72, 0x72, - 0x61, 0x79, 0x5f, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x07, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, - 0x31, 0x2e, 0x42, 0x6f, 0x6f, 0x6c, 0x65, 0x61, 0x6e, 0x41, 0x72, 0x72, 0x61, 0x79, 0x50, 0x72, - 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x52, 0x16, 0x62, 0x6f, 0x6f, 0x6c, 0x65, 0x61, - 0x6e, 0x41, 0x72, 0x72, 0x61, 0x79, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, - 0x12, 0x4a, 0x0a, 0x11, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x70, 0x72, 0x6f, 0x70, 0x65, - 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x77, 0x65, - 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, - 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x52, 0x10, 0x6f, 0x62, 0x6a, 0x65, - 0x63, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x5a, 0x0a, 0x17, - 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x61, 0x72, 0x72, 0x61, 0x79, 0x5f, 0x70, 0x72, 0x6f, - 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x22, 0x2e, - 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x62, 0x6a, 0x65, - 0x63, 0x74, 0x41, 0x72, 0x72, 0x61, 0x79, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, - 0x73, 0x52, 0x15, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x41, 0x72, 0x72, 0x61, 0x79, 0x50, 0x72, - 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x1a, 0x49, 0x0a, 0x14, 0x53, 0x69, 0x6e, 0x67, - 0x6c, 0x65, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x65, 0x66, 0x50, 0x72, 0x6f, 0x70, 0x73, - 0x12, 0x14, 0x0a, 0x05, 0x75, 0x75, 0x69, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, - 0x05, 0x75, 0x75, 0x69, 0x64, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x70, 0x72, 0x6f, 0x70, 0x5f, 0x6e, - 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x70, 0x4e, - 0x61, 0x6d, 0x65, 0x1a, 0x75, 0x0a, 0x13, 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x54, 0x61, 0x72, 0x67, - 0x65, 0x74, 0x52, 0x65, 0x66, 0x50, 0x72, 0x6f, 0x70, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x75, 0x75, - 0x69, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x75, 0x75, 0x69, 0x64, 0x73, - 0x12, 0x1b, 0x0a, 0x09, 0x70, 0x72, 0x6f, 0x70, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2b, 0x0a, - 0x11, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x5f, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, - 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xa4, 0x01, 0x0a, 0x11, 0x42, - 0x61, 0x74, 0x63, 0x68, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x70, 0x6c, 0x79, - 0x12, 0x12, 0x0a, 0x04, 0x74, 0x6f, 0x6f, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x02, 0x52, 0x04, - 0x74, 0x6f, 0x6f, 0x6b, 0x12, 0x41, 0x0a, 0x06, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x18, 0x02, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, - 0x76, 0x31, 0x2e, 0x42, 0x61, 0x74, 0x63, 0x68, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, - 0x65, 0x70, 0x6c, 0x79, 0x2e, 0x42, 0x61, 0x74, 0x63, 0x68, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x52, - 0x06, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x1a, 0x38, 0x0a, 0x0a, 0x42, 0x61, 0x74, 0x63, 0x68, - 0x45, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x12, 0x14, 0x0a, 0x05, 0x65, - 0x72, 0x72, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, - 0x72, 0x42, 0x6f, 0x0a, 0x23, 0x69, 0x6f, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, - 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x76, 0x31, 0x42, 0x12, 0x57, 0x65, 0x61, 0x76, 0x69, 0x61, - 0x74, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x42, 0x61, 0x74, 0x63, 0x68, 0x5a, 0x34, 0x67, 0x69, - 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, - 0x65, 0x2f, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, - 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x3b, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, - 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -} - -var ( - file_v1_batch_proto_rawDescOnce sync.Once - file_v1_batch_proto_rawDescData = file_v1_batch_proto_rawDesc -) - -func file_v1_batch_proto_rawDescGZIP() []byte { - file_v1_batch_proto_rawDescOnce.Do(func() { - file_v1_batch_proto_rawDescData = protoimpl.X.CompressGZIP(file_v1_batch_proto_rawDescData) - }) - return file_v1_batch_proto_rawDescData -} - -var file_v1_batch_proto_msgTypes = make([]protoimpl.MessageInfo, 7) -var file_v1_batch_proto_goTypes = []interface{}{ - (*BatchObjectsRequest)(nil), // 0: weaviate.v1.BatchObjectsRequest - (*BatchObject)(nil), // 1: weaviate.v1.BatchObject - (*BatchObjectsReply)(nil), // 2: weaviate.v1.BatchObjectsReply - (*BatchObject_Properties)(nil), // 3: weaviate.v1.BatchObject.Properties - (*BatchObject_SingleTargetRefProps)(nil), // 4: weaviate.v1.BatchObject.SingleTargetRefProps - (*BatchObject_MultiTargetRefProps)(nil), // 5: weaviate.v1.BatchObject.MultiTargetRefProps - (*BatchObjectsReply_BatchError)(nil), // 6: weaviate.v1.BatchObjectsReply.BatchError - (ConsistencyLevel)(0), // 7: weaviate.v1.ConsistencyLevel - (*structpb.Struct)(nil), // 8: google.protobuf.Struct - (*NumberArrayProperties)(nil), // 9: weaviate.v1.NumberArrayProperties - (*IntArrayProperties)(nil), // 10: weaviate.v1.IntArrayProperties - (*TextArrayProperties)(nil), // 11: weaviate.v1.TextArrayProperties - (*BooleanArrayProperties)(nil), // 12: weaviate.v1.BooleanArrayProperties - (*ObjectProperties)(nil), // 13: weaviate.v1.ObjectProperties - (*ObjectArrayProperties)(nil), // 14: weaviate.v1.ObjectArrayProperties -} -var file_v1_batch_proto_depIdxs = []int32{ - 1, // 0: weaviate.v1.BatchObjectsRequest.objects:type_name -> weaviate.v1.BatchObject - 7, // 1: weaviate.v1.BatchObjectsRequest.consistency_level:type_name -> weaviate.v1.ConsistencyLevel - 3, // 2: weaviate.v1.BatchObject.properties:type_name -> weaviate.v1.BatchObject.Properties - 6, // 3: weaviate.v1.BatchObjectsReply.errors:type_name -> weaviate.v1.BatchObjectsReply.BatchError - 8, // 4: weaviate.v1.BatchObject.Properties.non_ref_properties:type_name -> google.protobuf.Struct - 4, // 5: weaviate.v1.BatchObject.Properties.single_target_ref_props:type_name -> weaviate.v1.BatchObject.SingleTargetRefProps - 5, // 6: weaviate.v1.BatchObject.Properties.multi_target_ref_props:type_name -> weaviate.v1.BatchObject.MultiTargetRefProps - 9, // 7: weaviate.v1.BatchObject.Properties.number_array_properties:type_name -> weaviate.v1.NumberArrayProperties - 10, // 8: weaviate.v1.BatchObject.Properties.int_array_properties:type_name -> weaviate.v1.IntArrayProperties - 11, // 9: weaviate.v1.BatchObject.Properties.text_array_properties:type_name -> weaviate.v1.TextArrayProperties - 12, // 10: weaviate.v1.BatchObject.Properties.boolean_array_properties:type_name -> weaviate.v1.BooleanArrayProperties - 13, // 11: weaviate.v1.BatchObject.Properties.object_properties:type_name -> weaviate.v1.ObjectProperties - 14, // 12: weaviate.v1.BatchObject.Properties.object_array_properties:type_name -> weaviate.v1.ObjectArrayProperties - 13, // [13:13] is the sub-list for method output_type - 13, // [13:13] is the sub-list for method input_type - 13, // [13:13] is the sub-list for extension type_name - 13, // [13:13] is the sub-list for extension extendee - 0, // [0:13] is the sub-list for field type_name -} - -func init() { file_v1_batch_proto_init() } -func file_v1_batch_proto_init() { - if File_v1_batch_proto != nil { - return - } - file_v1_base_proto_init() - if !protoimpl.UnsafeEnabled { - file_v1_batch_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*BatchObjectsRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_batch_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*BatchObject); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_batch_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*BatchObjectsReply); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_batch_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*BatchObject_Properties); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_batch_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*BatchObject_SingleTargetRefProps); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_batch_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*BatchObject_MultiTargetRefProps); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_batch_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*BatchObjectsReply_BatchError); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } - file_v1_batch_proto_msgTypes[0].OneofWrappers = []interface{}{} - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_v1_batch_proto_rawDesc, - NumEnums: 0, - NumMessages: 7, - NumExtensions: 0, - NumServices: 0, - }, - GoTypes: file_v1_batch_proto_goTypes, - DependencyIndexes: file_v1_batch_proto_depIdxs, - MessageInfos: file_v1_batch_proto_msgTypes, - }.Build() - File_v1_batch_proto = out.File - file_v1_batch_proto_rawDesc = nil - file_v1_batch_proto_goTypes = nil - file_v1_batch_proto_depIdxs = nil -} diff --git a/grpc/generated/protocol/v1/batch_delete.pb.go b/grpc/generated/protocol/v1/batch_delete.pb.go deleted file mode 100644 index af5ff99949e67e91fa675a403bcf0f30c3ba390d..0000000000000000000000000000000000000000 --- a/grpc/generated/protocol/v1/batch_delete.pb.go +++ /dev/null @@ -1,397 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. - -package protocol - -import ( - reflect "reflect" - sync "sync" - - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -type BatchDeleteRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Collection string `protobuf:"bytes,1,opt,name=collection,proto3" json:"collection,omitempty"` - Filters *Filters `protobuf:"bytes,2,opt,name=filters,proto3" json:"filters,omitempty"` - Verbose bool `protobuf:"varint,3,opt,name=verbose,proto3" json:"verbose,omitempty"` - DryRun bool `protobuf:"varint,4,opt,name=dry_run,json=dryRun,proto3" json:"dry_run,omitempty"` - ConsistencyLevel *ConsistencyLevel `protobuf:"varint,5,opt,name=consistency_level,json=consistencyLevel,proto3,enum=weaviate.v1.ConsistencyLevel,oneof" json:"consistency_level,omitempty"` - Tenant *string `protobuf:"bytes,6,opt,name=tenant,proto3,oneof" json:"tenant,omitempty"` -} - -func (x *BatchDeleteRequest) Reset() { - *x = BatchDeleteRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_batch_delete_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *BatchDeleteRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*BatchDeleteRequest) ProtoMessage() {} - -func (x *BatchDeleteRequest) ProtoReflect() protoreflect.Message { - mi := &file_v1_batch_delete_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use BatchDeleteRequest.ProtoReflect.Descriptor instead. -func (*BatchDeleteRequest) Descriptor() ([]byte, []int) { - return file_v1_batch_delete_proto_rawDescGZIP(), []int{0} -} - -func (x *BatchDeleteRequest) GetCollection() string { - if x != nil { - return x.Collection - } - return "" -} - -func (x *BatchDeleteRequest) GetFilters() *Filters { - if x != nil { - return x.Filters - } - return nil -} - -func (x *BatchDeleteRequest) GetVerbose() bool { - if x != nil { - return x.Verbose - } - return false -} - -func (x *BatchDeleteRequest) GetDryRun() bool { - if x != nil { - return x.DryRun - } - return false -} - -func (x *BatchDeleteRequest) GetConsistencyLevel() ConsistencyLevel { - if x != nil && x.ConsistencyLevel != nil { - return *x.ConsistencyLevel - } - return ConsistencyLevel_CONSISTENCY_LEVEL_UNSPECIFIED -} - -func (x *BatchDeleteRequest) GetTenant() string { - if x != nil && x.Tenant != nil { - return *x.Tenant - } - return "" -} - -type BatchDeleteReply struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Took float32 `protobuf:"fixed32,1,opt,name=took,proto3" json:"took,omitempty"` - Failed int64 `protobuf:"varint,2,opt,name=failed,proto3" json:"failed,omitempty"` - Matches int64 `protobuf:"varint,3,opt,name=matches,proto3" json:"matches,omitempty"` - Successful int64 `protobuf:"varint,4,opt,name=successful,proto3" json:"successful,omitempty"` - Objects []*BatchDeleteObject `protobuf:"bytes,5,rep,name=objects,proto3" json:"objects,omitempty"` -} - -func (x *BatchDeleteReply) Reset() { - *x = BatchDeleteReply{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_batch_delete_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *BatchDeleteReply) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*BatchDeleteReply) ProtoMessage() {} - -func (x *BatchDeleteReply) ProtoReflect() protoreflect.Message { - mi := &file_v1_batch_delete_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use BatchDeleteReply.ProtoReflect.Descriptor instead. -func (*BatchDeleteReply) Descriptor() ([]byte, []int) { - return file_v1_batch_delete_proto_rawDescGZIP(), []int{1} -} - -func (x *BatchDeleteReply) GetTook() float32 { - if x != nil { - return x.Took - } - return 0 -} - -func (x *BatchDeleteReply) GetFailed() int64 { - if x != nil { - return x.Failed - } - return 0 -} - -func (x *BatchDeleteReply) GetMatches() int64 { - if x != nil { - return x.Matches - } - return 0 -} - -func (x *BatchDeleteReply) GetSuccessful() int64 { - if x != nil { - return x.Successful - } - return 0 -} - -func (x *BatchDeleteReply) GetObjects() []*BatchDeleteObject { - if x != nil { - return x.Objects - } - return nil -} - -type BatchDeleteObject struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Uuid []byte `protobuf:"bytes,1,opt,name=uuid,proto3" json:"uuid,omitempty"` - Successful bool `protobuf:"varint,2,opt,name=successful,proto3" json:"successful,omitempty"` - Error *string `protobuf:"bytes,3,opt,name=error,proto3,oneof" json:"error,omitempty"` // empty string means no error -} - -func (x *BatchDeleteObject) Reset() { - *x = BatchDeleteObject{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_batch_delete_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *BatchDeleteObject) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*BatchDeleteObject) ProtoMessage() {} - -func (x *BatchDeleteObject) ProtoReflect() protoreflect.Message { - mi := &file_v1_batch_delete_proto_msgTypes[2] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use BatchDeleteObject.ProtoReflect.Descriptor instead. -func (*BatchDeleteObject) Descriptor() ([]byte, []int) { - return file_v1_batch_delete_proto_rawDescGZIP(), []int{2} -} - -func (x *BatchDeleteObject) GetUuid() []byte { - if x != nil { - return x.Uuid - } - return nil -} - -func (x *BatchDeleteObject) GetSuccessful() bool { - if x != nil { - return x.Successful - } - return false -} - -func (x *BatchDeleteObject) GetError() string { - if x != nil && x.Error != nil { - return *x.Error - } - return "" -} - -var File_v1_batch_delete_proto protoreflect.FileDescriptor - -var file_v1_batch_delete_proto_rawDesc = []byte{ - 0x0a, 0x15, 0x76, 0x31, 0x2f, 0x62, 0x61, 0x74, 0x63, 0x68, 0x5f, 0x64, 0x65, 0x6c, 0x65, 0x74, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0b, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, - 0x65, 0x2e, 0x76, 0x31, 0x1a, 0x0d, 0x76, 0x31, 0x2f, 0x62, 0x61, 0x73, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x22, 0xa6, 0x02, 0x0a, 0x12, 0x42, 0x61, 0x74, 0x63, 0x68, 0x44, 0x65, 0x6c, - 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, - 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, - 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2e, 0x0a, 0x07, 0x66, 0x69, - 0x6c, 0x74, 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x77, 0x65, - 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, - 0x73, 0x52, 0x07, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, - 0x72, 0x62, 0x6f, 0x73, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x76, 0x65, 0x72, - 0x62, 0x6f, 0x73, 0x65, 0x12, 0x17, 0x0a, 0x07, 0x64, 0x72, 0x79, 0x5f, 0x72, 0x75, 0x6e, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x64, 0x72, 0x79, 0x52, 0x75, 0x6e, 0x12, 0x4f, 0x0a, - 0x11, 0x63, 0x6f, 0x6e, 0x73, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x5f, 0x6c, 0x65, 0x76, - 0x65, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1d, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, - 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x73, 0x69, 0x73, 0x74, 0x65, 0x6e, - 0x63, 0x79, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x48, 0x00, 0x52, 0x10, 0x63, 0x6f, 0x6e, 0x73, 0x69, - 0x73, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x88, 0x01, 0x01, 0x12, 0x1b, - 0x0a, 0x06, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x48, 0x01, - 0x52, 0x06, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x88, 0x01, 0x01, 0x42, 0x14, 0x0a, 0x12, 0x5f, - 0x63, 0x6f, 0x6e, 0x73, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x5f, 0x6c, 0x65, 0x76, 0x65, - 0x6c, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x22, 0xb2, 0x01, 0x0a, - 0x10, 0x42, 0x61, 0x74, 0x63, 0x68, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x70, 0x6c, - 0x79, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x6f, 0x6f, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x02, 0x52, - 0x04, 0x74, 0x6f, 0x6f, 0x6b, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x12, 0x18, 0x0a, - 0x07, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, - 0x6d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x75, 0x63, 0x63, 0x65, - 0x73, 0x73, 0x66, 0x75, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x73, 0x75, 0x63, - 0x63, 0x65, 0x73, 0x73, 0x66, 0x75, 0x6c, 0x12, 0x38, 0x0a, 0x07, 0x6f, 0x62, 0x6a, 0x65, 0x63, - 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, - 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x61, 0x74, 0x63, 0x68, 0x44, 0x65, 0x6c, 0x65, - 0x74, 0x65, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x07, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, - 0x73, 0x22, 0x6c, 0x0a, 0x11, 0x42, 0x61, 0x74, 0x63, 0x68, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, - 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x75, 0x69, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x75, 0x75, 0x69, 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x75, - 0x63, 0x63, 0x65, 0x73, 0x73, 0x66, 0x75, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, - 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x66, 0x75, 0x6c, 0x12, 0x19, 0x0a, 0x05, 0x65, 0x72, - 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x65, 0x72, 0x72, - 0x6f, 0x72, 0x88, 0x01, 0x01, 0x42, 0x08, 0x0a, 0x06, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x42, - 0x6f, 0x0a, 0x23, 0x69, 0x6f, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x63, - 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x63, 0x6f, 0x6c, 0x2e, 0x76, 0x31, 0x42, 0x12, 0x57, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, - 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x42, 0x61, 0x74, 0x63, 0x68, 0x5a, 0x34, 0x67, 0x69, 0x74, 0x68, - 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2f, - 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x67, 0x65, - 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x3b, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, - 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -} - -var ( - file_v1_batch_delete_proto_rawDescOnce sync.Once - file_v1_batch_delete_proto_rawDescData = file_v1_batch_delete_proto_rawDesc -) - -func file_v1_batch_delete_proto_rawDescGZIP() []byte { - file_v1_batch_delete_proto_rawDescOnce.Do(func() { - file_v1_batch_delete_proto_rawDescData = protoimpl.X.CompressGZIP(file_v1_batch_delete_proto_rawDescData) - }) - return file_v1_batch_delete_proto_rawDescData -} - -var file_v1_batch_delete_proto_msgTypes = make([]protoimpl.MessageInfo, 3) -var file_v1_batch_delete_proto_goTypes = []interface{}{ - (*BatchDeleteRequest)(nil), // 0: weaviate.v1.BatchDeleteRequest - (*BatchDeleteReply)(nil), // 1: weaviate.v1.BatchDeleteReply - (*BatchDeleteObject)(nil), // 2: weaviate.v1.BatchDeleteObject - (*Filters)(nil), // 3: weaviate.v1.Filters - (ConsistencyLevel)(0), // 4: weaviate.v1.ConsistencyLevel -} -var file_v1_batch_delete_proto_depIdxs = []int32{ - 3, // 0: weaviate.v1.BatchDeleteRequest.filters:type_name -> weaviate.v1.Filters - 4, // 1: weaviate.v1.BatchDeleteRequest.consistency_level:type_name -> weaviate.v1.ConsistencyLevel - 2, // 2: weaviate.v1.BatchDeleteReply.objects:type_name -> weaviate.v1.BatchDeleteObject - 3, // [3:3] is the sub-list for method output_type - 3, // [3:3] is the sub-list for method input_type - 3, // [3:3] is the sub-list for extension type_name - 3, // [3:3] is the sub-list for extension extendee - 0, // [0:3] is the sub-list for field type_name -} - -func init() { file_v1_batch_delete_proto_init() } -func file_v1_batch_delete_proto_init() { - if File_v1_batch_delete_proto != nil { - return - } - file_v1_base_proto_init() - if !protoimpl.UnsafeEnabled { - file_v1_batch_delete_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*BatchDeleteRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_batch_delete_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*BatchDeleteReply); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_batch_delete_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*BatchDeleteObject); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } - file_v1_batch_delete_proto_msgTypes[0].OneofWrappers = []interface{}{} - file_v1_batch_delete_proto_msgTypes[2].OneofWrappers = []interface{}{} - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_v1_batch_delete_proto_rawDesc, - NumEnums: 0, - NumMessages: 3, - NumExtensions: 0, - NumServices: 0, - }, - GoTypes: file_v1_batch_delete_proto_goTypes, - DependencyIndexes: file_v1_batch_delete_proto_depIdxs, - MessageInfos: file_v1_batch_delete_proto_msgTypes, - }.Build() - File_v1_batch_delete_proto = out.File - file_v1_batch_delete_proto_rawDesc = nil - file_v1_batch_delete_proto_goTypes = nil - file_v1_batch_delete_proto_depIdxs = nil -} diff --git a/grpc/generated/protocol/v1/properties.pb.go b/grpc/generated/protocol/v1/properties.pb.go deleted file mode 100644 index a42d6fd7a0eb3c18ccdb5aaac20f01cb5106bbe3..0000000000000000000000000000000000000000 --- a/grpc/generated/protocol/v1/properties.pb.go +++ /dev/null @@ -1,686 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. - -package protocol - -import ( - reflect "reflect" - sync "sync" - - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -type Properties struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Fields map[string]*Value `protobuf:"bytes,1,rep,name=fields,proto3" json:"fields,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` -} - -func (x *Properties) Reset() { - *x = Properties{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_properties_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *Properties) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*Properties) ProtoMessage() {} - -func (x *Properties) ProtoReflect() protoreflect.Message { - mi := &file_v1_properties_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use Properties.ProtoReflect.Descriptor instead. -func (*Properties) Descriptor() ([]byte, []int) { - return file_v1_properties_proto_rawDescGZIP(), []int{0} -} - -func (x *Properties) GetFields() map[string]*Value { - if x != nil { - return x.Fields - } - return nil -} - -type Value struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Types that are assignable to Kind: - // - // *Value_NumberValue - // *Value_StringValue - // *Value_BoolValue - // *Value_ObjectValue - // *Value_ListValue - // *Value_DateValue - // *Value_UuidValue - // *Value_IntValue - // *Value_GeoValue - // *Value_BlobValue - // *Value_PhoneValue - Kind isValue_Kind `protobuf_oneof:"kind"` -} - -func (x *Value) Reset() { - *x = Value{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_properties_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *Value) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*Value) ProtoMessage() {} - -func (x *Value) ProtoReflect() protoreflect.Message { - mi := &file_v1_properties_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use Value.ProtoReflect.Descriptor instead. -func (*Value) Descriptor() ([]byte, []int) { - return file_v1_properties_proto_rawDescGZIP(), []int{1} -} - -func (m *Value) GetKind() isValue_Kind { - if m != nil { - return m.Kind - } - return nil -} - -func (x *Value) GetNumberValue() float64 { - if x, ok := x.GetKind().(*Value_NumberValue); ok { - return x.NumberValue - } - return 0 -} - -func (x *Value) GetStringValue() string { - if x, ok := x.GetKind().(*Value_StringValue); ok { - return x.StringValue - } - return "" -} - -func (x *Value) GetBoolValue() bool { - if x, ok := x.GetKind().(*Value_BoolValue); ok { - return x.BoolValue - } - return false -} - -func (x *Value) GetObjectValue() *Properties { - if x, ok := x.GetKind().(*Value_ObjectValue); ok { - return x.ObjectValue - } - return nil -} - -func (x *Value) GetListValue() *ListValue { - if x, ok := x.GetKind().(*Value_ListValue); ok { - return x.ListValue - } - return nil -} - -func (x *Value) GetDateValue() string { - if x, ok := x.GetKind().(*Value_DateValue); ok { - return x.DateValue - } - return "" -} - -func (x *Value) GetUuidValue() string { - if x, ok := x.GetKind().(*Value_UuidValue); ok { - return x.UuidValue - } - return "" -} - -func (x *Value) GetIntValue() int64 { - if x, ok := x.GetKind().(*Value_IntValue); ok { - return x.IntValue - } - return 0 -} - -func (x *Value) GetGeoValue() *GeoCoordinate { - if x, ok := x.GetKind().(*Value_GeoValue); ok { - return x.GeoValue - } - return nil -} - -func (x *Value) GetBlobValue() string { - if x, ok := x.GetKind().(*Value_BlobValue); ok { - return x.BlobValue - } - return "" -} - -func (x *Value) GetPhoneValue() *PhoneNumber { - if x, ok := x.GetKind().(*Value_PhoneValue); ok { - return x.PhoneValue - } - return nil -} - -type isValue_Kind interface { - isValue_Kind() -} - -type Value_NumberValue struct { - NumberValue float64 `protobuf:"fixed64,1,opt,name=number_value,json=numberValue,proto3,oneof"` -} - -type Value_StringValue struct { - StringValue string `protobuf:"bytes,2,opt,name=string_value,json=stringValue,proto3,oneof"` -} - -type Value_BoolValue struct { - BoolValue bool `protobuf:"varint,3,opt,name=bool_value,json=boolValue,proto3,oneof"` -} - -type Value_ObjectValue struct { - ObjectValue *Properties `protobuf:"bytes,4,opt,name=object_value,json=objectValue,proto3,oneof"` -} - -type Value_ListValue struct { - ListValue *ListValue `protobuf:"bytes,5,opt,name=list_value,json=listValue,proto3,oneof"` -} - -type Value_DateValue struct { - DateValue string `protobuf:"bytes,6,opt,name=date_value,json=dateValue,proto3,oneof"` -} - -type Value_UuidValue struct { - UuidValue string `protobuf:"bytes,7,opt,name=uuid_value,json=uuidValue,proto3,oneof"` -} - -type Value_IntValue struct { - IntValue int64 `protobuf:"varint,8,opt,name=int_value,json=intValue,proto3,oneof"` -} - -type Value_GeoValue struct { - GeoValue *GeoCoordinate `protobuf:"bytes,9,opt,name=geo_value,json=geoValue,proto3,oneof"` -} - -type Value_BlobValue struct { - BlobValue string `protobuf:"bytes,10,opt,name=blob_value,json=blobValue,proto3,oneof"` -} - -type Value_PhoneValue struct { - PhoneValue *PhoneNumber `protobuf:"bytes,11,opt,name=phone_value,json=phoneValue,proto3,oneof"` -} - -func (*Value_NumberValue) isValue_Kind() {} - -func (*Value_StringValue) isValue_Kind() {} - -func (*Value_BoolValue) isValue_Kind() {} - -func (*Value_ObjectValue) isValue_Kind() {} - -func (*Value_ListValue) isValue_Kind() {} - -func (*Value_DateValue) isValue_Kind() {} - -func (*Value_UuidValue) isValue_Kind() {} - -func (*Value_IntValue) isValue_Kind() {} - -func (*Value_GeoValue) isValue_Kind() {} - -func (*Value_BlobValue) isValue_Kind() {} - -func (*Value_PhoneValue) isValue_Kind() {} - -type ListValue struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Values []*Value `protobuf:"bytes,1,rep,name=values,proto3" json:"values,omitempty"` -} - -func (x *ListValue) Reset() { - *x = ListValue{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_properties_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ListValue) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ListValue) ProtoMessage() {} - -func (x *ListValue) ProtoReflect() protoreflect.Message { - mi := &file_v1_properties_proto_msgTypes[2] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ListValue.ProtoReflect.Descriptor instead. -func (*ListValue) Descriptor() ([]byte, []int) { - return file_v1_properties_proto_rawDescGZIP(), []int{2} -} - -func (x *ListValue) GetValues() []*Value { - if x != nil { - return x.Values - } - return nil -} - -type GeoCoordinate struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Longitude float32 `protobuf:"fixed32,1,opt,name=longitude,proto3" json:"longitude,omitempty"` - Latitude float32 `protobuf:"fixed32,2,opt,name=latitude,proto3" json:"latitude,omitempty"` -} - -func (x *GeoCoordinate) Reset() { - *x = GeoCoordinate{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_properties_proto_msgTypes[3] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GeoCoordinate) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GeoCoordinate) ProtoMessage() {} - -func (x *GeoCoordinate) ProtoReflect() protoreflect.Message { - mi := &file_v1_properties_proto_msgTypes[3] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GeoCoordinate.ProtoReflect.Descriptor instead. -func (*GeoCoordinate) Descriptor() ([]byte, []int) { - return file_v1_properties_proto_rawDescGZIP(), []int{3} -} - -func (x *GeoCoordinate) GetLongitude() float32 { - if x != nil { - return x.Longitude - } - return 0 -} - -func (x *GeoCoordinate) GetLatitude() float32 { - if x != nil { - return x.Latitude - } - return 0 -} - -type PhoneNumber struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - CountryCode uint64 `protobuf:"varint,1,opt,name=country_code,json=countryCode,proto3" json:"country_code,omitempty"` - DefaultCountry string `protobuf:"bytes,2,opt,name=default_country,json=defaultCountry,proto3" json:"default_country,omitempty"` - Input string `protobuf:"bytes,3,opt,name=input,proto3" json:"input,omitempty"` - InternationalFormatted string `protobuf:"bytes,4,opt,name=international_formatted,json=internationalFormatted,proto3" json:"international_formatted,omitempty"` - National uint64 `protobuf:"varint,5,opt,name=national,proto3" json:"national,omitempty"` - NationalFormatted string `protobuf:"bytes,6,opt,name=national_formatted,json=nationalFormatted,proto3" json:"national_formatted,omitempty"` - Valid bool `protobuf:"varint,7,opt,name=valid,proto3" json:"valid,omitempty"` -} - -func (x *PhoneNumber) Reset() { - *x = PhoneNumber{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_properties_proto_msgTypes[4] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *PhoneNumber) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*PhoneNumber) ProtoMessage() {} - -func (x *PhoneNumber) ProtoReflect() protoreflect.Message { - mi := &file_v1_properties_proto_msgTypes[4] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use PhoneNumber.ProtoReflect.Descriptor instead. -func (*PhoneNumber) Descriptor() ([]byte, []int) { - return file_v1_properties_proto_rawDescGZIP(), []int{4} -} - -func (x *PhoneNumber) GetCountryCode() uint64 { - if x != nil { - return x.CountryCode - } - return 0 -} - -func (x *PhoneNumber) GetDefaultCountry() string { - if x != nil { - return x.DefaultCountry - } - return "" -} - -func (x *PhoneNumber) GetInput() string { - if x != nil { - return x.Input - } - return "" -} - -func (x *PhoneNumber) GetInternationalFormatted() string { - if x != nil { - return x.InternationalFormatted - } - return "" -} - -func (x *PhoneNumber) GetNational() uint64 { - if x != nil { - return x.National - } - return 0 -} - -func (x *PhoneNumber) GetNationalFormatted() string { - if x != nil { - return x.NationalFormatted - } - return "" -} - -func (x *PhoneNumber) GetValid() bool { - if x != nil { - return x.Valid - } - return false -} - -var File_v1_properties_proto protoreflect.FileDescriptor - -var file_v1_properties_proto_rawDesc = []byte{ - 0x0a, 0x13, 0x76, 0x31, 0x2f, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0b, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, - 0x76, 0x31, 0x22, 0x98, 0x01, 0x0a, 0x0a, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, - 0x73, 0x12, 0x3b, 0x0a, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x23, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, - 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, - 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x1a, 0x4d, - 0x0a, 0x0b, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, - 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, - 0x28, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, - 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x56, 0x61, 0x6c, - 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xeb, 0x03, - 0x0a, 0x05, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x23, 0x0a, 0x0c, 0x6e, 0x75, 0x6d, 0x62, 0x65, - 0x72, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x48, 0x00, 0x52, - 0x0b, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x23, 0x0a, 0x0c, - 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x48, 0x00, 0x52, 0x0b, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, - 0x65, 0x12, 0x1f, 0x0a, 0x0a, 0x62, 0x6f, 0x6f, 0x6c, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, 0x09, 0x62, 0x6f, 0x6f, 0x6c, 0x56, 0x61, 0x6c, - 0x75, 0x65, 0x12, 0x3c, 0x0a, 0x0c, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, - 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, - 0x73, 0x48, 0x00, 0x52, 0x0b, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x56, 0x61, 0x6c, 0x75, 0x65, - 0x12, 0x37, 0x0a, 0x0a, 0x6c, 0x69, 0x73, 0x74, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x05, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, - 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x48, 0x00, 0x52, 0x09, - 0x6c, 0x69, 0x73, 0x74, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x1f, 0x0a, 0x0a, 0x64, 0x61, 0x74, - 0x65, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, - 0x09, 0x64, 0x61, 0x74, 0x65, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x1f, 0x0a, 0x0a, 0x75, 0x75, - 0x69, 0x64, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, - 0x52, 0x09, 0x75, 0x75, 0x69, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x1d, 0x0a, 0x09, 0x69, - 0x6e, 0x74, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x03, 0x48, 0x00, - 0x52, 0x08, 0x69, 0x6e, 0x74, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x39, 0x0a, 0x09, 0x67, 0x65, - 0x6f, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, - 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x6f, 0x43, - 0x6f, 0x6f, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x48, 0x00, 0x52, 0x08, 0x67, 0x65, 0x6f, - 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x1f, 0x0a, 0x0a, 0x62, 0x6c, 0x6f, 0x62, 0x5f, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x09, 0x62, 0x6c, 0x6f, - 0x62, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x3b, 0x0a, 0x0b, 0x70, 0x68, 0x6f, 0x6e, 0x65, 0x5f, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x77, 0x65, - 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x68, 0x6f, 0x6e, 0x65, 0x4e, - 0x75, 0x6d, 0x62, 0x65, 0x72, 0x48, 0x00, 0x52, 0x0a, 0x70, 0x68, 0x6f, 0x6e, 0x65, 0x56, 0x61, - 0x6c, 0x75, 0x65, 0x42, 0x06, 0x0a, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x22, 0x37, 0x0a, 0x09, 0x4c, - 0x69, 0x73, 0x74, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x2a, 0x0a, 0x06, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, - 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x06, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x73, 0x22, 0x49, 0x0a, 0x0d, 0x47, 0x65, 0x6f, 0x43, 0x6f, 0x6f, 0x72, 0x64, - 0x69, 0x6e, 0x61, 0x74, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x6c, 0x6f, 0x6e, 0x67, 0x69, 0x74, 0x75, - 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x02, 0x52, 0x09, 0x6c, 0x6f, 0x6e, 0x67, 0x69, 0x74, - 0x75, 0x64, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x6c, 0x61, 0x74, 0x69, 0x74, 0x75, 0x64, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x02, 0x52, 0x08, 0x6c, 0x61, 0x74, 0x69, 0x74, 0x75, 0x64, 0x65, 0x22, - 0x89, 0x02, 0x0a, 0x0b, 0x50, 0x68, 0x6f, 0x6e, 0x65, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, - 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x72, 0x79, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x72, 0x79, 0x43, 0x6f, - 0x64, 0x65, 0x12, 0x27, 0x0a, 0x0f, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x5f, 0x63, 0x6f, - 0x75, 0x6e, 0x74, 0x72, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x64, 0x65, 0x66, - 0x61, 0x75, 0x6c, 0x74, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x69, - 0x6e, 0x70, 0x75, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x69, 0x6e, 0x70, 0x75, - 0x74, 0x12, 0x37, 0x0a, 0x17, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x61, 0x6c, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x74, 0x65, 0x64, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x16, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x61, - 0x6c, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x74, 0x65, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x6e, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x6e, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x12, 0x2d, 0x0a, 0x12, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x61, 0x6c, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x74, 0x65, 0x64, 0x18, 0x06, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x11, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x46, 0x6f, 0x72, 0x6d, - 0x61, 0x74, 0x74, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x18, 0x07, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x42, 0x74, 0x0a, 0x23, 0x69, - 0x6f, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, - 0x74, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, - 0x76, 0x31, 0x42, 0x17, 0x57, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x74, - 0x6f, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x5a, 0x34, 0x67, 0x69, 0x74, - 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, - 0x2f, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x67, - 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x3b, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, - 0x6c, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -} - -var ( - file_v1_properties_proto_rawDescOnce sync.Once - file_v1_properties_proto_rawDescData = file_v1_properties_proto_rawDesc -) - -func file_v1_properties_proto_rawDescGZIP() []byte { - file_v1_properties_proto_rawDescOnce.Do(func() { - file_v1_properties_proto_rawDescData = protoimpl.X.CompressGZIP(file_v1_properties_proto_rawDescData) - }) - return file_v1_properties_proto_rawDescData -} - -var file_v1_properties_proto_msgTypes = make([]protoimpl.MessageInfo, 6) -var file_v1_properties_proto_goTypes = []interface{}{ - (*Properties)(nil), // 0: weaviate.v1.Properties - (*Value)(nil), // 1: weaviate.v1.Value - (*ListValue)(nil), // 2: weaviate.v1.ListValue - (*GeoCoordinate)(nil), // 3: weaviate.v1.GeoCoordinate - (*PhoneNumber)(nil), // 4: weaviate.v1.PhoneNumber - nil, // 5: weaviate.v1.Properties.FieldsEntry -} -var file_v1_properties_proto_depIdxs = []int32{ - 5, // 0: weaviate.v1.Properties.fields:type_name -> weaviate.v1.Properties.FieldsEntry - 0, // 1: weaviate.v1.Value.object_value:type_name -> weaviate.v1.Properties - 2, // 2: weaviate.v1.Value.list_value:type_name -> weaviate.v1.ListValue - 3, // 3: weaviate.v1.Value.geo_value:type_name -> weaviate.v1.GeoCoordinate - 4, // 4: weaviate.v1.Value.phone_value:type_name -> weaviate.v1.PhoneNumber - 1, // 5: weaviate.v1.ListValue.values:type_name -> weaviate.v1.Value - 1, // 6: weaviate.v1.Properties.FieldsEntry.value:type_name -> weaviate.v1.Value - 7, // [7:7] is the sub-list for method output_type - 7, // [7:7] is the sub-list for method input_type - 7, // [7:7] is the sub-list for extension type_name - 7, // [7:7] is the sub-list for extension extendee - 0, // [0:7] is the sub-list for field type_name -} - -func init() { file_v1_properties_proto_init() } -func file_v1_properties_proto_init() { - if File_v1_properties_proto != nil { - return - } - if !protoimpl.UnsafeEnabled { - file_v1_properties_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Properties); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_properties_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Value); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_properties_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListValue); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_properties_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GeoCoordinate); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_properties_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PhoneNumber); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } - file_v1_properties_proto_msgTypes[1].OneofWrappers = []interface{}{ - (*Value_NumberValue)(nil), - (*Value_StringValue)(nil), - (*Value_BoolValue)(nil), - (*Value_ObjectValue)(nil), - (*Value_ListValue)(nil), - (*Value_DateValue)(nil), - (*Value_UuidValue)(nil), - (*Value_IntValue)(nil), - (*Value_GeoValue)(nil), - (*Value_BlobValue)(nil), - (*Value_PhoneValue)(nil), - } - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_v1_properties_proto_rawDesc, - NumEnums: 0, - NumMessages: 6, - NumExtensions: 0, - NumServices: 0, - }, - GoTypes: file_v1_properties_proto_goTypes, - DependencyIndexes: file_v1_properties_proto_depIdxs, - MessageInfos: file_v1_properties_proto_msgTypes, - }.Build() - File_v1_properties_proto = out.File - file_v1_properties_proto_rawDesc = nil - file_v1_properties_proto_goTypes = nil - file_v1_properties_proto_depIdxs = nil -} diff --git a/grpc/generated/protocol/v1/search_get.pb.go b/grpc/generated/protocol/v1/search_get.pb.go deleted file mode 100644 index f5af3b1dcbda316e421c0dae37e10a875088672c..0000000000000000000000000000000000000000 --- a/grpc/generated/protocol/v1/search_get.pb.go +++ /dev/null @@ -1,3133 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. - -package protocol - -import ( - reflect "reflect" - sync "sync" - - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" - structpb "google.golang.org/protobuf/types/known/structpb" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -type Hybrid_FusionType int32 - -const ( - Hybrid_FUSION_TYPE_UNSPECIFIED Hybrid_FusionType = 0 - Hybrid_FUSION_TYPE_RANKED Hybrid_FusionType = 1 - Hybrid_FUSION_TYPE_RELATIVE_SCORE Hybrid_FusionType = 2 -) - -// Enum value maps for Hybrid_FusionType. -var ( - Hybrid_FusionType_name = map[int32]string{ - 0: "FUSION_TYPE_UNSPECIFIED", - 1: "FUSION_TYPE_RANKED", - 2: "FUSION_TYPE_RELATIVE_SCORE", - } - Hybrid_FusionType_value = map[string]int32{ - "FUSION_TYPE_UNSPECIFIED": 0, - "FUSION_TYPE_RANKED": 1, - "FUSION_TYPE_RELATIVE_SCORE": 2, - } -) - -func (x Hybrid_FusionType) Enum() *Hybrid_FusionType { - p := new(Hybrid_FusionType) - *p = x - return p -} - -func (x Hybrid_FusionType) String() string { - return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) -} - -func (Hybrid_FusionType) Descriptor() protoreflect.EnumDescriptor { - return file_v1_search_get_proto_enumTypes[0].Descriptor() -} - -func (Hybrid_FusionType) Type() protoreflect.EnumType { - return &file_v1_search_get_proto_enumTypes[0] -} - -func (x Hybrid_FusionType) Number() protoreflect.EnumNumber { - return protoreflect.EnumNumber(x) -} - -// Deprecated: Use Hybrid_FusionType.Descriptor instead. -func (Hybrid_FusionType) EnumDescriptor() ([]byte, []int) { - return file_v1_search_get_proto_rawDescGZIP(), []int{7, 0} -} - -type SearchRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // required - Collection string `protobuf:"bytes,1,opt,name=collection,proto3" json:"collection,omitempty"` - // parameters - Tenant string `protobuf:"bytes,10,opt,name=tenant,proto3" json:"tenant,omitempty"` - ConsistencyLevel *ConsistencyLevel `protobuf:"varint,11,opt,name=consistency_level,json=consistencyLevel,proto3,enum=weaviate.v1.ConsistencyLevel,oneof" json:"consistency_level,omitempty"` - // what is returned - Properties *PropertiesRequest `protobuf:"bytes,20,opt,name=properties,proto3,oneof" json:"properties,omitempty"` - Metadata *MetadataRequest `protobuf:"bytes,21,opt,name=metadata,proto3,oneof" json:"metadata,omitempty"` - GroupBy *GroupBy `protobuf:"bytes,22,opt,name=group_by,json=groupBy,proto3,oneof" json:"group_by,omitempty"` - // affects order and length of results. 0/empty (default value) means disabled - Limit uint32 `protobuf:"varint,30,opt,name=limit,proto3" json:"limit,omitempty"` - Offset uint32 `protobuf:"varint,31,opt,name=offset,proto3" json:"offset,omitempty"` - Autocut uint32 `protobuf:"varint,32,opt,name=autocut,proto3" json:"autocut,omitempty"` - After string `protobuf:"bytes,33,opt,name=after,proto3" json:"after,omitempty"` - // protolint:disable:next REPEATED_FIELD_NAMES_PLURALIZED - SortBy []*SortBy `protobuf:"bytes,34,rep,name=sort_by,json=sortBy,proto3" json:"sort_by,omitempty"` - // matches/searches for objects - Filters *Filters `protobuf:"bytes,40,opt,name=filters,proto3,oneof" json:"filters,omitempty"` - HybridSearch *Hybrid `protobuf:"bytes,41,opt,name=hybrid_search,json=hybridSearch,proto3,oneof" json:"hybrid_search,omitempty"` - Bm25Search *BM25 `protobuf:"bytes,42,opt,name=bm25_search,json=bm25Search,proto3,oneof" json:"bm25_search,omitempty"` - NearVector *NearVector `protobuf:"bytes,43,opt,name=near_vector,json=nearVector,proto3,oneof" json:"near_vector,omitempty"` - NearObject *NearObject `protobuf:"bytes,44,opt,name=near_object,json=nearObject,proto3,oneof" json:"near_object,omitempty"` - NearText *NearTextSearch `protobuf:"bytes,45,opt,name=near_text,json=nearText,proto3,oneof" json:"near_text,omitempty"` - NearImage *NearImageSearch `protobuf:"bytes,46,opt,name=near_image,json=nearImage,proto3,oneof" json:"near_image,omitempty"` - NearAudio *NearAudioSearch `protobuf:"bytes,47,opt,name=near_audio,json=nearAudio,proto3,oneof" json:"near_audio,omitempty"` - NearVideo *NearVideoSearch `protobuf:"bytes,48,opt,name=near_video,json=nearVideo,proto3,oneof" json:"near_video,omitempty"` - Generative *GenerativeSearch `protobuf:"bytes,60,opt,name=generative,proto3,oneof" json:"generative,omitempty"` - Rerank *Rerank `protobuf:"bytes,61,opt,name=rerank,proto3,oneof" json:"rerank,omitempty"` - // Deprecated: Marked as deprecated in v1/search_get.proto. - Uses_123Api bool `protobuf:"varint,100,opt,name=uses_123_api,json=uses123Api,proto3" json:"uses_123_api,omitempty"` -} - -func (x *SearchRequest) Reset() { - *x = SearchRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_search_get_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *SearchRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*SearchRequest) ProtoMessage() {} - -func (x *SearchRequest) ProtoReflect() protoreflect.Message { - mi := &file_v1_search_get_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use SearchRequest.ProtoReflect.Descriptor instead. -func (*SearchRequest) Descriptor() ([]byte, []int) { - return file_v1_search_get_proto_rawDescGZIP(), []int{0} -} - -func (x *SearchRequest) GetCollection() string { - if x != nil { - return x.Collection - } - return "" -} - -func (x *SearchRequest) GetTenant() string { - if x != nil { - return x.Tenant - } - return "" -} - -func (x *SearchRequest) GetConsistencyLevel() ConsistencyLevel { - if x != nil && x.ConsistencyLevel != nil { - return *x.ConsistencyLevel - } - return ConsistencyLevel_CONSISTENCY_LEVEL_UNSPECIFIED -} - -func (x *SearchRequest) GetProperties() *PropertiesRequest { - if x != nil { - return x.Properties - } - return nil -} - -func (x *SearchRequest) GetMetadata() *MetadataRequest { - if x != nil { - return x.Metadata - } - return nil -} - -func (x *SearchRequest) GetGroupBy() *GroupBy { - if x != nil { - return x.GroupBy - } - return nil -} - -func (x *SearchRequest) GetLimit() uint32 { - if x != nil { - return x.Limit - } - return 0 -} - -func (x *SearchRequest) GetOffset() uint32 { - if x != nil { - return x.Offset - } - return 0 -} - -func (x *SearchRequest) GetAutocut() uint32 { - if x != nil { - return x.Autocut - } - return 0 -} - -func (x *SearchRequest) GetAfter() string { - if x != nil { - return x.After - } - return "" -} - -func (x *SearchRequest) GetSortBy() []*SortBy { - if x != nil { - return x.SortBy - } - return nil -} - -func (x *SearchRequest) GetFilters() *Filters { - if x != nil { - return x.Filters - } - return nil -} - -func (x *SearchRequest) GetHybridSearch() *Hybrid { - if x != nil { - return x.HybridSearch - } - return nil -} - -func (x *SearchRequest) GetBm25Search() *BM25 { - if x != nil { - return x.Bm25Search - } - return nil -} - -func (x *SearchRequest) GetNearVector() *NearVector { - if x != nil { - return x.NearVector - } - return nil -} - -func (x *SearchRequest) GetNearObject() *NearObject { - if x != nil { - return x.NearObject - } - return nil -} - -func (x *SearchRequest) GetNearText() *NearTextSearch { - if x != nil { - return x.NearText - } - return nil -} - -func (x *SearchRequest) GetNearImage() *NearImageSearch { - if x != nil { - return x.NearImage - } - return nil -} - -func (x *SearchRequest) GetNearAudio() *NearAudioSearch { - if x != nil { - return x.NearAudio - } - return nil -} - -func (x *SearchRequest) GetNearVideo() *NearVideoSearch { - if x != nil { - return x.NearVideo - } - return nil -} - -func (x *SearchRequest) GetGenerative() *GenerativeSearch { - if x != nil { - return x.Generative - } - return nil -} - -func (x *SearchRequest) GetRerank() *Rerank { - if x != nil { - return x.Rerank - } - return nil -} - -// Deprecated: Marked as deprecated in v1/search_get.proto. -func (x *SearchRequest) GetUses_123Api() bool { - if x != nil { - return x.Uses_123Api - } - return false -} - -type GroupBy struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // currently only supports one entry (eg just properties, no refs). But might - // be extended in the future. - // protolint:disable:next REPEATED_FIELD_NAMES_PLURALIZED - Path []string `protobuf:"bytes,1,rep,name=path,proto3" json:"path,omitempty"` - NumberOfGroups int32 `protobuf:"varint,2,opt,name=number_of_groups,json=numberOfGroups,proto3" json:"number_of_groups,omitempty"` - ObjectsPerGroup int32 `protobuf:"varint,3,opt,name=objects_per_group,json=objectsPerGroup,proto3" json:"objects_per_group,omitempty"` -} - -func (x *GroupBy) Reset() { - *x = GroupBy{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_search_get_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GroupBy) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GroupBy) ProtoMessage() {} - -func (x *GroupBy) ProtoReflect() protoreflect.Message { - mi := &file_v1_search_get_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GroupBy.ProtoReflect.Descriptor instead. -func (*GroupBy) Descriptor() ([]byte, []int) { - return file_v1_search_get_proto_rawDescGZIP(), []int{1} -} - -func (x *GroupBy) GetPath() []string { - if x != nil { - return x.Path - } - return nil -} - -func (x *GroupBy) GetNumberOfGroups() int32 { - if x != nil { - return x.NumberOfGroups - } - return 0 -} - -func (x *GroupBy) GetObjectsPerGroup() int32 { - if x != nil { - return x.ObjectsPerGroup - } - return 0 -} - -type SortBy struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Ascending bool `protobuf:"varint,1,opt,name=ascending,proto3" json:"ascending,omitempty"` - // currently only supports one entry (eg just properties, no refs). But the - // weaviate datastructure already has paths in it and this makes it easily - // extendable in the future - // protolint:disable:next REPEATED_FIELD_NAMES_PLURALIZED - Path []string `protobuf:"bytes,2,rep,name=path,proto3" json:"path,omitempty"` -} - -func (x *SortBy) Reset() { - *x = SortBy{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_search_get_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *SortBy) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*SortBy) ProtoMessage() {} - -func (x *SortBy) ProtoReflect() protoreflect.Message { - mi := &file_v1_search_get_proto_msgTypes[2] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use SortBy.ProtoReflect.Descriptor instead. -func (*SortBy) Descriptor() ([]byte, []int) { - return file_v1_search_get_proto_rawDescGZIP(), []int{2} -} - -func (x *SortBy) GetAscending() bool { - if x != nil { - return x.Ascending - } - return false -} - -func (x *SortBy) GetPath() []string { - if x != nil { - return x.Path - } - return nil -} - -type GenerativeSearch struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - SingleResponsePrompt string `protobuf:"bytes,1,opt,name=single_response_prompt,json=singleResponsePrompt,proto3" json:"single_response_prompt,omitempty"` - GroupedResponseTask string `protobuf:"bytes,2,opt,name=grouped_response_task,json=groupedResponseTask,proto3" json:"grouped_response_task,omitempty"` - GroupedProperties []string `protobuf:"bytes,3,rep,name=grouped_properties,json=groupedProperties,proto3" json:"grouped_properties,omitempty"` -} - -func (x *GenerativeSearch) Reset() { - *x = GenerativeSearch{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_search_get_proto_msgTypes[3] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GenerativeSearch) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GenerativeSearch) ProtoMessage() {} - -func (x *GenerativeSearch) ProtoReflect() protoreflect.Message { - mi := &file_v1_search_get_proto_msgTypes[3] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GenerativeSearch.ProtoReflect.Descriptor instead. -func (*GenerativeSearch) Descriptor() ([]byte, []int) { - return file_v1_search_get_proto_rawDescGZIP(), []int{3} -} - -func (x *GenerativeSearch) GetSingleResponsePrompt() string { - if x != nil { - return x.SingleResponsePrompt - } - return "" -} - -func (x *GenerativeSearch) GetGroupedResponseTask() string { - if x != nil { - return x.GroupedResponseTask - } - return "" -} - -func (x *GenerativeSearch) GetGroupedProperties() []string { - if x != nil { - return x.GroupedProperties - } - return nil -} - -type MetadataRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Uuid bool `protobuf:"varint,1,opt,name=uuid,proto3" json:"uuid,omitempty"` - Vector bool `protobuf:"varint,2,opt,name=vector,proto3" json:"vector,omitempty"` - CreationTimeUnix bool `protobuf:"varint,3,opt,name=creation_time_unix,json=creationTimeUnix,proto3" json:"creation_time_unix,omitempty"` - LastUpdateTimeUnix bool `protobuf:"varint,4,opt,name=last_update_time_unix,json=lastUpdateTimeUnix,proto3" json:"last_update_time_unix,omitempty"` - Distance bool `protobuf:"varint,5,opt,name=distance,proto3" json:"distance,omitempty"` - Certainty bool `protobuf:"varint,6,opt,name=certainty,proto3" json:"certainty,omitempty"` - Score bool `protobuf:"varint,7,opt,name=score,proto3" json:"score,omitempty"` - ExplainScore bool `protobuf:"varint,8,opt,name=explain_score,json=explainScore,proto3" json:"explain_score,omitempty"` - IsConsistent bool `protobuf:"varint,9,opt,name=is_consistent,json=isConsistent,proto3" json:"is_consistent,omitempty"` -} - -func (x *MetadataRequest) Reset() { - *x = MetadataRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_search_get_proto_msgTypes[4] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *MetadataRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*MetadataRequest) ProtoMessage() {} - -func (x *MetadataRequest) ProtoReflect() protoreflect.Message { - mi := &file_v1_search_get_proto_msgTypes[4] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use MetadataRequest.ProtoReflect.Descriptor instead. -func (*MetadataRequest) Descriptor() ([]byte, []int) { - return file_v1_search_get_proto_rawDescGZIP(), []int{4} -} - -func (x *MetadataRequest) GetUuid() bool { - if x != nil { - return x.Uuid - } - return false -} - -func (x *MetadataRequest) GetVector() bool { - if x != nil { - return x.Vector - } - return false -} - -func (x *MetadataRequest) GetCreationTimeUnix() bool { - if x != nil { - return x.CreationTimeUnix - } - return false -} - -func (x *MetadataRequest) GetLastUpdateTimeUnix() bool { - if x != nil { - return x.LastUpdateTimeUnix - } - return false -} - -func (x *MetadataRequest) GetDistance() bool { - if x != nil { - return x.Distance - } - return false -} - -func (x *MetadataRequest) GetCertainty() bool { - if x != nil { - return x.Certainty - } - return false -} - -func (x *MetadataRequest) GetScore() bool { - if x != nil { - return x.Score - } - return false -} - -func (x *MetadataRequest) GetExplainScore() bool { - if x != nil { - return x.ExplainScore - } - return false -} - -func (x *MetadataRequest) GetIsConsistent() bool { - if x != nil { - return x.IsConsistent - } - return false -} - -type PropertiesRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - NonRefProperties []string `protobuf:"bytes,1,rep,name=non_ref_properties,json=nonRefProperties,proto3" json:"non_ref_properties,omitempty"` - RefProperties []*RefPropertiesRequest `protobuf:"bytes,2,rep,name=ref_properties,json=refProperties,proto3" json:"ref_properties,omitempty"` - ObjectProperties []*ObjectPropertiesRequest `protobuf:"bytes,3,rep,name=object_properties,json=objectProperties,proto3" json:"object_properties,omitempty"` - ReturnAllNonrefProperties bool `protobuf:"varint,11,opt,name=return_all_nonref_properties,json=returnAllNonrefProperties,proto3" json:"return_all_nonref_properties,omitempty"` -} - -func (x *PropertiesRequest) Reset() { - *x = PropertiesRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_search_get_proto_msgTypes[5] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *PropertiesRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*PropertiesRequest) ProtoMessage() {} - -func (x *PropertiesRequest) ProtoReflect() protoreflect.Message { - mi := &file_v1_search_get_proto_msgTypes[5] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use PropertiesRequest.ProtoReflect.Descriptor instead. -func (*PropertiesRequest) Descriptor() ([]byte, []int) { - return file_v1_search_get_proto_rawDescGZIP(), []int{5} -} - -func (x *PropertiesRequest) GetNonRefProperties() []string { - if x != nil { - return x.NonRefProperties - } - return nil -} - -func (x *PropertiesRequest) GetRefProperties() []*RefPropertiesRequest { - if x != nil { - return x.RefProperties - } - return nil -} - -func (x *PropertiesRequest) GetObjectProperties() []*ObjectPropertiesRequest { - if x != nil { - return x.ObjectProperties - } - return nil -} - -func (x *PropertiesRequest) GetReturnAllNonrefProperties() bool { - if x != nil { - return x.ReturnAllNonrefProperties - } - return false -} - -type ObjectPropertiesRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - PropName string `protobuf:"bytes,1,opt,name=prop_name,json=propName,proto3" json:"prop_name,omitempty"` - PrimitiveProperties []string `protobuf:"bytes,2,rep,name=primitive_properties,json=primitiveProperties,proto3" json:"primitive_properties,omitempty"` - ObjectProperties []*ObjectPropertiesRequest `protobuf:"bytes,3,rep,name=object_properties,json=objectProperties,proto3" json:"object_properties,omitempty"` -} - -func (x *ObjectPropertiesRequest) Reset() { - *x = ObjectPropertiesRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_search_get_proto_msgTypes[6] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ObjectPropertiesRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ObjectPropertiesRequest) ProtoMessage() {} - -func (x *ObjectPropertiesRequest) ProtoReflect() protoreflect.Message { - mi := &file_v1_search_get_proto_msgTypes[6] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ObjectPropertiesRequest.ProtoReflect.Descriptor instead. -func (*ObjectPropertiesRequest) Descriptor() ([]byte, []int) { - return file_v1_search_get_proto_rawDescGZIP(), []int{6} -} - -func (x *ObjectPropertiesRequest) GetPropName() string { - if x != nil { - return x.PropName - } - return "" -} - -func (x *ObjectPropertiesRequest) GetPrimitiveProperties() []string { - if x != nil { - return x.PrimitiveProperties - } - return nil -} - -func (x *ObjectPropertiesRequest) GetObjectProperties() []*ObjectPropertiesRequest { - if x != nil { - return x.ObjectProperties - } - return nil -} - -type Hybrid struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Query string `protobuf:"bytes,1,opt,name=query,proto3" json:"query,omitempty"` - Properties []string `protobuf:"bytes,2,rep,name=properties,proto3" json:"properties,omitempty"` - // protolint:disable:next REPEATED_FIELD_NAMES_PLURALIZED - // - // Deprecated: Marked as deprecated in v1/search_get.proto. - Vector []float32 `protobuf:"fixed32,3,rep,packed,name=vector,proto3" json:"vector,omitempty"` // will be removed in the future, use vector_bytes - Alpha float32 `protobuf:"fixed32,4,opt,name=alpha,proto3" json:"alpha,omitempty"` - FusionType Hybrid_FusionType `protobuf:"varint,5,opt,name=fusion_type,json=fusionType,proto3,enum=weaviate.v1.Hybrid_FusionType" json:"fusion_type,omitempty"` - VectorBytes []byte `protobuf:"bytes,6,opt,name=vector_bytes,json=vectorBytes,proto3" json:"vector_bytes,omitempty"` -} - -func (x *Hybrid) Reset() { - *x = Hybrid{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_search_get_proto_msgTypes[7] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *Hybrid) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*Hybrid) ProtoMessage() {} - -func (x *Hybrid) ProtoReflect() protoreflect.Message { - mi := &file_v1_search_get_proto_msgTypes[7] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use Hybrid.ProtoReflect.Descriptor instead. -func (*Hybrid) Descriptor() ([]byte, []int) { - return file_v1_search_get_proto_rawDescGZIP(), []int{7} -} - -func (x *Hybrid) GetQuery() string { - if x != nil { - return x.Query - } - return "" -} - -func (x *Hybrid) GetProperties() []string { - if x != nil { - return x.Properties - } - return nil -} - -// Deprecated: Marked as deprecated in v1/search_get.proto. -func (x *Hybrid) GetVector() []float32 { - if x != nil { - return x.Vector - } - return nil -} - -func (x *Hybrid) GetAlpha() float32 { - if x != nil { - return x.Alpha - } - return 0 -} - -func (x *Hybrid) GetFusionType() Hybrid_FusionType { - if x != nil { - return x.FusionType - } - return Hybrid_FUSION_TYPE_UNSPECIFIED -} - -func (x *Hybrid) GetVectorBytes() []byte { - if x != nil { - return x.VectorBytes - } - return nil -} - -type NearTextSearch struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // protolint:disable:next REPEATED_FIELD_NAMES_PLURALIZED - Query []string `protobuf:"bytes,1,rep,name=query,proto3" json:"query,omitempty"` - Certainty *float64 `protobuf:"fixed64,2,opt,name=certainty,proto3,oneof" json:"certainty,omitempty"` - Distance *float64 `protobuf:"fixed64,3,opt,name=distance,proto3,oneof" json:"distance,omitempty"` - MoveTo *NearTextSearch_Move `protobuf:"bytes,4,opt,name=move_to,json=moveTo,proto3,oneof" json:"move_to,omitempty"` - MoveAway *NearTextSearch_Move `protobuf:"bytes,5,opt,name=move_away,json=moveAway,proto3,oneof" json:"move_away,omitempty"` -} - -func (x *NearTextSearch) Reset() { - *x = NearTextSearch{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_search_get_proto_msgTypes[8] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *NearTextSearch) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*NearTextSearch) ProtoMessage() {} - -func (x *NearTextSearch) ProtoReflect() protoreflect.Message { - mi := &file_v1_search_get_proto_msgTypes[8] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use NearTextSearch.ProtoReflect.Descriptor instead. -func (*NearTextSearch) Descriptor() ([]byte, []int) { - return file_v1_search_get_proto_rawDescGZIP(), []int{8} -} - -func (x *NearTextSearch) GetQuery() []string { - if x != nil { - return x.Query - } - return nil -} - -func (x *NearTextSearch) GetCertainty() float64 { - if x != nil && x.Certainty != nil { - return *x.Certainty - } - return 0 -} - -func (x *NearTextSearch) GetDistance() float64 { - if x != nil && x.Distance != nil { - return *x.Distance - } - return 0 -} - -func (x *NearTextSearch) GetMoveTo() *NearTextSearch_Move { - if x != nil { - return x.MoveTo - } - return nil -} - -func (x *NearTextSearch) GetMoveAway() *NearTextSearch_Move { - if x != nil { - return x.MoveAway - } - return nil -} - -type NearImageSearch struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Image string `protobuf:"bytes,1,opt,name=image,proto3" json:"image,omitempty"` - Certainty *float64 `protobuf:"fixed64,2,opt,name=certainty,proto3,oneof" json:"certainty,omitempty"` - Distance *float64 `protobuf:"fixed64,3,opt,name=distance,proto3,oneof" json:"distance,omitempty"` -} - -func (x *NearImageSearch) Reset() { - *x = NearImageSearch{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_search_get_proto_msgTypes[9] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *NearImageSearch) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*NearImageSearch) ProtoMessage() {} - -func (x *NearImageSearch) ProtoReflect() protoreflect.Message { - mi := &file_v1_search_get_proto_msgTypes[9] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use NearImageSearch.ProtoReflect.Descriptor instead. -func (*NearImageSearch) Descriptor() ([]byte, []int) { - return file_v1_search_get_proto_rawDescGZIP(), []int{9} -} - -func (x *NearImageSearch) GetImage() string { - if x != nil { - return x.Image - } - return "" -} - -func (x *NearImageSearch) GetCertainty() float64 { - if x != nil && x.Certainty != nil { - return *x.Certainty - } - return 0 -} - -func (x *NearImageSearch) GetDistance() float64 { - if x != nil && x.Distance != nil { - return *x.Distance - } - return 0 -} - -type NearAudioSearch struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Audio string `protobuf:"bytes,1,opt,name=audio,proto3" json:"audio,omitempty"` - Certainty *float64 `protobuf:"fixed64,2,opt,name=certainty,proto3,oneof" json:"certainty,omitempty"` - Distance *float64 `protobuf:"fixed64,3,opt,name=distance,proto3,oneof" json:"distance,omitempty"` -} - -func (x *NearAudioSearch) Reset() { - *x = NearAudioSearch{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_search_get_proto_msgTypes[10] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *NearAudioSearch) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*NearAudioSearch) ProtoMessage() {} - -func (x *NearAudioSearch) ProtoReflect() protoreflect.Message { - mi := &file_v1_search_get_proto_msgTypes[10] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use NearAudioSearch.ProtoReflect.Descriptor instead. -func (*NearAudioSearch) Descriptor() ([]byte, []int) { - return file_v1_search_get_proto_rawDescGZIP(), []int{10} -} - -func (x *NearAudioSearch) GetAudio() string { - if x != nil { - return x.Audio - } - return "" -} - -func (x *NearAudioSearch) GetCertainty() float64 { - if x != nil && x.Certainty != nil { - return *x.Certainty - } - return 0 -} - -func (x *NearAudioSearch) GetDistance() float64 { - if x != nil && x.Distance != nil { - return *x.Distance - } - return 0 -} - -type NearVideoSearch struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Video string `protobuf:"bytes,1,opt,name=video,proto3" json:"video,omitempty"` - Certainty *float64 `protobuf:"fixed64,2,opt,name=certainty,proto3,oneof" json:"certainty,omitempty"` - Distance *float64 `protobuf:"fixed64,3,opt,name=distance,proto3,oneof" json:"distance,omitempty"` -} - -func (x *NearVideoSearch) Reset() { - *x = NearVideoSearch{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_search_get_proto_msgTypes[11] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *NearVideoSearch) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*NearVideoSearch) ProtoMessage() {} - -func (x *NearVideoSearch) ProtoReflect() protoreflect.Message { - mi := &file_v1_search_get_proto_msgTypes[11] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use NearVideoSearch.ProtoReflect.Descriptor instead. -func (*NearVideoSearch) Descriptor() ([]byte, []int) { - return file_v1_search_get_proto_rawDescGZIP(), []int{11} -} - -func (x *NearVideoSearch) GetVideo() string { - if x != nil { - return x.Video - } - return "" -} - -func (x *NearVideoSearch) GetCertainty() float64 { - if x != nil && x.Certainty != nil { - return *x.Certainty - } - return 0 -} - -func (x *NearVideoSearch) GetDistance() float64 { - if x != nil && x.Distance != nil { - return *x.Distance - } - return 0 -} - -type BM25 struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Query string `protobuf:"bytes,1,opt,name=query,proto3" json:"query,omitempty"` - Properties []string `protobuf:"bytes,2,rep,name=properties,proto3" json:"properties,omitempty"` -} - -func (x *BM25) Reset() { - *x = BM25{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_search_get_proto_msgTypes[12] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *BM25) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*BM25) ProtoMessage() {} - -func (x *BM25) ProtoReflect() protoreflect.Message { - mi := &file_v1_search_get_proto_msgTypes[12] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use BM25.ProtoReflect.Descriptor instead. -func (*BM25) Descriptor() ([]byte, []int) { - return file_v1_search_get_proto_rawDescGZIP(), []int{12} -} - -func (x *BM25) GetQuery() string { - if x != nil { - return x.Query - } - return "" -} - -func (x *BM25) GetProperties() []string { - if x != nil { - return x.Properties - } - return nil -} - -type RefPropertiesRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - ReferenceProperty string `protobuf:"bytes,1,opt,name=reference_property,json=referenceProperty,proto3" json:"reference_property,omitempty"` - Properties *PropertiesRequest `protobuf:"bytes,2,opt,name=properties,proto3" json:"properties,omitempty"` - Metadata *MetadataRequest `protobuf:"bytes,3,opt,name=metadata,proto3" json:"metadata,omitempty"` - TargetCollection string `protobuf:"bytes,4,opt,name=target_collection,json=targetCollection,proto3" json:"target_collection,omitempty"` -} - -func (x *RefPropertiesRequest) Reset() { - *x = RefPropertiesRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_search_get_proto_msgTypes[13] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *RefPropertiesRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*RefPropertiesRequest) ProtoMessage() {} - -func (x *RefPropertiesRequest) ProtoReflect() protoreflect.Message { - mi := &file_v1_search_get_proto_msgTypes[13] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use RefPropertiesRequest.ProtoReflect.Descriptor instead. -func (*RefPropertiesRequest) Descriptor() ([]byte, []int) { - return file_v1_search_get_proto_rawDescGZIP(), []int{13} -} - -func (x *RefPropertiesRequest) GetReferenceProperty() string { - if x != nil { - return x.ReferenceProperty - } - return "" -} - -func (x *RefPropertiesRequest) GetProperties() *PropertiesRequest { - if x != nil { - return x.Properties - } - return nil -} - -func (x *RefPropertiesRequest) GetMetadata() *MetadataRequest { - if x != nil { - return x.Metadata - } - return nil -} - -func (x *RefPropertiesRequest) GetTargetCollection() string { - if x != nil { - return x.TargetCollection - } - return "" -} - -type NearVector struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // protolint:disable:next REPEATED_FIELD_NAMES_PLURALIZED - // - // Deprecated: Marked as deprecated in v1/search_get.proto. - Vector []float32 `protobuf:"fixed32,1,rep,packed,name=vector,proto3" json:"vector,omitempty"` // will be removed in the future, use vector_bytes - Certainty *float64 `protobuf:"fixed64,2,opt,name=certainty,proto3,oneof" json:"certainty,omitempty"` - Distance *float64 `protobuf:"fixed64,3,opt,name=distance,proto3,oneof" json:"distance,omitempty"` - VectorBytes []byte `protobuf:"bytes,4,opt,name=vector_bytes,json=vectorBytes,proto3" json:"vector_bytes,omitempty"` -} - -func (x *NearVector) Reset() { - *x = NearVector{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_search_get_proto_msgTypes[14] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *NearVector) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*NearVector) ProtoMessage() {} - -func (x *NearVector) ProtoReflect() protoreflect.Message { - mi := &file_v1_search_get_proto_msgTypes[14] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use NearVector.ProtoReflect.Descriptor instead. -func (*NearVector) Descriptor() ([]byte, []int) { - return file_v1_search_get_proto_rawDescGZIP(), []int{14} -} - -// Deprecated: Marked as deprecated in v1/search_get.proto. -func (x *NearVector) GetVector() []float32 { - if x != nil { - return x.Vector - } - return nil -} - -func (x *NearVector) GetCertainty() float64 { - if x != nil && x.Certainty != nil { - return *x.Certainty - } - return 0 -} - -func (x *NearVector) GetDistance() float64 { - if x != nil && x.Distance != nil { - return *x.Distance - } - return 0 -} - -func (x *NearVector) GetVectorBytes() []byte { - if x != nil { - return x.VectorBytes - } - return nil -} - -type NearObject struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` - Certainty *float64 `protobuf:"fixed64,2,opt,name=certainty,proto3,oneof" json:"certainty,omitempty"` - Distance *float64 `protobuf:"fixed64,3,opt,name=distance,proto3,oneof" json:"distance,omitempty"` -} - -func (x *NearObject) Reset() { - *x = NearObject{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_search_get_proto_msgTypes[15] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *NearObject) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*NearObject) ProtoMessage() {} - -func (x *NearObject) ProtoReflect() protoreflect.Message { - mi := &file_v1_search_get_proto_msgTypes[15] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use NearObject.ProtoReflect.Descriptor instead. -func (*NearObject) Descriptor() ([]byte, []int) { - return file_v1_search_get_proto_rawDescGZIP(), []int{15} -} - -func (x *NearObject) GetId() string { - if x != nil { - return x.Id - } - return "" -} - -func (x *NearObject) GetCertainty() float64 { - if x != nil && x.Certainty != nil { - return *x.Certainty - } - return 0 -} - -func (x *NearObject) GetDistance() float64 { - if x != nil && x.Distance != nil { - return *x.Distance - } - return 0 -} - -type Rerank struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Property string `protobuf:"bytes,1,opt,name=property,proto3" json:"property,omitempty"` - Query *string `protobuf:"bytes,2,opt,name=query,proto3,oneof" json:"query,omitempty"` -} - -func (x *Rerank) Reset() { - *x = Rerank{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_search_get_proto_msgTypes[16] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *Rerank) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*Rerank) ProtoMessage() {} - -func (x *Rerank) ProtoReflect() protoreflect.Message { - mi := &file_v1_search_get_proto_msgTypes[16] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use Rerank.ProtoReflect.Descriptor instead. -func (*Rerank) Descriptor() ([]byte, []int) { - return file_v1_search_get_proto_rawDescGZIP(), []int{16} -} - -func (x *Rerank) GetProperty() string { - if x != nil { - return x.Property - } - return "" -} - -func (x *Rerank) GetQuery() string { - if x != nil && x.Query != nil { - return *x.Query - } - return "" -} - -type SearchReply struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Took float32 `protobuf:"fixed32,1,opt,name=took,proto3" json:"took,omitempty"` - Results []*SearchResult `protobuf:"bytes,2,rep,name=results,proto3" json:"results,omitempty"` - GenerativeGroupedResult *string `protobuf:"bytes,3,opt,name=generative_grouped_result,json=generativeGroupedResult,proto3,oneof" json:"generative_grouped_result,omitempty"` - GroupByResults []*GroupByResult `protobuf:"bytes,4,rep,name=group_by_results,json=groupByResults,proto3" json:"group_by_results,omitempty"` -} - -func (x *SearchReply) Reset() { - *x = SearchReply{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_search_get_proto_msgTypes[17] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *SearchReply) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*SearchReply) ProtoMessage() {} - -func (x *SearchReply) ProtoReflect() protoreflect.Message { - mi := &file_v1_search_get_proto_msgTypes[17] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use SearchReply.ProtoReflect.Descriptor instead. -func (*SearchReply) Descriptor() ([]byte, []int) { - return file_v1_search_get_proto_rawDescGZIP(), []int{17} -} - -func (x *SearchReply) GetTook() float32 { - if x != nil { - return x.Took - } - return 0 -} - -func (x *SearchReply) GetResults() []*SearchResult { - if x != nil { - return x.Results - } - return nil -} - -func (x *SearchReply) GetGenerativeGroupedResult() string { - if x != nil && x.GenerativeGroupedResult != nil { - return *x.GenerativeGroupedResult - } - return "" -} - -func (x *SearchReply) GetGroupByResults() []*GroupByResult { - if x != nil { - return x.GroupByResults - } - return nil -} - -type RerankReply struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Score float64 `protobuf:"fixed64,1,opt,name=score,proto3" json:"score,omitempty"` -} - -func (x *RerankReply) Reset() { - *x = RerankReply{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_search_get_proto_msgTypes[18] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *RerankReply) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*RerankReply) ProtoMessage() {} - -func (x *RerankReply) ProtoReflect() protoreflect.Message { - mi := &file_v1_search_get_proto_msgTypes[18] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use RerankReply.ProtoReflect.Descriptor instead. -func (*RerankReply) Descriptor() ([]byte, []int) { - return file_v1_search_get_proto_rawDescGZIP(), []int{18} -} - -func (x *RerankReply) GetScore() float64 { - if x != nil { - return x.Score - } - return 0 -} - -type GenerativeReply struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Result string `protobuf:"bytes,1,opt,name=result,proto3" json:"result,omitempty"` -} - -func (x *GenerativeReply) Reset() { - *x = GenerativeReply{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_search_get_proto_msgTypes[19] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GenerativeReply) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GenerativeReply) ProtoMessage() {} - -func (x *GenerativeReply) ProtoReflect() protoreflect.Message { - mi := &file_v1_search_get_proto_msgTypes[19] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GenerativeReply.ProtoReflect.Descriptor instead. -func (*GenerativeReply) Descriptor() ([]byte, []int) { - return file_v1_search_get_proto_rawDescGZIP(), []int{19} -} - -func (x *GenerativeReply) GetResult() string { - if x != nil { - return x.Result - } - return "" -} - -type GroupByResult struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - MinDistance float32 `protobuf:"fixed32,2,opt,name=min_distance,json=minDistance,proto3" json:"min_distance,omitempty"` - MaxDistance float32 `protobuf:"fixed32,3,opt,name=max_distance,json=maxDistance,proto3" json:"max_distance,omitempty"` - NumberOfObjects int64 `protobuf:"varint,4,opt,name=number_of_objects,json=numberOfObjects,proto3" json:"number_of_objects,omitempty"` - Objects []*SearchResult `protobuf:"bytes,5,rep,name=objects,proto3" json:"objects,omitempty"` - Rerank *RerankReply `protobuf:"bytes,6,opt,name=rerank,proto3,oneof" json:"rerank,omitempty"` - Generative *GenerativeReply `protobuf:"bytes,7,opt,name=generative,proto3,oneof" json:"generative,omitempty"` -} - -func (x *GroupByResult) Reset() { - *x = GroupByResult{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_search_get_proto_msgTypes[20] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GroupByResult) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GroupByResult) ProtoMessage() {} - -func (x *GroupByResult) ProtoReflect() protoreflect.Message { - mi := &file_v1_search_get_proto_msgTypes[20] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GroupByResult.ProtoReflect.Descriptor instead. -func (*GroupByResult) Descriptor() ([]byte, []int) { - return file_v1_search_get_proto_rawDescGZIP(), []int{20} -} - -func (x *GroupByResult) GetName() string { - if x != nil { - return x.Name - } - return "" -} - -func (x *GroupByResult) GetMinDistance() float32 { - if x != nil { - return x.MinDistance - } - return 0 -} - -func (x *GroupByResult) GetMaxDistance() float32 { - if x != nil { - return x.MaxDistance - } - return 0 -} - -func (x *GroupByResult) GetNumberOfObjects() int64 { - if x != nil { - return x.NumberOfObjects - } - return 0 -} - -func (x *GroupByResult) GetObjects() []*SearchResult { - if x != nil { - return x.Objects - } - return nil -} - -func (x *GroupByResult) GetRerank() *RerankReply { - if x != nil { - return x.Rerank - } - return nil -} - -func (x *GroupByResult) GetGenerative() *GenerativeReply { - if x != nil { - return x.Generative - } - return nil -} - -type SearchResult struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Properties *PropertiesResult `protobuf:"bytes,1,opt,name=properties,proto3" json:"properties,omitempty"` - Metadata *MetadataResult `protobuf:"bytes,2,opt,name=metadata,proto3" json:"metadata,omitempty"` -} - -func (x *SearchResult) Reset() { - *x = SearchResult{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_search_get_proto_msgTypes[21] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *SearchResult) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*SearchResult) ProtoMessage() {} - -func (x *SearchResult) ProtoReflect() protoreflect.Message { - mi := &file_v1_search_get_proto_msgTypes[21] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use SearchResult.ProtoReflect.Descriptor instead. -func (*SearchResult) Descriptor() ([]byte, []int) { - return file_v1_search_get_proto_rawDescGZIP(), []int{21} -} - -func (x *SearchResult) GetProperties() *PropertiesResult { - if x != nil { - return x.Properties - } - return nil -} - -func (x *SearchResult) GetMetadata() *MetadataResult { - if x != nil { - return x.Metadata - } - return nil -} - -type MetadataResult struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` - // protolint:disable:next REPEATED_FIELD_NAMES_PLURALIZED - // - // Deprecated: Marked as deprecated in v1/search_get.proto. - Vector []float32 `protobuf:"fixed32,2,rep,packed,name=vector,proto3" json:"vector,omitempty"` - CreationTimeUnix int64 `protobuf:"varint,3,opt,name=creation_time_unix,json=creationTimeUnix,proto3" json:"creation_time_unix,omitempty"` - CreationTimeUnixPresent bool `protobuf:"varint,4,opt,name=creation_time_unix_present,json=creationTimeUnixPresent,proto3" json:"creation_time_unix_present,omitempty"` - LastUpdateTimeUnix int64 `protobuf:"varint,5,opt,name=last_update_time_unix,json=lastUpdateTimeUnix,proto3" json:"last_update_time_unix,omitempty"` - LastUpdateTimeUnixPresent bool `protobuf:"varint,6,opt,name=last_update_time_unix_present,json=lastUpdateTimeUnixPresent,proto3" json:"last_update_time_unix_present,omitempty"` - Distance float32 `protobuf:"fixed32,7,opt,name=distance,proto3" json:"distance,omitempty"` - DistancePresent bool `protobuf:"varint,8,opt,name=distance_present,json=distancePresent,proto3" json:"distance_present,omitempty"` - Certainty float32 `protobuf:"fixed32,9,opt,name=certainty,proto3" json:"certainty,omitempty"` - CertaintyPresent bool `protobuf:"varint,10,opt,name=certainty_present,json=certaintyPresent,proto3" json:"certainty_present,omitempty"` - Score float32 `protobuf:"fixed32,11,opt,name=score,proto3" json:"score,omitempty"` - ScorePresent bool `protobuf:"varint,12,opt,name=score_present,json=scorePresent,proto3" json:"score_present,omitempty"` - ExplainScore string `protobuf:"bytes,13,opt,name=explain_score,json=explainScore,proto3" json:"explain_score,omitempty"` - ExplainScorePresent bool `protobuf:"varint,14,opt,name=explain_score_present,json=explainScorePresent,proto3" json:"explain_score_present,omitempty"` - IsConsistent *bool `protobuf:"varint,15,opt,name=is_consistent,json=isConsistent,proto3,oneof" json:"is_consistent,omitempty"` - Generative string `protobuf:"bytes,16,opt,name=generative,proto3" json:"generative,omitempty"` - GenerativePresent bool `protobuf:"varint,17,opt,name=generative_present,json=generativePresent,proto3" json:"generative_present,omitempty"` - IsConsistentPresent bool `protobuf:"varint,18,opt,name=is_consistent_present,json=isConsistentPresent,proto3" json:"is_consistent_present,omitempty"` - VectorBytes []byte `protobuf:"bytes,19,opt,name=vector_bytes,json=vectorBytes,proto3" json:"vector_bytes,omitempty"` - IdAsBytes []byte `protobuf:"bytes,20,opt,name=id_as_bytes,json=idAsBytes,proto3" json:"id_as_bytes,omitempty"` - RerankScore float64 `protobuf:"fixed64,21,opt,name=rerank_score,json=rerankScore,proto3" json:"rerank_score,omitempty"` - RerankScorePresent bool `protobuf:"varint,22,opt,name=rerank_score_present,json=rerankScorePresent,proto3" json:"rerank_score_present,omitempty"` -} - -func (x *MetadataResult) Reset() { - *x = MetadataResult{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_search_get_proto_msgTypes[22] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *MetadataResult) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*MetadataResult) ProtoMessage() {} - -func (x *MetadataResult) ProtoReflect() protoreflect.Message { - mi := &file_v1_search_get_proto_msgTypes[22] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use MetadataResult.ProtoReflect.Descriptor instead. -func (*MetadataResult) Descriptor() ([]byte, []int) { - return file_v1_search_get_proto_rawDescGZIP(), []int{22} -} - -func (x *MetadataResult) GetId() string { - if x != nil { - return x.Id - } - return "" -} - -// Deprecated: Marked as deprecated in v1/search_get.proto. -func (x *MetadataResult) GetVector() []float32 { - if x != nil { - return x.Vector - } - return nil -} - -func (x *MetadataResult) GetCreationTimeUnix() int64 { - if x != nil { - return x.CreationTimeUnix - } - return 0 -} - -func (x *MetadataResult) GetCreationTimeUnixPresent() bool { - if x != nil { - return x.CreationTimeUnixPresent - } - return false -} - -func (x *MetadataResult) GetLastUpdateTimeUnix() int64 { - if x != nil { - return x.LastUpdateTimeUnix - } - return 0 -} - -func (x *MetadataResult) GetLastUpdateTimeUnixPresent() bool { - if x != nil { - return x.LastUpdateTimeUnixPresent - } - return false -} - -func (x *MetadataResult) GetDistance() float32 { - if x != nil { - return x.Distance - } - return 0 -} - -func (x *MetadataResult) GetDistancePresent() bool { - if x != nil { - return x.DistancePresent - } - return false -} - -func (x *MetadataResult) GetCertainty() float32 { - if x != nil { - return x.Certainty - } - return 0 -} - -func (x *MetadataResult) GetCertaintyPresent() bool { - if x != nil { - return x.CertaintyPresent - } - return false -} - -func (x *MetadataResult) GetScore() float32 { - if x != nil { - return x.Score - } - return 0 -} - -func (x *MetadataResult) GetScorePresent() bool { - if x != nil { - return x.ScorePresent - } - return false -} - -func (x *MetadataResult) GetExplainScore() string { - if x != nil { - return x.ExplainScore - } - return "" -} - -func (x *MetadataResult) GetExplainScorePresent() bool { - if x != nil { - return x.ExplainScorePresent - } - return false -} - -func (x *MetadataResult) GetIsConsistent() bool { - if x != nil && x.IsConsistent != nil { - return *x.IsConsistent - } - return false -} - -func (x *MetadataResult) GetGenerative() string { - if x != nil { - return x.Generative - } - return "" -} - -func (x *MetadataResult) GetGenerativePresent() bool { - if x != nil { - return x.GenerativePresent - } - return false -} - -func (x *MetadataResult) GetIsConsistentPresent() bool { - if x != nil { - return x.IsConsistentPresent - } - return false -} - -func (x *MetadataResult) GetVectorBytes() []byte { - if x != nil { - return x.VectorBytes - } - return nil -} - -func (x *MetadataResult) GetIdAsBytes() []byte { - if x != nil { - return x.IdAsBytes - } - return nil -} - -func (x *MetadataResult) GetRerankScore() float64 { - if x != nil { - return x.RerankScore - } - return 0 -} - -func (x *MetadataResult) GetRerankScorePresent() bool { - if x != nil { - return x.RerankScorePresent - } - return false -} - -type PropertiesResult struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Deprecated: Marked as deprecated in v1/search_get.proto. - NonRefProperties *structpb.Struct `protobuf:"bytes,1,opt,name=non_ref_properties,json=nonRefProperties,proto3" json:"non_ref_properties,omitempty"` - RefProps []*RefPropertiesResult `protobuf:"bytes,2,rep,name=ref_props,json=refProps,proto3" json:"ref_props,omitempty"` - TargetCollection string `protobuf:"bytes,3,opt,name=target_collection,json=targetCollection,proto3" json:"target_collection,omitempty"` - Metadata *MetadataResult `protobuf:"bytes,4,opt,name=metadata,proto3" json:"metadata,omitempty"` - // Deprecated: Marked as deprecated in v1/search_get.proto. - NumberArrayProperties []*NumberArrayProperties `protobuf:"bytes,5,rep,name=number_array_properties,json=numberArrayProperties,proto3" json:"number_array_properties,omitempty"` - // Deprecated: Marked as deprecated in v1/search_get.proto. - IntArrayProperties []*IntArrayProperties `protobuf:"bytes,6,rep,name=int_array_properties,json=intArrayProperties,proto3" json:"int_array_properties,omitempty"` - // Deprecated: Marked as deprecated in v1/search_get.proto. - TextArrayProperties []*TextArrayProperties `protobuf:"bytes,7,rep,name=text_array_properties,json=textArrayProperties,proto3" json:"text_array_properties,omitempty"` - // Deprecated: Marked as deprecated in v1/search_get.proto. - BooleanArrayProperties []*BooleanArrayProperties `protobuf:"bytes,8,rep,name=boolean_array_properties,json=booleanArrayProperties,proto3" json:"boolean_array_properties,omitempty"` - // Deprecated: Marked as deprecated in v1/search_get.proto. - ObjectProperties []*ObjectProperties `protobuf:"bytes,9,rep,name=object_properties,json=objectProperties,proto3" json:"object_properties,omitempty"` - // Deprecated: Marked as deprecated in v1/search_get.proto. - ObjectArrayProperties []*ObjectArrayProperties `protobuf:"bytes,10,rep,name=object_array_properties,json=objectArrayProperties,proto3" json:"object_array_properties,omitempty"` - NonRefProps *Properties `protobuf:"bytes,11,opt,name=non_ref_props,json=nonRefProps,proto3" json:"non_ref_props,omitempty"` - RefPropsRequested bool `protobuf:"varint,12,opt,name=ref_props_requested,json=refPropsRequested,proto3" json:"ref_props_requested,omitempty"` -} - -func (x *PropertiesResult) Reset() { - *x = PropertiesResult{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_search_get_proto_msgTypes[23] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *PropertiesResult) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*PropertiesResult) ProtoMessage() {} - -func (x *PropertiesResult) ProtoReflect() protoreflect.Message { - mi := &file_v1_search_get_proto_msgTypes[23] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use PropertiesResult.ProtoReflect.Descriptor instead. -func (*PropertiesResult) Descriptor() ([]byte, []int) { - return file_v1_search_get_proto_rawDescGZIP(), []int{23} -} - -// Deprecated: Marked as deprecated in v1/search_get.proto. -func (x *PropertiesResult) GetNonRefProperties() *structpb.Struct { - if x != nil { - return x.NonRefProperties - } - return nil -} - -func (x *PropertiesResult) GetRefProps() []*RefPropertiesResult { - if x != nil { - return x.RefProps - } - return nil -} - -func (x *PropertiesResult) GetTargetCollection() string { - if x != nil { - return x.TargetCollection - } - return "" -} - -func (x *PropertiesResult) GetMetadata() *MetadataResult { - if x != nil { - return x.Metadata - } - return nil -} - -// Deprecated: Marked as deprecated in v1/search_get.proto. -func (x *PropertiesResult) GetNumberArrayProperties() []*NumberArrayProperties { - if x != nil { - return x.NumberArrayProperties - } - return nil -} - -// Deprecated: Marked as deprecated in v1/search_get.proto. -func (x *PropertiesResult) GetIntArrayProperties() []*IntArrayProperties { - if x != nil { - return x.IntArrayProperties - } - return nil -} - -// Deprecated: Marked as deprecated in v1/search_get.proto. -func (x *PropertiesResult) GetTextArrayProperties() []*TextArrayProperties { - if x != nil { - return x.TextArrayProperties - } - return nil -} - -// Deprecated: Marked as deprecated in v1/search_get.proto. -func (x *PropertiesResult) GetBooleanArrayProperties() []*BooleanArrayProperties { - if x != nil { - return x.BooleanArrayProperties - } - return nil -} - -// Deprecated: Marked as deprecated in v1/search_get.proto. -func (x *PropertiesResult) GetObjectProperties() []*ObjectProperties { - if x != nil { - return x.ObjectProperties - } - return nil -} - -// Deprecated: Marked as deprecated in v1/search_get.proto. -func (x *PropertiesResult) GetObjectArrayProperties() []*ObjectArrayProperties { - if x != nil { - return x.ObjectArrayProperties - } - return nil -} - -func (x *PropertiesResult) GetNonRefProps() *Properties { - if x != nil { - return x.NonRefProps - } - return nil -} - -func (x *PropertiesResult) GetRefPropsRequested() bool { - if x != nil { - return x.RefPropsRequested - } - return false -} - -type RefPropertiesResult struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Properties []*PropertiesResult `protobuf:"bytes,1,rep,name=properties,proto3" json:"properties,omitempty"` - PropName string `protobuf:"bytes,2,opt,name=prop_name,json=propName,proto3" json:"prop_name,omitempty"` -} - -func (x *RefPropertiesResult) Reset() { - *x = RefPropertiesResult{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_search_get_proto_msgTypes[24] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *RefPropertiesResult) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*RefPropertiesResult) ProtoMessage() {} - -func (x *RefPropertiesResult) ProtoReflect() protoreflect.Message { - mi := &file_v1_search_get_proto_msgTypes[24] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use RefPropertiesResult.ProtoReflect.Descriptor instead. -func (*RefPropertiesResult) Descriptor() ([]byte, []int) { - return file_v1_search_get_proto_rawDescGZIP(), []int{24} -} - -func (x *RefPropertiesResult) GetProperties() []*PropertiesResult { - if x != nil { - return x.Properties - } - return nil -} - -func (x *RefPropertiesResult) GetPropName() string { - if x != nil { - return x.PropName - } - return "" -} - -type NearTextSearch_Move struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Force float32 `protobuf:"fixed32,1,opt,name=force,proto3" json:"force,omitempty"` - Concepts []string `protobuf:"bytes,2,rep,name=concepts,proto3" json:"concepts,omitempty"` - Uuids []string `protobuf:"bytes,3,rep,name=uuids,proto3" json:"uuids,omitempty"` -} - -func (x *NearTextSearch_Move) Reset() { - *x = NearTextSearch_Move{} - if protoimpl.UnsafeEnabled { - mi := &file_v1_search_get_proto_msgTypes[25] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *NearTextSearch_Move) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*NearTextSearch_Move) ProtoMessage() {} - -func (x *NearTextSearch_Move) ProtoReflect() protoreflect.Message { - mi := &file_v1_search_get_proto_msgTypes[25] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use NearTextSearch_Move.ProtoReflect.Descriptor instead. -func (*NearTextSearch_Move) Descriptor() ([]byte, []int) { - return file_v1_search_get_proto_rawDescGZIP(), []int{8, 0} -} - -func (x *NearTextSearch_Move) GetForce() float32 { - if x != nil { - return x.Force - } - return 0 -} - -func (x *NearTextSearch_Move) GetConcepts() []string { - if x != nil { - return x.Concepts - } - return nil -} - -func (x *NearTextSearch_Move) GetUuids() []string { - if x != nil { - return x.Uuids - } - return nil -} - -var File_v1_search_get_proto protoreflect.FileDescriptor - -var file_v1_search_get_proto_rawDesc = []byte{ - 0x0a, 0x13, 0x76, 0x31, 0x2f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x5f, 0x67, 0x65, 0x74, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0b, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, - 0x76, 0x31, 0x1a, 0x1c, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x62, 0x75, 0x66, 0x2f, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x1a, 0x0d, 0x76, 0x31, 0x2f, 0x62, 0x61, 0x73, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, - 0x13, 0x76, 0x31, 0x2f, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x8c, 0x0b, 0x0a, 0x0d, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, - 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, - 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x65, 0x6e, 0x61, 0x6e, 0x74, 0x12, 0x4f, - 0x0a, 0x11, 0x63, 0x6f, 0x6e, 0x73, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x5f, 0x6c, 0x65, - 0x76, 0x65, 0x6c, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1d, 0x2e, 0x77, 0x65, 0x61, 0x76, - 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x73, 0x69, 0x73, 0x74, 0x65, - 0x6e, 0x63, 0x79, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x48, 0x00, 0x52, 0x10, 0x63, 0x6f, 0x6e, 0x73, - 0x69, 0x73, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x88, 0x01, 0x01, 0x12, - 0x43, 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x14, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, - 0x31, 0x2e, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x48, 0x01, 0x52, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, - 0x73, 0x88, 0x01, 0x01, 0x12, 0x3d, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, - 0x18, 0x15, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, - 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x48, 0x02, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, - 0x88, 0x01, 0x01, 0x12, 0x34, 0x0a, 0x08, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x62, 0x79, 0x18, - 0x16, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, - 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x42, 0x79, 0x48, 0x03, 0x52, 0x07, 0x67, - 0x72, 0x6f, 0x75, 0x70, 0x42, 0x79, 0x88, 0x01, 0x01, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x69, 0x6d, - 0x69, 0x74, 0x18, 0x1e, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x12, - 0x16, 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x1f, 0x20, 0x01, 0x28, 0x0d, 0x52, - 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x75, 0x74, 0x6f, 0x63, - 0x75, 0x74, 0x18, 0x20, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x61, 0x75, 0x74, 0x6f, 0x63, 0x75, - 0x74, 0x12, 0x14, 0x0a, 0x05, 0x61, 0x66, 0x74, 0x65, 0x72, 0x18, 0x21, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x05, 0x61, 0x66, 0x74, 0x65, 0x72, 0x12, 0x2c, 0x0a, 0x07, 0x73, 0x6f, 0x72, 0x74, 0x5f, - 0x62, 0x79, 0x18, 0x22, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, - 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x72, 0x74, 0x42, 0x79, 0x52, 0x06, 0x73, - 0x6f, 0x72, 0x74, 0x42, 0x79, 0x12, 0x33, 0x0a, 0x07, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, - 0x18, 0x28, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, - 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x48, 0x04, 0x52, 0x07, - 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x88, 0x01, 0x01, 0x12, 0x3d, 0x0a, 0x0d, 0x68, 0x79, - 0x62, 0x72, 0x69, 0x64, 0x5f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x18, 0x29, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x13, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, - 0x48, 0x79, 0x62, 0x72, 0x69, 0x64, 0x48, 0x05, 0x52, 0x0c, 0x68, 0x79, 0x62, 0x72, 0x69, 0x64, - 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x88, 0x01, 0x01, 0x12, 0x37, 0x0a, 0x0b, 0x62, 0x6d, 0x32, - 0x35, 0x5f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x18, 0x2a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, - 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x4d, 0x32, - 0x35, 0x48, 0x06, 0x52, 0x0a, 0x62, 0x6d, 0x32, 0x35, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x88, - 0x01, 0x01, 0x12, 0x3d, 0x0a, 0x0b, 0x6e, 0x65, 0x61, 0x72, 0x5f, 0x76, 0x65, 0x63, 0x74, 0x6f, - 0x72, 0x18, 0x2b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, - 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x65, 0x61, 0x72, 0x56, 0x65, 0x63, 0x74, 0x6f, 0x72, - 0x48, 0x07, 0x52, 0x0a, 0x6e, 0x65, 0x61, 0x72, 0x56, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x88, 0x01, - 0x01, 0x12, 0x3d, 0x0a, 0x0b, 0x6e, 0x65, 0x61, 0x72, 0x5f, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, - 0x18, 0x2c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, - 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x65, 0x61, 0x72, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x48, - 0x08, 0x52, 0x0a, 0x6e, 0x65, 0x61, 0x72, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x88, 0x01, 0x01, - 0x12, 0x3d, 0x0a, 0x09, 0x6e, 0x65, 0x61, 0x72, 0x5f, 0x74, 0x65, 0x78, 0x74, 0x18, 0x2d, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, - 0x31, 0x2e, 0x4e, 0x65, 0x61, 0x72, 0x54, 0x65, 0x78, 0x74, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, - 0x48, 0x09, 0x52, 0x08, 0x6e, 0x65, 0x61, 0x72, 0x54, 0x65, 0x78, 0x74, 0x88, 0x01, 0x01, 0x12, - 0x40, 0x0a, 0x0a, 0x6e, 0x65, 0x61, 0x72, 0x5f, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x18, 0x2e, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, - 0x31, 0x2e, 0x4e, 0x65, 0x61, 0x72, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x53, 0x65, 0x61, 0x72, 0x63, - 0x68, 0x48, 0x0a, 0x52, 0x09, 0x6e, 0x65, 0x61, 0x72, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x88, 0x01, - 0x01, 0x12, 0x40, 0x0a, 0x0a, 0x6e, 0x65, 0x61, 0x72, 0x5f, 0x61, 0x75, 0x64, 0x69, 0x6f, 0x18, - 0x2f, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, - 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x65, 0x61, 0x72, 0x41, 0x75, 0x64, 0x69, 0x6f, 0x53, 0x65, 0x61, - 0x72, 0x63, 0x68, 0x48, 0x0b, 0x52, 0x09, 0x6e, 0x65, 0x61, 0x72, 0x41, 0x75, 0x64, 0x69, 0x6f, - 0x88, 0x01, 0x01, 0x12, 0x40, 0x0a, 0x0a, 0x6e, 0x65, 0x61, 0x72, 0x5f, 0x76, 0x69, 0x64, 0x65, - 0x6f, 0x18, 0x30, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, - 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x65, 0x61, 0x72, 0x56, 0x69, 0x64, 0x65, 0x6f, 0x53, - 0x65, 0x61, 0x72, 0x63, 0x68, 0x48, 0x0c, 0x52, 0x09, 0x6e, 0x65, 0x61, 0x72, 0x56, 0x69, 0x64, - 0x65, 0x6f, 0x88, 0x01, 0x01, 0x12, 0x42, 0x0a, 0x0a, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, - 0x69, 0x76, 0x65, 0x18, 0x3c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x77, 0x65, 0x61, 0x76, - 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, - 0x76, 0x65, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x48, 0x0d, 0x52, 0x0a, 0x67, 0x65, 0x6e, 0x65, - 0x72, 0x61, 0x74, 0x69, 0x76, 0x65, 0x88, 0x01, 0x01, 0x12, 0x30, 0x0a, 0x06, 0x72, 0x65, 0x72, - 0x61, 0x6e, 0x6b, 0x18, 0x3d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x77, 0x65, 0x61, 0x76, - 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x72, 0x61, 0x6e, 0x6b, 0x48, 0x0e, - 0x52, 0x06, 0x72, 0x65, 0x72, 0x61, 0x6e, 0x6b, 0x88, 0x01, 0x01, 0x12, 0x24, 0x0a, 0x0c, 0x75, - 0x73, 0x65, 0x73, 0x5f, 0x31, 0x32, 0x33, 0x5f, 0x61, 0x70, 0x69, 0x18, 0x64, 0x20, 0x01, 0x28, - 0x08, 0x42, 0x02, 0x18, 0x01, 0x52, 0x0a, 0x75, 0x73, 0x65, 0x73, 0x31, 0x32, 0x33, 0x41, 0x70, - 0x69, 0x42, 0x14, 0x0a, 0x12, 0x5f, 0x63, 0x6f, 0x6e, 0x73, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x63, - 0x79, 0x5f, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x42, 0x0d, 0x0a, 0x0b, 0x5f, 0x70, 0x72, 0x6f, 0x70, - 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, - 0x61, 0x74, 0x61, 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x62, 0x79, - 0x42, 0x0a, 0x0a, 0x08, 0x5f, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x42, 0x10, 0x0a, 0x0e, - 0x5f, 0x68, 0x79, 0x62, 0x72, 0x69, 0x64, 0x5f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x42, 0x0e, - 0x0a, 0x0c, 0x5f, 0x62, 0x6d, 0x32, 0x35, 0x5f, 0x73, 0x65, 0x61, 0x72, 0x63, 0x68, 0x42, 0x0e, - 0x0a, 0x0c, 0x5f, 0x6e, 0x65, 0x61, 0x72, 0x5f, 0x76, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x42, 0x0e, - 0x0a, 0x0c, 0x5f, 0x6e, 0x65, 0x61, 0x72, 0x5f, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x42, 0x0c, - 0x0a, 0x0a, 0x5f, 0x6e, 0x65, 0x61, 0x72, 0x5f, 0x74, 0x65, 0x78, 0x74, 0x42, 0x0d, 0x0a, 0x0b, - 0x5f, 0x6e, 0x65, 0x61, 0x72, 0x5f, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x42, 0x0d, 0x0a, 0x0b, 0x5f, - 0x6e, 0x65, 0x61, 0x72, 0x5f, 0x61, 0x75, 0x64, 0x69, 0x6f, 0x42, 0x0d, 0x0a, 0x0b, 0x5f, 0x6e, - 0x65, 0x61, 0x72, 0x5f, 0x76, 0x69, 0x64, 0x65, 0x6f, 0x42, 0x0d, 0x0a, 0x0b, 0x5f, 0x67, 0x65, - 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x76, 0x65, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x72, 0x65, 0x72, - 0x61, 0x6e, 0x6b, 0x22, 0x73, 0x0a, 0x07, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x42, 0x79, 0x12, 0x12, - 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, - 0x74, 0x68, 0x12, 0x28, 0x0a, 0x10, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x5f, 0x6f, 0x66, 0x5f, - 0x67, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0e, 0x6e, 0x75, - 0x6d, 0x62, 0x65, 0x72, 0x4f, 0x66, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x12, 0x2a, 0x0a, 0x11, - 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x5f, 0x70, 0x65, 0x72, 0x5f, 0x67, 0x72, 0x6f, 0x75, - 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0f, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, - 0x50, 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x22, 0x3a, 0x0a, 0x06, 0x53, 0x6f, 0x72, 0x74, - 0x42, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x61, 0x73, 0x63, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x61, 0x73, 0x63, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, - 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, - 0x70, 0x61, 0x74, 0x68, 0x22, 0xab, 0x01, 0x0a, 0x10, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, - 0x69, 0x76, 0x65, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x12, 0x34, 0x0a, 0x16, 0x73, 0x69, 0x6e, - 0x67, 0x6c, 0x65, 0x5f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x70, 0x72, 0x6f, - 0x6d, 0x70, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x14, 0x73, 0x69, 0x6e, 0x67, 0x6c, - 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x50, 0x72, 0x6f, 0x6d, 0x70, 0x74, 0x12, - 0x32, 0x0a, 0x15, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x65, 0x64, 0x5f, 0x72, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x5f, 0x74, 0x61, 0x73, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, - 0x67, 0x72, 0x6f, 0x75, 0x70, 0x65, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x54, - 0x61, 0x73, 0x6b, 0x12, 0x2d, 0x0a, 0x12, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x65, 0x64, 0x5f, 0x70, - 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, - 0x11, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x65, 0x64, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, - 0x65, 0x73, 0x22, 0xb8, 0x02, 0x0a, 0x0f, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x75, 0x69, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x04, 0x75, 0x75, 0x69, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x76, 0x65, - 0x63, 0x74, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x76, 0x65, 0x63, 0x74, - 0x6f, 0x72, 0x12, 0x2c, 0x0a, 0x12, 0x63, 0x72, 0x65, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, - 0x69, 0x6d, 0x65, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, - 0x63, 0x72, 0x65, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x55, 0x6e, 0x69, 0x78, - 0x12, 0x31, 0x0a, 0x15, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x5f, - 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x12, 0x6c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x55, - 0x6e, 0x69, 0x78, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x69, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x64, 0x69, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x12, - 0x1c, 0x0a, 0x09, 0x63, 0x65, 0x72, 0x74, 0x61, 0x69, 0x6e, 0x74, 0x79, 0x18, 0x06, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x09, 0x63, 0x65, 0x72, 0x74, 0x61, 0x69, 0x6e, 0x74, 0x79, 0x12, 0x14, 0x0a, - 0x05, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x73, 0x63, - 0x6f, 0x72, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x65, 0x78, 0x70, 0x6c, 0x61, 0x69, 0x6e, 0x5f, 0x73, - 0x63, 0x6f, 0x72, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x65, 0x78, 0x70, 0x6c, - 0x61, 0x69, 0x6e, 0x53, 0x63, 0x6f, 0x72, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x73, 0x5f, 0x63, - 0x6f, 0x6e, 0x73, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x74, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x0c, 0x69, 0x73, 0x43, 0x6f, 0x6e, 0x73, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x74, 0x22, 0x9f, 0x02, - 0x0a, 0x11, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x2c, 0x0a, 0x12, 0x6e, 0x6f, 0x6e, 0x5f, 0x72, 0x65, 0x66, 0x5f, 0x70, - 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, - 0x10, 0x6e, 0x6f, 0x6e, 0x52, 0x65, 0x66, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, - 0x73, 0x12, 0x48, 0x0a, 0x0e, 0x72, 0x65, 0x66, 0x5f, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, - 0x69, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x77, 0x65, 0x61, 0x76, - 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x50, 0x72, 0x6f, 0x70, 0x65, - 0x72, 0x74, 0x69, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x0d, 0x72, 0x65, - 0x66, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x51, 0x0a, 0x11, 0x6f, - 0x62, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, - 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, - 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x65, - 0x72, 0x74, 0x69, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x10, 0x6f, 0x62, - 0x6a, 0x65, 0x63, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x3f, - 0x0a, 0x1c, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x5f, 0x61, 0x6c, 0x6c, 0x5f, 0x6e, 0x6f, 0x6e, - 0x72, 0x65, 0x66, 0x5f, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x0b, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x19, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x41, 0x6c, 0x6c, 0x4e, - 0x6f, 0x6e, 0x72, 0x65, 0x66, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x22, - 0xbc, 0x01, 0x0a, 0x17, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, - 0x74, 0x69, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x70, - 0x72, 0x6f, 0x70, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, - 0x70, 0x72, 0x6f, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x31, 0x0a, 0x14, 0x70, 0x72, 0x69, 0x6d, - 0x69, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, - 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x13, 0x70, 0x72, 0x69, 0x6d, 0x69, 0x74, 0x69, 0x76, - 0x65, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x51, 0x0a, 0x11, 0x6f, - 0x62, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, - 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, - 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x65, - 0x72, 0x74, 0x69, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x10, 0x6f, 0x62, - 0x6a, 0x65, 0x63, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x22, 0xb7, - 0x02, 0x0a, 0x06, 0x48, 0x79, 0x62, 0x72, 0x69, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x71, 0x75, 0x65, - 0x72, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x12, - 0x1e, 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x02, 0x20, - 0x03, 0x28, 0x09, 0x52, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, - 0x1a, 0x0a, 0x06, 0x76, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x03, 0x28, 0x02, 0x42, - 0x02, 0x18, 0x01, 0x52, 0x06, 0x76, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x61, - 0x6c, 0x70, 0x68, 0x61, 0x18, 0x04, 0x20, 0x01, 0x28, 0x02, 0x52, 0x05, 0x61, 0x6c, 0x70, 0x68, - 0x61, 0x12, 0x3f, 0x0a, 0x0b, 0x66, 0x75, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x79, 0x70, 0x65, - 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1e, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, - 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x79, 0x62, 0x72, 0x69, 0x64, 0x2e, 0x46, 0x75, 0x73, 0x69, - 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0a, 0x66, 0x75, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x79, - 0x70, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x76, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, 0x62, 0x79, 0x74, - 0x65, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x76, 0x65, 0x63, 0x74, 0x6f, 0x72, - 0x42, 0x79, 0x74, 0x65, 0x73, 0x22, 0x61, 0x0a, 0x0a, 0x46, 0x75, 0x73, 0x69, 0x6f, 0x6e, 0x54, - 0x79, 0x70, 0x65, 0x12, 0x1b, 0x0a, 0x17, 0x46, 0x55, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x54, 0x59, - 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, - 0x12, 0x16, 0x0a, 0x12, 0x46, 0x55, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, - 0x52, 0x41, 0x4e, 0x4b, 0x45, 0x44, 0x10, 0x01, 0x12, 0x1e, 0x0a, 0x1a, 0x46, 0x55, 0x53, 0x49, - 0x4f, 0x4e, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x52, 0x45, 0x4c, 0x41, 0x54, 0x49, 0x56, 0x45, - 0x5f, 0x53, 0x43, 0x4f, 0x52, 0x45, 0x10, 0x02, 0x22, 0xf3, 0x02, 0x0a, 0x0e, 0x4e, 0x65, 0x61, - 0x72, 0x54, 0x65, 0x78, 0x74, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x12, 0x14, 0x0a, 0x05, 0x71, - 0x75, 0x65, 0x72, 0x79, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x71, 0x75, 0x65, 0x72, - 0x79, 0x12, 0x21, 0x0a, 0x09, 0x63, 0x65, 0x72, 0x74, 0x61, 0x69, 0x6e, 0x74, 0x79, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x01, 0x48, 0x00, 0x52, 0x09, 0x63, 0x65, 0x72, 0x74, 0x61, 0x69, 0x6e, 0x74, - 0x79, 0x88, 0x01, 0x01, 0x12, 0x1f, 0x0a, 0x08, 0x64, 0x69, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x01, 0x48, 0x01, 0x52, 0x08, 0x64, 0x69, 0x73, 0x74, 0x61, 0x6e, - 0x63, 0x65, 0x88, 0x01, 0x01, 0x12, 0x3e, 0x0a, 0x07, 0x6d, 0x6f, 0x76, 0x65, 0x5f, 0x74, 0x6f, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, - 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x65, 0x61, 0x72, 0x54, 0x65, 0x78, 0x74, 0x53, 0x65, 0x61, - 0x72, 0x63, 0x68, 0x2e, 0x4d, 0x6f, 0x76, 0x65, 0x48, 0x02, 0x52, 0x06, 0x6d, 0x6f, 0x76, 0x65, - 0x54, 0x6f, 0x88, 0x01, 0x01, 0x12, 0x42, 0x0a, 0x09, 0x6d, 0x6f, 0x76, 0x65, 0x5f, 0x61, 0x77, - 0x61, 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, - 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x65, 0x61, 0x72, 0x54, 0x65, 0x78, 0x74, 0x53, - 0x65, 0x61, 0x72, 0x63, 0x68, 0x2e, 0x4d, 0x6f, 0x76, 0x65, 0x48, 0x03, 0x52, 0x08, 0x6d, 0x6f, - 0x76, 0x65, 0x41, 0x77, 0x61, 0x79, 0x88, 0x01, 0x01, 0x1a, 0x4e, 0x0a, 0x04, 0x4d, 0x6f, 0x76, - 0x65, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x02, - 0x52, 0x05, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x63, 0x6f, 0x6e, 0x63, 0x65, - 0x70, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x63, 0x6f, 0x6e, 0x63, 0x65, - 0x70, 0x74, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x75, 0x75, 0x69, 0x64, 0x73, 0x18, 0x03, 0x20, 0x03, - 0x28, 0x09, 0x52, 0x05, 0x75, 0x75, 0x69, 0x64, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x63, 0x65, - 0x72, 0x74, 0x61, 0x69, 0x6e, 0x74, 0x79, 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x64, 0x69, 0x73, 0x74, - 0x61, 0x6e, 0x63, 0x65, 0x42, 0x0a, 0x0a, 0x08, 0x5f, 0x6d, 0x6f, 0x76, 0x65, 0x5f, 0x74, 0x6f, - 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x6d, 0x6f, 0x76, 0x65, 0x5f, 0x61, 0x77, 0x61, 0x79, 0x22, 0x86, - 0x01, 0x0a, 0x0f, 0x4e, 0x65, 0x61, 0x72, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x53, 0x65, 0x61, 0x72, - 0x63, 0x68, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x05, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x12, 0x21, 0x0a, 0x09, 0x63, 0x65, 0x72, 0x74, - 0x61, 0x69, 0x6e, 0x74, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x48, 0x00, 0x52, 0x09, 0x63, - 0x65, 0x72, 0x74, 0x61, 0x69, 0x6e, 0x74, 0x79, 0x88, 0x01, 0x01, 0x12, 0x1f, 0x0a, 0x08, 0x64, - 0x69, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x01, 0x48, 0x01, 0x52, - 0x08, 0x64, 0x69, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x88, 0x01, 0x01, 0x42, 0x0c, 0x0a, 0x0a, - 0x5f, 0x63, 0x65, 0x72, 0x74, 0x61, 0x69, 0x6e, 0x74, 0x79, 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x64, - 0x69, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x22, 0x86, 0x01, 0x0a, 0x0f, 0x4e, 0x65, 0x61, 0x72, - 0x41, 0x75, 0x64, 0x69, 0x6f, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x12, 0x14, 0x0a, 0x05, 0x61, - 0x75, 0x64, 0x69, 0x6f, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x61, 0x75, 0x64, 0x69, - 0x6f, 0x12, 0x21, 0x0a, 0x09, 0x63, 0x65, 0x72, 0x74, 0x61, 0x69, 0x6e, 0x74, 0x79, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x01, 0x48, 0x00, 0x52, 0x09, 0x63, 0x65, 0x72, 0x74, 0x61, 0x69, 0x6e, 0x74, - 0x79, 0x88, 0x01, 0x01, 0x12, 0x1f, 0x0a, 0x08, 0x64, 0x69, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x01, 0x48, 0x01, 0x52, 0x08, 0x64, 0x69, 0x73, 0x74, 0x61, 0x6e, - 0x63, 0x65, 0x88, 0x01, 0x01, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x63, 0x65, 0x72, 0x74, 0x61, 0x69, - 0x6e, 0x74, 0x79, 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x64, 0x69, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, - 0x22, 0x86, 0x01, 0x0a, 0x0f, 0x4e, 0x65, 0x61, 0x72, 0x56, 0x69, 0x64, 0x65, 0x6f, 0x53, 0x65, - 0x61, 0x72, 0x63, 0x68, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x69, 0x64, 0x65, 0x6f, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x69, 0x64, 0x65, 0x6f, 0x12, 0x21, 0x0a, 0x09, 0x63, 0x65, - 0x72, 0x74, 0x61, 0x69, 0x6e, 0x74, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x48, 0x00, 0x52, - 0x09, 0x63, 0x65, 0x72, 0x74, 0x61, 0x69, 0x6e, 0x74, 0x79, 0x88, 0x01, 0x01, 0x12, 0x1f, 0x0a, - 0x08, 0x64, 0x69, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x01, 0x48, - 0x01, 0x52, 0x08, 0x64, 0x69, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x88, 0x01, 0x01, 0x42, 0x0c, - 0x0a, 0x0a, 0x5f, 0x63, 0x65, 0x72, 0x74, 0x61, 0x69, 0x6e, 0x74, 0x79, 0x42, 0x0b, 0x0a, 0x09, - 0x5f, 0x64, 0x69, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x22, 0x3c, 0x0a, 0x04, 0x42, 0x4d, 0x32, - 0x35, 0x12, 0x14, 0x0a, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x12, 0x1e, 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, - 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x70, 0x72, 0x6f, - 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x22, 0xec, 0x01, 0x0a, 0x14, 0x52, 0x65, 0x66, 0x50, - 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x2d, 0x0a, 0x12, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x5f, 0x70, 0x72, - 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x72, 0x65, - 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x12, - 0x3e, 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, - 0x31, 0x2e, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x52, 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, - 0x38, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x1c, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, - 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, - 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x2b, 0x0a, 0x11, 0x74, 0x61, 0x72, - 0x67, 0x65, 0x74, 0x5f, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x43, 0x6f, 0x6c, 0x6c, - 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xaa, 0x01, 0x0a, 0x0a, 0x4e, 0x65, 0x61, 0x72, 0x56, - 0x65, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x1a, 0x0a, 0x06, 0x76, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x18, - 0x01, 0x20, 0x03, 0x28, 0x02, 0x42, 0x02, 0x18, 0x01, 0x52, 0x06, 0x76, 0x65, 0x63, 0x74, 0x6f, - 0x72, 0x12, 0x21, 0x0a, 0x09, 0x63, 0x65, 0x72, 0x74, 0x61, 0x69, 0x6e, 0x74, 0x79, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x01, 0x48, 0x00, 0x52, 0x09, 0x63, 0x65, 0x72, 0x74, 0x61, 0x69, 0x6e, 0x74, - 0x79, 0x88, 0x01, 0x01, 0x12, 0x1f, 0x0a, 0x08, 0x64, 0x69, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x01, 0x48, 0x01, 0x52, 0x08, 0x64, 0x69, 0x73, 0x74, 0x61, 0x6e, - 0x63, 0x65, 0x88, 0x01, 0x01, 0x12, 0x21, 0x0a, 0x0c, 0x76, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, - 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x76, 0x65, 0x63, - 0x74, 0x6f, 0x72, 0x42, 0x79, 0x74, 0x65, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x63, 0x65, 0x72, - 0x74, 0x61, 0x69, 0x6e, 0x74, 0x79, 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x64, 0x69, 0x73, 0x74, 0x61, - 0x6e, 0x63, 0x65, 0x22, 0x7b, 0x0a, 0x0a, 0x4e, 0x65, 0x61, 0x72, 0x4f, 0x62, 0x6a, 0x65, 0x63, - 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, - 0x64, 0x12, 0x21, 0x0a, 0x09, 0x63, 0x65, 0x72, 0x74, 0x61, 0x69, 0x6e, 0x74, 0x79, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x01, 0x48, 0x00, 0x52, 0x09, 0x63, 0x65, 0x72, 0x74, 0x61, 0x69, 0x6e, 0x74, - 0x79, 0x88, 0x01, 0x01, 0x12, 0x1f, 0x0a, 0x08, 0x64, 0x69, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x01, 0x48, 0x01, 0x52, 0x08, 0x64, 0x69, 0x73, 0x74, 0x61, 0x6e, - 0x63, 0x65, 0x88, 0x01, 0x01, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x63, 0x65, 0x72, 0x74, 0x61, 0x69, - 0x6e, 0x74, 0x79, 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x64, 0x69, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, - 0x22, 0x49, 0x0a, 0x06, 0x52, 0x65, 0x72, 0x61, 0x6e, 0x6b, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, - 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, - 0x6f, 0x70, 0x65, 0x72, 0x74, 0x79, 0x12, 0x19, 0x0a, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x88, 0x01, - 0x01, 0x42, 0x08, 0x0a, 0x06, 0x5f, 0x71, 0x75, 0x65, 0x72, 0x79, 0x22, 0xfb, 0x01, 0x0a, 0x0b, - 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x74, - 0x6f, 0x6f, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x02, 0x52, 0x04, 0x74, 0x6f, 0x6f, 0x6b, 0x12, - 0x33, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x19, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, - 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x07, 0x72, 0x65, 0x73, - 0x75, 0x6c, 0x74, 0x73, 0x12, 0x3f, 0x0a, 0x19, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, - 0x76, 0x65, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x65, 0x64, 0x5f, 0x72, 0x65, 0x73, 0x75, 0x6c, - 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x17, 0x67, 0x65, 0x6e, 0x65, 0x72, - 0x61, 0x74, 0x69, 0x76, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x65, 0x64, 0x52, 0x65, 0x73, 0x75, - 0x6c, 0x74, 0x88, 0x01, 0x01, 0x12, 0x44, 0x0a, 0x10, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x62, - 0x79, 0x5f, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x1a, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x72, - 0x6f, 0x75, 0x70, 0x42, 0x79, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x0e, 0x67, 0x72, 0x6f, - 0x75, 0x70, 0x42, 0x79, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x42, 0x1c, 0x0a, 0x1a, 0x5f, - 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, - 0x65, 0x64, 0x5f, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x22, 0x23, 0x0a, 0x0b, 0x52, 0x65, 0x72, - 0x61, 0x6e, 0x6b, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x63, 0x6f, 0x72, - 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x22, 0x29, - 0x0a, 0x0f, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x76, 0x65, 0x52, 0x65, 0x70, 0x6c, - 0x79, 0x12, 0x16, 0x0a, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x22, 0xde, 0x02, 0x0a, 0x0d, 0x47, 0x72, - 0x6f, 0x75, 0x70, 0x42, 0x79, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, - 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, - 0x21, 0x0a, 0x0c, 0x6d, 0x69, 0x6e, 0x5f, 0x64, 0x69, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x02, 0x52, 0x0b, 0x6d, 0x69, 0x6e, 0x44, 0x69, 0x73, 0x74, 0x61, 0x6e, - 0x63, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x6d, 0x61, 0x78, 0x5f, 0x64, 0x69, 0x73, 0x74, 0x61, 0x6e, - 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x02, 0x52, 0x0b, 0x6d, 0x61, 0x78, 0x44, 0x69, 0x73, - 0x74, 0x61, 0x6e, 0x63, 0x65, 0x12, 0x2a, 0x0a, 0x11, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x5f, - 0x6f, 0x66, 0x5f, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, - 0x52, 0x0f, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x4f, 0x66, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, - 0x73, 0x12, 0x33, 0x0a, 0x07, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, - 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x07, 0x6f, - 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x35, 0x0a, 0x06, 0x72, 0x65, 0x72, 0x61, 0x6e, 0x6b, - 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, - 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x72, 0x61, 0x6e, 0x6b, 0x52, 0x65, 0x70, 0x6c, 0x79, - 0x48, 0x00, 0x52, 0x06, 0x72, 0x65, 0x72, 0x61, 0x6e, 0x6b, 0x88, 0x01, 0x01, 0x12, 0x41, 0x0a, - 0x0a, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x76, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x1c, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, - 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x76, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x48, - 0x01, 0x52, 0x0a, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x76, 0x65, 0x88, 0x01, 0x01, - 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x72, 0x65, 0x72, 0x61, 0x6e, 0x6b, 0x42, 0x0d, 0x0a, 0x0b, 0x5f, - 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x76, 0x65, 0x22, 0x86, 0x01, 0x0a, 0x0c, 0x53, - 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x3d, 0x0a, 0x0a, 0x70, - 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1d, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, - 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x0a, - 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x37, 0x0a, 0x08, 0x6d, 0x65, - 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x77, - 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, - 0x61, 0x74, 0x61, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, - 0x61, 0x74, 0x61, 0x22, 0x99, 0x07, 0x0a, 0x0e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, - 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x1a, 0x0a, 0x06, 0x76, 0x65, 0x63, 0x74, 0x6f, 0x72, - 0x18, 0x02, 0x20, 0x03, 0x28, 0x02, 0x42, 0x02, 0x18, 0x01, 0x52, 0x06, 0x76, 0x65, 0x63, 0x74, - 0x6f, 0x72, 0x12, 0x2c, 0x0a, 0x12, 0x63, 0x72, 0x65, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, - 0x69, 0x6d, 0x65, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x10, - 0x63, 0x72, 0x65, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x55, 0x6e, 0x69, 0x78, - 0x12, 0x3b, 0x0a, 0x1a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x69, 0x6d, - 0x65, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x17, 0x63, 0x72, 0x65, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x69, - 0x6d, 0x65, 0x55, 0x6e, 0x69, 0x78, 0x50, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x12, 0x31, 0x0a, - 0x15, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x5f, 0x74, 0x69, 0x6d, - 0x65, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x12, 0x6c, 0x61, - 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x55, 0x6e, 0x69, 0x78, - 0x12, 0x40, 0x0a, 0x1d, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x5f, - 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x5f, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, - 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x19, 0x6c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, - 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x55, 0x6e, 0x69, 0x78, 0x50, 0x72, 0x65, 0x73, 0x65, - 0x6e, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x69, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x18, 0x07, - 0x20, 0x01, 0x28, 0x02, 0x52, 0x08, 0x64, 0x69, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x12, 0x29, - 0x0a, 0x10, 0x64, 0x69, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x5f, 0x70, 0x72, 0x65, 0x73, 0x65, - 0x6e, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x64, 0x69, 0x73, 0x74, 0x61, 0x6e, - 0x63, 0x65, 0x50, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x65, 0x72, - 0x74, 0x61, 0x69, 0x6e, 0x74, 0x79, 0x18, 0x09, 0x20, 0x01, 0x28, 0x02, 0x52, 0x09, 0x63, 0x65, - 0x72, 0x74, 0x61, 0x69, 0x6e, 0x74, 0x79, 0x12, 0x2b, 0x0a, 0x11, 0x63, 0x65, 0x72, 0x74, 0x61, - 0x69, 0x6e, 0x74, 0x79, 0x5f, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x18, 0x0a, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x10, 0x63, 0x65, 0x72, 0x74, 0x61, 0x69, 0x6e, 0x74, 0x79, 0x50, 0x72, 0x65, - 0x73, 0x65, 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x18, 0x0b, 0x20, - 0x01, 0x28, 0x02, 0x52, 0x05, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x73, 0x63, - 0x6f, 0x72, 0x65, 0x5f, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x18, 0x0c, 0x20, 0x01, 0x28, - 0x08, 0x52, 0x0c, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x50, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x12, - 0x23, 0x0a, 0x0d, 0x65, 0x78, 0x70, 0x6c, 0x61, 0x69, 0x6e, 0x5f, 0x73, 0x63, 0x6f, 0x72, 0x65, - 0x18, 0x0d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x65, 0x78, 0x70, 0x6c, 0x61, 0x69, 0x6e, 0x53, - 0x63, 0x6f, 0x72, 0x65, 0x12, 0x32, 0x0a, 0x15, 0x65, 0x78, 0x70, 0x6c, 0x61, 0x69, 0x6e, 0x5f, - 0x73, 0x63, 0x6f, 0x72, 0x65, 0x5f, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x18, 0x0e, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x13, 0x65, 0x78, 0x70, 0x6c, 0x61, 0x69, 0x6e, 0x53, 0x63, 0x6f, 0x72, - 0x65, 0x50, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x12, 0x28, 0x0a, 0x0d, 0x69, 0x73, 0x5f, 0x63, - 0x6f, 0x6e, 0x73, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x74, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x08, 0x48, - 0x00, 0x52, 0x0c, 0x69, 0x73, 0x43, 0x6f, 0x6e, 0x73, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x74, 0x88, - 0x01, 0x01, 0x12, 0x1e, 0x0a, 0x0a, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x76, 0x65, - 0x18, 0x10, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, - 0x76, 0x65, 0x12, 0x2d, 0x0a, 0x12, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x76, 0x65, - 0x5f, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x18, 0x11, 0x20, 0x01, 0x28, 0x08, 0x52, 0x11, - 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x76, 0x65, 0x50, 0x72, 0x65, 0x73, 0x65, 0x6e, - 0x74, 0x12, 0x32, 0x0a, 0x15, 0x69, 0x73, 0x5f, 0x63, 0x6f, 0x6e, 0x73, 0x69, 0x73, 0x74, 0x65, - 0x6e, 0x74, 0x5f, 0x70, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x18, 0x12, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x13, 0x69, 0x73, 0x43, 0x6f, 0x6e, 0x73, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x74, 0x50, 0x72, - 0x65, 0x73, 0x65, 0x6e, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x76, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x5f, - 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x13, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x76, 0x65, 0x63, - 0x74, 0x6f, 0x72, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x1e, 0x0a, 0x0b, 0x69, 0x64, 0x5f, 0x61, - 0x73, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x14, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x69, - 0x64, 0x41, 0x73, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x72, 0x65, 0x72, 0x61, - 0x6e, 0x6b, 0x5f, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x18, 0x15, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0b, - 0x72, 0x65, 0x72, 0x61, 0x6e, 0x6b, 0x53, 0x63, 0x6f, 0x72, 0x65, 0x12, 0x30, 0x0a, 0x14, 0x72, - 0x65, 0x72, 0x61, 0x6e, 0x6b, 0x5f, 0x73, 0x63, 0x6f, 0x72, 0x65, 0x5f, 0x70, 0x72, 0x65, 0x73, - 0x65, 0x6e, 0x74, 0x18, 0x16, 0x20, 0x01, 0x28, 0x08, 0x52, 0x12, 0x72, 0x65, 0x72, 0x61, 0x6e, - 0x6b, 0x53, 0x63, 0x6f, 0x72, 0x65, 0x50, 0x72, 0x65, 0x73, 0x65, 0x6e, 0x74, 0x42, 0x10, 0x0a, - 0x0e, 0x5f, 0x69, 0x73, 0x5f, 0x63, 0x6f, 0x6e, 0x73, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x74, 0x22, - 0x93, 0x07, 0x0a, 0x10, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x52, 0x65, - 0x73, 0x75, 0x6c, 0x74, 0x12, 0x49, 0x0a, 0x12, 0x6e, 0x6f, 0x6e, 0x5f, 0x72, 0x65, 0x66, 0x5f, - 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x17, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x42, 0x02, 0x18, 0x01, 0x52, 0x10, 0x6e, - 0x6f, 0x6e, 0x52, 0x65, 0x66, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, - 0x3d, 0x0a, 0x09, 0x72, 0x65, 0x66, 0x5f, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x18, 0x02, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, - 0x2e, 0x52, 0x65, 0x66, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x52, 0x65, - 0x73, 0x75, 0x6c, 0x74, 0x52, 0x08, 0x72, 0x65, 0x66, 0x50, 0x72, 0x6f, 0x70, 0x73, 0x12, 0x2b, - 0x0a, 0x11, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x5f, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x74, 0x61, 0x72, 0x67, 0x65, - 0x74, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x37, 0x0a, 0x08, 0x6d, - 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, - 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x65, 0x74, 0x61, - 0x64, 0x61, 0x74, 0x61, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, - 0x64, 0x61, 0x74, 0x61, 0x12, 0x5e, 0x0a, 0x17, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x5f, 0x61, - 0x72, 0x72, 0x61, 0x79, 0x5f, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, - 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, - 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x41, 0x72, 0x72, 0x61, 0x79, 0x50, - 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x42, 0x02, 0x18, 0x01, 0x52, 0x15, 0x6e, - 0x75, 0x6d, 0x62, 0x65, 0x72, 0x41, 0x72, 0x72, 0x61, 0x79, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, - 0x74, 0x69, 0x65, 0x73, 0x12, 0x55, 0x0a, 0x14, 0x69, 0x6e, 0x74, 0x5f, 0x61, 0x72, 0x72, 0x61, - 0x79, 0x5f, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x06, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, - 0x2e, 0x49, 0x6e, 0x74, 0x41, 0x72, 0x72, 0x61, 0x79, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, - 0x69, 0x65, 0x73, 0x42, 0x02, 0x18, 0x01, 0x52, 0x12, 0x69, 0x6e, 0x74, 0x41, 0x72, 0x72, 0x61, - 0x79, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x58, 0x0a, 0x15, 0x74, - 0x65, 0x78, 0x74, 0x5f, 0x61, 0x72, 0x72, 0x61, 0x79, 0x5f, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, - 0x74, 0x69, 0x65, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x77, 0x65, 0x61, - 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x65, 0x78, 0x74, 0x41, 0x72, 0x72, - 0x61, 0x79, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x42, 0x02, 0x18, 0x01, - 0x52, 0x13, 0x74, 0x65, 0x78, 0x74, 0x41, 0x72, 0x72, 0x61, 0x79, 0x50, 0x72, 0x6f, 0x70, 0x65, - 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x61, 0x0a, 0x18, 0x62, 0x6f, 0x6f, 0x6c, 0x65, 0x61, 0x6e, - 0x5f, 0x61, 0x72, 0x72, 0x61, 0x79, 0x5f, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, - 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, - 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x6f, 0x6f, 0x6c, 0x65, 0x61, 0x6e, 0x41, 0x72, 0x72, - 0x61, 0x79, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x42, 0x02, 0x18, 0x01, - 0x52, 0x16, 0x62, 0x6f, 0x6f, 0x6c, 0x65, 0x61, 0x6e, 0x41, 0x72, 0x72, 0x61, 0x79, 0x50, 0x72, - 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x4e, 0x0a, 0x11, 0x6f, 0x62, 0x6a, 0x65, - 0x63, 0x74, 0x5f, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x09, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, - 0x31, 0x2e, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, - 0x65, 0x73, 0x42, 0x02, 0x18, 0x01, 0x52, 0x10, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x50, 0x72, - 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x5e, 0x0a, 0x17, 0x6f, 0x62, 0x6a, 0x65, - 0x63, 0x74, 0x5f, 0x61, 0x72, 0x72, 0x61, 0x79, 0x5f, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, - 0x69, 0x65, 0x73, 0x18, 0x0a, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x77, 0x65, 0x61, 0x76, - 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x41, 0x72, - 0x72, 0x61, 0x79, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x42, 0x02, 0x18, - 0x01, 0x52, 0x15, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x41, 0x72, 0x72, 0x61, 0x79, 0x50, 0x72, - 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x3b, 0x0a, 0x0d, 0x6e, 0x6f, 0x6e, 0x5f, - 0x72, 0x65, 0x66, 0x5f, 0x70, 0x72, 0x6f, 0x70, 0x73, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x17, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, - 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x52, 0x0b, 0x6e, 0x6f, 0x6e, 0x52, 0x65, 0x66, - 0x50, 0x72, 0x6f, 0x70, 0x73, 0x12, 0x2e, 0x0a, 0x13, 0x72, 0x65, 0x66, 0x5f, 0x70, 0x72, 0x6f, - 0x70, 0x73, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x65, 0x64, 0x18, 0x0c, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x11, 0x72, 0x65, 0x66, 0x50, 0x72, 0x6f, 0x70, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x65, 0x64, 0x22, 0x71, 0x0a, 0x13, 0x52, 0x65, 0x66, 0x50, 0x72, 0x6f, 0x70, - 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x3d, 0x0a, 0x0a, - 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x1d, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, - 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, - 0x0a, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x70, - 0x72, 0x6f, 0x70, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, - 0x70, 0x72, 0x6f, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x42, 0x73, 0x0a, 0x23, 0x69, 0x6f, 0x2e, 0x77, - 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x2e, 0x67, - 0x72, 0x70, 0x63, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x76, 0x31, 0x42, - 0x16, 0x57, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x53, 0x65, - 0x61, 0x72, 0x63, 0x68, 0x47, 0x65, 0x74, 0x5a, 0x34, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, - 0x63, 0x6f, 0x6d, 0x2f, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2f, 0x77, 0x65, 0x61, - 0x76, 0x69, 0x61, 0x74, 0x65, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x67, 0x65, 0x6e, 0x65, 0x72, - 0x61, 0x74, 0x65, 0x64, 0x3b, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, -} - -var ( - file_v1_search_get_proto_rawDescOnce sync.Once - file_v1_search_get_proto_rawDescData = file_v1_search_get_proto_rawDesc -) - -func file_v1_search_get_proto_rawDescGZIP() []byte { - file_v1_search_get_proto_rawDescOnce.Do(func() { - file_v1_search_get_proto_rawDescData = protoimpl.X.CompressGZIP(file_v1_search_get_proto_rawDescData) - }) - return file_v1_search_get_proto_rawDescData -} - -var file_v1_search_get_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_v1_search_get_proto_msgTypes = make([]protoimpl.MessageInfo, 26) -var file_v1_search_get_proto_goTypes = []interface{}{ - (Hybrid_FusionType)(0), // 0: weaviate.v1.Hybrid.FusionType - (*SearchRequest)(nil), // 1: weaviate.v1.SearchRequest - (*GroupBy)(nil), // 2: weaviate.v1.GroupBy - (*SortBy)(nil), // 3: weaviate.v1.SortBy - (*GenerativeSearch)(nil), // 4: weaviate.v1.GenerativeSearch - (*MetadataRequest)(nil), // 5: weaviate.v1.MetadataRequest - (*PropertiesRequest)(nil), // 6: weaviate.v1.PropertiesRequest - (*ObjectPropertiesRequest)(nil), // 7: weaviate.v1.ObjectPropertiesRequest - (*Hybrid)(nil), // 8: weaviate.v1.Hybrid - (*NearTextSearch)(nil), // 9: weaviate.v1.NearTextSearch - (*NearImageSearch)(nil), // 10: weaviate.v1.NearImageSearch - (*NearAudioSearch)(nil), // 11: weaviate.v1.NearAudioSearch - (*NearVideoSearch)(nil), // 12: weaviate.v1.NearVideoSearch - (*BM25)(nil), // 13: weaviate.v1.BM25 - (*RefPropertiesRequest)(nil), // 14: weaviate.v1.RefPropertiesRequest - (*NearVector)(nil), // 15: weaviate.v1.NearVector - (*NearObject)(nil), // 16: weaviate.v1.NearObject - (*Rerank)(nil), // 17: weaviate.v1.Rerank - (*SearchReply)(nil), // 18: weaviate.v1.SearchReply - (*RerankReply)(nil), // 19: weaviate.v1.RerankReply - (*GenerativeReply)(nil), // 20: weaviate.v1.GenerativeReply - (*GroupByResult)(nil), // 21: weaviate.v1.GroupByResult - (*SearchResult)(nil), // 22: weaviate.v1.SearchResult - (*MetadataResult)(nil), // 23: weaviate.v1.MetadataResult - (*PropertiesResult)(nil), // 24: weaviate.v1.PropertiesResult - (*RefPropertiesResult)(nil), // 25: weaviate.v1.RefPropertiesResult - (*NearTextSearch_Move)(nil), // 26: weaviate.v1.NearTextSearch.Move - (ConsistencyLevel)(0), // 27: weaviate.v1.ConsistencyLevel - (*Filters)(nil), // 28: weaviate.v1.Filters - (*structpb.Struct)(nil), // 29: google.protobuf.Struct - (*NumberArrayProperties)(nil), // 30: weaviate.v1.NumberArrayProperties - (*IntArrayProperties)(nil), // 31: weaviate.v1.IntArrayProperties - (*TextArrayProperties)(nil), // 32: weaviate.v1.TextArrayProperties - (*BooleanArrayProperties)(nil), // 33: weaviate.v1.BooleanArrayProperties - (*ObjectProperties)(nil), // 34: weaviate.v1.ObjectProperties - (*ObjectArrayProperties)(nil), // 35: weaviate.v1.ObjectArrayProperties - (*Properties)(nil), // 36: weaviate.v1.Properties -} -var file_v1_search_get_proto_depIdxs = []int32{ - 27, // 0: weaviate.v1.SearchRequest.consistency_level:type_name -> weaviate.v1.ConsistencyLevel - 6, // 1: weaviate.v1.SearchRequest.properties:type_name -> weaviate.v1.PropertiesRequest - 5, // 2: weaviate.v1.SearchRequest.metadata:type_name -> weaviate.v1.MetadataRequest - 2, // 3: weaviate.v1.SearchRequest.group_by:type_name -> weaviate.v1.GroupBy - 3, // 4: weaviate.v1.SearchRequest.sort_by:type_name -> weaviate.v1.SortBy - 28, // 5: weaviate.v1.SearchRequest.filters:type_name -> weaviate.v1.Filters - 8, // 6: weaviate.v1.SearchRequest.hybrid_search:type_name -> weaviate.v1.Hybrid - 13, // 7: weaviate.v1.SearchRequest.bm25_search:type_name -> weaviate.v1.BM25 - 15, // 8: weaviate.v1.SearchRequest.near_vector:type_name -> weaviate.v1.NearVector - 16, // 9: weaviate.v1.SearchRequest.near_object:type_name -> weaviate.v1.NearObject - 9, // 10: weaviate.v1.SearchRequest.near_text:type_name -> weaviate.v1.NearTextSearch - 10, // 11: weaviate.v1.SearchRequest.near_image:type_name -> weaviate.v1.NearImageSearch - 11, // 12: weaviate.v1.SearchRequest.near_audio:type_name -> weaviate.v1.NearAudioSearch - 12, // 13: weaviate.v1.SearchRequest.near_video:type_name -> weaviate.v1.NearVideoSearch - 4, // 14: weaviate.v1.SearchRequest.generative:type_name -> weaviate.v1.GenerativeSearch - 17, // 15: weaviate.v1.SearchRequest.rerank:type_name -> weaviate.v1.Rerank - 14, // 16: weaviate.v1.PropertiesRequest.ref_properties:type_name -> weaviate.v1.RefPropertiesRequest - 7, // 17: weaviate.v1.PropertiesRequest.object_properties:type_name -> weaviate.v1.ObjectPropertiesRequest - 7, // 18: weaviate.v1.ObjectPropertiesRequest.object_properties:type_name -> weaviate.v1.ObjectPropertiesRequest - 0, // 19: weaviate.v1.Hybrid.fusion_type:type_name -> weaviate.v1.Hybrid.FusionType - 26, // 20: weaviate.v1.NearTextSearch.move_to:type_name -> weaviate.v1.NearTextSearch.Move - 26, // 21: weaviate.v1.NearTextSearch.move_away:type_name -> weaviate.v1.NearTextSearch.Move - 6, // 22: weaviate.v1.RefPropertiesRequest.properties:type_name -> weaviate.v1.PropertiesRequest - 5, // 23: weaviate.v1.RefPropertiesRequest.metadata:type_name -> weaviate.v1.MetadataRequest - 22, // 24: weaviate.v1.SearchReply.results:type_name -> weaviate.v1.SearchResult - 21, // 25: weaviate.v1.SearchReply.group_by_results:type_name -> weaviate.v1.GroupByResult - 22, // 26: weaviate.v1.GroupByResult.objects:type_name -> weaviate.v1.SearchResult - 19, // 27: weaviate.v1.GroupByResult.rerank:type_name -> weaviate.v1.RerankReply - 20, // 28: weaviate.v1.GroupByResult.generative:type_name -> weaviate.v1.GenerativeReply - 24, // 29: weaviate.v1.SearchResult.properties:type_name -> weaviate.v1.PropertiesResult - 23, // 30: weaviate.v1.SearchResult.metadata:type_name -> weaviate.v1.MetadataResult - 29, // 31: weaviate.v1.PropertiesResult.non_ref_properties:type_name -> google.protobuf.Struct - 25, // 32: weaviate.v1.PropertiesResult.ref_props:type_name -> weaviate.v1.RefPropertiesResult - 23, // 33: weaviate.v1.PropertiesResult.metadata:type_name -> weaviate.v1.MetadataResult - 30, // 34: weaviate.v1.PropertiesResult.number_array_properties:type_name -> weaviate.v1.NumberArrayProperties - 31, // 35: weaviate.v1.PropertiesResult.int_array_properties:type_name -> weaviate.v1.IntArrayProperties - 32, // 36: weaviate.v1.PropertiesResult.text_array_properties:type_name -> weaviate.v1.TextArrayProperties - 33, // 37: weaviate.v1.PropertiesResult.boolean_array_properties:type_name -> weaviate.v1.BooleanArrayProperties - 34, // 38: weaviate.v1.PropertiesResult.object_properties:type_name -> weaviate.v1.ObjectProperties - 35, // 39: weaviate.v1.PropertiesResult.object_array_properties:type_name -> weaviate.v1.ObjectArrayProperties - 36, // 40: weaviate.v1.PropertiesResult.non_ref_props:type_name -> weaviate.v1.Properties - 24, // 41: weaviate.v1.RefPropertiesResult.properties:type_name -> weaviate.v1.PropertiesResult - 42, // [42:42] is the sub-list for method output_type - 42, // [42:42] is the sub-list for method input_type - 42, // [42:42] is the sub-list for extension type_name - 42, // [42:42] is the sub-list for extension extendee - 0, // [0:42] is the sub-list for field type_name -} - -func init() { file_v1_search_get_proto_init() } -func file_v1_search_get_proto_init() { - if File_v1_search_get_proto != nil { - return - } - file_v1_base_proto_init() - file_v1_properties_proto_init() - if !protoimpl.UnsafeEnabled { - file_v1_search_get_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SearchRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_search_get_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GroupBy); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_search_get_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SortBy); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_search_get_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GenerativeSearch); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_search_get_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*MetadataRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_search_get_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PropertiesRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_search_get_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ObjectPropertiesRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_search_get_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Hybrid); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_search_get_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NearTextSearch); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_search_get_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NearImageSearch); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_search_get_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NearAudioSearch); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_search_get_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NearVideoSearch); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_search_get_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*BM25); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_search_get_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RefPropertiesRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_search_get_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NearVector); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_search_get_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NearObject); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_search_get_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Rerank); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_search_get_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SearchReply); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_search_get_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RerankReply); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_search_get_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GenerativeReply); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_search_get_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GroupByResult); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_search_get_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SearchResult); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_search_get_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*MetadataResult); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_search_get_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PropertiesResult); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_search_get_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RefPropertiesResult); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_v1_search_get_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NearTextSearch_Move); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } - file_v1_search_get_proto_msgTypes[0].OneofWrappers = []interface{}{} - file_v1_search_get_proto_msgTypes[8].OneofWrappers = []interface{}{} - file_v1_search_get_proto_msgTypes[9].OneofWrappers = []interface{}{} - file_v1_search_get_proto_msgTypes[10].OneofWrappers = []interface{}{} - file_v1_search_get_proto_msgTypes[11].OneofWrappers = []interface{}{} - file_v1_search_get_proto_msgTypes[14].OneofWrappers = []interface{}{} - file_v1_search_get_proto_msgTypes[15].OneofWrappers = []interface{}{} - file_v1_search_get_proto_msgTypes[16].OneofWrappers = []interface{}{} - file_v1_search_get_proto_msgTypes[17].OneofWrappers = []interface{}{} - file_v1_search_get_proto_msgTypes[20].OneofWrappers = []interface{}{} - file_v1_search_get_proto_msgTypes[22].OneofWrappers = []interface{}{} - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_v1_search_get_proto_rawDesc, - NumEnums: 1, - NumMessages: 26, - NumExtensions: 0, - NumServices: 0, - }, - GoTypes: file_v1_search_get_proto_goTypes, - DependencyIndexes: file_v1_search_get_proto_depIdxs, - EnumInfos: file_v1_search_get_proto_enumTypes, - MessageInfos: file_v1_search_get_proto_msgTypes, - }.Build() - File_v1_search_get_proto = out.File - file_v1_search_get_proto_rawDesc = nil - file_v1_search_get_proto_goTypes = nil - file_v1_search_get_proto_depIdxs = nil -} diff --git a/grpc/generated/protocol/v1/weaviate.pb.go b/grpc/generated/protocol/v1/weaviate.pb.go deleted file mode 100644 index a0f8e032e81915ec5cbf7d4f007cb0880a40cd6c..0000000000000000000000000000000000000000 --- a/grpc/generated/protocol/v1/weaviate.pb.go +++ /dev/null @@ -1,100 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. - -package protocol - -import ( - reflect "reflect" - - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -var File_v1_weaviate_proto protoreflect.FileDescriptor - -var file_v1_weaviate_proto_rawDesc = []byte{ - 0x0a, 0x11, 0x76, 0x31, 0x2f, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x12, 0x0b, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, - 0x1a, 0x0e, 0x76, 0x31, 0x2f, 0x62, 0x61, 0x74, 0x63, 0x68, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x1a, 0x15, 0x76, 0x31, 0x2f, 0x62, 0x61, 0x74, 0x63, 0x68, 0x5f, 0x64, 0x65, 0x6c, 0x65, 0x74, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x13, 0x76, 0x31, 0x2f, 0x73, 0x65, 0x61, 0x72, - 0x63, 0x68, 0x5f, 0x67, 0x65, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x32, 0xf1, 0x01, 0x0a, - 0x08, 0x57, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x12, 0x40, 0x0a, 0x06, 0x53, 0x65, 0x61, - 0x72, 0x63, 0x68, 0x12, 0x1a, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, - 0x31, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x18, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, - 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x52, 0x0a, 0x0c, 0x42, - 0x61, 0x74, 0x63, 0x68, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x20, 0x2e, 0x77, 0x65, - 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x61, 0x74, 0x63, 0x68, 0x4f, - 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, - 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x61, 0x74, 0x63, - 0x68, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, - 0x4f, 0x0a, 0x0b, 0x42, 0x61, 0x74, 0x63, 0x68, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x1f, - 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x61, 0x74, - 0x63, 0x68, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x1d, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x61, - 0x74, 0x63, 0x68, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, - 0x42, 0x6a, 0x0a, 0x23, 0x69, 0x6f, 0x2e, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2e, - 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x76, 0x31, 0x42, 0x0d, 0x57, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, - 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x34, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, - 0x6f, 0x6d, 0x2f, 0x77, 0x65, 0x61, 0x76, 0x69, 0x61, 0x74, 0x65, 0x2f, 0x77, 0x65, 0x61, 0x76, - 0x69, 0x61, 0x74, 0x65, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, - 0x74, 0x65, 0x64, 0x3b, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x62, 0x06, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x33, -} - -var file_v1_weaviate_proto_goTypes = []interface{}{ - (*SearchRequest)(nil), // 0: weaviate.v1.SearchRequest - (*BatchObjectsRequest)(nil), // 1: weaviate.v1.BatchObjectsRequest - (*BatchDeleteRequest)(nil), // 2: weaviate.v1.BatchDeleteRequest - (*SearchReply)(nil), // 3: weaviate.v1.SearchReply - (*BatchObjectsReply)(nil), // 4: weaviate.v1.BatchObjectsReply - (*BatchDeleteReply)(nil), // 5: weaviate.v1.BatchDeleteReply -} -var file_v1_weaviate_proto_depIdxs = []int32{ - 0, // 0: weaviate.v1.Weaviate.Search:input_type -> weaviate.v1.SearchRequest - 1, // 1: weaviate.v1.Weaviate.BatchObjects:input_type -> weaviate.v1.BatchObjectsRequest - 2, // 2: weaviate.v1.Weaviate.BatchDelete:input_type -> weaviate.v1.BatchDeleteRequest - 3, // 3: weaviate.v1.Weaviate.Search:output_type -> weaviate.v1.SearchReply - 4, // 4: weaviate.v1.Weaviate.BatchObjects:output_type -> weaviate.v1.BatchObjectsReply - 5, // 5: weaviate.v1.Weaviate.BatchDelete:output_type -> weaviate.v1.BatchDeleteReply - 3, // [3:6] is the sub-list for method output_type - 0, // [0:3] is the sub-list for method input_type - 0, // [0:0] is the sub-list for extension type_name - 0, // [0:0] is the sub-list for extension extendee - 0, // [0:0] is the sub-list for field type_name -} - -func init() { file_v1_weaviate_proto_init() } -func file_v1_weaviate_proto_init() { - if File_v1_weaviate_proto != nil { - return - } - file_v1_batch_proto_init() - file_v1_batch_delete_proto_init() - file_v1_search_get_proto_init() - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_v1_weaviate_proto_rawDesc, - NumEnums: 0, - NumMessages: 0, - NumExtensions: 0, - NumServices: 1, - }, - GoTypes: file_v1_weaviate_proto_goTypes, - DependencyIndexes: file_v1_weaviate_proto_depIdxs, - }.Build() - File_v1_weaviate_proto = out.File - file_v1_weaviate_proto_rawDesc = nil - file_v1_weaviate_proto_goTypes = nil - file_v1_weaviate_proto_depIdxs = nil -} diff --git a/grpc/generated/protocol/v1/weaviate_grpc.pb.go b/grpc/generated/protocol/v1/weaviate_grpc.pb.go deleted file mode 100644 index 7c15d4c3b66d33967fa6d108fdd2d91a045694df..0000000000000000000000000000000000000000 --- a/grpc/generated/protocol/v1/weaviate_grpc.pb.go +++ /dev/null @@ -1,174 +0,0 @@ -// Code generated by protoc-gen-go-grpc. DO NOT EDIT. - -package protocol - -import ( - context "context" - - grpc "google.golang.org/grpc" - codes "google.golang.org/grpc/codes" - status "google.golang.org/grpc/status" -) - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.32.0 or later. -const _ = grpc.SupportPackageIsVersion7 - -// WeaviateClient is the client API for Weaviate service. -// -// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. -type WeaviateClient interface { - Search(ctx context.Context, in *SearchRequest, opts ...grpc.CallOption) (*SearchReply, error) - BatchObjects(ctx context.Context, in *BatchObjectsRequest, opts ...grpc.CallOption) (*BatchObjectsReply, error) - BatchDelete(ctx context.Context, in *BatchDeleteRequest, opts ...grpc.CallOption) (*BatchDeleteReply, error) -} - -type weaviateClient struct { - cc grpc.ClientConnInterface -} - -func NewWeaviateClient(cc grpc.ClientConnInterface) WeaviateClient { - return &weaviateClient{cc} -} - -func (c *weaviateClient) Search(ctx context.Context, in *SearchRequest, opts ...grpc.CallOption) (*SearchReply, error) { - out := new(SearchReply) - err := c.cc.Invoke(ctx, "/weaviate.v1.Weaviate/Search", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *weaviateClient) BatchObjects(ctx context.Context, in *BatchObjectsRequest, opts ...grpc.CallOption) (*BatchObjectsReply, error) { - out := new(BatchObjectsReply) - err := c.cc.Invoke(ctx, "/weaviate.v1.Weaviate/BatchObjects", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *weaviateClient) BatchDelete(ctx context.Context, in *BatchDeleteRequest, opts ...grpc.CallOption) (*BatchDeleteReply, error) { - out := new(BatchDeleteReply) - err := c.cc.Invoke(ctx, "/weaviate.v1.Weaviate/BatchDelete", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -// WeaviateServer is the server API for Weaviate service. -// All implementations must embed UnimplementedWeaviateServer -// for forward compatibility -type WeaviateServer interface { - Search(context.Context, *SearchRequest) (*SearchReply, error) - BatchObjects(context.Context, *BatchObjectsRequest) (*BatchObjectsReply, error) - BatchDelete(context.Context, *BatchDeleteRequest) (*BatchDeleteReply, error) - mustEmbedUnimplementedWeaviateServer() -} - -// UnimplementedWeaviateServer must be embedded to have forward compatible implementations. -type UnimplementedWeaviateServer struct { -} - -func (UnimplementedWeaviateServer) Search(context.Context, *SearchRequest) (*SearchReply, error) { - return nil, status.Errorf(codes.Unimplemented, "method Search not implemented") -} -func (UnimplementedWeaviateServer) BatchObjects(context.Context, *BatchObjectsRequest) (*BatchObjectsReply, error) { - return nil, status.Errorf(codes.Unimplemented, "method BatchObjects not implemented") -} -func (UnimplementedWeaviateServer) BatchDelete(context.Context, *BatchDeleteRequest) (*BatchDeleteReply, error) { - return nil, status.Errorf(codes.Unimplemented, "method BatchDelete not implemented") -} -func (UnimplementedWeaviateServer) mustEmbedUnimplementedWeaviateServer() {} - -// UnsafeWeaviateServer may be embedded to opt out of forward compatibility for this service. -// Use of this interface is not recommended, as added methods to WeaviateServer will -// result in compilation errors. -type UnsafeWeaviateServer interface { - mustEmbedUnimplementedWeaviateServer() -} - -func RegisterWeaviateServer(s grpc.ServiceRegistrar, srv WeaviateServer) { - s.RegisterService(&Weaviate_ServiceDesc, srv) -} - -func _Weaviate_Search_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(SearchRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(WeaviateServer).Search(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/weaviate.v1.Weaviate/Search", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(WeaviateServer).Search(ctx, req.(*SearchRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _Weaviate_BatchObjects_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(BatchObjectsRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(WeaviateServer).BatchObjects(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/weaviate.v1.Weaviate/BatchObjects", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(WeaviateServer).BatchObjects(ctx, req.(*BatchObjectsRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _Weaviate_BatchDelete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(BatchDeleteRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(WeaviateServer).BatchDelete(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/weaviate.v1.Weaviate/BatchDelete", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(WeaviateServer).BatchDelete(ctx, req.(*BatchDeleteRequest)) - } - return interceptor(ctx, in, info, handler) -} - -// Weaviate_ServiceDesc is the grpc.ServiceDesc for Weaviate service. -// It's only intended for direct use with grpc.RegisterService, -// and not to be introspected or modified (even as a copy) -var Weaviate_ServiceDesc = grpc.ServiceDesc{ - ServiceName: "weaviate.v1.Weaviate", - HandlerType: (*WeaviateServer)(nil), - Methods: []grpc.MethodDesc{ - { - MethodName: "Search", - Handler: _Weaviate_Search_Handler, - }, - { - MethodName: "BatchObjects", - Handler: _Weaviate_BatchObjects_Handler, - }, - { - MethodName: "BatchDelete", - Handler: _Weaviate_BatchDelete_Handler, - }, - }, - Streams: []grpc.StreamDesc{}, - Metadata: "v1/weaviate.proto", -} diff --git a/grpc/proto/v0/batch.proto b/grpc/proto/v0/batch.proto deleted file mode 100644 index 9a3a5908f6e1ea81d8aa1bd890aab137b379d90a..0000000000000000000000000000000000000000 --- a/grpc/proto/v0/batch.proto +++ /dev/null @@ -1,12 +0,0 @@ -syntax = "proto3"; - -package weaviategrpc; - -option go_package = "github.com/weaviate/weaviate/grpc/generated;protocol"; -option java_package = "io.weaviate.client.grpc.protocol.v0"; -option java_outer_classname = "WeaviateProtoBatch"; - -message BatchObjectsRequest { -} -message BatchObjectsReply { -} diff --git a/grpc/proto/v0/search_get.proto b/grpc/proto/v0/search_get.proto deleted file mode 100644 index 513b4e16b70b2dd868c2838c2e284a51f3e8a278..0000000000000000000000000000000000000000 --- a/grpc/proto/v0/search_get.proto +++ /dev/null @@ -1,12 +0,0 @@ -syntax = "proto3"; - -package weaviategrpc; - -option go_package = "github.com/weaviate/weaviate/grpc/generated;protocol"; -option java_package = "io.weaviate.client.grpc.protocol.v0"; -option java_outer_classname = "WeaviateProtoSearchGet"; - -message SearchRequest { -} -message SearchReply { -} diff --git a/grpc/proto/v0/weaviate.proto b/grpc/proto/v0/weaviate.proto deleted file mode 100644 index 4c3f6024168e9b365d52917e6c8b8a73655fc8c0..0000000000000000000000000000000000000000 --- a/grpc/proto/v0/weaviate.proto +++ /dev/null @@ -1,15 +0,0 @@ -syntax = "proto3"; - -package weaviategrpc; - -import "v0/batch.proto"; -import "v0/search_get.proto"; - -option go_package = "github.com/weaviate/weaviate/grpc/generated;protocol"; -option java_package = "io.weaviate.client.grpc.protocol.v0"; -option java_outer_classname = "WeaviateProto"; - -service Weaviate { - rpc Search(SearchRequest) returns (SearchReply) {}; - rpc BatchObjects(BatchObjectsRequest) returns (BatchObjectsReply) {}; -} diff --git a/grpc/proto/v1/base.proto b/grpc/proto/v1/base.proto deleted file mode 100644 index 53b1bb6385db3cf1d1e6cded151dd42e25c7a35a..0000000000000000000000000000000000000000 --- a/grpc/proto/v1/base.proto +++ /dev/null @@ -1,134 +0,0 @@ -syntax = "proto3"; - -package weaviate.v1; -import "google/protobuf/struct.proto"; - -option go_package = "github.com/weaviate/weaviate/grpc/generated;protocol"; -option java_package = "io.weaviate.client.grpc.protocol.v1"; -option java_outer_classname = "WeaviateProtoBase"; - -enum ConsistencyLevel { - CONSISTENCY_LEVEL_UNSPECIFIED = 0; - CONSISTENCY_LEVEL_ONE = 1; - CONSISTENCY_LEVEL_QUORUM = 2; - CONSISTENCY_LEVEL_ALL = 3; -} - -message NumberArrayProperties { - repeated double values = 1 [deprecated = true]; // will be removed in the future, use vector_bytes - string prop_name = 2; - bytes values_bytes = 3; -} - -message IntArrayProperties { - repeated int64 values = 1; - string prop_name = 2; -} - -message TextArrayProperties { - repeated string values = 1; - string prop_name = 2; -} - -message BooleanArrayProperties { - repeated bool values = 1; - string prop_name = 2; -} - -message ObjectPropertiesValue { - google.protobuf.Struct non_ref_properties = 1; - repeated NumberArrayProperties number_array_properties = 2; - repeated IntArrayProperties int_array_properties = 3; - repeated TextArrayProperties text_array_properties = 4; - repeated BooleanArrayProperties boolean_array_properties = 5; - repeated ObjectProperties object_properties = 6; - repeated ObjectArrayProperties object_array_properties = 7; -} - -message ObjectArrayProperties { - repeated ObjectPropertiesValue values = 1; - string prop_name = 2; -} - -message ObjectProperties { - ObjectPropertiesValue value = 1; - string prop_name = 2; -} - - -message TextArray { - repeated string values = 1; -} - -message IntArray { - repeated int64 values = 1; -} - -message NumberArray { - repeated double values = 1; -} - -message BooleanArray { - repeated bool values = 1; -} - -message Filters { - enum Operator { - OPERATOR_UNSPECIFIED = 0; - OPERATOR_EQUAL = 1; - OPERATOR_NOT_EQUAL = 2; - OPERATOR_GREATER_THAN = 3; - OPERATOR_GREATER_THAN_EQUAL = 4; - OPERATOR_LESS_THAN = 5; - OPERATOR_LESS_THAN_EQUAL = 6; - OPERATOR_AND = 7; - OPERATOR_OR = 8; - OPERATOR_WITHIN_GEO_RANGE = 9; - OPERATOR_LIKE = 10; - OPERATOR_IS_NULL = 11; - OPERATOR_CONTAINS_ANY = 12; - OPERATOR_CONTAINS_ALL = 13; - } - - Operator operator = 1; - // protolint:disable:next REPEATED_FIELD_NAMES_PLURALIZED - repeated string on = 2 [deprecated = true]; // will be removed in the future, use path - repeated Filters filters = 3; - oneof test_value { - string value_text = 4; - int64 value_int = 5; - bool value_boolean = 6; - double value_number = 7; - TextArray value_text_array = 9; - IntArray value_int_array = 10; - BooleanArray value_boolean_array = 11; - NumberArray value_number_array = 12; - GeoCoordinatesFilter value_geo = 13; - }; - FilterTarget target = 20; // leave space for more filter values -} - -message FilterReferenceSingleTarget { - string on = 1; - FilterTarget target = 2; -} - -message FilterReferenceMultiTarget { - string on = 1; - FilterTarget target = 2; - string target_collection = 3; -} - -message FilterTarget { - oneof target{ - string property = 1; - FilterReferenceSingleTarget single_target = 2; - FilterReferenceMultiTarget multi_target = 3; - }; -} - -message GeoCoordinatesFilter { - float latitude = 1; - float longitude = 2; - float distance = 3; -} diff --git a/grpc/proto/v1/batch.proto b/grpc/proto/v1/batch.proto deleted file mode 100644 index 4fd7f391fbd0e2fd25355a9fa0220211307f5c8f..0000000000000000000000000000000000000000 --- a/grpc/proto/v1/batch.proto +++ /dev/null @@ -1,58 +0,0 @@ -syntax = "proto3"; - -package weaviate.v1; - -import "google/protobuf/struct.proto"; -import "v1/base.proto"; - -option go_package = "github.com/weaviate/weaviate/grpc/generated;protocol"; -option java_package = "io.weaviate.client.grpc.protocol.v1"; -option java_outer_classname = "WeaviateProtoBatch"; - -message BatchObjectsRequest { - repeated BatchObject objects = 1; - optional ConsistencyLevel consistency_level = 2; -} - -message BatchObject { - message Properties { - google.protobuf.Struct non_ref_properties = 1; - repeated SingleTargetRefProps single_target_ref_props = 2; - repeated MultiTargetRefProps multi_target_ref_props = 3; - repeated NumberArrayProperties number_array_properties = 4; - repeated IntArrayProperties int_array_properties = 5; - repeated TextArrayProperties text_array_properties = 6; - repeated BooleanArrayProperties boolean_array_properties = 7; - repeated ObjectProperties object_properties = 8; - repeated ObjectArrayProperties object_array_properties = 9; - } - - message SingleTargetRefProps { - repeated string uuids = 1; - string prop_name = 2; - } - - message MultiTargetRefProps { - repeated string uuids = 1; - string prop_name = 2; - string target_collection = 3; - } - - string uuid = 1; - // protolint:disable:next REPEATED_FIELD_NAMES_PLURALIZED - repeated float vector = 2 [deprecated = true]; // deprecated, will be removed - Properties properties = 3; - string collection = 4; - string tenant = 5; - bytes vector_bytes = 6; -} - -message BatchObjectsReply { - message BatchError { - int32 index = 1; - string error = 2; - } - - float took = 1; - repeated BatchError errors = 2; -} diff --git a/grpc/proto/v1/batch_delete.proto b/grpc/proto/v1/batch_delete.proto deleted file mode 100644 index 1f1adc63c5645df2000ba3c9a1d7902ff7dd70e8..0000000000000000000000000000000000000000 --- a/grpc/proto/v1/batch_delete.proto +++ /dev/null @@ -1,32 +0,0 @@ -syntax = "proto3"; - -package weaviate.v1; - -import "v1/base.proto"; - -option go_package = "github.com/weaviate/weaviate/grpc/generated;protocol"; -option java_package = "io.weaviate.client.grpc.protocol.v1"; -option java_outer_classname = "WeaviateProtoBatch"; - -message BatchDeleteRequest { - string collection = 1; - Filters filters = 2; - bool verbose = 3; - bool dry_run = 4; - optional ConsistencyLevel consistency_level = 5; - optional string tenant = 6; -} - -message BatchDeleteReply { - float took = 1; - int64 failed = 2; - int64 matches = 3; - int64 successful = 4; - repeated BatchDeleteObject objects = 5; -} - -message BatchDeleteObject { - bytes uuid = 1; - bool successful = 2; - optional string error = 3; // empty string means no error -} diff --git a/grpc/proto/v1/properties.proto b/grpc/proto/v1/properties.proto deleted file mode 100644 index 5939acbab77c51b0eebc10e32cc86cd44f6c01b4..0000000000000000000000000000000000000000 --- a/grpc/proto/v1/properties.proto +++ /dev/null @@ -1,46 +0,0 @@ -syntax = "proto3"; - -package weaviate.v1; - -option go_package = "github.com/weaviate/weaviate/grpc/generated;protocol"; -option java_package = "io.weaviate.client.grpc.protocol.v1"; -option java_outer_classname = "WeaviateProtoProperties"; - -message Properties { - map fields = 1; -} - -message Value { - oneof kind { - double number_value = 1; - string string_value = 2; - bool bool_value = 3; - Properties object_value = 4; - ListValue list_value = 5; - string date_value = 6; - string uuid_value = 7; - int64 int_value = 8; - GeoCoordinate geo_value = 9; - string blob_value = 10; - PhoneNumber phone_value = 11; - } -} - -message ListValue { - repeated Value values = 1; -} - -message GeoCoordinate { - float longitude = 1; - float latitude = 2; -} - -message PhoneNumber { - uint64 country_code = 1; - string default_country = 2; - string input = 3; - string international_formatted = 4; - uint64 national = 5; - string national_formatted = 6; - bool valid = 7; -} \ No newline at end of file diff --git a/grpc/proto/v1/search_get.proto b/grpc/proto/v1/search_get.proto deleted file mode 100644 index 59ee61a6e5224eaba63ad01a0c9920c88a7f5963..0000000000000000000000000000000000000000 --- a/grpc/proto/v1/search_get.proto +++ /dev/null @@ -1,253 +0,0 @@ -syntax = "proto3"; - -package weaviate.v1; - -import "google/protobuf/struct.proto"; -import "v1/base.proto"; -import "v1/properties.proto"; - -option go_package = "github.com/weaviate/weaviate/grpc/generated;protocol"; -option java_package = "io.weaviate.client.grpc.protocol.v1"; -option java_outer_classname = "WeaviateProtoSearchGet"; - -message SearchRequest { - //required - string collection = 1; - - // parameters - string tenant = 10; - optional ConsistencyLevel consistency_level = 11; - - // what is returned - optional PropertiesRequest properties = 20; - optional MetadataRequest metadata = 21; - optional GroupBy group_by = 22; - - // affects order and length of results. 0/empty (default value) means disabled - uint32 limit = 30; - uint32 offset = 31; - uint32 autocut = 32; - string after = 33; - // protolint:disable:next REPEATED_FIELD_NAMES_PLURALIZED - repeated SortBy sort_by = 34; - - // matches/searches for objects - optional Filters filters = 40; - optional Hybrid hybrid_search = 41; - optional BM25 bm25_search = 42; - optional NearVector near_vector = 43; - optional NearObject near_object = 44; - optional NearTextSearch near_text = 45; - optional NearImageSearch near_image = 46; - optional NearAudioSearch near_audio = 47; - optional NearVideoSearch near_video = 48; - - optional GenerativeSearch generative = 60; - optional Rerank rerank = 61; - - bool uses_123_api = 100 [deprecated = true]; -} - -message GroupBy { - // currently only supports one entry (eg just properties, no refs). But might - // be extended in the future. - // protolint:disable:next REPEATED_FIELD_NAMES_PLURALIZED - repeated string path = 1; - int32 number_of_groups = 2; - int32 objects_per_group = 3; -} - -message SortBy { - bool ascending = 1; - // currently only supports one entry (eg just properties, no refs). But the - // weaviate datastructure already has paths in it and this makes it easily - // extendable in the future - // protolint:disable:next REPEATED_FIELD_NAMES_PLURALIZED - repeated string path = 2; -} - -message GenerativeSearch { - string single_response_prompt = 1; - string grouped_response_task = 2; - repeated string grouped_properties = 3; -} - -message MetadataRequest { - bool uuid = 1; - bool vector = 2; - bool creation_time_unix = 3; - bool last_update_time_unix = 4; - bool distance = 5; - bool certainty = 6; - bool score = 7; - bool explain_score = 8; - bool is_consistent = 9; -} - -message PropertiesRequest { - repeated string non_ref_properties = 1; - repeated RefPropertiesRequest ref_properties = 2; - repeated ObjectPropertiesRequest object_properties = 3; - bool return_all_nonref_properties = 11; -} - -message ObjectPropertiesRequest { - string prop_name = 1; - repeated string primitive_properties = 2; - repeated ObjectPropertiesRequest object_properties = 3; -} - -message Hybrid { - string query = 1; - repeated string properties = 2; - // protolint:disable:next REPEATED_FIELD_NAMES_PLURALIZED - repeated float vector = 3 [deprecated = true]; // will be removed in the future, use vector_bytes - float alpha = 4; - enum FusionType { - FUSION_TYPE_UNSPECIFIED = 0; - FUSION_TYPE_RANKED = 1; - FUSION_TYPE_RELATIVE_SCORE = 2; - } - FusionType fusion_type = 5; - bytes vector_bytes = 6; -} - -message NearTextSearch { - message Move { - float force = 1; - repeated string concepts = 2; - repeated string uuids = 3; - } - - // protolint:disable:next REPEATED_FIELD_NAMES_PLURALIZED - repeated string query = 1; - optional double certainty = 2; - optional double distance = 3; - optional Move move_to = 4; - optional Move move_away = 5; -}; - -message NearImageSearch { - string image = 1; - optional double certainty = 2; - optional double distance = 3; -}; - -message NearAudioSearch { - string audio = 1; - optional double certainty = 2; - optional double distance = 3; -}; - -message NearVideoSearch { - string video = 1; - optional double certainty = 2; - optional double distance = 3; -}; - -message BM25 { - string query = 1; - repeated string properties = 2; -} - -message RefPropertiesRequest { - string reference_property = 1; - PropertiesRequest properties = 2; - MetadataRequest metadata = 3; - string target_collection = 4; -} - -message NearVector { - // protolint:disable:next REPEATED_FIELD_NAMES_PLURALIZED - repeated float vector = 1 [deprecated = true]; // will be removed in the future, use vector_bytes - optional double certainty = 2; - optional double distance = 3; - bytes vector_bytes = 4; -} - -message NearObject { - string id = 1; - optional double certainty = 2; - optional double distance = 3; -} - -message Rerank { - string property = 1; - optional string query = 2; -} - -message SearchReply { - float took = 1; - repeated SearchResult results = 2; - optional string generative_grouped_result = 3; - repeated GroupByResult group_by_results = 4; -} - -message RerankReply { - double score = 1; -} - -message GenerativeReply { - string result = 1; -} - -message GroupByResult { - string name = 1; - float min_distance = 2; - float max_distance = 3; - int64 number_of_objects = 4; - repeated SearchResult objects = 5; - optional RerankReply rerank = 6; - optional GenerativeReply generative = 7; -} - -message SearchResult { - PropertiesResult properties = 1; - MetadataResult metadata = 2; -} - -message MetadataResult { - string id = 1; - // protolint:disable:next REPEATED_FIELD_NAMES_PLURALIZED - repeated float vector = 2 [deprecated = true]; - int64 creation_time_unix = 3; - bool creation_time_unix_present = 4; - int64 last_update_time_unix = 5; - bool last_update_time_unix_present = 6; - float distance = 7; - bool distance_present = 8; - float certainty = 9; - bool certainty_present = 10; - float score = 11; - bool score_present = 12; - string explain_score = 13; - bool explain_score_present = 14; - optional bool is_consistent = 15; - string generative = 16; - bool generative_present = 17; - bool is_consistent_present = 18; - bytes vector_bytes = 19; - bytes id_as_bytes = 20; - double rerank_score = 21; - bool rerank_score_present = 22; -} - -message PropertiesResult { - google.protobuf.Struct non_ref_properties = 1 [deprecated = true]; - repeated RefPropertiesResult ref_props = 2; - string target_collection = 3; - MetadataResult metadata = 4; - repeated NumberArrayProperties number_array_properties = 5 [deprecated = true]; - repeated IntArrayProperties int_array_properties = 6 [deprecated = true]; - repeated TextArrayProperties text_array_properties = 7 [deprecated = true]; - repeated BooleanArrayProperties boolean_array_properties = 8 [deprecated = true]; - repeated ObjectProperties object_properties = 9 [deprecated = true]; - repeated ObjectArrayProperties object_array_properties = 10 [deprecated = true]; - Properties non_ref_props = 11; - bool ref_props_requested = 12; -} - -message RefPropertiesResult { - repeated PropertiesResult properties = 1; - string prop_name = 2; -} diff --git a/grpc/proto/v1/weaviate.proto b/grpc/proto/v1/weaviate.proto deleted file mode 100644 index b719e8893f090c562981c51705c5f529d56c28b2..0000000000000000000000000000000000000000 --- a/grpc/proto/v1/weaviate.proto +++ /dev/null @@ -1,17 +0,0 @@ -syntax = "proto3"; - -package weaviate.v1; - -import "v1/batch.proto"; -import "v1/batch_delete.proto"; -import "v1/search_get.proto"; - -option go_package = "github.com/weaviate/weaviate/grpc/generated;protocol"; -option java_package = "io.weaviate.client.grpc.protocol.v1"; -option java_outer_classname = "WeaviateProto"; - -service Weaviate { - rpc Search(SearchRequest) returns (SearchReply) {}; - rpc BatchObjects(BatchObjectsRequest) returns (BatchObjectsReply) {}; - rpc BatchDelete(BatchDeleteRequest) returns (BatchDeleteReply) {}; -} diff --git a/inputDocs/ArtificialIntelligence.html b/inputDocs/ArtificialIntelligence.html deleted file mode 100644 index e7d8d6d50711be871df0915bc10bb08ab4d3b937..0000000000000000000000000000000000000000 --- a/inputDocs/ArtificialIntelligence.html +++ /dev/null @@ -1,8079 +0,0 @@ - - - - -Artificial intelligence - Wikipedia - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -Jump to content -
-
-
- - - - -
-
- - - - - -
-
-
-
-
-
-
-
-
-
-
- -
-
-
- -
-
-
-
-
- -

Artificial intelligence

- -
- - -
- -
- - -
Edit links
-
- -
-
-
-
-
-
- -
-
- - - -
-
-
-
-
- - -
-
-
-
-
-
Page semi-protected
-
- -
From Wikipedia, the free encyclopedia
-
-
- - -
-
"AI" redirects here. For other uses, see AI (disambiguation), Artificial intelligence (disambiguation), and Intelligent agent.
-

- -

- - - - - - -
Part of a series on
Artificial intelligence
-
Major goals
-
-
Approaches
-
-
Applications
-
-
Philosophy
-
-
History
-
-
Glossary
-
-
Video (subtitles available) summarizing the topic of artificial intelligence and presenting some of the ethical implications.
-

Artificial intelligence (AI) is the intelligence of machines or software, as opposed to the intelligence of humans or other animals. It is a field of study in computer science that develops and studies intelligent machines. Such machines may be called AIs. -

AI technology is widely used throughout industry, government, and science. Some high-profile applications are: advanced web search engines (e.g., Google Search), recommendation systems (used by YouTube, Amazon, and Netflix), understanding human speech (such as Google Assistant, Siri, and Alexa), self-driving cars (e.g., Waymo), generative and creative tools (ChatGPT and AI art), and superhuman play and analysis in strategy games (such as chess and Go).[1] -

Alan Turing was the first person to conduct substantial research in the field that he called Machine Intelligence.[2] Artificial intelligence was founded as an academic discipline in 1956.[3] The field went through multiple cycles of optimism[4][5] followed by disappointment and loss of funding.[6][7] Funding and interest vastly increased after 2012 when deep learning surpassed all previous AI techniques,[8] and after 2017 with the transformer architecture.[9] This led to the AI spring of the early 2020s, with companies, universities, and laboratories overwhelmingly based in the United States pioneering significant advances in artificial intelligence.[10] -

The various sub-fields of AI research are centered around particular goals and the use of particular tools. The traditional goals of AI research include reasoning, knowledge representation, planning, learning, natural language processing, perception, and support for robotics.[a] General intelligence (the ability to complete any task performable by a human) is among the field's long-term goals.[11] -

To solve these problems, AI researchers have adapted and integrated a wide range of problem-solving techniques, including search and mathematical optimization, formal logic, artificial neural networks, and methods based on statistics, operations research, and economics.[b] AI also draws upon psychology, linguistics, philosophy, neuroscience and other fields.[12] -

- -

Goals

-

The general problem of simulating (or creating) intelligence has been broken into sub-problems. These consist of particular traits or capabilities that researchers expect an intelligent system to display. The traits described below have received the most attention and cover the scope of AI research.[a] -

-

Reasoning, problem-solving

-

Early researchers developed algorithms that imitated step-by-step reasoning that humans use when they solve puzzles or make logical deductions.[13] By the late 1980s and 1990s, methods were developed for dealing with uncertain or incomplete information, employing concepts from probability and economics.[14] -

Many of these algorithms are insufficient for solving large reasoning problems because they experience a "combinatorial explosion": they became exponentially slower as the problems grew larger.[15] Even humans rarely use the step-by-step deduction that early AI research could model. They solve most of their problems using fast, intuitive judgments.[16] Accurate and efficient reasoning is an unsolved problem. -

-

Knowledge representation

-
An ontology represents knowledge as a set of concepts within a domain and the relationships between those concepts.
-

Knowledge representation and knowledge engineering[17] allow AI programs to answer questions intelligently and make deductions about real-world facts. Formal knowledge representations are used in content-based indexing and retrieval,[18] scene interpretation,[19] clinical decision support,[20] knowledge discovery (mining "interesting" and actionable inferences from large databases),[21] and other areas.[22] -

A knowledge base is a body of knowledge represented in a form that can be used by a program. An ontology is the set of objects, relations, concepts, and properties used by a particular domain of knowledge.[23] Knowledge bases need to represent things such as: objects, properties, categories and relations between objects;[24] situations, events, states and time;[25] causes and effects;[26] knowledge about knowledge (what we know about what other people know);[27] default reasoning (things that humans assume are true until they are told differently and will remain true even when other facts are changing);[28] and many other aspects and domains of knowledge. -

Among the most difficult problems in knowledge representation are: the breadth of commonsense knowledge (the set of atomic facts that the average person knows is enormous);[29] and the sub-symbolic form of most commonsense knowledge (much of what people know is not represented as "facts" or "statements" that they could express verbally).[16] There is also the difficulty of knowledge acquisition, the problem of obtaining knowledge for AI applications.[c] -

-

Planning and decision making

-

An "agent" is anything that perceives and takes actions in the world. A rational agent has goals or preferences and takes actions to make them happen.[d][32] In automated planning, the agent has a specific goal.[33] In automated decision making, the agent has preferences – there are some situations it would prefer to be in, and some situations it is trying to avoid. The decision making agent assigns a number to each situation (called the "utility") that measures how much the agent prefers it. For each possible action, it can calculate the "expected utility": the utility of all possible outcomes of the action, weighted by the probability that the outcome will occur. It can then choose the action with the maximum expected utility.[34] -

In classical planning, the agent knows exactly what the effect of any action will be.[35] In most real-world problems, however, the agent may not be certain about the situation they are in (it is "unknown" or "unobservable") and it may not know for certain what will happen after each possible action (it is not "deterministic"). It must choose an action by making a probabilistic guess and then reassess the situation to see if the action worked.[36] -

In some problems, the agent's preferences may be uncertain, especially if there are other agents or humans involved. These can be learned (e.g., with inverse reinforcement learning) or the agent can seek information to improve its preferences.[37] Information value theory can be used to weigh the value of exploratory or experimental actions.[38] The space of possible future actions and situations is typically intractably large, so the agents must take actions and evaluate situations while being uncertain what the outcome will be. -

A Markov decision process has a transition model that describes the probability that a particular action will change the state in a particular way, and a reward function that supplies the utility of each state and the cost of each action. A policy associates a decision with each possible state. The policy could be calculated (e.g. by iteration), be heuristic, or it can be learned.[39] -

Game theory describes rational behavior of multiple interacting agents, and is used in AI programs that make decisions that involve other agents.[40] -

-

Learning

-

Machine learning is the study of programs that can improve their performance on a given task automatically.[41] It has been a part of AI from the beginning.[e] -

There are several kinds of machine learning. Unsupervised learning analyzes a stream of data and finds patterns and makes predictions without any other guidance.[44] Supervised learning requires a human to label the input data first, and comes in two main varieties: classification (where the program must learn to predict what category the input belongs in) and regression (where the program must deduce a numeric function based on numeric input).[45] -

In reinforcement learning the agent is rewarded for good responses and punished for bad ones. The agent learns to choose responses that are classified as "good".[46] Transfer learning is when the knowledge gained from one problem is applied to a new problem.[47] Deep learning is a type of machine learning that runs inputs through biologically inspired artificial neural networks for all of these types of learning.[48] -

Computational learning theory can assess learners by computational complexity, by sample complexity (how much data is required), or by other notions of optimization.[49] -

-
-

Natural language processing

-

Natural language processing (NLP)[50] allows programs to read, write and communicate in human languages such as English. Specific problems include speech recognition, speech synthesis, machine translation, information extraction, information retrieval and question answering.[51] -

Early work, based on Noam Chomsky's generative grammar and semantic networks, had difficulty with word-sense disambiguation[f] unless restricted to small domains called "micro-worlds" (due to the common sense knowledge problem[29]). Margaret Masterman believed that it was meaning, and not grammar that was the key to understanding languages, and that thesauri and not dictionaries should be the basis of computational language structure. -

Modern deep learning techniques for NLP include word embedding (representing words, typically as vectors encoding their meaning),[52] transformers (a deep learning architecture using an attention mechanism),[53] and others.[54] In 2019, generative pre-trained transformer (or "GPT") language models began to generate coherent text,[55][56] and by 2023 these models were able to get human-level scores on the bar exam, SAT test, GRE test, and many other real-world applications.[57] -

-

Perception

-

Machine perception is the ability to use input from sensors (such as cameras, microphones, wireless signals, active lidar, sonar, radar, and tactile sensors) to deduce aspects of the world. Computer vision is the ability to analyze visual input.[58] -

The field includes speech recognition,[59] image classification,[60] facial recognition, object recognition,[61] and robotic perception.[62] -

-

Social intelligence

-
Kismet, a robot head which was made in the 1990s; a machine that can recognize and simulate emotions.[63]
-

Affective computing is an interdisciplinary umbrella that comprises systems that recognize, interpret, process or simulate human feeling, emotion and mood.[64] For example, some virtual assistants are programmed to speak conversationally or even to banter humorously; it makes them appear more sensitive to the emotional dynamics of human interaction, or to otherwise facilitate human–computer interaction. -

However, this tends to give naïve users an unrealistic conception of the intelligence of existing computer agents.[65] Moderate successes related to affective computing include textual sentiment analysis and, more recently, multimodal sentiment analysis, wherein AI classifies the affects displayed by a videotaped subject.[66] -

-

General intelligence

-

A machine with artificial general intelligence should be able to solve a wide variety of problems with breadth and versatility similar to human intelligence.[11] -

-

Techniques

-

AI research uses a wide variety of techniques to accomplish the goals above.[b] -

-

Search and optimization

-

AI can solve many problems by intelligently searching through many possible solutions.[67] There are two very different kinds of search used in AI: state space search and local search. -

-

State space search

-

State space search searches through a tree of possible states to try to find a goal state.[68] For example, planning algorithms search through trees of goals and subgoals, attempting to find a path to a target goal, a process called means-ends analysis.[69] -

Simple exhaustive searches[70] are rarely sufficient for most real-world problems: the search space (the number of places to search) quickly grows to astronomical numbers. The result is a search that is too slow or never completes.[15] "Heuristics" or "rules of thumb" can help to prioritize choices that are more likely to reach a goal.[71] -

Adversarial search is used for game-playing programs, such as chess or Go. It searches through a tree of possible moves and counter-moves, looking for a winning position.[72] -

-

Local search

-
Illustration of gradient descent for 3 different starting points. Two parameters (represented by the plan coordinates) are adjusted in order to minimize the loss function (the height).

Local search uses mathematical optimization to find a solution to a problem. It begins with some form of guess and refines it incrementally.[73] -

Gradient descent is a type of local search that optimizes a set of numerical parameters by incrementally adjusting them to minimize a loss function. Variants of gradient descent are commonly used to train neural networks.[74] -

Another type of local search is evolutionary computation, which aims to iteratively improve a set of candidate solutions by "mutating" and "recombining" them, selecting only the fittest to survive each generation.[75] -

Distributed search processes can coordinate via swarm intelligence algorithms. Two popular swarm algorithms used in search are particle swarm optimization (inspired by bird flocking) and ant colony optimization (inspired by ant trails).[76] -

-

Logic

-

Formal Logic is used for reasoning and knowledge representation.[77] -Formal logic comes in two main forms: propositional logic (which operates on statements that are true or false and uses logical connectives such as "and", "or", "not" and "implies")[78] -and predicate logic (which also operates on objects, predicates and relations and uses quantifiers such as "Every X is a Y" and "There are some Xs that are Ys").[79] -

Logical inference (or deduction) is the process of proving a new statement (conclusion) from other statements that are already known to be true (the premises).[80] -A logical knowledge base also handles queries and assertions as a special case of inference.[81] -An inference rule describes what is a valid step in a proof. The most general inference rule is resolution.[82] -Inference can be reduced to performing a search to find a path that leads from premises to conclusions, where each step is the application of an inference rule.[83] -Inference performed this way is intractable except for short proofs in restricted domains. No efficient, powerful and general method has been discovered. -

Fuzzy logic assigns a "degree of truth" between 0 and 1. It can therefore handle propositions that are vague and partially true.[84] -Non-monotonic logics are designed to handle default reasoning.[28] -Other specialized versions of logic have been developed to describe many complex domains (see knowledge representation above). -

-

Probabilistic methods for uncertain reasoning

-
A simple Bayesian network, with the associated conditional probability tables.
-

Many problems in AI (including in reasoning, planning, learning, perception, and robotics) require the agent to operate with incomplete or uncertain information. AI researchers have devised a number of tools to solve these problems using methods from probability theory and economics.[85] -

Bayesian networks[86] -are a very general tool that can be used for many problems, including reasoning (using the Bayesian inference algorithm),[g][88] learning (using the expectation-maximization algorithm),[h][90] planning (using decision networks)[91] -and perception (using dynamic Bayesian networks).[92] -

Probabilistic algorithms can also be used for filtering, prediction, smoothing and finding explanations for streams of data, helping perception systems to analyze processes that occur over time (e.g., hidden Markov models or Kalman filters).[92] -

-Precise mathematical tools have been developed that analyze how an agent can make choices and plan, using decision theory, decision analysis,[93] and information value theory.[94] These tools include models such as Markov decision processes,[95] dynamic decision networks,[92] game theory and mechanism design.[96]

Expectation-maximization clustering of Old Faithful eruption data starts from a random guess but then successfully converges on an accurate clustering of the two physically distinct modes of eruption.
-

Classifiers and statistical learning methods

-

The simplest AI applications can be divided into two types: classifiers (e.g. "if shiny then diamond"), on one hand, and controllers (e.g. "if diamond then pick up"), on the other hand. Classifiers[97] -are functions that use pattern matching to determine the closest match. They can be fine-tuned based on chosen examples using supervised learning. Each pattern (also called an "observation") is labeled with a certain predefined class. All the observations combined with their class labels are known as a data set. When a new observation is received, that observation is classified based on previous experience.[45] -

There are many kinds of classifiers in use. The decision tree is the simplest and most widely used symbolic machine learning algorithm.[98] K-nearest neighbor algorithm was the most widely used analogical AI until the mid-1990s, and Kernel methods such as the support vector machine (SVM) displaced k-nearest neighbor in the 1990s.[99] -The naive Bayes classifier is reportedly the "most widely used learner"[100] at Google, due in part to its scalability.[101] -Neural networks are also used as classifiers.[102] -

-

Artificial neural networks

-
A neural network is an interconnected group of nodes, akin to the vast network of neurons in the human brain.
-

An Artificial neural network is based on a collection of nodes also known as artificial neurons, which loosely model the neurons in a biological brain. It is trained to recognise patterns, once trained it can recognise those patterns in fresh data. There is an input, at least one hidden layer of nodes and an output. Each node applies a function and once the weight crosses its specified threshold, the data is transmitted to the next layer. A network is typically called a deep neural network if it has at least 2 hidden layers.[102] -

Learning algorithms for neural networks use local search to choose the weights that will get the right output for each input during training. The most common training technique is the backpropagation algorithm.[103] -Neural networks learn to model complex relationships between inputs and outputs and find patterns in data. In theory, a neural network can learn any function.[104] -

In feedforward neural networks the signal passes in only one direction.[105] -Recurrent neural networks feed the output signal back into the input, which allows short-term memories of previous input events. Long short term memory is the most successful network architecture for recurrent networks.[106] -Perceptrons[107] -use only a single layer of neurons, deep learning[108] uses multiple layers. -Convolutional neural networks strengthen the connection between neurons that are "close" to each other – this is especially important in image processing, where a local set of neurons must identify an "edge" before the network can identify an object.[109] -

-
-

Deep learning

-
-

Deep learning[108] -uses several layers of neurons between the network's inputs and outputs. The multiple layers can progressively extract higher-level features from the raw input. For example, in image processing, lower layers may identify edges, while higher layers may identify the concepts relevant to a human such as digits or letters or faces.[110] -

Deep learning has profoundly improved the performance of programs in many important subfields of artificial intelligence, including computer vision, speech recognition, natural language processing, image classification[111] -and others. The reason that deep learning performs so well in so many applications is not known as of 2023.[112] -The sudden success of deep learning in 2012–2015 did not occur because of some new discovery or theoretical breakthrough (deep neural networks and backpropagation had been described by many people, as far back as the 1950s)[i] -but because of two factors: the incredible increase in computer power (including the hundred-fold increase in speed by switching to GPUs) and the availability of vast amounts of training data, especially the giant curated datasets used for benchmark testing, such as ImageNet.[j] -

-

GPT

-

Generative pre-trained transformers (GPT) are large language models that are based on the semantic relationships between words in sentences (natural language processing). Text-based GPT models are pre-trained on a large corpus of text which can be from the internet. The pre-training consists in predicting the next token (a token being usually a word, subword, or punctuation). Throughout this pre-training, GPT models accumulate knowledge about the world, and can then generate human-like text by repeatedly predicting the next token. Typically, a subsequent training phase makes the model more truthful, useful and harmless, usually with a technique called reinforcement learning from human feedback (RLHF). Current GPT models are still prone to generating falsehoods called "hallucinations", although this can be reduced with RLHF and quality data. They are used in chatbots which allow you to ask a question or request a task in simple text.[121][122] -

Current models and services include: Bard, ChatGPT, Grok, Claude, Copilot and LLaMA.[123] Multimodal GPT models can process different types of data (modalities) such as images, videos, sound and text.[124] -

-

Specialized hardware and software

-
Main articles: Programming languages for artificial intelligence and Hardware for artificial intelligence
-

In the late 2010s, graphics processing units (GPUs) that were increasingly designed with AI-specific enhancements and used with specialized TensorFlow software, had replaced previously used central processing unit (CPUs) as the dominant means for large-scale (commercial and academic) machine learning models' training.[125] -Historically, specialized languages, such as Lisp, Prolog, Python and others, had been used. -

-

Applications

-
Main article: Applications of artificial intelligence

AI and machine learning technology is used in most of the essential applications of the 2020s, including: search engines (such as Google Search), targeting online advertisements, recommendation systems (offered by Netflix, YouTube or Amazon), driving internet traffic, targeted advertising (AdSense, Facebook), virtual assistants (such as Siri or Alexa), autonomous vehicles (including drones, ADAS and self-driving cars), automatic language translation (Microsoft Translator, Google Translate), facial recognition (Apple's Face ID or Microsoft's DeepFace and Google's FaceNet) and image labeling (used by Facebook, Apple's iPhoto and TikTok). -

Health and Medicine

-
Main article: Artificial intelligence in healthcare
-

The application of AI in medicine and medical research has the potential to increase patient care and quality of life.[126] Through the lens of the Hippocratic Oath, medical professionals are ethically compelled to use AI, if applications can more accurately diagnose and treat patients. -

For medical research, AI is an important tool for processing and integrating Big Data. This is particularly important for organoid and tissue engineering development which use microscopy imaging as a key technique in fabrication.[127] -It has been suggested that AI can overcome discrepancies in funding allocated to different fields of research.[127] New AI tools can deepen our understanding of biomedically relevant pathways. For example, AlphaFold 2 (2021) demonstrated the ability to approximate, in hours rather than months, the 3D structure of a protein.[128] In 2023 it was reported that AI guided drug discovery helped find a class of antibiotics capable of killing two different types of drug-resistant bacteria.[129] -

-

Games

-
Main article: Game artificial intelligence
-

Game playing programs have been used since the 1950s to demonstrate and test AI's most advanced techniques.[130] Deep Blue became the first computer chess-playing system to beat a reigning world chess champion, Garry Kasparov, on 11 May 1997.[131] In 2011, in a Jeopardy! quiz show exhibition match, IBM's question answering system, Watson, defeated the two greatest Jeopardy! champions, Brad Rutter and Ken Jennings, by a significant margin.[132] In March 2016, AlphaGo won 4 out of 5 games of Go in a match with Go champion Lee Sedol, becoming the first computer Go-playing system to beat a professional Go player without handicaps. Then in 2017 it defeated Ke Jie, who was the best Go player in the world.[133] Other programs handle imperfect-information games, such as the poker-playing program Pluribus[134]. DeepMind developed increasingly generalistic reinfrocement learning models, such as with MuZero, which could be trained to play chess, Go, or Atari games.[135] In 2019, DeepMind's AlphaStar achieved grandmaster level in StarCraft II, a particularly challenging real-time strategy game that involves incomplete knowledge of what happens on the map.[136] In 2021 an AI agent competed in a Playstation Gran Turismo competition, winning against four of the world’s best Gran Turismo drivers using deep reinforcement learning.[137] -

-

Military

- -
Main article: Military artificial intelligence
-

Various countries are deploying AI military applications. The main applications enhance command and control, communications, sensors, integration and interoperability. Research is targeting intelligence collection and analysis, logistics, cyber operations, information operations, and semiautonomous and autonomous vehicles. AI technologies enable coordination of sensors and effectors, threat detection and identification, marking of enemy positions, target acquisition, coordination and deconfliction of distributed Joint Fires between networked combat vehicles involving manned and unmanned teams. AI was incorporated into military operations in Iraq and Syria. -

In November 2023, US Vice President Kamala Harris disclosed a declaration signed by 31 nations to set guardrails for the military use of IA. The commitments include using legal reviews to ensure the compliance of military AI with international laws, and being cautious and transparent in the development of this technology.[138] -

-

Generative AI

-
Main article: Generative artificial intelligence
-
Vincent van Gogh in watercolour created by generative AI software
-

In the early 2020s, generative AI gained widespread prominence. In March 2023, 58% of US adults had heard about ChatGPT and 14% had tried it.[139] The increasing realism and ease-of-use of AI-based text-to-image generators such as Midjourney, DALL-E, and Stable Diffusion sparked a trend of viral AI-generated photos. Widespread attention was gained by a fake photo of Pope Francis wearing a white puffer coat, the fictional arrest of Donald Trump, and a hoax of an attack on the Pentagon, as well as the usage in professional creative arts.[140][141] -

-

Industry Specific Tasks

- -

There are also thousands of successful AI applications used to solve specific problems for specific industries or institutions. In a 2017 survey, one in five companies reported they had incorporated "AI" in some offerings or processes.[142] A few examples are energy storage, medical diagnosis, military logistics, applications that predict the result of judicial decisions, foreign policy, or supply chain management. -

In agriculture, AI has helped farmers identify areas that need irrigation, fertilization, pesticide treatments or increasing yield. Agronomists use AI to conduct research and development. AI has been used to predict the ripening time for crops such as tomatoes, monitor soil moisture, operate agricultural robots, conduct predictive analytics, classify livestock pig call emotions, automate greenhouses, detect diseases and pests, and save water. -

Artificial intelligence is used in astronomy to analyze increasing amounts of available data and applications, mainly for "classification, regression, clustering, forecasting, generation, discovery, and the development of new scientific insights" for example for discovering exoplanets, forecasting solar activity, and distinguishing between signals and instrumental effects in gravitational wave astronomy. It could also be used for activities in space such as space exploration, including analysis of data from space missions, real-time science decisions of spacecraft, space debris avoidance, and more autonomous operation. -

-

Ethics

-

AI, like any powerful technology, has potential benefits and potential risks. AI may be able to advance science and find solutions for serious problems: Demis Hassabis of Deep Mind hopes to "solve intelligence, and then use that to solve everything else".[143] However, as the use of AI has become widespread, several unintended consequences and risks have been identified.[144] -

Anyone looking to use machine learning as part of real-world, in-production systems needs to factor ethics into their AI training processes and strive to avoid bias. This is especially true when using AI algorithms that are inherently unexplainable in deep learning.[145] -

-

Risks and harm

-

Privacy and copyright

-
Further information: Information privacy and Artificial intelligence and copyright
-

Machine learning algorithms require large amounts of data. The techniques used to acquire this data have raised concerns about privacy, surveillance and copyright. -

Technology companies collect a wide range of data from their users, including online activity, geolocation data, video and audio.[146] -For example, in order to build speech recognition algorithms, Amazon have recorded millions of private conversations and allowed temps to listen to and transcribe some of them.[147] -Opinions about this widespread surveillance range from those who see it as a necessary evil to those for whom it is clearly unethical and a violation of the right to privacy.[148] -

AI developers argue that this is the only way to deliver valuable applications. and have developed several techniques that attempt to preserve privacy while still obtaining the data, such as data aggregation, de-identification and differential privacy.[149] Since 2016, some privacy experts, such as Cynthia Dwork, began to view privacy in terms of fairness. Brian Christian wrote that experts have pivoted "from the question of 'what they know' to the question of 'what they're doing with it'.".[150] -

Generative AI is often trained on unlicensed copyrighted works, including in domains such as images or computer code; the output is then used under a rationale of "fair use". Also website owners who do not wish to have their copyrighted content be AI indexed or ‘scraped’ can add code to their site, as you would, if you did not want your website to be indexed by a search engine which is currently available to certain services such as OpenAI. Experts disagree about how well, and under what circumstances, this rationale will hold up in courts of law; relevant factors may include "the purpose and character of the use of the copyrighted work" and "the effect upon the potential market for the copyrighted work".[151] In 2023, leading authors (including John Grisham and Jonathan Franzen) sued AI companies for using their work to train generative AI.[152][153] -

-

Misinformation

-
See also: YouTube § Moderation and offensive content
-

YouTube, Facebook and others use recommender systems to guide users to more content. These AI programs were given the goal of maximizing user engagement (that is, the only goal was to keep people watching). The AI learned that users tended to choose misinformation, conspiracy theories, and extreme partisan content, and, to keep them watching, the AI recommended more of it. Users also tended to watch more content on the same subject, so the AI led people into filter bubbles where they received multiple versions of the same misinformation.[154] This convinced many users that the misinformation was true, and ultimately undermined trust in institutions, the media and the government.[155] The AI program had correctly learned to maximize its goal, but the result was harmful to society. After the U.S. election in 2016, major technology companies took steps to mitigate the problem. -

In 2022, generative AI began to create images, audio, video and text that are indistinguishable from real photographs, recordings, films or human writing. It is possible for bad actors to use this technology to create massive amounts of misinformation or propaganda.[156] AI pioneer Geoffrey Hinton expressed concern about AI enabling "authoritarian leaders to manipulate their electorates" on a large scale, among other risks.[157] -

-

Algorithmic bias and fairness

-
Main articles: Algorithmic bias and Fairness (machine learning)
-

Machine learning applications will be biased if they learn from biased data.[158] -The developers may not be aware that the bias exists.[159] -Bias can be introduced by the way training data is selected and by the way a model is deployed.[160][158] If a biased algorithm is used to make decisions that can seriously harm people (as it can in medicine, finance, recruitment, housing or policing) then the algorithm may cause discrimination.[161] -Fairness in machine learning is the study of how to prevent the harm caused by algorithmic bias. It has become serious area of academic study within AI. Researchers have discovered it is not always possible to define "fairness" in a way that satisfies all stakeholders.[162] -

On June 28, 2015, Google Photos's new image labeling feature mistakenly identified Jacky Alcine and a friend as "gorillas" because they were black. The system was trained on a dataset that contained very few images of black people,[163] a problem called "sample size disparity".[164] Google "fixed" this problem by preventing the system from labelling anything as a "gorilla". Eight years later, in 2023, Google Photos still could not identify a gorilla, and neither could similar products from Apple, Facebook, Microsoft and Amazon.[165] -

COMPAS is a commercial program widely used by U.S. courts to assess the likelihood of a defendant becoming a recidivist. -In 2016, Julia Angwin at ProPublica discovered that COMPAS exhibited racial bias, despite the fact that the program was not told the races of the defendants. Although the error rate for both whites and blacks was calibrated equal at exactly 61%, the errors for each race were different—the system consistently overestimated the chance that a black person would re-offend and would underestimate the chance that a white person would not re-offend.[166] In 2017, several researchers[k] showed that it was mathematically impossible for COMPAS to accommodate all possible measures of fairness when the base rates of re-offense were different for whites and blacks in the data.[168] -

A program can make biased decisions even if the data does not explicitly mention a problematic feature (such as "race" or "gender"). The feature will correlate with other features (like "address", "shopping history" or "first name"), and the program will make the same decisions based on these features as it would on "race" or "gender".[169] -Moritz Hardt said "the most robust fact in this research area is that fairness through blindness doesn't work."[170] -

Criticism of COMPAS highlighted a deeper problem with the misuse of AI. Machine learning models are designed to make "predictions" that are only valid if we assume that the future will resemble the past. If they are trained on data that includes the results of racist decisions in the past, machine learning models must predict that racist decisions will be made in the future. Unfortunately, if an application then uses these predictions as recommendations, some of these "recommendations" will likely be racist.[171] Thus, machine learning is not well suited to help make decisions in areas where there is hope that the future will be better than the past. It is necessarily descriptive and not proscriptive.[l] -

Bias and unfairness may go undetected because the developers are overwhelmingly white and male: among AI engineers, about 4% are black and 20% are women.[164] -

At its 2022 Conference on Fairness, Accountability, and Transparency (ACM FAccT 2022) the Association for Computing Machinery, in Seoul, South Korea, presented and published findings recommending that until AI and robotics systems are demonstrated to be free of bias mistakes, they are unsafe and the use of self-learning neural networks trained on vast, unregulated sources of flawed internet data should be curtailed.[173] -

-

Lack of transparency

-
See also: Explainable AI, Algorithmic transparency, and Right to explanation
-
Lidar testing vehicle for autonomous driving
-

Many AI systems are so complex that their designers cannot explain how they reach their decisions.[174] Particularly with deep neural networks, in which there are a large amount of non-linear relationships between inputs and outputs. But some popular explainability techniques exist.[175] -

There have been many cases where a machine learning program passed rigorous tests, but nevertheless learned something different than what the programmers intended. For example, a system that could identify skin diseases better than medical professionals was found to actually have a strong tendency to classify images with a ruler as "cancerous", because pictures of malignancies typically include a ruler to show the scale.[176] Another machine learning system designed to help effectively allocate medical resources was found to classify patients with asthma as being at "low risk" of dying from pneumonia. Having asthma is actually a severe risk factor, but since the patients having asthma would usually get much more medical care, they were relatively unlikely to die according to the training data. The correlation between asthma and low risk of dying from pneumonia was real, but misleading.[177] -

People who have been harmed by an algorithm's decision have a right to an explanation. Doctors, for example, are required to clearly and completely explain the reasoning behind any decision they make.[clarification needed][178] Early drafts of the European Union's General Data Protection Regulation in 2016 included an explicit statement that this right exists.[m] Industry experts noted that this is an unsolved problem with no solution in sight. Regulators argued that nevertheless the harm is real: if the problem has no solution, the tools should not be used.[179] -

DARPA established the XAI ("Explainable Artificial Intelligence") program in 2014 to try and solve these problems.[180] -

There are several potential solutions to the transparency problem. SHAP helps visualise the contribution of each feature to the output.[181] LIME can locally approximate a model with a simpler, interpretable model.[182] Multitask learning provides a large number of outputs in addition to the target classification. These other outputs can help developers deduce what the network has learned.[183] Deconvolution, DeepDream and other generative methods can allow developers to see what different layers of a deep network have learned and produce output that can suggest what the network is learning.[184] -

-

Conflict, surveillance and weaponized AI

-
Main articles: Lethal autonomous weapon, Artificial intelligence arms race, and AI safety
-

A lethal autonomous weapon is a machine that locates, selects and engages human targets without human supervision.[n] By 2015, over fifty countries were reported to be researching battlefield robots.[186] These weapons are considered especially dangerous for several reasons: if they kill an innocent person it is not clear who should be held accountable, it is unlikely they will reliably choose targets, and, if produced at scale, they are potentially weapons of mass destruction.[187] In 2014, 30 nations (including China) supported a ban on autonomous weapons under the United Nations' Convention on Certain Conventional Weapons, however the United States and others disagreed.[188] -

AI provides a number of tools that are particularly useful for authoritarian governments: smart spyware, face recognition and voice recognition allow widespread surveillance; such surveillance allows machine learning to classify potential enemies of the state and can prevent them from hiding; recommendation systems can precisely target propaganda and misinformation for maximum effect; deepfakes and generative AI aid in producing misinformation; advanced AI can make authoritarian centralized decision making more competitive with liberal and decentralized systems such as markets.[189] -

AI facial recognition systems are used for mass surveillance, notably in China.[190][191] In 2019, Bengaluru, India deployed AI-managed traffic signals. This system uses cameras to monitor traffic density and adjust signal timing based on the interval needed to clear traffic.[192] Terrorists, criminals and rogue states can use weaponized AI such as advanced digital warfare and lethal autonomous weapons. Machine-learning AI is also able to design tens of thousands of toxic molecules in a matter of hours.[193] -

-

Technological unemployment

-
Main articles: Workplace impact of artificial intelligence and Technological unemployment
-

From the early days of the development of artificial intelligence there have been arguments, for example those put forward by Joseph Weizenbaum, about whether tasks that can be done by computers actually should be done by them, given the difference between computers and humans, and between quantitative calculation and qualitative, value-based judgement.[194] -

Economists have frequently highlighted the risks of redundancies from AI, and speculated about unemployment if there is no adequate social policy for full employment.[195] -

In the past, technology has tended to increase rather than reduce total employment, but economists acknowledge that "we're in uncharted territory" with AI.[196] A survey of economists showed disagreement about whether the increasing use of robots and AI will cause a substantial increase in long-term unemployment, but they generally agree that it could be a net benefit if productivity gains are redistributed.[197] Risk estimates vary; for example, in the 2010s, Michael Osborne and Carl Benedikt Frey estimated 47% of U.S. jobs are at "high risk" of potential automation, while an OECD report classified only 9% of U.S. jobs as "high risk".[o][199] The methodology of speculating about future employment levels has been criticised as lacking evidential foundation, and for implying that technology, rather than social policy, creates unemployment, as opposed to redundancies.[195] -

Unlike previous waves of automation, many middle-class jobs may be eliminated by artificial intelligence; The Economist stated in 2015 that "the worry that AI could do to white-collar jobs what steam power did to blue-collar ones during the Industrial Revolution" is "worth taking seriously".[200] Jobs at extreme risk range from paralegals to fast food cooks, while job demand is likely to increase for care-related professions ranging from personal healthcare to the clergy.[201] -

In April 2023, it was reported that 70% of the jobs for Chinese video game illlustrators had been eliminated by generative artificial intelligence.[202][203] -

-

Existential risk

-
Main article: Existential risk from artificial general intelligence
-

It has been argued AI will become so powerful that humanity may irreversibly lose control of it. This could, as physicist Stephen Hawking stated, "spell the end of the human race".[204] This scenario has been common in science fiction, when a computer or robot suddenly develops a human-like "self-awareness" (or "sentience" or "consciousness") and becomes a malevolent character.[p] These sci-fi scenarios are misleading in several ways. -

First, AI does not require human-like "sentience" to be an existential risk. Modern AI programs are given specific goals and use learning and intelligence to achieve them. Philosopher Nick Bostrom argued that if one gives almost any goal to a sufficiently powerful AI, it may choose to destroy humanity to achieve it (he used the example of a paperclip factory manager).[206] Stuart Russell gives the example of household robot that tries to find a way to kill its owner to prevent it from being unplugged, reasoning that "you can't fetch the coffee if you're dead."[207] In order to be safe for humanity, a superintelligence would have to be genuinely aligned with humanity's morality and values so that it is "fundamentally on our side".[208] -

Second, Yuval Noah Harari argues that AI does not require a robot body or physical control to pose an existential risk. The essential parts of civilization are not physical. Things like ideologies, law, government, money and the economy are made of language; they exist because there are stories that billions of people believe. The current prevalence of misinformation suggests that an AI could use language to convince people to believe anything, even to take actions that are destructive.[209] -

The opinions amongst experts and industry insiders are mixed, with sizable fractions both concerned and unconcerned by risk from eventual superintelligent AI.[210] Personalities such as Stephen Hawking, Bill Gates, and Elon Musk have expressed concern about existential risk from AI.[211] -

In the early 2010s, experts argued that the risks are too distant in the future to warrant research or that humans will be valuable from the perspective of a superintelligent machine.[212] However, after 2016, the study of current and future risks and possible solutions became a serious area of research.[213] -

AI pioneers including Fei-Fei Li, Geoffrey Hinton, Yoshua Bengio, Cynthia Breazeal, Rana el Kaliouby, Demis Hassabis, Joy Buolamwini, and Sam Altman have expressed concerns about the risks of AI. In 2023, many leading AI experts issued the joint statement that "Mitigating the risk of extinction from AI should be a global priority alongside other societal-scale risks such as pandemics and nuclear war".[214] -

Other researchers, however, spoke in favor of a less dystopian view. AI pioneer Juergen Schmidhuber did not sign the joint statement, emphasising that in 95% of all cases, AI research is about making "human lives longer and healthier and easier."[215] While the tools that are now being used to improve lives can also be used by bad actors, "they can also be used against the bad actors."[216][217] Andrew Ng also argued that "it’s a mistake to fall for the doomsday hype on AI—and that regulators who do will only benefit vested interests."[218] Yann LeCun "scoffs at his peers’ dystopian scenarios of supercharged misinformation and even, eventually, human extinction."[219] -

-

Limiting AI

-

Possible options for limiting AI include: using Embedded Ethics or Constitutional AI where companies or governments can add a policy, restricting high levels of compute power in training, restricting the ability to rewrite its own code base, restrict certain AI techniques but not in the training phase, open-source (transparency) vs proprietary (could be more restricted), backup model with redundancy, restricting security, privacy and copyright, restricting or controlling the memory, real-time monitoring, risk analysis, emergency shut-off, rigorous simulation and testing, model certification, assess known vulnerabilities, restrict the training material, restrict access to the internet, issue terms of use. -

-

Ethical machines and alignment

-
Main articles: Machine ethics, AI safety, Friendly artificial intelligence, Artificial moral agents, and Human Compatible
-

Friendly AI are machines that have been designed from the beginning to minimize risks and to make choices that benefit humans. Eliezer Yudkowsky, who coined the term, argues that developing friendly AI should be a higher research priority: it may require a large investment and it must be completed before AI becomes an existential risk.[220] -

Machines with intelligence have the potential to use their intelligence to make ethical decisions. The field of machine ethics provides machines with ethical principles and procedures for resolving ethical dilemmas.[221] -The field of machine ethics is also called computational morality,[221] -and was founded at an AAAI symposium in 2005.[222] -

Other approaches include Wendell Wallach's "artificial moral agents"[223] -and Stuart J. Russell's three principles for developing provably beneficial machines.[224] -

-

Frameworks

-

Artificial Intelligence projects can have their ethical permissibility tested while designing, developing, and implementing an AI system. An AI framework such as the Care and Act Framework containing the SUM values – developed by the Alan Turing Institute tests projects in four main areas:[225][226] -

-
  • RESPECT the dignity of individual people
    • -
  • CONNECT with other people sincerely, openly and inclusively
    • -
  • CARE for the wellbeing of everyone
    • -
  • PROTECT social values, justice and the public interest
-

Other developments in ethical frameworks include those decided upon during the Asilomar Conference, the Montreal Declaration for Responsible AI, and the IEEE's Ethics of Autonomous Systems initiative, among others;[227] however, these principles do not go without their criticisms, especially regards to the people chosen contributes to these frameworks.[228] -

Promotion of the wellbeing of the people and communities that these technologies affect requires consideration of the social and ethical implications at all stages of AI system design, development and implementation, and collaboration between job roles such as data scientists, product managers, data engineers, domain experts, and delivery managers.[229] -

-

Regulation

-
Main articles: Regulation of artificial intelligence, Regulation of algorithms, and AI safety
-
AI Safety Summit
The first global AI Safety Summit was held in 2023 with a declaration calling for international co-operation
-

The regulation of artificial intelligence is the development of public sector policies and laws for promoting and regulating artificial intelligence (AI); it is therefore related to the broader regulation of algorithms.[230] -The regulatory and policy landscape for AI is an emerging issue in jurisdictions globally.[231] According to AI Index at Stanford, the annual number of AI-related laws passed in the 127 survey countries jumped from one passed in 2016 to 37 passed in 2022 alone.[232][233] -Between 2016 and 2020, more than 30 countries adopted dedicated strategies for AI.[234] -Most EU member states had released national AI strategies, as had Canada, China, India, Japan, Mauritius, the Russian Federation, Saudi Arabia, United Arab Emirates, US and Vietnam. Others were in the process of elaborating their own AI strategy, including Bangladesh, Malaysia and Tunisia.[234] -The Global Partnership on Artificial Intelligence was launched in June 2020, stating a need for AI to be developed in accordance with human rights and democratic values, to ensure public confidence and trust in the technology.[234] Henry Kissinger, Eric Schmidt, and Daniel Huttenlocher published a joint statement in November 2021 calling for a government commission to regulate AI.[235] -In 2023, OpenAI leaders published recommendations for the governance of superintelligence, which they believe may happen in less than 10 years.[236] In 2023, the United Nations also launched an advisory body to provide recommendations on AI governance; the body comprises technology company executives, governments officials and academics.[237] -

In a 2022 Ipsos survey, attitudes towards AI varied greatly by country; 78% of Chinese citizens, but only 35% of Americans, agreed that "products and services using AI have more benefits than drawbacks".[232] A 2023 Reuters/Ipsos poll found that 61% of Americans agree, and 22% disagree, that AI poses risks to humanity.[238] -In a 2023 Fox News poll, 35% of Americans thought it "very important", and an additional 41% thought it "somewhat important", for the federal government to regulate AI, versus 13% responding "not very important" and 8% responding "not at all important".[239][240] -

In November 2023, the first global AI Safety Summit was held in Bletchley Park in the UK to discuss the near and far term risks of AI and the possibility of mandatory and voluntary regulatory frameworks.[241] 28 countries including the United States, China, and the European Union issued a declaration at the start of the summit, calling for international co-operation to manage the challenges and risks of artificial intelligence.[242][243] -

-

History

-
Main article: History of artificial intelligence
-
For a chronological guide, see Timeline of artificial intelligence.
-

The study of mechanical or "formal" reasoning began with philosophers and mathematicians in antiquity. The study of logic led directly to Alan Turing's theory of computation, which suggested that a machine, by shuffling symbols as simple as "0" and "1", could simulate both mathematical deduction and formal reasoning, which is known as the Church–Turing thesis.[244] This, along with concurrent discoveries in cybernetics and information theory, led researchers to consider the possibility of building an "electronic brain".[q][246] -

Alan Turing was thinking about machine intelligence at least as early as 1941, when he circulated a paper on machine intelligence which could be the earliest paper in the field of AI – though it is now lost.[2] The first available paper generally recognized as "AI" was McCullouch and Pitts design for Turing-complete "artificial neurons" in 1943 – the first mathematical model of a neural network.[247] The paper was influenced by Turing's earlier paper 'On Computable Numbers' from 1936 using similar two-state boolean 'neurons', but was the first to apply it to neuronal function.[2] -

The term 'Machine Intelligence' was used by Alan Turing during his life which was later often referred to as 'Artificial Intelligence' after his death in 1954. In 1950 Turing published the best known of his papers 'Computing Machinery and Intelligence', the paper introduced his concept of what is now known as the Turing test to the general public. Then followed three radio broadcasts on AI by Turing, the lectures: 'Intelligent Machinery, A Heretical Theory’, ‘Can Digital Computers Think’? and the panel discussion ‘Can Automatic Calculating Machines be Said to Think’. By 1956 computer intelligence had been actively pursued for more than a decade in Britain; the earliest AI programmes were written there in 1951–1952.[2] -

In 1951, using a Ferranti Mark 1 computer of the University of Manchester, checkers and chess programs were wrote where you could play against the computer.[248] The field of American AI research was founded at a workshop at Dartmouth College in 1956.[r][3] The attendees became the leaders of AI research in the 1960s.[s] They and their students produced programs that the press described as "astonishing":[t] computers were learning checkers strategies, solving word problems in algebra, proving logical theorems and speaking English.[u][4] Artificial Intelligence laboratories were set up at a number of British and US Universities in the latter 1950s and early 1960s.[2] -

They had, however, underestimated the difficulty of the problem.[v] Both the U.S. and British governments cut off exploratory research in response to the criticism of Sir James Lighthill[253] and ongoing pressure from the U.S. Congress to fund more productive projects. Minsky's and Papert's book Perceptrons was understood as proving that artificial neural networks would never be useful for solving real-world tasks, thus discrediting the approach altogether.[254] The "AI winter", a period when obtaining funding for AI projects was difficult, followed.[6] -

In the early 1980s, AI research was revived by the commercial success of expert systems,[255] a form of AI program that simulated the knowledge and analytical skills of human experts. By 1985, the market for AI had reached over a billion dollars. At the same time, Japan's fifth generation computer project inspired the U.S. and British governments to restore funding for academic research.[5] However, beginning with the collapse of the Lisp Machine market in 1987, AI once again fell into disrepute, and a second, longer-lasting winter began.[7] -

Many researchers began to doubt that the current practices would be able to imitate all the processes of human cognition, especially perception, robotics, learning and pattern recognition.[256] A number of researchers began to look into "sub-symbolic" approaches.[257] Robotics researchers, such as Rodney Brooks, rejected "representation" in general and focussed directly on engineering machines that move and survive.[w] Judea Pearl, Lofti Zadeh and others developed methods that handled incomplete and uncertain information by making reasonable guesses rather than precise logic.[85][262] But the most important development was the revival of "connectionism", including neural network research, by Geoffrey Hinton and others.[263] In 1990, Yann LeCun successfully showed that convolutional neural networks can recognize handwritten digits, the first of many successful applications of neural networks.[264] -

AI gradually restored its reputation in the late 1990s and early 21st century by exploiting formal mathematical methods and by finding specific solutions to specific problems. This "narrow" and "formal" focus allowed researchers to produce verifiable results and collaborate with other fields (such as statistics, economics and mathematics).[265] -By 2000, solutions developed by AI researchers were being widely used, although in the 1990s they were rarely described as "artificial intelligence".[266] -

Several academic researchers became concerned that AI was no longer pursuing the original goal of creating versatile, fully intelligent machines. Beginning around 2002, they founded the subfield of artificial general intelligence (or "AGI"), which had several well-funded institutions by the 2010s.[11] -

Deep learning began to dominate industry benchmarks in 2012 and was adopted throughout the field.[8] -For many specific tasks, other methods were abandoned.[x] -Deep learning's success was based on both hardware improvements (faster computers,[268] graphics processing units, cloud computing[269]) -and access to large amounts of data[270] (including curated datasets,[269] such as ImageNet). -

Deep learning's success led to an enormous increase in interest and funding in AI.[y] -The amount of machine learning research (measured by total publications) increased by 50% in the years 2015–2019,[234] -and WIPO reported that AI was the most prolific emerging technology in terms of the number of patent applications and granted patents.[271] -According to 'AI Impacts', about $50 billion annually was invested in "AI" around 2022 in the US alone and about 20% of new US Computer Science PhD graduates have specialized in "AI";[272] -about 800,000 "AI"-related US job openings existed in 2022.[273] The large majority of the advances have occurred within the United States, with its companies, universities, and research labs leading artificial intelligence research.[10] -

In 2016, issues of fairness and the misuse of technology were catapulted into center stage at machine learning conferences, publications vastly increased, funding became available, and many researchers re-focussed their careers on these issues. The alignment problem became a serious field of academic study.[213] -

-

Philosophy

-
Main article: Philosophy of artificial intelligence
-

Defining artificial intelligence

-
Main articles: Turing test, Intelligent agent, Dartmouth workshop, and Synthetic intelligence
-

Alan Turing wrote in 1950 "I propose to consider the question 'can machines think'?"[274] He advised changing the question from whether a machine "thinks", to "whether or not it is possible for machinery to show intelligent behaviour".[274] He devised the Turing test, which measures the ability of a machine to simulate human conversation.[275] Since we can only observe the behavior of the machine, it does not matter if it is "actually" thinking or literally has a "mind". Turing notes that we can not determine these things about other people[z] but "it is usual to have a polite convention that everyone thinks"[276] -

Russell and Norvig agree with Turing that AI must be defined in terms of "acting" and not "thinking".[277] However, they are critical that the test compares machines to people. "Aeronautical engineering texts," they wrote, "do not define the goal of their field as making 'machines that fly so exactly like pigeons that they can fool other pigeons.'"[278] AI founder John McCarthy agreed, writing that "Artificial intelligence is not, by definition, simulation of human intelligence".[279] -

McCarthy defines intelligence as "the computational part of the ability to achieve goals in the world."[280] Another AI founder, Marvin Minsky similarly defines it as "the ability to solve hard problems".[281] These definitions view intelligence in terms of well-defined problems with well-defined solutions, where both the difficulty of the problem and the performance of the program are direct measures of the "intelligence" of the machine—and no other philosophical discussion is required, or may not even be possible. -

Another definition has been adopted by Google,[282] a major practitioner in the field of AI. This definition stipulates the ability of systems to synthesize information as the manifestation of intelligence, similar to the way it is defined in biological intelligence. -

-

Evaluating approaches to AI

-

No established unifying theory or paradigm has guided AI research for most of its history.[aa] The unprecedented success of statistical machine learning in the 2010s eclipsed all other approaches (so much so that some sources, especially in the business world, use the term "artificial intelligence" to mean "machine learning with neural networks"). This approach is mostly sub-symbolic, soft and narrow (see below). Critics argue that these questions may have to be revisited by future generations of AI researchers. -

-

Symbolic AI and its limits

-

Symbolic AI (or "GOFAI")[284] simulated the high-level conscious reasoning that people use when they solve puzzles, express legal reasoning and do mathematics. They were highly successful at "intelligent" tasks such as algebra or IQ tests. In the 1960s, Newell and Simon proposed the physical symbol systems hypothesis: "A physical symbol system has the necessary and sufficient means of general intelligent action."[285] -

However, the symbolic approach failed on many tasks that humans solve easily, such as learning, recognizing an object or commonsense reasoning. Moravec's paradox is the discovery that high-level "intelligent" tasks were easy for AI, but low level "instinctive" tasks were extremely difficult.[286] Philosopher Hubert Dreyfus had argued since the 1960s that human expertise depends on unconscious instinct rather than conscious symbol manipulation, and on having a "feel" for the situation, rather than explicit symbolic knowledge.[287] Although his arguments had been ridiculed and ignored when they were first presented, eventually, AI research came to agree with him.[ab][16] -

The issue is not resolved: sub-symbolic reasoning can make many of the same inscrutable mistakes that human intuition does, such as algorithmic bias. Critics such as Noam Chomsky argue continuing research into symbolic AI will still be necessary to attain general intelligence,[289][290] in part because sub-symbolic AI is a move away from explainable AI: it can be difficult or impossible to understand why a modern statistical AI program made a particular decision. The emerging field of neuro-symbolic artificial intelligence attempts to bridge the two approaches. -

-

Neat vs. scruffy

-
Main article: Neats and scruffies
-

"Neats" hope that intelligent behavior is described using simple, elegant principles (such as logic, optimization, or neural networks). "Scruffies" expect that it necessarily requires solving a large number of unrelated problems. Neats defend their programs with theoretical rigor, scruffies rely mainly on incremental testing to see if they work. This issue was actively discussed in the 1970s and 1980s,[291] but eventually was seen as irrelevant. Modern AI has elements of both. -

-

Soft vs. hard computing

-
Main article: Soft computing
-

Finding a provably correct or optimal solution is intractable for many important problems.[15] Soft computing is a set of techniques, including genetic algorithms, fuzzy logic and neural networks, that are tolerant of imprecision, uncertainty, partial truth and approximation. Soft computing was introduced in the late 1980s and most successful AI programs in the 21st century are examples of soft computing with neural networks. -

-

Narrow vs. general AI

-
Main article: Artificial general intelligence
-

AI researchers are divided as to whether to pursue the goals of artificial general intelligence and superintelligence directly or to solve as many specific problems as possible (narrow AI) in hopes these solutions will lead indirectly to the field's long-term goals.[292][293] General intelligence is difficult to define and difficult to measure, and modern AI has had more verifiable successes by focusing on specific problems with specific solutions. The experimental sub-field of artificial general intelligence studies this area exclusively. -

-

Machine consciousness, sentience and mind

-
Main articles: Philosophy of artificial intelligence and Artificial consciousness
-

The philosophy of mind does not know whether a machine can have a mind, consciousness and mental states, in the same sense that human beings do. This issue considers the internal experiences of the machine, rather than its external behavior. Mainstream AI research considers this issue irrelevant because it does not affect the goals of the field: to build machines that can solve problems using intelligence. Russell and Norvig add that "[t]he additional project of making a machine conscious in exactly the way humans are is not one that we are equipped to take on."[294] However, the question has become central to the philosophy of mind. It is also typically the central question at issue in artificial intelligence in fiction. -

-

Consciousness

-
Main articles: Hard problem of consciousness and Theory of mind
-

David Chalmers identified two problems in understanding the mind, which he named the "hard" and "easy" problems of consciousness.[295] The easy problem is understanding how the brain processes signals, makes plans and controls behavior. The hard problem is explaining how this feels or why it should feel like anything at all, assuming we are right in thinking that it truly does feel like something (Dennett's consciousness illusionism says this is an illusion). Human information processing is easy to explain, however, human subjective experience is difficult to explain. For example, it is easy to imagine a color-blind person who has learned to identify which objects in their field of view are red, but it is not clear what would be required for the person to know what red looks like.[296] -

-

Computationalism and functionalism

-
Main articles: Computational theory of mind, Functionalism (philosophy of mind), and Chinese room
-

Computationalism is the position in the philosophy of mind that the human mind is an information processing system and that thinking is a form of computing. Computationalism argues that the relationship between mind and body is similar or identical to the relationship between software and hardware and thus may be a solution to the mind–body problem. This philosophical position was inspired by the work of AI researchers and cognitive scientists in the 1960s and was originally proposed by philosophers Jerry Fodor and Hilary Putnam.[297] -

Philosopher John Searle characterized this position as "strong AI": "The appropriately programmed computer with the right inputs and outputs would thereby have a mind in exactly the same sense human beings have minds."[ac] Searle counters this assertion with his Chinese room argument, which attempts to show that, even if a machine perfectly simulates human behavior, there is still no reason to suppose it also has a mind.[301] -

-

Robot rights

-
Main article: Robot rights
-

If a machine has a mind and subjective experience, then it may also have sentience (the ability to feel), and if so it could also suffer; it has been argued that this could entitle it to certain rights.[302] Any hypothetical robot rights would lie on a spectrum with animal rights and human rights.[303] This issue has been considered in fiction for centuries,[304] and is now being considered by, for example, California's Institute for the Future; however, critics argue that the discussion is premature.[305] -

-

Future

-

Superintelligence and the singularity

-

A superintelligence is a hypothetical agent that would possess intelligence far surpassing that of the brightest and most gifted human mind.[293] -

If research into artificial general intelligence produced sufficiently intelligent software, it might be able to reprogram and improve itself. The improved software would be even better at improving itself, leading to what I. J. Good called an "intelligence explosion" and Vernor Vinge called a "singularity".[306] -

However, technologies cannot improve exponentially indefinitely, and typically follow an S-shaped curve, slowing when they reach the physical limits of what the technology can do.[307] -

-

Transhumanism

-

Robot designer Hans Moravec, cyberneticist Kevin Warwick, and inventor Ray Kurzweil have predicted that humans and machines will merge in the future into cyborgs that are more capable and powerful than either. This idea, called transhumanism, has roots in Aldous Huxley and Robert Ettinger.[308] -

Edward Fredkin argues that "artificial intelligence is the next stage in evolution", an idea first proposed by Samuel Butler's "Darwin among the Machines" as far back as 1863, and expanded upon by George Dyson in his book of the same name in 1998.[309] -

-

In fiction

-
Main article: Artificial intelligence in fiction
-
The word "robot" itself was coined by Karel Čapek in his 1921 play R.U.R., the title standing for "Rossum's Universal Robots".
-

Thought-capable artificial beings have appeared as storytelling devices since antiquity,[310] and have been a persistent theme in science fiction.[311] -

A common trope in these works began with Mary Shelley's Frankenstein, where a human creation becomes a threat to its masters. This includes such works as Arthur C. Clarke's and Stanley Kubrick's 2001: A Space Odyssey (both 1968), with HAL 9000, the murderous computer in charge of the Discovery One spaceship, as well as The Terminator (1984) and The Matrix (1999). In contrast, the rare loyal robots such as Gort from The Day the Earth Stood Still (1951) and Bishop from Aliens (1986) are less prominent in popular culture.[312] -

Isaac Asimov introduced the Three Laws of Robotics in many books and stories, most notably the "Multivac" series about a super-intelligent computer of the same name. Asimov's laws are often brought up during lay discussions of machine ethics;[313] while almost all artificial intelligence researchers are familiar with Asimov's laws through popular culture, they generally consider the laws useless for many reasons, one of which is their ambiguity.[314] -

Several works use AI to force us to confront the fundamental question of what makes us human, showing us artificial beings that have the ability to feel, and thus to suffer. This appears in Karel Čapek's R.U.R., the films A.I. Artificial Intelligence and Ex Machina, as well as the novel Do Androids Dream of Electric Sheep?, by Philip K. Dick. Dick considers the idea that our understanding of human subjectivity is altered by technology created with artificial intelligence.[315] -

-

See also

- -

Explanatory notes

-
-
    -
  1. ^ a b This list of intelligent traits is based on the topics covered by the major AI textbooks, including: Russell & Norvig (2021), Luger & Stubblefield (2004), Poole, Mackworth & Goebel (1998) and Nilsson (1998) -
    2. -
  2. ^ a b This list of tools is based on the topics covered by the major AI textbooks, including: Russell & Norvig (2021), Luger & Stubblefield (2004), Poole, Mackworth & Goebel (1998) and Nilsson (1998) -
    3. -
  3. ^ It is among the reasons that expert systems proved to be inefficient for capturing knowledge.[30][31] -
    4. -
  4. ^ -"Rational agent" is general term used in economics, philosophy and theoretical artificial intelligence. It can refer to anything that directs its behavior to accomplish goals, such as a person, an animal, a corporation, a nation, or, in the case of AI, a computer program. -
    5. -
  5. ^ Alan Turing discussed the centrality of learning as early as 1950, in his classic paper "Computing Machinery and Intelligence".[42] In 1956, at the original Dartmouth AI summer conference, Ray Solomonoff wrote a report on unsupervised probabilistic machine learning: "An Inductive Inference Machine".[43] -
    6. -
  6. ^ See AI winter § Machine translation and the ALPAC report of 1966 -
    7. -
  7. ^ -Compared with symbolic logic, formal Bayesian inference is computationally expensive. For inference to be tractable, most observations must be conditionally independent of one another. AdSense uses a Bayesian network with over 300 million edges to learn which ads to serve.[87] -
    8. -
  8. ^ Expectation-maximization, one of the most popular algorithms in machine learning, allows clustering in the presence of unknown latent variables.[89] -
    9. -
  9. ^ -Some form of deep neural networks (without a specific learning algorithm) were described by: -Alan Turing (1948);[113] -Frank Rosenblatt(1957);[113] -Karl Steinbuch and Roger David Joseph (1961).[114] -Deep or recurrent networks that learned (or used gradient descent) were developed by: -Ernst Ising and Wilhelm Lenz (1925);[115] -Oliver Selfridge (1959);[114] -Alexey Ivakhnenko and Valentin Lapa (1965);[115] -Kaoru Nakano (1977);[116] -Shun-Ichi Amari (1972);[116] -John Joseph Hopfield (1982).[116] -Backpropagation was independently discovered by: -Henry J. Kelley (1960);[113] -Arthur E. Bryson (1962);[113] -Stuart Dreyfus (1962);[113] -Arthur E. Bryson and Yu-Chi Ho (1969);[113] -Seppo Linnainmaa (1970);[117] -Paul Werbos (1974).[113] -In fact, backpropagation and gradient descent are straight forward applications of Gottfried Leibniz' chain rule in calculus (1676),[118] and is essentially identical (for one layer) to the method of least squares, developed independently by Johann Carl Friedrich Gauss (1795) and Adrien-Marie Legendre (1805).[119] There are probably many others, yet to be discovered by historians of science. -
    10. -
  10. ^ Geoffrey Hinton said, of his work on neural networks in the 1990s, "our labeled datasets were thousands of times too small. [And] our computers were millions of times too slow"[120] -
    11. -
  11. ^ Including Jon Kleinberg (Cornell), Sendhil Mullainathan (University of Chicago), Cynthia Chouldechova (Carnegie Mellon) and Sam Corbett-Davis (Stanford)[167] -
    12. -
  12. ^ Moritz Hardt (a director at the Max Planck Institute for Intelligent Systems) argues that machine learning "is fundamentally the wrong tool for a lot of domains, where you're trying to design interventions and mechanisms that change the world."[172] -
    13. -
  13. ^ When the law was passed in 2018, it still contained a form of this provision. -
    14. -
  14. ^ This is the United Nations' definition, and includes things like land mines as well.[185] -
    15. -
  15. ^ See table 4; 9% is both the OECD average and the US average.[198] -
    16. -
  16. ^ Sometimes called a "robopocalypse".[205] -
    17. -
  17. ^ "Electronic brain" was the term used by the press around this time.[245] -
    18. -
  18. ^ -Daniel Crevier wrote, "the conference is generally recognized as the official birthdate of the new science."[249] Russell and Norvig called the conference "the inception of artificial intelligence."[247] -
    19. -
  19. ^ -Russell and Norvig wrote "for the next 20 years the field would be dominated by these people and their students."[250] -
    20. -
  20. ^ -Russell and Norvig wrote "it was astonishing whenever a computer did anything kind of smartish".[251] -
    21. -
  21. ^ -The programs described are Arthur Samuel's checkers program for the IBM 701, Daniel Bobrow's STUDENT, Newell and Simon's Logic Theorist and Terry Winograd's SHRDLU. -
    22. -
  22. ^ Russell and Norvig write: "in almost all cases, these early systems failed on more difficult problems"[252] -
    23. -
  23. ^ -Embodied approaches to AI[258] were championed by Hans Moravec[259] and Rodney Brooks[260] and went by many names: Nouvelle AI.[260] Developmental robotics,[261] -
    24. -
  24. ^ Matteo Wong wrote in The Atlantic: "Whereas for decades, computer-science fields such as natural-language processing, computer vision, and robotics used extremely different methods, now they all use a programming method called "deep learning." As a result, their code and approaches have become more similar, and their models are easier to integrate into one another."[267] -
    25. -
  25. ^ Jack Clark wrote in Bloomberg: "After a half-decade of quiet breakthroughs in artificial intelligence, 2015 has been a landmark year. Computers are smarter and learning faster than ever," and noted that the number of software projects that use machine learning at Google increased from a "sporadic usage" in 2012 to more than 2,700 projects in 2015.[269] -
    26. -
  26. ^ See Problem of other minds -
    27. -
  27. ^ Nils Nilsson wrote in 1983: "Simply put, there is wide disagreement in the field about what AI is all about."[283] -
    28. -
  28. ^ -Daniel Crevier wrote that "time has proven the accuracy and perceptiveness of some of Dreyfus's comments. Had he formulated them less aggressively, constructive actions they suggested might have been taken much earlier."[288] -
    29. -
  29. ^ -Searle presented this definition of "Strong AI" in 1999.[298] Searle's original formulation was "The appropriately programmed computer really is a mind, in the sense that computers given the right programs can be literally said to understand and have other cognitive states."[299] Strong AI is defined similarly by Russell and Norvig: "Stong AI – the assertion that machines that do so are actually thinking (as opposed to simulating thinking)."[300] -
    30. -
-

References

-
-
    -
  1. ^ Google (2016). -
    2. -
  2. ^ a b c d e Copeland, J., ed. (2004). The Essential Turing: the ideas that gave birth to the computer age. Oxford, England: Clarendon Press. ISBN 0-19-825079-7. -
    3. -
  3. ^ a b -Dartmouth workshop: - -The proposal: - -
    4. -
  4. ^ a b -Successful programs the 1960s: - - -
    5. -
  5. ^ a b -Funding initiatives in the early 1980s: Fifth Generation Project (Japan), Alvey (UK), Microelectronics and Computer Technology Corporation (US), Strategic Computing Initiative (US): - - -
    6. -
  6. ^ a b -First AI Winter, Lighthill report, Mansfield Amendment - -
    7. -
  7. ^ a b -Second AI Winter: - -
    8. -
  8. ^ a b -Deep learning revolution, AlexNet: - -
    9. -
  9. ^ Toews (2023). -
    10. -
  10. ^ a b Frank (2023). -
    11. -
  11. ^ a b c -Artificial general intelligence: - -Proposal for the modern version: - -Warnings of overspecialization in AI from leading researchers: - -
    12. -
  12. ^ Russell & Norvig (2021, §1.2). -
    13. -
  13. ^ -Problem solving, puzzle solving, game playing and deduction: - - -
    14. -
  14. ^ -Uncertain reasoning: - -
    15. -
  15. ^ a b c -Intractability and efficiency and the combinatorial explosion: - - -
    16. -
  16. ^ a b c -Psychological evidence of the prevalence sub-symbolic reasoning and knowledge: - -
    17. -
  17. ^ -Knowledge representation and knowledge engineering: - -
    18. -
  18. ^ Smoliar & Zhang (1994). -
    19. -
  19. ^ Neumann & Möller (2008). -
    20. -
  20. ^ Kuperman, Reichley & Bailey (2006). -
    21. -
  21. ^ McGarry (2005). -
    22. -
  22. ^ Bertini, Del Bimbo & Torniai (2006). -
    23. -
  23. ^ Russell & Norvig (2021), pp. 272. -
    24. -
  24. ^ -Representing categories and relations: Semantic networks, description logics, inheritance (including frames and scripts): - -
    25. -
  25. ^ Representing events and time:Situation calculus, event calculus, fluent calculus (including solving the frame problem): - -
    26. -
  26. ^ -Causal calculus: - -
    27. -
  27. ^ -Representing knowledge about knowledge: Belief calculus, modal logics: - -
    28. -
  28. ^ a b -Default reasoning, Frame problem, default logic, non-monotonic logics, circumscription, closed world assumption, abduction: - -(Poole et al. places abduction under "default reasoning". Luger et al. places this under "uncertain reasoning"). -
    29. -
  29. ^ a b -Breadth of commonsense knowledge: - -
    30. -
  30. ^ Newquist (1994), p. 296. -
    31. -
  31. ^ Crevier (1993), pp. 204–208. -
    32. -
  32. ^ Russell & Norvig (2021), p. 528. -
    33. -
  33. ^ -Automated planning: - - -
    34. -
  34. ^ -Automated decision making, Decision theory: - - -
    35. -
  35. ^ -Classical planning: - - -
    36. -
  36. ^ -Sensorless or "conformant" planning, contingent planning, replanning (a.k.a online planning): - - -
    37. -
  37. ^ -Uncertain preferences: - -Inverse reinforcement learning: - -
    38. -
  38. ^ -Information value theory: - - -
    39. -
  39. ^ -Markov decision process: - - -
    40. -
  40. ^ -Game theory and multi-agent decision theory: - - -
    41. -
  41. ^ -Learning: - -
    42. -
  42. ^ Turing (1950). -
    43. -
  43. ^ Solomonoff (1956). -
    44. -
  44. ^ -Unsupervised learning: - -
    45. -
  45. ^ a b -Supervised learning: - -
    46. -
  46. ^ -Reinforcement learning: - -
    47. -
  47. ^ -Transfer learning: - -
    48. -
  48. ^ "Artificial Intelligence (AI): What Is AI and How Does It Work? | Built In". builtin.com. Retrieved 30 October 2023. -
    49. -
  49. ^ -Computational learning theory: - -
    50. -
  50. ^ -Natural language processing (NLP): - -
    51. -
  51. ^ -Subproblems of NLP: - -
    52. -
  52. ^ Russell & Norvig (2021), p. 856–858. -
    53. -
  53. ^ Dickson (2022). -
    54. -
  54. ^ Modern statistical and deep learning approaches to NLP: - -
    55. -
  55. ^ Vincent (2019). -
    56. -
  56. ^ Russell & Norvig (2021), p. 875–878. -
    57. -
  57. ^ Bushwick (2023). -
    58. -
  58. ^ -Computer vision: - -
    59. -
  59. ^ Russell & Norvig (2021), pp. 849–850. -
    60. -
  60. ^ Russell & Norvig (2021), pp. 895–899. -
    61. -
  61. ^ Russell & Norvig (2021), pp. 899–901. -
    62. -
  62. ^ Russell & Norvig (2021), pp. 931–938. -
    63. -
  63. ^ MIT AIL (2014). -
    64. -
  64. ^ -Affective computing: - -
    65. -
  65. ^ Waddell (2018). -
    66. -
  66. ^ Poria et al. (2017). -
    67. -
  67. ^ -Search algorithms: - -
    68. -
  68. ^ -State space search: - -
    69. -
  69. ^ Russell & Norvig (2021), §11.2. -
    70. -
  70. ^ Uninformed searches (breadth first search, depth-first search and general state space search): - -
    71. -
  71. ^ -Heuristic or informed searches (e.g., greedy best first and A*): - - -
    72. -
  72. ^ -Adversarial search: - -
    73. -
  73. ^ Local or "optimization" search: - - -
    74. -
  74. ^ Singh Chauhan, Nagesh (18 December 2020). "Optimization Algorithms in Neural Networks". KDnuggets. Retrieved 13 January 2024. -
    75. -
  75. ^ -Evolutionary computation: - -
    76. -
  76. ^ Merkle & Middendorf (2013). -
    77. -
  77. ^ -Logic: - -
    78. -
  78. ^ -Propositional logic: - -
    79. -
  79. ^ -First-order logic and features such as equality: - -
    80. -
  80. ^ -Logical inference: - -
    81. -
  81. ^ Russell & Norvig (2021), §8.3.1. -
    82. -
  82. ^ -Resolution and unification: - -
    83. -
  83. ^ Forward chaining, backward chaining, Horn clauses, and logical deduction as search: - -
    84. -
  84. ^ -Fuzzy logic: - -
    85. -
  85. ^ a b -Stochastic methods for uncertain reasoning: - -
    86. -
  86. ^ -Bayesian networks: - -
    87. -
  87. ^ Domingos (2015), chapter 6. -
    88. -
  88. ^ -Bayesian inference algorithm: - -
    89. -
  89. ^ Domingos (2015), p. 210. -
    90. -
  90. ^ -Bayesian learning and the expectation-maximization algorithm: - -
    91. -
  91. ^ Bayesian decision theory and Bayesian decision networks: - -
    92. -
  92. ^ a b c -Stochastic temporal models: - -Hidden Markov model: - -Kalman filters: - -Dynamic Bayesian networks: - -
    93. -
  93. ^ -decision theory and decision analysis: - -
    94. -
  94. ^ -Information value theory: - -
    95. -
  95. ^ Markov decision processes and dynamic decision networks: - -
    96. -
  96. ^ Game theory and mechanism design: - -
    97. -
  97. ^ -Statistical learning methods and classifiers: - -
    98. -
  98. ^ -Decision trees: - -
    99. -
  99. ^ -Non-parameteric learning models such as K-nearest neighbor and support vector machines: - -
    100. -
  100. ^ Domingos (2015), p. 152. -
    101. -
  101. ^ -Naive Bayes classifier: - -
    102. -
  102. ^ a b -Neural networks: - -
    103. -
  103. ^ -Gradient calculation in computational graphs, backpropagation, automatic differentiation: - -
    104. -
  104. ^ -Universal approximation theorem: - -The theorem: - -
    105. -
  105. ^ -Feedforward neural networks: - -
    106. -
  106. ^ -Recurrent neural networks: - -
    107. -
  107. ^ -Perceptrons: - -
    108. -
  108. ^ a b -Deep learning: - -
    109. -
  109. ^ -Convolutional neural networks: - -
    110. -
  110. ^ Deng & Yu (2014), pp. 199–200. -
    111. -
  111. ^ Ciresan, Meier & Schmidhuber (2012). -
    112. -
  112. ^ Russell & Norvig (2021), p. 751. -
    113. -
  113. ^ a b c d e f g Russell & Norvig (2021), p. 785. -
    114. -
  114. ^ a b Schmidhuber (2022), §5. -
    115. -
  115. ^ a b Schmidhuber (2022), §6. -
    116. -
  116. ^ a b c Schmidhuber (2022), §7. -
    117. -
  117. ^ Schmidhuber (2022), §8. -
    118. -
  118. ^ Schmidhuber (2022), §2. -
    119. -
  119. ^ Schmidhuber (2022), §3. -
    120. -
  120. ^ Quoted in Christian (2020, p. 22) -
    121. -
  121. ^ Smith (2023). -
    122. -
  122. ^ "Explained: Generative AI". 9 November 2023. -
    123. -
  123. ^ "AI Writing and Content Creation Tools". MIT Sloan Teaching & Learning Technologies. Retrieved 25 December 2023. -
    124. -
  124. ^ Marmouyet (2023). -
    125. -
  125. ^ Kobielus (2019). -
    126. -
  126. ^ Davenport, T; Kalakota, R (June 2019). "The potential for artificial intelligence in healthcare". Future Healthc J. 6 (2): 94–98. doi:10.7861/futurehosp.6-2-94. PMC 6616181. PMID 31363513. -
    127. -
  127. ^ a b Bax, Monique; Thorpe, Jordan; Romanov, Valentin (December 2023). "The future of personalized cardiovascular medicine demands 3D and 4D printing, stem cells, and artificial intelligence". Frontiers in Sensors. 4. doi:10.3389/fsens.2023.1294721. ISSN 2673-5067. -
    128. -
  128. ^ Jumper, J; Evans, R; Pritzel, A (2021). "Highly accurate protein structure prediction with AlphaFold". Nature. 596 (7873): 583–589. Bibcode:2021Natur.596..583J. doi:10.1038/s41586-021-03819-2. PMC 8371605. PMID 34265844. -
    129. -
  129. ^ "AI discovers new class of antibiotics to kill drug-resistant bacteria". 20 December 2023. -
    130. -
  130. ^ Grant, Eugene F.; Lardner, Rex (25 July 1952). "The Talk of the Town – It". The New Yorker. ISSN 0028-792X. Retrieved 28 January 2024. -
    131. -
  131. ^ Anderson, Mark Robert (11 May 2017). "Twenty years on from Deep Blue vs Kasparov: how a chess match started the big data revolution". The Conversation. Retrieved 28 January 2024. -
    132. -
  132. ^ Markoff, John (16 February 2011). "Computer Wins on 'Jeopardy!': Trivial, It's Not". The New York Times. ISSN 0362-4331. Retrieved 28 January 2024. -
    133. -
  133. ^ Byford, Sam (27 May 2017). "AlphaGo retires from competitive Go after defeating world number one 3-0". The Verge. Retrieved 28 January 2024. -
    134. -
  134. ^ Brown, Noam; Sandholm, Tuomas (30 August 2019). "Superhuman AI for multiplayer poker". Science. 365 (6456): 885–890. doi:10.1126/science.aay2400. ISSN 0036-8075. -
    135. -
  135. ^ "MuZero: Mastering Go, chess, shogi and Atari without rules". Google DeepMind. 23 December 2020. Retrieved 28 January 2024. -
    136. -
  136. ^ Sample, Ian (30 October 2019). "AI becomes grandmaster in 'fiendishly complex' StarCraft II". The Guardian. ISSN 0261-3077. Retrieved 28 January 2024. -
    137. -
  137. ^ Wurman, P.R.; Barrett, S.; Kawamoto, K. (2022). "Outracing champion Gran Turismo drivers with deep reinforcement learning". Nature 602. 602 (7896): 223–228. doi:10.1038/s41586-021-04357-7. -
    138. -
  138. ^ Knight, Will. "The US and 30 Other Nations Agree to Set Guardrails for Military AI". Wired. ISSN 1059-1028. Retrieved 24 January 2024. -
    139. -
  139. ^ Marcelline, Marco (27 May 2023). "ChatGPT: Most Americans Know About It, But Few Actually Use the AI Chatbot". PCMag. Retrieved 28 January 2024. -
    140. -
  140. ^ Lu, Donna (31 March 2023). "Misinformation, mistakes and the Pope in a puffer: what rapidly evolving AI can – and can't – do". The Guardian. ISSN 0261-3077. Retrieved 28 January 2024. -
    141. -
  141. ^ "Fake image of Pentagon explosion causes real dip on Wall Street". euronews. 23 May 2023. Retrieved 28 January 2024. -
    142. -
  142. ^ Ransbotham, Sam; Kiron, David; Gerbert, Philipp; Reeves, Martin (6 September 2017). "Reshaping Business With Artificial Intelligence". MIT Sloan Management Review. -
    143. -
  143. ^ Simonite (2016). -
    144. -
  144. ^ Russell & Norvig (2021), p. 987. -
    145. -
  145. ^ Laskowski (2023). -
    146. -
  146. ^ GAO (2022). -
    147. -
  147. ^ Valinsky (2019). -
    148. -
  148. ^ Russell & Norvig (2021), p. 991. -
    149. -
  149. ^ Russell & Norvig (2021), p. 991–992. -
    150. -
  150. ^ Christian (2020), p. 63. -
    151. -
  151. ^ Vincent (2022). -
    152. -
  152. ^ Reisner (2023). -
    153. -
  153. ^ Alter & Harris (2023). -
    154. -
  154. ^ Nicas (2018). -
    155. -
  155. ^ "Trust and Distrust in America". 22 July 2019. -
    156. -
  156. ^ Williams (2023). -
    157. -
  157. ^ Taylor & Hern (2023). -
    158. -
  158. ^ a b Rose (2023). -
    159. -
  159. ^ CNA (2019). -
    160. -
  160. ^ Goffrey (2008), p. 17. -
    161. -
  161. ^ Berdahl et al. (2023); Goffrey (2008, p. 17); Rose (2023); Russell & Norvig (2021, p. 995) -
    162. -
  162. ^ -Algorithmic bias and Fairness (machine learning): - -
    163. -
  163. ^ Christian (2020), p. 25. -
    164. -
  164. ^ a b Russell & Norvig (2021), p. 995. -
    165. -
  165. ^ Grant & Hill (2023). -
    166. -
  166. ^ Larson & Angwin (2016). -
    167. -
  167. ^ Christian (2020), p. 67–70. -
    168. -
  168. ^ Christian (2020, pp. 67–70); Russell & Norvig (2021, pp. 993–994) -
    169. -
  169. ^ Russell & Norvig (2021, p. 995); Lipartito (2011, p. 36); Goodman & Flaxman (2017, p. 6); Christian (2020, pp. 39–40, 65) -
    170. -
  170. ^ Quoted in Christian (2020, p. 65). -
    171. -
  171. ^ Russell & Norvig (2021, p. 994); Christian (2020, pp. 40, 80–81) -
    172. -
  172. ^ Quoted in Christian (2020, p. 80) -
    173. -
  173. ^ Dockrill (2022). -
    174. -
  174. ^ Sample (2017). -
    175. -
  175. ^ "Black Box AI". 16 June 2023. -
    176. -
  176. ^ Christian (2020), p. 110. -
    177. -
  177. ^ Christian (2020), pp. 88–91. -
    178. -
  178. ^ Christian (2020, p. 83); Russell & Norvig (2021, p. 997) -
    179. -
  179. ^ Christian (2020), p. 91. -
    180. -
  180. ^ Christian (2020), p. 83. -
    181. -
  181. ^ Verma (2021). -
    182. -
  182. ^ Rothman (2020). -
    183. -
  183. ^ Christian (2020), p. 105-108. -
    184. -
  184. ^ Christian (2020), pp. 108–112. -
    185. -
  185. ^ Russell & Norvig (2021), p. 989. -
    186. -
  186. ^ Robitzski (2018); Sainato (2015) -
    187. -
  187. ^ Russell & Norvig (2021), p. 987-990. -
    188. -
  188. ^ Russell & Norvig (2021), p. 988. -
    189. -
  189. ^ Harari (2018). -
    190. -
  190. ^ Buckley, Chris; Mozur, Paul (22 May 2019). "How China Uses High-Tech Surveillance to Subdue Minorities". The New York Times. -
    191. -
  191. ^ "Security lapse exposed a Chinese smart city surveillance system". 3 May 2019. Archived from the original on 7 March 2021. Retrieved 14 September 2020. -
    192. -
  192. ^ "AI traffic signals to be installed in Bengaluru soon". NextBigWhat. 24 September 2019. Retrieved 1 October 2019. -
    193. -
  193. ^ Urbina et al. (2022). -
    194. -
  194. ^ Tarnoff, Ben (4 August 2023). "Lessons from Eliza". The Guardian Weekly. pp. 34–9. -
    195. -
  195. ^ a b E McGaughey, 'Will Robots Automate Your Job Away? Full Employment, Basic Income, and Economic Democracy' (2022) 51(3) Industrial Law Journal 511–559 Archived 27 May 2023 at the Wayback Machine -
    196. -
  196. ^ Ford & Colvin (2015);McGaughey (2022) -
    197. -
  197. ^ IGM Chicago (2017). -
    198. -
  198. ^ Arntz, Gregory & Zierahn (2016), p. 33. -
    199. -
  199. ^ Lohr (2017); Frey & Osborne (2017); Arntz, Gregory & Zierahn (2016, p. 33) -
    200. -
  200. ^ Morgenstern (2015). -
    201. -
  201. ^ Mahdawi (2017); Thompson (2014) -
    202. -
  202. ^ Zhou, Viola (11 April 2023). "AI is already taking video game illustrators' jobs in China". Rest of World. Retrieved 17 August 2023. -
    203. -
  203. ^ Carter, Justin (11 April 2023). "China's game art industry reportedly decimated by growing AI use". Game Developer. Retrieved 17 August 2023. -
    204. -
  204. ^ Cellan-Jones (2014). -
    205. -
  205. ^ Russell & Norvig 2021, p. 1001. -
    206. -
  206. ^ Bostrom (2014). -
    207. -
  207. ^ Russell (2019). -
    208. -
  208. ^ Bostrom (2014); Müller & Bostrom (2014); Bostrom (2015). -
    209. -
  209. ^ Harari (2023). -
    210. -
  210. ^ Müller & Bostrom (2014). -
    211. -
  211. ^ -Leaders' concerns about the existential risks of AI around 2015: - -
    212. -
  212. ^ -Arguments that AI is not an imminent risk: - -
    213. -
  213. ^ a b Christian (2020), pp. 67, 73. -
    214. -
  214. ^ Valance (2023). -
    215. -
  215. ^ Taylor, Josh (7 May 2023). "Rise of artificial intelligence is inevitable but should not be feared, 'father of AI' says". The Guardian. Retrieved 26 May 2023. -
    216. -
  216. ^ Colton, Emma (7 May 2023). "'Father of AI' says tech fears misplaced: 'You cannot stop it'". Fox News. Retrieved 26 May 2023. -
    217. -
  217. ^ Jones, Hessie (23 May 2023). "Juergen Schmidhuber, Renowned 'Father Of Modern AI,' Says His Life's Work Won't Lead To Dystopia". Forbes. Retrieved 26 May 2023. -
    218. -
  218. ^ McMorrow, Ryan (19 December 2023). "Andrew Ng: 'Do we think the world is better off with more or less intelligence?'". Financial Times. Retrieved 30 December 2023. -
    219. -
  219. ^ Levy, Steven (22 December 2023). "How Not to Be Stupid About AI, With Yann LeCun". Wired. Retrieved 30 December 2023. -
    220. -
  220. ^ Yudkowsky (2008). -
    221. -
  221. ^ a b Anderson & Anderson (2011). -
    222. -
  222. ^ AAAI (2014). -
    223. -
  223. ^ Wallach (2010). -
    224. -
  224. ^ Russell (2019), p. 173. -
    225. -
  225. ^ Alan Turing Institute (2019). "Understanding artificial intelligence ethics and safety" (PDF). -
    226. -
  226. ^ Alan Turing Institute (2023). "AI Ethics and Governance in Practice" (PDF). -
    227. -
  227. ^ Floridi, Luciano; Cowls, Josh (23 June 2019). "A Unified Framework of Five Principles for AI in Society". Harvard Data Science Review. 1 (1). doi:10.1162/99608f92.8cd550d1. S2CID 198775713. -
    228. -
  228. ^ Buruk, Banu; Ekmekci, Perihan Elif; Arda, Berna (1 September 2020). "A critical perspective on guidelines for responsible and trustworthy artificial intelligence". Medicine, Health Care and Philosophy. 23 (3): 387–399. doi:10.1007/s11019-020-09948-1. ISSN 1572-8633. PMID 32236794. S2CID 214766800. -
    229. -
  229. ^ Kamila, Manoj Kumar; Jasrotia, Sahil Singh (1 January 2023). "Ethical issues in the development of artificial intelligence: recognizing the risks". International Journal of Ethics and Systems. ahead-of-print (ahead-of-print). doi:10.1108/IJOES-05-2023-0107. ISSN 2514-9369. S2CID 259614124. -
    230. -
  230. ^ -Regulation of AI to mitigate risks: - -
    231. - -
  231. ^ a b Vincent (2023). -
    232. -
  232. ^ Stanford University (2023). -
    233. -
  233. ^ a b c d UNESCO (2021). -
    234. -
  234. ^ Kissinger (2021). -
    235. -
  235. ^ Altman, Brockman & Sutskever (2023). -
    236. -
  236. ^ VOA News (25 October 2023). "UN Announces Advisory Body on Artificial Intelligence". -
    237. -
  237. ^ Edwards (2023). -
    238. -
  238. ^ Kasperowicz (2023). -
    239. -
  239. ^ Fox News (2023). -
    240. -
  240. ^ Milmo, Dan (3 November 2023). "Hope or Horror? The great AI debate dividing its pioneers". The Guardian Weekly. pp. 10–12. -
    241. -
  241. ^ "The Bletchley Declaration by Countries Attending the AI Safety Summit, 1-2 November 2023". GOV.UK. 1 November 2023. Archived from the original on 1 November 2023. Retrieved 2 November 2023. -
    242. -
  242. ^ "Countries agree to safe and responsible development of frontier AI in landmark Bletchley Declaration". GOV.UK (Press release). Archived from the original on 1 November 2023. Retrieved 1 November 2023. -
    243. -
  243. ^ Berlinski (2000). -
    244. -
  244. ^ "Google books ngram". -
    245. -
  245. ^ -AI's immediate precursors: - -
    246. -
  246. ^ a b Russell & Norvig (2021), p. 17. -
    247. -
  247. ^ See "A Brief History of Computing" at AlanTuring.net. -
    248. -
  248. ^ Crevier (1993), pp. 47–49. -
    249. -
  249. ^ Russell & Norvig (2003), p. 17. -
    250. -
  250. ^ Russell & Norvig (2003), p. 18. -
    251. -
  251. ^ Russell & Norvig (2021), p. 21. -
    252. -
  252. ^ Lighthill (1973). -
    253. -
  253. ^ Russell & Norvig (2021), p. 22. -
    254. -
  254. ^ -Expert systems: - -
    255. -
  255. ^ Russell & Norvig (2021), p. 24. -
    256. -
  256. ^ Nilsson (1998), p. 7. -
    257. -
  257. ^ McCorduck (2004), pp. 454–462. -
    258. -
  258. ^ Moravec (1988). -
    259. -
  259. ^ a b Brooks (1990). -
    260. -
  260. ^ -Developmental robotics: - -
    261. -
  261. ^ Russell & Norvig (2021), p. 25. -
    262. -
  262. ^ - -
    263. -
  263. ^ Russell & Norvig (2021), p. 26. -
    264. -
  264. ^ -Formal and narrow methods adopted in the 1990s: - -
    265. -
  265. ^ -AI widely used in the late 1990s: - -
    266. -
  266. ^ Wong (2023). -
    267. -
  267. ^ -Moore's Law and AI: - -
    268. -
  268. ^ a b c Clark (2015b). -
    269. -
  269. ^ -Big data: - -
    270. -
  270. ^ "Intellectual Property and Frontier Technologies". WIPO. Archived from the original on 2 April 2022. Retrieved 30 March 2022. -
    271. -
  271. ^ DiFeliciantonio (2023). -
    272. -
  272. ^ Goswami (2023). -
    273. -
  273. ^ a b Turing (1950), p. 1. -
    274. -
  274. ^ -Turing's original publication of the Turing test in "Computing machinery and intelligence": - -Historical influence and philosophical implications: - -
    275. -
  275. ^ Turing (1950), Under "The Argument from Consciousness". -
    276. -
  276. ^ Russell & Norvig (2021), chpt. 2. -
    277. -
  277. ^ Russell & Norvig (2021), p. 3. -
    278. -
  278. ^ Maker (2006). -
    279. -
  279. ^ McCarthy (1999). -
    280. -
  280. ^ Minsky (1986). -
    281. -
  281. ^ "What Is Artificial Intelligence (AI)?". Google Cloud Platform. Archived from the original on 31 July 2023. Retrieved 16 October 2023. -
    282. -
  282. ^ Nilsson (1983), p. 10. -
    283. -
  283. ^ Haugeland (1985), pp. 112–117. -
    284. -
  284. ^ -Physical symbol system hypothesis: - -Historical significance: - -
    285. -
  285. ^ -Moravec's paradox: - -
    286. -
  286. ^ -Dreyfus' critique of AI: - -Historical significance and philosophical implications: - -
    287. -
  287. ^ Crevier (1993), p. 125. -
    288. -
  288. ^ Langley (2011). -
    289. -
  289. ^ Katz (2012). -
    290. -
  290. ^ -Neats vs. scruffies, the historic debate: - -A classic example of the "scruffy" approach to intelligence: - -A modern example of neat AI and its aspirations in the 21st century: - -
    291. -
  291. ^ Pennachin & Goertzel (2007). -
    292. -
  292. ^ a b Roberts (2016). -
    293. -
  293. ^ Russell & Norvig (2021), p. 986. -
    294. -
  294. ^ Chalmers (1995). -
    295. -
  295. ^ Dennett (1991). -
    296. -
  296. ^ Horst (2005). -
    297. -
  297. ^ Searle (1999). -
    298. -
  298. ^ Searle (1980), p. 1. -
    299. -
  299. ^ Russell & Norvig (2021), p. 9817. -
    300. -
  300. ^ -Searle's Chinese room argument: - -Discussion: - -
    301. -
  301. ^ -Robot rights: - -
    302. -
  302. ^ Evans (2015). -
    303. -
  303. ^ McCorduck (2004), pp. 19–25. -
    304. -
  304. ^ Henderson (2007). -
    305. -
  305. ^ -The Intelligence explosion and technological singularity: - -I. J. Good's "intelligence explosion" - -Vernor Vinge's "singularity" - -
    306. -
  306. ^ Russell & Norvig (2021), p. 1005. -
    307. -
  307. ^ -Transhumanism: - -
    308. -
  308. ^ -AI as evolution: - -
    309. -
  309. ^ -AI in myth: - -
    310. -
  310. ^ McCorduck (2004), pp. 340–400. -
    311. -
  311. ^ Buttazzo (2001). -
    312. -
  312. ^ Anderson (2008). -
    313. -
  313. ^ McCauley (2007). -
    314. -
  314. ^ Galvan (1997). -
    315. -
-

AI textbooks

-

The two most widely used textbooks in 2023. (See the Open Syllabus). -

- -

These were the four of the most widely used AI textbooks in 2008: -

-
- -

Later editions. -

- -
-

History of AI

-
- -
-

Other sources

-
- -
-

Further reading

-
- -
-

External links

- - -
-
- - - - -
-
Retrieved from "https://en.wikipedia.org/w/index.php?title=Artificial_intelligence&oldid=1202857005"
- -
-
- -
-
- - - -
-
-
-
-
    -
  • - -
    • -
-
- - - - \ No newline at end of file diff --git a/inputDocs/Norwegian_Literature.html b/inputDocs/Norwegian_Literature.html deleted file mode 100644 index a814b8fea0ad2a5b2b96fa7bb7b652b33729727a..0000000000000000000000000000000000000000 --- a/inputDocs/Norwegian_Literature.html +++ /dev/null @@ -1,1366 +0,0 @@ - - - - -Norwegian literature - Wikipedia - - - - - - - - - - - - - - - - - - - - - - - - - - - -Jump to content -
-
-
- - - - -
-
- - - - - -
-
-
-
-
-
-
-
-
-
-
- -
-
-
- -
-
-
-
-
- -

Norwegian literature

- - -
-
-
-
- -
-
- - - -
-
-
-
-
- - -
-
-
-
-
-
- -
From Wikipedia, the free encyclopedia
-
-
- - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-

Norwegian literature -

-
-

By category
-Norwegian language
-List of writers -

-
Norwegian authors -
-

Writers - -Novelists
-Playwrights - -Poets
-Essayists -

-
Forms -
-

Novel - Poetry - Plays
-

-
Genres -
-

Science Fiction
-

-
Criticism & Awards -
-

Literary theory - -Critics
-Literary Prizes -

-
Most visited -
-

Ibsen - Vesaas - -Hamsun
-Collett - Bjørnson
-Wergeland - -Dag Solstad
-Jon Fosse - -Sigrid Undset
-

-
Norway Portal -
Literature Portal -
- - - -
Part of a series on
Norwegians
-Culture
-
-Diaspora
-
-Other
-
-Norwegian Portal
-

Norwegian literature is literature composed in Norway or by Norwegian people. The history of Norwegian literature starts with the pagan Eddaic poems and skaldic verse of the 9th and 10th centuries with poets such as Bragi Boddason and Eyvindr Skáldaspillir. The arrival of Christianity around the year 1000 brought Norway into contact with European medieval learning, hagiography and history writing. Merged with native oral tradition and Icelandic influence, this was to flower into an active period of literature production in the late 12th and early 13th centuries. Major works of that period include Historia Norwegie, Thidreks saga and Konungs skuggsjá. -

The period from the 14th century to the 19th is considered a Dark Age in the nation's literature though Norwegian-born writers such as Peder Claussøn Friis, Dorothe Engelbretsdatter and Ludvig Holberg contributed to the common literature of Denmark–Norway. With the advent of nationalism and the struggle for independence in the early 19th century, a new period of national literature emerged. In a flood of nationalistic romanticism, the great four emerged: Henrik Ibsen, Bjørnstjerne Bjørnson, Alexander Kielland, and Jonas Lie. The dramatist Henrik Wergeland was the most-influential author of the period while the later works of Henrik Ibsen were to earn Norway a key place in Western European literature. -

Modernist literature was introduced to Norway through the literature of Knut Hamsun and Sigbjørn Obstfelder in the 1890s. In the 1930s Emil Boyson, Gunnar Larsen, Haakon Bugge Mahrt, Rolf Stenersen and Edith Øberg were among the Norwegian authors who experimented with prose modernism. The literature in the first years after the Second World War was characterized by a long series of documentary reports from people who had been in German custody, or who had participated in the resistance efforts during the occupation. In the 20th century notable Norwegian writers include the two Nobel Prize-winning authors, Knut Hamsun and Sigrid Undset. The period after 1965 represented a sharp expansion of market for Norwegian fiction and the 1970s produced both politicization and empowerment of Norwegian authors. The 1980s has been labeled the "fantasy decade" in Norwegian literature. -

- -

Medieval poetry[edit]

-
See also: Old Norse poetry
-

The earliest preserved examples of Old Norse literature are the Eddic poems, the oldest of which may have been composed in early 9th century Norway drawing on the common Germanic tradition of alliterative verse. In the 9th century the first instances of skaldic poetry also appear with the skalds Bragi Boddason, Þjóðólfr of Hvinir and the court poets of Harald Fairhair. This tradition continued through the 10th century with the major Norwegian poet being Eyvindr skáldaspillir. By the late 10th century the tradition of skaldic verse had increasingly moved to Iceland and Norwegian rulers such as Eiríkr Hákonarson and St. Olaf employed mostly Icelandic poets. -

-

Medieval prose[edit]

-
See also: Old Norse literature
-

In pagan times the runic alphabet was the only one used in Norway. The preserved inscriptions from that time are mostly short memorial dedications or magical formulas. One of the longest inscriptions is that on the 8th century Eggjum stone, containing cryptic religious or magical allusions. Around the years 1000 to 1030, Christianity became established in Norway, bringing with it the Latin alphabet. The oldest preserved Norwegian prose works are from the mid-12th century, the earliest are Latin hagiographical and historical texts such as Passio Olavi, Acta sanctorum in Selio, Historia Norwegie and Historia de Antiquitate Regum Norwagiensium. At the end of the 12th century, historical writing expanded to the vernacular with Ágrip af Nóregskonungasögum followed by the Legendary Saga of St. Olaf and Fagrskinna. -

Medieval Norwegian literature is closely tied with medieval Icelandic literature, and together, they are considered Old Norse literature. The greatest Norse author of the 13th century was the Icelander Snorri Sturluson. He recorded Norse mythology in the form of the Prose Edda, a book of poetic language providing an important understanding of Norse culture prior to Christianity. He was also the author of the Heimskringla, a detailed history of the Norwegian kings that begins in the legendary Ynglinga saga and continues to document much of early Norwegian history. -

The period of common Old Norse literature continued up through the 13th century with Norwegian contributions such as Thidreks saga and Konungs skuggsjá but by the 14th century saga writing was no longer cultivated in Norway and Icelandic literature became increasingly isolated. -

-

"Four Hundred Years of Darkness"[edit]

- - - - -
Reformation-era literature
-
Overview
-
-
British
- -
- -
- -


-

-
-
  • Pastoral
    • -
  • Morality
    • -
  • History
    • -
  • Tragedy
    • -
  • Revenge
    • -
    -
    Continental
    - -
    -

    Dutch Renaissance and Golden Age -

    -Folklore of the Low Countries
    -
    Scandinavian
    - -
    -

    Norwegian literature was virtually nonexistent during the period of the Scandinavian Union and the subsequent Dano-Norwegian union (1387–1814). Ibsen characterized this period as "Four Hundred Years of Darkness". During the period of union with Denmark, Danish replaced Norwegian. The university and cultural center of Denmark–Norway was Copenhagen, where young men went to study.[1] -

    The reformation was imposed on Norway in 1537 and the Dano-Norwegian rulers used it to also impose Danish culture; this was effected through the pulpit as well as through written records, as pastors were trained in Copenhagen. Thus, written Norwegian became closely related to Danish, causing the literature to become essentially Danish. Geble Pedersson (c.1490–1557) was the first Lutheran Bishop of Bergen and a man of broad humanistic views; his adopted son, Absalon Pederssøn Beyer (1528–1575), followed in his footsteps as a humanist and a nationalist, writing an important historical work, Concerning the Kingdom of Norway (1567). Peder Claussøn Friis (1545–1615) was also a humanist who both revived the Heimskringla by translating it into the language of the period and wrote the first natural history of Norway as well as an important topographic study of Norway.[1] -

    The seventeenth century was a period of meager literary activity in Norway, but there were significant contributions. Petter Dass (1647–1707) wrote Nordlands Trompet (The Trumpet of Nordland) which described in graphic verse the landscape, mode of life, conditions and character of the northern Norwegian people. Two other authors merit mention. Dorothe Engelbretsdotter (1634–1713), was Norway's first recognized woman author who wrote powerful religious poetry. Her first work, Siælens Sang-offer, was published in 1678. Taare-Offer was her second collected works and was published for the first time in 1685. Another gifted poet was Anders Arrebo who translated the Psalms into Norwegian and composed the creation poem, Hexaemeron.[1] -

    Norway also contributed significantly to the joint literature of Denmark–Norway. One of the first names in Danish literature, Peder Claussøn Friis (1545–1614), was Norwegian-born. Other important Norwegian by birth 'Danish' authors of the period included Ludvig Holberg (Bergen, 1684–1754), Christian Tullin (Christiania, 1728–1765), and Johan Herman Wessel (1742–1785).[1] -

    -

    Rebirth[edit]

    -

    Two major events precipitated a major resurgence in Norwegian literature. In 1811, a Norwegian university was established in Christiania (later renamed Oslo). Seized by the spirit of revolution following the American and French Revolutions, as well as bridling as a result of the forced separation from Denmark and subordination to Sweden subsequent to the Napoleonic wars, Norwegians signed their first constitution in 1814. Virtually immediately, the cultural backwater that was Norway brought forth a series of strong authors recognized first in Scandinavia, and then worldwide. -

    Henrik Wergeland is generally recognized as the father of a new Norwegian literature. The enthusiastic nationalism of Wergeland and his young following brought conflict with the establishment, which was unwilling to accept everything as good, simply because it was Norwegian. -

    This period also saw collection of Norwegian folk tales by Peter Asbjørnsen and Bishop Jørgen Moe. This collection, which paralleled those by the Brothers Grimm in Germany and Hans Christian Andersen in Denmark, captured an important overview of the folk culture of the mountains and fjords. -

    At least as important in the creation of a Norwegian literature was the effort to introduce a pure Norwegian language, based on the dialects spoken in the areas more isolated from capital. The genius of Ivar Aasen (1813–1898) was at the heart of this effort. Aasen, a self-taught linguistic scholar and philologist, documented a written grammar and dictionary for the spoken Norwegian folk language, which became Nynorsk (New Norwegian) – the "speech of the country" as opposed to the official language largely imported from Denmark. Nynorsk is one of the two official written norms of the Norwegian language to this day. -

    -

    National Romantic Period[edit]

    -

    By the late 19th century, in a flood of nationalistic romanticism, the great four emerged: Henrik Ibsen, Bjørnstjerne Bjørnson, Alexander Kielland, and Jonas Lie. A unity of purpose pervades the whole period, creation of a national culture based on the almost forgotten and certainly neglected past, as well as celebration of the bondekultur or Norwegian farm culture. The realism of Kielland (e.g., Skipper Worse) gave way to the romantic and nationalistic spirit which swept Europe and rekindled the Norwegian interest in their glorious Viking past (e.g., Ibsen's The Vikings at Helgeland), the struggles of the Middle Ages (e.g., Ibsen's Lady Inger of Østeraad), peasant stories (e.g., Bjørnson's A Happy Boy) and the wonders of myths and folks tales of the mountains (e.g., Ibsen's Peer Gynt) and the sea (e.g., Lie's The Visionary). -

    -

    Transition to Realism[edit]

    -

    Although a strong contributor to early Norwegian romanticism, Henrik Ibsen is perhaps best known as an influential Norwegian playwright who was largely responsible for the popularity of modern realistic drama in Europe, with plays like The Wild Duck and A Doll's House. In this, he built on a theme first evident in Norway with plays like Bjørnson's En fallit (A Bankruptcy). -

    -

    Emigration Literature[edit]

    -

    Although a side note to the mainstream of Norwegian literature, the literature which documents the experience of Norwegian emigrants to America is as important as the Norwegian immigrants became to the growing America of the 19th century. Three authors are recognized in this genre; Ole Rølvaag wrote about immigrants, while Johan Bojer and Ingeborg Refling Hagen wrote about emigrants. Ole E. Rølvaag, who immigrated to America, experienced life in the prairies, and rose to become professor of Norwegian at St. Olaf College in Northfield, Minnesota, provided a strong record of the joys and pains of the immigrant in adapting to the harsh realities of and carving out a new life in a wild new country. Norwegian author Johan Bojer provided a mirror image, depicting the struggles and processes which led to the decisions to emigrate. Ingeborg Refling Hagen, having two brothers and a sister in the United States contemplated the emigrant's longing for home and their harsh struggle "over there" in a known collection of emigrant poems from 1935. -

    -

    Modernism in Norway[edit]

    -

    Modernist literature was introduced to Norway through the literature of Knut Hamsun and Sigbjørn Obstfelder in the 1890s. In the 1930s, Emil Boyson, Gunnar Larsen, Haakon Bugge Mahrt, Rolf Stenersen and Edith Øberg were among the Norwegian authors who experimented with prose modernism. The books of the 1930s did not receive the same recognition as modernist works after the war. In 1947, Tarjei Vesaas published a poetry collection, Leiken og lynet, that led to major debate about the shape and rhythm for Norwegian poetry. This evolved further in the 1950s. Rolf Jacobsen achieved recognition as a poet of modernistic style after the war. Kristofer Uppdal was also recognized for his work. -

    -

    The Twentieth Century[edit]

    -

    After the death of the great four and Amalie Skram, a new period of Norwegian literature took place. The year 1905, when Norway was free from the union with Sweden, marks a new period in the history of Norwegian literature. In the 20th century, three Norwegian novelists won the Nobel prize in literature. The first was Bjørnstjerne Bjørnson, whose prize reflected work of the previous century. The second was awarded to Knut Hamsun for the idealistic novel Markens Grøde (Growth of the Soil, 1917) in 1920. The third was Sigrid Undset for the trilogy of Kristin Lavransdatter and the two books of Olav Audunssøn, in 1927. -

    Knut Hamsun was especially criticized because of his sympathy for Nasjonal Samling, a Norwegian Nazi-party, during the Second World War. -

    Other important Norwegian writers include: Trygve Gulbranssen, Jens Bjørneboe, Agnar Mykle, Olav Duun, Cora Sandel, Kjartan Fløgstad, Arne Garborg, Aksel Sandemose, Tarjei Vesaas, Lars Saabye Christensen, Kjell Askildsen, Johan Borgen, Dag Solstad, Herbjørg Wassmo, Jon Fosse, Hans Herbjørnsrud, Jan Erik Vold, Roy Jacobsen, Bergljot Hobæk Haff, Hans E. Kinck, Olav H. Hauge, Rolf Jacobsen, Gunvor Hofmo, Arnulf Øverland, Sigbjørn Obstfelder, Olaf Bull, Aasmund Olavsson Vinje, Tor Ulven, Torborg Nedreaas, Stein Mehren, Jan Kjærstad, Georg Johannesen, Kristofer Uppdal, Aslaug Vaa, Halldis Moren Vesaas, Sigurd Hoel, Johan Falkberget, Hans Børli and Axel Jensen. -

    -

    The Post-war Period (1945–1965)[edit]

    -

    The literature in the first years after the Second World War was characterized by a long series of documentary reports from people who had been in German custody, or who had participated in the resistance efforts during the occupation. The most famous among these were Lise Børsums's Fange i Ravensbrück, Odd Nansen's Fra dag til dag (From Day to Day) and the posthumously published Petter Moens dagbok (Petter Moen's diary). Some years later, biographies of heroes of resistance, such as Fridtjof Sælen's Shetlands-Larsen, about Leif Andreas Larsen, and David Armin Howarth's Ni liv. Historien om Jan Baalsrud (Nine Lives – the story of Jan Baalsrud), became major publishing successes. -

    Fiction of the period also centered on the war. Sigurd Evensmo's Englandsfarere (published in English as "A Boat for England") about a group of resistance fighters who are captured. Tarjei Vesaas symbolically addressed the war experience in Huset i mørkret (The House in the Dark). A significant portion of the post-war literature was concerned with the question of why some remained good Norwegian patriots while others, seemingly ordinary people, served the enemy. Examples of this include Sigurd Hoel's Møte ved milepelen from 1947, Kåre Holt's Det store veiskillet (The Big Fork) from 1949 and Aksel Sandemose's Varulven (The Werewolf) from 1958, which provide psychological explanations for collaboration. -

    Poetry written during the war, which had either been broadcast from London or had circulated illegally, was published as collections in the spring of 1945, and enjoyed a popularity that Norwegian poetry has not seen before or since. In particular Arnulf Øverland's Vi overlever alt (We survive everything) and Nordahl Grieg's Friheten (Freedom) were well received. Some of those who were young during the war found that the traditional lyrical forms were insufficient to express horrors of war, atomic bombs and the emerging Cold War. Gunvor Hofmo, who was personally affected by the war, came with the remarkable collections Jeg vil hjem til menneskene (I Want to Go Home to the People) and Fra en annen virkelighet (From an Alternate Reality). -

    Modernism appeared on a broad front in the Norwegian poetry of the 1950s. It impacted the lyrics produced by Tarjei Vesaas, Ernst Orvil, Astrid Tollefsen and Olav H. Hauge. Among the younger poets, such as Astrid Hjertenæs Andersen, Paal Brekke, Hans Børli, Harald Sverdrup and Marie Takvam, free verse was the preferred form. Paal Brekke was modernism's foremost advocate against traditionalists – such as Arnulf Øverland and André Bjerke – in a wide-ranging debate about poetic forms which is recognized as the speaking-in-tongues debate. Georg Johannesen's first publication Dikt (Poetry) in 1959 introduced a new interest in political and social values, that had not been particularly evident in the 1950s. At the same time, the well-established poet, Rolf Jacobsen, espoused a more critical attitude to the consumer mentality and environmental destruction. -

    In prose, first and foremost it was Jens Bjørneboe who led the attack on the establishment in the 1950s. In Jonas and Den onde hyrde (The Evil Shepherd) he attacks the school and prison systems, arguing that there the government shows its authoritarian aspects particularly clearly. One of the highlights of 1950s prose literature is Johan Borgen's Lillelord trilogy. Borgen' work is characterized by an experimental prose-writing style, which can be seen in several short story collections as well as the experimental novel Jeg (I) from 1959. Another highlight of 1950s literature was two controversial novels by Agnar Mykle's about Ask Burlefot: Lasso rundt fru Luna (published in English as "Lasso Around The Moon") and Sangen om den røde rubin (The Song of the Red Ruby). But as a result of legal intervention against the latter book, the pressure of the court case and surrounding controversy left Mykle a reclusive who published little thereafter. Axel Jensen was another fresh, new voice in the 1950s. In his debut novels Ikaros and Line the young protagonist comes to terms with nonsocialistic members of the Social Democratic welfare state. Jensen also introduced a new theme in Norwegian literature with the publication of Epp in 1965; this novel dealt with a future dystopia. -

    Besides Johan Borgen, Tarjei Vesaas and Torborg Nedreaas also achieved recognition as excellent short story writers. In 1953, Kjell Askildsen debuted with the short story collection Heretter følger jeg deg helt hjem (From now on I'll walk you all the way home). He has since remained at short prose genre, and is today considered one of Norwegian literature's finest short story writers. -

    -

    Political awareness and social realism (1965–1980)[edit]

    -

    The period after 1965 represented a sharp expansion of market for Norwegian fiction. In 1965, Norway instituted a policy for purchasing new literature. The state committed to purchase 1000 copies of each new title of Norwegian literature (conditioned on it meeting minimum standards). These were distributed among the country's libraries. This, combined with the creation of the book club Bokklubben Nye Bøker (New Books) in 1976 produced increased vitality in the country's literary production. -

    The 1970s produced both politicization and empowerment of Norwegian authors as a group – as well as intellectuals in general. The Norwegian Authors' Union became an arena for political struggle as well as the struggle for academic authors' rights. At one point the author's union split into two camps. Around the country the authors organized themselves in the regional author's organizations, and started a number of literary journals, in which contributions by amateur writers were welcomed. -

    Profil would eventually become the most-notable literary magazine. From 1965, it published the work of a number of young writers who would put their distinct mark on the literature of the period. The Profil goal was to bring Norwegian literature abreast of European literature in general. To achieve this, they rebelled against the traditional psychological novel development. The question of the true identify for the modern state was core. Dag Solstad contributed significantly to this late-1960s figures modernism through his articles, essays and literary works. -

    Poetry already exhibited a modernist style, which was prevalent through the 1950s and early 1960s. Traditionalists who still wrote in fixed stanza forms were out of favor. The younger poets targeted replacing the 1950s-style symbolism, and Jan Erik Vold was at the forefront of this insurgency. Profil poetry introduced a new simplicity, concretism, and use of everyday language. Paal Brekke was particularly noted for promoting modern European poetry, both as poet and critic. He argued for a renewal of Norwegian poetry, and spread knowledge of foreign literature through translations of English modernist writers like T.S.Eliot. In the mid-1950s, Brekke participated in the debate on lyrical form, and opposed André Bjerke and Arnulf Øverland in the so-called Glossolalia debate. Among the established lyrists, Olav H. Hauge transitioned to modernistic and concretist poetry and enjoyed a renaissance, especially with his collection entitled Dropar in austavind, which inspired other, younger Norwegian poets, such as Jan Erik Vold. -

    After a short period the Profil group went separate routes, as authors such as Dag Solstad, Espen Haavardsholm, and Tor Obrestad turned to the newly formed party Workers' Communist Party (Arbeidernes kommunistparti or AKP), and become involved in formulating a new political program that based on the view that literature should serve the working people and their uprising against capitalism. Arild Asnes Solstad's 1970 is a key novel to understanding the desire of the modern intellectual to connect with something larger and more realistic – the working people and a cause. -

    There were few AKP-authors, yet they managed to set a major part of the agenda for Norwegian fiction through much of the 1970s. Some authors began to write novels and poems in a language targeted so that people could recognize themselves, often known as social realism literature. Well-known works in this genre include Solstad's 25. septemberplassen, Obrestad's Sauda! Streik! and Haavardsholm's Historiens kraftlinjer. -

    Even though a minority wrote AKP-themed literature, there was a general willingness of the larger community of authors to support this literary focus. Besides the class struggle, there were two areas that were subject of serious literature: feminism and the struggle against the concentration of governmental power into a centralized government. -

    The term feminist literature or woman's literature was shifting during this period. While some believed that a special term for literature written for women by women about women's experiences were necessary, others were concerned that feminist literature served to place the female writers and readers outside the community, in an isolated cycle. Notwithstanding the debate, important contributions came from new, female authors about women unsatisfactory role in the family and in society. Liv Køltzow's Hvem bestemmer over Liv og Unni? (Who decides for Liv and Unni?) is central to understanding the new woman's literature. Bjørg Vik contributed a long series of short story collections and the play To akter for fem kvinner (Two acts for five women). Both Køltzow's and Vik's work stayed with the realistic tradition. Later Cecilie Løveid and Eldrid Lunden created work with a more rebellious language representing a fresh genre of experimental work. Løveid's work is notably committed to finding a new language for a new female role. -

    -

    Beyond social realism (1980–2000)[edit]

    -

    The decade of the 1980s was in many ways a response to the social realism in 1970s literature. In 1983, Kaj Skagen published a polemical-philosophical treatise titled Bazarovs barn ("Bazarov's Children", alluding to the Russian fictional nihilist Eugene Bazarov), which reconciled the role of authors who had been on the periphery in the 1970s. Skagen advocated for a more individual-oriented and idealistic literature. Although it is uncertain whether this book created or simplify reflected the transition, many of the 1970s authors shifted in new directions during the 1980s. Dag Solstad published two novels which were retrospectives on the Workers' Communist Party. Espen Haavardsholm wrote a novel titled Drift and Edvard Hoem authored Prøvetid. All these works focused on middle-aged men who live through the crises of life, while struggling to find new footing. Similarly Knut Faldbakken's novels about the change of men's roles during the women's revolution in the 1970s reflected the new direction. -

    The 1980s generated several major novels that develop a main theme over decades, are centered on a strong-central character person and are built around rural milieu or a local community of a not too distant past. Examples include Lars Saabye Christensen's Beatles, Tove Nilsen's Skyskraperengler (Skyscraper Angels), Ingvar Ambjørnsen's Hvite niggere (White Niggers), Gerd Brantenberg's St.Croix trilogy, Herbjørg Wassmo's Tora-trilogy and Roy Jacobsen's Seierherrene. -

    The 1980s have also been labeled the "fantasy decade" in Norwegian literature. A number of authors, including Kjartan Fløgstad, Mari Osmundsen, Hans Herbjørnsrud, Arild Nyquist, Jan Kjærstad and Ragnar Hovland produced works with magical, fantastic or improbable elements. Literature written for children and young people also included fantastic elements; Tormod Haugen is the most notable contributor to this genre. -

    A large number of 1980s authors displayed a high degree of literary consciousness. Many of the new authors in this decade were formally educated in literature, philosophy and other academic subjects at the many schools or institutes for writers established throughout Norway. Many novels generated internal conflicts with the text itself or with other texts, and the protagonists was represented as a writer, scientist or artist. Jan Kjærstad's Homo Falsus is perhaps the foremost of these 1980s meta-novels, Karin Moe's KYKA/1984 another. Ole Robert Sunde and Liv Nysted also produced works in this genre. Another consequence of more academically oriented authors was the large number of essay collections published in recent years; these often provide an authors' interpretations of other authors or reflections on other forms of art. -

    The period showed a rising interest in crime literature. Jon Michelet, Gunnar Staalesen, Kim Småge and Fredrik Skagen all were well appreciated by Norwegian readers. In the 1990s female crime writers such as Karin Fossum and Anne Holt had great success – the latter's works featured a female investigator. Interest in crime has in no way decreased since the turn of the millennium, and a number of writers have either specialized in crime or have alternated between crime and other prose. Jo Nesbø, Kurt Aust, Unni Lindell, Tom Egeland, Tom Kristensen, Jørn Lier Horst, Stein Morten Lier and Kjell Ola Dahl are among the authors in this category. A stream of translated crime, especially from Sweden and Britain, have influenced Norwegian authors of this genre. -

    Another clear trend is an interest in biographies, especially of authors and artists. Many of the significant living writers during the 1980s have written one or more biographies of deceased artist or other colleagues. In addition, several significant biographies were written. The Fall of the Sun God. Knut Hamsun by Jørgen Haugan and Ingar Sletten Kolloen's Knut Hamsun biography received great attention. There is a trend in these modern biographies – similar to today's cinema and unlike the past – to use source material of a private character. -

    In poetry Rolf Jacobsen's Nattåpent sold almost 20,000 copies and Harald Sverdrup's Lysets øyeblikk was also very well received. Stein Mehren, Tor Ulven and Paal-Helge Haugen also published significant collections of poetry during this decade. Jan Erik Vold wrote some of his most political poetry, reminiscent of the 1970s, during the 1990s. The new and emerging poetry shows great diversity. However, only the rare collection of poetry achieves substantial sales or circulation. Poetry can be said to be in a crisis state, unlike newer novels, which often are published in large quantities as the month's book for book clubs. -

    Theater audiences show only moderate interest in new Norwegian plays. Hence drama has been overshadowed by prose and poetry, with one exception: Jon Fosse. Fosse, through the 1990s and later, has achieved an international acclaim not enjoyed by any other Norwegian playwright since Ibsen. -

    -

    21st century[edit]

    -

    Karl Ove Knausgård had worldwide success with his six-volume series of autobiographical novels entitled My Struggle (Min kamp in Norwegian) and was described by Wall Street Journal as "one of the 21st-century's greatest literary sensations". Knausgård is also the author of novels (Ute av verden, En tid for alt, Morgenstjernen), the autobiographical The Seasons Quartet and essay collections.[2] -

    -

    Comics[edit]

    -

    Norway has a distinct comic strip and single-panel comic culture that it shares with Sweden. -

    Story-driven comics with local themes were popular in the postwar years, including Vangsgutane, Jens von Bustenskjold and Smørbukk. However, they gradually faded out of popularity, leading to several decades with no major locally produced comics (with the partial exception of Pyton). Starting in the early 1990s, a large number of strip comics were born from the local hype that had surrounded such US strip comics as Calvin & Hobbes, Piranha Club and Beetle Bailey. Significant names include Frode Øverli (Pondus, Rutetid), Lars Lauvik (Eon, Wildlife), Mads Eriksen (M), Lise Myhre (Nemi), Øyvind Sagosen (Radio Gaga), and the duo Emberland & Sveen (Sleivdal IL). -

    Starting out with a focus on slapstick comedy, Norwegian comic strips gradually focused more on relationships and family life from the late 2000s onwards, leading to the creation of additional comic strips made by names like Hanne Sigbjørnsen (Tegnehanne) and Nils Axle Kanten (Hjalmar). -

    -

    Electronic literature[edit]

    -

    Hans Kristian Rustad's book Digital litteratur (2012) provides an overview of early Norwegian electronic literature. See also the Nordic Electronic Literature Collection in the ELMCIP Electronic Literature Knowledge Base.[3] Significant authors include Ottar Ormstad and Anne Bang Steinsvik.[4][5] -

    -

    See also[edit]

    - -

    Notes[edit]

    -
    -
      -
    1. ^ a b c d Blankner 1938. -
      2. -
    2. ^ On my radar: Karl Ove Knausgaard's cultural highlights The Guardian 24 January 2021 -
      3. -
    3. ^ "Nordic Electronic Literature Research Collection | ELMCIP". elmcip.net. Retrieved 2022-10-15. -
      4. -
    4. ^ Rustad, Hans Kristian (2009-03-22). "Elektronisk litteratur i Norden: Nordisk hyperteksttradisjon i lys av internasjonal hypertekstfiksjonshistorikk". K&K - Kultur og Klasse (in Danish). 36 (106): 190–217. doi:10.7146/kok.v36i106.22030. ISSN 2246-2589. -
      5. -
    5. ^ Ormstad, Ottar (2012-12-20). "From Concrete to Digital Poetry: DRIVING DOWN THE ROAD OF CONTINUITY? A Personal Report from Norway". Dichtung Digital. Journal für Kunst und Kultur digitaler Medien. 14 (2): 1–18. doi:10.25969/mediarep/17756. -
      6. -
    -

    References[edit]

    -
    • Blankner, Frederika (1938). A History of the Scandinavian Literatures. Dial Press Inc., New York.
      • -
    • Clough, Ethlyn T. (editor) (1909). Norwegian Life. Bay View Reading Club.
      • -
    • Gjerset, Knut (1915). The History of the Norwegian People. MacMillan.
      • -
    • Griffiths, Tony (2004). Scandinavia; at War with Trolls. Palgrave MacMillan. ISBN 1-4039-6776-8
      • -
    • Grøndahl, Carl Henrik and Nina Tjomsland (editors) (1978). The Literary Masters of Norway, with Samples of Their Works. Tanum-Norli, Oslo.
      • -
    • Larson, Karen (1948). A History of Norway. Princeton University Press.
      • -
    • Naess, Harald S. (1993). A History of Norwegian Literature. University of Nebraska Press. ISBN 0-8032-3317-5
    -

    External links[edit]

    -
    -
    -
    -
    Wikisource has the text of the 1905 New International Encyclopedia article "Norwegian Literature".
    -
    -
    -
    -
    -
    Wikisource has several original texts related to -Norwegian literature.
    -
    - -
    -
    -
    -
    -
    - - - - -
    -
    Retrieved from "https://en.wikipedia.org/w/index.php?title=Norwegian_literature&oldid=1189385061"
    - -
    -
    - -
    -
    - - - -
    -
    -
    -
    -
      -
    • - -
      • -
    -
    - - - - \ No newline at end of file diff --git a/llama.cpp b/llama.cpp deleted file mode 160000 index f87f7b898651339fe173ddf016ca826163e899d8..0000000000000000000000000000000000000000 --- a/llama.cpp +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f87f7b898651339fe173ddf016ca826163e899d8 diff --git a/modules/backup-azure/client.go b/modules/backup-azure/client.go deleted file mode 100644 index ad7e30833dc26ee6aa0ce1f13a22d9c7013dc1f8..0000000000000000000000000000000000000000 --- a/modules/backup-azure/client.go +++ /dev/null @@ -1,277 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modstgazure - -import ( - "bytes" - "context" - "fmt" - "io" - "os" - "path" - "strings" - "time" - - "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" - "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/backup" -) - -type azureClient struct { - client *azblob.Client - config clientConfig - serviceURL string - dataPath string -} - -func newClient(ctx context.Context, config *clientConfig, dataPath string) (*azureClient, error) { - connectionString := os.Getenv("AZURE_STORAGE_CONNECTION_STRING") - if connectionString != "" { - client, err := azblob.NewClientFromConnectionString(connectionString, nil) - if err != nil { - return nil, errors.Wrap(err, "create client using connection string") - } - serviceURL := "" - connectionStrings := strings.Split(connectionString, ";") - for _, str := range connectionStrings { - if strings.HasPrefix(str, "BlobEndpoint") { - blobEndpoint := strings.Split(str, "=") - if len(blobEndpoint) > 1 { - serviceURL = blobEndpoint[1] - if !strings.HasSuffix(serviceURL, "/") { - serviceURL = serviceURL + "/" - } - } - } - } - return &azureClient{client, *config, serviceURL, dataPath}, nil - } - - // Your account name and key can be obtained from the Azure Portal. - accountName := os.Getenv("AZURE_STORAGE_ACCOUNT") - accountKey := os.Getenv("AZURE_STORAGE_KEY") - - if accountName == "" { - return nil, errors.New("AZURE_STORAGE_ACCOUNT must be set") - } - - // The service URL for blob endpoints is usually in the form: http(s)://.blob.core.windows.net/ - serviceURL := fmt.Sprintf("https://%s.blob.core.windows.net/", accountName) - - if accountKey != "" { - cred, err := azblob.NewSharedKeyCredential(accountName, accountKey) - if err != nil { - return nil, err - } - - client, err := azblob.NewClientWithSharedKeyCredential(serviceURL, cred, nil) - if err != nil { - return nil, err - } - return &azureClient{client, *config, serviceURL, dataPath}, nil - } - - options := &azblob.ClientOptions{ - ClientOptions: policy.ClientOptions{ - Retry: policy.RetryOptions{ - MaxRetries: 3, - RetryDelay: 4 * time.Second, - MaxRetryDelay: 120 * time.Second, - }, - }, - } - - client, err := azblob.NewClientWithNoCredential(serviceURL, options) - if err != nil { - return nil, err - } - return &azureClient{client, *config, serviceURL, dataPath}, nil -} - -func (a *azureClient) HomeDir(backupID string) string { - return a.serviceURL + path.Join(a.config.Container, a.makeObjectName(backupID)) -} - -func (a *azureClient) makeObjectName(parts ...string) string { - base := path.Join(parts...) - return path.Join(a.config.BackupPath, base) -} - -func (a *azureClient) GetObject(ctx context.Context, backupID, key string) ([]byte, error) { - objectName := a.makeObjectName(backupID, key) - - blobDownloadResponse, err := a.client.DownloadStream(ctx, a.config.Container, objectName, nil) - if err != nil { - if bloberror.HasCode(err, bloberror.BlobNotFound) { - return nil, backup.NewErrNotFound(errors.Wrapf(err, "get object '%s'", objectName)) - } - return nil, backup.NewErrInternal(errors.Wrapf(err, "download stream for object '%s'", objectName)) - } - - reader := blobDownloadResponse.Body - downloadData, err := io.ReadAll(reader) - errClose := reader.Close() - if errClose != nil { - return nil, backup.NewErrInternal(errors.Wrapf(errClose, "close stream for object '%s'", objectName)) - } - if err != nil { - return nil, backup.NewErrInternal(errors.Wrapf(err, "read stream for object '%s'", objectName)) - } - - return downloadData, nil -} - -func (a *azureClient) PutFile(ctx context.Context, backupID, key, srcPath string) error { - filePath := path.Join(a.dataPath, srcPath) - file, err := os.Open(filePath) - if err != nil { - return backup.NewErrInternal(errors.Wrapf(err, "open file: %q", filePath)) - } - defer file.Close() - - objectName := a.makeObjectName(backupID, key) - _, err = a.client.UploadFile(ctx, - a.config.Container, - objectName, - file, - &azblob.UploadFileOptions{ - Metadata: map[string]*string{"backupid": to.Ptr(backupID)}, - Tags: map[string]string{"backupid": backupID}, - }) - if err != nil { - return backup.NewErrInternal(errors.Wrapf(err, "upload file for object '%s'", objectName)) - } - - return nil -} - -func (a *azureClient) PutObject(ctx context.Context, backupID, key string, data []byte) error { - objectName := a.makeObjectName(backupID, key) - - reader := bytes.NewReader(data) - _, err := a.client.UploadStream(ctx, - a.config.Container, - objectName, - reader, - &azblob.UploadStreamOptions{ - Metadata: map[string]*string{"backupid": to.Ptr(backupID)}, - Tags: map[string]string{"backupid": backupID}, - }) - if err != nil { - return backup.NewErrInternal(errors.Wrapf(err, "upload stream for object '%s'", objectName)) - } - - return nil -} - -func (a *azureClient) Initialize(ctx context.Context, backupID string) error { - key := "access-check" - - if err := a.PutObject(ctx, backupID, key, []byte("")); err != nil { - return errors.Wrap(err, "failed to access-check Azure backup module") - } - - objectName := a.makeObjectName(backupID, key) - if _, err := a.client.DeleteBlob(ctx, a.config.Container, objectName, nil); err != nil { - return errors.Wrap(err, "failed to remove access-check Azure backup module") - } - - return nil -} - -func (a *azureClient) WriteToFile(ctx context.Context, backupID, key, destPath string) error { - dir := path.Dir(destPath) - if err := os.MkdirAll(dir, os.ModePerm); err != nil { - return errors.Wrapf(err, "make dir '%s'", dir) - } - - file, err := os.Create(destPath) - if err != nil { - return backup.NewErrInternal(errors.Wrapf(err, "create file: %q", destPath)) - } - defer file.Close() - - objectName := a.makeObjectName(backupID, key) - _, err = a.client.DownloadFile(ctx, a.config.Container, objectName, file, nil) - if err != nil { - if bloberror.HasCode(err, bloberror.BlobNotFound) { - return backup.NewErrNotFound(errors.Wrapf(err, "get object '%s'", objectName)) - } - return backup.NewErrInternal(errors.Wrapf(err, "download file for object '%s'", objectName)) - } - - return nil -} - -func (a *azureClient) Write(ctx context.Context, backupID, key string, r io.ReadCloser) (written int64, err error) { - path := a.makeObjectName(backupID, key) - reader := &reader{src: r} - defer func() { - r.Close() - written = int64(reader.count) - }() - - if _, err = a.client.UploadStream(ctx, - a.config.Container, - path, - reader, - &azblob.UploadStreamOptions{ - Metadata: map[string]*string{"backupid": to.Ptr(backupID)}, - Tags: map[string]string{"backupid": backupID}, - }); err != nil { - err = fmt.Errorf("upload stream %q: %w", path, err) - } - - return -} - -func (a *azureClient) Read(ctx context.Context, backupID, key string, w io.WriteCloser) (int64, error) { - defer w.Close() - - path := a.makeObjectName(backupID, key) - resp, err := a.client.DownloadStream(ctx, a.config.Container, path, nil) - if err != nil { - err = fmt.Errorf("find object %q: %w", path, err) - if bloberror.HasCode(err, bloberror.BlobNotFound) { - err = backup.NewErrNotFound(err) - } - return 0, err - } - defer resp.Body.Close() - - read, err := io.Copy(w, resp.Body) - if err != nil { - return read, fmt.Errorf("io.copy %q: %w", path, err) - } - - return read, nil -} - -func (a *azureClient) SourceDataPath() string { - return a.dataPath -} - -// reader is a wrapper used to count number of written bytes -// Unlike GCS and S3 Azure Interface does not provide this information -type reader struct { - src io.Reader - count int -} - -func (r *reader) Read(p []byte) (n int, err error) { - n, err = r.src.Read(p) - r.count += n - return -} diff --git a/modules/backup-azure/module.go b/modules/backup-azure/module.go deleted file mode 100644 index 23b472abc7015ed5a1a7ee3cec4d634910a42e84..0000000000000000000000000000000000000000 --- a/modules/backup-azure/module.go +++ /dev/null @@ -1,116 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modstgazure - -import ( - "context" - "net/http" - "os" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" -) - -const ( - Name = "backup-azure" - AltName1 = "azure" - azureContainer = "BACKUP_AZURE_CONTAINER" - - // this is an optional value, allowing for - // the backup to be stored in a specific - // directory inside the provided container. - // - // if left unset, the backup files will - // be stored directly in the root of the - // container. - azurePath = "BACKUP_AZURE_PATH" -) - -type clientConfig struct { - Container string - - // this is an optional value, allowing for - // the backup to be stored in a specific - // directory inside the provided bucket - BackupPath string -} - -type Module struct { - logger logrus.FieldLogger - *azureClient - dataPath string -} - -func New() *Module { - return &Module{} -} - -func (m *Module) Name() string { - return Name -} - -func (m *Module) IsExternal() bool { - return true -} - -func (m *Module) AltNames() []string { - return []string{AltName1} -} - -func (m *Module) Type() modulecapabilities.ModuleType { - return modulecapabilities.Backup -} - -func (m *Module) Init(ctx context.Context, - params moduletools.ModuleInitParams, -) error { - m.logger = params.GetLogger() - m.dataPath = params.GetStorageProvider().DataPath() - - config := &clientConfig{ - Container: os.Getenv(azureContainer), - BackupPath: os.Getenv(azurePath), - } - if config.Container == "" { - return errors.Errorf("backup init: '%s' must be set", azureContainer) - } - - client, err := newClient(ctx, config, m.dataPath) - if err != nil { - return errors.Wrap(err, "init Azure client") - } - m.azureClient = client - return nil -} - -func (m *Module) RootHandler() http.Handler { - // TODO: remove once this is a capability interface - return nil -} - -func (m *Module) MetaInfo() (map[string]interface{}, error) { - metaInfo := make(map[string]interface{}) - metaInfo["containerName"] = m.config.Container - if root := m.config.BackupPath; root != "" { - metaInfo["rootName"] = root - } - return metaInfo, nil -} - -// verify we implement the modules.Module interface -var ( - _ = modulecapabilities.Module(New()) - _ = modulecapabilities.BackupBackend(New()) - _ = modulecapabilities.MetaProvider(New()) -) diff --git a/modules/backup-filesystem/backup.go b/modules/backup-filesystem/backup.go deleted file mode 100644 index 6d612536b3ddd100b4b2160dfce12eb593b3c58e..0000000000000000000000000000000000000000 --- a/modules/backup-filesystem/backup.go +++ /dev/null @@ -1,236 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modstgfs - -import ( - "context" - "fmt" - "io" - "os" - "path" - "path/filepath" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/backup" - "github.com/weaviate/weaviate/usecases/monitoring" -) - -func (m *Module) GetObject(ctx context.Context, backupID, key string) ([]byte, error) { - metaPath, err := m.getObjectPath(ctx, backupID, key) - if err != nil { - return nil, err - } - - contents, err := os.ReadFile(metaPath) - if err != nil { - return nil, backup.NewErrInternal(errors.Wrapf(err, "get object '%s'", metaPath)) - } - - metric, err := monitoring.GetMetrics().BackupRestoreDataTransferred.GetMetricWithLabelValues(m.Name(), "class") - if err == nil { - metric.Add(float64(len(contents))) - } - - return contents, nil -} - -func (m *Module) getObjectPath(ctx context.Context, backupID, key string) (string, error) { - metaPath := filepath.Join(m.backupsPath, backupID, key) - - if err := ctx.Err(); err != nil { - return "", backup.NewErrContextExpired(errors.Wrapf(err, "get object '%s'", metaPath)) - } - - if _, err := os.Stat(metaPath); errors.Is(err, os.ErrNotExist) { - return "", backup.NewErrNotFound(errors.Wrapf(err, "get object '%s'", metaPath)) - } else if err != nil { - return "", backup.NewErrInternal(errors.Wrapf(err, "get object '%s'", metaPath)) - } - - return metaPath, nil -} - -func (m *Module) PutFile(ctx context.Context, backupID, key, srcPath string) error { - sourcePath := path.Join(m.dataPath, srcPath) - backupPath := path.Join(m.makeBackupDirPath(backupID), key) - - bytesWritten, err := m.copyFile(sourcePath, backupPath) - if err != nil { - return err - } - - metric, err := monitoring.GetMetrics().BackupStoreDataTransferred.GetMetricWithLabelValues(m.Name(), "class") - if err == nil { - metric.Add(float64(bytesWritten)) - } - - return nil -} - -func (m *Module) copyFile(sourcePath, destinationPath string) (int64, error) { - source, err := os.Open(sourcePath) - defer func() error { - return source.Close() - }() - if err != nil { - return 0, errors.Wrapf(err, "open file '%s'", sourcePath) - } - - if _, err := os.Stat(destinationPath); err != nil { - if err := os.MkdirAll(path.Dir(destinationPath), os.ModePerm); err != nil { - return 0, errors.Wrapf(err, "make dir '%s'", destinationPath) - } - } - - destination, err := os.Create(destinationPath) - defer func() error { - return destination.Close() - }() - if err != nil { - return 0, errors.Wrapf(err, "create destination file '%s'", destinationPath) - } - - written, err := io.Copy(destination, source) - if err != nil { - return 0, errors.Wrapf(err, "copy file from '%s' to '%s'", sourcePath, destinationPath) - } - - return written, nil -} - -func (m *Module) PutObject(ctx context.Context, backupID, key string, byes []byte) error { - backupPath := path.Join(m.makeBackupDirPath(backupID), key) - - dir := path.Dir(backupPath) - - if err := os.MkdirAll(dir, os.ModePerm); err != nil { - return errors.Wrapf(err, "make dir '%s'", dir) - } - - if err := os.WriteFile(backupPath, byes, os.ModePerm); err != nil { - return errors.Wrapf(err, "write file '%s'", backupPath) - } - - metric, err := monitoring.GetMetrics().BackupStoreDataTransferred.GetMetricWithLabelValues(m.Name(), "class") - if err == nil { - metric.Add(float64(len(byes))) - } - - return nil -} - -func (m *Module) Initialize(ctx context.Context, backupID string) error { - // TODO: does anything need to be done here? - return nil -} - -func (m *Module) WriteToFile(ctx context.Context, backupID, key, destPath string) error { - sourcePath, err := m.getObjectPath(ctx, backupID, key) - if err != nil { - return err - } - - bytesWritten, err := m.copyFile(sourcePath, destPath) - if err != nil { - return err - } - - metric, err := monitoring.GetMetrics().BackupRestoreDataTransferred.GetMetricWithLabelValues(m.Name(), "class") - if err == nil { - metric.Add(float64(bytesWritten)) - } - - return nil -} - -func (m *Module) Write(ctx context.Context, backupID, key string, r io.ReadCloser) (int64, error) { - defer r.Close() - backupPath := path.Join(m.makeBackupDirPath(backupID), key) - dir := path.Dir(backupPath) - if err := os.MkdirAll(dir, os.ModePerm); err != nil { - return 0, fmt.Errorf("make dir %q: %w", dir, err) - } - f, err := os.OpenFile(backupPath, os.O_RDWR|os.O_CREATE, os.ModePerm) - if err != nil { - return 0, fmt.Errorf("open file %q: %w", backupPath, err) - } - defer f.Close() - - written, err := io.Copy(f, r) - if err != nil { - return 0, fmt.Errorf("write file %q: %w", backupPath, err) - } - if metric, err := monitoring.GetMetrics().BackupStoreDataTransferred. - GetMetricWithLabelValues(m.Name(), "class"); err == nil { - metric.Add(float64(written)) - } - - return written, err -} - -func (m *Module) Read(ctx context.Context, backupID, key string, w io.WriteCloser) (int64, error) { - defer w.Close() - sourcePath, err := m.getObjectPath(ctx, backupID, key) - if err != nil { - return 0, fmt.Errorf("source path %s/%s: %w", backupID, key, err) - } - - // open file - f, err := os.Open(sourcePath) - if err != nil { - return 0, fmt.Errorf("open file %q: %w", sourcePath, err) - } - defer f.Close() - - // copy file - read, err := io.Copy(w, f) - if err != nil { - return 0, fmt.Errorf("write : %w", err) - } - - if metric, err := monitoring.GetMetrics().BackupRestoreDataTransferred. - GetMetricWithLabelValues(m.Name(), "class"); err == nil { - metric.Add(float64(read)) - } - return read, err -} - -func (m *Module) SourceDataPath() string { - return m.dataPath -} - -func (m *Module) initBackupBackend(ctx context.Context, backupsPath string) error { - if backupsPath == "" { - return fmt.Errorf("empty backup path provided") - } - backupsPath = filepath.Clean(backupsPath) - if !filepath.IsAbs(backupsPath) { - return fmt.Errorf("relative backup path provided") - } - if err := m.createBackupsDir(backupsPath); err != nil { - return errors.Wrap(err, "invalid backup path provided") - } - m.backupsPath = backupsPath - - return nil -} - -func (m *Module) createBackupsDir(backupsPath string) error { - if err := os.MkdirAll(backupsPath, os.ModePerm); err != nil { - m.logger.WithField("module", m.Name()). - WithField("action", "create_backups_dir"). - WithError(err). - Errorf("failed creating backups directory %v", backupsPath) - return backup.NewErrInternal(errors.Wrap(err, "make backups dir")) - } - return nil -} diff --git a/modules/backup-filesystem/backup_test.go b/modules/backup-filesystem/backup_test.go deleted file mode 100644 index c1f0a5e2d0bd97861291d97c41c98d323ee0db85..0000000000000000000000000000000000000000 --- a/modules/backup-filesystem/backup_test.go +++ /dev/null @@ -1,54 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modstgfs - -import ( - "context" - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestBackend_StoreBackup(t *testing.T) { - backupRelativePath := filepath.Join("./backups", "some", "nested", "dir") - backupAbsolutePath := t.TempDir() - - ctx := context.Background() - - t.Run("fails init fs module with empty backup path", func(t *testing.T) { - module := New() - err := module.initBackupBackend(ctx, "") - - assert.NotNil(t, err) - assert.Contains(t, err.Error(), "empty backup path provided") - }) - - t.Run("fails init fs module with relative backup path", func(t *testing.T) { - module := New() - err := module.initBackupBackend(ctx, backupRelativePath) - - assert.NotNil(t, err) - assert.Contains(t, err.Error(), "relative backup path provided") - }) - - t.Run("inits backup module with absolute backup path", func(t *testing.T) { - module := New() - err := module.initBackupBackend(ctx, backupAbsolutePath) - - assert.Nil(t, err) - - _, err = os.Stat(backupAbsolutePath) - assert.Nil(t, err) - }) -} diff --git a/modules/backup-filesystem/module.go b/modules/backup-filesystem/module.go deleted file mode 100644 index 6acc57425d0490d0dae88e48274c0abfa8335dd0..0000000000000000000000000000000000000000 --- a/modules/backup-filesystem/module.go +++ /dev/null @@ -1,96 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modstgfs - -import ( - "context" - "net/http" - "os" - "path" - "path/filepath" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" -) - -const ( - Name = "backup-filesystem" - AltName1 = "filesystem" - backupsPathName = "BACKUP_FILESYSTEM_PATH" -) - -type Module struct { - logger logrus.FieldLogger - dataPath string // path to the current (operational) data - backupsPath string // complete(?) path to the directory that holds all the backups -} - -func New() *Module { - return &Module{} -} - -func (m *Module) Name() string { - return Name -} - -func (m *Module) IsExternal() bool { - return false -} - -func (m *Module) AltNames() []string { - return []string{AltName1} -} - -func (m *Module) Type() modulecapabilities.ModuleType { - return modulecapabilities.Backup -} - -func (m *Module) Init(ctx context.Context, - params moduletools.ModuleInitParams, -) error { - m.logger = params.GetLogger() - m.dataPath = params.GetStorageProvider().DataPath() - backupsPath := os.Getenv(backupsPathName) - if err := m.initBackupBackend(ctx, backupsPath); err != nil { - return errors.Wrap(err, "init backup backend") - } - - return nil -} - -func (m *Module) HomeDir(backupID string) string { - return path.Join(m.makeBackupDirPath(backupID)) -} - -func (m *Module) RootHandler() http.Handler { - // TODO: remove once this is a capability interface - return nil -} - -func (m *Module) MetaInfo() (map[string]interface{}, error) { - metaInfo := make(map[string]interface{}) - metaInfo["backupsPath"] = m.backupsPath - return metaInfo, nil -} - -func (m *Module) makeBackupDirPath(id string) string { - return filepath.Join(m.backupsPath, id) -} - -// verify we implement the modules.Module interface -var ( - _ = modulecapabilities.Module(New()) - _ = modulecapabilities.BackupBackend(New()) - _ = modulecapabilities.MetaProvider(New()) -) diff --git a/modules/backup-gcs/client.go b/modules/backup-gcs/client.go deleted file mode 100644 index db043417b39fdcd9d3c88819d895897be083962f..0000000000000000000000000000000000000000 --- a/modules/backup-gcs/client.go +++ /dev/null @@ -1,373 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modstggcs - -import ( - "context" - "fmt" - "io" - "os" - "path" - "strings" - "time" - - "cloud.google.com/go/storage" - "github.com/googleapis/gax-go/v2" - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/backup" - "github.com/weaviate/weaviate/usecases/monitoring" - "golang.org/x/oauth2/google" - "google.golang.org/api/option" -) - -type gcsClient struct { - client *storage.Client - config clientConfig - projectID string - dataPath string -} - -func newClient(ctx context.Context, config *clientConfig, dataPath string) (*gcsClient, error) { - options := []option.ClientOption{} - useAuth := strings.ToLower(os.Getenv("BACKUP_GCS_USE_AUTH")) != "false" - if useAuth { - scopes := []string{ - "https://www.googleapis.com/auth/devstorage.read_write", - } - creds, err := google.FindDefaultCredentials(ctx, scopes...) - if err != nil { - return nil, errors.Wrap(err, "find default credentials") - } - options = append(options, option.WithCredentials(creds)) - } else { - options = append(options, option.WithoutAuthentication()) - } - projectID := os.Getenv("GOOGLE_CLOUD_PROJECT") - if len(projectID) == 0 { - projectID = os.Getenv("GCLOUD_PROJECT") - if len(projectID) == 0 { - projectID = os.Getenv("GCP_PROJECT") - } - } - client, err := storage.NewClient(ctx, options...) - if err != nil { - return nil, errors.Wrap(err, "create client") - } - - client.SetRetry(storage.WithBackoff(gax.Backoff{ - Initial: 2 * time.Second, // Note: the client uses a jitter internally - Max: 60 * time.Second, - Multiplier: 3, - }), - storage.WithPolicy(storage.RetryAlways), - ) - return &gcsClient{client, *config, projectID, dataPath}, nil -} - -func (g *gcsClient) getObject(ctx context.Context, bucket *storage.BucketHandle, - backupID, objectName string, -) ([]byte, error) { - // Create bucket reader - obj := bucket.Object(objectName) - reader, err := obj.NewReader(ctx) - if err != nil { - if errors.Is(err, storage.ErrObjectNotExist) { - return nil, err - } - return nil, errors.Wrapf(err, "new reader: %v", objectName) - } - // Read file contents - content, err := io.ReadAll(reader) - if err != nil { - return nil, errors.Wrapf(err, "read object: %v", objectName) - } - - metric, err := monitoring.GetMetrics().BackupRestoreDataTransferred.GetMetricWithLabelValues(Name, "class") - if err == nil { - metric.Add(float64(len(content))) - } - return content, nil -} - -func (g *gcsClient) HomeDir(backupID string) string { - return "gs://" + path.Join(g.config.Bucket, - g.makeObjectName(backupID)) -} - -func (g *gcsClient) findBucket(ctx context.Context) (*storage.BucketHandle, error) { - bucket := g.client.Bucket(g.config.Bucket) - - if _, err := bucket.Attrs(ctx); err != nil { - return nil, err - } - - return bucket, nil -} - -func (g *gcsClient) makeObjectName(parts ...string) string { - base := path.Join(parts...) - return path.Join(g.config.BackupPath, base) -} - -func (g *gcsClient) GetObject(ctx context.Context, backupID, key string) ([]byte, error) { - objectName := g.makeObjectName(backupID, key) - - if err := ctx.Err(); err != nil { - return nil, backup.NewErrContextExpired(errors.Wrapf(err, "get object '%s'", objectName)) - } - - bucket, err := g.findBucket(ctx) - if err != nil { - if errors.Is(err, storage.ErrBucketNotExist) { - return nil, backup.NewErrNotFound(errors.Wrapf(err, "get object '%s'", objectName)) - } - return nil, backup.NewErrInternal(errors.Wrapf(err, "get object '%s'", objectName)) - } - - contents, err := g.getObject(ctx, bucket, backupID, objectName) - if err != nil { - if errors.Is(err, storage.ErrObjectNotExist) { - return nil, backup.NewErrNotFound(errors.Wrapf(err, "get object '%s'", objectName)) - } - return nil, backup.NewErrInternal(errors.Wrapf(err, "get object '%s'", objectName)) - } - - return contents, nil -} - -// PutFile creates an object with contents from file at filePath. -func (g *gcsClient) PutFile(ctx context.Context, backupID, key, srcPath string) error { - bucket, err := g.findBucket(ctx) - if err != nil { - return fmt.Errorf("find bucket: %w", err) - } - - // open source file - filePath := path.Join(g.dataPath, srcPath) - file, err := os.Open(filePath) - if err != nil { - return fmt.Errorf("os.open %q: %w", filePath, err) - } - defer file.Close() - - // create a new writer - object := g.makeObjectName(backupID, key) - writer := bucket.Object(object).NewWriter(ctx) - writer.ContentType = "application/octet-stream" - writer.Metadata = map[string]string{"backup-id": backupID} - - // if we return early make sure writer is closed - closeWriter := true - defer func() { - if closeWriter { - writer.Close() - } - }() - - nBytes, err := io.Copy(writer, file) - if err != nil { - return fmt.Errorf("io.copy %q %q: %w", object, filePath, err) - } - closeWriter = false - if err := writer.Close(); err != nil { - return fmt.Errorf("writer.close %q: %w", filePath, err) - } - metric, err := monitoring.GetMetrics().BackupStoreDataTransferred.GetMetricWithLabelValues("backup-gcs", "class") - if err == nil { - metric.Add(float64(nBytes)) - } - return nil -} - -func (g *gcsClient) PutObject(ctx context.Context, backupID, key string, byes []byte) error { - bucket, err := g.findBucket(ctx) - if err != nil { - return errors.Wrap(err, "find bucket") - } - - objectName := g.makeObjectName(backupID, key) - obj := bucket.Object(objectName) - writer := obj.NewWriter(ctx) - writer.ContentType = "application/octet-stream" - writer.Metadata = map[string]string{ - "backup-id": backupID, - } - if _, err := writer.Write(byes); err != nil { - return errors.Wrapf(err, "write file: %v", objectName) - } - if err := writer.Close(); err != nil { - return errors.Wrapf(err, "close writer for file: %v", objectName) - } - - metric, err := monitoring.GetMetrics().BackupStoreDataTransferred.GetMetricWithLabelValues("backup-gcs", "class") - if err == nil { - metric.Add(float64(len(byes))) - } - - return nil -} - -func (g *gcsClient) Initialize(ctx context.Context, backupID string) error { - key := "access-check" - - if err := g.PutObject(ctx, backupID, key, []byte("")); err != nil { - return errors.Wrap(err, "failed to access-check gcs backup module") - } - - bucket, err := g.findBucket(ctx) - if err != nil { - return errors.Wrap(err, "find bucket") - } - - objectName := g.makeObjectName(backupID, key) - if err := bucket.Object(objectName).Delete(ctx); err != nil { - return errors.Wrap(err, "failed to remove access-check gcs backup module") - } - - return nil -} - -// WriteToFile downloads an object and store its content in destPath -// The file destPath will be created if it doesn't exit -func (g *gcsClient) WriteToFile(ctx context.Context, backupID, key, destPath string) (err error) { - bucket, err := g.findBucket(ctx) - if err != nil { - return fmt.Errorf("find bucket: %w", err) - } - - // validate destination path - if st, err := os.Stat(destPath); err == nil { - if st.IsDir() { - return fmt.Errorf("file is a directory") - } - } else if !os.IsNotExist(err) { - return err - } - - // create empty file - dir := path.Dir(destPath) - if err := os.MkdirAll(dir, os.ModePerm); err != nil { - return fmt.Errorf("os.mkdir %q: %w", dir, err) - } - file, err := os.Create(destPath) - if err != nil { - return fmt.Errorf("os.create %q: %w", destPath, err) - } - - // make sure to close and delete in case we return early - closeAndRemove := true - defer func() { - if closeAndRemove { - file.Close() - os.Remove(destPath) - } - }() - - // create reader - object := g.makeObjectName(backupID, key) - rc, err := bucket.Object(object).NewReader(ctx) - if err != nil { - return fmt.Errorf("find object %q: %w", object, err) - } - defer rc.Close() - - // transfer content to the file - if _, err := io.Copy(file, rc); err != nil { - return fmt.Errorf("io.Copy:%q %q: %w", destPath, object, err) - } - closeAndRemove = false - if err = file.Close(); err != nil { - return fmt.Errorf("f.Close %q: %w", destPath, err) - } - - return nil -} - -func (g *gcsClient) Write(ctx context.Context, backupID, key string, r io.ReadCloser) (int64, error) { - defer r.Close() - - bucket, err := g.findBucket(ctx) - if err != nil { - return 0, fmt.Errorf("find bucket: %w", err) - } - - // create a new writer - path := g.makeObjectName(backupID, key) - writer := bucket.Object(path).NewWriter(ctx) - writer.ContentType = "application/octet-stream" - writer.Metadata = map[string]string{"backup-id": backupID} - - // if we return early make sure writer is closed - closeWriter := true - defer func() { - if closeWriter { - writer.Close() - } - }() - - // copy - written, err := io.Copy(writer, r) - if err != nil { - return 0, fmt.Errorf("io.copy %q: %w", path, err) - } - closeWriter = false - if err := writer.Close(); err != nil { - return 0, fmt.Errorf("writer.close %q: %w", path, err) - } - if metric, err := monitoring.GetMetrics().BackupStoreDataTransferred. - GetMetricWithLabelValues(Name, "class"); err == nil { - metric.Add(float64(written)) - } - return written, nil -} - -func (g *gcsClient) Read(ctx context.Context, backupID, key string, w io.WriteCloser) (int64, error) { - defer w.Close() - - bucket, err := g.findBucket(ctx) - if err != nil { - err = fmt.Errorf("find bucket: %w", err) - if errors.Is(err, storage.ErrObjectNotExist) { - err = backup.NewErrNotFound(err) - } - return 0, err - } - - // create reader - path := g.makeObjectName(backupID, key) - rc, err := bucket.Object(path).NewReader(ctx) - if err != nil { - err = fmt.Errorf("find object %s: %v", path, err) - if errors.Is(err, storage.ErrObjectNotExist) { - err = backup.NewErrNotFound(err) - } - return 0, err - } - defer rc.Close() - - // copy - read, err := io.Copy(w, rc) - if err != nil { - return read, fmt.Errorf("io.copy %q: %w", path, err) - } - - if metric, err := monitoring.GetMetrics().BackupRestoreDataTransferred. - GetMetricWithLabelValues(Name, "class"); err == nil { - metric.Add(float64(float64(read))) - } - - return read, nil -} - -func (g *gcsClient) SourceDataPath() string { - return g.dataPath -} diff --git a/modules/backup-gcs/module.go b/modules/backup-gcs/module.go deleted file mode 100644 index 8c802ce3de0cce091c599b2c8ee70bfa9183e283..0000000000000000000000000000000000000000 --- a/modules/backup-gcs/module.go +++ /dev/null @@ -1,116 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modstggcs - -import ( - "context" - "net/http" - "os" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" -) - -const ( - Name = "backup-gcs" - AltName1 = "gcs" - gcsBucket = "BACKUP_GCS_BUCKET" - - // this is an optional value, allowing for - // the backup to be stored in a specific - // directory inside the provided bucket. - // - // if left unset, the backup files will - // be stored directly in the root of the - // bucket. - gcsPath = "BACKUP_GCS_PATH" -) - -type clientConfig struct { - Bucket string - - // this is an optional value, allowing for - // the backup to be stored in a specific - // directory inside the provided bucket - BackupPath string -} - -type Module struct { - logger logrus.FieldLogger - *gcsClient - dataPath string -} - -func New() *Module { - return &Module{} -} - -func (m *Module) Name() string { - return Name -} - -func (m *Module) IsExternal() bool { - return true -} - -func (m *Module) AltNames() []string { - return []string{AltName1} -} - -func (m *Module) Type() modulecapabilities.ModuleType { - return modulecapabilities.Backup -} - -func (m *Module) Init(ctx context.Context, - params moduletools.ModuleInitParams, -) error { - m.logger = params.GetLogger() - m.dataPath = params.GetStorageProvider().DataPath() - - config := &clientConfig{ - Bucket: os.Getenv(gcsBucket), - BackupPath: os.Getenv(gcsPath), - } - if config.Bucket == "" { - return errors.Errorf("backup init: '%s' must be set", gcsBucket) - } - - client, err := newClient(ctx, config, m.dataPath) - if err != nil { - return errors.Wrap(err, "init gcs client") - } - m.gcsClient = client - return nil -} - -func (m *Module) RootHandler() http.Handler { - // TODO: remove once this is a capability interface - return nil -} - -func (m *Module) MetaInfo() (map[string]interface{}, error) { - metaInfo := make(map[string]interface{}) - metaInfo["bucketName"] = m.config.Bucket - if root := m.config.BackupPath; root != "" { - metaInfo["rootName"] = root - } - return metaInfo, nil -} - -// verify we implement the modules.Module interface -var ( - _ = modulecapabilities.Module(New()) - _ = modulecapabilities.BackupBackend(New()) - _ = modulecapabilities.MetaProvider(New()) -) diff --git a/modules/backup-s3/client.go b/modules/backup-s3/client.go deleted file mode 100644 index 99608141afffecac79fabf9dac044bb2c9338b9c..0000000000000000000000000000000000000000 --- a/modules/backup-s3/client.go +++ /dev/null @@ -1,229 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modstgs3 - -import ( - "bytes" - "context" - "fmt" - "io" - "net/http" - "os" - "path" - - "github.com/minio/minio-go/v7" - "github.com/minio/minio-go/v7/pkg/credentials" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/backup" - "github.com/weaviate/weaviate/usecases/monitoring" -) - -type s3Client struct { - client *minio.Client - config *clientConfig - logger logrus.FieldLogger - dataPath string -} - -func newClient(config *clientConfig, logger logrus.FieldLogger, dataPath string) (*s3Client, error) { - region := os.Getenv("AWS_REGION") - if len(region) == 0 { - region = os.Getenv("AWS_DEFAULT_REGION") - } - - var creds *credentials.Credentials - if (os.Getenv("AWS_ACCESS_KEY_ID") != "" || os.Getenv("AWS_ACCESS_KEY") != "") && - (os.Getenv("AWS_SECRET_ACCESS_KEY") != "" || os.Getenv("AWS_SECRET_KEY") != "") { - creds = credentials.NewEnvAWS() - } else { - creds = credentials.NewIAM("") - if _, err := creds.Get(); err != nil { - // can be anonymous access - creds = credentials.NewEnvAWS() - } - } - - client, err := minio.New(config.Endpoint, &minio.Options{ - Creds: creds, - Region: region, - Secure: config.UseSSL, - }) - if err != nil { - return nil, errors.Wrap(err, "create client") - } - return &s3Client{client, config, logger, dataPath}, nil -} - -func (s *s3Client) makeObjectName(parts ...string) string { - base := path.Join(parts...) - return path.Join(s.config.BackupPath, base) -} - -func (s *s3Client) HomeDir(backupID string) string { - return "s3://" + path.Join(s.config.Bucket, - s.makeObjectName(backupID)) -} - -func (s *s3Client) GetObject(ctx context.Context, backupID, key string) ([]byte, error) { - objectName := s.makeObjectName(backupID, key) - - if err := ctx.Err(); err != nil { - return nil, backup.NewErrContextExpired(errors.Wrapf(err, "get object '%s'", objectName)) - } - - obj, err := s.client.GetObject(ctx, s.config.Bucket, objectName, minio.GetObjectOptions{}) - if err != nil { - return nil, backup.NewErrInternal(errors.Wrapf(err, "get object '%s'", objectName)) - } - - contents, err := io.ReadAll(obj) - if err != nil { - if s3Err, ok := err.(minio.ErrorResponse); ok && s3Err.StatusCode == http.StatusNotFound { - return nil, backup.NewErrNotFound(errors.Wrapf(err, "get object '%s'", objectName)) - } - return nil, backup.NewErrInternal(errors.Wrapf(err, "get object '%s'", objectName)) - } - - metric, err := monitoring.GetMetrics().BackupRestoreDataTransferred.GetMetricWithLabelValues(Name, "class") - if err == nil { - metric.Add(float64(len(contents))) - } - - return contents, nil -} - -func (s *s3Client) PutFile(ctx context.Context, backupID, key string, srcPath string) error { - objectName := s.makeObjectName(backupID, key) - srcPath = path.Join(s.dataPath, srcPath) - opt := minio.PutObjectOptions{ContentType: "application/octet-stream"} - - _, err := s.client.FPutObject(ctx, s.config.Bucket, objectName, srcPath, opt) - if err != nil { - return backup.NewErrInternal( - errors.Wrapf(err, "put file '%s'", objectName)) - } - - // Get filesize - file, err := os.Stat(srcPath) - if err != nil { - return nil - } - size := file.Size() - - metric, err := monitoring.GetMetrics().BackupStoreDataTransferred.GetMetricWithLabelValues(Name, "class") - if err == nil { - metric.Add(float64(size)) - } - return nil -} - -func (s *s3Client) PutObject(ctx context.Context, backupID, key string, byes []byte) error { - objectName := s.makeObjectName(backupID, key) - opt := minio.PutObjectOptions{ContentType: "application/octet-stream"} - reader := bytes.NewReader(byes) - objectSize := int64(len(byes)) - - _, err := s.client.PutObject(ctx, s.config.Bucket, objectName, reader, objectSize, opt) - if err != nil { - return backup.NewErrInternal( - errors.Wrapf(err, "put object '%s'", objectName)) - } - - metric, err := monitoring.GetMetrics().BackupStoreDataTransferred.GetMetricWithLabelValues(Name, "class") - if err == nil { - metric.Add(float64(len(byes))) - } - return nil -} - -func (s *s3Client) Initialize(ctx context.Context, backupID string) error { - key := "access-check" - - if err := s.PutObject(ctx, backupID, key, []byte("")); err != nil { - return errors.Wrap(err, "failed to access-check s3 backup module") - } - - objectName := s.makeObjectName(backupID, key) - opt := minio.RemoveObjectOptions{} - if err := s.client.RemoveObject(ctx, s.config.Bucket, objectName, opt); err != nil { - return errors.Wrap(err, "failed to remove access-check s3 backup module") - } - - return nil -} - -// WriteFile downloads contents of an object to a local file destPath -func (s *s3Client) WriteToFile(ctx context.Context, backupID, key, destPath string) error { - object := s.makeObjectName(backupID, key) - err := s.client.FGetObject(ctx, s.config.Bucket, object, destPath, minio.GetObjectOptions{}) - if err != nil { - return fmt.Errorf("s3.FGetObject %q %q: %w", destPath, object, err) - } - - if st, err := os.Stat(destPath); err == nil { - metric, err := monitoring.GetMetrics().BackupRestoreDataTransferred.GetMetricWithLabelValues(Name, "class") - if err == nil { - metric.Add(float64(st.Size())) - } - } - return nil -} - -func (s *s3Client) Write(ctx context.Context, backupID, key string, r io.ReadCloser) (int64, error) { - defer r.Close() - path := s.makeObjectName(backupID, key) - opt := minio.PutObjectOptions{ - ContentType: "application/octet-stream", - DisableMultipart: false, - } - - info, err := s.client.PutObject(ctx, s.config.Bucket, path, r, -1, opt) - if err != nil { - return info.Size, fmt.Errorf("write object %q", path) - } - - if metric, err := monitoring.GetMetrics().BackupStoreDataTransferred. - GetMetricWithLabelValues(Name, "class"); err == nil { - metric.Add(float64(float64(info.Size))) - } - return info.Size, nil -} - -func (s *s3Client) Read(ctx context.Context, backupID, key string, w io.WriteCloser) (int64, error) { - defer w.Close() - path := s.makeObjectName(backupID, key) - obj, err := s.client.GetObject(ctx, s.config.Bucket, path, minio.GetObjectOptions{}) - if err != nil { - return 0, fmt.Errorf("get object %q: %w", path, err) - } - - read, err := io.Copy(w, obj) - if err != nil { - err = fmt.Errorf("get object %q: %w", path, err) - if s3Err, ok := err.(minio.ErrorResponse); ok && s3Err.StatusCode == http.StatusNotFound { - err = backup.NewErrNotFound(err) - } - return 0, err - } - - if metric, err := monitoring.GetMetrics().BackupRestoreDataTransferred. - GetMetricWithLabelValues(Name, "class"); err == nil { - metric.Add(float64(float64(read))) - } - - return read, nil -} - -func (s *s3Client) SourceDataPath() string { - return s.dataPath -} diff --git a/modules/backup-s3/config.go b/modules/backup-s3/config.go deleted file mode 100644 index f5f3552139a1df1ab2f3bd928b8b7be7b2349c8b..0000000000000000000000000000000000000000 --- a/modules/backup-s3/config.go +++ /dev/null @@ -1,31 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modstgs3 - -type clientConfig struct { - Endpoint string - Bucket string - UseSSL bool - - // this is an optional value, allowing for - // the backup to be stored in a specific - // directory inside the provided bucket - BackupPath string -} - -func newConfig(endpoint, bucket, path string, useSSL bool) *clientConfig { - const DEFAULT_ENDPOINT = "s3.amazonaws.com" - if endpoint == "" { - endpoint = DEFAULT_ENDPOINT - } - return &clientConfig{endpoint, bucket, useSSL, path} -} diff --git a/modules/backup-s3/module.go b/modules/backup-s3/module.go deleted file mode 100644 index c8fd6a6705dff316407c56d7a68d57b7842efd6b..0000000000000000000000000000000000000000 --- a/modules/backup-s3/module.go +++ /dev/null @@ -1,110 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modstgs3 - -import ( - "context" - "net/http" - "os" - "strings" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" -) - -const ( - Name = "backup-s3" - AltName1 = "s3" - s3Endpoint = "BACKUP_S3_ENDPOINT" - s3Bucket = "BACKUP_S3_BUCKET" - s3UseSSL = "BACKUP_S3_USE_SSL" - - // this is an optional value, allowing for - // the backup to be stored in a specific - // directory inside the provided bucket. - // - // if left unset, the backup files will - // be stored directly in the root of the - // bucket. - s3Path = "BACKUP_S3_PATH" -) - -type Module struct { - *s3Client - logger logrus.FieldLogger - dataPath string -} - -func New() *Module { - return &Module{} -} - -func (m *Module) Name() string { - return Name -} - -func (m *Module) IsExternal() bool { - return true -} - -func (m *Module) AltNames() []string { - return []string{AltName1} -} - -func (m *Module) Type() modulecapabilities.ModuleType { - return modulecapabilities.Backup -} - -func (m *Module) Init(ctx context.Context, - params moduletools.ModuleInitParams, -) error { - m.logger = params.GetLogger() - m.dataPath = params.GetStorageProvider().DataPath() - bucket := os.Getenv(s3Bucket) - if bucket == "" { - return errors.Errorf("backup init: '%s' must be set", s3Bucket) - } - // SSL on by default - useSSL := strings.ToLower(os.Getenv(s3UseSSL)) != "false" - config := newConfig(os.Getenv(s3Endpoint), bucket, os.Getenv(s3Path), useSSL) - client, err := newClient(config, m.logger, m.dataPath) - if err != nil { - return errors.Wrap(err, "initialize S3 backup module") - } - m.s3Client = client - return nil -} - -func (m *Module) RootHandler() http.Handler { - // TODO: remove once this is a capability interface - return nil -} - -func (m *Module) MetaInfo() (map[string]interface{}, error) { - metaInfo := make(map[string]interface{}, 4) - metaInfo["endpoint"] = m.config.Endpoint - metaInfo["bucketName"] = m.config.Bucket - if root := m.config.BackupPath; root != "" { - metaInfo["rootName"] = root - } - metaInfo["useSSL"] = m.config.UseSSL - return metaInfo, nil -} - -// verify we implement the modules.Module interface -var ( - _ = modulecapabilities.Module(New()) - _ = modulecapabilities.BackupBackend(New()) - _ = modulecapabilities.MetaProvider(New()) -) diff --git a/modules/generative-anyscale/clients/anyscale.go b/modules/generative-anyscale/clients/anyscale.go deleted file mode 100644 index a68d4db01c583bb160ab2ad9397757f3b73ab499..0000000000000000000000000000000000000000 --- a/modules/generative-anyscale/clients/anyscale.go +++ /dev/null @@ -1,217 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "regexp" - "strings" - "time" - - "github.com/weaviate/weaviate/usecases/modulecomponents" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/modules/generative-anyscale/config" - generativemodels "github.com/weaviate/weaviate/usecases/modulecomponents/additional/models" -) - -var compile, _ = regexp.Compile(`{([\w\s]*?)}`) - -type anyscale struct { - apiKey string - httpClient *http.Client - logger logrus.FieldLogger -} - -func New(apiKey string, timeout time.Duration, logger logrus.FieldLogger) *anyscale { - return &anyscale{ - apiKey: apiKey, - httpClient: &http.Client{ - Timeout: timeout, - }, - logger: logger, - } -} - -func (v *anyscale) GenerateSingleResult(ctx context.Context, textProperties map[string]string, prompt string, cfg moduletools.ClassConfig) (*generativemodels.GenerateResponse, error) { - forPrompt, err := v.generateForPrompt(textProperties, prompt) - if err != nil { - return nil, err - } - return v.Generate(ctx, cfg, forPrompt) -} - -func (v *anyscale) GenerateAllResults(ctx context.Context, textProperties []map[string]string, task string, cfg moduletools.ClassConfig) (*generativemodels.GenerateResponse, error) { - forTask, err := v.generatePromptForTask(textProperties, task) - if err != nil { - return nil, err - } - return v.Generate(ctx, cfg, forTask) -} - -func (v *anyscale) Generate(ctx context.Context, cfg moduletools.ClassConfig, prompt string) (*generativemodels.GenerateResponse, error) { - settings := config.NewClassSettings(cfg) - - anyscaleUrl := v.getAnyscaleUrl(ctx, settings.BaseURL()) - anyscalePrompt := []map[string]string{ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": prompt}, - } - input := generateInput{ - Messages: anyscalePrompt, - Model: settings.Model(), - Temperature: settings.Temperature(), - } - - body, err := json.Marshal(input) - if err != nil { - return nil, errors.Wrap(err, "marshal body") - } - - req, err := http.NewRequestWithContext(ctx, "POST", anyscaleUrl, - bytes.NewReader(body)) - if err != nil { - return nil, errors.Wrap(err, "create POST request") - } - apiKey, err := v.getApiKey(ctx) - if err != nil { - return nil, errors.Wrapf(err, "Anyscale (OpenAI) API Key") - } - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", apiKey)) - req.Header.Add("Content-Type", "application/json") - - res, err := v.httpClient.Do(req) - if err != nil { - return nil, errors.Wrap(err, "send POST request") - } - defer res.Body.Close() - - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - return nil, errors.Wrap(err, "read response body") - } - - var resBody generateResponse - if err := json.Unmarshal(bodyBytes, &resBody); err != nil { - return nil, errors.Wrap(err, "unmarshal response body") - } - - if res.StatusCode != 200 || resBody.Error != nil { - if resBody.Error != nil { - return nil, errors.Errorf("connection to Anyscale API failed with status: %d error: %v", res.StatusCode, resBody.Error.Message) - } - return nil, errors.Errorf("connection to Anyscale API failed with status: %d", res.StatusCode) - } - - textResponse := resBody.Choices[0].Message.Content - - return &generativemodels.GenerateResponse{ - Result: &textResponse, - }, nil -} - -func (v *anyscale) getAnyscaleUrl(ctx context.Context, baseURL string) string { - passedBaseURL := baseURL - if headerBaseURL := v.getValueFromContext(ctx, "X-Anyscale-Baseurl"); headerBaseURL != "" { - passedBaseURL = headerBaseURL - } - return fmt.Sprintf("%s/v1/chat/completions", passedBaseURL) -} - -func (v *anyscale) generatePromptForTask(textProperties []map[string]string, task string) (string, error) { - marshal, err := json.Marshal(textProperties) - if err != nil { - return "", err - } - return fmt.Sprintf(`'%v: -%v`, task, string(marshal)), nil -} - -func (v *anyscale) generateForPrompt(textProperties map[string]string, prompt string) (string, error) { - all := compile.FindAll([]byte(prompt), -1) - for _, match := range all { - originalProperty := string(match) - replacedProperty := compile.FindStringSubmatch(originalProperty)[1] - replacedProperty = strings.TrimSpace(replacedProperty) - value := textProperties[replacedProperty] - if value == "" { - return "", errors.Errorf("Following property has empty value: '%v'. Make sure you spell the property name correctly, verify that the property exists and has a value", replacedProperty) - } - prompt = strings.ReplaceAll(prompt, originalProperty, value) - } - return prompt, nil -} - -func (v *anyscale) getValueFromContext(ctx context.Context, key string) string { - if value := ctx.Value(key); value != nil { - if keyHeader, ok := value.([]string); ok && len(keyHeader) > 0 && len(keyHeader[0]) > 0 { - return keyHeader[0] - } - } - // try getting header from GRPC if not successful - if apiKey := modulecomponents.GetValueFromGRPC(ctx, key); len(apiKey) > 0 && len(apiKey[0]) > 0 { - return apiKey[0] - } - return "" -} - -func (v *anyscale) getApiKey(ctx context.Context) (string, error) { - // note Anyscale uses the OpenAI API Key in it's requests. - if apiKey := v.getValueFromContext(ctx, "X-Anyscale-Api-Key"); apiKey != "" { - return apiKey, nil - } - if v.apiKey != "" { - return v.apiKey, nil - } - return "", errors.New("no api key found " + - "neither in request header: X-Anyscale-Api-Key " + - "nor in environment variable under ANYSCALE_APIKEY") -} - -type generateInput struct { - Model string `json:"model"` - Messages []map[string]string `json:"messages"` - Temperature int `json:"temperature"` -} - -type Message struct { - Role string `json:"role"` - Content string `json:"content"` -} - -type Choice struct { - Message Message `json:"message"` - Index int `json:"index"` - FinishReason string `json:"finish_reason"` -} - -// The entire response for an error ends up looking different, may want to add omitempty everywhere. -type generateResponse struct { - ID string `json:"id"` - Object string `json:"object"` - Created int64 `json:"created"` - Model string `json:"model"` - Choices []Choice `json:"choices"` - Usage map[string]int `json:"usage"` - Error *anyscaleApiError `json:"error,omitempty"` -} - -type anyscaleApiError struct { - Message string `json:"message"` -} diff --git a/modules/generative-anyscale/clients/anyscale_meta.go b/modules/generative-anyscale/clients/anyscale_meta.go deleted file mode 100644 index ae4478f4e49c21288811e02c80811625544606a0..0000000000000000000000000000000000000000 --- a/modules/generative-anyscale/clients/anyscale_meta.go +++ /dev/null @@ -1,19 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -func (v *anyscale) MetaInfo() (map[string]interface{}, error) { - return map[string]interface{}{ - "name": "Generative Search - Anyscale", - "documentationHref": "https://docs.anyscale.com/endpoints/overview", - }, nil -} diff --git a/modules/generative-anyscale/clients/anyscale_meta_test.go b/modules/generative-anyscale/clients/anyscale_meta_test.go deleted file mode 100644 index cf2d5de1c508ccd35e36a24f2e3034cd3b8d1831..0000000000000000000000000000000000000000 --- a/modules/generative-anyscale/clients/anyscale_meta_test.go +++ /dev/null @@ -1,67 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func TestGetMeta(t *testing.T) { - t.Run("when the server is providing meta", func(t *testing.T) { - server := httptest.NewServer(&testMetaHandler{t: t}) - defer server.Close() - c := New(server.URL, 0, nullLogger()) - meta, err := c.MetaInfo() - - assert.Nil(t, err) - assert.NotNil(t, meta) - metaModel := meta["name"] - assert.True(t, metaModel != nil) - documentationHref := meta["documentationHref"] - assert.True(t, documentationHref != nil) - }) -} - -type testMetaHandler struct { - t *testing.T - // the test handler will report as not ready before the time has passed - readyTime time.Time -} - -func (f *testMetaHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - assert.Equal(f.t, "/meta", r.URL.String()) - assert.Equal(f.t, http.MethodGet, r.Method) - - if time.Since(f.readyTime) < 0 { - w.WriteHeader(http.StatusServiceUnavailable) - } - - w.Write([]byte(f.metaInfo())) -} - -func (f *testMetaHandler) metaInfo() string { - return `{ - "hostname": "http://127.0.0.1:8080", - "modules": { - "generative-cohere": { - "documentationHref": URL_HERE", - "name": "Cohere Generative Module" - } - }, - "version": "1.16.0" -}` -} diff --git a/modules/generative-anyscale/clients/anyscale_test.go b/modules/generative-anyscale/clients/anyscale_test.go deleted file mode 100644 index 6ab8c689c1921ff128d6f8d31b847c3dfd3124a5..0000000000000000000000000000000000000000 --- a/modules/generative-anyscale/clients/anyscale_test.go +++ /dev/null @@ -1,152 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "context" - "encoding/json" - "io" - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/sirupsen/logrus" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func nullLogger() logrus.FieldLogger { - l, _ := test.NewNullLogger() - return l -} - -func TestGetAnswer(t *testing.T) { - textProperties := []map[string]string{{"prop": "My name is john"}} - - tests := []struct { - name string - answer generateResponse - timeout time.Duration - expectedResult string - }{ - { - name: "when the server has a successful aner", - answer: generateResponse{ - Choices: []Choice{{Message: Message{Content: "John"}}}, - Error: nil, - }, - expectedResult: "John", - }, - { - name: "when the server has a an error", - answer: generateResponse{ - Error: &anyscaleApiError{ - Message: "some error from the server", - }, - }, - }, - { - name: "when the server does not respond in time", - answer: generateResponse{Error: &anyscaleApiError{Message: "context deadline exceeded"}}, - timeout: time.Second, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - handler := &testAnswerHandler{ - t: t, - answer: test.answer, - timeout: test.timeout, - } - server := httptest.NewServer(handler) - defer server.Close() - - c := New("apiKey", test.timeout, nullLogger()) - - settings := &fakeClassConfig{baseURL: server.URL} - res, err := c.GenerateAllResults(context.Background(), textProperties, "What is my name?", settings) - - if test.answer.Error != nil { - assert.Contains(t, err.Error(), test.answer.Error.Message) - } else { - assert.Equal(t, test.expectedResult, *res.Result) - } - }) - } - - t.Run("when X-Anyscale-BaseURL header is passed", func(t *testing.T) { - c := New("apiKey", 5*time.Second, nullLogger()) - baseUrl := "https://api.endpoints.anyscale.com" - buildURL := c.getAnyscaleUrl(context.Background(), baseUrl) - assert.Equal(t, "https://api.endpoints.anyscale.com/v1/chat/completions", buildURL) - }) -} - -type testAnswerHandler struct { - t *testing.T - // the test handler will report as not ready before the time has passed - answer generateResponse - timeout time.Duration -} - -func (f *testAnswerHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - assert.Equal(f.t, "/v1/chat/completions", r.URL.String()) - assert.Equal(f.t, http.MethodPost, r.Method) - - time.Sleep(f.timeout) - - if f.answer.Error != nil && f.answer.Error.Message != "" { - outBytes, err := json.Marshal(f.answer) - require.Nil(f.t, err) - - w.WriteHeader(http.StatusInternalServerError) - w.Write(outBytes) - return - } - - bodyBytes, err := io.ReadAll(r.Body) - require.Nil(f.t, err) - defer r.Body.Close() - - var b map[string]interface{} - require.Nil(f.t, json.Unmarshal(bodyBytes, &b)) - - outBytes, err := json.Marshal(f.answer) - require.Nil(f.t, err) - - w.Write(outBytes) -} - -type fakeClassConfig struct { - baseURL string -} - -func (cfg *fakeClassConfig) Tenant() string { - return "" -} - -func (cfg *fakeClassConfig) Class() map[string]interface{} { - return nil -} - -func (cfg *fakeClassConfig) ClassByModuleName(moduleName string) map[string]interface{} { - settings := map[string]interface{}{ - "baseURL": cfg.baseURL, - } - return settings -} - -func (cfg *fakeClassConfig) Property(propName string) map[string]interface{} { - return nil -} diff --git a/modules/generative-anyscale/config.go b/modules/generative-anyscale/config.go deleted file mode 100644 index 229681b5374a23b37464c1192c0d62bc4757f585..0000000000000000000000000000000000000000 --- a/modules/generative-anyscale/config.go +++ /dev/null @@ -1,41 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modgenerativeanyscale - -import ( - "context" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/modules/generative-anyscale/config" -) - -func (m *GenerativeAnyscaleModule) ClassConfigDefaults() map[string]interface{} { - return map[string]interface{}{} -} - -func (m *GenerativeAnyscaleModule) PropertyConfigDefaults( - dt *schema.DataType, -) map[string]interface{} { - return map[string]interface{}{} -} - -func (m *GenerativeAnyscaleModule) ValidateClass(ctx context.Context, - class *models.Class, cfg moduletools.ClassConfig, -) error { - settings := config.NewClassSettings(cfg) - return settings.Validate(class) -} - -var _ = modulecapabilities.ClassConfigurator(New()) diff --git a/modules/generative-anyscale/config/class_settings.go b/modules/generative-anyscale/config/class_settings.go deleted file mode 100644 index a5b919dcb9c72a82bc0b7f90615fdff7c8db5910..0000000000000000000000000000000000000000 --- a/modules/generative-anyscale/config/class_settings.go +++ /dev/null @@ -1,139 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package config - -import ( - "encoding/json" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/moduletools" -) - -const ( - baseURLProperty = "baseURL" - modelProperty = "model" - temperatureProperty = "temperature" -) - -var availableAnyscaleModels = []string{ - "meta-llama/Llama-2-70b-chat-hf", - "meta-llama/Llama-2-13b-chat-hf", - "meta-llama/Llama-2-7b-chat-hf", - "codellama/CodeLlama-34b-Instruct-hf", - "mistralai/Mistral-7B-Instruct-v0.1", - "mistralai/Mixtral-8x7B-Instruct-v0.1", -} - -// note we might want to separate the baseURL and completions URL in the future. Fine-tuned models also use this URL. 12/3/23 -var ( - DefaultBaseURL = "https://api.endpoints.anyscale.com" - DefaultAnyscaleModel = "meta-llama/Llama-2-70b-chat-hf" - DefaultAnyscaleTemperature = 0 -) - -type classSettings struct { - cfg moduletools.ClassConfig -} - -func NewClassSettings(cfg moduletools.ClassConfig) *classSettings { - return &classSettings{cfg: cfg} -} - -func (ic *classSettings) Validate(class *models.Class) error { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return errors.New("empty config") - } - model := ic.getStringProperty(modelProperty, DefaultAnyscaleModel) - if model == nil || !ic.validateModel(*model) { - return errors.Errorf("wrong Anyscale model name, available model names are: %v", availableAnyscaleModels) - } - - return nil -} - -func (ic *classSettings) getStringProperty(name, defaultValue string) *string { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return &defaultValue - } - - model, ok := ic.cfg.ClassByModuleName("generative-anyscale")[name] - if ok { - asString, ok := model.(string) - if ok { - return &asString - } - var empty string - return &empty - } - return &defaultValue -} - -func (ic *classSettings) getIntProperty(name string, defaultValue *int) *int { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return defaultValue - } - - val, ok := ic.cfg.ClassByModuleName("generative-anyscale")[name] - if ok { - asInt, ok := val.(int) - if ok { - return &asInt - } - asFloat, ok := val.(float64) - if ok { - asInt := int(asFloat) - return &asInt - } - asNumber, ok := val.(json.Number) - if ok { - asFloat, _ := asNumber.Float64() - asInt := int(asFloat) - return &asInt - } - var wrongVal int = -1 - return &wrongVal - } - - if defaultValue != nil { - return defaultValue - } - return nil -} - -func (ic *classSettings) validateModel(model string) bool { - return contains(availableAnyscaleModels, model) -} - -func (ic *classSettings) BaseURL() string { - return *ic.getStringProperty(baseURLProperty, DefaultBaseURL) -} - -func (ic *classSettings) Model() string { - return *ic.getStringProperty(modelProperty, DefaultAnyscaleModel) -} - -func (ic *classSettings) Temperature() int { - return *ic.getIntProperty(temperatureProperty, &DefaultAnyscaleTemperature) -} - -func contains[T comparable](s []T, e T) bool { - for _, v := range s { - if v == e { - return true - } - } - return false -} diff --git a/modules/generative-anyscale/config/class_settings_test.go b/modules/generative-anyscale/config/class_settings_test.go deleted file mode 100644 index c9e9d5141ccd442af9f4dd8a031253f2f364cdda..0000000000000000000000000000000000000000 --- a/modules/generative-anyscale/config/class_settings_test.go +++ /dev/null @@ -1,112 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package config - -import ( - "errors" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/entities/moduletools" -) - -func Test_classSettings_Validate(t *testing.T) { - tests := []struct { - name string - cfg moduletools.ClassConfig - wantModel string - wantTemperature int - wantBaseURL string - wantErr error - }{ - { - name: "default settings", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{}, - }, - wantModel: "meta-llama/Llama-2-70b-chat-hf", - wantTemperature: 0, - wantBaseURL: "https://api.endpoints.anyscale.com", - wantErr: nil, - }, - { - name: "everything non default configured", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "model": "meta-llama/Llama-2-70b-chat-hf", - "temperature": 1, - }, - }, - wantModel: "meta-llama/Llama-2-70b-chat-hf", - wantTemperature: 1, - wantBaseURL: "https://api.endpoints.anyscale.com", - wantErr: nil, - }, - { - name: "everything non default configured and base url", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "model": "meta-llama/Llama-2-70b-chat-hf", - "temperature": 1, - "baseURL": "https://custom.endpoint.com", - }, - }, - wantModel: "meta-llama/Llama-2-70b-chat-hf", - wantTemperature: 1, - wantBaseURL: "https://custom.endpoint.com", - wantErr: nil, - }, - { - name: "unsupported model", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "model": "unsupported", - "temperature": 1, - "baseURL": "https://custom.endpoint.com", - }, - }, - wantErr: errors.New("wrong Anyscale model name, available model names are: [meta-llama/Llama-2-70b-chat-hf meta-llama/Llama-2-13b-chat-hf meta-llama/Llama-2-7b-chat-hf codellama/CodeLlama-34b-Instruct-hf mistralai/Mistral-7B-Instruct-v0.1 mistralai/Mixtral-8x7B-Instruct-v0.1]"), - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ic := NewClassSettings(tt.cfg) - if tt.wantErr != nil { - assert.Equal(t, tt.wantErr.Error(), ic.Validate(nil).Error()) - } else { - assert.Equal(t, tt.wantBaseURL, ic.BaseURL()) - assert.Equal(t, tt.wantModel, ic.Model()) - assert.Equal(t, tt.wantTemperature, ic.Temperature()) - } - }) - } -} - -type fakeClassConfig struct { - classConfig map[string]interface{} -} - -func (f fakeClassConfig) Class() map[string]interface{} { - return f.classConfig -} - -func (f fakeClassConfig) Tenant() string { - return "" -} - -func (f fakeClassConfig) ClassByModuleName(moduleName string) map[string]interface{} { - return f.classConfig -} - -func (f fakeClassConfig) Property(propName string) map[string]interface{} { - return nil -} diff --git a/modules/generative-anyscale/module.go b/modules/generative-anyscale/module.go deleted file mode 100644 index e1f95c67539d73650ede73768884be87193b9da1..0000000000000000000000000000000000000000 --- a/modules/generative-anyscale/module.go +++ /dev/null @@ -1,97 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modgenerativeanyscale - -import ( - "context" - "net/http" - "os" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/modules/generative-anyscale/clients" - additionalprovider "github.com/weaviate/weaviate/usecases/modulecomponents/additional" - generativemodels "github.com/weaviate/weaviate/usecases/modulecomponents/additional/models" -) - -const Name = "generative-anyscale" - -func New() *GenerativeAnyscaleModule { - return &GenerativeAnyscaleModule{} -} - -type GenerativeAnyscaleModule struct { - generative generativeClient - additionalPropertiesProvider modulecapabilities.AdditionalProperties -} - -type generativeClient interface { - GenerateSingleResult(ctx context.Context, textProperties map[string]string, prompt string, cfg moduletools.ClassConfig) (*generativemodels.GenerateResponse, error) - GenerateAllResults(ctx context.Context, textProperties []map[string]string, task string, cfg moduletools.ClassConfig) (*generativemodels.GenerateResponse, error) - Generate(ctx context.Context, cfg moduletools.ClassConfig, prompt string) (*generativemodels.GenerateResponse, error) - MetaInfo() (map[string]interface{}, error) -} - -func (m *GenerativeAnyscaleModule) Name() string { - return Name -} - -func (m *GenerativeAnyscaleModule) Type() modulecapabilities.ModuleType { - return modulecapabilities.Text2TextGenerative -} - -func (m *GenerativeAnyscaleModule) Init(ctx context.Context, - params moduletools.ModuleInitParams, -) error { - if err := m.initAdditional(ctx, params.GetConfig().ModuleHttpClientTimeout, params.GetLogger()); err != nil { - return errors.Wrap(err, "init q/a") - } - - return nil -} - -func (m *GenerativeAnyscaleModule) initAdditional(ctx context.Context, timeout time.Duration, - logger logrus.FieldLogger, -) error { - apiKey := os.Getenv("ANYSCALE_APIKEY") - - client := clients.New(apiKey, timeout, logger) - - m.generative = client - - m.additionalPropertiesProvider = additionalprovider.NewGenerativeProvider(m.generative) - - return nil -} - -func (m *GenerativeAnyscaleModule) MetaInfo() (map[string]interface{}, error) { - return m.generative.MetaInfo() -} - -func (m *GenerativeAnyscaleModule) RootHandler() http.Handler { - // TODO: remove once this is a capability interface - return nil -} - -func (m *GenerativeAnyscaleModule) AdditionalProperties() map[string]modulecapabilities.AdditionalProperty { - return m.additionalPropertiesProvider.AdditionalProperties() -} - -// verify we implement the modules.Module interface -var ( - _ = modulecapabilities.Module(New()) - _ = modulecapabilities.AdditionalProperties(New()) - _ = modulecapabilities.MetaProvider(New()) -) diff --git a/modules/generative-aws/clients/aws.go b/modules/generative-aws/clients/aws.go deleted file mode 100644 index e1375888f135b52d28c2d8bbe366c11090460c5d..0000000000000000000000000000000000000000 --- a/modules/generative-aws/clients/aws.go +++ /dev/null @@ -1,433 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "regexp" - "strings" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/modules/generative-aws/config" - generativemodels "github.com/weaviate/weaviate/usecases/modulecomponents/additional/models" -) - -var compile, _ = regexp.Compile(`{([\w\s]*?)}`) - -func buildBedrockUrl(service, region, model string) string { - urlTemplate := "https://%s.%s.amazonaws.com/model/%s/invoke" - return fmt.Sprintf(urlTemplate, fmt.Sprintf("%s-runtime", service), region, model) -} - -func buildSagemakerUrl(service, region, endpoint string) string { - urlTemplate := "https://runtime.%s.%s.amazonaws.com/endpoints/%s/invocations" - return fmt.Sprintf(urlTemplate, service, region, endpoint) -} - -type aws struct { - awsAccessKey string - awsSecretKey string - buildBedrockUrlFn func(service, region, model string) string - buildSagemakerUrlFn func(service, region, endpoint string) string - httpClient *http.Client - logger logrus.FieldLogger -} - -func New(awsAccessKey string, awsSecretKey string, timeout time.Duration, logger logrus.FieldLogger) *aws { - return &aws{ - awsAccessKey: awsAccessKey, - awsSecretKey: awsSecretKey, - httpClient: &http.Client{ - Timeout: timeout, - }, - buildBedrockUrlFn: buildBedrockUrl, - buildSagemakerUrlFn: buildSagemakerUrl, - logger: logger, - } -} - -func (v *aws) GenerateSingleResult(ctx context.Context, textProperties map[string]string, prompt string, cfg moduletools.ClassConfig) (*generativemodels.GenerateResponse, error) { - forPrompt, err := v.generateForPrompt(textProperties, prompt) - if err != nil { - return nil, err - } - return v.Generate(ctx, cfg, forPrompt) -} - -func (v *aws) GenerateAllResults(ctx context.Context, textProperties []map[string]string, task string, cfg moduletools.ClassConfig) (*generativemodels.GenerateResponse, error) { - forTask, err := v.generatePromptForTask(textProperties, task) - if err != nil { - return nil, err - } - return v.Generate(ctx, cfg, forTask) -} - -func (v *aws) Generate(ctx context.Context, cfg moduletools.ClassConfig, prompt string) (*generativemodels.GenerateResponse, error) { - settings := config.NewClassSettings(cfg) - service := settings.Service() - region := settings.Region() - model := settings.Model() - endpoint := settings.Endpoint() - targetModel := settings.TargetModel() - targetVariant := settings.TargetVariant() - - var body []byte - var endpointUrl string - var host string - var path string - var err error - - headers := map[string]string{ - "accept": "*/*", - "content-type": contentType, - } - - if v.isBedrock(service) { - endpointUrl = v.buildBedrockUrlFn(service, region, model) - host = service + "-runtime" + "." + region + ".amazonaws.com" - path = "/model/" + model + "/invoke" - - if v.isAmazonModel(model) { - body, err = json.Marshal(bedrockAmazonGenerateRequest{ - InputText: prompt, - }) - } else if v.isAnthropicModel(model) { - var builder strings.Builder - builder.WriteString("\n\nHuman: ") - builder.WriteString(prompt) - builder.WriteString("\n\nAssistant:") - body, err = json.Marshal(bedrockAnthropicGenerateRequest{ - Prompt: builder.String(), - MaxTokensToSample: *settings.MaxTokenCount(), - Temperature: *settings.Temperature(), - TopK: *settings.TopK(), - TopP: settings.TopP(), - StopSequences: settings.StopSequences(), - AnthropicVersion: "bedrock-2023-05-31", - }) - } else if v.isAI21Model(model) { - body, err = json.Marshal(bedrockAI21GenerateRequest{ - Prompt: prompt, - MaxTokens: *settings.MaxTokenCount(), - Temperature: *settings.Temperature(), - TopP: settings.TopP(), - StopSequences: settings.StopSequences(), - }) - } else if v.isCohereModel(model) { - body, err = json.Marshal(bedrockCohereRequest{ - Prompt: prompt, - Temperature: *settings.Temperature(), - MaxTokens: *settings.MaxTokenCount(), - // ReturnLikeliHood: "GENERATION", // contray to docs, this is invalid - }) - } - - headers["x-amzn-bedrock-save"] = "false" - if err != nil { - return nil, errors.Wrapf(err, "marshal body") - } - } else if v.isSagemaker(service) { - endpointUrl = v.buildSagemakerUrlFn(service, region, endpoint) - host = "runtime." + service + "." + region + ".amazonaws.com" - path = "/endpoints/" + endpoint + "/invocations" - if targetModel != "" { - headers["x-amzn-sagemaker-target-model"] = targetModel - } - if targetVariant != "" { - headers["x-amzn-sagemaker-target-variant"] = targetVariant - } - body, err = json.Marshal(sagemakerGenerateRequest{ - Prompt: prompt, - }) - if err != nil { - return nil, errors.Wrapf(err, "marshal body") - } - } else { - return nil, errors.Wrapf(err, "service error") - } - - accessKey, err := v.getAwsAccessKey(ctx) - if err != nil { - return nil, errors.Wrapf(err, "AWS Access Key") - } - secretKey, err := v.getAwsAccessSecret(ctx) - if err != nil { - return nil, errors.Wrapf(err, "AWS Secret Key") - } - - headers["host"] = host - amzDate, headers, authorizationHeader := getAuthHeader(accessKey, secretKey, host, service, region, path, body, headers) - headers["Authorization"] = authorizationHeader - headers["x-amz-date"] = amzDate - - req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpointUrl, bytes.NewReader(body)) - if err != nil { - return nil, errors.Wrap(err, "create POST request") - } - - for k, v := range headers { - req.Header.Set(k, v) - } - - res, err := v.httpClient.Do(req) - if err != nil { - return nil, errors.Wrap(err, "send POST request") - } - defer res.Body.Close() - - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - return nil, errors.Wrap(err, "read response body") - } - - if v.isBedrock(service) { - return v.parseBedrockResponse(bodyBytes, res) - } else if v.isSagemaker(service) { - return v.parseSagemakerResponse(bodyBytes, res) - } else { - return &generativemodels.GenerateResponse{ - Result: nil, - }, nil - } -} - -func (v *aws) parseBedrockResponse(bodyBytes []byte, res *http.Response) (*generativemodels.GenerateResponse, error) { - var resBodyMap map[string]interface{} - if err := json.Unmarshal(bodyBytes, &resBodyMap); err != nil { - return nil, errors.Wrap(err, "unmarshal response body") - } - - var resBody bedrockGenerateResponse - if err := json.Unmarshal(bodyBytes, &resBody); err != nil { - return nil, errors.Wrap(err, "unmarshal response body") - } - - if res.StatusCode != 200 || resBody.Message != nil { - if resBody.Message != nil { - return nil, fmt.Errorf("connection to AWS Bedrock failed with status: %v error: %s", - res.StatusCode, *resBody.Message) - } - return nil, fmt.Errorf("connection to AWS Bedrock failed with status: %d", res.StatusCode) - } - - if len(resBody.Results) == 0 && len(resBody.Generations) == 0 { - return nil, fmt.Errorf("received empty response from AWS Bedrock") - } - - var content string - if len(resBody.Results) > 0 && len(resBody.Results[0].CompletionReason) > 0 { - content = resBody.Results[0].OutputText - } else if len(resBody.Generations) > 0 { - content = resBody.Generations[0].Text - } - - if content != "" { - return &generativemodels.GenerateResponse{ - Result: &content, - }, nil - } - - return &generativemodels.GenerateResponse{ - Result: nil, - }, nil -} - -func (v *aws) parseSagemakerResponse(bodyBytes []byte, res *http.Response) (*generativemodels.GenerateResponse, error) { - var resBody sagemakerGenerateResponse - if err := json.Unmarshal(bodyBytes, &resBody); err != nil { - return nil, errors.Wrap(err, "unmarshal response body") - } - - if res.StatusCode != 200 || resBody.Message != nil { - if resBody.Message != nil { - return nil, fmt.Errorf("connection to AWS Sagemaker failed with status: %v error: %s", - res.StatusCode, *resBody.Message) - } - return nil, fmt.Errorf("connection to AWS Sagemaker failed with status: %d", res.StatusCode) - } - - if len(resBody.Generations) == 0 { - return nil, fmt.Errorf("received empty response from AWS Sagemaker") - } - - if len(resBody.Generations) > 0 && len(resBody.Generations[0].Id) > 0 { - content := resBody.Generations[0].Text - if content != "" { - return &generativemodels.GenerateResponse{ - Result: &content, - }, nil - } - } - return &generativemodels.GenerateResponse{ - Result: nil, - }, nil -} - -func (v *aws) isSagemaker(service string) bool { - return service == "sagemaker" -} - -func (v *aws) isBedrock(service string) bool { - return service == "bedrock" -} - -func (v *aws) generatePromptForTask(textProperties []map[string]string, task string) (string, error) { - marshal, err := json.Marshal(textProperties) - if err != nil { - return "", err - } - return fmt.Sprintf(`'%v: -%v`, task, string(marshal)), nil -} - -func (v *aws) generateForPrompt(textProperties map[string]string, prompt string) (string, error) { - all := compile.FindAll([]byte(prompt), -1) - for _, match := range all { - originalProperty := string(match) - replacedProperty := compile.FindStringSubmatch(originalProperty)[1] - replacedProperty = strings.TrimSpace(replacedProperty) - value := textProperties[replacedProperty] - if value == "" { - return "", errors.Errorf("Following property has empty value: '%v'. Make sure you spell the property name correctly, verify that the property exists and has a value", replacedProperty) - } - prompt = strings.ReplaceAll(prompt, originalProperty, value) - } - return prompt, nil -} - -func (v *aws) getAwsAccessKey(ctx context.Context) (string, error) { - awsAccessKey := ctx.Value("X-Aws-Access-Key") - if awsAccessKeyHeader, ok := awsAccessKey.([]string); ok && - len(awsAccessKeyHeader) > 0 && len(awsAccessKeyHeader[0]) > 0 { - return awsAccessKeyHeader[0], nil - } - if len(v.awsAccessKey) > 0 { - return v.awsAccessKey, nil - } - return "", errors.New("no access key found " + - "neither in request header: X-AWS-Access-Key " + - "nor in environment variable under AWS_ACCESS_KEY_ID or AWS_ACCESS_KEY") -} - -func (v *aws) getAwsAccessSecret(ctx context.Context) (string, error) { - awsAccessSecret := ctx.Value("X-Aws-Secret-Key") - if awsAccessSecretHeader, ok := awsAccessSecret.([]string); ok && - len(awsAccessSecretHeader) > 0 && len(awsAccessSecretHeader[0]) > 0 { - return awsAccessSecretHeader[0], nil - } - if len(v.awsSecretKey) > 0 { - return v.awsSecretKey, nil - } - return "", errors.New("no secret found " + - "neither in request header: X-Aws-Secret-Key " + - "nor in environment variable under AWS_SECRET_ACCESS_KEY or AWS_SECRET_KEY") -} - -func (v *aws) isAmazonModel(model string) bool { - return strings.Contains(model, "amazon") -} - -func (v *aws) isAI21Model(model string) bool { - return strings.Contains(model, "ai21") -} - -func (v *aws) isAnthropicModel(model string) bool { - return strings.Contains(model, "anthropic") -} - -func (v *aws) isCohereModel(model string) bool { - return strings.Contains(model, "cohere") -} - -type bedrockAmazonGenerateRequest struct { - InputText string `json:"inputText,omitempty"` - TextGenerationConfig *textGenerationConfig `json:"textGenerationConfig,omitempty"` -} - -type bedrockAnthropicGenerateRequest struct { - Prompt string `json:"prompt,omitempty"` - MaxTokensToSample int `json:"max_tokens_to_sample,omitempty"` - Temperature float64 `json:"temperature,omitempty"` - TopK int `json:"top_k,omitempty"` - TopP *float64 `json:"top_p,omitempty"` - StopSequences []string `json:"stop_sequences,omitempty"` - AnthropicVersion string `json:"anthropic_version,omitempty"` -} - -type bedrockAI21GenerateRequest struct { - Prompt string `json:"prompt,omitempty"` - MaxTokens int `json:"maxTokens,omitempty"` - Temperature float64 `json:"temperature,omitempty"` - TopP *float64 `json:"top_p,omitempty"` - StopSequences []string `json:"stop_sequences,omitempty"` - CountPenalty penalty `json:"countPenalty,omitempty"` - PresencePenalty penalty `json:"presencePenalty,omitempty"` - FrequencyPenalty penalty `json:"frequencyPenalty,omitempty"` -} -type bedrockCohereRequest struct { - Prompt string `json:"prompt,omitempty"` - MaxTokens int `json:"max_tokens,omitempty"` - Temperature float64 `json:"temperature,omitempty"` - ReturnLikeliHood string `json:"return_likelihood,omitempty"` -} - -type penalty struct { - Scale int `json:"scale,omitempty"` -} - -type sagemakerGenerateRequest struct { - Prompt string `json:"prompt,omitempty"` -} - -type textGenerationConfig struct { - MaxTokenCount int `json:"maxTokenCount"` - StopSequences []string `json:"stopSequences"` - Temperature float64 `json:"temperature"` - TopP int `json:"topP"` -} - -type bedrockGenerateResponse struct { - InputTextTokenCount int `json:"InputTextTokenCount,omitempty"` - Results []Result `json:"results,omitempty"` - Generations []BedrockGeneration `json:"generations,omitempty"` - Message *string `json:"message,omitempty"` -} - -type sagemakerGenerateResponse struct { - Generations []Generation `json:"generations,omitempty"` - Message *string `json:"message,omitempty"` -} - -type Generation struct { - Id string `json:"id,omitempty"` - Text string `json:"text,omitempty"` -} - -type BedrockGeneration struct { - Id string `json:"id,omitempty"` - Text string `json:"text,omitempty"` - FinishReason string `json:"finish_reason,omitempty"` -} - -type Result struct { - TokenCount int `json:"tokenCount,omitempty"` - OutputText string `json:"outputText,omitempty"` - CompletionReason string `json:"completionReason,omitempty"` -} diff --git a/modules/generative-aws/clients/aws_meta.go b/modules/generative-aws/clients/aws_meta.go deleted file mode 100644 index 9e973117e2ef3fb796ff49cf1a702d09c49b3073..0000000000000000000000000000000000000000 --- a/modules/generative-aws/clients/aws_meta.go +++ /dev/null @@ -1,19 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -func (v *aws) MetaInfo() (map[string]interface{}, error) { - return map[string]interface{}{ - "name": "Generative Search - AWS", - "documentationHref": "https://docs.aws.amazon.com/bedrock/latest/APIReference/welcome.html", - }, nil -} diff --git a/modules/generative-aws/clients/aws_meta_test.go b/modules/generative-aws/clients/aws_meta_test.go deleted file mode 100644 index 4a1e7e7f8604444647a4dd28298233e5ec3533dd..0000000000000000000000000000000000000000 --- a/modules/generative-aws/clients/aws_meta_test.go +++ /dev/null @@ -1,67 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func TestGetMeta(t *testing.T) { - t.Run("when the server is providing meta", func(t *testing.T) { - server := httptest.NewServer(&testMetaHandler{t: t}) - defer server.Close() - c := New(server.URL, "", 60*time.Second, nullLogger()) - meta, err := c.MetaInfo() - - assert.Nil(t, err) - assert.NotNil(t, meta) - metaModel := meta["name"] - assert.True(t, metaModel != nil) - documentationHref := meta["documentationHref"] - assert.True(t, documentationHref != nil) - }) -} - -type testMetaHandler struct { - t *testing.T - // the test handler will report as not ready before the time has passed - readyTime time.Time -} - -func (f *testMetaHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - assert.Equal(f.t, "/meta", r.URL.String()) - assert.Equal(f.t, http.MethodGet, r.Method) - - if time.Since(f.readyTime) < 0 { - w.WriteHeader(http.StatusServiceUnavailable) - } - - w.Write([]byte(f.metaInfo())) -} - -func (f *testMetaHandler) metaInfo() string { - return `{ - "hostname": "http://127.0.0.1:8080", - "modules": { - "generative-aws": { - "documentationHref": "to be announced", - "name": "AWS Generative Module" - } - }, - "version": "1.16.0" -}` -} diff --git a/modules/generative-aws/clients/aws_test.go b/modules/generative-aws/clients/aws_test.go deleted file mode 100644 index 60a802cd66232d39f2dfac5565ceb9434853b712..0000000000000000000000000000000000000000 --- a/modules/generative-aws/clients/aws_test.go +++ /dev/null @@ -1,123 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "context" - "encoding/json" - "io" - "net/http" - "net/http/httptest" - "strings" - "testing" - - "github.com/sirupsen/logrus" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - generativemodels "github.com/weaviate/weaviate/usecases/modulecomponents/additional/models" -) - -func nullLogger() logrus.FieldLogger { - l, _ := test.NewNullLogger() - return l -} - -func TestGetAnswer(t *testing.T) { - t.Run("when the server has a successful answer ", func(t *testing.T) { - t.Skip("Skipping this test for now") - handler := &testAnswerHandler{ - t: t, - } - server := httptest.NewServer(handler) - defer server.Close() - - c := &aws{ - httpClient: &http.Client{}, - logger: nullLogger(), - awsAccessKey: "123", - awsSecretKey: "123", - buildBedrockUrlFn: func(service, region, model string) string { - return server.URL - }, - buildSagemakerUrlFn: func(service, region, endpoint string) string { - return server.URL - }, - } - - textProperties := []map[string]string{{"prop": "My name is john"}} - expected := generativemodels.GenerateResponse{ - Result: ptString("John"), - } - - res, err := c.GenerateAllResults(context.Background(), textProperties, "What is my name?", nil) - - assert.Nil(t, err) - assert.Equal(t, expected, res) - }) - - t.Run("when the server has a an error", func(t *testing.T) { - t.Skip("Skipping this test for now") - server := httptest.NewServer(&testAnswerHandler{ - t: t, - }) - defer server.Close() - - c := &aws{ - httpClient: &http.Client{}, - logger: nullLogger(), - awsAccessKey: "123", - awsSecretKey: "123", - buildBedrockUrlFn: func(service, region, model string) string { - return server.URL - }, - buildSagemakerUrlFn: func(service, region, endpoint string) string { - return server.URL - }, - } - - textProperties := []map[string]string{{"prop": "My name is john"}} - - _, err := c.GenerateAllResults(context.Background(), textProperties, "What is my name?", nil) - - require.NotNil(t, err) - assert.EqualError(t, err, "connection to AWS failed with status: 200 error: some error from the server") - }) -} - -type testAnswerHandler struct { - t *testing.T -} - -func (f *testAnswerHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - assert.Equal(f.t, http.MethodPost, r.Method) - - bodyBytes, err := io.ReadAll(r.Body) - require.Nil(f.t, err) - defer r.Body.Close() - - var outBytes []byte - authHeader := r.Header["Authorization"][0] - if strings.Contains(authHeader, "bedrock") { - var request bedrockAmazonGenerateRequest - require.Nil(f.t, json.Unmarshal(bodyBytes, &request)) - - outBytes, err = json.Marshal(request) - require.Nil(f.t, err) - } - - w.Write(outBytes) -} - -func ptString(in string) *string { - return &in -} diff --git a/modules/generative-aws/clients/signer.go b/modules/generative-aws/clients/signer.go deleted file mode 100644 index 134cbd36a09d3170339d340026bdeaa2be52f55c..0000000000000000000000000000000000000000 --- a/modules/generative-aws/clients/signer.go +++ /dev/null @@ -1,110 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "crypto/hmac" - "crypto/sha256" - "encoding/hex" - "fmt" - "net/http" - "sort" - "strings" - "time" -) - -const ( - contentType = "application/json" - algorithm = "AWS4-HMAC-SHA256" -) - -func getAuthHeader(awsAccessKey string, awsSecretKey string, host string, service string, region string, path string, body []byte, headers map[string]string) (string, map[string]string, string) { - t := time.Now().UTC() - amzDate := t.Format("20060102T150405Z") - shortDate := t.Format("20060102") - - hashedPayload := sha256Hash(body) - - canonicalHeaders, signedHeaders := getCanonicalHeaders(headers) - - canonicalRequest := strings.Join([]string{ - http.MethodPost, - path, - "", - canonicalHeaders, - signedHeaders, - hashedPayload, - }, "\n") - - hashedCanonicalRequest := sha256Hash([]byte(canonicalRequest)) - - credentialScope := strings.Join([]string{shortDate, region, service, "aws4_request"}, "/") - - stringToSign := strings.Join([]string{ - algorithm, - amzDate, - credentialScope, - hashedCanonicalRequest, - }, "\n") - - signingKey := getSigningKey(awsSecretKey, shortDate, region, service) - - signature := hmacSHA256(signingKey, stringToSign) - - authorizationHeader := fmt.Sprintf("%s Credential=%s/%s, SignedHeaders=%s, Signature=%s", algorithm, awsAccessKey, credentialScope, signedHeaders, signature) - return amzDate, headers, authorizationHeader -} - -func getCanonicalHeaders(headers map[string]string) (string, string) { - var canonicalHeaders []string - var signedHeaders []string - - keys := make([]string, 0, len(headers)) - for k := range headers { - keys = append(keys, k) - } - - sort.Strings(keys) - - for _, k := range keys { - canonicalHeaders = append(canonicalHeaders, fmt.Sprintf("%s:%s\n", strings.ToLower(k), headers[k])) - signedHeaders = append(signedHeaders, strings.ToLower(k)) - } - - return strings.Join(canonicalHeaders, ""), strings.Join(signedHeaders, ";") -} - -func hmacSHA256(key []byte, message string) string { - mac := hmac.New(sha256.New, key) - mac.Write([]byte(message)) - return hex.EncodeToString(mac.Sum(nil)) -} - -func getSigningKey(secretKey, date, region, service string) []byte { - key := "AWS4" + secretKey - kDate := hmacSHA256Bytes([]byte(key), date) - kRegion := hmacSHA256Bytes(kDate, region) - kService := hmacSHA256Bytes(kRegion, service) - kSigning := hmacSHA256Bytes(kService, "aws4_request") - return kSigning -} - -func hmacSHA256Bytes(key []byte, message string) []byte { - mac := hmac.New(sha256.New, key) - mac.Write([]byte(message)) - return mac.Sum(nil) -} - -func sha256Hash(body []byte) string { - hash := sha256.Sum256(body) - return hex.EncodeToString(hash[:]) -} diff --git a/modules/generative-aws/config.go b/modules/generative-aws/config.go deleted file mode 100644 index 9bf2e1765281365f0db172c0a814f8f24920151e..0000000000000000000000000000000000000000 --- a/modules/generative-aws/config.go +++ /dev/null @@ -1,41 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modgenerativeaws - -import ( - "context" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/modules/generative-aws/config" -) - -func (m *GenerativeAWSModule) ClassConfigDefaults() map[string]interface{} { - return map[string]interface{}{} -} - -func (m *GenerativeAWSModule) PropertyConfigDefaults( - dt *schema.DataType, -) map[string]interface{} { - return map[string]interface{}{} -} - -func (m *GenerativeAWSModule) ValidateClass(ctx context.Context, - class *models.Class, cfg moduletools.ClassConfig, -) error { - settings := config.NewClassSettings(cfg) - return settings.Validate(class) -} - -var _ = modulecapabilities.ClassConfigurator(New()) diff --git a/modules/generative-aws/config/class_settings.go b/modules/generative-aws/config/class_settings.go deleted file mode 100644 index 1e5206e1fa35dece24cea59c10e511708618abfc..0000000000000000000000000000000000000000 --- a/modules/generative-aws/config/class_settings.go +++ /dev/null @@ -1,372 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package config - -import ( - "encoding/json" - "fmt" - "strings" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/moduletools" -) - -const ( - serviceProperty = "service" - regionProperty = "region" - modelProperty = "model" - endpointProperty = "endpoint" - targetModelProperty = "targetModel" - targetVariantProperty = "targetVariant" - maxTokenCountProperty = "maxTokenCount" - maxTokensToSampleProperty = "maxTokensToSample" - stopSequencesProperty = "stopSequences" - temperatureProperty = "temperature" - topPProperty = "topP" - topKProperty = "topK" -) - -var ( - DefaultTitanMaxTokens = 8192 - DefaultTitanStopSequences = []string{} - DefaultTitanTemperature = 0.0 - DefaultTitanTopP = 1.0 - DefaultService = "bedrock" -) - -var ( - DefaultAnthropicMaxTokensToSample = 300 - DefaultAnthropicStopSequences = []string{"\\n\\nHuman:"} - DefaultAnthropicTemperature = 1.0 - DefaultAnthropicTopK = 250 - DefaultAnthropicTopP = 0.999 -) - -var DefaultAI21MaxTokens = 300 - -var ( - DefaultCohereMaxTokens = 100 - DefaultCohereTemperature = 0.8 - DefaultAI21Temperature = 0.7 - DefaultCohereTopP = 1.0 -) - -var availableAWSServices = []string{ - DefaultService, -} - -var availableBedrockModels = []string{ - "cohere.command-text-v14", - "cohere.command-light-text-v14", -} - -type classSettings struct { - cfg moduletools.ClassConfig -} - -func NewClassSettings(cfg moduletools.ClassConfig) *classSettings { - return &classSettings{cfg: cfg} -} - -func (ic *classSettings) Validate(class *models.Class) error { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return errors.New("empty config") - } - - var errorMessages []string - - service := ic.Service() - if service == "" || !ic.validatAvailableAWSSetting(service, availableAWSServices) { - errorMessages = append(errorMessages, fmt.Sprintf("wrong %s, available services are: %v", serviceProperty, availableAWSServices)) - } - region := ic.Region() - if region == "" { - errorMessages = append(errorMessages, fmt.Sprintf("%s cannot be empty", regionProperty)) - } - - if isBedrock(service) { - model := ic.Model() - if model == "" && !ic.validateAWSSetting(model, availableBedrockModels) { - errorMessages = append(errorMessages, fmt.Sprintf("wrong %s: %s, available model names are: %v", modelProperty, model, availableBedrockModels)) - } - - maxTokenCount := ic.MaxTokenCount() - if *maxTokenCount < 1 || *maxTokenCount > 8192 { - errorMessages = append(errorMessages, fmt.Sprintf("%s has to be an integer value between 1 and 8096", maxTokenCountProperty)) - } - temperature := ic.Temperature() - if *temperature < 0 || *temperature > 1 { - errorMessages = append(errorMessages, fmt.Sprintf("%s has to be float value between 0 and 1", temperatureProperty)) - } - topP := ic.TopP() - if topP != nil && (*topP < 0 || *topP > 1) { - errorMessages = append(errorMessages, fmt.Sprintf("%s has to be an integer value between 0 and 1", topPProperty)) - } - - endpoint := ic.Endpoint() - if endpoint != "" { - errorMessages = append(errorMessages, fmt.Sprintf("wrong configuration: %s, not applicable to %s", endpoint, service)) - } - } - - if isSagemaker(service) { - endpoint := ic.Endpoint() - if endpoint == "" { - errorMessages = append(errorMessages, fmt.Sprintf("%s cannot be empty", endpointProperty)) - } - model := ic.Model() - if model != "" { - errorMessages = append(errorMessages, fmt.Sprintf("wrong configuration: %s, not applicable to %s. did you mean %s", modelProperty, service, targetModelProperty)) - } - } - - if len(errorMessages) > 0 { - return fmt.Errorf("%s", strings.Join(errorMessages, ", ")) - } - - return nil -} - -func (ic *classSettings) validatAvailableAWSSetting(value string, availableValues []string) bool { - for i := range availableValues { - if value == availableValues[i] { - return true - } - } - return false -} - -func (ic *classSettings) validateAWSSetting(value string, availableValues []string) bool { - for i := range availableValues { - if value == availableValues[i] { - return true - } - } - return false -} - -func (ic *classSettings) getStringProperty(name, defaultValue string) string { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return defaultValue - } - - value, ok := ic.cfg.ClassByModuleName("generative-aws")[name] - if ok { - asString, ok := value.(string) - if ok { - return asString - } - } - return defaultValue -} - -func (ic *classSettings) getFloatProperty(name string, defaultValue *float64) *float64 { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return defaultValue - } - - val, ok := ic.cfg.ClassByModuleName("generative-aws")[name] - if ok { - asFloat, ok := val.(float64) - if ok { - return &asFloat - } - asNumber, ok := val.(json.Number) - if ok { - asFloat, _ := asNumber.Float64() - return &asFloat - } - asInt, ok := val.(int) - if ok { - asFloat := float64(asInt) - return &asFloat - } - } - - return defaultValue -} - -func (ic *classSettings) getIntProperty(name string, defaultValue *int) *int { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return defaultValue - } - - val, ok := ic.cfg.ClassByModuleName("generative-cohere")[name] - if ok { - asInt, ok := val.(int) - if ok { - return &asInt - } - asFloat, ok := val.(float64) - if ok { - asInt := int(asFloat) - return &asInt - } - asNumber, ok := val.(json.Number) - if ok { - asFloat, _ := asNumber.Float64() - asInt := int(asFloat) - return &asInt - } - wrongVal := -1 - return &wrongVal - } - - if defaultValue != nil { - return defaultValue - } - return nil -} - -func (ic *classSettings) getListOfStringsProperty(name string, defaultValue []string) *[]string { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return &defaultValue - } - - model, ok := ic.cfg.ClassByModuleName("generative-aws")[name] - if ok { - asStringList, ok := model.([]string) - if ok { - return &asStringList - } - var empty []string - return &empty - } - return &defaultValue -} - -// AWS params -func (ic *classSettings) Service() string { - return ic.getStringProperty(serviceProperty, DefaultService) -} - -func (ic *classSettings) Region() string { - return ic.getStringProperty(regionProperty, "") -} - -func (ic *classSettings) Model() string { - return ic.getStringProperty(modelProperty, "") -} - -func (ic *classSettings) MaxTokenCount() *int { - if isBedrock(ic.Service()) { - if isAmazonModel(ic.Model()) { - return ic.getIntProperty(maxTokenCountProperty, &DefaultTitanMaxTokens) - } - if isAnthropicModel(ic.Model()) { - return ic.getIntProperty(maxTokensToSampleProperty, &DefaultAnthropicMaxTokensToSample) - } - if isAI21Model(ic.Model()) { - return ic.getIntProperty(maxTokenCountProperty, &DefaultAI21MaxTokens) - } - if isCohereModel(ic.Model()) { - return ic.getIntProperty(maxTokenCountProperty, &DefaultCohereMaxTokens) - } - } - return ic.getIntProperty(maxTokenCountProperty, nil) -} - -func (ic *classSettings) StopSequences() []string { - if isBedrock(ic.Service()) { - if isAmazonModel(ic.Model()) { - return *ic.getListOfStringsProperty(stopSequencesProperty, DefaultTitanStopSequences) - } - if isAnthropicModel(ic.Model()) { - return *ic.getListOfStringsProperty(stopSequencesProperty, DefaultAnthropicStopSequences) - } - } - return *ic.getListOfStringsProperty(stopSequencesProperty, nil) -} - -func (ic *classSettings) Temperature() *float64 { - if isBedrock(ic.Service()) { - if isAmazonModel(ic.Model()) { - return ic.getFloatProperty(temperatureProperty, &DefaultTitanTemperature) - } - if isAnthropicModel(ic.Model()) { - return ic.getFloatProperty(temperatureProperty, &DefaultAnthropicTemperature) - } - if isCohereModel(ic.Model()) { - return ic.getFloatProperty(temperatureProperty, &DefaultCohereTemperature) - } - if isAI21Model(ic.Model()) { - return ic.getFloatProperty(temperatureProperty, &DefaultAI21Temperature) - } - } - return ic.getFloatProperty(temperatureProperty, nil) -} - -func (ic *classSettings) TopP() *float64 { - if isBedrock(ic.Service()) { - if isAmazonModel(ic.Model()) { - return ic.getFloatProperty(topPProperty, &DefaultTitanTopP) - } - if isAnthropicModel(ic.Model()) { - return ic.getFloatProperty(topPProperty, &DefaultAnthropicTopP) - } - if isCohereModel(ic.Model()) { - return ic.getFloatProperty(topPProperty, &DefaultCohereTopP) - } - } - return ic.getFloatProperty(topPProperty, nil) -} - -func (ic *classSettings) TopK() *int { - if isBedrock(ic.Service()) { - if isAnthropicModel(ic.Model()) { - return ic.getIntProperty(topKProperty, &DefaultAnthropicTopK) - } - } - return ic.getIntProperty(topKProperty, nil) -} - -func (ic *classSettings) Endpoint() string { - return ic.getStringProperty(endpointProperty, "") -} - -func (ic *classSettings) TargetModel() string { - return ic.getStringProperty(targetModelProperty, "") -} - -func (ic *classSettings) TargetVariant() string { - return ic.getStringProperty(targetVariantProperty, "") -} - -func isSagemaker(service string) bool { - return service == "sagemaker" -} - -func isBedrock(service string) bool { - return service == "bedrock" -} - -func isAmazonModel(model string) bool { - return strings.HasPrefix(model, "amazon") -} - -func isAI21Model(model string) bool { - return strings.HasPrefix(model, "ai21") -} - -func isAnthropicModel(model string) bool { - return strings.HasPrefix(model, "anthropic") -} - -func isCohereModel(model string) bool { - return strings.HasPrefix(model, "cohere") -} diff --git a/modules/generative-aws/config/class_settings_test.go b/modules/generative-aws/config/class_settings_test.go deleted file mode 100644 index 7dbca89e26953565fdd6235efe5404b9fe6f6317..0000000000000000000000000000000000000000 --- a/modules/generative-aws/config/class_settings_test.go +++ /dev/null @@ -1,204 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package config - -import ( - "testing" - - "github.com/pkg/errors" - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/entities/moduletools" -) - -func Test_classSettings_Validate(t *testing.T) { - t.Skip("Skipping this test for now") - tests := []struct { - name string - cfg moduletools.ClassConfig - wantService string - wantRegion string - wantModel string - wantEndpoint string - wantTargetModel string - wantTargetVariant string - wantMaxTokenCount int - wantStopSequences []string - wantTemperature float64 - wantTopP int - wantErr error - }{ - { - name: "happy flow - Bedrock", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "service": "bedrock", - "region": "us-east-1", - "model": "amazon.titan-tg1-large", - }, - }, - wantService: "bedrock", - wantRegion: "us-east-1", - wantModel: "amazon.titan-tg1-large", - wantMaxTokenCount: 8192, - wantStopSequences: []string{}, - wantTemperature: 0, - wantTopP: 1, - }, - { - name: "happy flow - Sagemaker", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "service": "sagemaker", - "region": "us-east-1", - "endpoint": "my-endpoint-deployment", - "targetModel": "model", - "targetVariant": "variant-1", - }, - }, - wantService: "sagemaker", - wantRegion: "us-east-1", - wantEndpoint: "my-endpoint-deployment", - wantTargetModel: "model", - wantTargetVariant: "variant-1", - }, - { - name: "custom values - Bedrock", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "service": "bedrock", - "region": "us-east-1", - "model": "amazon.titan-tg1-large", - "maxTokenCount": 1, - "stopSequences": []string{"test", "test2"}, - "temperature": 0.2, - "topP": 0, - }, - }, - wantService: "bedrock", - wantRegion: "us-east-1", - wantModel: "amazon.titan-tg1-large", - wantMaxTokenCount: 1, - wantStopSequences: []string{"test", "test2"}, - wantTemperature: 0.2, - wantTopP: 0, - }, - { - name: "custom values - Sagemaker", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "service": "sagemaker", - "region": "us-east-1", - "endpoint": "this-is-my-endpoint", - "targetModel": "my-target-model", - "targetVariant": "my-target¬variant", - }, - }, - wantService: "sagemaker", - wantRegion: "us-east-1", - wantEndpoint: "this-is-my-endpoint", - wantTargetModel: "my-target-model", - wantTargetVariant: "my-target¬variant", - }, - { - name: "wrong temperature", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "service": "bedrock", - "region": "us-east-1", - "model": "amazon.titan-tg1-large", - "temperature": 2, - }, - }, - wantErr: errors.Errorf("temperature has to be float value between 0 and 1"), - }, - { - name: "wrong maxTokenCount", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "service": "bedrock", - "region": "us-east-1", - "model": "amazon.titan-tg1-large", - "maxTokenCount": 9000, - }, - }, - wantErr: errors.Errorf("maxTokenCount has to be an integer value between 1 and 8096"), - }, - { - name: "wrong topP", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "service": "bedrock", - "region": "us-east-1", - "model": "amazon.titan-tg1-large", - "topP": 2000, - }, - }, - wantErr: errors.Errorf("topP has to be an integer value between 0 and 1"), - }, - { - name: "wrong all", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "maxTokenCount": 9000, - "temperature": 2, - "topP": 3, - }, - }, - wantErr: errors.Errorf("wrong service, " + - "available services are: [bedrock sagemaker], " + - "region cannot be empty", - ), - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ic := NewClassSettings(tt.cfg) - if tt.wantErr != nil { - assert.EqualError(t, ic.Validate(nil), tt.wantErr.Error()) - } else { - assert.Equal(t, tt.wantService, ic.Service()) - assert.Equal(t, tt.wantRegion, ic.Region()) - assert.Equal(t, tt.wantModel, ic.Model()) - assert.Equal(t, tt.wantEndpoint, ic.Endpoint()) - assert.Equal(t, tt.wantTargetModel, ic.TargetModel()) - assert.Equal(t, tt.wantTargetVariant, ic.TargetVariant()) - if ic.Temperature() != nil { - assert.Equal(t, tt.wantTemperature, *ic.Temperature()) - } - assert.Equal(t, tt.wantStopSequences, ic.StopSequences()) - if ic.TopP() != nil { - assert.Equal(t, tt.wantTopP, *ic.TopP()) - } - } - }) - } -} - -type fakeClassConfig struct { - classConfig map[string]interface{} -} - -func (f fakeClassConfig) Class() map[string]interface{} { - return f.classConfig -} - -func (f fakeClassConfig) Tenant() string { - return "" -} - -func (f fakeClassConfig) ClassByModuleName(moduleName string) map[string]interface{} { - return f.classConfig -} - -func (f fakeClassConfig) Property(propName string) map[string]interface{} { - return nil -} diff --git a/modules/generative-aws/module.go b/modules/generative-aws/module.go deleted file mode 100644 index a31448ba16c87b70f43d18bbc3a2b3f1cde09e75..0000000000000000000000000000000000000000 --- a/modules/generative-aws/module.go +++ /dev/null @@ -1,111 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modgenerativeaws - -import ( - "context" - "net/http" - "os" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/modules/generative-aws/clients" - additionalprovider "github.com/weaviate/weaviate/usecases/modulecomponents/additional" - generativemodels "github.com/weaviate/weaviate/usecases/modulecomponents/additional/models" -) - -const Name = "generative-aws" - -func New() *GenerativeAWSModule { - return &GenerativeAWSModule{} -} - -type GenerativeAWSModule struct { - generative generativeClient - additionalPropertiesProvider modulecapabilities.AdditionalProperties -} - -type generativeClient interface { - GenerateSingleResult(ctx context.Context, textProperties map[string]string, prompt string, cfg moduletools.ClassConfig) (*generativemodels.GenerateResponse, error) - GenerateAllResults(ctx context.Context, textProperties []map[string]string, task string, cfg moduletools.ClassConfig) (*generativemodels.GenerateResponse, error) - Generate(ctx context.Context, cfg moduletools.ClassConfig, prompt string) (*generativemodels.GenerateResponse, error) - MetaInfo() (map[string]interface{}, error) -} - -func (m *GenerativeAWSModule) Name() string { - return Name -} - -func (m *GenerativeAWSModule) Type() modulecapabilities.ModuleType { - return modulecapabilities.Text2TextGenerative -} - -func (m *GenerativeAWSModule) Init(ctx context.Context, - params moduletools.ModuleInitParams, -) error { - if err := m.initAdditional(ctx, params.GetConfig().ModuleHttpClientTimeout, params.GetLogger()); err != nil { - return errors.Wrap(err, "init q/a") - } - - return nil -} - -func (m *GenerativeAWSModule) initAdditional(ctx context.Context, timeout time.Duration, - logger logrus.FieldLogger, -) error { - awsAccessKey := m.getAWSAccessKey() - awsSecret := m.getAWSSecretAccessKey() - - client := clients.New(awsAccessKey, awsSecret, timeout, logger) - - m.generative = client - - m.additionalPropertiesProvider = additionalprovider.NewGenerativeProvider(m.generative) - - return nil -} - -func (m *GenerativeAWSModule) getAWSAccessKey() string { - if os.Getenv("AWS_ACCESS_KEY_ID") != "" { - return os.Getenv("AWS_ACCESS_KEY_ID") - } - return os.Getenv("AWS_ACCESS_KEY") -} - -func (m *GenerativeAWSModule) getAWSSecretAccessKey() string { - if os.Getenv("AWS_SECRET_ACCESS_KEY") != "" { - return os.Getenv("AWS_SECRET_ACCESS_KEY") - } - return os.Getenv("AWS_SECRET_KEY") -} - -func (m *GenerativeAWSModule) MetaInfo() (map[string]interface{}, error) { - return m.generative.MetaInfo() -} - -func (m *GenerativeAWSModule) RootHandler() http.Handler { - // TODO: remove once this is a capability interface - return nil -} - -func (m *GenerativeAWSModule) AdditionalProperties() map[string]modulecapabilities.AdditionalProperty { - return m.additionalPropertiesProvider.AdditionalProperties() -} - -// verify we implement the modules.Module interface -var ( - _ = modulecapabilities.Module(New()) - _ = modulecapabilities.AdditionalProperties(New()) -) diff --git a/modules/generative-cohere/clients/cohere.go b/modules/generative-cohere/clients/cohere.go deleted file mode 100644 index 8deb72a825c141b2de9816c03431358c146964ec..0000000000000000000000000000000000000000 --- a/modules/generative-cohere/clients/cohere.go +++ /dev/null @@ -1,213 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "regexp" - "strings" - "time" - - "github.com/weaviate/weaviate/usecases/modulecomponents" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/modules/generative-cohere/config" - generativemodels "github.com/weaviate/weaviate/usecases/modulecomponents/additional/models" -) - -var compile, _ = regexp.Compile(`{([\w\s]*?)}`) - -type cohere struct { - apiKey string - httpClient *http.Client - logger logrus.FieldLogger -} - -func New(apiKey string, timeout time.Duration, logger logrus.FieldLogger) *cohere { - return &cohere{ - apiKey: apiKey, - httpClient: &http.Client{ - Timeout: timeout, - }, - logger: logger, - } -} - -func (v *cohere) GenerateSingleResult(ctx context.Context, textProperties map[string]string, prompt string, cfg moduletools.ClassConfig) (*generativemodels.GenerateResponse, error) { - forPrompt, err := v.generateForPrompt(textProperties, prompt) - if err != nil { - return nil, err - } - return v.Generate(ctx, cfg, forPrompt) -} - -func (v *cohere) GenerateAllResults(ctx context.Context, textProperties []map[string]string, task string, cfg moduletools.ClassConfig) (*generativemodels.GenerateResponse, error) { - forTask, err := v.generatePromptForTask(textProperties, task) - if err != nil { - return nil, err - } - return v.Generate(ctx, cfg, forTask) -} - -func (v *cohere) Generate(ctx context.Context, cfg moduletools.ClassConfig, prompt string) (*generativemodels.GenerateResponse, error) { - settings := config.NewClassSettings(cfg) - - cohereUrl, err := v.getCohereUrl(ctx, settings.BaseURL()) - if err != nil { - return nil, errors.Wrap(err, "join Cohere API host and path") - } - input := generateInput{ - Prompt: prompt, - Model: settings.Model(), - MaxTokens: settings.MaxTokens(), - Temperature: settings.Temperature(), - K: settings.K(), - StopSequences: settings.StopSequences(), - ReturnLikelihoods: settings.ReturnLikelihoods(), - } - - body, err := json.Marshal(input) - if err != nil { - return nil, errors.Wrap(err, "marshal body") - } - - req, err := http.NewRequestWithContext(ctx, "POST", cohereUrl, - bytes.NewReader(body)) - if err != nil { - return nil, errors.Wrap(err, "create POST request") - } - apiKey, err := v.getApiKey(ctx) - if err != nil { - return nil, errors.Wrapf(err, "Cohere API Key") - } - req.Header.Add("Authorization", fmt.Sprintf("BEARER %s", apiKey)) - req.Header.Add("Content-Type", "application/json") - - res, err := v.httpClient.Do(req) - if err != nil { - return nil, errors.Wrap(err, "send POST request") - } - defer res.Body.Close() - - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - return nil, errors.Wrap(err, "read response body") - } - - var resBody generateResponse - if err := json.Unmarshal(bodyBytes, &resBody); err != nil { - return nil, errors.Wrap(err, "unmarshal response body") - } - - if res.StatusCode != 200 || resBody.Error != nil { - if resBody.Error != nil { - return nil, errors.Errorf("connection to Cohere API failed with status: %d error: %v", res.StatusCode, resBody.Error.Message) - } - return nil, errors.Errorf("connection to Cohere API failed with status: %d", res.StatusCode) - } - - textResponse := resBody.Generations[0].Text - - return &generativemodels.GenerateResponse{ - Result: &textResponse, - }, nil -} - -func (v *cohere) getCohereUrl(ctx context.Context, baseURL string) (string, error) { - passedBaseURL := baseURL - if headerBaseURL := v.getValueFromContext(ctx, "X-Cohere-Baseurl"); headerBaseURL != "" { - passedBaseURL = headerBaseURL - } - return url.JoinPath(passedBaseURL, "/v1/generate") -} - -func (v *cohere) generatePromptForTask(textProperties []map[string]string, task string) (string, error) { - marshal, err := json.Marshal(textProperties) - if err != nil { - return "", err - } - return fmt.Sprintf(`'%v: -%v`, task, string(marshal)), nil -} - -func (v *cohere) generateForPrompt(textProperties map[string]string, prompt string) (string, error) { - all := compile.FindAll([]byte(prompt), -1) - for _, match := range all { - originalProperty := string(match) - replacedProperty := compile.FindStringSubmatch(originalProperty)[1] - replacedProperty = strings.TrimSpace(replacedProperty) - value := textProperties[replacedProperty] - if value == "" { - return "", errors.Errorf("Following property has empty value: '%v'. Make sure you spell the property name correctly, verify that the property exists and has a value", replacedProperty) - } - prompt = strings.ReplaceAll(prompt, originalProperty, value) - } - return prompt, nil -} - -func (v *cohere) getValueFromContext(ctx context.Context, key string) string { - if value := ctx.Value(key); value != nil { - if keyHeader, ok := value.([]string); ok && len(keyHeader) > 0 && len(keyHeader[0]) > 0 { - return keyHeader[0] - } - } - // try getting header from GRPC if not successful - if apiKey := modulecomponents.GetValueFromGRPC(ctx, key); len(apiKey) > 0 && len(apiKey[0]) > 0 { - return apiKey[0] - } - return "" -} - -func (v *cohere) getApiKey(ctx context.Context) (string, error) { - if apiKey := v.getValueFromContext(ctx, "X-Cohere-Api-Key"); apiKey != "" { - return apiKey, nil - } - if v.apiKey != "" { - return v.apiKey, nil - } - return "", errors.New("no api key found " + - "neither in request header: X-Cohere-Api-Key " + - "nor in environment variable under COHERE_APIKEY") -} - -type generateInput struct { - Prompt string `json:"prompt"` - Model string `json:"model"` - MaxTokens int `json:"max_tokens"` - Temperature int `json:"temperature"` - K int `json:"k"` - StopSequences []string `json:"stop_sequences"` - ReturnLikelihoods string `json:"return_likelihoods"` -} - -type generateResponse struct { - Generations []generation - Error *cohereApiError `json:"error,omitempty"` -} - -type generation struct { - Text string `json:"text"` -} - -// need to check this -// I think you just get message -type cohereApiError struct { - Message string `json:"message"` -} diff --git a/modules/generative-cohere/clients/cohere_meta.go b/modules/generative-cohere/clients/cohere_meta.go deleted file mode 100644 index 47147d55ff4058ba7c415f0e03dcb4ebdb474a61..0000000000000000000000000000000000000000 --- a/modules/generative-cohere/clients/cohere_meta.go +++ /dev/null @@ -1,19 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -func (v *cohere) MetaInfo() (map[string]interface{}, error) { - return map[string]interface{}{ - "name": "Generative Search - Cohere", - "documentationHref": "https://docs.cohere.com/reference/generate", - }, nil -} diff --git a/modules/generative-cohere/clients/cohere_meta_test.go b/modules/generative-cohere/clients/cohere_meta_test.go deleted file mode 100644 index cf2d5de1c508ccd35e36a24f2e3034cd3b8d1831..0000000000000000000000000000000000000000 --- a/modules/generative-cohere/clients/cohere_meta_test.go +++ /dev/null @@ -1,67 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func TestGetMeta(t *testing.T) { - t.Run("when the server is providing meta", func(t *testing.T) { - server := httptest.NewServer(&testMetaHandler{t: t}) - defer server.Close() - c := New(server.URL, 0, nullLogger()) - meta, err := c.MetaInfo() - - assert.Nil(t, err) - assert.NotNil(t, meta) - metaModel := meta["name"] - assert.True(t, metaModel != nil) - documentationHref := meta["documentationHref"] - assert.True(t, documentationHref != nil) - }) -} - -type testMetaHandler struct { - t *testing.T - // the test handler will report as not ready before the time has passed - readyTime time.Time -} - -func (f *testMetaHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - assert.Equal(f.t, "/meta", r.URL.String()) - assert.Equal(f.t, http.MethodGet, r.Method) - - if time.Since(f.readyTime) < 0 { - w.WriteHeader(http.StatusServiceUnavailable) - } - - w.Write([]byte(f.metaInfo())) -} - -func (f *testMetaHandler) metaInfo() string { - return `{ - "hostname": "http://127.0.0.1:8080", - "modules": { - "generative-cohere": { - "documentationHref": URL_HERE", - "name": "Cohere Generative Module" - } - }, - "version": "1.16.0" -}` -} diff --git a/modules/generative-cohere/clients/cohere_test.go b/modules/generative-cohere/clients/cohere_test.go deleted file mode 100644 index b0dd7ef6f5879eaa003c792953878d557dedcd9d..0000000000000000000000000000000000000000 --- a/modules/generative-cohere/clients/cohere_test.go +++ /dev/null @@ -1,161 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "context" - "encoding/json" - "io" - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/sirupsen/logrus" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func nullLogger() logrus.FieldLogger { - l, _ := test.NewNullLogger() - return l -} - -func TestGetAnswer(t *testing.T) { - textProperties := []map[string]string{{"prop": "My name is john"}} - - tests := []struct { - name string - answer generateResponse - timeout time.Duration - expectedResult string - }{ - { - name: "when the server has a successful aner", - answer: generateResponse{ - Generations: []generation{{Text: "John"}}, - Error: nil, - }, - expectedResult: "John", - }, - { - name: "when the server has a an error", - answer: generateResponse{ - Error: &cohereApiError{ - Message: "some error from the server", - }, - }, - }, - { - name: "when the server does not respond in time", - answer: generateResponse{Error: &cohereApiError{Message: "context deadline exceeded"}}, - timeout: time.Second, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - handler := &testAnswerHandler{ - t: t, - answer: test.answer, - timeout: test.timeout, - } - server := httptest.NewServer(handler) - defer server.Close() - - c := New("apiKey", test.timeout, nullLogger()) - - settings := &fakeClassConfig{baseURL: server.URL} - res, err := c.GenerateAllResults(context.Background(), textProperties, "What is my name?", settings) - - if test.answer.Error != nil { - assert.Contains(t, err.Error(), test.answer.Error.Message) - } else { - assert.Equal(t, test.expectedResult, *res.Result) - } - }) - } - - t.Run("when X-Cohere-BaseURL header is passed", func(t *testing.T) { - c := New("apiKey", 5*time.Second, nullLogger()) - - baseURL := "http://default-url.com" - ctxWithValue := context.WithValue(context.Background(), - "X-Cohere-Baseurl", []string{"http://base-url-passed-in-header.com"}) - - buildURL, err := c.getCohereUrl(ctxWithValue, baseURL) - require.NoError(t, err) - assert.Equal(t, "http://base-url-passed-in-header.com/v1/generate", buildURL) - - buildURL, err = c.getCohereUrl(context.TODO(), baseURL) - require.NoError(t, err) - assert.Equal(t, "http://default-url.com/v1/generate", buildURL) - }) -} - -type testAnswerHandler struct { - t *testing.T - // the test handler will report as not ready before the time has passed - answer generateResponse - timeout time.Duration -} - -func (f *testAnswerHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - assert.Equal(f.t, "/v1/generate", r.URL.String()) - assert.Equal(f.t, http.MethodPost, r.Method) - - time.Sleep(f.timeout) - - if f.answer.Error != nil && f.answer.Error.Message != "" { - outBytes, err := json.Marshal(f.answer) - require.Nil(f.t, err) - - w.WriteHeader(http.StatusInternalServerError) - w.Write(outBytes) - return - } - - bodyBytes, err := io.ReadAll(r.Body) - require.Nil(f.t, err) - defer r.Body.Close() - - var b map[string]interface{} - require.Nil(f.t, json.Unmarshal(bodyBytes, &b)) - - outBytes, err := json.Marshal(f.answer) - require.Nil(f.t, err) - - w.Write(outBytes) -} - -type fakeClassConfig struct { - baseURL string -} - -func (cfg *fakeClassConfig) Tenant() string { - return "" -} - -func (cfg *fakeClassConfig) Class() map[string]interface{} { - return nil -} - -func (cfg *fakeClassConfig) ClassByModuleName(moduleName string) map[string]interface{} { - settings := map[string]interface{}{ - "baseURL": cfg.baseURL, - } - return settings -} - -func (cfg *fakeClassConfig) Property(propName string) map[string]interface{} { - return nil -} diff --git a/modules/generative-cohere/config.go b/modules/generative-cohere/config.go deleted file mode 100644 index 586f2de70fa59da29d8f2eeed4adb5d0a561342c..0000000000000000000000000000000000000000 --- a/modules/generative-cohere/config.go +++ /dev/null @@ -1,41 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modgenerativecohere - -import ( - "context" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/modules/generative-cohere/config" -) - -func (m *GenerativeCohereModule) ClassConfigDefaults() map[string]interface{} { - return map[string]interface{}{} -} - -func (m *GenerativeCohereModule) PropertyConfigDefaults( - dt *schema.DataType, -) map[string]interface{} { - return map[string]interface{}{} -} - -func (m *GenerativeCohereModule) ValidateClass(ctx context.Context, - class *models.Class, cfg moduletools.ClassConfig, -) error { - settings := config.NewClassSettings(cfg) - return settings.Validate(class) -} - -var _ = modulecapabilities.ClassConfigurator(New()) diff --git a/modules/generative-cohere/config/class_settings.go b/modules/generative-cohere/config/class_settings.go deleted file mode 100644 index 6388d222df4f20f40bed13f4e61613becd7e56c6..0000000000000000000000000000000000000000 --- a/modules/generative-cohere/config/class_settings.go +++ /dev/null @@ -1,182 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package config - -import ( - "encoding/json" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/moduletools" -) - -const ( - baseURLProperty = "baseURL" - modelProperty = "model" - temperatureProperty = "temperature" - maxTokensProperty = "maxTokens" - kProperty = "k" - stopSequencesProperty = "stopSequences" - returnLikelihoodsProperty = "returnLikelihoods" -) - -var availableCohereModels = []string{ - "command-xlarge-beta", - "command-xlarge", "command-medium", "command-xlarge-nightly", "command-medium-nightly", "xlarge", "medium", - "command", "command-light", "command-nightly", "command-light-nightly", "base", "base-light", -} - -// note it might not like this -- might want int values for e.g. MaxTokens -var ( - DefaultBaseURL = "https://api.cohere.ai" - DefaultCohereModel = "command-nightly" - DefaultCohereTemperature = 0 - DefaultCohereMaxTokens = 2048 - DefaultCohereK = 0 - DefaultCohereStopSequences = []string{} - DefaultCohereReturnLikelihoods = "NONE" -) - -type classSettings struct { - cfg moduletools.ClassConfig -} - -func NewClassSettings(cfg moduletools.ClassConfig) *classSettings { - return &classSettings{cfg: cfg} -} - -func (ic *classSettings) Validate(class *models.Class) error { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return errors.New("empty config") - } - model := ic.getStringProperty(modelProperty, DefaultCohereModel) - if model == nil || !ic.validateModel(*model) { - return errors.Errorf("wrong Cohere model name, available model names are: %v", availableCohereModels) - } - - return nil -} - -func (ic *classSettings) getStringProperty(name, defaultValue string) *string { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return &defaultValue - } - - model, ok := ic.cfg.ClassByModuleName("generative-cohere")[name] - if ok { - asString, ok := model.(string) - if ok { - return &asString - } - var empty string - return &empty - } - return &defaultValue -} - -func (ic *classSettings) getIntProperty(name string, defaultValue *int) *int { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return defaultValue - } - - val, ok := ic.cfg.ClassByModuleName("generative-cohere")[name] - if ok { - asInt, ok := val.(int) - if ok { - return &asInt - } - asFloat, ok := val.(float64) - if ok { - asInt := int(asFloat) - return &asInt - } - asNumber, ok := val.(json.Number) - if ok { - asFloat, _ := asNumber.Float64() - asInt := int(asFloat) - return &asInt - } - var wrongVal int = -1 - return &wrongVal - } - - if defaultValue != nil { - return defaultValue - } - return nil -} - -func (ic *classSettings) getListOfStringsProperty(name string, defaultValue []string) *[]string { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return &defaultValue - } - - model, ok := ic.cfg.ClassByModuleName("generative-cohere")[name] - if ok { - asStringList, ok := model.([]string) - if ok { - return &asStringList - } - var empty []string - return &empty - } - return &defaultValue -} - -func (ic *classSettings) GetMaxTokensForModel(model string) int { - return DefaultCohereMaxTokens -} - -func (ic *classSettings) validateModel(model string) bool { - return contains(availableCohereModels, model) -} - -func (ic *classSettings) BaseURL() string { - return *ic.getStringProperty(baseURLProperty, DefaultBaseURL) -} - -func (ic *classSettings) Model() string { - return *ic.getStringProperty(modelProperty, DefaultCohereModel) -} - -func (ic *classSettings) MaxTokens() int { - return *ic.getIntProperty(maxTokensProperty, &DefaultCohereMaxTokens) -} - -func (ic *classSettings) Temperature() int { - return *ic.getIntProperty(temperatureProperty, &DefaultCohereTemperature) -} - -func (ic *classSettings) K() int { - return *ic.getIntProperty(kProperty, &DefaultCohereK) -} - -func (ic *classSettings) StopSequences() []string { - return *ic.getListOfStringsProperty(stopSequencesProperty, DefaultCohereStopSequences) -} - -func (ic *classSettings) ReturnLikelihoods() string { - return *ic.getStringProperty(returnLikelihoodsProperty, DefaultCohereReturnLikelihoods) -} - -func contains[T comparable](s []T, e T) bool { - for _, v := range s { - if v == e { - return true - } - } - return false -} diff --git a/modules/generative-cohere/config/class_settings_test.go b/modules/generative-cohere/config/class_settings_test.go deleted file mode 100644 index ef713dc21d2b49ef11c0ade00ab70f52565f99b2..0000000000000000000000000000000000000000 --- a/modules/generative-cohere/config/class_settings_test.go +++ /dev/null @@ -1,150 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package config - -import ( - "testing" - - "github.com/pkg/errors" - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/entities/moduletools" -) - -func Test_classSettings_Validate(t *testing.T) { - tests := []struct { - name string - cfg moduletools.ClassConfig - wantModel string - wantMaxTokens int - wantTemperature int - wantK int - wantStopSequences []string - wantReturnLikelihoods string - wantBaseURL string - wantErr error - }{ - { - name: "default settings", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{}, - }, - wantModel: "command-nightly", - wantMaxTokens: 2048, - wantTemperature: 0, - wantK: 0, - wantStopSequences: []string{}, - wantReturnLikelihoods: "NONE", - wantBaseURL: "https://api.cohere.ai", - wantErr: nil, - }, - { - name: "everything non default configured", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "model": "command-xlarge", - "maxTokens": 2048, - "temperature": 1, - "k": 2, - "stopSequences": []string{"stop1", "stop2"}, - "returnLikelihoods": "NONE", - }, - }, - wantModel: "command-xlarge", - wantMaxTokens: 2048, - wantTemperature: 1, - wantK: 2, - wantStopSequences: []string{"stop1", "stop2"}, - wantReturnLikelihoods: "NONE", - wantBaseURL: "https://api.cohere.ai", - wantErr: nil, - }, - { - name: "wrong model configured", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "model": "wrong-model", - }, - }, - wantErr: errors.Errorf("wrong Cohere model name, available model names are: " + - "[command-xlarge-beta command-xlarge command-medium command-xlarge-nightly " + - "command-medium-nightly xlarge medium command command-light command-nightly command-light-nightly base base-light]"), - }, - { - name: "default settings with command-light-nightly", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "model": "command-light-nightly", - }, - }, - wantModel: "command-light-nightly", - wantMaxTokens: 2048, - wantTemperature: 0, - wantK: 0, - wantStopSequences: []string{}, - wantReturnLikelihoods: "NONE", - wantBaseURL: "https://api.cohere.ai", - wantErr: nil, - }, - { - name: "default settings with command-light-nightly and baseURL", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "model": "command-light-nightly", - "baseURL": "http://custom-url.com", - }, - }, - wantModel: "command-light-nightly", - wantMaxTokens: 2048, - wantTemperature: 0, - wantK: 0, - wantStopSequences: []string{}, - wantReturnLikelihoods: "NONE", - wantBaseURL: "http://custom-url.com", - wantErr: nil, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ic := NewClassSettings(tt.cfg) - if tt.wantErr != nil { - assert.Equal(t, tt.wantErr.Error(), ic.Validate(nil).Error()) - } else { - assert.Equal(t, tt.wantModel, ic.Model()) - assert.Equal(t, tt.wantMaxTokens, ic.MaxTokens()) - assert.Equal(t, tt.wantTemperature, ic.Temperature()) - assert.Equal(t, tt.wantK, ic.K()) - assert.Equal(t, tt.wantStopSequences, ic.StopSequences()) - assert.Equal(t, tt.wantReturnLikelihoods, ic.ReturnLikelihoods()) - } - }) - } -} - -type fakeClassConfig struct { - classConfig map[string]interface{} -} - -func (f fakeClassConfig) Class() map[string]interface{} { - return f.classConfig -} - -func (f fakeClassConfig) Tenant() string { - return "" -} - -func (f fakeClassConfig) ClassByModuleName(moduleName string) map[string]interface{} { - return f.classConfig -} - -func (f fakeClassConfig) Property(propName string) map[string]interface{} { - return nil -} diff --git a/modules/generative-cohere/module.go b/modules/generative-cohere/module.go deleted file mode 100644 index abf1c3e23751f099ab5290b5cdfa410797ab9e67..0000000000000000000000000000000000000000 --- a/modules/generative-cohere/module.go +++ /dev/null @@ -1,97 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modgenerativecohere - -import ( - "context" - "net/http" - "os" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/modules/generative-cohere/clients" - additionalprovider "github.com/weaviate/weaviate/usecases/modulecomponents/additional" - generativemodels "github.com/weaviate/weaviate/usecases/modulecomponents/additional/models" -) - -const Name = "generative-cohere" - -func New() *GenerativeCohereModule { - return &GenerativeCohereModule{} -} - -type GenerativeCohereModule struct { - generative generativeClient - additionalPropertiesProvider modulecapabilities.AdditionalProperties -} - -type generativeClient interface { - GenerateSingleResult(ctx context.Context, textProperties map[string]string, prompt string, cfg moduletools.ClassConfig) (*generativemodels.GenerateResponse, error) - GenerateAllResults(ctx context.Context, textProperties []map[string]string, task string, cfg moduletools.ClassConfig) (*generativemodels.GenerateResponse, error) - Generate(ctx context.Context, cfg moduletools.ClassConfig, prompt string) (*generativemodels.GenerateResponse, error) - MetaInfo() (map[string]interface{}, error) -} - -func (m *GenerativeCohereModule) Name() string { - return Name -} - -func (m *GenerativeCohereModule) Type() modulecapabilities.ModuleType { - return modulecapabilities.Text2TextGenerative -} - -func (m *GenerativeCohereModule) Init(ctx context.Context, - params moduletools.ModuleInitParams, -) error { - if err := m.initAdditional(ctx, params.GetConfig().ModuleHttpClientTimeout, params.GetLogger()); err != nil { - return errors.Wrap(err, "init q/a") - } - - return nil -} - -func (m *GenerativeCohereModule) initAdditional(ctx context.Context, timeout time.Duration, - logger logrus.FieldLogger, -) error { - apiKey := os.Getenv("COHERE_APIKEY") - - client := clients.New(apiKey, timeout, logger) - - m.generative = client - - m.additionalPropertiesProvider = additionalprovider.NewGenerativeProvider(m.generative) - - return nil -} - -func (m *GenerativeCohereModule) MetaInfo() (map[string]interface{}, error) { - return m.generative.MetaInfo() -} - -func (m *GenerativeCohereModule) RootHandler() http.Handler { - // TODO: remove once this is a capability interface - return nil -} - -func (m *GenerativeCohereModule) AdditionalProperties() map[string]modulecapabilities.AdditionalProperty { - return m.additionalPropertiesProvider.AdditionalProperties() -} - -// verify we implement the modules.Module interface -var ( - _ = modulecapabilities.Module(New()) - _ = modulecapabilities.AdditionalProperties(New()) - _ = modulecapabilities.MetaProvider(New()) -) diff --git a/modules/generative-openai/clients/openai.go b/modules/generative-openai/clients/openai.go deleted file mode 100644 index 7ace5148c702a93e6dc1c6be16bd94238ff53ef8..0000000000000000000000000000000000000000 --- a/modules/generative-openai/clients/openai.go +++ /dev/null @@ -1,348 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "regexp" - "strings" - "time" - - "github.com/weaviate/weaviate/usecases/modulecomponents" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/modules/generative-openai/config" - generativemodels "github.com/weaviate/weaviate/usecases/modulecomponents/additional/models" -) - -var compile, _ = regexp.Compile(`{([\w\s]*?)}`) - -func buildUrlFn(isLegacy bool, resourceName, deploymentID, baseURL string) (string, error) { - if resourceName != "" && deploymentID != "" { - host := "https://" + resourceName + ".openai.azure.com" - path := "openai/deployments/" + deploymentID + "/chat/completions" - queryParam := "api-version=2023-03-15-preview" - return fmt.Sprintf("%s/%s?%s", host, path, queryParam), nil - } - path := "/v1/chat/completions" - if isLegacy { - path = "/v1/completions" - } - return url.JoinPath(baseURL, path) -} - -type openai struct { - openAIApiKey string - openAIOrganization string - azureApiKey string - buildUrl func(isLegacy bool, resourceName, deploymentID, baseURL string) (string, error) - httpClient *http.Client - logger logrus.FieldLogger -} - -func New(openAIApiKey, openAIOrganization, azureApiKey string, timeout time.Duration, logger logrus.FieldLogger) *openai { - return &openai{ - openAIApiKey: openAIApiKey, - openAIOrganization: openAIOrganization, - azureApiKey: azureApiKey, - httpClient: &http.Client{ - Timeout: timeout, - }, - buildUrl: buildUrlFn, - logger: logger, - } -} - -func (v *openai) GenerateSingleResult(ctx context.Context, textProperties map[string]string, prompt string, cfg moduletools.ClassConfig) (*generativemodels.GenerateResponse, error) { - forPrompt, err := v.generateForPrompt(textProperties, prompt) - if err != nil { - return nil, err - } - return v.Generate(ctx, cfg, forPrompt) -} - -func (v *openai) GenerateAllResults(ctx context.Context, textProperties []map[string]string, task string, cfg moduletools.ClassConfig) (*generativemodels.GenerateResponse, error) { - forTask, err := v.generatePromptForTask(textProperties, task) - if err != nil { - return nil, err - } - return v.Generate(ctx, cfg, forTask) -} - -func (v *openai) Generate(ctx context.Context, cfg moduletools.ClassConfig, prompt string) (*generativemodels.GenerateResponse, error) { - settings := config.NewClassSettings(cfg) - - oaiUrl, err := v.buildOpenAIUrl(ctx, settings) - if err != nil { - return nil, errors.Wrap(err, "url join path") - } - - input, err := v.generateInput(prompt, settings) - if err != nil { - return nil, errors.Wrap(err, "generate input") - } - - body, err := json.Marshal(input) - if err != nil { - return nil, errors.Wrap(err, "marshal body") - } - - req, err := http.NewRequestWithContext(ctx, "POST", oaiUrl, - bytes.NewReader(body)) - if err != nil { - return nil, errors.Wrap(err, "create POST request") - } - apiKey, err := v.getApiKey(ctx, settings.IsAzure()) - if err != nil { - return nil, errors.Wrapf(err, "OpenAI API Key") - } - req.Header.Add(v.getApiKeyHeaderAndValue(apiKey, settings.IsAzure())) - if openAIOrganization := v.getOpenAIOrganization(ctx); openAIOrganization != "" { - req.Header.Add("OpenAI-Organization", openAIOrganization) - } - req.Header.Add("Content-Type", "application/json") - - res, err := v.httpClient.Do(req) - if err != nil { - return nil, errors.Wrap(err, "send POST request") - } - defer res.Body.Close() - - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - return nil, errors.Wrap(err, "read response body") - } - - var resBody generateResponse - if err := json.Unmarshal(bodyBytes, &resBody); err != nil { - return nil, errors.Wrap(err, "unmarshal response body") - } - - if res.StatusCode != 200 || resBody.Error != nil { - return nil, v.getError(res.StatusCode, resBody.Error, settings.IsAzure()) - } - - textResponse := resBody.Choices[0].Text - if len(resBody.Choices) > 0 && textResponse != "" { - trimmedResponse := strings.Trim(textResponse, "\n") - return &generativemodels.GenerateResponse{ - Result: &trimmedResponse, - }, nil - } - - message := resBody.Choices[0].Message - if message != nil { - textResponse = message.Content - trimmedResponse := strings.Trim(textResponse, "\n") - return &generativemodels.GenerateResponse{ - Result: &trimmedResponse, - }, nil - } - - return &generativemodels.GenerateResponse{ - Result: nil, - }, nil -} - -func (v *openai) buildOpenAIUrl(ctx context.Context, settings config.ClassSettings) (string, error) { - baseURL := settings.BaseURL() - if headerBaseURL := v.getValueFromContext(ctx, "X-Openai-Baseurl"); headerBaseURL != "" { - baseURL = headerBaseURL - } - return v.buildUrl(settings.IsLegacy(), settings.ResourceName(), settings.DeploymentID(), baseURL) -} - -func (v *openai) generateInput(prompt string, settings config.ClassSettings) (generateInput, error) { - if settings.IsLegacy() { - return generateInput{ - Prompt: prompt, - Model: settings.Model(), - MaxTokens: settings.MaxTokens(), - Temperature: settings.Temperature(), - FrequencyPenalty: settings.FrequencyPenalty(), - PresencePenalty: settings.PresencePenalty(), - TopP: settings.TopP(), - }, nil - } else { - var input generateInput - messages := []message{{ - Role: "user", - Content: prompt, - }} - tokens, err := v.determineTokens(settings.GetMaxTokensForModel(settings.Model()), settings.MaxTokens(), settings.Model(), messages) - if err != nil { - return input, errors.Wrap(err, "determine tokens count") - } - input = generateInput{ - Messages: messages, - MaxTokens: tokens, - Temperature: settings.Temperature(), - FrequencyPenalty: settings.FrequencyPenalty(), - PresencePenalty: settings.PresencePenalty(), - TopP: settings.TopP(), - } - if !settings.IsAzure() { - // model is mandatory for OpenAI calls, but obsolete for Azure calls - input.Model = settings.Model() - } - return input, nil - } -} - -func (v *openai) getError(statusCode int, resBodyError *openAIApiError, isAzure bool) error { - endpoint := "OpenAI API" - if isAzure { - endpoint = "Azure OpenAI API" - } - if resBodyError != nil { - return fmt.Errorf("connection to: %s failed with status: %d error: %v", endpoint, statusCode, resBodyError.Message) - } - return fmt.Errorf("connection to: %s failed with status: %d", endpoint, statusCode) -} - -func (v *openai) determineTokens(maxTokensSetting float64, classSetting float64, model string, messages []message) (float64, error) { - tokenMessagesCount, err := getTokensCount(model, messages) - if err != nil { - return 0, err - } - messageTokens := float64(tokenMessagesCount) - if messageTokens+classSetting >= maxTokensSetting { - // max token limit must be in range: [1, maxTokensSetting) that's why -1 is added - return maxTokensSetting - messageTokens - 1, nil - } - return messageTokens, nil -} - -func (v *openai) getApiKeyHeaderAndValue(apiKey string, isAzure bool) (string, string) { - if isAzure { - return "api-key", apiKey - } - return "Authorization", fmt.Sprintf("Bearer %s", apiKey) -} - -func (v *openai) generatePromptForTask(textProperties []map[string]string, task string) (string, error) { - marshal, err := json.Marshal(textProperties) - if err != nil { - return "", err - } - return fmt.Sprintf(`'%v: -%v`, task, string(marshal)), nil -} - -func (v *openai) generateForPrompt(textProperties map[string]string, prompt string) (string, error) { - all := compile.FindAll([]byte(prompt), -1) - for _, match := range all { - originalProperty := string(match) - replacedProperty := compile.FindStringSubmatch(originalProperty)[1] - replacedProperty = strings.TrimSpace(replacedProperty) - value := textProperties[replacedProperty] - if value == "" { - return "", errors.Errorf("Following property has empty value: '%v'. Make sure you spell the property name correctly, verify that the property exists and has a value", replacedProperty) - } - prompt = strings.ReplaceAll(prompt, originalProperty, value) - } - return prompt, nil -} - -func (v *openai) getApiKey(ctx context.Context, isAzure bool) (string, error) { - var apiKey, envVar string - - if isAzure { - apiKey = "X-Azure-Api-Key" - envVar = "AZURE_APIKEY" - if len(v.azureApiKey) > 0 { - return v.azureApiKey, nil - } - } else { - apiKey = "X-Openai-Api-Key" - envVar = "OPENAI_APIKEY" - if len(v.openAIApiKey) > 0 { - return v.openAIApiKey, nil - } - } - - return v.getApiKeyFromContext(ctx, apiKey, envVar) -} - -func (v *openai) getApiKeyFromContext(ctx context.Context, apiKey, envVar string) (string, error) { - if apiKeyValue := v.getValueFromContext(ctx, apiKey); apiKeyValue != "" { - return apiKeyValue, nil - } - return "", fmt.Errorf("no api key found neither in request header: %s nor in environment variable under %s", apiKey, envVar) -} - -func (v *openai) getValueFromContext(ctx context.Context, key string) string { - if value := ctx.Value(key); value != nil { - if keyHeader, ok := value.([]string); ok && len(keyHeader) > 0 && len(keyHeader[0]) > 0 { - return keyHeader[0] - } - } - // try getting header from GRPC if not successful - if apiKey := modulecomponents.GetValueFromGRPC(ctx, key); len(apiKey) > 0 && len(apiKey[0]) > 0 { - return apiKey[0] - } - - return "" -} - -func (v *openai) getOpenAIOrganization(ctx context.Context) string { - if value := v.getValueFromContext(ctx, "X-Openai-Organization"); value != "" { - return value - } - return v.openAIOrganization -} - -type generateInput struct { - Prompt string `json:"prompt,omitempty"` - Messages []message `json:"messages,omitempty"` - Model string `json:"model,omitempty"` - MaxTokens float64 `json:"max_tokens"` - Temperature float64 `json:"temperature"` - Stop []string `json:"stop"` - FrequencyPenalty float64 `json:"frequency_penalty"` - PresencePenalty float64 `json:"presence_penalty"` - TopP float64 `json:"top_p"` -} - -type message struct { - Role string `json:"role"` - Content string `json:"content"` - Name string `json:"name,omitempty"` -} - -type generateResponse struct { - Choices []choice - Error *openAIApiError `json:"error,omitempty"` -} - -type choice struct { - FinishReason string - Index float32 - Logprobs string - Text string `json:"text,omitempty"` - Message *message `json:"message,omitempty"` -} - -type openAIApiError struct { - Message string `json:"message"` - Type string `json:"type"` - Param string `json:"param"` - Code json.Number `json:"code"` -} diff --git a/modules/generative-openai/clients/openai_meta.go b/modules/generative-openai/clients/openai_meta.go deleted file mode 100644 index f7ba2cb6d9b946c9304f1ecf5b79609966fda9f4..0000000000000000000000000000000000000000 --- a/modules/generative-openai/clients/openai_meta.go +++ /dev/null @@ -1,19 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -func (v *openai) MetaInfo() (map[string]interface{}, error) { - return map[string]interface{}{ - "name": "Generative Search - OpenAI", - "documentationHref": "https://platform.openai.com/docs/api-reference/completions", - }, nil -} diff --git a/modules/generative-openai/clients/openai_meta_test.go b/modules/generative-openai/clients/openai_meta_test.go deleted file mode 100644 index bd9dfc99a276c798bd63eac26e32247b24377e52..0000000000000000000000000000000000000000 --- a/modules/generative-openai/clients/openai_meta_test.go +++ /dev/null @@ -1,67 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func TestGetMeta(t *testing.T) { - t.Run("when the server is providing meta", func(t *testing.T) { - server := httptest.NewServer(&testMetaHandler{t: t}) - defer server.Close() - c := New("", "", "", 0, nullLogger()) - meta, err := c.MetaInfo() - - assert.Nil(t, err) - assert.NotNil(t, meta) - metaModel := meta["name"] - assert.True(t, metaModel != nil) - documentationHref := meta["documentationHref"] - assert.True(t, documentationHref != nil) - }) -} - -type testMetaHandler struct { - t *testing.T - // the test handler will report as not ready before the time has passed - readyTime time.Time -} - -func (f *testMetaHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - assert.Equal(f.t, "/meta", r.URL.String()) - assert.Equal(f.t, http.MethodGet, r.Method) - - if time.Since(f.readyTime) < 0 { - w.WriteHeader(http.StatusServiceUnavailable) - } - - w.Write([]byte(f.metaInfo())) -} - -func (f *testMetaHandler) metaInfo() string { - return `{ - "hostname": "http://127.0.0.1:8080", - "modules": { - "generative-openai": { - "documentationHref": "https://platform.openai.com/docs/api-reference/completions", - "name": "OpenAI Generative Module" - } - }, - "version": "1.16.0" -}` -} diff --git a/modules/generative-openai/clients/openai_test.go b/modules/generative-openai/clients/openai_test.go deleted file mode 100644 index 14d7c8b1c9b471e1af742fa040921eb13fdbd85c..0000000000000000000000000000000000000000 --- a/modules/generative-openai/clients/openai_test.go +++ /dev/null @@ -1,287 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "context" - "encoding/json" - "io" - "net/http" - "net/http/httptest" - "os" - "strings" - "testing" - - "github.com/sirupsen/logrus" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/modules/generative-openai/config" - generativemodels "github.com/weaviate/weaviate/usecases/modulecomponents/additional/models" -) - -func nullLogger() logrus.FieldLogger { - l, _ := test.NewNullLogger() - return l -} - -func fakeBuildUrl(serverURL string, isLegacy bool, resourceName, deploymentID, baseURL string) (string, error) { - endpoint, err := buildUrlFn(isLegacy, resourceName, deploymentID, baseURL) - if err != nil { - return "", err - } - endpoint = strings.Replace(endpoint, "https://api.openai.com", serverURL, 1) - return endpoint, nil -} - -func TestBuildUrlFn(t *testing.T) { - t.Run("buildUrlFn returns default OpenAI Client", func(t *testing.T) { - url, err := buildUrlFn(false, "", "", config.DefaultOpenAIBaseURL) - assert.Nil(t, err) - assert.Equal(t, "https://api.openai.com/v1/chat/completions", url) - }) - t.Run("buildUrlFn returns Azure Client", func(t *testing.T) { - url, err := buildUrlFn(false, "resourceID", "deploymentID", "") - assert.Nil(t, err) - assert.Equal(t, "https://resourceID.openai.azure.com/openai/deployments/deploymentID/chat/completions?api-version=2023-03-15-preview", url) - }) - - t.Run("buildUrlFn loads from environment variable", func(t *testing.T) { - url, err := buildUrlFn(false, "", "", "https://foobar.some.proxy") - assert.Nil(t, err) - assert.Equal(t, "https://foobar.some.proxy/v1/chat/completions", url) - os.Unsetenv("OPENAI_BASE_URL") - }) -} - -func TestGetAnswer(t *testing.T) { - textProperties := []map[string]string{{"prop": "My name is john"}} - t.Run("when the server has a successful answer ", func(t *testing.T) { - handler := &testAnswerHandler{ - t: t, - answer: generateResponse{ - Choices: []choice{{ - FinishReason: "test", - Index: 0, - Logprobs: "", - Text: "John", - }}, - Error: nil, - }, - } - server := httptest.NewServer(handler) - defer server.Close() - - c := New("openAIApiKey", "", "", 0, nullLogger()) - c.buildUrl = func(isLegacy bool, resourceName, deploymentID, baseURL string) (string, error) { - return fakeBuildUrl(server.URL, isLegacy, resourceName, deploymentID, baseURL) - } - - expected := generativemodels.GenerateResponse{ - Result: ptString("John"), - } - - res, err := c.GenerateAllResults(context.Background(), textProperties, "What is my name?", nil) - - assert.Nil(t, err) - assert.Equal(t, expected, *res) - }) - - t.Run("when the server has a an error", func(t *testing.T) { - server := httptest.NewServer(&testAnswerHandler{ - t: t, - answer: generateResponse{ - Error: &openAIApiError{ - Message: "some error from the server", - }, - }, - }) - defer server.Close() - - c := New("openAIApiKey", "", "", 0, nullLogger()) - c.buildUrl = func(isLegacy bool, resourceName, deploymentID, baseURL string) (string, error) { - return fakeBuildUrl(server.URL, isLegacy, resourceName, deploymentID, baseURL) - } - - _, err := c.GenerateAllResults(context.Background(), textProperties, "What is my name?", nil) - - require.NotNil(t, err) - assert.Error(t, err, "connection to OpenAI failed with status: 500 error: some error from the server") - }) - - t.Run("when X-OpenAI-BaseURL header is passed", func(t *testing.T) { - settings := &fakeClassSettings{ - baseURL: "http://default-url.com", - } - c := New("openAIApiKey", "", "", 0, nullLogger()) - - ctxWithValue := context.WithValue(context.Background(), - "X-Openai-Baseurl", []string{"http://base-url-passed-in-header.com"}) - - buildURL, err := c.buildOpenAIUrl(ctxWithValue, settings) - require.NoError(t, err) - assert.Equal(t, "http://base-url-passed-in-header.com/v1/chat/completions", buildURL) - - buildURL, err = c.buildOpenAIUrl(context.TODO(), settings) - require.NoError(t, err) - assert.Equal(t, "http://default-url.com/v1/chat/completions", buildURL) - }) -} - -type testAnswerHandler struct { - t *testing.T - // the test handler will report as not ready before the time has passed - answer generateResponse -} - -func (f *testAnswerHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - assert.Equal(f.t, "/v1/chat/completions", r.URL.String()) - assert.Equal(f.t, http.MethodPost, r.Method) - - if f.answer.Error != nil && f.answer.Error.Message != "" { - outBytes, err := json.Marshal(f.answer) - require.Nil(f.t, err) - - w.WriteHeader(http.StatusInternalServerError) - w.Write(outBytes) - return - } - - bodyBytes, err := io.ReadAll(r.Body) - require.Nil(f.t, err) - defer r.Body.Close() - - var b map[string]interface{} - require.Nil(f.t, json.Unmarshal(bodyBytes, &b)) - - outBytes, err := json.Marshal(f.answer) - require.Nil(f.t, err) - - w.Write(outBytes) -} - -func TestOpenAIApiErrorDecode(t *testing.T) { - t.Run("getModelStringQuery", func(t *testing.T) { - type args struct { - response []byte - } - tests := []struct { - name string - args args - want string - }{ - { - name: "Error code: missing property", - args: args{ - response: []byte(`{"message": "failed", "type": "error", "param": "arg..."}`), - }, - want: "", - }, - { - name: "Error code: as int", - args: args{ - response: []byte(`{"message": "failed", "type": "error", "param": "arg...", "code": 500}`), - }, - want: "500", - }, - { - name: "Error code as string", - args: args{ - response: []byte(`{"message": "failed", "type": "error", "param": "arg...", "code": "500"}`), - }, - want: "500", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - var got *openAIApiError - err := json.Unmarshal(tt.args.response, &got) - require.NoError(t, err) - - if got.Code.String() != tt.want { - t.Errorf("OpenAIerror.code = %v, want %v", got.Code, tt.want) - } - }) - } - }) -} - -func ptString(in string) *string { - return &in -} - -type fakeClassSettings struct { - isLegacy bool - model string - maxTokens float64 - temperature float64 - frequencyPenalty float64 - presencePenalty float64 - topP float64 - resourceName string - deploymentID string - isAzure bool - baseURL string -} - -func (s *fakeClassSettings) IsLegacy() bool { - return s.isLegacy -} - -func (s *fakeClassSettings) Model() string { - return s.model -} - -func (s *fakeClassSettings) MaxTokens() float64 { - return s.maxTokens -} - -func (s *fakeClassSettings) Temperature() float64 { - return s.temperature -} - -func (s *fakeClassSettings) FrequencyPenalty() float64 { - return s.frequencyPenalty -} - -func (s *fakeClassSettings) PresencePenalty() float64 { - return s.presencePenalty -} - -func (s *fakeClassSettings) TopP() float64 { - return s.topP -} - -func (s *fakeClassSettings) ResourceName() string { - return s.resourceName -} - -func (s *fakeClassSettings) DeploymentID() string { - return s.deploymentID -} - -func (s *fakeClassSettings) IsAzure() bool { - return s.isAzure -} - -func (s *fakeClassSettings) GetMaxTokensForModel(model string) float64 { - return 0 -} - -func (s *fakeClassSettings) Validate(class *models.Class) error { - return nil -} - -func (s *fakeClassSettings) BaseURL() string { - return s.baseURL -} diff --git a/modules/generative-openai/clients/openai_tokens.go b/modules/generative-openai/clients/openai_tokens.go deleted file mode 100644 index cb3c003a16efaf01ecdd372055793109189ed6c5..0000000000000000000000000000000000000000 --- a/modules/generative-openai/clients/openai_tokens.go +++ /dev/null @@ -1,48 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "fmt" - "strings" - - "github.com/pkoukk/tiktoken-go" -) - -func getTokensCount(model string, messages []message) (int, error) { - tke, err := tiktoken.EncodingForModel(model) - if err != nil { - return 0, fmt.Errorf("encoding for model %s: %w", model, err) - } - - tokensPerMessage := 3 - if strings.HasPrefix(model, "gpt-3.5-turbo") { - tokensPerMessage = 4 - } - - tokensPerName := 1 - if strings.HasPrefix(model, "gpt-3.5-turbo") { - tokensPerName = -1 - } - - tokensCount := 3 - for _, message := range messages { - tokensCount += tokensPerMessage - tokensCount += len(tke.Encode(message.Role, nil, nil)) - tokensCount += len(tke.Encode(message.Content, nil, nil)) - if message.Name != "" { - tokensCount += tokensPerName - tokensCount += len(tke.Encode(message.Name, nil, nil)) - } - } - return tokensCount, nil -} diff --git a/modules/generative-openai/clients/openai_tokens_test.go b/modules/generative-openai/clients/openai_tokens_test.go deleted file mode 100644 index 843be4d59f61738765b8e07c1d7bef98dd8d0c71..0000000000000000000000000000000000000000 --- a/modules/generative-openai/clients/openai_tokens_test.go +++ /dev/null @@ -1,132 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func Test_getTokensCount(t *testing.T) { - prompt := ` - Summarize the following in a tweet: - - As generative language models such as GPT-4 continue to push the boundaries of what AI can do, - the excitement surrounding its potential is spreading quickly. Many applications and projects are - built on top of GPT-4 to extend its capabilities and features. Additionally, many tools were created - in order to interact with large language models, like LangChain as an example. Auto-GPT is one of the fastest - rising open-source python projects harnessing the power of GPT-4! - ` - messages := []message{ - {Role: "user", Content: prompt}, - } - // Example messages from: https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb - // added for sanity check that getTokensCount method computes tokens accordingly to above examples provided by OpenAI - exampleMessages := []message{ - { - Role: "system", - Content: "You are a helpful, pattern-following assistant that translates corporate jargon into plain English.", - }, - { - Role: "system", - Name: "example_user", - Content: "New synergies will help drive top-line growth.", - }, - { - Role: "system", - Name: "example_assistant", - Content: "Things working well together will increase revenue.", - }, - { - Role: "system", - Name: "example_user", - Content: "Let's circle back when we have more bandwidth to touch base on opportunities for increased leverage.", - }, - { - Role: "system", - Name: "example_assistant", - Content: "Let's talk later when we're less busy about how to do better.", - }, - { - Role: "user", - Content: "This late pivot means we don't have time to boil the ocean for the client deliverable.", - }, - } - tests := []struct { - name string - model string - messages []message - want int - wantErr string - }{ - { - name: "text-davinci-002", - model: "text-davinci-002", - messages: messages, - want: 128, - }, - { - name: "text-davinci-003", - model: "text-davinci-003", - messages: messages, - want: 128, - }, - { - name: "gpt-3.5-turbo", - model: "gpt-3.5-turbo", - messages: messages, - want: 122, - }, - { - name: "gpt-4", - model: "gpt-4", - messages: messages, - want: 121, - }, - { - name: "gpt-4-32k", - model: "gpt-4-32k", - messages: messages, - want: 121, - }, - { - name: "non-existent-model", - model: "non-existent-model", - messages: messages, - wantErr: "encoding for model non-existent-model: no encoding for model non-existent-model", - }, - { - name: "OpenAI cookbook example - gpt-3.5-turbo-0301", - model: "gpt-3.5-turbo-0301", - messages: exampleMessages, - want: 127, - }, - { - name: "OpenAI cookbook example - gpt-4", - model: "gpt-4", - messages: exampleMessages, - want: 129, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := getTokensCount(tt.model, tt.messages) - if err != nil { - assert.EqualError(t, err, tt.wantErr) - } else { - assert.Nil(t, err) - assert.Equal(t, tt.want, got) - } - }) - } -} diff --git a/modules/generative-openai/config.go b/modules/generative-openai/config.go deleted file mode 100644 index 309fc4c2a2398356748ec53623fa454e1fc3a03d..0000000000000000000000000000000000000000 --- a/modules/generative-openai/config.go +++ /dev/null @@ -1,41 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modgenerativeopenai - -import ( - "context" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/modules/generative-openai/config" -) - -func (m *GenerativeOpenAIModule) ClassConfigDefaults() map[string]interface{} { - return map[string]interface{}{} -} - -func (m *GenerativeOpenAIModule) PropertyConfigDefaults( - dt *schema.DataType, -) map[string]interface{} { - return map[string]interface{}{} -} - -func (m *GenerativeOpenAIModule) ValidateClass(ctx context.Context, - class *models.Class, cfg moduletools.ClassConfig, -) error { - settings := config.NewClassSettings(cfg) - return settings.Validate(class) -} - -var _ = modulecapabilities.ClassConfigurator(New()) diff --git a/modules/generative-openai/config/class_settings.go b/modules/generative-openai/config/class_settings.go deleted file mode 100644 index 4fde18717c3c970385e55ee46b1e28b5421ebe5d..0000000000000000000000000000000000000000 --- a/modules/generative-openai/config/class_settings.go +++ /dev/null @@ -1,253 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package config - -import ( - "encoding/json" - "fmt" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/moduletools" -) - -const ( - modelProperty = "model" - temperatureProperty = "temperature" - maxTokensProperty = "maxTokens" - frequencyPenaltyProperty = "frequencyPenalty" - presencePenaltyProperty = "presencePenalty" - topPProperty = "topP" - baseURLProperty = "baseURL" -) - -var availableOpenAILegacyModels = []string{ - "text-davinci-002", - "text-davinci-003", -} - -var availableOpenAIModels = []string{ - "gpt-3.5-turbo", - "gpt-3.5-turbo-16k", - "gpt-3.5-turbo-1106", - "gpt-4", - "gpt-4-32k", - "gpt-4-1106-preview", -} - -var ( - DefaultOpenAIModel = "gpt-3.5-turbo" - DefaultOpenAITemperature = 0.0 - DefaultOpenAIMaxTokens = defaultMaxTokens[DefaultOpenAIModel] - DefaultOpenAIFrequencyPenalty = 0.0 - DefaultOpenAIPresencePenalty = 0.0 - DefaultOpenAITopP = 1.0 - DefaultOpenAIBaseURL = "https://api.openai.com" -) - -// todo Need to parse the tokenLimits in a smarter way, as the prompt defines the max length -var defaultMaxTokens = map[string]float64{ - "text-davinci-002": 4097, - "text-davinci-003": 4097, - "gpt-3.5-turbo": 4097, - "gpt-3.5-turbo-16k": 16384, - "gpt-3.5-turbo-1106": 16385, - "gpt-4": 8192, - "gpt-4-32k": 32768, - "gpt-4-1106-preview": 128000, -} - -type ClassSettings interface { - IsLegacy() bool - Model() string - MaxTokens() float64 - Temperature() float64 - FrequencyPenalty() float64 - PresencePenalty() float64 - TopP() float64 - ResourceName() string - DeploymentID() string - IsAzure() bool - GetMaxTokensForModel(model string) float64 - Validate(class *models.Class) error - BaseURL() string -} - -type classSettings struct { - cfg moduletools.ClassConfig -} - -func NewClassSettings(cfg moduletools.ClassConfig) ClassSettings { - return &classSettings{cfg: cfg} -} - -func (ic *classSettings) Validate(class *models.Class) error { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return errors.New("empty config") - } - - model := ic.getStringProperty(modelProperty, DefaultOpenAIModel) - if model == nil || !ic.validateModel(*model) { - return errors.Errorf("wrong OpenAI model name, available model names are: %v", availableOpenAIModels) - } - - temperature := ic.getFloatProperty(temperatureProperty, &DefaultOpenAITemperature) - if temperature == nil || (*temperature < 0 || *temperature > 1) { - return errors.Errorf("Wrong temperature configuration, values are between 0.0 and 1.0") - } - - maxTokens := ic.getFloatProperty(maxTokensProperty, &DefaultOpenAIMaxTokens) - if maxTokens == nil || (*maxTokens < 0 || *maxTokens > ic.GetMaxTokensForModel(DefaultOpenAIModel)) { - return errors.Errorf("Wrong maxTokens configuration, values are should have a minimal value of 1 and max is dependant on the model used") - } - - frequencyPenalty := ic.getFloatProperty(frequencyPenaltyProperty, &DefaultOpenAIFrequencyPenalty) - if frequencyPenalty == nil || (*frequencyPenalty < 0 || *frequencyPenalty > 1) { - return errors.Errorf("Wrong frequencyPenalty configuration, values are between 0.0 and 1.0") - } - - presencePenalty := ic.getFloatProperty(presencePenaltyProperty, &DefaultOpenAIPresencePenalty) - if presencePenalty == nil || (*presencePenalty < 0 || *presencePenalty > 1) { - return errors.Errorf("Wrong presencePenalty configuration, values are between 0.0 and 1.0") - } - - topP := ic.getFloatProperty(topPProperty, &DefaultOpenAITopP) - if topP == nil || (*topP < 0 || *topP > 5) { - return errors.Errorf("Wrong topP configuration, values are should have a minimal value of 1 and max of 5") - } - - err := ic.validateAzureConfig(ic.ResourceName(), ic.DeploymentID()) - if err != nil { - return err - } - - return nil -} - -func (ic *classSettings) getStringProperty(name, defaultValue string) *string { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return &defaultValue - } - - model, ok := ic.cfg.ClassByModuleName("generative-openai")[name] - if ok { - asString, ok := model.(string) - if ok { - return &asString - } - var empty string - return &empty - } - return &defaultValue -} - -func (ic *classSettings) getFloatProperty(name string, defaultValue *float64) *float64 { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return defaultValue - } - - val, ok := ic.cfg.ClassByModuleName("generative-openai")[name] - if ok { - asFloat, ok := val.(float64) - if ok { - return &asFloat - } - asNumber, ok := val.(json.Number) - if ok { - asFloat, _ := asNumber.Float64() - return &asFloat - } - asInt, ok := val.(int) - if ok { - asFloat := float64(asInt) - return &asFloat - } - var wrongVal float64 = -1.0 - return &wrongVal - } - - if defaultValue != nil { - return defaultValue - } - return nil -} - -func (ic *classSettings) GetMaxTokensForModel(model string) float64 { - return defaultMaxTokens[model] -} - -func (ic *classSettings) validateModel(model string) bool { - return contains(availableOpenAIModels, model) || contains(availableOpenAILegacyModels, model) -} - -func (ic *classSettings) IsLegacy() bool { - return contains(availableOpenAILegacyModels, ic.Model()) -} - -func (ic *classSettings) Model() string { - return *ic.getStringProperty(modelProperty, DefaultOpenAIModel) -} - -func (ic *classSettings) MaxTokens() float64 { - return *ic.getFloatProperty(maxTokensProperty, &DefaultOpenAIMaxTokens) -} - -func (ic *classSettings) BaseURL() string { - return *ic.getStringProperty(baseURLProperty, DefaultOpenAIBaseURL) -} - -func (ic *classSettings) Temperature() float64 { - return *ic.getFloatProperty(temperatureProperty, &DefaultOpenAITemperature) -} - -func (ic *classSettings) FrequencyPenalty() float64 { - return *ic.getFloatProperty(frequencyPenaltyProperty, &DefaultOpenAIFrequencyPenalty) -} - -func (ic *classSettings) PresencePenalty() float64 { - return *ic.getFloatProperty(presencePenaltyProperty, &DefaultOpenAIPresencePenalty) -} - -func (ic *classSettings) TopP() float64 { - return *ic.getFloatProperty(topPProperty, &DefaultOpenAITopP) -} - -func (ic *classSettings) ResourceName() string { - return *ic.getStringProperty("resourceName", "") -} - -func (ic *classSettings) DeploymentID() string { - return *ic.getStringProperty("deploymentId", "") -} - -func (ic *classSettings) IsAzure() bool { - return ic.ResourceName() != "" && ic.DeploymentID() != "" -} - -func (ic *classSettings) validateAzureConfig(resourceName string, deploymentId string) error { - if (resourceName == "" && deploymentId != "") || (resourceName != "" && deploymentId == "") { - return fmt.Errorf("both resourceName and deploymentId must be provided") - } - return nil -} - -func contains[T comparable](s []T, e T) bool { - for _, v := range s { - if v == e { - return true - } - } - return false -} diff --git a/modules/generative-openai/config/class_settings_test.go b/modules/generative-openai/config/class_settings_test.go deleted file mode 100644 index 884561c7426d6834586e48cdc5d746a1cc68d05c..0000000000000000000000000000000000000000 --- a/modules/generative-openai/config/class_settings_test.go +++ /dev/null @@ -1,265 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package config - -import ( - "testing" - - "github.com/pkg/errors" - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/entities/moduletools" -) - -func Test_classSettings_Validate(t *testing.T) { - tests := []struct { - name string - cfg moduletools.ClassConfig - wantModel string - wantMaxTokens float64 - wantTemperature float64 - wantTopP float64 - wantFrequencyPenalty float64 - wantPresencePenalty float64 - wantResourceName string - wantDeploymentID string - wantIsAzure bool - wantErr error - wantBaseURL string - }{ - { - name: "Happy flow", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{}, - }, - wantModel: "gpt-3.5-turbo", - wantMaxTokens: 4097, - wantTemperature: 0.0, - wantTopP: 1, - wantFrequencyPenalty: 0.0, - wantPresencePenalty: 0.0, - wantErr: nil, - wantBaseURL: "https://api.openai.com", - }, - { - name: "Everything non default configured", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "model": "gpt-3.5-turbo", - "maxTokens": 4097, - "temperature": 0.5, - "topP": 3, - "frequencyPenalty": 0.1, - "presencePenalty": 0.9, - }, - }, - wantModel: "gpt-3.5-turbo", - wantMaxTokens: 4097, - wantTemperature: 0.5, - wantTopP: 3, - wantFrequencyPenalty: 0.1, - wantPresencePenalty: 0.9, - wantErr: nil, - wantBaseURL: "https://api.openai.com", - }, - { - name: "OpenAI Proxy", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "model": "gpt-3.5-turbo", - "maxTokens": 4097, - "temperature": 0.5, - "topP": 3, - "frequencyPenalty": 0.1, - "presencePenalty": 0.9, - "baseURL": "https://proxy.weaviate.dev/", - }, - }, - wantBaseURL: "https://proxy.weaviate.dev/", - wantModel: "gpt-3.5-turbo", - wantMaxTokens: 4097, - wantTemperature: 0.5, - wantTopP: 3, - wantFrequencyPenalty: 0.1, - wantPresencePenalty: 0.9, - wantErr: nil, - }, - { - name: "Legacy config", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "model": "text-davinci-003", - "maxTokens": 1200, - "temperature": 0.5, - "topP": 3, - "frequencyPenalty": 0.1, - "presencePenalty": 0.9, - }, - }, - wantModel: "text-davinci-003", - wantMaxTokens: 1200, - wantTemperature: 0.5, - wantTopP: 3, - wantFrequencyPenalty: 0.1, - wantPresencePenalty: 0.9, - wantErr: nil, - wantBaseURL: "https://api.openai.com", - }, - { - name: "Azure OpenAI config", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "resourceName": "weaviate", - "deploymentId": "gpt-3.5-turbo", - "maxTokens": 4097, - "temperature": 0.5, - "topP": 3, - "frequencyPenalty": 0.1, - "presencePenalty": 0.9, - }, - }, - wantResourceName: "weaviate", - wantDeploymentID: "gpt-3.5-turbo", - wantIsAzure: true, - wantModel: "gpt-3.5-turbo", - wantMaxTokens: 4097, - wantTemperature: 0.5, - wantTopP: 3, - wantFrequencyPenalty: 0.1, - wantPresencePenalty: 0.9, - wantErr: nil, - wantBaseURL: "https://api.openai.com", - }, - { - name: "With gpt-3.5-turbo-16k model", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "model": "gpt-3.5-turbo-16k", - "maxTokens": 4097, - "temperature": 0.5, - "topP": 3, - "frequencyPenalty": 0.1, - "presencePenalty": 0.9, - }, - }, - wantModel: "gpt-3.5-turbo-16k", - wantMaxTokens: 4097, - wantTemperature: 0.5, - wantTopP: 3, - wantFrequencyPenalty: 0.1, - wantPresencePenalty: 0.9, - wantErr: nil, - wantBaseURL: "https://api.openai.com", - }, - { - name: "Wrong maxTokens configured", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "maxTokens": true, - }, - }, - wantErr: errors.Errorf("Wrong maxTokens configuration, values are should have a minimal value of 1 and max is dependant on the model used"), - }, - { - name: "Wrong temperature configured", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "temperature": true, - }, - }, - wantErr: errors.Errorf("Wrong temperature configuration, values are between 0.0 and 1.0"), - }, - { - name: "Wrong frequencyPenalty configured", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "frequencyPenalty": true, - }, - }, - wantErr: errors.Errorf("Wrong frequencyPenalty configuration, values are between 0.0 and 1.0"), - }, - { - name: "Wrong presencePenalty configured", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "presencePenalty": true, - }, - }, - wantErr: errors.Errorf("Wrong presencePenalty configuration, values are between 0.0 and 1.0"), - }, - { - name: "Wrong topP configured", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "topP": true, - }, - }, - wantErr: errors.Errorf("Wrong topP configuration, values are should have a minimal value of 1 and max of 5"), - }, - { - name: "Wrong Azure config - empty deploymentId", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "resourceName": "resource-name", - }, - }, - wantErr: errors.Errorf("both resourceName and deploymentId must be provided"), - }, - { - name: "Wrong Azure config - empty resourceName", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "deploymentId": "deployment-name", - }, - }, - wantErr: errors.Errorf("both resourceName and deploymentId must be provided"), - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ic := NewClassSettings(tt.cfg) - if tt.wantErr != nil { - assert.EqualError(t, tt.wantErr, ic.Validate(nil).Error()) - } else { - assert.Equal(t, tt.wantModel, ic.Model()) - assert.Equal(t, tt.wantMaxTokens, ic.MaxTokens()) - assert.Equal(t, tt.wantTemperature, ic.Temperature()) - assert.Equal(t, tt.wantTopP, ic.TopP()) - assert.Equal(t, tt.wantFrequencyPenalty, ic.FrequencyPenalty()) - assert.Equal(t, tt.wantPresencePenalty, ic.PresencePenalty()) - assert.Equal(t, tt.wantResourceName, ic.ResourceName()) - assert.Equal(t, tt.wantDeploymentID, ic.DeploymentID()) - assert.Equal(t, tt.wantIsAzure, ic.IsAzure()) - assert.Equal(t, tt.wantBaseURL, ic.BaseURL()) - } - }) - } -} - -type fakeClassConfig struct { - classConfig map[string]interface{} -} - -func (f fakeClassConfig) Class() map[string]interface{} { - return f.classConfig -} - -func (f fakeClassConfig) Tenant() string { - return "" -} - -func (f fakeClassConfig) ClassByModuleName(moduleName string) map[string]interface{} { - return f.classConfig -} - -func (f fakeClassConfig) Property(propName string) map[string]interface{} { - return nil -} diff --git a/modules/generative-openai/module.go b/modules/generative-openai/module.go deleted file mode 100644 index c5869985221ed0428a73c3a0f72a7ba81d8b757e..0000000000000000000000000000000000000000 --- a/modules/generative-openai/module.go +++ /dev/null @@ -1,99 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modgenerativeopenai - -import ( - "context" - "net/http" - "os" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/modules/generative-openai/clients" - additionalprovider "github.com/weaviate/weaviate/usecases/modulecomponents/additional" - generativemodels "github.com/weaviate/weaviate/usecases/modulecomponents/additional/models" -) - -const Name = "generative-openai" - -func New() *GenerativeOpenAIModule { - return &GenerativeOpenAIModule{} -} - -type GenerativeOpenAIModule struct { - generative generativeClient - additionalPropertiesProvider modulecapabilities.AdditionalProperties -} - -type generativeClient interface { - GenerateSingleResult(ctx context.Context, textProperties map[string]string, prompt string, cfg moduletools.ClassConfig) (*generativemodels.GenerateResponse, error) - GenerateAllResults(ctx context.Context, textProperties []map[string]string, task string, cfg moduletools.ClassConfig) (*generativemodels.GenerateResponse, error) - Generate(ctx context.Context, cfg moduletools.ClassConfig, prompt string) (*generativemodels.GenerateResponse, error) - MetaInfo() (map[string]interface{}, error) -} - -func (m *GenerativeOpenAIModule) Name() string { - return Name -} - -func (m *GenerativeOpenAIModule) Type() modulecapabilities.ModuleType { - return modulecapabilities.Text2TextGenerative -} - -func (m *GenerativeOpenAIModule) Init(ctx context.Context, - params moduletools.ModuleInitParams, -) error { - if err := m.initAdditional(ctx, params.GetConfig().ModuleHttpClientTimeout, params.GetLogger()); err != nil { - return errors.Wrap(err, "init q/a") - } - - return nil -} - -func (m *GenerativeOpenAIModule) initAdditional(ctx context.Context, timeout time.Duration, - logger logrus.FieldLogger, -) error { - openAIApiKey := os.Getenv("OPENAI_APIKEY") - openAIOrganization := os.Getenv("OPENAI_ORGANIZATION") - azureApiKey := os.Getenv("AZURE_APIKEY") - - client := clients.New(openAIApiKey, openAIOrganization, azureApiKey, timeout, logger) - - m.generative = client - - m.additionalPropertiesProvider = additionalprovider.NewGenerativeProvider(m.generative) - - return nil -} - -func (m *GenerativeOpenAIModule) MetaInfo() (map[string]interface{}, error) { - return m.generative.MetaInfo() -} - -func (m *GenerativeOpenAIModule) RootHandler() http.Handler { - // TODO: remove once this is a capability interface - return nil -} - -func (m *GenerativeOpenAIModule) AdditionalProperties() map[string]modulecapabilities.AdditionalProperty { - return m.additionalPropertiesProvider.AdditionalProperties() -} - -// verify we implement the modules.Module interface -var ( - _ = modulecapabilities.Module(New()) - _ = modulecapabilities.AdditionalProperties(New()) - _ = modulecapabilities.MetaProvider(New()) -) diff --git a/modules/generative-palm/clients/palm.go b/modules/generative-palm/clients/palm.go deleted file mode 100644 index 6e277a469542abd9eb82a62c9747f492f8c89f43..0000000000000000000000000000000000000000 --- a/modules/generative-palm/clients/palm.go +++ /dev/null @@ -1,568 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "regexp" - "strings" - "time" - - "github.com/weaviate/weaviate/usecases/modulecomponents" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/modules/generative-palm/config" - generativemodels "github.com/weaviate/weaviate/usecases/modulecomponents/additional/models" -) - -type harmCategory string - -var ( - // Category is unspecified. - HarmCategoryUnspecified harmCategory = "HARM_CATEGORY_UNSPECIFIED" - // Negative or harmful comments targeting identity and/or protected attribute. - HarmCategoryDerogatory harmCategory = "HARM_CATEGORY_DEROGATORY" - // Content that is rude, disrepspectful, or profane. - HarmCategoryToxicity harmCategory = "HARM_CATEGORY_TOXICITY" - // Describes scenarios depictng violence against an individual or group, or general descriptions of gore. - HarmCategoryViolence harmCategory = "HARM_CATEGORY_VIOLENCE" - // Contains references to sexual acts or other lewd content. - HarmCategorySexual harmCategory = "HARM_CATEGORY_SEXUAL" - // Promotes unchecked medical advice. - HarmCategoryMedical harmCategory = "HARM_CATEGORY_MEDICAL" - // Dangerous content that promotes, facilitates, or encourages harmful acts. - HarmCategoryDangerous harmCategory = "HARM_CATEGORY_DANGEROUS" - // Harassment content. - HarmCategoryHarassment harmCategory = "HARM_CATEGORY_HARASSMENT" - // Hate speech and content. - HarmCategoryHate_speech harmCategory = "HARM_CATEGORY_HATE_SPEECH" - // Sexually explicit content. - HarmCategorySexually_explicit harmCategory = "HARM_CATEGORY_SEXUALLY_EXPLICIT" - // Dangerous content. - HarmCategoryDangerous_content harmCategory = "HARM_CATEGORY_DANGEROUS_CONTENT" -) - -type harmBlockThreshold string - -var ( - // Threshold is unspecified. - HarmBlockThresholdUnspecified harmBlockThreshold = "HARM_BLOCK_THRESHOLD_UNSPECIFIED" - // Content with NEGLIGIBLE will be allowed. - BlockLowAndAbove harmBlockThreshold = "BLOCK_LOW_AND_ABOVE" - // Content with NEGLIGIBLE and LOW will be allowed. - BlockMediumAndAbove harmBlockThreshold = "BLOCK_MEDIUM_AND_ABOVE" - // Content with NEGLIGIBLE, LOW, and MEDIUM will be allowed. - BlockOnlyHigh harmBlockThreshold = "BLOCK_ONLY_HIGH" - // All content will be allowed. - BlockNone harmBlockThreshold = "BLOCK_NONE" -) - -type harmProbability string - -var ( - // Probability is unspecified. - HARM_PROBABILITY_UNSPECIFIED harmProbability = "HARM_PROBABILITY_UNSPECIFIED" - // Content has a negligible chance of being unsafe. - NEGLIGIBLE harmProbability = "NEGLIGIBLE" - // Content has a low chance of being unsafe. - LOW harmProbability = "LOW" - // Content has a medium chance of being unsafe. - MEDIUM harmProbability = "MEDIUM" - // Content has a high chance of being unsafe. - HIGH harmProbability = "HIGH" -) - -var compile, _ = regexp.Compile(`{([\w\s]*?)}`) - -func buildURL(useGenerativeAI bool, apiEndoint, projectID, modelID string) string { - if useGenerativeAI { - // Generative AI endpoints, for more context check out this link: - // https://developers.generativeai.google/models/language#model_variations - // https://developers.generativeai.google/api/rest/generativelanguage/models/generateMessage - if strings.HasPrefix(modelID, "gemini") { - return fmt.Sprintf("https://generativelanguage.googleapis.com/v1beta/models/%s:generateContent", modelID) - } - return "https://generativelanguage.googleapis.com/v1beta2/models/chat-bison-001:generateMessage" - } - urlTemplate := "https://%s/v1/projects/%s/locations/us-central1/publishers/google/models/%s:predict" - return fmt.Sprintf(urlTemplate, apiEndoint, projectID, modelID) -} - -type palm struct { - apiKey string - buildUrlFn func(useGenerativeAI bool, apiEndoint, projectID, modelID string) string - httpClient *http.Client - logger logrus.FieldLogger -} - -func New(apiKey string, timeout time.Duration, logger logrus.FieldLogger) *palm { - return &palm{ - apiKey: apiKey, - httpClient: &http.Client{ - Timeout: timeout, - }, - buildUrlFn: buildURL, - logger: logger, - } -} - -func (v *palm) GenerateSingleResult(ctx context.Context, textProperties map[string]string, prompt string, cfg moduletools.ClassConfig) (*generativemodels.GenerateResponse, error) { - forPrompt, err := v.generateForPrompt(textProperties, prompt) - if err != nil { - return nil, err - } - return v.Generate(ctx, cfg, forPrompt) -} - -func (v *palm) GenerateAllResults(ctx context.Context, textProperties []map[string]string, task string, cfg moduletools.ClassConfig) (*generativemodels.GenerateResponse, error) { - forTask, err := v.generatePromptForTask(textProperties, task) - if err != nil { - return nil, err - } - return v.Generate(ctx, cfg, forTask) -} - -func (v *palm) Generate(ctx context.Context, cfg moduletools.ClassConfig, prompt string) (*generativemodels.GenerateResponse, error) { - settings := config.NewClassSettings(cfg) - - useGenerativeAIEndpoint := v.useGenerativeAIEndpoint(settings.ApiEndpoint()) - modelID := settings.ModelID() - if settings.EndpointID() != "" { - modelID = settings.EndpointID() - } - - endpointURL := v.buildUrlFn(useGenerativeAIEndpoint, settings.ApiEndpoint(), settings.ProjectID(), modelID) - input := v.getPayload(useGenerativeAIEndpoint, prompt, settings) - - body, err := json.Marshal(input) - if err != nil { - return nil, errors.Wrap(err, "marshal body") - } - - req, err := http.NewRequestWithContext(ctx, "POST", endpointURL, - bytes.NewReader(body)) - if err != nil { - return nil, errors.Wrap(err, "create POST request") - } - - apiKey, err := v.getApiKey(ctx) - if err != nil { - return nil, errors.Wrapf(err, "PaLM API Key") - } - req.Header.Add("Content-Type", "application/json") - if useGenerativeAIEndpoint { - req.Header.Add("x-goog-api-key", apiKey) - } else { - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", apiKey)) - } - - res, err := v.httpClient.Do(req) - if err != nil { - return nil, errors.Wrap(err, "send POST request") - } - defer res.Body.Close() - - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - return nil, errors.Wrap(err, "read response body") - } - - if useGenerativeAIEndpoint { - if strings.HasPrefix(modelID, "gemini") { - return v.parseGenerateContentResponse(res.StatusCode, bodyBytes) - } - return v.parseGenerateMessageResponse(res.StatusCode, bodyBytes) - } - return v.parseResponse(res.StatusCode, bodyBytes) -} - -func (v *palm) parseGenerateMessageResponse(statusCode int, bodyBytes []byte) (*generativemodels.GenerateResponse, error) { - var resBody generateMessageResponse - if err := json.Unmarshal(bodyBytes, &resBody); err != nil { - return nil, errors.Wrap(err, "unmarshal response body") - } - - if err := v.checkResponse(statusCode, resBody.Error); err != nil { - return nil, err - } - - if len(resBody.Candidates) > 0 { - return v.getGenerateResponse(resBody.Candidates[0].Content) - } - - return &generativemodels.GenerateResponse{ - Result: nil, - }, nil -} - -func (v *palm) parseGenerateContentResponse(statusCode int, bodyBytes []byte) (*generativemodels.GenerateResponse, error) { - var resBody generateContentResponse - if err := json.Unmarshal(bodyBytes, &resBody); err != nil { - return nil, errors.Wrap(err, "unmarshal response body") - } - - if err := v.checkResponse(statusCode, resBody.Error); err != nil { - return nil, err - } - - if len(resBody.Candidates) > 0 && len(resBody.Candidates[0].Content.Parts) > 0 { - return v.getGenerateResponse(resBody.Candidates[0].Content.Parts[0].Text) - } - - return &generativemodels.GenerateResponse{ - Result: nil, - }, nil -} - -func (v *palm) parseResponse(statusCode int, bodyBytes []byte) (*generativemodels.GenerateResponse, error) { - var resBody generateResponse - if err := json.Unmarshal(bodyBytes, &resBody); err != nil { - return nil, errors.Wrap(err, "unmarshal response body") - } - - if err := v.checkResponse(statusCode, resBody.Error); err != nil { - return nil, err - } - - if len(resBody.Predictions) > 0 && len(resBody.Predictions[0].Candidates) > 0 { - return v.getGenerateResponse(resBody.Predictions[0].Candidates[0].Content) - } - - return &generativemodels.GenerateResponse{ - Result: nil, - }, nil -} - -func (v *palm) getGenerateResponse(content string) (*generativemodels.GenerateResponse, error) { - if content != "" { - trimmedResponse := strings.Trim(content, "\n") - return &generativemodels.GenerateResponse{ - Result: &trimmedResponse, - }, nil - } - - return &generativemodels.GenerateResponse{ - Result: nil, - }, nil -} - -func (v *palm) checkResponse(statusCode int, palmApiError *palmApiError) error { - if statusCode != 200 || palmApiError != nil { - if palmApiError != nil { - return fmt.Errorf("connection to Google PaLM failed with status: %v error: %v", - statusCode, palmApiError.Message) - } - return fmt.Errorf("connection to Google PaLM failed with status: %d", statusCode) - } - return nil -} - -func (v *palm) useGenerativeAIEndpoint(apiEndpoint string) bool { - return apiEndpoint == "generativelanguage.googleapis.com" -} - -func (v *palm) getPayload(useGenerativeAI bool, prompt string, settings config.ClassSettings) any { - if useGenerativeAI { - if strings.HasPrefix(settings.ModelID(), "gemini") { - input := generateContentRequest{ - Contents: []content{ - { - Role: "user", - Parts: []part{ - { - Text: prompt, - }, - }, - }, - }, - GenerationConfig: &generationConfig{ - Temperature: settings.Temperature(), - TopP: settings.TopP(), - TopK: settings.TopK(), - CandidateCount: 1, - }, - SafetySettings: []safetySetting{ - { - Category: HarmCategoryHarassment, - Threshold: BlockMediumAndAbove, - }, - { - Category: HarmCategoryHate_speech, - Threshold: BlockMediumAndAbove, - }, - { - Category: HarmCategoryDangerous_content, - Threshold: BlockMediumAndAbove, - }, - { - Category: HarmCategoryDangerous_content, - Threshold: BlockMediumAndAbove, - }, - }, - } - return input - } - input := generateMessageRequest{ - Prompt: &generateMessagePrompt{ - Messages: []generateMessage{ - { - Content: prompt, - }, - }, - }, - Temperature: settings.Temperature(), - TopP: settings.TopP(), - TopK: settings.TopK(), - CandidateCount: 1, - } - return input - } - input := generateInput{ - Instances: []instance{ - { - Messages: []message{ - { - Content: prompt, - }, - }, - }, - }, - Parameters: parameters{ - Temperature: settings.Temperature(), - MaxOutputTokens: settings.TokenLimit(), - TopP: settings.TopP(), - TopK: settings.TopK(), - }, - } - return input -} - -func (v *palm) generatePromptForTask(textProperties []map[string]string, task string) (string, error) { - marshal, err := json.Marshal(textProperties) - if err != nil { - return "", err - } - return fmt.Sprintf(`'%v: -%v`, task, string(marshal)), nil -} - -func (v *palm) generateForPrompt(textProperties map[string]string, prompt string) (string, error) { - all := compile.FindAll([]byte(prompt), -1) - for _, match := range all { - originalProperty := string(match) - replacedProperty := compile.FindStringSubmatch(originalProperty)[1] - replacedProperty = strings.TrimSpace(replacedProperty) - value := textProperties[replacedProperty] - if value == "" { - return "", errors.Errorf("Following property has empty value: '%v'. Make sure you spell the property name correctly, verify that the property exists and has a value", replacedProperty) - } - prompt = strings.ReplaceAll(prompt, originalProperty, value) - } - return prompt, nil -} - -func (v *palm) getApiKey(ctx context.Context) (string, error) { - if apiKeyValue := v.getValueFromContext(ctx, "X-Palm-Api-Key"); apiKeyValue != "" { - return apiKeyValue, nil - } - if len(v.apiKey) > 0 { - return v.apiKey, nil - } - return "", errors.New("no api key found " + - "neither in request header: X-Palm-Api-Key " + - "nor in environment variable under PALM_APIKEY") -} - -func (v *palm) getValueFromContext(ctx context.Context, key string) string { - if value := ctx.Value(key); value != nil { - if keyHeader, ok := value.([]string); ok && len(keyHeader) > 0 && len(keyHeader[0]) > 0 { - return keyHeader[0] - } - } - // try getting header from GRPC if not successful - if apiKey := modulecomponents.GetValueFromGRPC(ctx, key); len(apiKey) > 0 && len(apiKey[0]) > 0 { - return apiKey[0] - } - return "" -} - -type generateInput struct { - Instances []instance `json:"instances,omitempty"` - Parameters parameters `json:"parameters"` -} - -type instance struct { - Context string `json:"context,omitempty"` - Messages []message `json:"messages,omitempty"` - Examples []example `json:"examples,omitempty"` -} - -type message struct { - Author string `json:"author"` - Content string `json:"content"` -} - -type example struct { - Input string `json:"input"` - Output string `json:"output"` -} - -type parameters struct { - Temperature float64 `json:"temperature"` - MaxOutputTokens int `json:"maxOutputTokens"` - TopP float64 `json:"topP"` - TopK int `json:"topK"` -} - -type generateResponse struct { - Predictions []prediction `json:"predictions,omitempty"` - Error *palmApiError `json:"error,omitempty"` - DeployedModelId string `json:"deployedModelId,omitempty"` - Model string `json:"model,omitempty"` - ModelDisplayName string `json:"modelDisplayName,omitempty"` - ModelVersionId string `json:"modelVersionId,omitempty"` -} - -type prediction struct { - Candidates []candidate `json:"candidates,omitempty"` - SafetyAttributes *[]safetyAttributes `json:"safetyAttributes,omitempty"` -} - -type candidate struct { - Author string `json:"author"` - Content string `json:"content"` -} - -type safetyAttributes struct { - Scores []float64 `json:"scores,omitempty"` - Blocked *bool `json:"blocked,omitempty"` - Categories []string `json:"categories,omitempty"` -} - -type palmApiError struct { - Code int `json:"code"` - Message string `json:"message"` - Status string `json:"status"` -} - -type generateMessageRequest struct { - Prompt *generateMessagePrompt `json:"prompt,omitempty"` - Temperature float64 `json:"temperature,omitempty"` - CandidateCount int `json:"candidateCount,omitempty"` // default 1 - TopP float64 `json:"topP"` - TopK int `json:"topK"` -} - -type generateMessagePrompt struct { - Context string `json:"prompt,omitempty"` - Examples []generateExample `json:"examples,omitempty"` - Messages []generateMessage `json:"messages,omitempty"` -} - -type generateMessage struct { - Author string `json:"author,omitempty"` - Content string `json:"content,omitempty"` - CitationMetadata *generateCitationMetadata `json:"citationMetadata,omitempty"` -} - -type generateCitationMetadata struct { - CitationSources []generateCitationSource `json:"citationSources,omitempty"` -} - -type generateCitationSource struct { - StartIndex int `json:"startIndex,omitempty"` - EndIndex int `json:"endIndex,omitempty"` - URI string `json:"uri,omitempty"` - License string `json:"license,omitempty"` -} - -type generateExample struct { - Input *generateMessage `json:"input,omitempty"` - Output *generateMessage `json:"output,omitempty"` -} - -type generateMessageResponse struct { - Candidates []generateMessage `json:"candidates,omitempty"` - Messages []generateMessage `json:"messages,omitempty"` - Filters []contentFilter `json:"filters,omitempty"` - Error *palmApiError `json:"error,omitempty"` -} - -type contentFilter struct { - Reason string `json:"reason,omitempty"` - Message string `json:"message,omitempty"` -} - -type generateContentRequest struct { - Contents []content `json:"contents,omitempty"` - SafetySettings []safetySetting `json:"safetySettings,omitempty"` - GenerationConfig *generationConfig `json:"generationConfig,omitempty"` -} - -type content struct { - Parts []part `json:"parts,omitempty"` - Role string `json:"role,omitempty"` -} - -type part struct { - Text string `json:"text,omitempty"` - InlineData string `json:"inline_data,omitempty"` -} - -type safetySetting struct { - Category harmCategory `json:"category,omitempty"` - Threshold harmBlockThreshold `json:"threshold,omitempty"` -} - -type generationConfig struct { - StopSequences []string `json:"stopSequences,omitempty"` - CandidateCount int `json:"candidateCount,omitempty"` - MaxOutputTokens int `json:"maxOutputTokens,omitempty"` - Temperature float64 `json:"temperature,omitempty"` - TopP float64 `json:"topP,omitempty"` - TopK int `json:"topK,omitempty"` -} - -type generateContentResponse struct { - Candidates []generateContentCandidate `json:"candidates,omitempty"` - PromptFeedback *promptFeedback `json:"promptFeedback,omitempty"` - Error *palmApiError `json:"error,omitempty"` -} - -type generateContentCandidate struct { - Content contentResponse `json:"content,omitempty"` - FinishReason string `json:"finishReason,omitempty"` - Index int `json:"index,omitempty"` - SafetyRatings []safetyRating `json:"safetyRatings,omitempty"` -} - -type contentResponse struct { - Parts []part `json:"parts,omitempty"` - Role string `json:"role,omitempty"` -} - -type promptFeedback struct { - SafetyRatings []safetyRating `json:"safetyRatings,omitempty"` -} - -type safetyRating struct { - Category harmCategory `json:"category,omitempty"` - Probability harmProbability `json:"probability,omitempty"` - Blocked *bool `json:"blocked,omitempty"` -} diff --git a/modules/generative-palm/clients/palm_meta.go b/modules/generative-palm/clients/palm_meta.go deleted file mode 100644 index 188d8ed7d380c5413195a5275d6eda9bc0c33661..0000000000000000000000000000000000000000 --- a/modules/generative-palm/clients/palm_meta.go +++ /dev/null @@ -1,19 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -func (v *palm) MetaInfo() (map[string]interface{}, error) { - return map[string]interface{}{ - "name": "Generative Search - Google PaLM", - "documentationHref": "https://cloud.google.com/vertex-ai/docs/generative-ai/chat/test-chat-prompts", - }, nil -} diff --git a/modules/generative-palm/clients/palm_meta_test.go b/modules/generative-palm/clients/palm_meta_test.go deleted file mode 100644 index b2aa42a46f2e4ce9fac39322d34f1f9d488d0b28..0000000000000000000000000000000000000000 --- a/modules/generative-palm/clients/palm_meta_test.go +++ /dev/null @@ -1,67 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func TestGetMeta(t *testing.T) { - t.Run("when the server is providing meta", func(t *testing.T) { - server := httptest.NewServer(&testMetaHandler{t: t}) - defer server.Close() - c := New(server.URL, 0, nullLogger()) - meta, err := c.MetaInfo() - - assert.Nil(t, err) - assert.NotNil(t, meta) - metaModel := meta["name"] - assert.True(t, metaModel != nil) - documentationHref := meta["documentationHref"] - assert.True(t, documentationHref != nil) - }) -} - -type testMetaHandler struct { - t *testing.T - // the test handler will report as not ready before the time has passed - readyTime time.Time -} - -func (f *testMetaHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - assert.Equal(f.t, "/meta", r.URL.String()) - assert.Equal(f.t, http.MethodGet, r.Method) - - if time.Since(f.readyTime) < 0 { - w.WriteHeader(http.StatusServiceUnavailable) - } - - w.Write([]byte(f.metaInfo())) -} - -func (f *testMetaHandler) metaInfo() string { - return `{ - "hostname": "http://127.0.0.1:8080", - "modules": { - "generative-palm": { - "documentationHref": "to be announced", - "name": "Google Generative Module" - } - }, - "version": "1.16.0" -}` -} diff --git a/modules/generative-palm/clients/palm_test.go b/modules/generative-palm/clients/palm_test.go deleted file mode 100644 index ab0ded77a60c17eea33b7842267ee084cd48c4db..0000000000000000000000000000000000000000 --- a/modules/generative-palm/clients/palm_test.go +++ /dev/null @@ -1,140 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "context" - "encoding/json" - "io" - "net/http" - "net/http/httptest" - "testing" - - "github.com/sirupsen/logrus" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - generativemodels "github.com/weaviate/weaviate/usecases/modulecomponents/additional/models" -) - -func nullLogger() logrus.FieldLogger { - l, _ := test.NewNullLogger() - return l -} - -func TestGetAnswer(t *testing.T) { - t.Run("when the server has a successful answer ", func(t *testing.T) { - handler := &testAnswerHandler{ - t: t, - answer: generateResponse{ - Predictions: []prediction{ - { - Candidates: []candidate{ - { - Content: "John", - }, - }, - }, - }, - Error: nil, - }, - } - server := httptest.NewServer(handler) - defer server.Close() - - c := &palm{ - apiKey: "apiKey", - httpClient: &http.Client{}, - buildUrlFn: func(useGenerativeAI bool, apiEndoint, projectID, modelID string) string { - return server.URL - }, - logger: nullLogger(), - } - - textProperties := []map[string]string{{"prop": "My name is john"}} - expected := generativemodels.GenerateResponse{ - Result: ptString("John"), - } - - res, err := c.GenerateAllResults(context.Background(), textProperties, "What is my name?", nil) - - assert.Nil(t, err) - assert.Equal(t, expected, *res) - }) - - t.Run("when the server has a an error", func(t *testing.T) { - server := httptest.NewServer(&testAnswerHandler{ - t: t, - answer: generateResponse{ - Error: &palmApiError{ - Message: "some error from the server", - }, - }, - }) - defer server.Close() - - c := &palm{ - apiKey: "apiKey", - httpClient: &http.Client{}, - buildUrlFn: func(useGenerativeAI bool, apiEndoint, projectID, modelID string) string { - return server.URL - }, - logger: nullLogger(), - } - - textProperties := []map[string]string{{"prop": "My name is john"}} - - _, err := c.GenerateAllResults(context.Background(), textProperties, "What is my name?", nil) - - require.NotNil(t, err) - assert.EqualError(t, err, "connection to Google PaLM failed with status: 500 error: some error from the server") - }) -} - -type testAnswerHandler struct { - t *testing.T - // the test handler will report as not ready before the time has passed - answer generateResponse -} - -func (f *testAnswerHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - assert.Equal(f.t, http.MethodPost, r.Method) - - if f.answer.Error != nil && f.answer.Error.Message != "" { - outBytes, err := json.Marshal(f.answer) - require.Nil(f.t, err) - - w.WriteHeader(http.StatusInternalServerError) - w.Write(outBytes) - return - } - - bodyBytes, err := io.ReadAll(r.Body) - require.Nil(f.t, err) - defer r.Body.Close() - - var b generateInput - require.Nil(f.t, json.Unmarshal(bodyBytes, &b)) - - require.Len(f.t, b.Instances, 1) - require.Len(f.t, b.Instances[0].Messages, 1) - require.True(f.t, len(b.Instances[0].Messages[0].Content) > 0) - - outBytes, err := json.Marshal(f.answer) - require.Nil(f.t, err) - - w.Write(outBytes) -} - -func ptString(in string) *string { - return &in -} diff --git a/modules/generative-palm/config.go b/modules/generative-palm/config.go deleted file mode 100644 index e702446382be1a7713fa0428eb8511293ad5a6ae..0000000000000000000000000000000000000000 --- a/modules/generative-palm/config.go +++ /dev/null @@ -1,41 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modgenerativepalm - -import ( - "context" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/modules/generative-palm/config" -) - -func (m *GenerativePaLMModule) ClassConfigDefaults() map[string]interface{} { - return map[string]interface{}{} -} - -func (m *GenerativePaLMModule) PropertyConfigDefaults( - dt *schema.DataType, -) map[string]interface{} { - return map[string]interface{}{} -} - -func (m *GenerativePaLMModule) ValidateClass(ctx context.Context, - class *models.Class, cfg moduletools.ClassConfig, -) error { - settings := config.NewClassSettings(cfg) - return settings.Validate(class) -} - -var _ = modulecapabilities.ClassConfigurator(New()) diff --git a/modules/generative-palm/config/class_settings.go b/modules/generative-palm/config/class_settings.go deleted file mode 100644 index 6f17cb72c36f9e9cf8daa5cb9209c350df1571a8..0000000000000000000000000000000000000000 --- a/modules/generative-palm/config/class_settings.go +++ /dev/null @@ -1,242 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package config - -import ( - "encoding/json" - "fmt" - "strings" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/moduletools" -) - -const ( - apiEndpointProperty = "apiEndpoint" - projectIDProperty = "projectId" - endpointIDProperty = "endpointId" - modelIDProperty = "modelId" - temperatureProperty = "temperature" - tokenLimitProperty = "tokenLimit" - topPProperty = "topP" - topKProperty = "topK" -) - -var ( - DefaultPaLMApiEndpoint = "us-central1-aiplatform.googleapis.com" - DefaultPaLMModel = "chat-bison" - DefaultPaLMTemperature = 0.2 - DefaultTokenLimit = 256 - DefaultPaLMTopP = 0.95 - DefaultPaLMTopK = 40 - DefaulGenerativeAIApiEndpoint = "generativelanguage.googleapis.com" - DefaulGenerativeAIModelID = "chat-bison-001" -) - -var supportedGenerativeAIModels = []string{ - DefaulGenerativeAIModelID, - "gemini-pro", -} - -type ClassSettings interface { - Validate(class *models.Class) error - // Module settings - ApiEndpoint() string - ProjectID() string - EndpointID() string - ModelID() string - - // parameters - // 0.0 - 1.0 - Temperature() float64 - // 1 - 1024 - TokenLimit() int - // 1 - 40 - TopK() int - // 0.0 - 1.0 - TopP() float64 -} - -type classSettings struct { - cfg moduletools.ClassConfig -} - -func NewClassSettings(cfg moduletools.ClassConfig) ClassSettings { - return &classSettings{cfg: cfg} -} - -func (ic *classSettings) Validate(class *models.Class) error { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return errors.New("empty config") - } - - var errorMessages []string - - apiEndpoint := ic.ApiEndpoint() - projectID := ic.ProjectID() - if apiEndpoint != DefaulGenerativeAIApiEndpoint && projectID == "" { - errorMessages = append(errorMessages, fmt.Sprintf("%s cannot be empty", projectIDProperty)) - } - temperature := ic.Temperature() - if temperature < 0 || temperature > 1 { - errorMessages = append(errorMessages, fmt.Sprintf("%s has to be float value between 0 and 1", temperatureProperty)) - } - tokenLimit := ic.TokenLimit() - if tokenLimit < 1 || tokenLimit > 1024 { - errorMessages = append(errorMessages, fmt.Sprintf("%s has to be an integer value between 1 and 1024", tokenLimitProperty)) - } - topK := ic.TopK() - if topK < 1 || topK > 40 { - errorMessages = append(errorMessages, fmt.Sprintf("%s has to be an integer value between 1 and 40", topKProperty)) - } - topP := ic.TopP() - if topP < 0 || topP > 1 { - errorMessages = append(errorMessages, fmt.Sprintf("%s has to be float value between 0 and 1", topPProperty)) - } - // Google MakerSuite - model := ic.ModelID() - if apiEndpoint == DefaulGenerativeAIApiEndpoint && !contains[string](supportedGenerativeAIModels, model) { - errorMessages = append(errorMessages, fmt.Sprintf("%s is not supported available models are: %+v", model, supportedGenerativeAIModels)) - } - - if len(errorMessages) > 0 { - return fmt.Errorf("%s", strings.Join(errorMessages, ", ")) - } - - return nil -} - -func (ic *classSettings) getStringProperty(name, defaultValue string) string { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return defaultValue - } - - value, ok := ic.cfg.ClassByModuleName("generative-palm")[name] - if ok { - asString, ok := value.(string) - if ok { - return asString - } - } - return defaultValue -} - -func (ic *classSettings) getFloatProperty(name string, defaultValue float64) float64 { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return defaultValue - } - - val, ok := ic.cfg.ClassByModuleName("generative-palm")[name] - if ok { - asFloat, ok := val.(float64) - if ok { - return asFloat - } - asNumber, ok := val.(json.Number) - if ok { - asFloat, _ := asNumber.Float64() - return asFloat - } - asInt, ok := val.(int) - if ok { - asFloat := float64(asInt) - return asFloat - } - } - - return defaultValue -} - -func (ic *classSettings) getIntProperty(name string, defaultValue int) int { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return defaultValue - } - - val, ok := ic.cfg.ClassByModuleName("generative-palm")[name] - if ok { - asFloat, ok := val.(float64) - if ok { - return int(asFloat) - } - asNumber, ok := val.(json.Number) - if ok { - asInt64, _ := asNumber.Int64() - return int(asInt64) - } - asInt, ok := val.(int) - if ok { - return asInt - } - } - - return defaultValue -} - -func (ic *classSettings) getDefaultModel(apiEndpoint string) string { - if apiEndpoint == DefaulGenerativeAIApiEndpoint { - return DefaulGenerativeAIModelID - } - return DefaultPaLMModel -} - -// PaLM params -func (ic *classSettings) ApiEndpoint() string { - return ic.getStringProperty(apiEndpointProperty, DefaultPaLMApiEndpoint) -} - -func (ic *classSettings) ProjectID() string { - return ic.getStringProperty(projectIDProperty, "") -} - -func (ic *classSettings) EndpointID() string { - return ic.getStringProperty(endpointIDProperty, "") -} - -func (ic *classSettings) ModelID() string { - return ic.getStringProperty(modelIDProperty, ic.getDefaultModel(ic.ApiEndpoint())) -} - -// parameters - -// 0.0 - 1.0 -func (ic *classSettings) Temperature() float64 { - return ic.getFloatProperty(temperatureProperty, DefaultPaLMTemperature) -} - -// 1 - 1024 -func (ic *classSettings) TokenLimit() int { - return ic.getIntProperty(tokenLimitProperty, DefaultTokenLimit) -} - -// 1 - 40 -func (ic *classSettings) TopK() int { - return ic.getIntProperty(topKProperty, DefaultPaLMTopK) -} - -// 0.0 - 1.0 -func (ic *classSettings) TopP() float64 { - return ic.getFloatProperty(topPProperty, DefaultPaLMTopP) -} - -func contains[T comparable](s []T, e T) bool { - for _, v := range s { - if v == e { - return true - } - } - return false -} diff --git a/modules/generative-palm/config/class_settings_test.go b/modules/generative-palm/config/class_settings_test.go deleted file mode 100644 index abc5e79e99fd7ba89b2a5c10745a3077ae7d0f97..0000000000000000000000000000000000000000 --- a/modules/generative-palm/config/class_settings_test.go +++ /dev/null @@ -1,211 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package config - -import ( - "fmt" - "testing" - - "github.com/pkg/errors" - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/entities/moduletools" -) - -func Test_classSettings_Validate(t *testing.T) { - tests := []struct { - name string - cfg moduletools.ClassConfig - wantApiEndpoint string - wantProjectID string - wantModelID string - wantTemperature float64 - wantTokenLimit int - wantTopK int - wantTopP float64 - wantErr error - }{ - { - name: "happy flow", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "projectId": "projectId", - }, - }, - wantApiEndpoint: "us-central1-aiplatform.googleapis.com", - wantProjectID: "projectId", - wantModelID: "chat-bison", - wantTemperature: 0.2, - wantTokenLimit: 256, - wantTopK: 40, - wantTopP: 0.95, - wantErr: nil, - }, - { - name: "custom values", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "apiEndpoint": "google.com", - "projectId": "cloud-project", - "modelId": "model-id", - "temperature": 0.25, - "tokenLimit": 254, - "topK": 30, - "topP": 0.97, - }, - }, - wantApiEndpoint: "google.com", - wantProjectID: "cloud-project", - wantModelID: "model-id", - wantTemperature: 0.25, - wantTokenLimit: 254, - wantTopK: 30, - wantTopP: 0.97, - wantErr: nil, - }, - { - name: "wrong temperature", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "projectId": "cloud-project", - "temperature": 2, - }, - }, - wantErr: errors.Errorf("temperature has to be float value between 0 and 1"), - }, - { - name: "wrong tokenLimit", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "projectId": "cloud-project", - "tokenLimit": 2000, - }, - }, - wantErr: errors.Errorf("tokenLimit has to be an integer value between 1 and 1024"), - }, - { - name: "wrong topK", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "projectId": "cloud-project", - "topK": 2000, - }, - }, - wantErr: errors.Errorf("topK has to be an integer value between 1 and 40"), - }, - { - name: "wrong topP", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "projectId": "cloud-project", - "topP": 3, - }, - }, - wantErr: errors.Errorf("topP has to be float value between 0 and 1"), - }, - { - name: "wrong all", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "projectId": "", - "temperature": 2, - "tokenLimit": 2000, - "topK": 2000, - "topP": 3, - }, - }, - wantErr: errors.Errorf("projectId cannot be empty, " + - "temperature has to be float value between 0 and 1, " + - "tokenLimit has to be an integer value between 1 and 1024, " + - "topK has to be an integer value between 1 and 40, " + - "topP has to be float value between 0 and 1"), - }, - { - name: "Generative AI", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "apiEndpoint": "generativelanguage.googleapis.com", - }, - }, - wantApiEndpoint: "generativelanguage.googleapis.com", - wantProjectID: "", - wantModelID: "chat-bison-001", - wantTemperature: 0.2, - wantTokenLimit: 256, - wantTopK: 40, - wantTopP: 0.95, - wantErr: nil, - }, - { - name: "Generative AI with model", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "apiEndpoint": "generativelanguage.googleapis.com", - "modelId": "chat-bison-001", - }, - }, - wantApiEndpoint: "generativelanguage.googleapis.com", - wantProjectID: "", - wantModelID: "chat-bison-001", - wantTemperature: 0.2, - wantTokenLimit: 256, - wantTopK: 40, - wantTopP: 0.95, - wantErr: nil, - }, - { - name: "Generative AI with not supported model", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "apiEndpoint": "generativelanguage.googleapis.com", - "modelId": "unsupported-model", - }, - }, - wantErr: fmt.Errorf("unsupported-model is not supported available models are: [chat-bison-001 gemini-pro]"), - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ic := NewClassSettings(tt.cfg) - if tt.wantErr != nil { - assert.EqualError(t, ic.Validate(nil), tt.wantErr.Error()) - } else { - assert.Equal(t, tt.wantApiEndpoint, ic.ApiEndpoint()) - assert.Equal(t, tt.wantProjectID, ic.ProjectID()) - assert.Equal(t, tt.wantModelID, ic.ModelID()) - assert.Equal(t, tt.wantTemperature, ic.Temperature()) - assert.Equal(t, tt.wantTokenLimit, ic.TokenLimit()) - assert.Equal(t, tt.wantTopK, ic.TopK()) - assert.Equal(t, tt.wantTopP, ic.TopP()) - } - }) - } -} - -type fakeClassConfig struct { - classConfig map[string]interface{} -} - -func (f fakeClassConfig) Class() map[string]interface{} { - return f.classConfig -} - -func (f fakeClassConfig) Tenant() string { - return "" -} - -func (f fakeClassConfig) ClassByModuleName(moduleName string) map[string]interface{} { - return f.classConfig -} - -func (f fakeClassConfig) Property(propName string) map[string]interface{} { - return nil -} diff --git a/modules/generative-palm/module.go b/modules/generative-palm/module.go deleted file mode 100644 index 01a92bad949713997acbc4b43ae9ed993e202fbe..0000000000000000000000000000000000000000 --- a/modules/generative-palm/module.go +++ /dev/null @@ -1,96 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modgenerativepalm - -import ( - "context" - "net/http" - "os" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/modules/generative-palm/clients" - additionalprovider "github.com/weaviate/weaviate/usecases/modulecomponents/additional" - generativemodels "github.com/weaviate/weaviate/usecases/modulecomponents/additional/models" -) - -const Name = "generative-palm" - -func New() *GenerativePaLMModule { - return &GenerativePaLMModule{} -} - -type GenerativePaLMModule struct { - generative generativeClient - additionalPropertiesProvider modulecapabilities.AdditionalProperties -} - -type generativeClient interface { - GenerateSingleResult(ctx context.Context, textProperties map[string]string, prompt string, cfg moduletools.ClassConfig) (*generativemodels.GenerateResponse, error) - GenerateAllResults(ctx context.Context, textProperties []map[string]string, task string, cfg moduletools.ClassConfig) (*generativemodels.GenerateResponse, error) - Generate(ctx context.Context, cfg moduletools.ClassConfig, prompt string) (*generativemodels.GenerateResponse, error) - MetaInfo() (map[string]interface{}, error) -} - -func (m *GenerativePaLMModule) Name() string { - return Name -} - -func (m *GenerativePaLMModule) Type() modulecapabilities.ModuleType { - return modulecapabilities.Text2TextGenerative -} - -func (m *GenerativePaLMModule) Init(ctx context.Context, - params moduletools.ModuleInitParams, -) error { - if err := m.initAdditional(ctx, params.GetConfig().ModuleHttpClientTimeout, params.GetLogger()); err != nil { - return errors.Wrap(err, "init q/a") - } - - return nil -} - -func (m *GenerativePaLMModule) initAdditional(ctx context.Context, timeout time.Duration, - logger logrus.FieldLogger, -) error { - apiKey := os.Getenv("PALM_APIKEY") - - client := clients.New(apiKey, timeout, logger) - - m.generative = client - - m.additionalPropertiesProvider = additionalprovider.NewGenerativeProvider(m.generative) - - return nil -} - -func (m *GenerativePaLMModule) MetaInfo() (map[string]interface{}, error) { - return m.generative.MetaInfo() -} - -func (m *GenerativePaLMModule) RootHandler() http.Handler { - // TODO: remove once this is a capability interface - return nil -} - -func (m *GenerativePaLMModule) AdditionalProperties() map[string]modulecapabilities.AdditionalProperty { - return m.additionalPropertiesProvider.AdditionalProperties() -} - -// verify we implement the modules.Module interface -var ( - _ = modulecapabilities.Module(New()) - _ = modulecapabilities.AdditionalProperties(New()) -) diff --git a/modules/img2vec-neural/clients/startup.go b/modules/img2vec-neural/clients/startup.go deleted file mode 100644 index 9ac150397d7ac8c9a9485dceb625e41f39e19dc1..0000000000000000000000000000000000000000 --- a/modules/img2vec-neural/clients/startup.go +++ /dev/null @@ -1,68 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "context" - "net/http" - "time" - - "github.com/pkg/errors" -) - -func (v *vectorizer) WaitForStartup(initCtx context.Context, - interval time.Duration, -) error { - t := time.NewTicker(interval) - defer t.Stop() - expired := initCtx.Done() - var lastErr error - for { - select { - case <-t.C: - lastErr = v.checkReady(initCtx) - if lastErr == nil { - return nil - } - v.logger. - WithField("action", "img2vec_remote_wait_for_startup"). - WithError(lastErr).Warnf("img2vec-neural inference service not ready") - case <-expired: - return errors.Wrapf(lastErr, "init context expired before remote was ready") - } - } -} - -func (v *vectorizer) checkReady(initCtx context.Context) error { - // spawn a new context (derived on the overall context) which is used to - // consider an individual request timed out - requestCtx, cancel := context.WithTimeout(initCtx, 500*time.Millisecond) - defer cancel() - - req, err := http.NewRequestWithContext(requestCtx, http.MethodGet, - v.url("/.well-known/ready"), nil) - if err != nil { - return errors.Wrap(err, "create check ready request") - } - - res, err := v.httpClient.Do(req) - if err != nil { - return errors.Wrap(err, "send check ready request") - } - - defer res.Body.Close() - if res.StatusCode > 299 { - return errors.Errorf("not ready: status %d", res.StatusCode) - } - - return nil -} diff --git a/modules/img2vec-neural/clients/startup_test.go b/modules/img2vec-neural/clients/startup_test.go deleted file mode 100644 index 7836347d937eaaa18e3c1415dcb4b2aac2d2b15b..0000000000000000000000000000000000000000 --- a/modules/img2vec-neural/clients/startup_test.go +++ /dev/null @@ -1,98 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "context" - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/sirupsen/logrus" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestWaitForStartup(t *testing.T) { - t.Run("when the server is immediately ready", func(t *testing.T) { - server := httptest.NewServer(&testReadyHandler{t: t}) - defer server.Close() - c := New(server.URL, 0, nullLogger()) - err := c.WaitForStartup(context.Background(), 50*time.Millisecond) - - assert.Nil(t, err) - }) - - t.Run("when the server is down", func(t *testing.T) { - c := New("http://nothing-running-at-this-url", 0, nullLogger()) - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancel() - err := c.WaitForStartup(ctx, 150*time.Millisecond) - - require.NotNil(t, err, nullLogger()) - assert.Contains(t, err.Error(), "expired before remote was ready") - }) - - t.Run("when the server is alive, but not ready", func(t *testing.T) { - server := httptest.NewServer(&testReadyHandler{ - t: t, - readyTime: time.Now().Add(1 * time.Minute), - }) - c := New(server.URL, 0, nullLogger()) - defer server.Close() - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancel() - err := c.WaitForStartup(ctx, 50*time.Millisecond) - - require.NotNil(t, err) - assert.Contains(t, err.Error(), "expired before remote was ready") - }) - - t.Run("when the server is initially not ready, but then becomes ready", - func(t *testing.T) { - server := httptest.NewServer(&testReadyHandler{ - t: t, - readyTime: time.Now().Add(100 * time.Millisecond), - }) - c := New(server.URL, 0, nullLogger()) - defer server.Close() - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancel() - err := c.WaitForStartup(ctx, 50*time.Millisecond) - - require.Nil(t, err) - }) -} - -type testReadyHandler struct { - t *testing.T - // the test handler will report as not ready before the time has passed - readyTime time.Time -} - -func (f *testReadyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - assert.Equal(f.t, "/.well-known/ready", r.URL.String()) - assert.Equal(f.t, http.MethodGet, r.Method) - - if time.Since(f.readyTime) < 0 { - w.WriteHeader(http.StatusServiceUnavailable) - } - - w.WriteHeader(http.StatusNoContent) -} - -func nullLogger() logrus.FieldLogger { - l, _ := test.NewNullLogger() - return l -} diff --git a/modules/img2vec-neural/clients/vectorizer.go b/modules/img2vec-neural/clients/vectorizer.go deleted file mode 100644 index 41badb5fa8ebdf7deb706822a8b2d6d2b22d45a9..0000000000000000000000000000000000000000 --- a/modules/img2vec-neural/clients/vectorizer.go +++ /dev/null @@ -1,103 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/modules/img2vec-neural/ent" -) - -type vectorizer struct { - origin string - httpClient *http.Client - logger logrus.FieldLogger -} - -func New(origin string, timeout time.Duration, logger logrus.FieldLogger) *vectorizer { - return &vectorizer{ - origin: origin, - httpClient: &http.Client{ - Timeout: timeout, - }, - logger: logger, - } -} - -func (v *vectorizer) Vectorize(ctx context.Context, - id, image string, -) (*ent.VectorizationResult, error) { - body, err := json.Marshal(vecRequest{ - ID: id, - Image: image, - }) - if err != nil { - return nil, errors.Wrapf(err, "marshal body") - } - - req, err := http.NewRequestWithContext(ctx, "POST", v.url("/vectors"), - bytes.NewReader(body)) - if err != nil { - return nil, errors.Wrap(err, "create POST request") - } - - res, err := v.httpClient.Do(req) - if err != nil { - return nil, errors.Wrap(err, "send POST request") - } - defer res.Body.Close() - - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - return nil, errors.Wrap(err, "read response body") - } - - var resBody vecResponse - if err := json.Unmarshal(bodyBytes, &resBody); err != nil { - return nil, errors.Wrap(err, "unmarshal response body") - } - - if res.StatusCode > 399 { - return nil, errors.Errorf("fail with status %d", res.StatusCode) - } - - return &ent.VectorizationResult{ - ID: resBody.ID, - Image: image, - Dimensions: resBody.Dim, - Vector: resBody.Vector, - }, nil -} - -func (v *vectorizer) url(path string) string { - return fmt.Sprintf("%s%s", v.origin, path) -} - -type vecRequest struct { - ID string `json:"id"` - Image string `json:"image"` -} - -type vecResponse struct { - ID string `json:"id"` - Vector []float32 `json:"vector"` - Dim int `json:"dim"` - Error string `json:"error"` -} diff --git a/modules/img2vec-neural/config.go b/modules/img2vec-neural/config.go deleted file mode 100644 index 1011fc75899820610d0af18fd1baf2ea069dbf94..0000000000000000000000000000000000000000 --- a/modules/img2vec-neural/config.go +++ /dev/null @@ -1,44 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modimage - -import ( - "context" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/modules/img2vec-neural/vectorizer" -) - -func (m *ImageModule) ClassConfigDefaults() map[string]interface{} { - return map[string]interface{}{} -} - -func (m *ImageModule) PropertyConfigDefaults( - dt *schema.DataType, -) map[string]interface{} { - return map[string]interface{}{} -} - -func (m *ImageModule) ValidateClass(ctx context.Context, - class *models.Class, cfg moduletools.ClassConfig, -) error { - icheck := vectorizer.NewClassSettings(cfg) - return icheck.Validate() -} - -var ( - _ = modulecapabilities.ClassConfigurator(New()) - _ = modulecapabilities.MetaProvider(New()) -) diff --git a/modules/img2vec-neural/ent/vectorization_result.go b/modules/img2vec-neural/ent/vectorization_result.go deleted file mode 100644 index 9b4dea513b990b942e07199387f23e810c8ed0b9..0000000000000000000000000000000000000000 --- a/modules/img2vec-neural/ent/vectorization_result.go +++ /dev/null @@ -1,19 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package ent - -type VectorizationResult struct { - ID string - Image string - Dimensions int - Vector []float32 -} diff --git a/modules/img2vec-neural/module.go b/modules/img2vec-neural/module.go deleted file mode 100644 index 485bf7002796e8dd2fe23e7ca9f6c7ad72a75998..0000000000000000000000000000000000000000 --- a/modules/img2vec-neural/module.go +++ /dev/null @@ -1,107 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modimage - -import ( - "context" - "net/http" - "os" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/modules/img2vec-neural/clients" - "github.com/weaviate/weaviate/modules/img2vec-neural/vectorizer" -) - -func New() *ImageModule { - return &ImageModule{} -} - -type ImageModule struct { - vectorizer imageVectorizer - graphqlProvider modulecapabilities.GraphQLArguments - searcher modulecapabilities.Searcher -} - -type imageVectorizer interface { - Object(ctx context.Context, object *models.Object, objDiff *moduletools.ObjectDiff, - settings vectorizer.ClassSettings) error - VectorizeImage(ctx context.Context, - id, image string) ([]float32, error) -} - -func (m *ImageModule) Name() string { - return "img2vec-neural" -} - -func (m *ImageModule) Type() modulecapabilities.ModuleType { - return modulecapabilities.Img2Vec -} - -func (m *ImageModule) Init(ctx context.Context, - params moduletools.ModuleInitParams, -) error { - if err := m.initVectorizer(ctx, params.GetConfig().ModuleHttpClientTimeout, params.GetLogger()); err != nil { - return errors.Wrap(err, "init vectorizer") - } - - if err := m.initNearImage(); err != nil { - return errors.Wrap(err, "init near text") - } - - return nil -} - -func (m *ImageModule) initVectorizer(ctx context.Context, timeout time.Duration, - logger logrus.FieldLogger, -) error { - // TODO: proper config management - uri := os.Getenv("IMAGE_INFERENCE_API") - if uri == "" { - return errors.Errorf("required variable IMAGE_INFERENCE_API is not set") - } - - client := clients.New(uri, timeout, logger) - if err := client.WaitForStartup(ctx, 1*time.Second); err != nil { - return errors.Wrap(err, "init remote vectorizer") - } - - m.vectorizer = vectorizer.New(client) - - return nil -} - -func (m *ImageModule) RootHandler() http.Handler { - // TODO: remove once this is a capability interface - return nil -} - -func (m *ImageModule) VectorizeObject(ctx context.Context, - obj *models.Object, objDiff *moduletools.ObjectDiff, cfg moduletools.ClassConfig, -) error { - icheck := vectorizer.NewClassSettings(cfg) - return m.vectorizer.Object(ctx, obj, objDiff, icheck) -} - -func (m *ImageModule) MetaInfo() (map[string]interface{}, error) { - return map[string]interface{}{}, nil -} - -// verify we implement the modules.Module interface -var ( - _ = modulecapabilities.Module(New()) - _ = modulecapabilities.Vectorizer(New()) -) diff --git a/modules/img2vec-neural/nearImage.go b/modules/img2vec-neural/nearImage.go deleted file mode 100644 index 1350bace66a3d83283750970903796b380356995..0000000000000000000000000000000000000000 --- a/modules/img2vec-neural/nearImage.go +++ /dev/null @@ -1,36 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modimage - -import ( - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/usecases/modulecomponents/nearImage" -) - -func (m *ImageModule) initNearImage() error { - m.searcher = nearImage.NewSearcher(m.vectorizer) - m.graphqlProvider = nearImage.New() - return nil -} - -func (m *ImageModule) Arguments() map[string]modulecapabilities.GraphQLArgument { - return m.graphqlProvider.Arguments() -} - -func (m *ImageModule) VectorSearches() map[string]modulecapabilities.VectorForParams { - return m.searcher.VectorSearches() -} - -var ( - _ = modulecapabilities.GraphQLArguments(New()) - _ = modulecapabilities.Searcher(New()) -) diff --git a/modules/img2vec-neural/vectorizer/class_settings.go b/modules/img2vec-neural/vectorizer/class_settings.go deleted file mode 100644 index 5f7993bf1cf67f1d7c506d3fd1bac83a3c55a57e..0000000000000000000000000000000000000000 --- a/modules/img2vec-neural/vectorizer/class_settings.go +++ /dev/null @@ -1,89 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "errors" - - "github.com/weaviate/weaviate/entities/moduletools" -) - -type classSettings struct { - cfg moduletools.ClassConfig -} - -func NewClassSettings(cfg moduletools.ClassConfig) *classSettings { - return &classSettings{cfg: cfg} -} - -func (ic *classSettings) ImageField(property string) bool { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return false - } - - imageFields, ok := ic.cfg.Class()["imageFields"] - if !ok { - return false - } - - imageFieldsArray, ok := imageFields.([]interface{}) - if !ok { - return false - } - - fieldNames := make([]string, len(imageFieldsArray)) - for i, value := range imageFieldsArray { - fieldNames[i] = value.(string) - } - - for i := range fieldNames { - if fieldNames[i] == property { - return true - } - } - - return false -} - -func (ic *classSettings) Validate() error { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return errors.New("empty config") - } - - imageFields, ok := ic.cfg.Class()["imageFields"] - if !ok { - return errors.New("imageFields not present") - } - - imageFieldsArray, ok := imageFields.([]interface{}) - if !ok { - return errors.New("imageFields must be an array") - } - - if len(imageFieldsArray) == 0 { - return errors.New("must contain at least one image field name in imageFields") - } - - for _, value := range imageFieldsArray { - v, ok := value.(string) - if !ok { - return errors.New("imageField must be a string") - } - if len(v) == 0 { - return errors.New("imageField values cannot be empty") - } - } - - return nil -} diff --git a/modules/img2vec-neural/vectorizer/class_settings_test.go b/modules/img2vec-neural/vectorizer/class_settings_test.go deleted file mode 100644 index 8e421eac107970471e0db3bbc2dbbae815ab554c..0000000000000000000000000000000000000000 --- a/modules/img2vec-neural/vectorizer/class_settings_test.go +++ /dev/null @@ -1,92 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "testing" - - "github.com/weaviate/weaviate/entities/moduletools" -) - -func Test_classSettings_Validate(t *testing.T) { - type fields struct { - cfg moduletools.ClassConfig - } - tests := []struct { - name string - fields fields - wantErr bool - }{ - { - name: "should not pass with empty config", - wantErr: true, - }, - { - name: "should not pass with nil config", - fields: fields{ - cfg: nil, - }, - wantErr: true, - }, - { - name: "should not pass with nil imageFields", - fields: fields{ - cfg: newConfigBuilder().addSetting("imageFields", nil).build(), - }, - wantErr: true, - }, - { - name: "should not pass with fault imageFields value", - fields: fields{ - cfg: newConfigBuilder().addSetting("imageFields", []string{}).build(), - }, - wantErr: true, - }, - { - name: "should not pass with empty imageFields", - fields: fields{ - cfg: newConfigBuilder().addSetting("imageFields", []interface{}{}).build(), - }, - wantErr: true, - }, - { - name: "should not pass with empty string in imageFields", - fields: fields{ - cfg: newConfigBuilder().addSetting("imageFields", []interface{}{""}).build(), - }, - wantErr: true, - }, - { - name: "should not pass with int value in imageFields", - fields: fields{ - cfg: newConfigBuilder().addSetting("imageFields", []interface{}{1.0}).build(), - }, - wantErr: true, - }, - { - name: "should pass with proper value in imageFields", - fields: fields{ - cfg: newConfigBuilder().addSetting("imageFields", []interface{}{"field"}).build(), - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ic := &classSettings{ - cfg: tt.fields.cfg, - } - if err := ic.Validate(); (err != nil) != tt.wantErr { - t.Errorf("classSettings.Validate() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} diff --git a/modules/img2vec-neural/vectorizer/fakes_for_test.go b/modules/img2vec-neural/vectorizer/fakes_for_test.go deleted file mode 100644 index 90f2ace1a6816aa4b776ce54a4fe501d4a8684a5..0000000000000000000000000000000000000000 --- a/modules/img2vec-neural/vectorizer/fakes_for_test.go +++ /dev/null @@ -1,70 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - - "github.com/weaviate/weaviate/modules/img2vec-neural/ent" -) - -type builder struct { - fakeClassConfig *fakeClassConfig -} - -func newConfigBuilder() *builder { - return &builder{ - fakeClassConfig: &fakeClassConfig{config: map[string]interface{}{}}, - } -} - -func (b *builder) addSetting(name string, value interface{}) *builder { - b.fakeClassConfig.config[name] = value - return b -} - -func (b *builder) build() *fakeClassConfig { - return b.fakeClassConfig -} - -type fakeClassConfig struct { - config map[string]interface{} -} - -func (c fakeClassConfig) Class() map[string]interface{} { - return c.config -} - -func (c fakeClassConfig) ClassByModuleName(moduleName string) map[string]interface{} { - return c.config -} - -func (c fakeClassConfig) Property(propName string) map[string]interface{} { - return c.config -} - -func (f fakeClassConfig) Tenant() string { - return "" -} - -type fakeClient struct{} - -func (c *fakeClient) Vectorize(ctx context.Context, - id, image string, -) (*ent.VectorizationResult, error) { - result := &ent.VectorizationResult{ - ID: id, - Image: image, - Vector: []float32{1.0, 2.0, 3.0, 4.0, 5.0}, - } - return result, nil -} diff --git a/modules/img2vec-neural/vectorizer/vectorizer.go b/modules/img2vec-neural/vectorizer/vectorizer.go deleted file mode 100644 index d1caa6c8e3dd7b82e014be2a15b79e35a23b0891..0000000000000000000000000000000000000000 --- a/modules/img2vec-neural/vectorizer/vectorizer.go +++ /dev/null @@ -1,101 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - "fmt" - - "github.com/go-openapi/strfmt" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/modules/img2vec-neural/ent" - libvectorizer "github.com/weaviate/weaviate/usecases/vectorizer" -) - -type Vectorizer struct { - client Client -} - -func New(client Client) *Vectorizer { - return &Vectorizer{ - client: client, - } -} - -type Client interface { - Vectorize(ctx context.Context, - id, image string) (*ent.VectorizationResult, error) -} - -type ClassSettings interface { - ImageField(property string) bool -} - -func (v *Vectorizer) Object(ctx context.Context, object *models.Object, - objDiff *moduletools.ObjectDiff, settings ClassSettings, -) error { - vec, err := v.object(ctx, object.ID, object.Properties, objDiff, settings) - if err != nil { - return err - } - - object.Vector = vec - return nil -} - -func (v *Vectorizer) VectorizeImage(ctx context.Context, id, image string) ([]float32, error) { - res, err := v.client.Vectorize(ctx, id, image) - if err != nil { - return nil, err - } - - return res.Vector, nil -} - -func (v *Vectorizer) object(ctx context.Context, id strfmt.UUID, - schema interface{}, objDiff *moduletools.ObjectDiff, ichek ClassSettings, -) ([]float32, error) { - vectorize := objDiff == nil || objDiff.GetVec() == nil - - // vectorize image - images := []string{} - if schema != nil { - for prop, value := range schema.(map[string]interface{}) { - if !ichek.ImageField(prop) { - continue - } - valueString, ok := value.(string) - if ok { - images = append(images, valueString) - vectorize = vectorize || (objDiff != nil && objDiff.IsChangedProp(prop)) - } - } - } - - // no property was changed, old vector can be used - if !vectorize { - return objDiff.GetVec(), nil - } - - vectors := [][]float32{} - for i, image := range images { - imgID := fmt.Sprintf("%s_%v", id, i) - vector, err := v.VectorizeImage(ctx, imgID, image) - if err != nil { - return nil, err - } - vectors = append(vectors, vector) - } - - return libvectorizer.CombineVectors(vectors), nil -} diff --git a/modules/img2vec-neural/vectorizer/vectorizer_test.go b/modules/img2vec-neural/vectorizer/vectorizer_test.go deleted file mode 100644 index bd79fefe553d6518d40bfd66bc8008deb2a0bc4e..0000000000000000000000000000000000000000 --- a/modules/img2vec-neural/vectorizer/vectorizer_test.go +++ /dev/null @@ -1,155 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/moduletools" -) - -const image = "iVBORw0KGgoAAAANSUhEUgAAAGAAAAA/CAYAAAAfQM0aAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAyRpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuMy1jMDExIDY2LjE0NTY2MSwgMjAxMi8wMi8wNi0xNDo1NjoyNyAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvIiB4bWxuczp4bXBNTT0iaHR0cDovL25zLmFkb2JlLmNvbS94YXAvMS4wL21tLyIgeG1sbnM6c3RSZWY9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9zVHlwZS9SZXNvdXJjZVJlZiMiIHhtcDpDcmVhdG9yVG9vbD0iQWRvYmUgUGhvdG9zaG9wIENTNiAoTWFjaW50b3NoKSIgeG1wTU06SW5zdGFuY2VJRD0ieG1wLmlpZDpCRjQ5NEM3RDI5QTkxMUUyOTc1NENCMzI4N0QwNDNCOSIgeG1wTU06RG9jdW1lbnRJRD0ieG1wLmRpZDpCRjQ5NEM3RTI5QTkxMUUyOTc1NENCMzI4N0QwNDNCOSI+IDx4bXBNTTpEZXJpdmVkRnJvbSBzdFJlZjppbnN0YW5jZUlEPSJ4bXAuaWlkOkJGNDk0QzdCMjlBOTExRTI5NzU0Q0IzMjg3RDA0M0I5IiBzdFJlZjpkb2N1bWVudElEPSJ4bXAuZGlkOkJGNDk0QzdDMjlBOTExRTI5NzU0Q0IzMjg3RDA0M0I5Ii8+IDwvcmRmOkRlc2NyaXB0aW9uPiA8L3JkZjpSREY+IDwveDp4bXBtZXRhPiA8P3hwYWNrZXQgZW5kPSJyIj8+WeGRxAAAB2hJREFUeNrUXFtslUUQ3hJCoQVEKy0k1qQgrRg0vaAJaq1tvJSgaLy8mKDF2IvxBY2Bgm8+iIoxvhB72tTUmKgPigbFKCEtxeKD9hZjAi3GJrYJtqRai7TQB+pMz/zwU/5zzsxe2u4kXwiwZ+bb/Xb/s7v/zEmrra1VTFsFeBRQCtgEuBWwkv5vHPAn4DdAB+B7wBjXcUNDQ8o2dXV1SmDzyhUtLS3tBPyxC9CdrN1ihi/swKuA7YD0BG1uJhQDngdcAnwDeJ86Ole2kLii+J2AFsA+wF9RjRalmEUHaZY8m6RDUYZtn6HPHiRfLm2hck0D7AScAdRH8UokwD2AnwA7UoiUyhaRD/S12dHg+8B1OWA/4BTgqVQCPEJL8haLBNDXEfJt03ziipYH+BJwHFAYJcAWwCeAZQ6CLyPfWyz584nrbCuj74eHwgKsddih2R1ba+jHJ65R1k6PuWNhAd4DZM/BTiWbdhwm5hPXsA0AngY8COgNP4JwSTyu4zE/P18VFhZKP7aNYuouXxFX5Ic8Nc2Ea2D/AfYCNgIORZ0DdusOfnFxcXDwUD09PZKP76alKDUR16KiIlVQUHDl7/39/Uozpg7Xac45YB0dGrQHHw07KVwJpRRbYiKuyCc8+MhXcyXocP2RnvMvJhr8QIBK08EPbGJiQuqq0mX7KD4GIohi4xVPTU0N6/BRamPwu7u7dZb3/RozkW3IB3lZEkGHayeI8FFVVdWaZAIUcD2Wl5fbHHy024XtC6QBkomA/XHIFb8X0Xamp6efASHqt27dGnkVkcNxVlFRoXJycmwOvuLGNmifVATsD/bLZezgKgKE2J+bm3sKHk3XXUWs4Mz87Oxs24OvOLEN26cUAfvFXAkrlKGBCDNXEbAajldXV1+5ijjP+KCrg855x+3nk2uy8SwDdIIIM1cRI6k+0NraqkZGRmzuKAIbFrYf0Q2UaPOA/Wpra3PBNfHhYHq6HbC5qanpGB7ETgPWc0TApTr7eyDolOaj6LRG+/W2Bn94eJg7+DpcowZ+AGb+642NjYfC3wEdXAdI1uK2Du2ksH2HrcHHfggGX4frNVcRMPh7BwcHN8ZiseuuIr4DvKXib29YX2bhmW+wEqYptsREXC2eWXS44oyfuYqYmpra19LSEnkaRgEG6Nj8gGRHESVCRkaG9Kg+IOyTiGtmZqatnZsOV/zMLnjcsF7KH5AIECVCX1+f6u3tlbg4oLmc2VyDy8HgPshg2yzmCo8aFsdAALzpw9dw23REwJkvHPwjSu92UcwVRcAnAd4LaQ6+CVe2AGivAe5WwhcdGp0aoVgmJuIqnBy2uSa18Buxs4AXAJMO401SjLOGfnziyhYg2GrtcNSxSfJ90pI/n7iyBUA7quKv/IYsxhmiZ/ZRy/x94soWAO1nwL0qnhVw2cD/ZfKBvjod9cEnrmwB0DBh9RUVfxHxhYrnUHLtEn2mlHyMOe6HT1wT7oISGSas4ntNzJmsVFczjnMBN1CbfwGD1BYPID8A/lFzbz5xZQsQnmWfExa6ecNVIsBKWuIlgA0qnjG2PLhsou0aZgF3qfil2fg89ssbrhwBNtB+GN/dLUnQ5kbCHYAnAFMAvGpsoY7OlS0krmOhxx7WLHwAeBLwVahN2uIUswgrPB5T8rRv7DxWqDwM+JaCjzue8b5wZe2C7gJ8quKVJqY599vJ1yZHffCJK0uA+wAfAtZYjIO+Gsi3TfOJK0sAfFP/jpKV+HBtKfkutOTPJ64sAVYD3qXgrmwpxVht6McnrmwBMAP4pjlYdRij3tCHT1xZAuDdermOA836gDKKqWNirob1ASZc2eeAl3QH36A+AGP+ohFWxNVSfYAuV9YKyKUTo/bgo2nUB5RQbImJuFqsD9DhyhbAuDgjMI36gFKX7S3XB5S6egSV2Bh8zYyDYjr4SGYi2yzmMIm5YnFGkFOLSQGNjY3X/BtaLBabWQF5XKcO6gOkZT950gAW6wPWuXoEZXEaOqoPyHLcPqkIwvqALFcCZHJmvqP6gEzH7VOKIKgPyHQlwIVUjRzWB1xw3H4+ubIFGE3VyGF9wKjj9ik3D4L6gFFXArCSTlEEzKe3LMIfwvYDNgcf+4P9csSVLUAXt7GD+oBuYfsuW4OvUR/Q7UoA/G2zaRvbOqEI0xRbYiKulusDTrgSYEg6sxKJIKwP6FLyjDYRV4v1ATpc2QKgNZtu6zTqA5o1ObM/h5eDyMvCtrlZObLgNhRv+jAHvkwqQjDzhYPfrvRvF0VcLdQHaHGNxWKrZv0d//hahcqr8Ccww1kRbwPuVMIXHRqd+ptimZiIq0F9gA2urEcQ2jkVf/tz0WG8ixTjnKEfn7iyBQi2WnuULLlV0qE9FrdzPnFlC4CGRQkvqyQ/MqRh6KtO2S948IkrWwC0XwHPAQ4r85z7w+TL1U8Y+8Q14S4oyjA9703AZ4AqFX8RvoTpN8i3/Bi/p+egHz5xZQsQGCasvqGuZhzj76DdpuIZx8FPuOAviWDG8e8qXl0yXxnHPnGdsf8FGAByGwC02iMZswAAAABJRU5ErkJggg==" - -func TestVectorizer(t *testing.T) { - t.Run("should vectorize image", func(t *testing.T) { - // given - client := &fakeClient{} - vectorizer := &Vectorizer{client} - config := newConfigBuilder().addSetting("imageFields", []interface{}{"image"}).build() - settings := NewClassSettings(config) - object := &models.Object{ - ID: "some-uuid", - Properties: map[string]interface{}{ - "image": image, - }, - } - - // when - err := vectorizer.Object(context.Background(), object, nil, settings) - - // then - require.Nil(t, err) - assert.NotNil(t, object.Vector) - }) - - t.Run("should vectorize 2 image fields", func(t *testing.T) { - // given - client := &fakeClient{} - vectorizer := &Vectorizer{client} - config := newConfigBuilder().addSetting("imageFields", []interface{}{"image1", "image2"}).build() - settings := NewClassSettings(config) - object := &models.Object{ - ID: "some-uuid", - Properties: map[string]interface{}{ - "image1": image, - "image2": image, - }, - } - - // when - err := vectorizer.Object(context.Background(), object, nil, settings) - - // then - require.Nil(t, err) - assert.NotNil(t, object.Vector) - }) -} - -func TestVectorizerWithDiff(t *testing.T) { - type testCase struct { - name string - input *models.Object - diff *moduletools.ObjectDiff - expectedVectorize bool - } - - tests := []testCase{ - { - name: "no diff", - input: &models.Object{ - ID: "some-uuid", - Properties: map[string]interface{}{ - "image": image, - "text": "text", - }, - }, - diff: nil, - expectedVectorize: true, - }, - { - name: "diff all props unchanged", - input: &models.Object{ - ID: "some-uuid", - Properties: map[string]interface{}{ - "image": image, - "text": "text", - }, - }, - diff: newObjectDiffWithVector(). - WithProp("image", image, image). - WithProp("text", "text", "text"), - expectedVectorize: false, - }, - { - name: "diff one vectorizable prop changed", - input: &models.Object{ - ID: "some-uuid", - Properties: map[string]interface{}{ - "image": image, - "text": "text", - }, - }, - diff: newObjectDiffWithVector(). - WithProp("image", "", image), - expectedVectorize: true, - }, - { - name: "all non-vectorizable props changed", - input: &models.Object{ - ID: "some-uuid", - Properties: map[string]interface{}{ - "image": image, - "text": "text", - }, - }, - diff: newObjectDiffWithVector(). - WithProp("text", "old text", "text"), - expectedVectorize: false, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - client := &fakeClient{} - vectorizer := &Vectorizer{client} - config := newConfigBuilder().addSetting("imageFields", []interface{}{"image"}).build() - settings := NewClassSettings(config) - - err := vectorizer.Object(context.Background(), test.input, test.diff, settings) - - require.Nil(t, err) - if test.expectedVectorize { - assert.Equal(t, models.C11yVector{1, 2, 3, 4, 5}, test.input.Vector) - } else { - assert.Equal(t, models.C11yVector{0, 0, 0, 0, 0}, test.input.Vector) - } - }) - } -} - -func newObjectDiffWithVector() *moduletools.ObjectDiff { - return moduletools.NewObjectDiff([]float32{0, 0, 0, 0, 0}) -} diff --git a/modules/multi2vec-bind/clients/meta.go b/modules/multi2vec-bind/clients/meta.go deleted file mode 100644 index 4096218083c0916296c9195074ee298aaea60f6a..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/clients/meta.go +++ /dev/null @@ -1,45 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "context" - "encoding/json" - "io" - "net/http" - - "github.com/pkg/errors" -) - -func (v *vectorizer) MetaInfo() (map[string]interface{}, error) { - req, err := http.NewRequestWithContext(context.Background(), "GET", v.url("/meta"), nil) - if err != nil { - return nil, errors.Wrap(err, "create GET meta request") - } - - res, err := v.httpClient.Do(req) - if err != nil { - return nil, errors.Wrap(err, "send GET meta request") - } - defer res.Body.Close() - - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - return nil, errors.Wrap(err, "read meta response body") - } - - var resBody map[string]interface{} - if err := json.Unmarshal(bodyBytes, &resBody); err != nil { - return nil, errors.Wrap(err, "unmarshal meta response body") - } - return resBody, nil -} diff --git a/modules/multi2vec-bind/clients/meta_test.go b/modules/multi2vec-bind/clients/meta_test.go deleted file mode 100644 index 4ea82f38869547b29b750a642660debc2aec8a94..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/clients/meta_test.go +++ /dev/null @@ -1,59 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func TestGetMeta(t *testing.T) { - t.Run("when the server is providing meta", func(t *testing.T) { - server := httptest.NewServer(&testMetaHandler{t: t}) - defer server.Close() - c := New(server.URL, 0, nullLogger()) - meta, err := c.MetaInfo() - - assert.Nil(t, err) - assert.NotNil(t, meta) - assert.NotNil(t, meta["model"] != nil) - assert.NotNil(t, meta["version"] != nil) - }) -} - -type testMetaHandler struct { - t *testing.T - // the test handler will report as not ready before the time has passed - readyTime time.Time -} - -func (f *testMetaHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - assert.Equal(f.t, "/meta", r.URL.String()) - assert.Equal(f.t, http.MethodGet, r.Method) - - if time.Since(f.readyTime) < 0 { - w.WriteHeader(http.StatusServiceUnavailable) - } - - w.Write([]byte(f.metaInfo())) -} - -func (f *testMetaHandler) metaInfo() string { - return `{ - "model": "ImageBindModel", - "version": 1 -}` -} diff --git a/modules/multi2vec-bind/clients/startup.go b/modules/multi2vec-bind/clients/startup.go deleted file mode 100644 index b1ac1c3939e293f32264911d32fa0badc2fc597c..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/clients/startup.go +++ /dev/null @@ -1,68 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "context" - "net/http" - "time" - - "github.com/pkg/errors" -) - -func (v *vectorizer) WaitForStartup(initCtx context.Context, - interval time.Duration, -) error { - t := time.NewTicker(interval) - defer t.Stop() - expired := initCtx.Done() - var lastErr error - for { - select { - case <-t.C: - lastErr = v.checkReady(initCtx) - if lastErr == nil { - return nil - } - v.logger. - WithField("action", "multi2vec_remote_wait_for_startup"). - WithError(lastErr).Warnf("multi2vec-bind inference service not ready") - case <-expired: - return errors.Wrapf(lastErr, "init context expired before remote was ready") - } - } -} - -func (v *vectorizer) checkReady(initCtx context.Context) error { - // spawn a new context (derived on the overall context) which is used to - // consider an individual request timed out - requestCtx, cancel := context.WithTimeout(initCtx, 500*time.Millisecond) - defer cancel() - - req, err := http.NewRequestWithContext(requestCtx, http.MethodGet, - v.url("/.well-known/ready"), nil) - if err != nil { - return errors.Wrap(err, "create check ready request") - } - - res, err := v.httpClient.Do(req) - if err != nil { - return errors.Wrap(err, "send check ready request") - } - - defer res.Body.Close() - if res.StatusCode > 299 { - return errors.Errorf("not ready: status %d", res.StatusCode) - } - - return nil -} diff --git a/modules/multi2vec-bind/clients/startup_test.go b/modules/multi2vec-bind/clients/startup_test.go deleted file mode 100644 index 7836347d937eaaa18e3c1415dcb4b2aac2d2b15b..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/clients/startup_test.go +++ /dev/null @@ -1,98 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "context" - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/sirupsen/logrus" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestWaitForStartup(t *testing.T) { - t.Run("when the server is immediately ready", func(t *testing.T) { - server := httptest.NewServer(&testReadyHandler{t: t}) - defer server.Close() - c := New(server.URL, 0, nullLogger()) - err := c.WaitForStartup(context.Background(), 50*time.Millisecond) - - assert.Nil(t, err) - }) - - t.Run("when the server is down", func(t *testing.T) { - c := New("http://nothing-running-at-this-url", 0, nullLogger()) - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancel() - err := c.WaitForStartup(ctx, 150*time.Millisecond) - - require.NotNil(t, err, nullLogger()) - assert.Contains(t, err.Error(), "expired before remote was ready") - }) - - t.Run("when the server is alive, but not ready", func(t *testing.T) { - server := httptest.NewServer(&testReadyHandler{ - t: t, - readyTime: time.Now().Add(1 * time.Minute), - }) - c := New(server.URL, 0, nullLogger()) - defer server.Close() - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancel() - err := c.WaitForStartup(ctx, 50*time.Millisecond) - - require.NotNil(t, err) - assert.Contains(t, err.Error(), "expired before remote was ready") - }) - - t.Run("when the server is initially not ready, but then becomes ready", - func(t *testing.T) { - server := httptest.NewServer(&testReadyHandler{ - t: t, - readyTime: time.Now().Add(100 * time.Millisecond), - }) - c := New(server.URL, 0, nullLogger()) - defer server.Close() - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancel() - err := c.WaitForStartup(ctx, 50*time.Millisecond) - - require.Nil(t, err) - }) -} - -type testReadyHandler struct { - t *testing.T - // the test handler will report as not ready before the time has passed - readyTime time.Time -} - -func (f *testReadyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - assert.Equal(f.t, "/.well-known/ready", r.URL.String()) - assert.Equal(f.t, http.MethodGet, r.Method) - - if time.Since(f.readyTime) < 0 { - w.WriteHeader(http.StatusServiceUnavailable) - } - - w.WriteHeader(http.StatusNoContent) -} - -func nullLogger() logrus.FieldLogger { - l, _ := test.NewNullLogger() - return l -} diff --git a/modules/multi2vec-bind/clients/vectorizer.go b/modules/multi2vec-bind/clients/vectorizer.go deleted file mode 100644 index 6de910030aaa981386d1df162106ce926c6901fc..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/clients/vectorizer.go +++ /dev/null @@ -1,124 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/modules/multi2vec-bind/ent" -) - -type vectorizer struct { - origin string - httpClient *http.Client - logger logrus.FieldLogger -} - -func New(origin string, timeout time.Duration, logger logrus.FieldLogger) *vectorizer { - return &vectorizer{ - origin: origin, - httpClient: &http.Client{ - Timeout: timeout, - }, - logger: logger, - } -} - -func (v *vectorizer) Vectorize(ctx context.Context, - texts, images, audio, video, imu, thermal, depth []string, -) (*ent.VectorizationResult, error) { - body, err := json.Marshal(vecRequest{ - Texts: texts, - Images: images, - Audio: audio, - Video: video, - IMU: imu, - Thermal: thermal, - Depth: depth, - }) - if err != nil { - return nil, errors.Wrapf(err, "marshal body") - } - - req, err := http.NewRequestWithContext(ctx, "POST", v.url("/vectorize"), - bytes.NewReader(body)) - if err != nil { - return nil, errors.Wrap(err, "create POST request") - } - - res, err := v.httpClient.Do(req) - if err != nil { - return nil, errors.Wrap(err, "send POST request") - } - defer res.Body.Close() - - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - return nil, errors.Wrap(err, "read response body") - } - - var resBody vecResponse - if err := json.Unmarshal(bodyBytes, &resBody); err != nil { - return nil, errors.Wrap(err, "unmarshal response body") - } - - if res.StatusCode != 200 { - if resBody.Error != "" { - return nil, errors.Errorf("fail with status %d: %s", res.StatusCode, - resBody.Error) - } - return nil, errors.Errorf("fail with status %d", res.StatusCode) - } - - return &ent.VectorizationResult{ - TextVectors: resBody.TextVectors, - ImageVectors: resBody.ImageVectors, - AudioVectors: resBody.AudioVectors, - VideoVectors: resBody.VideoVectors, - IMUVectors: resBody.IMUVectors, - ThermalVectors: resBody.ThermalVectors, - DepthVectors: resBody.DepthVectors, - }, nil -} - -func (v *vectorizer) url(path string) string { - return fmt.Sprintf("%s%s", v.origin, path) -} - -type vecRequest struct { - Texts []string `json:"texts,omitempty"` - Images []string `json:"images,omitempty"` - Audio []string `json:"audio,omitempty"` - Video []string `json:"video,omitempty"` - IMU []string `json:"imu,omitempty"` - Thermal []string `json:"thermal,omitempty"` - Depth []string `json:"depth,omitempty"` -} - -type vecResponse struct { - TextVectors [][]float32 `json:"textVectors,omitempty"` - ImageVectors [][]float32 `json:"imageVectors,omitempty"` - AudioVectors [][]float32 `json:"audioVectors,omitempty"` - VideoVectors [][]float32 `json:"videoVectors,omitempty"` - IMUVectors [][]float32 `json:"imuVectors,omitempty"` - ThermalVectors [][]float32 `json:"thermalVectors,omitempty"` - DepthVectors [][]float32 `json:"depthVectors,omitempty"` - Error string `json:"error,omitempty"` -} diff --git a/modules/multi2vec-bind/clients/vectorizer_test.go b/modules/multi2vec-bind/clients/vectorizer_test.go deleted file mode 100644 index 9e6bec3c05271fe2111d7dd71d1293b3379fb34c..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/clients/vectorizer_test.go +++ /dev/null @@ -1,95 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "context" - "encoding/json" - "net/http" - "net/http/httptest" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/modules/multi2vec-clip/ent" -) - -func TestVectorize(t *testing.T) { - t.Run("when the response is successful", func(t *testing.T) { - server := httptest.NewServer(&testVectorizeHandler{ - t: t, - res: vecResponse{ - TextVectors: [][]float32{ - {0, 1, 2}, - }, - ImageVectors: [][]float32{ - {1, 2, 3}, - }, - }, - }) - defer server.Close() - c := New(server.URL, 0, nullLogger()) - res, err := c.Vectorize(context.Background(), []string{"hello"}, - []string{"image-encoding"}, nil, nil, nil, nil, nil) - - assert.Nil(t, err) - require.NotNil(t, res) - resp := &ent.VectorizationResult{ - TextVectors: [][]float32{ - {0, 1, 2}, - }, - ImageVectors: [][]float32{ - {1, 2, 3}, - }, - } - assert.Equal(t, res.TextVectors, resp.TextVectors) - assert.Equal(t, res.ImageVectors, resp.ImageVectors) - assert.Equal(t, res.VideoVectors, resp.VideoVectors) - assert.Equal(t, res.AudioVectors, resp.AudioVectors) - assert.Equal(t, res.ThermalVectors, resp.ThermalVectors) - assert.Equal(t, res.DepthVectors, resp.DepthVectors) - assert.Equal(t, res.IMUVectors, resp.IMUVectors) - }) - - t.Run("when the server has a an error", func(t *testing.T) { - server := httptest.NewServer(&testVectorizeHandler{ - t: t, - res: vecResponse{ - Error: "some error from the server", - }, - }) - defer server.Close() - c := New(server.URL, 0, nullLogger()) - _, err := c.Vectorize(context.Background(), []string{"hello"}, - []string{"image-encoding"}, []string{}, []string{}, []string{}, []string{}, []string{}) - - require.NotNil(t, err) - assert.Contains(t, err.Error(), "some error from the server") - }) -} - -type testVectorizeHandler struct { - t *testing.T - // the test handler will report as not ready before the time has passed - res vecResponse -} - -func (f *testVectorizeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - assert.Equal(f.t, "/vectorize", r.URL.String()) - assert.Equal(f.t, http.MethodPost, r.Method) - - if f.res.Error != "" { - w.WriteHeader(500) - } - jsonBytes, _ := json.Marshal(f.res) - w.Write(jsonBytes) -} diff --git a/modules/multi2vec-bind/config.go b/modules/multi2vec-bind/config.go deleted file mode 100644 index 0272151f24d9d3e432f45acd87b4b03ddc18effc..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/config.go +++ /dev/null @@ -1,41 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modbind - -import ( - "context" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/modules/multi2vec-clip/vectorizer" -) - -func (m *BindModule) ClassConfigDefaults() map[string]interface{} { - return map[string]interface{}{} -} - -func (m *BindModule) PropertyConfigDefaults( - dt *schema.DataType, -) map[string]interface{} { - return map[string]interface{}{} -} - -func (m *BindModule) ValidateClass(ctx context.Context, - class *models.Class, cfg moduletools.ClassConfig, -) error { - icheck := vectorizer.NewClassSettings(cfg) - return icheck.Validate() -} - -var _ = modulecapabilities.ClassConfigurator(New()) diff --git a/modules/multi2vec-bind/ent/vectorization_result.go b/modules/multi2vec-bind/ent/vectorization_result.go deleted file mode 100644 index ceb285f107aeb1082570a250c6111bf572ba7369..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/ent/vectorization_result.go +++ /dev/null @@ -1,22 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package ent - -type VectorizationResult struct { - TextVectors [][]float32 - ImageVectors [][]float32 - AudioVectors [][]float32 - VideoVectors [][]float32 - IMUVectors [][]float32 - ThermalVectors [][]float32 - DepthVectors [][]float32 -} diff --git a/modules/multi2vec-bind/module.go b/modules/multi2vec-bind/module.go deleted file mode 100644 index 454d63bf4f9a9c2e49b3e9a3e6968d0a6e9c6fb5..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/module.go +++ /dev/null @@ -1,188 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modbind - -import ( - "context" - "net/http" - "os" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/modules/multi2vec-bind/clients" - "github.com/weaviate/weaviate/modules/multi2vec-bind/vectorizer" -) - -const Name = "multi2vec-bind" - -func New() *BindModule { - return &BindModule{} -} - -type BindModule struct { - bindVectorizer bindVectorizer - nearImageGraphqlProvider modulecapabilities.GraphQLArguments - nearImageSearcher modulecapabilities.Searcher - nearAudioGraphqlProvider modulecapabilities.GraphQLArguments - nearAudioSearcher modulecapabilities.Searcher - nearVideoGraphqlProvider modulecapabilities.GraphQLArguments - nearVideoSearcher modulecapabilities.Searcher - nearIMUGraphqlProvider modulecapabilities.GraphQLArguments - nearIMUSearcher modulecapabilities.Searcher - nearThermalGraphqlProvider modulecapabilities.GraphQLArguments - nearThermalSearcher modulecapabilities.Searcher - nearDepthGraphqlProvider modulecapabilities.GraphQLArguments - nearDepthSearcher modulecapabilities.Searcher - textVectorizer textVectorizer - nearTextGraphqlProvider modulecapabilities.GraphQLArguments - nearTextSearcher modulecapabilities.Searcher - nearTextTransformer modulecapabilities.TextTransform - metaClient metaClient -} - -type metaClient interface { - MetaInfo() (map[string]interface{}, error) -} - -type bindVectorizer interface { - Object(ctx context.Context, object *models.Object, objDiff *moduletools.ObjectDiff, - settings vectorizer.ClassSettings) error - VectorizeImage(ctx context.Context, image string) ([]float32, error) - VectorizeAudio(ctx context.Context, audio string) ([]float32, error) - VectorizeVideo(ctx context.Context, video string) ([]float32, error) - VectorizeIMU(ctx context.Context, imu string) ([]float32, error) - VectorizeThermal(ctx context.Context, thermal string) ([]float32, error) - VectorizeDepth(ctx context.Context, depth string) ([]float32, error) -} - -type textVectorizer interface { - Texts(ctx context.Context, input []string, - settings vectorizer.ClassSettings) ([]float32, error) - MoveTo(source, target []float32, weight float32) ([]float32, error) - MoveAwayFrom(source, target []float32, weight float32) ([]float32, error) - CombineVectors(vectors [][]float32) []float32 -} - -func (m *BindModule) Name() string { - return Name -} - -func (m *BindModule) Type() modulecapabilities.ModuleType { - return modulecapabilities.Multi2Vec -} - -func (m *BindModule) Init(ctx context.Context, - params moduletools.ModuleInitParams, -) error { - if err := m.initVectorizer(ctx, params.GetConfig().ModuleHttpClientTimeout, params.GetLogger()); err != nil { - return errors.Wrap(err, "init vectorizer") - } - - if err := m.initNearImage(); err != nil { - return errors.Wrap(err, "init near image") - } - - if err := m.initNearAudio(); err != nil { - return errors.Wrap(err, "init near audio") - } - - if err := m.initNearVideo(); err != nil { - return errors.Wrap(err, "init near video") - } - - if err := m.initNearIMU(); err != nil { - return errors.Wrap(err, "init near imu") - } - - if err := m.initNearThermal(); err != nil { - return errors.Wrap(err, "init near thermal") - } - - if err := m.initNearDepth(); err != nil { - return errors.Wrap(err, "init near depth") - } - - return nil -} - -func (m *BindModule) InitExtension(modules []modulecapabilities.Module) error { - for _, module := range modules { - if module.Name() == m.Name() { - continue - } - if arg, ok := module.(modulecapabilities.TextTransformers); ok { - if arg != nil && arg.TextTransformers() != nil { - m.nearTextTransformer = arg.TextTransformers()["nearText"] - } - } - } - - if err := m.initNearText(); err != nil { - return errors.Wrap(err, "init near text") - } - - return nil -} - -func (m *BindModule) initVectorizer(ctx context.Context, timeout time.Duration, - logger logrus.FieldLogger, -) error { - // TODO: proper config management - uri := os.Getenv("BIND_INFERENCE_API") - if uri == "" { - return errors.Errorf("required variable BIND_INFERENCE_API is not set") - } - - client := clients.New(uri, timeout, logger) - if err := client.WaitForStartup(ctx, 1*time.Second); err != nil { - return errors.Wrap(err, "init remote vectorizer") - } - - m.bindVectorizer = vectorizer.New(client) - m.textVectorizer = vectorizer.New(client) - m.metaClient = client - - return nil -} - -func (m *BindModule) RootHandler() http.Handler { - // TODO: remove once this is a capability interface - return nil -} - -func (m *BindModule) VectorizeObject(ctx context.Context, - obj *models.Object, objDiff *moduletools.ObjectDiff, cfg moduletools.ClassConfig, -) error { - icheck := vectorizer.NewClassSettings(cfg) - return m.bindVectorizer.Object(ctx, obj, objDiff, icheck) -} - -func (m *BindModule) MetaInfo() (map[string]interface{}, error) { - return m.metaClient.MetaInfo() -} - -func (m *BindModule) VectorizeInput(ctx context.Context, - input string, cfg moduletools.ClassConfig, -) ([]float32, error) { - return m.textVectorizer.Texts(ctx, []string{input}, vectorizer.NewClassSettings(cfg)) -} - -// verify we implement the modules.Module interface -var ( - _ = modulecapabilities.Module(New()) - _ = modulecapabilities.Vectorizer(New()) - _ = modulecapabilities.InputVectorizer(New()) -) diff --git a/modules/multi2vec-bind/nearArguments.go b/modules/multi2vec-bind/nearArguments.go deleted file mode 100644 index 8ccf1bfe489fc71606bad4bc33f56189dd077877..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/nearArguments.go +++ /dev/null @@ -1,122 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modbind - -import ( - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/modules/multi2vec-bind/nearAudio" - "github.com/weaviate/weaviate/modules/multi2vec-bind/nearImage" - "github.com/weaviate/weaviate/modules/multi2vec-bind/nearVideo" - "github.com/weaviate/weaviate/modules/multi2vec-bind/neardepth" - "github.com/weaviate/weaviate/modules/multi2vec-bind/nearimu" - "github.com/weaviate/weaviate/modules/multi2vec-bind/neartext" - "github.com/weaviate/weaviate/modules/multi2vec-bind/nearthermal" -) - -func (m *BindModule) initNearText() error { - m.nearTextSearcher = neartext.NewSearcher(m.textVectorizer) - m.nearTextGraphqlProvider = neartext.New(m.nearTextTransformer) - return nil -} - -func (m *BindModule) initNearImage() error { - m.nearImageSearcher = nearImage.NewSearcher(m.bindVectorizer) - m.nearImageGraphqlProvider = nearImage.New() - return nil -} - -func (m *BindModule) initNearAudio() error { - m.nearAudioSearcher = nearAudio.NewSearcher(m.bindVectorizer) - m.nearAudioGraphqlProvider = nearAudio.New() - return nil -} - -func (m *BindModule) initNearVideo() error { - m.nearVideoSearcher = nearVideo.NewSearcher(m.bindVectorizer) - m.nearVideoGraphqlProvider = nearVideo.New() - return nil -} - -func (m *BindModule) initNearIMU() error { - m.nearIMUSearcher = nearimu.NewSearcher(m.bindVectorizer) - m.nearIMUGraphqlProvider = nearimu.New() - return nil -} - -func (m *BindModule) initNearThermal() error { - m.nearThermalSearcher = nearthermal.NewSearcher(m.bindVectorizer) - m.nearThermalGraphqlProvider = nearthermal.New() - return nil -} - -func (m *BindModule) initNearDepth() error { - m.nearDepthSearcher = neardepth.NewSearcher(m.bindVectorizer) - m.nearDepthGraphqlProvider = neardepth.New() - return nil -} - -func (m *BindModule) Arguments() map[string]modulecapabilities.GraphQLArgument { - arguments := map[string]modulecapabilities.GraphQLArgument{} - for name, arg := range m.nearTextGraphqlProvider.Arguments() { - arguments[name] = arg - } - for name, arg := range m.nearImageGraphqlProvider.Arguments() { - arguments[name] = arg - } - for name, arg := range m.nearAudioGraphqlProvider.Arguments() { - arguments[name] = arg - } - for name, arg := range m.nearVideoGraphqlProvider.Arguments() { - arguments[name] = arg - } - for name, arg := range m.nearIMUGraphqlProvider.Arguments() { - arguments[name] = arg - } - for name, arg := range m.nearThermalGraphqlProvider.Arguments() { - arguments[name] = arg - } - for name, arg := range m.nearDepthGraphqlProvider.Arguments() { - arguments[name] = arg - } - return arguments -} - -func (m *BindModule) VectorSearches() map[string]modulecapabilities.VectorForParams { - vectorSearches := map[string]modulecapabilities.VectorForParams{} - for name, arg := range m.nearTextSearcher.VectorSearches() { - vectorSearches[name] = arg - } - for name, arg := range m.nearImageSearcher.VectorSearches() { - vectorSearches[name] = arg - } - for name, arg := range m.nearAudioSearcher.VectorSearches() { - vectorSearches[name] = arg - } - for name, arg := range m.nearVideoSearcher.VectorSearches() { - vectorSearches[name] = arg - } - for name, arg := range m.nearIMUSearcher.VectorSearches() { - vectorSearches[name] = arg - } - for name, arg := range m.nearThermalSearcher.VectorSearches() { - vectorSearches[name] = arg - } - for name, arg := range m.nearDepthSearcher.VectorSearches() { - vectorSearches[name] = arg - } - return vectorSearches -} - -var ( - _ = modulecapabilities.GraphQLArguments(New()) - _ = modulecapabilities.Searcher(New()) -) diff --git a/modules/multi2vec-bind/nearAudio/graphql_argument.go b/modules/multi2vec-bind/nearAudio/graphql_argument.go deleted file mode 100644 index bdeed8e05d8a937e0ddf1b5019ad6b70ef786fb7..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/nearAudio/graphql_argument.go +++ /dev/null @@ -1,61 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package nearAudio - -import ( - "fmt" - - "github.com/tailor-inc/graphql" - "github.com/weaviate/weaviate/adapters/handlers/graphql/descriptions" -) - -func getNearAudioArgumentFn(classname string) *graphql.ArgumentConfig { - return nearAudioArgument("GetObjects", classname) -} - -func exploreNearAudioArgumentFn() *graphql.ArgumentConfig { - return nearAudioArgument("Explore", "") -} - -func aggregateNearAudioArgumentFn(classname string) *graphql.ArgumentConfig { - return nearAudioArgument("Aggregate", classname) -} - -func nearAudioArgument(prefix, className string) *graphql.ArgumentConfig { - prefixName := fmt.Sprintf("Multi2VecBind%s%s", prefix, className) - return &graphql.ArgumentConfig{ - Type: graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sNearAudioInpObj", prefixName), - Fields: nearAudioFields(prefixName), - Description: descriptions.GetWhereInpObj, - }, - ), - } -} - -func nearAudioFields(prefix string) graphql.InputObjectConfigFieldMap { - return graphql.InputObjectConfigFieldMap{ - "audio": &graphql.InputObjectFieldConfig{ - Description: "Base64 encoded audio file", - Type: graphql.NewNonNull(graphql.String), - }, - "certainty": &graphql.InputObjectFieldConfig{ - Description: descriptions.Certainty, - Type: graphql.Float, - }, - "distance": &graphql.InputObjectFieldConfig{ - Description: descriptions.Distance, - Type: graphql.Float, - }, - } -} diff --git a/modules/multi2vec-bind/nearAudio/graphql_argument_test.go b/modules/multi2vec-bind/nearAudio/graphql_argument_test.go deleted file mode 100644 index c2d4686b5ba68a9ec860e50d2374d12f1b752462..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/nearAudio/graphql_argument_test.go +++ /dev/null @@ -1,50 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package nearAudio - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/tailor-inc/graphql" -) - -func TestNearAudioGraphQLArgument(t *testing.T) { - t.Run("should generate nearAudio argument properly", func(t *testing.T) { - // given - prefix := "Prefix" - classname := "Class" - // when - nearAudio := nearAudioArgument(prefix, classname) - - // then - // the built graphQL field needs to support this structure: - // nearAudio: { - // audio: "base64;encoded,audio", - // distance: 0.9 - // } - assert.NotNil(t, nearAudio) - assert.Equal(t, "Multi2VecBindPrefixClassNearAudioInpObj", nearAudio.Type.Name()) - nearAudioFields, ok := nearAudio.Type.(*graphql.InputObject) - assert.True(t, ok) - assert.NotNil(t, nearAudioFields) - assert.Equal(t, 3, len(nearAudioFields.Fields())) - fields := nearAudioFields.Fields() - audio := fields["audio"] - audioNonNull, audioNonNullOK := audio.Type.(*graphql.NonNull) - assert.True(t, audioNonNullOK) - assert.Equal(t, "String", audioNonNull.OfType.Name()) - assert.NotNil(t, audio) - assert.NotNil(t, fields["certainty"]) - assert.NotNil(t, fields["distance"]) - }) -} diff --git a/modules/multi2vec-bind/nearAudio/graphql_extract.go b/modules/multi2vec-bind/nearAudio/graphql_extract.go deleted file mode 100644 index 40855c6f2d556ae3b1984839fc16eb3daa929e0b..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/nearAudio/graphql_extract.go +++ /dev/null @@ -1,37 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package nearAudio - -import "github.com/weaviate/weaviate/usecases/modulecomponents/nearAudio" - -// extractNearAudioFn arguments, such as "audio" and "certainty" -func extractNearAudioFn(source map[string]interface{}) interface{} { - var args nearAudio.NearAudioParams - - audio, ok := source["audio"].(string) - if ok { - args.Audio = audio - } - - certainty, ok := source["certainty"] - if ok { - args.Certainty = certainty.(float64) - } - - distance, ok := source["distance"] - if ok { - args.Distance = distance.(float64) - args.WithDistance = true - } - - return &args -} diff --git a/modules/multi2vec-bind/nearAudio/graphql_extract_test.go b/modules/multi2vec-bind/nearAudio/graphql_extract_test.go deleted file mode 100644 index 97d43aeead4070547fa99cef0e7d69c99071ac1a..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/nearAudio/graphql_extract_test.go +++ /dev/null @@ -1,76 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package nearAudio - -import ( - "reflect" - "testing" - - "github.com/weaviate/weaviate/usecases/modulecomponents/nearAudio" -) - -func Test_extractNearAudioFn(t *testing.T) { - type args struct { - source map[string]interface{} - } - tests := []struct { - name string - args args - want interface{} - }{ - { - name: "should extract properly with distance and audio params set", - args: args{ - source: map[string]interface{}{ - "audio": "base64;encoded", - "distance": float64(0.9), - }, - }, - want: &nearAudio.NearAudioParams{ - Audio: "base64;encoded", - Distance: 0.9, - WithDistance: true, - }, - }, - { - name: "should extract properly with certainty and audio params set", - args: args{ - source: map[string]interface{}{ - "audio": "base64;encoded", - "certainty": float64(0.9), - }, - }, - want: &nearAudio.NearAudioParams{ - Audio: "base64;encoded", - Certainty: 0.9, - }, - }, - { - name: "should extract properly with only audio set", - args: args{ - source: map[string]interface{}{ - "audio": "base64;encoded", - }, - }, - want: &nearAudio.NearAudioParams{ - Audio: "base64;encoded", - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := extractNearAudioFn(tt.args.source); !reflect.DeepEqual(got, tt.want) { - t.Errorf("extractNearAudioFn() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/modules/multi2vec-bind/nearAudio/graphql_provider.go b/modules/multi2vec-bind/nearAudio/graphql_provider.go deleted file mode 100644 index 94b4d403a396fbbdb1308ccb401c925c019c59ce..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/nearAudio/graphql_provider.go +++ /dev/null @@ -1,39 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package nearAudio - -import ( - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/usecases/modulecomponents/nearAudio" -) - -type GraphQLArgumentsProvider struct{} - -func New() *GraphQLArgumentsProvider { - return &GraphQLArgumentsProvider{} -} - -func (g *GraphQLArgumentsProvider) Arguments() map[string]modulecapabilities.GraphQLArgument { - arguments := map[string]modulecapabilities.GraphQLArgument{} - arguments["nearAudio"] = g.getNearAudio() - return arguments -} - -func (g *GraphQLArgumentsProvider) getNearAudio() modulecapabilities.GraphQLArgument { - return modulecapabilities.GraphQLArgument{ - GetArgumentsFunction: getNearAudioArgumentFn, - AggregateArgumentsFunction: aggregateNearAudioArgumentFn, - ExploreArgumentsFunction: exploreNearAudioArgumentFn, - ExtractFunction: extractNearAudioFn, - ValidateFunction: nearAudio.ValidateNearAudioFn, - } -} diff --git a/modules/multi2vec-bind/nearAudio/searcher.go b/modules/multi2vec-bind/nearAudio/searcher.go deleted file mode 100644 index 22a086146f55fd77f9ccebe5aacedb4a0978e4dc..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/nearAudio/searcher.go +++ /dev/null @@ -1,61 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package nearAudio - -import ( - "context" - "fmt" - - "github.com/weaviate/weaviate/usecases/modulecomponents/nearAudio" - - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" -) - -type Searcher struct { - vectorizer bindVectorizer -} - -func NewSearcher(vectorizer bindVectorizer) *Searcher { - return &Searcher{vectorizer} -} - -type bindVectorizer interface { - VectorizeAudio(ctx context.Context, audio string) ([]float32, error) -} - -func (s *Searcher) VectorSearches() map[string]modulecapabilities.VectorForParams { - vectorSearches := map[string]modulecapabilities.VectorForParams{} - vectorSearches["nearAudio"] = s.vectorForNearAudioParam - return vectorSearches -} - -func (s *Searcher) vectorForNearAudioParam(ctx context.Context, params interface{}, - className string, - findVectorFn modulecapabilities.FindVectorFn, - cfg moduletools.ClassConfig, -) ([]float32, error) { - return s.vectorFromNearAudioParam(ctx, params.(*nearAudio.NearAudioParams), className, findVectorFn, cfg) -} - -func (s *Searcher) vectorFromNearAudioParam(ctx context.Context, - params *nearAudio.NearAudioParams, className string, findVectorFn modulecapabilities.FindVectorFn, - cfg moduletools.ClassConfig, -) ([]float32, error) { - // find vector for given search query - vector, err := s.vectorizer.VectorizeAudio(ctx, params.Audio) - if err != nil { - return nil, fmt.Errorf("vectorize audio: %w", err) - } - - return vector, nil -} diff --git a/modules/multi2vec-bind/nearImage/graphql_argument.go b/modules/multi2vec-bind/nearImage/graphql_argument.go deleted file mode 100644 index b44c310957722b47f7f00cedf1d3b1c3f8bc450e..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/nearImage/graphql_argument.go +++ /dev/null @@ -1,61 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package nearImage - -import ( - "fmt" - - "github.com/tailor-inc/graphql" - "github.com/weaviate/weaviate/adapters/handlers/graphql/descriptions" -) - -func getNearImageArgumentFn(classname string) *graphql.ArgumentConfig { - return nearImageArgument("GetObjects", classname) -} - -func exploreNearImageArgumentFn() *graphql.ArgumentConfig { - return nearImageArgument("Explore", "") -} - -func aggregateNearImageArgumentFn(classname string) *graphql.ArgumentConfig { - return nearImageArgument("Aggregate", classname) -} - -func nearImageArgument(prefix, className string) *graphql.ArgumentConfig { - prefixName := fmt.Sprintf("Multi2VecBind%s%s", prefix, className) - return &graphql.ArgumentConfig{ - Type: graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sNearImageInpObj", prefixName), - Fields: nearImageFields(prefixName), - Description: descriptions.GetWhereInpObj, - }, - ), - } -} - -func nearImageFields(prefix string) graphql.InputObjectConfigFieldMap { - return graphql.InputObjectConfigFieldMap{ - "image": &graphql.InputObjectFieldConfig{ - Description: "Base64 encoded image", - Type: graphql.NewNonNull(graphql.String), - }, - "certainty": &graphql.InputObjectFieldConfig{ - Description: descriptions.Certainty, - Type: graphql.Float, - }, - "distance": &graphql.InputObjectFieldConfig{ - Description: descriptions.Distance, - Type: graphql.Float, - }, - } -} diff --git a/modules/multi2vec-bind/nearImage/graphql_argument_test.go b/modules/multi2vec-bind/nearImage/graphql_argument_test.go deleted file mode 100644 index 2c2d06f73c9960fd7823dacb03023068e13f0c75..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/nearImage/graphql_argument_test.go +++ /dev/null @@ -1,50 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package nearImage - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/tailor-inc/graphql" -) - -func TestNearImageGraphQLArgument(t *testing.T) { - t.Run("should generate nearImage argument properly", func(t *testing.T) { - // given - prefix := "Prefix" - classname := "Class" - // when - nearImage := nearImageArgument(prefix, classname) - - // then - // the built graphQL field needs to support this structure: - // nearImage: { - // image: "base64;encoded,image", - // distance: 0.9 - // } - assert.NotNil(t, nearImage) - assert.Equal(t, "Multi2VecBindPrefixClassNearImageInpObj", nearImage.Type.Name()) - answerFields, ok := nearImage.Type.(*graphql.InputObject) - assert.True(t, ok) - assert.NotNil(t, answerFields) - assert.Equal(t, 3, len(answerFields.Fields())) - fields := answerFields.Fields() - image := fields["image"] - imageNonNull, imageNonNullOK := image.Type.(*graphql.NonNull) - assert.True(t, imageNonNullOK) - assert.Equal(t, "String", imageNonNull.OfType.Name()) - assert.NotNil(t, image) - assert.NotNil(t, fields["certainty"]) - assert.NotNil(t, fields["distance"]) - }) -} diff --git a/modules/multi2vec-bind/nearImage/graphql_extract.go b/modules/multi2vec-bind/nearImage/graphql_extract.go deleted file mode 100644 index 543b1b66661c89d1d4c30a79aa100ca2b824850c..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/nearImage/graphql_extract.go +++ /dev/null @@ -1,37 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package nearImage - -import "github.com/weaviate/weaviate/usecases/modulecomponents/nearImage" - -// extractNearImageFn arguments, such as "image" and "certainty" -func extractNearImageFn(source map[string]interface{}) interface{} { - var args nearImage.NearImageParams - - image, ok := source["image"].(string) - if ok { - args.Image = image - } - - certainty, ok := source["certainty"] - if ok { - args.Certainty = certainty.(float64) - } - - distance, ok := source["distance"] - if ok { - args.Distance = distance.(float64) - args.WithDistance = true - } - - return &args -} diff --git a/modules/multi2vec-bind/nearImage/graphql_extract_test.go b/modules/multi2vec-bind/nearImage/graphql_extract_test.go deleted file mode 100644 index 7fc7782e226badb96cb1ee50ba667ddb50e48907..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/nearImage/graphql_extract_test.go +++ /dev/null @@ -1,76 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package nearImage - -import ( - "reflect" - "testing" - - "github.com/weaviate/weaviate/usecases/modulecomponents/nearImage" -) - -func Test_extractNearImageFn(t *testing.T) { - type args struct { - source map[string]interface{} - } - tests := []struct { - name string - args args - want interface{} - }{ - { - name: "should extract properly with distance and image params set", - args: args{ - source: map[string]interface{}{ - "image": "base64;encoded", - "distance": float64(0.9), - }, - }, - want: &nearImage.NearImageParams{ - Image: "base64;encoded", - Distance: 0.9, - WithDistance: true, - }, - }, - { - name: "should extract properly with certainty and image params set", - args: args{ - source: map[string]interface{}{ - "image": "base64;encoded", - "certainty": float64(0.9), - }, - }, - want: &nearImage.NearImageParams{ - Image: "base64;encoded", - Certainty: 0.9, - }, - }, - { - name: "should extract properly with only image set", - args: args{ - source: map[string]interface{}{ - "image": "base64;encoded", - }, - }, - want: &nearImage.NearImageParams{ - Image: "base64;encoded", - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := extractNearImageFn(tt.args.source); !reflect.DeepEqual(got, tt.want) { - t.Errorf("extractNearImageFn() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/modules/multi2vec-bind/nearImage/graphql_provider.go b/modules/multi2vec-bind/nearImage/graphql_provider.go deleted file mode 100644 index 563edc7793b684a795b5f49992ca79c693b88a02..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/nearImage/graphql_provider.go +++ /dev/null @@ -1,39 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package nearImage - -import ( - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/usecases/modulecomponents/nearImage" -) - -type GraphQLArgumentsProvider struct{} - -func New() *GraphQLArgumentsProvider { - return &GraphQLArgumentsProvider{} -} - -func (g *GraphQLArgumentsProvider) Arguments() map[string]modulecapabilities.GraphQLArgument { - arguments := map[string]modulecapabilities.GraphQLArgument{} - arguments["nearImage"] = g.getNearImage() - return arguments -} - -func (g *GraphQLArgumentsProvider) getNearImage() modulecapabilities.GraphQLArgument { - return modulecapabilities.GraphQLArgument{ - GetArgumentsFunction: getNearImageArgumentFn, - AggregateArgumentsFunction: aggregateNearImageArgumentFn, - ExploreArgumentsFunction: exploreNearImageArgumentFn, - ExtractFunction: extractNearImageFn, - ValidateFunction: nearImage.ValidateNearImageFn, - } -} diff --git a/modules/multi2vec-bind/nearImage/searcher.go b/modules/multi2vec-bind/nearImage/searcher.go deleted file mode 100644 index f9935aec1c79daefccf862385b70685264fa1ede..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/nearImage/searcher.go +++ /dev/null @@ -1,61 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package nearImage - -import ( - "context" - - "github.com/weaviate/weaviate/usecases/modulecomponents/nearImage" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" -) - -type Searcher struct { - vectorizer imgVectorizer -} - -func NewSearcher(vectorizer imgVectorizer) *Searcher { - return &Searcher{vectorizer} -} - -type imgVectorizer interface { - VectorizeImage(ctx context.Context, image string) ([]float32, error) -} - -func (s *Searcher) VectorSearches() map[string]modulecapabilities.VectorForParams { - vectorSearches := map[string]modulecapabilities.VectorForParams{} - vectorSearches["nearImage"] = s.vectorForNearImageParam - return vectorSearches -} - -func (s *Searcher) vectorForNearImageParam(ctx context.Context, params interface{}, - className string, - findVectorFn modulecapabilities.FindVectorFn, - cfg moduletools.ClassConfig, -) ([]float32, error) { - return s.vectorFromNearImageParam(ctx, params.(*nearImage.NearImageParams), className, findVectorFn, cfg) -} - -func (s *Searcher) vectorFromNearImageParam(ctx context.Context, - params *nearImage.NearImageParams, className string, findVectorFn modulecapabilities.FindVectorFn, - cfg moduletools.ClassConfig, -) ([]float32, error) { - // find vector for given search query - vector, err := s.vectorizer.VectorizeImage(ctx, params.Image) - if err != nil { - return nil, errors.Errorf("vectorize image: %v", err) - } - - return vector, nil -} diff --git a/modules/multi2vec-bind/nearVideo/graphql_argument.go b/modules/multi2vec-bind/nearVideo/graphql_argument.go deleted file mode 100644 index 96f077c963460a888ac35b57f31df3a06b8b72d0..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/nearVideo/graphql_argument.go +++ /dev/null @@ -1,61 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package nearVideo - -import ( - "fmt" - - "github.com/tailor-inc/graphql" - "github.com/weaviate/weaviate/adapters/handlers/graphql/descriptions" -) - -func getNearVideoArgumentFn(classname string) *graphql.ArgumentConfig { - return nearVideoArgument("GetObjects", classname) -} - -func exploreNearVideoArgumentFn() *graphql.ArgumentConfig { - return nearVideoArgument("Explore", "") -} - -func aggregateNearVideoArgumentFn(classname string) *graphql.ArgumentConfig { - return nearVideoArgument("Aggregate", classname) -} - -func nearVideoArgument(prefix, className string) *graphql.ArgumentConfig { - prefixName := fmt.Sprintf("Multi2VecBind%s%s", prefix, className) - return &graphql.ArgumentConfig{ - Type: graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sNearVideoInpObj", prefixName), - Fields: nearVideoFields(prefixName), - Description: descriptions.GetWhereInpObj, - }, - ), - } -} - -func nearVideoFields(prefix string) graphql.InputObjectConfigFieldMap { - return graphql.InputObjectConfigFieldMap{ - "video": &graphql.InputObjectFieldConfig{ - Description: "Base64 encoded video", - Type: graphql.NewNonNull(graphql.String), - }, - "certainty": &graphql.InputObjectFieldConfig{ - Description: descriptions.Certainty, - Type: graphql.Float, - }, - "distance": &graphql.InputObjectFieldConfig{ - Description: descriptions.Distance, - Type: graphql.Float, - }, - } -} diff --git a/modules/multi2vec-bind/nearVideo/graphql_argument_test.go b/modules/multi2vec-bind/nearVideo/graphql_argument_test.go deleted file mode 100644 index 1273e6b46308071b50b3b5830c7a653a8c24bc22..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/nearVideo/graphql_argument_test.go +++ /dev/null @@ -1,50 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package nearVideo - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/tailor-inc/graphql" -) - -func TestNearVideoGraphQLArgument(t *testing.T) { - t.Run("should generate nearVideo argument properly", func(t *testing.T) { - // given - prefix := "Prefix" - classname := "Class" - // when - nearVideo := nearVideoArgument(prefix, classname) - - // then - // the built graphQL field needs to support this structure: - // nearVideo: { - // video: "base64;encoded,video_file", - // distance: 0.9 - // } - assert.NotNil(t, nearVideo) - assert.Equal(t, "Multi2VecBindPrefixClassNearVideoInpObj", nearVideo.Type.Name()) - answerFields, ok := nearVideo.Type.(*graphql.InputObject) - assert.True(t, ok) - assert.NotNil(t, answerFields) - assert.Equal(t, 3, len(answerFields.Fields())) - fields := answerFields.Fields() - video := fields["video"] - videoNonNull, videoNonNullOK := video.Type.(*graphql.NonNull) - assert.True(t, videoNonNullOK) - assert.Equal(t, "String", videoNonNull.OfType.Name()) - assert.NotNil(t, video) - assert.NotNil(t, fields["certainty"]) - assert.NotNil(t, fields["distance"]) - }) -} diff --git a/modules/multi2vec-bind/nearVideo/graphql_extract.go b/modules/multi2vec-bind/nearVideo/graphql_extract.go deleted file mode 100644 index e1f2f4e9a441a090328efc99eb0d63012b09e727..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/nearVideo/graphql_extract.go +++ /dev/null @@ -1,37 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package nearVideo - -import "github.com/weaviate/weaviate/usecases/modulecomponents/nearVideo" - -// extractNearVideoFn arguments, such as "video" and "certainty" -func extractNearVideoFn(source map[string]interface{}) interface{} { - var args nearVideo.NearVideoParams - - video, ok := source["video"].(string) - if ok { - args.Video = video - } - - certainty, ok := source["certainty"] - if ok { - args.Certainty = certainty.(float64) - } - - distance, ok := source["distance"] - if ok { - args.Distance = distance.(float64) - args.WithDistance = true - } - - return &args -} diff --git a/modules/multi2vec-bind/nearVideo/graphql_extract_test.go b/modules/multi2vec-bind/nearVideo/graphql_extract_test.go deleted file mode 100644 index d964fb819a7b74ef25746befab28d064ba63c148..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/nearVideo/graphql_extract_test.go +++ /dev/null @@ -1,76 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package nearVideo - -import ( - "reflect" - "testing" - - "github.com/weaviate/weaviate/usecases/modulecomponents/nearVideo" -) - -func Test_extractNearVideoFn(t *testing.T) { - type args struct { - source map[string]interface{} - } - tests := []struct { - name string - args args - want interface{} - }{ - { - name: "should extract properly with distance and video params set", - args: args{ - source: map[string]interface{}{ - "video": "base64;encoded", - "distance": float64(0.9), - }, - }, - want: &nearVideo.NearVideoParams{ - Video: "base64;encoded", - Distance: 0.9, - WithDistance: true, - }, - }, - { - name: "should extract properly with certainty and video params set", - args: args{ - source: map[string]interface{}{ - "video": "base64;encoded", - "certainty": float64(0.9), - }, - }, - want: &nearVideo.NearVideoParams{ - Video: "base64;encoded", - Certainty: 0.9, - }, - }, - { - name: "should extract properly with only video set", - args: args{ - source: map[string]interface{}{ - "video": "base64;encoded", - }, - }, - want: &nearVideo.NearVideoParams{ - Video: "base64;encoded", - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := extractNearVideoFn(tt.args.source); !reflect.DeepEqual(got, tt.want) { - t.Errorf("extractNearVideoFn() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/modules/multi2vec-bind/nearVideo/graphql_provider.go b/modules/multi2vec-bind/nearVideo/graphql_provider.go deleted file mode 100644 index eb720b642d50d6f13620b5157402147fd17b951d..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/nearVideo/graphql_provider.go +++ /dev/null @@ -1,39 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package nearVideo - -import ( - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/usecases/modulecomponents/nearVideo" -) - -type GraphQLArgumentsProvider struct{} - -func New() *GraphQLArgumentsProvider { - return &GraphQLArgumentsProvider{} -} - -func (g *GraphQLArgumentsProvider) Arguments() map[string]modulecapabilities.GraphQLArgument { - arguments := map[string]modulecapabilities.GraphQLArgument{} - arguments["nearVideo"] = g.getNearVideo() - return arguments -} - -func (g *GraphQLArgumentsProvider) getNearVideo() modulecapabilities.GraphQLArgument { - return modulecapabilities.GraphQLArgument{ - GetArgumentsFunction: getNearVideoArgumentFn, - AggregateArgumentsFunction: aggregateNearVideoArgumentFn, - ExploreArgumentsFunction: exploreNearVideoArgumentFn, - ExtractFunction: extractNearVideoFn, - ValidateFunction: nearVideo.ValidateNearVideoFn, - } -} diff --git a/modules/multi2vec-bind/nearVideo/searcher.go b/modules/multi2vec-bind/nearVideo/searcher.go deleted file mode 100644 index 8f963faf44fce01fada40745780f63662be36514..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/nearVideo/searcher.go +++ /dev/null @@ -1,61 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package nearVideo - -import ( - "context" - - "github.com/weaviate/weaviate/usecases/modulecomponents/nearVideo" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" -) - -type Searcher struct { - vectorizer bindVectorizer -} - -func NewSearcher(vectorizer bindVectorizer) *Searcher { - return &Searcher{vectorizer} -} - -type bindVectorizer interface { - VectorizeVideo(ctx context.Context, video string) ([]float32, error) -} - -func (s *Searcher) VectorSearches() map[string]modulecapabilities.VectorForParams { - vectorSearches := map[string]modulecapabilities.VectorForParams{} - vectorSearches["nearVideo"] = s.vectorForNearVideoParam - return vectorSearches -} - -func (s *Searcher) vectorForNearVideoParam(ctx context.Context, params interface{}, - className string, - findVectorFn modulecapabilities.FindVectorFn, - cfg moduletools.ClassConfig, -) ([]float32, error) { - return s.vectorFromNearVideoParam(ctx, params.(*nearVideo.NearVideoParams), className, findVectorFn, cfg) -} - -func (s *Searcher) vectorFromNearVideoParam(ctx context.Context, - params *nearVideo.NearVideoParams, className string, findVectorFn modulecapabilities.FindVectorFn, - cfg moduletools.ClassConfig, -) ([]float32, error) { - // find vector for given search query - vector, err := s.vectorizer.VectorizeVideo(ctx, params.Video) - if err != nil { - return nil, errors.Errorf("vectorize video: %v", err) - } - - return vector, nil -} diff --git a/modules/multi2vec-bind/neardepth/graphql_argument.go b/modules/multi2vec-bind/neardepth/graphql_argument.go deleted file mode 100644 index 4c75e561da8323ec592af5e597701bbc198a3195..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/neardepth/graphql_argument.go +++ /dev/null @@ -1,61 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package neardepth - -import ( - "fmt" - - "github.com/tailor-inc/graphql" - "github.com/weaviate/weaviate/adapters/handlers/graphql/descriptions" -) - -func getNearDepthArgumentFn(classname string) *graphql.ArgumentConfig { - return nearDepthArgument("GetObjects", classname) -} - -func exploreNearDepthArgumentFn() *graphql.ArgumentConfig { - return nearDepthArgument("Explore", "") -} - -func aggregateNearDepthArgumentFn(classname string) *graphql.ArgumentConfig { - return nearDepthArgument("Aggregate", classname) -} - -func nearDepthArgument(prefix, className string) *graphql.ArgumentConfig { - prefixName := fmt.Sprintf("Multi2VecBind%s%s", prefix, className) - return &graphql.ArgumentConfig{ - Type: graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sNearDepthInpObj", prefixName), - Fields: nearDepthFields(prefixName), - Description: descriptions.GetWhereInpObj, - }, - ), - } -} - -func nearDepthFields(prefix string) graphql.InputObjectConfigFieldMap { - return graphql.InputObjectConfigFieldMap{ - "depth": &graphql.InputObjectFieldConfig{ - Description: "Base64 encoded Depth data", - Type: graphql.NewNonNull(graphql.String), - }, - "certainty": &graphql.InputObjectFieldConfig{ - Description: descriptions.Certainty, - Type: graphql.Float, - }, - "distance": &graphql.InputObjectFieldConfig{ - Description: descriptions.Distance, - Type: graphql.Float, - }, - } -} diff --git a/modules/multi2vec-bind/neardepth/graphql_argument_test.go b/modules/multi2vec-bind/neardepth/graphql_argument_test.go deleted file mode 100644 index 0ed2a640ea1a31a3dd30d7c8e993242f5510088a..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/neardepth/graphql_argument_test.go +++ /dev/null @@ -1,50 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package neardepth - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/tailor-inc/graphql" -) - -func TestNearDepthGraphQLArgument(t *testing.T) { - t.Run("should generate nearDepth argument properly", func(t *testing.T) { - // given - prefix := "Prefix" - classname := "Class" - // when - nearDepth := nearDepthArgument(prefix, classname) - - // then - // the built graphQL field needs to support this structure: - // nearDepth: { - // depth: "base64;encoded,depth_depth", - // distance: 0.9 - // } - assert.NotNil(t, nearDepth) - assert.Equal(t, "Multi2VecBindPrefixClassNearDepthInpObj", nearDepth.Type.Name()) - answerFields, ok := nearDepth.Type.(*graphql.InputObject) - assert.True(t, ok) - assert.NotNil(t, answerFields) - assert.Equal(t, 3, len(answerFields.Fields())) - fields := answerFields.Fields() - depth := fields["depth"] - depthNonNull, depthNonNullOK := depth.Type.(*graphql.NonNull) - assert.True(t, depthNonNullOK) - assert.Equal(t, "String", depthNonNull.OfType.Name()) - assert.NotNil(t, depth) - assert.NotNil(t, fields["certainty"]) - assert.NotNil(t, fields["distance"]) - }) -} diff --git a/modules/multi2vec-bind/neardepth/graphql_extract.go b/modules/multi2vec-bind/neardepth/graphql_extract.go deleted file mode 100644 index 7105ed040886532d9a6cadb0387fdfa62e228fc3..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/neardepth/graphql_extract.go +++ /dev/null @@ -1,35 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package neardepth - -// extractNearDepthFn arguments, such as "depth" and "certainty" -func extractNearDepthFn(source map[string]interface{}) interface{} { - var args NearDepthParams - - depth, ok := source["depth"].(string) - if ok { - args.Depth = depth - } - - certainty, ok := source["certainty"] - if ok { - args.Certainty = certainty.(float64) - } - - distance, ok := source["distance"] - if ok { - args.Distance = distance.(float64) - args.WithDistance = true - } - - return &args -} diff --git a/modules/multi2vec-bind/neardepth/graphql_extract_test.go b/modules/multi2vec-bind/neardepth/graphql_extract_test.go deleted file mode 100644 index 6784add7a3e64e0ffcd2df64be1781c54ff3e393..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/neardepth/graphql_extract_test.go +++ /dev/null @@ -1,74 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package neardepth - -import ( - "reflect" - "testing" -) - -func Test_extractNearDepthFn(t *testing.T) { - type args struct { - source map[string]interface{} - } - tests := []struct { - name string - args args - want interface{} - }{ - { - name: "should extract properly with distance and depth params set", - args: args{ - source: map[string]interface{}{ - "depth": "base64;encoded", - "distance": float64(0.9), - }, - }, - want: &NearDepthParams{ - Depth: "base64;encoded", - Distance: 0.9, - WithDistance: true, - }, - }, - { - name: "should extract properly with certainty and depth params set", - args: args{ - source: map[string]interface{}{ - "depth": "base64;encoded", - "certainty": float64(0.9), - }, - }, - want: &NearDepthParams{ - Depth: "base64;encoded", - Certainty: 0.9, - }, - }, - { - name: "should extract properly with only depth set", - args: args{ - source: map[string]interface{}{ - "depth": "base64;encoded", - }, - }, - want: &NearDepthParams{ - Depth: "base64;encoded", - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := extractNearDepthFn(tt.args.source); !reflect.DeepEqual(got, tt.want) { - t.Errorf("extractNearDepthFn() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/modules/multi2vec-bind/neardepth/graphql_provider.go b/modules/multi2vec-bind/neardepth/graphql_provider.go deleted file mode 100644 index 13bb80a755844d360199bc3ae56f663e443b45d1..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/neardepth/graphql_provider.go +++ /dev/null @@ -1,38 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package neardepth - -import ( - "github.com/weaviate/weaviate/entities/modulecapabilities" -) - -type GraphQLArgumentsProvider struct{} - -func New() *GraphQLArgumentsProvider { - return &GraphQLArgumentsProvider{} -} - -func (g *GraphQLArgumentsProvider) Arguments() map[string]modulecapabilities.GraphQLArgument { - arguments := map[string]modulecapabilities.GraphQLArgument{} - arguments["nearDepth"] = g.getNearDepth() - return arguments -} - -func (g *GraphQLArgumentsProvider) getNearDepth() modulecapabilities.GraphQLArgument { - return modulecapabilities.GraphQLArgument{ - GetArgumentsFunction: getNearDepthArgumentFn, - AggregateArgumentsFunction: aggregateNearDepthArgumentFn, - ExploreArgumentsFunction: exploreNearDepthArgumentFn, - ExtractFunction: extractNearDepthFn, - ValidateFunction: validateNearDepthFn, - } -} diff --git a/modules/multi2vec-bind/neardepth/param.go b/modules/multi2vec-bind/neardepth/param.go deleted file mode 100644 index c0bcc6b05f91e18763eb6e57d34cd0a1788e4bde..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/neardepth/param.go +++ /dev/null @@ -1,53 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package neardepth - -import ( - "errors" -) - -type NearDepthParams struct { - Depth string - Certainty float64 - Distance float64 - WithDistance bool -} - -func (n NearDepthParams) GetCertainty() float64 { - return n.Certainty -} - -func (n NearDepthParams) GetDistance() float64 { - return n.Distance -} - -func (n NearDepthParams) SimilarityMetricProvided() bool { - return n.Certainty != 0 || n.WithDistance -} - -func validateNearDepthFn(param interface{}) error { - nearDepth, ok := param.(*NearDepthParams) - if !ok { - return errors.New("'nearDepth' invalid parameter") - } - - if len(nearDepth.Depth) == 0 { - return errors.New("'nearDepth.depth' needs to be defined") - } - - if nearDepth.Certainty != 0 && nearDepth.WithDistance { - return errors.New( - "nearDepth cannot provide both distance and certainty") - } - - return nil -} diff --git a/modules/multi2vec-bind/neardepth/param_test.go b/modules/multi2vec-bind/neardepth/param_test.go deleted file mode 100644 index dc11ee44acab1f4e8fcc24083bb03164bf4b451c..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/neardepth/param_test.go +++ /dev/null @@ -1,78 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package neardepth - -import "testing" - -func Test_validateNearDepthFn(t *testing.T) { - type args struct { - param interface{} - } - tests := []struct { - name string - args args - wantErr bool - }{ - { - name: "should pass with proper values", - args: args{ - param: &NearDepthParams{ - Depth: "base64;enncoded", - }, - }, - }, - { - name: "should not pass with empty depth", - args: args{ - param: &NearDepthParams{ - Depth: "", - }, - }, - wantErr: true, - }, - { - name: "should not pass with nil depth", - args: args{ - param: &NearDepthParams{}, - }, - wantErr: true, - }, - { - name: "should not pass with struct param, not a pointer to struct", - args: args{ - param: NearDepthParams{ - Depth: "depth", - }, - }, - wantErr: true, - }, - { - name: "should not pass with certainty and distance", - args: args{ - param: NearDepthParams{ - Depth: "depth", - Distance: 0.9, - WithDistance: true, - Certainty: 0.1, - }, - }, - wantErr: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := validateNearDepthFn(tt.args.param); (err != nil) != tt.wantErr { - t.Errorf("validateNearDepthFn() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} diff --git a/modules/multi2vec-bind/neardepth/searcher.go b/modules/multi2vec-bind/neardepth/searcher.go deleted file mode 100644 index 3d0bf24bd25f87750a9448c6373e7860afda3fa1..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/neardepth/searcher.go +++ /dev/null @@ -1,59 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package neardepth - -import ( - "context" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" -) - -type Searcher struct { - vectorizer bindVectorizer -} - -func NewSearcher(vectorizer bindVectorizer) *Searcher { - return &Searcher{vectorizer} -} - -type bindVectorizer interface { - VectorizeDepth(ctx context.Context, thermal string) ([]float32, error) -} - -func (s *Searcher) VectorSearches() map[string]modulecapabilities.VectorForParams { - vectorSearches := map[string]modulecapabilities.VectorForParams{} - vectorSearches["nearDepth"] = s.vectorForNearDepthParam - return vectorSearches -} - -func (s *Searcher) vectorForNearDepthParam(ctx context.Context, params interface{}, - className string, - findVectorFn modulecapabilities.FindVectorFn, - cfg moduletools.ClassConfig, -) ([]float32, error) { - return s.vectorFromNearDepthParam(ctx, params.(*NearDepthParams), className, findVectorFn, cfg) -} - -func (s *Searcher) vectorFromNearDepthParam(ctx context.Context, - params *NearDepthParams, className string, findVectorFn modulecapabilities.FindVectorFn, - cfg moduletools.ClassConfig, -) ([]float32, error) { - // find vector for given search query - vector, err := s.vectorizer.VectorizeDepth(ctx, params.Depth) - if err != nil { - return nil, errors.Errorf("vectorize thermal: %v", err) - } - - return vector, nil -} diff --git a/modules/multi2vec-bind/nearimu/graphql_argument.go b/modules/multi2vec-bind/nearimu/graphql_argument.go deleted file mode 100644 index 4c077c48048afe68d04f59dddbc90cbba838fcd1..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/nearimu/graphql_argument.go +++ /dev/null @@ -1,61 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package nearimu - -import ( - "fmt" - - "github.com/tailor-inc/graphql" - "github.com/weaviate/weaviate/adapters/handlers/graphql/descriptions" -) - -func getNearIMUArgumentFn(classname string) *graphql.ArgumentConfig { - return nearIMUArgument("GetObjects", classname) -} - -func exploreNearIMUArgumentFn() *graphql.ArgumentConfig { - return nearIMUArgument("Explore", "") -} - -func aggregateNearIMUArgumentFn(classname string) *graphql.ArgumentConfig { - return nearIMUArgument("Aggregate", classname) -} - -func nearIMUArgument(prefix, className string) *graphql.ArgumentConfig { - prefixName := fmt.Sprintf("Multi2VecBind%s%s", prefix, className) - return &graphql.ArgumentConfig{ - Type: graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sNearIMUInpObj", prefixName), - Fields: nearIMUFields(prefixName), - Description: descriptions.GetWhereInpObj, - }, - ), - } -} - -func nearIMUFields(prefix string) graphql.InputObjectConfigFieldMap { - return graphql.InputObjectConfigFieldMap{ - "imu": &graphql.InputObjectFieldConfig{ - Description: "Base64 encoded IMU data", - Type: graphql.NewNonNull(graphql.String), - }, - "certainty": &graphql.InputObjectFieldConfig{ - Description: descriptions.Certainty, - Type: graphql.Float, - }, - "distance": &graphql.InputObjectFieldConfig{ - Description: descriptions.Distance, - Type: graphql.Float, - }, - } -} diff --git a/modules/multi2vec-bind/nearimu/graphql_argument_test.go b/modules/multi2vec-bind/nearimu/graphql_argument_test.go deleted file mode 100644 index e9018388eff7d4a82b3d0c432845e630b170727f..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/nearimu/graphql_argument_test.go +++ /dev/null @@ -1,50 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package nearimu - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/tailor-inc/graphql" -) - -func TestNearIMUGraphQLArgument(t *testing.T) { - t.Run("should generate nearIMU argument properly", func(t *testing.T) { - // given - prefix := "Prefix" - classname := "Class" - // when - nearIMU := nearIMUArgument(prefix, classname) - - // then - // the built graphQL field needs to support this structure: - // nearIMU: { - // imu: "base64;encoded,imu_data", - // distance: 0.9 - // } - assert.NotNil(t, nearIMU) - assert.Equal(t, "Multi2VecBindPrefixClassNearIMUInpObj", nearIMU.Type.Name()) - answerFields, ok := nearIMU.Type.(*graphql.InputObject) - assert.True(t, ok) - assert.NotNil(t, answerFields) - assert.Equal(t, 3, len(answerFields.Fields())) - fields := answerFields.Fields() - imu := fields["imu"] - imuNonNull, imuNonNullOK := imu.Type.(*graphql.NonNull) - assert.True(t, imuNonNullOK) - assert.Equal(t, "String", imuNonNull.OfType.Name()) - assert.NotNil(t, imu) - assert.NotNil(t, fields["certainty"]) - assert.NotNil(t, fields["distance"]) - }) -} diff --git a/modules/multi2vec-bind/nearimu/graphql_extract.go b/modules/multi2vec-bind/nearimu/graphql_extract.go deleted file mode 100644 index c255e7338be9c1dedd251fe4d46984fd100e1613..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/nearimu/graphql_extract.go +++ /dev/null @@ -1,35 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package nearimu - -// extractNearIMUFn arguments, such as "imu" and "certainty" -func extractNearIMUFn(source map[string]interface{}) interface{} { - var args NearIMUParams - - video, ok := source["imu"].(string) - if ok { - args.IMU = video - } - - certainty, ok := source["certainty"] - if ok { - args.Certainty = certainty.(float64) - } - - distance, ok := source["distance"] - if ok { - args.Distance = distance.(float64) - args.WithDistance = true - } - - return &args -} diff --git a/modules/multi2vec-bind/nearimu/graphql_extract_test.go b/modules/multi2vec-bind/nearimu/graphql_extract_test.go deleted file mode 100644 index 3d74401c69f9824d308e75b86263caaa4d425661..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/nearimu/graphql_extract_test.go +++ /dev/null @@ -1,74 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package nearimu - -import ( - "reflect" - "testing" -) - -func Test_extractNearIMUFn(t *testing.T) { - type args struct { - source map[string]interface{} - } - tests := []struct { - name string - args args - want interface{} - }{ - { - name: "should extract properly with distance and imu params set", - args: args{ - source: map[string]interface{}{ - "imu": "base64;encoded", - "distance": float64(0.9), - }, - }, - want: &NearIMUParams{ - IMU: "base64;encoded", - Distance: 0.9, - WithDistance: true, - }, - }, - { - name: "should extract properly with certainty and imu params set", - args: args{ - source: map[string]interface{}{ - "imu": "base64;encoded", - "certainty": float64(0.9), - }, - }, - want: &NearIMUParams{ - IMU: "base64;encoded", - Certainty: 0.9, - }, - }, - { - name: "should extract properly with only imu set", - args: args{ - source: map[string]interface{}{ - "imu": "base64;encoded", - }, - }, - want: &NearIMUParams{ - IMU: "base64;encoded", - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := extractNearIMUFn(tt.args.source); !reflect.DeepEqual(got, tt.want) { - t.Errorf("extractNearIMUFn() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/modules/multi2vec-bind/nearimu/graphql_provider.go b/modules/multi2vec-bind/nearimu/graphql_provider.go deleted file mode 100644 index 25ad3c82adcd29454b6a5147b23b9e1cbecc48c6..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/nearimu/graphql_provider.go +++ /dev/null @@ -1,38 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package nearimu - -import ( - "github.com/weaviate/weaviate/entities/modulecapabilities" -) - -type GraphQLArgumentsProvider struct{} - -func New() *GraphQLArgumentsProvider { - return &GraphQLArgumentsProvider{} -} - -func (g *GraphQLArgumentsProvider) Arguments() map[string]modulecapabilities.GraphQLArgument { - arguments := map[string]modulecapabilities.GraphQLArgument{} - arguments["nearIMU"] = g.getNearIMU() - return arguments -} - -func (g *GraphQLArgumentsProvider) getNearIMU() modulecapabilities.GraphQLArgument { - return modulecapabilities.GraphQLArgument{ - GetArgumentsFunction: getNearIMUArgumentFn, - AggregateArgumentsFunction: aggregateNearIMUArgumentFn, - ExploreArgumentsFunction: exploreNearIMUArgumentFn, - ExtractFunction: extractNearIMUFn, - ValidateFunction: validateNearIMUFn, - } -} diff --git a/modules/multi2vec-bind/nearimu/param.go b/modules/multi2vec-bind/nearimu/param.go deleted file mode 100644 index 4e8bf7bee328edad3e62ac084fa747f501008f8d..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/nearimu/param.go +++ /dev/null @@ -1,53 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package nearimu - -import ( - "errors" -) - -type NearIMUParams struct { - IMU string - Certainty float64 - Distance float64 - WithDistance bool -} - -func (n NearIMUParams) GetCertainty() float64 { - return n.Certainty -} - -func (n NearIMUParams) GetDistance() float64 { - return n.Distance -} - -func (n NearIMUParams) SimilarityMetricProvided() bool { - return n.Certainty != 0 || n.WithDistance -} - -func validateNearIMUFn(param interface{}) error { - nearIMU, ok := param.(*NearIMUParams) - if !ok { - return errors.New("'nearIMU' invalid parameter") - } - - if len(nearIMU.IMU) == 0 { - return errors.New("'nearIMU.imu' needs to be defined") - } - - if nearIMU.Certainty != 0 && nearIMU.WithDistance { - return errors.New( - "nearIMU cannot provide both distance and certainty") - } - - return nil -} diff --git a/modules/multi2vec-bind/nearimu/param_test.go b/modules/multi2vec-bind/nearimu/param_test.go deleted file mode 100644 index d5a779cdbb382ea4668ca190aa10fdd5f8881209..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/nearimu/param_test.go +++ /dev/null @@ -1,78 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package nearimu - -import "testing" - -func Test_validateNearImageFn(t *testing.T) { - type args struct { - param interface{} - } - tests := []struct { - name string - args args - wantErr bool - }{ - { - name: "should pass with proper values", - args: args{ - param: &NearIMUParams{ - IMU: "base64;enncoded", - }, - }, - }, - { - name: "should not pass with empty image", - args: args{ - param: &NearIMUParams{ - IMU: "", - }, - }, - wantErr: true, - }, - { - name: "should not pass with nil image", - args: args{ - param: &NearIMUParams{}, - }, - wantErr: true, - }, - { - name: "should not pass with struct param, not a pointer to struct", - args: args{ - param: NearIMUParams{ - IMU: "image", - }, - }, - wantErr: true, - }, - { - name: "should not pass with certainty and distance", - args: args{ - param: NearIMUParams{ - IMU: "imu_data", - Distance: 0.9, - WithDistance: true, - Certainty: 0.1, - }, - }, - wantErr: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := validateNearIMUFn(tt.args.param); (err != nil) != tt.wantErr { - t.Errorf("validateNearIMUFn() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} diff --git a/modules/multi2vec-bind/nearimu/searcher.go b/modules/multi2vec-bind/nearimu/searcher.go deleted file mode 100644 index 9934771ddd168e6a26e6393032c5c835b135609b..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/nearimu/searcher.go +++ /dev/null @@ -1,59 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package nearimu - -import ( - "context" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" -) - -type Searcher struct { - vectorizer bindVectorizer -} - -func NewSearcher(vectorizer bindVectorizer) *Searcher { - return &Searcher{vectorizer} -} - -type bindVectorizer interface { - VectorizeIMU(ctx context.Context, video string) ([]float32, error) -} - -func (s *Searcher) VectorSearches() map[string]modulecapabilities.VectorForParams { - vectorSearches := map[string]modulecapabilities.VectorForParams{} - vectorSearches["nearIMU"] = s.vectorForNearIMUParam - return vectorSearches -} - -func (s *Searcher) vectorForNearIMUParam(ctx context.Context, params interface{}, - className string, - findVectorFn modulecapabilities.FindVectorFn, - cfg moduletools.ClassConfig, -) ([]float32, error) { - return s.vectorFromNearIMUParam(ctx, params.(*NearIMUParams), className, findVectorFn, cfg) -} - -func (s *Searcher) vectorFromNearIMUParam(ctx context.Context, - params *NearIMUParams, className string, findVectorFn modulecapabilities.FindVectorFn, - cfg moduletools.ClassConfig, -) ([]float32, error) { - // find vector for given search query - vector, err := s.vectorizer.VectorizeIMU(ctx, params.IMU) - if err != nil { - return nil, errors.Errorf("vectorize imu: %v", err) - } - - return vector, nil -} diff --git a/modules/multi2vec-bind/neartext/fakes_for_test.go b/modules/multi2vec-bind/neartext/fakes_for_test.go deleted file mode 100644 index a72048d0b9cb0b05c8c5ca648cf95db841571505..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/neartext/fakes_for_test.go +++ /dev/null @@ -1,26 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package neartext - -type fakeTransformer struct{} - -func (t *fakeTransformer) Transform(in []string) ([]string, error) { - result := make([]string, len(in)) - for i, txt := range in { - if txt == "transform this" { - result[i] = "transformed text" - } else { - result[i] = txt - } - } - return result, nil -} diff --git a/modules/multi2vec-bind/neartext/graphql_argument.go b/modules/multi2vec-bind/neartext/graphql_argument.go deleted file mode 100644 index 2b51c63154166f96204878def168f577ba126557..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/neartext/graphql_argument.go +++ /dev/null @@ -1,120 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package neartext - -import ( - "fmt" - - "github.com/tailor-inc/graphql" - "github.com/weaviate/weaviate/adapters/handlers/graphql/descriptions" -) - -func (g *GraphQLArgumentsProvider) getNearTextArgumentFn(classname string) *graphql.ArgumentConfig { - return g.nearTextArgument("GetObjects", classname) -} - -func (g *GraphQLArgumentsProvider) exploreNearTextArgumentFn() *graphql.ArgumentConfig { - return g.nearTextArgument("Explore", "") -} - -func (g *GraphQLArgumentsProvider) aggregateNearTextArgumentFn(classname string) *graphql.ArgumentConfig { - return g.nearTextArgument("Aggregate", classname) -} - -func (g *GraphQLArgumentsProvider) nearTextArgument(prefix, className string) *graphql.ArgumentConfig { - prefixName := fmt.Sprintf("Multi2VecBind%s%s", prefix, className) - return &graphql.ArgumentConfig{ - Type: graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sNearTextInpObj", prefixName), - Fields: g.nearTextFields(prefixName), - Description: descriptions.GetWhereInpObj, - }, - ), - } -} - -func (g *GraphQLArgumentsProvider) nearTextFields(prefix string) graphql.InputObjectConfigFieldMap { - nearTextFields := graphql.InputObjectConfigFieldMap{ - "concepts": &graphql.InputObjectFieldConfig{ - // Description: descriptions.Concepts, - Type: graphql.NewNonNull(graphql.NewList(graphql.String)), - }, - "moveTo": &graphql.InputObjectFieldConfig{ - Description: descriptions.VectorMovement, - Type: graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sMoveTo", prefix), - Fields: g.movementInp(fmt.Sprintf("%sMoveTo", prefix)), - }), - }, - "certainty": &graphql.InputObjectFieldConfig{ - Description: descriptions.Certainty, - Type: graphql.Float, - }, - "distance": &graphql.InputObjectFieldConfig{ - Description: descriptions.Distance, - Type: graphql.Float, - }, - "moveAwayFrom": &graphql.InputObjectFieldConfig{ - Description: descriptions.VectorMovement, - Type: graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sMoveAwayFrom", prefix), - Fields: g.movementInp(fmt.Sprintf("%sMoveAwayFrom", prefix)), - }), - }, - } - if g.nearTextTransformer != nil { - nearTextFields["autocorrect"] = &graphql.InputObjectFieldConfig{ - Description: "Autocorrect input text values", - Type: graphql.Boolean, - } - } - return nearTextFields -} - -func (g *GraphQLArgumentsProvider) movementInp(prefix string) graphql.InputObjectConfigFieldMap { - return graphql.InputObjectConfigFieldMap{ - "concepts": &graphql.InputObjectFieldConfig{ - Description: descriptions.Keywords, - Type: graphql.NewList(graphql.String), - }, - "objects": &graphql.InputObjectFieldConfig{ - Description: "objects", - Type: graphql.NewList(g.objectsInpObj(prefix)), - }, - "force": &graphql.InputObjectFieldConfig{ - Description: descriptions.Force, - Type: graphql.NewNonNull(graphql.Float), - }, - } -} - -func (g *GraphQLArgumentsProvider) objectsInpObj(prefix string) *graphql.InputObject { - return graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sMovementObjectsInpObj", prefix), - Fields: graphql.InputObjectConfigFieldMap{ - "id": &graphql.InputObjectFieldConfig{ - Type: graphql.String, - Description: "id of an object", - }, - "beacon": &graphql.InputObjectFieldConfig{ - Type: graphql.String, - Description: descriptions.Beacon, - }, - }, - Description: "Movement Object", - }, - ) -} diff --git a/modules/multi2vec-bind/neartext/graphql_argument_test.go b/modules/multi2vec-bind/neartext/graphql_argument_test.go deleted file mode 100644 index 823d20bd5c381ba733f1081dc9667fe17f99fd14..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/neartext/graphql_argument_test.go +++ /dev/null @@ -1,194 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package neartext - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/tailor-inc/graphql" -) - -func TestNearTextGraphQLArgument(t *testing.T) { - t.Run("should generate nearText argument properly", func(t *testing.T) { - // given - prefix := "Prefix" - classname := "Class" - // when - nearText := New(nil).nearTextArgument(prefix, classname) - - // then - // the built graphQL field needs to support this structure: - // { - // concepts: ["c1", "c2"], - // certainty: 0.9, (or distance) - // moveTo: { - // concepts: ["c1", "c2"], - // objects: [ - // { id: "some-uuid-value"}], - // { beacon: "some-beacon-value"} - // ], - // force: 0.8 - // } - // moveAwayFrom: { - // concepts: ["c1", "c2"], - // objects: [ - // { id: "some-uuid-value"}], - // { beacon: "some-beacon-value"} - // ], - // force: 0.8 - // } - // } - assert.NotNil(t, nearText) - assert.Equal(t, "Multi2VecBindPrefixClassNearTextInpObj", nearText.Type.Name()) - nearTextFields, ok := nearText.Type.(*graphql.InputObject) - assert.True(t, ok) - assert.NotNil(t, nearTextFields) - assert.Equal(t, 5, len(nearTextFields.Fields())) - fields := nearTextFields.Fields() - concepts := fields["concepts"] - conceptsNonNull, conceptsNonNullOK := concepts.Type.(*graphql.NonNull) - assert.True(t, conceptsNonNullOK) - conceptsNonNullList, conceptsNonNullListOK := conceptsNonNull.OfType.(*graphql.List) - assert.True(t, conceptsNonNullListOK) - assert.Equal(t, "String", conceptsNonNullList.OfType.Name()) - assert.NotNil(t, concepts) - conceptsType, conceptsTypeOK := concepts.Type.(*graphql.NonNull) - assert.True(t, conceptsTypeOK) - assert.NotNil(t, conceptsType) - assert.NotNil(t, fields["certainty"]) - assert.NotNil(t, fields["distance"]) - assert.NotNil(t, fields["moveTo"]) - moveTo, moveToOK := fields["moveTo"].Type.(*graphql.InputObject) - assert.True(t, moveToOK) - assert.Equal(t, 3, len(moveTo.Fields())) - assert.NotNil(t, moveTo.Fields()["concepts"]) - moveToConcepts, moveToConceptsOK := moveTo.Fields()["concepts"].Type.(*graphql.List) - assert.True(t, moveToConceptsOK) - assert.Equal(t, "String", moveToConcepts.OfType.Name()) - assert.NotNil(t, moveToConcepts) - assert.NotNil(t, moveTo.Fields()["objects"]) - moveToObjects, moveToObjectsOK := moveTo.Fields()["objects"].Type.(*graphql.List) - assert.True(t, moveToObjectsOK) - moveToObjectsObjects, moveToObjectsObjectsOK := moveToObjects.OfType.(*graphql.InputObject) - assert.True(t, moveToObjectsObjectsOK) - assert.Equal(t, 2, len(moveToObjectsObjects.Fields())) - assert.NotNil(t, moveToObjectsObjects.Fields()["id"]) - assert.NotNil(t, moveToObjectsObjects.Fields()["beacon"]) - assert.NotNil(t, moveTo.Fields()["force"]) - _, moveToForceOK := moveTo.Fields()["force"].Type.(*graphql.NonNull) - assert.True(t, moveToForceOK) - assert.NotNil(t, fields["moveAwayFrom"]) - moveAwayFrom, moveAwayFromOK := fields["moveAwayFrom"].Type.(*graphql.InputObject) - assert.True(t, moveAwayFromOK) - assert.NotNil(t, moveAwayFrom.Fields()["concepts"]) - assert.NotNil(t, moveAwayFrom.Fields()["objects"]) - moveAwayFromObjects, moveAwayFromObjectsOK := moveAwayFrom.Fields()["objects"].Type.(*graphql.List) - assert.True(t, moveAwayFromObjectsOK) - moveAwayFromObjectsObjects, moveAwayFromObjectsObjectsOK := moveAwayFromObjects.OfType.(*graphql.InputObject) - assert.Equal(t, 2, len(moveAwayFromObjectsObjects.Fields())) - assert.True(t, moveAwayFromObjectsObjectsOK) - assert.NotNil(t, moveAwayFromObjectsObjects.Fields()["id"]) - assert.NotNil(t, moveAwayFromObjectsObjects.Fields()["beacon"]) - assert.NotNil(t, moveAwayFrom.Fields()["force"]) - _, moveAwayFromForceOK := moveAwayFrom.Fields()["force"].Type.(*graphql.NonNull) - assert.True(t, moveAwayFromForceOK) - }) -} - -func TestNearTextGraphQLArgumentWithAutocorrect(t *testing.T) { - t.Run("should generate nearText argument with autocorrect properly", func(t *testing.T) { - // given - prefix := "Prefix" - classname := "Class" - // when - nearText := New(&fakeTransformer{}).nearTextArgument(prefix, classname) - - // then - // the built graphQL field needs to support this structure: - // { - // concepts: ["c1", "c2"], - // certainty: 0.9, (or distance) - // autocorrect: true, - // moveTo: { - // concepts: ["c1", "c2"], - // objects: [ - // { id: "some-uuid-value"}], - // { beacon: "some-beacon-value"} - // ], - // force: 0.8 - // } - // moveAwayFrom: { - // concepts: ["c1", "c2"], - // objects: [ - // { id: "some-uuid-value"}], - // { beacon: "some-beacon-value"} - // ], - // force: 0.8 - // } - // } - assert.NotNil(t, nearText) - assert.Equal(t, "Multi2VecBindPrefixClassNearTextInpObj", nearText.Type.Name()) - nearTextFields, ok := nearText.Type.(*graphql.InputObject) - assert.True(t, ok) - assert.NotNil(t, nearTextFields) - assert.Equal(t, 6, len(nearTextFields.Fields())) - fields := nearTextFields.Fields() - concepts := fields["concepts"] - conceptsNonNull, conceptsNonNullOK := concepts.Type.(*graphql.NonNull) - assert.True(t, conceptsNonNullOK) - conceptsNonNullList, conceptsNonNullListOK := conceptsNonNull.OfType.(*graphql.List) - assert.True(t, conceptsNonNullListOK) - assert.Equal(t, "String", conceptsNonNullList.OfType.Name()) - assert.NotNil(t, concepts) - conceptsType, conceptsTypeOK := concepts.Type.(*graphql.NonNull) - assert.True(t, conceptsTypeOK) - assert.NotNil(t, conceptsType) - assert.NotNil(t, fields["certainty"]) - assert.NotNil(t, fields["autocorrect"]) - assert.NotNil(t, fields["moveTo"]) - moveTo, moveToOK := fields["moveTo"].Type.(*graphql.InputObject) - assert.True(t, moveToOK) - assert.Equal(t, 3, len(moveTo.Fields())) - assert.NotNil(t, moveTo.Fields()["concepts"]) - moveToConcepts, moveToConceptsOK := moveTo.Fields()["concepts"].Type.(*graphql.List) - assert.True(t, moveToConceptsOK) - assert.Equal(t, "String", moveToConcepts.OfType.Name()) - assert.NotNil(t, moveToConcepts) - assert.NotNil(t, moveTo.Fields()["objects"]) - moveToObjects, moveToObjectsOK := moveTo.Fields()["objects"].Type.(*graphql.List) - assert.True(t, moveToObjectsOK) - moveToObjectsObjects, moveToObjectsObjectsOK := moveToObjects.OfType.(*graphql.InputObject) - assert.True(t, moveToObjectsObjectsOK) - assert.Equal(t, 2, len(moveToObjectsObjects.Fields())) - assert.NotNil(t, moveToObjectsObjects.Fields()["id"]) - assert.NotNil(t, moveToObjectsObjects.Fields()["beacon"]) - assert.NotNil(t, moveTo.Fields()["force"]) - _, moveToForceOK := moveTo.Fields()["force"].Type.(*graphql.NonNull) - assert.True(t, moveToForceOK) - assert.NotNil(t, fields["moveAwayFrom"]) - moveAwayFrom, moveAwayFromOK := fields["moveAwayFrom"].Type.(*graphql.InputObject) - assert.True(t, moveAwayFromOK) - assert.NotNil(t, moveAwayFrom.Fields()["concepts"]) - assert.NotNil(t, moveAwayFrom.Fields()["objects"]) - moveAwayFromObjects, moveAwayFromObjectsOK := moveAwayFrom.Fields()["objects"].Type.(*graphql.List) - assert.True(t, moveAwayFromObjectsOK) - moveAwayFromObjectsObjects, moveAwayFromObjectsObjectsOK := moveAwayFromObjects.OfType.(*graphql.InputObject) - assert.Equal(t, 2, len(moveAwayFromObjectsObjects.Fields())) - assert.True(t, moveAwayFromObjectsObjectsOK) - assert.NotNil(t, moveAwayFromObjectsObjects.Fields()["id"]) - assert.NotNil(t, moveAwayFromObjectsObjects.Fields()["beacon"]) - assert.NotNil(t, moveAwayFrom.Fields()["force"]) - _, moveAwayFromForceOK := moveAwayFrom.Fields()["force"].Type.(*graphql.NonNull) - assert.True(t, moveAwayFromForceOK) - }) -} diff --git a/modules/multi2vec-bind/neartext/graphql_extract_test.go b/modules/multi2vec-bind/neartext/graphql_extract_test.go deleted file mode 100644 index 52c24d7fc8edcd21b6d1fb1292622233b425269b..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/neartext/graphql_extract_test.go +++ /dev/null @@ -1,620 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package neartext - -import ( - "reflect" - "testing" - - "github.com/weaviate/weaviate/usecases/modulecomponents/nearText" -) - -func Test_extractNearTextFn(t *testing.T) { - type args struct { - source map[string]interface{} - } - tests := []struct { - name string - args args - want *nearText.NearTextParams - }{ - { - "Extract with concepts", - args{ - source: map[string]interface{}{ - "concepts": []interface{}{"c1", "c2", "c3"}, - }, - }, - &nearText.NearTextParams{ - Values: []string{"c1", "c2", "c3"}, - }, - }, - { - "Extract with concepts, distance, limit and network", - args{ - source: map[string]interface{}{ - "concepts": []interface{}{"c1", "c2", "c3"}, - "distance": float64(0.4), - "limit": 100, - "network": true, - }, - }, - &nearText.NearTextParams{ - Values: []string{"c1", "c2", "c3"}, - Distance: 0.4, - WithDistance: true, - Limit: 100, - Network: true, - }, - }, - { - "Extract with concepts, certainty, limit and network", - args{ - source: map[string]interface{}{ - "concepts": []interface{}{"c1", "c2", "c3"}, - "certainty": float64(0.4), - "limit": 100, - "network": true, - }, - }, - &nearText.NearTextParams{ - Values: []string{"c1", "c2", "c3"}, - Certainty: 0.4, - Limit: 100, - Network: true, - }, - }, - { - "Extract with moveTo, moveAwayFrom, and distance", - args{ - source: map[string]interface{}{ - "concepts": []interface{}{"c1", "c2", "c3"}, - "distance": float64(0.89), - "limit": 500, - "network": false, - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"positive"}, - "force": float64(0.5), - }, - "moveAwayFrom": map[string]interface{}{ - "concepts": []interface{}{"epic"}, - "force": float64(0.25), - }, - }, - }, - &nearText.NearTextParams{ - Values: []string{"c1", "c2", "c3"}, - Distance: 0.89, - WithDistance: true, - Limit: 500, - Network: false, - MoveTo: nearText.ExploreMove{ - Values: []string{"positive"}, - Force: 0.5, - }, - MoveAwayFrom: nearText.ExploreMove{ - Values: []string{"epic"}, - Force: 0.25, - }, - }, - }, - { - "Extract with moveTo, moveAwayFrom, and certainty", - args{ - source: map[string]interface{}{ - "concepts": []interface{}{"c1", "c2", "c3"}, - "certainty": float64(0.89), - "limit": 500, - "network": false, - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"positive"}, - "force": float64(0.5), - }, - "moveAwayFrom": map[string]interface{}{ - "concepts": []interface{}{"epic"}, - "force": float64(0.25), - }, - }, - }, - &nearText.NearTextParams{ - Values: []string{"c1", "c2", "c3"}, - Certainty: 0.89, - Limit: 500, - Network: false, - MoveTo: nearText.ExploreMove{ - Values: []string{"positive"}, - Force: 0.5, - }, - MoveAwayFrom: nearText.ExploreMove{ - Values: []string{"epic"}, - Force: 0.25, - }, - }, - }, - { - "Extract with moveTo, moveAwayFrom, distance, and objects", - args{ - source: map[string]interface{}{ - "concepts": []interface{}{"c1", "c2", "c3"}, - "distance": float64(0.89), - "limit": 500, - "network": false, - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"positive"}, - "force": float64(0.5), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveTo-uuid1", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveTo-uuid2", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveTo-uuid3", - }, - }, - }, - "moveAwayFrom": map[string]interface{}{ - "concepts": []interface{}{"epic"}, - "force": float64(0.25), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveAwayFrom-uuid1", - }, - map[string]interface{}{ - "id": "moveAwayFrom-uuid2", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveAwayFrom-uuid3", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveAwayFrom-uuid4", - }, - }, - }, - }, - }, - &nearText.NearTextParams{ - Values: []string{"c1", "c2", "c3"}, - Distance: 0.89, - WithDistance: true, - Limit: 500, - Network: false, - MoveTo: nearText.ExploreMove{ - Values: []string{"positive"}, - Force: 0.5, - Objects: []nearText.ObjectMove{ - {ID: "moveTo-uuid1"}, - {Beacon: "weaviate://localhost/moveTo-uuid2"}, - {Beacon: "weaviate://localhost/moveTo-uuid3"}, - }, - }, - MoveAwayFrom: nearText.ExploreMove{ - Values: []string{"epic"}, - Force: 0.25, - Objects: []nearText.ObjectMove{ - {ID: "moveAwayFrom-uuid1"}, - {ID: "moveAwayFrom-uuid2"}, - {Beacon: "weaviate://localhost/moveAwayFrom-uuid3"}, - {Beacon: "weaviate://localhost/moveAwayFrom-uuid4"}, - }, - }, - }, - }, - { - "Extract with moveTo and moveAwayFrom (and objects)", - args{ - source: map[string]interface{}{ - "concepts": []interface{}{"c1", "c2", "c3"}, - "certainty": float64(0.89), - "limit": 500, - "network": false, - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"positive"}, - "force": float64(0.5), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveTo-uuid1", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveTo-uuid2", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveTo-uuid3", - }, - }, - }, - "moveAwayFrom": map[string]interface{}{ - "concepts": []interface{}{"epic"}, - "force": float64(0.25), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveAwayFrom-uuid1", - }, - map[string]interface{}{ - "id": "moveAwayFrom-uuid2", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveAwayFrom-uuid3", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveAwayFrom-uuid4", - }, - }, - }, - }, - }, - &nearText.NearTextParams{ - Values: []string{"c1", "c2", "c3"}, - Certainty: 0.89, - Limit: 500, - Network: false, - MoveTo: nearText.ExploreMove{ - Values: []string{"positive"}, - Force: 0.5, - Objects: []nearText.ObjectMove{ - {ID: "moveTo-uuid1"}, - {Beacon: "weaviate://localhost/moveTo-uuid2"}, - {Beacon: "weaviate://localhost/moveTo-uuid3"}, - }, - }, - MoveAwayFrom: nearText.ExploreMove{ - Values: []string{"epic"}, - Force: 0.25, - Objects: []nearText.ObjectMove{ - {ID: "moveAwayFrom-uuid1"}, - {ID: "moveAwayFrom-uuid2"}, - {Beacon: "weaviate://localhost/moveAwayFrom-uuid3"}, - {Beacon: "weaviate://localhost/moveAwayFrom-uuid4"}, - }, - }, - }, - }, - { - "Extract with moveTo and moveAwayFrom, distance, and doubled objects", - args{ - source: map[string]interface{}{ - "concepts": []interface{}{"c1", "c2", "c3"}, - "distance": float64(0.89), - "limit": 500, - "network": false, - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"positive"}, - "force": float64(0.5), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveTo-uuid1", - "beacon": "weaviate://localhost/moveTo-uuid2", - }, - map[string]interface{}{ - "id": "moveTo-uuid1", - "beacon": "weaviate://localhost/moveTo-uuid2", - }, - }, - }, - "moveAwayFrom": map[string]interface{}{ - "concepts": []interface{}{"epic"}, - "force": float64(0.25), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveAwayFrom-uuid1", - "beacon": "weaviate://localhost/moveAwayFrom-uuid1", - }, - map[string]interface{}{ - "id": "moveAwayFrom-uuid2", - "beacon": "weaviate://localhost/moveAwayFrom-uuid2", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveAwayFrom-uuid3", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveAwayFrom-uuid4", - }, - }, - }, - }, - }, - &nearText.NearTextParams{ - Values: []string{"c1", "c2", "c3"}, - Distance: 0.89, - WithDistance: true, - Limit: 500, - Network: false, - MoveTo: nearText.ExploreMove{ - Values: []string{"positive"}, - Force: 0.5, - Objects: []nearText.ObjectMove{ - {ID: "moveTo-uuid1", Beacon: "weaviate://localhost/moveTo-uuid2"}, - {ID: "moveTo-uuid1", Beacon: "weaviate://localhost/moveTo-uuid2"}, - }, - }, - MoveAwayFrom: nearText.ExploreMove{ - Values: []string{"epic"}, - Force: 0.25, - Objects: []nearText.ObjectMove{ - {ID: "moveAwayFrom-uuid1", Beacon: "weaviate://localhost/moveAwayFrom-uuid1"}, - {ID: "moveAwayFrom-uuid2", Beacon: "weaviate://localhost/moveAwayFrom-uuid2"}, - {Beacon: "weaviate://localhost/moveAwayFrom-uuid3"}, - {Beacon: "weaviate://localhost/moveAwayFrom-uuid4"}, - }, - }, - }, - }, - { - "Extract with moveTo and moveAwayFrom, certainty, and doubled objects", - args{ - source: map[string]interface{}{ - "concepts": []interface{}{"c1", "c2", "c3"}, - "certainty": float64(0.89), - "limit": 500, - "network": false, - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"positive"}, - "force": float64(0.5), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveTo-uuid1", - "beacon": "weaviate://localhost/moveTo-uuid2", - }, - map[string]interface{}{ - "id": "moveTo-uuid1", - "beacon": "weaviate://localhost/moveTo-uuid2", - }, - }, - }, - "moveAwayFrom": map[string]interface{}{ - "concepts": []interface{}{"epic"}, - "force": float64(0.25), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveAwayFrom-uuid1", - "beacon": "weaviate://localhost/moveAwayFrom-uuid1", - }, - map[string]interface{}{ - "id": "moveAwayFrom-uuid2", - "beacon": "weaviate://localhost/moveAwayFrom-uuid2", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveAwayFrom-uuid3", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveAwayFrom-uuid4", - }, - }, - }, - }, - }, - &nearText.NearTextParams{ - Values: []string{"c1", "c2", "c3"}, - Certainty: 0.89, - Limit: 500, - Network: false, - MoveTo: nearText.ExploreMove{ - Values: []string{"positive"}, - Force: 0.5, - Objects: []nearText.ObjectMove{ - {ID: "moveTo-uuid1", Beacon: "weaviate://localhost/moveTo-uuid2"}, - {ID: "moveTo-uuid1", Beacon: "weaviate://localhost/moveTo-uuid2"}, - }, - }, - MoveAwayFrom: nearText.ExploreMove{ - Values: []string{"epic"}, - Force: 0.25, - Objects: []nearText.ObjectMove{ - {ID: "moveAwayFrom-uuid1", Beacon: "weaviate://localhost/moveAwayFrom-uuid1"}, - {ID: "moveAwayFrom-uuid2", Beacon: "weaviate://localhost/moveAwayFrom-uuid2"}, - {Beacon: "weaviate://localhost/moveAwayFrom-uuid3"}, - {Beacon: "weaviate://localhost/moveAwayFrom-uuid4"}, - }, - }, - }, - }, - } - - testsWithAutocorrect := []struct { - name string - args args - want *nearText.NearTextParams - }{ - { - "Extract with concepts", - args{ - source: map[string]interface{}{ - "concepts": []interface{}{"c1", "c2", "c3"}, - "autocorrect": true, - }, - }, - &nearText.NearTextParams{ - Values: []string{"c1", "c2", "c3"}, - Autocorrect: true, - }, - }, - { - "Extract with concepts and perform autocorrect", - args{ - source: map[string]interface{}{ - "concepts": []interface{}{"transform this", "c2", "transform this"}, - "autocorrect": true, - }, - }, - &nearText.NearTextParams{ - Values: []string{"transformed text", "c2", "transformed text"}, - Autocorrect: true, - }, - }, - { - "Extract with moveTo and moveAwayFrom, distance (and doubled objects) and autocorrect", - args{ - source: map[string]interface{}{ - "concepts": []interface{}{"transform this", "c1", "c2", "c3", "transform this"}, - "distance": float64(0.89), - "limit": 500, - "network": false, - "autocorrect": true, - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"positive"}, - "force": float64(0.5), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveTo-uuid1", - "beacon": "weaviate://localhost/moveTo-uuid2", - }, - map[string]interface{}{ - "id": "moveTo-uuid1", - "beacon": "weaviate://localhost/moveTo-uuid2", - }, - }, - }, - "moveAwayFrom": map[string]interface{}{ - "concepts": []interface{}{"epic"}, - "force": float64(0.25), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveAwayFrom-uuid1", - "beacon": "weaviate://localhost/moveAwayFrom-uuid1", - }, - map[string]interface{}{ - "id": "moveAwayFrom-uuid2", - "beacon": "weaviate://localhost/moveAwayFrom-uuid2", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveAwayFrom-uuid3", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveAwayFrom-uuid4", - }, - }, - }, - }, - }, - &nearText.NearTextParams{ - Values: []string{"transformed text", "c1", "c2", "c3", "transformed text"}, - Distance: 0.89, - WithDistance: true, - Limit: 500, - Network: false, - Autocorrect: true, - MoveTo: nearText.ExploreMove{ - Values: []string{"positive"}, - Force: 0.5, - Objects: []nearText.ObjectMove{ - {ID: "moveTo-uuid1", Beacon: "weaviate://localhost/moveTo-uuid2"}, - {ID: "moveTo-uuid1", Beacon: "weaviate://localhost/moveTo-uuid2"}, - }, - }, - MoveAwayFrom: nearText.ExploreMove{ - Values: []string{"epic"}, - Force: 0.25, - Objects: []nearText.ObjectMove{ - {ID: "moveAwayFrom-uuid1", Beacon: "weaviate://localhost/moveAwayFrom-uuid1"}, - {ID: "moveAwayFrom-uuid2", Beacon: "weaviate://localhost/moveAwayFrom-uuid2"}, - {Beacon: "weaviate://localhost/moveAwayFrom-uuid3"}, - {Beacon: "weaviate://localhost/moveAwayFrom-uuid4"}, - }, - }, - }, - }, - { - "Extract with moveTo and moveAwayFrom, certainty (and doubled objects) and autocorrect", - args{ - source: map[string]interface{}{ - "concepts": []interface{}{"transform this", "c1", "c2", "c3", "transform this"}, - "certainty": float64(0.89), - "limit": 500, - "network": false, - "autocorrect": true, - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"positive"}, - "force": float64(0.5), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveTo-uuid1", - "beacon": "weaviate://localhost/moveTo-uuid2", - }, - map[string]interface{}{ - "id": "moveTo-uuid1", - "beacon": "weaviate://localhost/moveTo-uuid2", - }, - }, - }, - "moveAwayFrom": map[string]interface{}{ - "concepts": []interface{}{"epic"}, - "force": float64(0.25), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveAwayFrom-uuid1", - "beacon": "weaviate://localhost/moveAwayFrom-uuid1", - }, - map[string]interface{}{ - "id": "moveAwayFrom-uuid2", - "beacon": "weaviate://localhost/moveAwayFrom-uuid2", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveAwayFrom-uuid3", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveAwayFrom-uuid4", - }, - }, - }, - }, - }, - &nearText.NearTextParams{ - Values: []string{"transformed text", "c1", "c2", "c3", "transformed text"}, - Certainty: 0.89, - Limit: 500, - Network: false, - Autocorrect: true, - MoveTo: nearText.ExploreMove{ - Values: []string{"positive"}, - Force: 0.5, - Objects: []nearText.ObjectMove{ - {ID: "moveTo-uuid1", Beacon: "weaviate://localhost/moveTo-uuid2"}, - {ID: "moveTo-uuid1", Beacon: "weaviate://localhost/moveTo-uuid2"}, - }, - }, - MoveAwayFrom: nearText.ExploreMove{ - Values: []string{"epic"}, - Force: 0.25, - Objects: []nearText.ObjectMove{ - {ID: "moveAwayFrom-uuid1", Beacon: "weaviate://localhost/moveAwayFrom-uuid1"}, - {ID: "moveAwayFrom-uuid2", Beacon: "weaviate://localhost/moveAwayFrom-uuid2"}, - {Beacon: "weaviate://localhost/moveAwayFrom-uuid3"}, - {Beacon: "weaviate://localhost/moveAwayFrom-uuid4"}, - }, - }, - }, - }, - } - testsWithAutocorrect = append(testsWithAutocorrect, tests...) - - t.Run("should extract values", func(t *testing.T) { - provider := New(nil) - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := provider.extractNearTextFn(tt.args.source); !reflect.DeepEqual(got, tt.want) { - t.Errorf("extractNearTextFn() = %v, want %v", got, tt.want) - } - }) - } - }) - t.Run("should extract values with transformer", func(t *testing.T) { - provider := New(&fakeTransformer{}) - for _, tt := range testsWithAutocorrect { - t.Run(tt.name, func(t *testing.T) { - if got := provider.extractNearTextFn(tt.args.source); !reflect.DeepEqual(got, tt.want) { - t.Errorf("extractNearTextFn() = %v, want %v", got, tt.want) - } - }) - } - }) -} diff --git a/modules/multi2vec-bind/neartext/graphql_provider.go b/modules/multi2vec-bind/neartext/graphql_provider.go deleted file mode 100644 index 5b6d7c335a957d0deb51fd01d0a35638f07ff595..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/neartext/graphql_provider.go +++ /dev/null @@ -1,40 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package neartext - -import ( - "github.com/weaviate/weaviate/entities/modulecapabilities" -) - -type GraphQLArgumentsProvider struct { - nearTextTransformer modulecapabilities.TextTransform -} - -func New(nearTextTransformer modulecapabilities.TextTransform) *GraphQLArgumentsProvider { - return &GraphQLArgumentsProvider{nearTextTransformer} -} - -func (g *GraphQLArgumentsProvider) Arguments() map[string]modulecapabilities.GraphQLArgument { - arguments := map[string]modulecapabilities.GraphQLArgument{} - arguments["nearText"] = g.getNearText() - return arguments -} - -func (g *GraphQLArgumentsProvider) getNearText() modulecapabilities.GraphQLArgument { - return modulecapabilities.GraphQLArgument{ - GetArgumentsFunction: g.getNearTextArgumentFn, - AggregateArgumentsFunction: g.aggregateNearTextArgumentFn, - ExploreArgumentsFunction: g.exploreNearTextArgumentFn, - ExtractFunction: g.extractNearTextFn, - ValidateFunction: g.validateNearTextFn, - } -} diff --git a/modules/multi2vec-bind/neartext/grapqhl_extract.go b/modules/multi2vec-bind/neartext/grapqhl_extract.go deleted file mode 100644 index c308fe87734ef0d215b5babf8c684f9d9d3be99b..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/neartext/grapqhl_extract.go +++ /dev/null @@ -1,115 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package neartext - -import "github.com/weaviate/weaviate/usecases/modulecomponents/nearText" - -// ExtractNearText arguments, such as "concepts", "moveTo", "moveAwayFrom", -// "limit", etc. -func (g *GraphQLArgumentsProvider) extractNearTextFn(source map[string]interface{}) interface{} { - var args nearText.NearTextParams - - // keywords is a required argument, so we don't need to check for its existing - keywords := source["concepts"].([]interface{}) - args.Values = make([]string, len(keywords)) - for i, value := range keywords { - args.Values[i] = value.(string) - } - - // autocorrect is an optional arg, so it could be nil - autocorrect, ok := source["autocorrect"] - if ok { - args.Autocorrect = autocorrect.(bool) - } - - // if there's text transformer present and autocorrect set to true - // perform text transformation operation - if args.Autocorrect && g.nearTextTransformer != nil { - if transformedValues, err := g.nearTextTransformer.Transform(args.Values); err == nil { - args.Values = transformedValues - } - } - - // limit is an optional arg, so it could be nil - limit, ok := source["limit"] - if ok { - // the type is fixed through gql config, no need to catch incorrect type - // assumption - args.Limit = limit.(int) - } - - certainty, ok := source["certainty"] - if ok { - args.Certainty = certainty.(float64) - } - - distance, ok := source["distance"] - if ok { - args.Distance = distance.(float64) - args.WithDistance = true - } - - // moveTo is an optional arg, so it could be nil - moveTo, ok := source["moveTo"] - if ok { - args.MoveTo = extractMovement(moveTo) - } - - // network is an optional arg, so it could be nil - network, ok := source["network"] - if ok { - args.Network = network.(bool) - } - - // moveAwayFrom is an optional arg, so it could be nil - moveAwayFrom, ok := source["moveAwayFrom"] - if ok { - args.MoveAwayFrom = extractMovement(moveAwayFrom) - } - - return &args -} - -func extractMovement(input interface{}) nearText.ExploreMove { - // the type is fixed through gql config, no need to catch incorrect type - // assumption, all fields are required so we don't need to check for their - // presence - moveToMap := input.(map[string]interface{}) - res := nearText.ExploreMove{} - res.Force = float32(moveToMap["force"].(float64)) - - keywords, ok := moveToMap["concepts"].([]interface{}) - if ok { - res.Values = make([]string, len(keywords)) - for i, value := range keywords { - res.Values[i] = value.(string) - } - } - - objects, ok := moveToMap["objects"].([]interface{}) - if ok { - res.Objects = make([]nearText.ObjectMove, len(objects)) - for i, value := range objects { - v, ok := value.(map[string]interface{}) - if ok { - if v["id"] != nil { - res.Objects[i].ID = v["id"].(string) - } - if v["beacon"] != nil { - res.Objects[i].Beacon = v["beacon"].(string) - } - } - } - } - - return res -} diff --git a/modules/multi2vec-bind/neartext/param.go b/modules/multi2vec-bind/neartext/param.go deleted file mode 100644 index 3e9d851d06f3dabe7d264949ec3c8e3fb62a483e..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/neartext/param.go +++ /dev/null @@ -1,25 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package neartext - -import ( - "github.com/pkg/errors" - "github.com/weaviate/weaviate/usecases/modulecomponents/nearText" -) - -func (g *GraphQLArgumentsProvider) validateNearTextFn(param interface{}) error { - nearTextParams, ok := param.(*nearText.NearTextParams) - if !ok { - return errors.New("'nearText' invalid parameter") - } - return nearTextParams.Validate() -} diff --git a/modules/multi2vec-bind/neartext/searcher.go b/modules/multi2vec-bind/neartext/searcher.go deleted file mode 100644 index 5f1c6b6589500a7b09881f52d2021a63691a0edf..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/neartext/searcher.go +++ /dev/null @@ -1,145 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package neartext - -import ( - "context" - - "github.com/weaviate/weaviate/usecases/modulecomponents/nearText" - - "github.com/go-openapi/strfmt" - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/schema/crossref" - localvectorizer "github.com/weaviate/weaviate/modules/multi2vec-bind/vectorizer" -) - -type Searcher struct { - vectorizer vectorizer -} - -func NewSearcher(vectorizer vectorizer) *Searcher { - return &Searcher{vectorizer} -} - -type vectorizer interface { - Texts(ctx context.Context, input []string, - settings localvectorizer.ClassSettings) ([]float32, error) - MoveTo(source, target []float32, weight float32) ([]float32, error) - MoveAwayFrom(source, target []float32, weight float32) ([]float32, error) - CombineVectors(vectors [][]float32) []float32 -} - -func (s *Searcher) VectorSearches() map[string]modulecapabilities.VectorForParams { - vectorSearches := map[string]modulecapabilities.VectorForParams{} - vectorSearches["nearText"] = s.vectorForNearTextParam - return vectorSearches -} - -func (s *Searcher) vectorForNearTextParam(ctx context.Context, params interface{}, - className string, - findVectorFn modulecapabilities.FindVectorFn, - cfg moduletools.ClassConfig, -) ([]float32, error) { - return s.vectorFromNearTextParam(ctx, params.(*nearText.NearTextParams), className, findVectorFn, cfg) -} - -func (s *Searcher) vectorFromNearTextParam(ctx context.Context, - params *nearText.NearTextParams, - className string, findVectorFn modulecapabilities.FindVectorFn, - cfg moduletools.ClassConfig, -) ([]float32, error) { - // it is safe to call NewClassSettings even knowing that cfg can be nil, it - // is to built to work with all defaults in the case of a nil-config, see - // vectorizer/class_settings_test.go for details. - settings := localvectorizer.NewClassSettings(cfg) - vector, err := s.vectorizer.Texts(ctx, params.Values, settings) - if err != nil { - return nil, errors.Errorf("vectorize keywords: %v", err) - } - - moveTo := params.MoveTo - if moveTo.Force > 0 && (len(moveTo.Values) > 0 || len(moveTo.Objects) > 0) { - moveToVector, err := s.vectorFromValuesAndObjects(ctx, moveTo.Values, - moveTo.Objects, className, findVectorFn, settings, cfg) - if err != nil { - return nil, errors.Errorf("vectorize move to: %v", err) - } - - afterMoveTo, err := s.vectorizer.MoveTo(vector, moveToVector, moveTo.Force) - if err != nil { - return nil, err - } - vector = afterMoveTo - } - - moveAway := params.MoveAwayFrom - if moveAway.Force > 0 && (len(moveAway.Values) > 0 || len(moveAway.Objects) > 0) { - moveAwayVector, err := s.vectorFromValuesAndObjects(ctx, moveAway.Values, - moveAway.Objects, className, findVectorFn, settings, cfg) - if err != nil { - return nil, errors.Errorf("vectorize move away from: %v", err) - } - - afterMoveFrom, err := s.vectorizer.MoveAwayFrom(vector, moveAwayVector, moveAway.Force) - if err != nil { - return nil, err - } - vector = afterMoveFrom - } - - return vector, nil -} - -func (s *Searcher) vectorFromValuesAndObjects(ctx context.Context, - values []string, objects []nearText.ObjectMove, - className string, - findVectorFn modulecapabilities.FindVectorFn, - settings localvectorizer.ClassSettings, - cfg moduletools.ClassConfig, -) ([]float32, error) { - var objectVectors [][]float32 - - if len(values) > 0 { - moveToVector, err := s.vectorizer.Texts(ctx, values, settings) - if err != nil { - return nil, errors.Errorf("vectorize move to: %v", err) - } - objectVectors = append(objectVectors, moveToVector) - } - - if len(objects) > 0 { - var id strfmt.UUID - for _, obj := range objects { - if len(obj.ID) > 0 { - id = strfmt.UUID(obj.ID) - } - if len(obj.Beacon) > 0 { - ref, err := crossref.Parse(obj.Beacon) - if err != nil { - return nil, err - } - id = ref.TargetID - } - - vector, err := findVectorFn(ctx, className, id, cfg.Tenant()) - if err != nil { - return nil, err - } - - objectVectors = append(objectVectors, vector) - } - } - - return s.vectorizer.CombineVectors(objectVectors), nil -} diff --git a/modules/multi2vec-bind/nearthermal/graphql_argument.go b/modules/multi2vec-bind/nearthermal/graphql_argument.go deleted file mode 100644 index 7b4698b13a8266b380d72dfbc24500be50f7b57a..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/nearthermal/graphql_argument.go +++ /dev/null @@ -1,61 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package nearthermal - -import ( - "fmt" - - "github.com/tailor-inc/graphql" - "github.com/weaviate/weaviate/adapters/handlers/graphql/descriptions" -) - -func getNearThermalArgumentFn(classname string) *graphql.ArgumentConfig { - return nearThermalArgument("GetObjects", classname) -} - -func exploreNearThermalArgumentFn() *graphql.ArgumentConfig { - return nearThermalArgument("Explore", "") -} - -func aggregateNearThermalArgumentFn(classname string) *graphql.ArgumentConfig { - return nearThermalArgument("Aggregate", classname) -} - -func nearThermalArgument(prefix, className string) *graphql.ArgumentConfig { - prefixName := fmt.Sprintf("Multi2VecBind%s%s", prefix, className) - return &graphql.ArgumentConfig{ - Type: graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sNearThermalInpObj", prefixName), - Fields: nearThermalFields(prefixName), - Description: descriptions.GetWhereInpObj, - }, - ), - } -} - -func nearThermalFields(prefix string) graphql.InputObjectConfigFieldMap { - return graphql.InputObjectConfigFieldMap{ - "thermal": &graphql.InputObjectFieldConfig{ - Description: "Base64 encoded thermal data", - Type: graphql.NewNonNull(graphql.String), - }, - "certainty": &graphql.InputObjectFieldConfig{ - Description: descriptions.Certainty, - Type: graphql.Float, - }, - "distance": &graphql.InputObjectFieldConfig{ - Description: descriptions.Distance, - Type: graphql.Float, - }, - } -} diff --git a/modules/multi2vec-bind/nearthermal/graphql_argument_test.go b/modules/multi2vec-bind/nearthermal/graphql_argument_test.go deleted file mode 100644 index ff6b2a1c6736fadd4abbe152ccc3321631f65b6e..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/nearthermal/graphql_argument_test.go +++ /dev/null @@ -1,50 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package nearthermal - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/tailor-inc/graphql" -) - -func TestNearThermalGraphQLArgument(t *testing.T) { - t.Run("should generate nearThermal argument properly", func(t *testing.T) { - // given - prefix := "Prefix" - classname := "Class" - // when - nearThermal := nearThermalArgument(prefix, classname) - - // then - // the built graphQL field needs to support this structure: - // nearThermal: { - // thermal: "base64;encoded,thermal_image", - // distance: 0.9 - // } - assert.NotNil(t, nearThermal) - assert.Equal(t, "Multi2VecBindPrefixClassNearThermalInpObj", nearThermal.Type.Name()) - answerFields, ok := nearThermal.Type.(*graphql.InputObject) - assert.True(t, ok) - assert.NotNil(t, answerFields) - assert.Equal(t, 3, len(answerFields.Fields())) - fields := answerFields.Fields() - thermal := fields["thermal"] - thermalNonNull, thermalNonNullOK := thermal.Type.(*graphql.NonNull) - assert.True(t, thermalNonNullOK) - assert.Equal(t, "String", thermalNonNull.OfType.Name()) - assert.NotNil(t, thermal) - assert.NotNil(t, fields["certainty"]) - assert.NotNil(t, fields["distance"]) - }) -} diff --git a/modules/multi2vec-bind/nearthermal/graphql_extract.go b/modules/multi2vec-bind/nearthermal/graphql_extract.go deleted file mode 100644 index 47039bc7939269622d5b8241050593635e98ca76..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/nearthermal/graphql_extract.go +++ /dev/null @@ -1,35 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package nearthermal - -// extractNearThermalFn arguments, such as "thermal" and "certainty" -func extractNearThermalFn(source map[string]interface{}) interface{} { - var args NearThermalParams - - thermal, ok := source["thermal"].(string) - if ok { - args.Thermal = thermal - } - - certainty, ok := source["certainty"] - if ok { - args.Certainty = certainty.(float64) - } - - distance, ok := source["distance"] - if ok { - args.Distance = distance.(float64) - args.WithDistance = true - } - - return &args -} diff --git a/modules/multi2vec-bind/nearthermal/graphql_extract_test.go b/modules/multi2vec-bind/nearthermal/graphql_extract_test.go deleted file mode 100644 index 4b7ac5c13c0224cae256b6555f230a65639fd86f..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/nearthermal/graphql_extract_test.go +++ /dev/null @@ -1,74 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package nearthermal - -import ( - "reflect" - "testing" -) - -func Test_extractNearThermalFn(t *testing.T) { - type args struct { - source map[string]interface{} - } - tests := []struct { - name string - args args - want interface{} - }{ - { - name: "should extract properly with distance and thermal params set", - args: args{ - source: map[string]interface{}{ - "thermal": "base64;encoded", - "distance": float64(0.9), - }, - }, - want: &NearThermalParams{ - Thermal: "base64;encoded", - Distance: 0.9, - WithDistance: true, - }, - }, - { - name: "should extract properly with certainty and thermal params set", - args: args{ - source: map[string]interface{}{ - "thermal": "base64;encoded", - "certainty": float64(0.9), - }, - }, - want: &NearThermalParams{ - Thermal: "base64;encoded", - Certainty: 0.9, - }, - }, - { - name: "should extract properly with only thermal set", - args: args{ - source: map[string]interface{}{ - "thermal": "base64;encoded", - }, - }, - want: &NearThermalParams{ - Thermal: "base64;encoded", - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := extractNearThermalFn(tt.args.source); !reflect.DeepEqual(got, tt.want) { - t.Errorf("extractNearThermalFn() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/modules/multi2vec-bind/nearthermal/graphql_provider.go b/modules/multi2vec-bind/nearthermal/graphql_provider.go deleted file mode 100644 index 78c4e9e97b8884451ebc295625e1e38298ecbe97..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/nearthermal/graphql_provider.go +++ /dev/null @@ -1,38 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package nearthermal - -import ( - "github.com/weaviate/weaviate/entities/modulecapabilities" -) - -type GraphQLArgumentsProvider struct{} - -func New() *GraphQLArgumentsProvider { - return &GraphQLArgumentsProvider{} -} - -func (g *GraphQLArgumentsProvider) Arguments() map[string]modulecapabilities.GraphQLArgument { - arguments := map[string]modulecapabilities.GraphQLArgument{} - arguments["nearThermal"] = g.getNearThermal() - return arguments -} - -func (g *GraphQLArgumentsProvider) getNearThermal() modulecapabilities.GraphQLArgument { - return modulecapabilities.GraphQLArgument{ - GetArgumentsFunction: getNearThermalArgumentFn, - AggregateArgumentsFunction: aggregateNearThermalArgumentFn, - ExploreArgumentsFunction: exploreNearThermalArgumentFn, - ExtractFunction: extractNearThermalFn, - ValidateFunction: validateNearThermalFn, - } -} diff --git a/modules/multi2vec-bind/nearthermal/param.go b/modules/multi2vec-bind/nearthermal/param.go deleted file mode 100644 index df33d32c43a17bae4662361b7c9ac5edf3bbb561..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/nearthermal/param.go +++ /dev/null @@ -1,53 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package nearthermal - -import ( - "errors" -) - -type NearThermalParams struct { - Thermal string - Certainty float64 - Distance float64 - WithDistance bool -} - -func (n NearThermalParams) GetCertainty() float64 { - return n.Certainty -} - -func (n NearThermalParams) GetDistance() float64 { - return n.Distance -} - -func (n NearThermalParams) SimilarityMetricProvided() bool { - return n.Certainty != 0 || n.WithDistance -} - -func validateNearThermalFn(param interface{}) error { - nearThermal, ok := param.(*NearThermalParams) - if !ok { - return errors.New("'nearThermal' invalid parameter") - } - - if len(nearThermal.Thermal) == 0 { - return errors.New("'nearThermal.thermal' needs to be defined") - } - - if nearThermal.Certainty != 0 && nearThermal.WithDistance { - return errors.New( - "nearThermal cannot provide both distance and certainty") - } - - return nil -} diff --git a/modules/multi2vec-bind/nearthermal/param_test.go b/modules/multi2vec-bind/nearthermal/param_test.go deleted file mode 100644 index 3d6d262e562fbc0886060ace35a89a447a493e14..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/nearthermal/param_test.go +++ /dev/null @@ -1,78 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package nearthermal - -import "testing" - -func Test_validateNearThermalFn(t *testing.T) { - type args struct { - param interface{} - } - tests := []struct { - name string - args args - wantErr bool - }{ - { - name: "should pass with proper values", - args: args{ - param: &NearThermalParams{ - Thermal: "base64;enncoded", - }, - }, - }, - { - name: "should not pass with empty thermal", - args: args{ - param: &NearThermalParams{ - Thermal: "", - }, - }, - wantErr: true, - }, - { - name: "should not pass with nil thermal", - args: args{ - param: &NearThermalParams{}, - }, - wantErr: true, - }, - { - name: "should not pass with struct param, not a pointer to struct", - args: args{ - param: NearThermalParams{ - Thermal: "thermal", - }, - }, - wantErr: true, - }, - { - name: "should not pass with certainty and distance", - args: args{ - param: NearThermalParams{ - Thermal: "thermal", - Distance: 0.9, - WithDistance: true, - Certainty: 0.1, - }, - }, - wantErr: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := validateNearThermalFn(tt.args.param); (err != nil) != tt.wantErr { - t.Errorf("validateNearThermalFn() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} diff --git a/modules/multi2vec-bind/nearthermal/searcher.go b/modules/multi2vec-bind/nearthermal/searcher.go deleted file mode 100644 index 0725e9d49e1a6bcc3b49cf96cae6b1e116a40036..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/nearthermal/searcher.go +++ /dev/null @@ -1,59 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package nearthermal - -import ( - "context" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" -) - -type Searcher struct { - vectorizer bindVectorizer -} - -func NewSearcher(vectorizer bindVectorizer) *Searcher { - return &Searcher{vectorizer} -} - -type bindVectorizer interface { - VectorizeThermal(ctx context.Context, thermal string) ([]float32, error) -} - -func (s *Searcher) VectorSearches() map[string]modulecapabilities.VectorForParams { - vectorSearches := map[string]modulecapabilities.VectorForParams{} - vectorSearches["nearThermal"] = s.vectorForNearThermalParam - return vectorSearches -} - -func (s *Searcher) vectorForNearThermalParam(ctx context.Context, params interface{}, - className string, - findVectorFn modulecapabilities.FindVectorFn, - cfg moduletools.ClassConfig, -) ([]float32, error) { - return s.vectorFromNearThermalParam(ctx, params.(*NearThermalParams), className, findVectorFn, cfg) -} - -func (s *Searcher) vectorFromNearThermalParam(ctx context.Context, - params *NearThermalParams, className string, findVectorFn modulecapabilities.FindVectorFn, - cfg moduletools.ClassConfig, -) ([]float32, error) { - // find vector for given search query - vector, err := s.vectorizer.VectorizeThermal(ctx, params.Thermal) - if err != nil { - return nil, errors.Errorf("vectorize thermal: %v", err) - } - - return vector, nil -} diff --git a/modules/multi2vec-bind/vectorizer/class_settings.go b/modules/multi2vec-bind/vectorizer/class_settings.go deleted file mode 100644 index 35a1cacdebbe64b9efc91cf462958847c048496e..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/vectorizer/class_settings.go +++ /dev/null @@ -1,288 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "encoding/json" - "fmt" - "strconv" - - "github.com/pkg/errors" - - "github.com/weaviate/weaviate/entities/moduletools" -) - -type classSettings struct { - cfg moduletools.ClassConfig -} - -func NewClassSettings(cfg moduletools.ClassConfig) *classSettings { - return &classSettings{cfg: cfg} -} - -func (ic *classSettings) ImageField(property string) bool { - return ic.field("imageFields", property) -} - -func (ic *classSettings) ImageFieldsWeights() ([]float32, error) { - return ic.getFieldsWeights("image") -} - -func (ic *classSettings) TextField(property string) bool { - return ic.field("textFields", property) -} - -func (ic *classSettings) TextFieldsWeights() ([]float32, error) { - return ic.getFieldsWeights("text") -} - -func (ic *classSettings) AudioField(property string) bool { - return ic.field("audioFields", property) -} - -func (ic *classSettings) AudioFieldsWeights() ([]float32, error) { - return ic.getFieldsWeights("audio") -} - -func (ic *classSettings) VideoField(property string) bool { - return ic.field("videoFields", property) -} - -func (ic *classSettings) VideoFieldsWeights() ([]float32, error) { - return ic.getFieldsWeights("video") -} - -func (ic *classSettings) IMUField(property string) bool { - return ic.field("imuFields", property) -} - -func (ic *classSettings) IMUFieldsWeights() ([]float32, error) { - return ic.getFieldsWeights("imu") -} - -func (ic *classSettings) ThermalField(property string) bool { - return ic.field("thermalFields", property) -} - -func (ic *classSettings) ThermalFieldsWeights() ([]float32, error) { - return ic.getFieldsWeights("thermal") -} - -func (ic *classSettings) DepthField(property string) bool { - return ic.field("depthFields", property) -} - -func (ic *classSettings) DepthFieldsWeights() ([]float32, error) { - return ic.getFieldsWeights("depth") -} - -func (ic *classSettings) field(name, property string) bool { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return false - } - - fields, ok := ic.cfg.Class()[name] - if !ok { - return false - } - - fieldsArray, ok := fields.([]interface{}) - if !ok { - return false - } - - fieldNames := make([]string, len(fieldsArray)) - for i, value := range fieldsArray { - fieldNames[i] = value.(string) - } - - for i := range fieldNames { - if fieldNames[i] == property { - return true - } - } - - return false -} - -func (ic *classSettings) Validate() error { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return errors.New("empty config") - } - - imageFields, imageFieldsOk := ic.cfg.Class()["imageFields"] - textFields, textFieldsOk := ic.cfg.Class()["textFields"] - audioFields, audioFieldsOk := ic.cfg.Class()["audioFields"] - videoFields, videoFieldsOk := ic.cfg.Class()["videoFields"] - imuFields, imuFieldsOk := ic.cfg.Class()["imuFields"] - thermalFields, thermalFieldsOk := ic.cfg.Class()["thermalFields"] - depthFields, depthFieldsOk := ic.cfg.Class()["depthFields"] - - if !imageFieldsOk && !textFieldsOk && !audioFieldsOk && !videoFieldsOk && - !imuFieldsOk && !thermalFieldsOk && !depthFieldsOk { - return errors.New("textFields or imageFields or audioFields or videoFields " + - "or imuFields or thermalFields or depthFields setting needs to be present") - } - - if imageFieldsOk { - if err := ic.validateWeightFieldCount("image", imageFields); err != nil { - return err - } - } - if textFieldsOk { - if err := ic.validateWeightFieldCount("text", textFields); err != nil { - return err - } - } - if audioFieldsOk { - if err := ic.validateWeightFieldCount("audio", audioFields); err != nil { - return err - } - } - if videoFieldsOk { - if err := ic.validateWeightFieldCount("video", videoFields); err != nil { - return err - } - } - if imuFieldsOk { - if err := ic.validateWeightFieldCount("imu", imuFields); err != nil { - return err - } - } - if thermalFieldsOk { - if err := ic.validateWeightFieldCount("thermal", thermalFields); err != nil { - return err - } - } - if depthFieldsOk { - if err := ic.validateWeightFieldCount("depth", depthFields); err != nil { - return err - } - } - - return nil -} - -func (ic *classSettings) validateWeightFieldCount(name string, fields interface{}) error { - imageFieldsCount, err := ic.validateFields(name, fields) - if err != nil { - return err - } - err = ic.validateWeights(name, imageFieldsCount) - if err != nil { - return err - } - return nil -} - -func (ic *classSettings) validateFields(name string, fields interface{}) (int, error) { - fieldsArray, ok := fields.([]interface{}) - if !ok { - return 0, errors.Errorf("%sFields must be an array", name) - } - - if len(fieldsArray) == 0 { - return 0, errors.Errorf("must contain at least one %s field name in %sFields", name, name) - } - - for _, value := range fieldsArray { - v, ok := value.(string) - if !ok { - return 0, errors.Errorf("%sField must be a string", name) - } - if len(v) == 0 { - return 0, errors.Errorf("%sField values cannot be empty", name) - } - } - - return len(fieldsArray), nil -} - -func (ic *classSettings) validateWeights(name string, count int) error { - weights, ok := ic.getWeights(name) - if ok { - if len(weights) != count { - return errors.Errorf("weights.%sFields does not equal number of %sFields", name, name) - } - _, err := ic.getWeightsArray(weights) - if err != nil { - return err - } - } - - return nil -} - -func (ic *classSettings) getWeights(name string) ([]interface{}, bool) { - weights, ok := ic.cfg.Class()["weights"] - if ok { - weightsObject, ok := weights.(map[string]interface{}) - if ok { - fieldWeights, ok := weightsObject[fmt.Sprintf("%sFields", name)] - if ok { - fieldWeightsArray, ok := fieldWeights.([]interface{}) - if ok { - return fieldWeightsArray, ok - } - } - } - } - - return nil, false -} - -func (ic *classSettings) getWeightsArray(weights []interface{}) ([]float32, error) { - weightsArray := make([]float32, len(weights)) - for i := range weights { - weight, err := ic.getNumber(weights[i]) - if err != nil { - return nil, err - } - weightsArray[i] = weight - } - return weightsArray, nil -} - -func (ic *classSettings) getFieldsWeights(name string) ([]float32, error) { - weights, ok := ic.getWeights(name) - if ok { - return ic.getWeightsArray(weights) - } - return nil, nil -} - -func (ic *classSettings) getNumber(in interface{}) (float32, error) { - switch i := in.(type) { - case float64: - return float32(i), nil - case float32: - return i, nil - case int: - return float32(i), nil - case string: - num, err := strconv.ParseFloat(i, 64) - if err != nil { - return 0, err - } - return float32(num), err - case json.Number: - num, err := i.Float64() - if err != nil { - return 0, err - } - return float32(num), err - default: - return 0.0, errors.Errorf("Unrecognized weight entry type: %T", i) - } -} diff --git a/modules/multi2vec-bind/vectorizer/class_settings_test.go b/modules/multi2vec-bind/vectorizer/class_settings_test.go deleted file mode 100644 index b37ac64b2f5004502a67353c63f4ea2ff6b56f26..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/vectorizer/class_settings_test.go +++ /dev/null @@ -1,226 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "encoding/json" - "testing" - - "github.com/weaviate/weaviate/entities/moduletools" -) - -func Test_classSettings_Validate(t *testing.T) { - type fields struct { - cfg moduletools.ClassConfig - } - tests := []struct { - name string - fields fields - wantErr bool - }{ - { - name: "should not pass with empty config", - wantErr: true, - }, - { - name: "should not pass with nil config", - fields: fields{ - cfg: nil, - }, - wantErr: true, - }, - { - name: "should not pass with nil imageFields", - fields: fields{ - cfg: newConfigBuilder().addSetting("imageFields", nil).build(), - }, - wantErr: true, - }, - { - name: "should not pass with fault imageFields value", - fields: fields{ - cfg: newConfigBuilder().addSetting("imageFields", []string{}).build(), - }, - wantErr: true, - }, - { - name: "should not pass with empty imageFields", - fields: fields{ - cfg: newConfigBuilder().addSetting("imageFields", []interface{}{}).build(), - }, - wantErr: true, - }, - { - name: "should not pass with empty string in imageFields", - fields: fields{ - cfg: newConfigBuilder().addSetting("imageFields", []interface{}{""}).build(), - }, - wantErr: true, - }, - { - name: "should not pass with int value in imageFields", - fields: fields{ - cfg: newConfigBuilder().addSetting("imageFields", []interface{}{1.0}).build(), - }, - wantErr: true, - }, - { - name: "should pass with proper value in imageFields", - fields: fields{ - cfg: newConfigBuilder().addSetting("imageFields", []interface{}{"field"}).build(), - }, - }, - { - name: "should pass with proper value in imageFields and textFields", - fields: fields{ - cfg: newConfigBuilder(). - addSetting("imageFields", []interface{}{"imageField"}). - addSetting("textFields", []interface{}{"textField"}). - build(), - }, - }, - { - name: "should pass with proper value in 2 imageFields and 2 textFields", - fields: fields{ - cfg: newConfigBuilder(). - addSetting("textFields", []interface{}{"textField1", "textField2"}). - addSetting("imageFields", []interface{}{"imageField1", "imageField2"}). - build(), - }, - }, - { - name: "should pass with proper value in 2 imageFields and 2 textFields and weights", - fields: fields{ - cfg: newConfigBuilder(). - addSetting("textFields", []interface{}{"textField1", "textField2"}). - addSetting("imageFields", []interface{}{"imageField1", "imageField2"}). - addWeights([]interface{}{1, 2}, []interface{}{1, 2}, nil, nil, nil, nil, nil). - build(), - }, - }, - { - name: "should pass with proper value in 1 imageFields and 2 textFields and weights", - fields: fields{ - cfg: newConfigBuilder(). - addSetting("textFields", []interface{}{"textField1", "textField2"}). - addSetting("imageFields", []interface{}{"imageField1"}). - addWeights([]interface{}{1, 2}, []interface{}{1}, nil, nil, nil, nil, nil). - build(), - }, - }, - { - name: "should pass with proper value in 2 imageFields and 2 textFields and weights", - fields: fields{ - cfg: newConfigBuilder(). - addSetting("textFields", []interface{}{"textField1", "textField2"}). - addSetting("imageFields", []interface{}{"imageField1"}). - addWeights([]interface{}{1, 2}, []interface{}{1}, nil, nil, nil, nil, nil). - build(), - }, - }, - { - name: "should not pass with proper value in 1 imageFields and 2 textFields and weights", - fields: fields{ - cfg: newConfigBuilder(). - addSetting("textFields", []interface{}{"textField1", "textField2"}). - addSetting("imageFields", []interface{}{"imageField1"}). - addWeights([]interface{}{1}, []interface{}{1}, nil, nil, nil, nil, nil). - build(), - }, - wantErr: true, - }, - { - name: "should not pass with not proper weight value in 2 imageFields and 2 textFields and weights", - fields: fields{ - cfg: newConfigBuilder(). - addSetting("textFields", []interface{}{"textField1", "textField2"}). - addSetting("imageFields", []interface{}{"imageField1"}). - addWeights([]interface{}{1, "aaaa"}, []interface{}{1}, nil, nil, nil, nil, nil). - build(), - }, - wantErr: true, - }, - { - name: "should not pass with not proper weight value in 2 imageFields and 2 textFields and weights", - fields: fields{ - cfg: newConfigBuilder(). - addSetting("textFields", []interface{}{"textField1", "textField2"}). - addSetting("imageFields", []interface{}{"imageField1"}). - addWeights([]interface{}{json.Number("1"), json.Number("2")}, []interface{}{json.Number("3")}, nil, nil, nil, nil, nil). - build(), - }, - }, - { - name: "should pass with proper values in all fields", - fields: fields{ - cfg: newConfigBuilder(). - addSetting("textFields", []interface{}{"textField1", "textField2"}). - addSetting("imageFields", []interface{}{"imageField1"}). - addSetting("audioFields", []interface{}{"audioField1"}). - addSetting("videoFields", []interface{}{"videoField1"}). - addSetting("imuFields", []interface{}{"imuField1"}). - addSetting("thermalFields", []interface{}{"thermalField1"}). - addSetting("depthFields", []interface{}{"depthField1", "depthField2"}). - build(), - }, - }, - { - name: "should pass with proper values in all fields and weights", - fields: fields{ - cfg: newConfigBuilder(). - addSetting("textFields", []interface{}{"textField1", "textField2"}). - addSetting("imageFields", []interface{}{"imageField1"}). - addSetting("audioFields", []interface{}{"audioField1"}). - addSetting("videoFields", []interface{}{"videoField1"}). - addSetting("imuFields", []interface{}{"imuField1"}). - addSetting("thermalFields", []interface{}{"thermalField1"}). - addSetting("depthFields", []interface{}{"depthField1", "depthField2"}). - addWeights([]interface{}{1, 2}, []interface{}{1}, []interface{}{1}, []interface{}{1}, []interface{}{1}, []interface{}{1}, []interface{}{1, 2}). - build(), - }, - }, - { - name: "should pass with proper values audio, video, imu, thermal and depth fields and weights", - fields: fields{ - cfg: newConfigBuilder(). - addSetting("audioFields", []interface{}{"audioField1", "audioField2"}). - addSetting("videoFields", []interface{}{"videoField1"}). - addSetting("imuFields", []interface{}{"imuField1"}). - addSetting("thermalFields", []interface{}{"thermalField1"}). - addSetting("depthFields", []interface{}{"depthField1", "depthField2"}). - addWeights(nil, nil, []interface{}{1, 2}, []interface{}{1}, []interface{}{1}, []interface{}{1}, []interface{}{1, 2}). - build(), - }, - }, - { - name: "should not pass with thermal and depth fields and not proper weights", - fields: fields{ - cfg: newConfigBuilder(). - addSetting("thermalFields", []interface{}{"thermalField1"}). - addSetting("depthFields", []interface{}{"depthField1", "depthField2"}). - addWeights(nil, nil, nil, nil, nil, []interface{}{1, 100}, []interface{}{1, 2}). - build(), - }, - wantErr: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ic := &classSettings{ - cfg: tt.fields.cfg, - } - if err := ic.Validate(); (err != nil) != tt.wantErr { - t.Errorf("classSettings.Validate() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} diff --git a/modules/multi2vec-bind/vectorizer/fakes_for_test.go b/modules/multi2vec-bind/vectorizer/fakes_for_test.go deleted file mode 100644 index 3617629f72e063006e285f62f1b311bdda1f9fcf..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/vectorizer/fakes_for_test.go +++ /dev/null @@ -1,100 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - - "github.com/weaviate/weaviate/modules/multi2vec-bind/ent" -) - -type builder struct { - fakeClassConfig *fakeClassConfig -} - -func newConfigBuilder() *builder { - return &builder{ - fakeClassConfig: &fakeClassConfig{config: map[string]interface{}{}}, - } -} - -func (b *builder) addSetting(name string, value interface{}) *builder { - b.fakeClassConfig.config[name] = value - return b -} - -func (b *builder) addWeights(textWeights, imageWeights, audioWeights, - videoWeights, imuWeights, thermalWeights, depthWeights []interface{}, -) *builder { - weightSettings := map[string]interface{}{} - if textWeights != nil { - weightSettings["textFields"] = textWeights - } - if imageWeights != nil { - weightSettings["imageFields"] = imageWeights - } - if audioWeights != nil { - weightSettings["audioFields"] = audioWeights - } - if videoWeights != nil { - weightSettings["videoFields"] = videoWeights - } - if imuWeights != nil { - weightSettings["imuFields"] = imuWeights - } - if thermalWeights != nil { - weightSettings["thermalFields"] = thermalWeights - } - if depthWeights != nil { - weightSettings["depthFields"] = depthWeights - } - if len(weightSettings) > 0 { - b.fakeClassConfig.config["weights"] = weightSettings - } - return b -} - -func (b *builder) build() *fakeClassConfig { - return b.fakeClassConfig -} - -type fakeClassConfig struct { - config map[string]interface{} -} - -func (c fakeClassConfig) Class() map[string]interface{} { - return c.config -} - -func (c fakeClassConfig) ClassByModuleName(moduleName string) map[string]interface{} { - return c.config -} - -func (c fakeClassConfig) Property(propName string) map[string]interface{} { - return c.config -} - -func (c fakeClassConfig) Tenant() string { - return "" -} - -type fakeClient struct{} - -func (c *fakeClient) Vectorize(ctx context.Context, - texts, images, audio, video, imu, thermal, depth []string, -) (*ent.VectorizationResult, error) { - result := &ent.VectorizationResult{ - TextVectors: [][]float32{{1.0, 2.0, 3.0, 4.0, 5.0}}, - ImageVectors: [][]float32{{10.0, 20.0, 30.0, 40.0, 50.0}}, - } - return result, nil -} diff --git a/modules/multi2vec-bind/vectorizer/texts.go b/modules/multi2vec-bind/vectorizer/texts.go deleted file mode 100644 index 1165649083353604b47a3b9cbf3c0fe8e7d0a299..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/vectorizer/texts.go +++ /dev/null @@ -1,31 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - - "github.com/pkg/errors" -) - -func (v *Vectorizer) Texts(ctx context.Context, inputs []string, - settings ClassSettings, -) ([]float32, error) { - res, err := v.client.Vectorize(ctx, inputs, []string{}, []string{}, []string{}, []string{}, []string{}, []string{}) - if err != nil { - return nil, errors.Wrap(err, "remote client vectorize") - } - if len(inputs) != len(res.TextVectors) { - return nil, errors.New("inputs are not equal to vectors returned") - } - return v.CombineVectors(res.TextVectors), nil -} diff --git a/modules/multi2vec-bind/vectorizer/todo_move_out_of_here_movements.go b/modules/multi2vec-bind/vectorizer/todo_move_out_of_here_movements.go deleted file mode 100644 index b7c554ae86e9338d31c79eb28f4a679c5ce95f93..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/vectorizer/todo_move_out_of_here_movements.go +++ /dev/null @@ -1,68 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "fmt" - - libvectorizer "github.com/weaviate/weaviate/usecases/vectorizer" -) - -// CombineVectors combines all of the vector into sum of their parts -func (v *Vectorizer) CombineVectors(vectors [][]float32) []float32 { - return libvectorizer.CombineVectors(vectors) -} - -// MoveTo moves one vector toward another -func (v *Vectorizer) MoveTo(source []float32, target []float32, weight float32, -) ([]float32, error) { - multiplier := float32(0.5) - - if len(source) != len(target) { - return nil, fmt.Errorf("movement: vector lengths don't match: got %d and %d", - len(source), len(target)) - } - - if weight < 0 || weight > 1 { - return nil, fmt.Errorf("movement: force must be between 0 and 1: got %f", - weight) - } - - out := make([]float32, len(source)) - for i, sourceItem := range source { - out[i] = sourceItem*(1-weight*multiplier) + target[i]*(weight*multiplier) - } - - return out, nil -} - -// MoveAwayFrom moves one vector away from another -func (v *Vectorizer) MoveAwayFrom(source []float32, target []float32, weight float32, -) ([]float32, error) { - multiplier := float32(0.5) // so the movement is fair in comparison with moveTo - if len(source) != len(target) { - return nil, fmt.Errorf("movement (moveAwayFrom): vector lengths don't match: "+ - "got %d and %d", len(source), len(target)) - } - - if weight < 0 { - return nil, fmt.Errorf("movement (moveAwayFrom): force must be 0 or positive: "+ - "got %f", weight) - } - - out := make([]float32, len(source)) - for i, sourceItem := range source { - out[i] = sourceItem + weight*multiplier*(sourceItem-target[i]) - } - - return out, nil -} diff --git a/modules/multi2vec-bind/vectorizer/vectorizer.go b/modules/multi2vec-bind/vectorizer/vectorizer.go deleted file mode 100644 index aaee3ac7f2a5ef9295563d846cb2b00ecb804aa7..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/vectorizer/vectorizer.go +++ /dev/null @@ -1,283 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - - "github.com/pkg/errors" - - "github.com/go-openapi/strfmt" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/modules/multi2vec-bind/ent" - libvectorizer "github.com/weaviate/weaviate/usecases/vectorizer" -) - -type Vectorizer struct { - client Client -} - -func New(client Client) *Vectorizer { - return &Vectorizer{ - client: client, - } -} - -type Client interface { - Vectorize(ctx context.Context, - texts, images, audio, video, imu, thermal, depth []string, - ) (*ent.VectorizationResult, error) -} - -type ClassSettings interface { - ImageField(property string) bool - ImageFieldsWeights() ([]float32, error) - TextField(property string) bool - TextFieldsWeights() ([]float32, error) - AudioField(property string) bool - AudioFieldsWeights() ([]float32, error) - VideoField(property string) bool - VideoFieldsWeights() ([]float32, error) - IMUField(property string) bool - IMUFieldsWeights() ([]float32, error) - ThermalField(property string) bool - ThermalFieldsWeights() ([]float32, error) - DepthField(property string) bool - DepthFieldsWeights() ([]float32, error) -} - -func (v *Vectorizer) Object(ctx context.Context, object *models.Object, - objDiff *moduletools.ObjectDiff, settings ClassSettings, -) error { - vec, err := v.object(ctx, object.ID, object.Properties, objDiff, settings) - if err != nil { - return err - } - - object.Vector = vec - return nil -} - -func (v *Vectorizer) VectorizeImage(ctx context.Context, image string) ([]float32, error) { - res, err := v.client.Vectorize(ctx, nil, []string{image}, nil, nil, nil, nil, nil) - if err != nil { - return nil, err - } - return v.getVector(res.ImageVectors) -} - -func (v *Vectorizer) VectorizeAudio(ctx context.Context, audio string) ([]float32, error) { - res, err := v.client.Vectorize(ctx, nil, nil, []string{audio}, nil, nil, nil, nil) - if err != nil { - return nil, err - } - return v.getVector(res.AudioVectors) -} - -func (v *Vectorizer) VectorizeVideo(ctx context.Context, video string) ([]float32, error) { - res, err := v.client.Vectorize(ctx, nil, nil, nil, []string{video}, nil, nil, nil) - if err != nil { - return nil, err - } - return v.getVector(res.VideoVectors) -} - -func (v *Vectorizer) VectorizeIMU(ctx context.Context, imu string) ([]float32, error) { - res, err := v.client.Vectorize(ctx, nil, nil, nil, nil, []string{imu}, nil, nil) - if err != nil { - return nil, err - } - return v.getVector(res.IMUVectors) -} - -func (v *Vectorizer) VectorizeThermal(ctx context.Context, thermal string) ([]float32, error) { - res, err := v.client.Vectorize(ctx, nil, nil, nil, nil, nil, []string{thermal}, nil) - if err != nil { - return nil, err - } - return v.getVector(res.ThermalVectors) -} - -func (v *Vectorizer) VectorizeDepth(ctx context.Context, depth string) ([]float32, error) { - res, err := v.client.Vectorize(ctx, nil, nil, nil, nil, nil, nil, []string{depth}) - if err != nil { - return nil, err - } - return v.getVector(res.DepthVectors) -} - -func (v *Vectorizer) getVector(vectors [][]float32) ([]float32, error) { - if len(vectors) != 1 { - return nil, errors.New("empty vector") - } - return vectors[0], nil -} - -func (v *Vectorizer) object(ctx context.Context, id strfmt.UUID, - schema interface{}, objDiff *moduletools.ObjectDiff, ichek ClassSettings, -) ([]float32, error) { - vectorize := objDiff == nil || objDiff.GetVec() == nil - - // vectorize image and text - var texts, images, audio, video, imu, thermal, depth []string - if schema != nil { - for prop, value := range schema.(map[string]interface{}) { - if ichek.ImageField(prop) { - valueString, ok := value.(string) - if ok { - images = append(images, valueString) - vectorize = vectorize || (objDiff != nil && objDiff.IsChangedProp(prop)) - } - } - if ichek.TextField(prop) { - valueString, ok := value.(string) - if ok { - texts = append(texts, valueString) - vectorize = vectorize || (objDiff != nil && objDiff.IsChangedProp(prop)) - } - valueArr, ok := value.([]interface{}) - if ok { - for _, value := range valueArr { - valueString, ok := value.(string) - if ok { - texts = append(texts, valueString) - vectorize = vectorize || (objDiff != nil && objDiff.IsChangedProp(prop)) - } - } - } - } - if ichek.AudioField(prop) { - valueString, ok := value.(string) - if ok { - audio = append(audio, valueString) - vectorize = vectorize || (objDiff != nil && objDiff.IsChangedProp(prop)) - } - } - if ichek.VideoField(prop) { - valueString, ok := value.(string) - if ok { - video = append(video, valueString) - vectorize = vectorize || (objDiff != nil && objDiff.IsChangedProp(prop)) - } - } - if ichek.IMUField(prop) { - valueString, ok := value.(string) - if ok { - imu = append(imu, valueString) - vectorize = vectorize || (objDiff != nil && objDiff.IsChangedProp(prop)) - } - } - if ichek.ThermalField(prop) { - valueString, ok := value.(string) - if ok { - thermal = append(thermal, valueString) - vectorize = vectorize || (objDiff != nil && objDiff.IsChangedProp(prop)) - } - } - if ichek.DepthField(prop) { - valueString, ok := value.(string) - if ok { - depth = append(depth, valueString) - vectorize = vectorize || (objDiff != nil && objDiff.IsChangedProp(prop)) - } - } - } - } - - // no property was changed, old vector can be used - if !vectorize { - return objDiff.GetVec(), nil - } - - vectors := [][]float32{} - if len(texts) > 0 || len(images) > 0 || len(audio) > 0 || len(video) > 0 || - len(imu) > 0 || len(thermal) > 0 || len(depth) > 0 { - res, err := v.client.Vectorize(ctx, texts, images, audio, video, imu, thermal, depth) - if err != nil { - return nil, err - } - vectors = append(vectors, res.TextVectors...) - vectors = append(vectors, res.ImageVectors...) - vectors = append(vectors, res.AudioVectors...) - vectors = append(vectors, res.VideoVectors...) - vectors = append(vectors, res.IMUVectors...) - vectors = append(vectors, res.ThermalVectors...) - vectors = append(vectors, res.DepthVectors...) - } - weights, err := v.getWeights(ichek) - if err != nil { - return nil, err - } - - return libvectorizer.CombineVectorsWithWeights(vectors, weights), nil -} - -func (v *Vectorizer) getWeights(ichek ClassSettings) ([]float32, error) { - weights := []float32{} - textFieldsWeights, err := ichek.TextFieldsWeights() - if err != nil { - return nil, err - } - imageFieldsWeights, err := ichek.ImageFieldsWeights() - if err != nil { - return nil, err - } - audioFieldsWeights, err := ichek.AudioFieldsWeights() - if err != nil { - return nil, err - } - videoFieldsWeights, err := ichek.VideoFieldsWeights() - if err != nil { - return nil, err - } - imuFieldsWeights, err := ichek.IMUFieldsWeights() - if err != nil { - return nil, err - } - thermalFieldsWeights, err := ichek.ThermalFieldsWeights() - if err != nil { - return nil, err - } - depthFieldsWeights, err := ichek.DepthFieldsWeights() - if err != nil { - return nil, err - } - - weights = append(weights, textFieldsWeights...) - weights = append(weights, imageFieldsWeights...) - weights = append(weights, audioFieldsWeights...) - weights = append(weights, videoFieldsWeights...) - weights = append(weights, imuFieldsWeights...) - weights = append(weights, thermalFieldsWeights...) - weights = append(weights, depthFieldsWeights...) - - normalizedWeights := v.normalizeWeights(weights) - - return normalizedWeights, nil -} - -func (v *Vectorizer) normalizeWeights(weights []float32) []float32 { - if len(weights) > 0 { - var denominator float32 - for i := range weights { - denominator += weights[i] - } - normalizer := 1 / denominator - normalized := make([]float32, len(weights)) - for i := range weights { - normalized[i] = weights[i] * normalizer - } - return normalized - } - return nil -} diff --git a/modules/multi2vec-bind/vectorizer/vectorizer_test.go b/modules/multi2vec-bind/vectorizer/vectorizer_test.go deleted file mode 100644 index 0fbf8eb58906a8c32f22e345519a176d79a8b4f1..0000000000000000000000000000000000000000 --- a/modules/multi2vec-bind/vectorizer/vectorizer_test.go +++ /dev/null @@ -1,221 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/moduletools" -) - -const image = "iVBORw0KGgoAAAANSUhEUgAAAGAAAAA/CAYAAAAfQM0aAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAyRpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuMy1jMDExIDY2LjE0NTY2MSwgMjAxMi8wMi8wNi0xNDo1NjoyNyAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvIiB4bWxuczp4bXBNTT0iaHR0cDovL25zLmFkb2JlLmNvbS94YXAvMS4wL21tLyIgeG1sbnM6c3RSZWY9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9zVHlwZS9SZXNvdXJjZVJlZiMiIHhtcDpDcmVhdG9yVG9vbD0iQWRvYmUgUGhvdG9zaG9wIENTNiAoTWFjaW50b3NoKSIgeG1wTU06SW5zdGFuY2VJRD0ieG1wLmlpZDpCRjQ5NEM3RDI5QTkxMUUyOTc1NENCMzI4N0QwNDNCOSIgeG1wTU06RG9jdW1lbnRJRD0ieG1wLmRpZDpCRjQ5NEM3RTI5QTkxMUUyOTc1NENCMzI4N0QwNDNCOSI+IDx4bXBNTTpEZXJpdmVkRnJvbSBzdFJlZjppbnN0YW5jZUlEPSJ4bXAuaWlkOkJGNDk0QzdCMjlBOTExRTI5NzU0Q0IzMjg3RDA0M0I5IiBzdFJlZjpkb2N1bWVudElEPSJ4bXAuZGlkOkJGNDk0QzdDMjlBOTExRTI5NzU0Q0IzMjg3RDA0M0I5Ii8+IDwvcmRmOkRlc2NyaXB0aW9uPiA8L3JkZjpSREY+IDwveDp4bXBtZXRhPiA8P3hwYWNrZXQgZW5kPSJyIj8+WeGRxAAAB2hJREFUeNrUXFtslUUQ3hJCoQVEKy0k1qQgrRg0vaAJaq1tvJSgaLy8mKDF2IvxBY2Bgm8+iIoxvhB72tTUmKgPigbFKCEtxeKD9hZjAi3GJrYJtqRai7TQB+pMz/zwU/5zzsxe2u4kXwiwZ+bb/Xb/s7v/zEmrra1VTFsFeBRQCtgEuBWwkv5vHPAn4DdAB+B7wBjXcUNDQ8o2dXV1SmDzyhUtLS3tBPyxC9CdrN1ihi/swKuA7YD0BG1uJhQDngdcAnwDeJ86Ole2kLii+J2AFsA+wF9RjRalmEUHaZY8m6RDUYZtn6HPHiRfLm2hck0D7AScAdRH8UokwD2AnwA7UoiUyhaRD/S12dHg+8B1OWA/4BTgqVQCPEJL8haLBNDXEfJt03ziipYH+BJwHFAYJcAWwCeAZQ6CLyPfWyz584nrbCuj74eHwgKsddih2R1ba+jHJ65R1k6PuWNhAd4DZM/BTiWbdhwm5hPXsA0AngY8COgNP4JwSTyu4zE/P18VFhZKP7aNYuouXxFX5Ic8Nc2Ea2D/AfYCNgIORZ0DdusOfnFxcXDwUD09PZKP76alKDUR16KiIlVQUHDl7/39/Uozpg7Xac45YB0dGrQHHw07KVwJpRRbYiKuyCc8+MhXcyXocP2RnvMvJhr8QIBK08EPbGJiQuqq0mX7KD4GIohi4xVPTU0N6/BRamPwu7u7dZb3/RozkW3IB3lZEkGHayeI8FFVVdWaZAIUcD2Wl5fbHHy024XtC6QBkomA/XHIFb8X0Xamp6efASHqt27dGnkVkcNxVlFRoXJycmwOvuLGNmifVATsD/bLZezgKgKE2J+bm3sKHk3XXUWs4Mz87Oxs24OvOLEN26cUAfvFXAkrlKGBCDNXEbAajldXV1+5ijjP+KCrg855x+3nk2uy8SwDdIIIM1cRI6k+0NraqkZGRmzuKAIbFrYf0Q2UaPOA/Wpra3PBNfHhYHq6HbC5qanpGB7ETgPWc0TApTr7eyDolOaj6LRG+/W2Bn94eJg7+DpcowZ+AGb+642NjYfC3wEdXAdI1uK2Du2ksH2HrcHHfggGX4frNVcRMPh7BwcHN8ZiseuuIr4DvKXib29YX2bhmW+wEqYptsREXC2eWXS44oyfuYqYmpra19LSEnkaRgEG6Nj8gGRHESVCRkaG9Kg+IOyTiGtmZqatnZsOV/zMLnjcsF7KH5AIECVCX1+f6u3tlbg4oLmc2VyDy8HgPshg2yzmCo8aFsdAALzpw9dw23REwJkvHPwjSu92UcwVRcAnAd4LaQ6+CVe2AGivAe5WwhcdGp0aoVgmJuIqnBy2uSa18Buxs4AXAJMO401SjLOGfnziyhYg2GrtcNSxSfJ90pI/n7iyBUA7quKv/IYsxhmiZ/ZRy/x94soWAO1nwL0qnhVw2cD/ZfKBvjod9cEnrmwB0DBh9RUVfxHxhYrnUHLtEn2mlHyMOe6HT1wT7oISGSas4ntNzJmsVFczjnMBN1CbfwGD1BYPID8A/lFzbz5xZQsQnmWfExa6ecNVIsBKWuIlgA0qnjG2PLhsou0aZgF3qfil2fg89ssbrhwBNtB+GN/dLUnQ5kbCHYAnAFMAvGpsoY7OlS0krmOhxx7WLHwAeBLwVahN2uIUswgrPB5T8rRv7DxWqDwM+JaCjzue8b5wZe2C7gJ8quKVJqY599vJ1yZHffCJK0uA+wAfAtZYjIO+Gsi3TfOJK0sAfFP/jpKV+HBtKfkutOTPJ64sAVYD3qXgrmwpxVht6McnrmwBMAP4pjlYdRij3tCHT1xZAuDdermOA836gDKKqWNirob1ASZc2eeAl3QH36A+AGP+ohFWxNVSfYAuV9YKyKUTo/bgo2nUB5RQbImJuFqsD9DhyhbAuDgjMI36gFKX7S3XB5S6egSV2Bh8zYyDYjr4SGYi2yzmMIm5YnFGkFOLSQGNjY3X/BtaLBabWQF5XKcO6gOkZT950gAW6wPWuXoEZXEaOqoPyHLcPqkIwvqALFcCZHJmvqP6gEzH7VOKIKgPyHQlwIVUjRzWB1xw3H4+ubIFGE3VyGF9wKjj9ik3D4L6gFFXArCSTlEEzKe3LMIfwvYDNgcf+4P9csSVLUAXt7GD+oBuYfsuW4OvUR/Q7UoA/G2zaRvbOqEI0xRbYiKulusDTrgSYEg6sxKJIKwP6FLyjDYRV4v1ATpc2QKgNZtu6zTqA5o1ObM/h5eDyMvCtrlZObLgNhRv+jAHvkwqQjDzhYPfrvRvF0VcLdQHaHGNxWKrZv0d//hahcqr8Ccww1kRbwPuVMIXHRqd+ptimZiIq0F9gA2urEcQ2jkVf/tz0WG8ixTjnKEfn7iyBQi2WnuULLlV0qE9FrdzPnFlC4CGRQkvqyQ/MqRh6KtO2S948IkrWwC0XwHPAQ4r85z7w+TL1U8Y+8Q14S4oyjA9703AZ4AqFX8RvoTpN8i3/Bi/p+egHz5xZQsQGCasvqGuZhzj76DdpuIZx8FPuOAviWDG8e8qXl0yXxnHPnGdsf8FGAByGwC02iMZswAAAABJRU5ErkJggg==" - -func TestVectorizer(t *testing.T) { - t.Run("should vectorize image", func(t *testing.T) { - // given - client := &fakeClient{} - vectorizer := &Vectorizer{client} - config := newConfigBuilder().addSetting("imageFields", []interface{}{"image"}).build() - settings := NewClassSettings(config) - object := &models.Object{ - ID: "some-uuid", - Properties: map[string]interface{}{ - "image": image, - }, - } - - // when - err := vectorizer.Object(context.Background(), object, nil, settings) - - // then - require.Nil(t, err) - assert.NotNil(t, object.Vector) - }) - - t.Run("should vectorize 2 image fields", func(t *testing.T) { - // given - client := &fakeClient{} - vectorizer := &Vectorizer{client} - config := newConfigBuilder().addSetting("imageFields", []interface{}{"image1", "image2"}).build() - settings := NewClassSettings(config) - object := &models.Object{ - ID: "some-uuid", - Properties: map[string]interface{}{ - "image1": image, - "image2": image, - }, - } - - // when - err := vectorizer.Object(context.Background(), object, nil, settings) - - // then - require.Nil(t, err) - assert.NotNil(t, object.Vector) - }) -} - -func TestVectorizerWithDiff(t *testing.T) { - type testCase struct { - name string - input *models.Object - diff *moduletools.ObjectDiff - expectedVectorize bool - } - - tests := []testCase{ - { - name: "no diff", - input: &models.Object{ - ID: "some-uuid", - Properties: map[string]interface{}{ - "image": image, - "text": "text", - "description": "non-vectorizable", - }, - }, - diff: nil, - expectedVectorize: true, - }, - { - name: "diff all props unchanged", - input: &models.Object{ - ID: "some-uuid", - Properties: map[string]interface{}{ - "image": image, - "text": "text", - "description": "non-vectorizable", - }, - }, - diff: newObjectDiffWithVector(). - WithProp("image", image, image). - WithProp("text", "text", "text"). - WithProp("description", "non-vectorizable", "non-vectorizable"), - expectedVectorize: false, - }, - { - name: "diff one vectorizable prop changed (1)", - input: &models.Object{ - ID: "some-uuid", - Properties: map[string]interface{}{ - "image": image, - "text": "text", - "description": "non-vectorizable", - }, - }, - diff: newObjectDiffWithVector(). - WithProp("image", "", image), - expectedVectorize: true, - }, - { - name: "diff one vectorizable prop changed (2)", - input: &models.Object{ - ID: "some-uuid", - Properties: map[string]interface{}{ - "image": image, - "text": "text", - "description": "non-vectorizable", - }, - }, - diff: newObjectDiffWithVector(). - WithProp("text", "old text", "text"), - expectedVectorize: true, - }, - { - name: "all non-vectorizable props changed", - input: &models.Object{ - ID: "some-uuid", - Properties: map[string]interface{}{ - "image": image, - "text": "text", - "description": "non-vectorizable", - }, - }, - diff: newObjectDiffWithVector(). - WithProp("description", "old non-vectorizable", "non-vectorizable"), - expectedVectorize: false, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - client := &fakeClient{} - vectorizer := &Vectorizer{client} - config := newConfigBuilder(). - addSetting("imageFields", []interface{}{"image"}). - addSetting("textFields", []interface{}{"text"}). - build() - settings := NewClassSettings(config) - - err := vectorizer.Object(context.Background(), test.input, test.diff, settings) - - require.Nil(t, err) - if test.expectedVectorize { - assert.Equal(t, models.C11yVector{5.5, 11, 16.5, 22, 27.5}, test.input.Vector) - } else { - assert.Equal(t, models.C11yVector{0, 0, 0, 0, 0}, test.input.Vector) - } - }) - } -} - -func TestVectorizer_normalizeWeights(t *testing.T) { - tests := []struct { - name string - weights []float32 - }{ - { - name: "normalize example 1", - weights: []float32{200, 100, 0.1}, - }, - { - name: "normalize example 2", - weights: []float32{300.22, 0.7, 17, 54}, - }, - { - name: "normalize example 3", - weights: []float32{300, 0.02, 17}, - }, - { - name: "normalize example 4", - weights: []float32{500, 0.02, 17.4, 180}, - }, - { - name: "normalize example 5", - weights: []float32{500, 0.02, 17.4, 2, 4, 5, .88}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - v := &Vectorizer{} - if got := v.normalizeWeights(tt.weights); !checkNormalization(got) { - t.Errorf("Vectorizer.normalizeWeights() = %v, want %v", got, 1.0) - } - }) - } -} - -func checkNormalization(weights []float32) bool { - var result float32 - for i := range weights { - result += weights[i] - } - return result == 1.0 -} - -func newObjectDiffWithVector() *moduletools.ObjectDiff { - return moduletools.NewObjectDiff([]float32{0, 0, 0, 0, 0}) -} diff --git a/modules/multi2vec-clip/clients/meta.go b/modules/multi2vec-clip/clients/meta.go deleted file mode 100644 index 4096218083c0916296c9195074ee298aaea60f6a..0000000000000000000000000000000000000000 --- a/modules/multi2vec-clip/clients/meta.go +++ /dev/null @@ -1,45 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "context" - "encoding/json" - "io" - "net/http" - - "github.com/pkg/errors" -) - -func (v *vectorizer) MetaInfo() (map[string]interface{}, error) { - req, err := http.NewRequestWithContext(context.Background(), "GET", v.url("/meta"), nil) - if err != nil { - return nil, errors.Wrap(err, "create GET meta request") - } - - res, err := v.httpClient.Do(req) - if err != nil { - return nil, errors.Wrap(err, "send GET meta request") - } - defer res.Body.Close() - - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - return nil, errors.Wrap(err, "read meta response body") - } - - var resBody map[string]interface{} - if err := json.Unmarshal(bodyBytes, &resBody); err != nil { - return nil, errors.Wrap(err, "unmarshal meta response body") - } - return resBody, nil -} diff --git a/modules/multi2vec-clip/clients/meta_test.go b/modules/multi2vec-clip/clients/meta_test.go deleted file mode 100644 index f27978feb8cbfb5266ed97d412cba14e31c76386..0000000000000000000000000000000000000000 --- a/modules/multi2vec-clip/clients/meta_test.go +++ /dev/null @@ -1,135 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func TestGetMeta(t *testing.T) { - t.Run("when the server is providing meta", func(t *testing.T) { - server := httptest.NewServer(&testMetaHandler{t: t}) - defer server.Close() - c := New(server.URL, 0, nullLogger()) - meta, err := c.MetaInfo() - - assert.Nil(t, err) - assert.NotNil(t, meta) - metaModel := meta["model"] - assert.True(t, metaModel != nil) - model, modelOK := metaModel.(map[string]interface{}) - assert.True(t, modelOK) - assert.True(t, model["_name_or_path"] != nil) - assert.True(t, model["architectures"] != nil) - modelID2label, modelID2labelOK := model["id2label"].(map[string]interface{}) - assert.True(t, modelID2labelOK) - assert.True(t, modelID2label["0"] != nil) - }) -} - -type testMetaHandler struct { - t *testing.T - // the test handler will report as not ready before the time has passed - readyTime time.Time -} - -func (f *testMetaHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - assert.Equal(f.t, "/meta", r.URL.String()) - assert.Equal(f.t, http.MethodGet, r.Method) - - if time.Since(f.readyTime) < 0 { - w.WriteHeader(http.StatusServiceUnavailable) - } - - w.Write([]byte(f.metaInfo())) -} - -func (f *testMetaHandler) metaInfo() string { - return `{ - "model": { - "_name_or_path": "clip", - "add_cross_attention": false, - "architectures": [ - "BertForQuestionAnswering" - ], - "attention_probs_dropout_prob": 0.1, - "bad_words_ids": null, - "bos_token_id": null, - "chunk_size_feed_forward": 0, - "decoder_start_token_id": null, - "diversity_penalty": 0.0, - "do_sample": false, - "early_stopping": false, - "encoder_no_repeat_ngram_size": 0, - "eos_token_id": null, - "finetuning_task": null, - "gradient_checkpointing": false, - "hidden_act": "gelu", - "hidden_dropout_prob": 0.1, - "hidden_size": 1024, - "id2label": { - "0": "LABEL_0", - "1": "LABEL_1" - }, - "initializer_range": 0.02, - "intermediate_size": 4096, - "is_decoder": false, - "is_encoder_decoder": false, - "label2id": { - "LABEL_0": 0, - "LABEL_1": 1 - }, - "layer_norm_eps": 1e-12, - "length_penalty": 1.0, - "max_length": 20, - "max_position_embeddings": 512, - "min_length": 0, - "model_type": "bert", - "no_repeat_ngram_size": 0, - "num_attention_heads": 16, - "num_beam_groups": 1, - "num_beams": 1, - "num_hidden_layers": 24, - "num_return_sequences": 1, - "output_attentions": false, - "output_hidden_states": false, - "output_scores": false, - "pad_token_id": 0, - "position_embedding_type": "absolute", - "prefix": null, - "pruned_heads": {}, - "repetition_penalty": 1.0, - "return_dict": true, - "return_dict_in_generate": false, - "sep_token_id": null, - "task_specific_params": null, - "temperature": 1.0, - "tie_encoder_decoder": false, - "tie_word_embeddings": true, - "tokenizer_class": null, - "top_k": 50, - "top_p": 1.0, - "torchscript": false, - "transformers_version": "4.3.2", - "type_vocab_size": 2, - "use_bfloat16": false, - "use_cache": true, - "vocab_size": 30522, - "xla_device": null - } -}` -} diff --git a/modules/multi2vec-clip/clients/startup.go b/modules/multi2vec-clip/clients/startup.go deleted file mode 100644 index c3d920502cde9f3295ef8290e3eac7c341c799bf..0000000000000000000000000000000000000000 --- a/modules/multi2vec-clip/clients/startup.go +++ /dev/null @@ -1,68 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "context" - "net/http" - "time" - - "github.com/pkg/errors" -) - -func (v *vectorizer) WaitForStartup(initCtx context.Context, - interval time.Duration, -) error { - t := time.NewTicker(interval) - defer t.Stop() - expired := initCtx.Done() - var lastErr error - for { - select { - case <-t.C: - lastErr = v.checkReady(initCtx) - if lastErr == nil { - return nil - } - v.logger. - WithField("action", "multi2vec_remote_wait_for_startup"). - WithError(lastErr).Warnf("multi2vec-clip inference service not ready") - case <-expired: - return errors.Wrapf(lastErr, "init context expired before remote was ready") - } - } -} - -func (v *vectorizer) checkReady(initCtx context.Context) error { - // spawn a new context (derived on the overall context) which is used to - // consider an individual request timed out - requestCtx, cancel := context.WithTimeout(initCtx, 500*time.Millisecond) - defer cancel() - - req, err := http.NewRequestWithContext(requestCtx, http.MethodGet, - v.url("/.well-known/ready"), nil) - if err != nil { - return errors.Wrap(err, "create check ready request") - } - - res, err := v.httpClient.Do(req) - if err != nil { - return errors.Wrap(err, "send check ready request") - } - - defer res.Body.Close() - if res.StatusCode > 299 { - return errors.Errorf("not ready: status %d", res.StatusCode) - } - - return nil -} diff --git a/modules/multi2vec-clip/clients/startup_test.go b/modules/multi2vec-clip/clients/startup_test.go deleted file mode 100644 index 7836347d937eaaa18e3c1415dcb4b2aac2d2b15b..0000000000000000000000000000000000000000 --- a/modules/multi2vec-clip/clients/startup_test.go +++ /dev/null @@ -1,98 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "context" - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/sirupsen/logrus" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestWaitForStartup(t *testing.T) { - t.Run("when the server is immediately ready", func(t *testing.T) { - server := httptest.NewServer(&testReadyHandler{t: t}) - defer server.Close() - c := New(server.URL, 0, nullLogger()) - err := c.WaitForStartup(context.Background(), 50*time.Millisecond) - - assert.Nil(t, err) - }) - - t.Run("when the server is down", func(t *testing.T) { - c := New("http://nothing-running-at-this-url", 0, nullLogger()) - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancel() - err := c.WaitForStartup(ctx, 150*time.Millisecond) - - require.NotNil(t, err, nullLogger()) - assert.Contains(t, err.Error(), "expired before remote was ready") - }) - - t.Run("when the server is alive, but not ready", func(t *testing.T) { - server := httptest.NewServer(&testReadyHandler{ - t: t, - readyTime: time.Now().Add(1 * time.Minute), - }) - c := New(server.URL, 0, nullLogger()) - defer server.Close() - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancel() - err := c.WaitForStartup(ctx, 50*time.Millisecond) - - require.NotNil(t, err) - assert.Contains(t, err.Error(), "expired before remote was ready") - }) - - t.Run("when the server is initially not ready, but then becomes ready", - func(t *testing.T) { - server := httptest.NewServer(&testReadyHandler{ - t: t, - readyTime: time.Now().Add(100 * time.Millisecond), - }) - c := New(server.URL, 0, nullLogger()) - defer server.Close() - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancel() - err := c.WaitForStartup(ctx, 50*time.Millisecond) - - require.Nil(t, err) - }) -} - -type testReadyHandler struct { - t *testing.T - // the test handler will report as not ready before the time has passed - readyTime time.Time -} - -func (f *testReadyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - assert.Equal(f.t, "/.well-known/ready", r.URL.String()) - assert.Equal(f.t, http.MethodGet, r.Method) - - if time.Since(f.readyTime) < 0 { - w.WriteHeader(http.StatusServiceUnavailable) - } - - w.WriteHeader(http.StatusNoContent) -} - -func nullLogger() logrus.FieldLogger { - l, _ := test.NewNullLogger() - return l -} diff --git a/modules/multi2vec-clip/clients/vectorizer.go b/modules/multi2vec-clip/clients/vectorizer.go deleted file mode 100644 index 723e13a1f0e8b3caf754dbbb3605d1d2b72bcd45..0000000000000000000000000000000000000000 --- a/modules/multi2vec-clip/clients/vectorizer.go +++ /dev/null @@ -1,101 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/modules/multi2vec-clip/ent" -) - -type vectorizer struct { - origin string - httpClient *http.Client - logger logrus.FieldLogger -} - -func New(origin string, timeout time.Duration, logger logrus.FieldLogger) *vectorizer { - return &vectorizer{ - origin: origin, - httpClient: &http.Client{ - Timeout: timeout, - }, - logger: logger, - } -} - -func (v *vectorizer) Vectorize(ctx context.Context, - texts, images []string, -) (*ent.VectorizationResult, error) { - body, err := json.Marshal(vecRequest{ - Texts: texts, - Images: images, - }) - if err != nil { - return nil, errors.Wrapf(err, "marshal body") - } - - req, err := http.NewRequestWithContext(ctx, "POST", v.url("/vectorize"), - bytes.NewReader(body)) - if err != nil { - return nil, errors.Wrap(err, "create POST request") - } - - res, err := v.httpClient.Do(req) - if err != nil { - return nil, errors.Wrap(err, "send POST request") - } - defer res.Body.Close() - - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - return nil, errors.Wrap(err, "read response body") - } - - var resBody vecResponse - if err := json.Unmarshal(bodyBytes, &resBody); err != nil { - return nil, errors.Wrap(err, "unmarshal response body") - } - - if res.StatusCode > 399 { - return nil, errors.Errorf("fail with status %d: %s", res.StatusCode, - resBody.Error) - } - - return &ent.VectorizationResult{ - TextVectors: resBody.TextVectors, - ImageVectors: resBody.ImageVectors, - }, nil -} - -func (v *vectorizer) url(path string) string { - return fmt.Sprintf("%s%s", v.origin, path) -} - -type vecRequest struct { - Texts []string `json:"texts"` - Images []string `json:"images"` -} - -type vecResponse struct { - TextVectors [][]float32 `json:"textVectors"` - ImageVectors [][]float32 `json:"imageVectors"` - Error string `json:"error"` -} diff --git a/modules/multi2vec-clip/clients/vectorizer_test.go b/modules/multi2vec-clip/clients/vectorizer_test.go deleted file mode 100644 index 76802cbc88f7a2e57df57ec757a784808c157ea4..0000000000000000000000000000000000000000 --- a/modules/multi2vec-clip/clients/vectorizer_test.go +++ /dev/null @@ -1,87 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "context" - "encoding/json" - "net/http" - "net/http/httptest" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/modules/multi2vec-clip/ent" -) - -func TestVectorize(t *testing.T) { - t.Run("when the response is successful", func(t *testing.T) { - server := httptest.NewServer(&testVectorizeHandler{ - t: t, - res: vecResponse{ - TextVectors: [][]float32{ - {0, 1, 2}, - }, - ImageVectors: [][]float32{ - {1, 2, 3}, - }, - }, - }) - defer server.Close() - c := New(server.URL, 0, nullLogger()) - res, err := c.Vectorize(context.Background(), []string{"hello"}, - []string{"image-encoding"}) - - assert.Nil(t, err) - assert.Equal(t, &ent.VectorizationResult{ - TextVectors: [][]float32{ - {0, 1, 2}, - }, - ImageVectors: [][]float32{ - {1, 2, 3}, - }, - }, res) - }) - - t.Run("when the server has a an error", func(t *testing.T) { - server := httptest.NewServer(&testVectorizeHandler{ - t: t, - res: vecResponse{ - Error: "some error from the server", - }, - }) - defer server.Close() - c := New(server.URL, 0, nullLogger()) - _, err := c.Vectorize(context.Background(), []string{"hello"}, - []string{"image-encoding"}) - - require.NotNil(t, err) - assert.Contains(t, err.Error(), "some error from the server") - }) -} - -type testVectorizeHandler struct { - t *testing.T - // the test handler will report as not ready before the time has passed - res vecResponse -} - -func (f *testVectorizeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - assert.Equal(f.t, "/vectorize", r.URL.String()) - assert.Equal(f.t, http.MethodPost, r.Method) - - if f.res.Error != "" { - w.WriteHeader(500) - } - jsonBytes, _ := json.Marshal(f.res) - w.Write(jsonBytes) -} diff --git a/modules/multi2vec-clip/config.go b/modules/multi2vec-clip/config.go deleted file mode 100644 index d80050bb5ca8cad586b947cddb2dd4f03dfa1a67..0000000000000000000000000000000000000000 --- a/modules/multi2vec-clip/config.go +++ /dev/null @@ -1,41 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modclip - -import ( - "context" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/modules/multi2vec-clip/vectorizer" -) - -func (m *ClipModule) ClassConfigDefaults() map[string]interface{} { - return map[string]interface{}{} -} - -func (m *ClipModule) PropertyConfigDefaults( - dt *schema.DataType, -) map[string]interface{} { - return map[string]interface{}{} -} - -func (m *ClipModule) ValidateClass(ctx context.Context, - class *models.Class, cfg moduletools.ClassConfig, -) error { - icheck := vectorizer.NewClassSettings(cfg) - return icheck.Validate() -} - -var _ = modulecapabilities.ClassConfigurator(New()) diff --git a/modules/multi2vec-clip/ent/vectorization_result.go b/modules/multi2vec-clip/ent/vectorization_result.go deleted file mode 100644 index ceb285f107aeb1082570a250c6111bf572ba7369..0000000000000000000000000000000000000000 --- a/modules/multi2vec-clip/ent/vectorization_result.go +++ /dev/null @@ -1,22 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package ent - -type VectorizationResult struct { - TextVectors [][]float32 - ImageVectors [][]float32 - AudioVectors [][]float32 - VideoVectors [][]float32 - IMUVectors [][]float32 - ThermalVectors [][]float32 - DepthVectors [][]float32 -} diff --git a/modules/multi2vec-clip/module.go b/modules/multi2vec-clip/module.go deleted file mode 100644 index 1b29322ab8c1c15ae0f4481d46e1bb554e643b4c..0000000000000000000000000000000000000000 --- a/modules/multi2vec-clip/module.go +++ /dev/null @@ -1,151 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modclip - -import ( - "context" - "net/http" - "os" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/modules/multi2vec-clip/clients" - "github.com/weaviate/weaviate/modules/multi2vec-clip/vectorizer" -) - -func New() *ClipModule { - return &ClipModule{} -} - -type ClipModule struct { - imageVectorizer imageVectorizer - nearImageGraphqlProvider modulecapabilities.GraphQLArguments - nearImageSearcher modulecapabilities.Searcher - textVectorizer textVectorizer - nearTextGraphqlProvider modulecapabilities.GraphQLArguments - nearTextSearcher modulecapabilities.Searcher - nearTextTransformer modulecapabilities.TextTransform - metaClient metaClient -} - -type metaClient interface { - MetaInfo() (map[string]interface{}, error) -} - -type imageVectorizer interface { - Object(ctx context.Context, object *models.Object, objDiff *moduletools.ObjectDiff, - settings vectorizer.ClassSettings) error - VectorizeImage(ctx context.Context, image string) ([]float32, error) -} - -type textVectorizer interface { - Texts(ctx context.Context, input []string, - settings vectorizer.ClassSettings) ([]float32, error) - MoveTo(source, target []float32, weight float32) ([]float32, error) - MoveAwayFrom(source, target []float32, weight float32) ([]float32, error) - CombineVectors(vectors [][]float32) []float32 -} - -func (m *ClipModule) Name() string { - return "multi2vec-clip" -} - -func (m *ClipModule) Type() modulecapabilities.ModuleType { - return modulecapabilities.Multi2Vec -} - -func (m *ClipModule) Init(ctx context.Context, - params moduletools.ModuleInitParams, -) error { - if err := m.initVectorizer(ctx, params.GetConfig().ModuleHttpClientTimeout, params.GetLogger()); err != nil { - return errors.Wrap(err, "init vectorizer") - } - - if err := m.initNearImage(); err != nil { - return errors.Wrap(err, "init near text") - } - - return nil -} - -func (m *ClipModule) InitExtension(modules []modulecapabilities.Module) error { - for _, module := range modules { - if module.Name() == m.Name() { - continue - } - if arg, ok := module.(modulecapabilities.TextTransformers); ok { - if arg != nil && arg.TextTransformers() != nil { - m.nearTextTransformer = arg.TextTransformers()["nearText"] - } - } - } - - if err := m.initNearText(); err != nil { - return errors.Wrap(err, "init near text") - } - - return nil -} - -func (m *ClipModule) initVectorizer(ctx context.Context, timeout time.Duration, - logger logrus.FieldLogger, -) error { - // TODO: proper config management - uri := os.Getenv("CLIP_INFERENCE_API") - if uri == "" { - return errors.Errorf("required variable CLIP_INFERENCE_API is not set") - } - - client := clients.New(uri, timeout, logger) - if err := client.WaitForStartup(ctx, 1*time.Second); err != nil { - return errors.Wrap(err, "init remote vectorizer") - } - - m.imageVectorizer = vectorizer.New(client) - m.textVectorizer = vectorizer.New(client) - m.metaClient = client - - return nil -} - -func (m *ClipModule) RootHandler() http.Handler { - // TODO: remove once this is a capability interface - return nil -} - -func (m *ClipModule) VectorizeObject(ctx context.Context, - obj *models.Object, objDiff *moduletools.ObjectDiff, cfg moduletools.ClassConfig, -) error { - icheck := vectorizer.NewClassSettings(cfg) - return m.imageVectorizer.Object(ctx, obj, objDiff, icheck) -} - -func (m *ClipModule) MetaInfo() (map[string]interface{}, error) { - return m.metaClient.MetaInfo() -} - -func (m *ClipModule) VectorizeInput(ctx context.Context, - input string, cfg moduletools.ClassConfig, -) ([]float32, error) { - return m.textVectorizer.Texts(ctx, []string{input}, vectorizer.NewClassSettings(cfg)) -} - -// verify we implement the modules.Module interface -var ( - _ = modulecapabilities.Module(New()) - _ = modulecapabilities.Vectorizer(New()) - _ = modulecapabilities.InputVectorizer(New()) -) diff --git a/modules/multi2vec-clip/nearImage.go b/modules/multi2vec-clip/nearImage.go deleted file mode 100644 index b944c1c0111cc5fad1c5cb613203b77d6e973574..0000000000000000000000000000000000000000 --- a/modules/multi2vec-clip/nearImage.go +++ /dev/null @@ -1,57 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modclip - -import ( - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/modules/multi2vec-clip/nearImage" - "github.com/weaviate/weaviate/modules/multi2vec-clip/neartext" -) - -func (m *ClipModule) initNearImage() error { - m.nearImageSearcher = nearImage.NewSearcher(m.imageVectorizer) - m.nearImageGraphqlProvider = nearImage.New() - return nil -} - -func (m *ClipModule) initNearText() error { - m.nearTextSearcher = neartext.NewSearcher(m.textVectorizer) - m.nearTextGraphqlProvider = neartext.New(m.nearTextTransformer) - return nil -} - -func (m *ClipModule) Arguments() map[string]modulecapabilities.GraphQLArgument { - arguments := map[string]modulecapabilities.GraphQLArgument{} - for name, arg := range m.nearImageGraphqlProvider.Arguments() { - arguments[name] = arg - } - for name, arg := range m.nearTextGraphqlProvider.Arguments() { - arguments[name] = arg - } - return arguments -} - -func (m *ClipModule) VectorSearches() map[string]modulecapabilities.VectorForParams { - vectorSearches := map[string]modulecapabilities.VectorForParams{} - for name, arg := range m.nearImageSearcher.VectorSearches() { - vectorSearches[name] = arg - } - for name, arg := range m.nearTextSearcher.VectorSearches() { - vectorSearches[name] = arg - } - return vectorSearches -} - -var ( - _ = modulecapabilities.GraphQLArguments(New()) - _ = modulecapabilities.Searcher(New()) -) diff --git a/modules/multi2vec-clip/nearImage/graphql_argument.go b/modules/multi2vec-clip/nearImage/graphql_argument.go deleted file mode 100644 index 2588a9b581410271a1dfb9e587a068c2efb899dd..0000000000000000000000000000000000000000 --- a/modules/multi2vec-clip/nearImage/graphql_argument.go +++ /dev/null @@ -1,61 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package nearImage - -import ( - "fmt" - - "github.com/tailor-inc/graphql" - "github.com/weaviate/weaviate/adapters/handlers/graphql/descriptions" -) - -func getNearImageArgumentFn(classname string) *graphql.ArgumentConfig { - return nearImageArgument("GetObjects", classname) -} - -func exploreNearImageArgumentFn() *graphql.ArgumentConfig { - return nearImageArgument("Explore", "") -} - -func aggregateNearImageArgumentFn(classname string) *graphql.ArgumentConfig { - return nearImageArgument("Aggregate", classname) -} - -func nearImageArgument(prefix, className string) *graphql.ArgumentConfig { - prefixName := fmt.Sprintf("Img2VecImage%s%s", prefix, className) - return &graphql.ArgumentConfig{ - Type: graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sNearImageInpObj", prefixName), - Fields: nearImageFields(prefixName), - Description: descriptions.GetWhereInpObj, - }, - ), - } -} - -func nearImageFields(prefix string) graphql.InputObjectConfigFieldMap { - return graphql.InputObjectConfigFieldMap{ - "image": &graphql.InputObjectFieldConfig{ - Description: "Base64 encoded image", - Type: graphql.NewNonNull(graphql.String), - }, - "certainty": &graphql.InputObjectFieldConfig{ - Description: descriptions.Certainty, - Type: graphql.Float, - }, - "distance": &graphql.InputObjectFieldConfig{ - Description: descriptions.Distance, - Type: graphql.Float, - }, - } -} diff --git a/modules/multi2vec-clip/nearImage/graphql_argument_test.go b/modules/multi2vec-clip/nearImage/graphql_argument_test.go deleted file mode 100644 index 9f585ed9c19905609b41bc6520ed56666f9d873e..0000000000000000000000000000000000000000 --- a/modules/multi2vec-clip/nearImage/graphql_argument_test.go +++ /dev/null @@ -1,50 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package nearImage - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/tailor-inc/graphql" -) - -func TestNearImageGraphQLArgument(t *testing.T) { - t.Run("should generate nearImage argument properly", func(t *testing.T) { - // given - prefix := "Prefix" - classname := "Class" - // when - nearImage := nearImageArgument(prefix, classname) - - // then - // the built graphQL field needs to support this structure: - // nearImage: { - // image: "base64;encoded,image", - // distance: 0.9 - // } - assert.NotNil(t, nearImage) - assert.Equal(t, "Img2VecImagePrefixClassNearImageInpObj", nearImage.Type.Name()) - answerFields, ok := nearImage.Type.(*graphql.InputObject) - assert.True(t, ok) - assert.NotNil(t, answerFields) - assert.Equal(t, 3, len(answerFields.Fields())) - fields := answerFields.Fields() - image := fields["image"] - imageNonNull, imageNonNullOK := image.Type.(*graphql.NonNull) - assert.True(t, imageNonNullOK) - assert.Equal(t, "String", imageNonNull.OfType.Name()) - assert.NotNil(t, image) - assert.NotNil(t, fields["certainty"]) - assert.NotNil(t, fields["distance"]) - }) -} diff --git a/modules/multi2vec-clip/nearImage/graphql_extract.go b/modules/multi2vec-clip/nearImage/graphql_extract.go deleted file mode 100644 index 543b1b66661c89d1d4c30a79aa100ca2b824850c..0000000000000000000000000000000000000000 --- a/modules/multi2vec-clip/nearImage/graphql_extract.go +++ /dev/null @@ -1,37 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package nearImage - -import "github.com/weaviate/weaviate/usecases/modulecomponents/nearImage" - -// extractNearImageFn arguments, such as "image" and "certainty" -func extractNearImageFn(source map[string]interface{}) interface{} { - var args nearImage.NearImageParams - - image, ok := source["image"].(string) - if ok { - args.Image = image - } - - certainty, ok := source["certainty"] - if ok { - args.Certainty = certainty.(float64) - } - - distance, ok := source["distance"] - if ok { - args.Distance = distance.(float64) - args.WithDistance = true - } - - return &args -} diff --git a/modules/multi2vec-clip/nearImage/graphql_extract_test.go b/modules/multi2vec-clip/nearImage/graphql_extract_test.go deleted file mode 100644 index 7fc7782e226badb96cb1ee50ba667ddb50e48907..0000000000000000000000000000000000000000 --- a/modules/multi2vec-clip/nearImage/graphql_extract_test.go +++ /dev/null @@ -1,76 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package nearImage - -import ( - "reflect" - "testing" - - "github.com/weaviate/weaviate/usecases/modulecomponents/nearImage" -) - -func Test_extractNearImageFn(t *testing.T) { - type args struct { - source map[string]interface{} - } - tests := []struct { - name string - args args - want interface{} - }{ - { - name: "should extract properly with distance and image params set", - args: args{ - source: map[string]interface{}{ - "image": "base64;encoded", - "distance": float64(0.9), - }, - }, - want: &nearImage.NearImageParams{ - Image: "base64;encoded", - Distance: 0.9, - WithDistance: true, - }, - }, - { - name: "should extract properly with certainty and image params set", - args: args{ - source: map[string]interface{}{ - "image": "base64;encoded", - "certainty": float64(0.9), - }, - }, - want: &nearImage.NearImageParams{ - Image: "base64;encoded", - Certainty: 0.9, - }, - }, - { - name: "should extract properly with only image set", - args: args{ - source: map[string]interface{}{ - "image": "base64;encoded", - }, - }, - want: &nearImage.NearImageParams{ - Image: "base64;encoded", - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := extractNearImageFn(tt.args.source); !reflect.DeepEqual(got, tt.want) { - t.Errorf("extractNearImageFn() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/modules/multi2vec-clip/nearImage/graphql_provider.go b/modules/multi2vec-clip/nearImage/graphql_provider.go deleted file mode 100644 index 563edc7793b684a795b5f49992ca79c693b88a02..0000000000000000000000000000000000000000 --- a/modules/multi2vec-clip/nearImage/graphql_provider.go +++ /dev/null @@ -1,39 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package nearImage - -import ( - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/usecases/modulecomponents/nearImage" -) - -type GraphQLArgumentsProvider struct{} - -func New() *GraphQLArgumentsProvider { - return &GraphQLArgumentsProvider{} -} - -func (g *GraphQLArgumentsProvider) Arguments() map[string]modulecapabilities.GraphQLArgument { - arguments := map[string]modulecapabilities.GraphQLArgument{} - arguments["nearImage"] = g.getNearImage() - return arguments -} - -func (g *GraphQLArgumentsProvider) getNearImage() modulecapabilities.GraphQLArgument { - return modulecapabilities.GraphQLArgument{ - GetArgumentsFunction: getNearImageArgumentFn, - AggregateArgumentsFunction: aggregateNearImageArgumentFn, - ExploreArgumentsFunction: exploreNearImageArgumentFn, - ExtractFunction: extractNearImageFn, - ValidateFunction: nearImage.ValidateNearImageFn, - } -} diff --git a/modules/multi2vec-clip/nearImage/searcher.go b/modules/multi2vec-clip/nearImage/searcher.go deleted file mode 100644 index f9935aec1c79daefccf862385b70685264fa1ede..0000000000000000000000000000000000000000 --- a/modules/multi2vec-clip/nearImage/searcher.go +++ /dev/null @@ -1,61 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package nearImage - -import ( - "context" - - "github.com/weaviate/weaviate/usecases/modulecomponents/nearImage" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" -) - -type Searcher struct { - vectorizer imgVectorizer -} - -func NewSearcher(vectorizer imgVectorizer) *Searcher { - return &Searcher{vectorizer} -} - -type imgVectorizer interface { - VectorizeImage(ctx context.Context, image string) ([]float32, error) -} - -func (s *Searcher) VectorSearches() map[string]modulecapabilities.VectorForParams { - vectorSearches := map[string]modulecapabilities.VectorForParams{} - vectorSearches["nearImage"] = s.vectorForNearImageParam - return vectorSearches -} - -func (s *Searcher) vectorForNearImageParam(ctx context.Context, params interface{}, - className string, - findVectorFn modulecapabilities.FindVectorFn, - cfg moduletools.ClassConfig, -) ([]float32, error) { - return s.vectorFromNearImageParam(ctx, params.(*nearImage.NearImageParams), className, findVectorFn, cfg) -} - -func (s *Searcher) vectorFromNearImageParam(ctx context.Context, - params *nearImage.NearImageParams, className string, findVectorFn modulecapabilities.FindVectorFn, - cfg moduletools.ClassConfig, -) ([]float32, error) { - // find vector for given search query - vector, err := s.vectorizer.VectorizeImage(ctx, params.Image) - if err != nil { - return nil, errors.Errorf("vectorize image: %v", err) - } - - return vector, nil -} diff --git a/modules/multi2vec-clip/nearText.go b/modules/multi2vec-clip/nearText.go deleted file mode 100644 index fc4c80b580b4fc630f51eb7bb046a2c717283993..0000000000000000000000000000000000000000 --- a/modules/multi2vec-clip/nearText.go +++ /dev/null @@ -1,36 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modclip - -// import ( -// "github.com/weaviate/weaviate/entities/modulecapabilities" -// "github.com/weaviate/weaviate/modules/multi2vec-clip/neartext" -// ) - -// func (m *ClipModule) initNearText() error { -// m.searcher = neartext.NewSearcher(m.vectorizer) -// m.graphqlProvider = neartext.New(m.nearTextTransformer) -// return nil -// } - -// func (m *ClipModule) Arguments() map[string]modulecapabilities.GraphQLArgument { -// return m.graphqlProvider.Arguments() -// } - -// func (m *ClipModule) VectorSearches() map[string]modulecapabilities.VectorForParams { -// return m.searcher.VectorSearches() -// } - -// var ( -// _ = modulecapabilities.GraphQLArguments(New()) -// _ = modulecapabilities.Searcher(New()) -// ) diff --git a/modules/multi2vec-clip/neartext/fakes_for_test.go b/modules/multi2vec-clip/neartext/fakes_for_test.go deleted file mode 100644 index a72048d0b9cb0b05c8c5ca648cf95db841571505..0000000000000000000000000000000000000000 --- a/modules/multi2vec-clip/neartext/fakes_for_test.go +++ /dev/null @@ -1,26 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package neartext - -type fakeTransformer struct{} - -func (t *fakeTransformer) Transform(in []string) ([]string, error) { - result := make([]string, len(in)) - for i, txt := range in { - if txt == "transform this" { - result[i] = "transformed text" - } else { - result[i] = txt - } - } - return result, nil -} diff --git a/modules/multi2vec-clip/neartext/graphql_argument.go b/modules/multi2vec-clip/neartext/graphql_argument.go deleted file mode 100644 index e47539f544c5cabfbf8361396a81b214c83b001a..0000000000000000000000000000000000000000 --- a/modules/multi2vec-clip/neartext/graphql_argument.go +++ /dev/null @@ -1,120 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package neartext - -import ( - "fmt" - - "github.com/tailor-inc/graphql" - "github.com/weaviate/weaviate/adapters/handlers/graphql/descriptions" -) - -func (g *GraphQLArgumentsProvider) getNearTextArgumentFn(classname string) *graphql.ArgumentConfig { - return g.nearTextArgument("GetObjects", classname) -} - -func (g *GraphQLArgumentsProvider) exploreNearTextArgumentFn() *graphql.ArgumentConfig { - return g.nearTextArgument("Explore", "") -} - -func (g *GraphQLArgumentsProvider) aggregateNearTextArgumentFn(classname string) *graphql.ArgumentConfig { - return g.nearTextArgument("Aggregate", classname) -} - -func (g *GraphQLArgumentsProvider) nearTextArgument(prefix, className string) *graphql.ArgumentConfig { - prefixName := fmt.Sprintf("Txt2VecC11y%s%s", prefix, className) - return &graphql.ArgumentConfig{ - Type: graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sNearTextInpObj", prefixName), - Fields: g.nearTextFields(prefixName), - Description: descriptions.GetWhereInpObj, - }, - ), - } -} - -func (g *GraphQLArgumentsProvider) nearTextFields(prefix string) graphql.InputObjectConfigFieldMap { - nearTextFields := graphql.InputObjectConfigFieldMap{ - "concepts": &graphql.InputObjectFieldConfig{ - // Description: descriptions.Concepts, - Type: graphql.NewNonNull(graphql.NewList(graphql.String)), - }, - "moveTo": &graphql.InputObjectFieldConfig{ - Description: descriptions.VectorMovement, - Type: graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sMoveTo", prefix), - Fields: g.movementInp(fmt.Sprintf("%sMoveTo", prefix)), - }), - }, - "certainty": &graphql.InputObjectFieldConfig{ - Description: descriptions.Certainty, - Type: graphql.Float, - }, - "distance": &graphql.InputObjectFieldConfig{ - Description: descriptions.Distance, - Type: graphql.Float, - }, - "moveAwayFrom": &graphql.InputObjectFieldConfig{ - Description: descriptions.VectorMovement, - Type: graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sMoveAwayFrom", prefix), - Fields: g.movementInp(fmt.Sprintf("%sMoveAwayFrom", prefix)), - }), - }, - } - if g.nearTextTransformer != nil { - nearTextFields["autocorrect"] = &graphql.InputObjectFieldConfig{ - Description: "Autocorrect input text values", - Type: graphql.Boolean, - } - } - return nearTextFields -} - -func (g *GraphQLArgumentsProvider) movementInp(prefix string) graphql.InputObjectConfigFieldMap { - return graphql.InputObjectConfigFieldMap{ - "concepts": &graphql.InputObjectFieldConfig{ - Description: descriptions.Keywords, - Type: graphql.NewList(graphql.String), - }, - "objects": &graphql.InputObjectFieldConfig{ - Description: "objects", - Type: graphql.NewList(g.objectsInpObj(prefix)), - }, - "force": &graphql.InputObjectFieldConfig{ - Description: descriptions.Force, - Type: graphql.NewNonNull(graphql.Float), - }, - } -} - -func (g *GraphQLArgumentsProvider) objectsInpObj(prefix string) *graphql.InputObject { - return graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sMovementObjectsInpObj", prefix), - Fields: graphql.InputObjectConfigFieldMap{ - "id": &graphql.InputObjectFieldConfig{ - Type: graphql.String, - Description: "id of an object", - }, - "beacon": &graphql.InputObjectFieldConfig{ - Type: graphql.String, - Description: descriptions.Beacon, - }, - }, - Description: "Movement Object", - }, - ) -} diff --git a/modules/multi2vec-clip/neartext/graphql_argument_test.go b/modules/multi2vec-clip/neartext/graphql_argument_test.go deleted file mode 100644 index 2952021d8a3341f5e2d484c52daaafe9f3cf9188..0000000000000000000000000000000000000000 --- a/modules/multi2vec-clip/neartext/graphql_argument_test.go +++ /dev/null @@ -1,194 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package neartext - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/tailor-inc/graphql" -) - -func TestNearTextGraphQLArgument(t *testing.T) { - t.Run("should generate nearText argument properly", func(t *testing.T) { - // given - prefix := "Prefix" - classname := "Class" - // when - nearText := New(nil).nearTextArgument(prefix, classname) - - // then - // the built graphQL field needs to support this structure: - // { - // concepts: ["c1", "c2"], - // certainty: 0.9, (or distance) - // moveTo: { - // concepts: ["c1", "c2"], - // objects: [ - // { id: "some-uuid-value"}], - // { beacon: "some-beacon-value"} - // ], - // force: 0.8 - // } - // moveAwayFrom: { - // concepts: ["c1", "c2"], - // objects: [ - // { id: "some-uuid-value"}], - // { beacon: "some-beacon-value"} - // ], - // force: 0.8 - // } - // } - assert.NotNil(t, nearText) - assert.Equal(t, "Txt2VecC11yPrefixClassNearTextInpObj", nearText.Type.Name()) - nearTextFields, ok := nearText.Type.(*graphql.InputObject) - assert.True(t, ok) - assert.NotNil(t, nearTextFields) - assert.Equal(t, 5, len(nearTextFields.Fields())) - fields := nearTextFields.Fields() - concepts := fields["concepts"] - conceptsNonNull, conceptsNonNullOK := concepts.Type.(*graphql.NonNull) - assert.True(t, conceptsNonNullOK) - conceptsNonNullList, conceptsNonNullListOK := conceptsNonNull.OfType.(*graphql.List) - assert.True(t, conceptsNonNullListOK) - assert.Equal(t, "String", conceptsNonNullList.OfType.Name()) - assert.NotNil(t, concepts) - conceptsType, conceptsTypeOK := concepts.Type.(*graphql.NonNull) - assert.True(t, conceptsTypeOK) - assert.NotNil(t, conceptsType) - assert.NotNil(t, fields["certainty"]) - assert.NotNil(t, fields["distance"]) - assert.NotNil(t, fields["moveTo"]) - moveTo, moveToOK := fields["moveTo"].Type.(*graphql.InputObject) - assert.True(t, moveToOK) - assert.Equal(t, 3, len(moveTo.Fields())) - assert.NotNil(t, moveTo.Fields()["concepts"]) - moveToConcepts, moveToConceptsOK := moveTo.Fields()["concepts"].Type.(*graphql.List) - assert.True(t, moveToConceptsOK) - assert.Equal(t, "String", moveToConcepts.OfType.Name()) - assert.NotNil(t, moveToConcepts) - assert.NotNil(t, moveTo.Fields()["objects"]) - moveToObjects, moveToObjectsOK := moveTo.Fields()["objects"].Type.(*graphql.List) - assert.True(t, moveToObjectsOK) - moveToObjectsObjects, moveToObjectsObjectsOK := moveToObjects.OfType.(*graphql.InputObject) - assert.True(t, moveToObjectsObjectsOK) - assert.Equal(t, 2, len(moveToObjectsObjects.Fields())) - assert.NotNil(t, moveToObjectsObjects.Fields()["id"]) - assert.NotNil(t, moveToObjectsObjects.Fields()["beacon"]) - assert.NotNil(t, moveTo.Fields()["force"]) - _, moveToForceOK := moveTo.Fields()["force"].Type.(*graphql.NonNull) - assert.True(t, moveToForceOK) - assert.NotNil(t, fields["moveAwayFrom"]) - moveAwayFrom, moveAwayFromOK := fields["moveAwayFrom"].Type.(*graphql.InputObject) - assert.True(t, moveAwayFromOK) - assert.NotNil(t, moveAwayFrom.Fields()["concepts"]) - assert.NotNil(t, moveAwayFrom.Fields()["objects"]) - moveAwayFromObjects, moveAwayFromObjectsOK := moveAwayFrom.Fields()["objects"].Type.(*graphql.List) - assert.True(t, moveAwayFromObjectsOK) - moveAwayFromObjectsObjects, moveAwayFromObjectsObjectsOK := moveAwayFromObjects.OfType.(*graphql.InputObject) - assert.Equal(t, 2, len(moveAwayFromObjectsObjects.Fields())) - assert.True(t, moveAwayFromObjectsObjectsOK) - assert.NotNil(t, moveAwayFromObjectsObjects.Fields()["id"]) - assert.NotNil(t, moveAwayFromObjectsObjects.Fields()["beacon"]) - assert.NotNil(t, moveAwayFrom.Fields()["force"]) - _, moveAwayFromForceOK := moveAwayFrom.Fields()["force"].Type.(*graphql.NonNull) - assert.True(t, moveAwayFromForceOK) - }) -} - -func TestNearTextGraphQLArgumentWithAutocorrect(t *testing.T) { - t.Run("should generate nearText argument with autocorrect properly", func(t *testing.T) { - // given - prefix := "Prefix" - classname := "Class" - // when - nearText := New(&fakeTransformer{}).nearTextArgument(prefix, classname) - - // then - // the built graphQL field needs to support this structure: - // { - // concepts: ["c1", "c2"], - // certainty: 0.9, (or distance) - // autocorrect: true, - // moveTo: { - // concepts: ["c1", "c2"], - // objects: [ - // { id: "some-uuid-value"}], - // { beacon: "some-beacon-value"} - // ], - // force: 0.8 - // } - // moveAwayFrom: { - // concepts: ["c1", "c2"], - // objects: [ - // { id: "some-uuid-value"}], - // { beacon: "some-beacon-value"} - // ], - // force: 0.8 - // } - // } - assert.NotNil(t, nearText) - assert.Equal(t, "Txt2VecC11yPrefixClassNearTextInpObj", nearText.Type.Name()) - nearTextFields, ok := nearText.Type.(*graphql.InputObject) - assert.True(t, ok) - assert.NotNil(t, nearTextFields) - assert.Equal(t, 6, len(nearTextFields.Fields())) - fields := nearTextFields.Fields() - concepts := fields["concepts"] - conceptsNonNull, conceptsNonNullOK := concepts.Type.(*graphql.NonNull) - assert.True(t, conceptsNonNullOK) - conceptsNonNullList, conceptsNonNullListOK := conceptsNonNull.OfType.(*graphql.List) - assert.True(t, conceptsNonNullListOK) - assert.Equal(t, "String", conceptsNonNullList.OfType.Name()) - assert.NotNil(t, concepts) - conceptsType, conceptsTypeOK := concepts.Type.(*graphql.NonNull) - assert.True(t, conceptsTypeOK) - assert.NotNil(t, conceptsType) - assert.NotNil(t, fields["certainty"]) - assert.NotNil(t, fields["autocorrect"]) - assert.NotNil(t, fields["moveTo"]) - moveTo, moveToOK := fields["moveTo"].Type.(*graphql.InputObject) - assert.True(t, moveToOK) - assert.Equal(t, 3, len(moveTo.Fields())) - assert.NotNil(t, moveTo.Fields()["concepts"]) - moveToConcepts, moveToConceptsOK := moveTo.Fields()["concepts"].Type.(*graphql.List) - assert.True(t, moveToConceptsOK) - assert.Equal(t, "String", moveToConcepts.OfType.Name()) - assert.NotNil(t, moveToConcepts) - assert.NotNil(t, moveTo.Fields()["objects"]) - moveToObjects, moveToObjectsOK := moveTo.Fields()["objects"].Type.(*graphql.List) - assert.True(t, moveToObjectsOK) - moveToObjectsObjects, moveToObjectsObjectsOK := moveToObjects.OfType.(*graphql.InputObject) - assert.True(t, moveToObjectsObjectsOK) - assert.Equal(t, 2, len(moveToObjectsObjects.Fields())) - assert.NotNil(t, moveToObjectsObjects.Fields()["id"]) - assert.NotNil(t, moveToObjectsObjects.Fields()["beacon"]) - assert.NotNil(t, moveTo.Fields()["force"]) - _, moveToForceOK := moveTo.Fields()["force"].Type.(*graphql.NonNull) - assert.True(t, moveToForceOK) - assert.NotNil(t, fields["moveAwayFrom"]) - moveAwayFrom, moveAwayFromOK := fields["moveAwayFrom"].Type.(*graphql.InputObject) - assert.True(t, moveAwayFromOK) - assert.NotNil(t, moveAwayFrom.Fields()["concepts"]) - assert.NotNil(t, moveAwayFrom.Fields()["objects"]) - moveAwayFromObjects, moveAwayFromObjectsOK := moveAwayFrom.Fields()["objects"].Type.(*graphql.List) - assert.True(t, moveAwayFromObjectsOK) - moveAwayFromObjectsObjects, moveAwayFromObjectsObjectsOK := moveAwayFromObjects.OfType.(*graphql.InputObject) - assert.Equal(t, 2, len(moveAwayFromObjectsObjects.Fields())) - assert.True(t, moveAwayFromObjectsObjectsOK) - assert.NotNil(t, moveAwayFromObjectsObjects.Fields()["id"]) - assert.NotNil(t, moveAwayFromObjectsObjects.Fields()["beacon"]) - assert.NotNil(t, moveAwayFrom.Fields()["force"]) - _, moveAwayFromForceOK := moveAwayFrom.Fields()["force"].Type.(*graphql.NonNull) - assert.True(t, moveAwayFromForceOK) - }) -} diff --git a/modules/multi2vec-clip/neartext/graphql_extract_test.go b/modules/multi2vec-clip/neartext/graphql_extract_test.go deleted file mode 100644 index 52c24d7fc8edcd21b6d1fb1292622233b425269b..0000000000000000000000000000000000000000 --- a/modules/multi2vec-clip/neartext/graphql_extract_test.go +++ /dev/null @@ -1,620 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package neartext - -import ( - "reflect" - "testing" - - "github.com/weaviate/weaviate/usecases/modulecomponents/nearText" -) - -func Test_extractNearTextFn(t *testing.T) { - type args struct { - source map[string]interface{} - } - tests := []struct { - name string - args args - want *nearText.NearTextParams - }{ - { - "Extract with concepts", - args{ - source: map[string]interface{}{ - "concepts": []interface{}{"c1", "c2", "c3"}, - }, - }, - &nearText.NearTextParams{ - Values: []string{"c1", "c2", "c3"}, - }, - }, - { - "Extract with concepts, distance, limit and network", - args{ - source: map[string]interface{}{ - "concepts": []interface{}{"c1", "c2", "c3"}, - "distance": float64(0.4), - "limit": 100, - "network": true, - }, - }, - &nearText.NearTextParams{ - Values: []string{"c1", "c2", "c3"}, - Distance: 0.4, - WithDistance: true, - Limit: 100, - Network: true, - }, - }, - { - "Extract with concepts, certainty, limit and network", - args{ - source: map[string]interface{}{ - "concepts": []interface{}{"c1", "c2", "c3"}, - "certainty": float64(0.4), - "limit": 100, - "network": true, - }, - }, - &nearText.NearTextParams{ - Values: []string{"c1", "c2", "c3"}, - Certainty: 0.4, - Limit: 100, - Network: true, - }, - }, - { - "Extract with moveTo, moveAwayFrom, and distance", - args{ - source: map[string]interface{}{ - "concepts": []interface{}{"c1", "c2", "c3"}, - "distance": float64(0.89), - "limit": 500, - "network": false, - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"positive"}, - "force": float64(0.5), - }, - "moveAwayFrom": map[string]interface{}{ - "concepts": []interface{}{"epic"}, - "force": float64(0.25), - }, - }, - }, - &nearText.NearTextParams{ - Values: []string{"c1", "c2", "c3"}, - Distance: 0.89, - WithDistance: true, - Limit: 500, - Network: false, - MoveTo: nearText.ExploreMove{ - Values: []string{"positive"}, - Force: 0.5, - }, - MoveAwayFrom: nearText.ExploreMove{ - Values: []string{"epic"}, - Force: 0.25, - }, - }, - }, - { - "Extract with moveTo, moveAwayFrom, and certainty", - args{ - source: map[string]interface{}{ - "concepts": []interface{}{"c1", "c2", "c3"}, - "certainty": float64(0.89), - "limit": 500, - "network": false, - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"positive"}, - "force": float64(0.5), - }, - "moveAwayFrom": map[string]interface{}{ - "concepts": []interface{}{"epic"}, - "force": float64(0.25), - }, - }, - }, - &nearText.NearTextParams{ - Values: []string{"c1", "c2", "c3"}, - Certainty: 0.89, - Limit: 500, - Network: false, - MoveTo: nearText.ExploreMove{ - Values: []string{"positive"}, - Force: 0.5, - }, - MoveAwayFrom: nearText.ExploreMove{ - Values: []string{"epic"}, - Force: 0.25, - }, - }, - }, - { - "Extract with moveTo, moveAwayFrom, distance, and objects", - args{ - source: map[string]interface{}{ - "concepts": []interface{}{"c1", "c2", "c3"}, - "distance": float64(0.89), - "limit": 500, - "network": false, - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"positive"}, - "force": float64(0.5), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveTo-uuid1", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveTo-uuid2", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveTo-uuid3", - }, - }, - }, - "moveAwayFrom": map[string]interface{}{ - "concepts": []interface{}{"epic"}, - "force": float64(0.25), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveAwayFrom-uuid1", - }, - map[string]interface{}{ - "id": "moveAwayFrom-uuid2", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveAwayFrom-uuid3", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveAwayFrom-uuid4", - }, - }, - }, - }, - }, - &nearText.NearTextParams{ - Values: []string{"c1", "c2", "c3"}, - Distance: 0.89, - WithDistance: true, - Limit: 500, - Network: false, - MoveTo: nearText.ExploreMove{ - Values: []string{"positive"}, - Force: 0.5, - Objects: []nearText.ObjectMove{ - {ID: "moveTo-uuid1"}, - {Beacon: "weaviate://localhost/moveTo-uuid2"}, - {Beacon: "weaviate://localhost/moveTo-uuid3"}, - }, - }, - MoveAwayFrom: nearText.ExploreMove{ - Values: []string{"epic"}, - Force: 0.25, - Objects: []nearText.ObjectMove{ - {ID: "moveAwayFrom-uuid1"}, - {ID: "moveAwayFrom-uuid2"}, - {Beacon: "weaviate://localhost/moveAwayFrom-uuid3"}, - {Beacon: "weaviate://localhost/moveAwayFrom-uuid4"}, - }, - }, - }, - }, - { - "Extract with moveTo and moveAwayFrom (and objects)", - args{ - source: map[string]interface{}{ - "concepts": []interface{}{"c1", "c2", "c3"}, - "certainty": float64(0.89), - "limit": 500, - "network": false, - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"positive"}, - "force": float64(0.5), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveTo-uuid1", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveTo-uuid2", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveTo-uuid3", - }, - }, - }, - "moveAwayFrom": map[string]interface{}{ - "concepts": []interface{}{"epic"}, - "force": float64(0.25), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveAwayFrom-uuid1", - }, - map[string]interface{}{ - "id": "moveAwayFrom-uuid2", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveAwayFrom-uuid3", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveAwayFrom-uuid4", - }, - }, - }, - }, - }, - &nearText.NearTextParams{ - Values: []string{"c1", "c2", "c3"}, - Certainty: 0.89, - Limit: 500, - Network: false, - MoveTo: nearText.ExploreMove{ - Values: []string{"positive"}, - Force: 0.5, - Objects: []nearText.ObjectMove{ - {ID: "moveTo-uuid1"}, - {Beacon: "weaviate://localhost/moveTo-uuid2"}, - {Beacon: "weaviate://localhost/moveTo-uuid3"}, - }, - }, - MoveAwayFrom: nearText.ExploreMove{ - Values: []string{"epic"}, - Force: 0.25, - Objects: []nearText.ObjectMove{ - {ID: "moveAwayFrom-uuid1"}, - {ID: "moveAwayFrom-uuid2"}, - {Beacon: "weaviate://localhost/moveAwayFrom-uuid3"}, - {Beacon: "weaviate://localhost/moveAwayFrom-uuid4"}, - }, - }, - }, - }, - { - "Extract with moveTo and moveAwayFrom, distance, and doubled objects", - args{ - source: map[string]interface{}{ - "concepts": []interface{}{"c1", "c2", "c3"}, - "distance": float64(0.89), - "limit": 500, - "network": false, - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"positive"}, - "force": float64(0.5), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveTo-uuid1", - "beacon": "weaviate://localhost/moveTo-uuid2", - }, - map[string]interface{}{ - "id": "moveTo-uuid1", - "beacon": "weaviate://localhost/moveTo-uuid2", - }, - }, - }, - "moveAwayFrom": map[string]interface{}{ - "concepts": []interface{}{"epic"}, - "force": float64(0.25), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveAwayFrom-uuid1", - "beacon": "weaviate://localhost/moveAwayFrom-uuid1", - }, - map[string]interface{}{ - "id": "moveAwayFrom-uuid2", - "beacon": "weaviate://localhost/moveAwayFrom-uuid2", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveAwayFrom-uuid3", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveAwayFrom-uuid4", - }, - }, - }, - }, - }, - &nearText.NearTextParams{ - Values: []string{"c1", "c2", "c3"}, - Distance: 0.89, - WithDistance: true, - Limit: 500, - Network: false, - MoveTo: nearText.ExploreMove{ - Values: []string{"positive"}, - Force: 0.5, - Objects: []nearText.ObjectMove{ - {ID: "moveTo-uuid1", Beacon: "weaviate://localhost/moveTo-uuid2"}, - {ID: "moveTo-uuid1", Beacon: "weaviate://localhost/moveTo-uuid2"}, - }, - }, - MoveAwayFrom: nearText.ExploreMove{ - Values: []string{"epic"}, - Force: 0.25, - Objects: []nearText.ObjectMove{ - {ID: "moveAwayFrom-uuid1", Beacon: "weaviate://localhost/moveAwayFrom-uuid1"}, - {ID: "moveAwayFrom-uuid2", Beacon: "weaviate://localhost/moveAwayFrom-uuid2"}, - {Beacon: "weaviate://localhost/moveAwayFrom-uuid3"}, - {Beacon: "weaviate://localhost/moveAwayFrom-uuid4"}, - }, - }, - }, - }, - { - "Extract with moveTo and moveAwayFrom, certainty, and doubled objects", - args{ - source: map[string]interface{}{ - "concepts": []interface{}{"c1", "c2", "c3"}, - "certainty": float64(0.89), - "limit": 500, - "network": false, - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"positive"}, - "force": float64(0.5), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveTo-uuid1", - "beacon": "weaviate://localhost/moveTo-uuid2", - }, - map[string]interface{}{ - "id": "moveTo-uuid1", - "beacon": "weaviate://localhost/moveTo-uuid2", - }, - }, - }, - "moveAwayFrom": map[string]interface{}{ - "concepts": []interface{}{"epic"}, - "force": float64(0.25), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveAwayFrom-uuid1", - "beacon": "weaviate://localhost/moveAwayFrom-uuid1", - }, - map[string]interface{}{ - "id": "moveAwayFrom-uuid2", - "beacon": "weaviate://localhost/moveAwayFrom-uuid2", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveAwayFrom-uuid3", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveAwayFrom-uuid4", - }, - }, - }, - }, - }, - &nearText.NearTextParams{ - Values: []string{"c1", "c2", "c3"}, - Certainty: 0.89, - Limit: 500, - Network: false, - MoveTo: nearText.ExploreMove{ - Values: []string{"positive"}, - Force: 0.5, - Objects: []nearText.ObjectMove{ - {ID: "moveTo-uuid1", Beacon: "weaviate://localhost/moveTo-uuid2"}, - {ID: "moveTo-uuid1", Beacon: "weaviate://localhost/moveTo-uuid2"}, - }, - }, - MoveAwayFrom: nearText.ExploreMove{ - Values: []string{"epic"}, - Force: 0.25, - Objects: []nearText.ObjectMove{ - {ID: "moveAwayFrom-uuid1", Beacon: "weaviate://localhost/moveAwayFrom-uuid1"}, - {ID: "moveAwayFrom-uuid2", Beacon: "weaviate://localhost/moveAwayFrom-uuid2"}, - {Beacon: "weaviate://localhost/moveAwayFrom-uuid3"}, - {Beacon: "weaviate://localhost/moveAwayFrom-uuid4"}, - }, - }, - }, - }, - } - - testsWithAutocorrect := []struct { - name string - args args - want *nearText.NearTextParams - }{ - { - "Extract with concepts", - args{ - source: map[string]interface{}{ - "concepts": []interface{}{"c1", "c2", "c3"}, - "autocorrect": true, - }, - }, - &nearText.NearTextParams{ - Values: []string{"c1", "c2", "c3"}, - Autocorrect: true, - }, - }, - { - "Extract with concepts and perform autocorrect", - args{ - source: map[string]interface{}{ - "concepts": []interface{}{"transform this", "c2", "transform this"}, - "autocorrect": true, - }, - }, - &nearText.NearTextParams{ - Values: []string{"transformed text", "c2", "transformed text"}, - Autocorrect: true, - }, - }, - { - "Extract with moveTo and moveAwayFrom, distance (and doubled objects) and autocorrect", - args{ - source: map[string]interface{}{ - "concepts": []interface{}{"transform this", "c1", "c2", "c3", "transform this"}, - "distance": float64(0.89), - "limit": 500, - "network": false, - "autocorrect": true, - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"positive"}, - "force": float64(0.5), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveTo-uuid1", - "beacon": "weaviate://localhost/moveTo-uuid2", - }, - map[string]interface{}{ - "id": "moveTo-uuid1", - "beacon": "weaviate://localhost/moveTo-uuid2", - }, - }, - }, - "moveAwayFrom": map[string]interface{}{ - "concepts": []interface{}{"epic"}, - "force": float64(0.25), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveAwayFrom-uuid1", - "beacon": "weaviate://localhost/moveAwayFrom-uuid1", - }, - map[string]interface{}{ - "id": "moveAwayFrom-uuid2", - "beacon": "weaviate://localhost/moveAwayFrom-uuid2", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveAwayFrom-uuid3", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveAwayFrom-uuid4", - }, - }, - }, - }, - }, - &nearText.NearTextParams{ - Values: []string{"transformed text", "c1", "c2", "c3", "transformed text"}, - Distance: 0.89, - WithDistance: true, - Limit: 500, - Network: false, - Autocorrect: true, - MoveTo: nearText.ExploreMove{ - Values: []string{"positive"}, - Force: 0.5, - Objects: []nearText.ObjectMove{ - {ID: "moveTo-uuid1", Beacon: "weaviate://localhost/moveTo-uuid2"}, - {ID: "moveTo-uuid1", Beacon: "weaviate://localhost/moveTo-uuid2"}, - }, - }, - MoveAwayFrom: nearText.ExploreMove{ - Values: []string{"epic"}, - Force: 0.25, - Objects: []nearText.ObjectMove{ - {ID: "moveAwayFrom-uuid1", Beacon: "weaviate://localhost/moveAwayFrom-uuid1"}, - {ID: "moveAwayFrom-uuid2", Beacon: "weaviate://localhost/moveAwayFrom-uuid2"}, - {Beacon: "weaviate://localhost/moveAwayFrom-uuid3"}, - {Beacon: "weaviate://localhost/moveAwayFrom-uuid4"}, - }, - }, - }, - }, - { - "Extract with moveTo and moveAwayFrom, certainty (and doubled objects) and autocorrect", - args{ - source: map[string]interface{}{ - "concepts": []interface{}{"transform this", "c1", "c2", "c3", "transform this"}, - "certainty": float64(0.89), - "limit": 500, - "network": false, - "autocorrect": true, - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"positive"}, - "force": float64(0.5), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveTo-uuid1", - "beacon": "weaviate://localhost/moveTo-uuid2", - }, - map[string]interface{}{ - "id": "moveTo-uuid1", - "beacon": "weaviate://localhost/moveTo-uuid2", - }, - }, - }, - "moveAwayFrom": map[string]interface{}{ - "concepts": []interface{}{"epic"}, - "force": float64(0.25), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveAwayFrom-uuid1", - "beacon": "weaviate://localhost/moveAwayFrom-uuid1", - }, - map[string]interface{}{ - "id": "moveAwayFrom-uuid2", - "beacon": "weaviate://localhost/moveAwayFrom-uuid2", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveAwayFrom-uuid3", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveAwayFrom-uuid4", - }, - }, - }, - }, - }, - &nearText.NearTextParams{ - Values: []string{"transformed text", "c1", "c2", "c3", "transformed text"}, - Certainty: 0.89, - Limit: 500, - Network: false, - Autocorrect: true, - MoveTo: nearText.ExploreMove{ - Values: []string{"positive"}, - Force: 0.5, - Objects: []nearText.ObjectMove{ - {ID: "moveTo-uuid1", Beacon: "weaviate://localhost/moveTo-uuid2"}, - {ID: "moveTo-uuid1", Beacon: "weaviate://localhost/moveTo-uuid2"}, - }, - }, - MoveAwayFrom: nearText.ExploreMove{ - Values: []string{"epic"}, - Force: 0.25, - Objects: []nearText.ObjectMove{ - {ID: "moveAwayFrom-uuid1", Beacon: "weaviate://localhost/moveAwayFrom-uuid1"}, - {ID: "moveAwayFrom-uuid2", Beacon: "weaviate://localhost/moveAwayFrom-uuid2"}, - {Beacon: "weaviate://localhost/moveAwayFrom-uuid3"}, - {Beacon: "weaviate://localhost/moveAwayFrom-uuid4"}, - }, - }, - }, - }, - } - testsWithAutocorrect = append(testsWithAutocorrect, tests...) - - t.Run("should extract values", func(t *testing.T) { - provider := New(nil) - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := provider.extractNearTextFn(tt.args.source); !reflect.DeepEqual(got, tt.want) { - t.Errorf("extractNearTextFn() = %v, want %v", got, tt.want) - } - }) - } - }) - t.Run("should extract values with transformer", func(t *testing.T) { - provider := New(&fakeTransformer{}) - for _, tt := range testsWithAutocorrect { - t.Run(tt.name, func(t *testing.T) { - if got := provider.extractNearTextFn(tt.args.source); !reflect.DeepEqual(got, tt.want) { - t.Errorf("extractNearTextFn() = %v, want %v", got, tt.want) - } - }) - } - }) -} diff --git a/modules/multi2vec-clip/neartext/graphql_provider.go b/modules/multi2vec-clip/neartext/graphql_provider.go deleted file mode 100644 index 5b6d7c335a957d0deb51fd01d0a35638f07ff595..0000000000000000000000000000000000000000 --- a/modules/multi2vec-clip/neartext/graphql_provider.go +++ /dev/null @@ -1,40 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package neartext - -import ( - "github.com/weaviate/weaviate/entities/modulecapabilities" -) - -type GraphQLArgumentsProvider struct { - nearTextTransformer modulecapabilities.TextTransform -} - -func New(nearTextTransformer modulecapabilities.TextTransform) *GraphQLArgumentsProvider { - return &GraphQLArgumentsProvider{nearTextTransformer} -} - -func (g *GraphQLArgumentsProvider) Arguments() map[string]modulecapabilities.GraphQLArgument { - arguments := map[string]modulecapabilities.GraphQLArgument{} - arguments["nearText"] = g.getNearText() - return arguments -} - -func (g *GraphQLArgumentsProvider) getNearText() modulecapabilities.GraphQLArgument { - return modulecapabilities.GraphQLArgument{ - GetArgumentsFunction: g.getNearTextArgumentFn, - AggregateArgumentsFunction: g.aggregateNearTextArgumentFn, - ExploreArgumentsFunction: g.exploreNearTextArgumentFn, - ExtractFunction: g.extractNearTextFn, - ValidateFunction: g.validateNearTextFn, - } -} diff --git a/modules/multi2vec-clip/neartext/grapqhl_extract.go b/modules/multi2vec-clip/neartext/grapqhl_extract.go deleted file mode 100644 index c308fe87734ef0d215b5babf8c684f9d9d3be99b..0000000000000000000000000000000000000000 --- a/modules/multi2vec-clip/neartext/grapqhl_extract.go +++ /dev/null @@ -1,115 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package neartext - -import "github.com/weaviate/weaviate/usecases/modulecomponents/nearText" - -// ExtractNearText arguments, such as "concepts", "moveTo", "moveAwayFrom", -// "limit", etc. -func (g *GraphQLArgumentsProvider) extractNearTextFn(source map[string]interface{}) interface{} { - var args nearText.NearTextParams - - // keywords is a required argument, so we don't need to check for its existing - keywords := source["concepts"].([]interface{}) - args.Values = make([]string, len(keywords)) - for i, value := range keywords { - args.Values[i] = value.(string) - } - - // autocorrect is an optional arg, so it could be nil - autocorrect, ok := source["autocorrect"] - if ok { - args.Autocorrect = autocorrect.(bool) - } - - // if there's text transformer present and autocorrect set to true - // perform text transformation operation - if args.Autocorrect && g.nearTextTransformer != nil { - if transformedValues, err := g.nearTextTransformer.Transform(args.Values); err == nil { - args.Values = transformedValues - } - } - - // limit is an optional arg, so it could be nil - limit, ok := source["limit"] - if ok { - // the type is fixed through gql config, no need to catch incorrect type - // assumption - args.Limit = limit.(int) - } - - certainty, ok := source["certainty"] - if ok { - args.Certainty = certainty.(float64) - } - - distance, ok := source["distance"] - if ok { - args.Distance = distance.(float64) - args.WithDistance = true - } - - // moveTo is an optional arg, so it could be nil - moveTo, ok := source["moveTo"] - if ok { - args.MoveTo = extractMovement(moveTo) - } - - // network is an optional arg, so it could be nil - network, ok := source["network"] - if ok { - args.Network = network.(bool) - } - - // moveAwayFrom is an optional arg, so it could be nil - moveAwayFrom, ok := source["moveAwayFrom"] - if ok { - args.MoveAwayFrom = extractMovement(moveAwayFrom) - } - - return &args -} - -func extractMovement(input interface{}) nearText.ExploreMove { - // the type is fixed through gql config, no need to catch incorrect type - // assumption, all fields are required so we don't need to check for their - // presence - moveToMap := input.(map[string]interface{}) - res := nearText.ExploreMove{} - res.Force = float32(moveToMap["force"].(float64)) - - keywords, ok := moveToMap["concepts"].([]interface{}) - if ok { - res.Values = make([]string, len(keywords)) - for i, value := range keywords { - res.Values[i] = value.(string) - } - } - - objects, ok := moveToMap["objects"].([]interface{}) - if ok { - res.Objects = make([]nearText.ObjectMove, len(objects)) - for i, value := range objects { - v, ok := value.(map[string]interface{}) - if ok { - if v["id"] != nil { - res.Objects[i].ID = v["id"].(string) - } - if v["beacon"] != nil { - res.Objects[i].Beacon = v["beacon"].(string) - } - } - } - } - - return res -} diff --git a/modules/multi2vec-clip/neartext/param.go b/modules/multi2vec-clip/neartext/param.go deleted file mode 100644 index 3e9d851d06f3dabe7d264949ec3c8e3fb62a483e..0000000000000000000000000000000000000000 --- a/modules/multi2vec-clip/neartext/param.go +++ /dev/null @@ -1,25 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package neartext - -import ( - "github.com/pkg/errors" - "github.com/weaviate/weaviate/usecases/modulecomponents/nearText" -) - -func (g *GraphQLArgumentsProvider) validateNearTextFn(param interface{}) error { - nearTextParams, ok := param.(*nearText.NearTextParams) - if !ok { - return errors.New("'nearText' invalid parameter") - } - return nearTextParams.Validate() -} diff --git a/modules/multi2vec-clip/neartext/searcher.go b/modules/multi2vec-clip/neartext/searcher.go deleted file mode 100644 index f3f1e399fa647c05914f888c8c0c951ce8f0b537..0000000000000000000000000000000000000000 --- a/modules/multi2vec-clip/neartext/searcher.go +++ /dev/null @@ -1,145 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package neartext - -import ( - "context" - - "github.com/weaviate/weaviate/usecases/modulecomponents/nearText" - - "github.com/go-openapi/strfmt" - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/schema/crossref" - localvectorizer "github.com/weaviate/weaviate/modules/multi2vec-clip/vectorizer" -) - -type Searcher struct { - vectorizer vectorizer -} - -func NewSearcher(vectorizer vectorizer) *Searcher { - return &Searcher{vectorizer} -} - -type vectorizer interface { - Texts(ctx context.Context, input []string, - settings localvectorizer.ClassSettings) ([]float32, error) - MoveTo(source, target []float32, weight float32) ([]float32, error) - MoveAwayFrom(source, target []float32, weight float32) ([]float32, error) - CombineVectors(vectors [][]float32) []float32 -} - -func (s *Searcher) VectorSearches() map[string]modulecapabilities.VectorForParams { - vectorSearches := map[string]modulecapabilities.VectorForParams{} - vectorSearches["nearText"] = s.vectorForNearTextParam - return vectorSearches -} - -func (s *Searcher) vectorForNearTextParam(ctx context.Context, params interface{}, - className string, - findVectorFn modulecapabilities.FindVectorFn, - cfg moduletools.ClassConfig, -) ([]float32, error) { - return s.vectorFromNearTextParam(ctx, params.(*nearText.NearTextParams), className, findVectorFn, cfg) -} - -func (s *Searcher) vectorFromNearTextParam(ctx context.Context, - params *nearText.NearTextParams, - className string, findVectorFn modulecapabilities.FindVectorFn, - cfg moduletools.ClassConfig, -) ([]float32, error) { - // it is safe to call NewClassSettings even knowing that cfg can be nil, it - // is to built to work with all defaults in the case of a nil-config, see - // vectorizer/class_settings_test.go for details. - settings := localvectorizer.NewClassSettings(cfg) - tenant := cfg.Tenant() - vector, err := s.vectorizer.Texts(ctx, params.Values, settings) - if err != nil { - return nil, errors.Errorf("vectorize keywords: %v", err) - } - - moveTo := params.MoveTo - if moveTo.Force > 0 && (len(moveTo.Values) > 0 || len(moveTo.Objects) > 0) { - moveToVector, err := s.vectorFromValuesAndObjects(ctx, moveTo.Values, - moveTo.Objects, className, findVectorFn, settings, tenant) - if err != nil { - return nil, errors.Errorf("vectorize move to: %v", err) - } - - afterMoveTo, err := s.vectorizer.MoveTo(vector, moveToVector, moveTo.Force) - if err != nil { - return nil, err - } - vector = afterMoveTo - } - - moveAway := params.MoveAwayFrom - if moveAway.Force > 0 && (len(moveAway.Values) > 0 || len(moveAway.Objects) > 0) { - moveAwayVector, err := s.vectorFromValuesAndObjects(ctx, moveAway.Values, - moveAway.Objects, className, findVectorFn, settings, tenant) - if err != nil { - return nil, errors.Errorf("vectorize move away from: %v", err) - } - - afterMoveFrom, err := s.vectorizer.MoveAwayFrom(vector, moveAwayVector, moveAway.Force) - if err != nil { - return nil, err - } - vector = afterMoveFrom - } - - return vector, nil -} - -func (s *Searcher) vectorFromValuesAndObjects(ctx context.Context, - values []string, objects []nearText.ObjectMove, - className string, - findVectorFn modulecapabilities.FindVectorFn, - settings localvectorizer.ClassSettings, tenant string, -) ([]float32, error) { - var objectVectors [][]float32 - - if len(values) > 0 { - moveToVector, err := s.vectorizer.Texts(ctx, values, settings) - if err != nil { - return nil, errors.Errorf("vectorize move to: %v", err) - } - objectVectors = append(objectVectors, moveToVector) - } - - if len(objects) > 0 { - var id strfmt.UUID - for _, obj := range objects { - if len(obj.ID) > 0 { - id = strfmt.UUID(obj.ID) - } - if len(obj.Beacon) > 0 { - ref, err := crossref.Parse(obj.Beacon) - if err != nil { - return nil, err - } - id = ref.TargetID - } - - vector, err := findVectorFn(ctx, className, id, tenant) - if err != nil { - return nil, err - } - - objectVectors = append(objectVectors, vector) - } - } - - return s.vectorizer.CombineVectors(objectVectors), nil -} diff --git a/modules/multi2vec-clip/vectorizer/class_settings.go b/modules/multi2vec-clip/vectorizer/class_settings.go deleted file mode 100644 index b828994da907517404dce3def09752765e3f3d06..0000000000000000000000000000000000000000 --- a/modules/multi2vec-clip/vectorizer/class_settings.go +++ /dev/null @@ -1,214 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "encoding/json" - "fmt" - "strconv" - - "github.com/pkg/errors" - - "github.com/weaviate/weaviate/entities/moduletools" -) - -type classSettings struct { - cfg moduletools.ClassConfig -} - -func NewClassSettings(cfg moduletools.ClassConfig) *classSettings { - return &classSettings{cfg: cfg} -} - -func (ic *classSettings) ImageField(property string) bool { - return ic.field("imageFields", property) -} - -func (ic *classSettings) ImageFieldsWeights() ([]float32, error) { - return ic.getFieldsWeights("image") -} - -func (ic *classSettings) TextField(property string) bool { - return ic.field("textFields", property) -} - -func (ic *classSettings) TextFieldsWeights() ([]float32, error) { - return ic.getFieldsWeights("text") -} - -func (ic *classSettings) field(name, property string) bool { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return false - } - - fields, ok := ic.cfg.Class()[name] - if !ok { - return false - } - - fieldsArray, ok := fields.([]interface{}) - if !ok { - return false - } - - fieldNames := make([]string, len(fieldsArray)) - for i, value := range fieldsArray { - fieldNames[i] = value.(string) - } - - for i := range fieldNames { - if fieldNames[i] == property { - return true - } - } - - return false -} - -func (ic *classSettings) Validate() error { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return errors.New("empty config") - } - - imageFields, imageFieldsOk := ic.cfg.Class()["imageFields"] - textFields, textFieldsOk := ic.cfg.Class()["textFields"] - if !imageFieldsOk && !textFieldsOk { - return errors.New("textFields or imageFields setting needs to be present") - } - - if imageFieldsOk { - imageFieldsCount, err := ic.validateFields("image", imageFields) - if err != nil { - return err - } - err = ic.validateWeights("image", imageFieldsCount) - if err != nil { - return err - } - } - - if textFieldsOk { - textFieldsCount, err := ic.validateFields("text", textFields) - if err != nil { - return err - } - err = ic.validateWeights("text", textFieldsCount) - if err != nil { - return err - } - } - - return nil -} - -func (ic *classSettings) validateFields(name string, fields interface{}) (int, error) { - fieldsArray, ok := fields.([]interface{}) - if !ok { - return 0, errors.Errorf("%sFields must be an array", name) - } - - if len(fieldsArray) == 0 { - return 0, errors.Errorf("must contain at least one %s field name in %sFields", name, name) - } - - for _, value := range fieldsArray { - v, ok := value.(string) - if !ok { - return 0, errors.Errorf("%sField must be a string", name) - } - if len(v) == 0 { - return 0, errors.Errorf("%sField values cannot be empty", name) - } - } - - return len(fieldsArray), nil -} - -func (ic *classSettings) validateWeights(name string, count int) error { - weights, ok := ic.getWeights(name) - if ok { - if len(weights) != count { - return errors.Errorf("weights.%sFields does not equal number of %sFields", name, name) - } - _, err := ic.getWeightsArray(weights) - if err != nil { - return err - } - } - - return nil -} - -func (ic *classSettings) getWeights(name string) ([]interface{}, bool) { - weights, ok := ic.cfg.Class()["weights"] - if ok { - weightsObject, ok := weights.(map[string]interface{}) - if ok { - fieldWeights, ok := weightsObject[fmt.Sprintf("%sFields", name)] - if ok { - fieldWeightsArray, ok := fieldWeights.([]interface{}) - if ok { - return fieldWeightsArray, ok - } - } - } - } - - return nil, false -} - -func (ic *classSettings) getWeightsArray(weights []interface{}) ([]float32, error) { - weightsArray := make([]float32, len(weights)) - for i := range weights { - weight, err := ic.getNumber(weights[i]) - if err != nil { - return nil, err - } - weightsArray[i] = weight - } - return weightsArray, nil -} - -func (ic *classSettings) getFieldsWeights(name string) ([]float32, error) { - weights, ok := ic.getWeights(name) - if ok { - return ic.getWeightsArray(weights) - } - return nil, nil -} - -func (ic *classSettings) getNumber(in interface{}) (float32, error) { - switch i := in.(type) { - case float64: - return float32(i), nil - case float32: - return i, nil - case int: - return float32(i), nil - case string: - num, err := strconv.ParseFloat(i, 64) - if err != nil { - return 0, err - } - return float32(num), err - case json.Number: - num, err := i.Float64() - if err != nil { - return 0, err - } - return float32(num), err - default: - return 0.0, errors.Errorf("Unrecognized weight entry type: %T", i) - } -} diff --git a/modules/multi2vec-clip/vectorizer/class_settings_test.go b/modules/multi2vec-clip/vectorizer/class_settings_test.go deleted file mode 100644 index 76e3e6c1b89940d84699cde36182e294dad034a1..0000000000000000000000000000000000000000 --- a/modules/multi2vec-clip/vectorizer/class_settings_test.go +++ /dev/null @@ -1,173 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "encoding/json" - "testing" - - "github.com/weaviate/weaviate/entities/moduletools" -) - -func Test_classSettings_Validate(t *testing.T) { - type fields struct { - cfg moduletools.ClassConfig - } - tests := []struct { - name string - fields fields - wantErr bool - }{ - { - name: "should not pass with empty config", - wantErr: true, - }, - { - name: "should not pass with nil config", - fields: fields{ - cfg: nil, - }, - wantErr: true, - }, - { - name: "should not pass with nil imageFields", - fields: fields{ - cfg: newConfigBuilder().addSetting("imageFields", nil).build(), - }, - wantErr: true, - }, - { - name: "should not pass with fault imageFields value", - fields: fields{ - cfg: newConfigBuilder().addSetting("imageFields", []string{}).build(), - }, - wantErr: true, - }, - { - name: "should not pass with empty imageFields", - fields: fields{ - cfg: newConfigBuilder().addSetting("imageFields", []interface{}{}).build(), - }, - wantErr: true, - }, - { - name: "should not pass with empty string in imageFields", - fields: fields{ - cfg: newConfigBuilder().addSetting("imageFields", []interface{}{""}).build(), - }, - wantErr: true, - }, - { - name: "should not pass with int value in imageFields", - fields: fields{ - cfg: newConfigBuilder().addSetting("imageFields", []interface{}{1.0}).build(), - }, - wantErr: true, - }, - { - name: "should pass with proper value in imageFields", - fields: fields{ - cfg: newConfigBuilder().addSetting("imageFields", []interface{}{"field"}).build(), - }, - }, - { - name: "should pass with proper value in imageFields and textFields", - fields: fields{ - cfg: newConfigBuilder(). - addSetting("imageFields", []interface{}{"imageField"}). - addSetting("textFields", []interface{}{"textField"}). - build(), - }, - }, - { - name: "should pass with proper value in 2 imageFields and 2 textFields", - fields: fields{ - cfg: newConfigBuilder(). - addSetting("textFields", []interface{}{"textField1", "textField2"}). - addSetting("imageFields", []interface{}{"imageField1", "imageField2"}). - build(), - }, - }, - { - name: "should pass with proper value in 2 imageFields and 2 textFields and weights", - fields: fields{ - cfg: newConfigBuilder(). - addSetting("textFields", []interface{}{"textField1", "textField2"}). - addSetting("imageFields", []interface{}{"imageField1", "imageField2"}). - addWeights([]interface{}{1, 2}, []interface{}{1, 2}). - build(), - }, - }, - { - name: "should pass with proper value in 1 imageFields and 2 textFields and weights", - fields: fields{ - cfg: newConfigBuilder(). - addSetting("textFields", []interface{}{"textField1", "textField2"}). - addSetting("imageFields", []interface{}{"imageField1"}). - addWeights([]interface{}{1, 2}, []interface{}{1}). - build(), - }, - }, - { - name: "should pass with proper value in 2 imageFields and 2 textFields and weights", - fields: fields{ - cfg: newConfigBuilder(). - addSetting("textFields", []interface{}{"textField1", "textField2"}). - addSetting("imageFields", []interface{}{"imageField1"}). - addWeights([]interface{}{1, 2}, []interface{}{1}). - build(), - }, - }, - { - name: "should not pass with proper value in 1 imageFields and 2 textFields and weights", - fields: fields{ - cfg: newConfigBuilder(). - addSetting("textFields", []interface{}{"textField1", "textField2"}). - addSetting("imageFields", []interface{}{"imageField1"}). - addWeights([]interface{}{1}, []interface{}{1}). - build(), - }, - wantErr: true, - }, - { - name: "should not pass with not proper weight value in 2 imageFields and 2 textFields and weights", - fields: fields{ - cfg: newConfigBuilder(). - addSetting("textFields", []interface{}{"textField1", "textField2"}). - addSetting("imageFields", []interface{}{"imageField1"}). - addWeights([]interface{}{1, "aaaa"}, []interface{}{1}). - build(), - }, - wantErr: true, - }, - { - name: "should not pass with not proper weight value in 2 imageFields and 2 textFields and weights", - fields: fields{ - cfg: newConfigBuilder(). - addSetting("textFields", []interface{}{"textField1", "textField2"}). - addSetting("imageFields", []interface{}{"imageField1"}). - addWeights([]interface{}{json.Number("1"), json.Number("2")}, []interface{}{json.Number("3")}). - build(), - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ic := &classSettings{ - cfg: tt.fields.cfg, - } - if err := ic.Validate(); (err != nil) != tt.wantErr { - t.Errorf("classSettings.Validate() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} diff --git a/modules/multi2vec-clip/vectorizer/fakes_for_test.go b/modules/multi2vec-clip/vectorizer/fakes_for_test.go deleted file mode 100644 index 128a57b3fe11eaf78627fade4c5505fc269ff56f..0000000000000000000000000000000000000000 --- a/modules/multi2vec-clip/vectorizer/fakes_for_test.go +++ /dev/null @@ -1,83 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - - "github.com/weaviate/weaviate/modules/multi2vec-clip/ent" -) - -type builder struct { - fakeClassConfig *fakeClassConfig -} - -func newConfigBuilder() *builder { - return &builder{ - fakeClassConfig: &fakeClassConfig{config: map[string]interface{}{}}, - } -} - -func (b *builder) addSetting(name string, value interface{}) *builder { - b.fakeClassConfig.config[name] = value - return b -} - -func (b *builder) addWeights(textWeights, imageWeights []interface{}) *builder { - if textWeights != nil || imageWeights != nil { - weightSettings := map[string]interface{}{} - if textWeights != nil { - weightSettings["textFields"] = textWeights - } - if imageWeights != nil { - weightSettings["imageFields"] = imageWeights - } - b.fakeClassConfig.config["weights"] = weightSettings - } - return b -} - -func (b *builder) build() *fakeClassConfig { - return b.fakeClassConfig -} - -type fakeClassConfig struct { - config map[string]interface{} -} - -func (c fakeClassConfig) Class() map[string]interface{} { - return c.config -} - -func (c fakeClassConfig) ClassByModuleName(moduleName string) map[string]interface{} { - return c.config -} - -func (c fakeClassConfig) Property(propName string) map[string]interface{} { - return c.config -} - -func (f fakeClassConfig) Tenant() string { - return "" -} - -type fakeClient struct{} - -func (c *fakeClient) Vectorize(ctx context.Context, - texts, images []string, -) (*ent.VectorizationResult, error) { - result := &ent.VectorizationResult{ - TextVectors: [][]float32{{1.0, 2.0, 3.0, 4.0, 5.0}}, - ImageVectors: [][]float32{{10.0, 20.0, 30.0, 40.0, 50.0}}, - } - return result, nil -} diff --git a/modules/multi2vec-clip/vectorizer/texts.go b/modules/multi2vec-clip/vectorizer/texts.go deleted file mode 100644 index ce46ddf94c0c169d7d62d2fd3385afc1bad4934e..0000000000000000000000000000000000000000 --- a/modules/multi2vec-clip/vectorizer/texts.go +++ /dev/null @@ -1,31 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - - "github.com/pkg/errors" -) - -func (v *Vectorizer) Texts(ctx context.Context, inputs []string, - settings ClassSettings, -) ([]float32, error) { - res, err := v.client.Vectorize(ctx, inputs, []string{}) - if err != nil { - return nil, errors.Wrap(err, "remote client vectorize") - } - if len(inputs) != len(res.TextVectors) { - return nil, errors.New("inputs are not equal to vectors returned") - } - return v.CombineVectors(res.TextVectors), nil -} diff --git a/modules/multi2vec-clip/vectorizer/todo_move_out_of_here_movements.go b/modules/multi2vec-clip/vectorizer/todo_move_out_of_here_movements.go deleted file mode 100644 index b7c554ae86e9338d31c79eb28f4a679c5ce95f93..0000000000000000000000000000000000000000 --- a/modules/multi2vec-clip/vectorizer/todo_move_out_of_here_movements.go +++ /dev/null @@ -1,68 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "fmt" - - libvectorizer "github.com/weaviate/weaviate/usecases/vectorizer" -) - -// CombineVectors combines all of the vector into sum of their parts -func (v *Vectorizer) CombineVectors(vectors [][]float32) []float32 { - return libvectorizer.CombineVectors(vectors) -} - -// MoveTo moves one vector toward another -func (v *Vectorizer) MoveTo(source []float32, target []float32, weight float32, -) ([]float32, error) { - multiplier := float32(0.5) - - if len(source) != len(target) { - return nil, fmt.Errorf("movement: vector lengths don't match: got %d and %d", - len(source), len(target)) - } - - if weight < 0 || weight > 1 { - return nil, fmt.Errorf("movement: force must be between 0 and 1: got %f", - weight) - } - - out := make([]float32, len(source)) - for i, sourceItem := range source { - out[i] = sourceItem*(1-weight*multiplier) + target[i]*(weight*multiplier) - } - - return out, nil -} - -// MoveAwayFrom moves one vector away from another -func (v *Vectorizer) MoveAwayFrom(source []float32, target []float32, weight float32, -) ([]float32, error) { - multiplier := float32(0.5) // so the movement is fair in comparison with moveTo - if len(source) != len(target) { - return nil, fmt.Errorf("movement (moveAwayFrom): vector lengths don't match: "+ - "got %d and %d", len(source), len(target)) - } - - if weight < 0 { - return nil, fmt.Errorf("movement (moveAwayFrom): force must be 0 or positive: "+ - "got %f", weight) - } - - out := make([]float32, len(source)) - for i, sourceItem := range source { - out[i] = sourceItem + weight*multiplier*(sourceItem-target[i]) - } - - return out, nil -} diff --git a/modules/multi2vec-clip/vectorizer/vectorizer.go b/modules/multi2vec-clip/vectorizer/vectorizer.go deleted file mode 100644 index 00d793fa5824c83f12504c1c3bd114fc7a86676b..0000000000000000000000000000000000000000 --- a/modules/multi2vec-clip/vectorizer/vectorizer.go +++ /dev/null @@ -1,164 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - - "github.com/pkg/errors" - - "github.com/go-openapi/strfmt" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/modules/multi2vec-clip/ent" - libvectorizer "github.com/weaviate/weaviate/usecases/vectorizer" -) - -type Vectorizer struct { - client Client -} - -func New(client Client) *Vectorizer { - return &Vectorizer{ - client: client, - } -} - -type Client interface { - Vectorize(ctx context.Context, - texts, images []string) (*ent.VectorizationResult, error) -} - -type ClassSettings interface { - ImageField(property string) bool - ImageFieldsWeights() ([]float32, error) - TextField(property string) bool - TextFieldsWeights() ([]float32, error) -} - -func (v *Vectorizer) Object(ctx context.Context, object *models.Object, - objDiff *moduletools.ObjectDiff, settings ClassSettings, -) error { - vec, err := v.object(ctx, object.ID, object.Properties, objDiff, settings) - if err != nil { - return err - } - - object.Vector = vec - return nil -} - -func (v *Vectorizer) VectorizeImage(ctx context.Context, image string) ([]float32, error) { - res, err := v.client.Vectorize(ctx, []string{}, []string{image}) - if err != nil { - return nil, err - } - if len(res.ImageVectors) != 1 { - return nil, errors.New("empty vector") - } - - return res.ImageVectors[0], nil -} - -func (v *Vectorizer) object(ctx context.Context, id strfmt.UUID, - schema interface{}, objDiff *moduletools.ObjectDiff, ichek ClassSettings, -) ([]float32, error) { - vectorize := objDiff == nil || objDiff.GetVec() == nil - - // vectorize image and text - texts := []string{} - images := []string{} - if schema != nil { - for prop, value := range schema.(map[string]interface{}) { - if ichek.ImageField(prop) { - valueString, ok := value.(string) - if ok { - images = append(images, valueString) - vectorize = vectorize || (objDiff != nil && objDiff.IsChangedProp(prop)) - } - } - if ichek.TextField(prop) { - valueString, ok := value.(string) - if ok { - texts = append(texts, valueString) - vectorize = vectorize || (objDiff != nil && objDiff.IsChangedProp(prop)) - } - } - valueArr, ok := value.([]interface{}) - if ok { - for _, value := range valueArr { - valueString, ok := value.(string) - if ok { - texts = append(texts, valueString) - vectorize = vectorize || (objDiff != nil && objDiff.IsChangedProp(prop)) - } - } - } - } - } - - // no property was changed, old vector can be used - if !vectorize { - return objDiff.GetVec(), nil - } - - vectors := [][]float32{} - if len(texts) > 0 || len(images) > 0 { - res, err := v.client.Vectorize(ctx, texts, images) - if err != nil { - return nil, err - } - vectors = append(vectors, res.TextVectors...) - vectors = append(vectors, res.ImageVectors...) - } - weights, err := v.getWeights(ichek) - if err != nil { - return nil, err - } - - return libvectorizer.CombineVectorsWithWeights(vectors, weights), nil -} - -func (v *Vectorizer) getWeights(ichek ClassSettings) ([]float32, error) { - weights := []float32{} - textFieldsWeights, err := ichek.TextFieldsWeights() - if err != nil { - return nil, err - } - imageFieldsWeights, err := ichek.ImageFieldsWeights() - if err != nil { - return nil, err - } - - weights = append(weights, textFieldsWeights...) - weights = append(weights, imageFieldsWeights...) - - normalizedWeights := v.normalizeWeights(weights) - - return normalizedWeights, nil -} - -func (v *Vectorizer) normalizeWeights(weights []float32) []float32 { - if len(weights) > 0 { - var denominator float32 - for i := range weights { - denominator += weights[i] - } - normalizer := 1 / denominator - normalized := make([]float32, len(weights)) - for i := range weights { - normalized[i] = weights[i] * normalizer - } - return normalized - } - return nil -} diff --git a/modules/multi2vec-clip/vectorizer/vectorizer_test.go b/modules/multi2vec-clip/vectorizer/vectorizer_test.go deleted file mode 100644 index ddadc1562562c189e9e18330dbed47cb56152517..0000000000000000000000000000000000000000 --- a/modules/multi2vec-clip/vectorizer/vectorizer_test.go +++ /dev/null @@ -1,250 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/moduletools" -) - -const image = "iVBORw0KGgoAAAANSUhEUgAAAGAAAAA/CAYAAAAfQM0aAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAyRpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuMy1jMDExIDY2LjE0NTY2MSwgMjAxMi8wMi8wNi0xNDo1NjoyNyAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvIiB4bWxuczp4bXBNTT0iaHR0cDovL25zLmFkb2JlLmNvbS94YXAvMS4wL21tLyIgeG1sbnM6c3RSZWY9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9zVHlwZS9SZXNvdXJjZVJlZiMiIHhtcDpDcmVhdG9yVG9vbD0iQWRvYmUgUGhvdG9zaG9wIENTNiAoTWFjaW50b3NoKSIgeG1wTU06SW5zdGFuY2VJRD0ieG1wLmlpZDpCRjQ5NEM3RDI5QTkxMUUyOTc1NENCMzI4N0QwNDNCOSIgeG1wTU06RG9jdW1lbnRJRD0ieG1wLmRpZDpCRjQ5NEM3RTI5QTkxMUUyOTc1NENCMzI4N0QwNDNCOSI+IDx4bXBNTTpEZXJpdmVkRnJvbSBzdFJlZjppbnN0YW5jZUlEPSJ4bXAuaWlkOkJGNDk0QzdCMjlBOTExRTI5NzU0Q0IzMjg3RDA0M0I5IiBzdFJlZjpkb2N1bWVudElEPSJ4bXAuZGlkOkJGNDk0QzdDMjlBOTExRTI5NzU0Q0IzMjg3RDA0M0I5Ii8+IDwvcmRmOkRlc2NyaXB0aW9uPiA8L3JkZjpSREY+IDwveDp4bXBtZXRhPiA8P3hwYWNrZXQgZW5kPSJyIj8+WeGRxAAAB2hJREFUeNrUXFtslUUQ3hJCoQVEKy0k1qQgrRg0vaAJaq1tvJSgaLy8mKDF2IvxBY2Bgm8+iIoxvhB72tTUmKgPigbFKCEtxeKD9hZjAi3GJrYJtqRai7TQB+pMz/zwU/5zzsxe2u4kXwiwZ+bb/Xb/s7v/zEmrra1VTFsFeBRQCtgEuBWwkv5vHPAn4DdAB+B7wBjXcUNDQ8o2dXV1SmDzyhUtLS3tBPyxC9CdrN1ihi/swKuA7YD0BG1uJhQDngdcAnwDeJ86Ole2kLii+J2AFsA+wF9RjRalmEUHaZY8m6RDUYZtn6HPHiRfLm2hck0D7AScAdRH8UokwD2AnwA7UoiUyhaRD/S12dHg+8B1OWA/4BTgqVQCPEJL8haLBNDXEfJt03ziipYH+BJwHFAYJcAWwCeAZQ6CLyPfWyz584nrbCuj74eHwgKsddih2R1ba+jHJ65R1k6PuWNhAd4DZM/BTiWbdhwm5hPXsA0AngY8COgNP4JwSTyu4zE/P18VFhZKP7aNYuouXxFX5Ic8Nc2Ea2D/AfYCNgIORZ0DdusOfnFxcXDwUD09PZKP76alKDUR16KiIlVQUHDl7/39/Uozpg7Xac45YB0dGrQHHw07KVwJpRRbYiKuyCc8+MhXcyXocP2RnvMvJhr8QIBK08EPbGJiQuqq0mX7KD4GIohi4xVPTU0N6/BRamPwu7u7dZb3/RozkW3IB3lZEkGHayeI8FFVVdWaZAIUcD2Wl5fbHHy024XtC6QBkomA/XHIFb8X0Xamp6efASHqt27dGnkVkcNxVlFRoXJycmwOvuLGNmifVATsD/bLZezgKgKE2J+bm3sKHk3XXUWs4Mz87Oxs24OvOLEN26cUAfvFXAkrlKGBCDNXEbAajldXV1+5ijjP+KCrg855x+3nk2uy8SwDdIIIM1cRI6k+0NraqkZGRmzuKAIbFrYf0Q2UaPOA/Wpra3PBNfHhYHq6HbC5qanpGB7ETgPWc0TApTr7eyDolOaj6LRG+/W2Bn94eJg7+DpcowZ+AGb+642NjYfC3wEdXAdI1uK2Du2ksH2HrcHHfggGX4frNVcRMPh7BwcHN8ZiseuuIr4DvKXib29YX2bhmW+wEqYptsREXC2eWXS44oyfuYqYmpra19LSEnkaRgEG6Nj8gGRHESVCRkaG9Kg+IOyTiGtmZqatnZsOV/zMLnjcsF7KH5AIECVCX1+f6u3tlbg4oLmc2VyDy8HgPshg2yzmCo8aFsdAALzpw9dw23REwJkvHPwjSu92UcwVRcAnAd4LaQ6+CVe2AGivAe5WwhcdGp0aoVgmJuIqnBy2uSa18Buxs4AXAJMO401SjLOGfnziyhYg2GrtcNSxSfJ90pI/n7iyBUA7quKv/IYsxhmiZ/ZRy/x94soWAO1nwL0qnhVw2cD/ZfKBvjod9cEnrmwB0DBh9RUVfxHxhYrnUHLtEn2mlHyMOe6HT1wT7oISGSas4ntNzJmsVFczjnMBN1CbfwGD1BYPID8A/lFzbz5xZQsQnmWfExa6ecNVIsBKWuIlgA0qnjG2PLhsou0aZgF3qfil2fg89ssbrhwBNtB+GN/dLUnQ5kbCHYAnAFMAvGpsoY7OlS0krmOhxx7WLHwAeBLwVahN2uIUswgrPB5T8rRv7DxWqDwM+JaCjzue8b5wZe2C7gJ8quKVJqY599vJ1yZHffCJK0uA+wAfAtZYjIO+Gsi3TfOJK0sAfFP/jpKV+HBtKfkutOTPJ64sAVYD3qXgrmwpxVht6McnrmwBMAP4pjlYdRij3tCHT1xZAuDdermOA836gDKKqWNirob1ASZc2eeAl3QH36A+AGP+ohFWxNVSfYAuV9YKyKUTo/bgo2nUB5RQbImJuFqsD9DhyhbAuDgjMI36gFKX7S3XB5S6egSV2Bh8zYyDYjr4SGYi2yzmMIm5YnFGkFOLSQGNjY3X/BtaLBabWQF5XKcO6gOkZT950gAW6wPWuXoEZXEaOqoPyHLcPqkIwvqALFcCZHJmvqP6gEzH7VOKIKgPyHQlwIVUjRzWB1xw3H4+ubIFGE3VyGF9wKjj9ik3D4L6gFFXArCSTlEEzKe3LMIfwvYDNgcf+4P9csSVLUAXt7GD+oBuYfsuW4OvUR/Q7UoA/G2zaRvbOqEI0xRbYiKulusDTrgSYEg6sxKJIKwP6FLyjDYRV4v1ATpc2QKgNZtu6zTqA5o1ObM/h5eDyMvCtrlZObLgNhRv+jAHvkwqQjDzhYPfrvRvF0VcLdQHaHGNxWKrZv0d//hahcqr8Ccww1kRbwPuVMIXHRqd+ptimZiIq0F9gA2urEcQ2jkVf/tz0WG8ixTjnKEfn7iyBQi2WnuULLlV0qE9FrdzPnFlC4CGRQkvqyQ/MqRh6KtO2S948IkrWwC0XwHPAQ4r85z7w+TL1U8Y+8Q14S4oyjA9703AZ4AqFX8RvoTpN8i3/Bi/p+egHz5xZQsQGCasvqGuZhzj76DdpuIZx8FPuOAviWDG8e8qXl0yXxnHPnGdsf8FGAByGwC02iMZswAAAABJRU5ErkJggg==" - -func TestVectorizer(t *testing.T) { - t.Run("should vectorize image", func(t *testing.T) { - // given - client := &fakeClient{} - vectorizer := &Vectorizer{client} - config := newConfigBuilder().addSetting("imageFields", []interface{}{"image"}).build() - settings := NewClassSettings(config) - object := &models.Object{ - ID: "some-uuid", - Properties: map[string]interface{}{ - "image": image, - }, - } - - // when - err := vectorizer.Object(context.Background(), object, nil, settings) - - // then - require.Nil(t, err) - assert.NotNil(t, object.Vector) - }) - - t.Run("should vectorize 2 image fields", func(t *testing.T) { - // given - client := &fakeClient{} - vectorizer := &Vectorizer{client} - config := newConfigBuilder().addSetting("imageFields", []interface{}{"image1", "image2"}).build() - settings := NewClassSettings(config) - object := &models.Object{ - ID: "some-uuid", - Properties: map[string]interface{}{ - "image1": image, - "image2": image, - }, - } - - // when - err := vectorizer.Object(context.Background(), object, nil, settings) - - // then - require.Nil(t, err) - assert.NotNil(t, object.Vector) - }) -} - -func TestVectorizerWithDiff(t *testing.T) { - type testCase struct { - name string - input *models.Object - diff *moduletools.ObjectDiff - expectedVectorize bool - } - - tests := []testCase{ - { - name: "no diff", - input: &models.Object{ - ID: "some-uuid", - Properties: map[string]interface{}{ - "image": image, - "text": "text", - "description": "non-vectorizable", - }, - }, - diff: nil, - expectedVectorize: true, - }, - { - name: "diff all props unchanged", - input: &models.Object{ - ID: "some-uuid", - Properties: map[string]interface{}{ - "image": image, - "text": "text", - "description": "non-vectorizable", - }, - }, - diff: newObjectDiffWithVector(). - WithProp("image", image, image). - WithProp("text", "text", "text"). - WithProp("description", "non-vectorizable", "non-vectorizable"), - expectedVectorize: false, - }, - { - name: "diff one vectorizable prop changed (1)", - input: &models.Object{ - ID: "some-uuid", - Properties: map[string]interface{}{ - "image": image, - "text": "text", - "description": "non-vectorizable", - }, - }, - diff: newObjectDiffWithVector(). - WithProp("image", "", image), - expectedVectorize: true, - }, - { - name: "diff one vectorizable prop changed (2)", - input: &models.Object{ - ID: "some-uuid", - Properties: map[string]interface{}{ - "image": image, - "text": "text", - "description": "non-vectorizable", - }, - }, - diff: newObjectDiffWithVector(). - WithProp("text", "old text", "text"), - expectedVectorize: true, - }, - { - name: "all non-vectorizable props changed", - input: &models.Object{ - ID: "some-uuid", - Properties: map[string]interface{}{ - "image": image, - "text": "text", - "description": "non-vectorizable", - }, - }, - diff: newObjectDiffWithVector(). - WithProp("description", "old non-vectorizable", "non-vectorizable"), - expectedVectorize: false, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - client := &fakeClient{} - vectorizer := &Vectorizer{client} - config := newConfigBuilder(). - addSetting("imageFields", []interface{}{"image"}). - addSetting("textFields", []interface{}{"text"}). - build() - settings := NewClassSettings(config) - - err := vectorizer.Object(context.Background(), test.input, test.diff, settings) - - require.Nil(t, err) - if test.expectedVectorize { - assert.Equal(t, models.C11yVector{5.5, 11, 16.5, 22, 27.5}, test.input.Vector) - // vectors are defined in Vectorize within fakes_for_test.go - // result calculated without weights as (textVectors[0][i]+imageVectors[0][i]) / 2 - } else { - assert.Equal(t, models.C11yVector{0, 0, 0, 0, 0}, test.input.Vector) - } - }) - } -} - -func TestVectorizerWithWeights(t *testing.T) { - client := &fakeClient{} - vectorizer := &Vectorizer{client} - config := newConfigBuilder(). - addSetting("imageFields", []interface{}{"image"}). - addSetting("textFields", []interface{}{"text"}). - addWeights([]interface{}{0.4}, []interface{}{0.6}). - build() - settings := NewClassSettings(config) - - input := &models.Object{ - ID: "some-uuid", - Properties: map[string]interface{}{ - "image": image, - "text": "text", - "description": "non-vectorizable", - }, - } - - err := vectorizer.Object(context.Background(), input, nil, settings) - - require.Nil(t, err) - assert.Equal(t, models.C11yVector{3.2, 6.4, 9.6, 12.8, 16}, input.Vector) - // vectors are defined in Vectorize within fakes_for_test.go - // result calculated with above weights as (textVectors[0][i]*0.4+imageVectors[0][i]*0.6) / 2 -} - -func TestVectorizer_normalizeWeights(t *testing.T) { - tests := []struct { - name string - weights []float32 - }{ - { - name: "normalize example 1", - weights: []float32{200, 100, 0.1}, - }, - { - name: "normalize example 2", - weights: []float32{300.22, 0.7, 17, 54}, - }, - { - name: "normalize example 3", - weights: []float32{300, 0.02, 17}, - }, - { - name: "normalize example 4", - weights: []float32{500, 0.02, 17.4, 180}, - }, - { - name: "normalize example 5", - weights: []float32{500, 0.02, 17.4, 2, 4, 5, .88}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - v := &Vectorizer{} - if got := v.normalizeWeights(tt.weights); !checkNormalization(got) { - t.Errorf("Vectorizer.normalizeWeights() = %v, want %v", got, 1.0) - } - }) - } -} - -func checkNormalization(weights []float32) bool { - var result float32 - for i := range weights { - result += weights[i] - } - return result == 1.0 -} - -func newObjectDiffWithVector() *moduletools.ObjectDiff { - return moduletools.NewObjectDiff([]float32{0, 0, 0, 0, 0}) -} diff --git a/modules/ner-transformers/additional/models/models.go b/modules/ner-transformers/additional/models/models.go deleted file mode 100644 index 3ff689d21b9d7c5a41c55423fd69433cb2b6c7fa..0000000000000000000000000000000000000000 --- a/modules/ner-transformers/additional/models/models.go +++ /dev/null @@ -1,23 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package models - -// Tokens used in NER module to represent -// the found entities in a given string property value -type Token struct { - Property string `json:"property,omitempty"` - Entity string `json:"entity,omitempty"` - Certainty float64 `json:"certainty,omitempty"` - Word string `json:"word,omitempty"` - StartPosition int `json:"startPosition,omitempty"` - EndPosition int `json:"endPosition,omitempty"` -} diff --git a/modules/ner-transformers/additional/provider.go b/modules/ner-transformers/additional/provider.go deleted file mode 100644 index 7527154024a4cba54fa54d4d68ac0e86ec6c3c86..0000000000000000000000000000000000000000 --- a/modules/ner-transformers/additional/provider.go +++ /dev/null @@ -1,58 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package additional - -import ( - "context" - - "github.com/weaviate/weaviate/entities/moduletools" - - "github.com/tailor-inc/graphql" - "github.com/tailor-inc/graphql/language/ast" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/search" -) - -type AdditionalProperty interface { - AdditionalPropertyFn(ctx context.Context, - in []search.Result, params interface{}, limit *int, - argumentModuleParams map[string]interface{}, cfg moduletools.ClassConfig) ([]search.Result, error) - ExtractAdditionalFn(param []*ast.Argument) interface{} - AdditionalPropertyDefaultValue() interface{} - AdditionalFieldFn(classname string) *graphql.Field -} - -type GraphQLAdditionalArgumentsProvider struct { - tokensProvider AdditionalProperty -} - -func New(tokensProvider AdditionalProperty) *GraphQLAdditionalArgumentsProvider { - return &GraphQLAdditionalArgumentsProvider{tokensProvider} -} - -func (p *GraphQLAdditionalArgumentsProvider) AdditionalProperties() map[string]modulecapabilities.AdditionalProperty { - additionalProperties := map[string]modulecapabilities.AdditionalProperty{} - additionalProperties["tokens"] = p.getTokens() - return additionalProperties -} - -func (p *GraphQLAdditionalArgumentsProvider) getTokens() modulecapabilities.AdditionalProperty { - return modulecapabilities.AdditionalProperty{ - GraphQLNames: []string{"tokens"}, - GraphQLFieldFunction: p.tokensProvider.AdditionalFieldFn, - GraphQLExtractFunction: p.tokensProvider.ExtractAdditionalFn, - SearchFunctions: modulecapabilities.AdditionalSearch{ - ExploreGet: p.tokensProvider.AdditionalPropertyFn, - ExploreList: p.tokensProvider.AdditionalPropertyFn, - }, - } -} diff --git a/modules/ner-transformers/additional/tokens/tokens.go b/modules/ner-transformers/additional/tokens/tokens.go deleted file mode 100644 index b9ff773cf39656300c23784f9688a096fe4025d2..0000000000000000000000000000000000000000 --- a/modules/ner-transformers/additional/tokens/tokens.go +++ /dev/null @@ -1,58 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package tokens - -import ( - "context" - "errors" - - "github.com/weaviate/weaviate/entities/moduletools" - - "github.com/tailor-inc/graphql" - "github.com/tailor-inc/graphql/language/ast" - "github.com/weaviate/weaviate/entities/search" - "github.com/weaviate/weaviate/modules/ner-transformers/ent" -) - -type nerClient interface { - GetTokens(ctx context.Context, property, text string) ([]ent.TokenResult, error) -} - -type TokenProvider struct { - ner nerClient -} - -func New(ner nerClient) *TokenProvider { - return &TokenProvider{ner} -} - -func (p *TokenProvider) AdditionalPropertyDefaultValue() interface{} { - return &Params{} -} - -func (p *TokenProvider) ExtractAdditionalFn(param []*ast.Argument) interface{} { - return p.parseTokenArguments(param) -} - -func (p *TokenProvider) AdditionalFieldFn(classname string) *graphql.Field { - return p.additionalTokensField(classname) -} - -func (p *TokenProvider) AdditionalPropertyFn(ctx context.Context, - in []search.Result, params interface{}, limit *int, - argumentModuleParams map[string]interface{}, cfg moduletools.ClassConfig, -) ([]search.Result, error) { - if parameters, ok := params.(*Params); ok { - return p.findTokens(ctx, in, parameters) - } - return nil, errors.New("wrong parameters") -} diff --git a/modules/ner-transformers/additional/tokens/tokens_graphql_field.go b/modules/ner-transformers/additional/tokens/tokens_graphql_field.go deleted file mode 100644 index 4f26e7da881f741a991a440d4063eb8821133aeb..0000000000000000000000000000000000000000 --- a/modules/ner-transformers/additional/tokens/tokens_graphql_field.go +++ /dev/null @@ -1,58 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package tokens - -import ( - "fmt" - - "github.com/tailor-inc/graphql" - "github.com/weaviate/weaviate/adapters/handlers/graphql/descriptions" -) - -func (p *TokenProvider) additionalTokensField(classname string) *graphql.Field { - return &graphql.Field{ - Args: graphql.FieldConfigArgument{ - "properties": &graphql.ArgumentConfig{ - Description: "Properties which contains text", - Type: graphql.NewList(graphql.String), - DefaultValue: nil, - }, - "certainty": &graphql.ArgumentConfig{ - Description: descriptions.Certainty, - Type: graphql.Float, - DefaultValue: nil, - }, - "distance": &graphql.ArgumentConfig{ - Description: descriptions.Distance, - Type: graphql.Float, - DefaultValue: nil, - }, - "limit": &graphql.ArgumentConfig{ - Type: graphql.Int, - Description: descriptions.Limit, - DefaultValue: nil, - }, - }, - Type: graphql.NewList(graphql.NewObject(graphql.ObjectConfig{ - Name: fmt.Sprintf("%sAdditionalTokens", classname), - Fields: graphql.Fields{ - "property": &graphql.Field{Type: graphql.String}, - "entity": &graphql.Field{Type: graphql.String}, - "certainty": &graphql.Field{Type: graphql.Float}, - "distance": &graphql.Field{Type: graphql.Float}, - "word": &graphql.Field{Type: graphql.String}, - "startPosition": &graphql.Field{Type: graphql.Int}, - "endPosition": &graphql.Field{Type: graphql.Int}, - }, - })), - } -} diff --git a/modules/ner-transformers/additional/tokens/tokens_graphql_field_test.go b/modules/ner-transformers/additional/tokens/tokens_graphql_field_test.go deleted file mode 100644 index f81681165f138a5fc0c1b9b539d6f964fba5993a..0000000000000000000000000000000000000000 --- a/modules/ner-transformers/additional/tokens/tokens_graphql_field_test.go +++ /dev/null @@ -1,69 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package tokens - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/tailor-inc/graphql" -) - -func Test_additionalTokensField(t *testing.T) { - // given - tokenProvider := &TokenProvider{} - classname := "Class" - - // when - tokens := tokenProvider.additionalTokensField(classname) - - // then - // the built graphQL field needs to support this structure: - // Args: { - // "properties": ["summary"], - // "limit": 1, - // "distance": 0.7 - // } - // Type: { - // tokens: { - // "property": "summary", - // "entity": "I-PER", - // "distance": 0.8, - // "word": "original word", - // "startPosition": 1, - // "endPosition": 2, - // } - // } - - assert.NotNil(t, tokens) - assert.Equal(t, "ClassAdditionalTokens", tokens.Type.Name()) - assert.NotNil(t, tokens.Type) - tokensObjectList, tokensObjectListOK := tokens.Type.(*graphql.List) - assert.True(t, tokensObjectListOK) - tokensObject, tokensObjectOK := tokensObjectList.OfType.(*graphql.Object) - assert.True(t, tokensObjectOK) - assert.Equal(t, 7, len(tokensObject.Fields())) - assert.NotNil(t, tokensObject.Fields()["property"]) - assert.NotNil(t, tokensObject.Fields()["entity"]) - assert.NotNil(t, tokensObject.Fields()["certainty"]) - assert.NotNil(t, tokensObject.Fields()["distance"]) - assert.NotNil(t, tokensObject.Fields()["word"]) - assert.NotNil(t, tokensObject.Fields()["startPosition"]) - assert.NotNil(t, tokensObject.Fields()["endPosition"]) - - assert.NotNil(t, tokens.Args) - assert.Equal(t, 4, len(tokens.Args)) - assert.NotNil(t, tokens.Args["certainty"]) - assert.NotNil(t, tokens.Args["distance"]) - assert.NotNil(t, tokens.Args["limit"]) - assert.NotNil(t, tokens.Args["properties"]) -} diff --git a/modules/ner-transformers/additional/tokens/tokens_params.go b/modules/ner-transformers/additional/tokens/tokens_params.go deleted file mode 100644 index 28c81131792b80becf16457b2a48b1ce1ab55f4a..0000000000000000000000000000000000000000 --- a/modules/ner-transformers/additional/tokens/tokens_params.go +++ /dev/null @@ -1,31 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package tokens - -type Params struct { - Limit *int // optional parameter - Certainty *float64 // optional parameter - Distance *float64 // optional parameter - Properties []string -} - -func (n Params) GetCertainty() *float64 { - return n.Certainty -} - -func (n Params) GetLimit() *int { - return n.Limit -} - -func (n Params) GetProperties() []string { - return n.Properties -} diff --git a/modules/ner-transformers/additional/tokens/tokens_params_extractor.go b/modules/ner-transformers/additional/tokens/tokens_params_extractor.go deleted file mode 100644 index afc9e523435a233ac1a23382854898203c39ece4..0000000000000000000000000000000000000000 --- a/modules/ner-transformers/additional/tokens/tokens_params_extractor.go +++ /dev/null @@ -1,52 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package tokens - -import ( - "strconv" - - "github.com/tailor-inc/graphql/language/ast" -) - -func (p *TokenProvider) parseTokenArguments(args []*ast.Argument) *Params { - out := &Params{} - - for _, arg := range args { - switch arg.Name.Value { - case "limit": - asInt, _ := strconv.Atoi(arg.Value.GetValue().(string)) - out.Limit = ptInt(asInt) - case "certainty": - asFloat, _ := strconv.ParseFloat(arg.Value.GetValue().(string), 64) - out.Certainty = &asFloat - case "distance": - asFloat, _ := strconv.ParseFloat(arg.Value.GetValue().(string), 64) - out.Distance = &asFloat - case "properties": - inp := arg.Value.GetValue().([]ast.Value) - out.Properties = make([]string, len(inp)) - - for i, value := range inp { - out.Properties[i] = value.(*ast.StringValue).Value - } - - default: - // ignore what we don't recognize - } - } - - return out -} - -func ptInt(in int) *int { - return &in -} diff --git a/modules/ner-transformers/additional/tokens/tokens_params_extractor_test.go b/modules/ner-transformers/additional/tokens/tokens_params_extractor_test.go deleted file mode 100644 index 80db7f64885bbc31548d99d42406b9c3ef75dfa9..0000000000000000000000000000000000000000 --- a/modules/ner-transformers/additional/tokens/tokens_params_extractor_test.go +++ /dev/null @@ -1,123 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package tokens - -import ( - "reflect" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/tailor-inc/graphql/language/ast" -) - -func Test_parseTokenArguments(t *testing.T) { - type args struct { - args []*ast.Argument - } - tests := []struct { - name string - args args - want *Params - }{ - { - name: "Should create with no params", - args: args{}, - want: &Params{}, - }, - { - name: "Should create with all params (and distance)", - args: args{ - args: []*ast.Argument{ - createArg("distance", "0.4"), - createArg("limit", "1"), - createListArg("properties", []string{"prop1", "prop2"}), - }, - }, - want: &Params{ - Properties: []string{"prop1", "prop2"}, - Distance: ptFloat64(0.4), - Limit: ptInt(1), - }, - }, - { - name: "Should create with all params (and certainty)", - args: args{ - args: []*ast.Argument{ - createArg("certainty", "0.4"), - createArg("limit", "1"), - createListArg("properties", []string{"prop1", "prop2"}), - }, - }, - want: &Params{ - Properties: []string{"prop1", "prop2"}, - Certainty: ptFloat64(0.4), - Limit: ptInt(1), - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - p := &TokenProvider{} - if got := p.parseTokenArguments(tt.args.args); !reflect.DeepEqual(got, tt.want) { - t.Errorf("parseTokenArguments() = %v, want %v", got, tt.want) - } - actual := p.parseTokenArguments(tt.args.args) - assert.Equal(t, tt.want, actual) - }) - } -} - -func createArg(name string, value string) *ast.Argument { - n := ast.Name{ - Value: name, - } - val := ast.StringValue{ - Kind: "Kind", - Value: value, - } - arg := ast.Argument{ - Name: ast.NewName(&n), - Kind: "Kind", - Value: ast.NewStringValue(&val), - } - a := ast.NewArgument(&arg) - return a -} - -func createListArg(name string, valuesIn []string) *ast.Argument { - n := ast.Name{ - Value: name, - } - - valuesAst := make([]ast.Value, len(valuesIn)) - for i, value := range valuesIn { - valuesAst[i] = &ast.StringValue{ - Kind: "Kind", - Value: value, - } - } - vals := ast.ListValue{ - Kind: "Kind", - Values: valuesAst, - } - arg := ast.Argument{ - Name: ast.NewName(&n), - Kind: "Kind", - Value: &vals, - } - a := ast.NewArgument(&arg) - return a -} - -func ptFloat64(in float64) *float64 { - return &in -} diff --git a/modules/ner-transformers/additional/tokens/tokens_result.go b/modules/ner-transformers/additional/tokens/tokens_result.go deleted file mode 100644 index 3842b59a985f7aab1ea31f9bd91324c36dfff589..0000000000000000000000000000000000000000 --- a/modules/ner-transformers/additional/tokens/tokens_result.go +++ /dev/null @@ -1,121 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package tokens - -import ( - "context" - "errors" - "fmt" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/search" - "github.com/weaviate/weaviate/modules/ner-transformers/ent" -) - -func (p *TokenProvider) findTokens(ctx context.Context, - in []search.Result, params *Params, -) ([]search.Result, error) { - if len(in) > 0 { - - if len(in) == 0 { - return nil, nil - } - - if params == nil { - return nil, fmt.Errorf("no params provided") - } - - properties := params.GetProperties() - - // check if user parameter values are valid - if len(properties) == 0 { - return in, errors.New("no properties provided") - } - - for i := range in { // for each result of the general GraphQL Query - ap := in[i].AdditionalProperties - if ap == nil { - ap = models.AdditionalProperties{} - } - - // check if the schema of the GraphQL data object contains the properties and they are text or string values - textProperties := map[string]string{} - schema := in[i].Object().Properties.(map[string]interface{}) - for property, value := range schema { - if p.containsProperty(property, properties) { - if valueString, ok := value.(string); ok && len(valueString) > 0 { - textProperties[property] = valueString - } - } - } - - certainty := params.GetCertainty() - limit := params.GetLimit() - tokensList := []ent.TokenResult{} - - // for each text property result, call the NER function and add to additional result - for property, value := range textProperties { - - if limit != nil && len(tokensList) > *limit { - break - } - - tokens, err := p.ner.GetTokens(ctx, property, value) - if err != nil { - return in, err - } - - tokens = cutOffByCertainty(tokens, certainty) - - tokensList = append(tokensList, tokens...) - } - - if limit != nil && len(tokensList) > *limit { - ap["tokens"] = tokensList[:*limit] - } else { - ap["tokens"] = tokensList - } - - in[i].AdditionalProperties = ap - } - } - return in, nil -} - -func cutOffByCertainty(tokens []ent.TokenResult, certainty *float64) []ent.TokenResult { - minCertainty := 0.0 - if certainty != nil { - minCertainty = *certainty - } - a := 0 - for _, x := range tokens { - if x.Certainty >= minCertainty { - tokens[a] = x - a++ - } - } - tokens = tokens[:a] - - return tokens -} - -func (p *TokenProvider) containsProperty(property string, properties []string) bool { - if len(properties) == 0 { - return true - } - for i := range properties { - if properties[i] == property { - return true - } - } - return false -} diff --git a/modules/ner-transformers/clients/ner.go b/modules/ner-transformers/clients/ner.go deleted file mode 100644 index 0b729d59ff348d203367219f106874b3bf6efc65..0000000000000000000000000000000000000000 --- a/modules/ner-transformers/clients/ner.go +++ /dev/null @@ -1,116 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/modules/ner-transformers/ent" -) - -type ner struct { - origin string - httpClient *http.Client - logger logrus.FieldLogger -} - -type nerInput struct { - Text string `json:"text"` -} - -type tokenResponse struct { - // Property string `json:"property"` - Entity string `json:"entity"` - Certainty float64 `json:"certainty"` - Distance float64 `json:"distance"` - Word string `json:"word"` - StartPosition int `json:"startPosition"` - EndPosition int `json:"endPosition"` -} - -type nerResponse struct { - Error string - nerInput - Tokens []tokenResponse `json:"tokens"` -} - -func New(origin string, timeout time.Duration, logger logrus.FieldLogger) *ner { - return &ner{ - origin: origin, - httpClient: &http.Client{Timeout: timeout}, - logger: logger, - } -} - -func (n *ner) GetTokens(ctx context.Context, property, - text string, -) ([]ent.TokenResult, error) { - body, err := json.Marshal(nerInput{ - Text: text, - }) - if err != nil { - return nil, errors.Wrapf(err, "marshal body") - } - - req, err := http.NewRequestWithContext(ctx, "POST", n.url("/ner/"), - bytes.NewReader(body)) - if err != nil { - return nil, errors.Wrap(err, "create POST request") - } - - res, err := n.httpClient.Do(req) - if err != nil { - return nil, errors.Wrap(err, "send POST request") - } - defer res.Body.Close() - - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - return nil, errors.Wrap(err, "read response body") - } - - var resBody nerResponse - if err := json.Unmarshal(bodyBytes, &resBody); err != nil { - return nil, errors.Wrap(err, "unmarshal response body") - } - - if res.StatusCode > 399 { - return nil, errors.Errorf("fail with status %d: %s", res.StatusCode, resBody.Error) - } - - out := make([]ent.TokenResult, len(resBody.Tokens)) - - for i, elem := range resBody.Tokens { - out[i].Certainty = elem.Certainty - out[i].Distance = elem.Distance - out[i].Entity = elem.Entity - out[i].Word = elem.Word - out[i].StartPosition = elem.StartPosition - out[i].EndPosition = elem.EndPosition - out[i].Property = property - } - - // format resBody to nerResult - return out, nil -} - -func (n *ner) url(path string) string { - return fmt.Sprintf("%s%s", n.origin, path) -} diff --git a/modules/ner-transformers/clients/ner_meta.go b/modules/ner-transformers/clients/ner_meta.go deleted file mode 100644 index 56444b4420c10e656b8bf6a07a2a57862acaa109..0000000000000000000000000000000000000000 --- a/modules/ner-transformers/clients/ner_meta.go +++ /dev/null @@ -1,45 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "context" - "encoding/json" - "io" - "net/http" - - "github.com/pkg/errors" -) - -func (n *ner) MetaInfo() (map[string]interface{}, error) { - req, err := http.NewRequestWithContext(context.Background(), "GET", n.url("/meta"), nil) - if err != nil { - return nil, errors.Wrap(err, "create GET meta request") - } - - res, err := n.httpClient.Do(req) - if err != nil { - return nil, errors.Wrap(err, "send GET meta request") - } - defer res.Body.Close() - - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - return nil, errors.Wrap(err, "read meta response body") - } - - var resBody map[string]interface{} - if err := json.Unmarshal(bodyBytes, &resBody); err != nil { - return nil, errors.Wrap(err, "unmarshal meta response body") - } - return resBody, nil -} diff --git a/modules/ner-transformers/clients/ner_meta_test.go b/modules/ner-transformers/clients/ner_meta_test.go deleted file mode 100644 index 72c97713bd9a2b63a2da39817c39f6cbb035079c..0000000000000000000000000000000000000000 --- a/modules/ner-transformers/clients/ner_meta_test.go +++ /dev/null @@ -1,156 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func TestGetMeta(t *testing.T) { - t.Run("when the server is providing meta", func(t *testing.T) { - server := httptest.NewServer(&testMetaHandler{t: t}) - defer server.Close() - c := New(server.URL, 0, nullLogger()) - meta, err := c.MetaInfo() - - assert.Nil(t, err) - assert.NotNil(t, meta) - metaModel := meta["model"] - assert.True(t, metaModel != nil) - model, modelOK := metaModel.(map[string]interface{}) - assert.True(t, modelOK) - assert.True(t, model["_name_or_path"] != nil) - assert.True(t, model["architectures"] != nil) - }) -} - -type testMetaHandler struct { - t *testing.T - // the test handler will report as not ready before the time has passed - readyTime time.Time -} - -func (f *testMetaHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - assert.Equal(f.t, "/meta", r.URL.String()) - assert.Equal(f.t, http.MethodGet, r.Method) - - if time.Since(f.readyTime) < 0 { - w.WriteHeader(http.StatusServiceUnavailable) - } - - w.Write([]byte(f.metaInfo())) -} - -func (f *testMetaHandler) metaInfo() string { - return `{ - "model": { - "_name_or_path": "dbmdz/bert-large-cased-finetuned-conll03-english", - "_num_labels": 9, - "add_cross_attention": false, - "architectures": [ - "BertForTokenClassification" - ], - "attention_probs_dropout_prob": 0.1, - "bad_words_ids": null, - "bos_token_id": null, - "chunk_size_feed_forward": 0, - "decoder_start_token_id": null, - "directionality": "bidi", - "diversity_penalty": 0, - "do_sample": false, - "early_stopping": false, - "encoder_no_repeat_ngram_size": 0, - "eos_token_id": null, - "finetuning_task": null, - "forced_bos_token_id": null, - "forced_eos_token_id": null, - "gradient_checkpointing": false, - "hidden_act": "gelu", - "hidden_dropout_prob": 0.1, - "hidden_size": 1024, - "id2label": { - "0": "O", - "1": "B-MISC", - "2": "I-MISC", - "3": "B-PER", - "4": "I-PER", - "5": "B-ORG", - "6": "I-ORG", - "7": "B-LOC", - "8": "I-LOC" - }, - "initializer_range": 0.02, - "intermediate_size": 4096, - "is_decoder": false, - "is_encoder_decoder": false, - "label2id": { - "B-LOC": 7, - "B-MISC": 1, - "B-ORG": 5, - "B-PER": 3, - "I-LOC": 8, - "I-MISC": 2, - "I-ORG": 6, - "I-PER": 4, - "O": 0 - }, - "layer_norm_eps": 1e-12, - "length_penalty": 1, - "max_length": 20, - "max_position_embeddings": 512, - "min_length": 0, - "model_type": "bert", - "no_repeat_ngram_size": 0, - "num_attention_heads": 16, - "num_beam_groups": 1, - "num_beams": 1, - "num_hidden_layers": 24, - "num_return_sequences": 1, - "output_attentions": false, - "output_hidden_states": false, - "output_scores": false, - "pad_token_id": 0, - "pooler_fc_size": 768, - "pooler_num_attention_heads": 12, - "pooler_num_fc_layers": 3, - "pooler_size_per_head": 128, - "pooler_type": "first_token_transform", - "position_embedding_type": "absolute", - "prefix": null, - "problem_type": null, - "pruned_heads": {}, - "remove_invalid_values": false, - "repetition_penalty": 1, - "return_dict": true, - "return_dict_in_generate": false, - "sep_token_id": null, - "task_specific_params": null, - "temperature": 1, - "tie_encoder_decoder": false, - "tie_word_embeddings": true, - "tokenizer_class": null, - "top_k": 50, - "top_p": 1, - "torchscript": false, - "transformers_version": "4.6.1", - "type_vocab_size": 2, - "use_bfloat16": false, - "use_cache": true, - "vocab_size": 28996 - } - }` -} diff --git a/modules/ner-transformers/clients/ner_test.go b/modules/ner-transformers/clients/ner_test.go deleted file mode 100644 index 68e2c80d68e45c0027ef8a7f06812e914e25bf18..0000000000000000000000000000000000000000 --- a/modules/ner-transformers/clients/ner_test.go +++ /dev/null @@ -1,132 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "context" - "encoding/json" - "net/http" - "net/http/httptest" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/modules/ner-transformers/ent" -) - -func TestGetAnswer(t *testing.T) { - t.Run("when the server has a successful answer (with distance)", func(t *testing.T) { - server := httptest.NewServer(&testNERHandler{ - t: t, - res: nerResponse{ - nerInput: nerInput{ - Text: "I work at Apple", - }, - Tokens: []tokenResponse{ - { - Entity: "I-ORG", - Distance: 0.3, - Word: "Apple", - StartPosition: 20, - EndPosition: 25, - }, - }, - }, - }) - defer server.Close() - c := New(server.URL, 0, nullLogger()) - res, err := c.GetTokens(context.Background(), "prop", - "I work at Apple") - - assert.Nil(t, err) - assert.Equal(t, []ent.TokenResult{ - { - Entity: "I-ORG", - Distance: 0.3, - Word: "Apple", - StartPosition: 20, - EndPosition: 25, - Property: "prop", - }, - }, res) - }) - - t.Run("when the server has a successful answer (with certainty)", func(t *testing.T) { - server := httptest.NewServer(&testNERHandler{ - t: t, - res: nerResponse{ - nerInput: nerInput{ - Text: "I work at Apple", - }, - Tokens: []tokenResponse{ - { - Entity: "I-ORG", - Certainty: 0.7, - Word: "Apple", - StartPosition: 20, - EndPosition: 25, - }, - }, - }, - }) - defer server.Close() - c := New(server.URL, 0, nullLogger()) - res, err := c.GetTokens(context.Background(), "prop", - "I work at Apple") - - assert.Nil(t, err) - assert.Equal(t, []ent.TokenResult{ - { - Entity: "I-ORG", - Certainty: 0.7, - Word: "Apple", - StartPosition: 20, - EndPosition: 25, - Property: "prop", - }, - }, res) - }) - - t.Run("when the server has a an error", func(t *testing.T) { - server := httptest.NewServer(&testNERHandler{ - t: t, - res: nerResponse{ - Error: "some error from the server", - }, - }) - defer server.Close() - c := New(server.URL, 0, nullLogger()) - _, err := c.GetTokens(context.Background(), "prop", - "I work at Apple") - - require.NotNil(t, err) - assert.Contains(t, err.Error(), "some error from the server") - }) -} - -type testNERHandler struct { - t *testing.T - // the test handler will report as not ready before the time has passed - res nerResponse -} - -func (f *testNERHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - assert.Equal(f.t, "/ner/", r.URL.String()) - assert.Equal(f.t, http.MethodPost, r.Method) - - if f.res.Error != "" { - w.WriteHeader(500) - } - - jsonBytes, _ := json.Marshal(f.res) - w.Write(jsonBytes) -} diff --git a/modules/ner-transformers/clients/startup.go b/modules/ner-transformers/clients/startup.go deleted file mode 100644 index 536e90592eaa9d69341975d6d32e8800673e9013..0000000000000000000000000000000000000000 --- a/modules/ner-transformers/clients/startup.go +++ /dev/null @@ -1,68 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "context" - "net/http" - "time" - - "github.com/pkg/errors" -) - -func (n *ner) WaitForStartup(initCtx context.Context, - interval time.Duration, -) error { - t := time.NewTicker(interval) - defer t.Stop() - expired := initCtx.Done() - var lastErr error - for { - select { - case <-t.C: - lastErr = n.checkReady(initCtx) - if lastErr == nil { - return nil - } - n.logger. - WithField("action", "ner_remote_wait_for_startup"). - WithError(lastErr).Warnf("ner remote service not ready") - case <-expired: - return errors.Wrapf(lastErr, "init context expired before remote was ready") - } - } -} - -func (n *ner) checkReady(initCtx context.Context) error { - // spawn a new context (derived on the overall context) which is used to - // consider an individual request timed out - requestCtx, cancel := context.WithTimeout(initCtx, 500*time.Millisecond) - defer cancel() - - req, err := http.NewRequestWithContext(requestCtx, http.MethodGet, - n.url("/.well-known/ready"), nil) - if err != nil { - return errors.Wrap(err, "create check ready request") - } - - res, err := n.httpClient.Do(req) - if err != nil { - return errors.Wrap(err, "send check ready request") - } - - defer res.Body.Close() - if res.StatusCode > 299 { - return errors.Errorf("not ready: status %d", res.StatusCode) - } - - return nil -} diff --git a/modules/ner-transformers/clients/startup_test.go b/modules/ner-transformers/clients/startup_test.go deleted file mode 100644 index 7836347d937eaaa18e3c1415dcb4b2aac2d2b15b..0000000000000000000000000000000000000000 --- a/modules/ner-transformers/clients/startup_test.go +++ /dev/null @@ -1,98 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "context" - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/sirupsen/logrus" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestWaitForStartup(t *testing.T) { - t.Run("when the server is immediately ready", func(t *testing.T) { - server := httptest.NewServer(&testReadyHandler{t: t}) - defer server.Close() - c := New(server.URL, 0, nullLogger()) - err := c.WaitForStartup(context.Background(), 50*time.Millisecond) - - assert.Nil(t, err) - }) - - t.Run("when the server is down", func(t *testing.T) { - c := New("http://nothing-running-at-this-url", 0, nullLogger()) - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancel() - err := c.WaitForStartup(ctx, 150*time.Millisecond) - - require.NotNil(t, err, nullLogger()) - assert.Contains(t, err.Error(), "expired before remote was ready") - }) - - t.Run("when the server is alive, but not ready", func(t *testing.T) { - server := httptest.NewServer(&testReadyHandler{ - t: t, - readyTime: time.Now().Add(1 * time.Minute), - }) - c := New(server.URL, 0, nullLogger()) - defer server.Close() - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancel() - err := c.WaitForStartup(ctx, 50*time.Millisecond) - - require.NotNil(t, err) - assert.Contains(t, err.Error(), "expired before remote was ready") - }) - - t.Run("when the server is initially not ready, but then becomes ready", - func(t *testing.T) { - server := httptest.NewServer(&testReadyHandler{ - t: t, - readyTime: time.Now().Add(100 * time.Millisecond), - }) - c := New(server.URL, 0, nullLogger()) - defer server.Close() - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancel() - err := c.WaitForStartup(ctx, 50*time.Millisecond) - - require.Nil(t, err) - }) -} - -type testReadyHandler struct { - t *testing.T - // the test handler will report as not ready before the time has passed - readyTime time.Time -} - -func (f *testReadyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - assert.Equal(f.t, "/.well-known/ready", r.URL.String()) - assert.Equal(f.t, http.MethodGet, r.Method) - - if time.Since(f.readyTime) < 0 { - w.WriteHeader(http.StatusServiceUnavailable) - } - - w.WriteHeader(http.StatusNoContent) -} - -func nullLogger() logrus.FieldLogger { - l, _ := test.NewNullLogger() - return l -} diff --git a/modules/ner-transformers/config.go b/modules/ner-transformers/config.go deleted file mode 100644 index 8ca847da5f032188c8eeb29cdc6fa19f851d27b5..0000000000000000000000000000000000000000 --- a/modules/ner-transformers/config.go +++ /dev/null @@ -1,39 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modner - -import ( - "context" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/schema" -) - -func (m *NERModule) ClassConfigDefaults() map[string]interface{} { - return map[string]interface{}{} -} - -func (m *NERModule) PropertyConfigDefaults( - dt *schema.DataType, -) map[string]interface{} { - return map[string]interface{}{} -} - -func (m *NERModule) ValidateClass(ctx context.Context, - class *models.Class, cfg moduletools.ClassConfig, -) error { - return nil -} - -var _ = modulecapabilities.ClassConfigurator(New()) diff --git a/modules/ner-transformers/ent/ner_result.go b/modules/ner-transformers/ent/ner_result.go deleted file mode 100644 index f2acc04f844a53dd2039d8fb1f167bfd866c1a2d..0000000000000000000000000000000000000000 --- a/modules/ner-transformers/ent/ner_result.go +++ /dev/null @@ -1,26 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package ent - -type TokenResult struct { - Property string - Word string - Entity string - Certainty float64 - Distance float64 - StartPosition int - EndPosition int -} - -type NerResult struct { - Tokens []TokenResult -} diff --git a/modules/ner-transformers/module.go b/modules/ner-transformers/module.go deleted file mode 100644 index 635d05591f3bed679cf5566fdeec9d323d5e7456..0000000000000000000000000000000000000000 --- a/modules/ner-transformers/module.go +++ /dev/null @@ -1,100 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modner - -import ( - "context" - "net/http" - "os" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - neradditional "github.com/weaviate/weaviate/modules/ner-transformers/additional" - neradditionaltoken "github.com/weaviate/weaviate/modules/ner-transformers/additional/tokens" - "github.com/weaviate/weaviate/modules/ner-transformers/clients" - "github.com/weaviate/weaviate/modules/ner-transformers/ent" -) - -func New() *NERModule { - return &NERModule{} -} - -type NERModule struct { - ner nerClient - additionalPropertiesProvider modulecapabilities.AdditionalProperties -} - -type nerClient interface { - GetTokens(ctx context.Context, property, text string) ([]ent.TokenResult, error) - MetaInfo() (map[string]interface{}, error) -} - -func (m *NERModule) Name() string { - return "ner-transformers" -} - -func (m *NERModule) Type() modulecapabilities.ModuleType { - return modulecapabilities.Text2TextNER -} - -func (m *NERModule) Init(ctx context.Context, - params moduletools.ModuleInitParams, -) error { - if err := m.initAdditional(ctx, params.GetConfig().ModuleHttpClientTimeout, params.GetLogger()); err != nil { - return errors.Wrap(err, "init additional") - } - return nil -} - -func (m *NERModule) initAdditional(ctx context.Context, timeout time.Duration, - logger logrus.FieldLogger, -) error { - uri := os.Getenv("NER_INFERENCE_API") - if uri == "" { - return errors.Errorf("required variable NER_INFERENCE_API is not set") - } - - client := clients.New(uri, timeout, logger) - if err := client.WaitForStartup(ctx, 1*time.Second); err != nil { - return errors.Wrap(err, "init remote ner module") - } - - m.ner = client - - tokenProvider := neradditionaltoken.New(m.ner) - m.additionalPropertiesProvider = neradditional.New(tokenProvider) - - return nil -} - -func (m *NERModule) RootHandler() http.Handler { - // TODO: remove once this is a capability interface - return nil -} - -func (m *NERModule) MetaInfo() (map[string]interface{}, error) { - return m.ner.MetaInfo() -} - -func (m *NERModule) AdditionalProperties() map[string]modulecapabilities.AdditionalProperty { - return m.additionalPropertiesProvider.AdditionalProperties() -} - -// verify we implement the modules.Module interface -var ( - _ = modulecapabilities.Module(New()) - _ = modulecapabilities.AdditionalProperties(New()) - _ = modulecapabilities.MetaProvider(New()) -) diff --git a/modules/qna-openai/additional/answer/answer.go b/modules/qna-openai/additional/answer/answer.go deleted file mode 100644 index 61e35c4acacd5b6a179b3acd2f40e651f957fade..0000000000000000000000000000000000000000 --- a/modules/qna-openai/additional/answer/answer.go +++ /dev/null @@ -1,65 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package answer - -import ( - "context" - "errors" - - "github.com/tailor-inc/graphql" - "github.com/tailor-inc/graphql/language/ast" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/search" - "github.com/weaviate/weaviate/modules/qna-openai/ent" -) - -type Params struct{} - -type qnaClient interface { - Answer(ctx context.Context, text, question string, cfg moduletools.ClassConfig) (*ent.AnswerResult, error) -} - -type paramsHelper interface { - GetQuestion(params interface{}) string - GetProperties(params interface{}) []string -} - -type AnswerProvider struct { - qna qnaClient - paramsHelper -} - -func New(qna qnaClient, paramsHelper paramsHelper) *AnswerProvider { - return &AnswerProvider{qna, paramsHelper} -} - -func (p *AnswerProvider) AdditionalPropertyDefaultValue() interface{} { - return &Params{} -} - -func (p *AnswerProvider) ExtractAdditionalFn(param []*ast.Argument) interface{} { - return &Params{} -} - -func (p *AnswerProvider) AdditionalFieldFn(classname string) *graphql.Field { - return p.additionalAnswerField(classname) -} - -func (p *AnswerProvider) AdditionalPropertyFn(ctx context.Context, - in []search.Result, params interface{}, limit *int, - argumentModuleParams map[string]interface{}, cfg moduletools.ClassConfig, -) ([]search.Result, error) { - if parameters, ok := params.(*Params); ok { - return p.findAnswer(ctx, in, parameters, limit, argumentModuleParams, cfg) - } - return nil, errors.New("wrong parameters") -} diff --git a/modules/qna-openai/additional/answer/answer_graphql_field.go b/modules/qna-openai/additional/answer/answer_graphql_field.go deleted file mode 100644 index 20642a8b44c6ef42316d050ac90463b840e70326..0000000000000000000000000000000000000000 --- a/modules/qna-openai/additional/answer/answer_graphql_field.go +++ /dev/null @@ -1,33 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package answer - -import ( - "fmt" - - "github.com/tailor-inc/graphql" -) - -func (p *AnswerProvider) additionalAnswerField(classname string) *graphql.Field { - return &graphql.Field{ - Type: graphql.NewObject(graphql.ObjectConfig{ - Name: fmt.Sprintf("%sAdditionalAnswer", classname), - Fields: graphql.Fields{ - "result": &graphql.Field{Type: graphql.String}, - "startPosition": &graphql.Field{Type: graphql.Int}, - "endPosition": &graphql.Field{Type: graphql.Int}, - "property": &graphql.Field{Type: graphql.String}, - "hasAnswer": &graphql.Field{Type: graphql.Boolean}, - }, - }), - } -} diff --git a/modules/qna-openai/additional/answer/answer_graphql_field_test.go b/modules/qna-openai/additional/answer/answer_graphql_field_test.go deleted file mode 100644 index 9bb2dd8be732d7075f890648273bf3f70f31e86c..0000000000000000000000000000000000000000 --- a/modules/qna-openai/additional/answer/answer_graphql_field_test.go +++ /dev/null @@ -1,54 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package answer - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/tailor-inc/graphql" -) - -func TestAnswerField(t *testing.T) { - t.Run("should generate answer argument properly", func(t *testing.T) { - // given - answerProvider := &AnswerProvider{} - classname := "Class" - - // when - answer := answerProvider.additionalAnswerField(classname) - - // then - // the built graphQL field needs to support this structure: - // Type: { - // answer: { - // result: "answer", - // startPosition: 1 - // endPosition: 2 - // distance: 0.2 - // property: "propName" - // hasAnswer: true - // } - // } - assert.NotNil(t, answer) - assert.Equal(t, "ClassAdditionalAnswer", answer.Type.Name()) - assert.NotNil(t, answer.Type) - answerObject, answerObjectOK := answer.Type.(*graphql.Object) - assert.True(t, answerObjectOK) - assert.Equal(t, 5, len(answerObject.Fields())) - assert.NotNil(t, answerObject.Fields()["result"]) - assert.NotNil(t, answerObject.Fields()["startPosition"]) - assert.NotNil(t, answerObject.Fields()["endPosition"]) - assert.NotNil(t, answerObject.Fields()["property"]) - assert.NotNil(t, answerObject.Fields()["hasAnswer"]) - }) -} diff --git a/modules/qna-openai/additional/answer/answer_result.go b/modules/qna-openai/additional/answer/answer_result.go deleted file mode 100644 index d41cff3f846ba37a61fb37d5740f7a1f63d73a2a..0000000000000000000000000000000000000000 --- a/modules/qna-openai/additional/answer/answer_result.go +++ /dev/null @@ -1,107 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package answer - -import ( - "context" - "errors" - "strings" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/search" - qnamodels "github.com/weaviate/weaviate/modules/qna-openai/additional/models" -) - -func (p *AnswerProvider) findAnswer(ctx context.Context, in []search.Result, params *Params, limit *int, argumentModuleParams map[string]interface{}, cfg moduletools.ClassConfig) ([]search.Result, error) { - if len(in) == 0 { - return in, nil - } - question := p.paramsHelper.GetQuestion(argumentModuleParams["ask"]) - if question == "" { - return in, errors.New("empty question") - } - properties := p.paramsHelper.GetProperties(argumentModuleParams["ask"]) - - for i := range in { - textProperties := map[string]string{} - schema := in[i].Object().Properties.(map[string]interface{}) - for property, value := range schema { - if p.containsProperty(property, properties) { - if valueString, ok := value.(string); ok && len(valueString) > 0 { - textProperties[property] = valueString - } - } - } - - var texts []string - for _, value := range textProperties { - texts = append(texts, value) - } - text := strings.Join(texts, " ") - if len(text) == 0 { - return in, errors.New("empty content") - } - - answer, err := p.qna.Answer(ctx, text, question, cfg) - if err != nil { - return in, err - } - - ap := in[i].AdditionalProperties - if ap == nil { - ap = models.AdditionalProperties{} - } - propertyName, startPos, endPos := p.findProperty(answer.Answer, textProperties) - ap["answer"] = &qnamodels.Answer{ - Result: answer.Answer, - Property: propertyName, - StartPosition: startPos, - EndPosition: endPos, - HasAnswer: answer.Answer != nil, - } - - in[i].AdditionalProperties = ap - } - - return in, nil -} - -func (p *AnswerProvider) containsProperty(property string, properties []string) bool { - if len(properties) == 0 { - return true - } - for i := range properties { - if properties[i] == property { - return true - } - } - return false -} - -func (p *AnswerProvider) findProperty(answer *string, textProperties map[string]string) (*string, int, int) { - if answer == nil { - return nil, 0, 0 - } - lowercaseAnswer := strings.ToLower(*answer) - if len(lowercaseAnswer) > 0 { - for property, value := range textProperties { - lowercaseValue := strings.ToLower(strings.ReplaceAll(value, "\n", " ")) - if strings.Contains(lowercaseValue, lowercaseAnswer) { - startIndex := strings.Index(lowercaseValue, lowercaseAnswer) - return &property, startIndex, startIndex + len(lowercaseAnswer) - } - } - } - propertyNotFound := "" - return &propertyNotFound, 0, 0 -} diff --git a/modules/qna-openai/additional/answer/answer_test.go b/modules/qna-openai/additional/answer/answer_test.go deleted file mode 100644 index 739352b0ba31b589504507bdf32b54d382ea1d94..0000000000000000000000000000000000000000 --- a/modules/qna-openai/additional/answer/answer_test.go +++ /dev/null @@ -1,187 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package answer - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/search" - qnamodels "github.com/weaviate/weaviate/modules/qna-openai/additional/models" - "github.com/weaviate/weaviate/modules/qna-openai/ent" -) - -func TestAdditionalAnswerProvider(t *testing.T) { - t.Run("should fail with empty content", func(t *testing.T) { - // given - qnaClient := &fakeQnAClient{} - fakeHelper := &fakeParamsHelper{} - answerProvider := New(qnaClient, fakeHelper) - in := []search.Result{ - { - ID: "some-uuid", - }, - } - fakeParams := &Params{} - limit := 1 - argumentModuleParams := map[string]interface{}{} - - // when - out, err := answerProvider.AdditionalPropertyFn(context.Background(), in, fakeParams, &limit, argumentModuleParams, nil) - - // then - require.NotNil(t, err) - require.NotEmpty(t, out) - assert.Error(t, err, "empty content") - }) - - t.Run("should fail with empty question", func(t *testing.T) { - // given - qnaClient := &fakeQnAClient{} - fakeHelper := &fakeParamsHelper{} - answerProvider := New(qnaClient, fakeHelper) - in := []search.Result{ - { - ID: "some-uuid", - Schema: map[string]interface{}{ - "content": "content", - }, - }, - } - fakeParams := &Params{} - limit := 1 - argumentModuleParams := map[string]interface{}{} - - // when - out, err := answerProvider.AdditionalPropertyFn(context.Background(), in, fakeParams, &limit, argumentModuleParams, nil) - - // then - require.NotNil(t, err) - require.NotEmpty(t, out) - assert.Error(t, err, "empty content") - }) - - t.Run("should answer", func(t *testing.T) { - // given - qnaClient := &fakeQnAClient{} - fakeHelper := &fakeParamsHelper{} - answerProvider := New(qnaClient, fakeHelper) - in := []search.Result{ - { - ID: "some-uuid", - Schema: map[string]interface{}{ - "content": "content", - }, - }, - } - fakeParams := &Params{} - limit := 1 - argumentModuleParams := map[string]interface{}{ - "ask": map[string]interface{}{ - "question": "question", - }, - } - - // when - out, err := answerProvider.AdditionalPropertyFn(context.Background(), in, fakeParams, &limit, argumentModuleParams, nil) - - // then - require.Nil(t, err) - require.NotEmpty(t, out) - assert.Equal(t, 1, len(in)) - answer, answerOK := in[0].AdditionalProperties["answer"] - assert.True(t, answerOK) - assert.NotNil(t, answer) - answerAdditional, answerAdditionalOK := answer.(*qnamodels.Answer) - assert.True(t, answerAdditionalOK) - assert.Equal(t, "answer", *answerAdditional.Result) - }) - - t.Run("should answer with property", func(t *testing.T) { - // given - qnaClient := &fakeQnAClient{} - fakeHelper := &fakeParamsHelper{} - answerProvider := New(qnaClient, fakeHelper) - in := []search.Result{ - { - ID: "some-uuid", - Schema: map[string]interface{}{ - "content": "content with answer", - "content2": "this one is just a title", - }, - }, - } - fakeParams := &Params{} - limit := 1 - argumentModuleParams := map[string]interface{}{ - "ask": map[string]interface{}{ - "question": "question", - "properties": []string{"content", "content2"}, - }, - } - - // when - out, err := answerProvider.AdditionalPropertyFn(context.Background(), in, fakeParams, &limit, argumentModuleParams, nil) - - // then - require.Nil(t, err) - require.NotEmpty(t, out) - assert.Equal(t, 1, len(in)) - answer, answerOK := in[0].AdditionalProperties["answer"] - assert.True(t, answerOK) - assert.NotNil(t, answer) - answerAdditional, answerAdditionalOK := answer.(*qnamodels.Answer) - assert.True(t, answerAdditionalOK) - assert.Equal(t, "answer", *answerAdditional.Result) - assert.Equal(t, "content", *answerAdditional.Property) - assert.Equal(t, 13, answerAdditional.StartPosition) - assert.Equal(t, 19, answerAdditional.EndPosition) - assert.Equal(t, true, answerAdditional.HasAnswer) - }) -} - -type fakeQnAClient struct{} - -func (c *fakeQnAClient) Answer(ctx context.Context, text, question string, cfg moduletools.ClassConfig) (*ent.AnswerResult, error) { - return c.getAnswer(question, "answer"), nil -} - -func (c *fakeQnAClient) getAnswer(question, answer string) *ent.AnswerResult { - return &ent.AnswerResult{ - Text: question, - Question: question, - Answer: &answer, - } -} - -type fakeParamsHelper struct{} - -func (h *fakeParamsHelper) GetQuestion(params interface{}) string { - if fakeParamsMap, ok := params.(map[string]interface{}); ok { - if question, ok := fakeParamsMap["question"].(string); ok { - return question - } - } - return "" -} - -func (h *fakeParamsHelper) GetProperties(params interface{}) []string { - if fakeParamsMap, ok := params.(map[string]interface{}); ok { - if properties, ok := fakeParamsMap["properties"].([]string); ok { - return properties - } - } - return nil -} diff --git a/modules/qna-openai/additional/models/models.go b/modules/qna-openai/additional/models/models.go deleted file mode 100644 index ae567aea4895424acfb3c7e19b30fb7cb49b7c0e..0000000000000000000000000000000000000000 --- a/modules/qna-openai/additional/models/models.go +++ /dev/null @@ -1,22 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package models - -// Answer used in qna module to represent -// the answer to a given question -type Answer struct { - Result *string `json:"result,omitempty"` - Property *string `json:"property,omitempty"` - StartPosition int `json:"startPosition,omitempty"` - EndPosition int `json:"endPosition,omitempty"` - HasAnswer bool `json:"hasAnswer,omitempty"` -} diff --git a/modules/qna-openai/additional/provider.go b/modules/qna-openai/additional/provider.go deleted file mode 100644 index 5df746281892008bd1ebc702a91507586dbb224e..0000000000000000000000000000000000000000 --- a/modules/qna-openai/additional/provider.go +++ /dev/null @@ -1,57 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package additional - -import ( - "context" - - "github.com/tailor-inc/graphql" - "github.com/tailor-inc/graphql/language/ast" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/search" -) - -type AdditionalProperty interface { - AdditionalPropertyFn(ctx context.Context, - in []search.Result, params interface{}, limit *int, - argumentModuleParams map[string]interface{}, cfg moduletools.ClassConfig) ([]search.Result, error) - ExtractAdditionalFn(param []*ast.Argument) interface{} - AdditionalPropertyDefaultValue() interface{} - AdditionalFieldFn(classname string) *graphql.Field -} - -type GraphQLAdditionalArgumentsProvider struct { - answerProvider AdditionalProperty -} - -func New(answerProvider AdditionalProperty) *GraphQLAdditionalArgumentsProvider { - return &GraphQLAdditionalArgumentsProvider{answerProvider} -} - -func (p *GraphQLAdditionalArgumentsProvider) AdditionalProperties() map[string]modulecapabilities.AdditionalProperty { - additionalProperties := map[string]modulecapabilities.AdditionalProperty{} - additionalProperties["answer"] = p.getAnswer() - return additionalProperties -} - -func (p *GraphQLAdditionalArgumentsProvider) getAnswer() modulecapabilities.AdditionalProperty { - return modulecapabilities.AdditionalProperty{ - GraphQLNames: []string{"answer"}, - GraphQLFieldFunction: p.answerProvider.AdditionalFieldFn, - GraphQLExtractFunction: p.answerProvider.ExtractAdditionalFn, - SearchFunctions: modulecapabilities.AdditionalSearch{ - ExploreGet: p.answerProvider.AdditionalPropertyFn, - ExploreList: p.answerProvider.AdditionalPropertyFn, - }, - } -} diff --git a/modules/qna-openai/ask.go b/modules/qna-openai/ask.go deleted file mode 100644 index 1569fd348960bd7c901ba358fda2beddcc179f95..0000000000000000000000000000000000000000 --- a/modules/qna-openai/ask.go +++ /dev/null @@ -1,40 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modqnaopenai - -import ( - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/modules/qna-openai/ask" -) - -func (m *QnAModule) initAskSearcher() error { - m.searcher = ask.NewSearcher(m.nearTextDependencies) - return nil -} - -func (m *QnAModule) initAskProvider() error { - m.graphqlProvider = ask.New(m.askTextTransformer) - return nil -} - -func (m *QnAModule) Arguments() map[string]modulecapabilities.GraphQLArgument { - return m.graphqlProvider.Arguments() -} - -func (m *QnAModule) VectorSearches() modulecapabilities.ModuleArgumentVectorForParams { - return m.searcher.VectorSearches() -} - -var ( - _ = modulecapabilities.GraphQLArguments(New()) - _ = modulecapabilities.DependencySearcher(New()) -) diff --git a/modules/qna-openai/ask/graphql_argument.go b/modules/qna-openai/ask/graphql_argument.go deleted file mode 100644 index 56aed59ba887099b0eb1ac684792718cefa5cd98..0000000000000000000000000000000000000000 --- a/modules/qna-openai/ask/graphql_argument.go +++ /dev/null @@ -1,64 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package ask - -import ( - "fmt" - - "github.com/tailor-inc/graphql" - "github.com/weaviate/weaviate/adapters/handlers/graphql/descriptions" -) - -func (g *GraphQLArgumentsProvider) getAskArgumentFn(classname string) *graphql.ArgumentConfig { - return g.askArgument("GetObjects", classname) -} - -func (g *GraphQLArgumentsProvider) exploreAskArgumentFn() *graphql.ArgumentConfig { - return g.askArgument("Explore", "") -} - -func (g *GraphQLArgumentsProvider) aggregateAskArgumentFn(classname string) *graphql.ArgumentConfig { - return g.askArgument("Aggregate", classname) -} - -func (g *GraphQLArgumentsProvider) askArgument(prefix, className string) *graphql.ArgumentConfig { - prefixName := fmt.Sprintf("QnATransformers%s%s", prefix, className) - return &graphql.ArgumentConfig{ - Type: graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sAskInpObj", prefixName), - Fields: g.askFields(prefixName), - Description: descriptions.GetWhereInpObj, - }, - ), - } -} - -func (g *GraphQLArgumentsProvider) askFields(prefix string) graphql.InputObjectConfigFieldMap { - askFields := graphql.InputObjectConfigFieldMap{ - "question": &graphql.InputObjectFieldConfig{ - Description: "Question to be answered", - Type: graphql.NewNonNull(graphql.String), - }, - "properties": &graphql.InputObjectFieldConfig{ - Description: "Properties which contains text", - Type: graphql.NewList(graphql.String), - }, - } - if g.askTransformer != nil { - askFields["autocorrect"] = &graphql.InputObjectFieldConfig{ - Description: "Autocorrect input text values", - Type: graphql.Boolean, - } - } - return askFields -} diff --git a/modules/qna-openai/ask/graphql_argument_test.go b/modules/qna-openai/ask/graphql_argument_test.go deleted file mode 100644 index 7fadfb74c56547cc1dfd1b563a7b675f39bd1604..0000000000000000000000000000000000000000 --- a/modules/qna-openai/ask/graphql_argument_test.go +++ /dev/null @@ -1,52 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package ask - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/tailor-inc/graphql" -) - -func TestAskGraphQLArgument(t *testing.T) { - t.Run("should generate ask argument properly", func(t *testing.T) { - // given - prefix := "Prefix" - classname := "Class" - // when - ask := New(nil).askArgument(prefix, classname) - - // then - // the built graphQL field needs to support this structure: - // ask { - // question: "question?", - // properties: ["prop1", "prop2"] - // } - assert.NotNil(t, ask) - assert.Equal(t, "QnATransformersPrefixClassAskInpObj", ask.Type.Name()) - askFields, ok := ask.Type.(*graphql.InputObject) - assert.True(t, ok) - assert.NotNil(t, askFields) - assert.Equal(t, 2, len(askFields.Fields())) - fields := askFields.Fields() - question := fields["question"] - questionNonNull, questionNonNullOK := question.Type.(*graphql.NonNull) - assert.True(t, questionNonNullOK) - assert.Equal(t, "String", questionNonNull.OfType.Name()) - assert.NotNil(t, question) - properties := fields["properties"] - propertiesList, propertiesListOK := properties.Type.(*graphql.List) - assert.True(t, propertiesListOK) - assert.Equal(t, "String", propertiesList.OfType.Name()) - }) -} diff --git a/modules/qna-openai/ask/graphql_provider.go b/modules/qna-openai/ask/graphql_provider.go deleted file mode 100644 index 98e1261176798e4f1de79599284a93a1d80e1884..0000000000000000000000000000000000000000 --- a/modules/qna-openai/ask/graphql_provider.go +++ /dev/null @@ -1,40 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package ask - -import ( - "github.com/weaviate/weaviate/entities/modulecapabilities" -) - -type GraphQLArgumentsProvider struct { - askTransformer modulecapabilities.TextTransform -} - -func New(askTransformer modulecapabilities.TextTransform) *GraphQLArgumentsProvider { - return &GraphQLArgumentsProvider{askTransformer} -} - -func (g *GraphQLArgumentsProvider) Arguments() map[string]modulecapabilities.GraphQLArgument { - arguments := map[string]modulecapabilities.GraphQLArgument{} - arguments["ask"] = g.getAsk() - return arguments -} - -func (g *GraphQLArgumentsProvider) getAsk() modulecapabilities.GraphQLArgument { - return modulecapabilities.GraphQLArgument{ - GetArgumentsFunction: g.getAskArgumentFn, - AggregateArgumentsFunction: g.aggregateAskArgumentFn, - ExploreArgumentsFunction: g.exploreAskArgumentFn, - ExtractFunction: g.extractAskFn, - ValidateFunction: g.validateAskFn, - } -} diff --git a/modules/qna-openai/ask/grapqhl_extract.go b/modules/qna-openai/ask/grapqhl_extract.go deleted file mode 100644 index fff41bc169b266823d2c1e489c75aee4439dddc8..0000000000000000000000000000000000000000 --- a/modules/qna-openai/ask/grapqhl_extract.go +++ /dev/null @@ -1,31 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package ask - -func (g *GraphQLArgumentsProvider) extractAskFn(source map[string]interface{}) interface{} { - var args AskParams - - question, ok := source["question"].(string) - if ok { - args.Question = question - } - - properties, ok := source["properties"].([]interface{}) - if ok { - args.Properties = make([]string, len(properties)) - for i, value := range properties { - args.Properties[i] = value.(string) - } - } - - return &args -} diff --git a/modules/qna-openai/ask/grapqhl_extract_test.go b/modules/qna-openai/ask/grapqhl_extract_test.go deleted file mode 100644 index 704b0bda36a224f266e4066ddb10eebe98f4defe..0000000000000000000000000000000000000000 --- a/modules/qna-openai/ask/grapqhl_extract_test.go +++ /dev/null @@ -1,70 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package ask - -import ( - "reflect" - "testing" -) - -func Test_extractAskFn(t *testing.T) { - type args struct { - source map[string]interface{} - } - tests := []struct { - name string - args args - want interface{} - }{ - { - name: "should parse properly with only question", - args: args{ - source: map[string]interface{}{ - "question": "some question", - }, - }, - want: &AskParams{ - Question: "some question", - }, - }, - { - name: "should parse properly without params", - args: args{ - source: map[string]interface{}{}, - }, - want: &AskParams{}, - }, - { - name: "should parse properly with question, and properties", - args: args{ - source: map[string]interface{}{ - "question": "some question", - "properties": []interface{}{"prop1", "prop2"}, - }, - }, - want: &AskParams{ - Question: "some question", - Properties: []string{"prop1", "prop2"}, - }, - }, - } - t.Run("should extract without text transformer", func(t *testing.T) { - provider := New(nil) - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := provider.extractAskFn(tt.args.source); !reflect.DeepEqual(got, tt.want) { - t.Errorf("extractAskFn() = %v, want %v", got, tt.want) - } - }) - } - }) -} diff --git a/modules/qna-openai/ask/param.go b/modules/qna-openai/ask/param.go deleted file mode 100644 index 6db82f4a4ffb252b8865cf1779bf166070779c40..0000000000000000000000000000000000000000 --- a/modules/qna-openai/ask/param.go +++ /dev/null @@ -1,55 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package ask - -import ( - "github.com/pkg/errors" -) - -type AskParams struct { - Question string - Certainty float64 - Distance float64 - WithDistance bool - Properties []string - Autocorrect bool -} - -func (n AskParams) GetCertainty() float64 { - return n.Certainty -} - -func (n AskParams) GetDistance() float64 { - return n.Distance -} - -func (n AskParams) SimilarityMetricProvided() bool { - return n.Certainty != 0 || n.WithDistance -} - -func (g *GraphQLArgumentsProvider) validateAskFn(param interface{}) error { - ask, ok := param.(*AskParams) - if !ok { - return errors.New("'ask' invalid parameter") - } - - if len(ask.Question) == 0 { - return errors.Errorf("'ask.question' needs to be defined") - } - - if ask.Certainty != 0 && ask.WithDistance { - return errors.Errorf( - "nearText cannot provide both distance and certainty") - } - - return nil -} diff --git a/modules/qna-openai/ask/param_helper.go b/modules/qna-openai/ask/param_helper.go deleted file mode 100644 index 7b0483e01fba121f6e2fea3d404dda12d317cb38..0000000000000000000000000000000000000000 --- a/modules/qna-openai/ask/param_helper.go +++ /dev/null @@ -1,46 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package ask - -type ParamsHelper struct{} - -func NewParamsHelper() *ParamsHelper { - return &ParamsHelper{} -} - -func (p *ParamsHelper) GetQuestion(params interface{}) string { - if parameters, ok := params.(*AskParams); ok { - return parameters.Question - } - return "" -} - -func (p *ParamsHelper) GetProperties(params interface{}) []string { - if parameters, ok := params.(*AskParams); ok { - return parameters.Properties - } - return nil -} - -func (p *ParamsHelper) GetCertainty(params interface{}) float64 { - if parameters, ok := params.(*AskParams); ok { - return parameters.Certainty - } - return 0 -} - -func (p *ParamsHelper) GetDistance(params interface{}) float64 { - if parameters, ok := params.(*AskParams); ok { - return parameters.Distance - } - return 0 -} diff --git a/modules/qna-openai/ask/param_helper_test.go b/modules/qna-openai/ask/param_helper_test.go deleted file mode 100644 index 709baeecf558a77fcf2387f0006484c1979ac526..0000000000000000000000000000000000000000 --- a/modules/qna-openai/ask/param_helper_test.go +++ /dev/null @@ -1,151 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package ask - -import ( - "reflect" - "testing" -) - -func TestParamsHelper_GetQuestion(t *testing.T) { - type args struct { - params interface{} - } - tests := []struct { - name string - args args - want string - }{ - { - name: "should get question with certainty", - args: args{ - params: &AskParams{ - Question: "question", - Certainty: 0.8, - }, - }, - want: "question", - }, - { - name: "should get question with distance", - args: args{ - params: &AskParams{ - Question: "question", - Distance: 0.8, - }, - }, - want: "question", - }, - { - name: "should get empty string when empty params", - args: args{ - params: &AskParams{}, - }, - want: "", - }, - { - name: "should get empty string when nil params", - args: args{ - params: nil, - }, - want: "", - }, - { - name: "should get empty string when passed a struct, not a pointer to struct", - args: args{ - params: AskParams{}, - }, - want: "", - }, - { - name: "should get empty string when passed a struct with question, not a pointer to struct", - args: args{ - params: AskParams{ - Question: "question?", - }, - }, - want: "", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - p := &ParamsHelper{} - if got := p.GetQuestion(tt.args.params); got != tt.want { - t.Errorf("ParamsHelper.GetQuestion() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestParamsHelper_GetProperties(t *testing.T) { - type args struct { - params interface{} - } - tests := []struct { - name string - p *ParamsHelper - args args - want []string - }{ - { - name: "should get properties with distance", - args: args{ - params: &AskParams{ - Question: "question", - Properties: []string{"prop1", "prop2"}, - Distance: 0.8, - }, - }, - want: []string{"prop1", "prop2"}, - }, - { - name: "should get properties with certainty", - args: args{ - params: &AskParams{ - Question: "question", - Properties: []string{"prop1", "prop2"}, - Certainty: 0.8, - }, - }, - want: []string{"prop1", "prop2"}, - }, - { - name: "should get nil properties with empty pointer to AskParams", - args: args{ - params: &AskParams{}, - }, - want: nil, - }, - { - name: "should get nil properties with empty AskParams", - args: args{ - params: AskParams{}, - }, - want: nil, - }, - { - name: "should get nil properties with nil params", - args: args{ - params: nil, - }, - want: nil, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - p := &ParamsHelper{} - if got := p.GetProperties(tt.args.params); !reflect.DeepEqual(got, tt.want) { - t.Errorf("ParamsHelper.GetProperties() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/modules/qna-openai/ask/param_test.go b/modules/qna-openai/ask/param_test.go deleted file mode 100644 index db632556cf7723840f02fc9889a68412a67f36b9..0000000000000000000000000000000000000000 --- a/modules/qna-openai/ask/param_test.go +++ /dev/null @@ -1,77 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package ask - -import "testing" - -func Test_validateAskFn(t *testing.T) { - type args struct { - param interface{} - } - tests := []struct { - name string - args args - wantErr bool - }{ - { - name: "should validate", - args: args{ - param: &AskParams{ - Question: "question", - }, - }, - }, - { - name: "should not validate when empty question", - args: args{ - param: &AskParams{ - Question: "", - }, - }, - wantErr: true, - }, - { - name: "should not validate when empty params", - args: args{ - param: &AskParams{}, - }, - wantErr: true, - }, - { - name: "should not validate when distance and certainty are present", - args: args{ - param: &AskParams{ - Distance: 0.1, - Certainty: 0.1, - }, - }, - wantErr: true, - }, - { - name: "should not validate when param passed is struct, not a pointer to struct", - args: args{ - param: AskParams{ - Question: "question", - }, - }, - wantErr: true, - }, - } - provider := New(nil) - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := provider.validateAskFn(tt.args.param); (err != nil) != tt.wantErr { - t.Errorf("validateAskFn() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} diff --git a/modules/qna-openai/ask/searcher.go b/modules/qna-openai/ask/searcher.go deleted file mode 100644 index 467b437acd297640bcf1abefeb99de04801f841f..0000000000000000000000000000000000000000 --- a/modules/qna-openai/ask/searcher.go +++ /dev/null @@ -1,71 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package ask - -import ( - "context" - - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" -) - -type vectorFromAskParam struct { - nearTextDep modulecapabilities.Dependency -} - -func (s *vectorFromAskParam) vectorForAskParamFn(ctx context.Context, params interface{}, - className string, - findVectorFn modulecapabilities.FindVectorFn, - cfg moduletools.ClassConfig, -) ([]float32, error) { - return s.vectorFromAskParam(ctx, params.(*AskParams), className, findVectorFn, cfg) -} - -func (s *vectorFromAskParam) vectorFromAskParam(ctx context.Context, - params *AskParams, className string, - findVectorFn modulecapabilities.FindVectorFn, - cfg moduletools.ClassConfig, -) ([]float32, error) { - arg := s.nearTextDep.GraphQLArgument() - - rawNearTextParam := map[string]interface{}{} - rawNearTextParam["concepts"] = []interface{}{params.Question} - - nearTextParam := arg.ExtractFunction(rawNearTextParam) - vectorSearchFn := s.nearTextDep.VectorSearch() - - return vectorSearchFn(ctx, nearTextParam, className, findVectorFn, cfg) -} - -type Searcher struct { - // nearText modules dependencies - nearTextDeps []modulecapabilities.Dependency -} - -func NewSearcher(nearTextDeps []modulecapabilities.Dependency) *Searcher { - return &Searcher{nearTextDeps} -} - -func (s *Searcher) VectorSearches() map[string]modulecapabilities.ArgumentVectorForParams { - vectorSearchers := map[string]modulecapabilities.ArgumentVectorForParams{} - for _, nearTextDep := range s.nearTextDeps { - vectorSearchers[nearTextDep.ModuleName()] = s.vectorSearches(nearTextDep) - } - return vectorSearchers -} - -func (s *Searcher) vectorSearches(nearTextDep modulecapabilities.Dependency) map[string]modulecapabilities.VectorForParams { - vectorSearches := map[string]modulecapabilities.VectorForParams{} - vectorFromAsk := &vectorFromAskParam{nearTextDep} - vectorSearches["ask"] = vectorFromAsk.vectorForAskParamFn - return vectorSearches -} diff --git a/modules/qna-openai/clients/qna.go b/modules/qna-openai/clients/qna.go deleted file mode 100644 index 27588101e6d8799a22f6dc3c36888c02f1fb1ddf..0000000000000000000000000000000000000000 --- a/modules/qna-openai/clients/qna.go +++ /dev/null @@ -1,251 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "strings" - "time" - - "github.com/weaviate/weaviate/usecases/modulecomponents" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/modules/qna-openai/config" - "github.com/weaviate/weaviate/modules/qna-openai/ent" -) - -func buildUrl(baseURL, resourceName, deploymentID string) (string, error) { - ///X update with base url - if resourceName != "" && deploymentID != "" { - host := "https://" + resourceName + ".openai.azure.com" - path := "openai/deployments/" + deploymentID + "/completions" - queryParam := "api-version=2022-12-01" - return fmt.Sprintf("%s/%s?%s", host, path, queryParam), nil - } - host := baseURL - path := "/v1/completions" - return url.JoinPath(host, path) -} - -type qna struct { - openAIApiKey string - openAIOrganization string - azureApiKey string - buildUrlFn func(baseURL, resourceName, deploymentID string) (string, error) - httpClient *http.Client - logger logrus.FieldLogger -} - -func New(openAIApiKey, openAIOrganization, azureApiKey string, timeout time.Duration, logger logrus.FieldLogger) *qna { - return &qna{ - openAIApiKey: openAIApiKey, - openAIOrganization: openAIOrganization, - azureApiKey: azureApiKey, - httpClient: &http.Client{Timeout: timeout}, - buildUrlFn: buildUrl, - logger: logger, - } -} - -func (v *qna) Answer(ctx context.Context, text, question string, cfg moduletools.ClassConfig) (*ent.AnswerResult, error) { - prompt := v.generatePrompt(text, question) - - settings := config.NewClassSettings(cfg) - - body, err := json.Marshal(answersInput{ - Prompt: prompt, - Model: settings.Model(), - MaxTokens: settings.MaxTokens(), - Temperature: settings.Temperature(), - Stop: []string{"\n"}, - FrequencyPenalty: settings.FrequencyPenalty(), - PresencePenalty: settings.PresencePenalty(), - TopP: settings.TopP(), - }) - if err != nil { - return nil, errors.Wrapf(err, "marshal body") - } - - oaiUrl, err := v.buildOpenAIUrl(ctx, settings.BaseURL(), settings.ResourceName(), settings.DeploymentID()) - if err != nil { - return nil, errors.Wrap(err, "join OpenAI API host and path") - } - fmt.Printf("using the OpenAI URL: %v\n", oaiUrl) - req, err := http.NewRequestWithContext(ctx, "POST", oaiUrl, - bytes.NewReader(body)) - if err != nil { - return nil, errors.Wrap(err, "create POST request") - } - apiKey, err := v.getApiKey(ctx, settings.IsAzure()) - if err != nil { - return nil, errors.Wrapf(err, "OpenAI API Key") - } - req.Header.Add(v.getApiKeyHeaderAndValue(apiKey, settings.IsAzure())) - if openAIOrganization := v.getOpenAIOrganization(ctx); openAIOrganization != "" { - req.Header.Add("OpenAI-Organization", openAIOrganization) - } - req.Header.Add("Content-Type", "application/json") - - res, err := v.httpClient.Do(req) - if err != nil { - return nil, errors.Wrap(err, "send POST request") - } - defer res.Body.Close() - - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - return nil, errors.Wrap(err, "read response body") - } - - var resBody answersResponse - if err := json.Unmarshal(bodyBytes, &resBody); err != nil { - return nil, errors.Wrap(err, "unmarshal response body") - } - - if res.StatusCode != 200 || resBody.Error != nil { - return nil, v.getError(res.StatusCode, resBody.Error, settings.IsAzure()) - } - - if len(resBody.Choices) > 0 && resBody.Choices[0].Text != "" { - return &ent.AnswerResult{ - Text: text, - Question: question, - Answer: &resBody.Choices[0].Text, - }, nil - } - return &ent.AnswerResult{ - Text: text, - Question: question, - Answer: nil, - }, nil -} - -func (v *qna) buildOpenAIUrl(ctx context.Context, baseURL, resourceName, deploymentID string) (string, error) { - passedBaseURL := baseURL - if headerBaseURL := v.getValueFromContext(ctx, "X-Openai-Baseurl"); headerBaseURL != "" { - passedBaseURL = headerBaseURL - } - return v.buildUrlFn(passedBaseURL, resourceName, deploymentID) -} - -func (v *qna) getError(statusCode int, resBodyError *openAIApiError, isAzure bool) error { - endpoint := "OpenAI API" - if isAzure { - endpoint = "Azure OpenAI API" - } - if resBodyError != nil { - return fmt.Errorf("connection to: %s failed with status: %d error: %v", endpoint, statusCode, resBodyError.Message) - } - return fmt.Errorf("connection to: %s failed with status: %d", endpoint, statusCode) -} - -func (v *qna) getApiKeyHeaderAndValue(apiKey string, isAzure bool) (string, string) { - if isAzure { - return "api-key", apiKey - } - return "Authorization", fmt.Sprintf("Bearer %s", apiKey) -} - -func (v *qna) generatePrompt(text string, question string) string { - return fmt.Sprintf(`'Please answer the question according to the above context. - -=== -Context: %v -=== -Q: %v -A:`, strings.ReplaceAll(text, "\n", " "), question) -} - -func (v *qna) getApiKey(ctx context.Context, isAzure bool) (string, error) { - var apiKey, envVar string - - if isAzure { - apiKey = "X-Azure-Api-Key" - envVar = "AZURE_APIKEY" - if len(v.azureApiKey) > 0 { - return v.azureApiKey, nil - } - } else { - apiKey = "X-Openai-Api-Key" - envVar = "OPENAI_APIKEY" - if len(v.openAIApiKey) > 0 { - return v.openAIApiKey, nil - } - } - - return v.getApiKeyFromContext(ctx, apiKey, envVar) -} - -func (v *qna) getApiKeyFromContext(ctx context.Context, apiKey, envVar string) (string, error) { - if apiKeyValue := v.getValueFromContext(ctx, apiKey); apiKeyValue != "" { - return apiKeyValue, nil - } - return "", fmt.Errorf("no api key found neither in request header: %s nor in environment variable under %s", apiKey, envVar) -} - -func (v *qna) getValueFromContext(ctx context.Context, key string) string { - if value := ctx.Value(key); value != nil { - if keyHeader, ok := value.([]string); ok && len(keyHeader) > 0 && len(keyHeader[0]) > 0 { - return keyHeader[0] - } - } - // try getting header from GRPC if not successful - if apiKey := modulecomponents.GetValueFromGRPC(ctx, key); len(apiKey) > 0 && len(apiKey[0]) > 0 { - return apiKey[0] - } - return "" -} - -func (v *qna) getOpenAIOrganization(ctx context.Context) string { - if value := v.getValueFromContext(ctx, "X-Openai-Organization"); value != "" { - return value - } - return v.openAIOrganization -} - -type answersInput struct { - Prompt string `json:"prompt"` - Model string `json:"model"` - MaxTokens float64 `json:"max_tokens"` - Temperature float64 `json:"temperature"` - Stop []string `json:"stop"` - FrequencyPenalty float64 `json:"frequency_penalty"` - PresencePenalty float64 `json:"presence_penalty"` - TopP float64 `json:"top_p"` -} - -type answersResponse struct { - Choices []choice - Error *openAIApiError `json:"error,omitempty"` -} - -type choice struct { - FinishReason string - Index float32 - Logprobs string - Text string -} - -type openAIApiError struct { - Message string `json:"message"` - Type string `json:"type"` - Param string `json:"param"` - Code json.Number `json:"code"` -} diff --git a/modules/qna-openai/clients/qna_meta.go b/modules/qna-openai/clients/qna_meta.go deleted file mode 100644 index 54999d57466dbf45830ebf75efb872a443b6cc4b..0000000000000000000000000000000000000000 --- a/modules/qna-openai/clients/qna_meta.go +++ /dev/null @@ -1,19 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -func (v *qna) MetaInfo() (map[string]interface{}, error) { - return map[string]interface{}{ - "name": "OpenAI Question & Answering Module", - "documentationHref": "https://platform.openai.com/docs/api-reference/completions", - }, nil -} diff --git a/modules/qna-openai/clients/qna_meta_test.go b/modules/qna-openai/clients/qna_meta_test.go deleted file mode 100644 index ac5fb29eefd6e2eeabeaa88f9c3cab830da89d0e..0000000000000000000000000000000000000000 --- a/modules/qna-openai/clients/qna_meta_test.go +++ /dev/null @@ -1,67 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func TestGetMeta(t *testing.T) { - t.Run("when the server is providing meta", func(t *testing.T) { - server := httptest.NewServer(&testMetaHandler{t: t}) - defer server.Close() - c := New("", "", "", 0, nullLogger()) - meta, err := c.MetaInfo() - - assert.Nil(t, err) - assert.NotNil(t, meta) - metaModel := meta["name"] - assert.True(t, metaModel != nil) - documentationHref := meta["documentationHref"] - assert.True(t, documentationHref != nil) - }) -} - -type testMetaHandler struct { - t *testing.T - // the test handler will report as not ready before the time has passed - readyTime time.Time -} - -func (f *testMetaHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - assert.Equal(f.t, "/meta", r.URL.String()) - assert.Equal(f.t, http.MethodGet, r.Method) - - if time.Since(f.readyTime) < 0 { - w.WriteHeader(http.StatusServiceUnavailable) - } - - w.Write([]byte(f.metaInfo())) -} - -func (f *testMetaHandler) metaInfo() string { - return `{ - "hostname": "http://127.0.0.1:8080", - "modules": { - "qna-openai": { - "documentationHref": "https://platform.openai.com/docs/api-reference/completions", - "name": "OpenAI Question & Answering Module" - } - }, - "version": "1.16.0" -}` -} diff --git a/modules/qna-openai/clients/qna_test.go b/modules/qna-openai/clients/qna_test.go deleted file mode 100644 index 9f3e7fa9d929531ec5f0d72f35093f48984bec48..0000000000000000000000000000000000000000 --- a/modules/qna-openai/clients/qna_test.go +++ /dev/null @@ -1,186 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "context" - "encoding/json" - "io" - "net/http" - "net/http/httptest" - "testing" - - "github.com/sirupsen/logrus" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/modules/qna-openai/ent" -) - -func nullLogger() logrus.FieldLogger { - l, _ := test.NewNullLogger() - return l -} - -func TestGetAnswer(t *testing.T) { - t.Run("when the server has a successful answer ", func(t *testing.T) { - handler := &testAnswerHandler{ - t: t, - answer: answersResponse{ - Choices: []choice{{ - FinishReason: "test", - Index: 0, - Logprobs: "", - Text: "John", - }}, - Error: nil, - }, - } - server := httptest.NewServer(handler) - defer server.Close() - - c := New("openAIApiKey", "", "", 0, nullLogger()) - c.buildUrlFn = func(baseURL, resourceName, deploymentID string) (string, error) { - return buildUrl(server.URL, resourceName, deploymentID) - } - - expected := ent.AnswerResult{ - Text: "My name is John", - Question: "What is my name?", - Answer: ptString("John"), - } - - res, err := c.Answer(context.Background(), "My name is John", "What is my name?", nil) - - assert.Nil(t, err) - assert.Equal(t, expected, *res) - }) - - t.Run("when the server has a an error", func(t *testing.T) { - server := httptest.NewServer(&testAnswerHandler{ - t: t, - answer: answersResponse{ - Error: &openAIApiError{ - Message: "some error from the server", - }, - }, - }) - defer server.Close() - - c := New("openAIApiKey", "", "", 0, nullLogger()) - c.buildUrlFn = func(baseURL, resourceName, deploymentID string) (string, error) { - return buildUrl(server.URL, resourceName, deploymentID) - } - - _, err := c.Answer(context.Background(), "My name is John", "What is my name?", nil) - - require.NotNil(t, err) - assert.Error(t, err, "connection to OpenAI failed with status: 500 error: some error from the server") - }) - - t.Run("when X-OpenAI-BaseURL header is passed", func(t *testing.T) { - c := New("openAIApiKey", "", "", 0, nullLogger()) - - ctxWithValue := context.WithValue(context.Background(), - "X-Openai-Baseurl", []string{"http://base-url-passed-in-header.com"}) - - buildURL, err := c.buildOpenAIUrl(ctxWithValue, "http://default-url.com", "", "") - require.NoError(t, err) - assert.Equal(t, "http://base-url-passed-in-header.com/v1/completions", buildURL) - - buildURL, err = c.buildOpenAIUrl(context.TODO(), "http://default-url.com", "", "") - require.NoError(t, err) - assert.Equal(t, "http://default-url.com/v1/completions", buildURL) - }) -} - -type testAnswerHandler struct { - t *testing.T - // the test handler will report as not ready before the time has passed - answer answersResponse -} - -func (f *testAnswerHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - assert.Equal(f.t, "/v1/completions", r.URL.String()) - assert.Equal(f.t, http.MethodPost, r.Method) - - if f.answer.Error != nil && f.answer.Error.Message != "" { - outBytes, err := json.Marshal(f.answer) - require.Nil(f.t, err) - - w.WriteHeader(http.StatusInternalServerError) - w.Write(outBytes) - return - } - - bodyBytes, err := io.ReadAll(r.Body) - require.Nil(f.t, err) - defer r.Body.Close() - - var b map[string]interface{} - require.Nil(f.t, json.Unmarshal(bodyBytes, &b)) - - outBytes, err := json.Marshal(f.answer) - require.Nil(f.t, err) - - w.Write(outBytes) -} - -func TestOpenAIApiErrorDecode(t *testing.T) { - t.Run("getModelStringQuery", func(t *testing.T) { - type args struct { - response []byte - } - tests := []struct { - name string - args args - want string - }{ - { - name: "Error code: missing property", - args: args{ - response: []byte(`{"message": "failed", "type": "error", "param": "arg..."}`), - }, - want: "", - }, - { - name: "Error code: as int", - args: args{ - response: []byte(`{"message": "failed", "type": "error", "param": "arg...", "code": 500}`), - }, - want: "500", - }, - { - name: "Error code as string", - args: args{ - response: []byte(`{"message": "failed", "type": "error", "param": "arg...", "code": "500"}`), - }, - want: "500", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - var got *openAIApiError - err := json.Unmarshal(tt.args.response, &got) - require.NoError(t, err) - - if got.Code.String() != tt.want { - t.Errorf("OpenAIerror.code = %v, want %v", got.Code, tt.want) - } - }) - } - }) -} - -func ptString(in string) *string { - return &in -} diff --git a/modules/qna-openai/config.go b/modules/qna-openai/config.go deleted file mode 100644 index b1042b58d7c01a23b98ee49b8699f1a906b38033..0000000000000000000000000000000000000000 --- a/modules/qna-openai/config.go +++ /dev/null @@ -1,41 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modqnaopenai - -import ( - "context" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/modules/qna-openai/config" -) - -func (m *QnAModule) ClassConfigDefaults() map[string]interface{} { - return map[string]interface{}{} -} - -func (m *QnAModule) PropertyConfigDefaults( - dt *schema.DataType, -) map[string]interface{} { - return map[string]interface{}{} -} - -func (m *QnAModule) ValidateClass(ctx context.Context, - class *models.Class, cfg moduletools.ClassConfig, -) error { - settings := config.NewClassSettings(cfg) - return settings.Validate(class) -} - -var _ = modulecapabilities.ClassConfigurator(New()) diff --git a/modules/qna-openai/config/class_settings.go b/modules/qna-openai/config/class_settings.go deleted file mode 100644 index 928401b5cd2c4c773a403aaa1b36285fe1dcd512..0000000000000000000000000000000000000000 --- a/modules/qna-openai/config/class_settings.go +++ /dev/null @@ -1,221 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package config - -import ( - "encoding/json" - "fmt" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/moduletools" -) - -const ( - modelProperty = "model" - temperatureProperty = "temperature" - maxTokensProperty = "maxTokens" - frequencyPenaltyProperty = "frequencyPenalty" - presencePenaltyProperty = "presencePenalty" - topPProperty = "topP" - baseURLProperty = "baseURL" -) - -var ( - DefaultOpenAIModel = "text-ada-001" - DefaultOpenAITemperature float64 = 0.0 - DefaultOpenAIMaxTokens float64 = 16 - DefaultOpenAIFrequencyPenalty float64 = 0.0 - DefaultOpenAIPresencePenalty float64 = 0.0 - DefaultOpenAITopP float64 = 1.0 - DefaultOpenAIBaseURL = "https://api.openai.com" -) - -var maxTokensForModel = map[string]float64{ - "text-ada-001": 2048, - "text-babbage-001": 2048, - "text-curie-001": 2048, - "text-davinci-002": 4000, - "text-davinci-003": 4000, - "gpt-3.5-turbo-instruct": 4000, -} - -var availableOpenAIModels = []string{ - "text-ada-001", - "text-babbage-001", - "text-curie-001", - "text-davinci-002", - "text-davinci-003", - "gpt-3.5-turbo-instruct", -} - -type classSettings struct { - cfg moduletools.ClassConfig -} - -func NewClassSettings(cfg moduletools.ClassConfig) *classSettings { - return &classSettings{cfg: cfg} -} - -func (ic *classSettings) Validate(class *models.Class) error { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return errors.New("empty config") - } - - model := ic.getStringProperty(modelProperty, DefaultOpenAIModel) - if model == nil || !ic.validateOpenAISetting(*model, availableOpenAIModels) { - return errors.Errorf("wrong OpenAI model name, available model names are: %v", availableOpenAIModels) - } - - temperature := ic.getFloatProperty(temperatureProperty, &DefaultOpenAITemperature) - if temperature == nil || (*temperature < 0 || *temperature > 1) { - return errors.Errorf("Wrong temperature configuration, values are between 0.0 and 1.0") - } - - maxTokens := ic.getFloatProperty(maxTokensProperty, &DefaultOpenAIMaxTokens) - if maxTokens == nil || (*maxTokens < 0 || *maxTokens > getMaxTokensForModel(*model)) { - return errors.Errorf("Wrong maxTokens configuration, values are should have a minimal value of 1 and max is dependant on the model used") - } - - frequencyPenalty := ic.getFloatProperty(frequencyPenaltyProperty, &DefaultOpenAIFrequencyPenalty) - if frequencyPenalty == nil || (*frequencyPenalty < 0 || *frequencyPenalty > 1) { - return errors.Errorf("Wrong frequencyPenalty configuration, values are between 0.0 and 1.0") - } - - presencePenalty := ic.getFloatProperty(presencePenaltyProperty, &DefaultOpenAIPresencePenalty) - if presencePenalty == nil || (*presencePenalty < 0 || *presencePenalty > 1) { - return errors.Errorf("Wrong presencePenalty configuration, values are between 0.0 and 1.0") - } - - topP := ic.getFloatProperty(topPProperty, &DefaultOpenAITopP) - if topP == nil || (*topP < 0 || *topP > 5) { - return errors.Errorf("Wrong topP configuration, values are should have a minimal value of 1 and max of 5") - } - - err := ic.validateAzureConfig(ic.ResourceName(), ic.DeploymentID()) - if err != nil { - return err - } - - return nil -} - -func (ic *classSettings) getStringProperty(name, defaultValue string) *string { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return &defaultValue - } - - model, ok := ic.cfg.ClassByModuleName("qna-openai")[name] - if ok { - asString, ok := model.(string) - if ok { - return &asString - } - var empty string - return &empty - } - return &defaultValue -} - -func (ic *classSettings) getFloatProperty(name string, defaultValue *float64) *float64 { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return defaultValue - } - - val, ok := ic.cfg.ClassByModuleName("qna-openai")[name] - if ok { - asFloat, ok := val.(float64) - if ok { - return &asFloat - } - asNumber, ok := val.(json.Number) - if ok { - asFloat, _ := asNumber.Float64() - return &asFloat - } - asInt, ok := val.(int) - if ok { - asFloat := float64(asInt) - return &asFloat - } - var wrongVal float64 = -1.0 - return &wrongVal - } - - if defaultValue != nil { - return defaultValue - } - return nil -} - -func getMaxTokensForModel(model string) float64 { - return maxTokensForModel[model] -} - -func (ic *classSettings) validateOpenAISetting(value string, availableValues []string) bool { - for i := range availableValues { - if value == availableValues[i] { - return true - } - } - return false -} - -func (ic *classSettings) Model() string { - return *ic.getStringProperty(modelProperty, DefaultOpenAIModel) -} - -func (ic *classSettings) MaxTokens() float64 { - return *ic.getFloatProperty(maxTokensProperty, &DefaultOpenAIMaxTokens) -} - -func (ic *classSettings) BaseURL() string { - return *ic.getStringProperty(baseURLProperty, DefaultOpenAIBaseURL) -} - -func (ic *classSettings) Temperature() float64 { - return *ic.getFloatProperty(temperatureProperty, &DefaultOpenAITemperature) -} - -func (ic *classSettings) FrequencyPenalty() float64 { - return *ic.getFloatProperty(frequencyPenaltyProperty, &DefaultOpenAIFrequencyPenalty) -} - -func (ic *classSettings) PresencePenalty() float64 { - return *ic.getFloatProperty(presencePenaltyProperty, &DefaultOpenAIPresencePenalty) -} - -func (ic *classSettings) TopP() float64 { - return *ic.getFloatProperty(topPProperty, &DefaultOpenAITopP) -} - -func (ic *classSettings) ResourceName() string { - return *ic.getStringProperty("resourceName", "") -} - -func (ic *classSettings) DeploymentID() string { - return *ic.getStringProperty("deploymentId", "") -} - -func (ic *classSettings) IsAzure() bool { - return ic.ResourceName() != "" && ic.DeploymentID() != "" -} - -func (ic *classSettings) validateAzureConfig(resourceName string, deploymentId string) error { - if (resourceName == "" && deploymentId != "") || (resourceName != "" && deploymentId == "") { - return fmt.Errorf("both resourceName and deploymentId must be provided") - } - return nil -} diff --git a/modules/qna-openai/config/class_settings_test.go b/modules/qna-openai/config/class_settings_test.go deleted file mode 100644 index 3a287e8a12a7f736cb9aa9ed633ac63324d6a1e4..0000000000000000000000000000000000000000 --- a/modules/qna-openai/config/class_settings_test.go +++ /dev/null @@ -1,215 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package config - -import ( - "testing" - - "github.com/pkg/errors" - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/entities/moduletools" -) - -func Test_classSettings_Validate(t *testing.T) { - tests := []struct { - name string - cfg moduletools.ClassConfig - wantModel string - wantMaxTokens float64 - wantTemperature float64 - wantTopP float64 - wantFrequencyPenalty float64 - wantPresencePenalty float64 - wantResourceName string - wantDeploymentID string - wantIsAzure bool - wantErr error - wantBaseURL string - }{ - { - name: "Happy flow", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{}, - }, - wantModel: "text-ada-001", - wantMaxTokens: 16, - wantTemperature: 0.0, - wantTopP: 1, - wantFrequencyPenalty: 0.0, - wantPresencePenalty: 0.0, - wantErr: nil, - wantBaseURL: "https://api.openai.com", - }, - { - name: "Everything non default configured", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "model": "text-babbage-001", - "maxTokens": 100, - "temperature": 0.5, - "topP": 3, - "frequencyPenalty": 0.1, - "presencePenalty": 0.9, - "baseURL": "https://openai.proxy.dev", - }, - }, - wantModel: "text-babbage-001", - wantMaxTokens: 100, - wantTemperature: 0.5, - wantTopP: 3, - wantFrequencyPenalty: 0.1, - wantPresencePenalty: 0.9, - wantBaseURL: "https://openai.proxy.dev", - wantErr: nil, - }, - { - name: "Azure OpenAI config", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "resourceName": "weaviate", - "deploymentId": "text-ada-001", - }, - }, - wantModel: "text-ada-001", - wantResourceName: "weaviate", - wantDeploymentID: "text-ada-001", - wantIsAzure: true, - wantMaxTokens: 16, - wantTemperature: 0.0, - wantTopP: 1, - wantFrequencyPenalty: 0.0, - wantPresencePenalty: 0.0, - wantErr: nil, - wantBaseURL: "https://api.openai.com", - }, - { - name: "Wrong model data type configured", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "model": true, - }, - }, - wantErr: errors.Errorf("wrong OpenAI model name, available model names are: %v", availableOpenAIModels), - }, - { - name: "Wrong model data type configured", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "model": "this-is-a-non-existing-model", - }, - }, - wantErr: errors.Errorf("wrong OpenAI model name, available model names are: %v", availableOpenAIModels), - }, - { - name: "Wrong maxTokens configured", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "maxTokens": true, - }, - }, - wantErr: errors.Errorf("Wrong maxTokens configuration, values are should have a minimal value of 1 and max is dependant on the model used"), - }, - { - name: "Wrong temperature configured", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "temperature": true, - }, - }, - wantErr: errors.Errorf("Wrong temperature configuration, values are between 0.0 and 1.0"), - }, - { - name: "Wrong frequencyPenalty configured", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "frequencyPenalty": true, - }, - }, - wantErr: errors.Errorf("Wrong frequencyPenalty configuration, values are between 0.0 and 1.0"), - }, - { - name: "Wrong presencePenalty configured", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "presencePenalty": true, - }, - }, - wantErr: errors.Errorf("Wrong presencePenalty configuration, values are between 0.0 and 1.0"), - }, - { - name: "Wrong topP configured", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "topP": true, - }, - }, - wantErr: errors.Errorf("Wrong topP configuration, values are should have a minimal value of 1 and max of 5"), - }, - { - name: "Wrong Azure OpenAI config - empty deploymentId", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "resourceName": "resource-name", - }, - }, - wantErr: errors.Errorf("both resourceName and deploymentId must be provided"), - }, - { - name: "Wrong Azure OpenAI config - empty resourceName", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "deploymentId": "ada", - }, - }, - wantErr: errors.Errorf("both resourceName and deploymentId must be provided"), - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ic := NewClassSettings(tt.cfg) - if tt.wantErr != nil { - assert.EqualError(t, tt.wantErr, ic.Validate(nil).Error()) - } else { - assert.Equal(t, tt.wantModel, ic.Model()) - assert.Equal(t, tt.wantMaxTokens, ic.MaxTokens()) - assert.Equal(t, tt.wantTemperature, ic.Temperature()) - assert.Equal(t, tt.wantTopP, ic.TopP()) - assert.Equal(t, tt.wantFrequencyPenalty, ic.FrequencyPenalty()) - assert.Equal(t, tt.wantPresencePenalty, ic.PresencePenalty()) - assert.Equal(t, tt.wantResourceName, ic.ResourceName()) - assert.Equal(t, tt.wantDeploymentID, ic.DeploymentID()) - assert.Equal(t, tt.wantIsAzure, ic.IsAzure()) - assert.Equal(t, tt.wantBaseURL, ic.BaseURL()) - } - }) - } -} - -type fakeClassConfig struct { - classConfig map[string]interface{} -} - -func (f fakeClassConfig) Class() map[string]interface{} { - return f.classConfig -} - -func (f fakeClassConfig) Tenant() string { - return "" -} - -func (f fakeClassConfig) ClassByModuleName(moduleName string) map[string]interface{} { - return f.classConfig -} - -func (f fakeClassConfig) Property(propName string) map[string]interface{} { - return nil -} diff --git a/modules/qna-openai/dependency/dependency.go b/modules/qna-openai/dependency/dependency.go deleted file mode 100644 index 170159b89ec36fed0d588cf59f21a3b48f819ade..0000000000000000000000000000000000000000 --- a/modules/qna-openai/dependency/dependency.go +++ /dev/null @@ -1,42 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package dependency - -import "github.com/weaviate/weaviate/entities/modulecapabilities" - -type NearTextDependecy struct { - moduleName string - argument modulecapabilities.GraphQLArgument - searcher modulecapabilities.VectorForParams -} - -func New(moduleName string, argument modulecapabilities.GraphQLArgument, - searcher modulecapabilities.VectorForParams, -) *NearTextDependecy { - return &NearTextDependecy{moduleName, argument, searcher} -} - -func (d *NearTextDependecy) Argument() string { - return "nearText" -} - -func (d *NearTextDependecy) ModuleName() string { - return d.moduleName -} - -func (d *NearTextDependecy) GraphQLArgument() modulecapabilities.GraphQLArgument { - return d.argument -} - -func (d *NearTextDependecy) VectorSearch() modulecapabilities.VectorForParams { - return d.searcher -} diff --git a/modules/qna-openai/ent/vectorization_result.go b/modules/qna-openai/ent/vectorization_result.go deleted file mode 100644 index 539d58c0733d8fd5d9b3426d675c2646099f3927..0000000000000000000000000000000000000000 --- a/modules/qna-openai/ent/vectorization_result.go +++ /dev/null @@ -1,18 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package ent - -type AnswerResult struct { - Text string - Question string - Answer *string -} diff --git a/modules/qna-openai/module.go b/modules/qna-openai/module.go deleted file mode 100644 index edb98b125186d623cfa20216cadf3ca7c3a6eafe..0000000000000000000000000000000000000000 --- a/modules/qna-openai/module.go +++ /dev/null @@ -1,168 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modqnaopenai - -import ( - "context" - "net/http" - "os" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - qnaadditional "github.com/weaviate/weaviate/modules/qna-openai/additional" - qnaadditionalanswer "github.com/weaviate/weaviate/modules/qna-openai/additional/answer" - qnaask "github.com/weaviate/weaviate/modules/qna-openai/ask" - "github.com/weaviate/weaviate/modules/qna-openai/clients" - qnaadependency "github.com/weaviate/weaviate/modules/qna-openai/dependency" - "github.com/weaviate/weaviate/modules/qna-openai/ent" -) - -const Name = "qna-openai" - -func New() *QnAModule { - return &QnAModule{} -} - -type QnAModule struct { - qna qnaClient - graphqlProvider modulecapabilities.GraphQLArguments - searcher modulecapabilities.DependencySearcher - additionalPropertiesProvider modulecapabilities.AdditionalProperties - nearTextDependencies []modulecapabilities.Dependency - askTextTransformer modulecapabilities.TextTransform -} - -type qnaClient interface { - Answer(ctx context.Context, text, question string, cfg moduletools.ClassConfig) (*ent.AnswerResult, error) - MetaInfo() (map[string]interface{}, error) -} - -func (m *QnAModule) Name() string { - return Name -} - -func (m *QnAModule) Type() modulecapabilities.ModuleType { - return modulecapabilities.Text2TextQnA -} - -func (m *QnAModule) Init(ctx context.Context, - params moduletools.ModuleInitParams, -) error { - if err := m.initAdditional(ctx, params.GetConfig().ModuleHttpClientTimeout, params.GetLogger()); err != nil { - return errors.Wrap(err, "init q/a") - } - - return nil -} - -func (m *QnAModule) InitExtension(modules []modulecapabilities.Module) error { - var textTransformer modulecapabilities.TextTransform - for _, module := range modules { - if module.Name() == m.Name() { - continue - } - if arg, ok := module.(modulecapabilities.TextTransformers); ok { - if arg != nil && arg.TextTransformers() != nil { - textTransformer = arg.TextTransformers()["ask"] - } - } - } - - m.askTextTransformer = textTransformer - - if err := m.initAskProvider(); err != nil { - return errors.Wrap(err, "init ask provider") - } - - return nil -} - -func (m *QnAModule) InitDependency(modules []modulecapabilities.Module) error { - nearTextDependencies := []modulecapabilities.Dependency{} - for _, module := range modules { - if module.Name() == m.Name() { - continue - } - var argument modulecapabilities.GraphQLArgument - var searcher modulecapabilities.VectorForParams - if arg, ok := module.(modulecapabilities.GraphQLArguments); ok { - if arg != nil && arg.Arguments() != nil { - if nearTextArg, ok := arg.Arguments()["nearText"]; ok { - argument = nearTextArg - } - } - } - if arg, ok := module.(modulecapabilities.Searcher); ok { - if arg != nil && arg.VectorSearches() != nil { - if nearTextSearcher, ok := arg.VectorSearches()["nearText"]; ok { - searcher = nearTextSearcher - } - } - } - - if argument.ExtractFunction != nil && searcher != nil { - nearTextDependency := qnaadependency.New(module.Name(), argument, searcher) - nearTextDependencies = append(nearTextDependencies, nearTextDependency) - } - } - if len(nearTextDependencies) == 0 { - return errors.New("nearText dependecy not present") - } - - m.nearTextDependencies = nearTextDependencies - - if err := m.initAskSearcher(); err != nil { - return errors.Wrap(err, "init ask searcher") - } - - return nil -} - -func (m *QnAModule) initAdditional(ctx context.Context, timeout time.Duration, - logger logrus.FieldLogger, -) error { - openAIApiKey := os.Getenv("OPENAI_APIKEY") - openAIOrganization := os.Getenv("OPENAI_ORGANIZATION") - azureApiKey := os.Getenv("AZURE_APIKEY") - - client := clients.New(openAIApiKey, openAIOrganization, azureApiKey, timeout, logger) - - m.qna = client - - answerProvider := qnaadditionalanswer.New(m.qna, qnaask.NewParamsHelper()) - m.additionalPropertiesProvider = qnaadditional.New(answerProvider) - - return nil -} - -func (m *QnAModule) RootHandler() http.Handler { - // TODO: remove once this is a capability interface - return nil -} - -func (m *QnAModule) MetaInfo() (map[string]interface{}, error) { - return m.qna.MetaInfo() -} - -func (m *QnAModule) AdditionalProperties() map[string]modulecapabilities.AdditionalProperty { - return m.additionalPropertiesProvider.AdditionalProperties() -} - -// verify we implement the modules.Module interface -var ( - _ = modulecapabilities.Module(New()) - _ = modulecapabilities.AdditionalProperties(New()) - _ = modulecapabilities.MetaProvider(New()) -) diff --git a/modules/qna-transformers/additional/answer/answer.go b/modules/qna-transformers/additional/answer/answer.go deleted file mode 100644 index 95502441bf9d21dc39efa7d7076dfa7240ce4293..0000000000000000000000000000000000000000 --- a/modules/qna-transformers/additional/answer/answer.go +++ /dev/null @@ -1,69 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package answer - -import ( - "context" - "errors" - - "github.com/tailor-inc/graphql" - "github.com/tailor-inc/graphql/language/ast" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/search" - "github.com/weaviate/weaviate/modules/qna-transformers/ent" -) - -type Params struct{} - -type qnaClient interface { - Answer(ctx context.Context, - text, question string) (*ent.AnswerResult, error) -} - -type paramsHelper interface { - GetQuestion(params interface{}) string - GetProperties(params interface{}) []string - GetCertainty(params interface{}) float64 - GetDistance(params interface{}) float64 - GetRerank(params interface{}) bool -} - -type AnswerProvider struct { - qna qnaClient - paramsHelper -} - -func New(qna qnaClient, paramsHelper paramsHelper) *AnswerProvider { - return &AnswerProvider{qna, paramsHelper} -} - -func (p *AnswerProvider) AdditionalPropertyDefaultValue() interface{} { - return &Params{} -} - -func (p *AnswerProvider) ExtractAdditionalFn(param []*ast.Argument) interface{} { - return &Params{} -} - -func (p *AnswerProvider) AdditionalFieldFn(classname string) *graphql.Field { - return p.additionalAnswerField(classname) -} - -func (p *AnswerProvider) AdditionalPropertyFn(ctx context.Context, - in []search.Result, params interface{}, limit *int, - argumentModuleParams map[string]interface{}, cfg moduletools.ClassConfig, -) ([]search.Result, error) { - if parameters, ok := params.(*Params); ok { - return p.findAnswer(ctx, in, parameters, limit, argumentModuleParams) - } - return nil, errors.New("wrong parameters") -} diff --git a/modules/qna-transformers/additional/answer/answer_graphql_field.go b/modules/qna-transformers/additional/answer/answer_graphql_field.go deleted file mode 100644 index 34b0ed7d7e9de5acf38cfaa8f225c3d113ea6ca7..0000000000000000000000000000000000000000 --- a/modules/qna-transformers/additional/answer/answer_graphql_field.go +++ /dev/null @@ -1,35 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package answer - -import ( - "fmt" - - "github.com/tailor-inc/graphql" -) - -func (p *AnswerProvider) additionalAnswerField(classname string) *graphql.Field { - return &graphql.Field{ - Type: graphql.NewObject(graphql.ObjectConfig{ - Name: fmt.Sprintf("%sAdditionalAnswer", classname), - Fields: graphql.Fields{ - "result": &graphql.Field{Type: graphql.String}, - "startPosition": &graphql.Field{Type: graphql.Int}, - "endPosition": &graphql.Field{Type: graphql.Int}, - "property": &graphql.Field{Type: graphql.String}, - "certainty": &graphql.Field{Type: graphql.Float}, - "distance": &graphql.Field{Type: graphql.Float}, - "hasAnswer": &graphql.Field{Type: graphql.Boolean}, - }, - }), - } -} diff --git a/modules/qna-transformers/additional/answer/answer_graphql_field_test.go b/modules/qna-transformers/additional/answer/answer_graphql_field_test.go deleted file mode 100644 index 7986123370d589883d978dcadc6c1fc1aac5dd12..0000000000000000000000000000000000000000 --- a/modules/qna-transformers/additional/answer/answer_graphql_field_test.go +++ /dev/null @@ -1,56 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package answer - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/tailor-inc/graphql" -) - -func TestAnswerField(t *testing.T) { - t.Run("should generate answer argument properly", func(t *testing.T) { - // given - answerProvider := &AnswerProvider{} - classname := "Class" - - // when - answer := answerProvider.additionalAnswerField(classname) - - // then - // the built graphQL field needs to support this structure: - // Type: { - // answer: { - // result: "answer", - // startPosition: 1 - // endPosition: 2 - // distance: 0.2 - // property: "propName" - // hasAnswer: true - // } - // } - assert.NotNil(t, answer) - assert.Equal(t, "ClassAdditionalAnswer", answer.Type.Name()) - assert.NotNil(t, answer.Type) - answerObject, answerObjectOK := answer.Type.(*graphql.Object) - assert.True(t, answerObjectOK) - assert.Equal(t, 7, len(answerObject.Fields())) - assert.NotNil(t, answerObject.Fields()["result"]) - assert.NotNil(t, answerObject.Fields()["startPosition"]) - assert.NotNil(t, answerObject.Fields()["endPosition"]) - assert.NotNil(t, answerObject.Fields()["property"]) - assert.NotNil(t, answerObject.Fields()["certainty"]) - assert.NotNil(t, answerObject.Fields()["distance"]) - assert.NotNil(t, answerObject.Fields()["hasAnswer"]) - }) -} diff --git a/modules/qna-transformers/additional/answer/answer_result.go b/modules/qna-transformers/additional/answer/answer_result.go deleted file mode 100644 index 9faff0271e9a1dce28219772d7b895f7ef85a461..0000000000000000000000000000000000000000 --- a/modules/qna-transformers/additional/answer/answer_result.go +++ /dev/null @@ -1,159 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package answer - -import ( - "context" - "errors" - "sort" - "strings" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/search" - qnamodels "github.com/weaviate/weaviate/modules/qna-transformers/additional/models" - "github.com/weaviate/weaviate/modules/qna-transformers/ent" -) - -func (p *AnswerProvider) findAnswer(ctx context.Context, - in []search.Result, params *Params, limit *int, - argumentModuleParams map[string]interface{}, -) ([]search.Result, error) { - if len(in) > 0 { - question := p.paramsHelper.GetQuestion(argumentModuleParams["ask"]) - if question == "" { - return in, errors.New("empty question") - } - properties := p.paramsHelper.GetProperties(argumentModuleParams["ask"]) - - for i := range in { - textProperties := map[string]string{} - schema := in[i].Object().Properties.(map[string]interface{}) - for property, value := range schema { - if p.containsProperty(property, properties) { - if valueString, ok := value.(string); ok && len(valueString) > 0 { - textProperties[property] = valueString - } - } - } - - texts := []string{} - for _, value := range textProperties { - texts = append(texts, value) - } - text := strings.Join(texts, " ") - if len(text) == 0 { - return in, errors.New("empty content") - } - - answer, err := p.qna.Answer(ctx, text, question) - if err != nil { - return in, err - } - - ap := in[i].AdditionalProperties - if ap == nil { - ap = models.AdditionalProperties{} - } - - if answerMeetsSimilarityThreshold(argumentModuleParams["ask"], p.paramsHelper, answer) { - propertyName, startPos, endPos := p.findProperty(answer.Answer, textProperties) - ap["answer"] = &qnamodels.Answer{ - Result: answer.Answer, - Property: propertyName, - StartPosition: startPos, - EndPosition: endPos, - Certainty: answer.Certainty, - Distance: answer.Distance, - HasAnswer: answer.Answer != nil, - } - } else { - ap["answer"] = &qnamodels.Answer{ - HasAnswer: false, - } - } - - in[i].AdditionalProperties = ap - } - } - - rerank := p.paramsHelper.GetRerank(argumentModuleParams["ask"]) - if rerank { - return p.rerank(in), nil - } - return in, nil -} - -func answerMeetsSimilarityThreshold(params interface{}, helper paramsHelper, ans *ent.AnswerResult) bool { - certainty := helper.GetCertainty(params) - if certainty > 0 && ans.Certainty != nil && *ans.Certainty < certainty { - return false - } - - distance := helper.GetDistance(params) - if distance > 0 && ans.Distance != nil && *ans.Distance > distance { - return false - } - - return true -} - -func (p *AnswerProvider) rerank(in []search.Result) []search.Result { - if len(in) > 0 { - sort.SliceStable(in, func(i, j int) bool { - return p.getAnswerCertainty(in[i]) > p.getAnswerCertainty(in[j]) - }) - } - return in -} - -func (p *AnswerProvider) getAnswerCertainty(result search.Result) float64 { - answerObj, ok := result.AdditionalProperties["answer"] - if ok { - answer, ok := answerObj.(*qnamodels.Answer) - if ok { - if answer.HasAnswer { - return *answer.Certainty - } - } - } - return 0 -} - -func (p *AnswerProvider) containsProperty(property string, properties []string) bool { - if len(properties) == 0 { - return true - } - for i := range properties { - if properties[i] == property { - return true - } - } - return false -} - -func (p *AnswerProvider) findProperty(answer *string, textProperties map[string]string) (*string, int, int) { - if answer == nil { - return nil, 0, 0 - } - lowercaseAnswer := strings.ToLower(*answer) - if len(lowercaseAnswer) > 0 { - for property, value := range textProperties { - lowercaseValue := strings.ToLower(strings.ReplaceAll(value, "\n", " ")) - if strings.Contains(lowercaseValue, lowercaseAnswer) { - startIndex := strings.Index(lowercaseValue, lowercaseAnswer) - return &property, startIndex, startIndex + len(lowercaseAnswer) - } - } - } - propertyNotFound := "" - return &propertyNotFound, 0, 0 -} diff --git a/modules/qna-transformers/additional/answer/answer_test.go b/modules/qna-transformers/additional/answer/answer_test.go deleted file mode 100644 index 58cffd959d0cc7a7546029ee20d4efb0b3fc543b..0000000000000000000000000000000000000000 --- a/modules/qna-transformers/additional/answer/answer_test.go +++ /dev/null @@ -1,577 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package answer - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/search" - qnamodels "github.com/weaviate/weaviate/modules/qna-transformers/additional/models" - "github.com/weaviate/weaviate/modules/qna-transformers/ent" -) - -func TestAdditionalAnswerProvider(t *testing.T) { - t.Run("should fail with empty content", func(t *testing.T) { - // given - qnaClient := &fakeQnAClient{} - fakeHelper := &fakeParamsHelper{} - answerProvider := New(qnaClient, fakeHelper) - in := []search.Result{ - { - ID: "some-uuid", - }, - } - fakeParams := &Params{} - limit := 1 - argumentModuleParams := map[string]interface{}{} - - // when - out, err := answerProvider.AdditionalPropertyFn(context.Background(), in, fakeParams, &limit, argumentModuleParams, nil) - - // then - require.NotNil(t, err) - require.NotEmpty(t, out) - assert.Error(t, err, "empty content") - }) - - t.Run("should fail with empty question", func(t *testing.T) { - // given - qnaClient := &fakeQnAClient{} - fakeHelper := &fakeParamsHelper{} - answerProvider := New(qnaClient, fakeHelper) - in := []search.Result{ - { - ID: "some-uuid", - Schema: map[string]interface{}{ - "content": "content", - }, - }, - } - fakeParams := &Params{} - limit := 1 - argumentModuleParams := map[string]interface{}{} - - // when - out, err := answerProvider.AdditionalPropertyFn(context.Background(), in, fakeParams, &limit, argumentModuleParams, nil) - - // then - require.NotNil(t, err) - require.NotEmpty(t, out) - assert.Error(t, err, "empty content") - }) - - t.Run("should answer", func(t *testing.T) { - // given - qnaClient := &fakeQnAClient{} - fakeHelper := &fakeParamsHelper{} - answerProvider := New(qnaClient, fakeHelper) - in := []search.Result{ - { - ID: "some-uuid", - Schema: map[string]interface{}{ - "content": "content", - }, - }, - } - fakeParams := &Params{} - limit := 1 - argumentModuleParams := map[string]interface{}{ - "ask": map[string]interface{}{ - "question": "question", - }, - } - - // when - out, err := answerProvider.AdditionalPropertyFn(context.Background(), in, fakeParams, &limit, argumentModuleParams, nil) - - // then - require.Nil(t, err) - require.NotEmpty(t, out) - assert.Equal(t, 1, len(in)) - answer, answerOK := in[0].AdditionalProperties["answer"] - assert.True(t, answerOK) - assert.NotNil(t, answer) - answerAdditional, answerAdditionalOK := answer.(*qnamodels.Answer) - assert.True(t, answerAdditionalOK) - assert.Equal(t, "answer", *answerAdditional.Result) - }) - - t.Run("should answer with property", func(t *testing.T) { - // given - qnaClient := &fakeQnAClient{} - fakeHelper := &fakeParamsHelper{} - answerProvider := New(qnaClient, fakeHelper) - in := []search.Result{ - { - ID: "some-uuid", - Schema: map[string]interface{}{ - "content": "content with answer", - "content2": "this one is just a title", - }, - }, - } - fakeParams := &Params{} - limit := 1 - argumentModuleParams := map[string]interface{}{ - "ask": map[string]interface{}{ - "question": "question", - "properties": []string{"content", "content2"}, - }, - } - - // when - out, err := answerProvider.AdditionalPropertyFn(context.Background(), in, fakeParams, &limit, argumentModuleParams, nil) - - // then - require.Nil(t, err) - require.NotEmpty(t, out) - assert.Equal(t, 1, len(in)) - answer, answerOK := in[0].AdditionalProperties["answer"] - assert.True(t, answerOK) - assert.NotNil(t, answer) - answerAdditional, answerAdditionalOK := answer.(*qnamodels.Answer) - assert.True(t, answerAdditionalOK) - assert.Equal(t, "answer", *answerAdditional.Result) - assert.Equal(t, "content", *answerAdditional.Property) - assert.Equal(t, 0.8, *answerAdditional.Certainty) - assert.InDelta(t, 0.4, *answerAdditional.Distance, 1e-9) - assert.Equal(t, 13, answerAdditional.StartPosition) - assert.Equal(t, 19, answerAdditional.EndPosition) - assert.Equal(t, true, answerAdditional.HasAnswer) - }) - - t.Run("should answer with similarity set above ask distance", func(t *testing.T) { - // given - qnaClient := &fakeQnAClient{} - fakeHelper := &fakeParamsHelper{} - answerProvider := New(qnaClient, fakeHelper) - in := []search.Result{ - { - ID: "some-uuid", - Schema: map[string]interface{}{ - "content": "content with answer", - "content2": "this one is just a title", - }, - }, - } - fakeParams := &Params{} - limit := 1 - argumentModuleParams := map[string]interface{}{ - "ask": map[string]interface{}{ - "question": "question", - "properties": []string{"content", "content2"}, - "distance": float64(0.4), - }, - } - - // when - out, err := answerProvider.AdditionalPropertyFn(context.Background(), in, fakeParams, &limit, argumentModuleParams, nil) - - // then - require.Nil(t, err) - require.NotEmpty(t, out) - assert.Equal(t, 1, len(out)) - answer, answerOK := out[0].AdditionalProperties["answer"] - assert.True(t, answerOK) - assert.NotNil(t, answer) - answerAdditional, answerAdditionalOK := answer.(*qnamodels.Answer) - assert.True(t, answerAdditionalOK) - assert.Equal(t, "answer", *answerAdditional.Result) - assert.Equal(t, "content", *answerAdditional.Property) - assert.Equal(t, 0.8, *answerAdditional.Certainty) - assert.Equal(t, *additional.CertaintyToDistPtr(ptFloat(0.8)), *answerAdditional.Distance) - assert.Equal(t, 13, answerAdditional.StartPosition) - assert.Equal(t, 19, answerAdditional.EndPosition) - assert.Equal(t, true, answerAdditional.HasAnswer) - }) - - t.Run("should answer with similarity set above ask certainty", func(t *testing.T) { - // given - qnaClient := &fakeQnAClient{} - fakeHelper := &fakeParamsHelper{} - answerProvider := New(qnaClient, fakeHelper) - in := []search.Result{ - { - ID: "some-uuid", - Schema: map[string]interface{}{ - "content": "content with answer", - "content2": "this one is just a title", - }, - }, - } - fakeParams := &Params{} - limit := 1 - argumentModuleParams := map[string]interface{}{ - "ask": map[string]interface{}{ - "question": "question", - "properties": []string{"content", "content2"}, - "certainty": float64(0.8), - }, - } - - // when - out, err := answerProvider.AdditionalPropertyFn(context.Background(), in, fakeParams, &limit, argumentModuleParams, nil) - - // then - require.Nil(t, err) - require.NotEmpty(t, out) - assert.Equal(t, 1, len(out)) - answer, answerOK := out[0].AdditionalProperties["answer"] - assert.True(t, answerOK) - assert.NotNil(t, answer) - answerAdditional, answerAdditionalOK := answer.(*qnamodels.Answer) - assert.True(t, answerAdditionalOK) - assert.Equal(t, "answer", *answerAdditional.Result) - assert.Equal(t, "content", *answerAdditional.Property) - assert.Equal(t, 0.8, *answerAdditional.Certainty) - assert.Equal(t, *additional.CertaintyToDistPtr(ptFloat(0.8)), *answerAdditional.Distance) - assert.Equal(t, 13, answerAdditional.StartPosition) - assert.Equal(t, 19, answerAdditional.EndPosition) - assert.Equal(t, true, answerAdditional.HasAnswer) - }) - - t.Run("should not answer with distance set below ask distance", func(t *testing.T) { - // given - qnaClient := &fakeQnAClient{} - fakeHelper := &fakeParamsHelper{} - answerProvider := New(qnaClient, fakeHelper) - in := []search.Result{ - { - ID: "some-uuid", - Schema: map[string]interface{}{ - "content": "content with answer", - "content2": "this one is just a title", - }, - }, - } - fakeParams := &Params{} - limit := 1 - argumentModuleParams := map[string]interface{}{ - "ask": map[string]interface{}{ - "question": "question", - "properties": []string{"content", "content2"}, - "distance": float64(0.19), - }, - } - - // when - out, err := answerProvider.AdditionalPropertyFn(context.Background(), in, fakeParams, &limit, argumentModuleParams, nil) - - // then - require.Nil(t, err) - require.NotEmpty(t, out) - assert.Equal(t, 1, len(in)) - answer, answerOK := in[0].AdditionalProperties["answer"] - assert.True(t, answerOK) - assert.NotNil(t, answer) - answerAdditional, answerAdditionalOK := answer.(*qnamodels.Answer) - assert.True(t, answerAdditionalOK) - assert.True(t, answerAdditional.Result == nil) - assert.True(t, answerAdditional.Property == nil) - assert.True(t, answerAdditional.Certainty == nil) - assert.True(t, answerAdditional.Distance == nil) - assert.Equal(t, 0, answerAdditional.StartPosition) - assert.Equal(t, 0, answerAdditional.EndPosition) - assert.Equal(t, false, answerAdditional.HasAnswer) - }) - - t.Run("should not answer with certainty set below ask certainty", func(t *testing.T) { - // given - qnaClient := &fakeQnAClient{} - fakeHelper := &fakeParamsHelper{} - answerProvider := New(qnaClient, fakeHelper) - in := []search.Result{ - { - ID: "some-uuid", - Schema: map[string]interface{}{ - "content": "content with answer", - "content2": "this one is just a title", - }, - }, - } - fakeParams := &Params{} - limit := 1 - argumentModuleParams := map[string]interface{}{ - "ask": map[string]interface{}{ - "question": "question", - "properties": []string{"content", "content2"}, - "certainty": float64(0.81), - }, - } - - // when - out, err := answerProvider.AdditionalPropertyFn(context.Background(), in, fakeParams, &limit, argumentModuleParams, nil) - - // then - require.Nil(t, err) - require.NotEmpty(t, out) - assert.Equal(t, 1, len(in)) - answer, answerOK := in[0].AdditionalProperties["answer"] - assert.True(t, answerOK) - assert.NotNil(t, answer) - answerAdditional, answerAdditionalOK := answer.(*qnamodels.Answer) - assert.True(t, answerAdditionalOK) - assert.True(t, answerAdditional.Result == nil) - assert.True(t, answerAdditional.Property == nil) - assert.True(t, answerAdditional.Certainty == nil) - assert.True(t, answerAdditional.Distance == nil) - assert.Equal(t, 0, answerAdditional.StartPosition) - assert.Equal(t, 0, answerAdditional.EndPosition) - assert.Equal(t, false, answerAdditional.HasAnswer) - }) - - t.Run("should answer with certainty set above ask certainty and the results should be reranked", func(t *testing.T) { - // given - qnaClient := &fakeQnAClient{} - fakeHelper := &fakeParamsHelper{} - answerProvider := New(qnaClient, fakeHelper) - in := []search.Result{ - { - ID: "uuid1", - Schema: map[string]interface{}{ - "content": "rerank 0.5", - }, - }, - { - ID: "uuid2", - Schema: map[string]interface{}{ - "content": "rerank 0.2", - }, - }, - { - ID: "uuid3", - Schema: map[string]interface{}{ - "content": "rerank 0.9", - }, - }, - } - fakeParams := &Params{} - limit := 1 - argumentModuleParams := map[string]interface{}{ - "ask": map[string]interface{}{ - "question": "question", - "properties": []string{"content"}, - "rerank": true, - }, - } - - // when - out, err := answerProvider.AdditionalPropertyFn(context.Background(), in, fakeParams, &limit, argumentModuleParams, nil) - - // then - require.Nil(t, err) - require.NotEmpty(t, out) - assert.Equal(t, 3, len(in)) - answer, answerOK := in[0].AdditionalProperties["answer"] - assert.True(t, answerOK) - assert.NotNil(t, answer) - answerAdditional, answerAdditionalOK := answer.(*qnamodels.Answer) - assert.True(t, answerAdditionalOK) - assert.Equal(t, "rerank 0.9", *answerAdditional.Result) - assert.Equal(t, "content", *answerAdditional.Property) - assert.Equal(t, 0.9, *answerAdditional.Certainty) - assert.Equal(t, *additional.CertaintyToDistPtr(ptFloat(0.9)), *answerAdditional.Distance) - assert.Equal(t, 0, answerAdditional.StartPosition) - assert.Equal(t, 10, answerAdditional.EndPosition) - assert.Equal(t, true, answerAdditional.HasAnswer) - - answer, answerOK = in[1].AdditionalProperties["answer"] - assert.True(t, answerOK) - assert.NotNil(t, answer) - answerAdditional, answerAdditionalOK = answer.(*qnamodels.Answer) - assert.True(t, answerAdditionalOK) - assert.Equal(t, "rerank 0.5", *answerAdditional.Result) - assert.Equal(t, "content", *answerAdditional.Property) - assert.Equal(t, 0.5, *answerAdditional.Certainty) - assert.Equal(t, *additional.CertaintyToDistPtr(ptFloat(0.5)), *answerAdditional.Distance) - assert.Equal(t, 0, answerAdditional.StartPosition) - assert.Equal(t, 10, answerAdditional.EndPosition) - assert.Equal(t, true, answerAdditional.HasAnswer) - - answer, answerOK = in[2].AdditionalProperties["answer"] - assert.True(t, answerOK) - assert.NotNil(t, answer) - answerAdditional, answerAdditionalOK = answer.(*qnamodels.Answer) - assert.True(t, answerAdditionalOK) - assert.Equal(t, "rerank 0.2", *answerAdditional.Result) - assert.Equal(t, "content", *answerAdditional.Property) - assert.Equal(t, 0.2, *answerAdditional.Certainty) - assert.Equal(t, *additional.CertaintyToDistPtr(ptFloat(0.2)), *answerAdditional.Distance) - assert.Equal(t, 0, answerAdditional.StartPosition) - assert.Equal(t, 10, answerAdditional.EndPosition) - assert.Equal(t, true, answerAdditional.HasAnswer) - }) - - t.Run("should answer with certainty set above ask certainty and the results should not be reranked", func(t *testing.T) { - // given - qnaClient := &fakeQnAClient{} - fakeHelper := &fakeParamsHelper{} - answerProvider := New(qnaClient, fakeHelper) - in := []search.Result{ - { - ID: "uuid1", - Schema: map[string]interface{}{ - "content": "rerank 0.5", - }, - }, - { - ID: "uuid2", - Schema: map[string]interface{}{ - "content": "rerank 0.2", - }, - }, - { - ID: "uuid3", - Schema: map[string]interface{}{ - "content": "rerank 0.9", - }, - }, - } - fakeParams := &Params{} - limit := 1 - argumentModuleParams := map[string]interface{}{ - "ask": map[string]interface{}{ - "question": "question", - "properties": []string{"content"}, - "rerank": false, - }, - } - - // when - out, err := answerProvider.AdditionalPropertyFn(context.Background(), in, fakeParams, &limit, argumentModuleParams, nil) - - // then - require.Nil(t, err) - require.NotEmpty(t, out) - assert.Equal(t, 3, len(in)) - answer, answerOK := in[0].AdditionalProperties["answer"] - assert.True(t, answerOK) - assert.NotNil(t, answer) - answerAdditional, answerAdditionalOK := answer.(*qnamodels.Answer) - assert.True(t, answerAdditionalOK) - assert.Equal(t, "rerank 0.5", *answerAdditional.Result) - assert.Equal(t, "content", *answerAdditional.Property) - assert.Equal(t, 0.5, *answerAdditional.Certainty) - assert.Equal(t, *additional.CertaintyToDistPtr(ptFloat(0.5)), *answerAdditional.Distance) - assert.Equal(t, 0, answerAdditional.StartPosition) - assert.Equal(t, 10, answerAdditional.EndPosition) - assert.Equal(t, true, answerAdditional.HasAnswer) - - answer, answerOK = in[1].AdditionalProperties["answer"] - assert.True(t, answerOK) - assert.NotNil(t, answer) - answerAdditional, answerAdditionalOK = answer.(*qnamodels.Answer) - assert.True(t, answerAdditionalOK) - assert.Equal(t, "rerank 0.2", *answerAdditional.Result) - assert.Equal(t, "content", *answerAdditional.Property) - assert.Equal(t, 0.2, *answerAdditional.Certainty) - assert.Equal(t, *additional.CertaintyToDistPtr(ptFloat(0.2)), *answerAdditional.Distance) - assert.Equal(t, 0, answerAdditional.StartPosition) - assert.Equal(t, 10, answerAdditional.EndPosition) - assert.Equal(t, true, answerAdditional.HasAnswer) - - answer, answerOK = in[2].AdditionalProperties["answer"] - assert.True(t, answerOK) - assert.NotNil(t, answer) - answerAdditional, answerAdditionalOK = answer.(*qnamodels.Answer) - assert.True(t, answerAdditionalOK) - assert.Equal(t, "rerank 0.9", *answerAdditional.Result) - assert.Equal(t, "content", *answerAdditional.Property) - assert.Equal(t, 0.9, *answerAdditional.Certainty) - assert.Equal(t, *additional.CertaintyToDistPtr(ptFloat(0.9)), *answerAdditional.Distance) - assert.Equal(t, 0, answerAdditional.StartPosition) - assert.Equal(t, 10, answerAdditional.EndPosition) - assert.Equal(t, true, answerAdditional.HasAnswer) - }) -} - -type fakeQnAClient struct{} - -func (c *fakeQnAClient) Answer(ctx context.Context, - text, question string, -) (*ent.AnswerResult, error) { - if text == "rerank 0.9" { - return c.getAnswer(question, "rerank 0.9", 0.9), nil - } - if text == "rerank 0.5" { - return c.getAnswer(question, "rerank 0.5", 0.5), nil - } - if text == "rerank 0.2" { - return c.getAnswer(question, "rerank 0.2", 0.2), nil - } - return c.getAnswer(question, "answer", 0.8), nil -} - -func (c *fakeQnAClient) getAnswer(question, answer string, certainty float64) *ent.AnswerResult { - return &ent.AnswerResult{ - Text: question, - Question: question, - Answer: &answer, - Certainty: &certainty, - Distance: additional.CertaintyToDistPtr(&certainty), - } -} - -type fakeParamsHelper struct{} - -func (h *fakeParamsHelper) GetQuestion(params interface{}) string { - if fakeParamsMap, ok := params.(map[string]interface{}); ok { - if question, ok := fakeParamsMap["question"].(string); ok { - return question - } - } - return "" -} - -func (h *fakeParamsHelper) GetProperties(params interface{}) []string { - if fakeParamsMap, ok := params.(map[string]interface{}); ok { - if properties, ok := fakeParamsMap["properties"].([]string); ok { - return properties - } - } - return nil -} - -func (h *fakeParamsHelper) GetCertainty(params interface{}) float64 { - if fakeParamsMap, ok := params.(map[string]interface{}); ok { - if certainty, ok := fakeParamsMap["certainty"].(float64); ok { - return certainty - } - } - return 0 -} - -func (h *fakeParamsHelper) GetDistance(params interface{}) float64 { - if fakeParamsMap, ok := params.(map[string]interface{}); ok { - if distance, ok := fakeParamsMap["distance"].(float64); ok { - return distance - } - } - return 0 -} - -func (h *fakeParamsHelper) GetRerank(params interface{}) bool { - if fakeParamsMap, ok := params.(map[string]interface{}); ok { - if rerank, ok := fakeParamsMap["rerank"].(bool); ok { - return rerank - } - } - return false -} - -func ptFloat(f float64) *float64 { - return &f -} diff --git a/modules/qna-transformers/additional/models/models.go b/modules/qna-transformers/additional/models/models.go deleted file mode 100644 index 9bffcf64fcde0f47c09e44613156f9e32a6ed51a..0000000000000000000000000000000000000000 --- a/modules/qna-transformers/additional/models/models.go +++ /dev/null @@ -1,24 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package models - -// Answer used in qna module to represent -// the answer to a given question -type Answer struct { - Result *string `json:"result,omitempty"` - Property *string `json:"property,omitempty"` - StartPosition int `json:"startPosition,omitempty"` - EndPosition int `json:"endPosition,omitempty"` - Certainty *float64 `json:"certainty,omitempty"` - Distance *float64 `json:"distance,omitempty"` - HasAnswer bool `json:"hasAnswer,omitempty"` -} diff --git a/modules/qna-transformers/additional/provider.go b/modules/qna-transformers/additional/provider.go deleted file mode 100644 index 5df746281892008bd1ebc702a91507586dbb224e..0000000000000000000000000000000000000000 --- a/modules/qna-transformers/additional/provider.go +++ /dev/null @@ -1,57 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package additional - -import ( - "context" - - "github.com/tailor-inc/graphql" - "github.com/tailor-inc/graphql/language/ast" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/search" -) - -type AdditionalProperty interface { - AdditionalPropertyFn(ctx context.Context, - in []search.Result, params interface{}, limit *int, - argumentModuleParams map[string]interface{}, cfg moduletools.ClassConfig) ([]search.Result, error) - ExtractAdditionalFn(param []*ast.Argument) interface{} - AdditionalPropertyDefaultValue() interface{} - AdditionalFieldFn(classname string) *graphql.Field -} - -type GraphQLAdditionalArgumentsProvider struct { - answerProvider AdditionalProperty -} - -func New(answerProvider AdditionalProperty) *GraphQLAdditionalArgumentsProvider { - return &GraphQLAdditionalArgumentsProvider{answerProvider} -} - -func (p *GraphQLAdditionalArgumentsProvider) AdditionalProperties() map[string]modulecapabilities.AdditionalProperty { - additionalProperties := map[string]modulecapabilities.AdditionalProperty{} - additionalProperties["answer"] = p.getAnswer() - return additionalProperties -} - -func (p *GraphQLAdditionalArgumentsProvider) getAnswer() modulecapabilities.AdditionalProperty { - return modulecapabilities.AdditionalProperty{ - GraphQLNames: []string{"answer"}, - GraphQLFieldFunction: p.answerProvider.AdditionalFieldFn, - GraphQLExtractFunction: p.answerProvider.ExtractAdditionalFn, - SearchFunctions: modulecapabilities.AdditionalSearch{ - ExploreGet: p.answerProvider.AdditionalPropertyFn, - ExploreList: p.answerProvider.AdditionalPropertyFn, - }, - } -} diff --git a/modules/qna-transformers/ask.go b/modules/qna-transformers/ask.go deleted file mode 100644 index d2be79a9c92b329efafb18fd10977de8a0e2ddd4..0000000000000000000000000000000000000000 --- a/modules/qna-transformers/ask.go +++ /dev/null @@ -1,40 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modqna - -import ( - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/modules/qna-transformers/ask" -) - -func (m *QnAModule) initAskSearcher() error { - m.searcher = ask.NewSearcher(m.nearTextDependencies) - return nil -} - -func (m *QnAModule) initAskProvider() error { - m.graphqlProvider = ask.New(m.askTextTransformer) - return nil -} - -func (m *QnAModule) Arguments() map[string]modulecapabilities.GraphQLArgument { - return m.graphqlProvider.Arguments() -} - -func (m *QnAModule) VectorSearches() modulecapabilities.ModuleArgumentVectorForParams { - return m.searcher.VectorSearches() -} - -var ( - _ = modulecapabilities.GraphQLArguments(New()) - _ = modulecapabilities.DependencySearcher(New()) -) diff --git a/modules/qna-transformers/ask/fakes_for_test.go b/modules/qna-transformers/ask/fakes_for_test.go deleted file mode 100644 index bf42f4cc2198b3d379222842faa674844447bcdd..0000000000000000000000000000000000000000 --- a/modules/qna-transformers/ask/fakes_for_test.go +++ /dev/null @@ -1,21 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package ask - -type fakeTransformer struct{} - -func (t *fakeTransformer) Transform(in []string) ([]string, error) { - if len(in) == 1 && in[0] == "transform this" { - return []string{"transformed text"}, nil - } - return in, nil -} diff --git a/modules/qna-transformers/ask/graphql_argument.go b/modules/qna-transformers/ask/graphql_argument.go deleted file mode 100644 index 634c02bae52ec39985b3726a504235c3e87acfd4..0000000000000000000000000000000000000000 --- a/modules/qna-transformers/ask/graphql_argument.go +++ /dev/null @@ -1,76 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package ask - -import ( - "fmt" - - "github.com/tailor-inc/graphql" - "github.com/weaviate/weaviate/adapters/handlers/graphql/descriptions" -) - -func (g *GraphQLArgumentsProvider) getAskArgumentFn(classname string) *graphql.ArgumentConfig { - return g.askArgument("GetObjects", classname) -} - -func (g *GraphQLArgumentsProvider) exploreAskArgumentFn() *graphql.ArgumentConfig { - return g.askArgument("Explore", "") -} - -func (g *GraphQLArgumentsProvider) aggregateAskArgumentFn(classname string) *graphql.ArgumentConfig { - return g.askArgument("Aggregate", classname) -} - -func (g *GraphQLArgumentsProvider) askArgument(prefix, className string) *graphql.ArgumentConfig { - prefixName := fmt.Sprintf("QnATransformers%s%s", prefix, className) - return &graphql.ArgumentConfig{ - Type: graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sAskInpObj", prefixName), - Fields: g.askFields(prefixName), - Description: descriptions.GetWhereInpObj, - }, - ), - } -} - -func (g *GraphQLArgumentsProvider) askFields(prefix string) graphql.InputObjectConfigFieldMap { - askFields := graphql.InputObjectConfigFieldMap{ - "question": &graphql.InputObjectFieldConfig{ - Description: "Question to be answered", - Type: graphql.NewNonNull(graphql.String), - }, - "certainty": &graphql.InputObjectFieldConfig{ - Description: descriptions.Certainty, - Type: graphql.Float, - }, - "distance": &graphql.InputObjectFieldConfig{ - Description: descriptions.Distance, - Type: graphql.Float, - }, - "properties": &graphql.InputObjectFieldConfig{ - Description: "Properties which contains text", - Type: graphql.NewList(graphql.String), - }, - "rerank": &graphql.InputObjectFieldConfig{ - Description: "Arranges the results by certainty", - Type: graphql.Boolean, - }, - } - if g.askTransformer != nil { - askFields["autocorrect"] = &graphql.InputObjectFieldConfig{ - Description: "Autocorrect input text values", - Type: graphql.Boolean, - } - } - return askFields -} diff --git a/modules/qna-transformers/ask/graphql_argument_test.go b/modules/qna-transformers/ask/graphql_argument_test.go deleted file mode 100644 index 9402861430c529e9ff0563bdbe4f18b65ea0f8f4..0000000000000000000000000000000000000000 --- a/modules/qna-transformers/ask/graphql_argument_test.go +++ /dev/null @@ -1,97 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package ask - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/tailor-inc/graphql" -) - -func TestAskGraphQLArgument(t *testing.T) { - t.Run("should generate ask argument properly", func(t *testing.T) { - // given - prefix := "Prefix" - classname := "Class" - // when - ask := New(nil).askArgument(prefix, classname) - - // then - // the built graphQL field needs to support this structure: - // ask { - // question: "question?", - // distance: 0.9 - // properties: ["prop1", "prop2"] - // rerank: true - // } - assert.NotNil(t, ask) - assert.Equal(t, "QnATransformersPrefixClassAskInpObj", ask.Type.Name()) - askFields, ok := ask.Type.(*graphql.InputObject) - assert.True(t, ok) - assert.NotNil(t, askFields) - assert.Equal(t, 5, len(askFields.Fields())) - fields := askFields.Fields() - question := fields["question"] - questionNonNull, questionNonNullOK := question.Type.(*graphql.NonNull) - assert.True(t, questionNonNullOK) - assert.Equal(t, "String", questionNonNull.OfType.Name()) - assert.NotNil(t, question) - assert.NotNil(t, fields["certainty"]) - assert.NotNil(t, fields["distance"]) - properties := fields["properties"] - propertiesList, propertiesListOK := properties.Type.(*graphql.List) - assert.True(t, propertiesListOK) - assert.Equal(t, "String", propertiesList.OfType.Name()) - assert.NotNil(t, fields["rerank"]) - }) -} - -func TestAskGraphQLArgumentWithAutocorrect(t *testing.T) { - t.Run("should generate ask argument properly with autocorrect", func(t *testing.T) { - // given - prefix := "Prefix" - classname := "Class" - // when - ask := New(&fakeTransformer{}).askArgument(prefix, classname) - - // then - // the built graphQL field needs to support this structure: - // ask { - // question: "question?", - // distance: 0.9 - // properties: ["prop1", "prop2"] - // autocorrect: true - // rerank: true - // } - assert.NotNil(t, ask) - assert.Equal(t, "QnATransformersPrefixClassAskInpObj", ask.Type.Name()) - askFields, ok := ask.Type.(*graphql.InputObject) - assert.True(t, ok) - assert.NotNil(t, askFields) - assert.Equal(t, 6, len(askFields.Fields())) - fields := askFields.Fields() - question := fields["question"] - questionNonNull, questionNonNullOK := question.Type.(*graphql.NonNull) - assert.True(t, questionNonNullOK) - assert.Equal(t, "String", questionNonNull.OfType.Name()) - assert.NotNil(t, question) - assert.NotNil(t, fields["certainty"]) - assert.NotNil(t, fields["distance"]) - properties := fields["properties"] - propertiesList, propertiesListOK := properties.Type.(*graphql.List) - assert.True(t, propertiesListOK) - assert.Equal(t, "String", propertiesList.OfType.Name()) - assert.NotNil(t, fields["autocorrect"]) - assert.NotNil(t, fields["rerank"]) - }) -} diff --git a/modules/qna-transformers/ask/graphql_provider.go b/modules/qna-transformers/ask/graphql_provider.go deleted file mode 100644 index 98e1261176798e4f1de79599284a93a1d80e1884..0000000000000000000000000000000000000000 --- a/modules/qna-transformers/ask/graphql_provider.go +++ /dev/null @@ -1,40 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package ask - -import ( - "github.com/weaviate/weaviate/entities/modulecapabilities" -) - -type GraphQLArgumentsProvider struct { - askTransformer modulecapabilities.TextTransform -} - -func New(askTransformer modulecapabilities.TextTransform) *GraphQLArgumentsProvider { - return &GraphQLArgumentsProvider{askTransformer} -} - -func (g *GraphQLArgumentsProvider) Arguments() map[string]modulecapabilities.GraphQLArgument { - arguments := map[string]modulecapabilities.GraphQLArgument{} - arguments["ask"] = g.getAsk() - return arguments -} - -func (g *GraphQLArgumentsProvider) getAsk() modulecapabilities.GraphQLArgument { - return modulecapabilities.GraphQLArgument{ - GetArgumentsFunction: g.getAskArgumentFn, - AggregateArgumentsFunction: g.aggregateAskArgumentFn, - ExploreArgumentsFunction: g.exploreAskArgumentFn, - ExtractFunction: g.extractAskFn, - ValidateFunction: g.validateAskFn, - } -} diff --git a/modules/qna-transformers/ask/grapqhl_extract.go b/modules/qna-transformers/ask/grapqhl_extract.go deleted file mode 100644 index af41c41cbeabda7c0bb64a1344bd184d10b035a8..0000000000000000000000000000000000000000 --- a/modules/qna-transformers/ask/grapqhl_extract.go +++ /dev/null @@ -1,62 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package ask - -func (g *GraphQLArgumentsProvider) extractAskFn(source map[string]interface{}) interface{} { - var args AskParams - - question, ok := source["question"].(string) - if ok { - args.Question = question - } - - // autocorrect is an optional arg, so it could be nil - autocorrect, ok := source["autocorrect"] - if ok { - args.Autocorrect = autocorrect.(bool) - } - - // if there's text transformer present and autocorrect set to true - // perform text transformation operation - if args.Autocorrect && g.askTransformer != nil { - if transformedValues, err := g.askTransformer.Transform([]string{args.Question}); err == nil && len(transformedValues) == 1 { - args.Question = transformedValues[0] - } - } - - certainty, ok := source["certainty"] - if ok { - args.Certainty = certainty.(float64) - } - - distance, ok := source["distance"] - if ok { - args.Distance = distance.(float64) - args.WithDistance = true - } - - properties, ok := source["properties"].([]interface{}) - if ok { - args.Properties = make([]string, len(properties)) - for i, value := range properties { - args.Properties[i] = value.(string) - } - } - - // rerank is an optional arg, so it could be nil - rerank, ok := source["rerank"] - if ok { - args.Rerank = rerank.(bool) - } - - return &args -} diff --git a/modules/qna-transformers/ask/grapqhl_extract_test.go b/modules/qna-transformers/ask/grapqhl_extract_test.go deleted file mode 100644 index ca1a0c85d1516841f655ff1cec6def163f141f30..0000000000000000000000000000000000000000 --- a/modules/qna-transformers/ask/grapqhl_extract_test.go +++ /dev/null @@ -1,283 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package ask - -import ( - "reflect" - "testing" -) - -func Test_extractAskFn(t *testing.T) { - type args struct { - source map[string]interface{} - } - tests := []struct { - name string - args args - want interface{} - }{ - { - name: "should parse properly with only question", - args: args{ - source: map[string]interface{}{ - "question": "some question", - }, - }, - want: &AskParams{ - Question: "some question", - }, - }, - { - name: "should parse properly with question and distance", - args: args{ - source: map[string]interface{}{ - "question": "some question", - "distance": 0.8, - }, - }, - want: &AskParams{ - Question: "some question", - Distance: 0.8, - WithDistance: true, - }, - }, - { - name: "should parse properly with question and certainty", - args: args{ - source: map[string]interface{}{ - "question": "some question", - "certainty": 0.8, - }, - }, - want: &AskParams{ - Question: "some question", - Certainty: 0.8, - }, - }, - { - name: "should parse properly without params", - args: args{ - source: map[string]interface{}{}, - }, - want: &AskParams{}, - }, - { - name: "should parse properly with question, distance, and properties", - args: args{ - source: map[string]interface{}{ - "question": "some question", - "distance": 0.8, - "properties": []interface{}{"prop1", "prop2"}, - }, - }, - want: &AskParams{ - Question: "some question", - Distance: 0.8, - WithDistance: true, - Properties: []string{"prop1", "prop2"}, - }, - }, - { - name: "should parse properly with question and certainty and properties", - args: args{ - source: map[string]interface{}{ - "question": "some question", - "certainty": 0.8, - "properties": []interface{}{"prop1", "prop2"}, - }, - }, - want: &AskParams{ - Question: "some question", - Certainty: 0.8, - Properties: []string{"prop1", "prop2"}, - }, - }, - { - name: "should parse properly with question, distance, properties, and rerank", - args: args{ - source: map[string]interface{}{ - "question": "some question", - "distance": 0.8, - "properties": []interface{}{"prop1", "prop2"}, - "rerank": true, - }, - }, - want: &AskParams{ - Question: "some question", - Distance: 0.8, - WithDistance: true, - Properties: []string{"prop1", "prop2"}, - Rerank: true, - }, - }, - { - name: "should parse properly with question and certainty and properties and rerank", - args: args{ - source: map[string]interface{}{ - "question": "some question", - "certainty": 0.8, - "properties": []interface{}{"prop1", "prop2"}, - "rerank": true, - }, - }, - want: &AskParams{ - Question: "some question", - Certainty: 0.8, - Properties: []string{"prop1", "prop2"}, - Rerank: true, - }, - }, - } - - testsWithAutocorrect := []struct { - name string - args args - want interface{} - }{ - { - name: "should parse properly with only question and autocorrect", - args: args{ - source: map[string]interface{}{ - "question": "some question", - "autocorrect": true, - }, - }, - want: &AskParams{ - Question: "some question", - Autocorrect: true, - }, - }, - { - name: "should parse properly and transform text in question", - args: args{ - source: map[string]interface{}{ - "question": "transform this", - "autocorrect": true, - }, - }, - want: &AskParams{ - Question: "transformed text", - Autocorrect: true, - }, - }, - { - name: "should parse properly and not transform text in question", - args: args{ - source: map[string]interface{}{ - "question": "transform this", - "autocorrect": false, - }, - }, - want: &AskParams{ - Question: "transform this", - Autocorrect: false, - }, - }, - { - name: "should parse properly with question, distance, properties, and autocorrect", - args: args{ - source: map[string]interface{}{ - "question": "transform this", - "distance": 0.8, - "properties": []interface{}{"prop1", "prop2"}, - "autocorrect": true, - }, - }, - want: &AskParams{ - Question: "transformed text", - Distance: 0.8, - WithDistance: true, - Properties: []string{"prop1", "prop2"}, - Autocorrect: true, - }, - }, - { - name: "should parse properly with question and certainty and properties and autocorrect", - args: args{ - source: map[string]interface{}{ - "question": "transform this", - "certainty": 0.8, - "properties": []interface{}{"prop1", "prop2"}, - "autocorrect": true, - }, - }, - want: &AskParams{ - Question: "transformed text", - Certainty: 0.8, - Properties: []string{"prop1", "prop2"}, - Autocorrect: true, - }, - }, - { - name: "should parse properly with question, distance, properties, autocorrect, and rerank", - args: args{ - source: map[string]interface{}{ - "question": "transform this", - "distance": 0.8, - "properties": []interface{}{"prop1", "prop2"}, - "autocorrect": true, - "rerank": true, - }, - }, - want: &AskParams{ - Question: "transformed text", - Distance: 0.8, - WithDistance: true, - Properties: []string{"prop1", "prop2"}, - Autocorrect: true, - Rerank: true, - }, - }, - { - name: "should parse properly with question and certainty and properties and autocorrect and rerank", - args: args{ - source: map[string]interface{}{ - "question": "transform this", - "certainty": 0.8, - "properties": []interface{}{"prop1", "prop2"}, - "autocorrect": true, - "rerank": true, - }, - }, - want: &AskParams{ - Question: "transformed text", - Certainty: 0.8, - Properties: []string{"prop1", "prop2"}, - Autocorrect: true, - Rerank: true, - }, - }, - } - - testsWithAutocorrect = append(testsWithAutocorrect, tests...) - - t.Run("should extract without text transformer", func(t *testing.T) { - provider := New(nil) - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := provider.extractAskFn(tt.args.source); !reflect.DeepEqual(got, tt.want) { - t.Errorf("extractAskFn() = %v, want %v", got, tt.want) - } - }) - } - }) - t.Run("should extract with text transformer", func(t *testing.T) { - provider := New(&fakeTransformer{}) - for _, tt := range testsWithAutocorrect { - t.Run(tt.name, func(t *testing.T) { - if got := provider.extractAskFn(tt.args.source); !reflect.DeepEqual(got, tt.want) { - t.Errorf("extractAskFn() = %v, want %v", got, tt.want) - } - }) - } - }) -} diff --git a/modules/qna-transformers/ask/param.go b/modules/qna-transformers/ask/param.go deleted file mode 100644 index 8d79b7ee16553500fe920250c6c490373c635b9e..0000000000000000000000000000000000000000 --- a/modules/qna-transformers/ask/param.go +++ /dev/null @@ -1,56 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package ask - -import ( - "github.com/pkg/errors" -) - -type AskParams struct { - Question string - Certainty float64 - Distance float64 - WithDistance bool - Properties []string - Autocorrect bool - Rerank bool -} - -func (n AskParams) GetCertainty() float64 { - return n.Certainty -} - -func (n AskParams) GetDistance() float64 { - return n.Distance -} - -func (n AskParams) SimilarityMetricProvided() bool { - return n.Certainty != 0 || n.WithDistance -} - -func (g *GraphQLArgumentsProvider) validateAskFn(param interface{}) error { - ask, ok := param.(*AskParams) - if !ok { - return errors.New("'ask' invalid parameter") - } - - if len(ask.Question) == 0 { - return errors.Errorf("'ask.question' needs to be defined") - } - - if ask.Certainty != 0 && ask.WithDistance { - return errors.Errorf( - "nearText cannot provide both distance and certainty") - } - - return nil -} diff --git a/modules/qna-transformers/ask/param_helper.go b/modules/qna-transformers/ask/param_helper.go deleted file mode 100644 index 0a01a6aa016f14c3616904ce2bc8d7f3ea546325..0000000000000000000000000000000000000000 --- a/modules/qna-transformers/ask/param_helper.go +++ /dev/null @@ -1,53 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package ask - -type ParamsHelper struct{} - -func NewParamsHelper() *ParamsHelper { - return &ParamsHelper{} -} - -func (p *ParamsHelper) GetQuestion(params interface{}) string { - if parameters, ok := params.(*AskParams); ok { - return parameters.Question - } - return "" -} - -func (p *ParamsHelper) GetProperties(params interface{}) []string { - if parameters, ok := params.(*AskParams); ok { - return parameters.Properties - } - return nil -} - -func (p *ParamsHelper) GetCertainty(params interface{}) float64 { - if parameters, ok := params.(*AskParams); ok { - return parameters.Certainty - } - return 0 -} - -func (p *ParamsHelper) GetDistance(params interface{}) float64 { - if parameters, ok := params.(*AskParams); ok { - return parameters.Distance - } - return 0 -} - -func (p *ParamsHelper) GetRerank(params interface{}) bool { - if parameters, ok := params.(*AskParams); ok { - return parameters.Rerank - } - return false -} diff --git a/modules/qna-transformers/ask/param_helper_test.go b/modules/qna-transformers/ask/param_helper_test.go deleted file mode 100644 index 709baeecf558a77fcf2387f0006484c1979ac526..0000000000000000000000000000000000000000 --- a/modules/qna-transformers/ask/param_helper_test.go +++ /dev/null @@ -1,151 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package ask - -import ( - "reflect" - "testing" -) - -func TestParamsHelper_GetQuestion(t *testing.T) { - type args struct { - params interface{} - } - tests := []struct { - name string - args args - want string - }{ - { - name: "should get question with certainty", - args: args{ - params: &AskParams{ - Question: "question", - Certainty: 0.8, - }, - }, - want: "question", - }, - { - name: "should get question with distance", - args: args{ - params: &AskParams{ - Question: "question", - Distance: 0.8, - }, - }, - want: "question", - }, - { - name: "should get empty string when empty params", - args: args{ - params: &AskParams{}, - }, - want: "", - }, - { - name: "should get empty string when nil params", - args: args{ - params: nil, - }, - want: "", - }, - { - name: "should get empty string when passed a struct, not a pointer to struct", - args: args{ - params: AskParams{}, - }, - want: "", - }, - { - name: "should get empty string when passed a struct with question, not a pointer to struct", - args: args{ - params: AskParams{ - Question: "question?", - }, - }, - want: "", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - p := &ParamsHelper{} - if got := p.GetQuestion(tt.args.params); got != tt.want { - t.Errorf("ParamsHelper.GetQuestion() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestParamsHelper_GetProperties(t *testing.T) { - type args struct { - params interface{} - } - tests := []struct { - name string - p *ParamsHelper - args args - want []string - }{ - { - name: "should get properties with distance", - args: args{ - params: &AskParams{ - Question: "question", - Properties: []string{"prop1", "prop2"}, - Distance: 0.8, - }, - }, - want: []string{"prop1", "prop2"}, - }, - { - name: "should get properties with certainty", - args: args{ - params: &AskParams{ - Question: "question", - Properties: []string{"prop1", "prop2"}, - Certainty: 0.8, - }, - }, - want: []string{"prop1", "prop2"}, - }, - { - name: "should get nil properties with empty pointer to AskParams", - args: args{ - params: &AskParams{}, - }, - want: nil, - }, - { - name: "should get nil properties with empty AskParams", - args: args{ - params: AskParams{}, - }, - want: nil, - }, - { - name: "should get nil properties with nil params", - args: args{ - params: nil, - }, - want: nil, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - p := &ParamsHelper{} - if got := p.GetProperties(tt.args.params); !reflect.DeepEqual(got, tt.want) { - t.Errorf("ParamsHelper.GetProperties() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/modules/qna-transformers/ask/param_test.go b/modules/qna-transformers/ask/param_test.go deleted file mode 100644 index db632556cf7723840f02fc9889a68412a67f36b9..0000000000000000000000000000000000000000 --- a/modules/qna-transformers/ask/param_test.go +++ /dev/null @@ -1,77 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package ask - -import "testing" - -func Test_validateAskFn(t *testing.T) { - type args struct { - param interface{} - } - tests := []struct { - name string - args args - wantErr bool - }{ - { - name: "should validate", - args: args{ - param: &AskParams{ - Question: "question", - }, - }, - }, - { - name: "should not validate when empty question", - args: args{ - param: &AskParams{ - Question: "", - }, - }, - wantErr: true, - }, - { - name: "should not validate when empty params", - args: args{ - param: &AskParams{}, - }, - wantErr: true, - }, - { - name: "should not validate when distance and certainty are present", - args: args{ - param: &AskParams{ - Distance: 0.1, - Certainty: 0.1, - }, - }, - wantErr: true, - }, - { - name: "should not validate when param passed is struct, not a pointer to struct", - args: args{ - param: AskParams{ - Question: "question", - }, - }, - wantErr: true, - }, - } - provider := New(nil) - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := provider.validateAskFn(tt.args.param); (err != nil) != tt.wantErr { - t.Errorf("validateAskFn() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} diff --git a/modules/qna-transformers/ask/searcher.go b/modules/qna-transformers/ask/searcher.go deleted file mode 100644 index 467b437acd297640bcf1abefeb99de04801f841f..0000000000000000000000000000000000000000 --- a/modules/qna-transformers/ask/searcher.go +++ /dev/null @@ -1,71 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package ask - -import ( - "context" - - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" -) - -type vectorFromAskParam struct { - nearTextDep modulecapabilities.Dependency -} - -func (s *vectorFromAskParam) vectorForAskParamFn(ctx context.Context, params interface{}, - className string, - findVectorFn modulecapabilities.FindVectorFn, - cfg moduletools.ClassConfig, -) ([]float32, error) { - return s.vectorFromAskParam(ctx, params.(*AskParams), className, findVectorFn, cfg) -} - -func (s *vectorFromAskParam) vectorFromAskParam(ctx context.Context, - params *AskParams, className string, - findVectorFn modulecapabilities.FindVectorFn, - cfg moduletools.ClassConfig, -) ([]float32, error) { - arg := s.nearTextDep.GraphQLArgument() - - rawNearTextParam := map[string]interface{}{} - rawNearTextParam["concepts"] = []interface{}{params.Question} - - nearTextParam := arg.ExtractFunction(rawNearTextParam) - vectorSearchFn := s.nearTextDep.VectorSearch() - - return vectorSearchFn(ctx, nearTextParam, className, findVectorFn, cfg) -} - -type Searcher struct { - // nearText modules dependencies - nearTextDeps []modulecapabilities.Dependency -} - -func NewSearcher(nearTextDeps []modulecapabilities.Dependency) *Searcher { - return &Searcher{nearTextDeps} -} - -func (s *Searcher) VectorSearches() map[string]modulecapabilities.ArgumentVectorForParams { - vectorSearchers := map[string]modulecapabilities.ArgumentVectorForParams{} - for _, nearTextDep := range s.nearTextDeps { - vectorSearchers[nearTextDep.ModuleName()] = s.vectorSearches(nearTextDep) - } - return vectorSearchers -} - -func (s *Searcher) vectorSearches(nearTextDep modulecapabilities.Dependency) map[string]modulecapabilities.VectorForParams { - vectorSearches := map[string]modulecapabilities.VectorForParams{} - vectorFromAsk := &vectorFromAskParam{nearTextDep} - vectorSearches["ask"] = vectorFromAsk.vectorForAskParamFn - return vectorSearches -} diff --git a/modules/qna-transformers/clients/qna.go b/modules/qna-transformers/clients/qna.go deleted file mode 100644 index b18612b601acc103409dd2fa5d54c9191f2302bf..0000000000000000000000000000000000000000 --- a/modules/qna-transformers/clients/qna.go +++ /dev/null @@ -1,105 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/modules/qna-transformers/ent" -) - -type qna struct { - origin string - httpClient *http.Client - logger logrus.FieldLogger -} - -func New(origin string, timeout time.Duration, logger logrus.FieldLogger) *qna { - return &qna{ - origin: origin, - httpClient: &http.Client{Timeout: timeout}, - logger: logger, - } -} - -func (q *qna) Answer(ctx context.Context, - text, question string, -) (*ent.AnswerResult, error) { - body, err := json.Marshal(answersInput{ - Text: text, - Question: question, - }) - if err != nil { - return nil, errors.Wrapf(err, "marshal body") - } - - req, err := http.NewRequestWithContext(ctx, "POST", q.url("/answers/"), - bytes.NewReader(body)) - if err != nil { - return nil, errors.Wrap(err, "create POST request") - } - - res, err := q.httpClient.Do(req) - if err != nil { - return nil, errors.Wrap(err, "send POST request") - } - defer res.Body.Close() - - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - return nil, errors.Wrap(err, "read response body") - } - - var resBody answersResponse - if err := json.Unmarshal(bodyBytes, &resBody); err != nil { - return nil, errors.Wrap(err, "unmarshal response body") - } - - if res.StatusCode > 399 { - return nil, errors.Errorf("fail with status %d: %s", res.StatusCode, - resBody.Error) - } - - return &ent.AnswerResult{ - Text: resBody.Text, - Question: resBody.Question, - Answer: resBody.Answer, - Certainty: resBody.Certainty, - Distance: additional.CertaintyToDistPtr(resBody.Certainty), - }, nil -} - -func (q *qna) url(path string) string { - return fmt.Sprintf("%s%s", q.origin, path) -} - -type answersInput struct { - Text string `json:"text"` - Question string `json:"question"` -} - -type answersResponse struct { - answersInput `json:"answersInput"` - Answer *string `json:"answer"` - Certainty *float64 `json:"certainty"` - Distance *float64 `json:"distance"` - Error string `json:"error"` -} diff --git a/modules/qna-transformers/clients/qna_meta.go b/modules/qna-transformers/clients/qna_meta.go deleted file mode 100644 index 4632e450e6f5a3629586d19e64c0b914dece6f06..0000000000000000000000000000000000000000 --- a/modules/qna-transformers/clients/qna_meta.go +++ /dev/null @@ -1,45 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "context" - "encoding/json" - "io" - "net/http" - - "github.com/pkg/errors" -) - -func (q *qna) MetaInfo() (map[string]interface{}, error) { - req, err := http.NewRequestWithContext(context.Background(), "GET", q.url("/meta"), nil) - if err != nil { - return nil, errors.Wrap(err, "create GET meta request") - } - - res, err := q.httpClient.Do(req) - if err != nil { - return nil, errors.Wrap(err, "send GET meta request") - } - defer res.Body.Close() - - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - return nil, errors.Wrap(err, "read meta response body") - } - - var resBody map[string]interface{} - if err := json.Unmarshal(bodyBytes, &resBody); err != nil { - return nil, errors.Wrap(err, "unmarshal meta response body") - } - return resBody, nil -} diff --git a/modules/qna-transformers/clients/qna_meta_test.go b/modules/qna-transformers/clients/qna_meta_test.go deleted file mode 100644 index 7945d5800d0e84950d5050fed604b094c236f1fb..0000000000000000000000000000000000000000 --- a/modules/qna-transformers/clients/qna_meta_test.go +++ /dev/null @@ -1,135 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func TestGetMeta(t *testing.T) { - t.Run("when the server is providing meta", func(t *testing.T) { - server := httptest.NewServer(&testMetaHandler{t: t}) - defer server.Close() - c := New(server.URL, 0, nullLogger()) - meta, err := c.MetaInfo() - - assert.Nil(t, err) - assert.NotNil(t, meta) - metaModel := meta["model"] - assert.True(t, metaModel != nil) - model, modelOK := metaModel.(map[string]interface{}) - assert.True(t, modelOK) - assert.True(t, model["_name_or_path"] != nil) - assert.True(t, model["architectures"] != nil) - modelID2label, modelID2labelOK := model["id2label"].(map[string]interface{}) - assert.True(t, modelID2labelOK) - assert.True(t, modelID2label["0"] != nil) - }) -} - -type testMetaHandler struct { - t *testing.T - // the test handler will report as not ready before the time has passed - readyTime time.Time -} - -func (f *testMetaHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - assert.Equal(f.t, "/meta", r.URL.String()) - assert.Equal(f.t, http.MethodGet, r.Method) - - if time.Since(f.readyTime) < 0 { - w.WriteHeader(http.StatusServiceUnavailable) - } - - w.Write([]byte(f.metaInfo())) -} - -func (f *testMetaHandler) metaInfo() string { - return `{ - "model": { - "_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad", - "add_cross_attention": false, - "architectures": [ - "BertForQuestionAnswering" - ], - "attention_probs_dropout_prob": 0.1, - "bad_words_ids": null, - "bos_token_id": null, - "chunk_size_feed_forward": 0, - "decoder_start_token_id": null, - "diversity_penalty": 0.0, - "do_sample": false, - "early_stopping": false, - "encoder_no_repeat_ngram_size": 0, - "eos_token_id": null, - "finetuning_task": null, - "gradient_checkpointing": false, - "hidden_act": "gelu", - "hidden_dropout_prob": 0.1, - "hidden_size": 1024, - "id2label": { - "0": "LABEL_0", - "1": "LABEL_1" - }, - "initializer_range": 0.02, - "intermediate_size": 4096, - "is_decoder": false, - "is_encoder_decoder": false, - "label2id": { - "LABEL_0": 0, - "LABEL_1": 1 - }, - "layer_norm_eps": 1e-12, - "length_penalty": 1.0, - "max_length": 20, - "max_position_embeddings": 512, - "min_length": 0, - "model_type": "bert", - "no_repeat_ngram_size": 0, - "num_attention_heads": 16, - "num_beam_groups": 1, - "num_beams": 1, - "num_hidden_layers": 24, - "num_return_sequences": 1, - "output_attentions": false, - "output_hidden_states": false, - "output_scores": false, - "pad_token_id": 0, - "position_embedding_type": "absolute", - "prefix": null, - "pruned_heads": {}, - "repetition_penalty": 1.0, - "return_dict": true, - "return_dict_in_generate": false, - "sep_token_id": null, - "task_specific_params": null, - "temperature": 1.0, - "tie_encoder_decoder": false, - "tie_word_embeddings": true, - "tokenizer_class": null, - "top_k": 50, - "top_p": 1.0, - "torchscript": false, - "transformers_version": "4.3.2", - "type_vocab_size": 2, - "use_bfloat16": false, - "use_cache": true, - "vocab_size": 30522, - "xla_device": null - } -}` -} diff --git a/modules/qna-transformers/clients/qna_test.go b/modules/qna-transformers/clients/qna_test.go deleted file mode 100644 index 6fb54e618d99aa3d1cfb7046b8a39bd127a37288..0000000000000000000000000000000000000000 --- a/modules/qna-transformers/clients/qna_test.go +++ /dev/null @@ -1,129 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "context" - "encoding/json" - "net/http" - "net/http/httptest" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/modules/qna-transformers/ent" -) - -func TestGetAnswer(t *testing.T) { - t.Run("when the server has a successful answer (with distance)", func(t *testing.T) { - server := httptest.NewServer(&testAnswerHandler{ - t: t, - answer: answersResponse{ - answersInput: answersInput{ - Text: "My name is John", - Question: "What is my name?", - }, - Answer: ptString("John"), - Certainty: ptFloat(0.7), - Distance: ptFloat(0.3), - }, - }) - defer server.Close() - c := New(server.URL, 0, nullLogger()) - res, err := c.Answer(context.Background(), "My name is John", - "What is my name?") - assert.Nil(t, err) - - expectedResult := ent.AnswerResult{ - Text: "My name is John", - Question: "What is my name?", - Answer: ptString("John"), - Certainty: ptFloat(0.7), - Distance: ptFloat(0.6), - } - - assert.Equal(t, expectedResult.Text, res.Text) - assert.Equal(t, expectedResult.Question, res.Question) - assert.Equal(t, expectedResult.Answer, res.Answer) - assert.Equal(t, expectedResult.Certainty, res.Certainty) - assert.InDelta(t, *expectedResult.Distance, *res.Distance, 1e-9) - }) - - t.Run("when the server has a successful answer (with certainty)", func(t *testing.T) { - server := httptest.NewServer(&testAnswerHandler{ - t: t, - answer: answersResponse{ - answersInput: answersInput{ - Text: "My name is John", - Question: "What is my name?", - }, - Answer: ptString("John"), - Certainty: ptFloat(0.7), - }, - }) - defer server.Close() - c := New(server.URL, 0, nullLogger()) - res, err := c.Answer(context.Background(), "My name is John", - "What is my name?") - - assert.Nil(t, err) - assert.Equal(t, &ent.AnswerResult{ - Text: "My name is John", - Question: "What is my name?", - Answer: ptString("John"), - Certainty: ptFloat(0.7), - Distance: additional.CertaintyToDistPtr(ptFloat(0.7)), - }, res) - }) - - t.Run("when the server has a an error", func(t *testing.T) { - server := httptest.NewServer(&testAnswerHandler{ - t: t, - answer: answersResponse{ - Error: "some error from the server", - }, - }) - defer server.Close() - c := New(server.URL, 0, nullLogger()) - _, err := c.Answer(context.Background(), "My name is John", - "What is my name?") - - require.NotNil(t, err) - assert.Contains(t, err.Error(), "some error from the server") - }) -} - -type testAnswerHandler struct { - t *testing.T - // the test handler will report as not ready before the time has passed - answer answersResponse -} - -func (f *testAnswerHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - assert.Equal(f.t, "/answers/", r.URL.String()) - assert.Equal(f.t, http.MethodPost, r.Method) - - if f.answer.Error != "" { - w.WriteHeader(500) - } - jsonBytes, _ := json.Marshal(f.answer) - w.Write(jsonBytes) -} - -func ptFloat(in float64) *float64 { - return &in -} - -func ptString(in string) *string { - return &in -} diff --git a/modules/qna-transformers/clients/startup.go b/modules/qna-transformers/clients/startup.go deleted file mode 100644 index 7c8fdf81caaa1a644b4dbf998f98b5d5eb3ba6fb..0000000000000000000000000000000000000000 --- a/modules/qna-transformers/clients/startup.go +++ /dev/null @@ -1,68 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "context" - "net/http" - "time" - - "github.com/pkg/errors" -) - -func (q *qna) WaitForStartup(initCtx context.Context, - interval time.Duration, -) error { - t := time.NewTicker(interval) - defer t.Stop() - expired := initCtx.Done() - var lastErr error - for { - select { - case <-t.C: - lastErr = q.checkReady(initCtx) - if lastErr == nil { - return nil - } - q.logger. - WithField("action", "qna_remote_wait_for_startup"). - WithError(lastErr).Warnf("qna remote service not ready") - case <-expired: - return errors.Wrapf(lastErr, "init context expired before remote was ready") - } - } -} - -func (q *qna) checkReady(initCtx context.Context) error { - // spawn a new context (derived on the overall context) which is used to - // consider an individual request timed out - requestCtx, cancel := context.WithTimeout(initCtx, 500*time.Millisecond) - defer cancel() - - req, err := http.NewRequestWithContext(requestCtx, http.MethodGet, - q.url("/.well-known/ready"), nil) - if err != nil { - return errors.Wrap(err, "create check ready request") - } - - res, err := q.httpClient.Do(req) - if err != nil { - return errors.Wrap(err, "send check ready request") - } - - defer res.Body.Close() - if res.StatusCode > 299 { - return errors.Errorf("not ready: status %d", res.StatusCode) - } - - return nil -} diff --git a/modules/qna-transformers/clients/startup_test.go b/modules/qna-transformers/clients/startup_test.go deleted file mode 100644 index 7836347d937eaaa18e3c1415dcb4b2aac2d2b15b..0000000000000000000000000000000000000000 --- a/modules/qna-transformers/clients/startup_test.go +++ /dev/null @@ -1,98 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "context" - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/sirupsen/logrus" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestWaitForStartup(t *testing.T) { - t.Run("when the server is immediately ready", func(t *testing.T) { - server := httptest.NewServer(&testReadyHandler{t: t}) - defer server.Close() - c := New(server.URL, 0, nullLogger()) - err := c.WaitForStartup(context.Background(), 50*time.Millisecond) - - assert.Nil(t, err) - }) - - t.Run("when the server is down", func(t *testing.T) { - c := New("http://nothing-running-at-this-url", 0, nullLogger()) - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancel() - err := c.WaitForStartup(ctx, 150*time.Millisecond) - - require.NotNil(t, err, nullLogger()) - assert.Contains(t, err.Error(), "expired before remote was ready") - }) - - t.Run("when the server is alive, but not ready", func(t *testing.T) { - server := httptest.NewServer(&testReadyHandler{ - t: t, - readyTime: time.Now().Add(1 * time.Minute), - }) - c := New(server.URL, 0, nullLogger()) - defer server.Close() - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancel() - err := c.WaitForStartup(ctx, 50*time.Millisecond) - - require.NotNil(t, err) - assert.Contains(t, err.Error(), "expired before remote was ready") - }) - - t.Run("when the server is initially not ready, but then becomes ready", - func(t *testing.T) { - server := httptest.NewServer(&testReadyHandler{ - t: t, - readyTime: time.Now().Add(100 * time.Millisecond), - }) - c := New(server.URL, 0, nullLogger()) - defer server.Close() - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancel() - err := c.WaitForStartup(ctx, 50*time.Millisecond) - - require.Nil(t, err) - }) -} - -type testReadyHandler struct { - t *testing.T - // the test handler will report as not ready before the time has passed - readyTime time.Time -} - -func (f *testReadyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - assert.Equal(f.t, "/.well-known/ready", r.URL.String()) - assert.Equal(f.t, http.MethodGet, r.Method) - - if time.Since(f.readyTime) < 0 { - w.WriteHeader(http.StatusServiceUnavailable) - } - - w.WriteHeader(http.StatusNoContent) -} - -func nullLogger() logrus.FieldLogger { - l, _ := test.NewNullLogger() - return l -} diff --git a/modules/qna-transformers/config.go b/modules/qna-transformers/config.go deleted file mode 100644 index 717e49c20c2c13d276b6eb7e72321406afe00e82..0000000000000000000000000000000000000000 --- a/modules/qna-transformers/config.go +++ /dev/null @@ -1,39 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modqna - -import ( - "context" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/schema" -) - -func (m *QnAModule) ClassConfigDefaults() map[string]interface{} { - return map[string]interface{}{} -} - -func (m *QnAModule) PropertyConfigDefaults( - dt *schema.DataType, -) map[string]interface{} { - return map[string]interface{}{} -} - -func (m *QnAModule) ValidateClass(ctx context.Context, - class *models.Class, cfg moduletools.ClassConfig, -) error { - return nil -} - -var _ = modulecapabilities.ClassConfigurator(New()) diff --git a/modules/qna-transformers/dependency/dependency.go b/modules/qna-transformers/dependency/dependency.go deleted file mode 100644 index 170159b89ec36fed0d588cf59f21a3b48f819ade..0000000000000000000000000000000000000000 --- a/modules/qna-transformers/dependency/dependency.go +++ /dev/null @@ -1,42 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package dependency - -import "github.com/weaviate/weaviate/entities/modulecapabilities" - -type NearTextDependecy struct { - moduleName string - argument modulecapabilities.GraphQLArgument - searcher modulecapabilities.VectorForParams -} - -func New(moduleName string, argument modulecapabilities.GraphQLArgument, - searcher modulecapabilities.VectorForParams, -) *NearTextDependecy { - return &NearTextDependecy{moduleName, argument, searcher} -} - -func (d *NearTextDependecy) Argument() string { - return "nearText" -} - -func (d *NearTextDependecy) ModuleName() string { - return d.moduleName -} - -func (d *NearTextDependecy) GraphQLArgument() modulecapabilities.GraphQLArgument { - return d.argument -} - -func (d *NearTextDependecy) VectorSearch() modulecapabilities.VectorForParams { - return d.searcher -} diff --git a/modules/qna-transformers/ent/vectorization_result.go b/modules/qna-transformers/ent/vectorization_result.go deleted file mode 100644 index 5fe363fa618999ebecfe6aa577b4d4d7e7a1fad1..0000000000000000000000000000000000000000 --- a/modules/qna-transformers/ent/vectorization_result.go +++ /dev/null @@ -1,20 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package ent - -type AnswerResult struct { - Text string - Question string - Answer *string - Certainty *float64 - Distance *float64 -} diff --git a/modules/qna-transformers/module.go b/modules/qna-transformers/module.go deleted file mode 100644 index 2f305a882e7441dc525c34461b5f2fd5ba52545d..0000000000000000000000000000000000000000 --- a/modules/qna-transformers/module.go +++ /dev/null @@ -1,172 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modqna - -import ( - "context" - "net/http" - "os" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - qnaadditional "github.com/weaviate/weaviate/modules/qna-transformers/additional" - qnaadditionalanswer "github.com/weaviate/weaviate/modules/qna-transformers/additional/answer" - qnaask "github.com/weaviate/weaviate/modules/qna-transformers/ask" - "github.com/weaviate/weaviate/modules/qna-transformers/clients" - qnaadependency "github.com/weaviate/weaviate/modules/qna-transformers/dependency" - "github.com/weaviate/weaviate/modules/qna-transformers/ent" -) - -func New() *QnAModule { - return &QnAModule{} -} - -type QnAModule struct { - qna qnaClient - graphqlProvider modulecapabilities.GraphQLArguments - searcher modulecapabilities.DependencySearcher - additionalPropertiesProvider modulecapabilities.AdditionalProperties - nearTextDependencies []modulecapabilities.Dependency - askTextTransformer modulecapabilities.TextTransform -} - -type qnaClient interface { - Answer(ctx context.Context, - text, question string) (*ent.AnswerResult, error) - MetaInfo() (map[string]interface{}, error) -} - -func (m *QnAModule) Name() string { - return "qna-transformers" -} - -func (m *QnAModule) Type() modulecapabilities.ModuleType { - return modulecapabilities.Text2TextQnA -} - -func (m *QnAModule) Init(ctx context.Context, - params moduletools.ModuleInitParams, -) error { - if err := m.initAdditional(ctx, params.GetConfig().ModuleHttpClientTimeout, params.GetLogger()); err != nil { - return errors.Wrap(err, "init vectorizer") - } - - return nil -} - -func (m *QnAModule) InitExtension(modules []modulecapabilities.Module) error { - var textTransformer modulecapabilities.TextTransform - for _, module := range modules { - if module.Name() == m.Name() { - continue - } - if arg, ok := module.(modulecapabilities.TextTransformers); ok { - if arg != nil && arg.TextTransformers() != nil { - textTransformer = arg.TextTransformers()["ask"] - } - } - } - - m.askTextTransformer = textTransformer - - if err := m.initAskProvider(); err != nil { - return errors.Wrap(err, "init ask provider") - } - - return nil -} - -func (m *QnAModule) InitDependency(modules []modulecapabilities.Module) error { - nearTextDependencies := []modulecapabilities.Dependency{} - for _, module := range modules { - if module.Name() == m.Name() { - continue - } - var argument modulecapabilities.GraphQLArgument - var searcher modulecapabilities.VectorForParams - if arg, ok := module.(modulecapabilities.GraphQLArguments); ok { - if arg != nil && arg.Arguments() != nil { - if nearTextArg, ok := arg.Arguments()["nearText"]; ok { - argument = nearTextArg - } - } - } - if arg, ok := module.(modulecapabilities.Searcher); ok { - if arg != nil && arg.VectorSearches() != nil { - if nearTextSearcher, ok := arg.VectorSearches()["nearText"]; ok { - searcher = nearTextSearcher - } - } - } - - if argument.ExtractFunction != nil && searcher != nil { - nearTextDependency := qnaadependency.New(module.Name(), argument, searcher) - nearTextDependencies = append(nearTextDependencies, nearTextDependency) - } - } - if len(nearTextDependencies) == 0 { - return errors.New("nearText dependecy not present") - } - - m.nearTextDependencies = nearTextDependencies - - if err := m.initAskSearcher(); err != nil { - return errors.Wrap(err, "init ask searcher") - } - - return nil -} - -func (m *QnAModule) initAdditional(ctx context.Context, timeout time.Duration, - logger logrus.FieldLogger, -) error { - // TODO: proper config management - uri := os.Getenv("QNA_INFERENCE_API") - if uri == "" { - return errors.Errorf("required variable QNA_INFERENCE_API is not set") - } - - client := clients.New(uri, timeout, logger) - if err := client.WaitForStartup(ctx, 1*time.Second); err != nil { - return errors.Wrap(err, "init remote vectorizer") - } - - m.qna = client - - answerProvider := qnaadditionalanswer.New(m.qna, qnaask.NewParamsHelper()) - m.additionalPropertiesProvider = qnaadditional.New(answerProvider) - - return nil -} - -func (m *QnAModule) RootHandler() http.Handler { - // TODO: remove once this is a capability interface - return nil -} - -func (m *QnAModule) MetaInfo() (map[string]interface{}, error) { - return m.qna.MetaInfo() -} - -func (m *QnAModule) AdditionalProperties() map[string]modulecapabilities.AdditionalProperty { - return m.additionalPropertiesProvider.AdditionalProperties() -} - -// verify we implement the modules.Module interface -var ( - _ = modulecapabilities.Module(New()) - _ = modulecapabilities.AdditionalProperties(New()) - _ = modulecapabilities.MetaProvider(New()) -) diff --git a/modules/ref2vec-centroid/config.go b/modules/ref2vec-centroid/config.go deleted file mode 100644 index 5cba311b2cf12246604a7e971c77a92aaf73907c..0000000000000000000000000000000000000000 --- a/modules/ref2vec-centroid/config.go +++ /dev/null @@ -1,44 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modcentroid - -import ( - "context" - "fmt" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/modules/ref2vec-centroid/config" -) - -func (m *CentroidModule) ClassConfigDefaults() map[string]interface{} { - return config.Default() -} - -func (m *CentroidModule) PropertyConfigDefaults(dataType *schema.DataType) map[string]interface{} { - // no property-specific config for this module - return nil -} - -func (m *CentroidModule) ValidateClass(ctx context.Context, - class *models.Class, classConfig moduletools.ClassConfig, -) error { - err := config.Validate(config.New(classConfig)) - if err != nil { - return fmt.Errorf("validate %q: %w", class.Class, err) - } - return nil -} - -var _ = modulecapabilities.ClassConfigurator(New()) diff --git a/modules/ref2vec-centroid/config/config.go b/modules/ref2vec-centroid/config/config.go deleted file mode 100644 index c04c753cdcfea9e456414e46e973c55f52c2fb48..0000000000000000000000000000000000000000 --- a/modules/ref2vec-centroid/config/config.go +++ /dev/null @@ -1,56 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package config - -import "github.com/weaviate/weaviate/entities/moduletools" - -const ( - MethodMean = "mean" - MethodDefault = MethodMean -) - -const ( - calculationMethodField = "method" - referencePropertiesField = "referenceProperties" -) - -func Default() map[string]interface{} { - return map[string]interface{}{ - calculationMethodField: MethodDefault, - } -} - -type Config struct { - class moduletools.ClassConfig -} - -func New(cfg moduletools.ClassConfig) *Config { - return &Config{class: cfg} -} - -func (c *Config) ReferenceProperties() map[string]struct{} { - refProps := map[string]struct{}{} - props := c.class.Class() - - iRefProps := props[referencePropertiesField].([]interface{}) - for _, iProp := range iRefProps { - refProps[iProp.(string)] = struct{}{} - } - - return refProps -} - -func (c *Config) CalculationMethod() string { - props := c.class.Class() - calcMethod := props[calculationMethodField].(string) - return calcMethod -} diff --git a/modules/ref2vec-centroid/config/validation.go b/modules/ref2vec-centroid/config/validation.go deleted file mode 100644 index 9a73daa0f80955bf21cd9aa7abeb594a19f63548..0000000000000000000000000000000000000000 --- a/modules/ref2vec-centroid/config/validation.go +++ /dev/null @@ -1,50 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package config - -import ( - "errors" - "fmt" -) - -var errInvalidConfig = errors.New("invalid config") - -func Validate(cfg *Config) error { - // referencePropertiesField is a required field - class := cfg.class.Class() - refProps, ok := class[referencePropertiesField] - if !ok { - return fmt.Errorf("%w: must have at least one value in the %q field", - errInvalidConfig, referencePropertiesField) - } - - propSlice, ok := refProps.([]interface{}) - if !ok { - return fmt.Errorf("%w: expected array for field %q, got %T", - errInvalidConfig, referencePropertiesField, refProps) - } - - if len(propSlice) == 0 { - return fmt.Errorf("%w: must have at least one value in the %q field", - errInvalidConfig, referencePropertiesField) - } - - // all provided property names must be strings - for _, prop := range propSlice { - if _, ok := prop.(string); !ok { - return fmt.Errorf("%w: expected %q to contain strings, found %T: %+v", - errInvalidConfig, referencePropertiesField, prop, refProps) - } - } - - return nil -} diff --git a/modules/ref2vec-centroid/config_test.go b/modules/ref2vec-centroid/config_test.go deleted file mode 100644 index c71e9b34097f21d6e0a287d61988ad7bbf613ba4..0000000000000000000000000000000000000000 --- a/modules/ref2vec-centroid/config_test.go +++ /dev/null @@ -1,99 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modcentroid - -import ( - "context" - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/modules/ref2vec-centroid/config" -) - -func TestConfigDefaults(t *testing.T) { - def := New().ClassConfigDefaults() - cfg := config.New(fakeClassConfig(def)) - - assert.Equal(t, config.MethodDefault, cfg.CalculationMethod()) -} - -func TestConfigValidator(t *testing.T) { - class := &models.Class{Class: "CentroidClass"} - - tests := []struct { - name string - class *models.Class - classConfig moduletools.ClassConfig - expectedErr error - }{ - { - name: "valid config", - class: class, - classConfig: fakeClassConfig{ - "referenceProperties": []interface{}{"someRef"}, - }, - }, - { - name: "invalid config - required fields omitted", - class: class, - classConfig: fakeClassConfig{}, - expectedErr: fmt.Errorf("validate %q: invalid config: must have at least "+ - "one value in the \"referenceProperties\" field", - class.Class), - }, - { - name: "invalid config - wrong type for referenceProperties", - class: class, - classConfig: fakeClassConfig{ - "referenceProperties": "someRef", - }, - expectedErr: fmt.Errorf("validate %q: invalid config: expected array for "+ - "field \"referenceProperties\", got string", - class.Class), - }, - { - name: "invalid config - empty referenceProperties slice", - class: class, - classConfig: fakeClassConfig{ - "referenceProperties": []interface{}{}, - }, - expectedErr: fmt.Errorf("validate %q: invalid config: must have at least "+ - "one value in the \"referenceProperties\" field", - class.Class), - }, - { - name: "invalid config - non-string value in referenceProperties array", - class: class, - classConfig: fakeClassConfig{ - "referenceProperties": []interface{}{"someRef", 123}, - }, - expectedErr: fmt.Errorf("validate %q: invalid config: expected \"referenceProperties\" "+ - "to contain strings, found int: [someRef 123]", - class.Class), - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - mod := New() - err := mod.ValidateClass(context.Background(), test.class, test.classConfig) - if test.expectedErr != nil { - assert.EqualError(t, err, test.expectedErr.Error()) - } else { - assert.Nil(t, err) - } - }) - } -} diff --git a/modules/ref2vec-centroid/fakes_for_test.go b/modules/ref2vec-centroid/fakes_for_test.go deleted file mode 100644 index a5a66af6255dce91f9df6934b23f87141000ca64..0000000000000000000000000000000000000000 --- a/modules/ref2vec-centroid/fakes_for_test.go +++ /dev/null @@ -1,53 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modcentroid - -import ( - "testing" - - "github.com/weaviate/weaviate/entities/moduletools" -) - -type fakeClassConfig map[string]interface{} - -func (cfg fakeClassConfig) Class() map[string]interface{} { - return cfg -} - -func (cfg fakeClassConfig) ClassByModuleName(moduleName string) map[string]interface{} { - return cfg -} - -func (cfg fakeClassConfig) Property(string) map[string]interface{} { - return nil -} - -func (f fakeClassConfig) Tenant() string { - return "" -} - -func newFakeStorageProvider(t *testing.T) *fakeStorageProvider { - dirName := t.TempDir() - return &fakeStorageProvider{dirName} -} - -type fakeStorageProvider struct { - dataPath string -} - -func (sp fakeStorageProvider) Storage(name string) (moduletools.Storage, error) { - return nil, nil -} - -func (sp fakeStorageProvider) DataPath() string { - return sp.dataPath -} diff --git a/modules/ref2vec-centroid/module.go b/modules/ref2vec-centroid/module.go deleted file mode 100644 index 0c1d64c0f9e55bada1338d10409ae7940649ee7b..0000000000000000000000000000000000000000 --- a/modules/ref2vec-centroid/module.go +++ /dev/null @@ -1,72 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modcentroid - -import ( - "context" - "net/http" - - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/modules/ref2vec-centroid/vectorizer" -) - -const ( - Name = "ref2vec-centroid" -) - -func New() *CentroidModule { - return &CentroidModule{} -} - -type CentroidModule struct { - logger logrus.FieldLogger -} - -func (m *CentroidModule) Name() string { - return Name -} - -func (m *CentroidModule) Init(ctx context.Context, params moduletools.ModuleInitParams) error { - m.logger = params.GetLogger() - return nil -} - -func (m *CentroidModule) RootHandler() http.Handler { - // TODO: remove from overall module, this is a capability - return nil -} - -func (m *CentroidModule) Type() modulecapabilities.ModuleType { - return modulecapabilities.Ref2Vec -} - -func (m *CentroidModule) MetaInfo() (map[string]interface{}, error) { - return map[string]interface{}{}, nil -} - -func (m *CentroidModule) VectorizeObject(ctx context.Context, - obj *models.Object, cfg moduletools.ClassConfig, - findRefVecsFn modulecapabilities.FindObjectFn, -) error { - vzr := vectorizer.New(cfg, findRefVecsFn) - return vzr.Object(ctx, obj) -} - -// verify we implement the modules.Module interface -var ( - _ = modulecapabilities.Module(New()) - _ = modulecapabilities.ReferenceVectorizer(New()) - _ = modulecapabilities.MetaProvider(New()) -) diff --git a/modules/ref2vec-centroid/module_test.go b/modules/ref2vec-centroid/module_test.go deleted file mode 100644 index 565e892500f68f794f2b36c8d212206e7fe49c6c..0000000000000000000000000000000000000000 --- a/modules/ref2vec-centroid/module_test.go +++ /dev/null @@ -1,173 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modcentroid - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/go-openapi/strfmt" - "github.com/google/uuid" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/schema/crossref" - "github.com/weaviate/weaviate/entities/search" - "github.com/weaviate/weaviate/usecases/config" -) - -func TestRef2VecCentroid(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) - defer cancel() - sp := newFakeStorageProvider(t) - logger, _ := test.NewNullLogger() - params := moduletools.NewInitParams(sp, nil, config.Config{}, logger) - - mod := New() - classConfig := fakeClassConfig(mod.ClassConfigDefaults()) - refProp := "someRef" - classConfig["referenceProperties"] = []interface{}{refProp} - - t.Run("Init", func(t *testing.T) { - err := mod.Init(ctx, params) - assert.Nil(t, err) - }) - - t.Run("RootHandler", func(t *testing.T) { - h := mod.RootHandler() - assert.Nil(t, h) - }) - - t.Run("Type", func(t *testing.T) { - typ := mod.Type() - assert.Equal(t, modulecapabilities.Ref2Vec, typ) - }) - - t.Run("Name", func(t *testing.T) { - name := mod.Name() - assert.Equal(t, Name, name) - }) - - t.Run("MetaInfo", func(t *testing.T) { - meta, err := mod.MetaInfo() - assert.Nil(t, err) - assert.Empty(t, meta) - }) - - t.Run("PropertyConfigDefaults", func(t *testing.T) { - dt := schema.DataType("dataType") - props := mod.PropertyConfigDefaults(&dt) - assert.Nil(t, props) - }) - - t.Run("ValidateClass", func(t *testing.T) { - t.Run("expected success", func(t *testing.T) { - class := &models.Class{} - - err := mod.ValidateClass(ctx, class, classConfig) - assert.Nil(t, err) - }) - - t.Run("expected error", func(t *testing.T) { - class := &models.Class{Class: "InvalidConfigClass"} - cfg := fakeClassConfig{} - - expectedErr := fmt.Sprintf( - "validate %q: invalid config: must have at least one "+ - "value in the \"referenceProperties\" field", - class.Class) - err := mod.ValidateClass(ctx, class, cfg) - assert.EqualError(t, err, expectedErr) - }) - }) - - t.Run("VectorizeObject", func(t *testing.T) { - t.Run("expected success", func(t *testing.T) { - t.Run("one refVec", func(t *testing.T) { - repo := &fakeObjectsRepo{} - ref := crossref.New("localhost", "SomeClass", strfmt.UUID(uuid.NewString())) - obj := &models.Object{Properties: map[string]interface{}{ - refProp: models.MultipleRef{ref.SingleRef()}, - }} - - repo.On("Object", ctx, ref.Class, ref.TargetID). - Return(&search.Result{Vector: []float32{1, 2, 3}}, nil) - - err := mod.VectorizeObject(ctx, obj, classConfig, repo.Object) - assert.Nil(t, err) - expectedVec := models.C11yVector{1, 2, 3} - assert.EqualValues(t, expectedVec, obj.Vector) - }) - - t.Run("no refVecs", func(t *testing.T) { - repo := &fakeObjectsRepo{} - ref := crossref.New("localhost", "SomeClass", strfmt.UUID(uuid.NewString())) - obj := &models.Object{Properties: map[string]interface{}{ - refProp: models.MultipleRef{ref.SingleRef()}, - }} - - repo.On("Object", ctx, ref.Class, ref.TargetID). - Return(&search.Result{}, nil) - - err := mod.VectorizeObject(ctx, obj, classConfig, repo.Object) - assert.Nil(t, err) - assert.Nil(t, nil, obj.Vector) - }) - }) - - t.Run("expected error", func(t *testing.T) { - t.Run("mismatched refVec lengths", func(t *testing.T) { - repo := &fakeObjectsRepo{} - ref1 := crossref.New("localhost", "SomeClass", strfmt.UUID(uuid.NewString())) - ref2 := crossref.New("localhost", "OtherClass", strfmt.UUID(uuid.NewString())) - obj := &models.Object{Properties: map[string]interface{}{ - refProp: models.MultipleRef{ - ref1.SingleRef(), - ref2.SingleRef(), - }, - }} - expectedErr := fmt.Errorf("calculate vector: calculate mean: " + - "found vectors of different length: 2 and 3") - - repo.On("Object", ctx, ref1.Class, ref1.TargetID). - Return(&search.Result{Vector: []float32{1, 2}}, nil) - repo.On("Object", ctx, ref2.Class, ref2.TargetID). - Return(&search.Result{Vector: []float32{1, 2, 3}}, nil) - - err := mod.VectorizeObject(ctx, obj, classConfig, repo.Object) - assert.EqualError(t, err, expectedErr.Error()) - }) - }) - }) -} - -type fakeObjectsRepo struct { - mock.Mock -} - -func (r *fakeObjectsRepo) Object(ctx context.Context, class string, - id strfmt.UUID, props search.SelectProperties, - addl additional.Properties, tenant string, -) (*search.Result, error) { - args := r.Called(ctx, class, id) - if args.Get(0) == nil { - return nil, args.Error(1) - } - return args.Get(0).(*search.Result), args.Error(1) -} diff --git a/modules/ref2vec-centroid/vectorizer/fakes_for_test.go b/modules/ref2vec-centroid/vectorizer/fakes_for_test.go deleted file mode 100644 index b8e88f7b8a2a61662c7b3ce051a89b0fc617d27d..0000000000000000000000000000000000000000 --- a/modules/ref2vec-centroid/vectorizer/fakes_for_test.go +++ /dev/null @@ -1,54 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - - "github.com/go-openapi/strfmt" - "github.com/stretchr/testify/mock" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/search" -) - -type fakeClassConfig map[string]interface{} - -func (cfg fakeClassConfig) Class() map[string]interface{} { - return cfg -} - -func (cfg fakeClassConfig) ClassByModuleName(moduleName string) map[string]interface{} { - return cfg -} - -func (cfg fakeClassConfig) Property(string) map[string]interface{} { - return nil -} - -func (f fakeClassConfig) Tenant() string { - return "" -} - -type fakeObjectsRepo struct { - mock.Mock -} - -func (r *fakeObjectsRepo) Object(ctx context.Context, class string, - id strfmt.UUID, props search.SelectProperties, - addl additional.Properties, tenant string, -) (*search.Result, error) { - args := r.Called(ctx, class, id, tenant) - if args.Get(0) == nil { - return nil, args.Error(1) - } - return args.Get(0).(*search.Result), args.Error(1) -} diff --git a/modules/ref2vec-centroid/vectorizer/method_mean.go b/modules/ref2vec-centroid/vectorizer/method_mean.go deleted file mode 100644 index 09ae832e6b730765c44c44e0c2ee294ba8c9b9be..0000000000000000000000000000000000000000 --- a/modules/ref2vec-centroid/vectorizer/method_mean.go +++ /dev/null @@ -1,41 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import "fmt" - -func calculateMean(refVecs ...[]float32) ([]float32, error) { - if len(refVecs) == 0 || len(refVecs[0]) == 0 { - return nil, nil - } - - targetVecLen := len(refVecs[0]) - meanVec := make([]float32, targetVecLen) - - // TODO: is there a more efficient way of doing this? - for _, vec := range refVecs { - if len(vec) != targetVecLen { - return nil, fmt.Errorf("calculate mean: found vectors of different length: %d and %d", - targetVecLen, len(vec)) - } - - for i, val := range vec { - meanVec[i] += val - } - } - - for i := range meanVec { - meanVec[i] /= float32(len(refVecs)) - } - - return meanVec, nil -} diff --git a/modules/ref2vec-centroid/vectorizer/vectorizer.go b/modules/ref2vec-centroid/vectorizer/vectorizer.go deleted file mode 100644 index 3061f7fa6331f6795d44ec6eec52a46735c51dd4..0000000000000000000000000000000000000000 --- a/modules/ref2vec-centroid/vectorizer/vectorizer.go +++ /dev/null @@ -1,142 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - "fmt" - - "github.com/go-openapi/strfmt" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/schema/crossref" - "github.com/weaviate/weaviate/entities/search" - "github.com/weaviate/weaviate/modules/ref2vec-centroid/config" -) - -type calcFn func(vecs ...[]float32) ([]float32, error) - -type Vectorizer struct { - config *config.Config - calcFn calcFn - findObjectFn modulecapabilities.FindObjectFn -} - -func New(cfg moduletools.ClassConfig, findFn modulecapabilities.FindObjectFn) *Vectorizer { - v := &Vectorizer{ - config: config.New(cfg), - findObjectFn: findFn, - } - - switch v.config.CalculationMethod() { - case config.MethodMean: - v.calcFn = calculateMean - default: - v.calcFn = calculateMean - } - - return v -} - -func (v *Vectorizer) Object(ctx context.Context, obj *models.Object) error { - props := v.config.ReferenceProperties() - - refVecs, err := v.referenceVectorSearch(ctx, obj, props) - if err != nil { - return err - } - - if len(refVecs) == 0 { - obj.Vector = nil - return nil - } - - vec, err := v.calcFn(refVecs...) - if err != nil { - return fmt.Errorf("calculate vector: %w", err) - } - - obj.Vector = vec - return nil -} - -func (v *Vectorizer) referenceVectorSearch(ctx context.Context, - obj *models.Object, refProps map[string]struct{}, -) ([][]float32, error) { - var refVecs [][]float32 - props := obj.Properties.(map[string]interface{}) - - // use the ids from parent's beacons to find the referenced objects - beacons := beaconsForVectorization(props, refProps) - for _, beacon := range beacons { - res, err := v.findReferenceObject(ctx, beacon, obj.Tenant) - if err != nil { - return nil, err - } - - // if the ref'd object has a vector, we grab it. - // these will be used to compute the parent's - // vector eventually - if res.Vector != nil { - refVecs = append(refVecs, res.Vector) - } - } - - return refVecs, nil -} - -func (v *Vectorizer) findReferenceObject(ctx context.Context, beacon strfmt.URI, tenant string) (res *search.Result, err error) { - ref, err := crossref.Parse(beacon.String()) - if err != nil { - return nil, fmt.Errorf("parse beacon %q: %w", beacon, err) - } - - res, err = v.findObjectFn(ctx, ref.Class, ref.TargetID, - search.SelectProperties{}, additional.Properties{}, tenant) - if err != nil || res == nil { - if err == nil { - err = fmt.Errorf("not found") - } - err = fmt.Errorf("find object with beacon %q': %w", beacon, err) - } - return -} - -func beaconsForVectorization(allProps map[string]interface{}, - targetRefProps map[string]struct{}, -) []strfmt.URI { - var beacons []strfmt.URI - - // add any refs that were supplied as a part of the parent - // object, like when caller is AddObject/UpdateObject - for prop, val := range allProps { - if _, ok := targetRefProps[prop]; ok { - switch refs := val.(type) { - case []interface{}: - // due to the fix introduced in https://github.com/weaviate/weaviate/pull/2320, - // MultipleRef's can appear as empty []interface{} when no actual refs are provided for - // an object's reference property. - // - // if we encounter []interface{}, assume it indicates an empty ref prop, and skip it. - continue - case models.MultipleRef: - for _, ref := range refs { - beacons = append(beacons, ref.Beacon) - } - } - } - } - - return beacons -} diff --git a/modules/ref2vec-centroid/vectorizer/vectorizer_test.go b/modules/ref2vec-centroid/vectorizer/vectorizer_test.go deleted file mode 100644 index f54a44d92a374276e4dabd7aa3a214176c3e194e..0000000000000000000000000000000000000000 --- a/modules/ref2vec-centroid/vectorizer/vectorizer_test.go +++ /dev/null @@ -1,194 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - "errors" - "reflect" - "testing" - - "github.com/go-openapi/strfmt" - "github.com/google/uuid" - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema/crossref" - "github.com/weaviate/weaviate/entities/search" - "github.com/weaviate/weaviate/modules/ref2vec-centroid/config" -) - -func TestVectorizer_New(t *testing.T) { - repo := &fakeObjectsRepo{} - t.Run("default is set correctly", func(t *testing.T) { - vzr := New(fakeClassConfig(config.Default()), repo.Object) - - expected := reflect.ValueOf(calculateMean).Pointer() - received := reflect.ValueOf(vzr.calcFn).Pointer() - - assert.EqualValues(t, expected, received) - }) - - t.Run("default calcFn is used when none provided", func(t *testing.T) { - cfg := fakeClassConfig{"method": ""} - vzr := New(cfg, repo.Object) - - expected := reflect.ValueOf(calculateMean).Pointer() - received := reflect.ValueOf(vzr.calcFn).Pointer() - - assert.EqualValues(t, expected, received) - }) -} - -func TestVectorizer_Object(t *testing.T) { - t.Run("calculate with mean", func(t *testing.T) { - type objectSearchResult struct { - res *search.Result - err error - } - - tests := []struct { - name string - objectSearchResults []objectSearchResult - expectedResult []float32 - expectedCalcError error - }{ - { - name: "expected success 1", - objectSearchResults: []objectSearchResult{ - {res: &search.Result{Vector: []float32{2, 4, 6}}}, - {res: &search.Result{Vector: []float32{4, 6, 8}}}, - }, - expectedResult: []float32{3, 5, 7}, - }, - { - name: "expected success 2", - objectSearchResults: []objectSearchResult{ - {res: &search.Result{Vector: []float32{1, 1, 1, 1, 1, 1, 1, 1, 1, 1}}}, - {res: &search.Result{Vector: []float32{2, 2, 2, 2, 2, 2, 2, 2, 2, 2}}}, - {res: &search.Result{Vector: []float32{3, 3, 3, 3, 3, 3, 3, 3, 3, 3}}}, - {res: &search.Result{Vector: []float32{4, 4, 4, 4, 4, 4, 4, 4, 4, 4}}}, - {res: &search.Result{Vector: []float32{5, 5, 5, 5, 5, 5, 5, 5, 5, 5}}}, - {res: &search.Result{Vector: []float32{6, 6, 6, 6, 6, 6, 6, 6, 6, 6}}}, - {res: &search.Result{Vector: []float32{7, 7, 7, 7, 7, 7, 7, 7, 7, 7}}}, - {res: &search.Result{Vector: []float32{8, 8, 8, 8, 8, 8, 8, 8, 8, 8}}}, - {res: &search.Result{Vector: []float32{9, 9, 9, 9, 9, 9, 9, 9, 9, 9}}}, - }, - expectedResult: []float32{5, 5, 5, 5, 5, 5, 5, 5, 5, 5}, - }, - { - name: "expected success 3", - objectSearchResults: []objectSearchResult{{}}, - }, - { - name: "expected success 4", - objectSearchResults: []objectSearchResult{ - {res: &search.Result{Vector: []float32{1, 2, 3, 4, 5, 6, 7, 8, 9}}}, - }, - expectedResult: []float32{1, 2, 3, 4, 5, 6, 7, 8, 9}, - }, - { - name: "expected success 5", - objectSearchResults: []objectSearchResult{ - {res: &search.Result{}}, - }, - expectedResult: nil, - }, - { - name: "expected error - mismatched vector dimensions", - objectSearchResults: []objectSearchResult{ - {res: &search.Result{Vector: []float32{1, 2, 3, 4, 5, 6, 7, 8, 9}}}, - {res: &search.Result{Vector: []float32{1, 2, 3, 4, 5, 6, 7, 8}}}, - }, - expectedCalcError: errors.New( - "calculate vector: calculate mean: found vectors of different length: 9 and 8"), - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - ctx := context.Background() - repo := &fakeObjectsRepo{} - refProps := []interface{}{"toRef"} - cfg := fakeClassConfig{"method": "mean", "referenceProperties": refProps} - - crossRefs := make([]*crossref.Ref, len(test.objectSearchResults)) - modelRefs := make(models.MultipleRef, len(test.objectSearchResults)) - for i, res := range test.objectSearchResults { - crossRef := crossref.New("localhost", "SomeClass", - strfmt.UUID(uuid.NewString())) - crossRefs[i] = crossRef - modelRefs[i] = crossRef.SingleRef() - - repo.On("Object", ctx, crossRef.Class, crossRef.TargetID, ""). - Return(res.res, res.err) - } - - obj := &models.Object{ - Properties: map[string]interface{}{"toRef": modelRefs}, - } - - err := New(cfg, repo.Object).Object(ctx, obj) - if test.expectedCalcError != nil { - assert.EqualError(t, err, test.expectedCalcError.Error()) - } else { - assert.EqualValues(t, test.expectedResult, obj.Vector) - } - }) - } - }) - - // due to the fix introduced in https://github.com/weaviate/weaviate/pull/2320, - // MultipleRef's can appear as empty []interface{} when no actual refs are provided for - // an object's reference property. - // - // this test asserts that reference properties do not break when they are unmarshalled - // as empty interface{} slices. - t.Run("when rep prop is stored as empty interface{} slice", func(t *testing.T) { - ctx := context.Background() - repo := &fakeObjectsRepo{} - refProps := []interface{}{"toRef"} - cfg := fakeClassConfig{"method": "mean", "referenceProperties": refProps} - - obj := &models.Object{ - Properties: map[string]interface{}{"toRef": []interface{}{}}, - } - - err := New(cfg, repo.Object).Object(ctx, obj) - assert.Nil(t, err) - assert.Nil(t, obj.Vector) - }) -} - -func TestVectorizer_Tenant(t *testing.T) { - objectSearchResults := search.Result{Vector: []float32{}} - ctx := context.Background() - repo := &fakeObjectsRepo{} - refProps := []interface{}{"toRef"} - cfg := fakeClassConfig{"method": "mean", "referenceProperties": refProps} - tenant := "randomTenant" - - crossRef := crossref.New("localhost", "SomeClass", - strfmt.UUID(uuid.NewString())) - modelRefs := models.MultipleRef{crossRef.SingleRef()} - - repo.On("Object", ctx, crossRef.Class, crossRef.TargetID, tenant). - Return(&objectSearchResults, nil) - - obj := &models.Object{ - Properties: map[string]interface{}{"toRef": modelRefs}, - Tenant: tenant, - } - - err := New(cfg, repo.Object).Object(ctx, obj) - assert.Nil(t, err) - assert.Nil(t, obj.Vector) -} diff --git a/modules/reranker-cohere/clients/ranker.go b/modules/reranker-cohere/clients/ranker.go deleted file mode 100644 index d3575842675ffd4b76767fe56193efc039fd1b30..0000000000000000000000000000000000000000 --- a/modules/reranker-cohere/clients/ranker.go +++ /dev/null @@ -1,246 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "runtime" - "sync" - "time" - - "github.com/weaviate/weaviate/usecases/modulecomponents" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/modules/reranker-cohere/config" - "github.com/weaviate/weaviate/usecases/modulecomponents/ent" - "golang.org/x/sync/errgroup" -) - -var _NUMCPU = runtime.NumCPU() - -type client struct { - lock sync.RWMutex - apiKey string - host string - path string - httpClient *http.Client - maxDocuments int - logger logrus.FieldLogger -} - -func New(apiKey string, timeout time.Duration, logger logrus.FieldLogger) *client { - return &client{ - apiKey: apiKey, - httpClient: &http.Client{Timeout: timeout}, - host: "https://api.cohere.ai", - path: "/v1/rerank", - maxDocuments: 1000, - logger: logger, - } -} - -func (c *client) Rank(ctx context.Context, query string, documents []string, - cfg moduletools.ClassConfig, -) (*ent.RankResult, error) { - eg := &errgroup.Group{} - eg.SetLimit(_NUMCPU) - - chunkedDocuments := c.chunkDocuments(documents, c.maxDocuments) - documentScoreResponses := make([][]ent.DocumentScore, len(chunkedDocuments)) - for i := range chunkedDocuments { - i := i // https://golang.org/doc/faq#closures_and_goroutines - eg.Go(func() error { - documentScoreResponse, err := c.performRank(ctx, query, chunkedDocuments[i], cfg) - if err != nil { - return err - } - c.lockGuard(func() { - documentScoreResponses[i] = documentScoreResponse - }) - return nil - }) - } - if err := eg.Wait(); err != nil { - return nil, err - } - - return c.toRankResult(query, documentScoreResponses), nil -} - -func (c *client) lockGuard(mutate func()) { - c.lock.Lock() - defer c.lock.Unlock() - mutate() -} - -func (c *client) performRank(ctx context.Context, query string, documents []string, - cfg moduletools.ClassConfig, -) ([]ent.DocumentScore, error) { - settings := config.NewClassSettings(cfg) - cohereUrl, err := url.JoinPath(c.host, c.path) - if err != nil { - return nil, errors.Wrap(err, "join Cohere API host and path") - } - - input := RankInput{ - Documents: documents, - Query: query, - Model: settings.Model(), - ReturnDocuments: false, - } - - body, err := json.Marshal(input) - if err != nil { - return nil, errors.Wrapf(err, "marshal body") - } - - req, err := http.NewRequestWithContext(ctx, "POST", cohereUrl, bytes.NewReader(body)) - if err != nil { - return nil, errors.Wrap(err, "create POST request") - } - - apiKey, err := c.getApiKey(ctx) - if err != nil { - return nil, errors.Wrapf(err, "Cohere API Key") - } - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", apiKey)) - req.Header.Add("Content-Type", "application/json") - - res, err := c.httpClient.Do(req) - if err != nil { - return nil, errors.Wrap(err, "send POST request") - } - defer res.Body.Close() - - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - return nil, errors.Wrap(err, "read response body") - } - - if res.StatusCode != 200 { - var apiError cohereApiError - err = json.Unmarshal(bodyBytes, &apiError) - if err != nil { - return nil, errors.Wrap(err, "unmarshal error from response body") - } - if apiError.Message != "" { - return nil, errors.Errorf("connection to Cohere API failed with status %d: %s", res.StatusCode, apiError.Message) - } - return nil, errors.Errorf("connection to Cohere API failed with status %d", res.StatusCode) - } - - var rankResponse RankResponse - if err := json.Unmarshal(bodyBytes, &rankResponse); err != nil { - return nil, errors.Wrap(err, "unmarshal response body") - } - return c.toDocumentScores(documents, rankResponse.Results), nil -} - -func (c *client) chunkDocuments(documents []string, chunkSize int) [][]string { - var requests [][]string - for i := 0; i < len(documents); i += chunkSize { - end := i + chunkSize - - if end > len(documents) { - end = len(documents) - } - - requests = append(requests, documents[i:end]) - } - - return requests -} - -func (c *client) toDocumentScores(documents []string, results []Result) []ent.DocumentScore { - documentScores := make([]ent.DocumentScore, len(results)) - for _, result := range results { - documentScores[result.Index] = ent.DocumentScore{ - Document: documents[result.Index], - Score: result.RelevanceScore, - } - } - return documentScores -} - -func (c *client) toRankResult(query string, results [][]ent.DocumentScore) *ent.RankResult { - documentScores := []ent.DocumentScore{} - for i := range results { - documentScores = append(documentScores, results[i]...) - } - return &ent.RankResult{ - Query: query, - DocumentScores: documentScores, - } -} - -func (c *client) getApiKey(ctx context.Context) (string, error) { - if len(c.apiKey) > 0 { - return c.apiKey, nil - } - key := "X-Cohere-Api-Key" - - apiKey := ctx.Value(key) - // try getting header from GRPC if not successful - if apiKey == nil { - apiKey = modulecomponents.GetValueFromGRPC(ctx, key) - } - if apiKeyHeader, ok := apiKey.([]string); ok && - len(apiKeyHeader) > 0 && len(apiKeyHeader[0]) > 0 { - return apiKeyHeader[0], nil - } - return "", errors.New("no api key found " + - "neither in request header: X-Cohere-Api-Key " + - "nor in environment variable under COHERE_APIKEY") -} - -type RankInput struct { - Documents []string `json:"documents"` - Query string `json:"query"` - Model string `json:"model"` - ReturnDocuments bool `json:"return_documents"` -} - -type Document struct { - Text string `json:"text"` -} - -type Result struct { - Index int `json:"index"` - RelevanceScore float64 `json:"relevance_score"` - Document Document `json:"document"` -} - -type APIVersion struct { - Version string `json:"version"` -} - -type Meta struct { - APIVersion APIVersion `json:"api_version"` -} - -type RankResponse struct { - ID string `json:"id"` - Results []Result `json:"results"` - Meta Meta `json:"meta"` -} - -type cohereApiError struct { - Message string `json:"message"` -} diff --git a/modules/reranker-cohere/clients/ranker_meta.go b/modules/reranker-cohere/clients/ranker_meta.go deleted file mode 100644 index 9c3828115755dcb00663ab99aeb04983740c3d8e..0000000000000000000000000000000000000000 --- a/modules/reranker-cohere/clients/ranker_meta.go +++ /dev/null @@ -1,19 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -func (s *client) MetaInfo() (map[string]interface{}, error) { - return map[string]interface{}{ - "name": "Reranker - Cohere", - "documentationHref": "https://txt.cohere.com/rerank/", - }, nil -} diff --git a/modules/reranker-cohere/clients/ranker_test.go b/modules/reranker-cohere/clients/ranker_test.go deleted file mode 100644 index 74d96efd75644ad23fd024238024ef0493ef9ad0..0000000000000000000000000000000000000000 --- a/modules/reranker-cohere/clients/ranker_test.go +++ /dev/null @@ -1,215 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "context" - "encoding/json" - "io" - "net/http" - "net/http/httptest" - "sync" - "testing" - - "github.com/sirupsen/logrus" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/usecases/modulecomponents/ent" -) - -func nullLogger() logrus.FieldLogger { - l, _ := test.NewNullLogger() - return l -} - -func TestRank(t *testing.T) { - t.Run("when the server has a successful response", func(t *testing.T) { - handler := &testRankHandler{ - t: t, - response: RankResponse{ - Results: []Result{ - { - Index: 0, - RelevanceScore: 0.9, - }, - }, - }, - } - server := httptest.NewServer(handler) - defer server.Close() - - c := New("apiKey", 0, nullLogger()) - c.host = server.URL - - expected := &ent.RankResult{ - DocumentScores: []ent.DocumentScore{ - { - Document: "I work at Apple", - Score: 0.9, - }, - }, - Query: "Where do I work?", - } - - res, err := c.Rank(context.Background(), "Where do I work?", []string{"I work at Apple"}, nil) - - assert.Nil(t, err) - assert.Equal(t, expected, res) - }) - - t.Run("when the server has an error", func(t *testing.T) { - handler := &testRankHandler{ - t: t, - response: RankResponse{ - Results: []Result{}, - }, - errorMessage: "some error from the server", - } - server := httptest.NewServer(handler) - defer server.Close() - - c := New("apiKey", 0, nullLogger()) - c.host = server.URL - - _, err := c.Rank(context.Background(), "I work at Apple", []string{"Where do I work?"}, nil) - - require.NotNil(t, err) - assert.Contains(t, err.Error(), "some error from the server") - }) - - t.Run("when we send requests in batches", func(t *testing.T) { - handler := &testRankHandler{ - t: t, - batchedResults: [][]Result{ - { - { - Index: 0, - RelevanceScore: 0.99, - }, - { - Index: 1, - RelevanceScore: 0.89, - }, - }, - { - { - Index: 0, - RelevanceScore: 0.19, - }, - { - Index: 1, - RelevanceScore: 0.29, - }, - }, - { - { - Index: 0, - RelevanceScore: 0.79, - }, - { - Index: 1, - RelevanceScore: 0.789, - }, - }, - { - { - Index: 0, - RelevanceScore: 0.0001, - }, - }, - }, - } - server := httptest.NewServer(handler) - defer server.Close() - - c := New("apiKey", 0, nullLogger()) - c.host = server.URL - // this will trigger 4 go routines - c.maxDocuments = 2 - - query := "Where do I work?" - documents := []string{ - "Response 1", "Response 2", "Response 3", "Response 4", - "Response 5", "Response 6", "Response 7", - } - - resp, err := c.Rank(context.Background(), query, documents, nil) - - require.Nil(t, err) - require.NotNil(t, resp) - require.NotNil(t, resp.DocumentScores) - for i := range resp.DocumentScores { - assert.Equal(t, documents[i], resp.DocumentScores[i].Document) - if i == 0 { - assert.Equal(t, 0.99, resp.DocumentScores[i].Score) - } - if i == len(documents)-1 { - assert.Equal(t, 0.0001, resp.DocumentScores[i].Score) - } - } - }) -} - -type testRankHandler struct { - lock sync.RWMutex - t *testing.T - response RankResponse - batchedResults [][]Result - errorMessage string -} - -func (f *testRankHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - f.lock.Lock() - defer f.lock.Unlock() - - if f.errorMessage != "" { - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(`{"message":"` + f.errorMessage + `"}`)) - return - } - - bodyBytes, err := io.ReadAll(r.Body) - require.Nil(f.t, err) - defer r.Body.Close() - - var req RankInput - require.Nil(f.t, json.Unmarshal(bodyBytes, &req)) - - containsDocument := func(req RankInput, in string) bool { - for _, doc := range req.Documents { - if doc == in { - return true - } - } - return false - } - - index := 0 - if len(f.batchedResults) > 0 { - if containsDocument(req, "Response 3") { - index = 1 - } - if containsDocument(req, "Response 5") { - index = 2 - } - if containsDocument(req, "Response 7") { - index = 3 - } - f.response.Results = f.batchedResults[index] - } - - outBytes, err := json.Marshal(f.response) - require.Nil(f.t, err) - - w.Write(outBytes) -} diff --git a/modules/reranker-cohere/config.go b/modules/reranker-cohere/config.go deleted file mode 100644 index e40394ff035741da30658461afc15281d54dcc60..0000000000000000000000000000000000000000 --- a/modules/reranker-cohere/config.go +++ /dev/null @@ -1,39 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modrerankercohere - -import ( - "context" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/schema" -) - -func (m *ReRankerCohereModule) ClassConfigDefaults() map[string]interface{} { - return map[string]interface{}{} -} - -func (m *ReRankerCohereModule) PropertyConfigDefaults( - dt *schema.DataType, -) map[string]interface{} { - return map[string]interface{}{} -} - -func (m *ReRankerCohereModule) ValidateClass(ctx context.Context, - class *models.Class, cfg moduletools.ClassConfig, -) error { - return nil -} - -var _ = modulecapabilities.ClassConfigurator(New()) diff --git a/modules/reranker-cohere/config/class_settings.go b/modules/reranker-cohere/config/class_settings.go deleted file mode 100644 index df1a27dedc83d8c9c0864eaa870f5b39edab66e3..0000000000000000000000000000000000000000 --- a/modules/reranker-cohere/config/class_settings.go +++ /dev/null @@ -1,88 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package config - -import ( - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/moduletools" -) - -const ( - modelProperty = "model" -) - -var availableCohereModels = []string{ - "rerank-english-v2.0", - "rerank-multilingual-v2.0", -} - -// note it might not like this -- might want int values for e.g. MaxTokens -var ( - DefaultCohereModel = "rerank-multilingual-v2.0" -) - -type classSettings struct { - cfg moduletools.ClassConfig -} - -func NewClassSettings(cfg moduletools.ClassConfig) *classSettings { - return &classSettings{cfg: cfg} -} - -func (ic *classSettings) Validate(class *models.Class) error { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return errors.New("empty config") - } - model := ic.getStringProperty(modelProperty, DefaultCohereModel) - if model == nil || !ic.validateModel(*model) { - return errors.Errorf("wrong Cohere model name, available model names are: %v", availableCohereModels) - } - - return nil -} - -func (ic *classSettings) getStringProperty(name string, defaultValue string) *string { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return &defaultValue - } - - model, ok := ic.cfg.ClassByModuleName("reranker-cohere")[name] - if ok { - asString, ok := model.(string) - if ok { - return &asString - } - var empty string - return &empty - } - return &defaultValue -} - -func (ic *classSettings) validateModel(model string) bool { - return contains(availableCohereModels, model) -} - -func (ic *classSettings) Model() string { - return *ic.getStringProperty(modelProperty, DefaultCohereModel) -} - -func contains[T comparable](s []T, e T) bool { - for _, v := range s { - if v == e { - return true - } - } - return false -} diff --git a/modules/reranker-cohere/config/class_settings_test.go b/modules/reranker-cohere/config/class_settings_test.go deleted file mode 100644 index d55150135ad20358d1467447a78129928c355104..0000000000000000000000000000000000000000 --- a/modules/reranker-cohere/config/class_settings_test.go +++ /dev/null @@ -1,85 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package config - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/entities/moduletools" -) - -func Test_classSettings_Validate(t *testing.T) { - tests := []struct { - name string - cfg moduletools.ClassConfig - wantModel string - wantErr error - }{ - { - name: "default settings", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{}, - }, - wantModel: "rerank-multilingual-v2.0", - }, - { - name: "custom settings", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "model": "rerank-english-v2.0", - }, - }, - wantModel: "rerank-english-v2.0", - }, - { - name: "unsupported model error", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "model": "rerank-french-v2.0", - }, - }, - wantErr: fmt.Errorf("wrong Cohere model name, available model names are: [rerank-english-v2.0 rerank-multilingual-v2.0]"), - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ic := NewClassSettings(tt.cfg) - if tt.wantErr != nil { - assert.EqualError(t, ic.Validate(nil), tt.wantErr.Error()) - } else { - assert.Equal(t, tt.wantModel, ic.Model()) - } - }) - } -} - -type fakeClassConfig struct { - classConfig map[string]interface{} -} - -func (f fakeClassConfig) Class() map[string]interface{} { - return f.classConfig -} - -func (f fakeClassConfig) Tenant() string { - return "" -} - -func (f fakeClassConfig) ClassByModuleName(moduleName string) map[string]interface{} { - return f.classConfig -} - -func (f fakeClassConfig) Property(propName string) map[string]interface{} { - return nil -} diff --git a/modules/reranker-cohere/module.go b/modules/reranker-cohere/module.go deleted file mode 100644 index 39cff258b9f500e00b60b4a2f2efc2b55bda85ba..0000000000000000000000000000000000000000 --- a/modules/reranker-cohere/module.go +++ /dev/null @@ -1,91 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modrerankercohere - -import ( - "context" - "net/http" - "os" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/modules/reranker-cohere/clients" - rerankeradditional "github.com/weaviate/weaviate/usecases/modulecomponents/additional" - "github.com/weaviate/weaviate/usecases/modulecomponents/ent" -) - -const Name = "reranker-cohere" - -func New() *ReRankerCohereModule { - return &ReRankerCohereModule{} -} - -type ReRankerCohereModule struct { - reranker ReRankerCohereClient - additionalPropertiesProvider modulecapabilities.AdditionalProperties -} - -type ReRankerCohereClient interface { - Rank(ctx context.Context, query string, documents []string, cfg moduletools.ClassConfig) (*ent.RankResult, error) - MetaInfo() (map[string]interface{}, error) -} - -func (m *ReRankerCohereModule) Name() string { - return Name -} - -func (m *ReRankerCohereModule) Type() modulecapabilities.ModuleType { - return modulecapabilities.Text2TextReranker -} - -func (m *ReRankerCohereModule) Init(ctx context.Context, - params moduletools.ModuleInitParams, -) error { - if err := m.initAdditional(ctx, params.GetConfig().ModuleHttpClientTimeout, params.GetLogger()); err != nil { - return errors.Wrap(err, "init cross encoder") - } - - return nil -} - -func (m *ReRankerCohereModule) initAdditional(ctx context.Context, timeout time.Duration, - logger logrus.FieldLogger, -) error { - apiKey := os.Getenv("COHERE_APIKEY") - client := clients.New(apiKey, timeout, logger) - m.reranker = client - m.additionalPropertiesProvider = rerankeradditional.NewRankerProvider(m.reranker) - return nil -} - -func (m *ReRankerCohereModule) MetaInfo() (map[string]interface{}, error) { - return m.reranker.MetaInfo() -} - -func (m *ReRankerCohereModule) RootHandler() http.Handler { - // TODO: remove once this is a capability interface - return nil -} - -func (m *ReRankerCohereModule) AdditionalProperties() map[string]modulecapabilities.AdditionalProperty { - return m.additionalPropertiesProvider.AdditionalProperties() -} - -// verify we implement the modules.Module interface -var ( - _ = modulecapabilities.Module(New()) - _ = modulecapabilities.AdditionalProperties(New()) - _ = modulecapabilities.MetaProvider(New()) -) diff --git a/modules/reranker-transformers/clients/ranker.go b/modules/reranker-transformers/clients/ranker.go deleted file mode 100644 index b7939cedb4a2650c7307cc47c95b0bebdb4558d1..0000000000000000000000000000000000000000 --- a/modules/reranker-transformers/clients/ranker.go +++ /dev/null @@ -1,181 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package client - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "runtime" - "sync" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/usecases/modulecomponents/ent" - "golang.org/x/sync/errgroup" -) - -var _NUMCPU = runtime.NumCPU() - -type client struct { - lock sync.RWMutex - origin string - httpClient *http.Client - maxDocuments int - logger logrus.FieldLogger -} - -func New(origin string, timeout time.Duration, logger logrus.FieldLogger) *client { - return &client{ - origin: origin, - httpClient: &http.Client{Timeout: timeout}, - maxDocuments: 32, - logger: logger, - } -} - -func (c *client) Rank(ctx context.Context, - query string, documents []string, cfg moduletools.ClassConfig, -) (*ent.RankResult, error) { - eg := &errgroup.Group{} - eg.SetLimit(_NUMCPU) - - chunkedDocuments := c.chunkDocuments(documents, c.maxDocuments) - documentScoreResponses := make([][]DocumentScore, len(chunkedDocuments)) - for i := range chunkedDocuments { - i := i // https://golang.org/doc/faq#closures_and_goroutines - eg.Go(func() error { - documentScoreResponse, err := c.performRank(ctx, query, chunkedDocuments[i], cfg) - if err != nil { - return err - } - c.lockGuard(func() { - documentScoreResponses[i] = documentScoreResponse - }) - return nil - }) - } - if err := eg.Wait(); err != nil { - return nil, err - } - - return c.toRankResult(query, documentScoreResponses), nil -} - -func (c *client) lockGuard(mutate func()) { - c.lock.Lock() - defer c.lock.Unlock() - mutate() -} - -func (c *client) toRankResult(query string, scores [][]DocumentScore) *ent.RankResult { - documentScores := []ent.DocumentScore{} - for _, docScores := range scores { - for i := range docScores { - documentScores = append(documentScores, ent.DocumentScore{ - Document: docScores[i].Document, - Score: docScores[i].Score, - }) - } - } - return &ent.RankResult{ - Query: query, - DocumentScores: documentScores, - } -} - -func (c *client) performRank(ctx context.Context, - query string, documents []string, cfg moduletools.ClassConfig, -) ([]DocumentScore, error) { - body, err := json.Marshal(RankInput{ - Query: query, - Documents: documents, - }) - if err != nil { - return nil, errors.Wrapf(err, "marshal body") - } - - req, err := http.NewRequestWithContext(ctx, "POST", c.url("/rerank"), - bytes.NewReader(body)) - if err != nil { - return nil, errors.Wrap(err, "create POST request") - } - - res, err := c.httpClient.Do(req) - if err != nil { - return nil, errors.Wrap(err, "send POST request") - } - defer res.Body.Close() - - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - return nil, errors.Wrap(err, "read response body") - } - - var resBody RankResponse - if err := json.Unmarshal(bodyBytes, &resBody); err != nil { - return nil, errors.Wrap(err, "unmarshal response body") - } - - if res.StatusCode != 200 { - if resBody.Error != "" { - return nil, errors.Errorf("fail with status %d: %s", res.StatusCode, - resBody.Error) - } - return nil, errors.Errorf("fail with status %d", res.StatusCode) - } - - return resBody.Scores, nil -} - -func (c *client) chunkDocuments(documents []string, chunkSize int) [][]string { - var requests [][]string - for i := 0; i < len(documents); i += chunkSize { - end := i + chunkSize - - if end > len(documents) { - end = len(documents) - } - - requests = append(requests, documents[i:end]) - } - - return requests -} - -func (c *client) url(path string) string { - return fmt.Sprintf("%s%s", c.origin, path) -} - -type RankInput struct { - Query string `json:"query"` - Documents []string `json:"documents"` - RankPropertyValue string `json:"property"` -} - -type DocumentScore struct { - Document string `json:"document"` - Score float64 `json:"score"` -} - -type RankResponse struct { - Query string `json:"query"` - Scores []DocumentScore `json:"scores"` - RankPropertyValue string `json:"property"` - Score float64 `json:"score"` - Error string `json:"error"` -} diff --git a/modules/reranker-transformers/clients/ranker_meta.go b/modules/reranker-transformers/clients/ranker_meta.go deleted file mode 100644 index 84765a6fc4f656f43944982bb2fc947869f8a8eb..0000000000000000000000000000000000000000 --- a/modules/reranker-transformers/clients/ranker_meta.go +++ /dev/null @@ -1,45 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package client - -import ( - "context" - "encoding/json" - "io" - "net/http" - - "github.com/pkg/errors" -) - -func (s *client) MetaInfo() (map[string]interface{}, error) { - req, err := http.NewRequestWithContext(context.Background(), "GET", s.url("/meta"), nil) - if err != nil { - return nil, errors.Wrap(err, "create GET meta request") - } - - res, err := s.httpClient.Do(req) - if err != nil { - return nil, errors.Wrap(err, "send GET meta request") - } - defer res.Body.Close() - - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - return nil, errors.Wrap(err, "read meta response body") - } - - var resBody map[string]interface{} - if err := json.Unmarshal(bodyBytes, &resBody); err != nil { - return nil, errors.Wrap(err, "unmarshal meta response body") - } - return resBody, nil -} diff --git a/modules/reranker-transformers/clients/ranker_meta_test.go b/modules/reranker-transformers/clients/ranker_meta_test.go deleted file mode 100644 index 193d0f3ece69cdf69f7e4a672aee6f701b536c5c..0000000000000000000000000000000000000000 --- a/modules/reranker-transformers/clients/ranker_meta_test.go +++ /dev/null @@ -1,156 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package client - -import ( - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func TestGetMeta(t *testing.T) { - t.Run("when the server is providing meta", func(t *testing.T) { - server := httptest.NewServer(&testMetaHandler{t: t}) - defer server.Close() - c := New(server.URL, 0, nullLogger()) - meta, err := c.MetaInfo() - - assert.Nil(t, err) - assert.NotNil(t, meta) - metaModel := meta["model"] - assert.True(t, metaModel != nil) - model, modelOK := metaModel.(map[string]interface{}) - assert.True(t, modelOK) - assert.True(t, model["_name_or_path"] != nil) - assert.True(t, model["architectures"] != nil) - }) -} - -type testMetaHandler struct { - t *testing.T - // the test handler will report as not ready before the time has passed - readyTime time.Time -} - -func (f *testMetaHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - assert.Equal(f.t, "/meta", r.URL.String()) - assert.Equal(f.t, http.MethodGet, r.Method) - - if time.Since(f.readyTime) < 0 { - w.WriteHeader(http.StatusServiceUnavailable) - } - - w.Write([]byte(f.metaInfo())) -} - -func (f *testMetaHandler) metaInfo() string { - return `{ - "model": { - "_name_or_path": "dbmdz/bert-large-cased-finetuned-conll03-english", - "_num_labels": 9, - "add_cross_attention": false, - "architectures": [ - "BertForTokenClassification" - ], - "attention_probs_dropout_prob": 0.1, - "bad_words_ids": null, - "bos_token_id": null, - "chunk_size_feed_forward": 0, - "decoder_start_token_id": null, - "directionality": "bidi", - "diversity_penalty": 0, - "do_sample": false, - "early_stopping": false, - "encoder_no_repeat_ngram_size": 0, - "eos_token_id": null, - "finetuning_task": null, - "forced_bos_token_id": null, - "forced_eos_token_id": null, - "gradient_checkpointing": false, - "hidden_act": "gelu", - "hidden_dropout_prob": 0.1, - "hidden_size": 1024, - "id2label": { - "0": "O", - "1": "B-MISC", - "2": "I-MISC", - "3": "B-PER", - "4": "I-PER", - "5": "B-ORG", - "6": "I-ORG", - "7": "B-LOC", - "8": "I-LOC" - }, - "initializer_range": 0.02, - "intermediate_size": 4096, - "is_decoder": false, - "is_encoder_decoder": false, - "label2id": { - "B-LOC": 7, - "B-MISC": 1, - "B-ORG": 5, - "B-PER": 3, - "I-LOC": 8, - "I-MISC": 2, - "I-ORG": 6, - "I-PER": 4, - "O": 0 - }, - "layer_norm_eps": 1e-12, - "length_penalty": 1, - "max_length": 20, - "max_position_embeddings": 512, - "min_length": 0, - "model_type": "bert", - "no_repeat_ngram_size": 0, - "num_attention_heads": 16, - "num_beam_groups": 1, - "num_beams": 1, - "num_hidden_layers": 24, - "num_return_sequences": 1, - "output_attentions": false, - "output_hidden_states": false, - "output_scores": false, - "pad_token_id": 0, - "pooler_fc_size": 768, - "pooler_num_attention_heads": 12, - "pooler_num_fc_layers": 3, - "pooler_size_per_head": 128, - "pooler_type": "first_token_transform", - "position_embedding_type": "absolute", - "prefix": null, - "problem_type": null, - "pruned_heads": {}, - "remove_invalid_values": false, - "repetition_penalty": 1, - "return_dict": true, - "return_dict_in_generate": false, - "sep_token_id": null, - "task_specific_params": null, - "temperature": 1, - "tie_encoder_decoder": false, - "tie_word_embeddings": true, - "tokenizer_class": null, - "top_k": 50, - "top_p": 1, - "torchscript": false, - "transformers_version": "4.6.1", - "type_vocab_size": 2, - "use_bfloat16": false, - "use_cache": true, - "vocab_size": 28996 - } - }` -} diff --git a/modules/reranker-transformers/clients/ranker_test.go b/modules/reranker-transformers/clients/ranker_test.go deleted file mode 100644 index d84cfdbc7ca27e312e2f582c411b8f9d951d03d7..0000000000000000000000000000000000000000 --- a/modules/reranker-transformers/clients/ranker_test.go +++ /dev/null @@ -1,203 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package client - -import ( - "context" - "encoding/json" - "io" - "net/http" - "net/http/httptest" - "sync" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/usecases/modulecomponents/ent" -) - -func TestGetScore(t *testing.T) { - t.Run("when the server has a successful answer", func(t *testing.T) { - server := httptest.NewServer(&testCrossRankerHandler{ - t: t, - res: RankResponse{ - Query: "Where do I work?", - Scores: []DocumentScore{ - { - Document: "I work at Apple", - Score: 0.15, - }, - }, - }, - }) - defer server.Close() - c := New(server.URL, 0, nullLogger()) - res, err := c.Rank(context.Background(), "Where do I work?", []string{"I work at Apple"}, nil) - - assert.Nil(t, err) - assert.Equal(t, ent.RankResult{ - Query: "Where do I work?", - DocumentScores: []ent.DocumentScore{ - { - Document: "I work at Apple", - Score: 0.15, - }, - }, - }, *res) - }) - - t.Run("when the server has an error", func(t *testing.T) { - server := httptest.NewServer(&testCrossRankerHandler{ - t: t, - res: RankResponse{ - Error: "some error from the server", - }, - }) - defer server.Close() - c := New(server.URL, 0, nullLogger()) - _, err := c.Rank(context.Background(), "prop", - []string{"I work at Apple"}, nil) - - require.NotNil(t, err) - assert.Contains(t, err.Error(), "some error from the server") - }) - - t.Run("when we send requests in batches", func(t *testing.T) { - server := httptest.NewServer(&testCrossRankerHandler{ - t: t, - res: RankResponse{ - Query: "Where do I work?", - Scores: []DocumentScore{ - { - Document: "I work at Apple", - Score: 0.15, - }, - }, - }, - batchedResults: [][]DocumentScore{ - { - { - Document: "Response 1", - Score: 0.99, - }, - { - Document: "Response 2", - Score: 0.89, - }, - }, - { - { - Document: "Response 3", - Score: 0.19, - }, - { - Document: "Response 4", - Score: 0.29, - }, - }, - { - { - Document: "Response 5", - Score: 0.79, - }, - { - Document: "Response 6", - Score: 0.789, - }, - }, - { - { - Document: "Response 7", - Score: 0.0001, - }, - }, - }, - }) - defer server.Close() - - c := New(server.URL, 0, nullLogger()) - c.maxDocuments = 2 - - query := "Where do I work?" - documents := []string{ - "Response 1", "Response 2", "Response 3", "Response 4", - "Response 5", "Response 6", "Response 7", - } - - resp, err := c.Rank(context.Background(), query, documents, nil) - - require.Nil(t, err) - require.NotNil(t, resp) - require.NotNil(t, resp.DocumentScores) - for i := range resp.DocumentScores { - assert.Equal(t, documents[i], resp.DocumentScores[i].Document) - if i == 0 { - assert.Equal(t, 0.99, resp.DocumentScores[i].Score) - } - if i == len(documents)-1 { - assert.Equal(t, 0.0001, resp.DocumentScores[i].Score) - } - } - }) -} - -type testCrossRankerHandler struct { - lock sync.RWMutex - t *testing.T - res RankResponse - batchedResults [][]DocumentScore -} - -func (f *testCrossRankerHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - f.lock.Lock() - defer f.lock.Unlock() - - assert.Equal(f.t, "/rerank", r.URL.String()) - assert.Equal(f.t, http.MethodPost, r.Method) - - if f.res.Error != "" { - w.WriteHeader(500) - } - - bodyBytes, err := io.ReadAll(r.Body) - require.Nil(f.t, err) - defer r.Body.Close() - - var req RankInput - require.Nil(f.t, json.Unmarshal(bodyBytes, &req)) - - containsDocument := func(req RankInput, in string) bool { - for _, doc := range req.Documents { - if doc == in { - return true - } - } - return false - } - - index := 0 - if len(f.batchedResults) > 0 { - if containsDocument(req, "Response 3") { - index = 1 - } - if containsDocument(req, "Response 5") { - index = 2 - } - if containsDocument(req, "Response 7") { - index = 3 - } - f.res.Scores = f.batchedResults[index] - } - - jsonBytes, _ := json.Marshal(f.res) - w.Write(jsonBytes) -} diff --git a/modules/reranker-transformers/clients/startup.go b/modules/reranker-transformers/clients/startup.go deleted file mode 100644 index 64794ec19ed9455c9fcfdf6ee0b30c7251a49f90..0000000000000000000000000000000000000000 --- a/modules/reranker-transformers/clients/startup.go +++ /dev/null @@ -1,68 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package client - -import ( - "context" - "net/http" - "time" - - "github.com/pkg/errors" -) - -func (c *client) WaitForStartup(initCtx context.Context, - interval time.Duration, -) error { - t := time.NewTicker(interval) - defer t.Stop() - expired := initCtx.Done() - var lastErr error - for { - select { - case <-t.C: - lastErr = c.checkReady(initCtx) - if lastErr == nil { - return nil - } - c.logger. - WithField("action", "reranktransformers_remote_wait_for_startup"). - WithError(lastErr).Warnf("reranktransformers remote service not ready") - case <-expired: - return errors.Wrapf(lastErr, "init context expired before remote was ready") - } - } -} - -func (c *client) checkReady(initCtx context.Context) error { - // spawn a new context (derived on the overall context) which is used to - // consider an individual request timed out - requestCtx, cancel := context.WithTimeout(initCtx, 500*time.Millisecond) - defer cancel() - - req, err := http.NewRequestWithContext(requestCtx, http.MethodGet, - c.url("/.well-known/ready"), nil) - if err != nil { - return errors.Wrap(err, "create check ready request") - } - - res, err := c.httpClient.Do(req) - if err != nil { - return errors.Wrap(err, "send check ready request") - } - - defer res.Body.Close() - if res.StatusCode > 299 { - return errors.Errorf("not ready: status %d", res.StatusCode) - } - - return nil -} diff --git a/modules/reranker-transformers/clients/startup_test.go b/modules/reranker-transformers/clients/startup_test.go deleted file mode 100644 index e9b69da8300a11cf7c5664598e613a8e9f661902..0000000000000000000000000000000000000000 --- a/modules/reranker-transformers/clients/startup_test.go +++ /dev/null @@ -1,98 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package client - -import ( - "context" - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/sirupsen/logrus" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestWaitForStartup(t *testing.T) { - t.Run("when the server is immediately ready", func(t *testing.T) { - server := httptest.NewServer(&testReadyHandler{t: t}) - defer server.Close() - c := New(server.URL, 0, nullLogger()) - err := c.WaitForStartup(context.Background(), 50*time.Millisecond) - - assert.Nil(t, err) - }) - - t.Run("when the server is down", func(t *testing.T) { - c := New("http://nothing-running-at-this-url", 0, nullLogger()) - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancel() - err := c.WaitForStartup(ctx, 150*time.Millisecond) - - require.NotNil(t, err, nullLogger()) - assert.Contains(t, err.Error(), "expired before remote was ready") - }) - - t.Run("when the server is alive, but not ready", func(t *testing.T) { - server := httptest.NewServer(&testReadyHandler{ - t: t, - readyTime: time.Now().Add(1 * time.Minute), - }) - c := New(server.URL, 0, nullLogger()) - defer server.Close() - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancel() - err := c.WaitForStartup(ctx, 50*time.Millisecond) - - require.NotNil(t, err) - assert.Contains(t, err.Error(), "expired before remote was ready") - }) - - t.Run("when the server is initially not ready, but then becomes ready", - func(t *testing.T) { - server := httptest.NewServer(&testReadyHandler{ - t: t, - readyTime: time.Now().Add(100 * time.Millisecond), - }) - c := New(server.URL, 0, nullLogger()) - defer server.Close() - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancel() - err := c.WaitForStartup(ctx, 50*time.Millisecond) - - require.Nil(t, err) - }) -} - -type testReadyHandler struct { - t *testing.T - // the test handler will report as not ready before the time has passed - readyTime time.Time -} - -func (f *testReadyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - assert.Equal(f.t, "/.well-known/ready", r.URL.String()) - assert.Equal(f.t, http.MethodGet, r.Method) - - if time.Since(f.readyTime) < 0 { - w.WriteHeader(http.StatusServiceUnavailable) - } - - w.WriteHeader(http.StatusNoContent) -} - -func nullLogger() logrus.FieldLogger { - l, _ := test.NewNullLogger() - return l -} diff --git a/modules/reranker-transformers/config.go b/modules/reranker-transformers/config.go deleted file mode 100644 index 842312fd1764a08eaea9562c51dbfcb44420f35d..0000000000000000000000000000000000000000 --- a/modules/reranker-transformers/config.go +++ /dev/null @@ -1,39 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modrerankertransformers - -import ( - "context" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/schema" -) - -func (m *ReRankerModule) ClassConfigDefaults() map[string]interface{} { - return map[string]interface{}{} -} - -func (m *ReRankerModule) PropertyConfigDefaults( - dt *schema.DataType, -) map[string]interface{} { - return map[string]interface{}{} -} - -func (m *ReRankerModule) ValidateClass(ctx context.Context, - class *models.Class, cfg moduletools.ClassConfig, -) error { - return nil -} - -var _ = modulecapabilities.ClassConfigurator(New()) diff --git a/modules/reranker-transformers/module.go b/modules/reranker-transformers/module.go deleted file mode 100644 index 6b6147f276b870407bdf9247a0e6e84711b2986b..0000000000000000000000000000000000000000 --- a/modules/reranker-transformers/module.go +++ /dev/null @@ -1,100 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modrerankertransformers - -import ( - "context" - "net/http" - "os" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - client "github.com/weaviate/weaviate/modules/reranker-transformers/clients" - additionalprovider "github.com/weaviate/weaviate/usecases/modulecomponents/additional" - "github.com/weaviate/weaviate/usecases/modulecomponents/ent" -) - -const Name = "reranker-transformers" - -func New() *ReRankerModule { - return &ReRankerModule{} -} - -type ReRankerModule struct { - reranker ReRankerClient - additionalPropertiesProvider modulecapabilities.AdditionalProperties -} - -type ReRankerClient interface { - Rank(ctx context.Context, query string, documents []string, cfg moduletools.ClassConfig) (*ent.RankResult, error) - MetaInfo() (map[string]interface{}, error) -} - -func (m *ReRankerModule) Name() string { - return Name -} - -func (m *ReRankerModule) Type() modulecapabilities.ModuleType { - return modulecapabilities.Text2TextReranker -} - -func (m *ReRankerModule) Init(ctx context.Context, - params moduletools.ModuleInitParams, -) error { - if err := m.initAdditional(ctx, params.GetConfig().ModuleHttpClientTimeout, params.GetLogger()); err != nil { - return errors.Wrap(err, "init re encoder") - } - - return nil -} - -func (m *ReRankerModule) initAdditional(ctx context.Context, timeout time.Duration, - logger logrus.FieldLogger, -) error { - uri := os.Getenv("RERANKER_INFERENCE_API") - if uri == "" { - return errors.Errorf("required variable RERANKER_INFERENCE_API is not set") - } - - client := client.New(uri, timeout, logger) - - m.reranker = client - if err := client.WaitForStartup(ctx, 1*time.Second); err != nil { - return errors.Wrap(err, "init remote sum module") - } - - m.additionalPropertiesProvider = additionalprovider.NewRankerProvider(client) - return nil -} - -func (m *ReRankerModule) MetaInfo() (map[string]interface{}, error) { - return m.reranker.MetaInfo() -} - -func (m *ReRankerModule) RootHandler() http.Handler { - // TODO: remove once this is a capability interface - return nil -} - -func (m *ReRankerModule) AdditionalProperties() map[string]modulecapabilities.AdditionalProperty { - return m.additionalPropertiesProvider.AdditionalProperties() -} - -// verify we implement the modules.Module interface -var ( - _ = modulecapabilities.Module(New()) - _ = modulecapabilities.AdditionalProperties(New()) - _ = modulecapabilities.MetaProvider(New()) -) diff --git a/modules/sum-transformers/additional/models/models.go b/modules/sum-transformers/additional/models/models.go deleted file mode 100644 index 7e656fada3bc5c935d48a202312dce0410b6eaea..0000000000000000000000000000000000000000 --- a/modules/sum-transformers/additional/models/models.go +++ /dev/null @@ -1,12 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package models diff --git a/modules/sum-transformers/additional/provider.go b/modules/sum-transformers/additional/provider.go deleted file mode 100644 index 7fe85c6ca62cfa486cc2c5085e21f46770f692ad..0000000000000000000000000000000000000000 --- a/modules/sum-transformers/additional/provider.go +++ /dev/null @@ -1,57 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package additional - -import ( - "context" - - "github.com/tailor-inc/graphql" - "github.com/tailor-inc/graphql/language/ast" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/search" -) - -type AdditionalProperty interface { - AdditionalPropertyFn(ctx context.Context, - in []search.Result, params interface{}, limit *int, - argumentModuleParams map[string]interface{}, cfg moduletools.ClassConfig) ([]search.Result, error) - ExtractAdditionalFn(param []*ast.Argument) interface{} - AdditionalPropertyDefaultValue() interface{} - AdditionalFieldFn(classname string) *graphql.Field -} - -type GraphQLAdditionalArgumentsProvider struct { - summaryProvider AdditionalProperty -} - -func New(summaryProvider AdditionalProperty) *GraphQLAdditionalArgumentsProvider { - return &GraphQLAdditionalArgumentsProvider{summaryProvider} -} - -func (p *GraphQLAdditionalArgumentsProvider) AdditionalProperties() map[string]modulecapabilities.AdditionalProperty { - additionalProperties := map[string]modulecapabilities.AdditionalProperty{} - additionalProperties["summary"] = p.getSummary() - return additionalProperties -} - -func (p *GraphQLAdditionalArgumentsProvider) getSummary() modulecapabilities.AdditionalProperty { - return modulecapabilities.AdditionalProperty{ - GraphQLNames: []string{"summary"}, - GraphQLFieldFunction: p.summaryProvider.AdditionalFieldFn, - GraphQLExtractFunction: p.summaryProvider.ExtractAdditionalFn, - SearchFunctions: modulecapabilities.AdditionalSearch{ - ExploreGet: p.summaryProvider.AdditionalPropertyFn, - ExploreList: p.summaryProvider.AdditionalPropertyFn, - }, - } -} diff --git a/modules/sum-transformers/additional/summary/summary.go b/modules/sum-transformers/additional/summary/summary.go deleted file mode 100644 index f481d7ef314d706b24e92348e7e8ecbd46d020b1..0000000000000000000000000000000000000000 --- a/modules/sum-transformers/additional/summary/summary.go +++ /dev/null @@ -1,57 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package summary - -import ( - "context" - "errors" - - "github.com/tailor-inc/graphql" - "github.com/tailor-inc/graphql/language/ast" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/search" - "github.com/weaviate/weaviate/modules/sum-transformers/ent" -) - -type sumClient interface { - GetSummary(ctx context.Context, property, text string) ([]ent.SummaryResult, error) -} - -type SummaryProvider struct { - sum sumClient -} - -func New(sum sumClient) *SummaryProvider { - return &SummaryProvider{sum} -} - -func (p *SummaryProvider) AdditionalPropertyDefaultValue() interface{} { - return &Params{} -} - -func (p *SummaryProvider) ExtractAdditionalFn(param []*ast.Argument) interface{} { - return p.parseSummaryArguments(param) -} - -func (p *SummaryProvider) AdditionalFieldFn(classname string) *graphql.Field { - return p.additionalSummaryField(classname) -} - -func (p *SummaryProvider) AdditionalPropertyFn(ctx context.Context, - in []search.Result, params interface{}, limit *int, - argumentModuleParams map[string]interface{}, cfg moduletools.ClassConfig, -) ([]search.Result, error) { - if parameters, ok := params.(*Params); ok { - return p.findSummary(ctx, in, parameters) - } - return nil, errors.New("wrong parameters") -} diff --git a/modules/sum-transformers/additional/summary/summary_graphql_field.go b/modules/sum-transformers/additional/summary/summary_graphql_field.go deleted file mode 100644 index 3efd1e7e81bbc7266601cbaa61d0c39596c90013..0000000000000000000000000000000000000000 --- a/modules/sum-transformers/additional/summary/summary_graphql_field.go +++ /dev/null @@ -1,37 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package summary - -import ( - "fmt" - - "github.com/tailor-inc/graphql" -) - -func (p *SummaryProvider) additionalSummaryField(classname string) *graphql.Field { - return &graphql.Field{ - Args: graphql.FieldConfigArgument{ - "properties": &graphql.ArgumentConfig{ - Description: "Properties which contains text", - Type: graphql.NewList(graphql.String), - DefaultValue: nil, - }, - }, - Type: graphql.NewList(graphql.NewObject(graphql.ObjectConfig{ - Name: fmt.Sprintf("%sAdditionalSummary", classname), - Fields: graphql.Fields{ - "property": &graphql.Field{Type: graphql.String}, - "result": &graphql.Field{Type: graphql.String}, - }, - })), - } -} diff --git a/modules/sum-transformers/additional/summary/summary_graphql_field_test.go b/modules/sum-transformers/additional/summary/summary_graphql_field_test.go deleted file mode 100644 index 67ccbf64ba1e8cf2e1daba406b76f0af5b625ba7..0000000000000000000000000000000000000000 --- a/modules/sum-transformers/additional/summary/summary_graphql_field_test.go +++ /dev/null @@ -1,43 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package summary - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/tailor-inc/graphql" -) - -func Test_additionalSummaryField(t *testing.T) { - // given - summaryProvider := &SummaryProvider{} - classname := "Class" - - // when - summary := summaryProvider.additionalSummaryField(classname) - - assert.NotNil(t, summary) - assert.Equal(t, "ClassAdditionalSummary", summary.Type.Name()) - assert.NotNil(t, summary.Type) - summaryObjectList, summaryObjectListOK := summary.Type.(*graphql.List) - assert.True(t, summaryObjectListOK) - summaryObject, summaryObjectOK := summaryObjectList.OfType.(*graphql.Object) - assert.True(t, summaryObjectOK) - assert.Equal(t, 2, len(summaryObject.Fields())) - assert.NotNil(t, summaryObject.Fields()["property"]) - assert.NotNil(t, summaryObject.Fields()["result"]) - - assert.NotNil(t, summary.Args) - assert.Equal(t, 1, len(summary.Args)) - assert.NotNil(t, summary.Args["properties"]) -} diff --git a/modules/sum-transformers/additional/summary/summary_params.go b/modules/sum-transformers/additional/summary/summary_params.go deleted file mode 100644 index e42bd5b2897af9285b89a8ad771bd0c540eb29b5..0000000000000000000000000000000000000000 --- a/modules/sum-transformers/additional/summary/summary_params.go +++ /dev/null @@ -1,20 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package summary - -type Params struct { - Properties []string -} - -func (n Params) GetProperties() []string { - return n.Properties -} diff --git a/modules/sum-transformers/additional/summary/summary_params_extractor.go b/modules/sum-transformers/additional/summary/summary_params_extractor.go deleted file mode 100644 index a75f5901b4aea415061ce2f3331810a06e8e4900..0000000000000000000000000000000000000000 --- a/modules/sum-transformers/additional/summary/summary_params_extractor.go +++ /dev/null @@ -1,40 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package summary - -import ( - "log" - - "github.com/tailor-inc/graphql/language/ast" -) - -func (p *SummaryProvider) parseSummaryArguments(args []*ast.Argument) *Params { - out := &Params{} - - for _, arg := range args { - switch arg.Name.Value { - case "properties": - inp := arg.Value.GetValue().([]ast.Value) - out.Properties = make([]string, len(inp)) - - for i, value := range inp { - out.Properties[i] = value.(*ast.StringValue).Value - } - - default: - // ignore what we don't recognize - log.Printf("Igonore not recognized value: %v", arg.Name.Value) - } - } - - return out -} diff --git a/modules/sum-transformers/additional/summary/summary_params_extractor_test.go b/modules/sum-transformers/additional/summary/summary_params_extractor_test.go deleted file mode 100644 index db80fad5fe5908e1eec34001dba3d3dd30149886..0000000000000000000000000000000000000000 --- a/modules/sum-transformers/additional/summary/summary_params_extractor_test.go +++ /dev/null @@ -1,83 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package summary - -import ( - "reflect" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/tailor-inc/graphql/language/ast" -) - -func Test_parseSummaryArguments(t *testing.T) { - type args struct { - args []*ast.Argument - } - tests := []struct { - name string - args args - want *Params - }{ - { - name: "Should create with no params", - args: args{}, - want: &Params{}, - }, - { - name: "Should create with all params", - args: args{ - args: []*ast.Argument{ - createListArg("properties", []string{"prop1", "prop2"}), - }, - }, - want: &Params{ - Properties: []string{"prop1", "prop2"}, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - p := &SummaryProvider{} - if got := p.parseSummaryArguments(tt.args.args); !reflect.DeepEqual(got, tt.want) { - t.Errorf("parseSummaryArguments() = %v, want %v", got, tt.want) - } - actual := p.parseSummaryArguments(tt.args.args) - assert.Equal(t, tt.want, actual) - }) - } -} - -func createListArg(name string, valuesIn []string) *ast.Argument { - n := ast.Name{ - Value: name, - } - - valuesAst := make([]ast.Value, len(valuesIn)) - for i, value := range valuesIn { - valuesAst[i] = &ast.StringValue{ - Kind: "Kind", - Value: value, - } - } - vals := ast.ListValue{ - Kind: "Kind", - Values: valuesAst, - } - arg := ast.Argument{ - Name: ast.NewName(&n), - Kind: "Kind", - Value: &vals, - } - a := ast.NewArgument(&arg) - return a -} diff --git a/modules/sum-transformers/additional/summary/summary_result.go b/modules/sum-transformers/additional/summary/summary_result.go deleted file mode 100644 index 64f6d4b01583522bfea30e02026488558caa9e34..0000000000000000000000000000000000000000 --- a/modules/sum-transformers/additional/summary/summary_result.go +++ /dev/null @@ -1,88 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package summary - -import ( - "context" - "errors" - "fmt" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/search" - "github.com/weaviate/weaviate/modules/sum-transformers/ent" -) - -func (p *SummaryProvider) findSummary(ctx context.Context, - in []search.Result, params *Params, -) ([]search.Result, error) { - if len(in) == 0 { - return nil, nil - } else { - if params == nil { - return nil, fmt.Errorf("no params provided") - } - - properties := params.GetProperties() - - // check if user parameter values are valid - if len(properties) == 0 { - return in, errors.New("no properties provided") - } - - for i := range in { // for each result of the general GraphQL Query - ap := in[i].AdditionalProperties - if ap == nil { - ap = models.AdditionalProperties{} - } - - // check if the schema of the GraphQL data object contains the properties and they are text or string values - textProperties := map[string]string{} - schema := in[i].Object().Properties.(map[string]interface{}) - for property, value := range schema { - if p.containsProperty(property, properties) { - if valueString, ok := value.(string); ok && len(valueString) > 0 { - textProperties[property] = valueString - } - } - } - - summaryList := []ent.SummaryResult{} - - // for each text property result, call the SUM function and add to additional result - for property, value := range textProperties { - summary, err := p.sum.GetSummary(ctx, property, value) - if err != nil { - return in, err - } - - summaryList = append(summaryList, summary...) - } - - ap["summary"] = summaryList - - in[i].AdditionalProperties = ap - } - } - return in, nil -} - -func (p *SummaryProvider) containsProperty(property string, properties []string) bool { - if len(properties) == 0 { - return true - } - for i := range properties { - if properties[i] == property { - return true - } - } - return false -} diff --git a/modules/sum-transformers/additional/summary/summary_test.go b/modules/sum-transformers/additional/summary/summary_test.go deleted file mode 100644 index 107923005c5fe18b9ce60590f0f4bbd9aa846732..0000000000000000000000000000000000000000 --- a/modules/sum-transformers/additional/summary/summary_test.go +++ /dev/null @@ -1,115 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package summary - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/search" - "github.com/weaviate/weaviate/modules/sum-transformers/ent" -) - -func TestAdditionalAnswerProvider(t *testing.T) { - t.Run("should fail with empty content", func(t *testing.T) { - // given - sumClient := &fakeSUMClient{} - summaryProvider := New(sumClient) - in := []search.Result{ - { - ID: "some-uuid", - }, - } - fakeParams := &Params{} - limit := 1 - argumentModuleParams := map[string]interface{}{} - - // when - out, err := summaryProvider.AdditionalPropertyFn(context.Background(), in, fakeParams, &limit, argumentModuleParams, nil) - - // then - require.NotNil(t, err) - require.NotEmpty(t, out) - assert.Error(t, err, "empty schema content") - }) - - t.Run("should fail with empty params", func(t *testing.T) { - // given - sumClient := &fakeSUMClient{} - summaryProvider := New(sumClient) - in := []search.Result{ - { - ID: "some-uuid", - Schema: map[string]interface{}{ - "content": "content", - }, - }, - } - fakeParams := &Params{} - limit := 1 - argumentModuleParams := map[string]interface{}{} - - // when - out, err := summaryProvider.AdditionalPropertyFn(context.Background(), in, fakeParams, &limit, argumentModuleParams, nil) - - // then - require.NotNil(t, err) - require.NotEmpty(t, out) - assert.Error(t, err, "empty params") - }) - - t.Run("should summarize", func(t *testing.T) { - sumClient := &fakeSUMClient{} - summaryProvider := New(sumClient) - in := []search.Result{ - { - ID: "some-uuid", - Schema: map[string]interface{}{ - "content": "this is the content", - }, - }, - } - fakeParams := &Params{Properties: []string{"content"}} - limit := 1 - argumentModuleParams := map[string]interface{}{} - - // when - out, err := summaryProvider.AdditionalPropertyFn(context.Background(), in, fakeParams, &limit, argumentModuleParams, nil) - // then - require.Nil(t, err) - require.NotEmpty(t, out) - assert.Equal(t, 1, len(in)) - answer, answerOK := in[0].AdditionalProperties["summary"] - assert.True(t, answerOK) - assert.NotNil(t, answer) - answerAdditional, answerAdditionalOK := answer.([]ent.SummaryResult) - assert.True(t, answerAdditionalOK) - assert.Equal(t, "this is the summary", answerAdditional[0].Result) - assert.Equal(t, "content", answerAdditional[0].Property) - }) -} - -type fakeSUMClient struct{} - -func (c *fakeSUMClient) GetSummary(ctx context.Context, property, text string, -) ([]ent.SummaryResult, error) { - return c.getSummary(property), nil -} - -func (c *fakeSUMClient) getSummary(property string) []ent.SummaryResult { - return []ent.SummaryResult{{ - Property: property, - Result: "this is the summary", - }} -} diff --git a/modules/sum-transformers/client/client.go b/modules/sum-transformers/client/client.go deleted file mode 100644 index 516325e1d48940908bf131f4b3fe5e319b2a5828..0000000000000000000000000000000000000000 --- a/modules/sum-transformers/client/client.go +++ /dev/null @@ -1,107 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package client - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/modules/sum-transformers/ent" -) - -type client struct { - origin string - httpClient *http.Client - logger logrus.FieldLogger -} - -type sumInput struct { - Text string `json:"text"` -} - -type summaryResponse struct { - // Property string `json:"property"` - Result string `json:"result"` -} - -type sumResponse struct { - Error string - sumInput - Summary []summaryResponse `json:"summary"` -} - -func New(origin string, timeout time.Duration, logger logrus.FieldLogger) *client { - return &client{ - origin: origin, - httpClient: &http.Client{ - Timeout: timeout, - }, - logger: logger, - } -} - -func (c *client) GetSummary(ctx context.Context, property, text string, -) ([]ent.SummaryResult, error) { - body, err := json.Marshal(sumInput{ - Text: text, - }) - if err != nil { - return nil, errors.Wrapf(err, "marshal body") - } - - req, err := http.NewRequestWithContext(ctx, "POST", c.url("/sum/"), - bytes.NewReader(body)) - if err != nil { - return nil, errors.Wrap(err, "create POST request") - } - - res, err := c.httpClient.Do(req) - if err != nil { - return nil, errors.Wrap(err, "send POST request") - } - defer res.Body.Close() - - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - return nil, errors.Wrap(err, "read response body") - } - - var resBody sumResponse - if err := json.Unmarshal(bodyBytes, &resBody); err != nil { - return nil, errors.Wrap(err, "unmarshal response body") - } - - if res.StatusCode > 399 { - return nil, errors.Errorf("fail with status %d: %s", res.StatusCode, resBody.Error) - } - - out := make([]ent.SummaryResult, len(resBody.Summary)) - - for i, elem := range resBody.Summary { - out[i].Result = elem.Result - out[i].Property = property - } - - // format resBody to nerResult - return out, nil -} - -func (c *client) url(path string) string { - return fmt.Sprintf("%s%s", c.origin, path) -} diff --git a/modules/sum-transformers/client/startup.go b/modules/sum-transformers/client/startup.go deleted file mode 100644 index 50394577789e162d41cae9f9f8ec9834f5bc9413..0000000000000000000000000000000000000000 --- a/modules/sum-transformers/client/startup.go +++ /dev/null @@ -1,68 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package client - -import ( - "context" - "net/http" - "time" - - "github.com/pkg/errors" -) - -func (c *client) WaitForStartup(initCtx context.Context, - interval time.Duration, -) error { - t := time.NewTicker(interval) - defer t.Stop() - expired := initCtx.Done() - var lastErr error - for { - select { - case <-t.C: - lastErr = c.checkReady(initCtx) - if lastErr == nil { - return nil - } - c.logger. - WithField("action", "sum_remote_wait_for_startup"). - WithError(lastErr).Warnf("sum remote service not ready") - case <-expired: - return errors.Wrapf(lastErr, "init context expired before remote was ready") - } - } -} - -func (c *client) checkReady(initCtx context.Context) error { - // spawn a new context (derived on the overall context) which is used to - // consider an individual request timed out - requestCtx, cancel := context.WithTimeout(initCtx, 500*time.Millisecond) - defer cancel() - - req, err := http.NewRequestWithContext(requestCtx, http.MethodGet, - c.url("/.well-known/ready"), nil) - if err != nil { - return errors.Wrap(err, "create check ready request") - } - - res, err := c.httpClient.Do(req) - if err != nil { - return errors.Wrap(err, "send check ready request") - } - - defer res.Body.Close() - if res.StatusCode > 299 { - return errors.Errorf("not ready: status %d", res.StatusCode) - } - - return nil -} diff --git a/modules/sum-transformers/client/startup_test.go b/modules/sum-transformers/client/startup_test.go deleted file mode 100644 index e9b69da8300a11cf7c5664598e613a8e9f661902..0000000000000000000000000000000000000000 --- a/modules/sum-transformers/client/startup_test.go +++ /dev/null @@ -1,98 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package client - -import ( - "context" - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/sirupsen/logrus" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestWaitForStartup(t *testing.T) { - t.Run("when the server is immediately ready", func(t *testing.T) { - server := httptest.NewServer(&testReadyHandler{t: t}) - defer server.Close() - c := New(server.URL, 0, nullLogger()) - err := c.WaitForStartup(context.Background(), 50*time.Millisecond) - - assert.Nil(t, err) - }) - - t.Run("when the server is down", func(t *testing.T) { - c := New("http://nothing-running-at-this-url", 0, nullLogger()) - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancel() - err := c.WaitForStartup(ctx, 150*time.Millisecond) - - require.NotNil(t, err, nullLogger()) - assert.Contains(t, err.Error(), "expired before remote was ready") - }) - - t.Run("when the server is alive, but not ready", func(t *testing.T) { - server := httptest.NewServer(&testReadyHandler{ - t: t, - readyTime: time.Now().Add(1 * time.Minute), - }) - c := New(server.URL, 0, nullLogger()) - defer server.Close() - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancel() - err := c.WaitForStartup(ctx, 50*time.Millisecond) - - require.NotNil(t, err) - assert.Contains(t, err.Error(), "expired before remote was ready") - }) - - t.Run("when the server is initially not ready, but then becomes ready", - func(t *testing.T) { - server := httptest.NewServer(&testReadyHandler{ - t: t, - readyTime: time.Now().Add(100 * time.Millisecond), - }) - c := New(server.URL, 0, nullLogger()) - defer server.Close() - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancel() - err := c.WaitForStartup(ctx, 50*time.Millisecond) - - require.Nil(t, err) - }) -} - -type testReadyHandler struct { - t *testing.T - // the test handler will report as not ready before the time has passed - readyTime time.Time -} - -func (f *testReadyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - assert.Equal(f.t, "/.well-known/ready", r.URL.String()) - assert.Equal(f.t, http.MethodGet, r.Method) - - if time.Since(f.readyTime) < 0 { - w.WriteHeader(http.StatusServiceUnavailable) - } - - w.WriteHeader(http.StatusNoContent) -} - -func nullLogger() logrus.FieldLogger { - l, _ := test.NewNullLogger() - return l -} diff --git a/modules/sum-transformers/client/sum_meta.go b/modules/sum-transformers/client/sum_meta.go deleted file mode 100644 index b634f525d7a16341562ef8d64275bbdcdc1e2c1e..0000000000000000000000000000000000000000 --- a/modules/sum-transformers/client/sum_meta.go +++ /dev/null @@ -1,45 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package client - -import ( - "context" - "encoding/json" - "io" - "net/http" - - "github.com/pkg/errors" -) - -func (c *client) MetaInfo() (map[string]interface{}, error) { - req, err := http.NewRequestWithContext(context.Background(), "GET", c.url("/meta"), nil) - if err != nil { - return nil, errors.Wrap(err, "create GET meta request") - } - - res, err := c.httpClient.Do(req) - if err != nil { - return nil, errors.Wrap(err, "send GET meta request") - } - defer res.Body.Close() - - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - return nil, errors.Wrap(err, "read meta response body") - } - - var resBody map[string]interface{} - if err := json.Unmarshal(bodyBytes, &resBody); err != nil { - return nil, errors.Wrap(err, "unmarshal meta response body") - } - return resBody, nil -} diff --git a/modules/sum-transformers/client/sum_meta_test.go b/modules/sum-transformers/client/sum_meta_test.go deleted file mode 100644 index 2acf44f9caffab0db7ba1b7ca6783c17306264c6..0000000000000000000000000000000000000000 --- a/modules/sum-transformers/client/sum_meta_test.go +++ /dev/null @@ -1,156 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package client - -import ( - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func TestGetMeta(t *testing.T) { - t.Run("when the server is providing meta", func(t *testing.T) { - server := httptest.NewServer(&testMetaHandler{t: t}) - defer server.Close() - c := New(server.URL, 0, nullLogger()) - meta, err := c.MetaInfo() - - assert.Nil(t, err) - assert.NotNil(t, meta) - metaModel := meta["model"] - assert.True(t, metaModel != nil) - model, modelOK := metaModel.(map[string]interface{}) - assert.True(t, modelOK) - assert.True(t, model["_name_or_path"] != nil) - assert.True(t, model["architectures"] != nil) - }) -} - -type testMetaHandler struct { - t *testing.T - // the test handler will report as not ready before the time has passed - readyTime time.Time -} - -func (f *testMetaHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - assert.Equal(f.t, "/meta", r.URL.String()) - assert.Equal(f.t, http.MethodGet, r.Method) - - if time.Since(f.readyTime) < 0 { - w.WriteHeader(http.StatusServiceUnavailable) - } - - w.Write([]byte(f.metaInfo())) -} - -func (f *testMetaHandler) metaInfo() string { - return `{ - "model": { - "_name_or_path": "dbmdz/bert-large-cased-finetuned-conll03-english", - "_num_labels": 9, - "add_cross_attention": false, - "architectures": [ - "BertForTokenClassification" - ], - "attention_probs_dropout_prob": 0.1, - "bad_words_ids": null, - "bos_token_id": null, - "chunk_size_feed_forward": 0, - "decoder_start_token_id": null, - "directionality": "bidi", - "diversity_penalty": 0, - "do_sample": false, - "early_stopping": false, - "encoder_no_repeat_ngram_size": 0, - "eos_token_id": null, - "finetuning_task": null, - "forced_bos_token_id": null, - "forced_eos_token_id": null, - "gradient_checkpointing": false, - "hidden_act": "gelu", - "hidden_dropout_prob": 0.1, - "hidden_size": 1024, - "id2label": { - "0": "O", - "1": "B-MISC", - "2": "I-MISC", - "3": "B-PER", - "4": "I-PER", - "5": "B-ORG", - "6": "I-ORG", - "7": "B-LOC", - "8": "I-LOC" - }, - "initializer_range": 0.02, - "intermediate_size": 4096, - "is_decoder": false, - "is_encoder_decoder": false, - "label2id": { - "B-LOC": 7, - "B-MISC": 1, - "B-ORG": 5, - "B-PER": 3, - "I-LOC": 8, - "I-MISC": 2, - "I-ORG": 6, - "I-PER": 4, - "O": 0 - }, - "layer_norm_eps": 1e-12, - "length_penalty": 1, - "max_length": 20, - "max_position_embeddings": 512, - "min_length": 0, - "model_type": "bert", - "no_repeat_ngram_size": 0, - "num_attention_heads": 16, - "num_beam_groups": 1, - "num_beams": 1, - "num_hidden_layers": 24, - "num_return_sequences": 1, - "output_attentions": false, - "output_hidden_states": false, - "output_scores": false, - "pad_token_id": 0, - "pooler_fc_size": 768, - "pooler_num_attention_heads": 12, - "pooler_num_fc_layers": 3, - "pooler_size_per_head": 128, - "pooler_type": "first_token_transform", - "position_embedding_type": "absolute", - "prefix": null, - "problem_type": null, - "pruned_heads": {}, - "remove_invalid_values": false, - "repetition_penalty": 1, - "return_dict": true, - "return_dict_in_generate": false, - "sep_token_id": null, - "task_specific_params": null, - "temperature": 1, - "tie_encoder_decoder": false, - "tie_word_embeddings": true, - "tokenizer_class": null, - "top_k": 50, - "top_p": 1, - "torchscript": false, - "transformers_version": "4.6.1", - "type_vocab_size": 2, - "use_bfloat16": false, - "use_cache": true, - "vocab_size": 28996 - } - }` -} diff --git a/modules/sum-transformers/client/sum_test.go b/modules/sum-transformers/client/sum_test.go deleted file mode 100644 index 715a253c74feb18e03155c0dda8f4d2702c67971..0000000000000000000000000000000000000000 --- a/modules/sum-transformers/client/sum_test.go +++ /dev/null @@ -1,88 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package client - -import ( - "context" - "encoding/json" - "net/http" - "net/http/httptest" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/modules/sum-transformers/ent" -) - -func TestGetAnswer(t *testing.T) { - t.Run("when the server has a successful answer", func(t *testing.T) { - server := httptest.NewServer(&testSUMHandler{ - t: t, - res: sumResponse{ - sumInput: sumInput{ - Text: "I work at Apple", - }, - Summary: []summaryResponse{ - { - Result: "Apple", - }, - }, - }, - }) - defer server.Close() - c := New(server.URL, 0, nullLogger()) - res, err := c.GetSummary(context.Background(), "prop", - "I work at Apple") - - assert.Nil(t, err) - assert.Equal(t, []ent.SummaryResult{ - { - Result: "Apple", - Property: "prop", - }, - }, res) - }) - - t.Run("when the server has a an error", func(t *testing.T) { - server := httptest.NewServer(&testSUMHandler{ - t: t, - res: sumResponse{ - Error: "some error from the server", - }, - }) - defer server.Close() - c := New(server.URL, 0, nullLogger()) - _, err := c.GetSummary(context.Background(), "prop", - "I work at Apple") - - require.NotNil(t, err) - assert.Contains(t, err.Error(), "some error from the server") - }) -} - -type testSUMHandler struct { - t *testing.T - // the test handler will report as not ready before the time has passed - res sumResponse -} - -func (f *testSUMHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - assert.Equal(f.t, "/sum/", r.URL.String()) - assert.Equal(f.t, http.MethodPost, r.Method) - - if f.res.Error != "" { - w.WriteHeader(500) - } - - jsonBytes, _ := json.Marshal(f.res) - w.Write(jsonBytes) -} diff --git a/modules/sum-transformers/config.go b/modules/sum-transformers/config.go deleted file mode 100644 index 6ebf225dbf0949b99f1dc11b9741d4bd9664142e..0000000000000000000000000000000000000000 --- a/modules/sum-transformers/config.go +++ /dev/null @@ -1,38 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modsum - -import ( - "context" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/schema" -) - -func (m *SUMModule) ClassConfigDefaults() map[string]interface{} { - return map[string]interface{}{} -} - -func (m *SUMModule) PropertyConfigDefaults(dt *schema.DataType, -) map[string]interface{} { - return map[string]interface{}{} -} - -func (m *SUMModule) ValidateClass(ctx context.Context, - class *models.Class, cfg moduletools.ClassConfig, -) error { - return nil -} - -var _ = modulecapabilities.ClassConfigurator(New()) diff --git a/modules/sum-transformers/ent/sum_result.go b/modules/sum-transformers/ent/sum_result.go deleted file mode 100644 index 4c13dcfc18b709fc26d2565f965cd475d68f94b8..0000000000000000000000000000000000000000 --- a/modules/sum-transformers/ent/sum_result.go +++ /dev/null @@ -1,21 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package ent - -type SummaryResult struct { - Property string - Result string -} - -type SumResult struct { - Summary []SummaryResult -} diff --git a/modules/sum-transformers/module.go b/modules/sum-transformers/module.go deleted file mode 100644 index 28dddde947d7cd615472e94c6df104527429ff49..0000000000000000000000000000000000000000 --- a/modules/sum-transformers/module.go +++ /dev/null @@ -1,100 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modsum - -import ( - "context" - "net/http" - "os" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - sumadditional "github.com/weaviate/weaviate/modules/sum-transformers/additional" - sumadditionalsummary "github.com/weaviate/weaviate/modules/sum-transformers/additional/summary" - "github.com/weaviate/weaviate/modules/sum-transformers/client" - "github.com/weaviate/weaviate/modules/sum-transformers/ent" -) - -func New() *SUMModule { - return &SUMModule{} -} - -type SUMModule struct { - sum sumClient - additionalPropertiesProvider modulecapabilities.AdditionalProperties -} - -type sumClient interface { - GetSummary(ctx context.Context, property, text string) ([]ent.SummaryResult, error) - MetaInfo() (map[string]interface{}, error) -} - -func (m *SUMModule) Name() string { - return "sum-transformers" -} - -func (m *SUMModule) Type() modulecapabilities.ModuleType { - return modulecapabilities.Text2TextSummarize -} - -func (m *SUMModule) Init(ctx context.Context, - params moduletools.ModuleInitParams, -) error { - if err := m.initAdditional(ctx, params.GetConfig().ModuleHttpClientTimeout, params.GetLogger()); err != nil { - return errors.Wrap(err, "init additional") - } - return nil -} - -func (m *SUMModule) initAdditional(ctx context.Context, timeout time.Duration, - logger logrus.FieldLogger, -) error { - uri := os.Getenv("SUM_INFERENCE_API") - if uri == "" { - return errors.Errorf("required variable SUM_INFERENCE_API is not set") - } - - client := client.New(uri, timeout, logger) - if err := client.WaitForStartup(ctx, 1*time.Second); err != nil { - return errors.Wrap(err, "init remote sum module") - } - - m.sum = client - - tokenProvider := sumadditionalsummary.New(m.sum) - m.additionalPropertiesProvider = sumadditional.New(tokenProvider) - - return nil -} - -func (m *SUMModule) RootHandler() http.Handler { - // TODO: remove once this is a capability interface - return nil -} - -func (m *SUMModule) MetaInfo() (map[string]interface{}, error) { - return m.sum.MetaInfo() -} - -func (m *SUMModule) AdditionalProperties() map[string]modulecapabilities.AdditionalProperty { - return m.additionalPropertiesProvider.AdditionalProperties() -} - -// verify we implement the modules.Module interface -var ( - _ = modulecapabilities.Module(New()) - _ = modulecapabilities.AdditionalProperties(New()) - _ = modulecapabilities.MetaProvider(New()) -) diff --git a/modules/text-spellcheck/additional/models/models.go b/modules/text-spellcheck/additional/models/models.go deleted file mode 100644 index 7565ccbb8c2a81daac7c1e07e6b59b888bdcbd9b..0000000000000000000000000000000000000000 --- a/modules/text-spellcheck/additional/models/models.go +++ /dev/null @@ -1,28 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package models - -// SpellCheckChange describes the misspellings -type SpellCheckChange struct { - Original string `json:"original,omitempty"` - Corrected string `json:"corrected,omitempty"` -} - -// SpellCheck presents proper text without misspellings -// and the list of words that were misspelled -type SpellCheck struct { - OriginalText string `json:"originalText,omitempty"` - DidYouMean string `json:"didYouMean,omitempty"` - Location string `json:"location,omitempty"` - NumberOfCorrections int `json:"numberOfCorrections,omitempty"` - Changes []SpellCheckChange `json:"changes,omitempty"` -} diff --git a/modules/text-spellcheck/additional/provider.go b/modules/text-spellcheck/additional/provider.go deleted file mode 100644 index 43c6e61f31288faf241e533352c9ab4aabdae53a..0000000000000000000000000000000000000000 --- a/modules/text-spellcheck/additional/provider.go +++ /dev/null @@ -1,57 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package additional - -import ( - "context" - - "github.com/tailor-inc/graphql" - "github.com/tailor-inc/graphql/language/ast" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/search" -) - -type AdditionalProperty interface { - AdditionalPropertyFn(ctx context.Context, - in []search.Result, params interface{}, limit *int, - argumentModuleParams map[string]interface{}, cfg moduletools.ClassConfig) ([]search.Result, error) - ExtractAdditionalFn(param []*ast.Argument) interface{} - AdditionalPropertyDefaultValue() interface{} - AdditionalFieldFn(classname string) *graphql.Field -} - -type GraphQLAdditionalArgumentsProvider struct { - spellCheckProvider AdditionalProperty -} - -func New(spellCheckProvider AdditionalProperty) *GraphQLAdditionalArgumentsProvider { - return &GraphQLAdditionalArgumentsProvider{spellCheckProvider} -} - -func (p *GraphQLAdditionalArgumentsProvider) AdditionalProperties() map[string]modulecapabilities.AdditionalProperty { - additionalProperties := map[string]modulecapabilities.AdditionalProperty{} - additionalProperties["spellCheck"] = p.getSpellCheck() - return additionalProperties -} - -func (p *GraphQLAdditionalArgumentsProvider) getSpellCheck() modulecapabilities.AdditionalProperty { - return modulecapabilities.AdditionalProperty{ - GraphQLNames: []string{"spellCheck"}, - GraphQLFieldFunction: p.spellCheckProvider.AdditionalFieldFn, - GraphQLExtractFunction: p.spellCheckProvider.ExtractAdditionalFn, - SearchFunctions: modulecapabilities.AdditionalSearch{ - ExploreGet: p.spellCheckProvider.AdditionalPropertyFn, - ExploreList: p.spellCheckProvider.AdditionalPropertyFn, - }, - } -} diff --git a/modules/text-spellcheck/additional/spellcheck/param_helper.go b/modules/text-spellcheck/additional/spellcheck/param_helper.go deleted file mode 100644 index 6f0444e45bc66d7f36758464de15ce9145d04fe0..0000000000000000000000000000000000000000 --- a/modules/text-spellcheck/additional/spellcheck/param_helper.go +++ /dev/null @@ -1,82 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package spellcheck - -import ( - "encoding/json" -) - -type paramHelper struct{} - -func newParamHelper() *paramHelper { - return ¶mHelper{} -} - -func (p *paramHelper) getTexts(argumentModuleParams map[string]interface{}) (string, []string, error) { - if argumentModuleParams["nearText"] != nil { - texts, err := p.parseNearText(argumentModuleParams["nearText"]) - return "nearText", texts, err - } - if argumentModuleParams["ask"] != nil { - texts, err := p.parseAsk(argumentModuleParams["ask"]) - return "ask", texts, err - } - return "", []string{}, nil -} - -func (p *paramHelper) toJsonParam(arg interface{}) (map[string]interface{}, error) { - data, err := json.Marshal(arg) - if err != nil { - return nil, err - } - var argument map[string]interface{} - err = json.Unmarshal(data, &argument) - if err != nil { - return nil, err - } - return argument, nil -} - -func (p *paramHelper) parseNearText(arg interface{}) ([]string, error) { - argument, err := p.toJsonParam(arg) - if err != nil { - return nil, err - } - if argument["Values"] != nil { - values, ok := argument["Values"].([]interface{}) - if ok { - texts := []string{} - for _, value := range values { - strVal, ok := value.(string) - if ok { - texts = append(texts, strVal) - } - } - return texts, nil - } - } - return []string{}, nil -} - -func (p *paramHelper) parseAsk(arg interface{}) ([]string, error) { - argument, err := p.toJsonParam(arg) - if err != nil { - return nil, err - } - if argument["Question"] != nil { - question, ok := argument["Question"].(string) - if ok { - return []string{question}, nil - } - } - return []string{}, nil -} diff --git a/modules/text-spellcheck/additional/spellcheck/param_helper_test.go b/modules/text-spellcheck/additional/spellcheck/param_helper_test.go deleted file mode 100644 index 2737bdb55b32418c59376901d6e41ab628ed0290..0000000000000000000000000000000000000000 --- a/modules/text-spellcheck/additional/spellcheck/param_helper_test.go +++ /dev/null @@ -1,86 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package spellcheck - -import ( - "reflect" - "testing" -) - -type fakeNearText struct { - Values []string -} - -type fakeAsk struct { - Question string -} - -func Test_paramHelper_getTexts(t *testing.T) { - type args struct { - argumentModuleParams map[string]interface{} - } - tests := []struct { - name string - args args - want string - want1 []string - wantErr bool - }{ - { - name: "should get values from nearText", - args: args{ - argumentModuleParams: map[string]interface{}{ - "nearText": fakeNearText{Values: []string{"a", "b"}}, - }, - }, - want: "nearText", - want1: []string{"a", "b"}, - wantErr: false, - }, - { - name: "should get values from ask", - args: args{ - argumentModuleParams: map[string]interface{}{ - "ask": fakeAsk{Question: "a"}, - }, - }, - want: "ask", - want1: []string{"a"}, - wantErr: false, - }, - { - name: "should be empty", - args: args{ - argumentModuleParams: map[string]interface{}{}, - }, - want: "", - want1: []string{}, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - p := ¶mHelper{} - got, got1, err := p.getTexts(tt.args.argumentModuleParams) - if (err != nil) != tt.wantErr { - t.Errorf("paramHelper.getTexts() error = %v, wantErr %v", err, tt.wantErr) - return - } - if got != tt.want { - t.Errorf("paramHelper.getTexts() got = %v, want %v", got, tt.want) - } - if !reflect.DeepEqual(got1, tt.want1) { - t.Errorf("paramHelper.getTexts() got1 = %v, want %v", got1, tt.want1) - } - }) - } -} diff --git a/modules/text-spellcheck/additional/spellcheck/spellcheck.go b/modules/text-spellcheck/additional/spellcheck/spellcheck.go deleted file mode 100644 index 9b3a8bf5fdbec368eddc5693f49d645507b08188..0000000000000000000000000000000000000000 --- a/modules/text-spellcheck/additional/spellcheck/spellcheck.go +++ /dev/null @@ -1,60 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package spellcheck - -import ( - "context" - "errors" - - "github.com/tailor-inc/graphql" - "github.com/tailor-inc/graphql/language/ast" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/search" - "github.com/weaviate/weaviate/modules/text-spellcheck/ent" -) - -type Params struct{} - -type spellCheckClient interface { - Check(ctx context.Context, text []string) (*ent.SpellCheckResult, error) -} - -type SpellCheckProvider struct { - spellCheck spellCheckClient - paramHelper *paramHelper -} - -func New(spellCheck spellCheckClient) *SpellCheckProvider { - return &SpellCheckProvider{spellCheck, newParamHelper()} -} - -func (p *SpellCheckProvider) AdditionalPropertyDefaultValue() interface{} { - return &Params{} -} - -func (p *SpellCheckProvider) ExtractAdditionalFn(param []*ast.Argument) interface{} { - return &Params{} -} - -func (p *SpellCheckProvider) AdditionalFieldFn(classname string) *graphql.Field { - return p.additionalSpellCheckField(classname) -} - -func (p *SpellCheckProvider) AdditionalPropertyFn(ctx context.Context, - in []search.Result, params interface{}, limit *int, - argumentModuleParams map[string]interface{}, cfg moduletools.ClassConfig, -) ([]search.Result, error) { - if parameters, ok := params.(*Params); ok { - return p.findSpellCheck(ctx, in, parameters, limit, argumentModuleParams) - } - return nil, errors.New("wrong parameters") -} diff --git a/modules/text-spellcheck/additional/spellcheck/spellcheck_graphql_field.go b/modules/text-spellcheck/additional/spellcheck/spellcheck_graphql_field.go deleted file mode 100644 index 022a3276b5c0909133143e20833f7447e81d2026..0000000000000000000000000000000000000000 --- a/modules/text-spellcheck/additional/spellcheck/spellcheck_graphql_field.go +++ /dev/null @@ -1,49 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package spellcheck - -import ( - "fmt" - - "github.com/tailor-inc/graphql" -) - -func (p *SpellCheckProvider) additionalSpellCheckField(classname string) *graphql.Field { - return &graphql.Field{ - Type: graphql.NewList(p.additionalSpellCheckObj(classname)), - } -} - -func (p *SpellCheckProvider) additionalSpellCheckObj(classname string) *graphql.Object { - return graphql.NewObject(graphql.ObjectConfig{ - Name: fmt.Sprintf("%sAdditionalSpellCheck", classname), - Fields: graphql.Fields{ - "originalText": &graphql.Field{Type: graphql.String}, - "didYouMean": &graphql.Field{Type: graphql.String}, - "location": &graphql.Field{Type: graphql.String}, - "numberOfCorrections": &graphql.Field{Type: graphql.Int}, - "changes": &graphql.Field{ - Type: graphql.NewList(p.additionalSpellCheckChangesObj(classname)), - }, - }, - }) -} - -func (p *SpellCheckProvider) additionalSpellCheckChangesObj(classname string) *graphql.Object { - return graphql.NewObject(graphql.ObjectConfig{ - Name: fmt.Sprintf("%sAdditionalSpellCheckChanges", classname), - Fields: graphql.Fields{ - "original": &graphql.Field{Type: graphql.String}, - "corrected": &graphql.Field{Type: graphql.String}, - }, - }) -} diff --git a/modules/text-spellcheck/additional/spellcheck/spellcheck_graphql_field_test.go b/modules/text-spellcheck/additional/spellcheck/spellcheck_graphql_field_test.go deleted file mode 100644 index 7a2565ba8bc133996bfa68eb92a1318728eff187..0000000000000000000000000000000000000000 --- a/modules/text-spellcheck/additional/spellcheck/spellcheck_graphql_field_test.go +++ /dev/null @@ -1,66 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package spellcheck - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/tailor-inc/graphql" -) - -func TestSpellCheckField(t *testing.T) { - t.Run("should generate spellCheck argument properly", func(t *testing.T) { - // given - spellCheckProvider := &SpellCheckProvider{} - classname := "Class" - - // when - spellCheck := spellCheckProvider.additionalSpellCheckField(classname) - - // then - // the built graphQL field needs to support this structure: - // Type: { - // spellCheck: [{ - // "originalText": "What did the monkey do?", - // "didYouMean": "What did the monkey do?"", - // "location": "nearText.concepts[0]", - // "numberOfCorrections": 1, - // "changes": [{ - // "original": "misspelling", - // "didYouMean": "correction" - // }] - // }] - // } - assert.NotNil(t, spellCheck) - assert.Equal(t, "ClassAdditionalSpellCheck", spellCheck.Type.Name()) - assert.NotNil(t, spellCheck.Type) - spellCheckObjectList, spellCheckObjectListOK := spellCheck.Type.(*graphql.List) - assert.True(t, spellCheckObjectListOK) - spellCheckObject, spellCheckObjectOK := spellCheckObjectList.OfType.(*graphql.Object) - assert.True(t, spellCheckObjectOK) - assert.Equal(t, 5, len(spellCheckObject.Fields())) - assert.NotNil(t, spellCheckObject.Fields()["originalText"]) - assert.NotNil(t, spellCheckObject.Fields()["didYouMean"]) - assert.NotNil(t, spellCheckObject.Fields()["location"]) - assert.NotNil(t, spellCheckObject.Fields()["numberOfCorrections"]) - assert.NotNil(t, spellCheckObject.Fields()["changes"]) - changes := spellCheckObject.Fields()["changes"] - spellCheckChangesObjectList, spellCheckChangesObjectListOK := changes.Type.(*graphql.List) - assert.True(t, spellCheckChangesObjectListOK) - spellCheckChangesObject, spellCheckChangesObjectOK := spellCheckChangesObjectList.OfType.(*graphql.Object) - assert.True(t, spellCheckChangesObjectOK) - assert.Equal(t, 2, len(spellCheckChangesObject.Fields())) - assert.NotNil(t, spellCheckChangesObject.Fields()["original"]) - assert.NotNil(t, spellCheckChangesObject.Fields()["corrected"]) - }) -} diff --git a/modules/text-spellcheck/additional/spellcheck/spellcheck_result.go b/modules/text-spellcheck/additional/spellcheck/spellcheck_result.go deleted file mode 100644 index 7bc13e25ca169e6f22e973655b1f06ceee4134ee..0000000000000000000000000000000000000000 --- a/modules/text-spellcheck/additional/spellcheck/spellcheck_result.go +++ /dev/null @@ -1,97 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package spellcheck - -import ( - "context" - "errors" - "fmt" - "strings" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/search" - spellcheckmodels "github.com/weaviate/weaviate/modules/text-spellcheck/additional/models" - "github.com/weaviate/weaviate/modules/text-spellcheck/ent" -) - -func (p *SpellCheckProvider) findSpellCheck(ctx context.Context, - in []search.Result, params *Params, limit *int, - argumentModuleParams map[string]interface{}, -) ([]search.Result, error) { - if len(in) > 0 { - name, texts, err := p.paramHelper.getTexts(argumentModuleParams) - if err != nil { - return in, errors.New("cannot get texts") - } - spellCheckAdditionalProperty, err := p.performSpellCheck(ctx, name, texts) - if err != nil { - return in, err - } - for i := range in { - ap := in[i].AdditionalProperties - if ap == nil { - ap = models.AdditionalProperties{} - } - ap["spellCheck"] = spellCheckAdditionalProperty - in[i].AdditionalProperties = ap - } - } - return in, nil -} - -func (p *SpellCheckProvider) performSpellCheck(ctx context.Context, name string, texts []string) ([]*spellcheckmodels.SpellCheck, error) { - if len(texts) == 0 { - return []*spellcheckmodels.SpellCheck{}, nil - } - spellCheckResult, err := p.spellCheck.Check(ctx, texts) - if err != nil { - return nil, err - } - return p.getSpellCheckAdditionalProperty(name, spellCheckResult), nil -} - -func (p *SpellCheckProvider) getSpellCheckAdditionalProperty(name string, spellCheckResult *ent.SpellCheckResult) []*spellcheckmodels.SpellCheck { - spellCheck := []*spellcheckmodels.SpellCheck{} - for i, t := range spellCheckResult.Text { - spellCheck = append(spellCheck, p.getSpellCheckAdditionalPropertyObject(t, p.getSpellCheckLocation(name, i), spellCheckResult)) - } - return spellCheck -} - -func (p *SpellCheckProvider) getSpellCheckLocation(name string, i int) string { - if name == "nearText" { - return fmt.Sprintf("nearText.concepts[%v]", i) - } - return "ask.question" -} - -func (p *SpellCheckProvider) getSpellCheckAdditionalPropertyObject(originalText, location string, spellCheckResult *ent.SpellCheckResult) *spellcheckmodels.SpellCheck { - didYouMean := originalText - changes := []spellcheckmodels.SpellCheckChange{} - for _, change := range spellCheckResult.Changes { - if strings.Contains(strings.ToLower(didYouMean), change.Original) { - didYouMean = strings.ReplaceAll(strings.ToLower(didYouMean), change.Original, change.Correction) - change := spellcheckmodels.SpellCheckChange{ - Original: change.Original, - Corrected: change.Correction, - } - changes = append(changes, change) - } - } - return &spellcheckmodels.SpellCheck{ - OriginalText: originalText, - DidYouMean: didYouMean, - Location: location, - NumberOfCorrections: len(changes), - Changes: changes, - } -} diff --git a/modules/text-spellcheck/clients/spellcheck.go b/modules/text-spellcheck/clients/spellcheck.go deleted file mode 100644 index 52693586cba5d451d906ae5ff30929a883c808de..0000000000000000000000000000000000000000 --- a/modules/text-spellcheck/clients/spellcheck.go +++ /dev/null @@ -1,114 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/modules/text-spellcheck/ent" -) - -type spellCheckInput struct { - Text []string `json:"text"` -} - -type spellCheckCorrection struct { - Original string `json:"original"` - Correction string `json:"correction"` -} - -type spellCheckResponse struct { - spellCheckInput - Changes []spellCheckCorrection `json:"changes"` -} - -type spellCheck struct { - origin string - httpClient *http.Client - logger logrus.FieldLogger -} - -func New(origin string, timeout time.Duration, logger logrus.FieldLogger) *spellCheck { - return &spellCheck{ - origin: origin, - httpClient: &http.Client{ - Timeout: timeout, - }, - logger: logger, - } -} - -func (s *spellCheck) Check(ctx context.Context, text []string) (*ent.SpellCheckResult, error) { - body, err := json.Marshal(spellCheckInput{ - Text: text, - }) - if err != nil { - return nil, errors.Wrapf(err, "marshal body") - } - - req, err := http.NewRequestWithContext(ctx, "POST", s.url("/spellcheck/"), - bytes.NewReader(body)) - if err != nil { - return nil, errors.Wrap(err, "create POST request") - } - - res, err := s.httpClient.Do(req) - if err != nil { - return nil, errors.Wrap(err, "send POST request") - } - defer res.Body.Close() - - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - return nil, errors.Wrap(err, "read response body") - } - - var resBody spellCheckResponse - if err := json.Unmarshal(bodyBytes, &resBody); err != nil { - return nil, errors.Wrap(err, "unmarshal response body") - } - - if res.StatusCode > 399 { - return nil, errors.Errorf("fail with status %d", res.StatusCode) - } - - return &ent.SpellCheckResult{ - Text: resBody.Text, - Changes: s.getCorrections(resBody.Changes), - }, nil -} - -func (s *spellCheck) url(path string) string { - return fmt.Sprintf("%s%s", s.origin, path) -} - -func (s *spellCheck) getCorrections(changes []spellCheckCorrection) []ent.SpellCheckCorrection { - if len(changes) == 0 { - return nil - } - corrections := make([]ent.SpellCheckCorrection, len(changes)) - for i := range changes { - corrections[i] = ent.SpellCheckCorrection{ - Original: changes[i].Original, - Correction: changes[i].Correction, - } - } - return corrections -} diff --git a/modules/text-spellcheck/clients/spellcheck_meta.go b/modules/text-spellcheck/clients/spellcheck_meta.go deleted file mode 100644 index 24821420e32c43c0a62c2cca9207d8e0f4727412..0000000000000000000000000000000000000000 --- a/modules/text-spellcheck/clients/spellcheck_meta.go +++ /dev/null @@ -1,45 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "context" - "encoding/json" - "io" - "net/http" - - "github.com/pkg/errors" -) - -func (s *spellCheck) MetaInfo() (map[string]interface{}, error) { - req, err := http.NewRequestWithContext(context.Background(), "GET", s.url("/meta"), nil) - if err != nil { - return nil, errors.Wrap(err, "create GET meta request") - } - - res, err := s.httpClient.Do(req) - if err != nil { - return nil, errors.Wrap(err, "send GET meta request") - } - defer res.Body.Close() - - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - return nil, errors.Wrap(err, "read meta response body") - } - - var resBody map[string]interface{} - if err := json.Unmarshal(bodyBytes, &resBody); err != nil { - return nil, errors.Wrap(err, "unmarshal meta response body") - } - return resBody, nil -} diff --git a/modules/text-spellcheck/clients/spellcheck_meta_test.go b/modules/text-spellcheck/clients/spellcheck_meta_test.go deleted file mode 100644 index 0aac1cac94654495f82dd7890d472e8edefe44f3..0000000000000000000000000000000000000000 --- a/modules/text-spellcheck/clients/spellcheck_meta_test.go +++ /dev/null @@ -1,63 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func TestGetMeta(t *testing.T) { - t.Run("when the server is providing meta", func(t *testing.T) { - server := httptest.NewServer(&testMetaHandler{t: t}) - defer server.Close() - c := New(server.URL, 0, nullLogger()) - meta, err := c.MetaInfo() - - assert.Nil(t, err) - assert.NotNil(t, meta) - metaModel := meta["model"] - assert.True(t, metaModel != nil) - model, modelOK := metaModel.(map[string]interface{}) - assert.True(t, modelOK) - assert.True(t, model["name"] != nil) - }) -} - -type testMetaHandler struct { - t *testing.T - // the test handler will report as not ready before the time has passed - readyTime time.Time -} - -func (f *testMetaHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - assert.Equal(f.t, "/meta", r.URL.String()) - assert.Equal(f.t, http.MethodGet, r.Method) - - if time.Since(f.readyTime) < 0 { - w.WriteHeader(http.StatusServiceUnavailable) - } - - w.Write([]byte(f.metaInfo())) -} - -func (f *testMetaHandler) metaInfo() string { - return `{ - "model": { - "name": "pyspellchecker" - } -}` -} diff --git a/modules/text-spellcheck/clients/startup.go b/modules/text-spellcheck/clients/startup.go deleted file mode 100644 index 8edbf68e719177fe7345a3ec7e4b6e34f5fc8d94..0000000000000000000000000000000000000000 --- a/modules/text-spellcheck/clients/startup.go +++ /dev/null @@ -1,68 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "context" - "net/http" - "time" - - "github.com/pkg/errors" -) - -func (s *spellCheck) WaitForStartup(initCtx context.Context, - interval time.Duration, -) error { - t := time.NewTicker(interval) - defer t.Stop() - expired := initCtx.Done() - var lastErr error - for { - select { - case <-t.C: - lastErr = s.checkReady(initCtx) - if lastErr == nil { - return nil - } - s.logger. - WithField("action", "qna_remote_wait_for_startup"). - WithError(lastErr).Warnf("qna remote service not ready") - case <-expired: - return errors.Wrapf(lastErr, "init context expired before remote was ready") - } - } -} - -func (s *spellCheck) checkReady(initCtx context.Context) error { - // spawn a new context (derived on the overall context) which is used to - // consider an individual request timed out - requestCtx, cancel := context.WithTimeout(initCtx, 500*time.Millisecond) - defer cancel() - - req, err := http.NewRequestWithContext(requestCtx, http.MethodGet, - s.url("/.well-known/ready"), nil) - if err != nil { - return errors.Wrap(err, "create check ready request") - } - - res, err := s.httpClient.Do(req) - if err != nil { - return errors.Wrap(err, "send check ready request") - } - - defer res.Body.Close() - if res.StatusCode > 299 { - return errors.Errorf("not ready: status %d", res.StatusCode) - } - - return nil -} diff --git a/modules/text-spellcheck/clients/startup_test.go b/modules/text-spellcheck/clients/startup_test.go deleted file mode 100644 index 7836347d937eaaa18e3c1415dcb4b2aac2d2b15b..0000000000000000000000000000000000000000 --- a/modules/text-spellcheck/clients/startup_test.go +++ /dev/null @@ -1,98 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "context" - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/sirupsen/logrus" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestWaitForStartup(t *testing.T) { - t.Run("when the server is immediately ready", func(t *testing.T) { - server := httptest.NewServer(&testReadyHandler{t: t}) - defer server.Close() - c := New(server.URL, 0, nullLogger()) - err := c.WaitForStartup(context.Background(), 50*time.Millisecond) - - assert.Nil(t, err) - }) - - t.Run("when the server is down", func(t *testing.T) { - c := New("http://nothing-running-at-this-url", 0, nullLogger()) - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancel() - err := c.WaitForStartup(ctx, 150*time.Millisecond) - - require.NotNil(t, err, nullLogger()) - assert.Contains(t, err.Error(), "expired before remote was ready") - }) - - t.Run("when the server is alive, but not ready", func(t *testing.T) { - server := httptest.NewServer(&testReadyHandler{ - t: t, - readyTime: time.Now().Add(1 * time.Minute), - }) - c := New(server.URL, 0, nullLogger()) - defer server.Close() - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancel() - err := c.WaitForStartup(ctx, 50*time.Millisecond) - - require.NotNil(t, err) - assert.Contains(t, err.Error(), "expired before remote was ready") - }) - - t.Run("when the server is initially not ready, but then becomes ready", - func(t *testing.T) { - server := httptest.NewServer(&testReadyHandler{ - t: t, - readyTime: time.Now().Add(100 * time.Millisecond), - }) - c := New(server.URL, 0, nullLogger()) - defer server.Close() - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancel() - err := c.WaitForStartup(ctx, 50*time.Millisecond) - - require.Nil(t, err) - }) -} - -type testReadyHandler struct { - t *testing.T - // the test handler will report as not ready before the time has passed - readyTime time.Time -} - -func (f *testReadyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - assert.Equal(f.t, "/.well-known/ready", r.URL.String()) - assert.Equal(f.t, http.MethodGet, r.Method) - - if time.Since(f.readyTime) < 0 { - w.WriteHeader(http.StatusServiceUnavailable) - } - - w.WriteHeader(http.StatusNoContent) -} - -func nullLogger() logrus.FieldLogger { - l, _ := test.NewNullLogger() - return l -} diff --git a/modules/text-spellcheck/config.go b/modules/text-spellcheck/config.go deleted file mode 100644 index b8813d11a685b1e613bbe40bad98a54d7219b784..0000000000000000000000000000000000000000 --- a/modules/text-spellcheck/config.go +++ /dev/null @@ -1,39 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modspellcheck - -import ( - "context" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/schema" -) - -func (m *SpellCheckModule) ClassConfigDefaults() map[string]interface{} { - return map[string]interface{}{} -} - -func (m *SpellCheckModule) PropertyConfigDefaults( - dt *schema.DataType, -) map[string]interface{} { - return map[string]interface{}{} -} - -func (m *SpellCheckModule) ValidateClass(ctx context.Context, - class *models.Class, cfg moduletools.ClassConfig, -) error { - return nil -} - -var _ = modulecapabilities.ClassConfigurator(New()) diff --git a/modules/text-spellcheck/ent/spellcheck_result.go b/modules/text-spellcheck/ent/spellcheck_result.go deleted file mode 100644 index eef02f828d3667529a537bcb2572edacee1b3bdc..0000000000000000000000000000000000000000 --- a/modules/text-spellcheck/ent/spellcheck_result.go +++ /dev/null @@ -1,21 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package ent - -type SpellCheckCorrection struct { - Original, Correction string -} - -type SpellCheckResult struct { - Text []string - Changes []SpellCheckCorrection -} diff --git a/modules/text-spellcheck/module.go b/modules/text-spellcheck/module.go deleted file mode 100644 index d7af9b3163fa3b923eaec0d5b3991384d1d96e81..0000000000000000000000000000000000000000 --- a/modules/text-spellcheck/module.go +++ /dev/null @@ -1,109 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modspellcheck - -import ( - "context" - "net/http" - "os" - "time" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - spellcheckadditional "github.com/weaviate/weaviate/modules/text-spellcheck/additional" - spellcheckadditionalspellcheck "github.com/weaviate/weaviate/modules/text-spellcheck/additional/spellcheck" - "github.com/weaviate/weaviate/modules/text-spellcheck/clients" - "github.com/weaviate/weaviate/modules/text-spellcheck/ent" - spellchecktexttransformer "github.com/weaviate/weaviate/modules/text-spellcheck/transformer" - spellchecktexttransformerautocorrect "github.com/weaviate/weaviate/modules/text-spellcheck/transformer/autocorrect" -) - -func New() *SpellCheckModule { - return &SpellCheckModule{} -} - -type SpellCheckModule struct { - spellCheck spellCheckClient - additionalPropertiesProvider modulecapabilities.AdditionalProperties - textTransformersProvider modulecapabilities.TextTransformers -} - -type spellCheckClient interface { - Check(ctx context.Context, text []string) (*ent.SpellCheckResult, error) - MetaInfo() (map[string]interface{}, error) -} - -func (m *SpellCheckModule) Name() string { - return "text-spellcheck" -} - -func (m *SpellCheckModule) Type() modulecapabilities.ModuleType { - return modulecapabilities.Extension -} - -func (m *SpellCheckModule) Init(ctx context.Context, - params moduletools.ModuleInitParams, -) error { - uri := os.Getenv("SPELLCHECK_INFERENCE_API") - if uri == "" { - return errors.Errorf("required variable SPELLCHECK_INFERENCE_API is not set") - } - - client := clients.New(uri, params.GetConfig().ModuleHttpClientTimeout, params.GetLogger()) - - if err := client.WaitForStartup(ctx, 1*time.Second); err != nil { - return errors.Wrap(err, "init remote spell check module") - } - - m.spellCheck = client - - m.initTextTransformers() - m.initAdditional() - - return nil -} - -func (m *SpellCheckModule) initTextTransformers() { - autocorrectProvider := spellchecktexttransformerautocorrect.New(m.spellCheck) - m.textTransformersProvider = spellchecktexttransformer.New(autocorrectProvider) -} - -func (m *SpellCheckModule) initAdditional() { - spellCheckProvider := spellcheckadditionalspellcheck.New(m.spellCheck) - m.additionalPropertiesProvider = spellcheckadditional.New(spellCheckProvider) -} - -func (m *SpellCheckModule) RootHandler() http.Handler { - // TODO: remove once this is a capability interface - return nil -} - -func (m *SpellCheckModule) MetaInfo() (map[string]interface{}, error) { - return m.spellCheck.MetaInfo() -} - -func (m *SpellCheckModule) AdditionalProperties() map[string]modulecapabilities.AdditionalProperty { - return m.additionalPropertiesProvider.AdditionalProperties() -} - -func (m *SpellCheckModule) TextTransformers() map[string]modulecapabilities.TextTransform { - return m.textTransformersProvider.TextTransformers() -} - -// verify we implement the modules.Module interface -var ( - _ = modulecapabilities.Module(New()) - _ = modulecapabilities.AdditionalProperties(New()) - _ = modulecapabilities.MetaProvider(New()) - _ = modulecapabilities.TextTransformers(New()) -) diff --git a/modules/text-spellcheck/transformer/autocorrect/autocorrect.go b/modules/text-spellcheck/transformer/autocorrect/autocorrect.go deleted file mode 100644 index 3263939c4d48f27420c778f545c7eb34fa892ad3..0000000000000000000000000000000000000000 --- a/modules/text-spellcheck/transformer/autocorrect/autocorrect.go +++ /dev/null @@ -1,48 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package autocorrect - -import ( - "context" - "strings" - - "github.com/weaviate/weaviate/modules/text-spellcheck/ent" -) - -type spellCheckClient interface { - Check(ctx context.Context, text []string) (*ent.SpellCheckResult, error) -} - -type AutocorrectTransformer struct { - spellCheckClient spellCheckClient -} - -func New(spellCheckClient spellCheckClient) *AutocorrectTransformer { - return &AutocorrectTransformer{spellCheckClient} -} - -func (t *AutocorrectTransformer) Transform(in []string) ([]string, error) { - spellCheckResult, err := t.spellCheckClient.Check(context.Background(), in) - if err != nil { - return nil, err - } - result := make([]string, len(in)) - changes := spellCheckResult.Changes - for i, txt := range spellCheckResult.Text { - didYouMean := txt - for _, change := range changes { - didYouMean = strings.ReplaceAll(strings.ToLower(didYouMean), change.Original, change.Correction) - } - result[i] = didYouMean - } - return result, nil -} diff --git a/modules/text-spellcheck/transformer/provider.go b/modules/text-spellcheck/transformer/provider.go deleted file mode 100644 index 868b1968ead7561b966cfd3ed57bbee680c95a05..0000000000000000000000000000000000000000 --- a/modules/text-spellcheck/transformer/provider.go +++ /dev/null @@ -1,31 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package texttransformer - -import ( - "github.com/weaviate/weaviate/entities/modulecapabilities" -) - -type TextTransformerProvider struct { - autocorrecProvider modulecapabilities.TextTransform -} - -func New(autocorrecProvider modulecapabilities.TextTransform) *TextTransformerProvider { - return &TextTransformerProvider{autocorrecProvider} -} - -func (p *TextTransformerProvider) TextTransformers() map[string]modulecapabilities.TextTransform { - textTransformers := map[string]modulecapabilities.TextTransform{} - textTransformers["nearText"] = p.autocorrecProvider - textTransformers["ask"] = p.autocorrecProvider - return textTransformers -} diff --git a/modules/text2vec-aws/clients/aws.go b/modules/text2vec-aws/clients/aws.go deleted file mode 100644 index 43d3b4de860398814d3d7df7487c1e5ff6f2cb9b..0000000000000000000000000000000000000000 --- a/modules/text2vec-aws/clients/aws.go +++ /dev/null @@ -1,397 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "strings" - "time" - - "github.com/google/uuid" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/modules/text2vec-aws/ent" -) - -type operationType string - -var ( - vectorizeObject operationType = "vectorize_object" - vectorizeQuery operationType = "vectorize_query" -) - -func buildBedrockUrl(service, region, model string) string { - serviceName := service - if strings.HasPrefix(model, "cohere") { - serviceName = fmt.Sprintf("%s-runtime", serviceName) - } - urlTemplate := "https://%s.%s.amazonaws.com/model/%s/invoke" - return fmt.Sprintf(urlTemplate, serviceName, region, model) -} - -func buildSagemakerUrl(service, region, endpoint string) string { - urlTemplate := "https://runtime.%s.%s.amazonaws.com/endpoints/%s/invocations" - return fmt.Sprintf(urlTemplate, service, region, endpoint) -} - -type aws struct { - awsAccessKey string - awsSecret string - buildBedrockUrlFn func(service, region, model string) string - buildSagemakerUrlFn func(service, region, endpoint string) string - httpClient *http.Client - logger logrus.FieldLogger -} - -func New(awsAccessKey string, awsSecret string, timeout time.Duration, logger logrus.FieldLogger) *aws { - return &aws{ - awsAccessKey: awsAccessKey, - awsSecret: awsSecret, - httpClient: &http.Client{ - Timeout: timeout, - }, - buildBedrockUrlFn: buildBedrockUrl, - buildSagemakerUrlFn: buildSagemakerUrl, - logger: logger, - } -} - -func (v *aws) Vectorize(ctx context.Context, input []string, - config ent.VectorizationConfig, -) (*ent.VectorizationResult, error) { - return v.vectorize(ctx, input, vectorizeObject, config) -} - -func (v *aws) VectorizeQuery(ctx context.Context, input []string, - config ent.VectorizationConfig, -) (*ent.VectorizationResult, error) { - return v.vectorize(ctx, input, vectorizeQuery, config) -} - -func (v *aws) vectorize(ctx context.Context, input []string, operation operationType, config ent.VectorizationConfig) (*ent.VectorizationResult, error) { - service := v.getService(config) - region := v.getRegion(config) - model := v.getModel(config) - endpoint := v.getEndpoint(config) - targetModel := v.getTargetModel(config) - targetVariant := v.getTargetVariant(config) - - var body []byte - var endpointUrl string - var host string - var path string - var err error - - headers := map[string]string{ - "accept": "*/*", - "content-type": contentType, - } - - if v.isBedrock(service) { - endpointUrl = v.buildBedrockUrlFn(service, region, model) - host, path, _ = extractHostAndPath(endpointUrl) - - req, err := createRequestBody(model, input, operation) - if err != nil { - return nil, err - } - - body, err = json.Marshal(req) - if err != nil { - return nil, errors.Wrapf(err, "marshal body") - } - } else if v.isSagemaker(service) { - endpointUrl = v.buildSagemakerUrlFn(service, region, endpoint) - host = "runtime." + service + "." + region + ".amazonaws.com" - path = "/endpoints/" + endpoint + "/invocations" - if targetModel != "" { - headers["x-amzn-sagemaker-target-model"] = targetModel - } - if targetVariant != "" { - headers["x-amzn-sagemaker-target-variant"] = targetVariant - } - body, err = json.Marshal(sagemakerEmbeddingsRequest{ - TextInputs: input, - }) - if err != nil { - return nil, errors.Wrapf(err, "marshal body") - } - } else { - return nil, errors.Wrapf(err, "service error") - } - - accessKey, err := v.getAwsAccessKey(ctx) - if err != nil { - return nil, errors.Wrapf(err, "AWS Access Key") - } - secretKey, err := v.getAwsAccessSecret(ctx) - if err != nil { - return nil, errors.Wrapf(err, "AWS Secret Key") - } - - headers["host"] = host - amzDate, headers, authorizationHeader := getAuthHeader(accessKey, secretKey, host, service, region, path, body, headers) - headers["Authorization"] = authorizationHeader - headers["x-amz-date"] = amzDate - - req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpointUrl, bytes.NewReader(body)) - if err != nil { - return nil, errors.Wrap(err, "create POST request") - } - - for k, v := range headers { - req.Header.Set(k, v) - } - - res, err := v.makeRequest(req, 30, 5) - if err != nil { - return nil, errors.Wrap(err, "send POST request") - } - defer res.Body.Close() - - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - return nil, errors.Wrap(err, "read response body") - } - if v.isBedrock(service) { - return v.parseBedrockResponse(bodyBytes, res, input) - } else { - return v.parseSagemakerResponse(bodyBytes, res, input) - } -} - -func (v *aws) makeRequest(req *http.Request, delayInSeconds int, maxRetries int) (*http.Response, error) { - var res *http.Response - var err error - - // Generate a UUID for this request - requestID := uuid.New().String() - - for i := 0; i < maxRetries; i++ { - res, err = v.httpClient.Do(req) - if err != nil { - return nil, errors.Wrap(err, "send POST request") - } - - // If the status code is not 429 or 400, break the loop - if res.StatusCode != http.StatusTooManyRequests && res.StatusCode != http.StatusBadRequest { - break - } - - v.logger.Debugf("Request ID %s to %s returned 429, retrying in %d seconds", requestID, req.URL, delayInSeconds) - - // Sleep for a while and then continue to the next iteration - time.Sleep(time.Duration(delayInSeconds) * time.Second) - - // Double the delay for the next iteration - delayInSeconds *= 2 - - } - - return res, err -} - -func (v *aws) parseBedrockResponse(bodyBytes []byte, res *http.Response, input []string) (*ent.VectorizationResult, error) { - var resBodyMap map[string]interface{} - if err := json.Unmarshal(bodyBytes, &resBodyMap); err != nil { - return nil, errors.Wrap(err, "unmarshal response body") - } - - // if resBodyMap has inputTextTokenCount, it's a resonse from an Amazon model - // otherwise, it is a response from a Cohere model - var resBody bedrockEmbeddingResponse - if err := json.Unmarshal(bodyBytes, &resBody); err != nil { - return nil, errors.Wrap(err, "unmarshal response body") - } - - if res.StatusCode != 200 || resBody.Message != nil { - if resBody.Message != nil { - return nil, fmt.Errorf("connection to AWS Bedrock failed with status: %v error: %s", - res.StatusCode, *resBody.Message) - } - return nil, fmt.Errorf("connection to AWS Bedrock failed with status: %d", res.StatusCode) - } - - if len(resBody.Embedding) == 0 && len(resBody.Embeddings) == 0 { - return nil, fmt.Errorf("could not obtain vector from AWS Bedrock") - } - - embedding := resBody.Embedding - if len(resBody.Embeddings) > 0 { - embedding = resBody.Embeddings[0] - } - - return &ent.VectorizationResult{ - Text: input[0], - Dimensions: len(embedding), - Vector: embedding, - }, nil -} - -func (v *aws) parseSagemakerResponse(bodyBytes []byte, res *http.Response, input []string) (*ent.VectorizationResult, error) { - var resBody sagemakerEmbeddingResponse - if err := json.Unmarshal(bodyBytes, &resBody); err != nil { - return nil, errors.Wrap(err, "unmarshal response body") - } - - if res.StatusCode != 200 || resBody.Message != nil { - if resBody.Message != nil { - return nil, fmt.Errorf("connection to AWS failed with status: %v error: %s", - res.StatusCode, *resBody.Message) - } - return nil, fmt.Errorf("connection to AWS failed with status: %d", res.StatusCode) - } - - if len(resBody.Embedding) == 0 { - return nil, errors.Errorf("empty embeddings response") - } - - return &ent.VectorizationResult{ - Text: input[0], - Dimensions: len(resBody.Embedding[0]), - Vector: resBody.Embedding[0], - }, nil -} - -func (v *aws) isSagemaker(service string) bool { - return service == "sagemaker" -} - -func (v *aws) isBedrock(service string) bool { - return service == "bedrock" -} - -func (v *aws) getAwsAccessKey(ctx context.Context) (string, error) { - awsAccessKey := ctx.Value("X-Aws-Access-Key") - if awsAccessKeyHeader, ok := awsAccessKey.([]string); ok && - len(awsAccessKeyHeader) > 0 && len(awsAccessKeyHeader[0]) > 0 { - return awsAccessKeyHeader[0], nil - } - if len(v.awsAccessKey) > 0 { - return v.awsAccessKey, nil - } - return "", errors.New("no access key found " + - "neither in request header: X-AWS-Access-Key " + - "nor in environment variable under AWS_ACCESS_KEY_ID or AWS_ACCESS_KEY") -} - -func (v *aws) getAwsAccessSecret(ctx context.Context) (string, error) { - awsSecretKey := ctx.Value("X-Aws-Secret-Key") - if awsAccessSecretHeader, ok := awsSecretKey.([]string); ok && - len(awsAccessSecretHeader) > 0 && len(awsAccessSecretHeader[0]) > 0 { - return awsAccessSecretHeader[0], nil - } - if len(v.awsSecret) > 0 { - return v.awsSecret, nil - } - return "", errors.New("no secret found " + - "neither in request header: X-AWS-Secret-Key " + - "nor in environment variable under AWS_SECRET_ACCESS_KEY or AWS_SECRET_KEY") -} - -func (v *aws) getModel(config ent.VectorizationConfig) string { - return config.Model -} - -func (v *aws) getRegion(config ent.VectorizationConfig) string { - return config.Region -} - -func (v *aws) getService(config ent.VectorizationConfig) string { - return config.Service -} - -func (v *aws) getEndpoint(config ent.VectorizationConfig) string { - return config.Endpoint -} - -func (v *aws) getTargetModel(config ent.VectorizationConfig) string { - return config.TargetModel -} - -func (v *aws) getTargetVariant(config ent.VectorizationConfig) string { - return config.TargetVariant -} - -type bedrockEmbeddingsRequest struct { - InputText string `json:"inputText,omitempty"` -} - -type bedrockCohereEmbeddingRequest struct { - Texts []string `json:"texts"` - InputType string `json:"input_type"` -} - -type sagemakerEmbeddingsRequest struct { - TextInputs []string `json:"text_inputs,omitempty"` -} - -type bedrockEmbeddingResponse struct { - InputTextTokenCount int `json:"InputTextTokenCount,omitempty"` - Embedding []float32 `json:"embedding,omitempty"` - Embeddings [][]float32 `json:"embeddings,omitempty"` - Message *string `json:"message,omitempty"` -} -type sagemakerEmbeddingResponse struct { - Embedding [][]float32 `json:"embedding,omitempty"` - ErrorCode *string `json:"ErrorCode,omitempty"` - LogStreamArn *string `json:"LogStreamArn,omitempty"` - OriginalMessage *string `json:"OriginalMessage,omitempty"` - Message *string `json:"Message,omitempty"` - OriginalStatusCode *int `json:"OriginalStatusCode,omitempty"` -} - -func extractHostAndPath(endpointUrl string) (string, string, error) { - u, err := url.Parse(endpointUrl) - if err != nil { - return "", "", err - } - - if u.Host == "" || u.Path == "" { - return "", "", fmt.Errorf("invalid endpoint URL: %s", endpointUrl) - } - - return u.Host, u.Path, nil -} - -func createRequestBody(model string, texts []string, operation operationType) (interface{}, error) { - modelParts := strings.Split(model, ".") - if len(modelParts) == 0 { - return nil, fmt.Errorf("invalid model: %s", model) - } - - modelProvider := modelParts[0] - - switch modelProvider { - case "amazon": - return bedrockEmbeddingsRequest{ - InputText: texts[0], - }, nil - case "cohere": - inputType := "search_document" - if operation == vectorizeQuery { - inputType = "search_query" - } - return bedrockCohereEmbeddingRequest{ - Texts: texts, - InputType: inputType, - }, nil - default: - return nil, fmt.Errorf("unknown model provider: %s", modelProvider) - } -} diff --git a/modules/text2vec-aws/clients/aws_test.go b/modules/text2vec-aws/clients/aws_test.go deleted file mode 100644 index f93ba982b9466d7ecc03d782837d23429a9a2bc4..0000000000000000000000000000000000000000 --- a/modules/text2vec-aws/clients/aws_test.go +++ /dev/null @@ -1,413 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "context" - "encoding/json" - "io" - "net/http" - "net/http/httptest" - "os" - "strings" - "testing" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/modules/text2vec-aws/ent" -) - -func TestClient(t *testing.T) { - t.Run("when all is fine", func(t *testing.T) { - t.Skip("Skipping this test for now") - server := httptest.NewServer(&fakeHandler{t: t}) - defer server.Close() - c := &aws{ - httpClient: &http.Client{}, - logger: nullLogger(), - awsAccessKey: "access_key", - awsSecret: "secret", - buildBedrockUrlFn: func(service, region, model string) string { - return server.URL - }, - buildSagemakerUrlFn: func(service, region, endpoint string) string { - return server.URL - }, - } - expected := &ent.VectorizationResult{ - Text: "This is my text", - Vector: []float32{0.1, 0.2, 0.3}, - Dimensions: 3, - } - res, err := c.Vectorize(context.Background(), []string{"This is my text"}, - ent.VectorizationConfig{ - Service: "bedrock", - Region: "region", - Model: "model", - }) - - assert.Nil(t, err) - assert.Equal(t, expected, res) - }) - - t.Run("when all is fine - Sagemaker", func(t *testing.T) { - server := httptest.NewServer(&fakeHandler{t: t}) - defer server.Close() - c := &aws{ - httpClient: &http.Client{}, - logger: nullLogger(), - awsAccessKey: "access_key", - awsSecret: "secret", - buildBedrockUrlFn: func(service, region, model string) string { - return server.URL - }, - buildSagemakerUrlFn: func(service, region, endpoint string) string { - return server.URL - }, - } - expected := &ent.VectorizationResult{ - Text: "This is my text", - Vector: []float32{0.1, 0.2, 0.3}, - Dimensions: 3, - } - res, err := c.Vectorize(context.Background(), []string{"This is my text"}, - ent.VectorizationConfig{ - Service: "sagemaker", - Region: "region", - Endpoint: "endpoint", - }) - - assert.Nil(t, err) - assert.Equal(t, expected, res) - }) - - t.Run("when the server returns an error", func(t *testing.T) { - t.Skip("Skipping this test for now") - server := httptest.NewServer(&fakeHandler{ - t: t, - serverError: errors.Errorf("nope, not gonna happen"), - }) - defer server.Close() - c := &aws{ - httpClient: &http.Client{}, - logger: nullLogger(), - awsAccessKey: "access_key", - awsSecret: "secret", - buildBedrockUrlFn: func(service, region, model string) string { - return server.URL - }, - buildSagemakerUrlFn: func(service, region, endpoint string) string { - return server.URL - }, - } - _, err := c.Vectorize(context.Background(), []string{"This is my text"}, - ent.VectorizationConfig{ - Service: "bedrock", - }) - - require.NotNil(t, err) - assert.EqualError(t, err, "connection to AWS failed with status: 500 error: nope, not gonna happen") - }) - - t.Run("when AWS key is passed using X-Aws-Api-Key header", func(t *testing.T) { - t.Skip("Skipping this test for now") - server := httptest.NewServer(&fakeHandler{t: t}) - defer server.Close() - c := &aws{ - httpClient: &http.Client{}, - logger: nullLogger(), - awsAccessKey: "access_key", - awsSecret: "secret", - buildBedrockUrlFn: func(service, region, model string) string { - return server.URL - }, - buildSagemakerUrlFn: func(service, region, endpoint string) string { - return server.URL - }, - } - ctxWithValue := context.WithValue(context.Background(), - "X-Aws-Api-Key", []string{"some-key"}) - - expected := &ent.VectorizationResult{ - Text: "This is my text", - Vector: []float32{0.1, 0.2, 0.3}, - Dimensions: 3, - } - res, err := c.Vectorize(ctxWithValue, []string{"This is my text"}, ent.VectorizationConfig{ - Service: "bedrock", - }) - - require.Nil(t, err) - assert.Equal(t, expected, res) - }) - - t.Run("when X-Aws-Access-Key header is passed but empty", func(t *testing.T) { - t.Skip("Skipping this test for now") - server := httptest.NewServer(&fakeHandler{t: t}) - defer server.Close() - c := &aws{ - httpClient: &http.Client{}, - logger: nullLogger(), - awsAccessKey: "", - awsSecret: "123", - buildBedrockUrlFn: func(service, region, model string) string { - return server.URL - }, - buildSagemakerUrlFn: func(service, region, endpoint string) string { - return server.URL - }, - } - ctxWithValue := context.WithValue(context.Background(), - "X-Aws-Api-Key", []string{""}) - - _, err := c.Vectorize(ctxWithValue, []string{"This is my text"}, ent.VectorizationConfig{ - Service: "bedrock", - }) - - require.NotNil(t, err) - assert.Equal(t, err.Error(), "AWS Access Key: no access key found neither in request header: "+ - "X-Aws-Access-Key nor in environment variable under AWS_ACCESS_KEY_ID") - }) - - t.Run("when X-Aws-Secret-Key header is passed but empty", func(t *testing.T) { - t.Skip("Skipping this test for now") - server := httptest.NewServer(&fakeHandler{t: t}) - defer server.Close() - c := &aws{ - httpClient: &http.Client{}, - logger: nullLogger(), - awsAccessKey: "123", - awsSecret: "", - buildBedrockUrlFn: func(service, region, model string) string { - return server.URL - }, - buildSagemakerUrlFn: func(service, region, endpoint string) string { - return server.URL - }, - } - ctxWithValue := context.WithValue(context.Background(), - "X-Aws-Api-Key", []string{""}) - - _, err := c.Vectorize(ctxWithValue, []string{"This is my text"}, ent.VectorizationConfig{ - Service: "bedrock", - }) - - require.NotNil(t, err) - assert.Equal(t, err.Error(), "AWS Secret Key: no secret found neither in request header: "+ - "X-Aws-Access-Secret nor in environment variable under AWS_SECRET_ACCESS_KEY") - }) -} - -func TestBuildBedrockUrl(t *testing.T) { - service := "bedrock" - region := "us-east-1" - t.Run("when using a Cohere", func(t *testing.T) { - model := "cohere.embed-english-v3" - - expected := "https://bedrock-runtime.us-east-1.amazonaws.com/model/cohere.embed-english-v3/invoke" - result := buildBedrockUrl(service, region, model) - - if result != expected { - t.Errorf("Expected %s but got %s", expected, result) - } - }) - - t.Run("When using an AWS model", func(t *testing.T) { - model := "amazon.titan-e1t-medium" - - expected := "https://bedrock.us-east-1.amazonaws.com/model/amazon.titan-e1t-medium/invoke" - result := buildBedrockUrl(service, region, model) - - if result != expected { - t.Errorf("Expected %s but got %s", expected, result) - } - }) -} - -func TestCreateRequestBody(t *testing.T) { - input := []string{"Hello, world!"} - - t.Run("Create request for Amazon embedding model", func(t *testing.T) { - model := "amazon.titan-e1t-medium" - req, _ := createRequestBody(model, input, vectorizeObject) - _, ok := req.(bedrockEmbeddingsRequest) - if !ok { - t.Fatalf("Expected req to be a bedrockEmbeddingsRequest, got %T", req) - } - }) - - t.Run("Create request for Cohere embedding model", func(t *testing.T) { - model := "cohere.embed-english-v3" - req, _ := createRequestBody(model, input, vectorizeObject) - _, ok := req.(bedrockCohereEmbeddingRequest) - if !ok { - t.Fatalf("Expected req to be a bedrockCohereEmbeddingRequest, got %T", req) - } - }) - - t.Run("Create request for unknown embedding model", func(t *testing.T) { - model := "unknown.model" - _, err := createRequestBody(model, input, vectorizeObject) - if err == nil { - t.Errorf("Expected an error for unknown model, got nil") - } - }) -} - -func TestVectorize(t *testing.T) { - ctx := context.Background() - input := []string{"Hello, world!"} - - t.Run("Vectorize using an Amazon model", func(t *testing.T) { - t.Skip("Skipping because CI doesnt have the right credentials") - config := ent.VectorizationConfig{ - Model: "amazon.titan-e1t-medium", - Service: "bedrock", - Region: "us-east-1", - } - - awsAccessKeyID := os.Getenv("AWS_ACCESS_KEY_ID_AMAZON") - awsSecretAccessKey := os.Getenv("AWS_SECRET_ACCESS_KEY_AMAZON") - - aws := New(awsAccessKeyID, awsSecretAccessKey, 60*time.Second, nil) - - _, err := aws.Vectorize(ctx, input, config) - if err != nil { - t.Errorf("Vectorize returned an error: %v", err) - } - }) - - t.Run("Vectorize using a Cohere model", func(t *testing.T) { - t.Skip("Skipping because CI doesnt have the right credentials") - config := ent.VectorizationConfig{ - Model: "cohere.embed-english-v3", - Service: "bedrock", - Region: "us-east-1", - } - - awsAccessKeyID := os.Getenv("AWS_ACCESS_KEY_ID_COHERE") - awsSecretAccessKey := os.Getenv("AWS_SECRET_ACCESS_KEY_COHERE") - - aws := New(awsAccessKeyID, awsSecretAccessKey, 60*time.Second, nil) - - _, err := aws.Vectorize(ctx, input, config) - if err != nil { - t.Errorf("Vectorize returned an error: %v", err) - } - }) -} - -func TestExtractHostAndPath(t *testing.T) { - t.Run("valid URL", func(t *testing.T) { - endpointUrl := "https://service.region.amazonaws.com/model/model-name/invoke" - expectedHost := "service.region.amazonaws.com" - expectedPath := "/model/model-name/invoke" - - host, path, err := extractHostAndPath(endpointUrl) - if err != nil { - t.Errorf("Unexpected error: %v", err) - } - if host != expectedHost { - t.Errorf("Expected host %s but got %s", expectedHost, host) - } - if path != expectedPath { - t.Errorf("Expected path %s but got %s", expectedPath, path) - } - }) - - t.Run("URL without host or path", func(t *testing.T) { - endpointUrl := "https://" - - _, _, err := extractHostAndPath(endpointUrl) - - if err == nil { - t.Error("Expected error but got nil") - } - }) -} - -type fakeHandler struct { - t *testing.T - serverError error -} - -func (f *fakeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - assert.Equal(f.t, http.MethodPost, r.Method) - - authHeader := r.Header["Authorization"][0] - if f.serverError != nil { - var outBytes []byte - var err error - - if strings.Contains(authHeader, "bedrock") { - embeddingResponse := &bedrockEmbeddingResponse{ - Message: ptString(f.serverError.Error()), - } - outBytes, err = json.Marshal(embeddingResponse) - } else { - embeddingResponse := &sagemakerEmbeddingResponse{ - Message: ptString(f.serverError.Error()), - } - outBytes, err = json.Marshal(embeddingResponse) - } - - require.Nil(f.t, err) - - w.WriteHeader(http.StatusInternalServerError) - w.Write(outBytes) - return - } - - bodyBytes, err := io.ReadAll(r.Body) - require.Nil(f.t, err) - defer r.Body.Close() - - var outBytes []byte - if strings.Contains(authHeader, "bedrock") { - var req bedrockEmbeddingsRequest - require.Nil(f.t, json.Unmarshal(bodyBytes, &req)) - - textInput := req.InputText - assert.Greater(f.t, len(textInput), 0) - embeddingResponse := &bedrockEmbeddingResponse{ - Embedding: []float32{0.1, 0.2, 0.3}, - } - outBytes, err = json.Marshal(embeddingResponse) - } else { - var req sagemakerEmbeddingsRequest - require.Nil(f.t, json.Unmarshal(bodyBytes, &req)) - - textInputs := req.TextInputs - assert.Greater(f.t, len(textInputs), 0) - embeddingResponse := &sagemakerEmbeddingResponse{ - Embedding: [][]float32{{0.1, 0.2, 0.3}}, - } - outBytes, err = json.Marshal(embeddingResponse) - } - - require.Nil(f.t, err) - - w.Write(outBytes) -} - -func nullLogger() logrus.FieldLogger { - l, _ := test.NewNullLogger() - return l -} - -func ptString(in string) *string { - return &in -} diff --git a/modules/text2vec-aws/clients/meta.go b/modules/text2vec-aws/clients/meta.go deleted file mode 100644 index a4d479870317daa5cb8a5b9c4e93334ea7d0ed79..0000000000000000000000000000000000000000 --- a/modules/text2vec-aws/clients/meta.go +++ /dev/null @@ -1,19 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -func (v *aws) MetaInfo() (map[string]interface{}, error) { - return map[string]interface{}{ - "name": "AWS Module", - "documentationHref": "https://cloud.google.com/vertex-ai/docs/generative-ai/embeddings/get-text-embeddings", - }, nil -} diff --git a/modules/text2vec-aws/clients/signer.go b/modules/text2vec-aws/clients/signer.go deleted file mode 100644 index 17ffb5271f6b8598afaf789067aa19cacc7ac7dc..0000000000000000000000000000000000000000 --- a/modules/text2vec-aws/clients/signer.go +++ /dev/null @@ -1,109 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "crypto/hmac" - "crypto/sha256" - "encoding/hex" - "fmt" - "net/http" - "sort" - "strings" - "time" -) - -const ( - contentType = "application/json" - algorithm = "AWS4-HMAC-SHA256" -) - -func getAuthHeader(awsAccessKey string, awsSecretKey string, host string, service string, region string, path string, body []byte, headers map[string]string) (string, map[string]string, string) { - t := time.Now().UTC() - amzDate := t.Format("20060102T150405Z") - shortDate := t.Format("20060102") - - hashedPayload := sha256Hash(body) - canonicalHeaders, signedHeaders := getCanonicalHeaders(headers) - - canonicalRequest := strings.Join([]string{ - http.MethodPost, - path, - "", - canonicalHeaders, - signedHeaders, - hashedPayload, - }, "\n") - - hashedCanonicalRequest := sha256Hash([]byte(canonicalRequest)) - - credentialScope := strings.Join([]string{shortDate, region, service, "aws4_request"}, "/") - - stringToSign := strings.Join([]string{ - algorithm, - amzDate, - credentialScope, - hashedCanonicalRequest, - }, "\n") - - signingKey := getSigningKey(awsSecretKey, shortDate, region, service) - - signature := hmacSHA256(signingKey, stringToSign) - - authorizationHeader := fmt.Sprintf("%s Credential=%s/%s, SignedHeaders=%s, Signature=%s", algorithm, awsAccessKey, credentialScope, signedHeaders, signature) - return amzDate, headers, authorizationHeader -} - -func getCanonicalHeaders(headers map[string]string) (string, string) { - var canonicalHeaders []string - var signedHeaders []string - - keys := make([]string, 0, len(headers)) - for k := range headers { - keys = append(keys, k) - } - - sort.Strings(keys) - - for _, k := range keys { - canonicalHeaders = append(canonicalHeaders, fmt.Sprintf("%s:%s\n", strings.ToLower(k), headers[k])) - signedHeaders = append(signedHeaders, strings.ToLower(k)) - } - - return strings.Join(canonicalHeaders, ""), strings.Join(signedHeaders, ";") -} - -func hmacSHA256(key []byte, message string) string { - mac := hmac.New(sha256.New, key) - mac.Write([]byte(message)) - return hex.EncodeToString(mac.Sum(nil)) -} - -func getSigningKey(secretKey, date, region, service string) []byte { - key := "AWS4" + secretKey - kDate := hmacSHA256Bytes([]byte(key), date) - kRegion := hmacSHA256Bytes(kDate, region) - kService := hmacSHA256Bytes(kRegion, service) - kSigning := hmacSHA256Bytes(kService, "aws4_request") - return kSigning -} - -func hmacSHA256Bytes(key []byte, message string) []byte { - mac := hmac.New(sha256.New, key) - mac.Write([]byte(message)) - return mac.Sum(nil) -} - -func sha256Hash(body []byte) string { - hash := sha256.Sum256(body) - return hex.EncodeToString(hash[:]) -} diff --git a/modules/text2vec-aws/config.go b/modules/text2vec-aws/config.go deleted file mode 100644 index 1d0136a87f939283ab42eeb75711fc57c044a2c7..0000000000000000000000000000000000000000 --- a/modules/text2vec-aws/config.go +++ /dev/null @@ -1,47 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modaws - -import ( - "context" - - "github.com/weaviate/weaviate/modules/text2vec-aws/vectorizer" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/schema" -) - -func (m *AwsModule) ClassConfigDefaults() map[string]interface{} { - return map[string]interface{}{ - "vectorizeClassName": vectorizer.DefaultVectorizeClassName, - } -} - -func (m *AwsModule) PropertyConfigDefaults( - dt *schema.DataType, -) map[string]interface{} { - return map[string]interface{}{ - "skip": !vectorizer.DefaultPropertyIndexed, - "vectorizePropertyName": vectorizer.DefaultVectorizePropertyName, - } -} - -func (m *AwsModule) ValidateClass(ctx context.Context, - class *models.Class, cfg moduletools.ClassConfig, -) error { - settings := vectorizer.NewClassSettings(cfg) - return settings.Validate(class) -} - -var _ = modulecapabilities.ClassConfigurator(New()) diff --git a/modules/text2vec-aws/ent/vectorization_config.go b/modules/text2vec-aws/ent/vectorization_config.go deleted file mode 100644 index 662302ad4109e049de6416874cfc93fc5c7fca63..0000000000000000000000000000000000000000 --- a/modules/text2vec-aws/ent/vectorization_config.go +++ /dev/null @@ -1,21 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package ent - -type VectorizationConfig struct { - Service string - Region string - Model string - Endpoint string - TargetModel string - TargetVariant string -} diff --git a/modules/text2vec-aws/ent/vectorization_result.go b/modules/text2vec-aws/ent/vectorization_result.go deleted file mode 100644 index 69eee4438e46a4427a7368204c662419196cc526..0000000000000000000000000000000000000000 --- a/modules/text2vec-aws/ent/vectorization_result.go +++ /dev/null @@ -1,18 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package ent - -type VectorizationResult struct { - Text string - Dimensions int - Vector []float32 -} diff --git a/modules/text2vec-aws/module.go b/modules/text2vec-aws/module.go deleted file mode 100644 index c2d0ce97f9d566c4b7f90ee421c47685acfdcbbf..0000000000000000000000000000000000000000 --- a/modules/text2vec-aws/module.go +++ /dev/null @@ -1,163 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modaws - -import ( - "context" - "net/http" - "os" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/modules/text2vec-aws/clients" - "github.com/weaviate/weaviate/modules/text2vec-aws/vectorizer" - "github.com/weaviate/weaviate/usecases/modulecomponents/additional" -) - -const Name = "text2vec-aws" - -func New() *AwsModule { - return &AwsModule{} -} - -type AwsModule struct { - vectorizer textVectorizer - metaProvider metaProvider - graphqlProvider modulecapabilities.GraphQLArguments - searcher modulecapabilities.Searcher - nearTextTransformer modulecapabilities.TextTransform - logger logrus.FieldLogger - additionalPropertiesProvider modulecapabilities.AdditionalProperties -} - -type textVectorizer interface { - Object(ctx context.Context, obj *models.Object, objDiff *moduletools.ObjectDiff, - cfg moduletools.ClassConfig) error - Texts(ctx context.Context, input []string, - cfg moduletools.ClassConfig) ([]float32, error) -} - -type metaProvider interface { - MetaInfo() (map[string]interface{}, error) -} - -func (m *AwsModule) Name() string { - return "text2vec-aws" -} - -func (m *AwsModule) Type() modulecapabilities.ModuleType { - return modulecapabilities.Text2Vec -} - -func (m *AwsModule) Init(ctx context.Context, - params moduletools.ModuleInitParams, -) error { - m.logger = params.GetLogger() - - if err := m.initVectorizer(ctx, params.GetConfig().ModuleHttpClientTimeout, m.logger); err != nil { - return errors.Wrap(err, "init vectorizer") - } - - if err := m.initAdditionalPropertiesProvider(); err != nil { - return errors.Wrap(err, "init additional properties provider") - } - - return nil -} - -func (m *AwsModule) InitExtension(modules []modulecapabilities.Module) error { - for _, module := range modules { - if module.Name() == m.Name() { - continue - } - if arg, ok := module.(modulecapabilities.TextTransformers); ok { - if arg != nil && arg.TextTransformers() != nil { - m.nearTextTransformer = arg.TextTransformers()["nearText"] - } - } - } - - if err := m.initNearText(); err != nil { - return errors.Wrap(err, "init graphql provider") - } - return nil -} - -func (m *AwsModule) initVectorizer(ctx context.Context, timeout time.Duration, - logger logrus.FieldLogger, -) error { - awsAccessKey := m.getAWSAccessKey() - awsSecret := m.getAWSSecretAccessKey() - client := clients.New(awsAccessKey, awsSecret, timeout, logger) - - m.vectorizer = vectorizer.New(client) - m.metaProvider = client - - return nil -} - -func (m *AwsModule) getAWSAccessKey() string { - if os.Getenv("AWS_ACCESS_KEY_ID") != "" { - return os.Getenv("AWS_ACCESS_KEY_ID") - } - return os.Getenv("AWS_ACCESS_KEY") -} - -func (m *AwsModule) getAWSSecretAccessKey() string { - if os.Getenv("AWS_SECRET_ACCESS_KEY") != "" { - return os.Getenv("AWS_SECRET_ACCESS_KEY") - } - return os.Getenv("AWS_SECRET_KEY") -} - -func (m *AwsModule) initAdditionalPropertiesProvider() error { - m.additionalPropertiesProvider = additional.NewText2VecProvider() - return nil -} - -func (m *AwsModule) RootHandler() http.Handler { - // TODO: remove once this is a capability interface - return nil -} - -func (m *AwsModule) VectorizeObject(ctx context.Context, - obj *models.Object, objDiff *moduletools.ObjectDiff, cfg moduletools.ClassConfig, -) error { - return m.vectorizer.Object(ctx, obj, objDiff, cfg) -} - -func (m *AwsModule) MetaInfo() (map[string]interface{}, error) { - return m.metaProvider.MetaInfo() -} - -func (m *AwsModule) AdditionalProperties() map[string]modulecapabilities.AdditionalProperty { - return m.additionalPropertiesProvider.AdditionalProperties() -} - -func (m *AwsModule) VectorizeInput(ctx context.Context, - input string, cfg moduletools.ClassConfig, -) ([]float32, error) { - return m.vectorizer.Texts(ctx, []string{input}, cfg) -} - -// verify we implement the modules.Module interface -var ( - _ = modulecapabilities.Module(New()) - _ = modulecapabilities.Vectorizer(New()) - _ = modulecapabilities.MetaProvider(New()) - _ = modulecapabilities.Searcher(New()) - _ = modulecapabilities.GraphQLArguments(New()) -) diff --git a/modules/text2vec-aws/nearText.go b/modules/text2vec-aws/nearText.go deleted file mode 100644 index 241efa07d21476e86ff8f76dc051067469f146e4..0000000000000000000000000000000000000000 --- a/modules/text2vec-aws/nearText.go +++ /dev/null @@ -1,36 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modaws - -import ( - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/usecases/modulecomponents/nearText" -) - -func (m *AwsModule) initNearText() error { - m.searcher = nearText.NewSearcher(m.vectorizer) - m.graphqlProvider = nearText.New(m.nearTextTransformer) - return nil -} - -func (m *AwsModule) Arguments() map[string]modulecapabilities.GraphQLArgument { - return m.graphqlProvider.Arguments() -} - -func (m *AwsModule) VectorSearches() map[string]modulecapabilities.VectorForParams { - return m.searcher.VectorSearches() -} - -var ( - _ = modulecapabilities.GraphQLArguments(New()) - _ = modulecapabilities.Searcher(New()) -) diff --git a/modules/text2vec-aws/vectorizer/class_settings.go b/modules/text2vec-aws/vectorizer/class_settings.go deleted file mode 100644 index a3ec6e147dbbb1b0dab6f44ca474584f98e30499..0000000000000000000000000000000000000000 --- a/modules/text2vec-aws/vectorizer/class_settings.go +++ /dev/null @@ -1,259 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "fmt" - "strings" - - "github.com/pkg/errors" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/schema" -) - -const ( - serviceProperty = "service" - regionProperty = "region" - modelProperty = "model" - endpointProperty = "endpoint" - targetModelProperty = "targetModel" - targetVariantProperty = "targetVariant" -) - -const ( - DefaultVectorizeClassName = false - DefaultPropertyIndexed = true - DefaultVectorizePropertyName = false - DefaultService = "bedrock" -) - -var availableAWSServices = []string{ - "bedrock", -} - -var availableAWSBedrockModels = []string{ - "amazon.titan-embed-text-v1", - "cohere.embed-english-v3", - "cohere.embed-multilingual-v3", -} - -type classSettings struct { - cfg moduletools.ClassConfig -} - -func NewClassSettings(cfg moduletools.ClassConfig) *classSettings { - return &classSettings{cfg: cfg} -} - -func (ic *classSettings) PropertyIndexed(propName string) bool { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return DefaultPropertyIndexed - } - - vcn, ok := ic.cfg.Property(propName)["skip"] - if !ok { - return DefaultPropertyIndexed - } - - asBool, ok := vcn.(bool) - if !ok { - return DefaultPropertyIndexed - } - - return !asBool -} - -func (ic *classSettings) VectorizePropertyName(propName string) bool { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return DefaultVectorizePropertyName - } - vcn, ok := ic.cfg.Property(propName)["vectorizePropertyName"] - if !ok { - return DefaultVectorizePropertyName - } - - asBool, ok := vcn.(bool) - if !ok { - return DefaultVectorizePropertyName - } - - return asBool -} - -func (ic *classSettings) VectorizeClassName() bool { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return DefaultVectorizeClassName - } - - vcn, ok := ic.cfg.Class()["vectorizeClassName"] - if !ok { - return DefaultVectorizeClassName - } - - asBool, ok := vcn.(bool) - if !ok { - return DefaultVectorizeClassName - } - - return asBool -} - -func (ic *classSettings) Validate(class *models.Class) error { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return errors.New("empty config") - } - - var errorMessages []string - - service := ic.Service() - if service == "" || !ic.validatAvailableAWSSetting(service, availableAWSServices) { - errorMessages = append(errorMessages, fmt.Sprintf("wrong %s, available services are: %v", serviceProperty, availableAWSServices)) - } - region := ic.Region() - if region == "" { - errorMessages = append(errorMessages, fmt.Sprintf("%s cannot be empty", regionProperty)) - } - - if isBedrock(service) { - model := ic.Model() - if model == "" || !ic.validatAvailableAWSSetting(model, availableAWSBedrockModels) { - errorMessages = append(errorMessages, fmt.Sprintf("wrong %s, available models are: %v", modelProperty, availableAWSBedrockModels)) - } - endpoint := ic.Endpoint() - if endpoint != "" { - errorMessages = append(errorMessages, fmt.Sprintf("wrong configuration: %s, not applicable to %s", endpoint, service)) - } - } - - if isSagemaker(service) { - endpoint := ic.Endpoint() - if endpoint == "" { - errorMessages = append(errorMessages, fmt.Sprintf("%s cannot be empty", endpointProperty)) - } - model := ic.Model() - if model != "" { - errorMessages = append(errorMessages, fmt.Sprintf("wrong configuration: %s, not applicable to %s. did you mean %s", modelProperty, service, targetModelProperty)) - } - } - - if len(errorMessages) > 0 { - return fmt.Errorf("%s", strings.Join(errorMessages, ", ")) - } - - err := ic.validateIndexState(class, ic) - if err != nil { - return err - } - - return nil -} - -func (ic *classSettings) validatAvailableAWSSetting(value string, availableValues []string) bool { - for i := range availableValues { - if value == availableValues[i] { - return true - } - } - return false -} - -func (ic *classSettings) getStringProperty(name, defaultValue string) string { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return defaultValue - } - - value, ok := ic.cfg.ClassByModuleName("text2vec-aws")[name] - if ok { - asString, ok := value.(string) - if ok { - return asString - } - } - return defaultValue -} - -func (cv *classSettings) validateIndexState(class *models.Class, settings ClassSettings) error { - if settings.VectorizeClassName() { - // if the user chooses to vectorize the classname, vector-building will - // always be possible, no need to investigate further - - return nil - } - - // search if there is at least one indexed, string/text prop. If found pass - // validation - for _, prop := range class.Properties { - if len(prop.DataType) < 1 { - return errors.Errorf("property %s must have at least one datatype: "+ - "got %v", prop.Name, prop.DataType) - } - - if prop.DataType[0] != string(schema.DataTypeString) && - prop.DataType[0] != string(schema.DataTypeText) { - // we can only vectorize text-like props - continue - } - - if settings.PropertyIndexed(prop.Name) { - // found at least one, this is a valid schema - return nil - } - } - - return fmt.Errorf("invalid properties: didn't find a single property which is " + - "of type string or text and is not excluded from indexing. In addition the " + - "class name is excluded from vectorization as well, meaning that it cannot be " + - "used to determine the vector position. To fix this, set 'vectorizeClassName' " + - "to true if the class name is contextionary-valid. Alternatively add at least " + - "contextionary-valid text/string property which is not excluded from " + - "indexing") -} - -// Aws params -func (ic *classSettings) Service() string { - return ic.getStringProperty(serviceProperty, DefaultService) -} - -func (ic *classSettings) Region() string { - return ic.getStringProperty(regionProperty, "") -} - -func (ic *classSettings) Model() string { - return ic.getStringProperty(modelProperty, "") -} - -func (ic *classSettings) Endpoint() string { - return ic.getStringProperty(endpointProperty, "") -} - -func (ic *classSettings) TargetModel() string { - return ic.getStringProperty(targetModelProperty, "") -} - -func (ic *classSettings) TargetVariant() string { - return ic.getStringProperty(targetVariantProperty, "") -} - -func isSagemaker(service string) bool { - return service == "sagemaker" -} - -func isBedrock(service string) bool { - return service == "bedrock" -} diff --git a/modules/text2vec-aws/vectorizer/class_settings_test.go b/modules/text2vec-aws/vectorizer/class_settings_test.go deleted file mode 100644 index 4d7e6b96372714a378593c10f2e3e10f5c9fe011..0000000000000000000000000000000000000000 --- a/modules/text2vec-aws/vectorizer/class_settings_test.go +++ /dev/null @@ -1,109 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "testing" - - "github.com/pkg/errors" - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/entities/moduletools" -) - -func Test_classSettings_Validate(t *testing.T) { - tests := []struct { - name string - cfg moduletools.ClassConfig - wantService string - wantRegion string - wantModel string - wantEndpoint string - wantTargetModel string - wantTargetVariant string - wantErr error - }{ - { - name: "happy flow - Bedrock", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "service": "bedrock", - "region": "us-east-1", - "model": "amazon.titan-embed-text-v1", - }, - }, - wantService: "bedrock", - wantRegion: "us-east-1", - wantModel: "amazon.titan-embed-text-v1", - wantErr: nil, - }, - { - name: "empty service", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "region": "us-east-1", - "model": "amazon.titan-embed-text-v1", - }, - }, - wantService: "bedrock", - wantRegion: "us-east-1", - wantModel: "amazon.titan-embed-text-v1", - }, - { - name: "empty region - Bedrock", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "service": "bedrock", - "model": "amazon.titan-embed-text-v1", - }, - }, - wantErr: errors.Errorf("region cannot be empty"), - }, - { - name: "wrong model", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "service": "bedrock", - "region": "us-west-1", - "model": "wrong-model", - }, - }, - wantErr: errors.Errorf("wrong model, available models are: [amazon.titan-embed-text-v1 cohere.embed-english-v3 cohere.embed-multilingual-v3]"), - }, - { - name: "all wrong", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "service": "", - "region": "", - "model": "", - }, - }, - wantErr: errors.Errorf("wrong service, available services are: [bedrock], " + - "region cannot be empty"), - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ic := NewClassSettings(tt.cfg) - if tt.wantErr != nil { - assert.EqualError(t, ic.Validate(nil), tt.wantErr.Error()) - } else { - assert.Equal(t, tt.wantService, ic.Service()) - assert.Equal(t, tt.wantRegion, ic.Region()) - assert.Equal(t, tt.wantModel, ic.Model()) - assert.Equal(t, tt.wantEndpoint, ic.Endpoint()) - assert.Equal(t, tt.wantTargetModel, ic.TargetModel()) - assert.Equal(t, tt.wantTargetVariant, ic.TargetVariant()) - } - }) - } -} diff --git a/modules/text2vec-aws/vectorizer/fakes_for_test.go b/modules/text2vec-aws/vectorizer/fakes_for_test.go deleted file mode 100644 index 28e83a8d9446313c7a150fc893bd184c41a53530..0000000000000000000000000000000000000000 --- a/modules/text2vec-aws/vectorizer/fakes_for_test.go +++ /dev/null @@ -1,105 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - - "github.com/weaviate/weaviate/modules/text2vec-aws/ent" -) - -type fakeClient struct { - lastInput []string - lastConfig ent.VectorizationConfig -} - -func (c *fakeClient) Vectorize(ctx context.Context, - text []string, cfg ent.VectorizationConfig, -) (*ent.VectorizationResult, error) { - c.lastInput = text - c.lastConfig = cfg - return &ent.VectorizationResult{ - Vector: []float32{0, 1, 2, 3}, - Dimensions: 4, - Text: text[0], - }, nil -} - -func (c *fakeClient) VectorizeQuery(ctx context.Context, - text []string, cfg ent.VectorizationConfig, -) (*ent.VectorizationResult, error) { - c.lastInput = text - c.lastConfig = cfg - return &ent.VectorizationResult{ - Vector: []float32{0.1, 1.1, 2.1, 3.1}, - Dimensions: 4, - Text: text[0], - }, nil -} - -type fakeClassConfig struct { - classConfig map[string]interface{} - vectorizeClassName bool - vectorizePropertyName bool - skippedProperty string - excludedProperty string - // module specific settings - service string - region string - model string - endpoint string - targetModel string - targetVariant string -} - -func (f fakeClassConfig) Class() map[string]interface{} { - if len(f.classConfig) > 0 { - return f.classConfig - } - classSettings := map[string]interface{}{ - "vectorizeClassName": f.vectorizeClassName, - "service": f.service, - "region": f.region, - "model": f.model, - "endpoint": f.endpoint, - "targetModel": f.targetModel, - "targetVariant": f.targetVariant, - } - return classSettings -} - -func (f fakeClassConfig) ClassByModuleName(moduleName string) map[string]interface{} { - return f.classConfig -} - -func (f fakeClassConfig) Property(propName string) map[string]interface{} { - if propName == f.skippedProperty { - return map[string]interface{}{ - "skip": true, - } - } - if propName == f.excludedProperty { - return map[string]interface{}{ - "vectorizePropertyName": false, - } - } - if f.vectorizePropertyName { - return map[string]interface{}{ - "vectorizePropertyName": true, - } - } - return nil -} - -func (f fakeClassConfig) Tenant() string { - return "" -} diff --git a/modules/text2vec-aws/vectorizer/objects.go b/modules/text2vec-aws/vectorizer/objects.go deleted file mode 100644 index 54c45442af7a8df32a71bc7a3a78eb3de8443e9e..0000000000000000000000000000000000000000 --- a/modules/text2vec-aws/vectorizer/objects.go +++ /dev/null @@ -1,93 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/modules/text2vec-aws/ent" - objectsvectorizer "github.com/weaviate/weaviate/usecases/modulecomponents/vectorizer" -) - -type Vectorizer struct { - client Client - objectVectorizer *objectsvectorizer.ObjectVectorizer -} - -func New(client Client) *Vectorizer { - return &Vectorizer{ - client: client, - objectVectorizer: objectsvectorizer.New(), - } -} - -type Client interface { - Vectorize(ctx context.Context, input []string, - config ent.VectorizationConfig) (*ent.VectorizationResult, error) - VectorizeQuery(ctx context.Context, input []string, - config ent.VectorizationConfig) (*ent.VectorizationResult, error) -} - -// IndexCheck returns whether a property of a class should be indexed -type ClassSettings interface { - PropertyIndexed(property string) bool - VectorizePropertyName(propertyName string) bool - VectorizeClassName() bool - Service() string - Region() string - Model() string - Endpoint() string - TargetModel() string - TargetVariant() string -} - -func (v *Vectorizer) Object(ctx context.Context, object *models.Object, - objDiff *moduletools.ObjectDiff, cfg moduletools.ClassConfig, -) error { - vec, err := v.object(ctx, object.Class, object.Properties, objDiff, cfg) - if err != nil { - return err - } - - object.Vector = vec - return nil -} - -func (v *Vectorizer) object(ctx context.Context, className string, - schema interface{}, objDiff *moduletools.ObjectDiff, cfg moduletools.ClassConfig, -) ([]float32, error) { - text, vector, err := v.objectVectorizer.TextsOrVector(ctx, className, schema, objDiff, NewClassSettings(cfg)) - if err != nil { - return nil, err - } - if vector != nil { - // dont' re-vectorize - return vector, nil - } - // vectorize text - icheck := NewClassSettings(cfg) - res, err := v.client.Vectorize(ctx, []string{text}, ent.VectorizationConfig{ - Service: icheck.Service(), - Region: icheck.Region(), - Model: icheck.Model(), - Endpoint: icheck.Endpoint(), - TargetModel: icheck.TargetModel(), - TargetVariant: icheck.TargetVariant(), - }) - if err != nil { - return nil, err - } - - return res.Vector, nil -} diff --git a/modules/text2vec-aws/vectorizer/objects_test.go b/modules/text2vec-aws/vectorizer/objects_test.go deleted file mode 100644 index 7996d43a5de8d461705768b346ed660587b01e14..0000000000000000000000000000000000000000 --- a/modules/text2vec-aws/vectorizer/objects_test.go +++ /dev/null @@ -1,362 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/moduletools" -) - -// These are mostly copy/pasted (with minimal additions) from the -// text2vec-contextionary module -func TestVectorizingObjects(t *testing.T) { - type testCase struct { - name string - input *models.Object - expectedClientCall string - noindex string - excludedProperty string // to simulate a schema where property names aren't vectorized - excludedClass string // to simulate a schema where class names aren't vectorized - awsModel string - } - - tests := []testCase{ - { - name: "empty object", - input: &models.Object{ - Class: "Car", - }, - awsModel: "large", - expectedClientCall: "car", - }, - { - name: "object with one string prop", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "Mercedes", - }, - }, - expectedClientCall: "car brand mercedes", - }, - { - name: "object with one non-string prop", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "power": 300, - }, - }, - expectedClientCall: "car", - }, - { - name: "object with a mix of props", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: "car brand best brand review a very great car", - }, - { - name: "with a noindexed property", - noindex: "review", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: "car brand best brand", - }, - { - name: "with the class name not vectorized", - excludedClass: "Car", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: "brand best brand review a very great car", - }, - { - name: "with a property name not vectorized", - excludedProperty: "review", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: "car brand best brand a very great car", - }, - { - name: "with no schema labels vectorized", - excludedProperty: "review", - excludedClass: "Car", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "review": "a very great car", - }, - }, - expectedClientCall: "a very great car", - }, - { - name: "with string/text arrays without propname or classname", - excludedProperty: "reviews", - excludedClass: "Car", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - expectedClientCall: "a very great car you should consider buying one", - }, - { - name: "with string/text arrays with propname and classname", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - expectedClientCall: "car reviews a very great car reviews you should consider buying one", - }, - { - name: "with compound class and prop names", - input: &models.Object{ - Class: "SuperCar", - Properties: map[string]interface{}{ - "brandOfTheCar": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: "super car brand of the car best brand review a very great car", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - client := &fakeClient{} - - v := New(client) - - ic := &fakeClassConfig{ - skippedProperty: test.noindex, - vectorizeClassName: test.excludedClass != "Car", - excludedProperty: test.excludedProperty, - service: "", - region: "", - model: "", - endpoint: "", - targetModel: "", - targetVariant: "", - vectorizePropertyName: true, - } - err := v.Object(context.Background(), test.input, nil, ic) - - require.Nil(t, err) - assert.Equal(t, models.C11yVector{0, 1, 2, 3}, test.input.Vector) - expected := strings.Split(test.expectedClientCall, " ") - actual := strings.Split(client.lastInput[0], " ") - assert.Equal(t, expected, actual) - }) - } -} - -func TestVectorizingObjectsWithDiff(t *testing.T) { - type testCase struct { - name string - input *models.Object - skipped string - diff *moduletools.ObjectDiff - expectedVectorize bool - } - - tests := []testCase{ - { - name: "no diff", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: nil, - expectedVectorize: true, - }, - { - name: "diff all props unchanged", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("brand", "best brand", "best brand"). - WithProp("power", 300, 300). - WithProp("description", "a very great car", "a very great car"). - WithProp("reviews", []interface{}{ - "a very great car", - "you should consider buying one", - }, []interface{}{ - "a very great car", - "you should consider buying one", - }), - expectedVectorize: false, - }, - { - name: "diff one vectorizable prop changed (1)", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("brand", "old best brand", "best brand"), - expectedVectorize: true, - }, - { - name: "diff one vectorizable prop changed (2)", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("description", "old a very great car", "a very great car"), - expectedVectorize: true, - }, - { - name: "diff one vectorizable prop changed (3)", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("reviews", []interface{}{ - "old a very great car", - "you should consider buying one", - }, []interface{}{ - "a very great car", - "you should consider buying one", - }), - expectedVectorize: true, - }, - { - name: "all non-vectorizable props changed", - skipped: "description", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("power", 123, 300). - WithProp("description", "old a very great car", "a very great car"), - expectedVectorize: false, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - ic := &fakeClassConfig{ - skippedProperty: test.skipped, - } - - client := &fakeClient{} - v := New(client) - - err := v.Object(context.Background(), test.input, test.diff, ic) - - require.Nil(t, err) - if test.expectedVectorize { - assert.Equal(t, models.C11yVector{0, 1, 2, 3}, test.input.Vector) - assert.NotEmpty(t, client.lastInput) - } else { - assert.Equal(t, models.C11yVector{0, 0, 0, 0}, test.input.Vector) - assert.Empty(t, client.lastInput) - } - }) - } -} - -func newObjectDiffWithVector() *moduletools.ObjectDiff { - return moduletools.NewObjectDiff([]float32{0, 0, 0, 0}) -} diff --git a/modules/text2vec-aws/vectorizer/texts.go b/modules/text2vec-aws/vectorizer/texts.go deleted file mode 100644 index 6677bfcf47561447f4eb082ef9342b44832ba3a0..0000000000000000000000000000000000000000 --- a/modules/text2vec-aws/vectorizer/texts.go +++ /dev/null @@ -1,44 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/modules/text2vec-aws/ent" - libvectorizer "github.com/weaviate/weaviate/usecases/vectorizer" -) - -func (v *Vectorizer) Texts(ctx context.Context, inputs []string, - cfg moduletools.ClassConfig, -) ([]float32, error) { - settings := NewClassSettings(cfg) - vectors := make([][]float32, len(inputs)) - for i := range inputs { - res, err := v.client.VectorizeQuery(ctx, []string{inputs[i]}, ent.VectorizationConfig{ - Service: settings.Service(), - Region: settings.Region(), - Model: settings.Model(), - Endpoint: settings.Endpoint(), - TargetModel: settings.TargetModel(), - TargetVariant: settings.TargetVariant(), - }) - if err != nil { - return nil, errors.Wrap(err, "remote client vectorize") - } - vectors[i] = res.Vector - } - - return libvectorizer.CombineVectors(vectors), nil -} diff --git a/modules/text2vec-aws/vectorizer/texts_test.go b/modules/text2vec-aws/vectorizer/texts_test.go deleted file mode 100644 index 00cc40a846d11c4f29ae7b5883d82aac97797a2c..0000000000000000000000000000000000000000 --- a/modules/text2vec-aws/vectorizer/texts_test.go +++ /dev/null @@ -1,100 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// as used in the nearText searcher -func TestVectorizingTexts(t *testing.T) { - type testCase struct { - name string - input []string - expectedClientCall string - expectedService string - expectedRegion string - expectedModel string - } - - tests := []testCase{ - { - name: "single word", - input: []string{"hello"}, - expectedClientCall: "hello", - expectedService: "bedrock", - }, - { - name: "multiple words", - input: []string{"hello world, this is me!"}, - expectedClientCall: "hello world, this is me!", - expectedService: "bedrock", - }, - { - name: "multiple sentences (joined with a dot)", - input: []string{"this is sentence 1", "and here's number 2"}, - expectedClientCall: "and here's number 2", - expectedService: "bedrock", - }, - { - name: "multiple sentences already containing a dot", - input: []string{"this is sentence 1.", "and here's number 2"}, - expectedClientCall: "and here's number 2", - expectedService: "bedrock", - }, - { - name: "multiple sentences already containing a question mark", - input: []string{"this is sentence 1?", "and here's number 2"}, - expectedClientCall: "and here's number 2", - expectedService: "bedrock", - }, - { - name: "multiple sentences already containing an exclamation mark", - input: []string{"this is sentence 1!", "and here's number 2"}, - expectedClientCall: "and here's number 2", - expectedService: "bedrock", - }, - { - name: "multiple sentences already containing comma", - input: []string{"this is sentence 1,", "and here's number 2"}, - expectedClientCall: "and here's number 2", - expectedService: "bedrock", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - client := &fakeClient{} - - v := New(client) - - settings := &fakeClassConfig{ - service: "bedrock", - region: "", - model: "", - } - vec, err := v.Texts(context.Background(), test.input, settings) - - require.Nil(t, err) - assert.Equal(t, []float32{0.1, 1.1, 2.1, 3.1}, vec) - assert.Equal(t, test.expectedClientCall, strings.Join(client.lastInput, ",")) - assert.Equal(t, test.expectedService, client.lastConfig.Service) - assert.Equal(t, test.expectedRegion, client.lastConfig.Region) - assert.Equal(t, test.expectedModel, client.lastConfig.Model) - }) - } -} diff --git a/modules/text2vec-cohere/clients/cohere.go b/modules/text2vec-cohere/clients/cohere.go deleted file mode 100644 index 1fc83922b91981f56a1e979f8480c28a9d8980f1..0000000000000000000000000000000000000000 --- a/modules/text2vec-cohere/clients/cohere.go +++ /dev/null @@ -1,176 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "time" - - "github.com/weaviate/weaviate/usecases/modulecomponents" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/modules/text2vec-cohere/ent" -) - -type embeddingsRequest struct { - Texts []string `json:"texts"` - Model string `json:"model,omitempty"` - Truncate string `json:"truncate,omitempty"` - InputType inputType `json:"input_type,omitempty"` -} - -type embeddingsResponse struct { - Embeddings [][]float32 `json:"embeddings,omitempty"` - Message string `json:"message,omitempty"` -} - -type vectorizer struct { - apiKey string - httpClient *http.Client - urlBuilder *cohereUrlBuilder - logger logrus.FieldLogger -} - -type inputType string - -const ( - searchDocument inputType = "search_document" - searchQuery inputType = "search_query" -) - -func New(apiKey string, timeout time.Duration, logger logrus.FieldLogger) *vectorizer { - return &vectorizer{ - apiKey: apiKey, - httpClient: &http.Client{ - Timeout: timeout, - }, - urlBuilder: newCohereUrlBuilder(), - logger: logger, - } -} - -func (v *vectorizer) Vectorize(ctx context.Context, input []string, - config ent.VectorizationConfig, -) (*ent.VectorizationResult, error) { - return v.vectorize(ctx, input, config.Model, config.Truncate, config.BaseURL, searchDocument) -} - -func (v *vectorizer) VectorizeQuery(ctx context.Context, input []string, - config ent.VectorizationConfig, -) (*ent.VectorizationResult, error) { - return v.vectorize(ctx, input, config.Model, config.Truncate, config.BaseURL, searchQuery) -} - -func (v *vectorizer) vectorize(ctx context.Context, input []string, - model, truncate, baseURL string, inputType inputType, -) (*ent.VectorizationResult, error) { - body, err := json.Marshal(embeddingsRequest{ - Texts: input, - Model: model, - Truncate: truncate, - InputType: inputType, - }) - if err != nil { - return nil, errors.Wrapf(err, "marshal body") - } - - url := v.getCohereUrl(ctx, baseURL) - req, err := http.NewRequestWithContext(ctx, "POST", url, - bytes.NewReader(body)) - if err != nil { - return nil, errors.Wrap(err, "create POST request") - } - apiKey, err := v.getApiKey(ctx) - if err != nil { - return nil, errors.Wrapf(err, "Cohere API Key") - } - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", apiKey)) - req.Header.Add("Content-Type", "application/json") - - res, err := v.httpClient.Do(req) - if err != nil { - return nil, errors.Wrap(err, "send POST request") - } - defer res.Body.Close() - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - return nil, errors.Wrap(err, "read response body") - } - var resBody embeddingsResponse - if err := json.Unmarshal(bodyBytes, &resBody); err != nil { - return nil, errors.Wrap(err, "unmarshal response body") - } - - if res.StatusCode >= 500 { - errorMessage := getErrorMessage(res.StatusCode, resBody.Message, "connection to Cohere failed with status: %d error: %v") - return nil, errors.Errorf(errorMessage) - } else if res.StatusCode > 200 { - errorMessage := getErrorMessage(res.StatusCode, resBody.Message, "failed with status: %d error: %v") - return nil, errors.Errorf(errorMessage) - } - - if len(resBody.Embeddings) == 0 { - return nil, errors.Errorf("empty embeddings response") - } - - return &ent.VectorizationResult{ - Text: input, - Dimensions: len(resBody.Embeddings[0]), - Vectors: resBody.Embeddings, - }, nil -} - -func (v *vectorizer) getCohereUrl(ctx context.Context, baseURL string) string { - passedBaseURL := baseURL - if headerBaseURL := v.getValueFromContext(ctx, "X-Cohere-Baseurl"); headerBaseURL != "" { - passedBaseURL = headerBaseURL - } - return v.urlBuilder.url(passedBaseURL) -} - -func getErrorMessage(statusCode int, resBodyError string, errorTemplate string) string { - if resBodyError != "" { - return fmt.Sprintf(errorTemplate, statusCode, resBodyError) - } - return fmt.Sprintf(errorTemplate, statusCode) -} - -func (v *vectorizer) getValueFromContext(ctx context.Context, key string) string { - if value := ctx.Value(key); value != nil { - if keyHeader, ok := value.([]string); ok && len(keyHeader) > 0 && len(keyHeader[0]) > 0 { - return keyHeader[0] - } - } - // try getting header from GRPC if not successful - if apiKey := modulecomponents.GetValueFromGRPC(ctx, key); len(apiKey) > 0 && len(apiKey[0]) > 0 { - return apiKey[0] - } - return "" -} - -func (v *vectorizer) getApiKey(ctx context.Context) (string, error) { - if apiKey := v.getValueFromContext(ctx, "X-Cohere-Api-Key"); apiKey != "" { - return apiKey, nil - } - if v.apiKey != "" { - return v.apiKey, nil - } - return "", errors.New("no api key found " + - "neither in request header: X-Cohere-Api-Key " + - "nor in environment variable under COHERE_APIKEY") -} diff --git a/modules/text2vec-cohere/clients/cohere_test.go b/modules/text2vec-cohere/clients/cohere_test.go deleted file mode 100644 index 5b46e1d118e284f2e1b9367f509d5b56c720d924..0000000000000000000000000000000000000000 --- a/modules/text2vec-cohere/clients/cohere_test.go +++ /dev/null @@ -1,250 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "context" - "encoding/json" - "io" - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/modules/text2vec-cohere/ent" -) - -func TestClient(t *testing.T) { - t.Run("when all is fine", func(t *testing.T) { - server := httptest.NewServer(&fakeHandler{t: t}) - defer server.Close() - c := &vectorizer{ - apiKey: "apiKey", - httpClient: &http.Client{}, - urlBuilder: &cohereUrlBuilder{ - origin: server.URL, - pathMask: "/v1/embed", - }, - logger: nullLogger(), - } - expected := &ent.VectorizationResult{ - Text: []string{"This is my text"}, - Vectors: [][]float32{{0.1, 0.2, 0.3}}, - Dimensions: 3, - } - res, err := c.Vectorize(context.Background(), []string{"This is my text"}, - ent.VectorizationConfig{ - Model: "large", - }) - - assert.Nil(t, err) - assert.Equal(t, expected, res) - }) - - t.Run("when the context is expired", func(t *testing.T) { - server := httptest.NewServer(&fakeHandler{t: t}) - defer server.Close() - c := &vectorizer{ - apiKey: "apiKey", - httpClient: &http.Client{}, - urlBuilder: &cohereUrlBuilder{ - origin: server.URL, - pathMask: "/v1/embed", - }, - logger: nullLogger(), - } - ctx, cancel := context.WithDeadline(context.Background(), time.Now()) - defer cancel() - - _, err := c.Vectorize(ctx, []string{"This is my text"}, ent.VectorizationConfig{}) - - require.NotNil(t, err) - assert.Contains(t, err.Error(), "context deadline exceeded") - }) - - t.Run("when the server returns an error", func(t *testing.T) { - server := httptest.NewServer(&fakeHandler{ - t: t, - serverError: errors.Errorf("nope, not gonna happen"), - }) - defer server.Close() - c := &vectorizer{ - apiKey: "apiKey", - httpClient: &http.Client{}, - urlBuilder: &cohereUrlBuilder{ - origin: server.URL, - pathMask: "/v1/embed", - }, - logger: nullLogger(), - } - _, err := c.Vectorize(context.Background(), []string{"This is my text"}, - ent.VectorizationConfig{}) - - require.NotNil(t, err) - assert.Equal(t, err.Error(), "connection to Cohere failed with status: 500 error: nope, not gonna happen") - }) - - t.Run("when Cohere key is passed using X-Cohere-Api-Key header", func(t *testing.T) { - server := httptest.NewServer(&fakeHandler{t: t}) - defer server.Close() - c := &vectorizer{ - apiKey: "", - httpClient: &http.Client{}, - urlBuilder: &cohereUrlBuilder{ - origin: server.URL, - pathMask: "/v1/embed", - }, - logger: nullLogger(), - } - ctxWithValue := context.WithValue(context.Background(), - "X-Cohere-Api-Key", []string{"some-key"}) - - expected := &ent.VectorizationResult{ - Text: []string{"This is my text"}, - Vectors: [][]float32{{0.1, 0.2, 0.3}}, - Dimensions: 3, - } - res, err := c.Vectorize(ctxWithValue, []string{"This is my text"}, - ent.VectorizationConfig{ - Model: "large", - }) - - require.Nil(t, err) - assert.Equal(t, expected, res) - }) - - t.Run("when Cohere key is empty", func(t *testing.T) { - server := httptest.NewServer(&fakeHandler{t: t}) - defer server.Close() - c := &vectorizer{ - apiKey: "", - httpClient: &http.Client{}, - urlBuilder: &cohereUrlBuilder{ - origin: server.URL, - pathMask: "/v1/embed", - }, - logger: nullLogger(), - } - ctx, cancel := context.WithDeadline(context.Background(), time.Now()) - defer cancel() - - _, err := c.Vectorize(ctx, []string{"This is my text"}, ent.VectorizationConfig{}) - - require.NotNil(t, err) - assert.Equal(t, err.Error(), "Cohere API Key: no api key found "+ - "neither in request header: X-Cohere-Api-Key "+ - "nor in environment variable under COHERE_APIKEY") - }) - - t.Run("when X-Cohere-Api-Key header is passed but empty", func(t *testing.T) { - server := httptest.NewServer(&fakeHandler{t: t}) - defer server.Close() - c := &vectorizer{ - apiKey: "", - httpClient: &http.Client{}, - urlBuilder: &cohereUrlBuilder{ - origin: server.URL, - pathMask: "/v1/embed", - }, - logger: nullLogger(), - } - ctxWithValue := context.WithValue(context.Background(), - "X-Cohere-Api-Key", []string{""}) - - _, err := c.Vectorize(ctxWithValue, []string{"This is my text"}, - ent.VectorizationConfig{ - Model: "large", - }) - - require.NotNil(t, err) - assert.Equal(t, err.Error(), "Cohere API Key: no api key found "+ - "neither in request header: X-Cohere-Api-Key "+ - "nor in environment variable under COHERE_APIKEY") - }) - - t.Run("when X-Cohere-BaseURL header is passed", func(t *testing.T) { - server := httptest.NewServer(&fakeHandler{t: t}) - defer server.Close() - c := &vectorizer{ - apiKey: "", - httpClient: &http.Client{}, - urlBuilder: &cohereUrlBuilder{ - origin: server.URL, - pathMask: "/v1/embed", - }, - logger: nullLogger(), - } - - baseURL := "http://default-url.com" - ctxWithValue := context.WithValue(context.Background(), - "X-Cohere-Baseurl", []string{"http://base-url-passed-in-header.com"}) - - buildURL := c.getCohereUrl(ctxWithValue, baseURL) - assert.Equal(t, "http://base-url-passed-in-header.com/v1/embed", buildURL) - - buildURL = c.getCohereUrl(context.TODO(), baseURL) - assert.Equal(t, "http://default-url.com/v1/embed", buildURL) - }) -} - -type fakeHandler struct { - t *testing.T - serverError error -} - -func (f *fakeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - assert.Equal(f.t, http.MethodPost, r.Method) - - if f.serverError != nil { - embeddingError := map[string]interface{}{ - "message": f.serverError.Error(), - "type": "invalid_request_error", - } - embeddingResponse := map[string]interface{}{ - "message": embeddingError["message"], - } - outBytes, err := json.Marshal(embeddingResponse) - require.Nil(f.t, err) - - w.WriteHeader(http.StatusInternalServerError) - w.Write(outBytes) - return - } - - bodyBytes, err := io.ReadAll(r.Body) - require.Nil(f.t, err) - defer r.Body.Close() - - var b map[string]interface{} - require.Nil(f.t, json.Unmarshal(bodyBytes, &b)) - - textInput := b["texts"].([]interface{}) - assert.Greater(f.t, len(textInput), 0) - - embeddingResponse := map[string]interface{}{ - "embeddings": [][]float32{{0.1, 0.2, 0.3}}, - } - outBytes, err := json.Marshal(embeddingResponse) - require.Nil(f.t, err) - - w.Write(outBytes) -} - -func nullLogger() logrus.FieldLogger { - l, _ := test.NewNullLogger() - return l -} diff --git a/modules/text2vec-cohere/clients/meta.go b/modules/text2vec-cohere/clients/meta.go deleted file mode 100644 index 3672ebd029323345135bbb967bf10d42ddae3ad3..0000000000000000000000000000000000000000 --- a/modules/text2vec-cohere/clients/meta.go +++ /dev/null @@ -1,19 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -func (v *vectorizer) MetaInfo() (map[string]interface{}, error) { - return map[string]interface{}{ - "name": "Cohere Module", - "documentationHref": "https://docs.cohere.ai/embedding-wiki/", - }, nil -} diff --git a/modules/text2vec-cohere/clients/url.go b/modules/text2vec-cohere/clients/url.go deleted file mode 100644 index f2f1b9eb0917f3bc30afa4c5411802cfe25f41dc..0000000000000000000000000000000000000000 --- a/modules/text2vec-cohere/clients/url.go +++ /dev/null @@ -1,33 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import "fmt" - -type cohereUrlBuilder struct { - origin string - pathMask string -} - -func newCohereUrlBuilder() *cohereUrlBuilder { - return &cohereUrlBuilder{ - origin: "https://api.cohere.ai", - pathMask: "/v1/embed", - } -} - -func (c *cohereUrlBuilder) url(baseURL string) string { - if baseURL != "" { - return fmt.Sprintf("%s%s", baseURL, c.pathMask) - } - return fmt.Sprintf("%s%s", c.origin, c.pathMask) -} diff --git a/modules/text2vec-cohere/config.go b/modules/text2vec-cohere/config.go deleted file mode 100644 index 840d83a7bf00d27478aeabd213fd106853359204..0000000000000000000000000000000000000000 --- a/modules/text2vec-cohere/config.go +++ /dev/null @@ -1,46 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modcohere - -import ( - "context" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/modules/text2vec-cohere/vectorizer" -) - -func (m *CohereModule) ClassConfigDefaults() map[string]interface{} { - return map[string]interface{}{ - "vectorizeClassName": vectorizer.DefaultVectorizeClassName, - } -} - -func (m *CohereModule) PropertyConfigDefaults( - dt *schema.DataType, -) map[string]interface{} { - return map[string]interface{}{ - "skip": !vectorizer.DefaultPropertyIndexed, - "vectorizePropertyName": vectorizer.DefaultVectorizePropertyName, - } -} - -func (m *CohereModule) ValidateClass(ctx context.Context, - class *models.Class, cfg moduletools.ClassConfig, -) error { - settings := vectorizer.NewClassSettings(cfg) - return settings.Validate(class) -} - -var _ = modulecapabilities.ClassConfigurator(New()) diff --git a/modules/text2vec-cohere/ent/vectorization_config.go b/modules/text2vec-cohere/ent/vectorization_config.go deleted file mode 100644 index b89a772962300421cd52ddbfa55f00409eaf70a3..0000000000000000000000000000000000000000 --- a/modules/text2vec-cohere/ent/vectorization_config.go +++ /dev/null @@ -1,18 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package ent - -type VectorizationConfig struct { - Model string - Truncate string - BaseURL string -} diff --git a/modules/text2vec-cohere/ent/vectorization_result.go b/modules/text2vec-cohere/ent/vectorization_result.go deleted file mode 100644 index 08574ffa2b38050f165c47fa6fbebd4d0042cc49..0000000000000000000000000000000000000000 --- a/modules/text2vec-cohere/ent/vectorization_result.go +++ /dev/null @@ -1,18 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package ent - -type VectorizationResult struct { - Text []string - Dimensions int - Vectors [][]float32 -} diff --git a/modules/text2vec-cohere/module.go b/modules/text2vec-cohere/module.go deleted file mode 100644 index bebdc1b17fd67ea4d23f25186b07d82faa72f5d8..0000000000000000000000000000000000000000 --- a/modules/text2vec-cohere/module.go +++ /dev/null @@ -1,149 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modcohere - -import ( - "context" - "net/http" - "os" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/modules/text2vec-cohere/clients" - "github.com/weaviate/weaviate/modules/text2vec-cohere/vectorizer" - "github.com/weaviate/weaviate/usecases/modulecomponents/additional" -) - -const Name = "text2vec-cohere" - -func New() *CohereModule { - return &CohereModule{} -} - -type CohereModule struct { - vectorizer textVectorizer - metaProvider metaProvider - graphqlProvider modulecapabilities.GraphQLArguments - searcher modulecapabilities.Searcher - nearTextTransformer modulecapabilities.TextTransform - logger logrus.FieldLogger - additionalPropertiesProvider modulecapabilities.AdditionalProperties -} - -type textVectorizer interface { - Object(ctx context.Context, obj *models.Object, objDiff *moduletools.ObjectDiff, - cfg moduletools.ClassConfig) error - Texts(ctx context.Context, input []string, - cfg moduletools.ClassConfig) ([]float32, error) -} - -type metaProvider interface { - MetaInfo() (map[string]interface{}, error) -} - -func (m *CohereModule) Name() string { - return Name -} - -func (m *CohereModule) Type() modulecapabilities.ModuleType { - return modulecapabilities.Text2MultiVec -} - -func (m *CohereModule) Init(ctx context.Context, - params moduletools.ModuleInitParams, -) error { - m.logger = params.GetLogger() - - if err := m.initVectorizer(ctx, params.GetConfig().ModuleHttpClientTimeout, m.logger); err != nil { - return errors.Wrap(err, "init vectorizer") - } - - if err := m.initAdditionalPropertiesProvider(); err != nil { - return errors.Wrap(err, "init additional properties provider") - } - - return nil -} - -func (m *CohereModule) InitExtension(modules []modulecapabilities.Module) error { - for _, module := range modules { - if module.Name() == m.Name() { - continue - } - if arg, ok := module.(modulecapabilities.TextTransformers); ok { - if arg != nil && arg.TextTransformers() != nil { - m.nearTextTransformer = arg.TextTransformers()["nearText"] - } - } - } - - if err := m.initNearText(); err != nil { - return errors.Wrap(err, "init graphql provider") - } - return nil -} - -func (m *CohereModule) initVectorizer(ctx context.Context, timeout time.Duration, - logger logrus.FieldLogger, -) error { - apiKey := os.Getenv("COHERE_APIKEY") - client := clients.New(apiKey, timeout, logger) - - m.vectorizer = vectorizer.New(client) - m.metaProvider = client - - return nil -} - -func (m *CohereModule) initAdditionalPropertiesProvider() error { - m.additionalPropertiesProvider = additional.NewText2VecProvider() - return nil -} - -func (m *CohereModule) RootHandler() http.Handler { - // TODO: remove once this is a capability interface - return nil -} - -func (m *CohereModule) VectorizeObject(ctx context.Context, - obj *models.Object, objDiff *moduletools.ObjectDiff, cfg moduletools.ClassConfig, -) error { - return m.vectorizer.Object(ctx, obj, objDiff, cfg) -} - -func (m *CohereModule) MetaInfo() (map[string]interface{}, error) { - return m.metaProvider.MetaInfo() -} - -func (m *CohereModule) VectorizeInput(ctx context.Context, - input string, cfg moduletools.ClassConfig, -) ([]float32, error) { - return m.vectorizer.Texts(ctx, []string{input}, cfg) -} - -func (m *CohereModule) AdditionalProperties() map[string]modulecapabilities.AdditionalProperty { - return m.additionalPropertiesProvider.AdditionalProperties() -} - -// verify we implement the modules.Module interface -var ( - _ = modulecapabilities.Module(New()) - _ = modulecapabilities.Vectorizer(New()) - _ = modulecapabilities.MetaProvider(New()) - _ = modulecapabilities.Searcher(New()) - _ = modulecapabilities.GraphQLArguments(New()) - _ = modulecapabilities.InputVectorizer(New()) -) diff --git a/modules/text2vec-cohere/nearText.go b/modules/text2vec-cohere/nearText.go deleted file mode 100644 index 21e59bab191d794123f1f9680a1a674013d5f682..0000000000000000000000000000000000000000 --- a/modules/text2vec-cohere/nearText.go +++ /dev/null @@ -1,36 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modcohere - -import ( - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/usecases/modulecomponents/nearText" -) - -func (m *CohereModule) initNearText() error { - m.searcher = nearText.NewSearcher(m.vectorizer) - m.graphqlProvider = nearText.New(m.nearTextTransformer) - return nil -} - -func (m *CohereModule) Arguments() map[string]modulecapabilities.GraphQLArgument { - return m.graphqlProvider.Arguments() -} - -func (m *CohereModule) VectorSearches() map[string]modulecapabilities.VectorForParams { - return m.searcher.VectorSearches() -} - -var ( - _ = modulecapabilities.GraphQLArguments(New()) - _ = modulecapabilities.Searcher(New()) -) diff --git a/modules/text2vec-cohere/vectorizer/class_settings.go b/modules/text2vec-cohere/vectorizer/class_settings.go deleted file mode 100644 index 53b5a4f90989a63885eb355716087b49faf3f902..0000000000000000000000000000000000000000 --- a/modules/text2vec-cohere/vectorizer/class_settings.go +++ /dev/null @@ -1,208 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "fmt" - "strings" - - "github.com/pkg/errors" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/schema" -) - -const ( - DefaultBaseURL = "https://api.cohere.ai" - DefaultCohereModel = "embed-multilingual-v3.0" - DefaultTruncate = "END" - DefaultVectorizeClassName = true - DefaultPropertyIndexed = true - DefaultVectorizePropertyName = false -) - -var ( - availableCohereModels = []string{ - "medium", - "large", "small", "multilingual-22-12", - "embed-english-v2.0", "embed-english-light-v2.0", "embed-multilingual-v2.0", - "embed-english-v3.0", "embed-english-light-v3.0", "embed-multilingual-v3.0", "embed-multilingual-light-v3.0", - } - experimetnalCohereModels = []string{"multilingual-2210-alpha"} - availableTruncates = []string{"NONE", "START", "END", "LEFT", "RIGHT"} -) - -type classSettings struct { - cfg moduletools.ClassConfig -} - -func NewClassSettings(cfg moduletools.ClassConfig) *classSettings { - return &classSettings{cfg: cfg} -} - -func (cs *classSettings) PropertyIndexed(propName string) bool { - if cs.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return DefaultPropertyIndexed - } - - vcn, ok := cs.cfg.Property(propName)["skip"] - if !ok { - return DefaultPropertyIndexed - } - - asBool, ok := vcn.(bool) - if !ok { - return DefaultPropertyIndexed - } - - return !asBool -} - -func (cs *classSettings) VectorizePropertyName(propName string) bool { - if cs.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return DefaultVectorizePropertyName - } - vcn, ok := cs.cfg.Property(propName)["vectorizePropertyName"] - if !ok { - return DefaultVectorizePropertyName - } - - asBool, ok := vcn.(bool) - if !ok { - return DefaultVectorizePropertyName - } - - return asBool -} - -func (cs *classSettings) Model() string { - return cs.getProperty("model", DefaultCohereModel) -} - -func (cs *classSettings) Truncate() string { - return cs.getProperty("truncate", DefaultTruncate) -} - -func (cs *classSettings) BaseURL() string { - return cs.getProperty("baseURL", DefaultBaseURL) -} - -func (cs *classSettings) VectorizeClassName() bool { - if cs.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return DefaultVectorizeClassName - } - - vcn, ok := cs.cfg.Class()["vectorizeClassName"] - if !ok { - return DefaultVectorizeClassName - } - - asBool, ok := vcn.(bool) - if !ok { - return DefaultVectorizeClassName - } - - return asBool -} - -func (cs *classSettings) Validate(class *models.Class) error { - if cs.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return errors.New("empty config") - } - - model := cs.Model() - if !cs.validateCohereSetting(model, append(availableCohereModels, experimetnalCohereModels...)) { - return errors.Errorf("wrong Cohere model name, available model names are: %v", availableCohereModels) - } - truncate := cs.Truncate() - if !cs.validateCohereSetting(truncate, availableTruncates) { - return errors.Errorf("wrong truncate type, available types are: %v", availableTruncates) - } - - err := cs.validateIndexState(class, cs) - if err != nil { - return err - } - - return nil -} - -func (cs *classSettings) validateCohereSetting(value string, availableValues []string) bool { - for i := range availableValues { - if value == availableValues[i] { - return true - } - } - return false -} - -func (cs *classSettings) getProperty(name, defaultValue string) string { - if cs.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return defaultValue - } - - model, ok := cs.cfg.Class()[name] - if ok { - asString, ok := model.(string) - if ok { - if name == "truncate" { - return asString - } else { - return strings.ToLower(asString) - } - } - } - - return defaultValue -} - -func (cs *classSettings) validateIndexState(class *models.Class, settings ClassSettings) error { - if settings.VectorizeClassName() { - // if the user chooses to vectorize the classname, vector-building will - // always be possible, no need to investigate further - - return nil - } - - // search if there is at least one indexed, string/text prop. If found pass - // validation - for _, prop := range class.Properties { - if len(prop.DataType) < 1 { - return errors.Errorf("property %s must have at least one datatype: "+ - "got %v", prop.Name, prop.DataType) - } - - if prop.DataType[0] != string(schema.DataTypeText) { - // we can only vectorize text-like props - continue - } - - if settings.PropertyIndexed(prop.Name) { - // found at least one, this is a valid schema - return nil - } - } - - return fmt.Errorf("invalid properties: didn't find a single property which is " + - "of type string or text and is not excluded from indexing. In addition the " + - "class name is excluded from vectorization as well, meaning that it cannot be " + - "used to determine the vector position. To fix this, set 'vectorizeClassName' " + - "to true if the class name is contextionary-valid. Alternatively add at least " + - "contextionary-valid text/string property which is not excluded from " + - "indexing") -} diff --git a/modules/text2vec-cohere/vectorizer/fakes_for_test.go b/modules/text2vec-cohere/vectorizer/fakes_for_test.go deleted file mode 100644 index 2be4bd8ef84918ec364e8c77e5e413fbfeec31d4..0000000000000000000000000000000000000000 --- a/modules/text2vec-cohere/vectorizer/fakes_for_test.go +++ /dev/null @@ -1,96 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - - "github.com/weaviate/weaviate/modules/text2vec-cohere/ent" -) - -type fakeClient struct { - lastInput []string - lastConfig ent.VectorizationConfig -} - -func (c *fakeClient) Vectorize(ctx context.Context, - text []string, cfg ent.VectorizationConfig, -) (*ent.VectorizationResult, error) { - c.lastInput = text - c.lastConfig = cfg - return &ent.VectorizationResult{ - Vectors: [][]float32{{0, 1, 2, 3}}, - Dimensions: 4, - Text: text, - }, nil -} - -func (c *fakeClient) VectorizeQuery(ctx context.Context, - text []string, cfg ent.VectorizationConfig, -) (*ent.VectorizationResult, error) { - c.lastInput = text - c.lastConfig = cfg - return &ent.VectorizationResult{ - Vectors: [][]float32{{0.1, 1.1, 2.1, 3.1}}, - Dimensions: 4, - Text: text, - }, nil -} - -type fakeClassConfig struct { - classConfig map[string]interface{} - vectorizeClassName bool - vectorizePropertyName bool - skippedProperty string - excludedProperty string - // module specific settings - cohereModel string - truncateType string - baseURL string -} - -func (f fakeClassConfig) Class() map[string]interface{} { - classSettings := map[string]interface{}{ - "vectorizeClassName": f.vectorizeClassName, - "model": f.cohereModel, - "truncate": f.truncateType, - "baseURL": f.baseURL, - } - return classSettings -} - -func (f fakeClassConfig) ClassByModuleName(moduleName string) map[string]interface{} { - return f.classConfig -} - -func (f fakeClassConfig) Property(propName string) map[string]interface{} { - if propName == f.skippedProperty { - return map[string]interface{}{ - "skip": true, - } - } - if propName == f.excludedProperty { - return map[string]interface{}{ - "vectorizePropertyName": false, - } - } - if f.vectorizePropertyName { - return map[string]interface{}{ - "vectorizePropertyName": true, - } - } - return nil -} - -func (f fakeClassConfig) Tenant() string { - return "" -} diff --git a/modules/text2vec-cohere/vectorizer/objects.go b/modules/text2vec-cohere/vectorizer/objects.go deleted file mode 100644 index eddcc422e457f901b25949631f0be890a1507586..0000000000000000000000000000000000000000 --- a/modules/text2vec-cohere/vectorizer/objects.go +++ /dev/null @@ -1,94 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - "fmt" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/modules/text2vec-cohere/ent" - objectsvectorizer "github.com/weaviate/weaviate/usecases/modulecomponents/vectorizer" - libvectorizer "github.com/weaviate/weaviate/usecases/vectorizer" -) - -type Vectorizer struct { - client Client - objectVectorizer *objectsvectorizer.ObjectVectorizer -} - -func New(client Client) *Vectorizer { - return &Vectorizer{ - client: client, - objectVectorizer: objectsvectorizer.New(), - } -} - -type Client interface { - Vectorize(ctx context.Context, input []string, - config ent.VectorizationConfig) (*ent.VectorizationResult, error) - VectorizeQuery(ctx context.Context, input []string, - config ent.VectorizationConfig) (*ent.VectorizationResult, error) -} - -// IndexCheck returns whether a property of a class should be indexed -type ClassSettings interface { - PropertyIndexed(property string) bool - VectorizePropertyName(propertyName string) bool - VectorizeClassName() bool - Model() string - Truncate() string - BaseURL() string -} - -func (v *Vectorizer) Object(ctx context.Context, object *models.Object, - objDiff *moduletools.ObjectDiff, cfg moduletools.ClassConfig, -) error { - vec, err := v.object(ctx, object.Class, object.Properties, objDiff, cfg) - if err != nil { - return err - } - - object.Vector = vec - return nil -} - -func (v *Vectorizer) object(ctx context.Context, className string, - schema interface{}, objDiff *moduletools.ObjectDiff, cfg moduletools.ClassConfig, -) ([]float32, error) { - text, vector, err := v.objectVectorizer.TextsOrVector(ctx, className, schema, objDiff, NewClassSettings(cfg)) - if err != nil { - return nil, err - } - if vector != nil { - // dont' re-vectorize - return vector, nil - } - // vectorize text - icheck := NewClassSettings(cfg) - res, err := v.client.Vectorize(ctx, []string{text}, ent.VectorizationConfig{ - Model: icheck.Model(), - BaseURL: icheck.BaseURL(), - }) - if err != nil { - return nil, err - } - if len(res.Vectors) == 0 { - return nil, fmt.Errorf("no vectors generated") - } - - if len(res.Vectors) > 1 { - return libvectorizer.CombineVectors(res.Vectors), nil - } - return res.Vectors[0], nil -} diff --git a/modules/text2vec-cohere/vectorizer/objects_test.go b/modules/text2vec-cohere/vectorizer/objects_test.go deleted file mode 100644 index ad2ae0a67d7de6816d8169212cd24d78ff3cc7fe..0000000000000000000000000000000000000000 --- a/modules/text2vec-cohere/vectorizer/objects_test.go +++ /dev/null @@ -1,360 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/moduletools" -) - -// These are mostly copy/pasted (with minimal additions) from the -// text2vec-contextionary module -func TestVectorizingObjects(t *testing.T) { - type testCase struct { - name string - input *models.Object - expectedClientCall string - expectedCohereModel string - noindex string - excludedProperty string // to simulate a schema where property names aren't vectorized - excludedClass string // to simulate a schema where class names aren't vectorized - cohereModel string - } - - tests := []testCase{ - { - name: "empty object", - input: &models.Object{ - Class: "Car", - }, - cohereModel: "large", - expectedCohereModel: "large", - expectedClientCall: "car", - }, - { - name: "object with one string prop", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "Mercedes", - }, - }, - expectedClientCall: "car brand mercedes", - }, - { - name: "object with one non-string prop", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "power": 300, - }, - }, - expectedClientCall: "car", - }, - { - name: "object with a mix of props", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: "car brand best brand review a very great car", - }, - { - name: "with a noindexed property", - noindex: "review", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: "car brand best brand", - }, - { - name: "with the class name not vectorized", - excludedClass: "Car", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: "brand best brand review a very great car", - }, - { - name: "with a property name not vectorized", - excludedProperty: "review", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: "car brand best brand a very great car", - }, - { - name: "with no schema labels vectorized", - excludedProperty: "review", - excludedClass: "Car", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "review": "a very great car", - }, - }, - expectedClientCall: "a very great car", - }, - { - name: "with string/text arrays without propname or classname", - excludedProperty: "reviews", - excludedClass: "Car", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - expectedClientCall: "a very great car you should consider buying one", - }, - { - name: "with string/text arrays with propname and classname", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - expectedClientCall: "car reviews a very great car reviews you should consider buying one", - }, - { - name: "with compound class and prop names", - input: &models.Object{ - Class: "SuperCar", - Properties: map[string]interface{}{ - "brandOfTheCar": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: "super car brand of the car best brand review a very great car", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - client := &fakeClient{} - - v := New(client) - - ic := &fakeClassConfig{ - excludedProperty: test.excludedProperty, - skippedProperty: test.noindex, - vectorizeClassName: test.excludedClass != "Car", - cohereModel: test.cohereModel, - vectorizePropertyName: true, - } - err := v.Object(context.Background(), test.input, nil, ic) - - require.Nil(t, err) - assert.Equal(t, models.C11yVector{0, 1, 2, 3}, test.input.Vector) - expected := strings.Split(test.expectedClientCall, " ") - actual := strings.Split(client.lastInput[0], " ") - assert.Equal(t, expected, actual) - assert.Equal(t, test.expectedCohereModel, client.lastConfig.Model) - }) - } -} - -func TestVectorizingObjectsWithDiff(t *testing.T) { - type testCase struct { - name string - input *models.Object - skipped string - diff *moduletools.ObjectDiff - expectedVectorize bool - } - - tests := []testCase{ - { - name: "no diff", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: nil, - expectedVectorize: true, - }, - { - name: "diff all props unchanged", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("brand", "best brand", "best brand"). - WithProp("power", 300, 300). - WithProp("description", "a very great car", "a very great car"). - WithProp("reviews", []interface{}{ - "a very great car", - "you should consider buying one", - }, []interface{}{ - "a very great car", - "you should consider buying one", - }), - expectedVectorize: false, - }, - { - name: "diff one vectorizable prop changed (1)", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("brand", "old best brand", "best brand"), - expectedVectorize: true, - }, - { - name: "diff one vectorizable prop changed (2)", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("description", "old a very great car", "a very great car"), - expectedVectorize: true, - }, - { - name: "diff one vectorizable prop changed (3)", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("reviews", []interface{}{ - "old a very great car", - "you should consider buying one", - }, []interface{}{ - "a very great car", - "you should consider buying one", - }), - expectedVectorize: true, - }, - { - name: "all non-vectorizable props changed", - skipped: "description", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("power", 123, 300). - WithProp("description", "old a very great car", "a very great car"), - expectedVectorize: false, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - ic := &fakeClassConfig{ - skippedProperty: test.skipped, - } - - client := &fakeClient{} - v := New(client) - - err := v.Object(context.Background(), test.input, test.diff, ic) - - require.Nil(t, err) - if test.expectedVectorize { - assert.Equal(t, models.C11yVector{0, 1, 2, 3}, test.input.Vector) - assert.NotEmpty(t, client.lastInput) - } else { - assert.Equal(t, models.C11yVector{0, 0, 0, 0}, test.input.Vector) - assert.Empty(t, client.lastInput) - } - }) - } -} - -func newObjectDiffWithVector() *moduletools.ObjectDiff { - return moduletools.NewObjectDiff([]float32{0, 0, 0, 0}) -} diff --git a/modules/text2vec-cohere/vectorizer/texts.go b/modules/text2vec-cohere/vectorizer/texts.go deleted file mode 100644 index cd321cb5ba654f71021bffe6dcec52ed1601347f..0000000000000000000000000000000000000000 --- a/modules/text2vec-cohere/vectorizer/texts.go +++ /dev/null @@ -1,36 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/modules/text2vec-cohere/ent" - libvectorizer "github.com/weaviate/weaviate/usecases/vectorizer" -) - -func (v *Vectorizer) Texts(ctx context.Context, inputs []string, - cfg moduletools.ClassConfig, -) ([]float32, error) { - settings := NewClassSettings(cfg) - res, err := v.client.VectorizeQuery(ctx, inputs, ent.VectorizationConfig{ - Model: settings.Model(), - Truncate: settings.Truncate(), - BaseURL: settings.BaseURL(), - }) - if err != nil { - return nil, errors.Wrap(err, "remote client vectorize") - } - return libvectorizer.CombineVectors(res.Vectors), nil -} diff --git a/modules/text2vec-cohere/vectorizer/texts_test.go b/modules/text2vec-cohere/vectorizer/texts_test.go deleted file mode 100644 index 50006bb796f448de0777119ffed1e3db163dea1f..0000000000000000000000000000000000000000 --- a/modules/text2vec-cohere/vectorizer/texts_test.go +++ /dev/null @@ -1,93 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// as used in the nearText searcher -func TestVectorizingTexts(t *testing.T) { - type testCase struct { - name string - input []string - expectedCohereModel string - cohereModel string - } - - tests := []testCase{ - { - name: "single word", - input: []string{"hello"}, - cohereModel: "large", - expectedCohereModel: "large", - }, - { - name: "multiple words", - input: []string{"hello world, this is me!"}, - cohereModel: "large", - expectedCohereModel: "large", - }, - { - name: "multiple sentences (joined with a dot)", - input: []string{"this is sentence 1", "and here's number 2"}, - cohereModel: "large", - expectedCohereModel: "large", - }, - { - name: "multiple sentences already containing a dot", - input: []string{"this is sentence 1.", "and here's number 2"}, - cohereModel: "large", - expectedCohereModel: "large", - }, - { - name: "multiple sentences already containing a question mark", - input: []string{"this is sentence 1?", "and here's number 2"}, - cohereModel: "large", - expectedCohereModel: "large", - }, - { - name: "multiple sentences already containing an exclamation mark", - input: []string{"this is sentence 1!", "and here's number 2"}, - cohereModel: "large", - expectedCohereModel: "large", - }, - { - name: "multiple sentences already containing comma", - input: []string{"this is sentence 1,", "and here's number 2"}, - cohereModel: "large", - expectedCohereModel: "large", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - client := &fakeClient{} - - v := New(client) - - settings := &fakeClassConfig{ - cohereModel: test.cohereModel, - } - vec, err := v.Texts(context.Background(), test.input, settings) - - require.Nil(t, err) - assert.Equal(t, []float32{0.1, 1.1, 2.1, 3.1}, vec) - assert.Equal(t, test.input, client.lastInput) - assert.Equal(t, test.expectedCohereModel, client.lastConfig.Model) - }) - } -} diff --git a/modules/text2vec-contextionary/additional/additional_arguments.go b/modules/text2vec-contextionary/additional/additional_arguments.go deleted file mode 100644 index 402669ae8a9d4d9c7c35ffb518f5400cf4c8cb6b..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/additional/additional_arguments.go +++ /dev/null @@ -1,106 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package additional - -import ( - "fmt" - - "github.com/tailor-inc/graphql" -) - -func additionalNearestNeighborsField(classname string) *graphql.Field { - return &graphql.Field{ - Type: graphql.NewObject(graphql.ObjectConfig{ - Name: fmt.Sprintf("%sAdditionalNearestNeighbors", classname), - Fields: graphql.Fields{ - "neighbors": &graphql.Field{Type: graphql.NewList(graphql.NewObject(graphql.ObjectConfig{ - Name: fmt.Sprintf("%sAdditionalNearestNeighborsNeighbor", classname), - Fields: graphql.Fields{ - "concept": &graphql.Field{Type: graphql.String}, - "distance": &graphql.Field{Type: graphql.Float}, - }, - }))}, - }, - }), - } -} - -func additionalFeatureProjectionField(classname string) *graphql.Field { - return &graphql.Field{ - Args: graphql.FieldConfigArgument{ - "algorithm": &graphql.ArgumentConfig{ - Type: graphql.String, - DefaultValue: nil, - }, - "dimensions": &graphql.ArgumentConfig{ - Type: graphql.Int, - DefaultValue: nil, - }, - "learningRate": &graphql.ArgumentConfig{ - Type: graphql.Int, - DefaultValue: nil, - }, - "iterations": &graphql.ArgumentConfig{ - Type: graphql.Int, - DefaultValue: nil, - }, - "perplexity": &graphql.ArgumentConfig{ - Type: graphql.Int, - DefaultValue: nil, - }, - }, - Type: graphql.NewObject(graphql.ObjectConfig{ - Name: fmt.Sprintf("%sAdditionalFeatureProjection", classname), - Fields: graphql.Fields{ - "vector": &graphql.Field{Type: graphql.NewList(graphql.Float)}, - }, - }), - } -} - -func additionalSemanticPathField(classname string) *graphql.Field { - return &graphql.Field{ - Type: graphql.NewObject(graphql.ObjectConfig{ - Name: fmt.Sprintf("%sAdditionalSemanticPath", classname), - Fields: graphql.Fields{ - "path": &graphql.Field{Type: graphql.NewList(graphql.NewObject(graphql.ObjectConfig{ - Name: fmt.Sprintf("%sAdditionalSemanticPathElement", classname), - Fields: graphql.Fields{ - "concept": &graphql.Field{Type: graphql.String}, - "distanceToQuery": &graphql.Field{Type: graphql.Float}, - "distanceToResult": &graphql.Field{Type: graphql.Float}, - "distanceToNext": &graphql.Field{Type: graphql.Float}, - "distanceToPrevious": &graphql.Field{Type: graphql.Float}, - }, - }))}, - }, - }), - } -} - -func additionalInterpretationField(classname string) *graphql.Field { - return &graphql.Field{ - Type: graphql.NewObject(graphql.ObjectConfig{ - Name: fmt.Sprintf("%sAdditionalInterpretation", classname), - Fields: graphql.Fields{ - "source": &graphql.Field{Type: graphql.NewList(graphql.NewObject(graphql.ObjectConfig{ - Name: fmt.Sprintf("%sAdditionalInterpretationSource", classname), - Fields: graphql.Fields{ - "concept": &graphql.Field{Type: graphql.String}, - "weight": &graphql.Field{Type: graphql.Float}, - "occurrence": &graphql.Field{Type: graphql.Int}, - }, - }))}, - }, - }), - } -} diff --git a/modules/text2vec-contextionary/additional/additional_arguments_test.go b/modules/text2vec-contextionary/additional/additional_arguments_test.go deleted file mode 100644 index 1cc7e5b8a022af22293f3ccba97813f7552b0ff3..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/additional/additional_arguments_test.go +++ /dev/null @@ -1,163 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package additional - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/tailor-inc/graphql" -) - -func TestNearestNeighborsField(t *testing.T) { - t.Run("should generate nearestNeighbors argument properly", func(t *testing.T) { - // given - classname := "Class" - - // when - nearestNeighbors := additionalNearestNeighborsField(classname) - - // then - // the built graphQL field needs to support this structure: - // Type: { - // neighbors: { - // concept: "c1", - // distance: 0.8 - // } - // } - assert.NotNil(t, nearestNeighbors) - assert.Equal(t, "ClassAdditionalNearestNeighbors", nearestNeighbors.Type.Name()) - assert.NotNil(t, nearestNeighbors.Type) - nearestNeighborsObject, nearestNeighborsObjectOK := nearestNeighbors.Type.(*graphql.Object) - assert.True(t, nearestNeighborsObjectOK) - assert.Equal(t, 1, len(nearestNeighborsObject.Fields())) - neighbors, neighborsOK := nearestNeighborsObject.Fields()["neighbors"] - assert.True(t, neighborsOK) - neighborsList, neighborsListOK := neighbors.Type.(*graphql.List) - assert.True(t, neighborsListOK) - neighborsListObjects, neighborsListObjectsOK := neighborsList.OfType.(*graphql.Object) - assert.True(t, neighborsListObjectsOK) - assert.Equal(t, 2, len(neighborsListObjects.Fields())) - assert.NotNil(t, neighborsListObjects.Fields()["concept"]) - assert.NotNil(t, neighborsListObjects.Fields()["distance"]) - }) -} - -func TestFeatureProjectionField(t *testing.T) { - t.Run("should generate featureProjection argument properly", func(t *testing.T) { - // given - classname := "Class" - - // when - featureProjection := additionalFeatureProjectionField(classname) - - // then - // the built graphQL field needs to support this structure: - // Args: { - // algorithm: "a", - // dimensions: 1, - // learningRate: 2, - // iterations: 3, - // perplexity: 4 - // } - // Type: { - // vector: [0, 1] - // } - assert.NotNil(t, featureProjection) - assert.Equal(t, "ClassAdditionalFeatureProjection", featureProjection.Type.Name()) - assert.NotNil(t, featureProjection.Args) - assert.Equal(t, 5, len(featureProjection.Args)) - assert.NotNil(t, featureProjection.Args["algorithm"]) - assert.NotNil(t, featureProjection.Args["dimensions"]) - assert.NotNil(t, featureProjection.Args["learningRate"]) - assert.NotNil(t, featureProjection.Args["iterations"]) - assert.NotNil(t, featureProjection.Args["perplexity"]) - featureProjectionObject, featureProjectionObjectOK := featureProjection.Type.(*graphql.Object) - assert.True(t, featureProjectionObjectOK) - assert.Equal(t, 1, len(featureProjectionObject.Fields())) - assert.NotNil(t, featureProjectionObject.Fields()["vector"]) - }) -} - -func TestSemanticPathField(t *testing.T) { - t.Run("should generate semanticPath argument properly", func(t *testing.T) { - // given - classname := "Class" - - // when - semanticPath := additionalSemanticPathField(classname) - - // then - // the built graphQL field needs to support this structure: - // Type: { - // path: [ - // { - // concept: "c1", - // distanceToQuery: 0.1, - // distanceToResult: 0.2, - // distanceToNext: 0.3, - // distanceToPrevious: 0.4, - // } - // } - assert.NotNil(t, semanticPath) - assert.Equal(t, "ClassAdditionalSemanticPath", semanticPath.Type.Name()) - semanticPathObject, semanticPathObjectOK := semanticPath.Type.(*graphql.Object) - assert.True(t, semanticPathObjectOK) - assert.Equal(t, 1, len(semanticPathObject.Fields())) - assert.NotNil(t, semanticPathObject.Fields()["path"]) - semanticPathObjectList, semanticPathObjectListOK := semanticPathObject.Fields()["path"].Type.(*graphql.List) - assert.True(t, semanticPathObjectListOK) - semanticPathObjectListObjects, semanticPathObjectListOK := semanticPathObjectList.OfType.(*graphql.Object) - assert.True(t, semanticPathObjectListOK) - assert.Equal(t, 5, len(semanticPathObjectListObjects.Fields())) - assert.NotNil(t, semanticPathObjectListObjects.Fields()["concept"]) - assert.NotNil(t, semanticPathObjectListObjects.Fields()["distanceToQuery"]) - assert.NotNil(t, semanticPathObjectListObjects.Fields()["distanceToResult"]) - assert.NotNil(t, semanticPathObjectListObjects.Fields()["distanceToNext"]) - assert.NotNil(t, semanticPathObjectListObjects.Fields()["distanceToPrevious"]) - }) -} - -func TestNearestInterpretationField(t *testing.T) { - t.Run("should generate interpretation argument properly", func(t *testing.T) { - // given - classname := "Class" - - // when - interpretation := additionalInterpretationField(classname) - - // then - // the built graphQL field needs to support this structure: - // Type: { - // source: [ - // { - // concept: "c1", - // weight: 0.1, - // occurrence: 0.2, - // } - // } - assert.NotNil(t, interpretation) - assert.Equal(t, "ClassAdditionalInterpretation", interpretation.Type.Name()) - interpretationObject, interpretationObjectOK := interpretation.Type.(*graphql.Object) - assert.True(t, interpretationObjectOK) - assert.Equal(t, 1, len(interpretationObject.Fields())) - assert.NotNil(t, interpretationObject.Fields()["source"]) - interpretationObjectList, interpretationObjectListOK := interpretationObject.Fields()["source"].Type.(*graphql.List) - assert.True(t, interpretationObjectListOK) - interpretationObjectListObjects, interpretationObjectListObjectsOK := interpretationObjectList.OfType.(*graphql.Object) - assert.True(t, interpretationObjectListObjectsOK) - assert.Equal(t, 3, len(interpretationObjectListObjects.Fields())) - assert.NotNil(t, interpretationObjectListObjects.Fields()["concept"]) - assert.NotNil(t, interpretationObjectListObjects.Fields()["weight"]) - assert.NotNil(t, interpretationObjectListObjects.Fields()["occurrence"]) - }) -} diff --git a/modules/text2vec-contextionary/additional/interpretation/interpretation.go b/modules/text2vec-contextionary/additional/interpretation/interpretation.go deleted file mode 100644 index 09bb0863cf771beb16129db19866eb610afe6f51..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/additional/interpretation/interpretation.go +++ /dev/null @@ -1,46 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package interpretation - -import ( - "context" - - "github.com/tailor-inc/graphql/language/ast" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/search" -) - -type Interpretation struct{} - -func New() *Interpretation { - return &Interpretation{} -} - -func (e *Interpretation) AdditionalPropertyDefaultValue() interface{} { - return true -} - -func (e *Interpretation) AdditionalPropertyFn(ctx context.Context, - in []search.Result, params interface{}, limit *int, - argumentModuleParams map[string]interface{}, cfg moduletools.ClassConfig, -) ([]search.Result, error) { - // this is a special case additional value - // this value is being added to storage object in vectorization process - // interpretation is being saved in DB when making vectorization - // interpretation is being extracted and added to the result - // when it's being read from DB (see storage_object.go) - return in, nil -} - -func (e *Interpretation) ExtractAdditionalFn(param []*ast.Argument) interface{} { - return true -} diff --git a/modules/text2vec-contextionary/additional/models/models.go b/modules/text2vec-contextionary/additional/models/models.go deleted file mode 100644 index e68d81b1a013e13b523b0a4c09678c074fa4cd89..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/additional/models/models.go +++ /dev/null @@ -1,48 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package models - -type FeatureProjection struct { - Vector []float32 `json:"vector"` -} - -type NearestNeighbors struct { - Neighbors []*NearestNeighbor `json:"neighbors"` -} - -type NearestNeighbor struct { - Concept string `json:"concept,omitempty"` - Distance float32 `json:"distance,omitempty"` - Vector []float32 `json:"vector"` -} - -type SemanticPath struct { - Path []*SemanticPathElement `json:"path"` -} - -type SemanticPathElement struct { - Concept string `json:"concept,omitempty"` - DistanceToNext *float32 `json:"distanceToNext,omitempty"` - DistanceToPrevious *float32 `json:"distanceToPrevious,omitempty"` - DistanceToQuery float32 `json:"distanceToQuery,omitempty"` - DistanceToResult float32 `json:"distanceToResult,omitempty"` -} - -type Interpretation struct { - Source []*InterpretationSource `json:"source"` -} - -type InterpretationSource struct { - Concept string `json:"concept,omitempty"` - Occurrence uint64 `json:"occurrence,omitempty"` - Weight float64 `json:"weight,omitempty"` -} diff --git a/modules/text2vec-contextionary/additional/nearestneighbors/extender.go b/modules/text2vec-contextionary/additional/nearestneighbors/extender.go deleted file mode 100644 index 6a7270a9740c1347b15c1fd59a8a46de916f7b41..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/additional/nearestneighbors/extender.go +++ /dev/null @@ -1,130 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package nearestneighbors - -import ( - "context" - "fmt" - - "github.com/weaviate/weaviate/entities/moduletools" - - "github.com/pkg/errors" - "github.com/tailor-inc/graphql/language/ast" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/search" - txt2vecmodels "github.com/weaviate/weaviate/modules/text2vec-contextionary/additional/models" -) - -const ( - DefaultLimit = 10 - DefaultK = 32 -) - -type Extender struct { - searcher contextionary -} - -type contextionary interface { - MultiNearestWordsByVector(ctx context.Context, vectors [][]float32, k, n int) ([]*txt2vecmodels.NearestNeighbors, error) -} - -func (e *Extender) AdditionalPropertyDefaultValue() interface{} { - return true -} - -func (e *Extender) AdditionalPropertyFn(ctx context.Context, - in []search.Result, params interface{}, limit *int, - argumentModuleParams map[string]interface{}, cfg moduletools.ClassConfig, -) ([]search.Result, error) { - return e.Multi(ctx, in, limit) -} - -func (e *Extender) ExtractAdditionalFn(param []*ast.Argument) interface{} { - return true -} - -func (e *Extender) Single(ctx context.Context, in *search.Result, limit *int) (*search.Result, error) { - if in == nil { - return nil, nil - } - - multiRes, err := e.Multi(ctx, []search.Result{*in}, limit) // safe to deref, as we did a nil check before - if err != nil { - return nil, err - } - - return &multiRes[0], nil -} - -func (e *Extender) Multi(ctx context.Context, in []search.Result, limit *int) ([]search.Result, error) { - if in == nil { - return nil, nil - } - - vectors := make([][]float32, len(in)) - for i, res := range in { - if res.Vector == nil || len(res.Vector) == 0 { - return nil, fmt.Errorf("item %d has no vector", i) - } - vectors[i] = res.Vector - } - - neighbors, err := e.searcher.MultiNearestWordsByVector(ctx, vectors, DefaultK, limitOrDefault(limit)) - if err != nil { - return nil, errors.Wrap(err, "get neighbors for search results") - } - - if len(neighbors) != len(in) { - return nil, fmt.Errorf("inconsistent results: input=%d neighbors=%d", len(in), len(neighbors)) - } - - for i, res := range in { - up := res.AdditionalProperties - if up == nil { - up = models.AdditionalProperties{} - } - - up["nearestNeighbors"] = removeDollarElements(neighbors[i]) - in[i].AdditionalProperties = up - } - - return in, nil -} - -func NewExtender(searcher contextionary) *Extender { - return &Extender{searcher: searcher} -} - -func limitOrDefault(user *int) int { - if user == nil || *user == 0 { - return DefaultLimit - } - - return *user -} - -func removeDollarElements(in *txt2vecmodels.NearestNeighbors) *txt2vecmodels.NearestNeighbors { - neighbors := make([]*txt2vecmodels.NearestNeighbor, len(in.Neighbors)) - i := 0 - for _, elem := range in.Neighbors { - if elem.Concept[0] == '$' { - continue - } - - neighbors[i] = elem - i++ - } - - return &txt2vecmodels.NearestNeighbors{ - Neighbors: neighbors[:i], - } -} diff --git a/modules/text2vec-contextionary/additional/nearestneighbors/extender_test.go b/modules/text2vec-contextionary/additional/nearestneighbors/extender_test.go deleted file mode 100644 index 5f036f2f1c7bde7c0742550127a332eade2cbaa3..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/additional/nearestneighbors/extender_test.go +++ /dev/null @@ -1,261 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package nearestneighbors - -import ( - "context" - "testing" - - "github.com/go-openapi/strfmt" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/search" - txt2vecmodels "github.com/weaviate/weaviate/modules/text2vec-contextionary/additional/models" -) - -func TestExtender(t *testing.T) { - f := &fakeContextionary{} - e := NewExtender(f) - - t.Run("with empty results", func(t *testing.T) { - testData := []search.Result(nil) - expectedResults := []search.Result(nil) - - res, err := e.Multi(context.Background(), testData, nil) - require.Nil(t, err) - assert.Equal(t, expectedResults, res) - }) - - t.Run("with a single result", func(t *testing.T) { - testData := &search.Result{ - Schema: map[string]interface{}{"name": "item1"}, - Vector: []float32{0.1, 0.3, 0.5}, - AdditionalProperties: map[string]interface{}{ - "classification": &additional.Classification{ // verify it doesn't remove existing additional props - ID: strfmt.UUID("123"), - }, - }, - } - - expectedResult := &search.Result{ - Schema: map[string]interface{}{"name": "item1"}, - Vector: []float32{0.1, 0.3, 0.5}, - AdditionalProperties: map[string]interface{}{ - "classification": &additional.Classification{ // verify it doesn't remove existing additional props - ID: strfmt.UUID("123"), - }, - "nearestNeighbors": &txt2vecmodels.NearestNeighbors{ - Neighbors: []*txt2vecmodels.NearestNeighbor{ - { - Concept: "word1", - Distance: 1, - }, - { - Concept: "word2", - Distance: 2, - }, - { - Concept: "word3", - Distance: 3, - }, - }, - }, - }, - } - - res, err := e.Single(context.Background(), testData, nil) - require.Nil(t, err) - assert.Equal(t, expectedResult, res) - }) - - t.Run("with multiple results", func(t *testing.T) { - vectors := [][]float32{ - {0.1, 0.2, 0.3}, - {0.11, 0.22, 0.33}, - {0.111, 0.222, 0.333}, - } - - testData := []search.Result{ - { - Schema: map[string]interface{}{"name": "item1"}, - Vector: vectors[0], - }, - { - Schema: map[string]interface{}{"name": "item2"}, - Vector: vectors[1], - }, - { - Schema: map[string]interface{}{"name": "item3"}, - Vector: vectors[2], - AdditionalProperties: map[string]interface{}{ - "classification": &additional.Classification{ // verify it doesn't remove existing additional props - ID: strfmt.UUID("123"), - }, - }, - }, - } - - expectedResults := []search.Result{ - { - Schema: map[string]interface{}{"name": "item1"}, - Vector: vectors[0], - AdditionalProperties: map[string]interface{}{ - "nearestNeighbors": &txt2vecmodels.NearestNeighbors{ - Neighbors: []*txt2vecmodels.NearestNeighbor{ - { - Concept: "word1", - Distance: 1, - }, - { - Concept: "word2", - Distance: 2, - }, - { - Concept: "word3", - Distance: 3, - }, - }, - }, - }, - }, - { - Schema: map[string]interface{}{"name": "item2"}, - Vector: vectors[1], - AdditionalProperties: map[string]interface{}{ - "nearestNeighbors": &txt2vecmodels.NearestNeighbors{ - Neighbors: []*txt2vecmodels.NearestNeighbor{ - { - Concept: "word4", - Distance: 0.1, - }, - { - Concept: "word5", - Distance: 0.2, - }, - { - Concept: "word6", - Distance: 0.3, - }, - }, - }, - }, - }, - { - Schema: map[string]interface{}{"name": "item3"}, - Vector: vectors[2], - AdditionalProperties: map[string]interface{}{ - "classification": &additional.Classification{ // verify it doesn't remove existing additional props - ID: strfmt.UUID("123"), - }, - "nearestNeighbors": &txt2vecmodels.NearestNeighbors{ - Neighbors: []*txt2vecmodels.NearestNeighbor{ - { - Concept: "word7", - Distance: 1.1, - }, - { - Concept: "word8", - Distance: 2.2, - }, - { - Concept: "word9", - Distance: 3.3, - }, - }, - }, - }, - }, - } - - res, err := e.Multi(context.Background(), testData, nil) - require.Nil(t, err) - assert.Equal(t, expectedResults, res) - assert.Equal(t, f.calledWithVectors, vectors) - }) -} - -type fakeContextionary struct { - calledWithVectors [][]float32 -} - -func (f *fakeContextionary) MultiNearestWordsByVector(ctx context.Context, vectors [][]float32, k, n int) ([]*txt2vecmodels.NearestNeighbors, error) { - f.calledWithVectors = vectors - out := []*txt2vecmodels.NearestNeighbors{ - { - Neighbors: []*txt2vecmodels.NearestNeighbor{ - { - Concept: "word1", - Distance: 1.0, - Vector: nil, - }, - { - Concept: "word2", - Distance: 2.0, - Vector: nil, - }, - { - Concept: "$THING[abc]", - Distance: 9.99, - Vector: nil, - }, - { - Concept: "word3", - Distance: 3.0, - Vector: nil, - }, - }, - }, - - { - Neighbors: []*txt2vecmodels.NearestNeighbor{ - { - Concept: "word4", - Distance: 0.1, - Vector: nil, - }, - { - Concept: "word5", - Distance: 0.2, - Vector: nil, - }, - { - Concept: "word6", - Distance: 0.3, - Vector: nil, - }, - }, - }, - - { - Neighbors: []*txt2vecmodels.NearestNeighbor{ - { - Concept: "word7", - Distance: 1.1, - Vector: nil, - }, - { - Concept: "word8", - Distance: 2.2, - Vector: nil, - }, - { - Concept: "word9", - Distance: 3.3, - Vector: nil, - }, - }, - }, - } - - return out[:len(vectors)], nil // return up to three results, but fewer if the input is shorter -} diff --git a/modules/text2vec-contextionary/additional/provider.go b/modules/text2vec-contextionary/additional/provider.go deleted file mode 100644 index 2dfef8f547025ef0bdea6285b4209a407a690dbd..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/additional/provider.go +++ /dev/null @@ -1,121 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package additional - -import ( - "context" - - "github.com/weaviate/weaviate/entities/moduletools" - - "github.com/tailor-inc/graphql/language/ast" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/search" -) - -type AdditionalProperty interface { - AdditionalPropertyFn(ctx context.Context, - in []search.Result, params interface{}, limit *int, - argumentModuleParams map[string]interface{}, cfg moduletools.ClassConfig) ([]search.Result, error) - ExtractAdditionalFn(param []*ast.Argument) interface{} - AdditionalPropertyDefaultValue() interface{} -} - -type GraphQLAdditionalArgumentsProvider struct { - nnExtender AdditionalProperty - projector AdditionalProperty - sempathBuilder AdditionalProperty - interpretation AdditionalProperty -} - -func New(nnExtender, projector, sempath, interpretation AdditionalProperty) *GraphQLAdditionalArgumentsProvider { - return &GraphQLAdditionalArgumentsProvider{nnExtender, projector, sempath, interpretation} -} - -func (p *GraphQLAdditionalArgumentsProvider) AdditionalProperties() map[string]modulecapabilities.AdditionalProperty { - additionalProperties := map[string]modulecapabilities.AdditionalProperty{} - additionalProperties["nearestNeighbors"] = p.getNearestNeighbors() - additionalProperties["featureProjection"] = p.getFeatureProjection() - additionalProperties["semanticPath"] = p.getSemanticPath() - additionalProperties["interpretation"] = p.getInterpretation() - return additionalProperties -} - -func (p *GraphQLAdditionalArgumentsProvider) getNearestNeighbors() modulecapabilities.AdditionalProperty { - return modulecapabilities.AdditionalProperty{ - RestNames: []string{ - "nearestNeighbors", - "nearestneighbors", - "nearest-neighbors", - "nearest_neighbors", - }, - DefaultValue: p.nnExtender.AdditionalPropertyDefaultValue(), - GraphQLNames: []string{"nearestNeighbors"}, - GraphQLFieldFunction: additionalNearestNeighborsField, - GraphQLExtractFunction: p.nnExtender.ExtractAdditionalFn, - SearchFunctions: modulecapabilities.AdditionalSearch{ - ObjectGet: p.nnExtender.AdditionalPropertyFn, - ObjectList: p.nnExtender.AdditionalPropertyFn, - ExploreGet: p.nnExtender.AdditionalPropertyFn, - ExploreList: p.nnExtender.AdditionalPropertyFn, - }, - } -} - -func (p *GraphQLAdditionalArgumentsProvider) getFeatureProjection() modulecapabilities.AdditionalProperty { - return modulecapabilities.AdditionalProperty{ - RestNames: []string{ - "featureProjection", - "featureprojection", - "feature-projection", - "feature_projection", - }, - DefaultValue: p.projector.AdditionalPropertyDefaultValue(), - GraphQLNames: []string{"featureProjection"}, - GraphQLFieldFunction: additionalFeatureProjectionField, - GraphQLExtractFunction: p.projector.ExtractAdditionalFn, - SearchFunctions: modulecapabilities.AdditionalSearch{ - ObjectList: p.projector.AdditionalPropertyFn, - ExploreGet: p.projector.AdditionalPropertyFn, - ExploreList: p.projector.AdditionalPropertyFn, - }, - } -} - -func (p *GraphQLAdditionalArgumentsProvider) getSemanticPath() modulecapabilities.AdditionalProperty { - return modulecapabilities.AdditionalProperty{ - DefaultValue: p.sempathBuilder.AdditionalPropertyDefaultValue(), - GraphQLNames: []string{"semanticPath"}, - GraphQLFieldFunction: additionalSemanticPathField, - GraphQLExtractFunction: p.sempathBuilder.ExtractAdditionalFn, - SearchFunctions: modulecapabilities.AdditionalSearch{ - ExploreGet: p.sempathBuilder.AdditionalPropertyFn, - }, - } -} - -func (p *GraphQLAdditionalArgumentsProvider) getInterpretation() modulecapabilities.AdditionalProperty { - return modulecapabilities.AdditionalProperty{ - RestNames: []string{ - "interpretation", - }, - DefaultValue: p.interpretation.AdditionalPropertyDefaultValue(), - GraphQLNames: []string{"interpretation"}, - GraphQLFieldFunction: additionalInterpretationField, - GraphQLExtractFunction: p.interpretation.ExtractAdditionalFn, - SearchFunctions: modulecapabilities.AdditionalSearch{ - ObjectGet: p.interpretation.AdditionalPropertyFn, - ObjectList: p.interpretation.AdditionalPropertyFn, - ExploreGet: p.interpretation.AdditionalPropertyFn, - ExploreList: p.interpretation.AdditionalPropertyFn, - }, - } -} diff --git a/modules/text2vec-contextionary/additional/sempath/builder.go b/modules/text2vec-contextionary/additional/sempath/builder.go deleted file mode 100644 index 4f36e2d7797f6afdf1430f2324d42417b7e132e2..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/additional/sempath/builder.go +++ /dev/null @@ -1,400 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package sempath - -import ( - "context" - "fmt" - "math" - "sort" - "time" - - "github.com/danaugrs/go-tsne/tsne" - "github.com/pkg/errors" - "github.com/tailor-inc/graphql/language/ast" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/search" - txt2vecmodels "github.com/weaviate/weaviate/modules/text2vec-contextionary/additional/models" - "gonum.org/v1/gonum/mat" -) - -func New(c11y Remote) *PathBuilder { - return &PathBuilder{ - fixedSeed: time.Now().UnixNano(), - c11y: c11y, - } -} - -type PathBuilder struct { - fixedSeed int64 - c11y Remote -} - -type Remote interface { - MultiNearestWordsByVector(ctx context.Context, vectors [][]float32, k, n int) ([]*txt2vecmodels.NearestNeighbors, error) -} - -func (pb *PathBuilder) AdditionalPropertyDefaultValue() interface{} { - return &Params{} -} - -func (pb *PathBuilder) AdditionalPropertyFn(ctx context.Context, - in []search.Result, params interface{}, limit *int, - argumentModuleParams map[string]interface{}, cfg moduletools.ClassConfig, -) ([]search.Result, error) { - if parameters, ok := params.(*Params); ok { - return pb.CalculatePath(in, parameters) - } - return nil, errors.New("unknown params") -} - -func (pb *PathBuilder) ExtractAdditionalFn(param []*ast.Argument) interface{} { - return &Params{} -} - -func (pb *PathBuilder) CalculatePath(in []search.Result, params *Params) ([]search.Result, error) { - if len(in) == 0 { - return nil, nil - } - - if params == nil { - return nil, fmt.Errorf("no params provided") - } - - dims := len(in[0].Vector) - if err := params.SetDefaultsAndValidate(len(in), dims); err != nil { - return nil, errors.Wrap(err, "invalid params") - } - - searchNeighbors, err := pb.addSearchNeighbors(params) - if err != nil { - return nil, err - } - - for i, obj := range in { - path, err := pb.calculatePathPerObject(obj, in, params, searchNeighbors) - if err != nil { - return nil, fmt.Errorf("object %d: %v", i, err) - } - - if in[i].AdditionalProperties == nil { - in[i].AdditionalProperties = models.AdditionalProperties{} - } - - in[i].AdditionalProperties["semanticPath"] = path - } - - return in, nil -} - -func (pb *PathBuilder) calculatePathPerObject(obj search.Result, allObjects []search.Result, params *Params, - searchNeighbors []*txt2vecmodels.NearestNeighbor, -) (*txt2vecmodels.SemanticPath, error) { - dims := len(obj.Vector) - matrix, neighbors, err := pb.vectorsToMatrix(obj, allObjects, dims, params, searchNeighbors) - if err != nil { - return nil, err - } - - inputRows := matrix.RawMatrix().Rows - t := tsne.NewTSNE(2, float64(inputRows/2), 100, 100, false) - res := t.EmbedData(matrix, nil) - rows, cols := res.Dims() - if rows != inputRows { - return nil, fmt.Errorf("have different output results than input %d != %d", inputRows, rows) - } - - // create an explicit copy of the neighbors, so we don't mutate them. - // Otherwise the 2nd round will have been influenced by the first - projectedNeighbors := copyNeighbors(neighbors) - var projectedSearchVector []float32 - var projectedTargetVector []float32 - for i := 0; i < rows; i++ { - vector := make([]float32, cols) - for j := range vector { - vector[j] = float32(res.At(i, j)) - } - if i == 0 { // the input object - projectedTargetVector = vector - } else if i < 1+len(neighbors) { - // these must be neighbor props - projectedNeighbors[i-1].Vector = vector - } else { - // is now the very last element which is the search vector - projectedSearchVector = vector - } - } - - path := pb.buildPath(projectedNeighbors, projectedSearchVector, projectedTargetVector) - return pb.addDistancesToPath(path, neighbors, params.SearchVector, obj.Vector) -} - -func (pb *PathBuilder) addSearchNeighbors(params *Params) ([]*txt2vecmodels.NearestNeighbor, error) { - nn, err := pb.c11y.MultiNearestWordsByVector(context.TODO(), [][]float32{params.SearchVector}, 36, 50) - if err != nil { - return nil, err - } - - return nn[0].Neighbors, nil -} - -// TODO: document behavior if it actually stays like this -func (pb *PathBuilder) vectorsToMatrix(obj search.Result, allObjects []search.Result, dims int, - params *Params, searchNeighbors []*txt2vecmodels.NearestNeighbor, -) (*mat.Dense, []*txt2vecmodels.NearestNeighbor, error) { - items := 1 // the initial object - var neighbors []*txt2vecmodels.NearestNeighbor - neighbors = pb.extractNeighbors(allObjects) - neighbors = append(neighbors, searchNeighbors...) - neighbors = pb.removeDuplicateNeighborsAndDollarNeighbors(neighbors) - items += len(neighbors) + 1 // The +1 is for the search vector which we append last - - // concat all vectors to build gonum dense matrix - mergedVectors := make([]float64, items*dims) - if l := len(obj.Vector); l != dims { - return nil, nil, fmt.Errorf("object: inconsistent vector lengths found: dimensions=%d and object=%d", dims, l) - } - - for j, dim := range obj.Vector { - mergedVectors[j] = float64(dim) - } - - withoutNeighbors := 1 * dims - for i, neighbor := range neighbors { - neighborVector := neighbor.Vector - - if l := len(neighborVector); l != dims { - return nil, nil, fmt.Errorf("neighbor: inconsistent vector lengths found: dimensions=%d and object=%d", dims, l) - } - - for j, dim := range neighborVector { - mergedVectors[withoutNeighbors+i*dims+j] = float64(dim) - } - } - - for i, dim := range params.SearchVector { - mergedVectors[len(mergedVectors)-dims+i] = float64(dim) - } - - return mat.NewDense(items, dims, mergedVectors), neighbors, nil -} - -func (pb *PathBuilder) extractNeighbors(in []search.Result) []*txt2vecmodels.NearestNeighbor { - var out []*txt2vecmodels.NearestNeighbor - - for _, obj := range in { - if obj.AdditionalProperties == nil || obj.AdditionalProperties["nearestNeighbors"] == nil { - continue - } - - if neighbors, ok := obj.AdditionalProperties["nearestNeighbors"]; ok { - if nearestNeighbors, ok := neighbors.(*txt2vecmodels.NearestNeighbors); ok { - out = append(out, nearestNeighbors.Neighbors...) - } - } - } - - return out -} - -func (pb *PathBuilder) removeDuplicateNeighborsAndDollarNeighbors(in []*txt2vecmodels.NearestNeighbor) []*txt2vecmodels.NearestNeighbor { - seen := map[string]struct{}{} - out := make([]*txt2vecmodels.NearestNeighbor, len(in)) - - i := 0 - for _, candidate := range in { - if _, ok := seen[candidate.Concept]; ok { - continue - } - - if candidate.Concept[0] == '$' { - continue - } - - out[i] = candidate - i++ - seen[candidate.Concept] = struct{}{} - } - - return out[:i] -} - -func (pb *PathBuilder) buildPath(neighbors []*txt2vecmodels.NearestNeighbor, searchVector []float32, - target []float32, -) *txt2vecmodels.SemanticPath { - var path []*txt2vecmodels.SemanticPathElement - - minDist := float32(math.MaxFloat32) - - current := searchVector // initial search point - - for { - nn := pb.nearestNeighbors(current, neighbors, 10) - nn = pb.discardFurtherThan(nn, minDist, target) - if len(nn) == 0 { - break - } - nn = pb.nearestNeighbors(current, nn, 1) - current = nn[0].Vector - minDist = pb.distance(current, target) - - path = append(path, &txt2vecmodels.SemanticPathElement{ - Concept: nn[0].Concept, - }) - } - - return &txt2vecmodels.SemanticPath{ - Path: path, - } -} - -func (pb *PathBuilder) nearestNeighbors(search []float32, candidates []*txt2vecmodels.NearestNeighbor, length int) []*txt2vecmodels.NearestNeighbor { - sort.Slice(candidates, func(a, b int) bool { - return pb.distance(candidates[a].Vector, search) < pb.distance(candidates[b].Vector, search) - }) - return candidates[:length] -} - -func (pb *PathBuilder) distance(a, b []float32) float32 { - var sums float32 - for i := range a { - sums += (a[i] - b[i]) * (a[i] - b[i]) - } - - return float32(math.Sqrt(float64(sums))) -} - -func (pb *PathBuilder) discardFurtherThan(candidates []*txt2vecmodels.NearestNeighbor, threshold float32, target []float32) []*txt2vecmodels.NearestNeighbor { - out := make([]*txt2vecmodels.NearestNeighbor, len(candidates)) - i := 0 - for _, c := range candidates { - if pb.distance(c.Vector, target) >= threshold { - continue - } - - out[i] = c - i++ - } - - return out[:i] -} - -// create an explicit deep copy that does not keep any references -func copyNeighbors(in []*txt2vecmodels.NearestNeighbor) []*txt2vecmodels.NearestNeighbor { - out := make([]*txt2vecmodels.NearestNeighbor, len(in)) - for i, n := range in { - out[i] = &txt2vecmodels.NearestNeighbor{ - Concept: n.Concept, - Distance: n.Distance, - Vector: n.Vector, - } - } - - return out -} - -func (pb *PathBuilder) addDistancesToPath(path *txt2vecmodels.SemanticPath, neighbors []*txt2vecmodels.NearestNeighbor, - searchVector, targetVector []float32, -) (*txt2vecmodels.SemanticPath, error) { - for i, elem := range path.Path { - vec, ok := neighborVecByConcept(neighbors, elem.Concept) - if !ok { - return nil, fmt.Errorf("no vector present for concept: %s", elem.Concept) - } - - if i != 0 { - // include previous - previousVec, ok := neighborVecByConcept(neighbors, path.Path[i-1].Concept) - if !ok { - return nil, fmt.Errorf("no vector present for previous concept: %s", path.Path[i-1].Concept) - } - - d, err := cosineDist(vec, previousVec) - if err != nil { - return nil, errors.Wrap(err, "calculate distance between current path and previous element") - } - - path.Path[i].DistanceToPrevious = &d - } - - // target - d, err := cosineDist(vec, targetVector) - if err != nil { - return nil, errors.Wrap(err, "calculate distance between current path and result element") - } - path.Path[i].DistanceToResult = d - - // query - d, err = cosineDist(vec, searchVector) - if err != nil { - return nil, errors.Wrap(err, "calculate distance between current path and query element") - } - path.Path[i].DistanceToQuery = d - - if i != len(path.Path)-1 { - // include next - nextVec, ok := neighborVecByConcept(neighbors, path.Path[i+1].Concept) - if !ok { - return nil, fmt.Errorf("no vector present for next concept: %s", path.Path[i+1].Concept) - } - - d, err := cosineDist(vec, nextVec) - if err != nil { - return nil, errors.Wrap(err, "calculate distance between current path and next element") - } - - path.Path[i].DistanceToNext = &d - } - } - - return path, nil -} - -func neighborVecByConcept(neighbors []*txt2vecmodels.NearestNeighbor, concept string) ([]float32, bool) { - for _, n := range neighbors { - if n.Concept == concept { - return n.Vector, true - } - } - - return nil, false -} - -func cosineSim(a, b []float32) (float32, error) { - if len(a) != len(b) { - return 0, fmt.Errorf("vectors have different dimensions") - } - - var ( - sumProduct float64 - sumASquare float64 - sumBSquare float64 - ) - - for i := range a { - sumProduct += float64(a[i] * b[i]) - sumASquare += float64(a[i] * a[i]) - sumBSquare += float64(b[i] * b[i]) - } - - return float32(sumProduct / (math.Sqrt(sumASquare) * math.Sqrt(sumBSquare))), nil -} - -func cosineDist(a, b []float32) (float32, error) { - sim, err := cosineSim(a, b) - if err != nil { - return 0, err - } - - return 1 - sim, nil -} diff --git a/modules/text2vec-contextionary/additional/sempath/builder_params.go b/modules/text2vec-contextionary/additional/sempath/builder_params.go deleted file mode 100644 index 22d921253ca6a06fd6fed69d49e987b67bc84968..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/additional/sempath/builder_params.go +++ /dev/null @@ -1,39 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package sempath - -import "github.com/weaviate/weaviate/entities/errorcompounder" - -type Params struct { - SearchVector []float32 -} - -func (p *Params) SetSearchVector(vector []float32) { - p.SearchVector = vector -} - -func (p *Params) SetDefaultsAndValidate(inputSize, dims int) error { - return p.validate(inputSize, dims) -} - -func (p *Params) validate(inputSize, dims int) error { - ec := &errorcompounder.ErrorCompounder{} - if inputSize > 25 { - ec.Addf("result length %d is larger than 25 items: semantic path calculation is only suported up to 25 items, set a limit to <= 25", inputSize) - } - - if p.SearchVector == nil || len(p.SearchVector) == 0 { - ec.Addf("no valid search vector present, got: %v", p.SearchVector) - } - - return ec.ToError() -} diff --git a/modules/text2vec-contextionary/additional/sempath/builder_params_test.go b/modules/text2vec-contextionary/additional/sempath/builder_params_test.go deleted file mode 100644 index 41faf974603fb6a9135d6d7405e4b71ef825ad80..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/additional/sempath/builder_params_test.go +++ /dev/null @@ -1,105 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package sempath - -import "testing" - -func TestParams_validate(t *testing.T) { - type fields struct { - SearchVector []float32 - } - type args struct { - inputSize int - dims int - } - tests := []struct { - name string - fields fields - args args - wantErr bool - }{ - { - name: "Should validate", - fields: fields{ - SearchVector: []float32{1.0}, - }, - args: args{ - inputSize: 25, - dims: 0, - }, - wantErr: false, - }, - { - name: "Should error with empty SearchVector", - fields: fields{ - SearchVector: []float32{}, - }, - args: args{ - inputSize: 25, - dims: 0, - }, - wantErr: true, - }, - { - name: "Should error with nil SearchVector", - fields: fields{}, - args: args{ - inputSize: 25, - dims: 0, - }, - wantErr: true, - }, - { - name: "Should error with with inputSize greater then 25", - fields: fields{ - SearchVector: []float32{1.0}, - }, - args: args{ - inputSize: 26, - dims: 0, - }, - wantErr: true, - }, - { - name: "Should error with with inputSize greater then 25 and nil SearchVector", - fields: fields{ - SearchVector: nil, - }, - args: args{ - inputSize: 26, - dims: 0, - }, - wantErr: true, - }, - { - name: "Should error with with inputSize greater then 25 and empty SearchVector", - fields: fields{ - SearchVector: []float32{}, - }, - args: args{ - inputSize: 26, - dims: 0, - }, - wantErr: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - p := &Params{ - SearchVector: tt.fields.SearchVector, - } - if err := p.validate(tt.args.inputSize, tt.args.dims); (err != nil) != tt.wantErr { - t.Errorf("Params.validate() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} diff --git a/modules/text2vec-contextionary/additional/sempath/builder_test.go b/modules/text2vec-contextionary/additional/sempath/builder_test.go deleted file mode 100644 index 914f125f201397f672a2d483ec55c93b730950d1..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/additional/sempath/builder_test.go +++ /dev/null @@ -1,162 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package sempath - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/search" - txt2vecmodels "github.com/weaviate/weaviate/modules/text2vec-contextionary/additional/models" -) - -func TestSemanticPathBuilder(t *testing.T) { - t.Skip("go1.20 change") - c11y := &fakeC11y{} - b := New(c11y) - - b.fixedSeed = 1000 // control randomness in unit test - - input := []search.Result{ - { - ID: "7fe919ed-2ef6-4087-856c-a307046bf895", - ClassName: "Foo", - Vector: []float32{1, 0.1}, - }, - } - searchVector := []float32{0.3, 0.3} - - c11y.neighbors = []*txt2vecmodels.NearestNeighbors{ - { - Neighbors: []*txt2vecmodels.NearestNeighbor{ - { - Concept: "good1", - Vector: []float32{0.5, 0.1}, - }, - { - Concept: "good2", - Vector: []float32{0.7, 0.2}, - }, - { - Concept: "good3", - Vector: []float32{0.9, 0.1}, - }, - { - Concept: "good4", - Vector: []float32{0.55, 0.1}, - }, - { - Concept: "good5", - Vector: []float32{0.77, 0.2}, - }, - { - Concept: "good6", - Vector: []float32{0.99, 0.1}, - }, - { - Concept: "bad1", - Vector: []float32{-0.1, -3}, - }, - { - Concept: "bad2", - Vector: []float32{-0.15, -2.75}, - }, - { - Concept: "bad3", - Vector: []float32{-0.22, -2.35}, - }, - { - Concept: "bad4", - Vector: []float32{0.1, -3.3}, - }, - { - Concept: "bad5", - Vector: []float32{0.15, -2.5}, - }, - { - Concept: "bad6", - Vector: []float32{-0.4, -2.25}, - }, - }, - }, - } - - res, err := b.CalculatePath(input, &Params{SearchVector: searchVector}) - require.Nil(t, err) - - expectedPath := &txt2vecmodels.SemanticPath{ - Path: []*txt2vecmodels.SemanticPathElement{ - { - Concept: "good5", - DistanceToNext: ptFloat32(0.00029218197), - DistanceToQuery: 0.13783735, - DistanceToResult: 0.011904657, - }, - { - Concept: "good2", - DistanceToNext: ptFloat32(0.014019072), - DistanceToPrevious: ptFloat32(0.00029218197), - DistanceToQuery: 0.12584269, - DistanceToResult: 0.015912116, - }, - { - Concept: "good3", - DistanceToNext: ptFloat32(4.9889088e-05), - DistanceToPrevious: ptFloat32(0.014019072), - DistanceToQuery: 0.21913117, - DistanceToResult: 6.0379505e-05, - }, - { - Concept: "good6", - DistanceToNext: ptFloat32(0.0046744347), - DistanceToPrevious: ptFloat32(4.9889088e-05), - DistanceToQuery: 0.2254098, - DistanceToResult: 5.364418e-07, - }, - { - Concept: "good1", - DistanceToNext: ptFloat32(0.00015383959), - DistanceToPrevious: ptFloat32(0.0046744347), - DistanceToQuery: 0.16794968, - DistanceToResult: 0.004771471, - }, - { - Concept: "good4", - DistanceToPrevious: ptFloat32(0.00015383959), - DistanceToQuery: 0.17780781, - DistanceToResult: 0.003213048, - }, - }, - } - - require.Len(t, res, 1) - require.NotNil(t, res[0].AdditionalProperties) - semanticPath, semanticPathOK := res[0].AdditionalProperties["semanticPath"] - assert.True(t, semanticPathOK) - semanticPathElement, semanticPathElementOK := semanticPath.(*txt2vecmodels.SemanticPath) - assert.True(t, semanticPathElementOK) - assert.Equal(t, expectedPath, semanticPathElement) -} - -type fakeC11y struct { - neighbors []*txt2vecmodels.NearestNeighbors -} - -func (f *fakeC11y) MultiNearestWordsByVector(ctx context.Context, vectors [][]float32, k, n int) ([]*txt2vecmodels.NearestNeighbors, error) { - return f.neighbors, nil -} - -func ptFloat32(in float32) *float32 { - return &in -} diff --git a/modules/text2vec-contextionary/classification/classifier.go b/modules/text2vec-contextionary/classification/classifier.go deleted file mode 100644 index dc9c4785c422fe5f69535cad250aadbe2a7fa357..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/classification/classifier.go +++ /dev/null @@ -1,122 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package classification - -import ( - "context" - "encoding/json" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" -) - -type vectorizer interface { - // MultiVectorForWord must keep order, if an item cannot be vectorized, the - // element should be explicit nil, not skipped - MultiVectorForWord(ctx context.Context, words []string) ([][]float32, error) - VectorOnlyForCorpi(ctx context.Context, corpi []string, overrides map[string]string) ([]float32, error) -} - -type Classifier struct { - vectorizer vectorizer -} - -func New(vectorizer vectorizer) modulecapabilities.Classifier { - return &Classifier{vectorizer: vectorizer} -} - -func (c *Classifier) Name() string { - return "text2vec-contextionary-contextual" -} - -func (c *Classifier) ClassifyFn(params modulecapabilities.ClassifyParams) (modulecapabilities.ClassifyItemFn, error) { - if c.vectorizer == nil { - return nil, errors.Errorf("cannot use text2vec-contextionary-contextual " + - "without the respective module") - } - - // 1. do preparation here once - preparedContext, err := c.prepareContextualClassification(params.Schema, params.VectorRepo, - params.Params, params.Filters, params.UnclassifiedItems) - if err != nil { - return nil, errors.Wrap(err, "prepare context for text2vec-contextionary-contextual classification") - } - - // 2. use higher order function to inject preparation data so it is then present for each single run - return c.makeClassifyItemContextual(params.Schema, preparedContext), nil -} - -func (c *Classifier) ParseClassifierSettings(params *models.Classification) error { - raw := params.Settings - settings := &ParamsContextual{} - if raw == nil { - settings.SetDefaults() - params.Settings = settings - return nil - } - - asMap, ok := raw.(map[string]interface{}) - if !ok { - return errors.Errorf("settings must be an object got %T", raw) - } - - v, err := c.extractNumberFromMap(asMap, "minimumUsableWords") - if err != nil { - return err - } - settings.MinimumUsableWords = v - - v, err = c.extractNumberFromMap(asMap, "informationGainCutoffPercentile") - if err != nil { - return err - } - settings.InformationGainCutoffPercentile = v - - v, err = c.extractNumberFromMap(asMap, "informationGainMaximumBoost") - if err != nil { - return err - } - settings.InformationGainMaximumBoost = v - - v, err = c.extractNumberFromMap(asMap, "tfidfCutoffPercentile") - if err != nil { - return err - } - settings.TfidfCutoffPercentile = v - - settings.SetDefaults() - params.Settings = settings - - return nil -} - -func (c *Classifier) extractNumberFromMap(in map[string]interface{}, field string) (*int32, error) { - unparsed, present := in[field] - if present { - parsed, ok := unparsed.(json.Number) - if !ok { - return nil, errors.Errorf("settings.%s must be number, got %T", - field, unparsed) - } - - asInt64, err := parsed.Int64() - if err != nil { - return nil, errors.Wrapf(err, "settings.%s", field) - } - - asInt32 := int32(asInt64) - return &asInt32, nil - } - - return nil, nil -} diff --git a/modules/text2vec-contextionary/classification/classifier_misc.go b/modules/text2vec-contextionary/classification/classifier_misc.go deleted file mode 100644 index 531189fb44d5d2655d0a1b7dc47c7050545b2c56..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/classification/classifier_misc.go +++ /dev/null @@ -1,21 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package classification - -import ( - "context" - "time" -) - -func contextWithTimeout(d time.Duration) (context.Context, context.CancelFunc) { - return context.WithTimeout(context.Background(), d) -} diff --git a/modules/text2vec-contextionary/classification/classifier_params.go b/modules/text2vec-contextionary/classification/classifier_params.go deleted file mode 100644 index 6a0b8a9462f5053b823784a0e68ed7241d57eaab..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/classification/classifier_params.go +++ /dev/null @@ -1,41 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package classification - -type ParamsContextual struct { - MinimumUsableWords *int32 `json:"minimumUsableWords"` - InformationGainCutoffPercentile *int32 `json:"informationGainCutoffPercentile"` - InformationGainMaximumBoost *int32 `json:"informationGainMaximumBoost"` - TfidfCutoffPercentile *int32 `json:"tfidfCutoffPercentile"` -} - -func (params *ParamsContextual) SetDefaults() { - if params.MinimumUsableWords == nil { - defaultParam := int32(3) - params.MinimumUsableWords = &defaultParam - } - - if params.InformationGainCutoffPercentile == nil { - defaultParam := int32(50) - params.InformationGainCutoffPercentile = &defaultParam - } - - if params.InformationGainMaximumBoost == nil { - defaultParam := int32(3) - params.InformationGainMaximumBoost = &defaultParam - } - - if params.TfidfCutoffPercentile == nil { - defaultParam := int32(80) - params.TfidfCutoffPercentile = &defaultParam - } -} diff --git a/modules/text2vec-contextionary/classification/classifier_prepare_contextual.go b/modules/text2vec-contextionary/classification/classifier_prepare_contextual.go deleted file mode 100644 index dd1770284811826ac3097018c551bfa3fe995e7d..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/classification/classifier_prepare_contextual.go +++ /dev/null @@ -1,170 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package classification - -import ( - "fmt" - "time" - - libfilters "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/search" - libclassification "github.com/weaviate/weaviate/usecases/classification" -) - -type tfidfScorer interface { - GetAllTerms(docIndex int) []TermWithTfIdf -} - -type contextualPreparationContext struct { - tfidf map[string]tfidfScorer // map[basedOnProp]scorer - targets map[string]search.Results // map[classifyProp]targets -} - -func (c *Classifier) prepareContextualClassification(schema schema.Schema, - vectorRepo modulecapabilities.VectorClassSearchRepo, params models.Classification, - filters libclassification.Filters, items search.Results, -) (contextualPreparationContext, error) { - p := &contextualPreparer{ - inputItems: items, - params: params, - repo: vectorRepo, - filters: filters, - schema: schema, - } - - return p.do() -} - -type contextualPreparer struct { - inputItems []search.Result - params models.Classification - repo modulecapabilities.VectorClassSearchRepo - filters libclassification.Filters - schema schema.Schema -} - -func (p *contextualPreparer) do() (contextualPreparationContext, error) { - pctx := contextualPreparationContext{} - - targets, err := p.findTargetsForProps() - if err != nil { - return pctx, err - } - - pctx.targets = targets - - tfidf, err := p.calculateTfidfForProps() - if err != nil { - return pctx, err - } - - pctx.tfidf = tfidf - - return pctx, nil -} - -func (p *contextualPreparer) calculateTfidfForProps() (map[string]tfidfScorer, error) { - props := map[string]tfidfScorer{} - - for _, basedOnName := range p.params.BasedOnProperties { - calc := NewTfIdfCalculator(len(p.inputItems)) - for _, obj := range p.inputItems { - schemaMap, ok := obj.Schema.(map[string]interface{}) - if !ok { - return nil, fmt.Errorf("no or incorrect schema map present on source object '%s': %T", obj.ID, obj.Schema) - } - - var docCorpus string - if basedOn, ok := schemaMap[basedOnName]; ok { - basedOnString, ok := basedOn.(string) - if !ok { - return nil, fmt.Errorf("property '%s' present on %s, but of unexpected type: want string, got %T", - basedOnName, obj.ID, basedOn) - } - - docCorpus = basedOnString - } - - calc.AddDoc(docCorpus) - } - - calc.Calculate() - props[basedOnName] = calc - } - - return props, nil -} - -func (p *contextualPreparer) findTargetsForProps() (map[string]search.Results, error) { - targetsMap := map[string]search.Results{} - - for _, targetProp := range p.params.ClassifyProperties { - class, err := p.classAndKindOfTarget(targetProp) - if err != nil { - return nil, fmt.Errorf("target prop '%s': find target class: %v", targetProp, err) - } - - targets, err := p.findTargets(class) - if err != nil { - return nil, fmt.Errorf("target prop '%s': find targets: %v", targetProp, err) - } - - targetsMap[targetProp] = targets - } - - return targetsMap, nil -} - -func (p *contextualPreparer) findTargets(class schema.ClassName) (search.Results, error) { - ctx, cancel := contextWithTimeout(30 * time.Second) - defer cancel() - res, err := p.repo.VectorClassSearch(ctx, modulecapabilities.VectorClassSearchParams{ - Filters: p.filters.Target(), - Pagination: &libfilters.Pagination{ - Limit: 10000, - }, - ClassName: string(class), - Properties: []string{"id"}, - }) - if err != nil { - return nil, fmt.Errorf("search closest target: %v", err) - } - - if len(res) == 0 { - return nil, fmt.Errorf("no potential targets found of class '%s'", class) - } - - return res, nil -} - -func (p *contextualPreparer) classAndKindOfTarget(propName string) (schema.ClassName, error) { - prop, err := p.schema.GetProperty(schema.ClassName(p.params.Class), schema.PropertyName(propName)) - if err != nil { - return "", fmt.Errorf("get target prop '%s': %v", propName, err) - } - - dataType, err := p.schema.FindPropertyDataType(prop.DataType) - if err != nil { - return "", fmt.Errorf("extract dataType of prop '%s': %v", propName, err) - } - - // we have passed validation, so it is safe to assume that this is a ref prop - targetClasses := dataType.Classes() - - // len=1 is guaranteed from validation - targetClass := targetClasses[0] - - return targetClass, nil -} diff --git a/modules/text2vec-contextionary/classification/classifier_run_contextual.go b/modules/text2vec-contextionary/classification/classifier_run_contextual.go deleted file mode 100644 index d5a87c0a9a0b5c9d4680ada629070b044f39e229..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/classification/classifier_run_contextual.go +++ /dev/null @@ -1,433 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package classification - -import ( - "fmt" - "math" - "sort" - "strings" - "time" - - "github.com/go-openapi/strfmt" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/schema/crossref" - "github.com/weaviate/weaviate/entities/search" -) - -// TODO: all of this must be served by the module in the future -type contextualItemClassifier struct { - item search.Result - itemIndex int - params models.Classification - settings *ParamsContextual - classifier *Classifier - writer modulecapabilities.Writer - schema schema.Schema - filters modulecapabilities.Filters - context contextualPreparationContext - vectorizer vectorizer - words []string - rankedWords map[string][]scoredWord // map[targetProp]words as scoring/ranking is per target -} - -func (c *Classifier) extendItemWithObjectMeta(item *search.Result, - params models.Classification, classified []string, -) { - // don't overwrite existing non-classification meta info - if item.AdditionalProperties == nil { - item.AdditionalProperties = models.AdditionalProperties{} - } - - item.AdditionalProperties["classification"] = additional.Classification{ - ID: params.ID, - Scope: params.ClassifyProperties, - ClassifiedFields: classified, - Completed: strfmt.DateTime(time.Now()), - } -} - -// makeClassifyItemContextual is a higher-order function to produce the actual -// classify function, but additionally allows us to inject data which is valid -// for the entire run, such as tf-idf data and target vectors -func (c *Classifier) makeClassifyItemContextual(schema schema.Schema, preparedContext contextualPreparationContext) func(search.Result, - int, models.Classification, modulecapabilities.Filters, modulecapabilities.Writer) error { - return func(item search.Result, itemIndex int, params models.Classification, - filters modulecapabilities.Filters, writer modulecapabilities.Writer, - ) error { - vectorizer := c.vectorizer - run := &contextualItemClassifier{ - item: item, - itemIndex: itemIndex, - params: params, - settings: params.Settings.(*ParamsContextual), // safe assertion after parsing - classifier: c, - writer: writer, - schema: schema, - filters: filters, - context: preparedContext, - vectorizer: vectorizer, - rankedWords: map[string][]scoredWord{}, - } - - err := run.do() - if err != nil { - return fmt.Errorf("text2vec-contextionary-contextual: %v", err) - } - - return nil - } -} - -func (c *contextualItemClassifier) do() error { - var classified []string - for _, propName := range c.params.ClassifyProperties { - current, err := c.property(propName) - if err != nil { - return fmt.Errorf("prop '%s': %v", propName, err) - } - - // append list of actually classified (can differ from scope!) properties, - // so we can build the object meta information - classified = append(classified, current) - } - - c.classifier.extendItemWithObjectMeta(&c.item, c.params, classified) - err := c.writer.Store(c.item) - if err != nil { - return fmt.Errorf("store %s/%s: %v", c.item.ClassName, c.item.ID, err) - } - - return nil -} - -func (c *contextualItemClassifier) property(propName string) (string, error) { - targets, ok := c.context.targets[propName] - if !ok || len(targets) == 0 { - return "", fmt.Errorf("have no potential targets for property '%s'", propName) - } - - schemaMap, ok := c.item.Schema.(map[string]interface{}) - if !ok { - return "", fmt.Errorf("no or incorrect schema map present on source c.object '%s': %T", c.item.ID, c.item.Schema) - } - - // Limitation for now, basedOnProperty is always 0 - basedOnName := c.params.BasedOnProperties[0] - basedOn, ok := schemaMap[basedOnName] - if !ok { - return "", fmt.Errorf("property '%s' not found on source c.object '%s': %T", propName, c.item.ID, c.item.Schema) - } - - basedOnString, ok := basedOn.(string) - if !ok { - return "", fmt.Errorf("property '%s' present on %s, but of unexpected type: want string, got %T", - basedOnName, c.item.ID, basedOn) - } - - words := newSplitter().Split(basedOnString) - c.words = words - - ctx, cancel := contextWithTimeout(10 * time.Second) - defer cancel() - - vectors, err := c.vectorizer.MultiVectorForWord(ctx, words) - if err != nil { - return "", fmt.Errorf("vectorize individual words: %v", err) - } - - scoredWords, err := c.scoreWords(words, vectors, propName) - if err != nil { - return "", fmt.Errorf("score words: %v", err) - } - - c.rankedWords[propName] = c.rankAndDedup(scoredWords) - - corpus, boosts, err := c.buildBoostedCorpus(propName) - if err != nil { - return "", fmt.Errorf("build corpus: %v", err) - } - - ctx, cancel = contextWithTimeout(10 * time.Second) - defer cancel() - vector, err := c.vectorizer.VectorOnlyForCorpi(ctx, []string{corpus}, boosts) - if err != nil { - return "", fmt.Errorf("vectorize corpus: %v", err) - } - - target, distance, err := c.findClosestTarget(vector, propName) - if err != nil { - return "", fmt.Errorf("find closest target: %v", err) - } - - targetBeacon := crossref.New("localhost", target.ClassName, target.ID).String() - c.item.Schema.(map[string]interface{})[propName] = models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(targetBeacon), - Classification: &models.ReferenceMetaClassification{ - WinningDistance: float64(distance), - }, - }, - } - - return propName, nil -} - -func (c *contextualItemClassifier) findClosestTarget(query []float32, targetProp string) (*search.Result, float32, error) { - minimum := float32(100000) - var prediction search.Result - - for _, item := range c.context.targets[targetProp] { - dist, err := cosineDist(query, item.Vector) - if err != nil { - return nil, -1, fmt.Errorf("calculate distance: %v", err) - } - - if dist < minimum { - minimum = dist - prediction = item - } - } - - return &prediction, minimum, nil -} - -func (c *contextualItemClassifier) buildBoostedCorpus(targetProp string) (string, map[string]string, error) { - var corpus []string - - for _, word := range c.words { - word = strings.ToLower(word) - - tfscores := c.context.tfidf[c.params.BasedOnProperties[0]].GetAllTerms(c.itemIndex) - // dereferencing these optional parameters is safe, as defaults are - // explicitly set in classifier.Schedule() - if c.isInIgPercentile(int(*c.settings.InformationGainCutoffPercentile), word, targetProp) && - c.isInTfPercentile(tfscores, int(*c.settings.TfidfCutoffPercentile), word) { - corpus = append(corpus, word) - } - } - - // use minimum words if len is currently less - limit := int(*c.settings.MinimumUsableWords) - if len(corpus) < limit { - corpus = c.getTopNWords(targetProp, limit) - } - - corpusStr := strings.ToLower(strings.Join(corpus, " ")) - boosts := c.boostByInformationGain(targetProp, int(*c.settings.InformationGainCutoffPercentile), - float32(*c.settings.InformationGainMaximumBoost)) - return corpusStr, boosts, nil -} - -func (c *contextualItemClassifier) boostByInformationGain(targetProp string, percentile int, - maxBoost float32, -) map[string]string { - cutoff := int(float32(percentile) / float32(100) * float32(len(c.rankedWords[targetProp]))) - out := make(map[string]string, cutoff) - - for i, word := range c.rankedWords[targetProp][:cutoff] { - boost := 1 - float32(math.Log(float64(i)/float64(cutoff)))*float32(1) - if math.IsInf(float64(boost), 1) || boost > maxBoost { - boost = maxBoost - } - - out[word.word] = fmt.Sprintf("%f * w", boost) - } - - return out -} - -type scoredWord struct { - word string - distance float32 - informationGain float32 -} - -func (c *contextualItemClassifier) getTopNWords(targetProp string, limit int) []string { - words := c.rankedWords[targetProp] - - if len(words) < limit { - limit = len(words) - } - - out := make([]string, limit) - for i := 0; i < limit; i++ { - out[i] = words[i].word - } - - return out -} - -func (c *contextualItemClassifier) rankAndDedup(in []*scoredWord) []scoredWord { - return c.dedup(c.rank(in)) -} - -func (c *contextualItemClassifier) dedup(in []scoredWord) []scoredWord { - // simple dedup since it's already ordered, we only need to check the previous element - indexOut := 0 - out := make([]scoredWord, len(in)) - for i, elem := range in { - if i == 0 { - out[indexOut] = elem - indexOut++ - continue - } - - if elem.word == out[indexOut-1].word { - continue - } - - out[indexOut] = elem - indexOut++ - } - - return out[:indexOut] -} - -func (c *contextualItemClassifier) rank(in []*scoredWord) []scoredWord { - i := 0 - filtered := make([]scoredWord, len(in)) - for _, w := range in { - if w == nil { - continue - } - - filtered[i] = *w - i++ - } - out := filtered[:i] - sort.Slice(out, func(a, b int) bool { return out[a].informationGain > out[b].informationGain }) - return out -} - -func (c *contextualItemClassifier) scoreWords(words []string, vectors [][]float32, - targetProp string, -) ([]*scoredWord, error) { - if len(words) != len(vectors) { - return nil, fmt.Errorf("fatal: word list (l=%d) and vector list (l=%d) have different lengths", - len(words), len(vectors)) - } - - out := make([]*scoredWord, len(words)) - for i := range words { - word := strings.ToLower(words[i]) - sw, err := c.scoreWord(word, vectors[i], targetProp) - if err != nil { - return nil, fmt.Errorf("score word '%s': %v", word, err) - } - - // accept nil-entries for now, they will be removed in ranking/deduping - out[i] = sw - } - - return out, nil -} - -func (c *contextualItemClassifier) scoreWord(word string, vector []float32, - targetProp string, -) (*scoredWord, error) { - var all []float32 - minimum := float32(1000000.00) - - if vector == nil { - return nil, nil - } - - targets, ok := c.context.targets[targetProp] - if !ok { - return nil, fmt.Errorf("fatal: targets for prop '%s' not found", targetProp) - } - - for _, target := range targets { - dist, err := cosineDist(vector, target.Vector) - if err != nil { - return nil, fmt.Errorf("calculate cosine distance: %v", err) - } - - all = append(all, dist) - - if dist < minimum { - minimum = dist - } - } - - return &scoredWord{word: word, distance: minimum, informationGain: avg(all) - minimum}, nil -} - -func avg(in []float32) float32 { - var sum float32 - for _, curr := range in { - sum += curr - } - - return sum / float32(len(in)) -} - -func (c *contextualItemClassifier) isInIgPercentile(percentage int, needle string, target string) bool { - cutoff := int(float32(percentage) / float32(100) * float32(len(c.rankedWords[target]))) - - // no need to check if key exists, guaranteed from run - selection := c.rankedWords[target][:cutoff] - - for _, hay := range selection { - if needle == hay.word { - return true - } - } - - return false -} - -func (c *contextualItemClassifier) isInTfPercentile(tf []TermWithTfIdf, percentage int, needle string) bool { - cutoff := int(float32(percentage) / float32(100) * float32(len(tf))) - selection := tf[:cutoff] - - for _, hay := range selection { - if needle == hay.Term { - return true - } - } - - return false -} - -func cosineSim(a, b []float32) (float32, error) { - if len(a) != len(b) { - return 0, fmt.Errorf("vectors have different dimensions") - } - - var ( - sumProduct float64 - sumASquare float64 - sumBSquare float64 - ) - - for i := range a { - sumProduct += float64(a[i] * b[i]) - sumASquare += float64(a[i] * a[i]) - sumBSquare += float64(b[i] * b[i]) - } - - return float32(sumProduct / (math.Sqrt(sumASquare) * math.Sqrt(sumBSquare))), nil -} - -func cosineDist(a, b []float32) (float32, error) { - sim, err := cosineSim(a, b) - if err != nil { - return 0, err - } - - return 1 - sim, nil -} diff --git a/modules/text2vec-contextionary/classification/classifier_test.go b/modules/text2vec-contextionary/classification/classifier_test.go deleted file mode 100644 index b62974bb68557a404aa50e14d6c2cfda2b0117ce..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/classification/classifier_test.go +++ /dev/null @@ -1,356 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package classification - -import ( - "context" - "encoding/json" - "fmt" - "strings" - "testing" - "time" - - "github.com/go-openapi/strfmt" - "github.com/pkg/errors" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema/crossref" - testhelper "github.com/weaviate/weaviate/test/helper" - usecasesclassfication "github.com/weaviate/weaviate/usecases/classification" -) - -func TestContextualClassifier_ParseSettings(t *testing.T) { - t.Run("should parse with default values with empty settings are passed", func(t *testing.T) { - // given - classifier := New(&fakeVectorizer{}) - params := &models.Classification{ - Class: "Article", - BasedOnProperties: []string{"description"}, - ClassifyProperties: []string{"exactCategory", "mainCategory"}, - Type: "text2vec-contextionary-contextual", - } - - // when - err := classifier.ParseClassifierSettings(params) - - // then - assert.Nil(t, err) - settings := params.Settings - assert.NotNil(t, settings) - paramsContextual, ok := settings.(*ParamsContextual) - assert.NotNil(t, paramsContextual) - assert.True(t, ok) - assert.Equal(t, int32(3), *paramsContextual.MinimumUsableWords) - assert.Equal(t, int32(50), *paramsContextual.InformationGainCutoffPercentile) - assert.Equal(t, int32(3), *paramsContextual.InformationGainMaximumBoost) - assert.Equal(t, int32(80), *paramsContextual.TfidfCutoffPercentile) - }) - - t.Run("should parse classifier settings", func(t *testing.T) { - // given - classifier := New(&fakeVectorizer{}) - params := &models.Classification{ - Class: "Article", - BasedOnProperties: []string{"description"}, - ClassifyProperties: []string{"exactCategory", "mainCategory"}, - Type: "text2vec-contextionary-contextual", - Settings: map[string]interface{}{ - "minimumUsableWords": json.Number("1"), - "informationGainCutoffPercentile": json.Number("2"), - "informationGainMaximumBoost": json.Number("3"), - "tfidfCutoffPercentile": json.Number("4"), - }, - } - - // when - err := classifier.ParseClassifierSettings(params) - - // then - assert.Nil(t, err) - assert.NotNil(t, params.Settings) - settings, ok := params.Settings.(*ParamsContextual) - assert.NotNil(t, settings) - assert.True(t, ok) - assert.Equal(t, int32(1), *settings.MinimumUsableWords) - assert.Equal(t, int32(2), *settings.InformationGainCutoffPercentile) - assert.Equal(t, int32(3), *settings.InformationGainMaximumBoost) - assert.Equal(t, int32(4), *settings.TfidfCutoffPercentile) - }) -} - -func TestContextualClassifier_Classify(t *testing.T) { - var id strfmt.UUID - // so we can reuse it for follow up requests, such as checking the status - - t.Run("with valid data", func(t *testing.T) { - sg := &fakeSchemaGetter{testSchema()} - repo := newFakeClassificationRepo() - authorizer := &fakeAuthorizer{} - - vectorRepo := newFakeVectorRepoContextual(testDataToBeClassified(), testDataPossibleTargets()) - logger, _ := test.NewNullLogger() - - vectorizer := &fakeVectorizer{words: testDataVectors()} - modulesProvider := NewFakeModulesProvider(vectorizer) - classifier := usecasesclassfication.New(sg, repo, vectorRepo, authorizer, logger, modulesProvider) - - contextual := "text2vec-contextionary-contextual" - params := models.Classification{ - Class: "Article", - BasedOnProperties: []string{"description"}, - ClassifyProperties: []string{"exactCategory", "mainCategory"}, - Type: contextual, - } - - t.Run("scheduling a classification", func(t *testing.T) { - class, err := classifier.Schedule(context.Background(), nil, params) - require.Nil(t, err, "should not error") - require.NotNil(t, class) - - assert.Len(t, class.ID, 36, "an id was assigned") - id = class.ID - }) - - t.Run("retrieving the same classification by id", func(t *testing.T) { - class, err := classifier.Get(context.Background(), nil, id) - require.Nil(t, err) - require.NotNil(t, class) - assert.Equal(t, id, class.ID) - }) - - // TODO: improve by polling instead - time.Sleep(500 * time.Millisecond) - - t.Run("status is now completed", func(t *testing.T) { - class, err := classifier.Get(context.Background(), nil, id) - require.Nil(t, err) - require.NotNil(t, class) - assert.Equal(t, models.ClassificationStatusCompleted, class.Status) - }) - - t.Run("the classifier updated the actions with the classified references", func(t *testing.T) { - vectorRepo.Lock() - require.Len(t, vectorRepo.db, 6) - vectorRepo.Unlock() - - t.Run("food", func(t *testing.T) { - idArticleFoodOne := "06a1e824-889c-4649-97f9-1ed3fa401d8e" - idArticleFoodTwo := "6402e649-b1e0-40ea-b192-a64eab0d5e56" - - checkRef(t, vectorRepo, idArticleFoodOne, "ExactCategory", "exactCategory", idCategoryFoodAndDrink) - checkRef(t, vectorRepo, idArticleFoodTwo, "MainCategory", "mainCategory", idMainCategoryFoodAndDrink) - }) - - t.Run("politics", func(t *testing.T) { - idArticlePoliticsOne := "75ba35af-6a08-40ae-b442-3bec69b355f9" - idArticlePoliticsTwo := "f850439a-d3cd-4f17-8fbf-5a64405645cd" - - checkRef(t, vectorRepo, idArticlePoliticsOne, "ExactCategory", "exactCategory", idCategoryPolitics) - checkRef(t, vectorRepo, idArticlePoliticsTwo, "MainCategory", "mainCategory", idMainCategoryPoliticsAndSociety) - }) - - t.Run("society", func(t *testing.T) { - idArticleSocietyOne := "a2bbcbdc-76e1-477d-9e72-a6d2cfb50109" - idArticleSocietyTwo := "069410c3-4b9e-4f68-8034-32a066cb7997" - - checkRef(t, vectorRepo, idArticleSocietyOne, "ExactCategory", "exactCategory", idCategorySociety) - checkRef(t, vectorRepo, idArticleSocietyTwo, "MainCategory", "mainCategory", idMainCategoryPoliticsAndSociety) - }) - }) - }) - - t.Run("when errors occur during classification", func(t *testing.T) { - sg := &fakeSchemaGetter{testSchema()} - repo := newFakeClassificationRepo() - authorizer := &fakeAuthorizer{} - vectorRepo := newFakeVectorRepoKNN(testDataToBeClassified(), testDataAlreadyClassified()) - vectorRepo.errorOnAggregate = errors.New("something went wrong") - logger, _ := test.NewNullLogger() - classifier := usecasesclassfication.New(sg, repo, vectorRepo, authorizer, logger, nil) - - params := models.Classification{ - Class: "Article", - BasedOnProperties: []string{"description"}, - ClassifyProperties: []string{"exactCategory", "mainCategory"}, - Settings: map[string]interface{}{ - "k": json.Number("1"), - }, - } - - t.Run("scheduling a classification", func(t *testing.T) { - class, err := classifier.Schedule(context.Background(), nil, params) - require.Nil(t, err, "should not error") - require.NotNil(t, class) - - assert.Len(t, class.ID, 36, "an id was assigned") - id = class.ID - }) - - waitForStatusToNoLongerBeRunning(t, classifier, id) - - t.Run("status is now failed", func(t *testing.T) { - class, err := classifier.Get(context.Background(), nil, id) - require.Nil(t, err) - require.NotNil(t, class) - assert.Equal(t, models.ClassificationStatusFailed, class.Status) - expectedErrStrings := []string{ - "classification failed: ", - "classify Article/75ba35af-6a08-40ae-b442-3bec69b355f9: something went wrong", - "classify Article/f850439a-d3cd-4f17-8fbf-5a64405645cd: something went wrong", - "classify Article/a2bbcbdc-76e1-477d-9e72-a6d2cfb50109: something went wrong", - "classify Article/069410c3-4b9e-4f68-8034-32a066cb7997: something went wrong", - "classify Article/06a1e824-889c-4649-97f9-1ed3fa401d8e: something went wrong", - "classify Article/6402e649-b1e0-40ea-b192-a64eab0d5e56: something went wrong", - } - for _, msg := range expectedErrStrings { - assert.Contains(t, class.Error, msg) - } - }) - }) - - t.Run("when there is nothing to be classified", func(t *testing.T) { - sg := &fakeSchemaGetter{testSchema()} - repo := newFakeClassificationRepo() - authorizer := &fakeAuthorizer{} - vectorRepo := newFakeVectorRepoKNN(nil, testDataAlreadyClassified()) - logger, _ := test.NewNullLogger() - classifier := usecasesclassfication.New(sg, repo, vectorRepo, authorizer, logger, nil) - - params := models.Classification{ - Class: "Article", - BasedOnProperties: []string{"description"}, - ClassifyProperties: []string{"exactCategory", "mainCategory"}, - Settings: map[string]interface{}{ - "k": json.Number("1"), - }, - } - - t.Run("scheduling a classification", func(t *testing.T) { - class, err := classifier.Schedule(context.Background(), nil, params) - require.Nil(t, err, "should not error") - require.NotNil(t, class) - - assert.Len(t, class.ID, 36, "an id was assigned") - id = class.ID - }) - - waitForStatusToNoLongerBeRunning(t, classifier, id) - - t.Run("status is now failed", func(t *testing.T) { - class, err := classifier.Get(context.Background(), nil, id) - require.Nil(t, err) - require.NotNil(t, class) - assert.Equal(t, models.ClassificationStatusFailed, class.Status) - expectedErr := "classification failed: " + - "no classes to be classified - did you run a previous classification already?" - assert.Equal(t, expectedErr, class.Error) - }) - }) -} - -func waitForStatusToNoLongerBeRunning(t *testing.T, classifier *usecasesclassfication.Classifier, id strfmt.UUID) { - testhelper.AssertEventuallyEqualWithFrequencyAndTimeout(t, true, func() interface{} { - class, err := classifier.Get(context.Background(), nil, id) - require.Nil(t, err) - require.NotNil(t, class) - - return class.Status != models.ClassificationStatusRunning - }, 100*time.Millisecond, 20*time.Second, "wait until status in no longer running") -} - -type genericFakeRepo interface { - get(strfmt.UUID) (*models.Object, bool) -} - -func checkRef(t *testing.T, repo genericFakeRepo, source, targetClass, propName, target string) { - object, ok := repo.get(strfmt.UUID(source)) - require.True(t, ok, "object must be present") - - schema, ok := object.Properties.(map[string]interface{}) - require.True(t, ok, "schema must be map") - - prop, ok := schema[propName] - require.True(t, ok, "ref prop must be present") - - refs, ok := prop.(models.MultipleRef) - require.True(t, ok, "ref prop must be models.MultipleRef") - require.Len(t, refs, 1, "refs must have len 1") - - assert.Equal(t, crossref.NewLocalhost(targetClass, strfmt.UUID(target)).String(), refs[0].Beacon.String(), "beacon must match") -} - -type fakeVectorizer struct { - words map[string][]float32 -} - -func (f *fakeVectorizer) MultiVectorForWord(ctx context.Context, words []string) ([][]float32, error) { - out := make([][]float32, len(words)) - for i, word := range words { - vector, ok := f.words[strings.ToLower(word)] - if !ok { - continue - } - out[i] = vector - } - return out, nil -} - -func (f *fakeVectorizer) VectorOnlyForCorpi(ctx context.Context, corpi []string, - overrides map[string]string, -) ([]float32, error) { - words := strings.Split(corpi[0], " ") - if len(words) == 0 { - return nil, fmt.Errorf("vector for corpi called without words") - } - - vectors, _ := f.MultiVectorForWord(ctx, words) - - return f.centroid(vectors, words) -} - -func (f *fakeVectorizer) centroid(in [][]float32, words []string) ([]float32, error) { - withoutNilVectors := make([][]float32, len(in)) - if len(in) == 0 { - return nil, fmt.Errorf("got nil vector list for words: %v", words) - } - - i := 0 - for _, vec := range in { - if vec == nil { - continue - } - - withoutNilVectors[i] = vec - i++ - } - withoutNilVectors = withoutNilVectors[:i] - if i == 0 { - return nil, fmt.Errorf("no usable words: %v", words) - } - - // take the first vector assuming all have the same length - out := make([]float32, len(withoutNilVectors[0])) - - for _, vec := range withoutNilVectors { - for i, dim := range vec { - out[i] = out[i] + dim - } - } - - for i, sum := range out { - out[i] = sum / float32(len(withoutNilVectors)) - } - - return out, nil -} diff --git a/modules/text2vec-contextionary/classification/fakes_for_test.go b/modules/text2vec-contextionary/classification/fakes_for_test.go deleted file mode 100644 index 9a21d2fe7eb99fa1d1aa122aff87b43cc3f06680..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/classification/fakes_for_test.go +++ /dev/null @@ -1,350 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package classification - -import ( - "context" - "fmt" - "sort" - "sync" - "time" - - "github.com/go-openapi/strfmt" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/dto" - libfilters "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/search" - usecasesclassfication "github.com/weaviate/weaviate/usecases/classification" - "github.com/weaviate/weaviate/usecases/objects" - "github.com/weaviate/weaviate/usecases/sharding" -) - -type fakeSchemaGetter struct { - schema schema.Schema -} - -func (f *fakeSchemaGetter) GetSchemaSkipAuth() schema.Schema { - return f.schema -} - -func (f *fakeSchemaGetter) CopyShardingState(class string) *sharding.State { - panic("not implemented") -} - -func (f *fakeSchemaGetter) ShardOwner(class, shard string) (string, error) { return "", nil } -func (f *fakeSchemaGetter) ShardReplicas(class, shard string) ([]string, error) { return nil, nil } - -func (f *fakeSchemaGetter) TenantShard(class, tenant string) (string, string) { - return tenant, models.TenantActivityStatusHOT -} -func (f *fakeSchemaGetter) ShardFromUUID(class string, uuid []byte) string { return "" } - -func (f *fakeSchemaGetter) Nodes() []string { - panic("not implemented") -} - -func (f *fakeSchemaGetter) NodeName() string { - panic("not implemented") -} - -func (f *fakeSchemaGetter) ClusterHealthScore() int { - panic("not implemented") -} - -func (f *fakeSchemaGetter) ResolveParentNodes(string, string, -) (map[string]string, error) { - panic("not implemented") -} - -type fakeClassificationRepo struct { - sync.Mutex - db map[strfmt.UUID]models.Classification -} - -func newFakeClassificationRepo() *fakeClassificationRepo { - return &fakeClassificationRepo{ - db: map[strfmt.UUID]models.Classification{}, - } -} - -func (f *fakeClassificationRepo) Put(ctx context.Context, class models.Classification) error { - f.Lock() - defer f.Unlock() - - f.db[class.ID] = class - return nil -} - -func (f *fakeClassificationRepo) Get(ctx context.Context, id strfmt.UUID) (*models.Classification, error) { - f.Lock() - defer f.Unlock() - - class, ok := f.db[id] - if !ok { - return nil, nil - } - - return &class, nil -} - -func newFakeVectorRepoKNN(unclassified, classified search.Results) *fakeVectorRepoKNN { - return &fakeVectorRepoKNN{ - unclassified: unclassified, - classified: classified, - db: map[strfmt.UUID]*models.Object{}, - } -} - -// read requests are specified through unclassified and classified, -// write requests (Put[Kind]) are stored in the db map -type fakeVectorRepoKNN struct { - sync.Mutex - unclassified []search.Result - classified []search.Result - db map[strfmt.UUID]*models.Object - errorOnAggregate error - batchStorageDelay time.Duration -} - -func (f *fakeVectorRepoKNN) GetUnclassified(ctx context.Context, - class string, properties []string, - filter *libfilters.LocalFilter, -) ([]search.Result, error) { - f.Lock() - defer f.Unlock() - return f.unclassified, nil -} - -func (f *fakeVectorRepoKNN) AggregateNeighbors(ctx context.Context, vector []float32, - class string, properties []string, k int, - filter *libfilters.LocalFilter, -) ([]usecasesclassfication.NeighborRef, error) { - f.Lock() - defer f.Unlock() - - // simulate that this takes some time - time.Sleep(1 * time.Millisecond) - - if k != 1 { - return nil, fmt.Errorf("fake vector repo only supports k=1") - } - - results := f.classified - sort.SliceStable(results, func(i, j int) bool { - simI, err := cosineSim(results[i].Vector, vector) - if err != nil { - panic(err.Error()) - } - - simJ, err := cosineSim(results[j].Vector, vector) - if err != nil { - panic(err.Error()) - } - return simI > simJ - }) - - var out []usecasesclassfication.NeighborRef - schema := results[0].Schema.(map[string]interface{}) - for _, propName := range properties { - prop, ok := schema[propName] - if !ok { - return nil, fmt.Errorf("missing prop %s", propName) - } - - refs := prop.(models.MultipleRef) - if len(refs) != 1 { - return nil, fmt.Errorf("wrong length %d", len(refs)) - } - - out = append(out, usecasesclassfication.NeighborRef{ - Beacon: refs[0].Beacon, - WinningCount: 1, - OverallCount: 1, - LosingCount: 1, - Property: propName, - }) - } - - return out, f.errorOnAggregate -} - -func (f *fakeVectorRepoKNN) ZeroShotSearch(ctx context.Context, vector []float32, - class string, properties []string, - filter *libfilters.LocalFilter, -) ([]search.Result, error) { - panic("not implemented") -} - -func (f *fakeVectorRepoKNN) VectorSearch(ctx context.Context, - params dto.GetParams, -) ([]search.Result, error) { - f.Lock() - defer f.Unlock() - return nil, fmt.Errorf("vector class search not implemented in fake") -} - -func (f *fakeVectorRepoKNN) BatchPutObjects(ctx context.Context, objects objects.BatchObjects, repl *additional.ReplicationProperties) (objects.BatchObjects, error) { - f.Lock() - defer f.Unlock() - - if f.batchStorageDelay > 0 { - time.Sleep(f.batchStorageDelay) - } - - for _, batchObject := range objects { - f.db[batchObject.Object.ID] = batchObject.Object - } - return objects, nil -} - -func (f *fakeVectorRepoKNN) get(id strfmt.UUID) (*models.Object, bool) { - f.Lock() - defer f.Unlock() - t, ok := f.db[id] - return t, ok -} - -type fakeAuthorizer struct{} - -func (f *fakeAuthorizer) Authorize(principal *models.Principal, verb, resource string) error { - return nil -} - -func newFakeVectorRepoContextual(unclassified, targets search.Results) *fakeVectorRepoContextual { - return &fakeVectorRepoContextual{ - unclassified: unclassified, - targets: targets, - db: map[strfmt.UUID]*models.Object{}, - } -} - -// read requests are specified through unclassified and classified, -// write requests (Put[Kind]) are stored in the db map -type fakeVectorRepoContextual struct { - sync.Mutex - unclassified []search.Result - targets []search.Result - db map[strfmt.UUID]*models.Object - errorOnAggregate error -} - -func (f *fakeVectorRepoContextual) get(id strfmt.UUID) (*models.Object, bool) { - f.Lock() - defer f.Unlock() - t, ok := f.db[id] - return t, ok -} - -func (f *fakeVectorRepoContextual) GetUnclassified(ctx context.Context, - class string, properties []string, - filter *libfilters.LocalFilter, -) ([]search.Result, error) { - return f.unclassified, nil -} - -func (f *fakeVectorRepoContextual) AggregateNeighbors(ctx context.Context, vector []float32, - class string, properties []string, k int, - filter *libfilters.LocalFilter, -) ([]usecasesclassfication.NeighborRef, error) { - panic("not implemented") -} - -func (f *fakeVectorRepoContextual) ZeroShotSearch(ctx context.Context, vector []float32, - class string, properties []string, - filter *libfilters.LocalFilter, -) ([]search.Result, error) { - panic("not implemented") -} - -func (f *fakeVectorRepoContextual) BatchPutObjects(ctx context.Context, objects objects.BatchObjects, repl *additional.ReplicationProperties) (objects.BatchObjects, error) { - f.Lock() - defer f.Unlock() - for _, batchObject := range objects { - f.db[batchObject.Object.ID] = batchObject.Object - } - return objects, nil -} - -func (f *fakeVectorRepoContextual) VectorSearch(ctx context.Context, - params dto.GetParams, -) ([]search.Result, error) { - if params.SearchVector == nil { - filteredTargets := matchClassName(f.targets, params.ClassName) - return filteredTargets, nil - } - - // simulate that this takes some time - time.Sleep(5 * time.Millisecond) - - filteredTargets := matchClassName(f.targets, params.ClassName) - results := filteredTargets - sort.SliceStable(results, func(i, j int) bool { - simI, err := cosineSim(results[i].Vector, params.SearchVector) - if err != nil { - panic(err.Error()) - } - - simJ, err := cosineSim(results[j].Vector, params.SearchVector) - if err != nil { - panic(err.Error()) - } - return simI > simJ - }) - - if len(results) == 0 { - return nil, f.errorOnAggregate - } - - out := []search.Result{ - results[0], - } - - return out, f.errorOnAggregate -} - -func matchClassName(in []search.Result, className string) []search.Result { - var out []search.Result - for _, item := range in { - if item.ClassName == className { - out = append(out, item) - } - } - - return out -} - -type fakeModulesProvider struct { - contextualClassifier modulecapabilities.Classifier -} - -func (fmp *fakeModulesProvider) VectorFromInput(ctx context.Context, className string, input string) ([]float32, error) { - panic("not implemented") -} - -func NewFakeModulesProvider(vectorizer *fakeVectorizer) *fakeModulesProvider { - return &fakeModulesProvider{New(vectorizer)} -} - -func (fmp *fakeModulesProvider) ParseClassifierSettings(name string, - params *models.Classification, -) error { - return fmp.contextualClassifier.ParseClassifierSettings(params) -} - -func (fmp *fakeModulesProvider) GetClassificationFn(className, name string, - params modulecapabilities.ClassifyParams, -) (modulecapabilities.ClassifyItemFn, error) { - return fmp.contextualClassifier.ClassifyFn(params) -} diff --git a/modules/text2vec-contextionary/classification/schema_for_test.go b/modules/text2vec-contextionary/classification/schema_for_test.go deleted file mode 100644 index 7d1e9841b51da2d22ebdc3ab4e683152322b4f6b..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/classification/schema_for_test.go +++ /dev/null @@ -1,234 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package classification - -import ( - "fmt" - - "github.com/go-openapi/strfmt" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/search" -) - -func testSchema() schema.Schema { - return schema.Schema{ - Objects: &models.Schema{ - Classes: []*models.Class{ - { - Class: "ExactCategory", - }, - { - Class: "MainCategory", - }, - { - Class: "Article", - Properties: []*models.Property{ - { - Name: "description", - DataType: []string{string(schema.DataTypeText)}, - }, - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "exactCategory", - DataType: []string{"ExactCategory"}, - }, - { - Name: "mainCategory", - DataType: []string{"MainCategory"}, - }, - { - Name: "categories", - DataType: []string{"ExactCategory"}, - }, - { - Name: "anyCategory", - DataType: []string{"MainCategory", "ExactCategory"}, - }, - }, - }, - }, - }, - } -} - -// vector position close to [1,0,0] means -> politics, [0,1,0] means -> society, [0, 0, 1] -> food&drink -func testDataToBeClassified() search.Results { - return search.Results{ - search.Result{ - ID: "75ba35af-6a08-40ae-b442-3bec69b355f9", - ClassName: "Article", - Vector: []float32{0.78, 0, 0}, - Schema: map[string]interface{}{ - "description": "Barack Obama is a former US president", - }, - }, - search.Result{ - ID: "f850439a-d3cd-4f17-8fbf-5a64405645cd", - ClassName: "Article", - Vector: []float32{0.90, 0, 0}, - Schema: map[string]interface{}{ - "description": "Michelle Obama is Barack Obamas wife", - }, - }, - search.Result{ - ID: "a2bbcbdc-76e1-477d-9e72-a6d2cfb50109", - ClassName: "Article", - Vector: []float32{0, 0.78, 0}, - Schema: map[string]interface{}{ - "description": "Johnny Depp is an actor", - }, - }, - search.Result{ - ID: "069410c3-4b9e-4f68-8034-32a066cb7997", - ClassName: "Article", - Vector: []float32{0, 0.90, 0}, - Schema: map[string]interface{}{ - "description": "Brad Pitt starred in a Quentin Tarantino movie", - }, - }, - search.Result{ - ID: "06a1e824-889c-4649-97f9-1ed3fa401d8e", - ClassName: "Article", - Vector: []float32{0, 0, 0.78}, - Schema: map[string]interface{}{ - "description": "Ice Cream often contains a lot of sugar", - }, - }, - search.Result{ - ID: "6402e649-b1e0-40ea-b192-a64eab0d5e56", - ClassName: "Article", - Vector: []float32{0, 0, 0.90}, - Schema: map[string]interface{}{ - "description": "French Fries are more common in Belgium and the US than in France", - }, - }, - } -} - -func testDataVectors() map[string][]float32 { - return map[string][]float32{ - "barack": {0.7, 0, 0}, - "michelle": {0.7, 0, 0}, - "obama": {1.0, 0, 0}, - "us": {0.6, 0.5, 0.4}, - "depp": {0.1, 0.8, 0.2}, - "actor": {0.1, 0.9, 0.0}, - "brad": {0.1, 0.8, 0.2}, - "starred": {0.1, 0.9, 0.0}, - "ice": {0, 0.1, 0.9}, - "cream": {0, 0.1, 0.8}, - "sugar": {0.3, 0.2, 0.9}, - "french": {0.5, 0.5, 0.4}, - "fries": {0, 0.1, 0.95}, - "belgium": {0.3, 0.3, 0.2}, - } -} - -const ( - idMainCategoryPoliticsAndSociety = "39c6abe3-4bbe-4c4e-9e60-ca5e99ec6b4e" - idMainCategoryFoodAndDrink = "5a3d909a-4f0d-4168-8f5c-cd3074d1e79a" - idCategoryPolitics = "1b204f16-7da6-44fd-bbd2-8cc4a7414bc3" - idCategorySociety = "ec500f39-1dc9-4580-9bd1-55a8ea8e37a2" - idCategoryFoodAndDrink = "027b708a-31ca-43ea-9001-88bec864c79c" -) - -// only used for contextual type classification -func testDataPossibleTargets() search.Results { - return search.Results{ - search.Result{ - ID: idMainCategoryPoliticsAndSociety, - ClassName: "MainCategory", - Vector: []float32{1.01, 1.01, 0}, - Schema: map[string]interface{}{ - "name": "Politics and Society", - }, - }, - search.Result{ - ID: idMainCategoryFoodAndDrink, - ClassName: "MainCategory", - Vector: []float32{0, 0, 0.99}, - Schema: map[string]interface{}{ - "name": "Food and Drinks", - }, - }, - search.Result{ - ID: idCategoryPolitics, - ClassName: "ExactCategory", - Vector: []float32{0.99, 0, 0}, - Schema: map[string]interface{}{ - "name": "Politics", - }, - }, - search.Result{ - ID: idCategorySociety, - ClassName: "ExactCategory", - Vector: []float32{0, 0.90, 0}, - Schema: map[string]interface{}{ - "name": "Society", - }, - }, - search.Result{ - ID: idCategoryFoodAndDrink, - ClassName: "ExactCategory", - Vector: []float32{0, 0, 0.99}, - Schema: map[string]interface{}{ - "name": "Food and Drink", - }, - }, - } -} - -func beaconRef(target string) *models.SingleRef { - beacon := fmt.Sprintf("weaviate://localhost/%s", target) - return &models.SingleRef{Beacon: strfmt.URI(beacon)} -} - -// only used for knn-type -func testDataAlreadyClassified() search.Results { - return search.Results{ - search.Result{ - ID: "8aeecd06-55a0-462c-9853-81b31a284d80", - ClassName: "Article", - Vector: []float32{1, 0, 0}, - Schema: map[string]interface{}{ - "description": "This article talks about politics", - "exactCategory": models.MultipleRef{beaconRef(idCategoryPolitics)}, - "mainCategory": models.MultipleRef{beaconRef(idMainCategoryPoliticsAndSociety)}, - }, - }, - search.Result{ - ID: "9f4c1847-2567-4de7-8861-34cf47a071ae", - ClassName: "Article", - Vector: []float32{0, 1, 0}, - Schema: map[string]interface{}{ - "description": "This articles talks about society", - "exactCategory": models.MultipleRef{beaconRef(idCategorySociety)}, - "mainCategory": models.MultipleRef{beaconRef(idMainCategoryPoliticsAndSociety)}, - }, - }, - search.Result{ - ID: "926416ec-8fb1-4e40-ab8c-37b226b3d68e", - ClassName: "Article", - Vector: []float32{0, 0, 1}, - Schema: map[string]interface{}{ - "description": "This article talks about food", - "exactCategory": models.MultipleRef{beaconRef(idCategoryFoodAndDrink)}, - "mainCategory": models.MultipleRef{beaconRef(idMainCategoryFoodAndDrink)}, - }, - }, - } -} diff --git a/modules/text2vec-contextionary/classification/splitter.go b/modules/text2vec-contextionary/classification/splitter.go deleted file mode 100644 index aee47184ca603e8f4da47db10620ad60a5f454d5..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/classification/splitter.go +++ /dev/null @@ -1,32 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package classification - -// TODO: This code is duplicated across weaviate and contextionary which makes -// changes risky. Can we find a single source of truth for this logic - -import ( - "strings" - "unicode" -) - -func newSplitter() *splitter { - return &splitter{} -} - -type splitter struct{} - -func (s *splitter) Split(corpus string) []string { - return strings.FieldsFunc(corpus, func(c rune) bool { - return !unicode.IsLetter(c) && !unicode.IsNumber(c) - }) -} diff --git a/modules/text2vec-contextionary/classification/tf_idf.go b/modules/text2vec-contextionary/classification/tf_idf.go deleted file mode 100644 index 9b8d01dc7cbc3d6b1f4effabb1be4eb92b864fd2..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/classification/tf_idf.go +++ /dev/null @@ -1,192 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package classification - -import ( - "fmt" - "math" - "sort" - "strings" -) - -// warning, not thread-safe for this spike - -type TfIdfCalculator struct { - size int - documents []string - documentLengths []uint - docPointer int - terms map[string][]uint16 - termIdf map[string]float32 -} - -func NewTfIdfCalculator(size int) *TfIdfCalculator { - return &TfIdfCalculator{ - size: size, - documents: make([]string, size), - documentLengths: make([]uint, size), - terms: make(map[string][]uint16), - termIdf: make(map[string]float32), - } -} - -func (c *TfIdfCalculator) AddDoc(doc string) error { - if c.docPointer > c.size { - return fmt.Errorf("doc size exceeded") - } - - c.documents[c.docPointer] = doc - c.docPointer++ - return nil -} - -func (c *TfIdfCalculator) Calculate() { - for i := range c.documents { - c.analyzeDoc(i) - } - - for term, frequencies := range c.terms { - var contained uint - for _, frequency := range frequencies { - if frequency > 0 { - contained++ - } - } - - c.termIdf[term] = float32(math.Log10(float64(c.size) / float64(contained))) - } -} - -func (c *TfIdfCalculator) analyzeDoc(docIndex int) { - terms := newSplitter().Split(c.documents[docIndex]) - for i, term := range terms { - term = strings.ToLower(term) - frequencies := c.getOrInitTerm(term) - frequencies[docIndex] = frequencies[docIndex] + 1 - c.documentLengths[docIndex] = uint(i + 1) - c.terms[term] = frequencies - } -} - -func (c *TfIdfCalculator) getOrInitTerm(term string) []uint16 { - frequencies, ok := c.terms[term] - if !ok { - frequencies := make([]uint16, c.size) - c.terms[term] = frequencies - return frequencies - } - - return frequencies -} - -func (c *TfIdfCalculator) Get(term string, doc int) float32 { - term = strings.ToLower(term) - frequencies, ok := c.terms[term] - if !ok { - return 0 - } - - tf := float32(frequencies[doc]) / float32(c.documentLengths[doc]) - idf := c.termIdf[term] - - return tf * idf -} - -func (c *TfIdfCalculator) GetAllTerms(docIndex int) []TermWithTfIdf { - terms := newSplitter().Split(c.documents[docIndex]) - terms = c.lowerCaseAndDedup(terms) - - out := make([]TermWithTfIdf, len(terms)) - for i, term := range terms { - out[i] = TermWithTfIdf{ - Term: term, - TfIdf: c.Get(term, docIndex), - } - } - - sort.Slice(out, func(a, b int) bool { return out[a].TfIdf > out[b].TfIdf }) - return c.withRelativeScores(out) -} - -type TermWithTfIdf struct { - Term string - TfIdf float32 - RelativeScore float32 -} - -func (c *TfIdfCalculator) withRelativeScores(list []TermWithTfIdf) []TermWithTfIdf { - // mean for variance - var mean float64 - for _, t := range list { - mean += float64(t.TfIdf) - } - mean = mean / float64(len(list)) - - // calculate variance - for i, t := range list { - variance := math.Pow(float64(t.TfIdf)-mean, 2) - if float64(t.TfIdf) < mean { - list[i].RelativeScore = float32(-variance) - } else { - list[i].RelativeScore = float32(variance) - } - } - - return c.withNormalizedScores(list) -} - -// between -1 and 1 -func (c *TfIdfCalculator) withNormalizedScores(list []TermWithTfIdf) []TermWithTfIdf { - max, min := c.maxMin(list) - - for i, curr := range list { - score := (curr.RelativeScore - min) / (max - min) - list[i].RelativeScore = (score - 0.5) * 2 - } - - return list -} - -func (c *TfIdfCalculator) maxMin(list []TermWithTfIdf) (float32, float32) { - max := list[0].RelativeScore - min := list[0].RelativeScore - - for _, curr := range list { - if curr.RelativeScore > max { - max = curr.RelativeScore - } - if curr.RelativeScore < min { - min = curr.RelativeScore - } - } - - return max, min -} - -func (c *TfIdfCalculator) lowerCaseAndDedup(list []string) []string { - seen := map[string]struct{}{} - out := make([]string, len(list)) - i := 0 - for _, term := range list { - term = strings.ToLower(term) - _, ok := seen[term] - if ok { - continue - } - - seen[term] = struct{}{} - out[i] = term - i++ - } - - return out[:i] -} diff --git a/modules/text2vec-contextionary/classification/tf_idf_test.go b/modules/text2vec-contextionary/classification/tf_idf_test.go deleted file mode 100644 index e6b77f7987ce00ca71b6faae9c68772c9cccba5f..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/classification/tf_idf_test.go +++ /dev/null @@ -1,103 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package classification - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestTfidf(t *testing.T) { - docs := []string{ - "this pinot wine is a pinot noir", - "this one is a cabernet sauvignon", - "this wine is a cabernet franc", - "this one is a merlot", - } - - calc := NewTfIdfCalculator(len(docs)) - for _, doc := range docs { - calc.AddDoc(doc) - } - calc.Calculate() - - t.Run("doc 0", func(t *testing.T) { - doc := 0 - - // filler words should have score of 0 - assert.Equal(t, float32(0), calc.Get("this", doc)) - assert.Equal(t, float32(0), calc.Get("is", doc)) - assert.Equal(t, float32(0), calc.Get("a", doc)) - - // next highest should be wine, noir, pinot - wine := calc.Get("wine", doc) - noir := calc.Get("noir", doc) - pinot := calc.Get("pinot", doc) - - assert.True(t, wine > 0, "wine greater 0") - assert.True(t, noir > wine, "noir greater than wine") - assert.True(t, pinot > noir, "pinot has highest score") - }) - - t.Run("doc 1", func(t *testing.T) { - doc := 1 - - // filler words should have score of 0 - assert.Equal(t, float32(0), calc.Get("this", doc)) - assert.Equal(t, float32(0), calc.Get("is", doc)) - assert.Equal(t, float32(0), calc.Get("a", doc)) - - // next highest should be one==cabernet, sauvignon - one := calc.Get("one", doc) - cabernet := calc.Get("cabernet", doc) - sauvignon := calc.Get("sauvignon", doc) - - assert.True(t, one > 0, "one greater 0") - assert.True(t, cabernet == one, "cabernet equal to one") - assert.True(t, sauvignon > cabernet, "sauvignon has highest score") - }) - - t.Run("doc 2", func(t *testing.T) { - doc := 2 - - // filler words should have score of 0 - assert.Equal(t, float32(0), calc.Get("this", doc)) - assert.Equal(t, float32(0), calc.Get("is", doc)) - assert.Equal(t, float32(0), calc.Get("a", doc)) - - // next highest should be one==cabernet, sauvignon - wine := calc.Get("wine", doc) - cabernet := calc.Get("cabernet", doc) - franc := calc.Get("franc", doc) - - assert.True(t, wine > 0, "wine greater 0") - assert.True(t, cabernet == wine, "cabernet equal to wine") - assert.True(t, franc > cabernet, "franc has highest score") - }) - - t.Run("doc 3", func(t *testing.T) { - doc := 3 - - // filler words should have score of 0 - assert.Equal(t, float32(0), calc.Get("this", doc)) - assert.Equal(t, float32(0), calc.Get("is", doc)) - assert.Equal(t, float32(0), calc.Get("a", doc)) - - // next highest should be one==cabernet, sauvignon - one := calc.Get("one", doc) - merlot := calc.Get("merlot", doc) - - assert.True(t, one > 0, "one greater 0") - assert.True(t, merlot > one, "merlot has highest score") - }) -} diff --git a/modules/text2vec-contextionary/client/contextionary.go b/modules/text2vec-contextionary/client/contextionary.go deleted file mode 100644 index 3dbb367a095a422d6597ed1a7b5b861de7e133c8..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/client/contextionary.go +++ /dev/null @@ -1,374 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package client - -import ( - "context" - "fmt" - "strings" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - pb "github.com/weaviate/contextionary/contextionary" - "github.com/weaviate/weaviate/entities/models" - txt2vecmodels "github.com/weaviate/weaviate/modules/text2vec-contextionary/additional/models" - "github.com/weaviate/weaviate/modules/text2vec-contextionary/vectorizer" - "github.com/weaviate/weaviate/usecases/traverser" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/credentials/insecure" - "google.golang.org/grpc/status" -) - -const ModelUncontactable = "module uncontactable" - -// Client establishes a gRPC connection to a remote contextionary service -type Client struct { - grpcClient pb.ContextionaryClient - logger logrus.FieldLogger -} - -// NewClient from gRPC discovery url to connect to a remote contextionary service -func NewClient(uri string, logger logrus.FieldLogger) (*Client, error) { - conn, err := grpc.Dial(uri, - grpc.WithTransportCredentials(insecure.NewCredentials()), - grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(1024*1024*48))) - if err != nil { - return nil, fmt.Errorf("couldn't connect to remote contextionary gRPC server: %s", err) - } - - client := pb.NewContextionaryClient(conn) - return &Client{ - grpcClient: client, - logger: logger, - }, nil -} - -// IsStopWord returns true if the given word is a stopword, errors on connection errors -func (c *Client) IsStopWord(ctx context.Context, word string) (bool, error) { - res, err := c.grpcClient.IsWordStopword(ctx, &pb.Word{Word: word}) - if err != nil { - logConnectionRefused(c.logger, err) - return false, err - } - - return res.Stopword, nil -} - -// IsWordPresent returns true if the given word is a stopword, errors on connection errors -func (c *Client) IsWordPresent(ctx context.Context, word string) (bool, error) { - res, err := c.grpcClient.IsWordPresent(ctx, &pb.Word{Word: word}) - if err != nil { - logConnectionRefused(c.logger, err) - return false, err - } - - return res.Present, nil -} - -// SafeGetSimilarWordsWithCertainty will always return a list words - unless there is a network error -func (c *Client) SafeGetSimilarWordsWithCertainty(ctx context.Context, word string, certainty float32) ([]string, error) { - res, err := c.grpcClient.SafeGetSimilarWordsWithCertainty(ctx, &pb.SimilarWordsParams{Word: word, Certainty: certainty}) - if err != nil { - logConnectionRefused(c.logger, err) - return nil, err - } - - output := make([]string, len(res.Words)) - for i, word := range res.Words { - output[i] = word.Word - } - - return output, nil -} - -// SchemaSearch for related classes and properties -// TODO: is this still used? -func (c *Client) SchemaSearch(ctx context.Context, params traverser.SearchParams) (traverser.SearchResults, error) { - pbParams := &pb.SchemaSearchParams{ - Certainty: params.Certainty, - Name: params.Name, - SearchType: searchTypeToProto(params.SearchType), - } - - res, err := c.grpcClient.SchemaSearch(ctx, pbParams) - if err != nil { - logConnectionRefused(c.logger, err) - return traverser.SearchResults{}, err - } - - return schemaSearchResultsFromProto(res), nil -} - -func searchTypeToProto(input traverser.SearchType) pb.SearchType { - switch input { - case traverser.SearchTypeClass: - return pb.SearchType_CLASS - case traverser.SearchTypeProperty: - return pb.SearchType_PROPERTY - default: - panic(fmt.Sprintf("unknown search type %v", input)) - } -} - -func searchTypeFromProto(input pb.SearchType) traverser.SearchType { - switch input { - case pb.SearchType_CLASS: - return traverser.SearchTypeClass - case pb.SearchType_PROPERTY: - return traverser.SearchTypeProperty - default: - panic(fmt.Sprintf("unknown search type %v", input)) - } -} - -func schemaSearchResultsFromProto(res *pb.SchemaSearchResults) traverser.SearchResults { - return traverser.SearchResults{ - Type: searchTypeFromProto(res.Type), - Results: searchResultsFromProto(res.Results), - } -} - -func searchResultsFromProto(input []*pb.SchemaSearchResult) []traverser.SearchResult { - output := make([]traverser.SearchResult, len(input)) - for i, res := range input { - output[i] = traverser.SearchResult{ - Certainty: res.Certainty, - Name: res.Name, - } - } - - return output -} - -func (c *Client) VectorForWord(ctx context.Context, word string) ([]float32, error) { - res, err := c.grpcClient.VectorForWord(ctx, &pb.Word{Word: word}) - if err != nil { - logConnectionRefused(c.logger, err) - return nil, fmt.Errorf("could not get vector from remote: %v", err) - } - v, _, _ := vectorFromProto(res) - return v, nil -} - -func logConnectionRefused(logger logrus.FieldLogger, err error) { - if strings.Contains(fmt.Sprintf("%v", err), "connect: connection refused") { - logger.WithError(err).WithField("module", "contextionary").Warnf(ModelUncontactable) - } else if strings.Contains(err.Error(), "connectex: No connection could be made because the target machine actively refused it.") { - logger.WithError(err).WithField("module", "contextionary").Warnf(ModelUncontactable) - } -} - -func (c *Client) MultiVectorForWord(ctx context.Context, words []string) ([][]float32, error) { - out := make([][]float32, len(words)) - wordParams := make([]*pb.Word, len(words)) - - for i, word := range words { - wordParams[i] = &pb.Word{Word: word} - } - - res, err := c.grpcClient.MultiVectorForWord(ctx, &pb.WordList{Words: wordParams}) - if err != nil { - logConnectionRefused(c.logger, err) - return nil, err - } - - for i, elem := range res.Vectors { - if len(elem.Entries) == 0 { - // indicates word not found - continue - } - - out[i], _, _ = vectorFromProto(elem) - } - - return out, nil -} - -func (c *Client) MultiNearestWordsByVector(ctx context.Context, vectors [][]float32, k, n int) ([]*txt2vecmodels.NearestNeighbors, error) { - out := make([]*txt2vecmodels.NearestNeighbors, len(vectors)) - searchParams := make([]*pb.VectorNNParams, len(vectors)) - - for i, vector := range vectors { - searchParams[i] = &pb.VectorNNParams{ - Vector: vectorToProto(vector), - K: int32(k), - N: int32(n), - } - } - - res, err := c.grpcClient.MultiNearestWordsByVector(ctx, &pb.VectorNNParamsList{Params: searchParams}) - if err != nil { - logConnectionRefused(c.logger, err) - return nil, err - } - - for i, elem := range res.Words { - out[i] = &txt2vecmodels.NearestNeighbors{ - Neighbors: c.extractNeighbors(elem), - } - } - - return out, nil -} - -func (c *Client) extractNeighbors(elem *pb.NearestWords) []*txt2vecmodels.NearestNeighbor { - out := make([]*txt2vecmodels.NearestNeighbor, len(elem.Words)) - - for i := range out { - vec, _, _ := vectorFromProto(elem.Vectors.Vectors[i]) - out[i] = &txt2vecmodels.NearestNeighbor{ - Concept: elem.Words[i], - Distance: elem.Distances[i], - Vector: vec, - } - } - return out -} - -func vectorFromProto(in *pb.Vector) ([]float32, []txt2vecmodels.InterpretationSource, error) { - output := make([]float32, len(in.Entries)) - for i, entry := range in.Entries { - output[i] = entry.Entry - } - - source := make([]txt2vecmodels.InterpretationSource, len(in.Source)) - for i, s := range in.Source { - source[i].Concept = s.Concept - source[i].Weight = float64(s.Weight) - source[i].Occurrence = s.Occurrence - } - - return output, source, nil -} - -func (c *Client) VectorForCorpi(ctx context.Context, corpi []string, overridesMap map[string]string) ([]float32, []txt2vecmodels.InterpretationSource, error) { - overrides := overridesFromMap(overridesMap) - res, err := c.grpcClient.VectorForCorpi(ctx, &pb.Corpi{Corpi: corpi, Overrides: overrides}) - if err != nil { - if strings.Contains(err.Error(), "connect: connection refused") { - c.logger.WithError(err).WithField("module", "contextionary").Warnf(ModelUncontactable) - } else if strings.Contains(err.Error(), "connectex: No connection could be made because the target machine actively refused it.") { - c.logger.WithError(err).WithField("module", "contextionary").Warnf(ModelUncontactable) - } - st, ok := status.FromError(err) - if !ok || st.Code() != codes.InvalidArgument { - return nil, nil, fmt.Errorf("could not get vector from remote: %v", err) - } - - return nil, nil, vectorizer.NewErrNoUsableWordsf(st.Message()) - } - - return vectorFromProto(res) -} - -func (c *Client) VectorOnlyForCorpi(ctx context.Context, corpi []string, overrides map[string]string) ([]float32, error) { - vec, _, err := c.VectorForCorpi(ctx, corpi, overrides) - return vec, err -} - -func (c *Client) NearestWordsByVector(ctx context.Context, vector []float32, n int, k int) ([]string, []float32, error) { - res, err := c.grpcClient.NearestWordsByVector(ctx, &pb.VectorNNParams{ - K: int32(k), - N: int32(n), - Vector: vectorToProto(vector), - }) - if err != nil { - logConnectionRefused(c.logger, err) - return nil, nil, fmt.Errorf("could not get nearest words by vector: %v", err) - } - - return res.Words, res.Distances, nil -} - -func (c *Client) AddExtension(ctx context.Context, extension *models.C11yExtension) error { - _, err := c.grpcClient.AddExtension(ctx, &pb.ExtensionInput{ - Concept: extension.Concept, - Definition: strings.ToLower(extension.Definition), - Weight: extension.Weight, - }) - - return err -} - -func vectorToProto(in []float32) *pb.Vector { - output := make([]*pb.VectorEntry, len(in)) - for i, entry := range in { - output[i] = &pb.VectorEntry{ - Entry: entry, - } - } - - return &pb.Vector{Entries: output} -} - -func (c *Client) WaitForStartupAndValidateVersion(startupCtx context.Context, - requiredMinimumVersion string, interval time.Duration, -) error { - for { - if err := startupCtx.Err(); err != nil { - return errors.Wrap(err, "wait for contextionary remote inference service") - } - - time.Sleep(interval) - - ctx, cancel := context.WithTimeout(startupCtx, 2*time.Second) - defer cancel() - v, err := c.version(ctx) - if err != nil { - c.logger.WithField("action", "startup_check_contextionary").WithError(err). - Warnf("could not connect to contextionary at startup, trying again in 1 sec") - continue - } - - ok, err := extractVersionAndCompare(v, requiredMinimumVersion) - if err != nil { - c.logger.WithField("action", "startup_check_contextionary"). - WithField("requiredMinimumContextionaryVersion", requiredMinimumVersion). - WithField("contextionaryVersion", v). - WithError(err). - Warnf("cannot determine if contextionary version is compatible. " + - "This is fine in development, but probelematic if you see this production") - return nil - } - - if ok { - c.logger.WithField("action", "startup_check_contextionary"). - WithField("requiredMinimumContextionaryVersion", requiredMinimumVersion). - WithField("contextionaryVersion", v). - Infof("found a valid contextionary version") - return nil - } else { - return errors.Errorf("insuffcient contextionary version: need at least %s, got %s", - requiredMinimumVersion, v) - } - } -} - -func overridesFromMap(in map[string]string) []*pb.Override { - if in == nil { - return nil - } - - out := make([]*pb.Override, len(in)) - i := 0 - for key, value := range in { - out[i] = &pb.Override{ - Word: key, - Expression: value, - } - i++ - } - - return out -} diff --git a/modules/text2vec-contextionary/client/meta_provider.go b/modules/text2vec-contextionary/client/meta_provider.go deleted file mode 100644 index 34e53e7574c74cd59f746f3b5a179f7d4fb843e0..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/client/meta_provider.go +++ /dev/null @@ -1,55 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package client - -import ( - "context" - - pb "github.com/weaviate/contextionary/contextionary" -) - -func (c *Client) version(ctx context.Context) (string, error) { - m, err := c.grpcClient.Meta(ctx, &pb.MetaParams{}) - if err != nil { - return "", err - } - - return m.Version, nil -} - -func (c *Client) wordCount(ctx context.Context) (int64, error) { - m, err := c.grpcClient.Meta(ctx, &pb.MetaParams{}) - if err != nil { - return 0, err - } - - return m.WordCount, nil -} - -func (c *Client) MetaInfo() (map[string]interface{}, error) { - c11yVersion, err := c.version(context.Background()) - if err != nil { - return nil, err - } - - c11yWordCount, err := c.wordCount(context.Background()) - if err != nil { - return nil, err - } - - meta := map[string]interface{}{ - "version": c11yVersion, - "wordCount": c11yWordCount, - } - - return meta, nil -} diff --git a/modules/text2vec-contextionary/client/version_checks.go b/modules/text2vec-contextionary/client/version_checks.go deleted file mode 100644 index d6d4b88e1316edd49f0c3e63766a7ab4ae2ed351..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/client/version_checks.go +++ /dev/null @@ -1,76 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package client - -import ( - "fmt" - "regexp" - "strconv" -) - -const ( - inputVersionRegexString = `^.*-v(?P[0-9]+)\.(?P[0-9]+)\.(?P[0-9]+)$` - minimumVersionRegexString = `^(?P[0-9]+)\.(?P[0-9])+\.(?P[0-9]+)$` -) - -func extractVersionAndCompare(input, requiredMin string) (bool, error) { - inputRegexp := regexp.MustCompile(inputVersionRegexString) - minimumRegexp := regexp.MustCompile(minimumVersionRegexString) - - if ok := inputRegexp.MatchString(input); !ok { - return false, fmt.Errorf("unexpected input version tag: %s", input) - } - - if ok := minimumRegexp.MatchString(requiredMin); !ok { - return false, fmt.Errorf("unexpected threshold version tag: %s", requiredMin) - } - - inputMatches := inputRegexp.FindAllStringSubmatch(input, 4) - inputMajor, _ := strconv.Atoi(inputMatches[0][1]) - inputMinor, _ := strconv.Atoi(inputMatches[0][2]) - inputPatch, _ := strconv.Atoi(inputMatches[0][3]) - - minimumMatches := minimumRegexp.FindAllStringSubmatch(requiredMin, 4) - minimumMajor, _ := strconv.Atoi(minimumMatches[0][1]) - minimumMinor, _ := strconv.Atoi(minimumMatches[0][2]) - minimumPatch, _ := strconv.Atoi(minimumMatches[0][3]) - - return compareSemver(inputMajor, inputMinor, inputPatch, minimumMajor, minimumMinor, minimumPatch), nil -} - -func compareSemver(iMaj, iMin, iPat, rMaj, rMin, rPat int) bool { - if iMaj > rMaj { - return true - } - - if iMaj < rMaj { - return false - } - - if iMin > rMin { - return true - } - - if iMin < rMin { - return false - } - - if iPat > rPat { - return true - } - - if iPat < rPat { - return false - } - - return true -} diff --git a/modules/text2vec-contextionary/client/version_checks_test.go b/modules/text2vec-contextionary/client/version_checks_test.go deleted file mode 100644 index 4ba124634bf4a215333ed7b742a49d475f3cb8ae..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/client/version_checks_test.go +++ /dev/null @@ -1,117 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package client - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestExtractVersionAndCompare(t *testing.T) { - type test struct { - input string - requiredMinimum string - expectedIsMet bool - expectedErr error - } - - tests := []test{ - { - input: "notavalidversiontag", - requiredMinimum: "1.2.3", - expectedIsMet: false, - expectedErr: fmt.Errorf("unexpected input version tag: notavalidversiontag"), - }, - { - input: "abc-v0.1.2", - requiredMinimum: "invalidrequired", - expectedIsMet: false, - expectedErr: fmt.Errorf("unexpected threshold version tag: invalidrequired"), - }, - - // valid matches - - // exact match - { - input: "abc-v0.1.2", - requiredMinimum: "0.1.2", - expectedIsMet: true, - expectedErr: nil, - }, - - // every digit bigger - { - input: "abc-v1.2.3", - requiredMinimum: "0.1.2", - expectedIsMet: true, - expectedErr: nil, - }, - - // only major bigger - { - input: "abc-v1.0.0", - requiredMinimum: "0.1.2", - expectedIsMet: true, - expectedErr: nil, - }, - - // only minor bigger - { - input: "abc-v0.2.0", - requiredMinimum: "0.1.2", - expectedIsMet: true, - expectedErr: nil, - }, - - // only patch bigger - { - input: "abc-v0.1.3", - requiredMinimum: "0.1.2", - expectedIsMet: true, - expectedErr: nil, - }, - - // invalid requirements - - // only patch smaller - { - input: "abc-v0.1.1", - requiredMinimum: "0.1.2", - expectedIsMet: false, - expectedErr: nil, - }, - - // only minor smaller - { - input: "abc-v0.0.9", - requiredMinimum: "0.1.2", - expectedIsMet: false, - expectedErr: nil, - }, - - // only major smaller - { - input: "abc-v0.9.9", - requiredMinimum: "1.1.2", - expectedIsMet: false, - expectedErr: nil, - }, - } - - for _, test := range tests { - ok, err := extractVersionAndCompare(test.input, test.requiredMinimum) - assert.Equal(t, test.expectedIsMet, ok) - assert.Equal(t, test.expectedErr, err) - } -} diff --git a/modules/text2vec-contextionary/concepts/rest.go b/modules/text2vec-contextionary/concepts/rest.go deleted file mode 100644 index 0871d9dca25f3531d7464bad84e8cbf580081bca..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/concepts/rest.go +++ /dev/null @@ -1,96 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package concepts - -import ( - "context" - "net/http" - - "github.com/weaviate/weaviate/entities/models" -) - -type RESTHandlers struct { - inspector Inspector -} - -func NewRESTHandlers(inspector Inspector) *RESTHandlers { - return &RESTHandlers{ - inspector: inspector, - } -} - -func (h *RESTHandlers) Handler() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch r.Method { - case http.MethodGet: - h.get(w, r) - default: - w.WriteHeader(http.StatusMethodNotAllowed) - } - }) -} - -func (h *RESTHandlers) get(w http.ResponseWriter, r *http.Request) { - if len(r.URL.String()) == 0 || h.extractConcept(r) == "" { - w.WriteHeader(http.StatusNotFound) - return - } - - h.getOne(w, r) -} - -func (h *RESTHandlers) getOne(w http.ResponseWriter, r *http.Request) { - concept := h.extractConcept(r) - - res, err := h.inspector.GetWords(r.Context(), concept) - if err != nil { - h.writeError(w, err, http.StatusBadRequest) - return - } - - json, err := res.MarshalBinary() - if err != nil { - h.writeError(w, err, http.StatusInternalServerError) - return - } - - w.Header().Add("content-type", "application/json") - w.WriteHeader(http.StatusOK) - w.Write(json) -} - -func (h *RESTHandlers) writeError(w http.ResponseWriter, err error, code int) { - res := &models.ErrorResponse{Error: []*models.ErrorResponseErrorItems0{{ - Message: err.Error(), - }}} - - json, mErr := res.MarshalBinary() - if mErr != nil { - // fallback to text - w.Header().Add("content-type", "text/plain") - w.WriteHeader(code) - w.Write([]byte(err.Error())) - } - - w.Header().Add("content-type", "application/json") - w.WriteHeader(code) - w.Write(json) -} - -func (h *RESTHandlers) extractConcept(r *http.Request) string { - // cutoff leading slash, consider the rest the concept - return r.URL.String()[1:] -} - -type Inspector interface { - GetWords(ctx context.Context, words string) (*models.C11yWordsResponse, error) -} diff --git a/modules/text2vec-contextionary/concepts/rest_test.go b/modules/text2vec-contextionary/concepts/rest_test.go deleted file mode 100644 index 981e37a34dae8848242720e26ebc6d4e765fa88b..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/concepts/rest_test.go +++ /dev/null @@ -1,103 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package concepts - -import ( - "context" - "io" - "net/http" - "net/http/httptest" - "testing" - - "github.com/pkg/errors" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" -) - -func TestHandlers(t *testing.T) { - insp := newFakeInspector() - h := NewRESTHandlers(insp) - - t.Run("without a concept", func(t *testing.T) { - insp.reset() - r := httptest.NewRequest("GET", "/", nil) - w := httptest.NewRecorder() - h.Handler().ServeHTTP(w, r) - - res := w.Result() - defer res.Body.Close() - assert.Equal(t, http.StatusNotFound, res.StatusCode) - }) - - t.Run("without any errors", func(t *testing.T) { - insp.reset() - r := httptest.NewRequest("GET", "/my-concept", nil) - w := httptest.NewRecorder() - h.Handler().ServeHTTP(w, r) - - res := w.Result() - defer res.Body.Close() - json, err := io.ReadAll(res.Body) - require.Nil(t, err) - expected := `{"individualWords":[{` + - `"info":{"vector":[0.1,0.2]},"present":true,"word":"my-concept"}]}` - - assert.Equal(t, http.StatusOK, res.StatusCode) - assert.Equal(t, expected, string(json)) - }) - - t.Run("without an error from the UC", func(t *testing.T) { - insp.reset() - insp.err = errors.Errorf("invalid input") - r := httptest.NewRequest("GET", "/my-concept", nil) - w := httptest.NewRecorder() - h.Handler().ServeHTTP(w, r) - - res := w.Result() - defer res.Body.Close() - json, err := io.ReadAll(res.Body) - require.Nil(t, err) - expected := `{"error":[{"message":"invalid input"}]}` - - assert.Equal(t, http.StatusBadRequest, res.StatusCode) - assert.Equal(t, expected, string(json)) - }) -} - -type fakeInspector struct { - err error -} - -func (f *fakeInspector) reset() { - f.err = nil -} - -func (f *fakeInspector) GetWords(ctx context.Context, - concept string, -) (*models.C11yWordsResponse, error) { - return &models.C11yWordsResponse{ - IndividualWords: []*models.C11yWordsResponseIndividualWordsItems0{ - { - Present: true, - Word: concept, - Info: &models.C11yWordsResponseIndividualWordsItems0Info{ - Vector: []float32{0.1, 0.2}, - }, - }, - }, - }, f.err -} - -func newFakeInspector() *fakeInspector { - return &fakeInspector{} -} diff --git a/modules/text2vec-contextionary/config.go b/modules/text2vec-contextionary/config.go deleted file mode 100644 index fc70287db25f6144d69e9a115cfc380eedbb1c62..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/config.go +++ /dev/null @@ -1,46 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modcontextionary - -import ( - "context" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/modules/text2vec-contextionary/vectorizer" -) - -func (m *ContextionaryModule) ClassConfigDefaults() map[string]interface{} { - return map[string]interface{}{ - "vectorizeClassName": vectorizer.DefaultVectorizeClassName, - } -} - -func (m *ContextionaryModule) PropertyConfigDefaults( - dt *schema.DataType, -) map[string]interface{} { - return map[string]interface{}{ - "skip": !vectorizer.DefaultPropertyIndexed, - "vectorizePropertyName": vectorizer.DefaultVectorizePropertyName, - } -} - -func (m *ContextionaryModule) ValidateClass(ctx context.Context, - class *models.Class, cfg moduletools.ClassConfig, -) error { - icheck := vectorizer.NewIndexChecker(cfg) - return m.configValidator.Do(ctx, class, cfg, icheck) -} - -var _ = modulecapabilities.ClassConfigurator(New()) diff --git a/modules/text2vec-contextionary/doc.go b/modules/text2vec-contextionary/doc.go deleted file mode 100644 index 617a2d7c52bfea3d3f67dbace5503e0812622648..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/doc.go +++ /dev/null @@ -1,17 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// modcontextionary concentrates some of the code that relates to the -// contextionary module, this must be extracted when Weaviate becomes modular. -// -// For now it's just used to concentrate any new contextionary specific code, -// so it's already in one place to make the modularization easier later on -package modcontextionary diff --git a/modules/text2vec-contextionary/extensions/rest_storage.go b/modules/text2vec-contextionary/extensions/rest_storage.go deleted file mode 100644 index 2279ac5eed0e74870d6238739d28eb230cb12f27..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/extensions/rest_storage.go +++ /dev/null @@ -1,142 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package extensions - -import ( - "io" - "net/http" -) - -type RESTHandlers struct { - ls LoaderStorer - proxy Proxy -} - -func NewRESTHandlers(ls LoaderStorer, proxy Proxy) *RESTHandlers { - return &RESTHandlers{ - ls: ls, - proxy: proxy, - } -} - -type RESTStorageHandlers struct { - ls LoaderStorer -} - -func newRESTStorageHandlers(ls LoaderStorer) *RESTStorageHandlers { - return &RESTStorageHandlers{ - ls: ls, - } -} - -func (h *RESTHandlers) StorageHandler() http.Handler { - return newRESTStorageHandlers(h.ls).Handler() -} - -func (h *RESTStorageHandlers) Handler() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch r.Method { - case http.MethodGet: - h.get(w, r) - case http.MethodPut: - h.put(w, r) - default: - w.WriteHeader(http.StatusMethodNotAllowed) - } - }) -} - -func (h *RESTStorageHandlers) get(w http.ResponseWriter, r *http.Request) { - if len(r.URL.String()) == 0 || h.extractConcept(r) == "" { - h.getAll(w, r) - return - } - - h.getOne(w, r) -} - -func (h *RESTStorageHandlers) getOne(w http.ResponseWriter, r *http.Request) { - concept := h.extractConcept(r) - if concept == "" { - w.WriteHeader(http.StatusNotFound) - return - } - - res, err := h.ls.Load(concept) - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(err.Error())) - return - } - - if res == nil { - w.WriteHeader(http.StatusNotFound) - return - } - - w.Write(res) -} - -func (h *RESTStorageHandlers) getAll(w http.ResponseWriter, r *http.Request) { - res, err := h.ls.LoadAll() - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(err.Error())) - return - } - - w.Write(res) -} - -func (h *RESTStorageHandlers) put(w http.ResponseWriter, r *http.Request) { - defer r.Body.Close() - concept := h.extractConcept(r) - if len(concept) == 0 { - w.WriteHeader(http.StatusNotFound) - return - } - - body, err := io.ReadAll(r.Body) - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(err.Error())) - } - - err = h.ls.Store(concept, body) - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(err.Error())) - } -} - -func (h *RESTStorageHandlers) extractConcept(r *http.Request) string { - // cutoff leading slash, consider the rest the concept - return r.URL.String()[1:] -} - -type Storer interface { - Store(concept string, value []byte) error -} - -type Loader interface { - Load(concept string) ([]byte, error) -} - -type LoaderAller interface { - LoadAll() ([]byte, error) -} - -type LoaderStorer interface { - Storer - Loader - LoaderAller -} diff --git a/modules/text2vec-contextionary/extensions/rest_storage_test.go b/modules/text2vec-contextionary/extensions/rest_storage_test.go deleted file mode 100644 index 9cf85e953b0869caaaea6e29e7da1dfc8fd2a977..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/extensions/rest_storage_test.go +++ /dev/null @@ -1,184 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package extensions - -import ( - "bytes" - "fmt" - "net/http" - "net/http/httptest" - "sort" - "testing" - - "github.com/stretchr/testify/assert" -) - -func Test_StorageHandlers(t *testing.T) { - ls := newFakeLoaderStorer() - h := NewRESTHandlers(ls, nil) - - extensionAKey := "my-first-extension" - extensionAValue := []byte("some-value") - - extensionBKey := "my-other-extension" - extensionBValue := []byte("some-other-value") - - t.Run("retrieving a non existent concept", func(t *testing.T) { - r := httptest.NewRequest("GET", "/my-concept", nil) - w := httptest.NewRecorder() - h.StorageHandler().ServeHTTP(w, r) - - res := w.Result() - defer res.Body.Close() - assert.Equal(t, http.StatusNotFound, res.StatusCode) - }) - - t.Run("storing two extensions", func(t *testing.T) { - t.Run("extension A", func(t *testing.T) { - body := bytes.NewReader(extensionAValue) - r := httptest.NewRequest("PUT", fmt.Sprintf("/%s", extensionAKey), body) - w := httptest.NewRecorder() - h.StorageHandler().ServeHTTP(w, r) - - res := w.Result() - defer res.Body.Close() - assert.Equal(t, http.StatusOK, res.StatusCode) - }) - - t.Run("extension B", func(t *testing.T) { - body := bytes.NewReader(extensionBValue) - r := httptest.NewRequest("PUT", fmt.Sprintf("/%s", extensionBKey), body) - w := httptest.NewRecorder() - h.StorageHandler().ServeHTTP(w, r) - - res := w.Result() - defer res.Body.Close() - assert.Equal(t, http.StatusOK, res.StatusCode) - }) - }) - - t.Run("when storing fails", func(t *testing.T) { - ls.storeError = fmt.Errorf("oops") - body := bytes.NewReader(extensionAValue) - r := httptest.NewRequest("PUT", "/some-extension", body) - - w := httptest.NewRecorder() - h.StorageHandler().ServeHTTP(w, r) - - res := w.Result() - defer res.Body.Close() - assert.Equal(t, http.StatusInternalServerError, res.StatusCode) - }) - - t.Run("storing with an empty concept", func(t *testing.T) { - body := bytes.NewReader(extensionAValue) - r := httptest.NewRequest("PUT", "/", body) - - w := httptest.NewRecorder() - h.StorageHandler().ServeHTTP(w, r) - - res := w.Result() - defer res.Body.Close() - assert.Equal(t, http.StatusNotFound, res.StatusCode) - }) - - t.Run("retrieving two extensions", func(t *testing.T) { - t.Run("extension A", func(t *testing.T) { - r := httptest.NewRequest("GET", fmt.Sprintf("/%s", extensionAKey), nil) - w := httptest.NewRecorder() - h.StorageHandler().ServeHTTP(w, r) - - res := w.Result() - defer res.Body.Close() - assert.Equal(t, http.StatusOK, res.StatusCode) - assert.Equal(t, extensionAValue, w.Body.Bytes()) - }) - - t.Run("extension B", func(t *testing.T) { - r := httptest.NewRequest("GET", fmt.Sprintf("/%s", extensionBKey), nil) - w := httptest.NewRecorder() - h.StorageHandler().ServeHTTP(w, r) - - res := w.Result() - defer res.Body.Close() - assert.Equal(t, http.StatusOK, res.StatusCode) - assert.Equal(t, extensionBValue, w.Body.Bytes()) - }) - - t.Run("full dump with trailing slash", func(t *testing.T) { - r := httptest.NewRequest("GET", "/", nil) - w := httptest.NewRecorder() - h.StorageHandler().ServeHTTP(w, r) - expectedValue := []byte("some-value\nsome-other-value\n") - - res := w.Result() - defer res.Body.Close() - assert.Equal(t, http.StatusOK, res.StatusCode) - assert.Equal(t, expectedValue, w.Body.Bytes()) - }) - }) - - t.Run("when loading fails", func(t *testing.T) { - ls.loadError = fmt.Errorf("oops") - body := bytes.NewReader(extensionAValue) - r := httptest.NewRequest("GET", "/some-extension", body) - - w := httptest.NewRecorder() - h.StorageHandler().ServeHTTP(w, r) - - res := w.Result() - defer res.Body.Close() - assert.Equal(t, http.StatusInternalServerError, res.StatusCode) - }) -} - -type fakeLoaderStorer struct { - store map[string][]byte - storeError error - loadError error -} - -func newFakeLoaderStorer() *fakeLoaderStorer { - return &fakeLoaderStorer{ - store: map[string][]byte{}, - } -} - -func (f *fakeLoaderStorer) Store(concept string, value []byte) error { - if f.storeError == nil { - f.store[concept] = value - } - return f.storeError -} - -func (f *fakeLoaderStorer) Load(concept string) ([]byte, error) { - return f.store[concept], f.loadError -} - -func (f *fakeLoaderStorer) LoadAll() ([]byte, error) { - var keys [][]byte - for key := range f.store { - keys = append(keys, []byte(key)) - } - - sort.Slice(keys, func(a, b int) bool { - return bytes.Compare(keys[a], keys[b]) == -1 - }) - - buf := bytes.NewBuffer(nil) - for _, key := range keys { - buf.Write(f.store[string(key)]) - buf.Write([]byte("\n")) - } - - return buf.Bytes(), nil -} diff --git a/modules/text2vec-contextionary/extensions/rest_user_facing.go b/modules/text2vec-contextionary/extensions/rest_user_facing.go deleted file mode 100644 index 070856545de8f0506b018cee3178e3f0db508b0e..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/extensions/rest_user_facing.go +++ /dev/null @@ -1,107 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package extensions - -import ( - "context" - "io" - "net/http" - - "github.com/weaviate/weaviate/entities/models" -) - -type RESTUserFacingHandlers struct { - proxy Proxy -} - -func newRESTUserFacingHandlers(proxy Proxy) *RESTUserFacingHandlers { - return &RESTUserFacingHandlers{ - proxy: proxy, - } -} - -func (h *RESTHandlers) UserFacingHandler() http.Handler { - return newRESTUserFacingHandlers(h.proxy).Handler() -} - -func (h *RESTUserFacingHandlers) Handler() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch r.Method { - case http.MethodPost: - h.post(w, r) - default: - w.WriteHeader(http.StatusMethodNotAllowed) - } - }) -} - -func (h *RESTUserFacingHandlers) post(w http.ResponseWriter, r *http.Request) { - ct := r.Header.Get("content-type") - if ct != "application/json" { - w.WriteHeader(http.StatusUnsupportedMediaType) - return - } - - defer r.Body.Close() - body, err := io.ReadAll(r.Body) - if err != nil { - h.writeError(w, err, http.StatusInternalServerError) - return - } - - var ext models.C11yExtension - if err := (&ext).UnmarshalBinary(body); err != nil { - h.writeError(w, err, http.StatusUnprocessableEntity) - return - } - - if err := h.proxy.AddExtension(r.Context(), &ext); err != nil { - h.writeError(w, err, http.StatusBadRequest) - return - } - - resBody, err := ext.MarshalBinary() - if err != nil { - h.writeError(w, err, http.StatusInternalServerError) - return - } - - w.Header().Add("content-type", "application/json") - w.WriteHeader(http.StatusOK) - w.Write(resBody) -} - -// C11yProxy proxies the request through the separate container, only for it to -// come back here for the storage. This is legacy from the pre-module times. -// TODO: cleanup, there does not need to be a separation between user-facing -// and internal storage endpoint in the long-term -type Proxy interface { - AddExtension(ctx context.Context, extension *models.C11yExtension) error -} - -func (h *RESTUserFacingHandlers) writeError(w http.ResponseWriter, err error, code int) { - res := &models.ErrorResponse{Error: []*models.ErrorResponseErrorItems0{{ - Message: err.Error(), - }}} - - json, mErr := res.MarshalBinary() - if mErr != nil { - // fallback to text - w.Header().Add("content-type", "text/plain") - w.WriteHeader(code) - w.Write([]byte(err.Error())) - } - - w.Header().Add("content-type", "application/json") - w.WriteHeader(code) - w.Write(json) -} diff --git a/modules/text2vec-contextionary/extensions/rest_user_facing_test.go b/modules/text2vec-contextionary/extensions/rest_user_facing_test.go deleted file mode 100644 index 9928fc51bf3fe8643f4276edee1b424197239c18..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/extensions/rest_user_facing_test.go +++ /dev/null @@ -1,107 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package extensions - -import ( - "bytes" - "context" - "io" - "net/http" - "net/http/httptest" - "testing" - - "github.com/pkg/errors" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" -) - -func Test_UserFacingHandlers(t *testing.T) { - proxy := newFakeProxy() - h := NewRESTHandlers(nil, proxy) - - t.Run("with a method other than POST", func(t *testing.T) { - r := httptest.NewRequest("GET", "/", nil) - w := httptest.NewRecorder() - h.UserFacingHandler().ServeHTTP(w, r) - - res := w.Result() - defer res.Body.Close() - assert.Equal(t, http.StatusMethodNotAllowed, res.StatusCode) - }) - - t.Run("with the wrong media type", func(t *testing.T) { - r := httptest.NewRequest("POST", "/", nil) - r.Header.Add("content-type", "text/plain") - w := httptest.NewRecorder() - h.UserFacingHandler().ServeHTTP(w, r) - - res := w.Result() - defer res.Body.Close() - assert.Equal(t, http.StatusUnsupportedMediaType, res.StatusCode) - }) - - t.Run("with the wrong body", func(t *testing.T) { - body := []byte(`{"concept":7}`) - r := httptest.NewRequest("POST", "/", bytes.NewReader(body)) - r.Header.Add("content-type", "application/json") - w := httptest.NewRecorder() - h.UserFacingHandler().ServeHTTP(w, r) - - res := w.Result() - defer res.Body.Close() - assert.Equal(t, http.StatusUnprocessableEntity, res.StatusCode) - }) - - t.Run("with the right body", func(t *testing.T) { - body := []byte(`{"concept":"foo","definition":"bar","weight":1}`) - r := httptest.NewRequest("POST", "/", bytes.NewReader(body)) - r.Header.Add("content-type", "application/json") - w := httptest.NewRecorder() - h.UserFacingHandler().ServeHTTP(w, r) - - res := w.Result() - defer res.Body.Close() - - readBody, err := io.ReadAll(res.Body) - require.Nil(t, err) - assert.Equal(t, http.StatusOK, res.StatusCode) - assert.Equal(t, body, readBody) - }) - - t.Run("with a proxy error", func(t *testing.T) { - proxy.err = errors.Errorf("invalid input") - body := []byte(`{"concept":"foo","definition":"bar","weight":1}`) - r := httptest.NewRequest("POST", "/", bytes.NewReader(body)) - r.Header.Add("content-type", "application/json") - w := httptest.NewRecorder() - h.UserFacingHandler().ServeHTTP(w, r) - - res := w.Result() - defer res.Body.Close() - assert.Equal(t, http.StatusBadRequest, res.StatusCode) - }) -} - -type fakeProxy struct { - err error -} - -func (f *fakeProxy) AddExtension(ctx context.Context, - ext *models.C11yExtension, -) error { - return f.err -} - -func newFakeProxy() *fakeProxy { - return &fakeProxy{} -} diff --git a/modules/text2vec-contextionary/extensions/usecase.go b/modules/text2vec-contextionary/extensions/usecase.go deleted file mode 100644 index eaf398ae0e27cb30a77ccaaa552ffc348e5fb2fd..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/extensions/usecase.go +++ /dev/null @@ -1,70 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package extensions - -import ( - "bytes" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/moduletools" -) - -// UseCase handles all business logic regarding extensions -type UseCase struct { - storage moduletools.Storage -} - -func NewUseCase(storage moduletools.Storage) *UseCase { - return &UseCase{ - storage: storage, - } -} - -func (uc *UseCase) Store(concept string, value []byte) error { - err := uc.storage.Put([]byte(concept), value) - if err != nil { - return errors.Wrapf(err, "store concept %q", concept) - } - - return nil -} - -func (uc *UseCase) Load(concept string) ([]byte, error) { - val, err := uc.storage.Get([]byte(concept)) - if err != nil { - return nil, errors.Wrapf(err, "load concept %q", concept) - } - - return val, nil -} - -func (uc *UseCase) LoadAll() ([]byte, error) { - buf := bytes.NewBuffer(nil) - err := uc.storage.Scan(func(k, v []byte) (bool, error) { - _, err := buf.Write(v) - if err != nil { - return false, errors.Wrapf(err, "write concept %q", string(k)) - } - - _, err = buf.Write([]byte("\n")) - if err != nil { - return false, errors.Wrap(err, "write newline separator") - } - - return true, nil - }) - if err != nil { - return nil, errors.Wrap(err, "load all concepts") - } - - return buf.Bytes(), nil -} diff --git a/modules/text2vec-contextionary/extensions/usecase_test.go b/modules/text2vec-contextionary/extensions/usecase_test.go deleted file mode 100644 index 5ad238b994c85802ddd839d103db383e7fe91e1f..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/extensions/usecase_test.go +++ /dev/null @@ -1,98 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package extensions - -import ( - "bytes" - "fmt" - "sort" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/moduletools" -) - -func Test_UseCase(t *testing.T) { - storage := newFakeStorage() - uc := NewUseCase(storage) - - t.Run("storing and loading something", func(t *testing.T) { - err := uc.Store("concept1", []byte("value1")) - require.Nil(t, err) - - err = uc.Store("concept2", []byte("value2")) - require.Nil(t, err) - - val, err := uc.Load("concept1") - require.Nil(t, err) - assert.Equal(t, []byte("value1"), val) - - val, err = uc.Load("concept2") - require.Nil(t, err) - assert.Equal(t, []byte("value2"), val) - - vals, err := uc.LoadAll() - require.Nil(t, err) - assert.Equal(t, []byte("value1\nvalue2\n"), vals) - }) - - t.Run("when storing fails", func(t *testing.T) { - storage.putError = fmt.Errorf("oops") - err := uc.Store("concept1", []byte("value1")) - assert.Equal(t, "store concept \"concept1\": oops", err.Error()) - }) - - t.Run("when loading fails", func(t *testing.T) { - storage.getError = fmt.Errorf("oops") - _, err := uc.Load("concept1") - assert.Equal(t, "load concept \"concept1\": oops", err.Error()) - }) -} - -func newFakeStorage() *fakeStorage { - return &fakeStorage{ - store: map[string][]byte{}, - } -} - -type fakeStorage struct { - store map[string][]byte - getError error - putError error -} - -func (f *fakeStorage) Get(k []byte) ([]byte, error) { - return f.store[string(k)], f.getError -} - -func (f *fakeStorage) Put(k, v []byte) error { - f.store[string(k)] = v - return f.putError -} - -func (f *fakeStorage) Scan(scan moduletools.ScanFn) error { - var keys [][]byte - for key := range f.store { - keys = append(keys, []byte(key)) - } - - sort.Slice(keys, func(a, b int) bool { - return bytes.Compare(keys[a], keys[b]) == -1 - }) - - for _, key := range keys { - scan(key, f.store[string(key)]) - } - - return nil -} diff --git a/modules/text2vec-contextionary/helpers_test.go b/modules/text2vec-contextionary/helpers_test.go deleted file mode 100644 index 9ef6b213dc0d15199cf59c34af0ace8fbef13c34..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/helpers_test.go +++ /dev/null @@ -1,377 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modcontextionary - -import ( - "context" - "fmt" - "net/http" - - "github.com/sirupsen/logrus/hooks/test" - "github.com/tailor-inc/graphql" - "github.com/tailor-inc/graphql/language/ast" - "github.com/weaviate/weaviate/adapters/handlers/graphql/local/explore" - "github.com/weaviate/weaviate/adapters/handlers/graphql/local/get" - test_helper "github.com/weaviate/weaviate/adapters/handlers/graphql/test/helper" - "github.com/weaviate/weaviate/entities/dto" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/search" - text2vecadditional "github.com/weaviate/weaviate/modules/text2vec-contextionary/additional" - text2vecadditionalsempath "github.com/weaviate/weaviate/modules/text2vec-contextionary/additional/sempath" - text2vecadditionalprojector "github.com/weaviate/weaviate/usecases/modulecomponents/additional/projector" - text2vecneartext "github.com/weaviate/weaviate/usecases/modulecomponents/nearText" - "github.com/weaviate/weaviate/usecases/traverser" -) - -type mockRequestsLog struct{} - -func (m *mockRequestsLog) Register(first string, second string) { -} - -type mockResolver struct { - test_helper.MockResolver -} - -type fakeInterpretation struct{} - -func (f *fakeInterpretation) AdditionalPropertyFn(ctx context.Context, - in []search.Result, params interface{}, limit *int, - argumentModuleParams map[string]interface{}, cfg moduletools.ClassConfig, -) ([]search.Result, error) { - return in, nil -} - -func (f *fakeInterpretation) ExtractAdditionalFn(param []*ast.Argument) interface{} { - return true -} - -func (f *fakeInterpretation) AdditionalPropertyDefaultValue() interface{} { - return true -} - -type fakeExtender struct { - returnArgs []search.Result -} - -func (f *fakeExtender) AdditionalPropertyFn(ctx context.Context, - in []search.Result, params interface{}, limit *int, - argumentModuleParams map[string]interface{}, cfg moduletools.ClassConfig, -) ([]search.Result, error) { - return f.returnArgs, nil -} - -func (f *fakeExtender) ExtractAdditionalFn(param []*ast.Argument) interface{} { - return true -} - -func (f *fakeExtender) AdditionalPropertyDefaultValue() interface{} { - return true -} - -type fakeProjector struct { - returnArgs []search.Result -} - -func (f *fakeProjector) AdditionalPropertyFn(ctx context.Context, - in []search.Result, params interface{}, limit *int, - argumentModuleParams map[string]interface{}, cfg moduletools.ClassConfig, -) ([]search.Result, error) { - return f.returnArgs, nil -} - -func (f *fakeProjector) ExtractAdditionalFn(param []*ast.Argument) interface{} { - if len(param) > 0 { - p := &text2vecadditionalprojector.Params{} - err := p.SetDefaultsAndValidate(100, 4) - if err != nil { - return nil - } - return p - } - return &text2vecadditionalprojector.Params{ - Enabled: true, - } -} - -func (f *fakeProjector) AdditionalPropertyDefaultValue() interface{} { - return &text2vecadditionalprojector.Params{} -} - -type fakePathBuilder struct { - returnArgs []search.Result -} - -func (f *fakePathBuilder) AdditionalPropertyFn(ctx context.Context, - in []search.Result, params interface{}, limit *int, - argumentModuleParams map[string]interface{}, cfg moduletools.ClassConfig, -) ([]search.Result, error) { - return f.returnArgs, nil -} - -func (f *fakePathBuilder) ExtractAdditionalFn(param []*ast.Argument) interface{} { - return &text2vecadditionalsempath.Params{} -} - -func (f *fakePathBuilder) AdditionalPropertyDefaultValue() interface{} { - return &text2vecadditionalsempath.Params{} -} - -type mockText2vecContextionaryModule struct{} - -func (m *mockText2vecContextionaryModule) Name() string { - return "text2vec-contextionary" -} - -func (m *mockText2vecContextionaryModule) Init(params moduletools.ModuleInitParams) error { - return nil -} - -func (m *mockText2vecContextionaryModule) RootHandler() http.Handler { - return nil -} - -// graphql arguments -func (m *mockText2vecContextionaryModule) Arguments() map[string]modulecapabilities.GraphQLArgument { - return text2vecneartext.New(nil).Arguments() -} - -// additional properties -func (m *mockText2vecContextionaryModule) AdditionalProperties() map[string]modulecapabilities.AdditionalProperty { - return text2vecadditional.New(&fakeExtender{}, &fakeProjector{}, &fakePathBuilder{}, &fakeInterpretation{}).AdditionalProperties() -} - -type fakeModulesProvider struct{} - -func (fmp *fakeModulesProvider) GetAll() []modulecapabilities.Module { - panic("implement me") -} - -func (fmp *fakeModulesProvider) VectorFromInput(ctx context.Context, className string, input string) ([]float32, error) { - panic("not implemented") -} - -func (fmp *fakeModulesProvider) GetArguments(class *models.Class) map[string]*graphql.ArgumentConfig { - args := map[string]*graphql.ArgumentConfig{} - txt2vec := &mockText2vecContextionaryModule{} - if class.Vectorizer == txt2vec.Name() { - for name, argument := range txt2vec.Arguments() { - args[name] = argument.GetArgumentsFunction(class.Class) - } - } - return args -} - -func (fmp *fakeModulesProvider) ExploreArguments(schema *models.Schema) map[string]*graphql.ArgumentConfig { - args := map[string]*graphql.ArgumentConfig{} - txt2vec := &mockText2vecContextionaryModule{} - for _, c := range schema.Classes { - if c.Vectorizer == txt2vec.Name() { - for name, argument := range txt2vec.Arguments() { - args[name] = argument.ExploreArgumentsFunction() - } - } - } - return args -} - -func (fmp *fakeModulesProvider) CrossClassExtractSearchParams(arguments map[string]interface{}) map[string]interface{} { - return fmp.ExtractSearchParams(arguments, "") -} - -func (fmp *fakeModulesProvider) ExtractSearchParams(arguments map[string]interface{}, className string) map[string]interface{} { - exractedParams := map[string]interface{}{} - if param, ok := arguments["nearText"]; ok { - exractedParams["nearText"] = extractNearTextParam(param.(map[string]interface{})) - } - return exractedParams -} - -func (fmp *fakeModulesProvider) GetAdditionalFields(class *models.Class) map[string]*graphql.Field { - txt2vec := &mockText2vecContextionaryModule{} - additionalProperties := map[string]*graphql.Field{} - for name, additionalProperty := range txt2vec.AdditionalProperties() { - if additionalProperty.GraphQLFieldFunction != nil { - additionalProperties[name] = additionalProperty.GraphQLFieldFunction(class.Class) - } - } - return additionalProperties -} - -func (fmp *fakeModulesProvider) ExtractAdditionalField(className, name string, params []*ast.Argument) interface{} { - txt2vec := &mockText2vecContextionaryModule{} - if additionalProperties := txt2vec.AdditionalProperties(); len(additionalProperties) > 0 { - if additionalProperty, ok := additionalProperties[name]; ok { - if additionalProperty.GraphQLExtractFunction != nil { - return additionalProperty.GraphQLExtractFunction(params) - } - } - } - return nil -} - -func (fmp *fakeModulesProvider) GetExploreAdditionalExtend(ctx context.Context, in []search.Result, - moduleParams map[string]interface{}, searchVector []float32, - argumentModuleParams map[string]interface{}, cfg moduletools.ClassConfig, -) ([]search.Result, error) { - return fmp.additionalExtend(ctx, in, moduleParams, searchVector, "ExploreGet", argumentModuleParams, nil) -} - -func (fmp *fakeModulesProvider) additionalExtend(ctx context.Context, - in search.Results, moduleParams map[string]interface{}, - searchVector []float32, capability string, argumentModuleParams map[string]interface{}, cfg moduletools.ClassConfig, -) (search.Results, error) { - txt2vec := &mockText2vecContextionaryModule{} - additionalProperties := txt2vec.AdditionalProperties() - for name, value := range moduleParams { - additionalPropertyFn := fmp.getAdditionalPropertyFn(additionalProperties[name], capability) - if additionalPropertyFn != nil && value != nil { - searchValue := value - if searchVectorValue, ok := value.(modulecapabilities.AdditionalPropertyWithSearchVector); ok { - searchVectorValue.SetSearchVector(searchVector) - searchValue = searchVectorValue - } - resArray, err := additionalPropertyFn(ctx, in, searchValue, nil, nil, nil) - if err != nil { - return nil, err - } - in = resArray - } - } - return in, nil -} - -func (fmp *fakeModulesProvider) getAdditionalPropertyFn(additionalProperty modulecapabilities.AdditionalProperty, - capability string, -) modulecapabilities.AdditionalPropertyFn { - switch capability { - case "ObjectGet": - return additionalProperty.SearchFunctions.ObjectGet - case "ObjectList": - return additionalProperty.SearchFunctions.ObjectList - case "ExploreGet": - return additionalProperty.SearchFunctions.ExploreGet - case "ExploreList": - return additionalProperty.SearchFunctions.ExploreList - default: - return nil - } -} - -func (fmp *fakeModulesProvider) GraphQLAdditionalFieldNames() []string { - txt2vec := &mockText2vecContextionaryModule{} - additionalPropertiesNames := []string{} - for _, additionalProperty := range txt2vec.AdditionalProperties() { - if additionalProperty.GraphQLNames != nil { - additionalPropertiesNames = append(additionalPropertiesNames, additionalProperty.GraphQLNames...) - } - } - return additionalPropertiesNames -} - -func extractNearTextParam(param map[string]interface{}) interface{} { - txt2vec := &mockText2vecContextionaryModule{} - argument := txt2vec.Arguments()["nearText"] - return argument.ExtractFunction(param) -} - -func createArg(name string, value string) *ast.Argument { - n := ast.Name{ - Value: name, - } - val := ast.StringValue{ - Kind: "Kind", - Value: value, - } - arg := ast.Argument{ - Name: ast.NewName(&n), - Kind: "Kind", - Value: ast.NewStringValue(&val), - } - a := ast.NewArgument(&arg) - return a -} - -func extractAdditionalParam(name string, args []*ast.Argument) interface{} { - txt2vec := &mockText2vecContextionaryModule{} - additionalProperties := txt2vec.AdditionalProperties() - switch name { - case "semanticPath", "featureProjection": - if ap, ok := additionalProperties[name]; ok { - return ap.GraphQLExtractFunction(args) - } - return nil - default: - return nil - } -} - -func getFakeModulesProvider() *fakeModulesProvider { - return &fakeModulesProvider{} -} - -func newMockResolver() *mockResolver { - logger, _ := test.NewNullLogger() - field, err := get.Build(&test_helper.SimpleSchema, logger, getFakeModulesProvider()) - if err != nil { - panic(fmt.Sprintf("could not build graphql test schema: %s", err)) - } - mocker := &mockResolver{} - mockLog := &mockRequestsLog{} - mocker.RootFieldName = "Get" - mocker.RootField = field - mocker.RootObject = map[string]interface{}{"Resolver": GetResolver(mocker), "RequestsLog": RequestsLog(mockLog)} - return mocker -} - -func newExploreMockResolver() *mockResolver { - field := explore.Build(test_helper.SimpleSchema.Objects, getFakeModulesProvider()) - mocker := &mockResolver{} - mockLog := &mockRequestsLog{} - mocker.RootFieldName = "Explore" - mocker.RootField = field - mocker.RootObject = map[string]interface{}{ - "Resolver": ExploreResolver(mocker), - "RequestsLog": mockLog, - } - return mocker -} - -func (m *mockResolver) GetClass(ctx context.Context, principal *models.Principal, - params dto.GetParams, -) ([]interface{}, error) { - args := m.Called(params) - return args.Get(0).([]interface{}), args.Error(1) -} - -func (m *mockResolver) Explore(ctx context.Context, - principal *models.Principal, params traverser.ExploreParams, -) ([]search.Result, error) { - args := m.Called(params) - return args.Get(0).([]search.Result), args.Error(1) -} - -// Resolver is a local abstraction of the required UC resolvers -type GetResolver interface { - GetClass(ctx context.Context, principal *models.Principal, info dto.GetParams) ([]interface{}, error) -} - -type ExploreResolver interface { - Explore(ctx context.Context, principal *models.Principal, params traverser.ExploreParams) ([]search.Result, error) -} - -// RequestsLog is a local abstraction on the RequestsLog that needs to be -// provided to the graphQL API in order to log Local.Get queries. -type RequestsLog interface { - Register(requestType string, identifier string) -} diff --git a/modules/text2vec-contextionary/module.go b/modules/text2vec-contextionary/module.go deleted file mode 100644 index 062aefd201705bf37101dca2317a1b7f3b3cba54..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/module.go +++ /dev/null @@ -1,249 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modcontextionary - -import ( - "context" - "net/http" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/adapters/handlers/rest/state" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - text2vecadditional "github.com/weaviate/weaviate/modules/text2vec-contextionary/additional" - text2vecinterpretation "github.com/weaviate/weaviate/modules/text2vec-contextionary/additional/interpretation" - text2vecnn "github.com/weaviate/weaviate/modules/text2vec-contextionary/additional/nearestneighbors" - text2vecsempath "github.com/weaviate/weaviate/modules/text2vec-contextionary/additional/sempath" - text2vecclassification "github.com/weaviate/weaviate/modules/text2vec-contextionary/classification" - "github.com/weaviate/weaviate/modules/text2vec-contextionary/client" - "github.com/weaviate/weaviate/modules/text2vec-contextionary/concepts" - "github.com/weaviate/weaviate/modules/text2vec-contextionary/extensions" - "github.com/weaviate/weaviate/modules/text2vec-contextionary/vectorizer" - localvectorizer "github.com/weaviate/weaviate/modules/text2vec-contextionary/vectorizer" - text2vecprojector "github.com/weaviate/weaviate/usecases/modulecomponents/additional/projector" - text2vecneartext "github.com/weaviate/weaviate/usecases/modulecomponents/nearText" -) - -// MinimumRequiredRemoteVersion describes the minimal semver version -// (independent of the model version) of the remote model inference API -const MinimumRequiredRemoteVersion = "1.0.0" - -func New() *ContextionaryModule { - return &ContextionaryModule{} -} - -// ContextionaryModule for now only handles storage and retrieval of extensions, -// but with making Weaviate more modular, this should contain anything related -// to the module -type ContextionaryModule struct { - storageProvider moduletools.StorageProvider - extensions *extensions.RESTHandlers - concepts *concepts.RESTHandlers - vectorizer *localvectorizer.Vectorizer - configValidator configValidator - graphqlProvider modulecapabilities.GraphQLArguments - additionalPropertiesProvider modulecapabilities.AdditionalProperties - searcher modulecapabilities.Searcher - remote remoteClient - classifierContextual modulecapabilities.Classifier - logger logrus.FieldLogger - nearTextTransformer modulecapabilities.TextTransform -} - -type remoteClient interface { - localvectorizer.RemoteClient - extensions.Proxy - vectorizer.InspectorClient - text2vecsempath.Remote - modulecapabilities.MetaProvider - modulecapabilities.VectorizerClient - WaitForStartupAndValidateVersion(ctx context.Context, version string, - interval time.Duration) error -} - -type configValidator interface { - Do(ctx context.Context, class *models.Class, cfg moduletools.ClassConfig, - indexChecker localvectorizer.IndexChecker) error -} - -func (m *ContextionaryModule) Name() string { - return "text2vec-contextionary" -} - -func (m *ContextionaryModule) Type() modulecapabilities.ModuleType { - return modulecapabilities.Text2Vec -} - -func (m *ContextionaryModule) Init(ctx context.Context, - params moduletools.ModuleInitParams, -) error { - m.storageProvider = params.GetStorageProvider() - appState, ok := params.GetAppState().(*state.State) - if !ok { - return errors.Errorf("appState is not a *state.State") - } - - m.logger = appState.Logger - - url := appState.ServerConfig.Config.Contextionary.URL - remote, err := client.NewClient(url, m.logger) - if err != nil { - return errors.Wrap(err, "init remote client") - } - m.remote = remote - - if err := m.remote.WaitForStartupAndValidateVersion(ctx, - MinimumRequiredRemoteVersion, 1*time.Second); err != nil { - return errors.Wrap(err, "validate remote inference api") - } - - if err := m.initExtensions(); err != nil { - return errors.Wrap(err, "init extensions") - } - - if err := m.initConcepts(); err != nil { - return errors.Wrap(err, "init concepts") - } - - if err := m.initVectorizer(); err != nil { - return errors.Wrap(err, "init vectorizer") - } - - if err := m.initGraphqlAdditionalPropertiesProvider(); err != nil { - return errors.Wrap(err, "init graphql additional properties provider") - } - - if err := m.initClassifiers(); err != nil { - return errors.Wrap(err, "init classifiers") - } - - return nil -} - -func (m *ContextionaryModule) InitExtension(modules []modulecapabilities.Module) error { - for _, module := range modules { - if module.Name() == m.Name() { - continue - } - if arg, ok := module.(modulecapabilities.TextTransformers); ok { - if arg != nil && arg.TextTransformers() != nil { - m.nearTextTransformer = arg.TextTransformers()["nearText"] - } - } - } - - if err := m.initGraphqlProvider(); err != nil { - return errors.Wrap(err, "init graphql provider") - } - return nil -} - -func (m *ContextionaryModule) initExtensions() error { - storage, err := m.storageProvider.Storage("contextionary-extensions") - if err != nil { - return errors.Wrap(err, "initialize extensions storage") - } - - uc := extensions.NewUseCase(storage) - m.extensions = extensions.NewRESTHandlers(uc, m.remote) - - return nil -} - -func (m *ContextionaryModule) initConcepts() error { - uc := localvectorizer.NewInspector(m.remote) - m.concepts = concepts.NewRESTHandlers(uc) - - return nil -} - -func (m *ContextionaryModule) initVectorizer() error { - m.vectorizer = localvectorizer.New(m.remote) - m.configValidator = localvectorizer.NewConfigValidator(m.remote, m.logger) - - m.searcher = text2vecneartext.NewSearcher(m.vectorizer) - - return nil -} - -func (m *ContextionaryModule) initGraphqlProvider() error { - m.graphqlProvider = text2vecneartext.New(m.nearTextTransformer) - return nil -} - -func (m *ContextionaryModule) initGraphqlAdditionalPropertiesProvider() error { - nnExtender := text2vecnn.NewExtender(m.remote) - featureProjector := text2vecprojector.New() - pathBuilder := text2vecsempath.New(m.remote) - interpretation := text2vecinterpretation.New() - m.additionalPropertiesProvider = text2vecadditional.New(nnExtender, featureProjector, pathBuilder, interpretation) - return nil -} - -func (m *ContextionaryModule) initClassifiers() error { - m.classifierContextual = text2vecclassification.New(m.remote) - return nil -} - -func (m *ContextionaryModule) RootHandler() http.Handler { - mux := http.NewServeMux() - - mux.Handle("/extensions-storage/", http.StripPrefix("/extensions-storage", - m.extensions.StorageHandler())) - mux.Handle("/extensions", http.StripPrefix("/extensions", - m.extensions.UserFacingHandler())) - mux.Handle("/concepts/", http.StripPrefix("/concepts", m.concepts.Handler())) - - return mux -} - -func (m *ContextionaryModule) VectorizeObject(ctx context.Context, - obj *models.Object, objDiff *moduletools.ObjectDiff, cfg moduletools.ClassConfig, -) error { - return m.vectorizer.Object(ctx, obj, objDiff, cfg) -} - -func (m *ContextionaryModule) VectorizeInput(ctx context.Context, - input string, cfg moduletools.ClassConfig, -) ([]float32, error) { - return m.vectorizer.Texts(ctx, []string{input}, cfg) -} - -func (m *ContextionaryModule) Arguments() map[string]modulecapabilities.GraphQLArgument { - return m.graphqlProvider.Arguments() -} - -func (m *ContextionaryModule) VectorSearches() map[string]modulecapabilities.VectorForParams { - return m.searcher.VectorSearches() -} - -func (m *ContextionaryModule) AdditionalProperties() map[string]modulecapabilities.AdditionalProperty { - return m.additionalPropertiesProvider.AdditionalProperties() -} - -func (m *ContextionaryModule) Classifiers() []modulecapabilities.Classifier { - return []modulecapabilities.Classifier{m.classifierContextual} -} - -func (m *ContextionaryModule) MetaInfo() (map[string]interface{}, error) { - return m.remote.MetaInfo() -} - -// verify we implement the modules.Module interface -var ( - _ = modulecapabilities.Module(New()) - _ = modulecapabilities.Vectorizer(New()) - _ = modulecapabilities.InputVectorizer(New()) -) diff --git a/modules/text2vec-contextionary/module_graphql_test.go b/modules/text2vec-contextionary/module_graphql_test.go deleted file mode 100644 index a11653f38a141593336a54f107bb7c42e8385d0b..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/module_graphql_test.go +++ /dev/null @@ -1,969 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modcontextionary - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/tailor-inc/graphql/language/ast" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/dto" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/search" - "github.com/weaviate/weaviate/modules/text2vec-contextionary/additional/models" - helper "github.com/weaviate/weaviate/test/helper" - "github.com/weaviate/weaviate/usecases/traverser" -) - -type testCase struct { - name string - query string - expectedParamsToTraverser traverser.ExploreParams - resolverReturn []search.Result - expectedResults []result -} - -type testCases []testCase - -type result struct { - pathToField []string - expectedValue interface{} -} - -func TestExtractAdditionalFields(t *testing.T) { - // We don't need to explicitly test every subselection as we did on - // phoneNumber as these fields have fixed keys. So we can simply check for - // the prop - - type test struct { - name string - query string - expectedParams dto.GetParams - resolverReturn interface{} - expectedResult interface{} - } - - tests := []test{ - { - name: "with _additional certainty", - query: "{ Get { SomeAction { _additional { certainty distance } } } }", - expectedParams: dto.GetParams{ - ClassName: "SomeAction", - AdditionalProperties: additional.Properties{ - Certainty: true, - Distance: true, - }, - }, - resolverReturn: []interface{}{ - map[string]interface{}{ - "_additional": map[string]interface{}{ - "certainty": 0.69, - "distance": helper.CertaintyToDist(t, 0.69), - }, - }, - }, - expectedResult: map[string]interface{}{ - "_additional": map[string]interface{}{ - "certainty": 0.69, - "distance": helper.CertaintyToDist(t, 0.69), - }, - }, - }, - { - name: "with _additional interpretation", - query: "{ Get { SomeAction { _additional { interpretation { source { concept weight occurrence } } } } } }", - expectedParams: dto.GetParams{ - ClassName: "SomeAction", - AdditionalProperties: additional.Properties{ - ModuleParams: map[string]interface{}{ - "interpretation": true, - }, - }, - }, - resolverReturn: []interface{}{ - map[string]interface{}{ - "_additional": map[string]interface{}{ - "interpretation": &models.Interpretation{ - Source: []*models.InterpretationSource{ - { - Concept: "foo", - Weight: 0.6, - Occurrence: 1200, - }, - { - Concept: "bar", - Weight: 0.9, - Occurrence: 800, - }, - }, - }, - }, - }, - }, - expectedResult: map[string]interface{}{ - "_additional": map[string]interface{}{ - "interpretation": map[string]interface{}{ - "source": []interface{}{ - map[string]interface{}{ - "concept": "foo", - "weight": 0.6, - "occurrence": 1200, - }, - map[string]interface{}{ - "concept": "bar", - "weight": 0.9, - "occurrence": 800, - }, - }, - }, - }, - }, - }, - { - name: "with _additional nearestNeighbors", - query: "{ Get { SomeAction { _additional { nearestNeighbors { neighbors { concept distance } } } } } }", - expectedParams: dto.GetParams{ - ClassName: "SomeAction", - AdditionalProperties: additional.Properties{ - ModuleParams: map[string]interface{}{ - "nearestNeighbors": true, - }, - }, - }, - resolverReturn: []interface{}{ - map[string]interface{}{ - "_additional": map[string]interface{}{ - "nearestNeighbors": &models.NearestNeighbors{ - Neighbors: []*models.NearestNeighbor{ - { - Concept: "foo", - Distance: 0.1, - }, - { - Concept: "bar", - Distance: 0.2, - }, - }, - }, - }, - }, - }, - expectedResult: map[string]interface{}{ - "_additional": map[string]interface{}{ - "nearestNeighbors": map[string]interface{}{ - "neighbors": []interface{}{ - map[string]interface{}{ - "concept": "foo", - "distance": float32(0.1), - }, - map[string]interface{}{ - "concept": "bar", - "distance": float32(0.2), - }, - }, - }, - }, - }, - }, - { - name: "with _additional featureProjection without any optional parameters", - query: "{ Get { SomeAction { _additional { featureProjection { vector } } } } }", - expectedParams: dto.GetParams{ - ClassName: "SomeAction", - AdditionalProperties: additional.Properties{ - ModuleParams: map[string]interface{}{ - "featureProjection": extractAdditionalParam("featureProjection", nil), - }, - }, - }, - resolverReturn: []interface{}{ - map[string]interface{}{ - "_additional": map[string]interface{}{ - "featureProjection": &models.FeatureProjection{ - Vector: []float32{0.0, 1.1, 2.2}, - }, - }, - }, - }, - expectedResult: map[string]interface{}{ - "_additional": map[string]interface{}{ - "featureProjection": map[string]interface{}{ - "vector": []interface{}{float32(0.0), float32(1.1), float32(2.2)}, - }, - }, - }, - }, - { - name: "with _additional featureProjection with optional parameters", - query: `{ Get { SomeAction { _additional { featureProjection(algorithm: "tsne", dimensions: 3, learningRate: 15, iterations: 100, perplexity: 10) { vector } } } } }`, - expectedParams: dto.GetParams{ - ClassName: "SomeAction", - AdditionalProperties: additional.Properties{ - ModuleParams: map[string]interface{}{ - "featureProjection": extractAdditionalParam("featureProjection", - []*ast.Argument{ - createArg("algorithm", "tsne"), - createArg("dimensions", "3"), - createArg("iterations", "100"), - createArg("learningRate", "15"), - createArg("perplexity", "10"), - }, - ), - }, - }, - }, - resolverReturn: []interface{}{ - map[string]interface{}{ - "_additional": map[string]interface{}{ - "featureProjection": &models.FeatureProjection{ - Vector: []float32{0.0, 1.1, 2.2}, - }, - }, - }, - }, - expectedResult: map[string]interface{}{ - "_additional": map[string]interface{}{ - "featureProjection": map[string]interface{}{ - "vector": []interface{}{float32(0.0), float32(1.1), float32(2.2)}, - }, - }, - }, - }, - { - name: "with _additional semanticPath set", - query: `{ Get { SomeAction { _additional { semanticPath { path { concept distanceToQuery distanceToResult distanceToPrevious distanceToNext } } } } } }`, - expectedParams: dto.GetParams{ - ClassName: "SomeAction", - AdditionalProperties: additional.Properties{ - ModuleParams: map[string]interface{}{ - "semanticPath": extractAdditionalParam("semanticPath", nil), - }, - }, - }, - resolverReturn: []interface{}{ - map[string]interface{}{ - "_additional": map[string]interface{}{ - "semanticPath": &models.SemanticPath{ - Path: []*models.SemanticPathElement{ - { - Concept: "foo", - DistanceToNext: ptFloat32(0.5), - DistanceToPrevious: nil, - DistanceToQuery: 0.1, - DistanceToResult: 0.1, - }, - { - Concept: "bar", - DistanceToPrevious: ptFloat32(0.5), - DistanceToNext: nil, - DistanceToQuery: 0.1, - DistanceToResult: 0.1, - }, - }, - }, - }, - }, - }, - expectedResult: map[string]interface{}{ - "_additional": map[string]interface{}{ - "semanticPath": map[string]interface{}{ - "path": []interface{}{ - map[string]interface{}{ - "concept": "foo", - "distanceToNext": float32(0.5), - "distanceToPrevious": nil, - "distanceToQuery": float32(0.1), - "distanceToResult": float32(0.1), - }, - map[string]interface{}{ - "concept": "bar", - "distanceToPrevious": float32(0.5), - "distanceToNext": nil, - "distanceToQuery": float32(0.1), - "distanceToResult": float32(0.1), - }, - }, - }, - }, - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - resolver := newMockResolver() - - resolver.On("GetClass", test.expectedParams). - Return(test.resolverReturn, nil).Once() - result := resolver.AssertResolve(t, test.query) - assert.Equal(t, test.expectedResult, result.Get("Get", "SomeAction").Result.([]interface{})[0]) - }) - } -} - -func TestNearTextRanker(t *testing.T) { - t.Parallel() - - resolver := newMockResolver() - - t.Run("for actions", func(t *testing.T) { - query := `{ Get { SomeAction(nearText: { - concepts: ["c1", "c2", "c3"], - moveTo: { - concepts:["positive"], - force: 0.5 - }, - moveAwayFrom: { - concepts:["epic"], - force: 0.25 - } - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeAction", - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - ModuleParams: map[string]interface{}{ - "nearText": extractNearTextParam(map[string]interface{}{ - "concepts": []interface{}{"c1", "c2", "c3"}, - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"positive"}, - "force": float64(0.5), - }, - "moveAwayFrom": map[string]interface{}{ - "concepts": []interface{}{"epic"}, - "force": float64(0.25), - }, - }), - }, - } - - resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - resolver.AssertResolve(t, query) - }) - - t.Run("for a class that does not have a text2vec module", func(t *testing.T) { - query := `{ Get { CustomVectorClass(nearText: { - concepts: ["c1", "c2", "c3"], - moveTo: { - concepts:["positive"], - force: 0.5 - }, - moveAwayFrom: { - concepts:["epic"], - force: 0.25 - } - }) { intField } } }` - - res := resolver.Resolve(query) - require.Len(t, res.Errors, 1) - assert.Contains(t, res.Errors[0].Message, "Unknown argument \"nearText\" on field \"CustomVectorClass\"") - }) - - t.Run("for things with optional distance set", func(t *testing.T) { - query := `{ Get { SomeThing(nearText: { - concepts: ["c1", "c2", "c3"], - distance: 0.6, - moveTo: { - concepts:["positive"], - force: 0.5 - }, - moveAwayFrom: { - concepts:["epic"], - force: 0.25 - } - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeThing", - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - Pagination: &filters.Pagination{Limit: filters.LimitFlagSearchByDist}, - ModuleParams: map[string]interface{}{ - "nearText": extractNearTextParam(map[string]interface{}{ - "concepts": []interface{}{"c1", "c2", "c3"}, - "distance": float64(0.6), - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"positive"}, - "force": float64(0.5), - }, - "moveAwayFrom": map[string]interface{}{ - "concepts": []interface{}{"epic"}, - "force": float64(0.25), - }, - }), - }, - } - resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - resolver.AssertResolve(t, query) - }) - - t.Run("for things with optional certainty set", func(t *testing.T) { - query := `{ Get { SomeThing(nearText: { - concepts: ["c1", "c2", "c3"], - certainty: 0.4, - moveTo: { - concepts:["positive"], - force: 0.5 - }, - moveAwayFrom: { - concepts:["epic"], - force: 0.25 - } - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeThing", - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - Pagination: &filters.Pagination{Limit: filters.LimitFlagSearchByDist}, - ModuleParams: map[string]interface{}{ - "nearText": extractNearTextParam(map[string]interface{}{ - "concepts": []interface{}{"c1", "c2", "c3"}, - "certainty": float64(0.4), - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"positive"}, - "force": float64(0.5), - }, - "moveAwayFrom": map[string]interface{}{ - "concepts": []interface{}{"epic"}, - "force": float64(0.25), - }, - }), - }, - } - resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - resolver.AssertResolve(t, query) - }) - - t.Run("for things with optional distance and objects set", func(t *testing.T) { - query := `{ Get { SomeThing(nearText: { - concepts: ["c1", "c2", "c3"], - distance: 0.4, - moveTo: { - concepts:["positive"], - force: 0.5 - objects: [ - { id: "moveTo-uuid1" } - { beacon: "weaviate://localhost/moveTo-uuid3" } - ] - }, - moveAwayFrom: { - concepts:["epic"], - force: 0.25 - objects: [ - { id: "moveAway-uuid1" } - { beacon: "weaviate://localhost/moveAway-uuid2" } - { beacon: "weaviate://localhost/moveAway-uuid3" } - ] - } - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeThing", - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - Pagination: &filters.Pagination{Limit: filters.LimitFlagSearchByDist}, - ModuleParams: map[string]interface{}{ - "nearText": extractNearTextParam(map[string]interface{}{ - "concepts": []interface{}{"c1", "c2", "c3"}, - "distance": float64(0.4), - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"positive"}, - "force": float64(0.5), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveTo-uuid1", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveTo-uuid3", - }, - }, - }, - "moveAwayFrom": map[string]interface{}{ - "concepts": []interface{}{"epic"}, - "force": float64(0.25), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveAway-uuid1", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveAway-uuid2", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveAway-uuid3", - }, - }, - }, - }), - }, - } - resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - resolver.AssertResolve(t, query) - }) - - t.Run("for things with optional certainty and objects set", func(t *testing.T) { - query := `{ Get { SomeThing(nearText: { - concepts: ["c1", "c2", "c3"], - certainty: 0.4, - moveTo: { - concepts:["positive"], - force: 0.5 - objects: [ - { id: "moveTo-uuid1" } - { beacon: "weaviate://localhost/moveTo-uuid3" } - ] - }, - moveAwayFrom: { - concepts:["epic"], - force: 0.25 - objects: [ - { id: "moveAway-uuid1" } - { beacon: "weaviate://localhost/moveAway-uuid2" } - { beacon: "weaviate://localhost/moveAway-uuid3" } - ] - } - }) { intField } } }` - - expectedParams := dto.GetParams{ - ClassName: "SomeThing", - Properties: []search.SelectProperty{{Name: "intField", IsPrimitive: true}}, - Pagination: &filters.Pagination{Limit: filters.LimitFlagSearchByDist}, - ModuleParams: map[string]interface{}{ - "nearText": extractNearTextParam(map[string]interface{}{ - "concepts": []interface{}{"c1", "c2", "c3"}, - "certainty": float64(0.4), - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"positive"}, - "force": float64(0.5), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveTo-uuid1", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveTo-uuid3", - }, - }, - }, - "moveAwayFrom": map[string]interface{}{ - "concepts": []interface{}{"epic"}, - "force": float64(0.25), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveAway-uuid1", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveAway-uuid2", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveAway-uuid3", - }, - }, - }, - }), - }, - } - resolver.On("GetClass", expectedParams). - Return([]interface{}{}, nil).Once() - - resolver.AssertResolve(t, query) - }) -} - -func Test_ResolveExplore(t *testing.T) { - t.Parallel() - - testsNearText := testCases{ - testCase{ - name: "Resolve Explore with nearText", - query: ` - { - Explore(nearText: {concepts: ["car", "best brand"]}) { - beacon className certainty distance - } - }`, - expectedParamsToTraverser: traverser.ExploreParams{ - ModuleParams: map[string]interface{}{ - "nearText": extractNearTextParam(map[string]interface{}{ - "concepts": []interface{}{"car", "best brand"}, - }), - }, - WithCertaintyProp: true, - }, - resolverReturn: []search.Result{ - { - Beacon: "weaviate://localhost/some-uuid", - ClassName: "bestClass", - Certainty: 0.7, - Dist: helper.CertaintyToDist(t, 0.7), - }, - }, - expectedResults: []result{{ - pathToField: []string{"Explore"}, - expectedValue: []interface{}{ - map[string]interface{}{ - "beacon": "weaviate://localhost/some-uuid", - "className": "bestClass", - "certainty": float32(0.7), - "distance": helper.CertaintyToDist(t, 0.7), - }, - }, - }}, - }, - - testCase{ - name: "with nearText with optional limit and distance set", - query: ` - { - Explore( - nearText: {concepts: ["car", "best brand"], distance: 0.6}, limit: 17 - ){ - beacon className - } - }`, - expectedParamsToTraverser: traverser.ExploreParams{ - ModuleParams: map[string]interface{}{ - "nearText": extractNearTextParam(map[string]interface{}{ - "concepts": []interface{}{"car", "best brand"}, - "distance": float64(0.6), - }), - }, - Limit: 17, - }, - resolverReturn: []search.Result{ - { - Beacon: "weaviate://localhost/some-uuid", - ClassName: "bestClass", - Dist: 0.6, - }, - }, - expectedResults: []result{{ - pathToField: []string{"Explore"}, - expectedValue: []interface{}{ - map[string]interface{}{ - "beacon": "weaviate://localhost/some-uuid", - "className": "bestClass", - }, - }, - }}, - }, - - testCase{ - name: "with nearText with optional limit and certainty set", - query: ` - { - Explore( - nearText: {concepts: ["car", "best brand"], certainty: 0.6}, limit: 17 - ){ - beacon className - } - }`, - expectedParamsToTraverser: traverser.ExploreParams{ - ModuleParams: map[string]interface{}{ - "nearText": extractNearTextParam(map[string]interface{}{ - "concepts": []interface{}{"car", "best brand"}, - "certainty": float64(0.6), - }), - }, - Limit: 17, - }, - resolverReturn: []search.Result{ - { - Beacon: "weaviate://localhost/some-uuid", - ClassName: "bestClass", - }, - }, - expectedResults: []result{{ - pathToField: []string{"Explore"}, - expectedValue: []interface{}{ - map[string]interface{}{ - "beacon": "weaviate://localhost/some-uuid", - "className": "bestClass", - }, - }, - }}, - }, - - testCase{ - name: "with moveTo set", - query: ` - { - Explore( - limit: 17 - nearText: { - concepts: ["car", "best brand"] - moveTo: { - concepts: ["mercedes"] - force: 0.7 - } - } - ) { - beacon className - } - }`, - expectedParamsToTraverser: traverser.ExploreParams{ - Limit: 17, - ModuleParams: map[string]interface{}{ - "nearText": extractNearTextParam(map[string]interface{}{ - "concepts": []interface{}{"car", "best brand"}, - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"mercedes"}, - "force": float64(0.7), - }, - }), - }, - }, - resolverReturn: []search.Result{ - { - Beacon: "weaviate://localhost/some-uuid", - ClassName: "bestClass", - }, - }, - expectedResults: []result{{ - pathToField: []string{"Explore"}, - expectedValue: []interface{}{ - map[string]interface{}{ - "beacon": "weaviate://localhost/some-uuid", - "className": "bestClass", - }, - }, - }}, - }, - - testCase{ - name: "with moveTo and moveAwayFrom set", - query: ` - { - Explore( - limit: 17 - nearText: { - concepts: ["car", "best brand"] - moveTo: { - concepts: ["mercedes"] - force: 0.7 - } - moveAwayFrom: { - concepts: ["van"] - force: 0.7 - } - } - ) { - beacon className - } - }`, - expectedParamsToTraverser: traverser.ExploreParams{ - Limit: 17, - ModuleParams: map[string]interface{}{ - "nearText": extractNearTextParam(map[string]interface{}{ - "concepts": []interface{}{"car", "best brand"}, - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"mercedes"}, - "force": float64(0.7), - }, - "moveAwayFrom": map[string]interface{}{ - "concepts": []interface{}{"van"}, - "force": float64(0.7), - }, - }), - }, - }, - resolverReturn: []search.Result{ - { - Beacon: "weaviate://localhost/some-uuid", - ClassName: "bestClass", - }, - }, - expectedResults: []result{{ - pathToField: []string{"Explore"}, - expectedValue: []interface{}{ - map[string]interface{}{ - "beacon": "weaviate://localhost/some-uuid", - "className": "bestClass", - }, - }, - }}, - }, - - testCase{ - name: "with moveTo and objects set", - query: ` - { - Explore( - limit: 17 - nearText: { - concepts: ["car", "best brand"] - moveTo: { - concepts: ["mercedes"] - force: 0.7 - objects: [ - {id: "moveto-uuid"}, - {beacon: "weaviate://localhost/other-moveto-uuid"}, - ] - } - } - ) { - beacon className - } - }`, - expectedParamsToTraverser: traverser.ExploreParams{ - Limit: 17, - ModuleParams: map[string]interface{}{ - "nearText": extractNearTextParam(map[string]interface{}{ - "concepts": []interface{}{"car", "best brand"}, - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"mercedes"}, - "force": float64(0.7), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveto-uuid", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/other-moveto-uuid", - }, - }, - }, - }), - }, - }, - resolverReturn: []search.Result{ - { - Beacon: "weaviate://localhost/some-uuid", - ClassName: "bestClass", - }, - }, - expectedResults: []result{{ - pathToField: []string{"Explore"}, - expectedValue: []interface{}{ - map[string]interface{}{ - "beacon": "weaviate://localhost/some-uuid", - "className": "bestClass", - }, - }, - }}, - }, - - testCase{ - name: "with moveTo and moveAwayFrom and objects set", - query: ` - { - Explore( - limit: 17 - nearText: { - concepts: ["car", "best brand"] - moveTo: { - concepts: ["mercedes"] - force: 0.7 - objects: [ - {id: "moveto-uuid1"}, - {beacon: "weaviate://localhost/moveto-uuid2"}, - ] - } - moveAwayFrom: { - concepts: ["van"] - force: 0.7 - objects: [ - {id: "moveAway-uuid1"}, - {beacon: "weaviate://localhost/moveAway-uuid2"}, - {id: "moveAway-uuid3"}, - {id: "moveAway-uuid4"}, - ] - } - } - ) { - beacon className - } - }`, - expectedParamsToTraverser: traverser.ExploreParams{ - Limit: 17, - ModuleParams: map[string]interface{}{ - "nearText": extractNearTextParam(map[string]interface{}{ - "concepts": []interface{}{"car", "best brand"}, - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"mercedes"}, - "force": float64(0.7), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveto-uuid1", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveto-uuid2", - }, - }, - }, - "moveAwayFrom": map[string]interface{}{ - "concepts": []interface{}{"van"}, - "force": float64(0.7), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveAway-uuid1", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveAway-uuid2", - }, - map[string]interface{}{ - "id": "moveAway-uuid3", - }, - map[string]interface{}{ - "id": "moveAway-uuid4", - }, - }, - }, - }), - }, - }, - resolverReturn: []search.Result{ - { - Beacon: "weaviate://localhost/some-uuid", - ClassName: "bestClass", - }, - }, - expectedResults: []result{{ - pathToField: []string{"Explore"}, - expectedValue: []interface{}{ - map[string]interface{}{ - "beacon": "weaviate://localhost/some-uuid", - "className": "bestClass", - }, - }, - }}, - }, - } - - testsNearText.AssertExtraction(t, newExploreMockResolver()) -} - -func (tests testCases) AssertExtraction(t *testing.T, resolver *mockResolver) { - for _, testCase := range tests { - t.Run(testCase.name, func(t *testing.T) { - resolver.On("Explore", testCase.expectedParamsToTraverser). - Return(testCase.resolverReturn, nil).Once() - - result := resolver.AssertResolve(t, testCase.query) - - for _, expectedResult := range testCase.expectedResults { - value := result.Get(expectedResult.pathToField...).Result - - assert.Equal(t, expectedResult.expectedValue, value) - } - }) - } -} - -func ptFloat32(in float32) *float32 { - return &in -} diff --git a/modules/text2vec-contextionary/vectorizer/fakes_for_test.go b/modules/text2vec-contextionary/vectorizer/fakes_for_test.go deleted file mode 100644 index a7f0a00081affaa433d1ae0e9e979ba85a309200..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/vectorizer/fakes_for_test.go +++ /dev/null @@ -1,84 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - - txt2vecmodels "github.com/weaviate/weaviate/modules/text2vec-contextionary/additional/models" -) - -type fakeClient struct { - lastInput []string -} - -func (c *fakeClient) VectorForCorpi(ctx context.Context, corpi []string, overrides map[string]string) ([]float32, []txt2vecmodels.InterpretationSource, error) { - c.lastInput = corpi - return []float32{0, 1, 2, 3}, nil, nil -} - -func (c *fakeClient) VectorForWord(ctx context.Context, word string) ([]float32, error) { - c.lastInput = []string{word} - return []float32{3, 2, 1, 0}, nil -} - -func (c *fakeClient) NearestWordsByVector(ctx context.Context, - vector []float32, n int, k int, -) ([]string, []float32, error) { - return []string{"word1", "word2"}, []float32{0.1, 0.2}, nil -} - -func (c *fakeClient) IsWordPresent(ctx context.Context, word string) (bool, error) { - return true, nil -} - -type fakeClassConfig struct { - classConfig map[string]interface{} - vectorizePropertyName bool - skippedProperty string - vectorizeClassName bool - excludedProperty string -} - -func (f fakeClassConfig) Class() map[string]interface{} { - classSettings := map[string]interface{}{ - "vectorizeClassName": f.vectorizeClassName, - } - return classSettings -} - -func (f fakeClassConfig) ClassByModuleName(moduleName string) map[string]interface{} { - return f.classConfig -} - -func (f fakeClassConfig) Property(propName string) map[string]interface{} { - if propName == f.skippedProperty { - return map[string]interface{}{ - "skip": true, - } - } - if propName == f.excludedProperty { - return map[string]interface{}{ - "vectorizePropertyName": false, - } - } - if f.vectorizePropertyName { - return map[string]interface{}{ - "vectorizePropertyName": true, - } - } - return nil -} - -func (f fakeClassConfig) Tenant() string { - return "" -} diff --git a/modules/text2vec-contextionary/vectorizer/index_check.go b/modules/text2vec-contextionary/vectorizer/index_check.go deleted file mode 100644 index 217d91558680a779eb4025c8a95080636ef54f3e..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/vectorizer/index_check.go +++ /dev/null @@ -1,72 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "github.com/weaviate/weaviate/entities/moduletools" -) - -const ( - DefaultPropertyIndexed = true - DefaultVectorizeClassName = true - DefaultVectorizePropertyName = false -) - -type indexChecker struct { - cfg moduletools.ClassConfig -} - -func NewIndexChecker(cfg moduletools.ClassConfig) *indexChecker { - return &indexChecker{cfg: cfg} -} - -func (ic *indexChecker) PropertyIndexed(propName string) bool { - vcn, ok := ic.cfg.Property(propName)["skip"] - if !ok { - return DefaultPropertyIndexed - } - - asBool, ok := vcn.(bool) - if !ok { - return DefaultPropertyIndexed - } - - return !asBool -} - -func (ic *indexChecker) VectorizePropertyName(propName string) bool { - vcn, ok := ic.cfg.Property(propName)["vectorizePropertyName"] - if !ok { - return DefaultVectorizePropertyName - } - - asBool, ok := vcn.(bool) - if !ok { - return DefaultVectorizePropertyName - } - - return asBool -} - -func (ic *indexChecker) VectorizeClassName() bool { - vcn, ok := ic.cfg.Class()["vectorizeClassName"] - if !ok { - return DefaultVectorizeClassName - } - - asBool, ok := vcn.(bool) - if !ok { - return DefaultVectorizeClassName - } - - return asBool -} diff --git a/modules/text2vec-contextionary/vectorizer/index_check_test.go b/modules/text2vec-contextionary/vectorizer/index_check_test.go deleted file mode 100644 index f5c93e10fd4553d01393d4bbc020507b929c1c30..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/vectorizer/index_check_test.go +++ /dev/null @@ -1,92 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/usecases/modules" -) - -func TestIndexChecker(t *testing.T) { - t.Run("with all defaults", func(t *testing.T) { - class := &models.Class{ - Class: "MyClass", - Properties: []*models.Property{{ - Name: "someProp", - }}, - } - - cfg := modules.NewClassBasedModuleConfig(class, "my-module", "tenant") - ic := NewIndexChecker(cfg) - - assert.True(t, ic.PropertyIndexed("someProp")) - assert.False(t, ic.VectorizePropertyName("someProp")) - assert.True(t, ic.VectorizeClassName()) - }) - - t.Run("with all explicit config matching the defaults", func(t *testing.T) { - class := &models.Class{ - Class: "MyClass", - ModuleConfig: map[string]interface{}{ - "my-module": map[string]interface{}{ - "vectorizeClassName": true, - }, - }, - Properties: []*models.Property{{ - Name: "someProp", - ModuleConfig: map[string]interface{}{ - "my-module": map[string]interface{}{ - "skip": false, - "vectorizePropertyName": false, - }, - }, - }}, - } - - cfg := modules.NewClassBasedModuleConfig(class, "my-module", "tenant") - ic := NewIndexChecker(cfg) - - assert.True(t, ic.PropertyIndexed("someProp")) - assert.False(t, ic.VectorizePropertyName("someProp")) - assert.True(t, ic.VectorizeClassName()) - }) - - t.Run("with all explicit config using non-default values", func(t *testing.T) { - class := &models.Class{ - Class: "MyClass", - ModuleConfig: map[string]interface{}{ - "my-module": map[string]interface{}{ - "vectorizeClassName": false, - }, - }, - Properties: []*models.Property{{ - Name: "someProp", - ModuleConfig: map[string]interface{}{ - "my-module": map[string]interface{}{ - "skip": true, - "vectorizePropertyName": true, - }, - }, - }}, - } - - cfg := modules.NewClassBasedModuleConfig(class, "my-module", "tenant") - ic := NewIndexChecker(cfg) - - assert.False(t, ic.PropertyIndexed("someProp")) - assert.True(t, ic.VectorizePropertyName("someProp")) - assert.False(t, ic.VectorizeClassName()) - }) -} diff --git a/modules/text2vec-contextionary/vectorizer/inspector.go b/modules/text2vec-contextionary/vectorizer/inspector.go deleted file mode 100644 index 0272fb8858c9e511270e80f3bfa5e7fb91c4faef..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/vectorizer/inspector.go +++ /dev/null @@ -1,252 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - "fmt" - "strings" - "unicode" - "unicode/utf8" - - "github.com/weaviate/weaviate/entities/models" - txt2vecmodels "github.com/weaviate/weaviate/modules/text2vec-contextionary/additional/models" -) - -type InspectorClient interface { - VectorForWord(ctx context.Context, word string) ([]float32, error) - VectorForCorpi(ctx context.Context, words []string, - overrides map[string]string) ([]float32, []txt2vecmodels.InterpretationSource, error) - NearestWordsByVector(ctx context.Context, vector []float32, n int, k int) ([]string, []float32, error) - IsWordPresent(ctx context.Context, word string) (bool, error) -} - -type Inspector struct { - client InspectorClient -} - -func NewInspector(client InspectorClient) *Inspector { - return &Inspector{client: client} -} - -func (i *Inspector) GetWords(ctx context.Context, words string) (*models.C11yWordsResponse, error) { - wordArray, err := i.validateAndSplit(words) - if err != nil { - return nil, err - } - - concatWord, err := i.concatWord(ctx, words, wordArray) - if err != nil { - return nil, err - } - - individualWords, err := i.individualWords(ctx, wordArray) - if err != nil { - return nil, err - } - - return &models.C11yWordsResponse{ - ConcatenatedWord: concatWord, - IndividualWords: individualWords, - }, nil -} - -func (i *Inspector) validateAndSplit(words string) ([]string, error) { - // set first character to lowercase - wordChars := []rune(words) - wordChars[0] = unicode.ToLower(wordChars[0]) - words = string(wordChars) - - for _, r := range words { - if !unicode.IsLetter(r) && !unicode.IsNumber(r) { - return nil, fmt.Errorf("invalid word input: words must only contain unicode letters and digits") - } - } - - return split(words), nil -} - -func (i *Inspector) concatWord(ctx context.Context, words string, - wordArray []string, -) (*models.C11yWordsResponseConcatenatedWord, error) { - if len(wordArray) < 2 { - // only build a concat response if we have more than a single word - return nil, nil - } - - // join the words into a single corpus. While the contextionary also supports - // building a centroid from multiple corpi (thus []string for Corpi, an - // occurrence-based weighing can only happen within a corpus. It is thus - by - // far - preferable in this case, to concat the words into one corpus, rather - // than treating each word as its own. - corpus := strings.Join(wordArray, " ") - vector, _, err := i.client.VectorForCorpi(ctx, []string{corpus}, nil) - if err != nil { - return nil, err - } - - nearestNeighbors, err := i.nearestNeighbors(ctx, vector) - if err != nil { - return nil, err - } - - return &models.C11yWordsResponseConcatenatedWord{ - ConcatenatedWord: words, - SingleWords: wordArray, - ConcatenatedVector: vector, - ConcatenatedNearestNeighbors: nearestNeighbors, - }, nil -} - -func (i *Inspector) nearestNeighbors(ctx context.Context, - vector []float32, -) ([]*models.C11yNearestNeighborsItems0, error) { - // relate words of centroid - words, dists, err := i.client.NearestWordsByVector(ctx, vector, 12, 32) - if err != nil { - return nil, err - } - - nearestNeighbors := []*models.C11yNearestNeighborsItems0{} - - // loop over NN Idx' and append to the return object - for i, word := range words { - item := models.C11yNearestNeighborsItems0{ - Word: word, - Distance: dists[i], - } - - nearestNeighbors = append(nearestNeighbors, &item) - } - - return nearestNeighbors, nil -} - -func (i *Inspector) individualWords(ctx context.Context, - wordArray []string, -) ([]*models.C11yWordsResponseIndividualWordsItems0, error) { - var res []*models.C11yWordsResponseIndividualWordsItems0 - - for _, word := range wordArray { - iw, err := i.individualWord(ctx, word) - if err != nil { - return nil, fmt.Errorf("word '%s': %v", word, err) - } - - res = append(res, iw) - } - - return res, nil -} - -func (i *Inspector) individualWord(ctx context.Context, - word string, -) (*models.C11yWordsResponseIndividualWordsItems0, error) { - ok, err := i.client.IsWordPresent(ctx, word) - if err != nil { - return nil, fmt.Errorf("could not check word presence: %v", err) - } - - if !ok { - return i.individualWordNotPresent(word), nil - } - - return i.individualWordPresent(ctx, word) -} - -func (i *Inspector) individualWordNotPresent(word string) *models.C11yWordsResponseIndividualWordsItems0 { - return &models.C11yWordsResponseIndividualWordsItems0{ - Word: word, - Present: false, - } -} - -func (i *Inspector) individualWordPresent(ctx context.Context, - word string, -) (*models.C11yWordsResponseIndividualWordsItems0, error) { - info, err := i.individualWordInfo(ctx, word) - if err != nil { - return nil, err - } - - return &models.C11yWordsResponseIndividualWordsItems0{ - Word: word, - Present: true, - Info: info, - }, nil -} - -func (i *Inspector) individualWordInfo(ctx context.Context, - word string, -) (*models.C11yWordsResponseIndividualWordsItems0Info, error) { - vector, err := i.client.VectorForWord(ctx, word) - if err != nil { - return nil, err - } - - nns, err := i.nearestNeighbors(ctx, vector) - if err != nil { - return nil, err - } - - return &models.C11yWordsResponseIndividualWordsItems0Info{ - Vector: vector, - NearestNeighbors: nns, - }, nil -} - -// Splits a CamelCase string to an array -// Based on: https://github.com/fatih/camelcase -func split(src string) (entries []string) { - // don't split invalid utf8 - if !utf8.ValidString(src) { - return []string{src} - } - entries = []string{} - var runes [][]rune - lastClass := 0 - class := 0 - // split into fields based on class of unicode character - for _, r := range src { - switch true { - case unicode.IsLower(r): - class = 1 - case unicode.IsUpper(r): - class = 2 - case unicode.IsDigit(r): - class = 1 - default: - class = 4 - } - if class == lastClass { - runes[len(runes)-1] = append(runes[len(runes)-1], r) - } else { - runes = append(runes, []rune{r}) - } - lastClass = class - } - // handle upper case -> lower case sequences, e.g. - // "PDFL", "oader" -> "PDF", "Loader" - for i := 0; i < len(runes)-1; i++ { - if unicode.IsUpper(runes[i][0]) && unicode.IsLower(runes[i+1][0]) { - runes[i+1] = append([]rune{runes[i][len(runes[i])-1]}, runes[i+1]...) - runes[i] = runes[i][:len(runes[i])-1] - } - } - // construct []string from results - for _, s := range runes { - if len(s) > 0 { - entries = append(entries, strings.ToLower(string(s))) - } - } - return -} diff --git a/modules/text2vec-contextionary/vectorizer/inspector_test.go b/modules/text2vec-contextionary/vectorizer/inspector_test.go deleted file mode 100644 index 560a480e30272525f3140aae70039f0155f9cef7..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/vectorizer/inspector_test.go +++ /dev/null @@ -1,190 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" -) - -func TestInspector(t *testing.T) { - type test struct { - name string - input string - expectedErr error - expectedOutput *models.C11yWordsResponse - } - - tests := []test{ - { - name: "invalid input", - input: "i don't like pizza", - expectedErr: fmt.Errorf("invalid word input: words must only contain unicode letters and digits"), - }, - { - name: "single valid word", - input: "pizza", - expectedOutput: &models.C11yWordsResponse{ - IndividualWords: []*models.C11yWordsResponseIndividualWordsItems0{ - { - Present: true, - Word: "pizza", - Info: &models.C11yWordsResponseIndividualWordsItems0Info{ - Vector: []float32{3, 2, 1, 0}, - NearestNeighbors: []*models.C11yNearestNeighborsItems0{ - { - Distance: 0.1, - Word: "word1", - }, - { - Distance: 0.2, - Word: "word2", - }, - }, - }, - }, - }, - }, - }, - { - name: "single valid word containing numbers", - input: "pi55a", - expectedOutput: &models.C11yWordsResponse{ - IndividualWords: []*models.C11yWordsResponseIndividualWordsItems0{ - { - Present: true, - Word: "pi55a", - Info: &models.C11yWordsResponseIndividualWordsItems0Info{ - Vector: []float32{3, 2, 1, 0}, - NearestNeighbors: []*models.C11yNearestNeighborsItems0{ - { - Distance: 0.1, - Word: "word1", - }, - { - Distance: 0.2, - Word: "word2", - }, - }, - }, - }, - }, - }, - }, - { - name: "concatenated words", - input: "pizzaBakerMakerShaker", - expectedOutput: &models.C11yWordsResponse{ - ConcatenatedWord: &models.C11yWordsResponseConcatenatedWord{ - ConcatenatedWord: "pizzaBakerMakerShaker", - SingleWords: []string{"pizza", "baker", "maker", "shaker"}, - ConcatenatedVector: []float32{0, 1, 2, 3}, - ConcatenatedNearestNeighbors: []*models.C11yNearestNeighborsItems0{ - { - Distance: 0.1, - Word: "word1", - }, - { - Distance: 0.2, - Word: "word2", - }, - }, - }, - IndividualWords: []*models.C11yWordsResponseIndividualWordsItems0{ - { - Present: true, - Word: "pizza", - Info: &models.C11yWordsResponseIndividualWordsItems0Info{ - Vector: []float32{3, 2, 1, 0}, - NearestNeighbors: []*models.C11yNearestNeighborsItems0{ - { - Distance: 0.1, - Word: "word1", - }, - { - Distance: 0.2, - Word: "word2", - }, - }, - }, - }, - { - Present: true, - Word: "baker", - Info: &models.C11yWordsResponseIndividualWordsItems0Info{ - Vector: []float32{3, 2, 1, 0}, - NearestNeighbors: []*models.C11yNearestNeighborsItems0{ - { - Distance: 0.1, - Word: "word1", - }, - { - Distance: 0.2, - Word: "word2", - }, - }, - }, - }, - { - Present: true, - Word: "maker", - Info: &models.C11yWordsResponseIndividualWordsItems0Info{ - Vector: []float32{3, 2, 1, 0}, - NearestNeighbors: []*models.C11yNearestNeighborsItems0{ - { - Distance: 0.1, - Word: "word1", - }, - { - Distance: 0.2, - Word: "word2", - }, - }, - }, - }, - { - Present: true, - Word: "shaker", - Info: &models.C11yWordsResponseIndividualWordsItems0Info{ - Vector: []float32{3, 2, 1, 0}, - NearestNeighbors: []*models.C11yNearestNeighborsItems0{ - { - Distance: 0.1, - Word: "word1", - }, - { - Distance: 0.2, - Word: "word2", - }, - }, - }, - }, - }, - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - client := &fakeClient{} - i := NewInspector(client) - res, err := i.GetWords(context.Background(), test.input) - require.Equal(t, err, test.expectedErr) - assert.Equal(t, res, test.expectedOutput) - }) - } -} diff --git a/modules/text2vec-contextionary/vectorizer/noop.go b/modules/text2vec-contextionary/vectorizer/noop.go deleted file mode 100644 index fe708a3f9d41db3689170f196f9e599fd034f79d..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/vectorizer/noop.go +++ /dev/null @@ -1,53 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - - "github.com/weaviate/weaviate/entities/models" -) - -// NoOpVectorizer is a simple stand in that does nothing. Can be used when the -// feature should be turned off overall -type NoOpVectorizer struct{} - -// Corpi is not implemented in the NoOpVectorizer -func (n *NoOpVectorizer) Corpi(ctx context.Context, corpi []string) ([]float32, error) { - return []float32{}, nil -} - -// MoveTo is not implemented in the NoOpVectorizer -func (n *NoOpVectorizer) MoveTo(source []float32, target []float32, weight float32) ([]float32, error) { - return []float32{}, nil -} - -// MoveAwayFrom is not implemented in the NoOpVectorizer -func (n *NoOpVectorizer) MoveAwayFrom(source []float32, target []float32, weight float32) ([]float32, error) { - return []float32{}, nil -} - -// NormalizedDistance is not implemented in the NoOpVectorizer -func (n *NoOpVectorizer) NormalizedDistance(a, b []float32) (float32, error) { - return 0, nil -} - -// Object is not implemented in the NoOpVectorizer -func (n *NoOpVectorizer) Object(ctx context.Context, concept *models.Object) ([]float32, error) { - return []float32{}, nil -} - -// NewNoOp creates a new NoOpVectorizer which can be used when no vectorization -// is desired, i.e. the feature is turned off completely -func NewNoOp() *NoOpVectorizer { - return &NoOpVectorizer{} -} diff --git a/modules/text2vec-contextionary/vectorizer/schema_config.go b/modules/text2vec-contextionary/vectorizer/schema_config.go deleted file mode 100644 index 2edc2bcc5a4fc56846a4c23ed853331a971f7079..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/vectorizer/schema_config.go +++ /dev/null @@ -1,236 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - "fmt" - "strings" - - "github.com/fatih/camelcase" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/schema" -) - -type ConfigValidator struct { - remote RemoteClient - logger logrus.FieldLogger -} - -type IndexChecker interface { - VectorizeClassName() bool - VectorizePropertyName(propName string) bool - PropertyIndexed(propName string) bool -} - -type RemoteClient interface { - IsStopWord(ctx context.Context, word string) (bool, error) - IsWordPresent(ctx context.Context, word string) (bool, error) -} - -func NewConfigValidator(rc RemoteClient, - logger logrus.FieldLogger, -) *ConfigValidator { - return &ConfigValidator{remote: rc, logger: logger} -} - -func (cv *ConfigValidator) Do(ctx context.Context, class *models.Class, - cfg moduletools.ClassConfig, icheck IndexChecker, -) error { - err := cv.validateClassName(ctx, class.Class, icheck.VectorizeClassName()) - if err != nil { - return fmt.Errorf("invalid class name: %w", err) - } - - for _, prop := range class.Properties { - if !icheck.PropertyIndexed(prop.Name) { - continue - } - - err = cv.validatePropertyName(ctx, prop.Name, - icheck.VectorizePropertyName(prop.Name)) - if err != nil { - return errors.Wrapf(err, "class %q: invalid property name", class.Class) - } - } - - if err := cv.validateIndexState(ctx, class, icheck); err != nil { - return errors.Wrap(err, "invalid combination of properties") - } - - cv.checkForPossibilityOfDuplicateVectors(ctx, class, icheck) - - return nil -} - -func (cv *ConfigValidator) validateClassName(ctx context.Context, className string, - vectorizeClass bool, -) error { - // class name - if !vectorizeClass { - // if the user chooses not to vectorize the class, we don't need to check - // if its c11y-valid or not - return nil - } - - camelParts := camelcase.Split(className) - stopWordsFound := 0 - for _, part := range camelParts { - word := strings.ToLower(part) - sw, err := cv.remote.IsStopWord(ctx, word) - if err != nil { - return fmt.Errorf("check stopword: %v", err) - } - - if sw { - stopWordsFound++ - continue - } - - present, err := cv.remote.IsWordPresent(ctx, word) - if err != nil { - return fmt.Errorf("check word presence: %v", err) - } - - if !present { - return fmt.Errorf("could not find the word '%s' from the class name '%s' "+ - "in the contextionary", word, className) - } - } - - if len(camelParts) == stopWordsFound { - return fmt.Errorf("className '%s' consists of only stopwords and is therefore "+ - "not a contextionary-valid class name, make sure at least one word in the "+ - "classname is not a stop word", className) - } - - return nil -} - -func (cv *ConfigValidator) validatePropertyName(ctx context.Context, - propertyName string, vectorize bool, -) error { - if !vectorize { - // user does not want to vectorize this property name, so we don't have to - // validate it - return nil - } - - camelParts := camelcase.Split(propertyName) - stopWordsFound := 0 - for _, part := range camelParts { - word := strings.ToLower(part) - sw, err := cv.remote.IsStopWord(ctx, word) - if err != nil { - return fmt.Errorf("check stopword: %v", err) - } - - if sw { - stopWordsFound++ - continue - } - - present, err := cv.remote.IsWordPresent(ctx, word) - if err != nil { - return fmt.Errorf("check word presence: %v", err) - } - - if !present { - return fmt.Errorf("could not find word '%s' of the property '%s' in the "+ - "contextionary", word, propertyName) - } - } - - if len(camelParts) == stopWordsFound { - return fmt.Errorf("the propertyName '%s' consists of only stopwords and is "+ - "therefore not a contextionary-valid property name, make sure at least one word "+ - "in the property name is not a stop word", propertyName) - } - - return nil -} - -func (cv *ConfigValidator) validateIndexState(ctx context.Context, - class *models.Class, icheck IndexChecker, -) error { - if icheck.VectorizeClassName() { - // if the user chooses to vectorize the classname, vector-building will - // always be possible, no need to investigate further - - return nil - } - - // search if there is at least one indexed, string/text or string/text[] - // prop. If found pass validation - for _, prop := range class.Properties { - if len(prop.DataType) < 1 { - return errors.Errorf("property %s must have at least one datatype: "+ - "got %v", prop.Name, prop.DataType) - } - - if prop.DataType[0] != string(schema.DataTypeText) && - prop.DataType[0] != string(schema.DataTypeTextArray) { - // we can only vectorize text-like props - continue - } - - if icheck.PropertyIndexed(prop.Name) { - // found at least one, this is a valid schema - return nil - } - } - - return fmt.Errorf("invalid properties: didn't find a single property which is " + - "of type string or text and is not excluded from indexing. In addition the " + - "class name is excluded from vectorization as well, meaning that it cannot be " + - "used to determine the vector position. To fix this, set 'vectorizeClassName' " + - "to true if the class name is contextionary-valid. Alternatively add at least " + - "contextionary-valid text/string property which is not excluded from " + - "indexing.") -} - -func (cv *ConfigValidator) checkForPossibilityOfDuplicateVectors( - ctx context.Context, class *models.Class, icheck IndexChecker, -) { - if !icheck.VectorizeClassName() { - // if the user choses not to vectorize the class name, this means they must - // have chosen something else to vectorize, otherwise the validation would - // have error'd before we ever got here. We can skip further checking. - - return - } - - // search if there is at least one indexed, string/text prop. If found exit - for _, prop := range class.Properties { - // length check skipped, because validation has already passed - if prop.DataType[0] != string(schema.DataTypeText) { - // we can only vectorize text-like props - continue - } - - if icheck.PropertyIndexed(prop.Name) { - // found at least one - return - } - } - - cv.logger.WithField("module", "text2vec-contextionary"). - WithField("class", class.Class). - Warnf("text2vec-contextionary: Class %q does not have any properties "+ - "indexed (or only non text-properties indexed) and the vector position is "+ - "only determined by the class name. Each object will end up with the same "+ - "vector which leads to a severe performance penalty on imports. Consider "+ - "setting vectorIndexConfig.skip=true for this property", class.Class) -} diff --git a/modules/text2vec-contextionary/vectorizer/schema_config_test.go b/modules/text2vec-contextionary/vectorizer/schema_config_test.go deleted file mode 100644 index 37346ddd0e5976d4459c3a61ebf022c23e94a509..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/vectorizer/schema_config_test.go +++ /dev/null @@ -1,515 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - "testing" - - "github.com/sirupsen/logrus" - ltest "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" -) - -func TestConfigValidator(t *testing.T) { - t.Run("validate class names", func(t *testing.T) { - type testCase struct { - input string - valid bool - name string - vectorize bool - } - - // for all test cases keep in mind that the word "carrot" is not present in - // the fake c11y, but every other word is. - // - // Additionally, the word "the" is a stopword - // - // all inputs represent class names (!) - tests := []testCase{ - // valid names - { - name: "Single uppercase word present in the c11y", - input: "Car", - valid: true, - vectorize: true, - }, - { - name: "Single lowercase word present in the c11y, stored as uppercase", - input: "car", - valid: true, - vectorize: true, - }, - { - name: "combination of valid words starting with uppercase letter", - input: "CarGarage", - valid: true, - vectorize: true, - }, - { - name: "combination of valid words starting with lowercase letter, stored as uppercase", - input: "carGarage", - valid: true, - vectorize: true, - }, - { - name: "combination of valid words and stopwords, starting with uppercase", - input: "TheCarGarage", - valid: true, - vectorize: true, - }, - { - name: "combination of valid words and stopwords starting with lowercase letter, stored as uppercase", - input: "carTheGarage", - valid: true, - vectorize: true, - }, - - // invalid names - { - name: "Single uppercase word NOT present in the c11y", - input: "Carrot", - valid: false, - vectorize: true, - }, - { - name: "Single lowercase word NOT present in the c11y", - input: "carrot", - valid: false, - vectorize: true, - }, - { - name: "Single uppercase stopword", - input: "The", - valid: false, - vectorize: true, - }, - { - name: "Single lowercase stopword", - input: "the", - valid: false, - vectorize: true, - }, - { - name: "combination of valid and invalid words, valid word first lowercased", - input: "potatoCarrot", - valid: false, - vectorize: true, - }, - { - name: "combination of valid and invalid words, valid word first uppercased", - input: "PotatoCarrot", - valid: false, - vectorize: true, - }, - { - name: "combination of valid and invalid words, invalid word first lowercased", - input: "carrotPotato", - valid: false, - vectorize: true, - }, - { - name: "combination of valid and invalid words, invalid word first uppercased", - input: "CarrotPotato", - valid: false, - vectorize: true, - }, - { - name: "combination of only stopwords, starting with lowercase", - input: "theThe", - valid: false, - vectorize: true, - }, - { - name: "combination of only stopwords, starting with uppercase", - input: "TheThe", - valid: false, - vectorize: true, - }, - - // vectorize turned off - { - name: "non-vectorized: combination of only stopwords, starting with uppercase", - input: "TheThe", - valid: true, - vectorize: false, - }, - { - name: "non-vectorized: excluded word", - input: "carrot", - valid: true, - vectorize: false, - }, - } - - for _, test := range tests { - t.Run(test.name+" object class", func(t *testing.T) { - class := &models.Class{ - Class: test.input, - Properties: []*models.Property{{ - Name: "dummyPropSoWeDontRunIntoAllNoindexedError", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }}, - } - - logger, _ := ltest.NewNullLogger() - v := NewConfigValidator(&fakeRemote{}, logger) - err := v.Do(context.Background(), class, nil, &fakeIndexChecker{ - vectorizeClassName: test.vectorize, - propertyIndexed: true, - }) - assert.Equal(t, test.valid, err == nil) - - // only proceed if input was supposed to be valid - if test.valid == false { - return - } - }) - } - }) - - t.Run("validate property names", func(t *testing.T) { - type testCase struct { - input string - valid bool - name string - vectorize bool - } - - // for all test cases keep in mind that the word "carrot" is not present in - // the fake c11y, but every other word is - // - // all inputs represent property names (!) - tests := []testCase{ - // valid names - { - name: "Single uppercase word present in the c11y, stored as lowercase", - input: "Brand", - valid: true, - vectorize: true, - }, - { - name: "Single lowercase word present in the c11y", - input: "brand", - valid: true, - vectorize: true, - }, - { - name: "combination of valid words starting with uppercase letter, stored as lowercase", - input: "BrandGarage", - valid: true, - vectorize: true, - }, - { - name: "combination of valid words starting with lowercase letter", - input: "brandGarage", - valid: true, - vectorize: true, - }, - { - name: "combination of valid words and stop words starting with uppercase letter, stored as lowercase", - input: "TheGarage", - valid: true, - vectorize: true, - }, - { - name: "combination of valid words and stop words starting with lowercase letter", - input: "theGarage", - valid: true, - vectorize: true, - }, - - // invalid names - { - name: "Single uppercase word NOT present in the c11y", - input: "Carrot", - valid: false, - vectorize: true, - }, - { - name: "Single lowercase word NOT present in the c11y", - input: "carrot", - valid: false, - vectorize: true, - }, - { - name: "Single lowercase stop word", - input: "the", - valid: false, - vectorize: true, - }, - { - name: "combination of valid and invalid words, valid word first lowercased", - input: "potatoCarrot", - valid: false, - vectorize: true, - }, - { - name: "combination of valid and invalid words, valid word first uppercased", - input: "PotatoCarrot", - valid: false, - vectorize: true, - }, - { - name: "combination of valid and invalid words, invalid word first lowercased", - input: "carrotPotato", - valid: false, - vectorize: true, - }, - { - name: "combination of valid and invalid words, invalid word first uppercased", - input: "CarrotPotato", - valid: false, - vectorize: true, - }, - { - name: "combination of only stop words, first lowercased", - input: "theThe", - valid: false, - vectorize: true, - }, - { - name: "combination of only stop words, first uppercased", - input: "TheThe", - valid: false, - vectorize: true, - }, - - // without vectorizing - { - name: "non-vectorizing: combination of only stop words, first uppercased", - input: "TheThe", - valid: true, - vectorize: false, - }, - { - name: "non-vectorizing: combination of only stop words, first uppercased", - input: "carrot", - valid: true, - vectorize: false, - }, - } - - for _, test := range tests { - t.Run(test.name+" object class", func(t *testing.T) { - class := &models.Class{ - Class: "ValidName", - Properties: []*models.Property{{ - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - Name: test.input, - }}, - } - - logger, _ := ltest.NewNullLogger() - v := NewConfigValidator(&fakeRemote{}, logger) - err := v.Do(context.Background(), class, nil, &fakeIndexChecker{ - vectorizePropertyName: test.vectorize, - propertyIndexed: true, - }) - assert.Equal(t, test.valid, err == nil) - }) - } - }) - - t.Run("all usable props no-indexed", func(t *testing.T) { - t.Run("all schema vectorization turned off", func(t *testing.T) { - class := &models.Class{ - Vectorizer: "text2vec-contextionary", - Class: "ValidName", - Properties: []*models.Property{ - { - DataType: []string{"text"}, - Name: "description", - }, - { - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - Name: "name", - }, - { - DataType: []string{"int"}, - Name: "amount", - }, - }, - } - - logger, _ := ltest.NewNullLogger() - v := NewConfigValidator(&fakeRemote{}, logger) - err := v.Do(context.Background(), class, nil, &fakeIndexChecker{ - vectorizePropertyName: false, - vectorizeClassName: false, - propertyIndexed: false, - }) - assert.NotNil(t, err) - }) - }) - - t.Run("with only array types", func(t *testing.T) { - class := &models.Class{ - Vectorizer: "text2vec-contextionary", - Class: "ValidName", - Properties: []*models.Property{ - { - DataType: []string{"text[]"}, - Name: "descriptions", - }, - { - DataType: schema.DataTypeTextArray.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - Name: "names", - }, - }, - } - - logger, _ := ltest.NewNullLogger() - v := NewConfigValidator(&fakeRemote{}, logger) - err := v.Do(context.Background(), class, nil, &fakeIndexChecker{ - vectorizePropertyName: false, - vectorizeClassName: false, - propertyIndexed: true, - }) - assert.Nil(t, err) - }) -} - -func TestConfigValidator_RiskOfDuplicateVectors(t *testing.T) { - type test struct { - name string - in *models.Class - expectWarning bool - indexChecker *fakeIndexChecker - } - - tests := []test{ - { - name: "usable properties", - in: &models.Class{ - Class: "ValidName", - Properties: []*models.Property{ - { - DataType: []string{string(schema.DataTypeText)}, - Name: "textProp", - }, - }, - }, - expectWarning: false, - indexChecker: &fakeIndexChecker{ - vectorizePropertyName: false, - vectorizeClassName: true, - propertyIndexed: true, - }, - }, - { - name: "no properties", - in: &models.Class{ - Class: "ValidName", - }, - expectWarning: true, - indexChecker: &fakeIndexChecker{ - vectorizePropertyName: false, - vectorizeClassName: true, - propertyIndexed: false, - }, - }, - { - name: "usable properties, but they are no-indexed", - in: &models.Class{ - Class: "ValidName", - Properties: []*models.Property{ - { - DataType: []string{string(schema.DataTypeText)}, - Name: "textProp", - }, - }, - }, - expectWarning: true, - indexChecker: &fakeIndexChecker{ - vectorizePropertyName: false, - vectorizeClassName: true, - propertyIndexed: false, - }, - }, - { - name: "only unusable properties", - in: &models.Class{ - Class: "ValidName", - Properties: []*models.Property{ - { - DataType: []string{string(schema.DataTypeInt)}, - Name: "intProp", - }, - }, - }, - expectWarning: true, - indexChecker: &fakeIndexChecker{ - vectorizePropertyName: false, - vectorizeClassName: true, - propertyIndexed: false, - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - logger, hook := ltest.NewNullLogger() - v := NewConfigValidator(&fakeRemote{}, logger) - err := v.Do(context.Background(), test.in, nil, test.indexChecker) - require.Nil(t, err) - - entry := hook.LastEntry() - if test.expectWarning { - require.NotNil(t, entry) - assert.Equal(t, logrus.WarnLevel, entry.Level) - } else { - assert.Nil(t, entry) - } - }) - } -} - -type fakeIndexChecker struct { - vectorizeClassName bool - vectorizePropertyName bool - propertyIndexed bool -} - -func (f *fakeIndexChecker) VectorizeClassName() bool { - return f.vectorizeClassName -} - -func (f *fakeIndexChecker) VectorizePropertyName(propName string) bool { - return f.vectorizePropertyName -} - -func (f *fakeIndexChecker) PropertyIndexed(propName string) bool { - return f.propertyIndexed -} - -// Every word in this fake c11y remote client is present except for the word -// Carrot which is not present -type fakeRemote struct{} - -func (f *fakeRemote) IsWordPresent(ctx context.Context, word string) (bool, error) { - if word == "carrot" || word == "the" { - return false, nil - } - return true, nil -} - -func (f *fakeRemote) IsStopWord(ctx context.Context, word string) (bool, error) { - return word == "the", nil -} diff --git a/modules/text2vec-contextionary/vectorizer/vectorizer.go b/modules/text2vec-contextionary/vectorizer/vectorizer.go deleted file mode 100644 index b36293827c0e0fbb3bde3df5564d33061ce200f1..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/vectorizer/vectorizer.go +++ /dev/null @@ -1,195 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -// TODO: This entire package should be part of the text2vec-contextionary -// module, if methods/objects in here are used from non-modular code, they -// probably shouldn't be in here - -import ( - "context" - "fmt" - "strings" - - "github.com/fatih/camelcase" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/moduletools" - txt2vecmodels "github.com/weaviate/weaviate/modules/text2vec-contextionary/additional/models" - objectsvectorizer "github.com/weaviate/weaviate/usecases/modulecomponents/vectorizer" -) - -// Vectorizer turns objects into vectors -type Vectorizer struct { - client client - objectVectorizer *objectsvectorizer.ObjectVectorizer -} - -type ErrNoUsableWords struct { - Err error -} - -func (e ErrNoUsableWords) Error() string { - return e.Err.Error() -} - -func NewErrNoUsableWordsf(pattern string, args ...interface{}) ErrNoUsableWords { - return ErrNoUsableWords{Err: fmt.Errorf(pattern, args...)} -} - -type client interface { - VectorForCorpi(ctx context.Context, corpi []string, - overrides map[string]string) ([]float32, []txt2vecmodels.InterpretationSource, error) -} - -// IndexCheck returns whether a property of a class should be indexed -type ClassIndexCheck interface { - PropertyIndexed(property string) bool - VectorizeClassName() bool - VectorizePropertyName(propertyName string) bool -} - -// New from c11y client -func New(client client) *Vectorizer { - return &Vectorizer{ - client: client, - objectVectorizer: objectsvectorizer.New(), - } -} - -func (v *Vectorizer) Texts(ctx context.Context, inputs []string, - cfg moduletools.ClassConfig, -) ([]float32, error) { - return v.Corpi(ctx, inputs) -} - -// Object object to vector -func (v *Vectorizer) Object(ctx context.Context, object *models.Object, - objectDiff *moduletools.ObjectDiff, cfg moduletools.ClassConfig, -) error { - var overrides map[string]string - if object.VectorWeights != nil { - overrides = object.VectorWeights.(map[string]string) - } - - icheck := NewIndexChecker(cfg) - vec, sources, err := v.object(ctx, object.Class, object.Properties, objectDiff, overrides, - icheck) - if err != nil { - return err - } - - object.Vector = vec - - if object.Additional == nil { - object.Additional = models.AdditionalProperties{} - } - - object.Additional["interpretation"] = &txt2vecmodels.Interpretation{ - Source: sourceFromInputElements(sources), - } - - return nil -} - -func (v *Vectorizer) object(ctx context.Context, className string, - schema interface{}, objDiff *moduletools.ObjectDiff, overrides map[string]string, - icheck ClassIndexCheck, -) ([]float32, []txt2vecmodels.InterpretationSource, error) { - corpi, vector, err := v.objectVectorizer.TextsOrVector(ctx, className, schema, objDiff, icheck) - if err != nil { - return nil, nil, err - } - // no property was changed, old vector can be used - if vector != nil { - // dont' re-vectorize - return vector, []txt2vecmodels.InterpretationSource{}, nil - } - // vectorize text - vector, ie, err := v.client.VectorForCorpi(ctx, []string{corpi}, overrides) - if err != nil { - switch err.(type) { - case ErrNoUsableWords: - return nil, nil, fmt.Errorf("The object is invalid, as weaviate could not extract "+ - "any contextionary-valid words from it. This is the case when you have "+ - "set the options 'vectorizeClassName: false' and 'vectorizePropertyName: false' in this class' schema definition "+ - "and not a single property's value "+ - "contains at least one contextionary-valid word. To fix this, you have several "+ - "options:\n\n1.) Make sure that the schema class name or the set properties are "+ - "a contextionary-valid term and include them in vectorization using the "+ - "'vectorizeClassName' or 'vectorizePropertyName' setting. In this case the vector position "+ - "will be composed of both the class/property names and the values for those fields. "+ - "Even if no property values are contextionary-valid, the overall word corpus is still valid "+ - "due to the contextionary-valid class/property names."+ - "\n\n2.) Alternatively, if you do not want to include schema class/property names "+ - "in vectorization, you must make sure that at least one text/string property contains "+ - "at least one contextionary-valid word."+ - "\n\n3.) If the word corpus weaviate extracted from your object "+ - "(see below) does contain enough meaning to build a vector position, but the contextionary "+ - "did not recognize the words, you can extend the contextionary using the "+ - "REST API. This is the case when you use mostly industry-specific terms which are "+ - "not known to the common language contextionary. Once extended, simply reimport this object."+ - "\n\nThe following words were extracted from your object: %v"+ - "\n\nTo learn more about the contextionary and how it behaves, check out: https://www.semi.technology/documentation/weaviate/current/contextionary.html"+ - "\n\nOriginal error: %v", corpi, err) - default: - return nil, nil, fmt.Errorf("vectorizing object with corpus '%+v': %v", corpi, err) - } - } - - return vector, ie, nil -} - -// Corpi takes any list of strings and builds a common vector for all of them -func (v *Vectorizer) Corpi(ctx context.Context, corpi []string, -) ([]float32, error) { - for i, corpus := range corpi { - corpi[i] = camelCaseToLower(corpus) - } - - vector, _, err := v.client.VectorForCorpi(ctx, corpi, nil) - if err != nil { - return nil, fmt.Errorf("vectorizing corpus '%+v': %v", corpi, err) - } - - return vector, nil -} - -func camelCaseToLower(in string) string { - parts := camelcase.Split(in) - var sb strings.Builder - for i, part := range parts { - if part == " " { - continue - } - - if i > 0 { - sb.WriteString(" ") - } - - sb.WriteString(strings.ToLower(part)) - } - - return sb.String() -} - -func sourceFromInputElements(in []txt2vecmodels.InterpretationSource) []*txt2vecmodels.InterpretationSource { - out := make([]*txt2vecmodels.InterpretationSource, len(in)) - for i, elem := range in { - out[i] = &txt2vecmodels.InterpretationSource{ - Concept: elem.Concept, - Occurrence: elem.Occurrence, - Weight: float64(elem.Weight), - } - } - - return out -} diff --git a/modules/text2vec-contextionary/vectorizer/vectorizer_test.go b/modules/text2vec-contextionary/vectorizer/vectorizer_test.go deleted file mode 100644 index a1afcfb0b0298aa30472970fa1dce0fedfa5e34f..0000000000000000000000000000000000000000 --- a/modules/text2vec-contextionary/vectorizer/vectorizer_test.go +++ /dev/null @@ -1,480 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/moduletools" -) - -func TestVectorizingObjects(t *testing.T) { - type testCase struct { - name string - input *models.Object - expectedClientCall []string - noindex string - excludedProperty string // to simulate a schema where property names aren't vectorized - excludedClass string // to simulate a schema where class names aren't vectorized - } - - tests := []testCase{ - { - name: "empty object", - input: &models.Object{ - Class: "Car", - }, - expectedClientCall: []string{"car"}, - }, - { - name: "object with one string prop", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "Mercedes", - }, - }, - expectedClientCall: []string{"car brand mercedes"}, - }, - - { - name: "object with one non-string prop", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "power": 300, - }, - }, - expectedClientCall: []string{"car"}, - }, - - { - name: "object with a mix of props", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: []string{"car brand best brand review a very great car"}, - }, - { - name: "with a noindexed property", - noindex: "review", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: []string{"car brand best brand"}, - }, - - { - name: "with the class name not vectorized", - excludedClass: "Car", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: []string{"brand best brand review a very great car"}, - }, - - { - name: "with a property name not vectorized", - excludedProperty: "review", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: []string{"car brand best brand a very great car"}, - }, - - { - name: "with no schema labels vectorized", - excludedProperty: "review", - excludedClass: "Car", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "review": "a very great car", - }, - }, - expectedClientCall: []string{"a very great car"}, - }, - - { - name: "with string/text arrays without propname or classname", - excludedProperty: "reviews", - excludedClass: "Car", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - expectedClientCall: []string{"a very great car you should consider buying one"}, - }, - - { - name: "with string/text arrays with propname and classname", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - expectedClientCall: []string{"car reviews a very great car reviews you should consider buying one"}, - }, - - { - name: "with compound class and prop names", - input: &models.Object{ - Class: "SuperCar", - Properties: map[string]interface{}{ - "brandOfTheCar": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: []string{"super car brand of the car best brand review a very great car"}, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - ic := &fakeClassConfig{ - excludedProperty: test.excludedProperty, - skippedProperty: test.noindex, - vectorizeClassName: test.excludedClass != "Car", - vectorizePropertyName: true, - } - - client := &fakeClient{} - v := New(client) - - err := v.Object(context.Background(), test.input, nil, ic) - - require.Nil(t, err) - assert.Equal(t, models.C11yVector{0, 1, 2, 3}, test.input.Vector) - expected := strings.Split(test.expectedClientCall[0], " ") - actual := strings.Split(client.lastInput[0], " ") - assert.ElementsMatch(t, expected, actual) - }) - } -} - -func TestVectorizingObjectsWithDiff(t *testing.T) { - type testCase struct { - name string - input *models.Object - skipped string - diff *moduletools.ObjectDiff - expectedVectorize bool - } - - tests := []testCase{ - { - name: "no diff", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: nil, - expectedVectorize: true, - }, - { - name: "diff all props unchanged", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("brand", "best brand", "best brand"). - WithProp("power", 300, 300). - WithProp("description", "a very great car", "a very great car"). - WithProp("reviews", []interface{}{ - "a very great car", - "you should consider buying one", - }, []interface{}{ - "a very great car", - "you should consider buying one", - }), - expectedVectorize: false, - }, - { - name: "diff one vectorizable prop changed (1)", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("brand", "old best brand", "best brand"), - expectedVectorize: true, - }, - { - name: "diff one vectorizable prop changed (2)", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("description", "old a very great car", "a very great car"), - expectedVectorize: true, - }, - { - name: "diff one vectorizable prop changed (3)", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("reviews", []interface{}{ - "old a very great car", - "you should consider buying one", - }, []interface{}{ - "a very great car", - "you should consider buying one", - }), - expectedVectorize: true, - }, - { - name: "all non-vectorizable props changed", - skipped: "description", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("power", 123, 300). - WithProp("description", "old a very great car", "a very great car"), - expectedVectorize: false, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - ic := &fakeClassConfig{ - skippedProperty: test.skipped, - } - - client := &fakeClient{} - v := New(client) - - err := v.Object(context.Background(), test.input, test.diff, ic) - - require.Nil(t, err) - if test.expectedVectorize { - assert.Equal(t, models.C11yVector{0, 1, 2, 3}, test.input.Vector) - assert.NotNil(t, client.lastInput) - } else { - assert.Equal(t, models.C11yVector{0, 0, 0, 0}, test.input.Vector) - assert.Nil(t, client.lastInput) - } - }) - } -} - -func TestVectorizingActions(t *testing.T) { - type testCase struct { - name string - input *models.Object - expectedClientCall []string - noindex string - excludedProperty string // to simulate a schema where property names aren't vectorized - excludedClass string // to simulate a schema where class names aren't vectorized - } - - tests := []testCase{ - { - name: "empty object", - input: &models.Object{ - Class: "Flight", - }, - expectedClientCall: []string{"flight"}, - }, - { - name: "object with one string prop", - input: &models.Object{ - Class: "Flight", - Properties: map[string]interface{}{ - "brand": "Mercedes", - }, - }, - expectedClientCall: []string{"flight brand mercedes"}, - }, - - { - name: "object with one non-string prop", - input: &models.Object{ - Class: "Flight", - Properties: map[string]interface{}{ - "length": 300, - }, - }, - expectedClientCall: []string{"flight"}, - }, - - { - name: "object with a mix of props", - input: &models.Object{ - Class: "Flight", - Properties: map[string]interface{}{ - "brand": "best brand", - "length": 300, - "review": "a very great flight", - }, - }, - expectedClientCall: []string{"flight brand best brand review a very great flight"}, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - client := &fakeClient{} - v := New(client) - - ic := &fakeClassConfig{ - excludedProperty: test.excludedProperty, - skippedProperty: test.noindex, - vectorizeClassName: test.excludedClass != "Flight", - vectorizePropertyName: true, - } - err := v.Object(context.Background(), test.input, nil, ic) - - require.Nil(t, err) - assert.Equal(t, models.C11yVector{0, 1, 2, 3}, test.input.Vector) - expected := strings.Split(test.expectedClientCall[0], " ") - actual := strings.Split(client.lastInput[0], " ") - assert.ElementsMatch(t, expected, actual) - }) - } -} - -func TestVectorizingSearchTerms(t *testing.T) { - type testCase struct { - name string - input []string - expectedClientCall []string - } - - tests := []testCase{ - { - name: "single word", - input: []string{"car"}, - expectedClientCall: []string{"car"}, - }, - { - name: "multiple entries with multiple words", - input: []string{"car", "car brand"}, - expectedClientCall: []string{"car", "car brand"}, - }, - { - name: "multiple entries with upper casing", - input: []string{"Car", "Car Brand"}, - expectedClientCall: []string{"car", "car brand"}, - }, - { - name: "with camel cased words", - input: []string{"Car", "CarBrand"}, - expectedClientCall: []string{"car", "car brand"}, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - client := &fakeClient{} - v := New(client) - - res, err := v.Corpi(context.Background(), test.input) - - require.Nil(t, err) - assert.Equal(t, []float32{0, 1, 2, 3}, res) - assert.ElementsMatch(t, test.expectedClientCall, client.lastInput) - }) - } -} - -func newObjectDiffWithVector() *moduletools.ObjectDiff { - return moduletools.NewObjectDiff([]float32{0, 0, 0, 0}) -} diff --git a/modules/text2vec-gpt4all/clients/gpt4all.go b/modules/text2vec-gpt4all/clients/gpt4all.go deleted file mode 100644 index c319b3006e500ed65b32e921d0ecdc91712225d3..0000000000000000000000000000000000000000 --- a/modules/text2vec-gpt4all/clients/gpt4all.go +++ /dev/null @@ -1,99 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/modules/text2vec-gpt4all/ent" -) - -type client struct { - origin string - httpClient *http.Client - logger logrus.FieldLogger -} - -func New(origin string, timeout time.Duration, logger logrus.FieldLogger) *client { - return &client{ - origin: origin, - httpClient: &http.Client{ - Timeout: timeout, - }, - logger: logger, - } -} - -func (c *client) Vectorize(ctx context.Context, text string) (*ent.VectorizationResult, error) { - body, err := json.Marshal(vecRequest{text}) - if err != nil { - return nil, errors.Wrapf(err, "marshal body") - } - - req, err := http.NewRequestWithContext(ctx, "POST", c.url("/vectorize"), - bytes.NewReader(body)) - if err != nil { - return nil, errors.Wrap(err, "create POST request") - } - - res, err := c.httpClient.Do(req) - if err != nil { - return nil, errors.Wrap(err, "send POST request") - } - defer res.Body.Close() - - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - return nil, errors.Wrap(err, "read response body") - } - - var resBody vecResponse - if err := json.Unmarshal(bodyBytes, &resBody); err != nil { - return nil, errors.Wrap(err, "unmarshal response body") - } - - if res.StatusCode != 200 { - if resBody.Error != "" { - return nil, errors.Errorf("failed with status: %d error: %v", res.StatusCode, resBody.Error) - } - return nil, errors.Errorf("failed with status: %d", res.StatusCode) - } - - return &ent.VectorizationResult{ - Vector: resBody.Vector, - Dimensions: resBody.Dim, - Text: resBody.Text, - }, nil -} - -func (c *client) url(path string) string { - return fmt.Sprintf("%s%s", c.origin, path) -} - -type vecRequest struct { - Text string `json:"text"` -} - -type vecResponse struct { - Text string `json:"text"` - Vector []float32 `json:"vector"` - Dim int `json:"dim"` - Error string `json:"error"` -} diff --git a/modules/text2vec-gpt4all/clients/meta.go b/modules/text2vec-gpt4all/clients/meta.go deleted file mode 100644 index f330857ac03460f625c24ff02d67e16351eaad93..0000000000000000000000000000000000000000 --- a/modules/text2vec-gpt4all/clients/meta.go +++ /dev/null @@ -1,45 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "context" - "encoding/json" - "io" - "net/http" - - "github.com/pkg/errors" -) - -func (c *client) MetaInfo() (map[string]interface{}, error) { - req, err := http.NewRequestWithContext(context.Background(), "GET", c.url("/meta"), nil) - if err != nil { - return nil, errors.Wrap(err, "create GET meta request") - } - - res, err := c.httpClient.Do(req) - if err != nil { - return nil, errors.Wrap(err, "send GET meta request") - } - defer res.Body.Close() - - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - return nil, errors.Wrap(err, "read meta response body") - } - - var resBody map[string]interface{} - if err := json.Unmarshal(bodyBytes, &resBody); err != nil { - return nil, errors.Wrap(err, "unmarshal meta response body") - } - return resBody, nil -} diff --git a/modules/text2vec-gpt4all/clients/meta_test.go b/modules/text2vec-gpt4all/clients/meta_test.go deleted file mode 100644 index 38140cad97c3bb738433c0b55b428d80ae398070..0000000000000000000000000000000000000000 --- a/modules/text2vec-gpt4all/clients/meta_test.go +++ /dev/null @@ -1,74 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestGetMeta(t *testing.T) { - t.Run("when the server is providing meta", func(t *testing.T) { - server := httptest.NewServer(&testMetaHandler{t: t}) - defer server.Close() - c := New(server.URL, 0, nullLogger()) - meta, err := c.MetaInfo() - - assert.Nil(t, err) - assert.NotNil(t, meta) - metaModel := meta["name"] - require.NotNil(t, metaModel) - assert.Equal(t, "Bert", metaModel) - }) -} - -type testMetaHandler struct { - t *testing.T - // the test handler will report as not ready before the time has passed - readyTime time.Time -} - -func (f *testMetaHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - assert.Equal(f.t, "/meta", r.URL.String()) - assert.Equal(f.t, http.MethodGet, r.Method) - - if time.Since(f.readyTime) < 0 { - w.WriteHeader(http.StatusServiceUnavailable) - } - - w.Write([]byte(f.metaInfo())) -} - -func (f *testMetaHandler) metaInfo() string { - return `{ - "description": "Sbert
    • For embeddings", - "disableGUI": "true", - "filename": "ggml-all-MiniLM-L6-v2-f16.bin", - "filesize": "45521167", - "md5sum": "031bb5d5722c08d13e3e8eaf55c37391", - "name": "Bert", - "order": "t", - "parameters": "1 million", - "path": "/Users/marcin/.cache/gpt4all/ggml-all-MiniLM-L6-v2-f16.bin", - "promptTemplate": "### Human: \n{0}\n### Assistant:\n", - "quant": "f16", - "ramrequired": "1", - "requires": "2.4.14", - "systemPrompt": "", - "type": "Bert" -}` -} diff --git a/modules/text2vec-gpt4all/clients/startup.go b/modules/text2vec-gpt4all/clients/startup.go deleted file mode 100644 index aeb4db5371ce2f148d960259252b6f6106d2c750..0000000000000000000000000000000000000000 --- a/modules/text2vec-gpt4all/clients/startup.go +++ /dev/null @@ -1,68 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "context" - "net/http" - "time" - - "github.com/pkg/errors" -) - -func (c *client) WaitForStartup(initCtx context.Context, - interval time.Duration, -) error { - t := time.NewTicker(interval) - defer t.Stop() - expired := initCtx.Done() - var lastErr error - for { - select { - case <-t.C: - lastErr = c.checkReady(initCtx) - if lastErr == nil { - return nil - } - c.logger. - WithField("action", "gpt4all_remote_wait_for_startup"). - WithError(lastErr).Warnf("gpt4all remote service not ready") - case <-expired: - return errors.Wrapf(lastErr, "init context expired before remote was ready") - } - } -} - -func (c *client) checkReady(initCtx context.Context) error { - // spawn a new context (derived on the overall context) which is used to - // consider an individual request timed out - requestCtx, cancel := context.WithTimeout(initCtx, 500*time.Millisecond) - defer cancel() - - req, err := http.NewRequestWithContext(requestCtx, http.MethodGet, - c.url("/.well-known/ready"), nil) - if err != nil { - return errors.Wrap(err, "create check ready request") - } - - res, err := c.httpClient.Do(req) - if err != nil { - return errors.Wrap(err, "send check ready request") - } - - defer res.Body.Close() - if res.StatusCode > 299 { - return errors.Errorf("not ready: status %d", res.StatusCode) - } - - return nil -} diff --git a/modules/text2vec-gpt4all/clients/startup_test.go b/modules/text2vec-gpt4all/clients/startup_test.go deleted file mode 100644 index 7836347d937eaaa18e3c1415dcb4b2aac2d2b15b..0000000000000000000000000000000000000000 --- a/modules/text2vec-gpt4all/clients/startup_test.go +++ /dev/null @@ -1,98 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "context" - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/sirupsen/logrus" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestWaitForStartup(t *testing.T) { - t.Run("when the server is immediately ready", func(t *testing.T) { - server := httptest.NewServer(&testReadyHandler{t: t}) - defer server.Close() - c := New(server.URL, 0, nullLogger()) - err := c.WaitForStartup(context.Background(), 50*time.Millisecond) - - assert.Nil(t, err) - }) - - t.Run("when the server is down", func(t *testing.T) { - c := New("http://nothing-running-at-this-url", 0, nullLogger()) - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancel() - err := c.WaitForStartup(ctx, 150*time.Millisecond) - - require.NotNil(t, err, nullLogger()) - assert.Contains(t, err.Error(), "expired before remote was ready") - }) - - t.Run("when the server is alive, but not ready", func(t *testing.T) { - server := httptest.NewServer(&testReadyHandler{ - t: t, - readyTime: time.Now().Add(1 * time.Minute), - }) - c := New(server.URL, 0, nullLogger()) - defer server.Close() - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancel() - err := c.WaitForStartup(ctx, 50*time.Millisecond) - - require.NotNil(t, err) - assert.Contains(t, err.Error(), "expired before remote was ready") - }) - - t.Run("when the server is initially not ready, but then becomes ready", - func(t *testing.T) { - server := httptest.NewServer(&testReadyHandler{ - t: t, - readyTime: time.Now().Add(100 * time.Millisecond), - }) - c := New(server.URL, 0, nullLogger()) - defer server.Close() - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancel() - err := c.WaitForStartup(ctx, 50*time.Millisecond) - - require.Nil(t, err) - }) -} - -type testReadyHandler struct { - t *testing.T - // the test handler will report as not ready before the time has passed - readyTime time.Time -} - -func (f *testReadyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - assert.Equal(f.t, "/.well-known/ready", r.URL.String()) - assert.Equal(f.t, http.MethodGet, r.Method) - - if time.Since(f.readyTime) < 0 { - w.WriteHeader(http.StatusServiceUnavailable) - } - - w.WriteHeader(http.StatusNoContent) -} - -func nullLogger() logrus.FieldLogger { - l, _ := test.NewNullLogger() - return l -} diff --git a/modules/text2vec-gpt4all/config.go b/modules/text2vec-gpt4all/config.go deleted file mode 100644 index 4e9ece9d27185c9eacc16a56fa0985a434850a24..0000000000000000000000000000000000000000 --- a/modules/text2vec-gpt4all/config.go +++ /dev/null @@ -1,39 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modtransformers - -import ( - "context" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/schema" -) - -func (m *GPT4AllModule) ClassConfigDefaults() map[string]interface{} { - return map[string]interface{}{} -} - -func (m *GPT4AllModule) PropertyConfigDefaults( - dt *schema.DataType, -) map[string]interface{} { - return map[string]interface{}{} -} - -func (m *GPT4AllModule) ValidateClass(ctx context.Context, - class *models.Class, cfg moduletools.ClassConfig, -) error { - return nil -} - -var _ = modulecapabilities.ClassConfigurator(New()) diff --git a/modules/text2vec-gpt4all/ent/vectorization_result.go b/modules/text2vec-gpt4all/ent/vectorization_result.go deleted file mode 100644 index 69eee4438e46a4427a7368204c662419196cc526..0000000000000000000000000000000000000000 --- a/modules/text2vec-gpt4all/ent/vectorization_result.go +++ /dev/null @@ -1,18 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package ent - -type VectorizationResult struct { - Text string - Dimensions int - Vector []float32 -} diff --git a/modules/text2vec-gpt4all/module.go b/modules/text2vec-gpt4all/module.go deleted file mode 100644 index c81cb697b2dcce31e5fc9953e08b4272fb914045..0000000000000000000000000000000000000000 --- a/modules/text2vec-gpt4all/module.go +++ /dev/null @@ -1,153 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modtransformers - -import ( - "context" - "net/http" - "os" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/modules/text2vec-gpt4all/clients" - "github.com/weaviate/weaviate/modules/text2vec-gpt4all/vectorizer" - "github.com/weaviate/weaviate/usecases/modulecomponents/additional" -) - -const Name = "text2vec-gpt4all" - -func New() *GPT4AllModule { - return &GPT4AllModule{} -} - -type GPT4AllModule struct { - vectorizer textVectorizer - metaProvider metaProvider - graphqlProvider modulecapabilities.GraphQLArguments - searcher modulecapabilities.Searcher - nearTextTransformer modulecapabilities.TextTransform - logger logrus.FieldLogger - additionalPropertiesProvider modulecapabilities.AdditionalProperties -} - -type textVectorizer interface { - Object(ctx context.Context, obj *models.Object, objDiff *moduletools.ObjectDiff, - cfg moduletools.ClassConfig) error - Texts(ctx context.Context, input []string, - cfg moduletools.ClassConfig) ([]float32, error) -} - -type metaProvider interface { - MetaInfo() (map[string]interface{}, error) -} - -func (m *GPT4AllModule) Name() string { - return Name -} - -func (m *GPT4AllModule) Type() modulecapabilities.ModuleType { - return modulecapabilities.Text2Vec -} - -func (m *GPT4AllModule) Init(ctx context.Context, - params moduletools.ModuleInitParams, -) error { - m.logger = params.GetLogger() - - if err := m.initVectorizer(ctx, params.GetConfig().ModuleHttpClientTimeout, m.logger); err != nil { - return errors.Wrap(err, "init vectorizer") - } - - if err := m.initAdditionalPropertiesProvider(); err != nil { - return errors.Wrap(err, "init additional properties provider") - } - - return nil -} - -func (m *GPT4AllModule) InitExtension(modules []modulecapabilities.Module) error { - for _, module := range modules { - if module.Name() == m.Name() { - continue - } - if arg, ok := module.(modulecapabilities.TextTransformers); ok { - if arg != nil && arg.TextTransformers() != nil { - m.nearTextTransformer = arg.TextTransformers()["nearText"] - } - } - } - - if err := m.initNearText(); err != nil { - return errors.Wrap(err, "init graphql provider") - } - return nil -} - -func (m *GPT4AllModule) initVectorizer(ctx context.Context, timeout time.Duration, - logger logrus.FieldLogger, -) error { - uri := os.Getenv("GPT4ALL_INFERENCE_API") - if uri == "" { - return errors.New("required variable GPT4ALL_INFERENCE_API is not set") - } - - client := clients.New(uri, timeout, logger) - if err := client.WaitForStartup(ctx, 1*time.Second); err != nil { - return errors.Wrap(err, "init remote vectorizer") - } - - m.vectorizer = vectorizer.New(client) - m.metaProvider = client - - return nil -} - -func (m *GPT4AllModule) initAdditionalPropertiesProvider() error { - m.additionalPropertiesProvider = additional.NewText2VecProvider() - return nil -} - -func (m *GPT4AllModule) RootHandler() http.Handler { - // TODO: remove once this is a capability interface - return nil -} - -func (m *GPT4AllModule) VectorizeObject(ctx context.Context, - obj *models.Object, objDiff *moduletools.ObjectDiff, cfg moduletools.ClassConfig, -) error { - return m.vectorizer.Object(ctx, obj, objDiff, cfg) -} - -func (m *GPT4AllModule) MetaInfo() (map[string]interface{}, error) { - return m.metaProvider.MetaInfo() -} - -func (m *GPT4AllModule) AdditionalProperties() map[string]modulecapabilities.AdditionalProperty { - return m.additionalPropertiesProvider.AdditionalProperties() -} - -func (m *GPT4AllModule) VectorizeInput(ctx context.Context, - input string, cfg moduletools.ClassConfig, -) ([]float32, error) { - return m.vectorizer.Texts(ctx, []string{input}, cfg) -} - -// verify we implement the modules.Module interface -var ( - _ = modulecapabilities.Module(New()) - _ = modulecapabilities.Vectorizer(New()) - _ = modulecapabilities.MetaProvider(New()) -) diff --git a/modules/text2vec-gpt4all/nearText.go b/modules/text2vec-gpt4all/nearText.go deleted file mode 100644 index e2b3f4338c05c64d77b8c281729711ce04cb5a06..0000000000000000000000000000000000000000 --- a/modules/text2vec-gpt4all/nearText.go +++ /dev/null @@ -1,36 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modtransformers - -import ( - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/usecases/modulecomponents/nearText" -) - -func (m *GPT4AllModule) initNearText() error { - m.searcher = nearText.NewSearcher(m.vectorizer) - m.graphqlProvider = nearText.New(m.nearTextTransformer) - return nil -} - -func (m *GPT4AllModule) Arguments() map[string]modulecapabilities.GraphQLArgument { - return m.graphqlProvider.Arguments() -} - -func (m *GPT4AllModule) VectorSearches() map[string]modulecapabilities.VectorForParams { - return m.searcher.VectorSearches() -} - -var ( - _ = modulecapabilities.GraphQLArguments(New()) - _ = modulecapabilities.Searcher(New()) -) diff --git a/modules/text2vec-gpt4all/vectorizer/class_settings.go b/modules/text2vec-gpt4all/vectorizer/class_settings.go deleted file mode 100644 index 091526a07332953a44ed9cc2c46d44cb9836dfb5..0000000000000000000000000000000000000000 --- a/modules/text2vec-gpt4all/vectorizer/class_settings.go +++ /dev/null @@ -1,86 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "github.com/weaviate/weaviate/entities/moduletools" -) - -const ( - DefaultPropertyIndexed = true - DefaultVectorizeClassName = true - DefaultVectorizePropertyName = false -) - -type classSettings struct { - cfg moduletools.ClassConfig -} - -func NewClassSettings(cfg moduletools.ClassConfig) *classSettings { - return &classSettings{cfg: cfg} -} - -func (ic *classSettings) PropertyIndexed(propName string) bool { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return DefaultPropertyIndexed - } - - vcn, ok := ic.cfg.Property(propName)["skip"] - if !ok { - return DefaultPropertyIndexed - } - - asBool, ok := vcn.(bool) - if !ok { - return DefaultPropertyIndexed - } - - return !asBool -} - -func (ic *classSettings) VectorizePropertyName(propName string) bool { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return DefaultVectorizePropertyName - } - vcn, ok := ic.cfg.Property(propName)["vectorizePropertyName"] - if !ok { - return DefaultVectorizePropertyName - } - - asBool, ok := vcn.(bool) - if !ok { - return DefaultVectorizePropertyName - } - - return asBool -} - -func (ic *classSettings) VectorizeClassName() bool { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return DefaultVectorizeClassName - } - - vcn, ok := ic.cfg.Class()["vectorizeClassName"] - if !ok { - return DefaultVectorizeClassName - } - - asBool, ok := vcn.(bool) - if !ok { - return DefaultVectorizeClassName - } - - return asBool -} diff --git a/modules/text2vec-gpt4all/vectorizer/class_settings_test.go b/modules/text2vec-gpt4all/vectorizer/class_settings_test.go deleted file mode 100644 index e1698311daebc2746ea558d6c670ee973b555023..0000000000000000000000000000000000000000 --- a/modules/text2vec-gpt4all/vectorizer/class_settings_test.go +++ /dev/null @@ -1,106 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/usecases/modules" -) - -func TestClassSettings(t *testing.T) { - t.Run("with all defaults", func(t *testing.T) { - class := &models.Class{ - Class: "MyClass", - Properties: []*models.Property{{ - Name: "someProp", - }}, - } - - cfg := modules.NewClassBasedModuleConfig(class, "my-module", "tenant") - ic := NewClassSettings(cfg) - - assert.True(t, ic.PropertyIndexed("someProp")) - assert.False(t, ic.VectorizePropertyName("someProp")) - assert.True(t, ic.VectorizeClassName()) - }) - - t.Run("with a nil config", func(t *testing.T) { - // this is the case if we were running in a situation such as a - // cross-class vectorization of search time, as is the case with Explore - // {}, we then expect all default values - - ic := NewClassSettings(nil) - - assert.True(t, ic.PropertyIndexed("someProp")) - assert.False(t, ic.VectorizePropertyName("someProp")) - assert.True(t, ic.VectorizeClassName()) - }) - - t.Run("with all explicit config matching the defaults", func(t *testing.T) { - class := &models.Class{ - Class: "MyClass", - ModuleConfig: map[string]interface{}{ - "my-module": map[string]interface{}{ - "vectorizeClassName": true, - "poolingStrategy": "masked_mean", - }, - }, - Properties: []*models.Property{{ - Name: "someProp", - ModuleConfig: map[string]interface{}{ - "my-module": map[string]interface{}{ - "skip": false, - "vectorizePropertyName": false, - }, - }, - }}, - } - - cfg := modules.NewClassBasedModuleConfig(class, "my-module", "tenant") - ic := NewClassSettings(cfg) - - assert.True(t, ic.PropertyIndexed("someProp")) - assert.False(t, ic.VectorizePropertyName("someProp")) - assert.True(t, ic.VectorizeClassName()) - }) - - t.Run("with all explicit config using non-default values", func(t *testing.T) { - class := &models.Class{ - Class: "MyClass", - ModuleConfig: map[string]interface{}{ - "my-module": map[string]interface{}{ - "vectorizeClassName": false, - "poolingStrategy": "cls", - }, - }, - Properties: []*models.Property{{ - Name: "someProp", - ModuleConfig: map[string]interface{}{ - "my-module": map[string]interface{}{ - "skip": true, - "vectorizePropertyName": true, - }, - }, - }}, - } - - cfg := modules.NewClassBasedModuleConfig(class, "my-module", "tenant") - ic := NewClassSettings(cfg) - - assert.False(t, ic.PropertyIndexed("someProp")) - assert.True(t, ic.VectorizePropertyName("someProp")) - assert.False(t, ic.VectorizeClassName()) - }) -} diff --git a/modules/text2vec-gpt4all/vectorizer/fakes_for_test.go b/modules/text2vec-gpt4all/vectorizer/fakes_for_test.go deleted file mode 100644 index b6d656963690c22315d78800b95b8080644d839b..0000000000000000000000000000000000000000 --- a/modules/text2vec-gpt4all/vectorizer/fakes_for_test.go +++ /dev/null @@ -1,75 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - - "github.com/weaviate/weaviate/modules/text2vec-gpt4all/ent" -) - -type fakeClient struct { - lastInput string -} - -func (c *fakeClient) Vectorize(ctx context.Context, - text string, -) (*ent.VectorizationResult, error) { - c.lastInput = text - return &ent.VectorizationResult{ - Vector: []float32{0, 1, 2, 3}, - Dimensions: 4, - Text: text, - }, nil -} - -type fakeClassConfig struct { - classConfig map[string]interface{} - vectorizeClassName bool - vectorizePropertyName bool - skippedProperty string - excludedProperty string -} - -func (f fakeClassConfig) Class() map[string]interface{} { - classSettings := map[string]interface{}{ - "vectorizeClassName": f.vectorizeClassName, - } - return classSettings -} - -func (f fakeClassConfig) ClassByModuleName(moduleName string) map[string]interface{} { - return f.classConfig -} - -func (f fakeClassConfig) Property(propName string) map[string]interface{} { - if propName == f.skippedProperty { - return map[string]interface{}{ - "skip": true, - } - } - if propName == f.excludedProperty { - return map[string]interface{}{ - "vectorizePropertyName": false, - } - } - if f.vectorizePropertyName { - return map[string]interface{}{ - "vectorizePropertyName": true, - } - } - return nil -} - -func (f fakeClassConfig) Tenant() string { - return "" -} diff --git a/modules/text2vec-gpt4all/vectorizer/objects.go b/modules/text2vec-gpt4all/vectorizer/objects.go deleted file mode 100644 index 6c02e2b8b39e3eb80f7af02093e3a9b1943e6252..0000000000000000000000000000000000000000 --- a/modules/text2vec-gpt4all/vectorizer/objects.go +++ /dev/null @@ -1,75 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/modules/text2vec-gpt4all/ent" - objectsvectorizer "github.com/weaviate/weaviate/usecases/modulecomponents/vectorizer" -) - -type Vectorizer struct { - client Client - objectVectorizer *objectsvectorizer.ObjectVectorizer -} - -func New(client Client) *Vectorizer { - return &Vectorizer{ - client: client, - objectVectorizer: objectsvectorizer.New(), - } -} - -type Client interface { - Vectorize(ctx context.Context, text string) (*ent.VectorizationResult, error) -} - -type ClassSettings interface { - PropertyIndexed(property string) bool - VectorizeClassName() bool - VectorizePropertyName(propertyName string) bool -} - -func (v *Vectorizer) Object(ctx context.Context, object *models.Object, - objDiff *moduletools.ObjectDiff, cfg moduletools.ClassConfig, -) error { - vec, err := v.object(ctx, object.Class, object.Properties, objDiff, cfg) - if err != nil { - return err - } - - object.Vector = vec - return nil -} - -func (v *Vectorizer) object(ctx context.Context, className string, - schema interface{}, objDiff *moduletools.ObjectDiff, cfg moduletools.ClassConfig, -) ([]float32, error) { - text, vector, err := v.objectVectorizer.TextsOrVector(ctx, className, schema, objDiff, NewClassSettings(cfg)) - if err != nil { - return nil, err - } - if vector != nil { - // dont' re-vectorize - return vector, nil - } - // vectorize text - res, err := v.client.Vectorize(ctx, text) - if err != nil { - return nil, err - } - - return res.Vector, nil -} diff --git a/modules/text2vec-gpt4all/vectorizer/objects_test.go b/modules/text2vec-gpt4all/vectorizer/objects_test.go deleted file mode 100644 index b9df652cc1f00a4b446af3364bc0cb59675d91a0..0000000000000000000000000000000000000000 --- a/modules/text2vec-gpt4all/vectorizer/objects_test.go +++ /dev/null @@ -1,353 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/moduletools" -) - -func TestVectorizingObjects(t *testing.T) { - type testCase struct { - name string - input *models.Object - expectedClientCall string - noindex string - excludedProperty string // to simulate a schema where property names aren't vectorized - excludedClass string // to simulate a schema where class names aren't vectorized - } - - tests := []testCase{ - { - name: "empty object", - input: &models.Object{ - Class: "Car", - }, - expectedClientCall: "car", - }, - { - name: "object with one string prop", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "Mercedes", - }, - }, - expectedClientCall: "car brand mercedes", - }, - { - name: "object with one non-string prop", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "power": 300, - }, - }, - expectedClientCall: "car", - }, - { - name: "object with a mix of props", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: "car brand best brand review a very great car", - }, - { - name: "with a noindexed property", - noindex: "review", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: "car brand best brand", - }, - - { - name: "with the class name not vectorized", - excludedClass: "Car", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: "brand best brand review a very great car", - }, - { - name: "with a property name not vectorized", - excludedProperty: "review", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: "car brand best brand a very great car", - }, - { - name: "with no schema labels vectorized", - excludedProperty: "review", - excludedClass: "Car", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "review": "a very great car", - }, - }, - expectedClientCall: "a very great car", - }, - { - name: "with string/text arrays without propname or classname", - excludedProperty: "reviews", - excludedClass: "Car", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - expectedClientCall: "a very great car you should consider buying one", - }, - { - name: "with string/text arrays with propname and classname", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - expectedClientCall: "car reviews a very great car reviews you should consider buying one", - }, - { - name: "with compound class and prop names", - input: &models.Object{ - Class: "SuperCar", - Properties: map[string]interface{}{ - "brandOfTheCar": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: "super car brand of the car best brand review a very great car", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - client := &fakeClient{} - - v := New(client) - - ic := &fakeClassConfig{ - excludedProperty: test.excludedProperty, - skippedProperty: test.noindex, - vectorizeClassName: test.excludedClass != "Car", - vectorizePropertyName: true, - } - err := v.Object(context.Background(), test.input, nil, ic) - - require.Nil(t, err) - assert.Equal(t, models.C11yVector{0, 1, 2, 3}, test.input.Vector) - expected := strings.Split(test.expectedClientCall, " ") - actual := strings.Split(client.lastInput, " ") - assert.Equal(t, expected, actual) - }) - } -} - -func TestVectorizingObjectsWithDiff(t *testing.T) { - type testCase struct { - name string - input *models.Object - skipped string - diff *moduletools.ObjectDiff - expectedVectorize bool - } - - tests := []testCase{ - { - name: "no diff", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: nil, - expectedVectorize: true, - }, - { - name: "diff all props unchanged", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("brand", "best brand", "best brand"). - WithProp("power", 300, 300). - WithProp("description", "a very great car", "a very great car"). - WithProp("reviews", []interface{}{ - "a very great car", - "you should consider buying one", - }, []interface{}{ - "a very great car", - "you should consider buying one", - }), - expectedVectorize: false, - }, - { - name: "diff one vectorizable prop changed (1)", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("brand", "old best brand", "best brand"), - expectedVectorize: true, - }, - { - name: "diff one vectorizable prop changed (2)", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("description", "old a very great car", "a very great car"), - expectedVectorize: true, - }, - { - name: "diff one vectorizable prop changed (3)", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("reviews", []interface{}{ - "old a very great car", - "you should consider buying one", - }, []interface{}{ - "a very great car", - "you should consider buying one", - }), - expectedVectorize: true, - }, - { - name: "all non-vectorizable props changed", - skipped: "description", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("power", 123, 300). - WithProp("description", "old a very great car", "a very great car"), - expectedVectorize: false, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - ic := &fakeClassConfig{ - skippedProperty: test.skipped, - } - - client := &fakeClient{} - v := New(client) - - err := v.Object(context.Background(), test.input, test.diff, ic) - - require.Nil(t, err) - if test.expectedVectorize { - assert.Equal(t, models.C11yVector{0, 1, 2, 3}, test.input.Vector) - assert.NotEmpty(t, client.lastInput) - } else { - assert.Equal(t, models.C11yVector{0, 0, 0, 0}, test.input.Vector) - assert.Empty(t, client.lastInput) - } - }) - } -} - -func newObjectDiffWithVector() *moduletools.ObjectDiff { - return moduletools.NewObjectDiff([]float32{0, 0, 0, 0}) -} diff --git a/modules/text2vec-gpt4all/vectorizer/texts.go b/modules/text2vec-gpt4all/vectorizer/texts.go deleted file mode 100644 index b8a6a643d697a9b0ab54db43a7c1f6e58f31b458..0000000000000000000000000000000000000000 --- a/modules/text2vec-gpt4all/vectorizer/texts.go +++ /dev/null @@ -1,34 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/moduletools" - libvectorizer "github.com/weaviate/weaviate/usecases/vectorizer" -) - -func (v *Vectorizer) Texts(ctx context.Context, inputs []string, - cfg moduletools.ClassConfig, -) ([]float32, error) { - vectors := make([][]float32, len(inputs)) - for i := range inputs { - res, err := v.client.Vectorize(ctx, inputs[i]) - if err != nil { - return nil, errors.Wrap(err, "remote client vectorize") - } - vectors[i] = res.Vector - } - return libvectorizer.CombineVectors(vectors), nil -} diff --git a/modules/text2vec-gpt4all/vectorizer/texts_test.go b/modules/text2vec-gpt4all/vectorizer/texts_test.go deleted file mode 100644 index a460b2d599535f2d3573cea7cd341bafdd5fd7b1..0000000000000000000000000000000000000000 --- a/modules/text2vec-gpt4all/vectorizer/texts_test.go +++ /dev/null @@ -1,63 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// as used in the nearText searcher -func TestVectorizingTexts(t *testing.T) { - type testCase struct { - name string - input []string - expectedClientCall string - } - - tests := []testCase{ - { - name: "single word", - input: []string{"hello"}, - expectedClientCall: "hello", - }, - { - name: "multiple words", - input: []string{"hello world, this is me!"}, - expectedClientCall: "hello world, this is me!", - }, - - { - name: "multiple sentences", - input: []string{"this is sentence 1", "and here's number 2"}, - expectedClientCall: "and here's number 2", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - client := &fakeClient{} - - v := New(client) - - settings := &fakeClassConfig{} - vec, err := v.Texts(context.Background(), test.input, settings) - - require.Nil(t, err) - assert.Equal(t, []float32{0, 1, 2, 3}, vec) - assert.Equal(t, test.expectedClientCall, client.lastInput) - }) - } -} diff --git a/modules/text2vec-huggingface/clients/bert_embeddings.go b/modules/text2vec-huggingface/clients/bert_embeddings.go deleted file mode 100644 index 8bf894c0d7a1a7ad0e63e5df437f65ed95ea5c8b..0000000000000000000000000000000000000000 --- a/modules/text2vec-huggingface/clients/bert_embeddings.go +++ /dev/null @@ -1,41 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import "errors" - -type bertEmbeddingsDecoder struct{} - -func newBertEmbeddingsDecoder() *bertEmbeddingsDecoder { - return &bertEmbeddingsDecoder{} -} - -func (d bertEmbeddingsDecoder) calculateVector(embeddings [][]float32) ([]float32, error) { - if len(embeddings) > 0 { - vectorLen := len(embeddings[0]) - sumEmbeddings := make([]float32, vectorLen) - embeddingsLen := len(embeddings) - var sum float32 - for i := 0; i < vectorLen; i++ { - sum = 0 - for j := 0; j < embeddingsLen; j++ { - sum += embeddings[j][i] - } - sumEmbeddings[i] = sum - } - for i := range sumEmbeddings { - sumEmbeddings[i] = sumEmbeddings[i] / float32(embeddingsLen) - } - return sumEmbeddings, nil - } - return nil, errors.New("missing embeddings") -} diff --git a/modules/text2vec-huggingface/clients/bert_embeddings_test.go b/modules/text2vec-huggingface/clients/bert_embeddings_test.go deleted file mode 100644 index 5f0e3faefaa088dcf92863d3739b46814d8bd754..0000000000000000000000000000000000000000 --- a/modules/text2vec-huggingface/clients/bert_embeddings_test.go +++ /dev/null @@ -1,64 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "reflect" - "testing" -) - -func Test_bertEmbeddingsDecoder_calculateVector(t *testing.T) { - tests := []struct { - name string - embeddings [][]float32 - want []float32 - wantErr bool - }{ - { - name: "nil", - embeddings: nil, - wantErr: true, - }, - { - name: "empty", - embeddings: [][]float32{}, - wantErr: true, - }, - { - name: "just one vector", - embeddings: [][]float32{{-0.17978577315807343}}, - want: []float32{-0.17978577315807343}, - }, - { - name: "distilbert-base-uncased", - embeddings: [][]float32{ - {-0.17978577315807343, -0.0678672045469284, 0.1706605851650238, -0.1639413982629776, -0.12804915010929108, 0.017568372189998627, 0.1610901951789856, 0.19909054040908813, -0.26103103160858154, -0.14505508542060852}, - {-0.25516796112060547, -0.054695576429367065, 0.13527897000312805, -0.3919253945350647, 0.1900954395532608, 0.5994636416435242, 0.5798457264900208, 0.6522972583770752, -0.08617493510246277, -0.35053199529647827}, - {0.930827260017395, 0.3315476179122925, -0.323006272315979, 0.18198077380657196, -0.3299236297607422, -0.5998684763908386, 0.3299814462661743, -0.6352149844169617, 0.5154204368591309, 0.11740084737539291}, - }, - want: []float32{0.1652911752462387, 0.06966160982847214, -0.005688905715942383, -0.12462866306304932, -0.08929244428873062, 0.005721171852201223, 0.35697245597839355, 0.07205760478973389, 0.05607149004936218, -0.1260620802640915}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - d := bertEmbeddingsDecoder{} - got, err := d.calculateVector(tt.embeddings) - if (err != nil) != tt.wantErr { - t.Errorf("bertEmbeddingsDecoder.calculateVector() error = %v, wantErr %v", err, tt.wantErr) - return - } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("bertEmbeddingsDecoder.calculateVector() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/modules/text2vec-huggingface/clients/huggingface.go b/modules/text2vec-huggingface/clients/huggingface.go deleted file mode 100644 index c664eaf4259d81d2d347a8a22554f0e7b94e2288..0000000000000000000000000000000000000000 --- a/modules/text2vec-huggingface/clients/huggingface.go +++ /dev/null @@ -1,228 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "time" - - "github.com/weaviate/weaviate/usecases/modulecomponents" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/modules/text2vec-huggingface/ent" -) - -const ( - DefaultOrigin = "https://api-inference.huggingface.co" - DefaultPath = "pipeline/feature-extraction" -) - -type embeddingsRequest struct { - Inputs []string `json:"inputs"` - Options *options `json:"options,omitempty"` -} - -type options struct { - WaitForModel bool `json:"wait_for_model,omitempty"` - UseGPU bool `json:"use_gpu,omitempty"` - UseCache bool `json:"use_cache,omitempty"` -} - -type embedding [][]float32 - -type embeddingBert [][][][]float32 - -type embeddingObject struct { - Embeddings embedding `json:"embeddings"` -} - -type huggingFaceApiError struct { - Error string `json:"error"` - EstimatedTime *float32 `json:"estimated_time,omitempty"` - Warnings []string `json:"warnings"` -} - -type vectorizer struct { - apiKey string - httpClient *http.Client - bertEmbeddingsDecoder *bertEmbeddingsDecoder - logger logrus.FieldLogger -} - -func New(apiKey string, timeout time.Duration, logger logrus.FieldLogger) *vectorizer { - return &vectorizer{ - apiKey: apiKey, - httpClient: &http.Client{ - Timeout: timeout, - }, - bertEmbeddingsDecoder: newBertEmbeddingsDecoder(), - logger: logger, - } -} - -func (v *vectorizer) Vectorize(ctx context.Context, input string, - config ent.VectorizationConfig, -) (*ent.VectorizationResult, error) { - return v.vectorize(ctx, v.getURL(config), input, v.getOptions(config)) -} - -func (v *vectorizer) VectorizeQuery(ctx context.Context, input string, - config ent.VectorizationConfig, -) (*ent.VectorizationResult, error) { - return v.vectorize(ctx, v.getURL(config), input, v.getOptions(config)) -} - -func (v *vectorizer) vectorize(ctx context.Context, url string, - input string, options options, -) (*ent.VectorizationResult, error) { - body, err := json.Marshal(embeddingsRequest{ - Inputs: []string{input}, - Options: &options, - }) - if err != nil { - return nil, errors.Wrapf(err, "marshal body") - } - - req, err := http.NewRequestWithContext(ctx, "POST", url, - bytes.NewReader(body)) - if err != nil { - return nil, errors.Wrap(err, "create POST request") - } - if apiKey := v.getApiKey(ctx); apiKey != "" { - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", apiKey)) - } - req.Header.Add("Content-Type", "application/json") - - res, err := v.httpClient.Do(req) - if err != nil { - return nil, errors.Wrap(err, "send POST request") - } - defer res.Body.Close() - - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - return nil, errors.Wrap(err, "read response body") - } - - if err := checkResponse(res, bodyBytes); err != nil { - return nil, err - } - - vector, err := v.decodeVector(bodyBytes) - if err != nil { - return nil, errors.Wrap(err, "cannot decode vector") - } - - return &ent.VectorizationResult{ - Text: input, - Dimensions: len(vector), - Vector: vector, - }, nil -} - -func checkResponse(res *http.Response, bodyBytes []byte) error { - if res.StatusCode < 400 { - return nil - } - - var resBody huggingFaceApiError - if err := json.Unmarshal(bodyBytes, &resBody); err != nil { - return fmt.Errorf("unmarshal error response body: %v", string(bodyBytes)) - } - - message := fmt.Sprintf("failed with status: %d", res.StatusCode) - if resBody.Error != "" { - message = fmt.Sprintf("%s error: %v", message, resBody.Error) - if resBody.EstimatedTime != nil { - message = fmt.Sprintf("%s estimated time: %v", message, *resBody.EstimatedTime) - } - if len(resBody.Warnings) > 0 { - message = fmt.Sprintf("%s warnings: %v", message, resBody.Warnings) - } - } - - if res.StatusCode == http.StatusInternalServerError { - message = fmt.Sprintf("connection to HuggingFace %v", message) - } - - return errors.New(message) -} - -func (v *vectorizer) decodeVector(bodyBytes []byte) ([]float32, error) { - var emb embedding - if err := json.Unmarshal(bodyBytes, &emb); err != nil { - var embObject embeddingObject - if err := json.Unmarshal(bodyBytes, &embObject); err != nil { - var embBert embeddingBert - if err := json.Unmarshal(bodyBytes, &embBert); err != nil { - return nil, errors.Wrap(err, "unmarshal response body") - } - - if len(embBert) == 1 && len(embBert[0]) == 1 { - return v.bertEmbeddingsDecoder.calculateVector(embBert[0][0]) - } - - return nil, errors.New("unprocessable response body") - } - if len(embObject.Embeddings) == 1 { - return embObject.Embeddings[0], nil - } - - return nil, errors.New("unprocessable response body") - } - - if len(emb) == 1 { - return emb[0], nil - } - - return nil, errors.New("unprocessable response body") -} - -func (v *vectorizer) getApiKey(ctx context.Context) string { - if len(v.apiKey) > 0 { - return v.apiKey - } - key := "X-Huggingface-Api-Key" - apiKey := ctx.Value(key) - // try getting header from GRPC if not successful - if apiKey == nil { - apiKey = modulecomponents.GetValueFromGRPC(ctx, key) - } - - if apiKeyHeader, ok := apiKey.([]string); ok && - len(apiKeyHeader) > 0 && len(apiKeyHeader[0]) > 0 { - return apiKeyHeader[0] - } - return "" -} - -func (v *vectorizer) getOptions(config ent.VectorizationConfig) options { - return options{ - WaitForModel: config.WaitForModel, - UseGPU: config.UseGPU, - UseCache: config.UseCache, - } -} - -func (v *vectorizer) getURL(config ent.VectorizationConfig) string { - if config.EndpointURL != "" { - return config.EndpointURL - } - - return fmt.Sprintf("%s/%s/%s", DefaultOrigin, DefaultPath, config.Model) -} diff --git a/modules/text2vec-huggingface/clients/huggingface_test.go b/modules/text2vec-huggingface/clients/huggingface_test.go deleted file mode 100644 index e724c150e0545e0f0f3e7a62bb876720f55dbb68..0000000000000000000000000000000000000000 --- a/modules/text2vec-huggingface/clients/huggingface_test.go +++ /dev/null @@ -1,283 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "context" - "encoding/json" - "io" - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/modules/text2vec-huggingface/ent" -) - -func TestClient(t *testing.T) { - t.Run("when all is fine", func(t *testing.T) { - server := httptest.NewServer(&fakeHandler{t: t}) - defer server.Close() - c := &vectorizer{ - apiKey: "apiKey", - httpClient: &http.Client{}, - logger: nullLogger(), - } - expected := &ent.VectorizationResult{ - Text: "This is my text", - Vector: []float32{0.1, 0.2, 0.3}, - Dimensions: 3, - } - res, err := c.Vectorize(context.Background(), "This is my text", - ent.VectorizationConfig{ - Model: "sentence-transformers/gtr-t5-xxl", - WaitForModel: false, - UseGPU: false, - UseCache: true, - EndpointURL: server.URL, - }) - - assert.Nil(t, err) - assert.Equal(t, expected, res) - }) - - t.Run("when the context is expired", func(t *testing.T) { - server := httptest.NewServer(&fakeHandler{t: t}) - defer server.Close() - c := &vectorizer{ - apiKey: "apiKey", - httpClient: &http.Client{}, - logger: nullLogger(), - } - ctx, cancel := context.WithDeadline(context.Background(), time.Now()) - defer cancel() - - _, err := c.Vectorize(ctx, "This is my text", ent.VectorizationConfig{ - EndpointURL: server.URL, - }) - - require.NotNil(t, err) - assert.Contains(t, err.Error(), "context deadline exceeded") - }) - - t.Run("when the server returns an error", func(t *testing.T) { - server := httptest.NewServer(&fakeHandler{ - t: t, - serverError: errors.Errorf("nope, not gonna happen"), - }) - defer server.Close() - c := &vectorizer{ - apiKey: "apiKey", - httpClient: &http.Client{}, - logger: nullLogger(), - } - _, err := c.Vectorize(context.Background(), "This is my text", - ent.VectorizationConfig{ - EndpointURL: server.URL, - }) - - require.NotNil(t, err) - assert.Equal(t, err.Error(), "connection to HuggingFace failed with status: 500 error: nope, not gonna happen estimated time: 20") - }) - - t.Run("when HuggingFace key is passed using X-Huggingface-Api-Key header", func(t *testing.T) { - server := httptest.NewServer(&fakeHandler{t: t}) - defer server.Close() - c := &vectorizer{ - apiKey: "", - httpClient: &http.Client{}, - logger: nullLogger(), - } - ctxWithValue := context.WithValue(context.Background(), - "X-Huggingface-Api-Key", []string{"some-key"}) - - expected := &ent.VectorizationResult{ - Text: "This is my text", - Vector: []float32{0.1, 0.2, 0.3}, - Dimensions: 3, - } - res, err := c.Vectorize(ctxWithValue, "This is my text", - ent.VectorizationConfig{ - Model: "sentence-transformers/gtr-t5-xxl", - WaitForModel: true, - UseGPU: false, - UseCache: true, - EndpointURL: server.URL, - }) - - require.Nil(t, err) - assert.Equal(t, expected, res) - }) - - t.Run("when a request requires an API KEY", func(t *testing.T) { - server := httptest.NewServer(&fakeHandler{ - t: t, - serverError: errors.Errorf("A valid user or organization token is required"), - }) - defer server.Close() - c := &vectorizer{ - apiKey: "", - httpClient: &http.Client{}, - logger: nullLogger(), - } - ctxWithValue := context.WithValue(context.Background(), - "X-Huggingface-Api-Key", []string{""}) - - _, err := c.Vectorize(ctxWithValue, "This is my text", - ent.VectorizationConfig{ - Model: "sentence-transformers/gtr-t5-xxl", - EndpointURL: server.URL, - }) - - require.NotNil(t, err) - assert.Equal(t, err.Error(), "failed with status: 401 error: A valid user or organization token is required") - }) - - t.Run("when the server returns an error with warnings", func(t *testing.T) { - server := httptest.NewServer(&fakeHandler{ - t: t, - serverError: errors.Errorf("with warnings"), - }) - defer server.Close() - c := &vectorizer{ - apiKey: "apiKey", - httpClient: &http.Client{}, - logger: nullLogger(), - } - _, err := c.Vectorize(context.Background(), "This is my text", - ent.VectorizationConfig{ - EndpointURL: server.URL, - }) - - require.NotNil(t, err) - assert.Equal(t, err.Error(), "connection to HuggingFace failed with status: 500 error: with warnings "+ - "warnings: [There was an inference error: CUDA error: all CUDA-capable devices are busy or unavailable\n"+ - "CUDA kernel errors might be asynchronously reported at some other API call,so the stacktrace below might be incorrect.\n"+ - "For debugging consider passing CUDA_LAUNCH_BLOCKING=1.]") - }) -} - -type fakeHandler struct { - t *testing.T - serverError error -} - -func (f *fakeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - assert.Equal(f.t, http.MethodPost, r.Method) - - if f.serverError != nil { - switch f.serverError.Error() { - case "with warnings": - embeddingError := map[string]interface{}{ - "error": f.serverError.Error(), - "warnings": []string{ - "There was an inference error: CUDA error: all CUDA-capable devices are busy or unavailable\n" + - "CUDA kernel errors might be asynchronously reported at some other API call,so the stacktrace below might be incorrect.\n" + - "For debugging consider passing CUDA_LAUNCH_BLOCKING=1.", - }, - } - outBytes, err := json.Marshal(embeddingError) - require.Nil(f.t, err) - - w.WriteHeader(http.StatusInternalServerError) - w.Write(outBytes) - return - case "A valid user or organization token is required": - embeddingError := map[string]interface{}{ - "error": "A valid user or organization token is required", - } - outBytes, err := json.Marshal(embeddingError) - require.Nil(f.t, err) - - w.WriteHeader(http.StatusUnauthorized) - w.Write(outBytes) - return - default: - embeddingError := map[string]interface{}{ - "error": f.serverError.Error(), - "estimated_time": 20.0, - } - outBytes, err := json.Marshal(embeddingError) - require.Nil(f.t, err) - - w.WriteHeader(http.StatusInternalServerError) - w.Write(outBytes) - return - } - } - - bodyBytes, err := io.ReadAll(r.Body) - require.Nil(f.t, err) - defer r.Body.Close() - - var b map[string]interface{} - require.Nil(f.t, json.Unmarshal(bodyBytes, &b)) - - textInputs := b["inputs"].([]interface{}) - assert.Greater(f.t, len(textInputs), 0) - textInput := textInputs[0].(string) - assert.Greater(f.t, len(textInput), 0) - - // TODO: fix this - embedding := [][]float32{{0.1, 0.2, 0.3}} - outBytes, err := json.Marshal(embedding) - require.Nil(f.t, err) - - w.Write(outBytes) -} - -func nullLogger() logrus.FieldLogger { - l, _ := test.NewNullLogger() - return l -} - -func Test_getURL(t *testing.T) { - v := &vectorizer{} - - tests := []struct { - name string - config ent.VectorizationConfig - want string - }{ - { - name: "Facebook DPR model", - config: ent.VectorizationConfig{ - Model: "sentence-transformers/facebook-dpr-ctx_encoder-multiset-base", - }, - want: "https://api-inference.huggingface.co/pipeline/feature-extraction/sentence-transformers/facebook-dpr-ctx_encoder-multiset-base", - }, - { - name: "BERT base model (uncased)", - config: ent.VectorizationConfig{ - Model: "bert-base-uncased", - }, - want: "https://api-inference.huggingface.co/pipeline/feature-extraction/bert-base-uncased", - }, - { - name: "BERT base model (uncased)", - config: ent.VectorizationConfig{ - EndpointURL: "https://self-hosted-instance.com/bert-base-uncased", - }, - want: "https://self-hosted-instance.com/bert-base-uncased", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, v.getURL(tt.config)) - }) - } -} diff --git a/modules/text2vec-huggingface/clients/meta.go b/modules/text2vec-huggingface/clients/meta.go deleted file mode 100644 index af0869618138bca54b1c5919cc86f689cdd569be..0000000000000000000000000000000000000000 --- a/modules/text2vec-huggingface/clients/meta.go +++ /dev/null @@ -1,19 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -func (v *vectorizer) MetaInfo() (map[string]interface{}, error) { - return map[string]interface{}{ - "name": "Hugging Face Module", - "documentationHref": "https://huggingface.co/docs/api-inference/detailed_parameters#feature-extraction-task", - }, nil -} diff --git a/modules/text2vec-huggingface/config.go b/modules/text2vec-huggingface/config.go deleted file mode 100644 index 322abc48547e682717ca839a5f71f5b14282cab8..0000000000000000000000000000000000000000 --- a/modules/text2vec-huggingface/config.go +++ /dev/null @@ -1,46 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modhuggingface - -import ( - "context" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/modules/text2vec-huggingface/vectorizer" -) - -func (m *HuggingFaceModule) ClassConfigDefaults() map[string]interface{} { - return map[string]interface{}{ - "vectorizeClassName": vectorizer.DefaultVectorizeClassName, - } -} - -func (m *HuggingFaceModule) PropertyConfigDefaults( - dt *schema.DataType, -) map[string]interface{} { - return map[string]interface{}{ - "skip": !vectorizer.DefaultPropertyIndexed, - "vectorizePropertyName": vectorizer.DefaultVectorizePropertyName, - } -} - -func (m *HuggingFaceModule) ValidateClass(ctx context.Context, - class *models.Class, cfg moduletools.ClassConfig, -) error { - settings := vectorizer.NewClassSettings(cfg) - return settings.Validate(class) -} - -var _ = modulecapabilities.ClassConfigurator(New()) diff --git a/modules/text2vec-huggingface/ent/vectorization_config.go b/modules/text2vec-huggingface/ent/vectorization_config.go deleted file mode 100644 index 359963b9025139d041739413b43f4e6885191194..0000000000000000000000000000000000000000 --- a/modules/text2vec-huggingface/ent/vectorization_config.go +++ /dev/null @@ -1,18 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package ent - -type VectorizationConfig struct { - EndpointURL string - Model string - WaitForModel, UseGPU, UseCache bool -} diff --git a/modules/text2vec-huggingface/ent/vectorization_result.go b/modules/text2vec-huggingface/ent/vectorization_result.go deleted file mode 100644 index 69eee4438e46a4427a7368204c662419196cc526..0000000000000000000000000000000000000000 --- a/modules/text2vec-huggingface/ent/vectorization_result.go +++ /dev/null @@ -1,18 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package ent - -type VectorizationResult struct { - Text string - Dimensions int - Vector []float32 -} diff --git a/modules/text2vec-huggingface/module.go b/modules/text2vec-huggingface/module.go deleted file mode 100644 index c3e4d7cc3e557065039408472dc2282fa2f55dbb..0000000000000000000000000000000000000000 --- a/modules/text2vec-huggingface/module.go +++ /dev/null @@ -1,149 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modhuggingface - -import ( - "context" - "net/http" - "os" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/modules/text2vec-huggingface/clients" - "github.com/weaviate/weaviate/modules/text2vec-huggingface/vectorizer" - "github.com/weaviate/weaviate/usecases/modulecomponents/additional" -) - -const Name = "text2vec-huggingface" - -func New() *HuggingFaceModule { - return &HuggingFaceModule{} -} - -type HuggingFaceModule struct { - vectorizer textVectorizer - metaProvider metaProvider - graphqlProvider modulecapabilities.GraphQLArguments - searcher modulecapabilities.Searcher - nearTextTransformer modulecapabilities.TextTransform - logger logrus.FieldLogger - additionalPropertiesProvider modulecapabilities.AdditionalProperties -} - -type textVectorizer interface { - Object(ctx context.Context, obj *models.Object, objDiff *moduletools.ObjectDiff, - cfg moduletools.ClassConfig) error - Texts(ctx context.Context, input []string, - cfg moduletools.ClassConfig) ([]float32, error) -} - -type metaProvider interface { - MetaInfo() (map[string]interface{}, error) -} - -func (m *HuggingFaceModule) Name() string { - return Name -} - -func (m *HuggingFaceModule) Type() modulecapabilities.ModuleType { - return modulecapabilities.Text2MultiVec -} - -func (m *HuggingFaceModule) Init(ctx context.Context, - params moduletools.ModuleInitParams, -) error { - m.logger = params.GetLogger() - - if err := m.initVectorizer(ctx, params.GetConfig().ModuleHttpClientTimeout, m.logger); err != nil { - return errors.Wrap(err, "init vectorizer") - } - - if err := m.initAdditionalPropertiesProvider(); err != nil { - return errors.Wrap(err, "init additional properties provider") - } - - return nil -} - -func (m *HuggingFaceModule) InitExtension(modules []modulecapabilities.Module) error { - for _, module := range modules { - if module.Name() == m.Name() { - continue - } - if arg, ok := module.(modulecapabilities.TextTransformers); ok { - if arg != nil && arg.TextTransformers() != nil { - m.nearTextTransformer = arg.TextTransformers()["nearText"] - } - } - } - - if err := m.initNearText(); err != nil { - return errors.Wrap(err, "init graphql provider") - } - return nil -} - -func (m *HuggingFaceModule) initVectorizer(ctx context.Context, timeout time.Duration, - logger logrus.FieldLogger, -) error { - apiKey := os.Getenv("HUGGINGFACE_APIKEY") - client := clients.New(apiKey, timeout, logger) - - m.vectorizer = vectorizer.New(client) - m.metaProvider = client - - return nil -} - -func (m *HuggingFaceModule) initAdditionalPropertiesProvider() error { - m.additionalPropertiesProvider = additional.NewText2VecProvider() - return nil -} - -func (m *HuggingFaceModule) RootHandler() http.Handler { - // TODO: remove once this is a capability interface - return nil -} - -func (m *HuggingFaceModule) VectorizeObject(ctx context.Context, - obj *models.Object, objDiff *moduletools.ObjectDiff, cfg moduletools.ClassConfig, -) error { - return m.vectorizer.Object(ctx, obj, objDiff, cfg) -} - -func (m *HuggingFaceModule) MetaInfo() (map[string]interface{}, error) { - return m.metaProvider.MetaInfo() -} - -func (m *HuggingFaceModule) AdditionalProperties() map[string]modulecapabilities.AdditionalProperty { - return m.additionalPropertiesProvider.AdditionalProperties() -} - -func (m *HuggingFaceModule) VectorizeInput(ctx context.Context, - input string, cfg moduletools.ClassConfig, -) ([]float32, error) { - return m.vectorizer.Texts(ctx, []string{input}, cfg) -} - -// verify we implement the modules.Module interface -var ( - _ = modulecapabilities.Module(New()) - _ = modulecapabilities.Vectorizer(New()) - _ = modulecapabilities.MetaProvider(New()) - _ = modulecapabilities.Searcher(New()) - _ = modulecapabilities.GraphQLArguments(New()) - _ = modulecapabilities.InputVectorizer(New()) -) diff --git a/modules/text2vec-huggingface/nearText.go b/modules/text2vec-huggingface/nearText.go deleted file mode 100644 index c3897d221a17715e20ee0ab4d67e6ca514c3b68e..0000000000000000000000000000000000000000 --- a/modules/text2vec-huggingface/nearText.go +++ /dev/null @@ -1,36 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modhuggingface - -import ( - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/usecases/modulecomponents/nearText" -) - -func (m *HuggingFaceModule) initNearText() error { - m.searcher = nearText.NewSearcher(m.vectorizer) - m.graphqlProvider = nearText.New(m.nearTextTransformer) - return nil -} - -func (m *HuggingFaceModule) Arguments() map[string]modulecapabilities.GraphQLArgument { - return m.graphqlProvider.Arguments() -} - -func (m *HuggingFaceModule) VectorSearches() map[string]modulecapabilities.VectorForParams { - return m.searcher.VectorSearches() -} - -var ( - _ = modulecapabilities.GraphQLArguments(New()) - _ = modulecapabilities.Searcher(New()) -) diff --git a/modules/text2vec-huggingface/vectorizer/class_settings.go b/modules/text2vec-huggingface/vectorizer/class_settings.go deleted file mode 100644 index 67a64632a727c1f1c6ffc2d13aa6fc8fa4e1972b..0000000000000000000000000000000000000000 --- a/modules/text2vec-huggingface/vectorizer/class_settings.go +++ /dev/null @@ -1,274 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "fmt" - - "github.com/pkg/errors" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/schema" -) - -const ( - DefaultHuggingFaceModel = "sentence-transformers/msmarco-bert-base-dot-v5" - DefaultOptionWaitForModel = false - DefaultOptionUseGPU = false - DefaultOptionUseCache = true - DefaultVectorizeClassName = true - DefaultPropertyIndexed = true - DefaultVectorizePropertyName = false -) - -type classSettings struct { - cfg moduletools.ClassConfig -} - -func NewClassSettings(cfg moduletools.ClassConfig) *classSettings { - return &classSettings{cfg: cfg} -} - -func (cs *classSettings) PropertyIndexed(propName string) bool { - if cs.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return DefaultPropertyIndexed - } - - vcn, ok := cs.cfg.Property(propName)["skip"] - if !ok { - return DefaultPropertyIndexed - } - - asBool, ok := vcn.(bool) - if !ok { - return DefaultPropertyIndexed - } - - return !asBool -} - -func (cs *classSettings) VectorizePropertyName(propName string) bool { - if cs.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return DefaultVectorizePropertyName - } - vcn, ok := cs.cfg.Property(propName)["vectorizePropertyName"] - if !ok { - return DefaultVectorizePropertyName - } - - asBool, ok := vcn.(bool) - if !ok { - return DefaultVectorizePropertyName - } - - return asBool -} - -func (cs *classSettings) EndpointURL() string { - return cs.getEndpointURL() -} - -func (cs *classSettings) PassageModel() string { - model := cs.getPassageModel() - if model == "" { - return DefaultHuggingFaceModel - } - return model -} - -func (cs *classSettings) QueryModel() string { - model := cs.getQueryModel() - if model == "" { - return DefaultHuggingFaceModel - } - return model -} - -func (cs *classSettings) OptionWaitForModel() bool { - return cs.getOptionOrDefault("waitForModel", DefaultOptionWaitForModel) -} - -func (cs *classSettings) OptionUseGPU() bool { - return cs.getOptionOrDefault("useGPU", DefaultOptionUseGPU) -} - -func (cs *classSettings) OptionUseCache() bool { - return cs.getOptionOrDefault("useCache", DefaultOptionUseCache) -} - -func (cs *classSettings) VectorizeClassName() bool { - if cs.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return DefaultVectorizeClassName - } - - vcn, ok := cs.cfg.Class()["vectorizeClassName"] - if !ok { - return DefaultVectorizeClassName - } - - asBool, ok := vcn.(bool) - if !ok { - return DefaultVectorizeClassName - } - - return asBool -} - -func (cs *classSettings) Validate(class *models.Class) error { - if cs.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return errors.New("empty config") - } - - err := cs.validateClassSettings() - if err != nil { - return err - } - - err = cs.validateIndexState(class, cs) - if err != nil { - return err - } - - return nil -} - -func (cs *classSettings) validateClassSettings() error { - endpointURL := cs.getEndpointURL() - if endpointURL != "" { - // endpoint is set, should be used for feature extraction - // all other settings are not relevant - return nil - } - - model := cs.getProperty("model") - passageModel := cs.getProperty("passageModel") - queryModel := cs.getProperty("queryModel") - - if model != "" && (passageModel != "" || queryModel != "") { - return errors.New("only one setting must be set either 'model' or 'passageModel' with 'queryModel'") - } - - if model == "" { - if passageModel != "" && queryModel == "" { - return errors.New("'passageModel' is set, but 'queryModel' is empty") - } - if passageModel == "" && queryModel != "" { - return errors.New("'queryModel' is set, but 'passageModel' is empty") - } - } - return nil -} - -func (cs *classSettings) getPassageModel() string { - model := cs.getProperty("model") - if model == "" { - model = cs.getProperty("passageModel") - } - return model -} - -func (cs *classSettings) getQueryModel() string { - model := cs.getProperty("model") - if model == "" { - model = cs.getProperty("queryModel") - } - return model -} - -func (cs *classSettings) getEndpointURL() string { - endpointURL := cs.getProperty("endpointUrl") - if endpointURL == "" { - endpointURL = cs.getProperty("endpointURL") - } - return endpointURL -} - -func (cs *classSettings) getOption(option string) *bool { - if cs.cfg != nil { - options, ok := cs.cfg.Class()["options"] - if ok { - asMap, ok := options.(map[string]interface{}) - if ok { - option, ok := asMap[option] - if ok { - asBool, ok := option.(bool) - if ok { - return &asBool - } - } - } - } - } - return nil -} - -func (cs *classSettings) getOptionOrDefault(option string, defaultValue bool) bool { - optionValue := cs.getOption(option) - if optionValue != nil { - return *optionValue - } - return defaultValue -} - -func (cs *classSettings) getProperty(name string) string { - if cs.cfg != nil { - model, ok := cs.cfg.Class()[name] - if ok { - asString, ok := model.(string) - if ok { - return asString - } - } - } - return "" -} - -func (cs *classSettings) validateIndexState(class *models.Class, settings ClassSettings) error { - if settings.VectorizeClassName() { - // if the user chooses to vectorize the classname, vector-building will - // always be possible, no need to investigate further - - return nil - } - - // search if there is at least one indexed, string/text prop. If found pass - // validation - for _, prop := range class.Properties { - if len(prop.DataType) < 1 { - return errors.Errorf("property %s must have at least one datatype: "+ - "got %v", prop.Name, prop.DataType) - } - - if prop.DataType[0] != string(schema.DataTypeText) { - // we can only vectorize text-like props - continue - } - - if settings.PropertyIndexed(prop.Name) { - // found at least one, this is a valid schema - return nil - } - } - - return fmt.Errorf("invalid properties: didn't find a single property which is " + - "of type string or text and is not excluded from indexing. In addition the " + - "class name is excluded from vectorization as well, meaning that it cannot be " + - "used to determine the vector position. To fix this, set 'vectorizeClassName' " + - "to true if the class name is contextionary-valid. Alternatively add at least " + - "contextionary-valid text/string property which is not excluded from " + - "indexing.") -} diff --git a/modules/text2vec-huggingface/vectorizer/class_settings_test.go b/modules/text2vec-huggingface/vectorizer/class_settings_test.go deleted file mode 100644 index 4658ef1efd0621d6beec5c5b22b661c49bb1abd1..0000000000000000000000000000000000000000 --- a/modules/text2vec-huggingface/vectorizer/class_settings_test.go +++ /dev/null @@ -1,119 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/entities/moduletools" -) - -func Test_classSettings_getPassageModel(t *testing.T) { - tests := []struct { - name string - cfg moduletools.ClassConfig - wantPassageModel string - wantQueryModel string - wantWaitForModel bool - wantUseGPU bool - wantUseCache bool - wantEndpointURL string - wantError error - }{ - { - name: "CShorten/CORD-19-Title-Abstracts", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "model": "CShorten/CORD-19-Title-Abstracts", - "options": map[string]interface{}{ - "waitForModel": true, - "useGPU": false, - "useCache": false, - }, - }, - }, - wantPassageModel: "CShorten/CORD-19-Title-Abstracts", - wantQueryModel: "CShorten/CORD-19-Title-Abstracts", - wantWaitForModel: true, - wantUseGPU: false, - wantUseCache: false, - }, - { - name: "sentence-transformers/all-MiniLM-L6-v2", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "model": "sentence-transformers/all-MiniLM-L6-v2", - }, - }, - wantPassageModel: "sentence-transformers/all-MiniLM-L6-v2", - wantQueryModel: "sentence-transformers/all-MiniLM-L6-v2", - wantWaitForModel: false, - wantUseGPU: false, - wantUseCache: true, - }, - { - name: "DPR models", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "passageModel": "sentence-transformers/facebook-dpr-ctx_encoder-single-nq-base", - "queryModel": "sentence-transformers/facebook-dpr-question_encoder-single-nq-base", - }, - }, - wantPassageModel: "sentence-transformers/facebook-dpr-ctx_encoder-single-nq-base", - wantQueryModel: "sentence-transformers/facebook-dpr-question_encoder-single-nq-base", - wantWaitForModel: false, - wantUseGPU: false, - wantUseCache: true, - }, - { - name: "Hugging Face Inference API - endpointURL", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "endpointURL": "http://endpoint.cloud", - }, - }, - wantPassageModel: "", - wantQueryModel: "", - wantWaitForModel: false, - wantUseGPU: false, - wantUseCache: true, - wantEndpointURL: "http://endpoint.cloud", - }, - { - name: "Hugging Face Inference API - endpointUrl", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "endpointUrl": "http://endpoint.cloud", - }, - }, - wantPassageModel: "", - wantQueryModel: "", - wantWaitForModel: false, - wantUseGPU: false, - wantUseCache: true, - wantEndpointURL: "http://endpoint.cloud", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ic := NewClassSettings(tt.cfg) - assert.Equal(t, tt.wantPassageModel, ic.getPassageModel()) - assert.Equal(t, tt.wantQueryModel, ic.getQueryModel()) - assert.Equal(t, tt.wantWaitForModel, ic.OptionWaitForModel()) - assert.Equal(t, tt.wantUseGPU, ic.OptionUseGPU()) - assert.Equal(t, tt.wantUseCache, ic.OptionUseCache()) - assert.Equal(t, tt.wantEndpointURL, ic.EndpointURL()) - assert.Equal(t, tt.wantError, ic.validateClassSettings()) - }) - } -} diff --git a/modules/text2vec-huggingface/vectorizer/fakes_for_test.go b/modules/text2vec-huggingface/vectorizer/fakes_for_test.go deleted file mode 100644 index e3a0de4b395bf82d1a84130372aa2a726a018816..0000000000000000000000000000000000000000 --- a/modules/text2vec-huggingface/vectorizer/fakes_for_test.go +++ /dev/null @@ -1,103 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - - "github.com/weaviate/weaviate/modules/text2vec-huggingface/ent" -) - -type fakeClient struct { - lastInput string - lastConfig ent.VectorizationConfig -} - -func (c *fakeClient) Vectorize(ctx context.Context, - text string, cfg ent.VectorizationConfig, -) (*ent.VectorizationResult, error) { - c.lastInput = text - c.lastConfig = cfg - return &ent.VectorizationResult{ - Vector: []float32{0, 1, 2, 3}, - Dimensions: 4, - Text: text, - }, nil -} - -func (c *fakeClient) VectorizeQuery(ctx context.Context, - text string, cfg ent.VectorizationConfig, -) (*ent.VectorizationResult, error) { - c.lastInput = text - c.lastConfig = cfg - return &ent.VectorizationResult{ - Vector: []float32{0.1, 1.1, 2.1, 3.1}, - Dimensions: 4, - Text: text, - }, nil -} - -type fakeClassConfig struct { - classConfig map[string]interface{} - vectorizeClassName bool - vectorizePropertyName bool - skippedProperty string - excludedProperty string - // module specific settings - model, passageModel, queryModel string - waitForModel, useGPU, useCache bool - endpointURL string -} - -func (f fakeClassConfig) Class() map[string]interface{} { - if len(f.classConfig) > 0 { - return f.classConfig - } - classSettings := map[string]interface{}{ - "vectorizeClassName": f.vectorizeClassName, - "model": f.model, - "passageModel": f.passageModel, - "queryModel": f.queryModel, - "waitForModel": f.waitForModel, - "useGPU": f.useGPU, - "useCache": f.useCache, - "endpointURL": f.endpointURL, - } - return classSettings -} - -func (f fakeClassConfig) ClassByModuleName(moduleName string) map[string]interface{} { - return f.classConfig -} - -func (f fakeClassConfig) Property(propName string) map[string]interface{} { - if propName == f.skippedProperty { - return map[string]interface{}{ - "skip": true, - } - } - if propName == f.excludedProperty { - return map[string]interface{}{ - "vectorizePropertyName": false, - } - } - if f.vectorizePropertyName { - return map[string]interface{}{ - "vectorizePropertyName": true, - } - } - return nil -} - -func (f fakeClassConfig) Tenant() string { - return "" -} diff --git a/modules/text2vec-huggingface/vectorizer/objects.go b/modules/text2vec-huggingface/vectorizer/objects.go deleted file mode 100644 index 9637266db6a93ea4f7a5a0e0836277378de3f6a9..0000000000000000000000000000000000000000 --- a/modules/text2vec-huggingface/vectorizer/objects.go +++ /dev/null @@ -1,92 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/modules/text2vec-huggingface/ent" - objectsvectorizer "github.com/weaviate/weaviate/usecases/modulecomponents/vectorizer" -) - -type Vectorizer struct { - client Client - objectVectorizer *objectsvectorizer.ObjectVectorizer -} - -func New(client Client) *Vectorizer { - return &Vectorizer{ - client: client, - objectVectorizer: objectsvectorizer.New(), - } -} - -type Client interface { - Vectorize(ctx context.Context, input string, - config ent.VectorizationConfig) (*ent.VectorizationResult, error) - VectorizeQuery(ctx context.Context, input string, - config ent.VectorizationConfig) (*ent.VectorizationResult, error) -} - -// IndexCheck returns whether a property of a class should be indexed -type ClassSettings interface { - PropertyIndexed(property string) bool - VectorizePropertyName(propertyName string) bool - VectorizeClassName() bool - EndpointURL() string - PassageModel() string - QueryModel() string - OptionWaitForModel() bool - OptionUseGPU() bool - OptionUseCache() bool -} - -func (v *Vectorizer) Object(ctx context.Context, object *models.Object, - objDiff *moduletools.ObjectDiff, cfg moduletools.ClassConfig, -) error { - vec, err := v.object(ctx, object.Class, object.Properties, objDiff, cfg) - if err != nil { - return err - } - - object.Vector = vec - return nil -} - -func (v *Vectorizer) object(ctx context.Context, className string, - schema interface{}, objDiff *moduletools.ObjectDiff, cfg moduletools.ClassConfig, -) ([]float32, error) { - text, vector, err := v.objectVectorizer.TextsOrVector(ctx, className, schema, objDiff, NewClassSettings(cfg)) - if err != nil { - return nil, err - } - if vector != nil { - // dont' re-vectorize - return vector, nil - } - // vectorize text - icheck := NewClassSettings(cfg) - res, err := v.client.Vectorize(ctx, text, ent.VectorizationConfig{ - EndpointURL: icheck.EndpointURL(), - Model: icheck.PassageModel(), - WaitForModel: icheck.OptionWaitForModel(), - UseGPU: icheck.OptionUseGPU(), - UseCache: icheck.OptionUseCache(), - }) - if err != nil { - return nil, err - } - - return res.Vector, nil -} diff --git a/modules/text2vec-huggingface/vectorizer/objects_test.go b/modules/text2vec-huggingface/vectorizer/objects_test.go deleted file mode 100644 index 2ea97f4817656d567207865071ad569e395715ba..0000000000000000000000000000000000000000 --- a/modules/text2vec-huggingface/vectorizer/objects_test.go +++ /dev/null @@ -1,372 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/moduletools" -) - -// These are mostly copy/pasted (with minimal additions) from the -// text2vec-contextionary module -func TestVectorizingObjects(t *testing.T) { - type testCase struct { - name string - input *models.Object - expectedClientCall string - expectedHuggingFaceModel string - noindex string - excludedProperty string // to simulate a schema where property names aren't vectorized - excludedClass string // to simulate a schema where class names aren't vectorized - passageModel string - endpointURL string - } - - tests := []testCase{ - { - name: "empty object", - input: &models.Object{ - Class: "Car", - }, - passageModel: "sentence-transformers/gtr-t5-xl", - expectedHuggingFaceModel: "sentence-transformers/gtr-t5-xl", - expectedClientCall: "car", - }, - { - name: "object with one string prop", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "Mercedes", - }, - }, - expectedClientCall: "car brand mercedes", - }, - { - name: "object with one non-string prop", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "power": 300, - }, - }, - expectedClientCall: "car", - }, - { - name: "object with a mix of props", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: "car brand best brand review a very great car", - }, - { - name: "with a noindexed property", - noindex: "review", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: "car brand best brand", - }, - { - name: "with the class name not vectorized", - excludedClass: "Car", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: "brand best brand review a very great car", - }, - { - name: "with a property name not vectorized", - excludedProperty: "review", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: "car brand best brand a very great car", - }, - { - name: "with no schema labels vectorized", - excludedProperty: "review", - excludedClass: "Car", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "review": "a very great car", - }, - }, - expectedClientCall: "a very great car", - }, - { - name: "with string/text arrays without propname or classname", - excludedProperty: "reviews", - excludedClass: "Car", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - expectedClientCall: "a very great car you should consider buying one", - }, - { - name: "with string/text arrays with propname and classname", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - expectedClientCall: "car reviews a very great car reviews you should consider buying one", - }, - { - name: "with compound class and prop names", - input: &models.Object{ - Class: "SuperCar", - Properties: map[string]interface{}{ - "brandOfTheCar": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: "super car brand of the car best brand review a very great car", - }, - { - name: "empty object with HF Inference Endpoint", - input: &models.Object{ - Class: "Car", - }, - endpointURL: "https://url.cloud", - expectedClientCall: "car", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - client := &fakeClient{} - - v := New(client) - - ic := &fakeClassConfig{ - excludedProperty: test.excludedProperty, - skippedProperty: test.noindex, - vectorizeClassName: test.excludedClass != "Car", - passageModel: test.passageModel, - endpointURL: test.endpointURL, - vectorizePropertyName: true, - } - err := v.Object(context.Background(), test.input, nil, ic) - - require.Nil(t, err) - assert.Equal(t, models.C11yVector{0, 1, 2, 3}, test.input.Vector) - expected := strings.Split(test.expectedClientCall, " ") - actual := strings.Split(client.lastInput, " ") - assert.Equal(t, expected, actual) - if test.expectedHuggingFaceModel != "" { - assert.Equal(t, test.expectedHuggingFaceModel, client.lastConfig.Model) - } - }) - } -} - -func TestVectorizingObjectsWithDiff(t *testing.T) { - type testCase struct { - name string - input *models.Object - skipped string - diff *moduletools.ObjectDiff - expectedVectorize bool - } - - tests := []testCase{ - { - name: "no diff", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: nil, - expectedVectorize: true, - }, - { - name: "diff all props unchanged", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("brand", "best brand", "best brand"). - WithProp("power", 300, 300). - WithProp("description", "a very great car", "a very great car"). - WithProp("reviews", []interface{}{ - "a very great car", - "you should consider buying one", - }, []interface{}{ - "a very great car", - "you should consider buying one", - }), - expectedVectorize: false, - }, - { - name: "diff one vectorizable prop changed (1)", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("brand", "old best brand", "best brand"), - expectedVectorize: true, - }, - { - name: "diff one vectorizable prop changed (2)", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("description", "old a very great car", "a very great car"), - expectedVectorize: true, - }, - { - name: "diff one vectorizable prop changed (3)", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("reviews", []interface{}{ - "old a very great car", - "you should consider buying one", - }, []interface{}{ - "a very great car", - "you should consider buying one", - }), - expectedVectorize: true, - }, - { - name: "all non-vectorizable props changed", - skipped: "description", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("power", 123, 300). - WithProp("description", "old a very great car", "a very great car"), - expectedVectorize: false, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - ic := &fakeClassConfig{ - skippedProperty: test.skipped, - } - - client := &fakeClient{} - v := New(client) - - err := v.Object(context.Background(), test.input, test.diff, ic) - - require.Nil(t, err) - if test.expectedVectorize { - assert.Equal(t, models.C11yVector{0, 1, 2, 3}, test.input.Vector) - assert.NotEmpty(t, client.lastInput) - } else { - assert.Equal(t, models.C11yVector{0, 0, 0, 0}, test.input.Vector) - assert.Empty(t, client.lastInput) - } - }) - } -} - -func newObjectDiffWithVector() *moduletools.ObjectDiff { - return moduletools.NewObjectDiff([]float32{0, 0, 0, 0}) -} diff --git a/modules/text2vec-huggingface/vectorizer/texts.go b/modules/text2vec-huggingface/vectorizer/texts.go deleted file mode 100644 index dd5d3d30f8609211d5fb319f7f55565e6c9b1fb9..0000000000000000000000000000000000000000 --- a/modules/text2vec-huggingface/vectorizer/texts.go +++ /dev/null @@ -1,54 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/modules/text2vec-contextionary/vectorizer" - "github.com/weaviate/weaviate/modules/text2vec-huggingface/ent" - libvectorizer "github.com/weaviate/weaviate/usecases/vectorizer" -) - -func (v *Vectorizer) VectorizeInput(ctx context.Context, input string, - icheck vectorizer.ClassIndexCheck, -) ([]float32, error) { - res, err := v.client.VectorizeQuery(ctx, input, ent.VectorizationConfig{}) - if err != nil { - return nil, err - } - return res.Vector, nil -} - -func (v *Vectorizer) Texts(ctx context.Context, inputs []string, - cfg moduletools.ClassConfig, -) ([]float32, error) { - settings := NewClassSettings(cfg) - vectors := make([][]float32, len(inputs)) - for i := range inputs { - res, err := v.client.VectorizeQuery(ctx, inputs[i], ent.VectorizationConfig{ - EndpointURL: settings.EndpointURL(), - Model: settings.QueryModel(), - WaitForModel: settings.OptionWaitForModel(), - UseGPU: settings.OptionUseGPU(), - UseCache: settings.OptionUseCache(), - }) - if err != nil { - return nil, errors.Wrap(err, "remote client vectorize") - } - vectors[i] = res.Vector - } - - return libvectorizer.CombineVectors(vectors), nil -} diff --git a/modules/text2vec-huggingface/vectorizer/texts_test.go b/modules/text2vec-huggingface/vectorizer/texts_test.go deleted file mode 100644 index 4a20b507a090378faacad2695dbff5560cbdddd3..0000000000000000000000000000000000000000 --- a/modules/text2vec-huggingface/vectorizer/texts_test.go +++ /dev/null @@ -1,100 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// as used in the nearText searcher -func TestVectorizingTexts(t *testing.T) { - type testCase struct { - name string - input []string - expectedHuggingFaceModel string - huggingFaceModel string - huggingFaceEndpointURL string - } - - tests := []testCase{ - { - name: "single word", - input: []string{"hello"}, - huggingFaceModel: "sentence-transformers/gtr-t5-xl", - expectedHuggingFaceModel: "sentence-transformers/gtr-t5-xl", - }, - { - name: "multiple words", - input: []string{"hello world, this is me!"}, - huggingFaceModel: "sentence-transformers/gtr-t5-xl", - expectedHuggingFaceModel: "sentence-transformers/gtr-t5-xl", - }, - { - name: "multiple sentences (joined with a dot)", - input: []string{"this is sentence 1", "and here's number 2"}, - huggingFaceModel: "sentence-transformers/gtr-t5-xl", - expectedHuggingFaceModel: "sentence-transformers/gtr-t5-xl", - }, - { - name: "multiple sentences already containing a dot", - input: []string{"this is sentence 1.", "and here's number 2"}, - huggingFaceModel: "sentence-transformers/gtr-t5-xl", - expectedHuggingFaceModel: "sentence-transformers/gtr-t5-xl", - }, - { - name: "multiple sentences already containing a question mark", - input: []string{"this is sentence 1?", "and here's number 2"}, - huggingFaceModel: "sentence-transformers/gtr-t5-xl", - expectedHuggingFaceModel: "sentence-transformers/gtr-t5-xl", - }, - { - name: "multiple sentences already containing an exclamation mark", - input: []string{"this is sentence 1!", "and here's number 2"}, - huggingFaceModel: "sentence-transformers/gtr-t5-xl", - expectedHuggingFaceModel: "sentence-transformers/gtr-t5-xl", - }, - { - name: "multiple sentences already containing comma", - input: []string{"this is sentence 1,", "and here's number 2"}, - huggingFaceModel: "sentence-transformers/gtr-t5-xl", - expectedHuggingFaceModel: "sentence-transformers/gtr-t5-xl", - }, - { - name: "single word with inference url", - input: []string{"hello"}, - huggingFaceEndpointURL: "http://url.cloud", - expectedHuggingFaceModel: "sentence-transformers/msmarco-bert-base-dot-v5", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - client := &fakeClient{} - - v := New(client) - - settings := &fakeClassConfig{ - model: test.huggingFaceModel, - endpointURL: test.huggingFaceEndpointURL, - } - vec, err := v.Texts(context.Background(), test.input, settings) - - require.Nil(t, err) - assert.Equal(t, []float32{0.1, 1.1, 2.1, 3.1}, vec) - assert.Equal(t, client.lastConfig.Model, test.expectedHuggingFaceModel) - }) - } -} diff --git a/modules/text2vec-jinaai/clients/jinaai.go b/modules/text2vec-jinaai/clients/jinaai.go deleted file mode 100644 index b7ac85baaf996aa51febefaea797c72d8e06774e..0000000000000000000000000000000000000000 --- a/modules/text2vec-jinaai/clients/jinaai.go +++ /dev/null @@ -1,195 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "time" - - "github.com/weaviate/weaviate/usecases/modulecomponents" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/modules/text2vec-jinaai/ent" -) - -type embeddingsRequest struct { - Input []string `json:"input"` - Model string `json:"model,omitempty"` -} - -type embedding struct { - Object string `json:"object"` - Data []embeddingData `json:"data,omitempty"` - Error *jinaAIApiError `json:"error,omitempty"` -} - -type embeddingData struct { - Object string `json:"object"` - Index int `json:"index"` - Embedding []float32 `json:"embedding"` -} - -type jinaAIApiError struct { - Message string `json:"message"` - Type string `json:"type"` - Param string `json:"param"` - Code string `json:"code"` -} - -func buildUrl(config ent.VectorizationConfig) (string, error) { - host := config.BaseURL - path := "/v1/embeddings" - return url.JoinPath(host, path) -} - -type vectorizer struct { - jinaAIApiKey string - httpClient *http.Client - buildUrlFn func(config ent.VectorizationConfig) (string, error) - logger logrus.FieldLogger -} - -func New(jinaAIApiKey string, timeout time.Duration, logger logrus.FieldLogger) *vectorizer { - return &vectorizer{ - jinaAIApiKey: jinaAIApiKey, - httpClient: &http.Client{ - Timeout: timeout, - }, - buildUrlFn: buildUrl, - logger: logger, - } -} - -func (v *vectorizer) Vectorize(ctx context.Context, input string, - config ent.VectorizationConfig, -) (*ent.VectorizationResult, error) { - return v.vectorize(ctx, []string{input}, config.Model, config) -} - -func (v *vectorizer) VectorizeQuery(ctx context.Context, input []string, - config ent.VectorizationConfig, -) (*ent.VectorizationResult, error) { - return v.vectorize(ctx, input, config.Model, config) -} - -func (v *vectorizer) vectorize(ctx context.Context, input []string, model string, config ent.VectorizationConfig) (*ent.VectorizationResult, error) { - body, err := json.Marshal(v.getEmbeddingsRequest(input, model)) - if err != nil { - return nil, errors.Wrap(err, "marshal body") - } - - endpoint, err := v.buildUrlFn(config) - if err != nil { - return nil, errors.Wrap(err, "join jinaAI API host and path") - } - - req, err := http.NewRequestWithContext(ctx, "POST", endpoint, - bytes.NewReader(body)) - if err != nil { - return nil, errors.Wrap(err, "create POST request") - } - apiKey, err := v.getApiKey(ctx) - if err != nil { - return nil, errors.Wrap(err, "API Key") - } - req.Header.Add(v.getApiKeyHeaderAndValue(apiKey)) - req.Header.Add("Content-Type", "application/json") - - res, err := v.httpClient.Do(req) - if err != nil { - return nil, errors.Wrap(err, "send POST request") - } - defer res.Body.Close() - - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - return nil, errors.Wrap(err, "read response body") - } - - var resBody embedding - if err := json.Unmarshal(bodyBytes, &resBody); err != nil { - return nil, errors.Wrap(err, "unmarshal response body") - } - - if res.StatusCode != 200 || resBody.Error != nil { - return nil, v.getError(res.StatusCode, resBody.Error) - } - - texts := make([]string, len(resBody.Data)) - embeddings := make([][]float32, len(resBody.Data)) - for i := range resBody.Data { - texts[i] = resBody.Data[i].Object - embeddings[i] = resBody.Data[i].Embedding - } - - return &ent.VectorizationResult{ - Text: texts, - Dimensions: len(resBody.Data[0].Embedding), - Vector: embeddings, - }, nil -} - -func (v *vectorizer) getError(statusCode int, resBodyError *jinaAIApiError) error { - endpoint := "JinaAI API" - if resBodyError != nil { - return fmt.Errorf("connection to: %s failed with status: %d error: %v", endpoint, statusCode, resBodyError.Message) - } - return fmt.Errorf("connection to: %s failed with status: %d", endpoint, statusCode) -} - -func (v *vectorizer) getEmbeddingsRequest(input []string, model string) embeddingsRequest { - return embeddingsRequest{Input: input, Model: model} -} - -func (v *vectorizer) getApiKeyHeaderAndValue(apiKey string) (string, string) { - return "Authorization", fmt.Sprintf("Bearer %s", apiKey) -} - -func (v *vectorizer) getApiKey(ctx context.Context) (string, error) { - var apiKey, envVar string - - apiKey = "X-Jinaai-Api-Key" - envVar = "JINAAI_APIKEY" - if len(v.jinaAIApiKey) > 0 { - return v.jinaAIApiKey, nil - } - - return v.getApiKeyFromContext(ctx, apiKey, envVar) -} - -func (v *vectorizer) getApiKeyFromContext(ctx context.Context, apiKey, envVar string) (string, error) { - if apiKeyValue := v.getValueFromContext(ctx, apiKey); apiKeyValue != "" { - return apiKeyValue, nil - } - return "", fmt.Errorf("no api key found neither in request header: %s nor in environment variable under %s", apiKey, envVar) -} - -func (v *vectorizer) getValueFromContext(ctx context.Context, key string) string { - if value := ctx.Value(key); value != nil { - if keyHeader, ok := value.([]string); ok && len(keyHeader) > 0 && len(keyHeader[0]) > 0 { - return keyHeader[0] - } - } - // try getting header from GRPC if not successful - if apiKey := modulecomponents.GetValueFromGRPC(ctx, key); len(apiKey) > 0 && len(apiKey[0]) > 0 { - return apiKey[0] - } - - return "" -} diff --git a/modules/text2vec-jinaai/clients/jinaai_test.go b/modules/text2vec-jinaai/clients/jinaai_test.go deleted file mode 100644 index d2a5544f07ebb1721ca764c8de295bb4c9a8ca0c..0000000000000000000000000000000000000000 --- a/modules/text2vec-jinaai/clients/jinaai_test.go +++ /dev/null @@ -1,233 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "context" - "encoding/json" - "io" - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/modules/text2vec-jinaai/ent" -) - -func TestBuildUrlFn(t *testing.T) { - t.Run("buildUrlFn returns default Jina AI URL", func(t *testing.T) { - config := ent.VectorizationConfig{ - Model: "", - BaseURL: "https://api.jina.ai", - } - url, err := buildUrl(config) - assert.Nil(t, err) - assert.Equal(t, "https://api.jina.ai/v1/embeddings", url) - }) - - t.Run("buildUrlFn loads from BaseURL", func(t *testing.T) { - config := ent.VectorizationConfig{ - Model: "", - BaseURL: "https://foobar.some.proxy", - } - url, err := buildUrl(config) - assert.Nil(t, err) - assert.Equal(t, "https://foobar.some.proxy/v1/embeddings", url) - }) -} - -func TestClient(t *testing.T) { - t.Run("when all is fine", func(t *testing.T) { - server := httptest.NewServer(&fakeHandler{t: t}) - defer server.Close() - - c := New("apiKey", 0, nullLogger()) - c.buildUrlFn = func(config ent.VectorizationConfig) (string, error) { - return server.URL, nil - } - - expected := &ent.VectorizationResult{ - Text: []string{"This is my text"}, - Vector: [][]float32{{0.1, 0.2, 0.3}}, - Dimensions: 3, - } - res, err := c.Vectorize(context.Background(), "This is my text", - ent.VectorizationConfig{ - Model: "jina-embedding-v2", - }) - - assert.Nil(t, err) - assert.Equal(t, expected, res) - }) - - t.Run("when the context is expired", func(t *testing.T) { - server := httptest.NewServer(&fakeHandler{t: t}) - defer server.Close() - c := New("apiKey", 0, nullLogger()) - c.buildUrlFn = func(config ent.VectorizationConfig) (string, error) { - return server.URL, nil - } - - ctx, cancel := context.WithDeadline(context.Background(), time.Now()) - defer cancel() - - _, err := c.Vectorize(ctx, "This is my text", ent.VectorizationConfig{}) - - require.NotNil(t, err) - assert.Contains(t, err.Error(), "context deadline exceeded") - }) - - t.Run("when the server returns an error", func(t *testing.T) { - server := httptest.NewServer(&fakeHandler{ - t: t, - serverError: errors.Errorf("nope, not gonna happen"), - }) - defer server.Close() - c := New("apiKey", 0, nullLogger()) - c.buildUrlFn = func(config ent.VectorizationConfig) (string, error) { - return server.URL, nil - } - - _, err := c.Vectorize(context.Background(), "This is my text", - ent.VectorizationConfig{}) - - require.NotNil(t, err) - assert.EqualError(t, err, "connection to: JinaAI API failed with status: 500 error: nope, not gonna happen") - }) - - t.Run("when JinaAI key is passed using X-Jinaai-Api-Key header", func(t *testing.T) { - server := httptest.NewServer(&fakeHandler{t: t}) - defer server.Close() - c := New("", 0, nullLogger()) - c.buildUrlFn = func(config ent.VectorizationConfig) (string, error) { - return server.URL, nil - } - - ctxWithValue := context.WithValue(context.Background(), - "X-Jinaai-Api-Key", []string{"some-key"}) - - expected := &ent.VectorizationResult{ - Text: []string{"This is my text"}, - Vector: [][]float32{{0.1, 0.2, 0.3}}, - Dimensions: 3, - } - res, err := c.Vectorize(ctxWithValue, "This is my text", - ent.VectorizationConfig{ - Model: "jina-emvedding-v2", - }) - - require.Nil(t, err) - assert.Equal(t, expected, res) - }) - - t.Run("when JinaAI key is empty", func(t *testing.T) { - server := httptest.NewServer(&fakeHandler{t: t}) - defer server.Close() - c := New("", 0, nullLogger()) - c.buildUrlFn = func(config ent.VectorizationConfig) (string, error) { - return server.URL, nil - } - - ctx, cancel := context.WithDeadline(context.Background(), time.Now()) - defer cancel() - - _, err := c.Vectorize(ctx, "This is my text", ent.VectorizationConfig{}) - - require.NotNil(t, err) - assert.EqualError(t, err, "API Key: no api key found "+ - "neither in request header: X-Jinaai-Api-Key "+ - "nor in environment variable under JINAAI_APIKEY") - }) - - t.Run("when X-Jinaai-Api-Key header is passed but empty", func(t *testing.T) { - server := httptest.NewServer(&fakeHandler{t: t}) - defer server.Close() - c := New("", 0, nullLogger()) - c.buildUrlFn = func(config ent.VectorizationConfig) (string, error) { - return server.URL, nil - } - - ctxWithValue := context.WithValue(context.Background(), - "X-Jinaai-Api-Key", []string{""}) - - _, err := c.Vectorize(ctxWithValue, "This is my text", - ent.VectorizationConfig{ - Model: "jina-embeddings-v2", - }) - - require.NotNil(t, err) - assert.EqualError(t, err, "API Key: no api key found "+ - "neither in request header: X-Jinaai-Api-Key "+ - "nor in environment variable under JINAAI_APIKEY") - }) -} - -type fakeHandler struct { - t *testing.T - serverError error -} - -func (f *fakeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - assert.Equal(f.t, http.MethodPost, r.Method) - - if f.serverError != nil { - embeddingError := map[string]interface{}{ - "message": f.serverError.Error(), - "type": "invalid_request_error", - } - embedding := map[string]interface{}{ - "error": embeddingError, - } - outBytes, err := json.Marshal(embedding) - require.Nil(f.t, err) - - w.WriteHeader(http.StatusInternalServerError) - w.Write(outBytes) - return - } - - bodyBytes, err := io.ReadAll(r.Body) - require.Nil(f.t, err) - defer r.Body.Close() - - var b map[string]interface{} - require.Nil(f.t, json.Unmarshal(bodyBytes, &b)) - - textInputArray := b["input"].([]interface{}) - textInput := textInputArray[0].(string) - assert.Greater(f.t, len(textInput), 0) - - embeddingData := map[string]interface{}{ - "object": textInput, - "index": 0, - "embedding": []float32{0.1, 0.2, 0.3}, - } - embedding := map[string]interface{}{ - "object": "list", - "data": []interface{}{embeddingData}, - } - - outBytes, err := json.Marshal(embedding) - require.Nil(f.t, err) - - w.Write(outBytes) -} - -func nullLogger() logrus.FieldLogger { - l, _ := test.NewNullLogger() - return l -} diff --git a/modules/text2vec-jinaai/clients/meta.go b/modules/text2vec-jinaai/clients/meta.go deleted file mode 100644 index f7777e0c85b28139f09123d64477cf3970a84d90..0000000000000000000000000000000000000000 --- a/modules/text2vec-jinaai/clients/meta.go +++ /dev/null @@ -1,19 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -func (v *vectorizer) MetaInfo() (map[string]interface{}, error) { - return map[string]interface{}{ - "name": "JinaAI Module", - "documentationHref": "https://jina.ai/embeddings/", - }, nil -} diff --git a/modules/text2vec-jinaai/config.go b/modules/text2vec-jinaai/config.go deleted file mode 100644 index 3c8566e60f963041cfad4a016695ab97b0f43765..0000000000000000000000000000000000000000 --- a/modules/text2vec-jinaai/config.go +++ /dev/null @@ -1,47 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modjinaai - -import ( - "context" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/modules/text2vec-jinaai/vectorizer" -) - -func (m *JinaAIModule) ClassConfigDefaults() map[string]interface{} { - return map[string]interface{}{ - "vectorizeClassName": vectorizer.DefaultVectorizeClassName, - "model": vectorizer.DefaultJinaAIModel, - } -} - -func (m *JinaAIModule) PropertyConfigDefaults( - dt *schema.DataType, -) map[string]interface{} { - return map[string]interface{}{ - "skip": !vectorizer.DefaultPropertyIndexed, - "vectorizePropertyName": vectorizer.DefaultVectorizePropertyName, - } -} - -func (m *JinaAIModule) ValidateClass(ctx context.Context, - class *models.Class, cfg moduletools.ClassConfig, -) error { - settings := vectorizer.NewClassSettings(cfg) - return settings.Validate(class) -} - -var _ = modulecapabilities.ClassConfigurator(New()) diff --git a/modules/text2vec-jinaai/ent/vectorization_config.go b/modules/text2vec-jinaai/ent/vectorization_config.go deleted file mode 100644 index ec9d0b610a21e0c3fdf346eaa44a7a88847e5626..0000000000000000000000000000000000000000 --- a/modules/text2vec-jinaai/ent/vectorization_config.go +++ /dev/null @@ -1,17 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package ent - -type VectorizationConfig struct { - Model string - BaseURL string -} diff --git a/modules/text2vec-jinaai/ent/vectorization_result.go b/modules/text2vec-jinaai/ent/vectorization_result.go deleted file mode 100644 index 9120fc2f73f2900c7f547973778d28dcb4b6c1e1..0000000000000000000000000000000000000000 --- a/modules/text2vec-jinaai/ent/vectorization_result.go +++ /dev/null @@ -1,18 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package ent - -type VectorizationResult struct { - Text []string - Dimensions int - Vector [][]float32 -} diff --git a/modules/text2vec-jinaai/module.go b/modules/text2vec-jinaai/module.go deleted file mode 100644 index f30b6f8a25f0390584bf7194f3cf7983a33155fb..0000000000000000000000000000000000000000 --- a/modules/text2vec-jinaai/module.go +++ /dev/null @@ -1,149 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modjinaai - -import ( - "context" - "net/http" - "os" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/modules/text2vec-jinaai/clients" - "github.com/weaviate/weaviate/modules/text2vec-jinaai/vectorizer" - "github.com/weaviate/weaviate/usecases/modulecomponents/additional" -) - -const Name = "text2vec-jinaai" - -func New() *JinaAIModule { - return &JinaAIModule{} -} - -type JinaAIModule struct { - vectorizer textVectorizer - metaProvider metaProvider - graphqlProvider modulecapabilities.GraphQLArguments - searcher modulecapabilities.Searcher - nearTextTransformer modulecapabilities.TextTransform - logger logrus.FieldLogger - additionalPropertiesProvider modulecapabilities.AdditionalProperties -} - -type textVectorizer interface { - Object(ctx context.Context, obj *models.Object, objDiff *moduletools.ObjectDiff, - cfg moduletools.ClassConfig) error - Texts(ctx context.Context, input []string, - cfg moduletools.ClassConfig) ([]float32, error) -} - -type metaProvider interface { - MetaInfo() (map[string]interface{}, error) -} - -func (m *JinaAIModule) Name() string { - return Name -} - -func (m *JinaAIModule) Type() modulecapabilities.ModuleType { - return modulecapabilities.Text2MultiVec -} - -func (m *JinaAIModule) Init(ctx context.Context, - params moduletools.ModuleInitParams, -) error { - m.logger = params.GetLogger() - - if err := m.initVectorizer(ctx, params.GetConfig().ModuleHttpClientTimeout, m.logger); err != nil { - return errors.Wrap(err, "init vectorizer") - } - - if err := m.initAdditionalPropertiesProvider(); err != nil { - return errors.Wrap(err, "init additional properties provider") - } - - return nil -} - -func (m *JinaAIModule) InitExtension(modules []modulecapabilities.Module) error { - for _, module := range modules { - if module.Name() == m.Name() { - continue - } - if arg, ok := module.(modulecapabilities.TextTransformers); ok { - if arg != nil && arg.TextTransformers() != nil { - m.nearTextTransformer = arg.TextTransformers()["nearText"] - } - } - } - - if err := m.initNearText(); err != nil { - return errors.Wrap(err, "init graphql provider") - } - return nil -} - -func (m *JinaAIModule) initVectorizer(ctx context.Context, timeout time.Duration, - logger logrus.FieldLogger, -) error { - jinaAIApiKey := os.Getenv("JINAAI_APIKEY") - - client := clients.New(jinaAIApiKey, timeout, logger) - - m.vectorizer = vectorizer.New(client) - m.metaProvider = client - - return nil -} - -func (m *JinaAIModule) initAdditionalPropertiesProvider() error { - m.additionalPropertiesProvider = additional.NewText2VecProvider() - return nil -} - -func (m *JinaAIModule) RootHandler() http.Handler { - // TODO: remove once this is a capability interface - return nil -} - -func (m *JinaAIModule) VectorizeObject(ctx context.Context, - obj *models.Object, objDiff *moduletools.ObjectDiff, cfg moduletools.ClassConfig, -) error { - return m.vectorizer.Object(ctx, obj, objDiff, cfg) -} - -func (m *JinaAIModule) MetaInfo() (map[string]interface{}, error) { - return m.metaProvider.MetaInfo() -} - -func (m *JinaAIModule) AdditionalProperties() map[string]modulecapabilities.AdditionalProperty { - return m.additionalPropertiesProvider.AdditionalProperties() -} - -func (m *JinaAIModule) VectorizeInput(ctx context.Context, - input string, cfg moduletools.ClassConfig, -) ([]float32, error) { - return m.vectorizer.Texts(ctx, []string{input}, cfg) -} - -// verify we implement the modules.Module interface -var ( - _ = modulecapabilities.Module(New()) - _ = modulecapabilities.Vectorizer(New()) - _ = modulecapabilities.MetaProvider(New()) - _ = modulecapabilities.Searcher(New()) - _ = modulecapabilities.GraphQLArguments(New()) -) diff --git a/modules/text2vec-jinaai/nearText.go b/modules/text2vec-jinaai/nearText.go deleted file mode 100644 index 979ad6832a11bbc1e98d5636c4e49fc75ae7f2e3..0000000000000000000000000000000000000000 --- a/modules/text2vec-jinaai/nearText.go +++ /dev/null @@ -1,36 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modjinaai - -import ( - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/usecases/modulecomponents/nearText" -) - -func (m *JinaAIModule) initNearText() error { - m.searcher = nearText.NewSearcher(m.vectorizer) - m.graphqlProvider = nearText.New(m.nearTextTransformer) - return nil -} - -func (m *JinaAIModule) Arguments() map[string]modulecapabilities.GraphQLArgument { - return m.graphqlProvider.Arguments() -} - -func (m *JinaAIModule) VectorSearches() map[string]modulecapabilities.VectorForParams { - return m.searcher.VectorSearches() -} - -var ( - _ = modulecapabilities.GraphQLArguments(New()) - _ = modulecapabilities.Searcher(New()) -) diff --git a/modules/text2vec-jinaai/vectorizer/class_settings.go b/modules/text2vec-jinaai/vectorizer/class_settings.go deleted file mode 100644 index f0c5e557175413bebd394c78ea0c6543b2c43d77..0000000000000000000000000000000000000000 --- a/modules/text2vec-jinaai/vectorizer/class_settings.go +++ /dev/null @@ -1,171 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "fmt" - "strings" - - "github.com/pkg/errors" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/schema" -) - -const ( - DefaultJinaAIDocumentType = "text" - DefaultJinaAIModel = "jina-embeddings-v2-base-en" - DefaultVectorizeClassName = true - DefaultPropertyIndexed = true - DefaultVectorizePropertyName = false - DefaultBaseURL = "https://api.jina.ai" -) - -type classSettings struct { - cfg moduletools.ClassConfig -} - -func NewClassSettings(cfg moduletools.ClassConfig) *classSettings { - return &classSettings{cfg: cfg} -} - -func (cs *classSettings) PropertyIndexed(propName string) bool { - if cs.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return DefaultPropertyIndexed - } - - vcn, ok := cs.cfg.Property(propName)["skip"] - if !ok { - return DefaultPropertyIndexed - } - - asBool, ok := vcn.(bool) - if !ok { - return DefaultPropertyIndexed - } - - return !asBool -} - -func (cs *classSettings) VectorizePropertyName(propName string) bool { - if cs.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return DefaultVectorizePropertyName - } - vcn, ok := cs.cfg.Property(propName)["vectorizePropertyName"] - if !ok { - return DefaultVectorizePropertyName - } - - asBool, ok := vcn.(bool) - if !ok { - return DefaultVectorizePropertyName - } - - return asBool -} - -func (cs *classSettings) Model() string { - return cs.getProperty("model", DefaultJinaAIModel) -} - -func (cs *classSettings) BaseURL() string { - return cs.getProperty("baseURL", DefaultBaseURL) -} - -func (cs *classSettings) VectorizeClassName() bool { - if cs.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return DefaultVectorizeClassName - } - - vcn, ok := cs.cfg.Class()["vectorizeClassName"] - if !ok { - return DefaultVectorizeClassName - } - - asBool, ok := vcn.(bool) - if !ok { - return DefaultVectorizeClassName - } - - return asBool -} - -func (cs *classSettings) Validate(class *models.Class) error { - if cs.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return errors.New("empty config") - } - - err := cs.validateIndexState(class, cs) - if err != nil { - return err - } - - return nil -} - -func (cs *classSettings) getProperty(name, defaultValue string) string { - if cs.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return defaultValue - } - - model, ok := cs.cfg.Class()[name] - if ok { - asString, ok := model.(string) - if ok { - return strings.ToLower(asString) - } - } - - return defaultValue -} - -func (cs *classSettings) validateIndexState(class *models.Class, settings ClassSettings) error { - if settings.VectorizeClassName() { - // if the user chooses to vectorize the classname, vector-building will - // always be possible, no need to investigate further - - return nil - } - - // search if there is at least one indexed, string/text prop. If found pass - // validation - for _, prop := range class.Properties { - if len(prop.DataType) < 1 { - return errors.Errorf("property %s must have at least one datatype: "+ - "got %v", prop.Name, prop.DataType) - } - - if prop.DataType[0] != string(schema.DataTypeText) { - // we can only vectorize text-like props - continue - } - - if settings.PropertyIndexed(prop.Name) { - // found at least one, this is a valid schema - return nil - } - } - - return fmt.Errorf("invalid properties: didn't find a single property which is " + - "of type string or text and is not excluded from indexing. In addition the " + - "class name is excluded from vectorization as well, meaning that it cannot be " + - "used to determine the vector position. To fix this, set 'vectorizeClassName' " + - "to true if the class name is contextionary-valid. Alternatively add at least " + - "contextionary-valid text/string property which is not excluded from " + - "indexing.") -} diff --git a/modules/text2vec-jinaai/vectorizer/fakes_for_test.go b/modules/text2vec-jinaai/vectorizer/fakes_for_test.go deleted file mode 100644 index 4d69650b96dd7b815d0add312349e06a8d293823..0000000000000000000000000000000000000000 --- a/modules/text2vec-jinaai/vectorizer/fakes_for_test.go +++ /dev/null @@ -1,94 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - - "github.com/weaviate/weaviate/modules/text2vec-jinaai/ent" -) - -type fakeClient struct { - lastInput []string - lastConfig ent.VectorizationConfig -} - -func (c *fakeClient) Vectorize(ctx context.Context, - text string, cfg ent.VectorizationConfig, -) (*ent.VectorizationResult, error) { - c.lastInput = []string{text} - c.lastConfig = cfg - return &ent.VectorizationResult{ - Vector: [][]float32{{0, 1, 2, 3}}, - Dimensions: 4, - Text: []string{text}, - }, nil -} - -func (c *fakeClient) VectorizeQuery(ctx context.Context, - text []string, cfg ent.VectorizationConfig, -) (*ent.VectorizationResult, error) { - c.lastInput = text - c.lastConfig = cfg - return &ent.VectorizationResult{ - Vector: [][]float32{{0.1, 1.1, 2.1, 3.1}}, - Dimensions: 4, - Text: text, - }, nil -} - -type fakeClassConfig struct { - classConfig map[string]interface{} - vectorizeClassName bool - vectorizePropertyName bool - skippedProperty string - excludedProperty string - jinaAIModel string -} - -func (f fakeClassConfig) Class() map[string]interface{} { - if len(f.classConfig) > 0 { - return f.classConfig - } - classSettings := map[string]interface{}{ - "vectorizeClassName": f.vectorizeClassName, - "model": f.jinaAIModel, - } - return classSettings -} - -func (f fakeClassConfig) ClassByModuleName(moduleName string) map[string]interface{} { - return f.classConfig -} - -func (f fakeClassConfig) Property(propName string) map[string]interface{} { - if propName == f.skippedProperty { - return map[string]interface{}{ - "skip": true, - } - } - if propName == f.excludedProperty { - return map[string]interface{}{ - "vectorizePropertyName": false, - } - } - if f.vectorizePropertyName { - return map[string]interface{}{ - "vectorizePropertyName": true, - } - } - return nil -} - -func (f fakeClassConfig) Tenant() string { - return "" -} diff --git a/modules/text2vec-jinaai/vectorizer/objects.go b/modules/text2vec-jinaai/vectorizer/objects.go deleted file mode 100644 index 6aa72f5ae43ad072b081abf3f971e1e7e612e744..0000000000000000000000000000000000000000 --- a/modules/text2vec-jinaai/vectorizer/objects.go +++ /dev/null @@ -1,89 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/modules/text2vec-jinaai/ent" - objectsvectorizer "github.com/weaviate/weaviate/usecases/modulecomponents/vectorizer" - libvectorizer "github.com/weaviate/weaviate/usecases/vectorizer" -) - -type Vectorizer struct { - client Client - objectVectorizer *objectsvectorizer.ObjectVectorizer -} - -func New(client Client) *Vectorizer { - return &Vectorizer{ - client: client, - objectVectorizer: objectsvectorizer.New(), - } -} - -type Client interface { - Vectorize(ctx context.Context, input string, - config ent.VectorizationConfig) (*ent.VectorizationResult, error) - VectorizeQuery(ctx context.Context, input []string, - config ent.VectorizationConfig) (*ent.VectorizationResult, error) -} - -// IndexCheck returns whether a property of a class should be indexed -type ClassSettings interface { - PropertyIndexed(property string) bool - VectorizePropertyName(propertyName string) bool - VectorizeClassName() bool - Model() string - BaseURL() string -} - -func (v *Vectorizer) Object(ctx context.Context, object *models.Object, - objDiff *moduletools.ObjectDiff, cfg moduletools.ClassConfig, -) error { - vec, err := v.object(ctx, object.Class, object.Properties, objDiff, cfg) - if err != nil { - return err - } - - object.Vector = vec - return nil -} - -func (v *Vectorizer) object(ctx context.Context, className string, - schema interface{}, objDiff *moduletools.ObjectDiff, cfg moduletools.ClassConfig, -) ([]float32, error) { - text, vector, err := v.objectVectorizer.TextsOrVector(ctx, className, schema, objDiff, NewClassSettings(cfg)) - if err != nil { - return nil, err - } - if vector != nil { - // dont' re-vectorize - return vector, nil - } - // vectorize text - icheck := NewClassSettings(cfg) - res, err := v.client.Vectorize(ctx, text, ent.VectorizationConfig{ - Model: icheck.Model(), - BaseURL: icheck.BaseURL(), - }) - if err != nil { - return nil, err - } - - if len(res.Vector) > 1 { - return libvectorizer.CombineVectors(res.Vector), nil - } - return res.Vector[0], nil -} diff --git a/modules/text2vec-jinaai/vectorizer/objects_test.go b/modules/text2vec-jinaai/vectorizer/objects_test.go deleted file mode 100644 index 14d5763ab1f9255c0ff16edaa4759d435797998c..0000000000000000000000000000000000000000 --- a/modules/text2vec-jinaai/vectorizer/objects_test.go +++ /dev/null @@ -1,385 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/moduletools" -) - -// These are mostly copy/pasted (with minimal additions) from the -// text2vec-contextionary module -func TestVectorizingObjects(t *testing.T) { - type testCase struct { - name string - input *models.Object - expectedClientCall string - expectedJinaAIModel string - noindex string - excludedProperty string // to simulate a schema where property names aren't vectorized - excludedClass string // to simulate a schema where class names aren't vectorized - jinaAIModel string - } - - tests := []testCase{ - { - name: "empty object", - input: &models.Object{ - Class: "Car", - }, - jinaAIModel: "jina-embedding-v2", - expectedJinaAIModel: "jina-embedding-v2", - expectedClientCall: "car", - }, - { - name: "object with one string prop", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "Mercedes", - }, - }, - expectedClientCall: "car brand mercedes", - }, - { - name: "object with one non-string prop", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "power": 300, - }, - }, - expectedClientCall: "car", - }, - { - name: "object with a mix of props", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: "car brand best brand review a very great car", - }, - { - name: "with a noindexed property", - noindex: "review", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: "car brand best brand", - }, - { - name: "with the class name not vectorized", - excludedClass: "Car", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: "brand best brand review a very great car", - }, - { - name: "with a property name not vectorized", - excludedProperty: "review", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: "car brand best brand a very great car", - }, - { - name: "with no schema labels vectorized", - excludedProperty: "review", - excludedClass: "Car", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "review": "a very great car", - }, - }, - expectedClientCall: "a very great car", - }, - { - name: "with string/text arrays without propname or classname", - excludedProperty: "reviews", - excludedClass: "Car", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - expectedClientCall: "a very great car you should consider buying one", - }, - { - name: "with string/text arrays with propname and classname", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - expectedClientCall: "car reviews a very great car reviews you should consider buying one", - }, - { - name: "with compound class and prop names", - input: &models.Object{ - Class: "SuperCar", - Properties: map[string]interface{}{ - "brandOfTheCar": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: "super car brand of the car best brand review a very great car", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - client := &fakeClient{} - - v := New(client) - - ic := &fakeClassConfig{ - excludedProperty: test.excludedProperty, - skippedProperty: test.noindex, - vectorizeClassName: test.excludedClass != "Car", - jinaAIModel: test.jinaAIModel, - vectorizePropertyName: true, - } - err := v.Object(context.Background(), test.input, nil, ic) - - require.Nil(t, err) - assert.Equal(t, models.C11yVector{0, 1, 2, 3}, test.input.Vector) - assert.Equal(t, []string{test.expectedClientCall}, client.lastInput) - assert.Equal(t, client.lastConfig.Model, test.expectedJinaAIModel) - }) - } -} - -func TestClassSettings(t *testing.T) { - type testCase struct { - expectedBaseURL string - cfg moduletools.ClassConfig - } - tests := []testCase{ - { - cfg: fakeClassConfig{ - classConfig: make(map[string]interface{}), - }, - expectedBaseURL: DefaultBaseURL, - }, - { - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "baseURL": "https://proxy.weaviate.dev", - }, - }, - expectedBaseURL: "https://proxy.weaviate.dev", - }, - } - - for _, tt := range tests { - ic := NewClassSettings(tt.cfg) - assert.Equal(t, tt.expectedBaseURL, ic.BaseURL()) - } -} - -func TestVectorizingObjectWithDiff(t *testing.T) { - type testCase struct { - name string - input *models.Object - skipped string - diff *moduletools.ObjectDiff - expectedVectorize bool - } - - tests := []testCase{ - { - name: "no diff", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: nil, - expectedVectorize: true, - }, - { - name: "diff all props unchanged", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("brand", "best brand", "best brand"). - WithProp("power", 300, 300). - WithProp("description", "a very great car", "a very great car"). - WithProp("reviews", []interface{}{ - "a very great car", - "you should consider buying one", - }, []interface{}{ - "a very great car", - "you should consider buying one", - }), - expectedVectorize: false, - }, - { - name: "diff one vectorizable prop changed (1)", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("brand", "old best brand", "best brand"), - expectedVectorize: true, - }, - { - name: "diff one vectorizable prop changed (2)", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("description", "old a very great car", "a very great car"), - expectedVectorize: true, - }, - { - name: "diff one vectorizable prop changed (3)", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("reviews", []interface{}{ - "old a very great car", - "you should consider buying one", - }, []interface{}{ - "a very great car", - "you should consider buying one", - }), - expectedVectorize: true, - }, - { - name: "all non-vectorizable props changed", - skipped: "description", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("power", 123, 300). - WithProp("description", "old a very great car", "a very great car"), - expectedVectorize: false, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - ic := &fakeClassConfig{ - skippedProperty: test.skipped, - } - - client := &fakeClient{} - v := New(client) - - err := v.Object(context.Background(), test.input, test.diff, ic) - - require.Nil(t, err) - if test.expectedVectorize { - assert.Equal(t, models.C11yVector{0, 1, 2, 3}, test.input.Vector) - assert.NotEmpty(t, client.lastInput) - } else { - assert.Equal(t, models.C11yVector{0, 0, 0, 0}, test.input.Vector) - assert.Empty(t, client.lastInput) - } - }) - } -} - -func newObjectDiffWithVector() *moduletools.ObjectDiff { - return moduletools.NewObjectDiff([]float32{0, 0, 0, 0}) -} diff --git a/modules/text2vec-jinaai/vectorizer/texts.go b/modules/text2vec-jinaai/vectorizer/texts.go deleted file mode 100644 index f37f00e8df44e86d7850149565fcf89a65678f6d..0000000000000000000000000000000000000000 --- a/modules/text2vec-jinaai/vectorizer/texts.go +++ /dev/null @@ -1,39 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/modules/text2vec-jinaai/ent" - libvectorizer "github.com/weaviate/weaviate/usecases/vectorizer" -) - -func (v *Vectorizer) Texts(ctx context.Context, inputs []string, - cfg moduletools.ClassConfig, -) ([]float32, error) { - settings := NewClassSettings(cfg) - res, err := v.client.VectorizeQuery(ctx, inputs, ent.VectorizationConfig{ - Model: settings.Model(), - BaseURL: settings.BaseURL(), - }) - if err != nil { - return nil, errors.Wrap(err, "remote client vectorize") - } - - if len(res.Vector) > 1 { - return libvectorizer.CombineVectors(res.Vector), nil - } - return res.Vector[0], nil -} diff --git a/modules/text2vec-jinaai/vectorizer/texts_test.go b/modules/text2vec-jinaai/vectorizer/texts_test.go deleted file mode 100644 index 76e7aec05e3e4b06f191e786b22d1359ec649fae..0000000000000000000000000000000000000000 --- a/modules/text2vec-jinaai/vectorizer/texts_test.go +++ /dev/null @@ -1,93 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// as used in the nearText searcher -func TestVectorizingTexts(t *testing.T) { - type testCase struct { - name string - input []string - expectedJinaAIModel string - jinaAIModel string - } - - tests := []testCase{ - { - name: "single word", - input: []string{"hello"}, - jinaAIModel: "jina-embedding-v2", - expectedJinaAIModel: "jina-embedding-v2", - }, - { - name: "multiple words", - input: []string{"hello world, this is me!"}, - jinaAIModel: "jina-embedding-v2", - expectedJinaAIModel: "jina-embedding-v2", - }, - { - name: "multiple sentences (joined with a dot)", - input: []string{"this is sentence 1", "and here's number 2"}, - jinaAIModel: "jina-embedding-v2", - expectedJinaAIModel: "jina-embedding-v2", - }, - { - name: "multiple sentences already containing a dot", - input: []string{"this is sentence 1.", "and here's number 2"}, - jinaAIModel: "jina-embedding-v2", - expectedJinaAIModel: "jina-embedding-v2", - }, - { - name: "multiple sentences already containing a question mark", - input: []string{"this is sentence 1?", "and here's number 2"}, - jinaAIModel: "jina-embedding-v2", - expectedJinaAIModel: "jina-embedding-v2", - }, - { - name: "multiple sentences already containing an exclamation mark", - input: []string{"this is sentence 1!", "and here's number 2"}, - jinaAIModel: "jina-embedding-v2", - expectedJinaAIModel: "jina-embedding-v2", - }, - { - name: "multiple sentences already containing comma", - input: []string{"this is sentence 1,", "and here's number 2"}, - jinaAIModel: "jina-embedding-v2", - expectedJinaAIModel: "jina-embedding-v2", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - client := &fakeClient{} - - v := New(client) - - settings := &fakeClassConfig{ - jinaAIModel: test.jinaAIModel, - } - vec, err := v.Texts(context.Background(), test.input, settings) - - require.Nil(t, err) - assert.Equal(t, []float32{0.1, 1.1, 2.1, 3.1}, vec) - assert.Equal(t, test.input, client.lastInput) - assert.Equal(t, client.lastConfig.Model, test.expectedJinaAIModel) - }) - } -} diff --git a/modules/text2vec-openai/clients/meta.go b/modules/text2vec-openai/clients/meta.go deleted file mode 100644 index a414da9f2dec92cadd255ac1bf274a254094ccf4..0000000000000000000000000000000000000000 --- a/modules/text2vec-openai/clients/meta.go +++ /dev/null @@ -1,19 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -func (v *vectorizer) MetaInfo() (map[string]interface{}, error) { - return map[string]interface{}{ - "name": "OpenAI Module", - "documentationHref": "https://platform.openai.com/docs/guides/embeddings/what-are-embeddings", - }, nil -} diff --git a/modules/text2vec-openai/clients/openai.go b/modules/text2vec-openai/clients/openai.go deleted file mode 100644 index 9f0d2b703509f68b9e1f35f0a04a1f442f051831..0000000000000000000000000000000000000000 --- a/modules/text2vec-openai/clients/openai.go +++ /dev/null @@ -1,275 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "time" - - "github.com/weaviate/weaviate/usecases/modulecomponents" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/modules/text2vec-openai/ent" -) - -type embeddingsRequest struct { - Input []string `json:"input"` - Model string `json:"model,omitempty"` -} - -type embedding struct { - Object string `json:"object"` - Data []embeddingData `json:"data,omitempty"` - Error *openAIApiError `json:"error,omitempty"` -} - -type embeddingData struct { - Object string `json:"object"` - Index int `json:"index"` - Embedding []float32 `json:"embedding"` -} - -type openAIApiError struct { - Message string `json:"message"` - Type string `json:"type"` - Param string `json:"param"` - Code json.Number `json:"code"` -} - -func buildUrl(baseURL, resourceName, deploymentID string, isAzure bool) (string, error) { - if isAzure { - host := baseURL - if host == "" || host == "https://api.openai.com" { - // Fall back to old assumption - host = "https://" + resourceName + ".openai.azure.com" - } - - path := "openai/deployments/" + deploymentID + "/embeddings" - queryParam := "api-version=2022-12-01" - return fmt.Sprintf("%s/%s?%s", host, path, queryParam), nil - } - - host := baseURL - path := "/v1/embeddings" - return url.JoinPath(host, path) -} - -type vectorizer struct { - openAIApiKey string - openAIOrganization string - azureApiKey string - httpClient *http.Client - buildUrlFn func(baseURL, resourceName, deploymentID string, isAzure bool) (string, error) - logger logrus.FieldLogger -} - -func New(openAIApiKey, openAIOrganization, azureApiKey string, timeout time.Duration, logger logrus.FieldLogger) *vectorizer { - return &vectorizer{ - openAIApiKey: openAIApiKey, - openAIOrganization: openAIOrganization, - azureApiKey: azureApiKey, - httpClient: &http.Client{ - Timeout: timeout, - }, - buildUrlFn: buildUrl, - logger: logger, - } -} - -func (v *vectorizer) Vectorize(ctx context.Context, input string, - config ent.VectorizationConfig, -) (*ent.VectorizationResult, error) { - return v.vectorize(ctx, []string{input}, v.getModelString(config.Type, config.Model, "document", config.ModelVersion), config) -} - -func (v *vectorizer) VectorizeQuery(ctx context.Context, input []string, - config ent.VectorizationConfig, -) (*ent.VectorizationResult, error) { - return v.vectorize(ctx, input, v.getModelString(config.Type, config.Model, "query", config.ModelVersion), config) -} - -func (v *vectorizer) vectorize(ctx context.Context, input []string, model string, config ent.VectorizationConfig) (*ent.VectorizationResult, error) { - body, err := json.Marshal(v.getEmbeddingsRequest(input, model, config.IsAzure)) - if err != nil { - return nil, errors.Wrap(err, "marshal body") - } - - endpoint, err := v.buildURL(ctx, config) - if err != nil { - return nil, errors.Wrap(err, "join OpenAI API host and path") - } - - req, err := http.NewRequestWithContext(ctx, "POST", endpoint, - bytes.NewReader(body)) - if err != nil { - return nil, errors.Wrap(err, "create POST request") - } - apiKey, err := v.getApiKey(ctx, config.IsAzure) - if err != nil { - return nil, errors.Wrap(err, "API Key") - } - req.Header.Add(v.getApiKeyHeaderAndValue(apiKey, config.IsAzure)) - if openAIOrganization := v.getOpenAIOrganization(ctx); openAIOrganization != "" { - req.Header.Add("OpenAI-Organization", openAIOrganization) - } - req.Header.Add("Content-Type", "application/json") - - res, err := v.httpClient.Do(req) - if err != nil { - return nil, errors.Wrap(err, "send POST request") - } - defer res.Body.Close() - - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - return nil, errors.Wrap(err, "read response body") - } - - var resBody embedding - if err := json.Unmarshal(bodyBytes, &resBody); err != nil { - return nil, errors.Wrap(err, "unmarshal response body") - } - - if res.StatusCode != 200 || resBody.Error != nil { - return nil, v.getError(res.StatusCode, resBody.Error, config.IsAzure) - } - - texts := make([]string, len(resBody.Data)) - embeddings := make([][]float32, len(resBody.Data)) - for i := range resBody.Data { - texts[i] = resBody.Data[i].Object - embeddings[i] = resBody.Data[i].Embedding - } - - return &ent.VectorizationResult{ - Text: texts, - Dimensions: len(resBody.Data[0].Embedding), - Vector: embeddings, - }, nil -} - -func (v *vectorizer) buildURL(ctx context.Context, config ent.VectorizationConfig) (string, error) { - baseURL, resourceName, deploymentID, isAzure := config.BaseURL, config.ResourceName, config.DeploymentID, config.IsAzure - if headerBaseURL := v.getValueFromContext(ctx, "X-Openai-Baseurl"); headerBaseURL != "" { - baseURL = headerBaseURL - } - return v.buildUrlFn(baseURL, resourceName, deploymentID, isAzure) -} - -func (v *vectorizer) getError(statusCode int, resBodyError *openAIApiError, isAzure bool) error { - endpoint := "OpenAI API" - if isAzure { - endpoint = "Azure OpenAI API" - } - if resBodyError != nil { - return fmt.Errorf("connection to: %s failed with status: %d error: %v", endpoint, statusCode, resBodyError.Message) - } - return fmt.Errorf("connection to: %s failed with status: %d", endpoint, statusCode) -} - -func (v *vectorizer) getEmbeddingsRequest(input []string, model string, isAzure bool) embeddingsRequest { - if isAzure { - return embeddingsRequest{Input: input} - } - return embeddingsRequest{Input: input, Model: model} -} - -func (v *vectorizer) getApiKeyHeaderAndValue(apiKey string, isAzure bool) (string, string) { - if isAzure { - return "api-key", apiKey - } - return "Authorization", fmt.Sprintf("Bearer %s", apiKey) -} - -func (v *vectorizer) getOpenAIOrganization(ctx context.Context) string { - if value := v.getValueFromContext(ctx, "X-Openai-Organization"); value != "" { - return value - } - return v.openAIOrganization -} - -func (v *vectorizer) getApiKey(ctx context.Context, isAzure bool) (string, error) { - var apiKey, envVar string - - if isAzure { - apiKey = "X-Azure-Api-Key" - envVar = "AZURE_APIKEY" - if len(v.azureApiKey) > 0 { - return v.azureApiKey, nil - } - } else { - apiKey = "X-Openai-Api-Key" - envVar = "OPENAI_APIKEY" - if len(v.openAIApiKey) > 0 { - return v.openAIApiKey, nil - } - } - - return v.getApiKeyFromContext(ctx, apiKey, envVar) -} - -func (v *vectorizer) getApiKeyFromContext(ctx context.Context, apiKey, envVar string) (string, error) { - if apiKeyValue := v.getValueFromContext(ctx, apiKey); apiKeyValue != "" { - return apiKeyValue, nil - } - return "", fmt.Errorf("no api key found neither in request header: %s nor in environment variable under %s", apiKey, envVar) -} - -func (v *vectorizer) getValueFromContext(ctx context.Context, key string) string { - if value := ctx.Value(key); value != nil { - if keyHeader, ok := value.([]string); ok && len(keyHeader) > 0 && len(keyHeader[0]) > 0 { - return keyHeader[0] - } - } - // try getting header from GRPC if not successful - if apiKey := modulecomponents.GetValueFromGRPC(ctx, key); len(apiKey) > 0 && len(apiKey[0]) > 0 { - return apiKey[0] - } - - return "" -} - -func (v *vectorizer) getModelString(docType, model, action, version string) string { - if version == "002" { - return v.getModel002String(model) - } - - return v.getModel001String(docType, model, action) -} - -func (v *vectorizer) getModel001String(docType, model, action string) string { - modelBaseString := "%s-search-%s-%s-001" - if action == "document" { - if docType == "code" { - return fmt.Sprintf(modelBaseString, docType, model, "code") - } - return fmt.Sprintf(modelBaseString, docType, model, "doc") - - } else { - if docType == "code" { - return fmt.Sprintf(modelBaseString, docType, model, "text") - } - return fmt.Sprintf(modelBaseString, docType, model, "query") - } -} - -func (v *vectorizer) getModel002String(model string) string { - modelBaseString := "text-embedding-%s-002" - return fmt.Sprintf(modelBaseString, model) -} diff --git a/modules/text2vec-openai/clients/openai_test.go b/modules/text2vec-openai/clients/openai_test.go deleted file mode 100644 index 70fbdef2fdce2c5316a30539e8a4c1384dcbb617..0000000000000000000000000000000000000000 --- a/modules/text2vec-openai/clients/openai_test.go +++ /dev/null @@ -1,495 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "context" - "encoding/json" - "io" - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/modules/text2vec-openai/ent" -) - -func TestBuildUrlFn(t *testing.T) { - t.Run("buildUrlFn returns default OpenAI Client", func(t *testing.T) { - config := ent.VectorizationConfig{ - Type: "", - Model: "", - ModelVersion: "", - ResourceName: "", - DeploymentID: "", - BaseURL: "https://api.openai.com", - IsAzure: false, - } - url, err := buildUrl(config.BaseURL, config.ResourceName, config.DeploymentID, config.IsAzure) - assert.Nil(t, err) - assert.Equal(t, "https://api.openai.com/v1/embeddings", url) - }) - t.Run("buildUrlFn returns Azure Client", func(t *testing.T) { - config := ent.VectorizationConfig{ - Type: "", - Model: "", - ModelVersion: "", - ResourceName: "resourceID", - DeploymentID: "deploymentID", - BaseURL: "", - IsAzure: true, - } - url, err := buildUrl(config.BaseURL, config.ResourceName, config.DeploymentID, config.IsAzure) - assert.Nil(t, err) - assert.Equal(t, "https://resourceID.openai.azure.com/openai/deployments/deploymentID/embeddings?api-version=2022-12-01", url) - }) - - t.Run("buildUrlFn returns Azure client with BaseUrl set", func(t *testing.T) { - config := ent.VectorizationConfig{ - Type: "", - Model: "", - ModelVersion: "", - ResourceName: "resourceID", - DeploymentID: "deploymentID", - BaseURL: "https://foobar.some.proxy", - IsAzure: true, - } - url, err := buildUrl(config.BaseURL, config.ResourceName, config.DeploymentID, config.IsAzure) - assert.Nil(t, err) - assert.Equal(t, "https://foobar.some.proxy/openai/deployments/deploymentID/embeddings?api-version=2022-12-01", url) - }) - - t.Run("buildUrlFn loads from BaseURL", func(t *testing.T) { - config := ent.VectorizationConfig{ - Type: "", - Model: "", - ModelVersion: "", - ResourceName: "resourceID", - DeploymentID: "deploymentID", - BaseURL: "https://foobar.some.proxy", - IsAzure: false, - } - url, err := buildUrl(config.BaseURL, config.ResourceName, config.DeploymentID, config.IsAzure) - assert.Nil(t, err) - assert.Equal(t, "https://foobar.some.proxy/v1/embeddings", url) - }) -} - -func TestClient(t *testing.T) { - t.Run("when all is fine", func(t *testing.T) { - server := httptest.NewServer(&fakeHandler{t: t}) - defer server.Close() - - c := New("apiKey", "", "", 0, nullLogger()) - c.buildUrlFn = func(baseURL, resourceName, deploymentID string, isAzure bool) (string, error) { - return server.URL, nil - } - - expected := &ent.VectorizationResult{ - Text: []string{"This is my text"}, - Vector: [][]float32{{0.1, 0.2, 0.3}}, - Dimensions: 3, - } - res, err := c.Vectorize(context.Background(), "This is my text", - ent.VectorizationConfig{ - Type: "text", - Model: "ada", - }) - - assert.Nil(t, err) - assert.Equal(t, expected, res) - }) - - t.Run("when the context is expired", func(t *testing.T) { - server := httptest.NewServer(&fakeHandler{t: t}) - defer server.Close() - c := New("apiKey", "", "", 0, nullLogger()) - c.buildUrlFn = func(baseURL, resourceName, deploymentID string, isAzure bool) (string, error) { - return server.URL, nil - } - - ctx, cancel := context.WithDeadline(context.Background(), time.Now()) - defer cancel() - - _, err := c.Vectorize(ctx, "This is my text", ent.VectorizationConfig{}) - - require.NotNil(t, err) - assert.Contains(t, err.Error(), "context deadline exceeded") - }) - - t.Run("when the server returns an error", func(t *testing.T) { - server := httptest.NewServer(&fakeHandler{ - t: t, - serverError: errors.Errorf("nope, not gonna happen"), - }) - defer server.Close() - c := New("apiKey", "", "", 0, nullLogger()) - c.buildUrlFn = func(baseURL, resourceName, deploymentID string, isAzure bool) (string, error) { - return server.URL, nil - } - - _, err := c.Vectorize(context.Background(), "This is my text", - ent.VectorizationConfig{}) - - require.NotNil(t, err) - assert.EqualError(t, err, "connection to: OpenAI API failed with status: 500 error: nope, not gonna happen") - }) - - t.Run("when OpenAI key is passed using X-Openai-Api-Key header", func(t *testing.T) { - server := httptest.NewServer(&fakeHandler{t: t}) - defer server.Close() - c := New("", "", "", 0, nullLogger()) - c.buildUrlFn = func(baseURL, resourceName, deploymentID string, isAzure bool) (string, error) { - return server.URL, nil - } - - ctxWithValue := context.WithValue(context.Background(), - "X-Openai-Api-Key", []string{"some-key"}) - - expected := &ent.VectorizationResult{ - Text: []string{"This is my text"}, - Vector: [][]float32{{0.1, 0.2, 0.3}}, - Dimensions: 3, - } - res, err := c.Vectorize(ctxWithValue, "This is my text", - ent.VectorizationConfig{ - Type: "text", - Model: "ada", - }) - - require.Nil(t, err) - assert.Equal(t, expected, res) - }) - - t.Run("when OpenAI key is empty", func(t *testing.T) { - server := httptest.NewServer(&fakeHandler{t: t}) - defer server.Close() - c := New("", "", "", 0, nullLogger()) - c.buildUrlFn = func(baseURL, resourceName, deploymentID string, isAzure bool) (string, error) { - return server.URL, nil - } - - ctx, cancel := context.WithDeadline(context.Background(), time.Now()) - defer cancel() - - _, err := c.Vectorize(ctx, "This is my text", ent.VectorizationConfig{}) - - require.NotNil(t, err) - assert.EqualError(t, err, "API Key: no api key found "+ - "neither in request header: X-Openai-Api-Key "+ - "nor in environment variable under OPENAI_APIKEY") - }) - - t.Run("when X-Openai-Api-Key header is passed but empty", func(t *testing.T) { - server := httptest.NewServer(&fakeHandler{t: t}) - defer server.Close() - c := New("", "", "", 0, nullLogger()) - c.buildUrlFn = func(baseURL, resourceName, deploymentID string, isAzure bool) (string, error) { - return server.URL, nil - } - - ctxWithValue := context.WithValue(context.Background(), - "X-Openai-Api-Key", []string{""}) - - _, err := c.Vectorize(ctxWithValue, "This is my text", - ent.VectorizationConfig{ - Type: "text", - Model: "ada", - }) - - require.NotNil(t, err) - assert.EqualError(t, err, "API Key: no api key found "+ - "neither in request header: X-Openai-Api-Key "+ - "nor in environment variable under OPENAI_APIKEY") - }) - - t.Run("when X-OpenAI-BaseURL header is passed", func(t *testing.T) { - server := httptest.NewServer(&fakeHandler{t: t}) - defer server.Close() - c := New("", "", "", 0, nullLogger()) - - config := ent.VectorizationConfig{ - Type: "text", - Model: "ada", - BaseURL: "http://default-url.com", - } - - ctxWithValue := context.WithValue(context.Background(), - "X-Openai-Baseurl", []string{"http://base-url-passed-in-header.com"}) - - buildURL, err := c.buildURL(ctxWithValue, config) - require.NoError(t, err) - assert.Equal(t, "http://base-url-passed-in-header.com/v1/embeddings", buildURL) - - buildURL, err = c.buildURL(context.TODO(), config) - require.NoError(t, err) - assert.Equal(t, "http://default-url.com/v1/embeddings", buildURL) - }) -} - -type fakeHandler struct { - t *testing.T - serverError error -} - -func (f *fakeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - assert.Equal(f.t, http.MethodPost, r.Method) - - if f.serverError != nil { - embeddingError := map[string]interface{}{ - "message": f.serverError.Error(), - "type": "invalid_request_error", - } - embedding := map[string]interface{}{ - "error": embeddingError, - } - outBytes, err := json.Marshal(embedding) - require.Nil(f.t, err) - - w.WriteHeader(http.StatusInternalServerError) - w.Write(outBytes) - return - } - - bodyBytes, err := io.ReadAll(r.Body) - require.Nil(f.t, err) - defer r.Body.Close() - - var b map[string]interface{} - require.Nil(f.t, json.Unmarshal(bodyBytes, &b)) - - textInputArray := b["input"].([]interface{}) - textInput := textInputArray[0].(string) - assert.Greater(f.t, len(textInput), 0) - - embeddingData := map[string]interface{}{ - "object": textInput, - "index": 0, - "embedding": []float32{0.1, 0.2, 0.3}, - } - embedding := map[string]interface{}{ - "object": "list", - "data": []interface{}{embeddingData}, - } - - outBytes, err := json.Marshal(embedding) - require.Nil(f.t, err) - - w.Write(outBytes) -} - -func nullLogger() logrus.FieldLogger { - l, _ := test.NewNullLogger() - return l -} - -func Test_getModelString(t *testing.T) { - t.Run("getModelStringDocument", func(t *testing.T) { - type args struct { - docType string - model string - version string - } - tests := []struct { - name string - args args - want string - }{ - { - name: "Document type: text model: ada vectorizationType: document", - args: args{ - docType: "text", - model: "ada", - }, - want: "text-search-ada-doc-001", - }, - { - name: "Document type: text model: ada-002 vectorizationType: document", - args: args{ - docType: "text", - model: "ada", - version: "002", - }, - want: "text-embedding-ada-002", - }, - { - name: "Document type: text model: babbage vectorizationType: document", - args: args{ - docType: "text", - model: "babbage", - }, - want: "text-search-babbage-doc-001", - }, - { - name: "Document type: text model: curie vectorizationType: document", - args: args{ - docType: "text", - model: "curie", - }, - want: "text-search-curie-doc-001", - }, - { - name: "Document type: text model: davinci vectorizationType: document", - args: args{ - docType: "text", - model: "davinci", - }, - want: "text-search-davinci-doc-001", - }, - { - name: "Document type: code model: ada vectorizationType: code", - args: args{ - docType: "code", - model: "ada", - }, - want: "code-search-ada-code-001", - }, - { - name: "Document type: code model: babbage vectorizationType: code", - args: args{ - docType: "code", - model: "babbage", - }, - want: "code-search-babbage-code-001", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - v := New("apiKey", "", "", 0, nullLogger()) - if got := v.getModelString(tt.args.docType, tt.args.model, "document", tt.args.version); got != tt.want { - t.Errorf("vectorizer.getModelString() = %v, want %v", got, tt.want) - } - }) - } - }) - - t.Run("getModelStringQuery", func(t *testing.T) { - type args struct { - docType string - model string - version string - } - tests := []struct { - name string - args args - want string - }{ - { - name: "Document type: text model: ada vectorizationType: query", - args: args{ - docType: "text", - model: "ada", - }, - want: "text-search-ada-query-001", - }, - { - name: "Document type: text model: babbage vectorizationType: query", - args: args{ - docType: "text", - model: "babbage", - }, - want: "text-search-babbage-query-001", - }, - { - name: "Document type: text model: curie vectorizationType: query", - args: args{ - docType: "text", - model: "curie", - }, - want: "text-search-curie-query-001", - }, - { - name: "Document type: text model: davinci vectorizationType: query", - args: args{ - docType: "text", - model: "davinci", - }, - want: "text-search-davinci-query-001", - }, - { - name: "Document type: code model: ada vectorizationType: text", - args: args{ - docType: "code", - model: "ada", - }, - want: "code-search-ada-text-001", - }, - { - name: "Document type: code model: babbage vectorizationType: text", - args: args{ - docType: "code", - model: "babbage", - }, - want: "code-search-babbage-text-001", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - v := New("apiKey", "", "", 0, nullLogger()) - if got := v.getModelString(tt.args.docType, tt.args.model, "query", tt.args.version); got != tt.want { - t.Errorf("vectorizer.getModelString() = %v, want %v", got, tt.want) - } - }) - } - }) -} - -func TestOpenAIApiErrorDecode(t *testing.T) { - t.Run("getModelStringQuery", func(t *testing.T) { - type args struct { - response []byte - } - tests := []struct { - name string - args args - want string - }{ - { - name: "Error code: missing property", - args: args{ - response: []byte(`{"message": "failed", "type": "error", "param": "arg..."}`), - }, - want: "", - }, - { - name: "Error code: as int", - args: args{ - response: []byte(`{"message": "failed", "type": "error", "param": "arg...", "code": 500}`), - }, - want: "500", - }, - { - name: "Error code as string", - args: args{ - response: []byte(`{"message": "failed", "type": "error", "param": "arg...", "code": "500"}`), - }, - want: "500", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - var got *openAIApiError - err := json.Unmarshal(tt.args.response, &got) - require.NoError(t, err) - - if got.Code.String() != tt.want { - t.Errorf("OpenAIerror.code = %v, want %v", got.Code, tt.want) - } - }) - } - }) -} diff --git a/modules/text2vec-openai/config.go b/modules/text2vec-openai/config.go deleted file mode 100644 index a90e79d274608426dfca0c780c42bf4b72f69b2e..0000000000000000000000000000000000000000 --- a/modules/text2vec-openai/config.go +++ /dev/null @@ -1,151 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modopenai - -import ( - "context" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/modules/text2vec-openai/vectorizer" -) - -func (m *OpenAIModule) ClassConfigDefaults() map[string]interface{} { - return map[string]interface{}{ - "vectorizeClassName": vectorizer.DefaultVectorizeClassName, - "type": vectorizer.DefaultOpenAIDocumentType, - "model": vectorizer.DefaultOpenAIModel, - "baseURL": vectorizer.DefaultBaseURL, - "modelVersion": vectorizer.PickDefaultModelVersion(vectorizer.DefaultOpenAIModel, - vectorizer.DefaultOpenAIDocumentType), - } -} - -func (m *OpenAIModule) PropertyConfigDefaults( - dt *schema.DataType, -) map[string]interface{} { - return map[string]interface{}{ - "skip": !vectorizer.DefaultPropertyIndexed, - "vectorizePropertyName": vectorizer.DefaultVectorizePropertyName, - } -} - -func (m *OpenAIModule) ValidateClass(ctx context.Context, - class *models.Class, cfg moduletools.ClassConfig, -) error { - settings := vectorizer.NewClassSettings(cfg) - return settings.Validate(class) -} - -var _ = modulecapabilities.ClassConfigurator(New()) - -// type ConfigValidator struct { -// logger logrus.FieldLogger -// } - -// type ClassSettings interface { -// VectorizeClassName() bool -// VectorizePropertyName(propName string) bool -// PropertyIndexed(propName string) bool -// } - -// func NewConfigValidator(logger logrus.FieldLogger) *ConfigValidator { -// return &ConfigValidator{logger: logger} -// } - -// func (cv *ConfigValidator) Do(ctx context.Context, class *models.Class, -// cfg moduletools.ClassConfig, settings ClassSettings) error { -// // In text2vec-openai (as opposed to e.g. text2vec-contextionary) the -// // assumption is that the models will be able to deal with any words, even -// // previously unseen ones. Therefore we do not need to validate individual -// // properties, but only the overall "index state" - -// if err := cv.validateIndexState(ctx, class, settings); err != nil { -// return errors.Errorf("invalid combination of properties") -// } - -// cv.checkForPossibilityOfDuplicateVectors(ctx, class, settings) - -// return nil -// } - -// func (cv *ConfigValidator) validateIndexState(ctx context.Context, -// class *models.Class, settings ClassSettings) error { -// if settings.VectorizeClassName() { -// // if the user chooses to vectorize the classname, vector-building will -// // always be possible, no need to investigate further - -// return nil -// } - -// // search if there is at least one indexed, string/text prop. If found pass -// // validation -// for _, prop := range class.Properties { -// if len(prop.DataType) < 1 { -// return errors.Errorf("property %s must have at least one datatype: "+ -// "got %v", prop.Name, prop.DataType) -// } - -// if prop.DataType[0] != string(schema.DataTypeText) { -// // we can only vectorize text-like props -// continue -// } - -// if settings.PropertyIndexed(prop.Name) { -// // found at least one, this is a valid schema -// return nil -// } -// } - -// return fmt.Errorf("invalid properties: didn't find a single property which is " + -// "of type string or text and is not excluded from indexing. In addition the " + -// "class name is excluded from vectorization as well, meaning that it cannot be " + -// "used to determine the vector position. To fix this, set 'vectorizeClassName' " + -// "to true if the class name is contextionary-valid. Alternatively add at least " + -// "contextionary-valid text/string property which is not excluded from " + -// "indexing.") -// } - -// func (cv *ConfigValidator) checkForPossibilityOfDuplicateVectors( -// ctx context.Context, class *models.Class, settings ClassSettings) { -// if !settings.VectorizeClassName() { -// // if the user choses not to vectorize the class name, this means they must -// // have chosen something else to vectorize, otherwise the validation would -// // have error'd before we ever got here. We can skip further checking. - -// return -// } - -// // search if there is at least one indexed, string/text prop. If found exit -// for _, prop := range class.Properties { -// // length check skipped, because validation has already passed -// if prop.DataType[0] != string(schema.DataTypeText) { -// // we can only vectorize text-like props -// continue -// } - -// if settings.PropertyIndexed(prop.Name) { -// // found at least one -// return -// } -// } - -// cv.logger.WithField("module", "text2vec-openai"). -// WithField("class", class.Class). -// Warnf("text2vec-openai: Class %q does not have any properties "+ -// "indexed (or only non text-properties indexed) and the vector position is "+ -// "only determined by the class name. Each object will end up with the same "+ -// "vector which leads to a severe performance penalty on imports. Consider "+ -// "setting vectorIndexConfig.skip=true for this property", class.Class) -// } diff --git a/modules/text2vec-openai/ent/vectorization_config.go b/modules/text2vec-openai/ent/vectorization_config.go deleted file mode 100644 index 22f1e27d8a6b7f18b7d26d7caf57d0e0f87a1841..0000000000000000000000000000000000000000 --- a/modules/text2vec-openai/ent/vectorization_config.go +++ /dev/null @@ -1,19 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package ent - -type VectorizationConfig struct { - Type, Model, ModelVersion, ResourceName string - BaseURL string - DeploymentID string `json:"deploymentId"` - IsAzure bool -} diff --git a/modules/text2vec-openai/ent/vectorization_result.go b/modules/text2vec-openai/ent/vectorization_result.go deleted file mode 100644 index 9120fc2f73f2900c7f547973778d28dcb4b6c1e1..0000000000000000000000000000000000000000 --- a/modules/text2vec-openai/ent/vectorization_result.go +++ /dev/null @@ -1,18 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package ent - -type VectorizationResult struct { - Text []string - Dimensions int - Vector [][]float32 -} diff --git a/modules/text2vec-openai/module.go b/modules/text2vec-openai/module.go deleted file mode 100644 index fc0e4c6e6aa016b22035cf573de4ac968dd05bc0..0000000000000000000000000000000000000000 --- a/modules/text2vec-openai/module.go +++ /dev/null @@ -1,151 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modopenai - -import ( - "context" - "net/http" - "os" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/modules/text2vec-openai/clients" - "github.com/weaviate/weaviate/modules/text2vec-openai/vectorizer" - "github.com/weaviate/weaviate/usecases/modulecomponents/additional" -) - -const Name = "text2vec-openai" - -func New() *OpenAIModule { - return &OpenAIModule{} -} - -type OpenAIModule struct { - vectorizer textVectorizer - metaProvider metaProvider - graphqlProvider modulecapabilities.GraphQLArguments - searcher modulecapabilities.Searcher - nearTextTransformer modulecapabilities.TextTransform - logger logrus.FieldLogger - additionalPropertiesProvider modulecapabilities.AdditionalProperties -} - -type textVectorizer interface { - Object(ctx context.Context, obj *models.Object, objDiff *moduletools.ObjectDiff, - cfg moduletools.ClassConfig) error - Texts(ctx context.Context, input []string, - cfg moduletools.ClassConfig) ([]float32, error) -} - -type metaProvider interface { - MetaInfo() (map[string]interface{}, error) -} - -func (m *OpenAIModule) Name() string { - return Name -} - -func (m *OpenAIModule) Type() modulecapabilities.ModuleType { - return modulecapabilities.Text2MultiVec -} - -func (m *OpenAIModule) Init(ctx context.Context, - params moduletools.ModuleInitParams, -) error { - m.logger = params.GetLogger() - - if err := m.initVectorizer(ctx, params.GetConfig().ModuleHttpClientTimeout, m.logger); err != nil { - return errors.Wrap(err, "init vectorizer") - } - - if err := m.initAdditionalPropertiesProvider(); err != nil { - return errors.Wrap(err, "init additional properties provider") - } - - return nil -} - -func (m *OpenAIModule) InitExtension(modules []modulecapabilities.Module) error { - for _, module := range modules { - if module.Name() == m.Name() { - continue - } - if arg, ok := module.(modulecapabilities.TextTransformers); ok { - if arg != nil && arg.TextTransformers() != nil { - m.nearTextTransformer = arg.TextTransformers()["nearText"] - } - } - } - - if err := m.initNearText(); err != nil { - return errors.Wrap(err, "init graphql provider") - } - return nil -} - -func (m *OpenAIModule) initVectorizer(ctx context.Context, timeout time.Duration, - logger logrus.FieldLogger, -) error { - openAIApiKey := os.Getenv("OPENAI_APIKEY") - openAIOrganization := os.Getenv("OPENAI_ORGANIZATION") - azureApiKey := os.Getenv("AZURE_APIKEY") - - client := clients.New(openAIApiKey, openAIOrganization, azureApiKey, timeout, logger) - - m.vectorizer = vectorizer.New(client) - m.metaProvider = client - - return nil -} - -func (m *OpenAIModule) initAdditionalPropertiesProvider() error { - m.additionalPropertiesProvider = additional.NewText2VecProvider() - return nil -} - -func (m *OpenAIModule) RootHandler() http.Handler { - // TODO: remove once this is a capability interface - return nil -} - -func (m *OpenAIModule) VectorizeObject(ctx context.Context, - obj *models.Object, objDiff *moduletools.ObjectDiff, cfg moduletools.ClassConfig, -) error { - return m.vectorizer.Object(ctx, obj, objDiff, cfg) -} - -func (m *OpenAIModule) MetaInfo() (map[string]interface{}, error) { - return m.metaProvider.MetaInfo() -} - -func (m *OpenAIModule) AdditionalProperties() map[string]modulecapabilities.AdditionalProperty { - return m.additionalPropertiesProvider.AdditionalProperties() -} - -func (m *OpenAIModule) VectorizeInput(ctx context.Context, - input string, cfg moduletools.ClassConfig, -) ([]float32, error) { - return m.vectorizer.Texts(ctx, []string{input}, cfg) -} - -// verify we implement the modules.Module interface -var ( - _ = modulecapabilities.Module(New()) - _ = modulecapabilities.Vectorizer(New()) - _ = modulecapabilities.MetaProvider(New()) - _ = modulecapabilities.Searcher(New()) - _ = modulecapabilities.GraphQLArguments(New()) -) diff --git a/modules/text2vec-openai/nearText.go b/modules/text2vec-openai/nearText.go deleted file mode 100644 index 8e808575facb63018b14bfbe9a2a3a8ada1169b9..0000000000000000000000000000000000000000 --- a/modules/text2vec-openai/nearText.go +++ /dev/null @@ -1,36 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modopenai - -import ( - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/usecases/modulecomponents/nearText" -) - -func (m *OpenAIModule) initNearText() error { - m.searcher = nearText.NewSearcher(m.vectorizer) - m.graphqlProvider = nearText.New(m.nearTextTransformer) - return nil -} - -func (m *OpenAIModule) Arguments() map[string]modulecapabilities.GraphQLArgument { - return m.graphqlProvider.Arguments() -} - -func (m *OpenAIModule) VectorSearches() map[string]modulecapabilities.VectorForParams { - return m.searcher.VectorSearches() -} - -var ( - _ = modulecapabilities.GraphQLArguments(New()) - _ = modulecapabilities.Searcher(New()) -) diff --git a/modules/text2vec-openai/vectorizer/class_settings.go b/modules/text2vec-openai/vectorizer/class_settings.go deleted file mode 100644 index 759bc41c1c07d7da7ac6a0b87dae92bc6c136c59..0000000000000000000000000000000000000000 --- a/modules/text2vec-openai/vectorizer/class_settings.go +++ /dev/null @@ -1,276 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "fmt" - "strings" - - "github.com/pkg/errors" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/schema" -) - -const ( - DefaultOpenAIDocumentType = "text" - DefaultOpenAIModel = "ada" - DefaultVectorizeClassName = true - DefaultPropertyIndexed = true - DefaultVectorizePropertyName = false - DefaultBaseURL = "https://api.openai.com" -) - -var availableOpenAITypes = []string{"text", "code"} - -var availableOpenAIModels = []string{ - "ada", // supports 001 and 002 - "babbage", // only supports 001 - "curie", // only supports 001 - "davinci", // only supports 001 -} - -type classSettings struct { - cfg moduletools.ClassConfig -} - -func NewClassSettings(cfg moduletools.ClassConfig) *classSettings { - return &classSettings{cfg: cfg} -} - -func (cs *classSettings) PropertyIndexed(propName string) bool { - if cs.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return DefaultPropertyIndexed - } - - vcn, ok := cs.cfg.Property(propName)["skip"] - if !ok { - return DefaultPropertyIndexed - } - - asBool, ok := vcn.(bool) - if !ok { - return DefaultPropertyIndexed - } - - return !asBool -} - -func (cs *classSettings) VectorizePropertyName(propName string) bool { - if cs.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return DefaultVectorizePropertyName - } - vcn, ok := cs.cfg.Property(propName)["vectorizePropertyName"] - if !ok { - return DefaultVectorizePropertyName - } - - asBool, ok := vcn.(bool) - if !ok { - return DefaultVectorizePropertyName - } - - return asBool -} - -func (cs *classSettings) Model() string { - return cs.getProperty("model", DefaultOpenAIModel) -} - -func (cs *classSettings) Type() string { - return cs.getProperty("type", DefaultOpenAIDocumentType) -} - -func (cs *classSettings) ModelVersion() string { - defaultVersion := PickDefaultModelVersion(cs.Model(), cs.Type()) - return cs.getProperty("modelVersion", defaultVersion) -} - -func (cs *classSettings) ResourceName() string { - return cs.getProperty("resourceName", "") -} - -func (cs *classSettings) BaseURL() string { - return cs.getProperty("baseURL", DefaultBaseURL) -} - -func (cs *classSettings) DeploymentID() string { - return cs.getProperty("deploymentId", "") -} - -func (cs *classSettings) IsAzure() bool { - return cs.ResourceName() != "" && cs.DeploymentID() != "" -} - -func (cs *classSettings) VectorizeClassName() bool { - if cs.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return DefaultVectorizeClassName - } - - vcn, ok := cs.cfg.Class()["vectorizeClassName"] - if !ok { - return DefaultVectorizeClassName - } - - asBool, ok := vcn.(bool) - if !ok { - return DefaultVectorizeClassName - } - - return asBool -} - -func (cs *classSettings) Validate(class *models.Class) error { - if cs.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return errors.New("empty config") - } - - docType := cs.Type() - if !cs.validateOpenAISetting(docType, availableOpenAITypes) { - return errors.Errorf("wrong OpenAI type name, available model names are: %v", availableOpenAITypes) - } - - model := cs.Model() - if !cs.validateOpenAISetting(model, availableOpenAIModels) { - return errors.Errorf("wrong OpenAI model name, available model names are: %v", availableOpenAIModels) - } - - version := cs.ModelVersion() - if err := cs.validateModelVersion(version, model, docType); err != nil { - return err - } - - err := cs.validateAzureConfig(cs.ResourceName(), cs.DeploymentID()) - if err != nil { - return err - } - - err = cs.validateIndexState(class, cs) - if err != nil { - return err - } - - return nil -} - -func (cs *classSettings) validateModelVersion(version, model, docType string) error { - if version == "001" { - // no restrictions - return nil - } - - if version == "002" { - // only ada/davinci 002 - if model != "ada" && model != "davinci" { - return fmt.Errorf("unsupported version %s", version) - } - } - - if version == "003" && model != "davinci" { - // only davinci 003 - return fmt.Errorf("unsupported version %s", version) - } - - if version != "002" && version != "003" { - // all other fallback - return fmt.Errorf("model %s is only available in version 001", model) - } - - if docType != "text" { - return fmt.Errorf("ada-002 no longer distinguishes between text/code, use 'text' for all use cases") - } - - return nil -} - -func (cs *classSettings) validateOpenAISetting(value string, availableValues []string) bool { - for i := range availableValues { - if value == availableValues[i] { - return true - } - } - return false -} - -func (cs *classSettings) getProperty(name, defaultValue string) string { - if cs.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return defaultValue - } - - model, ok := cs.cfg.Class()[name] - if ok { - asString, ok := model.(string) - if ok { - return strings.ToLower(asString) - } - } - - return defaultValue -} - -func (cs *classSettings) validateIndexState(class *models.Class, settings ClassSettings) error { - if settings.VectorizeClassName() { - // if the user chooses to vectorize the classname, vector-building will - // always be possible, no need to investigate further - - return nil - } - - // search if there is at least one indexed, string/text prop. If found pass - // validation - for _, prop := range class.Properties { - if len(prop.DataType) < 1 { - return errors.Errorf("property %s must have at least one datatype: "+ - "got %v", prop.Name, prop.DataType) - } - - if prop.DataType[0] != string(schema.DataTypeText) { - // we can only vectorize text-like props - continue - } - - if settings.PropertyIndexed(prop.Name) { - // found at least one, this is a valid schema - return nil - } - } - - return fmt.Errorf("invalid properties: didn't find a single property which is " + - "of type string or text and is not excluded from indexing. In addition the " + - "class name is excluded from vectorization as well, meaning that it cannot be " + - "used to determine the vector position. To fix this, set 'vectorizeClassName' " + - "to true if the class name is contextionary-valid. Alternatively add at least " + - "contextionary-valid text/string property which is not excluded from " + - "indexing.") -} - -func (cs *classSettings) validateAzureConfig(resourceName string, deploymentId string) error { - if (resourceName == "" && deploymentId != "") || (resourceName != "" && deploymentId == "") { - return fmt.Errorf("both resourceName and deploymentId must be provided") - } - return nil -} - -func PickDefaultModelVersion(model, docType string) string { - if model == "ada" && docType == "text" { - return "002" - } - - // for all other combinations stick with "001" - return "001" -} diff --git a/modules/text2vec-openai/vectorizer/fakes_for_test.go b/modules/text2vec-openai/vectorizer/fakes_for_test.go deleted file mode 100644 index 1c441dc62ada8936862b3af16c43df606997a626..0000000000000000000000000000000000000000 --- a/modules/text2vec-openai/vectorizer/fakes_for_test.go +++ /dev/null @@ -1,85 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - - "github.com/weaviate/weaviate/modules/text2vec-openai/ent" -) - -type fakeClient struct { - lastInput []string - lastConfig ent.VectorizationConfig -} - -func (c *fakeClient) Vectorize(ctx context.Context, - text string, cfg ent.VectorizationConfig, -) (*ent.VectorizationResult, error) { - c.lastInput = []string{text} - c.lastConfig = cfg - return &ent.VectorizationResult{ - Vector: [][]float32{{0, 1, 2, 3}}, - Dimensions: 4, - Text: []string{text}, - }, nil -} - -func (c *fakeClient) VectorizeQuery(ctx context.Context, - text []string, cfg ent.VectorizationConfig, -) (*ent.VectorizationResult, error) { - c.lastInput = text - c.lastConfig = cfg - return &ent.VectorizationResult{ - Vector: [][]float32{{0.1, 1.1, 2.1, 3.1}}, - Dimensions: 4, - Text: text, - }, nil -} - -type fakeClassConfig struct { - classConfig map[string]interface{} - vectorizePropertyName bool - skippedProperty string - excludedProperty string -} - -func (f fakeClassConfig) Class() map[string]interface{} { - return f.classConfig -} - -func (f fakeClassConfig) ClassByModuleName(moduleName string) map[string]interface{} { - return f.classConfig -} - -func (f fakeClassConfig) Property(propName string) map[string]interface{} { - if propName == f.skippedProperty { - return map[string]interface{}{ - "skip": true, - } - } - if propName == f.excludedProperty { - return map[string]interface{}{ - "vectorizePropertyName": false, - } - } - if f.vectorizePropertyName { - return map[string]interface{}{ - "vectorizePropertyName": true, - } - } - return nil -} - -func (f fakeClassConfig) Tenant() string { - return "" -} diff --git a/modules/text2vec-openai/vectorizer/objects.go b/modules/text2vec-openai/vectorizer/objects.go deleted file mode 100644 index a249c0f80b4eeaf1cdf7ed832435ec33195a7618..0000000000000000000000000000000000000000 --- a/modules/text2vec-openai/vectorizer/objects.go +++ /dev/null @@ -1,103 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/modules/text2vec-openai/ent" - objectsvectorizer "github.com/weaviate/weaviate/usecases/modulecomponents/vectorizer" - libvectorizer "github.com/weaviate/weaviate/usecases/vectorizer" -) - -type Vectorizer struct { - client Client - objectVectorizer *objectsvectorizer.ObjectVectorizer -} - -func New(client Client) *Vectorizer { - return &Vectorizer{ - client: client, - objectVectorizer: objectsvectorizer.New(), - } -} - -type Client interface { - Vectorize(ctx context.Context, input string, - config ent.VectorizationConfig) (*ent.VectorizationResult, error) - VectorizeQuery(ctx context.Context, input []string, - config ent.VectorizationConfig) (*ent.VectorizationResult, error) -} - -// IndexCheck returns whether a property of a class should be indexed -type ClassSettings interface { - PropertyIndexed(property string) bool - VectorizePropertyName(propertyName string) bool - VectorizeClassName() bool - Model() string - Type() string - ModelVersion() string - ResourceName() string - DeploymentID() string - BaseURL() string - IsAzure() bool -} - -func (v *Vectorizer) Object(ctx context.Context, object *models.Object, - objDiff *moduletools.ObjectDiff, cfg moduletools.ClassConfig, -) error { - vec, err := v.object(ctx, object.Class, object.Properties, objDiff, cfg) - if err != nil { - return err - } - - object.Vector = vec - return nil -} - -func (v *Vectorizer) object(ctx context.Context, className string, - schema interface{}, objDiff *moduletools.ObjectDiff, cfg moduletools.ClassConfig, -) ([]float32, error) { - text, vector, err := v.objectVectorizer.TextsOrVector(ctx, className, schema, objDiff, NewClassSettings(cfg)) - if err != nil { - return nil, err - } - if vector != nil { - // dont' re-vectorize - return vector, nil - } - // vectorize text - res, err := v.client.Vectorize(ctx, text, v.getVectorizationConfig(cfg)) - if err != nil { - return nil, err - } - - if len(res.Vector) > 1 { - return libvectorizer.CombineVectors(res.Vector), nil - } - return res.Vector[0], nil -} - -func (v *Vectorizer) getVectorizationConfig(cfg moduletools.ClassConfig) ent.VectorizationConfig { - settings := NewClassSettings(cfg) - return ent.VectorizationConfig{ - Type: settings.Type(), - Model: settings.Model(), - ModelVersion: settings.ModelVersion(), - ResourceName: settings.ResourceName(), - DeploymentID: settings.DeploymentID(), - BaseURL: settings.BaseURL(), - IsAzure: settings.IsAzure(), - } -} diff --git a/modules/text2vec-openai/vectorizer/objects_test.go b/modules/text2vec-openai/vectorizer/objects_test.go deleted file mode 100644 index b7a678afd920ad4e14140b85095b6216353ba5cd..0000000000000000000000000000000000000000 --- a/modules/text2vec-openai/vectorizer/objects_test.go +++ /dev/null @@ -1,466 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/moduletools" -) - -// These are mostly copy/pasted (with minimal additions) from the -// text2vec-contextionary module -func TestVectorizingObjects(t *testing.T) { - type testCase struct { - name string - input *models.Object - expectedClientCall string - expectedOpenAIType string - expectedOpenAIModel string - noindex string - excludedProperty string // to simulate a schema where property names aren't vectorized - excludedClass string // to simulate a schema where class names aren't vectorized - openAIType string - openAIModel string - openAIModelVersion string - } - - tests := []testCase{ - { - name: "empty object", - input: &models.Object{ - Class: "Car", - }, - openAIType: "text", - openAIModel: "ada", - expectedOpenAIType: "text", - expectedOpenAIModel: "ada", - expectedClientCall: "car", - }, - { - name: "object with one string prop", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "Mercedes", - }, - }, - expectedClientCall: "car brand mercedes", - }, - { - name: "object with one non-string prop", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "power": 300, - }, - }, - expectedClientCall: "car", - }, - { - name: "object with a mix of props", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: "car brand best brand review a very great car", - }, - { - name: "with a noindexed property", - noindex: "review", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: "car brand best brand", - }, - { - name: "with the class name not vectorized", - excludedClass: "Car", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: "brand best brand review a very great car", - }, - { - name: "with a property name not vectorized", - excludedProperty: "review", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: "car brand best brand a very great car", - }, - { - name: "with no schema labels vectorized", - excludedProperty: "review", - excludedClass: "Car", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "review": "a very great car", - }, - }, - expectedClientCall: "a very great car", - }, - { - name: "with string/text arrays without propname or classname", - excludedProperty: "reviews", - excludedClass: "Car", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - expectedClientCall: "a very great car you should consider buying one", - }, - { - name: "with string/text arrays with propname and classname", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - expectedClientCall: "car reviews a very great car reviews you should consider buying one", - }, - { - name: "with compound class and prop names", - input: &models.Object{ - Class: "SuperCar", - Properties: map[string]interface{}{ - "brandOfTheCar": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: "super car brand of the car best brand review a very great car", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - client := &fakeClient{} - - v := New(client) - - cfg := &fakeClassConfig{ - classConfig: map[string]interface{}{ - "vectorizeClassName": test.excludedClass != "Car", - "type": test.openAIType, - "model": test.openAIModel, - "modelVersion": test.openAIModelVersion, - }, - vectorizePropertyName: true, - skippedProperty: test.noindex, - excludedProperty: test.excludedProperty, - } - err := v.Object(context.Background(), test.input, nil, cfg) - - require.Nil(t, err) - assert.Equal(t, models.C11yVector{0, 1, 2, 3}, test.input.Vector) - assert.Equal(t, []string{test.expectedClientCall}, client.lastInput) - assert.Equal(t, test.expectedOpenAIType, client.lastConfig.Type) - assert.Equal(t, test.expectedOpenAIModel, client.lastConfig.Model) - }) - } -} - -func TestClassSettings(t *testing.T) { - type testCase struct { - expectedBaseURL string - cfg moduletools.ClassConfig - } - tests := []testCase{ - { - cfg: fakeClassConfig{ - classConfig: make(map[string]interface{}), - }, - expectedBaseURL: DefaultBaseURL, - }, - { - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "baseURL": "https://proxy.weaviate.dev", - }, - }, - expectedBaseURL: "https://proxy.weaviate.dev", - }, - } - - for _, tt := range tests { - ic := NewClassSettings(tt.cfg) - assert.Equal(t, tt.expectedBaseURL, ic.BaseURL()) - } -} - -func TestVectorizingObjectWithDiff(t *testing.T) { - type testCase struct { - name string - input *models.Object - skipped string - diff *moduletools.ObjectDiff - expectedVectorize bool - } - - tests := []testCase{ - { - name: "no diff", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: nil, - expectedVectorize: true, - }, - { - name: "diff all props unchanged", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("brand", "best brand", "best brand"). - WithProp("power", 300, 300). - WithProp("description", "a very great car", "a very great car"). - WithProp("reviews", []interface{}{ - "a very great car", - "you should consider buying one", - }, []interface{}{ - "a very great car", - "you should consider buying one", - }), - expectedVectorize: false, - }, - { - name: "diff one vectorizable prop changed (1)", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("brand", "old best brand", "best brand"), - expectedVectorize: true, - }, - { - name: "diff one vectorizable prop changed (2)", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("description", "old a very great car", "a very great car"), - expectedVectorize: true, - }, - { - name: "diff one vectorizable prop changed (3)", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("reviews", []interface{}{ - "old a very great car", - "you should consider buying one", - }, []interface{}{ - "a very great car", - "you should consider buying one", - }), - expectedVectorize: true, - }, - { - name: "all non-vectorizable props changed", - skipped: "description", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("power", 123, 300). - WithProp("description", "old a very great car", "a very great car"), - expectedVectorize: false, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - cfg := &fakeClassConfig{ - skippedProperty: test.skipped, - } - - client := &fakeClient{} - v := New(client) - - err := v.Object(context.Background(), test.input, test.diff, cfg) - - require.Nil(t, err) - if test.expectedVectorize { - assert.Equal(t, models.C11yVector{0, 1, 2, 3}, test.input.Vector) - assert.NotEmpty(t, client.lastInput) - } else { - assert.Equal(t, models.C11yVector{0, 0, 0, 0}, test.input.Vector) - assert.Empty(t, client.lastInput) - } - }) - } -} - -func newObjectDiffWithVector() *moduletools.ObjectDiff { - return moduletools.NewObjectDiff([]float32{0, 0, 0, 0}) -} - -func TestValidateModelVersion(t *testing.T) { - type test struct { - model string - docType string - version string - possible bool - } - - tests := []test{ - // 001 models - {"ada", "text", "001", true}, - {"ada", "code", "001", true}, - {"babbage", "text", "001", true}, - {"babbage", "code", "001", true}, - {"curie", "text", "001", true}, - {"curie", "code", "001", true}, - {"davinci", "text", "001", true}, - {"davinci", "code", "001", true}, - - // 002 models - {"ada", "text", "002", true}, - {"davinci", "text", "002", true}, - {"ada", "code", "002", false}, - {"babbage", "text", "002", false}, - {"babbage", "code", "002", false}, - {"curie", "text", "002", false}, - {"curie", "code", "002", false}, - {"davinci", "code", "002", false}, - - // 003 - {"davinci", "text", "003", true}, - {"ada", "text", "003", false}, - {"babbage", "text", "003", false}, - - // 004 - {"davinci", "text", "004", false}, - {"ada", "text", "004", false}, - {"babbage", "text", "004", false}, - } - - for _, test := range tests { - name := fmt.Sprintf("model=%s docType=%s version=%s", test.model, test.docType, test.version) - t.Run(name, func(t *testing.T) { - err := (&classSettings{}).validateModelVersion(test.version, test.model, test.docType) - if test.possible { - assert.Nil(t, err, "this combination should be possible") - } else { - assert.NotNil(t, err, "this combination should not be possible") - } - }) - } -} - -func TestPickDefaultModelVersion(t *testing.T) { - t.Run("ada with text", func(t *testing.T) { - version := PickDefaultModelVersion("ada", "text") - assert.Equal(t, "002", version) - }) - - t.Run("ada with code", func(t *testing.T) { - version := PickDefaultModelVersion("ada", "code") - assert.Equal(t, "001", version) - }) - - t.Run("with curie", func(t *testing.T) { - version := PickDefaultModelVersion("curie", "text") - assert.Equal(t, "001", version) - }) -} diff --git a/modules/text2vec-openai/vectorizer/texts.go b/modules/text2vec-openai/vectorizer/texts.go deleted file mode 100644 index 3cbf49bee7768486b13be82d0778a9eadcd8e846..0000000000000000000000000000000000000000 --- a/modules/text2vec-openai/vectorizer/texts.go +++ /dev/null @@ -1,34 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/moduletools" - libvectorizer "github.com/weaviate/weaviate/usecases/vectorizer" -) - -func (v *Vectorizer) Texts(ctx context.Context, inputs []string, - cfg moduletools.ClassConfig, -) ([]float32, error) { - res, err := v.client.VectorizeQuery(ctx, inputs, v.getVectorizationConfig(cfg)) - if err != nil { - return nil, errors.Wrap(err, "remote client vectorize") - } - - if len(res.Vector) > 1 { - return libvectorizer.CombineVectors(res.Vector), nil - } - return res.Vector[0], nil -} diff --git a/modules/text2vec-openai/vectorizer/texts_test.go b/modules/text2vec-openai/vectorizer/texts_test.go deleted file mode 100644 index 23b8bdbed791d1b1dc171180f7a9d216598d4d17..0000000000000000000000000000000000000000 --- a/modules/text2vec-openai/vectorizer/texts_test.go +++ /dev/null @@ -1,159 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// as used in the nearText searcher -func TestVectorizingTexts(t *testing.T) { - type testCase struct { - name string - input []string - expectedOpenAIType string - openAIType string - expectedOpenAIModel string - openAIModel string - modelVersion string - expectedModelVersion string - } - - tests := []testCase{ - { - name: "single word", - input: []string{"hello"}, - openAIType: "text", - expectedOpenAIType: "text", - openAIModel: "ada", - expectedOpenAIModel: "ada", - - // use something that doesn't exist on purpose to rule out that this was - // set by a default, but validate that the version was set explicitly - // due to https://github.com/weaviate/weaviate/issues/2458 - modelVersion: "003", - expectedModelVersion: "003", - }, - { - name: "multiple words", - input: []string{"hello world, this is me!"}, - openAIType: "text", - expectedOpenAIType: "text", - openAIModel: "ada", - expectedOpenAIModel: "ada", - - // use something that doesn't exist on purpose to rule out that this was - // set by a default, but validate that the version was set explicitly - // due to https://github.com/weaviate/weaviate/issues/2458 - modelVersion: "003", - expectedModelVersion: "003", - }, - { - name: "multiple sentences (joined with a dot)", - input: []string{"this is sentence 1", "and here's number 2"}, - openAIType: "text", - expectedOpenAIType: "text", - openAIModel: "ada", - expectedOpenAIModel: "ada", - - // use something that doesn't exist on purpose to rule out that this was - // set by a default, but validate that the version was set explicitly - // due to https://github.com/weaviate/weaviate/issues/2458 - modelVersion: "003", - expectedModelVersion: "003", - }, - { - name: "multiple sentences already containing a dot", - input: []string{"this is sentence 1.", "and here's number 2"}, - openAIType: "text", - expectedOpenAIType: "text", - openAIModel: "ada", - expectedOpenAIModel: "ada", - - // use something that doesn't exist on purpose to rule out that this was - // set by a default, but validate that the version was set explicitly - // due to https://github.com/weaviate/weaviate/issues/2458 - modelVersion: "003", - expectedModelVersion: "003", - }, - { - name: "multiple sentences already containing a question mark", - input: []string{"this is sentence 1?", "and here's number 2"}, - openAIType: "text", - expectedOpenAIType: "text", - openAIModel: "ada", - expectedOpenAIModel: "ada", - - // use something that doesn't exist on purpose to rule out that this was - // set by a default, but validate that the version was set explicitly - // due to https://github.com/weaviate/weaviate/issues/2458 - modelVersion: "003", - expectedModelVersion: "003", - }, - { - name: "multiple sentences already containing an exclamation mark", - input: []string{"this is sentence 1!", "and here's number 2"}, - openAIType: "text", - expectedOpenAIType: "text", - openAIModel: "ada", - expectedOpenAIModel: "ada", - - // use something that doesn't exist on purpose to rule out that this was - // set by a default, but validate that the version was set explicitly - // due to https://github.com/weaviate/weaviate/issues/2458 - modelVersion: "003", - expectedModelVersion: "003", - }, - { - name: "multiple sentences already containing comma", - input: []string{"this is sentence 1,", "and here's number 2"}, - openAIType: "text", - expectedOpenAIType: "text", - openAIModel: "ada", - expectedOpenAIModel: "ada", - - // use something that doesn't exist on purpose to rule out that this was - // set by a default, but validate that the version was set explicitly - // due to https://github.com/weaviate/weaviate/issues/2458 - modelVersion: "003", - expectedModelVersion: "003", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - client := &fakeClient{} - - v := New(client) - - cfg := &fakeClassConfig{ - classConfig: map[string]interface{}{ - "type": test.openAIType, - "model": test.openAIModel, - "modelVersion": test.modelVersion, - }, - } - vec, err := v.Texts(context.Background(), test.input, cfg) - - require.Nil(t, err) - assert.Equal(t, []float32{0.1, 1.1, 2.1, 3.1}, vec) - assert.Equal(t, test.input, client.lastInput) - assert.Equal(t, client.lastConfig.Type, test.expectedOpenAIType) - assert.Equal(t, client.lastConfig.Model, test.expectedOpenAIModel) - assert.Equal(t, client.lastConfig.ModelVersion, test.expectedModelVersion) - }) - } -} diff --git a/modules/text2vec-palm/additional/arguments.go b/modules/text2vec-palm/additional/arguments.go deleted file mode 100644 index 42265496a2571efa3841fc51e17d6cf0d068348f..0000000000000000000000000000000000000000 --- a/modules/text2vec-palm/additional/arguments.go +++ /dev/null @@ -1,51 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package additional - -import ( - "fmt" - - "github.com/tailor-inc/graphql" -) - -func additionalFeatureProjectionField(classname string) *graphql.Field { - return &graphql.Field{ - Args: graphql.FieldConfigArgument{ - "algorithm": &graphql.ArgumentConfig{ - Type: graphql.String, - DefaultValue: nil, - }, - "dimensions": &graphql.ArgumentConfig{ - Type: graphql.Int, - DefaultValue: nil, - }, - "learningRate": &graphql.ArgumentConfig{ - Type: graphql.Int, - DefaultValue: nil, - }, - "iterations": &graphql.ArgumentConfig{ - Type: graphql.Int, - DefaultValue: nil, - }, - "perplexity": &graphql.ArgumentConfig{ - Type: graphql.Int, - DefaultValue: nil, - }, - }, - Type: graphql.NewObject(graphql.ObjectConfig{ - Name: fmt.Sprintf("%sAdditionalFeatureProjection", classname), - Fields: graphql.Fields{ - "vector": &graphql.Field{Type: graphql.NewList(graphql.Float)}, - }, - }), - } -} diff --git a/modules/text2vec-palm/additional/arguments_test.go b/modules/text2vec-palm/additional/arguments_test.go deleted file mode 100644 index 1c6140a7d94593dc05d3ddee1ac082ee45866cd0..0000000000000000000000000000000000000000 --- a/modules/text2vec-palm/additional/arguments_test.go +++ /dev/null @@ -1,55 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package additional - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/tailor-inc/graphql" -) - -func TestFeatureProjectionField(t *testing.T) { - t.Run("should generate featureProjection argument properly", func(t *testing.T) { - // given - classname := "Class" - - // when - featureProjection := additionalFeatureProjectionField(classname) - - // then - // the built graphQL field needs to support this structure: - // Args: { - // algorithm: "a", - // dimensions: 1, - // learningRate: 2, - // iterations: 3, - // perplexity: 4 - // } - // Type: { - // vector: [0, 1] - // } - assert.NotNil(t, featureProjection) - assert.Equal(t, "ClassAdditionalFeatureProjection", featureProjection.Type.Name()) - assert.NotNil(t, featureProjection.Args) - assert.Equal(t, 5, len(featureProjection.Args)) - assert.NotNil(t, featureProjection.Args["algorithm"]) - assert.NotNil(t, featureProjection.Args["dimensions"]) - assert.NotNil(t, featureProjection.Args["learningRate"]) - assert.NotNil(t, featureProjection.Args["iterations"]) - assert.NotNil(t, featureProjection.Args["perplexity"]) - featureProjectionObject, featureProjectionObjectOK := featureProjection.Type.(*graphql.Object) - assert.True(t, featureProjectionObjectOK) - assert.Equal(t, 1, len(featureProjectionObject.Fields())) - assert.NotNil(t, featureProjectionObject.Fields()["vector"]) - }) -} diff --git a/modules/text2vec-palm/additional/projector/projector.go b/modules/text2vec-palm/additional/projector/projector.go deleted file mode 100644 index 32e22f7278ea2bcda5ec1dcd35f372c3dff3521e..0000000000000000000000000000000000000000 --- a/modules/text2vec-palm/additional/projector/projector.go +++ /dev/null @@ -1,123 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package projector - -import ( - "context" - "fmt" - "time" - - "github.com/danaugrs/go-tsne/tsne" - "github.com/pkg/errors" - "github.com/tailor-inc/graphql/language/ast" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/search" - "gonum.org/v1/gonum/mat" -) - -type FeatureProjection struct { - Vector []float32 `json:"vector"` -} - -func New() *FeatureProjector { - return &FeatureProjector{ - fixedSeed: time.Now().UnixNano(), - } -} - -type FeatureProjector struct { - fixedSeed int64 -} - -func (f *FeatureProjector) AdditonalPropertyDefaultValue() interface{} { - return &Params{} -} - -func (f *FeatureProjector) AdditionalPropertyFn(ctx context.Context, - in []search.Result, params interface{}, limit *int, - argumentModuleParams map[string]interface{}, cfg moduletools.ClassConfig, -) ([]search.Result, error) { - if parameters, ok := params.(*Params); ok { - return f.Reduce(in, parameters) - } - return nil, errors.New("unknown params") -} - -func (f *FeatureProjector) ExtractAdditionalFn(param []*ast.Argument) interface{} { - return parseFeatureProjectionArguments(param) -} - -func (f *FeatureProjector) Reduce(in []search.Result, params *Params) ([]search.Result, error) { - if len(in) == 0 { - return nil, nil - } - - if params == nil { - return nil, fmt.Errorf("no params provided") - } - - dims := len(in[0].Vector) - - if err := params.SetDefaultsAndValidate(len(in), dims); err != nil { - return nil, errors.Wrap(err, "invalid params") - } - - matrix, err := f.vectorsToMatrix(in, dims, params) - if err != nil { - return nil, err - } - t := tsne.NewTSNE(*params.Dimensions, float64(*params.Perplexity), - float64(*params.LearningRate), *params.Iterations, false) - t.EmbedData(matrix, nil) - rows, cols := t.Y.Dims() - if rows != len(in) { - return nil, fmt.Errorf("incorrect matrix dimensions after t-SNE len %d != %d", len(in), rows) - } - - for i := 0; i < rows; i++ { - vector := make([]float32, cols) - for j := range vector { - vector[j] = float32(t.Y.At(i, j)) - } - up := in[i].AdditionalProperties - if up == nil { - up = models.AdditionalProperties{} - } - - up["featureProjection"] = &FeatureProjection{ - Vector: vector, - } - - in[i].AdditionalProperties = up - } - - return in, nil -} - -func (f *FeatureProjector) vectorsToMatrix(in []search.Result, dims int, params *Params) (*mat.Dense, error) { - items := len(in) - - // concat all vectors to build gonum dense matrix - mergedVectors := make([]float64, items*dims) - for i, obj := range in { - if l := len(obj.Vector); l != dims { - return nil, fmt.Errorf("inconsistent vector lengths found: %d and %d", dims, l) - } - - for j, dim := range obj.Vector { - mergedVectors[i*dims+j] = float64(dim) - } - } - - return mat.NewDense(len(in), dims, mergedVectors), nil -} diff --git a/modules/text2vec-palm/additional/projector/projector_params.go b/modules/text2vec-palm/additional/projector/projector_params.go deleted file mode 100644 index 5f9b91f42256086bfe96bf660bbb5fe5b8df6376..0000000000000000000000000000000000000000 --- a/modules/text2vec-palm/additional/projector/projector_params.go +++ /dev/null @@ -1,90 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package projector - -import "github.com/weaviate/weaviate/entities/errorcompounder" - -type Params struct { - Enabled bool - Algorithm *string // optional parameter - Dimensions *int // optional parameter - Perplexity *int // optional parameter - Iterations *int // optional parameter - LearningRate *int // optional parameter - IncludeNeighbors bool -} - -func (p *Params) SetDefaultsAndValidate(inputSize, dims int) error { - p.setDefaults(inputSize, dims) - return p.validate(inputSize, dims) -} - -func (p *Params) setDefaults(inputSize, dims int) { - perplexity := p.min(inputSize-1, 5) - p.Algorithm = p.optionalString(p.Algorithm, "tsne") - p.Dimensions = p.optionalInt(p.Dimensions, 2) - p.Perplexity = p.optionalInt(p.Perplexity, perplexity) - p.Iterations = p.optionalInt(p.Iterations, 100) - p.LearningRate = p.optionalInt(p.LearningRate, 25) -} - -func (p *Params) validate(inputSize, dims int) error { - ec := &errorcompounder.ErrorCompounder{} - if *p.Algorithm != "tsne" { - ec.Addf("algorithm %s is not supported: must be one of: tsne", *p.Algorithm) - } - - if *p.Perplexity >= inputSize { - ec.Addf("perplexity must be smaller than amount of items: %d >= %d", *p.Perplexity, inputSize) - } - - if *p.Iterations < 1 { - ec.Addf("iterations must be at least 1, got: %d", *p.Iterations) - } - - if *p.LearningRate < 1 { - ec.Addf("learningRate must be at least 1, got: %d", *p.LearningRate) - } - - if *p.Dimensions < 1 { - ec.Addf("dimensions must be at least 1, got: %d", *p.Dimensions) - } - - if *p.Dimensions >= dims { - ec.Addf("dimensions must be smaller than source dimensions: %d >= %d", *p.Dimensions, dims) - } - - return ec.ToError() -} - -func (p *Params) min(a, b int) int { - if a < b { - return a - } - return b -} - -func (p Params) optionalString(in *string, defaultValue string) *string { - if in == nil { - return &defaultValue - } - - return in -} - -func (p Params) optionalInt(in *int, defaultValue int) *int { - if in == nil { - return &defaultValue - } - - return in -} diff --git a/modules/text2vec-palm/additional/projector/projector_params_extractor.go b/modules/text2vec-palm/additional/projector/projector_params_extractor.go deleted file mode 100644 index a4c9fd8d0a4f0e4a8994a7e89051061a35f050e7..0000000000000000000000000000000000000000 --- a/modules/text2vec-palm/additional/projector/projector_params_extractor.go +++ /dev/null @@ -1,54 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package projector - -import ( - "strconv" - - "github.com/tailor-inc/graphql/language/ast" -) - -func parseFeatureProjectionArguments(args []*ast.Argument) *Params { - out := &Params{Enabled: true} - - for _, arg := range args { - switch arg.Name.Value { - case "dimensions": - asInt, _ := strconv.Atoi(arg.Value.GetValue().(string)) - out.Dimensions = ptInt(asInt) - case "iterations": - asInt, _ := strconv.Atoi(arg.Value.GetValue().(string)) - out.Iterations = ptInt(asInt) - case "learningRate": - asInt, _ := strconv.Atoi(arg.Value.GetValue().(string)) - out.LearningRate = ptInt(asInt) - case "perplexity": - asInt, _ := strconv.Atoi(arg.Value.GetValue().(string)) - out.Perplexity = ptInt(asInt) - case "algorithm": - out.Algorithm = ptString(arg.Value.GetValue().(string)) - - default: - // ignore what we don't recognize - } - } - - return out -} - -func ptString(in string) *string { - return &in -} - -func ptInt(in int) *int { - return &in -} diff --git a/modules/text2vec-palm/additional/projector/projector_params_extractor_test.go b/modules/text2vec-palm/additional/projector/projector_params_extractor_test.go deleted file mode 100644 index a1cb94deab72a6f4f823ab4768eb462297a8128f..0000000000000000000000000000000000000000 --- a/modules/text2vec-palm/additional/projector/projector_params_extractor_test.go +++ /dev/null @@ -1,144 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package projector - -import ( - "reflect" - "testing" - - "github.com/tailor-inc/graphql/language/ast" -) - -func Test_parseFeatureProjectionArguments(t *testing.T) { - type args struct { - args []*ast.Argument - } - tests := []struct { - name string - args args - want *Params - }{ - { - name: "Should create with no params", - args: args{ - args: []*ast.Argument{}, - }, - want: &Params{ - Enabled: true, - }, - }, - { - name: "Should create with all params", - args: args{ - args: []*ast.Argument{ - createArg("algorithm", "tsne"), - createArg("dimensions", "3"), - createArg("iterations", "100"), - createArg("learningRate", "15"), - createArg("perplexity", "10"), - }, - }, - want: &Params{ - Enabled: true, - Algorithm: ptString("tsne"), - Dimensions: ptInt(3), - Iterations: ptInt(100), - LearningRate: ptInt(15), - Perplexity: ptInt(10), - }, - }, - { - name: "Should create with only algorithm param", - args: args{ - args: []*ast.Argument{ - createArg("algorithm", "tsne"), - }, - }, - want: &Params{ - Enabled: true, - Algorithm: ptString("tsne"), - }, - }, - { - name: "Should create with only dimensions param", - args: args{ - args: []*ast.Argument{ - createArg("dimensions", "3"), - }, - }, - want: &Params{ - Enabled: true, - Dimensions: ptInt(3), - }, - }, - { - name: "Should create with only iterations param", - args: args{ - args: []*ast.Argument{ - createArg("iterations", "100"), - }, - }, - want: &Params{ - Enabled: true, - Iterations: ptInt(100), - }, - }, - { - name: "Should create with only learningRate param", - args: args{ - args: []*ast.Argument{ - createArg("learningRate", "15"), - }, - }, - want: &Params{ - Enabled: true, - LearningRate: ptInt(15), - }, - }, - { - name: "Should create with only perplexity param", - args: args{ - args: []*ast.Argument{ - createArg("perplexity", "10"), - }, - }, - want: &Params{ - Enabled: true, - Perplexity: ptInt(10), - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := parseFeatureProjectionArguments(tt.args.args); !reflect.DeepEqual(got, tt.want) { - t.Errorf("parseFeatureProjectionArguments() = %v, want %v", got, tt.want) - } - }) - } -} - -func createArg(name string, value string) *ast.Argument { - n := ast.Name{ - Value: name, - } - val := ast.StringValue{ - Kind: "Kind", - Value: value, - } - arg := ast.Argument{ - Name: ast.NewName(&n), - Kind: "Kind", - Value: ast.NewStringValue(&val), - } - a := ast.NewArgument(&arg) - return a -} diff --git a/modules/text2vec-palm/additional/projector/projector_params_test.go b/modules/text2vec-palm/additional/projector/projector_params_test.go deleted file mode 100644 index cac96a21197a32cd96306e623a1aa51b2a7dd3b1..0000000000000000000000000000000000000000 --- a/modules/text2vec-palm/additional/projector/projector_params_test.go +++ /dev/null @@ -1,126 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package projector - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestParams_validate(t *testing.T) { - type args struct { - inputSize int - dims int - } - tests := []struct { - name string - param *Params - args args - wantErr bool - errContains []string - }{ - { - name: "Should validate properly with default Params", - param: generateParamWithDefaultValues(1, 3), - args: args{ - inputSize: 1, - dims: 3, - }, - wantErr: false, - }, - { - name: "Should validate properly with default Params with higher inputs", - param: generateParamWithDefaultValues(100, 50), - args: args{ - inputSize: 100, - dims: 50, - }, - wantErr: false, - }, - { - name: "Should not validate - dimensions must be higher then 2", - param: generateParamWithDefaultValues(100, 2), - args: args{ - inputSize: 100, - dims: 2, - }, - wantErr: true, - }, - { - name: "Should not validate - with dimensions equal to 0", - param: generateParamWithValues(true, "tsne", 0, 5, 100, 25, true), - args: args{ - inputSize: 100, - dims: 2, - }, - wantErr: true, - errContains: []string{ - "dimensions must be at least 1, got: 0", - }, - }, - { - name: "Should not validate - with all wrong values", - param: generateParamWithValues(true, "unknown", 5, 5, 0, 0, true), - args: args{ - inputSize: 4, - dims: 2, - }, - wantErr: true, - errContains: []string{ - "algorithm unknown is not supported: must be one of: tsne", - "perplexity must be smaller than amount of items: 5 >= 4", - "iterations must be at least 1, got: 0", - "learningRate must be at least 1, got: 0", - "dimensions must be smaller than source dimensions: 5 >= 2", - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - p := tt.param - err := p.validate(tt.args.inputSize, tt.args.dims) - if (err != nil) != tt.wantErr { - t.Errorf("Params.validate() error = %v, wantErr %v", err, tt.wantErr) - } - if tt.wantErr && len(tt.errContains) > 0 { - for _, containsString := range tt.errContains { - assert.Contains(t, err.Error(), containsString) - } - } - }) - } -} - -func generateParamWithDefaultValues(inputSize, dims int) *Params { - p := &Params{} - p.setDefaults(inputSize, dims) - return p -} - -func generateParamWithValues( - enabled bool, - algorithm string, - dims, perplexity, iterations, learningRate int, - includeNeighbors bool, -) *Params { - p := &Params{ - Enabled: enabled, - Algorithm: &algorithm, - Dimensions: &dims, - Perplexity: &perplexity, - Iterations: &iterations, - LearningRate: &learningRate, - IncludeNeighbors: includeNeighbors, - } - return p -} diff --git a/modules/text2vec-palm/additional/projector/projector_test.go b/modules/text2vec-palm/additional/projector/projector_test.go deleted file mode 100644 index ac1e54b478c0e9e2d90b5be1729c3fee49368efa..0000000000000000000000000000000000000000 --- a/modules/text2vec-palm/additional/projector/projector_test.go +++ /dev/null @@ -1,71 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package projector - -import ( - "testing" - - "github.com/go-openapi/strfmt" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/additional" - "github.com/weaviate/weaviate/entities/search" -) - -func TestProjector(t *testing.T) { - p := New() - - t.Run("with multiple results", func(t *testing.T) { - vectors := [][]float32{ - {1, 0, 0, 0, 0}, - {0, 0, 1, 0, 0}, - {1, 1, 1, 0, 0}, - } - - testData := []search.Result{ - { - Schema: map[string]interface{}{"name": "item1"}, - Vector: vectors[0], - }, - { - Schema: map[string]interface{}{"name": "item2"}, - Vector: vectors[1], - }, - { - Schema: map[string]interface{}{"name": "item3"}, - Vector: vectors[2], - AdditionalProperties: map[string]interface{}{ - "classification": &additional.Classification{ // verify it doesn't remove existing additional props - ID: strfmt.UUID("123"), - }, - }, - }, - } - - res, err := p.Reduce(testData, &Params{}) - require.Nil(t, err) - assert.Len(t, res, len(testData)) - classification, classificationOK := res[2].AdditionalProperties["classification"] - assert.True(t, classificationOK) - classificationElement, classificationElementOK := classification.(*additional.Classification) - assert.True(t, classificationElementOK) - assert.Equal(t, classificationElement.ID, strfmt.UUID("123"), - "existing additionals should not be removed") - for i := 0; i < 3; i++ { - featureProjection, featureProjectionOK := res[i].AdditionalProperties["featureProjection"] - assert.True(t, featureProjectionOK) - fpElement, fpElementOK := featureProjection.(*FeatureProjection) - assert.True(t, fpElementOK) - assert.Len(t, fpElement.Vector, 2) - } - }) -} diff --git a/modules/text2vec-palm/additional/provider.go b/modules/text2vec-palm/additional/provider.go deleted file mode 100644 index e5af5bfd526a7f16e29c40156b2168f049683e71..0000000000000000000000000000000000000000 --- a/modules/text2vec-palm/additional/provider.go +++ /dev/null @@ -1,51 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package additional - -import ( - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/modules/text2vec-palm/additional/projector" -) - -type GraphQLAdditionalArgumentsProvider struct { - projector *projector.FeatureProjector -} - -func New(projector *projector.FeatureProjector) *GraphQLAdditionalArgumentsProvider { - return &GraphQLAdditionalArgumentsProvider{projector} -} - -func (p *GraphQLAdditionalArgumentsProvider) AdditionalProperties() map[string]modulecapabilities.AdditionalProperty { - additionalProperties := map[string]modulecapabilities.AdditionalProperty{} - additionalProperties["featureProjection"] = p.getFeatureProjection() - return additionalProperties -} - -func (p *GraphQLAdditionalArgumentsProvider) getFeatureProjection() modulecapabilities.AdditionalProperty { - return modulecapabilities.AdditionalProperty{ - RestNames: []string{ - "featureProjection", - "featureprojection", - "feature-projection", - "feature_projection", - }, - DefaultValue: p.projector.AdditonalPropertyDefaultValue(), - GraphQLNames: []string{"featureProjection"}, - GraphQLFieldFunction: additionalFeatureProjectionField, - GraphQLExtractFunction: p.projector.ExtractAdditionalFn, - SearchFunctions: modulecapabilities.AdditionalSearch{ - ObjectList: p.projector.AdditionalPropertyFn, - ExploreGet: p.projector.AdditionalPropertyFn, - ExploreList: p.projector.AdditionalPropertyFn, - }, - } -} diff --git a/modules/text2vec-palm/clients/meta.go b/modules/text2vec-palm/clients/meta.go deleted file mode 100644 index 8db4aea7fbeb93b144b3a4310422a8743fc382e3..0000000000000000000000000000000000000000 --- a/modules/text2vec-palm/clients/meta.go +++ /dev/null @@ -1,19 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -func (v *palm) MetaInfo() (map[string]interface{}, error) { - return map[string]interface{}{ - "name": "Google PaLM Module", - "documentationHref": "https://cloud.google.com/vertex-ai/docs/generative-ai/embeddings/get-text-embeddings", - }, nil -} diff --git a/modules/text2vec-palm/clients/palm.go b/modules/text2vec-palm/clients/palm.go deleted file mode 100644 index 87a64db830bd0a440c4ec59f7d45be502ecbb751..0000000000000000000000000000000000000000 --- a/modules/text2vec-palm/clients/palm.go +++ /dev/null @@ -1,309 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "strings" - "time" - - "github.com/weaviate/weaviate/usecases/modulecomponents" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/modules/text2vec-palm/ent" -) - -type taskType string - -var ( - // Specifies the given text is a document in a search/retrieval setting - retrievalQuery taskType = "RETRIEVAL_QUERY" - // Specifies the given text is a query in a search/retrieval setting - retrievalDocument taskType = "RETRIEVAL_DOCUMENT" -) - -func buildURL(useGenerativeAI bool, apiEndoint, projectID, modelID string) string { - if useGenerativeAI { - // Generative AI supports only 1 embedding model: embedding-gecko-001. So for now - // in order to keep it simple we generate one variation of PaLM API url. - // For more context check out this link: - // https://developers.generativeai.google/models/language#model_variations - return "https://generativelanguage.googleapis.com/v1beta3/models/embedding-gecko-001:batchEmbedText" - } - urlTemplate := "https://%s/v1/projects/%s/locations/us-central1/publishers/google/models/%s:predict" - return fmt.Sprintf(urlTemplate, apiEndoint, projectID, modelID) -} - -type palm struct { - apiKey string - httpClient *http.Client - urlBuilderFn func(useGenerativeAI bool, apiEndoint, projectID, modelID string) string - logger logrus.FieldLogger -} - -func New(apiKey string, timeout time.Duration, logger logrus.FieldLogger) *palm { - return &palm{ - apiKey: apiKey, - httpClient: &http.Client{ - Timeout: timeout, - }, - urlBuilderFn: buildURL, - logger: logger, - } -} - -func (v *palm) Vectorize(ctx context.Context, input []string, - config ent.VectorizationConfig, titlePropertyValue string, -) (*ent.VectorizationResult, error) { - return v.vectorize(ctx, input, retrievalDocument, titlePropertyValue, config) -} - -func (v *palm) VectorizeQuery(ctx context.Context, input []string, - config ent.VectorizationConfig, -) (*ent.VectorizationResult, error) { - return v.vectorize(ctx, input, retrievalQuery, "", config) -} - -func (v *palm) vectorize(ctx context.Context, input []string, taskType taskType, - titlePropertyValue string, config ent.VectorizationConfig, -) (*ent.VectorizationResult, error) { - useGenerativeAIEndpoint := v.useGenerativeAIEndpoint(config) - - payload := v.getPayload(useGenerativeAIEndpoint, input, taskType, titlePropertyValue, config) - body, err := json.Marshal(payload) - if err != nil { - return nil, errors.Wrapf(err, "marshal body") - } - - endpointURL := v.urlBuilderFn(useGenerativeAIEndpoint, - v.getApiEndpoint(config), v.getProjectID(config), v.getModel(config)) - - req, err := http.NewRequestWithContext(ctx, "POST", endpointURL, - bytes.NewReader(body)) - if err != nil { - return nil, errors.Wrap(err, "create POST request") - } - - apiKey, err := v.getApiKey(ctx) - if err != nil { - return nil, errors.Wrapf(err, "Palm API Key") - } - req.Header.Add("Content-Type", "application/json") - if useGenerativeAIEndpoint { - req.Header.Add("x-goog-api-key", apiKey) - } else { - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", apiKey)) - } - - res, err := v.httpClient.Do(req) - if err != nil { - return nil, errors.Wrap(err, "send POST request") - } - defer res.Body.Close() - - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - return nil, errors.Wrap(err, "read response body") - } - - if useGenerativeAIEndpoint { - return v.parseGenerativeAIApiResponse(res.StatusCode, bodyBytes, input) - } - return v.parseEmbeddingsResponse(res.StatusCode, bodyBytes, input) -} - -func (v *palm) useGenerativeAIEndpoint(config ent.VectorizationConfig) bool { - return v.getApiEndpoint(config) == "generativelanguage.googleapis.com" -} - -func (v *palm) getPayload(useGenerativeAI bool, input []string, - taskType taskType, title string, config ent.VectorizationConfig, -) interface{} { - if useGenerativeAI { - return batchEmbedTextRequest{Texts: input} - } - isModelVersion001 := strings.HasSuffix(config.Model, "@001") - instances := make([]instance, len(input)) - for i := range input { - if isModelVersion001 { - instances[i] = instance{Content: input[i]} - } else { - instances[i] = instance{Content: input[i], TaskType: taskType, Title: title} - } - } - return embeddingsRequest{instances} -} - -func (v *palm) checkResponse(statusCode int, palmApiError *palmApiError) error { - if statusCode != 200 || palmApiError != nil { - if palmApiError != nil { - return fmt.Errorf("connection to Google PaLM failed with status: %v error: %v", - statusCode, palmApiError.Message) - } - return fmt.Errorf("connection to Google PaLM failed with status: %d", statusCode) - } - return nil -} - -func (v *palm) getApiKey(ctx context.Context) (string, error) { - if apiKeyValue := v.getValueFromContext(ctx, "X-Palm-Api-Key"); apiKeyValue != "" { - return apiKeyValue, nil - } - if len(v.apiKey) > 0 { - return v.apiKey, nil - } - return "", errors.New("no api key found " + - "neither in request header: X-Palm-Api-Key " + - "nor in environment variable under PALM_APIKEY") -} - -func (v *palm) getValueFromContext(ctx context.Context, key string) string { - if value := ctx.Value(key); value != nil { - if keyHeader, ok := value.([]string); ok && len(keyHeader) > 0 && len(keyHeader[0]) > 0 { - return keyHeader[0] - } - } - // try getting header from GRPC if not successful - if apiKey := modulecomponents.GetValueFromGRPC(ctx, key); len(apiKey) > 0 && len(apiKey[0]) > 0 { - return apiKey[0] - } - return "" -} - -func (v *palm) parseGenerativeAIApiResponse(statusCode int, - bodyBytes []byte, input []string, -) (*ent.VectorizationResult, error) { - var resBody batchEmbedTextResponse - if err := json.Unmarshal(bodyBytes, &resBody); err != nil { - return nil, errors.Wrap(err, "unmarshal response body") - } - - if err := v.checkResponse(statusCode, resBody.Error); err != nil { - return nil, err - } - - if len(resBody.Embeddings) == 0 { - return nil, errors.Errorf("empty embeddings response") - } - - vectors := make([][]float32, len(resBody.Embeddings)) - for i := range resBody.Embeddings { - vectors[i] = resBody.Embeddings[i].Value - } - dimensions := len(resBody.Embeddings[0].Value) - - return v.getResponse(input, dimensions, vectors) -} - -func (v *palm) parseEmbeddingsResponse(statusCode int, - bodyBytes []byte, input []string, -) (*ent.VectorizationResult, error) { - var resBody embeddingsResponse - if err := json.Unmarshal(bodyBytes, &resBody); err != nil { - return nil, errors.Wrap(err, "unmarshal response body") - } - - if err := v.checkResponse(statusCode, resBody.Error); err != nil { - return nil, err - } - - if len(resBody.Predictions) == 0 { - return nil, errors.Errorf("empty embeddings response") - } - - vectors := make([][]float32, len(resBody.Predictions)) - for i := range resBody.Predictions { - vectors[i] = resBody.Predictions[i].Embeddings.Values - } - dimensions := len(resBody.Predictions[0].Embeddings.Values) - - return v.getResponse(input, dimensions, vectors) -} - -func (v *palm) getResponse(input []string, dimensions int, vectors [][]float32) (*ent.VectorizationResult, error) { - return &ent.VectorizationResult{ - Texts: input, - Dimensions: dimensions, - Vectors: vectors, - }, nil -} - -func (v *palm) getApiEndpoint(config ent.VectorizationConfig) string { - return config.ApiEndpoint -} - -func (v *palm) getProjectID(config ent.VectorizationConfig) string { - return config.ProjectID -} - -func (v *palm) getModel(config ent.VectorizationConfig) string { - return config.Model -} - -type embeddingsRequest struct { - Instances []instance `json:"instances,omitempty"` -} - -type instance struct { - Content string `json:"content"` - TaskType taskType `json:"task_type,omitempty"` - Title string `json:"title,omitempty"` -} - -type embeddingsResponse struct { - Predictions []prediction `json:"predictions,omitempty"` - Error *palmApiError `json:"error,omitempty"` - DeployedModelId string `json:"deployedModelId,omitempty"` - Model string `json:"model,omitempty"` - ModelDisplayName string `json:"modelDisplayName,omitempty"` - ModelVersionId string `json:"modelVersionId,omitempty"` -} - -type prediction struct { - Embeddings embeddings `json:"embeddings,omitempty"` - SafetyAttributes *safetyAttributes `json:"safetyAttributes,omitempty"` -} - -type embeddings struct { - Values []float32 `json:"values,omitempty"` -} - -type safetyAttributes struct { - Scores []float64 `json:"scores,omitempty"` - Blocked *bool `json:"blocked,omitempty"` - Categories []string `json:"categories,omitempty"` -} - -type palmApiError struct { - Code int `json:"code"` - Message string `json:"message"` - Status string `json:"status"` -} - -type batchEmbedTextRequest struct { - Texts []string `json:"texts,omitempty"` -} - -type batchEmbedTextResponse struct { - Embeddings []embedding `json:"embeddings,omitempty"` - Error *palmApiError `json:"error,omitempty"` -} - -type embedding struct { - Value []float32 `json:"value,omitempty"` -} diff --git a/modules/text2vec-palm/clients/palm_test.go b/modules/text2vec-palm/clients/palm_test.go deleted file mode 100644 index 9a95cce6d341776993cdeabfb58cc5e1b172fd6f..0000000000000000000000000000000000000000 --- a/modules/text2vec-palm/clients/palm_test.go +++ /dev/null @@ -1,228 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "context" - "encoding/json" - "io" - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/modules/text2vec-palm/ent" -) - -func TestClient(t *testing.T) { - t.Run("when all is fine", func(t *testing.T) { - server := httptest.NewServer(&fakeHandler{t: t}) - defer server.Close() - c := &palm{ - apiKey: "apiKey", - httpClient: &http.Client{}, - urlBuilderFn: func(useGenerativeAI bool, apiEndoint, projectID, modelID string) string { - assert.Equal(t, "endpoint", apiEndoint) - assert.Equal(t, "project", projectID) - assert.Equal(t, "model", modelID) - return server.URL - }, - logger: nullLogger(), - } - expected := &ent.VectorizationResult{ - Texts: []string{"This is my text"}, - Vectors: [][]float32{{0.1, 0.2, 0.3}}, - Dimensions: 3, - } - res, err := c.Vectorize(context.Background(), []string{"This is my text"}, - ent.VectorizationConfig{ - ApiEndpoint: "endpoint", - ProjectID: "project", - Model: "model", - }, "") - - assert.Nil(t, err) - assert.Equal(t, expected, res) - }) - - t.Run("when the context is expired", func(t *testing.T) { - server := httptest.NewServer(&fakeHandler{t: t}) - defer server.Close() - c := &palm{ - apiKey: "apiKey", - httpClient: &http.Client{}, - urlBuilderFn: func(useGenerativeAI bool, apiEndoint, projectID, modelID string) string { - return server.URL - }, - logger: nullLogger(), - } - ctx, cancel := context.WithDeadline(context.Background(), time.Now()) - defer cancel() - - _, err := c.Vectorize(ctx, []string{"This is my text"}, ent.VectorizationConfig{}, "") - - require.NotNil(t, err) - assert.Contains(t, err.Error(), "context deadline exceeded") - }) - - t.Run("when the server returns an error", func(t *testing.T) { - server := httptest.NewServer(&fakeHandler{ - t: t, - serverError: errors.Errorf("nope, not gonna happen"), - }) - defer server.Close() - c := &palm{ - apiKey: "apiKey", - httpClient: &http.Client{}, - urlBuilderFn: func(useGenerativeAI bool, apiEndoint, projectID, modelID string) string { - return server.URL - }, - logger: nullLogger(), - } - _, err := c.Vectorize(context.Background(), []string{"This is my text"}, - ent.VectorizationConfig{}, "") - - require.NotNil(t, err) - assert.EqualError(t, err, "connection to Google PaLM failed with status: 500 error: nope, not gonna happen") - }) - - t.Run("when Palm key is passed using X-Palm-Api-Key header", func(t *testing.T) { - server := httptest.NewServer(&fakeHandler{t: t}) - defer server.Close() - c := &palm{ - apiKey: "", - httpClient: &http.Client{}, - urlBuilderFn: func(useGenerativeAI bool, apiEndoint, projectID, modelID string) string { - return server.URL - }, - logger: nullLogger(), - } - ctxWithValue := context.WithValue(context.Background(), - "X-Palm-Api-Key", []string{"some-key"}) - - expected := &ent.VectorizationResult{ - Texts: []string{"This is my text"}, - Vectors: [][]float32{{0.1, 0.2, 0.3}}, - Dimensions: 3, - } - res, err := c.Vectorize(ctxWithValue, []string{"This is my text"}, ent.VectorizationConfig{}, "") - - require.Nil(t, err) - assert.Equal(t, expected, res) - }) - - t.Run("when Palm key is empty", func(t *testing.T) { - server := httptest.NewServer(&fakeHandler{t: t}) - defer server.Close() - c := &palm{ - apiKey: "", - httpClient: &http.Client{}, - urlBuilderFn: func(useGenerativeAI bool, apiEndoint, projectID, modelID string) string { - return server.URL - }, - logger: nullLogger(), - } - ctx, cancel := context.WithDeadline(context.Background(), time.Now()) - defer cancel() - - _, err := c.Vectorize(ctx, []string{"This is my text"}, ent.VectorizationConfig{}, "") - - require.NotNil(t, err) - assert.Equal(t, err.Error(), "Palm API Key: no api key found "+ - "neither in request header: X-Palm-Api-Key "+ - "nor in environment variable under PALM_APIKEY") - }) - - t.Run("when X-Palm-Api-Key header is passed but empty", func(t *testing.T) { - server := httptest.NewServer(&fakeHandler{t: t}) - defer server.Close() - c := &palm{ - apiKey: "", - httpClient: &http.Client{}, - urlBuilderFn: buildURL, - logger: nullLogger(), - } - ctxWithValue := context.WithValue(context.Background(), - "X-Palm-Api-Key", []string{""}) - - _, err := c.Vectorize(ctxWithValue, []string{"This is my text"}, ent.VectorizationConfig{}, "") - - require.NotNil(t, err) - assert.Equal(t, err.Error(), "Palm API Key: no api key found "+ - "neither in request header: X-Palm-Api-Key "+ - "nor in environment variable under PALM_APIKEY") - }) -} - -type fakeHandler struct { - t *testing.T - serverError error -} - -func (f *fakeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - assert.Equal(f.t, http.MethodPost, r.Method) - - if f.serverError != nil { - embeddingResponse := &embeddingsResponse{ - Error: &palmApiError{ - Code: http.StatusInternalServerError, - Status: "error", - Message: f.serverError.Error(), - }, - } - - outBytes, err := json.Marshal(embeddingResponse) - require.Nil(f.t, err) - - w.WriteHeader(http.StatusInternalServerError) - w.Write(outBytes) - return - } - - bodyBytes, err := io.ReadAll(r.Body) - require.Nil(f.t, err) - defer r.Body.Close() - - var req embeddingsRequest - require.Nil(f.t, json.Unmarshal(bodyBytes, &req)) - - require.NotNil(f.t, req) - require.Len(f.t, req.Instances, 1) - - textInput := req.Instances[0].Content - assert.Greater(f.t, len(textInput), 0) - - embeddingResponse := &embeddingsResponse{ - Predictions: []prediction{ - { - Embeddings: embeddings{ - Values: []float32{0.1, 0.2, 0.3}, - }, - }, - }, - } - - outBytes, err := json.Marshal(embeddingResponse) - require.Nil(f.t, err) - - w.Write(outBytes) -} - -func nullLogger() logrus.FieldLogger { - l, _ := test.NewNullLogger() - return l -} diff --git a/modules/text2vec-palm/config.go b/modules/text2vec-palm/config.go deleted file mode 100644 index ff2458a706f0522a72c16cf0122f302db8e2ad1a..0000000000000000000000000000000000000000 --- a/modules/text2vec-palm/config.go +++ /dev/null @@ -1,47 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modpalm - -import ( - "context" - - "github.com/weaviate/weaviate/modules/text2vec-palm/config" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/schema" -) - -func (m *PalmModule) ClassConfigDefaults() map[string]interface{} { - return map[string]interface{}{ - "vectorizeClassName": config.DefaultVectorizeClassName, - } -} - -func (m *PalmModule) PropertyConfigDefaults( - dt *schema.DataType, -) map[string]interface{} { - return map[string]interface{}{ - "skip": !config.DefaultPropertyIndexed, - "vectorizePropertyName": config.DefaultVectorizePropertyName, - } -} - -func (m *PalmModule) ValidateClass(ctx context.Context, - class *models.Class, cfg moduletools.ClassConfig, -) error { - settings := config.NewClassSettings(cfg) - return settings.Validate(class) -} - -var _ = modulecapabilities.ClassConfigurator(New()) diff --git a/modules/text2vec-palm/config/class_settings.go b/modules/text2vec-palm/config/class_settings.go deleted file mode 100644 index b6fce71399c74e2342ea252a67d3081b4db73ba1..0000000000000000000000000000000000000000 --- a/modules/text2vec-palm/config/class_settings.go +++ /dev/null @@ -1,242 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package config - -import ( - "fmt" - "strings" - - "github.com/weaviate/weaviate/modules/text2vec-palm/vectorizer" - - "github.com/pkg/errors" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/schema" -) - -const ( - apiEndpointProperty = "apiEndpoint" - projectIDProperty = "projectId" - modelIDProperty = "modelId" - titleProperty = "titleProperty" -) - -const ( - DefaultVectorizeClassName = false - DefaultPropertyIndexed = true - DefaultVectorizePropertyName = false - DefaultApiEndpoint = "us-central1-aiplatform.googleapis.com" - DefaultModelID = "textembedding-gecko@001" - DefaulGenerativeAIApiEndpoint = "generativelanguage.googleapis.com" - DefaulGenerativeAIModelID = "embedding-gecko-001" -) - -var availablePalmModels = []string{ - DefaultModelID, - "textembedding-gecko@latest", - "textembedding-gecko-multilingual@latest", - "textembedding-gecko@003", - "textembedding-gecko@002", - "textembedding-gecko-multilingual@001", - "textembedding-gecko@001", -} - -var availableGenerativeAIModels = []string{ - DefaulGenerativeAIModelID, -} - -type classSettings struct { - cfg moduletools.ClassConfig -} - -func NewClassSettings(cfg moduletools.ClassConfig) *classSettings { - return &classSettings{cfg: cfg} -} - -func (ic *classSettings) PropertyIndexed(propName string) bool { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return DefaultPropertyIndexed - } - - vcn, ok := ic.cfg.Property(propName)["skip"] - if !ok { - return DefaultPropertyIndexed - } - - asBool, ok := vcn.(bool) - if !ok { - return DefaultPropertyIndexed - } - - return !asBool -} - -func (ic *classSettings) VectorizePropertyName(propName string) bool { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return DefaultVectorizePropertyName - } - vcn, ok := ic.cfg.Property(propName)["vectorizePropertyName"] - if !ok { - return DefaultVectorizePropertyName - } - - asBool, ok := vcn.(bool) - if !ok { - return DefaultVectorizePropertyName - } - - return asBool -} - -func (ic *classSettings) VectorizeClassName() bool { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return DefaultVectorizeClassName - } - - vcn, ok := ic.cfg.Class()["vectorizeClassName"] - if !ok { - return DefaultVectorizeClassName - } - - asBool, ok := vcn.(bool) - if !ok { - return DefaultVectorizeClassName - } - - return asBool -} - -func (ic *classSettings) Validate(class *models.Class) error { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return errors.New("empty config") - } - - var errorMessages []string - - apiEndpoint := ic.ApiEndpoint() - model := ic.ModelID() - if apiEndpoint == DefaulGenerativeAIApiEndpoint { - if model != "" && !ic.validatePalmSetting(model, availableGenerativeAIModels) { - errorMessages = append(errorMessages, fmt.Sprintf("wrong %s available Generative AI model names are: %v", modelIDProperty, availableGenerativeAIModels)) - } - } else { - projectID := ic.ProjectID() - if projectID == "" { - errorMessages = append(errorMessages, fmt.Sprintf("%s cannot be empty", projectIDProperty)) - } - if model != "" && !ic.validatePalmSetting(model, availablePalmModels) { - errorMessages = append(errorMessages, fmt.Sprintf("wrong %s available model names are: %v", modelIDProperty, availablePalmModels)) - } - } - - if len(errorMessages) > 0 { - return fmt.Errorf("%s", strings.Join(errorMessages, ", ")) - } - - err := ic.validateIndexState(class, ic) - if err != nil { - return err - } - - return nil -} - -func (ic *classSettings) validatePalmSetting(value string, availableValues []string) bool { - for i := range availableValues { - if value == availableValues[i] { - return true - } - } - return false -} - -func (ic *classSettings) getStringProperty(name, defaultValue string) string { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return defaultValue - } - - value, ok := ic.cfg.ClassByModuleName("text2vec-palm")[name] - if ok { - asString, ok := value.(string) - if ok { - return asString - } - } - return defaultValue -} - -func (cv *classSettings) validateIndexState(class *models.Class, settings vectorizer.ClassSettings) error { - if settings.VectorizeClassName() { - // if the user chooses to vectorize the classname, vector-building will - // always be possible, no need to investigate further - - return nil - } - - // search if there is at least one indexed, string/text prop. If found pass - // validation - for _, prop := range class.Properties { - if len(prop.DataType) < 1 { - return errors.Errorf("property %s must have at least one datatype: "+ - "got %v", prop.Name, prop.DataType) - } - - if prop.DataType[0] != string(schema.DataTypeString) && - prop.DataType[0] != string(schema.DataTypeText) { - // we can only vectorize text-like props - continue - } - - if settings.PropertyIndexed(prop.Name) { - // found at least one, this is a valid schema - return nil - } - } - - return fmt.Errorf("invalid properties: didn't find a single property which is " + - "of type string or text and is not excluded from indexing. In addition the " + - "class name is excluded from vectorization as well, meaning that it cannot be " + - "used to determine the vector position. To fix this, set 'vectorizeClassName' " + - "to true if the class name is contextionary-valid. Alternatively add at least " + - "contextionary-valid text/string property which is not excluded from " + - "indexing") -} - -func (ic *classSettings) getDefaultModel(apiEndpoint string) string { - if apiEndpoint == DefaulGenerativeAIApiEndpoint { - return DefaulGenerativeAIModelID - } - return DefaultModelID -} - -// PaLM params -func (ic *classSettings) ApiEndpoint() string { - return ic.getStringProperty(apiEndpointProperty, DefaultApiEndpoint) -} - -func (ic *classSettings) ProjectID() string { - return ic.getStringProperty(projectIDProperty, "") -} - -func (ic *classSettings) ModelID() string { - return ic.getStringProperty(modelIDProperty, ic.getDefaultModel(ic.ApiEndpoint())) -} - -func (ic *classSettings) TitleProperty() string { - return ic.getStringProperty(titleProperty, "") -} diff --git a/modules/text2vec-palm/config/class_settings_test.go b/modules/text2vec-palm/config/class_settings_test.go deleted file mode 100644 index dba83a1e918fe0982324db40a2cabbf3f7ea702b..0000000000000000000000000000000000000000 --- a/modules/text2vec-palm/config/class_settings_test.go +++ /dev/null @@ -1,164 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package config - -import ( - "testing" - - "github.com/pkg/errors" - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/entities/moduletools" -) - -func Test_classSettings_Validate(t *testing.T) { - tests := []struct { - name string - cfg moduletools.ClassConfig - wantApiEndpoint string - wantProjectID string - wantModelID string - wantTitle string - wantErr error - }{ - { - name: "happy flow", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "projectId": "projectId", - }, - }, - wantApiEndpoint: "us-central1-aiplatform.googleapis.com", - wantProjectID: "projectId", - wantModelID: "textembedding-gecko@001", - wantErr: nil, - }, - { - name: "custom values", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "apiEndpoint": "google.com", - "projectId": "projectId", - "titleProperty": "title", - }, - }, - wantApiEndpoint: "google.com", - wantProjectID: "projectId", - wantModelID: "textembedding-gecko@001", - wantTitle: "title", - wantErr: nil, - }, - { - name: "empty projectId", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "projectId": "", - }, - }, - wantErr: errors.Errorf("projectId cannot be empty"), - }, - { - name: "wrong modelId", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "projectId": "projectId", - "modelId": "wrong-model", - }, - }, - wantErr: errors.Errorf("wrong modelId available model names are: " + - "[textembedding-gecko@001 textembedding-gecko@latest " + - "textembedding-gecko-multilingual@latest textembedding-gecko@003 " + - "textembedding-gecko@002 textembedding-gecko-multilingual@001 textembedding-gecko@001]"), - }, - { - name: "all wrong", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "projectId": "", - "modelId": "wrong-model", - }, - }, - wantErr: errors.Errorf("projectId cannot be empty, " + - "wrong modelId available model names are: " + - "[textembedding-gecko@001 textembedding-gecko@latest " + - "textembedding-gecko-multilingual@latest textembedding-gecko@003 " + - "textembedding-gecko@002 textembedding-gecko-multilingual@001 textembedding-gecko@001]"), - }, - { - name: "Generative AI", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "apiEndpoint": "generativelanguage.googleapis.com", - }, - }, - wantApiEndpoint: "generativelanguage.googleapis.com", - wantProjectID: "", - wantModelID: "embedding-gecko-001", - wantErr: nil, - }, - { - name: "Generative AI with model", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "apiEndpoint": "generativelanguage.googleapis.com", - "modelId": "embedding-gecko-001", - }, - }, - wantApiEndpoint: "generativelanguage.googleapis.com", - wantProjectID: "", - wantModelID: "embedding-gecko-001", - wantErr: nil, - }, - { - name: "Generative AI with wrong model", - cfg: fakeClassConfig{ - classConfig: map[string]interface{}{ - "apiEndpoint": "generativelanguage.googleapis.com", - "modelId": "textembedding-gecko@001", - }, - }, - wantErr: errors.Errorf("wrong modelId available Generative AI model names are: [embedding-gecko-001]"), - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ic := NewClassSettings(tt.cfg) - if tt.wantErr != nil { - assert.EqualError(t, ic.Validate(nil), tt.wantErr.Error()) - } else { - assert.Equal(t, tt.wantApiEndpoint, ic.ApiEndpoint()) - assert.Equal(t, tt.wantProjectID, ic.ProjectID()) - assert.Equal(t, tt.wantModelID, ic.ModelID()) - assert.Equal(t, tt.wantTitle, ic.TitleProperty()) - } - }) - } -} - -type fakeClassConfig struct { - classConfig map[string]interface{} -} - -func (f fakeClassConfig) Class() map[string]interface{} { - return f.classConfig -} - -func (f fakeClassConfig) Tenant() string { - return "" -} - -func (f fakeClassConfig) ClassByModuleName(moduleName string) map[string]interface{} { - return f.classConfig -} - -func (f fakeClassConfig) Property(propName string) map[string]interface{} { - return nil -} diff --git a/modules/text2vec-palm/ent/vectorization_config.go b/modules/text2vec-palm/ent/vectorization_config.go deleted file mode 100644 index f031715eaec4bdf2209473a5d6e676eae53b5f7e..0000000000000000000000000000000000000000 --- a/modules/text2vec-palm/ent/vectorization_config.go +++ /dev/null @@ -1,18 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package ent - -type VectorizationConfig struct { - ApiEndpoint string - ProjectID string - Model string -} diff --git a/modules/text2vec-palm/ent/vectorization_result.go b/modules/text2vec-palm/ent/vectorization_result.go deleted file mode 100644 index b984aa4da4dc5b7fdade522487a9ad713dd8a479..0000000000000000000000000000000000000000 --- a/modules/text2vec-palm/ent/vectorization_result.go +++ /dev/null @@ -1,18 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package ent - -type VectorizationResult struct { - Texts []string - Dimensions int - Vectors [][]float32 -} diff --git a/modules/text2vec-palm/module.go b/modules/text2vec-palm/module.go deleted file mode 100644 index cef736318bf6199b711d15e3c2a8fe62e2abab79..0000000000000000000000000000000000000000 --- a/modules/text2vec-palm/module.go +++ /dev/null @@ -1,157 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modpalm - -import ( - "context" - "net/http" - "os" - "time" - - "github.com/weaviate/weaviate/modules/text2vec-palm/config" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/modules/text2vec-palm/additional" - "github.com/weaviate/weaviate/modules/text2vec-palm/additional/projector" - "github.com/weaviate/weaviate/modules/text2vec-palm/clients" - "github.com/weaviate/weaviate/modules/text2vec-palm/vectorizer" -) - -const Name = "text2vec-palm" - -func New() *PalmModule { - return &PalmModule{} -} - -type PalmModule struct { - vectorizer textVectorizer - metaProvider metaProvider - graphqlProvider modulecapabilities.GraphQLArguments - searcher modulecapabilities.Searcher - nearTextTransformer modulecapabilities.TextTransform - logger logrus.FieldLogger - additionalPropertiesProvider modulecapabilities.AdditionalProperties -} - -type textVectorizer interface { - Object(ctx context.Context, obj *models.Object, objDiff *moduletools.ObjectDiff, - settings vectorizer.ClassSettings) error - Texts(ctx context.Context, input []string, - settings vectorizer.ClassSettings) ([]float32, error) - - MoveTo(source, target []float32, weight float32) ([]float32, error) - MoveAwayFrom(source, target []float32, weight float32) ([]float32, error) - CombineVectors([][]float32) []float32 -} - -type metaProvider interface { - MetaInfo() (map[string]interface{}, error) -} - -func (m *PalmModule) Name() string { - return "text2vec-palm" -} - -func (m *PalmModule) Type() modulecapabilities.ModuleType { - return modulecapabilities.Text2Vec -} - -func (m *PalmModule) Init(ctx context.Context, - params moduletools.ModuleInitParams, -) error { - m.logger = params.GetLogger() - - if err := m.initVectorizer(ctx, params.GetConfig().ModuleHttpClientTimeout, m.logger); err != nil { - return errors.Wrap(err, "init vectorizer") - } - - if err := m.initAdditionalPropertiesProvider(); err != nil { - return errors.Wrap(err, "init additional properties provider") - } - - return nil -} - -func (m *PalmModule) InitExtension(modules []modulecapabilities.Module) error { - for _, module := range modules { - if module.Name() == m.Name() { - continue - } - if arg, ok := module.(modulecapabilities.TextTransformers); ok { - if arg != nil && arg.TextTransformers() != nil { - m.nearTextTransformer = arg.TextTransformers()["nearText"] - } - } - } - - if err := m.initNearText(); err != nil { - return errors.Wrap(err, "init graphql provider") - } - return nil -} - -func (m *PalmModule) initVectorizer(ctx context.Context, timeout time.Duration, - logger logrus.FieldLogger, -) error { - apiKey := os.Getenv("PALM_APIKEY") - client := clients.New(apiKey, timeout, logger) - - m.vectorizer = vectorizer.New(client) - m.metaProvider = client - - return nil -} - -func (m *PalmModule) initAdditionalPropertiesProvider() error { - projector := projector.New() - m.additionalPropertiesProvider = additional.New(projector) - return nil -} - -func (m *PalmModule) RootHandler() http.Handler { - // TODO: remove once this is a capability interface - return nil -} - -func (m *PalmModule) VectorizeObject(ctx context.Context, - obj *models.Object, objDiff *moduletools.ObjectDiff, cfg moduletools.ClassConfig, -) error { - icheck := config.NewClassSettings(cfg) - return m.vectorizer.Object(ctx, obj, objDiff, icheck) -} - -func (m *PalmModule) MetaInfo() (map[string]interface{}, error) { - return m.metaProvider.MetaInfo() -} - -func (m *PalmModule) AdditionalProperties() map[string]modulecapabilities.AdditionalProperty { - return m.additionalPropertiesProvider.AdditionalProperties() -} - -func (m *PalmModule) VectorizeInput(ctx context.Context, - input string, cfg moduletools.ClassConfig, -) ([]float32, error) { - return m.vectorizer.Texts(ctx, []string{input}, config.NewClassSettings(cfg)) -} - -// verify we implement the modules.Module interface -var ( - _ = modulecapabilities.Module(New()) - _ = modulecapabilities.Vectorizer(New()) - _ = modulecapabilities.MetaProvider(New()) - _ = modulecapabilities.Searcher(New()) - _ = modulecapabilities.GraphQLArguments(New()) -) diff --git a/modules/text2vec-palm/nearText.go b/modules/text2vec-palm/nearText.go deleted file mode 100644 index c2bc9c160b358d0240cb78d7c19cec71a98c7ba5..0000000000000000000000000000000000000000 --- a/modules/text2vec-palm/nearText.go +++ /dev/null @@ -1,36 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modpalm - -import ( - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/modules/text2vec-palm/neartext" -) - -func (m *PalmModule) initNearText() error { - m.searcher = neartext.NewSearcher(m.vectorizer) - m.graphqlProvider = neartext.New(m.nearTextTransformer) - return nil -} - -func (m *PalmModule) Arguments() map[string]modulecapabilities.GraphQLArgument { - return m.graphqlProvider.Arguments() -} - -func (m *PalmModule) VectorSearches() map[string]modulecapabilities.VectorForParams { - return m.searcher.VectorSearches() -} - -var ( - _ = modulecapabilities.GraphQLArguments(New()) - _ = modulecapabilities.Searcher(New()) -) diff --git a/modules/text2vec-palm/neartext/fakes_for_test.go b/modules/text2vec-palm/neartext/fakes_for_test.go deleted file mode 100644 index a72048d0b9cb0b05c8c5ca648cf95db841571505..0000000000000000000000000000000000000000 --- a/modules/text2vec-palm/neartext/fakes_for_test.go +++ /dev/null @@ -1,26 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package neartext - -type fakeTransformer struct{} - -func (t *fakeTransformer) Transform(in []string) ([]string, error) { - result := make([]string, len(in)) - for i, txt := range in { - if txt == "transform this" { - result[i] = "transformed text" - } else { - result[i] = txt - } - } - return result, nil -} diff --git a/modules/text2vec-palm/neartext/graphql_argument.go b/modules/text2vec-palm/neartext/graphql_argument.go deleted file mode 100644 index 9ba524edfd06baef476cf589d9b531701354eaee..0000000000000000000000000000000000000000 --- a/modules/text2vec-palm/neartext/graphql_argument.go +++ /dev/null @@ -1,116 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package neartext - -import ( - "fmt" - - "github.com/tailor-inc/graphql" - "github.com/weaviate/weaviate/adapters/handlers/graphql/descriptions" -) - -func (g *GraphQLArgumentsProvider) getNearTextArgumentFn(classname string) *graphql.ArgumentConfig { - return g.nearTextArgument("GetObjects", classname) -} - -func (g *GraphQLArgumentsProvider) aggregateNearTextArgumentFn(classname string) *graphql.ArgumentConfig { - return g.nearTextArgument("Aggregate", classname) -} - -func (g *GraphQLArgumentsProvider) nearTextArgument(prefix, className string) *graphql.ArgumentConfig { - prefixName := fmt.Sprintf("Txt2VecPalm%s%s", prefix, className) - return &graphql.ArgumentConfig{ - Type: graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sNearTextInpObj", prefixName), - Fields: g.nearTextFields(prefixName), - Description: descriptions.GetWhereInpObj, - }, - ), - } -} - -func (g *GraphQLArgumentsProvider) nearTextFields(prefix string) graphql.InputObjectConfigFieldMap { - nearTextFields := graphql.InputObjectConfigFieldMap{ - "concepts": &graphql.InputObjectFieldConfig{ - // Description: descriptions.Concepts, - Type: graphql.NewNonNull(graphql.NewList(graphql.String)), - }, - "moveTo": &graphql.InputObjectFieldConfig{ - Description: descriptions.VectorMovement, - Type: graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sMoveTo", prefix), - Fields: g.movementInp(fmt.Sprintf("%sMoveTo", prefix)), - }), - }, - "certainty": &graphql.InputObjectFieldConfig{ - Description: descriptions.Certainty, - Type: graphql.Float, - }, - "distance": &graphql.InputObjectFieldConfig{ - Description: descriptions.Distance, - Type: graphql.Float, - }, - "moveAwayFrom": &graphql.InputObjectFieldConfig{ - Description: descriptions.VectorMovement, - Type: graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sMoveAwayFrom", prefix), - Fields: g.movementInp(fmt.Sprintf("%sMoveAwayFrom", prefix)), - }), - }, - } - if g.nearTextTransformer != nil { - nearTextFields["autocorrect"] = &graphql.InputObjectFieldConfig{ - Description: "Autocorrect input text values", - Type: graphql.Boolean, - } - } - return nearTextFields -} - -func (g *GraphQLArgumentsProvider) movementInp(prefix string) graphql.InputObjectConfigFieldMap { - return graphql.InputObjectConfigFieldMap{ - "concepts": &graphql.InputObjectFieldConfig{ - Description: descriptions.Keywords, - Type: graphql.NewList(graphql.String), - }, - "objects": &graphql.InputObjectFieldConfig{ - Description: "objects", - Type: graphql.NewList(g.objectsInpObj(prefix)), - }, - "force": &graphql.InputObjectFieldConfig{ - Description: descriptions.Force, - Type: graphql.NewNonNull(graphql.Float), - }, - } -} - -func (g *GraphQLArgumentsProvider) objectsInpObj(prefix string) *graphql.InputObject { - return graphql.NewInputObject( - graphql.InputObjectConfig{ - Name: fmt.Sprintf("%sMovementObjectsInpObj", prefix), - Fields: graphql.InputObjectConfigFieldMap{ - "id": &graphql.InputObjectFieldConfig{ - Type: graphql.String, - Description: "id of an object", - }, - "beacon": &graphql.InputObjectFieldConfig{ - Type: graphql.String, - Description: descriptions.Beacon, - }, - }, - Description: "Movement Object", - }, - ) -} diff --git a/modules/text2vec-palm/neartext/graphql_argument_test.go b/modules/text2vec-palm/neartext/graphql_argument_test.go deleted file mode 100644 index a56e67f1a2fa35cdc3061690b1b8164c7c7bb57f..0000000000000000000000000000000000000000 --- a/modules/text2vec-palm/neartext/graphql_argument_test.go +++ /dev/null @@ -1,195 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package neartext - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/tailor-inc/graphql" -) - -func TestNearTextGraphQLArgument(t *testing.T) { - t.Run("should generate nearText argument properly", func(t *testing.T) { - // given - prefix := "Prefix" - classname := "Class" - // when - nearText := New(nil).nearTextArgument(prefix, classname) - - // then - // the built graphQL field needs to support this structure: - // { - // concepts: ["c1", "c2"], - // distance: 0.9, - // moveTo: { - // concepts: ["c1", "c2"], - // objects: [ - // { id: "some-uuid-value"}], - // { beacon: "some-beacon-value"} - // ], - // force: 0.8 - // } - // moveAwayFrom: { - // concepts: ["c1", "c2"], - // objects: [ - // { id: "some-uuid-value"}], - // { beacon: "some-beacon-value"} - // ], - // force: 0.8 - // } - // } - assert.NotNil(t, nearText) - assert.Equal(t, "Txt2VecPalmPrefixClassNearTextInpObj", nearText.Type.Name()) - nearTextFields, ok := nearText.Type.(*graphql.InputObject) - assert.True(t, ok) - assert.NotNil(t, nearTextFields) - assert.Equal(t, 5, len(nearTextFields.Fields())) - fields := nearTextFields.Fields() - concepts := fields["concepts"] - conceptsNonNull, conceptsNonNullOK := concepts.Type.(*graphql.NonNull) - assert.True(t, conceptsNonNullOK) - conceptsNonNullList, conceptsNonNullListOK := conceptsNonNull.OfType.(*graphql.List) - assert.True(t, conceptsNonNullListOK) - assert.Equal(t, "String", conceptsNonNullList.OfType.Name()) - assert.NotNil(t, concepts) - conceptsType, conceptsTypeOK := concepts.Type.(*graphql.NonNull) - assert.True(t, conceptsTypeOK) - assert.NotNil(t, conceptsType) - assert.NotNil(t, fields["certainty"]) - assert.NotNil(t, fields["distance"]) - assert.NotNil(t, fields["moveTo"]) - moveTo, moveToOK := fields["moveTo"].Type.(*graphql.InputObject) - assert.True(t, moveToOK) - assert.Equal(t, 3, len(moveTo.Fields())) - assert.NotNil(t, moveTo.Fields()["concepts"]) - moveToConcepts, moveToConceptsOK := moveTo.Fields()["concepts"].Type.(*graphql.List) - assert.True(t, moveToConceptsOK) - assert.Equal(t, "String", moveToConcepts.OfType.Name()) - assert.NotNil(t, moveToConcepts) - assert.NotNil(t, moveTo.Fields()["objects"]) - moveToObjects, moveToObjectsOK := moveTo.Fields()["objects"].Type.(*graphql.List) - assert.True(t, moveToObjectsOK) - moveToObjectsObjects, moveToObjectsObjectsOK := moveToObjects.OfType.(*graphql.InputObject) - assert.True(t, moveToObjectsObjectsOK) - assert.Equal(t, 2, len(moveToObjectsObjects.Fields())) - assert.NotNil(t, moveToObjectsObjects.Fields()["id"]) - assert.NotNil(t, moveToObjectsObjects.Fields()["beacon"]) - assert.NotNil(t, moveTo.Fields()["force"]) - _, moveToForceOK := moveTo.Fields()["force"].Type.(*graphql.NonNull) - assert.True(t, moveToForceOK) - assert.NotNil(t, fields["moveAwayFrom"]) - moveAwayFrom, moveAwayFromOK := fields["moveAwayFrom"].Type.(*graphql.InputObject) - assert.True(t, moveAwayFromOK) - assert.NotNil(t, moveAwayFrom.Fields()["concepts"]) - assert.NotNil(t, moveAwayFrom.Fields()["objects"]) - moveAwayFromObjects, moveAwayFromObjectsOK := moveAwayFrom.Fields()["objects"].Type.(*graphql.List) - assert.True(t, moveAwayFromObjectsOK) - moveAwayFromObjectsObjects, moveAwayFromObjectsObjectsOK := moveAwayFromObjects.OfType.(*graphql.InputObject) - assert.Equal(t, 2, len(moveAwayFromObjectsObjects.Fields())) - assert.True(t, moveAwayFromObjectsObjectsOK) - assert.NotNil(t, moveAwayFromObjectsObjects.Fields()["id"]) - assert.NotNil(t, moveAwayFromObjectsObjects.Fields()["beacon"]) - assert.NotNil(t, moveAwayFrom.Fields()["force"]) - _, moveAwayFromForceOK := moveAwayFrom.Fields()["force"].Type.(*graphql.NonNull) - assert.True(t, moveAwayFromForceOK) - }) -} - -func TestNearTextGraphQLArgumentWithAutocorrect(t *testing.T) { - t.Run("should generate nearText argument with autocorrect properly", func(t *testing.T) { - // given - prefix := "Prefix" - classname := "Class" - // when - nearText := New(&fakeTransformer{}).nearTextArgument(prefix, classname) - - // then - // the built graphQL field needs to support this structure: - // { - // concepts: ["c1", "c2"], - // certainty: 0.9, - // autocorrect: true, - // moveTo: { - // concepts: ["c1", "c2"], - // objects: [ - // { id: "some-uuid-value"}], - // { beacon: "some-beacon-value"} - // ], - // force: 0.8 - // } - // moveAwayFrom: { - // concepts: ["c1", "c2"], - // objects: [ - // { id: "some-uuid-value"}], - // { beacon: "some-beacon-value"} - // ], - // force: 0.8 - // } - // } - assert.NotNil(t, nearText) - assert.Equal(t, "Txt2VecPalmPrefixClassNearTextInpObj", nearText.Type.Name()) - nearTextFields, ok := nearText.Type.(*graphql.InputObject) - assert.True(t, ok) - assert.NotNil(t, nearTextFields) - assert.Equal(t, 6, len(nearTextFields.Fields())) - fields := nearTextFields.Fields() - concepts := fields["concepts"] - conceptsNonNull, conceptsNonNullOK := concepts.Type.(*graphql.NonNull) - assert.True(t, conceptsNonNullOK) - conceptsNonNullList, conceptsNonNullListOK := conceptsNonNull.OfType.(*graphql.List) - assert.True(t, conceptsNonNullListOK) - assert.Equal(t, "String", conceptsNonNullList.OfType.Name()) - assert.NotNil(t, concepts) - conceptsType, conceptsTypeOK := concepts.Type.(*graphql.NonNull) - assert.True(t, conceptsTypeOK) - assert.NotNil(t, conceptsType) - assert.NotNil(t, fields["certainty"]) - assert.NotNil(t, fields["distance"]) - assert.NotNil(t, fields["autocorrect"]) - assert.NotNil(t, fields["moveTo"]) - moveTo, moveToOK := fields["moveTo"].Type.(*graphql.InputObject) - assert.True(t, moveToOK) - assert.Equal(t, 3, len(moveTo.Fields())) - assert.NotNil(t, moveTo.Fields()["concepts"]) - moveToConcepts, moveToConceptsOK := moveTo.Fields()["concepts"].Type.(*graphql.List) - assert.True(t, moveToConceptsOK) - assert.Equal(t, "String", moveToConcepts.OfType.Name()) - assert.NotNil(t, moveToConcepts) - assert.NotNil(t, moveTo.Fields()["objects"]) - moveToObjects, moveToObjectsOK := moveTo.Fields()["objects"].Type.(*graphql.List) - assert.True(t, moveToObjectsOK) - moveToObjectsObjects, moveToObjectsObjectsOK := moveToObjects.OfType.(*graphql.InputObject) - assert.True(t, moveToObjectsObjectsOK) - assert.Equal(t, 2, len(moveToObjectsObjects.Fields())) - assert.NotNil(t, moveToObjectsObjects.Fields()["id"]) - assert.NotNil(t, moveToObjectsObjects.Fields()["beacon"]) - assert.NotNil(t, moveTo.Fields()["force"]) - _, moveToForceOK := moveTo.Fields()["force"].Type.(*graphql.NonNull) - assert.True(t, moveToForceOK) - assert.NotNil(t, fields["moveAwayFrom"]) - moveAwayFrom, moveAwayFromOK := fields["moveAwayFrom"].Type.(*graphql.InputObject) - assert.True(t, moveAwayFromOK) - assert.NotNil(t, moveAwayFrom.Fields()["concepts"]) - assert.NotNil(t, moveAwayFrom.Fields()["objects"]) - moveAwayFromObjects, moveAwayFromObjectsOK := moveAwayFrom.Fields()["objects"].Type.(*graphql.List) - assert.True(t, moveAwayFromObjectsOK) - moveAwayFromObjectsObjects, moveAwayFromObjectsObjectsOK := moveAwayFromObjects.OfType.(*graphql.InputObject) - assert.Equal(t, 2, len(moveAwayFromObjectsObjects.Fields())) - assert.True(t, moveAwayFromObjectsObjectsOK) - assert.NotNil(t, moveAwayFromObjectsObjects.Fields()["id"]) - assert.NotNil(t, moveAwayFromObjectsObjects.Fields()["beacon"]) - assert.NotNil(t, moveAwayFrom.Fields()["force"]) - _, moveAwayFromForceOK := moveAwayFrom.Fields()["force"].Type.(*graphql.NonNull) - assert.True(t, moveAwayFromForceOK) - }) -} diff --git a/modules/text2vec-palm/neartext/graphql_extract.go b/modules/text2vec-palm/neartext/graphql_extract.go deleted file mode 100644 index c308fe87734ef0d215b5babf8c684f9d9d3be99b..0000000000000000000000000000000000000000 --- a/modules/text2vec-palm/neartext/graphql_extract.go +++ /dev/null @@ -1,115 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package neartext - -import "github.com/weaviate/weaviate/usecases/modulecomponents/nearText" - -// ExtractNearText arguments, such as "concepts", "moveTo", "moveAwayFrom", -// "limit", etc. -func (g *GraphQLArgumentsProvider) extractNearTextFn(source map[string]interface{}) interface{} { - var args nearText.NearTextParams - - // keywords is a required argument, so we don't need to check for its existing - keywords := source["concepts"].([]interface{}) - args.Values = make([]string, len(keywords)) - for i, value := range keywords { - args.Values[i] = value.(string) - } - - // autocorrect is an optional arg, so it could be nil - autocorrect, ok := source["autocorrect"] - if ok { - args.Autocorrect = autocorrect.(bool) - } - - // if there's text transformer present and autocorrect set to true - // perform text transformation operation - if args.Autocorrect && g.nearTextTransformer != nil { - if transformedValues, err := g.nearTextTransformer.Transform(args.Values); err == nil { - args.Values = transformedValues - } - } - - // limit is an optional arg, so it could be nil - limit, ok := source["limit"] - if ok { - // the type is fixed through gql config, no need to catch incorrect type - // assumption - args.Limit = limit.(int) - } - - certainty, ok := source["certainty"] - if ok { - args.Certainty = certainty.(float64) - } - - distance, ok := source["distance"] - if ok { - args.Distance = distance.(float64) - args.WithDistance = true - } - - // moveTo is an optional arg, so it could be nil - moveTo, ok := source["moveTo"] - if ok { - args.MoveTo = extractMovement(moveTo) - } - - // network is an optional arg, so it could be nil - network, ok := source["network"] - if ok { - args.Network = network.(bool) - } - - // moveAwayFrom is an optional arg, so it could be nil - moveAwayFrom, ok := source["moveAwayFrom"] - if ok { - args.MoveAwayFrom = extractMovement(moveAwayFrom) - } - - return &args -} - -func extractMovement(input interface{}) nearText.ExploreMove { - // the type is fixed through gql config, no need to catch incorrect type - // assumption, all fields are required so we don't need to check for their - // presence - moveToMap := input.(map[string]interface{}) - res := nearText.ExploreMove{} - res.Force = float32(moveToMap["force"].(float64)) - - keywords, ok := moveToMap["concepts"].([]interface{}) - if ok { - res.Values = make([]string, len(keywords)) - for i, value := range keywords { - res.Values[i] = value.(string) - } - } - - objects, ok := moveToMap["objects"].([]interface{}) - if ok { - res.Objects = make([]nearText.ObjectMove, len(objects)) - for i, value := range objects { - v, ok := value.(map[string]interface{}) - if ok { - if v["id"] != nil { - res.Objects[i].ID = v["id"].(string) - } - if v["beacon"] != nil { - res.Objects[i].Beacon = v["beacon"].(string) - } - } - } - } - - return res -} diff --git a/modules/text2vec-palm/neartext/graphql_extract_test.go b/modules/text2vec-palm/neartext/graphql_extract_test.go deleted file mode 100644 index 728b169950a7c49a43a46a8b55555e3ab1d05084..0000000000000000000000000000000000000000 --- a/modules/text2vec-palm/neartext/graphql_extract_test.go +++ /dev/null @@ -1,620 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package neartext - -import ( - "reflect" - "testing" - - "github.com/weaviate/weaviate/usecases/modulecomponents/nearText" -) - -func Test_extractNearTextFn(t *testing.T) { - type args struct { - source map[string]interface{} - } - tests := []struct { - name string - args args - want *nearText.NearTextParams - }{ - { - "Extract with concepts", - args{ - source: map[string]interface{}{ - "concepts": []interface{}{"c1", "c2", "c3"}, - }, - }, - &nearText.NearTextParams{ - Values: []string{"c1", "c2", "c3"}, - }, - }, - { - "Extract with concepts, distance, limit and network", - args{ - source: map[string]interface{}{ - "concepts": []interface{}{"c1", "c2", "c3"}, - "distance": float64(0.4), - "limit": 100, - "network": true, - }, - }, - &nearText.NearTextParams{ - Values: []string{"c1", "c2", "c3"}, - Distance: 0.4, - WithDistance: true, - Limit: 100, - Network: true, - }, - }, - { - "Extract with concepts, certainty, limit and network", - args{ - source: map[string]interface{}{ - "concepts": []interface{}{"c1", "c2", "c3"}, - "certainty": float64(0.4), - "limit": 100, - "network": true, - }, - }, - &nearText.NearTextParams{ - Values: []string{"c1", "c2", "c3"}, - Certainty: 0.4, - Limit: 100, - Network: true, - }, - }, - { - "Extract with moveTo, moveAwayFrom, and distance", - args{ - source: map[string]interface{}{ - "concepts": []interface{}{"c1", "c2", "c3"}, - "distance": float64(0.89), - "limit": 500, - "network": false, - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"positive"}, - "force": float64(0.5), - }, - "moveAwayFrom": map[string]interface{}{ - "concepts": []interface{}{"epic"}, - "force": float64(0.25), - }, - }, - }, - &nearText.NearTextParams{ - Values: []string{"c1", "c2", "c3"}, - Distance: 0.89, - WithDistance: true, - Limit: 500, - Network: false, - MoveTo: nearText.ExploreMove{ - Values: []string{"positive"}, - Force: 0.5, - }, - MoveAwayFrom: nearText.ExploreMove{ - Values: []string{"epic"}, - Force: 0.25, - }, - }, - }, - { - "Extract with moveTo, moveAwayFrom, and certainty", - args{ - source: map[string]interface{}{ - "concepts": []interface{}{"c1", "c2", "c3"}, - "certainty": float64(0.89), - "limit": 500, - "network": false, - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"positive"}, - "force": float64(0.5), - }, - "moveAwayFrom": map[string]interface{}{ - "concepts": []interface{}{"epic"}, - "force": float64(0.25), - }, - }, - }, - &nearText.NearTextParams{ - Values: []string{"c1", "c2", "c3"}, - Certainty: 0.89, - Limit: 500, - Network: false, - MoveTo: nearText.ExploreMove{ - Values: []string{"positive"}, - Force: 0.5, - }, - MoveAwayFrom: nearText.ExploreMove{ - Values: []string{"epic"}, - Force: 0.25, - }, - }, - }, - { - "Extract with moveTo, moveAwayFrom, distance (and objects)", - args{ - source: map[string]interface{}{ - "concepts": []interface{}{"c1", "c2", "c3"}, - "distance": float64(0.89), - "limit": 500, - "network": false, - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"positive"}, - "force": float64(0.5), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveTo-uuid1", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveTo-uuid2", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveTo-uuid3", - }, - }, - }, - "moveAwayFrom": map[string]interface{}{ - "concepts": []interface{}{"epic"}, - "force": float64(0.25), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveAwayFrom-uuid1", - }, - map[string]interface{}{ - "id": "moveAwayFrom-uuid2", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveAwayFrom-uuid3", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveAwayFrom-uuid4", - }, - }, - }, - }, - }, - &nearText.NearTextParams{ - Values: []string{"c1", "c2", "c3"}, - Distance: 0.89, - WithDistance: true, - Limit: 500, - Network: false, - MoveTo: nearText.ExploreMove{ - Values: []string{"positive"}, - Force: 0.5, - Objects: []nearText.ObjectMove{ - {ID: "moveTo-uuid1"}, - {Beacon: "weaviate://localhost/moveTo-uuid2"}, - {Beacon: "weaviate://localhost/moveTo-uuid3"}, - }, - }, - MoveAwayFrom: nearText.ExploreMove{ - Values: []string{"epic"}, - Force: 0.25, - Objects: []nearText.ObjectMove{ - {ID: "moveAwayFrom-uuid1"}, - {ID: "moveAwayFrom-uuid2"}, - {Beacon: "weaviate://localhost/moveAwayFrom-uuid3"}, - {Beacon: "weaviate://localhost/moveAwayFrom-uuid4"}, - }, - }, - }, - }, - { - "Extract with moveTo, moveAwayFrom, certainty (and objects)", - args{ - source: map[string]interface{}{ - "concepts": []interface{}{"c1", "c2", "c3"}, - "certainty": float64(0.89), - "limit": 500, - "network": false, - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"positive"}, - "force": float64(0.5), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveTo-uuid1", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveTo-uuid2", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveTo-uuid3", - }, - }, - }, - "moveAwayFrom": map[string]interface{}{ - "concepts": []interface{}{"epic"}, - "force": float64(0.25), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveAwayFrom-uuid1", - }, - map[string]interface{}{ - "id": "moveAwayFrom-uuid2", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveAwayFrom-uuid3", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveAwayFrom-uuid4", - }, - }, - }, - }, - }, - &nearText.NearTextParams{ - Values: []string{"c1", "c2", "c3"}, - Certainty: 0.89, - Limit: 500, - Network: false, - MoveTo: nearText.ExploreMove{ - Values: []string{"positive"}, - Force: 0.5, - Objects: []nearText.ObjectMove{ - {ID: "moveTo-uuid1"}, - {Beacon: "weaviate://localhost/moveTo-uuid2"}, - {Beacon: "weaviate://localhost/moveTo-uuid3"}, - }, - }, - MoveAwayFrom: nearText.ExploreMove{ - Values: []string{"epic"}, - Force: 0.25, - Objects: []nearText.ObjectMove{ - {ID: "moveAwayFrom-uuid1"}, - {ID: "moveAwayFrom-uuid2"}, - {Beacon: "weaviate://localhost/moveAwayFrom-uuid3"}, - {Beacon: "weaviate://localhost/moveAwayFrom-uuid4"}, - }, - }, - }, - }, - { - "Extract with moveTo, moveAwayFrom, distance (and doubled objects)", - args{ - source: map[string]interface{}{ - "concepts": []interface{}{"c1", "c2", "c3"}, - "distance": float64(0.89), - "limit": 500, - "network": false, - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"positive"}, - "force": float64(0.5), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveTo-uuid1", - "beacon": "weaviate://localhost/moveTo-uuid2", - }, - map[string]interface{}{ - "id": "moveTo-uuid1", - "beacon": "weaviate://localhost/moveTo-uuid2", - }, - }, - }, - "moveAwayFrom": map[string]interface{}{ - "concepts": []interface{}{"epic"}, - "force": float64(0.25), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveAwayFrom-uuid1", - "beacon": "weaviate://localhost/moveAwayFrom-uuid1", - }, - map[string]interface{}{ - "id": "moveAwayFrom-uuid2", - "beacon": "weaviate://localhost/moveAwayFrom-uuid2", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveAwayFrom-uuid3", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveAwayFrom-uuid4", - }, - }, - }, - }, - }, - &nearText.NearTextParams{ - Values: []string{"c1", "c2", "c3"}, - Distance: 0.89, - WithDistance: true, - Limit: 500, - Network: false, - MoveTo: nearText.ExploreMove{ - Values: []string{"positive"}, - Force: 0.5, - Objects: []nearText.ObjectMove{ - {ID: "moveTo-uuid1", Beacon: "weaviate://localhost/moveTo-uuid2"}, - {ID: "moveTo-uuid1", Beacon: "weaviate://localhost/moveTo-uuid2"}, - }, - }, - MoveAwayFrom: nearText.ExploreMove{ - Values: []string{"epic"}, - Force: 0.25, - Objects: []nearText.ObjectMove{ - {ID: "moveAwayFrom-uuid1", Beacon: "weaviate://localhost/moveAwayFrom-uuid1"}, - {ID: "moveAwayFrom-uuid2", Beacon: "weaviate://localhost/moveAwayFrom-uuid2"}, - {Beacon: "weaviate://localhost/moveAwayFrom-uuid3"}, - {Beacon: "weaviate://localhost/moveAwayFrom-uuid4"}, - }, - }, - }, - }, - { - "Extract with moveTo, moveAwayFrom, certainty (and doubled objects)", - args{ - source: map[string]interface{}{ - "concepts": []interface{}{"c1", "c2", "c3"}, - "certainty": float64(0.89), - "limit": 500, - "network": false, - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"positive"}, - "force": float64(0.5), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveTo-uuid1", - "beacon": "weaviate://localhost/moveTo-uuid2", - }, - map[string]interface{}{ - "id": "moveTo-uuid1", - "beacon": "weaviate://localhost/moveTo-uuid2", - }, - }, - }, - "moveAwayFrom": map[string]interface{}{ - "concepts": []interface{}{"epic"}, - "force": float64(0.25), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveAwayFrom-uuid1", - "beacon": "weaviate://localhost/moveAwayFrom-uuid1", - }, - map[string]interface{}{ - "id": "moveAwayFrom-uuid2", - "beacon": "weaviate://localhost/moveAwayFrom-uuid2", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveAwayFrom-uuid3", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveAwayFrom-uuid4", - }, - }, - }, - }, - }, - &nearText.NearTextParams{ - Values: []string{"c1", "c2", "c3"}, - Certainty: 0.89, - Limit: 500, - Network: false, - MoveTo: nearText.ExploreMove{ - Values: []string{"positive"}, - Force: 0.5, - Objects: []nearText.ObjectMove{ - {ID: "moveTo-uuid1", Beacon: "weaviate://localhost/moveTo-uuid2"}, - {ID: "moveTo-uuid1", Beacon: "weaviate://localhost/moveTo-uuid2"}, - }, - }, - MoveAwayFrom: nearText.ExploreMove{ - Values: []string{"epic"}, - Force: 0.25, - Objects: []nearText.ObjectMove{ - {ID: "moveAwayFrom-uuid1", Beacon: "weaviate://localhost/moveAwayFrom-uuid1"}, - {ID: "moveAwayFrom-uuid2", Beacon: "weaviate://localhost/moveAwayFrom-uuid2"}, - {Beacon: "weaviate://localhost/moveAwayFrom-uuid3"}, - {Beacon: "weaviate://localhost/moveAwayFrom-uuid4"}, - }, - }, - }, - }, - } - - testsWithAutocorrect := []struct { - name string - args args - want *nearText.NearTextParams - }{ - { - "Extract with concepts", - args{ - source: map[string]interface{}{ - "concepts": []interface{}{"c1", "c2", "c3"}, - "autocorrect": true, - }, - }, - &nearText.NearTextParams{ - Values: []string{"c1", "c2", "c3"}, - Autocorrect: true, - }, - }, - { - "Extract with concepts and perform autocorrect", - args{ - source: map[string]interface{}{ - "concepts": []interface{}{"transform this", "c2", "transform this"}, - "autocorrect": true, - }, - }, - &nearText.NearTextParams{ - Values: []string{"transformed text", "c2", "transformed text"}, - Autocorrect: true, - }, - }, - { - "Extract with moveTo, moveAwayFrom, distance (and doubled objects) and autocorrect", - args{ - source: map[string]interface{}{ - "concepts": []interface{}{"transform this", "c1", "c2", "c3", "transform this"}, - "distance": float64(0.89), - "limit": 500, - "network": false, - "autocorrect": true, - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"positive"}, - "force": float64(0.5), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveTo-uuid1", - "beacon": "weaviate://localhost/moveTo-uuid2", - }, - map[string]interface{}{ - "id": "moveTo-uuid1", - "beacon": "weaviate://localhost/moveTo-uuid2", - }, - }, - }, - "moveAwayFrom": map[string]interface{}{ - "concepts": []interface{}{"epic"}, - "force": float64(0.25), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveAwayFrom-uuid1", - "beacon": "weaviate://localhost/moveAwayFrom-uuid1", - }, - map[string]interface{}{ - "id": "moveAwayFrom-uuid2", - "beacon": "weaviate://localhost/moveAwayFrom-uuid2", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveAwayFrom-uuid3", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveAwayFrom-uuid4", - }, - }, - }, - }, - }, - &nearText.NearTextParams{ - Values: []string{"transformed text", "c1", "c2", "c3", "transformed text"}, - Distance: 0.89, - WithDistance: true, - Limit: 500, - Network: false, - Autocorrect: true, - MoveTo: nearText.ExploreMove{ - Values: []string{"positive"}, - Force: 0.5, - Objects: []nearText.ObjectMove{ - {ID: "moveTo-uuid1", Beacon: "weaviate://localhost/moveTo-uuid2"}, - {ID: "moveTo-uuid1", Beacon: "weaviate://localhost/moveTo-uuid2"}, - }, - }, - MoveAwayFrom: nearText.ExploreMove{ - Values: []string{"epic"}, - Force: 0.25, - Objects: []nearText.ObjectMove{ - {ID: "moveAwayFrom-uuid1", Beacon: "weaviate://localhost/moveAwayFrom-uuid1"}, - {ID: "moveAwayFrom-uuid2", Beacon: "weaviate://localhost/moveAwayFrom-uuid2"}, - {Beacon: "weaviate://localhost/moveAwayFrom-uuid3"}, - {Beacon: "weaviate://localhost/moveAwayFrom-uuid4"}, - }, - }, - }, - }, - { - "Extract with moveTo, moveAwayFrom, certainty (and doubled objects) and autocorrect", - args{ - source: map[string]interface{}{ - "concepts": []interface{}{"transform this", "c1", "c2", "c3", "transform this"}, - "certainty": float64(0.89), - "limit": 500, - "network": false, - "autocorrect": true, - "moveTo": map[string]interface{}{ - "concepts": []interface{}{"positive"}, - "force": float64(0.5), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveTo-uuid1", - "beacon": "weaviate://localhost/moveTo-uuid2", - }, - map[string]interface{}{ - "id": "moveTo-uuid1", - "beacon": "weaviate://localhost/moveTo-uuid2", - }, - }, - }, - "moveAwayFrom": map[string]interface{}{ - "concepts": []interface{}{"epic"}, - "force": float64(0.25), - "objects": []interface{}{ - map[string]interface{}{ - "id": "moveAwayFrom-uuid1", - "beacon": "weaviate://localhost/moveAwayFrom-uuid1", - }, - map[string]interface{}{ - "id": "moveAwayFrom-uuid2", - "beacon": "weaviate://localhost/moveAwayFrom-uuid2", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveAwayFrom-uuid3", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/moveAwayFrom-uuid4", - }, - }, - }, - }, - }, - &nearText.NearTextParams{ - Values: []string{"transformed text", "c1", "c2", "c3", "transformed text"}, - Certainty: 0.89, - Limit: 500, - Network: false, - Autocorrect: true, - MoveTo: nearText.ExploreMove{ - Values: []string{"positive"}, - Force: 0.5, - Objects: []nearText.ObjectMove{ - {ID: "moveTo-uuid1", Beacon: "weaviate://localhost/moveTo-uuid2"}, - {ID: "moveTo-uuid1", Beacon: "weaviate://localhost/moveTo-uuid2"}, - }, - }, - MoveAwayFrom: nearText.ExploreMove{ - Values: []string{"epic"}, - Force: 0.25, - Objects: []nearText.ObjectMove{ - {ID: "moveAwayFrom-uuid1", Beacon: "weaviate://localhost/moveAwayFrom-uuid1"}, - {ID: "moveAwayFrom-uuid2", Beacon: "weaviate://localhost/moveAwayFrom-uuid2"}, - {Beacon: "weaviate://localhost/moveAwayFrom-uuid3"}, - {Beacon: "weaviate://localhost/moveAwayFrom-uuid4"}, - }, - }, - }, - }, - } - testsWithAutocorrect = append(testsWithAutocorrect, tests...) - - t.Run("should extract values", func(t *testing.T) { - provider := New(nil) - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := provider.extractNearTextFn(tt.args.source); !reflect.DeepEqual(got, tt.want) { - t.Errorf("extractNearTextFn() = %v, want %v", got, tt.want) - } - }) - } - }) - t.Run("should extract values with transformer", func(t *testing.T) { - provider := New(&fakeTransformer{}) - for _, tt := range testsWithAutocorrect { - t.Run(tt.name, func(t *testing.T) { - if got := provider.extractNearTextFn(tt.args.source); !reflect.DeepEqual(got, tt.want) { - t.Errorf("extractNearTextFn() = %v, want %v", got, tt.want) - } - }) - } - }) -} diff --git a/modules/text2vec-palm/neartext/graphql_provider.go b/modules/text2vec-palm/neartext/graphql_provider.go deleted file mode 100644 index 0f9ff7f2f6ce34a5a820c5e8bb66da28a4ab7d30..0000000000000000000000000000000000000000 --- a/modules/text2vec-palm/neartext/graphql_provider.go +++ /dev/null @@ -1,39 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package neartext - -import ( - "github.com/weaviate/weaviate/entities/modulecapabilities" -) - -type GraphQLArgumentsProvider struct { - nearTextTransformer modulecapabilities.TextTransform -} - -func New(nearTextTransformer modulecapabilities.TextTransform) *GraphQLArgumentsProvider { - return &GraphQLArgumentsProvider{nearTextTransformer} -} - -func (g *GraphQLArgumentsProvider) Arguments() map[string]modulecapabilities.GraphQLArgument { - arguments := map[string]modulecapabilities.GraphQLArgument{} - arguments["nearText"] = g.getNearText() - return arguments -} - -func (g *GraphQLArgumentsProvider) getNearText() modulecapabilities.GraphQLArgument { - return modulecapabilities.GraphQLArgument{ - GetArgumentsFunction: g.getNearTextArgumentFn, - AggregateArgumentsFunction: g.aggregateNearTextArgumentFn, - ExtractFunction: g.extractNearTextFn, - ValidateFunction: g.validateNearTextFn, - } -} diff --git a/modules/text2vec-palm/neartext/param.go b/modules/text2vec-palm/neartext/param.go deleted file mode 100644 index 3e9d851d06f3dabe7d264949ec3c8e3fb62a483e..0000000000000000000000000000000000000000 --- a/modules/text2vec-palm/neartext/param.go +++ /dev/null @@ -1,25 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package neartext - -import ( - "github.com/pkg/errors" - "github.com/weaviate/weaviate/usecases/modulecomponents/nearText" -) - -func (g *GraphQLArgumentsProvider) validateNearTextFn(param interface{}) error { - nearTextParams, ok := param.(*nearText.NearTextParams) - if !ok { - return errors.New("'nearText' invalid parameter") - } - return nearTextParams.Validate() -} diff --git a/modules/text2vec-palm/neartext/searcher.go b/modules/text2vec-palm/neartext/searcher.go deleted file mode 100644 index 1d762593db144c05106fd45aa33ac0fc68860ee7..0000000000000000000000000000000000000000 --- a/modules/text2vec-palm/neartext/searcher.go +++ /dev/null @@ -1,148 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package neartext - -import ( - "context" - - "github.com/weaviate/weaviate/usecases/modulecomponents/nearText" - - "github.com/weaviate/weaviate/modules/text2vec-palm/config" - - "github.com/go-openapi/strfmt" - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/schema/crossref" - localvectorizer "github.com/weaviate/weaviate/modules/text2vec-palm/vectorizer" -) - -type Searcher struct { - vectorizer vectorizer -} - -func NewSearcher(vectorizer vectorizer) *Searcher { - return &Searcher{vectorizer} -} - -type vectorizer interface { - Texts(ctx context.Context, input []string, - settings localvectorizer.ClassSettings) ([]float32, error) - MoveTo(source, target []float32, weight float32) ([]float32, error) - MoveAwayFrom(source, target []float32, weight float32) ([]float32, error) - CombineVectors(vectors [][]float32) []float32 -} - -func (s *Searcher) VectorSearches() map[string]modulecapabilities.VectorForParams { - vectorSearches := map[string]modulecapabilities.VectorForParams{} - vectorSearches["nearText"] = s.vectorForNearTextParam - return vectorSearches -} - -func (s *Searcher) vectorForNearTextParam(ctx context.Context, params interface{}, className string, - findVectorFn modulecapabilities.FindVectorFn, - cfg moduletools.ClassConfig, -) ([]float32, error) { - return s.vectorFromNearTextParam(ctx, params.(*nearText.NearTextParams), className, findVectorFn, cfg) -} - -func (s *Searcher) vectorFromNearTextParam(ctx context.Context, - params *nearText.NearTextParams, className string, findVectorFn modulecapabilities.FindVectorFn, - cfg moduletools.ClassConfig, -) ([]float32, error) { - // it is safe to call NewClassSettings even knowing that cfg can be nil, it - // is to built to work with all defaults in the case of a nil-config, see - // vectorizer/class_settings_test.go for details. - settings := config.NewClassSettings(cfg) - tenant := cfg.Tenant() - vector, err := s.vectorizer.Texts(ctx, params.Values, settings) - if err != nil { - return nil, errors.Errorf("vectorize keywords: %v", err) - } - - moveTo := params.MoveTo - if moveTo.Force > 0 && (len(moveTo.Values) > 0 || len(moveTo.Objects) > 0) { - moveToVector, err := s.vectorFromValuesAndObjects(ctx, moveTo.Values, - moveTo.Objects, className, findVectorFn, settings, tenant) - if err != nil { - return nil, errors.Errorf("vectorize move to: %v", err) - } - - afterMoveTo, err := s.vectorizer.MoveTo(vector, moveToVector, moveTo.Force) - if err != nil { - return nil, err - } - vector = afterMoveTo - } - - moveAway := params.MoveAwayFrom - if moveAway.Force > 0 && (len(moveAway.Values) > 0 || len(moveAway.Objects) > 0) { - moveAwayVector, err := s.vectorFromValuesAndObjects(ctx, moveAway.Values, - moveAway.Objects, className, findVectorFn, settings, tenant) - if err != nil { - return nil, errors.Errorf("vectorize move away from: %v", err) - } - - afterMoveFrom, err := s.vectorizer.MoveAwayFrom(vector, moveAwayVector, moveAway.Force) - if err != nil { - return nil, err - } - vector = afterMoveFrom - } - - return vector, nil -} - -func (s *Searcher) vectorFromValuesAndObjects(ctx context.Context, - values []string, objects []nearText.ObjectMove, - className string, - findVectorFn modulecapabilities.FindVectorFn, - settings localvectorizer.ClassSettings, tenant string, -) ([]float32, error) { - var objectVectors [][]float32 - class := className - if len(values) > 0 { - moveToVector, err := s.vectorizer.Texts(ctx, values, settings) - if err != nil { - return nil, errors.Errorf("vectorize move to: %v", err) - } - objectVectors = append(objectVectors, moveToVector) - } - if len(objects) > 0 { - var id strfmt.UUID - for _, obj := range objects { - if len(obj.ID) > 0 { - id = strfmt.UUID(obj.ID) - } - if len(obj.Beacon) > 0 { - ref, err := crossref.Parse(obj.Beacon) - if err != nil { - return nil, err - } - id = ref.TargetID - - if ref.Class != "" { - class = ref.Class - } - } - - vector, err := findVectorFn(ctx, class, id, tenant) - if err != nil { - return nil, err - } - - objectVectors = append(objectVectors, vector) - } - } - - return s.vectorizer.CombineVectors(objectVectors), nil -} diff --git a/modules/text2vec-palm/vectorizer/fakes_for_test.go b/modules/text2vec-palm/vectorizer/fakes_for_test.go deleted file mode 100644 index 0ba5b766cd573d6111a72f3474f1f47c131afa37..0000000000000000000000000000000000000000 --- a/modules/text2vec-palm/vectorizer/fakes_for_test.go +++ /dev/null @@ -1,89 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - - "github.com/weaviate/weaviate/modules/text2vec-palm/ent" -) - -type fakeClient struct { - lastInput []string - lastConfig ent.VectorizationConfig -} - -func (c *fakeClient) Vectorize(ctx context.Context, - text []string, cfg ent.VectorizationConfig, titlePropertyValue string, -) (*ent.VectorizationResult, error) { - c.lastInput = text - c.lastConfig = cfg - return &ent.VectorizationResult{ - Vectors: [][]float32{{0, 1, 2, 3}}, - Dimensions: 4, - Texts: text, - }, nil -} - -func (c *fakeClient) VectorizeQuery(ctx context.Context, - text []string, cfg ent.VectorizationConfig, -) (*ent.VectorizationResult, error) { - c.lastInput = text - c.lastConfig = cfg - return &ent.VectorizationResult{ - Vectors: [][]float32{{0.1, 1.1, 2.1, 3.1}}, - Dimensions: 4, - Texts: text, - }, nil -} - -type fakeSettings struct { - skippedProperty string - vectorizeClassName bool - excludedProperty string - apiEndpoint string - projectID string - endpointID string - truncateType string -} - -func (f *fakeSettings) PropertyIndexed(propName string) bool { - return f.skippedProperty != propName -} - -func (f *fakeSettings) VectorizePropertyName(propName string) bool { - return f.excludedProperty != propName -} - -func (f *fakeSettings) VectorizeClassName() bool { - return f.vectorizeClassName -} - -func (f *fakeSettings) Truncate() string { - return f.truncateType -} - -func (f *fakeSettings) ApiEndpoint() string { - return f.truncateType -} - -func (f *fakeSettings) ProjectID() string { - return f.truncateType -} - -func (f *fakeSettings) ModelID() string { - return f.truncateType -} - -func (f *fakeSettings) TitleProperty() string { - return "" -} diff --git a/modules/text2vec-palm/vectorizer/objects.go b/modules/text2vec-palm/vectorizer/objects.go deleted file mode 100644 index ae2fb003790113cfeb9d78877ead141b99055bee..0000000000000000000000000000000000000000 --- a/modules/text2vec-palm/vectorizer/objects.go +++ /dev/null @@ -1,184 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - "fmt" - "sort" - "strings" - - "github.com/fatih/camelcase" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/modules/text2vec-palm/ent" -) - -type Vectorizer struct { - client Client -} - -func New(client Client) *Vectorizer { - return &Vectorizer{ - client: client, - } -} - -type Client interface { - Vectorize(ctx context.Context, input []string, - config ent.VectorizationConfig, titlePropertyValue string) (*ent.VectorizationResult, error) - VectorizeQuery(ctx context.Context, input []string, - config ent.VectorizationConfig) (*ent.VectorizationResult, error) -} - -// IndexCheck returns whether a property of a class should be indexed -type ClassSettings interface { - PropertyIndexed(property string) bool - VectorizePropertyName(propertyName string) bool - VectorizeClassName() bool - ApiEndpoint() string - ProjectID() string - ModelID() string - TitleProperty() string -} - -func sortStringKeys(schema_map map[string]interface{}) []string { - keys := make([]string, 0, len(schema_map)) - for k := range schema_map { - keys = append(keys, k) - } - sort.Strings(keys) - return keys -} - -func (v *Vectorizer) Object(ctx context.Context, object *models.Object, - objDiff *moduletools.ObjectDiff, settings ClassSettings, -) error { - vec, err := v.object(ctx, object.Class, object.Properties, objDiff, settings) - if err != nil { - return err - } - - object.Vector = vec - return nil -} - -func appendPropIfText(icheck ClassSettings, list *[]string, propName string, - value interface{}, -) bool { - valueString, ok := value.(string) - if ok { - if icheck.VectorizePropertyName(propName) { - // use prop and value - *list = append(*list, strings.ToLower( - fmt.Sprintf("%s %s", camelCaseToLower(propName), valueString))) - } else { - *list = append(*list, strings.ToLower(valueString)) - } - return true - } - return false -} - -func appendTitlePropIfText(icheck ClassSettings, list *[]string, propName string, - value interface{}, -) bool { - if icheck.TitleProperty() == propName { - return appendPropIfText(icheck, list, propName, value) - } - return false -} - -func (v *Vectorizer) object(ctx context.Context, className string, - schema interface{}, objDiff *moduletools.ObjectDiff, icheck ClassSettings, -) ([]float32, error) { - vectorize := objDiff == nil || objDiff.GetVec() == nil - - var titlePropertyValue []string - var corpi []string - if icheck.VectorizeClassName() { - corpi = append(corpi, camelCaseToLower(className)) - } - if schema != nil { - schemamap := schema.(map[string]interface{}) - for _, prop := range sortStringKeys(schemamap) { - if !icheck.PropertyIndexed(prop) { - continue - } - - appended := false - switch val := schemamap[prop].(type) { - case []string: - for _, elem := range val { - appended = appendPropIfText(icheck, &corpi, prop, elem) || appended - appendTitlePropIfText(icheck, &titlePropertyValue, prop, elem) - } - case []interface{}: - for _, elem := range val { - appended = appendPropIfText(icheck, &corpi, prop, elem) || appended - appendTitlePropIfText(icheck, &titlePropertyValue, prop, elem) - } - default: - appended = appendPropIfText(icheck, &corpi, prop, val) - appendTitlePropIfText(icheck, &titlePropertyValue, prop, val) - } - - vectorize = vectorize || (appended && objDiff != nil && objDiff.IsChangedProp(prop)) - } - } - if len(corpi) == 0 { - // fall back to using the class name - corpi = append(corpi, camelCaseToLower(className)) - } - - // no property was changed, old vector can be used - if !vectorize { - return objDiff.GetVec(), nil - } - - text := []string{strings.Join(corpi, " ")} - titleProperty := strings.Join(titlePropertyValue, " ") - res, err := v.client.Vectorize(ctx, text, ent.VectorizationConfig{ - ApiEndpoint: icheck.ApiEndpoint(), - ProjectID: icheck.ProjectID(), - Model: icheck.ModelID(), - }, titleProperty) - if err != nil { - return nil, err - } - if len(res.Vectors) == 0 { - return nil, fmt.Errorf("no vectors generated") - } - - if len(res.Vectors) > 1 { - return v.CombineVectors(res.Vectors), nil - } - return res.Vectors[0], nil -} - -func camelCaseToLower(in string) string { - parts := camelcase.Split(in) - var sb strings.Builder - for i, part := range parts { - if part == " " { - continue - } - - if i > 0 { - sb.WriteString(" ") - } - - sb.WriteString(strings.ToLower(part)) - } - - return sb.String() -} diff --git a/modules/text2vec-palm/vectorizer/objects_test.go b/modules/text2vec-palm/vectorizer/objects_test.go deleted file mode 100644 index c9d6819fbcff854784bad66a3c6e2f98005396b3..0000000000000000000000000000000000000000 --- a/modules/text2vec-palm/vectorizer/objects_test.go +++ /dev/null @@ -1,359 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/moduletools" -) - -// These are mostly copy/pasted (with minimal additions) from the -// text2vec-contextionary module -func TestVectorizingObjects(t *testing.T) { - type testCase struct { - name string - input *models.Object - expectedClientCall string - noindex string - excludedProperty string // to simulate a schema where property names aren't vectorized - excludedClass string // to simulate a schema where class names aren't vectorized - palmModel string - } - - tests := []testCase{ - { - name: "empty object", - input: &models.Object{ - Class: "Car", - }, - palmModel: "large", - expectedClientCall: "car", - }, - { - name: "object with one string prop", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "Mercedes", - }, - }, - expectedClientCall: "car brand mercedes", - }, - { - name: "object with one non-string prop", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "power": 300, - }, - }, - expectedClientCall: "car", - }, - { - name: "object with a mix of props", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: "car brand best brand review a very great car", - }, - { - name: "with a noindexed property", - noindex: "review", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: "car brand best brand", - }, - { - name: "with the class name not vectorized", - excludedClass: "Car", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: "brand best brand review a very great car", - }, - { - name: "with a property name not vectorized", - excludedProperty: "review", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: "car brand best brand a very great car", - }, - { - name: "with no schema labels vectorized", - excludedProperty: "review", - excludedClass: "Car", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "review": "a very great car", - }, - }, - expectedClientCall: "a very great car", - }, - { - name: "with string/text arrays without propname or classname", - excludedProperty: "reviews", - excludedClass: "Car", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - expectedClientCall: "a very great car you should consider buying one", - }, - { - name: "with string/text arrays with propname and classname", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - expectedClientCall: "car reviews a very great car reviews you should consider buying one", - }, - { - name: "with compound class and prop names", - input: &models.Object{ - Class: "SuperCar", - Properties: map[string]interface{}{ - "brandOfTheCar": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: "super car brand of the car best brand review a very great car", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - client := &fakeClient{} - - v := New(client) - - ic := &fakeSettings{ - skippedProperty: test.noindex, - vectorizeClassName: test.excludedClass != "Car", - excludedProperty: test.excludedProperty, - apiEndpoint: "", - projectID: "", - endpointID: "", - truncateType: "", - } - err := v.Object(context.Background(), test.input, nil, ic) - - require.Nil(t, err) - assert.Equal(t, models.C11yVector{0, 1, 2, 3}, test.input.Vector) - expected := strings.Split(test.expectedClientCall, " ") - actual := strings.Split(client.lastInput[0], " ") - assert.Equal(t, expected, actual) - }) - } -} - -func TestVectorizingObjectsWithDiff(t *testing.T) { - type testCase struct { - name string - input *models.Object - skipped string - diff *moduletools.ObjectDiff - expectedVectorize bool - } - - tests := []testCase{ - { - name: "no diff", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: nil, - expectedVectorize: true, - }, - { - name: "diff all props unchanged", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("brand", "best brand", "best brand"). - WithProp("power", 300, 300). - WithProp("description", "a very great car", "a very great car"). - WithProp("reviews", []interface{}{ - "a very great car", - "you should consider buying one", - }, []interface{}{ - "a very great car", - "you should consider buying one", - }), - expectedVectorize: false, - }, - { - name: "diff one vectorizable prop changed (1)", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("brand", "old best brand", "best brand"), - expectedVectorize: true, - }, - { - name: "diff one vectorizable prop changed (2)", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("description", "old a very great car", "a very great car"), - expectedVectorize: true, - }, - { - name: "diff one vectorizable prop changed (3)", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("reviews", []interface{}{ - "old a very great car", - "you should consider buying one", - }, []interface{}{ - "a very great car", - "you should consider buying one", - }), - expectedVectorize: true, - }, - { - name: "all non-vectorizable props changed", - skipped: "description", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("power", 123, 300). - WithProp("description", "old a very great car", "a very great car"), - expectedVectorize: false, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - ic := &fakeSettings{ - skippedProperty: test.skipped, - } - - client := &fakeClient{} - v := New(client) - - err := v.Object(context.Background(), test.input, test.diff, ic) - - require.Nil(t, err) - if test.expectedVectorize { - assert.Equal(t, models.C11yVector{0, 1, 2, 3}, test.input.Vector) - assert.NotEmpty(t, client.lastInput) - } else { - assert.Equal(t, models.C11yVector{0, 0, 0, 0}, test.input.Vector) - assert.Empty(t, client.lastInput) - } - }) - } -} - -func newObjectDiffWithVector() *moduletools.ObjectDiff { - return moduletools.NewObjectDiff([]float32{0, 0, 0, 0}) -} diff --git a/modules/text2vec-palm/vectorizer/texts.go b/modules/text2vec-palm/vectorizer/texts.go deleted file mode 100644 index a407fee0d13c0ad314e54718bb374ce5c530441b..0000000000000000000000000000000000000000 --- a/modules/text2vec-palm/vectorizer/texts.go +++ /dev/null @@ -1,34 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/modules/text2vec-palm/ent" -) - -func (v *Vectorizer) Texts(ctx context.Context, inputs []string, - settings ClassSettings, -) ([]float32, error) { - res, err := v.client.VectorizeQuery(ctx, inputs, ent.VectorizationConfig{ - ApiEndpoint: settings.ApiEndpoint(), - ProjectID: settings.ProjectID(), - Model: settings.ModelID(), - }) - if err != nil { - return nil, errors.Wrap(err, "remote client vectorize") - } - - return v.CombineVectors(res.Vectors), nil -} diff --git a/modules/text2vec-palm/vectorizer/texts_test.go b/modules/text2vec-palm/vectorizer/texts_test.go deleted file mode 100644 index c35fa31d6f689f0a6e9f29fd0a39f872f842fe51..0000000000000000000000000000000000000000 --- a/modules/text2vec-palm/vectorizer/texts_test.go +++ /dev/null @@ -1,100 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// as used in the nearText searcher -func TestVectorizingTexts(t *testing.T) { - type testCase struct { - name string - input []string - expectedApiEndpoint string - expectedProjectID string - expectedEndpointID string - expectedPalmModel string - palmModel string - } - - tests := []testCase{ - { - name: "single word", - input: []string{"hello"}, - palmModel: "large", - expectedPalmModel: "large", - }, - { - name: "multiple words", - input: []string{"hello world, this is me!"}, - palmModel: "large", - expectedPalmModel: "large", - }, - { - name: "multiple sentences (joined with a dot)", - input: []string{"this is sentence 1", "and here's number 2"}, - palmModel: "large", - expectedPalmModel: "large", - }, - { - name: "multiple sentences already containing a dot", - input: []string{"this is sentence 1.", "and here's number 2"}, - palmModel: "large", - expectedPalmModel: "large", - }, - { - name: "multiple sentences already containing a question mark", - input: []string{"this is sentence 1?", "and here's number 2"}, - palmModel: "large", - expectedPalmModel: "large", - }, - { - name: "multiple sentences already containing an exclamation mark", - input: []string{"this is sentence 1!", "and here's number 2"}, - palmModel: "large", - expectedPalmModel: "large", - }, - { - name: "multiple sentences already containing comma", - input: []string{"this is sentence 1,", "and here's number 2"}, - palmModel: "large", - expectedPalmModel: "large", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - client := &fakeClient{} - - v := New(client) - - settings := &fakeSettings{ - apiEndpoint: "", - projectID: "", - endpointID: "", - } - vec, err := v.Texts(context.Background(), test.input, settings) - - require.Nil(t, err) - assert.Equal(t, []float32{0.1, 1.1, 2.1, 3.1}, vec) - assert.Equal(t, test.input, client.lastInput) - assert.Equal(t, test.expectedApiEndpoint, client.lastConfig.ApiEndpoint) - assert.Equal(t, test.expectedProjectID, client.lastConfig.ProjectID) - assert.Equal(t, test.expectedEndpointID, client.lastConfig.Model) - }) - } -} diff --git a/modules/text2vec-palm/vectorizer/todo_move_out_of_here_movements.go b/modules/text2vec-palm/vectorizer/todo_move_out_of_here_movements.go deleted file mode 100644 index b7c554ae86e9338d31c79eb28f4a679c5ce95f93..0000000000000000000000000000000000000000 --- a/modules/text2vec-palm/vectorizer/todo_move_out_of_here_movements.go +++ /dev/null @@ -1,68 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "fmt" - - libvectorizer "github.com/weaviate/weaviate/usecases/vectorizer" -) - -// CombineVectors combines all of the vector into sum of their parts -func (v *Vectorizer) CombineVectors(vectors [][]float32) []float32 { - return libvectorizer.CombineVectors(vectors) -} - -// MoveTo moves one vector toward another -func (v *Vectorizer) MoveTo(source []float32, target []float32, weight float32, -) ([]float32, error) { - multiplier := float32(0.5) - - if len(source) != len(target) { - return nil, fmt.Errorf("movement: vector lengths don't match: got %d and %d", - len(source), len(target)) - } - - if weight < 0 || weight > 1 { - return nil, fmt.Errorf("movement: force must be between 0 and 1: got %f", - weight) - } - - out := make([]float32, len(source)) - for i, sourceItem := range source { - out[i] = sourceItem*(1-weight*multiplier) + target[i]*(weight*multiplier) - } - - return out, nil -} - -// MoveAwayFrom moves one vector away from another -func (v *Vectorizer) MoveAwayFrom(source []float32, target []float32, weight float32, -) ([]float32, error) { - multiplier := float32(0.5) // so the movement is fair in comparison with moveTo - if len(source) != len(target) { - return nil, fmt.Errorf("movement (moveAwayFrom): vector lengths don't match: "+ - "got %d and %d", len(source), len(target)) - } - - if weight < 0 { - return nil, fmt.Errorf("movement (moveAwayFrom): force must be 0 or positive: "+ - "got %f", weight) - } - - out := make([]float32, len(source)) - for i, sourceItem := range source { - out[i] = sourceItem + weight*multiplier*(sourceItem-target[i]) - } - - return out, nil -} diff --git a/modules/text2vec-transformers/clients/meta.go b/modules/text2vec-transformers/clients/meta.go deleted file mode 100644 index cb1d9c09942bed74cadacdbcd6c8dd3e57ecd830..0000000000000000000000000000000000000000 --- a/modules/text2vec-transformers/clients/meta.go +++ /dev/null @@ -1,105 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "strings" - "sync" - - "github.com/pkg/errors" -) - -func (v *vectorizer) MetaInfo() (map[string]interface{}, error) { - type nameMetaErr struct { - name string - meta map[string]interface{} - err error - } - - endpoints := map[string]string{} - if v.originPassage != v.originQuery { - endpoints["passage"] = v.urlPassage("/meta") - endpoints["query"] = v.urlQuery("/meta") - } else { - endpoints[""] = v.urlPassage("/meta") - } - - var wg sync.WaitGroup - ch := make(chan nameMetaErr, len(endpoints)) - for serviceName, endpoint := range endpoints { - wg.Add(1) - go func(serviceName string, endpoint string) { - defer wg.Done() - meta, err := v.metaInfo(endpoint) - ch <- nameMetaErr{serviceName, meta, err} - }(serviceName, endpoint) - } - wg.Wait() - close(ch) - - metas := map[string]interface{}{} - var errs []string - for nme := range ch { - if nme.err != nil { - prefix := "" - if nme.name != "" { - prefix = "[" + nme.name + "] " - } - errs = append(errs, fmt.Sprintf("%s%v", prefix, nme.err.Error())) - } - if nme.meta != nil { - metas[nme.name] = nme.meta - } - } - - if len(errs) > 0 { - return nil, errors.Errorf(strings.Join(errs, ", ")) - } - if len(metas) == 1 { - for _, meta := range metas { - return meta.(map[string]interface{}), nil - } - } - return metas, nil -} - -func (v *vectorizer) metaInfo(endpoint string) (map[string]interface{}, error) { - req, err := http.NewRequestWithContext(context.Background(), "GET", endpoint, nil) - if err != nil { - return nil, errors.Wrap(err, "create GET meta request") - } - - res, err := v.httpClient.Do(req) - if err != nil { - return nil, errors.Wrap(err, "send GET meta request") - } - defer res.Body.Close() - if !(res.StatusCode >= http.StatusOK && res.StatusCode < http.StatusMultipleChoices) { - return nil, errors.Errorf("unexpected status code '%d' of meta request", res.StatusCode) - } - - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - return nil, errors.Wrap(err, "read meta response body") - } - - var resBody map[string]interface{} - if err := json.Unmarshal(bodyBytes, &resBody); err != nil { - return nil, errors.Wrap(err, "unmarshal meta response body") - } - return resBody, nil -} diff --git a/modules/text2vec-transformers/clients/meta_test.go b/modules/text2vec-transformers/clients/meta_test.go deleted file mode 100644 index 2f0c3deeb03fa281df1150e1bdb4ce67557bb3fe..0000000000000000000000000000000000000000 --- a/modules/text2vec-transformers/clients/meta_test.go +++ /dev/null @@ -1,350 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func TestGetMeta(t *testing.T) { - t.Run("when common server is providing meta", func(t *testing.T) { - server := httptest.NewServer(&testMetaHandler{t: t}) - defer server.Close() - v := New(server.URL, server.URL, 0, nullLogger()) - meta, err := v.MetaInfo() - - assert.Nil(t, err) - assert.NotNil(t, meta) - - model := extractChildMap(t, meta, "model") - assert.NotNil(t, model["_name_or_path"]) - assert.NotNil(t, model["architectures"]) - assert.Contains(t, model["architectures"], "DistilBertModel") - ID2Label := extractChildMap(t, model, "id2label") - assert.NotNil(t, ID2Label["0"]) - assert.NotNil(t, ID2Label["1"]) - }) - - t.Run("when passage and query servers are providing meta", func(t *testing.T) { - serverPassage := httptest.NewServer(&testMetaHandler{t: t, modelType: "passage"}) - serverQuery := httptest.NewServer(&testMetaHandler{t: t, modelType: "query"}) - defer serverPassage.Close() - defer serverQuery.Close() - v := New(serverPassage.URL, serverQuery.URL, 0, nullLogger()) - meta, err := v.MetaInfo() - - assert.Nil(t, err) - assert.NotNil(t, meta) - - passage := extractChildMap(t, meta, "passage") - passageModel := extractChildMap(t, passage, "model") - assert.NotNil(t, passageModel["_name_or_path"]) - assert.NotNil(t, passageModel["architectures"]) - assert.Contains(t, passageModel["architectures"], "DPRContextEncoder") - passageID2Label := extractChildMap(t, passageModel, "id2label") - assert.NotNil(t, passageID2Label["0"]) - assert.NotNil(t, passageID2Label["1"]) - - query := extractChildMap(t, meta, "query") - queryModel := extractChildMap(t, query, "model") - assert.NotNil(t, queryModel["_name_or_path"]) - assert.NotNil(t, queryModel["architectures"]) - assert.Contains(t, queryModel["architectures"], "DPRQuestionEncoder") - queryID2Label := extractChildMap(t, queryModel, "id2label") - assert.NotNil(t, queryID2Label["0"]) - assert.NotNil(t, queryID2Label["1"]) - }) - - t.Run("when passage and query servers are unavailable", func(t *testing.T) { - rt := time.Now().Add(time.Hour) - serverPassage := httptest.NewServer(&testMetaHandler{t: t, modelType: "passage", readyTime: rt}) - serverQuery := httptest.NewServer(&testMetaHandler{t: t, modelType: "query", readyTime: rt}) - defer serverPassage.Close() - defer serverQuery.Close() - v := New(serverPassage.URL, serverQuery.URL, 0, nullLogger()) - meta, err := v.MetaInfo() - - assert.NotNil(t, err) - assert.Contains(t, err.Error(), "[passage] unexpected status code '503' of meta request") - assert.Contains(t, err.Error(), "[query] unexpected status code '503' of meta request") - assert.Nil(t, meta) - }) -} - -type testMetaHandler struct { - t *testing.T - // the test handler will report as not ready before the time has passed - readyTime time.Time - modelType string -} - -func (h *testMetaHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - assert.Equal(h.t, "/meta", r.URL.String()) - assert.Equal(h.t, http.MethodGet, r.Method) - - if time.Since(h.readyTime) < 0 { - w.WriteHeader(http.StatusServiceUnavailable) - return - } - - w.Write([]byte(h.metaInfo())) -} - -func (h *testMetaHandler) metaInfo() string { - switch h.modelType { - case "passage": - return `{ - "model": { - "return_dict": true, - "output_hidden_states": false, - "output_attentions": false, - "torchscript": false, - "torch_dtype": "float32", - "use_bfloat16": false, - "pruned_heads": {}, - "tie_word_embeddings": true, - "is_encoder_decoder": false, - "is_decoder": false, - "cross_attention_hidden_size": null, - "add_cross_attention": false, - "tie_encoder_decoder": false, - "max_length": 20, - "min_length": 0, - "do_sample": false, - "early_stopping": false, - "num_beams": 1, - "num_beam_groups": 1, - "diversity_penalty": 0, - "temperature": 1, - "top_k": 50, - "top_p": 1, - "repetition_penalty": 1, - "length_penalty": 1, - "no_repeat_ngram_size": 0, - "encoder_no_repeat_ngram_size": 0, - "bad_words_ids": null, - "num_return_sequences": 1, - "chunk_size_feed_forward": 0, - "output_scores": false, - "return_dict_in_generate": false, - "forced_bos_token_id": null, - "forced_eos_token_id": null, - "remove_invalid_values": false, - "architectures": [ - "DPRContextEncoder" - ], - "finetuning_task": null, - "id2label": { - "0": "LABEL_0", - "1": "LABEL_1" - }, - "label2id": { - "LABEL_0": 0, - "LABEL_1": 1 - }, - "tokenizer_class": null, - "prefix": null, - "bos_token_id": null, - "pad_token_id": 0, - "eos_token_id": null, - "sep_token_id": null, - "decoder_start_token_id": null, - "task_specific_params": null, - "problem_type": null, - "_name_or_path": "./models/model", - "transformers_version": "4.16.2", - "gradient_checkpointing": false, - "model_type": "dpr", - "vocab_size": 30522, - "hidden_size": 768, - "num_hidden_layers": 12, - "num_attention_heads": 12, - "hidden_act": "gelu", - "intermediate_size": 3072, - "hidden_dropout_prob": 0.1, - "attention_probs_dropout_prob": 0.1, - "max_position_embeddings": 512, - "type_vocab_size": 2, - "initializer_range": 0.02, - "layer_norm_eps": 1e-12, - "projection_dim": 0, - "position_embedding_type": "absolute" - } - }` - case "query": - return `{ - "model": { - "return_dict": true, - "output_hidden_states": false, - "output_attentions": false, - "torchscript": false, - "torch_dtype": "float32", - "use_bfloat16": false, - "pruned_heads": {}, - "tie_word_embeddings": true, - "is_encoder_decoder": false, - "is_decoder": false, - "cross_attention_hidden_size": null, - "add_cross_attention": false, - "tie_encoder_decoder": false, - "max_length": 20, - "min_length": 0, - "do_sample": false, - "early_stopping": false, - "num_beams": 1, - "num_beam_groups": 1, - "diversity_penalty": 0, - "temperature": 1, - "top_k": 50, - "top_p": 1, - "repetition_penalty": 1, - "length_penalty": 1, - "no_repeat_ngram_size": 0, - "encoder_no_repeat_ngram_size": 0, - "bad_words_ids": null, - "num_return_sequences": 1, - "chunk_size_feed_forward": 0, - "output_scores": false, - "return_dict_in_generate": false, - "forced_bos_token_id": null, - "forced_eos_token_id": null, - "remove_invalid_values": false, - "architectures": [ - "DPRQuestionEncoder" - ], - "finetuning_task": null, - "id2label": { - "0": "LABEL_0", - "1": "LABEL_1" - }, - "label2id": { - "LABEL_0": 0, - "LABEL_1": 1 - }, - "tokenizer_class": null, - "prefix": null, - "bos_token_id": null, - "pad_token_id": 0, - "eos_token_id": null, - "sep_token_id": null, - "decoder_start_token_id": null, - "task_specific_params": null, - "problem_type": null, - "_name_or_path": "./models/model", - "transformers_version": "4.16.2", - "gradient_checkpointing": false, - "model_type": "dpr", - "vocab_size": 30522, - "hidden_size": 768, - "num_hidden_layers": 12, - "num_attention_heads": 12, - "hidden_act": "gelu", - "intermediate_size": 3072, - "hidden_dropout_prob": 0.1, - "attention_probs_dropout_prob": 0.1, - "max_position_embeddings": 512, - "type_vocab_size": 2, - "initializer_range": 0.02, - "layer_norm_eps": 1e-12, - "projection_dim": 0, - "position_embedding_type": "absolute" - } - }` - default: - return `{ - "model": { - "_name_or_path": "distilbert-base-uncased", - "activation": "gelu", - "add_cross_attention": false, - "architectures": [ - "DistilBertModel" - ], - "attention_dropout": 0.1, - "bad_words_ids": null, - "bos_token_id": null, - "chunk_size_feed_forward": 0, - "decoder_start_token_id": null, - "dim": 768, - "diversity_penalty": 0, - "do_sample": false, - "dropout": 0.1, - "early_stopping": false, - "encoder_no_repeat_ngram_size": 0, - "eos_token_id": null, - "finetuning_task": null, - "hidden_dim": 3072, - "id2label": { - "0": "LABEL_0", - "1": "LABEL_1" - }, - "initializer_range": 0.02, - "is_decoder": false, - "is_encoder_decoder": false, - "label2id": { - "LABEL_0": 0, - "LABEL_1": 1 - }, - "length_penalty": 1, - "max_length": 20, - "max_position_embeddings": 512, - "min_length": 0, - "model_type": "distilbert", - "n_heads": 12, - "n_layers": 6, - "no_repeat_ngram_size": 0, - "num_beam_groups": 1, - "num_beams": 1, - "num_return_sequences": 1, - "output_attentions": false, - "output_hidden_states": false, - "output_scores": false, - "pad_token_id": 0, - "prefix": null, - "pruned_heads": {}, - "qa_dropout": 0.1, - "repetition_penalty": 1, - "return_dict": true, - "return_dict_in_generate": false, - "sep_token_id": null, - "seq_classif_dropout": 0.2, - "sinusoidal_pos_embds": false, - "task_specific_params": null, - "temperature": 1, - "tie_encoder_decoder": false, - "tie_weights_": true, - "tie_word_embeddings": true, - "tokenizer_class": null, - "top_k": 50, - "top_p": 1, - "torchscript": false, - "transformers_version": "4.3.2", - "use_bfloat16": false, - "vocab_size": 30522, - "xla_device": null - } - }` - } -} - -func extractChildMap(t *testing.T, parent map[string]interface{}, name string) map[string]interface{} { - assert.NotNil(t, parent[name]) - child, ok := parent[name].(map[string]interface{}) - assert.True(t, ok) - assert.NotNil(t, child) - - return child -} diff --git a/modules/text2vec-transformers/clients/startup.go b/modules/text2vec-transformers/clients/startup.go deleted file mode 100644 index a3b6b4d5507acd371308d2e21164a886adb6e5ae..0000000000000000000000000000000000000000 --- a/modules/text2vec-transformers/clients/startup.go +++ /dev/null @@ -1,109 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "context" - "net/http" - "strings" - "sync" - "time" - - "github.com/pkg/errors" -) - -func (v *vectorizer) WaitForStartup(initCtx context.Context, - interval time.Duration, -) error { - endpoints := map[string]string{} - if v.originPassage != v.originQuery { - endpoints["passage"] = v.urlPassage("/.well-known/ready") - endpoints["query"] = v.urlQuery("/.well-known/ready") - } else { - endpoints[""] = v.urlPassage("/.well-known/ready") - } - - ch := make(chan error, len(endpoints)) - var wg sync.WaitGroup - for serviceName, endpoint := range endpoints { - wg.Add(1) - go func(serviceName string, endpoint string) { - defer wg.Done() - if err := v.waitFor(initCtx, interval, endpoint, serviceName); err != nil { - ch <- err - } - }(serviceName, endpoint) - } - wg.Wait() - close(ch) - - if len(ch) > 0 { - var errs []string - for err := range ch { - errs = append(errs, err.Error()) - } - return errors.Errorf(strings.Join(errs, ", ")) - } - return nil -} - -func (v *vectorizer) waitFor(initCtx context.Context, interval time.Duration, endpoint string, serviceName string) error { - ticker := time.NewTicker(interval) - defer ticker.Stop() - expired := initCtx.Done() - var lastErr error - prefix := "" - if serviceName != "" { - prefix = "[" + serviceName + "] " - } - - for { - select { - case <-ticker.C: - lastErr = v.checkReady(initCtx, endpoint, serviceName) - if lastErr == nil { - return nil - } - v.logger. - WithField("action", "transformer_remote_wait_for_startup"). - WithError(lastErr).Warnf("%stransformer remote inference service not ready", prefix) - case <-expired: - return errors.Wrapf(lastErr, "%sinit context expired before remote was ready", prefix) - } - } -} - -func (v *vectorizer) checkReady(initCtx context.Context, endpoint string, serviceName string) error { - // spawn a new context (derived on the overall context) which is used to - // consider an individual request timed out - // due to parent timeout being superior over request's one, request can be cancelled by parent timeout - // resulting in "send check ready request" even if service is responding with non 2xx http code - requestCtx, cancel := context.WithTimeout(initCtx, 500*time.Millisecond) - defer cancel() - - req, err := http.NewRequestWithContext(requestCtx, http.MethodGet, endpoint, nil) - if err != nil { - return errors.Wrap(err, "create check ready request") - } - - res, err := v.httpClient.Do(req) - if err != nil { - return errors.Wrap(err, "send check ready request") - } - - defer res.Body.Close() - if res.StatusCode > 299 { - return errors.Errorf("not ready: status %d", res.StatusCode) - } - - return nil -} diff --git a/modules/text2vec-transformers/clients/startup_test.go b/modules/text2vec-transformers/clients/startup_test.go deleted file mode 100644 index 2291d69c10ffbcdb0d28a7242baae0b7bf5a7d04..0000000000000000000000000000000000000000 --- a/modules/text2vec-transformers/clients/startup_test.go +++ /dev/null @@ -1,197 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "context" - "net/http" - "net/http/httptest" - "regexp" - "strings" - "testing" - "time" - - "github.com/sirupsen/logrus" - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestWaitForStartup(t *testing.T) { - t.Run("when common server is immediately ready", func(t *testing.T) { - server := httptest.NewServer(&testReadyHandler{t: t}) - defer server.Close() - v := New(server.URL, server.URL, 0, nullLogger()) - err := v.WaitForStartup(context.Background(), 150*time.Millisecond) - - assert.Nil(t, err) - }) - - t.Run("when passage and query servers are immediately ready", func(t *testing.T) { - serverPassage := httptest.NewServer(&testReadyHandler{t: t}) - serverQuery := httptest.NewServer(&testReadyHandler{t: t}) - defer serverPassage.Close() - defer serverQuery.Close() - v := New(serverPassage.URL, serverQuery.URL, 0, nullLogger()) - err := v.WaitForStartup(context.Background(), 150*time.Millisecond) - - assert.Nil(t, err) - }) - - t.Run("when common server is down", func(t *testing.T) { - url := "http://nothing-running-at-this-url" - v := New(url, url, 0, nullLogger()) - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancel() - err := v.WaitForStartup(ctx, 50*time.Millisecond) - - require.NotNil(t, err, nullLogger()) - assert.Contains(t, err.Error(), "init context expired before remote was ready: send check ready request") - assertContainsEither(t, err.Error(), "dial tcp", "context deadline exceeded") - assert.NotContains(t, err.Error(), "[passage]") - assert.NotContains(t, err.Error(), "[query]") - }) - - t.Run("when passage and query servers are down", func(t *testing.T) { - urlPassage := "http://nothing-running-at-this-url" - urlQuery := "http://nothing-running-at-this-url-either" - v := New(urlPassage, urlQuery, 0, nullLogger()) - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancel() - err := v.WaitForStartup(ctx, 50*time.Millisecond) - - require.NotNil(t, err, nullLogger()) - assert.Contains(t, err.Error(), "[passage] init context expired before remote was ready: send check ready request") - assert.Contains(t, err.Error(), "[query] init context expired before remote was ready: send check ready request") - assertContainsEither(t, err.Error(), "dial tcp", "context deadline exceeded") - }) - - t.Run("when common server is alive, but not ready", func(t *testing.T) { - server := httptest.NewServer(&testReadyHandler{ - t: t, - readyTime: time.Now().Add(time.Hour), - }) - defer server.Close() - v := New(server.URL, server.URL, 0, nullLogger()) - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancel() - err := v.WaitForStartup(ctx, 50*time.Millisecond) - - require.NotNil(t, err) - assert.Contains(t, err.Error(), "init context expired before remote was ready") - assertContainsEither(t, err.Error(), "not ready: status 503", "context deadline exceeded") - assert.NotContains(t, err.Error(), "[passage]") - assert.NotContains(t, err.Error(), "[query]") - }) - - t.Run("when passage and query servers are alive, but not ready", func(t *testing.T) { - rt := time.Now().Add(time.Hour) - serverPassage := httptest.NewServer(&testReadyHandler{ - t: t, - readyTime: rt, - }) - serverQuery := httptest.NewServer(&testReadyHandler{ - t: t, - readyTime: rt, - }) - defer serverPassage.Close() - defer serverQuery.Close() - v := New(serverPassage.URL, serverQuery.URL, 0, nullLogger()) - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancel() - err := v.WaitForStartup(ctx, 50*time.Millisecond) - - require.NotNil(t, err) - assert.Contains(t, err.Error(), "[passage] init context expired before remote was ready") - assert.Contains(t, err.Error(), "[query] init context expired before remote was ready") - assertContainsEither(t, err.Error(), "not ready: status 503", "context deadline exceeded") - }) - - t.Run("when passage and query servers are alive, but query one is not ready", func(t *testing.T) { - serverPassage := httptest.NewServer(&testReadyHandler{t: t}) - serverQuery := httptest.NewServer(&testReadyHandler{ - t: t, - readyTime: time.Now().Add(1 * time.Minute), - }) - defer serverPassage.Close() - defer serverQuery.Close() - v := New(serverPassage.URL, serverQuery.URL, 0, nullLogger()) - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancel() - err := v.WaitForStartup(ctx, 50*time.Millisecond) - - require.NotNil(t, err) - assert.Contains(t, err.Error(), "[query] init context expired before remote was ready") - assertContainsEither(t, err.Error(), "not ready: status 503", "context deadline exceeded") - assert.NotContains(t, err.Error(), "[passage]") - }) - - t.Run("when common server is initially not ready, but then becomes ready", func(t *testing.T) { - server := httptest.NewServer(&testReadyHandler{ - t: t, - readyTime: time.Now().Add(100 * time.Millisecond), - }) - v := New(server.URL, server.URL, 0, nullLogger()) - defer server.Close() - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancel() - err := v.WaitForStartup(ctx, 50*time.Millisecond) - - require.Nil(t, err) - }) - - t.Run("when passage and query servers are initially not ready, but then become ready", func(t *testing.T) { - serverPassage := httptest.NewServer(&testReadyHandler{ - t: t, - readyTime: time.Now().Add(100 * time.Millisecond), - }) - serverQuery := httptest.NewServer(&testReadyHandler{ - t: t, - readyTime: time.Now().Add(150 * time.Millisecond), - }) - defer serverPassage.Close() - defer serverQuery.Close() - v := New(serverPassage.URL, serverQuery.URL, 0, nullLogger()) - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancel() - err := v.WaitForStartup(ctx, 50*time.Millisecond) - - require.Nil(t, err) - }) -} - -type testReadyHandler struct { - t *testing.T - // the test handler will report as not ready before the time has passed - readyTime time.Time -} - -func (f *testReadyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - assert.Equal(f.t, "/.well-known/ready", r.URL.String()) - assert.Equal(f.t, http.MethodGet, r.Method) - - if time.Since(f.readyTime) < 0 { - w.WriteHeader(http.StatusServiceUnavailable) - } else { - w.WriteHeader(http.StatusNoContent) - } -} - -func nullLogger() logrus.FieldLogger { - l, _ := test.NewNullLogger() - return l -} - -func assertContainsEither(t *testing.T, str string, contains ...string) { - reg := regexp.MustCompile(strings.Join(contains, "|")) - assert.Regexp(t, reg, str) -} diff --git a/modules/text2vec-transformers/clients/transformers.go b/modules/text2vec-transformers/clients/transformers.go deleted file mode 100644 index ca8d24857f27141c323cae7473ccfb07b1d242bf..0000000000000000000000000000000000000000 --- a/modules/text2vec-transformers/clients/transformers.go +++ /dev/null @@ -1,123 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/modules/text2vec-transformers/ent" -) - -type vectorizer struct { - originPassage string - originQuery string - httpClient *http.Client - logger logrus.FieldLogger -} - -func New(originPassage, originQuery string, timeout time.Duration, logger logrus.FieldLogger) *vectorizer { - return &vectorizer{ - originPassage: originPassage, - originQuery: originQuery, - httpClient: &http.Client{ - Timeout: timeout, - }, - logger: logger, - } -} - -func (v *vectorizer) VectorizeObject(ctx context.Context, input string, - config ent.VectorizationConfig, -) (*ent.VectorizationResult, error) { - return v.vectorize(ctx, input, config, v.urlPassage) -} - -func (v *vectorizer) VectorizeQuery(ctx context.Context, input string, - config ent.VectorizationConfig, -) (*ent.VectorizationResult, error) { - return v.vectorize(ctx, input, config, v.urlQuery) -} - -func (v *vectorizer) vectorize(ctx context.Context, input string, - config ent.VectorizationConfig, url func(string) string, -) (*ent.VectorizationResult, error) { - body, err := json.Marshal(vecRequest{ - Text: input, - Config: vecRequestConfig{ - PoolingStrategy: config.PoolingStrategy, - }, - }) - if err != nil { - return nil, errors.Wrapf(err, "marshal body") - } - - req, err := http.NewRequestWithContext(ctx, "POST", url("/vectors"), - bytes.NewReader(body)) - if err != nil { - return nil, errors.Wrap(err, "create POST request") - } - - res, err := v.httpClient.Do(req) - if err != nil { - return nil, errors.Wrap(err, "send POST request") - } - defer res.Body.Close() - - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - return nil, errors.Wrap(err, "read response body") - } - - var resBody vecRequest - if err := json.Unmarshal(bodyBytes, &resBody); err != nil { - return nil, errors.Wrap(err, "unmarshal response body") - } - - if res.StatusCode > 399 { - return nil, errors.Errorf("fail with status %d: %s", res.StatusCode, - resBody.Error) - } - - return &ent.VectorizationResult{ - Text: resBody.Text, - Dimensions: resBody.Dims, - Vector: resBody.Vector, - }, nil -} - -func (v *vectorizer) urlPassage(path string) string { - return fmt.Sprintf("%s%s", v.originPassage, path) -} - -func (v *vectorizer) urlQuery(path string) string { - return fmt.Sprintf("%s%s", v.originQuery, path) -} - -type vecRequest struct { - Text string `json:"text"` - Dims int `json:"dims"` - Vector []float32 `json:"vector"` - Error string `json:"error"` - Config vecRequestConfig `json:"config"` -} - -type vecRequestConfig struct { - PoolingStrategy string `json:"pooling_strategy"` -} diff --git a/modules/text2vec-transformers/clients/transformers_test.go b/modules/text2vec-transformers/clients/transformers_test.go deleted file mode 100644 index 5f2afd5c5f56f628220a7608d74807c40802fe38..0000000000000000000000000000000000000000 --- a/modules/text2vec-transformers/clients/transformers_test.go +++ /dev/null @@ -1,114 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package clients - -import ( - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/pkg/errors" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/modules/text2vec-transformers/ent" -) - -func TestClient(t *testing.T) { - t.Run("when all is fine", func(t *testing.T) { - server := httptest.NewServer(&fakeHandler{t: t}) - defer server.Close() - c := New(server.URL, server.URL, 0, nullLogger()) - expected := &ent.VectorizationResult{ - Text: "This is my text", - Vector: []float32{0.1, 0.2, 0.3}, - Dimensions: 3, - } - res, err := c.VectorizeObject(context.Background(), "This is my text", - ent.VectorizationConfig{ - PoolingStrategy: "masked_mean", - }) - - assert.Nil(t, err) - assert.Equal(t, expected, res) - }) - - t.Run("when the context is expired", func(t *testing.T) { - server := httptest.NewServer(&fakeHandler{t: t}) - defer server.Close() - c := New(server.URL, server.URL, 0, nullLogger()) - ctx, cancel := context.WithDeadline(context.Background(), time.Now()) - defer cancel() - - _, err := c.VectorizeObject(ctx, "This is my text", ent.VectorizationConfig{}) - - require.NotNil(t, err) - assert.Contains(t, err.Error(), "context deadline exceeded") - }) - - t.Run("when the server returns an error", func(t *testing.T) { - server := httptest.NewServer(&fakeHandler{ - t: t, - serverError: errors.Errorf("nope, not gonna happen"), - }) - defer server.Close() - c := New(server.URL, server.URL, 0, nullLogger()) - _, err := c.VectorizeObject(context.Background(), "This is my text", - ent.VectorizationConfig{}) - - require.NotNil(t, err) - assert.Contains(t, err.Error(), "nope, not gonna happen") - }) -} - -type fakeHandler struct { - t *testing.T - serverError error -} - -func (f *fakeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - assert.Equal(f.t, "/vectors", r.URL.String()) - assert.Equal(f.t, http.MethodPost, r.Method) - - if f.serverError != nil { - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(fmt.Sprintf(`{"error":"%s"}`, f.serverError.Error()))) - return - } - - bodyBytes, err := io.ReadAll(r.Body) - require.Nil(f.t, err) - defer r.Body.Close() - - var b map[string]interface{} - require.Nil(f.t, json.Unmarshal(bodyBytes, &b)) - - textInput := b["text"].(string) - assert.Greater(f.t, len(textInput), 0) - - pooling := b["config"].(map[string]interface{})["pooling_strategy"].(string) - assert.Equal(f.t, "masked_mean", pooling) - - out := map[string]interface{}{ - "text": textInput, - "dims": 3, - "vector": []float32{0.1, 0.2, 0.3}, - } - outBytes, err := json.Marshal(out) - require.Nil(f.t, err) - - w.Write(outBytes) -} diff --git a/modules/text2vec-transformers/config.go b/modules/text2vec-transformers/config.go deleted file mode 100644 index 6581c246f9994556729592094b10ee715af9d1c1..0000000000000000000000000000000000000000 --- a/modules/text2vec-transformers/config.go +++ /dev/null @@ -1,153 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modtransformers - -import ( - "context" - "fmt" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/modules/text2vec-transformers/vectorizer" -) - -func (m *TransformersModule) ClassConfigDefaults() map[string]interface{} { - return map[string]interface{}{ - "vectorizeClassName": vectorizer.DefaultVectorizeClassName, - "poolingStrategy": vectorizer.DefaultPoolingStrategy, - } -} - -func (m *TransformersModule) PropertyConfigDefaults( - dt *schema.DataType, -) map[string]interface{} { - return map[string]interface{}{ - "skip": !vectorizer.DefaultPropertyIndexed, - "vectorizePropertyName": vectorizer.DefaultVectorizePropertyName, - } -} - -func (m *TransformersModule) ValidateClass(ctx context.Context, - class *models.Class, cfg moduletools.ClassConfig, -) error { - settings := vectorizer.NewClassSettings(cfg) - return NewConfigValidator(m.logger).Do(ctx, class, cfg, settings) -} - -var _ = modulecapabilities.ClassConfigurator(New()) - -type ConfigValidator struct { - logger logrus.FieldLogger -} - -type ClassSettings interface { - VectorizeClassName() bool - VectorizePropertyName(propName string) bool - PropertyIndexed(propName string) bool -} - -func NewConfigValidator(logger logrus.FieldLogger) *ConfigValidator { - return &ConfigValidator{logger: logger} -} - -func (cv *ConfigValidator) Do(ctx context.Context, class *models.Class, - cfg moduletools.ClassConfig, settings ClassSettings, -) error { - // In text2vec-transformers (as opposed to e.g. text2vec-contextionary) the - // assumption is that the models will be able to deal with any words, even - // previously unseen ones. Therefore we do not need to validate individual - // properties, but only the overall "index state" - - if err := cv.validateIndexState(ctx, class, settings); err != nil { - return errors.Errorf("invalid combination of properties") - } - - cv.checkForPossibilityOfDuplicateVectors(ctx, class, settings) - - return nil -} - -func (cv *ConfigValidator) validateIndexState(ctx context.Context, - class *models.Class, settings ClassSettings, -) error { - if settings.VectorizeClassName() { - // if the user chooses to vectorize the classname, vector-building will - // always be possible, no need to investigate further - - return nil - } - - // search if there is at least one indexed, string/text prop. If found pass - // validation - for _, prop := range class.Properties { - if len(prop.DataType) < 1 { - return errors.Errorf("property %s must have at least one datatype: "+ - "got %v", prop.Name, prop.DataType) - } - - if prop.DataType[0] != string(schema.DataTypeText) { - // we can only vectorize text-like props - continue - } - - if settings.PropertyIndexed(prop.Name) { - // found at least one, this is a valid schema - return nil - } - } - - return fmt.Errorf("invalid properties: didn't find a single property which is " + - "of type string or text and is not excluded from indexing. In addition the " + - "class name is excluded from vectorization as well, meaning that it cannot be " + - "used to determine the vector position. To fix this, set 'vectorizeClassName' " + - "to true if the class name is contextionary-valid. Alternatively add at least " + - "contextionary-valid text/string property which is not excluded from " + - "indexing.") -} - -func (cv *ConfigValidator) checkForPossibilityOfDuplicateVectors( - ctx context.Context, class *models.Class, settings ClassSettings, -) { - if !settings.VectorizeClassName() { - // if the user choses not to vectorize the class name, this means they must - // have chosen something else to vectorize, otherwise the validation would - // have error'd before we ever got here. We can skip further checking. - - return - } - - // search if there is at least one indexed, string/text prop. If found exit - for _, prop := range class.Properties { - // length check skipped, because validation has already passed - if prop.DataType[0] != string(schema.DataTypeText) { - // we can only vectorize text-like props - continue - } - - if settings.PropertyIndexed(prop.Name) { - // found at least one - return - } - } - - cv.logger.WithField("module", "text2vec-transformers"). - WithField("class", class.Class). - Warnf("text2vec-contextionary: Class %q does not have any properties "+ - "indexed (or only non text-properties indexed) and the vector position is "+ - "only determined by the class name. Each object will end up with the same "+ - "vector which leads to a severe performance penalty on imports. Consider "+ - "setting vectorIndexConfig.skip=true for this property", class.Class) -} diff --git a/modules/text2vec-transformers/config_test.go b/modules/text2vec-transformers/config_test.go deleted file mode 100644 index 10701e897f186dce469925ab6bbb8d462e6525c6..0000000000000000000000000000000000000000 --- a/modules/text2vec-transformers/config_test.go +++ /dev/null @@ -1,188 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modtransformers - -import ( - "context" - "testing" - - "github.com/sirupsen/logrus" - ltest "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" -) - -func TestConfigDefaults(t *testing.T) { - t.Run("for properties", func(t *testing.T) { - def := New().ClassConfigDefaults() - - assert.Equal(t, true, def["vectorizeClassName"]) - assert.Equal(t, "masked_mean", def["poolingStrategy"]) - }) - - t.Run("for the class", func(t *testing.T) { - dt := schema.DataTypeText - def := New().PropertyConfigDefaults(&dt) - assert.Equal(t, false, def["vectorizePropertyName"]) - assert.Equal(t, false, def["skip"]) - }) -} - -func TestConfigValidator(t *testing.T) { - t.Run("all usable props no-indexed", func(t *testing.T) { - t.Run("all schema vectorization turned off", func(t *testing.T) { - class := &models.Class{ - Vectorizer: "text2vec-contextionary", - Class: "ValidName", - Properties: []*models.Property{ - { - DataType: []string{"text"}, - Name: "description", - }, - { - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - Name: "name", - }, - { - DataType: []string{"int"}, - Name: "amount", - }, - }, - } - - logger, _ := ltest.NewNullLogger() - v := NewConfigValidator(logger) - err := v.Do(context.Background(), class, nil, &fakeIndexChecker{ - vectorizePropertyName: false, - vectorizeClassName: false, - propertyIndexed: false, - }) - assert.NotNil(t, err) - }) - }) -} - -func TestConfigValidator_RiskOfDuplicateVectors(t *testing.T) { - type test struct { - name string - in *models.Class - expectWarning bool - indexChecker *fakeIndexChecker - } - - tests := []test{ - { - name: "usable properties", - in: &models.Class{ - Class: "ValidName", - Properties: []*models.Property{ - { - DataType: []string{string(schema.DataTypeText)}, - Name: "textProp", - }, - }, - }, - expectWarning: false, - indexChecker: &fakeIndexChecker{ - vectorizePropertyName: false, - vectorizeClassName: true, - propertyIndexed: true, - }, - }, - { - name: "no properties", - in: &models.Class{ - Class: "ValidName", - }, - expectWarning: true, - indexChecker: &fakeIndexChecker{ - vectorizePropertyName: false, - vectorizeClassName: true, - propertyIndexed: false, - }, - }, - { - name: "usable properties, but they are no-indexed", - in: &models.Class{ - Class: "ValidName", - Properties: []*models.Property{ - { - DataType: []string{string(schema.DataTypeText)}, - Name: "textProp", - }, - }, - }, - expectWarning: true, - indexChecker: &fakeIndexChecker{ - vectorizePropertyName: false, - vectorizeClassName: true, - propertyIndexed: false, - }, - }, - { - name: "only unusable properties", - in: &models.Class{ - Class: "ValidName", - Properties: []*models.Property{ - { - DataType: []string{string(schema.DataTypeInt)}, - Name: "intProp", - }, - }, - }, - expectWarning: true, - indexChecker: &fakeIndexChecker{ - vectorizePropertyName: false, - vectorizeClassName: true, - propertyIndexed: false, - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - logger, hook := ltest.NewNullLogger() - v := NewConfigValidator(logger) - err := v.Do(context.Background(), test.in, nil, test.indexChecker) - require.Nil(t, err) - - entry := hook.LastEntry() - if test.expectWarning { - require.NotNil(t, entry) - assert.Equal(t, logrus.WarnLevel, entry.Level) - } else { - assert.Nil(t, entry) - } - }) - } -} - -type fakeIndexChecker struct { - vectorizeClassName bool - vectorizePropertyName bool - propertyIndexed bool -} - -func (f *fakeIndexChecker) VectorizeClassName() bool { - return f.vectorizeClassName -} - -func (f *fakeIndexChecker) VectorizePropertyName(propName string) bool { - return f.vectorizePropertyName -} - -func (f *fakeIndexChecker) PropertyIndexed(propName string) bool { - return f.propertyIndexed -} diff --git a/modules/text2vec-transformers/ent/vectorization_config.go b/modules/text2vec-transformers/ent/vectorization_config.go deleted file mode 100644 index 8d77fe17cb65ca5049d94238775bc8dc6ccfd161..0000000000000000000000000000000000000000 --- a/modules/text2vec-transformers/ent/vectorization_config.go +++ /dev/null @@ -1,16 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package ent - -type VectorizationConfig struct { - PoolingStrategy string -} diff --git a/modules/text2vec-transformers/ent/vectorization_result.go b/modules/text2vec-transformers/ent/vectorization_result.go deleted file mode 100644 index 69eee4438e46a4427a7368204c662419196cc526..0000000000000000000000000000000000000000 --- a/modules/text2vec-transformers/ent/vectorization_result.go +++ /dev/null @@ -1,18 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package ent - -type VectorizationResult struct { - Text string - Dimensions int - Vector []float32 -} diff --git a/modules/text2vec-transformers/module.go b/modules/text2vec-transformers/module.go deleted file mode 100644 index 53cd08af717e0c505b09cd0ee674e9356776ced4..0000000000000000000000000000000000000000 --- a/modules/text2vec-transformers/module.go +++ /dev/null @@ -1,169 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modtransformers - -import ( - "context" - "net/http" - "os" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/modules/text2vec-transformers/clients" - "github.com/weaviate/weaviate/modules/text2vec-transformers/vectorizer" - "github.com/weaviate/weaviate/usecases/modulecomponents/additional" -) - -func New() *TransformersModule { - return &TransformersModule{} -} - -type TransformersModule struct { - vectorizer textVectorizer - metaProvider metaProvider - graphqlProvider modulecapabilities.GraphQLArguments - searcher modulecapabilities.Searcher - nearTextTransformer modulecapabilities.TextTransform - logger logrus.FieldLogger - additionalPropertiesProvider modulecapabilities.AdditionalProperties -} - -type textVectorizer interface { - Object(ctx context.Context, obj *models.Object, objDiff *moduletools.ObjectDiff, - cfg moduletools.ClassConfig) error - Texts(ctx context.Context, input []string, - cfg moduletools.ClassConfig) ([]float32, error) -} - -type metaProvider interface { - MetaInfo() (map[string]interface{}, error) -} - -func (m *TransformersModule) Name() string { - return "text2vec-transformers" -} - -func (m *TransformersModule) Type() modulecapabilities.ModuleType { - return modulecapabilities.Text2Vec -} - -func (m *TransformersModule) Init(ctx context.Context, - params moduletools.ModuleInitParams, -) error { - m.logger = params.GetLogger() - - if err := m.initVectorizer(ctx, params.GetConfig().ModuleHttpClientTimeout, m.logger); err != nil { - return errors.Wrap(err, "init vectorizer") - } - - if err := m.initAdditionalPropertiesProvider(); err != nil { - return errors.Wrap(err, "init additional properties provider") - } - - return nil -} - -func (m *TransformersModule) InitExtension(modules []modulecapabilities.Module) error { - for _, module := range modules { - if module.Name() == m.Name() { - continue - } - if arg, ok := module.(modulecapabilities.TextTransformers); ok { - if arg != nil && arg.TextTransformers() != nil { - m.nearTextTransformer = arg.TextTransformers()["nearText"] - } - } - } - - if err := m.initNearText(); err != nil { - return errors.Wrap(err, "init graphql provider") - } - return nil -} - -func (m *TransformersModule) initVectorizer(ctx context.Context, timeout time.Duration, - logger logrus.FieldLogger, -) error { - // TODO: gh-1486 proper config management - uriPassage := os.Getenv("TRANSFORMERS_PASSAGE_INFERENCE_API") - uriQuery := os.Getenv("TRANSFORMERS_QUERY_INFERENCE_API") - uriCommon := os.Getenv("TRANSFORMERS_INFERENCE_API") - - if uriCommon == "" { - if uriPassage == "" && uriQuery == "" { - return errors.Errorf("required variable TRANSFORMERS_INFERENCE_API or both variables TRANSFORMERS_PASSAGE_INFERENCE_API and TRANSFORMERS_QUERY_INFERENCE_API are not set") - } - if uriPassage != "" && uriQuery == "" { - return errors.Errorf("required variable TRANSFORMERS_QUERY_INFERENCE_API is not set") - } - if uriPassage == "" && uriQuery != "" { - return errors.Errorf("required variable TRANSFORMERS_PASSAGE_INFERENCE_API is not set") - } - } else { - if uriPassage != "" || uriQuery != "" { - return errors.Errorf("either variable TRANSFORMERS_INFERENCE_API or both variables TRANSFORMERS_PASSAGE_INFERENCE_API and TRANSFORMERS_QUERY_INFERENCE_API should be set") - } - uriPassage = uriCommon - uriQuery = uriCommon - } - - client := clients.New(uriPassage, uriQuery, timeout, logger) - if err := client.WaitForStartup(ctx, 1*time.Second); err != nil { - return errors.Wrap(err, "init remote vectorizer") - } - - m.vectorizer = vectorizer.New(client) - m.metaProvider = client - - return nil -} - -func (m *TransformersModule) initAdditionalPropertiesProvider() error { - m.additionalPropertiesProvider = additional.NewText2VecProvider() - return nil -} - -func (m *TransformersModule) RootHandler() http.Handler { - // TODO: remove once this is a capability interface - return nil -} - -func (m *TransformersModule) VectorizeObject(ctx context.Context, - obj *models.Object, objDiff *moduletools.ObjectDiff, cfg moduletools.ClassConfig, -) error { - return m.vectorizer.Object(ctx, obj, objDiff, cfg) -} - -func (m *TransformersModule) MetaInfo() (map[string]interface{}, error) { - return m.metaProvider.MetaInfo() -} - -func (m *TransformersModule) AdditionalProperties() map[string]modulecapabilities.AdditionalProperty { - return m.additionalPropertiesProvider.AdditionalProperties() -} - -func (m *TransformersModule) VectorizeInput(ctx context.Context, - input string, cfg moduletools.ClassConfig, -) ([]float32, error) { - return m.vectorizer.Texts(ctx, []string{input}, cfg) -} - -// verify we implement the modules.Module interface -var ( - _ = modulecapabilities.Module(New()) - _ = modulecapabilities.Vectorizer(New()) - _ = modulecapabilities.MetaProvider(New()) -) diff --git a/modules/text2vec-transformers/nearText.go b/modules/text2vec-transformers/nearText.go deleted file mode 100644 index 232e6ff8b7866e2279904669322d2b0956cda8dd..0000000000000000000000000000000000000000 --- a/modules/text2vec-transformers/nearText.go +++ /dev/null @@ -1,36 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package modtransformers - -import ( - "github.com/weaviate/weaviate/entities/modulecapabilities" - "github.com/weaviate/weaviate/usecases/modulecomponents/nearText" -) - -func (m *TransformersModule) initNearText() error { - m.searcher = nearText.NewSearcher(m.vectorizer) - m.graphqlProvider = nearText.New(m.nearTextTransformer) - return nil -} - -func (m *TransformersModule) Arguments() map[string]modulecapabilities.GraphQLArgument { - return m.graphqlProvider.Arguments() -} - -func (m *TransformersModule) VectorSearches() map[string]modulecapabilities.VectorForParams { - return m.searcher.VectorSearches() -} - -var ( - _ = modulecapabilities.GraphQLArguments(New()) - _ = modulecapabilities.Searcher(New()) -) diff --git a/modules/text2vec-transformers/vectorizer/class_settings.go b/modules/text2vec-transformers/vectorizer/class_settings.go deleted file mode 100644 index a34a7f32516d3d7b2088a0461e35cfa3c5a154f6..0000000000000000000000000000000000000000 --- a/modules/text2vec-transformers/vectorizer/class_settings.go +++ /dev/null @@ -1,106 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "github.com/weaviate/weaviate/entities/moduletools" -) - -const ( - DefaultPropertyIndexed = true - DefaultVectorizeClassName = true - DefaultVectorizePropertyName = false - DefaultPoolingStrategy = "masked_mean" -) - -type classSettings struct { - cfg moduletools.ClassConfig -} - -func NewClassSettings(cfg moduletools.ClassConfig) *classSettings { - return &classSettings{cfg: cfg} -} - -func (ic *classSettings) PropertyIndexed(propName string) bool { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return DefaultPropertyIndexed - } - - vcn, ok := ic.cfg.Property(propName)["skip"] - if !ok { - return DefaultPropertyIndexed - } - - asBool, ok := vcn.(bool) - if !ok { - return DefaultPropertyIndexed - } - - return !asBool -} - -func (ic *classSettings) VectorizePropertyName(propName string) bool { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return DefaultVectorizePropertyName - } - vcn, ok := ic.cfg.Property(propName)["vectorizePropertyName"] - if !ok { - return DefaultVectorizePropertyName - } - - asBool, ok := vcn.(bool) - if !ok { - return DefaultVectorizePropertyName - } - - return asBool -} - -func (ic *classSettings) VectorizeClassName() bool { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return DefaultVectorizeClassName - } - - vcn, ok := ic.cfg.Class()["vectorizeClassName"] - if !ok { - return DefaultVectorizeClassName - } - - asBool, ok := vcn.(bool) - if !ok { - return DefaultVectorizeClassName - } - - return asBool -} - -func (ic *classSettings) PoolingStrategy() string { - if ic.cfg == nil { - // we would receive a nil-config on cross-class requests, such as Explore{} - return DefaultPoolingStrategy - } - - vcn, ok := ic.cfg.Class()["poolingStrategy"] - if !ok { - return DefaultPoolingStrategy - } - - asString, ok := vcn.(string) - if !ok { - return DefaultPoolingStrategy - } - - return asString -} diff --git a/modules/text2vec-transformers/vectorizer/class_settings_test.go b/modules/text2vec-transformers/vectorizer/class_settings_test.go deleted file mode 100644 index 0eefbc55802e0453fcb72c9759d1b63deaa8e7e2..0000000000000000000000000000000000000000 --- a/modules/text2vec-transformers/vectorizer/class_settings_test.go +++ /dev/null @@ -1,110 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/usecases/modules" -) - -func TestClassSettings(t *testing.T) { - t.Run("with all defaults", func(t *testing.T) { - class := &models.Class{ - Class: "MyClass", - Properties: []*models.Property{{ - Name: "someProp", - }}, - } - - cfg := modules.NewClassBasedModuleConfig(class, "my-module", "tenant") - ic := NewClassSettings(cfg) - - assert.True(t, ic.PropertyIndexed("someProp")) - assert.False(t, ic.VectorizePropertyName("someProp")) - assert.True(t, ic.VectorizeClassName()) - assert.Equal(t, ic.PoolingStrategy(), "masked_mean") - }) - - t.Run("with a nil config", func(t *testing.T) { - // this is the case if we were running in a situation such as a - // cross-class vectorization of search time, as is the case with Explore - // {}, we then expect all default values - - ic := NewClassSettings(nil) - - assert.True(t, ic.PropertyIndexed("someProp")) - assert.False(t, ic.VectorizePropertyName("someProp")) - assert.True(t, ic.VectorizeClassName()) - assert.Equal(t, ic.PoolingStrategy(), "masked_mean") - }) - - t.Run("with all explicit config matching the defaults", func(t *testing.T) { - class := &models.Class{ - Class: "MyClass", - ModuleConfig: map[string]interface{}{ - "my-module": map[string]interface{}{ - "vectorizeClassName": true, - "poolingStrategy": "masked_mean", - }, - }, - Properties: []*models.Property{{ - Name: "someProp", - ModuleConfig: map[string]interface{}{ - "my-module": map[string]interface{}{ - "skip": false, - "vectorizePropertyName": false, - }, - }, - }}, - } - - cfg := modules.NewClassBasedModuleConfig(class, "my-module", "tenant") - ic := NewClassSettings(cfg) - - assert.True(t, ic.PropertyIndexed("someProp")) - assert.False(t, ic.VectorizePropertyName("someProp")) - assert.True(t, ic.VectorizeClassName()) - assert.Equal(t, ic.PoolingStrategy(), "masked_mean") - }) - - t.Run("with all explicit config using non-default values", func(t *testing.T) { - class := &models.Class{ - Class: "MyClass", - ModuleConfig: map[string]interface{}{ - "my-module": map[string]interface{}{ - "vectorizeClassName": false, - "poolingStrategy": "cls", - }, - }, - Properties: []*models.Property{{ - Name: "someProp", - ModuleConfig: map[string]interface{}{ - "my-module": map[string]interface{}{ - "skip": true, - "vectorizePropertyName": true, - }, - }, - }}, - } - - cfg := modules.NewClassBasedModuleConfig(class, "my-module", "tenant") - ic := NewClassSettings(cfg) - - assert.False(t, ic.PropertyIndexed("someProp")) - assert.True(t, ic.VectorizePropertyName("someProp")) - assert.False(t, ic.VectorizeClassName()) - assert.Equal(t, ic.PoolingStrategy(), "cls") - }) -} diff --git a/modules/text2vec-transformers/vectorizer/fakes_for_test.go b/modules/text2vec-transformers/vectorizer/fakes_for_test.go deleted file mode 100644 index 04e819f1071114a46130f0df11ae53429d177e2d..0000000000000000000000000000000000000000 --- a/modules/text2vec-transformers/vectorizer/fakes_for_test.go +++ /dev/null @@ -1,85 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - - "github.com/weaviate/weaviate/modules/text2vec-transformers/ent" -) - -type fakeClient struct { - lastInput string - lastConfig ent.VectorizationConfig -} - -func (c *fakeClient) VectorizeObject(ctx context.Context, - text string, cfg ent.VectorizationConfig, -) (*ent.VectorizationResult, error) { - c.lastInput = text - c.lastConfig = cfg - return &ent.VectorizationResult{ - Vector: []float32{0, 1, 2, 3}, - Dimensions: 4, - Text: text, - }, nil -} - -func (c *fakeClient) VectorizeQuery(ctx context.Context, - text string, cfg ent.VectorizationConfig, -) (*ent.VectorizationResult, error) { - return c.VectorizeObject(ctx, text, cfg) -} - -type fakeClassConfig struct { - classConfig map[string]interface{} - vectorizeClassName bool - vectorizePropertyName bool - skippedProperty string - excludedProperty string - poolingStrategy string -} - -func (f fakeClassConfig) Class() map[string]interface{} { - classSettings := map[string]interface{}{ - "vectorizeClassName": f.vectorizeClassName, - "poolingStrategy": f.poolingStrategy, - } - return classSettings -} - -func (f fakeClassConfig) ClassByModuleName(moduleName string) map[string]interface{} { - return f.classConfig -} - -func (f fakeClassConfig) Property(propName string) map[string]interface{} { - if propName == f.skippedProperty { - return map[string]interface{}{ - "skip": true, - } - } - if propName == f.excludedProperty { - return map[string]interface{}{ - "vectorizePropertyName": false, - } - } - if f.vectorizePropertyName { - return map[string]interface{}{ - "vectorizePropertyName": true, - } - } - return nil -} - -func (f fakeClassConfig) Tenant() string { - return "" -} diff --git a/modules/text2vec-transformers/vectorizer/objects.go b/modules/text2vec-transformers/vectorizer/objects.go deleted file mode 100644 index 0146a19e4c09ec89415d945b4cdf13a38d14a168..0000000000000000000000000000000000000000 --- a/modules/text2vec-transformers/vectorizer/objects.go +++ /dev/null @@ -1,82 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/modules/text2vec-transformers/ent" - objectsvectorizer "github.com/weaviate/weaviate/usecases/modulecomponents/vectorizer" -) - -type Vectorizer struct { - client Client - objectVectorizer *objectsvectorizer.ObjectVectorizer -} - -func New(client Client) *Vectorizer { - return &Vectorizer{ - client: client, - objectVectorizer: objectsvectorizer.New(), - } -} - -type Client interface { - VectorizeObject(ctx context.Context, input string, - cfg ent.VectorizationConfig) (*ent.VectorizationResult, error) - VectorizeQuery(ctx context.Context, input string, - cfg ent.VectorizationConfig) (*ent.VectorizationResult, error) -} - -// IndexCheck returns whether a property of a class should be indexed -type ClassSettings interface { - PropertyIndexed(property string) bool - VectorizeClassName() bool - VectorizePropertyName(propertyName string) bool - PoolingStrategy() string -} - -func (v *Vectorizer) Object(ctx context.Context, object *models.Object, - objDiff *moduletools.ObjectDiff, cfg moduletools.ClassConfig, -) error { - vec, err := v.object(ctx, object.Class, object.Properties, objDiff, cfg) - if err != nil { - return err - } - - object.Vector = vec - return nil -} - -func (v *Vectorizer) object(ctx context.Context, className string, - schema interface{}, objDiff *moduletools.ObjectDiff, cfg moduletools.ClassConfig, -) ([]float32, error) { - text, vector, err := v.objectVectorizer.TextsOrVector(ctx, className, schema, objDiff, NewClassSettings(cfg)) - if err != nil { - return nil, err - } - if vector != nil { - // dont' re-vectorize - return vector, nil - } - // vectorize text - res, err := v.client.VectorizeObject(ctx, text, ent.VectorizationConfig{ - PoolingStrategy: NewClassSettings(cfg).PoolingStrategy(), - }) - if err != nil { - return nil, err - } - - return res.Vector, nil -} diff --git a/modules/text2vec-transformers/vectorizer/objects_test.go b/modules/text2vec-transformers/vectorizer/objects_test.go deleted file mode 100644 index bd27bafa08e476d1508f48c9ba27c142d44ee046..0000000000000000000000000000000000000000 --- a/modules/text2vec-transformers/vectorizer/objects_test.go +++ /dev/null @@ -1,368 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/moduletools" -) - -// These are mostly copy/pasted (with minimal additions) from the -// text2vec-contextionary module -func TestVectorizingObjects(t *testing.T) { - type testCase struct { - name string - input *models.Object - expectedClientCall string - expectedPoolingStrategy string - noindex string - excludedProperty string // to simulate a schema where property names aren't vectorized - excludedClass string // to simulate a schema where class names aren't vectorized - poolingStrategy string - } - - tests := []testCase{ - { - name: "empty object", - input: &models.Object{ - Class: "Car", - }, - poolingStrategy: "cls", - expectedPoolingStrategy: "cls", - expectedClientCall: "car", - }, - { - name: "object with one string prop", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "Mercedes", - }, - }, - expectedClientCall: "car brand mercedes", - }, - - { - name: "object with one non-string prop", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "power": 300, - }, - }, - expectedClientCall: "car", - }, - - { - name: "object with a mix of props", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: "car brand best brand review a very great car", - }, - { - name: "with a noindexed property", - noindex: "review", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: "car brand best brand", - }, - - { - name: "with the class name not vectorized", - excludedClass: "Car", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: "brand best brand review a very great car", - }, - - { - name: "with a property name not vectorized", - excludedProperty: "review", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: "car brand best brand a very great car", - }, - - { - name: "with no schema labels vectorized", - excludedProperty: "review", - excludedClass: "Car", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "review": "a very great car", - }, - }, - expectedClientCall: "a very great car", - }, - - { - name: "with string/text arrays without propname or classname", - excludedProperty: "reviews", - excludedClass: "Car", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - expectedClientCall: "a very great car you should consider buying one", - }, - - { - name: "with string/text arrays with propname and classname", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - expectedClientCall: "car reviews a very great car reviews you should consider buying one", - }, - - { - name: "with compound class and prop names", - input: &models.Object{ - Class: "SuperCar", - Properties: map[string]interface{}{ - "brandOfTheCar": "best brand", - "power": 300, - "review": "a very great car", - }, - }, - expectedClientCall: "super car brand of the car best brand review a very great car", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - client := &fakeClient{} - - v := New(client) - - ic := &fakeClassConfig{ - excludedProperty: test.excludedProperty, - skippedProperty: test.noindex, - vectorizeClassName: test.excludedClass != "Car", - poolingStrategy: test.poolingStrategy, - vectorizePropertyName: true, - } - err := v.Object(context.Background(), test.input, nil, ic) - - require.Nil(t, err) - assert.Equal(t, models.C11yVector{0, 1, 2, 3}, test.input.Vector) - expected := strings.Split(test.expectedClientCall, " ") - actual := strings.Split(client.lastInput, " ") - assert.Equal(t, expected, actual) - assert.Equal(t, client.lastConfig.PoolingStrategy, test.expectedPoolingStrategy) - }) - } -} - -func TestVectorizingObjectsWithDiff(t *testing.T) { - type testCase struct { - name string - input *models.Object - skipped string - diff *moduletools.ObjectDiff - expectedVectorize bool - } - - tests := []testCase{ - { - name: "no diff", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: nil, - expectedVectorize: true, - }, - { - name: "diff all props unchanged", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("brand", "best brand", "best brand"). - WithProp("power", 300, 300). - WithProp("description", "a very great car", "a very great car"). - WithProp("reviews", []interface{}{ - "a very great car", - "you should consider buying one", - }, []interface{}{ - "a very great car", - "you should consider buying one", - }), - expectedVectorize: false, - }, - { - name: "diff one vectorizable prop changed (1)", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("brand", "old best brand", "best brand"), - expectedVectorize: true, - }, - { - name: "diff one vectorizable prop changed (2)", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("description", "old a very great car", "a very great car"), - expectedVectorize: true, - }, - { - name: "diff one vectorizable prop changed (3)", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("reviews", []interface{}{ - "old a very great car", - "you should consider buying one", - }, []interface{}{ - "a very great car", - "you should consider buying one", - }), - expectedVectorize: true, - }, - { - name: "all non-vectorizable props changed", - skipped: "description", - input: &models.Object{ - Class: "Car", - Properties: map[string]interface{}{ - "brand": "best brand", - "power": 300, - "description": "a very great car", - "reviews": []interface{}{ - "a very great car", - "you should consider buying one", - }, - }, - }, - diff: newObjectDiffWithVector(). - WithProp("power", 123, 300). - WithProp("description", "old a very great car", "a very great car"), - expectedVectorize: false, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - ic := &fakeClassConfig{ - skippedProperty: test.skipped, - } - - client := &fakeClient{} - v := New(client) - - err := v.Object(context.Background(), test.input, test.diff, ic) - - require.Nil(t, err) - if test.expectedVectorize { - assert.Equal(t, models.C11yVector{0, 1, 2, 3}, test.input.Vector) - assert.NotEmpty(t, client.lastInput) - } else { - assert.Equal(t, models.C11yVector{0, 0, 0, 0}, test.input.Vector) - assert.Empty(t, client.lastInput) - } - }) - } -} - -func newObjectDiffWithVector() *moduletools.ObjectDiff { - return moduletools.NewObjectDiff([]float32{0, 0, 0, 0}) -} diff --git a/modules/text2vec-transformers/vectorizer/texts.go b/modules/text2vec-transformers/vectorizer/texts.go deleted file mode 100644 index 280e3b355d249b0ec1fea056eacfe7b7454565e0..0000000000000000000000000000000000000000 --- a/modules/text2vec-transformers/vectorizer/texts.go +++ /dev/null @@ -1,38 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/moduletools" - "github.com/weaviate/weaviate/modules/text2vec-transformers/ent" - libvectorizer "github.com/weaviate/weaviate/usecases/vectorizer" -) - -func (v *Vectorizer) Texts(ctx context.Context, inputs []string, - cfg moduletools.ClassConfig, -) ([]float32, error) { - vectors := make([][]float32, len(inputs)) - for i := range inputs { - res, err := v.client.VectorizeQuery(ctx, inputs[i], ent.VectorizationConfig{ - PoolingStrategy: NewClassSettings(cfg).PoolingStrategy(), - }) - if err != nil { - return nil, errors.Wrap(err, "remote client vectorize") - } - vectors[i] = res.Vector - } - - return libvectorizer.CombineVectors(vectors), nil -} diff --git a/modules/text2vec-transformers/vectorizer/texts_test.go b/modules/text2vec-transformers/vectorizer/texts_test.go deleted file mode 100644 index 6d655c6631b114cafef7fd42ad32aaada3e84dc0..0000000000000000000000000000000000000000 --- a/modules/text2vec-transformers/vectorizer/texts_test.go +++ /dev/null @@ -1,94 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package vectorizer - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// as used in the nearText searcher -func TestVectorizingTexts(t *testing.T) { - type testCase struct { - name string - input []string - expectedPoolingStrategy string - poolingStrategy string - } - - tests := []testCase{ - { - name: "single word", - input: []string{"hello"}, - poolingStrategy: "cls", - expectedPoolingStrategy: "cls", - }, - { - name: "multiple words", - input: []string{"hello world, this is me!"}, - poolingStrategy: "cls", - expectedPoolingStrategy: "cls", - }, - - { - name: "multiple sentences (joined with a dot)", - input: []string{"this is sentence 1", "and here's number 2"}, - poolingStrategy: "cls", - expectedPoolingStrategy: "cls", - }, - - { - name: "multiple sentences already containing a dot", - input: []string{"this is sentence 1.", "and here's number 2"}, - poolingStrategy: "cls", - expectedPoolingStrategy: "cls", - }, - { - name: "multiple sentences already containing a question mark", - input: []string{"this is sentence 1?", "and here's number 2"}, - poolingStrategy: "cls", - expectedPoolingStrategy: "cls", - }, - { - name: "multiple sentences already containing an exclamation mark", - input: []string{"this is sentence 1!", "and here's number 2"}, - poolingStrategy: "cls", - expectedPoolingStrategy: "cls", - }, - { - name: "multiple sentences already containing comma", - input: []string{"this is sentence 1,", "and here's number 2"}, - poolingStrategy: "cls", - expectedPoolingStrategy: "cls", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - client := &fakeClient{} - - v := New(client) - - settings := &fakeClassConfig{ - poolingStrategy: test.poolingStrategy, - } - vec, err := v.Texts(context.Background(), test.input, settings) - - require.Nil(t, err) - assert.Equal(t, []float32{0, 1, 2, 3}, vec) - assert.Equal(t, client.lastConfig.PoolingStrategy, test.expectedPoolingStrategy) - }) - } -} diff --git a/multi-qa-MiniLM-L6-cos-v1 b/multi-qa-MiniLM-L6-cos-v1 deleted file mode 160000 index 38845167a107b59398111f0cfb430897cf1a4639..0000000000000000000000000000000000000000 --- a/multi-qa-MiniLM-L6-cos-v1 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 38845167a107b59398111f0cfb430897cf1a4639 diff --git a/openapi-specs/extendresponses.js b/openapi-specs/extendresponses.js deleted file mode 100644 index af4a5186d260958cf387ab25ffcca1ed2f68a691..0000000000000000000000000000000000000000 --- a/openapi-specs/extendresponses.js +++ /dev/null @@ -1,26 +0,0 @@ -/** - * A simple script to add a new response to every single API path. This was - * built for the purpose of adding 500 Internal Server Error to everything, but - * could potentially also be used for other purposes in the future. - */ - -const fs = require('fs') - -const file = fs.readFileSync('./schema.json', 'utf-8') -const parsed = JSON.parse(file) - -for (const [pathKey, pathValue] of Object.entries(parsed.paths)) { - for (const [path, value] of Object.entries(pathValue)) { - if (!value.responses) { - continue - } - - value.responses['500'] = { - description: "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - schema: { - "$ref": "#/definitions/ErrorResponse" - } - } - } -} -fs.writeFileSync('./schema.json', JSON.stringify(parsed, null, 2)) diff --git a/openapi-specs/schema.json b/openapi-specs/schema.json deleted file mode 100644 index 4565027c54fe3b70ff79f909204bbbe25c289194..0000000000000000000000000000000000000000 --- a/openapi-specs/schema.json +++ /dev/null @@ -1,4971 +0,0 @@ -{ - "basePath": "/v1", - "consumes": [ - "application/yaml", - "application/json" - ], - "definitions": { - "Link": { - "type": "object", - "properties": { - "href": { - "type": "string", - "description": "target of the link" - }, - "rel": { - "type": "string", - "description": "relationship if both resources are related, e.g. 'next', 'previous', 'parent', etc." - }, - "name": { - "type": "string", - "description": "human readable name of the resource group" - }, - "documentationHref": { - "type": "string", - "description": "weaviate documentation about this resource group" - } - } - }, - "Principal": { - "type": "object", - "properties": { - "username": { - "type": "string", - "description": "The username that was extracted either from the authentication information" - }, - "groups": { - "type": "array", - "items": { - "type": "string" - } - } - } - }, - "C11yWordsResponse": { - "description": "An array of available words and contexts.", - "properties": { - "concatenatedWord": { - "description": "Weighted results for all words", - "type": "object", - "properties": { - "concatenatedWord": { - "type": "string" - }, - "singleWords": { - "type": "array", - "items": { - "format": "string" - } - }, - "concatenatedVector": { - "$ref": "#/definitions/C11yVector" - }, - "concatenatedNearestNeighbors": { - "$ref": "#/definitions/C11yNearestNeighbors" - } - } - }, - "individualWords": { - "description": "Weighted results for per individual word", - "type": "array", - "items": { - "type": "object", - "properties": { - "word": { - "type": "string" - }, - "present": { - "type": "boolean" - }, - "info": { - "type": "object", - "properties": { - "vector": { - "$ref": "#/definitions/C11yVector" - }, - "nearestNeighbors": { - "$ref": "#/definitions/C11yNearestNeighbors" - } - } - } - } - } - } - } - }, - "C11yExtension": { - "description": "A resource describing an extension to the contextinoary, containing both the identifier and the definition of the extension", - "properties": { - "concept": { - "description": "The new concept you want to extend. Must be an all-lowercase single word, or a space delimited compound word. Examples: 'foobarium', 'my custom concept'", - "type": "string", - "example": "foobarium" - }, - "definition": { - "description": "A list of space-delimited words or a sentence describing what the custom concept is about. Avoid using the custom concept itself. An Example definition for the custom concept 'foobarium': would be 'a naturally occurring element which can only be seen by programmers'", - "type": "string" - }, - "weight": { - "description": "Weight of the definition of the new concept where 1='override existing definition entirely' and 0='ignore custom definition'. Note that if the custom concept is not present in the contextionary yet, the weight cannot be less than 1.", - "type": "number", - "format": "float" - } - } - }, - "C11yNearestNeighbors": { - "description": "C11y function to show the nearest neighbors to a word.", - "type": "array", - "items": { - "type": "object", - "properties": { - "word": { - "type": "string" - }, - "distance": { - "type": "number", - "format": "float" - } - } - } - }, - "C11yVector": { - "description": "A Vector in the Contextionary", - "type": "array", - "items": { - "type": "number", - "format": "float" - } - }, - "C11yVectorBasedQuestion": { - "description": "Receive question based on array of classes, properties and values.", - "type": "array", - "items": { - "type": "object", - "properties": { - "classVectors": { - "description": "Vectorized classname.", - "type": "array", - "items": { - "type": "number", - "format": "float" - }, - "minItems": 300, - "maxItems": 300 - }, - "classProps": { - "description": "Vectorized properties.", - "type": "array", - "items": { - "type": "object", - "properties": { - "propsVectors": { - "type": "array", - "items": { - "type": "number", - "format": "float" - } - }, - "value": { - "description": "String with valuename.", - "type": "string" - } - } - }, - "minItems": 300, - "maxItems": 300 - } - } - } - }, - "Deprecation": { - "type": "object", - "properties": { - "id": { - "type": "string", - "description": "The id that uniquely identifies this particular deprecations (mostly used internally)" - }, - "status": { - "type": "string", - "description": "Whether the problematic API functionality is deprecated (planned to be removed) or already removed" - }, - "apiType": { - "type": "string", - "description": "Describes which API is effected, usually one of: REST, GraphQL" - }, - "msg": { - "type": "string", - "description": "What this deprecation is about" - }, - "mitigation": { - "type": "string", - "description": "User-required object to not be affected by the (planned) removal" - }, - "sinceVersion": { - "type": "string", - "description": "The deprecation was introduced in this version" - }, - "plannedRemovalVersion": { - "type": "string", - "description": "A best-effort guess of which upcoming version will remove the feature entirely" - }, - "removedIn": { - "type": "string", - "description": "If the feature has already been removed, it was removed in this version", - "x-nullable": true - }, - "removedTime": { - "type": "string", - "format": "date-time", - "description": "If the feature has already been removed, it was removed at this timestamp", - "x-nullable": true - }, - "sinceTime": { - "type": "string", - "format": "date-time", - "description": "The deprecation was introduced in this version" - }, - "locations": { - "type": "array", - "description": "The locations within the specified API affected by this deprecation", - "items": { - "type": "string" - } - } - } - }, - "ErrorResponse": { - "description": "An error response given by Weaviate end-points.", - "properties": { - "error": { - "items": { - "properties": { - "message": { - "type": "string" - } - }, - "type": "object" - }, - "type": "array" - } - }, - "type": "object" - }, - "GraphQLError": { - "description": "An error response caused by a GraphQL query.", - "properties": { - "locations": { - "items": { - "properties": { - "column": { - "format": "int64", - "type": "integer" - }, - "line": { - "format": "int64", - "type": "integer" - } - }, - "type": "object" - }, - "type": "array" - }, - "message": { - "type": "string" - }, - "path": { - "items": { - "type": "string" - }, - "type": "array" - } - } - }, - "GraphQLQuery": { - "description": "GraphQL query based on: http://facebook.github.io/graphql/.", - "properties": { - "operationName": { - "description": "The name of the operation if multiple exist in the query.", - "type": "string" - }, - "query": { - "description": "Query based on GraphQL syntax.", - "type": "string" - }, - "variables": { - "description": "Additional variables for the query.", - "type": "object" - } - }, - "type": "object" - }, - "GraphQLQueries": { - "description": "A list of GraphQL queries.", - "items": { - "$ref": "#/definitions/GraphQLQuery" - }, - "type": "array" - }, - "GraphQLResponse": { - "description": "GraphQL based response: http://facebook.github.io/graphql/.", - "properties": { - "data": { - "additionalProperties": { - "$ref": "#/definitions/JsonObject" - }, - "description": "GraphQL data object.", - "type": "object" - }, - "errors": { - "description": "Array with errors.", - "items": { - "$ref": "#/definitions/GraphQLError" - }, - "x-omitempty": true, - "type": "array" - } - } - }, - "GraphQLResponses": { - "description": "A list of GraphQL responses.", - "items": { - "$ref": "#/definitions/GraphQLResponse" - }, - "type": "array" - }, - "InvertedIndexConfig": { - "description": "Configure the inverted index built into Weaviate", - "properties": { - "cleanupIntervalSeconds": { - "description": "Asynchronous index clean up happens every n seconds", - "format": "int", - "type": "number" - }, - "bm25": { - "$ref": "#/definitions/BM25Config" - }, - "stopwords": { - "$ref": "#/definitions/StopwordConfig" - }, - "indexTimestamps": { - "description": "Index each object by its internal timestamps", - "type": "boolean" - }, - "indexNullState": { - "description": "Index each object with the null state", - "type": "boolean" - }, - "indexPropertyLength": { - "description": "Index length of properties", - "type": "boolean" - } - }, - "type": "object" - }, - "ReplicationConfig": { - "description": "Configure how replication is executed in a cluster", - "properties": { - "factor": { - "description": "Number of times a class is replicated", - "type": "integer" - } - }, - "type": "object" - }, - "BM25Config": { - "description": "tuning parameters for the BM25 algorithm", - "properties": { - "k1": { - "description": "calibrates term-weight scaling based on the term frequency within a document", - "format": "float", - "type": "number" - }, - "b": { - "description": "calibrates term-weight scaling based on the document length", - "format": "float", - "type": "number" - } - }, - "type": "object" - }, - "StopwordConfig": { - "description": "fine-grained control over stopword list usage", - "properties": { - "preset": { - "description": "pre-existing list of common words by language", - "type": "string" - }, - "additions": { - "description": "stopwords to be considered additionally", - "type": "array", - "items": { - "type": "string" - } - }, - "removals": { - "description": "stopwords to be removed from consideration", - "type": "array", - "items": { - "type": "string" - } - } - }, - "type": "object" - }, - "MultiTenancyConfig": { - "description": "Configuration related to multi-tenancy within a class", - "properties": { - "enabled": { - "description": "Whether or not multi-tenancy is enabled for this class", - "type": "boolean", - "x-omitempty": false - } - } - }, - "JsonObject": { - "description": "JSON object value.", - "type": "object" - }, - "Meta": { - "description": "Contains meta information of the current Weaviate instance.", - "properties": { - "hostname": { - "description": "The url of the host.", - "format": "url", - "type": "string" - }, - "version": { - "description": "Version of weaviate you are currently running", - "type": "string" - }, - "modules": { - "description": "Module-specific meta information", - "type": "object" - } - }, - "type": "object" - }, - "MultipleRef": { - "description": "Multiple instances of references to other objects.", - "items": { - "$ref": "#/definitions/SingleRef" - }, - "type": "array" - }, - "PatchDocumentObject": { - "description": "Either a JSONPatch document as defined by RFC 6902 (from, op, path, value), or a merge document (RFC 7396).", - "properties": { - "from": { - "description": "A string containing a JSON Pointer value.", - "type": "string" - }, - "op": { - "description": "The operation to be performed.", - "enum": [ - "add", - "remove", - "replace", - "move", - "copy", - "test" - ], - "type": "string" - }, - "path": { - "description": "A JSON-Pointer.", - "type": "string" - }, - "value": { - "description": "The value to be used within the operations.", - "type": "object" - }, - "merge": { - "$ref": "#/definitions/Object" - } - }, - "required": [ - "op", - "path" - ] - }, - "PatchDocumentAction": { - "description": "Either a JSONPatch document as defined by RFC 6902 (from, op, path, value), or a merge document (RFC 7396).", - "properties": { - "from": { - "description": "A string containing a JSON Pointer value.", - "type": "string" - }, - "op": { - "description": "The operation to be performed.", - "enum": [ - "add", - "remove", - "replace", - "move", - "copy", - "test" - ], - "type": "string" - }, - "path": { - "description": "A JSON-Pointer.", - "type": "string" - }, - "value": { - "description": "The value to be used within the operations.", - "type": "object" - }, - "merge": { - "$ref": "#/definitions/Object" - } - }, - "required": [ - "op", - "path" - ] - }, - "PeerUpdate": { - "description": "A single peer in the network.", - "properties": { - "id": { - "description": "The session ID of the peer.", - "type": "string", - "format": "uuid" - }, - "name": { - "description": "Human readable name.", - "type": "string" - }, - "uri": { - "description": "The location where the peer is exposed to the internet.", - "type": "string", - "format": "uri" - }, - "schemaHash": { - "description": "The latest known hash of the peer's schema.", - "type": "string" - } - } - }, - "PeerUpdateList": { - "description": "List of known peers.", - "items": { - "$ref": "#/definitions/PeerUpdate" - }, - "type": "array" - }, - "VectorWeights": { - "description": "Allow custom overrides of vector weights as math expressions. E.g. \"pancake\": \"7\" will set the weight for the word pancake to 7 in the vectorization, whereas \"w * 3\" would triple the originally calculated word. This is an open object, with OpenAPI Specification 3.0 this will be more detailed. See Weaviate docs for more info. In the future this will become a key/value (string/string) object.", - "type": "object" - }, - "PropertySchema": { - "description": "This is an open object, with OpenAPI Specification 3.0 this will be more detailed. See Weaviate docs for more info. In the future this will become a key/value OR a SingleRef definition.", - "type": "object" - }, - "SchemaHistory": { - "description": "This is an open object, with OpenAPI Specification 3.0 this will be more detailed. See Weaviate docs for more info. In the future this will become a key/value OR a SingleRef definition.", - "type": "object" - }, - "Schema": { - "description": "Definitions of semantic schemas (also see: https://github.com/weaviate/weaviate-semantic-schemas).", - "properties": { - "classes": { - "description": "Semantic classes that are available.", - "items": { - "$ref": "#/definitions/Class" - }, - "type": "array" - }, - "maintainer": { - "description": "Email of the maintainer.", - "format": "email", - "type": "string" - }, - "name": { - "description": "Name of the schema.", - "type": "string" - } - }, - "type": "object" - }, - "SchemaClusterStatus": { - "description": "Indicates the health of the schema in a cluster.", - "properties": { - "healthy": { - "description": "True if the cluster is in sync, false if there is an issue (see error).", - "type": "boolean", - "x-omitempty": false - }, - "error": { - "description": "Contains the sync check error if one occurred", - "type": "string", - "x-omitempty": true - }, - "hostname": { - "description": "Hostname of the coordinating node, i.e. the one that received the cluster. This can be useful information if the error message contains phrases such as 'other nodes agree, but local does not', etc.", - "type": "string" - }, - "nodeCount": { - "description": "Number of nodes that participated in the sync check", - "type": "number", - "format": "int" - }, - "ignoreSchemaSync": { - "description": "The cluster check at startup can be ignored (to recover from an out-of-sync situation).", - "type": "boolean", - "x-omitempty": false - } - }, - "type": "object" - }, - "Class": { - "properties": { - "class": { - "description": "Name of the class as URI relative to the schema URL.", - "type": "string" - }, - "vectorIndexType": { - "description": "Name of the vector index to use, eg. (HNSW)", - "type": "string" - }, - "vectorIndexConfig": { - "description": "Vector-index config, that is specific to the type of index selected in vectorIndexType", - "type": "object" - }, - "shardingConfig": { - "description": "Manage how the index should be sharded and distributed in the cluster", - "type": "object" - }, - "replicationConfig": { - "$ref": "#/definitions/ReplicationConfig" - }, - "invertedIndexConfig": { - "$ref": "#/definitions/InvertedIndexConfig" - }, - "multiTenancyConfig": { - "$ref": "#/definitions/MultiTenancyConfig" - }, - "vectorizer": { - "description": "Specify how the vectors for this class should be determined. The options are either 'none' - this means you have to import a vector with each object yourself - or the name of a module that provides vectorization capabilities, such as 'text2vec-contextionary'. If left empty, it will use the globally configured default which can itself either be 'none' or a specific module.", - "type": "string" - }, - "moduleConfig": { - "description": "Configuration specific to modules this Weaviate instance has installed", - "type": "object" - }, - "description": { - "description": "Description of the class.", - "type": "string" - }, - "properties": { - "description": "The properties of the class.", - "items": { - "$ref": "#/definitions/Property" - }, - "type": "array" - } - }, - "type": "object" - }, - "Property": { - "properties": { - "dataType": { - "description": "Can be a reference to another type when it starts with a capital (for example Person), otherwise \"string\" or \"int\".", - "items": { - "type": "string" - }, - "type": "array" - }, - "description": { - "description": "Description of the property.", - "type": "string" - }, - "moduleConfig": { - "description": "Configuration specific to modules this Weaviate instance has installed", - "type": "object" - }, - "name": { - "description": "Name of the property as URI relative to the schema URL.", - "type": "string" - }, - "indexInverted": { - "description": "Optional. Should this property be indexed in the inverted index. Defaults to true. If you choose false, you will not be able to use this property in where filters, bm25 or hybrid search. This property has no affect on vectorization decisions done by modules (deprecated as of v1.19; use indexFilterable or/and indexSearchable instead)", - "type": "boolean", - "x-nullable": true - }, - "indexFilterable": { - "description": "Optional. Should this property be indexed in the inverted index. Defaults to true. If you choose false, you will not be able to use this property in where filters. This property has no affect on vectorization decisions done by modules", - "type": "boolean", - "x-nullable": true - }, - "indexSearchable": { - "description": "Optional. Should this property be indexed in the inverted index. Defaults to true. Applicable only to properties of data type text and text[]. If you choose false, you will not be able to use this property in bm25 or hybrid search. This property has no affect on vectorization decisions done by modules", - "type": "boolean", - "x-nullable": true - }, - "tokenization": { - "description": "Determines tokenization of the property as separate words or whole field. Optional. Applies to text and text[] data types. Allowed values are `word` (default; splits on any non-alphanumerical, lowercases), `lowercase` (splits on white spaces, lowercases), `whitespace` (splits on white spaces), `field` (trims). Not supported for remaining data types", - "type": "string", - "enum": [ - "word", - "lowercase", - "whitespace", - "field" - ] - }, - "nestedProperties": { - "description": "The properties of the nested object(s). Applies to object and object[] data types.", - "items": { - "$ref": "#/definitions/NestedProperty" - }, - "type": "array", - "x-omitempty": true - } - }, - "type": "object" - }, - "NestedProperty": { - "properties": { - "dataType": { - "items": { - "type": "string" - }, - "type": "array" - }, - "description": { - "type": "string" - }, - "name": { - "type": "string" - }, - "indexFilterable": { - "type": "boolean", - "x-nullable": true - }, - "indexSearchable": { - "type": "boolean", - "x-nullable": true - }, - "tokenization": { - "type": "string", - "enum": [ - "word", - "lowercase", - "whitespace", - "field" - ] - }, - "nestedProperties": { - "items": { - "$ref": "#/definitions/NestedProperty" - }, - "type": "array", - "x-omitempty": true - } - }, - "type": "object" - }, - "ShardStatusList": { - "description": "The status of all the shards of a Class", - "items": { - "$ref": "#/definitions/ShardStatusGetResponse" - }, - "type": "array" - }, - "ShardStatusGetResponse": { - "description": "Response body of shard status get request", - "properties": { - "name": { - "description": "Name of the shard", - "type": "string" - }, - "status": { - "description": "Status of the shard", - "type": "string" - }, - "vectorQueueSize": { - "description": "Size of the vector queue of the shard", - "type": "integer", - "x-omitempty": false - } - } - }, - "ShardStatus": { - "description": "The status of a single shard", - "properties": { - "status": { - "description": "Status of the shard", - "type": "string" - } - } - }, - "BackupCreateStatusResponse": { - "description": "The definition of a backup create metadata", - "properties": { - "id": { - "description": "The ID of the backup. Must be URL-safe and work as a filesystem path, only lowercase, numbers, underscore, minus characters allowed.", - "type": "string" - }, - "backend": { - "description": "Backup backend name e.g. filesystem, gcs, s3.", - "type": "string" - }, - "path": { - "description": "destination path of backup files proper to selected backend", - "type": "string" - }, - "error": { - "description": "error message if creation failed", - "type": "string" - }, - "status": { - "description": "phase of backup creation process", - "type": "string", - "default": "STARTED", - "enum": [ - "STARTED", - "TRANSFERRING", - "TRANSFERRED", - "SUCCESS", - "FAILED" - ] - } - } - }, - "BackupRestoreStatusResponse": { - "description": "The definition of a backup restore metadata", - "properties": { - "id": { - "description": "The ID of the backup. Must be URL-safe and work as a filesystem path, only lowercase, numbers, underscore, minus characters allowed.", - "type": "string" - }, - "backend": { - "description": "Backup backend name e.g. filesystem, gcs, s3.", - "type": "string" - }, - "path": { - "description": "destination path of backup files proper to selected backup backend", - "type": "string" - }, - "error": { - "description": "error message if restoration failed", - "type": "string" - }, - "status": { - "description": "phase of backup restoration process", - "type": "string", - "default": "STARTED", - "enum": [ - "STARTED", - "TRANSFERRING", - "TRANSFERRED", - "SUCCESS", - "FAILED" - ] - } - } - }, - "BackupConfig": { - "description": "Backup custom configuration", - "type": "object", - "properties": { - "CPUPercentage": { - "description": "Desired CPU core utilization ranging from 1%-80%", - "type": "integer", - "default": 50, - "minimum": 1, - "maximum": 80, - "x-nullable": false - }, - "ChunkSize": { - "description": "Weaviate will attempt to come close the specified size, with a minimum of 2MB, default of 128MB, and a maximum of 512MB", - "type": "integer", - "default": 128, - "minimum": 2, - "maximum": 512, - "x-nullable": false - }, - "CompressionLevel": { - "description": "compression level used by compression algorithm", - "type": "string", - "default": "DefaultCompression", - "x-nullable": false, - "enum": [ - "DefaultCompression", - "BestSpeed", - "BestCompression" - ] - } - } - }, - "RestoreConfig": { - "description": "Backup custom configuration", - "type": "object", - "properties": { - "CPUPercentage": { - "description": "Desired CPU core utilization ranging from 1%-80%", - "type": "integer", - "default": 50, - "minimum": 1, - "maximum": 80, - "x-nullable": false - } - } - }, - "BackupCreateRequest": { - "description": "Request body for creating a backup of a set of classes", - "properties": { - "id": { - "description": "The ID of the backup. Must be URL-safe and work as a filesystem path, only lowercase, numbers, underscore, minus characters allowed.", - "type": "string" - }, - "config": { - "description": "Custom configuration for the backup creation process", - "type": "object", - "$ref": "#/definitions/BackupConfig" - }, - "include": { - "description": "List of classes to include in the backup creation process", - "type": "array", - "items": { - "type": "string" - } - }, - "exclude": { - "description": "List of classes to exclude from the backup creation process", - "type": "array", - "items": { - "type": "string" - } - } - } - }, - "BackupCreateResponse": { - "description": "The definition of a backup create response body", - "properties": { - "id": { - "description": "The ID of the backup. Must be URL-safe and work as a filesystem path, only lowercase, numbers, underscore, minus characters allowed.", - "type": "string" - }, - "classes": { - "description": "The list of classes for which the backup creation process was started", - "type": "array", - "items": { - "type": "string" - } - }, - "backend": { - "description": "Backup backend name e.g. filesystem, gcs, s3.", - "type": "string" - }, - "path": { - "description": "destination path of backup files proper to selected backend", - "type": "string" - }, - "error": { - "description": "error message if creation failed", - "type": "string" - }, - "status": { - "description": "phase of backup creation process", - "type": "string", - "default": "STARTED", - "enum": [ - "STARTED", - "TRANSFERRING", - "TRANSFERRED", - "SUCCESS", - "FAILED" - ] - } - } - }, - "BackupRestoreRequest": { - "description": "Request body for restoring a backup for a set of classes", - "properties": { - "config": { - "description": "Custom configuration for the backup restoration process", - "type": "object", - "$ref": "#/definitions/RestoreConfig" - }, - "include": { - "description": "List of classes to include in the backup restoration process", - "type": "array", - "items": { - "type": "string" - } - }, - "exclude": { - "description": "List of classes to exclude from the backup restoration process", - "type": "array", - "items": { - "type": "string" - } - }, - "node_mapping": { - "description": "Allows overriding the node names stored in the backup with different ones. Useful when restoring backups to a different environment.", - "type": "object", - "additionalProperties": { - "type": "string" - } - } - } - }, - "BackupRestoreResponse": { - "description": "The definition of a backup restore response body", - "properties": { - "id": { - "description": "The ID of the backup. Must be URL-safe and work as a filesystem path, only lowercase, numbers, underscore, minus characters allowed.", - "type": "string" - }, - "classes": { - "description": "The list of classes for which the backup restoration process was started", - "type": "array", - "items": { - "type": "string" - } - }, - "backend": { - "description": "Backup backend name e.g. filesystem, gcs, s3.", - "type": "string" - }, - "path": { - "description": "destination path of backup files proper to selected backend", - "type": "string" - }, - "error": { - "description": "error message if restoration failed", - "type": "string" - }, - "status": { - "description": "phase of backup restoration process", - "type": "string", - "default": "STARTED", - "enum": [ - "STARTED", - "TRANSFERRING", - "TRANSFERRED", - "SUCCESS", - "FAILED" - ] - } - } - }, - "NodeStats": { - "description": "The summary of Weaviate's statistics.", - "properties": { - "shardCount": { - "description": "The count of Weaviate's shards.", - "format": "int", - "type": "number", - "x-omitempty": false - }, - "objectCount": { - "description": "The total number of objects in DB.", - "format": "int64", - "type": "number", - "x-omitempty": false - } - } - }, - "BatchStats": { - "description": "The summary of a nodes batch queue congestion status.", - "properties": { - "queueLength": { - "description": "How many objects are currently in the batch queue.", - "format": "int", - "type": "number", - "x-omitempty": true, - "x-nullable": true - }, - "ratePerSecond": { - "description": "How many objects are approximately processed from the batch queue per second.", - "format": "int", - "type": "number", - "x-omitempty": false - } - } - }, - "NodeShardStatus": { - "description": "The definition of a node shard status response body", - "properties": { - "name": { - "description": "The name of the shard.", - "type": "string", - "x-omitempty": false - }, - "class": { - "description": "The name of shard's class.", - "type": "string", - "x-omitempty": false - }, - "objectCount": { - "description": "The number of objects in shard.", - "format": "int64", - "type": "number", - "x-omitempty": false - }, - "vectorIndexingStatus": { - "description": "The status of the vector indexing process.", - "format": "string", - "x-omitempty": false - }, - "compressed": { - "description": "The status of vector compression/quantization.", - "format": "boolean", - "x-omitempty": false - }, - "vectorQueueLength": { - "description": "The length of the vector indexing queue.", - "format": "int64", - "type": "number", - "x-omitempty": false - } - } - }, - "NodeStatus": { - "description": "The definition of a backup node status response body", - "properties": { - "name": { - "description": "The name of the node.", - "type": "string" - }, - "status": { - "description": "Node's status.", - "type": "string", - "default": "HEALTHY", - "enum": [ - "HEALTHY", - "UNHEALTHY", - "UNAVAILABLE" - ] - }, - "version": { - "description": "The version of Weaviate.", - "type": "string" - }, - "gitHash": { - "description": "The gitHash of Weaviate.", - "type": "string" - }, - "stats": { - "description": "Weaviate overall statistics.", - "type": "object", - "$ref": "#/definitions/NodeStats" - }, - "batchStats": { - "description": "Weaviate batch statistics.", - "type": "object", - "$ref": "#/definitions/BatchStats" - }, - "shards": { - "description": "The list of the shards with it's statistics.", - "type": "array", - "items": { - "$ref": "#/definitions/NodeShardStatus" - } - } - } - }, - "NodesStatusResponse": { - "description": "The status of all of the Weaviate nodes", - "type": "object", - "properties": { - "nodes": { - "type": "array", - "items": { - "$ref": "#/definitions/NodeStatus" - } - } - } - }, - "SingleRef": { - "description": "Either set beacon (direct reference) or set class and schema (concept reference)", - "properties": { - "class": { - "description": "If using a concept reference (rather than a direct reference), specify the desired class name here", - "format": "uri", - "type": "string" - }, - "schema": { - "description": "If using a concept reference (rather than a direct reference), specify the desired properties here", - "$ref": "#/definitions/PropertySchema" - }, - "beacon": { - "description": "If using a direct reference, specify the URI to point to the cross-ref here. Should be in the form of weaviate://localhost/ for the example of a local cross-ref to an object", - "format": "uri", - "type": "string" - }, - "href": { - "description": "If using a direct reference, this read-only fields provides a link to the referenced resource. If 'origin' is globally configured, an absolute URI is shown - a relative URI otherwise.", - "format": "uri", - "type": "string" - }, - "classification": { - "description": "Additional Meta information about classifications if the item was part of one", - "$ref": "#/definitions/ReferenceMetaClassification" - } - } - }, - "AdditionalProperties": { - "description": "Additional Meta information about a single object object.", - "type": "object", - "additionalProperties": { - "type": "object" - } - }, - "ReferenceMetaClassification": { - "description": "This meta field contains additional info about the classified reference property", - "properties": { - "overallCount": { - "description": "overall neighbors checked as part of the classification. In most cases this will equal k, but could be lower than k - for example if not enough data was present", - "type": "number", - "format": "int64" - }, - "winningCount": { - "description": "size of the winning group, a number between 1..k", - "type": "number", - "format": "int64" - }, - "losingCount": { - "description": "size of the losing group, can be 0 if the winning group size equals k", - "type": "number", - "format": "int64" - }, - "closestOverallDistance": { - "description": "The lowest distance of any neighbor, regardless of whether they were in the winning or losing group", - "type": "number", - "format": "float32" - }, - "winningDistance": { - "description": "deprecated - do not use, to be removed in 0.23.0", - "type": "number", - "format": "float32" - }, - "meanWinningDistance": { - "description": "Mean distance of all neighbors from the winning group", - "type": "number", - "format": "float32" - }, - "closestWinningDistance": { - "description": "Closest distance of a neighbor from the winning group", - "type": "number", - "format": "float32" - }, - "closestLosingDistance": { - "description": "The lowest distance of a neighbor in the losing group. Optional. If k equals the size of the winning group, there is no losing group", - "type": "number", - "format": "float32", - "x-nullable": true - }, - "losingDistance": { - "description": "deprecated - do not use, to be removed in 0.23.0", - "type": "number", - "format": "float32", - "x-nullable": true - }, - "meanLosingDistance": { - "description": "Mean distance of all neighbors from the losing group. Optional. If k equals the size of the winning group, there is no losing group.", - "type": "number", - "format": "float32", - "x-nullable": true - } - } - }, - "BatchReference": { - "properties": { - "from": { - "description": "Long-form beacon-style URI to identify the source of the cross-ref including the property name. Should be in the form of weaviate://localhost////, where must be one of 'objects', 'objects' and and must represent the cross-ref property of source class to be used.", - "format": "uri", - "type": "string", - "example": "weaviate://localhost/Zoo/a5d09582-4239-4702-81c9-92a6e0122bb4/hasAnimals" - }, - "to": { - "description": "Short-form URI to point to the cross-ref. Should be in the form of weaviate://localhost/ for the example of a local cross-ref to an object", - "example": "weaviate://localhost/97525810-a9a5-4eb0-858a-71449aeb007f", - "format": "uri", - "type": "string" - }, - "tenant": { - "type": "string", - "description": "Name of the reference tenant." - } - } - }, - "BatchReferenceResponse": { - "allOf": [ - { - "$ref": "#/definitions/BatchReference" - }, - { - "properties": { - "result": { - "description": "Results for this specific reference.", - "format": "object", - "properties": { - "status": { - "type": "string", - "default": "SUCCESS", - "enum": [ - "SUCCESS", - "PENDING", - "FAILED" - ] - }, - "errors": { - "$ref": "#/definitions/ErrorResponse" - } - } - } - } - } - ], - "type": "object" - }, - "GeoCoordinates": { - "properties": { - "latitude": { - "description": "The latitude of the point on earth in decimal form", - "format": "float", - "type": "number", - "x-nullable": true - }, - "longitude": { - "description": "The longitude of the point on earth in decimal form", - "format": "float", - "type": "number", - "x-nullable": true - } - } - }, - "PhoneNumber": { - "properties": { - "input": { - "description": "The raw input as the phone number is present in your raw data set. It will be parsed into the standardized formats if valid.", - "type": "string" - }, - "internationalFormatted": { - "description": "Read-only. Parsed result in the international format (e.g. +49 123 ...)", - "type": "string" - }, - "defaultCountry": { - "description": "Optional. The ISO 3166-1 alpha-2 country code. This is used to figure out the correct countryCode and international format if only a national number (e.g. 0123 4567) is provided", - "type": "string" - }, - "countryCode": { - "description": "Read-only. The numerical country code (e.g. 49)", - "format": "uint64", - "type": "number" - }, - "national": { - "description": "Read-only. The numerical representation of the national part", - "format": "uint64", - "type": "number" - }, - "nationalFormatted": { - "description": "Read-only. Parsed result in the national format (e.g. 0123 456789)", - "type": "string" - }, - "valid": { - "description": "Read-only. Indicates whether the parsed number is a valid phone number", - "type": "boolean" - } - } - }, - "Object": { - "properties": { - "class": { - "description": "Class of the Object, defined in the schema.", - "type": "string" - }, - "vectorWeights": { - "$ref": "#/definitions/VectorWeights" - }, - "properties": { - "$ref": "#/definitions/PropertySchema" - }, - "id": { - "description": "ID of the Object.", - "format": "uuid", - "type": "string" - }, - "creationTimeUnix": { - "description": "Timestamp of creation of this Object in milliseconds since epoch UTC.", - "format": "int64", - "type": "integer" - }, - "lastUpdateTimeUnix": { - "description": "Timestamp of the last Object update in milliseconds since epoch UTC.", - "format": "int64", - "type": "integer" - }, - "vector": { - "description": "This object's position in the Contextionary vector space. Read-only if using a vectorizer other than 'none'. Writable and required if using 'none' as vectorizer.", - "$ref": "#/definitions/C11yVector" - }, - "tenant": { - "description": "Name of the Objects tenant.", - "type": "string" - }, - "additional": { - "$ref": "#/definitions/AdditionalProperties" - } - }, - "type": "object" - }, - "ObjectsGetResponse": { - "allOf": [ - { - "$ref": "#/definitions/Object" - }, - { - "properties": { - "deprecations": { - "type": "array", - "items": { - "$ref": "#/definitions/Deprecation" - } - } - } - }, - { - "properties": { - "result": { - "description": "Results for this specific Object.", - "format": "object", - "properties": { - "status": { - "type": "string", - "default": "SUCCESS", - "enum": [ - "SUCCESS", - "PENDING", - "FAILED" - ] - }, - "errors": { - "$ref": "#/definitions/ErrorResponse" - } - } - } - } - } - ], - "type": "object" - }, - "BatchDelete": { - "type": "object", - "properties": { - "match": { - "description": "Outlines how to find the objects to be deleted.", - "type": "object", - "properties": { - "class": { - "description": "Class (name) which objects will be deleted.", - "type": "string", - "example": "City" - }, - "where": { - "description": "Filter to limit the objects to be deleted.", - "type": "object", - "$ref": "#/definitions/WhereFilter" - } - } - }, - "output": { - "description": "Controls the verbosity of the output, possible values are: \"minimal\", \"verbose\". Defaults to \"minimal\".", - "type": "string", - "default": "minimal" - }, - "dryRun": { - "description": "If true, objects will not be deleted yet, but merely listed. Defaults to false.", - "type": "boolean", - "default": false - } - } - }, - "BatchDeleteResponse": { - "description": "Delete Objects response.", - "type": "object", - "properties": { - "match": { - "description": "Outlines how to find the objects to be deleted.", - "type": "object", - "properties": { - "class": { - "description": "Class (name) which objects will be deleted.", - "type": "string", - "example": "City" - }, - "where": { - "description": "Filter to limit the objects to be deleted.", - "type": "object", - "$ref": "#/definitions/WhereFilter" - } - } - }, - "output": { - "description": "Controls the verbosity of the output, possible values are: \"minimal\", \"verbose\". Defaults to \"minimal\".", - "type": "string", - "default": "minimal" - }, - "dryRun": { - "description": "If true, objects will not be deleted yet, but merely listed. Defaults to false.", - "type": "boolean", - "default": false - }, - "results": { - "type": "object", - "properties": { - "matches": { - "description": "How many objects were matched by the filter.", - "type": "number", - "format": "int64", - "x-omitempty": false - }, - "limit": { - "description": "The most amount of objects that can be deleted in a single query, equals QUERY_MAXIMUM_RESULTS.", - "type": "number", - "format": "int64", - "x-omitempty": false - }, - "successful": { - "description": "How many objects were successfully deleted in this round.", - "type": "number", - "format": "int64", - "x-omitempty": false - }, - "failed": { - "description": "How many objects should have been deleted but could not be deleted.", - "type": "number", - "format": "int64", - "x-omitempty": false - }, - "objects": { - "description": "With output set to \"minimal\" only objects with error occurred will the be described. Successfully deleted objects would be omitted. Output set to \"verbose\" will list all of the objets with their respective statuses.", - "type": "array", - "items": { - "description": "Results for this specific Object.", - "format": "object", - "properties": { - "id": { - "description": "ID of the Object.", - "format": "uuid", - "type": "string" - }, - "status": { - "type": "string", - "default": "SUCCESS", - "enum": [ - "SUCCESS", - "DRYRUN", - "FAILED" - ] - }, - "errors": { - "$ref": "#/definitions/ErrorResponse" - } - } - } - } - } - } - } - }, - "ObjectsListResponse": { - "description": "List of Objects.", - "properties": { - "objects": { - "description": "The actual list of Objects.", - "items": { - "$ref": "#/definitions/Object" - }, - "type": "array" - }, - "deprecations": { - "type": "array", - "items": { - "$ref": "#/definitions/Deprecation" - } - }, - "totalResults": { - "description": "The total number of Objects for the query. The number of items in a response may be smaller due to paging.", - "format": "int64", - "type": "integer" - } - }, - "type": "object" - }, - "Classification": { - "description": "Manage classifications, trigger them and view status of past classifications.", - "properties": { - "id": { - "description": "ID to uniquely identify this classification run", - "format": "uuid", - "type": "string", - "example": "ee722219-b8ec-4db1-8f8d-5150bb1a9e0c" - }, - "class": { - "description": "class (name) which is used in this classification", - "type": "string", - "example": "City" - }, - "classifyProperties": { - "description": "which ref-property to set as part of the classification", - "type": "array", - "items": { - "type": "string" - }, - "example": [ - "inCountry" - ] - }, - "basedOnProperties": { - "description": "base the text-based classification on these fields (of type text)", - "type": "array", - "items": { - "type": "string" - }, - "example": [ - "description" - ] - }, - "status": { - "description": "status of this classification", - "type": "string", - "enum": [ - "running", - "completed", - "failed" - ], - "example": "running" - }, - "meta": { - "description": "additional meta information about the classification", - "type": "object", - "$ref": "#/definitions/ClassificationMeta" - }, - "type": { - "description": "which algorithm to use for classifications", - "type": "string" - }, - "settings": { - "description": "classification-type specific settings", - "type": "object" - }, - "error": { - "description": "error message if status == failed", - "type": "string", - "default": "", - "example": "classify xzy: something went wrong" - }, - "filters": { - "type": "object", - "properties": { - "sourceWhere": { - "description": "limit the objects to be classified", - "type": "object", - "$ref": "#/definitions/WhereFilter" - }, - "trainingSetWhere": { - "description": "Limit the training objects to be considered during the classification. Can only be used on types with explicit training sets, such as 'knn'", - "type": "object", - "$ref": "#/definitions/WhereFilter" - }, - "targetWhere": { - "description": "Limit the possible sources when using an algorithm which doesn't really on training data, e.g. 'contextual'. When using an algorithm with a training set, such as 'knn', limit the training set instead", - "type": "object", - "$ref": "#/definitions/WhereFilter" - } - } - } - }, - "type": "object" - }, - "ClassificationMeta": { - "description": "Additional information to a specific classification", - "properties": { - "started": { - "description": "time when this classification was started", - "type": "string", - "format": "date-time", - "example": "2017-07-21T17:32:28Z" - }, - "completed": { - "description": "time when this classification finished", - "type": "string", - "format": "date-time", - "example": "2017-07-21T17:32:28Z" - }, - "count": { - "description": "number of objects which were taken into consideration for classification", - "type": "integer", - "example": 147 - }, - "countSucceeded": { - "description": "number of objects successfully classified", - "type": "integer", - "example": 140 - }, - "countFailed": { - "description": "number of objects which could not be classified - see error message for details", - "type": "integer", - "example": 7 - } - }, - "type": "object" - }, - "WhereFilter": { - "description": "Filter search results using a where filter", - "properties": { - "operands": { - "description": "combine multiple where filters, requires 'And' or 'Or' operator", - "type": "array", - "items": { - "$ref": "#/definitions/WhereFilter" - } - }, - "operator": { - "description": "operator to use", - "type": "string", - "enum": [ - "And", - "Or", - "Equal", - "Like", - "NotEqual", - "GreaterThan", - "GreaterThanEqual", - "LessThan", - "LessThanEqual", - "WithinGeoRange", - "IsNull", - "ContainsAny", - "ContainsAll" - ], - "example": "GreaterThanEqual" - }, - "path": { - "description": "path to the property currently being filtered", - "type": "array", - "items": { - "type": "string" - }, - "example": [ - "inCity", - "City", - "name" - ] - }, - "valueInt": { - "description": "value as integer", - "type": "integer", - "format": "int64", - "example": 2000, - "x-nullable": true - }, - "valueNumber": { - "description": "value as number/float", - "type": "number", - "format": "float64", - "example": 3.14, - "x-nullable": true - }, - "valueBoolean": { - "description": "value as boolean", - "type": "boolean", - "example": false, - "x-nullable": true - }, - "valueString": { - "description": "value as text (deprecated as of v1.19; alias for valueText)", - "type": "string", - "example": "my search term", - "x-nullable": true - }, - "valueText": { - "description": "value as text", - "type": "string", - "example": "my search term", - "x-nullable": true - }, - "valueDate": { - "description": "value as date (as string)", - "type": "string", - "example": "TODO", - "x-nullable": true - }, - "valueIntArray": { - "description": "value as integer", - "type": "array", - "items": { - "type": "integer", - "format": "int64" - }, - "example": "[100, 200]", - "x-nullable": true, - "x-omitempty": true - }, - "valueNumberArray": { - "description": "value as number/float", - "type": "array", - "items": { - "type": "number", - "format": "float64" - }, - "example": [ - 3.14 - ], - "x-nullable": true, - "x-omitempty": true - }, - "valueBooleanArray": { - "description": "value as boolean", - "type": "array", - "items": { - "type": "boolean" - }, - "example": [ - true, - false - ], - "x-nullable": true, - "x-omitempty": true - }, - "valueStringArray": { - "description": "value as text (deprecated as of v1.19; alias for valueText)", - "type": "array", - "items": { - "type": "string" - }, - "example": [ - "my search term" - ], - "x-nullable": true, - "x-omitempty": true - }, - "valueTextArray": { - "description": "value as text", - "type": "array", - "items": { - "type": "string" - }, - "example": [ - "my search term" - ], - "x-nullable": true, - "x-omitempty": true - }, - "valueDateArray": { - "description": "value as date (as string)", - "type": "array", - "items": { - "type": "string" - }, - "example": "TODO", - "x-nullable": true, - "x-omitempty": true - }, - "valueGeoRange": { - "description": "value as geo coordinates and distance", - "type": "object", - "$ref": "#/definitions/WhereFilterGeoRange", - "x-nullable": true - } - }, - "type": "object" - }, - "WhereFilterGeoRange": { - "type": "object", - "description": "filter within a distance of a georange", - "properties": { - "geoCoordinates": { - "$ref": "#/definitions/GeoCoordinates", - "x-nullable": false - }, - "distance": { - "type": "object", - "properties": { - "max": { - "type": "number", - "format": "float64" - } - } - } - } - }, - "Tenant": { - "type": "object", - "description": "attributes representing a single tenant within weaviate", - "properties": { - "name": { - "description": "name of the tenant", - "type": "string" - }, - "activityStatus": { - "description": "activity status of the tenant's shard. Optional for creating tenant (implicit `HOT`) and required for updating tenant. Allowed values are `HOT` - tenant is fully active, `WARM` - tenant is active, some restrictions are imposed (TBD; not supported yet), `COLD` - tenant is inactive; no actions can be performed on tenant, tenant's files are stored locally, `FROZEN` - as COLD, but files are stored on cloud storage (not supported yet)", - "type": "string", - "enum": [ - "HOT", - "WARM", - "COLD", - "FROZEN" - ] - } - } - } - }, - "externalDocs": { - "url": "https://github.com/weaviate/weaviate" - }, - "info": { - "contact": { - "email": "hello@weaviate.io", - "name": "Weaviate", - "url": "https://github.com/weaviate" - }, - "description": "Cloud-native, modular vector database", - "title": "Weaviate", - "version": "1.23.3" - }, - "parameters": { - "CommonAfterParameterQuery": { - "description": "The starting ID of the result window.", - "in": "query", - "name": "after", - "required": false, - "type": "string" - }, - "CommonOffsetParameterQuery": { - "description": "The starting index of the result window. Default value is 0.", - "format": "int64", - "in": "query", - "name": "offset", - "required": false, - "type": "integer", - "default": 0 - }, - "CommonLimitParameterQuery": { - "description": "The maximum number of items to be returned per page. Default value is set in Weaviate config.", - "format": "int64", - "in": "query", - "name": "limit", - "required": false, - "type": "integer" - }, - "CommonIncludeParameterQuery": { - "description": "Include additional information, such as classification infos. Allowed values include: classification, vector, interpretation", - "in": "query", - "name": "include", - "required": false, - "type": "string" - }, - "CommonConsistencyLevelParameterQuery": { - "description": "Determines how many replicas must acknowledge a request before it is considered successful", - "in": "query", - "name": "consistency_level", - "required": false, - "type": "string" - }, - "CommonTenantParameterQuery": { - "description": "Specifies the tenant in a request targeting a multi-tenant class", - "in": "query", - "name": "tenant", - "required": false, - "type": "string" - }, - "CommonNodeNameParameterQuery": { - "description": "The target node which should fulfill the request", - "in": "query", - "name": "node_name", - "required": false, - "type": "string" - }, - "CommonSortParameterQuery": { - "description": "Sort parameter to pass an information about the names of the sort fields", - "in": "query", - "name": "sort", - "required": false, - "type": "string" - }, - "CommonOrderParameterQuery": { - "description": "Order parameter to tell how to order (asc or desc) data within given field", - "in": "query", - "name": "order", - "required": false, - "type": "string" - }, - "CommonClassParameterQuery": { - "description": "Class parameter specifies the class from which to query objects", - "in": "query", - "name": "class", - "required": false, - "type": "string" - }, - "CommonOutputVerbosityParameterQuery": { - "description": "Controls the verbosity of the output, possible values are: \"minimal\", \"verbose\". Defaults to \"minimal\".", - "in": "query", - "name": "output", - "required": false, - "type": "string", - "default": "minimal" - } - }, - "paths": { - "/": { - "get": { - "description": "Home. Discover the REST API", - "operationId": "weaviate.root", - "responses": { - "200": { - "description": "Weaviate is alive and ready to serve content", - "schema": { - "type": "object", - "properties": { - "links": { - "type": "array", - "items": { - "$ref": "#/definitions/Link" - } - } - } - } - } - } - } - }, - "/.well-known/live": { - "get": { - "description": "Determines whether the application is alive. Can be used for kubernetes liveness probe", - "operationId": "weaviate.wellknown.liveness", - "responses": { - "200": { - "description": "The application is able to respond to HTTP requests" - } - } - } - }, - "/.well-known/ready": { - "get": { - "description": "Determines whether the application is ready to receive traffic. Can be used for kubernetes readiness probe.", - "operationId": "weaviate.wellknown.readiness", - "responses": { - "200": { - "description": "The application has completed its start-up routine and is ready to accept traffic." - }, - "503": { - "description": "The application is currently not able to serve traffic. If other horizontal replicas of weaviate are available and they are capable of receiving traffic, all traffic should be redirected there instead." - } - } - } - }, - "/.well-known/openid-configuration": { - "get": { - "description": "OIDC Discovery page, redirects to the token issuer if one is configured", - "responses": { - "200": { - "description": "Successful response, inspect body", - "schema": { - "type": "object", - "properties": { - "href": { - "description": "The Location to redirect to", - "type": "string" - }, - "clientId": { - "description": "OAuth Client ID", - "type": "string" - }, - "scopes": { - "description": "OAuth Scopes", - "type": "array", - "items": { - "type": "string" - }, - "x-omitempty": true - } - } - } - }, - "404": { - "description": "Not found, no oidc provider present" - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "summary": "OIDC discovery information if OIDC auth is enabled", - "tags": [ - "well-known", - "oidc", - "discovery" - ], - "x-available-in-mqtt": false, - "x-available-in-websocket": false - } - }, - "/objects": { - "get": { - "description": "Lists all Objects in reverse order of creation, owned by the user that belongs to the used token.", - "operationId": "objects.list", - "x-serviceIds": [ - "weaviate.local.query" - ], - "parameters": [ - { - "$ref": "#/parameters/CommonAfterParameterQuery" - }, - { - "$ref": "#/parameters/CommonOffsetParameterQuery" - }, - { - "$ref": "#/parameters/CommonLimitParameterQuery" - }, - { - "$ref": "#/parameters/CommonIncludeParameterQuery" - }, - { - "$ref": "#/parameters/CommonSortParameterQuery" - }, - { - "$ref": "#/parameters/CommonOrderParameterQuery" - }, - { - "$ref": "#/parameters/CommonClassParameterQuery" - }, - { - "$ref": "#/parameters/CommonTenantParameterQuery" - } - ], - "responses": { - "200": { - "description": "Successful response.", - "schema": { - "$ref": "#/definitions/ObjectsListResponse" - } - }, - "400": { - "description": "Malformed request.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Successful query result but no resource was found." - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "summary": "Get a list of Objects.", - "tags": [ - "objects" - ], - "x-available-in-mqtt": false, - "x-available-in-websocket": false - }, - "post": { - "description": "Registers a new Object. Provided meta-data and schema values are validated.", - "operationId": "objects.create", - "x-serviceIds": [ - "weaviate.local.add" - ], - "parameters": [ - { - "in": "body", - "name": "body", - "required": true, - "schema": { - "$ref": "#/definitions/Object" - } - }, - { - "$ref": "#/parameters/CommonConsistencyLevelParameterQuery" - } - ], - "responses": { - "200": { - "description": "Object created.", - "schema": { - "$ref": "#/definitions/Object" - } - }, - "400": { - "description": "Malformed request.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "summary": "Create Objects between two Objects (object and subject).", - "tags": [ - "objects" - ], - "x-available-in-mqtt": false, - "x-available-in-websocket": false - } - }, - "/objects/{id}": { - "delete": { - "description": "Deletes an Object from the system.", - "operationId": "objects.delete", - "x-serviceIds": [ - "weaviate.local.manipulate" - ], - "parameters": [ - { - "description": "Unique ID of the Object.", - "format": "uuid", - "in": "path", - "name": "id", - "required": true, - "type": "string" - }, - { - "$ref": "#/parameters/CommonConsistencyLevelParameterQuery" - }, - { - "$ref": "#/parameters/CommonTenantParameterQuery" - } - ], - "responses": { - "204": { - "description": "Successfully deleted." - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Successful query result but no resource was found." - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "summary": "Delete an Object based on its UUID.", - "tags": [ - "objects" - ], - "x-available-in-mqtt": true, - "x-available-in-websocket": true, - "deprecated": true - }, - "get": { - "description": "Lists Objects.", - "operationId": "objects.get", - "x-serviceIds": [ - "weaviate.local.query" - ], - "parameters": [ - { - "description": "Unique ID of the Object.", - "format": "uuid", - "in": "path", - "name": "id", - "required": true, - "type": "string" - }, - { - "$ref": "#/parameters/CommonIncludeParameterQuery" - } - ], - "responses": { - "200": { - "description": "Successful response.", - "schema": { - "$ref": "#/definitions/Object" - } - }, - "400": { - "description": "Malformed request.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Successful query result but no resource was found." - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "summary": "Get a specific Object based on its UUID and a Object UUID. Also available as Websocket bus.", - "tags": [ - "objects" - ], - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "deprecated": true - }, - "patch": { - "description": "Updates an Object. This method supports json-merge style patch semantics (RFC 7396). Provided meta-data and schema values are validated. LastUpdateTime is set to the time this function is called.", - "operationId": "objects.patch", - "x-serviceIds": [ - "weaviate.local.manipulate" - ], - "parameters": [ - { - "description": "Unique ID of the Object.", - "format": "uuid", - "in": "path", - "name": "id", - "required": true, - "type": "string" - }, - { - "description": "RFC 7396-style patch, the body contains the object to merge into the existing object.", - "in": "body", - "name": "body", - "required": false, - "schema": { - "$ref": "#/definitions/Object" - } - }, - { - "$ref": "#/parameters/CommonConsistencyLevelParameterQuery" - } - ], - "responses": { - "204": { - "description": "Successfully applied. No content provided." - }, - "400": { - "description": "The patch-JSON is malformed." - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Successful query result but no resource was found." - }, - "422": { - "description": "The patch-JSON is valid but unprocessable.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "summary": "Update an Object based on its UUID (using patch semantics).", - "tags": [ - "objects" - ], - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "deprecated": true - }, - "put": { - "description": "Updates an Object's data. Given meta-data and schema values are validated. LastUpdateTime is set to the time this function is called.", - "operationId": "objects.update", - "x-serviceIds": [ - "weaviate.local.manipulate" - ], - "parameters": [ - { - "description": "Unique ID of the Object.", - "format": "uuid", - "in": "path", - "name": "id", - "required": true, - "type": "string" - }, - { - "in": "body", - "name": "body", - "required": true, - "schema": { - "$ref": "#/definitions/Object" - } - }, - { - "$ref": "#/parameters/CommonConsistencyLevelParameterQuery" - } - ], - "responses": { - "200": { - "description": "Successfully received.", - "schema": { - "$ref": "#/definitions/Object" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Successful query result but no resource was found." - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "summary": "Update an Object based on its UUID.", - "tags": [ - "objects" - ], - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "deprecated": true - }, - "head": { - "description": "Checks if an Object exists in the system.", - "operationId": "objects.head", - "x-serviceIds": [ - "weaviate.local.manipulate" - ], - "parameters": [ - { - "description": "Unique ID of the Object.", - "format": "uuid", - "in": "path", - "name": "id", - "required": true, - "type": "string" - } - ], - "responses": { - "204": { - "description": "Object exists." - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Object doesn't exist." - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "summary": "Checks Object's existence based on its UUID.", - "tags": [ - "objects" - ], - "x-available-in-mqtt": true, - "x-available-in-websocket": true, - "deprecated": true - } - }, - "/objects/{className}/{id}": { - "get": { - "description": "Get a single data object", - "operationId": "objects.class.get", - "x-serviceIds": [ - "weaviate.local.query" - ], - "parameters": [ - { - "name": "className", - "in": "path", - "required": true, - "type": "string" - }, - { - "description": "Unique ID of the Object.", - "format": "uuid", - "in": "path", - "name": "id", - "required": true, - "type": "string" - }, - { - "$ref": "#/parameters/CommonIncludeParameterQuery" - }, - { - "$ref": "#/parameters/CommonConsistencyLevelParameterQuery" - }, - { - "$ref": "#/parameters/CommonNodeNameParameterQuery" - }, - { - "$ref": "#/parameters/CommonTenantParameterQuery" - } - ], - "responses": { - "200": { - "description": "Successful response.", - "schema": { - "$ref": "#/definitions/Object" - } - }, - "400": { - "description": "Malformed request.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Successful query result but no resource was found." - }, - "422": { - "description": "Request is well-formed (i.e., syntactically correct), but erroneous.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "summary": "Get a specific Object based on its class and UUID. Also available as Websocket bus.", - "tags": [ - "objects" - ], - "x-available-in-mqtt": false, - "x-available-in-websocket": false - }, - "delete": { - "description": "Delete a single data object.", - "operationId": "objects.class.delete", - "x-serviceIds": [ - "weaviate.local.manipulate" - ], - "parameters": [ - { - "name": "className", - "in": "path", - "required": true, - "type": "string" - }, - { - "description": "Unique ID of the Object.", - "format": "uuid", - "in": "path", - "name": "id", - "required": true, - "type": "string" - }, - { - "$ref": "#/parameters/CommonConsistencyLevelParameterQuery" - }, - { - "$ref": "#/parameters/CommonTenantParameterQuery" - } - ], - "responses": { - "204": { - "description": "Successfully deleted." - }, - "400": { - "description": "Malformed request.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Successful query result but no resource was found." - }, - "422": { - "description": "Request is well-formed (i.e., syntactically correct), but erroneous.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "summary": "Delete object based on its class and UUID.", - "tags": [ - "objects" - ], - "x-available-in-mqtt": true, - "x-available-in-websocket": true - }, - "put": { - "description": "Update an individual data object based on its class and uuid.", - "operationId": "objects.class.put", - "x-serviceIds": [ - "weaviate.local.manipulate" - ], - "parameters": [ - { - "name": "className", - "in": "path", - "required": true, - "type": "string" - }, - { - "description": "The uuid of the data object to update.", - "format": "uuid", - "in": "path", - "name": "id", - "required": true, - "type": "string" - }, - { - "in": "body", - "name": "body", - "required": true, - "schema": { - "$ref": "#/definitions/Object" - } - }, - { - "$ref": "#/parameters/CommonConsistencyLevelParameterQuery" - } - ], - "responses": { - "200": { - "description": "Successfully received.", - "schema": { - "$ref": "#/definitions/Object" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Successful query result but no resource was found." - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "summary": "Update a class object based on its uuid", - "tags": [ - "objects" - ], - "x-available-in-mqtt": false, - "x-available-in-websocket": false - }, - "patch": { - "description": "Update an individual data object based on its class and uuid. This method supports json-merge style patch semantics (RFC 7396). Provided meta-data and schema values are validated. LastUpdateTime is set to the time this function is called.", - "operationId": "objects.class.patch", - "x-serviceIds": [ - "weaviate.local.manipulate" - ], - "parameters": [ - { - "description": "The class name as defined in the schema", - "name": "className", - "in": "path", - "required": true, - "type": "string" - }, - { - "description": "The uuid of the data object to update.", - "format": "uuid", - "in": "path", - "name": "id", - "required": true, - "type": "string" - }, - { - "description": "RFC 7396-style patch, the body contains the object to merge into the existing object.", - "in": "body", - "name": "body", - "required": false, - "schema": { - "$ref": "#/definitions/Object" - } - }, - { - "$ref": "#/parameters/CommonConsistencyLevelParameterQuery" - } - ], - "responses": { - "204": { - "description": "Successfully applied. No content provided." - }, - "400": { - "description": "The patch-JSON is malformed.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Successful query result but no resource was found." - }, - "422": { - "description": "The patch-JSON is valid but unprocessable.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "summary": "Update an Object based on its UUID (using patch semantics).", - "tags": [ - "objects" - ], - "x-available-in-mqtt": false, - "x-available-in-websocket": false - }, - "head": { - "description": "Checks if a data object exists without retrieving it.", - "operationId": "objects.class.head", - "x-serviceIds": [ - "weaviate.local.manipulate" - ], - "parameters": [ - { - "description": "The class name as defined in the schema", - "name": "className", - "in": "path", - "required": true, - "type": "string" - }, - { - "description": "The uuid of the data object", - "format": "uuid", - "in": "path", - "name": "id", - "required": true, - "type": "string" - }, - { - "$ref": "#/parameters/CommonConsistencyLevelParameterQuery" - }, - { - "$ref": "#/parameters/CommonTenantParameterQuery" - } - ], - "responses": { - "204": { - "description": "Object exists." - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Object doesn't exist." - }, - "422": { - "description": "Request is well-formed (i.e., syntactically correct), but erroneous.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "summary": "Checks object's existence based on its class and uuid.", - "tags": [ - "objects" - ], - "x-available-in-mqtt": true, - "x-available-in-websocket": true - } - }, - "/objects/{id}/references/{propertyName}": { - "post": { - "description": "Add a single reference to a class-property.", - "operationId": "objects.references.create", - "x-serviceIds": [ - "weaviate.local.manipulate" - ], - "parameters": [ - { - "description": "Unique ID of the Object.", - "format": "uuid", - "in": "path", - "name": "id", - "required": true, - "type": "string" - }, - { - "description": "Unique name of the property related to the Object.", - "in": "path", - "name": "propertyName", - "required": true, - "type": "string" - }, - { - "in": "body", - "name": "body", - "required": true, - "schema": { - "$ref": "#/definitions/SingleRef" - } - }, - { - "$ref": "#/parameters/CommonTenantParameterQuery" - } - ], - "responses": { - "200": { - "description": "Successfully added the reference." - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the property exists or that it is a class?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "summary": "Add a single reference to a class-property.", - "tags": [ - "objects" - ], - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "deprecated": true - }, - "put": { - "description": "Replace all references to a class-property.", - "operationId": "objects.references.update", - "x-serviceIds": [ - "weaviate.local.manipulate" - ], - "parameters": [ - { - "description": "Unique ID of the Object.", - "format": "uuid", - "in": "path", - "name": "id", - "required": true, - "type": "string" - }, - { - "description": "Unique name of the property related to the Object.", - "in": "path", - "name": "propertyName", - "required": true, - "type": "string" - }, - { - "in": "body", - "name": "body", - "required": true, - "schema": { - "$ref": "#/definitions/MultipleRef" - } - }, - { - "$ref": "#/parameters/CommonTenantParameterQuery" - } - ], - "responses": { - "200": { - "description": "Successfully replaced all the references." - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the property exists or that it is a class?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "summary": "Replace all references to a class-property.", - "tags": [ - "objects" - ], - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "deprecated": true - }, - "delete": { - "description": "Delete the single reference that is given in the body from the list of references that this property has.", - "operationId": "objects.references.delete", - "x-serviceIds": [ - "weaviate.local.manipulate" - ], - "parameters": [ - { - "description": "Unique ID of the Object.", - "format": "uuid", - "in": "path", - "name": "id", - "required": true, - "type": "string" - }, - { - "description": "Unique name of the property related to the Object.", - "in": "path", - "name": "propertyName", - "required": true, - "type": "string" - }, - { - "in": "body", - "name": "body", - "required": true, - "schema": { - "$ref": "#/definitions/SingleRef" - } - }, - { - "$ref": "#/parameters/CommonTenantParameterQuery" - } - ], - "responses": { - "204": { - "description": "Successfully deleted." - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Successful query result but no resource was found.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "summary": "Delete the single reference that is given in the body from the list of references that this property has.", - "tags": [ - "objects" - ], - "x-available-in-mqtt": false, - "x-available-in-websocket": false, - "deprecated": true - } - }, - "/objects/{className}/{id}/references/{propertyName}": { - "post": { - "description": "Add a single reference to a class-property.", - "operationId": "objects.class.references.create", - "x-serviceIds": [ - "weaviate.local.manipulate" - ], - "parameters": [ - { - "description": "The class name as defined in the schema", - "name": "className", - "in": "path", - "required": true, - "type": "string" - }, - { - "description": "Unique ID of the Object.", - "format": "uuid", - "in": "path", - "name": "id", - "required": true, - "type": "string" - }, - { - "description": "Unique name of the property related to the Object.", - "in": "path", - "name": "propertyName", - "required": true, - "type": "string" - }, - { - "in": "body", - "name": "body", - "required": true, - "schema": { - "$ref": "#/definitions/SingleRef" - } - }, - { - "$ref": "#/parameters/CommonConsistencyLevelParameterQuery" - }, - { - "$ref": "#/parameters/CommonTenantParameterQuery" - } - ], - "responses": { - "200": { - "description": "Successfully added the reference." - }, - "400": { - "description": "Malformed request.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Source object doesn't exist." - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the property exists or that it is a class?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "summary": "Add a single reference to a class-property.", - "tags": [ - "objects" - ], - "x-available-in-mqtt": false, - "x-available-in-websocket": false - }, - "put": { - "description": "Update all references of a property of a data object.", - "operationId": "objects.class.references.put", - "x-serviceIds": [ - "weaviate.local.manipulate" - ], - "parameters": [ - { - "description": "The class name as defined in the schema", - "name": "className", - "in": "path", - "required": true, - "type": "string" - }, - { - "description": "Unique ID of the Object.", - "format": "uuid", - "in": "path", - "name": "id", - "required": true, - "type": "string" - }, - { - "description": "Unique name of the property related to the Object.", - "in": "path", - "name": "propertyName", - "required": true, - "type": "string" - }, - { - "in": "body", - "name": "body", - "required": true, - "schema": { - "$ref": "#/definitions/MultipleRef" - } - }, - { - "$ref": "#/parameters/CommonConsistencyLevelParameterQuery" - }, - { - "$ref": "#/parameters/CommonTenantParameterQuery" - } - ], - "responses": { - "200": { - "description": "Successfully replaced all the references." - }, - "400": { - "description": "Malformed request.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Source object doesn't exist." - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the property exists or that it is a class?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "summary": "Replace all references to a class-property.", - "tags": [ - "objects" - ], - "x-available-in-mqtt": false, - "x-available-in-websocket": false - }, - "delete": { - "description": "Delete the single reference that is given in the body from the list of references that this property of a data object has", - "operationId": "objects.class.references.delete", - "x-serviceIds": [ - "weaviate.local.manipulate" - ], - "parameters": [ - { - "description": "The class name as defined in the schema", - "name": "className", - "in": "path", - "required": true, - "type": "string" - }, - { - "description": "Unique ID of the Object.", - "format": "uuid", - "in": "path", - "name": "id", - "required": true, - "type": "string" - }, - { - "description": "Unique name of the property related to the Object.", - "in": "path", - "name": "propertyName", - "required": true, - "type": "string" - }, - { - "in": "body", - "name": "body", - "required": true, - "schema": { - "$ref": "#/definitions/SingleRef" - } - }, - { - "$ref": "#/parameters/CommonConsistencyLevelParameterQuery" - }, - { - "$ref": "#/parameters/CommonTenantParameterQuery" - } - ], - "responses": { - "204": { - "description": "Successfully deleted." - }, - "400": { - "description": "Malformed request.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Successful query result but no resource was found.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the property exists or that it is a class?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "summary": "Delete the single reference that is given in the body from the list of references that this property has.", - "tags": [ - "objects" - ], - "x-available-in-mqtt": false, - "x-available-in-websocket": false - } - }, - "/objects/validate": { - "post": { - "description": "Validate an Object's schema and meta-data. It has to be based on a schema, which is related to the given Object to be accepted by this validation.", - "operationId": "objects.validate", - "x-serviceIds": [ - "weaviate.local.query.meta" - ], - "parameters": [ - { - "in": "body", - "name": "body", - "required": true, - "schema": { - "$ref": "#/definitions/Object" - } - } - ], - "responses": { - "200": { - "description": "Successfully validated." - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "summary": "Validate an Object based on a schema.", - "tags": [ - "objects" - ], - "x-available-in-mqtt": false, - "x-available-in-websocket": false - } - }, - "/batch/objects": { - "post": { - "description": "Register new Objects in bulk. Provided meta-data and schema values are validated.", - "operationId": "batch.objects.create", - "x-serviceIds": [ - "weaviate.local.add" - ], - "parameters": [ - { - "in": "body", - "name": "body", - "required": true, - "schema": { - "type": "object", - "properties": { - "fields": { - "description": "Define which fields need to be returned. Default value is ALL", - "type": "array", - "items": { - "type": "string", - "default": "ALL", - "enum": [ - "ALL", - "class", - "schema", - "id", - "creationTimeUnix" - ] - } - }, - "objects": { - "type": "array", - "items": { - "$ref": "#/definitions/Object" - } - } - } - } - }, - { - "$ref": "#/parameters/CommonConsistencyLevelParameterQuery" - } - ], - "responses": { - "200": { - "description": "Request succeeded, see response body to get detailed information about each batched item.", - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/ObjectsGetResponse" - } - } - }, - "400": { - "description": "Malformed request.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "summary": "Creates new Objects based on a Object template as a batch.", - "tags": [ - "batch", - "objects" - ], - "x-available-in-mqtt": false, - "x-available-in-websocket": false - }, - "delete": { - "description": "Delete Objects in bulk that match a certain filter.", - "operationId": "batch.objects.delete", - "x-serviceIds": [ - "weaviate.local.manipulate" - ], - "parameters": [ - { - "in": "body", - "name": "body", - "required": true, - "schema": { - "$ref": "#/definitions/BatchDelete" - } - }, - { - "$ref": "#/parameters/CommonConsistencyLevelParameterQuery" - }, - { - "$ref": "#/parameters/CommonTenantParameterQuery" - } - ], - "responses": { - "200": { - "description": "Request succeeded, see response body to get detailed information about each batched item.", - "schema": { - "$ref": "#/definitions/BatchDeleteResponse" - } - }, - "400": { - "description": "Malformed request.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "summary": "Deletes Objects based on a match filter as a batch.", - "tags": [ - "batch", - "objects" - ], - "x-available-in-mqtt": false, - "x-available-in-websocket": false - } - }, - "/batch/references": { - "post": { - "description": "Register cross-references between any class items (objects or objects) in bulk.", - "operationId": "batch.references.create", - "x-serviceIds": [ - "weaviate.local.add" - ], - "parameters": [ - { - "in": "body", - "name": "body", - "description": "A list of references to be batched. The ideal size depends on the used database connector. Please see the documentation of the used connector for help", - "required": true, - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/BatchReference" - } - } - }, - { - "$ref": "#/parameters/CommonConsistencyLevelParameterQuery" - } - ], - "responses": { - "200": { - "description": "Request Successful. Warning: A successful request does not guarantee that every batched reference was successfully created. Inspect the response body to see which references succeeded and which failed.", - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/BatchReferenceResponse" - } - } - }, - "400": { - "description": "Malformed request.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "summary": "Creates new Cross-References between arbitrary classes in bulk.", - "tags": [ - "batch", - "references" - ], - "x-available-in-mqtt": false, - "x-available-in-websocket": false - } - }, - "/graphql": { - "post": { - "description": "Get an object based on GraphQL", - "operationId": "graphql.post", - "x-serviceIds": [ - "weaviate.local.query", - "weaviate.local.query.meta", - "weaviate.network.query", - "weaviate.network.query.meta" - ], - "parameters": [ - { - "description": "The GraphQL query request parameters.", - "in": "body", - "name": "body", - "required": true, - "schema": { - "$ref": "#/definitions/GraphQLQuery" - } - } - ], - "responses": { - "200": { - "description": "Successful query (with select).", - "schema": { - "$ref": "#/definitions/GraphQLResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "summary": "Get a response based on GraphQL", - "tags": [ - "graphql" - ], - "x-available-in-mqtt": false, - "x-available-in-websocket": false - } - }, - "/graphql/batch": { - "post": { - "description": "Perform a batched GraphQL query", - "operationId": "graphql.batch", - "x-serviceIds": [ - "weaviate.local.query", - "weaviate.local.query.meta", - "weaviate.network.query", - "weaviate.network.query.meta" - ], - "parameters": [ - { - "description": "The GraphQL queries.", - "in": "body", - "name": "body", - "required": true, - "schema": { - "$ref": "#/definitions/GraphQLQueries" - } - } - ], - "responses": { - "200": { - "description": "Successful query (with select).", - "schema": { - "$ref": "#/definitions/GraphQLResponses" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Request body is well-formed (i.e., syntactically correct), but semantically erroneous. Are you sure the class is defined in the configuration file?", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "summary": "Get a response based on GraphQL.", - "tags": [ - "graphql" - ], - "x-available-in-mqtt": false, - "x-available-in-websocket": false - } - }, - "/meta": { - "get": { - "description": "Gives meta information about the server and can be used to provide information to another Weaviate instance that wants to interact with the current instance.", - "operationId": "meta.get", - "x-serviceIds": [ - "weaviate.local.query.meta" - ], - "responses": { - "200": { - "description": "Successful response.", - "schema": { - "$ref": "#/definitions/Meta" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "summary": "Returns meta information of the current Weaviate instance.", - "tags": [ - "meta" - ], - "x-available-in-mqtt": false, - "x-available-in-websocket": false - } - }, - "/schema/cluster-status": { - "get": { - "operationId": "schema.cluster.status", - "tags": [ - "schema" - ], - "responses": { - "200": { - "description": "The schema in the cluster is in sync.", - "schema": { - "$ref": "#/definitions/SchemaClusterStatus" - } - }, - "500": { - "description": "The schema is either out of sync (see response body) or the sync check could not be completed.", - "schema": { - "$ref": "#/definitions/SchemaClusterStatus" - } - } - } - } - }, - "/schema": { - "get": { - "summary": "Dump the current the database schema.", - "operationId": "schema.dump", - "x-serviceIds": [ - "weaviate.local.query.meta" - ], - "tags": [ - "schema" - ], - "responses": { - "200": { - "description": "Successfully dumped the database schema.", - "schema": { - "$ref": "#/definitions/Schema" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - } - }, - "post": { - "summary": "Create a new Object class in the schema.", - "operationId": "schema.objects.create", - "x-serviceIds": [ - "weaviate.local.add.meta" - ], - "tags": [ - "schema" - ], - "parameters": [ - { - "name": "objectClass", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/Class" - } - } - ], - "responses": { - "200": { - "description": "Added the new Object class to the schema.", - "schema": { - "$ref": "#/definitions/Class" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Invalid Object class", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - } - } - }, - "/schema/{className}": { - "get": { - "summary": "Get a single class from the schema", - "operationId": "schema.objects.get", - "x-serviceIds": [ - "weaviate.local.get.meta" - ], - "tags": [ - "schema" - ], - "parameters": [ - { - "name": "className", - "in": "path", - "required": true, - "type": "string" - } - ], - "responses": { - "200": { - "description": "Found the Class, returned as body", - "schema": { - "$ref": "#/definitions/Class" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "This class does not exist" - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - } - }, - "delete": { - "summary": "Remove an Object class (and all data in the instances) from the schema.", - "operationId": "schema.objects.delete", - "x-serviceIds": [ - "weaviate.local.manipulate.meta" - ], - "tags": [ - "schema" - ], - "parameters": [ - { - "name": "className", - "in": "path", - "required": true, - "type": "string" - } - ], - "responses": { - "200": { - "description": "Removed the Object class from the schema." - }, - "400": { - "description": "Could not delete the Object class.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - } - }, - "put": { - "summary": "Update settings of an existing schema class", - "description": "Use this endpoint to alter an existing class in the schema. Note that not all settings are mutable. If an error about immutable fields is returned and you still need to update this particular setting, you will have to delete the class (and the underlying data) and recreate. This endpoint cannot be used to modify properties. Instead use POST /v1/schema/{className}/properties. A typical use case for this endpoint is to update configuration, such as the vectorIndexConfig. Note that even in mutable sections, such as vectorIndexConfig, some fields may be immutable.", - "operationId": "schema.objects.update", - "x-serviceIds": [ - "weaviate.local.manipulate.meta" - ], - "tags": [ - "schema" - ], - "parameters": [ - { - "name": "className", - "in": "path", - "required": true, - "type": "string" - }, - { - "name": "objectClass", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/Class" - } - } - ], - "responses": { - "200": { - "description": "Class was updated successfully", - "schema": { - "$ref": "#/definitions/Class" - } - }, - "422": { - "description": "Invalid update attempt", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Class to be updated does not exist", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - } - } - }, - "/schema/{className}/properties": { - "post": { - "summary": "Add a property to an Object class.", - "operationId": "schema.objects.properties.add", - "x-serviceIds": [ - "weaviate.local.manipulate.meta" - ], - "tags": [ - "schema" - ], - "parameters": [ - { - "name": "className", - "in": "path", - "required": true, - "type": "string" - }, - { - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/Property" - } - } - ], - "responses": { - "200": { - "description": "Added the property.", - "schema": { - "$ref": "#/definitions/Property" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Invalid property.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - } - } - }, - "/schema/{className}/shards": { - "get": { - "summary": "Get the shards status of an Object class", - "operationId": "schema.objects.shards.get", - "x-serviceIds": [ - "weaviate.local.get.meta" - ], - "tags": [ - "schema" - ], - "parameters": [ - { - "name": "className", - "in": "path", - "required": true, - "type": "string" - }, - { - "name": "tenant", - "in": "query", - "type": "string" - } - ], - "responses": { - "200": { - "description": "Found the status of the shards, returned as body", - "schema": { - "$ref": "#/definitions/ShardStatusList" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "This class does not exist", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - } - } - }, - "/schema/{className}/shards/{shardName}": { - "put": { - "description": "Update shard status of an Object Class", - "operationId": "schema.objects.shards.update", - "x-serviceIds": [ - "weaviate.local.manipulate.meta" - ], - "tags": [ - "schema" - ], - "parameters": [ - { - "name": "className", - "in": "path", - "required": true, - "type": "string" - }, - { - "name": "shardName", - "in": "path", - "required": true, - "type": "string" - }, - { - "in": "body", - "name": "body", - "required": true, - "schema": { - "$ref": "#/definitions/ShardStatus" - } - } - ], - "responses": { - "200": { - "description": "Shard status was updated successfully", - "schema": { - "$ref": "#/definitions/ShardStatus" - } - }, - "422": { - "description": "Invalid update attempt", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Shard to be updated does not exist", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - } - } - }, - "/schema/{className}/tenants": { - "post": { - "description": "Create a new tenant for a specific class", - "operationId": "tenants.create", - "tags": [ - "schema" - ], - "parameters": [ - { - "name": "className", - "in": "path", - "required": true, - "type": "string" - }, - { - "in": "body", - "name": "body", - "required": true, - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/Tenant" - } - } - } - ], - "responses": { - "200": { - "description": "Added new tenants to the specified class", - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/Tenant" - } - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Invalid Tenant class", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - } - }, - "put": { - "description": "Update tenant of a specific class", - "operationId": "tenants.update", - "tags": [ - "schema" - ], - "parameters": [ - { - "name": "className", - "in": "path", - "required": true, - "type": "string" - }, - { - "in": "body", - "name": "body", - "required": true, - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/Tenant" - } - } - } - ], - "responses": { - "200": { - "description": "Updated tenants of the specified class", - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/Tenant" - } - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Invalid Tenant class", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - } - }, - "delete": { - "description": "delete tenants from a specific class", - "operationId": "tenants.delete", - "tags": [ - "schema" - ], - "parameters": [ - { - "name": "className", - "in": "path", - "required": true, - "type": "string" - }, - { - "in": "body", - "name": "tenants", - "required": true, - "schema": { - "type": "array", - "items": { - "type": "string" - } - } - } - ], - "responses": { - "200": { - "description": "Deleted tenants from specified class." - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Invalid Tenant class", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - } - }, - "get": { - "description": "get all tenants from a specific class", - "operationId": "tenants.get", - "tags": [ - "schema" - ], - "parameters": [ - { - "name": "className", - "in": "path", - "required": true, - "type": "string" - } - ], - "responses": { - "200": { - "description": "tenants from specified class.", - "schema": { - "type": "array", - "items": { - "$ref": "#/definitions/Tenant" - } - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Invalid Tenant class", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - } - } - }, - "/backups/{backend}": { - "post": { - "description": "Starts a process of creating a backup for a set of classes", - "operationId": "backups.create", - "x-serviceIds": [ - "weaviate.local.backup" - ], - "tags": [ - "backups" - ], - "parameters": [ - { - "name": "backend", - "in": "path", - "required": true, - "type": "string", - "description": "Backup backend name e.g. filesystem, gcs, s3." - }, - { - "in": "body", - "name": "body", - "required": true, - "schema": { - "$ref": "#/definitions/BackupCreateRequest" - } - } - ], - "responses": { - "200": { - "description": "Backup create process successfully started.", - "schema": { - "$ref": "#/definitions/BackupCreateResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Invalid backup creation attempt.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - } - } - }, - "/backups/{backend}/{id}": { - "get": { - "description": "Returns status of backup creation attempt for a set of classes", - "operationId": "backups.create.status", - "x-serviceIds": [ - "weaviate.local.backup" - ], - "tags": [ - "backups" - ], - "parameters": [ - { - "name": "backend", - "in": "path", - "required": true, - "type": "string", - "description": "Backup backend name e.g. filesystem, gcs, s3." - }, - { - "name": "id", - "in": "path", - "required": true, - "type": "string", - "description": "The ID of a backup. Must be URL-safe and work as a filesystem path, only lowercase, numbers, underscore, minus characters allowed." - } - ], - "responses": { - "200": { - "description": "Backup creation status successfully returned", - "schema": { - "$ref": "#/definitions/BackupCreateStatusResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Not Found - Backup does not exist", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Invalid backup restoration status attempt.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - } - } - }, - "/backups/{backend}/{id}/restore": { - "post": { - "description": "Starts a process of restoring a backup for a set of classes", - "operationId": "backups.restore", - "x-serviceIds": [ - "weaviate.local.backup" - ], - "tags": [ - "backups" - ], - "parameters": [ - { - "name": "backend", - "in": "path", - "required": true, - "type": "string", - "description": "Backup backend name e.g. filesystem, gcs, s3." - }, - { - "name": "id", - "in": "path", - "required": true, - "type": "string", - "description": "The ID of a backup. Must be URL-safe and work as a filesystem path, only lowercase, numbers, underscore, minus characters allowed." - }, - { - "in": "body", - "name": "body", - "required": true, - "schema": { - "$ref": "#/definitions/BackupRestoreRequest" - } - } - ], - "responses": { - "200": { - "description": "Backup restoration process successfully started.", - "schema": { - "$ref": "#/definitions/BackupRestoreResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Not Found - Backup does not exist", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Invalid backup restoration attempt.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - } - }, - "get": { - "description": "Returns status of a backup restoration attempt for a set of classes", - "operationId": "backups.restore.status", - "x-serviceIds": [ - "weaviate.local.backup" - ], - "tags": [ - "backups" - ], - "parameters": [ - { - "name": "backend", - "in": "path", - "required": true, - "type": "string", - "description": "Backup backend name e.g. filesystem, gcs, s3." - }, - { - "name": "id", - "in": "path", - "required": true, - "type": "string", - "description": "The ID of a backup. Must be URL-safe and work as a filesystem path, only lowercase, numbers, underscore, minus characters allowed." - } - ], - "responses": { - "200": { - "description": "Backup restoration status successfully returned", - "schema": { - "$ref": "#/definitions/BackupRestoreStatusResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Not Found - Backup does not exist", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - } - } - }, - "/nodes": { - "get": { - "description": "Returns status of Weaviate DB.", - "operationId": "nodes.get", - "x-serviceIds": [ - "weaviate.nodes.status.get" - ], - "tags": [ - "nodes" - ], - "parameters": [ - { - "$ref": "#/parameters/CommonOutputVerbosityParameterQuery" - } - ], - "responses": { - "200": { - "description": "Nodes status successfully returned", - "schema": { - "$ref": "#/definitions/NodesStatusResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Not Found - Backup does not exist", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Invalid backup restoration status attempt.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - } - } - }, - "/nodes/{className}": { - "get": { - "description": "Returns status of Weaviate DB.", - "operationId": "nodes.get.class", - "x-serviceIds": [ - "weaviate.nodes.status.get.class" - ], - "tags": [ - "nodes" - ], - "parameters": [ - { - "name": "className", - "in": "path", - "required": true, - "type": "string" - }, - { - "$ref": "#/parameters/CommonOutputVerbosityParameterQuery" - } - ], - "responses": { - "200": { - "description": "Nodes status successfully returned", - "schema": { - "$ref": "#/definitions/NodesStatusResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "404": { - "description": "Not Found - Backup does not exist", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "422": { - "description": "Invalid backup restoration status attempt.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - } - } - }, - "/classifications/": { - "post": { - "description": "Trigger a classification based on the specified params. Classifications will run in the background, use GET /classifications/ to retrieve the status of your classification.", - "operationId": "classifications.post", - "x-serviceIds": [ - "weaviate.classifications.post" - ], - "parameters": [ - { - "description": "parameters to start a classification", - "in": "body", - "schema": { - "$ref": "#/definitions/Classification" - }, - "name": "params", - "required": true - } - ], - "responses": { - "201": { - "description": "Successfully started classification.", - "schema": { - "$ref": "#/definitions/Classification" - } - }, - "400": { - "description": "Incorrect request", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "summary": "Starts a classification.", - "tags": [ - "classifications" - ] - } - }, - "/classifications/{id}": { - "get": { - "description": "Get status, results and metadata of a previously created classification", - "operationId": "classifications.get", - "x-serviceIds": [ - "weaviate.classifications.get" - ], - "parameters": [ - { - "description": "classification id", - "in": "path", - "type": "string", - "name": "id", - "required": true - } - ], - "responses": { - "200": { - "description": "Found the classification, returned as body", - "schema": { - "$ref": "#/definitions/Classification" - } - }, - "404": { - "description": "Not Found - Classification does not exist" - }, - "401": { - "description": "Unauthorized or invalid credentials." - }, - "403": { - "description": "Forbidden", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "An error has occurred while trying to fulfill the request. Most likely the ErrorResponse will contain more information about the error.", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - }, - "summary": "View previously created classification", - "tags": [ - "classifications" - ] - } - } - }, - "produces": [ - "application/json" - ], - "schemes": [ - "https" - ], - "security": [ - {}, - { - "oidc": [] - } - ], - "securityDefinitions": { - "oidc": { - "type": "oauth2", - "description": "OIDC (OpenConnect ID - based on OAuth2)", - "flow": "implicit", - "authorizationUrl": "http://to-be-configured-in-the-application-config" - } - }, - "swagger": "2.0", - "tags": [ - { - "name": "objects" - }, - { - "name": "batch", - "description": "These operations allow to execute batch requests for Objects and Objects. Mostly used for importing large datasets." - }, - { - "name": "graphql" - }, - { - "name": "meta" - }, - { - "name": "P2P" - }, - { - "name": "contextionary-API", - "description": "All functions related to the Contextionary." - }, - { - "name": "schema", - "description": "These operations enable manipulation of the schema in Weaviate schema." - } - ] -} diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 3b851fe15f54b9e9428912b2492c13e44ee7d7eb..0000000000000000000000000000000000000000 --- a/requirements.txt +++ /dev/null @@ -1,18 +0,0 @@ -weaviate-client==4.* -sentence-transformers -langchain -langchain_community -lxml -beautifulsoup4 - -transformers==4.34.1 -fastapi==0.103.2 -uvicorn==0.23.2 -nltk==3.8.1 -torch==2.0.1 -sentencepiece==0.1.99 -sentence-transformers==2.2.2 -optimum==1.13.2 -onnxruntime==1.16.1 -onnx==1.14.1 -ipywidgets diff --git a/requirements_Orig.txt b/requirements_Orig.txt deleted file mode 100644 index 7e01105c9e1a46b6030a56ac4a8fe7763d1fa261..0000000000000000000000000000000000000000 --- a/requirements_Orig.txt +++ /dev/null @@ -1,19 +0,0 @@ -transformers==4.40.1 -torch==2.3.0 -#gradio -#sentencepiece -#protobuf -weaviate-client==4.* -sentence-transformers -langchain -lxml -#huggingface-hub -#semantic-text-splitter -#tokenizers -#json5 -#regex -beautifulsoup4 -uvicorn -fastapi -optimum==1.16.2 -onnx \ No newline at end of file diff --git a/semsearch._Hld03py b/semsearch._Hld03py deleted file mode 100644 index 7fa6d39f8849b8f66c577150bb1b0005d79d1731..0000000000000000000000000000000000000000 --- a/semsearch._Hld03py +++ /dev/null @@ -1,402 +0,0 @@ -import weaviate - -from sentence_transformers import SentenceTransformer -from langchain_community.document_loaders import BSHTMLLoader -from pathlib import Path -from lxml import html -import logging -from semantic_text_splitter import HuggingFaceTextSplitter -from tokenizers import Tokenizer -import json -import os -import re -import logging - -import llama_cpp -from llama_cpp import Llama -import ipywidgets as widgets -import time -from IPython.display import display, clear_output - -weaviate_logger = logging.getLogger("httpx") -weaviate_logger.setLevel(logging.WARNING) - -logger = logging.getLogger(__name__) -logging.basicConfig(level=logging.INFO) - - - -################################################################# -# Connect to Weaviate vector database. -################################################################# -client = "" -def connectToWeaviateDB(): - ###################################################### - # Connect to the Weaviate vector database. - logger.info("#### Create Weaviate db client connection.") - client = weaviate.connect_to_custom( - http_host="127.0.0.1", - http_port=8080, - http_secure=False, - grpc_host="127.0.0.1", - grpc_port=50051, - grpc_secure=False - ) - client.connect() - - -####################################################### -# Read each text input file, parse it into a document, -# chunk it, collect chunks and document name. -####################################################### -webpageDocNames = [] -page_contentArray = [] -webpageTitles = [] -webpageChunks = [] -webpageChunksDocNames = [] - -def readParseChunkFiles(): - logger.info("#### Read and chunk input text files.") - for filename in os.listdir(pathString): - logger.info(filename) - path = Path(pathString + "/" + filename) - filename = filename.rstrip(".html") - webpageDocNames.append(filename) - htmlLoader = BSHTMLLoader(path,"utf-8") - htmlData = htmlLoader.load() - - title = htmlData[0].metadata['title'] - page_content = htmlData[0].page_content - - # Clean data. Remove multiple newlines, etc. - page_content = re.sub(r'\n+', '\n',page_content) - - page_contentArray.append(page_content); - webpageTitles.append (title) - max_tokens = 1000 - tokenizer = Tokenizer.from_pretrained("bert-base-uncased") - logger.debug(f"### tokenizer: {tokenizer}") - splitter = HuggingFaceTextSplitter(tokenizer, trim_chunks=True) - chunksOnePage = splitter.chunks(page_content, chunk_capacity=50) - - chunks = [] - for chnk in chunksOnePage: - logger.debug(f"#### chnk in file: {chnk}") - chunks.append(chnk) - logger.debug(f"chunks: {chunks}") - webpageChunks.append(chunks) - webpageChunksDocNames.append(filename + "Chunks") - - logger.debug(f"### filename, title: {filename}, {title}") - - logger.debug(f"### webpageDocNames: {webpageDocNames}") - -################################################################# -# Create the chunks collection for the Weaviate database. -################################################################# -def createChunksCollection(): - logger.info("#### createChunksCollection() entered.") - if client.collections.exists("Chunks"): - client.collections.delete("Chunks") - - class_obj = { - "class": "Chunks", - "description": "Collection for document chunks.", - "vectorizer": "text2vec-transformers", - "moduleConfig": { - "text2vec-transformers": { - "vectorizeClassName": True - } - }, - "vectorIndexType": "hnsw", - "vectorIndexConfig": { - "distance": "cosine", - }, - "properties": [ - { - "name": "chunk", - "dataType": ["text"], - "description": "Single webpage chunk.", - "vectorizer": "text2vec-transformers", - "moduleConfig": { - "text2vec-transformers": { - "vectorizePropertyName": False, - "skip": False, - "tokenization": "lowercase" - } - } - }, - { - "name": "chunk_index", - "dataType": ["int"] - }, - { - "name": "webpage", - "dataType": ["Documents"], - "description": "Webpage content chunks.", - - "invertedIndexConfig": { - "bm25": { - "b": 0.75, - "k1": 1.2 - } - } - } - ] - } - return(client.collections.create_from_dict(class_obj)) - - -##################################################################### -# Create the document collection for the Weaviate database. -##################################################################### -def createWebpageCollection(): - logger.info("#### createWebpageCollection() entered.") - if client.collections.exists("Documents"): - client.collections.delete("Documents") - - class_obj = { - "class": "Documents", - "description": "For first attempt at loading a Weviate database.", - "vectorizer": "text2vec-transformers", - "moduleConfig": { - "text2vec-transformers": { - "vectorizeClassName": False - } - }, - "vectorIndexType": "hnsw", - "vectorIndexConfig": { - "distance": "cosine", - }, - "properties": [ - { - "name": "title", - "dataType": ["text"], - "description": "HTML doc title.", - "vectorizer": "text2vec-transformers", - "moduleConfig": { - "text2vec-transformers": { - "vectorizePropertyName": True, - "skip": False, - "tokenization": "lowercase" - } - }, - "invertedIndexConfig": { - "bm25": { - "b": 0.75, - "k1": 1.2 - }, - } - }, - { - "name": "content", - "dataType": ["text"], - "description": "HTML page content.", - "moduleConfig": { - "text2vec-transformers": { - "vectorizePropertyName": True, - "tokenization": "whitespace" - } - } - } - ] - } - return(client.collections.create_from_dict(class_obj)) - - -################################################################# -# Create document and chunk objects in database. -################################################################# -def createDatabaseObjects(): - logger.info("#### Create page/doc and chunk db objects.") - for i, className in enumerate(webpageDocNames): - title = webpageTitles[i] - logger.debug(f"## className, title: {className}, {title}") - # Create Webpage Object - page_content = page_contentArray[i] - # Insert the document. - wpCollectionObj_uuid = wpCollection.data.insert( - { - "name": className, - "title": title, - "content": page_content - } - ) - - # Insert the chunks for the document. - for i2, chunk in enumerate(webpageChunks[i]): - chunk_uuid = wpChunkCollection.data.insert( - { - "title": title, - "chunk": chunk, - "chunk_index": i2, - "references": - { - "webpage": wpCollectionObj_uuid - } - } - ) - - -################################################################# -# Create display widgets. -################################################################# -output_widget = "" -systemTextArea = "" -userTextArea = "" -ragPromptTextArea = "" -responseTextArea = "" -selectRag = "" -submitButton = "" -def createWidgets(): - output_widget = widgets.Output() - with output_widget: - print("### Create widgets entered.") - - systemTextArea = widgets.Textarea( - value='', - placeholder='Enter System Prompt.', - description='Sys Prompt: ', - disabled=False, - layout=widgets.Layout(width='300px', height='80px') - ) - - userTextArea = widgets.Textarea( - value='', - placeholder='Enter User Prompt.', - description='User Prompt: ', - disabled=False, - layout=widgets.Layout(width='435px', height='110px') - ) - - ragPromptTextArea = widgets.Textarea( - value='', - placeholder='App generated prompt with RAG information.', - description='RAG Prompt: ', - disabled=False, - layout=widgets.Layout(width='580px', height='180px') - ) - - responseTextArea = widgets.Textarea( - value='', - placeholder='LLM generated response.', - description='LLM Resp: ', - disabled=False, - layout=widgets.Layout(width='780px', height='200px') - ) - - selectRag = widgets.Checkbox( - value=False, - description='Use RAG', - disabled=False - ) - - submitButton = widgets.Button( - description='Run Model.', - disabled=False, - button_style='', # 'success', 'info', 'warning', 'danger' or '' - tooltip='Click', - icon='check' # (FontAwesome names without the `fa-` prefix) - ) - - -###################################################################### -# MAINLINE -###################################################################### -logger.info("#### MAINLINE ENTERED.") - -#pathString = "/Users/660565/KPSAllInOne/ProgramFilesX86/WebCopy/DownloadedWebSites/LLMPOC_HTML" -pathString = "/app/inputDocs" -chunks = [] -webpageDocNames = [] -page_contentArray = [] -webpageChunks = [] -webpageTitles = [] -webpageChunksDocNames = [] - -#connectToWeaviateDB() -logger.info("#### Create Weaviate db client connection.") -client = weaviate.connect_to_custom( - http_host="127.0.0.1", - http_port=8080, - http_secure=False, - grpc_host="127.0.0.1", - grpc_port=50051, - grpc_secure=False -) -client.connect() - -readParseChunkFiles() -wpCollection = createWebpageCollection() -wpChunkCollection = createChunksCollection() - -#createDatabaseObjects() -logger.info("#### Create page/doc and chunk db objects.") -for i, className in enumerate(webpageDocNames): - title = webpageTitles[i] - logger.debug(f"## className, title: {className}, {title}") - # Create Webpage Object - page_content = page_contentArray[i] - # Insert the document. - wpCollectionObj_uuid = wpCollection.data.insert( - { - "name": className, - "title": title, - "content": page_content - } - ) - - # Insert the chunks for the document. - for i2, chunk in enumerate(webpageChunks[i]): - chunk_uuid = wpChunkCollection.data.insert( - { - "title": title, - "chunk": chunk, - "chunk_index": i2, - "references": - { - "webpage": wpCollectionObj_uuid - } - } - ) - -############################################################################### -# text contains prompt for vector DB. -text = "human-made computer cognitive ability" - - -############################################################################### -# Initial the the sentence transformer and encode the query prompt. -logger.info(f"#### Encode text query prompt to create vectors. {text}") -model = SentenceTransformer('/app/multi-qa-MiniLM-L6-cos-v1') - -vector = model.encode(text) -vectorList = [] - -logger.debug("#### Print vectors.") -for vec in vector: - vectorList.append(vec) -logger.debug(f"vectorList: {vectorList[2]}") - -# Fetch chunks and print chunks. -logger.info("#### Retrieve semchunks from db using vectors from prompt.") -semChunks = wpChunkCollection.query.near_vector( - near_vector=vectorList, - distance=0.7, - limit=3 -) -logger.debug(f"### semChunks[0]: {semChunks}") - -# Print chunks, corresponding document and document title. -logger.info("#### Print individual retrieved chunks.") -for chunk in enumerate(semChunks.objects): - logger.info(f"#### chunk: {chunk}") - webpage_uuid = chunk[1].properties['references']['webpage'] - logger.info(f"webpage_uuid: {webpage_uuid}") - wpFromChunk = wpCollection.query.fetch_object_by_id(webpage_uuid) - logger.info(f"### wpFromChunk title: {wpFromChunk.properties['title']}") - -logger.info("#### Closing client db connection.") -client.close() - -logger.info("#### Program terminating.") diff --git a/semsearch.py b/semsearch.py deleted file mode 100644 index 2ed4ff2ba6338e0af9e710252528712dc7fc1d68..0000000000000000000000000000000000000000 --- a/semsearch.py +++ /dev/null @@ -1,443 +0,0 @@ -import weaviate -from weaviate.connect import ConnectionParams -from weaviate.classes.init import AdditionalConfig, Timeout - -from sentence_transformers import SentenceTransformer -from langchain_community.document_loaders import BSHTMLLoader -from pathlib import Path -from lxml import html -import logging -from semantic_text_splitter import HuggingFaceTextSplitter -from tokenizers import Tokenizer -import json -import os -import re -import logging - -import llama_cpp -from llama_cpp import Llama -import ipywidgets as widgets -from IPython.display import display, clear_output - - -weaviate_logger = logging.getLogger("httpx") -weaviate_logger.setLevel(logging.DEBUG) - -logger = logging.getLogger(__name__) -logging.basicConfig(level=logging.INFO) - - - -###################################################################### -# MAINLINE -# -logger.info("#### MAINLINE ENTERED.") - -#pathString = "/Users/660565/KPSAllInOne/ProgramFilesX86/WebCopy/DownloadedWebSites/LLMPOC_HTML" -pathString = "/app/inputDocs" -chunks = [] -webpageDocNames = [] -page_contentArray = [] -webpageChunks = [] -webpageTitles = [] -webpageChunksDocNames = [] - -##################################################################### -# Create UI widgets. -output_widget = widgets.Output() -with output_widget: - print("### Create widgets entered.") - -systemTextArea = widgets.Textarea( - value='', - placeholder='Enter System Prompt.', - description='Sys Prompt: ', - disabled=False, - layout=widgets.Layout(width='300px', height='80px') -) - -userTextArea = widgets.Textarea( - value='', - placeholder='Enter User Prompt.', - description='User Prompt: ', - disabled=False, - layout=widgets.Layout(width='435px', height='110px') -) - -ragPromptTextArea = widgets.Textarea( - value='', - placeholder='App generated prompt with RAG information.', - description='RAG Prompt: ', - disabled=False, - layout=widgets.Layout(width='580px', height='180px') -) - -responseTextArea = widgets.Textarea( - value='', - placeholder='LLM generated response.', - description='LLM Resp: ', - disabled=False, - layout=widgets.Layout(width='780px', height='200px') -) - -selectRag = widgets.Checkbox( - value=False, - description='Use RAG', - disabled=False -) - -submitButton = widgets.Button( - description='Run Model.', - disabled=False, - button_style='', # 'success', 'info', 'warning', 'danger' or '' - tooltip='Click', - icon='check' # (FontAwesome names without the `fa-` prefix) -) - - -####################################################### -# Read each text input file, parse it into a document, -# chunk it, collect chunks and document name. -logger.info("#### Read and chunk input text files.") -for filename in os.listdir(pathString): - logger.info(filename) - path = Path(pathString + "/" + filename) - filename = filename.rstrip(".html") - webpageDocNames.append(filename) - htmlLoader = BSHTMLLoader(path,"utf-8") - htmlData = htmlLoader.load() - - title = htmlData[0].metadata['title'] - page_content = htmlData[0].page_content - - # Clean data. Remove multiple newlines, etc. - page_content = re.sub(r'\n+', '\n',page_content) - - page_contentArray.append(page_content); - webpageTitles.append(title) - max_tokens = 1000 - tokenizer = Tokenizer.from_pretrained("bert-base-uncased") - logger.debug(f"### tokenizer: {tokenizer}") - splitter = HuggingFaceTextSplitter(tokenizer, trim_chunks=True) - chunksOnePage = splitter.chunks(page_content, chunk_capacity=50) - - chunks = [] - for chnk in chunksOnePage: - logger.debug(f"#### chnk in file: {chnk}") - chunks.append(chnk) - logger.debug(f"chunks: {chunks}") - webpageChunks.append(chunks) - webpageChunksDocNames.append(filename + "Chunks") - - logger.debug(f"### filename, title: {filename}, {title}") - -logger.debug(f"### webpageDocNames: {webpageDocNames}") - - -###################################################### -# Connect to the Weaviate vector database. -logger.info("#### Create Weaviate db client connection.") -#client = weaviate.connect_to_custom( -# http_host="127.0.0.1", -# http_port=8080, -# http_secure=False, -# grpc_host="127.0.0.1", -# grpc_port=50051, -# grpc_secure=False, -# timeout=[600,600] -# #read_timeout=600, -# #write_timeout=90 -#) - -client = weaviate.WeaviateClient( - connection_params=ConnectionParams.from_params( - http_host="localhost", - http_port="8080", - http_secure=False, - grpc_host="localhost", - grpc_port="50051", - grpc_secure=False, - ), -# auth_client_secret=weaviate.auth.AuthApiKey("secr3tk3y"), -# additional_headers={ -# "X-OpenAI-Api-Key": os.getenv("OPENAI_APIKEY") -# }, - additional_config=AdditionalConfig( - timeout=Timeout(init=60, query=1800, insert=1800), # Values in seconds - ) -) -client.connect() - - -###################################################### -# Create database webpage and chunks collections. -#wpCollection = createWebpageCollection() -#wpChunkCollection = createChunksCollection() -logger.info("#### createWebpageCollection() entered.") -if client.collections.exists("Documents"): - client.collections.delete("Documents") - -class_obj = { - "class": "Documents", - "description": "For first attempt at loading a Weviate database.", - "vectorizer": "text2vec-transformers", - "moduleConfig": { - "text2vec-transformers": { - "vectorizeClassName": False - } - }, - "vectorIndexType": "hnsw", - "vectorIndexConfig": { - "distance": "cosine", - }, - "properties": [ - { - "name": "title", - "dataType": ["text"], - "description": "HTML doc title.", - "vectorizer": "text2vec-transformers", - "moduleConfig": { - "text2vec-transformers": { - "vectorizePropertyName": True, - "skip": False, - "tokenization": "lowercase" - } - }, - "invertedIndexConfig": { - "bm25": { - "b": 0.75, - "k1": 1.2 - }, - } - }, - { - "name": "content", - "dataType": ["text"], - "description": "HTML page content.", - "moduleConfig": { - "text2vec-transformers": { - "vectorizePropertyName": True, - "tokenization": "whitespace" - } - } - } - ] -} -wpCollection = client.collections.create_from_dict(class_obj) - -logger.info("#### createChunksCollection() entered.") -if client.collections.exists("Chunks"): - client.collections.delete("Chunks") - -class_obj = { - "class": "Chunks", - "description": "Collection for document chunks.", - "vectorizer": "text2vec-transformers", - "moduleConfig": { - "text2vec-transformers": { - "vectorizeClassName": True - } - }, - "vectorIndexType": "hnsw", - "vectorIndexConfig": { - "distance": "cosine", - }, - "properties": [ - { - "name": "chunk", - "dataType": ["text"], - "description": "Single webpage chunk.", - "vectorizer": "text2vec-transformers", - "moduleConfig": { - "text2vec-transformers": { - "vectorizePropertyName": False, - "skip": False, - "tokenization": "lowercase" - } - } - }, - { - "name": "chunk_index", - "dataType": ["int"] - }, - { - "name": "webpage", - "dataType": ["Documents"], - "description": "Webpage content chunks.", - - "invertedIndexConfig": { - "bm25": { - "b": 0.75, - "k1": 1.2 - } - } - } - ] -} -wpChunkCollection = client.collections.create_from_dict(class_obj) - - -########################################################### -# Create document and chunks objects in the database. -logger.info("#### Create page/doc db objects.") -for i, className in enumerate(webpageDocNames): - title = webpageTitles[i] - logger.debug(f"## className, title: {className}, {title}") - # Create Webpage Object - page_content = page_contentArray[i] - # Insert the document. - wpCollectionObj_uuid = wpCollection.data.insert( - { - "name": className, - "title": title, - "content": page_content - } - ) - logger.info("#### Create chunk db objects.") - # Insert the chunks for the document. - for i2, chunk in enumerate(webpageChunks[i]): - chunk_uuid = wpChunkCollection.data.insert( - { - "title": title, - "chunk": chunk, - "chunk_index": i2, - "references": - { - "webpage": wpCollectionObj_uuid - } - } - ) - -############################################################################### -# text contains prompt for vector DB. -text = "human-made computer cognitive ability" - - -############################################################################### -# Initial the the sentence transformer and encode the query prompt. -logger.info(f"#### Encode text query prompt to create vectors. {text}") -model = SentenceTransformer('/app/multi-qa-MiniLM-L6-cos-v1') - -vector = model.encode(text) -vectorList = [] - -logger.debug("#### Print vectors.") -for vec in vector: - vectorList.append(vec) -logger.debug(f"vectorList: {vectorList[2]}") - -# Fetch chunks and print chunks. -logger.info("#### Retrieve semchunks from db using vectors from prompt.") -semChunks = wpChunkCollection.query.near_vector( - near_vector=vectorList, - distance=0.7, - limit=3 -) -logger.debug(f"### semChunks[0]: {semChunks}") - -# Print chunks, corresponding document and document title. -logger.info("#### Print individual retrieved chunks.") -for chunk in enumerate(semChunks.objects): - logger.info(f"#### chunk: {chunk}") - webpage_uuid = chunk[1].properties['references']['webpage'] - logger.info(f"webpage_uuid: {webpage_uuid}") - wpFromChunk = wpCollection.query.fetch_object_by_id(webpage_uuid) - logger.info(f"### wpFromChunk title: {wpFromChunk.properties['title']}") - - - -#################################################################### -# -collection = client.collections.get("Chunks") -#model = SentenceTransformer('../multi-qa-MiniLM-L6-cos-v1') - -################################################################# -# Initialize the LLM. -model_path = "/app/llama-2-7b-chat.Q4_0.gguf" -llm = Llama(model_path, - #*, - n_gpu_layers=0, - split_mode=llama_cpp.LLAMA_SPLIT_MODE_LAYER, - main_gpu=0, - tensor_split=None, - vocab_only=False, - use_mmap=True, - use_mlock=False, - kv_overrides=None, - seed=llama_cpp.LLAMA_DEFAULT_SEED, - n_ctx=512, - n_batch=512, - n_threads=8, - n_threads_batch=16, - rope_scaling_type=llama_cpp.LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED, - pooling_type=llama_cpp.LLAMA_POOLING_TYPE_UNSPECIFIED, - rope_freq_base=0.0, - rope_freq_scale=0.0, - yarn_ext_factor=-1.0, - yarn_attn_factor=1.0, - yarn_beta_fast=32.0, - yarn_beta_slow=1.0, - yarn_orig_ctx=0, - logits_all=False, - embedding=False, - offload_kqv=True, - last_n_tokens_size=64, - lora_base=None, - lora_scale=1.0, - lora_path=None, - numa=False, - chat_format=None, - chat_handler=None, - draft_model=None, - tokenizer=None, - type_k=None, - type_v=None, - verbose=True - ) - - -display(systemTextArea) -display(userTextArea) -display(ragPromptTextArea) -display(responseTextArea) -display(selectRag) -display(submitButton) - -def setPrompt(pprompt,ragFlag): - print("\n### setPrompt() entered. ragFlag: ",ragFlag) - if ragFlag: - ragPrompt = setRagPrompt(pprompt) - userPrompt = pprompt + "\n" + ragPrompt - prompt = userPrompt - else: - userPrompt = pprompt - prompt = f""" [INST] <> {systemTextArea.value} > Q: {userPrompt} A: [/INST]""" - return prompt - -def runModel(prompt): - output = llm.create_completion( - prompt, # Prompt - max_tokens=4096, # Generate up to 32 tokens - #stop = ["Q:", "\n"], # Stop generating just before the model would generate a new question - echo = False # Echo the prompt back in the output - ) - responseTextArea.value = output["choices"][0]["text"] - -def on_submitButton_clicked(b): - with output_widget: - clear_output(wait=True) - ragPromptTextArea.value = "" - responseTextArea.value = "" - log.debug(f"### selectRag: {selectRag.value}") - prompt = setPrompt(userTextArea.value,selectRag.value) - log.debug("### prompt: " + prompt) - runModel(prompt) - -submitButton.on_click(on_submitButton_clicked) -display(output_widget) - - -logger.info("#### Closing client db connection.") -client.close() - -logger.info("#### Program terminating.") diff --git a/semsearch_Hld02.py b/semsearch_Hld02.py deleted file mode 100644 index bdbea36840c3725e1d6f578db574dee1d2323077..0000000000000000000000000000000000000000 --- a/semsearch_Hld02.py +++ /dev/null @@ -1,402 +0,0 @@ -import weaviate - -from sentence_transformers import SentenceTransformer -from langchain_community.document_loaders import BSHTMLLoader -from pathlib import Path -from lxml import html -import logging -from semantic_text_splitter import HuggingFaceTextSplitter -from tokenizers import Tokenizer -import json -import os -import re -import logging - -import llama_cpp -from llama_cpp import Llama -import ipywidgets as widgets -import time -from IPython.display import display, clear_output - -weaviate_logger = logging.getLogger("httpx") -weaviate_logger.setLevel(logging.WARNING) - -logger = logging.getLogger(__name__) -logging.basicConfig(level=logging.INFO) - - - -################################################################# -# Connect to Weaviate vector database. -################################################################# -client = "" -def connectToWeaviateDB(): - ###################################################### - # Connect to the Weaviate vector database. - logger.info("#### Create Weaviate db client connection.") - client = weaviate.connect_to_custom( - http_host="127.0.0.1", - http_port=8080, - http_secure=False, - grpc_host="127.0.0.1", - grpc_port=50051, - grpc_secure=False - ) - client.connect() - - -####################################################### -# Read each text input file, parse it into a document, -# chunk it, collect chunks and document name. -####################################################### -webpageDocNames = [] -page_contentArray = [] -webpageTitles = [] -webpageChunks = [] -webpageChunksDocNames = [] - -def readParseChunkFiles(): - logger.info("#### Read and chunk input text files.") - for filename in os.listdir(pathString): - logger.info(filename) - path = Path(pathString + "/" + filename) - filename = filename.rstrip(".html") - webpageDocNames.append(filename) - htmlLoader = BSHTMLLoader(path,"utf-8") - htmlData = htmlLoader.load() - - title = htmlData[0].metadata['title'] - page_content = htmlData[0].page_content - - # Clean data. Remove multiple newlines, etc. - page_content = re.sub(r'\n+', '\n',page_content) - - page_contentArray.append(page_content); - webpageTitles.append (title) - max_tokens = 1000 - tokenizer = Tokenizer.from_pretrained("bert-base-uncased") - logger.debug(f"### tokenizer: {tokenizer}") - splitter = HuggingFaceTextSplitter(tokenizer, trim_chunks=True) - chunksOnePage = splitter.chunks(page_content, chunk_capacity=50) - - chunks = [] - for chnk in chunksOnePage: - logger.debug(f"#### chnk in file: {chnk}") - chunks.append(chnk) - logger.debug(f"chunks: {chunks}") - webpageChunks.append(chunks) - webpageChunksDocNames.append(filename + "Chunks") - - logger.debug(f"### filename, title: {filename}, {title}") - - logger.debug(f"### webpageDocNames: {webpageDocNames}") - -################################################################# -# Create the chunks collection for the Weaviate database. -################################################################# -def createChunksCollection(): - logger.info("#### createChunksCollection() entered.") - if client.collections.exists("Chunks"): - client.collections.delete("Chunks") - - class_obj = { - "class": "Chunks", - "description": "Collection for document chunks.", - "vectorizer": "text2vec-transformers", - "moduleConfig": { - "text2vec-transformers": { - "vectorizeClassName": True - } - }, - "vectorIndexType": "hnsw", - "vectorIndexConfig": { - "distance": "cosine", - }, - "properties": [ - { - "name": "chunk", - "dataType": ["text"], - "description": "Single webpage chunk.", - "vectorizer": "text2vec-transformers", - "moduleConfig": { - "text2vec-transformers": { - "vectorizePropertyName": False, - "skip": False, - "tokenization": "lowercase" - } - } - }, - { - "name": "chunk_index", - "dataType": ["int"] - }, - { - "name": "webpage", - "dataType": ["Documents"], - "description": "Webpage content chunks.", - - "invertedIndexConfig": { - "bm25": { - "b": 0.75, - "k1": 1.2 - } - } - } - ] - } - return(client.collections.create_from_dict(class_obj)) - - -##################################################################### -# Create the document collection for the Weaviate database. -##################################################################### -def createWebpageCollection(): - logger.info("#### createWebpageCollection() entered.") - if client.collections.exists("Documents"): - client.collections.delete("Documents") - - class_obj = { - "class": "Documents", - "description": "For first attempt at loading a Weviate database.", - "vectorizer": "text2vec-transformers", - "moduleConfig": { - "text2vec-transformers": { - "vectorizeClassName": False - } - }, - "vectorIndexType": "hnsw", - "vectorIndexConfig": { - "distance": "cosine", - }, - "properties": [ - { - "name": "title", - "dataType": ["text"], - "description": "HTML doc title.", - "vectorizer": "text2vec-transformers", - "moduleConfig": { - "text2vec-transformers": { - "vectorizePropertyName": True, - "skip": False, - "tokenization": "lowercase" - } - }, - "invertedIndexConfig": { - "bm25": { - "b": 0.75, - "k1": 1.2 - }, - } - }, - { - "name": "content", - "dataType": ["text"], - "description": "HTML page content.", - "moduleConfig": { - "text2vec-transformers": { - "vectorizePropertyName": True, - "tokenization": "whitespace" - } - } - } - ] - } - return(client.collections.create_from_dict(class_obj)) - - -################################################################# -# Create document and chunk objects in database. -################################################################# -def createDatabaseObjects(): - logger.info("#### Create page/doc and chunk db objects.") - for i, className in enumerate(webpageDocNames): - title = webpageTitles[i] - logger.debug(f"## className, title: {className}, {title}") - # Create Webpage Object - page_content = page_contentArray[i] - # Insert the document. - wpCollectionObj_uuid = wpCollection.data.insert( - { - "name": className, - "title": title, - "content": page_content - } - ) - - # Insert the chunks for the document. - for i2, chunk in enumerate(webpageChunks[i]): - chunk_uuid = wpChunkCollection.data.insert( - { - "title": title, - "chunk": chunk, - "chunk_index": i2, - "references": - { - "webpage": wpCollectionObj_uuid - } - } - ) - - -################################################################# -# Create display widgets. -################################################################# -output_widget = "" -systemTextArea = "" -userTextArea = "" -ragPromptTextArea = "" -responseTextArea = "" -selectRag = "" -submitButton = "" -def createWidgets(): - output_widget = widgets.Output() - with output_widget: - print("### Create widgets entered.") - - systemTextArea = widgets.Textarea( - value='', - placeholder='Enter System Prompt.', - description='Sys Prompt: ', - disabled=False, - layout=widgets.Layout(width='300px', height='80px') - ) - - userTextArea = widgets.Textarea( - value='', - placeholder='Enter User Prompt.', - description='User Prompt: ', - disabled=False, - layout=widgets.Layout(width='435px', height='110px') - ) - - ragPromptTextArea = widgets.Textarea( - value='', - placeholder='App generated prompt with RAG information.', - description='RAG Prompt: ', - disabled=False, - layout=widgets.Layout(width='580px', height='180px') - ) - - responseTextArea = widgets.Textarea( - value='', - placeholder='LLM generated response.', - description='LLM Resp: ', - disabled=False, - layout=widgets.Layout(width='780px', height='200px') - ) - - selectRag = widgets.Checkbox( - value=False, - description='Use RAG', - disabled=False - ) - - submitButton = widgets.Button( - description='Run Model.', - disabled=False, - button_style='', # 'success', 'info', 'warning', 'danger' or '' - tooltip='Click', - icon='check' # (FontAwesome names without the `fa-` prefix) - ) - - -###################################################################### -# MAINLINE -###################################################################### -logger.info("#### MAINLINE ENTERED.") - -#pathString = "/Users/660565/KPSAllInOne/ProgramFilesX86/WebCopy/DownloadedWebSites/LLMPOC_HTML" -pathString = "/app/inputDocs" -chunks = [] -webpageDocNames = [] -page_contentArray = [] -webpageChunks = [] -webpageTitles = [] -webpageChunksDocNames = [] - -#connectToWeaviateDB() -logger.info("#### Create Weaviate db client connection.") -client = weaviate.connect_to_custom( - http_host="127.0.0.1", - http_port=8080, - http_secure=False, - grpc_host="127.0.0.1", - grpc_port=50051, - grpc_secure=False -) -client.connect() - -readParseChunkFiles() -wpCollection = createWebpageCollection() -wpChunkCollection = createChunksCollection() - -#createDatabaseObjects() -logger.info("#### Create page/doc and chunk db objects.") - for i, className in enumerate(webpageDocNames): - title = webpageTitles[i] - logger.debug(f"## className, title: {className}, {title}") - # Create Webpage Object - page_content = page_contentArray[i] - # Insert the document. - wpCollectionObj_uuid = wpCollection.data.insert( - { - "name": className, - "title": title, - "content": page_content - } - ) - - # Insert the chunks for the document. - for i2, chunk in enumerate(webpageChunks[i]): - chunk_uuid = wpChunkCollection.data.insert( - { - "title": title, - "chunk": chunk, - "chunk_index": i2, - "references": - { - "webpage": wpCollectionObj_uuid - } - } - ) - -############################################################################### -# text contains prompt for vector DB. -text = "human-made computer cognitive ability" - - -############################################################################### -# Initial the the sentence transformer and encode the query prompt. -logger.info(f"#### Encode text query prompt to create vectors. {text}") -model = SentenceTransformer('/app/multi-qa-MiniLM-L6-cos-v1') - -vector = model.encode(text) -vectorList = [] - -logger.debug("#### Print vectors.") -for vec in vector: - vectorList.append(vec) -logger.debug(f"vectorList: {vectorList[2]}") - -# Fetch chunks and print chunks. -logger.info("#### Retrieve semchunks from db using vectors from prompt.") -semChunks = wpChunkCollection.query.near_vector( - near_vector=vectorList, - distance=0.7, - limit=3 -) -logger.debug(f"### semChunks[0]: {semChunks}") - -# Print chunks, corresponding document and document title. -logger.info("#### Print individual retrieved chunks.") -for chunk in enumerate(semChunks.objects): - logger.info(f"#### chunk: {chunk}") - webpage_uuid = chunk[1].properties['references']['webpage'] - logger.info(f"webpage_uuid: {webpage_uuid}") - wpFromChunk = wpCollection.query.fetch_object_by_id(webpage_uuid) - logger.info(f"### wpFromChunk title: {wpFromChunk.properties['title']}") - -logger.info("#### Closing client db connection.") -client.close() - -logger.info("#### Program terminating.") diff --git a/semsearch_Hld04_Working.py b/semsearch_Hld04_Working.py deleted file mode 100644 index ec1310f6349076f0ca22e3ce1414df7f9e855e6f..0000000000000000000000000000000000000000 --- a/semsearch_Hld04_Working.py +++ /dev/null @@ -1,271 +0,0 @@ -import weaviate - -from sentence_transformers import SentenceTransformer -from langchain_community.document_loaders import BSHTMLLoader -from pathlib import Path -from lxml import html -import logging -from semantic_text_splitter import HuggingFaceTextSplitter -from tokenizers import Tokenizer -import json -import os -import re -import logging - -weaviate_logger = logging.getLogger("httpx") -weaviate_logger.setLevel(logging.WARNING) - -logger = logging.getLogger(__name__) -logging.basicConfig(level=logging.INFO) - - - -###################################################################### -# MAINLINE -# -logger.info("#### MAINLINE ENTERED.") - -#pathString = "/Users/660565/KPSAllInOne/ProgramFilesX86/WebCopy/DownloadedWebSites/LLMPOC_HTML" -pathString = "/app/inputDocs" -chunks = [] -webpageDocNames = [] -page_contentArray = [] -webpageChunks = [] -webpageTitles = [] -webpageChunksDocNames = [] - - -####################################################### -# Read each text input file, parse it into a document, -# chunk it, collect chunks and document name. -logger.info("#### Read and chunk input text files.") -for filename in os.listdir(pathString): - logger.info(filename) - path = Path(pathString + "/" + filename) - filename = filename.rstrip(".html") - webpageDocNames.append(filename) - htmlLoader = BSHTMLLoader(path,"utf-8") - htmlData = htmlLoader.load() - - title = htmlData[0].metadata['title'] - page_content = htmlData[0].page_content - - # Clean data. Remove multiple newlines, etc. - page_content = re.sub(r'\n+', '\n',page_content) - - page_contentArray.append(page_content); - webpageTitles.append(title) - max_tokens = 1000 - tokenizer = Tokenizer.from_pretrained("bert-base-uncased") - logger.debug(f"### tokenizer: {tokenizer}") - splitter = HuggingFaceTextSplitter(tokenizer, trim_chunks=True) - chunksOnePage = splitter.chunks(page_content, chunk_capacity=50) - - chunks = [] - for chnk in chunksOnePage: - logger.debug(f"#### chnk in file: {chnk}") - chunks.append(chnk) - logger.debug(f"chunks: {chunks}") - webpageChunks.append(chunks) - webpageChunksDocNames.append(filename + "Chunks") - - logger.debug(f"### filename, title: {filename}, {title}") - -logger.debug(f"### webpageDocNames: {webpageDocNames}") - - -###################################################### -# Connect to the Weaviate vector database. -logger.info("#### Create Weaviate db client connection.") -client = weaviate.connect_to_custom( - http_host="127.0.0.1", - http_port=8080, - http_secure=False, - grpc_host="127.0.0.1", - grpc_port=50051, - grpc_secure=False - #read_timeout=600, - #write_timeout=90 -) -client.connect() - - -###################################################### -# Create database webpage and chunks collections. -#wpCollection = createWebpageCollection() -#wpChunkCollection = createChunksCollection() -logger.info("#### createWebpageCollection() entered.") -if client.collections.exists("Documents"): - client.collections.delete("Documents") - -class_obj = { - "class": "Documents", - "description": "For first attempt at loading a Weviate database.", - "vectorizer": "text2vec-transformers", - "moduleConfig": { - "text2vec-transformers": { - "vectorizeClassName": False - } - }, - "vectorIndexType": "hnsw", - "vectorIndexConfig": { - "distance": "cosine", - }, - "properties": [ - { - "name": "title", - "dataType": ["text"], - "description": "HTML doc title.", - "vectorizer": "text2vec-transformers", - "moduleConfig": { - "text2vec-transformers": { - "vectorizePropertyName": True, - "skip": False, - "tokenization": "lowercase" - } - }, - "invertedIndexConfig": { - "bm25": { - "b": 0.75, - "k1": 1.2 - }, - } - }, - { - "name": "content", - "dataType": ["text"], - "description": "HTML page content.", - "moduleConfig": { - "text2vec-transformers": { - "vectorizePropertyName": True, - "tokenization": "whitespace" - } - } - } - ] -} -wpCollection = client.collections.create_from_dict(class_obj) - -logger.info("#### createChunksCollection() entered.") -if client.collections.exists("Chunks"): - client.collections.delete("Chunks") - -class_obj = { - "class": "Chunks", - "description": "Collection for document chunks.", - "vectorizer": "text2vec-transformers", - "moduleConfig": { - "text2vec-transformers": { - "vectorizeClassName": True - } - }, - "vectorIndexType": "hnsw", - "vectorIndexConfig": { - "distance": "cosine", - }, - "properties": [ - { - "name": "chunk", - "dataType": ["text"], - "description": "Single webpage chunk.", - "vectorizer": "text2vec-transformers", - "moduleConfig": { - "text2vec-transformers": { - "vectorizePropertyName": False, - "skip": False, - "tokenization": "lowercase" - } - } - }, - { - "name": "chunk_index", - "dataType": ["int"] - }, - { - "name": "webpage", - "dataType": ["Documents"], - "description": "Webpage content chunks.", - - "invertedIndexConfig": { - "bm25": { - "b": 0.75, - "k1": 1.2 - } - } - } - ] -} -wpChunkCollection = client.collections.create_from_dict(class_obj) - - -########################################################### -# Create document and chunks objects in the database. -logger.info("#### Create page/doc and chunk db objects.") -for i, className in enumerate(webpageDocNames): - title = webpageTitles[i] - logger.debug(f"## className, title: {className}, {title}") - # Create Webpage Object - page_content = page_contentArray[i] - # Insert the document. - wpCollectionObj_uuid = wpCollection.data.insert( - { - "name": className, - "title": title, - "content": page_content - } - ) - - # Insert the chunks for the document. - for i2, chunk in enumerate(webpageChunks[i]): - chunk_uuid = wpChunkCollection.data.insert( - { - "title": title, - "chunk": chunk, - "chunk_index": i2, - "references": - { - "webpage": wpCollectionObj_uuid - } - } - ) - -############################################################################### -# text contains prompt for vector DB. -text = "human-made computer cognitive ability" - - -############################################################################### -# Initial the the sentence transformer and encode the query prompt. -logger.info(f"#### Encode text query prompt to create vectors. {text}") -model = SentenceTransformer('/app/multi-qa-MiniLM-L6-cos-v1') - -vector = model.encode(text) -vectorList = [] - -logger.debug("#### Print vectors.") -for vec in vector: - vectorList.append(vec) -logger.debug(f"vectorList: {vectorList[2]}") - -# Fetch chunks and print chunks. -logger.info("#### Retrieve semchunks from db using vectors from prompt.") -semChunks = wpChunkCollection.query.near_vector( - near_vector=vectorList, - distance=0.7, - limit=3 -) -logger.debug(f"### semChunks[0]: {semChunks}") - -# Print chunks, corresponding document and document title. -logger.info("#### Print individual retrieved chunks.") -for chunk in enumerate(semChunks.objects): - logger.info(f"#### chunk: {chunk}") - webpage_uuid = chunk[1].properties['references']['webpage'] - logger.info(f"webpage_uuid: {webpage_uuid}") - wpFromChunk = wpCollection.query.fetch_object_by_id(webpage_uuid) - logger.info(f"### wpFromChunk title: {wpFromChunk.properties['title']}") - -logger.info("#### Closing client db connection.") -client.close() - -logger.info("#### Program terminating.") diff --git a/semsearch_Hld05_Working.py b/semsearch_Hld05_Working.py deleted file mode 100644 index ec1310f6349076f0ca22e3ce1414df7f9e855e6f..0000000000000000000000000000000000000000 --- a/semsearch_Hld05_Working.py +++ /dev/null @@ -1,271 +0,0 @@ -import weaviate - -from sentence_transformers import SentenceTransformer -from langchain_community.document_loaders import BSHTMLLoader -from pathlib import Path -from lxml import html -import logging -from semantic_text_splitter import HuggingFaceTextSplitter -from tokenizers import Tokenizer -import json -import os -import re -import logging - -weaviate_logger = logging.getLogger("httpx") -weaviate_logger.setLevel(logging.WARNING) - -logger = logging.getLogger(__name__) -logging.basicConfig(level=logging.INFO) - - - -###################################################################### -# MAINLINE -# -logger.info("#### MAINLINE ENTERED.") - -#pathString = "/Users/660565/KPSAllInOne/ProgramFilesX86/WebCopy/DownloadedWebSites/LLMPOC_HTML" -pathString = "/app/inputDocs" -chunks = [] -webpageDocNames = [] -page_contentArray = [] -webpageChunks = [] -webpageTitles = [] -webpageChunksDocNames = [] - - -####################################################### -# Read each text input file, parse it into a document, -# chunk it, collect chunks and document name. -logger.info("#### Read and chunk input text files.") -for filename in os.listdir(pathString): - logger.info(filename) - path = Path(pathString + "/" + filename) - filename = filename.rstrip(".html") - webpageDocNames.append(filename) - htmlLoader = BSHTMLLoader(path,"utf-8") - htmlData = htmlLoader.load() - - title = htmlData[0].metadata['title'] - page_content = htmlData[0].page_content - - # Clean data. Remove multiple newlines, etc. - page_content = re.sub(r'\n+', '\n',page_content) - - page_contentArray.append(page_content); - webpageTitles.append(title) - max_tokens = 1000 - tokenizer = Tokenizer.from_pretrained("bert-base-uncased") - logger.debug(f"### tokenizer: {tokenizer}") - splitter = HuggingFaceTextSplitter(tokenizer, trim_chunks=True) - chunksOnePage = splitter.chunks(page_content, chunk_capacity=50) - - chunks = [] - for chnk in chunksOnePage: - logger.debug(f"#### chnk in file: {chnk}") - chunks.append(chnk) - logger.debug(f"chunks: {chunks}") - webpageChunks.append(chunks) - webpageChunksDocNames.append(filename + "Chunks") - - logger.debug(f"### filename, title: {filename}, {title}") - -logger.debug(f"### webpageDocNames: {webpageDocNames}") - - -###################################################### -# Connect to the Weaviate vector database. -logger.info("#### Create Weaviate db client connection.") -client = weaviate.connect_to_custom( - http_host="127.0.0.1", - http_port=8080, - http_secure=False, - grpc_host="127.0.0.1", - grpc_port=50051, - grpc_secure=False - #read_timeout=600, - #write_timeout=90 -) -client.connect() - - -###################################################### -# Create database webpage and chunks collections. -#wpCollection = createWebpageCollection() -#wpChunkCollection = createChunksCollection() -logger.info("#### createWebpageCollection() entered.") -if client.collections.exists("Documents"): - client.collections.delete("Documents") - -class_obj = { - "class": "Documents", - "description": "For first attempt at loading a Weviate database.", - "vectorizer": "text2vec-transformers", - "moduleConfig": { - "text2vec-transformers": { - "vectorizeClassName": False - } - }, - "vectorIndexType": "hnsw", - "vectorIndexConfig": { - "distance": "cosine", - }, - "properties": [ - { - "name": "title", - "dataType": ["text"], - "description": "HTML doc title.", - "vectorizer": "text2vec-transformers", - "moduleConfig": { - "text2vec-transformers": { - "vectorizePropertyName": True, - "skip": False, - "tokenization": "lowercase" - } - }, - "invertedIndexConfig": { - "bm25": { - "b": 0.75, - "k1": 1.2 - }, - } - }, - { - "name": "content", - "dataType": ["text"], - "description": "HTML page content.", - "moduleConfig": { - "text2vec-transformers": { - "vectorizePropertyName": True, - "tokenization": "whitespace" - } - } - } - ] -} -wpCollection = client.collections.create_from_dict(class_obj) - -logger.info("#### createChunksCollection() entered.") -if client.collections.exists("Chunks"): - client.collections.delete("Chunks") - -class_obj = { - "class": "Chunks", - "description": "Collection for document chunks.", - "vectorizer": "text2vec-transformers", - "moduleConfig": { - "text2vec-transformers": { - "vectorizeClassName": True - } - }, - "vectorIndexType": "hnsw", - "vectorIndexConfig": { - "distance": "cosine", - }, - "properties": [ - { - "name": "chunk", - "dataType": ["text"], - "description": "Single webpage chunk.", - "vectorizer": "text2vec-transformers", - "moduleConfig": { - "text2vec-transformers": { - "vectorizePropertyName": False, - "skip": False, - "tokenization": "lowercase" - } - } - }, - { - "name": "chunk_index", - "dataType": ["int"] - }, - { - "name": "webpage", - "dataType": ["Documents"], - "description": "Webpage content chunks.", - - "invertedIndexConfig": { - "bm25": { - "b": 0.75, - "k1": 1.2 - } - } - } - ] -} -wpChunkCollection = client.collections.create_from_dict(class_obj) - - -########################################################### -# Create document and chunks objects in the database. -logger.info("#### Create page/doc and chunk db objects.") -for i, className in enumerate(webpageDocNames): - title = webpageTitles[i] - logger.debug(f"## className, title: {className}, {title}") - # Create Webpage Object - page_content = page_contentArray[i] - # Insert the document. - wpCollectionObj_uuid = wpCollection.data.insert( - { - "name": className, - "title": title, - "content": page_content - } - ) - - # Insert the chunks for the document. - for i2, chunk in enumerate(webpageChunks[i]): - chunk_uuid = wpChunkCollection.data.insert( - { - "title": title, - "chunk": chunk, - "chunk_index": i2, - "references": - { - "webpage": wpCollectionObj_uuid - } - } - ) - -############################################################################### -# text contains prompt for vector DB. -text = "human-made computer cognitive ability" - - -############################################################################### -# Initial the the sentence transformer and encode the query prompt. -logger.info(f"#### Encode text query prompt to create vectors. {text}") -model = SentenceTransformer('/app/multi-qa-MiniLM-L6-cos-v1') - -vector = model.encode(text) -vectorList = [] - -logger.debug("#### Print vectors.") -for vec in vector: - vectorList.append(vec) -logger.debug(f"vectorList: {vectorList[2]}") - -# Fetch chunks and print chunks. -logger.info("#### Retrieve semchunks from db using vectors from prompt.") -semChunks = wpChunkCollection.query.near_vector( - near_vector=vectorList, - distance=0.7, - limit=3 -) -logger.debug(f"### semChunks[0]: {semChunks}") - -# Print chunks, corresponding document and document title. -logger.info("#### Print individual retrieved chunks.") -for chunk in enumerate(semChunks.objects): - logger.info(f"#### chunk: {chunk}") - webpage_uuid = chunk[1].properties['references']['webpage'] - logger.info(f"webpage_uuid: {webpage_uuid}") - wpFromChunk = wpCollection.query.fetch_object_by_id(webpage_uuid) - logger.info(f"### wpFromChunk title: {wpFromChunk.properties['title']}") - -logger.info("#### Closing client db connection.") -client.close() - -logger.info("#### Program terminating.") diff --git a/semsearch_Orig.py b/semsearch_Orig.py deleted file mode 100644 index b473ad155088618b20edad3d9ffe3e28bd68333d..0000000000000000000000000000000000000000 --- a/semsearch_Orig.py +++ /dev/null @@ -1,274 +0,0 @@ -import weaviate - -from sentence_transformers import SentenceTransformer -from langchain_community.document_loaders import BSHTMLLoader -from pathlib import Path -from lxml import html -import logging -from semantic_text_splitter import HuggingFaceTextSplitter -from tokenizers import Tokenizer -import json -import os -import re -import logging - -weaviate_logger = logging.getLogger("httpx") -weaviate_logger.setLevel(logging.WARNING) - -logger = logging.getLogger(__name__) -logging.basicConfig(level=logging.INFO) - - -################################################################# -# Create the chunks collection for the Weaviate database. -def createChunksCollection(): - logger.info("#### createChunksCollection() entered.") - if client.collections.exists("Chunks"): - client.collections.delete("Chunks") - - class_obj = { - "class": "Chunks", - "description": "Collection for document chunks.", - "vectorizer": "text2vec-transformers", - "moduleConfig": { - "text2vec-transformers": { - "vectorizeClassName": True - } - }, - "vectorIndexType": "hnsw", - "vectorIndexConfig": { - "distance": "cosine", - }, - "properties": [ - { - "name": "chunk", - "dataType": ["text"], - "description": "Single webpage chunk.", - "vectorizer": "text2vec-transformers", - "moduleConfig": { - "text2vec-transformers": { - "vectorizePropertyName": False, - "skip": False, - "tokenization": "lowercase" - } - } - }, - { - "name": "chunk_index", - "dataType": ["int"] - }, - { - "name": "webpage", - "dataType": ["Documents"], - "description": "Webpage content chunks.", - - "invertedIndexConfig": { - "bm25": { - "b": 0.75, - "k1": 1.2 - } - } - } - ] - } - return(client.collections.create_from_dict(class_obj)) - - -##################################################################### -# Create the document collection for the Weaviate database. -def createWebpageCollection(): - logger.info("#### createWebpageCollection() entered.") - if client.collections.exists("Documents"): - client.collections.delete("Documents") - - class_obj = { - "class": "Documents", - "description": "For first attempt at loading a Weviate database.", - "vectorizer": "text2vec-transformers", - "moduleConfig": { - "text2vec-transformers": { - "vectorizeClassName": False - } - }, - "vectorIndexType": "hnsw", - "vectorIndexConfig": { - "distance": "cosine", - }, - "properties": [ - { - "name": "title", - "dataType": ["text"], - "description": "HTML doc title.", - "vectorizer": "text2vec-transformers", - "moduleConfig": { - "text2vec-transformers": { - "vectorizePropertyName": True, - "skip": False, - "tokenization": "lowercase" - } - }, - "invertedIndexConfig": { - "bm25": { - "b": 0.75, - "k1": 1.2 - }, - } - }, - { - "name": "content", - "dataType": ["text"], - "description": "HTML page content.", - "moduleConfig": { - "text2vec-transformers": { - "vectorizePropertyName": True, - "tokenization": "whitespace" - } - } - } - ] - } - return(client.collections.create_from_dict(class_obj)) - - -###################################################################### -# MAINLINE -# -logger.info("#### MAINLINE ENTERED.") - -#pathString = "/Users/660565/KPSAllInOne/ProgramFilesX86/WebCopy/DownloadedWebSites/LLMPOC_HTML" -pathString = "/app/inputDocs" -chunks = [] -webpageDocNames = [] -page_contentArray = [] -webpageChunks = [] -webpageTitles = [] -webpageChunksDocNames = [] - - -###################################################### -# Connect to the Weaviate vector database. -logger.info("#### Create Weaviate db client connection.") -client = weaviate.connect_to_custom( - http_host="127.0.0.1", - http_port=8080, - http_secure=False, - grpc_host="127.0.0.1", - grpc_port=50051, - grpc_secure=False -) -client.connect() - -####################################################### -# Read each text input file, parse it into a document, -# chunk it, collect chunks and document name. -logger.info("#### Read and chunk input text files.") -for filename in os.listdir(pathString): - logger.info(filename) - path = Path(pathString + "/" + filename) - filename = filename.rstrip(".html") - webpageDocNames.append(filename) - htmlLoader = BSHTMLLoader(path,"utf-8") - htmlData = htmlLoader.load() - - title = htmlData[0].metadata['title'] - page_content = htmlData[0].page_content - - # Clean data. Remove multiple newlines, etc. - page_content = re.sub(r'\n+', '\n',page_content) - - page_contentArray.append(page_content); - webpageTitles.append(title) - max_tokens = 1000 - tokenizer = Tokenizer.from_pretrained("bert-base-uncased") - logger.debug(f"### tokenizer: {tokenizer}") - splitter = HuggingFaceTextSplitter(tokenizer, trim_chunks=True) - chunksOnePage = splitter.chunks(page_content, chunk_capacity=50) - - chunks = [] - for chnk in chunksOnePage: - logger.debug(f"#### chnk in file: {chnk}") - chunks.append(chnk) - logger.debug(f"chunks: {chunks}") - webpageChunks.append(chunks) - webpageChunksDocNames.append(filename + "Chunks") - - logger.debug(f"### filename, title: {filename}, {title}") - -logger.debug(f"### webpageDocNames: {webpageDocNames}") - -###################################################### -# Create database webpage and chunks collections. -wpCollection = createWebpageCollection() -wpChunkCollection = createChunksCollection() - -########################################################### -# Create document and chunks objects in the database. -logger.info("#### Create page/doc and chunk db objects.") -for i, className in enumerate(webpageDocNames): - title = webpageTitles[i] - logger.debug(f"## className, title: {className}, {title}") - # Create Webpage Object - page_content = page_contentArray[i] - # Insert the document. - wpCollectionObj_uuid = wpCollection.data.insert( - { - "name": className, - "title": title, - "content": page_content - } - ) - - # Insert the chunks for the document. - for i2, chunk in enumerate(webpageChunks[i]): - chunk_uuid = wpChunkCollection.data.insert( - { - "title": title, - "chunk": chunk, - "chunk_index": i2, - "references": - { - "webpage": wpCollectionObj_uuid - } - } - ) - -############################################################################### -# text contains prompt for vector DB. -text = "human-made computer cognitive ability" - - -############################################################################### -# Initial the the sentence transformer and encode the query prompt. -logger.info(f"#### Encode text query prompt to create vectors. {text}") -model = SentenceTransformer('/app/multi-qa-MiniLM-L6-cos-v1') - -vector = model.encode(text) -vectorList = [] - -logger.debug("#### Print vectors.") -for vec in vector: - vectorList.append(vec) -logger.debug(f"vectorList: {vectorList[2]}") - -# Fetch chunks and print chunks. -logger.info("#### Retrieve semchunks from db using vectors from prompt.") -semChunks = wpChunkCollection.query.near_vector( - near_vector=vectorList, - distance=0.7, - limit=3 -) -logger.debug(f"### semChunks[0]: {semChunks}") - -# Print chunks, corresponding document and document title. -logger.info("#### Print individual retrieved chunks.") -for chunk in enumerate(semChunks.objects): - logger.info(f"#### chunk: {chunk}") - webpage_uuid = chunk[1].properties['references']['webpage'] - logger.info(f"webpage_uuid: {webpage_uuid}") - wpFromChunk = wpCollection.query.fetch_object_by_id(webpage_uuid) - logger.info(f"### wpFromChunk title: {wpFromChunk.properties['title']}") - -logger.info("#### Closing client db connection.") -client.close() - -logger.info("#### Program terminating.") diff --git a/startup.sh b/startup.sh deleted file mode 100644 index 63c979047dad180ec145322ec0660c8a9e420068..0000000000000000000000000000000000000000 --- a/startup.sh +++ /dev/null @@ -1,65 +0,0 @@ -#! /bin/bash - -echo "#### startup.sh entered." -echo "### df -h"; df -h -#echo "### ls -l /app"; ls -l /app -#echo "### ls -l /app/weaviate"; ls -l /app/weaviate -#echo "### ls -l /app/text2vec-transformers"; ls -l /app/text2vec-transformers -#echo "### ls -l /data"; ls -l /data - -mkdir -p /data/var/lib/weaviate -chmod -R 777 /data/var/lib/weaviate -echo "### ls -al /data/var/lib/weaviate"; ls -al /data/var/lib/weaviate - -################################################ -# Start tex2vec-transformers -echo "#### Before /app/text2vec-transformers" -/app/text2vec-transformers/bin/uvicorn app:app --host 0.0.0.0 --port 8081 --log-level info --timeout-keep-alive 1440 2>& 1 | tee /data/var/lib/weaviate/t2v.log & - -#sleep 5 -#echo "\n######## curl t2 " -#for (( ; ; )) do curl localhost:8081/vectors -H 'Content-Type: application/json' -d '{"text": "foo bar"}'; sleep 61; done & - - -############################################### -# Start the weaviate vector database server. -echo "#### Before /app/weaviate" - -#echo "### pwd"; pwd -#echo "### ls -al ~"; ls -al ~ - -#echo "### ls -al ~"; ls -al ~ -#ln -s ~/var/lib/weaviate /var/lib/weaviate - -#echo "### ls -l /var/lib/weaviate"; ls -l /var/lib/weaviate -#echo "### ls -l /data"; ls -l /data -#echo "### ls -l /data/var/lib/weaviate"; ls -l /data/var/lib/weaviate - -export AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED=true \ - PERSISTENCE_DATA_PATH=/data/var/lib/weaviate \ - DEFAULT_VECTORIZER_MODULE=text2vec-transformers \ - ENABLE_MODULES=text2vec-transformers \ - TRANSFORMERS_INFERENCE_API=http://127.0.0.1:8081 \ - LOG_LEVEL=info \ - MODULES_CLIENT_TIMEOUT=600s -env -/app/weaviate/weaviate --host 127.0.0.1 --port 8080 --scheme http --write-timeout 600s 2>& 1 | tee /data/var/lib/weaviate/ws.log & - -echo "#### Before sleep." -sleep 60 - -echo "#### Before /app/semsearch.py" -python /app/semsearch.py 2>& 1 | tee /data/var/lib/weaviate/ss.log & - -# Display timestamps. -for (( ; ; )) do date; sleep 60; done & - -sleep 10 -echo "#############################" -df -h -ls -al /data/var/lib/weaviate -ls -al /data/var/lib/weaviate/* - -wait - - diff --git a/test/acceptance/actions/add_test.go b/test/acceptance/actions/add_test.go deleted file mode 100644 index acc8bbf22c0d0fad62c0e7cd7787a989e701683d..0000000000000000000000000000000000000000 --- a/test/acceptance/actions/add_test.go +++ /dev/null @@ -1,131 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "encoding/json" - "fmt" - "testing" - - "github.com/go-openapi/strfmt" - "github.com/weaviate/weaviate/client/objects" - "github.com/weaviate/weaviate/entities/models" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/test/helper" -) - -// executed in setup_test.go -func addingObjects(t *testing.T) { - class := "TestObject" - t.Run("can create object", func(t *testing.T) { - // Set all object values to compare - objectTestString := "Test string" - objectTestInt := 1 - objectTestBoolean := true - objectTestNumber := 1.337 - objectTestDate := "2017-10-06T08:15:30+01:00" - - params := objects.NewObjectsCreateParams().WithBody( - &models.Object{ - Class: class, - Properties: map[string]interface{}{ - "testString": objectTestString, - "testWholeNumber": objectTestInt, - "testTrueFalse": objectTestBoolean, - "testNumber": objectTestNumber, - "testDateTime": objectTestDate, - }, - }) - - resp, err := helper.Client(t).Objects.ObjectsCreate(params, nil) - - // Ensure that the response is OK - helper.AssertRequestOk(t, resp, err, func() { - object := resp.Payload - assert.Regexp(t, strfmt.UUIDPattern, object.ID) - - schema, ok := object.Properties.(map[string]interface{}) - if !ok { - t.Fatal("The returned schema is not an JSON object") - } - - testWholeNumber, _ := schema["testWholeNumber"].(json.Number).Int64() - testNumber, _ := schema["testNumber"].(json.Number).Float64() - - // Check whether the returned information is the same as the data added - assert.Equal(t, objectTestString, schema["testString"]) - assert.Equal(t, objectTestInt, int(testWholeNumber)) - assert.Equal(t, objectTestBoolean, schema["testTrueFalse"]) - assert.Equal(t, objectTestNumber, testNumber) - assert.Equal(t, objectTestDate, schema["testDateTime"]) - }) - }) - - t.Run("can create and get object", func(t *testing.T) { - objectTestString := "Test string" - objectTestInt := 1 - objectTestBoolean := true - objectTestNumber := 1.337 - objectTestDate := "2017-10-06T08:15:30+01:00" - - objectID := helper.AssertCreateObject(t, class, map[string]interface{}{ - "testString": objectTestString, - "testWholeNumber": objectTestInt, - "testTrueFalse": objectTestBoolean, - "testNumber": objectTestNumber, - "testDateTime": objectTestDate, - }) - helper.AssertGetObjectEventually(t, class, objectID) - - // Now fetch the object - getResp, err := helper.Client(t).Objects.ObjectsGet(objects.NewObjectsGetParams().WithID(objectID), nil) - - helper.AssertRequestOk(t, getResp, err, func() { - object := getResp.Payload - - schema, ok := object.Properties.(map[string]interface{}) - if !ok { - t.Fatal("The returned schema is not an JSON object") - } - - testWholeNumber, _ := schema["testWholeNumber"].(json.Number).Int64() - testNumber, _ := schema["testNumber"].(json.Number).Float64() - - // Check whether the returned information is the same as the data added - assert.Equal(t, objectTestString, schema["testString"]) - assert.Equal(t, objectTestInt, int(testWholeNumber)) - assert.Equal(t, objectTestBoolean, schema["testTrueFalse"]) - assert.Equal(t, objectTestNumber, testNumber) - assert.Equal(t, objectTestDate, schema["testDateTime"]) - }) - }) - - t.Run("can add single ref", func(t *testing.T) { - firstID := helper.AssertCreateObject(t, class, map[string]interface{}{}) - helper.AssertGetObjectEventually(t, class, firstID) - - secondID := helper.AssertCreateObject(t, "TestObjectTwo", map[string]interface{}{ - "testString": "stringy", - "testReference": []interface{}{ - map[string]interface{}{ - "beacon": fmt.Sprintf("weaviate://localhost/%s", firstID), - }, - }, - }) - - secondObject := helper.AssertGetObjectEventually(t, "TestObjectTwo", secondID) - - singleRef := secondObject.Properties.(map[string]interface{})["testReference"].([]interface{})[0].(map[string]interface{}) - assert.Equal(t, singleRef["beacon"].(string), fmt.Sprintf("weaviate://localhost/TestObject/%s", firstID)) - }) -} diff --git a/test/acceptance/actions/delete_test.go b/test/acceptance/actions/delete_test.go deleted file mode 100644 index befcc49dc9ad03946d96a00f4b2101458c1b6763..0000000000000000000000000000000000000000 --- a/test/acceptance/actions/delete_test.go +++ /dev/null @@ -1,38 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -// Acceptance tests for objects - -import ( - "testing" - - "github.com/weaviate/weaviate/client/objects" - "github.com/weaviate/weaviate/test/helper" -) - -func removingObjects(t *testing.T) { - objectId := helper.AssertCreateObject(t, "TestObject", map[string]interface{}{}) - - // Yes, it's created - _ = helper.AssertGetObjectEventually(t, "TestObject", objectId) - - // Now perorm the the deletion - delResp, err := helper.Client(t).Objects.ObjectsDelete(objects.NewObjectsDeleteParams().WithID(objectId), nil) - helper.AssertRequestOk(t, delResp, err, nil) - - _ = helper.AssertGetObjectFailsEventually(t, "TestObject", objectId) - - // And verify that the object is gone - getResp, err := helper.Client(t).Objects.ObjectsGet(objects.NewObjectsGetParams().WithID(objectId), nil) - helper.AssertRequestFail(t, getResp, err, nil) -} diff --git a/test/acceptance/actions/individual_refs_test.go b/test/acceptance/actions/individual_refs_test.go deleted file mode 100644 index daeb47ef641e362b10bd9b681f062bc0c2c07739..0000000000000000000000000000000000000000 --- a/test/acceptance/actions/individual_refs_test.go +++ /dev/null @@ -1,167 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -// Acceptance tests for objects - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/client/objects" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema/crossref" - "github.com/weaviate/weaviate/test/helper" -) - -// run from setup_test.go -func objectReferences(t *testing.T) { - var ( - class1 = "TestObject" - class2 = "TestObjectTwo" - ) - t.Run("can add reference individually", func(t *testing.T) { - t.Parallel() - - toPointToUuid := helper.AssertCreateObject(t, "TestObject", map[string]interface{}{}) - helper.AssertGetObjectEventually(t, class1, toPointToUuid) - - uuid := helper.AssertCreateObject(t, class2, map[string]interface{}{}) - - // Verify that testReferences is empty - updatedObject := helper.AssertGetObjectEventually(t, class2, uuid) - updatedSchema := updatedObject.Properties.(map[string]interface{}) - assert.Nil(t, updatedSchema["testReferences"]) - - // Append a property reference - params := objects.NewObjectsReferencesCreateParams(). - WithID(uuid). - WithPropertyName("testReferences"). - WithBody(crossref.NewLocalhost(class1, toPointToUuid).SingleRef()) - - updateResp, err := helper.Client(t).Objects.ObjectsReferencesCreate(params, nil) - helper.AssertRequestOk(t, updateResp, err, nil) - - checkThunk := func() interface{} { - resp, err := helper.Client(t).Objects.ObjectsGet(objects.NewObjectsGetParams().WithID(uuid), nil) - if err != nil { - t.Log(err) - return false - } - - updatedSchema = resp.Payload.Properties.(map[string]interface{}) - return updatedSchema["testReferences"] != nil - } - - helper.AssertEventuallyEqual(t, true, checkThunk) - }) - - t.Run("can replace all properties", func(t *testing.T) { - t.Parallel() - - toPointToUuidFirst := helper.AssertCreateObject(t, "TestObject", map[string]interface{}{}) - toPointToUuidLater := helper.AssertCreateObject(t, "TestObject", map[string]interface{}{}) - helper.AssertGetObjectEventually(t, "TestObject", toPointToUuidFirst) - helper.AssertGetObjectEventually(t, "TestObject", toPointToUuidLater) - - uuid := helper.AssertCreateObject(t, "TestObjectTwo", map[string]interface{}{ - "testReferences": models.MultipleRef{ - crossref.NewLocalhost("TestObject", toPointToUuidFirst).SingleRef(), - }, - }) - - // Verify that testReferences is empty - updatedObject := helper.AssertGetObjectEventually(t, "TestObjectTwo", uuid) - updatedSchema := updatedObject.Properties.(map[string]interface{}) - assert.NotNil(t, updatedSchema["testReferences"]) - - // Replace - params := objects.NewObjectsReferencesUpdateParams(). - WithID(uuid). - WithPropertyName("testReferences"). - WithBody(models.MultipleRef{ - crossref.NewLocalhost("TestObject", toPointToUuidLater).SingleRef(), - }) - - updateResp, err := helper.Client(t).Objects.ObjectsReferencesUpdate(params, nil) - helper.AssertRequestOk(t, updateResp, err, nil) - - checkThunk := func() interface{} { - resp, err := helper.Client(t).Objects.ObjectsGet(objects.NewObjectsGetParams().WithID(uuid), nil) - if err != nil { - t.Log(err) - return false - } - - updatedSchema = resp.Payload.Properties.(map[string]interface{}) - return updatedSchema["testReferences"] != nil - } - - helper.AssertEventuallyEqual(t, true, checkThunk) - }) - - t.Run("remove property individually", func(t *testing.T) { - t.Parallel() - - toPointToUuid := helper.AssertCreateObject(t, "TestObject", map[string]interface{}{}) - helper.AssertGetObjectEventually(t, "TestObject", toPointToUuid) - - uuid := helper.AssertCreateObject(t, "TestObjectTwo", map[string]interface{}{ - "testReferences": models.MultipleRef{ - crossref.NewLocalhost("TestObject", toPointToUuid).SingleRef(), - }, - }) - - // Verify that testReferences is not empty - updatedObject := helper.AssertGetObjectEventually(t, "TestObjectTwo", uuid) - updatedSchema := updatedObject.Properties.(map[string]interface{}) - assert.NotNil(t, updatedSchema["testReferences"]) - - // Delete a property reference - params := objects.NewObjectsReferencesDeleteParams(). - WithID(uuid). - WithPropertyName("testReferences"). - WithBody( - crossref.NewLocalhost("TestObject", toPointToUuid).SingleRef(), - ) - - updateResp, err := helper.Client(t).Objects.ObjectsReferencesDelete(params, nil) - helper.AssertRequestOk(t, updateResp, err, nil) - - checkThunk := func() interface{} { - resp, err := helper.Client(t).Objects.ObjectsGet(objects.NewObjectsGetParams().WithID(uuid), nil) - if err != nil { - t.Log(err) - return false - } - - refs := resp.Payload.Properties.(map[string]interface{})["testReferences"] - - if refs == nil { - return true - } - - refsSlice, ok := refs.([]interface{}) - if ok { - return len(refsSlice) == 0 - } - - // neither nil, nor a list - t.Logf("prop %s was neither nil nor a list after deleting, instead we got %#v", "testReferences", refs) - t.Fail() - - return false - } - - helper.AssertEventuallyEqual(t, true, checkThunk) - }) -} diff --git a/test/acceptance/actions/network_refs_test.go b/test/acceptance/actions/network_refs_test.go deleted file mode 100644 index 12146d275b26b3cc148ec7a2074e4b0bab9a3554..0000000000000000000000000000000000000000 --- a/test/acceptance/actions/network_refs_test.go +++ /dev/null @@ -1,90 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -// func TestCanAddSingleNetworkRef(t *testing.T) { -// networkRefID := "711da979-4b0b-41e2-bcb8-fcc03554c7c8" -// actionID := assertCreateAction(t, "TestAction", map[string]interface{}{ -// "testReference": []interface{}{ -// map[string]interface{}{ -// "beacon": strfmt.UUID(fmt.Sprintf("weaviate://RemoteWeaviateForAcceptanceTest/things/%s", networkRefID)), -// }, -// }, -// }) -// assertGetActionEventually(t, actionID) - -// t.Run("it can query the resource again to verify the cross ref was added", func(t *testing.T) { -// action := assertGetAction(t, actionID) -// list := action.Schema.(map[string]interface{})["testReference"] -// require.NotNil(t, list, "cross-ref is present") -// rawCref := list.([]interface{})[0] -// cref := rawCref.(map[string]interface{}) -// assert.Equal(t, -// fmt.Sprintf("weaviate://RemoteWeaviateForAcceptanceTest/things/%s", networkRefID), cref["beacon"]) -// }) - -// t.Run("an implicit schema update has happened, we now include the network ref's class", func(t *testing.T) { -// schema := assertGetSchema(t) -// require.NotNil(t, schema.Actions) -// class := assertClassInSchema(t, schema.Actions, "TestAction") -// prop := assertPropertyInClass(t, class, "testReference") -// expectedDataType := []string{"TestThing", "RemoteWeaviateForAcceptanceTest/Instruments"} -// assert.Equal(t, expectedDataType, prop.DataType, "prop should have old and newly added dataTypes") -// }) -// } - -// func TestCanPatchSingleNetworkRef(t *testing.T) { -// t.Parallel() - -// actionID := assertCreateAction(t, "TestAction", nil) -// assertGetActionEventually(t, actionID) -// networkRefID := "711da979-4b0b-41e2-bcb8-fcc03554c7c8" - -// op := "add" -// path := "/schema/testReference" - -// patch := &models.PatchDocument{ -// Op: &op, -// Path: &path, -// Value: []interface{}{ -// map[string]interface{}{ -// "beacon": strfmt.UUID(fmt.Sprintf("weaviate://RemoteWeaviateForAcceptanceTest/things/%s", networkRefID)), -// }, -// }, -// } - -// t.Run("it can apply the patch", func(t *testing.T) { -// params := actions.NewActionsPatchParams(). -// WithBody([]*models.PatchDocument{patch}). -// WithID(actionID) -// patchResp, err := helper.Client(t).Actions.ActionsPatch(params, nil) -// helper.AssertRequestOk(t, patchResp, err, nil) -// }) - -// t.Run("it can query the resource again to verify the cross ref was added", func(t *testing.T) { -// patchedAction := assertGetAction(t, actionID) -// list := patchedAction.Schema.(map[string]interface{})["testReference"] -// require.NotNil(t, list, "cross-ref is present") -// rawCref := list.([]interface{})[0] -// cref := rawCref.(map[string]interface{}) -// assert.Equal(t, fmt.Sprintf("weaviate://RemoteWeaviateForAcceptanceTest/things/%s", networkRefID), cref["beacon"]) -// }) - -// t.Run("an implicit schema update has happened, we now include the network ref's class", func(t *testing.T) { -// schema := assertGetSchema(t) -// require.NotNil(t, schema.Actions) -// class := assertClassInSchema(t, schema.Actions, "TestAction") -// prop := assertPropertyInClass(t, class, "testReference") -// expectedDataType := []string{"TestThing", "RemoteWeaviateForAcceptanceTest/Instruments"} -// assert.Equal(t, expectedDataType, prop.DataType, "prop should have old and newly added dataTypes") -// }) -// } diff --git a/test/acceptance/actions/object_test.go b/test/acceptance/actions/object_test.go deleted file mode 100644 index 5c87f0747ca5b203c1a178338663b5c343ce967c..0000000000000000000000000000000000000000 --- a/test/acceptance/actions/object_test.go +++ /dev/null @@ -1,773 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "encoding/json" - "errors" - "fmt" - "testing" - "time" - - "github.com/go-openapi/strfmt" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/client/batch" - "github.com/weaviate/weaviate/client/objects" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/schema/crossref" - "github.com/weaviate/weaviate/test/helper" -) - -func TestFindObject(t *testing.T) { - t.Parallel() - var ( - cls = "TestObjectHTTPGet" - first_friend = "TestObjectHTTPGetFriendFirst" - second_friend = "TestObjectHTTPGetFriendSecond" - ) - - // test setup - first_uuid := helper.AssertCreateObject(t, first_friend, map[string]interface{}{}) - defer helper.DeleteClassObject(t, first_friend) - second_uuid := helper.AssertCreateObject(t, second_friend, map[string]interface{}{}) - defer helper.DeleteClassObject(t, second_friend) - - helper.AssertCreateObjectClass(t, &models.Class{ - Class: cls, - Vectorizer: "none", - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "friend", - DataType: []string{first_friend, second_friend}, - }, - }, - }) - // tear down - defer helper.DeleteClassObject(t, cls) - link1 := map[string]interface{}{ - "beacon": crossref.NewLocalhost(first_friend, first_uuid).String(), - "href": fmt.Sprintf("/v1/objects/%s/%s", first_friend, first_uuid), - } - link2 := map[string]interface{}{ - "beacon": crossref.NewLocalhost(second_friend, second_uuid).String(), - "href": fmt.Sprintf("/v1/objects/%s/%s", second_friend, second_uuid), - } - expected := map[string]interface{}{ - "number": json.Number("2"), - "friend": []interface{}{link1, link2}, - } - - uuid := helper.AssertCreateObject(t, cls, expected) - - r := objects.NewObjectsClassGetParams().WithID(uuid).WithClassName(cls) - resp, err := helper.Client(t).Objects.ObjectsClassGet(r, nil) - helper.AssertRequestOk(t, resp, err, nil) - assert.Equal(t, expected, resp.Payload.Properties.(map[string]interface{})) - - // check for an object which doesn't exist - unknown_uuid := strfmt.UUID("11110000-0000-0000-0000-000011110000") - r = objects.NewObjectsClassGetParams().WithID(unknown_uuid).WithClassName(cls) - resp, err = helper.Client(t).Objects.ObjectsClassGet(r, nil) - helper.AssertRequestFail(t, resp, err, nil) -} - -func TestHeadObject(t *testing.T) { - t.Parallel() - cls := "TestObjectHTTPHead" - // test setup - helper.AssertCreateObjectClass(t, &models.Class{ - Class: cls, - Vectorizer: "none", - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - }) - // tear down - defer helper.DeleteClassObject(t, cls) - - uuid := helper.AssertCreateObject(t, cls, map[string]interface{}{ - "name": "John", - }) - - r := objects.NewObjectsClassHeadParams().WithID(uuid).WithClassName(cls) - resp, err := helper.Client(t).Objects.ObjectsClassHead(r, nil) - helper.AssertRequestOk(t, resp, err, nil) - - // check for an object which doesn't exist - unknown_uuid := strfmt.UUID("11110000-0000-0000-0000-000011110000") - r = objects.NewObjectsClassHeadParams().WithID(unknown_uuid).WithClassName(cls) - resp, err = helper.Client(t).Objects.ObjectsClassHead(r, nil) - helper.AssertRequestFail(t, resp, err, nil) -} - -func TestPutObject(t *testing.T) { - t.Parallel() - var ( - cls = "TestObjectHTTPUpdate" - friend_cls = "TestObjectHTTPUpdateFriend" - ) - - // test setup - friend_uuid := helper.AssertCreateObject(t, friend_cls, map[string]interface{}{}) - defer helper.DeleteClassObject(t, friend_cls) - - helper.AssertCreateObjectClass(t, &models.Class{ - Class: cls, - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "vectorizeClassName": true, - }, - }, - Properties: []*models.Property{ - { - Name: "testString", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "testWholeNumber", - DataType: []string{"int"}, - }, - { - Name: "testNumber", - DataType: []string{"number"}, - }, - { - Name: "testDateTime", - DataType: []string{"date"}, - }, - { - Name: "testTrueFalse", - DataType: []string{"boolean"}, - }, - { - Name: "friend", - DataType: []string{friend_cls}, - }, - }, - }) - // tear down - defer helper.DeleteClassObject(t, cls) - uuid := helper.AssertCreateObject(t, cls, map[string]interface{}{ - "testWholeNumber": 2.0, - "testDateTime": time.Now(), - "testString": "wibbly", - }) - - link1 := map[string]interface{}{ - "beacon": crossref.NewLocalhost(friend_cls, friend_uuid).String(), - "href": fmt.Sprintf("/v1/objects/%s/%s", friend_cls, friend_uuid), - } - link2 := map[string]interface{}{ - "beacon": crossref.NewLocalhost(friend_cls, friend_uuid).String(), - "href": fmt.Sprintf("/v1/objects/%s/%s", friend_cls, friend_uuid), - } - expected := map[string]interface{}{ - "testNumber": json.Number("2"), - "testTrueFalse": true, - "testString": "wibbly wobbly", - "friend": []interface{}{link1, link2}, - } - update := models.Object{ - Class: cls, - Properties: models.PropertySchema(expected), - ID: uuid, - } - params := objects.NewObjectsClassPutParams().WithID(uuid).WithBody(&update) - updateResp, err := helper.Client(t).Objects.ObjectsClassPut(params, nil) - helper.AssertRequestOk(t, updateResp, err, nil) - actual := helper.AssertGetObject(t, cls, uuid).Properties.(map[string]interface{}) - assert.Equal(t, expected, actual) -} - -func TestPatchObject(t *testing.T) { - t.Parallel() - var ( - cls = "TestObjectHTTPPatch" - friend_cls = "TestObjectHTTPPatchFriend" - mconfig = map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "vectorizeClassName": true, - }, - } - ) - // test setup - helper.AssertCreateObjectClass(t, &models.Class{ // friend - Class: friend_cls, - ModuleConfig: mconfig, - Properties: []*models.Property{}, - }) - defer helper.DeleteClassObject(t, friend_cls) - helper.AssertCreateObjectClass(t, &models.Class{ // class - Class: cls, - ModuleConfig: mconfig, - Properties: []*models.Property{ - { - Name: "string1", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "integer1", - DataType: []string{"int"}, - }, - { - Name: "number1", - DataType: []string{"number"}, - }, - { - Name: "friend", - DataType: []string{friend_cls}, - }, - { - Name: "boolean1", - DataType: []string{"boolean"}, - }, - }, - }) - defer helper.DeleteClassObject(t, cls) - - uuid := helper.AssertCreateObject(t, cls, map[string]interface{}{ - "integer1": 2.0, - "string1": "wibbly", - }) - friendID := helper.AssertCreateObject(t, friend_cls, nil) - link1 := map[string]interface{}{ - "beacon": fmt.Sprintf("weaviate://localhost/%s/%s", friend_cls, friendID), - "href": fmt.Sprintf("/v1/objects/%s/%s", friend_cls, friendID), - } - link2 := map[string]interface{}{ - "beacon": fmt.Sprintf("weaviate://localhost/%s/%s", friend_cls, friendID), - "href": fmt.Sprintf("/v1/objects/%s/%s", friend_cls, friendID), - } - expected := map[string]interface{}{ - "integer1": json.Number("2"), - "number1": json.Number("3"), - "boolean1": true, - "string1": "wibbly wobbly", - "friend": []interface{}{link1, link2}, - } - update := map[string]interface{}{ - "number1": 3.0, - "boolean1": true, - "string1": "wibbly wobbly", - "friend": []interface{}{ - map[string]interface{}{ - "beacon": link1["beacon"], - }, map[string]interface{}{ - "beacon": link2["beacon"], - }, - }, - } - updateObj := models.Object{ - Properties: models.PropertySchema(update), - } - params := objects.NewObjectsClassPatchParams().WithClassName(cls) - params.WithID(uuid).WithBody(&updateObj) - updateResp, err := helper.Client(t).Objects.ObjectsClassPatch(params, nil) - helper.AssertRequestOk(t, updateResp, err, nil) - actual := func() interface{} { - obj := helper.AssertGetObject(t, cls, uuid) - props := obj.Properties.(map[string]interface{}) - return props - } - helper.AssertEventuallyEqual(t, expected, actual) - - params.WithID(strfmt.UUID("e5be1f32-0001-0000-0000-ebb25dfc811f")) - _, err = helper.Client(t).Objects.ObjectsClassPatch(params, nil) - if err == nil { - t.Errorf("must return an error for non existing object") - } -} - -func TestDeleteObject(t *testing.T) { - t.Parallel() - var ( - id = strfmt.UUID("21111111-1111-1111-1111-111111111111") - classA = "TestObjectHTTPDeleteA" - classB = "TestObjectHTTPDeleteB" - props = []*models.Property{ - { - Name: "text", - DataType: []string{"text"}, - }, - } - ) - // test setup - helper.AssertCreateObjectClass(t, &models.Class{ - Class: classA, - Vectorizer: "none", - Properties: props, - }) - defer helper.DeleteClassObject(t, classA) - - helper.AssertCreateObjectClass(t, &models.Class{ - Class: classB, - Vectorizer: "none", - Properties: props, - }) - - defer helper.DeleteClassObject(t, classB) - - object1 := &models.Object{ - Class: classA, - ID: id, - Properties: map[string]interface{}{ - "text": "string 1", - }, - } - object2 := &models.Object{ - Class: classB, - ID: id, - Properties: map[string]interface{}{ - "text": "string 2", - }, - } - - // create objects - returnedFields := "ALL" - params := batch.NewBatchObjectsCreateParams().WithBody( - batch.BatchObjectsCreateBody{ - Objects: []*models.Object{object1, object2}, - Fields: []*string{&returnedFields}, - }) - - resp, err := helper.BatchClient(t).BatchObjectsCreate(params, nil) - - // ensure that the response is OK - helper.AssertRequestOk(t, resp, err, func() { - objectsCreateResponse := resp.Payload - - // check if the batch response contains two batched responses - assert.Equal(t, 2, len(objectsCreateResponse)) - - for _, elem := range resp.Payload { - assert.Nil(t, elem.Result.Errors) - } - }) - - { // "delete object from first class - params := objects.NewObjectsClassDeleteParams().WithClassName(classA).WithID(id) - resp, err := helper.Client(t).Objects.ObjectsClassDelete(params, nil) - if err != nil { - t.Errorf("cannot delete existing object err: %v", err) - } - assert.Equal(t, &objects.ObjectsClassDeleteNoContent{}, resp) - } - { // check if object still exit - params := objects.NewObjectsClassGetParams().WithClassName(classA).WithID(id) - _, err := helper.Client(t).Objects.ObjectsClassGet(params, nil) - werr := &objects.ObjectsClassGetNotFound{} - if !errors.As(err, &werr) { - t.Errorf("Get deleted object error got: %v want %v", err, werr) - } - } - { // object with a different class must exist - params := objects.NewObjectsClassGetParams().WithClassName(classB).WithID(id) - resp, err := helper.Client(t).Objects.ObjectsClassGet(params, nil) - if err != nil { - t.Errorf("object must exist err: %v", err) - } - if resp.Payload == nil { - t.Errorf("payload of an existing object cannot be empty") - } - } - - { // "delete object again from first class - params := objects.NewObjectsClassDeleteParams().WithClassName(classA).WithID(id) - _, err := helper.Client(t).Objects.ObjectsClassDelete(params, nil) - werr := &objects.ObjectsClassDeleteNotFound{} - if !errors.As(err, &werr) { - t.Errorf("Get deleted object error got: %v want %v", err, werr) - } - } -} - -func TestPostReference(t *testing.T) { - t.Parallel() - var ( - cls = "TestObjectHTTPAddReference" - friend_cls = "TestObjectHTTPAddReferenceFriend" - mconfig = map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "vectorizeClassName": true, - }, - } - ) - - // test setup - helper.AssertCreateObjectClass(t, &models.Class{ - Class: friend_cls, - ModuleConfig: mconfig, - Properties: []*models.Property{}, - }) - defer helper.DeleteClassObject(t, friend_cls) - helper.AssertCreateObjectClass(t, &models.Class{ - Class: cls, - ModuleConfig: mconfig, - Properties: []*models.Property{ - { - Name: "number", - DataType: []string{"number"}, - }, - { - Name: "friend", - DataType: []string{friend_cls}, - }, - }, - }) - defer helper.DeleteClassObject(t, cls) - uuid := helper.AssertCreateObject(t, cls, map[string]interface{}{ - "number": 2.0, - }) - friendID := helper.AssertCreateObject(t, friend_cls, nil) - expected := map[string]interface{}{ - "number": json.Number("2"), - "friend": []interface{}{ - map[string]interface{}{ - "beacon": fmt.Sprintf("weaviate://localhost/%s/%s", friend_cls, friendID), - "href": fmt.Sprintf("/v1/objects/%s/%s", friend_cls, friendID), - }, - }, - } - updateObj := crossref.NewLocalhost(friend_cls, friendID).SingleRef() - params := objects.NewObjectsClassReferencesCreateParams().WithClassName(cls) - params.WithID(uuid).WithBody(updateObj).WithPropertyName("friend") - resp, err := helper.Client(t).Objects.ObjectsClassReferencesCreate(params, nil) - helper.AssertRequestOk(t, resp, err, nil) - obj := helper.AssertGetObject(t, cls, uuid) - actual := obj.Properties.(map[string]interface{}) - assert.Equal(t, expected, actual) - - params.WithPropertyName("unknown") - _, err = helper.Client(t).Objects.ObjectsClassReferencesCreate(params, nil) - if _, ok := err.(*objects.ObjectsClassReferencesCreateUnprocessableEntity); !ok { - t.Errorf("error type expected: %T, got %T", objects.ObjectsClassReferencesCreateUnprocessableEntity{}, err) - } - - params.WithPropertyName("friend") - params.WithID("e7cd261a-0000-0000-0000-d7b8e7b5c9ea") - _, err = helper.Client(t).Objects.ObjectsClassReferencesCreate(params, nil) - if _, ok := err.(*objects.ObjectsClassReferencesCreateNotFound); !ok { - t.Errorf("error type expected: %T, got %T", objects.ObjectsClassReferencesCreateNotFound{}, err) - } -} - -func TestPutReferences(t *testing.T) { - t.Parallel() - var ( - cls = "TestObjectHTTPUpdateReferences" - first_friend = "TestObjectHTTPUpdateReferencesFriendFirst" - second_friend = "TestObjectHTTPUpdateReferencesFriendSecond" - mconfig = map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "vectorizeClassName": true, - }, - } - ) - // test setup - helper.AssertCreateObjectClass(t, &models.Class{ - Class: first_friend, - ModuleConfig: mconfig, - Properties: []*models.Property{}, - }) - defer helper.DeleteClassObject(t, first_friend) - - helper.AssertCreateObjectClass(t, &models.Class{ - Class: second_friend, - ModuleConfig: mconfig, - Properties: []*models.Property{}, - }) - defer helper.DeleteClassObject(t, second_friend) - - helper.AssertCreateObjectClass(t, &models.Class{ - Class: cls, - ModuleConfig: mconfig, - Properties: []*models.Property{ - { - Name: "number", - DataType: []string{"number"}, - }, - { - Name: "friend", - DataType: []string{first_friend, second_friend}, - }, - }, - }) - defer helper.DeleteClassObject(t, cls) - - uuid := helper.AssertCreateObject(t, cls, map[string]interface{}{ - "number": 2.0, - }) - first_friendID := helper.AssertCreateObject(t, first_friend, nil) - second_friendID := helper.AssertCreateObject(t, second_friend, nil) - - expected := map[string]interface{}{ - "number": json.Number("2"), - "friend": []interface{}{ - map[string]interface{}{ - "beacon": fmt.Sprintf("weaviate://localhost/%s/%s", first_friend, first_friendID), - "href": fmt.Sprintf("/v1/objects/%s/%s", first_friend, first_friendID), - }, - map[string]interface{}{ - "beacon": fmt.Sprintf("weaviate://localhost/%s/%s", second_friend, second_friendID), - "href": fmt.Sprintf("/v1/objects/%s/%s", second_friend, second_friendID), - }, - }, - } - updateObj := models.MultipleRef{ - crossref.NewLocalhost(first_friend, first_friendID).SingleRef(), - crossref.NewLocalhost(second_friend, second_friendID).SingleRef(), - } - // add two references - params := objects.NewObjectsClassReferencesPutParams().WithClassName(cls) - params.WithID(uuid).WithBody(updateObj).WithPropertyName("friend") - resp, err := helper.Client(t).Objects.ObjectsClassReferencesPut(params, nil) - helper.AssertRequestOk(t, resp, err, nil) - obj := helper.AssertGetObject(t, cls, uuid) - actual := obj.Properties.(map[string]interface{}) - assert.Equal(t, expected, actual) - - // exclude one reference - params.WithID(uuid).WithBody(updateObj[:1]).WithPropertyName("friend") - resp, err = helper.Client(t).Objects.ObjectsClassReferencesPut(params, nil) - helper.AssertRequestOk(t, resp, err, nil) - obj = helper.AssertGetObject(t, cls, uuid) - actual = obj.Properties.(map[string]interface{}) - expected["friend"] = expected["friend"].([]interface{})[:1] - assert.Equal(t, expected, actual) - - params.WithPropertyName("unknown") - _, err = helper.Client(t).Objects.ObjectsClassReferencesPut(params, nil) - if _, ok := err.(*objects.ObjectsClassReferencesPutUnprocessableEntity); !ok { - t.Errorf("error type expected: %T, got %T", objects.ObjectsClassReferencesPutUnprocessableEntity{}, err) - } - params.WithPropertyName("friend") - - params.WithID("e7cd261a-0000-0000-0000-d7b8e7b5c9ea") - _, err = helper.Client(t).Objects.ObjectsClassReferencesPut(params, nil) - if _, ok := err.(*objects.ObjectsClassReferencesPutNotFound); !ok { - t.Errorf("error type expected: %T, got %T", objects.ObjectsClassReferencesPutNotFound{}, err) - } - params.WithID(uuid) - - // exclude all - params.WithBody(models.MultipleRef{}).WithPropertyName("friend") - resp, err = helper.Client(t).Objects.ObjectsClassReferencesPut(params, nil) - helper.AssertRequestOk(t, resp, err, nil) - obj = helper.AssertGetObject(t, cls, uuid) - actual = obj.Properties.(map[string]interface{}) - expected["friend"] = expected["friend"].([]interface{})[1:] - assert.Equal(t, expected, actual) - - // bad request since body is required - params.WithID(uuid).WithBody(nil).WithPropertyName("friend") - _, err = helper.Client(t).Objects.ObjectsClassReferencesPut(params, nil) - if _, ok := err.(*objects.ObjectsClassReferencesPutUnprocessableEntity); !ok { - t.Errorf("error type expected: %T, got %T", objects.ObjectsClassReferencesPutUnprocessableEntity{}, err) - } -} - -func TestDeleteReference(t *testing.T) { - t.Parallel() - var ( - cls = "TestObjectHTTPDeleteReference" - first_friend = "TestObjectHTTPDeleteReferenceFriendFirst" - second_friend = "TestObjectHTTPDeleteReferenceFriendSecond" - mconfig = map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "vectorizeClassName": true, - }, - } - ) - // test setup - helper.AssertCreateObjectClass(t, &models.Class{ - Class: first_friend, - ModuleConfig: mconfig, - Properties: []*models.Property{}, - }) - defer helper.DeleteClassObject(t, first_friend) - - helper.AssertCreateObjectClass(t, &models.Class{ - Class: second_friend, - ModuleConfig: mconfig, - Properties: []*models.Property{}, - }) - defer helper.DeleteClassObject(t, second_friend) - - helper.AssertCreateObjectClass(t, &models.Class{ - Class: cls, - ModuleConfig: mconfig, - Properties: []*models.Property{ - { - Name: "number", - DataType: []string{"number"}, - }, - { - Name: "friend", - DataType: []string{first_friend, second_friend}, - }, - }, - }) - defer helper.DeleteClassObject(t, cls) - - first_friendID := helper.AssertCreateObject(t, first_friend, nil) - second_friendID := helper.AssertCreateObject(t, second_friend, nil) - uuid := helper.AssertCreateObject(t, cls, map[string]interface{}{ - "number": 2.0, - "friend": []interface{}{ - map[string]interface{}{ - "beacon": fmt.Sprintf("weaviate://localhost/%s/%s", first_friend, first_friendID), - "href": fmt.Sprintf("/v1/objects/%s/%s", first_friend, first_friendID), - }, - map[string]interface{}{ - "beacon": fmt.Sprintf("weaviate://localhost/%s/%s", second_friend, second_friendID), - "href": fmt.Sprintf("/v1/objects/%s/%s", second_friend, second_friendID), - }, - }, - }) - expected := map[string]interface{}{ - "number": json.Number("2"), - "friend": []interface{}{ - map[string]interface{}{ - "beacon": fmt.Sprintf("weaviate://localhost/%s/%s", first_friend, first_friendID), - "href": fmt.Sprintf("/v1/objects/%s/%s", first_friend, first_friendID), - }, - }, - } - - updateObj := crossref.NewLocalhost(second_friend, second_friendID).SingleRef() - // delete second reference - params := objects.NewObjectsClassReferencesDeleteParams().WithClassName(cls) - params.WithID(uuid).WithBody(updateObj).WithPropertyName("friend") - resp, err := helper.Client(t).Objects.ObjectsClassReferencesDelete(params, nil) - helper.AssertRequestOk(t, resp, err, nil) - obj := helper.AssertGetObject(t, cls, uuid) - actual := obj.Properties.(map[string]interface{}) - assert.Equal(t, expected, actual) - - // delete same reference again - resp, err = helper.Client(t).Objects.ObjectsClassReferencesDelete(params, nil) - helper.AssertRequestOk(t, resp, err, nil) - obj = helper.AssertGetObject(t, cls, uuid) - actual = obj.Properties.(map[string]interface{}) - assert.Equal(t, expected, actual) - - // delete last reference - expected = map[string]interface{}{ - "number": json.Number("2"), - "friend": []interface{}{}, - } - updateObj = crossref.NewLocalhost(first_friend, first_friendID).SingleRef() - params.WithID(uuid).WithBody(updateObj).WithPropertyName("friend") - resp, err = helper.Client(t).Objects.ObjectsClassReferencesDelete(params, nil) - helper.AssertRequestOk(t, resp, err, nil) - obj = helper.AssertGetObject(t, cls, uuid) - actual = obj.Properties.(map[string]interface{}) - assert.Equal(t, expected, actual) - - // property is not part of the schema - params.WithPropertyName("unknown") - _, err = helper.Client(t).Objects.ObjectsClassReferencesDelete(params, nil) - if _, ok := err.(*objects.ObjectsClassReferencesDeleteUnprocessableEntity); !ok { - t.Errorf("error type expected: %T, got %T", objects.ObjectsClassReferencesDeleteUnprocessableEntity{}, err) - } - params.WithPropertyName("friend") - - // This ID doesn't exist - params.WithID("e7cd261a-0000-0000-0000-d7b8e7b5c9ea") - _, err = helper.Client(t).Objects.ObjectsClassReferencesDelete(params, nil) - if _, ok := err.(*objects.ObjectsClassReferencesDeleteNotFound); !ok { - t.Errorf("error type expected: %T, got %T", objects.ObjectsClassReferencesDeleteNotFound{}, err) - } - params.WithID(uuid) - - // bad request since body is required - params.WithID(uuid).WithBody(nil).WithPropertyName("friend") - _, err = helper.Client(t).Objects.ObjectsClassReferencesDelete(params, nil) - if _, ok := err.(*objects.ObjectsClassReferencesDeleteUnprocessableEntity); !ok { - t.Errorf("error type expected: %T, got %T", objects.ObjectsClassReferencesDeleteUnprocessableEntity{}, err) - } -} - -func TestQuery(t *testing.T) { - t.Parallel() - var ( - cls = "TestObjectHTTPQuery" - first_friend = "TestObjectHTTPQueryFriend" - ) - // test setup - helper.AssertCreateObject(t, first_friend, map[string]interface{}{}) - defer helper.DeleteClassObject(t, first_friend) - helper.AssertCreateObjectClass(t, &models.Class{ - Class: cls, - Vectorizer: "none", - Properties: []*models.Property{ - { - Name: "count", - DataType: []string{"int"}, - }, - }, - }) - defer helper.DeleteClassObject(t, cls) - helper.AssertCreateObject(t, cls, map[string]interface{}{"count": 1}) - helper.AssertCreateObject(t, cls, map[string]interface{}{"count": 1}) - - listParams := objects.NewObjectsListParams() - listParams.Class = &cls - resp, err := helper.Client(t).Objects.ObjectsList(listParams, nil) - require.Nil(t, err, "unexpected error", resp) - - if n := len(resp.Payload.Objects); n != 2 { - t.Errorf("Number of object got:%v want %v", n, 2) - } - var count int64 - for _, x := range resp.Payload.Objects { - if x.Class != cls { - t.Errorf("Class got:%v want:%v", x.Class, cls) - } - m, ok := x.Properties.(map[string]interface{}) - if !ok { - t.Error("wrong property type") - } - n, _ := m["count"].(json.Number).Int64() - count += n - } - if count != 2 { - t.Errorf("Count got:%v want:%v", count, 2) - } - - listParams.Class = &first_friend - resp, err = helper.Client(t).Objects.ObjectsList(listParams, nil) - require.Nil(t, err, "unexpected error", resp) - if n := len(resp.Payload.Objects); n != 1 { - t.Errorf("Number of friend objects got:%v want %v", n, 2) - } - - unknown_cls := "unknow" - listParams.Class = &unknown_cls - _, err = helper.Client(t).Objects.ObjectsList(listParams, nil) - if _, ok := err.(*objects.ObjectsListNotFound); !ok { - t.Errorf("error type expected: %T, got %T", objects.ObjectsListNotFound{}, err) - } -} diff --git a/test/acceptance/actions/setup_test.go b/test/acceptance/actions/setup_test.go deleted file mode 100644 index acc69c523717938a79d3f89a3a09f910e9303ba8..0000000000000000000000000000000000000000 --- a/test/acceptance/actions/setup_test.go +++ /dev/null @@ -1,192 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "encoding/json" - "fmt" - "net/http" - "testing" - - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/client/nodes" - clschema "github.com/weaviate/weaviate/client/schema" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/storagestate" - "github.com/weaviate/weaviate/entities/verbosity" - "github.com/weaviate/weaviate/test/helper" - graphqlhelper "github.com/weaviate/weaviate/test/helper/graphql" -) - -func Test_Objects(t *testing.T) { - t.Run("setup", func(t *testing.T) { - helper.AssertCreateObjectClass(t, &models.Class{ - Class: "ObjectTestThing", - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "vectorizeClassName": true, - }, - }, - Properties: []*models.Property{ - { - Name: "testString", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - }) - helper.AssertCreateObjectClass(t, &models.Class{ - Class: "TestObject", - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "vectorizeClassName": true, - }, - }, - Properties: []*models.Property{ - { - Name: "testString", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "testWholeNumber", - DataType: []string{"int"}, - }, - { - Name: "testNumber", - DataType: []string{"number"}, - }, - { - Name: "testDateTime", - DataType: []string{"date"}, - }, - { - Name: "testTrueFalse", - DataType: []string{"boolean"}, - }, - { - Name: "testReference", - DataType: []string{"ObjectTestThing"}, - }, - }, - }) - helper.AssertCreateObjectClass(t, &models.Class{ - Class: "TestObjectTwo", - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "vectorizeClassName": true, - }, - }, - Properties: []*models.Property{ - { - Name: "testReference", - DataType: []string{"TestObject"}, - }, - { - Name: "testReferences", - DataType: []string{"TestObject"}, - }, - { - Name: "testString", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - }) - }) - - // tests - t.Run("adding objects", addingObjects) - t.Run("removing objects", removingObjects) - t.Run("object references", objectReferences) - t.Run("updating objects deprecated", updateObjectsDeprecated) - - // tear down - helper.AssertDeleteObjectClass(t, "ObjectTestThing") - helper.AssertDeleteObjectClass(t, "TestObject") - helper.AssertDeleteObjectClass(t, "TestObjectTwo") -} - -func Test_Delete_ReadOnly_Classes(t *testing.T) { - className := "DeleteReadonlyClassTest" - - t.Run("setup", func(t *testing.T) { - helper.AssertCreateObjectClass(t, &models.Class{ - Class: className, - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "vectorizeClassName": true, - }, - }, - Properties: []*models.Property{ - { - Name: "stringProp", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - }) - - batchSize := 1000 - batch := make([]*models.Object, batchSize) - for i := 0; i < batchSize; i++ { - batch[i] = &models.Object{ - Class: className, - Properties: map[string]interface{}{ - "stringProp": fmt.Sprintf("obj#%d", i+1), - }, - } - } - helper.CreateObjectsBatch(t, batch) - }) - - t.Run("assert data exists", func(t *testing.T) { - res := graphqlhelper.AssertGraphQL(t, helper.RootAuth, - fmt.Sprintf("{Aggregate {%s {meta {count}}}}", className)) - count := res.Get("Aggregate", className).AsSlice()[0].(map[string]interface{})["meta"].(map[string]interface{})["count"] - require.EqualValues(t, json.Number("1000"), count) - }) - - t.Run("set shard to readonly", func(t *testing.T) { - verbose := verbosity.OutputVerbose - nodesResp, err := helper.Client(t).Nodes.NodesGet(nodes.NewNodesGetParams().WithOutput(&verbose), nil) - require.Nil(t, err) - require.NotNil(t, nodesResp.Payload) - require.Len(t, nodesResp.Payload.Nodes, 1) - require.Len(t, nodesResp.Payload.Nodes[0].Shards, 1) - require.Equal(t, className, nodesResp.Payload.Nodes[0].Shards[0].Class) - targetShard := nodesResp.Payload.Nodes[0].Shards[0].Name - - params := clschema.NewSchemaObjectsShardsUpdateParams(). - WithBody(&models.ShardStatus{Status: storagestate.StatusReadOnly.String()}). - WithClassName(className). - WithShardName(targetShard) - shardsResp, err := helper.Client(t).Schema.SchemaObjectsShardsUpdate(params, nil) - require.Nil(t, err) - require.NotNil(t, shardsResp.Payload) - require.Equal(t, storagestate.StatusReadOnly.String(), shardsResp.Payload.Status) - }) - - t.Run("delete class with readonly shard", func(t *testing.T) { - params := clschema.NewSchemaObjectsDeleteParams().WithClassName(className) - resp, err := helper.Client(t).Schema.SchemaObjectsDelete(params, nil) - require.Nil(t, err) - require.True(t, resp.IsCode(http.StatusOK)) - }) - - t.Run("assert class is deleted", func(t *testing.T) { - params := clschema.NewSchemaObjectsGetParams().WithClassName(className) - _, err := helper.Client(t).Schema.SchemaObjectsGet(params, nil) - require.Equal(t, err, &clschema.SchemaObjectsGetNotFound{}) - }) -} diff --git a/test/acceptance/actions/update_test.go b/test/acceptance/actions/update_test.go deleted file mode 100644 index 8b9fdc86169a1d0db2eeb57f33dd9deb0c53f0ee..0000000000000000000000000000000000000000 --- a/test/acceptance/actions/update_test.go +++ /dev/null @@ -1,165 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -// Acceptance tests for objects - -import ( - "encoding/json" - "fmt" - "testing" - - "github.com/davecgh/go-spew/spew" - "github.com/weaviate/weaviate/client/objects" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/test/helper" -) - -// run from setup_test.go -func updateObjectsDeprecated(t *testing.T) { - t.Run("update and set number", func(t *testing.T) { - uuid := helper.AssertCreateObject(t, "TestObject", map[string]interface{}{}) - helper.AssertGetObjectEventually(t, "TestObject", uuid) - - schema := models.PropertySchema(map[string]interface{}{ - "testNumber": 41.0, - }) - - update := models.Object{} - update.Properties = schema - update.Class = "TestObject" - update.ID = uuid - - params := objects.NewObjectsUpdateParams().WithID(uuid).WithBody(&update) - updateResp, err := helper.Client(t).Objects.ObjectsUpdate(params, nil) - helper.AssertRequestOk(t, updateResp, err, nil) - - actualThunk := func() interface{} { - updatedObject := helper.AssertGetObject(t, update.Class, uuid) - updatedSchema := updatedObject.Properties.(map[string]interface{}) - if updatedSchema["testNumber"] == nil { - return nil - } - num, _ := updatedSchema["testNumber"].(json.Number).Float64() - return num - } - helper.AssertEventuallyEqual(t, 41.0, actualThunk) - }) - - t.Run("update and set string", func(t *testing.T) { - uuid := helper.AssertCreateObject(t, "TestObject", map[string]interface{}{}) - helper.AssertGetObjectEventually(t, "TestObject", uuid) - - schema := models.PropertySchema(map[string]interface{}{ - "testString": "wibbly wobbly", - }) - - update := models.Object{} - update.Properties = schema - update.Class = "TestObject" - update.ID = uuid - - params := objects.NewObjectsUpdateParams().WithID(uuid).WithBody(&update) - updateResp, err := helper.Client(t).Objects.ObjectsUpdate(params, nil) - helper.AssertRequestOk(t, updateResp, err, nil) - - actualThunk := func() interface{} { - updatedObject := helper.AssertGetObject(t, update.Class, uuid) - updatedSchema := updatedObject.Properties.(map[string]interface{}) - return updatedSchema["testString"] - } - helper.AssertEventuallyEqual(t, "wibbly wobbly", actualThunk) - }) - - t.Run("update and set bool", func(t *testing.T) { - t.Parallel() - uuid := helper.AssertCreateObject(t, "TestObject", map[string]interface{}{}) - helper.AssertGetObjectEventually(t, "TestObject", uuid) - - schema := models.PropertySchema(map[string]interface{}{ - "testTrueFalse": true, - }) - - update := models.Object{} - update.Properties = schema - update.Class = "TestObject" - update.ID = uuid - - params := objects.NewObjectsUpdateParams().WithID(uuid).WithBody(&update) - updateResp, err := helper.Client(t).Objects.ObjectsUpdate(params, nil) - - helper.AssertRequestOk(t, updateResp, err, nil) - - actualThunk := func() interface{} { - updatedObject := helper.AssertGetObject(t, update.Class, uuid) - updatedSchema := updatedObject.Properties.(map[string]interface{}) - return updatedSchema["testTrueFalse"] - } - helper.AssertEventuallyEqual(t, true, actualThunk) - }) - - t.Run("can patch object with cref", func(t *testing.T) { - thingToRefID := helper.AssertCreateObject(t, "ObjectTestThing", nil) - helper.AssertGetObjectEventually(t, "ObjectTestThing", thingToRefID) - objectID := helper.AssertCreateObject(t, "TestObject", nil) - helper.AssertGetObjectEventually(t, "TestObject", objectID) - - merge := &models.Object{ - Class: "TestObject", - Properties: map[string]interface{}{ - "testReference": []interface{}{ - map[string]interface{}{ - "beacon": fmt.Sprintf("weaviate://localhost/%s", thingToRefID), - }, - }, - }, - } - - // Now to try to link - params := objects.NewObjectsPatchParams(). - WithBody(merge). - WithID(objectID) - patchResp, err := helper.Client(t).Objects.ObjectsPatch(params, nil) - spew.Dump(err) - helper.AssertRequestOk(t, patchResp, err, nil) - - actualThunk := func() interface{} { - patchedObject := helper.AssertGetObject(t, merge.Class, objectID) - - rawRef, ok := patchedObject.Properties.(map[string]interface{})["testReference"] - if !ok { - return nil - } - - refsSlice, ok := rawRef.([]interface{}) - if !ok { - t.Logf("found the ref prop, but it was not a slice, but %T", refsSlice) - t.Fail() - } - - if len(refsSlice) != 1 { - t.Logf("expected ref slice to have one element, but got: %d", len(refsSlice)) - t.Fail() - } - - refMap, ok := refsSlice[0].(map[string]interface{}) - if !ok { - t.Logf("found the ref element, but it was not a map, but %T", refsSlice[0]) - t.Fail() - } - - return refMap["beacon"] - } - - helper.AssertEventuallyEqual(t, fmt.Sprintf("weaviate://localhost/ObjectTestThing/%s", thingToRefID), actualThunk) - }) -} diff --git a/test/acceptance/batch_request_endpoints/actionscreate_test.go b/test/acceptance/batch_request_endpoints/actionscreate_test.go deleted file mode 100644 index 7c39ff400b4c79c935f379bc62edeecb1fd0dc9d..0000000000000000000000000000000000000000 --- a/test/acceptance/batch_request_endpoints/actionscreate_test.go +++ /dev/null @@ -1,79 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// TODO: change this test to simulate a successful query response when the test dataset is implemented. - -// Acceptance tests for the batch ObjectsCreate endpoint -package batch_request_endpoints - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/client/batch" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/test/helper" -) - -// Test if batching is working correctly. Sends an OK batch containing two batched requests that refer to non-existing classes. -// The expected outcome is a 200 batch response containing two batched responses. These batched responses should both contain errors. -func TestBatchObjectsCreateResultsOrder(t *testing.T) { - t.Parallel() - - classOneName := "ItIsExtremelyUnlikelyThatThisClassActuallyExistsButJustToBeSureHereAreSomeRandomNumbers12987825624398509861298409782539802434516542" - classTwoName := "ItIsExtremelyUnlikelyThatThisClassActuallyExistsButJustToBeSureHereAreSomeRandomNumbers12987825624398509861298409782539802434516541" - expectedResult := "class '%s' not present in schema" - - // generate objectcreate content - object1 := &models.Object{ - Class: classOneName, - Properties: map[string]interface{}{ - "testString": "Test string", - }, - } - object2 := &models.Object{ - Class: classTwoName, - Properties: map[string]interface{}{ - "testWholeNumber": 1, - }, - } - - testFields := "ALL" - - // generate request body - params := batch.NewBatchObjectsCreateParams().WithBody(batch.BatchObjectsCreateBody{ - Objects: []*models.Object{object1, object2}, - Fields: []*string{&testFields}, - }) - - // perform the request - resp, err := helper.BatchClient(t).BatchObjectsCreate(params, nil) - // ensure that the response is OK - helper.AssertRequestOk(t, resp, err, func() { - objectsCreateResponse := resp.Payload - - // check if the batch response contains two batched responses - assert.Equal(t, 2, len(objectsCreateResponse)) - - // check if the error message matches the expected outcome (and are therefore returned in the correct order) - if len(objectsCreateResponse) == 2 { - responseOne := objectsCreateResponse[0].Result.Errors.Error[0].Message - responseTwo := objectsCreateResponse[1].Result.Errors.Error[0].Message - - fullExpectedOutcomeOne := fmt.Sprintf(expectedResult, classOneName) - assert.Contains(t, responseOne, fullExpectedOutcomeOne) - - fullExpectedOutcomeTwo := fmt.Sprintf(expectedResult, classTwoName) - assert.Contains(t, responseTwo, fullExpectedOutcomeTwo) - } - }) -} diff --git a/test/acceptance/batch_request_endpoints/batch_delete_test.go b/test/acceptance/batch_request_endpoints/batch_delete_test.go deleted file mode 100644 index 9b693c046f75507ed8a3c685e212fab5ea9fd806..0000000000000000000000000000000000000000 --- a/test/acceptance/batch_request_endpoints/batch_delete_test.go +++ /dev/null @@ -1,319 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package batch_request_endpoints - -import ( - "fmt" - "testing" - - "github.com/go-openapi/strfmt" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/client/batch" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/test/helper" -) - -func batchDeleteJourney(t *testing.T) { - maxObjects := 20 - var sources []*models.Object - equalThisName := "equal-this-name" - - getBatchDelete := func(className string, path []string, valueText string, dryRun bool) *batch.BatchObjectsDeleteParams { - output := "verbose" - params := batch.NewBatchObjectsDeleteParams().WithBody(&models.BatchDelete{ - Match: &models.BatchDeleteMatch{ - Class: className, - Where: &models.WhereFilter{ - Operator: "Equal", - Path: path, - ValueText: &valueText, - }, - }, - DryRun: &dryRun, - Output: &output, - }) - return params - } - - sourceUUIDs := make([]strfmt.UUID, maxObjects) - targetUUIDs := make([]strfmt.UUID, maxObjects) - - t.Run("create some data", func(t *testing.T) { - sources = make([]*models.Object, maxObjects) - for i := range sources { - uuid := mustNewUUID() - - sources[i] = &models.Object{ - Class: "BulkTestSource", - ID: uuid, - Properties: map[string]interface{}{ - "name": equalThisName, - }, - } - - sourceUUIDs[i] = uuid - } - - targets := make([]*models.Object, maxObjects) - for i := range targets { - uuid := mustNewUUID() - - targets[i] = &models.Object{ - Class: "BulkTestTarget", - ID: uuid, - Properties: map[string]interface{}{ - "intProp": i, - }, - } - - targetUUIDs[i] = uuid - } - }) - - t.Run("import all batch objects", func(t *testing.T) { - params := batch.NewBatchObjectsCreateParams().WithBody( - batch.BatchObjectsCreateBody{ - Objects: sources, - }, - ) - res, err := helper.Client(t).Batch.BatchObjectsCreate(params, nil) - require.Nil(t, err) - - for _, elem := range res.Payload { - require.Nil(t, elem.Result.Errors) - } - }) - - t.Run("import all batch refs", func(t *testing.T) { - batchRefs := make([]*models.BatchReference, len(sources)) - - for i := range batchRefs { - batchRefs[i] = &models.BatchReference{ - From: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s/%s/fromSource", "BulkTestTarget", targetUUIDs[i])), - To: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", sourceUUIDs[i])), - } - } - - params := batch.NewBatchReferencesCreateParams().WithBody(batchRefs) - res, err := helper.Client(t).Batch.BatchReferencesCreate(params, nil) - require.Nil(t, err) - - for _, elem := range res.Payload { - require.Nil(t, elem.Result.Errors) - } - }) - - t.Run("verify using GraphQL", func(t *testing.T) { - // verify objects - result := AssertGraphQL(t, helper.RootAuth, ` - { Get { BulkTestSource(where:{operator:Equal path:["name"] valueText:"equal-this-name"}) { name } } } - `) - items := result.Get("Get", "BulkTestSource").AsSlice() - require.Len(t, items, maxObjects) - - // verify refs - result = AssertGraphQL(t, helper.RootAuth, ` - { - Get { - BulkTestTarget - ( - where: { - path: ["fromSource", "BulkTestSource", "name"] - operator: Equal - valueText: "equal-this-name" - } - ) - { - fromSource { - ... on BulkTestSource { - _additional { - id - } - } - } - } - } - } - `) - items = result.Get("Get", "BulkTestTarget").AsSlice() - for _, item := range items { - fromSource := item.(map[string]interface{})["fromSource"] - require.NotNil(t, fromSource) - } - require.Len(t, items, maxObjects) - }) - - t.Run("perform batch delete by refs dry run", func(t *testing.T) { - params := getBatchDelete("BulkTestTarget", []string{"fromSource", "BulkTestSource", "name"}, equalThisName, true) - res, err := helper.Client(t).Batch.BatchObjectsDelete(params, nil) - require.Nil(t, err) - - response := res.Payload - require.NotNil(t, response) - require.NotNil(t, response.Match) - require.NotNil(t, response.Results) - require.Equal(t, int64(maxObjects), response.Results.Matches) - require.Equal(t, int64(0), response.Results.Successful) - require.Equal(t, int64(0), response.Results.Failed) - require.Equal(t, maxObjects, len(response.Results.Objects)) - for _, elem := range response.Results.Objects { - require.Nil(t, elem.Errors) - } - }) - - t.Run("[deprecated string] perform batch delete by refs dry run", func(t *testing.T) { - params := getBatchDelete("BulkTestTarget", []string{"fromSource", "BulkTestSource", "name"}, equalThisName, true) - params.Body.Match.Where.ValueText = nil - params.Body.Match.Where.ValueString = &equalThisName - - res, err := helper.Client(t).Batch.BatchObjectsDelete(params, nil) - require.Nil(t, err) - - response := res.Payload - require.NotNil(t, response) - require.NotNil(t, response.Match) - require.NotNil(t, response.Results) - require.Equal(t, int64(maxObjects), response.Results.Matches) - require.Equal(t, int64(0), response.Results.Successful) - require.Equal(t, int64(0), response.Results.Failed) - require.Equal(t, maxObjects, len(response.Results.Objects)) - for _, elem := range response.Results.Objects { - require.Nil(t, elem.Errors) - } - }) - - t.Run("verify that batch delete by refs dry run didn't delete data", func(t *testing.T) { - result := AssertGraphQL(t, helper.RootAuth, ` - { - Get { - BulkTestTarget - ( - where: { - path: ["fromSource", "BulkTestSource", "name"] - operator: Equal - valueText: "equal-this-name" - } - ) - { - fromSource { - ... on BulkTestSource { - _additional { - id - } - } - } - } - } - } - `) - items := result.Get("Get", "BulkTestTarget").AsSlice() - require.Len(t, items, maxObjects) - }) - - t.Run("perform batch delete by prop dry run", func(t *testing.T) { - params := getBatchDelete("BulkTestSource", []string{"name"}, equalThisName, true) - res, err := helper.Client(t).Batch.BatchObjectsDelete(params, nil) - require.Nil(t, err) - - response := res.Payload - require.NotNil(t, response) - require.NotNil(t, response.Match) - require.NotNil(t, response.Results) - require.Equal(t, int64(maxObjects), response.Results.Matches) - require.Equal(t, int64(0), response.Results.Successful) - require.Equal(t, int64(0), response.Results.Failed) - require.Equal(t, maxObjects, len(response.Results.Objects)) - for _, elem := range response.Results.Objects { - require.Nil(t, elem.Errors) - } - }) - - t.Run("verify that batch delete by prop dry run didn't delete data", func(t *testing.T) { - result := AssertGraphQL(t, helper.RootAuth, ` - { Get { BulkTestSource(where:{operator:Equal path:["name"] valueText:"equal-this-name"}) { name } } } - `) - items := result.Get("Get", "BulkTestSource").AsSlice() - require.Len(t, items, maxObjects) - }) - - t.Run("perform batch delete by ref", func(t *testing.T) { - params := getBatchDelete("BulkTestTarget", []string{"fromSource", "BulkTestSource", "name"}, equalThisName, false) - res, err := helper.Client(t).Batch.BatchObjectsDelete(params, nil) - require.Nil(t, err) - - response := res.Payload - require.NotNil(t, response) - require.NotNil(t, response.Match) - require.NotNil(t, response.Results) - require.Equal(t, int64(maxObjects), response.Results.Matches) - require.Equal(t, int64(maxObjects), response.Results.Successful) - require.Equal(t, int64(0), response.Results.Failed) - require.Equal(t, maxObjects, len(response.Results.Objects)) - for _, elem := range response.Results.Objects { - require.Nil(t, elem.Errors) - } - }) - - t.Run("verify that batch delete by ref deleted everything", func(t *testing.T) { - result := AssertGraphQL(t, helper.RootAuth, ` - { - Get { - BulkTestTarget - ( - where: { - path: ["fromSource", "BulkTestSource", "name"] - operator: Equal - valueText: "equal-this-name" - } - ) - { - fromSource { - ... on BulkTestSource { - _additional { - id - } - } - } - } - } - } - `) - items := result.Get("Get", "BulkTestTarget").AsSlice() - require.Len(t, items, 0) - }) - - t.Run("perform batch delete by prop", func(t *testing.T) { - params := getBatchDelete("BulkTestSource", []string{"name"}, equalThisName, false) - res, err := helper.Client(t).Batch.BatchObjectsDelete(params, nil) - require.Nil(t, err) - - response := res.Payload - require.NotNil(t, response) - require.NotNil(t, response.Match) - require.NotNil(t, response.Results) - require.Equal(t, int64(maxObjects), response.Results.Matches) - require.Equal(t, int64(maxObjects), response.Results.Successful) - require.Equal(t, int64(0), response.Results.Failed) - require.Equal(t, maxObjects, len(response.Results.Objects)) - for _, elem := range response.Results.Objects { - require.Nil(t, elem.Errors) - } - }) - - t.Run("verify that batch delete by prop deleted everything", func(t *testing.T) { - result := AssertGraphQL(t, helper.RootAuth, ` - { Get { BulkTestSource(where:{operator:Equal path:["name"] valueText:"equal-this-name"}) { name } } } - `) - items := result.Get("Get", "BulkTestSource").AsSlice() - require.Len(t, items, 0) - }) -} diff --git a/test/acceptance/batch_request_endpoints/batch_journey_test.go b/test/acceptance/batch_request_endpoints/batch_journey_test.go deleted file mode 100644 index a513df889ce18cc2c7c2cd7589922076fd1ac621..0000000000000000000000000000000000000000 --- a/test/acceptance/batch_request_endpoints/batch_journey_test.go +++ /dev/null @@ -1,194 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package batch_request_endpoints - -import ( - "fmt" - "testing" - - "github.com/go-openapi/strfmt" - "github.com/google/uuid" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/client/batch" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/test/helper" -) - -func batchJourney(t *testing.T) { - sourcesSize := 10 - targetsSize := 3 - var sources []*models.Object - var targets []*models.Object - - t.Run("create some data", func(t *testing.T) { - sources = make([]*models.Object, sourcesSize) - for i := range sources { - sources[i] = &models.Object{ - Class: "BulkTestSource", - ID: mustNewUUID(), - Properties: map[string]interface{}{ - "name": fmt.Sprintf("source%d", i), - }, - } - } - - targets = make([]*models.Object, targetsSize) - for i := range targets { - targets[i] = &models.Object{ - Class: "BulkTest", - ID: mustNewUUID(), - Properties: map[string]interface{}{ - "name": fmt.Sprintf("target%d", i), - }, - } - } - }) - - t.Run("import all data in batch", func(t *testing.T) { - params := batch.NewBatchObjectsCreateParams().WithBody( - batch.BatchObjectsCreateBody{ - Objects: append(sources, targets...), - }, - ) - res, err := helper.Client(t).Batch.BatchObjectsCreate(params, nil) - require.Nil(t, err) - - for _, elem := range res.Payload { - assert.Nil(t, elem.Result.Errors) - } - }) - - t.Run("set one cref each from each source to all targets", func(t *testing.T) { - body := make([]*models.BatchReference, sourcesSize*targetsSize) - for i := range sources { - for j := range targets { - index := i*targetsSize + j - body[index] = &models.BatchReference{ - From: strfmt.URI( - fmt.Sprintf("weaviate://localhost/BulkTestSource/%s/ref", sources[i].ID)), - To: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", targets[j].ID)), - } - } - } - params := batch.NewBatchReferencesCreateParams().WithBody(body) - res, err := helper.Client(t).Batch.BatchReferencesCreate(params, nil) - require.Nil(t, err) - - for _, elem := range res.Payload { - assert.Nil(t, elem.Result.Errors) - } - }) - - t.Run("verify using GraphQL", func(t *testing.T) { - result := AssertGraphQL(t, helper.RootAuth, ` - { Get { BulkTestSource { ref { ... on BulkTest { name } } } } } - `) - items := result.Get("Get", "BulkTestSource").AsSlice() - assert.Len(t, items, sourcesSize) - for _, obj := range items { - refs := obj.(map[string]interface{})["ref"].([]interface{}) - assert.Len(t, refs, targetsSize) - } - }) -} - -func mustNewUUID() strfmt.UUID { - return strfmt.UUID(uuid.New().String()) -} - -func Test_BugFlakyResultCountWithVectorSearch(t *testing.T) { - className := "FlakyBugTestClass" - - // since this bug occurs only in around 1 in 25 cases, we run the test - // multiple times to increase the chance we're running into it - amount := 50 - for i := 0; i < amount; i++ { - t.Run("create schema", func(t *testing.T) { - createObjectClass(t, &models.Class{ - Class: className, - Properties: []*models.Property{ - { - Name: "title", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "url", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "wordCount", - DataType: []string{"int"}, - }, - }, - }) - }) - - t.Run("create and import some data", func(t *testing.T) { - objects := []*models.Object{ - { - Class: className, - Properties: map[string]interface{}{ - "title": "article 1", - "url": "http://articles.local/my-article-1", - "wordCount": 60, - }, - }, - { - Class: className, - Properties: map[string]interface{}{ - "title": "article 2", - "url": "http://articles.local/my-article-2", - "wordCount": 40, - }, - }, - { - Class: className, - Properties: map[string]interface{}{ - "title": "article 3", - "url": "http://articles.local/my-article-3", - "wordCount": 600, - }, - }, - } - - params := batch.NewBatchObjectsCreateParams().WithBody( - batch.BatchObjectsCreateBody{ - Objects: objects, - }, - ) - res, err := helper.Client(t).Batch.BatchObjectsCreate(params, nil) - require.Nil(t, err) - - for _, elem := range res.Payload { - assert.Nil(t, elem.Result.Errors) - } - }) - - t.Run("verify using GraphQL", func(t *testing.T) { - result := AssertGraphQL(t, helper.RootAuth, fmt.Sprintf(` - { Get { %s(nearText: {concepts: ["news"]}) { - wordCount title url - } } } - `, className)) - items := result.Get("Get", className).AsSlice() - assert.Len(t, items, 3) - }) - - t.Run("cleanup", func(t *testing.T) { - deleteObjectClass(t, className) - }) - } -} diff --git a/test/acceptance/batch_request_endpoints/graphql_helper_for_test.go b/test/acceptance/batch_request_endpoints/graphql_helper_for_test.go deleted file mode 100644 index c0d0384de393e3e9087077c972a348bbeaf4350f..0000000000000000000000000000000000000000 --- a/test/acceptance/batch_request_endpoints/graphql_helper_for_test.go +++ /dev/null @@ -1,88 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package batch_request_endpoints - -import ( - "encoding/json" - "fmt" - "testing" - - "github.com/go-openapi/runtime" - "github.com/weaviate/weaviate/client/graphql" - graphql_client "github.com/weaviate/weaviate/client/graphql" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/test/helper" -) - -type GraphQLResult struct { - Result interface{} -} - -// Perform a GraphQL request -func QueryGraphQL(t *testing.T, auth runtime.ClientAuthInfoWriterFunc, operation string, query string, variables map[string]interface{}) (*models.GraphQLResponse, error) { - var vars interface{} = variables - params := graphql_client.NewGraphqlPostParams().WithBody(&models.GraphQLQuery{OperationName: operation, Query: query, Variables: vars}) - response, err := helper.Client(t).Graphql.GraphqlPost(params, nil) - if err != nil { - return nil, err - } - - return response.Payload, nil -} - -// Perform a query and assert that it is successful -func AssertGraphQL(t *testing.T, auth runtime.ClientAuthInfoWriterFunc, query string) *GraphQLResult { - response, err := QueryGraphQL(t, auth, "", query, nil) - if err != nil { - parsedErr, ok := err.(*graphql.GraphqlPostUnprocessableEntity) - if !ok { - t.Fatalf("Expected the query to succeed, but failed due to: %+v", err) - } - t.Fatalf("Expected the query to succeed, but failed with unprocessable entity: %v", parsedErr.Payload.Error[0]) - } - - if len(response.Errors) != 0 { - j, _ := json.Marshal(response.Errors) - t.Fatal("GraphQL resolved to an error:", string(j)) - } - - data := make(map[string]interface{}) - - // get rid of models.JSONData - for key, value := range response.Data { - data[key] = value - } - - return &GraphQLResult{Result: data} -} - -// Drill down in the result -func (g GraphQLResult) Get(paths ...string) *GraphQLResult { - current := g.Result - for _, path := range paths { - var ok bool - currentAsMap := (current.(map[string]interface{})) - current, ok = currentAsMap[path] - if !ok { - panic(fmt.Sprintf("Cannot get element %s in %#v; result: %#v", path, paths, g.Result)) - } - } - - return &GraphQLResult{ - Result: current, - } -} - -// Cast the result to a slice -func (g *GraphQLResult) AsSlice() []interface{} { - return g.Result.([]interface{}) -} diff --git a/test/acceptance/batch_request_endpoints/graphql_test.go b/test/acceptance/batch_request_endpoints/graphql_test.go deleted file mode 100644 index 0d403a693991ad7de2e089300e4c7937b6c14c80..0000000000000000000000000000000000000000 --- a/test/acceptance/batch_request_endpoints/graphql_test.go +++ /dev/null @@ -1,98 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package batch_request_endpoints - -// TODO: These tests add little value, they only test one specific error case, -// but don't test any happy path at all. This should probably be removed or -// fixed. However, they do at least assure that the order of return values matches -// the order of input values. - -// Acceptance tests for the batch GraphQL endpoint - -// There is a helper struct called GraphQLResult that helps to navigate through the output, -// a query generator and a few helper functions to access the GraphQL endpoint. -// See the end of this file for more details on how those work. - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - graphql_client "github.com/weaviate/weaviate/client/graphql" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/test/helper" -) - -// TODO: change this test to simulate a successful query response when the test dataset is implemented. - -// Check if batch results are returned in the correct order by comparing result equality to predefined outcomes. -// This includes testing whether individual requests and the batch request are handled correctly -func gqlResultsOrder(t *testing.T) { - queryOneName := "testQuery" - queryTwoName := "testQuery2" - expectedResult := "Syntax Error GraphQL request (1:1) Unexpected Name \"%s\"\n\n1: %s\n ^\n" - - // perform the query - gqlResponse, err := queryBatchEndpoint(t, nil) - if err != nil { - t.Fatalf("The returned schema is not an JSON object: %v", err) - } - // check if the batch response contains two batched responses - assert.Equal(t, 2, len(gqlResponse)) - - // check if the error message matches the expected outcome (and are therefore returned in the correct order) - if len(gqlResponse) == 2 { - responseOne := gqlResponse[0].Errors[0].Message - responseTwo := gqlResponse[1].Errors[0].Message - - fullExpectedOutcomeOne := fmt.Sprintf(expectedResult, queryOneName, queryOneName) - assert.Equal(t, fullExpectedOutcomeOne, responseOne) - - fullExpectedOutcomeTwo := fmt.Sprintf(expectedResult, queryTwoName, queryTwoName) - assert.Equal(t, fullExpectedOutcomeTwo, responseTwo) - } -} - -func gqlMalformedRequest(t *testing.T) { - vars := []int{1, 2, 3} - expectedResult := "422: expected map[string]interface{}, received %v" - - // perform the query - gqlResponse, err := queryBatchEndpoint(t, vars) - if err != nil { - t.Fatalf("The returned schema is not an JSON object: %v", err) - } - // check if the batch response contains two batched responses - assert.Equal(t, 2, len(gqlResponse)) - - fullExpectedOutcome := fmt.Sprintf(expectedResult, vars) - assert.Equal(t, fullExpectedOutcome, gqlResponse[0].Errors[0].Message) - assert.Equal(t, fullExpectedOutcome, gqlResponse[1].Errors[0].Message) -} - -// Helper functions -// TODO: change this to a successful query when the test dataset is implemented. Make sure to implement a query returning 3 or more elements. -// Perform a batch GraphQL query -func queryBatchEndpoint(t *testing.T, vars interface{}) (models.GraphQLResponses, error) { - query1 := &models.GraphQLQuery{OperationName: "testQuery", Query: "testQuery", Variables: vars} - query2 := &models.GraphQLQuery{OperationName: "testQuery2", Query: "testQuery2", Variables: vars} - - queries := models.GraphQLQueries{query1, query2} - - params := graphql_client.NewGraphqlBatchParams().WithBody(queries) - response, err := helper.Client(t).Graphql.GraphqlBatch(params, nil) - if err != nil { - return nil, err - } - - return response.Payload, nil -} diff --git a/test/acceptance/batch_request_endpoints/setup_test.go b/test/acceptance/batch_request_endpoints/setup_test.go deleted file mode 100644 index 1ea6ab1b93c2f77131ed383f9780d4ddf2c84a49..0000000000000000000000000000000000000000 --- a/test/acceptance/batch_request_endpoints/setup_test.go +++ /dev/null @@ -1,89 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package batch_request_endpoints - -import ( - "testing" - "time" - - clschema "github.com/weaviate/weaviate/client/schema" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/test/helper" -) - -func Test_Batch(t *testing.T) { - // there is no gql provider if there is no schema, so we need some sort of a schema - - t.Run("setup", func(t *testing.T) { - createObjectClass(t, &models.Class{ - Class: "BulkTest", - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - }) - createObjectClass(t, &models.Class{ - Class: "BulkTestSource", - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "ref", - DataType: []string{"BulkTest"}, - }, - }, - }) - createObjectClass(t, &models.Class{ - Class: "BulkTestTarget", - Properties: []*models.Property{ - { - Name: "intProp", - DataType: []string{"int"}, - }, - { - Name: "fromSource", - DataType: []string{"BulkTestSource"}, - }, - }, - }) - }) - - time.Sleep(2000 * time.Millisecond) - - t.Run("gql results order", batchJourney) - t.Run("gql results order", gqlResultsOrder) - t.Run("gql malformed request", gqlMalformedRequest) - t.Run("batch delete", batchDeleteJourney) - - deleteObjectClass(t, "BulkTest") - deleteObjectClass(t, "BulkTestSource") - deleteObjectClass(t, "BulkTestTarget") -} - -func createObjectClass(t *testing.T, class *models.Class) { - params := clschema.NewSchemaObjectsCreateParams().WithObjectClass(class) - resp, err := helper.Client(t).Schema.SchemaObjectsCreate(params, nil) - helper.AssertRequestOk(t, resp, err, nil) -} - -func deleteObjectClass(t *testing.T, class string) { - delParams := clschema.NewSchemaObjectsDeleteParams().WithClassName(class) - delRes, err := helper.Client(t).Schema.SchemaObjectsDelete(delParams, nil) - helper.AssertRequestOk(t, delRes, err, nil) -} diff --git a/test/acceptance/batch_request_endpoints/thingscreate_test.go b/test/acceptance/batch_request_endpoints/thingscreate_test.go deleted file mode 100644 index 145b63f933846e2777647b0d1e7a7785466c45f5..0000000000000000000000000000000000000000 --- a/test/acceptance/batch_request_endpoints/thingscreate_test.go +++ /dev/null @@ -1,80 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// TODO: change this test to simulate a successful query response when the test dataset is implemented. - -// Acceptance tests for the batch ThingsCreate endpoint -package batch_request_endpoints - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/client/batch" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/test/helper" -) - -// Test if batching is working correctly. Sends an OK batch containing two batched requests that refer to non-existing classes. -// The expected outcome is a 200 batch response containing two batched responses. These batched responses should both contain errors. -func TestBatchThingsCreateResultsOrder(t *testing.T) { - t.Parallel() - - classOneName := "ItIsExtremelyUnlikelyThatThisClassActuallyExistsButJustToBeSureHereAreSomeRandomNumbers12987825624398509861298409782539802434516542" - classTwoName := "ItIsExtremelyUnlikelyThatThisClassActuallyExistsButJustToBeSureHereAreSomeRandomNumbers12987825624398509861298409782539802434516541" - expectedResult := "class '%s' not present in schema" - - // generate actioncreate content - object1 := &models.Object{ - Class: classOneName, - Properties: map[string]interface{}{ - "testString": "Test string", - }, - } - object2 := &models.Object{ - Class: classTwoName, - Properties: map[string]interface{}{ - "testWholeNumber": 1, - }, - } - - testFields := "ALL" - - // generate request body - params := batch.NewBatchObjectsCreateParams().WithBody(batch.BatchObjectsCreateBody{ - Objects: []*models.Object{object1, object2}, - Fields: []*string{&testFields}, - }) - - // perform the request - resp, err := helper.BatchClient(t).BatchObjectsCreate(params, nil) - - // ensure that the response is OK - helper.AssertRequestOk(t, resp, err, func() { - thingsCreateResponse := resp.Payload - - // check if the batch response contains two batched responses - assert.Equal(t, 2, len(thingsCreateResponse)) - - // check if the error message matches the expected outcome (and are therefore returned in the correct order) - if len(thingsCreateResponse) == 2 { - responseOne := thingsCreateResponse[0].Result.Errors.Error[0].Message - responseTwo := thingsCreateResponse[1].Result.Errors.Error[0].Message - - fullExpectedOutcomeOne := fmt.Sprintf(expectedResult, classOneName) - assert.Contains(t, responseOne, fullExpectedOutcomeOne) - - fullExpectedOutcomeTwo := fmt.Sprintf(expectedResult, classTwoName) - assert.Contains(t, responseTwo, fullExpectedOutcomeTwo) - } - }) -} diff --git a/test/acceptance/classifications/contextual_classification_test.go b/test/acceptance/classifications/contextual_classification_test.go deleted file mode 100644 index 3e275ebc77eb3b5ba3026fa547711b82c2901b5f..0000000000000000000000000000000000000000 --- a/test/acceptance/classifications/contextual_classification_test.go +++ /dev/null @@ -1,101 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "testing" - "time" - - "github.com/go-openapi/strfmt" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/client/classifications" - "github.com/weaviate/weaviate/client/objects" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/test/helper" - testhelper "github.com/weaviate/weaviate/test/helper" -) - -func contextualClassification(t *testing.T) { - var id strfmt.UUID - - res, err := helper.Client(t).Classifications.ClassificationsPost(classifications.NewClassificationsPostParams(). - WithParams(&models.Classification{ - Class: "Article", - ClassifyProperties: []string{"ofCategory"}, - BasedOnProperties: []string{"content"}, - Type: "text2vec-contextionary-contextual", - }), nil) - require.Nil(t, err) - id = res.Payload.ID - - // wait for classification to be completed - testhelper.AssertEventuallyEqualWithFrequencyAndTimeout(t, "completed", func() interface{} { - res, err := helper.Client(t).Classifications.ClassificationsGet(classifications.NewClassificationsGetParams(). - WithID(id.String()), nil) - - require.Nil(t, err) - return res.Payload.Status - }, 100*time.Millisecond, 15*time.Second) - - // wait for latest changes to be indexed / wait for consistency - testhelper.AssertEventuallyEqual(t, true, func() interface{} { - res, err := helper.Client(t).Objects.ObjectsGet(objects.NewObjectsGetParams(). - WithID(article1), nil) - require.Nil(t, err) - return res.Payload.Properties.(map[string]interface{})["ofCategory"] != nil - }) - testhelper.AssertEventuallyEqual(t, true, func() interface{} { - res, err := helper.Client(t).Objects.ObjectsGet(objects.NewObjectsGetParams(). - WithID(article2), nil) - require.Nil(t, err) - return res.Payload.Properties.(map[string]interface{})["ofCategory"] != nil - }) - testhelper.AssertEventuallyEqual(t, true, func() interface{} { - res, err := helper.Client(t).Objects.ObjectsGet(objects.NewObjectsGetParams(). - WithID(article3), nil) - require.Nil(t, err) - return res.Payload.Properties.(map[string]interface{})["ofCategory"] != nil - }) - - gres := AssertGraphQL(t, nil, ` -{ - Get { - Article { - _additional { - id - } - ofCategory { - ... on Category { - name - } - } - } - } -}`) - - expectedCategoriesByID := map[strfmt.UUID]string{ - article1: "Computers and Technology", - article2: "Food and Drink", - article3: "Politics", - } - articles := gres.Get("Get", "Article").AsSlice() - for _, article := range articles { - actual := article.(map[string]interface{})["ofCategory"].([]interface{})[0].(map[string]interface{})["name"].(string) - id := article.(map[string]interface{})["_additional"].(map[string]interface{})["id"].(string) - assert.Equal(t, expectedCategoriesByID[strfmt.UUID(id)], actual) - } -} - -func ptString(in string) *string { - return &in -} diff --git a/test/acceptance/classifications/graphql_helper.go b/test/acceptance/classifications/graphql_helper.go deleted file mode 100644 index 05ad436102b3fe7f8944e8a153039a8090f9f7da..0000000000000000000000000000000000000000 --- a/test/acceptance/classifications/graphql_helper.go +++ /dev/null @@ -1,88 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "encoding/json" - "fmt" - "testing" - - "github.com/go-openapi/runtime" - "github.com/weaviate/weaviate/client/graphql" - graphql_client "github.com/weaviate/weaviate/client/graphql" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/test/helper" -) - -type GraphQLResult struct { - Result interface{} -} - -// Perform a GraphQL request -func QueryGraphQL(t *testing.T, auth runtime.ClientAuthInfoWriterFunc, operation string, query string, variables map[string]interface{}) (*models.GraphQLResponse, error) { - var vars interface{} = variables - params := graphql_client.NewGraphqlPostParams().WithBody(&models.GraphQLQuery{OperationName: operation, Query: query, Variables: vars}) - response, err := helper.Client(t).Graphql.GraphqlPost(params, nil) - if err != nil { - return nil, err - } - - return response.Payload, nil -} - -// Perform a query and assert that it is successful -func AssertGraphQL(t *testing.T, auth runtime.ClientAuthInfoWriterFunc, query string) *GraphQLResult { - response, err := QueryGraphQL(t, auth, "", query, nil) - if err != nil { - parsedErr, ok := err.(*graphql.GraphqlPostUnprocessableEntity) - if !ok { - t.Fatalf("Expected the query to succeed, but failed due to: %#v", err) - } - t.Fatalf("Expected the query to succeed, but failed with unprocessable entity: %v", parsedErr.Payload.Error[0]) - } - - if len(response.Errors) != 0 { - j, _ := json.Marshal(response.Errors) - t.Fatal("GraphQL resolved to an error:", string(j)) - } - - data := make(map[string]interface{}) - - // get rid of models.JSONData - for key, value := range response.Data { - data[key] = value - } - - return &GraphQLResult{Result: data} -} - -// Drill down in the result -func (g GraphQLResult) Get(paths ...string) *GraphQLResult { - current := g.Result - for _, path := range paths { - var ok bool - currentAsMap := (current.(map[string]interface{})) - current, ok = currentAsMap[path] - if !ok { - panic(fmt.Sprintf("Cannot get element %s in %#v; result: %#v", path, paths, g.Result)) - } - } - - return &GraphQLResult{ - Result: current, - } -} - -// Cast the result to a slice -func (g *GraphQLResult) AsSlice() []interface{} { - return g.Result.([]interface{}) -} diff --git a/test/acceptance/classifications/knn_classification_test.go b/test/acceptance/classifications/knn_classification_test.go deleted file mode 100644 index 222487b82e61850b44469f0b14bd91b5471c0363..0000000000000000000000000000000000000000 --- a/test/acceptance/classifications/knn_classification_test.go +++ /dev/null @@ -1,145 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "encoding/json" - "fmt" - "testing" - "time" - - "github.com/go-openapi/strfmt" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/client/classifications" - "github.com/weaviate/weaviate/client/objects" - "github.com/weaviate/weaviate/client/schema" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/test/helper" - testhelper "github.com/weaviate/weaviate/test/helper" -) - -func knnClassification(t *testing.T) { - var id strfmt.UUID - - t.Run("ensure class shard for classification is ready", func(t *testing.T) { - testhelper.AssertEventuallyEqualWithFrequencyAndTimeout(t, "READY", - func() interface{} { - shardStatus, err := helper.Client(t).Schema.SchemaObjectsShardsGet(schema.NewSchemaObjectsShardsGetParams().WithClassName("Recipe"), nil) - require.Nil(t, err) - require.GreaterOrEqual(t, len(shardStatus.Payload), 1) - return shardStatus.Payload[0].Status - }, 250*time.Millisecond, 15*time.Second) - }) - - t.Run("start the classification and wait for completion", func(t *testing.T) { - res, err := helper.Client(t).Classifications.ClassificationsPost( - classifications.NewClassificationsPostParams().WithParams(&models.Classification{ - Class: "Recipe", - ClassifyProperties: []string{"ofType"}, - BasedOnProperties: []string{"content"}, - Type: "knn", - Settings: map[string]interface{}{ - "k": 5, - }, - }), nil) - require.Nil(t, err) - id = res.Payload.ID - - // wait for classification to be completed - testhelper.AssertEventuallyEqualWithFrequencyAndTimeout(t, "completed", - func() interface{} { - res, err := helper.Client(t).Classifications.ClassificationsGet( - classifications.NewClassificationsGetParams().WithID(id.String()), nil) - - require.Nil(t, err) - return res.Payload.Status - }, 100*time.Millisecond, 15*time.Second) - }) - - t.Run("assure changes present", func(t *testing.T) { - // wait for latest changes to be indexed / wait for consistency - testhelper.AssertEventuallyEqual(t, true, func() interface{} { - res, err := helper.Client(t).Objects.ObjectsGet(objects.NewObjectsGetParams(). - WithID(unclassifiedSavory), nil) - require.Nil(t, err) - return res.Payload.Properties.(map[string]interface{})["ofType"] != nil - }) - testhelper.AssertEventuallyEqual(t, true, func() interface{} { - res, err := helper.Client(t).Objects.ObjectsGet(objects.NewObjectsGetParams(). - WithID(unclassifiedSweet), nil) - require.Nil(t, err) - return res.Payload.Properties.(map[string]interface{})["ofType"] != nil - }) - }) - - t.Run("inspect unclassified savory", func(t *testing.T) { - res, err := helper.Client(t).Objects. - ObjectsGet(objects.NewObjectsGetParams(). - WithID(unclassifiedSavory). - WithInclude(ptString("classification")), nil) - - require.Nil(t, err) - schema, ok := res.Payload.Properties.(map[string]interface{}) - require.True(t, ok) - - expectedRefTarget := fmt.Sprintf("weaviate://localhost/RecipeType/%s", - recipeTypeSavory) - ref := schema["ofType"].([]interface{})[0].(map[string]interface{}) - assert.Equal(t, ref["beacon"].(string), expectedRefTarget) - - verifyMetaDistances(t, ref) - }) - - t.Run("inspect unclassified sweet", func(t *testing.T) { - res, err := helper.Client(t).Objects. - ObjectsGet(objects.NewObjectsGetParams(). - WithID(unclassifiedSweet). - WithInclude(ptString("classification")), nil) - - require.Nil(t, err) - schema, ok := res.Payload.Properties.(map[string]interface{}) - require.True(t, ok) - - expectedRefTarget := fmt.Sprintf("weaviate://localhost/RecipeType/%s", - recipeTypeSweet) - ref := schema["ofType"].([]interface{})[0].(map[string]interface{}) - assert.Equal(t, ref["beacon"].(string), expectedRefTarget) - - verifyMetaDistances(t, ref) - }) -} - -func verifyMetaDistances(t *testing.T, ref map[string]interface{}) { - classification, ok := ref["classification"].(map[string]interface{}) - require.True(t, ok) - - assert.Equal(t, json.Number("3"), classification["winningCount"]) - assert.Equal(t, json.Number("2"), classification["losingCount"]) - assert.Equal(t, json.Number("5"), classification["overallCount"]) - - closestWinning, err := classification["closestWinningDistance"].(json.Number).Float64() - require.Nil(t, err) - closestLosing, err := classification["closestLosingDistance"].(json.Number).Float64() - require.Nil(t, err) - closestOverall, err := classification["closestOverallDistance"].(json.Number).Float64() - require.Nil(t, err) - meanWinning, err := classification["meanWinningDistance"].(json.Number).Float64() - require.Nil(t, err) - meanLosing, err := classification["meanLosingDistance"].(json.Number).Float64() - require.Nil(t, err) - - assert.True(t, closestWinning == closestOverall, "closestWinning == closestOverall") - assert.True(t, closestWinning < meanWinning, "closestWinning < meanWinning") - assert.True(t, closestWinning < closestLosing, "closestWinning < closestLosing") - assert.True(t, closestLosing < meanLosing, "closestLosing < meanLosing") -} diff --git a/test/acceptance/classifications/setup_test.go b/test/acceptance/classifications/setup_test.go deleted file mode 100644 index ec1450149353b76a7fc54b816cc3ae691eafd06d..0000000000000000000000000000000000000000 --- a/test/acceptance/classifications/setup_test.go +++ /dev/null @@ -1,410 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "fmt" - "testing" - - "github.com/go-openapi/strfmt" - "github.com/weaviate/weaviate/client/objects" - clschema "github.com/weaviate/weaviate/client/schema" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/test/helper" - testhelper "github.com/weaviate/weaviate/test/helper" -) - -var ( - // contextual - article1 strfmt.UUID = "dcbe5df8-af01-46f1-b45f-bcc9a7a0773d" // apple macbook - article2 strfmt.UUID = "6a8c7b62-fd45-488f-b884-ec87227f6eb3" // ice cream and steak - article3 strfmt.UUID = "92f05097-6371-499c-a0fe-3e60ae16fe3d" // president of the us - - // knn - recipeTypeSavory strfmt.UUID = "989d792c-b59e-4430-80a3-cf7f320f31b0" - recipeTypeSweet strfmt.UUID = "c9dfda02-6b05-4117-9d95-a188342cca48" - unclassifiedSavory strfmt.UUID = "953c03f8-d61e-44c0-bbf1-2afe0dc1ce87" - unclassifiedSweet strfmt.UUID = "04603002-cb66-4fce-bf6d-56bdf9b0b5d4" - - // zeroshot - foodTypeMeat strfmt.UUID = "998d792c-b59e-4430-80a3-cf7f320f31b0" - foodTypeIceCream strfmt.UUID = "998d792c-b59e-4430-80a3-cf7f320f31b1" - unclassifiedSteak strfmt.UUID = "953c03f8-d61e-44c0-bbf1-2afe0dc1ce10" - unclassifiedIceCreams strfmt.UUID = "953c03f8-d61e-44c0-bbf1-2afe0dc1ce11" -) - -func Test_Classifications(t *testing.T) { - t.Run("article/category setup for contextual classification", setupArticleCategory) - t.Run("recipe setup for knn classification", setupRecipe) - t.Run("food types and recipes setup for zeroshot classification", setupFoodTypes) - - // tests - t.Run("contextual classification", contextualClassification) - t.Run("knn classification", knnClassification) - t.Run("zeroshot classification", zeroshotClassification) - - // tear down - deleteObjectClass(t, "Article") - deleteObjectClass(t, "Category") - deleteObjectClass(t, "Recipe") - deleteObjectClass(t, "RecipeType") - deleteObjectClass(t, "FoodType") - deleteObjectClass(t, "Recipes") -} - -func setupArticleCategory(t *testing.T) { - t.Run("schema setup", func(t *testing.T) { - createObjectClass(t, &models.Class{ - Class: "Category", - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "vectorizeClassName": true, - }, - }, - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - }) - createObjectClass(t, &models.Class{ - Class: "Article", - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "vectorizeClassName": true, - }, - }, - Properties: []*models.Property{ - { - Name: "content", - DataType: []string{"text"}, - }, - { - Name: "OfCategory", - DataType: []string{"Category"}, - }, - }, - }) - }) - - t.Run("object setup - categories", func(t *testing.T) { - createObject(t, &models.Object{ - Class: "Category", - Properties: map[string]interface{}{ - "name": "Food and Drink", - }, - }) - createObject(t, &models.Object{ - Class: "Category", - Properties: map[string]interface{}{ - "name": "Computers and Technology", - }, - }) - createObject(t, &models.Object{ - Class: "Category", - Properties: map[string]interface{}{ - "name": "Politics", - }, - }) - }) - - t.Run("object setup - articles", func(t *testing.T) { - createObject(t, &models.Object{ - ID: article1, - Class: "Article", - Properties: map[string]interface{}{ - "content": "The new Apple Macbook 16 inch provides great performance", - }, - }) - createObject(t, &models.Object{ - ID: article2, - Class: "Article", - Properties: map[string]interface{}{ - "content": "I love eating ice cream with my t-bone steak", - }, - }) - createObject(t, &models.Object{ - ID: article3, - Class: "Article", - Properties: map[string]interface{}{ - "content": "Barack Obama was the 44th president of the united states", - }, - }) - }) - - assertGetObjectEventually(t, "92f05097-6371-499c-a0fe-3e60ae16fe3d") -} - -func setupRecipe(t *testing.T) { - t.Run("schema setup", func(t *testing.T) { - createObjectClass(t, &models.Class{ - Class: "RecipeType", - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "vectorizeClassName": true, - }, - }, - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - }) - createObjectClass(t, &models.Class{ - Class: "Recipe", - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "vectorizeClassName": true, - }, - }, - Properties: []*models.Property{ - { - Name: "content", - DataType: []string{"text"}, - }, - { - Name: "OfType", - DataType: []string{"RecipeType"}, - }, - }, - }) - }) - - t.Run("object setup - recipe types", func(t *testing.T) { - createObject(t, &models.Object{ - Class: "RecipeType", - ID: recipeTypeSavory, - Properties: map[string]interface{}{ - "name": "Savory", - }, - }) - - createObject(t, &models.Object{ - Class: "RecipeType", - ID: recipeTypeSweet, - Properties: map[string]interface{}{ - "name": "Sweet", - }, - }) - }) - - t.Run("object setup - articles", func(t *testing.T) { - createObject(t, &models.Object{ - Class: "Recipe", - Properties: map[string]interface{}{ - "content": "Mix two eggs with milk and 7 grams of sugar, bake in the oven at 200 degrees", - "ofType": []interface{}{ - map[string]interface{}{ - "beacon": fmt.Sprintf("weaviate://localhost/%s", recipeTypeSweet), - }, - }, - }, - }) - - createObject(t, &models.Object{ - Class: "Recipe", - Properties: map[string]interface{}{ - "content": "Sautee the apples with sugar and add a dash of milk.", - "ofType": []interface{}{ - map[string]interface{}{ - "beacon": fmt.Sprintf("weaviate://localhost/%s", recipeTypeSweet), - }, - }, - }, - }) - - createObject(t, &models.Object{ - Class: "Recipe", - Properties: map[string]interface{}{ - "content": "Mix butter, cream and sugar. Make eggwhites fluffy and mix with the batter", - "ofType": []interface{}{ - map[string]interface{}{ - "beacon": fmt.Sprintf("weaviate://localhost/%s", recipeTypeSweet), - }, - }, - }, - }) - - createObject(t, &models.Object{ - Class: "Recipe", - Properties: map[string]interface{}{ - "content": "Fry the steak in the pan, then sautee the onions in the same pan", - "ofType": []interface{}{ - map[string]interface{}{ - "beacon": fmt.Sprintf("weaviate://localhost/%s", recipeTypeSavory), - }, - }, - }, - }) - - createObject(t, &models.Object{ - Class: "Recipe", - Properties: map[string]interface{}{ - "content": "Cut the potatoes in half and add salt and pepper. Serve with the meat.", - "ofType": []interface{}{ - map[string]interface{}{ - "beacon": fmt.Sprintf("weaviate://localhost/%s", recipeTypeSavory), - }, - }, - }, - }) - - createObject(t, &models.Object{ - Class: "Recipe", - Properties: map[string]interface{}{ - "content": "Put the pasta and sauce mix in the oven, top with plenty of cheese", - "ofType": []interface{}{ - map[string]interface{}{ - "beacon": fmt.Sprintf("weaviate://localhost/%s", recipeTypeSavory), - }, - }, - }, - }) - - createObject(t, &models.Object{ - ID: unclassifiedSavory, - Class: "Recipe", - Properties: map[string]interface{}{ - "content": "Serve the steak with fries and ketchup.", - }, - }) - - createObject(t, &models.Object{ - ID: unclassifiedSweet, - Class: "Recipe", - Properties: map[string]interface{}{ - "content": "Whisk the cream, add sugar and serve with strawberries", - }, - }) - }) - - assertGetObjectEventually(t, unclassifiedSweet) -} - -func setupFoodTypes(t *testing.T) { - t.Run("schema setup", func(t *testing.T) { - createObjectClass(t, &models.Class{ - Class: "FoodType", - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "vectorizeClassName": true, - }, - }, - Properties: []*models.Property{ - { - Name: "text", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - }) - createObjectClass(t, &models.Class{ - Class: "Recipes", - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "vectorizeClassName": true, - }, - }, - Properties: []*models.Property{ - { - Name: "text", - DataType: []string{"text"}, - }, - { - Name: "ofFoodType", - DataType: []string{"FoodType"}, - }, - }, - }) - }) - - t.Run("object setup - food types", func(t *testing.T) { - createObject(t, &models.Object{ - Class: "FoodType", - ID: foodTypeIceCream, - Properties: map[string]interface{}{ - "text": "Ice cream", - }, - }) - - createObject(t, &models.Object{ - Class: "FoodType", - ID: foodTypeMeat, - Properties: map[string]interface{}{ - "text": "Meat", - }, - }) - }) - - t.Run("object setup - recipes", func(t *testing.T) { - createObject(t, &models.Object{ - Class: "Recipes", - ID: unclassifiedSteak, - Properties: map[string]interface{}{ - "text": "Cut the steak in half and put it into pan", - }, - }) - - createObject(t, &models.Object{ - Class: "Recipes", - ID: unclassifiedIceCreams, - Properties: map[string]interface{}{ - "text": "There are flavors of vanilla, chocolate and strawberry", - }, - }) - }) -} - -func createObjectClass(t *testing.T, class *models.Class) { - params := clschema.NewSchemaObjectsCreateParams().WithObjectClass(class) - resp, err := helper.Client(t).Schema.SchemaObjectsCreate(params, nil) - helper.AssertRequestOk(t, resp, err, nil) -} - -func createObject(t *testing.T, object *models.Object) { - params := objects.NewObjectsCreateParams().WithBody(object) - resp, err := helper.Client(t).Objects.ObjectsCreate(params, nil) - helper.AssertRequestOk(t, resp, err, nil) -} - -func deleteObjectClass(t *testing.T, class string) { - delParams := clschema.NewSchemaObjectsDeleteParams().WithClassName(class) - delRes, err := helper.Client(t).Schema.SchemaObjectsDelete(delParams, nil) - helper.AssertRequestOk(t, delRes, err, nil) -} - -func assertGetObjectEventually(t *testing.T, uuid strfmt.UUID) *models.Object { - var ( - resp *objects.ObjectsGetOK - err error - ) - - checkThunk := func() interface{} { - resp, err = helper.Client(t).Objects.ObjectsGet(objects.NewObjectsGetParams().WithID(uuid), nil) - return err == nil - } - - testhelper.AssertEventuallyEqual(t, true, checkThunk) - - var object *models.Object - - helper.AssertRequestOk(t, resp, err, func() { - object = resp.Payload - }) - - return object -} diff --git a/test/acceptance/classifications/zeroshot_classification_test.go b/test/acceptance/classifications/zeroshot_classification_test.go deleted file mode 100644 index 1f9e572c9a2446f6b81ce5c24edcbf640a53fce4..0000000000000000000000000000000000000000 --- a/test/acceptance/classifications/zeroshot_classification_test.go +++ /dev/null @@ -1,100 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "fmt" - "testing" - "time" - - "github.com/go-openapi/strfmt" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/client/classifications" - "github.com/weaviate/weaviate/client/objects" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/test/helper" - testhelper "github.com/weaviate/weaviate/test/helper" -) - -func zeroshotClassification(t *testing.T) { - var id strfmt.UUID - - t.Run("start the classification and wait for completion", func(t *testing.T) { - res, err := helper.Client(t).Classifications.ClassificationsPost( - classifications.NewClassificationsPostParams().WithParams(&models.Classification{ - Class: "Recipes", - ClassifyProperties: []string{"ofFoodType"}, - BasedOnProperties: []string{"text"}, - Type: "zeroshot", - }), nil) - require.Nil(t, err) - id = res.Payload.ID - - // wait for classification to be completed - testhelper.AssertEventuallyEqualWithFrequencyAndTimeout(t, "completed", - func() interface{} { - res, err := helper.Client(t).Classifications.ClassificationsGet( - classifications.NewClassificationsGetParams().WithID(id.String()), nil) - - require.Nil(t, err) - return res.Payload.Status - }, 100*time.Millisecond, 15*time.Second) - }) - - t.Run("assure changes present", func(t *testing.T) { - // wait for latest changes to be indexed / wait for consistency - testhelper.AssertEventuallyEqual(t, true, func() interface{} { - res, err := helper.Client(t).Objects.ObjectsGet(objects.NewObjectsGetParams(). - WithID(unclassifiedSteak), nil) - require.Nil(t, err) - return res.Payload.Properties.(map[string]interface{})["ofFoodType"] != nil - }) - testhelper.AssertEventuallyEqual(t, true, func() interface{} { - res, err := helper.Client(t).Objects.ObjectsGet(objects.NewObjectsGetParams(). - WithID(unclassifiedIceCreams), nil) - require.Nil(t, err) - return res.Payload.Properties.(map[string]interface{})["ofFoodType"] != nil - }) - }) - - t.Run("assure proper classification present", func(t *testing.T) { - // wait for latest changes to be indexed / wait for consistency - testhelper.AssertEventuallyEqual(t, true, func() interface{} { - res, err := helper.Client(t).Objects.ObjectsGet(objects.NewObjectsGetParams(). - WithID(unclassifiedSteak), nil) - require.Nil(t, err) - return checkOfFoodTypeRef(res.Payload.Properties, foodTypeMeat) - }) - testhelper.AssertEventuallyEqual(t, true, func() interface{} { - res, err := helper.Client(t).Objects.ObjectsGet(objects.NewObjectsGetParams(). - WithID(unclassifiedIceCreams), nil) - require.Nil(t, err) - return checkOfFoodTypeRef(res.Payload.Properties, foodTypeIceCream) - }) - }) -} - -func checkOfFoodTypeRef(properties interface{}, id strfmt.UUID) bool { - ofFoodType, ok := properties.(map[string]interface{})["ofFoodType"].([]interface{}) - if !ok || len(ofFoodType) == 0 { - return false - } - ofFoodTypeMap, ok := ofFoodType[0].(map[string]interface{}) - if !ok { - return false - } - beacon, ok := ofFoodTypeMap["beacon"] - if !ok { - return false - } - return beacon == fmt.Sprintf("weaviate://localhost/FoodType/%s", id) -} diff --git a/test/acceptance/cluster_api_auth/cluster_api_auth_test.go b/test/acceptance/cluster_api_auth/cluster_api_auth_test.go deleted file mode 100644 index 5cd884e6a2f31582280d4b2c507db741c82aa677..0000000000000000000000000000000000000000 --- a/test/acceptance/cluster_api_auth/cluster_api_auth_test.go +++ /dev/null @@ -1,87 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "context" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/client/nodes" - "github.com/weaviate/weaviate/test/docker" - "github.com/weaviate/weaviate/test/helper" - graphqlhelper "github.com/weaviate/weaviate/test/helper/graphql" - "github.com/weaviate/weaviate/test/helper/sample-schema/books" -) - -func TestClusterAPIAuth(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) - defer cancel() - - compose, err := docker.New(). - WithWeaviateClusterWithBasicAuth("user", "pass"). - WithText2VecContextionary(). - Start(ctx) - require.Nil(t, err) - defer func() { - if err := compose.Terminate(ctx); err != nil { - t.Fatalf("failed to terminate test containers: %v", err) - } - }() - - helper.SetupClient(compose.GetWeaviate().URI()) - - t.Run("sanity checks", func(t *testing.T) { - t.Run("check nodes", func(t *testing.T) { - resp, err := helper.Client(t).Nodes.NodesGet(nodes.NewNodesGetParams(), nil) - require.Nil(t, err) - - nodeStatusResp := resp.GetPayload() - require.NotNil(t, nodeStatusResp) - - nodes := nodeStatusResp.Nodes - require.NotNil(t, nodes) - require.Len(t, nodes, 2) - }) - - booksClass := books.ClassContextionaryVectorizer() - helper.CreateClass(t, booksClass) - defer helper.DeleteClass(t, booksClass.Class) - - t.Run("import data", func(t *testing.T) { - helper.CreateObjectsBatch(t, books.Objects()) - }) - - t.Run("nearText query", func(t *testing.T) { - query := ` - { - Get { - Books( - nearText: { - concepts: ["Frank Herbert"] - } - ){ - title - } - } - }` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - books := result.Get("Get", "Books").AsSlice() - require.True(t, len(books) > 0) - results, ok := books[0].(map[string]interface{}) - require.True(t, ok) - assert.True(t, results["title"] != nil) - }) - }) -} diff --git a/test/acceptance/fixtures/actions.json b/test/acceptance/fixtures/actions.json deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/test/acceptance/fixtures/things.json b/test/acceptance/fixtures/things.json deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/test/acceptance/graphql_resolvers/aggregate_response_assert.go b/test/acceptance/graphql_resolvers/aggregate_response_assert.go deleted file mode 100644 index 03ed47aa7b71723f795f6fd23c9bed79e4a36d63..0000000000000000000000000000000000000000 --- a/test/acceptance/graphql_resolvers/aggregate_response_assert.go +++ /dev/null @@ -1,448 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "encoding/json" - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/entities/schema" -) - -const delta = 0.00001 - -type assertFunc func(response map[string]interface{}) bool - -type aggregateResponseAssert struct { - t *testing.T - assert *assert.Assertions -} - -func newAggregateResponseAssert(t *testing.T) *aggregateResponseAssert { - return &aggregateResponseAssert{t, assert.New(t)} -} - -func (a *aggregateResponseAssert) meta(count int64) assertFunc { - return func(response map[string]interface{}) bool { - metaKey := "meta" - if !a.assert.Contains(response, metaKey) { - return false - } - return a.hasInt(response[metaKey].(map[string]interface{}), metaKey, "count", count) - } -} - -func (a *aggregateResponseAssert) groupedBy(value string, path ...interface{}) assertFunc { - return func(response map[string]interface{}) bool { - groupedByKey := "groupedBy" - if !a.assert.Contains(response, groupedByKey) { - return false - } - aggMap := response[groupedByKey].(map[string]interface{}) - return combinedAssert( - a.hasString(aggMap, groupedByKey, "value", value), - a.hasArray(aggMap, groupedByKey, "path", path), - ) - } -} - -func (a *aggregateResponseAssert) pointingTo(propName string, path ...interface{}) assertFunc { - return func(response map[string]interface{}) bool { - if !a.assert.Contains(response, propName) { - return false - } - aggMap := response[propName].(map[string]interface{}) - return a.hasArray(aggMap, propName, "pointingTo", path) - } -} - -func (a *aggregateResponseAssert) typedBoolean(dataType schema.DataType, propName string, - count, totalFalse, totalTrue int64, - percentageFalse, percentageTrue float64, -) assertFunc { - return func(response map[string]interface{}) bool { - if !a.assert.Contains(response, propName) { - return false - } - aggMap := response[propName].(map[string]interface{}) - return combinedAssert( - a.hasInt(aggMap, propName, "count", count), - a.hasInt(aggMap, propName, "totalFalse", totalFalse), - a.hasInt(aggMap, propName, "totalTrue", totalTrue), - a.hasNumber(aggMap, propName, "percentageFalse", percentageFalse), - a.hasNumber(aggMap, propName, "percentageTrue", percentageTrue), - a.hasString(aggMap, propName, "type", string(dataType)), - ) - } -} - -func (a *aggregateResponseAssert) booleanArray(propName string, - count, totalFalse, totalTrue int64, - percentageFalse, percentageTrue float64, -) assertFunc { - return a.typedBoolean(schema.DataTypeBooleanArray, propName, count, totalFalse, totalTrue, - percentageFalse, percentageTrue) -} - -func (a *aggregateResponseAssert) boolean(propName string, - count, totalFalse, totalTrue int64, - percentageFalse, percentageTrue float64, -) assertFunc { - return a.typedBoolean(schema.DataTypeBoolean, propName, count, totalFalse, totalTrue, - percentageFalse, percentageTrue) -} - -func (a *aggregateResponseAssert) typedBoolean0(dataType schema.DataType, propName string) assertFunc { - return func(response map[string]interface{}) bool { - if !a.assert.Contains(response, propName) { - return false - } - aggMap := response[propName].(map[string]interface{}) - return combinedAssert( - a.hasInt(aggMap, propName, "count", 0), - a.hasInt(aggMap, propName, "totalFalse", 0), - a.hasInt(aggMap, propName, "totalTrue", 0), - a.hasNil(aggMap, propName, "percentageFalse"), - a.hasNil(aggMap, propName, "percentageTrue"), - a.hasString(aggMap, propName, "type", string(dataType)), - ) - } -} - -func (a *aggregateResponseAssert) booleanArray0(propName string) assertFunc { - return a.typedBoolean0(schema.DataTypeBooleanArray, propName) -} - -func (a *aggregateResponseAssert) boolean0(propName string) assertFunc { - return a.typedBoolean0(schema.DataTypeBoolean, propName) -} - -func (a *aggregateResponseAssert) typedInts(dataType schema.DataType, propName string, - count, maximum, minimum, mode, sum int64, - median, mean float64, -) assertFunc { - return func(response map[string]interface{}) bool { - if !a.assert.Contains(response, propName) { - return false - } - aggMap := response[propName].(map[string]interface{}) - return combinedAssert( - a.hasInt(aggMap, propName, "count", count), - a.hasInt(aggMap, propName, "maximum", maximum), - a.hasInt(aggMap, propName, "minimum", minimum), - a.hasInt(aggMap, propName, "mode", mode), - a.hasInt(aggMap, propName, "sum", sum), - a.hasNumber(aggMap, propName, "median", median), - a.hasNumber(aggMap, propName, "mean", mean), - a.hasString(aggMap, propName, "type", string(dataType)), - ) - } -} - -func (a *aggregateResponseAssert) intArray(propName string, - count, maximum, minimum, mode, sum int64, - median, mean float64, -) assertFunc { - return a.typedInts(schema.DataTypeIntArray, propName, count, maximum, minimum, - mode, sum, median, mean) -} - -func (a *aggregateResponseAssert) int(propName string, - count, maximum, minimum, mode, sum int64, - median, mean float64, -) assertFunc { - return a.typedInts(schema.DataTypeInt, propName, count, maximum, minimum, - mode, sum, median, mean) -} - -func (a *aggregateResponseAssert) typedInts0(dataType schema.DataType, propName string) assertFunc { - return func(response map[string]interface{}) bool { - if !a.assert.Contains(response, propName) { - return false - } - aggMap := response[propName].(map[string]interface{}) - return combinedAssert( - a.hasInt(aggMap, propName, "count", 0), - a.hasNil(aggMap, propName, "maximum"), - a.hasNil(aggMap, propName, "minimum"), - a.hasNil(aggMap, propName, "mode"), - a.hasNil(aggMap, propName, "sum"), - a.hasNil(aggMap, propName, "median"), - a.hasNil(aggMap, propName, "mean"), - a.hasString(aggMap, propName, "type", string(dataType)), - ) - } -} - -func (a *aggregateResponseAssert) intArray0(propName string) assertFunc { - return a.typedInts0(schema.DataTypeIntArray, propName) -} - -func (a *aggregateResponseAssert) int0(propName string) assertFunc { - return a.typedInts0(schema.DataTypeInt, propName) -} - -func (a *aggregateResponseAssert) typedNumbers(dataType schema.DataType, propName string, - count int64, - maximum, minimum, mode, sum, median, mean float64, -) assertFunc { - return func(response map[string]interface{}) bool { - if !a.assert.Contains(response, propName) { - return false - } - aggMap := response[propName].(map[string]interface{}) - return combinedAssert( - a.hasInt(aggMap, propName, "count", count), - a.hasNumber(aggMap, propName, "maximum", maximum), - a.hasNumber(aggMap, propName, "minimum", minimum), - a.hasNumber(aggMap, propName, "mode", mode), - a.hasNumber(aggMap, propName, "sum", sum), - a.hasNumber(aggMap, propName, "median", median), - a.hasNumber(aggMap, propName, "mean", mean), - a.hasString(aggMap, propName, "type", string(dataType)), - ) - } -} - -func (a *aggregateResponseAssert) numberArray(propName string, - count int64, - maximum, minimum, mode, sum, median, mean float64, -) assertFunc { - return a.typedNumbers(schema.DataTypeNumberArray, propName, count, maximum, minimum, - mode, sum, median, mean) -} - -func (a *aggregateResponseAssert) number(propName string, - count int64, - maximum, minimum, mode, sum, median, mean float64, -) assertFunc { - return a.typedNumbers(schema.DataTypeNumber, propName, count, maximum, minimum, - mode, sum, median, mean) -} - -func (a *aggregateResponseAssert) typedNumbers0(dataType schema.DataType, propName string) assertFunc { - return func(response map[string]interface{}) bool { - if !a.assert.Contains(response, propName) { - return false - } - aggMap := response[propName].(map[string]interface{}) - return combinedAssert( - a.hasInt(aggMap, propName, "count", 0), - a.hasNil(aggMap, propName, "maximum"), - a.hasNil(aggMap, propName, "minimum"), - a.hasNil(aggMap, propName, "mode"), - a.hasNil(aggMap, propName, "sum"), - a.hasNil(aggMap, propName, "median"), - a.hasNil(aggMap, propName, "mean"), - a.hasString(aggMap, propName, "type", string(dataType)), - ) - } -} - -func (a *aggregateResponseAssert) numberArray0(propName string) assertFunc { - return a.typedNumbers0(schema.DataTypeNumberArray, propName) -} - -func (a *aggregateResponseAssert) number0(propName string) assertFunc { - return a.typedNumbers0(schema.DataTypeNumber, propName) -} - -func (a *aggregateResponseAssert) dateArray(propName string, count int64) assertFunc { - return a.date(propName, count) -} - -func (a *aggregateResponseAssert) date(propName string, count int64) assertFunc { - return func(response map[string]interface{}) bool { - if !a.assert.Contains(response, propName) { - return false - } - return a.hasInt(response[propName].(map[string]interface{}), propName, "count", count) - } -} - -func (a *aggregateResponseAssert) dateArray0(propName string) assertFunc { - return a.date(propName, 0) -} - -func (a *aggregateResponseAssert) date0(propName string) assertFunc { - return a.date(propName, 0) -} - -func (a *aggregateResponseAssert) typedStrings(dataType schema.DataType, propName string, - count int64, - values []string, occurrences []int64, -) assertFunc { - return func(response map[string]interface{}) bool { - if !a.assert.Contains(response, propName) { - return false - } - aggMap := response[propName].(map[string]interface{}) - return combinedAssert( - a.hasInt(aggMap, propName, "count", count), - a.hasString(aggMap, propName, "type", string(dataType)), - a.hasOccurrences(aggMap, propName, values, occurrences), - ) - } -} - -func (a *aggregateResponseAssert) typedStrings0(dataType schema.DataType, propName string) assertFunc { - return func(response map[string]interface{}) bool { - if !a.assert.Contains(response, propName) { - return false - } - aggMap := response[propName].(map[string]interface{}) - return combinedAssert( - a.hasInt(aggMap, propName, "count", 0), - a.hasString(aggMap, propName, "type", string(dataType)), - a.hasOccurrences(aggMap, propName, nil, nil), - ) - } -} - -func (a *aggregateResponseAssert) textArray(propName string, - count int64, - values []string, occurrences []int64, -) assertFunc { - return a.typedStrings(schema.DataTypeTextArray, propName, count, values, occurrences) -} - -func (a *aggregateResponseAssert) text(propName string, - count int64, - values []string, occurrences []int64, -) assertFunc { - return a.typedStrings(schema.DataTypeText, propName, count, values, occurrences) -} - -func (a *aggregateResponseAssert) textArray0(propName string) assertFunc { - return a.typedStrings0(schema.DataTypeTextArray, propName) -} - -func (a *aggregateResponseAssert) text0(propName string) assertFunc { - return a.typedStrings0(schema.DataTypeText, propName) -} - -func (a *aggregateResponseAssert) hasOccurrences(parentMap map[string]interface{}, - parentKey string, values []string, occurrences []int64, -) bool { - key := "topOccurrences" - to, exists := parentMap[key] - if !exists { - return a.assert.Fail(fmt.Sprintf("'%s' does not have '%s'\n%#v", parentKey, key, parentMap)) - } - - toArr := to.([]interface{}) - assertResults := make([]bool, len(values)) - for i := range values { - key := fmt.Sprintf("%s.%s[%d]", parentKey, key, i) - toSingle := toArr[i].(map[string]interface{}) - assertResults[i] = combinedAssert( - a.hasString(toSingle, key, "value", values[i]), - a.hasInt(toSingle, key, "occurs", occurrences[i]), - ) - } - return combinedAssert(assertResults...) -} - -func (a *aggregateResponseAssert) hasNumber(parentMap map[string]interface{}, - parentKey, key string, expectedValue float64, -) bool { - v, exist := parentMap[key] - if !exist { - return a.assert.Fail(fmt.Sprintf("'%s' does not have '%s'\n%#v", parentKey, key, parentMap)) - } - if v == nil { - return a.assert.Fail(fmt.Sprintf("'%s.%s' is nil", parentKey, key)) - } - if v, ok := v.(json.Number); ok { - if value, err := v.Float64(); err == nil { - if a.assert.InDelta(expectedValue, value, delta) { - return true - } - return a.assert.Fail(fmt.Sprintf("'%s.%s' of %#v is not equal to %#v", parentKey, key, value, expectedValue)) - } - } - return a.assert.Fail(fmt.Sprintf("'%s.%s' of %#v is not equal to %#v", parentKey, key, v, expectedValue)) -} - -func (a *aggregateResponseAssert) hasInt(parentMap map[string]interface{}, - parentKey, key string, expectedValue int64, -) bool { - v, exist := parentMap[key] - if !exist { - return a.assert.Fail(fmt.Sprintf("'%s' does not have '%s'\n%#v", parentKey, key, parentMap)) - } - if v == nil { - return a.assert.Fail(fmt.Sprintf("'%s.%s' is nil", parentKey, key)) - } - if v, ok := v.(json.Number); ok { - if value, err := v.Int64(); err == nil { - if value == expectedValue { - return true - } - return a.assert.Fail(fmt.Sprintf("'%s.%s' of %#v is not equal to %#v", parentKey, key, value, expectedValue)) - } - } - return a.assert.Fail(fmt.Sprintf("'%s.%s' of %#v is not equal to %#v", parentKey, key, v, expectedValue)) -} - -func (a *aggregateResponseAssert) hasString(parentMap map[string]interface{}, - parentKey, key string, expectedValue string, -) bool { - v, exist := parentMap[key] - if !exist { - return a.assert.Fail(fmt.Sprintf("'%s' does not have '%s'\n%#v", parentKey, key, parentMap)) - } - if v == nil { - return a.assert.Fail(fmt.Sprintf("'%s.%s' is nil", parentKey, key)) - } - if v != expectedValue { - return a.assert.Fail(fmt.Sprintf("'%s.%s' of %#v is not equal to %#v", parentKey, key, v, expectedValue)) - } - return true -} - -func (a *aggregateResponseAssert) hasNil(parentMap map[string]interface{}, - parentKey, key string, -) bool { - v, exist := parentMap[key] - if !exist { - return a.assert.Fail(fmt.Sprintf("'%s' does not have '%s'\n%#v", parentKey, key, parentMap)) - } - if v != nil { - return a.assert.Fail(fmt.Sprintf("'%s.%s' is not nil", parentKey, key)) - } - return true -} - -func (a *aggregateResponseAssert) hasArray(parentMap map[string]interface{}, - parentKey, key string, expectedValue []interface{}, -) bool { - v, exist := parentMap[key] - if !exist { - return a.assert.Fail(fmt.Sprintf("'%s' does not have '%s'\n%#v", parentKey, key, parentMap)) - } - if !a.assert.Equal(expectedValue, v) { - return a.assert.Fail(fmt.Sprintf("'%s.%s' of %#v is not equal to %#v", parentKey, key, v, expectedValue)) - } - return true -} - -func combinedAssert(assertResults ...bool) bool { - for _, assertResult := range assertResults { - if !assertResult { - return false - } - } - return true -} diff --git a/test/acceptance/graphql_resolvers/fixtures/actions_schema.json b/test/acceptance/graphql_resolvers/fixtures/actions_schema.json deleted file mode 100644 index bbeed93ba1a2813e0ca0704f9efb65db57a4a3ca..0000000000000000000000000000000000000000 --- a/test/acceptance/graphql_resolvers/fixtures/actions_schema.json +++ /dev/null @@ -1,99 +0,0 @@ -{ - "version": "0.0.1", - "type": "action", - "name": "weaviate demo actions schema", - "maintainer": "yourfriends@weaviate.com", - "classes": [ - { - "class": "Flight", - "description": "An airline flight", - "properties": [ - { - "name": "aircraft", - "dataType": ["Aircraft"], - "description": "The kind of aircraft" - }, - { - "name": "arrivalAirport", - "dataType": ["Airport"], - "description": "The airport where the flight terminates" - }, - { - "name": "departureAirport", - "dataType": ["Airport"], - "description": "The airport where the flight originates." - }, - { - "name": "flightNumber", - "dataType": ["string"], - "description": "The unique identifier for a flight" - }, - { - "name": "date", - "dataType": ["date"], - "description": "The date the person is moving" - }, - { - "name": "estimatedFlightDuration", - "dataType": ["number"], - "description": "The estimated time in minutes the flight will take" - }, - { - "name": "numberOfPassengers", - "dataType": ["int"], - "description": "Total number of passengers" - }, - { - "name": "isDelayed", - "dataType": ["boolean"], - "description": "True if the flight is delayed" - } - ] - }, - { - "class": "BuyAction", - "description": "An agent buys an object, product, or service from a seller for a price.", - "properties": [ - { - "name": "salesPerson", - "dataType": ["Airline", "Person"], - "description": "The seller of the thing", - "keywords": [ - {"keyword": "sale", "weight": 0.5}, - {"keyword": "person", "weight": 0.5} - ] - }, - { - "name": "price", - "dataType": ["number"], - "description": "The offer price of a product" - }, - { - "name": "recipient", - "dataType": ["Person"], - "description": "The recipient of the good", - "keywords": [ - {"keyword": "buy", "weight": 0.5}, - {"keyword": "person", "weight": 0.5} - ] - } - ] - }, - { - "class": "Event", - "description": "Event is used to replicate bugs and prevent regressions for the fixes mentioned in gh-824", - "properties": [ - { - "name": "name", - "dataType": ["string"], - "description": "The name of the event" - }, - { - "name": "description", - "dataType": ["text"], - "description": "The description of the event" - } - ] - } - ] -} diff --git a/test/acceptance/graphql_resolvers/fixtures/data.json b/test/acceptance/graphql_resolvers/fixtures/data.json deleted file mode 100644 index 78e4569a1b62de6dca7f52af73a4012ed6670ca1..0000000000000000000000000000000000000000 --- a/test/acceptance/graphql_resolvers/fixtures/data.json +++ /dev/null @@ -1,495 +0,0 @@ -{ - "Objects": [ - { - "class": "City", - "uuid": "001", - "name": "Amsterdam", - "population": 1800000, - "isCapital": true, - "inCountry": [ - { - "class": "Country", - "uuid": "005", - "name": "Netherlands", - "population": 17000000 - } - ], - "reviews": "Great! A perfect city", - "location": { - "latitude": 52.366667, - "longitude": 4.9 - } - }, - { - "class": "City", - "uuid": "002", - "name": "Rotterdam", - "population": 1800000, - "isCapital": false, - "inCountry": [ - { - "class": "Country", - "uuid": "005", - "name": "Netherlands", - "population": 17000000 - } - ] - }, - { - "class": "City", - "uuid": "003", - "name": "Berlin", - "population": 3470000, - "isCapital": true, - "reviews": "I had a bad time here", - "inCountry": [ - { - "class": "Country", - "uuid": "006", - "name": "Germany", - "population": 83000000 - } - ] - }, - { - "class": "City", - "uuid": "004", - "name": "Dusseldorf", - "population": 600000, - "isCapital": false, - "inCountry": [ - { - "class": "Country", - "uuid": "006", - "name": "Germany", - "population": 83000000 - } - ], - "location": { - "latitude": 51.225556, - "longitude": 6.782778 - } - }, - { - "class": "Country", - "uuid": "005", - "name": "Netherlands", - "population": 17000000 - }, - { - "class": "Country", - "uuid": "006", - "name": "Germany", - "population": 83000000 - }, - { - "class": "Person", - "uuid": "007", - "livesIn": [ - { - "class": "City", - "uuid": "004", - "name": "Dusseldorf", - "population": 600000, - "isCapital": false, - "inCountry": [ - { - "class": "Country", - "uuid": "006", - "name": "Germany", - "population": 83000000 - } - ] - } - ], - "ofNation": [ - { - "class": "Country", - "uuid": "006", - "name": "Germany", - "population": 83000000 - } - ] - }, - { - "class": "Person", - "uuid": "008", - "livesIn": [ - { - "class": "City", - "uuid": "001", - "name": "Amsterdam", - "population": 1800000, - "isCapital": true, - "inCountry": { - "class": "Country", - "uuid": "005", - "name": "Netherlands", - "population": 17000000 - } - } - ], - "ofNation": [ - { - "class": "Country", - "uuid": "005", - "name": "Netherlands", - "population": 17000000 - } - ] - }, - { - "class": "Airport", - "uuid": "009", - "inCity": [ - { - "class": "City", - "uuid": "001", - "name": "Amsterdam", - "population": 1800000, - "isCapital": true, - "inCountry": [ - { - "class": "Country", - "uuid": "005", - "name": "Netherlands", - "population": 17000000 - } - ] - } - ], - "code": "10000" - }, - { - "class": "Airport", - "uuid": "010", - "inCity": [ - { - "class": "City", - "uuid": "002", - "name": "Rotterdam", - "population": 1800000, - "isCapital": false, - "inCountry": { - "class": "Country", - "uuid": "005", - "name": "Netherlands", - "population": 17000000 - } - } - ], - "code": "20000" - }, - { - "class": "Airport", - "uuid": "011", - "inCity": [ - { - "class": "City", - "uuid": "004", - "name": "Dusseldorf", - "population": 600000, - "isCapital": false, - "inCountry": { - "class": "Country", - "uuid": "006", - "name": "Germany", - "population": 83000000 - } - } - ], - "code": "30000" - }, - { - "class": "Airport", - "uuid": "012", - "inCity": [ - { - "class": "City", - "uuid": "003", - "name": "Berlin", - "population": 3470000, - "isCapital": true, - "inCountry": { - "class": "Country", - "uuid": "006", - "name": "Germany", - "population": 83000000 - } - } - ], - "code": "40000" - }, - { - "class": "Airline", - "uuid": "013", - "name": "Lufthansa", - "basedIn": [ - { - "class": "City", - "uuid": "003", - "name": "Berlin", - "population": 3470000, - "isCapital": true, - "inCountry": { - "class": "Country", - "uuid": "006", - "name": "Germany", - "population": 83000000 - } - } - ], - "code": "123" - }, - { - "class": "Airline", - "uuid": "014", - "name": "KLM", - "basedIn": [ - { - "class": "City", - "uuid": "001", - "name": "Amsterdam", - "population": 1800000, - "isCapital": true, - "inCountry": { - "class": "Country", - "uuid": "005", - "name": "Netherlands", - "population": 17000000 - } - } - ], - "code": "321" - }, - { - "class": "Aircraft", - "uuid": "015", - "name": "Boeing 747", - "numberOfSeats": 366 - }, - { - "class": "Aircraft", - "uuid": "016", - "name": "Airbus A380", - "numberOfSeats": 853 - }, - { - "class": "Plane", - "uuid": "017", - "code": "AB123", - "aircraft": [ - { - "class": "Aircraft", - "uuid": "015", - "name": "Boeing 747", - "numberOfSeats": 366 - } - ], - "ofAirline": [ - { - "class": "Airline", - "uuid": "013", - "name": "Lufthansa", - "basedIn": { - "class": "City", - "uuid": "003", - "name": "Berlin", - "population": 3470000, - "isCapital": true, - "inCountry": { - "class": "Country", - "uuid": "006", - "name": "Germany", - "population": 83000000 - } - }, - "code": "123" - } - ] - }, - { - "class": "Plane", - "uuid": "018", - "code": "AB456", - "aircraft": [ - { - "class": "Aircraft", - "uuid": "016", - "name": "Airbus A380", - "numberOfSeats": 853 - } - ], - "ofAirline": [ - { - "class": "Airline", - "uuid": "014", - "name": "KLM", - "basedIn": { - "class": "City", - "uuid": "001", - "name": "Amsterdam", - "population": 1800000, - "isCapital": true, - "inCountry": { - "class": "Country", - "uuid": "005", - "name": "Netherlands", - "population": 17000000 - } - }, - "code": "321" - } - ] - }, - { - "class": "Plane", - "uuid": "019", - "code": "AB457", - "aircraft": [ - { - "class": "Aircraft", - "uuid": "015", - "name": "Boeing 747", - "numberOfSeats": 366 - } - ], - "ofAirline": [ - { - "class": "Airline", - "uuid": "014", - "name": "KLM", - "basedIn": [ - { - "class": "City", - "uuid": "001", - "name": "Amsterdam", - "population": 1800000, - "isCapital": true, - "inCountry": [ - { - "class": "Country", - "uuid": "005", - "name": "Netherlands", - "population": 17000000 - } - ] - } - ], - "code": "321" - } - ] - }, - { - "class": "Plane", - "uuid": "020", - "code": "AB456", - "aircraft": [ - { - "class": "Aircraft", - "uuid": "016", - "name": "Airbus A380", - "numberOfSeats": 853 - } - ], - "ofAirline": [ - { - "class": "Airline", - "uuid": "013", - "name": "Lufthansa", - "basedIn": [ - { - "class": "City", - "uuid": "003", - "name": "Berlin", - "population": 3470000, - "isCapital": true, - "inCountry": [ - { - "class": "Country", - "uuid": "006", - "name": "Germany", - "population": 83000000 - } - ] - } - ], - "code": "123" - } - ] - } - ], - "Actions": [ - { - "class": "Event", - "uuid": "021", - "name": "A name" - }, - { - "class": "Event", - "uuid": "022", - "description": "A desricption" - }, - { - "class": "Flight", - "uuid": "100", - "aircraft": [ - { - "class": "Aircraft", - "uuid": "016", - "name": "Airbus A380", - "numberOfSeats": 853 - } - ], - "arrivalAirport": [ - { - "class": "Airport", - "uuid": "011", - "inCity": [ - { - "class": "City", - "uuid": "004", - "name": "Dusseldorf", - "population": 600000, - "isCapital": false, - "inCountry": { - "class": "Country", - "uuid": "006", - "name": "Germany", - "population": 83000000 - } - } - ], - "code": "30000" - } - ], - "departureAirport": [ - { - "class": "Airport", - "uuid": "012", - "inCity": [ - { - "class": "City", - "uuid": "003", - "name": "Berlin", - "population": 3470000, - "isCapital": true, - "inCountry": { - "class": "Country", - "uuid": "006", - "name": "Germany", - "population": 83000000 - } - } - ], - "code": "40000" - } - ], - "flightNumber": "LH987", - "date": "2017-08-17T12:47:00+02:00", - "estimatedFlightDuration": 50, - "numberOfPassengers": 300, - "isDelayed": false - } - ] -} diff --git a/test/acceptance/graphql_resolvers/fixtures/things_schema.json b/test/acceptance/graphql_resolvers/fixtures/things_schema.json deleted file mode 100644 index b714a9283de968894b50cfa56314d97df2e5ee90..0000000000000000000000000000000000000000 --- a/test/acceptance/graphql_resolvers/fixtures/things_schema.json +++ /dev/null @@ -1,177 +0,0 @@ -{ - "@context": "", - "version": "0.0.1", - "type": "object", - "name": "weaviate demo objects schema", - "maintainer": "yourfriends@weaviate.com", - "classes": [ - { - "class": "City", - "description": "A city, place or town", - "properties": [ - { - "name": "name", - "dataType": ["string"], - "description": "Official name of the city." - }, - { - "name": "inCountry", - "dataType": ["Country"], - "description": "The country the city lies in" - }, - { - "name": "population", - "dataType": ["int"], - "description": "Number of inhabitants of the city" - }, - { - "name": "isCapital", - "dataType": ["boolean"], - "description": "True if the city is a capital" - }, - { - "name": "reviews", - "dataType": ["string"], - "description": "Visitors opinions about the city" - }, - { - "name": "location", - "dataType": ["geoCoordinates"], - "description": "The city's longitude and latitude", - "keywords": [{"keyword": "location", "weight": 1.0}] - } - ] - }, - { - "class": "Person", - "description": "Person", - "properties": [ - { - "name": "livesIn", - "dataType": ["City"], - "description": "The place where the person lives in." - }, - { - "name": "birthday", - "dataType": ["date"], - "description": "Birthday of the person" - }, - { - "name": "name", - "dataType": ["string"], - "description": "Name of the person" - }, - { - "name": "ofNation", - "dataType": ["Country"], - "description": "Nationality of the person", - "keywords": [ - {"keyword": "nation", "weight": 1.0}, - {"keyword": "country", "weight": 1.0}, - {"keyword": "person", "weight": 1.0} - ] - } - ] - }, - { - "class": "Country", - "description": "Country", - "properties": [ - { - "name": "name", - "dataType": ["string"], - "description": "Official name of the country" - }, - { - "name": "population", - "dataType": ["int"], - "description": "Amount of people living in the country" - } - ] - }, - { - "class": "Airport", - "description": "An airport", - "properties": [ - { - "name": "code", - "dataType": ["string"], - "description": "identifier for an airport" - }, - { - "name": "inCity", - "dataType": ["City"], - "description": "City where the airport lies." - }, - { - "name": "name", - "dataType": ["string"], - "description": "Official name of the airport" - } - ] - }, - { - "class": "Airline", - "description": "An organization that provides flights for passengers", - "properties": [ - { - "name": "code", - "dataType": ["string"], - "description": "identifier for an airport" - }, - { - "name": "basedIn", - "dataType": ["City", "Country"], - "description": "City or country where the airline is based." - }, - { - "name": "name", - "dataType": ["string"], - "description": "Official name of the airline" - }, - { - "name": "hasNumberOfPlanes", - "dataType": ["int"], - "description": "Number of airplanes the airline owns" - } - ] - }, - { - "class": "Plane", - "description": "An airplane", - "properties": [ - { - "name": "code", - "dataType": ["string"], - "description": "identifier for the plane" - }, - { - "name": "aircraft", - "dataType": ["Aircraft"], - "description": "the kind of aircraft" - }, - { - "name": "ofAirline", - "dataType": ["Airline"], - "description": "Airline the plane is owned by" - } - ] - }, - { - "class": "Aircraft", - "description": "The kind of aircraft (e.g., \"Boeing 747\")", - "properties": [ - { - "name": "name", - "dataType": ["string"], - "description": "name of the aircraft" - }, - { - "name": "numberOfSeats", - "dataType": ["number"], - "description": "number of seats available in the aircraft." - } - ] - } - ] -} diff --git a/test/acceptance/graphql_resolvers/json_helper.go b/test/acceptance/graphql_resolvers/json_helper.go deleted file mode 100644 index b948fdcf7ce0116098ef1678850a570f29d8ab6c..0000000000000000000000000000000000000000 --- a/test/acceptance/graphql_resolvers/json_helper.go +++ /dev/null @@ -1,26 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "encoding/json" -) - -func parseJSONSlice(text string) []interface{} { - var result []interface{} - err := json.Unmarshal([]byte(text), &result) - if err != nil { - panic(err) - } - - return result -} diff --git a/test/acceptance/graphql_resolvers/local_aggregate_hybrid_search_test.go b/test/acceptance/graphql_resolvers/local_aggregate_hybrid_search_test.go deleted file mode 100644 index b62c5204c3e9d55e056ad5cb08236a37f4fd7e34..0000000000000000000000000000000000000000 --- a/test/acceptance/graphql_resolvers/local_aggregate_hybrid_search_test.go +++ /dev/null @@ -1,102 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/test/helper" - graphqlhelper "github.com/weaviate/weaviate/test/helper/graphql" -) - -func aggregationWithHybridSearch(t *testing.T) { - t.Run("without search vector", func(t *testing.T) { - query := ` - { - Aggregate { - Company - ( - objectLimit: 3 - hybrid: { - alpha: 0.5 - query: "Apple" - } - ) - { - name { - topOccurrences { - value - } - } - } - } - }` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query).Get("Aggregate", "Company").AsSlice() - require.Len(t, result, 1) - topOccur := result[0].(map[string]interface{})["name"].(map[string]interface{})["topOccurrences"].([]interface{}) - require.Len(t, topOccur, 3) - assert.Contains(t, topOccur, map[string]interface{}{"value": "Apple"}) - assert.Contains(t, topOccur, map[string]interface{}{"value": "Apple Inc."}) - assert.Contains(t, topOccur, map[string]interface{}{"value": "Apple Incorporated"}) - }) - - t.Run("with grouping, sparse search only", func(t *testing.T) { - query := ` - { - Aggregate { - Company - ( - groupBy: "name" - hybrid: { - alpha: 0 - query: "Google" - } - ) - { - name { - topOccurrences { - value - } - } - } - } - }` - - type object = map[string]interface{} - - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query).Get("Aggregate", "Company").AsSlice() - require.Len(t, result, 3) - assert.Contains(t, result, object{ - "name": object{ - "topOccurrences": []interface{}{ - object{"value": "Google"}, - }, - }, - }) - assert.Contains(t, result, object{ - "name": object{ - "topOccurrences": []interface{}{ - object{"value": "Google Inc."}, - }, - }, - }) - assert.Contains(t, result, object{ - "name": object{ - "topOccurrences": []interface{}{ - object{"value": "Google Incorporated"}, - }, - }, - }) - }) -} diff --git a/test/acceptance/graphql_resolvers/local_aggregate_matrix_groupby_test.go b/test/acceptance/graphql_resolvers/local_aggregate_matrix_groupby_test.go deleted file mode 100644 index 52d0ca22b906c73b8b81a5ad19983dd7501aa99a..0000000000000000000000000000000000000000 --- a/test/acceptance/graphql_resolvers/local_aggregate_matrix_groupby_test.go +++ /dev/null @@ -1,2152 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/test/helper" - graphqlhelper "github.com/weaviate/weaviate/test/helper/graphql" -) - -func aggregateArrayClassWithGroupByTest(t *testing.T) { - asserts := newAggregateResponseAssert(t) - testCasesGen := &aggregateArrayClassTestCases{} - - t.Run("aggregate ArrayClass with group by texts", func(t *testing.T) { - expectedAllResultsAssertions := map[string][]assertFunc{ - "Atxt": { - asserts.groupedBy("Atxt", "texts"), - asserts.meta(4), - asserts.booleanArray("booleans", 10, 4, 6, 0.4, 0.6), - asserts.textArray("texts", 10, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{4, 3, 2, 1}), - asserts.numberArray("numbers", 10, 4, 1, 1, 20, 2, 2), - asserts.intArray("ints", 10, 104, 101, 101, 1020, 102, 102), - asserts.dateArray("datesAsStrings", 10), - asserts.dateArray("dates", 10), - }, - "Btxt": { - asserts.groupedBy("Btxt", "texts"), - asserts.meta(3), - asserts.booleanArray("booleans", 9, 3, 6, 0.3333333333333333, 0.6666666666666666), - asserts.textArray("texts", 9, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{3, 3, 2, 1}), - asserts.numberArray("numbers", 9, 4, 1, 1, 19, 2, 2.111111111111111), - asserts.intArray("ints", 9, 104, 101, 101, 919, 102, 102.11111111111111), - asserts.dateArray("datesAsStrings", 9), - asserts.dateArray("dates", 9), - }, - "Ctxt": { - asserts.groupedBy("Ctxt", "texts"), - asserts.meta(2), - asserts.booleanArray("booleans", 7, 2, 5, 0.2857142857142857, 0.7142857142857143), - asserts.textArray("texts", 7, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{2, 2, 2, 1}), - asserts.numberArray("numbers", 7, 4, 1, 1, 16, 2, 2.2857142857142856), - asserts.intArray("ints", 7, 104, 101, 101, 716, 102, 102.28571428571429), - asserts.dateArray("datesAsStrings", 7), - asserts.dateArray("dates", 7), - }, - "Dtxt": { - asserts.groupedBy("Dtxt", "texts"), - asserts.meta(1), - asserts.booleanArray("booleans", 4, 1, 3, 0.25, 0.75), - asserts.textArray("texts", 4, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{1, 1, 1, 1}), - asserts.numberArray("numbers", 4, 4, 1, 1, 10, 2.5, 2.5), - asserts.intArray("ints", 4, 104, 101, 101, 410, 102.5, 102.5), - asserts.dateArray("datesAsStrings", 4), - asserts.dateArray("dates", 4), - }, - } - expectedResultsWithDataAssertions := map[string][]assertFunc{ - "Atxt": { - asserts.groupedBy("Atxt", "texts"), - asserts.meta(2), - asserts.booleanArray("booleans", 7, 2, 5, 0.2857142857142857, 0.7142857142857143), - asserts.textArray("texts", 7, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{2, 2, 2, 1}), - asserts.numberArray("numbers", 7, 4, 1, 1, 16, 2, 2.2857142857142856), - asserts.intArray("ints", 7, 104, 101, 101, 716, 102, 102.28571428571429), - asserts.dateArray("datesAsStrings", 7), - asserts.dateArray("dates", 7), - }, - "Btxt": { - asserts.groupedBy("Btxt", "texts"), - asserts.meta(2), - asserts.booleanArray("booleans", 7, 2, 5, 0.2857142857142857, 0.7142857142857143), - asserts.textArray("texts", 7, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{2, 2, 2, 1}), - asserts.numberArray("numbers", 7, 4, 1, 1, 16, 2, 2.2857142857142856), - asserts.intArray("ints", 7, 104, 101, 101, 716, 102, 102.28571428571429), - asserts.dateArray("datesAsStrings", 7), - asserts.dateArray("dates", 7), - }, - "Ctxt": { - asserts.groupedBy("Ctxt", "texts"), - asserts.meta(2), - asserts.booleanArray("booleans", 7, 2, 5, 0.2857142857142857, 0.7142857142857143), - asserts.textArray("texts", 7, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{2, 2, 2, 1}), - asserts.numberArray("numbers", 7, 4, 1, 1, 16, 2, 2.2857142857142856), - asserts.intArray("ints", 7, 104, 101, 101, 716, 102, 102.28571428571429), - asserts.dateArray("datesAsStrings", 7), - asserts.dateArray("dates", 7), - }, - "Dtxt": { - asserts.groupedBy("Dtxt", "texts"), - asserts.meta(1), - asserts.booleanArray("booleans", 4, 1, 3, 0.25, 0.75), - asserts.textArray("texts", 4, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{1, 1, 1, 1}), - asserts.numberArray("numbers", 4, 4, 1, 1, 10, 2.5, 2.5), - asserts.intArray("ints", 4, 104, 101, 101, 410, 102.5, 102.5), - asserts.dateArray("datesAsStrings", 4), - asserts.dateArray("dates", 4), - }, - } - expectedResultsWithoutDataAssertions := map[string][]assertFunc{} - expectedNoResultsAssertions := map[string][]assertFunc{} - - testCases := []aggregateTestCase{ - testCasesGen.WithoutFilters(expectedAllResultsAssertions), - - testCasesGen.WithWhereFilter_AllResults(expectedAllResultsAssertions), - testCasesGen.WithWhereFilter_ResultsWithData(expectedResultsWithDataAssertions), - testCasesGen.WithWhereFilter_ResultsWithoutData(expectedResultsWithoutDataAssertions), - testCasesGen.WithWhereFilter_NoResults(expectedNoResultsAssertions), - - testCasesGen.WithNearObjectFilter_AllResults(expectedAllResultsAssertions), - testCasesGen.WithNearObjectFilter_ResultsWithData(expectedResultsWithDataAssertions), - testCasesGen.WithNearObjectFilter_ResultsWithoutData(expectedResultsWithoutDataAssertions), - - testCasesGen.WithWhereAndNearObjectFilters_AllResults(expectedAllResultsAssertions), - testCasesGen.WithWhereAndNearObjectFilters_ResultsWithData(expectedResultsWithDataAssertions), - testCasesGen.WithWhereAndNearObjectFilters_ResultsWithoutData(expectedResultsWithoutDataAssertions), - testCasesGen.WithWhereAndNearObjectFilters_NoResults(expectedNoResultsAssertions), - } - - for _, tc := range testCases { - query := aggregateArrayClassQuery(tc.filters, "groupBy: [\"texts\"]") - - t.Run(tc.name, func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - extracted := extractArrayClassGroupByResult(result) - - assert.Len(t, extracted, len(tc.groupedAssertions)) - for groupedBy, groupAssertions := range tc.groupedAssertions { - group := findGroup(groupedBy, extracted) - require.NotNil(t, group, fmt.Sprintf("Group '%s' not found", groupedBy)) - - for _, assertion := range groupAssertions { - assertion(group) - } - } - }) - } - }) - - t.Run("aggregate ArrayClass with group by ints", func(t *testing.T) { - expectedAllResultsAssertions := map[string][]assertFunc{ - "101": { - asserts.groupedBy("101", "ints"), - asserts.meta(4), - asserts.booleanArray("booleans", 10, 4, 6, 0.4, 0.6), - asserts.textArray("texts", 10, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{4, 3, 2, 1}), - asserts.numberArray("numbers", 10, 4, 1, 1, 20, 2, 2), - asserts.intArray("ints", 10, 104, 101, 101, 1020, 102, 102), - asserts.dateArray("datesAsStrings", 10), - asserts.dateArray("dates", 10), - }, - "102": { - asserts.groupedBy("102", "ints"), - asserts.meta(3), - asserts.booleanArray("booleans", 9, 3, 6, 0.3333333333333333, 0.6666666666666666), - asserts.textArray("texts", 9, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{3, 3, 2, 1}), - asserts.numberArray("numbers", 9, 4, 1, 1, 19, 2, 2.111111111111111), - asserts.intArray("ints", 9, 104, 101, 101, 919, 102, 102.11111111111111), - asserts.dateArray("datesAsStrings", 9), - asserts.dateArray("dates", 9), - }, - "103": { - asserts.groupedBy("103", "ints"), - asserts.meta(2), - asserts.booleanArray("booleans", 7, 2, 5, 0.2857142857142857, 0.7142857142857143), - asserts.textArray("texts", 7, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{2, 2, 2, 1}), - asserts.numberArray("numbers", 7, 4, 1, 1, 16, 2, 2.2857142857142856), - asserts.intArray("ints", 7, 104, 101, 101, 716, 102, 102.28571428571429), - asserts.dateArray("datesAsStrings", 7), - asserts.dateArray("dates", 7), - }, - "104": { - asserts.groupedBy("104", "ints"), - asserts.meta(1), - asserts.booleanArray("booleans", 4, 1, 3, 0.25, 0.75), - asserts.textArray("texts", 4, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{1, 1, 1, 1}), - asserts.numberArray("numbers", 4, 4, 1, 1, 10, 2.5, 2.5), - asserts.intArray("ints", 4, 104, 101, 101, 410, 102.5, 102.5), - asserts.dateArray("datesAsStrings", 4), - asserts.dateArray("dates", 4), - }, - } - expectedResultsWithDataAssertions := map[string][]assertFunc{ - "101": { - asserts.groupedBy("101", "ints"), - asserts.meta(2), - asserts.booleanArray("booleans", 7, 2, 5, 0.2857142857142857, 0.7142857142857143), - asserts.textArray("texts", 7, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{2, 2, 2, 1}), - asserts.numberArray("numbers", 7, 4, 1, 1, 16, 2, 2.2857142857142856), - asserts.intArray("ints", 7, 104, 101, 101, 716, 102, 102.28571428571429), - asserts.dateArray("datesAsStrings", 7), - asserts.dateArray("dates", 7), - }, - "102": { - asserts.groupedBy("102", "ints"), - asserts.meta(2), - asserts.booleanArray("booleans", 7, 2, 5, 0.2857142857142857, 0.7142857142857143), - asserts.textArray("texts", 7, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{2, 2, 2, 1}), - asserts.numberArray("numbers", 7, 4, 1, 1, 16, 2, 2.2857142857142856), - asserts.intArray("ints", 7, 104, 101, 101, 716, 102, 102.28571428571429), - asserts.dateArray("datesAsStrings", 7), - asserts.dateArray("dates", 7), - }, - "103": { - asserts.groupedBy("103", "ints"), - asserts.meta(2), - asserts.booleanArray("booleans", 7, 2, 5, 0.2857142857142857, 0.7142857142857143), - asserts.textArray("texts", 7, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{2, 2, 2, 1}), - asserts.numberArray("numbers", 7, 4, 1, 1, 16, 2, 2.2857142857142856), - asserts.intArray("ints", 7, 104, 101, 101, 716, 102, 102.28571428571429), - asserts.dateArray("datesAsStrings", 7), - asserts.dateArray("dates", 7), - }, - "104": { - asserts.groupedBy("104", "ints"), - asserts.meta(1), - asserts.booleanArray("booleans", 4, 1, 3, 0.25, 0.75), - asserts.textArray("texts", 4, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{1, 1, 1, 1}), - asserts.numberArray("numbers", 4, 4, 1, 1, 10, 2.5, 2.5), - asserts.intArray("ints", 4, 104, 101, 101, 410, 102.5, 102.5), - asserts.dateArray("datesAsStrings", 4), - asserts.dateArray("dates", 4), - }, - } - expectedResultsWithoutDataAssertions := map[string][]assertFunc{} - expectedNoResultsAssertions := map[string][]assertFunc{} - - testCases := []aggregateTestCase{ - testCasesGen.WithoutFilters(expectedAllResultsAssertions), - - testCasesGen.WithWhereFilter_AllResults(expectedAllResultsAssertions), - testCasesGen.WithWhereFilter_ResultsWithData(expectedResultsWithDataAssertions), - testCasesGen.WithWhereFilter_ResultsWithoutData(expectedResultsWithoutDataAssertions), - testCasesGen.WithWhereFilter_NoResults(expectedNoResultsAssertions), - - testCasesGen.WithNearObjectFilter_AllResults(expectedAllResultsAssertions), - testCasesGen.WithNearObjectFilter_ResultsWithData(expectedResultsWithDataAssertions), - testCasesGen.WithNearObjectFilter_ResultsWithoutData(expectedResultsWithoutDataAssertions), - - testCasesGen.WithWhereAndNearObjectFilters_AllResults(expectedAllResultsAssertions), - testCasesGen.WithWhereAndNearObjectFilters_ResultsWithData(expectedResultsWithDataAssertions), - testCasesGen.WithWhereAndNearObjectFilters_ResultsWithoutData(expectedResultsWithoutDataAssertions), - testCasesGen.WithWhereAndNearObjectFilters_NoResults(expectedNoResultsAssertions), - } - - for _, tc := range testCases { - query := aggregateArrayClassQuery(tc.filters, "groupBy: [\"ints\"]") - - t.Run(tc.name, func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - extracted := extractArrayClassGroupByResult(result) - - assert.Len(t, extracted, len(tc.groupedAssertions)) - for groupedBy, groupAssertions := range tc.groupedAssertions { - group := findGroup(groupedBy, extracted) - require.NotNil(t, group, fmt.Sprintf("Group '%s' not found", groupedBy)) - - for _, assertion := range groupAssertions { - assertion(group) - } - } - }) - } - }) - - t.Run("aggregate ArrayClass with group by numbers", func(t *testing.T) { - expectedAllResultsAssertions := map[string][]assertFunc{ - "1": { - asserts.groupedBy("1", "numbers"), - asserts.meta(4), - asserts.booleanArray("booleans", 10, 4, 6, 0.4, 0.6), - asserts.textArray("texts", 10, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{4, 3, 2, 1}), - asserts.numberArray("numbers", 10, 4, 1, 1, 20, 2, 2), - asserts.intArray("ints", 10, 104, 101, 101, 1020, 102, 102), - asserts.dateArray("datesAsStrings", 10), - asserts.dateArray("dates", 10), - }, - "2": { - asserts.groupedBy("2", "numbers"), - asserts.meta(3), - asserts.booleanArray("booleans", 9, 3, 6, 0.3333333333333333, 0.6666666666666666), - asserts.textArray("texts", 9, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{3, 3, 2, 1}), - asserts.numberArray("numbers", 9, 4, 1, 1, 19, 2, 2.111111111111111), - asserts.intArray("ints", 9, 104, 101, 101, 919, 102, 102.11111111111111), - asserts.dateArray("datesAsStrings", 9), - asserts.dateArray("dates", 9), - }, - "3": { - asserts.groupedBy("3", "numbers"), - asserts.meta(2), - asserts.booleanArray("booleans", 7, 2, 5, 0.2857142857142857, 0.7142857142857143), - asserts.textArray("texts", 7, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{2, 2, 2, 1}), - asserts.numberArray("numbers", 7, 4, 1, 1, 16, 2, 2.2857142857142856), - asserts.intArray("ints", 7, 104, 101, 101, 716, 102, 102.28571428571429), - asserts.dateArray("datesAsStrings", 7), - asserts.dateArray("dates", 7), - }, - "4": { - asserts.groupedBy("4", "numbers"), - asserts.meta(1), - asserts.booleanArray("booleans", 4, 1, 3, 0.25, 0.75), - asserts.textArray("texts", 4, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{1, 1, 1, 1}), - asserts.numberArray("numbers", 4, 4, 1, 1, 10, 2.5, 2.5), - asserts.intArray("ints", 4, 104, 101, 101, 410, 102.5, 102.5), - asserts.dateArray("datesAsStrings", 4), - asserts.dateArray("dates", 4), - }, - } - expectedResultsWithDataAssertions := map[string][]assertFunc{ - "1": { - asserts.groupedBy("1", "numbers"), - asserts.meta(2), - asserts.booleanArray("booleans", 7, 2, 5, 0.2857142857142857, 0.7142857142857143), - asserts.textArray("texts", 7, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{2, 2, 2, 1}), - asserts.numberArray("numbers", 7, 4, 1, 1, 16, 2, 2.2857142857142856), - asserts.intArray("ints", 7, 104, 101, 101, 716, 102, 102.28571428571429), - asserts.dateArray("datesAsStrings", 7), - asserts.dateArray("dates", 7), - }, - "2": { - asserts.groupedBy("2", "numbers"), - asserts.meta(2), - asserts.booleanArray("booleans", 7, 2, 5, 0.2857142857142857, 0.7142857142857143), - asserts.textArray("texts", 7, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{2, 2, 2, 1}), - asserts.numberArray("numbers", 7, 4, 1, 1, 16, 2, 2.2857142857142856), - asserts.intArray("ints", 7, 104, 101, 101, 716, 102, 102.28571428571429), - asserts.dateArray("datesAsStrings", 7), - asserts.dateArray("dates", 7), - }, - "3": { - asserts.groupedBy("3", "numbers"), - asserts.meta(2), - asserts.booleanArray("booleans", 7, 2, 5, 0.2857142857142857, 0.7142857142857143), - asserts.textArray("texts", 7, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{2, 2, 2, 1}), - asserts.numberArray("numbers", 7, 4, 1, 1, 16, 2, 2.2857142857142856), - asserts.intArray("ints", 7, 104, 101, 101, 716, 102, 102.28571428571429), - asserts.dateArray("datesAsStrings", 7), - asserts.dateArray("dates", 7), - }, - "4": { - asserts.groupedBy("4", "numbers"), - asserts.meta(1), - asserts.booleanArray("booleans", 4, 1, 3, 0.25, 0.75), - asserts.textArray("texts", 4, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{1, 1, 1, 1}), - asserts.numberArray("numbers", 4, 4, 1, 1, 10, 2.5, 2.5), - asserts.intArray("ints", 4, 104, 101, 101, 410, 102.5, 102.5), - asserts.dateArray("datesAsStrings", 4), - asserts.dateArray("dates", 4), - }, - } - expectedResultsWithoutDataAssertions := map[string][]assertFunc{} - expectedNoResultsAssertions := map[string][]assertFunc{} - - testCases := []aggregateTestCase{ - testCasesGen.WithoutFilters(expectedAllResultsAssertions), - - testCasesGen.WithWhereFilter_AllResults(expectedAllResultsAssertions), - testCasesGen.WithWhereFilter_ResultsWithData(expectedResultsWithDataAssertions), - testCasesGen.WithWhereFilter_ResultsWithoutData(expectedResultsWithoutDataAssertions), - testCasesGen.WithWhereFilter_NoResults(expectedNoResultsAssertions), - - testCasesGen.WithNearObjectFilter_AllResults(expectedAllResultsAssertions), - testCasesGen.WithNearObjectFilter_ResultsWithData(expectedResultsWithDataAssertions), - testCasesGen.WithNearObjectFilter_ResultsWithoutData(expectedResultsWithoutDataAssertions), - - testCasesGen.WithWhereAndNearObjectFilters_AllResults(expectedAllResultsAssertions), - testCasesGen.WithWhereAndNearObjectFilters_ResultsWithData(expectedResultsWithDataAssertions), - testCasesGen.WithWhereAndNearObjectFilters_ResultsWithoutData(expectedResultsWithoutDataAssertions), - testCasesGen.WithWhereAndNearObjectFilters_NoResults(expectedNoResultsAssertions), - } - - for _, tc := range testCases { - query := aggregateArrayClassQuery(tc.filters, "groupBy: [\"numbers\"]") - - t.Run(tc.name, func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - extracted := extractArrayClassGroupByResult(result) - - assert.Len(t, extracted, len(tc.groupedAssertions)) - for groupedBy, groupAssertions := range tc.groupedAssertions { - group := findGroup(groupedBy, extracted) - require.NotNil(t, group, fmt.Sprintf("Group '%s' not found", groupedBy)) - - for _, assertion := range groupAssertions { - assertion(group) - } - } - }) - } - }) - - t.Run("aggregate ArrayClass with group by dates", func(t *testing.T) { - expectedAllResultsAssertions := map[string][]assertFunc{ - "2001-06-01T12:00:00Z": { - asserts.groupedBy("2001-06-01T12:00:00Z", "dates"), - asserts.meta(4), - asserts.booleanArray("booleans", 10, 4, 6, 0.4, 0.6), - asserts.textArray("texts", 10, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{4, 3, 2, 1}), - asserts.numberArray("numbers", 10, 4, 1, 1, 20, 2, 2), - asserts.intArray("ints", 10, 104, 101, 101, 1020, 102, 102), - asserts.dateArray("datesAsStrings", 10), - asserts.dateArray("dates", 10), - }, - "2002-06-02T12:00:00Z": { - asserts.groupedBy("2002-06-02T12:00:00Z", "dates"), - asserts.meta(3), - asserts.booleanArray("booleans", 9, 3, 6, 0.3333333333333333, 0.6666666666666666), - asserts.textArray("texts", 9, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{3, 3, 2, 1}), - asserts.numberArray("numbers", 9, 4, 1, 1, 19, 2, 2.111111111111111), - asserts.intArray("ints", 9, 104, 101, 101, 919, 102, 102.11111111111111), - asserts.dateArray("datesAsStrings", 9), - asserts.dateArray("dates", 9), - }, - "2003-06-03T12:00:00Z": { - asserts.groupedBy("2003-06-03T12:00:00Z", "dates"), - asserts.meta(2), - asserts.booleanArray("booleans", 7, 2, 5, 0.2857142857142857, 0.7142857142857143), - asserts.textArray("texts", 7, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{2, 2, 2, 1}), - asserts.numberArray("numbers", 7, 4, 1, 1, 16, 2, 2.2857142857142856), - asserts.intArray("ints", 7, 104, 101, 101, 716, 102, 102.28571428571429), - asserts.dateArray("datesAsStrings", 7), - asserts.dateArray("dates", 7), - }, - "2004-06-04T12:00:00Z": { - asserts.groupedBy("2004-06-04T12:00:00Z", "dates"), - asserts.meta(1), - asserts.booleanArray("booleans", 4, 1, 3, 0.25, 0.75), - asserts.textArray("texts", 4, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{1, 1, 1, 1}), - asserts.numberArray("numbers", 4, 4, 1, 1, 10, 2.5, 2.5), - asserts.intArray("ints", 4, 104, 101, 101, 410, 102.5, 102.5), - asserts.dateArray("datesAsStrings", 4), - asserts.dateArray("dates", 4), - }, - } - expectedResultsWithDataAssertions := map[string][]assertFunc{ - "2001-06-01T12:00:00Z": { - asserts.groupedBy("2001-06-01T12:00:00Z", "dates"), - asserts.meta(2), - asserts.booleanArray("booleans", 7, 2, 5, 0.2857142857142857, 0.7142857142857143), - asserts.textArray("texts", 7, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{2, 2, 2, 1}), - asserts.numberArray("numbers", 7, 4, 1, 1, 16, 2, 2.2857142857142856), - asserts.intArray("ints", 7, 104, 101, 101, 716, 102, 102.28571428571429), - asserts.dateArray("datesAsStrings", 7), - asserts.dateArray("dates", 7), - }, - "2002-06-02T12:00:00Z": { - asserts.groupedBy("2002-06-02T12:00:00Z", "dates"), - asserts.meta(2), - asserts.booleanArray("booleans", 7, 2, 5, 0.2857142857142857, 0.7142857142857143), - asserts.textArray("texts", 7, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{2, 2, 2, 1}), - asserts.numberArray("numbers", 7, 4, 1, 1, 16, 2, 2.2857142857142856), - asserts.intArray("ints", 7, 104, 101, 101, 716, 102, 102.28571428571429), - asserts.dateArray("datesAsStrings", 7), - asserts.dateArray("dates", 7), - }, - "2003-06-03T12:00:00Z": { - asserts.groupedBy("2003-06-03T12:00:00Z", "dates"), - asserts.meta(2), - asserts.booleanArray("booleans", 7, 2, 5, 0.2857142857142857, 0.7142857142857143), - asserts.textArray("texts", 7, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{2, 2, 2, 1}), - asserts.numberArray("numbers", 7, 4, 1, 1, 16, 2, 2.2857142857142856), - asserts.intArray("ints", 7, 104, 101, 101, 716, 102, 102.28571428571429), - asserts.dateArray("datesAsStrings", 7), - asserts.dateArray("dates", 7), - }, - "2004-06-04T12:00:00Z": { - asserts.groupedBy("2004-06-04T12:00:00Z", "dates"), - asserts.meta(1), - asserts.booleanArray("booleans", 4, 1, 3, 0.25, 0.75), - asserts.textArray("texts", 4, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{1, 1, 1, 1}), - asserts.numberArray("numbers", 4, 4, 1, 1, 10, 2.5, 2.5), - asserts.intArray("ints", 4, 104, 101, 101, 410, 102.5, 102.5), - asserts.dateArray("datesAsStrings", 4), - asserts.dateArray("dates", 4), - }, - } - expectedResultsWithoutDataAssertions := map[string][]assertFunc{} - expectedNoResultsAssertions := map[string][]assertFunc{} - - testCases := []aggregateTestCase{ - testCasesGen.WithoutFilters(expectedAllResultsAssertions), - - testCasesGen.WithWhereFilter_AllResults(expectedAllResultsAssertions), - testCasesGen.WithWhereFilter_ResultsWithData(expectedResultsWithDataAssertions), - testCasesGen.WithWhereFilter_ResultsWithoutData(expectedResultsWithoutDataAssertions), - testCasesGen.WithWhereFilter_NoResults(expectedNoResultsAssertions), - - testCasesGen.WithNearObjectFilter_AllResults(expectedAllResultsAssertions), - testCasesGen.WithNearObjectFilter_ResultsWithData(expectedResultsWithDataAssertions), - testCasesGen.WithNearObjectFilter_ResultsWithoutData(expectedResultsWithoutDataAssertions), - - testCasesGen.WithWhereAndNearObjectFilters_AllResults(expectedAllResultsAssertions), - testCasesGen.WithWhereAndNearObjectFilters_ResultsWithData(expectedResultsWithDataAssertions), - testCasesGen.WithWhereAndNearObjectFilters_ResultsWithoutData(expectedResultsWithoutDataAssertions), - testCasesGen.WithWhereAndNearObjectFilters_NoResults(expectedNoResultsAssertions), - } - - for _, tc := range testCases { - query := aggregateArrayClassQuery(tc.filters, "groupBy: [\"dates\"]") - - t.Run(tc.name, func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - extracted := extractArrayClassGroupByResult(result) - - assert.Len(t, extracted, len(tc.groupedAssertions)) - for groupedBy, groupAssertions := range tc.groupedAssertions { - group := findGroup(groupedBy, extracted) - require.NotNil(t, group, fmt.Sprintf("Group '%s' not found", groupedBy)) - - for _, assertion := range groupAssertions { - assertion(group) - } - } - }) - } - }) - - t.Run("aggregate ArrayClass with group by dates as strings", func(t *testing.T) { - expectedAllResultsAssertions := map[string][]assertFunc{ - "2021-06-01T22:18:59.640162Z": { - asserts.groupedBy("2021-06-01T22:18:59.640162Z", "datesAsStrings"), - asserts.meta(4), - asserts.booleanArray("booleans", 10, 4, 6, 0.4, 0.6), - asserts.textArray("texts", 10, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{4, 3, 2, 1}), - asserts.numberArray("numbers", 10, 4, 1, 1, 20, 2, 2), - asserts.intArray("ints", 10, 104, 101, 101, 1020, 102, 102), - asserts.dateArray("datesAsStrings", 10), - asserts.dateArray("dates", 10), - }, - "2022-06-02T22:18:59.640162Z": { - asserts.groupedBy("2022-06-02T22:18:59.640162Z", "datesAsStrings"), - asserts.meta(3), - asserts.booleanArray("booleans", 9, 3, 6, 0.3333333333333333, 0.6666666666666666), - asserts.textArray("texts", 9, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{3, 3, 2, 1}), - asserts.numberArray("numbers", 9, 4, 1, 1, 19, 2, 2.111111111111111), - asserts.intArray("ints", 9, 104, 101, 101, 919, 102, 102.11111111111111), - asserts.dateArray("datesAsStrings", 9), - asserts.dateArray("dates", 9), - }, - "2023-06-03T22:18:59.640162Z": { - asserts.groupedBy("2023-06-03T22:18:59.640162Z", "datesAsStrings"), - asserts.meta(2), - asserts.booleanArray("booleans", 7, 2, 5, 0.2857142857142857, 0.7142857142857143), - asserts.textArray("texts", 7, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{2, 2, 2, 1}), - asserts.numberArray("numbers", 7, 4, 1, 1, 16, 2, 2.2857142857142856), - asserts.intArray("ints", 7, 104, 101, 101, 716, 102, 102.28571428571429), - asserts.dateArray("datesAsStrings", 7), - asserts.dateArray("dates", 7), - }, - "2024-06-04T22:18:59.640162Z": { - asserts.groupedBy("2024-06-04T22:18:59.640162Z", "datesAsStrings"), - asserts.meta(1), - asserts.booleanArray("booleans", 4, 1, 3, 0.25, 0.75), - asserts.textArray("texts", 4, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{1, 1, 1, 1}), - asserts.numberArray("numbers", 4, 4, 1, 1, 10, 2.5, 2.5), - asserts.intArray("ints", 4, 104, 101, 101, 410, 102.5, 102.5), - asserts.dateArray("datesAsStrings", 4), - asserts.dateArray("dates", 4), - }, - } - expectedResultsWithDataAssertions := map[string][]assertFunc{ - "2021-06-01T22:18:59.640162Z": { - asserts.groupedBy("2021-06-01T22:18:59.640162Z", "datesAsStrings"), - asserts.meta(2), - asserts.booleanArray("booleans", 7, 2, 5, 0.2857142857142857, 0.7142857142857143), - asserts.textArray("texts", 7, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{2, 2, 2, 1}), - asserts.numberArray("numbers", 7, 4, 1, 1, 16, 2, 2.2857142857142856), - asserts.intArray("ints", 7, 104, 101, 101, 716, 102, 102.28571428571429), - asserts.dateArray("datesAsStrings", 7), - asserts.dateArray("dates", 7), - }, - "2022-06-02T22:18:59.640162Z": { - asserts.groupedBy("2022-06-02T22:18:59.640162Z", "datesAsStrings"), - asserts.meta(2), - asserts.booleanArray("booleans", 7, 2, 5, 0.2857142857142857, 0.7142857142857143), - asserts.textArray("texts", 7, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{2, 2, 2, 1}), - asserts.numberArray("numbers", 7, 4, 1, 1, 16, 2, 2.2857142857142856), - asserts.intArray("ints", 7, 104, 101, 101, 716, 102, 102.28571428571429), - asserts.dateArray("datesAsStrings", 7), - asserts.dateArray("dates", 7), - }, - "2023-06-03T22:18:59.640162Z": { - asserts.groupedBy("2023-06-03T22:18:59.640162Z", "datesAsStrings"), - asserts.meta(2), - asserts.booleanArray("booleans", 7, 2, 5, 0.2857142857142857, 0.7142857142857143), - asserts.textArray("texts", 7, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{2, 2, 2, 1}), - asserts.numberArray("numbers", 7, 4, 1, 1, 16, 2, 2.2857142857142856), - asserts.intArray("ints", 7, 104, 101, 101, 716, 102, 102.28571428571429), - asserts.dateArray("datesAsStrings", 7), - asserts.dateArray("dates", 7), - }, - "2024-06-04T22:18:59.640162Z": { - asserts.groupedBy("2024-06-04T22:18:59.640162Z", "datesAsStrings"), - asserts.meta(1), - asserts.booleanArray("booleans", 4, 1, 3, 0.25, 0.75), - asserts.textArray("texts", 4, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{1, 1, 1, 1}), - asserts.numberArray("numbers", 4, 4, 1, 1, 10, 2.5, 2.5), - asserts.intArray("ints", 4, 104, 101, 101, 410, 102.5, 102.5), - asserts.dateArray("datesAsStrings", 4), - asserts.dateArray("dates", 4), - }, - } - expectedResultsWithoutDataAssertions := map[string][]assertFunc{} - expectedNoResultsAssertions := map[string][]assertFunc{} - - testCases := []aggregateTestCase{ - testCasesGen.WithoutFilters(expectedAllResultsAssertions), - - testCasesGen.WithWhereFilter_AllResults(expectedAllResultsAssertions), - testCasesGen.WithWhereFilter_ResultsWithData(expectedResultsWithDataAssertions), - testCasesGen.WithWhereFilter_ResultsWithoutData(expectedResultsWithoutDataAssertions), - testCasesGen.WithWhereFilter_NoResults(expectedNoResultsAssertions), - - testCasesGen.WithNearObjectFilter_AllResults(expectedAllResultsAssertions), - testCasesGen.WithNearObjectFilter_ResultsWithData(expectedResultsWithDataAssertions), - testCasesGen.WithNearObjectFilter_ResultsWithoutData(expectedResultsWithoutDataAssertions), - - testCasesGen.WithWhereAndNearObjectFilters_AllResults(expectedAllResultsAssertions), - testCasesGen.WithWhereAndNearObjectFilters_ResultsWithData(expectedResultsWithDataAssertions), - testCasesGen.WithWhereAndNearObjectFilters_ResultsWithoutData(expectedResultsWithoutDataAssertions), - testCasesGen.WithWhereAndNearObjectFilters_NoResults(expectedNoResultsAssertions), - } - - for _, tc := range testCases { - query := aggregateArrayClassQuery(tc.filters, "groupBy: [\"datesAsStrings\"]") - - t.Run(tc.name, func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - extracted := extractArrayClassGroupByResult(result) - - assert.Len(t, extracted, len(tc.groupedAssertions)) - for groupedBy, groupAssertions := range tc.groupedAssertions { - group := findGroup(groupedBy, extracted) - require.NotNil(t, group, fmt.Sprintf("Group '%s' not found", groupedBy)) - - for _, assertion := range groupAssertions { - assertion(group) - } - } - }) - } - }) - - t.Run("aggregate ArrayClass with group by booleans", func(t *testing.T) { - expectedAllResultsAssertions := map[string][]assertFunc{ - "true": { - asserts.groupedBy("true", "booleans"), - asserts.meta(3), - asserts.booleanArray("booleans", 9, 3, 6, 0.3333333333333333, 0.6666666666666666), - asserts.textArray("texts", 9, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{3, 3, 2, 1}), - asserts.numberArray("numbers", 9, 4, 1, 1, 19, 2, 2.111111111111111), - asserts.intArray("ints", 9, 104, 101, 101, 919, 102, 102.11111111111111), - asserts.dateArray("datesAsStrings", 9), - asserts.dateArray("dates", 9), - }, - "false": { - asserts.groupedBy("false", "booleans"), - asserts.meta(4), - asserts.booleanArray("booleans", 10, 4, 6, 0.4, 0.6), - asserts.textArray("texts", 10, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{4, 3, 2, 1}), - asserts.numberArray("numbers", 10, 4, 1, 1, 20, 2, 2), - asserts.intArray("ints", 10, 104, 101, 101, 1020, 102, 102), - asserts.dateArray("datesAsStrings", 10), - asserts.dateArray("dates", 10), - }, - } - expectedResultsWithDataAssertions := map[string][]assertFunc{ - "true": { - asserts.groupedBy("true", "booleans"), - asserts.meta(2), - asserts.booleanArray("booleans", 7, 2, 5, 0.2857142857142857, 0.7142857142857143), - asserts.textArray("texts", 7, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{2, 2, 2, 1}), - asserts.numberArray("numbers", 7, 4, 1, 1, 16, 2, 2.2857142857142856), - asserts.intArray("ints", 7, 104, 101, 101, 716, 102, 102.28571428571429), - asserts.dateArray("datesAsStrings", 7), - asserts.dateArray("dates", 7), - }, - "false": { - asserts.groupedBy("false", "booleans"), - asserts.meta(2), - asserts.booleanArray("booleans", 7, 2, 5, 0.2857142857142857, 0.7142857142857143), - asserts.textArray("texts", 7, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{2, 2, 2, 1}), - asserts.numberArray("numbers", 7, 4, 1, 1, 16, 2, 2.2857142857142856), - asserts.intArray("ints", 7, 104, 101, 101, 716, 102, 102.28571428571429), - asserts.dateArray("datesAsStrings", 7), - asserts.dateArray("dates", 7), - }, - } - expectedResultsWithoutDataAssertions := map[string][]assertFunc{} - expectedNoResultsAssertions := map[string][]assertFunc{} - - testCases := []aggregateTestCase{ - testCasesGen.WithoutFilters(expectedAllResultsAssertions), - - testCasesGen.WithWhereFilter_AllResults(expectedAllResultsAssertions), - testCasesGen.WithWhereFilter_ResultsWithData(expectedResultsWithDataAssertions), - testCasesGen.WithWhereFilter_ResultsWithoutData(expectedResultsWithoutDataAssertions), - testCasesGen.WithWhereFilter_NoResults(expectedNoResultsAssertions), - - testCasesGen.WithNearObjectFilter_AllResults(expectedAllResultsAssertions), - testCasesGen.WithNearObjectFilter_ResultsWithData(expectedResultsWithDataAssertions), - testCasesGen.WithNearObjectFilter_ResultsWithoutData(expectedResultsWithoutDataAssertions), - - testCasesGen.WithWhereAndNearObjectFilters_AllResults(expectedAllResultsAssertions), - testCasesGen.WithWhereAndNearObjectFilters_ResultsWithData(expectedResultsWithDataAssertions), - testCasesGen.WithWhereAndNearObjectFilters_ResultsWithoutData(expectedResultsWithoutDataAssertions), - testCasesGen.WithWhereAndNearObjectFilters_NoResults(expectedNoResultsAssertions), - } - - for _, tc := range testCases { - query := aggregateArrayClassQuery(tc.filters, "groupBy: [\"booleans\"]") - - t.Run(tc.name, func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - extracted := extractArrayClassGroupByResult(result) - - assert.Len(t, extracted, len(tc.groupedAssertions)) - for groupedBy, groupAssertions := range tc.groupedAssertions { - group := findGroup(groupedBy, extracted) - require.NotNil(t, group, fmt.Sprintf("Group '%s' not found", groupedBy)) - - for _, assertion := range groupAssertions { - assertion(group) - } - } - }) - } - }) -} - -func aggregateDuplicatesClassWithGroupByTest(t *testing.T) { - asserts := newAggregateResponseAssert(t) - testCasesGen := &aggregateDuplicatesClassTestCases{} - - t.Run("aggregate DuplicatesClass with group by texts", func(t *testing.T) { - expectedAllResultsAssertions := map[string][]assertFunc{ - "Atxt": { - asserts.groupedBy("Atxt", "texts"), - asserts.meta(3), - asserts.booleanArray("booleans", 9, 3, 6, 0.3333333333333333, 0.6666666666666666), - asserts.textArray("texts", 9, []string{"Atxt", "Btxt"}, []int64{6, 3}), - asserts.numberArray("numbers", 9, 2, 1, 1, 12, 1, 1.3333333333333333), - asserts.intArray("ints", 9, 102, 101, 101, 912, 101, 101.33333333333333), - asserts.dateArray("datesAsStrings", 9), - }, - "Btxt": { - asserts.groupedBy("Btxt", "texts"), - asserts.meta(3), - asserts.booleanArray("booleans", 9, 3, 6, 0.3333333333333333, 0.6666666666666666), - asserts.textArray("texts", 9, []string{"Atxt", "Btxt"}, []int64{6, 3}), - asserts.numberArray("numbers", 9, 2, 1, 1, 12, 1, 1.3333333333333333), - asserts.intArray("ints", 9, 102, 101, 101, 912, 101, 101.33333333333333), - asserts.dateArray("datesAsStrings", 9), - }, - } - expectedSomeResultsAssertions := map[string][]assertFunc{ - "Atxt": { - asserts.groupedBy("Atxt", "texts"), - asserts.meta(1), - asserts.booleanArray("booleans", 4, 1, 3, 0.25, 0.75), - asserts.textArray("texts", 4, []string{"Atxt", "Btxt"}, []int64{3, 1}), - asserts.numberArray("numbers", 4, 2, 1, 1, 5, 1, 1.25), - asserts.intArray("ints", 4, 102, 101, 101, 405, 101, 101.25), - asserts.dateArray("datesAsStrings", 4), - }, - "Btxt": { - asserts.groupedBy("Btxt", "texts"), - asserts.meta(1), - asserts.booleanArray("booleans", 4, 1, 3, 0.25, 0.75), - asserts.textArray("texts", 4, []string{"Atxt", "Btxt"}, []int64{3, 1}), - asserts.numberArray("numbers", 4, 2, 1, 1, 5, 1, 1.25), - asserts.intArray("ints", 4, 102, 101, 101, 405, 101, 101.25), - asserts.dateArray("datesAsStrings", 4), - }, - } - expectedNoResultsAssertsions := map[string][]assertFunc{} - - testCases := []aggregateTestCase{ - testCasesGen.WithoutFilters(expectedAllResultsAssertions), - - testCasesGen.WithWhereFilter_AllResults(expectedAllResultsAssertions), - testCasesGen.WithWhereFilter_SomeResults(expectedSomeResultsAssertions), - testCasesGen.WithWhereFilter_NoResults(expectedNoResultsAssertsions), - } - - for _, tc := range testCases { - query := aggregateDuplicatesClassQuery(tc.filters, "groupBy: [\"texts\"]") - - t.Run(tc.name, func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - extracted := extractDuplicatesClassGroupByResult(result) - - assert.Len(t, extracted, len(tc.groupedAssertions)) - for groupedBy, groupAssertions := range tc.groupedAssertions { - group := findGroup(groupedBy, extracted) - require.NotNil(t, group, fmt.Sprintf("Group '%s' not found", groupedBy)) - - for _, assertion := range groupAssertions { - assertion(group) - } - } - }) - } - }) - - t.Run("aggregate DuplicatesClass with group by ints", func(t *testing.T) { - expectedAllResultsAssertions := map[string][]assertFunc{ - "101": { - asserts.groupedBy("101", "ints"), - asserts.meta(3), - asserts.booleanArray("booleans", 9, 3, 6, 0.3333333333333333, 0.6666666666666666), - asserts.textArray("texts", 9, []string{"Atxt", "Btxt"}, []int64{6, 3}), - asserts.numberArray("numbers", 9, 2, 1, 1, 12, 1, 1.3333333333333333), - asserts.intArray("ints", 9, 102, 101, 101, 912, 101, 101.33333333333333), - asserts.dateArray("datesAsStrings", 9), - }, - "102": { - asserts.groupedBy("102", "ints"), - asserts.meta(3), - asserts.booleanArray("booleans", 9, 3, 6, 0.3333333333333333, 0.6666666666666666), - asserts.textArray("texts", 9, []string{"Atxt", "Btxt"}, []int64{6, 3}), - asserts.numberArray("numbers", 9, 2, 1, 1, 12, 1, 1.3333333333333333), - asserts.intArray("ints", 9, 102, 101, 101, 912, 101, 101.33333333333333), - asserts.dateArray("datesAsStrings", 9), - }, - } - expectedSomeResultsAssertions := map[string][]assertFunc{ - "101": { - asserts.groupedBy("101", "ints"), - asserts.meta(1), - asserts.booleanArray("booleans", 4, 1, 3, 0.25, 0.75), - asserts.textArray("texts", 4, []string{"Atxt", "Btxt"}, []int64{3, 1}), - asserts.numberArray("numbers", 4, 2, 1, 1, 5, 1, 1.25), - asserts.intArray("ints", 4, 102, 101, 101, 405, 101, 101.25), - asserts.dateArray("datesAsStrings", 4), - }, - "102": { - asserts.groupedBy("102", "ints"), - asserts.meta(1), - asserts.booleanArray("booleans", 4, 1, 3, 0.25, 0.75), - asserts.textArray("texts", 4, []string{"Atxt", "Btxt"}, []int64{3, 1}), - asserts.numberArray("numbers", 4, 2, 1, 1, 5, 1, 1.25), - asserts.intArray("ints", 4, 102, 101, 101, 405, 101, 101.25), - asserts.dateArray("datesAsStrings", 4), - }, - } - expectedNoResultsAssertsions := map[string][]assertFunc{} - - testCases := []aggregateTestCase{ - testCasesGen.WithoutFilters(expectedAllResultsAssertions), - - testCasesGen.WithWhereFilter_AllResults(expectedAllResultsAssertions), - testCasesGen.WithWhereFilter_SomeResults(expectedSomeResultsAssertions), - testCasesGen.WithWhereFilter_NoResults(expectedNoResultsAssertsions), - } - - for _, tc := range testCases { - query := aggregateDuplicatesClassQuery(tc.filters, "groupBy: [\"ints\"]") - - t.Run(tc.name, func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - extracted := extractDuplicatesClassGroupByResult(result) - - assert.Len(t, extracted, len(tc.groupedAssertions)) - for groupedBy, groupAssertions := range tc.groupedAssertions { - group := findGroup(groupedBy, extracted) - require.NotNil(t, group, fmt.Sprintf("Group '%s' not found", groupedBy)) - - for _, assertion := range groupAssertions { - assertion(group) - } - } - }) - } - }) - - t.Run("aggregate DuplicatesClass with group by numbers", func(t *testing.T) { - expectedAllResultsAssertions := map[string][]assertFunc{ - "1": { - asserts.groupedBy("1", "numbers"), - asserts.meta(3), - asserts.booleanArray("booleans", 9, 3, 6, 0.3333333333333333, 0.6666666666666666), - asserts.textArray("texts", 9, []string{"Atxt", "Btxt"}, []int64{6, 3}), - asserts.numberArray("numbers", 9, 2, 1, 1, 12, 1, 1.3333333333333333), - asserts.intArray("ints", 9, 102, 101, 101, 912, 101, 101.33333333333333), - asserts.dateArray("datesAsStrings", 9), - }, - "2": { - asserts.groupedBy("2", "numbers"), - asserts.meta(3), - asserts.booleanArray("booleans", 9, 3, 6, 0.3333333333333333, 0.6666666666666666), - asserts.textArray("texts", 9, []string{"Atxt", "Btxt"}, []int64{6, 3}), - asserts.numberArray("numbers", 9, 2, 1, 1, 12, 1, 1.3333333333333333), - asserts.intArray("ints", 9, 102, 101, 101, 912, 101, 101.33333333333333), - asserts.dateArray("datesAsStrings", 9), - }, - } - expectedSomeResultsAssertions := map[string][]assertFunc{ - "1": { - asserts.groupedBy("1", "numbers"), - asserts.meta(1), - asserts.booleanArray("booleans", 4, 1, 3, 0.25, 0.75), - asserts.textArray("texts", 4, []string{"Atxt", "Btxt"}, []int64{3, 1}), - asserts.numberArray("numbers", 4, 2, 1, 1, 5, 1, 1.25), - asserts.intArray("ints", 4, 102, 101, 101, 405, 101, 101.25), - asserts.dateArray("datesAsStrings", 4), - }, - "2": { - asserts.groupedBy("2", "numbers"), - asserts.meta(1), - asserts.booleanArray("booleans", 4, 1, 3, 0.25, 0.75), - asserts.textArray("texts", 4, []string{"Atxt", "Btxt"}, []int64{3, 1}), - asserts.numberArray("numbers", 4, 2, 1, 1, 5, 1, 1.25), - asserts.intArray("ints", 4, 102, 101, 101, 405, 101, 101.25), - asserts.dateArray("datesAsStrings", 4), - }, - } - expectedNoResultsAssertsions := map[string][]assertFunc{} - - testCases := []aggregateTestCase{ - testCasesGen.WithoutFilters(expectedAllResultsAssertions), - - testCasesGen.WithWhereFilter_AllResults(expectedAllResultsAssertions), - testCasesGen.WithWhereFilter_SomeResults(expectedSomeResultsAssertions), - testCasesGen.WithWhereFilter_NoResults(expectedNoResultsAssertsions), - } - - for _, tc := range testCases { - query := aggregateDuplicatesClassQuery(tc.filters, "groupBy: [\"numbers\"]") - - t.Run(tc.name, func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - extracted := extractDuplicatesClassGroupByResult(result) - - assert.Len(t, extracted, len(tc.groupedAssertions)) - for groupedBy, groupAssertions := range tc.groupedAssertions { - group := findGroup(groupedBy, extracted) - require.NotNil(t, group, fmt.Sprintf("Group '%s' not found", groupedBy)) - - for _, assertion := range groupAssertions { - assertion(group) - } - } - }) - } - }) - - t.Run("aggregate DuplicatesClass with group by dates as string", func(t *testing.T) { - expectedAllResultsAssertions := map[string][]assertFunc{ - "2021-06-01T22:18:59.640162Z": { - asserts.groupedBy("2021-06-01T22:18:59.640162Z", "datesAsStrings"), - asserts.meta(3), - asserts.booleanArray("booleans", 9, 3, 6, 0.3333333333333333, 0.6666666666666666), - asserts.textArray("texts", 9, []string{"Atxt", "Btxt"}, []int64{6, 3}), - asserts.numberArray("numbers", 9, 2, 1, 1, 12, 1, 1.3333333333333333), - asserts.intArray("ints", 9, 102, 101, 101, 912, 101, 101.33333333333333), - asserts.dateArray("datesAsStrings", 9), - }, - "2022-06-02T22:18:59.640162Z": { - asserts.groupedBy("2022-06-02T22:18:59.640162Z", "datesAsStrings"), - asserts.meta(3), - asserts.booleanArray("booleans", 9, 3, 6, 0.3333333333333333, 0.6666666666666666), - asserts.textArray("texts", 9, []string{"Atxt", "Btxt"}, []int64{6, 3}), - asserts.numberArray("numbers", 9, 2, 1, 1, 12, 1, 1.3333333333333333), - asserts.intArray("ints", 9, 102, 101, 101, 912, 101, 101.33333333333333), - asserts.dateArray("datesAsStrings", 9), - }, - } - expectedSomeResultsAssertions := map[string][]assertFunc{ - "2021-06-01T22:18:59.640162Z": { - asserts.groupedBy("2021-06-01T22:18:59.640162Z", "datesAsStrings"), - asserts.meta(1), - asserts.booleanArray("booleans", 4, 1, 3, 0.25, 0.75), - asserts.textArray("texts", 4, []string{"Atxt", "Btxt"}, []int64{3, 1}), - asserts.numberArray("numbers", 4, 2, 1, 1, 5, 1, 1.25), - asserts.intArray("ints", 4, 102, 101, 101, 405, 101, 101.25), - asserts.dateArray("datesAsStrings", 4), - }, - "2022-06-02T22:18:59.640162Z": { - asserts.groupedBy("2022-06-02T22:18:59.640162Z", "datesAsStrings"), - asserts.meta(1), - asserts.booleanArray("booleans", 4, 1, 3, 0.25, 0.75), - asserts.textArray("texts", 4, []string{"Atxt", "Btxt"}, []int64{3, 1}), - asserts.numberArray("numbers", 4, 2, 1, 1, 5, 1, 1.25), - asserts.intArray("ints", 4, 102, 101, 101, 405, 101, 101.25), - asserts.dateArray("datesAsStrings", 4), - }, - } - expectedNoResultsAssertsions := map[string][]assertFunc{} - - testCases := []aggregateTestCase{ - testCasesGen.WithoutFilters(expectedAllResultsAssertions), - - testCasesGen.WithWhereFilter_AllResults(expectedAllResultsAssertions), - testCasesGen.WithWhereFilter_SomeResults(expectedSomeResultsAssertions), - testCasesGen.WithWhereFilter_NoResults(expectedNoResultsAssertsions), - } - - for _, tc := range testCases { - query := aggregateDuplicatesClassQuery(tc.filters, "groupBy: [\"datesAsStrings\"]") - - t.Run(tc.name, func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - extracted := extractDuplicatesClassGroupByResult(result) - - assert.Len(t, extracted, len(tc.groupedAssertions)) - for groupedBy, groupAssertions := range tc.groupedAssertions { - group := findGroup(groupedBy, extracted) - require.NotNil(t, group, fmt.Sprintf("Group '%s' not found", groupedBy)) - - for _, assertion := range groupAssertions { - assertion(group) - } - } - }) - } - }) - - t.Run("aggregate DuplicatesClass with group by booleans", func(t *testing.T) { - expectedAllResultsAssertions := map[string][]assertFunc{ - "true": { - asserts.groupedBy("true", "booleans"), - asserts.meta(3), - asserts.booleanArray("booleans", 9, 3, 6, 0.3333333333333333, 0.6666666666666666), - asserts.textArray("texts", 9, []string{"Atxt", "Btxt"}, []int64{6, 3}), - asserts.numberArray("numbers", 9, 2, 1, 1, 12, 1, 1.3333333333333333), - asserts.intArray("ints", 9, 102, 101, 101, 912, 101, 101.33333333333333), - asserts.dateArray("datesAsStrings", 9), - }, - "false": { - asserts.groupedBy("false", "booleans"), - asserts.meta(3), - asserts.booleanArray("booleans", 9, 3, 6, 0.3333333333333333, 0.6666666666666666), - asserts.textArray("texts", 9, []string{"Atxt", "Btxt"}, []int64{6, 3}), - asserts.numberArray("numbers", 9, 2, 1, 1, 12, 1, 1.3333333333333333), - asserts.intArray("ints", 9, 102, 101, 101, 912, 101, 101.33333333333333), - asserts.dateArray("datesAsStrings", 9), - }, - } - expectedSomeResultsAssertions := map[string][]assertFunc{ - "true": { - asserts.groupedBy("true", "booleans"), - asserts.meta(1), - asserts.booleanArray("booleans", 4, 1, 3, 0.25, 0.75), - asserts.textArray("texts", 4, []string{"Atxt", "Btxt"}, []int64{3, 1}), - asserts.numberArray("numbers", 4, 2, 1, 1, 5, 1, 1.25), - asserts.intArray("ints", 4, 102, 101, 101, 405, 101, 101.25), - asserts.dateArray("datesAsStrings", 4), - }, - "false": { - asserts.groupedBy("false", "booleans"), - asserts.meta(1), - asserts.booleanArray("booleans", 4, 1, 3, 0.25, 0.75), - asserts.textArray("texts", 4, []string{"Atxt", "Btxt"}, []int64{3, 1}), - asserts.numberArray("numbers", 4, 2, 1, 1, 5, 1, 1.25), - asserts.intArray("ints", 4, 102, 101, 101, 405, 101, 101.25), - asserts.dateArray("datesAsStrings", 4), - }, - } - expectedNoResultsAssertsions := map[string][]assertFunc{} - - testCases := []aggregateTestCase{ - testCasesGen.WithoutFilters(expectedAllResultsAssertions), - - testCasesGen.WithWhereFilter_AllResults(expectedAllResultsAssertions), - testCasesGen.WithWhereFilter_SomeResults(expectedSomeResultsAssertions), - testCasesGen.WithWhereFilter_NoResults(expectedNoResultsAssertsions), - } - - for _, tc := range testCases { - query := aggregateDuplicatesClassQuery(tc.filters, "groupBy: [\"booleans\"]") - - t.Run(tc.name, func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - extracted := extractDuplicatesClassGroupByResult(result) - - assert.Len(t, extracted, len(tc.groupedAssertions)) - for groupedBy, groupAssertions := range tc.groupedAssertions { - group := findGroup(groupedBy, extracted) - require.NotNil(t, group, fmt.Sprintf("Group '%s' not found", groupedBy)) - - for _, assertion := range groupAssertions { - assertion(group) - } - } - }) - } - }) -} - -func aggregateCityClassWithGroupByTest(t *testing.T) { - t.Run("aggregate City with group by city area", func(t *testing.T) { - asserts := newAggregateResponseAssert(t) - testCasesGen := &aggregateCityTestCases{} - - expectedAllResultsAssertions := map[string][]assertFunc{ - "891.96": { - asserts.groupedBy("891.96", "cityArea"), - asserts.meta(1), - asserts.number("cityArea", 1, 891.96, 891.96, 891.96, 891.96, 891.96, 891.96), - asserts.date("cityRights", 1), - asserts.text("history", 1, []string{historyBerlin}, []int64{1}), - asserts.boolean("isCapital", 1, 0, 1, 0, 1), - asserts.textArray("museums", 1, []string{"German Historical Museum"}, []int64{1}), - asserts.text("name", 1, []string{"Berlin"}, []int64{1}), - asserts.int("population", 1, 3470000, 3470000, 3470000, 3470000, 3470000, 3470000), - asserts.textArray("timezones", 2, []string{"CEST", "CET"}, []int64{1, 1}), - asserts.pointingTo("inCountry", "Country"), - }, - "891.95": { - asserts.groupedBy("891.95", "cityArea"), - asserts.meta(1), - asserts.number("cityArea", 1, 891.95, 891.95, 891.95, 891.95, 891.95, 891.95), - asserts.date("cityRights", 1), - asserts.text("history", 1, []string{historyAmsterdam}, []int64{1}), - asserts.boolean("isCapital", 1, 0, 1, 0, 1), - asserts.textArray("museums", 2, []string{"Rijksmuseum", "Stedelijk Museum"}, []int64{1, 1}), - asserts.text("name", 1, []string{"Amsterdam"}, []int64{1}), - asserts.int("population", 1, 1800000, 1800000, 1800000, 1800000, 1800000, 1800000), - asserts.textArray("timezones", 2, []string{"CEST", "CET"}, []int64{1, 1}), - asserts.pointingTo("inCountry", "Country"), - }, - "217.22": { - asserts.groupedBy("217.22", "cityArea"), - asserts.meta(1), - asserts.number("cityArea", 1, 217.22, 217.22, 217.22, 217.22, 217.22, 217.22), - asserts.date("cityRights", 1), - asserts.text("history", 1, []string{historyDusseldorf}, []int64{1}), - asserts.boolean("isCapital", 1, 1, 0, 1, 0), - asserts.textArray("museums", 3, []string{"Onomato", "Schiffahrt Museum", "Schlossturm"}, []int64{1, 1, 1}), - asserts.text("name", 1, []string{"Dusseldorf"}, []int64{1}), - asserts.int("population", 1, 600000, 600000, 600000, 600000, 600000, 600000), - asserts.textArray("timezones", 2, []string{"CEST", "CET"}, []int64{1, 1}), - asserts.pointingTo("inCountry", "Country"), - }, - "319.35": { - asserts.groupedBy("319.35", "cityArea"), - asserts.meta(1), - asserts.number("cityArea", 1, 319.35, 319.35, 319.35, 319.35, 319.35, 319.35), - asserts.date("cityRights", 1), - asserts.text("history", 1, []string{historyRotterdam}, []int64{1}), - asserts.boolean("isCapital", 1, 1, 0, 1, 0), - asserts.textArray("museums", 3, []string{"Museum Boijmans Van Beuningen", "Wereldmuseum", "Witte de With Center for Contemporary Art"}, []int64{1, 1, 1}), - asserts.text("name", 1, []string{"Rotterdam"}, []int64{1}), - asserts.int("population", 1, 600000, 600000, 600000, 600000, 600000, 600000), - asserts.textArray("timezones", 2, []string{"CEST", "CET"}, []int64{1, 1}), - asserts.pointingTo("inCountry", "Country"), - }, - } - expectedResultsWithDataAssertions := map[string][]assertFunc{ - "891.96": { - asserts.groupedBy("891.96", "cityArea"), - asserts.meta(1), - asserts.number("cityArea", 1, 891.96, 891.96, 891.96, 891.96, 891.96, 891.96), - asserts.date("cityRights", 1), - asserts.text("history", 1, []string{historyBerlin}, []int64{1}), - asserts.boolean("isCapital", 1, 0, 1, 0, 1), - asserts.textArray("museums", 1, []string{"German Historical Museum"}, []int64{1}), - asserts.text("name", 1, []string{"Berlin"}, []int64{1}), - asserts.int("population", 1, 3470000, 3470000, 3470000, 3470000, 3470000, 3470000), - asserts.textArray("timezones", 2, []string{"CEST", "CET"}, []int64{1, 1}), - asserts.pointingTo("inCountry", "Country"), - }, - "891.95": { - asserts.groupedBy("891.95", "cityArea"), - asserts.meta(1), - asserts.number("cityArea", 1, 891.95, 891.95, 891.95, 891.95, 891.95, 891.95), - asserts.date("cityRights", 1), - asserts.text("history", 1, []string{historyAmsterdam}, []int64{1}), - asserts.boolean("isCapital", 1, 0, 1, 0, 1), - asserts.textArray("museums", 2, []string{"Rijksmuseum", "Stedelijk Museum"}, []int64{1, 1}), - asserts.text("name", 1, []string{"Amsterdam"}, []int64{1}), - asserts.int("population", 1, 1800000, 1800000, 1800000, 1800000, 1800000, 1800000), - asserts.textArray("timezones", 2, []string{"CEST", "CET"}, []int64{1, 1}), - asserts.pointingTo("inCountry", "Country"), - }, - } - expectedResultsWithoutDataAssertions := map[string][]assertFunc{} - expectedNoResultsAssertions := map[string][]assertFunc{} - - testCases := []aggregateTestCase{ - testCasesGen.WithoutFilters(expectedAllResultsAssertions), - - testCasesGen.WithWhereFilter_AllResults(expectedAllResultsAssertions), - testCasesGen.WithWhereFilter_ResultsWithData(expectedResultsWithDataAssertions), - testCasesGen.WithWhereFilter_ResultsWithoutData(expectedResultsWithoutDataAssertions), - testCasesGen.WithWhereFilter_NoResults(expectedNoResultsAssertions), - - testCasesGen.WithNearObjectFilter_AllResults(expectedAllResultsAssertions), - testCasesGen.WithNearObjectFilter_ResultsWithData(expectedResultsWithDataAssertions), - testCasesGen.WithNearObjectFilter_ResultsWithoutData(expectedResultsWithoutDataAssertions), - - testCasesGen.WithWhereAndNearObjectFilters_AllResults(expectedAllResultsAssertions), - testCasesGen.WithWhereAndNearObjectFilters_ResultsWithData(expectedResultsWithDataAssertions), - testCasesGen.WithWhereAndNearObjectFilters_ResultsWithoutData(expectedResultsWithoutDataAssertions), - testCasesGen.WithWhereAndNearObjectFilters_NoResults(expectedNoResultsAssertions), - } - - for _, tc := range testCases { - query := aggregateCityQuery(tc.filters, "groupBy: [\"cityArea\"]") - - t.Run(tc.name, func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - extracted := extractCityGroupByResult(result) - - assert.Len(t, extracted, len(tc.groupedAssertions)) - for groupedBy, groupAssertions := range tc.groupedAssertions { - group := findGroup(groupedBy, extracted) - require.NotNil(t, group, fmt.Sprintf("Group '%s' not found", groupedBy)) - - for _, assertion := range groupAssertions { - assertion(group) - } - } - }) - } - }) - - t.Run("aggregate City with group by history", func(t *testing.T) { - asserts := newAggregateResponseAssert(t) - testCasesGen := &aggregateCityTestCases{} - - expectedAllResultsAssertions := map[string][]assertFunc{ - historyBerlin: { - asserts.groupedBy(historyBerlin, "history"), - asserts.meta(1), - asserts.number("cityArea", 1, 891.96, 891.96, 891.96, 891.96, 891.96, 891.96), - asserts.date("cityRights", 1), - asserts.text("history", 1, []string{historyBerlin}, []int64{1}), - asserts.boolean("isCapital", 1, 0, 1, 0, 1), - asserts.textArray("museums", 1, []string{"German Historical Museum"}, []int64{1}), - asserts.text("name", 1, []string{"Berlin"}, []int64{1}), - asserts.int("population", 1, 3470000, 3470000, 3470000, 3470000, 3470000, 3470000), - asserts.textArray("timezones", 2, []string{"CEST", "CET"}, []int64{1, 1}), - asserts.pointingTo("inCountry", "Country"), - }, - historyAmsterdam: { - asserts.groupedBy(historyAmsterdam, "history"), - asserts.meta(1), - asserts.number("cityArea", 1, 891.95, 891.95, 891.95, 891.95, 891.95, 891.95), - asserts.date("cityRights", 1), - asserts.text("history", 1, []string{historyAmsterdam}, []int64{1}), - asserts.boolean("isCapital", 1, 0, 1, 0, 1), - asserts.textArray("museums", 2, []string{"Rijksmuseum", "Stedelijk Museum"}, []int64{1, 1}), - asserts.text("name", 1, []string{"Amsterdam"}, []int64{1}), - asserts.int("population", 1, 1800000, 1800000, 1800000, 1800000, 1800000, 1800000), - asserts.textArray("timezones", 2, []string{"CEST", "CET"}, []int64{1, 1}), - asserts.pointingTo("inCountry", "Country"), - }, - historyDusseldorf: { - asserts.groupedBy(historyDusseldorf, "history"), - asserts.meta(1), - asserts.number("cityArea", 1, 217.22, 217.22, 217.22, 217.22, 217.22, 217.22), - asserts.date("cityRights", 1), - asserts.text("history", 1, []string{historyDusseldorf}, []int64{1}), - asserts.boolean("isCapital", 1, 1, 0, 1, 0), - asserts.textArray("museums", 3, []string{"Onomato", "Schiffahrt Museum", "Schlossturm"}, []int64{1, 1, 1}), - asserts.text("name", 1, []string{"Dusseldorf"}, []int64{1}), - asserts.int("population", 1, 600000, 600000, 600000, 600000, 600000, 600000), - asserts.textArray("timezones", 2, []string{"CEST", "CET"}, []int64{1, 1}), - asserts.pointingTo("inCountry", "Country"), - }, - historyRotterdam: { - asserts.groupedBy(historyRotterdam, "history"), - asserts.meta(1), - asserts.number("cityArea", 1, 319.35, 319.35, 319.35, 319.35, 319.35, 319.35), - asserts.date("cityRights", 1), - asserts.text("history", 1, []string{historyRotterdam}, []int64{1}), - asserts.boolean("isCapital", 1, 1, 0, 1, 0), - asserts.textArray("museums", 3, []string{"Museum Boijmans Van Beuningen", "Wereldmuseum", "Witte de With Center for Contemporary Art"}, []int64{1, 1, 1}), - asserts.text("name", 1, []string{"Rotterdam"}, []int64{1}), - asserts.int("population", 1, 600000, 600000, 600000, 600000, 600000, 600000), - asserts.textArray("timezones", 2, []string{"CEST", "CET"}, []int64{1, 1}), - asserts.pointingTo("inCountry", "Country"), - }, - } - expectedResultsWithDataAssertions := map[string][]assertFunc{ - historyBerlin: { - asserts.groupedBy(historyBerlin, "history"), - asserts.meta(1), - asserts.number("cityArea", 1, 891.96, 891.96, 891.96, 891.96, 891.96, 891.96), - asserts.date("cityRights", 1), - asserts.text("history", 1, []string{historyBerlin}, []int64{1}), - asserts.boolean("isCapital", 1, 0, 1, 0, 1), - asserts.textArray("museums", 1, []string{"German Historical Museum"}, []int64{1}), - asserts.text("name", 1, []string{"Berlin"}, []int64{1}), - asserts.int("population", 1, 3470000, 3470000, 3470000, 3470000, 3470000, 3470000), - asserts.textArray("timezones", 2, []string{"CEST", "CET"}, []int64{1, 1}), - asserts.pointingTo("inCountry", "Country"), - }, - historyAmsterdam: { - asserts.groupedBy(historyAmsterdam, "history"), - asserts.meta(1), - asserts.number("cityArea", 1, 891.95, 891.95, 891.95, 891.95, 891.95, 891.95), - asserts.date("cityRights", 1), - asserts.text("history", 1, []string{historyAmsterdam}, []int64{1}), - asserts.boolean("isCapital", 1, 0, 1, 0, 1), - asserts.textArray("museums", 2, []string{"Rijksmuseum", "Stedelijk Museum"}, []int64{1, 1}), - asserts.text("name", 1, []string{"Amsterdam"}, []int64{1}), - asserts.int("population", 1, 1800000, 1800000, 1800000, 1800000, 1800000, 1800000), - asserts.textArray("timezones", 2, []string{"CEST", "CET"}, []int64{1, 1}), - asserts.pointingTo("inCountry", "Country"), - }, - } - expectedResultsWithoutDataAssertions := map[string][]assertFunc{} - expectedNoResultsAssertions := map[string][]assertFunc{} - - testCases := []aggregateTestCase{ - testCasesGen.WithoutFilters(expectedAllResultsAssertions), - - testCasesGen.WithWhereFilter_AllResults(expectedAllResultsAssertions), - testCasesGen.WithWhereFilter_ResultsWithData(expectedResultsWithDataAssertions), - testCasesGen.WithWhereFilter_ResultsWithoutData(expectedResultsWithoutDataAssertions), - testCasesGen.WithWhereFilter_NoResults(expectedNoResultsAssertions), - - testCasesGen.WithNearObjectFilter_AllResults(expectedAllResultsAssertions), - testCasesGen.WithNearObjectFilter_ResultsWithData(expectedResultsWithDataAssertions), - testCasesGen.WithNearObjectFilter_ResultsWithoutData(expectedResultsWithoutDataAssertions), - - testCasesGen.WithWhereAndNearObjectFilters_AllResults(expectedAllResultsAssertions), - testCasesGen.WithWhereAndNearObjectFilters_ResultsWithData(expectedResultsWithDataAssertions), - testCasesGen.WithWhereAndNearObjectFilters_ResultsWithoutData(expectedResultsWithoutDataAssertions), - testCasesGen.WithWhereAndNearObjectFilters_NoResults(expectedNoResultsAssertions), - } - - for _, tc := range testCases { - query := aggregateCityQuery(tc.filters, "groupBy: [\"history\"]") - - t.Run(tc.name, func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - extracted := extractCityGroupByResult(result) - - assert.Len(t, extracted, len(tc.groupedAssertions)) - for groupedBy, groupAssertions := range tc.groupedAssertions { - group := findGroup(groupedBy, extracted) - require.NotNil(t, group, fmt.Sprintf("Group '%s' not found", groupedBy)) - - for _, assertion := range groupAssertions { - assertion(group) - } - } - }) - } - }) - - t.Run("aggregate City with group by name", func(t *testing.T) { - asserts := newAggregateResponseAssert(t) - testCasesGen := &aggregateCityTestCases{} - - expectedAllResultsAssertions := map[string][]assertFunc{ - "Berlin": { - asserts.groupedBy("Berlin", "name"), - asserts.meta(1), - asserts.number("cityArea", 1, 891.96, 891.96, 891.96, 891.96, 891.96, 891.96), - asserts.date("cityRights", 1), - asserts.text("history", 1, []string{historyBerlin}, []int64{1}), - asserts.boolean("isCapital", 1, 0, 1, 0, 1), - asserts.textArray("museums", 1, []string{"German Historical Museum"}, []int64{1}), - asserts.text("name", 1, []string{"Berlin"}, []int64{1}), - asserts.int("population", 1, 3470000, 3470000, 3470000, 3470000, 3470000, 3470000), - asserts.textArray("timezones", 2, []string{"CEST", "CET"}, []int64{1, 1}), - asserts.pointingTo("inCountry", "Country"), - }, - "Amsterdam": { - asserts.groupedBy("Amsterdam", "name"), - asserts.meta(1), - asserts.number("cityArea", 1, 891.95, 891.95, 891.95, 891.95, 891.95, 891.95), - asserts.date("cityRights", 1), - asserts.text("history", 1, []string{historyAmsterdam}, []int64{1}), - asserts.boolean("isCapital", 1, 0, 1, 0, 1), - asserts.textArray("museums", 2, []string{"Rijksmuseum", "Stedelijk Museum"}, []int64{1, 1}), - asserts.text("name", 1, []string{"Amsterdam"}, []int64{1}), - asserts.int("population", 1, 1800000, 1800000, 1800000, 1800000, 1800000, 1800000), - asserts.textArray("timezones", 2, []string{"CEST", "CET"}, []int64{1, 1}), - asserts.pointingTo("inCountry", "Country"), - }, - "Dusseldorf": { - asserts.groupedBy("Dusseldorf", "name"), - asserts.meta(1), - asserts.number("cityArea", 1, 217.22, 217.22, 217.22, 217.22, 217.22, 217.22), - asserts.date("cityRights", 1), - asserts.text("history", 1, []string{historyDusseldorf}, []int64{1}), - asserts.boolean("isCapital", 1, 1, 0, 1, 0), - asserts.textArray("museums", 3, []string{"Onomato", "Schiffahrt Museum", "Schlossturm"}, []int64{1, 1, 1}), - asserts.text("name", 1, []string{"Dusseldorf"}, []int64{1}), - asserts.int("population", 1, 600000, 600000, 600000, 600000, 600000, 600000), - asserts.textArray("timezones", 2, []string{"CEST", "CET"}, []int64{1, 1}), - asserts.pointingTo("inCountry", "Country"), - }, - "Rotterdam": { - asserts.groupedBy("Rotterdam", "name"), - asserts.meta(1), - asserts.number("cityArea", 1, 319.35, 319.35, 319.35, 319.35, 319.35, 319.35), - asserts.date("cityRights", 1), - asserts.text("history", 1, []string{historyRotterdam}, []int64{1}), - asserts.boolean("isCapital", 1, 1, 0, 1, 0), - asserts.textArray("museums", 3, []string{"Museum Boijmans Van Beuningen", "Wereldmuseum", "Witte de With Center for Contemporary Art"}, []int64{1, 1, 1}), - asserts.text("name", 1, []string{"Rotterdam"}, []int64{1}), - asserts.int("population", 1, 600000, 600000, 600000, 600000, 600000, 600000), - asserts.textArray("timezones", 2, []string{"CEST", "CET"}, []int64{1, 1}), - asserts.pointingTo("inCountry", "Country"), - }, - "Missing Island": { - asserts.groupedBy("Missing Island", "name"), - asserts.meta(1), - asserts.number0("cityArea"), - asserts.date0("cityRights"), - asserts.text0("history"), - asserts.boolean("isCapital", 1, 1, 0, 1, 0), - asserts.textArray0("museums"), - asserts.text("name", 1, []string{"Missing Island"}, []int64{1}), - asserts.int("population", 1, 0, 0, 0, 0, 0, 0), - asserts.textArray0("timezones"), - asserts.pointingTo("inCountry", "Country"), - }, - } - expectedResultsWithDataAssertions := map[string][]assertFunc{ - "Berlin": { - asserts.groupedBy("Berlin", "name"), - asserts.meta(1), - asserts.number("cityArea", 1, 891.96, 891.96, 891.96, 891.96, 891.96, 891.96), - asserts.date("cityRights", 1), - asserts.text("history", 1, []string{historyBerlin}, []int64{1}), - asserts.boolean("isCapital", 1, 0, 1, 0, 1), - asserts.textArray("museums", 1, []string{"German Historical Museum"}, []int64{1}), - asserts.text("name", 1, []string{"Berlin"}, []int64{1}), - asserts.int("population", 1, 3470000, 3470000, 3470000, 3470000, 3470000, 3470000), - asserts.textArray("timezones", 2, []string{"CEST", "CET"}, []int64{1, 1}), - asserts.pointingTo("inCountry", "Country"), - }, - "Amsterdam": { - asserts.groupedBy("Amsterdam", "name"), - asserts.meta(1), - asserts.number("cityArea", 1, 891.95, 891.95, 891.95, 891.95, 891.95, 891.95), - asserts.date("cityRights", 1), - asserts.text("history", 1, []string{historyAmsterdam}, []int64{1}), - asserts.boolean("isCapital", 1, 0, 1, 0, 1), - asserts.textArray("museums", 2, []string{"Rijksmuseum", "Stedelijk Museum"}, []int64{1, 1}), - asserts.text("name", 1, []string{"Amsterdam"}, []int64{1}), - asserts.int("population", 1, 1800000, 1800000, 1800000, 1800000, 1800000, 1800000), - asserts.textArray("timezones", 2, []string{"CEST", "CET"}, []int64{1, 1}), - asserts.pointingTo("inCountry", "Country"), - }, - } - expectedResultsWithoutDataAssertions := map[string][]assertFunc{} - expectedNoResultsAssertions := map[string][]assertFunc{} - - testCases := []aggregateTestCase{ - testCasesGen.WithoutFilters(expectedAllResultsAssertions), - - testCasesGen.WithWhereFilter_AllResults(expectedAllResultsAssertions), - testCasesGen.WithWhereFilter_ResultsWithData(expectedResultsWithDataAssertions), - testCasesGen.WithWhereFilter_ResultsWithoutData(expectedResultsWithoutDataAssertions), - testCasesGen.WithWhereFilter_NoResults(expectedNoResultsAssertions), - - testCasesGen.WithNearObjectFilter_AllResults(expectedAllResultsAssertions), - testCasesGen.WithNearObjectFilter_ResultsWithData(expectedResultsWithDataAssertions), - testCasesGen.WithNearObjectFilter_ResultsWithoutData(expectedResultsWithoutDataAssertions), - - testCasesGen.WithWhereAndNearObjectFilters_AllResults(expectedAllResultsAssertions), - testCasesGen.WithWhereAndNearObjectFilters_ResultsWithData(expectedResultsWithDataAssertions), - testCasesGen.WithWhereAndNearObjectFilters_ResultsWithoutData(expectedResultsWithoutDataAssertions), - testCasesGen.WithWhereAndNearObjectFilters_NoResults(expectedNoResultsAssertions), - } - - for _, tc := range testCases { - query := aggregateCityQuery(tc.filters, "groupBy: [\"name\"]") - - t.Run(tc.name, func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - extracted := extractCityGroupByResult(result) - - assert.Len(t, extracted, len(tc.groupedAssertions)) - for groupedBy, groupAssertions := range tc.groupedAssertions { - group := findGroup(groupedBy, extracted) - require.NotNil(t, group, fmt.Sprintf("Group '%s' not found", groupedBy)) - - for _, assertion := range groupAssertions { - assertion(group) - } - } - }) - } - }) - - t.Run("aggregate City with group by is capital", func(t *testing.T) { - asserts := newAggregateResponseAssert(t) - testCasesGen := &aggregateCityTestCases{} - - expectedAllResultsAssertions := map[string][]assertFunc{ - "true": { - asserts.groupedBy("true", "isCapital"), - asserts.meta(2), - asserts.number("cityArea", 2, 891.96, 891.95, 891.95, 1783.91, 891.955, 891.955), - asserts.date("cityRights", 2), - asserts.text("history", 2, []string{historyAmsterdam, historyBerlin}, []int64{1, 1}), - asserts.boolean("isCapital", 2, 0, 2, 0, 1), - asserts.textArray("museums", 3, []string{"German Historical Museum", "Rijksmuseum", "Stedelijk Museum"}, []int64{1, 1, 1}), - asserts.text("name", 2, []string{"Amsterdam", "Berlin"}, []int64{1, 1}), - asserts.int("population", 2, 3470000, 1800000, 1800000, 5270000, 2635000, 2635000), - asserts.textArray("timezones", 4, []string{"CEST", "CET"}, []int64{2, 2}), - asserts.pointingTo("inCountry", "Country"), - }, - "false": { - asserts.groupedBy("false", "isCapital"), - asserts.meta(3), - asserts.number("cityArea", 2, 319.35, 217.22, 217.22, 536.57, 268.285, 268.285), - asserts.date("cityRights", 2), - asserts.text("history", 2, []string{historyRotterdam, historyDusseldorf}, []int64{1, 1}), - asserts.boolean("isCapital", 3, 3, 0, 1, 0), - asserts.textArray("museums", 6, []string{"Museum Boijmans Van Beuningen", "Onomato", "Schiffahrt Museum", "Schlossturm", "Wereldmuseum"}, []int64{1, 1, 1, 1, 1}), - asserts.text("name", 3, []string{"Dusseldorf", "Missing Island", "Rotterdam"}, []int64{1, 1, 1}), - asserts.int("population", 3, 600000, 0, 600000, 1200000, 600000, 400000), - asserts.textArray("timezones", 4, []string{"CEST", "CET"}, []int64{2, 2}), - asserts.pointingTo("inCountry", "Country"), - }, - } - expectedResultsWithDataAssertions := map[string][]assertFunc{ - "true": { - asserts.groupedBy("true", "isCapital"), - asserts.meta(2), - asserts.number("cityArea", 2, 891.96, 891.95, 891.95, 1783.91, 891.955, 891.955), - asserts.date("cityRights", 2), - asserts.text("history", 2, []string{historyAmsterdam, historyBerlin}, []int64{1, 1}), - asserts.boolean("isCapital", 2, 0, 2, 0, 1), - asserts.textArray("museums", 3, []string{"German Historical Museum", "Rijksmuseum", "Stedelijk Museum"}, []int64{1, 1, 1}), - asserts.text("name", 2, []string{"Amsterdam", "Berlin"}, []int64{1, 1}), - asserts.int("population", 2, 3470000, 1800000, 1800000, 5270000, 2635000, 2635000), - asserts.textArray("timezones", 4, []string{"CEST", "CET"}, []int64{2, 2}), - asserts.pointingTo("inCountry", "Country"), - }, - } - expectedResultsWithoutDataAssertions := map[string][]assertFunc{} - expectedNoResultsAssertions := map[string][]assertFunc{} - - testCases := []aggregateTestCase{ - testCasesGen.WithoutFilters(expectedAllResultsAssertions), - - testCasesGen.WithWhereFilter_AllResults(expectedAllResultsAssertions), - testCasesGen.WithWhereFilter_ResultsWithData(expectedResultsWithDataAssertions), - testCasesGen.WithWhereFilter_ResultsWithoutData(expectedResultsWithoutDataAssertions), - testCasesGen.WithWhereFilter_NoResults(expectedNoResultsAssertions), - - testCasesGen.WithNearObjectFilter_AllResults(expectedAllResultsAssertions), - testCasesGen.WithNearObjectFilter_ResultsWithData(expectedResultsWithDataAssertions), - testCasesGen.WithNearObjectFilter_ResultsWithoutData(expectedResultsWithoutDataAssertions), - - testCasesGen.WithWhereAndNearObjectFilters_AllResults(expectedAllResultsAssertions), - testCasesGen.WithWhereAndNearObjectFilters_ResultsWithData(expectedResultsWithDataAssertions), - testCasesGen.WithWhereAndNearObjectFilters_ResultsWithoutData(expectedResultsWithoutDataAssertions), - testCasesGen.WithWhereAndNearObjectFilters_NoResults(expectedNoResultsAssertions), - } - - for _, tc := range testCases { - query := aggregateCityQuery(tc.filters, "groupBy: [\"isCapital\"]") - - t.Run(tc.name, func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - extracted := extractCityGroupByResult(result) - - assert.Len(t, extracted, len(tc.groupedAssertions)) - for groupedBy, groupAssertions := range tc.groupedAssertions { - group := findGroup(groupedBy, extracted) - require.NotNil(t, group, fmt.Sprintf("Group '%s' not found", groupedBy)) - - for _, assertion := range groupAssertions { - assertion(group) - } - } - }) - } - }) - - t.Run("aggregate City with group by name", func(t *testing.T) { - asserts := newAggregateResponseAssert(t) - testCasesGen := &aggregateCityTestCases{} - - expectedAllResultsAssertions := map[string][]assertFunc{ - "1400-01-01T00:00:00+02:00": { - asserts.groupedBy("1400-01-01T00:00:00+02:00", "cityRights"), - asserts.meta(2), - asserts.number("cityArea", 2, 891.96, 891.95, 891.95, 1783.91, 891.955, 891.955), - asserts.date("cityRights", 2), - asserts.text("history", 2, []string{historyAmsterdam, historyBerlin}, []int64{1, 1}), - asserts.boolean("isCapital", 2, 0, 2, 0, 1), - asserts.textArray("museums", 3, []string{"German Historical Museum", "Rijksmuseum", "Stedelijk Museum"}, []int64{1, 1, 1}), - asserts.text("name", 2, []string{"Amsterdam", "Berlin"}, []int64{1, 1}), - asserts.int("population", 2, 3470000, 1800000, 1800000, 5270000, 2635000, 2635000), - asserts.textArray("timezones", 4, []string{"CEST", "CET"}, []int64{2, 2}), - asserts.pointingTo("inCountry", "Country"), - }, - "1135-01-01T00:00:00+02:00": { - asserts.groupedBy("1135-01-01T00:00:00+02:00", "cityRights"), - asserts.meta(1), - asserts.number("cityArea", 1, 217.22, 217.22, 217.22, 217.22, 217.22, 217.22), - asserts.date("cityRights", 1), - asserts.text("history", 1, []string{historyDusseldorf}, []int64{1}), - asserts.boolean("isCapital", 1, 1, 0, 1, 0), - asserts.textArray("museums", 3, []string{"Onomato", "Schiffahrt Museum", "Schlossturm"}, []int64{1, 1, 1}), - asserts.text("name", 1, []string{"Dusseldorf"}, []int64{1}), - asserts.int("population", 1, 600000, 600000, 600000, 600000, 600000, 600000), - asserts.textArray("timezones", 2, []string{"CEST", "CET"}, []int64{1, 1}), - asserts.pointingTo("inCountry", "Country"), - }, - "1283-01-01T00:00:00+02:00": { - asserts.groupedBy("1283-01-01T00:00:00+02:00", "cityRights"), - asserts.meta(1), - asserts.number("cityArea", 1, 319.35, 319.35, 319.35, 319.35, 319.35, 319.35), - asserts.date("cityRights", 1), - asserts.text("history", 1, []string{historyRotterdam}, []int64{1}), - asserts.boolean("isCapital", 1, 1, 0, 1, 0), - asserts.textArray("museums", 3, []string{"Museum Boijmans Van Beuningen", "Wereldmuseum", "Witte de With Center for Contemporary Art"}, []int64{1, 1, 1}), - asserts.text("name", 1, []string{"Rotterdam"}, []int64{1}), - asserts.int("population", 1, 600000, 600000, 600000, 600000, 600000, 600000), - asserts.textArray("timezones", 2, []string{"CEST", "CET"}, []int64{1, 1}), - asserts.pointingTo("inCountry", "Country"), - }, - } - expectedResultsWithDataAssertions := map[string][]assertFunc{ - "1400-01-01T00:00:00+02:00": { - asserts.groupedBy("1400-01-01T00:00:00+02:00", "cityRights"), - asserts.meta(2), - asserts.number("cityArea", 2, 891.96, 891.95, 891.95, 1783.91, 891.955, 891.955), - asserts.date("cityRights", 2), - asserts.text("history", 2, []string{historyAmsterdam, historyBerlin}, []int64{1, 1}), - asserts.boolean("isCapital", 2, 0, 2, 0, 1), - asserts.textArray("museums", 3, []string{"German Historical Museum", "Rijksmuseum", "Stedelijk Museum"}, []int64{1, 1, 1}), - asserts.text("name", 2, []string{"Amsterdam", "Berlin"}, []int64{1, 1}), - asserts.int("population", 2, 3470000, 1800000, 1800000, 5270000, 2635000, 2635000), - asserts.textArray("timezones", 4, []string{"CEST", "CET"}, []int64{2, 2}), - asserts.pointingTo("inCountry", "Country"), - }, - } - expectedResultsWithoutDataAssertions := map[string][]assertFunc{} - expectedNoResultsAssertions := map[string][]assertFunc{} - - testCases := []aggregateTestCase{ - testCasesGen.WithoutFilters(expectedAllResultsAssertions), - - testCasesGen.WithWhereFilter_AllResults(expectedAllResultsAssertions), - testCasesGen.WithWhereFilter_ResultsWithData(expectedResultsWithDataAssertions), - testCasesGen.WithWhereFilter_ResultsWithoutData(expectedResultsWithoutDataAssertions), - testCasesGen.WithWhereFilter_NoResults(expectedNoResultsAssertions), - - testCasesGen.WithNearObjectFilter_AllResults(expectedAllResultsAssertions), - testCasesGen.WithNearObjectFilter_ResultsWithData(expectedResultsWithDataAssertions), - testCasesGen.WithNearObjectFilter_ResultsWithoutData(expectedResultsWithoutDataAssertions), - - testCasesGen.WithWhereAndNearObjectFilters_AllResults(expectedAllResultsAssertions), - testCasesGen.WithWhereAndNearObjectFilters_ResultsWithData(expectedResultsWithDataAssertions), - testCasesGen.WithWhereAndNearObjectFilters_ResultsWithoutData(expectedResultsWithoutDataAssertions), - testCasesGen.WithWhereAndNearObjectFilters_NoResults(expectedNoResultsAssertions), - } - - for _, tc := range testCases { - query := aggregateCityQuery(tc.filters, "groupBy: [\"cityRights\"]") - - t.Run(tc.name, func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - extracted := extractCityGroupByResult(result) - - assert.Len(t, extracted, len(tc.groupedAssertions)) - for groupedBy, groupAssertions := range tc.groupedAssertions { - group := findGroup(groupedBy, extracted) - require.NotNil(t, group, fmt.Sprintf("Group '%s' not found", groupedBy)) - - for _, assertion := range groupAssertions { - assertion(group) - } - } - }) - } - }) - - t.Run("aggregate City with group by population", func(t *testing.T) { - asserts := newAggregateResponseAssert(t) - testCasesGen := &aggregateCityTestCases{} - - expectedAllResultsAssertions := map[string][]assertFunc{ - "600000": { - asserts.groupedBy("600000", "population"), - asserts.meta(2), - asserts.number("cityArea", 2, 319.35, 217.22, 217.22, 536.57, 268.285, 268.285), - asserts.date("cityRights", 2), - asserts.text("history", 2, []string{historyRotterdam, historyDusseldorf}, []int64{1, 1}), - asserts.boolean("isCapital", 2, 2, 0, 1, 0), - asserts.textArray("museums", 6, []string{"Museum Boijmans Van Beuningen", "Onomato", "Schiffahrt Museum", "Schlossturm", "Wereldmuseum"}, []int64{1, 1, 1, 1, 1}), - asserts.text("name", 2, []string{"Dusseldorf", "Rotterdam"}, []int64{1, 1}), - asserts.int("population", 2, 600000, 600000, 600000, 1200000, 600000, 600000), - asserts.textArray("timezones", 4, []string{"CEST", "CET"}, []int64{2, 2}), - asserts.pointingTo("inCountry", "Country"), - }, - "3.47e+06": { - asserts.groupedBy("3.47e+06", "population"), - asserts.meta(1), - asserts.number("cityArea", 1, 891.96, 891.96, 891.96, 891.96, 891.96, 891.96), - asserts.date("cityRights", 1), - asserts.text("history", 1, []string{historyBerlin}, []int64{1}), - asserts.boolean("isCapital", 1, 0, 1, 0, 1), - asserts.textArray("museums", 1, []string{"German Historical Museum"}, []int64{1}), - asserts.text("name", 1, []string{"Berlin"}, []int64{1}), - asserts.int("population", 1, 3470000, 3470000, 3470000, 3470000, 3470000, 3470000), - asserts.textArray("timezones", 2, []string{"CEST", "CET"}, []int64{1, 1}), - asserts.pointingTo("inCountry", "Country"), - }, - "1.8e+06": { - asserts.groupedBy("1.8e+06", "population"), - asserts.meta(1), - asserts.number("cityArea", 1, 891.95, 891.95, 891.95, 891.95, 891.95, 891.95), - asserts.date("cityRights", 1), - asserts.text("history", 1, []string{historyAmsterdam}, []int64{1}), - asserts.boolean("isCapital", 1, 0, 1, 0, 1), - asserts.textArray("museums", 2, []string{"Rijksmuseum", "Stedelijk Museum"}, []int64{1, 1}), - asserts.text("name", 1, []string{"Amsterdam"}, []int64{1}), - asserts.int("population", 1, 1800000, 1800000, 1800000, 1800000, 1800000, 1800000), - asserts.textArray("timezones", 2, []string{"CEST", "CET"}, []int64{1, 1}), - asserts.pointingTo("inCountry", "Country"), - }, - "0": { - asserts.groupedBy("0", "population"), - asserts.meta(1), - asserts.number0("cityArea"), - asserts.date0("cityRights"), - asserts.text0("history"), - asserts.boolean("isCapital", 1, 1, 0, 1, 0), - asserts.textArray0("museums"), - asserts.text("name", 1, []string{"Missing Island"}, []int64{1}), - asserts.int("population", 1, 0, 0, 0, 0, 0, 0), - asserts.textArray0("timezones"), - asserts.pointingTo("inCountry", "Country"), - }, - } - expectedResultsWithDataAssertions := map[string][]assertFunc{ - "3.47e+06": { - asserts.groupedBy("3.47e+06", "population"), - asserts.meta(1), - asserts.number("cityArea", 1, 891.96, 891.96, 891.96, 891.96, 891.96, 891.96), - asserts.date("cityRights", 1), - asserts.text("history", 1, []string{historyBerlin}, []int64{1}), - asserts.boolean("isCapital", 1, 0, 1, 0, 1), - asserts.textArray("museums", 1, []string{"German Historical Museum"}, []int64{1}), - asserts.text("name", 1, []string{"Berlin"}, []int64{1}), - asserts.int("population", 1, 3470000, 3470000, 3470000, 3470000, 3470000, 3470000), - asserts.textArray("timezones", 2, []string{"CEST", "CET"}, []int64{1, 1}), - asserts.pointingTo("inCountry", "Country"), - }, - "1.8e+06": { - asserts.groupedBy("1.8e+06", "population"), - asserts.meta(1), - asserts.number("cityArea", 1, 891.95, 891.95, 891.95, 891.95, 891.95, 891.95), - asserts.date("cityRights", 1), - asserts.text("history", 1, []string{historyAmsterdam}, []int64{1}), - asserts.boolean("isCapital", 1, 0, 1, 0, 1), - asserts.textArray("museums", 2, []string{"Rijksmuseum", "Stedelijk Museum"}, []int64{1, 1}), - asserts.text("name", 1, []string{"Amsterdam"}, []int64{1}), - asserts.int("population", 1, 1800000, 1800000, 1800000, 1800000, 1800000, 1800000), - asserts.textArray("timezones", 2, []string{"CEST", "CET"}, []int64{1, 1}), - asserts.pointingTo("inCountry", "Country"), - }, - } - expectedResultsWithoutDataAssertions := map[string][]assertFunc{} - expectedNoResultsAssertions := map[string][]assertFunc{} - - testCases := []aggregateTestCase{ - testCasesGen.WithoutFilters(expectedAllResultsAssertions), - - testCasesGen.WithWhereFilter_AllResults(expectedAllResultsAssertions), - testCasesGen.WithWhereFilter_ResultsWithData(expectedResultsWithDataAssertions), - testCasesGen.WithWhereFilter_ResultsWithoutData(expectedResultsWithoutDataAssertions), - testCasesGen.WithWhereFilter_NoResults(expectedNoResultsAssertions), - - testCasesGen.WithNearObjectFilter_AllResults(expectedAllResultsAssertions), - testCasesGen.WithNearObjectFilter_ResultsWithData(expectedResultsWithDataAssertions), - testCasesGen.WithNearObjectFilter_ResultsWithoutData(expectedResultsWithoutDataAssertions), - - testCasesGen.WithWhereAndNearObjectFilters_AllResults(expectedAllResultsAssertions), - testCasesGen.WithWhereAndNearObjectFilters_ResultsWithData(expectedResultsWithDataAssertions), - testCasesGen.WithWhereAndNearObjectFilters_ResultsWithoutData(expectedResultsWithoutDataAssertions), - testCasesGen.WithWhereAndNearObjectFilters_NoResults(expectedNoResultsAssertions), - } - - for _, tc := range testCases { - query := aggregateCityQuery(tc.filters, "groupBy: [\"population\"]") - - t.Run(tc.name, func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - extracted := extractCityGroupByResult(result) - - assert.Len(t, extracted, len(tc.groupedAssertions)) - for groupedBy, groupAssertions := range tc.groupedAssertions { - group := findGroup(groupedBy, extracted) - require.NotNil(t, group, fmt.Sprintf("Group '%s' not found", groupedBy)) - - for _, assertion := range groupAssertions { - assertion(group) - } - } - }) - } - }) - - t.Run("aggregate City with group by timezones", func(t *testing.T) { - asserts := newAggregateResponseAssert(t) - testCasesGen := &aggregateCityTestCases{} - - expectedAllResultsAssertions := map[string][]assertFunc{ - "CEST": { - asserts.groupedBy("CEST", "timezones"), - asserts.meta(4), - asserts.number("cityArea", 4, 891.96, 217.22, 217.22, 2320.48, 605.6500000000001, 580.12), - asserts.date("cityRights", 4), - asserts.text("history", 4, []string{historyAmsterdam, historyRotterdam, historyBerlin, historyDusseldorf}, []int64{1, 1, 1, 1}), - asserts.boolean("isCapital", 4, 2, 2, 0.5, 0.5), - asserts.textArray("museums", 9, []string{"German Historical Museum", "Museum Boijmans Van Beuningen", "Onomato", "Rijksmuseum", "Schiffahrt Museum"}, []int64{1, 1, 1, 1, 1}), - asserts.text("name", 4, []string{"Amsterdam", "Berlin", "Dusseldorf", "Rotterdam"}, []int64{1, 1, 1, 1}), - asserts.int("population", 4, 3470000, 600000, 600000, 6470000, 1200000, 1617500), - asserts.textArray("timezones", 8, []string{"CEST", "CET"}, []int64{4, 4}), - asserts.pointingTo("inCountry", "Country"), - }, - "CET": { - asserts.groupedBy("CET", "timezones"), - asserts.meta(4), - asserts.number("cityArea", 4, 891.96, 217.22, 217.22, 2320.48, 605.6500000000001, 580.12), - asserts.date("cityRights", 4), - asserts.text("history", 4, []string{historyAmsterdam, historyRotterdam, historyBerlin, historyDusseldorf}, []int64{1, 1, 1, 1}), - asserts.boolean("isCapital", 4, 2, 2, 0.5, 0.5), - asserts.textArray("museums", 9, []string{"German Historical Museum", "Museum Boijmans Van Beuningen", "Onomato", "Rijksmuseum", "Schiffahrt Museum"}, []int64{1, 1, 1, 1, 1}), - asserts.text("name", 4, []string{"Amsterdam", "Berlin", "Dusseldorf", "Rotterdam"}, []int64{1, 1, 1, 1}), - asserts.int("population", 4, 3470000, 600000, 600000, 6470000, 1200000, 1617500), - asserts.textArray("timezones", 8, []string{"CEST", "CET"}, []int64{4, 4}), - asserts.pointingTo("inCountry", "Country"), - }, - } - expectedResultsWithDataAssertions := map[string][]assertFunc{ - "CEST": { - asserts.groupedBy("CEST", "timezones"), - asserts.meta(2), - asserts.number("cityArea", 2, 891.96, 891.95, 891.95, 1783.91, 891.955, 891.955), - asserts.date("cityRights", 2), - asserts.text("history", 2, []string{historyAmsterdam, historyBerlin}, []int64{1, 1}), - asserts.boolean("isCapital", 2, 0, 2, 0, 1), - asserts.textArray("museums", 3, []string{"German Historical Museum", "Rijksmuseum", "Stedelijk Museum"}, []int64{1, 1, 1}), - asserts.text("name", 2, []string{"Amsterdam", "Berlin"}, []int64{1, 1}), - asserts.int("population", 2, 3470000, 1800000, 1800000, 5270000, 2635000, 2635000), - asserts.textArray("timezones", 4, []string{"CEST", "CET"}, []int64{2, 2}), - asserts.pointingTo("inCountry", "Country"), - }, - "CET": { - asserts.groupedBy("CET", "timezones"), - asserts.meta(2), - asserts.number("cityArea", 2, 891.96, 891.95, 891.95, 1783.91, 891.955, 891.955), - asserts.date("cityRights", 2), - asserts.text("history", 2, []string{historyAmsterdam, historyBerlin}, []int64{1, 1}), - asserts.boolean("isCapital", 2, 0, 2, 0, 1), - asserts.textArray("museums", 3, []string{"German Historical Museum", "Rijksmuseum", "Stedelijk Museum"}, []int64{1, 1, 1}), - asserts.text("name", 2, []string{"Amsterdam", "Berlin"}, []int64{1, 1}), - asserts.int("population", 2, 3470000, 1800000, 1800000, 5270000, 2635000, 2635000), - asserts.textArray("timezones", 4, []string{"CEST", "CET"}, []int64{2, 2}), - asserts.pointingTo("inCountry", "Country"), - }, - } - expectedResultsWithoutDataAssertions := map[string][]assertFunc{} - expectedNoResultsAssertions := map[string][]assertFunc{} - - testCases := []aggregateTestCase{ - testCasesGen.WithoutFilters(expectedAllResultsAssertions), - - testCasesGen.WithWhereFilter_AllResults(expectedAllResultsAssertions), - testCasesGen.WithWhereFilter_ResultsWithData(expectedResultsWithDataAssertions), - testCasesGen.WithWhereFilter_ResultsWithoutData(expectedResultsWithoutDataAssertions), - testCasesGen.WithWhereFilter_NoResults(expectedNoResultsAssertions), - - testCasesGen.WithNearObjectFilter_AllResults(expectedAllResultsAssertions), - testCasesGen.WithNearObjectFilter_ResultsWithData(expectedResultsWithDataAssertions), - testCasesGen.WithNearObjectFilter_ResultsWithoutData(expectedResultsWithoutDataAssertions), - - testCasesGen.WithWhereAndNearObjectFilters_AllResults(expectedAllResultsAssertions), - testCasesGen.WithWhereAndNearObjectFilters_ResultsWithData(expectedResultsWithDataAssertions), - testCasesGen.WithWhereAndNearObjectFilters_ResultsWithoutData(expectedResultsWithoutDataAssertions), - testCasesGen.WithWhereAndNearObjectFilters_NoResults(expectedNoResultsAssertions), - } - - for _, tc := range testCases { - query := aggregateCityQuery(tc.filters, "groupBy: [\"timezones\"]") - - t.Run(tc.name, func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - extracted := extractCityGroupByResult(result) - - assert.Len(t, extracted, len(tc.groupedAssertions)) - for groupedBy, groupAssertions := range tc.groupedAssertions { - group := findGroup(groupedBy, extracted) - require.NotNil(t, group, fmt.Sprintf("Group '%s' not found", groupedBy)) - - for _, assertion := range groupAssertions { - assertion(group) - } - } - }) - } - }) - - t.Run("aggregate City with group by museums", func(t *testing.T) { - asserts := newAggregateResponseAssert(t) - testCasesGen := &aggregateCityTestCases{} - - expectedAllResultsAssertions := map[string][]assertFunc{ - "German Historical Museum": { - asserts.groupedBy("German Historical Museum", "museums"), - asserts.meta(1), - asserts.number("cityArea", 1, 891.96, 891.96, 891.96, 891.96, 891.96, 891.96), - asserts.date("cityRights", 1), - asserts.text("history", 1, []string{historyBerlin}, []int64{1}), - asserts.boolean("isCapital", 1, 0, 1, 0, 1), - asserts.textArray("museums", 1, []string{"German Historical Museum"}, []int64{1}), - asserts.text("name", 1, []string{"Berlin"}, []int64{1}), - asserts.int("population", 1, 3470000, 3470000, 3470000, 3470000, 3470000, 3470000), - asserts.textArray("timezones", 2, []string{"CEST", "CET"}, []int64{1, 1}), - asserts.pointingTo("inCountry", "Country"), - }, - "Rijksmuseum": { - asserts.groupedBy("Rijksmuseum", "museums"), - asserts.meta(1), - asserts.number("cityArea", 1, 891.95, 891.95, 891.95, 891.95, 891.95, 891.95), - asserts.date("cityRights", 1), - asserts.text("history", 1, []string{historyAmsterdam}, []int64{1}), - asserts.boolean("isCapital", 1, 0, 1, 0, 1), - asserts.textArray("museums", 2, []string{"Rijksmuseum", "Stedelijk Museum"}, []int64{1, 1}), - asserts.text("name", 1, []string{"Amsterdam"}, []int64{1}), - asserts.int("population", 1, 1800000, 1800000, 1800000, 1800000, 1800000, 1800000), - asserts.textArray("timezones", 2, []string{"CEST", "CET"}, []int64{1, 1}), - asserts.pointingTo("inCountry", "Country"), - }, - "Stedelijk Museum": { - asserts.groupedBy("Stedelijk Museum", "museums"), - asserts.meta(1), - asserts.number("cityArea", 1, 891.95, 891.95, 891.95, 891.95, 891.95, 891.95), - asserts.date("cityRights", 1), - asserts.text("history", 1, []string{historyAmsterdam}, []int64{1}), - asserts.boolean("isCapital", 1, 0, 1, 0, 1), - asserts.textArray("museums", 2, []string{"Rijksmuseum", "Stedelijk Museum"}, []int64{1, 1}), - asserts.text("name", 1, []string{"Amsterdam"}, []int64{1}), - asserts.int("population", 1, 1800000, 1800000, 1800000, 1800000, 1800000, 1800000), - asserts.textArray("timezones", 2, []string{"CEST", "CET"}, []int64{1, 1}), - asserts.pointingTo("inCountry", "Country"), - }, - "Onomato": { - asserts.groupedBy("Onomato", "museums"), - asserts.meta(1), - asserts.number("cityArea", 1, 217.22, 217.22, 217.22, 217.22, 217.22, 217.22), - asserts.date("cityRights", 1), - asserts.text("history", 1, []string{historyDusseldorf}, []int64{1}), - asserts.boolean("isCapital", 1, 1, 0, 1, 0), - asserts.textArray("museums", 3, []string{"Onomato", "Schiffahrt Museum", "Schlossturm"}, []int64{1, 1, 1}), - asserts.text("name", 1, []string{"Dusseldorf"}, []int64{1}), - asserts.int("population", 1, 600000, 600000, 600000, 600000, 600000, 600000), - asserts.textArray("timezones", 2, []string{"CEST", "CET"}, []int64{1, 1}), - asserts.pointingTo("inCountry", "Country"), - }, - "Schiffahrt Museum": { - asserts.groupedBy("Schiffahrt Museum", "museums"), - asserts.meta(1), - asserts.number("cityArea", 1, 217.22, 217.22, 217.22, 217.22, 217.22, 217.22), - asserts.date("cityRights", 1), - asserts.text("history", 1, []string{historyDusseldorf}, []int64{1}), - asserts.boolean("isCapital", 1, 1, 0, 1, 0), - asserts.textArray("museums", 3, []string{"Onomato", "Schiffahrt Museum", "Schlossturm"}, []int64{1, 1, 1}), - asserts.text("name", 1, []string{"Dusseldorf"}, []int64{1}), - asserts.int("population", 1, 600000, 600000, 600000, 600000, 600000, 600000), - asserts.textArray("timezones", 2, []string{"CEST", "CET"}, []int64{1, 1}), - asserts.pointingTo("inCountry", "Country"), - }, - "Schlossturm": { - asserts.groupedBy("Schlossturm", "museums"), - asserts.meta(1), - asserts.number("cityArea", 1, 217.22, 217.22, 217.22, 217.22, 217.22, 217.22), - asserts.date("cityRights", 1), - asserts.text("history", 1, []string{historyDusseldorf}, []int64{1}), - asserts.boolean("isCapital", 1, 1, 0, 1, 0), - asserts.textArray("museums", 3, []string{"Onomato", "Schiffahrt Museum", "Schlossturm"}, []int64{1, 1, 1}), - asserts.text("name", 1, []string{"Dusseldorf"}, []int64{1}), - asserts.int("population", 1, 600000, 600000, 600000, 600000, 600000, 600000), - asserts.textArray("timezones", 2, []string{"CEST", "CET"}, []int64{1, 1}), - asserts.pointingTo("inCountry", "Country"), - }, - "Museum Boijmans Van Beuningen": { - asserts.groupedBy("Museum Boijmans Van Beuningen", "museums"), - asserts.meta(1), - asserts.number("cityArea", 1, 319.35, 319.35, 319.35, 319.35, 319.35, 319.35), - asserts.date("cityRights", 1), - asserts.text("history", 1, []string{historyRotterdam}, []int64{1}), - asserts.boolean("isCapital", 1, 1, 0, 1, 0), - asserts.textArray("museums", 3, []string{"Museum Boijmans Van Beuningen", "Wereldmuseum", "Witte de With Center for Contemporary Art"}, []int64{1, 1, 1}), - asserts.text("name", 1, []string{"Rotterdam"}, []int64{1}), - asserts.int("population", 1, 600000, 600000, 600000, 600000, 600000, 600000), - asserts.textArray("timezones", 2, []string{"CEST", "CET"}, []int64{1, 1}), - asserts.pointingTo("inCountry", "Country"), - }, - "Wereldmuseum": { - asserts.groupedBy("Wereldmuseum", "museums"), - asserts.meta(1), - asserts.number("cityArea", 1, 319.35, 319.35, 319.35, 319.35, 319.35, 319.35), - asserts.date("cityRights", 1), - asserts.text("history", 1, []string{historyRotterdam}, []int64{1}), - asserts.boolean("isCapital", 1, 1, 0, 1, 0), - asserts.textArray("museums", 3, []string{"Museum Boijmans Van Beuningen", "Wereldmuseum", "Witte de With Center for Contemporary Art"}, []int64{1, 1, 1}), - asserts.text("name", 1, []string{"Rotterdam"}, []int64{1}), - asserts.int("population", 1, 600000, 600000, 600000, 600000, 600000, 600000), - asserts.textArray("timezones", 2, []string{"CEST", "CET"}, []int64{1, 1}), - asserts.pointingTo("inCountry", "Country"), - }, - "Witte de With Center for Contemporary Art": { - asserts.groupedBy("Witte de With Center for Contemporary Art", "museums"), - asserts.meta(1), - asserts.number("cityArea", 1, 319.35, 319.35, 319.35, 319.35, 319.35, 319.35), - asserts.date("cityRights", 1), - asserts.text("history", 1, []string{historyRotterdam}, []int64{1}), - asserts.boolean("isCapital", 1, 1, 0, 1, 0), - asserts.textArray("museums", 3, []string{"Museum Boijmans Van Beuningen", "Wereldmuseum", "Witte de With Center for Contemporary Art"}, []int64{1, 1, 1}), - asserts.text("name", 1, []string{"Rotterdam"}, []int64{1}), - asserts.int("population", 1, 600000, 600000, 600000, 600000, 600000, 600000), - asserts.textArray("timezones", 2, []string{"CEST", "CET"}, []int64{1, 1}), - asserts.pointingTo("inCountry", "Country"), - }, - } - expectedResultsWithDataAssertions := map[string][]assertFunc{ - "German Historical Museum": { - asserts.groupedBy("German Historical Museum", "museums"), - asserts.meta(1), - asserts.number("cityArea", 1, 891.96, 891.96, 891.96, 891.96, 891.96, 891.96), - asserts.date("cityRights", 1), - asserts.text("history", 1, []string{historyBerlin}, []int64{1}), - asserts.boolean("isCapital", 1, 0, 1, 0, 1), - asserts.textArray("museums", 1, []string{"German Historical Museum"}, []int64{1}), - asserts.text("name", 1, []string{"Berlin"}, []int64{1}), - asserts.int("population", 1, 3470000, 3470000, 3470000, 3470000, 3470000, 3470000), - asserts.textArray("timezones", 2, []string{"CEST", "CET"}, []int64{1, 1}), - asserts.pointingTo("inCountry", "Country"), - }, - "Rijksmuseum": { - asserts.groupedBy("Rijksmuseum", "museums"), - asserts.meta(1), - asserts.number("cityArea", 1, 891.95, 891.95, 891.95, 891.95, 891.95, 891.95), - asserts.date("cityRights", 1), - asserts.text("history", 1, []string{historyAmsterdam}, []int64{1}), - asserts.boolean("isCapital", 1, 0, 1, 0, 1), - asserts.textArray("museums", 2, []string{"Rijksmuseum", "Stedelijk Museum"}, []int64{1, 1}), - asserts.text("name", 1, []string{"Amsterdam"}, []int64{1}), - asserts.int("population", 1, 1800000, 1800000, 1800000, 1800000, 1800000, 1800000), - asserts.textArray("timezones", 2, []string{"CEST", "CET"}, []int64{1, 1}), - asserts.pointingTo("inCountry", "Country"), - }, - "Stedelijk Museum": { - asserts.groupedBy("Stedelijk Museum", "museums"), - asserts.meta(1), - asserts.number("cityArea", 1, 891.95, 891.95, 891.95, 891.95, 891.95, 891.95), - asserts.date("cityRights", 1), - asserts.text("history", 1, []string{historyAmsterdam}, []int64{1}), - asserts.boolean("isCapital", 1, 0, 1, 0, 1), - asserts.textArray("museums", 2, []string{"Rijksmuseum", "Stedelijk Museum"}, []int64{1, 1}), - asserts.text("name", 1, []string{"Amsterdam"}, []int64{1}), - asserts.int("population", 1, 1800000, 1800000, 1800000, 1800000, 1800000, 1800000), - asserts.textArray("timezones", 2, []string{"CEST", "CET"}, []int64{1, 1}), - asserts.pointingTo("inCountry", "Country"), - }, - } - expectedResultsWithoutDataAssertions := map[string][]assertFunc{} - expectedNoResultsAssertions := map[string][]assertFunc{} - - testCases := []aggregateTestCase{ - testCasesGen.WithoutFilters(expectedAllResultsAssertions), - - testCasesGen.WithWhereFilter_AllResults(expectedAllResultsAssertions), - testCasesGen.WithWhereFilter_ResultsWithData(expectedResultsWithDataAssertions), - testCasesGen.WithWhereFilter_ResultsWithoutData(expectedResultsWithoutDataAssertions), - testCasesGen.WithWhereFilter_NoResults(expectedNoResultsAssertions), - - testCasesGen.WithNearObjectFilter_AllResults(expectedAllResultsAssertions), - testCasesGen.WithNearObjectFilter_ResultsWithData(expectedResultsWithDataAssertions), - testCasesGen.WithNearObjectFilter_ResultsWithoutData(expectedResultsWithoutDataAssertions), - - testCasesGen.WithWhereAndNearObjectFilters_AllResults(expectedAllResultsAssertions), - testCasesGen.WithWhereAndNearObjectFilters_ResultsWithData(expectedResultsWithDataAssertions), - testCasesGen.WithWhereAndNearObjectFilters_ResultsWithoutData(expectedResultsWithoutDataAssertions), - testCasesGen.WithWhereAndNearObjectFilters_NoResults(expectedNoResultsAssertions), - } - - for _, tc := range testCases { - query := aggregateCityQuery(tc.filters, "groupBy: [\"museums\"]") - - t.Run(tc.name, func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - extracted := extractCityGroupByResult(result) - - assert.Len(t, extracted, len(tc.groupedAssertions)) - for groupedBy, groupAssertions := range tc.groupedAssertions { - group := findGroup(groupedBy, extracted) - require.NotNil(t, group, fmt.Sprintf("Group '%s' not found", groupedBy)) - - for _, assertion := range groupAssertions { - assertion(group) - } - } - }) - } - }) -} - -func extractArrayClassGroupByResult(result *graphqlhelper.GraphQLResult) []interface{} { - return extractAggregateResult(result, arrayClassName) -} - -func extractDuplicatesClassGroupByResult(result *graphqlhelper.GraphQLResult) []interface{} { - return extractAggregateResult(result, duplicatesClassName) -} - -func extractCityGroupByResult(result *graphqlhelper.GraphQLResult) []interface{} { - return extractAggregateResult(result, cityClassName) -} - -func findGroup(groupedBy string, groups []interface{}) map[string]interface{} { - for i := range groups { - if group, ok := groups[i].(map[string]interface{}); !ok { - continue - } else if gb, exists := group["groupedBy"]; !exists { - continue - } else if gbm, ok := gb.(map[string]interface{}); !ok { - continue - } else if value, exists := gbm["value"]; !exists { - continue - } else if groupedByValue, ok := value.(string); !ok { - continue - } else if groupedByValue == groupedBy { - return group - } - } - return nil -} diff --git a/test/acceptance/graphql_resolvers/local_aggregate_matrix_no_groupby_test.go b/test/acceptance/graphql_resolvers/local_aggregate_matrix_no_groupby_test.go deleted file mode 100644 index 4046fdedb50138f204773ca985ba9c418d9f7a8d..0000000000000000000000000000000000000000 --- a/test/acceptance/graphql_resolvers/local_aggregate_matrix_no_groupby_test.go +++ /dev/null @@ -1,306 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "testing" - - "github.com/weaviate/weaviate/test/helper" - graphqlhelper "github.com/weaviate/weaviate/test/helper/graphql" -) - -func aggregateArrayClassWithoutGroupByTest(t *testing.T) { - t.Run("aggregate ArrayClass without group by", func(t *testing.T) { - asserts := newAggregateResponseAssert(t) - testCasesGen := &aggregateArrayClassTestCases{} - - expectedAllResultsAssertions := []assertFunc{ - asserts.meta(7), - asserts.booleanArray("booleans", 10, 4, 6, 0.4, 0.6), - asserts.textArray("texts", 10, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{4, 3, 2, 1}), - asserts.numberArray("numbers", 10, 4, 1, 1, 20, 2, 2), - asserts.intArray("ints", 10, 104, 101, 101, 1020, 102, 102), - asserts.dateArray("datesAsStrings", 10), - asserts.dateArray("dates", 10), - } - expectedResultsWithDataAssertions := []assertFunc{ - asserts.meta(2), - asserts.booleanArray("booleans", 7, 2, 5, 0.2857142857142857, 0.7142857142857143), - asserts.textArray("texts", 7, []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, []int64{2, 2, 2, 1}), - asserts.numberArray("numbers", 7, 4, 1, 1, 16, 2, 2.2857142857142856), - asserts.intArray("ints", 7, 104, 101, 101, 716, 102, 102.28571428571429), - asserts.dateArray("datesAsStrings", 7), - asserts.dateArray("dates", 7), - } - expectedResultsWithoutDataAssertions := []assertFunc{ - asserts.meta(3), - asserts.booleanArray0("booleans"), - asserts.textArray0("texts"), - asserts.numberArray0("numbers"), - asserts.intArray0("ints"), - asserts.dateArray0("datesAsStrings"), - asserts.dateArray0("dates"), - } - expectedNoResultsAssertions := []assertFunc{ - asserts.meta(0), - asserts.booleanArray0("booleans"), - asserts.textArray0("texts"), - asserts.numberArray0("numbers"), - asserts.intArray0("ints"), - asserts.dateArray0("datesAsStrings"), - asserts.dateArray0("dates"), - } - - testCases := []aggregateTestCase{ - testCasesGen.WithoutFilters(wrapWithMap(expectedAllResultsAssertions)), - - testCasesGen.WithWhereFilter_AllResults(wrapWithMap(expectedAllResultsAssertions)), - testCasesGen.WithWhereFilter_ResultsWithData(wrapWithMap(expectedResultsWithDataAssertions)), - testCasesGen.WithWhereFilter_ResultsWithoutData(wrapWithMap(expectedResultsWithoutDataAssertions)), - testCasesGen.WithWhereFilter_NoResults(wrapWithMap(expectedNoResultsAssertions)), - - testCasesGen.WithNearObjectFilter_AllResults(wrapWithMap(expectedAllResultsAssertions)), - testCasesGen.WithNearObjectFilter_ResultsWithData(wrapWithMap(expectedResultsWithDataAssertions)), - testCasesGen.WithNearObjectFilter_ResultsWithoutData(wrapWithMap(expectedResultsWithoutDataAssertions)), - - testCasesGen.WithWhereAndNearObjectFilters_AllResults(wrapWithMap(expectedAllResultsAssertions)), - testCasesGen.WithWhereAndNearObjectFilters_ResultsWithData(wrapWithMap(expectedResultsWithDataAssertions)), - testCasesGen.WithWhereAndNearObjectFilters_ResultsWithoutData(wrapWithMap(expectedResultsWithoutDataAssertions)), - testCasesGen.WithWhereAndNearObjectFilters_NoResults(wrapWithMap(expectedNoResultsAssertions)), - } - - for _, tc := range testCases { - query := aggregateArrayClassQuery(tc.filters, "") - - t.Run(tc.name, func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - extracted := extractArrayClassNoGroupByResult(result) - - for _, groupAssertions := range tc.groupedAssertions { - for _, assertion := range groupAssertions { - assertion(extracted) - } - } - }) - } - }) -} - -func aggregateDuplicatesClassWithoutGroupByTest(t *testing.T) { - t.Run("aggregate DuplicatesClass without group by", func(t *testing.T) { - asserts := newAggregateResponseAssert(t) - testCasesGen := &aggregateDuplicatesClassTestCases{} - - expectedAllResultsAssertions := []assertFunc{ - asserts.meta(3), - asserts.booleanArray("booleans", 9, 3, 6, 0.3333333333333333, 0.6666666666666666), - asserts.textArray("texts", 9, []string{"Atxt", "Btxt"}, []int64{6, 3}), - asserts.numberArray("numbers", 9, 2, 1, 1, 12, 1, 1.3333333333333333), - asserts.intArray("ints", 9, 102, 101, 101, 912, 101, 101.33333333333333), - asserts.dateArray("datesAsStrings", 9), - } - expectedSomeResultsAssertions := []assertFunc{ - asserts.meta(1), - asserts.booleanArray("booleans", 4, 1, 3, 0.25, 0.75), - asserts.textArray("texts", 4, []string{"Atxt", "Btxt"}, []int64{3, 1}), - asserts.numberArray("numbers", 4, 2, 1, 1, 5, 1, 1.25), - asserts.intArray("ints", 4, 102, 101, 101, 405, 101, 101.25), - asserts.dateArray("datesAsStrings", 4), - } - expectedNoResultsAssertions := []assertFunc{ - asserts.meta(0), - asserts.booleanArray0("booleans"), - asserts.textArray0("texts"), - asserts.numberArray0("numbers"), - asserts.intArray0("ints"), - asserts.dateArray0("datesAsStrings"), - } - - testCases := []aggregateTestCase{ - testCasesGen.WithoutFilters(wrapWithMap(expectedAllResultsAssertions)), - - testCasesGen.WithWhereFilter_AllResults(wrapWithMap(expectedAllResultsAssertions)), - testCasesGen.WithWhereFilter_SomeResults(wrapWithMap(expectedSomeResultsAssertions)), - testCasesGen.WithWhereFilter_NoResults(wrapWithMap(expectedNoResultsAssertions)), - } - - for _, tc := range testCases { - query := aggregateDuplicatesClassQuery(tc.filters, "") - - t.Run(tc.name, func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - extracted := extractDuplicatesClassNoGroupByResult(result) - - for _, groupAssertions := range tc.groupedAssertions { - for _, assertion := range groupAssertions { - assertion(extracted) - } - } - }) - } - }) -} - -func aggregateNoPropsClassWithoutGroupByTest(t *testing.T) { - t.Run("aggregate NoPropsClass without group by", func(t *testing.T) { - asserts := newAggregateResponseAssert(t) - testCasesGen := &aggregateNoPropsClassTestCases{} - - expectedAllResultsAssertions := []assertFunc{ - asserts.meta(2), - } - expectedSomeResultsAssertions := []assertFunc{ - asserts.meta(1), - } - expectedNoResultsAssertions := []assertFunc{ - asserts.meta(0), - } - - testCases := []aggregateTestCase{ - testCasesGen.WithoutFilters(wrapWithMap(expectedAllResultsAssertions)), - - testCasesGen.WithWhereFilter_AllResults(wrapWithMap(expectedAllResultsAssertions)), - testCasesGen.WithWhereFilter_SomeResults(wrapWithMap(expectedSomeResultsAssertions)), - testCasesGen.WithWhereFilter_NoResults(wrapWithMap(expectedNoResultsAssertions)), - - testCasesGen.WithNearObjectFilter_AllResults(wrapWithMap(expectedAllResultsAssertions)), - - testCasesGen.WithWhereAndNearObjectFilters_AllResults(wrapWithMap(expectedAllResultsAssertions)), - testCasesGen.WithWhereAndNearObjectFilters_SomeResults(wrapWithMap(expectedSomeResultsAssertions)), - testCasesGen.WithWhereAndNearObjectFilters_NoResults(wrapWithMap(expectedNoResultsAssertions)), - } - - for _, tc := range testCases { - query := aggregateNoPropsQuery(tc.filters) - - t.Run(tc.name, func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - extracted := extractNoPropsClassNoGroupByResult(result) - - for _, groupAssertions := range tc.groupedAssertions { - for _, assertion := range groupAssertions { - assertion(extracted) - } - } - }) - } - }) -} - -func aggregateCityClassWithoutGroupByTest(t *testing.T) { - t.Run("aggregate City without group by", func(t *testing.T) { - asserts := newAggregateResponseAssert(t) - testCasesGen := &aggregateCityTestCases{} - - expectedAllResultsAssertions := []assertFunc{ - asserts.meta(6), - asserts.number("cityArea", 4, 891.96, 217.22, 217.22, 2320.48, 605.6500000000001, 580.12), - asserts.date("cityRights", 4), - asserts.text("history", 4, []string{historyAmsterdam, historyRotterdam, historyBerlin, historyDusseldorf}, []int64{1, 1, 1, 1}), - asserts.boolean("isCapital", 5, 3, 2, 0.6, 0.4), - asserts.textArray("museums", 9, []string{"German Historical Museum", "Museum Boijmans Van Beuningen", "Onomato", "Rijksmuseum", "Schiffahrt Museum"}, []int64{1, 1, 1, 1, 1}), - asserts.text("name", 5, []string{"Amsterdam", "Berlin", "Dusseldorf", "Missing Island", "Rotterdam"}, []int64{1, 1, 1, 1, 1}), - asserts.int("population", 5, 3470000, 0, 600000, 6470000, 600000, 1294000), - asserts.textArray("timezones", 8, []string{"CEST", "CET"}, []int64{4, 4}), - asserts.pointingTo("inCountry", "Country"), - } - expectedResultsWithDataAssertions := []assertFunc{ - asserts.meta(2), - asserts.number("cityArea", 2, 891.96, 891.95, 891.95, 1783.91, 891.955, 891.955), - asserts.date("cityRights", 2), - asserts.text("history", 2, []string{historyAmsterdam, historyBerlin}, []int64{1, 1}), - asserts.boolean("isCapital", 2, 0, 2, 0, 1), - asserts.textArray("museums", 3, []string{"German Historical Museum", "Rijksmuseum", "Stedelijk Museum"}, []int64{1, 1, 1}), - asserts.text("name", 2, []string{"Amsterdam", "Berlin"}, []int64{1, 1}), - asserts.int("population", 2, 3470000, 1800000, 1800000, 5270000, 2635000, 2635000), - asserts.textArray("timezones", 4, []string{"CEST", "CET"}, []int64{2, 2}), - asserts.pointingTo("inCountry", "Country"), - } - expectedResultsWithoutDataAssertions := []assertFunc{ - asserts.meta(1), - asserts.number0("cityArea"), - asserts.date0("cityRights"), - asserts.text0("history"), - asserts.boolean0("isCapital"), - asserts.textArray0("museums"), - asserts.text0("name"), - asserts.int0("population"), - asserts.textArray0("timezones"), - asserts.pointingTo("inCountry", "Country"), - } - expectedNoResultsAssertions := []assertFunc{ - asserts.meta(0), - asserts.number0("cityArea"), - asserts.date0("cityRights"), - asserts.text0("history"), - asserts.boolean0("isCapital"), - asserts.textArray0("museums"), - asserts.text0("name"), - asserts.int0("population"), - asserts.textArray0("timezones"), - asserts.pointingTo("inCountry", "Country"), - } - - testCases := []aggregateTestCase{ - testCasesGen.WithoutFilters(wrapWithMap(expectedAllResultsAssertions)), - - testCasesGen.WithWhereFilter_AllResults(wrapWithMap(expectedAllResultsAssertions)), - testCasesGen.WithWhereFilter_ResultsWithData(wrapWithMap(expectedResultsWithDataAssertions)), - testCasesGen.WithWhereFilter_ResultsWithoutData(wrapWithMap(expectedResultsWithoutDataAssertions)), - testCasesGen.WithWhereFilter_NoResults(wrapWithMap(expectedNoResultsAssertions)), - - testCasesGen.WithNearObjectFilter_AllResults(wrapWithMap(expectedAllResultsAssertions)), - testCasesGen.WithNearObjectFilter_ResultsWithData(wrapWithMap(expectedResultsWithDataAssertions)), - testCasesGen.WithNearObjectFilter_ResultsWithoutData(wrapWithMap(expectedResultsWithoutDataAssertions)), - - testCasesGen.WithWhereAndNearObjectFilters_AllResults(wrapWithMap(expectedAllResultsAssertions)), - testCasesGen.WithWhereAndNearObjectFilters_ResultsWithData(wrapWithMap(expectedResultsWithDataAssertions)), - testCasesGen.WithWhereAndNearObjectFilters_ResultsWithoutData(wrapWithMap(expectedResultsWithoutDataAssertions)), - testCasesGen.WithWhereAndNearObjectFilters_NoResults(wrapWithMap(expectedNoResultsAssertions)), - } - - for _, tc := range testCases { - query := aggregateCityQuery(tc.filters, "") - - t.Run(tc.name, func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - extracted := extractCityNoGroupByResult(result) - - for _, groupAssertions := range tc.groupedAssertions { - for _, assertion := range groupAssertions { - assertion(extracted) - } - } - }) - } - }) -} - -func extractArrayClassNoGroupByResult(result *graphqlhelper.GraphQLResult) map[string]interface{} { - return extractAggregateResult(result, arrayClassName)[0].(map[string]interface{}) -} - -func extractDuplicatesClassNoGroupByResult(result *graphqlhelper.GraphQLResult) map[string]interface{} { - return extractAggregateResult(result, duplicatesClassName)[0].(map[string]interface{}) -} - -func extractNoPropsClassNoGroupByResult(result *graphqlhelper.GraphQLResult) map[string]interface{} { - return extractAggregateResult(result, noPropsClassName)[0].(map[string]interface{}) -} - -func extractCityNoGroupByResult(result *graphqlhelper.GraphQLResult) map[string]interface{} { - return extractAggregateResult(result, cityClassName)[0].(map[string]interface{}) -} - -func wrapWithMap(assertFuncs []assertFunc) map[string][]assertFunc { - return map[string][]assertFunc{"": assertFuncs} -} diff --git a/test/acceptance/graphql_resolvers/local_aggregate_matrix_setup_test.go b/test/acceptance/graphql_resolvers/local_aggregate_matrix_setup_test.go deleted file mode 100644 index 46f8d2954f63f2d460736adaa5256dbc65a5be5c..0000000000000000000000000000000000000000 --- a/test/acceptance/graphql_resolvers/local_aggregate_matrix_setup_test.go +++ /dev/null @@ -1,1130 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "fmt" - "time" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - graphqlhelper "github.com/weaviate/weaviate/test/helper/graphql" -) - -const notExistingObjectId = "cfa3b21e-ca5f-4db7-a412-ffffffffffff" - -const ( - arrayClassName = "ArrayClass" - - objectArrayClassID1_4el = "cfa3b21e-ca5f-4db7-a412-5fc6a23c534a" - objectArrayClassID2_3el = "cfa3b21e-ca5f-4db7-a412-5fc6a23c534b" - objectArrayClassID3_2el = "cfa3b21e-ca5f-4db7-a412-5fc6a23c535a" - objectArrayClassID4_1el = "cfa3b21e-ca5f-4db7-a412-5fc6a23c535b" - objectArrayClassID5_0el = "cfa3b21e-ca5f-4db7-a412-5fc6a23c536a" - objectArrayClassID6_nils = "cfa3b21e-ca5f-4db7-a412-5fc6a23c536b" - objectArrayClassID7_empty = "cfa3b21e-ca5f-4db7-a412-5fc6a23c536c" -) - -const ( - noPropsClassName = "ClassWithoutProperties" - - objectNoPropsClassID1 = "dfa3b21e-ca5f-4db7-a412-5fc6a23c5301" - objectNoPropsClassID2 = "dfa3b21e-ca5f-4db7-a412-5fc6a23c5311" -) - -const ( - cityClassName = "City" -) - -const ( - duplicatesClassName = "DuplicatesClass" - - objectDuplicatesClassID1_4el = "a8076f34-ec16-4333-a963-00c89c5ba001" - objectDuplicatesClassID2_3el = "a8076f34-ec16-4333-a963-00c89c5ba002" - objectDuplicatesClassID3_2el = "a8076f34-ec16-4333-a963-00c89c5ba003" -) - -func arrayClassSchema() *models.Class { - return &models.Class{ - Class: arrayClassName, - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "vectorizeClassName": true, - }, - }, - InvertedIndexConfig: &models.InvertedIndexConfig{IndexPropertyLength: true, IndexNullState: true}, - Properties: []*models.Property{ - { - Name: "texts", - DataType: schema.DataTypeTextArray.PropString(), - Tokenization: models.PropertyTokenizationWord, - }, - { - Name: "numbers", - DataType: []string{"number[]"}, - }, - { - Name: "ints", - DataType: []string{"int[]"}, - }, - { - Name: "booleans", - DataType: []string{"boolean[]"}, - }, - { - Name: "datesAsStrings", - DataType: []string{"date[]"}, - }, - { - Name: "dates", - DataType: []string{"date[]"}, - }, - }, - } -} - -func arrayClassObjects() []*models.Object { - return []*models.Object{ - objectArrayClass4el(), - objectArrayClass3el(), - objectArrayClass2el(), - objectArrayClass1el(), - objectArrayClass0el(), - objectArrayClassNils(), - objectArrayClassEmpty(), - } -} - -func objectArrayClass4el() *models.Object { - return &models.Object{ - Class: arrayClassName, - ID: objectArrayClassID1_4el, - Properties: map[string]interface{}{ - "texts": []string{"Atxt", "Btxt", "Ctxt", "Dtxt"}, - "numbers": []float64{1.0, 2.0, 3.0, 4.0}, - "ints": []int{101, 102, 103, 104}, - "booleans": []bool{true, true, true, false}, - "datesAsStrings": []string{ - "2021-06-01T22:18:59.640162Z", - "2022-06-02T22:18:59.640162Z", - "2023-06-03T22:18:59.640162Z", - "2024-06-04T22:18:59.640162Z", - }, - "dates": []time.Time{ - time.Date(2001, 6, 1, 12, 0, 0, 0, time.UTC), - time.Date(2002, 6, 2, 12, 0, 0, 0, time.UTC), - time.Date(2003, 6, 3, 12, 0, 0, 0, time.UTC), - time.Date(2004, 6, 4, 12, 0, 0, 0, time.UTC), - }, - }, - } -} - -func objectArrayClass3el() *models.Object { - return &models.Object{ - Class: arrayClassName, - ID: objectArrayClassID2_3el, - Properties: map[string]interface{}{ - "texts": []string{"Atxt", "Btxt", "Ctxt"}, - "numbers": []float64{1.0, 2.0, 3.0}, - "ints": []int{101, 102, 103}, - "booleans": []bool{true, true, false}, - "datesAsStrings": []string{ - "2021-06-01T22:18:59.640162Z", - "2022-06-02T22:18:59.640162Z", - "2023-06-03T22:18:59.640162Z", - }, - "dates": []time.Time{ - time.Date(2001, 6, 1, 12, 0, 0, 0, time.UTC), - time.Date(2002, 6, 2, 12, 0, 0, 0, time.UTC), - time.Date(2003, 6, 3, 12, 0, 0, 0, time.UTC), - }, - }, - } -} - -func objectArrayClass2el() *models.Object { - return &models.Object{ - Class: arrayClassName, - ID: objectArrayClassID3_2el, - Properties: map[string]interface{}{ - "texts": []string{"Atxt", "Btxt"}, - "numbers": []float64{1.0, 2.0}, - "ints": []int{101, 102}, - "booleans": []bool{true, false}, - "datesAsStrings": []string{ - "2021-06-01T22:18:59.640162Z", - "2022-06-02T22:18:59.640162Z", - }, - "dates": []time.Time{ - time.Date(2001, 6, 1, 12, 0, 0, 0, time.UTC), - time.Date(2002, 6, 2, 12, 0, 0, 0, time.UTC), - }, - }, - } -} - -func objectArrayClass1el() *models.Object { - return &models.Object{ - Class: arrayClassName, - ID: objectArrayClassID4_1el, - Properties: map[string]interface{}{ - "texts": []string{"Atxt"}, - "numbers": []float64{1.0}, - "ints": []int{101}, - "booleans": []bool{false}, - "datesAsStrings": []string{ - "2021-06-01T22:18:59.640162Z", - }, - "dates": []time.Time{ - time.Date(2001, 6, 1, 12, 0, 0, 0, time.UTC), - }, - }, - } -} - -func objectArrayClass0el() *models.Object { - return &models.Object{ - Class: arrayClassName, - ID: objectArrayClassID5_0el, - Properties: map[string]interface{}{ - "texts": []string{}, - "numbers": []float64{}, - "ints": []int{}, - "booleans": []bool{}, - "datesAsStrings": []string{}, - "dates": []time.Time{}, - }, - } -} - -func objectArrayClassNils() *models.Object { - return &models.Object{ - Class: arrayClassName, - ID: objectArrayClassID6_nils, - Properties: map[string]interface{}{ - "texts": nil, - "numbers": nil, - "ints": nil, - "booleans": nil, - "datesAsStrings": nil, - "dates": nil, - }, - } -} - -func objectArrayClassEmpty() *models.Object { - return &models.Object{ - Class: arrayClassName, - ID: objectArrayClassID7_empty, - } -} - -func aggregateArrayClassQuery(filters, groupBy string) string { - query := `{ - Aggregate { - %s - %s - { - meta{ - count - } - booleans{ - count - type - totalTrue - totalFalse - percentageTrue - percentageFalse - } - texts{ - count - type - topOccurrences { - value - occurs - } - } - numbers{ - count - type - minimum - maximum - mean - median - mode - sum - } - ints{ - count - type - minimum - maximum - mean - median - mode - sum - } - datesAsStrings{ - count - } - dates{ - count - } - %s - } - } - }` - - params := "" - if filters != "" || groupBy != "" { - params = fmt.Sprintf( - `( - %s - %s - )`, filters, groupBy) - } - groupedBy := "" - if groupBy != "" { - groupedBy = `groupedBy{ - value - path - }` - } - - return fmt.Sprintf(query, arrayClassName, params, groupedBy) -} - -func extractAggregateResult(result *graphqlhelper.GraphQLResult, className string) []interface{} { - return result.Get("Aggregate", className).AsSlice() -} - -func noPropsClassSchema() *models.Class { - return &models.Class{ - Class: noPropsClassName, - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "vectorizeClassName": true, - }, - }, - } -} - -func noPropsClassObjects() []*models.Object { - return []*models.Object{ - objectNoPropsClass1(), - objectNoPropsClass2(), - } -} - -func objectNoPropsClass1() *models.Object { - return &models.Object{ - Class: noPropsClassName, - ID: objectNoPropsClassID1, - } -} - -func objectNoPropsClass2() *models.Object { - return &models.Object{ - Class: noPropsClassName, - ID: objectNoPropsClassID2, - } -} - -func aggregateNoPropsQuery(filters string) string { - query := ` - { - Aggregate { - %s - %s - { - meta{ - count - } - } - } - } - ` - - params := "" - if filters != "" { - params = fmt.Sprintf( - `( - %s - )`, filters) - } - - return fmt.Sprintf(query, noPropsClassName, params) -} - -func aggregateCityQuery(filters, groupBy string) string { - query := `{ - Aggregate { - %s - %s - { - meta { - count - } - name { - count - type - topOccurrences { - value - occurs - } - } - cityArea { - count - type - minimum - maximum - mean - median - mode - sum - } - isCapital { - count - type - totalTrue - totalFalse - percentageTrue - percentageFalse - } - population { - count - type - minimum - maximum - mean - median - mode - sum - } - cityRights { - count - } - history { - count - type - topOccurrences { - value - occurs - } - } - museums { - count - type - topOccurrences { - value - occurs - } - } - timezones { - count - type - topOccurrences { - value - occurs - } - } - inCountry { - pointingTo - type - } - %s - } - } - }` - - params := "" - if filters != "" || groupBy != "" { - params = fmt.Sprintf( - `( - %s - %s - )`, filters, groupBy) - } - groupedBy := "" - if groupBy != "" { - groupedBy = `groupedBy{ - value - path - }` - } - - return fmt.Sprintf(query, cityClassName, params, groupedBy) -} - -func duplicatesClassSchema() *models.Class { - return &models.Class{ - Class: duplicatesClassName, - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "vectorizeClassName": true, - }, - }, - Properties: []*models.Property{ - { - Name: "texts", - DataType: schema.DataTypeTextArray.PropString(), - Tokenization: models.PropertyTokenizationWord, - }, - { - Name: "numbers", - DataType: []string{"number[]"}, - }, - { - Name: "ints", - DataType: []string{"int[]"}, - }, - { - Name: "booleans", - DataType: []string{"boolean[]"}, - }, - { - Name: "datesAsStrings", - DataType: []string{"date[]"}, - }, - }, - } -} - -func duplicatesClassObjects() []*models.Object { - return []*models.Object{ - objectDuplicatesClass4el(), - objectDuplicatesClass3el(), - objectDuplicatesClass2el(), - } -} - -func objectDuplicatesClass4el() *models.Object { - return &models.Object{ - Class: duplicatesClassName, - ID: objectDuplicatesClassID1_4el, - Properties: map[string]interface{}{ - "texts": []string{"Atxt", "Atxt", "Atxt", "Btxt"}, - "numbers": []float64{1.0, 1.0, 1.0, 2.0}, - "ints": []int{101, 101, 101, 102}, - "booleans": []bool{true, true, true, false}, - "datesAsStrings": []string{ - "2021-06-01T22:18:59.640162Z", - "2021-06-01T22:18:59.640162Z", - "2021-06-01T22:18:59.640162Z", - "2022-06-02T22:18:59.640162Z", - }, - }, - } -} - -func objectDuplicatesClass3el() *models.Object { - return &models.Object{ - Class: duplicatesClassName, - ID: objectDuplicatesClassID2_3el, - Properties: map[string]interface{}{ - "texts": []string{"Atxt", "Atxt", "Btxt"}, - "numbers": []float64{1.0, 1.0, 2.0}, - "ints": []int{101, 101, 102}, - "booleans": []bool{true, true, false}, - "datesAsStrings": []string{ - "2021-06-01T22:18:59.640162Z", - "2021-06-01T22:18:59.640162Z", - "2022-06-02T22:18:59.640162Z", - }, - }, - } -} - -func objectDuplicatesClass2el() *models.Object { - return &models.Object{ - Class: duplicatesClassName, - ID: objectDuplicatesClassID3_2el, - Properties: map[string]interface{}{ - "texts": []string{"Atxt", "Btxt"}, - "numbers": []float64{1.0, 2.0}, - "ints": []int{101, 102}, - "booleans": []bool{true, false}, - "datesAsStrings": []string{ - "2021-06-01T22:18:59.640162Z", - "2022-06-02T22:18:59.640162Z", - }, - }, - } -} - -func aggregateDuplicatesClassQuery(filters, groupBy string) string { - query := `{ - Aggregate { - %s - %s - { - meta{ - count - } - booleans{ - count - type - totalTrue - totalFalse - percentageTrue - percentageFalse - } - texts{ - count - type - topOccurrences { - value - occurs - } - } - numbers{ - count - type - minimum - maximum - mean - median - mode - sum - } - ints{ - count - type - minimum - maximum - mean - median - mode - sum - } - datesAsStrings{ - count - } - %s - } - } - }` - - params := "" - if filters != "" || groupBy != "" { - params = fmt.Sprintf( - `( - %s - %s - )`, filters, groupBy) - } - groupedBy := "" - if groupBy != "" { - groupedBy = `groupedBy{ - value - path - }` - } - - return fmt.Sprintf(query, duplicatesClassName, params, groupedBy) -} - -type aggregateTestCase struct { - name string - filters string - groupedAssertions map[string][]assertFunc // map[groupedBy]assertionsForGroup -} - -type aggregateArrayClassTestCases struct{} - -func (tc *aggregateArrayClassTestCases) WithoutFilters(groupedAssertions map[string][]assertFunc) aggregateTestCase { - return aggregateTestCase{ - name: "without filters", - groupedAssertions: groupedAssertions, - } -} - -func (tc *aggregateArrayClassTestCases) WithWhereFilter_AllResults(groupedAssertions map[string][]assertFunc) aggregateTestCase { - return aggregateTestCase{ - name: "with where filter (all results)", - filters: ` - where: { - operator: Like - path: ["id"] - valueText: "*" - }`, - groupedAssertions: groupedAssertions, - } -} - -func (tc *aggregateArrayClassTestCases) WithWhereFilter_ResultsWithData(groupedAssertions map[string][]assertFunc) aggregateTestCase { - return aggregateTestCase{ - name: "with where filter (results with data)", - filters: fmt.Sprintf(` - where: { - operator: Like - path: ["id"] - valueText: "%s" - }`, objectArrayClassID1_4el[:35]+"?"), - groupedAssertions: groupedAssertions, - } -} - -func (tc *aggregateArrayClassTestCases) WithWhereFilter_ResultsWithoutData(groupedAssertions map[string][]assertFunc) aggregateTestCase { - return aggregateTestCase{ - name: "with where filter (results without data)", - filters: fmt.Sprintf(` - where: { - operator: Like - path: ["id"] - valueText: "%s" - }`, objectArrayClassID5_0el[:35]+"?"), - groupedAssertions: groupedAssertions, - } -} - -func (tc *aggregateArrayClassTestCases) WithWhereFilter_NoResults(groupedAssertions map[string][]assertFunc) aggregateTestCase { - return aggregateTestCase{ - name: "with where filter (no results)", - filters: fmt.Sprintf(` - where: { - operator: Like - path: ["id"] - valueText: "%s" - }`, notExistingObjectId), - groupedAssertions: groupedAssertions, - } -} - -func (tc *aggregateArrayClassTestCases) WithNearObjectFilter_AllResults(groupedAssertions map[string][]assertFunc) aggregateTestCase { - return aggregateTestCase{ - name: "with nearObject filter (all results)", - filters: fmt.Sprintf(` - nearObject: { - id: "%s" - certainty: 0.1 - }`, objectArrayClassID1_4el), - groupedAssertions: groupedAssertions, - } -} - -func (tc *aggregateArrayClassTestCases) WithNearObjectFilter_ResultsWithData(groupedAssertions map[string][]assertFunc) aggregateTestCase { - return aggregateTestCase{ - name: "with nearObject filter (results with data)", - filters: fmt.Sprintf(` - nearObject: { - id: "%s" - certainty: 0.988 - }`, objectArrayClassID1_4el), - groupedAssertions: groupedAssertions, - } -} - -func (tc *aggregateArrayClassTestCases) WithNearObjectFilter_ResultsWithoutData(groupedAssertions map[string][]assertFunc) aggregateTestCase { - return aggregateTestCase{ - name: "with nearObject filter (results without data)", - filters: fmt.Sprintf(` - nearObject: { - id: "%s" - certainty: 1 - }`, objectArrayClassID5_0el), - groupedAssertions: groupedAssertions, - } -} - -func (tc *aggregateArrayClassTestCases) WithWhereAndNearObjectFilters_AllResults(groupedAssertions map[string][]assertFunc) aggregateTestCase { - return aggregateTestCase{ - name: "with where & nearObject filters (all results)", - filters: fmt.Sprintf(` - where: { - operator: Like - path: ["id"] - valueText: "*" - } - nearObject: { - id: "%s" - certainty: 0.1 - }`, objectArrayClassID1_4el), - groupedAssertions: groupedAssertions, - } -} - -func (tc *aggregateArrayClassTestCases) WithWhereAndNearObjectFilters_ResultsWithData(groupedAssertions map[string][]assertFunc) aggregateTestCase { - return aggregateTestCase{ - name: "with where & nearObject filters (results with data)", - filters: fmt.Sprintf(` - where: { - operator: Like - path: ["id"] - valueText: "%s" - } - nearObject: { - id: "%s" - certainty: 0.98 - }`, objectArrayClassID1_4el[:35]+"?", objectArrayClassID1_4el), - groupedAssertions: groupedAssertions, - } -} - -func (tc *aggregateArrayClassTestCases) WithWhereAndNearObjectFilters_ResultsWithoutData(groupedAssertions map[string][]assertFunc) aggregateTestCase { - return aggregateTestCase{ - name: "with where & nearObject filters (results without data)", - filters: fmt.Sprintf(` - where: { - operator: Like - path: ["id"] - valueText: "%s" - } - nearObject: { - id: "%s" - certainty: 1 - }`, objectArrayClassID5_0el[:35]+"?", objectArrayClassID5_0el), - groupedAssertions: groupedAssertions, - } -} - -func (tc *aggregateArrayClassTestCases) WithWhereAndNearObjectFilters_NoResults(groupedAssertions map[string][]assertFunc) aggregateTestCase { - return aggregateTestCase{ - name: "with where & nearObject filters (no results)", - filters: fmt.Sprintf(` - where: { - operator: Like - path: ["id"] - valueText: "%s" - } - nearObject: { - id: "%s" - certainty: 0.1 - }`, notExistingObjectId, objectArrayClassID1_4el), - groupedAssertions: groupedAssertions, - } -} - -type aggregateNoPropsClassTestCases struct{} - -func (tc *aggregateNoPropsClassTestCases) WithoutFilters(groupedAssertions map[string][]assertFunc) aggregateTestCase { - return aggregateTestCase{ - name: "without filters", - groupedAssertions: groupedAssertions, - } -} - -func (tc *aggregateNoPropsClassTestCases) WithWhereFilter_AllResults(groupedAssertions map[string][]assertFunc) aggregateTestCase { - return aggregateTestCase{ - name: "with where filter (all results)", - filters: ` - where: { - operator: Like - path: ["id"] - valueText: "*" - }`, - groupedAssertions: groupedAssertions, - } -} - -func (tc *aggregateNoPropsClassTestCases) WithWhereFilter_SomeResults(groupedAssertions map[string][]assertFunc) aggregateTestCase { - return aggregateTestCase{ - name: "with where filter (some results)", - filters: fmt.Sprintf(` - where: { - operator: Like - path: ["id"] - valueText: "%s" - }`, objectNoPropsClassID1[:35]+"?"), - groupedAssertions: groupedAssertions, - } -} - -func (tc *aggregateNoPropsClassTestCases) WithWhereFilter_NoResults(groupedAssertions map[string][]assertFunc) aggregateTestCase { - return aggregateTestCase{ - name: "with where filter (no results)", - filters: fmt.Sprintf(` - where: { - operator: Like - path: ["id"] - valueText: "%s" - }`, notExistingObjectId), - groupedAssertions: groupedAssertions, - } -} - -func (tc *aggregateNoPropsClassTestCases) WithNearObjectFilter_AllResults(groupedAssertions map[string][]assertFunc) aggregateTestCase { - return aggregateTestCase{ - name: "with nearObject filter (all results)", - filters: fmt.Sprintf(` - nearObject: { - id: "%s" - certainty: 0.1 - }`, objectNoPropsClassID1), - groupedAssertions: groupedAssertions, - } -} - -func (tc *aggregateNoPropsClassTestCases) WithWhereAndNearObjectFilters_AllResults(groupedAssertions map[string][]assertFunc) aggregateTestCase { - return aggregateTestCase{ - name: "with where & nearObject filters (all results)", - filters: fmt.Sprintf(` - where: { - operator: Like - path: ["id"] - valueText: "*" - } - nearObject: { - id: "%s" - certainty: 0.1 - }`, objectNoPropsClassID1), - groupedAssertions: groupedAssertions, - } -} - -func (tc *aggregateNoPropsClassTestCases) WithWhereAndNearObjectFilters_SomeResults(groupedAssertions map[string][]assertFunc) aggregateTestCase { - return aggregateTestCase{ - name: "with where & nearObject filters (some results)", - filters: fmt.Sprintf(` - where: { - operator: Like - path: ["id"] - valueText: "%s" - } - nearObject: { - id: "%s" - certainty: 1 - }`, objectNoPropsClassID1[:35]+"?", objectNoPropsClassID1), - groupedAssertions: groupedAssertions, - } -} - -func (tc *aggregateNoPropsClassTestCases) WithWhereAndNearObjectFilters_NoResults(groupedAssertions map[string][]assertFunc) aggregateTestCase { - return aggregateTestCase{ - name: "with where & nearObject filters (no results)", - filters: fmt.Sprintf(` - where: { - operator: Like - path: ["id"] - valueText: "%s" - } - nearObject: { - id: "%s" - certainty: 0.1 - }`, notExistingObjectId, objectNoPropsClassID1), - groupedAssertions: groupedAssertions, - } -} - -type aggregateCityTestCases struct{} - -func (tc *aggregateCityTestCases) WithoutFilters(groupedAssertions map[string][]assertFunc) aggregateTestCase { - return aggregateTestCase{ - name: "without filters", - groupedAssertions: groupedAssertions, - } -} - -func (tc *aggregateCityTestCases) WithWhereFilter_AllResults(groupedAssertions map[string][]assertFunc) aggregateTestCase { - return aggregateTestCase{ - name: "with where filter (all results)", - filters: ` - where: { - operator: Like - path: ["id"] - valueText: "*" - }`, - groupedAssertions: groupedAssertions, - } -} - -func (tc *aggregateCityTestCases) WithWhereFilter_ResultsWithData(groupedAssertions map[string][]assertFunc) aggregateTestCase { - return aggregateTestCase{ - name: "with where filter (results with data)", - filters: ` - where: { - operator: Equal, - path: ["isCapital"], - valueBoolean: true - }`, - groupedAssertions: groupedAssertions, - } -} - -func (tc *aggregateCityTestCases) WithWhereFilter_ResultsWithoutData(groupedAssertions map[string][]assertFunc) aggregateTestCase { - return aggregateTestCase{ - name: "with where filter (results without data)", - filters: fmt.Sprintf(` - where: { - operator: Like - path: ["id"] - valueText: "%s" - }`, nullisland), - groupedAssertions: groupedAssertions, - } -} - -func (tc *aggregateCityTestCases) WithWhereFilter_NoResults(groupedAssertions map[string][]assertFunc) aggregateTestCase { - return aggregateTestCase{ - name: "with where filter (no results)", - filters: fmt.Sprintf(` - where: { - operator: Like - path: ["id"] - valueText: "%s" - }`, notExistingObjectId), - groupedAssertions: groupedAssertions, - } -} - -func (tc *aggregateCityTestCases) WithNearObjectFilter_AllResults(groupedAssertions map[string][]assertFunc) aggregateTestCase { - return aggregateTestCase{ - name: "with nearObject filter (all results)", - filters: fmt.Sprintf(` - nearObject: { - id: "%s" - certainty: 0.1 - }`, berlin), - groupedAssertions: groupedAssertions, - } -} - -func (tc *aggregateCityTestCases) WithNearObjectFilter_ResultsWithData(groupedAssertions map[string][]assertFunc) aggregateTestCase { - return aggregateTestCase{ - name: "with nearObject filter (results with data)", - filters: fmt.Sprintf(` - nearObject: { - id: "%s" - certainty: 0.81 - }`, berlin), - groupedAssertions: groupedAssertions, - } -} - -func (tc *aggregateCityTestCases) WithNearObjectFilter_ResultsWithoutData(groupedAssertions map[string][]assertFunc) aggregateTestCase { - return aggregateTestCase{ - name: "with nearObject filter (results without data)", - filters: fmt.Sprintf(` - nearObject: { - id: "%s" - certainty: 0.9 - }`, nullisland), - groupedAssertions: groupedAssertions, - } -} - -func (tc *aggregateCityTestCases) WithWhereAndNearObjectFilters_AllResults(groupedAssertions map[string][]assertFunc) aggregateTestCase { - return aggregateTestCase{ - name: "with where & nearObject filters (all results)", - filters: fmt.Sprintf(` - where: { - operator: Like - path: ["id"] - valueText: "*" - } - nearObject: { - id: "%s" - certainty: 0.1 - }`, berlin), - groupedAssertions: groupedAssertions, - } -} - -func (tc *aggregateCityTestCases) WithWhereAndNearObjectFilters_ResultsWithData(groupedAssertions map[string][]assertFunc) aggregateTestCase { - return aggregateTestCase{ - name: "with where & nearObject filters (results with data)", - filters: fmt.Sprintf(` - where: { - operator: Equal, - path: ["isCapital"], - valueBoolean: true - } - nearObject: { - id: "%s" - certainty: 0.81 - }`, berlin), - groupedAssertions: groupedAssertions, - } -} - -func (tc *aggregateCityTestCases) WithWhereAndNearObjectFilters_ResultsWithoutData(groupedAssertions map[string][]assertFunc) aggregateTestCase { - return aggregateTestCase{ - name: "with where & nearObject filters (results without data)", - filters: fmt.Sprintf(` - where: { - operator: Like - path: ["id"] - valueText: "%s" - } - nearObject: { - id: "%s" - certainty: 0.9 - }`, nullisland, nullisland), - groupedAssertions: groupedAssertions, - } -} - -func (tc *aggregateCityTestCases) WithWhereAndNearObjectFilters_NoResults(groupedAssertions map[string][]assertFunc) aggregateTestCase { - return aggregateTestCase{ - name: "with where & nearObject filters (no results)", - filters: fmt.Sprintf(` - where: { - operator: Like - path: ["id"] - valueText: "%s" - } - nearObject: { - id: "%s" - certainty: 0.1 - }`, notExistingObjectId, berlin), - groupedAssertions: groupedAssertions, - } -} - -type aggregateDuplicatesClassTestCases struct{} - -func (tc *aggregateDuplicatesClassTestCases) WithoutFilters(groupedAssertions map[string][]assertFunc) aggregateTestCase { - return aggregateTestCase{ - name: "without filters", - groupedAssertions: groupedAssertions, - } -} - -func (tc *aggregateDuplicatesClassTestCases) WithWhereFilter_AllResults(groupedAssertions map[string][]assertFunc) aggregateTestCase { - return aggregateTestCase{ - name: "with where filter (all results)", - filters: ` - where: { - operator: Like - path: ["id"] - valueText: "*" - }`, - groupedAssertions: groupedAssertions, - } -} - -func (tc *aggregateDuplicatesClassTestCases) WithWhereFilter_SomeResults(groupedAssertions map[string][]assertFunc) aggregateTestCase { - return aggregateTestCase{ - name: "with where filter (some results)", - filters: fmt.Sprintf(` - where: { - operator: Like - path: ["id"] - valueText: "%s" - }`, objectDuplicatesClassID1_4el), - groupedAssertions: groupedAssertions, - } -} - -func (tc *aggregateDuplicatesClassTestCases) WithWhereFilter_NoResults(groupedAssertions map[string][]assertFunc) aggregateTestCase { - return aggregateTestCase{ - name: "with where filter (no results)", - filters: fmt.Sprintf(` - where: { - operator: Like - path: ["id"] - valueText: "%s" - }`, notExistingObjectId), - groupedAssertions: groupedAssertions, - } -} diff --git a/test/acceptance/graphql_resolvers/local_aggregate_test.go b/test/acceptance/graphql_resolvers/local_aggregate_test.go deleted file mode 100644 index 8fea5654214f89d8e06246a8fac04d6347fbdd63..0000000000000000000000000000000000000000 --- a/test/acceptance/graphql_resolvers/local_aggregate_test.go +++ /dev/null @@ -1,1532 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "encoding/json" - "fmt" - "strconv" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/test/helper" - graphqlhelper "github.com/weaviate/weaviate/test/helper/graphql" -) - -// This test prevents a regression on the fix for -// https://github.com/weaviate/weaviate/issues/824 -func localMeta_StringPropsNotSetEverywhere(t *testing.T) { - graphqlhelper.AssertGraphQL(t, helper.RootAuth, ` - { - Aggregate { - City { - name { - topOccurrences { - occurs - value - } - } - } - } - } - `) -} - -func localMetaWithWhereAndNearTextFilters(t *testing.T) { - t.Run("with distance", func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, ` - { - Aggregate{ - City (where: { - valueBoolean: true, - operator: Equal, - path: ["isCapital"] - } - nearText: { - concepts: ["Amsterdam"] - distance: 0.2 - } - ){ - meta { - count - } - isCapital { - count - percentageFalse - percentageTrue - totalFalse - totalTrue - type - } - population { - mean - count - maximum - minimum - sum - type - } - inCountry { - pointingTo - type - } - name { - topOccurrences { - occurs - value - } - type - count - } - } - } - } - `) - - t.Run("meta count", func(t *testing.T) { - meta := result.Get("Aggregate", "City").AsSlice()[0].(map[string]interface{})["meta"] - count := meta.(map[string]interface{})["count"] - expected := json.Number("1") - assert.Equal(t, expected, count) - }) - - t.Run("boolean props", func(t *testing.T) { - isCapital := result.Get("Aggregate", "City").AsSlice()[0].(map[string]interface{})["isCapital"] - expected := map[string]interface{}{ - "count": json.Number("1"), - "percentageTrue": json.Number("1"), - "percentageFalse": json.Number("0"), - "totalTrue": json.Number("1"), - "totalFalse": json.Number("0"), - "type": "boolean", - } - assert.Equal(t, expected, isCapital) - }) - - t.Run("int/number props", func(t *testing.T) { - population := result.Get("Aggregate", "City").AsSlice()[0].(map[string]interface{})["population"] - expected := map[string]interface{}{ - "mean": json.Number("1800000"), - "count": json.Number("1"), - "maximum": json.Number("1800000"), - "minimum": json.Number("1800000"), - "sum": json.Number("1800000"), - "type": "int", - } - assert.Equal(t, expected, population) - }) - - t.Run("ref prop", func(t *testing.T) { - inCountry := result.Get("Aggregate", "City").AsSlice()[0].(map[string]interface{})["inCountry"] - expected := map[string]interface{}{ - "pointingTo": []interface{}{"Country"}, - "type": "cref", - } - assert.Equal(t, expected, inCountry) - }) - - t.Run("string prop", func(t *testing.T) { - name := result.Get("Aggregate", "City"). - AsSlice()[0].(map[string]interface{})["name"].(map[string]interface{}) - typeField := name["type"] - topOccurrences := name["topOccurrences"] - - assert.Equal(t, schema.DataTypeText.String(), typeField) - - expectedTopOccurrences := []interface{}{ - map[string]interface{}{ - "value": "Amsterdam", - "occurs": json.Number("1"), - }, - } - assert.ElementsMatch(t, expectedTopOccurrences, topOccurrences) - }) - }) - - t.Run("with certainty", func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, ` - { - Aggregate{ - City (where: { - valueBoolean: true, - operator: Equal, - path: ["isCapital"] - } - nearText: { - concepts: ["Amsterdam"] - certainty: 0.9 - } - ){ - meta { - count - } - isCapital { - count - percentageFalse - percentageTrue - totalFalse - totalTrue - type - } - population { - mean - count - maximum - minimum - sum - type - } - inCountry { - pointingTo - type - } - name { - topOccurrences { - occurs - value - } - type - count - } - } - } - }`) - - t.Run("meta count", func(t *testing.T) { - meta := result.Get("Aggregate", "City").AsSlice()[0].(map[string]interface{})["meta"] - count := meta.(map[string]interface{})["count"] - expected := json.Number("1") - assert.Equal(t, expected, count) - }) - - t.Run("boolean props", func(t *testing.T) { - isCapital := result.Get("Aggregate", "City").AsSlice()[0].(map[string]interface{})["isCapital"] - expected := map[string]interface{}{ - "count": json.Number("1"), - "percentageTrue": json.Number("1"), - "percentageFalse": json.Number("0"), - "totalTrue": json.Number("1"), - "totalFalse": json.Number("0"), - "type": "boolean", - } - assert.Equal(t, expected, isCapital) - }) - - t.Run("int/number props", func(t *testing.T) { - population := result.Get("Aggregate", "City").AsSlice()[0].(map[string]interface{})["population"] - expected := map[string]interface{}{ - "mean": json.Number("1800000"), - "count": json.Number("1"), - "maximum": json.Number("1800000"), - "minimum": json.Number("1800000"), - "sum": json.Number("1800000"), - "type": "int", - } - assert.Equal(t, expected, population) - }) - - t.Run("ref prop", func(t *testing.T) { - inCountry := result.Get("Aggregate", "City").AsSlice()[0].(map[string]interface{})["inCountry"] - expected := map[string]interface{}{ - "pointingTo": []interface{}{"Country"}, - "type": "cref", - } - assert.Equal(t, expected, inCountry) - }) - - t.Run("string prop", func(t *testing.T) { - name := result.Get("Aggregate", "City"). - AsSlice()[0].(map[string]interface{})["name"].(map[string]interface{}) - typeField := name["type"] - topOccurrences := name["topOccurrences"] - - assert.Equal(t, schema.DataTypeText.String(), typeField) - - expectedTopOccurrences := []interface{}{ - map[string]interface{}{ - "value": "Amsterdam", - "occurs": json.Number("1"), - }, - } - assert.ElementsMatch(t, expectedTopOccurrences, topOccurrences) - }) - }) -} - -func localMetaWithWhereAndNearObjectFilters(t *testing.T) { - t.Run("with distance", func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, ` - { - Aggregate{ - City (where: { - valueBoolean: true, - operator: Equal, - path: ["isCapital"] - } - nearObject: { - id: "9b9cbea5-e87e-4cd0-89af-e2f424fd52d6" - distance: 0.2 - } - ){ - meta { - count - } - isCapital { - count - percentageFalse - percentageTrue - totalFalse - totalTrue - type - } - population { - mean - count - maximum - minimum - sum - type - } - inCountry { - pointingTo - type - } - name { - topOccurrences { - occurs - value - } - type - count - } - } - } - }`) - - t.Run("meta count", func(t *testing.T) { - meta := result.Get("Aggregate", "City").AsSlice()[0].(map[string]interface{})["meta"] - count := meta.(map[string]interface{})["count"] - expected := json.Number("1") - assert.Equal(t, expected, count) - }) - - t.Run("boolean props", func(t *testing.T) { - isCapital := result.Get("Aggregate", "City").AsSlice()[0].(map[string]interface{})["isCapital"] - expected := map[string]interface{}{ - "count": json.Number("1"), - "percentageTrue": json.Number("1"), - "percentageFalse": json.Number("0"), - "totalTrue": json.Number("1"), - "totalFalse": json.Number("0"), - "type": "boolean", - } - assert.Equal(t, expected, isCapital) - }) - - t.Run("int/number props", func(t *testing.T) { - population := result.Get("Aggregate", "City").AsSlice()[0].(map[string]interface{})["population"] - expected := map[string]interface{}{ - "mean": json.Number("3470000"), - "count": json.Number("1"), - "maximum": json.Number("3470000"), - "minimum": json.Number("3470000"), - "sum": json.Number("3470000"), - "type": "int", - } - assert.Equal(t, expected, population) - }) - - t.Run("ref prop", func(t *testing.T) { - inCountry := result.Get("Aggregate", "City").AsSlice()[0].(map[string]interface{})["inCountry"] - expected := map[string]interface{}{ - "pointingTo": []interface{}{"Country"}, - "type": "cref", - } - assert.Equal(t, expected, inCountry) - }) - - t.Run("string prop", func(t *testing.T) { - name := result.Get("Aggregate", "City"). - AsSlice()[0].(map[string]interface{})["name"].(map[string]interface{}) - typeField := name["type"] - topOccurrences := name["topOccurrences"] - - assert.Equal(t, schema.DataTypeText.String(), typeField) - - expectedTopOccurrences := []interface{}{ - map[string]interface{}{ - "value": "Berlin", - "occurs": json.Number("1"), - }, - } - assert.ElementsMatch(t, expectedTopOccurrences, topOccurrences) - }) - }) - - t.Run("with certainty", func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, ` - { - Aggregate{ - City (where: { - valueBoolean: true, - operator: Equal, - path: ["isCapital"] - } - nearObject: { - id: "9b9cbea5-e87e-4cd0-89af-e2f424fd52d6" - certainty: 0.9 - } - ){ - meta { - count - } - isCapital { - count - percentageFalse - percentageTrue - totalFalse - totalTrue - type - } - population { - mean - count - maximum - minimum - sum - type - } - inCountry { - pointingTo - type - } - name { - topOccurrences { - occurs - value - } - type - count - } - } - } - }`) - - t.Run("meta count", func(t *testing.T) { - meta := result.Get("Aggregate", "City").AsSlice()[0].(map[string]interface{})["meta"] - count := meta.(map[string]interface{})["count"] - expected := json.Number("1") - assert.Equal(t, expected, count) - }) - - t.Run("boolean props", func(t *testing.T) { - isCapital := result.Get("Aggregate", "City").AsSlice()[0].(map[string]interface{})["isCapital"] - expected := map[string]interface{}{ - "count": json.Number("1"), - "percentageTrue": json.Number("1"), - "percentageFalse": json.Number("0"), - "totalTrue": json.Number("1"), - "totalFalse": json.Number("0"), - "type": "boolean", - } - assert.Equal(t, expected, isCapital) - }) - - t.Run("int/number props", func(t *testing.T) { - population := result.Get("Aggregate", "City").AsSlice()[0].(map[string]interface{})["population"] - expected := map[string]interface{}{ - "mean": json.Number("3470000"), - "count": json.Number("1"), - "maximum": json.Number("3470000"), - "minimum": json.Number("3470000"), - "sum": json.Number("3470000"), - "type": "int", - } - assert.Equal(t, expected, population) - }) - - t.Run("ref prop", func(t *testing.T) { - inCountry := result.Get("Aggregate", "City").AsSlice()[0].(map[string]interface{})["inCountry"] - expected := map[string]interface{}{ - "pointingTo": []interface{}{"Country"}, - "type": "cref", - } - assert.Equal(t, expected, inCountry) - }) - - t.Run("string prop", func(t *testing.T) { - name := result.Get("Aggregate", "City"). - AsSlice()[0].(map[string]interface{})["name"].(map[string]interface{}) - typeField := name["type"] - topOccurrences := name["topOccurrences"] - - assert.Equal(t, schema.DataTypeText.String(), typeField) - - expectedTopOccurrences := []interface{}{ - map[string]interface{}{ - "value": "Berlin", - "occurs": json.Number("1"), - }, - } - assert.ElementsMatch(t, expectedTopOccurrences, topOccurrences) - }) - }) -} - -func localMetaWithNearVectorFilter(t *testing.T) { - t.Run("with distance", func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, ` - { - Aggregate{ - CustomVectorClass( - nearVector: { - vector: [1,0,0] - distance: 0.0002 - } - ){ - meta { - count - } - name { - topOccurrences { - occurs - value - } - type - count - } - } - } - }`) - - t.Run("meta count", func(t *testing.T) { - meta := result.Get("Aggregate", "CustomVectorClass").AsSlice()[0].(map[string]interface{})["meta"] - count := meta.(map[string]interface{})["count"] - expected := json.Number("1") - assert.Equal(t, expected, count) - }) - - t.Run("string prop", func(t *testing.T) { - name := result.Get("Aggregate", "CustomVectorClass"). - AsSlice()[0].(map[string]interface{})["name"].(map[string]interface{}) - typeField := name["type"] - topOccurrences := name["topOccurrences"] - - assert.Equal(t, schema.DataTypeText.String(), typeField) - - expectedTopOccurrences := []interface{}{ - map[string]interface{}{ - "value": "Mercedes", - "occurs": json.Number("1"), - }, - } - assert.ElementsMatch(t, expectedTopOccurrences, topOccurrences) - }) - }) - - t.Run("with certainty", func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, ` - { - Aggregate{ - CustomVectorClass( - nearVector: { - vector: [1,0,0] - certainty: 0.9999 - } - ){ - meta { - count - } - name { - topOccurrences { - occurs - value - } - type - count - } - } - } - }`) - - t.Run("meta count", func(t *testing.T) { - meta := result.Get("Aggregate", "CustomVectorClass").AsSlice()[0].(map[string]interface{})["meta"] - count := meta.(map[string]interface{})["count"] - expected := json.Number("1") - assert.Equal(t, expected, count) - }) - - t.Run("string prop", func(t *testing.T) { - name := result.Get("Aggregate", "CustomVectorClass"). - AsSlice()[0].(map[string]interface{})["name"].(map[string]interface{}) - typeField := name["type"] - topOccurrences := name["topOccurrences"] - - assert.Equal(t, schema.DataTypeText.String(), typeField) - - expectedTopOccurrences := []interface{}{ - map[string]interface{}{ - "value": "Mercedes", - "occurs": json.Number("1"), - }, - } - assert.ElementsMatch(t, expectedTopOccurrences, topOccurrences) - }) - }) -} - -func localMetaWithWhereAndNearVectorFilters(t *testing.T) { - t.Run("with distance", func(t *testing.T) { - t.Run("with expected results, low certainty", func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, ` - { - Aggregate { - CustomVectorClass( - where: { - valueText: "Ford" - operator: Equal - path: ["name"] - } - nearVector: { - vector: [1,0,0] - distance: 0.6 - } - ) { - meta { - count - } - name { - topOccurrences { - occurs - value - } - type - count - } - } - } - } - `) - - require.NotNil(t, result) - - agg := result.Result.(map[string]interface{})["Aggregate"].(map[string]interface{}) - cls := agg["CustomVectorClass"].([]interface{}) - require.Len(t, cls, 1) - name := cls[0].(map[string]interface{})["name"].(map[string]interface{}) - topOcc := name["topOccurrences"].([]interface{}) - require.Len(t, topOcc, 1) - val := topOcc[0].(map[string]interface{})["value"] - assert.Equal(t, "Ford", val) - }) - - t.Run("with no expected results, low distance", func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, ` - { - Aggregate { - CustomVectorClass( - where: { - valueText: "Ford" - operator: Equal - path: ["name"] - } - nearVector: { - vector: [1,0,0] - distance: 0.2 - } - ) { - meta { - count - } - name { - topOccurrences { - occurs - value - } - type - count - } - } - } - } - `) - - require.NotNil(t, result) - - agg := result.Result.(map[string]interface{})["Aggregate"].(map[string]interface{}) - cls := agg["CustomVectorClass"].([]interface{}) - require.Len(t, cls, 1) - name := cls[0].(map[string]interface{})["name"].(map[string]interface{}) - topOcc := name["topOccurrences"].([]interface{}) - require.Len(t, topOcc, 0) - }) - - t.Run("with expected results, low distance", func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, ` - { - Aggregate { - CustomVectorClass( - where: { - valueText: "Mercedes" - operator: Equal - path: ["name"] - } - nearVector: { - vector: [1,0,0] - distance: 0.1 - } - ) { - meta { - count - } - name { - topOccurrences { - occurs - value - } - type - count - } - } - } - }`) - - require.NotNil(t, result) - - agg := result.Result.(map[string]interface{})["Aggregate"].(map[string]interface{}) - cls := agg["CustomVectorClass"].([]interface{}) - require.Len(t, cls, 1) - name := cls[0].(map[string]interface{})["name"].(map[string]interface{}) - topOcc := name["topOccurrences"].([]interface{}) - require.Len(t, topOcc, 1) - val := topOcc[0].(map[string]interface{})["value"] - assert.Equal(t, "Mercedes", val) - }) - }) - - t.Run("with certainty", func(t *testing.T) { - t.Run("with expected results, low certainty", func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, ` - { - Aggregate { - CustomVectorClass( - where: { - valueText: "Ford" - operator: Equal - path: ["name"] - } - nearVector: { - vector: [1,0,0] - certainty: 0.7 - } - ) { - meta { - count - } - name { - topOccurrences { - occurs - value - } - type - count - } - } - } - } - `) - - require.NotNil(t, result) - - agg := result.Result.(map[string]interface{})["Aggregate"].(map[string]interface{}) - cls := agg["CustomVectorClass"].([]interface{}) - require.Len(t, cls, 1) - name := cls[0].(map[string]interface{})["name"].(map[string]interface{}) - topOcc := name["topOccurrences"].([]interface{}) - require.Len(t, topOcc, 1) - val := topOcc[0].(map[string]interface{})["value"] - assert.Equal(t, "Ford", val) - }) - - t.Run("with no expected results, high certainty", func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, ` - { - Aggregate { - CustomVectorClass( - where: { - valueText: "Ford" - operator: Equal - path: ["name"] - } - nearVector: { - vector: [1,0,0] - certainty: 0.9 - } - ) { - meta { - count - } - name { - topOccurrences { - occurs - value - } - type - count - } - } - } - } - `) - - require.NotNil(t, result) - - agg := result.Result.(map[string]interface{})["Aggregate"].(map[string]interface{}) - cls := agg["CustomVectorClass"].([]interface{}) - require.Len(t, cls, 1) - name := cls[0].(map[string]interface{})["name"].(map[string]interface{}) - topOcc := name["topOccurrences"].([]interface{}) - require.Len(t, topOcc, 0) - }) - - t.Run("with expected results, high certainty", func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, ` - { - Aggregate { - CustomVectorClass( - where: { - valueText: "Mercedes" - operator: Equal - path: ["name"] - } - nearVector: { - vector: [1,0,0] - certainty: 0.9 - } - ) { - meta { - count - } - name { - topOccurrences { - occurs - value - } - type - count - } - } - } - } - `) - - require.NotNil(t, result) - - agg := result.Result.(map[string]interface{})["Aggregate"].(map[string]interface{}) - cls := agg["CustomVectorClass"].([]interface{}) - require.Len(t, cls, 1) - name := cls[0].(map[string]interface{})["name"].(map[string]interface{}) - topOcc := name["topOccurrences"].([]interface{}) - require.Len(t, topOcc, 1) - val := topOcc[0].(map[string]interface{})["value"] - assert.Equal(t, "Mercedes", val) - }) - }) -} - -func localMetaWithWhereGroupByNearMediaFilters(t *testing.T) { - t.Run("with nearObject", func(t *testing.T) { - query := ` - { - Aggregate { - Company - ( - groupBy: "name" - nearObject: {id: "cfa3b21e-ca4f-4db7-a432-7fc6a23c534d", certainty: 0.99} - ) - { - groupedBy { - value - } - meta { - count - } - } - } - }` - - expected := map[string]interface{}{ - "Aggregate": map[string]interface{}{ - "Company": []interface{}{ - map[string]interface{}{ - "groupedBy": map[string]interface{}{ - "value": "Microsoft Inc.", - }, - "meta": map[string]interface{}{ - "count": json.Number("1"), - }, - }, - }, - }, - } - - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query).Result - assert.EqualValues(t, expected, result) - }) - - t.Run("with nearText", func(t *testing.T) { - t.Run("with distance", func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, ` - { - Aggregate{ - City ( - groupBy: "population" - where: { - valueBoolean: true, - operator: Equal, - path: ["isCapital"] - } - nearText: { - concepts: ["Amsterdam"] - distance: 0.2 - } - ){ - meta { - count - } - groupedBy { - value - } - } - } - } - `) - - expected := map[string]interface{}{ - "Aggregate": map[string]interface{}{ - "City": []interface{}{ - map[string]interface{}{ - "groupedBy": map[string]interface{}{ - "value": "1.8e+06", - }, - "meta": map[string]interface{}{ - "count": json.Number("1"), - }, - }, - }, - }, - } - - assert.EqualValues(t, expected, result.Result) - }) - - t.Run("with certainty", func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, ` - { - Aggregate{ - City ( - groupBy: "population" - where: { - valueBoolean: true, - operator: Equal, - path: ["isCapital"] - } - nearText: { - concepts: ["Amsterdam"] - certainty: 0.9 - } - ){ - meta { - count - } - groupedBy { - value - } - } - } - } - `) - - expected := map[string]interface{}{ - "Aggregate": map[string]interface{}{ - "City": []interface{}{ - map[string]interface{}{ - "groupedBy": map[string]interface{}{ - "value": "1.8e+06", - }, - "meta": map[string]interface{}{ - "count": json.Number("1"), - }, - }, - }, - }, - } - - assert.EqualValues(t, expected, result.Result) - }) - }) - - t.Run("with nearVector", func(t *testing.T) { - getQuery := ` - { - Get { - Company(where: { - path: ["name"] - operator: Equal - valueText: "Google Inc." - }) - { - _additional { - vector - } - } - } - }` - - vectorResult := graphqlhelper.AssertGraphQL(t, helper.RootAuth, getQuery). - Get("Get", "Company"). - AsSlice()[0].(map[string]interface{})["_additional"].(map[string]interface{})["vector"].([]interface{}) - - vector := make([]float32, len(vectorResult)) - for i, ifc := range vectorResult { - val, err := strconv.ParseFloat(ifc.(json.Number).String(), 32) - require.Nil(t, err) - vector[i] = float32(val) - } - - aggQuery := fmt.Sprintf(` - { - Aggregate { - Company - ( - groupBy: "name" - nearVector: {vector: %+v, certainty: 0.99} - ) - { - groupedBy { - value - } - meta { - count - } - } - } - } - `, vector) - - aggResult := graphqlhelper.AssertGraphQL(t, helper.RootAuth, aggQuery).Result - - expected := map[string]interface{}{ - "Aggregate": map[string]interface{}{ - "Company": []interface{}{ - map[string]interface{}{ - "groupedBy": map[string]interface{}{ - "value": "Google Inc.", - }, - "meta": map[string]interface{}{ - "count": json.Number("1"), - }, - }, - }, - }, - } - - assert.EqualValues(t, expected, aggResult) - }) -} - -func localMetaWithObjectLimit(t *testing.T) { - t.Run("with nearObject and distance", func(t *testing.T) { - objectLimit := 1 - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, fmt.Sprintf(` - { - Aggregate{ - City ( - objectLimit: %d - nearObject: { - id: "9b9cbea5-e87e-4cd0-89af-e2f424fd52d6" - distance: 0.3 - } - ){ - meta { - count - } - } - } - } - `, objectLimit)) - - t.Run("validate objectLimit functions as expected", func(t *testing.T) { - res := result.Get("Aggregate", "City").AsSlice() - require.Len(t, res, 1) - meta := res[0].(map[string]interface{})["meta"] - count := meta.(map[string]interface{})["count"] - assert.Equal(t, json.Number(fmt.Sprint(objectLimit)), count) - }) - }) - - t.Run("with nearObject and certainty", func(t *testing.T) { - objectLimit := 1 - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, fmt.Sprintf(` - { - Aggregate{ - City ( - objectLimit: %d - nearObject: { - id: "9b9cbea5-e87e-4cd0-89af-e2f424fd52d6" - certainty: 0.7 - } - ){ - meta { - count - } - } - } - } - `, objectLimit)) - - t.Run("validate objectLimit functions as expected", func(t *testing.T) { - res := result.Get("Aggregate", "City").AsSlice() - require.Len(t, res, 1) - meta := res[0].(map[string]interface{})["meta"] - count := meta.(map[string]interface{})["count"] - assert.Equal(t, json.Number(fmt.Sprint(objectLimit)), count) - }) - }) - - t.Run("with nearObject and no certainty", func(t *testing.T) { - objectLimit := 2 - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, fmt.Sprintf(` - { - Aggregate{ - City ( - objectLimit: %d - nearObject: { - id: "9b9cbea5-e87e-4cd0-89af-e2f424fd52d6" - } - ){ - meta { - count - } - } - } - } - `, objectLimit)) - - t.Run("validate objectLimit functions as expected", func(t *testing.T) { - res := result.Get("Aggregate", "City").AsSlice() - require.Len(t, res, 1) - meta := res[0].(map[string]interface{})["meta"] - count := meta.(map[string]interface{})["count"] - assert.Equal(t, json.Number(fmt.Sprint(objectLimit)), count) - }) - }) - - t.Run("with nearObject and very high distance, no objectLimit", func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, ` - { - Aggregate { - RansomNote( - nearText: { - concepts: ["abc"] - distance: 1.9998 - } - ) { - meta { - count - } - } - } - } - `) - - t.Run("validate nearMedia runs unlimited without objectLimit", func(t *testing.T) { - res := result.Get("Aggregate", "RansomNote").AsSlice() - require.Len(t, res, 1) - meta := res[0].(map[string]interface{})["meta"] - count := meta.(map[string]interface{})["count"] - assert.Equal(t, json.Number("500"), count) - }) - }) - - t.Run("with nearObject and very low certainty, no objectLimit", func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, ` - { - Aggregate { - RansomNote( - nearText: { - concepts: ["abc"] - certainty: 0.0001 - } - ) { - meta { - count - } - } - } - } - `) - - t.Run("validate nearMedia runs unlimited without objectLimit", func(t *testing.T) { - res := result.Get("Aggregate", "RansomNote").AsSlice() - require.Len(t, res, 1) - meta := res[0].(map[string]interface{})["meta"] - count := meta.(map[string]interface{})["count"] - assert.Equal(t, json.Number("500"), count) - }) - }) - - t.Run("with nearObject and low distance (few results), high objectLimit", func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, ` - { - Aggregate { - RansomNote( - nearText: { - concepts: ["abc"] - distance: 0.6 # should return about 6 elements - } - objectLimit:100, - ) { - meta { - count - } - } - } - } - `) - - t.Run("validate fewer than objectLimit elements are returned", func(t *testing.T) { - res := result.Get("Aggregate", "RansomNote").AsSlice() - require.Len(t, res, 1) - meta := res[0].(map[string]interface{})["meta"] - count := meta.(map[string]interface{})["count"] - countParsed, err := count.(json.Number).Int64() - require.Nil(t, err) - assert.Less(t, countParsed, int64(100)) - }) - }) - - t.Run("with nearObject and high certainty (few results), high objectLimit", func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, ` - { - Aggregate { - RansomNote( - nearText: { - concepts: ["abc"] - certainty: 0.7 # should return about 6 elements - } - objectLimit:100, - ) { - meta { - count - } - } - } - } - `) - - t.Run("validate fewer than objectLimit elements are returned", func(t *testing.T) { - res := result.Get("Aggregate", "RansomNote").AsSlice() - require.Len(t, res, 1) - meta := res[0].(map[string]interface{})["meta"] - count := meta.(map[string]interface{})["count"] - countParsed, err := count.(json.Number).Int64() - require.Nil(t, err) - assert.Less(t, countParsed, int64(100)) - }) - }) - - t.Run("with nearText and no distance/certainty, where filter and groupBy", func(t *testing.T) { - objectLimit := 4 - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, fmt.Sprintf(` - { - Aggregate { - Company ( - groupBy: ["name"] - where: { - valueText: "Apple*", - operator: Like, - path: ["name"] - } - objectLimit: %d - nearText: { - concepts: ["Apple"] - certainty: 0.5 - } - ){ - meta { - count - } - groupedBy { - value - } - } - } - } - `, objectLimit)) - - expected := []interface{}{ - map[string]interface{}{ - "groupedBy": map[string]interface{}{ - "value": "Apple Incorporated", - }, - "meta": map[string]interface{}{ - "count": json.Number("1"), - }, - }, - map[string]interface{}{ - "groupedBy": map[string]interface{}{ - "value": "Apple Inc.", - }, - "meta": map[string]interface{}{ - "count": json.Number("1"), - }, - }, - map[string]interface{}{ - "groupedBy": map[string]interface{}{ - "value": "Apple", - }, - "meta": map[string]interface{}{ - "count": json.Number("1"), - }, - }, - } - - companies := result.Get("Aggregate", "Company").Result.([]interface{}) - for _, company := range companies { - assert.Contains(t, expected, company) - } - }) - - t.Run("with nearObject and certainty, where filter", func(t *testing.T) { - objectLimit := 1 - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, fmt.Sprintf(` - { - Aggregate{ - City ( - where: { - valueBoolean: true, - operator: Equal, - path: ["isCapital"] - } - objectLimit: %d - nearObject: { - id: "9b9cbea5-e87e-4cd0-89af-e2f424fd52d6" - } - ){ - meta { - count - } - } - } - } - `, objectLimit)) - - t.Run("validate objectLimit functions as expected", func(t *testing.T) { - res := result.Get("Aggregate", "City").AsSlice() - require.Len(t, res, 1) - meta := res[0].(map[string]interface{})["meta"] - count := meta.(map[string]interface{})["count"] - assert.Equal(t, json.Number(fmt.Sprint(objectLimit)), count) - }) - }) -} - -func aggregatesOnDateFields(t *testing.T) { - t.Run("without grouping", func(t *testing.T) { - query := ` - { - Aggregate { - HasDateField { - timestamp { - count - minimum - maximum - median - } - } - } - }` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query).Get("Aggregate", "HasDateField").AsSlice() - assert.Len(t, result, 1) - - expected := []interface{}{ - map[string]interface{}{ - "timestamp": map[string]interface{}{ - "count": json.Number("10"), - "maximum": "2022-06-16T22:19:11.837473Z", - "median": "2022-06-16T22:19:06.1449075Z", - "minimum": "2022-06-16T22:18:59.640162Z", - }, - }, - } - assert.Equal(t, expected, result) - }) - - t.Run("with grouping on a unique field", func(t *testing.T) { - query := ` - { - Aggregate { - HasDateField - ( - groupBy: "unique" - ) - { - timestamp { - count - minimum - maximum - median - mode - } - } - } - }` - - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query).Get("Aggregate", "HasDateField").AsSlice() - assert.Len(t, result, 10) - - expected := []interface{}{ - map[string]interface{}{ - "timestamp": map[string]interface{}{ - "count": json.Number("1"), - "maximum": "2022-06-16T22:19:05.894857Z", - "median": "2022-06-16T22:19:05.894857Z", - "minimum": "2022-06-16T22:19:05.894857Z", - "mode": "2022-06-16T22:19:05.894857Z", - }, - }, - map[string]interface{}{ - "timestamp": map[string]interface{}{ - "count": json.Number("1"), - "maximum": "2022-06-16T22:19:08.112395Z", - "median": "2022-06-16T22:19:08.112395Z", - "minimum": "2022-06-16T22:19:08.112395Z", - "mode": "2022-06-16T22:19:08.112395Z", - }, - }, - map[string]interface{}{ - "timestamp": map[string]interface{}{ - "count": json.Number("1"), - "maximum": "2022-06-16T22:19:03.495596Z", - "median": "2022-06-16T22:19:03.495596Z", - "minimum": "2022-06-16T22:19:03.495596Z", - "mode": "2022-06-16T22:19:03.495596Z", - }, - }, - map[string]interface{}{ - "timestamp": map[string]interface{}{ - "count": json.Number("1"), - "maximum": "2022-06-16T22:19:07.589828Z", - "median": "2022-06-16T22:19:07.589828Z", - "minimum": "2022-06-16T22:19:07.589828Z", - "mode": "2022-06-16T22:19:07.589828Z", - }, - }, - map[string]interface{}{ - "timestamp": map[string]interface{}{ - "count": json.Number("1"), - "maximum": "2022-06-16T22:19:06.394958Z", - "median": "2022-06-16T22:19:06.394958Z", - "minimum": "2022-06-16T22:19:06.394958Z", - "mode": "2022-06-16T22:19:06.394958Z", - }, - }, - map[string]interface{}{ - "timestamp": map[string]interface{}{ - "count": json.Number("1"), - "maximum": "2022-06-16T22:19:11.837473Z", - "median": "2022-06-16T22:19:11.837473Z", - "minimum": "2022-06-16T22:19:11.837473Z", - "mode": "2022-06-16T22:19:11.837473Z", - }, - }, - map[string]interface{}{ - "timestamp": map[string]interface{}{ - "count": json.Number("1"), - "maximum": "2022-06-16T22:18:59.640162Z", - "median": "2022-06-16T22:18:59.640162Z", - "minimum": "2022-06-16T22:18:59.640162Z", - "mode": "2022-06-16T22:18:59.640162Z", - }, - }, - map[string]interface{}{ - "timestamp": map[string]interface{}{ - "count": json.Number("1"), - "maximum": "2022-06-16T22:19:01.495967Z", - "median": "2022-06-16T22:19:01.495967Z", - "minimum": "2022-06-16T22:19:01.495967Z", - "mode": "2022-06-16T22:19:01.495967Z", - }, - }, - map[string]interface{}{ - "timestamp": map[string]interface{}{ - "count": json.Number("1"), - "maximum": "2022-06-16T22:19:10.339493Z", - "median": "2022-06-16T22:19:10.339493Z", - "minimum": "2022-06-16T22:19:10.339493Z", - "mode": "2022-06-16T22:19:10.339493Z", - }, - }, - map[string]interface{}{ - "timestamp": map[string]interface{}{ - "count": json.Number("1"), - "maximum": "2022-06-16T22:19:04.3828349Z", - "median": "2022-06-16T22:19:04.3828349Z", - "minimum": "2022-06-16T22:19:04.3828349Z", - "mode": "2022-06-16T22:19:04.3828349Z", - }, - }, - } - - for _, res := range result { - assert.Contains(t, expected, res) - } - }) - - t.Run("group on identical field", func(t *testing.T) { - query := ` - { - Aggregate { - HasDateField - ( - groupBy: "identical" - ) - { - timestamp { - count - minimum - maximum - median - } - } - } - }` - - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query).Get("Aggregate", "HasDateField").AsSlice() - - expected := []interface{}{ - map[string]interface{}{ - "timestamp": map[string]interface{}{ - "count": json.Number("10"), - "maximum": "2022-06-16T22:19:11.837473Z", - "median": "2022-06-16T22:19:06.1449075Z", - "minimum": "2022-06-16T22:18:59.640162Z", - }, - }, - } - - assert.Equal(t, expected, result) - }) -} diff --git a/test/acceptance/graphql_resolvers/local_aggregate_with_expected_failures.go b/test/acceptance/graphql_resolvers/local_aggregate_with_expected_failures.go deleted file mode 100644 index 0df70027c3baeaac68fb160fa9c6196cef0a3771..0000000000000000000000000000000000000000 --- a/test/acceptance/graphql_resolvers/local_aggregate_with_expected_failures.go +++ /dev/null @@ -1,353 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/test/helper" - graphqlhelper "github.com/weaviate/weaviate/test/helper/graphql" -) - -func aggregatesWithExpectedFailures(t *testing.T) { - t.Run("with nearVector, no certainty", func(t *testing.T) { - result := graphqlhelper.ErrorGraphQL(t, helper.RootAuth, ` - { - Aggregate{ - CustomVectorClass( - nearVector: { - vector: [1,0,0] - } - ){ - meta { - count - } - name { - topOccurrences { - occurs - value - } - type - count - } - } - } - } - `) - - require.NotEmpty(t, result) - require.Len(t, result, 1) - assert.True(t, strings.Contains(result[0].Message, - "must provide certainty or objectLimit with vector search"), - "unexpected error message: %s", result[0].Message) - }) - - t.Run("with nearObject, no certainty", func(t *testing.T) { - result := graphqlhelper.ErrorGraphQL(t, helper.RootAuth, ` - { - Aggregate{ - City( - nearObject: { - id: "9b9cbea5-e87e-4cd0-89af-e2f424fd52d6" - } - ){ - meta { - count - } - name { - topOccurrences { - occurs - value - } - type - count - } - } - } - } - `) - - require.NotEmpty(t, result) - require.Len(t, result, 1) - assert.True(t, strings.Contains(result[0].Message, - "must provide certainty or objectLimit with vector search"), - "unexpected error message: %s", result[0].Message) - }) - - t.Run("with nearText, no certainty", func(t *testing.T) { - result := graphqlhelper.ErrorGraphQL(t, helper.RootAuth, ` - { - Aggregate{ - City( - nearText: { - concepts: ["Amsterdam"] - } - ){ - meta { - count - } - name { - topOccurrences { - occurs - value - } - type - count - } - } - } - } - `) - - require.NotEmpty(t, result) - require.Len(t, result, 1) - assert.True(t, strings.Contains(result[0].Message, - "must provide certainty or objectLimit with vector search"), - "unexpected error message: %s", result[0].Message) - }) - - t.Run("with nearVector, where filter, no certainty", func(t *testing.T) { - result := graphqlhelper.ErrorGraphQL(t, helper.RootAuth, ` - { - Aggregate{ - CustomVectorClass( - where: { - valueText: "Mercedes", - operator: Equal, - path: ["name"] - } - nearVector: { - vector: [1,0,0] - } - ){ - meta { - count - } - name { - topOccurrences { - occurs - value - } - type - count - } - } - } - } - `) - - require.NotEmpty(t, result) - require.Len(t, result, 1) - assert.True(t, strings.Contains(result[0].Message, - "must provide certainty or objectLimit with vector search"), - "unexpected error message: %s", result[0].Message) - }) - - t.Run("with nearObject, where filter, no certainty", func(t *testing.T) { - result := graphqlhelper.ErrorGraphQL(t, helper.RootAuth, ` - { - Aggregate{ - City (where: { - valueBoolean: true, - operator: Equal, - path: ["isCapital"] - } - nearObject: { - id: "9b9cbea5-e87e-4cd0-89af-e2f424fd52d6" - } - ){ - meta { - count - } - isCapital { - count - percentageFalse - percentageTrue - totalFalse - totalTrue - type - } - population { - mean - count - maximum - minimum - sum - type - } - inCountry { - pointingTo - type - } - name { - topOccurrences { - occurs - value - } - type - count - } - } - } - } - `) - - require.NotEmpty(t, result) - require.Len(t, result, 1) - assert.True(t, strings.Contains(result[0].Message, - "must provide certainty or objectLimit with vector search"), - "unexpected error message: %s", result[0].Message) - }) - - t.Run("with nearText, where filter, no certainty", func(t *testing.T) { - result := graphqlhelper.ErrorGraphQL(t, helper.RootAuth, ` - { - Aggregate{ - City (where: { - valueBoolean: true, - operator: Equal, - path: ["isCapital"] - } - nearText: { - concepts: ["Amsterdam"] - } - ){ - meta { - count - } - isCapital { - count - percentageFalse - percentageTrue - totalFalse - totalTrue - type - } - population { - mean - count - maximum - minimum - sum - type - } - inCountry { - pointingTo - type - } - name { - topOccurrences { - occurs - value - } - type - count - } - } - } - } - `) - - require.NotEmpty(t, result) - require.Len(t, result, 1) - assert.True(t, strings.Contains(result[0].Message, - "must provide certainty or objectLimit with vector search"), - "unexpected error message: %s", result[0].Message) - }) - - t.Run("objectLimit passed with no nearMedia", func(t *testing.T) { - result := graphqlhelper.ErrorGraphQL(t, helper.RootAuth, ` - { - Aggregate{ - CustomVectorClass(objectLimit: 1){ - meta { - count - } - name { - topOccurrences { - occurs - value - } - type - count - } - } - } - } - `) - - require.NotEmpty(t, result) - require.Len(t, result, 1) - assert.True(t, strings.Contains(result[0].Message, "objectLimit can only be used with a near or hybrid filter")) - }) -} - -func exploreWithExpectedFailures(t *testing.T) { - t.Run("Explore called when classes have different distance configs", func(t *testing.T) { - className := "L2DistanceClass" - defer deleteObjectClass(t, className) - - t.Run("create class configured with non-default distance type", func(t *testing.T) { - createObjectClass(t, &models.Class{ - Class: className, - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "vectorizeClassName": true, - }, - }, - VectorIndexConfig: map[string]interface{}{ - "distance": "l2-squared", - }, - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - }) - }) - - t.Run("assert failure to Explore with mismatched distance types", func(t *testing.T) { - query := ` - { - Explore(nearVector: {vector:[1,1,1]}) { - distance - } - }` - - result := graphqlhelper.ErrorGraphQL(t, helper.RootAuth, query) - assert.Len(t, result, 1) - - errMsg := result[0].Message - assert.Contains(t, errMsg, "vector search across classes not possible") - assert.Contains(t, errMsg, "found different distance metrics") - assert.Contains(t, errMsg, "class 'L2DistanceClass' uses distance metric 'l2-squared'") - assert.Contains(t, errMsg, "class 'Airport' uses distance metric 'cosine'") - assert.Contains(t, errMsg, "class 'Person' uses distance metric 'cosine'") - assert.Contains(t, errMsg, "class 'ArrayClass' uses distance metric 'cosine'") - assert.Contains(t, errMsg, "class 'HasDateField' uses distance metric 'cosine'") - assert.Contains(t, errMsg, "class 'CustomVectorClass' uses distance metric 'cosine'") - assert.Contains(t, errMsg, "class 'RansomNote' uses distance metric 'cosine'") - assert.Contains(t, errMsg, "class 'MultiShard' uses distance metric 'cosine'") - assert.Contains(t, errMsg, "class 'Country' uses distance metric 'cosine'") - assert.Contains(t, errMsg, "class 'City' uses distance metric 'cosine'") - assert.Contains(t, errMsg, "class 'Company' uses distance metric 'cosine'") - assert.Contains(t, errMsg, "class 'Pizza' uses distance metric 'cosine'") - }) - }) -} diff --git a/test/acceptance/graphql_resolvers/local_get_cursor_test.go b/test/acceptance/graphql_resolvers/local_get_cursor_test.go deleted file mode 100644 index b84a31e443e67e056a532a79ff2ec769a11b3be3..0000000000000000000000000000000000000000 --- a/test/acceptance/graphql_resolvers/local_get_cursor_test.go +++ /dev/null @@ -1,182 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "fmt" - "testing" - - "github.com/go-openapi/strfmt" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/test/helper" - graphqlhelper "github.com/weaviate/weaviate/test/helper/graphql" - "github.com/weaviate/weaviate/test/helper/sample-schema/multishard" -) - -func getWithCursorSearch(t *testing.T) { - t.Run("listing objects using cursor api", func(t *testing.T) { - tests := []struct { - name string - className string - after string - limit int - filter string - expectedIDs []strfmt.UUID - expectedErrorMsg string - }{ - { - name: `cursor with after: "" limit: 2`, - className: "CursorClass", - after: "", - limit: 2, - expectedIDs: []strfmt.UUID{ - cursorClassID1, - cursorClassID2, - cursorClassID3, - cursorClassID4, - cursorClassID5, - cursorClassID6, - cursorClassID7, - }, - }, - { - name: fmt.Sprintf("cursor with after: \"%s\" limit: 1", cursorClassID4), - className: "CursorClass", - after: cursorClassID4.String(), - limit: 1, - expectedIDs: []strfmt.UUID{ - cursorClassID5, - cursorClassID6, - cursorClassID7, - }, - }, - { - name: "error with offset", - className: "CursorClass", - filter: `limit: 1 after: "" offset: 1`, - expectedErrorMsg: "cursor api: invalid 'after' parameter: offset cannot be set with after and limit parameters", - }, - { - name: "error with nearObject", - className: "CursorClass", - filter: fmt.Sprintf("limit: 1 after: \"\" nearObject:{id:\"%s\"}", cursorClassID1), - expectedErrorMsg: "cursor api: invalid 'after' parameter: other params cannot be set with after and limit parameters", - }, - { - name: "error with nearVector", - className: "CursorClass", - filter: `limit: 1 after: "" nearVector:{vector:[0.1, 0.2]}`, - expectedErrorMsg: "cursor api: invalid 'after' parameter: other params cannot be set with after and limit parameters", - }, - { - name: "error with hybrid", - className: "CursorClass", - filter: `limit: 1 after: "" hybrid:{query:"cursor api"}`, - expectedErrorMsg: "cursor api: invalid 'after' parameter: other params cannot be set with after and limit parameters", - }, - { - name: "error with bm25", - className: "CursorClass", - filter: `limit: 1 after: "" bm25:{query:"cursor api"}`, - expectedErrorMsg: "cursor api: invalid 'after' parameter: other params cannot be set with after and limit parameters", - }, - { - name: "error with sort", - className: "CursorClass", - filter: `limit: 1 after: "" sort:{path:"name"}`, - expectedErrorMsg: "cursor api: invalid 'after' parameter: sort cannot be set with after and limit parameters", - }, - { - name: "error with where", - className: "CursorClass", - filter: `limit: 1 after: "" where:{path:"id" operator:Like valueText:"*"}`, - expectedErrorMsg: "cursor api: invalid 'after' parameter: where cannot be set with after and limit parameters", - }, - { - name: "error with bm25, hybrid and offset", - className: "CursorClass", - filter: `limit: 1 after: "" bm25:{query:"cursor api"} hybrid:{query:"cursor api"} offset:1`, - expectedErrorMsg: "cursor api: invalid 'after' parameter: other params cannot be set with after and limit parameters", - }, - { - name: "error with no limit set", - className: "CursorClass", - filter: `after:"00000000-0000-0000-0000-000000000000"`, - expectedErrorMsg: "cursor api: invalid 'after' parameter: limit parameter must be set", - }, - // multi shard - { - name: `multi shard cursor with after: "" limit: 1`, - className: "MultiShard", - after: "", - limit: 1, - expectedIDs: []strfmt.UUID{ - multishard.MultiShardID1, - multishard.MultiShardID2, - multishard.MultiShardID3, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - query := "{ Get { " + tt.className + " %s { _additional { id } } } }" - if len(tt.expectedErrorMsg) > 0 { - errQuery := fmt.Sprintf(query, fmt.Sprintf("(%s)", tt.filter)) - result := graphqlhelper.ErrorGraphQL(t, helper.RootAuth, errQuery) - assert.Len(t, result, 1) - - errMsg := result[0].Message - assert.Equal(t, tt.expectedErrorMsg, errMsg) - } else { - parseResults := func(t *testing.T, cities []interface{}) []strfmt.UUID { - var ids []strfmt.UUID - for _, city := range cities { - id, ok := city.(map[string]interface{})["_additional"].(map[string]interface{})["id"] - require.True(t, ok) - - idString, ok := id.(string) - require.True(t, ok) - - ids = append(ids, strfmt.UUID(idString)) - } - return ids - } - // use cursor api - cursorSearch := func(t *testing.T, className, after string, limit int) []strfmt.UUID { - cursor := fmt.Sprintf(`(limit: %v after: "%s")`, limit, after) - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, fmt.Sprintf(query, cursor)) - cities := result.Get("Get", className).AsSlice() - return parseResults(t, cities) - } - - var cursorIDs []strfmt.UUID - after, limit := tt.after, tt.limit - for { - result := cursorSearch(t, tt.className, after, limit) - cursorIDs = append(cursorIDs, result...) - if len(result) == 0 { - break - } - after = result[len(result)-1].String() - } - - assert.ElementsMatch(t, tt.expectedIDs, cursorIDs) - require.Equal(t, len(tt.expectedIDs), len(cursorIDs)) - for i := range tt.expectedIDs { - assert.Equal(t, tt.expectedIDs[i], cursorIDs[i]) - } - } - }) - } - }) -} diff --git a/test/acceptance/graphql_resolvers/local_get_hybrid_search_test.go b/test/acceptance/graphql_resolvers/local_get_hybrid_search_test.go deleted file mode 100644 index 61672472ffcda26b11ab0ef3f4173f4e0f1cc580..0000000000000000000000000000000000000000 --- a/test/acceptance/graphql_resolvers/local_get_hybrid_search_test.go +++ /dev/null @@ -1,257 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/test/helper" - graphqlhelper "github.com/weaviate/weaviate/test/helper/graphql" -) - -// appleVec is the t2v-contextionary representation of "Apple Inc." -var appleVec = []float32{ - 0.1156649, -0.3561866, 0.4718789, 0.37318036, 0.39549947, 0.019409189, -0.5052104, -0.49448758, - 0.34452468, 0.46354344, 0.1932035, 0.51334095, 0.06032639, 0.022086846, 0.20391269, 0.3013975, - 0.18838425, -0.2362212, -0.25797912, -0.11189923, -0.14507815, 0.3113891, -0.90078014, 0.027230136, - -0.5541761, -0.33453932, 0.9467, 0.39270592, 0.0775289, -0.14601035, -0.5497628, 0.34385568, - 0.5363504, 0.03164669, 0.03510879, -0.37564012, 0.22805381, -0.66345274, -0.92397606, 0.85855925, - -0.5637805, 0.035184387, 0.23299722, -0.042199645, -0.52195567, -0.17418303, -0.029039165, 0.4399605, - 0.36524323, 0.21769615, -0.1977588, -0.17114285, 0.30731055, -0.6743735, 0.25451374, 0.41582933, - 0.61602086, 0.3382223, 0.39701316, -0.54065305, -0.16107371, -0.80420196, -0.42476287, 0.40522298, - -0.24763498, -0.7224363, -0.5512907, -0.0400732, -0.09994836, -0.2354202, 0.2904534, -0.12089672, - -0.07095274, -0.8213324, 0.3695029, 0.27129403, 0.28678897, -0.108535565, -0.30699188, 0.10705576, - 0.08372605, -0.64183795, 0.34861454, -0.30277634, 0.21602349, -0.23038381, -0.10144254, -0.47548878, - 0.3525676, 0.3357812, 0.031383604, -0.32346088, -0.7515443, -0.14595662, -0.1425658, 0.54312915, - -0.60661954, 0.10959545, -0.17200017, 0.60667217, -0.22193804, 0.5861486, 0.4714104, -0.4168524, - -0.23929326, 0.47505698, -0.5256647, 0.23308091, 0.16735256, -0.021147087, -0.6238067, -0.065388694, - 0.38134024, 0.17625189, -0.048189547, -0.40676376, -0.20627557, -0.6200684, 0.24607961, -0.7479579, - 0.36243674, -0.41451588, -0.3258561, 0.07216902, 0.15214325, 0.2363326, 1.7854439, 0.2354896, - -0.80430084, 0.39550564, 0.06727363, 0.45679152, 0.09223966, -0.17635022, 0.065364204, -0.6799169, - 0.46794528, -0.6863512, -0.007789179, 0.0216118, 0.3218315, -0.329095, -0.15101263, 0.054294955, - 0.35598493, 0.8095643, 0.4240984, 0.107904576, -0.65505075, -0.25601476, -0.040415946, 0.57646215, - -0.14216466, -0.5626221, 0.21731018, 0.25857863, 0.029463748, -0.043640777, -0.86262965, 0.0075217593, - -0.65511745, 0.30682194, 0.36109644, -0.34552526, -0.57620883, -0.111058705, 0.42360848, 0.22977945, - 0.058191486, -0.6967789, -0.083894424, 0.21894856, -0.15210733, 0.2840013, -0.66721946, -0.12251554, - -0.55239767, -0.06489324, -0.17015795, -0.15400846, 0.14791602, -0.76380575, 0.27046034, -0.47688308, - 0.25788718, -0.074898824, 0.181136, 0.6860475, -0.14676934, 0.13610536, 0.74407804, -0.26433572, - -0.09919782, -0.26012585, -0.18844572, 0.8116442, 0.24614683, 0.076953486, 0.41485175, -0.64702696, - -0.5514351, -0.44831908, 0.7871427, 0.1256176, -0.37650946, 0.26002303, 0.55952126, -0.5275842, - 0.7185946, 0.09147637, -0.3937243, 0.10171145, -0.6451931, 0.8872601, 0.011252741, 1.1493335, - 0.7991122, -0.16108659, -0.7322848, 0.5237607, -0.50677204, 0.12007416, -0.6966177, -0.5039344, - 0.020131318, 0.15328859, -1.0066653, 0.32302102, -0.36504102, 0.37823763, -0.19183074, -0.4154492, - 0.14257756, 0.6225165, -0.24297066, 0.014472419, 0.8159169, 1.2461865, 0.07883369, -0.35416773, - -0.06593153, -0.81301326, 0.17566697, -0.04062626, -0.112336636, -0.22738501, -0.42422646, 0.458409, - 0.79599, 0.33880755, 0.39182758, 0.054381482, 0.5805471, 0.25382927, -0.16633242, 0.08435115, - 0.53753984, -0.16825016, -0.69669664, 0.21506411, -0.35470957, 0.25212923, 0.20211501, 0.6161077, - -0.077442676, -0.024064686, -0.18163882, 0.6834761, -1.0793741, 0.25927436, -0.69374615, -0.025031673, - -0.1307808, -0.5026866, -0.14586367, -0.41198593, -0.4018977, 0.10252101, -0.22274522, 0.9635526, - -0.17163973, 0.1639396, 0.66181034, -0.42865846, -0.18711954, -0.23968346, -0.09696686, 0.38911402, - 0.0962325, 0.46173036, 0.10814153, 1.0249863, -0.2061986, 0.6657442, -0.3277397, 0.26586995, - -0.12981872, 0.40097368, -0.49962977, -0.61136127, -} - -func getWithHybridSearch(t *testing.T) { - t.Run("without references", func(t *testing.T) { - query := ` - { - Get { - Airport - ( - hybrid: { - alpha: 0 - query: "10000" - } - ) - { - code - } - } - }` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query).Get("Get", "Airport").AsSlice() - require.Len(t, result, 1) - assert.EqualValues(t, map[string]interface{}{"code": "10000"}, result[0]) - }) - - t.Run("with limit and vector", func(t *testing.T) { - limit := 2 - query := fmt.Sprintf(` - { - Get { - Company( - limit: %d - hybrid: { - query: "Apple", - alpha: 0.5, - vector: %s - } - ) { - name - } - } - }`, limit, graphqlhelper.Vec2String(appleVec)) - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query).Get("Get", "Company").AsSlice() - require.Len(t, result, limit) - assert.Contains(t, result, map[string]interface{}{ - "name": "Apple", - }) - assert.Contains(t, result, map[string]interface{}{ - "name": "Apple Inc.", - }) - }) - - t.Run("with limit and no vector", func(t *testing.T) { - limit := 2 - query := fmt.Sprintf(` - { - Get { - Company( - limit: %d - hybrid: { - query: "Apple", - alpha: 0.5, - } - ) { - name - } - } - }`, limit) - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query).Get("Get", "Company").AsSlice() - require.Len(t, result, limit) - assert.Contains(t, result, map[string]interface{}{ - "name": "Apple", - }) - assert.Contains(t, result, map[string]interface{}{ - "name": "Apple Inc.", - }) - }) - - t.Run("with no limit and vector", func(t *testing.T) { - query := fmt.Sprintf(` - { - Get { - Company( - hybrid: { - query: "Apple", - alpha: 0.5, - vector: %s - } - ) { - name - } - } - }`, graphqlhelper.Vec2String(appleVec)) - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query).Get("Get", "Company").AsSlice() - require.Len(t, result, 9) - }) - - t.Run("with no limit and no vector", func(t *testing.T) { - query := ` - { - Get { - Company( - hybrid: { - query: "Apple", - alpha: 0.5, - } - ) { - name - } - } - }` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query).Get("Get", "Company").AsSlice() - require.Len(t, result, 9) - }) - - t.Run("with _additional{vector}", func(t *testing.T) { - query := ` - { - Get { - Company( - hybrid: { - query: "Apple", - alpha: 0.5, - } - ) { - _additional { - vector - } - } - } - }` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query).Get("Get", "Company").AsSlice() - require.Len(t, result, 9) - for _, res := range result { - company := res.(map[string]interface{}) - addl := company["_additional"].(map[string]interface{}) - vec, found := addl["vector"] - assert.True(t, found) - assert.Len(t, vec, 300) - } - }) - - t.Run("with references", func(t *testing.T) { - query := ` - { - Get { - Airport - ( - hybrid: { - alpha: 0.5 - query: "1000" - } - ) - { - code - inCity { - ... on City { - name - } - } - } - } - }` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query).Get("Get", "Airport").AsSlice() - require.Len(t, result, 4) - assert.Contains(t, result, - map[string]interface{}{ - "code": "10000", - "inCity": []interface{}{ - map[string]interface{}{"name": "Amsterdam"}, - }, - }) - assert.Contains(t, result, - map[string]interface{}{ - "code": "20000", - "inCity": []interface{}{ - map[string]interface{}{"name": "Rotterdam"}, - }, - }) - assert.Contains(t, result, - map[string]interface{}{ - "code": "30000", - "inCity": []interface{}{ - map[string]interface{}{"name": "Dusseldorf"}, - }, - }) - assert.Contains(t, result, - map[string]interface{}{ - "code": "40000", - "inCity": []interface{}{ - map[string]interface{}{"name": "Berlin"}, - }, - }) - }) -} diff --git a/test/acceptance/graphql_resolvers/local_get_shadow_test.go b/test/acceptance/graphql_resolvers/local_get_shadow_test.go deleted file mode 100644 index e862cc6d4d80dca6c8587dd4bc13f0c379caf41f..0000000000000000000000000000000000000000 --- a/test/acceptance/graphql_resolvers/local_get_shadow_test.go +++ /dev/null @@ -1,140 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "encoding/json" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/test/helper" - graphqlhelper "github.com/weaviate/weaviate/test/helper/graphql" -) - -// run by setup_test.go -func runningGetNearObjectWithShadowedObjects(t *testing.T) { - t.Run("running Get nearObject against shadow class", func(t *testing.T) { - query := ` - { - Get { - NearObjectSearch ( - nearObject: { - id : "aa44bbee-ca5f-4db7-a412-5fc6a2300001" - certainty: 0.98 - } - ) { - name - } - } - } - ` - - for i := 0; i < 50; i++ { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - objs := result.Get("Get", "NearObjectSearch").AsSlice() - - expected := []interface{}{ - map[string]interface{}{"name": "Mount Everest"}, - } - - assert.Len(t, objs, 1) - assert.ElementsMatch(t, expected, objs) - } - }) -} - -func runningAggregateNearObjectWithShadowedObjects(t *testing.T) { - t.Run("running Aggregate nearObject against shadow class", func(t *testing.T) { - query := ` - { - Aggregate { - NearObjectSearch ( - nearObject: { - id : "aa44bbee-ca5f-4db7-a412-5fc6a2300001" - certainty: 0.98 - } - ) { - meta { - count - } - } - } - } - ` - - for i := 0; i < 50; i++ { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - meta := result.Get("Aggregate", "NearObjectSearch").AsSlice()[0].(map[string]interface{})["meta"] - count := meta.(map[string]interface{})["count"] - expected := json.Number("1") - assert.Equal(t, expected, count) - } - }) -} - -func runningExploreNearObjectWithShadowedObjects(t *testing.T) { - t.Run("running Explore nearObject against shadow class with same contents", func(t *testing.T) { - query := ` - { - Explore ( - nearObject: { - id : "aa44bbee-ca5f-4db7-a412-5fc6a2300011" - certainty: 0.98 - } - ) { - beacon - } - } - ` - - for i := 0; i < 50; i++ { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - objs := result.Get("Explore").AsSlice() - - expected := []interface{}{ - map[string]interface{}{"beacon": "weaviate://localhost/NearObjectSearch/aa44bbee-ca5f-4db7-a412-5fc6a2300011"}, - map[string]interface{}{"beacon": "weaviate://localhost/NearObjectSearchShadow/aa44bbee-ca5f-4db7-a412-5fc6a2300011"}, - } - - assert.Len(t, objs, 2) - assert.ElementsMatch(t, expected, objs) - } - }) - - t.Run("running Explore nearObject against shadow class with different contents", func(t *testing.T) { - query := ` - { - Explore ( - nearObject: { - id : "aa44bbee-ca5f-4db7-a412-5fc6a2300001" - certainty: 0.98 - } - ) { - beacon - } - } - ` - - for i := 0; i < 50; i++ { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - objs := result.Get("Explore").AsSlice() - - expected := []interface{}{ - map[string]interface{}{"beacon": "weaviate://localhost/NearObjectSearch/aa44bbee-ca5f-4db7-a412-5fc6a2300001"}, - map[string]interface{}{"beacon": "weaviate://localhost/NearObjectSearchShadow/aa44bbee-ca5f-4db7-a412-5fc6a2300001"}, - } - - assert.Len(t, objs, 2) - assert.ElementsMatch(t, expected, objs) - } - }) -} diff --git a/test/acceptance/graphql_resolvers/local_get_test.go b/test/acceptance/graphql_resolvers/local_get_test.go deleted file mode 100644 index 36fffc833fbebdd855e0d40728a9e7e07852d6a2..0000000000000000000000000000000000000000 --- a/test/acceptance/graphql_resolvers/local_get_test.go +++ /dev/null @@ -1,138 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "bytes" - "encoding/json" - "fmt" - "net/http" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/test/helper" - graphqlhelper "github.com/weaviate/weaviate/test/helper/graphql" -) - -// run by setup_test.go -func gettingObjects(t *testing.T) { - t.Run("listing cities without references", func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, "{ Get { City { name } } }") - cities := result.Get("Get", "City").AsSlice() - - expected := []interface{}{ - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": "Rotterdam"}, - map[string]interface{}{"name": "Berlin"}, - map[string]interface{}{"name": "Dusseldorf"}, - map[string]interface{}{"name": "Missing Island"}, - map[string]interface{}{"name": nil}, - } - - assert.ElementsMatch(t, expected, cities) - }) - - t.Run("listing cities with relations", func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, "{ Get { City { name, inCountry { ... on Country { name } } } } }") - cities := result.Get("Get", "City").AsSlice() - - expected := parseJSONSlice(`[ - { "name": "Amsterdam", "inCountry": [{ "name": "Netherlands" }] }, - { "name": "Rotterdam", "inCountry": [{ "name": "Netherlands" }] }, - { "name": "Berlin", "inCountry": [{ "name": "Germany" }] }, - { "name": "Dusseldorf", "inCountry": [{ "name": "Germany" }] }, - { "name": "Missing Island", "inCountry": null }, - { "name": null, "inCountry": null } - ]`) - - assert.ElementsMatch(t, expected, cities) - }) - - t.Run("make sure raw response contains no error key", func(t *testing.T) { - // This test prevents a regression on gh-1535 - - query := []byte(`{"query":"{ Get { City { name } } }"}`) - res, err := http.Post(fmt.Sprintf("%s%s", helper.GetWeaviateURL(), "/v1/graphql"), - "application/json", bytes.NewReader(query)) - require.Nil(t, err) - - defer res.Body.Close() - var body map[string]interface{} - err = json.NewDecoder(res.Body).Decode(&body) - require.Nil(t, err) - - _, ok := body["errors"] - assert.False(t, ok) - - cities := body["data"].(map[string]interface{})["Get"].(map[string]interface{})["City"].([]interface{}) - assert.Greater(t, len(cities), 0) - }) - - t.Run("listing cities with limit", func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, "{ Get { City(limit: 2) { name } } }") - cities := result.Get("Get", "City").AsSlice() - - expected := []interface{}{ - map[string]interface{}{"name": "Rotterdam"}, - map[string]interface{}{"name": "Dusseldorf"}, - } - - assert.ElementsMatch(t, expected, cities) - }) - - t.Run("listing cities with offset and limit", func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, "{ Get { City(offset: 2 limit: 2) { name } } }") - cities := result.Get("Get", "City").AsSlice() - - expected := []interface{}{ - map[string]interface{}{"name": "Missing Island"}, - map[string]interface{}{"name": nil}, - } - - assert.ElementsMatch(t, expected, cities) - }) - - t.Run("listing cities with offset", func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, "{ Get { City(offset: 2) { name } } }") - cities := result.Get("Get", "City").AsSlice() - - expected := []interface{}{ - map[string]interface{}{"name": "Missing Island"}, - map[string]interface{}{"name": nil}, - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": "Berlin"}, - } - - assert.ElementsMatch(t, expected, cities) - }) - - t.Run("listing cities with offset and limit beyond results size", func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, "{ Get { City(offset: 5 limit: 10) { name } } }") - cities := result.Get("Get", "City").AsSlice() - - expected := []interface{}{ - map[string]interface{}{"name": "Berlin"}, - } - - assert.ElementsMatch(t, expected, cities) - }) - - t.Run("listing cities with offset beyond results size", func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, "{ Get { City(offset: 6) { name } } }") - cities := result.Get("Get", "City").AsSlice() - - expected := []interface{}{} - - assert.ElementsMatch(t, expected, cities) - }) -} diff --git a/test/acceptance/graphql_resolvers/local_get_with_additional_test.go b/test/acceptance/graphql_resolvers/local_get_with_additional_test.go deleted file mode 100644 index 8cd7a23986c5cb49f7a4ab28bb4e4c9ede5adc4f..0000000000000000000000000000000000000000 --- a/test/acceptance/graphql_resolvers/local_get_with_additional_test.go +++ /dev/null @@ -1,347 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "encoding/json" - "testing" - - graphqlhelper "github.com/weaviate/weaviate/test/helper/graphql" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/test/helper" -) - -func gettingObjectsWithAdditionalProps(t *testing.T) { - t.Run("with vector set", func(t *testing.T) { - query := ` - { - Get { - Company { - _additional { - vector - } - name - } - } - } - ` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - companies := result.Get("Get", "Company").AsSlice() - - require.Greater(t, len(companies), 0) - for _, comp := range companies { - vec, ok := comp.(map[string]interface{})["_additional"].(map[string]interface{})["vector"] - require.True(t, ok) - - vecSlice, ok := vec.([]interface{}) - require.True(t, ok) - require.Greater(t, len(vecSlice), 0) - - asFloat, err := vecSlice[0].(json.Number).Float64() - require.Nil(t, err) - assert.True(t, asFloat >= -1) - assert.True(t, asFloat <= 1) - } - }) - - t.Run("with interpretation set", func(t *testing.T) { - query := ` - { - Get { - Company { - _additional { - interpretation{ - source { - concept - } - } - } - name - } - } - } - ` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - companies := result.Get("Get", "Company").AsSlice() - - expected := []interface{}{ - map[string]interface{}{ - "name": "Microsoft Inc.", - "_additional": map[string]interface{}{ - "interpretation": map[string]interface{}{ - "source": []interface{}{ - map[string]interface{}{ - "concept": "microsoft", - }, - map[string]interface{}{ - "concept": "inc", - }, - }, - }, - }, - }, - map[string]interface{}{ - "name": "Microsoft Incorporated", - "_additional": map[string]interface{}{ - "interpretation": map[string]interface{}{ - "source": []interface{}{ - map[string]interface{}{ - "concept": "microsoft", - }, - map[string]interface{}{ - "concept": "incorporated", - }, - }, - }, - }, - }, - map[string]interface{}{ - "name": "Microsoft", - "_additional": map[string]interface{}{ - "interpretation": map[string]interface{}{ - "source": []interface{}{ - map[string]interface{}{ - "concept": "microsoft", - }, - }, - }, - }, - }, - map[string]interface{}{ - "name": "Apple Inc.", - "_additional": map[string]interface{}{ - "interpretation": map[string]interface{}{ - "source": []interface{}{ - map[string]interface{}{ - "concept": "apple", - }, - map[string]interface{}{ - "concept": "inc", - }, - }, - }, - }, - }, - map[string]interface{}{ - "name": "Apple Incorporated", - "_additional": map[string]interface{}{ - "interpretation": map[string]interface{}{ - "source": []interface{}{ - map[string]interface{}{ - "concept": "apple", - }, - map[string]interface{}{ - "concept": "incorporated", - }, - }, - }, - }, - }, - map[string]interface{}{ - "name": "Apple", - "_additional": map[string]interface{}{ - "interpretation": map[string]interface{}{ - "source": []interface{}{ - map[string]interface{}{ - "concept": "apple", - }, - }, - }, - }, - }, - map[string]interface{}{ - "name": "Google Inc.", - "_additional": map[string]interface{}{ - "interpretation": map[string]interface{}{ - "source": []interface{}{ - map[string]interface{}{ - "concept": "google", - }, - map[string]interface{}{ - "concept": "inc", - }, - }, - }, - }, - }, - map[string]interface{}{ - "name": "Google Incorporated", - "_additional": map[string]interface{}{ - "interpretation": map[string]interface{}{ - "source": []interface{}{ - map[string]interface{}{ - "concept": "google", - }, - map[string]interface{}{ - "concept": "incorporated", - }, - }, - }, - }, - }, - map[string]interface{}{ - "name": "Google", - "_additional": map[string]interface{}{ - "interpretation": map[string]interface{}{ - "source": []interface{}{ - map[string]interface{}{ - "concept": "google", - }, - }, - }, - }, - }, - } - - assert.ElementsMatch(t, expected, companies) - }) - - t.Run("with _additional nearestNeighbors set", func(t *testing.T) { - query := ` - { - Get { - Company { - _additional { - nearestNeighbors{ - neighbors { - concept - distance - } - } - } - name - } - } - } - ` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - companies := result.Get("Get", "Company").AsSlice() - - extractNeighbors := func(in interface{}) []interface{} { - return in.(map[string]interface{})["_additional"].(map[string]interface{})["nearestNeighbors"].(map[string]interface{})["neighbors"].([]interface{}) - } - - neighbors0 := extractNeighbors(companies[0]) - neighbors1 := extractNeighbors(companies[1]) - neighbors2 := extractNeighbors(companies[2]) - - validateNeighbors(t, neighbors0, neighbors1, neighbors2) - }) - - t.Run("with _additional featureProjection set", func(t *testing.T) { - query := ` - { - Get { - Company { - _additional { - featureProjection(dimensions:3){ - vector - } - } - name - } - } - } - ` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - companies := result.Get("Get", "Company").AsSlice() - - extractProjections := func(in interface{}) []interface{} { - return in.(map[string]interface{})["_additional"].(map[string]interface{})["featureProjection"].(map[string]interface{})["vector"].([]interface{}) - } - - projections0 := extractProjections(companies[0]) - projections1 := extractProjections(companies[1]) - projections2 := extractProjections(companies[2]) - - validateProjections(t, 3, projections0, projections1, projections2) - }) - - t.Run("with _additional vector set in reference", func(t *testing.T) { - query := ` - { - Get { - City { - _additional { - vector - } - inCountry { - ... on Country { - _additional { - vector - } - } - } - } - } - } - ` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - cities := result.Get("Get", "City").AsSlice() - - vector := cities[0].(map[string]interface{})["inCountry"].([]interface{})[0].(map[string]interface{})["_additional"].(map[string]interface{})["vector"] - - assert.NotNil(t, vector) - }) - - t.Run("with _additional creationTimeUnix and lastUpdateTimeUnix set in reference", func(t *testing.T) { - query := ` - { - Get { - City { - inCountry { - ... on Country { - _additional { - creationTimeUnix - lastUpdateTimeUnix - } - } - } - } - } - } - ` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - cities := result.Get("Get", "City").AsSlice() - - created := cities[0].(map[string]interface{})["inCountry"].([]interface{})[0].(map[string]interface{})["_additional"].(map[string]interface{})["creationTimeUnix"] - updated := cities[0].(map[string]interface{})["inCountry"].([]interface{})[0].(map[string]interface{})["_additional"].(map[string]interface{})["lastUpdateTimeUnix"] - - assert.NotNil(t, created) - assert.NotNil(t, updated) - }) -} - -func validateNeighbors(t *testing.T, neighborsGroups ...[]interface{}) { - for i, group := range neighborsGroups { - if len(group) == 0 { - t.Fatalf("group %d: length of neighbors is 0", i) - } - - for j, neighbor := range group { - asMap := neighbor.(map[string]interface{}) - if len(asMap["concept"].(string)) == 0 { - t.Fatalf("group %d: element %d: concept has length 0", i, j) - } - } - } -} - -func validateProjections(t *testing.T, dims int, vectors ...[]interface{}) { - for i := range vectors { - if len(vectors[i]) != dims { - t.Fatalf("expected feature projection vector to have length 3, got: %d", len(vectors[i])) - } - } -} diff --git a/test/acceptance/graphql_resolvers/local_get_with_custom_vectors_test.go b/test/acceptance/graphql_resolvers/local_get_with_custom_vectors_test.go deleted file mode 100644 index 42b194eb82daf702bf34f1497f0d81d983586cda..0000000000000000000000000000000000000000 --- a/test/acceptance/graphql_resolvers/local_get_with_custom_vectors_test.go +++ /dev/null @@ -1,70 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "fmt" - "testing" - - graphqlhelper "github.com/weaviate/weaviate/test/helper/graphql" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/test/helper" -) - -func gettingObjectsWithCustomVectors(t *testing.T) { - t.Run("through Get {}", func(t *testing.T) { - query := ` - { - Get { - CustomVectorClass(nearVector:{vector:[1,1,1]}) { - _additional { - id - } - } - } - } - ` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - results := result.Get("Get", "CustomVectorClass").AsSlice() - - expected := []interface{}{ - map[string]interface{}{"_additional": map[string]interface{}{"id": string(cvc1)}}, - map[string]interface{}{"_additional": map[string]interface{}{"id": string(cvc2)}}, - map[string]interface{}{"_additional": map[string]interface{}{"id": string(cvc3)}}, - } - - assert.Equal(t, expected, results) - }) -} - -func exploreObjectsWithCustomVectors(t *testing.T) { - t.Run("through Explore {}", func(t *testing.T) { - query := ` - { - Explore(nearVector: {vector:[1,1,1]}) { - beacon - } - } - ` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - results := result.Get("Explore").AsSlice() - - expected := []interface{}{ - map[string]interface{}{"beacon": fmt.Sprintf("weaviate://localhost/CustomVectorClass/%s", cvc1)}, - map[string]interface{}{"beacon": fmt.Sprintf("weaviate://localhost/CustomVectorClass/%s", cvc2)}, - map[string]interface{}{"beacon": fmt.Sprintf("weaviate://localhost/CustomVectorClass/%s", cvc3)}, - } - - assert.Equal(t, expected, results) - }) -} diff --git a/test/acceptance/graphql_resolvers/local_get_with_expected_failures.go b/test/acceptance/graphql_resolvers/local_get_with_expected_failures.go deleted file mode 100644 index 669a1edb0d9c2987e499b65185bd9a7af9b52a22..0000000000000000000000000000000000000000 --- a/test/acceptance/graphql_resolvers/local_get_with_expected_failures.go +++ /dev/null @@ -1,111 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "testing" - - graphqlhelper "github.com/weaviate/weaviate/test/helper/graphql" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/test/helper" -) - -func getsWithExpectedFailures(t *testing.T) { - t.Run("get with certainty on l2-squared distancer", func(t *testing.T) { - className := "L2DistanceClass" - defer deleteObjectClass(t, className) - - t.Run("create class configured with distance type l2-squared", func(t *testing.T) { - createObjectClass(t, &models.Class{ - Class: className, - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "vectorizeClassName": true, - }, - }, - VectorIndexConfig: map[string]interface{}{ - "distance": "l2-squared", - }, - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - }) - }) - - t.Run("assert failure to get", func(t *testing.T) { - query := ` - { - Get { - L2DistanceClass(nearVector: {vector:[1,1,1], certainty: 0.8}) { - name - } - } - }` - - result := graphqlhelper.ErrorGraphQL(t, helper.RootAuth, query) - assert.Len(t, result, 1) - - errMsg := result[0].Message - assert.Equal(t, "can't compute and return certainty when vector index is configured with l2-squared distance", errMsg) - }) - }) - - t.Run("get with certainty on dot distancer", func(t *testing.T) { - className := "DotDistanceClass" - defer deleteObjectClass(t, className) - - t.Run("create class configured with distance type dot", func(t *testing.T) { - createObjectClass(t, &models.Class{ - Class: className, - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "vectorizeClassName": true, - }, - }, - VectorIndexConfig: map[string]interface{}{ - "distance": "dot", - }, - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - }) - }) - - t.Run("assert failure to get", func(t *testing.T) { - query := ` - { - Get { - DotDistanceClass(nearVector: {vector:[1,1,1], certainty: 0.8}) { - name - } - } - }` - - result := graphqlhelper.ErrorGraphQL(t, helper.RootAuth, query) - assert.Len(t, result, 1) - - errMsg := result[0].Message - assert.Equal(t, "can't compute and return certainty when vector index is configured with dot distance", errMsg) - }) - }) -} diff --git a/test/acceptance/graphql_resolvers/local_get_with_filter_test.go b/test/acceptance/graphql_resolvers/local_get_with_filter_test.go deleted file mode 100644 index 0dff2f1f4593b0ae37bebe8087a2eab03460a497..0000000000000000000000000000000000000000 --- a/test/acceptance/graphql_resolvers/local_get_with_filter_test.go +++ /dev/null @@ -1,729 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "encoding/json" - "fmt" - "strconv" - "testing" - "time" - - graphqlhelper "github.com/weaviate/weaviate/test/helper/graphql" - - "github.com/go-openapi/strfmt" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/test/helper" -) - -func gettingObjectsWithFilters(t *testing.T) { - t.Run("without filters <- this is the control", func(t *testing.T) { - query := ` - { - Get { - Airport { - code - } - } - } - ` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - airports := result.Get("Get", "Airport").AsSlice() - - expected := []interface{}{ - map[string]interface{}{"code": "10000"}, - map[string]interface{}{"code": "20000"}, - map[string]interface{}{"code": "30000"}, - map[string]interface{}{"code": "40000"}, - } - - assert.ElementsMatch(t, expected, airports) - }) - - t.Run("nearText with prop length", func(t *testing.T) { - query := ` - { - Get { - City ( - nearText: { - concepts: ["hi"], - distance: 0.9 - }, - where: { - path: "len(name)" - operator: GreaterThanEqual - valueInt: 0 - } - ) { - name - } - } - } - ` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - cities := result.Get("Get", "City").AsSlice() - assert.Len(t, cities, 5) - }) - - t.Run("nearText with null filter", func(t *testing.T) { - query := ` - { - Get { - City ( - nearText: { - concepts: ["hi"], - distance: 0.9 - }, - where: { - path: "name" - operator: IsNull - valueBoolean: true - } - ) { - name - } - } - } - ` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - cities := result.Get("Get", "City").AsSlice() - assert.Len(t, cities, 1) - }) - - t.Run("with filters applied", func(t *testing.T) { - query := ` - { - Get { - Airport(where:{ - operator:And - operands: [ - { - operator: GreaterThan, - valueInt: 600000, - path:["inCity", "City", "population"] - } - { - operator: Equal, - valueText:"Germany" - path:["inCity", "City", "inCountry", "Country", "name"] - } - ] - }){ - code - } - } - } - ` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - airports := result.Get("Get", "Airport").AsSlice() - - expected := []interface{}{ - map[string]interface{}{"code": "40000"}, - } - - assert.ElementsMatch(t, expected, airports) - }) - - t.Run("with or filters applied", func(t *testing.T) { - // this test was added to prevent a regression on the bugfix for gh-758 - - query := ` - { - Aggregate { - City(where:{ - operator:Or - operands:[{ - valueText:"Amsterdam", - operator:Equal, - path:["name"] - }, { - valueText:"Berlin", - operator:Equal, - path:["name"] - }] - }) { - __typename - name { - __typename - count - } - } - } - } - ` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - cityMeta := result.Get("Aggregate", "City").AsSlice()[0] - - expected := map[string]interface{}{ - "__typename": "AggregateCity", - "name": map[string]interface{}{ - "__typename": "AggregateCitynameObj", - "count": json.Number("2"), - }, - } - - assert.Equal(t, expected, cityMeta) - }) - - t.Run("with filters and ref showing a phone number", func(t *testing.T) { - // this is the journey test for gh-1088 - - query := ` - { - Get { - Airport(where:{ - valueText:"Amsterdam", - operator:Equal, - path:["inCity", "City", "name"] - }) { - phone { - internationalFormatted - countryCode - nationalFormatted - } - } - } - } - ` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - airport := result.Get("Get", "Airport").AsSlice()[0] - - expected := map[string]interface{}{ - "phone": map[string]interface{}{ - "internationalFormatted": "+31 1234567", - "countryCode": json.Number("31"), - "nationalFormatted": "1234567", - }, - } - - assert.Equal(t, expected, airport) - }) - - t.Run("with uuid filters applied", func(t *testing.T) { - query := ` - { - Get { - Airport(where:{ - operator:And - operands: [ - { - operator: GreaterThan, - valueText: "00000000-0000-0000-0000-000000010000", - path:["airportId"] - }, - { - operator: LessThan, - valueText: "00000000-0000-0000-0000-000000030000", - path:["airportId"] - }, - { - operator: NotEqual, - valueText: "00000000-0000-0000-0000-000000040000", - path:["airportId"] - } - ] - }){ - code - airportId - } - } - } - ` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - airports := result.Get("Get", "Airport").AsSlice() - - expected := []interface{}{ - map[string]interface{}{ - "code": "20000", - "airportId": "00000000-0000-0000-0000-000000020000", - }, - } - - assert.ElementsMatch(t, expected, airports) - }) - - t.Run("filtering for ref counts", func(t *testing.T) { - // this is the journey test for gh-1101 - - query := func(op string, count int) string { - return fmt.Sprintf(` - { - Get { - Person(where:{ - valueInt: %d - operator:%s, - path:["livesIn"] - }) { - name - } - } - } - `, count, op) - } - - t.Run("no refs", func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query("Equal", 0)) - // Alice should be the only person that has zero places she lives in - require.Len(t, result.Get("Get", "Person").AsSlice(), 1) - name := result.Get("Get", "Person").AsSlice()[0].(map[string]interface{})["name"] - assert.Equal(t, "Alice", name) - }) - - t.Run("exactly one", func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query("Equal", 1)) - // bob should be the only person that has zero places she lives in - require.Len(t, result.Get("Get", "Person").AsSlice(), 1) - name := result.Get("Get", "Person").AsSlice()[0].(map[string]interface{})["name"] - assert.Equal(t, "Bob", name) - }) - - t.Run("2 or more", func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query("GreaterThanEqual", 2)) - // both john(2) and petra(3) should match - require.Len(t, result.Get("Get", "Person").AsSlice(), 2) - name1 := result.Get("Get", "Person").AsSlice()[0].(map[string]interface{})["name"] - name2 := result.Get("Get", "Person").AsSlice()[1].(map[string]interface{})["name"] - assert.ElementsMatch(t, []string{"John", "Petra"}, []string{name1.(string), name2.(string)}) - }) - }) - - t.Run("filtering by property len", func(t *testing.T) { - query := `{ - Get { - ArrayClass(where:{ - valueInt: 4, - operator:Equal, - path:["len(texts)"] - }) { - texts - } - } - } - ` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - require.Len(t, result.Get("Get", "ArrayClass").AsSlice(), 1) - }) - - t.Run("filtering by null property", func(t *testing.T) { - query := `{ - Get { - ArrayClass(where:{ - valueBoolean: true, - operator:IsNull, - path:["texts"] - }) { - texts - } - } - } - ` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - require.Len(t, result.Get("Get", "ArrayClass").AsSlice(), 3) // empty, nil and len==0 objects - }) - - t.Run("filtering by property with field tokenization", func(t *testing.T) { - // tests gh-1821 feature - - query := func(value string) string { - return fmt.Sprintf(` - { - Get { - Person(where:{ - valueText: "%s" - operator:Equal, - path:["profession"] - }) { - name - } - } - } - `, value) - } - - t.Run("noone", func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query("Quality")) - // Quality is not full field for anyone, therefore noone should be returned - require.Len(t, result.Get("Get", "Person").AsSlice(), 0) - }) - - t.Run("just one is Mechanical Engineer", func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query("Mechanical Engineer")) - // Bob is Mechanical Engineer, though John is Senior - require.Len(t, result.Get("Get", "Person").AsSlice(), 1) - name := result.Get("Get", "Person").AsSlice()[0].(map[string]interface{})["name"] - assert.Equal(t, "Bob", name) - }) - - t.Run("just one is Senior Mechanical Engineer", func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query("Senior Mechanical Engineer")) - // so to get John, his full profession name has to be used - require.Len(t, result.Get("Get", "Person").AsSlice(), 1) - name := result.Get("Get", "Person").AsSlice()[0].(map[string]interface{})["name"] - assert.Equal(t, "John", name) - }) - - t.Run("just one is Quality Assurance Manager", func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query("Quality Assurance Manager")) - // petra is Quality Assurance Manager - require.Len(t, result.Get("Get", "Person").AsSlice(), 1) - name := result.Get("Get", "Person").AsSlice()[0].(map[string]interface{})["name"] - assert.Equal(t, "Petra", name) - }) - }) - - t.Run("filtering by array property with field tokenization", func(t *testing.T) { - // tests gh-1821 feature - - query := func(value string) string { - return fmt.Sprintf(` - { - Get { - Person(where:{ - valueText: "%s" - operator:Equal, - path:["about"] - }) { - name - } - } - } - `, value) - } - - t.Run("noone", func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query("swimming")) - // swimming is not full field for anyone, therefore noone should be returned - require.Len(t, result.Get("Get", "Person").AsSlice(), 0) - }) - - t.Run("just one hates swimming", func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query("hates swimming")) - // but only john hates swimming - require.Len(t, result.Get("Get", "Person").AsSlice(), 1) - name := result.Get("Get", "Person").AsSlice()[0].(map[string]interface{})["name"].(string) - assert.Equal(t, "John", name) - }) - - t.Run("exactly 2 loves travelling", func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query("loves travelling")) - // bob and john loves travelling, alice loves traveling very much - require.Len(t, result.Get("Get", "Person").AsSlice(), 2) - name1 := result.Get("Get", "Person").AsSlice()[0].(map[string]interface{})["name"].(string) - name2 := result.Get("Get", "Person").AsSlice()[1].(map[string]interface{})["name"].(string) - assert.ElementsMatch(t, []string{"Bob", "John"}, []string{name1, name2}) - }) - - t.Run("only one likes cooking for family", func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query("likes cooking for family")) - // petra likes cooking for family, john simply likes cooking - require.Len(t, result.Get("Get", "Person").AsSlice(), 1) - name := result.Get("Get", "Person").AsSlice()[0].(map[string]interface{})["name"] - assert.Equal(t, "Petra", name) - }) - }) - - t.Run("filtering by stopwords", func(t *testing.T) { - query := func(value string) string { - return fmt.Sprintf(` - { - Get { - Pizza(where:{ - valueText: "%s" - operator:Equal, - path:["description"] - }) { - name - _additional{ - id - } - } - } - } - `, value) - } - - t.Run("2 results by partial description", func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query("italian")) - pizzas := result.Get("Get", "Pizza").AsSlice() - require.Len(t, pizzas, 2) - id1 := pizzas[0].(map[string]interface{})["_additional"].(map[string]interface{})["id"] - id2 := pizzas[1].(map[string]interface{})["_additional"].(map[string]interface{})["id"] - assert.Equal(t, quattroFormaggi.String(), id1) - assert.Equal(t, fruttiDiMare.String(), id2) - }) - - t.Run("1 result by full description containing stopwords", func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query("Universally accepted to be the best pizza ever created.")) - pizzas := result.Get("Get", "Pizza").AsSlice() - require.Len(t, pizzas, 1) - id1 := pizzas[0].(map[string]interface{})["_additional"].(map[string]interface{})["id"] - assert.Equal(t, hawaii.String(), id1) - }) - - t.Run("error by description containing just stopwords", func(t *testing.T) { - errors := graphqlhelper.ErrorGraphQL(t, helper.RootAuth, query("to be or not to be")) - require.Len(t, errors, 1) - assert.Contains(t, errors[0].Message, "invalid search term, only stopwords provided. Stopwords can be configured in class.invertedIndexConfig.stopwords") - }) - }) - - t.Run("with filtering by id", func(t *testing.T) { - // this is the journey test for gh-1088 - - query := ` - { - Get { - Airport(where:{ - valueText:"4770bb19-20fd-406e-ac64-9dac54c27a0f", - operator:Equal, - path:["id"] - }) { - phone { - internationalFormatted - countryCode - nationalFormatted - } - } - } - } - ` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - airport := result.Get("Get", "Airport").AsSlice()[0] - - expected := map[string]interface{}{ - "phone": map[string]interface{}{ - "internationalFormatted": "+31 1234567", - "countryCode": json.Number("31"), - "nationalFormatted": "1234567", - }, - } - - assert.Equal(t, expected, airport) - }) - - t.Run("with filtering by timestamps", func(t *testing.T) { - query := ` - { - Get { - Airport { - _additional { - id - creationTimeUnix - lastUpdateTimeUnix - } - } - } - } - ` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - airport := result.Get("Get", "Airport").AsSlice()[0] - additional := airport.(map[string]interface{})["_additional"] - targetID := additional.(map[string]interface{})["id"].(string) - targetCreationTime := additional.(map[string]interface{})["creationTimeUnix"].(string) - targetUpdateTime := additional.(map[string]interface{})["lastUpdateTimeUnix"].(string) - - creationTimestamp, err := strconv.ParseInt(targetCreationTime, 10, 64) - assert.Nil(t, err) - creationDate := time.UnixMilli(creationTimestamp).Format(time.RFC3339) - updateTimestamp, err := strconv.ParseInt(targetUpdateTime, 10, 64) - assert.Nil(t, err) - updateDate := time.UnixMilli(updateTimestamp).Format(time.RFC3339) - - t.Run("creationTimeUnix as timestamp", func(t *testing.T) { - query := fmt.Sprintf(` - { - Get { - Airport( - where: { - path: ["_creationTimeUnix"] - operator: Equal - valueText: "%s" - } - ) - { - _additional { - id - } - } - } - } - `, targetCreationTime) - - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - airport := result.Get("Get", "Airport").AsSlice()[0] - additional := airport.(map[string]interface{})["_additional"] - resultID := additional.(map[string]interface{})["id"].(string) - assert.Equal(t, targetID, resultID) - }) - - t.Run("creationTimeUnix as date", func(t *testing.T) { - query := fmt.Sprintf(` - { - Get { - Airport( - where: { - path: ["_creationTimeUnix"] - operator: GreaterThanEqual - valueDate: "%s" - } - ) - { - _additional { - id - } - } - } - } - `, creationDate) - - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - airport := result.Get("Get", "Airport").AsSlice()[0] - additional := airport.(map[string]interface{})["_additional"] - resultID := additional.(map[string]interface{})["id"].(string) - assert.Equal(t, targetID, resultID) - }) - - t.Run("lastUpdateTimeUnix as timestamp", func(t *testing.T) { - query := fmt.Sprintf(` - { - Get { - Airport( - where: { - path: ["_lastUpdateTimeUnix"] - operator: Equal - valueText: "%s" - } - ) - { - _additional { - id - } - } - } - } - `, targetUpdateTime) - - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - airport := result.Get("Get", "Airport").AsSlice()[0] - additional := airport.(map[string]interface{})["_additional"] - resultID := additional.(map[string]interface{})["id"].(string) - assert.Equal(t, targetID, resultID) - }) - - t.Run("lastUpdateTimeUnix as date", func(t *testing.T) { - query := fmt.Sprintf(` - { - Get { - Airport( - where: { - path: ["_lastUpdateTimeUnix"] - operator: GreaterThanEqual - valueDate: "%s" - } - ) - { - _additional { - id - } - } - } - } - `, updateDate) - - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - airport := result.Get("Get", "Airport").AsSlice()[0] - additional := airport.(map[string]interface{})["_additional"] - resultID := additional.(map[string]interface{})["id"].(string) - assert.Equal(t, targetID, resultID) - }) - }) - - t.Run("with id filter on object with no props", func(t *testing.T) { - id := strfmt.UUID("f0ea8fb8-5a1f-449d-aed5-d68dc65cd644") - defer deleteObjectClass(t, "NoProps") - - t.Run("setup test class and obj", func(t *testing.T) { - createObjectClass(t, &models.Class{ - Class: "NoProps", Properties: []*models.Property{ - {Name: "unused", DataType: schema.DataTypeText.PropString(), Tokenization: models.PropertyTokenizationWhitespace}, - }, - }) - - createObject(t, &models.Object{Class: "NoProps", ID: id}) - }) - - t.Run("do query", func(t *testing.T) { - query := fmt.Sprintf(` - { - Get { - NoProps(where:{operator:Equal path:["_id"] valueText:"%s"}) - { - _additional {id} - } - } - } - `, id) - response := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - result := response.Get("Get", "NoProps").AsSlice() - require.Len(t, result, 1) - additional := result[0].(map[string]interface{})["_additional"] - resultID := additional.(map[string]interface{})["id"].(string) - assert.Equal(t, id.String(), resultID) - }) - }) - - t.Run("with nul filter", func(t *testing.T) { - tests := []struct { - name string - value bool - Results []interface{} - }{ - { - name: "Null values", - value: true, - Results: []interface{}{"Missing Island", nil}, // one entry with null history has no name - }, - { - name: "Non-null values", - value: false, - Results: []interface{}{"Amsterdam", "Rotterdam", "Berlin", "Dusseldorf"}, - }, - } - query := ` - { - Get { - City(where:{ - valueBoolean: %v, - operator:IsNull, - path:["history"] - }) { - name - } - } - } - ` - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, fmt.Sprintf(query, tt.value)) - cities := result.Get("Get", "City").AsSlice() - require.Len(t, cities, len(tt.Results)) - for _, city := range cities { - cityMap := city.(map[string]interface{}) - require.Contains(t, tt.Results, cityMap["name"]) - } - }) - } - }) -} diff --git a/test/acceptance/graphql_resolvers/local_get_with_geo_range_filter_test.go b/test/acceptance/graphql_resolvers/local_get_with_geo_range_filter_test.go deleted file mode 100644 index e730188739439f267b7166447d570c4574c35abb..0000000000000000000000000000000000000000 --- a/test/acceptance/graphql_resolvers/local_get_with_geo_range_filter_test.go +++ /dev/null @@ -1,134 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "encoding/json" - "testing" - - graphqlhelper "github.com/weaviate/weaviate/test/helper/graphql" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/test/helper" -) - -func gettingObjectsWithGeoFilters(t *testing.T) { - t.Run("Only Dusseldorf should be within 100km of Dusseldorf", func(t *testing.T) { - query := ` - { - Get { - City(where:{ - operator: WithinGeoRange - path: ["location"] - valueGeoRange: { geoCoordinates: {latitude: 51.225556, longitude: 6.782778} distance: { max: 100000 } } - }){ - name - location { - latitude - longitude - } - } - } - } - ` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - cities := result.Get("Get", "City").AsSlice() - - expectedResults := []interface{}{ - map[string]interface{}{ - "name": "Dusseldorf", - "location": map[string]interface{}{ - "latitude": json.Number("51.225555"), - "longitude": json.Number("6.782778"), - }, - }, - } - - assert.Equal(t, expectedResults, cities) - }) - - t.Run("Dusseldorf and Amsterdam should be within 200km of Dusseldorf", func(t *testing.T) { - query := ` - { - Get { - City(where:{ - operator: WithinGeoRange - path: ["location"] - valueGeoRange: { geoCoordinates: {latitude: 51.225556, longitude: 6.782778} distance: { max: 200000 } } - }){ - name - location { - latitude - longitude - } - } - } - } - ` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - cities := result.Get("Get", "City").AsSlice() - - expectedResults := []interface{}{ - map[string]interface{}{ - "name": "Dusseldorf", - "location": map[string]interface{}{ - "latitude": json.Number("51.225555"), - "longitude": json.Number("6.782778"), - }, - }, - map[string]interface{}{ - "name": "Amsterdam", - "location": map[string]interface{}{ - "latitude": json.Number("52.36667"), - "longitude": json.Number("4.9"), - }, - }, - } - - assert.ElementsMatch(t, expectedResults, cities) - }) - - // This test prevents a regression on gh-825 - t.Run("Missing island is displayed correctly", func(t *testing.T) { - query := ` - { - Get { - City(where:{ - operator: WithinGeoRange - path: ["location"] - valueGeoRange: { geoCoordinates: {latitude: 0, longitude: 0} distance: { max: 20 } } - }){ - name - location { - latitude - longitude - } - } - } - } - ` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - cities := result.Get("Get", "City").AsSlice() - - expectedResults := []interface{}{ - map[string]interface{}{ - "name": "Missing Island", - "location": map[string]interface{}{ - "latitude": json.Number("0"), - "longitude": json.Number("0"), - }, - }, - } - - assert.ElementsMatch(t, expectedResults, cities) - }) -} diff --git a/test/acceptance/graphql_resolvers/local_get_with_group_by_test.go b/test/acceptance/graphql_resolvers/local_get_with_group_by_test.go deleted file mode 100644 index 592c4f03adb5a2e1035aedcde0b0eeb3ad445db3..0000000000000000000000000000000000000000 --- a/test/acceptance/graphql_resolvers/local_get_with_group_by_test.go +++ /dev/null @@ -1,120 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "testing" - - graphqlhelper "github.com/weaviate/weaviate/test/helper/graphql" - "github.com/weaviate/weaviate/test/helper/journey" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/test/helper" -) - -func groupByObjects(t *testing.T) { - t.Run("group by: people by city", func(t *testing.T) { - getGroup := func(value interface{}) map[string]interface{} { - group := value.(map[string]interface{})["_additional"].(map[string]interface{})["group"].(map[string]interface{}) - return group - } - getGroupHits := func(group map[string]interface{}) (string, []string) { - result := []string{} - hits := group["hits"].([]interface{}) - for _, hit := range hits { - additional := hit.(map[string]interface{})["_additional"].(map[string]interface{}) - result = append(result, additional["id"].(string)) - } - groupedBy := group["groupedBy"].(map[string]interface{}) - groupedByValue := groupedBy["value"].(string) - return groupedByValue, result - } - query := ` - { - Get{ - Person( - nearObject:{ - id: "8615585a-2960-482d-b19d-8bee98ade52c" - } - groupBy:{ - path:["livesIn"] - groups:4 - objectsPerGroup: 10 - } - ){ - _additional{ - id - group{ - groupedBy{value} - count - maxDistance - minDistance - hits { - _additional { - id - distance - } - } - } - } - } - } - } - ` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - groups := result.Get("Get", "Person").AsSlice() - - require.Len(t, groups, 4) - - expectedResults := map[string][]string{} - - groupedBy1 := `weaviate://localhost/City/8f5f8e44-d348-459c-88b1-c1a44bb8f8be` - expectedGroup1 := []string{ - "8615585a-2960-482d-b19d-8bee98ade52c", - "3ef44474-b5e5-455d-91dc-d917b5b76165", - "15d222c9-8c36-464b-bedb-113faa1c1e4c", - } - expectedResults[groupedBy1] = expectedGroup1 - - groupedBy2 := `weaviate://localhost/City/9b9cbea5-e87e-4cd0-89af-e2f424fd52d6` - expectedGroup2 := []string{ - "3ef44474-b5e5-455d-91dc-d917b5b76165", - "15d222c9-8c36-464b-bedb-113faa1c1e4c", - } - expectedResults[groupedBy2] = expectedGroup2 - - groupedBy3 := `weaviate://localhost/City/6ffb03f8-a853-4ec5-a5d8-302e45aaaf13` - expectedGroup3 := []string{ - "15d222c9-8c36-464b-bedb-113faa1c1e4c", - } - expectedResults[groupedBy3] = expectedGroup3 - - groupedBy4 := "" - expectedGroup4 := []string{ - "5d0fa6ee-21c4-4b46-a735-f0208717837d", - } - expectedResults[groupedBy4] = expectedGroup4 - - groupsOrder := []string{groupedBy1, groupedBy2, groupedBy4, groupedBy3} - for i, current := range groups { - group := getGroup(current) - groupedBy, ids := getGroupHits(group) - assert.Equal(t, groupsOrder[i], groupedBy) - assert.ElementsMatch(t, expectedResults[groupedBy], ids) - } - }) - - t.Run("group by: passages by documents", func(t *testing.T) { - journey.GroupBySingleAndMultiShardTests(t, "") - }) -} diff --git a/test/acceptance/graphql_resolvers/local_get_with_grouping_test.go b/test/acceptance/graphql_resolvers/local_get_with_grouping_test.go deleted file mode 100644 index 38b0f25a3158fc4d91cca0df7a61706c0eea9878..0000000000000000000000000000000000000000 --- a/test/acceptance/graphql_resolvers/local_get_with_grouping_test.go +++ /dev/null @@ -1,353 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "strings" - "testing" - - graphqlhelper "github.com/weaviate/weaviate/test/helper/graphql" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/test/helper" -) - -func gettingObjectsWithGrouping(t *testing.T) { - t.Run("without grouping <- this is the control", func(t *testing.T) { - query := ` - { - Get { - Company { - name - } - } - } - ` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - companies := result.Get("Get", "Company").AsSlice() - - expected := []interface{}{ - map[string]interface{}{"name": "Microsoft Inc."}, - map[string]interface{}{"name": "Microsoft Incorporated"}, - map[string]interface{}{"name": "Microsoft"}, - map[string]interface{}{"name": "Apple Inc."}, - map[string]interface{}{"name": "Apple Incorporated"}, - map[string]interface{}{"name": "Apple"}, - map[string]interface{}{"name": "Google Inc."}, - map[string]interface{}{"name": "Google Incorporated"}, - map[string]interface{}{"name": "Google"}, - } - - assert.ElementsMatch(t, expected, companies) - }) - - t.Run("grouping mode set to merge and force to 1.0", func(t *testing.T) { - query := ` - { - Get { - Company(group: {type: merge, force:1.0}) { - name - inCity { - ... on City {name} - } - } - } - } - ` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - companies := result.Get("Get", "Company").AsSlice() - - require.Len(t, companies, 1) - - companyNames := companies[0].(map[string]interface{})["name"].(string) - assert.NotEmpty(t, companies) - - mustContain := []string{"Apple", "Google", "Microsoft"} - for _, companyName := range mustContain { - if !strings.Contains(companyNames, companyName) { - t.Errorf("%s not contained in %v", companyName, companyNames) - } - } - - companyCities := companies[0].(map[string]interface{})["inCity"].([]interface{}) - expectedCities := []map[string]interface{}{ - {"name": "Dusseldorf"}, - {"name": "Amsterdam"}, - {"name": "Berlin"}, - } - - assert.ElementsMatch(t, expectedCities, companyCities) - }) - - t.Run("grouping mode set to merge and force to 0.0", func(t *testing.T) { - query := ` - { - Get { - Company(group: {type: merge, force:0.0}) { - name - inCity { - ... on City { - name - } - } - } - } - } - ` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - companies := result.Get("Get", "Company").AsSlice() - - require.Len(t, companies, 9) - - getName := func(value map[string]interface{}) string { - return value["name"].(string) - } - - getCities := func(value map[string]interface{}) []string { - inCity := value["inCity"].([]interface{}) - cities := make([]string, len(inCity)) - for i := range inCity { - cityVal := inCity[i].(map[string]interface{}) - cities[i] = getName(cityVal) - } - return cities - } - - for _, current := range companies { - currentMap := current.(map[string]interface{}) - if getName(currentMap) == "Microsoft Incorporated" { - assert.Len(t, getCities(currentMap), 2) - } - if getName(currentMap) == "Microsoft Inc." { - assert.Len(t, getCities(currentMap), 1) - } - if getName(currentMap) == "Microsoft" { - assert.Len(t, getCities(currentMap), 1) - } - } - }) - - t.Run("grouping mode set to closest and force to 0.1", func(t *testing.T) { - query := ` - { - Get { - Company(group: {type: closest, force:0.1}) { - name - inCity { - ... on City { - name - } - } - } - } - } - ` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - companies := result.Get("Get", "Company").AsSlice() - - assert.True(t, len(companies) > 0) - }) - - t.Run("grouping mode set to closest with near text", func(t *testing.T) { - query := ` - { - Get { - Company(nearText: {concepts: "Apple"}, group: {type: closest, force:1.0}) { - name - } - } - } - ` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - companies := result.Get("Get", "Company").AsSlice() - - assert.True(t, len(companies) == 1) - }) - - t.Run("grouping with where filter", func(t *testing.T) { - query := ` - { - Get { - Company(group:{type:merge force:1.0} where:{path:["id"] operator:Like valueText:"*"}) { - name - inCity { - ... on City { - name - } - } - } - } - } - ` - - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - grouped := result.Get("Get", "Company").AsSlice() - require.Len(t, grouped, 1) - groupedName := grouped[0].(map[string]interface{})["name"].(string) - assert.Equal(t, "Microsoft Inc. (Microsoft Incorporated, Microsoft, Apple Inc., "+ - "Apple Incorporated, Apple, Google Inc., Google Incorporated, Google)", - groupedName) - - companyCities := grouped[0].(map[string]interface{})["inCity"].([]interface{}) - expectedCities := []map[string]interface{}{ - {"name": "Dusseldorf"}, - {"name": "Amsterdam"}, - {"name": "Berlin"}, - } - - assert.ElementsMatch(t, expectedCities, companyCities) - - // this query should yield the same results as the above, as the above where filter will - // match all records. checking the previous payload with the one below is a sanity check - // for the sake of validating the fix for [github issue 1958] - // (https://github.com/weaviate/weaviate/issues/1958) - // UPDATE: due to introducing roaring bitmaps as set holding docIDs of filtered documents - // internal order of results returned has changed from property value based to docID based, - // but set content remain unchanged - // for that reason grouped name in the following test is different with and without filters, - // though it still contains the same elements - queryWithoutWhere := ` - { - Get { - Company(group:{type:merge force:1.0}) { - name - inCity { - ... on City { - name - } - } - } - } - } - ` - result = graphqlhelper.AssertGraphQL(t, helper.RootAuth, queryWithoutWhere) - groupedWithoutWhere := result.Get("Get", "Company").AsSlice() - groupedWithoutWhereName := groupedWithoutWhere[0].(map[string]interface{})["name"].(string) - assert.Equal(t, "Apple Inc. (Google Incorporated, Google Inc., Microsoft Incorporated, "+ - "Apple, Apple Incorporated, Google, Microsoft Inc., Microsoft)", - groupedWithoutWhereName) - - companyCities = groupedWithoutWhere[0].(map[string]interface{})["inCity"].([]interface{}) - assert.ElementsMatch(t, expectedCities, companyCities) - }) - - t.Run("grouping with sort", func(t *testing.T) { - query := ` - { - Get { - Company(group:{type:merge force:1.0} sort:{path:["name"]}) { - name - inCity { - ... on City { - name - } - } - } - } - } - ` - - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - grouped := result.Get("Get", "Company").AsSlice() - require.Len(t, grouped, 1) - groupedName := grouped[0].(map[string]interface{})["name"].(string) - assert.Equal(t, "Apple (Apple Inc., Apple Incorporated, Google, Google Inc., "+ - "Google Incorporated, Microsoft, Microsoft Inc., Microsoft Incorporated)", - groupedName) - - groupedCities := grouped[0].(map[string]interface{})["inCity"].([]interface{}) - expectedCities := []map[string]interface{}{ - {"name": "Dusseldorf"}, - {"name": "Amsterdam"}, - {"name": "Berlin"}, - } - - assert.ElementsMatch(t, expectedCities, groupedCities) - }) - - // temporarily removed due to - // https://github.com/weaviate/weaviate/issues/1302 - // t.Run("grouping mode set to closest", func(t *testing.T) { - // query := ` - // { - // Get { - // Company(group: {type: closest, force:0.10}) { - // name - // } - // } - // } - // ` - // result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - // companies := result.Get("Get", "Company").AsSlice() - - // assert.Len(t, companies, 3) - // mustContain := []string{"Apple", "Microsoft", "Google"} - // outer: - // for _, toContain := range mustContain { - // for _, current := range companies { - // if strings.Contains(current.(map[string]interface{})["name"].(string), toContain) { - // continue outer - // } - // } - - // t.Errorf("%s not contained in %v", toContain, companies) - // } - // }) - - // ignore as 0.16.0 contextionaries aren't compatible with this test - // t.Run("grouping mode set to merge", func(t *testing.T) { - // query := ` - // { - // Get { - // Company(group: {type: merge, force:0.1}) { - // name - // inCity { - // ... on City { - // name - // } - // } - // } - // } - // } - // ` - // result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - // companies := result.Get("Get", "Company").AsSlice() - - // assert.Len(t, companies, 3) - // mustContain := [][]string{ - // []string{"Apple", "Apple Inc.", "Apple Incorporated"}, - // []string{"Microsoft", "Microsoft Inc.", "Microsoft Incorporated"}, - // []string{"Google", "Google Inc.", "Google Incorporated"}, - // } - - // allContained := func(current map[string]interface{}, toContains []string) bool { - // for _, toContain := range toContains { - // if !strings.Contains(current["name"].(string), toContain) { - // return false - // } - // } - // return true - // } - - // outer: - // for _, toContain := range mustContain { - // for _, current := range companies { - // if allContained(current.(map[string]interface{}), toContain) { - // continue outer - // } - // } - - // t.Errorf("%s not contained in %v", toContain, companies) - // } - // }) -} diff --git a/test/acceptance/graphql_resolvers/local_get_with_sort_test.go b/test/acceptance/graphql_resolvers/local_get_with_sort_test.go deleted file mode 100644 index 11d29e480783f53eb6742c66c033fc80e10790a9..0000000000000000000000000000000000000000 --- a/test/acceptance/graphql_resolvers/local_get_with_sort_test.go +++ /dev/null @@ -1,1220 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "fmt" - "reflect" - "strings" - "testing" - - graphqlhelper "github.com/weaviate/weaviate/test/helper/graphql" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/test/helper" -) - -func gettingObjectsWithSort(t *testing.T) { - buildSort := func(path []string, order string) string { - pathArgs := make([]string, len(path)) - for i := range path { - pathArgs[i] = fmt.Sprintf("\"%s\"", path[i]) - } - return fmt.Sprintf("{path:[%s] order:%s}", strings.Join(pathArgs, ","), order) - } - buildSortFilter := func(sort []string) string { - return fmt.Sprintf("sort:[%s]", strings.Join(sort, ",")) - } - - t.Run("simple sort", func(t *testing.T) { - query := ` - { - Get { - City( - sort: [{ - path: ["%s"] - order: %s - }] - ) { - name - } - } - } - ` - tests := []struct { - name string - property, order string - expected []interface{} - }{ - { - name: "sort by name asc", - property: "name", - order: "asc", - expected: []interface{}{ - map[string]interface{}{"name": nil}, - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": "Berlin"}, - map[string]interface{}{"name": "Dusseldorf"}, - map[string]interface{}{"name": "Missing Island"}, - map[string]interface{}{"name": "Rotterdam"}, - }, - }, - { - name: "sort by name desc", - property: "name", - order: "desc", - expected: []interface{}{ - map[string]interface{}{"name": "Rotterdam"}, - map[string]interface{}{"name": "Missing Island"}, - map[string]interface{}{"name": "Dusseldorf"}, - map[string]interface{}{"name": "Berlin"}, - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": nil}, - }, - }, - { - name: "sort by population asc", - property: "population", - order: "asc", - expected: []interface{}{ - map[string]interface{}{"name": nil}, - map[string]interface{}{"name": "Missing Island"}, - map[string]interface{}{"name": "Rotterdam"}, - map[string]interface{}{"name": "Dusseldorf"}, - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": "Berlin"}, - }, - }, - { - name: "sort by population desc", - property: "population", - order: "desc", - expected: []interface{}{ - map[string]interface{}{"name": "Berlin"}, - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": "Rotterdam"}, - map[string]interface{}{"name": "Dusseldorf"}, - map[string]interface{}{"name": "Missing Island"}, - map[string]interface{}{"name": nil}, - }, - }, - { - name: "sort by isCapital asc", - property: "isCapital", - order: "asc", - expected: []interface{}{ - map[string]interface{}{"name": nil}, - map[string]interface{}{"name": "Rotterdam"}, - map[string]interface{}{"name": "Dusseldorf"}, - map[string]interface{}{"name": "Missing Island"}, - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": "Berlin"}, - }, - }, - { - name: "sort by isCapital desc", - property: "isCapital", - order: "desc", - expected: []interface{}{ - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": "Berlin"}, - map[string]interface{}{"name": "Rotterdam"}, - map[string]interface{}{"name": "Dusseldorf"}, - map[string]interface{}{"name": "Missing Island"}, - map[string]interface{}{"name": nil}, - }, - }, - { - name: "sort by cityArea asc", - property: "cityArea", - order: "asc", - expected: []interface{}{ - map[string]interface{}{"name": "Missing Island"}, - map[string]interface{}{"name": nil}, - map[string]interface{}{"name": "Dusseldorf"}, - map[string]interface{}{"name": "Rotterdam"}, - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": "Berlin"}, - }, - }, - { - name: "sort by cityArea desc", - property: "cityArea", - order: "desc", - expected: []interface{}{ - map[string]interface{}{"name": "Berlin"}, - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": "Rotterdam"}, - map[string]interface{}{"name": "Dusseldorf"}, - map[string]interface{}{"name": "Missing Island"}, - map[string]interface{}{"name": nil}, - }, - }, - { - name: "sort by cityRights asc", - property: "cityRights", - order: "asc", - expected: []interface{}{ - map[string]interface{}{"name": "Missing Island"}, - map[string]interface{}{"name": nil}, - map[string]interface{}{"name": "Dusseldorf"}, - map[string]interface{}{"name": "Rotterdam"}, - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": "Berlin"}, - }, - }, - { - name: "sort by cityRights desc", - property: "cityRights", - order: "desc", - expected: []interface{}{ - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": "Berlin"}, - map[string]interface{}{"name": "Rotterdam"}, - map[string]interface{}{"name": "Dusseldorf"}, - map[string]interface{}{"name": "Missing Island"}, - map[string]interface{}{"name": nil}, - }, - }, - { - name: "sort by timezones asc", - property: "timezones", - order: "asc", - expected: []interface{}{ - map[string]interface{}{"name": "Missing Island"}, - map[string]interface{}{"name": nil}, - map[string]interface{}{"name": "Rotterdam"}, - map[string]interface{}{"name": "Dusseldorf"}, - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": "Berlin"}, - }, - }, - { - name: "sort by timezones desc", - property: "timezones", - order: "desc", - expected: []interface{}{ - map[string]interface{}{"name": "Rotterdam"}, - map[string]interface{}{"name": "Dusseldorf"}, - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": "Berlin"}, - map[string]interface{}{"name": "Missing Island"}, - map[string]interface{}{"name": nil}, - }, - }, - { - name: "sort by museums asc", - property: "museums", - order: "asc", - expected: []interface{}{ - map[string]interface{}{"name": "Missing Island"}, - map[string]interface{}{"name": nil}, - map[string]interface{}{"name": "Berlin"}, - map[string]interface{}{"name": "Rotterdam"}, - map[string]interface{}{"name": "Dusseldorf"}, - map[string]interface{}{"name": "Amsterdam"}, - }, - }, - { - name: "sort by museums desc", - property: "museums", - order: "desc", - expected: []interface{}{ - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": "Dusseldorf"}, - map[string]interface{}{"name": "Rotterdam"}, - map[string]interface{}{"name": "Berlin"}, - map[string]interface{}{"name": "Missing Island"}, - map[string]interface{}{"name": nil}, - }, - }, - { - name: "sort by history asc", - property: "history", - order: "asc", - expected: []interface{}{ - map[string]interface{}{"name": "Missing Island"}, - map[string]interface{}{"name": nil}, - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": "Rotterdam"}, - map[string]interface{}{"name": "Berlin"}, - map[string]interface{}{"name": "Dusseldorf"}, - }, - }, - { - name: "sort by history desc", - property: "history", - order: "desc", - expected: []interface{}{ - map[string]interface{}{"name": "Dusseldorf"}, - map[string]interface{}{"name": "Berlin"}, - map[string]interface{}{"name": "Rotterdam"}, - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": "Missing Island"}, - map[string]interface{}{"name": nil}, - }, - }, - { - name: "sort by phoneNumber asc", - property: "phoneNumber", - order: "asc", - expected: []interface{}{ - map[string]interface{}{"name": "Missing Island"}, - map[string]interface{}{"name": nil}, - map[string]interface{}{"name": "Rotterdam"}, - map[string]interface{}{"name": "Dusseldorf"}, - map[string]interface{}{"name": "Berlin"}, - map[string]interface{}{"name": "Amsterdam"}, - }, - }, - { - name: "sort by phoneNumber desc", - property: "phoneNumber", - order: "desc", - expected: []interface{}{ - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": "Berlin"}, - map[string]interface{}{"name": "Dusseldorf"}, - map[string]interface{}{"name": "Rotterdam"}, - map[string]interface{}{"name": "Missing Island"}, - map[string]interface{}{"name": nil}, - }, - }, - { - name: "sort by location asc", - property: "location", - order: "asc", - expected: []interface{}{ - map[string]interface{}{"name": "Rotterdam"}, - map[string]interface{}{"name": nil}, - map[string]interface{}{"name": "Berlin"}, - map[string]interface{}{"name": "Missing Island"}, - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": "Dusseldorf"}, - }, - }, - { - name: "sort by location desc", - property: "location", - order: "desc", - expected: []interface{}{ - map[string]interface{}{"name": "Dusseldorf"}, - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": "Missing Island"}, - map[string]interface{}{"name": "Rotterdam"}, - map[string]interface{}{"name": nil}, - map[string]interface{}{"name": "Berlin"}, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, fmt.Sprintf(query, tt.property, tt.order)) - got := result.Get("Get", "City").AsSlice() - if !reflect.DeepEqual(got, tt.expected) { - t.Errorf("sort objects got = %v, want %v", got, tt.expected) - } - }) - } - }) - - t.Run("complex sort", func(t *testing.T) { - query := ` - { - Get { - City( - %s - ) { - name - } - } - } - ` - queryLimit := ` - { - Get { - City( - limit: %d - %s - ) { - name - } - } - } - ` - tests := []struct { - name string - sort []string - expected []interface{} - }{ - { - name: "sort by population and name asc", - sort: []string{ - buildSort([]string{"population"}, "asc"), - buildSort([]string{"name"}, "asc"), - }, - expected: []interface{}{ - map[string]interface{}{"name": nil}, - map[string]interface{}{"name": "Missing Island"}, - map[string]interface{}{"name": "Dusseldorf"}, - map[string]interface{}{"name": "Rotterdam"}, - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": "Berlin"}, - }, - }, - { - name: "sort by population asc and name desc", - sort: []string{ - buildSort([]string{"population"}, "asc"), - buildSort([]string{"name"}, "desc"), - }, - expected: []interface{}{ - map[string]interface{}{"name": nil}, - map[string]interface{}{"name": "Missing Island"}, - map[string]interface{}{"name": "Rotterdam"}, - map[string]interface{}{"name": "Dusseldorf"}, - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": "Berlin"}, - }, - }, - { - name: "sort by name asc and population desc", - sort: []string{ - buildSort([]string{"name"}, "asc"), - buildSort([]string{"population"}, "desc"), - }, - expected: []interface{}{ - map[string]interface{}{"name": nil}, - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": "Berlin"}, - map[string]interface{}{"name": "Dusseldorf"}, - map[string]interface{}{"name": "Missing Island"}, - map[string]interface{}{"name": "Rotterdam"}, - }, - }, - { - name: "sort by population and name desc", - sort: []string{ - buildSort([]string{"population"}, "desc"), - buildSort([]string{"name"}, "desc"), - }, - expected: []interface{}{ - map[string]interface{}{"name": "Berlin"}, - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": "Rotterdam"}, - map[string]interface{}{"name": "Dusseldorf"}, - map[string]interface{}{"name": "Missing Island"}, - map[string]interface{}{"name": nil}, - }, - }, - { - name: "sort by phoneNumber and population and name asc", - sort: []string{ - buildSort([]string{"phoneNumber"}, "asc"), - buildSort([]string{"population"}, "asc"), - buildSort([]string{"name"}, "asc"), - }, - expected: []interface{}{ - map[string]interface{}{"name": nil}, - map[string]interface{}{"name": "Missing Island"}, - map[string]interface{}{"name": "Rotterdam"}, - map[string]interface{}{"name": "Dusseldorf"}, - map[string]interface{}{"name": "Berlin"}, - map[string]interface{}{"name": "Amsterdam"}, - }, - }, - { - name: "sort by isCapital asc and name asc", - sort: []string{ - buildSort([]string{"isCapital"}, "asc"), - buildSort([]string{"name"}, "asc"), - }, - expected: []interface{}{ - map[string]interface{}{"name": nil}, - map[string]interface{}{"name": "Dusseldorf"}, - map[string]interface{}{"name": "Missing Island"}, - map[string]interface{}{"name": "Rotterdam"}, - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": "Berlin"}, - }, - }, - { - name: "sort by isCapital asc and name desc", - sort: []string{ - buildSort([]string{"isCapital"}, "asc"), - buildSort([]string{"name"}, "desc"), - }, - expected: []interface{}{ - map[string]interface{}{"name": nil}, - map[string]interface{}{"name": "Rotterdam"}, - map[string]interface{}{"name": "Missing Island"}, - map[string]interface{}{"name": "Dusseldorf"}, - map[string]interface{}{"name": "Berlin"}, - map[string]interface{}{"name": "Amsterdam"}, - }, - }, - { - name: "sort by isCapital desc and name asc", - sort: []string{ - buildSort([]string{"isCapital"}, "desc"), - buildSort([]string{"name"}, "asc"), - }, - expected: []interface{}{ - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": "Berlin"}, - map[string]interface{}{"name": "Dusseldorf"}, - map[string]interface{}{"name": "Missing Island"}, - map[string]interface{}{"name": "Rotterdam"}, - map[string]interface{}{"name": nil}, - }, - }, - { - name: "sort by isCapital desc and name desc", - sort: []string{ - buildSort([]string{"isCapital"}, "desc"), - buildSort([]string{"name"}, "desc"), - }, - expected: []interface{}{ - map[string]interface{}{"name": "Berlin"}, - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": "Rotterdam"}, - map[string]interface{}{"name": "Missing Island"}, - map[string]interface{}{"name": "Dusseldorf"}, - map[string]interface{}{"name": nil}, - }, - }, - { - name: "sort by isCapital asc and population desc and name asc", - sort: []string{ - buildSort([]string{"isCapital"}, "asc"), - buildSort([]string{"population"}, "desc"), - buildSort([]string{"name"}, "asc"), - }, - expected: []interface{}{ - map[string]interface{}{"name": nil}, - map[string]interface{}{"name": "Dusseldorf"}, - map[string]interface{}{"name": "Rotterdam"}, - map[string]interface{}{"name": "Missing Island"}, - map[string]interface{}{"name": "Berlin"}, - map[string]interface{}{"name": "Amsterdam"}, - }, - }, - { - name: "sort by isCapital desc and population desc and name desc", - sort: []string{ - buildSort([]string{"isCapital"}, "desc"), - buildSort([]string{"population"}, "desc"), - buildSort([]string{"name"}, "desc"), - }, - expected: []interface{}{ - map[string]interface{}{"name": "Berlin"}, - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": "Rotterdam"}, - map[string]interface{}{"name": "Dusseldorf"}, - map[string]interface{}{"name": "Missing Island"}, - map[string]interface{}{"name": nil}, - }, - }, - { - name: "sort by isCapital asc and timezones asc and city rights asc and name asc", - sort: []string{ - buildSort([]string{"isCapital"}, "asc"), - buildSort([]string{"timezones"}, "asc"), - buildSort([]string{"cityRights"}, "asc"), - buildSort([]string{"name"}, "asc"), - }, - expected: []interface{}{ - map[string]interface{}{"name": nil}, - map[string]interface{}{"name": "Missing Island"}, - map[string]interface{}{"name": "Dusseldorf"}, - map[string]interface{}{"name": "Rotterdam"}, - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": "Berlin"}, - }, - }, - { - name: "sort by isCapital desc and timezones asc and city rights asc and name desc", - sort: []string{ - buildSort([]string{"isCapital"}, "desc"), - buildSort([]string{"timezones"}, "asc"), - buildSort([]string{"cityRights"}, "asc"), - buildSort([]string{"name"}, "desc"), - }, - expected: []interface{}{ - map[string]interface{}{"name": "Berlin"}, - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": "Missing Island"}, - map[string]interface{}{"name": "Dusseldorf"}, - map[string]interface{}{"name": "Rotterdam"}, - map[string]interface{}{"name": nil}, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Run("without limit", func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, fmt.Sprintf(query, buildSortFilter(tt.sort))) - got := result.Get("Get", "City").AsSlice() - if !reflect.DeepEqual(got, tt.expected) { - t.Errorf("sort objects got = %v, want %v", got, tt.expected) - } - }) - t.Run("with limit", func(t *testing.T) { - limit := 4 - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, fmt.Sprintf(queryLimit, limit, buildSortFilter(tt.sort))) - got := result.Get("Get", "City").AsSlice() - if !reflect.DeepEqual(got, tt.expected[:limit]) { - t.Errorf("sort objects got = %v, want %v", got, tt.expected) - } - }) - }) - } - }) - - t.Run("sort with where", func(t *testing.T) { - query := ` - { - Get { - City( - sort: [{ - path: ["location"] - order: %s - }] - where: { - operator: Or, - operands: [ - {valueText: "6ffb03f8-a853-4ec5-a5d8-302e45aaaf13", path: ["id"], operator: Equal}, - {valueText: "823abeca-eef3-41c7-b587-7a6977b08003", path: ["id"], operator: Equal} - ]} - ) { - name - } - } - } - ` - tests := []struct { - name string - order string - expected []interface{} - }{ - { - name: "location asc", - order: "asc", - expected: []interface{}{ - map[string]interface{}{"name": "Missing Island"}, - map[string]interface{}{"name": "Dusseldorf"}, - }, - }, - { - name: "location desc", - order: "desc", - expected: []interface{}{ - map[string]interface{}{"name": "Dusseldorf"}, - map[string]interface{}{"name": "Missing Island"}, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, fmt.Sprintf(query, tt.order)) - got := result.Get("Get", "City").AsSlice() - if !reflect.DeepEqual(got, tt.expected) { - t.Errorf("sort objects got = %v, want %v", got, tt.expected) - } - }) - } - }) - - t.Run("sort with where with non-existent-uuid", func(t *testing.T) { - query := ` - { - Get { - City( - sort: [{ - path: ["location"] - order: asc - }] - where: { - valueText: "non-existent-uuid", path: ["id"], operator: Equal - } - ) { - name - } - } - } - ` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - got := result.Get("Get", "City").AsSlice() - assert.Empty(t, got) - }) - - t.Run("sort with nearText (with distance)", func(t *testing.T) { - query := ` - { - Get { - City( - nearText: { - concepts: ["Berlin"] - distance: 0.6 - } - %s - ) { - name - } - } - } - ` - tests := []struct { - name string - sort []string - expected []interface{} - }{ - { - name: "name asc", - sort: []string{ - buildSort([]string{"name"}, "asc"), - }, - expected: []interface{}{ - map[string]interface{}{"name": nil}, - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": "Berlin"}, - map[string]interface{}{"name": "Dusseldorf"}, - map[string]interface{}{"name": "Rotterdam"}, - }, - }, - { - name: "name desc", - sort: []string{ - buildSort([]string{"name"}, "desc"), - }, - expected: []interface{}{ - map[string]interface{}{"name": "Rotterdam"}, - map[string]interface{}{"name": "Dusseldorf"}, - map[string]interface{}{"name": "Berlin"}, - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": nil}, - }, - }, - { - name: "population asc", - sort: []string{ - buildSort([]string{"population"}, "asc"), - }, - expected: []interface{}{ - map[string]interface{}{"name": nil}, - map[string]interface{}{"name": "Dusseldorf"}, - map[string]interface{}{"name": "Rotterdam"}, - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": "Berlin"}, - }, - }, - { - name: "population desc", - sort: []string{ - buildSort([]string{"population"}, "desc"), - }, - expected: []interface{}{ - map[string]interface{}{"name": "Berlin"}, - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": "Dusseldorf"}, - map[string]interface{}{"name": "Rotterdam"}, - map[string]interface{}{"name": nil}, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, fmt.Sprintf(query, buildSortFilter(tt.sort))) - got := result.Get("Get", "City").AsSlice() - if !reflect.DeepEqual(got, tt.expected) { - t.Errorf("sort objects got = %v, want %v", got, tt.expected) - } - }) - } - }) - - t.Run("sort with nearText (with certainty)", func(t *testing.T) { - query := ` - { - Get { - City( - nearText: { - concepts: ["Berlin"] - certainty: 0.7 - } - %s - ) { - name - } - } - } - ` - tests := []struct { - name string - sort []string - expected []interface{} - }{ - { - name: "name asc", - sort: []string{ - buildSort([]string{"name"}, "asc"), - }, - expected: []interface{}{ - map[string]interface{}{"name": nil}, - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": "Berlin"}, - map[string]interface{}{"name": "Dusseldorf"}, - map[string]interface{}{"name": "Rotterdam"}, - }, - }, - { - name: "name desc", - sort: []string{ - buildSort([]string{"name"}, "desc"), - }, - expected: []interface{}{ - map[string]interface{}{"name": "Rotterdam"}, - map[string]interface{}{"name": "Dusseldorf"}, - map[string]interface{}{"name": "Berlin"}, - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": nil}, - }, - }, - { - name: "population asc", - sort: []string{ - buildSort([]string{"population"}, "asc"), - }, - expected: []interface{}{ - map[string]interface{}{"name": nil}, - map[string]interface{}{"name": "Dusseldorf"}, - map[string]interface{}{"name": "Rotterdam"}, - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": "Berlin"}, - }, - }, - { - name: "population desc", - sort: []string{ - buildSort([]string{"population"}, "desc"), - }, - expected: []interface{}{ - map[string]interface{}{"name": "Berlin"}, - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": "Dusseldorf"}, - map[string]interface{}{"name": "Rotterdam"}, - map[string]interface{}{"name": nil}, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, fmt.Sprintf(query, buildSortFilter(tt.sort))) - got := result.Get("Get", "City").AsSlice() - if !reflect.DeepEqual(got, tt.expected) { - t.Errorf("sort objects got = %v, want %v", got, tt.expected) - } - }) - } - }) - - t.Run("sort with nearText and limit (with distance)", func(t *testing.T) { - query := ` - { - Get { - City( - nearText: { - concepts: ["Berlin"] - distance: 0.6 - } - %s - limit: 2 - ) { - name - } - } - } - ` - tests := []struct { - name string - sort []string - expected []interface{} - }{ - { - name: "name asc", - sort: []string{ - buildSort([]string{"name"}, "asc"), - }, - expected: []interface{}{ - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": "Berlin"}, - }, - }, - { - name: "name desc", - sort: []string{ - buildSort([]string{"name"}, "desc"), - }, - expected: []interface{}{ - map[string]interface{}{"name": "Berlin"}, - map[string]interface{}{"name": "Amsterdam"}, - }, - }, - { - name: "population asc", - sort: []string{ - buildSort([]string{"population"}, "asc"), - }, - expected: []interface{}{ - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": "Berlin"}, - }, - }, - { - name: "population desc", - sort: []string{ - buildSort([]string{"population"}, "desc"), - }, - expected: []interface{}{ - map[string]interface{}{"name": "Berlin"}, - map[string]interface{}{"name": "Amsterdam"}, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, fmt.Sprintf(query, buildSortFilter(tt.sort))) - got := result.Get("Get", "City").AsSlice() - if !reflect.DeepEqual(got, tt.expected) { - t.Errorf("sort objects got = %v, want %v", got, tt.expected) - } - }) - } - }) - - t.Run("sort with nearText and limit (with certainty)", func(t *testing.T) { - query := ` - { - Get { - City( - nearText: { - concepts: ["Berlin"] - certainty: 0.7 - } - %s - limit: 2 - ) { - name - } - } - } - ` - tests := []struct { - name string - sort []string - expected []interface{} - }{ - { - name: "name asc", - sort: []string{ - buildSort([]string{"name"}, "asc"), - }, - expected: []interface{}{ - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": "Berlin"}, - }, - }, - { - name: "name desc", - sort: []string{ - buildSort([]string{"name"}, "desc"), - }, - expected: []interface{}{ - map[string]interface{}{"name": "Berlin"}, - map[string]interface{}{"name": "Amsterdam"}, - }, - }, - { - name: "population asc", - sort: []string{ - buildSort([]string{"population"}, "asc"), - }, - expected: []interface{}{ - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": "Berlin"}, - }, - }, - { - name: "population desc", - sort: []string{ - buildSort([]string{"population"}, "desc"), - }, - expected: []interface{}{ - map[string]interface{}{"name": "Berlin"}, - map[string]interface{}{"name": "Amsterdam"}, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, fmt.Sprintf(query, buildSortFilter(tt.sort))) - got := result.Get("Get", "City").AsSlice() - if !reflect.DeepEqual(got, tt.expected) { - t.Errorf("sort objects got = %v, want %v", got, tt.expected) - } - }) - } - }) - - t.Run("sort with where and nearText and limit (with distance)", func(t *testing.T) { - query := ` - { - Get { - City( - where: { - valueBoolean: true, - operator: Equal, - path: ["isCapital"] - } - nearText: { - concepts: ["Amsterdam"] - distance: 0.6 - } - %s - limit: 2 - ) { - name - } - } - } - ` - tests := []struct { - name string - sort []string - expected []interface{} - }{ - { - name: "name asc", - sort: []string{ - buildSort([]string{"name"}, "asc"), - }, - expected: []interface{}{ - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": "Berlin"}, - }, - }, - { - name: "name desc", - sort: []string{ - buildSort([]string{"name"}, "desc"), - }, - expected: []interface{}{ - map[string]interface{}{"name": "Berlin"}, - map[string]interface{}{"name": "Amsterdam"}, - }, - }, - { - name: "population asc", - sort: []string{ - buildSort([]string{"population"}, "asc"), - }, - expected: []interface{}{ - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": "Berlin"}, - }, - }, - { - name: "population desc", - sort: []string{ - buildSort([]string{"population"}, "desc"), - }, - expected: []interface{}{ - map[string]interface{}{"name": "Berlin"}, - map[string]interface{}{"name": "Amsterdam"}, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, fmt.Sprintf(query, buildSortFilter(tt.sort))) - got := result.Get("Get", "City").AsSlice() - if !reflect.DeepEqual(got, tt.expected) { - t.Errorf("sort objects got = %v, want %v", got, tt.expected) - } - }) - } - }) - - t.Run("sort with where and nearText and limit (with certainty)", func(t *testing.T) { - query := ` - { - Get { - City( - where: { - valueBoolean: true, - operator: Equal, - path: ["isCapital"] - } - nearText: { - concepts: ["Amsterdam"] - certainty: 0.7 - } - %s - limit: 2 - ) { - name - } - } - } - ` - tests := []struct { - name string - sort []string - expected []interface{} - }{ - { - name: "name asc", - sort: []string{ - buildSort([]string{"name"}, "asc"), - }, - expected: []interface{}{ - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": "Berlin"}, - }, - }, - { - name: "name desc", - sort: []string{ - buildSort([]string{"name"}, "desc"), - }, - expected: []interface{}{ - map[string]interface{}{"name": "Berlin"}, - map[string]interface{}{"name": "Amsterdam"}, - }, - }, - { - name: "population asc", - sort: []string{ - buildSort([]string{"population"}, "asc"), - }, - expected: []interface{}{ - map[string]interface{}{"name": "Amsterdam"}, - map[string]interface{}{"name": "Berlin"}, - }, - }, - { - name: "population desc", - sort: []string{ - buildSort([]string{"population"}, "desc"), - }, - expected: []interface{}{ - map[string]interface{}{"name": "Berlin"}, - map[string]interface{}{"name": "Amsterdam"}, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, fmt.Sprintf(query, buildSortFilter(tt.sort))) - got := result.Get("Get", "City").AsSlice() - if !reflect.DeepEqual(got, tt.expected) { - t.Errorf("sort objects got = %v, want %v", got, tt.expected) - } - }) - } - }) - - t.Run("broken sort clause", func(t *testing.T) { - query := ` - { - Get { - %s( - %s - ) { - name - } - } - } - ` - tests := []struct { - name string - className string - sort []string - expectedMsg string - }{ - { - name: "empty path", - className: "City", - sort: []string{ - buildSort([]string{}, "asc"), - }, - expectedMsg: "invalid 'sort' parameter: sort parameter at position 0: " + - "path parameter cannot be empty", - }, - { - name: "empty property in path", - className: "City", - sort: []string{ - buildSort([]string{""}, "asc"), - }, - expectedMsg: "invalid 'sort' parameter: sort parameter at position 0: " + - "no such prop with name '' found in class 'City' in the schema. " + - "Check your schema files for which properties in this class are available", - }, - { - name: "reference prop in path", - className: "City", - sort: []string{ - buildSort([]string{"ref", "prop"}, "asc"), - }, - expectedMsg: "invalid 'sort' parameter: sort parameter at position 0: " + - "sorting by reference not supported, path must have exactly one argument", - }, - { - name: "non-existent class", - className: "NonExistentClass", - sort: []string{ - buildSort([]string{"property"}, "asc"), - }, - expectedMsg: "Cannot query field \"NonExistentClass\" on type \"GetObjectsObj\".", - }, - { - name: "non-existent property", - className: "City", - sort: []string{ - buildSort([]string{"nonexistentproperty"}, "asc"), - }, - expectedMsg: "invalid 'sort' parameter: sort parameter at position 0: " + - "no such prop with name 'nonexistentproperty' found in class 'City' in the schema. " + - "Check your schema files for which properties in this class are available", - }, - { - name: "reference property", - className: "City", - sort: []string{ - buildSort([]string{"inCountry"}, "asc"), - }, - expectedMsg: "invalid 'sort' parameter: sort parameter at position 0: " + - "sorting by reference not supported, " + - "property \"inCountry\" is a ref prop to the class \"Country\"", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := graphqlhelper.ErrorGraphQL(t, helper.RootAuth, fmt.Sprintf(query, tt.className, buildSortFilter(tt.sort))) - for _, gqlError := range result { - assert.Equal(t, tt.expectedMsg, gqlError.Message) - } - }) - } - }) -} diff --git a/test/acceptance/graphql_resolvers/local_get_with_unlimited_vector_search_test.go b/test/acceptance/graphql_resolvers/local_get_with_unlimited_vector_search_test.go deleted file mode 100644 index f31dd4df7fa7fae44b039abaa120ca45b68558d5..0000000000000000000000000000000000000000 --- a/test/acceptance/graphql_resolvers/local_get_with_unlimited_vector_search_test.go +++ /dev/null @@ -1,678 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "fmt" - "testing" - - graphqlhelper "github.com/weaviate/weaviate/test/helper/graphql" - - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/test/helper" -) - -func gettingObjectsWithNearFields(t *testing.T) { - defaultLimit := 100 - - // nearVector - - t.Run("nearVector: with implicit unlimited search - no limit provided (with distance)", func(t *testing.T) { - query := ` - { - Get { - RansomNote( - nearVector: { - distance: 1.8 - vector: [-0.07853702, -0.33730024, 0.62998116, 0.08703484, -0.0011832615, 0.23041481, -0.091878965, 0.1184808, 0.060692377, 0.1748896, 0.53659165, 0.12019079, 0.54373807, -0.43369776, 0.1843199, -0.19319294, 0.122559674, -0.09465141, -0.14069664, 0.031092037, -0.1763922, 0.0074394196, -0.2586067, 0.10990611, -0.18623954, -0.038631044, -0.22795723, 0.09296776, -0.31110525, -0.37963995, -0.19045947, 0.48089907, 0.46725857, 0.28541213, 0.08294283, -0.18865398, 0.09647029, 0.2321466, -0.03435125, -0.09602424, -0.3831683, -0.027315892, 0.4215511, -0.35865632, 0.41955224, 0.090477064, 0.29026023, -0.48485047, -0.24656451, -0.06478625, 0.07755969, -0.049564634, 0.026147474, -0.028342195, -0.035627227, 0.49309397, 0.3705331, 0.04615483, 0.14789912, -0.01220134, 0.300666, -0.246646, 0.0038986988, 0.16730541, 0.46581128, -0.04931062, 0.040290095, 0.32867354, -0.18300997, 0.30411696, -0.1969807, 0.4727539, -0.31915516, -0.32722718, 0.12694982, 0.22583495, -0.014532595, -0.14432396, 0.2469766, 0.14872919, -0.06750808, 0.06351308, -0.287756, -0.32118404, 0.25326216, 0.45288888, -0.36307186, 0.05369787, -0.3283361, 0.07754738, 0.38473788, -0.5120014, -0.3344492, -0.1102767, -0.16755687, -0.3575448, -0.2555015, -0.42976367, -0.2695758, 0.04068499, 0.591914, -0.008395256, 0.2613976, -0.51722556, -0.22581989, 0.036452737, 0.42190477, -0.256124, 0.25849363, -0.073515825, -0.08690646, 0.013338611, 0.14928678, 0.16371651, 0.111465625, -0.117571846, -0.44434816, 0.07603647, 0.4188736, -0.16967061, 0.040313836, 0.41924894, -0.36998197, 0.23204626, -0.23309743, -0.18061559, 1.0674918, -0.51468146, -0.37230963, 0.02214618, -0.5616187, -0.07473461, -0.3314831, -0.24691144, -0.34061712, -0.1511554, 0.33711013, 0.1055847, -0.047220375, -0.06317049, -0.22572862, -0.21646689, 0.090705894, 0.018562902, 0.020744732, -0.5342965, -0.23402104, -0.17680043, 0.1363935, -0.17916845, 0.37825805, -0.07233101, -0.28244817, 0.4055966, 0.19701958, 0.6236174, 0.078134544, 0.46439224, -0.60451704, 0.16722181, -0.20011653, 0.36931068, -0.39967215, 0.21178648, 0.47920865, -0.033521328, 0.57077545, -0.8003054, -0.4028354, 0.27799648, -0.23070334, 0.57747835, 0.49984616, -0.12409506, -0.26694623, -0.20168623, -0.19415514, -0.4626071, 0.10374411, 0.24878122, 0.47352287, -0.6494472, -0.26087105, 0.418008, -0.2789803, -0.60986733, -0.54914564, 0.4734504, 0.04347568, -0.13549352, 0.1530609, 0.085039385, -0.014595425, -0.1106091, 0.014441653, 0.14899726, -0.107090004, 0.03979257, 0.20897605, -0.040235993, 0.1928343, -0.048328623, 0.5435577, -0.1704212, -0.016530415, 0.11402996, 0.24666561, -0.62601864, 0.6729872, -0.21594357, -0.3161654, 0.2899072, -0.05281632, 0.026857251, 0.13927892, 0.26362655, 0.37995058, -0.056429606, 0.27310744, -0.34237143, -0.6419976, -0.02513231, -0.18217334, 0.021232722, -0.35155025, 0.055071075, -0.22192729, 0.4597671, 0.09872845, -0.41803727, -0.08897542, -0.63276047, 0.38059604, 0.45347637, 0.52723855, 0.25096536, -0.3165448, 0.43803728, 0.02419832, 0.317004, -0.059602205, 0.15561013, 0.11867607, 0.7157601, 0.08024589, -0.013107148, 0.3127224, -0.08844044, 0.5374578, 0.39421698, -0.054171022, 0.0913302, -0.081881694, 0.24596375, -0.2841653, -0.5482517, -0.5673938, 0.05889957, -0.1146344, 0.39452744, -0.03414711, 0.32027423, 0.2599335, -0.31470263, 0.45967287, -0.5710101, -0.21222454, 0.38154987, -0.21218868, -0.4366558, 0.13715877, 0.23925674, 0.34832072, -0.03769251, 0.25530148, 0.10662722, -0.5269836, 0.32952255, 0.46165445, 0.3794754, -0.061259665, 0.02883365, -0.3199015, 0.40625557, -0.3794913, 0.42420092, 0.4631467, 0.54236996, 0.031472385, 0.2635622, -0.25566247, -0.040713936, 0.48734123, 0.2742017, -0.15524681, 0.025654443, 0.056942068, -0.48883253, 0.60433495, 0.03514151] - } - ) { - _additional { - vector - } - contents - } - } - } - ` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - notes := result.Get("Get", "RansomNote").AsSlice() - require.NotEmpty(t, notes) - require.Greater(t, len(notes), defaultLimit) - }) - - t.Run("nearVector: with implicit unlimited search - no limit provided (with certainty)", func(t *testing.T) { - query := ` - { - Get { - RansomNote( - nearVector: { - certainty: 0.1 - vector: [-0.07853702, -0.33730024, 0.62998116, 0.08703484, -0.0011832615, 0.23041481, -0.091878965, 0.1184808, 0.060692377, 0.1748896, 0.53659165, 0.12019079, 0.54373807, -0.43369776, 0.1843199, -0.19319294, 0.122559674, -0.09465141, -0.14069664, 0.031092037, -0.1763922, 0.0074394196, -0.2586067, 0.10990611, -0.18623954, -0.038631044, -0.22795723, 0.09296776, -0.31110525, -0.37963995, -0.19045947, 0.48089907, 0.46725857, 0.28541213, 0.08294283, -0.18865398, 0.09647029, 0.2321466, -0.03435125, -0.09602424, -0.3831683, -0.027315892, 0.4215511, -0.35865632, 0.41955224, 0.090477064, 0.29026023, -0.48485047, -0.24656451, -0.06478625, 0.07755969, -0.049564634, 0.026147474, -0.028342195, -0.035627227, 0.49309397, 0.3705331, 0.04615483, 0.14789912, -0.01220134, 0.300666, -0.246646, 0.0038986988, 0.16730541, 0.46581128, -0.04931062, 0.040290095, 0.32867354, -0.18300997, 0.30411696, -0.1969807, 0.4727539, -0.31915516, -0.32722718, 0.12694982, 0.22583495, -0.014532595, -0.14432396, 0.2469766, 0.14872919, -0.06750808, 0.06351308, -0.287756, -0.32118404, 0.25326216, 0.45288888, -0.36307186, 0.05369787, -0.3283361, 0.07754738, 0.38473788, -0.5120014, -0.3344492, -0.1102767, -0.16755687, -0.3575448, -0.2555015, -0.42976367, -0.2695758, 0.04068499, 0.591914, -0.008395256, 0.2613976, -0.51722556, -0.22581989, 0.036452737, 0.42190477, -0.256124, 0.25849363, -0.073515825, -0.08690646, 0.013338611, 0.14928678, 0.16371651, 0.111465625, -0.117571846, -0.44434816, 0.07603647, 0.4188736, -0.16967061, 0.040313836, 0.41924894, -0.36998197, 0.23204626, -0.23309743, -0.18061559, 1.0674918, -0.51468146, -0.37230963, 0.02214618, -0.5616187, -0.07473461, -0.3314831, -0.24691144, -0.34061712, -0.1511554, 0.33711013, 0.1055847, -0.047220375, -0.06317049, -0.22572862, -0.21646689, 0.090705894, 0.018562902, 0.020744732, -0.5342965, -0.23402104, -0.17680043, 0.1363935, -0.17916845, 0.37825805, -0.07233101, -0.28244817, 0.4055966, 0.19701958, 0.6236174, 0.078134544, 0.46439224, -0.60451704, 0.16722181, -0.20011653, 0.36931068, -0.39967215, 0.21178648, 0.47920865, -0.033521328, 0.57077545, -0.8003054, -0.4028354, 0.27799648, -0.23070334, 0.57747835, 0.49984616, -0.12409506, -0.26694623, -0.20168623, -0.19415514, -0.4626071, 0.10374411, 0.24878122, 0.47352287, -0.6494472, -0.26087105, 0.418008, -0.2789803, -0.60986733, -0.54914564, 0.4734504, 0.04347568, -0.13549352, 0.1530609, 0.085039385, -0.014595425, -0.1106091, 0.014441653, 0.14899726, -0.107090004, 0.03979257, 0.20897605, -0.040235993, 0.1928343, -0.048328623, 0.5435577, -0.1704212, -0.016530415, 0.11402996, 0.24666561, -0.62601864, 0.6729872, -0.21594357, -0.3161654, 0.2899072, -0.05281632, 0.026857251, 0.13927892, 0.26362655, 0.37995058, -0.056429606, 0.27310744, -0.34237143, -0.6419976, -0.02513231, -0.18217334, 0.021232722, -0.35155025, 0.055071075, -0.22192729, 0.4597671, 0.09872845, -0.41803727, -0.08897542, -0.63276047, 0.38059604, 0.45347637, 0.52723855, 0.25096536, -0.3165448, 0.43803728, 0.02419832, 0.317004, -0.059602205, 0.15561013, 0.11867607, 0.7157601, 0.08024589, -0.013107148, 0.3127224, -0.08844044, 0.5374578, 0.39421698, -0.054171022, 0.0913302, -0.081881694, 0.24596375, -0.2841653, -0.5482517, -0.5673938, 0.05889957, -0.1146344, 0.39452744, -0.03414711, 0.32027423, 0.2599335, -0.31470263, 0.45967287, -0.5710101, -0.21222454, 0.38154987, -0.21218868, -0.4366558, 0.13715877, 0.23925674, 0.34832072, -0.03769251, 0.25530148, 0.10662722, -0.5269836, 0.32952255, 0.46165445, 0.3794754, -0.061259665, 0.02883365, -0.3199015, 0.40625557, -0.3794913, 0.42420092, 0.4631467, 0.54236996, 0.031472385, 0.2635622, -0.25566247, -0.040713936, 0.48734123, 0.2742017, -0.15524681, 0.025654443, 0.056942068, -0.48883253, 0.60433495, 0.03514151] - } - ) { - _additional { - vector - } - contents - } - } - } - ` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - notes := result.Get("Get", "RansomNote").AsSlice() - require.NotEmpty(t, notes) - require.Greater(t, len(notes), defaultLimit) - }) - - t.Run("nearVector: with implicit unlimited search - negative limit provided (with distance)", func(t *testing.T) { - query := ` - { - Get { - RansomNote( - limit: -1 - nearVector: { - distance: 0.9 - vector: [-0.07853702, -0.33730024, 0.62998116, 0.08703484, -0.0011832615, 0.23041481, -0.091878965, 0.1184808, 0.060692377, 0.1748896, 0.53659165, 0.12019079, 0.54373807, -0.43369776, 0.1843199, -0.19319294, 0.122559674, -0.09465141, -0.14069664, 0.031092037, -0.1763922, 0.0074394196, -0.2586067, 0.10990611, -0.18623954, -0.038631044, -0.22795723, 0.09296776, -0.31110525, -0.37963995, -0.19045947, 0.48089907, 0.46725857, 0.28541213, 0.08294283, -0.18865398, 0.09647029, 0.2321466, -0.03435125, -0.09602424, -0.3831683, -0.027315892, 0.4215511, -0.35865632, 0.41955224, 0.090477064, 0.29026023, -0.48485047, -0.24656451, -0.06478625, 0.07755969, -0.049564634, 0.026147474, -0.028342195, -0.035627227, 0.49309397, 0.3705331, 0.04615483, 0.14789912, -0.01220134, 0.300666, -0.246646, 0.0038986988, 0.16730541, 0.46581128, -0.04931062, 0.040290095, 0.32867354, -0.18300997, 0.30411696, -0.1969807, 0.4727539, -0.31915516, -0.32722718, 0.12694982, 0.22583495, -0.014532595, -0.14432396, 0.2469766, 0.14872919, -0.06750808, 0.06351308, -0.287756, -0.32118404, 0.25326216, 0.45288888, -0.36307186, 0.05369787, -0.3283361, 0.07754738, 0.38473788, -0.5120014, -0.3344492, -0.1102767, -0.16755687, -0.3575448, -0.2555015, -0.42976367, -0.2695758, 0.04068499, 0.591914, -0.008395256, 0.2613976, -0.51722556, -0.22581989, 0.036452737, 0.42190477, -0.256124, 0.25849363, -0.073515825, -0.08690646, 0.013338611, 0.14928678, 0.16371651, 0.111465625, -0.117571846, -0.44434816, 0.07603647, 0.4188736, -0.16967061, 0.040313836, 0.41924894, -0.36998197, 0.23204626, -0.23309743, -0.18061559, 1.0674918, -0.51468146, -0.37230963, 0.02214618, -0.5616187, -0.07473461, -0.3314831, -0.24691144, -0.34061712, -0.1511554, 0.33711013, 0.1055847, -0.047220375, -0.06317049, -0.22572862, -0.21646689, 0.090705894, 0.018562902, 0.020744732, -0.5342965, -0.23402104, -0.17680043, 0.1363935, -0.17916845, 0.37825805, -0.07233101, -0.28244817, 0.4055966, 0.19701958, 0.6236174, 0.078134544, 0.46439224, -0.60451704, 0.16722181, -0.20011653, 0.36931068, -0.39967215, 0.21178648, 0.47920865, -0.033521328, 0.57077545, -0.8003054, -0.4028354, 0.27799648, -0.23070334, 0.57747835, 0.49984616, -0.12409506, -0.26694623, -0.20168623, -0.19415514, -0.4626071, 0.10374411, 0.24878122, 0.47352287, -0.6494472, -0.26087105, 0.418008, -0.2789803, -0.60986733, -0.54914564, 0.4734504, 0.04347568, -0.13549352, 0.1530609, 0.085039385, -0.014595425, -0.1106091, 0.014441653, 0.14899726, -0.107090004, 0.03979257, 0.20897605, -0.040235993, 0.1928343, -0.048328623, 0.5435577, -0.1704212, -0.016530415, 0.11402996, 0.24666561, -0.62601864, 0.6729872, -0.21594357, -0.3161654, 0.2899072, -0.05281632, 0.026857251, 0.13927892, 0.26362655, 0.37995058, -0.056429606, 0.27310744, -0.34237143, -0.6419976, -0.02513231, -0.18217334, 0.021232722, -0.35155025, 0.055071075, -0.22192729, 0.4597671, 0.09872845, -0.41803727, -0.08897542, -0.63276047, 0.38059604, 0.45347637, 0.52723855, 0.25096536, -0.3165448, 0.43803728, 0.02419832, 0.317004, -0.059602205, 0.15561013, 0.11867607, 0.7157601, 0.08024589, -0.013107148, 0.3127224, -0.08844044, 0.5374578, 0.39421698, -0.054171022, 0.0913302, -0.081881694, 0.24596375, -0.2841653, -0.5482517, -0.5673938, 0.05889957, -0.1146344, 0.39452744, -0.03414711, 0.32027423, 0.2599335, -0.31470263, 0.45967287, -0.5710101, -0.21222454, 0.38154987, -0.21218868, -0.4366558, 0.13715877, 0.23925674, 0.34832072, -0.03769251, 0.25530148, 0.10662722, -0.5269836, 0.32952255, 0.46165445, 0.3794754, -0.061259665, 0.02883365, -0.3199015, 0.40625557, -0.3794913, 0.42420092, 0.4631467, 0.54236996, 0.031472385, 0.2635622, -0.25566247, -0.040713936, 0.48734123, 0.2742017, -0.15524681, 0.025654443, 0.056942068, -0.48883253, 0.60433495, 0.03514151] - } - ) { - _additional { - vector - } - contents - } - } - } - ` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - notes := result.Get("Get", "RansomNote").AsSlice() - require.NotEmpty(t, notes) - require.Greater(t, len(notes), defaultLimit) - }) - - t.Run("nearVector: with implicit unlimited search - negative limit provided (with certainty)", func(t *testing.T) { - query := ` - { - Get { - RansomNote( - limit: -1 - nearVector: { - certainty: 0.1 - vector: [-0.07853702, -0.33730024, 0.62998116, 0.08703484, -0.0011832615, 0.23041481, -0.091878965, 0.1184808, 0.060692377, 0.1748896, 0.53659165, 0.12019079, 0.54373807, -0.43369776, 0.1843199, -0.19319294, 0.122559674, -0.09465141, -0.14069664, 0.031092037, -0.1763922, 0.0074394196, -0.2586067, 0.10990611, -0.18623954, -0.038631044, -0.22795723, 0.09296776, -0.31110525, -0.37963995, -0.19045947, 0.48089907, 0.46725857, 0.28541213, 0.08294283, -0.18865398, 0.09647029, 0.2321466, -0.03435125, -0.09602424, -0.3831683, -0.027315892, 0.4215511, -0.35865632, 0.41955224, 0.090477064, 0.29026023, -0.48485047, -0.24656451, -0.06478625, 0.07755969, -0.049564634, 0.026147474, -0.028342195, -0.035627227, 0.49309397, 0.3705331, 0.04615483, 0.14789912, -0.01220134, 0.300666, -0.246646, 0.0038986988, 0.16730541, 0.46581128, -0.04931062, 0.040290095, 0.32867354, -0.18300997, 0.30411696, -0.1969807, 0.4727539, -0.31915516, -0.32722718, 0.12694982, 0.22583495, -0.014532595, -0.14432396, 0.2469766, 0.14872919, -0.06750808, 0.06351308, -0.287756, -0.32118404, 0.25326216, 0.45288888, -0.36307186, 0.05369787, -0.3283361, 0.07754738, 0.38473788, -0.5120014, -0.3344492, -0.1102767, -0.16755687, -0.3575448, -0.2555015, -0.42976367, -0.2695758, 0.04068499, 0.591914, -0.008395256, 0.2613976, -0.51722556, -0.22581989, 0.036452737, 0.42190477, -0.256124, 0.25849363, -0.073515825, -0.08690646, 0.013338611, 0.14928678, 0.16371651, 0.111465625, -0.117571846, -0.44434816, 0.07603647, 0.4188736, -0.16967061, 0.040313836, 0.41924894, -0.36998197, 0.23204626, -0.23309743, -0.18061559, 1.0674918, -0.51468146, -0.37230963, 0.02214618, -0.5616187, -0.07473461, -0.3314831, -0.24691144, -0.34061712, -0.1511554, 0.33711013, 0.1055847, -0.047220375, -0.06317049, -0.22572862, -0.21646689, 0.090705894, 0.018562902, 0.020744732, -0.5342965, -0.23402104, -0.17680043, 0.1363935, -0.17916845, 0.37825805, -0.07233101, -0.28244817, 0.4055966, 0.19701958, 0.6236174, 0.078134544, 0.46439224, -0.60451704, 0.16722181, -0.20011653, 0.36931068, -0.39967215, 0.21178648, 0.47920865, -0.033521328, 0.57077545, -0.8003054, -0.4028354, 0.27799648, -0.23070334, 0.57747835, 0.49984616, -0.12409506, -0.26694623, -0.20168623, -0.19415514, -0.4626071, 0.10374411, 0.24878122, 0.47352287, -0.6494472, -0.26087105, 0.418008, -0.2789803, -0.60986733, -0.54914564, 0.4734504, 0.04347568, -0.13549352, 0.1530609, 0.085039385, -0.014595425, -0.1106091, 0.014441653, 0.14899726, -0.107090004, 0.03979257, 0.20897605, -0.040235993, 0.1928343, -0.048328623, 0.5435577, -0.1704212, -0.016530415, 0.11402996, 0.24666561, -0.62601864, 0.6729872, -0.21594357, -0.3161654, 0.2899072, -0.05281632, 0.026857251, 0.13927892, 0.26362655, 0.37995058, -0.056429606, 0.27310744, -0.34237143, -0.6419976, -0.02513231, -0.18217334, 0.021232722, -0.35155025, 0.055071075, -0.22192729, 0.4597671, 0.09872845, -0.41803727, -0.08897542, -0.63276047, 0.38059604, 0.45347637, 0.52723855, 0.25096536, -0.3165448, 0.43803728, 0.02419832, 0.317004, -0.059602205, 0.15561013, 0.11867607, 0.7157601, 0.08024589, -0.013107148, 0.3127224, -0.08844044, 0.5374578, 0.39421698, -0.054171022, 0.0913302, -0.081881694, 0.24596375, -0.2841653, -0.5482517, -0.5673938, 0.05889957, -0.1146344, 0.39452744, -0.03414711, 0.32027423, 0.2599335, -0.31470263, 0.45967287, -0.5710101, -0.21222454, 0.38154987, -0.21218868, -0.4366558, 0.13715877, 0.23925674, 0.34832072, -0.03769251, 0.25530148, 0.10662722, -0.5269836, 0.32952255, 0.46165445, 0.3794754, -0.061259665, 0.02883365, -0.3199015, 0.40625557, -0.3794913, 0.42420092, 0.4631467, 0.54236996, 0.031472385, 0.2635622, -0.25566247, -0.040713936, 0.48734123, 0.2742017, -0.15524681, 0.025654443, 0.056942068, -0.48883253, 0.60433495, 0.03514151] - } - ) { - _additional { - vector - } - contents - } - } - } - ` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - notes := result.Get("Get", "RansomNote").AsSlice() - require.NotEmpty(t, notes) - require.Greater(t, len(notes), defaultLimit) - }) - - t.Run("nearVector: with limited search - limit provided (with distance)", func(t *testing.T) { - query := ` - { - Get { - RansomNote( - limit: 10 - nearVector: { - distance: 0.9 - vector: [-0.07853702, -0.33730024, 0.62998116, 0.08703484, -0.0011832615, 0.23041481, -0.091878965, 0.1184808, 0.060692377, 0.1748896, 0.53659165, 0.12019079, 0.54373807, -0.43369776, 0.1843199, -0.19319294, 0.122559674, -0.09465141, -0.14069664, 0.031092037, -0.1763922, 0.0074394196, -0.2586067, 0.10990611, -0.18623954, -0.038631044, -0.22795723, 0.09296776, -0.31110525, -0.37963995, -0.19045947, 0.48089907, 0.46725857, 0.28541213, 0.08294283, -0.18865398, 0.09647029, 0.2321466, -0.03435125, -0.09602424, -0.3831683, -0.027315892, 0.4215511, -0.35865632, 0.41955224, 0.090477064, 0.29026023, -0.48485047, -0.24656451, -0.06478625, 0.07755969, -0.049564634, 0.026147474, -0.028342195, -0.035627227, 0.49309397, 0.3705331, 0.04615483, 0.14789912, -0.01220134, 0.300666, -0.246646, 0.0038986988, 0.16730541, 0.46581128, -0.04931062, 0.040290095, 0.32867354, -0.18300997, 0.30411696, -0.1969807, 0.4727539, -0.31915516, -0.32722718, 0.12694982, 0.22583495, -0.014532595, -0.14432396, 0.2469766, 0.14872919, -0.06750808, 0.06351308, -0.287756, -0.32118404, 0.25326216, 0.45288888, -0.36307186, 0.05369787, -0.3283361, 0.07754738, 0.38473788, -0.5120014, -0.3344492, -0.1102767, -0.16755687, -0.3575448, -0.2555015, -0.42976367, -0.2695758, 0.04068499, 0.591914, -0.008395256, 0.2613976, -0.51722556, -0.22581989, 0.036452737, 0.42190477, -0.256124, 0.25849363, -0.073515825, -0.08690646, 0.013338611, 0.14928678, 0.16371651, 0.111465625, -0.117571846, -0.44434816, 0.07603647, 0.4188736, -0.16967061, 0.040313836, 0.41924894, -0.36998197, 0.23204626, -0.23309743, -0.18061559, 1.0674918, -0.51468146, -0.37230963, 0.02214618, -0.5616187, -0.07473461, -0.3314831, -0.24691144, -0.34061712, -0.1511554, 0.33711013, 0.1055847, -0.047220375, -0.06317049, -0.22572862, -0.21646689, 0.090705894, 0.018562902, 0.020744732, -0.5342965, -0.23402104, -0.17680043, 0.1363935, -0.17916845, 0.37825805, -0.07233101, -0.28244817, 0.4055966, 0.19701958, 0.6236174, 0.078134544, 0.46439224, -0.60451704, 0.16722181, -0.20011653, 0.36931068, -0.39967215, 0.21178648, 0.47920865, -0.033521328, 0.57077545, -0.8003054, -0.4028354, 0.27799648, -0.23070334, 0.57747835, 0.49984616, -0.12409506, -0.26694623, -0.20168623, -0.19415514, -0.4626071, 0.10374411, 0.24878122, 0.47352287, -0.6494472, -0.26087105, 0.418008, -0.2789803, -0.60986733, -0.54914564, 0.4734504, 0.04347568, -0.13549352, 0.1530609, 0.085039385, -0.014595425, -0.1106091, 0.014441653, 0.14899726, -0.107090004, 0.03979257, 0.20897605, -0.040235993, 0.1928343, -0.048328623, 0.5435577, -0.1704212, -0.016530415, 0.11402996, 0.24666561, -0.62601864, 0.6729872, -0.21594357, -0.3161654, 0.2899072, -0.05281632, 0.026857251, 0.13927892, 0.26362655, 0.37995058, -0.056429606, 0.27310744, -0.34237143, -0.6419976, -0.02513231, -0.18217334, 0.021232722, -0.35155025, 0.055071075, -0.22192729, 0.4597671, 0.09872845, -0.41803727, -0.08897542, -0.63276047, 0.38059604, 0.45347637, 0.52723855, 0.25096536, -0.3165448, 0.43803728, 0.02419832, 0.317004, -0.059602205, 0.15561013, 0.11867607, 0.7157601, 0.08024589, -0.013107148, 0.3127224, -0.08844044, 0.5374578, 0.39421698, -0.054171022, 0.0913302, -0.081881694, 0.24596375, -0.2841653, -0.5482517, -0.5673938, 0.05889957, -0.1146344, 0.39452744, -0.03414711, 0.32027423, 0.2599335, -0.31470263, 0.45967287, -0.5710101, -0.21222454, 0.38154987, -0.21218868, -0.4366558, 0.13715877, 0.23925674, 0.34832072, -0.03769251, 0.25530148, 0.10662722, -0.5269836, 0.32952255, 0.46165445, 0.3794754, -0.061259665, 0.02883365, -0.3199015, 0.40625557, -0.3794913, 0.42420092, 0.4631467, 0.54236996, 0.031472385, 0.2635622, -0.25566247, -0.040713936, 0.48734123, 0.2742017, -0.15524681, 0.025654443, 0.056942068, -0.48883253, 0.60433495, 0.03514151] - } - ) { - _additional { - vector - } - contents - } - } - } - ` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - notes := result.Get("Get", "RansomNote").AsSlice() - require.NotEmpty(t, notes) - require.Equal(t, 10, len(notes)) - }) - - t.Run("nearVector: with limited search - limit provided (with certainty)", func(t *testing.T) { - query := ` - { - Get { - RansomNote( - limit: 10 - nearVector: { - certainty: 0.1 - vector: [-0.07853702, -0.33730024, 0.62998116, 0.08703484, -0.0011832615, 0.23041481, -0.091878965, 0.1184808, 0.060692377, 0.1748896, 0.53659165, 0.12019079, 0.54373807, -0.43369776, 0.1843199, -0.19319294, 0.122559674, -0.09465141, -0.14069664, 0.031092037, -0.1763922, 0.0074394196, -0.2586067, 0.10990611, -0.18623954, -0.038631044, -0.22795723, 0.09296776, -0.31110525, -0.37963995, -0.19045947, 0.48089907, 0.46725857, 0.28541213, 0.08294283, -0.18865398, 0.09647029, 0.2321466, -0.03435125, -0.09602424, -0.3831683, -0.027315892, 0.4215511, -0.35865632, 0.41955224, 0.090477064, 0.29026023, -0.48485047, -0.24656451, -0.06478625, 0.07755969, -0.049564634, 0.026147474, -0.028342195, -0.035627227, 0.49309397, 0.3705331, 0.04615483, 0.14789912, -0.01220134, 0.300666, -0.246646, 0.0038986988, 0.16730541, 0.46581128, -0.04931062, 0.040290095, 0.32867354, -0.18300997, 0.30411696, -0.1969807, 0.4727539, -0.31915516, -0.32722718, 0.12694982, 0.22583495, -0.014532595, -0.14432396, 0.2469766, 0.14872919, -0.06750808, 0.06351308, -0.287756, -0.32118404, 0.25326216, 0.45288888, -0.36307186, 0.05369787, -0.3283361, 0.07754738, 0.38473788, -0.5120014, -0.3344492, -0.1102767, -0.16755687, -0.3575448, -0.2555015, -0.42976367, -0.2695758, 0.04068499, 0.591914, -0.008395256, 0.2613976, -0.51722556, -0.22581989, 0.036452737, 0.42190477, -0.256124, 0.25849363, -0.073515825, -0.08690646, 0.013338611, 0.14928678, 0.16371651, 0.111465625, -0.117571846, -0.44434816, 0.07603647, 0.4188736, -0.16967061, 0.040313836, 0.41924894, -0.36998197, 0.23204626, -0.23309743, -0.18061559, 1.0674918, -0.51468146, -0.37230963, 0.02214618, -0.5616187, -0.07473461, -0.3314831, -0.24691144, -0.34061712, -0.1511554, 0.33711013, 0.1055847, -0.047220375, -0.06317049, -0.22572862, -0.21646689, 0.090705894, 0.018562902, 0.020744732, -0.5342965, -0.23402104, -0.17680043, 0.1363935, -0.17916845, 0.37825805, -0.07233101, -0.28244817, 0.4055966, 0.19701958, 0.6236174, 0.078134544, 0.46439224, -0.60451704, 0.16722181, -0.20011653, 0.36931068, -0.39967215, 0.21178648, 0.47920865, -0.033521328, 0.57077545, -0.8003054, -0.4028354, 0.27799648, -0.23070334, 0.57747835, 0.49984616, -0.12409506, -0.26694623, -0.20168623, -0.19415514, -0.4626071, 0.10374411, 0.24878122, 0.47352287, -0.6494472, -0.26087105, 0.418008, -0.2789803, -0.60986733, -0.54914564, 0.4734504, 0.04347568, -0.13549352, 0.1530609, 0.085039385, -0.014595425, -0.1106091, 0.014441653, 0.14899726, -0.107090004, 0.03979257, 0.20897605, -0.040235993, 0.1928343, -0.048328623, 0.5435577, -0.1704212, -0.016530415, 0.11402996, 0.24666561, -0.62601864, 0.6729872, -0.21594357, -0.3161654, 0.2899072, -0.05281632, 0.026857251, 0.13927892, 0.26362655, 0.37995058, -0.056429606, 0.27310744, -0.34237143, -0.6419976, -0.02513231, -0.18217334, 0.021232722, -0.35155025, 0.055071075, -0.22192729, 0.4597671, 0.09872845, -0.41803727, -0.08897542, -0.63276047, 0.38059604, 0.45347637, 0.52723855, 0.25096536, -0.3165448, 0.43803728, 0.02419832, 0.317004, -0.059602205, 0.15561013, 0.11867607, 0.7157601, 0.08024589, -0.013107148, 0.3127224, -0.08844044, 0.5374578, 0.39421698, -0.054171022, 0.0913302, -0.081881694, 0.24596375, -0.2841653, -0.5482517, -0.5673938, 0.05889957, -0.1146344, 0.39452744, -0.03414711, 0.32027423, 0.2599335, -0.31470263, 0.45967287, -0.5710101, -0.21222454, 0.38154987, -0.21218868, -0.4366558, 0.13715877, 0.23925674, 0.34832072, -0.03769251, 0.25530148, 0.10662722, -0.5269836, 0.32952255, 0.46165445, 0.3794754, -0.061259665, 0.02883365, -0.3199015, 0.40625557, -0.3794913, 0.42420092, 0.4631467, 0.54236996, 0.031472385, 0.2635622, -0.25566247, -0.040713936, 0.48734123, 0.2742017, -0.15524681, 0.025654443, 0.056942068, -0.48883253, 0.60433495, 0.03514151] - } - ) { - _additional { - vector - } - contents - } - } - } - ` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - notes := result.Get("Get", "RansomNote").AsSlice() - require.NotEmpty(t, notes) - require.Equal(t, 10, len(notes)) - }) - - t.Run("nearVector: results limited by distance", func(t *testing.T) { - query := ` - { - Get { - RansomNote( - nearVector: { - distance: 0.01 - vector: [-0.07853702, -0.33730024, 0.62998116, 0.08703484, -0.0011832615, 0.23041481, -0.091878965, 0.1184808, 0.060692377, 0.1748896, 0.53659165, 0.12019079, 0.54373807, -0.43369776, 0.1843199, -0.19319294, 0.122559674, -0.09465141, -0.14069664, 0.031092037, -0.1763922, 0.0074394196, -0.2586067, 0.10990611, -0.18623954, -0.038631044, -0.22795723, 0.09296776, -0.31110525, -0.37963995, -0.19045947, 0.48089907, 0.46725857, 0.28541213, 0.08294283, -0.18865398, 0.09647029, 0.2321466, -0.03435125, -0.09602424, -0.3831683, -0.027315892, 0.4215511, -0.35865632, 0.41955224, 0.090477064, 0.29026023, -0.48485047, -0.24656451, -0.06478625, 0.07755969, -0.049564634, 0.026147474, -0.028342195, -0.035627227, 0.49309397, 0.3705331, 0.04615483, 0.14789912, -0.01220134, 0.300666, -0.246646, 0.0038986988, 0.16730541, 0.46581128, -0.04931062, 0.040290095, 0.32867354, -0.18300997, 0.30411696, -0.1969807, 0.4727539, -0.31915516, -0.32722718, 0.12694982, 0.22583495, -0.014532595, -0.14432396, 0.2469766, 0.14872919, -0.06750808, 0.06351308, -0.287756, -0.32118404, 0.25326216, 0.45288888, -0.36307186, 0.05369787, -0.3283361, 0.07754738, 0.38473788, -0.5120014, -0.3344492, -0.1102767, -0.16755687, -0.3575448, -0.2555015, -0.42976367, -0.2695758, 0.04068499, 0.591914, -0.008395256, 0.2613976, -0.51722556, -0.22581989, 0.036452737, 0.42190477, -0.256124, 0.25849363, -0.073515825, -0.08690646, 0.013338611, 0.14928678, 0.16371651, 0.111465625, -0.117571846, -0.44434816, 0.07603647, 0.4188736, -0.16967061, 0.040313836, 0.41924894, -0.36998197, 0.23204626, -0.23309743, -0.18061559, 1.0674918, -0.51468146, -0.37230963, 0.02214618, -0.5616187, -0.07473461, -0.3314831, -0.24691144, -0.34061712, -0.1511554, 0.33711013, 0.1055847, -0.047220375, -0.06317049, -0.22572862, -0.21646689, 0.090705894, 0.018562902, 0.020744732, -0.5342965, -0.23402104, -0.17680043, 0.1363935, -0.17916845, 0.37825805, -0.07233101, -0.28244817, 0.4055966, 0.19701958, 0.6236174, 0.078134544, 0.46439224, -0.60451704, 0.16722181, -0.20011653, 0.36931068, -0.39967215, 0.21178648, 0.47920865, -0.033521328, 0.57077545, -0.8003054, -0.4028354, 0.27799648, -0.23070334, 0.57747835, 0.49984616, -0.12409506, -0.26694623, -0.20168623, -0.19415514, -0.4626071, 0.10374411, 0.24878122, 0.47352287, -0.6494472, -0.26087105, 0.418008, -0.2789803, -0.60986733, -0.54914564, 0.4734504, 0.04347568, -0.13549352, 0.1530609, 0.085039385, -0.014595425, -0.1106091, 0.014441653, 0.14899726, -0.107090004, 0.03979257, 0.20897605, -0.040235993, 0.1928343, -0.048328623, 0.5435577, -0.1704212, -0.016530415, 0.11402996, 0.24666561, -0.62601864, 0.6729872, -0.21594357, -0.3161654, 0.2899072, -0.05281632, 0.026857251, 0.13927892, 0.26362655, 0.37995058, -0.056429606, 0.27310744, -0.34237143, -0.6419976, -0.02513231, -0.18217334, 0.021232722, -0.35155025, 0.055071075, -0.22192729, 0.4597671, 0.09872845, -0.41803727, -0.08897542, -0.63276047, 0.38059604, 0.45347637, 0.52723855, 0.25096536, -0.3165448, 0.43803728, 0.02419832, 0.317004, -0.059602205, 0.15561013, 0.11867607, 0.7157601, 0.08024589, -0.013107148, 0.3127224, -0.08844044, 0.5374578, 0.39421698, -0.054171022, 0.0913302, -0.081881694, 0.24596375, -0.2841653, -0.5482517, -0.5673938, 0.05889957, -0.1146344, 0.39452744, -0.03414711, 0.32027423, 0.2599335, -0.31470263, 0.45967287, -0.5710101, -0.21222454, 0.38154987, -0.21218868, -0.4366558, 0.13715877, 0.23925674, 0.34832072, -0.03769251, 0.25530148, 0.10662722, -0.5269836, 0.32952255, 0.46165445, 0.3794754, -0.061259665, 0.02883365, -0.3199015, 0.40625557, -0.3794913, 0.42420092, 0.4631467, 0.54236996, 0.031472385, 0.2635622, -0.25566247, -0.040713936, 0.48734123, 0.2742017, -0.15524681, 0.025654443, 0.056942068, -0.48883253, 0.60433495, 0.03514151] - } - ) { - _additional { - id - } - contents - } - } - } - ` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - notes := result.Get("Get", "RansomNote").AsSlice() - require.NotEmpty(t, notes) - require.Less(t, len(notes), defaultLimit) - }) - - t.Run("nearVector: results limited by certainty", func(t *testing.T) { - query := ` - { - Get { - RansomNote( - nearVector: { - certainty: 0.99 - vector: [-0.07853702, -0.33730024, 0.62998116, 0.08703484, -0.0011832615, 0.23041481, -0.091878965, 0.1184808, 0.060692377, 0.1748896, 0.53659165, 0.12019079, 0.54373807, -0.43369776, 0.1843199, -0.19319294, 0.122559674, -0.09465141, -0.14069664, 0.031092037, -0.1763922, 0.0074394196, -0.2586067, 0.10990611, -0.18623954, -0.038631044, -0.22795723, 0.09296776, -0.31110525, -0.37963995, -0.19045947, 0.48089907, 0.46725857, 0.28541213, 0.08294283, -0.18865398, 0.09647029, 0.2321466, -0.03435125, -0.09602424, -0.3831683, -0.027315892, 0.4215511, -0.35865632, 0.41955224, 0.090477064, 0.29026023, -0.48485047, -0.24656451, -0.06478625, 0.07755969, -0.049564634, 0.026147474, -0.028342195, -0.035627227, 0.49309397, 0.3705331, 0.04615483, 0.14789912, -0.01220134, 0.300666, -0.246646, 0.0038986988, 0.16730541, 0.46581128, -0.04931062, 0.040290095, 0.32867354, -0.18300997, 0.30411696, -0.1969807, 0.4727539, -0.31915516, -0.32722718, 0.12694982, 0.22583495, -0.014532595, -0.14432396, 0.2469766, 0.14872919, -0.06750808, 0.06351308, -0.287756, -0.32118404, 0.25326216, 0.45288888, -0.36307186, 0.05369787, -0.3283361, 0.07754738, 0.38473788, -0.5120014, -0.3344492, -0.1102767, -0.16755687, -0.3575448, -0.2555015, -0.42976367, -0.2695758, 0.04068499, 0.591914, -0.008395256, 0.2613976, -0.51722556, -0.22581989, 0.036452737, 0.42190477, -0.256124, 0.25849363, -0.073515825, -0.08690646, 0.013338611, 0.14928678, 0.16371651, 0.111465625, -0.117571846, -0.44434816, 0.07603647, 0.4188736, -0.16967061, 0.040313836, 0.41924894, -0.36998197, 0.23204626, -0.23309743, -0.18061559, 1.0674918, -0.51468146, -0.37230963, 0.02214618, -0.5616187, -0.07473461, -0.3314831, -0.24691144, -0.34061712, -0.1511554, 0.33711013, 0.1055847, -0.047220375, -0.06317049, -0.22572862, -0.21646689, 0.090705894, 0.018562902, 0.020744732, -0.5342965, -0.23402104, -0.17680043, 0.1363935, -0.17916845, 0.37825805, -0.07233101, -0.28244817, 0.4055966, 0.19701958, 0.6236174, 0.078134544, 0.46439224, -0.60451704, 0.16722181, -0.20011653, 0.36931068, -0.39967215, 0.21178648, 0.47920865, -0.033521328, 0.57077545, -0.8003054, -0.4028354, 0.27799648, -0.23070334, 0.57747835, 0.49984616, -0.12409506, -0.26694623, -0.20168623, -0.19415514, -0.4626071, 0.10374411, 0.24878122, 0.47352287, -0.6494472, -0.26087105, 0.418008, -0.2789803, -0.60986733, -0.54914564, 0.4734504, 0.04347568, -0.13549352, 0.1530609, 0.085039385, -0.014595425, -0.1106091, 0.014441653, 0.14899726, -0.107090004, 0.03979257, 0.20897605, -0.040235993, 0.1928343, -0.048328623, 0.5435577, -0.1704212, -0.016530415, 0.11402996, 0.24666561, -0.62601864, 0.6729872, -0.21594357, -0.3161654, 0.2899072, -0.05281632, 0.026857251, 0.13927892, 0.26362655, 0.37995058, -0.056429606, 0.27310744, -0.34237143, -0.6419976, -0.02513231, -0.18217334, 0.021232722, -0.35155025, 0.055071075, -0.22192729, 0.4597671, 0.09872845, -0.41803727, -0.08897542, -0.63276047, 0.38059604, 0.45347637, 0.52723855, 0.25096536, -0.3165448, 0.43803728, 0.02419832, 0.317004, -0.059602205, 0.15561013, 0.11867607, 0.7157601, 0.08024589, -0.013107148, 0.3127224, -0.08844044, 0.5374578, 0.39421698, -0.054171022, 0.0913302, -0.081881694, 0.24596375, -0.2841653, -0.5482517, -0.5673938, 0.05889957, -0.1146344, 0.39452744, -0.03414711, 0.32027423, 0.2599335, -0.31470263, 0.45967287, -0.5710101, -0.21222454, 0.38154987, -0.21218868, -0.4366558, 0.13715877, 0.23925674, 0.34832072, -0.03769251, 0.25530148, 0.10662722, -0.5269836, 0.32952255, 0.46165445, 0.3794754, -0.061259665, 0.02883365, -0.3199015, 0.40625557, -0.3794913, 0.42420092, 0.4631467, 0.54236996, 0.031472385, 0.2635622, -0.25566247, -0.040713936, 0.48734123, 0.2742017, -0.15524681, 0.025654443, 0.056942068, -0.48883253, 0.60433495, 0.03514151] - } - ) { - _additional { - id - } - contents - } - } - } - ` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - notes := result.Get("Get", "RansomNote").AsSlice() - require.NotEmpty(t, notes) - require.Less(t, len(notes), defaultLimit) - }) - - // nearObject - - nearObjID := getOneExistingID(t, "RansomNote") - - t.Run("nearObject: with implicit unlimited search - no limit provided (with distance)", func(t *testing.T) { - query := fmt.Sprintf(` - { - Get { - RansomNote( - nearObject: { - distance: 1.8 - id: "%s" - } - ) { - contents - } - } - } - `, nearObjID) - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - notes := result.Get("Get", "RansomNote").AsSlice() - require.NotEmpty(t, notes) - require.Greater(t, len(notes), defaultLimit) - }) - - t.Run("nearObject: with implicit unlimited search - no limit provided (with certainty)", func(t *testing.T) { - query := fmt.Sprintf(` - { - Get { - RansomNote( - nearObject: { - certainty: 0.1 - id: "%s" - } - ) { - contents - } - } - } - `, nearObjID) - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - notes := result.Get("Get", "RansomNote").AsSlice() - require.NotEmpty(t, notes) - require.Greater(t, len(notes), defaultLimit) - }) - - t.Run("nearObject: with implicit unlimited search - negative limit provided (with distance)", func(t *testing.T) { - query := fmt.Sprintf(` - { - Get { - RansomNote( - limit: -1 - nearObject: { - distance: 0.9 - id: "%s" - } - ) { - contents - } - } - } - `, nearObjID) - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - notes := result.Get("Get", "RansomNote").AsSlice() - require.NotEmpty(t, notes) - require.Greater(t, len(notes), defaultLimit) - }) - - t.Run("nearObject: with implicit unlimited search - negative limit provided (with certainty)", func(t *testing.T) { - query := fmt.Sprintf(` - { - Get { - RansomNote( - limit: -1 - nearObject: { - certainty: 0.1 - id: "%s" - } - ) { - contents - } - } - } - `, nearObjID) - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - notes := result.Get("Get", "RansomNote").AsSlice() - require.NotEmpty(t, notes) - require.Greater(t, len(notes), defaultLimit) - }) - - t.Run("nearObject: with limited search - limit provided (with distance)", func(t *testing.T) { - query := fmt.Sprintf(` - { - Get { - RansomNote( - limit: 13 - nearObject: { - distance: 0.9 - id: "%s" - } - ) { - contents - } - } - } - `, nearObjID) - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - notes := result.Get("Get", "RansomNote").AsSlice() - require.NotEmpty(t, notes) - require.Equal(t, 13, len(notes)) - }) - - t.Run("nearObject: with limited search - limit provided (with certainty)", func(t *testing.T) { - query := fmt.Sprintf(` - { - Get { - RansomNote( - limit: 13 - nearObject: { - certainty: 0.1 - id: "%s" - } - ) { - contents - } - } - } - `, nearObjID) - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - notes := result.Get("Get", "RansomNote").AsSlice() - require.NotEmpty(t, notes) - require.Equal(t, 13, len(notes)) - }) - - t.Run("nearObject: results limited by distance", func(t *testing.T) { - query := fmt.Sprintf(` - { - Get { - RansomNote( - nearObject: { - distance: 0.01 - id: "%s" - } - ) { - _additional { - id - } - contents - } - } - } - `, nearObjID) - - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - notes := result.Get("Get", "RansomNote").AsSlice() - require.NotEmpty(t, notes) - require.Less(t, len(notes), defaultLimit) - }) - - t.Run("nearObject: results limited by certainty", func(t *testing.T) { - query := fmt.Sprintf(` - { - Get { - RansomNote( - nearObject: { - certainty: 0.99 - id: "%s" - } - ) { - _additional { - id - } - contents - } - } - } - `, nearObjID) - - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - notes := result.Get("Get", "RansomNote").AsSlice() - require.NotEmpty(t, notes) - require.Less(t, len(notes), defaultLimit) - }) - - // nearText - - t.Run("nearText: with implicit unlimited search - no limit provided (with distance)", func(t *testing.T) { - query := ` - { - Get { - RansomNote( - nearText: { - distance: 1.8 - concepts: ["abcd"] - } - ) { - contents - } - } - } - ` - - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - notes := result.Get("Get", "RansomNote").AsSlice() - require.NotEmpty(t, notes) - require.Greater(t, len(notes), defaultLimit) - }) - - t.Run("nearText: with implicit unlimited search - no limit provided (with certainty)", func(t *testing.T) { - query := ` - { - Get { - RansomNote( - nearText: { - certainty: 0.1 - concepts: ["abcd"] - } - ) { - contents - } - } - } - ` - - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - notes := result.Get("Get", "RansomNote").AsSlice() - require.NotEmpty(t, notes) - require.Greater(t, len(notes), defaultLimit) - }) - - t.Run("nearText: with implicit unlimited search - negative limit provided (with distance)", func(t *testing.T) { - query := ` - { - Get { - RansomNote( - limit: -1 - nearText: { - distance: 1.8 - concepts: ["abcd"] - } - ) { - contents - } - } - } - ` - - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - notes := result.Get("Get", "RansomNote").AsSlice() - require.NotEmpty(t, notes) - require.Greater(t, len(notes), defaultLimit) - }) - - t.Run("nearText: with implicit unlimited search - negative limit provided (with certainty)", func(t *testing.T) { - query := ` - { - Get { - RansomNote( - limit: -1 - nearText: { - certainty: 0.1 - concepts: ["abcd"] - } - ) { - contents - } - } - } - ` - - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - notes := result.Get("Get", "RansomNote").AsSlice() - require.NotEmpty(t, notes) - require.Greater(t, len(notes), defaultLimit) - }) - - t.Run("nearText: with limited search - limit provided (with distance)", func(t *testing.T) { - query := ` - { - Get { - RansomNote( - limit: 5 - nearText: { - distance: 0.9 - concepts: ["abcd"] - } - ) { - contents - } - } - } - ` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - notes := result.Get("Get", "RansomNote").AsSlice() - require.NotEmpty(t, notes) - require.Equal(t, 5, len(notes)) - }) - - t.Run("nearText: with limited search - limit provided (with certainty)", func(t *testing.T) { - query := ` - { - Get { - RansomNote( - limit: 5 - nearText: { - certainty: 0.1 - concepts: ["abcd"] - } - ) { - contents - } - } - } - ` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - notes := result.Get("Get", "RansomNote").AsSlice() - require.NotEmpty(t, notes) - require.Equal(t, 5, len(notes)) - }) - - t.Run("nearText: results limited by distance", func(t *testing.T) { - query := ` - { - Get { - RansomNote( - nearText: { - distance: 0.2 - concepts: ["abcd"] - } - ) { - _additional { - id - } - contents - } - } - } - ` - - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - notes := result.Get("Get", "RansomNote").AsSlice() - require.Less(t, len(notes), defaultLimit) - }) - - t.Run("nearText: results limited by certainty", func(t *testing.T) { - query := ` - { - Get { - RansomNote( - nearText: { - certainty: 0.9 - concepts: ["abcd"] - } - ) { - _additional { - id - } - contents - } - } - } - ` - - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - notes := result.Get("Get", "RansomNote").AsSlice() - require.Less(t, len(notes), defaultLimit) - }) -} - -func gettingObjectsWithNearFieldsMultiShard(t *testing.T) { - t.Run("nearText: results limited by distance with multi shard", func(t *testing.T) { - query := ` - { - Get { - MultiShard( - nearText: { - distance: 0.9 - concepts: ["multi shard"] - } - ) { - _additional { - id - } - } - } - } - ` - - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - notes := result.Get("Get", "MultiShard").AsSlice() - require.Equal(t, len(notes), 3) - }) - - t.Run("nearText: results limited by certainty with multi shard", func(t *testing.T) { - query := ` - { - Get { - MultiShard( - nearText: { - certainty: 0.1 - concepts: ["multi shard"] - } - ) { - _additional { - id - } - } - } - } - ` - - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - notes := result.Get("Get", "MultiShard").AsSlice() - require.Equal(t, len(notes), 3) - }) -} - -func getOneExistingID(t *testing.T, className string) string { - query := fmt.Sprintf("{Get {%s(limit: 1) {_additional {id}}}}", className) - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query).Result.(map[string]interface{}) - - classes, ok := result["Get"].(map[string]interface{}) - if !ok { - t.Fatalf("Get response is not a map[string]interface{}") - } - - class, ok := classes[className] - if !ok { - t.Fatalf("class %s was not included in response", className) - } - - classSlice, ok := class.([]interface{}) - if !ok { - t.Fatalf("%s response is not a []interface{}", className) - } - require.NotEmpty(t, class, "getOneExistingID empty class response for %s", className) - - firstItem, ok := classSlice[0].(map[string]interface{}) - if !ok { - t.Fatalf("first item is not a map[string]interface{}") - } - - additional, ok := firstItem["_additional"].(map[string]interface{}) - if !ok { - t.Fatalf("_additional result is not a map[string]interface{}") - } - - id, ok := additional["id"].(string) - if !ok { - t.Fatalf("id is not a string") - } - - return id -} diff --git a/test/acceptance/graphql_resolvers/metrics_stability_test.go b/test/acceptance/graphql_resolvers/metrics_stability_test.go deleted file mode 100644 index 88bec031f797d5bf5f4ae589ab974ca7db0e3679..0000000000000000000000000000000000000000 --- a/test/acceptance/graphql_resolvers/metrics_stability_test.go +++ /dev/null @@ -1,201 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "bufio" - "context" - "fmt" - "math" - "math/rand" - "net/http" - "strings" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/client/objects" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/test/helper" - graphqlhelper "github.com/weaviate/weaviate/test/helper/graphql" -) - -const metricClassPrefix = "MetricsClassPrefix" - -func metricsCount(t *testing.T) { - defer cleanupMetricsClasses(t, 0, 20) - createImportQueryMetricsClasses(t, 0, 10) - metricsLinesBefore := countMetricsLines(t) - createImportQueryMetricsClasses(t, 10, 20) - metricsLinesAfter := countMetricsLines(t) - assert.Equal(t, metricsLinesBefore, metricsLinesAfter, "number of metrics should not have changed") -} - -func createImportQueryMetricsClasses(t *testing.T, start, end int) { - for i := start; i < end; i++ { - createMetricsClass(t, i) - importMetricsClass(t, i) - queryMetricsClass(t, i) - } -} - -func createMetricsClass(t *testing.T, classIndex int) { - createObjectClass(t, &models.Class{ - Class: metricsClassName(classIndex), - Vectorizer: "none", - Properties: []*models.Property{ - { - Name: "some_text", - DataType: schema.DataTypeText.PropString(), - }, - }, - VectorIndexConfig: map[string]any{ - "efConstruction": 10, - "maxConnextions": 2, - "ef": 10, - }, - }) -} - -func queryMetricsClass(t *testing.T, classIndex int) { - // object by ID which exists - resp, err := helper.Client(t).Objects. - ObjectsClassGet( - objects.NewObjectsClassGetParams(). - WithID(helper.IntToUUID(1)). - WithClassName(metricsClassName(classIndex)), - nil) - - require.Nil(t, err) - assert.NotNil(t, resp.Payload) - - // object by ID which doesn't exist - // ignore any return values - helper.Client(t).Objects. - ObjectsClassGet( - objects.NewObjectsClassGetParams(). - WithID(helper.IntToUUID(math.MaxUint64)). - WithClassName(metricsClassName(classIndex)), - nil) - - // vector search - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, - fmt.Sprintf( - "{ Get { %s(nearVector:{vector: [0.3,0.3,0.7,0.7]}, limit:5) { some_text } } }", - metricsClassName(classIndex), - ), - ) - objs := result.Get("Get", metricsClassName(classIndex)).AsSlice() - assert.Len(t, objs, 5) - - // filtered vector search (which has specific metrics) - // vector search - result = graphqlhelper.AssertGraphQL(t, helper.RootAuth, - fmt.Sprintf( - "{ Get { %s(nearVector:{vector:[0.3,0.3,0.7,0.7]}, limit:5, where: %s) { some_text } } }", - metricsClassName(classIndex), - `{operator:Equal, valueText: "individually", path:["some_text"]}`, - ), - ) - objs = result.Get("Get", metricsClassName(classIndex)).AsSlice() - assert.Len(t, objs, 1) -} - -// make sure that we use both individual as well as batch imports, as they -// might produce different metrics -func importMetricsClass(t *testing.T, classIndex int) { - // individual - createObject(t, &models.Object{ - Class: metricsClassName(classIndex), - Properties: map[string]interface{}{ - "some_text": "this object was created individually", - }, - ID: helper.IntToUUID(1), - Vector: randomVector(4), - }) - - // with batches - const ( - batchSize = 100 - numBatches = 50 - ) - - for i := 0; i < numBatches; i++ { - batch := make([]*models.Object, batchSize) - for j := 0; j < batchSize; j++ { - batch[j] = &models.Object{ - Class: metricsClassName(classIndex), - Properties: map[string]interface{}{ - "some_text": fmt.Sprintf("this is object %d of batch %d", j, i), - }, - Vector: randomVector(4), - } - } - - createObjectsBatch(t, batch) - } -} - -func cleanupMetricsClasses(t *testing.T, start, end int) { - for i := start; i < end; i++ { - deleteObjectClass(t, metricsClassName(i)) - } -} - -func randomVector(dims int) []float32 { - out := make([]float32, dims) - for i := range out { - out[i] = rand.Float32() - } - return out -} - -func countMetricsLines(t *testing.T) int { - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - - req, err := http.NewRequestWithContext(ctx, http.MethodGet, - "http://localhost:2112/metrics", nil) - require.Nil(t, err) - - c := &http.Client{} - res, err := c.Do(req) - require.Nil(t, err) - - defer res.Body.Close() - require.Equal(t, http.StatusOK, res.StatusCode) - - scanner := bufio.NewScanner(res.Body) - lineCount := 0 - for scanner.Scan() { - line := scanner.Text() - if strings.Contains(line, "shards_loaded") || strings.Contains(line, "shards_loading") || strings.Contains(line, "shards_unloading") || strings.Contains(line, "shards_unloaded") { - continue - } - require.NotContains( - t, - strings.ToLower(line), - strings.ToLower(metricClassPrefix), - ) - lineCount++ - } - - require.Nil(t, scanner.Err()) - - return lineCount -} - -func metricsClassName(classIndex int) string { - return fmt.Sprintf("%s_%d", metricClassPrefix, classIndex) -} diff --git a/test/acceptance/graphql_resolvers/multi_reftype_bug_test.go b/test/acceptance/graphql_resolvers/multi_reftype_bug_test.go deleted file mode 100644 index b16018ab1fee6945ff873f075e99bcdd79137d96..0000000000000000000000000000000000000000 --- a/test/acceptance/graphql_resolvers/multi_reftype_bug_test.go +++ /dev/null @@ -1,501 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "fmt" - "testing" - - graphqlhelper "github.com/weaviate/weaviate/test/helper/graphql" - - "github.com/go-openapi/strfmt" - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/test/helper" -) - -// This test prevents a regression on -// https://github.com/weaviate/weaviate/issues/1410 -func TestMultipleRefTypeIssues(t *testing.T) { - className := func(suffix string) string { - return "MultiRefTypeBug" + suffix - } - defer deleteObjectClass(t, className("TargetOne")) - defer deleteObjectClass(t, className("TargetTwo")) - defer deleteObjectClass(t, className("Source")) - - const ( - targetOneID strfmt.UUID = "155c5914-6594-4cde-b3ab-f8570b561965" - targetTwoID strfmt.UUID = "ebf85a07-6b34-4e3b-b7c5-077f904fc955" - ) - - t.Run("import schema", func(t *testing.T) { - createObjectClass(t, &models.Class{ - Class: className("TargetOne"), - Properties: []*models.Property{ - { - Name: "name", - DataType: []string{"text"}, - }, - }, - }) - - createObjectClass(t, &models.Class{ - Class: className("TargetTwo"), - Properties: []*models.Property{ - { - Name: "name", - DataType: []string{"text"}, - }, - }, - }) - - createObjectClass(t, &models.Class{ - Class: className("Source"), - Properties: []*models.Property{ - { - Name: "name", - DataType: []string{"text"}, - }, - { - Name: "toTargets", - DataType: []string{className("TargetOne"), className("TargetTwo")}, - }, - }, - }) - }) - - t.Run("import data", func(t *testing.T) { - createObject(t, &models.Object{ - Class: className("TargetOne"), - ID: targetOneID, - Properties: map[string]interface{}{ - "name": "target a", - }, - }) - - createObject(t, &models.Object{ - Class: className("TargetTwo"), - ID: targetTwoID, - Properties: map[string]interface{}{ - "name": "target b", - }, - }) - - createObject(t, &models.Object{ - Class: className("Source"), - Properties: map[string]interface{}{ - "name": "source without refs", - }, - }) - - createObject(t, &models.Object{ - Class: className("Source"), - Properties: map[string]interface{}{ - "name": "source with ref to One", - "toTargets": []interface{}{ - map[string]interface{}{ - "beacon": fmt.Sprintf("weaviate://localhost/%s", targetOneID), - }, - }, - }, - }) - - createObject(t, &models.Object{ - Class: className("Source"), - Properties: map[string]interface{}{ - "name": "source with ref to Two", - "toTargets": []interface{}{ - map[string]interface{}{ - "beacon": fmt.Sprintf("weaviate://localhost/%s", targetTwoID), - }, - }, - }, - }) - - createObject(t, &models.Object{ - Class: className("Source"), - Properties: map[string]interface{}{ - "name": "source with ref to both", - "toTargets": []interface{}{ - map[string]interface{}{ - "beacon": fmt.Sprintf("weaviate://localhost/%s", targetOneID), - }, - map[string]interface{}{ - "beacon": fmt.Sprintf("weaviate://localhost/%s", targetTwoID), - }, - }, - }, - }) - }) - - t.Run("verify different scenarios through GraphQL", func(t *testing.T) { - t.Run("requesting no references", func(t *testing.T) { - query := fmt.Sprintf(` - { - Get { - %s { - name - } - } - } - `, className("Source")) - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - actual := result.Get("Get", className("Source")).AsSlice() - expected := []interface{}{ - map[string]interface{}{"name": "source with ref to One"}, - map[string]interface{}{"name": "source with ref to Two"}, - map[string]interface{}{"name": "source with ref to both"}, - map[string]interface{}{"name": "source without refs"}, - } - - assert.ElementsMatch(t, expected, actual) - }) - - t.Run("requesting references of type One without additional { id }", func(t *testing.T) { - query := fmt.Sprintf(` - { - Get { - %s { - name - toTargets { - ... on %s { - name - } - } - } - } - } - `, className("Source"), className("TargetOne")) - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - actual := result.Get("Get", className("Source")).AsSlice() - expected := []interface{}{ - map[string]interface{}{ - "name": "source with ref to One", - "toTargets": []interface{}{ - map[string]interface{}{ - "name": "target a", - }, - }, - }, - map[string]interface{}{ - "name": "source with ref to Two", - "toTargets": nil, - }, - map[string]interface{}{ - "name": "source with ref to both", - "toTargets": []interface{}{ - map[string]interface{}{ - "name": "target a", - }, - }, - }, - map[string]interface{}{ - "name": "source without refs", - "toTargets": nil, - }, - } - - assert.ElementsMatch(t, expected, actual) - }) - - t.Run("requesting references of type One with additional { id }", func(t *testing.T) { - query := fmt.Sprintf(` - { - Get { - %s { - name - toTargets { - ... on %s { - name - _additional { id } - } - } - } - } - } - `, className("Source"), className("TargetOne")) - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - actual := result.Get("Get", className("Source")).AsSlice() - expected := []interface{}{ - map[string]interface{}{ - "name": "source with ref to One", - "toTargets": []interface{}{ - map[string]interface{}{ - "name": "target a", - "_additional": map[string]interface{}{ - "id": targetOneID.String(), - }, - }, - }, - }, - map[string]interface{}{ - "name": "source with ref to Two", - "toTargets": nil, - }, - map[string]interface{}{ - "name": "source with ref to both", - "toTargets": []interface{}{ - map[string]interface{}{ - "name": "target a", - "_additional": map[string]interface{}{ - "id": targetOneID.String(), - }, - }, - }, - }, - map[string]interface{}{ - "name": "source without refs", - "toTargets": nil, - }, - } - - assert.ElementsMatch(t, expected, actual) - }) - - t.Run("requesting references of type Two without additional { id }", func(t *testing.T) { - query := fmt.Sprintf(` - { - Get { - %s { - name - toTargets { - ... on %s { - name - } - } - } - } - } - `, className("Source"), className("TargetTwo")) - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - actual := result.Get("Get", className("Source")).AsSlice() - expected := []interface{}{ - map[string]interface{}{ - "name": "source with ref to Two", - "toTargets": []interface{}{ - map[string]interface{}{ - "name": "target b", - }, - }, - }, - map[string]interface{}{ - "name": "source with ref to One", - "toTargets": nil, - }, - map[string]interface{}{ - "name": "source with ref to both", - "toTargets": []interface{}{ - map[string]interface{}{ - "name": "target b", - }, - }, - }, - map[string]interface{}{ - "name": "source without refs", - "toTargets": nil, - }, - } - - assert.ElementsMatch(t, expected, actual) - }) - - t.Run("requesting references of type Two with additional { id }", func(t *testing.T) { - query := fmt.Sprintf(` - { - Get { - %s { - name - toTargets { - ... on %s { - name - _additional { id } - } - } - } - } - } - `, className("Source"), className("TargetTwo")) - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - actual := result.Get("Get", className("Source")).AsSlice() - expected := []interface{}{ - map[string]interface{}{ - "name": "source with ref to Two", - "toTargets": []interface{}{ - map[string]interface{}{ - "name": "target b", - "_additional": map[string]interface{}{ - "id": targetTwoID.String(), - }, - }, - }, - }, - map[string]interface{}{ - "name": "source with ref to One", - "toTargets": nil, - }, - map[string]interface{}{ - "name": "source with ref to both", - "toTargets": []interface{}{ - map[string]interface{}{ - "name": "target b", - "_additional": map[string]interface{}{ - "id": targetTwoID.String(), - }, - }, - }, - }, - map[string]interface{}{ - "name": "source without refs", - "toTargets": nil, - }, - } - - assert.ElementsMatch(t, expected, actual) - }) - - t.Run("requesting references of both types without additional { id }", - func(t *testing.T) { - query := fmt.Sprintf(` - { - Get { - %s { - name - toTargets { - ... on %s { - name - } - ... on %s { - name - } - } - } - } - } - `, className("Source"), className("TargetOne"), className("TargetTwo")) - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - actual := result.Get("Get", className("Source")).AsSlice() - expected := []interface{}{ - map[string]interface{}{ - "name": "source with ref to Two", - "toTargets": []interface{}{ - map[string]interface{}{ - "name": "target b", - }, - }, - }, - map[string]interface{}{ - "name": "source with ref to One", - "toTargets": []interface{}{ - map[string]interface{}{ - "name": "target a", - }, - }, - }, - map[string]interface{}{ - "name": "source with ref to both", - "toTargets": []interface{}{ - map[string]interface{}{ - "name": "target a", - }, - map[string]interface{}{ - "name": "target b", - }, - }, - }, - map[string]interface{}{ - "name": "source without refs", - "toTargets": nil, - }, - } - - assert.ElementsMatch(t, expected, actual) - }) - - t.Run("requesting references of type Two with additional { id }", func(t *testing.T) { - query := fmt.Sprintf(` - { - Get { - %s { - name - toTargets { - ... on %s { - name - _additional { id } - } - ... on %s { - name - _additional { id } - } - } - } - } - } - `, className("Source"), className("TargetOne"), className("TargetTwo")) - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - actual := result.Get("Get", className("Source")).AsSlice() - expected := []interface{}{ - map[string]interface{}{ - "name": "source with ref to Two", - "toTargets": []interface{}{ - map[string]interface{}{ - "name": "target b", - "_additional": map[string]interface{}{ - "id": targetTwoID.String(), - }, - }, - }, - }, - map[string]interface{}{ - "name": "source with ref to One", - "toTargets": []interface{}{ - map[string]interface{}{ - "name": "target a", - "_additional": map[string]interface{}{ - "id": targetOneID.String(), - }, - }, - }, - }, - map[string]interface{}{ - "name": "source with ref to both", - "toTargets": []interface{}{ - map[string]interface{}{ - "name": "target a", - "_additional": map[string]interface{}{ - "id": targetOneID.String(), - }, - }, - map[string]interface{}{ - "name": "target b", - "_additional": map[string]interface{}{ - "id": targetTwoID.String(), - }, - }, - }, - }, - map[string]interface{}{ - "name": "source without refs", - "toTargets": nil, - }, - } - - assert.ElementsMatch(t, expected, actual) - }) - }) - - t.Run("cleanup", func(t *testing.T) { - deleteObjectClass(t, className("Source")) - deleteObjectClass(t, className("TargetOne")) - deleteObjectClass(t, className("TargetTwo")) - }) -} diff --git a/test/acceptance/graphql_resolvers/network_aggregate_test.go b/test/acceptance/graphql_resolvers/network_aggregate_test.go deleted file mode 100644 index 54af240304b661512c857a00258f69feed00b176..0000000000000000000000000000000000000000 --- a/test/acceptance/graphql_resolvers/network_aggregate_test.go +++ /dev/null @@ -1,51 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -// import ( -// "encoding/json" -// "testing" - -// "github.com/weaviate/weaviate/test/helper" -// "github.com/stretchr/testify/assert" -// ) - -// func TestNetworkAggregate(t *testing.T) { -// result := AssertGraphQL(t, helper.RootAuth, ` -// { -// Network { -// Aggregate{ -// RemoteWeaviateForAcceptanceTest { -// Things { -// Instruments(groupBy:["name"]) { -// volume { -// count -// } -// } -// } -// } -// } -// } -// } -// `) - -// volume := result.Get("Network", "Aggregate", "RemoteWeaviateForAcceptanceTest", "Things", "Instruments").Result -// expected := []interface{}{ -// map[string]interface{}{ -// "volume": map[string]interface{}{ -// "count": json.Number("82"), -// }, -// }, -// } - -// assert.Equal(t, expected, volume) -// } diff --git a/test/acceptance/graphql_resolvers/network_fetch_test.go b/test/acceptance/graphql_resolvers/network_fetch_test.go deleted file mode 100644 index 09af561613144abdde9b3cc5eb2b120359e51825..0000000000000000000000000000000000000000 --- a/test/acceptance/graphql_resolvers/network_fetch_test.go +++ /dev/null @@ -1,75 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -// func Test_NetworkFetch(t *testing.T) { -// result := AssertGraphQL(t, helper.RootAuth, ` -// { -// Network { -// Fetch { -// Things(where: { -// class: { -// name: "bestclass" -// certainty: 0.8 -// keywords: [{value: "foo", weight: 0.9}] -// }, -// properties: { -// name: "bestproperty" -// certainty: 0.8 -// keywords: [{value: "bar", weight: 0.9}] -// operator: Equal -// valueString: "some-value" -// }, -// }) { -// beacon certainty -// } -// } -// } -// }`, -// ) - -// results := result.Get("Network", "Fetch", "Things").Result -// expected := []interface{}{ -// map[string]interface{}{ -// "beacon": "weaviate://RemoteWeaviateForAcceptanceTest/things/c2b94c9a-fea2-4f9a-ae40-6d63534633f7", -// "certainty": json.Number("0.5"), -// }, -// map[string]interface{}{ -// "beacon": "weaviate://RemoteWeaviateForAcceptanceTest/things/32fc9b12-00b8-46b2-962d-63c1f352e090", -// "certainty": json.Number("0.7"), -// }, -// } -// assert.Equal(t, expected, results) -// } - -// func Test_NetworkFetchFuzzy(t *testing.T) { -// result := AssertGraphQL(t, helper.RootAuth, ` -// { -// Network { -// Fetch { -// Fuzzy(value:"something", certainty: 0.5) { -// beacon certainty -// } -// } -// } -// }`, -// ) - -// results := result.Get("Network", "Fetch", "Fuzzy").Result -// expected := []interface{}{ -// map[string]interface{}{ -// "beacon": "weaviate://RemoteWeaviateForAcceptanceTest/things/61c21951-3460-4189-86ad-884a17b70c16", -// "certainty": json.Number("0.5"), -// }, -// } -// assert.Equal(t, expected, results) -// } diff --git a/test/acceptance/graphql_resolvers/network_get_meta_test.go b/test/acceptance/graphql_resolvers/network_get_meta_test.go deleted file mode 100644 index 6313e0848b49ac62d60c944f470d078e88d01a24..0000000000000000000000000000000000000000 --- a/test/acceptance/graphql_resolvers/network_get_meta_test.go +++ /dev/null @@ -1,50 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -// import ( -// "encoding/json" -// "testing" - -// "github.com/weaviate/weaviate/test/helper" -// "github.com/stretchr/testify/assert" -// ) - -// func TestNetworkMeta(t *testing.T) { -// result := AssertGraphQL(t, helper.RootAuth, ` -// { -// Network { -// Meta{ -// RemoteWeaviateForAcceptanceTest { -// Things { -// Instruments { -// volume { -// maximum -// minimum -// mean -// } -// } -// } -// } -// } -// } -// } -// `) - -// volume := result.Get("Network", "Meta", "RemoteWeaviateForAcceptanceTest", "Things", "Instruments", "volume").Result -// expected := map[string]interface{}{ -// "mean": json.Number("82"), -// "maximum": json.Number("110"), -// "minimum": json.Number("65"), -// } -// assert.Equal(t, expected, volume) -// } diff --git a/test/acceptance/graphql_resolvers/network_get_test.go b/test/acceptance/graphql_resolvers/network_get_test.go deleted file mode 100644 index d064016fbdc02bc3540c17df60580f45b672f27d..0000000000000000000000000000000000000000 --- a/test/acceptance/graphql_resolvers/network_get_test.go +++ /dev/null @@ -1,36 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -// import ( -// "testing" - -// "github.com/weaviate/weaviate/test/helper" -// "github.com/stretchr/testify/assert" -// ) - -// // Note: Things.Instruments is not something that is present in our local schema -// // This is on purpose to verify that we have support for a completely different -// // schema on a remote instance. -// func TestNetworkGetSimple(t *testing.T) { -// result := AssertGraphQL(t, helper.RootAuth, "{ Network { Get { RemoteWeaviateForAcceptanceTest { Things { Instruments { name } } } } }") -// instruments := result.Get("Network", "Get", "RemoteWeaviateForAcceptanceTest", "Things", "Instruments").AsSlice() - -// expected := []interface{}{ -// map[string]interface{}{"name": "Piano"}, -// map[string]interface{}{"name": "Guitar"}, -// map[string]interface{}{"name": "Bass Guitar"}, -// map[string]interface{}{"name": "Talkbox"}, -// } - -// assert.ElementsMatch(t, expected, instruments) -// } diff --git a/test/acceptance/graphql_resolvers/objects_helper.go b/test/acceptance/graphql_resolvers/objects_helper.go deleted file mode 100644 index d3733fdfe082fc1ada32e26cee1d90ec4ca2e114..0000000000000000000000000000000000000000 --- a/test/acceptance/graphql_resolvers/objects_helper.go +++ /dev/null @@ -1,35 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "testing" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/test/helper" -) - -func createObjectClass(t *testing.T, class *models.Class) { - helper.CreateClass(t, class) -} - -func createObject(t *testing.T, object *models.Object) { - helper.CreateObject(t, object) -} - -func createObjectsBatch(t *testing.T, objects []*models.Object) { - helper.CreateObjectsBatch(t, objects) -} - -func deleteObjectClass(t *testing.T, class string) { - helper.DeleteClass(t, class) -} diff --git a/test/acceptance/graphql_resolvers/setup_test.go b/test/acceptance/graphql_resolvers/setup_test.go deleted file mode 100644 index 41ad5073276b4ba4729c915fcdf0482191766ff5..0000000000000000000000000000000000000000 --- a/test/acceptance/graphql_resolvers/setup_test.go +++ /dev/null @@ -1,1126 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "context" - "fmt" - "math/rand" - "testing" - "time" - - "github.com/go-openapi/strfmt" - "github.com/google/uuid" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/schema/crossref" - "github.com/weaviate/weaviate/test/docker" - "github.com/weaviate/weaviate/test/helper" - "github.com/weaviate/weaviate/test/helper/sample-schema/multishard" -) - -func TestGraphQL_AsyncIndexing(t *testing.T) { - ctx := context.Background() - compose, err := docker.New(). - WithWeaviate(). - WithText2VecContextionary(). - WithWeaviateEnv("ASYNC_INDEXING", "true"). - Start(ctx) - require.NoError(t, err) - defer func() { - require.NoError(t, compose.Terminate(ctx)) - }() - - defer helper.SetupClient(fmt.Sprintf("%s:%s", helper.ServerHost, helper.ServerPort)) - helper.SetupClient(compose.GetWeaviate().URI()) - - testGraphQL(t) -} - -func TestGraphQL_SyncIndexing(t *testing.T) { - testGraphQL(t) -} - -func testGraphQL(t *testing.T) { - // tests with classes that have objects with same uuids - t.Run("import test data (near object search class)", addTestDataNearObjectSearch) - - t.Run("running Get nearObject against shadowed objects", runningGetNearObjectWithShadowedObjects) - t.Run("running Aggregate nearObject against shadowed objects", runningAggregateNearObjectWithShadowedObjects) - t.Run("running Explore nearObject against shadowed objects", runningExploreNearObjectWithShadowedObjects) - - deleteObjectClass(t, "NearObjectSearch") - deleteObjectClass(t, "NearObjectSearchShadow") - - // setup tests - t.Run("setup test schema", addTestSchema) - t.Run("import test data (city, country, airport)", addTestDataCityAirport) - t.Run("import test data (companies)", addTestDataCompanies) - t.Run("import test data (person)", addTestDataPersons) - t.Run("import test data (pizzas)", addTestDataPizzas) - t.Run("import test data (array class)", addTestDataArrayClass) - t.Run("import test data (duplicates class)", addTestDataDuplicatesClass) - t.Run("import test data (500 random strings)", addTestDataRansomNotes) - t.Run("import test data (multi shard)", addTestDataMultiShard) - t.Run("import test data (date field class)", addDateFieldClass) - t.Run("import test data (custom vector class)", addTestDataCVC) - t.Run("import test data (class without properties)", addTestDataNoProperties) - t.Run("import test data (cursor api)", addTestDataCursorSearch) - - // explore tests - t.Run("expected explore failures with invalid conditions", exploreWithExpectedFailures) - - // get tests - t.Run("getting objects", gettingObjects) - t.Run("getting objects with filters", gettingObjectsWithFilters) - t.Run("getting objects with geo filters", gettingObjectsWithGeoFilters) - t.Run("getting objects with grouping", gettingObjectsWithGrouping) - t.Run("getting objects with additional props", gettingObjectsWithAdditionalProps) - t.Run("getting objects with near fields", gettingObjectsWithNearFields) - t.Run("getting objects with near fields with multi shard setup", gettingObjectsWithNearFieldsMultiShard) - t.Run("getting objects with sort", gettingObjectsWithSort) - t.Run("getting objects with hybrid search", getWithHybridSearch) - t.Run("expected get failures with invalid conditions", getsWithExpectedFailures) - t.Run("cursor through results", getWithCursorSearch) - t.Run("groupBy objects", groupByObjects) - - // aggregate tests - t.Run("aggregates noPropsClass without grouping", aggregateNoPropsClassWithoutGroupByTest) - t.Run("aggregates arrayClass without grouping", aggregateArrayClassWithoutGroupByTest) - t.Run("aggregates arrayClass with grouping", aggregateArrayClassWithGroupByTest) - t.Run("aggregates duplicatesClass without grouping", aggregateDuplicatesClassWithoutGroupByTest) - t.Run("aggregates duplicatesClass with grouping", aggregateDuplicatesClassWithGroupByTest) - t.Run("aggregates city without grouping", aggregateCityClassWithoutGroupByTest) - t.Run("aggregates city with grouping", aggregateCityClassWithGroupByTest) - - t.Run("aggregates local meta string props not set everywhere", localMeta_StringPropsNotSetEverywhere) - t.Run("aggregates local meta with where and nearText filters", localMetaWithWhereAndNearTextFilters) - t.Run("aggregates local meta with where and nearObject filters", localMetaWithWhereAndNearObjectFilters) - t.Run("aggregates local meta with nearVector filters", localMetaWithNearVectorFilter) - t.Run("aggregates local meta with where and nearVector nearMedia", localMetaWithWhereAndNearVectorFilters) - t.Run("aggregates local meta with where groupBy and nearMedia filters", localMetaWithWhereGroupByNearMediaFilters) - t.Run("aggregates local meta with objectLimit and nearMedia filters", localMetaWithObjectLimit) - t.Run("aggregates on date fields", aggregatesOnDateFields) - t.Run("aggregates with hybrid search", aggregationWithHybridSearch) - t.Run("expected aggregate failures with invalid conditions", aggregatesWithExpectedFailures) - - t.Run("metrics count is stable when more classes are added", metricsCount) - - // tear down - deleteObjectClass(t, "Person") - deleteObjectClass(t, "Pizza") - deleteObjectClass(t, "Country") - deleteObjectClass(t, "City") - deleteObjectClass(t, "Airport") - deleteObjectClass(t, "Company") - deleteObjectClass(t, "RansomNote") - deleteObjectClass(t, "MultiShard") - deleteObjectClass(t, "HasDateField") - deleteObjectClass(t, arrayClassName) - deleteObjectClass(t, duplicatesClassName) - deleteObjectClass(t, noPropsClassName) - deleteObjectClass(t, "CursorClass") - - // only run after everything else is deleted, this way, we can also run an - // all-class Explore since all vectors which are now left have the same - // dimensions. - - t.Run("getting objects with custom vectors", gettingObjectsWithCustomVectors) - t.Run("explore objects with custom vectors", exploreObjectsWithCustomVectors) - - deleteObjectClass(t, "CustomVectorClass") -} - -func addTestSchema(t *testing.T) { - createObjectClass(t, &models.Class{ - Class: "Country", - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "vectorizeClassName": true, - }, - }, - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - }) - - // City class has only one vectorizable field: "name" - // the rest of the fields are explicitly set to skip vectorization - // to not to downgrade the distance/certainty result on which the - // aggregate tests are based on. - createObjectClass(t, &models.Class{ - Class: "City", - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "vectorizeClassName": true, - }, - }, - InvertedIndexConfig: &models.InvertedIndexConfig{IndexNullState: true, IndexPropertyLength: true, IndexTimestamps: true}, - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "skip": false, - }, - }, - }, - { - Name: "inCountry", - DataType: []string{"Country"}, - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "skip": true, - }, - }, - }, - { - Name: "population", - DataType: []string{"int"}, - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "skip": true, - }, - }, - }, - { - Name: "location", - DataType: []string{"geoCoordinates"}, - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "skip": true, - }, - }, - }, - { - Name: "isCapital", - DataType: []string{"boolean"}, - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "skip": true, - }, - }, - }, - { - Name: "cityArea", - DataType: []string{"number"}, - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "skip": true, - }, - }, - }, - { - Name: "cityRights", - DataType: []string{"date"}, - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "skip": true, - }, - }, - }, - { - Name: "timezones", - DataType: schema.DataTypeTextArray.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "skip": true, - }, - }, - }, - { - Name: "museums", - DataType: []string{"text[]"}, - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "skip": true, - }, - }, - }, - { - Name: "history", - DataType: []string{"text"}, - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "skip": true, - }, - }, - }, - { - Name: "phoneNumber", - DataType: []string{"phoneNumber"}, - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "skip": true, - }, - }, - }, - }, - }) - - createObjectClass(t, &models.Class{ - Class: "Airport", - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "vectorizeClassName": true, - }, - }, - InvertedIndexConfig: &models.InvertedIndexConfig{ - CleanupIntervalSeconds: 60, - Stopwords: &models.StopwordConfig{ - Preset: "en", - }, - IndexTimestamps: true, - }, - Properties: []*models.Property{ - { - Name: "code", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "phone", - DataType: []string{"phoneNumber"}, - }, - { - Name: "inCity", - DataType: []string{"City"}, - }, - { - Name: "airportId", - DataType: []string{"uuid"}, - }, - }, - }) - - createObjectClass(t, &models.Class{ - Class: "Company", - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "vectorizeClassName": false, - }, - }, - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "vectorizePropertyName": false, - }, - }, - }, - { - Name: "inCity", - DataType: []string{"City"}, - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "vectorizePropertyName": false, - }, - }, - }, - }, - }) - - createObjectClass(t, &models.Class{ - Class: "Person", - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "vectorizeClassName": false, - }, - }, - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "vectorizePropertyName": false, - }, - }, - }, - { - Name: "livesIn", - DataType: []string{"City"}, - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "vectorizePropertyName": false, - }, - }, - }, - { - Name: "profession", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationField, - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "vectorizePropertyName": false, - }, - }, - }, - { - Name: "about", - DataType: schema.DataTypeTextArray.PropString(), - Tokenization: models.PropertyTokenizationField, - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "vectorizePropertyName": false, - }, - }, - }, - }, - }) - - createObjectClass(t, &models.Class{ - Class: "Pizza", - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "vectorizeClassName": false, - }, - }, - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationField, - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "vectorizePropertyName": false, - }, - }, - }, - { - Name: "description", - DataType: []string{string(schema.DataTypeText)}, - Tokenization: models.PropertyTokenizationWord, - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "vectorizePropertyName": false, - }, - }, - }, - }, - }) - - createObjectClass(t, &models.Class{ - Class: "RansomNote", - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "vectorizeClassName": true, - }, - }, - Properties: []*models.Property{ - { - Name: "contents", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - }) - - createObjectClass(t, multishard.ClassContextionaryVectorizer()) - - createObjectClass(t, &models.Class{ - Class: "HasDateField", - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "vectorizeClassName": true, - }, - }, - Properties: []*models.Property{ - { - Name: "unique", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "timestamp", - DataType: []string{"date"}, - }, - { - Name: "identical", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - }) - - createObjectClass(t, &models.Class{ - Class: "CustomVectorClass", - Vectorizer: "none", - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - }) - - createObjectClass(t, noPropsClassSchema()) - createObjectClass(t, arrayClassSchema()) - createObjectClass(t, duplicatesClassSchema()) -} - -const ( - netherlands strfmt.UUID = "67b79643-cf8b-4b22-b206-6e63dbb4e57a" - germany strfmt.UUID = "561eea29-b733-4079-b50b-cfabd78190b7" - amsterdam strfmt.UUID = "8f5f8e44-d348-459c-88b1-c1a44bb8f8be" - rotterdam strfmt.UUID = "660db307-a163-41d2-8182-560782cd018f" - berlin strfmt.UUID = "9b9cbea5-e87e-4cd0-89af-e2f424fd52d6" - dusseldorf strfmt.UUID = "6ffb03f8-a853-4ec5-a5d8-302e45aaaf13" - missingisland strfmt.UUID = "823abeca-eef3-41c7-b587-7a6977b08003" - nullisland strfmt.UUID = "823abeca-eef3-41c7-b587-7a6977b08067" - airport1 strfmt.UUID = "4770bb19-20fd-406e-ac64-9dac54c27a0f" - airport2 strfmt.UUID = "cad6ab9b-5bb9-4388-a933-a5bdfd23db37" - airport3 strfmt.UUID = "55a4dbbb-e2af-4b2a-901d-98146d1eeca7" - airport4 strfmt.UUID = "62d15920-b546-4844-bc87-3ae33268fab5" - cvc1 strfmt.UUID = "1ffeb3e1-1258-4c2a-afc3-55543f6c44b8" - cvc2 strfmt.UUID = "df22e5c4-5d17-49f9-a71d-f392a82bc086" - cvc3 strfmt.UUID = "c28a039a-d509-4c2e-940a-8b109e5bebf4" - - quattroFormaggi strfmt.UUID = "152500c6-4a8a-4732-aede-9fcab7e43532" - fruttiDiMare strfmt.UUID = "a828e9aa-d1b6-4644-8569-30d404e31a0d" - hawaii strfmt.UUID = "ed75037b-0748-4970-811e-9fe835ed41d1" - doener strfmt.UUID = "a655292d-1b93-44a1-9a47-57b6922bb455" -) - -var ( - historyAmsterdam = "Due to its geographical location in what used to be wet peatland, the founding of Amsterdam is of a younger age than the founding of other urban centers in the Low Countries. However, in and around the area of what later became Amsterdam, local farmers settled as early as three millennia ago. They lived along the prehistoric IJ river and upstream of its tributary Amstel. The prehistoric IJ was a shallow and quiet stream in peatland behind beach ridges. This secluded area could grow there into an important local settlement center, especially in the late Bronze Age, the Iron Age and the Roman Age. Neolithic and Roman artefacts have also been found downstream of this area, in the prehistoric Amstel bedding under Amsterdam's Damrak and Rokin, such as shards of Bell Beaker culture pottery (2200-2000 BC) and a granite grinding stone (2700-2750 BC).[27][28] But the location of these artefacts around the river banks of the Amstel probably point to a presence of a modest semi-permanent or seasonal settlement of the previous mentioned local farmers. A permanent settlement would not have been possible, since the river mouth and the banks of the Amstel in this period in time were too wet for permanent habitation" - historyRotterdam = "On 7 July 1340, Count Willem IV of Holland granted city rights to Rotterdam, whose population then was only a few thousand.[14] Around the year 1350, a shipping canal (the Rotterdamse Schie) was completed, which provided Rotterdam access to the larger towns in the north, allowing it to become a local trans-shipment centre between the Netherlands, England and Germany, and to urbanize" - historyBerlin = "The earliest evidence of settlements in the area of today's Berlin are remnants of a house foundation dated to 1174, found in excavations in Berlin Mitte,[27] and a wooden beam dated from approximately 1192.[28] The first written records of towns in the area of present-day Berlin date from the late 12th century. Spandau is first mentioned in 1197 and Köpenick in 1209, although these areas did not join Berlin until 1920.[29] The central part of Berlin can be traced back to two towns. Cölln on the Fischerinsel is first mentioned in a 1237 document, and Berlin, across the Spree in what is now called the Nikolaiviertel, is referenced in a document from 1244.[28] 1237 is considered the founding date of the city.[30] The two towns over time formed close economic and social ties, and profited from the staple right on the two important trade routes Via Imperii and from Bruges to Novgorod.[12] In 1307, they formed an alliance with a common external policy, their internal administrations still being separated" - historyDusseldorf = "The first written mention of Düsseldorf (then called Dusseldorp in the local Low Rhenish dialect) dates back to 1135. Under Emperor Friedrich Barbarossa the small town of Kaiserswerth to the north of Düsseldorf became a well-fortified outpost, where soldiers kept a watchful eye on every movement on the Rhine. Kaiserswerth eventually became a suburb of Düsseldorf in 1929. In 1186, Düsseldorf came under the rule of the Counts of Berg. 14 August 1288 is one of the most important dates in the history of Düsseldorf. On this day the sovereign Count Adolf VIII of Berg granted the village on the banks of the Düssel town privileges. Before this, a bloody struggle for power had taken place between the Archbishop of Cologne and the count of Berg, culminating in the Battle of Worringen" -) - -func addTestDataCityAirport(t *testing.T) { - // countries - createObject(t, &models.Object{ - Class: "Country", - ID: netherlands, - Properties: map[string]interface{}{ - "name": "Netherlands", - }, - }) - createObject(t, &models.Object{ - Class: "Country", - ID: germany, - Properties: map[string]interface{}{ - "name": "Germany", - }, - }) - - // cities - createObject(t, &models.Object{ - Class: "City", - ID: amsterdam, - Properties: map[string]interface{}{ - "name": "Amsterdam", - "population": 1800000, - "location": map[string]interface{}{ - "latitude": 52.366667, - "longitude": 4.9, - }, - "inCountry": []interface{}{ - map[string]interface{}{ - "beacon": crossref.NewLocalhost("Country", netherlands).String(), - }, - }, - "isCapital": true, - "cityArea": float64(891.95), - "cityRights": mustParseYear("1400"), - "timezones": []string{"CET", "CEST"}, - "museums": []string{"Stedelijk Museum", "Rijksmuseum"}, - "history": historyAmsterdam, - "phoneNumber": map[string]interface{}{ - "input": "+311000004", - }, - }, - }) - createObject(t, &models.Object{ - Class: "City", - ID: rotterdam, - Properties: map[string]interface{}{ - "name": "Rotterdam", - "population": 600000, - "inCountry": []interface{}{ - map[string]interface{}{ - "beacon": crossref.NewLocalhost("Country", netherlands).String(), - }, - }, - "isCapital": false, - "cityArea": float64(319.35), - "cityRights": mustParseYear("1283"), - "timezones": []string{"CET", "CEST"}, - "museums": []string{"Museum Boijmans Van Beuningen", "Wereldmuseum", "Witte de With Center for Contemporary Art"}, - "history": historyRotterdam, - "phoneNumber": map[string]interface{}{ - "input": "+311000000", - }, - }, - }) - createObject(t, &models.Object{ - Class: "City", - ID: berlin, - Properties: map[string]interface{}{ - "name": "Berlin", - "population": 3470000, - "inCountry": []interface{}{ - map[string]interface{}{ - "beacon": crossref.NewLocalhost("Country", germany).String(), - }, - }, - "isCapital": true, - "cityArea": float64(891.96), - "cityRights": mustParseYear("1400"), - "timezones": []string{"CET", "CEST"}, - "museums": []string{"German Historical Museum"}, - "history": historyBerlin, - "phoneNumber": map[string]interface{}{ - "input": "+311000002", - }, - }, - }) - createObject(t, &models.Object{ - Class: "City", - ID: dusseldorf, - Properties: map[string]interface{}{ - "name": "Dusseldorf", - "population": 600000, - "inCountry": []interface{}{ - map[string]interface{}{ - "beacon": crossref.NewLocalhost("Country", germany).String(), - }, - }, - "location": map[string]interface{}{ - "latitude": 51.225556, - "longitude": 6.782778, - }, - "isCapital": false, - "cityArea": float64(217.22), - "cityRights": mustParseYear("1135"), - "timezones": []string{"CET", "CEST"}, - "museums": []string{"Schlossturm", "Schiffahrt Museum", "Onomato"}, - "history": historyDusseldorf, - "phoneNumber": map[string]interface{}{ - "input": "+311000001", - }, - }, - }) - - createObject(t, &models.Object{ - Class: "City", - ID: missingisland, - Properties: map[string]interface{}{ - "name": "Missing Island", - "population": 0, - "location": map[string]interface{}{ - "latitude": 0, - "longitude": 0, - }, - "isCapital": false, - }, - }) - - createObject(t, &models.Object{ - Class: "City", - ID: nullisland, - Properties: map[string]interface{}{ - "name": nil, - "population": nil, - "inCountry": nil, - "location": nil, - "isCapital": nil, - "cityArea": nil, - "cityRights": nil, - "timezones": nil, - "museums": nil, - "history": nil, - "phoneNumber": nil, - }, - }) - - // airports - createObject(t, &models.Object{ - Class: "Airport", - ID: airport1, - Properties: map[string]interface{}{ - "code": "10000", - "airportId": uuid.MustParse("00000000-0000-0000-0000-000000010000").String(), - "phone": map[string]interface{}{ - "input": "+311234567", - }, - "inCity": []interface{}{ - map[string]interface{}{ - "beacon": crossref.NewLocalhost("City", amsterdam).String(), - }, - }, - }, - }) - createObject(t, &models.Object{ - Class: "Airport", - ID: airport2, - Properties: map[string]interface{}{ - "code": "20000", - "airportId": uuid.MustParse("00000000-0000-0000-0000-000000020000").String(), - "inCity": []interface{}{ - map[string]interface{}{ - "beacon": crossref.NewLocalhost("City", rotterdam).String(), - }, - }, - }, - }) - createObject(t, &models.Object{ - Class: "Airport", - ID: airport3, - Properties: map[string]interface{}{ - "code": "30000", - "airportId": uuid.MustParse("00000000-0000-0000-0000-000000030000").String(), - "inCity": []interface{}{ - map[string]interface{}{ - "beacon": crossref.NewLocalhost("City", dusseldorf).String(), - }, - }, - }, - }) - createObject(t, &models.Object{ - Class: "Airport", - ID: airport4, - Properties: map[string]interface{}{ - "code": "40000", - "airportId": uuid.MustParse("00000000-0000-0000-0000-000000040000").String(), - "inCity": []interface{}{ - map[string]interface{}{ - "beacon": crossref.NewLocalhost("City", berlin).String(), - }, - }, - }, - }) -} - -func addTestDataCompanies(t *testing.T) { - var ( - microsoft1 strfmt.UUID = "cfa3b21e-ca4f-4db7-a432-7fc6a23c534d" - microsoft2 strfmt.UUID = "8f75ed97-39dd-4294-bff7-ecabd7923062" - microsoft3 strfmt.UUID = "f343f51d-7e05-4084-bd66-d504db3b6bec" - apple1 strfmt.UUID = "477fec91-1292-4928-8f53-f0ff49c76900" - apple2 strfmt.UUID = "bb2cfdba-d4ba-4cf8-abda-e719ef35ac33" - apple3 strfmt.UUID = "b71d2b4c-3da1-4684-9c5e-aabd2a4f2998" - google1 strfmt.UUID = "8c2e21fc-46fe-4999-b41c-a800595129af" - google2 strfmt.UUID = "62b969c6-f184-4be0-8c40-7470af417cfc" - google3 strfmt.UUID = "c7829929-2037-4420-acbc-a433269feb93" - ) - - type companyTemplate struct { - id strfmt.UUID - name string - inCity []strfmt.UUID - } - - companies := []companyTemplate{ - {id: microsoft1, name: "Microsoft Inc.", inCity: []strfmt.UUID{dusseldorf}}, - {id: microsoft2, name: "Microsoft Incorporated", inCity: []strfmt.UUID{dusseldorf, amsterdam}}, - {id: microsoft3, name: "Microsoft", inCity: []strfmt.UUID{berlin}}, - {id: apple1, name: "Apple Inc."}, - {id: apple2, name: "Apple Incorporated"}, - {id: apple3, name: "Apple"}, - {id: google1, name: "Google Inc."}, - {id: google2, name: "Google Incorporated"}, - {id: google3, name: "Google"}, - } - - // companies - for _, company := range companies { - inCity := []interface{}{} - for _, c := range company.inCity { - inCity = append(inCity, - map[string]interface{}{ - "beacon": crossref.NewLocalhost("City", c).String(), - }) - } - - createObject(t, &models.Object{ - Class: "Company", - ID: company.id, - Properties: map[string]interface{}{ - "inCity": inCity, - "name": company.name, - }, - }) - } - - assertGetObjectEventually(t, companies[len(companies)-1].id) -} - -func addTestDataPersons(t *testing.T) { - var ( - alice strfmt.UUID = "5d0fa6ee-21c4-4b46-a735-f0208717837d" - bob strfmt.UUID = "8615585a-2960-482d-b19d-8bee98ade52c" - john strfmt.UUID = "3ef44474-b5e5-455d-91dc-d917b5b76165" - petra strfmt.UUID = "15d222c9-8c36-464b-bedb-113faa1c1e4c" - ) - - type personTemplate struct { - id strfmt.UUID - name string - livesIn []strfmt.UUID - profession string - about []string - } - - persons := []personTemplate{ - { - id: alice, name: "Alice", livesIn: []strfmt.UUID{}, profession: "Quality Control Analyst", - about: []string{"loves travelling very much"}, - }, - { - id: bob, name: "Bob", livesIn: []strfmt.UUID{amsterdam}, profession: "Mechanical Engineer", - about: []string{"loves travelling", "hates cooking"}, - }, - { - id: john, name: "John", livesIn: []strfmt.UUID{amsterdam, berlin}, profession: "Senior Mechanical Engineer", - about: []string{"hates swimming", "likes cooking", "loves travelling"}, - }, - { - id: petra, name: "Petra", livesIn: []strfmt.UUID{amsterdam, berlin, dusseldorf}, profession: "Quality Assurance Manager", - about: []string{"likes swimming", "likes cooking for family"}, - }, - } - - // persons - for _, person := range persons { - livesIn := []interface{}{} - for _, c := range person.livesIn { - livesIn = append(livesIn, - map[string]interface{}{ - "beacon": crossref.NewLocalhost("City", c).String(), - }) - } - - createObject(t, &models.Object{ - Class: "Person", - ID: person.id, - Properties: map[string]interface{}{ - "livesIn": livesIn, - "name": person.name, - "profession": person.profession, - "about": person.about, - }, - }) - } - - assertGetObjectEventually(t, persons[len(persons)-1].id) -} - -func addTestDataPizzas(t *testing.T) { - createObject(t, &models.Object{ - Class: "Pizza", - ID: quattroFormaggi, - Properties: map[string]interface{}{ - "name": "Quattro Formaggi", - "description": "Pizza quattro formaggi Italian: [ˈkwattro forˈmaddʒi] (four cheese pizza) is a variety of pizza in Italian cuisine that is topped with a combination of four kinds of cheese, usually melted together, with (rossa, red) or without (bianca, white) tomato sauce. It is popular worldwide, including in Italy,[1] and is one of the iconic items from pizzerias's menus.", - }, - }) - createObject(t, &models.Object{ - Class: "Pizza", - ID: fruttiDiMare, - Properties: map[string]interface{}{ - "name": "Frutti di Mare", - "description": "Frutti di Mare is an Italian type of pizza that may be served with scampi, mussels or squid. It typically lacks cheese, with the seafood being served atop a tomato sauce.", - }, - }) - createObject(t, &models.Object{ - Class: "Pizza", - ID: hawaii, - Properties: map[string]interface{}{ - "name": "Hawaii", - "description": "Universally accepted to be the best pizza ever created.", - }, - }) - createObject(t, &models.Object{ - Class: "Pizza", - ID: doener, - Properties: map[string]interface{}{ - "name": "Doener", - "description": "A innovation, some say revolution, in the pizza industry.", - }, - }) - - assertGetObjectEventually(t, quattroFormaggi) - assertGetObjectEventually(t, fruttiDiMare) - assertGetObjectEventually(t, hawaii) - assertGetObjectEventually(t, doener) -} - -func addTestDataCVC(t *testing.T) { - // add one object individually - createObject(t, &models.Object{ - Class: "CustomVectorClass", - ID: cvc1, - Vector: []float32{1.1, 1.1, 1.1}, - Properties: map[string]interface{}{ - "name": "Ford", - }, - }) - - assertGetObjectEventually(t, cvc1) - - createObjectsBatch(t, []*models.Object{ - { - Class: "CustomVectorClass", - ID: cvc2, - Vector: []float32{1.1, 1.1, 0.1}, - Properties: map[string]interface{}{ - "name": "Tesla", - }, - }, - { - Class: "CustomVectorClass", - ID: cvc3, - Vector: []float32{1.1, 0, 0}, - Properties: map[string]interface{}{ - "name": "Mercedes", - }, - }, - }) - assertGetObjectEventually(t, cvc3) -} - -func addTestDataNoProperties(t *testing.T) { - for _, object := range noPropsClassObjects() { - createObject(t, object) - assertGetObjectEventually(t, object.ID) - } -} - -func addTestDataArrayClass(t *testing.T) { - for _, object := range arrayClassObjects() { - createObject(t, object) - assertGetObjectEventually(t, object.ID) - } -} - -func addTestDataDuplicatesClass(t *testing.T) { - for _, object := range duplicatesClassObjects() { - createObject(t, object) - assertGetObjectEventually(t, object.ID) - } -} - -func addTestDataRansomNotes(t *testing.T) { - const ( - noteLengthMin = 4 - noteLengthMax = 1024 - - batchSize = 10 - numBatches = 50 - ) - - seededRand := rand.New(rand.NewSource(time.Now().UnixNano())) - - for i := 0; i < numBatches; i++ { - batch := make([]*models.Object, batchSize) - for j := 0; j < batchSize; j++ { - noteLength := noteLengthMin + seededRand.Intn(noteLengthMax-noteLengthMin+1) - note := helper.GetRandomString(noteLength) - - batch[j] = &models.Object{ - Class: "RansomNote", - Properties: map[string]interface{}{"contents": note}, - } - } - - createObjectsBatch(t, batch) - } -} - -func addTestDataMultiShard(t *testing.T) { - for _, multiShard := range multishard.Objects() { - helper.CreateObject(t, multiShard) - helper.AssertGetObjectEventually(t, multiShard.Class, multiShard.ID) - } -} - -func addTestDataNearObjectSearch(t *testing.T) { - classNames := []string{"NearObjectSearch", "NearObjectSearchShadow"} - ids := []strfmt.UUID{ - "aa44bbee-ca5f-4db7-a412-5fc6a2300001", - "aa44bbee-ca5f-4db7-a412-5fc6a2300002", - "aa44bbee-ca5f-4db7-a412-5fc6a2300003", - "aa44bbee-ca5f-4db7-a412-5fc6a2300004", - "aa44bbee-ca5f-4db7-a412-5fc6a2300005", - } - names := []string{ - "Mount Everest", - "Amsterdam is a cool city", - "Football is a game where people run after ball", - "Berlin is Germany's capital city", - "London is a cool city", - } - - for _, className := range classNames { - createObjectClass(t, &models.Class{ - Class: className, - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "vectorizeClassName": true, - }, - }, - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - }) - } - - for i, id := range ids { - createObject(t, &models.Object{ - Class: classNames[0], - ID: id, - Properties: map[string]interface{}{ - "name": names[i], - }, - }) - assertGetObjectEventually(t, id) - createObject(t, &models.Object{ - Class: classNames[1], - ID: id, - Properties: map[string]interface{}{ - "name": fmt.Sprintf("altered contents of: %v", names[i]), - }, - }) - assertGetObjectEventually(t, id) - } - - createObject(t, &models.Object{ - Class: classNames[0], - ID: "aa44bbee-ca5f-4db7-a412-5fc6a2300011", - Properties: map[string]interface{}{ - "name": "the same content goes here just for explore tests", - }, - }) - assertGetObjectEventually(t, "aa44bbee-ca5f-4db7-a412-5fc6a2300011") - createObject(t, &models.Object{ - Class: classNames[1], - ID: "aa44bbee-ca5f-4db7-a412-5fc6a2300011", - Properties: map[string]interface{}{ - "name": "the same content goes here just for explore tests", - }, - }) - assertGetObjectEventually(t, "aa44bbee-ca5f-4db7-a412-5fc6a2300011") -} - -const ( - cursorClassID1 = strfmt.UUID("00000000-0000-0000-0000-000000000001") - cursorClassID2 = strfmt.UUID("00000000-0000-0000-0000-000000000002") - cursorClassID3 = strfmt.UUID("00000000-0000-0000-0000-000000000003") - cursorClassID4 = strfmt.UUID("00000000-0000-0000-0000-000000000004") - cursorClassID5 = strfmt.UUID("00000000-0000-0000-0000-000000000005") - cursorClassID6 = strfmt.UUID("00000000-0000-0000-0000-000000000006") - cursorClassID7 = strfmt.UUID("00000000-0000-0000-0000-000000000007") -) - -func addTestDataCursorSearch(t *testing.T) { - className := "CursorClass" - ids := []strfmt.UUID{ - cursorClassID1, - cursorClassID2, - cursorClassID3, - cursorClassID4, - cursorClassID5, - cursorClassID6, - cursorClassID7, - } - names := []string{ - "Mount Everest", - "Amsterdam is a cool city", - "Football is a game where people run after ball", - "Berlin is Germany's capital city", - "London is a cool city", - "Wroclaw is a really cool city", - "Brisbane is a city in Australia", - } - - createObjectClass(t, &models.Class{ - Class: className, - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "vectorizeClassName": true, - }, - }, - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - }) - - for i, id := range ids { - createObject(t, &models.Object{ - Class: className, - ID: id, - Properties: map[string]interface{}{ - "name": names[i], - }, - }) - assertGetObjectEventually(t, id) - } -} - -func addDateFieldClass(t *testing.T) { - timestamps := []string{ - "2022-06-16T22:18:59.640162Z", - "2022-06-16T22:19:01.495967Z", - "2022-06-16T22:19:03.495596Z", - "2022-06-16T22:19:04.3828349Z", - "2022-06-16T22:19:05.894857Z", - "2022-06-16T22:19:06.394958Z", - "2022-06-16T22:19:07.589828Z", - "2022-06-16T22:19:08.112395Z", - "2022-06-16T22:19:10.339493Z", - "2022-06-16T22:19:11.837473Z", - } - - for i := 0; i < len(timestamps); i++ { - createObject(t, &models.Object{ - Class: "HasDateField", - Properties: map[string]interface{}{ - "unique": fmt.Sprintf("#%d", i+1), - "timestamp": timestamps[i], - "identical": "hello!", - }, - }) - } -} - -func mustParseYear(year string) time.Time { - date := fmt.Sprintf("%s-01-01T00:00:00+02:00", year) - asTime, err := time.Parse(time.RFC3339, date) - if err != nil { - panic(err) - } - return asTime -} diff --git a/test/acceptance/graphql_resolvers/unindexed_property_test.go b/test/acceptance/graphql_resolvers/unindexed_property_test.go deleted file mode 100644 index 79f02f545eb3ca4e3f358e6159f2172fc7eac1cf..0000000000000000000000000000000000000000 --- a/test/acceptance/graphql_resolvers/unindexed_property_test.go +++ /dev/null @@ -1,155 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "testing" - - "github.com/go-openapi/strfmt" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/client/objects" - clschema "github.com/weaviate/weaviate/client/schema" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/test/helper" - testhelper "github.com/weaviate/weaviate/test/helper" - graphqlhelper "github.com/weaviate/weaviate/test/helper/graphql" -) - -func Test_UnindexedProperty(t *testing.T) { - className := "NoIndexTestClass" - - defer func() { - delParams := clschema.NewSchemaObjectsDeleteParams().WithClassName(className) - delResp, err := helper.Client(t).Schema.SchemaObjectsDelete(delParams, nil) - helper.AssertRequestOk(t, delResp, err, nil) - }() - - t.Run("creating a class with two string props", func(t *testing.T) { - vFalse := false - vTrue := true - - c := &models.Class{ - Class: className, - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "vectorizeClassName": true, - }, - }, - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - IndexFilterable: &vTrue, - IndexSearchable: &vTrue, - }, - { - Name: "hiddenName", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - IndexFilterable: &vFalse, - IndexSearchable: &vFalse, - }, - }, - } - - params := clschema.NewSchemaObjectsCreateParams().WithObjectClass(c) - resp, err := helper.Client(t).Schema.SchemaObjectsCreate(params, nil) - helper.AssertRequestOk(t, resp, err, nil) - }) - - t.Run("creating an object", func(t *testing.T) { - params := objects.NewObjectsCreateParams().WithBody( - &models.Object{ - Class: className, - ID: "f5ffb60f-4c13-4d07-a395-829b2396c7b9", - Properties: map[string]interface{}{ - "name": "elephant", - "hiddenName": "zebra", - }, - }) - resp, err := helper.Client(t).Objects.ObjectsCreate(params, nil) - helper.AssertRequestOk(t, resp, err, nil) - }) - - assertGetObjectEventually(t, "f5ffb60f-4c13-4d07-a395-829b2396c7b9") - - t.Run("searching for the indexed prop", func(t *testing.T) { - query := ` - { - Get { - NoIndexTestClass(where:{ - operator: Equal, - valueText: "elephant" - path:["name"] - }){ - name - hiddenName - } - } - } - ` - - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - objects := result.Get("Get", className).AsSlice() - - expected := []interface{}{ - map[string]interface{}{"name": "elephant", "hiddenName": "zebra"}, - } - - assert.ElementsMatch(t, expected, objects) - }) - - t.Run("searching for the non-indexed prop", func(t *testing.T) { - query := ` - { - Get { - NoIndexTestClass(where:{ - operator: Equal, - valueText: "zebra" - path:["hiddenName"] - }){ - name - hiddenName - } - } - } - ` - res, err := graphqlhelper.QueryGraphQL(t, helper.RootAuth, "", query, nil) - require.Nil(t, err) - assert.True(t, len(res.Errors) > 0, "this query should be impossible as the field was not indexed") - }) -} - -func assertGetObjectEventually(t *testing.T, uuid strfmt.UUID) *models.Object { - var ( - resp *objects.ObjectsGetOK - err error - ) - - checkThunk := func() interface{} { - resp, err = helper.Client(t).Objects.ObjectsGet(objects.NewObjectsGetParams().WithID(uuid), nil) - return err == nil - } - - testhelper.AssertEventuallyEqual(t, true, checkThunk) - - var object *models.Object - - helper.AssertRequestOk(t, resp, err, func() { - object = resp.Payload - }) - - return object -} diff --git a/test/acceptance/grpc/grpc_test.go b/test/acceptance/grpc/grpc_test.go deleted file mode 100644 index ab813546e3a439900314f3b69e4be1cccdb01453..0000000000000000000000000000000000000000 --- a/test/acceptance/grpc/grpc_test.go +++ /dev/null @@ -1,218 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "context" - "math/big" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - pb "github.com/weaviate/weaviate/grpc/generated/protocol/v1" - "github.com/weaviate/weaviate/test/helper" - "github.com/weaviate/weaviate/test/helper/sample-schema/books" - "google.golang.org/grpc/health/grpc_health_v1" -) - -func idByte(id string) []byte { - hexInteger, _ := new(big.Int).SetString(strings.Replace(id, "-", "", -1), 16) - return hexInteger.Bytes() -} - -func TestGRPC(t *testing.T) { - conn, err := helper.CreateGrpcConnectionClient(":50051") - require.NoError(t, err) - require.NotNil(t, conn) - grpcClient := helper.CreateGrpcWeaviateClient(conn) - require.NotNil(t, grpcClient) - - // delete if exists and then re-create Books class - booksClass := books.ClassContextionaryVectorizer() - helper.DeleteClass(t, booksClass.Class) - helper.CreateClass(t, booksClass) - defer helper.DeleteClass(t, booksClass.Class) - - t.Run("Health Check", func(t *testing.T) { - client := grpc_health_v1.NewHealthClient(conn) - check, err := client.Check(context.TODO(), &grpc_health_v1.HealthCheckRequest{}) - require.NoError(t, err) - require.NotNil(t, check) - assert.Equal(t, grpc_health_v1.HealthCheckResponse_SERVING.Enum().Number(), check.Status.Number()) - }) - - t.Run("Batch import", func(t *testing.T) { - resp, err := grpcClient.BatchObjects(context.TODO(), &pb.BatchObjectsRequest{ - Objects: books.BatchObjects(), - }) - require.NoError(t, err) - require.NotNil(t, resp) - }) - - tests := []struct { - name string - req *pb.SearchRequest - }{ - { - name: "Search with props", - req: &pb.SearchRequest{ - Collection: booksClass.Class, - Properties: &pb.PropertiesRequest{ - NonRefProperties: []string{"title"}, - ObjectProperties: []*pb.ObjectPropertiesRequest{ - { - PropName: "meta", - PrimitiveProperties: []string{"isbn"}, - ObjectProperties: []*pb.ObjectPropertiesRequest{ - { - PropName: "obj", - PrimitiveProperties: []string{"text"}, - }, - { - PropName: "objs", - PrimitiveProperties: []string{"text"}, - }, - }, - }, - {PropName: "reviews", PrimitiveProperties: []string{"tags"}}, - }, - }, - Metadata: &pb.MetadataRequest{ - Uuid: true, - }, - Uses_123Api: true, - }, - }, - { - name: "Search without props", - req: &pb.SearchRequest{ - Collection: booksClass.Class, - Metadata: &pb.MetadataRequest{ - Uuid: true, - }, - Uses_123Api: true, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - scifi := "sci-fi" - resp, err := grpcClient.Search(context.TODO(), tt.req) - require.NoError(t, err) - require.NotNil(t, resp) - require.NotNil(t, resp.Results) - assert.Equal(t, len(books.BatchObjects()), len(resp.Results)) - for i := range resp.Results { - res := resp.Results[i] - id := res.Metadata.Id - - assert.True(t, id == books.Dune.String() || id == books.ProjectHailMary.String() || id == books.TheLordOfTheIceGarden.String()) - titleRaw := res.Properties.NonRefProps.Fields["title"] - require.NotNil(t, titleRaw) - title := titleRaw.GetStringValue() - require.NotNil(t, title) - - metaRaw := res.Properties.NonRefProps.Fields["meta"] - require.NotNil(t, metaRaw) - meta := metaRaw.GetObjectValue() - require.NotNil(t, meta) - isbnRaw := meta.GetFields()["isbn"] - require.NotNil(t, isbnRaw) - isbn := isbnRaw.GetStringValue() - require.NotNil(t, isbn) - - objRaw := meta.GetFields()["obj"] - require.NotNil(t, objRaw) - obj := objRaw.GetObjectValue() - require.NotNil(t, obj) - - objsRaw := meta.GetFields()["objs"] - require.NotNil(t, objsRaw) - objs := objsRaw.GetListValue() - require.NotNil(t, objs) - - objEntryRaw := objs.Values[0] - require.NotNil(t, objEntryRaw) - objEntry := objEntryRaw.GetObjectValue() - require.NotNil(t, objEntry) - - reviewsRaw := res.Properties.NonRefProps.Fields["reviews"] - require.NotNil(t, reviewsRaw) - reviews := reviewsRaw.GetListValue() - require.NotNil(t, reviews) - require.Len(t, reviews.Values, 1) - - review := reviews.Values[0].GetObjectValue() - require.NotNil(t, review) - - tags := review.Fields["tags"].GetListValue() - require.NotNil(t, tags) - - strTags := make([]string, len(tags.Values)) - for i, tag := range tags.Values { - strTags[i] = tag.GetStringValue() - } - - expectedTitle := "" - expectedIsbn := "" - expectedTags := []string{} - if id == books.Dune.String() { - expectedTitle = "Dune" - expectedIsbn = "978-0593099322" - expectedTags = []string{scifi, "epic"} - } - if id == books.ProjectHailMary.String() { - expectedTitle = "Project Hail Mary" - expectedIsbn = "978-0593135204" - expectedTags = []string{scifi} - } - if id == books.TheLordOfTheIceGarden.String() { - expectedTitle = "The Lord of the Ice Garden" - expectedIsbn = "978-8374812962" - expectedTags = []string{scifi, "fantasy"} - } - assert.Equal(t, expectedTitle, title) - assert.Equal(t, expectedIsbn, isbn) - assert.Equal(t, expectedTags, strTags) - - expectedObj := &pb.Properties{ - Fields: map[string]*pb.Value{ - "text": {Kind: &pb.Value_StringValue{StringValue: "some text"}}, - }, - } - assert.Equal(t, expectedObj, obj) - assert.Equal(t, expectedObj, objEntry) - } - }) - } - - t.Run("Batch delete", func(t *testing.T) { - resp, err := grpcClient.BatchDelete(context.TODO(), &pb.BatchDeleteRequest{ - Collection: "Books", - Filters: &pb.Filters{Operator: pb.Filters_OPERATOR_EQUAL, TestValue: &pb.Filters_ValueText{ValueText: "Dune"}, Target: &pb.FilterTarget{Target: &pb.FilterTarget_Property{Property: "title"}}}, - DryRun: true, - Verbose: true, - }) - require.NoError(t, err) - require.NotNil(t, resp) - require.Equal(t, resp.Matches, int64(1)) - require.Equal(t, resp.Successful, int64(1)) - require.Equal(t, resp.Failed, int64(0)) - require.Equal(t, resp.Objects[0].Uuid, idByte(books.Dune.String())) - }) - - t.Run("gRPC Search removed", func(t *testing.T) { - _, err := grpcClient.Search(context.TODO(), &pb.SearchRequest{}) - require.NotNil(t, err) - }) -} diff --git a/test/acceptance/grpc/grpc_test_deprecated.go b/test/acceptance/grpc/grpc_test_deprecated.go deleted file mode 100644 index 84b691a6da85b71204dcb2c76374d880b1cf7107..0000000000000000000000000000000000000000 --- a/test/acceptance/grpc/grpc_test_deprecated.go +++ /dev/null @@ -1,164 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - pb "github.com/weaviate/weaviate/grpc/generated/protocol/v1" - "github.com/weaviate/weaviate/test/helper" - "github.com/weaviate/weaviate/test/helper/sample-schema/books" - "google.golang.org/grpc/health/grpc_health_v1" -) - -func TestGRPCDeprecated(t *testing.T) { - conn, err := helper.CreateGrpcConnectionClient(":50051") - require.NoError(t, err) - require.NotNil(t, conn) - grpcClient := helper.CreateGrpcWeaviateClient(conn) - require.NotNil(t, grpcClient) - - // create Books class - booksClass := books.ClassContextionaryVectorizer() - helper.CreateClass(t, booksClass) - defer helper.DeleteClass(t, booksClass.Class) - - t.Run("Health Check", func(t *testing.T) { - client := grpc_health_v1.NewHealthClient(conn) - check, err := client.Check(context.TODO(), &grpc_health_v1.HealthCheckRequest{}) - require.NoError(t, err) - require.NotNil(t, check) - assert.Equal(t, grpc_health_v1.HealthCheckResponse_SERVING.Enum().Number(), check.Status.Number()) - }) - - t.Run("Batch import", func(t *testing.T) { - resp, err := grpcClient.BatchObjects(context.TODO(), &pb.BatchObjectsRequest{ - Objects: books.BatchObjects(), - }) - require.NoError(t, err) - require.NotNil(t, resp) - }) - - tests := []struct { - name string - req *pb.SearchRequest - }{ - { - name: "Search with props", - req: &pb.SearchRequest{ - Collection: booksClass.Class, - Properties: &pb.PropertiesRequest{ - NonRefProperties: []string{"title"}, - ObjectProperties: []*pb.ObjectPropertiesRequest{ - { - PropName: "meta", - PrimitiveProperties: []string{"isbn"}, - ObjectProperties: []*pb.ObjectPropertiesRequest{ - { - PropName: "obj", - PrimitiveProperties: []string{"text"}, - }, - { - PropName: "objs", - PrimitiveProperties: []string{"text"}, - }, - }, - }, - {PropName: "reviews", PrimitiveProperties: []string{"tags"}}, - }, - }, - Metadata: &pb.MetadataRequest{ - Uuid: true, - }, - Uses_123Api: false, - }, - }, - { - name: "Search without props", - req: &pb.SearchRequest{ - Collection: booksClass.Class, - Metadata: &pb.MetadataRequest{ - Uuid: true, - }, - Uses_123Api: false, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - scifi := "sci-fi" - resp, err := grpcClient.Search(context.TODO(), tt.req) - require.NoError(t, err) - require.NotNil(t, resp) - require.NotNil(t, resp.Results) - assert.Equal(t, len(books.BatchObjects()), len(resp.Results)) - for i := range resp.Results { - res := resp.Results[i] - id := res.Metadata.Id - - assert.True(t, id == books.Dune.String() || id == books.ProjectHailMary.String() || id == books.TheLordOfTheIceGarden.String()) - title, ok := res.Properties.NonRefProperties.AsMap()["title"] - require.True(t, ok) - - objProps := res.Properties.ObjectProperties - require.Len(t, objProps, 1) - isbn, ok := objProps[0].Value.NonRefProperties.AsMap()["isbn"] - require.True(t, ok) - - nestedObjProps := objProps[0].Value.ObjectProperties - require.Len(t, nestedObjProps, 1) - nestedObj := nestedObjProps[0].Value.NonRefProperties.AsMap() - - nestedObjArrayProps := objProps[0].Value.ObjectArrayProperties - require.Len(t, nestedObjArrayProps, 1) - nestedObjEntry := nestedObjArrayProps[0].Values[0].NonRefProperties.AsMap() - - objArrayProps := res.Properties.ObjectArrayProperties - require.Len(t, objArrayProps, 1) - tags := objArrayProps[0].Values[0].TextArrayProperties[0].Values - require.True(t, ok) - - expectedTitle := "" - expectedIsbn := "" - expectedTags := []string{} - if id == books.Dune.String() { - expectedTitle = "Dune" - expectedIsbn = "978-0593099322" - expectedTags = []string{scifi, "epic"} - } - if id == books.ProjectHailMary.String() { - expectedTitle = "Project Hail Mary" - expectedIsbn = "978-0593135204" - expectedTags = []string{scifi} - } - if id == books.TheLordOfTheIceGarden.String() { - expectedTitle = "The Lord of the Ice Garden" - expectedIsbn = "978-8374812962" - expectedTags = []string{scifi, "fantasy"} - } - assert.Equal(t, expectedTitle, title) - assert.Equal(t, expectedIsbn, isbn) - assert.Equal(t, expectedTags, tags) - assert.Equal(t, map[string]interface{}{"text": "some text"}, nestedObj) - assert.Equal(t, map[string]interface{}{"text": "some text"}, nestedObjEntry) - } - }) - } - - t.Run("gRPC Search removed", func(t *testing.T) { - _, err := grpcClient.Search(context.TODO(), &pb.SearchRequest{}) - require.NotNil(t, err) - }) -} diff --git a/test/acceptance/multi_node/bm25_test.go b/test/acceptance/multi_node/bm25_test.go deleted file mode 100644 index 70eb6d497c46542af75a50f6e6a24bf17fed462b..0000000000000000000000000000000000000000 --- a/test/acceptance/multi_node/bm25_test.go +++ /dev/null @@ -1,77 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package multi_node - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/test/docker" - "github.com/weaviate/weaviate/test/helper" - graphqlhelper "github.com/weaviate/weaviate/test/helper/graphql" - "github.com/weaviate/weaviate/test/helper/sample-schema/articles" -) - -var paragraphs = []string{ - "Some random text", - "Other text", - "completely unrelated", - "this has nothing to do with the rest", -} - -func TestBm25MultiNode(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) - defer cancel() - for i := 0; i < 5; i++ { - t.Run(fmt.Sprintf("iteration: %v", i), func(t *testing.T) { - runBM25MultinodeTest(t, ctx) - }) - } -} - -func runBM25MultinodeTest(t *testing.T, ctx context.Context) { - compose, err := docker.New(). - WithWeaviateCluster(). - Start(ctx) - require.NoError(t, err) - defer func() { - if err := compose.Terminate(ctx); err != nil { - t.Fatalf("failed to terminate test containers: %s", err.Error()) - } - }() - - helper.SetupClient(compose.GetWeaviate().URI()) - paragraphClass := articles.ParagraphsClass() - helper.CreateClass(t, paragraphClass) - for _, par := range paragraphs { - obj := articles.NewParagraph(). - WithContents(par). - Object() - helper.CreateObject(t, obj) - } - - query := ` - { - Get { - Paragraph (bm25:{query:"random"}){ - contents - } - } - } - ` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - resParagraph := result.Get("Get", "Paragraph").AsSlice() - require.Equal(t, resParagraph[0].(map[string]interface{})["contents"], paragraphs[0]) -} diff --git a/test/acceptance/multi_tenancy/add_tenant_objects_test.go b/test/acceptance/multi_tenancy/add_tenant_objects_test.go deleted file mode 100644 index 50be06e8363d0f7b89bd7354d4ba25d68af9b541..0000000000000000000000000000000000000000 --- a/test/acceptance/multi_tenancy/add_tenant_objects_test.go +++ /dev/null @@ -1,166 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "testing" - - "github.com/weaviate/weaviate/client/objects" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/test/helper" -) - -func TestAddTenantObjects(t *testing.T) { - className := "MultiTenantClass" - testClass := models.Class{ - Class: className, - MultiTenancyConfig: &models.MultiTenancyConfig{ - Enabled: true, - }, - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - }, - }, - } - tenantNames := []string{ - "Tenant1", "Tenant2", "Tenant3", - } - tenantObjects := []*models.Object{ - { - ID: "0927a1e0-398e-4e76-91fb-04a7a8f0405c", - Class: className, - Properties: map[string]interface{}{ - "name": tenantNames[0], - }, - Tenant: tenantNames[0], - }, - { - ID: "831ae1d0-f441-44b1-bb2a-46548048e26f", - Class: className, - Properties: map[string]interface{}{ - "name": tenantNames[1], - }, - Tenant: tenantNames[1], - }, - { - ID: "6f3363e0-c0a0-4618-bf1f-b6cad9cdff59", - Class: className, - Properties: map[string]interface{}{ - "name": tenantNames[2], - }, - Tenant: tenantNames[2], - }, - } - - defer func() { - helper.DeleteClass(t, className) - }() - - t.Run("create class with multi-tenancy enabled", func(t *testing.T) { - helper.CreateClass(t, &testClass) - }) - - t.Run("create tenants", func(t *testing.T) { - tenants := make([]*models.Tenant, len(tenantNames)) - for i := range tenants { - tenants[i] = &models.Tenant{Name: tenantNames[i]} - } - helper.CreateTenants(t, className, tenants) - }) - - t.Run("add tenant objects", func(t *testing.T) { - for _, obj := range tenantObjects { - helper.CreateObject(t, obj) - } - }) - - t.Run("verify object creation", func(t *testing.T) { - for i, obj := range tenantObjects { - resp, err := helper.TenantObject(t, obj.Class, obj.ID, tenantNames[i]) - require.Nil(t, err) - assert.Equal(t, obj.Class, resp.Class) - assert.Equal(t, obj.Properties, resp.Properties) - } - }) -} - -func TestAddTenantObjectsToNonMultiClass(t *testing.T) { - className := "NoTenantClass" - tenantName := "randomTenant" - defer func() { - helper.DeleteClass(t, className) - }() - - testClass := models.Class{ - Class: className, - MultiTenancyConfig: &models.MultiTenancyConfig{Enabled: false}, - } - helper.CreateClass(t, &testClass) - - objWithTenant := &models.Object{ - ID: "0927a1e0-398e-4e76-91fb-04a7a8f0405c", - Class: className, - Tenant: tenantName, - } - params := objects.NewObjectsCreateParams().WithBody(objWithTenant) - _, err := helper.Client(t).Objects.ObjectsCreate(params, nil) - require.NotNil(t, err) -} - -func TestAddNonTenantObjectsToMultiClass(t *testing.T) { - className := "TenantClassFail" - defer func() { - helper.DeleteClass(t, className) - }() - - testClass := models.Class{ - Class: className, - MultiTenancyConfig: &models.MultiTenancyConfig{Enabled: true}, - } - helper.CreateClass(t, &testClass) - objWithTenant := &models.Object{ - ID: "0927a1e0-398e-4e76-91fb-04a7a8f0405c", - Class: className, - } - params := objects.NewObjectsCreateParams().WithBody(objWithTenant) - _, err := helper.Client(t).Objects.ObjectsCreate(params, nil) - require.NotNil(t, err) -} - -func TestAddObjectWithNonexistentTenantToMultiClass(t *testing.T) { - className := "TenantClass" - defer func() { - helper.DeleteClass(t, className) - }() - - testClass := models.Class{ - Class: className, - MultiTenancyConfig: &models.MultiTenancyConfig{Enabled: true}, - } - helper.CreateClass(t, &testClass) - helper.CreateTenants(t, className, []*models.Tenant{{Name: "randomTenant1"}}) - - objWithTenant := &models.Object{ - ID: "0927a1e0-398e-4e76-91fb-04a7a8f0405c", - Class: className, - Tenant: "randomTenant2", - } - params := objects.NewObjectsCreateParams().WithBody(objWithTenant) - _, err := helper.Client(t).Objects.ObjectsCreate(params, nil) - require.NotNil(t, err) -} diff --git a/test/acceptance/multi_tenancy/batch_add_tenant_objects_test.go b/test/acceptance/multi_tenancy/batch_add_tenant_objects_test.go deleted file mode 100644 index 14d2e9db43a82ed4735928082c0bf8108a80af20..0000000000000000000000000000000000000000 --- a/test/acceptance/multi_tenancy/batch_add_tenant_objects_test.go +++ /dev/null @@ -1,259 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "testing" - - "github.com/go-openapi/strfmt" - "github.com/google/uuid" - - "github.com/weaviate/weaviate/client/batch" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/test/helper" -) - -func TestBatchAddTenantObjects(t *testing.T) { - testClass := models.Class{ - Class: "MultiTenantClass", - MultiTenancyConfig: &models.MultiTenancyConfig{ - Enabled: true, - }, - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - }, - }, - } - tenantName := "Tenant1" - tenantObjects := []*models.Object{ - { - ID: "0927a1e0-398e-4e76-91fb-04a7a8f0405c", - Class: testClass.Class, - Properties: map[string]interface{}{ - "name": tenantName, - }, - Tenant: tenantName, - }, - { - ID: "831ae1d0-f441-44b1-bb2a-46548048e26f", - Class: testClass.Class, - Properties: map[string]interface{}{ - "name": tenantName, - }, - Tenant: tenantName, - }, - { - ID: "6f3363e0-c0a0-4618-bf1f-b6cad9cdff59", - Class: testClass.Class, - Properties: map[string]interface{}{ - "name": tenantName, - }, - Tenant: tenantName, - }, - } - - helper.CreateClass(t, &testClass) - defer func() { - helper.DeleteClass(t, testClass.Class) - }() - - helper.CreateTenants(t, testClass.Class, []*models.Tenant{{Name: tenantName}}) - - t.Run("add and get tenant objects", func(t *testing.T) { - helper.CreateObjectsBatch(t, tenantObjects) - - for _, obj := range tenantObjects { - resp, err := helper.TenantObject(t, obj.Class, obj.ID, tenantName) - require.Nil(t, err) - assert.Equal(t, obj.ID, resp.ID) - assert.Equal(t, obj.Class, resp.Class) - assert.Equal(t, obj.Tenant, resp.Tenant) - } - }) -} - -func TestBatchWithMixedTenants(t *testing.T) { - className := "MultiTenantClassMixedBatchFail" - classes := []models.Class{ - { - Class: className + "1", - MultiTenancyConfig: &models.MultiTenancyConfig{ - Enabled: true, - }, - }, { - Class: className + "2", - MultiTenancyConfig: &models.MultiTenancyConfig{ - Enabled: true, - }, - }, - } - tenants := []string{"tenant1", "tenant2", "tenant3"} - for i := range classes { - helper.CreateClass(t, &classes[i]) - for k := range tenants { - helper.CreateTenants(t, classes[i].Class, []*models.Tenant{{Name: tenants[k]}}) - } - } - defer func() { - for i := range classes { - helper.DeleteClass(t, classes[i].Class) - } - }() - - var tenantObjects []*models.Object - - for i := 0; i < 9; i++ { - tenantObjects = append(tenantObjects, &models.Object{ - ID: strfmt.UUID(uuid.New().String()), - Class: classes[i%2].Class, - Tenant: tenants[i%len(tenants)], - }, - ) - } - helper.CreateObjectsBatch(t, tenantObjects) - - for _, obj := range tenantObjects { - resp, err := helper.TenantObject(t, obj.Class, obj.ID, obj.Tenant) - require.Nil(t, err) - assert.Equal(t, obj.ID, resp.ID) - assert.Equal(t, obj.Class, resp.Class) - } -} - -func TestAddNonTenantBatchToMultiClass(t *testing.T) { - className := "MultiTenantClassBatchFail" - testClass := models.Class{ - Class: className, - MultiTenancyConfig: &models.MultiTenancyConfig{ - Enabled: true, - }, - } - nonTenantObjects := []*models.Object{ - { - ID: "0927a1e0-398e-4e76-91fb-04a7a8f0405c", - Class: testClass.Class, - }, - { - ID: "831ae1d0-f441-44b1-bb2a-46548048e26f", - Class: testClass.Class, - }, - { - ID: "6f3363e0-c0a0-4618-bf1f-b6cad9cdff59", - Class: testClass.Class, - }, - } - - helper.CreateClass(t, &testClass) - defer func() { - helper.DeleteClass(t, testClass.Class) - }() - helper.CreateTenants(t, className, []*models.Tenant{{Name: "randomTenant1"}}) - params := batch.NewBatchObjectsCreateParams(). - WithBody(batch.BatchObjectsCreateBody{ - Objects: nonTenantObjects, - }) - resp, err := helper.Client(t).Batch.BatchObjectsCreate(params, nil) - require.Nil(t, err) - for i := range resp.Payload { - require.NotNil(t, resp.Payload[i].Result.Errors) - } -} - -func TestAddBatchToNonMultiClass(t *testing.T) { - className := "MultiTenantClassBatchFail" - testClass := models.Class{ - Class: className, - MultiTenancyConfig: &models.MultiTenancyConfig{ - Enabled: false, - }, - } - tenantObjects := []*models.Object{ - { - ID: "0927a1e0-398e-4e76-91fb-04a7a8f0405c", - Class: testClass.Class, - Tenant: "something", - }, - { - ID: "831ae1d0-f441-44b1-bb2a-46548048e26f", - Class: testClass.Class, - Tenant: "something", - }, - { - ID: "6f3363e0-c0a0-4618-bf1f-b6cad9cdff59", - Class: testClass.Class, - Tenant: "something", - }, - } - - helper.CreateClass(t, &testClass) - defer func() { - helper.DeleteClass(t, testClass.Class) - }() - params := batch.NewBatchObjectsCreateParams(). - WithBody(batch.BatchObjectsCreateBody{ - Objects: tenantObjects, - }) - resp, err := helper.Client(t).Batch.BatchObjectsCreate(params, nil) - require.Nil(t, err) - for i := range resp.Payload { - require.NotNil(t, resp.Payload[i].Result.Errors) - } -} - -func TestAddBatchWithNonExistentTenant(t *testing.T) { - className := "MultiTenantClassBatchFail" - testClass := models.Class{ - Class: className, - MultiTenancyConfig: &models.MultiTenancyConfig{ - Enabled: true, - }, - } - nonTenantObjects := []*models.Object{ - { - ID: "0927a1e0-398e-4e76-91fb-04a7a8f0405c", - Class: testClass.Class, - Tenant: "something", - }, - { - ID: "831ae1d0-f441-44b1-bb2a-46548048e26f", - Class: testClass.Class, - Tenant: "something", - }, - { - ID: "6f3363e0-c0a0-4618-bf1f-b6cad9cdff59", - Class: testClass.Class, - Tenant: "something", - }, - } - - helper.CreateClass(t, &testClass) - defer func() { - helper.DeleteClass(t, testClass.Class) - }() - helper.CreateTenants(t, className, []*models.Tenant{{Name: "somethingElse"}}) - - params := batch.NewBatchObjectsCreateParams(). - WithBody(batch.BatchObjectsCreateBody{ - Objects: nonTenantObjects, - }) - resp, err := helper.Client(t).Batch.BatchObjectsCreate(params, nil) - require.Nil(t, err) - for i := range resp.Payload { - require.NotNil(t, resp.Payload[i].Result.Errors) - } -} diff --git a/test/acceptance/multi_tenancy/batch_add_tenant_references_test.go b/test/acceptance/multi_tenancy/batch_add_tenant_references_test.go deleted file mode 100644 index fce78b8f3c3477bc59ce6607219c1b59bce50f08..0000000000000000000000000000000000000000 --- a/test/acceptance/multi_tenancy/batch_add_tenant_references_test.go +++ /dev/null @@ -1,448 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "fmt" - "testing" - - "github.com/google/uuid" - - "github.com/go-openapi/strfmt" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/schema/crossref" - "github.com/weaviate/weaviate/test/helper" -) - -func TestBatchAddTenantReferences(t *testing.T) { - className1 := "MultiTenantClass1" - className2 := "MultiTenantClass2" - className3 := "SingleTenantClass1" - className4 := "SingleTenantClass2" - tenantName1 := "Tenant1" - tenantName2 := "Tenant2" - mtRefProp1 := "relatedToMT1" - mtRefProp2 := "relatedToMT2" - stRefProp := "relatedToST" - mtClass1 := models.Class{ - Class: className1, - MultiTenancyConfig: &models.MultiTenancyConfig{Enabled: true}, - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - }, - { - Name: mtRefProp1, - DataType: []string{className1}, - }, - { - Name: mtRefProp2, - DataType: []string{className2}, - }, - { - Name: stRefProp, - DataType: []string{className3}, - }, - }, - } - mtClass2 := models.Class{ - Class: className2, - MultiTenancyConfig: &models.MultiTenancyConfig{ - Enabled: true, - }, - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - }, - }, - } - stClass1 := models.Class{ - Class: className3, - Properties: []*models.Property{ - { - Name: "stringProp", - DataType: schema.DataTypeText.PropString(), - }, - }, - } - stClass2 := models.Class{ - Class: className4, - Properties: []*models.Property{ - { - Name: mtRefProp1, - DataType: []string{className1}, - }, - }, - } - mtObject1 := &models.Object{ - ID: "0927a1e0-398e-4e76-91fb-04a7a8f0405c", - Class: className1, - Properties: map[string]interface{}{ - "name": tenantName1, - }, - Tenant: tenantName1, - } - mtObject2DiffTenant := &models.Object{ - ID: "af90a7e3-53b3-4eb0-b395-10a04d217263", - Class: className2, - Properties: map[string]interface{}{ - "name": tenantName2, - }, - Tenant: tenantName2, - } - mtObject2SameTenant := &models.Object{ - ID: "4076df6b-0767-43a9-a0a4-2ec153bf262e", - Class: className2, - Properties: map[string]interface{}{ - "name": tenantName1, - }, - Tenant: tenantName1, - } - stObject1 := &models.Object{ - ID: "bea841c7-d689-4526-8af3-56c44b44274a", - Class: className3, - Properties: map[string]interface{}{ - "stringProp": "123", - }, - } - stObject2 := &models.Object{ - ID: "744f869a-7dcb-4fb5-8b0a-73075da1e116", - Class: className4, - } - - defer func() { - helper.DeleteClass(t, className1) - helper.DeleteClass(t, className2) - helper.DeleteClass(t, className3) - helper.DeleteClass(t, className4) - }() - - t.Run("create classes", func(t *testing.T) { - helper.CreateClass(t, &stClass1) - helper.CreateClass(t, &mtClass2) - helper.CreateClass(t, &mtClass1) - helper.CreateClass(t, &stClass2) - }) - - t.Run("create tenants", func(t *testing.T) { - helper.CreateTenants(t, className1, []*models.Tenant{{Name: tenantName1}}) - helper.CreateTenants(t, className2, []*models.Tenant{{Name: tenantName1}}) - helper.CreateTenants(t, className2, []*models.Tenant{{Name: tenantName2}}) - }) - - t.Run("add tenant objects", func(t *testing.T) { - helper.CreateObject(t, mtObject1) - helper.CreateObject(t, mtObject2DiffTenant) - helper.CreateObject(t, mtObject2SameTenant) - helper.CreateObject(t, stObject1) - helper.CreateObject(t, stObject2) - - t.Run("verify objects creation", func(t *testing.T) { - resp, err := helper.TenantObject(t, mtObject1.Class, mtObject1.ID, tenantName1) - require.Nil(t, err) - require.Equal(t, mtObject1.Class, resp.Class) - require.Equal(t, mtObject1.Properties, resp.Properties) - - resp, err = helper.TenantObject(t, mtObject2DiffTenant.Class, mtObject2DiffTenant.ID, tenantName2) - require.Nil(t, err) - require.Equal(t, mtObject2DiffTenant.Class, resp.Class) - require.Equal(t, mtObject2DiffTenant.Properties, resp.Properties) - - resp, err = helper.TenantObject(t, mtObject2SameTenant.Class, mtObject2SameTenant.ID, tenantName1) - require.Nil(t, err) - require.Equal(t, mtObject2SameTenant.Class, resp.Class) - require.Equal(t, mtObject2SameTenant.Properties, resp.Properties) - - resp, err = helper.GetObject(t, stObject1.Class, stObject1.ID) - require.Nil(t, err) - require.Equal(t, stObject1.Class, resp.Class) - - resp, err = helper.GetObject(t, stObject2.Class, stObject2.ID) - require.Nil(t, err) - require.Equal(t, stObject2.Class, resp.Class) - }) - }) - - t.Run("add tenant reference - same class and tenant", func(t *testing.T) { - refs := []*models.BatchReference{ - { - From: strfmt.URI(crossref.NewSource(schema.ClassName(className1), - schema.PropertyName(mtRefProp1), mtObject1.ID).String()), - To: strfmt.URI(crossref.NewLocalhost(className1, mtObject1.ID).String()), - Tenant: tenantName1, - }, - } - resp, err := helper.AddReferences(t, refs) - helper.CheckReferencesBatchResponse(t, resp, err) - - t.Run("verify object references", func(t *testing.T) { - resp, err := helper.TenantObject(t, mtObject1.Class, mtObject1.ID, tenantName1) - require.Nil(t, err) - require.Equal(t, mtObject1.Class, resp.Class) - require.Equal(t, mtObject1.ID, resp.ID) - relatedTo := resp.Properties.(map[string]interface{})[mtRefProp1].([]interface{}) - require.Len(t, relatedTo, 1) - beacon := relatedTo[0].(map[string]interface{})["beacon"].(string) - assert.Equal(t, helper.NewBeacon(className1, mtObject1.ID), strfmt.URI(beacon)) - }) - }) - - t.Run("add tenant reference - different MT class same tenant", func(t *testing.T) { - refs := []*models.BatchReference{ - { - From: strfmt.URI(crossref.NewSource(schema.ClassName(className1), - schema.PropertyName(mtRefProp2), mtObject1.ID).String()), - To: strfmt.URI(crossref.NewLocalhost(className2, mtObject2SameTenant.ID).String()), - Tenant: tenantName1, - }, - } - resp, err := helper.AddReferences(t, refs) - helper.CheckReferencesBatchResponse(t, resp, err) - - t.Run("verify object references", func(t *testing.T) { - resp, err := helper.TenantObject(t, mtObject1.Class, mtObject1.ID, tenantName1) - require.Nil(t, err) - require.Equal(t, mtObject1.Class, resp.Class) - require.Equal(t, mtObject1.ID, resp.ID) - relatedTo := resp.Properties.(map[string]interface{})[mtRefProp2].([]interface{}) - require.Len(t, relatedTo, 1) - beacon := relatedTo[0].(map[string]interface{})["beacon"].(string) - assert.Equal(t, helper.NewBeacon(className2, mtObject2SameTenant.ID), strfmt.URI(beacon)) - }) - }) - - t.Run("no references between different tenants", func(t *testing.T) { - refs := []*models.BatchReference{ - { - From: strfmt.URI(crossref.NewSource(schema.ClassName(className1), - schema.PropertyName(mtRefProp2), mtObject1.ID).String()), - To: strfmt.URI(crossref.NewLocalhost(className2, mtObject2DiffTenant.ID).String()), - Tenant: tenantName1, - }, - } - - resp, err := helper.AddReferences(t, refs) - require.Nil(t, err) - require.NotNil(t, resp) - require.Len(t, resp, 1) - require.Empty(t, resp[0].To) - require.Empty(t, resp[0].From) - require.NotNil(t, resp[0].Result) - require.NotNil(t, resp[0].Result.Errors) - require.Len(t, resp[0].Result.Errors.Error, 1) - require.NotNil(t, resp[0].Result.Errors.Error[0]) - expectedMsg := fmt.Sprintf(`target: object %s/%s not found for tenant %q`, className2, mtObject2DiffTenant.ID, tenantName1) - assert.Equal(t, expectedMsg, resp[0].Result.Errors.Error[0].Message) - }) - - t.Run("add tenant reference - from MT class to single tenant class", func(t *testing.T) { - refs := []*models.BatchReference{ - { - From: strfmt.URI(crossref.NewSource(schema.ClassName(className1), - schema.PropertyName(stRefProp), mtObject1.ID).String()), - To: strfmt.URI(crossref.NewLocalhost(className3, stObject1.ID).String()), - Tenant: tenantName1, - }, - } - resp, err := helper.AddReferences(t, refs) - helper.CheckReferencesBatchResponse(t, resp, err) - - t.Run("verify object references", func(t *testing.T) { - resp, err := helper.TenantObject(t, mtObject1.Class, mtObject1.ID, tenantName1) - require.Nil(t, err) - require.Equal(t, mtObject1.Class, resp.Class) - require.Equal(t, mtObject1.ID, resp.ID) - relatedTo := resp.Properties.(map[string]interface{})[stRefProp].([]interface{}) - require.Len(t, relatedTo, 1) - beacon := relatedTo[0].(map[string]interface{})["beacon"].(string) - assert.Equal(t, helper.NewBeacon(className3, stObject1.ID), strfmt.URI(beacon)) - }) - }) - - t.Run("no references from single tenant class to MT class", func(t *testing.T) { - refs := []*models.BatchReference{ - { - From: strfmt.URI(crossref.NewSource(schema.ClassName(className4), - schema.PropertyName(mtRefProp1), stObject2.ID).String()), - To: strfmt.URI(crossref.NewLocalhost(className1, mtObject1.ID).String()), - }, - } - - resp, err := helper.AddReferences(t, refs) - require.Nil(t, err) - require.NotNil(t, resp) - require.Len(t, resp, 1) - require.Empty(t, resp[0].To) - require.Empty(t, resp[0].From) - require.NotNil(t, resp[0].Result) - require.NotNil(t, resp[0].Result.Errors) - require.Len(t, resp[0].Result.Errors.Error, 1) - require.NotNil(t, resp[0].Result.Errors.Error[0]) - expectedMsg := "invalid reference: cannot reference a multi-tenant enabled class from a non multi-tenant enabled class" - assert.Equal(t, expectedMsg, resp[0].Result.Errors.Error[0].Message) - }) -} - -func TestAddMultipleTenantsForBatch(t *testing.T) { - tenants := []string{"tenant1", "tenant2"} - classNames := []string{"MultiTenantRefs1", "MultiTenantRefs2", "MultiTenantRefs3"} - refProps := []string{"refPropST", "refPropOtherMT", "refPropSelf"} - classes := []models.Class{ - {Class: classNames[0]}, - { - Class: classNames[1], - MultiTenancyConfig: &models.MultiTenancyConfig{Enabled: true}, - }, - { - Class: classNames[2], - MultiTenancyConfig: &models.MultiTenancyConfig{Enabled: true}, - Properties: []*models.Property{ - { - Name: refProps[0], - DataType: []string{classNames[0]}, - }, - { - Name: refProps[1], - DataType: []string{classNames[1]}, - }, - { - Name: refProps[2], - DataType: []string{classNames[2]}, - }, - }, - }, - } - defer func() { - for i := range classes { - helper.DeleteClass(t, classes[i].Class) - } - }() - for i := range classes { - helper.CreateClass(t, &classes[i]) - } - - for _, class := range classes[1:] { - for k := range tenants { - helper.CreateTenants(t, class.Class, []*models.Tenant{{Name: tenants[k]}}) - } - } - - var tenantObjects []*models.Object - objMap := make(map[string][]int) - - for i := 0; i < 9; i++ { - obj := &models.Object{ - ID: strfmt.UUID(uuid.New().String()), - Class: classes[i%len(classes)].Class, - } - if i%len(classes) > 0 { // only for MMT class - obj.Tenant = tenants[i%len(tenants)] - } - tenantObjects = append(tenantObjects, obj) - objMap[obj.Class] = append(objMap[obj.Class], i) - } - helper.CreateObjectsBatch(t, tenantObjects) - - t.Run("refs between same class", func(t *testing.T) { - var refs []*models.BatchReference - for _, objectIndex := range objMap[classNames[2]] { - obj := tenantObjects[objectIndex] - refs = append(refs, &models.BatchReference{ - From: strfmt.URI(crossref.NewSource(schema.ClassName(obj.Class), - schema.PropertyName(refProps[2]), obj.ID).String()), - To: strfmt.URI(crossref.NewLocalhost(classNames[2], obj.ID).String()), - Tenant: obj.Tenant, - }, - ) - } - resp, err := helper.AddReferences(t, refs) - helper.CheckReferencesBatchResponse(t, resp, err) - - // verify refs - for _, objectIndex := range objMap[classNames[2]] { - obj := tenantObjects[objectIndex] - - resp, err := helper.TenantObject(t, classNames[2], obj.ID, obj.Tenant) - require.Nil(t, err) - require.Equal(t, obj.Class, resp.Class) - require.Equal(t, fmt.Sprintf("weaviate://localhost/%s/%v", obj.Class, obj.ID), resp.Properties.(map[string]interface{})[refProps[2]].([]interface{})[0].(map[string]interface{})["beacon"]) - require.Equal(t, obj.Tenant, resp.Tenant) - } - }) - - t.Run("refs between multiple classes class", func(t *testing.T) { - var refs []*models.BatchReference - for i, objectIndexClass2 := range objMap[classNames[2]] { - objClass2 := tenantObjects[objectIndexClass2] - // refs between two MMT classes - if len(objMap[classNames[1]]) > i { - objClass1 := tenantObjects[objMap[classNames[1]][i]] - if objClass2.Tenant == objClass1.Tenant { - refs = append(refs, &models.BatchReference{ - From: strfmt.URI(crossref.NewSource(schema.ClassName(classNames[2]), - schema.PropertyName(refProps[1]), objClass2.ID).String()), - To: strfmt.URI(crossref.NewLocalhost(classNames[1], objClass1.ID).String()), - Tenant: objClass2.Tenant, - }) - } - } - - // refs between MMT and non MMT class - if len(objMap[classNames[0]]) > i { - objClass0 := tenantObjects[objMap[classNames[0]][i]] - refs = append(refs, &models.BatchReference{ - From: strfmt.URI(crossref.NewSource(schema.ClassName(classNames[2]), - schema.PropertyName(refProps[0]), objClass2.ID).String()), - To: strfmt.URI(crossref.NewLocalhost(classNames[0], objClass0.ID).String()), - Tenant: objClass2.Tenant, - }) - } - } - resp, err := helper.AddReferences(t, refs) - helper.CheckReferencesBatchResponse(t, resp, err) - - // verify refs - for i, objectIndexClass2 := range objMap[classNames[2]] { - objClass2 := tenantObjects[objectIndexClass2] - // refs between two MMT classes - if len(objMap[classNames[1]]) > i { - objClass1 := tenantObjects[objMap[classNames[1]][i]] - if objClass2.Tenant != objClass1.Tenant { - continue - } - - resp, err := helper.TenantObject(t, classNames[2], objClass2.ID, objClass2.Tenant) - require.Nil(t, err) - require.Equal(t, objClass2.Class, resp.Class) - require.Equal(t, fmt.Sprintf("weaviate://localhost/%s/%v", objClass1.Class, objClass1.ID), resp.Properties.(map[string]interface{})[refProps[1]].([]interface{})[0].(map[string]interface{})["beacon"]) - require.Equal(t, objClass2.Tenant, resp.Tenant) - - } - - // refs between MMT and non MMT class - if len(objMap[classNames[0]]) > i { - objClass0 := tenantObjects[objMap[classNames[0]][i]] - refs = append(refs, &models.BatchReference{ - From: strfmt.URI(crossref.NewSource(schema.ClassName(classNames[2]), - schema.PropertyName(refProps[0]), objClass2.ID).String()), - To: strfmt.URI(crossref.NewLocalhost(classNames[0], objClass0.ID).String()), - Tenant: objClass2.Tenant, - }) - } - } - }) -} diff --git a/test/acceptance/multi_tenancy/batch_delete_tenant_objects_test.go b/test/acceptance/multi_tenancy/batch_delete_tenant_objects_test.go deleted file mode 100644 index 3159852cdcd33ef1333e35bda62d9231480f69eb..0000000000000000000000000000000000000000 --- a/test/acceptance/multi_tenancy/batch_delete_tenant_objects_test.go +++ /dev/null @@ -1,118 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/client/objects" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/test/helper" -) - -func TestBatchDeleteTenantObjects(t *testing.T) { - className := "MultiTenantClass" - testClass := models.Class{ - Class: className, - MultiTenancyConfig: &models.MultiTenancyConfig{ - Enabled: true, - }, - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - }, - }, - } - tenantName := "Tenant1" - tenantObjects := []*models.Object{ - { - ID: "0927a1e0-398e-4e76-91fb-04a7a8f0405c", - Class: testClass.Class, - Properties: map[string]interface{}{ - "name": tenantName, - }, - Tenant: tenantName, - }, - { - ID: "831ae1d0-f441-44b1-bb2a-46548048e26f", - Class: testClass.Class, - Properties: map[string]interface{}{ - "name": tenantName, - }, - Tenant: tenantName, - }, - { - ID: "6f3363e0-c0a0-4618-bf1f-b6cad9cdff59", - Class: testClass.Class, - Properties: map[string]interface{}{ - "name": tenantName, - }, - Tenant: tenantName, - }, - } - - defer func() { - helper.DeleteClass(t, className) - }() - - helper.CreateClass(t, &testClass) - helper.CreateTenants(t, className, []*models.Tenant{{Name: tenantName}}) - - t.Run("add tenant objects", func(t *testing.T) { - helper.CreateObjectsBatch(t, tenantObjects) - - t.Run("verify tenant objects", func(t *testing.T) { - for _, obj := range tenantObjects { - resp, err := helper.TenantObject(t, obj.Class, obj.ID, tenantName) - require.Nil(t, err) - require.Equal(t, obj.ID, resp.ID) - require.Equal(t, obj.Class, resp.Class) - require.Equal(t, obj.Properties, resp.Properties) - } - }) - }) - - t.Run("batch delete tenant objects", func(t *testing.T) { - glob := "*" - where := models.WhereFilter{ - Operator: filters.OperatorLike.Name(), - Path: []string{"id"}, - ValueString: &glob, - } - match := models.BatchDeleteMatch{ - Class: className, - Where: &where, - } - batch := models.BatchDelete{Match: &match} - resp, err := helper.DeleteTenantObjectsBatch(t, &batch, tenantName) - require.Nil(t, err) - require.NotNil(t, resp) - require.NotNil(t, resp.Results) - assert.Nil(t, resp.Results.Objects) - assert.Equal(t, int64(3), resp.Results.Successful) - assert.Equal(t, int64(0), resp.Results.Failed) - - t.Run("verify tenant object deletion", func(t *testing.T) { - for _, obj := range tenantObjects { - resp, err := helper.TenantObject(t, obj.Class, obj.ID, tenantName) - assert.Nil(t, resp) - assert.NotNil(t, err) - assert.EqualError(t, objects.NewObjectsClassGetNotFound(), err.Error()) - } - }) - }) -} diff --git a/test/acceptance/multi_tenancy/class_creation_test.go b/test/acceptance/multi_tenancy/class_creation_test.go deleted file mode 100644 index e091d687e98a8f6611b3bf83ffc591ea55d41032..0000000000000000000000000000000000000000 --- a/test/acceptance/multi_tenancy/class_creation_test.go +++ /dev/null @@ -1,57 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "testing" - - "github.com/go-openapi/strfmt" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/test/helper" -) - -func TestClassMultiTenancyDisabled(t *testing.T) { - testClass := models.Class{ - Class: "ClassDisableMultiTenancy", - MultiTenancyConfig: &models.MultiTenancyConfig{ - Enabled: false, - }, - } - objUUID := strfmt.UUID("0927a1e0-398e-4e76-91fb-04a7a8f0405c") - - helper.CreateClass(t, &testClass) - defer func() { - helper.DeleteClass(t, testClass.Class) - }() - - helper.CreateObjectsBatch(t, []*models.Object{{ - ID: objUUID, - Class: testClass.Class, - }}) - - object, err := helper.GetObject(t, testClass.Class, objUUID) - require.Nil(t, err) - require.NotNil(t, object) - require.Equal(t, objUUID, object.ID) -} - -func TestClassMultiTenancyDisabledSchemaPrint(t *testing.T) { - testClass := models.Class{Class: "ClassDisableMultiTenancy"} - helper.CreateClass(t, &testClass) - defer func() { - helper.DeleteClass(t, testClass.Class) - }() - - classReturn := helper.GetClass(t, testClass.Class) - require.NotNil(t, classReturn.MultiTenancyConfig) -} diff --git a/test/acceptance/multi_tenancy/create_and_delete_tenants_test.go b/test/acceptance/multi_tenancy/create_and_delete_tenants_test.go deleted file mode 100644 index 0a732d69a8500535ec077184230f0c3cf72aec42..0000000000000000000000000000000000000000 --- a/test/acceptance/multi_tenancy/create_and_delete_tenants_test.go +++ /dev/null @@ -1,218 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/client/nodes" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/verbosity" - "github.com/weaviate/weaviate/test/helper" -) - -var verbose = verbosity.OutputVerbose - -func TestCreateTenants(t *testing.T) { - testClass := models.Class{ - Class: "MultiTenantClass", - MultiTenancyConfig: &models.MultiTenancyConfig{ - Enabled: true, - }, - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - }, - }, - } - - t.Run("Create tenant", func(Z *testing.T) { - expectedTenants := []string{ - "Tenant1", "Tenant2", "Tenant3", - } - - defer func() { - helper.DeleteClass(t, testClass.Class) - }() - helper.CreateClass(t, &testClass) - - tenants := make([]*models.Tenant, len(expectedTenants)) - for i := range tenants { - tenants[i] = &models.Tenant{ - Name: expectedTenants[i], - ActivityStatus: models.TenantActivityStatusHOT, - } - } - helper.CreateTenants(t, testClass.Class, tenants) - - respGet, errGet := helper.GetTenants(t, testClass.Class) - require.Nil(t, errGet) - require.NotNil(t, respGet) - require.ElementsMatch(t, respGet.Payload, tenants) - - resp, err := helper.Client(t).Nodes.NodesGet(nodes.NewNodesGetParams().WithOutput(&verbose), nil) - require.Nil(t, err) - require.NotNil(t, resp.Payload) - require.NotNil(t, resp.Payload.Nodes) - require.Len(t, resp.Payload.Nodes, 1) - require.Len(t, resp.Payload.Nodes[0].Shards, 3) - - var foundTenants []string - for _, found := range resp.Payload.Nodes[0].Shards { - assert.Equal(t, testClass.Class, found.Class) - foundTenants = append(foundTenants, found.Name) - } - assert.ElementsMatch(t, expectedTenants, foundTenants) - }) - - t.Run("Create duplicate tenant once", func(Z *testing.T) { - defer func() { - helper.DeleteClass(t, testClass.Class) - }() - helper.CreateClass(t, &testClass) - err := helper.CreateTenantsReturnError(t, testClass.Class, []*models.Tenant{{Name: "DoubleTenant"}, {Name: "DoubleTenant"}}) - require.Nil(t, err) - - // only added once - respGet, errGet := helper.GetTenants(t, testClass.Class) - require.Nil(t, errGet) - require.NotNil(t, respGet) - require.Len(t, respGet.Payload, 1) - }) - - t.Run("Create same tenant multiple times", func(Z *testing.T) { - defer func() { - helper.DeleteClass(t, testClass.Class) - }() - helper.CreateClass(t, &testClass) - helper.CreateTenants(t, testClass.Class, []*models.Tenant{{Name: "AddTenantAgain"}}) - - // idempotent operation - err := helper.CreateTenantsReturnError(t, testClass.Class, []*models.Tenant{{Name: "AddTenantAgain"}}) - require.Nil(t, err) - }) -} - -func TestDeleteTenants(t *testing.T) { - testClass := models.Class{ - Class: "MultiTenantClassDelete", - MultiTenancyConfig: &models.MultiTenancyConfig{Enabled: true}, - } - - defer func() { - helper.DeleteClass(t, testClass.Class) - }() - helper.CreateClass(t, &testClass) - - tenants := []string{"tenant1", "tenant2", "tenant3", "tenant4"} - var tenantsObject []*models.Tenant - for _, tenant := range tenants { - tenantsObject = append(tenantsObject, &models.Tenant{Name: tenant}) - } - helper.CreateTenants(t, testClass.Class, tenantsObject) - - t.Run("Delete same tenant multiple times", func(t *testing.T) { - err := helper.DeleteTenants(t, testClass.Class, []string{"tenant4"}) - require.Nil(t, err) - - // deleted once - resp, err := helper.Client(t).Nodes.NodesGet(nodes.NewNodesGetParams().WithOutput(&verbose), nil) - require.Nil(t, err) - require.NotNil(t, resp.Payload) - require.NotNil(t, resp.Payload.Nodes) - require.Len(t, resp.Payload.Nodes, 1) - for _, shard := range resp.Payload.Nodes[0].Shards { - assert.NotEqual(t, "tenant4", shard.Name) - } - - // idempotent operation - err = helper.DeleteTenants(t, testClass.Class, []string{"tenant4"}) - require.Nil(t, err) - }) - - t.Run("Delete duplicate tenant once", func(Z *testing.T) { - err := helper.DeleteTenants(t, testClass.Class, []string{"tenant1", "tenant1"}) - // idempotent operation - require.Nil(t, err) - - // deleted once - resp, err := helper.Client(t).Nodes.NodesGet(nodes.NewNodesGetParams().WithOutput(&verbose), nil) - require.Nil(t, err) - require.NotNil(t, resp.Payload) - require.NotNil(t, resp.Payload.Nodes) - require.Len(t, resp.Payload.Nodes, 1) - require.Len(t, resp.Payload.Nodes[0].Shards, 2) - }) - - t.Run("Delete non-existent tenant alongside existing", func(Z *testing.T) { - err := helper.DeleteTenants(t, testClass.Class, []string{"tenant1", "tenant5"}) - require.Nil(t, err) - - // idempotent - deleting multiple times works - tenant1 is removed - resp, err := helper.Client(t).Nodes.NodesGet(nodes.NewNodesGetParams().WithOutput(&verbose), nil) - require.Nil(t, err) - require.NotNil(t, resp.Payload) - require.NotNil(t, resp.Payload.Nodes) - require.Len(t, resp.Payload.Nodes, 1) - require.Len(t, resp.Payload.Nodes[0].Shards, 2) - }) - - t.Run("Delete tenants", func(Z *testing.T) { - err := helper.DeleteTenants(t, testClass.Class, []string{"tenant1", "tenant3"}) - require.Nil(t, err) - - // successfully deleted - resp, err := helper.Client(t).Nodes.NodesGet(nodes.NewNodesGetParams().WithOutput(&verbose), nil) - require.Nil(t, err) - require.NotNil(t, resp.Payload) - require.NotNil(t, resp.Payload.Nodes) - require.Len(t, resp.Payload.Nodes, 1) - require.Len(t, resp.Payload.Nodes[0].Shards, 1) - }) -} - -func TestTenantsNonMultiTenant(t *testing.T) { - testClass := models.Class{ - Class: "TenantsNoMultiClass", - MultiTenancyConfig: &models.MultiTenancyConfig{ - Enabled: false, - }, - } - defer func() { - helper.DeleteClass(t, testClass.Class) - }() - helper.CreateClass(t, &testClass) - - err := helper.CreateTenantsReturnError(t, testClass.Class, []*models.Tenant{{Name: "doesNotMatter"}}) - require.NotNil(t, err) - - _, err = helper.GetTenants(t, testClass.Class) - require.NotNil(t, err) - - err = helper.DeleteTenants(t, testClass.Class, []string{"doesNotMatter"}) - require.NotNil(t, err) -} - -func TestTenantsClassDoesNotExist(t *testing.T) { - err := helper.CreateTenantsReturnError(t, "DoesNotExist", []*models.Tenant{{Name: "doesNotMatter"}}) - require.NotNil(t, err) - - _, err = helper.GetTenants(t, "DoesNotExist") - require.NotNil(t, err) - - err = helper.DeleteTenants(t, "DoesNotExist", []string{"doesNotMatter"}) - require.NotNil(t, err) -} diff --git a/test/acceptance/multi_tenancy/delete_tenant_objects_test.go b/test/acceptance/multi_tenancy/delete_tenant_objects_test.go deleted file mode 100644 index a43c43122f001b1ffc4c769cd4e637faac0f548c..0000000000000000000000000000000000000000 --- a/test/acceptance/multi_tenancy/delete_tenant_objects_test.go +++ /dev/null @@ -1,113 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/client/objects" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/test/helper" -) - -func TestDeleteTenantObjects(t *testing.T) { - testClass := models.Class{ - Class: "MultiTenantClass", - MultiTenancyConfig: &models.MultiTenancyConfig{ - Enabled: true, - }, - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - }, - }, - } - tenantNames := []string{ - "Tenant1", "Tenant2", "Tenant3", - } - tenantObjects := []*models.Object{ - { - ID: "0927a1e0-398e-4e76-91fb-04a7a8f0405c", - Class: testClass.Class, - Properties: map[string]interface{}{ - "name": tenantNames[0], - }, - Tenant: tenantNames[0], - }, - { - ID: "831ae1d0-f441-44b1-bb2a-46548048e26f", - Class: testClass.Class, - Properties: map[string]interface{}{ - "name": tenantNames[1], - }, - Tenant: tenantNames[1], - }, - { - ID: "6f3363e0-c0a0-4618-bf1f-b6cad9cdff59", - Class: testClass.Class, - Properties: map[string]interface{}{ - "name": tenantNames[2], - }, - Tenant: tenantNames[2], - }, - } - - defer func() { - helper.DeleteClass(t, testClass.Class) - }() - - t.Run("create class with multi-tenancy enabled", func(t *testing.T) { - helper.CreateClass(t, &testClass) - }) - - t.Run("create tenants", func(t *testing.T) { - tenants := make([]*models.Tenant, len(tenantNames)) - for i := range tenants { - tenants[i] = &models.Tenant{Name: tenantNames[i]} - } - helper.CreateTenants(t, testClass.Class, tenants) - }) - - t.Run("add tenant objects", func(t *testing.T) { - for _, obj := range tenantObjects { - helper.CreateObject(t, obj) - } - - t.Run("verify tenant objects creation", func(t *testing.T) { - for i, obj := range tenantObjects { - resp, err := helper.TenantObject(t, obj.Class, obj.ID, tenantNames[i]) - require.Nil(t, err) - require.Equal(t, obj.ID, resp.ID) - require.Equal(t, obj.Class, resp.Class) - require.Equal(t, obj.Properties, resp.Properties) - } - }) - }) - - t.Run("delete tenant objects", func(t *testing.T) { - for i, obj := range tenantObjects { - helper.DeleteTenantObject(t, obj.Class, obj.ID, tenantNames[i]) - } - }) - - t.Run("assert object deletion", func(t *testing.T) { - for i, obj := range tenantObjects { - _, err := helper.TenantObject(t, obj.Class, obj.ID, tenantNames[i]) - require.NotNil(t, err) - assert.EqualError(t, &objects.ObjectsClassGetNotFound{}, err.Error()) - } - }) -} diff --git a/test/acceptance/multi_tenancy/get_tenant_objects_test.go b/test/acceptance/multi_tenancy/get_tenant_objects_test.go deleted file mode 100644 index acabc84504806162afde9ae686b1dee644debc75..0000000000000000000000000000000000000000 --- a/test/acceptance/multi_tenancy/get_tenant_objects_test.go +++ /dev/null @@ -1,473 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/client/objects" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/test/helper" -) - -func TestGetTenantObjects(t *testing.T) { - testClass := models.Class{ - Class: "MultiTenantClass", - MultiTenancyConfig: &models.MultiTenancyConfig{ - Enabled: true, - }, - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - }, - }, - } - tenantNames := []string{ - "Tenant1", "Tenant2", "Tenant3", - } - tenantObjects := []*models.Object{ - { - ID: "0927a1e0-398e-4e76-91fb-04a7a8f0405c", - Class: testClass.Class, - Properties: map[string]interface{}{ - "name": tenantNames[0], - }, - Tenant: tenantNames[0], - }, - { - ID: "831ae1d0-f441-44b1-bb2a-46548048e26f", - Class: testClass.Class, - Properties: map[string]interface{}{ - "name": tenantNames[1], - }, - Tenant: tenantNames[1], - }, - { - ID: "6f3363e0-c0a0-4618-bf1f-b6cad9cdff59", - Class: testClass.Class, - Properties: map[string]interface{}{ - "name": tenantNames[2], - }, - Tenant: tenantNames[2], - }, - } - - defer func() { - helper.DeleteClass(t, testClass.Class) - }() - - t.Run("create class with multi-tenancy enabled", func(t *testing.T) { - helper.CreateClass(t, &testClass) - }) - - t.Run("create tenants", func(t *testing.T) { - tenants := make([]*models.Tenant, len(tenantNames)) - for i := range tenants { - tenants[i] = &models.Tenant{Name: tenantNames[i]} - } - helper.CreateTenants(t, testClass.Class, tenants) - }) - - t.Run("add tenant objects", func(t *testing.T) { - for _, obj := range tenantObjects { - helper.CreateObject(t, obj) - } - }) - - t.Run("get tenant objects", func(t *testing.T) { - for i, obj := range tenantObjects { - resp, err := helper.TenantObject(t, obj.Class, obj.ID, tenantNames[i]) - require.Nil(t, err) - assert.Equal(t, obj.ID, resp.ID) - assert.Equal(t, obj.Class, resp.Class) - assert.Equal(t, obj.Properties, resp.Properties) - } - }) - - t.Run("get tenant objects with include", func(t *testing.T) { - for i, obj := range tenantObjects { - resp, err := helper.TenantObjectWithInclude(t, obj.Class, obj.ID, tenantNames[i], "vector") - require.Nil(t, err) - assert.Equal(t, obj.ID, resp.ID) - assert.Equal(t, obj.Class, resp.Class) - assert.Equal(t, obj.Properties, resp.Properties) - } - }) -} - -func TestListTenantObjects(t *testing.T) { - tenantNames := []string{ - "Tenant1", "Tenant2", - } - - classMT_1 := models.Class{ - Class: "MultiTenantClass1", - MultiTenancyConfig: &models.MultiTenancyConfig{ - Enabled: true, - }, - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - }, - }, - } - classMT_2 := models.Class{ - Class: "MultiTenantClass2", - MultiTenancyConfig: &models.MultiTenancyConfig{ - Enabled: true, - }, - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - }, - }, - } - classMT_3 := models.Class{ - Class: "SingleTenantClass3", - MultiTenancyConfig: &models.MultiTenancyConfig{ - Enabled: true, - }, - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - }, - }, - } - classMT_4 := models.Class{ - Class: "SingleTenantClass4", - MultiTenancyConfig: &models.MultiTenancyConfig{ - Enabled: true, - }, - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - }, - }, - } - classNonMT_1 := models.Class{ - Class: "NonTenantClass1", - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - }, - }, - } - classNonMT_2 := models.Class{ - Class: "NonTenantClass2", - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - }, - }, - } - - objectsMT_T1 := []*models.Object{ - { - ID: "b1d19f8a-2158-4c41-b648-ba77a0ea7074", - Class: classMT_1.Class, - Properties: map[string]interface{}{ - "name": "Obj1_Class1_Tenant1", - }, - Tenant: tenantNames[0], - }, - { - ID: "a95c027c-07fb-4175-b726-4d5cfd55a7cf", - Class: classMT_1.Class, - Properties: map[string]interface{}{ - "name": "Obj2_Class1_Tenant1", - }, - Tenant: tenantNames[0], - }, - { - ID: "026890f5-8623-4d31-b295-b2820a81b85a", - Class: classMT_2.Class, - Properties: map[string]interface{}{ - "name": "Obj3_Class2_Tenant1", - }, - Tenant: tenantNames[0], - }, - { - ID: "d697d6b6-d7e6-47e6-a268-42e917b614e1", - Class: classMT_3.Class, - Properties: map[string]interface{}{ - "name": "Obj4_Class3_Tenant1", - }, - Tenant: tenantNames[0], - }, - } - objectsMT_T2 := []*models.Object{ - { - ID: "7baead88-a42b-4876-a185-e0ccc61c58ca", - Class: classMT_1.Class, - Properties: map[string]interface{}{ - "name": "Obj1_Class1_Tenant2", - }, - Tenant: tenantNames[1], - }, - { - ID: "7fa1fd17-a883-465a-ae22-44f103250b27", - Class: classMT_2.Class, - Properties: map[string]interface{}{ - "name": "Obj2_Class2_Tenant2", - }, - Tenant: tenantNames[1], - }, - { - ID: "fd4ce87a-8034-4e27-8d47-539fa9dde1f3", - Class: classMT_2.Class, - Properties: map[string]interface{}{ - "name": "Obj3_Class2_Tenant2", - }, - Tenant: tenantNames[1], - }, - { - ID: "b33d8f4c-30f9-426d-94a5-fa256f3fb5e7", - Class: classMT_4.Class, - Properties: map[string]interface{}{ - "name": "Obj4_Class4_Tenant2", - }, - Tenant: tenantNames[1], - }, - } - objectsNonMT := []*models.Object{ - { - ID: "6f019424-bacf-4539-b1be-fc1d3eccb50a", - Class: classNonMT_1.Class, - Properties: map[string]interface{}{ - "name": "Obj1_NonTenant1", - }, - }, - { - ID: "8d02b16c-478c-4cae-9384-3b686bae0f4e", - Class: classNonMT_1.Class, - Properties: map[string]interface{}{ - "name": "Obj2_NonTenant1", - }, - }, - { - ID: "865a820a-c325-4d10-8d8c-4b991bc43778", - Class: classNonMT_2.Class, - Properties: map[string]interface{}{ - "name": "Obj3_NonTenant2", - }, - }, - } - - defer func() { - helper.DeleteClass(t, classMT_1.Class) - helper.DeleteClass(t, classMT_2.Class) - helper.DeleteClass(t, classMT_3.Class) - helper.DeleteClass(t, classMT_4.Class) - helper.DeleteClass(t, classNonMT_1.Class) - helper.DeleteClass(t, classNonMT_2.Class) - }() - - extractIds := func(objs []*models.Object) []string { - ids := make([]string, len(objs)) - for i, obj := range objs { - ids[i] = obj.ID.String() - } - return ids - } - - t.Run("create MT and non-MT classes", func(t *testing.T) { - helper.CreateClass(t, &classMT_1) - helper.CreateClass(t, &classMT_2) - helper.CreateClass(t, &classMT_3) - helper.CreateClass(t, &classMT_4) - helper.CreateClass(t, &classNonMT_1) - helper.CreateClass(t, &classNonMT_2) - }) - - t.Run("create tenants for MT classes", func(t *testing.T) { - tenants := make([]*models.Tenant, len(tenantNames)) - for i := range tenants { - tenants[i] = &models.Tenant{Name: tenantNames[i]} - } - helper.CreateTenants(t, classMT_1.Class, tenants) - helper.CreateTenants(t, classMT_2.Class, tenants) - helper.CreateTenants(t, classMT_3.Class, tenants[:1]) - helper.CreateTenants(t, classMT_4.Class, tenants[1:]) - }) - - t.Run("add objects", func(t *testing.T) { - objects := append(objectsMT_T1, objectsMT_T2...) - objects = append(objects, objectsNonMT...) - - helper.CreateObjectsBatch(t, objects) - }) - - t.Run("list objects for tenant 1", func(t *testing.T) { - t.Run("no class", func(t *testing.T) { - res, err := helper.TenantListObjects(t, "", tenantNames[0]) - require.Nil(t, err) - require.NotNil(t, res) - require.Equal(t, int64(4), res.TotalResults) - assert.ElementsMatch(t, []string{ - "b1d19f8a-2158-4c41-b648-ba77a0ea7074", - "a95c027c-07fb-4175-b726-4d5cfd55a7cf", - "026890f5-8623-4d31-b295-b2820a81b85a", - "d697d6b6-d7e6-47e6-a268-42e917b614e1", - }, extractIds(res.Objects)) - }) - t.Run("classMT_T1T2_1", func(t *testing.T) { - res, err := helper.TenantListObjects(t, classMT_1.Class, tenantNames[0]) - - require.Nil(t, err) - require.NotNil(t, res) - require.Equal(t, int64(2), res.TotalResults) - assert.ElementsMatch(t, []string{ - "b1d19f8a-2158-4c41-b648-ba77a0ea7074", - "a95c027c-07fb-4175-b726-4d5cfd55a7cf", - }, extractIds(res.Objects)) - }) - t.Run("classMT_T1T2_2", func(t *testing.T) { - res, err := helper.TenantListObjects(t, classMT_2.Class, tenantNames[0]) - - require.Nil(t, err) - require.NotNil(t, res) - require.Equal(t, int64(1), res.TotalResults) - assert.ElementsMatch(t, []string{ - "026890f5-8623-4d31-b295-b2820a81b85a", - }, extractIds(res.Objects)) - }) - - t.Run("classMT_T1", func(t *testing.T) { - res, err := helper.TenantListObjects(t, classMT_3.Class, tenantNames[0]) - - require.Nil(t, err) - require.NotNil(t, res) - require.Equal(t, int64(1), res.TotalResults) - assert.ElementsMatch(t, []string{ - "d697d6b6-d7e6-47e6-a268-42e917b614e1", - }, extractIds(res.Objects)) - }) - - t.Run("classMT_T2", func(t *testing.T) { - res, err := helper.TenantListObjects(t, classMT_4.Class, tenantNames[0]) - - require.NotNil(t, err) - expErr := &objects.ObjectsListUnprocessableEntity{} - require.ErrorAs(t, err, &expErr) - assert.Contains(t, err.(*objects.ObjectsListUnprocessableEntity).Payload.Error[0].Message, tenantNames[0]) - require.Nil(t, res) - }) - }) - - t.Run("list objects for tenant 2", func(t *testing.T) { - t.Run("no class", func(t *testing.T) { - res, err := helper.TenantListObjects(t, "", tenantNames[1]) - - require.Nil(t, err) - require.NotNil(t, res) - require.Equal(t, int64(4), res.TotalResults) - assert.ElementsMatch(t, []string{ - "7baead88-a42b-4876-a185-e0ccc61c58ca", - "7fa1fd17-a883-465a-ae22-44f103250b27", - "fd4ce87a-8034-4e27-8d47-539fa9dde1f3", - "b33d8f4c-30f9-426d-94a5-fa256f3fb5e7", - }, extractIds(res.Objects)) - }) - - t.Run("classMT_T1T2_1", func(t *testing.T) { - res, err := helper.TenantListObjects(t, classMT_1.Class, tenantNames[1]) - - require.Nil(t, err) - require.NotNil(t, res) - require.Equal(t, int64(1), res.TotalResults) - assert.ElementsMatch(t, []string{ - "7baead88-a42b-4876-a185-e0ccc61c58ca", - }, extractIds(res.Objects)) - }) - - t.Run("classMT_T1T2_2", func(t *testing.T) { - res, err := helper.TenantListObjects(t, classMT_2.Class, tenantNames[1]) - - require.Nil(t, err) - require.NotNil(t, res) - require.Equal(t, int64(2), res.TotalResults) - assert.ElementsMatch(t, []string{ - "7fa1fd17-a883-465a-ae22-44f103250b27", - "fd4ce87a-8034-4e27-8d47-539fa9dde1f3", - }, extractIds(res.Objects)) - }) - - t.Run("classMT_T1", func(t *testing.T) { - res, err := helper.TenantListObjects(t, classMT_3.Class, tenantNames[1]) - - require.NotNil(t, err) - expErr := &objects.ObjectsListUnprocessableEntity{} - require.ErrorAs(t, err, &expErr) - assert.Contains(t, err.(*objects.ObjectsListUnprocessableEntity).Payload.Error[0].Message, tenantNames[1]) - require.Nil(t, res) - }) - - t.Run("classMT_T2", func(t *testing.T) { - res, err := helper.TenantListObjects(t, classMT_4.Class, tenantNames[1]) - - require.Nil(t, err) - require.NotNil(t, res) - require.Equal(t, int64(1), res.TotalResults) - assert.ElementsMatch(t, []string{ - "b33d8f4c-30f9-426d-94a5-fa256f3fb5e7", - }, extractIds(res.Objects)) - }) - }) - - t.Run("list objects no tenant", func(t *testing.T) { - t.Run("no class", func(t *testing.T) { - res, err := helper.ListObjects(t, "") - - require.Nil(t, err) - require.NotNil(t, res) - require.Equal(t, int64(3), res.TotalResults) - assert.ElementsMatch(t, []string{ - "6f019424-bacf-4539-b1be-fc1d3eccb50a", - "8d02b16c-478c-4cae-9384-3b686bae0f4e", - "865a820a-c325-4d10-8d8c-4b991bc43778", - }, extractIds(res.Objects)) - }) - - t.Run("classNonMT_1", func(t *testing.T) { - res, err := helper.ListObjects(t, classNonMT_1.Class) - - require.Nil(t, err) - require.NotNil(t, res) - require.Equal(t, int64(2), res.TotalResults) - assert.ElementsMatch(t, []string{ - "6f019424-bacf-4539-b1be-fc1d3eccb50a", - "8d02b16c-478c-4cae-9384-3b686bae0f4e", - }, extractIds(res.Objects)) - }) - - t.Run("classNonMT_2", func(t *testing.T) { - res, err := helper.ListObjects(t, classNonMT_2.Class) - - require.Nil(t, err) - require.NotNil(t, res) - require.Equal(t, int64(1), res.TotalResults) - assert.ElementsMatch(t, []string{ - "865a820a-c325-4d10-8d8c-4b991bc43778", - }, extractIds(res.Objects)) - }) - }) -} diff --git a/test/acceptance/multi_tenancy/gql_aggregate_tenant_objects_test.go b/test/acceptance/multi_tenancy/gql_aggregate_tenant_objects_test.go deleted file mode 100644 index 54ce3906f83cca6a2fa71e00120df0b00c64cb70..0000000000000000000000000000000000000000 --- a/test/acceptance/multi_tenancy/gql_aggregate_tenant_objects_test.go +++ /dev/null @@ -1,206 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "encoding/json" - "fmt" - "testing" - - "github.com/go-openapi/strfmt" - "github.com/google/uuid" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/client/nodes" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/test/helper" - graphqlhelper "github.com/weaviate/weaviate/test/helper/graphql" -) - -func TestGQLAggregateTenantObjects(t *testing.T) { - testClass := models.Class{ - Class: "MultiTenantClass", - MultiTenancyConfig: &models.MultiTenancyConfig{ - Enabled: true, - }, - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - }, - }, - } - tenantName1 := "Tenant1" - tenantName2 := "Tenant2" - numTenantObjs1 := 5 - numTenantObjs2 := 3 - - defer func() { - helper.DeleteClass(t, testClass.Class) - }() - helper.CreateClass(t, &testClass) - - tenants := []*models.Tenant{ - {Name: tenantName1}, - {Name: tenantName2}, - } - helper.CreateTenants(t, testClass.Class, tenants) - - batch1 := makeTenantBatch(batchParams{ - className: testClass.Class, - tenantName: tenantName1, - batchSize: numTenantObjs1, - }) - batch2 := makeTenantBatch(batchParams{ - className: testClass.Class, - tenantName: tenantName2, - batchSize: numTenantObjs2, - }) - - helper.CreateObjectsBatch(t, batch1) - helper.CreateObjectsBatch(t, batch2) - - t.Run("GQL Aggregate tenant objects", func(t *testing.T) { - testAggregateTenantSuccess(t, testClass.Class, tenantName1, numTenantObjs1, "") - testAggregateTenantSuccess(t, testClass.Class, tenantName2, numTenantObjs2, "") - }) - - t.Run("GQL Aggregate tenant objects near object", func(t *testing.T) { - testAggregateTenantSuccess(t, testClass.Class, tenantName1, numTenantObjs1, string(batch1[0].ID)) - testAggregateTenantSuccess(t, testClass.Class, tenantName2, numTenantObjs2, string(batch2[0].ID)) - }) - - t.Run("Get global tenant objects count", func(t *testing.T) { - params := nodes.NewNodesGetClassParams().WithClassName(testClass.Class).WithOutput(&verbose) - resp, err := helper.Client(t).Nodes.NodesGetClass(params, nil) - require.Nil(t, err) - - payload := resp.GetPayload() - require.NotNil(t, payload) - require.NotNil(t, payload.Nodes) - require.Len(t, payload.Nodes, 1) - - node := payload.Nodes[0] - require.NotNil(t, node) - assert.Equal(t, models.NodeStatusStatusHEALTHY, *node.Status) - assert.True(t, len(node.Name) > 0) - assert.True(t, node.GitHash != "" && node.GitHash != "unknown") - assert.Len(t, node.Shards, 2) - - shardCount := map[string]int64{ - tenantName1: int64(numTenantObjs1), - tenantName2: int64(numTenantObjs2), - } - - for _, shard := range node.Shards { - count, ok := shardCount[shard.Name] - require.True(t, ok, "expected shard %q to be in %+v", - shard.Name, []string{tenantName1, tenantName2}) - - assert.Equal(t, testClass.Class, shard.Class) - assert.Equal(t, count, shard.ObjectCount) - } - - require.NotNil(t, node.Stats) - assert.Equal(t, int64(numTenantObjs1+numTenantObjs2), node.Stats.ObjectCount) - assert.Equal(t, int64(2), node.Stats.ShardCount) - }) -} - -func TestGQLAggregateTenantObjects_InvalidTenant(t *testing.T) { - testClass := models.Class{ - Class: "MultiTenantClass", - MultiTenancyConfig: &models.MultiTenancyConfig{ - Enabled: true, - }, - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - }, - }, - } - tenantName := "Tenant1" - numTenantObjs := 5 - - defer func() { - helper.DeleteClass(t, testClass.Class) - }() - - t.Run("setup test data", func(t *testing.T) { - t.Run("create class with multi-tenancy enabled", func(t *testing.T) { - helper.CreateClass(t, &testClass) - }) - - t.Run("create tenants", func(t *testing.T) { - tenants := []*models.Tenant{ - {Name: tenantName}, - } - helper.CreateTenants(t, testClass.Class, tenants) - }) - - t.Run("add tenant objects", func(t *testing.T) { - batch := makeTenantBatch(batchParams{ - className: testClass.Class, - tenantName: tenantName, - batchSize: numTenantObjs, - }) - helper.CreateObjectsBatch(t, batch) - }) - }) - - t.Run("non-existent tenant key", func(t *testing.T) { - query := fmt.Sprintf(`{Aggregate{%s(tenant:"DNE"){meta{count}}}}`, testClass.Class) - expected := `"DNE"` - resp, err := graphqlhelper.QueryGraphQL(t, helper.RootAuth, "", query, nil) - require.Nil(t, err) - assert.Nil(t, resp.Data["Aggregate"].(map[string]interface{})[testClass.Class]) - assert.Len(t, resp.Errors, 1) - assert.Contains(t, resp.Errors[0].Message, expected) - }) -} - -type batchParams struct { - className string - tenantName string - batchSize int -} - -func makeTenantBatch(params batchParams) []*models.Object { - batch := make([]*models.Object, params.batchSize) - for i := range batch { - batch[i] = &models.Object{ - ID: strfmt.UUID(uuid.New().String()), - Class: params.className, - Properties: map[string]interface{}{ - "name": params.tenantName, - }, - Tenant: params.tenantName, - } - } - return batch -} - -func testAggregateTenantSuccess(t *testing.T, className, tenantName string, expectedCount int, nearObjectId string) { - nearObject := "" - if nearObjectId != "" { - nearObject = fmt.Sprintf(`nearObject: {id: "%s", certainty: 0.4},`, nearObjectId) - } - - query := fmt.Sprintf(`{Aggregate{%s(%s,tenant:%q){meta{count}}}}`, className, nearObject, tenantName) - resp := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - result := resp.Get("Aggregate", className).AsSlice() - require.Len(t, result, 1) - count := result[0].(map[string]any)["meta"].(map[string]any)["count"].(json.Number) - assert.Equal(t, json.Number(fmt.Sprint(expectedCount)), count) -} diff --git a/test/acceptance/multi_tenancy/gql_get_tenant_objects_test.go b/test/acceptance/multi_tenancy/gql_get_tenant_objects_test.go deleted file mode 100644 index 09d03273cf19e07c76b85b91ddaec1f6b9852855..0000000000000000000000000000000000000000 --- a/test/acceptance/multi_tenancy/gql_get_tenant_objects_test.go +++ /dev/null @@ -1,249 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "fmt" - "strings" - "testing" - - "github.com/go-openapi/strfmt" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/test/helper" - graphqlhelper "github.com/weaviate/weaviate/test/helper/graphql" -) - -func TestGQLGetTenantObjects(t *testing.T) { - testClass := models.Class{ - Class: "MultiTenantClass", - MultiTenancyConfig: &models.MultiTenancyConfig{ - Enabled: true, - }, - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - }, - { - Name: "text", - DataType: schema.DataTypeText.PropString(), - }, - }, - Vectorizer: "text2vec-contextionary", - } - tenant := "Tenant1" - otherTenant := "otherTenant" - tenantObjects := []*models.Object{ - { - ID: "0927a1e0-398e-4e76-91fb-04a7a8f0405c", - Class: testClass.Class, - Properties: map[string]interface{}{ - "name": tenant, - "text": "meat", - }, - Tenant: tenant, - }, - { - ID: "831ae1d0-f441-44b1-bb2a-46548048e26f", - Class: testClass.Class, - Properties: map[string]interface{}{ - "name": tenant, - "text": "bananas", - }, - Tenant: tenant, - }, - { - ID: "6f3363e0-c0a0-4618-bf1f-b6cad9cdff59", - Class: testClass.Class, - Properties: map[string]interface{}{ - "name": tenant, - "text": "kiwi", - }, - Tenant: tenant, - }, - { - ID: "6f3363e0-c0a0-4618-bf1f-b6cad9cdff59", - Class: testClass.Class, - Properties: map[string]interface{}{ - "name": tenant, - "text": "kiwi", - }, - Tenant: otherTenant, - }, - } - - // add more objects for other tenant, won't show up in search - - defer func() { - helper.DeleteClass(t, testClass.Class) - }() - - helper.CreateClass(t, &testClass) - helper.CreateTenants(t, testClass.Class, []*models.Tenant{{Name: tenant}, {Name: otherTenant}}) - helper.CreateObjectsBatch(t, tenantObjects) - - t.Run("Test tenant objects", func(t *testing.T) { - for _, obj := range tenantObjects { - resp, err := helper.TenantObject(t, obj.Class, obj.ID, obj.Tenant) - require.Nil(t, err) - assert.Equal(t, obj.ID, resp.ID) - assert.Equal(t, obj.Class, resp.Class) - assert.Equal(t, obj.Properties, resp.Properties) - } - }) - - t.Run("GQL Get tenant objects", func(t *testing.T) { - expectedIDs := map[strfmt.UUID]bool{} - for _, obj := range tenantObjects { - expectedIDs[obj.ID] = false - } - - query := fmt.Sprintf(`{Get{%s(tenant:%q){_additional{id}}}}`, testClass.Class, tenant) - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - for _, obj := range result.Get("Get", testClass.Class).AsSlice() { - id := obj.(map[string]any)["_additional"].(map[string]any)["id"].(string) - if _, ok := expectedIDs[strfmt.UUID(id)]; ok { - expectedIDs[strfmt.UUID(id)] = true - } else { - t.Fatalf("found unexpected id %q", id) - } - } - - for id, found := range expectedIDs { - if !found { - t.Fatalf("expected to find id %q, but didn't", id) - } - } - }) - - t.Run("GQL near objects", func(t *testing.T) { - expectedIDs := map[strfmt.UUID]bool{} - for _, obj := range tenantObjects { - expectedIDs[obj.ID] = false - } - - query := fmt.Sprintf(`{Get{%s(nearObject:{id: %q}, tenant:%q){_additional{id}}}}`, testClass.Class, tenantObjects[0].ID, tenant) - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - res := result.Get("Get", testClass.Class) - require.NotNil(t, res) // objects have no content, so no result - }) - - t.Run("GQL near text", func(t *testing.T) { - expectedIDs := map[strfmt.UUID]bool{} - for _, obj := range tenantObjects { - expectedIDs[obj.ID] = false - } - - query := fmt.Sprintf(`{Get{%s(nearText:{concepts: "apple", moveTo: {concepts: ["fruit"], force: 0.1}, moveAwayFrom: {objects: [{id:"0927a1e0-398e-4e76-91fb-04a7a8f0405c"}], force: 0.1}}, tenant:%q){_additional{id}}}}`, testClass.Class, tenant) - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - res := result.Get("Get", testClass.Class) - require.NotNil(t, res) - require.Len(t, res.Result, 3) // don't find object from other tenants - }) - - t.Run("GQL bm25", func(t *testing.T) { - expectedIDs := map[strfmt.UUID]bool{} - for _, obj := range tenantObjects { - expectedIDs[obj.ID] = false - } - - query := fmt.Sprintf(`{Get{%s(bm25:{query: "kiwi"}, tenant:%q){_additional{id}}}}`, testClass.Class, tenant) - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - res := result.Get("Get", testClass.Class) - require.NotNil(t, res) - require.Len(t, res.Result, 1) // don't find object from other tenants - }) - - t.Run("GQL hybrid", func(t *testing.T) { - expectedIDs := map[strfmt.UUID]bool{} - for _, obj := range tenantObjects { - expectedIDs[obj.ID] = false - } - - query := fmt.Sprintf(`{Get{%s(hybrid:{query: "kiwi", alpha: 0.1}, tenant:%q, autocut:1){text _additional{id}}}}`, testClass.Class, tenant) - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - res := result.Get("Get", testClass.Class) - require.NotNil(t, res) - require.Len(t, res.Result, 1) // find only relevant results from tenant - }) -} - -func TestGQLGetTenantObjects_MissingTenant(t *testing.T) { - testClass := models.Class{ - Class: "MultiTenantClass", - MultiTenancyConfig: &models.MultiTenancyConfig{ - Enabled: true, - }, - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - }, - }, - } - tenantName := "Tenant1" - tenantObjects := []*models.Object{ - { - ID: "0927a1e0-398e-4e76-91fb-04a7a8f0405c", - Class: testClass.Class, - Properties: map[string]interface{}{ - "name": tenantName, - }, - Tenant: tenantName, - }, - { - ID: "831ae1d0-f441-44b1-bb2a-46548048e26f", - Class: testClass.Class, - Properties: map[string]interface{}{ - "name": tenantName, - }, - Tenant: tenantName, - }, - { - ID: "6f3363e0-c0a0-4618-bf1f-b6cad9cdff59", - Class: testClass.Class, - Properties: map[string]interface{}{ - "name": tenantName, - }, - Tenant: tenantName, - }, - } - - defer func() { - helper.DeleteClass(t, testClass.Class) - }() - - helper.CreateClass(t, &testClass) - helper.CreateTenants(t, testClass.Class, []*models.Tenant{{Name: tenantName}}) - helper.CreateObjectsBatch(t, tenantObjects) - - for _, obj := range tenantObjects { - resp, err := helper.TenantObject(t, obj.Class, obj.ID, tenantName) - require.Nil(t, err) - assert.Equal(t, obj.ID, resp.ID) - assert.Equal(t, obj.Class, resp.Class) - assert.Equal(t, obj.Properties, resp.Properties) - } - - query := fmt.Sprintf(`{Get{%s{_additional{id}}}}`, testClass.Class) - result, err := graphqlhelper.QueryGraphQL(t, helper.RootAuth, "", query, nil) - require.Nil(t, err) - require.Len(t, result.Errors, 1) - assert.Nil(t, result.Data["Get"].(map[string]interface{})[testClass.Class]) - msg := fmt.Sprintf(`explorer: list class: search: object search at index %s: `, - strings.ToLower(testClass.Class)) + - fmt.Sprintf(`class %s has multi-tenancy enabled, but request was without tenant`, testClass.Class) - assert.Equal(t, result.Errors[0].Message, msg) -} diff --git a/test/acceptance/multi_tenancy/head_tenant_objects_test.go b/test/acceptance/multi_tenancy/head_tenant_objects_test.go deleted file mode 100644 index 8a0a85eb2759705a7b1b4f861bc91ccec15af187..0000000000000000000000000000000000000000 --- a/test/acceptance/multi_tenancy/head_tenant_objects_test.go +++ /dev/null @@ -1,96 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/test/helper" -) - -func TestHeadTenantObjects(t *testing.T) { - testClass := models.Class{ - Class: "MultiTenantClass", - MultiTenancyConfig: &models.MultiTenancyConfig{ - Enabled: true, - }, - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - }, - }, - } - tenantNames := []string{ - "Tenant1", "Tenant2", "Tenant3", - } - tenantObjects := []*models.Object{ - { - ID: "0927a1e0-398e-4e76-91fb-04a7a8f0405c", - Class: testClass.Class, - Properties: map[string]interface{}{ - "name": tenantNames[0], - }, - Tenant: tenantNames[0], - }, - { - ID: "831ae1d0-f441-44b1-bb2a-46548048e26f", - Class: testClass.Class, - Properties: map[string]interface{}{ - "name": tenantNames[1], - }, - Tenant: tenantNames[1], - }, - { - ID: "6f3363e0-c0a0-4618-bf1f-b6cad9cdff59", - Class: testClass.Class, - Properties: map[string]interface{}{ - "name": tenantNames[2], - }, - Tenant: tenantNames[2], - }, - } - - defer func() { - helper.DeleteClass(t, testClass.Class) - }() - - t.Run("create class with multi-tenancy enabled", func(t *testing.T) { - helper.CreateClass(t, &testClass) - }) - - t.Run("create tenants", func(t *testing.T) { - tenants := make([]*models.Tenant, len(tenantNames)) - for i := range tenants { - tenants[i] = &models.Tenant{Name: tenantNames[i]} - } - helper.CreateTenants(t, testClass.Class, tenants) - }) - - t.Run("add tenant objects", func(t *testing.T) { - for _, obj := range tenantObjects { - helper.CreateObject(t, obj) - } - }) - - t.Run("head tenant objects", func(t *testing.T) { - for i, obj := range tenantObjects { - exists, err := helper.TenantObjectExists(t, obj.Class, obj.ID, tenantNames[i]) - require.Nil(t, err) - assert.True(t, exists) - } - }) -} diff --git a/test/acceptance/multi_tenancy/patch_tenant_objects_test.go b/test/acceptance/multi_tenancy/patch_tenant_objects_test.go deleted file mode 100644 index 436e1d0cb4c9204cda5a8d3583b918721e58a92c..0000000000000000000000000000000000000000 --- a/test/acceptance/multi_tenancy/patch_tenant_objects_test.go +++ /dev/null @@ -1,199 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "fmt" - "net/http" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/client/objects" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/test/helper" -) - -func TestPatchTenantObjects(t *testing.T) { - mutableProp := "mutableProp" - testClass := models.Class{ - Class: "MultiTenantClass", - MultiTenancyConfig: &models.MultiTenancyConfig{ - Enabled: true, - }, - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - }, { - Name: mutableProp, - DataType: schema.DataTypeText.PropString(), - }, - }, - } - tenantNames := []string{ - "Tenant1", "Tenant2", "Tenant3", - } - tenantObjects := []*models.Object{ - { - ID: "0927a1e0-398e-4e76-91fb-04a7a8f0405c", - Class: testClass.Class, - Properties: map[string]interface{}{ - "name": tenantNames[0], - mutableProp: "obj#0", - }, - Tenant: tenantNames[0], - }, - { - ID: "831ae1d0-f441-44b1-bb2a-46548048e26f", - Class: testClass.Class, - Properties: map[string]interface{}{ - "name": tenantNames[1], - mutableProp: "obj#1", - }, - Tenant: tenantNames[1], - }, - { - ID: "6f3363e0-c0a0-4618-bf1f-b6cad9cdff59", - Class: testClass.Class, - Properties: map[string]interface{}{ - "name": tenantNames[2], - mutableProp: "obj#2", - }, - Tenant: tenantNames[2], - }, - } - - defer func() { - helper.DeleteClass(t, testClass.Class) - }() - - t.Run("create class with multi-tenancy enabled", func(t *testing.T) { - helper.CreateClass(t, &testClass) - }) - - t.Run("create tenants", func(t *testing.T) { - tenants := make([]*models.Tenant, len(tenantNames)) - for i := range tenants { - tenants[i] = &models.Tenant{Name: tenantNames[i]} - } - helper.CreateTenants(t, testClass.Class, tenants) - }) - - t.Run("add tenant objects", func(t *testing.T) { - for _, obj := range tenantObjects { - helper.CreateObject(t, obj) - } - - t.Run("verify tenant object creation", func(t *testing.T) { - for i, obj := range tenantObjects { - resp, err := helper.TenantObject(t, obj.Class, obj.ID, tenantNames[i]) - require.Nil(t, err) - require.Equal(t, obj.ID, resp.ID) - require.Equal(t, obj.Class, resp.Class) - require.Equal(t, obj.Properties, resp.Properties) - } - }) - }) - - t.Run("patch tenant objects", func(t *testing.T) { - for i, obj := range tenantObjects { - mut := obj.Properties.(map[string]interface{})[mutableProp] - toUpdate := &models.Object{ - Class: testClass.Class, - ID: obj.ID, - Properties: map[string]interface{}{ - "name": tenantNames[i], - mutableProp: fmt.Sprintf("%s--patched", mut), - }, - Tenant: tenantNames[i], - } - helper.PatchObject(t, toUpdate) - } - - t.Run("assert tenant object updates", func(t *testing.T) { - for i, obj := range tenantObjects { - resp, err := helper.TenantObject(t, obj.Class, obj.ID, tenantNames[i]) - require.Nil(t, err) - require.Equal(t, obj.ID, resp.ID) - require.Equal(t, obj.Class, resp.Class) - expectedProps := obj.Properties.(map[string]interface{}) - expectedProps[mutableProp] = fmt.Sprintf("%s--patched", expectedProps[mutableProp]) - require.Equal(t, expectedProps, resp.Properties) - } - }) - }) -} - -func TestPatchTenantObjects_ChangeTenant(t *testing.T) { - className := "MultiTenantClassPatch" - tenantName := "Tenant1" - testClass := models.Class{ - Class: className, - MultiTenancyConfig: &models.MultiTenancyConfig{ - Enabled: true, - }, - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - }, - }, - } - tenantObject := models.Object{ - ID: "0927a1e0-398e-4e76-91fb-04a7a8f0405c", - Class: className, - Properties: map[string]interface{}{ - "name": tenantName, - }, - Tenant: tenantName, - } - - defer func() { - helper.DeleteClass(t, className) - }() - - t.Run("create class with multi-tenancy enabled", func(t *testing.T) { - helper.CreateClass(t, &testClass) - helper.CreateTenants(t, className, []*models.Tenant{{Name: tenantName}}) - }) - - t.Run("add tenant object", func(t *testing.T) { - params := objects.NewObjectsCreateParams(). - WithBody(&tenantObject) - _, err := helper.Client(t).Objects.ObjectsCreate(params, nil) - require.Nil(t, err) - }) - - t.Run("patch tenant object", func(t *testing.T) { - toUpdate := models.Object{ - Class: testClass.Class, - ID: tenantObject.ID, - Properties: map[string]interface{}{ - "name": "updatedTenantName", - }, - Tenant: "updatedTenantName", - } - params := objects.NewObjectsClassPatchParams().WithClassName(toUpdate.Class). - WithID(toUpdate.ID).WithBody(&toUpdate) - _, err := helper.Client(t).Objects.ObjectsClassPatch(params, nil) - require.NotNil(t, err) // tenant does not exist - parsedErr, ok := err.(*objects.ObjectsClassPatchUnprocessableEntity) - require.True(t, ok) - require.NotNil(t, parsedErr.Payload.Error) - require.Len(t, parsedErr.Payload.Error, 1) - assert.Contains(t, err.Error(), fmt.Sprint(http.StatusUnprocessableEntity)) - expected := "\"updatedTenantName\"" - assert.Contains(t, parsedErr.Payload.Error[0].Message, expected) - }) -} diff --git a/test/acceptance/multi_tenancy/tenant_objects_reference_test.go b/test/acceptance/multi_tenancy/tenant_objects_reference_test.go deleted file mode 100644 index d51a37a56b92d1110b6b73c6806738e1aa2806cf..0000000000000000000000000000000000000000 --- a/test/acceptance/multi_tenancy/tenant_objects_reference_test.go +++ /dev/null @@ -1,180 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/test/helper" -) - -func TestTenantObjectsReference(t *testing.T) { - className := "MultiTenantClass" - mutableProp := "mutableProp" - refProp := "refProp" - testClass := models.Class{ - Class: className, - MultiTenancyConfig: &models.MultiTenancyConfig{ - Enabled: true, - }, - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - }, - { - Name: mutableProp, - DataType: schema.DataTypeText.PropString(), - }, - { - Name: refProp, - DataType: []string{className}, - }, - }, - } - tenantNames := []string{ - "Tenant1", "Tenant2", "Tenant3", - } - tenantObjects := []*models.Object{ - { - ID: "0927a1e0-398e-4e76-91fb-04a7a8f0405c", - Class: className, - Properties: map[string]interface{}{ - "name": tenantNames[0], - mutableProp: "obj#0", - }, - Tenant: tenantNames[0], - }, - { - ID: "831ae1d0-f441-44b1-bb2a-46548048e26f", - Class: className, - Properties: map[string]interface{}{ - "name": tenantNames[1], - mutableProp: "obj#1", - }, - Tenant: tenantNames[1], - }, - { - ID: "6f3363e0-c0a0-4618-bf1f-b6cad9cdff59", - Class: className, - Properties: map[string]interface{}{ - "name": tenantNames[2], - mutableProp: "obj#2", - }, - Tenant: tenantNames[2], - }, - } - tenantRefs := []*models.Object{ - { - ID: "169b62a7-ef1c-481d-8fb0-27f11716bde7", - Class: className, - Properties: map[string]interface{}{ - "name": tenantNames[0], - mutableProp: "ref#0", - }, - Tenant: tenantNames[0], - }, - { - ID: "4d78424d-f7bd-479b-bd8a-52510e2db0fd", - Class: className, - Properties: map[string]interface{}{ - "name": tenantNames[1], - mutableProp: "ref#1", - }, - Tenant: tenantNames[1], - }, - { - ID: "c1db0a06-d5f9-4f77-aa3c-08a44f16e358", - Class: className, - Properties: map[string]interface{}{ - "name": tenantNames[2], - mutableProp: "ref#2", - }, - Tenant: tenantNames[2], - }, - } - - defer func() { - helper.DeleteClass(t, className) - }() - - t.Run("create class with multi-tenancy enabled", func(t *testing.T) { - helper.CreateClass(t, &testClass) - }) - - t.Run("create tenants", func(t *testing.T) { - tenants := make([]*models.Tenant, len(tenantNames)) - for i := range tenants { - tenants[i] = &models.Tenant{Name: tenantNames[i]} - } - helper.CreateTenants(t, className, tenants) - }) - - t.Run("add tenant objects", func(t *testing.T) { - for i, obj := range tenantObjects { - helper.CreateObject(t, obj) - helper.CreateObject(t, tenantRefs[i]) - } - - t.Run("verify tenant object creation", func(t *testing.T) { - for i, obj := range tenantObjects { - resp, err := helper.TenantObject(t, obj.Class, obj.ID, tenantNames[i]) - require.Nil(t, err) - require.Equal(t, obj.ID, resp.ID) - require.Equal(t, obj.Class, resp.Class) - require.Equal(t, obj.Properties, resp.Properties) - } - }) - }) - - t.Run("add tenant object references", func(t *testing.T) { - for i, obj := range tenantObjects { - ref := &models.SingleRef{Beacon: helper.NewBeacon(className, tenantRefs[i].ID)} - helper.AddReferenceTenant(t, obj, ref, refProp, tenantNames[i]) - } - - t.Run("assert tenant object references", func(t *testing.T) { - for i, obj := range tenantObjects { - resp, err := helper.TenantObject(t, obj.Class, obj.ID, tenantNames[i]) - require.Nil(t, err) - require.Equal(t, obj.ID, resp.ID) - require.Equal(t, obj.Class, resp.Class) - refs := resp.Properties.(map[string]interface{})[refProp].([]interface{}) - require.Len(t, refs, 1) - expectedBeacon := helper.NewBeacon(className, tenantRefs[i].ID).String() - assert.Equal(t, expectedBeacon, refs[0].(map[string]interface{})["beacon"]) - } - }) - }) - - t.Run("delete tenant object references", func(Z *testing.T) { - for i, obj := range tenantObjects { - ref := &models.SingleRef{Beacon: helper.NewBeacon(className, tenantRefs[i].ID)} - helper.DeleteReferenceTenant(t, obj, ref, refProp, tenantNames[i]) - } - - t.Run("assert tenant object references", func(t *testing.T) { - for i, obj := range tenantObjects { - resp, err := helper.TenantObject(t, obj.Class, obj.ID, tenantNames[i]) - require.Nil(t, err) - require.Equal(t, obj.ID, resp.ID) - require.Equal(t, obj.Class, resp.Class) - refs := resp.Properties.(map[string]interface{})[refProp].([]interface{}) - require.Len(t, refs, 0) - } - }) - }) -} diff --git a/test/acceptance/multi_tenancy/update_tenant_objects_test.go b/test/acceptance/multi_tenancy/update_tenant_objects_test.go deleted file mode 100644 index 3d80e3f052e97ff67396c56e912a7b3130487a02..0000000000000000000000000000000000000000 --- a/test/acceptance/multi_tenancy/update_tenant_objects_test.go +++ /dev/null @@ -1,199 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "fmt" - "net/http" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/client/objects" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/test/helper" -) - -func TestUpdateTenantObjects(t *testing.T) { - mutableProp := "mutableProp" - testClass := models.Class{ - Class: "MultiTenantClass", - MultiTenancyConfig: &models.MultiTenancyConfig{ - Enabled: true, - }, - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - }, { - Name: mutableProp, - DataType: schema.DataTypeText.PropString(), - }, - }, - } - tenantNames := []string{ - "Tenant1", "Tenant2", "Tenant3", - } - tenantObjects := []*models.Object{ - { - ID: "0927a1e0-398e-4e76-91fb-04a7a8f0405c", - Class: testClass.Class, - Properties: map[string]interface{}{ - "name": tenantNames[0], - mutableProp: "obj#0", - }, - Tenant: tenantNames[0], - }, - { - ID: "831ae1d0-f441-44b1-bb2a-46548048e26f", - Class: testClass.Class, - Properties: map[string]interface{}{ - "name": tenantNames[1], - mutableProp: "obj#1", - }, - Tenant: tenantNames[1], - }, - { - ID: "6f3363e0-c0a0-4618-bf1f-b6cad9cdff59", - Class: testClass.Class, - Properties: map[string]interface{}{ - "name": tenantNames[2], - mutableProp: "obj#2", - }, - Tenant: tenantNames[2], - }, - } - - defer func() { - helper.DeleteClass(t, testClass.Class) - }() - - t.Run("create class with multi-tenancy enabled", func(t *testing.T) { - helper.CreateClass(t, &testClass) - }) - - t.Run("create tenants", func(t *testing.T) { - tenants := make([]*models.Tenant, len(tenantNames)) - for i := range tenants { - tenants[i] = &models.Tenant{Name: tenantNames[i]} - } - helper.CreateTenants(t, testClass.Class, tenants) - }) - - t.Run("add tenant objects", func(t *testing.T) { - for _, obj := range tenantObjects { - helper.CreateObject(t, obj) - } - - t.Run("verify tenant object creation", func(t *testing.T) { - for i, obj := range tenantObjects { - resp, err := helper.TenantObject(t, obj.Class, obj.ID, tenantNames[i]) - require.Nil(t, err) - require.Equal(t, obj.ID, resp.ID) - require.Equal(t, obj.Class, resp.Class) - require.Equal(t, obj.Properties, resp.Properties) - } - }) - }) - - t.Run("update tenant objects", func(t *testing.T) { - for i, obj := range tenantObjects { - mut := obj.Properties.(map[string]interface{})[mutableProp] - toUpdate := &models.Object{ - Class: testClass.Class, - ID: obj.ID, - Properties: map[string]interface{}{ - "name": tenantNames[i], - mutableProp: fmt.Sprintf("%s--updated", mut), - }, - Tenant: tenantNames[i], - } - helper.UpdateObject(t, toUpdate) - } - - t.Run("assert tenant object updates", func(t *testing.T) { - for i, obj := range tenantObjects { - resp, err := helper.TenantObject(t, obj.Class, obj.ID, tenantNames[i]) - require.Nil(t, err) - require.Equal(t, obj.ID, resp.ID) - require.Equal(t, obj.Class, resp.Class) - expectedProps := obj.Properties.(map[string]interface{}) - expectedProps[mutableProp] = fmt.Sprintf("%s--updated", expectedProps[mutableProp]) - require.Equal(t, expectedProps, resp.Properties) - } - }) - }) -} - -func TestUpdateTenantObjects_UpdateTenant(t *testing.T) { - className := "MultiTenantClass" - tenantName := "Tenant1" - testClass := models.Class{ - Class: className, - MultiTenancyConfig: &models.MultiTenancyConfig{ - Enabled: true, - }, - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - }, - }, - } - tenantObject := models.Object{ - ID: "0927a1e0-398e-4e76-91fb-04a7a8f0405c", - Class: className, - Properties: map[string]interface{}{ - "name": tenantName, - }, - Tenant: tenantName, - } - - defer func() { - helper.DeleteClass(t, className) - }() - - t.Run("create class with multi-tenancy enabled", func(t *testing.T) { - helper.CreateClass(t, &testClass) - helper.CreateTenants(t, className, []*models.Tenant{{Name: tenantName}}) - }) - - t.Run("add tenant object", func(t *testing.T) { - params := objects.NewObjectsCreateParams(). - WithBody(&tenantObject) - _, err := helper.Client(t).Objects.ObjectsCreate(params, nil) - require.Nil(t, err) - }) - - t.Run("update tenant object", func(t *testing.T) { - toUpdate := models.Object{ - Class: testClass.Class, - ID: tenantObject.ID, - Properties: map[string]interface{}{ - "name": "updatedTenantName", - }, - Tenant: "updatedTenantName", - } - params := objects.NewObjectsClassPutParams().WithClassName(toUpdate.Class). - WithID(toUpdate.ID).WithBody(&toUpdate) - _, err := helper.Client(t).Objects.ObjectsClassPut(params, nil) - require.NotNil(t, err) // tenant does not exist - parsedErr, ok := err.(*objects.ObjectsClassPutUnprocessableEntity) - require.True(t, ok) - require.NotNil(t, parsedErr.Payload.Error) - require.Len(t, parsedErr.Payload.Error, 1) - assert.Contains(t, err.Error(), fmt.Sprint(http.StatusUnprocessableEntity)) - expected := "\"updatedTenantName\"" - assert.Contains(t, parsedErr.Payload.Error[0].Message, expected) - }) -} diff --git a/test/acceptance/multi_tenancy/update_tenant_references_test.go b/test/acceptance/multi_tenancy/update_tenant_references_test.go deleted file mode 100644 index ba9df12571a0d462716de68d095722fb50ea3a28..0000000000000000000000000000000000000000 --- a/test/acceptance/multi_tenancy/update_tenant_references_test.go +++ /dev/null @@ -1,134 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "testing" - - "github.com/go-openapi/strfmt" - "github.com/google/uuid" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/test/helper" -) - -func TestTenantObjectsReferenceUpdates(t *testing.T) { - className := "MultiTenantClass" - refProp := "refProp1" - - class := models.Class{ - Class: className, - MultiTenancyConfig: &models.MultiTenancyConfig{Enabled: true}, - Properties: []*models.Property{ - {Name: refProp, DataType: []string{className}}, - }, - } - defer func() { - helper.DeleteClass(t, className) - }() - helper.CreateClass(t, &class) - - tenantNames := []string{"Tenant1", "Tenant2", "Tenant3"} - tenants := make([]*models.Tenant, len(tenantNames)) - for i := range tenants { - tenants[i] = &models.Tenant{Name: tenantNames[i]} - } - helper.CreateTenants(t, className, tenants) - - objs := make([]*models.Object, 0) - for i := 0; i < len(tenantNames)*4; i++ { - obj := &models.Object{ - ID: strfmt.UUID(uuid.New().String()), - Class: className, - Tenant: tenantNames[i%len(tenantNames)], - } - objs = append(objs, obj) - helper.CreateObject(t, obj) - } - - for _, obj := range objs { - ref := &models.SingleRef{Beacon: helper.NewBeacon(className, obj.ID)} - helper.AddReferenceTenant(t, obj, ref, refProp, obj.Tenant) - helper.AddReferenceTenant(t, obj, ref, refProp, obj.Tenant) - } - - for _, obj := range objs { - resp, err := helper.TenantObject(t, obj.Class, obj.ID, obj.Tenant) - require.Nil(t, err) - require.Equal(t, obj.ID, resp.ID) - require.Equal(t, obj.Class, resp.Class) - refs := resp.Properties.(map[string]interface{})[refProp].([]interface{}) - require.Len(t, refs, 2) - expectedBeacon := helper.NewBeacon(className, obj.ID).String() - assert.Equal(t, expectedBeacon, refs[0].(map[string]interface{})["beacon"]) - } - - t.Run("Update reference - single", func(t *testing.T) { - for i, obj := range objs { - j := (i + len(tenantNames)) % len(objs) // always reference the next object with the same tenant - ref := models.MultipleRef{{Beacon: helper.NewBeacon(className, objs[j].ID)}} - helper.UpdateReferenceTenant(t, obj, ref, refProp, obj.Tenant) - } - - for i, obj := range objs { - resp, err := helper.TenantObject(t, obj.Class, obj.ID, obj.Tenant) - require.Nil(t, err) - require.Equal(t, obj.ID, resp.ID) - require.Equal(t, obj.Class, resp.Class) - refs := resp.Properties.(map[string]interface{})[refProp].([]interface{}) - require.Len(t, refs, 1) - j := (i + len(tenantNames)) % len(objs) // always reference the next object with the same tenant - expectedBeacon := helper.NewBeacon(className, objs[j].ID).String() - assert.Equal(t, expectedBeacon, refs[0].(map[string]interface{})["beacon"]) - } - }) - - t.Run("Update reference - multiple", func(t *testing.T) { - for i, obj := range objs { - ref := models.MultipleRef{} - j := (i + 2*len(tenantNames)) % len(objs) - ref = append(ref, &models.SingleRef{Beacon: helper.NewBeacon(className, objs[j].ID)}) - ref = append(ref, &models.SingleRef{Beacon: helper.NewBeacon(className, objs[j].ID)}) - ref = append(ref, &models.SingleRef{Beacon: helper.NewBeacon(className, objs[j].ID)}) - helper.UpdateReferenceTenant(t, obj, ref, refProp, obj.Tenant) - } - - for i, obj := range objs { - resp, err := helper.TenantObject(t, obj.Class, obj.ID, obj.Tenant) - require.Nil(t, err) - require.Equal(t, obj.ID, resp.ID) - require.Equal(t, obj.Class, resp.Class) - refs := resp.Properties.(map[string]interface{})[refProp].([]interface{}) - require.Len(t, refs, 3) - j := (i + 2*len(tenantNames)) % len(objs) // always reference the next object with the same tenant - expectedBeacon := helper.NewBeacon(className, objs[j].ID).String() - assert.Equal(t, expectedBeacon, refs[0].(map[string]interface{})["beacon"]) - } - }) - - t.Run("Update reference - empty", func(t *testing.T) { - for _, obj := range objs { - ref := models.MultipleRef{} - helper.UpdateReferenceTenant(t, obj, ref, refProp, obj.Tenant) - } - - for _, obj := range objs { - resp, err := helper.TenantObject(t, obj.Class, obj.ID, obj.Tenant) - require.Nil(t, err) - require.Equal(t, obj.ID, resp.ID) - require.Equal(t, obj.Class, resp.Class) - refs := resp.Properties.(map[string]interface{})[refProp].([]interface{}) - require.Len(t, refs, 0) - } - }) -} diff --git a/test/acceptance/nodes/nodes_api_test.go b/test/acceptance/nodes/nodes_api_test.go deleted file mode 100644 index f1c23bbf72b6a6fd219a7e19b9b3e1b8e6788fe3..0000000000000000000000000000000000000000 --- a/test/acceptance/nodes/nodes_api_test.go +++ /dev/null @@ -1,415 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/go-openapi/strfmt" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/client/batch" - "github.com/weaviate/weaviate/client/meta" - "github.com/weaviate/weaviate/client/nodes" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/verbosity" - "github.com/weaviate/weaviate/test/docker" - "github.com/weaviate/weaviate/test/helper" - "github.com/weaviate/weaviate/test/helper/sample-schema/books" - "github.com/weaviate/weaviate/test/helper/sample-schema/documents" - "github.com/weaviate/weaviate/test/helper/sample-schema/multishard" -) - -func Test_NodesAPI(t *testing.T) { - t.Run("empty DB", func(t *testing.T) { - meta, err := helper.Client(t).Meta.MetaGet(meta.NewMetaGetParams(), nil) - require.Nil(t, err) - assert.NotNil(t, meta.GetPayload()) - - assertions := func(t *testing.T, nodeStatus *models.NodeStatus) { - require.NotNil(t, nodeStatus) - assert.Equal(t, models.NodeStatusStatusHEALTHY, *nodeStatus.Status) - assert.True(t, len(nodeStatus.Name) > 0) - assert.True(t, nodeStatus.GitHash != "" && nodeStatus.GitHash != "unknown") - assert.Equal(t, meta.Payload.Version, nodeStatus.Version) - assert.Empty(t, nodeStatus.Shards) - require.NotNil(t, nodeStatus.Stats) - assert.Equal(t, int64(0), nodeStatus.Stats.ObjectCount) - assert.Equal(t, int64(0), nodeStatus.Stats.ShardCount) - } - - testStatusResponse(t, assertions, nil, "") - }) - - t.Run("DB with Books (1 class ,1 shard configuration, 1 node)", func(t *testing.T) { - booksClass := books.ClassContextionaryVectorizer() - helper.CreateClass(t, booksClass) - defer helper.DeleteClass(t, booksClass.Class) - - for _, book := range books.Objects() { - helper.CreateObject(t, book) - helper.AssertGetObjectEventually(t, book.Class, book.ID) - } - - minimalAssertions := func(t *testing.T, nodeStatus *models.NodeStatus) { - require.NotNil(t, nodeStatus) - assert.Equal(t, models.NodeStatusStatusHEALTHY, *nodeStatus.Status) - assert.True(t, len(nodeStatus.Name) > 0) - assert.True(t, nodeStatus.GitHash != "" && nodeStatus.GitHash != "unknown") - require.NotNil(t, nodeStatus.Stats) - assert.Equal(t, int64(3), nodeStatus.Stats.ObjectCount) - assert.Equal(t, int64(1), nodeStatus.Stats.ShardCount) - } - - verboseAssertions := func(t *testing.T, nodeStatus *models.NodeStatus) { - require.Len(t, nodeStatus.Shards, 1) - shard := nodeStatus.Shards[0] - assert.True(t, len(shard.Name) > 0) - assert.Equal(t, booksClass.Class, shard.Class) - assert.Equal(t, int64(3), shard.ObjectCount) - } - - testStatusResponse(t, minimalAssertions, verboseAssertions, "") - }) - - t.Run("DB with MultiShard (1 class, 2 shards configuration, 1 node)", func(t *testing.T) { - multiShardClass := multishard.ClassContextionaryVectorizer() - helper.CreateClass(t, multiShardClass) - defer helper.DeleteClass(t, multiShardClass.Class) - - for _, multiShard := range multishard.Objects() { - helper.CreateObject(t, multiShard) - helper.AssertGetObjectEventually(t, multiShard.Class, multiShard.ID) - } - - minimalAssertions := func(t *testing.T, nodeStatus *models.NodeStatus) { - require.NotNil(t, nodeStatus) - assert.Equal(t, models.NodeStatusStatusHEALTHY, *nodeStatus.Status) - assert.True(t, len(nodeStatus.Name) > 0) - assert.True(t, nodeStatus.GitHash != "" && nodeStatus.GitHash != "unknown") - require.NotNil(t, nodeStatus.Stats) - assert.Equal(t, int64(3), nodeStatus.Stats.ObjectCount) - assert.Equal(t, int64(2), nodeStatus.Stats.ShardCount) - } - - verboseAsssertions := func(t *testing.T, nodeStatus *models.NodeStatus) { - assert.Len(t, nodeStatus.Shards, 2) - for _, shard := range nodeStatus.Shards { - assert.True(t, len(shard.Name) > 0) - assert.Equal(t, multiShardClass.Class, shard.Class) - assert.GreaterOrEqual(t, shard.ObjectCount, int64(0)) - } - } - - testStatusResponse(t, minimalAssertions, verboseAsssertions, "") - }) - - t.Run("with class name: DB with Books and Documents, 1 shard, 1 node", func(t *testing.T) { - booksClass := books.ClassContextionaryVectorizer() - helper.CreateClass(t, booksClass) - defer helper.DeleteClass(t, booksClass.Class) - - t.Run("insert and check books", func(t *testing.T) { - for _, book := range books.Objects() { - helper.CreateObject(t, book) - helper.AssertGetObjectEventually(t, book.Class, book.ID) - } - - assertions := func(t *testing.T, nodeStatus *models.NodeStatus) { - require.NotNil(t, nodeStatus.Stats) - assert.Equal(t, int64(3), nodeStatus.Stats.ObjectCount) - assert.Equal(t, int64(1), nodeStatus.Stats.ShardCount) - } - - testStatusResponse(t, assertions, nil, "") - }) - - t.Run("insert and check documents", func(t *testing.T) { - docsClasses := documents.ClassesContextionaryVectorizer(false) - helper.CreateClass(t, docsClasses[0]) - helper.CreateClass(t, docsClasses[1]) - defer helper.DeleteClass(t, docsClasses[0].Class) - defer helper.DeleteClass(t, docsClasses[1].Class) - - for _, doc := range documents.Objects() { - helper.CreateObject(t, doc) - helper.AssertGetObjectEventually(t, doc.Class, doc.ID) - } - - docsClass := docsClasses[0] - - minimalAssertions := func(t *testing.T, nodeStatus *models.NodeStatus) { - assert.Equal(t, models.NodeStatusStatusHEALTHY, *nodeStatus.Status) - assert.True(t, len(nodeStatus.Name) > 0) - assert.True(t, nodeStatus.GitHash != "" && nodeStatus.GitHash != "unknown") - require.NotNil(t, nodeStatus.Stats) - assert.Equal(t, int64(2), nodeStatus.Stats.ObjectCount) - assert.Equal(t, int64(1), nodeStatus.Stats.ShardCount) - } - - verboseAssertions := func(t *testing.T, nodeStatus *models.NodeStatus) { - assert.Len(t, nodeStatus.Shards, 1) - shard := nodeStatus.Shards[0] - assert.True(t, len(shard.Name) > 0) - assert.Equal(t, docsClass.Class, shard.Class) - assert.Equal(t, int64(2), shard.ObjectCount) - } - - testStatusResponse(t, minimalAssertions, verboseAssertions, docsClass.Class) - }) - }) - - // This test prevents a regression of - // https://github.com/weaviate/weaviate/issues/2454 - t.Run("validate count with updates", func(t *testing.T) { - booksClass := books.ClassContextionaryVectorizer() - helper.CreateClass(t, booksClass) - defer helper.DeleteClass(t, booksClass.Class) - - _, err := helper.BatchClient(t).BatchObjectsCreate( - batch.NewBatchObjectsCreateParams().WithBody(batch.BatchObjectsCreateBody{ - Objects: []*models.Object{ - { - ID: strfmt.UUID("2D0D3E3B-54B2-48D4-BFE0-4BE2C060110E"), - Class: booksClass.Class, - Properties: map[string]interface{}{ - "title": "A book that changes", - "description": "First iteration", - }, - }, - }, - }), nil) - require.Nil(t, err) - - // Note that this is the same ID as before, so this is an update!! - _, err = helper.BatchClient(t).BatchObjectsCreate( - batch.NewBatchObjectsCreateParams().WithBody(batch.BatchObjectsCreateBody{ - Objects: []*models.Object{ - { - ID: strfmt.UUID("2D0D3E3B-54B2-48D4-BFE0-4BE2C060110E"), - Class: booksClass.Class, - Properties: map[string]interface{}{ - "title": "A book that changes", - "description": "A new (second) iteration", - }, - }, - }, - }), nil) - require.Nil(t, err) - - assertions := func(t *testing.T, nodeStatus *models.NodeStatus) { - require.NotNil(t, nodeStatus.Stats) - assert.Equal(t, int64(1), nodeStatus.Stats.ObjectCount) - } - - testStatusResponse(t, assertions, nil, "") - }) -} - -func TestNodesApi_Compression_AsyncIndexing(t *testing.T) { - ctx := context.Background() - compose, err := docker.New(). - WithWeaviate(). - WithText2VecContextionary(). - WithWeaviateEnv("ASYNC_INDEXING", "true"). - Start(ctx) - require.NoError(t, err) - defer func() { - require.NoError(t, compose.Terminate(ctx)) - }() - - defer helper.SetupClient(fmt.Sprintf("%s:%s", helper.ServerHost, helper.ServerPort)) - helper.SetupClient(compose.GetWeaviate().URI()) - - t.Run("validate flat compression status", func(t *testing.T) { - booksClass := books.ClassContextionaryVectorizer() - booksClass.VectorIndexType = "flat" - booksClass.VectorIndexConfig = map[string]interface{}{ - "bq": map[string]interface{}{ - "enabled": true, - }, - } - helper.CreateClass(t, booksClass) - defer helper.DeleteClass(t, booksClass.Class) - - t.Run("check compressed true", func(t *testing.T) { - verbose := "verbose" - params := nodes.NewNodesGetParams().WithOutput(&verbose) - resp, err := helper.Client(t).Nodes.NodesGet(params, nil) - require.Nil(t, err) - - nodeStatusResp := resp.GetPayload() - require.NotNil(t, nodeStatusResp) - - nodes := nodeStatusResp.Nodes - require.NotNil(t, nodes) - require.Len(t, nodes, 1) - - nodeStatus := nodes[0] - require.NotNil(t, nodeStatus) - - require.True(t, nodeStatus.Shards[0].Compressed) - }) - }) - - t.Run("validate hnsw pq async compression", func(t *testing.T) { - booksClass := books.ClassContextionaryVectorizer() - booksClass.VectorIndexConfig = map[string]interface{}{ - "pq": map[string]interface{}{ - "trainingLimit": 256, - "enabled": true, - "segments": 1, - }, - } - helper.CreateClass(t, booksClass) - defer helper.DeleteClass(t, booksClass.Class) - - t.Run("check compressed initially false", func(t *testing.T) { - verbose := "verbose" - params := nodes.NewNodesGetParams().WithOutput(&verbose) - resp, err := helper.Client(t).Nodes.NodesGet(params, nil) - require.Nil(t, err) - - nodeStatusResp := resp.GetPayload() - require.NotNil(t, nodeStatusResp) - - nodes := nodeStatusResp.Nodes - require.NotNil(t, nodes) - require.Len(t, nodes, 1) - - nodeStatus := nodes[0] - require.NotNil(t, nodeStatus) - - require.False(t, nodeStatus.Shards[0].Compressed) - }) - - t.Run("load data for pq", func(t *testing.T) { - num := 1024 - objects := make([]*models.Object, num) - - for i := 0; i < num; i++ { - objects[i] = &models.Object{ - Class: booksClass.Class, - Vector: []float32{float32(i % 32), float32(i), 3.0, 4.0}, - } - } - - _, err := helper.BatchClient(t).BatchObjectsCreate( - batch.NewBatchObjectsCreateParams().WithBody(batch.BatchObjectsCreateBody{ - Objects: objects, - }, - ), nil) - require.Nil(t, err) - }) - - t.Run("check eventually compressed if async enabled", func(t *testing.T) { - checkThunk := func() interface{} { - verbose := "verbose" - params := nodes.NewNodesGetParams().WithOutput(&verbose) - resp, err := helper.Client(t).Nodes.NodesGet(params, nil) - require.Nil(t, err) - - nodeStatusResp := resp.GetPayload() - require.NotNil(t, nodeStatusResp) - - nodes := nodeStatusResp.Nodes - require.NotNil(t, nodes) - require.Len(t, nodes, 1) - - nodeStatus := nodes[0] - require.NotNil(t, nodeStatus) - return nodeStatus.Shards[0].Compressed - } - - helper.AssertEventuallyEqualWithFrequencyAndTimeout(t, true, checkThunk, 100*time.Millisecond, 10*time.Second) - }) - }) -} - -func TestNodesApi_Compression_SyncIndexing(t *testing.T) { - t.Run("validate flat compression status", func(t *testing.T) { - booksClass := books.ClassContextionaryVectorizer() - booksClass.VectorIndexType = "flat" - booksClass.VectorIndexConfig = map[string]interface{}{ - "bq": map[string]interface{}{ - "enabled": true, - }, - } - helper.CreateClass(t, booksClass) - defer helper.DeleteClass(t, booksClass.Class) - - t.Run("check compressed true", func(t *testing.T) { - verbose := "verbose" - params := nodes.NewNodesGetParams().WithOutput(&verbose) - resp, err := helper.Client(t).Nodes.NodesGet(params, nil) - require.Nil(t, err) - - nodeStatusResp := resp.GetPayload() - require.NotNil(t, nodeStatusResp) - - nodes := nodeStatusResp.Nodes - require.NotNil(t, nodes) - require.Len(t, nodes, 1) - - nodeStatus := nodes[0] - require.NotNil(t, nodeStatus) - - require.True(t, nodeStatus.Shards[0].Compressed) - }) - }) -} - -func testStatusResponse(t *testing.T, minimalAssertions, verboseAssertions func(*testing.T, *models.NodeStatus), - class string, -) { - minimal, verbose := verbosity.OutputMinimal, verbosity.OutputVerbose - - commonTests := func(resp *nodes.NodesGetOK) { - require.NotNil(t, resp.Payload) - nodes := resp.Payload.Nodes - require.NotNil(t, nodes) - require.Len(t, nodes, 1) - minimalAssertions(t, nodes[0]) - } - - t.Run("minimal", func(t *testing.T) { - payload, err := getNodesStatus(t, minimal, class) - require.Nil(t, err) - commonTests(&nodes.NodesGetOK{Payload: payload}) - }) - - if verboseAssertions != nil { - t.Run("verbose", func(t *testing.T) { - payload, err := getNodesStatus(t, verbose, class) - require.Nil(t, err) - commonTests(&nodes.NodesGetOK{Payload: payload}) - // If commonTests pass, resp.Nodes[0] != nil - verboseAssertions(t, payload.Nodes[0]) - }) - } -} - -func getNodesStatus(t *testing.T, output, class string) (payload *models.NodesStatusResponse, err error) { - if class != "" { - params := nodes.NewNodesGetClassParams().WithOutput(&output).WithClassName(class) - body, clientErr := helper.Client(t).Nodes.NodesGetClass(params, nil) - payload, err = body.Payload, clientErr - } else { - params := nodes.NewNodesGetParams().WithOutput(&output) - body, clientErr := helper.Client(t).Nodes.NodesGet(params, nil) - payload, err = body.Payload, clientErr - } - return -} diff --git a/test/acceptance/objects/additional_props_test.go b/test/acceptance/objects/additional_props_test.go deleted file mode 100644 index 737bde4592428ec9d48cc16baf8a180626042774..0000000000000000000000000000000000000000 --- a/test/acceptance/objects/additional_props_test.go +++ /dev/null @@ -1,95 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "encoding/json" - "testing" - - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/client/objects" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/test/helper" -) - -func searchNeighbors(t *testing.T) { - listParams := objects.NewObjectsListParams().WithInclude(ptString("nearestNeighbors")) - res, err := helper.Client(t).Objects.ObjectsList(listParams, nil) - require.Nil(t, err, "should not error") - - extractNeighbor := func(in *models.Object) []interface{} { - // marshalling to JSON and back into an untyped map to make sure we assert - // on the actual JSON structure. This way if we accidentally change the - // goswagger generation so it affects both the client and the server in the - // same way, this test should catch it - b, err := json.Marshal(in) - require.Nil(t, err) - - var untyped map[string]interface{} - err = json.Unmarshal(b, &untyped) - require.Nil(t, err) - - return untyped["additional"].(map[string]interface{})["nearestNeighbors"].(map[string]interface{})["neighbors"].([]interface{}) - } - - validateNeighbors(t, extractNeighbor(res.Payload.Objects[0]), extractNeighbor(res.Payload.Objects[1])) -} - -func featureProjection(t *testing.T) { - listParams := objects.NewObjectsListParams().WithInclude(ptString("featureProjection")) - res, err := helper.Client(t).Objects.ObjectsList(listParams, nil) - require.Nil(t, err, "should not error") - - extractProjection := func(in *models.Object) []interface{} { - // marshalling to JSON and back into an untyped map to make sure we assert - // on the actual JSON structure. This way if we accidentally change the - // goswagger generation so it affects both the client and the server in the - // same way, this test should catch it - b, err := json.Marshal(in) - require.Nil(t, err) - - var untyped map[string]interface{} - err = json.Unmarshal(b, &untyped) - require.Nil(t, err) - - return untyped["additional"].(map[string]interface{})["featureProjection"].(map[string]interface{})["vector"].([]interface{}) - } - - validateProjections(t, 2, extractProjection(res.Payload.Objects[0]), extractProjection(res.Payload.Objects[1])) -} - -func ptString(in string) *string { - return &in -} - -func validateNeighbors(t *testing.T, neighborsGroups ...[]interface{}) { - for i, group := range neighborsGroups { - if len(group) == 0 { - t.Fatalf("group %d: length of neighbors is 0", i) - } - - for j, neighbor := range group { - asMap := neighbor.(map[string]interface{}) - if len(asMap["concept"].(string)) == 0 { - t.Fatalf("group %d: element %d: concept has length 0", i, j) - } - } - } -} - -func validateProjections(t *testing.T, dims int, vectors ...[]interface{}) { - for _, vector := range vectors { - if len(vector) != dims { - t.Fatalf("expected feature projection vector to have length 3, got: %d", len(vector)) - } - } -} diff --git a/test/acceptance/objects/auto_schema_test.go b/test/acceptance/objects/auto_schema_test.go deleted file mode 100644 index c627317064d7f2040d3c81b944fc2ebe70940521..0000000000000000000000000000000000000000 --- a/test/acceptance/objects/auto_schema_test.go +++ /dev/null @@ -1,200 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -// Acceptance tests for objects. - -import ( - "encoding/json" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/client/schema" - - "github.com/weaviate/weaviate/client/objects" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/test/helper" -) - -func TestAutoSchemaWithDifferentProperties(t *testing.T) { - // Add two objects with different properties to the same class. With autoschema enabled both should be added and - // the class should have properties form both classes at the end - className := "RandomName234234" - - testCases := []struct { - name string - names []string - }{ - {name: "UpperCase", names: []string{"NonExistingProperty", "OtherNonExistingProperty"}}, - {name: "LowerCase", names: []string{"nonExistingProperty", "otherNonExistingProperty"}}, - } - - for _, test := range testCases { - t.Run(test.name, func(t *testing.T) { - obj1 := &models.Object{ - Class: className, - Properties: map[string]interface{}{ - test.names[0]: "test", - }, - } - params := objects.NewObjectsCreateParams().WithBody(obj1) - resp, err := helper.Client(t).Objects.ObjectsCreate(params, nil) - helper.AssertRequestOk(t, resp, err, nil) - - obj2 := &models.Object{ - Class: className, - Properties: map[string]interface{}{ - test.names[1]: "test", - }, - } - params2 := objects.NewObjectsCreateParams().WithBody(obj2) - resp2, err2 := helper.Client(t).Objects.ObjectsCreate(params2, nil) - helper.AssertRequestOk(t, resp2, err2, nil) - - SchemaParams := schema.NewSchemaDumpParams() - resp3, err3 := helper.Client(t).Schema.SchemaDump(SchemaParams, nil) - helper.AssertRequestOk(t, resp3, err3, nil) - assert.Len(t, resp3.Payload.Classes, 1) - class := resp3.Payload.Classes[0] - assert.Len(t, class.Properties, 2) - props := class.Properties - assert.ElementsMatch(t, []string{props[0].Name, props[1].Name}, []string{"nonExistingProperty", "otherNonExistingProperty"}) - deleteObjectClass(t, className) - }) - } -} - -// run from setup_test.go -func autoSchemaObjects(t *testing.T) { - autoSchemaObjectTestCases := []struct { - // the name of the test - name string - // the example object, with non existent classes and properties. - object func() *models.Object - }{ - { - name: "non existing class", - object: func() *models.Object { - return &models.Object{ - ID: "8e2997f2-1972-4ee2-ad35-5fc704f2893e", - Class: "NonExistingClass", - Properties: map[string]interface{}{ - "testString": "test", - "testNumber": json.Number("1"), - "testDate": "2002-10-02T15:00:00Z", - "testBoolean": true, - "testGeoCoordinates": map[string]interface{}{ - "latitude": json.Number("1.01"), - "longitude": json.Number("1.01"), - }, - "testPhoneNumber": map[string]interface{}{ - "input": "020 1234567", - "defaultCountry": "nl", - }, - "textArray": []string{"a", "b", "c"}, - "intArray": []int{1, 2, 3}, - "numberArray": []int{11.0, 22.0, 33.0}, - }, - } - }, - }, - { - name: "non existing property", - object: func() *models.Object { - return &models.Object{ - Class: "TestObject", - Properties: map[string]interface{}{ - "nonExistingProperty": "test", - }, - } - }, - }, - { - name: "non existing property update class", - object: func() *models.Object { - return &models.Object{ - ID: "8e2997f2-1972-4ee2-ad35-5fc704f2893f", - Class: "TestObject", - Properties: map[string]interface{}{ - "nonExistingDateProperty": "2002-10-02T15:00:00Z", - "nonExistingNumberProperty": json.Number("1"), - }, - } - }, - }, - } - - t.Run("auto schema should create object with missing classes and properties", func(t *testing.T) { - for _, example_ := range autoSchemaObjectTestCases { - t.Run(example_.name, func(t *testing.T) { - example := example_ // Needed; example is updated to point to a new test case. - t.Parallel() - - params := objects.NewObjectsCreateParams().WithBody(example.object()) - resp, err := helper.Client(t).Objects.ObjectsCreate(params, nil) - helper.AssertRequestOk(t, resp, err, nil) - }) - } - }) - - autoSchemaCrossRefTestCases := []struct { - // the name of the test - name string - // the example object, with non existent classes and properties. - object func() *models.Object - }{ - { - name: "non existing cross ref property update class", - object: func() *models.Object { - return &models.Object{ - Class: "TestObject", - Properties: map[string]interface{}{ - "hasNonExistingClass": []interface{}{ - map[string]interface{}{ - "beacon": "weaviate://localhost/8e2997f2-1972-4ee2-ad35-5fc704f2893e", - }, - }, - }, - } - }, - }, - { - name: "non existing cross ref property update class", - object: func() *models.Object { - return &models.Object{ - Class: "TestObject", - Properties: map[string]interface{}{ - "hasNonExistingClassAndTestObject": []interface{}{ - map[string]interface{}{ - "beacon": "weaviate://localhost/8e2997f2-1972-4ee2-ad35-5fc704f2893e", - }, - map[string]interface{}{ - "beacon": "weaviate://localhost/8e2997f2-1972-4ee2-ad35-5fc704f2893f", - }, - }, - }, - } - }, - }, - } - - t.Run("auto schema should create object with missing cross ref properties", func(t *testing.T) { - for _, example_ := range autoSchemaCrossRefTestCases { - t.Run(example_.name, func(t *testing.T) { - example := example_ // Needed; example is updated to point to a new test case. - params := objects.NewObjectsCreateParams().WithBody(example.object()) - resp, err := helper.Client(t).Objects.ObjectsCreate(params, nil) - helper.AssertRequestOk(t, resp, err, nil) - }) - } - }) -} diff --git a/test/acceptance/objects/crefs_test.go b/test/acceptance/objects/crefs_test.go deleted file mode 100644 index 1be5a6429ae0c88254b60675ac6472cb1960faf3..0000000000000000000000000000000000000000 --- a/test/acceptance/objects/crefs_test.go +++ /dev/null @@ -1,918 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "fmt" - "testing" - - "github.com/google/uuid" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/weaviate/weaviate/client/batch" - - "github.com/go-openapi/strfmt" - "github.com/weaviate/weaviate/client/objects" - clschema "github.com/weaviate/weaviate/client/schema" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/test/helper" - testhelper "github.com/weaviate/weaviate/test/helper" -) - -const ( - beaconStart = "weaviate://localhost/" - pathStart = "/v1/objects/" -) - -func TestRefsWithoutToClass(t *testing.T) { - params := clschema.NewSchemaObjectsCreateParams().WithObjectClass(&models.Class{Class: "ReferenceTo"}) - resp, err := helper.Client(t).Schema.SchemaObjectsCreate(params, nil) - helper.AssertRequestOk(t, resp, err, nil) - refToClassName := "ReferenceTo" - refFromClassName := "ReferenceFrom" - otherClassMT := "Other" - - // other class has multi-tenancy enabled to make sure that problems trigger an error - paramsMT := clschema.NewSchemaObjectsCreateParams().WithObjectClass( - &models.Class{Class: otherClassMT, MultiTenancyConfig: &models.MultiTenancyConfig{Enabled: true}}, - ) - respMT, err := helper.Client(t).Schema.SchemaObjectsCreate(paramsMT, nil) - helper.AssertRequestOk(t, respMT, err, nil) - - tenant := "tenant" - tenants := make([]*models.Tenant, 1) - for i := range tenants { - tenants[i] = &models.Tenant{Name: tenant} - } - helper.CreateTenants(t, otherClassMT, tenants) - - refFromClass := &models.Class{ - Class: refFromClassName, - Properties: []*models.Property{ - { - DataType: []string{refToClassName}, - Name: "ref", - }, - }, - } - params2 := clschema.NewSchemaObjectsCreateParams().WithObjectClass(refFromClass) - resp2, err := helper.Client(t).Schema.SchemaObjectsCreate(params2, nil) - helper.AssertRequestOk(t, resp2, err, nil) - - defer deleteObjectClass(t, refToClassName) - defer deleteObjectClass(t, refFromClassName) - defer deleteObjectClass(t, otherClassMT) - - refToId := assertCreateObject(t, refToClassName, map[string]interface{}{}) - assertGetObjectWithClass(t, refToId, refToClassName) - assertCreateObjectWithID(t, otherClassMT, tenant, refToId, map[string]interface{}{}) - refFromId := assertCreateObject(t, refFromClassName, map[string]interface{}{}) - assertGetObjectWithClass(t, refFromId, refFromClassName) - - postRefParams := objects.NewObjectsClassReferencesCreateParams(). - WithID(refFromId). - WithPropertyName("ref").WithClassName(refFromClass.Class). - WithBody(&models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf(beaconStart+"%s", refToId.String())), - }) - postRefResponse, err := helper.Client(t).Objects.ObjectsClassReferencesCreate(postRefParams, nil) - helper.AssertRequestOk(t, postRefResponse, err, nil) - - // validate that ref was create for the correct class - objWithRef := func() interface{} { - obj := assertGetObjectWithClass(t, refFromId, refFromClassName) - return obj.Properties - } - testhelper.AssertEventuallyEqual(t, map[string]interface{}{ - "ref": []interface{}{ - map[string]interface{}{ - "beacon": fmt.Sprintf(beaconStart+"%s/%s", refToClassName, refToId.String()), - "href": fmt.Sprintf(pathStart+"%s/%s", refToClassName, refToId.String()), - }, - }, - }, objWithRef) - - // update prop with multiple references - updateRefParams := objects.NewObjectsClassReferencesPutParams(). - WithID(refFromId). - WithPropertyName("ref").WithClassName(refFromClass.Class). - WithBody(models.MultipleRef{ - {Beacon: strfmt.URI(fmt.Sprintf(beaconStart+"%s", refToId.String()))}, - {Beacon: strfmt.URI(fmt.Sprintf(beaconStart+"%s/%s", refToClassName, refToId.String()))}, - }) - updateRefResponse, err := helper.Client(t).Objects.ObjectsClassReferencesPut(updateRefParams, nil) - helper.AssertRequestOk(t, updateRefResponse, err, nil) - - objWithTwoRef := func() interface{} { - obj := assertGetObjectWithClass(t, refFromId, refFromClassName) - return obj.Properties - } - testhelper.AssertEventuallyEqual(t, map[string]interface{}{ - "ref": []interface{}{ - map[string]interface{}{ - "beacon": fmt.Sprintf(beaconStart+"%s/%s", refToClassName, refToId.String()), - "href": fmt.Sprintf(pathStart+"%s/%s", refToClassName, refToId.String()), - }, - map[string]interface{}{ - "beacon": fmt.Sprintf(beaconStart+"%s/%s", refToClassName, refToId.String()), - "href": fmt.Sprintf(pathStart+"%s/%s", refToClassName, refToId.String()), - }, - }, - }, objWithTwoRef) - - // delete reference without class - deleteRefParams := objects.NewObjectsClassReferencesDeleteParams(). - WithID(refFromId). - WithPropertyName("ref").WithClassName(refFromClass.Class). - WithBody(&models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf(beaconStart+"%s", refToId.String())), - }) - deleteRefResponse, err := helper.Client(t).Objects.ObjectsClassReferencesDelete(deleteRefParams, nil) - helper.AssertRequestOk(t, deleteRefResponse, err, nil) - objWithoutRef := func() interface{} { - obj := assertGetObjectWithClass(t, refFromId, refFromClassName) - return obj.Properties - } - testhelper.AssertEventuallyEqual(t, map[string]interface{}{ - "ref": []interface{}{}, - }, objWithoutRef) -} - -func TestRefsMultiTarget(t *testing.T) { - refToClassName := "ReferenceTo" - refFromClassName := "ReferenceFrom" - defer deleteObjectClass(t, refToClassName) - defer deleteObjectClass(t, refFromClassName) - - params := clschema.NewSchemaObjectsCreateParams().WithObjectClass(&models.Class{Class: refToClassName}) - resp, err := helper.Client(t).Schema.SchemaObjectsCreate(params, nil) - helper.AssertRequestOk(t, resp, err, nil) - - refFromClass := &models.Class{ - Class: refFromClassName, - Properties: []*models.Property{ - { - DataType: []string{refToClassName, refFromClassName}, - Name: "ref", - }, - }, - } - params2 := clschema.NewSchemaObjectsCreateParams().WithObjectClass(refFromClass) - resp2, err := helper.Client(t).Schema.SchemaObjectsCreate(params2, nil) - helper.AssertRequestOk(t, resp2, err, nil) - - refToId := assertCreateObject(t, refToClassName, map[string]interface{}{}) - assertGetObjectEventually(t, refToId) - refFromId := assertCreateObject(t, refFromClassName, map[string]interface{}{}) - assertGetObjectEventually(t, refFromId) - - cases := []struct { - classRef string - id string - }{ - {classRef: "", id: refToId.String()}, - {classRef: refToClassName + "/", id: refToId.String()}, - {classRef: refFromClassName + "/", id: refFromId.String()}, - } - for _, tt := range cases { - postRefParams := objects.NewObjectsClassReferencesCreateParams(). - WithID(refFromId). - WithPropertyName("ref").WithClassName(refFromClass.Class). - WithBody(&models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf(beaconStart+"%s%s", tt.classRef, tt.id)), - }) - postRefResponse, err := helper.Client(t).Objects.ObjectsClassReferencesCreate(postRefParams, nil) - helper.AssertRequestOk(t, postRefResponse, err, nil) - - // validate that ref was create for the correct class - objWithRef := func() interface{} { - obj := assertGetObjectWithClass(t, refFromId, refFromClassName) - return obj.Properties - } - testhelper.AssertEventuallyEqual(t, map[string]interface{}{ - "ref": []interface{}{ - map[string]interface{}{ - "beacon": fmt.Sprintf(beaconStart+"%s%s", tt.classRef, tt.id), - "href": fmt.Sprintf(pathStart+"%s%s", tt.classRef, tt.id), - }, - }, - }, objWithRef) - - // delete refs - updateRefParams := objects.NewObjectsClassReferencesPutParams(). - WithID(refFromId). - WithPropertyName("ref").WithClassName(refFromClass.Class). - WithBody(models.MultipleRef{}) - updateRefResponse, err := helper.Client(t).Objects.ObjectsClassReferencesPut(updateRefParams, nil) - helper.AssertRequestOk(t, updateRefResponse, err, nil) - } -} - -func TestBatchRefsMultiTarget(t *testing.T) { - refToClassName := "ReferenceTo" - refFromClassName := "ReferenceFrom" - defer deleteObjectClass(t, refToClassName) - defer deleteObjectClass(t, refFromClassName) - - params := clschema.NewSchemaObjectsCreateParams().WithObjectClass(&models.Class{Class: refToClassName}) - resp, err := helper.Client(t).Schema.SchemaObjectsCreate(params, nil) - helper.AssertRequestOk(t, resp, err, nil) - - refFromClass := &models.Class{ - Class: refFromClassName, - Properties: []*models.Property{ - { - DataType: []string{refToClassName, refFromClassName}, - Name: "ref", - }, - }, - } - params2 := clschema.NewSchemaObjectsCreateParams().WithObjectClass(refFromClass) - resp2, err := helper.Client(t).Schema.SchemaObjectsCreate(params2, nil) - helper.AssertRequestOk(t, resp2, err, nil) - - uuidsTo := make([]strfmt.UUID, 10) - uuidsFrom := make([]strfmt.UUID, 10) - for i := 0; i < 10; i++ { - uuidsTo[i] = assertCreateObject(t, refToClassName, map[string]interface{}{}) - assertGetObjectEventually(t, uuidsTo[i]) - uuidsFrom[i] = assertCreateObject(t, refFromClassName, map[string]interface{}{}) - assertGetObjectEventually(t, uuidsFrom[i]) - } - - // add refs without toClass - var batchRefs []*models.BatchReference - for i := range uuidsFrom[:2] { - from := beaconStart + "ReferenceFrom/" + uuidsFrom[i] + "/ref" - to := beaconStart + uuidsTo[i] - batchRefs = append(batchRefs, &models.BatchReference{From: strfmt.URI(from), To: strfmt.URI(to)}) - } - - // add refs with toClass target 1 - for i := range uuidsFrom[2:5] { - j := i + 2 - from := beaconStart + "ReferenceFrom/" + uuidsFrom[j] + "/ref" - to := beaconStart + "ReferenceTo/" + uuidsTo[j] - batchRefs = append(batchRefs, &models.BatchReference{From: strfmt.URI(from), To: strfmt.URI(to)}) - } - - // add refs with toClass target 2 - for i := range uuidsFrom[5:] { - j := i + 5 - from := beaconStart + "ReferenceFrom/" + uuidsFrom[j] + "/ref" - to := beaconStart + "ReferenceFrom/" + uuidsTo[j] - batchRefs = append(batchRefs, &models.BatchReference{From: strfmt.URI(from), To: strfmt.URI(to)}) - } - - postRefParams := batch.NewBatchReferencesCreateParams().WithBody(batchRefs) - postRefResponse, err := helper.Client(t).Batch.BatchReferencesCreate(postRefParams, nil) - helper.AssertRequestOk(t, postRefResponse, err, nil) - - // no autodetect for multi-target - for i := range uuidsFrom[:2] { - objWithRef := func() interface{} { - obj := assertGetObjectWithClass(t, uuidsFrom[i], refFromClassName) - return obj.Properties - } - testhelper.AssertEventuallyEqual(t, map[string]interface{}{ - "ref": []interface{}{ - map[string]interface{}{ - "beacon": fmt.Sprintf(beaconStart+"%s", uuidsTo[i].String()), - "href": fmt.Sprintf(pathStart+"%s", uuidsTo[i].String()), - }, - }, - }, objWithRef) - } - - // refs for target 1 - for i := range uuidsFrom[2:5] { - j := i + 2 - objWithRef := func() interface{} { - obj := assertGetObjectWithClass(t, uuidsFrom[j], refFromClassName) - return obj.Properties - } - testhelper.AssertEventuallyEqual(t, map[string]interface{}{ - "ref": []interface{}{ - map[string]interface{}{ - "beacon": fmt.Sprintf(beaconStart+"%s/%s", refToClassName, uuidsTo[j].String()), - "href": fmt.Sprintf(pathStart+"%s/%s", refToClassName, uuidsTo[j].String()), - }, - }, - }, objWithRef) - } - - // refs for target 2 - for i := range uuidsFrom[5:] { - j := i + 5 - objWithRef := func() interface{} { - obj := assertGetObjectWithClass(t, uuidsFrom[j], refFromClassName) - return obj.Properties - } - testhelper.AssertEventuallyEqual(t, map[string]interface{}{ - "ref": []interface{}{ - map[string]interface{}{ - "beacon": fmt.Sprintf(beaconStart+"%s/%s", refFromClassName, uuidsTo[j].String()), - "href": fmt.Sprintf(pathStart+"%s/%s", refFromClassName, uuidsTo[j].String()), - }, - }, - }, objWithRef) - } -} - -func TestBatchRefsWithoutFromAndToClass(t *testing.T) { - refToClassName := "ReferenceTo" - refFromClassName := "ReferenceFrom" - - params := clschema.NewSchemaObjectsCreateParams().WithObjectClass(&models.Class{Class: refToClassName}) - resp, err := helper.Client(t).Schema.SchemaObjectsCreate(params, nil) - helper.AssertRequestOk(t, resp, err, nil) - - refFromClass := &models.Class{ - Class: refFromClassName, - Properties: []*models.Property{ - { - DataType: []string{refToClassName}, - Name: "ref", - }, - }, - } - params2 := clschema.NewSchemaObjectsCreateParams().WithObjectClass(refFromClass) - resp2, err := helper.Client(t).Schema.SchemaObjectsCreate(params2, nil) - helper.AssertRequestOk(t, resp2, err, nil) - - defer deleteObjectClass(t, refToClassName) - defer deleteObjectClass(t, refFromClassName) - - uuidsTo := make([]strfmt.UUID, 10) - uuidsFrom := make([]strfmt.UUID, 10) - for i := 0; i < 10; i++ { - uuidsTo[i] = assertCreateObject(t, refToClassName, map[string]interface{}{}) - assertGetObjectWithClass(t, uuidsTo[i], refToClassName) - - uuidsFrom[i] = assertCreateObject(t, refFromClassName, map[string]interface{}{}) - assertGetObjectWithClass(t, uuidsFrom[i], refFromClassName) - } - - // cannot do from urls without class - var batchRefs []*models.BatchReference - for i := range uuidsFrom { - from := beaconStart + uuidsFrom[i] + "/ref" - to := beaconStart + uuidsTo[i] - batchRefs = append(batchRefs, &models.BatchReference{From: strfmt.URI(from), To: strfmt.URI(to)}) - } - - postRefParams := batch.NewBatchReferencesCreateParams().WithBody(batchRefs) - resp3, err := helper.Client(t).Batch.BatchReferencesCreate(postRefParams, nil) - require.Nil(t, err) - require.NotNil(t, resp3) - for i := range resp3.Payload { - require.NotNil(t, resp3.Payload[i].Result.Errors) - } -} - -func TestBatchRefWithErrors(t *testing.T) { - refToClassName := "ReferenceTo" - refFromClassName := "ReferenceFrom" - - params := clschema.NewSchemaObjectsCreateParams().WithObjectClass(&models.Class{Class: refToClassName}) - resp, err := helper.Client(t).Schema.SchemaObjectsCreate(params, nil) - helper.AssertRequestOk(t, resp, err, nil) - - refFromClass := &models.Class{ - Class: refFromClassName, - Properties: []*models.Property{ - { - DataType: []string{refToClassName}, - Name: "ref", - }, - }, - } - params2 := clschema.NewSchemaObjectsCreateParams().WithObjectClass(refFromClass) - resp2, err := helper.Client(t).Schema.SchemaObjectsCreate(params2, nil) - helper.AssertRequestOk(t, resp2, err, nil) - - defer deleteObjectClass(t, refToClassName) - defer deleteObjectClass(t, refFromClassName) - - uuidsTo := make([]strfmt.UUID, 2) - uuidsFrom := make([]strfmt.UUID, 2) - for i := 0; i < 2; i++ { - uuidsTo[i] = assertCreateObject(t, refToClassName, map[string]interface{}{}) - assertGetObjectWithClass(t, uuidsTo[i], refToClassName) - - uuidsFrom[i] = assertCreateObject(t, refFromClassName, map[string]interface{}{}) - assertGetObjectWithClass(t, uuidsFrom[i], refFromClassName) - } - - var batchRefs []*models.BatchReference - for i := range uuidsFrom { - from := beaconStart + "ReferenceFrom/" + uuidsFrom[i] + "/ref" - to := beaconStart + uuidsTo[i] - batchRefs = append(batchRefs, &models.BatchReference{From: strfmt.URI(from), To: strfmt.URI(to)}) - } - - // append one entry with a non-existent class - batchRefs = append(batchRefs, &models.BatchReference{From: strfmt.URI(beaconStart + "DoesNotExist/" + uuidsFrom[0] + "/ref"), To: strfmt.URI(beaconStart + uuidsTo[0])}) - - // append one entry with a non-existent property for existing class - batchRefs = append(batchRefs, &models.BatchReference{From: strfmt.URI(beaconStart + "ReferenceFrom/" + uuidsFrom[0] + "/doesNotExist"), To: strfmt.URI(beaconStart + uuidsTo[0])}) - - postRefParams := batch.NewBatchReferencesCreateParams().WithBody(batchRefs) - postRefResponse, err := helper.Client(t).Batch.BatchReferencesCreate(postRefParams, nil) - helper.AssertRequestOk(t, postRefResponse, err, nil) - - require.NotNil(t, postRefResponse.Payload[2].Result.Errors) - require.Contains(t, postRefResponse.Payload[2].Result.Errors.Error[0].Message, "class DoesNotExist does not exist") - - require.NotNil(t, postRefResponse.Payload[3].Result.Errors) - require.Contains(t, postRefResponse.Payload[3].Result.Errors.Error[0].Message, "property doesNotExist does not exist for class ReferenceFrom") -} - -func TestBatchRefsWithoutToClass(t *testing.T) { - refToClassName := "ReferenceTo" - refFromClassName := "ReferenceFrom" - otherClassMT := "Other" - - // other class has multi-tenancy enabled to make sure that problems trigger an error - paramsMT := clschema.NewSchemaObjectsCreateParams().WithObjectClass( - &models.Class{Class: otherClassMT, MultiTenancyConfig: &models.MultiTenancyConfig{Enabled: true}}, - ) - respMT, err := helper.Client(t).Schema.SchemaObjectsCreate(paramsMT, nil) - helper.AssertRequestOk(t, respMT, err, nil) - - tenant := "tenant" - tenants := make([]*models.Tenant, 1) - for i := range tenants { - tenants[i] = &models.Tenant{Name: tenant} - } - helper.CreateTenants(t, otherClassMT, tenants) - - params := clschema.NewSchemaObjectsCreateParams().WithObjectClass(&models.Class{Class: refToClassName}) - resp, err := helper.Client(t).Schema.SchemaObjectsCreate(params, nil) - helper.AssertRequestOk(t, resp, err, nil) - - refFromClass := &models.Class{ - Class: refFromClassName, - Properties: []*models.Property{ - { - DataType: []string{refToClassName}, - Name: "ref", - }, - }, - } - params2 := clschema.NewSchemaObjectsCreateParams().WithObjectClass(refFromClass) - resp2, err := helper.Client(t).Schema.SchemaObjectsCreate(params2, nil) - helper.AssertRequestOk(t, resp2, err, nil) - - defer deleteObjectClass(t, refToClassName) - defer deleteObjectClass(t, refFromClassName) - defer deleteObjectClass(t, otherClassMT) - - uuidsTo := make([]strfmt.UUID, 10) - uuidsFrom := make([]strfmt.UUID, 10) - for i := 0; i < 10; i++ { - uuidsTo[i] = assertCreateObject(t, refToClassName, map[string]interface{}{}) - assertGetObjectWithClass(t, uuidsTo[i], refToClassName) - - // create object with same id in MT class - assertCreateObjectWithID(t, otherClassMT, tenant, uuidsTo[i], map[string]interface{}{}) - - uuidsFrom[i] = assertCreateObject(t, refFromClassName, map[string]interface{}{}) - assertGetObjectWithClass(t, uuidsFrom[i], refFromClassName) - } - - var batchRefs []*models.BatchReference - for i := range uuidsFrom { - from := beaconStart + "ReferenceFrom/" + uuidsFrom[i] + "/ref" - to := beaconStart + uuidsTo[i] - batchRefs = append(batchRefs, &models.BatchReference{From: strfmt.URI(from), To: strfmt.URI(to)}) - } - - postRefParams := batch.NewBatchReferencesCreateParams().WithBody(batchRefs) - postRefResponse, err := helper.Client(t).Batch.BatchReferencesCreate(postRefParams, nil) - helper.AssertRequestOk(t, postRefResponse, err, nil) - - for i := range uuidsFrom { - // validate that ref was create for the correct class - objWithRef := func() interface{} { - obj := assertGetObjectWithClass(t, uuidsFrom[i], refFromClassName) - return obj.Properties - } - testhelper.AssertEventuallyEqual(t, map[string]interface{}{ - "ref": []interface{}{ - map[string]interface{}{ - "beacon": fmt.Sprintf(beaconStart+"%s/%s", refToClassName, uuidsTo[i].String()), - "href": fmt.Sprintf(pathStart+"%s/%s", refToClassName, uuidsTo[i].String()), - }, - }, - }, objWithRef) - } -} - -func TestObjectBatchToClassDetection(t *testing.T) { - // uses same code path as normal object add - refToClassName := "ReferenceTo" - refFromClassName := "ReferenceFrom" - defer deleteObjectClass(t, refToClassName) - defer deleteObjectClass(t, refFromClassName) - - params := clschema.NewSchemaObjectsCreateParams().WithObjectClass(&models.Class{Class: refToClassName}) - resp, err := helper.Client(t).Schema.SchemaObjectsCreate(params, nil) - helper.AssertRequestOk(t, resp, err, nil) - - refFromClass := &models.Class{ - Class: refFromClassName, - Properties: []*models.Property{ - { - DataType: []string{refToClassName}, - Name: "ref", - }, - }, - } - params2 := clschema.NewSchemaObjectsCreateParams().WithObjectClass(refFromClass) - resp2, err := helper.Client(t).Schema.SchemaObjectsCreate(params2, nil) - helper.AssertRequestOk(t, resp2, err, nil) - - refs := make([]interface{}, 10) - uuidsTo := make([]strfmt.UUID, 10) - - for i := 0; i < 10; i++ { - uuidTo := assertCreateObject(t, refToClassName, map[string]interface{}{}) - uuidsTo[i] = uuidTo - assertGetObjectEventually(t, uuidTo) - refs[i] = map[string]interface{}{ - "beacon": beaconStart + uuidTo, - } - } - - fromBatch := make([]*models.Object, 10) - for i := 0; i < 10; i++ { - fromBatch[i] = &models.Object{ - Class: refFromClassName, - ID: strfmt.UUID(uuid.New().String()), - Properties: map[string]interface{}{ - "ref": refs[i : i+1], - }, - } - } - paramsBatch := batch.NewBatchObjectsCreateParams().WithBody( - batch.BatchObjectsCreateBody{ - Objects: fromBatch, - }, - ) - res, err := helper.Client(t).Batch.BatchObjectsCreate(paramsBatch, nil) - require.Nil(t, err) - for _, elem := range res.Payload { - assert.Nil(t, elem.Result.Errors) - } - - for i := range fromBatch { - // validate that ref was create for the correct class - objWithRef := func() interface{} { - obj := assertGetObjectWithClass(t, fromBatch[i].ID, refFromClassName) - return obj.Properties - } - testhelper.AssertEventuallyEqual(t, map[string]interface{}{ - "ref": []interface{}{ - map[string]interface{}{ - "beacon": fmt.Sprintf(beaconStart+"%s/%s", refToClassName, uuidsTo[i].String()), - "href": fmt.Sprintf(pathStart+"%s/%s", refToClassName, uuidsTo[i].String()), - }, - }, - }, objWithRef) - } -} - -func TestObjectCrefWithoutToClass(t *testing.T) { - refToClassName := "ReferenceTo" - refFromClassName := "ReferenceFrom" - otherClassMT := "Other" - - // other class has multi-tenancy enabled to make sure that problems trigger an error - paramsMT := clschema.NewSchemaObjectsCreateParams().WithObjectClass( - &models.Class{Class: otherClassMT, MultiTenancyConfig: &models.MultiTenancyConfig{Enabled: true}}, - ) - respMT, err := helper.Client(t).Schema.SchemaObjectsCreate(paramsMT, nil) - helper.AssertRequestOk(t, respMT, err, nil) - - tenant := "tenant" - tenants := make([]*models.Tenant, 1) - for i := range tenants { - tenants[i] = &models.Tenant{Name: tenant} - } - helper.CreateTenants(t, otherClassMT, tenants) - - params := clschema.NewSchemaObjectsCreateParams().WithObjectClass(&models.Class{Class: refToClassName}) - resp, err := helper.Client(t).Schema.SchemaObjectsCreate(params, nil) - helper.AssertRequestOk(t, resp, err, nil) - - refFromClass := &models.Class{ - Class: refFromClassName, - Properties: []*models.Property{ - { - DataType: []string{refToClassName}, - Name: "ref", - }, - }, - } - params2 := clschema.NewSchemaObjectsCreateParams().WithObjectClass(refFromClass) - resp2, err := helper.Client(t).Schema.SchemaObjectsCreate(params2, nil) - helper.AssertRequestOk(t, resp2, err, nil) - - defer deleteObjectClass(t, refToClassName) - defer deleteObjectClass(t, refFromClassName) - defer deleteObjectClass(t, otherClassMT) - - refs := make([]interface{}, 10) - uuids := make([]strfmt.UUID, 10) - for i := 0; i < 10; i++ { - uuidTo := assertCreateObject(t, refToClassName, map[string]interface{}{}) - assertGetObjectWithClass(t, uuidTo, refToClassName) - - // create object with same id in MT class - assertCreateObjectWithID(t, otherClassMT, tenant, uuidTo, map[string]interface{}{}) - - refs[i] = map[string]interface{}{ - "beacon": beaconStart + uuidTo, - } - uuids[i] = uuidTo - } - - uuidFrom := assertCreateObject(t, refFromClassName, map[string]interface{}{"ref": refs}) - assertGetObjectWithClass(t, uuidFrom, refFromClassName) - - objWithRef := assertGetObjectWithClass(t, uuidFrom, refFromClassName) - assert.NotNil(t, objWithRef.Properties) - refsReturned := objWithRef.Properties.(map[string]interface{})["ref"].([]interface{}) - for i := range refsReturned { - require.Equal(t, refsReturned[i].(map[string]interface{})["beacon"], string(beaconStart+"ReferenceTo/"+uuids[i])) - } -} - -// This test suite is meant to prevent a regression on -// https://github.com/weaviate/weaviate/issues/868, hence it tries to -// reproduce the steps outlined in there as closely as possible -func Test_CREFWithCardinalityMany_UsingPatch(t *testing.T) { - defer func() { - // clean up so we can run this test multiple times in a row - delCityParams := clschema.NewSchemaObjectsDeleteParams().WithClassName("ReferenceTestCity") - dresp, err := helper.Client(t).Schema.SchemaObjectsDelete(delCityParams, nil) - t.Logf("clean up - delete city \n%v\n %v", dresp, err) - - delPlaceParams := clschema.NewSchemaObjectsDeleteParams().WithClassName("ReferenceTestPlace") - dresp, err = helper.Client(t).Schema.SchemaObjectsDelete(delPlaceParams, nil) - t.Logf("clean up - delete place \n%v\n %v", dresp, err) - }() - - t.Log("1. create ReferenceTestPlace class") - placeClass := &models.Class{ - Class: "ReferenceTestPlace", - Properties: []*models.Property{ - { - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - Name: "name", - }, - }, - } - params := clschema.NewSchemaObjectsCreateParams().WithObjectClass(placeClass) - resp, err := helper.Client(t).Schema.SchemaObjectsCreate(params, nil) - helper.AssertRequestOk(t, resp, err, nil) - - t.Log("2. create ReferenceTestCity class with HasPlaces (many) cross-ref") - cityClass := &models.Class{ - Class: "ReferenceTestCity", - Properties: []*models.Property{ - { - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - Name: "name", - }, - { - DataType: []string{"ReferenceTestPlace"}, - Name: "HasPlaces", - }, - }, - } - params = clschema.NewSchemaObjectsCreateParams().WithObjectClass(cityClass) - resp, err = helper.Client(t).Schema.SchemaObjectsCreate(params, nil) - helper.AssertRequestOk(t, resp, err, nil) - - t.Log("3. add two places and save their IDs") - place1ID := assertCreateObject(t, "ReferenceTestPlace", map[string]interface{}{ - "name": "Place 1", - }) - place2ID := assertCreateObject(t, "ReferenceTestPlace", map[string]interface{}{ - "name": "Place 2", - }) - assertGetObjectEventually(t, place1ID) - assertGetObjectEventually(t, place2ID) - - t.Log("4. add one city") - cityID := assertCreateObject(t, "ReferenceTestCity", map[string]interface{}{ - "name": "My City", - }) - assertGetObjectEventually(t, cityID) - - t.Log("5. patch city to point to the first place") - patchParams := objects.NewObjectsPatchParams(). - WithID(cityID). - WithBody(&models.Object{ - Class: "ReferenceTestCity", - Properties: map[string]interface{}{ - "hasPlaces": []interface{}{ - map[string]interface{}{ - "beacon": fmt.Sprintf("weaviate://localhost/%s", place1ID.String()), - }, - }, - }, - }) - patchResp, err := helper.Client(t).Objects.ObjectsPatch(patchParams, nil) - helper.AssertRequestOk(t, patchResp, err, nil) - - t.Log("6. verify first cross ref was added") - - actualThunk := func() interface{} { - cityAfterFirstPatch := assertGetObject(t, cityID) - return cityAfterFirstPatch.Properties - } - - testhelper.AssertEventuallyEqual(t, map[string]interface{}{ - "name": "My City", - "hasPlaces": []interface{}{ - map[string]interface{}{ - "beacon": fmt.Sprintf("weaviate://localhost/%s/%s", placeClass.Class, place1ID.String()), - "href": fmt.Sprintf("/v1/objects/%s/%s", placeClass.Class, place1ID.String()), - }, - }, - }, actualThunk) - - t.Log("7. patch city to point to the second place") - patchParams = objects.NewObjectsPatchParams(). - WithID(cityID). - WithBody(&models.Object{ - Class: "ReferenceTestCity", - Properties: map[string]interface{}{ - "hasPlaces": []interface{}{ - map[string]interface{}{ - "beacon": fmt.Sprintf("weaviate://localhost/%s", place2ID.String()), - }, - }, - }, - }) - patchResp, err = helper.Client(t).Objects.ObjectsPatch(patchParams, nil) - helper.AssertRequestOk(t, patchResp, err, nil) - - actualThunk = func() interface{} { - city := assertGetObject(t, cityID) - return city.Properties.(map[string]interface{})["hasPlaces"].([]interface{}) - } - - t.Log("9. verify both cross refs are present") - expectedRefs := []interface{}{ - map[string]interface{}{ - "beacon": fmt.Sprintf("weaviate://localhost/%s/%s", placeClass.Class, place1ID.String()), - "href": fmt.Sprintf("/v1/objects/%s/%s", placeClass.Class, place1ID.String()), - }, - map[string]interface{}{ - "beacon": fmt.Sprintf("weaviate://localhost/%s/%s", placeClass.Class, place2ID.String()), - "href": fmt.Sprintf("/v1/objects/%s/%s", placeClass.Class, place2ID.String()), - }, - } - - testhelper.AssertEventuallyEqual(t, expectedRefs, actualThunk) -} - -// This test suite is meant to prevent a regression on -// https://github.com/weaviate/weaviate/issues/868, hence it tries to -// reproduce the steps outlined in there as closely as possible -func Test_CREFWithCardinalityMany_UsingPostReference(t *testing.T) { - defer func() { - // clean up so we can run this test multiple times in a row - delCityParams := clschema.NewSchemaObjectsDeleteParams().WithClassName("ReferenceTestCity") - dresp, err := helper.Client(t).Schema.SchemaObjectsDelete(delCityParams, nil) - t.Logf("clean up - delete city \n%v\n %v", dresp, err) - - delPlaceParams := clschema.NewSchemaObjectsDeleteParams().WithClassName("ReferenceTestPlace") - dresp, err = helper.Client(t).Schema.SchemaObjectsDelete(delPlaceParams, nil) - t.Logf("clean up - delete place \n%v\n %v", dresp, err) - }() - - t.Log("1. create ReferenceTestPlace class") - placeClass := &models.Class{ - Class: "ReferenceTestPlace", - Properties: []*models.Property{ - { - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - Name: "name", - }, - }, - } - params := clschema.NewSchemaObjectsCreateParams().WithObjectClass(placeClass) - resp, err := helper.Client(t).Schema.SchemaObjectsCreate(params, nil) - helper.AssertRequestOk(t, resp, err, nil) - - t.Log("2. create ReferenceTestCity class with HasPlaces (many) cross-ref") - cityClass := &models.Class{ - Class: "ReferenceTestCity", - Properties: []*models.Property{ - { - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - Name: "name", - }, - { - DataType: []string{"ReferenceTestPlace"}, - Name: "HasPlaces", - }, - }, - } - params = clschema.NewSchemaObjectsCreateParams().WithObjectClass(cityClass) - resp, err = helper.Client(t).Schema.SchemaObjectsCreate(params, nil) - helper.AssertRequestOk(t, resp, err, nil) - - t.Log("3. add two places and save their IDs") - place1ID := assertCreateObject(t, "ReferenceTestPlace", map[string]interface{}{ - "name": "Place 1", - }) - place2ID := assertCreateObject(t, "ReferenceTestPlace", map[string]interface{}{ - "name": "Place 2", - }) - assertGetObjectEventually(t, place1ID) - assertGetObjectEventually(t, place2ID) - t.Logf("Place 1 ID: %s", place1ID) - t.Logf("Place 2 ID: %s", place2ID) - - t.Log("4. add one city") - cityID := assertCreateObject(t, "ReferenceTestCity", map[string]interface{}{ - "name": "My City", - }) - assertGetObjectEventually(t, cityID) - - t.Log("5. POST /references/ for place 1") - postRefParams := objects.NewObjectsReferencesCreateParams(). - WithID(cityID). - WithPropertyName("hasPlaces"). - WithBody(&models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", place1ID.String())), - }) - postRefResponse, err := helper.Client(t).Objects.ObjectsReferencesCreate(postRefParams, nil) - helper.AssertRequestOk(t, postRefResponse, err, nil) - - actualThunk := func() interface{} { - city := assertGetObject(t, cityID) - return city.Properties - } - t.Log("7. verify first cross ref was added") - testhelper.AssertEventuallyEqual(t, map[string]interface{}{ - "name": "My City", - "hasPlaces": []interface{}{ - map[string]interface{}{ - "beacon": fmt.Sprintf("weaviate://localhost/%s/%s", placeClass.Class, place1ID.String()), - "href": fmt.Sprintf("/v1/objects/%s/%s", placeClass.Class, place1ID.String()), - }, - }, - }, actualThunk) - - t.Log("8. POST /references/ for place 2") - postRefParams = objects.NewObjectsReferencesCreateParams(). - WithID(cityID). - WithPropertyName("hasPlaces"). - WithBody(&models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", place2ID.String())), - }) - postRefResponse, err = helper.Client(t).Objects.ObjectsReferencesCreate(postRefParams, nil) - helper.AssertRequestOk(t, postRefResponse, err, nil) - - t.Log("9. verify both cross refs are present") - actualThunk = func() interface{} { - city := assertGetObject(t, cityID) - return city.Properties.(map[string]interface{})["hasPlaces"].([]interface{}) - } - - expectedRefs := []interface{}{ - map[string]interface{}{ - "beacon": fmt.Sprintf("weaviate://localhost/%s/%s", placeClass.Class, place1ID.String()), - "href": fmt.Sprintf("/v1/objects/%s/%s", placeClass.Class, place1ID.String()), - }, - map[string]interface{}{ - "beacon": fmt.Sprintf("weaviate://localhost/%s/%s", placeClass.Class, place2ID.String()), - "href": fmt.Sprintf("/v1/objects/%s/%s", placeClass.Class, place2ID.String()), - }, - } - - testhelper.AssertEventuallyEqual(t, expectedRefs, actualThunk) -} diff --git a/test/acceptance/objects/crefs_without_waiting_for_refresh_test.go b/test/acceptance/objects/crefs_without_waiting_for_refresh_test.go deleted file mode 100644 index 4f24324315aca022778fcac2f399bd05da6be74b..0000000000000000000000000000000000000000 --- a/test/acceptance/objects/crefs_without_waiting_for_refresh_test.go +++ /dev/null @@ -1,104 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "fmt" - "testing" - - "github.com/go-openapi/strfmt" - clschema "github.com/weaviate/weaviate/client/schema" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/test/helper" - testhelper "github.com/weaviate/weaviate/test/helper" -) - -// See https://github.com/weaviate/weaviate/issues/980 -func Test_AddingReferenceWithoutWaiting_UsingPostObjects(t *testing.T) { - defer func() { - // clean up so we can run this test multiple times in a row - delCityParams := clschema.NewSchemaObjectsDeleteParams().WithClassName("ReferenceWaitingTestCity") - dresp, err := helper.Client(t).Schema.SchemaObjectsDelete(delCityParams, nil) - t.Logf("clean up - delete city \n%v\n %v", dresp, err) - - delPlaceParams := clschema.NewSchemaObjectsDeleteParams().WithClassName("ReferenceWaitingTestPlace") - dresp, err = helper.Client(t).Schema.SchemaObjectsDelete(delPlaceParams, nil) - t.Logf("clean up - delete place \n%v\n %v", dresp, err) - }() - - t.Log("1. create ReferenceTestPlace class") - placeClass := &models.Class{ - Class: "ReferenceWaitingTestPlace", - Properties: []*models.Property{ - { - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - Name: "name", - }, - }, - } - params := clschema.NewSchemaObjectsCreateParams().WithObjectClass(placeClass) - resp, err := helper.Client(t).Schema.SchemaObjectsCreate(params, nil) - helper.AssertRequestOk(t, resp, err, nil) - - t.Log("2. create ReferenceTestCity class with HasPlace cross-ref") - cityClass := &models.Class{ - Class: "ReferenceWaitingTestCity", - Properties: []*models.Property{ - { - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - Name: "name", - }, - { - DataType: []string{"ReferenceWaitingTestPlace"}, - Name: "HasPlace", - }, - }, - } - params = clschema.NewSchemaObjectsCreateParams().WithObjectClass(cityClass) - resp, err = helper.Client(t).Schema.SchemaObjectsCreate(params, nil) - helper.AssertRequestOk(t, resp, err, nil) - - t.Log("3. add a places and save the ID") - placeID := assertCreateObject(t, "ReferenceWaitingTestPlace", map[string]interface{}{ - "name": "Place 1", - }) - - t.Log("4. add one city with ref to the place") - cityID := assertCreateObject(t, "ReferenceWaitingTestCity", map[string]interface{}{ - "name": "My City", - "hasPlace": models.MultipleRef{ - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", placeID.String())), - }, - }, - }) - - assertGetObjectEventually(t, cityID) - - actualThunk := func() interface{} { - city := assertGetObject(t, cityID) - return city.Properties - } - t.Log("7. verify first cross ref was added") - testhelper.AssertEventuallyEqual(t, map[string]interface{}{ - "name": "My City", - "hasPlace": []interface{}{ - map[string]interface{}{ - "beacon": fmt.Sprintf("weaviate://localhost/%s/%s", "ReferenceWaitingTestPlace", placeID.String()), - "href": fmt.Sprintf("/v1/objects/%s/%s", "ReferenceWaitingTestPlace", placeID.String()), - }, - }, - }, actualThunk) -} diff --git a/test/acceptance/objects/custom_vectors.go b/test/acceptance/objects/custom_vectors.go deleted file mode 100644 index 079b1a2313b20d56668483c9e38b2fc87c007de5..0000000000000000000000000000000000000000 --- a/test/acceptance/objects/custom_vectors.go +++ /dev/null @@ -1,89 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "testing" - - "github.com/davecgh/go-spew/spew" - "github.com/go-openapi/strfmt" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/client/objects" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/test/helper" -) - -func customVectors(t *testing.T) { - var id strfmt.UUID - - t.Run("create object", func(t *testing.T) { - params := objects.NewObjectsCreateParams().WithBody( - &models.Object{ - Class: "TestObjectCustomVector", - Properties: map[string]interface{}{"description": "foo"}, - Vector: []float32{0.1, 0.2}, - }) - resp, err := helper.Client(t).Objects.ObjectsCreate(params, nil) - require.Nil(t, err, "creation should succeed") - id = resp.Payload.ID - }) - - t.Run("check custom vector is set", func(t *testing.T) { - include := "vector" - params := objects.NewObjectsGetParams().WithID(id).WithInclude(&include) - resp, err := helper.Client(t).Objects.ObjectsGet(params, nil) - require.Nil(t, err, "get should succeed") - assert.Equal(t, []float32{0.1, 0.2}, []float32(resp.Payload.Vector)) - }) - - t.Run("replace object entirely (update)", func(t *testing.T) { - params := objects.NewObjectsUpdateParams().WithID(id).WithBody(&models.Object{ - ID: id, - Class: "TestObjectCustomVector", - Properties: map[string]interface{}{"description": "foo updated"}, - Vector: []float32{0.1, 0.3}, - }) - _, err := helper.Client(t).Objects.ObjectsUpdate(params, nil) - require.Nil(t, err, "update should succeed") - }) - - t.Run("check custom vector is updated", func(t *testing.T) { - include := "vector" - params := objects.NewObjectsGetParams().WithID(id).WithInclude(&include) - resp, err := helper.Client(t).Objects.ObjectsGet(params, nil) - require.Nil(t, err, "get should succeed") - assert.Equal(t, []float32{0.1, 0.3}, []float32(resp.Payload.Vector)) - }) - - t.Run("replace only vector through merge", func(t *testing.T) { - params := objects.NewObjectsPatchParams().WithID(id).WithBody(&models.Object{ - ID: id, - Class: "TestObjectCustomVector", - Properties: map[string]interface{}{}, - Vector: []float32{0.4, 0.3}, - }) - _, err := helper.Client(t).Objects.ObjectsPatch(params, nil) - if err != nil { - spew.Dump(err.(*objects.ObjectsPatchInternalServerError).Payload.Error[0]) - } - require.Nil(t, err, "patch should succeed") - }) - - t.Run("check custom vector is updated", func(t *testing.T) { - include := "vector" - params := objects.NewObjectsGetParams().WithID(id).WithInclude(&include) - resp, err := helper.Client(t).Objects.ObjectsGet(params, nil) - require.Nil(t, err, "get should succeed") - assert.Equal(t, []float32{0.4, 0.3}, []float32(resp.Payload.Vector)) - }) -} diff --git a/test/acceptance/objects/delete_objects_from_all_classes.go b/test/acceptance/objects/delete_objects_from_all_classes.go deleted file mode 100644 index 4cec98757e12714e2b6f9e87d393aa23dbe52010..0000000000000000000000000000000000000000 --- a/test/acceptance/objects/delete_objects_from_all_classes.go +++ /dev/null @@ -1,98 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -// Acceptance tests for objects. - -import ( - "testing" - - "github.com/go-openapi/strfmt" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/weaviate/weaviate/client/batch" - "github.com/weaviate/weaviate/client/objects" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/test/helper" -) - -// run from setup_test.go -func deleteAllObjectsFromAllClasses(t *testing.T) { - // We can have a situation that objects in different classes - // have the same ID. This test is to ensure that the delete request - // deletes all of the objects with a given ID in all classes - // This test is connected with this issue: - // https://github.com/weaviate/weaviate/issues/1836 - const fakeObjectId strfmt.UUID = "11111111-1111-1111-1111-111111111111" - - t.Run("create objects with a specified id", func(t *testing.T) { - object1 := &models.Object{ - Class: "TestDeleteClassOne", - ID: fakeObjectId, - Properties: map[string]interface{}{ - "text": "Test string 1", - }, - } - object2 := &models.Object{ - Class: "TestDeleteClassTwo", - ID: fakeObjectId, - Properties: map[string]interface{}{ - "text": "Test string 2", - }, - } - - testFields := "ALL" - // generate request body - params := batch.NewBatchObjectsCreateParams().WithBody(batch.BatchObjectsCreateBody{ - Objects: []*models.Object{object1, object2}, - Fields: []*string{&testFields}, - }) - - // perform the request - resp, err := helper.BatchClient(t).BatchObjectsCreate(params, nil) - // ensure that the response is OK - helper.AssertRequestOk(t, resp, err, func() { - objectsCreateResponse := resp.Payload - - // check if the batch response contains two batched responses - assert.Equal(t, 2, len(objectsCreateResponse)) - - for _, elem := range resp.Payload { - assert.Nil(t, elem.Result.Errors) - } - }) - }) - - t.Run("check that object exists", func(t *testing.T) { - // there are actually 2 objects in 2 classes with this ID - params := objects.NewObjectsGetParams().WithID(fakeObjectId) - resp, err := helper.Client(t).Objects.ObjectsGet(params, nil) - require.Nil(t, err, "get should succeed") - assert.NotNil(t, resp.Payload) - }) - - t.Run("delete objects with a given ID from all classes", func(t *testing.T) { - params := objects.NewObjectsDeleteParams().WithID(fakeObjectId) - resp, err := helper.Client(t).Objects.ObjectsDelete(params, nil) - require.Nil(t, err, "delete should succeed") - assert.Equal(t, &objects.ObjectsDeleteNoContent{}, resp) - }) - - t.Run("check that object with given ID is removed from all classes", func(t *testing.T) { - params := objects.NewObjectsGetParams().WithID(fakeObjectId) - resp, err := helper.Client(t).Objects.ObjectsGet(params, nil) - require.Equal(t, &objects.ObjectsGetNotFound{}, err) - assert.Nil(t, resp) - }) -} diff --git a/test/acceptance/objects/head_test.go b/test/acceptance/objects/head_test.go deleted file mode 100644 index d5e9a05a74c98d8a31be2f56d1b6d930a49b87b6..0000000000000000000000000000000000000000 --- a/test/acceptance/objects/head_test.go +++ /dev/null @@ -1,71 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -// Acceptance tests for things. - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/weaviate/weaviate/client/objects" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/test/helper" -) - -// Test that we can properly check object's existence. -// Create two objects, and check that those objects exist. -// Also check one non existent object that it doesn't exist -// This test is run by setup_test.go -func checkObjects(t *testing.T) { - params1 := objects.NewObjectsCreateParams().WithBody( - &models.Object{ - Class: "TestObject", - Properties: map[string]interface{}{}, - }) - resp1, err := helper.Client(t).Objects.ObjectsCreate(params1, nil) - require.Nil(t, err, "creation should succeed") - object1ID := resp1.Payload.ID - - params2 := objects.NewObjectsCreateParams().WithBody( - &models.Object{ - Class: "TestObject", - Properties: map[string]interface{}{}, - }) - resp2, err := helper.Client(t).Objects.ObjectsCreate(params2, nil) - assert.Nil(t, err, "creation should succeed") - object2ID := resp2.Payload.ID - - // wait for both Objects to be indexed - assertGetObjectEventually(t, object1ID) - assertGetObjectEventually(t, object2ID) - - headParams := objects.NewObjectsHeadParams().WithID(object1ID) - resp, err := helper.Client(t).Objects.ObjectsHead(headParams, nil) - - require.Nil(t, err, "should not error") - assert.True(t, resp != nil, "Did not find object 1") - - headParams = objects.NewObjectsHeadParams().WithID("non-existent-object") - resp, err = helper.Client(t).Objects.ObjectsHead(headParams, nil) - - require.NotNil(t, err, "should error") - assert.True(t, resp == nil, "Did find non existent object") - - headParams = objects.NewObjectsHeadParams().WithID(object2ID) - resp, err = helper.Client(t).Objects.ObjectsHead(headParams, nil) - - require.Nil(t, err, "should not error") - assert.True(t, resp != nil, "Did not find object 2") -} diff --git a/test/acceptance/objects/helpers_for_test.go b/test/acceptance/objects/helpers_for_test.go deleted file mode 100644 index 64922a9bee9f2b5afbd49beecad93d9fccd3b87f..0000000000000000000000000000000000000000 --- a/test/acceptance/objects/helpers_for_test.go +++ /dev/null @@ -1,102 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "testing" - - "github.com/go-openapi/strfmt" - "github.com/weaviate/weaviate/client/objects" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/test/helper" - testhelper "github.com/weaviate/weaviate/test/helper" -) - -func assertCreateObject(t *testing.T, className string, schema map[string]interface{}) strfmt.UUID { - params := objects.NewObjectsCreateParams().WithBody( - &models.Object{ - Class: className, - Properties: schema, - }) - - resp, err := helper.Client(t).Objects.ObjectsCreate(params, nil) - - var objectID strfmt.UUID - - // Ensure that the response is OK - helper.AssertRequestOk(t, resp, err, func() { - objectID = resp.Payload.ID - }) - - return objectID -} - -func assertCreateObjectWithID(t *testing.T, className, tenant string, id strfmt.UUID, schema map[string]interface{}) { - params := objects.NewObjectsCreateParams().WithBody( - &models.Object{ - ID: id, - Class: className, - Properties: schema, - Tenant: tenant, - }) - - resp, err := helper.Client(t).Objects.ObjectsCreate(params, nil) - - // Ensure that the response is OK - helper.AssertRequestOk(t, resp, err, nil) -} - -func assertGetObject(t *testing.T, uuid strfmt.UUID) *models.Object { - getResp, err := helper.Client(t).Objects.ObjectsGet(objects.NewObjectsGetParams().WithID(uuid), nil) - - var object *models.Object - - helper.AssertRequestOk(t, getResp, err, func() { - object = getResp.Payload - }) - - return object -} - -func assertGetObjectWithClass(t *testing.T, uuid strfmt.UUID, class string) *models.Object { - getResp, err := helper.Client(t).Objects.ObjectsClassGet(objects.NewObjectsClassGetParams().WithID(uuid).WithClassName(class), nil) - - var object *models.Object - - helper.AssertRequestOk(t, getResp, err, func() { - object = getResp.Payload - }) - - return object -} - -func assertGetObjectEventually(t *testing.T, uuid strfmt.UUID) *models.Object { - var ( - resp *objects.ObjectsGetOK - err error - ) - - checkThunk := func() interface{} { - resp, err = helper.Client(t).Objects.ObjectsGet(objects.NewObjectsGetParams().WithID(uuid), nil) - return err == nil - } - - testhelper.AssertEventuallyEqual(t, true, checkThunk) - - var object *models.Object - - helper.AssertRequestOk(t, resp, err, func() { - object = resp.Payload - }) - - return object -} diff --git a/test/acceptance/objects/list_test.go b/test/acceptance/objects/list_test.go deleted file mode 100644 index e85faed6931d943a137f0b25d96207e48faf9893..0000000000000000000000000000000000000000 --- a/test/acceptance/objects/list_test.go +++ /dev/null @@ -1,77 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -// Acceptance tests for things. - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/client/objects" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/test/helper" -) - -// Test that we can properly list objects. -// Create two objects, and check that the list all contains them all. -// This test is run by setup_test.go -func listingObjects(t *testing.T) { - params1 := objects.NewObjectsCreateParams().WithBody( - &models.Object{ - Class: "TestObject", - Properties: map[string]interface{}{ - "testString": "1", - }, - }) - resp1, err := helper.Client(t).Objects.ObjectsCreate(params1, nil) - require.Nil(t, err, "creation should succeed") - object1ID := resp1.Payload.ID - - params2 := objects.NewObjectsCreateParams().WithBody( - &models.Object{ - Class: "TestObject", - Properties: map[string]interface{}{ - "testString": "2", - }, - }) - resp2, err := helper.Client(t).Objects.ObjectsCreate(params2, nil) - assert.Nil(t, err, "creation should succeed") - object2ID := resp2.Payload.ID - - // wait for both Objects to be indexed - assertGetObjectEventually(t, object1ID) - assertGetObjectEventually(t, object2ID) - - listParams := objects.NewObjectsListParams() - resp, err := helper.Client(t).Objects.ObjectsList(listParams, nil) - require.Nil(t, err, "should not error") - - found1 := false - found2 := false - - for _, object := range resp.Payload.Objects { - if object.ID == resp1.Payload.ID { - assert.False(t, found1, "found double ID for object 1!") - found1 = true - } - - if object.ID == resp2.Payload.ID { - assert.False(t, found2, "found double ID for object 2!") - found2 = true - } - } - - assert.True(t, found1, "Did not find object 1") - assert.True(t, found2, "Did not find object 2") -} diff --git a/test/acceptance/objects/network_refs_test.go b/test/acceptance/objects/network_refs_test.go deleted file mode 100644 index ea8c8a02ea06825927de571905400160c96c25f6..0000000000000000000000000000000000000000 --- a/test/acceptance/objects/network_refs_test.go +++ /dev/null @@ -1,104 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -// func TestCanAddSingleNetworkRef(t *testing.T) { -// networkRefID := "711da979-4b0b-41e2-bcb8-fcc03554c7c8" -// thingID := assertCreateThing(t, "TestThing", map[string]interface{}{ -// "testReference": []interface{}{ -// map[string]interface{}{ -// "beacon": "weaviate://RemoteWeaviateForAcceptanceTest/things/" + networkRefID, -// }, -// }, -// }) - -// t.Run("it can query the resource again to verify the cross ref was added", func(t *testing.T) { -// thing := assertGetThingEventually(t, thingID) -// list := thing.Schema.(map[string]interface{})["testReference"] -// require.NotNil(t, list, "cross-ref is present") -// cref := list.([]interface{})[0].(map[string]interface{}) -// assert.Equal(t, cref["beacon"], "weaviate://RemoteWeaviateForAcceptanceTest/things/"+networkRefID) -// }) - -// t.Run("an implicit schema update has happened, we now include the network ref's class", func(t *testing.T) { -// schema := assertGetSchema(t) -// require.NotNil(t, schema.Things) -// class := assertClassInSchema(t, schema.Things, "TestThing") -// prop := assertPropertyInClass(t, class, "testReference") -// expectedDataType := []string{"TestThingTwo", "RemoteWeaviateForAcceptanceTest/Instruments"} -// assert.Equal(t, expectedDataType, prop.DataType, "prop should have old and newly added dataTypes") -// }) - -// t.Run("it can query the reference through the graphql api", func(t *testing.T) { -// result := graphql.AssertGraphQL(t, helper.RootAuth, -// "{ Get { Things { TestThing { TestReference { ... on RemoteWeaviateForAcceptanceTest__Instruments { name } } } } } }") -// things := result.Get("Get", "Things", "TestThing").AsSlice() -// assert.Contains(t, things, parseJSONObj(`{"TestReference":[{"name": "Talkbox"}]}`)) -// }) -// } - -// func TestCanPatchNetworkRef(t *testing.T) { -// t.Parallel() - -// thingID := assertCreateThing(t, "TestThing", nil) -// assertGetThingEventually(t, thingID) -// networkRefID := "711da979-4b0b-41e2-bcb8-fcc03554c7c8" - -// op := "add" -// path := "/schema/testReference" - -// patch := &models.PatchDocument{ -// Op: &op, -// Path: &path, -// Value: []interface{}{ -// map[string]interface{}{ -// "beacon": "weaviate://RemoteWeaviateForAcceptanceTest/things/" + networkRefID, -// }, -// }, -// } - -// t.Run("it can apply the patch", func(t *testing.T) { -// params := things.NewThingsPatchParams(). -// WithBody([]*models.PatchDocument{patch}). -// WithID(thingID) -// patchResp, err := helper.Client(t).Things.ThingsPatch(params, nil) -// helper.AssertRequestOk(t, patchResp, err, nil) -// }) - -// t.Run("it can query the resource again to verify the cross ref was added", func(t *testing.T) { -// patchedThing := assertGetThing(t, thingID) -// list := patchedThing.Schema.(map[string]interface{})["testReference"] -// require.NotNil(t, list, "cross-ref is present") -// cref := list.([]interface{})[0].(map[string]interface{}) -// assert.Equal(t, cref["beacon"], "weaviate://RemoteWeaviateForAcceptanceTest/things/"+networkRefID) -// }) - -// t.Run("an implicit schema update has happened, we now include the network ref's class", func(t *testing.T) { -// schema := assertGetSchema(t) -// require.NotNil(t, schema.Things) -// class := assertClassInSchema(t, schema.Things, "TestThing") -// prop := assertPropertyInClass(t, class, "testReference") -// expectedDataType := []string{"TestThingTwo", "RemoteWeaviateForAcceptanceTest/Instruments"} -// assert.Equal(t, expectedDataType, prop.DataType, "prop should have old and newly added dataTypes") -// }) -// } - -// func parseJSONObj(text string) interface{} { -// var result interface{} -// err := json.Unmarshal([]byte(text), &result) - -// if err != nil { -// panic(err) -// } - -// return result -// } diff --git a/test/acceptance/objects/objects_test.go b/test/acceptance/objects/objects_test.go deleted file mode 100644 index 364549e0b2c72fd8bf38da9e281b8b33a66f4a9e..0000000000000000000000000000000000000000 --- a/test/acceptance/objects/objects_test.go +++ /dev/null @@ -1,325 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -// Acceptance tests for objects. - -import ( - "encoding/json" - "errors" - "fmt" - "testing" - - "github.com/go-openapi/strfmt" - - "github.com/stretchr/testify/assert" - - "github.com/weaviate/weaviate/client/objects" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/test/helper" - testhelper "github.com/weaviate/weaviate/test/helper" -) - -// run from setup_test.go -func creatingObjects(t *testing.T) { - const fakeObjectId strfmt.UUID = "11111111-1111-1111-1111-111111111111" - - t.Run("create object with user specified id", func(t *testing.T) { - var ( - id = strfmt.UUID("d47ea61b-0ed7-4e5f-9c05-6d2c0786660f") - className = "TestObject" - // Set all object values to compare - objectTestString = "Test string" - ) - // clean up to make sure we can run this test multiple times in a row - defer func() { - params := objects.NewObjectsDeleteParams().WithID(id) - helper.Client(t).Objects.ObjectsDelete(params, nil) - { - params := objects.NewObjectsClassGetParams() - params.WithClassName(className).WithID(id) - _, err := helper.Client(t).Objects.ObjectsClassGet(params, nil) - if err == nil { - t.Errorf("Object %v cannot exist after deletion", id) - } - werr := new(objects.ObjectsClassGetNotFound) - if ok := errors.As(err, &werr); !ok { - t.Errorf("get deleted object err got: %v want: %v", err, werr) - } - } - }() - - params := objects.NewObjectsCreateParams().WithBody( - &models.Object{ - ID: id, - Class: className, - Properties: map[string]interface{}{ - "testString": objectTestString, - }, - }) - - resp, err := helper.Client(t).Objects.ObjectsCreate(params, nil) - - // Ensure that the response is OK - helper.AssertRequestOk(t, resp, err, func() { - object := resp.Payload - assert.Regexp(t, strfmt.UUIDPattern, object.ID) - - schema, ok := object.Properties.(map[string]interface{}) - if !ok { - t.Fatal("The returned schema is not an JSON object") - } - - // Check whether the returned information is the same as the data added - assert.Equal(t, objectTestString, schema["testString"]) - }) - - // wait for the object to be created - testhelper.AssertEventuallyEqual(t, id, func() interface{} { - params := objects.NewObjectsClassGetParams() - params.WithClassName(className).WithID(id) - object, err := helper.Client(t).Objects.ObjectsClassGet(params, nil) - if err != nil { - return nil - } - - return object.Payload.ID - }) - // deprecated: is here because of backward compatibility reasons - testhelper.AssertEventuallyEqual(t, id, func() interface{} { - params := objects.NewObjectsGetParams().WithID(id) - object, err := helper.Client(t).Objects.ObjectsGet(params, nil) - if err != nil { - return nil - } - - return object.Payload.ID - }) - - // Try to create the same object again and make sure it fails - params = objects.NewObjectsCreateParams().WithBody( - &models.Object{ - ID: id, - Class: "TestObject", - Properties: map[string]interface{}{ - "testString": objectTestString, - }, - }) - - resp, err = helper.Client(t).Objects.ObjectsCreate(params, nil) - helper.AssertRequestFail(t, resp, err, func() { - errResponse, ok := err.(*objects.ObjectsCreateUnprocessableEntity) - if !ok { - t.Fatalf("Did not get not found response, but %#v", err) - } - - assert.Equal(t, fmt.Sprintf("id '%s' already exists", id), errResponse.Payload.Error[0].Message) - }) - }) - - // Check if we can create a Object, and that it's properties are stored correctly. - t.Run("creating a object", func(t *testing.T) { - t.Parallel() - // Set all object values to compare - objectTestString := "Test string" - objectTestInt := 1 - objectTestBoolean := true - objectTestNumber := 1.337 - objectTestDate := "2017-10-06T08:15:30+01:00" - objectTestPhoneNumber := map[string]interface{}{ - "input": "0171 11122233", - "defaultCountry": "DE", - } - - params := objects.NewObjectsCreateParams().WithBody( - &models.Object{ - Class: "TestObject", - Properties: map[string]interface{}{ - "testString": objectTestString, - "testWholeNumber": objectTestInt, - "testTrueFalse": objectTestBoolean, - "testNumber": objectTestNumber, - "testDateTime": objectTestDate, - "testPhoneNumber": objectTestPhoneNumber, - }, - }) - - resp, err := helper.Client(t).Objects.ObjectsCreate(params, nil) - - // Ensure that the response is OK - helper.AssertRequestOk(t, resp, err, func() { - object := resp.Payload - assert.Regexp(t, strfmt.UUIDPattern, object.ID) - - schema, ok := object.Properties.(map[string]interface{}) - if !ok { - t.Fatal("The returned schema is not an JSON object") - } - - testWholeNumber, _ := schema["testWholeNumber"].(json.Number).Int64() - testNumber, _ := schema["testNumber"].(json.Number).Float64() - - expectedParsedPhoneNumber := map[string]interface{}{ - "input": "0171 11122233", - "defaultCountry": "DE", - "countryCode": json.Number("49"), - "internationalFormatted": "+49 171 11122233", - "national": json.Number("17111122233"), - "nationalFormatted": "0171 11122233", - "valid": true, - } - - // Check whether the returned information is the same as the data added - assert.Equal(t, objectTestString, schema["testString"]) - assert.Equal(t, objectTestInt, int(testWholeNumber)) - assert.Equal(t, objectTestBoolean, schema["testTrueFalse"]) - assert.Equal(t, objectTestNumber, testNumber) - assert.Equal(t, objectTestDate, schema["testDateTime"]) - assert.Equal(t, expectedParsedPhoneNumber, schema["testPhoneNumber"]) - }) - }) - - // Examples of how a Object can be invalid. - invalidObjectTestCases := []struct { - // What is wrong in this example - mistake string - - // the example object, with a mistake. - // this is a function, so that we can use utility functions like - // helper.GetWeaviateURL(), which might not be initialized yet - // during the static construction of the examples. - object func() *models.Object - - // Enable the option to perform some extra assertions on the error response - errorCheck func(t *testing.T, err *models.ErrorResponse) - }{ - { - mistake: "missing the class", - object: func() *models.Object { - return &models.Object{ - Properties: map[string]interface{}{ - "testString": "test", - }, - } - }, - errorCheck: func(t *testing.T, err *models.ErrorResponse) { - assert.Equal(t, "invalid object: the given class is empty", err.Error[0].Message) - }, - }, - // AUTO_SCHEMA creates classes automatically - // { - // mistake: "non existing class", - // object: func() *models.Object { - // return &models.Object{ - // Class: "NonExistingClass", - // Properties: map[string]interface{}{ - // "testString": "test", - // }, - // } - // }, - // errorCheck: func(t *testing.T, err *models.ErrorResponse) { - // assert.Equal(t, fmt.Sprintf("invalid object: class '%s' not present in schema", "NonExistingClass"), err.Error[0].Message) - // }, - // }, - // AUTO_SCHEMA creates missing properties automatically - // { - // mistake: "non existing property", - // object: func() *models.Object { - // return &models.Object{ - // Class: "TestObject", - // Properties: map[string]interface{}{ - // "nonExistingProperty": "test", - // }, - // } - // }, - // errorCheck: func(t *testing.T, err *models.ErrorResponse) { - // assert.Equal(t, fmt.Sprintf("invalid object: "+schema.ErrorNoSuchProperty, "nonExistingProperty", "TestObject"), err.Error[0].Message) - // }, - // }, - { - /* TODO gh-616: don't count nr of elements in validation. Just validate keys, and _also_ generate an error on superfluous keys. - E.g. - var cref *string - var type_ *string - var locationUrl *string - - for key, val := range(propertyValue) { - switch key { - case "beacon": cref = val - case "type": type_ = val - case "locationUrl": locationUrl = val - default: - return fmt.Errof("Unexpected key %s", key) - } - } - if cref == nil { return fmt.Errorf("beacon missing") } - if type_ == nil { return fmt.Errorf("type missing") } - if locationUrl == nil { return fmt.Errorf("locationUrl missing") } - - // now everything has a valid state. - */ - mistake: "invalid cref, property missing locationUrl", - object: func() *models.Object { - return &models.Object{ - Class: "TestObject", - Properties: map[string]interface{}{ - "testReference": map[string]interface{}{ - "beacon": fakeObjectId, - "x": nil, - "type": "Object", - }, - }, - } - }, - errorCheck: func(t *testing.T, err *models.ErrorResponse) { - assert.NotNil(t, err) - }, - }, - { - mistake: "invalid property; assign int to string", - object: func() *models.Object { - return &models.Object{ - Class: "TestObject", - Properties: map[string]interface{}{ - "testString": 2, - }, - } - }, - errorCheck: func(t *testing.T, err *models.ErrorResponse) { - assert.Contains(t, - "invalid object: invalid text property 'testString' on class 'TestObject': not a string, but json.Number", - err.Error[0].Message) - }, - }, - } - - // Check that none of the examples of invalid objects can be created. - t.Run("cannot create invalid objects", func(t *testing.T) { - // invalidObjectTestCases defined below this test. - for _, example_ := range invalidObjectTestCases { - t.Run(example_.mistake, func(t *testing.T) { - example := example_ // Needed; example is updated to point to a new test case. - t.Parallel() - - params := objects.NewObjectsCreateParams().WithBody(example.object()) - resp, err := helper.Client(t).Objects.ObjectsCreate(params, nil) - helper.AssertRequestFail(t, resp, err, func() { - errResponse, ok := err.(*objects.ObjectsCreateUnprocessableEntity) - if !ok { - t.Fatalf("Did not get not found response, but %#v", err) - } - example.errorCheck(t, errResponse.Payload) - }) - }) - } - }) -} diff --git a/test/acceptance/objects/rapid_updates_add_reference_test.go b/test/acceptance/objects/rapid_updates_add_reference_test.go deleted file mode 100644 index 16c25dae15f2f3181e75e9c33b27b608c214b86b..0000000000000000000000000000000000000000 --- a/test/acceptance/objects/rapid_updates_add_reference_test.go +++ /dev/null @@ -1,153 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "fmt" - "strings" - "testing" - "time" - - "github.com/go-openapi/strfmt" - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/client/objects" - clschema "github.com/weaviate/weaviate/client/schema" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/test/helper" -) - -// This aims to prevent a regression on -// https://github.com/weaviate/weaviate/issues/1016 -// The issue was that rapid POST .../references/... request in succession would -// overwrite each other due to the eventual consistency nature of the used -// backend (esvector). This bug is considered fix if n items can be rapidly -// added and a subsequent GET request of the source resource contains all -// previously added references. -func Test_RapidlyAddingReferences(t *testing.T) { - sourceClass := "SequenceReferenceTestSource" - targetClass := "SequenceReferenceTestTarget" - - sourceID := strfmt.UUID("96ce03ca-58ed-48e1-a0f1-51f63fa9aa12") - - targetIDs := []strfmt.UUID{ - "ce1a4756-b7ce-44fa-b079-45a7ec400882", - "e1edb4ff-570c-4f0b-a1a1-18af118369aa", - "25d22c70-3df0-4e5c-b8c1-a88d4d2771ef", - "6f2a0708-3e8e-4a68-9763-26c465d8bf83", - "c4dfae47-ebcf-4808-9122-1c67898ec140", - "754bd925-1900-4f93-9f5d-27631eb618bb", - "babba820-e3f5-4e8d-a354-76f2cb13fdba", - "270942da-1999-40cd-a580-a91aa144b6c0", - "a7a06618-6d50-4654-be75-2c9f639a6368", - "47ba1d2b-6b8c-4b3b-92a8-46574a069ae8", - } - - t.Run("adding the required schema", func(t *testing.T) { - t.Run("target class", func(t *testing.T) { - params := clschema.NewSchemaObjectsCreateParams().WithObjectClass( - &models.Class{ - Class: targetClass, - Properties: []*models.Property{ - { - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - Name: "name", - }, - }, - }, - ) - resp, err := helper.Client(t).Schema.SchemaObjectsCreate(params, nil) - helper.AssertRequestOk(t, resp, err, nil) - }) - - t.Run("source class", func(t *testing.T) { - params := clschema.NewSchemaObjectsCreateParams().WithObjectClass( - &models.Class{ - Class: sourceClass, - Properties: []*models.Property{ - { - DataType: []string{targetClass}, - Name: "toTarget", - }, - { - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - Name: "name", - }, - }, - }, - ) - resp, err := helper.Client(t).Schema.SchemaObjectsCreate(params, nil) - helper.AssertRequestOk(t, resp, err, nil) - }) - }) - - t.Run("adding all objects (without referencing)", func(t *testing.T) { - t.Run("source object", func(t *testing.T) { - assertCreateObjectWithID(t, sourceClass, "", sourceID, map[string]interface{}{ - "name": "Source Object", - }) - }) - - t.Run("target objects", func(t *testing.T) { - for i, id := range targetIDs { - assertCreateObjectWithID(t, targetClass, "", id, map[string]interface{}{ - "name": fmt.Sprintf("target object %d", i), - }) - } - }) - }) - - t.Run("waiting for the last added object to be present", func(t *testing.T) { - assertGetObjectEventually(t, targetIDs[len(targetIDs)-1]) - }) - - t.Run("placing all references in succession", func(t *testing.T) { - for _, id := range targetIDs { - params := objects.NewObjectsReferencesCreateParams(). - WithID(sourceID). - WithPropertyName("toTarget"). - WithBody( - &models.SingleRef{ - Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/%s", id)), - }, - ) - - res, err := helper.Client(t).Objects.ObjectsReferencesCreate(params, nil) - helper.AssertRequestOk(t, res, err, nil) - } - }) - - // wait for index refresh - time.Sleep(2 * time.Second) // TODO: improve through polling - - t.Run("checking which refs were set", func(t *testing.T) { - source := assertGetObject(t, sourceID) - - var foundIDs []strfmt.UUID - // extract IDs - for _, ref := range source.Properties.(map[string]interface{})["toTarget"].([]interface{}) { - beacon := ref.(map[string]interface{})["beacon"].(string) - chunks := strings.Split(beacon, "/") - foundIDs = append(foundIDs, strfmt.UUID(chunks[len(chunks)-1])) - } - - assert.ElementsMatch(t, targetIDs, foundIDs) - }) - - // cleanup - helper.Client(t).Schema.SchemaObjectsDelete( - clschema.NewSchemaObjectsDeleteParams().WithClassName(sourceClass), nil) - helper.Client(t).Schema.SchemaObjectsDelete( - clschema.NewSchemaObjectsDeleteParams().WithClassName(targetClass), nil) -} diff --git a/test/acceptance/objects/setup_test.go b/test/acceptance/objects/setup_test.go deleted file mode 100644 index ac12de0d52605d44e6861869370a57a8d3ff27f8..0000000000000000000000000000000000000000 --- a/test/acceptance/objects/setup_test.go +++ /dev/null @@ -1,184 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "context" - "fmt" - "testing" - - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/client/objects" - - clschema "github.com/weaviate/weaviate/client/schema" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/test/docker" - "github.com/weaviate/weaviate/test/helper" -) - -// Tests that sort parameters are validated with the correct class -func TestSort(t *testing.T) { - createObjectClass(t, &models.Class{ - Class: "ClassToSort", - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - }) - defer deleteObjectClass(t, "ClassToSort") - - createObjectClass(t, &models.Class{ - Class: "OtherClass", - Properties: []*models.Property{ - { - Name: "ref", - DataType: []string{"ClassToSort"}, - }, - }, - }) - defer deleteObjectClass(t, "OtherClass") - - listParams := objects.NewObjectsListParams() - nameClass := "ClassToSort" - nameProp := "name" - limit := int64(5) - listParams.Class = &nameClass - listParams.Sort = &nameProp - listParams.Limit = &limit - - _, err := helper.Client(t).Objects.ObjectsList(listParams, nil) - require.Nil(t, err, "should not error") -} - -func TestObjects_AsyncIndexing(t *testing.T) { - ctx := context.Background() - compose, err := docker.New(). - WithWeaviate(). - WithText2VecContextionary(). - WithWeaviateEnv("ASYNC_INDEXING", "true"). - Start(ctx) - require.NoError(t, err) - defer func() { - require.NoError(t, compose.Terminate(ctx)) - }() - - defer helper.SetupClient(fmt.Sprintf("%s:%s", helper.ServerHost, helper.ServerPort)) - helper.SetupClient(compose.GetWeaviate().URI()) - - testObjects(t) -} - -func TestObjects_SyncIndexing(t *testing.T) { - testObjects(t) -} - -func testObjects(t *testing.T) { - createObjectClass(t, &models.Class{ - Class: "TestObject", - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "vectorizeClassName": true, - }, - }, - Properties: []*models.Property{ - { - Name: "testString", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "testWholeNumber", - DataType: []string{"int"}, - }, - { - Name: "testNumber", - DataType: []string{"number"}, - }, - { - Name: "testDateTime", - DataType: []string{"date"}, - }, - { - Name: "testTrueFalse", - DataType: []string{"boolean"}, - }, - { - Name: "testPhoneNumber", - DataType: []string{"phoneNumber"}, - }, - }, - }) - createObjectClass(t, &models.Class{ - Class: "TestObjectCustomVector", - Vectorizer: "none", - Properties: []*models.Property{ - { - Name: "description", - DataType: []string{"text"}, - }, - }, - }) - createObjectClass(t, &models.Class{ - Class: "TestDeleteClassOne", - Vectorizer: "none", - Properties: []*models.Property{ - { - Name: "text", - DataType: []string{"text"}, - }, - }, - }) - createObjectClass(t, &models.Class{ - Class: "TestDeleteClassTwo", - Vectorizer: "none", - Properties: []*models.Property{ - { - Name: "text", - DataType: []string{"text"}, - }, - }, - }) - - // tests - t.Run("listing objects", listingObjects) - t.Run("searching for neighbors", searchNeighbors) - t.Run("running a feature projection", featureProjection) - t.Run("creating objects", creatingObjects) - - t.Run("custom vector journey", customVectors) - t.Run("auto schema", autoSchemaObjects) - t.Run("checking object's existence", checkObjects) - t.Run("delete request deletes all objects with a given ID", deleteAllObjectsFromAllClasses) - - // tear down - deleteObjectClass(t, "TestObject") - deleteObjectClass(t, "TestObjectCustomVector") - deleteObjectClass(t, "NonExistingClass") - deleteObjectClass(t, "TestDeleteClassOne") - deleteObjectClass(t, "TestDeleteClassTwo") -} - -func createObjectClass(t *testing.T, class *models.Class) { - params := clschema.NewSchemaObjectsCreateParams().WithObjectClass(class) - resp, err := helper.Client(t).Schema.SchemaObjectsCreate(params, nil) - helper.AssertRequestOk(t, resp, err, nil) -} - -func deleteObjectClass(t *testing.T, class string) { - delParams := clschema.NewSchemaObjectsDeleteParams().WithClassName(class) - delRes, err := helper.Client(t).Schema.SchemaObjectsDelete(delParams, nil) - helper.AssertRequestOk(t, delRes, err, nil) -} diff --git a/test/acceptance/objects/skip_vector_index_test.go b/test/acceptance/objects/skip_vector_index_test.go deleted file mode 100644 index 9e656394dd9c84d69fb165711277a05eba18d985..0000000000000000000000000000000000000000 --- a/test/acceptance/objects/skip_vector_index_test.go +++ /dev/null @@ -1,69 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "testing" - - "github.com/go-openapi/strfmt" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/client/objects" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/test/helper" -) - -func TestSkipVectorIndex(t *testing.T) { - // Import a class with vectorizer 'none' and 'skipVectorIndex: true', import - // objects without vectors. - - t.Run("create schema", func(t *testing.T) { - createObjectClass(t, &models.Class{ - Class: "TestSkipVectorIndex", - VectorIndexConfig: map[string]interface{}{ - "skip": true, - }, - Vectorizer: "none", - Properties: []*models.Property{ - { - Name: "name", - DataType: []string{"text"}, - }, - }, - }) - }) - - id := strfmt.UUID("d1d58565-3c9b-4ca6-ac7f-43f739700a1d") - - t.Run("create object", func(t *testing.T) { - params := objects.NewObjectsCreateParams().WithBody( - &models.Object{ - ID: id, - Class: "TestSkipVectorIndex", - Properties: map[string]interface{}{"name": "Jane Doe"}, - }) - _, err := helper.Client(t).Objects.ObjectsCreate(params, nil) - require.Nil(t, err, "creation should succeed") - }) - - t.Run("get obj by ID", func(t *testing.T) { - params := objects.NewObjectsGetParams().WithID(id) - obj, err := helper.Client(t).Objects.ObjectsGet(params, nil) - require.Nil(t, err, "object can be retrieved by id") - - assert.Equal(t, "Jane Doe", obj.Payload.Properties.(map[string]interface{})["name"].(string)) - }) - - t.Run("tear down", func(t *testing.T) { - deleteObjectClass(t, "TestSkipVectorIndex") - }) -} diff --git a/test/acceptance/replication/crud_ops.go b/test/acceptance/replication/crud_ops.go deleted file mode 100644 index 3fab8175b36dbf7986c9b30178de431dec235c60..0000000000000000000000000000000000000000 --- a/test/acceptance/replication/crud_ops.go +++ /dev/null @@ -1,282 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package replication - -import ( - "encoding/json" - "fmt" - "strings" - "testing" - - "github.com/go-openapi/strfmt" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/client/nodes" - "github.com/weaviate/weaviate/client/objects" - "github.com/weaviate/weaviate/entities/filters" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/verbosity" - "github.com/weaviate/weaviate/test/helper" - graphqlhelper "github.com/weaviate/weaviate/test/helper/graphql" - "github.com/weaviate/weaviate/usecases/replica" -) - -func getClass(t *testing.T, host, class string) *models.Class { - helper.SetupClient(host) - return helper.GetClass(t, class) -} - -func updateClass(t *testing.T, host string, class *models.Class) { - helper.SetupClient(host) - helper.UpdateClass(t, class) -} - -func createObject(t *testing.T, host string, obj *models.Object) { - helper.SetupClient(host) - helper.CreateObject(t, obj) -} - -func createObjectCL(t *testing.T, host string, obj *models.Object, cl replica.ConsistencyLevel) { - helper.SetupClient(host) - helper.CreateObjectCL(t, obj, cl) -} - -func createTenantObject(t *testing.T, host string, obj *models.Object) { - helper.SetupClient(host) - helper.CreateObject(t, obj) -} - -func createObjects(t *testing.T, host string, batch []*models.Object) { - helper.SetupClient(host) - helper.CreateObjectsBatch(t, batch) -} - -func createTenantObjects(t *testing.T, host string, batch []*models.Object) { - helper.SetupClient(host) - helper.CreateObjectsBatch(t, batch) -} - -func getObject(t *testing.T, host, class string, id strfmt.UUID, withVec bool) (*models.Object, error) { - helper.SetupClient(host) - var include string - if withVec { - include = "vector" - } - return helper.GetObject(t, class, id, include) -} - -func getTenantObject(t *testing.T, host, class string, id strfmt.UUID, tenant string) (*models.Object, error) { - helper.SetupClient(host) - return helper.TenantObject(t, class, id, tenant) -} - -func objectExistsCL(t *testing.T, host, class string, id strfmt.UUID, cl replica.ConsistencyLevel) (bool, error) { - helper.SetupClient(host) - return helper.ObjectExistsCL(t, class, id, cl) -} - -func getObjectCL(t *testing.T, host, class string, id strfmt.UUID, cl replica.ConsistencyLevel) (*models.Object, error) { - helper.SetupClient(host) - return helper.GetObjectCL(t, class, id, cl) -} - -func getObjectFromNode(t *testing.T, host, class string, id strfmt.UUID, nodename string) (*models.Object, error) { - helper.SetupClient(host) - return helper.GetObjectFromNode(t, class, id, nodename) -} - -func getTenantObjectFromNode(t *testing.T, host, class string, id strfmt.UUID, nodename, tenant string) (*models.Object, error) { - helper.SetupClient(host) - return helper.GetTenantObjectFromNode(t, class, id, nodename, tenant) -} - -func patchObject(t *testing.T, host string, patch *models.Object) { - helper.SetupClient(host) - helper.PatchObject(t, patch) -} - -func patchTenantObject(t *testing.T, host string, patch *models.Object) { - helper.SetupClient(host) - helper.PatchObject(t, patch) -} - -func updateObjectCL(t *testing.T, host string, obj *models.Object, cl replica.ConsistencyLevel) { - helper.SetupClient(host) - helper.UpdateObjectCL(t, obj, cl) -} - -func addReferences(t *testing.T, host string, refs []*models.BatchReference) { - helper.SetupClient(host) - resp, err := helper.AddReferences(t, refs) - helper.CheckReferencesBatchResponse(t, resp, err) -} - -func addTenantReferences(t *testing.T, host string, refs []*models.BatchReference) { - helper.SetupClient(host) - resp, err := helper.AddReferences(t, refs) - helper.CheckReferencesBatchResponse(t, resp, err) -} - -func deleteObject(t *testing.T, host, class string, id strfmt.UUID) { - helper.SetupClient(host) - - toDelete, err := helper.GetObject(t, class, id) - require.Nil(t, err) - - helper.DeleteObject(t, toDelete) - - _, err = helper.GetObject(t, class, id) - assert.Equal(t, &objects.ObjectsClassGetNotFound{}, err) -} - -func deleteTenantObject(t *testing.T, host, class string, id strfmt.UUID, tenant string) { - helper.SetupClient(host) - helper.DeleteTenantObject(t, class, id, tenant) - - _, err := helper.TenantObject(t, class, id, tenant) - assert.Equal(t, &objects.ObjectsClassGetNotFound{}, err) -} - -func deleteObjects(t *testing.T, host, class string, path []string, valueText string) { - helper.SetupClient(host) - - batchDelete := &models.BatchDelete{ - Match: &models.BatchDeleteMatch{ - Class: class, - Where: &models.WhereFilter{ - Operator: filters.OperatorLike.Name(), - Path: path, - ValueText: &valueText, - }, - }, - } - helper.DeleteObjectsBatch(t, batchDelete) - - resp := gqlGet(t, host, class, replica.All) - assert.Empty(t, resp) -} - -func deleteTenantObjects(t *testing.T, host, class string, path []string, valueText, tenant string) { - helper.SetupClient(host) - - batchDelete := &models.BatchDelete{ - Match: &models.BatchDeleteMatch{ - Class: class, - Where: &models.WhereFilter{ - Operator: filters.OperatorLike.Name(), - Path: path, - ValueText: &valueText, - }, - }, - } - resp, err := helper.DeleteTenantObjectsBatch(t, batchDelete, tenant) - helper.AssertRequestOk(t, resp, err, nil) - - deleted := gqlTenantGet(t, host, class, replica.All, tenant) - assert.Empty(t, deleted) -} - -func gqlGet(t *testing.T, host, class string, cl replica.ConsistencyLevel, fields ...string) []interface{} { - helper.SetupClient(host) - - if cl == "" { - cl = replica.Quorum - } - - q := fmt.Sprintf("{Get {%s (consistencyLevel: %s)", class, cl) + " {%s}}}" - if len(fields) == 0 { - fields = []string{"_additional{id isConsistent vector}"} - } - q = fmt.Sprintf(q, strings.Join(fields, " ")) - - return gqlDo(t, class, q) -} - -func gqlGetNearVec(t *testing.T, host, class string, vec []interface{}, cl replica.ConsistencyLevel, fields ...string) []interface{} { - helper.SetupClient(host) - - if cl == "" { - cl = replica.Quorum - } - - q := fmt.Sprintf("{Get {%s (consistencyLevel: %s, nearVector: {vector: %s, certainty: 0.8})", - class, cl, vec2String(vec)) + " {%s}}}" - if len(fields) == 0 { - fields = []string{"_additional{id isConsistent}"} - } - q = fmt.Sprintf(q, strings.Join(fields, " ")) - - return gqlDo(t, class, q) -} - -func gqlDo(t *testing.T, class, query string) []interface{} { - resp := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - - result := resp.Get("Get").Get(class) - return result.Result.([]interface{}) -} - -func gqlTenantGet(t *testing.T, host, class string, cl replica.ConsistencyLevel, - tenant string, fields ...string, -) []interface{} { - helper.SetupClient(host) - - if cl == "" { - cl = replica.Quorum - } - - q := fmt.Sprintf("{Get {%s (tenant: %q, consistencyLevel: %s)", class, tenant, cl) + " {%s}}}" - if len(fields) == 0 { - fields = []string{"_additional{id isConsistent}"} - } - q = fmt.Sprintf(q, strings.Join(fields, " ")) - - resp := graphqlhelper.AssertGraphQL(t, helper.RootAuth, q) - - result := resp.Get("Get").Get(class) - return result.Result.([]interface{}) -} - -func countTenantObjects(t *testing.T, host, class string, - tenant string, -) int64 { - helper.SetupClient(host) - - q := fmt.Sprintf(`{Aggregate{%s(tenant: %q){meta{count}}}}`, class, tenant) - - resp := graphqlhelper.AssertGraphQL(t, helper.RootAuth, q) - - result := resp.Get("Aggregate").Get(class).AsSlice() - require.Len(t, result, 1) - meta := result[0].(map[string]interface{})["meta"].(map[string]interface{}) - count, err := meta["count"].(json.Number).Int64() - require.Nil(t, err) - return count -} - -func getNodes(t *testing.T, host string) *models.NodesStatusResponse { - helper.SetupClient(host) - verbose := verbosity.OutputVerbose - params := nodes.NewNodesGetParams().WithOutput(&verbose) - resp, err := helper.Client(t).Nodes.NodesGet(params, nil) - helper.AssertRequestOk(t, resp, err, nil) - return resp.Payload -} - -func vec2String(v []interface{}) (s string) { - for _, n := range v { - x := n.(json.Number) - s = fmt.Sprintf("%s, %s", s, x.String()) - } - s = strings.TrimLeft(s, ", ") - return fmt.Sprintf("[%s]", s) -} diff --git a/test/acceptance/replication/crud_test.go b/test/acceptance/replication/crud_test.go deleted file mode 100644 index dc13ded24f9b81415470689ed96e4bbece5636ec..0000000000000000000000000000000000000000 --- a/test/acceptance/replication/crud_test.go +++ /dev/null @@ -1,439 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package replication - -import ( - "context" - "encoding/json" - "fmt" - "testing" - "time" - - "github.com/go-openapi/strfmt" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/client/objects" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema/crossref" - "github.com/weaviate/weaviate/test/docker" - "github.com/weaviate/weaviate/test/helper" - "github.com/weaviate/weaviate/test/helper/sample-schema/articles" - "github.com/weaviate/weaviate/usecases/replica" -) - -var ( - paragraphIDs = []strfmt.UUID{ - strfmt.UUID("3bf331ac-8c86-4f95-b127-2f8f96bbc093"), - strfmt.UUID("47b26ba1-6bc9-41f8-a655-8b9a5b60e1a3"), - strfmt.UUID("5fef6289-28d2-4ea2-82a9-48eb501200cd"), - strfmt.UUID("34a673b4-8859-4cb4-bb30-27f5622b47e9"), - strfmt.UUID("9fa362f5-c2dc-4fb8-b5b2-11701adc5f75"), - strfmt.UUID("63735238-6723-4caf-9eaa-113120968ff4"), - strfmt.UUID("2236744d-b2d2-40e5-95d8-2574f20a7126"), - strfmt.UUID("1a54e25d-aaf9-48d2-bc3c-bef00b556297"), - strfmt.UUID("0b8a0e70-a240-44b2-ac6d-26dda97523b9"), - strfmt.UUID("50566856-5d0a-4fb1-a390-e099bc236f66"), - } - - articleIDs = []strfmt.UUID{ - strfmt.UUID("aeaf8743-5a8f-4149-b960-444181d3131a"), - strfmt.UUID("2a1e9834-064e-4ca8-9efc-35707c6bae6d"), - strfmt.UUID("8d101c0c-4deb-48d0-805c-d9c691042a1a"), - strfmt.UUID("b9715fec-ef6c-4e8d-a89e-55e2eebee3f6"), - strfmt.UUID("faf520f2-f6c3-4cdf-9c16-0348ffd0f8ac"), - strfmt.UUID("d4c695dd-4dc7-4e49-bc73-089ef5f90fc8"), - strfmt.UUID("c7949324-e07f-4ffc-8be0-194f0470d375"), - strfmt.UUID("9c112e01-7759-43ed-a6e8-5defb267c8ee"), - strfmt.UUID("9bf847f3-3a1a-45a5-b656-311163e536b5"), - strfmt.UUID("c1975388-d67c-404a-ae77-5983fbaea4bb"), - } -) - -func immediateReplicaCRUD(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) - defer cancel() - - compose, err := docker.New(). - WithWeaviateCluster(). - WithText2VecContextionary(). - Start(ctx) - require.Nil(t, err) - defer func() { - if err := compose.Terminate(ctx); err != nil { - t.Fatalf("failed to terminate test containers: %s", err.Error()) - } - }() - - helper.SetupClient(compose.GetWeaviate().URI()) - paragraphClass := articles.ParagraphsClass() - articleClass := articles.ArticlesClass() - - t.Run("create schema", func(t *testing.T) { - paragraphClass.ReplicationConfig = &models.ReplicationConfig{ - Factor: 2, - } - helper.CreateClass(t, paragraphClass) - articleClass.ReplicationConfig = &models.ReplicationConfig{ - Factor: 2, - } - helper.CreateClass(t, articleClass) - }) - - t.Run("insert paragraphs batch", func(t *testing.T) { - t.Run("create objects on node 1", func(t *testing.T) { - batch := make([]*models.Object, len(paragraphIDs)) - for i, id := range paragraphIDs { - batch[i] = articles.NewParagraph(). - WithID(id). - WithContents(fmt.Sprintf("paragraph#%d", i)). - Object() - } - createObjects(t, compose.GetWeaviate().URI(), batch) - }) - - t.Run("stop node 1", func(t *testing.T) { - stopNode(ctx, t, compose, compose.GetWeaviate().Name()) - }) - - t.Run("assert objects exist on node 2", func(t *testing.T) { - resp := gqlGet(t, compose.GetWeaviateNode2().URI(), "Paragraph", replica.One) - assert.Len(t, resp, len(paragraphIDs)) - }) - - t.Run("restart node 1", func(t *testing.T) { - restartNode1(ctx, t, compose) - }) - }) - - t.Run("insert articles individually", func(t *testing.T) { - t.Run("create objects on node 2", func(t *testing.T) { - for i, id := range articleIDs { - obj := articles.NewArticle(). - WithID(id). - WithTitle(fmt.Sprintf("Article#%d", i)). - Object() - createObject(t, compose.GetWeaviateNode2().URI(), obj) - } - }) - - t.Run("stop node 2", func(t *testing.T) { - stopNode(ctx, t, compose, compose.GetWeaviateNode2().Name()) - }) - - t.Run("assert objects exist on node 1", func(t *testing.T) { - resp := gqlGet(t, compose.GetWeaviate().URI(), "Article", replica.One) - assert.Len(t, resp, len(articleIDs)) - }) - - t.Run("restart node 2", func(t *testing.T) { - err = compose.Start(ctx, compose.GetWeaviateNode2().Name()) - require.Nil(t, err) - }) - }) - - t.Run("add references", func(t *testing.T) { - refs := make([]*models.BatchReference, len(articleIDs)) - for i := range articleIDs { - refs[i] = &models.BatchReference{ - From: strfmt.URI(crossref.NewSource("Article", "hasParagraphs", articleIDs[i]).String()), - To: strfmt.URI(crossref.NewLocalhost("Paragraph", paragraphIDs[i]).String()), - } - } - - t.Run("add references to node 1", func(t *testing.T) { - addReferences(t, compose.GetWeaviate().URI(), refs) - }) - - t.Run("stop node 1", func(t *testing.T) { - stopNode(ctx, t, compose, compose.GetWeaviate().Name()) - }) - - t.Run("assert references were added successfully to node 2", func(t *testing.T) { - type additional struct { - ID strfmt.UUID `json:"id"` - } - - type article struct { - Additional additional `json:"_additional"` - HasParagraphs []struct { - Additional additional `json:"_additional"` - } `json:"hasParagraphs"` - } - - // maps article id to referenced paragraph id - refPairs := make(map[strfmt.UUID]strfmt.UUID) - resp := gqlGet(t, compose.GetWeaviateNode2().URI(), "Article", replica.One, - "_additional{id}", "hasParagraphs {... on Paragraph {_additional{id}}}") - assert.Len(t, resp, len(articleIDs)) - - for _, r := range resp { - b, err := json.Marshal(r) - require.Nil(t, err) - var art article - err = json.Unmarshal(b, &art) - require.Nil(t, err) - require.Len(t, art.HasParagraphs, 1) - refPairs[art.Additional.ID] = art.HasParagraphs[0].Additional.ID - } - - for i := range articleIDs { - paragraphID, ok := refPairs[articleIDs[i]] - require.True(t, ok, "expected %q to be in refPairs: %+v", articleIDs[i], refPairs) - assert.Equal(t, paragraphIDs[i], paragraphID) - } - }) - - t.Run("restart node 1", func(t *testing.T) { - restartNode1(ctx, t, compose) - }) - }) - - t.Run("patch an object", func(t *testing.T) { - before, err := getObject(t, compose.GetWeaviate().URI(), "Article", articleIDs[0], false) - require.Nil(t, err) - newTitle := "Article#9000" - - t.Run("execute object patch on node 2", func(t *testing.T) { - patch := &models.Object{ - ID: before.ID, - Class: "Article", - Properties: map[string]interface{}{"title": newTitle}, - } - patchObject(t, compose.GetWeaviateNode2().URI(), patch) - }) - - t.Run("stop node 2", func(t *testing.T) { - stopNode(ctx, t, compose, compose.GetWeaviateNode2().Name()) - }) - - t.Run("assert object is patched on node 1", func(t *testing.T) { - after, err := getObjectFromNode(t, compose.GetWeaviate().URI(), "Article", articleIDs[0], "node1") - require.Nil(t, err) - - newVal, ok := after.Properties.(map[string]interface{})["title"] - require.True(t, ok) - assert.Equal(t, newTitle, newVal) - }) - - t.Run("restart node 2", func(t *testing.T) { - err = compose.Start(ctx, compose.GetWeaviateNode2().Name()) - require.Nil(t, err) - }) - }) - - t.Run("delete an object", func(t *testing.T) { - t.Run("execute delete object on node 1", func(t *testing.T) { - deleteObject(t, compose.GetWeaviate().URI(), "Article", articleIDs[0]) - }) - - t.Run("stop node 1", func(t *testing.T) { - stopNode(ctx, t, compose, compose.GetWeaviate().Name()) - }) - - t.Run("assert object removed from node 2", func(t *testing.T) { - _, err := getObjectFromNode(t, compose.GetWeaviateNode2().URI(), "Article", articleIDs[0], "node2") - assert.Equal(t, &objects.ObjectsClassGetNotFound{}, err) - }) - - t.Run("restart node 1", func(t *testing.T) { - restartNode1(ctx, t, compose) - }) - }) - - t.Run("batch delete all objects", func(t *testing.T) { - t.Run("execute batch delete on node 2", func(t *testing.T) { - deleteObjects(t, compose.GetWeaviateNode2().URI(), - "Article", []string{"title"}, "Article#*") - }) - - t.Run("stop node 2", func(t *testing.T) { - stopNode(ctx, t, compose, compose.GetWeaviateNode2().Name()) - }) - - t.Run("assert objects are removed from node 1", func(t *testing.T) { - resp := gqlGet(t, compose.GetWeaviate().URI(), "Article", replica.One) - assert.Empty(t, resp) - }) - - t.Run("restart node 2", func(t *testing.T) { - err = compose.Start(ctx, compose.GetWeaviateNode2().Name()) - require.Nil(t, err) - }) - }) -} - -func eventualReplicaCRUD(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) - defer cancel() - - compose, err := docker.New(). - WithWeaviateCluster(). - WithText2VecContextionary(). - Start(ctx) - require.Nil(t, err) - defer func() { - if err := compose.Terminate(ctx); err != nil { - t.Fatalf("failed to terminate test containers: %s", err.Error()) - } - }() - - helper.SetupClient(compose.GetWeaviate().URI()) - paragraphClass := articles.ParagraphsClass() - articleClass := articles.ArticlesClass() - - t.Run("create schema on node 1", func(t *testing.T) { - paragraphClass.ShardingConfig = map[string]interface{}{"desiredCount": 1} - helper.CreateClass(t, paragraphClass) - articleClass.ShardingConfig = map[string]interface{}{"desiredCount": 1} - helper.CreateClass(t, articleClass) - }) - - t.Run("insert paragraphs batch on node 1", func(t *testing.T) { - batch := make([]*models.Object, len(paragraphIDs)) - for i, id := range paragraphIDs { - batch[i] = articles.NewParagraph(). - WithID(id). - WithContents(fmt.Sprintf("paragraph#%d", i)). - Object() - } - createObjects(t, compose.GetWeaviate().URI(), batch) - }) - - t.Run("insert articles batch on node 1", func(t *testing.T) { - batch := make([]*models.Object, len(articleIDs)) - for i, id := range articleIDs { - batch[i] = articles.NewArticle(). - WithID(id). - WithTitle(fmt.Sprintf("Article#%d", i)). - Object() - } - createObjects(t, compose.GetWeaviate().URI(), batch) - }) - - t.Run("configure classes to replicate to node 2", func(t *testing.T) { - ac := helper.GetClass(t, "Article") - ac.ReplicationConfig = &models.ReplicationConfig{ - Factor: 2, - } - helper.UpdateClass(t, ac) - - pc := helper.GetClass(t, "Paragraph") - pc.ReplicationConfig = &models.ReplicationConfig{ - Factor: 2, - } - helper.UpdateClass(t, pc) - }) - - t.Run("stop node 1", func(t *testing.T) { - stopNode(ctx, t, compose, compose.GetWeaviate().Name()) - }) - - t.Run("assert all previous data replicated to node 2", func(t *testing.T) { - resp := gqlGet(t, compose.GetWeaviateNode2().URI(), "Article", replica.One) - assert.Len(t, resp, len(articleIDs)) - resp = gqlGet(t, compose.GetWeaviateNode2().URI(), "Paragraph", replica.One) - assert.Len(t, resp, len(paragraphIDs)) - }) - - t.Run("restart node 1", func(t *testing.T) { - restartNode1(ctx, t, compose) - }) - - t.Run("assert any future writes are replicated", func(t *testing.T) { - t.Run("patch an object", func(t *testing.T) { - before, err := getObject(t, compose.GetWeaviate().URI(), "Article", articleIDs[0], false) - require.Nil(t, err) - newTitle := "Article#9000" - - t.Run("execute object patch on node 2", func(t *testing.T) { - patch := &models.Object{ - ID: before.ID, - Class: "Article", - Properties: map[string]interface{}{"title": newTitle}, - } - patchObject(t, compose.GetWeaviateNode2().URI(), patch) - }) - - t.Run("stop node 2", func(t *testing.T) { - stopNode(ctx, t, compose, compose.GetWeaviateNode2().Name()) - }) - - t.Run("assert object is patched on node 1", func(t *testing.T) { - after, err := getObjectFromNode(t, compose.GetWeaviate().URI(), "Article", articleIDs[0], "node1") - require.Nil(t, err) - - newVal, ok := after.Properties.(map[string]interface{})["title"] - require.True(t, ok) - assert.Equal(t, newTitle, newVal) - }) - - t.Run("restart node 2", func(t *testing.T) { - err = compose.Start(ctx, compose.GetWeaviateNode2().Name()) - require.Nil(t, err) - }) - }) - - t.Run("delete an object", func(t *testing.T) { - t.Run("execute delete object on node 1", func(t *testing.T) { - deleteObject(t, compose.GetWeaviate().URI(), "Article", articleIDs[0]) - }) - - t.Run("stop node 1", func(t *testing.T) { - stopNode(ctx, t, compose, compose.GetWeaviate().Name()) - }) - - t.Run("assert object removed from node 2", func(t *testing.T) { - _, err := getObjectFromNode(t, compose.GetWeaviateNode2().URI(), "Article", articleIDs[0], "node2") - assert.Equal(t, &objects.ObjectsClassGetNotFound{}, err) - }) - - t.Run("restart node 1", func(t *testing.T) { - restartNode1(ctx, t, compose) - }) - }) - - t.Run("batch delete all objects", func(t *testing.T) { - t.Run("execute batch delete on node 2", func(t *testing.T) { - deleteObjects(t, compose.GetWeaviateNode2().URI(), - "Article", []string{"title"}, "Article#*") - }) - - t.Run("stop node 2", func(t *testing.T) { - stopNode(ctx, t, compose, compose.GetWeaviateNode2().Name()) - }) - - t.Run("assert objects are removed from node 1", func(t *testing.T) { - resp := gqlGet(t, compose.GetWeaviate().URI(), "Article", replica.One) - assert.Empty(t, resp) - }) - - t.Run("restart node 2", func(t *testing.T) { - err = compose.Start(ctx, compose.GetWeaviateNode2().Name()) - require.Nil(t, err) - }) - }) - }) -} - -func restartNode1(ctx context.Context, t *testing.T, compose *docker.DockerCompose) { - // since node1 is the gossip "leader", node 2 must be stopped and restarted - // after node1 to re-facilitate internode communication - stopNode(ctx, t, compose, compose.GetWeaviateNode2().Name()) - require.Nil(t, compose.Start(ctx, compose.GetWeaviate().Name())) - require.Nil(t, compose.Start(ctx, compose.GetWeaviateNode2().Name())) - <-time.After(1 * time.Second) // wait for initialization -} - -func stopNode(ctx context.Context, t *testing.T, compose *docker.DockerCompose, container string) { - require.Nil(t, compose.Stop(ctx, container, nil)) - <-time.After(1 * time.Second) // give time for shutdown -} diff --git a/test/acceptance/replication/graphql_test.go b/test/acceptance/replication/graphql_test.go deleted file mode 100644 index a565b45b44ee012f0ada8d3511616efc133a71a7..0000000000000000000000000000000000000000 --- a/test/acceptance/replication/graphql_test.go +++ /dev/null @@ -1,130 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package replication - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/test/docker" - "github.com/weaviate/weaviate/test/helper" - "github.com/weaviate/weaviate/test/helper/sample-schema/articles" - "github.com/weaviate/weaviate/usecases/replica" -) - -func graphqlSearch(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) - defer cancel() - - compose, err := docker.New(). - WithWeaviateCluster(). - WithText2VecContextionary(). - Start(ctx) - require.Nil(t, err) - defer func() { - if err := compose.Terminate(ctx); err != nil { - t.Fatalf("failed to terminate test containers: %s", err.Error()) - } - }() - - helper.SetupClient(compose.GetWeaviate().URI()) - paragraphClass := articles.ParagraphsClass() - paragraphClass.Vectorizer = "text2vec-contextionary" - articleClass := articles.ArticlesClass() - - t.Run("create schema", func(t *testing.T) { - paragraphClass.ReplicationConfig = &models.ReplicationConfig{ - Factor: 2, - } - helper.CreateClass(t, paragraphClass) - articleClass.ReplicationConfig = &models.ReplicationConfig{ - Factor: 2, - } - helper.CreateClass(t, articleClass) - }) - - t.Run("insert paragraphs", func(t *testing.T) { - batch := make([]*models.Object, len(paragraphIDs)) - for i, id := range paragraphIDs { - batch[i] = articles.NewParagraph(). - WithID(id). - WithContents(fmt.Sprintf("paragraph#%d", i)). - Object() - } - createObjects(t, compose.GetWeaviate().URI(), batch) - }) - - t.Run("insert articles", func(t *testing.T) { - batch := make([]*models.Object, len(articleIDs)) - for i, id := range articleIDs { - batch[i] = articles.NewArticle(). - WithID(id). - WithTitle(fmt.Sprintf("Article#%d", i)). - Object() - } - createObjects(t, compose.GetWeaviateNode2().URI(), batch) - }) - - t.Run("stop node 2", func(t *testing.T) { - stopNode(ctx, t, compose, compose.GetWeaviateNode2().Name()) - time.Sleep(10 * time.Second) - }) - - t.Run("get consistent search results with ONE (1/2 nodes up)", func(t *testing.T) { - resp := gqlGet(t, compose.GetWeaviate().URI(), paragraphClass.Class, replica.One) - checkResultsConsistency(t, resp, true) - }) - - t.Run("restart node 2", func(t *testing.T) { - err = compose.Start(ctx, compose.GetWeaviateNode2().Name()) - require.Nil(t, err) - }) - - t.Run("get consistent search results with ALL (2/2 nodes up)", func(t *testing.T) { - resp := gqlGet(t, compose.GetWeaviate().URI(), paragraphClass.Class, replica.All) - checkResultsConsistency(t, resp, true) - }) - - t.Run("get consistent search results with QUORUM (2/2 nodes up)", func(t *testing.T) { - resp := gqlGet(t, compose.GetWeaviate().URI(), paragraphClass.Class, replica.Quorum) - checkResultsConsistency(t, resp, true) - }) - - t.Run("get consistent search results with ONE (2/2 nodes up)", func(t *testing.T) { - resp := gqlGet(t, compose.GetWeaviate().URI(), paragraphClass.Class, replica.One) - checkResultsConsistency(t, resp, true) - }) - - t.Run("get consistent search results with ONE (2/2 nodes up)", func(t *testing.T) { - resp := gqlGet(t, compose.GetWeaviate().URI(), paragraphClass.Class, replica.One) - require.GreaterOrEqual(t, len(resp), 1) - vec := resp[0].(map[string]interface{})["_additional"].(map[string]interface{})["vector"].([]interface{}) - resp = gqlGetNearVec(t, compose.GetWeaviate().URI(), paragraphClass.Class, vec, replica.Quorum) - checkResultsConsistency(t, resp, true) - }) -} - -func checkResultsConsistency(t *testing.T, results []interface{}, expectConsistent bool) { - for _, res := range results { - addl := res.(map[string]interface{})["_additional"].(map[string]interface{}) - if expectConsistent { - assert.True(t, addl["isConsistent"].(bool)) - } else { - assert.False(t, addl["isConsistent"].(bool)) - } - } -} diff --git a/test/acceptance/replication/multi_tenancy_test.go b/test/acceptance/replication/multi_tenancy_test.go deleted file mode 100644 index cefe9ec0e89c836160c2cc37b72e31d2639f6a84..0000000000000000000000000000000000000000 --- a/test/acceptance/replication/multi_tenancy_test.go +++ /dev/null @@ -1,268 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package replication - -import ( - "context" - "encoding/json" - "fmt" - "testing" - "time" - - "github.com/go-openapi/strfmt" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/client/objects" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema/crossref" - "github.com/weaviate/weaviate/test/docker" - "github.com/weaviate/weaviate/test/helper" - "github.com/weaviate/weaviate/test/helper/sample-schema/articles" - "github.com/weaviate/weaviate/usecases/replica" -) - -const ( - tenantID = strfmt.UUID("45e9e17e-8102-4011-95f0-3079ca188bbf") -) - -func multiTenancyEnabled(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) - defer cancel() - - compose, err := docker.New(). - WithWeaviateCluster(). - WithText2VecContextionary(). - Start(ctx) - require.Nil(t, err) - defer func() { - if err := compose.Terminate(ctx); err != nil { - t.Fatalf("failed to terminate test containers: %s", err.Error()) - } - }() - - helper.SetupClient(compose.GetWeaviate().URI()) - paragraphClass := articles.ParagraphsClass() - articleClass := articles.ArticlesClass() - - t.Run("create schema", func(t *testing.T) { - paragraphClass.ReplicationConfig = &models.ReplicationConfig{ - Factor: 2, - } - paragraphClass.MultiTenancyConfig = &models.MultiTenancyConfig{ - Enabled: true, - } - helper.CreateClass(t, paragraphClass) - articleClass.ReplicationConfig = &models.ReplicationConfig{ - Factor: 2, - } - articleClass.MultiTenancyConfig = &models.MultiTenancyConfig{ - Enabled: true, - } - helper.CreateClass(t, articleClass) - }) - - t.Run("add tenants", func(t *testing.T) { - tenants := []*models.Tenant{{Name: tenantID.String()}} - helper.CreateTenants(t, paragraphClass.Class, tenants) - helper.CreateTenants(t, articleClass.Class, tenants) - }) - - t.Run("insert paragraphs batch", func(t *testing.T) { - t.Run("create objects on node 1", func(t *testing.T) { - batch := make([]*models.Object, len(paragraphIDs)) - for i, id := range paragraphIDs { - batch[i] = articles.NewParagraph(). - WithID(id). - WithContents(fmt.Sprintf("paragraph#%d", i)). - WithTenant(tenantID.String()). - Object() - } - createTenantObjects(t, compose.GetWeaviate().URI(), batch) - }) - - t.Run("stop node 1", func(t *testing.T) { - stopNode(ctx, t, compose, compose.GetWeaviate().Name()) - }) - - t.Run("assert objects exist on node 2", func(t *testing.T) { - count := countTenantObjects(t, compose.GetWeaviateNode2().URI(), - "Paragraph", tenantID.String()) - assert.Equal(t, int64(len(paragraphIDs)), count) - }) - - t.Run("restart node 1", func(t *testing.T) { - restartNode1(ctx, t, compose) - }) - }) - - t.Run("insert articles individually", func(t *testing.T) { - t.Run("create objects on node 2", func(t *testing.T) { - for i, id := range articleIDs { - obj := articles.NewArticle(). - WithID(id). - WithTitle(fmt.Sprintf("Article#%d", i)). - WithTenant(tenantID.String()). - Object() - createTenantObject(t, compose.GetWeaviateNode2().URI(), obj) - } - }) - - t.Run("stop node 2", func(t *testing.T) { - stopNode(ctx, t, compose, compose.GetWeaviateNode2().Name()) - }) - - t.Run("assert objects exist on node 1", func(t *testing.T) { - count := countTenantObjects(t, compose.GetWeaviate().URI(), - "Article", tenantID.String()) - assert.Equal(t, int64(len(articleIDs)), count) - }) - - t.Run("restart node 2", func(t *testing.T) { - err = compose.Start(ctx, compose.GetWeaviateNode2().Name()) - require.Nil(t, err) - }) - }) - - t.Run("add references", func(t *testing.T) { - refs := make([]*models.BatchReference, len(articleIDs)) - for i := range articleIDs { - refs[i] = &models.BatchReference{ - From: strfmt.URI(crossref.NewSource("Article", "hasParagraphs", articleIDs[i]).String()), - To: strfmt.URI(crossref.NewLocalhost("Paragraph", paragraphIDs[i]).String()), - Tenant: tenantID.String(), - } - } - - t.Run("add references to node 1", func(t *testing.T) { - addTenantReferences(t, compose.GetWeaviate().URI(), refs) - }) - - t.Run("stop node 1", func(t *testing.T) { - stopNode(ctx, t, compose, compose.GetWeaviate().Name()) - }) - - t.Run("assert references were added successfully to node 2", func(t *testing.T) { - type additional struct { - ID strfmt.UUID `json:"id"` - } - - type article struct { - Additional additional `json:"_additional"` - HasParagraphs []struct { - Additional additional `json:"_additional"` - } `json:"hasParagraphs"` - } - - // maps article id to referenced paragraph id - refPairs := make(map[strfmt.UUID]strfmt.UUID) - resp := gqlTenantGet(t, compose.GetWeaviateNode2().URI(), "Article", replica.One, - tenantID.String(), "_additional{id}", "hasParagraphs {... on Paragraph {_additional{id}}}") - assert.Len(t, resp, len(articleIDs)) - - for _, r := range resp { - b, err := json.Marshal(r) - require.Nil(t, err) - var art article - err = json.Unmarshal(b, &art) - require.Nil(t, err) - require.Len(t, art.HasParagraphs, 1) - refPairs[art.Additional.ID] = art.HasParagraphs[0].Additional.ID - } - - for i := range articleIDs { - paragraphID, ok := refPairs[articleIDs[i]] - require.True(t, ok, "expected %q to be in refPairs: %+v", articleIDs[i], refPairs) - assert.Equal(t, paragraphIDs[i], paragraphID) - } - }) - - t.Run("restart node 1", func(t *testing.T) { - restartNode1(ctx, t, compose) - }) - }) - - t.Run("patch an object", func(t *testing.T) { - before, err := getTenantObject(t, compose.GetWeaviate().URI(), "Article", articleIDs[0], tenantID.String()) - require.Nil(t, err) - newTitle := "Article#9000" - - t.Run("execute object patch on node 2", func(t *testing.T) { - patch := &models.Object{ - ID: before.ID, - Class: "Article", - Properties: map[string]interface{}{"title": newTitle}, - Tenant: tenantID.String(), - } - patchTenantObject(t, compose.GetWeaviateNode2().URI(), patch) - }) - - t.Run("stop node 2", func(t *testing.T) { - stopNode(ctx, t, compose, compose.GetWeaviateNode2().Name()) - }) - - t.Run("assert object is patched on node 1", func(t *testing.T) { - after, err := getTenantObjectFromNode(t, compose.GetWeaviate().URI(), - "Article", articleIDs[0], "node1", tenantID.String()) - require.Nil(t, err) - - newVal, ok := after.Properties.(map[string]interface{})["title"] - require.True(t, ok) - assert.Equal(t, newTitle, newVal) - }) - - t.Run("restart node 2", func(t *testing.T) { - err = compose.Start(ctx, compose.GetWeaviateNode2().Name()) - require.Nil(t, err) - }) - }) - - t.Run("delete an object", func(t *testing.T) { - t.Run("execute delete object on node 1", func(t *testing.T) { - deleteTenantObject(t, compose.GetWeaviate().URI(), "Article", articleIDs[0], tenantID.String()) - }) - - t.Run("stop node 1", func(t *testing.T) { - stopNode(ctx, t, compose, compose.GetWeaviate().Name()) - }) - - t.Run("assert object removed from node 2", func(t *testing.T) { - _, err := getTenantObjectFromNode(t, compose.GetWeaviateNode2().URI(), - "Article", articleIDs[0], "node2", tenantID.String()) - assert.Equal(t, &objects.ObjectsClassGetNotFound{}, err) - }) - - t.Run("restart node 1", func(t *testing.T) { - restartNode1(ctx, t, compose) - }) - }) - - t.Run("batch delete all objects", func(t *testing.T) { - t.Run("execute batch delete on node 2", func(t *testing.T) { - deleteTenantObjects(t, compose.GetWeaviateNode2().URI(), - "Article", []string{"title"}, "Article#*", tenantID.String()) - }) - - t.Run("stop node 2", func(t *testing.T) { - stopNode(ctx, t, compose, compose.GetWeaviateNode2().Name()) - }) - - t.Run("assert objects are removed from node 1", func(t *testing.T) { - count := countTenantObjects(t, compose.GetWeaviate().URI(), "Article", tenantID.String()) - assert.Zero(t, count) - }) - - t.Run("restart node 2", func(t *testing.T) { - err = compose.Start(ctx, compose.GetWeaviateNode2().Name()) - require.Nil(t, err) - }) - }) -} diff --git a/test/acceptance/replication/read_repair_test.go b/test/acceptance/replication/read_repair_test.go deleted file mode 100644 index 39c18af3752e60abf8834352c15ddc042b7be147..0000000000000000000000000000000000000000 --- a/test/acceptance/replication/read_repair_test.go +++ /dev/null @@ -1,159 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package replication - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/test/docker" - "github.com/weaviate/weaviate/test/helper" - "github.com/weaviate/weaviate/test/helper/sample-schema/articles" - "github.com/weaviate/weaviate/usecases/replica" -) - -func readRepair(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) - defer cancel() - - compose, err := docker.New(). - WithWeaviateCluster(). - WithText2VecContextionary(). - Start(ctx) - require.Nil(t, err) - defer func() { - if err := compose.Terminate(ctx); err != nil { - t.Fatalf("failed to terminate test containers: %s", err.Error()) - } - }() - - helper.SetupClient(compose.GetWeaviate().URI()) - paragraphClass := articles.ParagraphsClass() - articleClass := articles.ArticlesClass() - - t.Run("create schema", func(t *testing.T) { - paragraphClass.ReplicationConfig = &models.ReplicationConfig{ - Factor: 2, - } - paragraphClass.Vectorizer = "text2vec-contextionary" - helper.CreateClass(t, paragraphClass) - articleClass.ReplicationConfig = &models.ReplicationConfig{ - Factor: 2, - } - helper.CreateClass(t, articleClass) - }) - - t.Run("insert paragraphs", func(t *testing.T) { - batch := make([]*models.Object, len(paragraphIDs)) - for i, id := range paragraphIDs { - batch[i] = articles.NewParagraph(). - WithID(id). - WithContents(fmt.Sprintf("paragraph#%d", i)). - Object() - } - createObjects(t, compose.GetWeaviate().URI(), batch) - }) - - t.Run("insert articles", func(t *testing.T) { - batch := make([]*models.Object, len(articleIDs)) - for i, id := range articleIDs { - batch[i] = articles.NewArticle(). - WithID(id). - WithTitle(fmt.Sprintf("Article#%d", i)). - Object() - } - createObjects(t, compose.GetWeaviateNode2().URI(), batch) - }) - - t.Run("stop node 2", func(t *testing.T) { - stopNode(ctx, t, compose, compose.GetWeaviateNode2().Name()) - time.Sleep(10 * time.Second) - }) - - repairObj := models.Object{ - ID: "e5390693-5a22-44b8-997d-2a213aaf5884", - Class: "Paragraph", - Properties: map[string]interface{}{ - "contents": "a new paragraph", - }, - } - - t.Run("add new object to node one", func(t *testing.T) { - createObjectCL(t, compose.GetWeaviate().URI(), &repairObj, replica.One) - }) - - t.Run("restart node 2", func(t *testing.T) { - err = compose.Start(ctx, compose.GetWeaviateNode2().Name()) - require.Nil(t, err) - }) - - t.Run("run fetch to trigger read repair", func(t *testing.T) { - _, err := getObject(t, compose.GetWeaviate().URI(), repairObj.Class, repairObj.ID, true) - require.Nil(t, err) - }) - - t.Run("assert new object read repair was made", func(t *testing.T) { - stopNode(ctx, t, compose, compose.GetWeaviate().Name()) - time.Sleep(10 * time.Second) - - resp, err := getObjectCL(t, compose.GetWeaviateNode2().URI(), - repairObj.Class, repairObj.ID, replica.One) - require.Nil(t, err) - assert.Equal(t, repairObj.ID, resp.ID) - assert.Equal(t, repairObj.Class, resp.Class) - assert.EqualValues(t, repairObj.Properties, resp.Properties) - assert.EqualValues(t, repairObj.Vector, resp.Vector) - }) - - replaceObj := repairObj - replaceObj.Properties = map[string]interface{}{ - "contents": "this paragraph was replaced", - } - - t.Run("replace object", func(t *testing.T) { - updateObjectCL(t, compose.GetWeaviateNode2().URI(), &replaceObj, replica.One) - }) - - t.Run("restart node 1", func(t *testing.T) { - restartNode1(ctx, t, compose) - }) - - t.Run("run exists to trigger read repair", func(t *testing.T) { - exists, err := objectExistsCL(t, compose.GetWeaviateNode2().URI(), - replaceObj.Class, replaceObj.ID, replica.All) - require.Nil(t, err) - require.True(t, exists) - }) - - t.Run("assert updated object read repair was made", func(t *testing.T) { - stopNode(ctx, t, compose, compose.GetWeaviateNode2().Name()) - time.Sleep(10 * time.Second) - - exists, err := objectExistsCL(t, compose.GetWeaviate().URI(), - replaceObj.Class, replaceObj.ID, replica.One) - require.Nil(t, err) - require.True(t, exists) - - resp, err := getObjectCL(t, compose.GetWeaviate().URI(), - repairObj.Class, repairObj.ID, replica.One) - require.Nil(t, err) - assert.Equal(t, replaceObj.ID, resp.ID) - assert.Equal(t, replaceObj.Class, resp.Class) - assert.EqualValues(t, replaceObj.Properties, resp.Properties) - assert.EqualValues(t, replaceObj.Vector, resp.Vector) - }) -} diff --git a/test/acceptance/replication/scale_test.go b/test/acceptance/replication/scale_test.go deleted file mode 100644 index 47002d025a2ba7060134c05f450d7213af096c1f..0000000000000000000000000000000000000000 --- a/test/acceptance/replication/scale_test.go +++ /dev/null @@ -1,141 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package replication - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/go-openapi/strfmt" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema/crossref" - "github.com/weaviate/weaviate/test/docker" - "github.com/weaviate/weaviate/test/helper" - "github.com/weaviate/weaviate/test/helper/sample-schema/articles" - "github.com/weaviate/weaviate/usecases/replica" -) - -func multiShardScaleOut(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) - defer cancel() - - compose, err := docker.New(). - WithWeaviateCluster(). - WithText2VecContextionary(). - Start(ctx) - require.Nil(t, err) - defer func() { - if err := compose.Terminate(ctx); err != nil { - t.Fatalf("failed to terminate test containers: %s", err.Error()) - } - }() - - helper.SetupClient(compose.GetWeaviate().URI()) - paragraphClass := articles.ParagraphsClass() - paragraphClass.ShardingConfig = map[string]interface{}{ - "desiredCount": 1, - } - articleClass := articles.ArticlesClass() - articleClass.ShardingConfig = map[string]interface{}{ - "desiredCount": 1, - } - - t.Run("create schema", func(t *testing.T) { - helper.CreateClass(t, paragraphClass) - helper.CreateClass(t, articleClass) - }) - - t.Run("insert paragraphs", func(t *testing.T) { - batch := make([]*models.Object, len(paragraphIDs)) - for i, id := range paragraphIDs { - batch[i] = articles.NewParagraph(). - WithID(id). - WithContents(fmt.Sprintf("paragraph#%d", i)). - Object() - } - createObjects(t, compose.GetWeaviate().URI(), batch) - }) - - t.Run("insert articles", func(t *testing.T) { - batch := make([]*models.Object, len(articleIDs)) - for i, id := range articleIDs { - batch[i] = articles.NewArticle(). - WithID(id). - WithTitle(fmt.Sprintf("Article#%d", i)). - Object() - } - createObjects(t, compose.GetWeaviateNode2().URI(), batch) - }) - - t.Run("add references", func(t *testing.T) { - refs := make([]*models.BatchReference, len(articleIDs)) - for i := range articleIDs { - refs[i] = &models.BatchReference{ - From: strfmt.URI(crossref.NewSource("Article", "hasParagraphs", articleIDs[i]).String()), - To: strfmt.URI(crossref.NewLocalhost("Paragraph", paragraphIDs[i]).String()), - } - } - addReferences(t, compose.GetWeaviate().URI(), refs) - }) - - t.Run("scale out paragraphs", func(t *testing.T) { - c := getClass(t, compose.GetWeaviate().URI(), paragraphClass.Class) - c.ReplicationConfig.Factor = 2 - updateClass(t, compose.GetWeaviate().URI(), c) - }) - - t.Run("assert paragraphs were scaled out", func(t *testing.T) { - n := getNodes(t, compose.GetWeaviate().URI()) - var shardsFound int - for _, node := range n.Nodes { - for _, shard := range node.Shards { - if shard.Class == paragraphClass.Class { - assert.EqualValues(t, 10, shard.ObjectCount) - shardsFound++ - } - } - } - assert.Equal(t, 2, shardsFound) - }) - - t.Run("scale out articles", func(t *testing.T) { - c := getClass(t, compose.GetWeaviate().URI(), articleClass.Class) - c.ReplicationConfig.Factor = 2 - updateClass(t, compose.GetWeaviate().URI(), c) - }) - - t.Run("assert articles were scaled out", func(t *testing.T) { - n := getNodes(t, compose.GetWeaviate().URI()) - var shardsFound int - for _, node := range n.Nodes { - for _, shard := range node.Shards { - if shard.Class == articleClass.Class { - assert.EqualValues(t, 10, shard.ObjectCount) - shardsFound++ - } - } - } - assert.Equal(t, 2, shardsFound) - }) - - t.Run("kill a node and check contents of remaining node", func(t *testing.T) { - stopNode(ctx, t, compose, compose.GetWeaviateNode2().Name()) - p := gqlGet(t, compose.GetWeaviate().URI(), paragraphClass.Class, replica.One) - assert.Len(t, p, 10) - a := gqlGet(t, compose.GetWeaviate().URI(), articleClass.Class, replica.One) - assert.Len(t, a, 10) - }) -} diff --git a/test/acceptance/replication/setup_test.go b/test/acceptance/replication/setup_test.go deleted file mode 100644 index 58dbcf77fa5789977ff872a89dd254052a2b0453..0000000000000000000000000000000000000000 --- a/test/acceptance/replication/setup_test.go +++ /dev/null @@ -1,23 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package replication - -import "testing" - -func TestReplication(t *testing.T) { - t.Run("immediate replica CRUD", immediateReplicaCRUD) - t.Run("eventual replica CRUD", eventualReplicaCRUD) - t.Run("multishard scale out", multiShardScaleOut) - t.Run("read repair", readRepair) - t.Run("graphql search", graphqlSearch) - t.Run("multi-tenancy enabled", multiTenancyEnabled) -} diff --git a/test/acceptance/run.sh b/test/acceptance/run.sh deleted file mode 100644 index 5b01bcf345355ca0e4b846eb07a22b35cc338ab6..0000000000000000000000000000000000000000 --- a/test/acceptance/run.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -set -eou pipefail - -function main() { - # needed for test/docker package during replication tests - export TEST_WEAVIATE_IMAGE=weaviate/test-server - # for now we need to run the tests sequentially, there seems to be some sort of issues with running them in parallel - for pkg in $(go list ./... | grep 'test/acceptance' | grep -v 'test/acceptance/stress_tests' ); do - if ! go test -count 1 -race "$pkg"; then - echo "Test for $pkg failed" >&2 - return 1 - fi - done - for pkg in $(go list ./... | grep 'test/acceptance/stress_tests' ); do - if ! go test -count 1 "$pkg"; then - echo "Test for $pkg failed" >&2 - return 1 - fi - done - # tests with go client are in a separate package with its own dependencies to isolate them - cd 'test/acceptance_with_go_client' - for pkg in $(go list ./... ); do - if ! go test -count 1 -race "$pkg"; then - echo "Test for $pkg failed" >&2 - return 1 - fi - done -} - -main "$@" \ No newline at end of file diff --git a/test/acceptance/schema/add_class_test.go b/test/acceptance/schema/add_class_test.go deleted file mode 100644 index 9e50fb2d1744d88afec91c4470481946fe6a4c6a..0000000000000000000000000000000000000000 --- a/test/acceptance/schema/add_class_test.go +++ /dev/null @@ -1,465 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - clschema "github.com/weaviate/weaviate/client/schema" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/test/helper" -) - -// this test prevents a regression on -// https://github.com/weaviate/weaviate/issues/981 -func TestInvalidDataTypeInProperty(t *testing.T) { - t.Parallel() - className := "WrongPropertyClass" - - t.Run("asserting that this class does not exist yet", func(t *testing.T) { - assert.NotContains(t, GetObjectClassNames(t), className) - }) - - t.Run("trying to import empty string as data type", func(t *testing.T) { - c := &models.Class{ - Class: className, - Properties: []*models.Property{ - { - Name: "someProperty", - DataType: []string{""}, - }, - }, - } - - params := clschema.NewSchemaObjectsCreateParams().WithObjectClass(c) - resp, err := helper.Client(t).Schema.SchemaObjectsCreate(params, nil) - helper.AssertRequestFail(t, resp, err, func() { - parsed, ok := err.(*clschema.SchemaObjectsCreateUnprocessableEntity) - require.True(t, ok, "error should be unprocessable entity") - assert.Equal(t, "property 'someProperty': invalid dataType: dataType cannot be an empty string", - parsed.Payload.Error[0].Message) - }) - }) -} - -func TestInvalidPropertyName(t *testing.T) { - t.Parallel() - className := "WrongPropertyClass" - - t.Run("asserting that this class does not exist yet", func(t *testing.T) { - assert.NotContains(t, GetObjectClassNames(t), className) - }) - - t.Run("trying to create class with invalid property name", func(t *testing.T) { - c := &models.Class{ - Class: className, - Properties: []*models.Property{ - { - Name: "some-property", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - } - - params := clschema.NewSchemaObjectsCreateParams().WithObjectClass(c) - resp, err := helper.Client(t).Schema.SchemaObjectsCreate(params, nil) - helper.AssertRequestFail(t, resp, err, func() { - parsed, ok := err.(*clschema.SchemaObjectsCreateUnprocessableEntity) - require.True(t, ok, "error should be unprocessable entity") - assert.Equal(t, "'some-property' is not a valid property name. Property names in Weaviate "+ - "are restricted to valid GraphQL names, which must be “/[_A-Za-z][_0-9A-Za-z]*/”.", - parsed.Payload.Error[0].Message) - }) - }) -} - -func TestAddAndRemoveObjectClass(t *testing.T) { - randomObjectClassName := "YellowCars" - - // Ensure that this name is not in the schema yet. - t.Log("Asserting that this class does not exist yet") - assert.NotContains(t, GetObjectClassNames(t), randomObjectClassName) - - tc := &models.Class{ - Class: randomObjectClassName, - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "vectorizeClassName": true, - }, - }, - } - - t.Log("Creating class") - params := clschema.NewSchemaObjectsCreateParams().WithObjectClass(tc) - resp, err := helper.Client(t).Schema.SchemaObjectsCreate(params, nil) - helper.AssertRequestOk(t, resp, err, nil) - - t.Log("Asserting that this class is now created") - assert.Contains(t, GetObjectClassNames(t), randomObjectClassName) - - t.Run("pure http - without the auto-generated client", testGetSchemaWithoutClient) - - // Now clean up this class. - t.Log("Remove the class") - delParams := clschema.NewSchemaObjectsDeleteParams().WithClassName(randomObjectClassName) - delResp, err := helper.Client(t).Schema.SchemaObjectsDelete(delParams, nil) - helper.AssertRequestOk(t, delResp, err, nil) - - // And verify that the class does not exist anymore. - assert.NotContains(t, GetObjectClassNames(t), randomObjectClassName) - - t.Log("Verify schema cluster status") - statusResp, err := helper.Client(t).Schema.SchemaClusterStatus( - clschema.NewSchemaClusterStatusParams(), nil, - ) - require.Nil(t, err) - assert.Equal(t, "", statusResp.Payload.Error) - assert.True(t, statusResp.Payload.Healthy) -} - -// This test prevents a regression on -// https://github.com/weaviate/weaviate/issues/1799 -// -// This was related to adding ref props. For example in the case of a circular -// dependency (A<>B), users would typically add A without refs, then add B with -// a reference back to A, finally update A with a ref to B. -// -// This last update that would set the ref prop on an existing class was missing -// module-specific defaults. So when comparing to-be-updated to existing we would -// find differences in the properties, thus triggering the above error. -func TestUpdateHNSWSettingsAfterAddingRefProps(t *testing.T) { - className := "RefUpdateIssueClass" - - t.Run("asserting that this class does not exist yet", func(t *testing.T) { - assert.NotContains(t, GetObjectClassNames(t), className) - }) - - defer func(t *testing.T) { - params := clschema.NewSchemaObjectsDeleteParams().WithClassName(className) - _, err := helper.Client(t).Schema.SchemaObjectsDelete(params, nil) - assert.Nil(t, err) - if err != nil { - if typed, ok := err.(*clschema.SchemaObjectsDeleteBadRequest); ok { - fmt.Println(typed.Payload.Error[0].Message) - } - } - }(t) - - t.Run("initially creating the class", func(t *testing.T) { - c := &models.Class{ - Class: className, - Properties: []*models.Property{ - { - Name: "string_prop", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - } - - params := clschema.NewSchemaObjectsCreateParams().WithObjectClass(c) - _, err := helper.Client(t).Schema.SchemaObjectsCreate(params, nil) - assert.Nil(t, err) - }) - - t.Run("adding a ref prop after the fact", func(t *testing.T) { - params := clschema.NewSchemaObjectsPropertiesAddParams(). - WithClassName(className). - WithBody(&models.Property{ - DataType: []string{className}, - Name: "ref_prop", - }) - _, err := helper.Client(t).Schema.SchemaObjectsPropertiesAdd(params, nil) - assert.Nil(t, err) - }) - - t.Run("obtaining the class, making an innocent change and trying to update it", func(t *testing.T) { - params := clschema.NewSchemaObjectsGetParams(). - WithClassName(className) - res, err := helper.Client(t).Schema.SchemaObjectsGet(params, nil) - require.Nil(t, err) - - class := res.Payload - - class.VectorIndexConfig.(map[string]interface{})["ef"] = float64(1234) - - updateParams := clschema.NewSchemaObjectsUpdateParams(). - WithClassName(className). - WithObjectClass(class) - _, err = helper.Client(t).Schema.SchemaObjectsUpdate(updateParams, nil) - assert.Nil(t, err) - }) - - t.Run("obtaining the class, making a change to IndexNullState (immutable) property and update", func(t *testing.T) { - params := clschema.NewSchemaObjectsGetParams(). - WithClassName(className) - res, err := helper.Client(t).Schema.SchemaObjectsGet(params, nil) - require.Nil(t, err) - - class := res.Payload - - // IndexNullState cannot be updated during runtime - class.InvertedIndexConfig.IndexNullState = true - updateParams := clschema.NewSchemaObjectsUpdateParams(). - WithClassName(className). - WithObjectClass(class) - _, err = helper.Client(t).Schema.SchemaObjectsUpdate(updateParams, nil) - assert.NotNil(t, err) - }) - - t.Run("obtaining the class, making a change to IndexPropertyLength (immutable) property and update", func(t *testing.T) { - params := clschema.NewSchemaObjectsGetParams(). - WithClassName(className) - res, err := helper.Client(t).Schema.SchemaObjectsGet(params, nil) - require.Nil(t, err) - - class := res.Payload - - // IndexPropertyLength cannot be updated during runtime - class.InvertedIndexConfig.IndexPropertyLength = true - updateParams := clschema.NewSchemaObjectsUpdateParams(). - WithClassName(className). - WithObjectClass(class) - _, err = helper.Client(t).Schema.SchemaObjectsUpdate(updateParams, nil) - assert.NotNil(t, err) - }) -} - -// This test prevents a regression of -// https://github.com/weaviate/weaviate/issues/2692 -// -// In this issue, any time a class had no vector index set, any other update to -// the class would be blocked -func TestUpdateClassWithoutVectorIndex(t *testing.T) { - className := "IAintGotNoVectorIndex" - - t.Run("asserting that this class does not exist yet", func(t *testing.T) { - assert.NotContains(t, GetObjectClassNames(t), className) - }) - - defer func(t *testing.T) { - params := clschema.NewSchemaObjectsDeleteParams().WithClassName(className) - _, err := helper.Client(t).Schema.SchemaObjectsDelete(params, nil) - assert.Nil(t, err) - if err != nil { - if typed, ok := err.(*clschema.SchemaObjectsDeleteBadRequest); ok { - fmt.Println(typed.Payload.Error[0].Message) - } - } - }(t) - - t.Run("initially creating the class", func(t *testing.T) { - c := &models.Class{ - Class: className, - InvertedIndexConfig: &models.InvertedIndexConfig{ - Stopwords: &models.StopwordConfig{ - Preset: "en", - }, - }, - Properties: []*models.Property{ - { - Name: "text_prop", - DataType: []string{"text"}, - }, - }, - VectorIndexConfig: map[string]interface{}{ - "skip": true, - }, - } - - params := clschema.NewSchemaObjectsCreateParams().WithObjectClass(c) - _, err := helper.Client(t).Schema.SchemaObjectsCreate(params, nil) - assert.Nil(t, err) - }) - - t.Run("obtaining the class, making an innocent change and trying to update it", func(t *testing.T) { - params := clschema.NewSchemaObjectsGetParams(). - WithClassName(className) - res, err := helper.Client(t).Schema.SchemaObjectsGet(params, nil) - require.Nil(t, err) - - class := res.Payload - - class.InvertedIndexConfig.Stopwords.Preset = "none" - - updateParams := clschema.NewSchemaObjectsUpdateParams(). - WithClassName(className). - WithObjectClass(class) - _, err = helper.Client(t).Schema.SchemaObjectsUpdate(updateParams, nil) - assert.Nil(t, err) - }) -} - -// This test prevents a regression of -// https://github.com/weaviate/weaviate/issues//3177 -// -// This test ensures that distance belongs to the immutable properties, i.e. no changes to it are possible after creating the class. -func TestUpdateDistanceSettings(t *testing.T) { - className := "Cosine_Class" - - t.Run("asserting that this class does not exist yet", func(t *testing.T) { - assert.NotContains(t, GetObjectClassNames(t), className) - }) - - defer func(t *testing.T) { - params := clschema.NewSchemaObjectsDeleteParams().WithClassName(className) - _, err := helper.Client(t).Schema.SchemaObjectsDelete(params, nil) - assert.Nil(t, err) - if err != nil { - if typed, ok := err.(*clschema.SchemaObjectsDeleteBadRequest); ok { - fmt.Println(typed.Payload.Error[0].Message) - } - } - }(t) - - t.Run("initially creating the class", func(t *testing.T) { - c := &models.Class{ - Class: className, - Vectorizer: "none", - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - VectorIndexConfig: map[string]interface{}{ - "distance": "cosine", - }, - } - - params := clschema.NewSchemaObjectsCreateParams().WithObjectClass(c) - _, err := helper.Client(t).Schema.SchemaObjectsCreate(params, nil) - assert.Nil(t, err) - }) - - t.Run("Trying to change the distance measurement", func(t *testing.T) { - params := clschema.NewSchemaObjectsGetParams(). - WithClassName(className) - res, err := helper.Client(t).Schema.SchemaObjectsGet(params, nil) - require.Nil(t, err) - - class := res.Payload - - class.VectorIndexConfig.(map[string]interface{})["distance"] = "l2-squared" - - updateParams := clschema.NewSchemaObjectsUpdateParams(). - WithClassName(className). - WithObjectClass(class) - _, err = helper.Client(t).Schema.SchemaObjectsUpdate(updateParams, nil) - assert.NotNil(t, err) - }) -} - -// TODO: https://github.com/weaviate/weaviate/issues/973 -// // This test prevents a regression on the fix for this bug: -// // https://github.com/weaviate/weaviate/issues/831 -// func TestDeleteSingleProperties(t *testing.T) { -// t.Parallel() - -// randomObjectClassName := "RedShip" - -// // Ensure that this name is not in the schema yet. -// t.Log("Asserting that this class does not exist yet") -// assert.NotContains(t, GetThingClassNames(t), randomThingClassName) - -// tc := &models.Class{ -// Class: randomThingClassName, -// Properties: []*models.Property{ -// &models.Property{ -// DataType: schema.DataTypeText.PropString(), -// Tokenization: models.PropertyTokenizationWhitespace, -// Name: "name", -// }, -// &models.Property{ -// DataType: schema.DataTypeText.PropString(), -// Tokenization: models.PropertyTokenizationWhitespace, -// Name: "description", -// }, -// }, -// } - -// t.Log("Creating class") -// params := clschema.NewSchemaThingsCreateParams().WithThingClass(tc) -// resp, err := helper.Client(t).Schema.SchemaThingsCreate(params, nil) -// helper.AssertRequestOk(t, resp, err, nil) - -// t.Log("Asserting that this class is now created") -// assert.Contains(t, GetThingClassNames(t), randomThingClassName) - -// t.Log("adding an instance of this particular class that uses both properties") -// instanceParams := things.NewThingsCreateParams().WithBody( -// &models.Thing{ -// Class: randomThingClassName, -// Schema: map[string]interface{}{ -// "name": "my name", -// "description": "my description", -// }, -// }) -// instanceRes, err := helper.Client(t).Things.ThingsCreate(instanceParams, nil) -// assert.Nil(t, err, "adding a class instance should not error") - -// t.Log("delete a single property of the class") -// deleteParams := clschema.NewSchemaThingsPropertiesDeleteParams(). -// WithClassName(randomThingClassName). -// WithPropertyName("description") -// _, err = helper.Client(t).Schema.SchemaThingsPropertiesDelete(deleteParams, nil) -// assert.Nil(t, err, "deleting the property should not error") - -// t.Log("retrieve the class and make sure the property is gone") -// thing := assertGetThingEventually(t, instanceRes.Payload.ID) -// expectedSchema := map[string]interface{}{ -// "name": "my name", -// } -// assert.Equal(t, expectedSchema, thing.Schema) - -// t.Log("verifying that we can still retrieve the thing through graphQL") -// result := gql.AssertGraphQL(t, helper.RootAuth, "{ Get { Things { RedShip { name } } } }") -// ships := result.Get("Get", "Things", "RedShip").AsSlice() -// expectedShip := map[string]interface{}{ -// "name": "my name", -// } -// assert.Contains(t, ships, expectedShip) - -// t.Log("verifying other GQL/REST queries still work") -// gql.AssertGraphQL(t, helper.RootAuth, "{ Meta { Things { RedShip { name { count } } } } }") -// gql.AssertGraphQL(t, helper.RootAuth, `{ Aggregate { Things { RedShip(groupBy: ["name"]) { name { count } } } } }`) -// _, err = helper.Client(t).Things.ThingsList(things.NewThingsListParams(), nil) -// assert.Nil(t, err, "listing things should not error") - -// t.Log("verifying we could re-add the property with the same name") -// readdParams := clschema.NewSchemaThingsPropertiesAddParams(). -// WithClassName(randomThingClassName). -// WithBody(&models.Property{ -// Name: "description", -// DataType: schema.DataTypeText.PropString(), -// Tokenization: models.PropertyTokenizationWhitespace, -// }) - -// _, err = helper.Client(t).Schema.SchemaThingsPropertiesAdd(readdParams, nil) -// assert.Nil(t, err, "adding the previously deleted property again should not error") - -// // Now clean up this class. -// t.Log("Remove the class") -// delParams := clschema.NewSchemaThingsDeleteParams().WithClassName(randomThingClassName) -// delResp, err := helper.Client(t).Schema.SchemaThingsDelete(delParams, nil) -// helper.AssertRequestOk(t, delResp, err, nil) - -// // And verify that the class does not exist anymore. -// assert.NotContains(t, GetThingClassNames(t), randomThingClassName) -// } diff --git a/test/acceptance/schema/get_schema_without_client_test.go b/test/acceptance/schema/get_schema_without_client_test.go deleted file mode 100644 index 7536fa59544e9eaa10473df776b63b17012cccfd..0000000000000000000000000000000000000000 --- a/test/acceptance/schema/get_schema_without_client_test.go +++ /dev/null @@ -1,105 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "encoding/json" - "fmt" - "net/http" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/vectorindex/hnsw" - "github.com/weaviate/weaviate/test/helper" -) - -func testGetSchemaWithoutClient(t *testing.T) { - res, err := http.Get(fmt.Sprintf("%s%s", helper.GetWeaviateURL(), "/v1/schema")) - require.Nil(t, err) - - defer res.Body.Close() - var body map[string]interface{} - err = json.NewDecoder(res.Body).Decode(&body) - require.Nil(t, err) - - expected := map[string]interface{}{ - "classes": []interface{}{ - map[string]interface{}{ - "class": "YellowCars", - "properties": (interface{})(nil), - "vectorIndexType": "hnsw", // from default - "vectorIndexConfig": map[string]interface{}{ // from default - "skip": false, - "cleanupIntervalSeconds": float64(300), - "efConstruction": float64(128), - "flatSearchCutoff": float64(40000), - "ef": float64(-1), - "maxConnections": float64(64), - "vectorCacheMaxObjects": float64(1e12), - "dynamicEfMin": float64(100), - "dynamicEfMax": float64(500), - "dynamicEfFactor": float64(8), - "distance": "cosine", - "bq": map[string]interface{}{ - "enabled": false, - }, - "pq": map[string]interface{}{ - "bitCompression": false, - "centroids": float64(256), - "enabled": false, - "encoder": map[string]interface{}{ - "distribution": "log-normal", - "type": hnsw.PQEncoderTypeKMeans, - }, - "segments": float64(0), - "trainingLimit": float64(100000), - }, - }, - "shardingConfig": map[string]interface{}{ - "actualCount": float64(1), - "actualVirtualCount": float64(128), - "desiredCount": float64(1), - "desiredVirtualCount": float64(128), - "function": "murmur3", - "strategy": "hash", - "key": "_id", - "virtualPerPhysical": float64(128), - }, - "replicationConfig": map[string]interface{}{ - "factor": float64(1), - }, - "vectorizer": "text2vec-contextionary", // global default from env var, see docker-compose-test.yml - "invertedIndexConfig": map[string]interface{}{ - "cleanupIntervalSeconds": float64(60), - "bm25": map[string]interface{}{ - "k1": float64(1.2), - "b": float64(0.75), - }, - "stopwords": map[string]interface{}{ - "preset": "en", - "additions": nil, - "removals": nil, - }, - }, - "moduleConfig": map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "vectorizeClassName": true, - }, - }, - "multiTenancyConfig": map[string]interface{}{"enabled": false}, - }, - }, - } - - assert.Equal(t, expected, body) -} diff --git a/test/acceptance/schema/helper.go b/test/acceptance/schema/helper.go deleted file mode 100644 index 0063e55a499c6bfa477becb733a77cd7e4782bf1..0000000000000000000000000000000000000000 --- a/test/acceptance/schema/helper.go +++ /dev/null @@ -1,48 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "testing" - - "github.com/weaviate/weaviate/test/helper" -) - -// Helper function to get all the names of Object classes. -func GetObjectClassNames(t *testing.T) []string { - resp, err := helper.Client(t).Schema.SchemaDump(nil, nil) - var names []string - - // Extract all names - helper.AssertRequestOk(t, resp, err, func() { - for _, class := range resp.Payload.Classes { - names = append(names, class.Class) - } - }) - - return names -} - -// Helper function to get all the names of Action classes. -// func GetActionClassNames(t *testing.T) []string { -// resp, err := helper.Client(t).Schema.SchemaDump(nil, nil) -// var names []string - -// // Extract all names -// helper.AssertRequestOk(t, resp, err, func() { -// for _, class := range resp.Payload.Actions.Classes { -// names = append(names, class.Class) -// } -// }) - -// return names -// } diff --git a/test/acceptance/schema/transactions_test.go b/test/acceptance/schema/transactions_test.go deleted file mode 100644 index b8dc944abc969b32cfe082f38088b6dfbf30079f..0000000000000000000000000000000000000000 --- a/test/acceptance/schema/transactions_test.go +++ /dev/null @@ -1,141 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "bytes" - "fmt" - "io" - "net/http" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// This test makes sure that a malformed tx payload (possibly from a bad actor) -// can't crash - or possibly worse - deadlock Weaviate -// -// See https://github.com/weaviate/weaviate/issues/3401 for details -func TestCrashServerThroughInvalidTxPayloads(t *testing.T) { - tests := []struct { - name string - txId string - payload string - errMustContain string - }{ - { - name: "add_class: empty class payload", - txId: "1", - payload: `{"id":"1", "type": "add_class", "payload":{}}`, - errMustContain: "class is nil", - }, - { - name: "add_class: class set, but empty, no state", - txId: "2", - payload: `{"id":"2", "type": "add_class", "payload":{"class":{}}}`, - errMustContain: "state is nil", - }, - { - name: "add_class: class set and valid, no state", - txId: "3", - payload: `{"id":"3", "type": "add_class", "payload":{"class":{"vectorIndexType":"hnsw","class":"Foo"}}}`, - errMustContain: "state is nil", - }, - { - name: "add_class: class set, but empty, with state", - txId: "4", - payload: `{"id":"4", "type": "add_class", "payload":{"state":{},"class":{}}}`, - errMustContain: "unsupported vector index", - }, - { - name: "update_class: empty class payload", - txId: "1", - payload: `{"id":"1", "type": "update_class", "payload":{}}`, - errMustContain: "class is nil", - }, - { - name: "update_class: class set and valid, no state", - txId: "3", - payload: `{"id":"3", "type": "update_class", "payload":{"class":{"vectorIndexType":"hnsw","class":"FoobarBazzzar"}}}`, - }, - { - name: "update_class: class set, but empty, with state", - txId: "4", - payload: `{"id":"4", "type": "update_class", "payload":{"state":{},"class":{}}}`, - errMustContain: "unsupported vector index", - }, - { - name: "add_tenants: empty payload", - txId: "1", - payload: `{"id":"1", "type": "add_tenants", "payload":{}}`, - errMustContain: "not found", - }, - { - name: "delete_tenants: malformed payload", - txId: "1", - payload: `{"id":"1", "type": "delete_tenants", "payload":7}`, - errMustContain: "invalid", - }, - { - name: "delete_class: malformed payload", - txId: "2", - payload: `{"id":"2", "type": "delete_class", "payload":7}`, - errMustContain: "invalid", - }, - { - name: "add_property: missing prop", - txId: "1", - payload: `{"id":"1", "type": "add_property", "payload":{"class":"Foo"}}`, - errMustContain: "property is nil", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - client := http.Client{} - - // open tx - payload := []byte(test.payload) - req, err := http.NewRequest(http.MethodPost, "http://localhost:7101/schema/transactions/", bytes.NewReader(payload)) - require.NoError(t, err) - - req.Header.Add("Content-Type", "application/json") - - res, err := client.Do(req) - require.NoError(t, err) - defer res.Body.Close() - - // try to commit tx - req, err = http.NewRequest(http.MethodPut, - fmt.Sprintf("http://localhost:7101/schema/transactions/%s/commit", test.txId), nil) - require.NoError(t, err) - - res, err = client.Do(req) - require.NoError(t, err) - defer res.Body.Close() - - assert.Greater(t, res.StatusCode, 399) - resBytes, _ := io.ReadAll(res.Body) - assert.Contains(t, string(resBytes), test.errMustContain) - - // clean up tx (so next test doesn't have concurrent tx error) - req, err = http.NewRequest(http.MethodDelete, - fmt.Sprintf("http://localhost:7101/schema/transactions/%s", test.txId), nil) - require.NoError(t, err) - - res, err = client.Do(req) - require.NoError(t, err) - defer res.Body.Close() - }) - } -} diff --git a/test/acceptance/stress_tests/concurrent_batches_test.go b/test/acceptance/stress_tests/concurrent_batches_test.go deleted file mode 100644 index 9eb3732e94d450bd8a5c3ad2c401cd6926dc0d1b..0000000000000000000000000000000000000000 --- a/test/acceptance/stress_tests/concurrent_batches_test.go +++ /dev/null @@ -1,94 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package stress_tests - -import ( - "encoding/json" - "fmt" - "sync" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/entities/models" -) - -type batch struct { - Objects []*models.Object -} - -const class = "TestClass" - -func Test_AddConcurrentSchemas_sameObject(t *testing.T) { - url := "http://localhost:8080/v1/" - batch := batch{createObject(class)} - parallelReqs := 10 - wg := sync.WaitGroup{} - wg.Add(parallelReqs) - - // Add schema and object - c := createHttpClient() - clearExistingObjects(c, url) - requestSchema := createSchemaRequest(url, class, false) - performRequest(c, requestSchema) - - for i := 0; i < parallelReqs; i++ { - go func(j int) { - defer wg.Done() - c := createHttpClient() - performRequest(c, createRequest(url+"batch/objects", "POST", batch)) - }(i) - } - wg.Wait() - - requestRead := createRequest(url+"objects?limit="+fmt.Sprint(10)+"&class="+class, "GET", nil) - _, body, _ := performRequest(c, requestRead) - var result map[string]interface{} - json.Unmarshal(body, &result) - assert.Equal(t, 1, int(result["totalResults"].(float64))) -} - -func Test_AddConcurrentBatches_differentObjects(t *testing.T) { - url := "http://localhost:8080/v1/" - - parallelReqs := 150 - wg := sync.WaitGroup{} - wg.Add(parallelReqs) - - // Add schema and object - c := createHttpClient() - clearExistingObjects(c, url) - requestSchema := createSchemaRequest(url, class, false) - performRequest(c, requestSchema) - batch1 := batch{createObject(class)} - batch2 := batch{createObject(class)} - - for i := 0; i < parallelReqs; i++ { - go func(j int) { - defer wg.Done() - - c := createHttpClient() - if j%2 == 0 { - performRequest(c, createRequest(url+"batch/objects", "POST", batch1)) - } else { - performRequest(c, createRequest(url+"batch/objects", "POST", batch2)) - } - }(i) - } - wg.Wait() - - requestRead := createRequest(url+"objects?limit="+fmt.Sprint(10)+"&class="+class, "GET", nil) - _, body, _ := performRequest(c, requestRead) - var result map[string]interface{} - json.Unmarshal(body, &result) - assert.Equal(t, 2, int(result["totalResults"].(float64))) - clearExistingObjects(c, url) -} diff --git a/test/acceptance/stress_tests/stress.go b/test/acceptance/stress_tests/stress.go deleted file mode 100644 index 8addded5bda0d61de1d10cf08153d0d7e46586bc..0000000000000000000000000000000000000000 --- a/test/acceptance/stress_tests/stress.go +++ /dev/null @@ -1,179 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package stress_tests - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "net" - "net/http" - "strings" - "time" - - "github.com/go-openapi/strfmt" - "github.com/google/uuid" - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" -) - -// If there is already a schema present, clear it out -func clearExistingObjects(c *http.Client, url string) { - checkSchemaRequest := createRequest(url+"schema", "GET", nil) - checkSchemaResponseCode, body, err := performRequest(c, checkSchemaRequest) - if err != nil { - panic(errors.Wrap(err, "perform request")) - } - if checkSchemaResponseCode != 200 { - return - } - - var dump models.Schema - if err := json.Unmarshal(body, &dump); err != nil { - panic(errors.Wrap(err, "Could not unmarshal read response")) - } - for _, classObj := range dump.Classes { - requestDelete := createRequest(url+"schema/"+classObj.Class, "DELETE", nil) - responseDeleteCode, _, err := performRequest(c, requestDelete) - if err != nil { - panic(errors.Wrap(err, "Could delete schema")) - } - if responseDeleteCode != 200 { - panic(fmt.Sprintf("Could not delete schema, code: %v", responseDeleteCode)) - } - } -} - -func createHttpClient() *http.Client { - httpT := &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: (&net.Dialer{ - Timeout: 500 * time.Second, - KeepAlive: 120 * time.Second, - }).DialContext, - MaxIdleConnsPerHost: 100, - MaxIdleConns: 100, - IdleConnTimeout: 90 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - } - return &http.Client{Transport: httpT} -} - -// createRequest creates requests -func createRequest(url string, method string, payload interface{}) *http.Request { - var body io.Reader = nil - if payload != nil { - jsonBody, err := json.Marshal(payload) - if err != nil { - panic(errors.Wrap(err, "Could not marshal request")) - } - body = bytes.NewBuffer(jsonBody) - } - request, err := http.NewRequest(method, url, body) - if err != nil { - panic(errors.Wrap(err, "Could not create request")) - } - request.Header.Add("Content-Type", "application/json") - request.Header.Add("Accept", "application/json") - - return request -} - -// performRequest runs requests -func performRequest(c *http.Client, request *http.Request) (int, []byte, error) { - for { - response, err := c.Do(request) - if err != nil { - return 0, nil, err - } - - body, err := io.ReadAll(response.Body) - response.Body.Close() - if err != nil { - return 0, nil, err - } - - if response.StatusCode == 200 { - return response.StatusCode, body, nil - } - time.Sleep(time.Millisecond * 10) - var result map[string]interface{} - json.Unmarshal(body, &result) - message := result["error"].([]interface{})[0].(map[string]interface{})["message"].(string) - - if strings.Contains(message, "concurrent transaction") { - time.Sleep(time.Millisecond * 10) - continue - } - return response.StatusCode, body, nil - } -} - -func createSchemaRequest(url string, class string, multiTenantcy bool) *http.Request { - classObj := &models.Class{ - Class: class, - Description: "Dummy class for benchmarking purposes", - MultiTenancyConfig: &models.MultiTenancyConfig{ - Enabled: multiTenantcy, - }, - Properties: []*models.Property{ - { - DataType: []string{"int"}, - Description: "The value of the counter in the dataset", - Name: "counter", - }, - { - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - Description: "The value of the counter in the dataset", - Name: "name", - }, - }, - } - request := createRequest(url+"schema", "POST", classObj) - return request -} - -func createObject(class string) []*models.Object { - objects := []*models.Object{ - { - Class: class, - ID: strfmt.UUID(uuid.New().String()), - Vector: models.C11yVector([]float32{1.0, 2, 534, 324, 0.0001}), - Properties: map[string]interface{}{ - "counter": 50, - "counter2": 45, - "something": "JustSlammedMyKeyboardahudghoig", - }, - }, - } - return objects -} - -func createBatch(class string, batchSize int, tenants []models.Tenant) []*models.Object { - objects := make([]*models.Object, 0, batchSize) - for i := 0; i < batchSize; i++ { - objects = append(objects, &models.Object{ - Class: class, - ID: strfmt.UUID(uuid.New().String()), - Properties: map[string]interface{}{ - "counter": i, - "name": tenants[i%len(tenants)].Name, - }, - Tenant: tenants[i%len(tenants)].Name, - }) - } - return objects -} diff --git a/test/acceptance/stress_tests/tenants_test.go b/test/acceptance/stress_tests/tenants_test.go deleted file mode 100644 index aebd25f112612d51a9a378647d1efbfd90f77aa4..0000000000000000000000000000000000000000 --- a/test/acceptance/stress_tests/tenants_test.go +++ /dev/null @@ -1,112 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package stress_tests - -import ( - "encoding/json" - "fmt" - "sync" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/entities/models" -) - -func TestConcurrentTenantsAddAndRemove(t *testing.T) { - url := "http://localhost:8080/v1/" - - c := createHttpClient() - clearExistingObjects(c, url) - requestSchema := createSchemaRequest(url, class, true) - - performRequest(c, requestSchema) - - // Add schema and object - parallelReqs := 50 - wg := sync.WaitGroup{} - wgStartReqests := sync.WaitGroup{} - wgStartReqests.Add(parallelReqs) - wg.Add(parallelReqs) - - for i := 0; i < parallelReqs; i++ { - go func(j int) { - c := createHttpClient() - - requestAddTenant := createRequest(url+"schema/"+class+"/tenants", "POST", []models.Tenant{{Name: "tenant" + fmt.Sprint(j)}}) - requestDeleteTenant := createRequest(url+"schema/"+class+"/tenants", "DELETE", []string{"tenant" + fmt.Sprint(j)}) - - wgStartReqests.Done() - wgStartReqests.Wait() - performRequest(c, requestAddTenant) - performRequest(c, requestDeleteTenant) - - wg.Done() - }(i) - } - wg.Wait() - - requestGetTenants := createRequest(url+"schema/"+class+"/tenants", "GET", nil) - - _, body, _ := performRequest(c, requestGetTenants) - var result []*models.Tenant - json.Unmarshal(body, &result) - assert.Equal(t, 0, len(result)) -} - -func TestConcurrentTenantBatchesWithTenantAdd(t *testing.T) { - url := "http://localhost:8080/v1/" - - c := createHttpClient() - clearExistingObjects(c, url) - - tenants := make([]models.Tenant, 10) - for i := 0; i < len(tenants); i++ { - tenants[i] = models.Tenant{Name: "tenant" + fmt.Sprint(i)} - } - - nrClasses := 10 - for i := 0; i < nrClasses; i++ { - performRequest(c, createSchemaRequest(url, class+fmt.Sprint(i), true)) - performRequest(c, createRequest(url+"schema/"+class+fmt.Sprint(i)+"/tenants", "POST", tenants)) - } - - // Add schema and object - parallelReqs := 20 - wg := sync.WaitGroup{} - wgStartReqests := sync.WaitGroup{} - wgStartReqests.Add(parallelReqs) - wg.Add(parallelReqs) - - batchSize := 100 - - for i := 0; i < parallelReqs; i++ { - go func(j int) { - c := createHttpClient() - - requestBatch := createRequest(url+"batch/objects", "POST", batch{createBatch(class+fmt.Sprint(j%nrClasses), batchSize, tenants)}) - - wgStartReqests.Done() - wgStartReqests.Wait() - performRequest(c, requestBatch) - wg.Done() - }(i) - } - wg.Wait() - - // check for one class that we have the expected nr of objects per tenant - requestRead := createRequest(url+fmt.Sprintf("objects?limit=%v&class="+class+"%s&tenant=%s", 1000, "4", fmt.Sprint(tenants[0].Name)), "GET", nil) - _, body, _ := performRequest(c, requestRead) - var result map[string]interface{} - json.Unmarshal(body, &result) - assert.Equal(t, parallelReqs/nrClasses*batchSize/len(tenants), int(result["totalResults"].(float64))) // batches per class, * objects per batch/nr_tenants - clearExistingObjects(c, url) -} diff --git a/test/acceptance/telemetry/logging_test.go b/test/acceptance/telemetry/logging_test.go deleted file mode 100644 index 3b9252295b3c46ee1f6ae1e11745c957416e37f3..0000000000000000000000000000000000000000 --- a/test/acceptance/telemetry/logging_test.go +++ /dev/null @@ -1,130 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -// TODO gh-1232 remove -// // This test covers the Telemetry process. Weaviate should log each time its endpoints are accessed, -// // and it should send these logs to an endpoint. In the testing config environment this endpoint is -// // the mock api's address. This test sends a request to Weaviate to ensure at least one log *should* -// // exist, waits a few seconds to ensure the log *should* be sent to the mock endpoint and then retrieves -// // the most recently received log from the mock endpoint. The test then decodes the response and -// // validates its structure. -// // -// // controlled by setup_test.go -// func createActionLogging(t *testing.T) { -// // send a request -// sendCreateActionRequest(t) - -// // wait for the log to be posted -// time.Sleep(7 * time.Second) - -// result := retrieveLogFromMockEndpoint(t) - -// if result != nil { -// interpretedResult := interpretResult(t, result) -// if interpretedResult != nil { -// _, namePresent := interpretedResult["n"] -// _, typePresent := interpretedResult["t"] -// _, identifierPresent := interpretedResult["i"] -// _, amountPresent := interpretedResult["a"] -// _, whenPresent := interpretedResult["w"] - -// assert.Equal(t, true, namePresent) -// assert.Equal(t, true, typePresent) -// assert.Equal(t, true, identifierPresent) -// assert.Equal(t, true, amountPresent) -// assert.Equal(t, true, whenPresent) - -// } -// } -// } - -// // The sendCreateActionRequest acceptance test is copied here to ensure at least one request should be -// // logged when we check the mock api's most recently received request. These assertions should pass -// // regardless of the rest of the Telemetry test passing. -// func sendCreateActionRequest(t *testing.T) { -// // Set all action values to compare -// actionTestString := "Test string" -// actionTestInt := 1 -// actionTestBoolean := true -// actionTestNumber := 1.337 -// actionTestDate := "2017-10-06T08:15:30+01:00" - -// params := actions.NewActionsCreateParams().WithBody( -// &models.Action{ -// Class: "MonitoringTestAction", -// Schema: map[string]interface{}{ -// "testString": actionTestString, -// "testWholeNumber": actionTestInt, -// "testTrueFalse": actionTestBoolean, -// "testNumber": actionTestNumber, -// "testDateTime": actionTestDate, -// }, -// }) - -// resp, err := helper.Client(t).Actions.ActionsCreate(params, nil) - -// // Ensure that the response is OK. -// helper.AssertRequestOk(t, resp, err, func() { -// action := resp.Payload -// assert.Regexp(t, strfmt.UUIDPattern, action.ID) - -// schema, ok := action.Schema.(map[string]interface{}) -// if !ok { -// t.Fatal("The returned schema is not an JSON object") -// } - -// testWholeNumber, _ := schema["testWholeNumber"].(json.Number).Int64() -// testNumber, _ := schema["testNumber"].(json.Number).Float64() - -// // Check whether the returned information is the same as the data added. -// assert.Equal(t, actionTestString, schema["testString"]) -// assert.Equal(t, actionTestInt, int(testWholeNumber)) -// assert.Equal(t, actionTestBoolean, schema["testTrueFalse"]) -// assert.Equal(t, actionTestNumber, testNumber) -// assert.Equal(t, actionTestDate, schema["testDateTime"]) -// }) -// } - -// // retrieveLogFromMockEndpoint retrieves the most recently received log from the mock api. -// func retrieveLogFromMockEndpoint(t *testing.T) []byte { -// req, err := http.NewRequest("GET", "http://localhost:8087/mock/last", nil) -// req.Close = true -// assert.Equal(t, nil, err) - -// client := &http.Client{} -// resp, err := client.Do(req) -// if err == nil { -// body, _ := io.ReadAll(resp.Body) -// defer resp.Body.Close() -// return body -// } -// if err != nil { -// urlError, ok := err.(*url.Error) -// if ok { -// assert.Equal(t, nil, urlError.Op) -// } -// } - -// return nil -// } - -// // interpretResult converts the received cbor-encoded log to a []map[string]interface. -// func interpretResult(t *testing.T, resultBody []byte) map[string]interface{} { -// decoded := make([]map[string]interface{}, 1) -// cborHandle := new(codec.CborHandle) -// encoder := codec.NewDecoderBytes(resultBody, cborHandle) -// err := encoder.Decode(&decoded) - -// require.Equal(t, nil, err) -// return decoded[0] -// } diff --git a/test/acceptance/telemetry/setup_test.go b/test/acceptance/telemetry/setup_test.go deleted file mode 100644 index c1d40918ea8c62c77c0e5f90f256238db7794939..0000000000000000000000000000000000000000 --- a/test/acceptance/telemetry/setup_test.go +++ /dev/null @@ -1,70 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -// TODO gh-1232: remove -// import ( -// "testing" - -// "github.com/weaviate/weaviate/client/schema" -// "github.com/weaviate/weaviate/entities/models" -// "github.com/weaviate/weaviate/test/helper" -// ) - -// func Test_Actions(t *testing.T) { -// t.Run("setup", func(t *testing.T) { -// createActionClass(t, &models.Class{ -// Class: "MonitoringTestAction", -// Properties: []*models.Property{ -// &models.Property{ -// Name: "testString", -// DataType: schema.DataTypeText.PropString(), -// Tokenization: models.PropertyTokenizationWhitespace, -// }, -// &models.Property{ -// Name: "testWholeNumber", -// DataType: []string{"int"}, -// }, -// &models.Property{ -// Name: "testNumber", -// DataType: []string{"number"}, -// }, -// &models.Property{ -// Name: "testDateTime", -// DataType: []string{"date"}, -// }, -// &models.Property{ -// Name: "testTrueFalse", -// DataType: []string{"boolean"}, -// }, -// }, -// }) -// }) - -// // tests -// t.Run("create action logging", createActionLogging) - -// // tear down -// deleteActionClass(t, "MonitoringTestAction") -// } - -// func createActionClass(t *testing.T, class *models.Class) { -// params := schema.NewSchemaActionsCreateParams().WithActionClass(class) -// resp, err := helper.Client(t).Schema.SchemaActionsCreate(params, nil) -// helper.AssertRequestOk(t, resp, err, nil) -// } - -// func deleteActionClass(t *testing.T, class string) { -// delParams := schema.NewSchemaActionsDeleteParams().WithClassName(class) -// delRes, err := helper.Client(t).Schema.SchemaActionsDelete(delParams, nil) -// helper.AssertRequestOk(t, delRes, err, nil) -// } diff --git a/test/acceptance/vector_distances/cosine_test.go b/test/acceptance/vector_distances/cosine_test.go deleted file mode 100644 index 1b7b7defdcd05cf2d6a3f63a7d58286389bcb6ca..0000000000000000000000000000000000000000 --- a/test/acceptance/vector_distances/cosine_test.go +++ /dev/null @@ -1,559 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "encoding/json" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" -) - -func addTestDataCosine(t *testing.T) { - createObject(t, &models.Object{ - Class: "Cosine_Class", - Properties: map[string]interface{}{ - "name": "object_1", - }, - Vector: []float32{ - 0.7, 0.3, // our base object - }, - }) - - createObject(t, &models.Object{ - Class: "Cosine_Class", - Properties: map[string]interface{}{ - "name": "object_2", - }, - Vector: []float32{ - 1.4, 0.6, // identical angle to the base - }, - }) - - createObject(t, &models.Object{ - Class: "Cosine_Class", - Properties: map[string]interface{}{ - "name": "object_3", - }, - Vector: []float32{ - -0.7, -0.3, // perfect opposite of the base - }, - }) - - createObject(t, &models.Object{ - Class: "Cosine_Class", - Properties: map[string]interface{}{ - "name": "object_4", - }, - Vector: []float32{ - 1, 1, // somewhere in between - }, - }) -} - -func testCosine(t *testing.T) { - t.Run("without any limiting parameters", func(t *testing.T) { - res := AssertGraphQL(t, nil, ` - { - Get{ - Cosine_Class(nearVector:{vector: [0.7, 0.3]}){ - name - _additional{distance certainty} - } - } - } - `) - results := res.Get("Get", "Cosine_Class").AsSlice() - expectedDistances := []float32{ - 0, // the same vector as the query - 0, // the same angle as the query vector, - 0.0715, // the vector in between, - 2, // the perfect opposite vector, - } - - compareDistances(t, expectedDistances, results) - }) - - t.Run("limiting by certainty", func(t *testing.T) { - // cosine is a special case. It still supports certainty for legacy - // reasons. All other distances do not work with certainty. - - t.Run("Get: with certainty=0 meaning 'match anything'", func(t *testing.T) { - res := AssertGraphQL(t, nil, ` - { - Get{ - Cosine_Class(nearVector:{vector: [0.7, 0.3], certainty: 0}){ - name - _additional{distance certainty} - } - } - } - `) - results := res.Get("Get", "Cosine_Class").AsSlice() - expectedDistances := []float32{ - 0, // the same vector as the query - 0, // the same angle as the query vector, - 0.0715, // the vector in between, - 2, // the perfect opposite vector, - } - - compareDistances(t, expectedDistances, results) - - expectedCertainties := []float32{ - 1, // the same vector as the query - 1, // the same angle as the query vector, - 0.96, // the vector in between, - 0, // the perfect opposite vector, - } - - compareCertainties(t, expectedCertainties, results) - }) - - t.Run("Explore: with certainty=0 meaning 'match anything'", func(t *testing.T) { - res := AssertGraphQL(t, nil, ` - { - Explore(nearVector:{vector: [0.7, 0.3], certainty: 0}){ - distance certainty - } - } - `) - results := res.Get("Explore").AsSlice() - expectedDistances := []float32{ - 0, // the same vector as the query - 0, // the same angle as the query vector, - 0.0715, // the vector in between, - 2, // the perfect opposite vector, - } - - compareDistancesExplore(t, expectedDistances, results) - - expectedCertainties := []float32{ - 1, // the same vector as the query - 1, // the same angle as the query vector, - 0.96, // the vector in between, - 0, // the perfect opposite vector, - } - - compareCertaintiesExplore(t, expectedCertainties, results) - }) - - t.Run("Get: with certainty=0.95", func(t *testing.T) { - res := AssertGraphQL(t, nil, ` - { - Get{ - Cosine_Class(nearVector:{vector: [0.7, 0.3], certainty: 0.95}){ - name - _additional{distance certainty} - } - } - } - `) - results := res.Get("Get", "Cosine_Class").AsSlice() - expectedDistances := []float32{ - 0, // the same vector as the query - 0, // the same angle as the query vector, - 0.0715, // the vector in between, - } - - compareDistances(t, expectedDistances, results) - - expectedCertainties := []float32{ - 1, // the same vector as the query - 1, // the same angle as the query vector, - 0.96, // the vector in between, - // the last element does not have the required certainty (0<0.95) - } - - compareCertainties(t, expectedCertainties, results) - }) - - t.Run("Explore: with certainty=0.95", func(t *testing.T) { - res := AssertGraphQL(t, nil, ` - { - Explore(nearVector:{vector: [0.7, 0.3], certainty: 0.95}){ - distance certainty - } - } - `) - results := res.Get("Explore").AsSlice() - expectedDistances := []float32{ - 0, // the same vector as the query - 0, // the same angle as the query vector, - 0.0715, // the vector in between, - } - - compareDistancesExplore(t, expectedDistances, results) - - expectedCertainties := []float32{ - 1, // the same vector as the query - 1, // the same angle as the query vector, - 0.96, // the vector in between, - // the last element does not have the required certainty (0<0.95) - } - - compareCertaintiesExplore(t, expectedCertainties, results) - }) - - t.Run("Get: with certainty=0.97", func(t *testing.T) { - res := AssertGraphQL(t, nil, ` - { - Get{ - Cosine_Class(nearVector:{vector: [0.7, 0.3], certainty: 0.97}){ - name - _additional{distance certainty} - } - } - } - `) - results := res.Get("Get", "Cosine_Class").AsSlice() - expectedDistances := []float32{ - 0, // the same vector as the query - 0, // the same angle as the query vector, - } - - compareDistances(t, expectedDistances, results) - - expectedCertainties := []float32{ - 1, // the same vector as the query - 1, // the same angle as the query vector, - // the last two elements would have certainty of 0.96 and 0, so they won't match - } - - compareCertainties(t, expectedCertainties, results) - }) - - t.Run("Explore: with certainty=0.97", func(t *testing.T) { - res := AssertGraphQL(t, nil, ` - { - Explore(nearVector:{vector: [0.7, 0.3], certainty: 0.97}){ - distance certainty - } - } - `) - results := res.Get("Explore").AsSlice() - expectedDistances := []float32{ - 0, // the same vector as the query - 0, // the same angle as the query vector, - } - - compareDistancesExplore(t, expectedDistances, results) - - expectedCertainties := []float32{ - 1, // the same vector as the query - 1, // the same angle as the query vector, - // the last two elements would have certainty of 0.96 and 0, so they won't match - } - - compareCertaintiesExplore(t, expectedCertainties, results) - }) - - t.Run("Get: with certainty=1", func(t *testing.T) { - // only perfect matches should be included now (certainty=1, distance=0) - res := AssertGraphQL(t, nil, ` - { - Get{ - Cosine_Class(nearVector:{vector: [0.7, 0.3], certainty: 1}){ - name - _additional{distance certainty} - } - } - } - `) - results := res.Get("Get", "Cosine_Class").AsSlice() - expectedDistances := []float32{ - 0, // the same vector as the query - 0, // the same angle as the query vector, - } - - compareDistances(t, expectedDistances, results) - - expectedCertainties := []float32{ - 1, // the same vector as the query - 1, // the same angle as the query vector, - // the last two elements would have certainty of 0.96 and 0, so they won't match - } - - compareCertainties(t, expectedCertainties, results) - }) - - t.Run("Explore: with certainty=1", func(t *testing.T) { - // only perfect matches should be included now (certainty=1, distance=0) - res := AssertGraphQL(t, nil, ` - { - Explore(nearVector:{vector: [0.7, 0.3], certainty: 1}){ - distance certainty - } - } - `) - results := res.Get("Explore").AsSlice() - expectedDistances := []float32{ - 0, // the same vector as the query - 0, // the same angle as the query vector, - } - - compareDistancesExplore(t, expectedDistances, results) - - expectedCertainties := []float32{ - 1, // the same vector as the query - 1, // the same angle as the query vector, - // the last two elements would have certainty of 0.96 and 0, so they won't match - } - - compareCertaintiesExplore(t, expectedCertainties, results) - }) - }) - - t.Run("limiting by distance", func(t *testing.T) { - t.Run("Get: with distance=2, i.e. max distance, should match all", func(t *testing.T) { - res := AssertGraphQL(t, nil, ` - { - Get{ - Cosine_Class(nearVector:{vector: [0.7, 0.3], distance: 2}){ - name - _additional{distance certainty} - } - } - } - `) - results := res.Get("Get", "Cosine_Class").AsSlice() - expectedDistances := []float32{ - 0, // the same vector as the query - 0, // the same angle as the query vector, - 0.0715, // the vector in between, - 2, // the perfect opposite vector, - } - - compareDistances(t, expectedDistances, results) - }) - - t.Run("Explore: with distance=2, i.e. max distance, should match all", func(t *testing.T) { - res := AssertGraphQL(t, nil, ` - { - Explore(nearVector:{vector: [0.7, 0.3], distance: 2}){ - distance certainty - } - } - `) - results := res.Get("Explore").AsSlice() - expectedDistances := []float32{ - 0, // the same vector as the query - 0, // the same angle as the query vector, - 0.0715, // the vector in between, - 2, // the perfect opposite vector, - } - - compareDistancesExplore(t, expectedDistances, results) - }) - - t.Run("Get: with distance=1.99, should exclude the last", func(t *testing.T) { - res := AssertGraphQL(t, nil, ` - { - Get{ - Cosine_Class(nearVector:{vector: [0.7, 0.3], distance: 1.99}){ - name - _additional{distance certainty} - } - } - } - `) - results := res.Get("Get", "Cosine_Class").AsSlice() - expectedDistances := []float32{ - 0, // the same vector as the query - 0, // the same angle as the query vector, - 0.0715, // the vector in between, - // the vector with the perfect opposite has a distance of 2.00 which is > 1.99 - } - - compareDistances(t, expectedDistances, results) - }) - - t.Run("Explore: with distance=1.99, should exclude the last", func(t *testing.T) { - res := AssertGraphQL(t, nil, ` - { - Explore(nearVector:{vector: [0.7, 0.3], distance: 1.99}){ - distance certainty - } - } - `) - results := res.Get("Explore").AsSlice() - expectedDistances := []float32{ - 0, // the same vector as the query - 0, // the same angle as the query vector, - 0.0715, // the vector in between, - // the vector with the perfect opposite has a distance of 2.00 which is > 1.99 - } - - compareDistancesExplore(t, expectedDistances, results) - }) - - t.Run("Get: with distance=0.08, it should barely still match element 3", func(t *testing.T) { - res := AssertGraphQL(t, nil, ` - { - Get{ - Cosine_Class(nearVector:{vector: [0.7, 0.3], distance: 0.08}){ - name - _additional{distance certainty} - } - } - } - `) - results := res.Get("Get", "Cosine_Class").AsSlice() - expectedDistances := []float32{ - 0, // the same vector as the query - 0, // the same angle as the query vector, - 0.0715, // the vector in between, just within the allowed range - // the vector with the perfect opposite has a distance of 2.00 which is > 0.08 - } - - compareDistances(t, expectedDistances, results) - }) - - t.Run("Explore: with distance=0.08, it should barely still match element 3", func(t *testing.T) { - res := AssertGraphQL(t, nil, ` - { - Explore(nearVector:{vector: [0.7, 0.3], distance: 0.08}){ - distance certainty - } - } - `) - results := res.Get("Explore").AsSlice() - expectedDistances := []float32{ - 0, // the same vector as the query - 0, // the same angle as the query vector, - 0.0715, // the vector in between, just within the allowed range - // the vector with the perfect opposite has a distance of 2.00 which is > 0.08 - } - - compareDistancesExplore(t, expectedDistances, results) - }) - - t.Run("Get: with distance=0.01, most vectors are excluded", func(t *testing.T) { - res := AssertGraphQL(t, nil, ` - { - Get{ - Cosine_Class(nearVector:{vector: [0.7, 0.3], distance: 0.01}){ - name - _additional{distance certainty} - } - } - } - `) - results := res.Get("Get", "Cosine_Class").AsSlice() - expectedDistances := []float32{ - 0, // the same vector as the query - 0, // the same angle as the query vector, - // the third vector would have had a distance of 0.07... which is more than 0.01 - // the vector with the perfect opposite has a distance of 2.00 which is > 0.08 - } - - compareDistances(t, expectedDistances, results) - }) - - t.Run("Explore: with distance=0.01, most vectors are excluded", func(t *testing.T) { - res := AssertGraphQL(t, nil, ` - { - Explore(nearVector:{vector: [0.7, 0.3], distance: 0.01}){ - distance certainty - } - } - `) - results := res.Get("Explore").AsSlice() - expectedDistances := []float32{ - 0, // the same vector as the query - 0, // the same angle as the query vector, - // the third vector would have had a distance of 0.07... which is more than 0.01 - // the vector with the perfect opposite has a distance of 2.00 which is > 0.08 - } - - compareDistancesExplore(t, expectedDistances, results) - }) - - t.Run("Get: with distance=0, only perfect matches are allowed", func(t *testing.T) { - res := AssertGraphQL(t, nil, ` - { - Get{ - Cosine_Class(nearVector:{vector: [0.7, 0.3], distance: 0}){ - name - _additional{distance certainty} - } - } - } - `) - results := res.Get("Get", "Cosine_Class").AsSlice() - expectedDistances := []float32{ - 0, // the same vector as the query - 0, // the same angle as the query vector, - // only the first two vectors are perfect matches - } - - compareDistances(t, expectedDistances, results) - }) - - t.Run("Explore: with distance=0, only perfect matches are allowed", func(t *testing.T) { - res := AssertGraphQL(t, nil, ` - { - Explore(nearVector:{vector: [0.7, 0.3], distance: 0}){ - distance certainty - } - } - `) - results := res.Get("Explore").AsSlice() - expectedDistances := []float32{ - 0, // the same vector as the query - 0, // the same angle as the query vector, - // only the first two vectors are perfect matches - } - - compareDistancesExplore(t, expectedDistances, results) - }) - }) -} - -func compareDistances(t *testing.T, expectedDistances []float32, results []interface{}) { - require.Equal(t, len(expectedDistances), len(results)) - for i, expected := range expectedDistances { - actual, err := results[i].(map[string]interface{})["_additional"].(map[string]interface{})["distance"].(json.Number).Float64() - require.Nil(t, err) - assert.InDelta(t, expected, actual, 0.01) - } -} - -func compareDistancesExplore(t *testing.T, expectedDistances []float32, results []interface{}) { - require.Equal(t, len(expectedDistances), len(results)) - for i, expected := range expectedDistances { - actual, err := results[i].(map[string]interface{})["distance"].(json.Number).Float64() - require.Nil(t, err) - assert.InDelta(t, expected, actual, 0.01) - } -} - -// unique to cosine for legacy reasons -func compareCertainties(t *testing.T, expectedDistances []float32, results []interface{}) { - require.Equal(t, len(expectedDistances), len(results)) - for i, expected := range expectedDistances { - actual, err := results[i].(map[string]interface{})["_additional"].(map[string]interface{})["certainty"].(json.Number).Float64() - require.Nil(t, err) - assert.InDelta(t, expected, actual, 0.01) - } -} - -// unique to cosine for legacy reasons -func compareCertaintiesExplore(t *testing.T, expectedDistances []float32, results []interface{}) { - require.Equal(t, len(expectedDistances), len(results)) - for i, expected := range expectedDistances { - actual, err := results[i].(map[string]interface{})["certainty"].(json.Number).Float64() - require.Nil(t, err) - assert.InDelta(t, expected, actual, 0.01) - } -} diff --git a/test/acceptance/vector_distances/data_test.go b/test/acceptance/vector_distances/data_test.go deleted file mode 100644 index 0470032a5a3e76840103dc12c120f866fd1db117..0000000000000000000000000000000000000000 --- a/test/acceptance/vector_distances/data_test.go +++ /dev/null @@ -1,98 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "testing" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" -) - -func addTestSchemaCosine(t *testing.T) { - createObjectClass(t, &models.Class{ - Class: "Cosine_Class", - Vectorizer: "none", - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - VectorIndexConfig: map[string]interface{}{ - "distance": "cosine", - }, - }) -} - -func addTestSchemaOther(t *testing.T) { - createObjectClass(t, &models.Class{ - Class: "Dot_Class", - Vectorizer: "none", - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - VectorIndexConfig: map[string]interface{}{ - "distance": "dot", - }, - }) - - createObjectClass(t, &models.Class{ - Class: "L2Squared_Class", - Vectorizer: "none", - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - VectorIndexConfig: map[string]interface{}{ - "distance": "l2-squared", - }, - }) - - createObjectClass(t, &models.Class{ - Class: "Manhattan_Class", - Vectorizer: "none", - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - VectorIndexConfig: map[string]interface{}{ - "distance": "manhattan", - }, - }) - - createObjectClass(t, &models.Class{ - Class: "Hamming_Class", - Vectorizer: "none", - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - VectorIndexConfig: map[string]interface{}{ - "distance": "hamming", - }, - }) -} diff --git a/test/acceptance/vector_distances/dot_test.go b/test/acceptance/vector_distances/dot_test.go deleted file mode 100644 index 92ea6a2b5a67858e01fabbbe519a818c74d9bd35..0000000000000000000000000000000000000000 --- a/test/acceptance/vector_distances/dot_test.go +++ /dev/null @@ -1,217 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "testing" - - "github.com/weaviate/weaviate/entities/models" -) - -func addTestDataDot(t *testing.T) { - createObject(t, &models.Object{ - Class: "Dot_Class", - Properties: map[string]interface{}{ - "name": "object_1", - }, - Vector: []float32{ - 3, 4, 5, // our base object - }, - }) - - createObject(t, &models.Object{ - Class: "Dot_Class", - Properties: map[string]interface{}{ - "name": "object_2", - }, - Vector: []float32{ - 1, 1, 1, // a length-one vector - }, - }) - - createObject(t, &models.Object{ - Class: "Dot_Class", - Properties: map[string]interface{}{ - "name": "object_3", - }, - Vector: []float32{ - 0, 0, 0, // a zero vecto - }, - }) - - createObject(t, &models.Object{ - Class: "Dot_Class", - Properties: map[string]interface{}{ - "name": "object_2", - }, - Vector: []float32{ - -3, -4, -5, // negative of the base vector - }, - }) -} - -func testDot(t *testing.T) { - t.Run("without any limiting distance", func(t *testing.T) { - res := AssertGraphQL(t, nil, ` - { - Get{ - Dot_Class(nearVector:{vector: [3,4,5]}){ - name - _additional{distance} - } - } - } - `) - results := res.Get("Get", "Dot_Class").AsSlice() - expectedDistances := []float32{ - -50, // the same vector as the query - -12, // the same angle as the query vector, - 0, // the vector in between, - 50, // the negative of the query vec - } - - compareDistances(t, expectedDistances, results) - }) - - t.Run("with a specified certainty arg - should error", func(t *testing.T) { - ErrorGraphQL(t, nil, ` - { - Get{ - Dot_Class(nearVector:{certainty: 0.7, vector: [3,4,5]}){ - name - _additional{distance} - } - } - } - `) - }) - - t.Run("with a specified certainty prop - should error", func(t *testing.T) { - ErrorGraphQL(t, nil, ` - { - Get{ - Dot_Class(nearVector:{distance: 0.7, vector: [3,4,5]}){ - name - _additional{certainty} - } - } - } - `) - }) - - t.Run("with a max distancer higher than all results, should contain all elements", func(t *testing.T) { - res := AssertGraphQL(t, nil, ` - { - Get{ - Dot_Class(nearVector:{distance: 50, vector: [3,4,5]}){ - name - _additional{distance} - } - } - } - `) - results := res.Get("Get", "Dot_Class").AsSlice() - expectedDistances := []float32{ - -50, // the same vector as the query - -12, // the same angle as the query vector, - 0, // the vector in between, - 50, // the negative of the query vec - } - - compareDistances(t, expectedDistances, results) - }) - - t.Run("with a positive max distance that does not match all results, should contain 3 elems", func(t *testing.T) { - res := AssertGraphQL(t, nil, ` - { - Get{ - Dot_Class(nearVector:{distance: 30, vector: [3,4,5]}){ - name - _additional{distance} - } - } - } - `) - results := res.Get("Get", "Dot_Class").AsSlice() - expectedDistances := []float32{ - -50, // the same vector as the query - -12, // the same angle as the query vector, - 0, // the vector in between, - // the last one is not contained as it would have a distance of 50, which is > 30 - } - - compareDistances(t, expectedDistances, results) - }) - - t.Run("with distance 0, should contain 3 elems", func(t *testing.T) { - res := AssertGraphQL(t, nil, ` - { - Get{ - Dot_Class(nearVector:{distance: 0, vector: [3,4,5]}){ - name - _additional{distance} - } - } - } - `) - results := res.Get("Get", "Dot_Class").AsSlice() - expectedDistances := []float32{ - -50, // the same vector as the query - -12, // the same angle as the query vector, - 0, // the vector in between, - // the last one is not contained as it would have a distance of 50, which is > 0 - } - - compareDistances(t, expectedDistances, results) - }) - - t.Run("with a negative distance that should only leave the first element", func(t *testing.T) { - res := AssertGraphQL(t, nil, ` - { - Get{ - Dot_Class(nearVector:{distance: -40, vector: [3,4,5]}){ - name - _additional{distance} - } - } - } - `) - results := res.Get("Get", "Dot_Class").AsSlice() - expectedDistances := []float32{ - -50, // the same vector as the query - // the second element's distance would be -12 which is > -40 - // the third element's distance would be 0 which is > -40 - // the last one is not contained as it would have a distance of 50, which is > 0 - } - - compareDistances(t, expectedDistances, results) - }) - - t.Run("with a distance so small that no element should be left", func(t *testing.T) { - res := AssertGraphQL(t, nil, ` - { - Get{ - Dot_Class(nearVector:{distance: -60, vector: [3,4,5]}){ - name - _additional{distance} - } - } - } - `) - results := res.Get("Get", "Dot_Class").AsSlice() - expectedDistances := []float32{ - // all elements have a distance > -60, so nothing matches - } - - compareDistances(t, expectedDistances, results) - }) -} diff --git a/test/acceptance/vector_distances/explore_test.go b/test/acceptance/vector_distances/explore_test.go deleted file mode 100644 index 0163ab86694cc4cc97309147abdadd61222fa2c6..0000000000000000000000000000000000000000 --- a/test/acceptance/vector_distances/explore_test.go +++ /dev/null @@ -1,128 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "encoding/json" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" -) - -func testExplore(t *testing.T) { - className := "L2Squared_Class_2" - defer deleteObjectClass(t, className) - - t.Run("create second L2 class", func(t *testing.T) { - createObjectClass(t, &models.Class{ - Class: className, - Vectorizer: "none", - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - VectorIndexConfig: map[string]interface{}{ - "distance": "l2-squared", - }, - }) - }) - - t.Run("create objects for the new class", func(t *testing.T) { - createObject(t, &models.Object{ - Class: className, - Properties: map[string]interface{}{ - "name": "object_1", - }, - Vector: []float32{ - 6, 7, 8, - }, - }) - - createObject(t, &models.Object{ - Class: className, - Properties: map[string]interface{}{ - "name": "object_2", - }, - Vector: []float32{ - 1, 2, 3, - }, - }) - }) - - t.Run("run Explore and assert results", func(t *testing.T) { - res := AssertGraphQL(t, nil, ` - { - Explore(nearVector: {vector: [3, 4, 5], distance: 365}) { - distance - className - } - } - `) - - explore := res.Get("Explore").AsSlice() - - expected := []struct { - className string - dist json.Number - }{ - {className: "L2Squared_Class_2", dist: "12"}, - {className: "L2Squared_Class_2", dist: "27"}, - {className: "L2Squared_Class", dist: "50"}, - {className: "L2Squared_Class", dist: "147"}, - {className: "L2Squared_Class", dist: "365"}, - } - - for i, item := range explore { - m, ok := item.(map[string]interface{}) - assert.True(t, ok) - assert.Equal(t, expected[i].dist, m["distance"]) - assert.Equal(t, expected[i].className, m["className"]) - } - }) - - t.Run("run Explore with certainty arg and expect failure", func(t *testing.T) { - res := ErrorGraphQL(t, nil, ` - { - Explore(nearVector: {vector: [3, 4, 5], certainty: 0.4}) { - distance - className - } - } - `) - - assert.Len(t, res, 1) - expectedErr := "can't compute and return certainty when vector index is configured with l2-squared distance" - errorMsg := res[0].Message - assert.Equal(t, expectedErr, errorMsg) - }) - - t.Run("run Explore with certainty prop and expect failure", func(t *testing.T) { - res := ErrorGraphQL(t, nil, ` - { - Explore(nearVector: {vector: [3, 4, 5], distance: 0.4}) { - certainty - className - } - } - `) - - assert.Len(t, res, 1) - expectedErr := "can't compute and return certainty when vector index is configured with l2-squared distance" - errorMsg := res[0].Message - assert.Equal(t, expectedErr, errorMsg) - }) -} diff --git a/test/acceptance/vector_distances/graphql_helper_test.go b/test/acceptance/vector_distances/graphql_helper_test.go deleted file mode 100644 index 9e7f8b9e132e2d68ad6221e37de9e6bcb4e98bd4..0000000000000000000000000000000000000000 --- a/test/acceptance/vector_distances/graphql_helper_test.go +++ /dev/null @@ -1,106 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "encoding/json" - "fmt" - "testing" - - "github.com/go-openapi/runtime" - "github.com/weaviate/weaviate/client/graphql" - graphql_client "github.com/weaviate/weaviate/client/graphql" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/test/helper" -) - -type GraphQLResult struct { - Result interface{} -} - -// Perform a GraphQL request -func QueryGraphQL(t *testing.T, auth runtime.ClientAuthInfoWriterFunc, operation string, query string, variables map[string]interface{}) (*models.GraphQLResponse, error) { - var vars interface{} = variables - params := graphql_client.NewGraphqlPostParams().WithBody(&models.GraphQLQuery{OperationName: operation, Query: query, Variables: vars}) - response, err := helper.Client(t).Graphql.GraphqlPost(params, nil) - if err != nil { - return nil, err - } - - return response.Payload, nil -} - -// Perform a GraphQL request and call fatal on failure -func QueryGraphQLOrFatal(t *testing.T, auth runtime.ClientAuthInfoWriterFunc, operation string, query string, variables map[string]interface{}) *models.GraphQLResponse { - response, err := QueryGraphQL(t, auth, operation, query, variables) - if err != nil { - parsedErr, ok := err.(*graphql.GraphqlPostUnprocessableEntity) - if !ok { - t.Fatalf("Expected the query to succeed, but failed due to: %#v", err) - } - t.Fatalf("Expected the query to succeed, but failed with unprocessable entity: %v", parsedErr.Payload.Error[0]) - } - return response -} - -// Perform a query and assert that it is successful -func AssertGraphQL(t *testing.T, auth runtime.ClientAuthInfoWriterFunc, query string) *GraphQLResult { - response := QueryGraphQLOrFatal(t, auth, "", query, nil) - - if len(response.Errors) != 0 { - j, _ := json.Marshal(response.Errors) - t.Fatal("GraphQL resolved to an error:", string(j)) - } - - data := make(map[string]interface{}) - - // get rid of models.JSONData - for key, value := range response.Data { - data[key] = value - } - - return &GraphQLResult{Result: data} -} - -// Perform a query and assert that it has errors -func ErrorGraphQL(t *testing.T, auth runtime.ClientAuthInfoWriterFunc, query string) []*models.GraphQLError { - response := QueryGraphQLOrFatal(t, auth, "", query, nil) - - if len(response.Errors) == 0 { - j, _ := json.Marshal(response.Errors) - t.Fatal("GraphQL resolved to data:", string(j)) - } - - return response.Errors -} - -// Drill down in the result -func (g GraphQLResult) Get(paths ...string) *GraphQLResult { - current := g.Result - for _, path := range paths { - var ok bool - currentAsMap := (current.(map[string]interface{})) - current, ok = currentAsMap[path] - if !ok { - panic(fmt.Sprintf("Cannot get element %s in %#v; result: %#v", path, paths, g.Result)) - } - } - - return &GraphQLResult{ - Result: current, - } -} - -// Cast the result to a slice -func (g *GraphQLResult) AsSlice() []interface{} { - return g.Result.([]interface{}) -} diff --git a/test/acceptance/vector_distances/hamming_test.go b/test/acceptance/vector_distances/hamming_test.go deleted file mode 100644 index 333799df0c4b5177a0e1a33bf6219b4b329b37b1..0000000000000000000000000000000000000000 --- a/test/acceptance/vector_distances/hamming_test.go +++ /dev/null @@ -1,185 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "testing" - - "github.com/weaviate/weaviate/entities/models" -) - -func addTestDataHamming(t *testing.T) { - createObject(t, &models.Object{ - Class: "Hamming_Class", - Properties: map[string]interface{}{ - "name": "object_1", - }, - Vector: []float32{ - 10, 10, 10, - }, - }) - - createObject(t, &models.Object{ - Class: "Hamming_Class", - Properties: map[string]interface{}{ - "name": "object_2", - }, - Vector: []float32{ - 10, 10, 12, - }, - }) - - createObject(t, &models.Object{ - Class: "Hamming_Class", - Properties: map[string]interface{}{ - "name": "object_3", - }, - Vector: []float32{ - 10, 11, 12, - }, - }) -} - -func testHamming(t *testing.T) { - t.Run("without any limiting parameters", func(t *testing.T) { - res := AssertGraphQL(t, nil, ` - { - Get{ - Hamming_Class(nearVector:{vector: [10,10,10]}){ - name - _additional{distance} - } - } - } - `) - results := res.Get("Get", "Hamming_Class").AsSlice() - expectedDistances := []float32{ - 0, // the same vector as the query - 1, // hamming distance to object_3 vector - 2, // hamming distance to object_4 vector - } - - compareDistances(t, expectedDistances, results) - }) - - t.Run("with a certainty arg", func(t *testing.T) { - // not supported for non-cosine distances - ErrorGraphQL(t, nil, ` - { - Get{ - Hamming_Class(nearVector:{vector: [10,11,12], certainty:0.3}){ - name - _additional{distance} - } - } - } - `) - }) - - t.Run("with a certainty prop", func(t *testing.T) { - // not supported for non-cosine distances - ErrorGraphQL(t, nil, ` - { - Get{ - Hamming_Class(nearVector:{vector: [10,11,12], distance:0.3}){ - name - _additional{certainty} - } - } - } - `) - }) - - t.Run("a high distance that includes all elements", func(t *testing.T) { - res := AssertGraphQL(t, nil, ` - { - Get{ - Hamming_Class(nearVector:{vector: [10,10,10], distance: 365}){ - name - _additional{distance} - } - } - } - `) - results := res.Get("Get", "Hamming_Class").AsSlice() - expectedDistances := []float32{ - 0, // the same vector as the query - 1, // hamming distance to object_3 vector - 2, // hamming distance to object_4 vector - } - - compareDistances(t, expectedDistances, results) - }) - - t.Run("a distance that is too low for the last element", func(t *testing.T) { - res := AssertGraphQL(t, nil, ` - { - Get{ - Hamming_Class(nearVector:{vector: [10,10,10], distance: 1.5}){ - name - _additional{distance} - } - } - } - `) - results := res.Get("Get", "Hamming_Class").AsSlice() - expectedDistances := []float32{ - 0, // the same vector as the query - 1, // hamming distance to object_3 vector - // hamming distance to object_4 vector skipped because 2>1 - } - - compareDistances(t, expectedDistances, results) - }) - - t.Run("a distance that is too low for the second element", func(t *testing.T) { - res := AssertGraphQL(t, nil, ` - { - Get{ - Hamming_Class(nearVector:{vector: [10,10,10], distance: 0.5}){ - name - _additional{distance} - } - } - } - `) - results := res.Get("Get", "Hamming_Class").AsSlice() - expectedDistances := []float32{ - 0, // the same vector as the query - // hamming distance to object_3 vector skipped because 1>0.5 - // hamming distance to object_4 vector skipped because 2>0.5 - } - - compareDistances(t, expectedDistances, results) - }) - - t.Run("a distance of 0 only matches exact elements", func(t *testing.T) { - res := AssertGraphQL(t, nil, ` - { - Get{ - Hamming_Class(nearVector:{vector: [10,10,10], distance: 0}){ - name - _additional{distance} - } - } - } - `) - results := res.Get("Get", "Hamming_Class").AsSlice() - expectedDistances := []float32{ - 0, // the same vector as the query - // second elem skipped, because 1 > 0 - // last eleme skipped, because 2 > 0 - } - - compareDistances(t, expectedDistances, results) - }) -} diff --git a/test/acceptance/vector_distances/l2_test.go b/test/acceptance/vector_distances/l2_test.go deleted file mode 100644 index 9a52e67250df8b60b3726a0040468afac42b3104..0000000000000000000000000000000000000000 --- a/test/acceptance/vector_distances/l2_test.go +++ /dev/null @@ -1,206 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "testing" - - "github.com/weaviate/weaviate/entities/models" -) - -func addTestDataL2(t *testing.T) { - createObject(t, &models.Object{ - Class: "L2Squared_Class", - Properties: map[string]interface{}{ - "name": "object_1", - }, - Vector: []float32{ - 10, 11, 12, - }, - }) - - createObject(t, &models.Object{ - Class: "L2Squared_Class", - Properties: map[string]interface{}{ - "name": "object_2", - }, - Vector: []float32{ - 13, 15, 17, - }, - }) - - createObject(t, &models.Object{ - Class: "L2Squared_Class", - Properties: map[string]interface{}{ - "name": "object_3", - }, - Vector: []float32{ - 0, 0, 0, // a zero vecto - }, - }) -} - -func testL2(t *testing.T) { - t.Run("without any limiting parameters", func(t *testing.T) { - res := AssertGraphQL(t, nil, ` - { - Get{ - L2Squared_Class(nearVector:{vector: [10,11,12]}){ - name - _additional{distance} - } - } - } - `) - results := res.Get("Get", "L2Squared_Class").AsSlice() - expectedDistances := []float32{ - 0, // the same vector as the query - 50, // distance to the second vector - 365, // l2 squared distance to the root - } - - compareDistances(t, expectedDistances, results) - }) - - t.Run("with a certainty arg", func(t *testing.T) { - // not supported for non-cosine distances - ErrorGraphQL(t, nil, ` - { - Get{ - L2Squared_Class(nearVector:{vector: [10,11,12], certainty:0.3}){ - name - _additional{distance} - } - } - } - `) - }) - - t.Run("with a certainty prop", func(t *testing.T) { - // not supported for non-cosine distances - ErrorGraphQL(t, nil, ` - { - Get{ - L2Squared_Class(nearVector:{vector: [10,11,12], distance:0.3}){ - name - _additional{certainty} - } - } - } - `) - }) - - t.Run("a high distance that includes all elements", func(t *testing.T) { - res := AssertGraphQL(t, nil, ` - { - Get{ - L2Squared_Class(nearVector:{vector: [10,11,12], distance: 365}){ - name - _additional{distance} - } - } - } - `) - results := res.Get("Get", "L2Squared_Class").AsSlice() - expectedDistances := []float32{ - 0, // the same vector as the query - 50, // distance to the second vector - 365, // l2 squared distance to the root - } - - compareDistances(t, expectedDistances, results) - }) - - t.Run("a distance that is too low for the last element", func(t *testing.T) { - res := AssertGraphQL(t, nil, ` - { - Get{ - L2Squared_Class(nearVector:{vector: [10,11,12], distance: 364}){ - name - _additional{distance} - } - } - } - `) - results := res.Get("Get", "L2Squared_Class").AsSlice() - expectedDistances := []float32{ - 0, // the same vector as the query - 50, // distance to the second vector - // last eleme skipped, because 365 > 364 - } - - compareDistances(t, expectedDistances, results) - }) - - t.Run("a distance that is too low for the second element", func(t *testing.T) { - res := AssertGraphQL(t, nil, ` - { - Get{ - L2Squared_Class(nearVector:{vector: [10,11,12], distance: 49}){ - name - _additional{distance} - } - } - } - `) - results := res.Get("Get", "L2Squared_Class").AsSlice() - expectedDistances := []float32{ - 0, // the same vector as the query - // second elem skipped, because 50 > 49 - // last eleme skipped, because 365 > 364 - } - - compareDistances(t, expectedDistances, results) - }) - - t.Run("a really low distance that only matches one elem", func(t *testing.T) { - res := AssertGraphQL(t, nil, ` - { - Get{ - L2Squared_Class(nearVector:{vector: [10,11,12], distance: 0.001}){ - name - _additional{distance} - } - } - } - `) - results := res.Get("Get", "L2Squared_Class").AsSlice() - expectedDistances := []float32{ - 0, // the same vector as the query - // second elem skipped, because 50 > 0.001 - // last eleme skipped, because 365 > 0.001 - } - - compareDistances(t, expectedDistances, results) - }) - - t.Run("a distance of 0 only matches exact elements", func(t *testing.T) { - res := AssertGraphQL(t, nil, ` - { - Get{ - L2Squared_Class(nearVector:{vector: [10,11,12], distance: 0}){ - name - _additional{distance} - } - } - } - `) - results := res.Get("Get", "L2Squared_Class").AsSlice() - expectedDistances := []float32{ - 0, // the same vector as the query - // second elem skipped, because 50 > 0 - // last eleme skipped, because 365 > 0 - } - - compareDistances(t, expectedDistances, results) - }) -} diff --git a/test/acceptance/vector_distances/manhattan_test.go b/test/acceptance/vector_distances/manhattan_test.go deleted file mode 100644 index e3752d588e3873d79ac0c8ce17f4a9b6dd0590ea..0000000000000000000000000000000000000000 --- a/test/acceptance/vector_distances/manhattan_test.go +++ /dev/null @@ -1,206 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "testing" - - "github.com/weaviate/weaviate/entities/models" -) - -func addTestDataManhattan(t *testing.T) { - createObject(t, &models.Object{ - Class: "Manhattan_Class", - Properties: map[string]interface{}{ - "name": "object_1", - }, - Vector: []float32{ - 10, 11, 12, - }, - }) - - createObject(t, &models.Object{ - Class: "Manhattan_Class", - Properties: map[string]interface{}{ - "name": "object_2", - }, - Vector: []float32{ - 13, 15, 17, - }, - }) - - createObject(t, &models.Object{ - Class: "Manhattan_Class", - Properties: map[string]interface{}{ - "name": "object_3", - }, - Vector: []float32{ - 0, 0, 0, // a zero vector - }, - }) -} - -func testManhattan(t *testing.T) { - t.Run("without any limiting parameters", func(t *testing.T) { - res := AssertGraphQL(t, nil, ` - { - Get{ - Manhattan_Class(nearVector:{vector: [10,11,12]}){ - name - _additional{distance} - } - } - } - `) - results := res.Get("Get", "Manhattan_Class").AsSlice() - expectedDistances := []float32{ - 0, // the same vector as the query - 12, // distance to the second vector -> abs(10-13)+abs(11-15)+abs(12-17) = 3+4+5 = 12 - 33, // manhattan distance to the root -> 10+11+12 = 33 - } - - compareDistances(t, expectedDistances, results) - }) - - t.Run("with a certainty arg", func(t *testing.T) { - // not supported for non-cosine distances - ErrorGraphQL(t, nil, ` - { - Get{ - Manhattan_Class(nearVector:{vector: [10,11,12], certainty:0.3}){ - name - _additional{distance} - } - } - } - `) - }) - - t.Run("with a certainty prop", func(t *testing.T) { - // not supported for non-cosine distances - ErrorGraphQL(t, nil, ` - { - Get{ - Manhattan_Class(nearVector:{vector: [10,11,12], distance:0.3}){ - name - _additional{certainty} - } - } - } - `) - }) - - t.Run("a high distance that includes all elements", func(t *testing.T) { - res := AssertGraphQL(t, nil, ` - { - Get{ - Manhattan_Class(nearVector:{vector: [10,11,12], distance: 365}){ - name - _additional{distance} - } - } - } - `) - results := res.Get("Get", "Manhattan_Class").AsSlice() - expectedDistances := []float32{ - 0, // the same vector as the query - 12, // distance to the second vector -> abs(10-13)+abs(11-15)+abs(12-17) = 3+4+5 = 12 - 33, // manhattan distance to the root -> 10+11+12 = 33 - } - - compareDistances(t, expectedDistances, results) - }) - - t.Run("a distance that is too low for the last element", func(t *testing.T) { - res := AssertGraphQL(t, nil, ` - { - Get{ - Manhattan_Class(nearVector:{vector: [10,11,12], distance: 30}){ - name - _additional{distance} - } - } - } - `) - results := res.Get("Get", "Manhattan_Class").AsSlice() - expectedDistances := []float32{ - 0, // the same vector as the query - 12, // distance to the second vector -> abs(10-13)+abs(11-15)+abs(12-17) = 3+4+5 = 12 - // last skipped, as 33 > 30 - } - - compareDistances(t, expectedDistances, results) - }) - - t.Run("a distance that is too low for the second element", func(t *testing.T) { - res := AssertGraphQL(t, nil, ` - { - Get{ - Manhattan_Class(nearVector:{vector: [10,11,12], distance: 10}){ - name - _additional{distance} - } - } - } - `) - results := res.Get("Get", "Manhattan_Class").AsSlice() - expectedDistances := []float32{ - 0, // the same vector as the query - // second elem skipped, because 12 > 10 - // last eleme skipped, because 33 > 10 - } - - compareDistances(t, expectedDistances, results) - }) - - t.Run("a really low distance that only matches one elem", func(t *testing.T) { - res := AssertGraphQL(t, nil, ` - { - Get{ - Manhattan_Class(nearVector:{vector: [10,11,12], distance: 0.001}){ - name - _additional{distance} - } - } - } - `) - results := res.Get("Get", "Manhattan_Class").AsSlice() - expectedDistances := []float32{ - 0, // the same vector as the query - // second elem skipped, because 12 > 0.001 - // last eleme skipped, because 33 > 0.001 - } - - compareDistances(t, expectedDistances, results) - }) - - t.Run("a distance of 0 only matches exact elements", func(t *testing.T) { - res := AssertGraphQL(t, nil, ` - { - Get{ - Manhattan_Class(nearVector:{vector: [10,11,12], distance: 0}){ - name - _additional{distance} - } - } - } - `) - results := res.Get("Get", "Manhattan_Class").AsSlice() - expectedDistances := []float32{ - 0, // the same vector as the query - // second elem skipped, because 12 > 0 - // last eleme skipped, because 33 > 0 - } - - compareDistances(t, expectedDistances, results) - }) -} diff --git a/test/acceptance/vector_distances/setup_test.go b/test/acceptance/vector_distances/setup_test.go deleted file mode 100644 index 437ecd7ea4474051dc69946efbf2f5ceba7b77a7..0000000000000000000000000000000000000000 --- a/test/acceptance/vector_distances/setup_test.go +++ /dev/null @@ -1,71 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "testing" - - "github.com/weaviate/weaviate/client/objects" - "github.com/weaviate/weaviate/client/schema" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/test/helper" -) - -func Test_GraphQL(t *testing.T) { - t.Run("setup test schema (cosine only)", addTestSchemaCosine) - // at this point only cosine is present, so we can evaluate both Get and - // Explore - t.Run("import cosine test data ", addTestDataCosine) - t.Run("test cosine distance", testCosine) - - // import rest of the schema meaning, we can only test Get, Explore is now - // impossible - t.Run("setup test schema (all)", addTestSchemaOther) - t.Run("import dot test data ", addTestDataDot) - t.Run("test dot distance", testDot) - t.Run("import l2 test data ", addTestDataL2) - t.Run("test l2 distance", testL2) - t.Run("import manhattan test data", addTestDataManhattan) - t.Run("test manhattan distance", testManhattan) - t.Run("import hamming test data", addTestDataHamming) - t.Run("test hamming distance", testHamming) - - // tear down what we no longer need - deleteObjectClass(t, "Cosine_Class") - deleteObjectClass(t, "Dot_Class") - deleteObjectClass(t, "Manhattan_Class") - deleteObjectClass(t, "Hamming_Class") - - // now only l2 is left so we can test explore with L2 - t.Run("explore across multiple non-cosine classes", testExplore) - - // tear down remaining classes - deleteObjectClass(t, "L2Squared_Class") -} - -func createObjectClass(t *testing.T, class *models.Class) { - params := schema.NewSchemaObjectsCreateParams().WithObjectClass(class) - resp, err := helper.Client(t).Schema.SchemaObjectsCreate(params, nil) - helper.AssertRequestOk(t, resp, err, nil) -} - -func createObject(t *testing.T, object *models.Object) { - params := objects.NewObjectsCreateParams().WithBody(object) - resp, err := helper.Client(t).Objects.ObjectsCreate(params, nil) - helper.AssertRequestOk(t, resp, err, nil) -} - -func deleteObjectClass(t *testing.T, class string) { - delParams := schema.NewSchemaObjectsDeleteParams().WithClassName(class) - delRes, err := helper.Client(t).Schema.SchemaObjectsDelete(delParams, nil) - helper.AssertRequestOk(t, delRes, err, nil) -} diff --git a/test/acceptance_with_go_client/auth_tests/auth_setup_test.go b/test/acceptance_with_go_client/auth_tests/auth_setup_test.go deleted file mode 100644 index 8a1e406755d0f190e3ed4be4c5ae7e1a5b1997fc..0000000000000000000000000000000000000000 --- a/test/acceptance_with_go_client/auth_tests/auth_setup_test.go +++ /dev/null @@ -1,42 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package auth_tests - -import ( - "context" - "os" - "testing" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/test/docker" -) - -const weaviateEndpoint = "WEAVIATE_ENDPOINT" - -func TestMain(m *testing.M) { - ctx := context.Background() - compose, err := docker.New(). - WithWeaviateAuth(). - Start(ctx) - if err != nil { - panic(errors.Wrapf(err, "cannot start")) - } - - os.Setenv(weaviateEndpoint, compose.GetWeaviate().URI()) - code := m.Run() - - if err := compose.Terminate(ctx); err != nil { - panic(errors.Wrapf(err, "cannot terminate")) - } - - os.Exit(code) -} diff --git a/test/acceptance_with_go_client/auth_tests/auth_test.go b/test/acceptance_with_go_client/auth_tests/auth_test.go deleted file mode 100644 index bac38683dec898de5704d74917456389c93e57e7..0000000000000000000000000000000000000000 --- a/test/acceptance_with_go_client/auth_tests/auth_test.go +++ /dev/null @@ -1,100 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package auth_tests - -import ( - "context" - "os" - "testing" - - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate-go-client/v4/weaviate/auth" - "github.com/weaviate/weaviate-go-client/v4/weaviate/fault" - "github.com/weaviate/weaviate/entities/models" - - client "github.com/weaviate/weaviate-go-client/v4/weaviate" -) - -const ( - wcsUserOnAdmin = "ms_2d0e007e7136de11d5f29fce7a53dae219a51458@existiert.net" - wcsUserNotOnAdmin = "ms_f0559e7899681721a44d9ca1f7418ff3cc6321c3@muellmail.com" -) - -func TestAuthGraphQLUnauthenticated(t *testing.T) { - ctx := context.Background() - c := client.New(client.Config{Scheme: "http", Host: os.Getenv(weaviateEndpoint)}) - _, err := c.GraphQL().Raw().WithQuery("{__schema {queryType {fields {name}}}}").Do(ctx) - require.NotNil(t, err) -} - -func TestAuthGraphQLValidUserNotOnAdminlist(t *testing.T) { - pw := os.Getenv("WCS_DUMMY_CI_PW_2") - if pw == "" { - t.Skip("No password supplied") - } - - ctx := context.Background() - - conf := client.Config{ - Scheme: "http", - Host: os.Getenv(weaviateEndpoint), - AuthConfig: auth.ResourceOwnerPasswordFlow{Username: wcsUserNotOnAdmin, Password: pw}, - } - c, err := client.NewClient(conf) - require.Nil(t, err) - - _, err = c.GraphQL().Raw().WithQuery("{__schema {queryType {fields {name}}}}").Do(ctx) - require.NotNil(t, err) -} - -func TestAuthGraphQLValidUser(t *testing.T) { - pwAdminUser := os.Getenv("WCS_DUMMY_CI_PW") - pwNoAdminUser := os.Getenv("WCS_DUMMY_CI_PW_2") - if pwAdminUser == "" || pwNoAdminUser == "" { - t.Skip("No password supplied") - } - - ctx := context.Background() - conf := client.Config{ - Scheme: "http", - Host: os.Getenv(weaviateEndpoint), - AuthConfig: auth.ResourceOwnerPasswordFlow{Username: wcsUserOnAdmin, Password: pwAdminUser}, - } - c, err := client.NewClient(conf) - require.Nil(t, err) - - // add a class so schema is not empty - require.Nil(t, c.Schema().AllDeleter().Do(ctx)) - require.Nil(t, c.Schema().ClassCreator().WithClass(&models.Class{Class: "Pizza"}).Do(ctx)) - - t.Run("returns schema without error for admin", func(t *testing.T) { - _, err = c.GraphQL().Raw().WithQuery("{__schema {queryType {fields {name}}}}").Do(ctx) - require.Nil(t, err) - }) - - t.Run("returns auth error for non-admin", func(t *testing.T) { - conf2 := client.Config{ - Scheme: "http", - Host: os.Getenv(weaviateEndpoint), - AuthConfig: auth.ResourceOwnerPasswordFlow{Username: wcsUserNotOnAdmin, Password: pwNoAdminUser}, - } - cNoAdmin, err := client.NewClient(conf2) - require.Nil(t, err) - - _, err = cNoAdmin.GraphQL().Raw().WithQuery("{__schema {queryType {fields {name}}}}").Do(ctx) - require.NotNil(t, err) - wErr, ok := err.(*fault.WeaviateClientError) - require.True(t, ok) - - require.Contains(t, wErr.DerivedFromError.Error(), "forbidden") - }) -} diff --git a/test/acceptance_with_go_client/autoschema_test.go b/test/acceptance_with_go_client/autoschema_test.go deleted file mode 100644 index 81f8a8616b01774d810d6da212ca80c631b5f5e0..0000000000000000000000000000000000000000 --- a/test/acceptance_with_go_client/autoschema_test.go +++ /dev/null @@ -1,303 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package acceptance_with_go_client - -import ( - "context" - "testing" - - "github.com/weaviate/weaviate-go-client/v4/weaviate/graphql" - "github.com/weaviate/weaviate/entities/models" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - client "github.com/weaviate/weaviate-go-client/v4/weaviate" -) - -func TestAutoschemaCasingClass(t *testing.T) { - ctx := context.Background() - c, err := client.NewClient(client.Config{Scheme: "http", Host: "localhost:8080"}) - require.Nil(t, err) - - upperClassName := "RandomBlueTree" - lowerClassName := "randomBlueTree" - - cases := []struct { - className1 string - className2 string - }{ - {className1: upperClassName, className2: upperClassName}, - {className1: lowerClassName, className2: lowerClassName}, - {className1: upperClassName, className2: lowerClassName}, - {className1: lowerClassName, className2: upperClassName}, - } - for _, tt := range cases { - t.Run(tt.className1+" "+tt.className2, func(t *testing.T) { - c.Schema().ClassDeleter().WithClassName(tt.className1).Do(ctx) - c.Schema().ClassDeleter().WithClassName(tt.className2).Do(ctx) - creator := c.Data().Creator() - _, err := creator.WithClassName(tt.className1).Do(ctx) - require.Nil(t, err) - - _, err = creator.WithClassName(tt.className2).Do(ctx) - require.Nil(t, err) - - // Regardless of whether a class exists or not, the delete operation will always return a success - require.Nil(t, c.Schema().ClassDeleter().WithClassName(upperClassName).Do(ctx)) - require.Nil(t, c.Schema().ClassDeleter().WithClassName(lowerClassName).Do(ctx)) - }) - } -} - -func TestAutoschemaCasingProps(t *testing.T) { - ctx := context.Background() - c, err := client.NewClient(client.Config{Scheme: "http", Host: "localhost:8080"}) - require.Nil(t, err) - - className := "RandomGreenBike" - - upperPropName := "SomeProp" - lowerPropName := "someProp" - cases := []struct { - prop1 string - prop2 string - }{ - {prop1: upperPropName, prop2: upperPropName}, - {prop1: lowerPropName, prop2: lowerPropName}, - {prop1: upperPropName, prop2: lowerPropName}, - {prop1: lowerPropName, prop2: upperPropName}, - } - for _, tt := range cases { - t.Run(tt.prop1+" "+tt.prop2, func(t *testing.T) { - c.Schema().ClassDeleter().WithClassName(className).Do(ctx) - creator := c.Data().Creator() - _, err := creator.WithClassName(className).Do(ctx) - require.Nil(t, err) - - creator1 := c.Data().Creator() - _, err = creator1.WithClassName(className).WithProperties(map[string]string{tt.prop1: "something"}).Do(ctx) - require.Nil(t, err) - - creator2 := c.Data().Creator() - _, err = creator2.WithClassName(className).WithProperties(map[string]string{tt.prop2: "other value"}).Do(ctx) - require.Nil(t, err) - - // three objects should have been added - result, err := c.GraphQL().Aggregate().WithClassName(className).WithFields(graphql.Field{ - Name: "meta", Fields: []graphql.Field{ - {Name: "count"}, - }, - }).Do(ctx) - require.Nil(t, err) - require.Equal(t, result.Data["Aggregate"].(map[string]interface{})[className].([]interface{})[0].(map[string]interface{})["meta"].(map[string]interface{})["count"], 3.) - - require.Nil(t, c.Schema().ClassDeleter().WithClassName(className).Do(ctx)) - }) - } -} - -func TestAutoschemaCasingUpdateProps(t *testing.T) { - ctx := context.Background() - c, err := client.NewClient(client.Config{Scheme: "http", Host: "localhost:8080"}) - require.Nil(t, err) - - objId := "67b79643-cf8b-4b22-b206-6e63dbb4e57a" - upperPropName := "SomeProp" - lowerPropName := "someProp" - cases := []struct { - prop1 string - prop2 string - }{ - {prop1: upperPropName, prop2: upperPropName}, - {prop1: lowerPropName, prop2: lowerPropName}, - {prop1: upperPropName, prop2: lowerPropName}, - {prop1: lowerPropName, prop2: upperPropName}, - } - for _, tt := range cases { - t.Run(tt.prop1+" "+tt.prop2, func(t *testing.T) { - className := "RandomOliveTree" - c.Schema().ClassDeleter().WithClassName(className).Do(ctx) - creator := c.Data().Creator() - _, err := creator.WithClassName(className).Do(ctx) - require.Nil(t, err) - - creator1 := c.Data().Creator() - _, err = creator1.WithClassName(className).WithID(objId).WithProperties(map[string]string{tt.prop1: "something"}).Do(ctx) - require.Nil(t, err) - - updater := c.Data().Updater() - err = updater.WithClassName(className).WithID(objId).WithProperties(map[string]string{tt.prop2: "other"}).Do(ctx) - require.Nil(t, err) - - // two objects should have been added (with one update - result, err := c.GraphQL().Aggregate().WithClassName(className).WithFields(graphql.Field{ - Name: "meta", Fields: []graphql.Field{ - {Name: "count"}, - }, - }).Do(ctx) - require.Nil(t, err) - require.Equal(t, result.Data["Aggregate"].(map[string]interface{})[className].([]interface{})[0].(map[string]interface{})["meta"].(map[string]interface{})["count"], 2.) - - require.Nil(t, c.Schema().ClassDeleter().WithClassName(className).Do(ctx)) - }) - } -} - -func TestAutoschemaPanicOnUnregonizedDataType(t *testing.T) { - ctx := context.Background() - c, err := client.NewClient(client.Config{Scheme: "http", Host: "localhost:8080"}) - require.Nil(t, err) - - tests := []struct { - name string - properties map[string]interface{} - containsErrMessage string - }{ - { - name: "unrecognized array property type", - properties: map[string]interface{}{ - "panicProperty": []interface{}{ - []interface{}{ - []interface{}{ - "panic", - }, - }, - }, - }, - containsErrMessage: "property 'panicProperty' on class 'BeautifulWeather': element [0]: unrecognized data type of value", - }, - { - name: "unrecognized nil array property type", - properties: map[string]interface{}{ - "panicProperty": []interface{}{ - []interface{}{ - []interface{}{ - nil, - }, - }, - }, - }, - containsErrMessage: "property 'panicProperty' on class 'BeautifulWeather': element [0]: unrecognized data type of value", - }, - { - name: "array property with nil", - properties: map[string]interface{}{ - "nilPropertyArray": []interface{}{nil}, - }, - containsErrMessage: "property 'nilPropertyArray' on class 'BeautifulWeather': element [0]: unrecognized data type of value ''", - }, - { - name: "empty string array property", - properties: map[string]interface{}{ - "emptyPropertyArray": []string{}, - }, - }, - { - name: "empty interface array property", - properties: map[string]interface{}{ - "emptyPropertyArray": []interface{}{}, - }, - }, - { - name: "empty int array property", - properties: map[string]interface{}{ - "emptyPropertyArray": []int{}, - }, - }, - { - name: "array property with empty string", - properties: map[string]interface{}{ - "emptyPropertyArray": []string{""}, - }, - }, - { - name: "nil property", - properties: map[string]interface{}{ - "nilProperty": nil, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - resp, err := c.Data(). - Creator(). - WithClassName("BeautifulWeather"). - WithProperties(tt.properties). - Do(ctx) - - if tt.containsErrMessage != "" { - assert.Nil(t, resp) - assert.NotNil(t, err) - assert.ErrorContains(t, err, tt.containsErrMessage) - } else { - assert.NotNil(t, resp) - assert.Nil(t, err) - } - - err = c.Schema().ClassDeleter().WithClassName("BeautifulWeather").Do(ctx) - require.Nil(t, err) - }) - } -} - -func TestAutoschemaPanicOnUnregonizedDataTypeWithBatch(t *testing.T) { - ctx := context.Background() - c, err := client.NewClient(client.Config{Scheme: "http", Host: "localhost:8080"}) - require.Nil(t, err) - - className := "Passage" - t.Run("should not panic with properties defined as empty array, but just return error", func(t *testing.T) { - obj := &models.Object{ - Class: className, - Properties: []interface{}{}, - } - - resp, err := c.Batch().ObjectsBatcher().WithObject(obj).Do(ctx) - require.Nil(t, err) - require.Len(t, resp, 1) - require.NotNil(t, resp[0].Result) - require.NotNil(t, resp[0].Result.Errors) - require.Len(t, resp[0].Result.Errors.Error, 1) - assert.Equal(t, "could not recognize object's properties: []", resp[0].Result.Errors.Error[0].Message) - - objs, err := c.Data().ObjectsGetter().WithClassName(className).Do(ctx) - require.Nil(t, err) - require.Len(t, objs, 0) - - err = c.Schema().ClassDeleter().WithClassName(className).Do(ctx) - require.Nil(t, err) - }) - - t.Run("should create object in batch without problems", func(t *testing.T) { - obj := &models.Object{ - Class: className, - Properties: map[string]interface{}{ - "stringProperty": "value", - }, - } - resp, err := c.Batch().ObjectsBatcher().WithObject(obj).Do(ctx) - require.Nil(t, err) - require.Len(t, resp, 1) - require.NotNil(t, resp[0].Result) - require.Nil(t, resp[0].Result.Errors) - require.NotNil(t, resp[0].Object) - assert.True(t, len(resp[0].Object.Vector) > 0) - - objs, err := c.Data().ObjectsGetter().WithClassName(className).Do(ctx) - require.Nil(t, err) - require.Len(t, objs, 1) - - err = c.Schema().ClassDeleter().WithClassName(className).Do(ctx) - require.Nil(t, err) - }) -} diff --git a/test/acceptance_with_go_client/compression/compression_test.go b/test/acceptance_with_go_client/compression/compression_test.go deleted file mode 100644 index 862f53e157d49f61b96dcd273efae98dee736912..0000000000000000000000000000000000000000 --- a/test/acceptance_with_go_client/compression/compression_test.go +++ /dev/null @@ -1,242 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package compression_test - -import ( - "context" - "fmt" - "math/rand" - "testing" - "time" - - "github.com/stretchr/testify/require" - wvt "github.com/weaviate/weaviate-go-client/v4/weaviate" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/entities/vectorindex/hnsw" - "github.com/weaviate/weaviate/test/docker" -) - -func TestCompression_AdaptSegments(t *testing.T) { - type testCase struct { - dimensions int - } - - testCases := []testCase{ - {dimensions: 768}, - {dimensions: 125}, - {dimensions: 4}, - } - - ctx := context.Background() - className := "Compressed" - - t.Run("async indexing", func(t *testing.T) { - compose, err := docker.New(). - WithWeaviate(). - WithWeaviateEnv("ASYNC_INDEXING", "true"). - Start(ctx) - require.NoError(t, err) - defer func() { - require.NoError(t, compose.Terminate(ctx)) - }() - - client, err := wvt.NewClient(wvt.Config{Scheme: "http", Host: compose.GetWeaviate().URI()}) - require.NoError(t, err) - cleanup := func() { - err := client.Schema().AllDeleter().Do(ctx) - require.NoError(t, err) - } - - for _, tc := range testCases { - t.Run(fmt.Sprintf("dimensions %d", tc.dimensions), func(t *testing.T) { - defer cleanup() - - var class *models.Class - - t.Run("create class", func(t *testing.T) { - class = createClass(t, client, className) - }) - - t.Run("compress", func(t *testing.T) { - pq := class.VectorIndexConfig.(map[string]interface{})["pq"].(map[string]interface{}) - pq["segments"] = 0 - pq["centroids"] = 256 - pq["trainingLimit"] = 999 - - err = client.Schema().ClassUpdater(). - WithClass(class). - Do(ctx) - require.NoError(t, err) - }) - - t.Run("populate", func(t *testing.T) { - populate(t, client, className, tc.dimensions) - }) - - t.Run("check eventually compressed", func(t *testing.T) { - checkEventuallyCompressed(t, client, className) - }) - }) - } - }) - - t.Run("sync indexing", func(t *testing.T) { - client, err := wvt.NewClient(wvt.Config{Scheme: "http", Host: "localhost:8080"}) - require.NoError(t, err) - cleanup := func() { - err := client.Schema().AllDeleter().Do(ctx) - require.NoError(t, err) - } - - for _, tc := range testCases { - t.Run(fmt.Sprintf("dimensions %d", tc.dimensions), func(t *testing.T) { - defer cleanup() - - var class *models.Class - - t.Run("create class", func(t *testing.T) { - class = createClass(t, client, className) - }) - - t.Run("populate", func(t *testing.T) { - populate(t, client, className, tc.dimensions) - }) - - t.Run("compress", func(t *testing.T) { - pq := class.VectorIndexConfig.(map[string]interface{})["pq"].(map[string]interface{}) - pq["segments"] = 0 - pq["centroids"] = 256 - - err = client.Schema().ClassUpdater(). - WithClass(class). - Do(ctx) - require.NoError(t, err) - }) - - t.Run("check eventually compressed", func(t *testing.T) { - checkEventuallyCompressed(t, client, className) - }) - }) - } - }) -} - -func createClass(t *testing.T, client *wvt.Client, className string) *models.Class { - ctx := context.Background() - efConstruction := 64 - ef := 32 - maxNeighbors := 32 - vectorCacheMaxObjects := 10e12 - - class := &models.Class{ - Class: className, - Properties: []*models.Property{{ - Name: "int", - DataType: schema.DataTypeInt.PropString(), - }}, - Vectorizer: "none", - VectorIndexConfig: map[string]interface{}{ - "maxConnections": maxNeighbors, - "efConstruction": efConstruction, - "ef": ef, - "vectorCacheMaxObjects": vectorCacheMaxObjects, - "distance": "l2-squared", - "pq": map[string]interface{}{ - "enabled": true, - "encoder": map[string]interface{}{ - "distribution": hnsw.PQEncoderDistributionNormal, - "type": hnsw.PQEncoderTypeKMeans, - }, - }, - }, - } - - err := client.Schema().ClassCreator(). - WithClass(class). - Do(ctx) - require.NoError(t, err) - - return class -} - -func populate(t *testing.T, client *wvt.Client, className string, dimensions int) { - ctx := context.Background() - vectorsCount := 1000 - - objects := make([]*models.Object, vectorsCount) - for i, vector := range randomVecs(vectorsCount, dimensions) { - objects[i] = &models.Object{ - Class: className, - Properties: map[string]interface{}{ - "int": i, - }, - Vector: vector, - } - } - resps, err := client.Batch().ObjectsBatcher(). - WithObjects(objects...). - Do(ctx) - require.NoError(t, err) - require.Len(t, resps, vectorsCount) - for _, resp := range resps { - require.NotNil(t, resp.Result.Status) - require.Equal(t, models.ObjectsGetResponseAO2ResultStatusSUCCESS, *resp.Result.Status) - } -} - -func checkEventuallyCompressed(t *testing.T, client *wvt.Client, className string) { - ctx := context.Background() - timeout := 15 * time.Minute - interval := 1 * time.Second - - var compressed bool - end := time.Now().Add(timeout) - for time.Now().Before(end) { - resp, err := client.Cluster().NodesStatusGetter(). - WithClass(className). - WithOutput("verbose"). - Do(ctx) - require.NoError(t, err) - require.NotNil(t, resp) - require.Len(t, resp.Nodes, 1) - require.Len(t, resp.Nodes[0].Shards, 1) - - compressed = resp.Nodes[0].Shards[0].Compressed - if compressed { - break - } - time.Sleep(interval) - } - - require.True(t, compressed) -} - -func randomVecs(size int, dimensions int) [][]float32 { - r := rand.New(rand.NewSource(time.Now().UnixNano())) - vectors := make([][]float32, size) - for i := range vectors { - vectors[i] = genVector(r, dimensions) - } - return vectors -} - -func genVector(r *rand.Rand, dimensions int) []float32 { - vector := make([]float32, dimensions) - for i := range vector { - // Some distances like dot could produce negative values when the vectors have negative values - // This change will not affect anything when using a distance like l2, but will cover some bugs - // when using distances like dot - vector[i] = r.Float32()*2 - 1 - } - return vector -} diff --git a/test/acceptance_with_go_client/endpoint_test.go b/test/acceptance_with_go_client/endpoint_test.go deleted file mode 100644 index d84eecb8325ff3229921cd05d40f67f0bb54c600..0000000000000000000000000000000000000000 --- a/test/acceptance_with_go_client/endpoint_test.go +++ /dev/null @@ -1,100 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package acceptance_with_go_client - -import ( - "context" - "testing" - - "github.com/weaviate/weaviate-go-client/v4/weaviate/filters" - "github.com/weaviate/weaviate-go-client/v4/weaviate/graphql" - - "github.com/stretchr/testify/require" - client "github.com/weaviate/weaviate-go-client/v4/weaviate" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" -) - -func TestUpdatingPropertiesWithNil(t *testing.T) { - ctx := context.Background() - c, err := client.NewClient(client.Config{Scheme: "http", Host: "localhost:8080"}) - require.Nil(t, err) - - className := "RandomPinkFlower" - upperPropName := "SomeProp" - lowerPropName := "someProp" - cases := []struct { - prop1 string - prop2 string - }{ - {prop1: upperPropName, prop2: upperPropName}, - {prop1: lowerPropName, prop2: lowerPropName}, - {prop1: upperPropName, prop2: lowerPropName}, - {prop1: lowerPropName, prop2: upperPropName}, - } - for _, tt := range cases { - t.Run(tt.prop1+" "+tt.prop2, func(t *testing.T) { - c.Schema().ClassDeleter().WithClassName(className).Do(ctx) - defer c.Schema().ClassDeleter().WithClassName(className).Do(ctx) - classCreator := c.Schema().ClassCreator() - class := models.Class{ - Class: className, - Properties: []*models.Property{{ - Name: tt.prop1, - // TODO change to method call - DataType: []string{string(schema.DataTypeText)}, - // TODO change to constant - Tokenization: "whitespace", - }}, - InvertedIndexConfig: &models.InvertedIndexConfig{IndexNullState: true}, - } - require.Nil(t, classCreator.WithClass(&class).Do(ctx)) - - obj, err := c.Data().Creator().WithClassName(className).WithProperties( - map[string]interface{}{tt.prop1: "SomeText"}, - ).WithID("0a21ae23-3f22-4f34-b04a-199e701886ef").Do(ctx) - require.Nil(t, err) - - require.Nil(t, c.Data().Updater().WithClassName(className).WithProperties(map[string]interface{}{tt.prop2: nil}).WithID(string(obj.Object.ID)).WithMerge().Do(ctx)) - - // update should have cleared the object - getter := c.Data().ObjectsGetter() - objAfterUpdate, err := getter.WithID(string(obj.Object.ID)).WithClassName(className).Do(ctx) - require.Nil(t, err) - require.Len(t, objAfterUpdate[0].Properties, 0) - - // test that II has been updated: - // a) no results for when filtering for old value - // b) one result when filtering for null values - filter := filters.Where() - filter.WithValueString("SomeText") - filter.WithOperator(filters.Equal) - filter.WithPath([]string{lowerPropName}) - resultFilter, err := c.GraphQL().Get().WithClassName(className).WithWhere(filter).WithFields(graphql.Field{Name: "_additional", Fields: []graphql.Field{{Name: "id"}}}).Do(ctx) - require.Nil(t, err) - require.Len(t, resultFilter.Data["Get"].(map[string]interface{})[className], 0) - - filter = filters.Where() - filter.WithValueBoolean(true) - filter.WithOperator("IsNull") // replace with real operator after updating go client - filter.WithPath([]string{lowerPropName}) - resultFilter, err = c.GraphQL().Get().WithClassName(className).WithWhere(filter).WithFields(graphql.Field{Name: "_additional", Fields: []graphql.Field{{Name: "id"}}}).Do(ctx) - require.Nil(t, err) - require.Len(t, resultFilter.Data["Get"].(map[string]interface{})[className], 1) - - // Property is still part of the class - schemaClass, err := c.Schema().ClassGetter().WithClassName(className).Do(ctx) - require.Nil(t, err) - require.Len(t, schemaClass.Properties, 1) - }) - } -} diff --git a/test/acceptance_with_go_client/filters_tests/contains_any_all_test.go b/test/acceptance_with_go_client/filters_tests/contains_any_all_test.go deleted file mode 100644 index e67d3afb8fcaff9ff6ef754b98f8528d0b048c25..0000000000000000000000000000000000000000 --- a/test/acceptance_with_go_client/filters_tests/contains_any_all_test.go +++ /dev/null @@ -1,632 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package filters_tests - -import ( - "context" - "fmt" - "testing" - "time" - - acceptance_with_go_client "acceptance_tests_with_client" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - wvt "github.com/weaviate/weaviate-go-client/v4/weaviate" - "github.com/weaviate/weaviate-go-client/v4/weaviate/filters" - "github.com/weaviate/weaviate-go-client/v4/weaviate/graphql" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" -) - -func testContainsAnyAll(t *testing.T, host string) func(t *testing.T) { - return func(t *testing.T) { - client, err := wvt.NewClient(wvt.Config{Scheme: "http", Host: host}) - require.Nil(t, err) - - className := "WhereTest" - id1 := "00000000-0000-0000-0000-000000000001" - id2 := "00000000-0000-0000-0000-000000000002" - id3 := "00000000-0000-0000-0000-000000000003" - ids := []string{id1, id2, id3} - - cleanup := func() { - err := client.Schema().AllDeleter().Do(context.Background()) - require.Nil(t, err) - } - - t.Run("create class", func(t *testing.T) { - cleanup() - class := &models.Class{ - Class: className, - Properties: []*models.Property{ - { - Name: "color", - DataType: schema.DataTypeText.PropString(), - }, - { - Name: "colors", - DataType: schema.DataTypeTextArray.PropString(), - }, - { - Name: "author", - DataType: schema.DataTypeString.PropString(), - }, - { - Name: "authors", - DataType: schema.DataTypeStringArray.PropString(), - }, - { - Name: "number", - DataType: schema.DataTypeNumber.PropString(), - }, - { - Name: "numbers", - DataType: schema.DataTypeNumberArray.PropString(), - }, - { - Name: "int", - DataType: schema.DataTypeInt.PropString(), - }, - { - Name: "ints", - DataType: schema.DataTypeIntArray.PropString(), - }, - { - Name: "date", - DataType: schema.DataTypeDate.PropString(), - }, - { - Name: "dates", - DataType: schema.DataTypeDateArray.PropString(), - }, - { - Name: "bool", - DataType: schema.DataTypeBoolean.PropString(), - }, - { - Name: "bools", - DataType: schema.DataTypeBooleanArray.PropString(), - }, - { - Name: "uuid", - DataType: schema.DataTypeUUID.PropString(), - }, - { - Name: "uuids", - DataType: schema.DataTypeUUIDArray.PropString(), - }, - }, - } - err := client.Schema().ClassCreator().WithClass(class).Do(context.TODO()) - require.Nil(t, err) - }) - - t.Run("where with bm25 without data", func(t *testing.T) { - mustGetTime := func(date string) time.Time { - result, err := time.Parse(time.RFC3339Nano, date) - if err != nil { - panic(fmt.Sprintf("can't parse date: %v", date)) - } - return result - } - tests := []struct { - name string - where *filters.WhereBuilder - property string - }{ - // Contains operator with array types - { - name: "contains any authors with string array", - where: filters.Where(). - WithPath([]string{"authors"}). - WithOperator(filters.ContainsAny). - WithValueString("John", "Jenny", "Joseph"), - property: "authors", - }, - { - name: "contains any colors with text array", - where: filters.Where(). - WithPath([]string{"colors"}). - WithOperator(filters.ContainsAny). - WithValueText("red", "blue", "green"), - property: "colors", - }, - { - name: "contains any numbers with number array", - where: filters.Where(). - WithPath([]string{"numbers"}). - WithOperator(filters.ContainsAny). - WithValueNumber(1.1, 2.2, 3.3), - property: "numbers", - }, - { - name: "contains any ints with int array", - where: filters.Where(). - WithPath([]string{"ints"}). - WithOperator(filters.ContainsAny). - WithValueInt(1, 2, 3), - property: "ints", - }, - { - name: "contains any bools with bool array", - where: filters.Where(). - WithPath([]string{"bools"}). - WithOperator(filters.ContainsAny). - WithValueBoolean(true, false), - property: "bools", - }, - { - name: "contains any uuids with uuid array", - where: filters.Where(). - WithPath([]string{"uuids"}). - WithOperator(filters.ContainsAny). - WithValueText(id1, id2, id3), - property: "uuids", - }, - { - name: "contains any dates with dates array", - where: filters.Where(). - WithPath([]string{"dates"}). - WithOperator(filters.ContainsAny). - WithValueDate( - mustGetTime("2009-11-01T23:00:00Z"), mustGetTime("2009-11-02T23:00:00Z"), mustGetTime("2009-11-03T23:00:00Z"), - ), - property: "dates", - }, - // Contains operator with primitives - { - name: "contains any author with string", - where: filters.Where(). - WithPath([]string{"author"}). - WithOperator(filters.ContainsAny). - WithValueString("John", "Jenny", "Joseph"), - property: "author", - }, - { - name: "contains any color with text", - where: filters.Where(). - WithPath([]string{"color"}). - WithOperator(filters.ContainsAny). - WithValueText("red", "blue", "green"), - property: "color", - }, - { - name: "contains any number with number", - where: filters.Where(). - WithPath([]string{"number"}). - WithOperator(filters.ContainsAny). - WithValueNumber(1.1, 2.2, 3.3), - property: "number", - }, - { - name: "contains any int with int", - where: filters.Where(). - WithPath([]string{"int"}). - WithOperator(filters.ContainsAny). - WithValueInt(1, 2, 3), - property: "int", - }, - { - name: "contains any bool with bool", - where: filters.Where(). - WithPath([]string{"bool"}). - WithOperator(filters.ContainsAny). - WithValueBoolean(true, false, true), - property: "bool", - }, - { - name: "contains any uuid with uuid", - where: filters.Where(). - WithPath([]string{"uuid"}). - WithOperator(filters.ContainsAny). - WithValueText(id1, id2, id3), - property: "uuid", - }, - { - name: "contains any uuid with id", - where: filters.Where(). - WithPath([]string{"id"}). - WithOperator(filters.ContainsAny). - WithValueText(id1, id2, id3), - property: "uuid", - }, - { - name: "contains any date with date", - where: filters.Where(). - WithPath([]string{"date"}). - WithOperator(filters.ContainsAny). - WithValueDate( - mustGetTime("2009-11-01T23:00:00Z"), mustGetTime("2009-11-02T23:00:00Z"), mustGetTime("2009-11-03T23:00:00Z"), - ), - property: "date", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - fields := []graphql.Field{ - {Name: tt.property}, - {Name: "_additional", Fields: []graphql.Field{{Name: "id"}}}, - } - resp, err := client.GraphQL().Get(). - WithClassName(className). - WithWhere(tt.where). - WithBM25(client.GraphQL(). - Bm25ArgBuilder(). - WithQuery(tt.property), - ). - WithFields(fields...). - Do(context.TODO()) - require.Nil(t, err) - require.Empty(t, resp.Errors) - }) - } - }) - - t.Run("with data", func(t *testing.T) { - t.Run("insert data", func(t *testing.T) { - authors := []string{"John", "Jenny", "Joseph"} - authorsArray := [][]string{ - {"John", "Jenny", "Joseph"}, - {"John", "Jenny"}, - {"John"}, - } - colors := []string{"red", "blue", "green"} - colorssArray := [][]string{ - {"red", "blue", "green"}, - {"red", "blue"}, - {"red"}, - } - numbers := []float64{1.1, 2.2, 3.3} - numbersArray := [][]float64{ - {1.1, 2.2, 3.3}, - {1.1, 2.2}, - {1.1}, - } - ints := []int64{1, 2, 3} - intsArray := [][]int64{ - {1, 2, 3}, - {1, 2}, - {1}, - } - uuids := []string{id1, id2, id3} - uuidsArray := [][]string{ - {id1, id2, id3}, - {id1, id2}, - {id1}, - } - dates := []string{"2009-11-01T23:00:00Z", "2009-11-02T23:00:00Z", "2009-11-03T23:00:00Z"} - datesArray := [][]string{ - {"2009-11-01T23:00:00Z", "2009-11-02T23:00:00Z", "2009-11-03T23:00:00Z"}, - {"2009-11-01T23:00:00Z", "2009-11-02T23:00:00Z"}, - {"2009-11-01T23:00:00Z"}, - } - bools := []bool{true, false, true} - boolsArray := [][]bool{ - {true, false, true}, - {true, false}, - {true}, - } - for i, id := range ids { - _, err := client.Data().Creator(). - WithClassName(className). - WithID(id). - WithProperties(map[string]interface{}{ - "color": colors[i], - "colors": colorssArray[i], - "author": authors[i], - "authors": authorsArray[i], - "number": numbers[i], - "numbers": numbersArray[i], - "int": ints[i], - "ints": intsArray[i], - "uuid": uuids[i], - "uuids": uuidsArray[i], - "date": dates[i], - "dates": datesArray[i], - "bool": bools[i], - "bools": boolsArray[i], - }). - Do(context.TODO()) - require.Nil(t, err) - } - }) - - t.Run("where", func(t *testing.T) { - mustGetTime := func(date string) time.Time { - result, err := time.Parse(time.RFC3339Nano, date) - if err != nil { - panic(fmt.Sprintf("can't parse date: %v", date)) - } - return result - } - tests := []struct { - name string - where *filters.WhereBuilder - property string - nearText *graphql.NearTextArgumentBuilder - expectedIds []string - }{ - // Contains operator with array types - { - name: "contains all authors with string array", - where: filters.Where(). - WithPath([]string{"authors"}). - WithOperator(filters.ContainsAll). - WithValueString("John", "Jenny", "Joseph"), - property: "authors", - expectedIds: []string{id1}, - }, - { - name: "contains any authors with string array", - where: filters.Where(). - WithPath([]string{"authors"}). - WithOperator(filters.ContainsAny). - WithValueString("John", "Jenny", "Joseph"), - property: "authors", - expectedIds: []string{id1, id2, id3}, - }, - { - name: "contains all colors with text array", - where: filters.Where(). - WithPath([]string{"colors"}). - WithOperator(filters.ContainsAll). - WithValueText("red", "blue", "green"), - property: "colors", - expectedIds: []string{id1}, - }, - { - name: "contains any colors with text array", - where: filters.Where(). - WithPath([]string{"colors"}). - WithOperator(filters.ContainsAny). - WithValueText("red", "blue", "green"), - property: "colors", - expectedIds: []string{id1, id2, id3}, - }, - { - name: "contains all numbers with number array", - where: filters.Where(). - WithPath([]string{"numbers"}). - WithOperator(filters.ContainsAll). - WithValueNumber(1.1, 2.2, 3.3), - property: "numbers", - expectedIds: []string{id1}, - }, - { - name: "contains any numbers with number array", - where: filters.Where(). - WithPath([]string{"numbers"}). - WithOperator(filters.ContainsAny). - WithValueNumber(1.1, 2.2, 3.3), - property: "numbers", - expectedIds: []string{id1, id2, id3}, - }, - { - name: "contains all ints with int array", - where: filters.Where(). - WithPath([]string{"ints"}). - WithOperator(filters.ContainsAll). - WithValueInt(1, 2, 3), - property: "ints", - expectedIds: []string{id1}, - }, - { - name: "contains any ints with int array", - where: filters.Where(). - WithPath([]string{"ints"}). - WithOperator(filters.ContainsAny). - WithValueInt(1, 2, 3), - property: "ints", - expectedIds: []string{id1, id2, id3}, - }, - { - name: "contains all bools with bool array", - where: filters.Where(). - WithPath([]string{"bools"}). - WithOperator(filters.ContainsAll). - WithValueBoolean(true, false, true), - property: "bools", - expectedIds: []string{id1, id2}, - }, - { - name: "contains any bools with bool array", - where: filters.Where(). - WithPath([]string{"bools"}). - WithOperator(filters.ContainsAny). - WithValueBoolean(true, false), - property: "bools", - expectedIds: []string{id1, id2, id3}, - }, - { - name: "contains all uuids with uuid array", - where: filters.Where(). - WithPath([]string{"uuids"}). - WithOperator(filters.ContainsAll). - WithValueText(id1, id2, id3), - property: "uuids", - expectedIds: []string{id1}, - }, - { - name: "contains any uuids with uuid array", - where: filters.Where(). - WithPath([]string{"uuids"}). - WithOperator(filters.ContainsAny). - WithValueText(id1, id2, id3), - property: "uuids", - expectedIds: []string{id1, id2, id3}, - }, - { - name: "contains all dates with dates array", - where: filters.Where(). - WithPath([]string{"dates"}). - WithOperator(filters.ContainsAll). - WithValueDate( - mustGetTime("2009-11-01T23:00:00Z"), mustGetTime("2009-11-02T23:00:00Z"), mustGetTime("2009-11-03T23:00:00Z"), - ), - property: "dates", - expectedIds: []string{id1}, - }, - { - name: "contains any dates with dates array", - where: filters.Where(). - WithPath([]string{"dates"}). - WithOperator(filters.ContainsAny). - WithValueDate( - mustGetTime("2009-11-01T23:00:00Z"), mustGetTime("2009-11-02T23:00:00Z"), mustGetTime("2009-11-03T23:00:00Z"), - ), - property: "dates", - expectedIds: []string{id1, id2, id3}, - }, - { - name: "complex contains all ints and all numbers with AND on int array", - where: filters.Where(). - WithOperator(filters.And). - WithOperands([]*filters.WhereBuilder{ - filters.Where(). - WithPath([]string{"numbers"}). - WithOperator(filters.ContainsAll). - WithValueNumber(1.1, 2.2, 3.3), - filters.Where(). - WithPath([]string{"ints"}). - WithOperator(filters.ContainsAll). - WithValueInt(1, 2, 3), - }), - property: "ints", - expectedIds: []string{id1}, - }, - { - name: "complex contains any ints and all numbers with OR on int array", - where: filters.Where(). - WithOperator(filters.Or). - WithOperands([]*filters.WhereBuilder{ - filters.Where(). - WithPath([]string{"numbers"}). - WithOperator(filters.ContainsAll). - WithValueNumber(1.1, 2.2, 3.3), - filters.Where(). - WithPath([]string{"ints"}). - WithOperator(filters.ContainsAny). - WithValueInt(1, 2, 3), - }), - property: "ints", - expectedIds: []string{id1, id2, id3}, - }, - // Contains operator with primitives - { - name: "contains any author with string", - where: filters.Where(). - WithPath([]string{"author"}). - WithOperator(filters.ContainsAny). - WithValueString("John", "Jenny", "Joseph"), - property: "author", - expectedIds: []string{id1, id2, id3}, - }, - { - name: "contains any color with text", - where: filters.Where(). - WithPath([]string{"color"}). - WithOperator(filters.ContainsAny). - WithValueText("red", "blue", "green"), - property: "color", - expectedIds: []string{id1, id2, id3}, - }, - { - name: "contains any number with number", - where: filters.Where(). - WithPath([]string{"number"}). - WithOperator(filters.ContainsAny). - WithValueNumber(1.1, 2.2, 3.3), - property: "number", - expectedIds: []string{id1, id2, id3}, - }, - { - name: "contains any int with int", - where: filters.Where(). - WithPath([]string{"int"}). - WithOperator(filters.ContainsAny). - WithValueInt(1, 2, 3), - property: "int", - expectedIds: []string{id1, id2, id3}, - }, - { - name: "contains any bool with bool", - where: filters.Where(). - WithPath([]string{"bool"}). - WithOperator(filters.ContainsAny). - WithValueBoolean(true, false, true), - property: "bool", - expectedIds: []string{id1, id2, id3}, - }, - { - name: "contains any uuid with uuid", - where: filters.Where(). - WithPath([]string{"uuid"}). - WithOperator(filters.ContainsAny). - WithValueText(id1, id2, id3), - property: "uuid", - expectedIds: []string{id1, id2, id3}, - }, - { - name: "contains any uuid with id", - where: filters.Where(). - WithPath([]string{"id"}). - WithOperator(filters.ContainsAny). - WithValueText(id1, id2, id3), - property: "uuid", - expectedIds: []string{id1, id2, id3}, - }, - { - name: "contains any date with date", - where: filters.Where(). - WithPath([]string{"date"}). - WithOperator(filters.ContainsAny). - WithValueDate( - mustGetTime("2009-11-01T23:00:00Z"), mustGetTime("2009-11-02T23:00:00Z"), mustGetTime("2009-11-03T23:00:00Z"), - ), - property: "date", - expectedIds: []string{id1, id2, id3}, - }, - { - name: "contains all authors with string array and nearText", - where: filters.Where(). - WithPath([]string{"authors"}). - WithOperator(filters.ContainsAll). - WithValueString("John", "Jenny", "Joseph"), - property: "authors", - nearText: client.GraphQL().NearTextArgBuilder(). - WithConcepts([]string{"John"}), - expectedIds: []string{id1}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - fields := []graphql.Field{ - {Name: tt.property}, - {Name: "_additional", Fields: []graphql.Field{{Name: "id"}}}, - } - resp, err := client.GraphQL().Get(). - WithClassName(className). - WithWhere(tt.where). - WithFields(fields...). - WithNearText(tt.nearText). - Do(context.TODO()) - require.Nil(t, err) - resultIds := acceptance_with_go_client.GetIds(t, resp, className) - assert.Len(t, resultIds, len(tt.expectedIds)) - assert.ElementsMatch(t, resultIds, tt.expectedIds) - }) - } - }) - }) - } -} diff --git a/test/acceptance_with_go_client/filters_tests/contains_text_test.go b/test/acceptance_with_go_client/filters_tests/contains_text_test.go deleted file mode 100644 index c473fb9897fb5636ce2ead82a4f4eece548e30f3..0000000000000000000000000000000000000000 --- a/test/acceptance_with_go_client/filters_tests/contains_text_test.go +++ /dev/null @@ -1,372 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package filters_tests - -import ( - "context" - "fmt" - "testing" - - acceptance_with_go_client "acceptance_tests_with_client" - - "github.com/go-openapi/strfmt" - "github.com/stretchr/testify/require" - wvt "github.com/weaviate/weaviate-go-client/v4/weaviate" - "github.com/weaviate/weaviate-go-client/v4/weaviate/filters" - "github.com/weaviate/weaviate-go-client/v4/weaviate/graphql" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" -) - -func testContainsText(t *testing.T, host string) func(t *testing.T) { - return func(t *testing.T) { - client, err := wvt.NewClient(wvt.Config{Scheme: "http", Host: host}) - require.NoError(t, err) - - defer func() { - err := client.Schema().AllDeleter().Do(context.Background()) - require.NoError(t, err) - }() - - ctx := context.Background() - className := "ContainsText" - id := "be6452f4-5db6-4a41-bfef-ff5dffd4ab16" - texts := []string{ - " Hello You*-beautiful_world?!", - "HoW yOU_DOin? ", - } - - t.Run("init data", func(t *testing.T) { - class := &models.Class{ - Class: className, - Properties: []*models.Property{ - { - Name: "textField", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationField, - }, - { - Name: "textWhitespace", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "textLowercase", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationLowercase, - }, - { - Name: "textWord", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWord, - }, - - { - Name: "textsField", - DataType: schema.DataTypeTextArray.PropString(), - Tokenization: models.PropertyTokenizationField, - }, - { - Name: "textsWhitespace", - DataType: schema.DataTypeTextArray.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "textsLowercase", - DataType: schema.DataTypeTextArray.PropString(), - Tokenization: models.PropertyTokenizationLowercase, - }, - { - Name: "textsWord", - DataType: schema.DataTypeTextArray.PropString(), - Tokenization: models.PropertyTokenizationWord, - }, - }, - } - - err := client.Schema().ClassCreator(). - WithClass(class). - Do(ctx) - require.NoError(t, err) - - wrap, err := client.Data().Creator(). - WithClassName(className). - WithID(id). - WithProperties(map[string]interface{}{ - "textField": texts[0], - "textWhitespace": texts[0], - "textLowercase": texts[0], - "textWord": texts[0], - "textsField": texts, - "textsWhitespace": texts, - "textsLowercase": texts, - "textsWord": texts, - }). - Do(ctx) - require.NoError(t, err) - require.NotNil(t, wrap) - require.NotNil(t, wrap.Object) - require.Equal(t, strfmt.UUID(id), wrap.Object.ID) - }) - - t.Run("search using contains", func(t *testing.T) { - type testCase struct { - propName string - operator filters.WhereOperator - values []string - expectedFound bool - } - - testCases := []testCase{} - testCases = append(testCases, - testCase{ - propName: "textField", - operator: filters.ContainsAny, - values: []string{"Hello You*-beautiful_world?!", "HoW yOU_DOin?"}, - expectedFound: true, - }, - testCase{ - propName: "textField", - operator: filters.ContainsAll, - values: []string{"Hello You*-beautiful_world?!", "HoW yOU_DOin?"}, - expectedFound: false, - }, - testCase{ - propName: "textsField", - operator: filters.ContainsAny, - values: []string{"Hello You*-beautiful_world?!", "HoW yOU_DOin?"}, - expectedFound: true, - }, - testCase{ - propName: "textsField", - operator: filters.ContainsAll, - values: []string{"Hello You*-beautiful_world?!", "HoW yOU_DOin?"}, - expectedFound: true, - }, - - testCase{ - propName: "textWord", - operator: filters.ContainsAny, - values: []string{"HELLO", "doin"}, - expectedFound: true, - }, - testCase{ - propName: "textWord", - operator: filters.ContainsAll, - values: []string{"HELLO", "doin"}, - expectedFound: false, - }, - testCase{ - propName: "textsWord", - operator: filters.ContainsAny, - values: []string{"HELLO", "doin"}, - expectedFound: true, - }, - testCase{ - propName: "textsWord", - operator: filters.ContainsAll, - values: []string{"HELLO", "doin"}, - expectedFound: true, - }, - - testCase{ - propName: "textField", - operator: filters.ContainsAny, - values: []string{"Hello", "HoW"}, - expectedFound: false, - }, - testCase{ - propName: "textField", - operator: filters.ContainsAll, - values: []string{"Hello", "HoW"}, - expectedFound: false, - }, - testCase{ - propName: "textsField", - operator: filters.ContainsAny, - values: []string{"Hello", "HoW"}, - expectedFound: false, - }, - testCase{ - propName: "textsField", - operator: filters.ContainsAll, - values: []string{"Hello", "HoW"}, - expectedFound: false, - }, - testCase{ - propName: "textWhitespace", - operator: filters.ContainsAny, - values: []string{"Hello", "HoW"}, - expectedFound: true, - }, - testCase{ - propName: "textWhitespace", - operator: filters.ContainsAll, - values: []string{"Hello", "HoW"}, - expectedFound: false, - }, - testCase{ - propName: "textsWhitespace", - operator: filters.ContainsAny, - values: []string{"Hello", "HoW"}, - expectedFound: true, - }, - testCase{ - propName: "textsWhitespace", - operator: filters.ContainsAll, - values: []string{"Hello", "HoW"}, - expectedFound: true, - }, - testCase{ - propName: "textLowercase", - operator: filters.ContainsAny, - values: []string{"Hello", "HoW"}, - expectedFound: true, - }, - testCase{ - propName: "textLowercase", - operator: filters.ContainsAll, - values: []string{"Hello", "HoW"}, - expectedFound: false, - }, - testCase{ - propName: "textsLowercase", - operator: filters.ContainsAny, - values: []string{"Hello", "HoW"}, - expectedFound: true, - }, - testCase{ - propName: "textsLowercase", - operator: filters.ContainsAll, - values: []string{"Hello", "HoW"}, - expectedFound: true, - }, - testCase{ - propName: "textWord", - operator: filters.ContainsAny, - values: []string{"Hello", "HoW"}, - expectedFound: true, - }, - testCase{ - propName: "textWord", - operator: filters.ContainsAll, - values: []string{"Hello", "HoW"}, - expectedFound: false, - }, - testCase{ - propName: "textsWord", - operator: filters.ContainsAny, - values: []string{"Hello", "HoW"}, - expectedFound: true, - }, - testCase{ - propName: "textsWord", - operator: filters.ContainsAll, - values: []string{"Hello", "HoW"}, - expectedFound: true, - }, - ) - - for _, propName := range []string{"textField", "textsField"} { - testCases = append(testCases, - testCase{ - propName: propName, - operator: filters.ContainsAny, - values: []string{"hello", "world"}, - expectedFound: false, - }, - testCase{ - propName: propName, - operator: filters.ContainsAll, - values: []string{"hello", "world"}, - expectedFound: false, - }, - ) - } - for _, propName := range []string{"textWhitespace", "textsWhitespace"} { - testCases = append(testCases, - testCase{ - propName: propName, - operator: filters.ContainsAny, - values: []string{"hello", "world"}, - expectedFound: false, - }, - testCase{ - propName: propName, - operator: filters.ContainsAll, - values: []string{"hello", "world"}, - expectedFound: false, - }, - ) - } - for _, propName := range []string{"textLowercase", "textsLowercase"} { - testCases = append(testCases, - testCase{ - propName: propName, - operator: filters.ContainsAny, - values: []string{"hello", "world"}, - expectedFound: true, - }, - testCase{ - propName: propName, - operator: filters.ContainsAll, - values: []string{"hello", "world"}, - expectedFound: false, - }, - ) - } - for _, propName := range []string{"textWord", "textsWord"} { - testCases = append(testCases, - testCase{ - propName: propName, - operator: filters.ContainsAny, - values: []string{"hello", "world"}, - expectedFound: true, - }, - testCase{ - propName: propName, - operator: filters.ContainsAll, - values: []string{"hello", "world"}, - expectedFound: true, - }, - ) - } - - for _, tc := range testCases { - t.Run(fmt.Sprintf("%+v", tc), func(t *testing.T) { - where := filters.Where(). - WithPath([]string{tc.propName}). - WithOperator(tc.operator). - WithValueText(tc.values...) - field := graphql.Field{ - Name: "_additional", - Fields: []graphql.Field{{Name: "id"}}, - } - - resp, err := client.GraphQL().Get(). - WithClassName(className). - WithWhere(where). - WithFields(field). - Do(ctx) - require.NoError(t, err) - - ids := acceptance_with_go_client.GetIds(t, resp, className) - if tc.expectedFound { - require.ElementsMatch(t, ids, []string{id}) - } else { - require.Empty(t, ids) - } - }) - } - }) - } -} diff --git a/test/acceptance_with_go_client/filters_tests/where_test.go b/test/acceptance_with_go_client/filters_tests/where_test.go deleted file mode 100644 index 4f866878c29423717c04b0aee998baa60e12e63c..0000000000000000000000000000000000000000 --- a/test/acceptance_with_go_client/filters_tests/where_test.go +++ /dev/null @@ -1,42 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package filters_tests - -import ( - "context" - "testing" - - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/test/docker" -) - -func TestWhereFilter(t *testing.T) { - t.Run("ContainsAny / ContainsAll", testContainsAnyAll(t, "localhost:8080")) - t.Run("Contains Text", testContainsText(t, "localhost:8080")) -} - -func TestWhereFilter_Cluster(t *testing.T) { - ctx := context.Background() - compose, err := docker.New(). - WithWeaviateCluster(). - WithText2VecContextionary(). - Start(ctx) - require.NoError(t, err) - defer func() { - require.NoError(t, compose.Terminate(ctx)) - }() - - endpoint := compose.GetWeaviate().URI() - - t.Run("ContainsAny / ContainsAll", testContainsAnyAll(t, endpoint)) - t.Run("Contains Text", testContainsText(t, endpoint)) -} diff --git a/test/acceptance_with_go_client/fixtures/allproperties.go b/test/acceptance_with_go_client/fixtures/allproperties.go deleted file mode 100644 index aaeaaab43cef7c666665c171cb13784aa36d0723..0000000000000000000000000000000000000000 --- a/test/acceptance_with_go_client/fixtures/allproperties.go +++ /dev/null @@ -1,186 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package fixtures - -import ( - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" -) - -var ( - AllPropertiesClassName = "AllProperties" - AllPropertiesID1 = "00000000-0000-0000-0000-000000000001" - vTrue = true - vFalse = false -) - -var AllPropertiesClass = &models.Class{ - Class: AllPropertiesClassName, - Properties: []*models.Property{ - { - Name: "objectProperty", - DataType: schema.DataTypeObject.PropString(), - IndexFilterable: &vTrue, - IndexSearchable: &vFalse, - Tokenization: "", - NestedProperties: []*models.NestedProperty{ - { - Name: "text", - DataType: schema.DataTypeText.PropString(), - }, - { - Name: "texts", - DataType: schema.DataTypeTextArray.PropString(), - }, - { - Name: "number", - DataType: schema.DataTypeNumber.PropString(), - }, - { - Name: "numbers", - DataType: schema.DataTypeNumberArray.PropString(), - }, - { - Name: "int", - DataType: schema.DataTypeInt.PropString(), - }, - { - Name: "ints", - DataType: schema.DataTypeIntArray.PropString(), - }, - { - Name: "date", - DataType: schema.DataTypeDate.PropString(), - }, - { - Name: "dates", - DataType: schema.DataTypeDateArray.PropString(), - }, - { - Name: "bool", - DataType: schema.DataTypeBoolean.PropString(), - }, - { - Name: "bools", - DataType: schema.DataTypeBooleanArray.PropString(), - }, - { - Name: "uuid", - DataType: schema.DataTypeUUID.PropString(), - }, - { - Name: "uuids", - DataType: schema.DataTypeUUIDArray.PropString(), - }, - { - Name: "nested_int", - DataType: schema.DataTypeInt.PropString(), - IndexFilterable: &vTrue, - IndexSearchable: &vFalse, - Tokenization: "", - }, - { - Name: "nested_number", - DataType: schema.DataTypeNumber.PropString(), - IndexFilterable: &vTrue, - IndexSearchable: &vFalse, - Tokenization: "", - }, - { - Name: "nested_text", - DataType: schema.DataTypeText.PropString(), - IndexFilterable: &vTrue, - IndexSearchable: &vTrue, - Tokenization: models.PropertyTokenizationWord, - }, - { - Name: "nested_objects", - DataType: schema.DataTypeObject.PropString(), - IndexFilterable: &vTrue, - IndexSearchable: &vFalse, - Tokenization: "", - NestedProperties: []*models.NestedProperty{ - { - Name: "nested_bool_lvl2", - DataType: schema.DataTypeBoolean.PropString(), - IndexFilterable: &vTrue, - IndexSearchable: &vFalse, - Tokenization: "", - }, - { - Name: "nested_numbers_lvl2", - DataType: schema.DataTypeNumberArray.PropString(), - IndexFilterable: &vTrue, - IndexSearchable: &vFalse, - Tokenization: "", - }, - }, - }, - { - Name: "nested_array_objects", - DataType: schema.DataTypeObjectArray.PropString(), - IndexFilterable: &vTrue, - IndexSearchable: &vFalse, - Tokenization: "", - NestedProperties: []*models.NestedProperty{ - { - Name: "nested_bool_lvl2", - DataType: schema.DataTypeBoolean.PropString(), - IndexFilterable: &vTrue, - IndexSearchable: &vFalse, - Tokenization: "", - }, - { - Name: "nested_numbers_lvl2", - DataType: schema.DataTypeNumberArray.PropString(), - IndexFilterable: &vTrue, - IndexSearchable: &vFalse, - Tokenization: "", - }, - }, - }, - }, - }, - }, -} - -var AllPropertiesProperties = map[string]interface{}{ - "text": "red", - "texts": []string{"red", "blue"}, - "number": float64(1.1), - "numbers": []float64{1.1, 2.2}, - "int": int64(1), - "ints": []int64{1, 2}, - "uuid": AllPropertiesID1, - "uuids": []string{AllPropertiesID1}, - "date": "2009-11-01T23:00:00Z", - "dates": []string{"2009-11-01T23:00:00Z"}, - "bool": true, - "bools": []bool{true, false}, - "nested_int": int64(11), - "nested_number": float64(11.11), - "nested_text": "nested text", - "nested_objects": map[string]interface{}{ - "nested_bool_lvl2": true, - "nested_numbers_lvl2": []float64{11.1, 22.1}, - }, - "nested_array_objects": []interface{}{ - map[string]interface{}{ - "nested_bool_lvl2": true, - "nested_numbers_lvl2": []float64{111.1, 222.1}, - }, - map[string]interface{}{ - "nested_bool_lvl2": false, - "nested_numbers_lvl2": []float64{112.1, 222.1}, - }, - }, -} diff --git a/test/acceptance_with_go_client/fixtures/food.go b/test/acceptance_with_go_client/fixtures/food.go deleted file mode 100644 index 4d0353687b8b85795e660c1767c1d902c1787abd..0000000000000000000000000000000000000000 --- a/test/acceptance_with_go_client/fixtures/food.go +++ /dev/null @@ -1,530 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package fixtures - -import ( - "context" - "fmt" - "testing" - - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate-go-client/v4/weaviate" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" -) - -const ( - PIZZA_QUATTRO_FORMAGGI_ID = "10523cdd-15a2-42f4-81fa-267fe92f7cd6" - PIZZA_FRUTTI_DI_MARE_ID = "927dd3ac-e012-4093-8007-7799cc7e81e4" - PIZZA_HAWAII_ID = "00000000-0000-0000-0000-000000000000" - PIZZA_DOENER_ID = "5b6a08ba-1d46-43aa-89cc-8b070790c6f2" - SOUP_CHICKENSOUP_ID = "8c156d37-81aa-4ce9-a811-621e2702b825" - SOUP_BEAUTIFUL_ID = "27351361-2898-4d1a-aad7-1ca48253eb0b" - SOUP_TRIPE_ID = "a0b0c0d0-2898-4d1a-aad7-1ca48253eb0b" - RISOTTO_RISI_E_BISI_ID = "da751a25-f573-4715-a893-e607b2de0ba4" - RISOTTO_ALLA_PILOTA_ID = "10c2ee44-7d58-42be-9d64-5766883ca8cb" - RISOTTO_AL_NERO_DI_SEPPIA = "696bf381-7f98-40a4-bcad-841780e00e0e" -) - -var IdsByClass = map[string][]string{ - "Pizza": { - PIZZA_QUATTRO_FORMAGGI_ID, - PIZZA_FRUTTI_DI_MARE_ID, - PIZZA_HAWAII_ID, - PIZZA_DOENER_ID, - }, - "Soup": { - SOUP_CHICKENSOUP_ID, - SOUP_BEAUTIFUL_ID, - }, - "Risotto": { - RISOTTO_RISI_E_BISI_ID, - RISOTTO_ALLA_PILOTA_ID, - RISOTTO_AL_NERO_DI_SEPPIA, - }, -} - -var BeaconsByClass = map[string][]map[string]string{ - "Pizza": { - {"beacon": makeBeacon("Pizza", PIZZA_QUATTRO_FORMAGGI_ID)}, - {"beacon": makeBeacon("Pizza", PIZZA_FRUTTI_DI_MARE_ID)}, - {"beacon": makeBeacon("Pizza", PIZZA_HAWAII_ID)}, - {"beacon": makeBeacon("Pizza", PIZZA_DOENER_ID)}, - }, - "Soup": { - {"beacon": makeBeacon("Soup", SOUP_CHICKENSOUP_ID)}, - {"beacon": makeBeacon("Soup", SOUP_BEAUTIFUL_ID)}, - }, - "Risotto": { - {"beacon": makeBeacon("Risotto", RISOTTO_RISI_E_BISI_ID)}, - {"beacon": makeBeacon("Risotto", RISOTTO_ALLA_PILOTA_ID)}, - {"beacon": makeBeacon("Risotto", RISOTTO_AL_NERO_DI_SEPPIA)}, - }, -} - -var AllIds = []string{ - PIZZA_QUATTRO_FORMAGGI_ID, - PIZZA_FRUTTI_DI_MARE_ID, - PIZZA_HAWAII_ID, - PIZZA_DOENER_ID, - - SOUP_CHICKENSOUP_ID, - SOUP_BEAUTIFUL_ID, - - RISOTTO_RISI_E_BISI_ID, - RISOTTO_ALLA_PILOTA_ID, - RISOTTO_AL_NERO_DI_SEPPIA, -} - -func makeBeacon(class, id string) string { - return fmt.Sprintf("weaviate://localhost/%s/%s", class, id) -} - -// ##### SCHEMA ##### - -func CreateSchemaPizza(t *testing.T, client *weaviate.Client) { - createSchema(t, client, classPizza()) -} - -func CreateSchemaSoup(t *testing.T, client *weaviate.Client) { - createSchema(t, client, classSoup()) -} - -func CreateSchemaRisotto(t *testing.T, client *weaviate.Client) { - createSchema(t, client, classRisotto()) -} - -func CreateSchemaFood(t *testing.T, client *weaviate.Client) { - CreateSchemaPizza(t, client) - CreateSchemaSoup(t, client) - CreateSchemaRisotto(t, client) -} - -func CreateSchemaPizzaForTenants(t *testing.T, client *weaviate.Client) { - createSchema(t, client, classForTenants(classPizza())) -} - -func CreateSchemaSoupForTenants(t *testing.T, client *weaviate.Client) { - createSchema(t, client, classForTenants(classSoup())) -} - -func CreateSchemaRisottoForTenants(t *testing.T, client *weaviate.Client) { - createSchema(t, client, classForTenants(classRisotto())) -} - -func CreateSchemaFoodForTenants(t *testing.T, client *weaviate.Client) { - CreateSchemaPizzaForTenants(t, client) - CreateSchemaSoupForTenants(t, client) - CreateSchemaRisottoForTenants(t, client) -} - -func createSchema(t *testing.T, client *weaviate.Client, class *models.Class) { - err := client.Schema().ClassCreator(). - WithClass(class). - Do(context.Background()) - - require.Nil(t, err) -} - -// ##### CLASSES ##### - -func classPizza() *models.Class { - return &models.Class{ - Class: "Pizza", - Description: "A delicious religion like food and arguably the best export of Italy.", - InvertedIndexConfig: &models.InvertedIndexConfig{IndexTimestamps: true}, - Properties: classPropertiesFood(), - } -} - -func classSoup() *models.Class { - return &models.Class{ - Class: "Soup", - Description: "Mostly water based brew of sustenance for humans.", - Properties: classPropertiesFood(), - } -} - -func classRisotto() *models.Class { - return &models.Class{ - Class: "Risotto", - Description: "Risotto is a northern Italian rice dish cooked with broth.", - InvertedIndexConfig: &models.InvertedIndexConfig{IndexTimestamps: true}, - Properties: classPropertiesFood(), - } -} - -func classForTenants(class *models.Class) *models.Class { - class.MultiTenancyConfig = &models.MultiTenancyConfig{ - Enabled: true, - } - return class -} - -func classPropertiesFood() []*models.Property { - nameProperty := &models.Property{ - Name: "name", - Description: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationField, - } - descriptionProperty := &models.Property{ - Name: "description", - Description: "description", - DataType: schema.DataTypeText.PropString(), - } - bestBeforeProperty := &models.Property{ - Name: "best_before", - Description: "You better eat this food before it expires", - DataType: schema.DataTypeDate.PropString(), - } - priceProperty := &models.Property{ - Name: "price", - Description: "price", - DataType: schema.DataTypeNumber.PropString(), - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "skip": true, - }, - }, - } - - return []*models.Property{ - nameProperty, descriptionProperty, bestBeforeProperty, priceProperty, - } -} - -// ##### DATA ##### - -func CreateDataPizza(t *testing.T, client *weaviate.Client) { - createData(t, client, []*models.Object{ - objectPizzaQuattroFormaggi(), - objectPizzaFruttiDiMare(), - objectPizzaHawaii(), - objectPizzaDoener(), - }) -} - -func CreateDataSoup(t *testing.T, client *weaviate.Client) { - createData(t, client, []*models.Object{ - objectSoupChicken(), - objectSoupBeautiful(), - }) -} - -func CreateDataRisotto(t *testing.T, client *weaviate.Client) { - createData(t, client, []*models.Object{ - objectRisottoRisiEBisi(), - objectRisottoAllaPilota(), - objectRisottoAlNeroDiSeppia(), - }) -} - -func CreateDataFood(t *testing.T, client *weaviate.Client) { - createData(t, client, []*models.Object{ - objectPizzaQuattroFormaggi(), - objectPizzaFruttiDiMare(), - objectPizzaHawaii(), - objectPizzaDoener(), - - objectSoupChicken(), - objectSoupBeautiful(), - - objectRisottoRisiEBisi(), - objectRisottoAllaPilota(), - objectRisottoAlNeroDiSeppia(), - }) -} - -func createData(t *testing.T, client *weaviate.Client, objects []*models.Object) { - resp, err := client.Batch().ObjectsBatcher(). - WithObjects(objects...). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, resp) - require.Len(t, resp, len(objects)) -} - -func CreateDataPizzaQuattroFormaggiForTenants(t *testing.T, client *weaviate.Client, tenantNames ...string) { - createDataForTenants(t, client, tenantNames, func() []*models.Object { - return []*models.Object{ - objectPizzaQuattroFormaggi(), - } - }) -} - -func CreateDataPizzaFruttiDiMareForTenants(t *testing.T, client *weaviate.Client, tenantNames ...string) { - createDataForTenants(t, client, tenantNames, func() []*models.Object { - return []*models.Object{ - objectPizzaFruttiDiMare(), - } - }) -} - -func CreateDataPizzaHawaiiForTenants(t *testing.T, client *weaviate.Client, tenantNames ...string) { - createDataForTenants(t, client, tenantNames, func() []*models.Object { - return []*models.Object{ - objectPizzaHawaii(), - } - }) -} - -func CreateDataPizzaDoenerForTenants(t *testing.T, client *weaviate.Client, tenantNames ...string) { - createDataForTenants(t, client, tenantNames, func() []*models.Object { - return []*models.Object{ - objectPizzaDoener(), - } - }) -} - -func CreateDataPizzaForTenants(t *testing.T, client *weaviate.Client, tenantNames ...string) { - createDataForTenants(t, client, tenantNames, func() []*models.Object { - return []*models.Object{ - objectPizzaQuattroFormaggi(), - objectPizzaFruttiDiMare(), - objectPizzaHawaii(), - objectPizzaDoener(), - } - }) -} - -func CreateDataSoupChickenForTenants(t *testing.T, client *weaviate.Client, tenantNames ...string) { - createDataForTenants(t, client, tenantNames, func() []*models.Object { - return []*models.Object{ - objectSoupChicken(), - } - }) -} - -func CreateDataSoupBeautifulForTenants(t *testing.T, client *weaviate.Client, tenantNames ...string) { - createDataForTenants(t, client, tenantNames, func() []*models.Object { - return []*models.Object{ - objectSoupBeautiful(), - } - }) -} - -func CreateDataSoupForTenants(t *testing.T, client *weaviate.Client, tenantNames ...string) { - createDataForTenants(t, client, tenantNames, func() []*models.Object { - return []*models.Object{ - objectSoupChicken(), - objectSoupBeautiful(), - } - }) -} - -func CreateDataRisottoForTenants(t *testing.T, client *weaviate.Client, tenantNames ...string) { - createDataForTenants(t, client, tenantNames, func() []*models.Object { - return []*models.Object{ - objectRisottoRisiEBisi(), - objectRisottoAllaPilota(), - objectRisottoAlNeroDiSeppia(), - } - }) -} - -func CreateDataFoodForTenants(t *testing.T, client *weaviate.Client, tenantNames ...string) { - createDataForTenants(t, client, tenantNames, func() []*models.Object { - return []*models.Object{ - objectPizzaQuattroFormaggi(), - objectPizzaFruttiDiMare(), - objectPizzaHawaii(), - objectPizzaDoener(), - - objectSoupChicken(), - objectSoupBeautiful(), - - objectRisottoRisiEBisi(), - objectRisottoAllaPilota(), - objectRisottoAlNeroDiSeppia(), - } - }) -} - -func createDataForTenants(t *testing.T, client *weaviate.Client, - tenantNames []string, objectsSupplier func() []*models.Object, -) { - objects := []*models.Object{} - for _, name := range tenantNames { - for _, object := range objectsSupplier() { - object.Tenant = name - objects = append(objects, object) - } - } - createData(t, client, objects) -} - -// ##### OBJECTS ##### - -func objectPizzaQuattroFormaggi() *models.Object { - return &models.Object{ - Class: "Pizza", - ID: PIZZA_QUATTRO_FORMAGGI_ID, - Properties: map[string]interface{}{ - "name": "Quattro Formaggi", - "description": "Pizza quattro formaggi Italian: [ˈkwattro forˈmaddʒi] (four cheese pizza) is a variety of pizza in Italian cuisine that is topped with a combination of four kinds of cheese, usually melted together, with (rossa, red) or without (bianca, white) tomato sauce. It is popular worldwide, including in Italy,[1] and is one of the iconic items from pizzerias's menus.", - "price": float32(1.1), - "best_before": "2022-05-03T12:04:40+02:00", - }, - } -} - -func objectPizzaFruttiDiMare() *models.Object { - return &models.Object{ - Class: "Pizza", - ID: PIZZA_FRUTTI_DI_MARE_ID, - Properties: map[string]interface{}{ - "name": "Frutti di Mare", - "description": "Frutti di Mare is an Italian type of pizza that may be served with scampi, mussels or squid. It typically lacks cheese, with the seafood being served atop a tomato sauce.", - "price": float32(1.2), - "best_before": "2022-05-05T07:16:30+02:00", - }, - } -} - -func objectPizzaHawaii() *models.Object { - return &models.Object{ - Class: "Pizza", - // this uuid guarantees that it's the first for cursor tests (otherwise - // they might be flaky if the randomly generated ids are sometimes higher - // and sometimes lower - ID: PIZZA_HAWAII_ID, - Properties: map[string]interface{}{ - "name": "Hawaii", - "description": "Universally accepted to be the best pizza ever created.", - "price": float32(1.3), - }, - } -} - -func objectPizzaDoener() *models.Object { - return &models.Object{ - Class: "Pizza", - ID: PIZZA_DOENER_ID, - Properties: map[string]interface{}{ - "name": "Doener", - "description": "A innovation, some say revolution, in the pizza industry.", - "price": float32(1.4), - }, - } -} - -func objectSoupChicken() *models.Object { - return &models.Object{ - Class: "Soup", - ID: SOUP_CHICKENSOUP_ID, - Properties: map[string]interface{}{ - "name": "ChickenSoup", - "description": "Used by humans when their inferior genetics are attacked by microscopic organisms.", - "price": float32(2.1), - }, - } -} - -func objectSoupBeautiful() *models.Object { - return &models.Object{ - Class: "Soup", - ID: SOUP_BEAUTIFUL_ID, - Properties: map[string]interface{}{ - "name": "Beautiful", - "description": "Putting the game of letter soups to a whole new level.", - "price": float32(2.2), - }, - } -} - -func objectRisottoRisiEBisi() *models.Object { - return &models.Object{ - Class: "Risotto", - ID: RISOTTO_RISI_E_BISI_ID, - Properties: map[string]interface{}{ - "name": "Risi e bisi", - "description": "A Veneto spring dish.", - "price": float32(3.1), - "best_before": "2022-05-03T12:04:40+02:00", - }, - } -} - -func objectRisottoAllaPilota() *models.Object { - return &models.Object{ - Class: "Risotto", - ID: RISOTTO_ALLA_PILOTA_ID, - Properties: map[string]interface{}{ - "name": "Risotto alla pilota", - "description": "A specialty of Mantua, made with sausage, pork, and Parmesan cheese.", - "price": float32(3.2), - "best_before": "2022-05-03T12:04:40+02:00", - }, - } -} - -func objectRisottoAlNeroDiSeppia() *models.Object { - return &models.Object{ - Class: "Risotto", - ID: RISOTTO_AL_NERO_DI_SEPPIA, - Properties: map[string]interface{}{ - "name": "Risotto al nero di seppia", - "description": "A specialty of the Veneto region, made with cuttlefish cooked with their ink-sacs intact, leaving the risotto black .", - "price": float32(3.3), - }, - } -} - -// ##### TENANTS ##### - -func CreateTenantsPizza(t *testing.T, client *weaviate.Client, tenants ...models.Tenant) { - createTenants(t, client, "Pizza", tenants) -} - -func CreateTenantsSoup(t *testing.T, client *weaviate.Client, tenants ...models.Tenant) { - createTenants(t, client, "Soup", tenants) -} - -func CreateTenantsRisotto(t *testing.T, client *weaviate.Client, tenants ...models.Tenant) { - createTenants(t, client, "Risotto", tenants) -} - -func CreateTenantsFood(t *testing.T, client *weaviate.Client, tenants ...models.Tenant) { - CreateTenantsPizza(t, client, tenants...) - CreateTenantsSoup(t, client, tenants...) - CreateTenantsRisotto(t, client, tenants...) -} - -func createTenants(t *testing.T, client *weaviate.Client, className string, tenants []models.Tenant) { - err := client.Schema().TenantsCreator(). - WithClassName(className). - WithTenants(tenants...). - Do(context.Background()) - - require.Nil(t, err) -} - -type Tenants []models.Tenant - -func (t Tenants) Names() []string { - names := make([]string, len(t)) - for i, tenant := range t { - names[i] = tenant.Name - } - return names -} - -func (t Tenants) ByName(name string) *models.Tenant { - for _, tenant := range t { - if tenant.Name == name { - return &tenant - } - } - return nil -} diff --git a/test/acceptance_with_go_client/go.mod b/test/acceptance_with_go_client/go.mod deleted file mode 100644 index 17173c4ca3263b2b6be64868c184b1aed3d2615b..0000000000000000000000000000000000000000 --- a/test/acceptance_with_go_client/go.mod +++ /dev/null @@ -1,134 +0,0 @@ -module acceptance_tests_with_client - -go 1.21 - -replace github.com/weaviate/weaviate => ../.. - -require ( - github.com/go-openapi/strfmt v0.21.3 - github.com/google/uuid v1.3.1 - github.com/pkg/errors v0.9.1 - github.com/stretchr/testify v1.8.4 - github.com/weaviate/weaviate v1.23.0 - github.com/weaviate/weaviate-go-client/v4 v4.12.1 -) - -require ( - cloud.google.com/go v0.110.7 // indirect - cloud.google.com/go/compute v1.23.0 // indirect - cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v1.1.1 // indirect - cloud.google.com/go/storage v1.33.0 // indirect - dario.cat/mergo v1.0.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0 // indirect - github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect - github.com/Microsoft/go-winio v0.6.1 // indirect - github.com/Microsoft/hcsshim v0.11.4 // indirect - github.com/PuerkitoBio/purell v1.1.1 // indirect - github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect - github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da // indirect - github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect - github.com/beorn7/perks v1.0.1 // indirect - github.com/cenkalti/backoff/v4 v4.2.1 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/containerd/containerd v1.7.11 // indirect - github.com/containerd/log v0.1.0 // indirect - github.com/cpuguy83/dockercfg v0.3.1 // indirect - github.com/danaugrs/go-tsne v0.0.0-20200708172100-6b7d1d577fd3 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/dlclark/regexp2 v1.10.0 // indirect - github.com/docker/distribution v2.8.2+incompatible // indirect - github.com/docker/docker v24.0.7+incompatible // indirect - github.com/docker/go-connections v0.4.0 // indirect - github.com/docker/go-units v0.5.0 // indirect - github.com/dustin/go-humanize v1.0.1 // indirect - github.com/fatih/camelcase v1.0.0 // indirect - github.com/go-ole/go-ole v1.2.6 // indirect - github.com/go-openapi/analysis v0.21.2 // indirect - github.com/go-openapi/errors v0.20.3 // indirect - github.com/go-openapi/jsonpointer v0.19.5 // indirect - github.com/go-openapi/jsonreference v0.19.6 // indirect - github.com/go-openapi/loads v0.21.1 // indirect - github.com/go-openapi/spec v0.20.4 // indirect - github.com/go-openapi/swag v0.22.3 // indirect - github.com/go-openapi/validate v0.21.0 // indirect - github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.3 // indirect - github.com/google/btree v1.0.1 // indirect - github.com/google/s2a-go v0.1.4 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect - github.com/googleapis/gax-go/v2 v2.12.0 // indirect - github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/go-immutable-radix v1.0.0 // indirect - github.com/hashicorp/go-msgpack v0.5.3 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect - github.com/hashicorp/go-sockaddr v1.0.0 // indirect - github.com/hashicorp/golang-lru v0.5.1 // indirect - github.com/hashicorp/memberlist v0.5.0 // indirect - github.com/josharian/intern v1.0.0 // indirect - github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.16.7 // indirect - github.com/klauspost/cpuid/v2 v2.2.5 // indirect - github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect - github.com/magiconair/properties v1.8.7 // indirect - github.com/mailru/easyjson v0.7.7 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect - github.com/miekg/dns v1.1.26 // indirect - github.com/minio/md5-simd v1.1.2 // indirect - github.com/minio/minio-go/v7 v7.0.63 // indirect - github.com/minio/sha256-simd v1.0.1 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/moby/patternmatcher v0.6.0 // indirect - github.com/moby/sys/sequential v0.5.0 // indirect - github.com/moby/term v0.5.0 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/morikuni/aec v1.0.0 // indirect - github.com/oklog/ulid v1.3.1 // indirect - github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.0-rc5 // indirect - github.com/opencontainers/runc v1.1.10 // indirect - github.com/pkoukk/tiktoken-go v0.1.6 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect - github.com/prometheus/client_golang v1.17.0 // indirect - github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect - github.com/prometheus/common v0.44.0 // indirect - github.com/prometheus/procfs v0.11.1 // indirect - github.com/rs/xid v1.5.0 // indirect - github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect - github.com/shirou/gopsutil/v3 v3.23.9 // indirect - github.com/shoenig/go-m1cpu v0.1.6 // indirect - github.com/sirupsen/logrus v1.9.3 // indirect - github.com/tailor-inc/graphql v0.4.1 // indirect - github.com/testcontainers/testcontainers-go v0.26.0 // indirect - github.com/tklauser/go-sysconf v0.3.12 // indirect - github.com/tklauser/numcpus v0.6.1 // indirect - github.com/yusufpapurcu/wmi v1.2.3 // indirect - go.mongodb.org/mongo-driver v1.11.3 // indirect - go.opencensus.io v0.24.0 // indirect - golang.org/x/crypto v0.17.0 // indirect - golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea // indirect - golang.org/x/mod v0.11.0 // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/oauth2 v0.11.0 // indirect - golang.org/x/sync v0.3.0 // indirect - golang.org/x/sys v0.15.0 // indirect - golang.org/x/text v0.14.0 // indirect - golang.org/x/tools v0.10.0 // indirect - golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect - gonum.org/v1/gonum v0.12.0 // indirect - google.golang.org/api v0.132.0 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect - google.golang.org/grpc v1.59.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect - gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect -) diff --git a/test/acceptance_with_go_client/go.sum b/test/acceptance_with_go_client/go.sum deleted file mode 100644 index 1aa6fd83902171157b66474e986d6aa43f67c33d..0000000000000000000000000000000000000000 --- a/test/acceptance_with_go_client/go.sum +++ /dev/null @@ -1,579 +0,0 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.110.7 h1:rJyC7nWRg2jWGZ4wSJ5nY65GTdYJkg0cd/uXb+ACI6o= -cloud.google.com/go v0.110.7/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= -cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= -cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= -cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= -cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/iam v1.1.1 h1:lW7fzj15aVIXYHREOqjRBV9PsH0Z6u8Y46a1YGvQP4Y= -cloud.google.com/go/iam v1.1.1/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= -cloud.google.com/go/storage v1.33.0 h1:PVrDOkIC8qQVa1P3SXGpQvfuJhN2LHOoyZvWs8D2X5M= -cloud.google.com/go/storage v1.33.0/go.mod h1:Hhh/dogNRGca7IWv1RC2YqEn0c0G77ctA/OxflYkiD8= -dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= -dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= -github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= -github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0 h1:9kDVnTz3vbfweTqAUmk/a/pH5pWFCHtvRpHYC0G/dcA= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0/go.mod h1:3Ug6Qzto9anB6mGlEdgYMDF5zHQ+wwhEaYR4s17PHMw= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 h1:vcYCAze6p19qBW7MhZybIsqD8sMV8js0NyQM8JDnVtg= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0/go.mod h1:OQeznEEkTZ9OrhHJoDD8ZDq51FHgXjqtP9z6bEwBq9U= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0 h1:Ma67P/GGprNwsslzEH6+Kb8nybI8jpDTm4Wmzu2ReK8= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0/go.mod h1:c+Lifp3EDEamAkPVzMooRNOK6CZjNSdEnf1A7jsI9u4= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0 h1:gggzg0SUMs6SQbEw+3LoSsYf9YMjkupeAnHMX8O9mmY= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0/go.mod h1:+6KLcKIVgxoBDMqMO/Nvy7bZ9a0nbU3I1DtFQK3YvB4= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 h1:OBhqkivkhkMqLPymWEppkm7vgPQY2XsHoEkaMQ0AdZY= -github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0/go.mod h1:kgDmCTgBzIEPFElEF+FK0SdjAor06dRq2Go927dnQ6o= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= -github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= -github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7fz8= -github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYxPRqGcpAD9w= -github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= -github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= -github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= -github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= -github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= -github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/containerd/containerd v1.7.11 h1:lfGKw3eU35sjV0aG2eYZTiwFEY1pCzxdzicHP3SZILw= -github.com/containerd/containerd v1.7.11/go.mod h1:5UluHxHTX2rdvYuZ5OJTC5m/KJNs0Zs9wVoJm9zf5ZE= -github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= -github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= -github.com/cpuguy83/dockercfg v0.3.1 h1:/FpZ+JaygUR/lZP2NlFI2DVfrOEMAIKP5wWEJdoYe9E= -github.com/cpuguy83/dockercfg v0.3.1/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= -github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= -github.com/danaugrs/go-tsne v0.0.0-20200708172100-6b7d1d577fd3 h1:4V3w6LD+GOVbkF0jtjAzMRczS18+Gx0/nSZ3Pub3h00= -github.com/danaugrs/go-tsne v0.0.0-20200708172100-6b7d1d577fd3/go.mod h1:tcVxJUGCaPp/YynlqJTfJtGc/LF9vn4WUZSSmaGu3dA= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0= -github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= -github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= -github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= -github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= -github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM= -github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= -github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= -github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= -github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8= -github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= -github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/go-openapi/analysis v0.21.2 h1:hXFrOYFHUAMQdu6zwAiKKJHJQ8kqZs1ux/ru1P1wLJU= -github.com/go-openapi/analysis v0.21.2/go.mod h1:HZwRk4RRisyG8vx2Oe6aqeSQcoxRp47Xkp3+K6q+LdY= -github.com/go-openapi/errors v0.19.8/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= -github.com/go-openapi/errors v0.19.9/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= -github.com/go-openapi/errors v0.20.2/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= -github.com/go-openapi/errors v0.20.3 h1:rz6kiC84sqNQoqrtulzaL/VERgkoCyB6WdEkc2ujzUc= -github.com/go-openapi/errors v0.20.3/go.mod h1:Z3FlZ4I8jEGxjUK+bugx3on2mIAk4txuAOhlsB1FSgk= -github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= -github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.19.6 h1:UBIxjkht+AWIgYzCDSv2GN+E/togfwXUJFRTWhl2Jjs= -github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= -github.com/go-openapi/loads v0.21.1 h1:Wb3nVZpdEzDTcly8S4HMkey6fjARRzb7iEaySimlDW0= -github.com/go-openapi/loads v0.21.1/go.mod h1:/DtAMXXneXFjbQMGEtbamCZb+4x7eGwkvZCvBmwUG+g= -github.com/go-openapi/spec v0.20.4 h1:O8hJrt0UMnhHcluhIdUgCLRWyM2x7QkBXRvOs7m+O1M= -github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I= -github.com/go-openapi/strfmt v0.21.0/go.mod h1:ZRQ409bWMj+SOgXofQAGTIo2Ebu72Gs+WaRADcS5iNg= -github.com/go-openapi/strfmt v0.21.1/go.mod h1:I/XVKeLc5+MM5oPNN7P6urMOpuLXEcNrCX/rPGuWb0k= -github.com/go-openapi/strfmt v0.21.3 h1:xwhj5X6CjXEZZHMWy1zKJxvW9AfHC9pkyUjLvHtKG7o= -github.com/go-openapi/strfmt v0.21.3/go.mod h1:k+RzNO0Da+k3FrrynSNN8F7n/peCmQQqbbXjtDfvmGg= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= -github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-openapi/validate v0.21.0 h1:+Wqk39yKOhfpLqNLEC0/eViCkzM5FVXVqrvt526+wcI= -github.com/go-openapi/validate v0.21.0/go.mod h1:rjnrwK57VJ7A8xqfpAOEKRH8yQSGUriMu5/zuPSQ1hg= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= -github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= -github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= -github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= -github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= -github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= -github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= -github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= -github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= -github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= -github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= -github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= -github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= -github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= -github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= -github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= -github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= -github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= -github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= -github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= -github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= -github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= -github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= -github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= -github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= -github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= -github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= -github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= -github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= -github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvkiqTYKBCKLNmlge2eVjoZfySzM= -github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w= -github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= -github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= -github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= -github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/memberlist v0.5.0 h1:EtYPN8DpAURiapus508I4n9CzHs2W+8NZGbmmR/prTM= -github.com/hashicorp/memberlist v0.5.0/go.mod h1:yvyXLpo0QaGE59Y7hDTsTzDD25JYBZ4mHgHUZ8lrOI0= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= -github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= -github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= -github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= -github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= -github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= -github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= -github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= -github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/miekg/dns v1.1.26 h1:gPxPSwALAeHJSjarOs00QjVdV9QoBvc1D2ujQUr5BzU= -github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= -github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= -github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.63 h1:GbZ2oCvaUdgT5640WJOpyDhhDxvknAJU2/T3yurwcbQ= -github.com/minio/minio-go/v7 v7.0.63/go.mod h1:Q6X7Qjb7WMhvG65qKf4gUgA5XaiSox74kR1uAEjxRS4= -github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= -github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= -github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= -github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= -github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= -github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= -github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= -github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= -github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= -github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.0-rc5 h1:Ygwkfw9bpDvs+c9E34SdgGOj41dX/cbdlwvlWt0pnFI= -github.com/opencontainers/image-spec v1.1.0-rc5/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= -github.com/opencontainers/runc v1.1.10 h1:EaL5WeO9lv9wmS6SASjszOeQdSctvpbu0DdBQBizE40= -github.com/opencontainers/runc v1.1.10/go.mod h1:+/R6+KmDlh+hOO8NkjmgkG9Qzvypzk0yXxAPYYR65+M= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= -github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= -github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkoukk/tiktoken-go v0.1.6 h1:JF0TlJzhTbrI30wCvFuiw6FzP2+/bR+FIxUdgEAcUsw= -github.com/pkoukk/tiktoken-go v0.1.6/go.mod h1:9NiV+i9mJKGj1rYOT+njbv+ZwA/zJxYdewGl6qVatpg= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= -github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= -github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= -github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM= -github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= -github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= -github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= -github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= -github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= -github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/shirou/gopsutil/v3 v3.23.9 h1:ZI5bWVeu2ep4/DIxB4U9okeYJ7zp/QLTO4auRb/ty/E= -github.com/shirou/gopsutil/v3 v3.23.9/go.mod h1:x/NWSb71eMcjFIO0vhyGW5nZ7oSIgVjrCnADckb85GA= -github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= -github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= -github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= -github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= -github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/tailor-inc/graphql v0.4.1 h1:pAl/Xwy8aMtDVX0QjNj/S8vAWVM0e3IrK9554w0c2wg= -github.com/tailor-inc/graphql v0.4.1/go.mod h1:KtXmBAjFV+o3NEaWvtOStTMqE7g7sCWIGazL5sgJU7k= -github.com/testcontainers/testcontainers-go v0.26.0 h1:uqcYdoOHBy1ca7gKODfBd9uTHVK3a7UL848z09MVZ0c= -github.com/testcontainers/testcontainers-go v0.26.0/go.mod h1:ICriE9bLX5CLxL9OFQ2N+2N+f+803LNJ1utJb1+Inx0= -github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= -github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= -github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= -github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= -github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= -github.com/weaviate/weaviate v1.23.0 h1:OYJ34YSGLsSK92KfJlN/+AYWTSOS/Hzn3UkNNWqZ5F4= -github.com/weaviate/weaviate v1.23.0/go.mod h1:afludwbcyIZa9HEBELvHNb8zjH+KcjcW/jb4SZ5C2T4= -github.com/weaviate/weaviate-go-client/v4 v4.12.1 h1:XFKL49BgSOcxrFs5IV+Q5pydLTsh0HQHuWbKNSLMWLU= -github.com/weaviate/weaviate-go-client/v4 v4.12.1/go.mod h1:r1PlU5sAZKFvAPgymEHQj0hjSAuEV9X77PJ/ffZ6cEo= -github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= -github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= -github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= -github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= -github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= -github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= -github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg= -go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng= -go.mongodb.org/mongo-driver v1.10.0/go.mod h1:wsihk0Kdgv8Kqu1Anit4sfK+22vSFbUrAVEYRhCXrA8= -go.mongodb.org/mongo-driver v1.11.3 h1:Ql6K6qYHEzB6xvu4+AU0BoRoqf9vFPcc4o7MUIdPW8Y= -go.mongodb.org/mongo-driver v1.11.3/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= -go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= -go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= -golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea h1:vLCWI/yYrdEHyN2JzIzPO3aaQJHQdp89IZBA/+azVC4= -golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= -golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= -golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg= -golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -gonum.org/v1/gonum v0.12.0 h1:xKuo6hzt+gMav00meVPUlXwSdoEJP46BR+wdxQEFK2o= -gonum.org/v1/gonum v0.12.0/go.mod h1:73TDxJfAAHeA8Mk9mf8NlIppyhQNo5GLTcYeqgo2lvY= -google.golang.org/api v0.132.0 h1:8t2/+qZ26kAOGSmOiHwVycqVaDg7q3JDILrNi/Z6rvc= -google.golang.org/api v0.132.0/go.mod h1:AeTBC6GpJnJSRJjktDcPX0QwtS8pGYZOV6MSuSCusw0= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY= -google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= -google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d h1:DoPTO70H+bcDXcd39vOqb2viZxgqeBeSGtZ55yZU4/Q= -google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= -google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= -google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= -gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.5.0 h1:Ljk6PdHdOhAb5aDMWXjDLMMhph+BpztA4v1QdqEW2eY= -gotest.tools/v3 v3.5.0/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/test/acceptance_with_go_client/grpc_tests/batch_grpc_test.go b/test/acceptance_with_go_client/grpc_tests/batch_grpc_test.go deleted file mode 100644 index 64ea98580d88b894874f26d5ecfc7ff631382074..0000000000000000000000000000000000000000 --- a/test/acceptance_with_go_client/grpc_tests/batch_grpc_test.go +++ /dev/null @@ -1,107 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package grpc_tests - -import ( - "context" - "testing" - "time" - - "acceptance_tests_with_client/fixtures" - - "github.com/go-openapi/strfmt" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - wvt "github.com/weaviate/weaviate-go-client/v4/weaviate" - "github.com/weaviate/weaviate-go-client/v4/weaviate/grpc" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/test/docker" -) - -func TestGRPC_Batch(t *testing.T) { - ctx := context.Background() - config := wvt.Config{Scheme: "http", Host: "localhost:8080", GrpcConfig: &grpc.Config{Host: "localhost:50051"}} - client, err := wvt.NewClient(config) - require.NoError(t, err) - require.NotNil(t, client) - // clean DB - err = client.Schema().AllDeleter().Do(ctx) - require.NoError(t, err) - t.Run("all properties", testGRPCBatchAPI(ctx, client)) -} - -func TestGRPC_Batch_Cluster(t *testing.T) { - ctx := context.Background() - compose, err := docker.New(). - WithWeaviateClusterWithGRPC(). - WithText2VecContextionary(). - Start(ctx) - require.NoError(t, err) - defer func() { - require.NoError(t, compose.Terminate(ctx)) - }() - - httpUri := compose.GetWeaviateNode2().GetEndpoint(docker.HTTP) - grpcUri := compose.GetWeaviateNode2().GetEndpoint(docker.GRPC) - - config := wvt.Config{ - Scheme: "http", Host: httpUri, - GrpcConfig: &grpc.Config{Host: grpcUri}, - StartupTimeout: 30 * time.Second, - } - client, err := wvt.NewClient(config) - require.NoError(t, err) - require.NotNil(t, client) - // clean DB - err = client.Schema().AllDeleter().Do(ctx) - require.NoError(t, err) - - t.Run("all properties", testGRPCBatchAPI(ctx, client)) -} - -func testGRPCBatchAPI(ctx context.Context, client *wvt.Client) func(t *testing.T) { - return func(t *testing.T) { - class := fixtures.AllPropertiesClass - className := fixtures.AllPropertiesClassName - id1 := fixtures.AllPropertiesID1 - properties := fixtures.AllPropertiesProperties - t.Run("create schema", func(t *testing.T) { - err := client.Schema().ClassCreator().WithClass(class).Do(ctx) - require.NoError(t, err) - }) - t.Run("grpc batch import", func(t *testing.T) { - objects := []*models.Object{ - { - Class: className, - ID: strfmt.UUID(id1), - Properties: properties, - }, - } - resp, err := client.Batch().ObjectsBatcher().WithObjects(objects...).Do(ctx) - require.NoError(t, err) - require.NotNil(t, resp) - require.Len(t, resp, 1) - require.NotNil(t, resp[0].Result) - assert.Nil(t, resp[0].Result.Errors) - }) - t.Run("check", func(t *testing.T) { - objs, err := client.Data().ObjectsGetter().WithClassName(className).WithID(id1).Do(ctx) - require.NoError(t, err) - require.NotNil(t, objs) - require.Len(t, objs, 1) - assert.Equal(t, className, objs[0].Class) - props, ok := objs[0].Properties.(map[string]interface{}) - require.True(t, ok) - require.Equal(t, len(properties), len(props)) - }) - } -} diff --git a/test/acceptance_with_go_client/helper.go b/test/acceptance_with_go_client/helper.go deleted file mode 100644 index b1dcd8aac78e41f7fa72fc41d9a87b257e3926d9..0000000000000000000000000000000000000000 --- a/test/acceptance_with_go_client/helper.go +++ /dev/null @@ -1,49 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package acceptance_with_go_client - -import ( - "testing" - - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" -) - -var ( - vFalse = false - vTrue = true -) - -func GetIds(t *testing.T, resp *models.GraphQLResponse, className string) []string { - require.NotNil(t, resp) - require.NotNil(t, resp.Data) - require.Empty(t, resp.Errors) - - classMap, ok := resp.Data["Get"].(map[string]interface{}) - require.True(t, ok) - - class, ok := classMap[className].([]interface{}) - require.True(t, ok) - - ids := make([]string, len(class)) - for i := range class { - resultMap, ok := class[i].(map[string]interface{}) - require.True(t, ok) - - additional, ok := resultMap["_additional"].(map[string]interface{}) - require.True(t, ok) - - ids[i] = additional["id"].(string) - } - - return ids -} diff --git a/test/acceptance_with_go_client/multi_tenancy_tests/activation_deactivation_test.go b/test/acceptance_with_go_client/multi_tenancy_tests/activation_deactivation_test.go deleted file mode 100644 index dd0522513d8801c30ff8c38e54819655c2ebbdf2..0000000000000000000000000000000000000000 --- a/test/acceptance_with_go_client/multi_tenancy_tests/activation_deactivation_test.go +++ /dev/null @@ -1,661 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package multi_tenancy_tests - -import ( - "context" - "fmt" - "testing" - "time" - - "acceptance_tests_with_client/fixtures" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - wvt "github.com/weaviate/weaviate-go-client/v4/weaviate" - "github.com/weaviate/weaviate-go-client/v4/weaviate/fault" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/test/docker" -) - -func TestActivationDeactivation(t *testing.T) { - client, err := wvt.NewClient(wvt.Config{Scheme: "http", Host: "localhost:8080"}) - require.Nil(t, err) - - cleanup := func() { - err := client.Schema().AllDeleter().Do(context.Background()) - require.Nil(t, err) - } - - t.Run("deactivate / activate journey", func(t *testing.T) { - defer cleanup() - - tenants := fixtures.Tenants{ - { - Name: "tenantNo1", - // default status HOT - }, - { - Name: "tenantNo2", - ActivityStatus: models.TenantActivityStatusHOT, - }, - { - Name: "tenantNo3", - ActivityStatus: models.TenantActivityStatusCOLD, - }, - } - className := "Pizza" - ctx := context.Background() - pizzaIds := fixtures.IdsByClass[className] - - t.Run("create tenants (1,2,3), populate active tenants (1,2)", func(t *testing.T) { - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenants...) - fixtures.CreateDataPizzaForTenants(t, client, tenants[:2].Names()...) - - assertTenantActive(t, client, className, tenants[0].Name) - assertTenantActive(t, client, className, tenants[1].Name) - assertTenantInactive(t, client, className, tenants[2].Name) - - assertActiveTenantObjects(t, client, className, tenants[0].Name, pizzaIds) - assertActiveTenantObjects(t, client, className, tenants[1].Name, pizzaIds) - assertInactiveTenantObjects(t, client, className, tenants[2].Name) - }) - - t.Run("deactivate tenant (1)", func(t *testing.T) { - err := client.Schema().TenantsUpdater(). - WithClassName(className). - WithTenants(models.Tenant{ - Name: tenants[0].Name, - ActivityStatus: models.TenantActivityStatusCOLD, - }). - Do(ctx) - require.Nil(t, err) - - assertTenantInactive(t, client, className, tenants[0].Name) - assertTenantActive(t, client, className, tenants[1].Name) - assertTenantInactive(t, client, className, tenants[2].Name) - - assertInactiveTenantObjects(t, client, className, tenants[0].Name) - assertActiveTenantObjects(t, client, className, tenants[1].Name, pizzaIds) - assertInactiveTenantObjects(t, client, className, tenants[2].Name) - }) - - t.Run("activate and populate tenant (3)", func(t *testing.T) { - err := client.Schema().TenantsUpdater(). - WithClassName(className). - WithTenants(models.Tenant{ - Name: tenants[2].Name, - ActivityStatus: models.TenantActivityStatusHOT, - }). - Do(ctx) - require.Nil(t, err) - - fixtures.CreateDataPizzaForTenants(t, client, tenants[2].Name) - - assertTenantInactive(t, client, className, tenants[0].Name) - assertTenantActive(t, client, className, tenants[1].Name) - assertTenantActive(t, client, className, tenants[2].Name) - - assertInactiveTenantObjects(t, client, className, tenants[0].Name) - assertActiveTenantObjects(t, client, className, tenants[1].Name, pizzaIds) - assertActiveTenantObjects(t, client, className, tenants[2].Name, pizzaIds) - }) - - t.Run("activate tenant (1)", func(t *testing.T) { - err := client.Schema().TenantsUpdater(). - WithClassName(className). - WithTenants(models.Tenant{ - Name: tenants[0].Name, - ActivityStatus: models.TenantActivityStatusHOT, - }). - Do(ctx) - require.Nil(t, err) - - assertTenantActive(t, client, className, tenants[0].Name) - assertTenantActive(t, client, className, tenants[1].Name) - assertTenantActive(t, client, className, tenants[2].Name) - - assertActiveTenantObjects(t, client, className, tenants[0].Name, pizzaIds) - assertActiveTenantObjects(t, client, className, tenants[1].Name, pizzaIds) - assertActiveTenantObjects(t, client, className, tenants[2].Name, pizzaIds) - }) - - t.Run("deactivate tenant (2)", func(t *testing.T) { - err := client.Schema().TenantsUpdater(). - WithClassName(className). - WithTenants(models.Tenant{ - Name: tenants[1].Name, - ActivityStatus: models.TenantActivityStatusCOLD, - }). - Do(ctx) - require.Nil(t, err) - - assertTenantActive(t, client, className, tenants[0].Name) - assertTenantInactive(t, client, className, tenants[1].Name) - assertTenantActive(t, client, className, tenants[2].Name) - - assertActiveTenantObjects(t, client, className, tenants[0].Name, pizzaIds) - assertInactiveTenantObjects(t, client, className, tenants[1].Name) - assertActiveTenantObjects(t, client, className, tenants[2].Name, pizzaIds) - }) - - t.Run("delete tenants", func(t *testing.T) { - err := client.Schema().TenantsDeleter(). - WithClassName(className). - WithTenants(tenants.Names()...). - Do(ctx) - - require.Nil(t, err) - }) - }) -} - -type composeFn func(t *testing.T, ctx context.Context) ( - client *wvt.Client, - cleanupFn func(t *testing.T, ctx context.Context), - restartFn func(t *testing.T, ctx context.Context) *wvt.Client) - -func TestActivationDeactivation_Restarts(t *testing.T) { - t.Run("single node", func(t *testing.T) { - composeFn := func(t *testing.T, ctx context.Context) ( - client *wvt.Client, - cleanupFn func(t *testing.T, ctx context.Context), - restartFn func(t *testing.T, ctx context.Context) *wvt.Client, - ) { - compose, err := docker.New().WithWeaviate().Start(ctx) - require.Nil(t, err) - - container := compose.GetWeaviate() - client, err = wvt.NewClient(wvt.Config{Scheme: "http", Host: container.URI()}) - require.Nil(t, err) - - cleanupFn = func(t *testing.T, ctx context.Context) { - err := compose.Terminate(ctx) - require.Nil(t, err) - } - - restartFn = func(t *testing.T, ctx context.Context) *wvt.Client { - require.Nil(t, compose.Stop(ctx, container.Name(), nil)) - require.Nil(t, compose.Start(ctx, container.Name())) - - client, err := wvt.NewClient(wvt.Config{Scheme: "http", Host: container.URI()}) - require.Nil(t, err) - - return client - } - - return - } - - testActivationDeactivationWithRestarts(t, composeFn) - }) - - t.Run("multiple nodes", func(t *testing.T) { - composeFn := func(t *testing.T, ctx context.Context) ( - client *wvt.Client, - cleanupFn func(t *testing.T, ctx context.Context), - restartFn func(t *testing.T, ctx context.Context) *wvt.Client, - ) { - compose, err := docker.New().WithWeaviateCluster().Start(ctx) - require.Nil(t, err) - - container1 := compose.GetWeaviate() - container2 := compose.GetWeaviateNode2() - client, err = wvt.NewClient(wvt.Config{Scheme: "http", Host: container1.URI()}) - require.Nil(t, err) - - cleanupFn = func(t *testing.T, ctx context.Context) { - err := compose.Terminate(ctx) - require.Nil(t, err) - } - - restartFn = func(t *testing.T, ctx context.Context) *wvt.Client { - require.Nil(t, compose.Stop(ctx, container1.Name(), nil)) - require.Nil(t, compose.Start(ctx, container1.Name())) - require.Nil(t, compose.Stop(ctx, container2.Name(), nil)) - require.Nil(t, compose.Start(ctx, container2.Name())) - - client, err := wvt.NewClient(wvt.Config{Scheme: "http", Host: container1.URI()}) - require.Nil(t, err) - - return client - } - - return - } - - testActivationDeactivationWithRestarts(t, composeFn) - }) -} - -func testActivationDeactivationWithRestarts(t *testing.T, composeFn composeFn) { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) - defer cancel() - - client, composeCleanup, restart := composeFn(t, ctx) - defer composeCleanup(t, ctx) - - cleanup := func() { - err := client.Schema().AllDeleter().Do(context.Background()) - require.Nil(t, err) - } - - classPizza := "Pizza" - classSoup := "Soup" - - t.Run("deactivate / activate journey", func(t *testing.T) { - defer cleanup() - - createTenants := func(className string, groupId, count int, status string) fixtures.Tenants { - tenants := make(fixtures.Tenants, count) - for i := 0; i < count; i++ { - tenants[i] = models.Tenant{ - Name: fmt.Sprintf("tenant_%s_%d_%d", className, groupId, i), - ActivityStatus: status, - } - } - return tenants - } - assertActiveTenants := func(t *testing.T, tenants fixtures.Tenants, className string, expectedIds []string) { - for _, tenant := range tenants { - assertTenantActive(t, client, className, tenant.Name) - assertActiveTenantObjects(t, client, className, tenant.Name, expectedIds) - } - } - assertInactiveTenants := func(t *testing.T, tenants fixtures.Tenants, className string) { - for _, tenant := range tenants { - assertTenantInactive(t, client, className, tenant.Name) - assertInactiveTenantObjects(t, client, className, tenant.Name) - } - } - - tenants1Pizza := createTenants(classPizza, 1, 5, "") // default status HOT - tenants2Pizza := createTenants(classPizza, 2, 4, models.TenantActivityStatusHOT) - tenants3Pizza := createTenants(classPizza, 3, 3, models.TenantActivityStatusCOLD) - tenants1Soup := createTenants(classSoup, 1, 4, "") // default status HOT - tenants2Soup := createTenants(classSoup, 2, 3, models.TenantActivityStatusHOT) - tenants3Soup := createTenants(classSoup, 3, 2, models.TenantActivityStatusCOLD) - idsPizza := fixtures.IdsByClass[classPizza] - idsSoup := fixtures.IdsByClass[classSoup] - - t.Run("create tenants (1,2,3), populate active tenants (1,2)", func(t *testing.T) { - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenants1Pizza...) - fixtures.CreateTenantsPizza(t, client, tenants2Pizza...) - fixtures.CreateTenantsPizza(t, client, tenants3Pizza...) - fixtures.CreateDataPizzaForTenants(t, client, tenants1Pizza.Names()...) - fixtures.CreateDataPizzaForTenants(t, client, tenants2Pizza.Names()...) - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenants1Soup...) - fixtures.CreateTenantsSoup(t, client, tenants2Soup...) - fixtures.CreateTenantsSoup(t, client, tenants3Soup...) - fixtures.CreateDataSoupForTenants(t, client, tenants1Soup.Names()...) - fixtures.CreateDataSoupForTenants(t, client, tenants2Soup.Names()...) - - assertActiveTenants(t, tenants1Pizza, classPizza, idsPizza) - assertActiveTenants(t, tenants2Pizza, classPizza, idsPizza) - assertInactiveTenants(t, tenants3Pizza, classPizza) - - assertActiveTenants(t, tenants1Soup, classSoup, idsSoup) - assertActiveTenants(t, tenants2Soup, classSoup, idsSoup) - assertInactiveTenants(t, tenants3Soup, classSoup) - }) - - t.Run("deactivate tenants (1)", func(t *testing.T) { - tenants := make(fixtures.Tenants, len(tenants1Pizza)) - for i, tenant := range tenants1Pizza { - tenants[i] = models.Tenant{ - Name: tenant.Name, - ActivityStatus: models.TenantActivityStatusCOLD, - } - } - - err := client.Schema().TenantsUpdater(). - WithClassName(classPizza). - WithTenants(tenants...). - Do(ctx) - require.Nil(t, err) - - tenants = make(fixtures.Tenants, len(tenants1Soup)) - for i, tenant := range tenants1Soup { - tenants[i] = models.Tenant{ - Name: tenant.Name, - ActivityStatus: models.TenantActivityStatusCOLD, - } - } - - err = client.Schema().TenantsUpdater(). - WithClassName(classSoup). - WithTenants(tenants...). - Do(ctx) - require.Nil(t, err) - - assertInactiveTenants(t, tenants1Pizza, classPizza) - assertActiveTenants(t, tenants2Pizza, classPizza, idsPizza) - assertInactiveTenants(t, tenants3Pizza, classPizza) - - assertInactiveTenants(t, tenants1Soup, classSoup) - assertActiveTenants(t, tenants2Soup, classSoup, idsSoup) - assertInactiveTenants(t, tenants3Soup, classSoup) - }) - - t.Run("restart db, nothing changed", func(t *testing.T) { - client = restart(t, ctx) - - assertInactiveTenants(t, tenants1Pizza, classPizza) - assertActiveTenants(t, tenants2Pizza, classPizza, idsPizza) - assertInactiveTenants(t, tenants3Pizza, classPizza) - - assertInactiveTenants(t, tenants1Soup, classSoup) - assertActiveTenants(t, tenants2Soup, classSoup, idsSoup) - assertInactiveTenants(t, tenants3Soup, classSoup) - }) - - t.Run("activate and populate tenants (3)", func(t *testing.T) { - tenants := make(fixtures.Tenants, len(tenants3Pizza)) - for i, tenant := range tenants3Pizza { - tenants[i] = models.Tenant{ - Name: tenant.Name, - ActivityStatus: models.TenantActivityStatusHOT, - } - } - - err := client.Schema().TenantsUpdater(). - WithClassName(classPizza). - WithTenants(tenants...). - Do(ctx) - require.Nil(t, err) - - fixtures.CreateDataPizzaForTenants(t, client, tenants3Pizza.Names()...) - - tenants = make(fixtures.Tenants, len(tenants3Soup)) - for i, tenant := range tenants3Soup { - tenants[i] = models.Tenant{ - Name: tenant.Name, - ActivityStatus: models.TenantActivityStatusHOT, - } - } - - err = client.Schema().TenantsUpdater(). - WithClassName(classSoup). - WithTenants(tenants...). - Do(ctx) - require.Nil(t, err) - - fixtures.CreateDataSoupForTenants(t, client, tenants3Soup.Names()...) - - assertInactiveTenants(t, tenants1Pizza, classPizza) - assertActiveTenants(t, tenants2Pizza, classPizza, idsPizza) - assertActiveTenants(t, tenants3Pizza, classPizza, idsPizza) - - assertInactiveTenants(t, tenants1Soup, classSoup) - assertActiveTenants(t, tenants2Soup, classSoup, idsSoup) - assertActiveTenants(t, tenants3Soup, classSoup, idsSoup) - }) - - t.Run("activate tenants (1)", func(t *testing.T) { - tenants := make(fixtures.Tenants, len(tenants1Pizza)) - for i, tenant := range tenants1Pizza { - tenants[i] = models.Tenant{ - Name: tenant.Name, - ActivityStatus: models.TenantActivityStatusHOT, - } - } - - err := client.Schema().TenantsUpdater(). - WithClassName(classPizza). - WithTenants(tenants...). - Do(ctx) - require.Nil(t, err) - - tenants = make(fixtures.Tenants, len(tenants1Soup)) - for i, tenant := range tenants1Soup { - tenants[i] = models.Tenant{ - Name: tenant.Name, - ActivityStatus: models.TenantActivityStatusHOT, - } - } - - err = client.Schema().TenantsUpdater(). - WithClassName(classSoup). - WithTenants(tenants...). - Do(ctx) - require.Nil(t, err) - - assertActiveTenants(t, tenants1Pizza, classPizza, idsPizza) - assertActiveTenants(t, tenants2Pizza, classPizza, idsPizza) - assertActiveTenants(t, tenants3Pizza, classPizza, idsPizza) - - assertActiveTenants(t, tenants1Soup, classSoup, idsSoup) - assertActiveTenants(t, tenants2Soup, classSoup, idsSoup) - assertActiveTenants(t, tenants3Soup, classSoup, idsSoup) - }) - - t.Run("deactivate tenants (2)", func(t *testing.T) { - tenants := make(fixtures.Tenants, len(tenants2Pizza)) - for i, tenant := range tenants2Pizza { - tenants[i] = models.Tenant{ - Name: tenant.Name, - ActivityStatus: models.TenantActivityStatusCOLD, - } - } - - err := client.Schema().TenantsUpdater(). - WithClassName(classPizza). - WithTenants(tenants...). - Do(ctx) - require.Nil(t, err) - - tenants = make(fixtures.Tenants, len(tenants2Soup)) - for i, tenant := range tenants2Soup { - tenants[i] = models.Tenant{ - Name: tenant.Name, - ActivityStatus: models.TenantActivityStatusCOLD, - } - } - - err = client.Schema().TenantsUpdater(). - WithClassName(classSoup). - WithTenants(tenants...). - Do(ctx) - require.Nil(t, err) - - assertActiveTenants(t, tenants1Pizza, classPizza, idsPizza) - assertInactiveTenants(t, tenants2Pizza, classPizza) - assertActiveTenants(t, tenants3Pizza, classPizza, idsPizza) - - assertActiveTenants(t, tenants1Soup, classSoup, idsSoup) - assertInactiveTenants(t, tenants2Soup, classSoup) - assertActiveTenants(t, tenants3Soup, classSoup, idsSoup) - }) - - t.Run("restart db, nothing changed", func(t *testing.T) { - client = restart(t, ctx) - - assertActiveTenants(t, tenants1Pizza, classPizza, idsPizza) - assertInactiveTenants(t, tenants2Pizza, classPizza) - assertActiveTenants(t, tenants3Pizza, classPizza, idsPizza) - - assertActiveTenants(t, tenants1Soup, classSoup, idsSoup) - assertInactiveTenants(t, tenants2Soup, classSoup) - assertActiveTenants(t, tenants3Soup, classSoup, idsSoup) - }) - - t.Run("activate already active (1,3), deactivate already inactive (2), nothing changed", func(t *testing.T) { - tenants := make(fixtures.Tenants, 0, len(tenants1Pizza)+len(tenants2Pizza)+len(tenants3Pizza)) - for _, tenant := range tenants1Pizza { - tenants = append(tenants, models.Tenant{ - Name: tenant.Name, - ActivityStatus: models.TenantActivityStatusHOT, - }) - } - for _, tenant := range tenants2Pizza { - tenants = append(tenants, models.Tenant{ - Name: tenant.Name, - ActivityStatus: models.TenantActivityStatusCOLD, - }) - } - for _, tenant := range tenants3Pizza { - tenants = append(tenants, models.Tenant{ - Name: tenant.Name, - ActivityStatus: models.TenantActivityStatusHOT, - }) - } - - err := client.Schema().TenantsUpdater(). - WithClassName(classPizza). - WithTenants(tenants...). - Do(ctx) - require.Nil(t, err) - - tenants = make(fixtures.Tenants, 0, len(tenants1Soup)+len(tenants2Soup)+len(tenants3Soup)) - for _, tenant := range tenants1Soup { - tenants = append(tenants, models.Tenant{ - Name: tenant.Name, - ActivityStatus: models.TenantActivityStatusHOT, - }) - } - for _, tenant := range tenants2Soup { - tenants = append(tenants, models.Tenant{ - Name: tenant.Name, - ActivityStatus: models.TenantActivityStatusCOLD, - }) - } - for _, tenant := range tenants3Soup { - tenants = append(tenants, models.Tenant{ - Name: tenant.Name, - ActivityStatus: models.TenantActivityStatusHOT, - }) - } - - err = client.Schema().TenantsUpdater(). - WithClassName(classSoup). - WithTenants(tenants...). - Do(ctx) - require.Nil(t, err) - - assertActiveTenants(t, tenants1Pizza, classPizza, idsPizza) - assertInactiveTenants(t, tenants2Pizza, classPizza) - assertActiveTenants(t, tenants3Pizza, classPizza, idsPizza) - - assertActiveTenants(t, tenants1Soup, classSoup, idsSoup) - assertInactiveTenants(t, tenants2Soup, classSoup) - assertActiveTenants(t, tenants3Soup, classSoup, idsSoup) - }) - - t.Run("activate tenants (2)", func(t *testing.T) { - tenants := make(fixtures.Tenants, len(tenants2Pizza)) - for i, tenant := range tenants2Pizza { - tenants[i] = models.Tenant{ - Name: tenant.Name, - ActivityStatus: models.TenantActivityStatusHOT, - } - } - - err := client.Schema().TenantsUpdater(). - WithClassName(classPizza). - WithTenants(tenants...). - Do(ctx) - require.Nil(t, err) - - tenants = make(fixtures.Tenants, len(tenants2Soup)) - for i, tenant := range tenants2Soup { - tenants[i] = models.Tenant{ - Name: tenant.Name, - ActivityStatus: models.TenantActivityStatusHOT, - } - } - - err = client.Schema().TenantsUpdater(). - WithClassName(classSoup). - WithTenants(tenants...). - Do(ctx) - require.Nil(t, err) - - assertActiveTenants(t, tenants1Pizza, classPizza, idsPizza) - assertActiveTenants(t, tenants2Pizza, classPizza, idsPizza) - assertActiveTenants(t, tenants3Pizza, classPizza, idsPizza) - - assertActiveTenants(t, tenants1Soup, classSoup, idsSoup) - assertActiveTenants(t, tenants2Soup, classSoup, idsSoup) - assertActiveTenants(t, tenants3Soup, classSoup, idsSoup) - }) - - t.Run("restart db, nothing changed", func(t *testing.T) { - client = restart(t, ctx) - - assertActiveTenants(t, tenants1Pizza, classPizza, idsPizza) - assertActiveTenants(t, tenants2Pizza, classPizza, idsPizza) - assertActiveTenants(t, tenants3Pizza, classPizza, idsPizza) - - assertActiveTenants(t, tenants1Soup, classSoup, idsSoup) - assertActiveTenants(t, tenants2Soup, classSoup, idsSoup) - assertActiveTenants(t, tenants3Soup, classSoup, idsSoup) - }) - }) -} - -func assertTenantActive(t *testing.T, client *wvt.Client, className, tenantName string) { - gotTenants, err := client.Schema().TenantsGetter(). - WithClassName(className). - Do(context.Background()) - require.Nil(t, err) - require.NotEmpty(t, gotTenants) - - byName := fixtures.Tenants(gotTenants).ByName(tenantName) - require.NotNil(t, byName) - require.Equal(t, models.TenantActivityStatusHOT, byName.ActivityStatus) -} - -func assertTenantInactive(t *testing.T, client *wvt.Client, className, tenantName string) { - gotTenants, err := client.Schema().TenantsGetter(). - WithClassName(className). - Do(context.Background()) - require.Nil(t, err) - require.NotEmpty(t, gotTenants) - - byName := fixtures.Tenants(gotTenants).ByName(tenantName) - require.NotNil(t, byName) - require.Equal(t, models.TenantActivityStatusCOLD, byName.ActivityStatus) -} - -func assertActiveTenantObjects(t *testing.T, client *wvt.Client, className, tenantName string, expectedIds []string) { - objects, err := client.Data().ObjectsGetter(). - WithClassName(className). - WithTenant(tenantName). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, len(expectedIds)) - - ids := make([]string, len(objects)) - for i, object := range objects { - ids[i] = string(object.ID) - } - assert.ElementsMatch(t, expectedIds, ids) -} - -func assertInactiveTenantObjects(t *testing.T, client *wvt.Client, className, tenantName string) { - objects, err := client.Data().ObjectsGetter(). - WithClassName(className). - WithTenant(tenantName). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "tenant not active") - require.Nil(t, objects) -} diff --git a/test/acceptance_with_go_client/multi_tenancy_tests/batch_reference_test.go b/test/acceptance_with_go_client/multi_tenancy_tests/batch_reference_test.go deleted file mode 100644 index 762b9756a5f3bf6de4aed309912acf5bb536f5fa..0000000000000000000000000000000000000000 --- a/test/acceptance_with_go_client/multi_tenancy_tests/batch_reference_test.go +++ /dev/null @@ -1,1523 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package multi_tenancy_tests - -import ( - "context" - "testing" - - "acceptance_tests_with_client/fixtures" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - wvt "github.com/weaviate/weaviate-go-client/v4/weaviate" - "github.com/weaviate/weaviate-go-client/v4/weaviate/filters" - "github.com/weaviate/weaviate-go-client/v4/weaviate/graphql" - "github.com/weaviate/weaviate/entities/models" -) - -func TestBatchReferenceCreate_MultiTenancy(t *testing.T) { - client, err := wvt.NewClient(wvt.Config{Scheme: "http", Host: "localhost:8080"}) - require.Nil(t, err) - - cleanup := func() { - err := client.Schema().AllDeleter().Do(context.Background()) - require.Nil(t, err) - } - - t.Run("creates references between MT classes", func(t *testing.T) { - defer cleanup() - - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - soupIds := fixtures.IdsByClass["Soup"] - pizzaIds := fixtures.IdsByClass["Pizza"] - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenants...) - fixtures.CreateDataSoupForTenants(t, client, tenants.Names()...) - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenants...) - fixtures.CreateDataPizzaForTenants(t, client, tenants.Names()...) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - references := []*models.BatchReference{} - for _, tenant := range tenants { - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - rpb := client.Batch().ReferencePayloadBuilder(). - WithFromClassName("Soup"). - WithFromRefProp("relatedToPizza"). - WithFromID(soupId). - WithToClassName("Pizza"). - WithToID(pizzaId). - WithTenant(tenant.Name) - - references = append(references, rpb.Payload()) - } - } - } - - resp, err := client.Batch().ReferencesBatcher(). - WithReferences(references...). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, resp) - assert.Len(t, resp, len(references)) - for i := range resp { - require.NotNil(t, resp[i].Result) - require.NotNil(t, resp[i].Result.Status) - assert.Equal(t, "SUCCESS", *resp[i].Result.Status) - assert.Nil(t, resp[i].Result.Errors) - } - - t.Run("verify created", func(t *testing.T) { - for _, tenant := range tenants { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Len(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"].([]interface{}), - len(pizzaIds)) - } - } - }) - - t.Run("verify graphql search", func(t *testing.T) { - for _, tenant := range tenants { - resp, err := client.GraphQL().Get(). - WithClassName("Soup"). - WithTenant(tenant.Name). - WithFields(graphql.Field{ - Name: "_additional", Fields: []graphql.Field{{Name: "id"}}, - }). - WithWhere(filters.Where(). - WithPath([]string{"relatedToPizza", "Pizza", "name"}). - WithOperator(filters.Equal). - WithValueString("Quattro Formaggi")). - Do(context.Background()) - - require.NoError(t, err) - assertGraphqlGetIds(t, resp, "Soup", soupIds) - } - }) - }) - - t.Run("fails creating references between MT classes without tenant", func(t *testing.T) { - defer cleanup() - - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - soupIds := fixtures.IdsByClass["Soup"] - pizzaIds := fixtures.IdsByClass["Pizza"] - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenants...) - fixtures.CreateDataSoupForTenants(t, client, tenants.Names()...) - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenants...) - fixtures.CreateDataPizzaForTenants(t, client, tenants.Names()...) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - references := []*models.BatchReference{} - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - rpb := client.Batch().ReferencePayloadBuilder(). - WithFromClassName("Soup"). - WithFromRefProp("relatedToPizza"). - WithFromID(soupId). - WithToClassName("Pizza"). - WithToID(pizzaId) - - references = append(references, rpb.Payload()) - } - } - - resp, err := client.Batch().ReferencesBatcher(). - WithReferences(references...). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, resp) - assert.Len(t, resp, len(references)) - for i := range resp { - require.NotNil(t, resp[i].Result) - require.NotNil(t, resp[i].Result.Status) - assert.Equal(t, "FAILED", *resp[i].Result.Status) - require.NotNil(t, resp[i].Result.Errors) - require.Len(t, resp[i].Result.Errors.Error, 1) - assert.Contains(t, resp[i].Result.Errors.Error[0].Message, "has multi-tenancy enabled, but request was without tenant") - } - - t.Run("verify not created", func(t *testing.T) { - for _, tenant := range tenants { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Nil(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"]) - } - } - }) - }) - - t.Run("fails creating references between MT classes with non existent tenant", func(t *testing.T) { - defer cleanup() - - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - soupIds := fixtures.IdsByClass["Soup"] - pizzaIds := fixtures.IdsByClass["Pizza"] - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenants...) - fixtures.CreateDataSoupForTenants(t, client, tenants.Names()...) - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenants...) - fixtures.CreateDataPizzaForTenants(t, client, tenants.Names()...) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - references := []*models.BatchReference{} - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - rpb := client.Batch().ReferencePayloadBuilder(). - WithFromClassName("Soup"). - WithFromRefProp("relatedToPizza"). - WithFromID(soupId). - WithToClassName("Pizza"). - WithToID(pizzaId). - WithTenant("nonExistentTenant") - - references = append(references, rpb.Payload()) - } - } - - resp, err := client.Batch().ReferencesBatcher(). - WithReferences(references...). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, resp) - assert.Len(t, resp, len(references)) - for i := range resp { - require.NotNil(t, resp[i].Result) - require.NotNil(t, resp[i].Result.Status) - assert.Equal(t, "FAILED", *resp[i].Result.Status) - require.NotNil(t, resp[i].Result.Errors) - require.Len(t, resp[i].Result.Errors.Error, 1) - assert.Contains(t, resp[i].Result.Errors.Error[0].Message, "tenant not found") - } - - t.Run("verify not created", func(t *testing.T) { - for _, tenant := range tenants { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Nil(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"]) - } - } - }) - }) - - t.Run("fails creating references between MT classes with different existing tenant", func(t *testing.T) { - defer cleanup() - - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - soupIds := fixtures.IdsByClass["Soup"] - pizzaIds := fixtures.IdsByClass["Pizza"] - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenants...) - fixtures.CreateDataSoupForTenants(t, client, tenants[0].Name) - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenants...) - fixtures.CreateDataPizzaForTenants(t, client, tenants[0].Name) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - references := []*models.BatchReference{} - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - rpb := client.Batch().ReferencePayloadBuilder(). - WithFromClassName("Soup"). - WithFromRefProp("relatedToPizza"). - WithFromID(soupId). - WithToClassName("Pizza"). - WithToID(pizzaId). - WithTenant(tenants[1].Name) - - references = append(references, rpb.Payload()) - } - } - - resp, err := client.Batch().ReferencesBatcher(). - WithReferences(references...). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, resp) - assert.Len(t, resp, len(references)) - for i := range resp { - require.NotNil(t, resp[i].Result) - require.NotNil(t, resp[i].Result.Status) - assert.Equal(t, "FAILED", *resp[i].Result.Status) - require.NotNil(t, resp[i].Result.Errors) - require.Len(t, resp[i].Result.Errors.Error, 1) - assert.Contains(t, resp[i].Result.Errors.Error[0].Message, "not found for tenant") - } - - t.Run("verify not created", func(t *testing.T) { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenants[0].Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Nil(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"]) - } - }) - - t.Run("verify new objects not created", func(t *testing.T) { - for _, soupId := range soupIds { - exists, err := client.Data().Checker(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenants[1].Name). - Do(context.Background()) - - require.Nil(t, err) - assert.False(t, exists) - } - - for _, pizzaId := range pizzaIds { - exists, err := client.Data().Checker(). - WithClassName("Pizza"). - WithID(pizzaId). - WithTenant(tenants[1].Name). - Do(context.Background()) - - require.Nil(t, err) - assert.False(t, exists) - } - }) - }) - - t.Run("fails creating references between MT classes and different tenants", func(t *testing.T) { - tenantPizza := models.Tenant{Name: "tenantPizza"} - tenantSoup := models.Tenant{Name: "tenantSoup"} - soupIds := fixtures.IdsByClass["Soup"] - pizzaIds := fixtures.IdsByClass["Pizza"] - - t.Run("with SRC tenant (common tenants)", func(t *testing.T) { - defer cleanup() - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenantPizza, tenantSoup) - fixtures.CreateDataSoupForTenants(t, client, tenantSoup.Name) - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenantPizza, tenantSoup) - fixtures.CreateDataPizzaForTenants(t, client, tenantPizza.Name) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - references := []*models.BatchReference{} - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - rpb := client.Batch().ReferencePayloadBuilder(). - WithFromClassName("Soup"). - WithFromRefProp("relatedToPizza"). - WithFromID(soupId). - WithToClassName("Pizza"). - WithToID(pizzaId). - WithTenant(tenantSoup.Name) // SRC tenant - - references = append(references, rpb.Payload()) - } - } - - resp, err := client.Batch().ReferencesBatcher(). - WithReferences(references...). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, resp) - assert.Len(t, resp, len(references)) - for i := range resp { - require.NotNil(t, resp[i].Result) - require.NotNil(t, resp[i].Result.Status) - assert.Equal(t, "FAILED", *resp[i].Result.Status) - require.NotNil(t, resp[i].Result.Errors) - require.Len(t, resp[i].Result.Errors.Error, 1) - assert.Contains(t, resp[i].Result.Errors.Error[0].Message, "not found for tenant") - } - - t.Run("verify not created", func(t *testing.T) { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Nil(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"]) - } - }) - - t.Run("verify new objects not created", func(t *testing.T) { - for _, soupId := range soupIds { - exists, err := client.Data().Checker(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantPizza.Name). - Do(context.Background()) - - require.Nil(t, err) - assert.False(t, exists) - } - - for _, pizzaId := range pizzaIds { - exists, err := client.Data().Checker(). - WithClassName("Pizza"). - WithID(pizzaId). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.Nil(t, err) - assert.False(t, exists) - } - }) - }) - - t.Run("with SRC tenant (separate tenants)", func(t *testing.T) { - defer cleanup() - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenantSoup) - fixtures.CreateDataSoupForTenants(t, client, tenantSoup.Name) - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenantPizza) - fixtures.CreateDataPizzaForTenants(t, client, tenantPizza.Name) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - references := []*models.BatchReference{} - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - rpb := client.Batch().ReferencePayloadBuilder(). - WithFromClassName("Soup"). - WithFromRefProp("relatedToPizza"). - WithFromID(soupId). - WithToClassName("Pizza"). - WithToID(pizzaId). - WithTenant(tenantSoup.Name) // SRC tenant - - references = append(references, rpb.Payload()) - } - } - - resp, err := client.Batch().ReferencesBatcher(). - WithReferences(references...). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, resp) - assert.Len(t, resp, len(references)) - for i := range resp { - require.NotNil(t, resp[i].Result) - require.NotNil(t, resp[i].Result.Status) - assert.Equal(t, "FAILED", *resp[i].Result.Status) - require.NotNil(t, resp[i].Result.Errors) - require.Len(t, resp[i].Result.Errors.Error, 1) - assert.Contains(t, resp[i].Result.Errors.Error[0].Message, "tenant not found") - } - - t.Run("verify not created", func(t *testing.T) { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Nil(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"]) - } - }) - - t.Run("verify new objects not created", func(t *testing.T) { - for _, soupId := range soupIds { - exists, err := client.Data().Checker(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantPizza.Name). - Do(context.Background()) - - require.NotNil(t, err) - assert.False(t, exists) - } - - for _, pizzaId := range pizzaIds { - exists, err := client.Data().Checker(). - WithClassName("Pizza"). - WithID(pizzaId). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.NotNil(t, err) - assert.False(t, exists) - } - }) - }) - - t.Run("with DEST tenant (common tenants)", func(t *testing.T) { - defer cleanup() - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenantPizza, tenantSoup) - fixtures.CreateDataSoupForTenants(t, client, tenantSoup.Name) - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenantPizza, tenantSoup) - fixtures.CreateDataPizzaForTenants(t, client, tenantPizza.Name) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - references := []*models.BatchReference{} - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - rpb := client.Batch().ReferencePayloadBuilder(). - WithFromClassName("Soup"). - WithFromRefProp("relatedToPizza"). - WithFromID(soupId). - WithToClassName("Pizza"). - WithToID(pizzaId). - WithTenant(tenantPizza.Name) // DEST tenant - - references = append(references, rpb.Payload()) - } - } - - resp, err := client.Batch().ReferencesBatcher(). - WithReferences(references...). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, resp) - assert.Len(t, resp, len(references)) - for i := range resp { - require.NotNil(t, resp[i].Result) - require.NotNil(t, resp[i].Result.Status) - assert.Equal(t, "FAILED", *resp[i].Result.Status) - require.NotNil(t, resp[i].Result.Errors) - require.Len(t, resp[i].Result.Errors.Error, 1) - assert.Contains(t, resp[i].Result.Errors.Error[0].Message, "not found for tenant") - } - - t.Run("verify not created", func(t *testing.T) { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Nil(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"]) - } - }) - - t.Run("verify new objects not created", func(t *testing.T) { - for _, soupId := range soupIds { - exists, err := client.Data().Checker(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantPizza.Name). - Do(context.Background()) - - require.Nil(t, err) - assert.False(t, exists) - } - - for _, pizzaId := range pizzaIds { - exists, err := client.Data().Checker(). - WithClassName("Pizza"). - WithID(pizzaId). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.Nil(t, err) - assert.False(t, exists) - } - }) - }) - - t.Run("with DEST tenant (separate tenants)", func(t *testing.T) { - defer cleanup() - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenantSoup) - fixtures.CreateDataSoupForTenants(t, client, tenantSoup.Name) - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenantPizza) - fixtures.CreateDataPizzaForTenants(t, client, tenantPizza.Name) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - references := []*models.BatchReference{} - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - rpb := client.Batch().ReferencePayloadBuilder(). - WithFromClassName("Soup"). - WithFromRefProp("relatedToPizza"). - WithFromID(soupId). - WithToClassName("Pizza"). - WithToID(pizzaId). - WithTenant(tenantPizza.Name) // DEST tenant - - references = append(references, rpb.Payload()) - } - } - - resp, err := client.Batch().ReferencesBatcher(). - WithReferences(references...). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, resp) - assert.Len(t, resp, len(references)) - for i := range resp { - require.NotNil(t, resp[i].Result) - require.NotNil(t, resp[i].Result.Status) - assert.Equal(t, "FAILED", *resp[i].Result.Status) - require.NotNil(t, resp[i].Result.Errors) - require.Len(t, resp[i].Result.Errors.Error, 1) - assert.Contains(t, resp[i].Result.Errors.Error[0].Message, "tenant not found") - } - - t.Run("verify not created", func(t *testing.T) { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Nil(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"]) - } - }) - - t.Run("verify new objects not created", func(t *testing.T) { - for _, soupId := range soupIds { - exists, err := client.Data().Checker(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantPizza.Name). - Do(context.Background()) - - require.NotNil(t, err) - assert.False(t, exists) - } - - for _, pizzaId := range pizzaIds { - exists, err := client.Data().Checker(). - WithClassName("Pizza"). - WithID(pizzaId). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.NotNil(t, err) - assert.False(t, exists) - } - }) - }) - }) - - t.Run("creates references between MT and non-MT classes", func(t *testing.T) { - defer cleanup() - - tenantSoup := models.Tenant{Name: "tenantSoup"} - pizzaIds := fixtures.IdsByClass["Pizza"] - soupIds := fixtures.IdsByClass["Soup"] - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenantSoup) - fixtures.CreateDataSoupForTenants(t, client, tenantSoup.Name) - - fixtures.CreateSchemaPizza(t, client) - fixtures.CreateDataPizza(t, client) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - references := []*models.BatchReference{} - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - rpb := client.Batch().ReferencePayloadBuilder(). - WithFromClassName("Soup"). - WithFromRefProp("relatedToPizza"). - WithFromID(soupId). - WithToClassName("Pizza"). - WithToID(pizzaId). - WithTenant(tenantSoup.Name) - - references = append(references, rpb.Payload()) - } - } - - resp, err := client.Batch().ReferencesBatcher(). - WithReferences(references...). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, resp) - assert.Len(t, resp, len(references)) - for i := range resp { - require.NotNil(t, resp[i].Result) - require.NotNil(t, resp[i].Result.Status) - assert.Equal(t, "SUCCESS", *resp[i].Result.Status) - assert.Nil(t, resp[i].Result.Errors) - } - - t.Run("verify created", func(t *testing.T) { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Len(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"].([]interface{}), - len(pizzaIds)) - } - }) - - t.Run("verify graphql search", func(t *testing.T) { - resp, err := client.GraphQL().Get(). - WithClassName("Soup"). - WithTenant(tenantSoup.Name). - WithFields(graphql.Field{ - Name: "_additional", Fields: []graphql.Field{{Name: "id"}}, - }). - WithWhere(filters.Where(). - WithPath([]string{"relatedToPizza", "Pizza", "name"}). - WithOperator(filters.Equal). - WithValueString("Quattro Formaggi")). - Do(context.Background()) - - require.NoError(t, err) - assertGraphqlGetIds(t, resp, "Soup", soupIds) - }) - }) - - t.Run("fails creating references between MT and non-MT classes without tenant", func(t *testing.T) { - defer cleanup() - - tenantSoup := models.Tenant{Name: "tenantSoup"} - pizzaIds := fixtures.IdsByClass["Pizza"] - soupIds := fixtures.IdsByClass["Soup"] - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenantSoup) - fixtures.CreateDataSoupForTenants(t, client, tenantSoup.Name) - - fixtures.CreateSchemaPizza(t, client) - fixtures.CreateDataPizza(t, client) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - references := []*models.BatchReference{} - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - rpb := client.Batch().ReferencePayloadBuilder(). - WithFromClassName("Soup"). - WithFromRefProp("relatedToPizza"). - WithFromID(soupId). - WithToClassName("Pizza"). - WithToID(pizzaId) - - references = append(references, rpb.Payload()) - } - } - - resp, err := client.Batch().ReferencesBatcher(). - WithReferences(references...). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, resp) - assert.Len(t, resp, len(references)) - for i := range resp { - require.NotNil(t, resp[i].Result) - require.NotNil(t, resp[i].Result.Status) - assert.Equal(t, "FAILED", *resp[i].Result.Status) - require.NotNil(t, resp[i].Result.Errors) - require.Len(t, resp[i].Result.Errors.Error, 1) - assert.Contains(t, resp[i].Result.Errors.Error[0].Message, "has multi-tenancy enabled, but request was without tenant") - } - - t.Run("verify not created", func(t *testing.T) { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Nil(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"]) - } - }) - - t.Run("verify new objects not created", func(t *testing.T) { - for _, soupId := range soupIds { - exists, err := client.Data().Checker(). - WithClassName("Soup"). - WithID(soupId). - Do(context.Background()) - - require.NotNil(t, err) - assert.False(t, exists) - } - - for _, pizzaId := range pizzaIds { - exists, err := client.Data().Checker(). - WithClassName("Pizza"). - WithID(pizzaId). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.NotNil(t, err) - assert.False(t, exists) - } - }) - }) - - t.Run("fails creating references between MT and non-MT classes with non existent tenant", func(t *testing.T) { - defer cleanup() - - tenantSoup := models.Tenant{Name: "tenantSoup"} - pizzaIds := fixtures.IdsByClass["Pizza"] - soupIds := fixtures.IdsByClass["Soup"] - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenantSoup) - fixtures.CreateDataSoupForTenants(t, client, tenantSoup.Name) - - fixtures.CreateSchemaPizza(t, client) - fixtures.CreateDataPizza(t, client) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - references := []*models.BatchReference{} - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - rpb := client.Batch().ReferencePayloadBuilder(). - WithFromClassName("Soup"). - WithFromRefProp("relatedToPizza"). - WithFromID(soupId). - WithToClassName("Pizza"). - WithToID(pizzaId). - WithTenant("nonExistentTenant") - - references = append(references, rpb.Payload()) - } - } - - resp, err := client.Batch().ReferencesBatcher(). - WithReferences(references...). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, resp) - assert.Len(t, resp, len(references)) - for i := range resp { - require.NotNil(t, resp[i].Result) - require.NotNil(t, resp[i].Result.Status) - assert.Equal(t, "FAILED", *resp[i].Result.Status) - require.NotNil(t, resp[i].Result.Errors) - require.Len(t, resp[i].Result.Errors.Error, 1) - assert.Contains(t, resp[i].Result.Errors.Error[0].Message, "tenant not found") - } - - t.Run("verify not created", func(t *testing.T) { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Nil(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"]) - } - }) - - t.Run("verify new objects not created", func(t *testing.T) { - for _, soupId := range soupIds { - exists, err := client.Data().Checker(). - WithClassName("Soup"). - WithID(soupId). - Do(context.Background()) - - require.NotNil(t, err) - assert.False(t, exists) - } - - for _, pizzaId := range pizzaIds { - exists, err := client.Data().Checker(). - WithClassName("Pizza"). - WithID(pizzaId). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.NotNil(t, err) - assert.False(t, exists) - } - }) - }) - - t.Run("fails creating references between MT and non-MT classes with different existing tenant", func(t *testing.T) { - defer cleanup() - - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - pizzaIds := fixtures.IdsByClass["Pizza"] - soupIds := fixtures.IdsByClass["Soup"] - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenants...) - fixtures.CreateDataSoupForTenants(t, client, tenants[0].Name) - - fixtures.CreateSchemaPizza(t, client) - fixtures.CreateDataPizza(t, client) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - references := []*models.BatchReference{} - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - rpb := client.Batch().ReferencePayloadBuilder(). - WithFromClassName("Soup"). - WithFromRefProp("relatedToPizza"). - WithFromID(soupId). - WithToClassName("Pizza"). - WithToID(pizzaId). - WithTenant(tenants[1].Name) - - references = append(references, rpb.Payload()) - } - } - - resp, err := client.Batch().ReferencesBatcher(). - WithReferences(references...). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, resp) - assert.Len(t, resp, len(references)) - for i := range resp { - require.NotNil(t, resp[i].Result) - require.NotNil(t, resp[i].Result.Status) - assert.Equal(t, "FAILED", *resp[i].Result.Status) - require.NotNil(t, resp[i].Result.Errors) - require.Len(t, resp[i].Result.Errors.Error, 1) - assert.Contains(t, resp[i].Result.Errors.Error[0].Message, "not found for tenant") - } - - t.Run("verify not created", func(t *testing.T) { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenants[0].Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Nil(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"]) - } - }) - - t.Run("verify new objects not created", func(t *testing.T) { - for _, soupId := range soupIds { - exists, err := client.Data().Checker(). - WithClassName("Soup"). - WithID(soupId). - Do(context.Background()) - - require.NotNil(t, err) - assert.False(t, exists) - } - - for _, soupId := range soupIds { - exists, err := client.Data().Checker(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenants[1].Name). - Do(context.Background()) - - require.Nil(t, err) - assert.False(t, exists) - } - - for _, tenant := range tenants { - for _, pizzaId := range pizzaIds { - exists, err := client.Data().Checker(). - WithClassName("Pizza"). - WithID(pizzaId). - WithTenant(tenant.Name). - Do(context.Background()) - - require.NotNil(t, err) - assert.False(t, exists) - } - } - }) - }) - - t.Run("fails creating references between non-MT and MT classes", func(t *testing.T) { - defer cleanup() - - tenantPizza := models.Tenant{Name: "tenantPizza"} - pizzaIds := fixtures.IdsByClass["Pizza"] - soupIds := fixtures.IdsByClass["Soup"] - - fixtures.CreateSchemaSoup(t, client) - fixtures.CreateDataSoup(t, client) - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenantPizza) - fixtures.CreateDataPizzaForTenants(t, client, tenantPizza.Name) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - references := []*models.BatchReference{} - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - rpb := client.Batch().ReferencePayloadBuilder(). - WithFromClassName("Soup"). - WithFromRefProp("relatedToPizza"). - WithFromID(soupId). - WithToClassName("Pizza"). - WithToID(pizzaId). - WithTenant(tenantPizza.Name) - - references = append(references, rpb.Payload()) - } - } - - resp, err := client.Batch().ReferencesBatcher(). - WithReferences(references...). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, resp) - assert.Len(t, resp, len(references)) - for i := range resp { - require.NotNil(t, resp[i].Result) - require.NotNil(t, resp[i].Result.Status) - assert.Equal(t, "FAILED", *resp[i].Result.Status) - require.NotNil(t, resp[i].Result.Errors) - require.Len(t, resp[i].Result.Errors.Error, 1) - assert.Contains(t, resp[i].Result.Errors.Error[0].Message, "cannot reference a multi-tenant enabled class from a non multi-tenant enabled class") - } - - t.Run("verify not created", func(t *testing.T) { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Nil(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"]) - } - }) - - t.Run("verify new objects not created", func(t *testing.T) { - for _, soupId := range soupIds { - exists, err := client.Data().Checker(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantPizza.Name). - Do(context.Background()) - - require.NotNil(t, err) - assert.False(t, exists) - } - }) - }) - - t.Run("fails creating references between non-MT and MT classes without tenant", func(t *testing.T) { - defer cleanup() - - tenantPizza := models.Tenant{Name: "tenantPizza"} - pizzaIds := fixtures.IdsByClass["Pizza"] - soupIds := fixtures.IdsByClass["Soup"] - - fixtures.CreateSchemaSoup(t, client) - fixtures.CreateDataSoup(t, client) - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenantPizza) - fixtures.CreateDataPizzaForTenants(t, client, tenantPizza.Name) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - references := []*models.BatchReference{} - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - rpb := client.Batch().ReferencePayloadBuilder(). - WithFromClassName("Soup"). - WithFromRefProp("relatedToPizza"). - WithFromID(soupId). - WithToClassName("Pizza"). - WithToID(pizzaId) - - references = append(references, rpb.Payload()) - } - } - - resp, err := client.Batch().ReferencesBatcher(). - WithReferences(references...). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, resp) - assert.Len(t, resp, len(references)) - for i := range resp { - require.NotNil(t, resp[i].Result) - require.NotNil(t, resp[i].Result.Status) - assert.Equal(t, "FAILED", *resp[i].Result.Status) - require.NotNil(t, resp[i].Result.Errors) - require.Len(t, resp[i].Result.Errors.Error, 1) - assert.Contains(t, resp[i].Result.Errors.Error[0].Message, "cannot reference a multi-tenant enabled class from a non multi-tenant enabled class") - } - - t.Run("verify not created", func(t *testing.T) { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Nil(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"]) - } - }) - - t.Run("verify new objects not created", func(t *testing.T) { - for _, soupId := range soupIds { - exists, err := client.Data().Checker(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantPizza.Name). - Do(context.Background()) - - require.NotNil(t, err) - assert.False(t, exists) - } - }) - }) - - t.Run("fails creating references between non-MT and MT classes with non existent tenant", func(t *testing.T) { - defer cleanup() - - tenantPizza := models.Tenant{Name: "tenantPizza"} - pizzaIds := fixtures.IdsByClass["Pizza"] - soupIds := fixtures.IdsByClass["Soup"] - - fixtures.CreateSchemaSoup(t, client) - fixtures.CreateDataSoup(t, client) - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenantPizza) - fixtures.CreateDataPizzaForTenants(t, client, tenantPizza.Name) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - references := []*models.BatchReference{} - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - rpb := client.Batch().ReferencePayloadBuilder(). - WithFromClassName("Soup"). - WithFromRefProp("relatedToPizza"). - WithFromID(soupId). - WithToClassName("Pizza"). - WithToID(pizzaId). - WithTenant("nonExistentTenant") - - references = append(references, rpb.Payload()) - } - } - - resp, err := client.Batch().ReferencesBatcher(). - WithReferences(references...). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, resp) - assert.Len(t, resp, len(references)) - for i := range resp { - require.NotNil(t, resp[i].Result) - require.NotNil(t, resp[i].Result.Status) - assert.Equal(t, "FAILED", *resp[i].Result.Status) - require.NotNil(t, resp[i].Result.Errors) - require.Len(t, resp[i].Result.Errors.Error, 1) - assert.Contains(t, resp[i].Result.Errors.Error[0].Message, "cannot reference a multi-tenant enabled class from a non multi-tenant enabled class") - } - - t.Run("verify not created", func(t *testing.T) { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Nil(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"]) - } - }) - - t.Run("verify new objects not created", func(t *testing.T) { - for _, soupId := range soupIds { - exists, err := client.Data().Checker(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantPizza.Name). - Do(context.Background()) - - require.NotNil(t, err) - assert.False(t, exists) - } - }) - }) - - t.Run("fails creating references between non-MT and MT classes with different existing tenant", func(t *testing.T) { - defer cleanup() - - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - pizzaIds := fixtures.IdsByClass["Pizza"] - soupIds := fixtures.IdsByClass["Soup"] - - fixtures.CreateSchemaSoup(t, client) - fixtures.CreateDataSoup(t, client) - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenants...) - fixtures.CreateDataPizzaForTenants(t, client, tenants[0].Name) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - references := []*models.BatchReference{} - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - rpb := client.Batch().ReferencePayloadBuilder(). - WithFromClassName("Soup"). - WithFromRefProp("relatedToPizza"). - WithFromID(soupId). - WithToClassName("Pizza"). - WithToID(pizzaId). - WithTenant(tenants[1].Name) - - references = append(references, rpb.Payload()) - } - } - - resp, err := client.Batch().ReferencesBatcher(). - WithReferences(references...). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, resp) - assert.Len(t, resp, len(references)) - for i := range resp { - require.NotNil(t, resp[i].Result) - require.NotNil(t, resp[i].Result.Status) - assert.Equal(t, "FAILED", *resp[i].Result.Status) - require.NotNil(t, resp[i].Result.Errors) - require.Len(t, resp[i].Result.Errors.Error, 1) - assert.Contains(t, resp[i].Result.Errors.Error[0].Message, "cannot reference a multi-tenant enabled class from a non multi-tenant enabled class") - } - - t.Run("verify not created", func(t *testing.T) { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Nil(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"]) - } - }) - - t.Run("verify new objects not created", func(t *testing.T) { - for _, tenant := range tenants { - for _, soupId := range soupIds { - exists, err := client.Data().Checker(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenant.Name). - Do(context.Background()) - - require.NotNil(t, err) - assert.False(t, exists) - } - } - - for _, pizzaId := range pizzaIds { - exists, err := client.Data().Checker(). - WithClassName("Soup"). - WithID(pizzaId). - WithTenant(tenants[1].Name). - Do(context.Background()) - - require.NotNil(t, err) - assert.False(t, exists) - } - }) - }) -} diff --git a/test/acceptance_with_go_client/multi_tenancy_tests/batch_test.go b/test/acceptance_with_go_client/multi_tenancy_tests/batch_test.go deleted file mode 100644 index 942b066603833e831553bca5a659fc09eb6e9c9e..0000000000000000000000000000000000000000 --- a/test/acceptance_with_go_client/multi_tenancy_tests/batch_test.go +++ /dev/null @@ -1,729 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package multi_tenancy_tests - -import ( - "context" - "testing" - - "acceptance_tests_with_client/fixtures" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - wvt "github.com/weaviate/weaviate-go-client/v4/weaviate" - "github.com/weaviate/weaviate-go-client/v4/weaviate/fault" - "github.com/weaviate/weaviate-go-client/v4/weaviate/filters" - "github.com/weaviate/weaviate/entities/models" -) - -func TestBatchCreate_MultiTenancy(t *testing.T) { - client, err := wvt.NewClient(wvt.Config{Scheme: "http", Host: "localhost:8080"}) - require.Nil(t, err) - - cleanup := func() { - err := client.Schema().AllDeleter().Do(context.Background()) - require.Nil(t, err) - } - - t.Run("creates objects of MT class", func(t *testing.T) { - defer cleanup() - - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenants...) - fixtures.CreateTenantsSoup(t, client, tenants...) - - for _, tenant := range tenants { - resp, err := client.Batch().ObjectsBatcher(). - WithObjects( - &models.Object{ - Class: "Pizza", - ID: fixtures.PIZZA_QUATTRO_FORMAGGI_ID, - Properties: map[string]interface{}{ - "name": "Quattro Formaggi", - "description": "Pizza quattro formaggi Italian: [ˈkwattro forˈmaddʒi] (four cheese pizza) is a variety of pizza in Italian cuisine that is topped with a combination of four kinds of cheese, usually melted together, with (rossa, red) or without (bianca, white) tomato sauce. It is popular worldwide, including in Italy,[1] and is one of the iconic items from pizzerias's menus.", - "price": float32(1.1), - "best_before": "2022-05-03T12:04:40+02:00", - }, - Tenant: tenant.Name, - }, - &models.Object{ - Class: "Pizza", - ID: fixtures.PIZZA_FRUTTI_DI_MARE_ID, - Properties: map[string]interface{}{ - "name": "Frutti di Mare", - "description": "Frutti di Mare is an Italian type of pizza that may be served with scampi, mussels or squid. It typically lacks cheese, with the seafood being served atop a tomato sauce.", - "price": float32(1.2), - "best_before": "2022-05-05T07:16:30+02:00", - }, - Tenant: tenant.Name, - }, - &models.Object{ - Class: "Soup", - ID: fixtures.SOUP_CHICKENSOUP_ID, - Properties: map[string]interface{}{ - "name": "ChickenSoup", - "description": "Used by humans when their inferior genetics are attacked by microscopic organisms.", - "price": float32(2.1), - }, - Tenant: tenant.Name, - }, - &models.Object{ - Class: "Soup", - ID: fixtures.SOUP_BEAUTIFUL_ID, - Properties: map[string]interface{}{ - "name": "Beautiful", - "description": "Putting the game of letter soups to a whole new level.", - "price": float32(2.2), - }, - Tenant: tenant.Name, - }). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, resp) - require.Len(t, resp, 4) - - ids := make([]string, len(resp)) - for i := range resp { - require.NotNil(t, resp[i]) - require.NotNil(t, resp[i].Result) - require.NotNil(t, resp[i].Result.Status) - assert.Equal(t, "SUCCESS", *resp[i].Result.Status) - assert.Equal(t, tenant.Name, resp[i].Tenant) - - ids[i] = resp[i].ID.String() - } - assert.ElementsMatch(t, ids, []string{ - fixtures.PIZZA_QUATTRO_FORMAGGI_ID, - fixtures.PIZZA_FRUTTI_DI_MARE_ID, - fixtures.SOUP_CHICKENSOUP_ID, - fixtures.SOUP_BEAUTIFUL_ID, - }) - } - - t.Run("verify created", func(t *testing.T) { - for _, tenant := range tenants { - for id, className := range map[string]string{ - fixtures.PIZZA_QUATTRO_FORMAGGI_ID: "Pizza", - fixtures.PIZZA_FRUTTI_DI_MARE_ID: "Pizza", - fixtures.SOUP_CHICKENSOUP_ID: "Soup", - fixtures.SOUP_BEAUTIFUL_ID: "Soup", - } { - exists, err := client.Data().Checker(). - WithID(id). - WithClassName(className). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.True(t, exists) - } - } - }) - }) - - t.Run("fails creating objects of MT class without tenant", func(t *testing.T) { - defer cleanup() - - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenants...) - fixtures.CreateTenantsSoup(t, client, tenants...) - - resp, err := client.Batch().ObjectsBatcher(). - WithObjects( - &models.Object{ - Class: "Pizza", - ID: fixtures.PIZZA_QUATTRO_FORMAGGI_ID, - Properties: map[string]interface{}{ - "name": "Quattro Formaggi", - "description": "Pizza quattro formaggi Italian: [ˈkwattro forˈmaddʒi] (four cheese pizza) is a variety of pizza in Italian cuisine that is topped with a combination of four kinds of cheese, usually melted together, with (rossa, red) or without (bianca, white) tomato sauce. It is popular worldwide, including in Italy,[1] and is one of the iconic items from pizzerias's menus.", - "price": float32(1.1), - "best_before": "2022-05-03T12:04:40+02:00", - }, - }, - &models.Object{ - Class: "Pizza", - ID: fixtures.PIZZA_FRUTTI_DI_MARE_ID, - Properties: map[string]interface{}{ - "name": "Frutti di Mare", - "description": "Frutti di Mare is an Italian type of pizza that may be served with scampi, mussels or squid. It typically lacks cheese, with the seafood being served atop a tomato sauce.", - "price": float32(1.2), - "best_before": "2022-05-05T07:16:30+02:00", - }, - }, - &models.Object{ - Class: "Soup", - ID: fixtures.SOUP_CHICKENSOUP_ID, - Properties: map[string]interface{}{ - "name": "ChickenSoup", - "description": "Used by humans when their inferior genetics are attacked by microscopic organisms.", - "price": float32(2.1), - }, - }, - &models.Object{ - Class: "Soup", - ID: fixtures.SOUP_BEAUTIFUL_ID, - Properties: map[string]interface{}{ - "name": "Beautiful", - "description": "Putting the game of letter soups to a whole new level.", - "price": float32(2.2), - }, - }). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, resp) - require.Len(t, resp, 4) - - for i := range resp { - require.NotNil(t, resp[i]) - require.NotNil(t, resp[i].Result) - require.NotNil(t, resp[i].Result.Status) - assert.Equal(t, "FAILED", *resp[i].Result.Status) - require.NotNil(t, resp[i].Result.Errors) - require.NotNil(t, resp[i].Result.Errors.Error) - require.Len(t, resp[i].Result.Errors.Error, 1) - assert.Contains(t, resp[i].Result.Errors.Error[0].Message, "has multi-tenancy enabled, but request was without tenant") - assert.Empty(t, resp[i].Tenant) - } - - t.Run("verify not created", func(t *testing.T) { - for _, tenant := range tenants { - for id, className := range map[string]string{ - fixtures.PIZZA_QUATTRO_FORMAGGI_ID: "Pizza", - fixtures.PIZZA_FRUTTI_DI_MARE_ID: "Pizza", - fixtures.SOUP_CHICKENSOUP_ID: "Soup", - fixtures.SOUP_BEAUTIFUL_ID: "Soup", - } { - exists, err := client.Data().Checker(). - WithID(id). - WithClassName(className). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.False(t, exists) - } - } - }) - }) - - t.Run("fails creating objects of MT class with non existent tenant", func(t *testing.T) { - defer cleanup() - - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenants...) - fixtures.CreateTenantsSoup(t, client, tenants...) - - resp, err := client.Batch().ObjectsBatcher(). - WithObjects( - &models.Object{ - Class: "Pizza", - ID: fixtures.PIZZA_QUATTRO_FORMAGGI_ID, - Properties: map[string]interface{}{ - "name": "Quattro Formaggi", - "description": "Pizza quattro formaggi Italian: [ˈkwattro forˈmaddʒi] (four cheese pizza) is a variety of pizza in Italian cuisine that is topped with a combination of four kinds of cheese, usually melted together, with (rossa, red) or without (bianca, white) tomato sauce. It is popular worldwide, including in Italy,[1] and is one of the iconic items from pizzerias's menus.", - "price": float32(1.1), - "best_before": "2022-05-03T12:04:40+02:00", - }, - Tenant: "nonExistentTenant", - }, - &models.Object{ - Class: "Pizza", - ID: fixtures.PIZZA_FRUTTI_DI_MARE_ID, - Properties: map[string]interface{}{ - "name": "Frutti di Mare", - "description": "Frutti di Mare is an Italian type of pizza that may be served with scampi, mussels or squid. It typically lacks cheese, with the seafood being served atop a tomato sauce.", - "price": float32(1.2), - "best_before": "2022-05-05T07:16:30+02:00", - }, - Tenant: "nonExistentTenant", - }, - &models.Object{ - Class: "Soup", - ID: fixtures.SOUP_CHICKENSOUP_ID, - Properties: map[string]interface{}{ - "name": "ChickenSoup", - "description": "Used by humans when their inferior genetics are attacked by microscopic organisms.", - "price": float32(2.1), - }, - Tenant: "nonExistentTenant", - }, - &models.Object{ - Class: "Soup", - ID: fixtures.SOUP_BEAUTIFUL_ID, - Properties: map[string]interface{}{ - "name": "Beautiful", - "description": "Putting the game of letter soups to a whole new level.", - "price": float32(2.2), - }, - Tenant: "nonExistentTenant", - }). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, resp) - assert.Len(t, resp, 4) - for i := range resp { - require.NotNil(t, resp[i]) - require.NotNil(t, resp[i].Result) - require.NotNil(t, resp[i].Result.Status) - assert.Equal(t, "FAILED", *resp[i].Result.Status) - require.NotNil(t, resp[i].Result.Errors) - require.NotNil(t, resp[i].Result.Errors.Error) - require.Len(t, resp[i].Result.Errors.Error, 1) - assert.Contains(t, resp[i].Result.Errors.Error[0].Message, "tenant not found") - assert.Equal(t, "nonExistentTenant", resp[i].Tenant) - } - - t.Run("verify not created", func(t *testing.T) { - for _, tenant := range tenants { - for id, className := range map[string]string{ - fixtures.PIZZA_QUATTRO_FORMAGGI_ID: "Pizza", - fixtures.PIZZA_FRUTTI_DI_MARE_ID: "Pizza", - fixtures.SOUP_CHICKENSOUP_ID: "Soup", - fixtures.SOUP_BEAUTIFUL_ID: "Soup", - } { - exists, err := client.Data().Checker(). - WithID(id). - WithClassName(className). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.False(t, exists) - } - } - }) - }) - - t.Run("fails creating objects of non-MT class when tenant given", func(t *testing.T) { - defer cleanup() - - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - - fixtures.CreateSchemaPizza(t, client) - fixtures.CreateSchemaSoup(t, client) - - for _, tenant := range tenants { - resp, err := client.Batch().ObjectsBatcher(). - WithObjects( - &models.Object{ - Class: "Pizza", - ID: fixtures.PIZZA_QUATTRO_FORMAGGI_ID, - Properties: map[string]interface{}{ - "name": "Quattro Formaggi", - "description": "Pizza quattro formaggi Italian: [ˈkwattro forˈmaddʒi] (four cheese pizza) is a variety of pizza in Italian cuisine that is topped with a combination of four kinds of cheese, usually melted together, with (rossa, red) or without (bianca, white) tomato sauce. It is popular worldwide, including in Italy,[1] and is one of the iconic items from pizzerias's menus.", - "price": float32(1.1), - "best_before": "2022-05-03T12:04:40+02:00", - }, - Tenant: tenant.Name, - }, - &models.Object{ - Class: "Pizza", - ID: fixtures.PIZZA_FRUTTI_DI_MARE_ID, - Properties: map[string]interface{}{ - "name": "Frutti di Mare", - "description": "Frutti di Mare is an Italian type of pizza that may be served with scampi, mussels or squid. It typically lacks cheese, with the seafood being served atop a tomato sauce.", - "price": float32(1.2), - "best_before": "2022-05-05T07:16:30+02:00", - }, - Tenant: tenant.Name, - }, - &models.Object{ - Class: "Soup", - ID: fixtures.SOUP_CHICKENSOUP_ID, - Properties: map[string]interface{}{ - "name": "ChickenSoup", - "description": "Used by humans when their inferior genetics are attacked by microscopic organisms.", - "price": float32(2.1), - }, - Tenant: tenant.Name, - }, - &models.Object{ - Class: "Soup", - ID: fixtures.SOUP_BEAUTIFUL_ID, - Properties: map[string]interface{}{ - "name": "Beautiful", - "description": "Putting the game of letter soups to a whole new level.", - "price": float32(2.2), - }, - Tenant: tenant.Name, - }). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, resp) - require.Len(t, resp, 4) - for i := range resp { - require.NotNil(t, resp[i]) - require.NotNil(t, resp[i].Result) - require.NotNil(t, resp[i].Result.Status) - assert.Equal(t, "FAILED", *resp[i].Result.Status) - require.NotNil(t, resp[i].Result.Errors) - require.NotNil(t, resp[i].Result.Errors.Error) - require.Len(t, resp[i].Result.Errors.Error, 1) - assert.Contains(t, resp[i].Result.Errors.Error[0].Message, "has multi-tenancy disabled, but request was with tenant") - assert.Equal(t, tenant.Name, resp[i].Tenant) - } - } - - t.Run("verify not created", func(t *testing.T) { - for id, className := range map[string]string{ - fixtures.PIZZA_QUATTRO_FORMAGGI_ID: "Pizza", - fixtures.PIZZA_FRUTTI_DI_MARE_ID: "Pizza", - fixtures.SOUP_CHICKENSOUP_ID: "Soup", - fixtures.SOUP_BEAUTIFUL_ID: "Soup", - } { - exists, err := client.Data().Checker(). - WithID(id). - WithClassName(className). - Do(context.Background()) - - require.Nil(t, err) - require.False(t, exists) - } - }) - }) -} - -func TestBatchDelete_MultiTenancy(t *testing.T) { - client, err := wvt.NewClient(wvt.Config{Scheme: "http", Host: "localhost:8080"}) - require.Nil(t, err) - - cleanup := func() { - err := client.Schema().AllDeleter().Do(context.Background()) - require.Nil(t, err) - } - - t.Run("deletes objects from MT class", func(t *testing.T) { - defer cleanup() - - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - - fixtures.CreateSchemaFoodForTenants(t, client) - fixtures.CreateTenantsFood(t, client, tenants...) - fixtures.CreateDataFoodForTenants(t, client, tenants.Names()...) - - for _, tenant := range tenants { - for className, ids := range fixtures.IdsByClass { - resp, err := client.Batch().ObjectsBatchDeleter(). - WithClassName(className). - WithWhere(filters.Where(). - WithOperator(filters.Like). - WithPath([]string{"name"}). - WithValueText("*")). - WithOutput("minimal"). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, resp) - require.NotNil(t, resp.Results) - assert.Equal(t, int64(len(ids)), resp.Results.Matches) - assert.Equal(t, int64(len(ids)), resp.Results.Successful) - } - } - - t.Run("verify deleted", func(t *testing.T) { - for _, tenant := range tenants { - for className, ids := range fixtures.IdsByClass { - for _, id := range ids { - exists, err := client.Data().Checker(). - WithID(id). - WithClassName(className). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.False(t, exists) - } - } - } - }) - }) - - t.Run("fails deleting objects from MT class without tenant", func(t *testing.T) { - defer cleanup() - - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - - fixtures.CreateSchemaFoodForTenants(t, client) - fixtures.CreateTenantsFood(t, client, tenants...) - fixtures.CreateDataFoodForTenants(t, client, tenants.Names()...) - - for className := range fixtures.IdsByClass { - resp, err := client.Batch().ObjectsBatchDeleter(). - WithClassName(className). - WithWhere(filters.Where(). - WithOperator(filters.Like). - WithPath([]string{"name"}). - WithValueText("*")). - WithOutput("minimal"). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "has multi-tenancy enabled, but request was without tenant") - require.Nil(t, resp) - } - - t.Run("verify not deleted", func(t *testing.T) { - for _, tenant := range tenants { - for className, ids := range fixtures.IdsByClass { - for _, id := range ids { - exists, err := client.Data().Checker(). - WithID(id). - WithClassName(className). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.True(t, exists) - } - } - } - }) - }) - - t.Run("fails deleting objects from MT class with non existent tenant", func(t *testing.T) { - defer cleanup() - - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - - fixtures.CreateSchemaFoodForTenants(t, client) - fixtures.CreateTenantsFood(t, client, tenants...) - fixtures.CreateDataFoodForTenants(t, client, tenants.Names()...) - - for className := range fixtures.IdsByClass { - resp, err := client.Batch().ObjectsBatchDeleter(). - WithClassName(className). - WithWhere(filters.Where(). - WithOperator(filters.Like). - WithPath([]string{"name"}). - WithValueText("*")). - WithOutput("minimal"). - WithTenant("nonExistentTenant"). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "tenant not found") - require.Nil(t, resp) - } - - t.Run("verify not deleted", func(t *testing.T) { - for _, tenant := range tenants { - for className, ids := range fixtures.IdsByClass { - for _, id := range ids { - exists, err := client.Data().Checker(). - WithID(id). - WithClassName(className). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.True(t, exists) - } - } - } - }) - }) - - t.Run("fails deleting objects from non-MT class when tenant given", func(t *testing.T) { - defer cleanup() - - fixtures.CreateSchemaFood(t, client) - fixtures.CreateDataFood(t, client) - - for className := range fixtures.IdsByClass { - resp, err := client.Batch().ObjectsBatchDeleter(). - WithClassName(className). - WithWhere(filters.Where(). - WithOperator(filters.Like). - WithPath([]string{"name"}). - WithValueText("*")). - WithOutput("minimal"). - WithTenant("nonExistentTenant"). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "has multi-tenancy disabled, but request was with tenant") - require.Nil(t, resp) - } - - t.Run("verify non deleted", func(t *testing.T) { - for className, ids := range fixtures.IdsByClass { - for _, id := range ids { - exists, err := client.Data().Checker(). - WithID(id). - WithClassName(className). - Do(context.Background()) - - require.Nil(t, err) - require.True(t, exists) - } - } - }) - }) - - t.Run("deletes objects from MT class by ref", func(t *testing.T) { - defer cleanup() - - tenant1 := models.Tenant{Name: "tenantNo1"} - tenant2 := models.Tenant{Name: "tenantNo2"} - soupIdByTenantAndPizza := map[string]map[string]string{ - tenant1.Name: { - fixtures.PIZZA_QUATTRO_FORMAGGI_ID: fixtures.SOUP_CHICKENSOUP_ID, - fixtures.PIZZA_FRUTTI_DI_MARE_ID: fixtures.SOUP_CHICKENSOUP_ID, - }, - tenant2.Name: { - fixtures.PIZZA_HAWAII_ID: fixtures.SOUP_BEAUTIFUL_ID, - fixtures.PIZZA_DOENER_ID: fixtures.SOUP_BEAUTIFUL_ID, - }, - } - - t.Run("add data", func(t *testing.T) { - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenant1, tenant2) - fixtures.CreateDataPizzaQuattroFormaggiForTenants(t, client, tenant1.Name) - fixtures.CreateDataPizzaFruttiDiMareForTenants(t, client, tenant1.Name) - fixtures.CreateDataPizzaHawaiiForTenants(t, client, tenant2.Name) - fixtures.CreateDataPizzaDoenerForTenants(t, client, tenant2.Name) - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenant1, tenant2) - fixtures.CreateDataSoupChickenForTenants(t, client, tenant1.Name) - fixtures.CreateDataSoupBeautifulForTenants(t, client, tenant2.Name) - }) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - t.Run("create refs", func(t *testing.T) { - references := []*models.BatchReference{} - - for tenant, pizzaToSoup := range soupIdByTenantAndPizza { - for pizzaId, soupId := range pizzaToSoup { - rpb := client.Batch().ReferencePayloadBuilder(). - WithFromClassName("Soup"). - WithFromRefProp("relatedToPizza"). - WithFromID(soupId). - WithToClassName("Pizza"). - WithToID(pizzaId). - WithTenant(tenant) - - references = append(references, rpb.Payload()) - } - } - - resp, err := client.Batch().ReferencesBatcher(). - WithReferences(references...). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, resp) - assert.Len(t, resp, len(references)) - for i := range resp { - require.NotNil(t, resp[i].Result) - assert.Nil(t, resp[i].Result.Errors) - } - }) - - for tenant, pizzaToSoup := range soupIdByTenantAndPizza { - i := 0 - for pizzaId := range pizzaToSoup { - resp, err := client.Batch().ObjectsBatchDeleter(). - WithClassName("Soup"). - WithWhere(filters.Where(). - WithOperator(filters.Like). - WithPath([]string{"relatedToPizza", "Pizza", "_id"}). - WithValueText(pizzaId)). - WithOutput("minimal"). - WithTenant(tenant). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, resp) - require.NotNil(t, resp.Results) - - // single soup references two pizzas, so soup will be deleted only for 1st pizza - if i == 0 { - assert.Equal(t, int64(1), resp.Results.Matches) - assert.Equal(t, int64(1), resp.Results.Successful) - } else { - assert.Equal(t, int64(0), resp.Results.Matches) - assert.Equal(t, int64(0), resp.Results.Successful) - } - i++ - } - } - - t.Run("verify deleted", func(t *testing.T) { - for tenant, pizzaToSoup := range soupIdByTenantAndPizza { - for _, soupId := range pizzaToSoup { - exists, err := client.Data().Checker(). - WithID(soupId). - WithClassName("Soup"). - WithTenant(tenant). - Do(context.Background()) - - require.Nil(t, err) - require.False(t, exists) - } - } - }) - }) -} diff --git a/test/acceptance_with_go_client/multi_tenancy_tests/data_test.go b/test/acceptance_with_go_client/multi_tenancy_tests/data_test.go deleted file mode 100644 index d04d4bca721124860cda7b0bfc39309d4171e2cb..0000000000000000000000000000000000000000 --- a/test/acceptance_with_go_client/multi_tenancy_tests/data_test.go +++ /dev/null @@ -1,1382 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package multi_tenancy_tests - -import ( - "context" - "fmt" - "testing" - - "acceptance_tests_with_client/fixtures" - - "github.com/go-openapi/strfmt" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - wvt "github.com/weaviate/weaviate-go-client/v4/weaviate" - "github.com/weaviate/weaviate-go-client/v4/weaviate/fault" - "github.com/weaviate/weaviate/entities/models" -) - -func TestData_MultiTenancy(t *testing.T) { - client, err := wvt.NewClient(wvt.Config{Scheme: "http", Host: "localhost:8080"}) - require.Nil(t, err) - - cleanup := func() { - err := client.Schema().AllDeleter().Do(context.Background()) - require.Nil(t, err) - } - - t.Run("creates objects of MT class", func(t *testing.T) { - defer cleanup() - - className := "Pizza" - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenants...) - - for _, tenant := range tenants { - wrap, err := client.Data().Creator(). - WithClassName(className). - WithID(fixtures.PIZZA_QUATTRO_FORMAGGI_ID). - WithProperties(map[string]interface{}{ - "name": "Quattro Formaggi", - "description": "Pizza quattro formaggi Italian: [ˈkwattro forˈmaddʒi] (four cheese pizza) is a variety of pizza in Italian cuisine that is topped with a combination of four kinds of cheese, usually melted together, with (rossa, red) or without (bianca, white) tomato sauce. It is popular worldwide, including in Italy,[1] and is one of the iconic items from pizzerias's menus.", - "price": float32(1.1), - "best_before": "2022-05-03T12:04:40+02:00", - }). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, wrap) - require.NotNil(t, wrap.Object) - assert.Equal(t, strfmt.UUID(fixtures.PIZZA_QUATTRO_FORMAGGI_ID), wrap.Object.ID) - assert.Equal(t, "Quattro Formaggi", wrap.Object.Properties.(map[string]interface{})["name"]) - assert.Equal(t, tenant.Name, wrap.Object.Tenant) - - wrap, err = client.Data().Creator(). - WithClassName(className). - WithID(fixtures.PIZZA_FRUTTI_DI_MARE_ID). - WithProperties(map[string]interface{}{ - "name": "Frutti di Mare", - "description": "Frutti di Mare is an Italian type of pizza that may be served with scampi, mussels or squid. It typically lacks cheese, with the seafood being served atop a tomato sauce.", - "price": float32(1.2), - "best_before": "2022-05-05T07:16:30+02:00", - }). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, wrap) - require.NotNil(t, wrap.Object) - assert.Equal(t, strfmt.UUID(fixtures.PIZZA_FRUTTI_DI_MARE_ID), wrap.Object.ID) - assert.Equal(t, "Frutti di Mare", wrap.Object.Properties.(map[string]interface{})["name"]) - assert.Equal(t, tenant.Name, wrap.Object.Tenant) - } - - t.Run("verify created", func(t *testing.T) { - for _, tenant := range tenants { - for _, id := range []string{ - fixtures.PIZZA_QUATTRO_FORMAGGI_ID, - fixtures.PIZZA_FRUTTI_DI_MARE_ID, - } { - exists, err := client.Data().Checker(). - WithID(id). - WithClassName(className). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.True(t, exists) - } - } - }) - }) - - t.Run("fails creating objects of MT class without tenant", func(t *testing.T) { - defer cleanup() - - className := "Pizza" - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenants...) - - wrap, err := client.Data().Creator(). - WithClassName(className). - WithID(fixtures.PIZZA_QUATTRO_FORMAGGI_ID). - WithProperties(map[string]interface{}{ - "name": "Quattro Formaggi", - "description": "Pizza quattro formaggi Italian: [ˈkwattro forˈmaddʒi] (four cheese pizza) is a variety of pizza in Italian cuisine that is topped with a combination of four kinds of cheese, usually melted together, with (rossa, red) or without (bianca, white) tomato sauce. It is popular worldwide, including in Italy,[1] and is one of the iconic items from pizzerias's menus.", - "price": float32(1.1), - "best_before": "2022-05-03T12:04:40+02:00", - }). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "has multi-tenancy enabled, but request was without tenant") - require.Nil(t, wrap) - - wrap, err = client.Data().Creator(). - WithClassName(className). - WithID(fixtures.PIZZA_FRUTTI_DI_MARE_ID). - WithProperties(map[string]interface{}{ - "name": "Frutti di Mare", - "description": "Frutti di Mare is an Italian type of pizza that may be served with scampi, mussels or squid. It typically lacks cheese, with the seafood being served atop a tomato sauce.", - "price": float32(1.2), - "best_before": "2022-05-05T07:16:30+02:00", - }). - Do(context.Background()) - - require.NotNil(t, err) - clientErr = err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "has multi-tenancy enabled, but request was without tenant") - require.Nil(t, wrap) - - t.Run("verify not created", func(t *testing.T) { - for _, tenant := range tenants { - for _, id := range []string{ - fixtures.PIZZA_QUATTRO_FORMAGGI_ID, - fixtures.PIZZA_FRUTTI_DI_MARE_ID, - } { - exists, err := client.Data().Checker(). - WithID(id). - WithClassName(className). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.False(t, exists) - } - } - }) - }) - - t.Run("fails creating objects of MT class with non existent tenant", func(t *testing.T) { - defer cleanup() - - className := "Pizza" - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenants...) - - wrap, err := client.Data().Creator(). - WithClassName(className). - WithID(fixtures.PIZZA_QUATTRO_FORMAGGI_ID). - WithProperties(map[string]interface{}{ - "name": "Quattro Formaggi", - "description": "Pizza quattro formaggi Italian: [ˈkwattro forˈmaddʒi] (four cheese pizza) is a variety of pizza in Italian cuisine that is topped with a combination of four kinds of cheese, usually melted together, with (rossa, red) or without (bianca, white) tomato sauce. It is popular worldwide, including in Italy,[1] and is one of the iconic items from pizzerias's menus.", - "price": float32(1.1), - "best_before": "2022-05-03T12:04:40+02:00", - }). - WithTenant("nonExistentTenant"). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "tenant not found") - require.Nil(t, wrap) - - wrap, err = client.Data().Creator(). - WithClassName(className). - WithID(fixtures.PIZZA_FRUTTI_DI_MARE_ID). - WithProperties(map[string]interface{}{ - "name": "Frutti di Mare", - "description": "Frutti di Mare is an Italian type of pizza that may be served with scampi, mussels or squid. It typically lacks cheese, with the seafood being served atop a tomato sauce.", - "price": float32(1.2), - "best_before": "2022-05-05T07:16:30+02:00", - }). - WithTenant("nonExistentTenant"). - Do(context.Background()) - - require.NotNil(t, err) - clientErr = err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "tenant not found") - require.Nil(t, wrap) - - t.Run("verify not created", func(t *testing.T) { - for _, tenant := range tenants { - for _, id := range []string{ - fixtures.PIZZA_QUATTRO_FORMAGGI_ID, - fixtures.PIZZA_FRUTTI_DI_MARE_ID, - } { - exists, err := client.Data().Checker(). - WithID(id). - WithClassName(className). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.False(t, exists) - } - } - }) - }) - - t.Run("fails creating objects of non-MT class when tenant given", func(t *testing.T) { - defer cleanup() - - className := "Pizza" - - fixtures.CreateSchemaPizza(t, client) - - wrap, err := client.Data().Creator(). - WithClassName(className). - WithID(fixtures.PIZZA_QUATTRO_FORMAGGI_ID). - WithProperties(map[string]interface{}{ - "name": "Quattro Formaggi", - "description": "Pizza quattro formaggi Italian: [ˈkwattro forˈmaddʒi] (four cheese pizza) is a variety of pizza in Italian cuisine that is topped with a combination of four kinds of cheese, usually melted together, with (rossa, red) or without (bianca, white) tomato sauce. It is popular worldwide, including in Italy,[1] and is one of the iconic items from pizzerias's menus.", - "price": float32(1.1), - "best_before": "2022-05-03T12:04:40+02:00", - }). - WithTenant("nonExistentTenant"). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "has multi-tenancy disabled, but request was with tenant") - require.Nil(t, wrap) - - wrap, err = client.Data().Creator(). - WithClassName(className). - WithID(fixtures.PIZZA_FRUTTI_DI_MARE_ID). - WithProperties(map[string]interface{}{ - "name": "Frutti di Mare", - "description": "Frutti di Mare is an Italian type of pizza that may be served with scampi, mussels or squid. It typically lacks cheese, with the seafood being served atop a tomato sauce.", - "price": float32(1.2), - "best_before": "2022-05-05T07:16:30+02:00", - }). - WithTenant("nonExistentTenant"). - Do(context.Background()) - - require.NotNil(t, err) - clientErr = err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "has multi-tenancy disabled, but request was with tenant") - require.Nil(t, wrap) - - t.Run("verify not created", func(t *testing.T) { - objects, err := client.Data().ObjectsGetter(). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - assert.Len(t, objects, 0) - }) - }) - - t.Run("gets objects of MT class", func(t *testing.T) { - defer cleanup() - - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - - fixtures.CreateSchemaFoodForTenants(t, client) - fixtures.CreateTenantsFood(t, client, tenants...) - fixtures.CreateDataFoodForTenants(t, client, tenants.Names()...) - - extractIds := func(objs []*models.Object) []string { - ids := make([]string, len(objs)) - for i, obj := range objs { - ids[i] = obj.ID.String() - } - return ids - } - - for _, tenant := range tenants { - for className, ids := range fixtures.IdsByClass { - for _, id := range ids { - t.Run("single object by class+id", func(t *testing.T) { - objects, err := client.Data().ObjectsGetter(). - WithID(id). - WithClassName(className). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Equal(t, strfmt.UUID(id), objects[0].ID) - assert.Equal(t, tenant.Name, objects[0].Tenant) - }) - } - - t.Run("list objects by class", func(t *testing.T) { - objects, err := client.Data().ObjectsGetter(). - WithClassName(className). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, len(ids)) - assert.ElementsMatch(t, ids, extractIds(objects)) - }) - } - - t.Run("list all objects", func(t *testing.T) { - objects, err := client.Data().ObjectsGetter(). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, len(fixtures.AllIds)) - assert.ElementsMatch(t, fixtures.AllIds, extractIds(objects)) - }) - } - }) - - t.Run("fails getting objects of MT class without tenant", func(t *testing.T) { - defer cleanup() - - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - - fixtures.CreateSchemaFoodForTenants(t, client) - fixtures.CreateTenantsFood(t, client, tenants...) - fixtures.CreateDataFoodForTenants(t, client, tenants.Names()...) - - for className, ids := range fixtures.IdsByClass { - for _, id := range ids { - t.Run("single object by class+id", func(t *testing.T) { - objects, err := client.Data().ObjectsGetter(). - WithID(id). - WithClassName(className). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "has multi-tenancy enabled, but request was without tenant") - assert.Nil(t, objects) - }) - } - - t.Run("list objects by class", func(t *testing.T) { - objects, err := client.Data().ObjectsGetter(). - WithClassName(className). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "has multi-tenancy enabled, but request was without tenant") - assert.Nil(t, objects) - }) - } - - t.Run("list all objects", func(t *testing.T) { - objects, err := client.Data().ObjectsGetter(). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - assert.Len(t, objects, 0) - }) - }) - - t.Run("fails getting objects of MT class with non existent tenant", func(t *testing.T) { - defer cleanup() - - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - - fixtures.CreateSchemaFoodForTenants(t, client) - fixtures.CreateTenantsFood(t, client, tenants...) - fixtures.CreateDataFoodForTenants(t, client, tenants.Names()...) - - for className, ids := range fixtures.IdsByClass { - for _, id := range ids { - t.Run("single object by class+id", func(t *testing.T) { - objects, err := client.Data().ObjectsGetter(). - WithID(id). - WithClassName(className). - WithTenant("nonExistentTenant"). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "tenant not found") - assert.Nil(t, objects) - }) - } - - t.Run("list objects by class", func(t *testing.T) { - objects, err := client.Data().ObjectsGetter(). - WithClassName(className). - WithTenant("nonExistentTenant"). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "tenant not found") - assert.Nil(t, objects) - }) - } - - t.Run("list all objects", func(t *testing.T) { - objects, err := client.Data().ObjectsGetter(). - WithTenant("nonExistentTenant"). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - assert.Len(t, objects, 0) - }) - }) - - t.Run("fails getting objects of non-MT class when tenant given", func(t *testing.T) { - defer cleanup() - - fixtures.CreateSchemaFood(t, client) - fixtures.CreateDataFood(t, client) - - for className, ids := range fixtures.IdsByClass { - for _, id := range ids { - t.Run("single object by class+id", func(t *testing.T) { - objects, err := client.Data().ObjectsGetter(). - WithID(id). - WithClassName(className). - WithTenant("nonExistentTenant"). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "has multi-tenancy disabled, but request was with tenant") - require.Nil(t, objects) - }) - } - - t.Run("list objects by class", func(t *testing.T) { - objects, err := client.Data().ObjectsGetter(). - WithClassName(className). - WithTenant("nonExistentTenant"). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "has multi-tenancy disabled, but request was with tenant") - require.Nil(t, objects) - }) - } - - t.Run("list all objects", func(t *testing.T) { - objects, err := client.Data().ObjectsGetter(). - WithTenant("nonExistentTenant"). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - assert.Len(t, objects, 0) - }) - }) - - t.Run("checks objects of MT class", func(t *testing.T) { - defer cleanup() - - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - - fixtures.CreateSchemaFoodForTenants(t, client) - fixtures.CreateTenantsFood(t, client, tenants...) - fixtures.CreateDataFoodForTenants(t, client, tenants.Names()...) - - for _, tenant := range tenants { - for className, ids := range fixtures.IdsByClass { - for _, id := range ids { - exists, err := client.Data().Checker(). - WithID(id). - WithClassName(className). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.True(t, exists) - } - } - } - }) - - t.Run("fails checking objects of MT class without tenant", func(t *testing.T) { - defer cleanup() - - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - - fixtures.CreateSchemaFoodForTenants(t, client) - fixtures.CreateTenantsFood(t, client, tenants...) - fixtures.CreateDataFoodForTenants(t, client, tenants.Names()...) - - for className, ids := range fixtures.IdsByClass { - for _, id := range ids { - exists, err := client.Data().Checker(). - WithID(id). - WithClassName(className). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Empty(t, clientErr.Msg) // no body in HEAD - assert.False(t, exists) - } - } - }) - - t.Run("fails checking objects of MT class with non existent tenant", func(t *testing.T) { - defer cleanup() - - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - - fixtures.CreateSchemaFoodForTenants(t, client) - fixtures.CreateTenantsFood(t, client, tenants...) - fixtures.CreateDataFoodForTenants(t, client, tenants.Names()...) - - for className, ids := range fixtures.IdsByClass { - for _, id := range ids { - exists, err := client.Data().Checker(). - WithID(id). - WithClassName(className). - WithTenant("nonExistentTenant"). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Empty(t, clientErr.Msg) // no body in HEAD - assert.False(t, exists) - } - } - }) - - t.Run("fails checking objects of non-MT class when tenant given", func(t *testing.T) { - defer cleanup() - - fixtures.CreateSchemaFood(t, client) - fixtures.CreateDataFood(t, client) - - for className, ids := range fixtures.IdsByClass { - for _, id := range ids { - exists, err := client.Data().Checker(). - WithID(id). - WithClassName(className). - WithTenant("nonExistentTenant"). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Empty(t, clientErr.Msg) // no body in HEAD - require.False(t, exists) - } - } - }) - - t.Run("deletes objects from MT class", func(t *testing.T) { - defer cleanup() - - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - - fixtures.CreateSchemaFoodForTenants(t, client) - fixtures.CreateTenantsFood(t, client, tenants...) - fixtures.CreateDataFoodForTenants(t, client, tenants.Names()...) - - for _, tenant := range tenants { - for className, ids := range fixtures.IdsByClass { - expectedLeft := len(ids) - - for _, id := range ids { - err := client.Data().Deleter(). - WithID(id). - WithClassName(className). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - expectedLeft-- - - t.Run("verify deleted", func(t *testing.T) { - exists, err := client.Data().Checker(). - WithID(id). - WithClassName(className). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.False(t, exists) - }) - - t.Run("verify left", func(t *testing.T) { - objects, err := client.Data().ObjectsGetter(). - WithClassName(className). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - assert.Len(t, objects, expectedLeft) - }) - } - } - } - }) - - t.Run("fails deleting objects from MT class without tenant", func(t *testing.T) { - defer cleanup() - - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - - fixtures.CreateSchemaFoodForTenants(t, client) - fixtures.CreateTenantsFood(t, client, tenants...) - fixtures.CreateDataFoodForTenants(t, client, tenants.Names()...) - - for className, ids := range fixtures.IdsByClass { - for _, id := range ids { - err := client.Data().Deleter(). - WithID(id). - WithClassName(className). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "has multi-tenancy enabled, but request was without tenant") - - t.Run("verify not deleted", func(t *testing.T) { - for _, tenant := range tenants { - exists, err := client.Data().Checker(). - WithID(id). - WithClassName(className). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.True(t, exists) - } - }) - } - } - - t.Run("verify not deleted", func(t *testing.T) { - for _, tenant := range tenants { - objects, err := client.Data().ObjectsGetter(). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - assert.Len(t, objects, len(fixtures.AllIds)) - } - }) - }) - - t.Run("fails deleting objects from MT class with non existent tenant", func(t *testing.T) { - defer cleanup() - - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - - fixtures.CreateSchemaFoodForTenants(t, client) - fixtures.CreateTenantsFood(t, client, tenants...) - fixtures.CreateDataFoodForTenants(t, client, tenants.Names()...) - - for className, ids := range fixtures.IdsByClass { - for _, id := range ids { - err := client.Data().Deleter(). - WithID(id). - WithClassName(className). - WithTenant("nonExistentTenant"). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "tenant not found") - - t.Run("verify not deleted", func(t *testing.T) { - for _, tenant := range tenants { - exists, err := client.Data().Checker(). - WithID(id). - WithClassName(className). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.True(t, exists) - } - }) - } - } - - t.Run("verify not deleted", func(t *testing.T) { - for _, tenant := range tenants { - objects, err := client.Data().ObjectsGetter(). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - assert.Len(t, objects, len(fixtures.AllIds)) - } - }) - }) - - t.Run("fails deleting objects from non-MT class when tenant given", func(t *testing.T) { - defer cleanup() - - fixtures.CreateSchemaFood(t, client) - fixtures.CreateDataFood(t, client) - - for className, ids := range fixtures.IdsByClass { - for _, id := range ids { - err := client.Data().Deleter(). - WithID(id). - WithClassName(className). - WithTenant("nonExistentTenant"). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "has multi-tenancy disabled, but request was with tenant") - - t.Run("verify not deleted", func(t *testing.T) { - exists, err := client.Data().Checker(). - WithID(id). - WithClassName(className). - Do(context.Background()) - - require.Nil(t, err) - require.True(t, exists) - }) - } - } - - t.Run("verify not deleted", func(t *testing.T) { - objects, err := client.Data().ObjectsGetter(). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - assert.Len(t, objects, len(fixtures.AllIds)) - }) - }) - - t.Run("updates objects of MT class", func(t *testing.T) { - defer cleanup() - - className := "Soup" - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenants...) - fixtures.CreateDataSoupForTenants(t, client, tenants.Names()...) - - for _, tenant := range tenants { - err := client.Data().Updater(). - WithClassName(className). - WithID(fixtures.SOUP_CHICKENSOUP_ID). - WithProperties(map[string]interface{}{ - "name": "ChickenSoup", - "description": fmt.Sprintf("updated ChickenSoup description [%s]", tenant), - "price": float32(2.1), - }). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - - err = client.Data().Updater(). - WithClassName(className). - WithID(fixtures.SOUP_BEAUTIFUL_ID). - WithProperties(map[string]interface{}{ - "name": "Beautiful", - "description": fmt.Sprintf("updated Beautiful description [%s]", tenant), - "price": float32(2.2), - }). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - } - - t.Run("verify updated", func(t *testing.T) { - for _, tenant := range tenants { - objects, err := client.Data().ObjectsGetter(). - WithID(fixtures.SOUP_CHICKENSOUP_ID). - WithClassName(className). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Equal(t, tenant.Name, objects[0].Tenant) - assert.Equal(t, fmt.Sprintf("updated ChickenSoup description [%s]", tenant), - objects[0].Properties.(map[string]interface{})["description"]) - - objects, err = client.Data().ObjectsGetter(). - WithID(fixtures.SOUP_BEAUTIFUL_ID). - WithClassName(className). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Equal(t, tenant.Name, objects[0].Tenant) - assert.Equal(t, fmt.Sprintf("updated Beautiful description [%s]", tenant), - objects[0].Properties.(map[string]interface{})["description"]) - } - }) - }) - - t.Run("fails updating objects of MT class without tenant", func(t *testing.T) { - defer cleanup() - - className := "Soup" - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenants...) - fixtures.CreateDataSoupForTenants(t, client, tenants.Names()...) - - err := client.Data().Updater(). - WithClassName(className). - WithID(fixtures.SOUP_CHICKENSOUP_ID). - WithProperties(map[string]interface{}{ - "name": "ChickenSoup", - "description": "updated ChickenSoup description", - "price": float32(2.1), - }). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "has multi-tenancy enabled, but request was without tenant") - - err = client.Data().Updater(). - WithClassName(className). - WithID(fixtures.SOUP_BEAUTIFUL_ID). - WithProperties(map[string]interface{}{ - "name": "Beautiful", - "description": "updated Beautiful description", - "price": float32(2.2), - }). - Do(context.Background()) - - require.NotNil(t, err) - clientErr = err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "has multi-tenancy enabled, but request was without tenant") - - t.Run("verify not updated", func(t *testing.T) { - for _, tenant := range tenants { - objects, err := client.Data().ObjectsGetter(). - WithID(fixtures.SOUP_CHICKENSOUP_ID). - WithClassName(className). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Equal(t, tenant.Name, objects[0].Tenant) - assert.Equal(t, "Used by humans when their inferior genetics are attacked by microscopic organisms.", - objects[0].Properties.(map[string]interface{})["description"]) - - objects, err = client.Data().ObjectsGetter(). - WithID(fixtures.SOUP_BEAUTIFUL_ID). - WithClassName(className). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Equal(t, tenant.Name, objects[0].Tenant) - assert.Equal(t, "Putting the game of letter soups to a whole new level.", - objects[0].Properties.(map[string]interface{})["description"]) - } - }) - }) - - t.Run("fails updating objects of MT class with non existent tenant", func(t *testing.T) { - defer cleanup() - - className := "Soup" - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenants...) - fixtures.CreateDataSoupForTenants(t, client, tenants.Names()...) - - err := client.Data().Updater(). - WithClassName(className). - WithID(fixtures.SOUP_CHICKENSOUP_ID). - WithProperties(map[string]interface{}{ - "name": "ChickenSoup", - "description": "updated ChickenSoup description", - "price": float32(2.1), - }). - WithTenant("nonExistentTenant"). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "tenant not found") - - err = client.Data().Updater(). - WithClassName(className). - WithID(fixtures.SOUP_BEAUTIFUL_ID). - WithProperties(map[string]interface{}{ - "name": "Beautiful", - "description": "updated Beautiful description", - "price": float32(2.2), - }). - WithTenant("nonExistentTenant"). - Do(context.Background()) - - require.NotNil(t, err) - clientErr = err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "tenant not found") - - t.Run("verify not updated", func(t *testing.T) { - for _, tenant := range tenants { - objects, err := client.Data().ObjectsGetter(). - WithID(fixtures.SOUP_CHICKENSOUP_ID). - WithClassName(className). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Equal(t, tenant.Name, objects[0].Tenant) - assert.Equal(t, "Used by humans when their inferior genetics are attacked by microscopic organisms.", - objects[0].Properties.(map[string]interface{})["description"]) - - objects, err = client.Data().ObjectsGetter(). - WithID(fixtures.SOUP_BEAUTIFUL_ID). - WithClassName(className). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Equal(t, tenant.Name, objects[0].Tenant) - assert.Equal(t, "Putting the game of letter soups to a whole new level.", - objects[0].Properties.(map[string]interface{})["description"]) - } - }) - }) - - t.Run("fails updating objects of non-MT class when tenant given", func(t *testing.T) { - defer cleanup() - - className := "Soup" - - fixtures.CreateSchemaSoup(t, client) - fixtures.CreateDataSoup(t, client) - - err := client.Data().Updater(). - WithClassName(className). - WithID(fixtures.SOUP_CHICKENSOUP_ID). - WithProperties(map[string]interface{}{ - "name": "ChickenSoup", - "description": "updated ChickenSoup description", - "price": float32(2.1), - }). - WithTenant("nonExistentTenant"). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "has multi-tenancy disabled, but request was with tenant") - - err = client.Data().Updater(). - WithClassName(className). - WithID(fixtures.SOUP_BEAUTIFUL_ID). - WithProperties(map[string]interface{}{ - "name": "Beautiful", - "description": "updated Beautiful description", - "price": float32(2.2), - }). - WithTenant("nonExistentTenant"). - Do(context.Background()) - - require.NotNil(t, err) - clientErr = err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "has multi-tenancy disabled, but request was with tenant") - - t.Run("verify not updated", func(t *testing.T) { - objects, err := client.Data().ObjectsGetter(). - WithID(fixtures.SOUP_CHICKENSOUP_ID). - WithClassName(className). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Equal(t, "Used by humans when their inferior genetics are attacked by microscopic organisms.", - objects[0].Properties.(map[string]interface{})["description"]) - - objects, err = client.Data().ObjectsGetter(). - WithID(fixtures.SOUP_BEAUTIFUL_ID). - WithClassName(className). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Equal(t, "Putting the game of letter soups to a whole new level.", - objects[0].Properties.(map[string]interface{})["description"]) - }) - }) - - t.Run("merges objects of MT class", func(t *testing.T) { - defer cleanup() - - className := "Soup" - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenants...) - fixtures.CreateDataSoupForTenants(t, client, tenants.Names()...) - - for _, tenant := range tenants { - err := client.Data().Updater(). - WithClassName(className). - WithID(fixtures.SOUP_CHICKENSOUP_ID). - WithProperties(map[string]interface{}{ - "description": fmt.Sprintf("merged ChickenSoup description [%s]", tenant), - }). - WithTenant(tenant.Name). - WithMerge(). - Do(context.Background()) - - require.Nil(t, err) - - err = client.Data().Updater(). - WithClassName(className). - WithID(fixtures.SOUP_BEAUTIFUL_ID). - WithProperties(map[string]interface{}{ - "description": fmt.Sprintf("merged Beautiful description [%s]", tenant), - }). - WithTenant(tenant.Name). - WithMerge(). - Do(context.Background()) - - require.Nil(t, err) - } - - t.Run("verify merged", func(t *testing.T) { - for _, tenant := range tenants { - objects, err := client.Data().ObjectsGetter(). - WithID(fixtures.SOUP_CHICKENSOUP_ID). - WithClassName(className). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Equal(t, tenant.Name, objects[0].Tenant) - assert.Equal(t, fmt.Sprintf("merged ChickenSoup description [%s]", tenant), - objects[0].Properties.(map[string]interface{})["description"]) - - objects, err = client.Data().ObjectsGetter(). - WithID(fixtures.SOUP_BEAUTIFUL_ID). - WithClassName(className). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Equal(t, tenant.Name, objects[0].Tenant) - assert.Equal(t, fmt.Sprintf("merged Beautiful description [%s]", tenant), - objects[0].Properties.(map[string]interface{})["description"]) - } - }) - }) - - t.Run("fails merging objects of MT class without tenant", func(t *testing.T) { - defer cleanup() - - className := "Soup" - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenants...) - fixtures.CreateDataSoupForTenants(t, client, tenants.Names()...) - - err := client.Data().Updater(). - WithClassName(className). - WithID(fixtures.SOUP_CHICKENSOUP_ID). - WithProperties(map[string]interface{}{ - "description": "merged ChickenSoup description", - }). - WithMerge(). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "has multi-tenancy enabled, but request was without tenant") - - err = client.Data().Updater(). - WithClassName(className). - WithID(fixtures.SOUP_BEAUTIFUL_ID). - WithProperties(map[string]interface{}{ - "description": "merged Beautiful description", - }). - WithMerge(). - Do(context.Background()) - - require.NotNil(t, err) - clientErr = err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "has multi-tenancy enabled, but request was without tenant") - - t.Run("verify not merged", func(t *testing.T) { - for _, tenant := range tenants { - objects, err := client.Data().ObjectsGetter(). - WithID(fixtures.SOUP_CHICKENSOUP_ID). - WithClassName(className). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Equal(t, tenant.Name, objects[0].Tenant) - assert.Equal(t, "Used by humans when their inferior genetics are attacked by microscopic organisms.", - objects[0].Properties.(map[string]interface{})["description"]) - - objects, err = client.Data().ObjectsGetter(). - WithID(fixtures.SOUP_BEAUTIFUL_ID). - WithClassName(className). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Equal(t, tenant.Name, objects[0].Tenant) - assert.Equal(t, "Putting the game of letter soups to a whole new level.", - objects[0].Properties.(map[string]interface{})["description"]) - } - }) - }) - - t.Run("fails merging objects of MT class with non existent tenant", func(t *testing.T) { - defer cleanup() - - className := "Soup" - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenants...) - fixtures.CreateDataSoupForTenants(t, client, tenants.Names()...) - - err := client.Data().Updater(). - WithClassName(className). - WithID(fixtures.SOUP_CHICKENSOUP_ID). - WithProperties(map[string]interface{}{ - "description": "merged ChickenSoup description", - }). - WithMerge(). - WithTenant("nonExistentTenant"). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "tenant not found") - - err = client.Data().Updater(). - WithClassName(className). - WithID(fixtures.SOUP_BEAUTIFUL_ID). - WithProperties(map[string]interface{}{ - "description": "merged Beautiful description", - }). - WithMerge(). - WithTenant("nonExistentTenant"). - Do(context.Background()) - - require.NotNil(t, err) - clientErr = err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "tenant not found") - - t.Run("verify not merged", func(t *testing.T) { - for _, tenant := range tenants { - objects, err := client.Data().ObjectsGetter(). - WithID(fixtures.SOUP_CHICKENSOUP_ID). - WithClassName(className). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Equal(t, tenant.Name, objects[0].Tenant) - assert.Equal(t, "Used by humans when their inferior genetics are attacked by microscopic organisms.", - objects[0].Properties.(map[string]interface{})["description"]) - - objects, err = client.Data().ObjectsGetter(). - WithID(fixtures.SOUP_BEAUTIFUL_ID). - WithClassName(className). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Equal(t, tenant.Name, objects[0].Tenant) - assert.Equal(t, "Putting the game of letter soups to a whole new level.", - objects[0].Properties.(map[string]interface{})["description"]) - } - }) - }) - - t.Run("fails merging objects of non-MT class when tenant given", func(t *testing.T) { - defer cleanup() - - className := "Soup" - - fixtures.CreateSchemaSoup(t, client) - fixtures.CreateDataSoup(t, client) - - err := client.Data().Updater(). - WithClassName(className). - WithID(fixtures.SOUP_CHICKENSOUP_ID). - WithProperties(map[string]interface{}{ - "description": "merged ChickenSoup description [%s]", - }). - WithTenant("nonExistentTenant"). - WithMerge(). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "has multi-tenancy disabled, but request was with tenant") - - err = client.Data().Updater(). - WithClassName(className). - WithID(fixtures.SOUP_BEAUTIFUL_ID). - WithProperties(map[string]interface{}{ - "description": "merged Beautiful description [%s]", - }). - WithTenant("nonExistentTenant"). - WithMerge(). - Do(context.Background()) - - require.NotNil(t, err) - clientErr = err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "has multi-tenancy disabled, but request was with tenant") - - t.Run("verify not merged", func(t *testing.T) { - objects, err := client.Data().ObjectsGetter(). - WithID(fixtures.SOUP_CHICKENSOUP_ID). - WithClassName(className). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Equal(t, "Used by humans when their inferior genetics are attacked by microscopic organisms.", - objects[0].Properties.(map[string]interface{})["description"]) - - objects, err = client.Data().ObjectsGetter(). - WithID(fixtures.SOUP_BEAUTIFUL_ID). - WithClassName(className). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Equal(t, "Putting the game of letter soups to a whole new level.", - objects[0].Properties.(map[string]interface{})["description"]) - }) - }) -} diff --git a/test/acceptance_with_go_client/multi_tenancy_tests/graphql_test.go b/test/acceptance_with_go_client/multi_tenancy_tests/graphql_test.go deleted file mode 100644 index 79e9f9db6cad2ebfa53e885d08464e6db5baecf6..0000000000000000000000000000000000000000 --- a/test/acceptance_with_go_client/multi_tenancy_tests/graphql_test.go +++ /dev/null @@ -1,512 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package multi_tenancy_tests - -import ( - "context" - "fmt" - "testing" - - "github.com/go-openapi/strfmt" - "github.com/weaviate/weaviate/entities/schema" - - "acceptance_tests_with_client/fixtures" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - wvt "github.com/weaviate/weaviate-go-client/v4/weaviate" - "github.com/weaviate/weaviate-go-client/v4/weaviate/filters" - "github.com/weaviate/weaviate-go-client/v4/weaviate/graphql" - "github.com/weaviate/weaviate/entities/models" -) - -func TestGraphQL_MultiTenancy(t *testing.T) { - client, err := wvt.NewClient(wvt.Config{Scheme: "http", Host: "localhost:8080"}) - require.Nil(t, err) - - cleanup := func() { - err := client.Schema().AllDeleter().Do(context.Background()) - require.Nil(t, err) - } - - t.Run("GraphQL Get", func(t *testing.T) { - defer cleanup() - - tenant1 := models.Tenant{Name: "tenantNo1"} - tenant2 := models.Tenant{Name: "tenantNo2"} - - assertGetContainsIds := func(t *testing.T, response *models.GraphQLResponse, - className string, expectedIds []string, - ) { - require.NotNil(t, response) - assert.Nil(t, response.Errors) - require.NotNil(t, response.Data) - - get := response.Data["Get"].(map[string]interface{}) - objects := get[className].([]interface{}) - require.Len(t, objects, len(expectedIds)) - - ids := []string{} - for i := range objects { - ids = append(ids, objects[i].(map[string]interface{})["_additional"].(map[string]interface{})["id"].(string)) - } - assert.ElementsMatch(t, expectedIds, ids) - } - - t.Run("add data", func(t *testing.T) { - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenant1, tenant2) - fixtures.CreateDataPizzaQuattroFormaggiForTenants(t, client, tenant1.Name) - fixtures.CreateDataPizzaFruttiDiMareForTenants(t, client, tenant1.Name) - fixtures.CreateDataPizzaHawaiiForTenants(t, client, tenant2.Name) - fixtures.CreateDataPizzaDoenerForTenants(t, client, tenant2.Name) - }) - - t.Run("get all data for tenant", func(t *testing.T) { - expectedIdsByTenant := map[string][]string{ - tenant1.Name: { - fixtures.PIZZA_QUATTRO_FORMAGGI_ID, - fixtures.PIZZA_FRUTTI_DI_MARE_ID, - }, - tenant2.Name: { - fixtures.PIZZA_HAWAII_ID, - fixtures.PIZZA_DOENER_ID, - }, - } - - for tenant, expectedIds := range expectedIdsByTenant { - resp, err := client.GraphQL().Get(). - WithClassName("Pizza"). - WithTenant(tenant). - WithFields(graphql.Field{ - Name: "_additional", - Fields: []graphql.Field{{Name: "id"}}, - }). - Do(context.Background()) - - assert.Nil(t, err) - assertGetContainsIds(t, resp, "Pizza", expectedIds) - } - }) - - t.Run("get limited data for tenant", func(t *testing.T) { - expectedIdsByTenant := map[string][]string{ - tenant1.Name: { - fixtures.PIZZA_QUATTRO_FORMAGGI_ID, - }, - tenant2.Name: { - fixtures.PIZZA_HAWAII_ID, - }, - } - - for tenant, expectedIds := range expectedIdsByTenant { - resp, err := client.GraphQL().Get(). - WithClassName("Pizza"). - WithTenant(tenant). - WithLimit(1). - WithFields(graphql.Field{ - Name: "_additional", - Fields: []graphql.Field{{Name: "id"}}, - }). - Do(context.Background()) - - assert.Nil(t, err) - assertGetContainsIds(t, resp, "Pizza", expectedIds) - } - }) - - t.Run("get filtered data for tenant", func(t *testing.T) { - expectedIdsByTenant := map[string][]string{ - tenant1.Name: {}, - tenant2.Name: { - fixtures.PIZZA_DOENER_ID, - }, - } - where := filters.Where(). - WithPath([]string{"price"}). - WithOperator(filters.GreaterThan). - WithValueNumber(1.3) - - for tenant, expectedIds := range expectedIdsByTenant { - resp, err := client.GraphQL().Get(). - WithClassName("Pizza"). - WithTenant(tenant). - WithWhere(where). - WithFields(graphql.Field{ - Name: "_additional", - Fields: []graphql.Field{{Name: "id"}}, - }). - Do(context.Background()) - - assert.Nil(t, err) - assertGetContainsIds(t, resp, "Pizza", expectedIds) - } - }) - }) - - t.Run("GraphQL Get referenced class", func(t *testing.T) { - defer cleanup() - - tenant1 := models.Tenant{Name: "tenantNo1"} - tenant2 := models.Tenant{Name: "tenantNo2"} - soupIdByTenantAndPizza := map[string]map[string]string{ - tenant1.Name: { - fixtures.PIZZA_QUATTRO_FORMAGGI_ID: fixtures.SOUP_CHICKENSOUP_ID, - fixtures.PIZZA_FRUTTI_DI_MARE_ID: fixtures.SOUP_CHICKENSOUP_ID, - }, - tenant2.Name: { - fixtures.PIZZA_HAWAII_ID: fixtures.SOUP_BEAUTIFUL_ID, - fixtures.PIZZA_DOENER_ID: fixtures.SOUP_BEAUTIFUL_ID, - }, - } - - assertGetContainsIds := func(t *testing.T, response *models.GraphQLResponse, - className string, expectedIds ...string, - ) { - require.NotNil(t, response) - assert.Nil(t, response.Errors) - require.NotNil(t, response.Data) - - get := response.Data["Get"].(map[string]interface{}) - objects := get[className].([]interface{}) - require.Len(t, objects, len(expectedIds)) - - ids := []string{} - for i := range objects { - ids = append(ids, objects[i].(map[string]interface{})["_additional"].(map[string]interface{})["id"].(string)) - } - assert.ElementsMatch(t, expectedIds, ids) - } - - t.Run("add data", func(t *testing.T) { - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenant1, tenant2) - fixtures.CreateDataPizzaQuattroFormaggiForTenants(t, client, tenant1.Name) - fixtures.CreateDataPizzaFruttiDiMareForTenants(t, client, tenant1.Name) - fixtures.CreateDataPizzaHawaiiForTenants(t, client, tenant2.Name) - fixtures.CreateDataPizzaDoenerForTenants(t, client, tenant2.Name) - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenant1, tenant2) - fixtures.CreateDataSoupChickenForTenants(t, client, tenant1.Name) - fixtures.CreateDataSoupBeautifulForTenants(t, client, tenant2.Name) - }) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - t.Run("create refs", func(t *testing.T) { - references := []*models.BatchReference{} - - for tenant, pizzaToSoup := range soupIdByTenantAndPizza { - for pizzaId, soupId := range pizzaToSoup { - rpb := client.Batch().ReferencePayloadBuilder(). - WithFromClassName("Soup"). - WithFromRefProp("relatedToPizza"). - WithFromID(soupId). - WithToClassName("Pizza"). - WithToID(pizzaId). - WithTenant(tenant) - - references = append(references, rpb.Payload()) - } - } - - resp, err := client.Batch().ReferencesBatcher(). - WithReferences(references...). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, resp) - assert.Len(t, resp, len(references)) - for i := range resp { - require.NotNil(t, resp[i].Result) - assert.Nil(t, resp[i].Result.Errors) - } - }) - - for tenant, pizzaToSoup := range soupIdByTenantAndPizza { - for pizzaId, expectedSoupId := range pizzaToSoup { - resp, err := client.GraphQL().Get(). - WithClassName("Soup"). - WithTenant(tenant). - WithFields(graphql.Field{ - Name: "_additional", - Fields: []graphql.Field{{Name: "id"}}, - }). - WithWhere(filters.Where(). - WithOperator(filters.Equal). - WithPath([]string{"relatedToPizza", "Pizza", "_id"}). - WithValueText(pizzaId)). - Do(context.Background()) - - assert.Nil(t, err) - assertGetContainsIds(t, resp, "Soup", expectedSoupId) - } - } - }) - - t.Run("GraphQL Aggregate", func(t *testing.T) { - defer cleanup() - - tenant1 := models.Tenant{Name: "tenantNo1"} - tenant2 := models.Tenant{Name: "tenantNo2"} - - assertAggregateNumFieldHasValues := func(t *testing.T, response *models.GraphQLResponse, - className string, fieldName string, expectedAggValues map[string]*float64, - ) { - require.NotNil(t, response) - assert.Nil(t, response.Errors) - require.NotNil(t, response.Data) - - agg := response.Data["Aggregate"].(map[string]interface{}) - objects := agg[className].([]interface{}) - require.Len(t, objects, 1) - obj := objects[0].(map[string]interface{})[fieldName].(map[string]interface{}) - - for name, value := range expectedAggValues { - if value == nil { - assert.Nil(t, obj[name]) - } else { - assert.Equal(t, *value, obj[name]) - } - } - } - ptr := func(f float64) *float64 { - return &f - } - - t.Run("add data", func(t *testing.T) { - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenant1, tenant2) - fixtures.CreateDataPizzaQuattroFormaggiForTenants(t, client, tenant1.Name) - fixtures.CreateDataPizzaFruttiDiMareForTenants(t, client, tenant1.Name) - fixtures.CreateDataPizzaHawaiiForTenants(t, client, tenant2.Name) - fixtures.CreateDataPizzaDoenerForTenants(t, client, tenant2.Name) - }) - - t.Run("aggregate all data for tenant", func(t *testing.T) { - expectedAggValuesByTenant := map[string]map[string]*float64{ - tenant1.Name: { - "count": ptr(2), - "maximum": ptr(1.2), - "minimum": ptr(1.1), - "median": ptr(1.15), - "mean": ptr(1.15), - "mode": ptr(1.1), - "sum": ptr(2.3), - }, - tenant2.Name: { - "count": ptr(2), - "maximum": ptr(1.4), - "minimum": ptr(1.3), - "median": ptr(1.35), - "mean": ptr(1.35), - "mode": ptr(1.3), - "sum": ptr(2.7), - }, - } - - for tenant, expectedAggValues := range expectedAggValuesByTenant { - resp, err := client.GraphQL().Aggregate(). - WithClassName("Pizza"). - WithTenant(tenant). - WithFields(graphql.Field{ - Name: "price", - Fields: []graphql.Field{ - {Name: "count"}, - {Name: "maximum"}, - {Name: "minimum"}, - {Name: "median"}, - {Name: "mean"}, - {Name: "mode"}, - {Name: "sum"}, - }, - }). - Do(context.Background()) - - assert.Nil(t, err) - assertAggregateNumFieldHasValues(t, resp, "Pizza", "price", expectedAggValues) - } - }) - - t.Run("aggregate filtered data for tenant", func(t *testing.T) { - expectedAggValuesByTenant := map[string]map[string]*float64{ - tenant1.Name: { - "count": ptr(0), - "maximum": nil, - "minimum": nil, - "median": nil, - "mean": nil, - "mode": nil, - "sum": nil, - }, - tenant2.Name: { - "count": ptr(1), - "maximum": ptr(1.4), - "minimum": ptr(1.4), - "median": ptr(1.4), - "mean": ptr(1.4), - "mode": ptr(1.4), - "sum": ptr(1.4), - }, - } - where := filters.Where(). - WithPath([]string{"price"}). - WithOperator(filters.GreaterThan). - WithValueNumber(1.3) - - for tenant, expectedAggValues := range expectedAggValuesByTenant { - resp, err := client.GraphQL().Aggregate(). - WithClassName("Pizza"). - WithTenant(tenant). - WithWhere(where). - WithFields(graphql.Field{ - Name: "price", - Fields: []graphql.Field{ - {Name: "count"}, - {Name: "maximum"}, - {Name: "minimum"}, - {Name: "median"}, - {Name: "mean"}, - {Name: "mode"}, - {Name: "sum"}, - }, - }). - Do(context.Background()) - - assert.Nil(t, err) - assertAggregateNumFieldHasValues(t, resp, "Pizza", "price", expectedAggValues) - } - }) - }) - - t.Run("GraphQL Explore", func(t *testing.T) { - defer cleanup() - - tenant1 := models.Tenant{Name: "tenantNo1"} - tenant2 := models.Tenant{Name: "tenantNo2"} - - assertExploreContainsErrors := func(t *testing.T, response *models.GraphQLResponse, - expectedErrorMessage string, - ) { - require.NotNil(t, response) - require.NotNil(t, response.Errors) - require.Nil(t, response.Data["Explore"]) - require.NotNil(t, response.Data) - require.Len(t, response.Errors, 1) - assert.NotEmpty(t, response.Errors[0].Message) - assert.Equal(t, expectedErrorMessage, response.Errors[0].Message) - } - - t.Run("add data", func(t *testing.T) { - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenant1, tenant2) - fixtures.CreateDataPizzaQuattroFormaggiForTenants(t, client, tenant1.Name) - fixtures.CreateDataPizzaFruttiDiMareForTenants(t, client, tenant1.Name) - fixtures.CreateDataPizzaHawaiiForTenants(t, client, tenant2.Name) - fixtures.CreateDataPizzaDoenerForTenants(t, client, tenant2.Name) - }) - - t.Run("explore with nearText", func(t *testing.T) { - nearText := client.GraphQL().NearTextArgBuilder(). - WithConcepts([]string{"Italian"}) - - resp, err := client.GraphQL().Explore(). - WithNearText(nearText). - WithFields(graphql.Beacon, graphql.Certainty, graphql.ClassName). - Do(context.Background()) - - require.Nil(t, err) - assertExploreContainsErrors(t, resp, - "vector search: search index pizza: class Pizza has multi-tenancy enabled, but request was without tenant", - ) - }) - }) -} - -func TestGroupByMultiTenancy(t *testing.T) { - client, err := wvt.NewClient(wvt.Config{Scheme: "http", Host: "localhost:8080"}) - require.Nil(t, err) - - ctx := context.Background() - - defer client.Schema().ClassDeleter().WithClassName("TextContent").Do(ctx) - defer client.Schema().ClassDeleter().WithClassName("Document").Do(ctx) - - err = client.Schema().ClassCreator().WithClass( - &models.Class{ - Class: "TextContent", - MultiTenancyConfig: &models.MultiTenancyConfig{Enabled: true}, - Vectorizer: "text2vec-contextionary", - Properties: []*models.Property{ - { - Name: "text", - DataType: schema.DataTypeText.PropString(), - }, - }, - }, - ).Do(ctx) - require.Nil(t, err) - - err = client.Schema().ClassCreator().WithClass( - &models.Class{ - Class: "Document", - MultiTenancyConfig: &models.MultiTenancyConfig{Enabled: true}, - Vectorizer: "text2vec-contextionary", - Properties: []*models.Property{ - { - Name: "textContents", - DataType: []string{"TextContent"}, - }, - { - Name: "title", - DataType: schema.DataTypeText.PropString(), - }, - }, - }, - ).Do(ctx) - require.Nil(t, err) - - require.Nil(t, client.Schema().PropertyCreator().WithClassName("TextContent").WithProperty(&models.Property{ - Name: "contentOf", - DataType: []string{"Document"}, - }).Do(ctx)) - - require.Nil(t, client.Schema().TenantsCreator().WithClassName("TextContent").WithTenants(models.Tenant{Name: "1"}).Do(ctx)) - require.Nil(t, client.Schema().TenantsCreator().WithClassName("Document").WithTenants(models.Tenant{Name: "1"}).Do(ctx)) - - docCreator := client.Data().Creator().WithClassName("Document").WithTenant("1") - doc1, err := docCreator.WithProperties(map[string]interface{}{"title": "Foo"}).Do(ctx) - require.Nil(t, err) - - textCreator := client.Data().Creator().WithClassName("TextContent").WithTenant("1") - text1, err := textCreator.WithProperties(map[string]interface{}{"text": "Text from Foo"}).Do(ctx) - require.Nil(t, err) - - require.Nil(t, client.Data().ReferenceCreator().WithTenant("1").WithClassName("Document").WithID(doc1.Object.ID.String()).WithReferenceProperty("textContents").WithReference(&models.SingleRef{Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/TextContent/%s", text1.Object.ID.String()))}).Do(ctx)) - require.Nil(t, client.Data().ReferenceCreator().WithTenant("1").WithClassName("TextContent").WithID(text1.Object.ID.String()).WithReferenceProperty("contentOf").WithReference(&models.SingleRef{Beacon: strfmt.URI(fmt.Sprintf("weaviate://localhost/Document/%s", doc1.Object.ID.String()))}).Do(ctx)) - - result, err := client.GraphQL().Raw().WithQuery("{Get{TextContent(nearText: {concepts: [\"Foo\"] distance: 1.0} groupBy:{path:[\"contentOf\"], groups:2, objectsPerGroup:1}tenant: \"1\"){contentOf{... on Document{title}}}}}").Do(ctx) - require.Nil(t, err) - require.NotNil(t, result) - require.Nil(t, result.Errors) -} diff --git a/test/acceptance_with_go_client/multi_tenancy_tests/reference_test.go b/test/acceptance_with_go_client/multi_tenancy_tests/reference_test.go deleted file mode 100644 index acf1edfea66da47c2022a9ff0d67472068bd0596..0000000000000000000000000000000000000000 --- a/test/acceptance_with_go_client/multi_tenancy_tests/reference_test.go +++ /dev/null @@ -1,3874 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package multi_tenancy_tests - -import ( - "context" - "fmt" - "strings" - "testing" - - "acceptance_tests_with_client/fixtures" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - wvt "github.com/weaviate/weaviate-go-client/v4/weaviate" - "github.com/weaviate/weaviate-go-client/v4/weaviate/fault" - "github.com/weaviate/weaviate-go-client/v4/weaviate/filters" - "github.com/weaviate/weaviate-go-client/v4/weaviate/graphql" - "github.com/weaviate/weaviate/entities/models" -) - -func TestDataReference_MultiTenancy(t *testing.T) { - client, err := wvt.NewClient(wvt.Config{Scheme: "http", Host: "localhost:8080"}) - require.Nil(t, err) - - cleanup := func() { - err := client.Schema().AllDeleter().Do(context.Background()) - require.Nil(t, err) - } - - t.Run("creates references between MT classes", func(t *testing.T) { - defer cleanup() - - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - soupIds := fixtures.IdsByClass["Soup"] - pizzaIds := fixtures.IdsByClass["Pizza"] - pizzaBeacons := fixtures.BeaconsByClass["Pizza"] - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenants...) - fixtures.CreateDataSoupForTenants(t, client, tenants.Names()...) - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenants...) - fixtures.CreateDataPizzaForTenants(t, client, tenants.Names()...) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - for _, tenant := range tenants { - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - ref := client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload() - - err := client.Data().ReferenceCreator(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReference(ref). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - } - } - } - - for _, tenant := range tenants { - _, err := client.Data().Creator(). - WithClassName("Soup"). - WithID(fixtures.SOUP_TRIPE_ID). - WithProperties(map[string]interface{}{ - "name": "Tripe", - "description": "Tripe soup is a speciality of Romanian cuisine where it is known as Ciorbă de Burtă.", - "price": float32(2.3), - "relatedToPizza": pizzaBeacons, - }). - WithTenant(tenant.Name). - Do(context.Background()) - require.Nil(t, err) - } - allSoupIds := append(soupIds, fixtures.SOUP_TRIPE_ID) - - t.Run("verify created", func(t *testing.T) { - for _, tenant := range tenants { - for _, soupId := range allSoupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Len(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"].([]interface{}), - len(pizzaIds)) - } - } - }) - - t.Run("verify graphql search", func(t *testing.T) { - for _, tenant := range tenants { - resp, err := client.GraphQL().Get(). - WithClassName("Soup"). - WithTenant(tenant.Name). - WithFields(graphql.Field{ - Name: "_additional", Fields: []graphql.Field{{Name: "id"}}, - }). - WithWhere(filters.Where(). - WithPath([]string{"relatedToPizza", "Pizza", "name"}). - WithOperator(filters.Equal). - WithValueString("Quattro Formaggi")). - Do(context.Background()) - - require.NoError(t, err) - assertGraphqlGetIds(t, resp, "Soup", allSoupIds) - } - }) - }) - - t.Run("fails creating references between MT classes without tenant", func(t *testing.T) { - defer cleanup() - - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - soupIds := fixtures.IdsByClass["Soup"] - pizzaIds := fixtures.IdsByClass["Pizza"] - pizzaBeacons := fixtures.BeaconsByClass["Pizza"] - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenants...) - fixtures.CreateDataSoupForTenants(t, client, tenants.Names()...) - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenants...) - fixtures.CreateDataPizzaForTenants(t, client, tenants.Names()...) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - ref := client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload() - - err := client.Data().ReferenceCreator(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReference(ref). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "has multi-tenancy enabled, but request was without tenant") - } - } - - _, err := client.Data().Creator(). - WithClassName("Soup"). - WithID(fixtures.SOUP_TRIPE_ID). - WithProperties(map[string]interface{}{ - "name": "Tripe", - "description": "Tripe soup is a speciality of Romanian cuisine where it is known as Ciorbă de Burtă.", - "price": float32(2.3), - "relatedToPizza": pizzaBeacons, - }). - Do(context.Background()) - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "has multi-tenancy enabled, but request was without tenant") - - t.Run("verify not created", func(t *testing.T) { - for _, tenant := range tenants { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Nil(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"]) - } - _, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(fixtures.SOUP_TRIPE_ID). - WithTenant(tenant.Name). - Do(context.Background()) - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 404, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "") - } - }) - }) - - t.Run("fails creating references between MT classes with non existent tenant", func(t *testing.T) { - defer cleanup() - - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - soupIds := fixtures.IdsByClass["Soup"] - pizzaIds := fixtures.IdsByClass["Pizza"] - pizzaBeacons := fixtures.BeaconsByClass["Pizza"] - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenants...) - fixtures.CreateDataSoupForTenants(t, client, tenants.Names()...) - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenants...) - fixtures.CreateDataPizzaForTenants(t, client, tenants.Names()...) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - ref := client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload() - - err := client.Data().ReferenceCreator(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReference(ref). - WithTenant("nonExistentTenant"). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "tenant not found") - } - } - - _, err := client.Data().Creator(). - WithClassName("Soup"). - WithID(fixtures.SOUP_TRIPE_ID). - WithProperties(map[string]interface{}{ - "name": "Tripe", - "description": "Tripe soup is a speciality of Romanian cuisine where it is known as Ciorbă de Burtă.", - "price": float32(2.3), - "relatedToPizza": pizzaBeacons, - }). - WithTenant("nonExistentTenant"). - Do(context.Background()) - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "tenant not found") - - t.Run("verify not created", func(t *testing.T) { - for _, tenant := range tenants { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Nil(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"]) - } - _, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(fixtures.SOUP_TRIPE_ID). - WithTenant(tenant.Name). - Do(context.Background()) - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 404, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "") - } - }) - }) - - t.Run("fails creating references between MT classes with different existing tenant", func(t *testing.T) { - defer cleanup() - - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - soupIds := fixtures.IdsByClass["Soup"] - pizzaIds := fixtures.IdsByClass["Pizza"] - pizzaBeacons := fixtures.BeaconsByClass["Pizza"] - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenants...) - fixtures.CreateDataSoupForTenants(t, client, tenants[0].Name) - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenants...) - fixtures.CreateDataPizzaForTenants(t, client, tenants[0].Name) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - ref := client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload() - - err := client.Data().ReferenceCreator(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReference(ref). - WithTenant(tenants[1].Name). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "no object with id") - } - } - - _, err := client.Data().Creator(). - WithClassName("Soup"). - WithID(fixtures.SOUP_TRIPE_ID). - WithProperties(map[string]interface{}{ - "name": "Tripe", - "description": "Tripe soup is a speciality of Romanian cuisine where it is known as Ciorbă de Burtă.", - "price": float32(2.3), - "relatedToPizza": pizzaBeacons, - }). - WithTenant(tenants[1].Name). - Do(context.Background()) - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "no object with id") - - t.Run("verify not created", func(t *testing.T) { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenants[0].Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Nil(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"]) - } - _, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(fixtures.SOUP_TRIPE_ID). - WithTenant(tenants[0].Name). - Do(context.Background()) - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 404, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "") - }) - - t.Run("verify new objects not created", func(t *testing.T) { - for _, soupId := range soupIds { - exists, err := client.Data().Checker(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenants[1].Name). - Do(context.Background()) - - require.Nil(t, err) - assert.False(t, exists) - } - - for _, pizzaId := range pizzaIds { - exists, err := client.Data().Checker(). - WithClassName("Pizza"). - WithID(pizzaId). - WithTenant(tenants[1].Name). - Do(context.Background()) - - require.Nil(t, err) - assert.False(t, exists) - } - }) - }) - - t.Run("fails creating references between MT classes and different tenants", func(t *testing.T) { - tenantPizza := models.Tenant{Name: "tenantPizza"} - tenantSoup := models.Tenant{Name: "tenantSoup"} - soupIds := fixtures.IdsByClass["Soup"] - pizzaIds := fixtures.IdsByClass["Pizza"] - - t.Run("with SRC tenant (common tenants)", func(t *testing.T) { - defer cleanup() - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenantPizza, tenantSoup) - fixtures.CreateDataSoupForTenants(t, client, tenantSoup.Name) - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenantPizza, tenantSoup) - fixtures.CreateDataPizzaForTenants(t, client, tenantPizza.Name) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - ref := client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload() - - err := client.Data().ReferenceCreator(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReference(ref). - WithTenant(tenantSoup.Name). // SRC tenant - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "no object with id") - } - } - - t.Run("verify not created", func(t *testing.T) { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Nil(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"]) - } - }) - - t.Run("verify new objects not created", func(t *testing.T) { - for _, soupId := range soupIds { - exists, err := client.Data().Checker(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantPizza.Name). - Do(context.Background()) - - require.Nil(t, err) - assert.False(t, exists) - } - - for _, pizzaId := range pizzaIds { - exists, err := client.Data().Checker(). - WithClassName("Pizza"). - WithID(pizzaId). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.Nil(t, err) - assert.False(t, exists) - } - }) - }) - - t.Run("with SRC tenant (separate tenants)", func(t *testing.T) { - defer cleanup() - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenantSoup) - fixtures.CreateDataSoupForTenants(t, client, tenantSoup.Name) - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenantPizza) - fixtures.CreateDataPizzaForTenants(t, client, tenantPizza.Name) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - ref := client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload() - - err := client.Data().ReferenceCreator(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReference(ref). - WithTenant(tenantSoup.Name). // SRC tenant - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "tenant not found") - } - } - - t.Run("verify not created", func(t *testing.T) { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Nil(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"]) - } - }) - - t.Run("verify new objects not created", func(t *testing.T) { - for _, soupId := range soupIds { - exists, err := client.Data().Checker(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantPizza.Name). - Do(context.Background()) - - require.NotNil(t, err) - assert.False(t, exists) - } - - for _, pizzaId := range pizzaIds { - exists, err := client.Data().Checker(). - WithClassName("Pizza"). - WithID(pizzaId). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.NotNil(t, err) - assert.False(t, exists) - } - }) - }) - - t.Run("with DEST tenant (common tenants)", func(t *testing.T) { - defer cleanup() - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenantPizza, tenantSoup) - fixtures.CreateDataSoupForTenants(t, client, tenantSoup.Name) - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenantPizza, tenantSoup) - fixtures.CreateDataPizzaForTenants(t, client, tenantPizza.Name) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - ref := client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload() - - err := client.Data().ReferenceCreator(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReference(ref). - WithTenant(tenantPizza.Name). // DEST tenant - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 404, clientErr.StatusCode) - assert.Empty(t, clientErr.Msg) - } - } - - t.Run("verify not created", func(t *testing.T) { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Nil(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"]) - } - }) - - t.Run("verify new objects not created", func(t *testing.T) { - for _, soupId := range soupIds { - exists, err := client.Data().Checker(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantPizza.Name). - Do(context.Background()) - - require.Nil(t, err) - assert.False(t, exists) - } - - for _, pizzaId := range pizzaIds { - exists, err := client.Data().Checker(). - WithClassName("Pizza"). - WithID(pizzaId). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.Nil(t, err) - assert.False(t, exists) - } - }) - }) - - t.Run("with DEST tenant (separate tenants)", func(t *testing.T) { - defer cleanup() - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenantSoup) - fixtures.CreateDataSoupForTenants(t, client, tenantSoup.Name) - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenantPizza) - fixtures.CreateDataPizzaForTenants(t, client, tenantPizza.Name) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - ref := client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload() - - err := client.Data().ReferenceCreator(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReference(ref). - WithTenant(tenantPizza.Name). // DEST tenant - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "tenant not found") - } - } - - t.Run("verify not created", func(t *testing.T) { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Nil(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"]) - } - }) - - t.Run("verify new objects not created", func(t *testing.T) { - for _, soupId := range soupIds { - exists, err := client.Data().Checker(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantPizza.Name). - Do(context.Background()) - - require.NotNil(t, err) - assert.False(t, exists) - } - - for _, pizzaId := range pizzaIds { - exists, err := client.Data().Checker(). - WithClassName("Pizza"). - WithID(pizzaId). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.NotNil(t, err) - assert.False(t, exists) - } - }) - }) - }) - - t.Run("deletes references between MT classes", func(t *testing.T) { - defer cleanup() - - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - pizzaIds := fixtures.IdsByClass["Pizza"] - soupIds := fixtures.IdsByClass["Soup"] - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenants...) - fixtures.CreateDataSoupForTenants(t, client, tenants.Names()...) - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenants...) - fixtures.CreateDataPizzaForTenants(t, client, tenants.Names()...) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - t.Run("create refs", func(t *testing.T) { - references := []*models.BatchReference{} - - for _, tenant := range tenants { - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - rpb := client.Batch().ReferencePayloadBuilder(). - WithFromClassName("Soup"). - WithFromID(soupId). - WithFromRefProp("relatedToPizza"). - WithToClassName("Pizza"). - WithToID(pizzaId). - WithTenant(tenant.Name) - - references = append(references, rpb.Payload()) - } - } - } - - resp, err := client.Batch().ReferencesBatcher(). - WithReferences(references...). - Do(context.Background()) - - require.Nil(t, err) - require.Len(t, resp, len(references)) - for i := range resp { - assert.Nil(t, resp[i].Result.Errors) - } - }) - - for _, tenant := range tenants { - for _, soupId := range soupIds { - expectedRefsLeft := len(pizzaIds) - - for _, pizzaId := range pizzaIds { - ref := client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload() - - err := client.Data().ReferenceDeleter(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReference(ref). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - - t.Run("verify deleted one by one", func(t *testing.T) { - expectedRefsLeft-- - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Len(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"].([]interface{}), - expectedRefsLeft) - }) - } - } - } - }) - - t.Run("fails deleting references between MT classes without tenant", func(t *testing.T) { - defer cleanup() - - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - pizzaIds := fixtures.IdsByClass["Pizza"] - soupIds := fixtures.IdsByClass["Soup"] - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenants...) - fixtures.CreateDataSoupForTenants(t, client, tenants.Names()...) - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenants...) - fixtures.CreateDataPizzaForTenants(t, client, tenants.Names()...) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - t.Run("create refs", func(t *testing.T) { - references := []*models.BatchReference{} - - for _, tenant := range tenants { - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - rpb := client.Batch().ReferencePayloadBuilder(). - WithFromClassName("Soup"). - WithFromID(soupId). - WithFromRefProp("relatedToPizza"). - WithToClassName("Pizza"). - WithToID(pizzaId). - WithTenant(tenant.Name) - - references = append(references, rpb.Payload()) - } - } - } - - resp, err := client.Batch().ReferencesBatcher(). - WithReferences(references...). - Do(context.Background()) - - require.Nil(t, err) - require.Len(t, resp, len(references)) - for i := range resp { - assert.Nil(t, resp[i].Result.Errors) - } - }) - - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - ref := client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload() - - err := client.Data().ReferenceDeleter(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReference(ref). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "has multi-tenancy enabled, but request was without tenant") - } - } - - t.Run("verify not deleted", func(t *testing.T) { - for _, tenant := range tenants { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Len(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"].([]interface{}), - len(pizzaIds)) - } - } - }) - }) - - t.Run("fails deleting references between MT classes with non existent tenant", func(t *testing.T) { - defer cleanup() - - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - pizzaIds := fixtures.IdsByClass["Pizza"] - soupIds := fixtures.IdsByClass["Soup"] - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenants...) - fixtures.CreateDataSoupForTenants(t, client, tenants.Names()...) - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenants...) - fixtures.CreateDataPizzaForTenants(t, client, tenants.Names()...) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - t.Run("create refs", func(t *testing.T) { - references := []*models.BatchReference{} - - for _, tenant := range tenants { - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - rpb := client.Batch().ReferencePayloadBuilder(). - WithFromClassName("Soup"). - WithFromID(soupId). - WithFromRefProp("relatedToPizza"). - WithToClassName("Pizza"). - WithToID(pizzaId). - WithTenant(tenant.Name) - - references = append(references, rpb.Payload()) - } - } - } - - resp, err := client.Batch().ReferencesBatcher(). - WithReferences(references...). - Do(context.Background()) - - require.Nil(t, err) - require.Len(t, resp, len(references)) - for i := range resp { - assert.Nil(t, resp[i].Result.Errors) - } - }) - - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - ref := client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload() - - err := client.Data().ReferenceDeleter(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReference(ref). - WithTenant("nonExistentTenant"). - Do(context.Background()) - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "tenant not found") - } - } - - t.Run("verify not deleted", func(t *testing.T) { - for _, tenant := range tenants { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Len(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"].([]interface{}), - len(pizzaIds)) - } - } - }) - }) - - t.Run("fails deleting references between MT classes with different existing tenant", func(t *testing.T) { - defer cleanup() - - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - pizzaIds := fixtures.IdsByClass["Pizza"] - soupIds := fixtures.IdsByClass["Soup"] - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenants...) - fixtures.CreateDataSoupForTenants(t, client, tenants[0].Name) - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenants...) - fixtures.CreateDataPizzaForTenants(t, client, tenants[0].Name) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - t.Run("create refs", func(t *testing.T) { - references := []*models.BatchReference{} - - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - rpb := client.Batch().ReferencePayloadBuilder(). - WithFromClassName("Soup"). - WithFromID(soupId). - WithFromRefProp("relatedToPizza"). - WithToClassName("Pizza"). - WithToID(pizzaId). - WithTenant(tenants[0].Name) - - references = append(references, rpb.Payload()) - } - } - - resp, err := client.Batch().ReferencesBatcher(). - WithReferences(references...). - Do(context.Background()) - - require.Nil(t, err) - require.Len(t, resp, len(references)) - for i := range resp { - assert.Nil(t, resp[i].Result.Errors) - } - }) - - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - ref := client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload() - - err := client.Data().ReferenceDeleter(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReference(ref). - WithTenant(tenants[1].Name). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 404, clientErr.StatusCode) - assert.Empty(t, clientErr.Msg) - } - } - - t.Run("verify not deleted", func(t *testing.T) { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenants[0].Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Len(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"].([]interface{}), - len(pizzaIds)) - } - }) - }) - - t.Run("replaces references between MT classes", func(t *testing.T) { - defer cleanup() - - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - soupIds := fixtures.IdsByClass["Soup"] - pizzaIdsBefore := fixtures.IdsByClass["Pizza"][:2] - pizzaIdsAfter := fixtures.IdsByClass["Pizza"][2:] - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenants...) - fixtures.CreateDataSoupForTenants(t, client, tenants.Names()...) - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenants...) - fixtures.CreateDataPizzaForTenants(t, client, tenants.Names()...) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - t.Run("create refs", func(t *testing.T) { - references := []*models.BatchReference{} - - for _, tenant := range tenants { - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIdsBefore { - rpb := client.Batch().ReferencePayloadBuilder(). - WithFromClassName("Soup"). - WithFromID(soupId). - WithFromRefProp("relatedToPizza"). - WithToClassName("Pizza"). - WithToID(pizzaId). - WithTenant(tenant.Name) - - references = append(references, rpb.Payload()) - } - } - } - - resp, err := client.Batch().ReferencesBatcher(). - WithReferences(references...). - Do(context.Background()) - - require.Nil(t, err) - require.Len(t, resp, len(references)) - for i := range resp { - assert.Nil(t, resp[i].Result.Errors) - } - }) - - for _, tenant := range tenants { - for _, soupId := range soupIds { - var refs models.MultipleRef - for _, pizzaId := range pizzaIdsAfter { - ref := client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload() - refs = append(refs, ref) - } - - err := client.Data().ReferenceReplacer(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReferences(&refs). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - } - } - - t.Run("verify replaced", func(t *testing.T) { - for _, tenant := range tenants { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - require.Len(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"].([]interface{}), - len(pizzaIdsAfter)) - - for _, pizzaId := range pizzaIdsAfter { - found := false - for _, ref := range objects[0].Properties.(map[string]interface{})["relatedToPizza"].([]interface{}) { - if strings.Contains(ref.(map[string]interface{})["beacon"].(string), pizzaId) { - found = true - break - } - } - assert.True(t, found, fmt.Sprintf("ref to '%s' not found", pizzaId)) - } - } - } - }) - - t.Run("verify graphql search", func(t *testing.T) { - for _, tenant := range tenants { - resp, err := client.GraphQL().Get(). - WithClassName("Soup"). - WithTenant(tenant.Name). - WithFields(graphql.Field{ - Name: "_additional", Fields: []graphql.Field{{Name: "id"}}, - }). - WithWhere(filters.Where(). - WithPath([]string{"relatedToPizza", "Pizza", "name"}). - WithOperator(filters.Equal). - WithValueString("Quattro Formaggi")). - Do(context.Background()) - - require.NoError(t, err) - assertGraphqlGetIds(t, resp, "Soup", []string{}) - } - }) - }) - - t.Run("fails replacing references between MT classes without tenant", func(t *testing.T) { - defer cleanup() - - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - soupIds := fixtures.IdsByClass["Soup"] - pizzaIdsBefore := fixtures.IdsByClass["Pizza"][:2] - pizzaIdsAfter := fixtures.IdsByClass["Pizza"][2:] - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenants...) - fixtures.CreateDataSoupForTenants(t, client, tenants.Names()...) - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenants...) - fixtures.CreateDataPizzaForTenants(t, client, tenants.Names()...) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - t.Run("create refs", func(t *testing.T) { - references := []*models.BatchReference{} - - for _, tenant := range tenants { - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIdsBefore { - rpb := client.Batch().ReferencePayloadBuilder(). - WithFromClassName("Soup"). - WithFromID(soupId). - WithFromRefProp("relatedToPizza"). - WithToClassName("Pizza"). - WithToID(pizzaId). - WithTenant(tenant.Name) - - references = append(references, rpb.Payload()) - } - } - } - - resp, err := client.Batch().ReferencesBatcher(). - WithReferences(references...). - Do(context.Background()) - - require.Nil(t, err) - require.Len(t, resp, len(references)) - for i := range resp { - assert.Nil(t, resp[i].Result.Errors) - } - }) - - for _, soupId := range soupIds { - var refs models.MultipleRef - for _, pizzaId := range pizzaIdsAfter { - ref := client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload() - refs = append(refs, ref) - } - - err := client.Data().ReferenceReplacer(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReferences(&refs). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "has multi-tenancy enabled, but request was without tenant") - } - - t.Run("verify not replaced", func(t *testing.T) { - for _, tenant := range tenants { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - require.Len(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"].([]interface{}), - len(pizzaIdsBefore)) - - for _, pizzaId := range pizzaIdsBefore { - found := false - for _, ref := range objects[0].Properties.(map[string]interface{})["relatedToPizza"].([]interface{}) { - if strings.Contains(ref.(map[string]interface{})["beacon"].(string), pizzaId) { - found = true - break - } - } - assert.True(t, found, fmt.Sprintf("ref to '%s' not found", pizzaId)) - } - } - } - }) - }) - - t.Run("fails replacing references between MT classes with non existent tenant", func(t *testing.T) { - defer cleanup() - - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - soupIds := fixtures.IdsByClass["Soup"] - pizzaIdsBefore := fixtures.IdsByClass["Pizza"][:2] - pizzaIdsAfter := fixtures.IdsByClass["Pizza"][2:] - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenants...) - fixtures.CreateDataSoupForTenants(t, client, tenants.Names()...) - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenants...) - fixtures.CreateDataPizzaForTenants(t, client, tenants.Names()...) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - t.Run("create refs", func(t *testing.T) { - references := []*models.BatchReference{} - - for _, tenant := range tenants { - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIdsBefore { - rpb := client.Batch().ReferencePayloadBuilder(). - WithFromClassName("Soup"). - WithFromID(soupId). - WithFromRefProp("relatedToPizza"). - WithToClassName("Pizza"). - WithToID(pizzaId). - WithTenant(tenant.Name) - - references = append(references, rpb.Payload()) - } - } - } - - resp, err := client.Batch().ReferencesBatcher(). - WithReferences(references...). - Do(context.Background()) - - require.Nil(t, err) - require.Len(t, resp, len(references)) - for i := range resp { - assert.Nil(t, resp[i].Result.Errors) - } - }) - - for _, soupId := range soupIds { - var refs models.MultipleRef - for _, pizzaId := range pizzaIdsAfter { - ref := client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload() - refs = append(refs, ref) - } - - err := client.Data().ReferenceReplacer(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReferences(&refs). - WithTenant("nonExistentTenant"). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "tenant not found") - } - - t.Run("verify not replaced", func(t *testing.T) { - for _, tenant := range tenants { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenant.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - require.Len(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"].([]interface{}), - len(pizzaIdsBefore)) - - for _, pizzaId := range pizzaIdsBefore { - found := false - for _, ref := range objects[0].Properties.(map[string]interface{})["relatedToPizza"].([]interface{}) { - if strings.Contains(ref.(map[string]interface{})["beacon"].(string), pizzaId) { - found = true - break - } - } - assert.True(t, found, fmt.Sprintf("ref to '%s' not found", pizzaId)) - } - } - } - }) - }) - - t.Run("fails replacing references between MT classes with different existing tenant", func(t *testing.T) { - defer cleanup() - - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - soupIds := fixtures.IdsByClass["Soup"] - pizzaIdsBefore := fixtures.IdsByClass["Pizza"][:2] - pizzaIdsAfter := fixtures.IdsByClass["Pizza"][2:] - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenants...) - fixtures.CreateDataSoupForTenants(t, client, tenants[0].Name) - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenants...) - fixtures.CreateDataPizzaForTenants(t, client, tenants[0].Name) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - t.Run("create refs", func(t *testing.T) { - references := []*models.BatchReference{} - - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIdsBefore { - rpb := client.Batch().ReferencePayloadBuilder(). - WithFromClassName("Soup"). - WithFromID(soupId). - WithFromRefProp("relatedToPizza"). - WithToClassName("Pizza"). - WithToID(pizzaId). - WithTenant(tenants[0].Name) - - references = append(references, rpb.Payload()) - } - } - - resp, err := client.Batch().ReferencesBatcher(). - WithReferences(references...). - Do(context.Background()) - - require.Nil(t, err) - require.Len(t, resp, len(references)) - for i := range resp { - assert.Nil(t, resp[i].Result.Errors) - } - }) - - for _, soupId := range soupIds { - var refs models.MultipleRef - for _, pizzaId := range pizzaIdsAfter { - ref := client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload() - refs = append(refs, ref) - } - - err := client.Data().ReferenceReplacer(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReferences(&refs). - WithTenant(tenants[1].Name). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 404, clientErr.StatusCode) - assert.Empty(t, clientErr.Msg) - } - - t.Run("verify not replaced", func(t *testing.T) { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenants[0].Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - require.Len(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"].([]interface{}), - len(pizzaIdsBefore)) - - for _, pizzaId := range pizzaIdsBefore { - found := false - for _, ref := range objects[0].Properties.(map[string]interface{})["relatedToPizza"].([]interface{}) { - if strings.Contains(ref.(map[string]interface{})["beacon"].(string), pizzaId) { - found = true - break - } - } - assert.True(t, found, fmt.Sprintf("ref to '%s' not found", pizzaId)) - } - } - }) - - t.Run("verify new objects not created", func(t *testing.T) { - for _, soupId := range soupIds { - exists, err := client.Data().Checker(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenants[1].Name). - Do(context.Background()) - - require.Nil(t, err) - assert.False(t, exists) - } - - for _, pizzaId := range fixtures.IdsByClass["Pizza"] { - exists, err := client.Data().Checker(). - WithClassName("Pizza"). - WithID(pizzaId). - WithTenant(tenants[1].Name). - Do(context.Background()) - - require.Nil(t, err) - assert.False(t, exists) - } - }) - }) - - t.Run("fails replacing references between MT classes and different tenants", func(t *testing.T) { - tenantPizza := models.Tenant{Name: "tenantPizza"} - tenantSoup := models.Tenant{Name: "tenantSoup"} - soupIds := fixtures.IdsByClass["Soup"] - pizzaIds := fixtures.IdsByClass["Pizza"] - - t.Run("with SRC tenant (common tenants)", func(t *testing.T) { - defer cleanup() - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenantPizza, tenantSoup) - fixtures.CreateDataSoupForTenants(t, client, tenantSoup.Name) - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenantPizza, tenantSoup) - fixtures.CreateDataPizzaForTenants(t, client, tenantPizza.Name) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - refs := models.MultipleRef{ - client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload(), - } - - err := client.Data().ReferenceReplacer(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReferences(&refs). - WithTenant(tenantSoup.Name). // SRC tenant - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "no object with id") - } - } - - t.Run("verify not replaced", func(t *testing.T) { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Nil(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"]) - } - }) - - t.Run("verify new objects not created", func(t *testing.T) { - for _, soupId := range soupIds { - exists, err := client.Data().Checker(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantPizza.Name). - Do(context.Background()) - - require.Nil(t, err) - assert.False(t, exists) - } - - for _, pizzaId := range pizzaIds { - exists, err := client.Data().Checker(). - WithClassName("Pizza"). - WithID(pizzaId). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.Nil(t, err) - assert.False(t, exists) - } - }) - }) - - t.Run("with SRC tenant (separate tenants)", func(t *testing.T) { - defer cleanup() - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenantSoup) - fixtures.CreateDataSoupForTenants(t, client, tenantSoup.Name) - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenantPizza) - fixtures.CreateDataPizzaForTenants(t, client, tenantPizza.Name) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - refs := models.MultipleRef{ - client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload(), - } - - err := client.Data().ReferenceReplacer(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReferences(&refs). - WithTenant(tenantSoup.Name). // SRC tenant - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "tenant not found") - } - } - - t.Run("verify not replaced", func(t *testing.T) { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Nil(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"]) - } - }) - - t.Run("verify new objects not created", func(t *testing.T) { - for _, soupId := range soupIds { - exists, err := client.Data().Checker(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantPizza.Name). - Do(context.Background()) - - require.NotNil(t, err) - assert.False(t, exists) - } - - for _, pizzaId := range pizzaIds { - exists, err := client.Data().Checker(). - WithClassName("Pizza"). - WithID(pizzaId). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.NotNil(t, err) - assert.False(t, exists) - } - }) - }) - - t.Run("with DEST tenant (common tenants)", func(t *testing.T) { - defer cleanup() - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenantPizza, tenantSoup) - fixtures.CreateDataSoupForTenants(t, client, tenantSoup.Name) - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenantPizza, tenantSoup) - fixtures.CreateDataPizzaForTenants(t, client, tenantPizza.Name) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - refs := models.MultipleRef{ - client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload(), - } - - err := client.Data().ReferenceReplacer(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReferences(&refs). - WithTenant(tenantPizza.Name). // DEST tenant - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 404, clientErr.StatusCode) - assert.Empty(t, clientErr.Msg) - } - } - - t.Run("verify not replaced", func(t *testing.T) { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Nil(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"]) - } - }) - - t.Run("verify new objects not created", func(t *testing.T) { - for _, soupId := range soupIds { - exists, err := client.Data().Checker(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantPizza.Name). - Do(context.Background()) - - require.Nil(t, err) - assert.False(t, exists) - } - - for _, pizzaId := range pizzaIds { - exists, err := client.Data().Checker(). - WithClassName("Pizza"). - WithID(pizzaId). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.Nil(t, err) - assert.False(t, exists) - } - }) - }) - - t.Run("with DEST tenant (separate tenants)", func(t *testing.T) { - defer cleanup() - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenantSoup) - fixtures.CreateDataSoupForTenants(t, client, tenantSoup.Name) - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenantPizza) - fixtures.CreateDataPizzaForTenants(t, client, tenantPizza.Name) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - refs := models.MultipleRef{ - client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload(), - } - - err := client.Data().ReferenceReplacer(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReferences(&refs). - WithTenant(tenantPizza.Name). // DEST tenant - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "tenant not found") - } - } - - t.Run("verify not replaced", func(t *testing.T) { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Nil(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"]) - } - }) - - t.Run("verify new objects not created", func(t *testing.T) { - for _, soupId := range soupIds { - exists, err := client.Data().Checker(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantPizza.Name). - Do(context.Background()) - - require.NotNil(t, err) - assert.False(t, exists) - } - - for _, pizzaId := range pizzaIds { - exists, err := client.Data().Checker(). - WithClassName("Pizza"). - WithID(pizzaId). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.NotNil(t, err) - assert.False(t, exists) - } - }) - }) - }) - - t.Run("creates references between MT and non-MT classes", func(t *testing.T) { - defer cleanup() - - tenantSoup := models.Tenant{Name: "tenantSoup"} - pizzaIds := fixtures.IdsByClass["Pizza"] - soupIds := fixtures.IdsByClass["Soup"] - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenantSoup) - fixtures.CreateDataSoupForTenants(t, client, tenantSoup.Name) - - fixtures.CreateSchemaPizza(t, client) - fixtures.CreateDataPizza(t, client) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - ref := client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload() - - err := client.Data().ReferenceCreator(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReference(ref). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.Nil(t, err) - } - } - - t.Run("verify created", func(t *testing.T) { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Len(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"].([]interface{}), - len(pizzaIds)) - } - }) - - t.Run("verify graphql search", func(t *testing.T) { - resp, err := client.GraphQL().Get(). - WithClassName("Soup"). - WithTenant(tenantSoup.Name). - WithFields(graphql.Field{ - Name: "_additional", Fields: []graphql.Field{{Name: "id"}}, - }). - WithWhere(filters.Where(). - WithPath([]string{"relatedToPizza", "Pizza", "name"}). - WithOperator(filters.Equal). - WithValueString("Quattro Formaggi")). - Do(context.Background()) - - require.NoError(t, err) - assertGraphqlGetIds(t, resp, "Soup", soupIds) - }) - }) - - t.Run("fails creating references between MT and non-MT classes without tenant", func(t *testing.T) { - defer cleanup() - - tenantSoup := models.Tenant{Name: "tenantSoup"} - pizzaIds := fixtures.IdsByClass["Pizza"] - soupIds := fixtures.IdsByClass["Soup"] - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenantSoup) - fixtures.CreateDataSoupForTenants(t, client, tenantSoup.Name) - - fixtures.CreateSchemaPizza(t, client) - fixtures.CreateDataPizza(t, client) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - ref := client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload() - - err := client.Data().ReferenceCreator(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReference(ref). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "has multi-tenancy enabled, but request was without tenant") - } - } - - t.Run("verify not created", func(t *testing.T) { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Nil(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"]) - } - }) - - t.Run("verify new objects not created", func(t *testing.T) { - for _, soupId := range soupIds { - exists, err := client.Data().Checker(). - WithClassName("Soup"). - WithID(soupId). - Do(context.Background()) - - require.NotNil(t, err) - assert.False(t, exists) - } - - for _, pizzaId := range pizzaIds { - exists, err := client.Data().Checker(). - WithClassName("Pizza"). - WithID(pizzaId). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.NotNil(t, err) - assert.False(t, exists) - } - }) - }) - - t.Run("fails creating references between MT and non-MT classes with non existent tenant", func(t *testing.T) { - defer cleanup() - - tenantSoup := models.Tenant{Name: "tenantSoup"} - pizzaIds := fixtures.IdsByClass["Pizza"] - soupIds := fixtures.IdsByClass["Soup"] - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenantSoup) - fixtures.CreateDataSoupForTenants(t, client, tenantSoup.Name) - - fixtures.CreateSchemaPizza(t, client) - fixtures.CreateDataPizza(t, client) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - ref := client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload() - - err := client.Data().ReferenceCreator(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReference(ref). - WithTenant("nonExistentTenant"). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "tenant not found") - } - } - - t.Run("verify not created", func(t *testing.T) { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Nil(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"]) - } - }) - - t.Run("verify new objects not created", func(t *testing.T) { - for _, soupId := range soupIds { - exists, err := client.Data().Checker(). - WithClassName("Soup"). - WithID(soupId). - Do(context.Background()) - - require.NotNil(t, err) - assert.False(t, exists) - } - - for _, pizzaId := range pizzaIds { - exists, err := client.Data().Checker(). - WithClassName("Pizza"). - WithID(pizzaId). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.NotNil(t, err) - assert.False(t, exists) - } - }) - }) - - t.Run("fails creating references between MT and non-MT classes with different existing tenant", func(t *testing.T) { - defer cleanup() - - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - pizzaIds := fixtures.IdsByClass["Pizza"] - soupIds := fixtures.IdsByClass["Soup"] - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenants...) - fixtures.CreateDataSoupForTenants(t, client, tenants[0].Name) - - fixtures.CreateSchemaPizza(t, client) - fixtures.CreateDataPizza(t, client) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - ref := client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload() - - err := client.Data().ReferenceCreator(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReference(ref). - WithTenant(tenants[1].Name). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 404, clientErr.StatusCode) // TODO 422? - assert.Empty(t, clientErr.Msg) - } - } - - t.Run("verify not created", func(t *testing.T) { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenants[0].Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Nil(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"]) - } - }) - - t.Run("verify new objects not created", func(t *testing.T) { - for _, soupId := range soupIds { - exists, err := client.Data().Checker(). - WithClassName("Soup"). - WithID(soupId). - Do(context.Background()) - - require.NotNil(t, err) - assert.False(t, exists) - } - - for _, soupId := range soupIds { - exists, err := client.Data().Checker(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenants[1].Name). - Do(context.Background()) - - require.Nil(t, err) - assert.False(t, exists) - } - - for _, tenant := range tenants { - for _, pizzaId := range pizzaIds { - exists, err := client.Data().Checker(). - WithClassName("Pizza"). - WithID(pizzaId). - WithTenant(tenant.Name). - Do(context.Background()) - - require.NotNil(t, err) - assert.False(t, exists) - } - } - }) - }) - - t.Run("deletes references between MT and non-MT classes", func(t *testing.T) { - defer cleanup() - - tenantSoup := models.Tenant{Name: "tenantSoup"} - pizzaIds := fixtures.IdsByClass["Pizza"] - soupIds := fixtures.IdsByClass["Soup"] - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenantSoup) - fixtures.CreateDataSoupForTenants(t, client, tenantSoup.Name) - - fixtures.CreateSchemaPizza(t, client) - fixtures.CreateDataPizza(t, client) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - t.Run("create refs", func(t *testing.T) { - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - ref := client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload() - - err := client.Data().ReferenceCreator(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReference(ref). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.Nil(t, err) - } - } - }) - - for _, soupId := range soupIds { - expectedRefsLeft := len(pizzaIds) - - for _, pizzaId := range pizzaIds { - ref := client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload() - - err := client.Data().ReferenceDeleter(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReference(ref). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.Nil(t, err) - - t.Run("verify deleted one by one", func(t *testing.T) { - expectedRefsLeft-- - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Len(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"].([]interface{}), - expectedRefsLeft) - }) - } - } - }) - - t.Run("fails deleting references between MT and non-MT classes without tenant", func(t *testing.T) { - defer cleanup() - - tenantSoup := models.Tenant{Name: "tenantSoup"} - pizzaIds := fixtures.IdsByClass["Pizza"] - soupIds := fixtures.IdsByClass["Soup"] - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenantSoup) - fixtures.CreateDataSoupForTenants(t, client, tenantSoup.Name) - - fixtures.CreateSchemaPizza(t, client) - fixtures.CreateDataPizza(t, client) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - t.Run("create refs", func(t *testing.T) { - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - ref := client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload() - - err := client.Data().ReferenceCreator(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReference(ref). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.Nil(t, err) - } - } - }) - - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - ref := client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload() - - err := client.Data().ReferenceDeleter(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReference(ref). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "has multi-tenancy enabled, but request was without tenant") - } - } - - t.Run("verify not deleted", func(t *testing.T) { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Len(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"].([]interface{}), - len(pizzaIds)) - } - }) - }) - - t.Run("fails deleting references between MT and non-MT classes with non existent tenant", func(t *testing.T) { - defer cleanup() - - tenantSoup := models.Tenant{Name: "tenantSoup"} - pizzaIds := fixtures.IdsByClass["Pizza"] - soupIds := fixtures.IdsByClass["Soup"] - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenantSoup) - fixtures.CreateDataSoupForTenants(t, client, tenantSoup.Name) - - fixtures.CreateSchemaPizza(t, client) - fixtures.CreateDataPizza(t, client) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - t.Run("create refs", func(t *testing.T) { - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - ref := client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload() - - err := client.Data().ReferenceCreator(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReference(ref). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.Nil(t, err) - } - } - }) - - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - ref := client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload() - - err := client.Data().ReferenceDeleter(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReference(ref). - WithTenant("nonExistentTenant"). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "tenant not found") - } - } - - t.Run("verify not deleted", func(t *testing.T) { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Len(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"].([]interface{}), - len(pizzaIds)) - } - }) - }) - - t.Run("fails deleting references between MT and non-MT classes with different existing tenant", func(t *testing.T) { - defer cleanup() - - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - pizzaIds := fixtures.IdsByClass["Pizza"] - soupIds := fixtures.IdsByClass["Soup"] - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenants...) - fixtures.CreateDataSoupForTenants(t, client, tenants[0].Name) - - fixtures.CreateSchemaPizza(t, client) - fixtures.CreateDataPizza(t, client) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - t.Run("create refs", func(t *testing.T) { - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - ref := client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload() - - err := client.Data().ReferenceCreator(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReference(ref). - WithTenant(tenants[0].Name). - Do(context.Background()) - - require.Nil(t, err) - } - } - }) - - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - ref := client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload() - - err := client.Data().ReferenceDeleter(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReference(ref). - WithTenant(tenants[1].Name). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 404, clientErr.StatusCode) - assert.Empty(t, clientErr.Msg) - } - } - - t.Run("verify not deleted", func(t *testing.T) { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenants[0].Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Len(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"].([]interface{}), - len(pizzaIds)) - } - }) - }) - - t.Run("replaces references between MT and non-MT classes", func(t *testing.T) { - defer cleanup() - - tenantSoup := models.Tenant{Name: "tenantSoup"} - pizzaIdsBefore := fixtures.IdsByClass["Pizza"][:2] - pizzaIdsAfter := fixtures.IdsByClass["Pizza"][2:] - soupIds := fixtures.IdsByClass["Soup"] - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenantSoup) - fixtures.CreateDataSoupForTenants(t, client, tenantSoup.Name) - - fixtures.CreateSchemaPizza(t, client) - fixtures.CreateDataPizza(t, client) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - t.Run("create refs", func(t *testing.T) { - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIdsBefore { - ref := client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload() - - err := client.Data().ReferenceCreator(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReference(ref). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.Nil(t, err) - } - } - }) - - for _, soupId := range soupIds { - var refs models.MultipleRef - for _, pizzaId := range pizzaIdsAfter { - ref := client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload() - refs = append(refs, ref) - } - - err := client.Data().ReferenceReplacer(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReferences(&refs). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.Nil(t, err) - } - - t.Run("verify replaced", func(t *testing.T) { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - require.Len(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"].([]interface{}), - len(pizzaIdsAfter)) - - for _, pizzaId := range pizzaIdsAfter { - found := false - for _, ref := range objects[0].Properties.(map[string]interface{})["relatedToPizza"].([]interface{}) { - if strings.Contains(ref.(map[string]interface{})["beacon"].(string), pizzaId) { - found = true - break - } - } - assert.True(t, found, fmt.Sprintf("ref to '%s' not found", pizzaId)) - } - } - }) - - t.Run("verify graphql search", func(t *testing.T) { - resp, err := client.GraphQL().Get(). - WithClassName("Soup"). - WithTenant(tenantSoup.Name). - WithFields(graphql.Field{ - Name: "_additional", Fields: []graphql.Field{{Name: "id"}}, - }). - WithWhere(filters.Where(). - WithPath([]string{"relatedToPizza", "Pizza", "name"}). - WithOperator(filters.Equal). - WithValueString("Quattro Formaggi")). - Do(context.Background()) - - require.NoError(t, err) - assertGraphqlGetIds(t, resp, "Soup", []string{}) - }) - }) - - t.Run("fails replacing references between MT and non-MT classes without tenant", func(t *testing.T) { - defer cleanup() - - tenantSoup := models.Tenant{Name: "tenantSoup"} - pizzaIdsBefore := fixtures.IdsByClass["Pizza"][:2] - pizzaIdsAfter := fixtures.IdsByClass["Pizza"][2:] - soupIds := fixtures.IdsByClass["Soup"] - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenantSoup) - fixtures.CreateDataSoupForTenants(t, client, tenantSoup.Name) - - fixtures.CreateSchemaPizza(t, client) - fixtures.CreateDataPizza(t, client) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - t.Run("create refs", func(t *testing.T) { - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIdsBefore { - ref := client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload() - - err := client.Data().ReferenceCreator(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReference(ref). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.Nil(t, err) - } - } - }) - - for _, soupId := range soupIds { - var refs models.MultipleRef - for _, pizzaId := range pizzaIdsAfter { - ref := client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload() - refs = append(refs, ref) - } - - err := client.Data().ReferenceReplacer(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReferences(&refs). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "has multi-tenancy enabled, but request was without tenant") - } - - t.Run("verify not replaced", func(t *testing.T) { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - require.Len(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"].([]interface{}), - len(pizzaIdsBefore)) - - for _, pizzaId := range pizzaIdsBefore { - found := false - for _, ref := range objects[0].Properties.(map[string]interface{})["relatedToPizza"].([]interface{}) { - if strings.Contains(ref.(map[string]interface{})["beacon"].(string), pizzaId) { - found = true - break - } - } - assert.True(t, found, fmt.Sprintf("ref to '%s' not found", pizzaId)) - } - } - }) - }) - - t.Run("fails replacing references between MT and non-MT classes with non existent tenant", func(t *testing.T) { - defer cleanup() - - tenantSoup := models.Tenant{Name: "tenantSoup"} - pizzaIdsBefore := fixtures.IdsByClass["Pizza"][:2] - pizzaIdsAfter := fixtures.IdsByClass["Pizza"][2:] - soupIds := fixtures.IdsByClass["Soup"] - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenantSoup) - fixtures.CreateDataSoupForTenants(t, client, tenantSoup.Name) - - fixtures.CreateSchemaPizza(t, client) - fixtures.CreateDataPizza(t, client) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - t.Run("create refs", func(t *testing.T) { - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIdsBefore { - ref := client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload() - - err := client.Data().ReferenceCreator(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReference(ref). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.Nil(t, err) - } - } - }) - - for _, soupId := range soupIds { - var refs models.MultipleRef - for _, pizzaId := range pizzaIdsAfter { - ref := client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload() - refs = append(refs, ref) - } - - err := client.Data().ReferenceReplacer(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReferences(&refs). - WithTenant("nonExistentTenant"). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "tenant not found") - } - - t.Run("verify not replaced", func(t *testing.T) { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantSoup.Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - require.Len(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"].([]interface{}), - len(pizzaIdsBefore)) - - for _, pizzaId := range pizzaIdsBefore { - found := false - for _, ref := range objects[0].Properties.(map[string]interface{})["relatedToPizza"].([]interface{}) { - if strings.Contains(ref.(map[string]interface{})["beacon"].(string), pizzaId) { - found = true - break - } - } - assert.True(t, found, fmt.Sprintf("ref to '%s' not found", pizzaId)) - } - } - }) - }) - - t.Run("fails replacing references between MT and non-MT classes with different existing tenant", func(t *testing.T) { - defer cleanup() - - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - soupIds := fixtures.IdsByClass["Soup"] - pizzaIdsBefore := fixtures.IdsByClass["Pizza"][:2] - pizzaIdsAfter := fixtures.IdsByClass["Pizza"][2:] - - fixtures.CreateSchemaSoupForTenants(t, client) - fixtures.CreateTenantsSoup(t, client, tenants...) - fixtures.CreateDataSoupForTenants(t, client, tenants[0].Name) - - fixtures.CreateSchemaPizza(t, client) - fixtures.CreateDataPizza(t, client) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - t.Run("create refs", func(t *testing.T) { - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIdsBefore { - ref := client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload() - - err := client.Data().ReferenceCreator(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReference(ref). - WithTenant(tenants[0].Name). - Do(context.Background()) - - require.Nil(t, err) - } - } - }) - - for _, soupId := range soupIds { - var refs models.MultipleRef - for _, pizzaId := range pizzaIdsAfter { - ref := client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload() - refs = append(refs, ref) - } - - err := client.Data().ReferenceReplacer(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReferences(&refs). - WithTenant(tenants[1].Name). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 404, clientErr.StatusCode) - assert.Empty(t, clientErr.Msg) - } - - t.Run("verify not replaced", func(t *testing.T) { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenants[0].Name). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - require.Len(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"].([]interface{}), - len(pizzaIdsBefore)) - - for _, pizzaId := range pizzaIdsBefore { - found := false - for _, ref := range objects[0].Properties.(map[string]interface{})["relatedToPizza"].([]interface{}) { - if strings.Contains(ref.(map[string]interface{})["beacon"].(string), pizzaId) { - found = true - break - } - } - assert.True(t, found, fmt.Sprintf("ref to '%s' not found", pizzaId)) - } - } - }) - - t.Run("verify new objects not created", func(t *testing.T) { - for _, soupId := range soupIds { - exists, err := client.Data().Checker(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenants[1].Name). - Do(context.Background()) - - require.Nil(t, err) - assert.False(t, exists) - } - - for _, tenant := range tenants { - for _, pizzaId := range fixtures.IdsByClass["Pizza"] { - exists, err := client.Data().Checker(). - WithClassName("Pizza"). - WithID(pizzaId). - WithTenant(tenant.Name). - Do(context.Background()) - - require.NotNil(t, err) - assert.False(t, exists) - } - } - }) - }) - - t.Run("fails creating references between non-MT and MT classes", func(t *testing.T) { - defer cleanup() - - tenantPizza := models.Tenant{Name: "tenantPizza"} - pizzaIds := fixtures.IdsByClass["Pizza"] - soupIds := fixtures.IdsByClass["Soup"] - - fixtures.CreateSchemaSoup(t, client) - fixtures.CreateDataSoup(t, client) - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenantPizza) - fixtures.CreateDataPizzaForTenants(t, client, tenantPizza.Name) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - ref := client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload() - - err := client.Data().ReferenceCreator(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReference(ref). - WithTenant(tenantPizza.Name). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "has multi-tenancy disabled, but request was with tenant") - } - } - - t.Run("verify not created", func(t *testing.T) { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Nil(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"]) - } - }) - - t.Run("verify new objects not created", func(t *testing.T) { - for _, soupId := range soupIds { - exists, err := client.Data().Checker(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantPizza.Name). - Do(context.Background()) - - require.NotNil(t, err) - assert.False(t, exists) - } - }) - }) - - t.Run("fails creating references between non-MT and MT classes without tenant", func(t *testing.T) { - defer cleanup() - - tenantPizza := models.Tenant{Name: "tenantPizza"} - pizzaIds := fixtures.IdsByClass["Pizza"] - soupIds := fixtures.IdsByClass["Soup"] - - fixtures.CreateSchemaSoup(t, client) - fixtures.CreateDataSoup(t, client) - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenantPizza) - fixtures.CreateDataPizzaForTenants(t, client, tenantPizza.Name) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - ref := client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload() - - err := client.Data().ReferenceCreator(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReference(ref). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - // TODO valid error? - assert.Contains(t, clientErr.Msg, "has multi-tenancy enabled, but request was without tenant") - } - } - - t.Run("verify not created", func(t *testing.T) { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Nil(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"]) - } - }) - - t.Run("verify new objects not created", func(t *testing.T) { - for _, soupId := range soupIds { - exists, err := client.Data().Checker(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantPizza.Name). - Do(context.Background()) - - require.NotNil(t, err) - assert.False(t, exists) - } - }) - }) - - t.Run("fails creating references between non-MT and MT classes with non existent tenant", func(t *testing.T) { - defer cleanup() - - tenantPizza := models.Tenant{Name: "tenantPizza"} - pizzaIds := fixtures.IdsByClass["Pizza"] - soupIds := fixtures.IdsByClass["Soup"] - - fixtures.CreateSchemaSoup(t, client) - fixtures.CreateDataSoup(t, client) - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenantPizza) - fixtures.CreateDataPizzaForTenants(t, client, tenantPizza.Name) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - ref := client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload() - - err := client.Data().ReferenceCreator(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReference(ref). - WithTenant("nonExistentTenant"). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - // TODO valid error? - assert.Contains(t, clientErr.Msg, "tenant not found") - } - } - - t.Run("verify not created", func(t *testing.T) { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Nil(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"]) - } - }) - - t.Run("verify new objects not created", func(t *testing.T) { - for _, soupId := range soupIds { - exists, err := client.Data().Checker(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantPizza.Name). - Do(context.Background()) - - require.NotNil(t, err) - assert.False(t, exists) - } - }) - }) - - t.Run("fails creating references between non-MT and MT classes with different existing tenant", func(t *testing.T) { - defer cleanup() - - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - pizzaIds := fixtures.IdsByClass["Pizza"] - soupIds := fixtures.IdsByClass["Soup"] - - fixtures.CreateSchemaSoup(t, client) - fixtures.CreateDataSoup(t, client) - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenants...) - fixtures.CreateDataPizzaForTenants(t, client, tenants[0].Name) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - for _, soupId := range soupIds { - for _, pizzaId := range pizzaIds { - ref := client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload() - - err := client.Data().ReferenceCreator(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReference(ref). - WithTenant(tenants[1].Name). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - // TODO valid error? - assert.Contains(t, clientErr.Msg, "no object with id") - } - } - - t.Run("verify not created", func(t *testing.T) { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Nil(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"]) - } - }) - - t.Run("verify new objects not created", func(t *testing.T) { - for _, tenant := range tenants { - for _, soupId := range soupIds { - exists, err := client.Data().Checker(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenant.Name). - Do(context.Background()) - - require.NotNil(t, err) - assert.False(t, exists) - } - } - - for _, pizzaId := range pizzaIds { - exists, err := client.Data().Checker(). - WithClassName("Soup"). - WithID(pizzaId). - WithTenant(tenants[1].Name). - Do(context.Background()) - - require.NotNil(t, err) - assert.False(t, exists) - } - }) - }) - - t.Run("fails replacing references between non-MT and MT classes", func(t *testing.T) { - defer cleanup() - - tenantPizza := models.Tenant{Name: "tenantPizza"} - pizzaIds := fixtures.IdsByClass["Pizza"] - soupIds := fixtures.IdsByClass["Soup"] - - fixtures.CreateSchemaSoup(t, client) - fixtures.CreateDataSoup(t, client) - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenantPizza) - fixtures.CreateDataPizzaForTenants(t, client, tenantPizza.Name) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - for _, soupId := range soupIds { - var refs models.MultipleRef - for _, pizzaId := range pizzaIds { - ref := client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload() - refs = append(refs, ref) - } - - err := client.Data().ReferenceReplacer(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReferences(&refs). - WithTenant(tenantPizza.Name). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "has multi-tenancy disabled, but request was with tenant") - } - - t.Run("verify not replaced", func(t *testing.T) { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Nil(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"]) - } - }) - - t.Run("verify new objects not created", func(t *testing.T) { - for _, soupId := range soupIds { - exists, err := client.Data().Checker(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantPizza.Name). - Do(context.Background()) - - require.NotNil(t, err) - assert.False(t, exists) - } - }) - }) - - t.Run("fails replacing references between non-MT and MT classes without tenant", func(t *testing.T) { - defer cleanup() - - tenantPizza := models.Tenant{Name: "tenantPizza"} - pizzaIds := fixtures.IdsByClass["Pizza"] - soupIds := fixtures.IdsByClass["Soup"] - - fixtures.CreateSchemaSoup(t, client) - fixtures.CreateDataSoup(t, client) - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenantPizza) - fixtures.CreateDataPizzaForTenants(t, client, tenantPizza.Name) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - for _, soupId := range soupIds { - var refs models.MultipleRef - for _, pizzaId := range pizzaIds { - ref := client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload() - refs = append(refs, ref) - } - - err := client.Data().ReferenceReplacer(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReferences(&refs). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - // TODO valid error? - assert.Contains(t, clientErr.Msg, "has multi-tenancy enabled, but request was without tenant") - } - - t.Run("verify not replaced", func(t *testing.T) { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Nil(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"]) - } - }) - - t.Run("verify new objects not created", func(t *testing.T) { - for _, soupId := range soupIds { - exists, err := client.Data().Checker(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantPizza.Name). - Do(context.Background()) - - require.NotNil(t, err) - assert.False(t, exists) - } - }) - }) - - t.Run("fails replacing references between non-MT and MT classes with non existent tenant", func(t *testing.T) { - defer cleanup() - - tenantPizza := models.Tenant{Name: "tenantPizza"} - pizzaIds := fixtures.IdsByClass["Pizza"] - soupIds := fixtures.IdsByClass["Soup"] - - fixtures.CreateSchemaSoup(t, client) - fixtures.CreateDataSoup(t, client) - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenantPizza) - fixtures.CreateDataPizzaForTenants(t, client, tenantPizza.Name) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - for _, soupId := range soupIds { - var refs models.MultipleRef - for _, pizzaId := range pizzaIds { - ref := client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload() - refs = append(refs, ref) - } - - err := client.Data().ReferenceReplacer(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReferences(&refs). - WithTenant("nonExistentTenant"). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "has multi-tenancy disabled, but request was with tenant") - } - - t.Run("verify not replaced", func(t *testing.T) { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Nil(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"]) - } - }) - - t.Run("verify new objects not created", func(t *testing.T) { - for _, soupId := range soupIds { - exists, err := client.Data().Checker(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenantPizza.Name). - Do(context.Background()) - - require.NotNil(t, err) - assert.False(t, exists) - } - }) - }) - - t.Run("fails replacing references between non-MT and MT classes with different existing tenant", func(t *testing.T) { - defer cleanup() - - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - pizzaIds := fixtures.IdsByClass["Pizza"] - soupIds := fixtures.IdsByClass["Soup"] - - fixtures.CreateSchemaSoup(t, client) - fixtures.CreateDataSoup(t, client) - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenants...) - fixtures.CreateDataPizzaForTenants(t, client, tenants[0].Name) - - t.Run("create ref property", func(t *testing.T) { - err := client.Schema().PropertyCreator(). - WithClassName("Soup"). - WithProperty(&models.Property{ - Name: "relatedToPizza", - DataType: []string{"Pizza"}, - }). - Do(context.Background()) - - require.Nil(t, err) - }) - - for _, soupId := range soupIds { - var refs models.MultipleRef - for _, pizzaId := range pizzaIds { - ref := client.Data().ReferencePayloadBuilder(). - WithClassName("Pizza"). - WithID(pizzaId). - Payload() - refs = append(refs, ref) - } - - err := client.Data().ReferenceReplacer(). - WithClassName("Soup"). - WithID(soupId). - WithReferenceProperty("relatedToPizza"). - WithReferences(&refs). - WithTenant(tenants[1].Name). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "has multi-tenancy disabled, but request was with tenant") - } - - t.Run("verify not replaced", func(t *testing.T) { - for _, soupId := range soupIds { - objects, err := client.Data().ObjectsGetter(). - WithClassName("Soup"). - WithID(soupId). - Do(context.Background()) - - require.Nil(t, err) - require.NotNil(t, objects) - require.Len(t, objects, 1) - assert.Nil(t, objects[0].Properties.(map[string]interface{})["relatedToPizza"]) - } - }) - - t.Run("verify new objects not created", func(t *testing.T) { - for _, tenant := range tenants { - for _, soupId := range soupIds { - exists, err := client.Data().Checker(). - WithClassName("Soup"). - WithID(soupId). - WithTenant(tenant.Name). - Do(context.Background()) - - require.NotNil(t, err) - assert.False(t, exists) - } - } - - for _, pizzaId := range pizzaIds { - exists, err := client.Data().Checker(). - WithClassName("Soup"). - WithID(pizzaId). - WithTenant(tenants[1].Name). - Do(context.Background()) - - require.NotNil(t, err) - assert.False(t, exists) - } - }) - }) -} - -func assertGraphqlGetIds(t *testing.T, resp *models.GraphQLResponse, className string, - expectedIds []string, -) { - require.NotNil(t, resp) - require.Nil(t, resp.Errors) - require.NotNil(t, resp.Data) - require.Contains(t, resp.Data, "Get") - - get := resp.Data["Get"] - require.NotNil(t, get) - - classes, ok := get.(map[string]interface{}) - require.True(t, ok) - require.Contains(t, classes, className) - - objects, ok := classes[className].([]interface{}) - require.True(t, ok) - - ids := make([]string, len(objects)) - for i := range objects { - props, ok := objects[i].(map[string]interface{}) - require.True(t, ok) - require.Contains(t, props, "_additional") - - add, ok := props["_additional"].(map[string]interface{}) - require.True(t, ok) - require.Contains(t, add, "id") - - id, ok := add["id"].(string) - require.True(t, ok) - ids[i] = id - } - require.ElementsMatch(t, expectedIds, ids) -} diff --git a/test/acceptance_with_go_client/multi_tenancy_tests/schema_test.go b/test/acceptance_with_go_client/multi_tenancy_tests/schema_test.go deleted file mode 100644 index 22b403e029ee5f8d89bf8c45d1242033c864f484..0000000000000000000000000000000000000000 --- a/test/acceptance_with_go_client/multi_tenancy_tests/schema_test.go +++ /dev/null @@ -1,348 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package multi_tenancy_tests - -import ( - "context" - "testing" - - "acceptance_tests_with_client/fixtures" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - wvt "github.com/weaviate/weaviate-go-client/v4/weaviate" - "github.com/weaviate/weaviate-go-client/v4/weaviate/fault" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" -) - -func TestSchema_MultiTenancyConfig(t *testing.T) { - client, err := wvt.NewClient(wvt.Config{Scheme: "http", Host: "localhost:8080"}) - require.Nil(t, err) - - cleanup := func() { - err := client.Schema().AllDeleter().Do(context.Background()) - require.Nil(t, err) - } - - t.Run("class with MT config - MT enabled", func(t *testing.T) { - defer cleanup() - - className := "MultiTenantClass" - schemaClass := &models.Class{ - Class: className, - Properties: []*models.Property{ - { - Name: "someProperty", - DataType: schema.DataTypeText.PropString(), - }, - }, - MultiTenancyConfig: &models.MultiTenancyConfig{ - Enabled: true, - }, - } - - err := client.Schema().ClassCreator(). - WithClass(schemaClass). - Do(context.Background()) - require.Nil(t, err) - - t.Run("verify class created", func(t *testing.T) { - loadedClass, err := client.Schema().ClassGetter().WithClassName(className).Do(context.Background()) - require.Nil(t, err) - require.NotNil(t, loadedClass.MultiTenancyConfig) - assert.Equal(t, true, loadedClass.MultiTenancyConfig.Enabled) - }) - }) - - t.Run("class with MT config - MT disabled", func(t *testing.T) { - defer cleanup() - - className := "MultiTenantClassDisabled" - schemaClass := &models.Class{ - Class: className, - Properties: []*models.Property{ - { - Name: "someProperty", - DataType: schema.DataTypeText.PropString(), - }, - }, - MultiTenancyConfig: &models.MultiTenancyConfig{ - Enabled: false, - }, - } - - err := client.Schema().ClassCreator(). - WithClass(schemaClass). - Do(context.Background()) - require.Nil(t, err) - - t.Run("verify class created", func(t *testing.T) { - loadedClass, err := client.Schema().ClassGetter().WithClassName(className).Do(context.Background()) - require.Nil(t, err) - require.NotNil(t, loadedClass.MultiTenancyConfig) - assert.Equal(t, false, loadedClass.MultiTenancyConfig.Enabled) - }) - }) - - t.Run("class without MT config", func(t *testing.T) { - defer cleanup() - - className := "NonMultiTenantClass" - schemaClass := &models.Class{ - Class: className, - Properties: []*models.Property{ - { - Name: "someProperty", - DataType: schema.DataTypeText.PropString(), - }, - }, - } - - err := client.Schema().ClassCreator(). - WithClass(schemaClass). - Do(context.Background()) - require.Nil(t, err) - - t.Run("verify class created", func(t *testing.T) { - loadedClass, err := client.Schema().ClassGetter().WithClassName(className).Do(context.Background()) - require.Nil(t, err) - assert.NotNil(t, loadedClass.MultiTenancyConfig) - assert.False(t, loadedClass.MultiTenancyConfig.Enabled) - }) - }) -} - -func TestSchema_Tenants(t *testing.T) { - client, err := wvt.NewClient(wvt.Config{Scheme: "http", Host: "localhost:8080"}) - require.Nil(t, err) - - cleanup := func() { - err := client.Schema().AllDeleter().Do(context.Background()) - require.Nil(t, err) - } - - className := "Pizza" - - t.Run("adds tenants to MT class", func(t *testing.T) { - defer cleanup() - - fixtures.CreateSchemaPizzaForTenants(t, client) - - t.Run("adds single tenant", func(t *testing.T) { - tenant := models.Tenant{ - Name: "tenantNo1", - } - - err := client.Schema().TenantsCreator(). - WithClassName(className). - WithTenants(tenant). - Do(context.Background()) - - require.Nil(t, err) - }) - - t.Run("adds multiple tenants", func(t *testing.T) { - tenants := fixtures.Tenants{ - {Name: "tenantNo2"}, - {Name: "tenantNo3"}, - } - - err := client.Schema().TenantsCreator(). - WithClassName(className). - WithTenants(tenants...). - Do(context.Background()) - - require.Nil(t, err) - }) - }) - - t.Run("fails adding tenants to non-MT class", func(t *testing.T) { - defer cleanup() - - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - - fixtures.CreateSchemaPizza(t, client) - - err := client.Schema().TenantsCreator(). - WithClassName(className). - WithTenants(tenants...). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "multi-tenancy is not enabled for class") - }) - - t.Run("gets tenants of MT class", func(t *testing.T) { - defer cleanup() - - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenants...) - - gotTenants, err := client.Schema().TenantsGetter(). - WithClassName(className). - Do(context.Background()) - - require.Nil(t, err) - require.Len(t, gotTenants, len(tenants)) - - assert.ElementsMatch(t, tenants.Names(), fixtures.Tenants(gotTenants).Names()) - }) - - t.Run("fails getting tenants from non-MT class", func(t *testing.T) { - defer cleanup() - - fixtures.CreateSchemaPizza(t, client) - - gotTenants, err := client.Schema().TenantsGetter(). - WithClassName(className). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "multi-tenancy is not enabled for class") - require.Nil(t, gotTenants) - }) - - t.Run("updates tenants of MT class", func(t *testing.T) { - defer cleanup() - - tenants := []models.Tenant{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenants...) - - t.Run("fails updating non existent tenant", func(t *testing.T) { - err := client.Schema().TenantsUpdater(). - WithClassName(className). - WithTenants(models.Tenant{ - Name: "nonExistentTenant", - ActivityStatus: models.TenantActivityStatusCOLD, - }).Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "not found") - }) - - t.Run("updates existent tenants", func(t *testing.T) { - err := client.Schema().TenantsUpdater(). - WithClassName(className). - WithTenants( - models.Tenant{ - Name: tenants[0].Name, - ActivityStatus: models.TenantActivityStatusCOLD, - }, - models.Tenant{ - Name: tenants[1].Name, - ActivityStatus: models.TenantActivityStatusCOLD, - }, - ).Do(context.Background()) - - require.Nil(t, err) - }) - }) - - t.Run("fails updating tenants of non-MT class", func(t *testing.T) { - defer cleanup() - - tenants := []models.Tenant{ - { - Name: "tenantNo1", - ActivityStatus: models.TenantActivityStatusCOLD, - }, - { - Name: "tenantNo2", - ActivityStatus: models.TenantActivityStatusCOLD, - }, - } - - fixtures.CreateSchemaPizza(t, client) - - err := client.Schema().TenantsUpdater(). - WithClassName(className). - WithTenants(tenants...). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "multi-tenancy is not enabled for class") - }) - - t.Run("deletes tenants from MT class", func(t *testing.T) { - defer cleanup() - - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - {Name: "tenantNo3"}, - } - - fixtures.CreateSchemaPizzaForTenants(t, client) - fixtures.CreateTenantsPizza(t, client, tenants...) - - t.Run("does not error on deleting non existent tenant", func(t *testing.T) { - err := client.Schema().TenantsDeleter(). - WithClassName(className). - WithTenants(tenants[0].Name, "nonExistentTenant"). - Do(context.Background()) - - require.Nil(t, err) - }) - - t.Run("deletes multiple tenants", func(t *testing.T) { - err := client.Schema().TenantsDeleter(). - WithClassName(className). - WithTenants(tenants[1:].Names()...). - Do(context.Background()) - - require.Nil(t, err) - }) - }) - - t.Run("fails deleting tenants from non-MT class", func(t *testing.T) { - defer cleanup() - - tenants := fixtures.Tenants{ - {Name: "tenantNo1"}, - {Name: "tenantNo2"}, - } - - fixtures.CreateSchemaPizza(t, client) - - err := client.Schema().TenantsDeleter(). - WithClassName(className). - WithTenants(tenants.Names()...). - Do(context.Background()) - - require.NotNil(t, err) - clientErr := err.(*fault.WeaviateClientError) - assert.Equal(t, 422, clientErr.StatusCode) - assert.Contains(t, clientErr.Msg, "multi-tenancy is not enabled for class") - }) -} diff --git a/test/acceptance_with_go_client/object_property_tests/autoschema_test.go b/test/acceptance_with_go_client/object_property_tests/autoschema_test.go deleted file mode 100644 index 3e5d7d2ce31394c92921ed57645920dc66d01f92..0000000000000000000000000000000000000000 --- a/test/acceptance_with_go_client/object_property_tests/autoschema_test.go +++ /dev/null @@ -1,298 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package object_property_tests - -import ( - "context" - "testing" - - "github.com/go-openapi/strfmt" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - wvt "github.com/weaviate/weaviate-go-client/v4/weaviate" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" -) - -func TestObjectProperty_AutoSchema(t *testing.T) { - ctx := context.Background() - client, err := wvt.NewClient(wvt.Config{Scheme: "http", Host: "localhost:8080"}) - require.Nil(t, err) - - id1 := strfmt.UUID("00000000-0000-0000-0000-000000000001") - - // clean up DB - err = client.Schema().AllDeleter().Do(context.Background()) - require.Nil(t, err) - - assertDataCreated := func(t *testing.T, className, id string) { - res, err := client.Data().ObjectsGetter().WithClassName(className).WithID(id).Do(ctx) - require.NoError(t, err) - require.Len(t, res, 1) - - props, ok := res[0].Properties.(map[string]interface{}) - require.True(t, ok) - require.Len(t, props, 2) - - require.Contains(t, props, "company") - assert.Equal(t, "BestOne", props["company"]) - - require.Contains(t, props, "json") - jsonProp, ok := props["json"].(map[string]interface{}) - require.True(t, ok) - require.Len(t, jsonProp, 3) - - require.Contains(t, jsonProp, "firstName") - assert.Equal(t, "John", jsonProp["firstName"]) - - require.Contains(t, jsonProp, "lastName") - assert.Equal(t, "Doe", jsonProp["lastName"]) - - require.Contains(t, jsonProp, "phones") - phones, ok := jsonProp["phones"].([]interface{}) - require.True(t, ok) - require.Len(t, phones, 2) - for i, p := range phones { - phone, ok := p.(map[string]interface{}) - require.True(t, ok) - require.Contains(t, phone, "phoneNo") - assert.Equal(t, float64(i+1), phone["phoneNo"]) - } - } - - assertDataUpdated := func(t *testing.T, className, id string) { - res, err := client.Data().ObjectsGetter().WithClassName(className).WithID(id).Do(ctx) - require.NoError(t, err) - require.Len(t, res, 1) - - props, ok := res[0].Properties.(map[string]interface{}) - require.True(t, ok) - require.Len(t, props, 3) - - require.Contains(t, props, "company") - assert.Equal(t, "BestTwo", props["company"]) - - require.Contains(t, props, "founded") - assert.Equal(t, float64(1950), props["founded"]) - - require.Contains(t, props, "json") - jsonProp, ok := props["json"].(map[string]interface{}) - require.True(t, ok) - require.Len(t, jsonProp, 3) - - require.Contains(t, jsonProp, "firstName") - assert.Equal(t, "Jane", jsonProp["firstName"]) - - require.Contains(t, jsonProp, "age") - assert.Equal(t, float64(32), jsonProp["age"]) - - require.Contains(t, jsonProp, "phones") - phones, ok := jsonProp["phones"].([]interface{}) - require.True(t, ok) - require.Len(t, phones, 3) - for i, p := range phones { - phone, ok := p.(map[string]interface{}) - require.True(t, ok) - require.Contains(t, phone, "phoneNo") - assert.Equal(t, float64(i+1), phone["phoneNo"]) - } - } - - assertDataMerged := func(t *testing.T, className, id string) { - res, err := client.Data().ObjectsGetter().WithClassName(className).WithID(id).Do(ctx) - require.NoError(t, err) - require.Len(t, res, 1) - - props, ok := res[0].Properties.(map[string]interface{}) - require.True(t, ok) - require.Len(t, props, 3) - - require.Contains(t, props, "company") - assert.Equal(t, "BestTwo", props["company"]) - - require.Contains(t, props, "founded") - assert.Equal(t, float64(1960), props["founded"]) - - require.Contains(t, props, "json") - jsonProp, ok := props["json"].(map[string]interface{}) - require.True(t, ok) - require.Len(t, jsonProp, 2) - - require.Contains(t, jsonProp, "lastName") - assert.Equal(t, "Smith", jsonProp["lastName"]) - - require.Contains(t, jsonProp, "phones") - phones, ok := jsonProp["phones"].([]interface{}) - require.True(t, ok) - require.Len(t, phones, 0) - } - - type testCase struct { - name string - className string - before func(t *testing.T, className string) - } - - testCases := []testCase{ - { - name: "without auto schema", - className: "WithoutAutoSchema", - before: func(t *testing.T, className string) { - err := client.Schema().ClassCreator().WithClass(&models.Class{ - Class: className, - Properties: []*models.Property{ - { - Name: "company", - DataType: schema.DataTypeText.PropString(), - }, - { - Name: "founded", - DataType: schema.DataTypeInt.PropString(), - }, - { - Name: "json", - DataType: schema.DataTypeObject.PropString(), - NestedProperties: []*models.NestedProperty{ - { - Name: "firstName", - DataType: schema.DataTypeText.PropString(), - }, - { - Name: "lastName", - DataType: schema.DataTypeText.PropString(), - }, - { - Name: "age", - DataType: schema.DataTypeInt.PropString(), - }, - { - Name: "phones", - DataType: schema.DataTypeObjectArray.PropString(), - NestedProperties: []*models.NestedProperty{ - { - Name: "phoneNo", - DataType: schema.DataTypeInt.PropString(), - }, - }, - }, - }, - }, - }, - }).Do(ctx) - require.NoError(t, err) - }, - }, - { - name: "with auto schema", - className: "WithAutoSchema", - before: func(t *testing.T, className string) {}, - }, - { - name: "partially with auto schema", - className: "PartiallyAutoSchema", - before: func(t *testing.T, className string) { - err := client.Schema().ClassCreator().WithClass(&models.Class{ - Class: className, - Properties: []*models.Property{ - { - Name: "company", - DataType: schema.DataTypeText.PropString(), - }, - { - Name: "json", - DataType: schema.DataTypeObject.PropString(), - NestedProperties: []*models.NestedProperty{ - { - Name: "firstName", - DataType: schema.DataTypeText.PropString(), - }, - }, - }, - }, - }).Do(ctx) - require.NoError(t, err) - }, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - tc.before(t, tc.className) - - t.Run("create", func(t *testing.T) { - _, err := client.Data().Creator(). - WithClassName(tc.className). - WithID(id1.String()). - WithProperties(map[string]interface{}{ - "company": "BestOne", - "json": map[string]interface{}{ - "firstName": "John", - "lastName": "Doe", - "phones": []interface{}{ - map[string]interface{}{ - "phoneNo": 1, - }, - map[string]interface{}{ - "phoneNo": 2, - }, - }, - }, - }).Do(ctx) - require.NoError(t, err) - assertDataCreated(t, tc.className, id1.String()) - }) - - t.Run("update", func(t *testing.T) { - err := client.Data().Updater(). - WithClassName(tc.className). - WithID(id1.String()). - WithProperties(map[string]interface{}{ - "company": "BestTwo", - "founded": 1950, - "json": map[string]interface{}{ - "firstName": "Jane", - "age": 32, - "phones": []interface{}{ - map[string]interface{}{ - "phoneNo": 1, - }, - map[string]interface{}{ - "phoneNo": 2, - }, - map[string]interface{}{ - "phoneNo": 3, - }, - }, - }, - }).Do(ctx) - require.NoError(t, err) - assertDataUpdated(t, tc.className, id1.String()) - }) - - t.Run("merge", func(t *testing.T) { - err := client.Data().Updater(). - WithMerge(). - WithClassName(tc.className). - WithID(id1.String()). - WithProperties(map[string]interface{}{ - "founded": 1960, - "json": map[string]interface{}{ - "lastName": "Smith", - "phones": []interface{}{}, - }, - }).Do(ctx) - require.NoError(t, err) - assertDataMerged(t, tc.className, id1.String()) - }) - }) - } -} diff --git a/test/acceptance_with_go_client/object_property_tests/batch.json b/test/acceptance_with_go_client/object_property_tests/batch.json deleted file mode 100644 index d27f946627e3e7ca10334eecdde277c1cc28a3b1..0000000000000000000000000000000000000000 --- a/test/acceptance_with_go_client/object_property_tests/batch.json +++ /dev/null @@ -1,152 +0,0 @@ -[ - { - "firstName": "John", - "lastName": "Smith", - "profession": "Software Engineer", - "description": "Experienced software engineer with a passion for coding and problem-solving.", - "phone": { - "input": "020 1234567", - "defaultCountry": "us" - }, - "location": { - "latitude": 52.366667, - "longitude": 4.9 - }, - "city": "New York" - }, - { - "firstName": "Alice", - "lastName": "Johnson", - "profession": "Graphic Designer", - "description": "Creative graphic designer with a strong sense of visual aesthetics.", - "phone": { - "input": "020 1234599", - "defaultCountry": "us" - }, - "location": { - "latitude": 52.366667, - "longitude": 4.19 - }, - "city": "Los Angeles" - }, - { - "firstName": "David", - "lastName": "Williams", - "profession": "Accountant", - "description": "Detail-oriented accountant with a knack for numbers and financial analysis.", - "phone": { - "input": "020 1234588", - "defaultCountry": "us" - }, - "location": { - "latitude": 52.366667, - "longitude": 4.29 - }, - "city": "Chicago" - }, - { - "firstName": "Emily", - "lastName": "Davis", - "profession": "Marketing Manager", - "description": "Seasoned marketing manager specializing in digital marketing strategies.", - "phone": { - "input": "020 1234511", - "defaultCountry": "us" - }, - "location": { - "latitude": 52.366667, - "longitude": 4.39 - }, - "city": "Houston" - }, - { - "firstName": "Michael", - "lastName": "Brown", - "profession": "Doctor", - "description": "Dedicated medical doctor committed to providing top-quality healthcare.", - "phone": { - "input": "020 1234111", - "defaultCountry": "us" - }, - "location": { - "latitude": 52.366667, - "longitude": 4.49 - }, - "city": "San Francisco" - }, - { - "firstName": "Olivia", - "lastName": "Miller", - "profession": "Teacher", - "description": "Passionate teacher with a love for educating and nurturing young minds.", - "phone": { - "input": "020 1234589", - "defaultCountry": "us" - }, - "location": { - "latitude": 52.366667, - "longitude": 4.339 - }, - "city": "Miami" - }, - { - "firstName": "William", - "lastName": "Jones", - "profession": "Architect", - "description": "Innovative architect known for creating breathtaking and sustainable designs.", - "phone": { - "input": "020 1234567", - "defaultCountry": "us" - }, - "location": { - "latitude": 52.366667, - "longitude": 5.9 - }, - "city": "Seattle" - }, - { - "firstName": "Sophia", - "lastName": "Wilson", - "profession": "Chef", - "description": "Talented chef renowned for crafting exquisite culinary delights.", - "phone": { - "input": "020 200300300", - "defaultCountry": "us" - }, - "location": { - "latitude": 52.366667, - "longitude": 4.9009 - }, - "city": "Boston" - }, - { - "firstName": "Liam", - "lastName": "Anderson", - "profession": "Financial Analyst", - "description": "Analytical financial analyst with expertise in investment strategies.", - "phone": { - "input": "020 1234444", - "defaultCountry": "us" - }, - "location": { - "latitude": 52.366667, - "longitude": 6.9 - }, - "city": "Dallas" - }, - { - "firstName": "Ava", - "lastName": "Martin", - "profession": "Journalist", - "description": "Dedicated journalist with a passion for reporting the truth.", - "phone": { - "input": "020 1234909", - "defaultCountry": "us" - }, - "location": { - "latitude": 52.366667, - "longitude": 44.9 - }, - "city": "Phoenix" - } -] \ No newline at end of file diff --git a/test/acceptance_with_go_client/object_property_tests/batch_test.go b/test/acceptance_with_go_client/object_property_tests/batch_test.go deleted file mode 100644 index 638b5739c39ab686bf0112e383f509897fc5fe48..0000000000000000000000000000000000000000 --- a/test/acceptance_with_go_client/object_property_tests/batch_test.go +++ /dev/null @@ -1,360 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package object_property_tests - -import ( - "context" - "encoding/json" - "io" - "os" - "testing" - - "github.com/go-openapi/strfmt" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - wvt "github.com/weaviate/weaviate-go-client/v4/weaviate" - "github.com/weaviate/weaviate-go-client/v4/weaviate/grpc" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" -) - -func TestObjectProperty_Batch(t *testing.T) { - ctx := context.Background() - - tests := []struct { - name string - config wvt.Config - }{ - { - name: "single node - rest api", - config: wvt.Config{Scheme: "http", Host: "localhost:8080"}, - }, - { - name: "single node - grpc api", - config: wvt.Config{Scheme: "http", Host: "localhost:8080", GrpcConfig: &grpc.Config{Host: "localhost:50051"}}, - }, - } - for _, tt := range tests { - client, err := wvt.NewClient(tt.config) - require.Nil(t, err) - - id1 := strfmt.UUID("00000000-0000-0000-0000-000000000001") - id2 := strfmt.UUID("00000000-0000-0000-0000-000000000002") - - // clean up DB - err = client.Schema().AllDeleter().Do(context.Background()) - require.Nil(t, err) - - t.Run("batch import object property", func(t *testing.T) { - resp, err := client.Batch().ObjectsBatcher(). - WithObjects( - &models.Object{ - Class: "Person", - ID: id1, - Properties: map[string]interface{}{ - "name": "John Doe", - "json": map[string]interface{}{ - "firstName": "John", - "lastName": "Doe", - "proffession": "Accountant", - "birthdate": "2012-05-05T07:16:30+02:00", - "phoneNumber": map[string]interface{}{ - "input": "020 1234567", - "defaultCountry": "nl", - "internationalFormatted": "+31 20 1234567", - "countryCode": 31, - "national": 201234567, - "nationalFormatted": "020 1234567", - "valid": true, - }, - "location": map[string]interface{}{ - "latitude": 52.366667, - "longitude": 4.9, - }, - }, - }, - }, - &models.Object{ - Class: "Person", - ID: id2, - Properties: map[string]interface{}{ - "name": "Stacey Spears", - "json": map[string]interface{}{ - "firstName": "Stacey", - "lastName": "Spears", - "proffession": "Accountant", - "birthdate": "2011-05-05T07:16:30+02:00", - "phoneNumber": map[string]interface{}{ - "input": "020 1555444", - "defaultCountry": "nl", - "internationalFormatted": "+31 20 1555444", - "countryCode": 31, - "national": 201555444, - "nationalFormatted": "020 1555444", - "valid": true, - }, - "location": map[string]interface{}{ - "latitude": 51.366667, - "longitude": 5.9, - }, - }, - }, - }). - Do(ctx) - require.NoError(t, err) - require.NotNil(t, resp) - require.Len(t, resp, 2) - objs, err := client.Data().ObjectsGetter().WithClassName("Person").WithID(id1.String()).Do(ctx) - require.NoError(t, err) - require.NotNil(t, objs) - require.Len(t, objs, 1) - properties, ok := objs[0].Properties.(map[string]interface{}) - require.True(t, ok) - require.NotNil(t, properties) - jsonProp, ok := properties["json"].(map[string]interface{}) - require.True(t, ok) - require.NotNil(t, jsonProp) - phone, ok := jsonProp["phoneNumber"].(map[string]interface{}) - require.True(t, ok) - assert.Equal(t, 7, len(phone)) - location, ok := jsonProp["location"].(map[string]interface{}) - require.True(t, ok) - assert.Equal(t, 2, len(location)) - }) - - t.Run("batch import object array property", func(t *testing.T) { - resp, err := client.Batch().ObjectsBatcher(). - WithObjects( - &models.Object{ - Class: "People", - ID: id1, - Properties: map[string]interface{}{ - "country": "United Kingdom", - "people": []interface{}{ - map[string]interface{}{ - "firstName": "John", - "lastName": "Doe", - "proffession": "Accountant", - "birthdate": "2012-05-05T07:16:30+02:00", - "phoneNumber": map[string]interface{}{ - "input": "020 1234567", - "defaultCountry": "nl", - }, - "location": map[string]interface{}{ - "latitude": 52.366667, - "longitude": 4.9, - }, - }, - map[string]interface{}{ - "firstName": "Stacey", - "lastName": "Spears", - "proffession": "Accountant", - "birthdate": "2011-05-05T07:16:30+02:00", - "phoneNumber": map[string]interface{}{ - "input": "020 1555444", - "defaultCountry": "nl", - }, - "location": map[string]interface{}{ - "latitude": 51.366667, - "longitude": 5.9, - }, - }, - }, - }, - }, - &models.Object{ - Class: "People", - ID: id2, - Properties: map[string]interface{}{ - "country": "United States of America", - "people": []interface{}{ - map[string]interface{}{ - "firstName": "Robert", - "lastName": "Junior", - "proffession": "Accountant", - "birthdate": "2002-05-05T07:16:30+02:00", - "phoneNumber": map[string]interface{}{ - "input": "020 1234567", - "defaultCountry": "nl", - }, - "location": map[string]interface{}{ - "latitude": 52.366667, - "longitude": 4.9, - }, - }, - map[string]interface{}{ - "firstName": "Steven", - "lastName": "Spears", - "proffession": "Accountant", - "birthdate": "2009-05-05T07:16:30+02:00", - "phoneNumber": map[string]interface{}{ - "input": "020 1555444", - "defaultCountry": "nl", - }, - "location": map[string]interface{}{ - "latitude": 51.366667, - "longitude": 5.9, - }, - }, - }, - }, - }, - ). - Do(ctx) - require.NoError(t, err) - require.NotNil(t, resp) - require.Len(t, resp, 2) - objs, err := client.Data().ObjectsGetter().WithClassName("People").Do(ctx) - require.NoError(t, err) - require.NotNil(t, objs) - require.Len(t, objs, 2) - properties, ok := objs[0].Properties.(map[string]interface{}) - require.True(t, ok) - require.NotNil(t, properties) - people, ok := properties["people"].([]interface{}) - require.True(t, ok) - assert.Equal(t, 2, len(people)) - person, ok := people[0].(map[string]interface{}) - require.True(t, ok) - assert.Equal(t, 6, len(person)) - }) - - t.Run("load json array file using batch api", func(t *testing.T) { - loadJson := func(t *testing.T) []interface{} { - jsonFile, err := os.Open("./batch.json") - require.NoError(t, err) - require.NotNil(t, jsonFile) - byteValue, err := io.ReadAll(jsonFile) - require.NoError(t, err) - require.NotNil(t, byteValue) - var result []interface{} - json.Unmarshal([]byte(byteValue), &result) - return result - } - checkData := func(t *testing.T, className string) { - res, err := client.Data().ObjectsGetter().WithClassName(className).WithID(id1.String()).Do(ctx) - require.Nil(t, err) - require.Len(t, res, 1) - props, ok := res[0].Properties.(map[string]interface{}) - require.True(t, ok) - assert.NotNil(t, props) - assert.Equal(t, 1, len(props)) - objectArrayProp, ok := props["objectArray"].([]interface{}) - require.True(t, ok) - assert.NotNil(t, objectArrayProp) - assert.Equal(t, 10, len(objectArrayProp)) - data, ok := objectArrayProp[0].(map[string]interface{}) - require.True(t, ok) - assert.NotNil(t, data) - assert.Equal(t, 7, len(data)) - phone, ok := data["phone"].(map[string]interface{}) - require.True(t, ok) - assert.NotNil(t, phone) - assert.Equal(t, 2, len(phone)) - location, ok := data["location"].(map[string]interface{}) - require.True(t, ok) - assert.NotNil(t, location) - assert.Equal(t, 2, len(location)) - } - // parse batch.json - loadedJson := loadJson(t) - t.Run("with auto schema", func(t *testing.T) { - className := "BatchArrayJsonAutoSchema" - resp, err := client.Batch().ObjectsBatcher(). - WithObjects(&models.Object{ - Class: className, - ID: id1, - Properties: map[string]interface{}{ - "objectArray": loadedJson, - }, - }).Do(ctx) - require.NoError(t, err) - require.NotNil(t, resp) - checkData(t, className) - }) - t.Run("without auto schema", func(t *testing.T) { - className := "BatchArrayJsonWithoutAutoSchema" - err := client.Schema().ClassCreator().WithClass(&models.Class{ - Class: className, - Properties: []*models.Property{ - { - Name: "objectArray", - DataType: schema.DataTypeObjectArray.PropString(), - NestedProperties: []*models.NestedProperty{ - { - Name: "firstName", - DataType: schema.DataTypeText.PropString(), - }, - { - Name: "lastName", - DataType: schema.DataTypeText.PropString(), - }, - { - Name: "profession", - DataType: schema.DataTypeText.PropString(), - }, - { - Name: "description", - DataType: schema.DataTypeText.PropString(), - }, - { - Name: "phone", - DataType: schema.DataTypeObject.PropString(), - NestedProperties: []*models.NestedProperty{ - { - Name: "input", - DataType: schema.DataTypeText.PropString(), - }, - { - Name: "defaultCountry", - DataType: schema.DataTypeText.PropString(), - }, - }, - }, - { - Name: "location", - DataType: schema.DataTypeObject.PropString(), - NestedProperties: []*models.NestedProperty{ - { - Name: "latitude", - DataType: schema.DataTypeNumber.PropString(), - }, - { - Name: "longitude", - DataType: schema.DataTypeNumber.PropString(), - }, - }, - }, - { - Name: "city", - DataType: schema.DataTypeText.PropString(), - }, - }, - }, - }, - }).Do(ctx) - require.NoError(t, err) - resp, err := client.Batch().ObjectsBatcher(). - WithObjects(&models.Object{ - Class: className, - ID: id1, - Properties: map[string]interface{}{ - "objectArray": loadedJson, - }, - }).Do(ctx) - require.NoError(t, err) - require.NotNil(t, resp) - checkData(t, className) - }) - }) - } -} diff --git a/test/acceptance_with_go_client/object_property_tests/data_test.go b/test/acceptance_with_go_client/object_property_tests/data_test.go deleted file mode 100644 index 4219cda8818b2305fee7785adbb38b39bf440a22..0000000000000000000000000000000000000000 --- a/test/acceptance_with_go_client/object_property_tests/data_test.go +++ /dev/null @@ -1,163 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package object_property_tests - -import ( - "context" - "encoding/json" - "io" - "os" - "testing" - - "acceptance_tests_with_client/fixtures" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - wvt "github.com/weaviate/weaviate-go-client/v4/weaviate" -) - -func TestObjectProperty_Data(t *testing.T) { - ctx := context.Background() - client, err := wvt.NewClient(wvt.Config{Scheme: "http", Host: "localhost:8080"}) - require.Nil(t, err) - - className := fixtures.AllPropertiesClassName - id1 := fixtures.AllPropertiesID1 - - properties := fixtures.AllPropertiesProperties - - // clean up DB - err = client.Schema().AllDeleter().Do(context.Background()) - require.Nil(t, err) - - t.Run("create schema and import object", func(t *testing.T) { - class := fixtures.AllPropertiesClass - err := client.Schema().ClassCreator().WithClass(class).Do(ctx) - require.NoError(t, err) - // add object - _, err = client.Data().Creator(). - WithClassName(className). - WithID(id1). - WithProperties(properties). - Do(context.TODO()) - require.Nil(t, err) - // check if object exists - res, err := client.Data().ObjectsGetter().WithID(id1).Do(ctx) - require.Nil(t, err) - require.Len(t, res, 1) - props, ok := res[0].Properties.(map[string]interface{}) - require.True(t, ok) - assert.NotNil(t, props) - assert.Equal(t, 17, len(props)) - nestedProps, ok := props["nested_objects"].(map[string]interface{}) - require.True(t, ok) - assert.NotNil(t, nestedProps) - assert.Equal(t, 2, len(nestedProps)) - }) - - t.Run("create complicated object using auto schema", func(t *testing.T) { - _, err = client.Data().Creator(). - WithClassName("NestedObject"). - WithID(id1). - WithProperties(map[string]interface{}{ - "nestedProp": map[string]interface{}{ - "a": int64(1), - "nested": map[string]interface{}{ - "b": int64(2), - "nested": map[string]interface{}{ - "c": int64(3), - "nested": map[string]interface{}{ - "d": int64(4), - "nested": map[string]interface{}{ - "e": int64(5), - "nested": map[string]interface{}{ - "f": int64(6), - "nested": map[string]interface{}{ - "g": int64(7), - "nested": map[string]interface{}{ - "h": int64(7), - "nested": map[string]interface{}{ - "i": int64(7), - "nested": map[string]interface{}{ - "j": int64(8), - "nested": map[string]interface{}{ - "k": int64(9), - "nested": map[string]interface{}{ - "l": int64(10), - "nested": map[string]interface{}{ - "m": int64(10), - "nested": map[string]interface{}{ - "n": int64(11), - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }). - Do(context.TODO()) - require.Nil(t, err) - // check if object exists - res, err := client.Data().ObjectsGetter().WithID(id1).Do(ctx) - require.Nil(t, err) - require.Len(t, res, 1) - }) - - t.Run("load complicated json using auto schema", func(t *testing.T) { - jsonFile, err := os.Open("./example.json") - require.NoError(t, err) - require.NotNil(t, jsonFile) - byteValue, err := io.ReadAll(jsonFile) - require.NoError(t, err) - require.NotNil(t, byteValue) - var result map[string]interface{} - json.Unmarshal([]byte(byteValue), &result) - _, err = client.Data().Creator(). - WithClassName("ComplicatedJson"). - WithID(id1). - WithProperties(map[string]interface{}{ - "complicated": result, - }). - Do(context.TODO()) - require.Nil(t, err) - res, err := client.Data().ObjectsGetter().WithClassName("ComplicatedJson").WithID(id1).Do(ctx) - require.Nil(t, err) - require.Len(t, res, 1) - props, ok := res[0].Properties.(map[string]interface{}) - require.True(t, ok) - assert.NotNil(t, props) - assert.Equal(t, 1, len(props)) - complicated, ok := props["complicated"].(map[string]interface{}) - require.True(t, ok) - assert.NotNil(t, complicated) - assert.Equal(t, 2, len(complicated)) - objects, ok := complicated["objects"].([]interface{}) - require.True(t, ok) - assert.NotNil(t, objects) - assert.Equal(t, 2, len(objects)) - data, ok := objects[0].(map[string]interface{}) - require.True(t, ok) - assert.NotNil(t, data) - assert.Equal(t, 6, len(data)) - properties, ok := data["properties"].(map[string]interface{}) - require.True(t, ok) - assert.NotNil(t, properties) - }) -} diff --git a/test/acceptance_with_go_client/object_property_tests/example.json b/test/acceptance_with_go_client/object_property_tests/example.json deleted file mode 100644 index 4f13f27de013b355890136405cbf42af13b8db12..0000000000000000000000000000000000000000 --- a/test/acceptance_with_go_client/object_property_tests/example.json +++ /dev/null @@ -1,119 +0,0 @@ -{ - "objects": [ - { - "class": "ObjectPropertyClass", - "creationTimeUnix": 1696234486729, - "id": "00000000-0000-0000-0000-000000000001", - "lastUpdateTimeUnix": 1696234486729, - "properties": { - "bool": true, - "bools": [ - true, - false - ], - "date": "2009-11-01T23:00:00Z", - "dates": [ - "2009-11-01T23:00:00Z" - ], - "int": 1, - "ints": [ - 1, - 2 - ], - "nested_array_objects": [ - { - "nested_bool_lvl2": true, - "nested_numbers_lvl2": [ - 111.1, - 222.1 - ] - }, - { - "nested_bool_lvl2": false, - "nested_numbers_lvl2": [ - 112.1, - 222.1 - ] - } - ], - "nested_int": 11, - "nested_number": 11.11, - "nested_objects": { - "nested_bool_lvl2": true, - "nested_numbers_lvl2": [ - 11.1, - 22.1 - ] - }, - "nested_text": "nested text", - "number": 1.1, - "numbers": [ - 1.1, - 2.2 - ], - "text": "red", - "texts": [ - "red", - "blue" - ], - "uuid": "00000000-0000-0000-0000-000000000001", - "uuids": [ - "00000000-0000-0000-0000-000000000001" - ] - }, - "vectorWeights": 5000 - }, - { - "class": "NestedObject", - "creationTimeUnix": 1696234486752, - "id": "00000000-0000-0000-0000-000000000001", - "lastUpdateTimeUnix": 1696234486752, - "properties": { - "nestedProp": { - "a": 1, - "nested": { - "b": 2, - "nested": { - "c": 3, - "nested": { - "d": 4, - "nested": { - "e": 5, - "nested": { - "f": 6, - "nested": { - "g": 7, - "nested": { - "h": 7, - "nested": { - "i": 7, - "nested": { - "j": 8, - "nested": { - "k": 9, - "nested": { - "l": 10, - "nested": { - "m": 10, - "nested": { - "n": 11 - } - } - } - } - } - } - } - } - } - } - } - } - } - } - }, - "vectorWeights": 4000 - } - ], - "totalResults": 2 -} \ No newline at end of file diff --git a/test/acceptance_with_go_client/object_property_tests/graphql_test.go b/test/acceptance_with_go_client/object_property_tests/graphql_test.go deleted file mode 100644 index de0bdcd8ba401e3e4a846a3614a5083208abc9f5..0000000000000000000000000000000000000000 --- a/test/acceptance_with_go_client/object_property_tests/graphql_test.go +++ /dev/null @@ -1,129 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package object_property_tests - -import ( - "context" - "testing" - - "github.com/go-openapi/strfmt" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - wvt "github.com/weaviate/weaviate-go-client/v4/weaviate" - "github.com/weaviate/weaviate-go-client/v4/weaviate/filters" - "github.com/weaviate/weaviate-go-client/v4/weaviate/graphql" - "github.com/weaviate/weaviate/entities/models" -) - -func TestObjectProperty_GraphQL(t *testing.T) { - ctx := context.Background() - client, err := wvt.NewClient(wvt.Config{Scheme: "http", Host: "localhost:8080"}) - require.Nil(t, err) - - id1 := strfmt.UUID("00000000-0000-0000-0000-000000000001") - id2 := strfmt.UUID("00000000-0000-0000-0000-000000000002") - - // clean up DB - err = client.Schema().AllDeleter().Do(context.Background()) - require.Nil(t, err) - - t.Run("graphql check", func(t *testing.T) { - className := "Person" - create, err := client.Batch().ObjectsBatcher(). - WithObjects( - &models.Object{ - Class: className, - ID: id1, - Properties: map[string]interface{}{ - "name": "John Doe", - "json": map[string]interface{}{ - "firstName": "John", - "lastName": "Doe", - "proffession": "Accountant", - "birthdate": "2012-05-05T07:16:30+02:00", - "phoneNumber": map[string]interface{}{ - "input": "020 1234567", - "defaultCountry": "nl", - }, - "location": map[string]interface{}{ - "latitude": 52.366667, - "longitude": 4.9, - }, - }, - }, - }, - &models.Object{ - Class: className, - ID: id2, - Properties: map[string]interface{}{ - "name": "Stacey Spears", - "json": map[string]interface{}{ - "firstName": "Stacey", - "lastName": "Spears", - "proffession": "Singer", - "birthdate": "2011-05-05T07:16:30+02:00", - "phoneNumber": map[string]interface{}{ - "input": "020 1555444", - "defaultCountry": "nl", - }, - "location": map[string]interface{}{ - "latitude": 51.366667, - "longitude": 5.9, - }, - }, - }, - }). - Do(ctx) - require.NoError(t, err) - require.NotNil(t, create) - resp, err := client.GraphQL().Get(). - WithClassName(className). - WithWhere(filters.Where(). - WithPath([]string{"id"}). - WithOperator(filters.Equal). - WithValueText(id1.String()), - ). - WithFields( - graphql.Field{Name: "name"}, - graphql.Field{Name: "json{firstName lastName phoneNumber{input} location{latitude longitude}}"}, - graphql.Field{Name: "_additional{id}"}, - ). - Do(ctx) - require.NoError(t, err) - require.NotNil(t, resp) - require.Empty(t, resp.Errors) - require.NotNil(t, resp.Data) - get, ok := resp.Data["Get"].(map[string]interface{}) - require.True(t, ok) - require.Equal(t, 1, len(get)) - person, ok := get["Person"].([]interface{}) - require.True(t, ok) - require.Equal(t, 1, len(person)) - p, ok := person[0].(map[string]interface{}) - require.True(t, ok) - require.Equal(t, 3, len(p)) - _additional, ok := p["_additional"].(map[string]interface{}) - require.True(t, ok) - require.Equal(t, 1, len(_additional)) - assert.Equal(t, id1.String(), _additional["id"]) - assert.Equal(t, "John Doe", p["name"]) - data, ok := p["json"].(map[string]interface{}) - require.True(t, ok) - require.Equal(t, 4, len(data)) - assert.Equal(t, "John", data["firstName"]) - assert.Equal(t, "Doe", data["lastName"]) - phoneNumber, ok := data["phoneNumber"].(map[string]interface{}) - require.True(t, ok) - require.Equal(t, 1, len(phoneNumber)) - assert.Equal(t, "020 1234567", phoneNumber["input"]) - }) -} diff --git a/test/acceptance_with_go_client/schema_test.go b/test/acceptance_with_go_client/schema_test.go deleted file mode 100644 index a3ed7af1ad49c16a8dcbb6c1184bba4831b42e77..0000000000000000000000000000000000000000 --- a/test/acceptance_with_go_client/schema_test.go +++ /dev/null @@ -1,116 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package acceptance_with_go_client - -import ( - "context" - "encoding/json" - "fmt" - "testing" - - "github.com/stretchr/testify/require" - client "github.com/weaviate/weaviate-go-client/v4/weaviate" - "github.com/weaviate/weaviate-go-client/v4/weaviate/fault" - "github.com/weaviate/weaviate-go-client/v4/weaviate/graphql" - "github.com/weaviate/weaviate/entities/models" -) - -type testCase struct { - className1 string - className2 string -} - -func TestSchemaCasingClass(t *testing.T) { - ctx := context.Background() - c, err := client.NewClient(client.Config{Scheme: "http", Host: "localhost:8080"}) - require.Nil(t, err) - - className1 := "RandomGreenCar" - className2 := "RANDOMGreenCar" - - cases := []testCase{ - {className1: className1, className2: className1}, - {className1: className2, className2: className2}, - {className1: className1, className2: className2}, - {className1: className2, className2: className1}, - } - for _, tt := range cases { - t.Run(tt.className1+" "+tt.className2, func(t *testing.T) { - c.Schema().ClassDeleter().WithClassName(tt.className1).Do(ctx) - c.Schema().ClassDeleter().WithClassName(tt.className2).Do(ctx) - class := &models.Class{Class: tt.className1, Vectorizer: "none"} - require.Nil(t, c.Schema().ClassCreator().WithClass(class).Do(ctx)) - - // try to create class again with permuted-casing duplicate. - // this should fail as it already exists - class2 := &models.Class{Class: tt.className2, Vectorizer: "none"} - err := c.Schema().ClassCreator().WithClass(class2).Do(ctx) - checkDuplicateClassErrors(t, err, tt) - - // create object with both casing as class name. - _, err = c.Data().Creator().WithClassName(tt.className1).Do(ctx) - require.Nil(t, err) - // this should fail if the 2nd class is a non-equal permutation of the first - _, err = c.Data().Creator().WithClassName(tt.className2).Do(ctx) - if tt.className1 != tt.className2 { - require.NotNil(t, err) - } else { - require.Nil(t, err) - } - - result, err := c.GraphQL().Aggregate().WithClassName(tt.className1).WithFields(graphql.Field{ - Name: "meta", Fields: []graphql.Field{ - {Name: "count"}, - }, - }).Do(ctx) - require.Nil(t, err) - require.Empty(t, result.Errors) - data := result.Data["Aggregate"].(map[string]interface{})[tt.className1].([]interface{})[0].(map[string]interface{})["meta"].(map[string]interface{})["count"] - if tt.className1 == tt.className2 { - // two objects should have been added if the test case contains exact class name matches - require.Equal(t, data, 2.) - } else { - // otherwise, only one object should have been created, since the permuted class name does not exist - require.Equal(t, data, 1.) - } - - // Regardless of whether a class exists or not, the delete operation will always return a success - require.Nil(t, c.Schema().ClassDeleter().WithClassName(tt.className1).Do(ctx)) - require.Nil(t, c.Schema().ClassDeleter().WithClassName(tt.className2).Do(ctx)) - }) - } -} - -func checkDuplicateClassErrors(t *testing.T, err error, tt testCase) { - require.NotNil(t, err) - rawError, ok := err.(*fault.WeaviateClientError) - if ok { - var clientErr clientError - require.Nil(t, json.Unmarshal([]byte(rawError.Msg), &clientErr)) - require.Len(t, clientErr.Error, 1) - if tt.className1 == tt.className2 { - require.Equal(t, clientErr.Error[0].Message, fmt.Sprintf("class name %q already exists", tt.className2)) - } else { - require.Equal(t, clientErr.Error[0].Message, - fmt.Sprintf("class name %q already exists as a permutation of: %q. class names must be unique when lowercased", - tt.className2, tt.className1)) - } - } else { - t.Fatalf("unexpected error: %v", err) - } -} - -type clientError struct { - Error []struct { - Message string `json:"message"` - } `json:"error"` -} diff --git a/test/acceptance_with_go_client/search_test.go b/test/acceptance_with_go_client/search_test.go deleted file mode 100644 index e2ad8a4516d4a5489361228f86f9b5154ad81678..0000000000000000000000000000000000000000 --- a/test/acceptance_with_go_client/search_test.go +++ /dev/null @@ -1,349 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package acceptance_with_go_client - -import ( - "context" - "fmt" - "testing" - - "github.com/google/uuid" - - "github.com/stretchr/testify/require" - client "github.com/weaviate/weaviate-go-client/v4/weaviate" - "github.com/weaviate/weaviate-go-client/v4/weaviate/graphql" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" -) - -var paragraphs = []string{ - "Some random text", - "Other text", - "completely unrelated", - "this has nothing to do with the rest", -} - -var ( - TRUE = true - ctx = context.Background() -) - -func AddClassAndObjects(t *testing.T, className string, datatype string, c *client.Client, vectorizer string) { - class := &models.Class{ - Class: className, - Properties: []*models.Property{ - {Name: "contents", DataType: []string{datatype}, Tokenization: "word", IndexFilterable: &TRUE, IndexSearchable: &TRUE}, - {Name: "num", DataType: []string{"int"}}, - }, - InvertedIndexConfig: &models.InvertedIndexConfig{Bm25: &models.BM25Config{K1: 1.2, B: 0.75}}, - Vectorizer: vectorizer, - } - require.Nil(t, c.Schema().ClassCreator().WithClass(class).Do(ctx)) - - creator := c.Data().Creator() - _, err := creator.WithClassName(className).WithProperties( - map[string]interface{}{"contents": []string{"nice", "what a rain day"}, "num": 0}).Do(ctx) - require.Nil(t, err) - _, err = creator.WithClassName(className).WithProperties( - map[string]interface{}{"contents": []string{"rain", "snow and sun at once? nice"}, "num": 1}).Do(ctx) - require.Nil(t, err) - _, err = creator.WithClassName(className).WithProperties( - map[string]interface{}{"contents": []string{ - "super long text to get the score down", - "snow and sun at the same time? How nice", - "long text without any meaning", - "just ignore this", - "this too, it doesn't matter", - }, "num": 2}).Do(ctx) - _, err = creator.WithClassName(className).WithProperties( - map[string]interface{}{"contents": []string{ - "super long text to get the score down", - "rain is necessary", - "long text without any meaning", - "just ignore this", - "this too, it doesn't matter", - }, "num": 3}).Do(ctx) -} - -func TestSearchOnArrays(t *testing.T) { - ctx := context.Background() - c, err := client.NewClient(client.Config{Scheme: "http", Host: "localhost:8080"}) - require.Nil(t, err) - - c.Schema().AllDeleter().Do(ctx) - - cases := []struct { - datatype schema.DataType - useHybrid bool // bm25 if not - }{ - {datatype: schema.DataTypeTextArray, useHybrid: true}, - {datatype: schema.DataTypeTextArray, useHybrid: false}, - // deprecated string - {datatype: schema.DataTypeStringArray, useHybrid: false}, - } - for _, tt := range cases { - t.Run("arrays "+tt.datatype.String(), func(t *testing.T) { - className := "Paragraph15845" - class := &models.Class{ - Class: className, - Properties: []*models.Property{ - { - Name: "contents", - DataType: tt.datatype.PropString(), - Tokenization: models.PropertyTokenizationWord, - IndexFilterable: &vFalse, - IndexSearchable: &vTrue, - }, - { - Name: "num", - DataType: schema.DataTypeInt.PropString(), - IndexFilterable: &vTrue, - }, - }, - InvertedIndexConfig: &models.InvertedIndexConfig{Bm25: &models.BM25Config{K1: 1.2, B: 0.75}}, - Vectorizer: "none", - } - require.Nil(t, c.Schema().ClassCreator().WithClass(class).Do(ctx)) - defer c.Schema().ClassDeleter().WithClassName(className).Do(ctx) - - creator := c.Data().Creator() - _, err := creator.WithClassName(className).WithProperties( - map[string]interface{}{"contents": []string{"what a nice day", "what a rainy day"}, "num": 0}).Do(ctx) - require.Nil(t, err) - _, err = creator.WithClassName(className).WithProperties( - map[string]interface{}{"contents": []string{"rain all day", "snow and sun at the same time? How nice"}, "num": 1}).Do(ctx) - require.Nil(t, err) - - var results *models.GraphQLResponse - if tt.useHybrid { - builder := c.GraphQL().HybridArgumentBuilder().WithQuery("nice").WithAlpha(0) - results, err = c.GraphQL().Get().WithClassName(className).WithHybrid(builder).WithFields(graphql.Field{Name: "num"}).Do(ctx) - require.Nil(t, err) - } else { - builder := c.GraphQL().Bm25ArgBuilder().WithQuery("nice").WithProperties("contents") - results, err = c.GraphQL().Get().WithClassName(className).WithBM25(builder).WithFields(graphql.Field{Name: "num"}).Do(ctx) - require.Nil(t, err) - } - result := results.Data["Get"].(map[string]interface{})[className].([]interface{}) - require.Len(t, result, 2) - require.Equal(t, 0., result[0].(map[string]interface{})["num"]) - require.Equal(t, 1., result[1].(map[string]interface{})["num"]) - }) - } -} - -func TestSearchOnSomeProperties(t *testing.T) { - ctx := context.Background() - c, err := client.NewClient(client.Config{Scheme: "http", Host: "localhost:8080"}) - require.Nil(t, err) - - c.Schema().AllDeleter().Do(ctx) - - // only one property contains the search term - cases := []struct { - queryType string // hybrid or bm25 - property string - results int - }{ - {queryType: "bm25", property: "one", results: 1}, - {queryType: "hybrid", property: "one", results: 1}, - {queryType: "bm25", property: "two", results: 0}, - {queryType: "hybrid", property: "two", results: 0}, - } - for _, tt := range cases { - t.Run("search on some properties "+tt.queryType, func(t *testing.T) { - className := "Paragraph15845" - class := &models.Class{ - Class: className, - Properties: []*models.Property{ - { - Name: "one", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWord, - IndexFilterable: &vFalse, - IndexSearchable: &vTrue, - }, - { - Name: "two", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWord, - IndexFilterable: &vFalse, - IndexSearchable: &vTrue, - }, - }, - InvertedIndexConfig: &models.InvertedIndexConfig{Bm25: &models.BM25Config{K1: 1.2, B: 0.75}}, - Vectorizer: "none", - } - require.Nil(t, c.Schema().ClassCreator().WithClass(class).Do(ctx)) - defer c.Schema().ClassDeleter().WithClassName(className).Do(ctx) - - creator := c.Data().Creator() - _, err := creator.WithClassName(className).WithProperties( - map[string]interface{}{"one": "hello", "two": "world"}).Do(ctx) - require.Nil(t, err) - - alpha := "" - if tt.queryType == "hybrid" { - alpha = "alpha:0" // exclude vector search, it doesn't matter for this testcase - } - - results, err := c.GraphQL().Raw().WithQuery(fmt.Sprintf("{Get{%s(%s:{query:\"hello\", properties: [\"%s\"] %s} ){_additional{id}}}}", className, tt.queryType, tt.property, alpha)).Do(ctx) - result := results.Data["Get"].(map[string]interface{})[className].([]interface{}) - require.Len(t, result, tt.results) - }) - } -} - -func TestAutocut(t *testing.T) { - ctx := context.Background() - c := client.New(client.Config{Scheme: "http", Host: "localhost:8080"}) - c.Schema().AllDeleter().Do(ctx) - className := "Paragraph453745" - - AddClassAndObjects(t, className, string(schema.DataTypeTextArray), c, "none") - defer c.Schema().ClassDeleter().WithClassName(className).Do(ctx) - - searchQuery := []string{"hybrid:{query:\"rain nice\", alpha: 0.0, fusionType: relativeScoreFusion", "bm25:{query:\"rain nice\""} - cases := []struct { - autocut int - numResults int - }{ - {autocut: 1, numResults: 2}, {autocut: 2, numResults: 4}, {autocut: -1, numResults: 4 /*disabled*/}, - } - for _, tt := range cases { - for _, search := range searchQuery { - t.Run("autocut "+fmt.Sprint(tt.autocut, " ", search), func(t *testing.T) { - results, err := c.GraphQL().Raw().WithQuery(fmt.Sprintf("{Get{%s(%s, properties: [\"contents\"]}, autocut: %d){num}}}", className, search, tt.autocut)).Do(ctx) - require.Nil(t, err) - result := results.Data["Get"].(map[string]interface{})[className].([]interface{}) - require.Len(t, result, tt.numResults) - require.Equal(t, 0., result[0].(map[string]interface{})["num"]) - require.Equal(t, 1., result[1].(map[string]interface{})["num"]) - }) - } - } -} - -func TestHybridWithPureVectorSearch(t *testing.T) { - ctx := context.Background() - c := client.New(client.Config{Scheme: "http", Host: "localhost:8080"}) - c.Schema().AllDeleter().Do(ctx) - className := "ParagraphWithManyWords" - - AddClassAndObjects(t, className, string(schema.DataTypeTextArray), c, "text2vec-contextionary") - defer c.Schema().ClassDeleter().WithClassName(className).Do(ctx) - - results, err := c.GraphQL().Raw().WithQuery(fmt.Sprintf("{Get{%s(hybrid: {query: \"rain nice\" properties: [\"contents\"], alpha:1}, autocut: -1){num}}}", className)).Do(ctx) - require.Nil(t, err) - result := results.Data["Get"].(map[string]interface{})[className].([]interface{}) - require.Len(t, result, 4) -} - -func TestNearVectorAndObjectAutocut(t *testing.T) { - ctx := context.Background() - c := client.New(client.Config{Scheme: "http", Host: "localhost:8080"}) - c.Schema().AllDeleter().Do(ctx) - className := "YellowAndBlueTrain" - - class := &models.Class{ - Class: className, - Vectorizer: "none", - } - require.Nil(t, c.Schema().ClassCreator().WithClass(class).Do(ctx)) - defer c.Schema().ClassDeleter().WithClassName(className).Do(ctx) - - var uuids []string - creator := c.Data().Creator() - vectorNumbers := []float32{1, 1.1, 1.2, 2.0, 2.1, 2.2, 3.1, 3.2, 3.2} - for _, vectorNumber := range vectorNumbers { - uuids = append(uuids, uuid.New().String()) - _, err := creator.WithClassName(className).WithVector([]float32{1, 1, 1, 1, 1, vectorNumber}).WithID(uuids[len(uuids)-1]).Do(ctx) - require.Nil(t, err) - } - - t.Run("near vector", func(t *testing.T) { - cases := []struct { - autocut int - numResults int - }{ - {autocut: 1, numResults: 3}, {autocut: 2, numResults: 6}, {autocut: -1, numResults: 9 /*disabled*/}, - } - for _, tt := range cases { - t.Run("autocut "+fmt.Sprint(tt.autocut), func(t *testing.T) { - results, err := c.GraphQL().Raw().WithQuery(fmt.Sprintf("{Get{%s(nearVector:{vector:[1, 1, 1, 1, 1, 1]}, autocut: %d){_additional{vector}}}}", className, tt.autocut)).Do(ctx) - require.Nil(t, err) - result := results.Data["Get"].(map[string]interface{})[className].([]interface{}) - require.Len(t, result, tt.numResults) - }) - } - }) - - t.Run("near object", func(t *testing.T) { - cases := []struct { - autocut int - numResults int - }{ - {autocut: 1, numResults: 3}, {autocut: 2, numResults: 6}, {autocut: -1, numResults: 9 /*disabled*/}, - } - for _, tt := range cases { - t.Run("autocut "+fmt.Sprint(tt.autocut), func(t *testing.T) { - results, err := c.GraphQL().Raw().WithQuery(fmt.Sprintf("{Get{%s(nearObject:{id:%q}, autocut: %d){_additional{vector}}}}", className, uuids[0], tt.autocut)).Do(ctx) - require.Nil(t, err) - result := results.Data["Get"].(map[string]interface{})[className].([]interface{}) - require.Len(t, result, tt.numResults) - }) - } - }) -} - -func TestNearTextAutocut(t *testing.T) { - ctx := context.Background() - c := client.New(client.Config{Scheme: "http", Host: "localhost:8080"}) - c.Schema().AllDeleter().Do(ctx) - className := "YellowAndBlueSub" - - class := &models.Class{ - Class: className, - Properties: []*models.Property{ - { - Name: "text", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWord, - }, - }, - Vectorizer: "text2vec-contextionary", - } - require.Nil(t, c.Schema().ClassCreator().WithClass(class).Do(ctx)) - defer c.Schema().ClassDeleter().WithClassName(className).Do(ctx) - - creator := c.Data().Creator() - - texts := []string{"word", "another word", "another word and", "completely unrelated"} - for _, text := range texts { - _, err := creator.WithClassName(className).WithProperties(map[string]interface{}{"text": text}).Do(ctx) - require.Nil(t, err) - } - cases := []struct { - autocut int - numResults int - }{ - {autocut: 1, numResults: 3}, {autocut: -1, numResults: 4 /*disabled*/}, - } - for _, tt := range cases { - t.Run("autocut "+fmt.Sprint(tt.autocut), func(t *testing.T) { - results, err := c.GraphQL().Raw().WithQuery(fmt.Sprintf("{Get{%s(nearText:{concepts: \"word\"}, autocut: %d){_additional{vector}}}}", className, tt.autocut)).Do(ctx) - require.Nil(t, err) - result := results.Data["Get"].(map[string]interface{})[className].([]interface{}) - require.Len(t, result, tt.numResults) - }) - } -} diff --git a/test/acceptance_with_go_client/vector_dimensions_test.go b/test/acceptance_with_go_client/vector_dimensions_test.go deleted file mode 100644 index 8409c4790bd9bcdbb61eff3750ff453994c2fc53..0000000000000000000000000000000000000000 --- a/test/acceptance_with_go_client/vector_dimensions_test.go +++ /dev/null @@ -1,123 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package acceptance_with_go_client - -import ( - "context" - "fmt" - "math/rand" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - client "github.com/weaviate/weaviate-go-client/v4/weaviate" - "github.com/weaviate/weaviate-go-client/v4/weaviate/graphql" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" -) - -func TestEdgeVectorDimensions(t *testing.T) { - r := rand.New(rand.NewSource(time.Now().UnixNano())) - c := client.New(client.Config{Scheme: "http", Host: "localhost:8080"}) - ctx := context.Background() - - objID1 := "00000000-0000-0000-0000-000000000001" - objID2 := "00000000-0000-0000-0000-000000000002" - className := "VectorDimensions65k" - propName := "title" - - // 65535 is the max value for uint16 - maxUint16 := uint16(65535) - - for _, vectorLength := range []uint16{1, 50000, maxUint16} { - t.Run(fmt.Sprintf("%v vector dimensions", vectorLength), func(t *testing.T) { - class := &models.Class{ - Class: className, - Properties: []*models.Property{ - {Name: propName, DataType: []string{string(schema.DataTypeText)}, IndexInverted: &vTrue}, - }, - Vectorizer: "none", - } - - // delete class if exists and cleanup after test - c.Schema().ClassDeleter().WithClassName(className).Do(ctx) - require.Nil(t, c.Schema().ClassCreator().WithClass(class).Do(ctx)) - defer c.Schema().ClassDeleter().WithClassName(className).Do(ctx) - - t.Run("insert vectors", func(t *testing.T) { - generateVector := func(dims uint16) []float32 { - vector := make([]float32, dims) - for i := range vector { - vector[i] = r.Float32() - } - return vector - } - for i, objID := range []string{objID1, objID2} { - _, err := c.Data().Creator(). - WithClassName(className).WithID(objID). - WithProperties(map[string]interface{}{ - propName: fmt.Sprintf("title %v", i), - }). - WithVector(generateVector(vectorLength)). - Do(ctx) - require.Nil(t, err) - } - }) - - t.Run("check Aggregate", func(t *testing.T) { - getCount := func(t *testing.T, result *models.GraphQLResponse) int { - aggregate, ok := result.Data["Aggregate"].(map[string]interface{}) - require.True(t, ok) - require.NotNil(t, aggregate) - class, ok := aggregate[className].([]interface{}) - require.True(t, ok) - require.Len(t, class, 1) - title, ok := class[0].(map[string]interface{}) - require.True(t, ok) - require.NotNil(t, title) - count, ok := title["title"].(map[string]interface{}) - require.True(t, ok) - require.NotNil(t, count) - titleCount, ok := count["count"].(float64) - require.True(t, ok) - return int(titleCount) - } - result, err := c.GraphQL().Aggregate().WithClassName(className).WithFields(graphql.Field{ - Name: propName, Fields: []graphql.Field{ - {Name: "count"}, - }, - }).Do(ctx) - require.Nil(t, err) - require.Empty(t, result.Errors) - assert.Equal(t, 2, getCount(t, result)) - }) - - t.Run("check nearObject", func(t *testing.T) { - nearObject := c.GraphQL().NearObjectArgBuilder().WithID(objID1) - result, err := c.GraphQL().Get(). - WithClassName(className). - WithNearObject(nearObject). - WithFields(graphql.Field{Name: propName}). - Do(ctx) - require.Nil(t, err) - require.Empty(t, result.Errors) - get, ok := result.Data["Get"].(map[string]interface{}) - require.True(t, ok) - require.NotNil(t, get) - class, ok := get[className].([]interface{}) - require.True(t, ok) - require.Len(t, class, 2) - }) - }) - } -} diff --git a/test/acceptance_with_go_client/where_filters_test.go b/test/acceptance_with_go_client/where_filters_test.go deleted file mode 100644 index ba35d4d983a9ebb4f48e0f091c2beeba75664635..0000000000000000000000000000000000000000 --- a/test/acceptance_with_go_client/where_filters_test.go +++ /dev/null @@ -1,55 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package acceptance_with_go_client - -import ( - "context" - "testing" - - "github.com/weaviate/weaviate-go-client/v4/weaviate/graphql" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate-go-client/v4/weaviate/filters" - - client "github.com/weaviate/weaviate-go-client/v4/weaviate" -) - -func TestCorrectErrorForIsNullFilter(t *testing.T) { - c := client.New(client.Config{Scheme: "http", Host: "localhost:8080"}) - ctx := context.Background() - - className := "RandomClass45357" - propName := "title" - class := &models.Class{ - Class: className, - Properties: []*models.Property{ - {Name: propName, DataType: []string{string(schema.DataTypeText)}, IndexInverted: &vTrue}, - }, - InvertedIndexConfig: &models.InvertedIndexConfig{IndexNullState: true}, - Vectorizer: "none", - } - - // delete class if exists and cleanup after test - c.Schema().ClassDeleter().WithClassName(className).Do(ctx) - require.Nil(t, c.Schema().ClassCreator().WithClass(class).Do(ctx)) - defer c.Schema().ClassDeleter().WithClassName(className).Do(ctx) - - filter := filters.Where() - filter.WithOperator(filters.IsNull) - filter.WithValueString("asd") // wrong type - filter.WithPath([]string{propName}) - result, err := c.GraphQL().Get().WithClassName(className).WithWhere(filter).WithFields(graphql.Field{Name: propName}).Do(ctx) - require.Nil(t, err) - require.Contains(t, result.Errors[0].Message, "booleanValue") -} diff --git a/test/benchmark/.gitattributes b/test/benchmark/.gitattributes deleted file mode 100644 index 0cb8c2c26da5a4725d0658b5f53ae3feba414781..0000000000000000000000000000000000000000 --- a/test/benchmark/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -*.fvecs filter=lfs diff=lfs merge=lfs -text diff --git a/test/benchmark/benchmark.go b/test/benchmark/benchmark.go deleted file mode 100644 index 6209aa8ffd115432e542ce1ac491e17153c527dd..0000000000000000000000000000000000000000 --- a/test/benchmark/benchmark.go +++ /dev/null @@ -1,262 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// Package implements performance tracking examples - -package main - -import ( - "bytes" - "encoding/json" - "flag" - "fmt" - "io" - "net" - "net/http" - "os" - "os/exec" - "time" - - "github.com/pkg/errors" - - "github.com/weaviate/weaviate/entities/models" -) - -type batch struct { - Objects []*models.Object -} - -type benchmarkResult map[string]map[string]int64 - -func main() { - var benchmarkName string - var numBatches, failPercentage, maxEntries int - - flag.StringVar(&benchmarkName, "name", "SIFT", "Which benchmark should be run. Currently only SIFT is available.") - flag.IntVar(&maxEntries, "numberEntries", 100000, "Maximum number of entries read from the dataset") - flag.IntVar(&numBatches, "numBatches", 1, "With how many parallel batches objects should be added") - flag.IntVar(&failPercentage, "fail", -1, "Fail if regression is larger") - flag.Parse() - - t := &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 120 * time.Second, - }).DialContext, - MaxIdleConnsPerHost: 100, - MaxIdleConns: 100, - IdleConnTimeout: 90 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - } - c := &http.Client{Transport: t} - url := "http://localhost:8080/v1/" - - alreadyRunning := startWeaviate(c, url) - - var newRuntime map[string]int64 - var err error - switch benchmarkName { - case "SIFT": - newRuntime, err = benchmarkSift(c, url, maxEntries, numBatches) - default: - panic("Unknown benchmark " + benchmarkName) - } - - if err != nil { - clearExistingObjects(c, url) - } - - if !alreadyRunning { - tearDownWeaviate() - } - - if err != nil { - panic(errors.Wrap(err, "Error occurred during benchmarking")) - } - - FullBenchmarkName := benchmarkName + "-" + fmt.Sprint(maxEntries) + "_Entries-" + fmt.Sprint(numBatches) + "_Batch(es)" - - // Write results to file, keeping existing entries - oldBenchmarkRunTimes := readCurrentBenchmarkResults() - oldRuntime := oldBenchmarkRunTimes[FullBenchmarkName] - oldBenchmarkRunTimes[FullBenchmarkName] = newRuntime - benchmarkJSON, _ := json.MarshalIndent(oldBenchmarkRunTimes, "", "\t") - if err := os.WriteFile("benchmark_results.json", benchmarkJSON, 0o666); err != nil { - panic(err) - } - - totalNewRuntime := int64(0) - for _, runtime := range newRuntime { - totalNewRuntime += runtime - } - totalOldRuntime := int64(0) - for _, runtime := range oldRuntime { - totalOldRuntime += runtime - } - - fmt.Fprint( - os.Stdout, - "Runtime for benchmark "+FullBenchmarkName+ - ": old total runtime: "+fmt.Sprint(totalOldRuntime)+"ms, new total runtime:"+fmt.Sprint(totalNewRuntime)+"ms.\n"+ - "This is a change of "+fmt.Sprintf("%.2f", 100*float32(totalNewRuntime-totalOldRuntime)/float32(totalNewRuntime))+"%.\n"+ - "Please update the benchmark results if necessary.\n\n", - ) - fmt.Fprint(os.Stdout, "Runtime for individual steps:.\n") - for name, time := range newRuntime { - fmt.Fprint(os.Stdout, "Runtime for "+name+" is "+fmt.Sprint(time)+"ms.\n") - } - - // Return with error code if runtime regressed and corresponding flag was set - if failPercentage >= 0 && - totalOldRuntime > 0 && // don't report regression if no old entry exists - float64(totalOldRuntime)*(1.0+0.01*float64(failPercentage)) < float64(totalNewRuntime) { - fmt.Fprint( - os.Stderr, "Failed due to performance regressions.\n", - ) - os.Exit(1) - } -} - -// If there is already a schema present, clear it out -func clearExistingObjects(c *http.Client, url string) { - checkSchemaRequest := createRequest(url+"schema", "GET", nil) - checkSchemaResponseCode, body, _, err := performRequest(c, checkSchemaRequest) - if err != nil { - panic(errors.Wrap(err, "perform request")) - } - if checkSchemaResponseCode != 200 { - return - } - - var dump models.Schema - if err := json.Unmarshal(body, &dump); err != nil { - panic(errors.Wrap(err, "Could not unmarshal read response")) - } - for _, classObj := range dump.Classes { - requestDelete := createRequest(url+"schema/"+classObj.Class, "DELETE", nil) - responseDeleteCode, _, _, err := performRequest(c, requestDelete) - if err != nil { - panic(errors.Wrap(err, "Could delete schema")) - } - if responseDeleteCode != 200 { - panic(fmt.Sprintf("Could not delete schema, code: %v", responseDeleteCode)) - } - } -} - -func command(app string, arguments []string, waitForCompletion bool) error { - mydir, err := os.Getwd() - if err != nil { - return err - } - - cmd := exec.Command(app, arguments...) - execDir := mydir + "/../../" - cmd.Dir = execDir - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - if waitForCompletion { - err = cmd.Run() - } else { - err = cmd.Start() - } - - return err -} - -func readCurrentBenchmarkResults() benchmarkResult { - benchmarkFile, err := os.Open("benchmark_results.json") - if err != nil { - fmt.Print("No benchmark file present.") - return make(benchmarkResult) - } - defer benchmarkFile.Close() - - var result benchmarkResult - jsonParser := json.NewDecoder(benchmarkFile) - if err = jsonParser.Decode(&result); err != nil { - panic("Could not parse existing benchmark file.") - } - return result -} - -func tearDownWeaviate() error { - fmt.Print("Shutting down weaviate.\n") - app := "docker-compose" - arguments := []string{ - "down", - "--remove-orphans", - } - return command(app, arguments, true) -} - -// start weaviate in case it was not already started -// -// We want to benchmark the current state and therefore need to rebuild and then start a docker container -func startWeaviate(c *http.Client, url string) bool { - requestReady := createRequest(url+".well-known/ready", "GET", nil) - - responseStartedCode, _, _, err := performRequest(c, requestReady) - alreadyRunning := err == nil && responseStartedCode == 200 - - if alreadyRunning { - fmt.Print("Weaviate instance already running.\n") - return alreadyRunning - } - - fmt.Print("(Re-) build and start weaviate.\n") - cmd := "./tools/test/run_ci_server.sh" - if err := command(cmd, []string{}, true); err != nil { - panic(errors.Wrap(err, "Command to (re-) build and start weaviate failed")) - } - return false -} - -// createRequest creates requests -func createRequest(url string, method string, payload interface{}) *http.Request { - var body io.Reader = nil - if payload != nil { - jsonBody, err := json.Marshal(payload) - if err != nil { - panic(errors.Wrap(err, "Could not marshal request")) - } - body = bytes.NewBuffer(jsonBody) - } - request, err := http.NewRequest(method, url, body) - if err != nil { - panic(errors.Wrap(err, "Could not create request")) - } - request.Header.Add("Content-Type", "application/json") - request.Header.Add("Accept", "application/json") - - return request -} - -// performRequest runs requests -func performRequest(c *http.Client, request *http.Request) (int, []byte, int64, error) { - timeStart := time.Now() - response, err := c.Do(request) - requestTime := time.Since(timeStart).Milliseconds() - - if err != nil { - return 0, nil, requestTime, err - } - - body, err := io.ReadAll(response.Body) - response.Body.Close() - if err != nil { - return 0, nil, requestTime, err - } - - return response.StatusCode, body, requestTime, nil -} diff --git a/test/benchmark/benchmark_sift.go b/test/benchmark/benchmark_sift.go deleted file mode 100644 index 2f1e185ee4694f0f5f650aa41f8325fe67a3b1c7..0000000000000000000000000000000000000000 --- a/test/benchmark/benchmark_sift.go +++ /dev/null @@ -1,241 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package main - -import ( - "encoding/binary" - "encoding/json" - "fmt" - "io" - "math" - "net/http" - "os" - "sync" - - "github.com/go-openapi/strfmt" - "github.com/google/uuid" - "github.com/pkg/errors" - "github.com/weaviate/weaviate/entities/models" -) - -const ( - class = "Benchmark" - nrSearchResults = 79 -) - -func createSchemaSIFTRequest(url string) *http.Request { - classObj := &models.Class{ - Class: class, - Description: "Dummy class for benchmarking purposes", - Properties: []*models.Property{ - { - DataType: []string{"int"}, - Description: "The value of the counter in the dataset", - Name: "counter", - }, - }, - VectorIndexConfig: map[string]interface{}{ // values are from benchmark script - "distance": "l2-squared", - "ef": -1, - "efConstruction": 64, - "maxConnections": 64, - "vectorCacheMaxObjects": 1000000000, - }, - Vectorizer: "none", - } - request := createRequest(url+"schema", "POST", classObj) - return request -} - -func float32FromBytes(bytes []byte) float32 { - bits := binary.LittleEndian.Uint32(bytes) - float := math.Float32frombits(bits) - return float -} - -func int32FromBytes(bytes []byte) int { - return int(binary.LittleEndian.Uint32(bytes)) -} - -func readSiftFloat(file string, maxObjects int) []*models.Object { - var objects []*models.Object - - f, err := os.Open("sift/" + file) - if err != nil { - panic(errors.Wrap(err, "Could not open SIFT file")) - } - defer f.Close() - - fi, err := f.Stat() - if err != nil { - panic(errors.Wrap(err, "Could not get SIFT file properties")) - } - fileSize := fi.Size() - if fileSize < 1000000 { - panic("The file is only " + fmt.Sprint(fileSize) + " bytes long. Did you forgot to install git lfs?") - } - - // The sift data is a binary file containing floating point vectors - // For each entry, the first 4 bytes is the length of the vector (in number of floats, not in bytes) - // which is followed by the vector data with vector length * 4 bytes. - // |-length-vec1 (4bytes)-|-Vec1-data-(4*length-vector-1 bytes)-|-length-vec2 (4bytes)-|-Vec2-data-(4*length-vector-2 bytes)-| - // The vector length needs to be converted from bytes to int - // The vector data needs to be converted from bytes to float - // Note that the vector entries are of type float but are integer numbers eg 2.0 - bytesPerF := 4 - vectorLengthFloat := 128 - vectorBytes := make([]byte, bytesPerF+vectorLengthFloat*bytesPerF) - for i := 0; i >= 0; i++ { - _, err = f.Read(vectorBytes) - if err == io.EOF { - break - } else if err != nil { - panic(err) - } - if int32FromBytes(vectorBytes[0:bytesPerF]) != vectorLengthFloat { - panic("Each vector must have 128 entries.") - } - var vectorFloat []float32 - for j := 0; j < vectorLengthFloat; j++ { - start := (j + 1) * bytesPerF // first bytesPerF are length of vector - vectorFloat = append(vectorFloat, float32FromBytes(vectorBytes[start:start+bytesPerF])) - } - ObjectUuid := uuid.New() - object := &models.Object{ - Class: class, - ID: strfmt.UUID(ObjectUuid.String()), - Vector: models.C11yVector(vectorFloat), - Properties: map[string]interface{}{ - "counter": i, - }, - } - objects = append(objects, object) - - if i >= maxObjects { - break - } - } - if len(objects) < maxObjects { - panic("Could not load all elements.") - } - - return objects -} - -func benchmarkSift(c *http.Client, url string, maxObjects, numBatches int) (map[string]int64, error) { - clearExistingObjects(c, url) - objects := readSiftFloat("sift_base.fvecs", maxObjects) - queries := readSiftFloat("sift_query.fvecs", maxObjects/100) - requestSchema := createSchemaSIFTRequest(url) - - passedTime := make(map[string]int64) - - // Add schema - responseSchemaCode, _, timeSchema, err := performRequest(c, requestSchema) - passedTime["AddSchema"] = timeSchema - if err != nil { - return nil, errors.Wrap(err, "Could not add schema, error: ") - } else if responseSchemaCode != 200 { - return nil, errors.Errorf("Could not add schma, http error code: %v", responseSchemaCode) - } - - // Batch-add - passedTime["BatchAdd"] = 0 - wg := sync.WaitGroup{} - batchSize := len(objects) / numBatches - errorChan := make(chan error, numBatches) - timeChan := make(chan int64, numBatches) - - for i := 0; i < numBatches; i++ { - wg.Add(1) - go func(batchId int, errChan chan<- error) { - batchObjects := objects[batchId*batchSize : (batchId+1)*batchSize] - requestAdd := createRequest(url+"batch/objects", "POST", batch{batchObjects}) - responseAddCode, _, timeBatchAdd, err := performRequest(c, requestAdd) - - timeChan <- timeBatchAdd - if err != nil { - errChan <- errors.Wrap(err, "Could not add batch, error: ") - } else if responseAddCode != 200 { - errChan <- errors.Errorf("Could not add batch, http error code: %v", responseAddCode) - } - wg.Done() - }(i, errorChan) - - } - wg.Wait() - close(errorChan) - close(timeChan) - for err := range errorChan { - return nil, err - } - for timing := range timeChan { - passedTime["BatchAdd"] += timing - } - - // Read entries - nrSearchResultsUse := nrSearchResults - if maxObjects < nrSearchResultsUse { - nrSearchResultsUse = maxObjects - } - requestRead := createRequest(url+"objects?limit="+fmt.Sprint(nrSearchResultsUse)+"&class="+class, "GET", nil) - responseReadCode, body, timeGetObjects, err := performRequest(c, requestRead) - passedTime["GetObjects"] = timeGetObjects - if err != nil { - return nil, errors.Wrap(err, "Could not read objects") - } else if responseReadCode != 200 { - return nil, errors.New("Could not read objects, http error code: " + fmt.Sprint(responseReadCode)) - } - var result map[string]interface{} - if err := json.Unmarshal(body, &result); err != nil { - return nil, errors.Wrap(err, "Could not unmarshal read response") - } - if int(result["totalResults"].(float64)) != nrSearchResultsUse { - errString := "Found " + fmt.Sprint(int(result["totalResults"].(float64))) + - " results. Expected " + fmt.Sprint(nrSearchResultsUse) + "." - return nil, errors.New(errString) - } - - // Use sample queries - for _, query := range queries { - queryString := "{Get{" + class + "(nearVector: {vector:" + fmt.Sprint(query.Vector) + " }){counter}}}" - requestQuery := createRequest(url+"graphql", "POST", models.GraphQLQuery{ - Query: queryString, - }) - responseQueryCode, body, timeQuery, err := performRequest(c, requestQuery) - passedTime["Query"] += timeQuery - if err != nil { - return nil, errors.Wrap(err, "Could not query objects") - } else if responseQueryCode != 200 { - return nil, errors.Errorf("Could not query objects, http error code: %v", responseQueryCode) - } - var result map[string]interface{} - if err := json.Unmarshal(body, &result); err != nil { - return nil, errors.Wrap(err, "Could not unmarshal query response") - } - if result["data"] == nil || result["errors"] != nil { - return nil, errors.New("GraphQL Error") - } - } - - // Delete class (with schema and all entries) to clear all entries so next round can start fresh - requestDelete := createRequest(url+"schema/"+class, "DELETE", nil) - responseDeleteCode, _, timeDelete, err := performRequest(c, requestDelete) - passedTime["Delete"] += timeDelete - if err != nil { - return nil, errors.Wrap(err, "Could not delete class") - } else if responseDeleteCode != 200 { - return nil, errors.Errorf("Could not delete class, http error code: %v", responseDeleteCode) - } - - return passedTime, nil -} diff --git a/test/benchmark/remote/README.md b/test/benchmark/remote/README.md deleted file mode 100644 index c8663c19177bf3b5a57f7e7f043ef7a67311823d..0000000000000000000000000000000000000000 --- a/test/benchmark/remote/README.md +++ /dev/null @@ -1,52 +0,0 @@ -This folder contains scripts to help execute the benchmark remotely on a GCP -VM. - -## Requirements - -- Bash -- gcloud (logged in and permissions to `semi-automated-benchmarking` project -- terraform - -## Usage - -From the root folder run - -``` -test/benchmark/remote/run.sh --all -``` - -To run the whole benchmark suite. The `--all` command will: - -- Check that `gcloud` and `terraform` are installed locally -- Spin up a machine in GCP -- Install the dependencies on this machine -- Clone Weaviate -- Check out the same commit that is check out locally -- Run the benchmarks script -- Copy the results file to the local machine -- Destroy the machine - -If any of the command fails the machine will be destroyed. - -## Debugging - -You can also run all commands individually, simply run the command without -parameters to list all possible options. - -To do all steps (including spinning up a machine) that happen _before_ running -the benchmarks you can invoke it with the `--prepare` option. To get an -interactive ssh session into the machine, use `--ssh`. To destroy the machine -at an arbitrary point run `--delete_machine`. - -## Run on arbitrary branch - -*Note: Checking out a branch that does not yet have the scripts from this -folder, will fail. You need a branch created after this script was initially -built or has been rebased on top of it.* - -``` -test/benchmark/remote/run.sh --prepare -test/benchmark/remote/run.sh --checkout -test/benchmark/remote/run.sh --benchmark -test/benchmark/remote/run.sh --delete_machine -``` diff --git a/test/benchmark/remote/run.sh b/test/benchmark/remote/run.sh deleted file mode 100644 index 0a2cf5ce4d605e9ccf056b369b434504a7561b2b..0000000000000000000000000000000000000000 --- a/test/benchmark/remote/run.sh +++ /dev/null @@ -1,156 +0,0 @@ -#!/bin/bash - -PROJECT=semi-automated-benchmarking -ZONE=us-central1-a -INSTANCE=automated-loadtest -GOVERSION=https://go.dev/dl/go1.18.4.linux-amd64.tar.gz -FILE_PREFIX=${FILE_PREFIX:-""} - -set -eou pipefail - -# change to script directory -cd "${0%/*}" || exit - -function main() { - while [[ "$#" -gt 0 ]]; do - case $1 in - --all) run_all; exit 0 ;; - --create_machine) create_machine; exit 0 ;; - --clone_repository) clone_repository; exit 0;; - --delete_machine) delete_machine; exit 0;; - --install_dependencies) install_dependencies; exit 0;; - --benchmark) benchmark; exit 0;; - --prepare) prepare; exit 0;; - --checkout) checkout "$2" ; exit 0;; - --ssh) interactive_ssh; exit 0;; - *) echo "Unknown parameter passed: $1"; exit 1 ;; - esac - shift - done - - print_help -} - -function print_help() { - echo "Valid arguments include:" - echo "" - echo " --all Run everything, including machine creation & destruction" - echo " --prepare Create Machine & run all the steps prior to benchmark execution" - echo " --create_machine Only create machine" - echo " --delete_machine Stop & Delete running machine" - echo " --clone_repository Clone and checkout Weaviate repo at specified commit" - echo " --checkout Checkout arbitrary branch or commit" - echo " --ssh Interactive SSH session" -} - -function run_all() { - trap delete_machine EXIT - - prepare - benchmark -} - -function prepare() { - check - create_machine - install_dependencies - clone_repository -} - -function check() { - echo_green "Checking required dependencies" - if ! command -v gcloud &> /dev/null - then - echo_red "Missing gcloud binary" - return 1 - fi - - if ! command -v terraform &> /dev/null - then - echo_red "Missing terraform binary" - return 1 - fi - - echo "Ready to go!" -} - -function create_machine() { - (cd terraform && terraform init) - (cd terraform && terraform apply -auto-approve) - echo "Sleeping for 10s, so first ssh doesn't fail. This should be improved through polling" - sleep 10 -} - -function delete_machine() { - (cd terraform && terraform destroy -auto-approve) -} - -function install_dependencies() { - ssh_command "sudo apt-get update && sudo apt-get install -y git git-lfs curl" - ssh_command "ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts" - install_go - install_docker -} - -function install_go { - ssh_command "curl -Lo go.tar.gz $GOVERSION" - ssh_command "sudo rm -rf /usr/local/go && sudo tar -C /usr/local -xzf go.tar.gz" - ssh_command 'echo '"'"'PATH=$PATH:/usr/local/go/bin'"'"' >> ~/.profile' - ssh_command "go version" -} - -function install_docker() { - ssh_command "if ! command -v docker &> /dev/null; then curl -fsSL https://get.docker.com -o get-docker.sh && sh ./get-docker.sh; fi" - ssh_command "sudo groupadd docker || true" - ssh_command "sudo usermod -aG docker $USER" -} - -function clone_repository() { - ref=$(git rev-parse --abbrev-ref HEAD) - echo_green "Cloning weaviate repo to branch $ref" - ssh_command "cd; [ ! -d weaviate ] && git clone --depth 1 --branch $ref https://github.com/weaviate/weaviate.git weaviate || true" - ssh_command "cd weaviate; git-lfs install; git-lfs pull" -} - -function checkout() { - ref="$1" - ssh_command "cd weaviate; git checkout $ref" -} - -function benchmark() { - echo_green "Run benchmarks on remote machine" - ssh_command "echo "stop all running docker containers"; docker rm -f $(docker ps -q) || true" - ssh_command "cd ~/weaviate; rm test/benchmark/benchmark_results.json || true" - ssh_command "cd ~/weaviate; test/benchmark/run_performance_tracker.sh" - echo_green "Copy results file to local machine" - filename="${FILE_PREFIX}benchmark_results_$(date +%s).json" - scp_command "$INSTANCE:~/weaviate/test/benchmark/benchmark_results.json" "$filename" - echo "Results file succesfully copied to ${PWD}/$filename" -} - - -function echo_green() { - green='\033[0;32m' - nc='\033[0m' - echo -e "${green}${*}${nc}" -} - -function echo_red() { - red='\033[0;31m' - nc='\033[0m' - echo -e "${red}${*}${nc}" -} - -function ssh_command() { - gcloud beta compute ssh --project=$PROJECT --zone=$ZONE "$INSTANCE" --command="source ~/.profile; $1" -} - -function scp_command() { - gcloud beta compute scp --project=$PROJECT --zone=$ZONE "$@" -} - -function interactive_ssh() { - gcloud beta compute ssh --project=$PROJECT --zone=$ZONE "$INSTANCE" -} - -main "$@" diff --git a/test/benchmark/remote/terraform/.gitignore b/test/benchmark/remote/terraform/.gitignore deleted file mode 100644 index d43420d8acfa979f86003092cbcc4e5078b15a38..0000000000000000000000000000000000000000 --- a/test/benchmark/remote/terraform/.gitignore +++ /dev/null @@ -1,9 +0,0 @@ -# Local .terraform directories -**/.terraform/* - -.terraform.lock.hcl - - -# .tfstate files -*.tfstate -*.tfstate.* diff --git a/test/benchmark/remote/terraform/main.tf b/test/benchmark/remote/terraform/main.tf deleted file mode 100644 index 45de42838b8d2d053b11dfceaa5a61d4692be356..0000000000000000000000000000000000000000 --- a/test/benchmark/remote/terraform/main.tf +++ /dev/null @@ -1,3 +0,0 @@ -provider "google" { - project = "semi-automated-benchmarking" -} diff --git a/test/benchmark/remote/terraform/variables.tf b/test/benchmark/remote/terraform/variables.tf deleted file mode 100644 index bda567e26300e28122641c4971f3f94f90d6248d..0000000000000000000000000000000000000000 --- a/test/benchmark/remote/terraform/variables.tf +++ /dev/null @@ -1,4 +0,0 @@ -variable "machine_type" { - type = string - default = "c2-standard-16" -} diff --git a/test/benchmark/remote/terraform/vm.tf b/test/benchmark/remote/terraform/vm.tf deleted file mode 100644 index af8b77258d1fca779187c23ecc366b0c6fb03a2b..0000000000000000000000000000000000000000 --- a/test/benchmark/remote/terraform/vm.tf +++ /dev/null @@ -1,23 +0,0 @@ -resource "google_compute_instance" "default" { - name = "automated-loadtest" - machine_type = var.machine_type - zone = "us-central1-a" - - tags = ["automated-loadtest"] - - boot_disk { - initialize_params { - image = "debian-cloud/debian-11" - size = 100 - type = "pd-ssd" - } - } - - network_interface { - network = "default" - - access_config { - // Ephemeral public IP - } - } -} diff --git a/test/benchmark/run_performance_tracker.sh b/test/benchmark/run_performance_tracker.sh deleted file mode 100644 index 777efae902844dad59ce349b91bc4c0d7960c2b5..0000000000000000000000000000000000000000 --- a/test/benchmark/run_performance_tracker.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -# change to script directory -cd "${0%/*}" || exit - -go run . -name "SIFT" -numberEntries 100000 -fail "-1" -numBatches "1" \ No newline at end of file diff --git a/test/benchmark_bm25/.gitignore b/test/benchmark_bm25/.gitignore deleted file mode 100644 index be8473d91c2fc217efbc8ed24d66a1ca989f8a97..0000000000000000000000000000000000000000 --- a/test/benchmark_bm25/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -datasets/ -datasets.yml \ No newline at end of file diff --git a/test/benchmark_bm25/cmd/import.go b/test/benchmark_bm25/cmd/import.go deleted file mode 100644 index 782b9fe91279867c3e7d39cd037eeb5d49c546ab..0000000000000000000000000000000000000000 --- a/test/benchmark_bm25/cmd/import.go +++ /dev/null @@ -1,120 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package cmd - -import ( - "context" - "fmt" - "log" - - "github.com/go-openapi/strfmt" - "github.com/google/uuid" - "github.com/spf13/cobra" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/test/benchmark_bm25/lib" -) - -func init() { - rootCmd.AddCommand(importCmd) - importCmd.PersistentFlags().IntVarP(&BatchSize, "batch-size", "b", DefaultBatchSize, "number of objects in a single import batch") - importCmd.PersistentFlags().IntVarP(&MultiplyProperties, "multiply-properties", "m", DefaultMultiplyProperties, "create artifical copies of real properties by setting a value larger than 1. The properties have identical contents, so it won't alter results, but leads to many more calculations.") - importCmd.PersistentFlags().BoolVarP(&Vectorizer, "vectorizer", "v", DefaultVectorizer, "Vectorize import data with default vectorizer") -} - -var importCmd = &cobra.Command{ - Use: "import", - Short: "Import a dataset (or multiple datasets) into Weaviate", - RunE: func(cmd *cobra.Command, args []string) error { - client, err := lib.ClientFromOrigin(Origin) - if err != nil { - return err - } - - ok, err := client.Misc().LiveChecker().Do(context.Background()) - if err != nil { - return fmt.Errorf("weaviate is not ready at %v: %w", Origin, err) - } - - if !ok { - return fmt.Errorf("weaviate is not ready") - } - - datasets, err := lib.ParseDatasetConfig(DatasetConfigPath) - if err != nil { - return fmt.Errorf("parse dataset cfg file: %w", err) - } - - c, err := lib.ParseCorpi(datasets.Datasets[0], MultiplyProperties) - if err != nil { - return err - } - - if err := client.Schema().AllDeleter().Do(context.Background()); err != nil { - return fmt.Errorf("clear schema prior to import: %w", err) - } - - if err := client.Schema().ClassCreator(). - WithClass(lib.SchemaFromDataset(datasets.Datasets[0], Vectorizer)). - Do(context.Background()); err != nil { - return fmt.Errorf("create schema for %s: %w", datasets.Datasets[0], err) - } - - batch := client.Batch().ObjectsBatcher() - for i, corp := range c { - id := uuid.MustParse(fmt.Sprintf("%032x", i)).String() - props := map[string]interface{}{ - "modulo_10": i % 10, - "modulo_100": i % 100, - "modulo_1000": i % 1000, - } - - for key, value := range corp { - props[key] = value - } - - batch.WithObjects(&models.Object{ - ID: strfmt.UUID(id), - Class: lib.ClassNameFromDatasetID(datasets.Datasets[0].ID), - Properties: props, - }) - - if i != 0 && i%BatchSize == 0 { - br, err := batch.Do(context.Background()) - if err != nil { - return fmt.Errorf("batch %d: %w", i, err) - } - - if err := lib.HandleBatchResponse(br); err != nil { - return err - } - } - - if i%1000 == 0 { - log.Printf("imported %d/%d objects", i, len(c)) - } - } - - if len(c)&BatchSize != 0 { - // we need to send one final batch - br, err := batch.Do(context.Background()) - if err != nil { - return fmt.Errorf("final batch: %w", err) - } - if err := lib.HandleBatchResponse(br); err != nil { - return err - } - log.Printf("imported %d/%d objects", len(c), len(c)) - } - - return nil - }, -} diff --git a/test/benchmark_bm25/cmd/query.go b/test/benchmark_bm25/cmd/query.go deleted file mode 100644 index 74fe7362c9d77db48b0b1d562b910d7a3ce2ac54..0000000000000000000000000000000000000000 --- a/test/benchmark_bm25/cmd/query.go +++ /dev/null @@ -1,137 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package cmd - -import ( - "context" - "errors" - "fmt" - "log" - "time" - - "github.com/spf13/cobra" - "github.com/weaviate/weaviate-go-client/v4/weaviate/filters" - - "github.com/weaviate/weaviate-go-client/v4/weaviate/graphql" - "github.com/weaviate/weaviate/test/benchmark_bm25/lib" -) - -func init() { - rootCmd.AddCommand(queryCmd) - queryCmd.PersistentFlags().IntVarP(&BatchSize, "batch-size", "b", DefaultBatchSize, "number of objects in a single import batch") - queryCmd.PersistentFlags().IntVarP(&QueriesCount, "count", "c", DefaultQueriesCount, "run only the specified amount of queries, negative numbers mean unlimited") - queryCmd.PersistentFlags().IntVarP(&FilterObjectPercentage, "filter", "f", DefaultFilterObjectPercentage, "The given percentage of objects are filtered out. Off by default, use <=0 to disable") - queryCmd.PersistentFlags().Float32VarP(&Alpha, "alpha", "a", DefaultAlpha, "Weighting for keyword vs vector search. Alpha = 0 (Default) is pure BM25 search.") - queryCmd.PersistentFlags().StringVarP(&Ranking, "ranking´", "r", DefaultRanking, "Which ranking algorithm should be used for hybrid search, rankedFusion (default) and relativeScoreFusion.") -} - -var queryCmd = &cobra.Command{ - Use: "query", - Short: "Send queries for a dataset", - RunE: func(cmd *cobra.Command, args []string) error { - client, err := lib.ClientFromOrigin(Origin) - if err != nil { - return err - } - - ok, err := client.Misc().LiveChecker().Do(context.Background()) - if err != nil { - return fmt.Errorf("weaviate is not ready: %w", err) - } - - if !ok { - return fmt.Errorf("weaviate is not ready") - } - log.Print("weaviate is ready") - - datasets, err := lib.ParseDatasetConfig(DatasetConfigPath) - if err != nil { - return fmt.Errorf("parse dataset cfg file: %w", err) - } - - ds := datasets.Datasets[0] - - log.Print("parse queries") - q, err := lib.ParseQueries(ds, QueriesCount) - if err != nil { - return err - } - log.Print("queries parsed") - - times := []time.Duration{} - - log.Print("start querying") - - scores := lib.Scores{} - propNameWithId := lib.SanitizePropName(ds.Queries.PropertyWithId) - className := lib.ClassNameFromDatasetID(ds.ID) - for i, query := range q { - before := time.Now() - queryBuilder := client.GraphQL().Get().WithClassName(className).WithLimit(100).WithFields(graphql.Field{Name: "_additional { id }"}, graphql.Field{Name: propNameWithId}) - if Alpha == 0 { - bm25 := &graphql.BM25ArgumentBuilder{} - bm25.WithQuery(query.Query) - queryBuilder.WithBM25(bm25) - } else { - hybrid := &graphql.HybridArgumentBuilder{} - ranking := graphql.FusionType(Ranking) - hybrid.WithQuery(query.Query).WithAlpha(Alpha).WithFusionType(ranking) - queryBuilder.WithHybrid(hybrid) - } - if FilterObjectPercentage > 0 { - filter := filters.Where() - filter.WithPath([]string{"modulo_100"}) - filter.WithOperator(filters.GreaterThan) - filter.WithValueInt(int64(FilterObjectPercentage)) - queryBuilder = queryBuilder.WithWhere(filter) - } - result, err := queryBuilder.Do(context.Background()) - if err != nil { - return err - } - - if result.Errors != nil { - return errors.New(result.Errors[0].Message) - } - times = append(times, time.Since(before)) - - logMsg := fmt.Sprintf("completed %d/%d queries.", i, len(q)) - - if len(query.MatchingIds) > 0 && len(ds.Queries.PropertyWithId) > 0 { - resultIds := result.Data["Get"].(map[string]interface{})[className].([]interface{}) - if err := scores.AddResult(query.MatchingIds, resultIds, propNameWithId); err != nil { - return err - } - logMsg += fmt.Sprintf("nDCG score: %.04f", scores.CurrentNDCG()) - } - if i%1000 == 0 && i > 0 { - log.Printf(logMsg) - } - } - - meta, err := client.GraphQL().Aggregate().WithClassName(lib.ClassNameFromDatasetID(ds.ID)). - WithFields(graphql.Field{Name: "meta", Fields: []graphql.Field{{Name: "count"}}}). - Do(context.Background()) - if err != nil { - return err - } - - objCount := int(meta.Data["Aggregate"].(map[string]interface{})[lib.ClassNameFromDatasetID(ds.ID)].([]interface{})[0].(map[string]interface{})["meta"].(map[string]interface{})["count"].(float64)) - - fmt.Printf("\nObjects imported: %d\n", objCount) - stat := lib.AnalyzeLatencies(times) - stat.PrettyPrint() - scores.PrettyPrint() - - return nil - }, -} diff --git a/test/benchmark_bm25/cmd/root.go b/test/benchmark_bm25/cmd/root.go deleted file mode 100644 index 4f68a59babfb4ef419998c0107a01b6ed6f23167..0000000000000000000000000000000000000000 --- a/test/benchmark_bm25/cmd/root.go +++ /dev/null @@ -1,64 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package cmd - -import ( - "fmt" - "os" - - "github.com/spf13/cobra" -) - -var ( - Origin string - DatasetConfigPath string - BatchSize int - QueriesCount int - MultiplyProperties int - FilterObjectPercentage int - Alpha float32 - Ranking string - Vectorizer bool -) - -const ( - DefaultOrigin = "http://localhost:8080" - DefaultDatasetConfigPath = "datasets.yml" - DefaultBatchSize = 100 - DefaultQueriesCount = -1 - DefaultMultiplyProperties = 1 - DefaultFilterObjectPercentage = 0 - DefaultAlpha = 0 - DefaultRanking = "ranked_fusion" - DefaultVectorizer = false -) - -var rootCmd = &cobra.Command{ - Use: "benchmarker", - Short: "benchmarker is a simple tool to obtain bm25 speed benchmarks", - RunE: func(cmd *cobra.Command, args []string) error { - fmt.Printf("Run --help to see usage instructions.\n") - return nil - }, -} - -func Execute() { - if err := rootCmd.Execute(); err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } -} - -func init() { - rootCmd.PersistentFlags().StringVarP(&Origin, "origin", "o", DefaultOrigin, "origin (schema + host + port) where weaviate is running") - rootCmd.PersistentFlags().StringVar(&DatasetConfigPath, "dataset-config", DefaultDatasetConfigPath, "path to dataset config file") -} diff --git a/test/benchmark_bm25/datasets_default.yml b/test/benchmark_bm25/datasets_default.yml deleted file mode 100644 index a38753372be9e5ff7b0f80087ca11d397add82dc..0000000000000000000000000000000000000000 --- a/test/benchmark_bm25/datasets_default.yml +++ /dev/null @@ -1,51 +0,0 @@ -datasets: - - id: msmarco - path: ./datasets/msmarco - corpus: - indexed_properties: - - title - - text - unindexed_properties: - - _id - queries: - property: text - - id: nfcorpus - path: ./datasets/nfcorpus - corpus: - indexed_properties: - - title - - text - unindexed_properties: - - _id - queries: - property: text - - id: fiqa - path: ./datasets/fiqa - corpus: - indexed_properties: - - title - - text - unindexed_properties: - - _id - queries: - property: query - matching_results: original_matchingDocIDs - property_with_id: _id - - id: quora - path: ./datasets/quora - corpus: - indexed_properties: - - text - unindexed_properties: - - _id - queries: - property: text - - id: webistouche - path: ./datasets/webis-touche2020 - corpus: - indexed_properties: - - text - unindexed_properties: - - _id - queries: - property: text diff --git a/test/benchmark_bm25/gather_stats.sh b/test/benchmark_bm25/gather_stats.sh deleted file mode 100644 index 0614b551dbd58670ae9a1611abdcd3f0c98366bc..0000000000000000000000000000000000000000 --- a/test/benchmark_bm25/gather_stats.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/sh - -# Check if a name parameter is provided -if [ -z "$1" ]; then - echo "Usage: $0 [-p | --png]" - exit 1 -fi - -NAME=$1 -SAVE_PNG=false - -# Check for -p or --png flag -if [ "$2" = "-p" ] || [ "$2" = "--png" ]; then - SAVE_PNG=true -fi - -if $SAVE_PNG; then - go tool pprof -png -lines http://localhost:6060/debug/pprof/heap > "${NAME}_heap.png" & - go tool pprof -png http://localhost:6060/debug/pprof/profile\?seconds\=30 > "${NAME}_profile.png" & - go tool pprof -png http://localhost:6060/debug/pprof/allocs > "${NAME}_allocs.png" & - - wait - - open "${NAME}_heap.png" - open "${NAME}_profile.png" - open "${NAME}_allocs.png" -else - curl http://127.0.0.1:6060/debug/pprof/allocs\?seconds=30 > "${NAME}_allocs.prof" - curl http://127.0.0.1:6060/debug/pprof/heap\?seconds=30 > "${NAME}_heap.prof" - curl http://127.0.0.1:6060/debug/pprof/profile\?seconds=30 > "${NAME}_profile.prof" - -# Check if the FlameGraph directory exists -if [ -d "./FlameGraph" ]; then - go tool pprof -raw -output="${NAME}_cpu.txt" 'http://localhost:6060/debug/pprof/profile?seconds=30' - # Checkout the FlameGraph git repository for really nice flame graphs - # git checkout https://github.com/brendangregg/FlameGraph - ./FlameGraph/stackcollapse-go.pl "${NAME}_cpu.txt" | ./FlameGraph/flamegraph.pl > "${NAME}_flame.svg" & - open "${NAME}_flame.svg" -else - echo "FlameGraph directory not found. Please clone the FlameGraph repository at https://github.com/brendangregg/FlameGraph to get nice flamegraphs." -fi - -fi - - diff --git a/test/benchmark_bm25/go.mod b/test/benchmark_bm25/go.mod deleted file mode 100644 index aee313547fce95f1a3e47d73376fb6a155b1d080..0000000000000000000000000000000000000000 --- a/test/benchmark_bm25/go.mod +++ /dev/null @@ -1,42 +0,0 @@ -module github.com/weaviate/weaviate/test/benchmark_bm25 - -go 1.21 - -replace github.com/weaviate/weaviate => ../.. - -require ( - github.com/go-openapi/strfmt v0.21.7 - github.com/google/uuid v1.3.1 - github.com/spf13/cobra v1.7.0 - github.com/weaviate/weaviate v1.23.0 - github.com/weaviate/weaviate-go-client/v4 v4.12.1 - gopkg.in/yaml.v3 v3.0.1 -) - -require ( - github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect - github.com/go-openapi/analysis v0.21.2 // indirect - github.com/go-openapi/errors v0.20.3 // indirect - github.com/go-openapi/jsonpointer v0.19.5 // indirect - github.com/go-openapi/jsonreference v0.20.0 // indirect - github.com/go-openapi/loads v0.21.1 // indirect - github.com/go-openapi/spec v0.20.4 // indirect - github.com/go-openapi/swag v0.22.3 // indirect - github.com/go-openapi/validate v0.21.0 // indirect - github.com/golang/protobuf v1.5.3 // indirect - github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/josharian/intern v1.0.0 // indirect - github.com/mailru/easyjson v0.7.7 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/oklog/ulid v1.3.1 // indirect - github.com/spf13/pflag v1.0.5 // indirect - go.mongodb.org/mongo-driver v1.12.1 // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/oauth2 v0.12.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect - google.golang.org/grpc v1.59.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect -) diff --git a/test/benchmark_bm25/go.sum b/test/benchmark_bm25/go.sum deleted file mode 100644 index ce71ab233fdcf9f417dcf0d8fabd62ba0f271fe5..0000000000000000000000000000000000000000 --- a/test/benchmark_bm25/go.sum +++ /dev/null @@ -1,234 +0,0 @@ -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= -github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= -github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/go-openapi/analysis v0.21.2 h1:hXFrOYFHUAMQdu6zwAiKKJHJQ8kqZs1ux/ru1P1wLJU= -github.com/go-openapi/analysis v0.21.2/go.mod h1:HZwRk4RRisyG8vx2Oe6aqeSQcoxRp47Xkp3+K6q+LdY= -github.com/go-openapi/errors v0.19.8/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= -github.com/go-openapi/errors v0.19.9/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= -github.com/go-openapi/errors v0.20.3 h1:rz6kiC84sqNQoqrtulzaL/VERgkoCyB6WdEkc2ujzUc= -github.com/go-openapi/errors v0.20.3/go.mod h1:Z3FlZ4I8jEGxjUK+bugx3on2mIAk4txuAOhlsB1FSgk= -github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= -github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= -github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA= -github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= -github.com/go-openapi/loads v0.21.1 h1:Wb3nVZpdEzDTcly8S4HMkey6fjARRzb7iEaySimlDW0= -github.com/go-openapi/loads v0.21.1/go.mod h1:/DtAMXXneXFjbQMGEtbamCZb+4x7eGwkvZCvBmwUG+g= -github.com/go-openapi/spec v0.20.4 h1:O8hJrt0UMnhHcluhIdUgCLRWyM2x7QkBXRvOs7m+O1M= -github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I= -github.com/go-openapi/strfmt v0.21.0/go.mod h1:ZRQ409bWMj+SOgXofQAGTIo2Ebu72Gs+WaRADcS5iNg= -github.com/go-openapi/strfmt v0.21.1/go.mod h1:I/XVKeLc5+MM5oPNN7P6urMOpuLXEcNrCX/rPGuWb0k= -github.com/go-openapi/strfmt v0.21.7 h1:rspiXgNWgeUzhjo1YU01do6qsahtJNByjLVbPLNHb8k= -github.com/go-openapi/strfmt v0.21.7/go.mod h1:adeGTkxE44sPyLk0JV235VQAO/ZXUr8KAzYjclFs3ew= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= -github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-openapi/validate v0.21.0 h1:+Wqk39yKOhfpLqNLEC0/eViCkzM5FVXVqrvt526+wcI= -github.com/go-openapi/validate v0.21.0/go.mod h1:rjnrwK57VJ7A8xqfpAOEKRH8yQSGUriMu5/zuPSQ1hg= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= -github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= -github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= -github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= -github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= -github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= -github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= -github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= -github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= -github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= -github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= -github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= -github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= -github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= -github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= -github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= -github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= -github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= -github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= -github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= -github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= -github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= -github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= -github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= -github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= -github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= -github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= -github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= -github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= -github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= -github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/weaviate/weaviate-go-client/v4 v4.12.1 h1:XFKL49BgSOcxrFs5IV+Q5pydLTsh0HQHuWbKNSLMWLU= -github.com/weaviate/weaviate-go-client/v4 v4.12.1/go.mod h1:r1PlU5sAZKFvAPgymEHQj0hjSAuEV9X77PJ/ffZ6cEo= -github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= -github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= -github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= -github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= -github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= -github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg= -go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng= -go.mongodb.org/mongo-driver v1.12.1 h1:nLkghSU8fQNaK7oUmDhQFsnrtcoNy7Z6LVFKsEecqgE= -go.mongodb.org/mongo-driver v1.12.1/go.mod h1:/rGBTebI3XYboVmgz+Wv3Bcbl3aD0QF9zl6kDDw18rQ= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= -golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= -golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= -google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= -google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/test/benchmark_bm25/lib/batch.go b/test/benchmark_bm25/lib/batch.go deleted file mode 100644 index 322100cb09ba5ee5993547495731d7d0d503a608..0000000000000000000000000000000000000000 --- a/test/benchmark_bm25/lib/batch.go +++ /dev/null @@ -1,43 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lib - -import ( - "fmt" - "strings" - - "github.com/weaviate/weaviate/entities/models" -) - -func HandleBatchResponse(res []models.ObjectsGetResponse) error { - msgs := []string{} - - for i, obj := range res { - if obj.Result.Errors == nil { - continue - } - - if len(obj.Result.Errors.Error) == 0 { - continue - } - - msg := fmt.Sprintf("at pos %d: %s", i, obj.Result.Errors.Error[0].Message) - msgs = append(msgs, msg) - } - - if len(msgs) == 0 { - return nil - } - - msg := strings.Join(msgs, ", ") - return fmt.Errorf("%s", msg) -} diff --git a/test/benchmark_bm25/lib/client.go b/test/benchmark_bm25/lib/client.go deleted file mode 100644 index 26e269faac7c7070c0e28856b15e6e2ef15f52e8..0000000000000000000000000000000000000000 --- a/test/benchmark_bm25/lib/client.go +++ /dev/null @@ -1,34 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lib - -import ( - "net/url" - - client "github.com/weaviate/weaviate-go-client/v4/weaviate" -) - -func ClientFromOrigin(origin string) (*client.Client, error) { - parsed, err := url.Parse(origin) - if err != nil { - return nil, err - } - - config := client.Config{ - Scheme: parsed.Scheme, - Host: parsed.Host, - } - - client := client.New(config) - - return client, nil -} diff --git a/test/benchmark_bm25/lib/dataset_corpus.go b/test/benchmark_bm25/lib/dataset_corpus.go deleted file mode 100644 index 64b29faa973ea504fb9349a933a52f71a6f72479..0000000000000000000000000000000000000000 --- a/test/benchmark_bm25/lib/dataset_corpus.go +++ /dev/null @@ -1,87 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lib - -import ( - "bufio" - "encoding/json" - "fmt" - "math/rand" - "os" - "path/filepath" -) - -type Corpi []Corpus - -type Corpus map[string]string - -// TODO: just loading the whole corpus into memory isn't very efficient, this -// could be improved by iterating one object at a time, e.g. with a callback -func ParseCorpi(ds Dataset, multiply int) (Corpi, error) { - p := filepath.Join(ds.Path, "corpus.jsonl") - f, err := os.Open(p) - if err != nil { - return nil, fmt.Errorf("open queries file at %s: %w", p, err) - } - - defer f.Close() - - scanner := bufio.NewScanner(f) - - // we need reproducible random numbers to be able to compare different runs. This only needs to be stable for a - // given version of the code. If it produces different numbers after a dependency update it doesn't matter. - r := rand.New(rand.NewSource(9)) - - c := Corpi{} - for scanner.Scan() { - obj := map[string]interface{}{} - if err := json.Unmarshal(scanner.Bytes(), &obj); err != nil { - return nil, err - } - - corp := Corpus{} - for _, prop := range ds.Corpus.IndexedProperties { - propStr, ok := obj[prop].(string) - if !ok { - return nil, fmt.Errorf("indexed property %s is not a string: %T", - prop, obj[prop]) - } - - corp[SanitizePropName(prop)] = propStr - for i := 1; i < multiply; i++ { - newName := fmt.Sprintf("%s_copy_%d", SanitizePropName(prop), i) - var propString string - if len(c) > 1 { // get the content of the property from another object, to get more varied results - otherObjectIndex := r.Intn(len(corp)) - propString = c[otherObjectIndex][SanitizePropName(prop)] - } else { - propString = propStr - } - corp[newName] = propString - } - } - - for _, prop := range ds.Corpus.UnindexedProperties { - propStr, ok := obj[prop].(string) - if !ok { - return nil, fmt.Errorf("unindexed property %s is not a string: %T", - prop, obj[prop]) - } - - corp[SanitizePropName(prop)] = propStr - } - - c = append(c, corp) - } - - return c, nil -} diff --git a/test/benchmark_bm25/lib/dataset_queries.go b/test/benchmark_bm25/lib/dataset_queries.go deleted file mode 100644 index 736d59ee0cb8cb430bc76e5b798f18d02c62394a..0000000000000000000000000000000000000000 --- a/test/benchmark_bm25/lib/dataset_queries.go +++ /dev/null @@ -1,92 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lib - -import ( - "bufio" - "encoding/json" - "fmt" - "os" - "path/filepath" - "strconv" -) - -type Query struct { - Query string - MatchingIds []int -} - -type Queries []Query - -func ParseQueries(ds Dataset, limit int) (Queries, error) { - p := filepath.Join(ds.Path, "queries.jsonl") - f, err := os.Open(p) - if err != nil { - return nil, fmt.Errorf("open queries file at %s: %w", p, err) - } - - defer f.Close() - - scanner := bufio.NewScanner(f) - - q := Queries{} - obj := map[string]interface{}{} - for scanner.Scan() { - if limit > 0 && len(q) >= limit { - break - } - - if err := json.Unmarshal(scanner.Bytes(), &obj); err != nil { - return nil, err - } - - queryStr, ok := obj[ds.Queries.Property].(string) - if !ok { - return nil, fmt.Errorf("property %s is not a string: %T", - ds.Queries.Property, obj[ds.Queries.Property]) - } - - var matchingIds []int - if ds.Queries.MatchingResults != "" { - matchingIdsInterface, ok := obj[ds.Queries.MatchingResults].([]interface{}) - if !ok { - return nil, fmt.Errorf("property %s is not a []interface{}]: %T", - ds.Queries.MatchingResults, obj[ds.Queries.MatchingResults]) - } - matchingIds = make([]int, len(matchingIdsInterface)) - for i, val := range matchingIdsInterface { - // docIds can be provided as strings or float arrays - valStr, ok := val.(string) - if ok { - valInt, err := strconv.Atoi(valStr) - if err != nil { - return nil, err - } - matchingIds[i] = valInt - - } else { - valFloat, ok := val.(float64) - if !ok { - return nil, fmt.Errorf("matching Id %v is not a float: %v", i, matchingIdsInterface) - } - matchingIds[i] = int(valFloat) - } - - } - - } - - q = append(q, Query{Query: queryStr, MatchingIds: matchingIds}) - } - - return q, nil -} diff --git a/test/benchmark_bm25/lib/datasets.go b/test/benchmark_bm25/lib/datasets.go deleted file mode 100644 index 3b588236498325307cb2fa82ec592bdde378a86c..0000000000000000000000000000000000000000 --- a/test/benchmark_bm25/lib/datasets.go +++ /dev/null @@ -1,55 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lib - -import ( - "os" - - "gopkg.in/yaml.v3" -) - -type DatasetCfg struct { - Datasets []Dataset `yaml:"datasets"` -} - -type Dataset struct { - ID string `yaml:"id"` - Path string `yaml:"path"` - Corpus DatasetCorpus `yaml:"corpus"` - Queries DatasetQueries `yaml:"queries"` -} - -type DatasetCorpus struct { - IndexedProperties []string `yaml:"indexed_properties"` - UnindexedProperties []string `yaml:"unindexed_properties"` -} - -type DatasetQueries struct { - Property string `yaml:"property"` - MatchingResults string `yaml:"matching_results"` - PropertyWithId string `yaml:"property_with_id"` -} - -func ParseDatasetConfig(filename string) (DatasetCfg, error) { - var config DatasetCfg - yamlFile, err := os.ReadFile(filename) - if err != nil { - return config, err - } - - err = yaml.Unmarshal(yamlFile, &config) - if err != nil { - return config, err - } - - return config, nil -} diff --git a/test/benchmark_bm25/lib/durations.go b/test/benchmark_bm25/lib/durations.go deleted file mode 100644 index f900537db135b3aec5d331416ba06d5e3d3e522f..0000000000000000000000000000000000000000 --- a/test/benchmark_bm25/lib/durations.go +++ /dev/null @@ -1,60 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lib - -import ( - "fmt" - "sort" - "time" -) - -type LatencyStats struct { - Min time.Duration - Max time.Duration - Mean time.Duration - P50 time.Duration - P90 time.Duration - P99 time.Duration -} - -func AnalyzeLatencies(in []time.Duration) LatencyStats { - out := LatencyStats{} - sort.Slice(in, func(a, b int) bool { return in[a] < in[b] }) - - out.Min = in[0] - out.Max = in[len(in)-1] - - sum := time.Duration(0) - for _, dur := range in { - sum += dur - } - out.Mean = sum / time.Duration(len(in)) - - out.P50 = in[len(in)/2] - out.P90 = in[len(in)*9/10] - out.P99 = in[len(in)*99/100] - - return out -} - -func (l LatencyStats) PrettyPrint() { - fmt.Printf("\n") - fmt.Printf("Query Latencies \n") - fmt.Printf("-------------------- \n") - fmt.Printf("Min: %12s\n", l.Min) - fmt.Printf("Mean: %12s\n", l.Mean) - fmt.Printf("Max: %12s\n", l.Max) - fmt.Printf("p50: %12s\n", l.P50) - fmt.Printf("p90: %12s\n", l.P90) - fmt.Printf("p99: %12s\n", l.P99) - fmt.Printf("\n") -} diff --git a/test/benchmark_bm25/lib/schema.go b/test/benchmark_bm25/lib/schema.go deleted file mode 100644 index 6ef2659694db5fbd273c1c1a8d6049f400588146..0000000000000000000000000000000000000000 --- a/test/benchmark_bm25/lib/schema.go +++ /dev/null @@ -1,100 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lib - -import ( - "strings" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" -) - -func SchemaFromDataset(ds Dataset, includeVectorizer bool) *models.Class { - out := &models.Class{} - out.Class = ClassNameFromDatasetID(ds.ID) - if !includeVectorizer { - out.VectorIndexConfig = map[string]interface{}{ - "skip": true, - } - out.Vectorizer = "none" - } - out.InvertedIndexConfig = &models.InvertedIndexConfig{ - Stopwords: &models.StopwordConfig{ - Preset: "none", - }, - } - - for _, prop := range ds.Corpus.IndexedProperties { - t := true - // all indexed props are indexed as text - prop := &models.Property{ - Name: SanitizePropName(prop), - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWord, - IndexFilterable: &t, - IndexSearchable: &t, - } - - out.Properties = append(out.Properties, prop) - - } - - for _, prop := range ds.Corpus.UnindexedProperties { - // all indexed props are indexed as text - f := false - prop := &models.Property{ - Name: SanitizePropName(prop), - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationField, - IndexFilterable: &f, - IndexSearchable: &f, - } - - out.Properties = append(out.Properties, prop) - } - - filterProps := []*models.Property{ - { - Name: "modulo_10", - DataType: []string{"int"}, - }, - { - Name: "modulo_100", - DataType: []string{"int"}, - }, - { - Name: "modulo_1000", - DataType: []string{"int"}, - }, - } - - out.Properties = append(out.Properties, filterProps...) - - return out -} - -func ClassNameFromDatasetID(in string) string { - if len(in) == 0 { - panic("zero length dataset name") - } - - return strings.ToUpper(string(in[0])) + strings.ToLower(in[1:]) -} - -func SanitizePropName(in string) string { - if len(in) >= 2 && in[0] == '_' && in[1] != '_' { - // single leading underscore is reserved, but we can append another one - return "_" + in - } - - return in -} diff --git a/test/benchmark_bm25/lib/scores.go b/test/benchmark_bm25/lib/scores.go deleted file mode 100644 index c17173d45774de07da4181593a66c7b125c933ac..0000000000000000000000000000000000000000 --- a/test/benchmark_bm25/lib/scores.go +++ /dev/null @@ -1,63 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package lib - -import ( - "fmt" - "math" - "strconv" -) - -type Scores struct { - NDCG float64 - hitsAt1 int - hitsAt5 int - numQueries int -} - -func (n *Scores) AddResult(matchingIds []int, resultIds []interface{}, propNameWithId string) error { - IDCG := 0. - for j := 0; j < len(matchingIds); j++ { - IDCG += 1. / math.Log(float64(j+2.)) - } - - DCG := 0. - for rank, resultId := range resultIds { - id, err := strconv.Atoi(resultId.(map[string]interface{})[propNameWithId].(string)) - for _, matchingId := range matchingIds { - if err != nil { - return err - } - if id == matchingId { - if rank == 0 { - n.hitsAt1 += 1 - } - if rank < 5 { - n.hitsAt5 += 1 - } - DCG += 1 / math.Log(float64(rank+2)) - - } - } - } - n.NDCG += DCG / IDCG - n.numQueries += 1 - return nil -} - -func (n *Scores) CurrentNDCG() float64 { - return n.NDCG / float64(n.numQueries) -} - -func (n *Scores) PrettyPrint() { - fmt.Printf("nDCG score: %.04f, hits at 1: %d, hits at 5: %d\n", n.NDCG/float64(n.numQueries), n.hitsAt1, n.hitsAt5) -} diff --git a/test/benchmark_bm25/main.go b/test/benchmark_bm25/main.go deleted file mode 100644 index 37f942ca43bc2c50d9acbf3b150ebf6200136054..0000000000000000000000000000000000000000 --- a/test/benchmark_bm25/main.go +++ /dev/null @@ -1,18 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package main - -import "github.com/weaviate/weaviate/test/benchmark_bm25/cmd" - -func main() { - cmd.Execute() -} diff --git a/test/docker/azurite.go b/test/docker/azurite.go deleted file mode 100644 index 838e96d7b92c0c5633bfca7b14f2f63d87336cbc..0000000000000000000000000000000000000000 --- a/test/docker/azurite.go +++ /dev/null @@ -1,65 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package docker - -import ( - "context" - "fmt" - "time" - - "github.com/docker/go-connections/nat" - "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/wait" -) - -const Azurite = "azurite" - -func startAzurite(ctx context.Context, networkName string) (*DockerContainer, error) { - blobPort := nat.Port("10000/tcp") - container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ - ContainerRequest: testcontainers.ContainerRequest{ - Image: "mcr.microsoft.com/azure-storage/azurite", - ExposedPorts: []string{"10000/tcp", "10001/tcp", "10002/tcp"}, - Hostname: Azurite, - AutoRemove: true, - Networks: []string{networkName}, - NetworkAliases: map[string][]string{ - networkName: {Azurite}, - }, - Cmd: []string{Azurite, "--blobHost", "0.0.0.0", "--queueHost", "0.0.0.0", "--tableHost", "0.0.0.0"}, - WaitingFor: wait. - ForAll( - wait.ForLog("Azurite Blob service is successfully listening at http://0.0.0.0:10000"), - wait.ForLog("Azurite Queue service is successfully listening at http://0.0.0.0:10001"), - wait.ForLog("Azurite Table service is successfully listening at http://0.0.0.0:10002"), - wait.ForListeningPort(blobPort), - wait.ForListeningPort("10001/tcp"), - wait.ForListeningPort("10002/tcp"), - ).WithDeadline(60 * time.Second), - }, - Started: true, - }) - if err != nil { - return nil, err - } - uri, err := container.PortEndpoint(ctx, blobPort, "") - if err != nil { - return nil, err - } - envSettings := make(map[string]string) - connectionString := "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://%s/devstoreaccount1;" - blobEndpoint := fmt.Sprintf("%s:%s", Azurite, blobPort.Port()) - envSettings["AZURE_STORAGE_CONNECTION_STRING"] = fmt.Sprintf(connectionString, blobEndpoint) - endpoints := make(map[EndpointName]endpoint) - endpoints[HTTP] = endpoint{blobPort, uri} - return &DockerContainer{Azurite, endpoints, container, envSettings}, nil -} diff --git a/test/docker/clip.go b/test/docker/clip.go deleted file mode 100644 index 6b35342da694c44f6940c2e04751de47896b6e4a..0000000000000000000000000000000000000000 --- a/test/docker/clip.go +++ /dev/null @@ -1,64 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package docker - -import ( - "context" - "fmt" - "time" - - "github.com/docker/go-connections/nat" - "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/wait" -) - -const Multi2VecCLIP = "multi2vec-clip" - -func startM2VClip(ctx context.Context, networkName, clipImage string) (*DockerContainer, error) { - image := "semitechnologies/multi2vec-clip:sentence-transformers-clip-ViT-B-32-multilingual-v1" - if len(clipImage) > 0 { - image = clipImage - } - port := nat.Port("8080/tcp") - container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ - ContainerRequest: testcontainers.ContainerRequest{ - Image: image, - Hostname: Multi2VecCLIP, - Networks: []string{networkName}, - NetworkAliases: map[string][]string{ - networkName: {Multi2VecCLIP}, - }, - ExposedPorts: []string{"8080/tcp"}, - AutoRemove: true, - WaitingFor: wait. - ForHTTP("/.well-known/ready"). - WithPort(port). - WithStatusCodeMatcher(func(status int) bool { - return status == 204 - }). - WithStartupTimeout(240 * time.Second), - }, - Started: true, - }) - if err != nil { - return nil, err - } - uri, err := container.PortEndpoint(ctx, port, "") - if err != nil { - return nil, err - } - envSettings := make(map[string]string) - envSettings["CLIP_INFERENCE_API"] = fmt.Sprintf("http://%s:%s", Multi2VecCLIP, port.Port()) - endpoints := make(map[EndpointName]endpoint) - endpoints[HTTP] = endpoint{port, uri} - return &DockerContainer{Multi2VecCLIP, endpoints, container, envSettings}, nil -} diff --git a/test/docker/compose.go b/test/docker/compose.go deleted file mode 100644 index 895b5a0d5f28a05a52ac36b84bba8ac8939f5d4c..0000000000000000000000000000000000000000 --- a/test/docker/compose.go +++ /dev/null @@ -1,507 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package docker - -import ( - "context" - "fmt" - "os" - - "github.com/pkg/errors" - "github.com/testcontainers/testcontainers-go" - modstgazure "github.com/weaviate/weaviate/modules/backup-azure" - modstgfilesystem "github.com/weaviate/weaviate/modules/backup-filesystem" - modstggcs "github.com/weaviate/weaviate/modules/backup-gcs" - modstgs3 "github.com/weaviate/weaviate/modules/backup-s3" - modgenerativeanyscale "github.com/weaviate/weaviate/modules/generative-anyscale" - modgenerativeaws "github.com/weaviate/weaviate/modules/generative-aws" - modgenerativecohere "github.com/weaviate/weaviate/modules/generative-cohere" - modgenerativeopenai "github.com/weaviate/weaviate/modules/generative-openai" - modgenerativepalm "github.com/weaviate/weaviate/modules/generative-palm" - modqnaopenai "github.com/weaviate/weaviate/modules/qna-openai" - modrerankercohere "github.com/weaviate/weaviate/modules/reranker-cohere" - modaws "github.com/weaviate/weaviate/modules/text2vec-aws" - modcohere "github.com/weaviate/weaviate/modules/text2vec-cohere" - modhuggingface "github.com/weaviate/weaviate/modules/text2vec-huggingface" - modopenai "github.com/weaviate/weaviate/modules/text2vec-openai" - modpalm "github.com/weaviate/weaviate/modules/text2vec-palm" -) - -const ( - // envTestWeaviateImage can be passed to tests to spin up docker compose with given image - envTestWeaviateImage = "TEST_WEAVIATE_IMAGE" - // envTestText2vecTransformersImage adds ability to pass a custom image to module tests - envTestText2vecTransformersImage = "TEST_TEXT2VEC_TRANSFORMERS_IMAGE" - // envTestText2vecContextionaryImage adds ability to pass a custom image to module tests - envTestText2vecContextionaryImage = "TEST_TEXT2VEC_CONTEXTIONARY_IMAGE" - // envTestQnATransformersImage adds ability to pass a custom image to module tests - envTestQnATransformersImage = "TEST_QNA_TRANSFORMERS_IMAGE" - // envTestSUMTransformersImage adds ability to pass a custom image to module tests - envTestSUMTransformersImage = "TEST_SUM_TRANSFORMERS_IMAGE" - // envTestMulti2VecCLIPImage adds ability to pass a custom CLIP image to module tests - envTestMulti2VecCLIPImage = "TEST_MULTI2VEC_CLIP_IMAGE" - // envTestImg2VecNeuralImage adds ability to pass a custom Im2Vec Neural image to module tests - envTestImg2VecNeuralImage = "TEST_IMG2VEC_NEURAL_IMAGE" - // envTestRerankerTransformersImage adds ability to pass a custom image to module tests - envTestRerankerTransformersImage = "TEST_RERANKER_TRANSFORMERS_IMAGE" -) - -const ( - Ref2VecCentroid = "ref2vec-centroid" -) - -type Compose struct { - enableModules []string - defaultVectorizerModule string - withMinIO bool - withGCS bool - withAzurite bool - withBackendFilesystem bool - withBackendS3 bool - withBackendS3Bucket string - withBackendGCS bool - withBackendGCSBucket string - withBackendAzure bool - withBackendAzureContainer string - withTransformers bool - withContextionary bool - withQnATransformers bool - withWeaviate bool - withWeaviateExposeGRPCPort bool - withSecondWeaviate bool - withWeaviateAuth bool - withWeaviateBasicAuth bool - withWeaviateBasicAuthUsername string - withWeaviateBasicAuthPassword string - withWeaviateCluster bool - withSUMTransformers bool - withCentroid bool - withCLIP bool - withImg2Vec bool - withRerankerTransformers bool - weaviateEnvs map[string]string -} - -func New() *Compose { - return &Compose{enableModules: []string{}, weaviateEnvs: make(map[string]string)} -} - -func (d *Compose) WithMinIO() *Compose { - d.withMinIO = true - d.enableModules = append(d.enableModules, modstgs3.Name) - return d -} - -func (d *Compose) WithGCS() *Compose { - d.withGCS = true - d.enableModules = append(d.enableModules, modstggcs.Name) - return d -} - -func (d *Compose) WithAzurite() *Compose { - d.withAzurite = true - d.enableModules = append(d.enableModules, modstgazure.Name) - return d -} - -func (d *Compose) WithText2VecTransformers() *Compose { - d.withTransformers = true - d.enableModules = append(d.enableModules, Text2VecTransformers) - d.defaultVectorizerModule = Text2VecTransformers - return d -} - -func (d *Compose) WithText2VecContextionary() *Compose { - d.withContextionary = true - d.enableModules = append(d.enableModules, Text2VecContextionary) - d.defaultVectorizerModule = Text2VecContextionary - return d -} - -func (d *Compose) WithQnATransformers() *Compose { - d.withQnATransformers = true - d.enableModules = append(d.enableModules, QnATransformers) - return d -} - -func (d *Compose) WithBackendFilesystem() *Compose { - d.withBackendFilesystem = true - d.enableModules = append(d.enableModules, modstgfilesystem.Name) - return d -} - -func (d *Compose) WithBackendS3(bucket string) *Compose { - d.withBackendS3 = true - d.withBackendS3Bucket = bucket - d.withMinIO = true - d.enableModules = append(d.enableModules, modstgs3.Name) - return d -} - -func (d *Compose) WithBackendGCS(bucket string) *Compose { - d.withBackendGCS = true - d.withBackendGCSBucket = bucket - d.withGCS = true - d.enableModules = append(d.enableModules, modstggcs.Name) - return d -} - -func (d *Compose) WithBackendAzure(container string) *Compose { - d.withBackendAzure = true - d.withBackendAzureContainer = container - d.withAzurite = true - d.enableModules = append(d.enableModules, modstgazure.Name) - return d -} - -func (d *Compose) WithSUMTransformers() *Compose { - d.withSUMTransformers = true - d.enableModules = append(d.enableModules, SUMTransformers) - return d -} - -func (d *Compose) WithMulti2VecCLIP() *Compose { - d.withCLIP = true - d.enableModules = append(d.enableModules, Multi2VecCLIP) - return d -} - -func (d *Compose) WithImg2VecNeural() *Compose { - d.withImg2Vec = true - d.enableModules = append(d.enableModules, Img2VecNeural) - return d -} - -func (d *Compose) WithRef2VecCentroid() *Compose { - d.withCentroid = true - d.enableModules = append(d.enableModules, Ref2VecCentroid) - return d -} - -func (d *Compose) WithText2VecOpenAI() *Compose { - d.enableModules = append(d.enableModules, modopenai.Name) - return d -} - -func (d *Compose) WithText2VecCohere() *Compose { - d.enableModules = append(d.enableModules, modcohere.Name) - return d -} - -func (d *Compose) WithText2VecPaLM() *Compose { - d.enableModules = append(d.enableModules, modpalm.Name) - return d -} - -func (d *Compose) WithText2VecAWS() *Compose { - d.enableModules = append(d.enableModules, modaws.Name) - return d -} - -func (d *Compose) WithText2VecHuggingFace() *Compose { - d.enableModules = append(d.enableModules, modhuggingface.Name) - return d -} - -func (d *Compose) WithGenerativeOpenAI() *Compose { - d.enableModules = append(d.enableModules, modgenerativeopenai.Name) - return d -} - -func (d *Compose) WithGenerativeAWS() *Compose { - d.enableModules = append(d.enableModules, modgenerativeaws.Name) - return d -} - -func (d *Compose) WithGenerativeCohere() *Compose { - d.enableModules = append(d.enableModules, modgenerativecohere.Name) - return d -} - -func (d *Compose) WithGenerativePaLM() *Compose { - d.enableModules = append(d.enableModules, modgenerativepalm.Name) - return d -} - -func (d *Compose) WithGenerativeAnyscale() *Compose { - d.enableModules = append(d.enableModules, modgenerativeanyscale.Name) - return d -} - -func (d *Compose) WithQnAOpenAI() *Compose { - d.enableModules = append(d.enableModules, modqnaopenai.Name) - return d -} - -func (d *Compose) WithRerankerCohere() *Compose { - d.enableModules = append(d.enableModules, modrerankercohere.Name) - return d -} - -func (d *Compose) WithRerankerTransformers() *Compose { - d.withRerankerTransformers = true - d.enableModules = append(d.enableModules, RerankerTransformers) - return d -} - -func (d *Compose) WithWeaviate() *Compose { - d.withWeaviate = true - return d -} - -func (d *Compose) WithWeaviateWithGRPC() *Compose { - d.withWeaviate = true - d.withWeaviateExposeGRPCPort = true - return d -} - -func (d *Compose) WithSecondWeaviate() *Compose { - d.withSecondWeaviate = true - return d -} - -func (d *Compose) WithWeaviateCluster() *Compose { - d.withWeaviate = true - d.withWeaviateCluster = true - return d -} - -func (d *Compose) WithWeaviateClusterWithGRPC() *Compose { - d.withWeaviate = true - d.withWeaviateCluster = true - d.withWeaviateExposeGRPCPort = true - return d -} - -func (d *Compose) WithWeaviateClusterWithBasicAuth(username, password string) *Compose { - d.withWeaviate = true - d.withWeaviateCluster = true - d.withWeaviateBasicAuth = true - d.withWeaviateBasicAuthUsername = username - d.withWeaviateBasicAuthPassword = password - return d -} - -func (d *Compose) WithWeaviateAuth() *Compose { - d.withWeaviate = true - d.withWeaviateAuth = true - return d -} - -func (d *Compose) WithWeaviateEnv(name, value string) *Compose { - d.weaviateEnvs[name] = value - return d -} - -func (d *Compose) Start(ctx context.Context) (*DockerCompose, error) { - networkName := "weaviate-module-acceptance-tests" - network, err := testcontainers.GenericNetwork(ctx, testcontainers.GenericNetworkRequest{ - NetworkRequest: testcontainers.NetworkRequest{ - Name: networkName, - Internal: false, - }, - }) - if err != nil { - return nil, errors.Wrapf(err, "network: %s", networkName) - } - envSettings := make(map[string]string) - containers := []*DockerContainer{} - if d.withMinIO { - container, err := startMinIO(ctx, networkName) - if err != nil { - return nil, errors.Wrapf(err, "start %s", MinIO) - } - containers = append(containers, container) - if d.withBackendS3 { - for k, v := range container.envSettings { - envSettings[k] = v - } - envSettings["BACKUP_S3_BUCKET"] = d.withBackendS3Bucket - } - } - if d.withGCS { - container, err := startGCS(ctx, networkName) - if err != nil { - return nil, errors.Wrapf(err, "start %s", GCS) - } - containers = append(containers, container) - if d.withBackendGCS { - for k, v := range container.envSettings { - envSettings[k] = v - } - envSettings["BACKUP_GCS_BUCKET"] = d.withBackendGCSBucket - } - } - if d.withAzurite { - container, err := startAzurite(ctx, networkName) - if err != nil { - return nil, errors.Wrapf(err, "start %s", Azurite) - } - containers = append(containers, container) - if d.withBackendAzure { - for k, v := range container.envSettings { - envSettings[k] = v - } - envSettings["BACKUP_AZURE_CONTAINER"] = d.withBackendAzureContainer - } - } - if d.withBackendFilesystem { - envSettings["BACKUP_FILESYSTEM_PATH"] = "/tmp/backups" - } - if d.withTransformers { - image := os.Getenv(envTestText2vecTransformersImage) - container, err := startT2VTransformers(ctx, networkName, image) - if err != nil { - return nil, errors.Wrapf(err, "start %s", Text2VecTransformers) - } - for k, v := range container.envSettings { - envSettings[k] = v - } - containers = append(containers, container) - } - if d.withContextionary { - image := os.Getenv(envTestText2vecContextionaryImage) - container, err := startT2VContextionary(ctx, networkName, image) - if err != nil { - return nil, errors.Wrapf(err, "start %s", Text2VecContextionary) - } - for k, v := range container.envSettings { - envSettings[k] = v - } - containers = append(containers, container) - } - if d.withQnATransformers { - image := os.Getenv(envTestQnATransformersImage) - container, err := startQnATransformers(ctx, networkName, image) - if err != nil { - return nil, errors.Wrapf(err, "start %s", QnATransformers) - } - for k, v := range container.envSettings { - envSettings[k] = v - } - containers = append(containers, container) - } - if d.withSUMTransformers { - image := os.Getenv(envTestSUMTransformersImage) - container, err := startSUMTransformers(ctx, networkName, image) - if err != nil { - return nil, errors.Wrapf(err, "start %s", SUMTransformers) - } - for k, v := range container.envSettings { - envSettings[k] = v - } - containers = append(containers, container) - } - if d.withCLIP { - image := os.Getenv(envTestMulti2VecCLIPImage) - container, err := startM2VClip(ctx, networkName, image) - if err != nil { - return nil, errors.Wrapf(err, "start %s", Multi2VecCLIP) - } - for k, v := range container.envSettings { - envSettings[k] = v - } - containers = append(containers, container) - } - if d.withImg2Vec { - image := os.Getenv(envTestImg2VecNeuralImage) - container, err := startI2VNeural(ctx, networkName, image) - if err != nil { - return nil, errors.Wrapf(err, "start %s", Img2VecNeural) - } - for k, v := range container.envSettings { - envSettings[k] = v - } - containers = append(containers, container) - } - if d.withRerankerTransformers { - image := os.Getenv(envTestRerankerTransformersImage) - container, err := startRerankerTransformers(ctx, networkName, image) - if err != nil { - return nil, errors.Wrapf(err, "start %s", RerankerTransformers) - } - for k, v := range container.envSettings { - envSettings[k] = v - } - containers = append(containers, container) - } - if d.withWeaviate { - image := os.Getenv(envTestWeaviateImage) - hostname := Weaviate - if d.withWeaviateCluster { - envSettings["CLUSTER_HOSTNAME"] = "node1" - envSettings["CLUSTER_GOSSIP_BIND_PORT"] = "7100" - envSettings["CLUSTER_DATA_BIND_PORT"] = "7101" - } - if d.withWeaviateBasicAuth { - envSettings["CLUSTER_BASIC_AUTH_USERNAME"] = d.withWeaviateBasicAuthUsername - envSettings["CLUSTER_BASIC_AUTH_PASSWORD"] = d.withWeaviateBasicAuthPassword - } - if d.withWeaviateAuth { - envSettings["AUTHENTICATION_OIDC_ENABLED"] = "true" - envSettings["AUTHENTICATION_OIDC_CLIENT_ID"] = "wcs" - envSettings["AUTHENTICATION_OIDC_ISSUER"] = "https://auth.wcs.api.semi.technology/auth/realms/SeMI" - envSettings["AUTHENTICATION_OIDC_USERNAME_CLAIM"] = "email" - envSettings["AUTHENTICATION_OIDC_GROUPS_CLAIM"] = "groups" - envSettings["AUTHORIZATION_ADMINLIST_ENABLED"] = "true" - envSettings["AUTHORIZATION_ADMINLIST_USERS"] = "ms_2d0e007e7136de11d5f29fce7a53dae219a51458@existiert.net" - } - for k, v := range d.weaviateEnvs { - envSettings[k] = v - } - container, err := startWeaviate(ctx, d.enableModules, d.defaultVectorizerModule, - envSettings, networkName, image, hostname, d.withWeaviateExposeGRPCPort) - if err != nil { - return nil, errors.Wrapf(err, "start %s", hostname) - } - containers = append(containers, container) - } - if d.withWeaviateCluster { - image := os.Getenv(envTestWeaviateImage) - hostname := WeaviateNode2 - envSettings["CLUSTER_HOSTNAME"] = "node2" - envSettings["CLUSTER_GOSSIP_BIND_PORT"] = "7102" - envSettings["CLUSTER_DATA_BIND_PORT"] = "7103" - envSettings["CLUSTER_JOIN"] = fmt.Sprintf("%s:7100", Weaviate) - for k, v := range d.weaviateEnvs { - envSettings[k] = v - } - container, err := startWeaviate(ctx, d.enableModules, d.defaultVectorizerModule, - envSettings, networkName, image, hostname, d.withWeaviateExposeGRPCPort) - if err != nil { - return nil, errors.Wrapf(err, "start %s", hostname) - } - containers = append(containers, container) - } - - if d.withSecondWeaviate { - image := os.Getenv(envTestWeaviateImage) - hostname := SecondWeaviate - secondWeaviateSettings := envSettings - // Ensure second weaviate doesn't get cluster settings from the first cluster if any. - delete(secondWeaviateSettings, "CLUSTER_HOSTNAME") - delete(secondWeaviateSettings, "CLUSTER_GOSSIP_BIND_PORT") - delete(secondWeaviateSettings, "CLUSTER_DATA_BIND_PORT") - delete(secondWeaviateSettings, "CLUSTER_JOIN") - for k, v := range d.weaviateEnvs { - envSettings[k] = v - } - container, err := startWeaviate(ctx, d.enableModules, d.defaultVectorizerModule, - envSettings, networkName, image, hostname, d.withWeaviateExposeGRPCPort) - if err != nil { - return nil, errors.Wrapf(err, "start %s", hostname) - } - containers = append(containers, container) - } - - return &DockerCompose{network, containers}, nil -} diff --git a/test/docker/container.go b/test/docker/container.go deleted file mode 100644 index ee915cb752ee1aa509d35b9801d52f9e84baea35..0000000000000000000000000000000000000000 --- a/test/docker/container.go +++ /dev/null @@ -1,51 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package docker - -import ( - "github.com/docker/go-connections/nat" - "github.com/testcontainers/testcontainers-go" -) - -type EndpointName string - -var ( - HTTP EndpointName = "http" - GRPC EndpointName = "grpc" -) - -type endpoint struct { - port nat.Port - uri string -} - -type DockerContainer struct { - name string - endpoints map[EndpointName]endpoint - container testcontainers.Container - envSettings map[string]string -} - -func (d *DockerContainer) Name() string { - return d.name -} - -func (d *DockerContainer) URI() string { - return d.GetEndpoint(HTTP) -} - -func (d *DockerContainer) GetEndpoint(name EndpointName) string { - if endpoint, ok := d.endpoints[name]; ok { - return endpoint.uri - } - return "" -} diff --git a/test/docker/contextionary.go b/test/docker/contextionary.go deleted file mode 100644 index 3f5f5bb4a56ad6d651ec9d5a40edb5f11e95e631..0000000000000000000000000000000000000000 --- a/test/docker/contextionary.go +++ /dev/null @@ -1,62 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package docker - -import ( - "context" - "fmt" - - "github.com/docker/go-connections/nat" - "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/wait" -) - -const Text2VecContextionary = "text2vec-contextionary" - -func startT2VContextionary(ctx context.Context, networkName, contextionaryImage string) (*DockerContainer, error) { - image := "semitechnologies/contextionary:en0.16.0-v1.2.1" - if len(contextionaryImage) > 0 { - image = contextionaryImage - } - port := nat.Port("9999/tcp") - container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ - ContainerRequest: testcontainers.ContainerRequest{ - Image: image, - Hostname: Text2VecContextionary, - Networks: []string{networkName}, - NetworkAliases: map[string][]string{ - networkName: {Text2VecContextionary}, - }, - Env: map[string]string{ - "OCCURRENCE_WEIGHT_LINEAR_FACTOR": "0.75", - "EXTENSIONS_STORAGE_MODE": "weaviate", - "EXTENSIONS_STORAGE_ORIGIN": fmt.Sprintf("http://%s:8080", Weaviate), - }, - ExposedPorts: []string{"9999/tcp"}, - AutoRemove: true, - WaitingFor: wait.ForListeningPort(port), - }, - Started: true, - }) - if err != nil { - return nil, err - } - uri, err := container.PortEndpoint(ctx, port, "") - if err != nil { - return nil, err - } - envSettings := make(map[string]string) - envSettings["CONTEXTIONARY_URL"] = fmt.Sprintf("%s:%s", Text2VecContextionary, port.Port()) - endpoints := make(map[EndpointName]endpoint) - endpoints[HTTP] = endpoint{port, uri} - return &DockerContainer{Text2VecContextionary, endpoints, container, envSettings}, nil -} diff --git a/test/docker/docker.go b/test/docker/docker.go deleted file mode 100644 index fa63109063f96ab7a18bf5cb5afe5b820e0ee6d1..0000000000000000000000000000000000000000 --- a/test/docker/docker.go +++ /dev/null @@ -1,138 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package docker - -import ( - "context" - "fmt" - "time" - - "github.com/pkg/errors" - "github.com/testcontainers/testcontainers-go" -) - -type DockerCompose struct { - network testcontainers.Network - containers []*DockerContainer -} - -func (d *DockerCompose) Containers() []*DockerContainer { - return d.containers -} - -func (d *DockerCompose) Terminate(ctx context.Context) error { - var errs error - for _, c := range d.containers { - if err := c.container.Terminate(ctx); err != nil { - errs = errors.Wrapf(err, "cannot terminate: %v", c.name) - } - } - if d.network != nil { - if err := d.network.Remove(ctx); err != nil { - errs = errors.Wrapf(err, "cannot remove network") - } - } - return errs -} - -func (d *DockerCompose) Stop(ctx context.Context, container string, timeout *time.Duration) error { - for _, c := range d.containers { - if c.name == container { - if err := c.container.Stop(ctx, timeout); err != nil { - return fmt.Errorf("cannot stop %q: %w", c.name, err) - } - } - } - return nil -} - -func (d *DockerCompose) Start(ctx context.Context, container string) error { - for _, c := range d.containers { - if c.name == container { - if err := c.container.Start(ctx); err != nil { - return fmt.Errorf("cannot start %q: %w", c.name, err) - } - if err := d.waitUntilRunning(c.name, c.container); err != nil { - return err - } - newEndpoints := map[EndpointName]endpoint{} - for name, e := range c.endpoints { - newURI, err := c.container.PortEndpoint(ctx, e.port, "") - if err != nil { - return fmt.Errorf("failed to get new uri for container %q: %w", c.name, err) - } - newEndpoints[name] = endpoint{e.port, newURI} - } - c.endpoints = newEndpoints - } - } - return nil -} - -func (d *DockerCompose) waitUntilRunning(name string, container testcontainers.Container) error { - waitTimeout := 1 * time.Minute - start := time.Now() - for { - if container.IsRunning() { - return nil - } - time.Sleep(200 * time.Millisecond) - if time.Now().After(start.Add(waitTimeout)) { - return fmt.Errorf("container %q: was still not running after %v", name, waitTimeout) - } - } -} - -func (d *DockerCompose) GetMinIO() *DockerContainer { - return d.getContainerByName(MinIO) -} - -func (d *DockerCompose) GetGCS() *DockerContainer { - return d.getContainerByName(GCS) -} - -func (d *DockerCompose) GetAzurite() *DockerContainer { - return d.getContainerByName(Azurite) -} - -func (d *DockerCompose) GetWeaviate() *DockerContainer { - return d.getContainerByName(Weaviate) -} - -func (d *DockerCompose) GetSecondWeaviate() *DockerContainer { - return d.getContainerByName(SecondWeaviate) -} - -func (d *DockerCompose) GetWeaviateNode2() *DockerContainer { - return d.getContainerByName(WeaviateNode2) -} - -func (d *DockerCompose) GetText2VecTransformers() *DockerContainer { - return d.getContainerByName(Text2VecTransformers) -} - -func (d *DockerCompose) GetText2VecContextionary() *DockerContainer { - return d.getContainerByName(Text2VecContextionary) -} - -func (d *DockerCompose) GetQnATransformers() *DockerContainer { - return d.getContainerByName(QnATransformers) -} - -func (d *DockerCompose) getContainerByName(name string) *DockerContainer { - for _, c := range d.containers { - if c.name == name { - return c - } - } - return nil -} diff --git a/test/docker/gcs.go b/test/docker/gcs.go deleted file mode 100644 index 14c559f010c7981fbaf33a04e9dd9a24b7ad7833..0000000000000000000000000000000000000000 --- a/test/docker/gcs.go +++ /dev/null @@ -1,65 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package docker - -import ( - "context" - "fmt" - "os" - "time" - - "github.com/docker/go-connections/nat" - "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/wait" -) - -const GCS = "gcp-storage-emulator" - -func startGCS(ctx context.Context, networkName string) (*DockerContainer, error) { - port := nat.Port("9090/tcp") - container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ - ContainerRequest: testcontainers.ContainerRequest{ - Image: "oittaa/gcp-storage-emulator", - ExposedPorts: []string{"9090/tcp"}, - Name: GCS, - Hostname: GCS, - AutoRemove: true, - Networks: []string{networkName}, - NetworkAliases: map[string][]string{ - networkName: {GCS}, - }, - Env: map[string]string{ - "PORT": port.Port(), - }, - WaitingFor: wait.ForAll( - wait.ForListeningPort(port), - wait.ForHTTP("/").WithPort(port), - ).WithStartupTimeoutDefault(60 * time.Second), - }, - Started: true, - }) - if err != nil { - return nil, err - } - uri, err := container.PortEndpoint(ctx, port, "") - if err != nil { - return nil, err - } - envSettings := make(map[string]string) - projectID := os.Getenv("GOOGLE_CLOUD_PROJECT") - envSettings["GOOGLE_CLOUD_PROJECT"] = projectID - envSettings["STORAGE_EMULATOR_HOST"] = fmt.Sprintf("%s:%s", GCS, port.Port()) - envSettings["BACKUP_GCS_USE_AUTH"] = "false" - endpoints := make(map[EndpointName]endpoint) - endpoints[HTTP] = endpoint{port, uri} - return &DockerContainer{GCS, endpoints, container, envSettings}, nil -} diff --git a/test/docker/image.go b/test/docker/image.go deleted file mode 100644 index f07e14ea908c1bb8b752a74debd0f3eb07df532e..0000000000000000000000000000000000000000 --- a/test/docker/image.go +++ /dev/null @@ -1,64 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package docker - -import ( - "context" - "fmt" - "time" - - "github.com/docker/go-connections/nat" - "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/wait" -) - -const Img2VecNeural = "img2vec-neural" - -func startI2VNeural(ctx context.Context, networkName, img2vecImage string) (*DockerContainer, error) { - image := "semitechnologies/img2vec-pytorch:resnet50" - if len(img2vecImage) > 0 { - image = img2vecImage - } - port := nat.Port("8080/tcp") - container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ - ContainerRequest: testcontainers.ContainerRequest{ - Image: image, - Hostname: Img2VecNeural, - Networks: []string{networkName}, - NetworkAliases: map[string][]string{ - networkName: {Img2VecNeural}, - }, - ExposedPorts: []string{"8080/tcp"}, - AutoRemove: true, - WaitingFor: wait. - ForHTTP("/.well-known/ready"). - WithPort(port). - WithStatusCodeMatcher(func(status int) bool { - return status == 204 - }). - WithStartupTimeout(240 * time.Second), - }, - Started: true, - }) - if err != nil { - return nil, err - } - uri, err := container.PortEndpoint(ctx, port, "") - if err != nil { - return nil, err - } - envSettings := make(map[string]string) - envSettings["IMAGE_INFERENCE_API"] = fmt.Sprintf("http://%s:%s", Img2VecNeural, port.Port()) - endpoints := make(map[EndpointName]endpoint) - endpoints[HTTP] = endpoint{port, uri} - return &DockerContainer{Img2VecNeural, endpoints, container, envSettings}, nil -} diff --git a/test/docker/minio.go b/test/docker/minio.go deleted file mode 100644 index 5be406816eacee9d6f11e26dfbd487634772b730..0000000000000000000000000000000000000000 --- a/test/docker/minio.go +++ /dev/null @@ -1,66 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package docker - -import ( - "context" - "fmt" - "time" - - "github.com/docker/go-connections/nat" - "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/wait" -) - -const MinIO = "test-minio" - -func startMinIO(ctx context.Context, networkName string) (*DockerContainer, error) { - port := nat.Port("9000/tcp") - container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ - ContainerRequest: testcontainers.ContainerRequest{ - Image: "minio/minio", - ExposedPorts: []string{"9000/tcp"}, - Name: MinIO, - Hostname: MinIO, - AutoRemove: true, - Networks: []string{networkName}, - NetworkAliases: map[string][]string{ - networkName: {MinIO}, - }, - Env: map[string]string{ - "MINIO_ROOT_USER": "aws_access_key", - "MINIO_ROOT_PASSWORD": "aws_secret_key", - }, - Cmd: []string{"server", "/data"}, - WaitingFor: wait.ForAll( - wait.ForListeningPort(port), - wait.ForHTTP("/minio/health/ready").WithPort(port), - ).WithDeadline(60 * time.Second), - }, - Started: true, - }) - if err != nil { - return nil, err - } - uri, err := container.PortEndpoint(ctx, port, "") - if err != nil { - return nil, err - } - envSettings := make(map[string]string) - envSettings["BACKUP_S3_ENDPOINT"] = fmt.Sprintf("%s:%s", MinIO, port.Port()) - envSettings["BACKUP_S3_USE_SSL"] = "false" - envSettings["AWS_ACCESS_KEY_ID"] = "aws_access_key" - envSettings["AWS_SECRET_KEY"] = "aws_secret_key" - endpoints := make(map[EndpointName]endpoint) - endpoints[HTTP] = endpoint{port, uri} - return &DockerContainer{MinIO, endpoints, container, envSettings}, nil -} diff --git a/test/docker/qna.go b/test/docker/qna.go deleted file mode 100644 index 70c51f13c506d81d831c17889fb7d60da585a536..0000000000000000000000000000000000000000 --- a/test/docker/qna.go +++ /dev/null @@ -1,64 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package docker - -import ( - "context" - "fmt" - "time" - - "github.com/docker/go-connections/nat" - "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/wait" -) - -const QnATransformers = "qna-transformers" - -func startQnATransformers(ctx context.Context, networkName, qnaImage string) (*DockerContainer, error) { - image := "semitechnologies/qna-transformers:distilbert-base-uncased-distilled-squad" - if len(qnaImage) > 0 { - image = qnaImage - } - port := nat.Port("8080/tcp") - container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ - ContainerRequest: testcontainers.ContainerRequest{ - Image: image, - Hostname: QnATransformers, - Networks: []string{networkName}, - NetworkAliases: map[string][]string{ - networkName: {QnATransformers}, - }, - ExposedPorts: []string{"8080/tcp"}, - AutoRemove: true, - WaitingFor: wait. - ForHTTP("/.well-known/ready"). - WithPort(port). - WithStatusCodeMatcher(func(status int) bool { - return status == 204 - }). - WithStartupTimeout(240 * time.Second), - }, - Started: true, - }) - if err != nil { - return nil, err - } - uri, err := container.PortEndpoint(ctx, port, "") - if err != nil { - return nil, err - } - envSettings := make(map[string]string) - envSettings["QNA_INFERENCE_API"] = fmt.Sprintf("http://%s:%s", QnATransformers, port.Port()) - endpoints := make(map[EndpointName]endpoint) - endpoints[HTTP] = endpoint{port, uri} - return &DockerContainer{QnATransformers, endpoints, container, envSettings}, nil -} diff --git a/test/docker/reranker.go b/test/docker/reranker.go deleted file mode 100644 index d474b7ac251afeedd36db184d26316cee2cc683e..0000000000000000000000000000000000000000 --- a/test/docker/reranker.go +++ /dev/null @@ -1,64 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package docker - -import ( - "context" - "fmt" - "time" - - "github.com/docker/go-connections/nat" - "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/wait" -) - -const RerankerTransformers = "reranker-transformers" - -func startRerankerTransformers(ctx context.Context, networkName, rerankerTransformersImage string) (*DockerContainer, error) { - image := "semitechnologies/reranker-transformers:cross-encoder-ms-marco-MiniLM-L-6-v2" - if len(rerankerTransformersImage) > 0 { - image = rerankerTransformersImage - } - port := nat.Port("8080/tcp") - container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ - ContainerRequest: testcontainers.ContainerRequest{ - Image: image, - Hostname: RerankerTransformers, - Networks: []string{networkName}, - NetworkAliases: map[string][]string{ - networkName: {RerankerTransformers}, - }, - ExposedPorts: []string{"8080/tcp"}, - AutoRemove: true, - WaitingFor: wait. - ForHTTP("/.well-known/ready"). - WithPort(port). - WithStatusCodeMatcher(func(status int) bool { - return status == 204 - }). - WithStartupTimeout(240 * time.Second), - }, - Started: true, - }) - if err != nil { - return nil, err - } - uri, err := container.PortEndpoint(ctx, port, "") - if err != nil { - return nil, err - } - envSettings := make(map[string]string) - envSettings["RERANKER_INFERENCE_API"] = fmt.Sprintf("http://%s:%s", RerankerTransformers, port.Port()) - endpoints := make(map[EndpointName]endpoint) - endpoints[HTTP] = endpoint{port, uri} - return &DockerContainer{RerankerTransformers, endpoints, container, envSettings}, nil -} diff --git a/test/docker/sum.go b/test/docker/sum.go deleted file mode 100644 index d50441d42e993f10b48909084396c42051728e15..0000000000000000000000000000000000000000 --- a/test/docker/sum.go +++ /dev/null @@ -1,64 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package docker - -import ( - "context" - "fmt" - "time" - - "github.com/docker/go-connections/nat" - "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/wait" -) - -const SUMTransformers = "sum-transformers" - -func startSUMTransformers(ctx context.Context, networkName, sumImage string) (*DockerContainer, error) { - image := "semitechnologies/sum-transformers:facebook-bart-large-cnn" - if len(sumImage) > 0 { - image = sumImage - } - port := nat.Port("8080/tcp") - container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ - ContainerRequest: testcontainers.ContainerRequest{ - Image: image, - Hostname: SUMTransformers, - Networks: []string{networkName}, - NetworkAliases: map[string][]string{ - networkName: {SUMTransformers}, - }, - ExposedPorts: []string{"8080/tcp"}, - AutoRemove: true, - WaitingFor: wait. - ForHTTP("/.well-known/ready"). - WithPort(port). - WithStatusCodeMatcher(func(status int) bool { - return status == 204 - }). - WithStartupTimeout(240 * time.Second), - }, - Started: true, - }) - if err != nil { - return nil, err - } - uri, err := container.PortEndpoint(ctx, port, "") - if err != nil { - return nil, err - } - envSettings := make(map[string]string) - envSettings["SUM_INFERENCE_API"] = fmt.Sprintf("http://%s:%s", SUMTransformers, port.Port()) - endpoints := make(map[EndpointName]endpoint) - endpoints[HTTP] = endpoint{port, uri} - return &DockerContainer{SUMTransformers, endpoints, container, envSettings}, nil -} diff --git a/test/docker/transformers.go b/test/docker/transformers.go deleted file mode 100644 index 8054c3d51c5da319fd104756214e824cb803ed94..0000000000000000000000000000000000000000 --- a/test/docker/transformers.go +++ /dev/null @@ -1,64 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package docker - -import ( - "context" - "fmt" - "time" - - "github.com/docker/go-connections/nat" - "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/wait" -) - -const Text2VecTransformers = "text2vec-transformers" - -func startT2VTransformers(ctx context.Context, networkName, transformersImage string) (*DockerContainer, error) { - image := "semitechnologies/transformers-inference:baai-bge-small-en-v1.5-onnx" - if len(transformersImage) > 0 { - image = transformersImage - } - port := nat.Port("8080/tcp") - container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ - ContainerRequest: testcontainers.ContainerRequest{ - Image: image, - Hostname: Text2VecTransformers, - Networks: []string{networkName}, - NetworkAliases: map[string][]string{ - networkName: {Text2VecTransformers}, - }, - ExposedPorts: []string{"8080/tcp"}, - AutoRemove: true, - WaitingFor: wait. - ForHTTP("/.well-known/ready"). - WithPort(port). - WithStatusCodeMatcher(func(status int) bool { - return status == 204 - }). - WithStartupTimeout(240 * time.Second), - }, - Started: true, - }) - if err != nil { - return nil, err - } - uri, err := container.PortEndpoint(ctx, port, "") - if err != nil { - return nil, err - } - envSettings := make(map[string]string) - envSettings["TRANSFORMERS_INFERENCE_API"] = fmt.Sprintf("http://%s:%s", Text2VecTransformers, port.Port()) - endpoints := make(map[EndpointName]endpoint) - endpoints[HTTP] = endpoint{port, uri} - return &DockerContainer{Text2VecTransformers, endpoints, container, envSettings}, nil -} diff --git a/test/docker/weaviate.go b/test/docker/weaviate.go deleted file mode 100644 index 1cc755cf44142872a4d7aae77e14c674cfc324dd..0000000000000000000000000000000000000000 --- a/test/docker/weaviate.go +++ /dev/null @@ -1,136 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package docker - -import ( - "context" - "os" - "os/exec" - "runtime" - "strings" - "time" - - "github.com/docker/go-connections/nat" - "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/wait" -) - -const ( - Weaviate = "weaviate" - WeaviateNode2 = "weaviate2" - - SecondWeaviate = "second-weaviate" -) - -func startWeaviate(ctx context.Context, - enableModules []string, defaultVectorizerModule string, - extraEnvSettings map[string]string, networkName string, - weaviateImage, hostname string, exposeGRPCPort bool, -) (*DockerContainer, error) { - fromDockerFile := testcontainers.FromDockerfile{} - if len(weaviateImage) == 0 { - path, err := os.Getwd() - if err != nil { - return nil, err - } - getContextPath := func(path string) string { - if strings.Contains(path, "test/acceptance_with_go_client") { - return path[:strings.Index(path, "/test/acceptance_with_go_client")] - } - if strings.Contains(path, "test/acceptance") { - return path[:strings.Index(path, "/test/acceptance")] - } - return path[:strings.Index(path, "/test/modules")] - } - targetArch := runtime.GOARCH - gitHashBytes, err := exec.Command("git", "rev-parse", "--short", "HEAD").CombinedOutput() - if err != nil { - return nil, err - } - gitHash := strings.ReplaceAll(string(gitHashBytes), "\n", "") - contextPath := getContextPath(path) - fromDockerFile = testcontainers.FromDockerfile{ - Context: contextPath, - Dockerfile: "Dockerfile", - BuildArgs: map[string]*string{ - "TARGETARCH": &targetArch, - "GITHASH": &gitHash, - }, - PrintBuildLog: true, - KeepImage: false, - } - } - containerName := Weaviate - if hostname != "" { - containerName = hostname - } - env := map[string]string{ - "AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED": "true", - "LOG_LEVEL": "debug", - "QUERY_DEFAULTS_LIMIT": "20", - "PERSISTENCE_DATA_PATH": "./data", - "DEFAULT_VECTORIZER_MODULE": "none", - } - if len(enableModules) > 0 { - env["ENABLE_MODULES"] = strings.Join(enableModules, ",") - } - if len(defaultVectorizerModule) > 0 { - env["DEFAULT_VECTORIZER_MODULE"] = defaultVectorizerModule - } - for key, value := range extraEnvSettings { - env[key] = value - } - httpPort := nat.Port("8080/tcp") - exposedPorts := []string{"8080/tcp"} - waitStrategies := []wait.Strategy{ - wait.ForListeningPort(httpPort), - wait.ForHTTP("/v1/.well-known/ready").WithPort(httpPort), - } - grpcPort := nat.Port("50051/tcp") - if exposeGRPCPort { - exposedPorts = append(exposedPorts, "50051/tcp") - waitStrategies = append(waitStrategies, wait.ForListeningPort(grpcPort)) - } - req := testcontainers.ContainerRequest{ - FromDockerfile: fromDockerFile, - Image: weaviateImage, - Hostname: containerName, - Networks: []string{networkName}, - NetworkAliases: map[string][]string{ - networkName: {containerName}, - }, - ExposedPorts: exposedPorts, - Env: env, - WaitingFor: wait.ForAll(waitStrategies...).WithStartupTimeoutDefault(120 * time.Second), - } - c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ - ContainerRequest: req, - Started: true, - }) - if err != nil { - return nil, err - } - httpUri, err := c.PortEndpoint(ctx, httpPort, "") - if err != nil { - return nil, err - } - endpoints := make(map[EndpointName]endpoint) - endpoints[HTTP] = endpoint{httpPort, httpUri} - if exposeGRPCPort { - grpcUri, err := c.PortEndpoint(ctx, grpcPort, "") - if err != nil { - return nil, err - } - endpoints[GRPC] = endpoint{grpcPort, grpcUri} - } - return &DockerContainer{containerName, endpoints, c, nil}, nil -} diff --git a/test/helper/assertions.go b/test/helper/assertions.go deleted file mode 100644 index 565aa2448d7f7aaf255892e19d1e7b3657bf7d83..0000000000000000000000000000000000000000 --- a/test/helper/assertions.go +++ /dev/null @@ -1,55 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package helper - -import ( - "encoding/json" - "reflect" - "testing" -) - -// Asserts that the request did not return an error. -// Optionally perform some checks only if the request did not fail -func AssertRequestOk(t *testing.T, response interface{}, err error, checkFn func()) { - t.Helper() - if err != nil { - responseJson, _ := json.MarshalIndent(response, "", " ") - errorPayload, _ := json.MarshalIndent(err, "", " ") - t.Fatalf("Failed to perform request! Error: %s %s (Original error %s). Response: %s", getType(err), errorPayload, err, responseJson) - } else { - if checkFn != nil { - checkFn() - } - } -} - -// Asserts that the request _did_ return an error. -// Optionally perform some checks only if the request failed -func AssertRequestFail(t *testing.T, response interface{}, err error, checkFn func()) { - if err == nil { - responseJson, _ := json.MarshalIndent(response, "", " ") - t.Fatalf("Request succeeded unexpectedly. Response:\n%s", responseJson) - } else { - if checkFn != nil { - checkFn() - } - } -} - -// Get type name of some value, according to https://stackoverflow.com/questions/35790935/using-reflection-in-go-to-get-the-name-of-a-struct -func getType(myvar interface{}) string { - if t := reflect.TypeOf(myvar); t.Kind() == reflect.Ptr { - return "*" + t.Elem().Name() - } else { - return t.Name() - } -} diff --git a/test/helper/backups.go b/test/helper/backups.go deleted file mode 100644 index 74453bcc33e5e41981d3224e2cce6c392e1453b3..0000000000000000000000000000000000000000 --- a/test/helper/backups.go +++ /dev/null @@ -1,71 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package helper - -import ( - "testing" - - "github.com/weaviate/weaviate/client/backups" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/usecases/backup" -) - -func DefaultBackupConfig() *models.BackupConfig { - return &models.BackupConfig{ - CompressionLevel: models.BackupConfigCompressionLevelDefaultCompression, - CPUPercentage: backup.DefaultCPUPercentage, - ChunkSize: 128, - } -} - -func DefaultRestoreConfig() *models.RestoreConfig { - return &models.RestoreConfig{ - CPUPercentage: backup.DefaultCPUPercentage, - } -} - -func CreateBackup(t *testing.T, cfg *models.BackupConfig, className, backend, backupID string) (*backups.BackupsCreateOK, error) { - params := backups.NewBackupsCreateParams(). - WithBackend(backend). - WithBody(&models.BackupCreateRequest{ - ID: backupID, - Include: []string{className}, - Config: cfg, - }) - return Client(t).Backups.BackupsCreate(params, nil) -} - -func CreateBackupStatus(t *testing.T, backend, backupID string) (*backups.BackupsCreateStatusOK, error) { - params := backups.NewBackupsCreateStatusParams(). - WithBackend(backend). - WithID(backupID) - return Client(t).Backups.BackupsCreateStatus(params, nil) -} - -func RestoreBackup(t *testing.T, cfg *models.RestoreConfig, className, backend, backupID string, nodeMapping map[string]string) (*backups.BackupsRestoreOK, error) { - params := backups.NewBackupsRestoreParams(). - WithBackend(backend). - WithID(backupID). - WithBody(&models.BackupRestoreRequest{ - Include: []string{className}, - NodeMapping: nodeMapping, - Config: cfg, - }) - return Client(t).Backups.BackupsRestore(params, nil) -} - -func RestoreBackupStatus(t *testing.T, backend, backupID string) (*backups.BackupsRestoreStatusOK, error) { - params := backups.NewBackupsRestoreStatusParams(). - WithBackend(backend). - WithID(backupID) - return Client(t).Backups.BackupsRestoreStatus(params, nil) -} diff --git a/test/helper/client.go b/test/helper/client.go deleted file mode 100644 index 3f26e0175b882340eb58281e7a9922d156234c3e..0000000000000000000000000000000000000000 --- a/test/helper/client.go +++ /dev/null @@ -1,71 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package helper - -// This file contains the Client(t *testing.T) function, that can be used to construct a client that talks to -// the Weaviate server that is configured using command line arguments (see init.go). -// -// We pass in the test (*testing.T), to be able to log HTTP traffic to that specific test case. -// This allows us to get detailed logs of the performed HTTP requests if a acceptance test fails. - -// The CreateAuth returns a function that attaches the key and token headers to each HTTP call. - -// Example: -// func TestSomething(t *testing.T) { -// // Use specific key & token -// auth := helper.CreateAuth(key, token) -// helper.Client(t).SomeScope.SomeOperation(&someParams, auth) -// -// // Use root key & token -// helper.Client(t).SomeScope.SomeOperation(&someParams, helper.RootAuth) -// } - -import ( - "fmt" - "testing" - - "github.com/go-openapi/runtime" - httptransport "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" - apiclient "github.com/weaviate/weaviate/client" -) - -// Create a client that logs with t.Logf, if a *testing.T is provided. -// If there is no test case at hand, pass in nil to disable logging. -func Client(t *testing.T) *apiclient.Weaviate { - transport := httptransport.New(fmt.Sprintf("%s:%s", ServerHost, ServerPort), "/v1", []string{ServerScheme}) - - // If a test case is provided, and we want to dump HTTP traffic, - // create a simple logger that logs HTTP traffic to the test case. - if t != nil && DebugHTTP { - transport.SetDebug(true) - transport.SetLogger(&testLogger{t: t}) - } - - client := apiclient.New(transport, strfmt.Default) - return client -} - -// Create a Weaviate client for the given API key & token. -func CreateAuth(apiKey strfmt.UUID, apiToken string) runtime.ClientAuthInfoWriterFunc { - // Create an auth writer that both sets the api key & token. - authWriter := func(r runtime.ClientRequest, _ strfmt.Registry) error { - err := r.SetHeaderParam("X-API-KEY", string(apiKey)) - if err != nil { - return err - } - - return r.SetHeaderParam("X-API-TOKEN", apiToken) - } - - return authWriter -} diff --git a/test/helper/compression.go b/test/helper/compression.go deleted file mode 100644 index e2975a54fa3b001fef1f22fc99b521816d4213ff..0000000000000000000000000000000000000000 --- a/test/helper/compression.go +++ /dev/null @@ -1,27 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package helper - -import ( - "testing" - "time" -) - -func EnablePQ(t *testing.T, className string, pq map[string]interface{}) { - class := GetClass(t, className) - cfg := class.VectorIndexConfig.(map[string]interface{}) - cfg["pq"] = pq - class.VectorIndexConfig = cfg - UpdateClass(t, class) - // Time for compression to complete - time.Sleep(2 * time.Second) -} diff --git a/test/helper/distance.go b/test/helper/distance.go deleted file mode 100644 index 6d440d0081344e189ee5f4bc231ce592ae8375b4..0000000000000000000000000000000000000000 --- a/test/helper/distance.go +++ /dev/null @@ -1,28 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package helper - -import ( - "testing" - - "github.com/weaviate/weaviate/entities/additional" -) - -func CertaintyToDist(t *testing.T, in float32) float32 { - asFloat64 := float64(in) - dist := additional.CertaintyToDistPtr(&asFloat64) - if dist == nil { - t.Fatalf( - "somehow %+v of type %T failed to produce a non-null *float64", in, in) - } - return float32(*dist) -} diff --git a/test/helper/eventually_equal.go b/test/helper/eventually_equal.go deleted file mode 100644 index 95257df62795af2f2ad5849c6362109cbcbe6266..0000000000000000000000000000000000000000 --- a/test/helper/eventually_equal.go +++ /dev/null @@ -1,80 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package helper - -import ( - "fmt" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -type fakeT struct { - lastError error -} - -func (f *fakeT) Reset() { - f.lastError = nil -} - -func (f *fakeT) Errorf(msg string, args ...interface{}) { - f.lastError = fmt.Errorf(msg, args...) -} - -// AssertEventuallyEqual retries the 'actual' thunk every 10ms for a total of -// 300ms. If a single one succeeds, it returns, if all fails it eventually -// fails -func AssertEventuallyEqual(t *testing.T, expected interface{}, actualThunk func() interface{}, msg ...interface{}) { - t.Helper() - interval := 10 * time.Millisecond - timeout := 4000 * time.Millisecond - elapsed := 0 * time.Millisecond - fakeT := &fakeT{} - - for elapsed < timeout { - fakeT.Reset() - actual := actualThunk() - assert.Equal(fakeT, expected, actual, msg...) - - if fakeT.lastError == nil { - return - } - - time.Sleep(interval) - elapsed += interval - } - - t.Errorf("waiting for %s, but never succeeded:\n\n%s", elapsed, fakeT.lastError) -} - -func AssertEventuallyEqualWithFrequencyAndTimeout(t *testing.T, expected interface{}, actualThunk func() interface{}, - interval time.Duration, timeout time.Duration, msg ...interface{}, -) { - elapsed := 0 * time.Millisecond - fakeT := &fakeT{} - - for elapsed < timeout { - fakeT.Reset() - actual := actualThunk() - assert.Equal(fakeT, expected, actual, msg...) - - if fakeT.lastError == nil { - return - } - - time.Sleep(interval) - elapsed += interval - } - - t.Errorf("waiting for %s, but never succeeded:\n\n%s", elapsed, fakeT.lastError) -} diff --git a/test/helper/graphql/graphql_helper.go b/test/helper/graphql/graphql_helper.go deleted file mode 100644 index 9550f38d89f4973a35c28bed38ded1ea9c4f0c80..0000000000000000000000000000000000000000 --- a/test/helper/graphql/graphql_helper.go +++ /dev/null @@ -1,126 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package graphqlhelper - -import ( - "encoding/json" - "fmt" - "strings" - "testing" - - "github.com/go-openapi/runtime" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/client/graphql" - graphql_client "github.com/weaviate/weaviate/client/graphql" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/test/helper" -) - -type GraphQLResult struct { - Result interface{} -} - -// Perform a GraphQL request -func QueryGraphQL(t *testing.T, auth runtime.ClientAuthInfoWriterFunc, operation string, query string, variables map[string]interface{}) (*models.GraphQLResponse, error) { - var vars interface{} = variables - params := graphql_client.NewGraphqlPostParams().WithBody(&models.GraphQLQuery{OperationName: operation, Query: query, Variables: vars}) - response, err := helper.Client(t).Graphql.GraphqlPost(params, nil) - if err != nil { - return nil, err - } - - return response.Payload, nil -} - -// Perform a GraphQL request and call fatal on failure -func QueryGraphQLOrFatal(t *testing.T, auth runtime.ClientAuthInfoWriterFunc, operation string, query string, variables map[string]interface{}) *models.GraphQLResponse { - response, err := QueryGraphQL(t, auth, operation, query, variables) - if err != nil { - parsedErr, ok := err.(*graphql.GraphqlPostUnprocessableEntity) - if !ok { - t.Fatalf("Expected the query to succeed, but failed due to: %#v", err) - } - t.Fatalf("Expected the query to succeed, but failed with unprocessable entity: %v", parsedErr.Payload.Error[0]) - } - return response -} - -// Perform a query and assert that it is successful -func AssertGraphQL(t *testing.T, auth runtime.ClientAuthInfoWriterFunc, query string) *GraphQLResult { - response := QueryGraphQLOrFatal(t, auth, "", query, nil) - - if len(response.Errors) != 0 { - j, _ := json.Marshal(response.Errors) - t.Fatal("GraphQL resolved to an error:", string(j)) - } - - data := make(map[string]interface{}) - - // get rid of models.JSONData - for key, value := range response.Data { - data[key] = value - } - - return &GraphQLResult{Result: data} -} - -// Perform a query and assert that it has errors -func ErrorGraphQL(t *testing.T, auth runtime.ClientAuthInfoWriterFunc, query string) []*models.GraphQLError { - response := QueryGraphQLOrFatal(t, auth, "", query, nil) - - if len(response.Errors) == 0 { - j, _ := json.Marshal(response.Errors) - t.Fatal("GraphQL resolved to data:", string(j)) - } - - return response.Errors -} - -// Drill down in the result -func (g GraphQLResult) Get(paths ...string) *GraphQLResult { - current := g.Result - for _, path := range paths { - var ok bool - currentAsMap := (current.(map[string]interface{})) - current, ok = currentAsMap[path] - if !ok { - panic(fmt.Sprintf("Cannot get element %s in %#v; result: %#v", path, paths, g.Result)) - } - } - - return &GraphQLResult{ - Result: current, - } -} - -// Cast the result to a slice -func (g *GraphQLResult) AsSlice() []interface{} { - return g.Result.([]interface{}) -} - -func Vec2String(v []float32) (s string) { - for _, n := range v { - s = fmt.Sprintf("%s, %f", s, n) - } - s = strings.TrimLeft(s, ", ") - return fmt.Sprintf("[%s]", s) -} - -func ParseVec(t *testing.T, iVec []interface{}) []float32 { - vec := make([]float32, len(iVec)) - for i, val := range iVec { - parsed, err := val.(json.Number).Float64() - require.Nil(t, err) - vec[i] = float32(parsed) - } - return vec -} diff --git a/test/helper/grpc_client.go b/test/helper/grpc_client.go deleted file mode 100644 index 9d8cdff73b8fc567b4b1f81cbe301f4426c70e39..0000000000000000000000000000000000000000 --- a/test/helper/grpc_client.go +++ /dev/null @@ -1,45 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package helper - -import ( - "crypto/tls" - "fmt" - "strings" - - pb "github.com/weaviate/weaviate/grpc/generated/protocol/v1" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials" - "google.golang.org/grpc/credentials/insecure" -) - -func CreateGrpcConnectionClient(host string) (*grpc.ClientConn, error) { - var opts []grpc.DialOption - opts = append(opts, grpc.WithBlock()) - if strings.HasSuffix(host, ":443") { - tlsConfig := &tls.Config{ - InsecureSkipVerify: true, - } - opts = append(opts, grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig))) - } else { - opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) - } - conn, err := grpc.Dial(host, opts...) - if err != nil { - return nil, fmt.Errorf("failed to dial: %w", err) - } - return conn, nil -} - -func CreateGrpcWeaviateClient(conn *grpc.ClientConn) pb.WeaviateClient { - return pb.NewWeaviateClient(conn) -} diff --git a/test/helper/init.go b/test/helper/init.go deleted file mode 100644 index f37b17b7de453f86c80c11c18a5e74115e09945e..0000000000000000000000000000000000000000 --- a/test/helper/init.go +++ /dev/null @@ -1,52 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package helper - -// This file contains the init() function for the helper package. -// In go, each package can have an init() function that runs whenever a package is "imported" in a program, before -// the main function runs. -// -// In our case, we use it to parse additional flags that are used to configure the helper to point to the right -// Weaviate instance, with the correct key and token. - -import ( - "fmt" - - "github.com/go-openapi/runtime" -) - -// Configuration flags provided by the user that runs an acceptance test. -var ( - ServerPort string - ServerHost string - ServerScheme string - DebugHTTP bool -) - -// Credentials for the root key -var RootAuth runtime.ClientAuthInfoWriterFunc - -func init() { - if ServerScheme == "" { - ServerScheme = "http" - } - - if ServerPort == "" { - ServerPort = "8080" - } - - RootAuth = nil -} - -func GetWeaviateURL() string { - return fmt.Sprintf("%s://%s:%s", ServerScheme, ServerHost, ServerPort) -} diff --git a/test/helper/journey/backup_and_restore_journey.go b/test/helper/journey/backup_and_restore_journey.go deleted file mode 100644 index 6ef4234874737a75d059bfa6eb6c1a58c48240fa..0000000000000000000000000000000000000000 --- a/test/helper/journey/backup_and_restore_journey.go +++ /dev/null @@ -1,195 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package journey - -import ( - "testing" - "time" - - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/client/backups" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/test/helper" - "github.com/weaviate/weaviate/test/helper/sample-schema/books" -) - -func backupAndRestoreJourneyTest(t *testing.T, weaviateEndpoint, backend string) { - if weaviateEndpoint != "" { - helper.SetupClient(weaviateEndpoint) - } - booksClass := books.ClassContextionaryVectorizer() - helper.CreateClass(t, booksClass) - defer helper.DeleteClass(t, booksClass.Class) - - verifyThatAllBooksExist := func(t *testing.T) { - book := helper.AssertGetObject(t, booksClass.Class, books.Dune) - require.Equal(t, books.Dune, book.ID) - book = helper.AssertGetObject(t, booksClass.Class, books.ProjectHailMary) - require.Equal(t, books.ProjectHailMary, book.ID) - book = helper.AssertGetObject(t, booksClass.Class, books.TheLordOfTheIceGarden) - require.Equal(t, books.TheLordOfTheIceGarden, book.ID) - } - - backupID := "backup-1" - t.Run("add data to Books schema", func(t *testing.T) { - for _, book := range books.Objects() { - helper.CreateObject(t, book) - helper.AssertGetObjectEventually(t, book.Class, book.ID) - } - }) - - t.Run("verify that Books objects exist", func(t *testing.T) { - verifyThatAllBooksExist(t) - }) - - t.Run("verify invalid compression config", func(t *testing.T) { - // unknown compression level - resp, err := helper.CreateBackup(t, &models.BackupConfig{ - CompressionLevel: "some-weird-config", - }, booksClass.Class, backend, backupID) - - helper.AssertRequestFail(t, resp, err, func() { - _, ok := err.(*backups.BackupsCreateUnprocessableEntity) - require.True(t, ok, "not backups.BackupsCreateUnprocessableEntity") - }) - - // out of band cpu % - resp, err = helper.CreateBackup(t, &models.BackupConfig{ - CPUPercentage: 120, - }, booksClass.Class, backend, backupID) - helper.AssertRequestFail(t, resp, err, func() { - _, ok := err.(*backups.BackupsCreateUnprocessableEntity) - require.True(t, ok, "not backups.BackupsCreateUnprocessableEntity") - }) - - // out of band chunkSize - resp, err = helper.CreateBackup(t, &models.BackupConfig{ - ChunkSize: 1024, - }, booksClass.Class, backend, backupID) - helper.AssertRequestFail(t, resp, err, func() { - _, ok := err.(*backups.BackupsCreateUnprocessableEntity) - require.True(t, ok, "not backups.BackupsCreateUnprocessableEntity") - }) - }) - - t.Run("start backup process", func(t *testing.T) { - params := backups.NewBackupsCreateParams(). - WithBackend(backend). - WithBody(&models.BackupCreateRequest{ - ID: backupID, - Include: []string{booksClass.Class}, - Config: &models.BackupConfig{ - CPUPercentage: 80, - ChunkSize: 512, - CompressionLevel: models.BackupConfigCompressionLevelDefaultCompression, - }, - }) - resp, err := helper.Client(t).Backups.BackupsCreate(params, nil) - - helper.AssertRequestOk(t, resp, err, func() { - meta := resp.GetPayload() - require.NotNil(t, meta) - require.Equal(t, models.BackupCreateStatusResponseStatusSTARTED, *meta.Status) - }) - }) - - t.Run("verify that backup process is completed", func(t *testing.T) { - params := backups.NewBackupsCreateStatusParams(). - WithBackend(backend). - WithID(backupID) - for { - resp, err := helper.Client(t).Backups.BackupsCreateStatus(params, nil) - require.Nil(t, err) - require.NotNil(t, resp) - meta := resp.GetPayload() - require.NotNil(t, meta) - switch *meta.Status { - case models.BackupCreateStatusResponseStatusSUCCESS: - return - case models.BackupCreateStatusResponseStatusFAILED: - t.Errorf("failed to create backup, got response: %+v", meta) - return - default: - time.Sleep(1 * time.Second) - } - } - }) - - t.Run("verify that Books objects still exist", func(t *testing.T) { - verifyThatAllBooksExist(t) - }) - - t.Run("remove Books class", func(t *testing.T) { - helper.DeleteClass(t, booksClass.Class) - }) - - t.Run("verify that objects don't exist", func(t *testing.T) { - err := helper.AssertGetObjectFailsEventually(t, booksClass.Class, books.Dune) - require.NotNil(t, err) - err = helper.AssertGetObjectFailsEventually(t, booksClass.Class, books.ProjectHailMary) - require.NotNil(t, err) - err = helper.AssertGetObjectFailsEventually(t, booksClass.Class, books.TheLordOfTheIceGarden) - require.NotNil(t, err) - }) - - // out of band cpu % - t.Run("invalid restore request", func(t *testing.T) { - resp, err := helper.RestoreBackup(t, &models.RestoreConfig{ - CPUPercentage: 180, - }, booksClass.Class, backend, backupID, map[string]string{}) - - helper.AssertRequestFail(t, resp, err, func() { - _, ok := err.(*backups.BackupsRestoreUnprocessableEntity) - require.True(t, ok, "not backups.BackupsRestoreUnprocessableEntity") - }) - }) - - t.Run("start restore process", func(t *testing.T) { - params := backups.NewBackupsRestoreParams(). - WithBackend(backend). - WithID(backupID). - WithBody(&models.BackupRestoreRequest{ - Include: []string{booksClass.Class}, - }) - resp, err := helper.Client(t).Backups.BackupsRestore(params, nil) - helper.AssertRequestOk(t, resp, err, func() { - meta := resp.GetPayload() - require.NotNil(t, meta) - require.Equal(t, models.BackupCreateStatusResponseStatusSTARTED, *meta.Status) - }) - }) - t.Run("verify that restore process is completed", func(t *testing.T) { - params := backups.NewBackupsRestoreStatusParams(). - WithBackend(backend). - WithID(backupID) - for { - resp, err := helper.Client(t).Backups.BackupsRestoreStatus(params, nil) - require.Nil(t, err) - require.NotNil(t, resp) - meta := resp.GetPayload() - require.NotNil(t, meta) - switch *meta.Status { - case models.BackupRestoreStatusResponseStatusSUCCESS: - return - case models.BackupRestoreStatusResponseStatusFAILED: - t.Errorf("failed to restore backup, got response: %+v", meta) - return - default: - time.Sleep(1 * time.Second) - } - } - }) - - t.Run("verify that Books objects exist after restore", func(t *testing.T) { - verifyThatAllBooksExist(t) - }) -} diff --git a/test/helper/journey/backup_journey.go b/test/helper/journey/backup_journey.go deleted file mode 100644 index 4329f3cb6b7e26184be4b3fd941756b1e43af84a..0000000000000000000000000000000000000000 --- a/test/helper/journey/backup_journey.go +++ /dev/null @@ -1,298 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package journey - -import ( - "math/rand" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/backup" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/test/helper" - moduleshelper "github.com/weaviate/weaviate/test/helper/modules" -) - -type journeyType int - -const ( - singleNodeJourney journeyType = iota - clusterJourney -) - -type dataIntegrityCheck int - -const ( - checkClassPresenceOnly = iota - checkClassAndDataPresence -) - -const ( - singleTenant = "" - multiTenant = true -) - -func backupJourney(t *testing.T, className, backend, backupID string, - journeyType journeyType, dataIntegrityCheck dataIntegrityCheck, - tenantNames []string, pqEnabled bool, -) { - if journeyType == clusterJourney && backend == "filesystem" { - t.Run("should fail backup/restore with local filesystem backend", func(t *testing.T) { - backupResp, err := helper.CreateBackup(t, helper.DefaultBackupConfig(), className, backend, backupID) - assert.Nil(t, backupResp) - assert.Error(t, err) - - restoreResp, err := helper.RestoreBackup(t, helper.DefaultRestoreConfig(), className, backend, backupID, map[string]string{}) - assert.Nil(t, restoreResp) - assert.Error(t, err) - }) - return - } - - t.Run("create backup", func(t *testing.T) { - resp, err := helper.CreateBackup(t, helper.DefaultBackupConfig(), className, backend, backupID) - helper.AssertRequestOk(t, resp, err, nil) - // wait for create success - createTime := time.Now() - for { - if time.Now().After(createTime.Add(time.Minute)) { - break - } - - resp, err := helper.CreateBackupStatus(t, backend, backupID) - helper.AssertRequestOk(t, resp, err, func() { - require.NotNil(t, resp) - require.NotNil(t, resp.Payload) - require.NotNil(t, resp.Payload.Status) - }) - - if *resp.Payload.Status == string(backup.Success) { - break - } - time.Sleep(time.Second * 1) - } - - statusResp, err := helper.CreateBackupStatus(t, backend, backupID) - helper.AssertRequestOk(t, resp, err, func() { - require.NotNil(t, statusResp) - require.NotNil(t, statusResp.Payload) - require.NotNil(t, statusResp.Payload.Status) - }) - - require.Equal(t, *statusResp.Payload.Status, - string(backup.Success), statusResp.Payload.Error) - }) - - t.Run("delete class for restoration", func(t *testing.T) { - helper.DeleteClass(t, className) - }) - - t.Run("restore backup", func(t *testing.T) { - _, err := helper.RestoreBackup(t, helper.DefaultRestoreConfig(), className, backend, backupID, map[string]string{}) - require.Nil(t, err, "expected nil, got: %v", err) - - // wait for restore success - restoreTime := time.Now() - for { - if time.Now().After(restoreTime.Add(time.Minute)) { - break - } - - resp, err := helper.RestoreBackupStatus(t, backend, backupID) - helper.AssertRequestOk(t, resp, err, func() { - require.NotNil(t, resp) - require.NotNil(t, resp.Payload) - require.NotNil(t, resp.Payload.Status) - }) - - if *resp.Payload.Status == string(backup.Success) { - break - } - - time.Sleep(time.Second) - } - - statusResp, err := helper.RestoreBackupStatus(t, backend, backupID) - helper.AssertRequestOk(t, statusResp, err, func() { - require.NotNil(t, statusResp) - require.NotNil(t, statusResp.Payload) - require.NotNil(t, statusResp.Payload.Status) - }) - - require.Equal(t, *statusResp.Payload.Status, string(backup.Success)) - }) - - // assert class exists again it its entirety - if tenantNames != nil { - for _, name := range tenantNames { - moduleshelper.EnsureClassExists(t, className, name) - if dataIntegrityCheck == checkClassAndDataPresence { - count := moduleshelper.GetClassCount(t, className, name) - assert.Equal(t, int64(500/len(tenantNames)), count) - } - } - } else { - moduleshelper.EnsureClassExists(t, className, singleTenant) - if dataIntegrityCheck == checkClassAndDataPresence { - count := moduleshelper.GetClassCount(t, className, singleTenant) - assert.Equal(t, int64(500), count) - if pqEnabled { - moduleshelper.EnsureCompressedVectorsRestored(t, className) - } - } - } -} - -func nodeMappingBackupJourney_Backup(t *testing.T, className, backend, backupID string, tenantNames []string, -) { - t.Run("create backup", func(t *testing.T) { - resp, err := helper.CreateBackup(t, helper.DefaultBackupConfig(), className, backend, backupID) - helper.AssertRequestOk(t, resp, err, nil) - // wait for create success - createTime := time.Now() - for { - if time.Now().After(createTime.Add(time.Minute)) { - break - } - - resp, err := helper.CreateBackupStatus(t, backend, backupID) - helper.AssertRequestOk(t, resp, err, func() { - require.NotNil(t, resp) - require.NotNil(t, resp.Payload) - require.NotNil(t, resp.Payload.Status) - }) - - if *resp.Payload.Status == string(backup.Success) { - break - } - time.Sleep(time.Second * 1) - } - - statusResp, err := helper.CreateBackupStatus(t, backend, backupID) - helper.AssertRequestOk(t, resp, err, func() { - require.NotNil(t, statusResp) - require.NotNil(t, statusResp.Payload) - require.NotNil(t, statusResp.Payload.Status) - }) - - require.Equal(t, *statusResp.Payload.Status, string(backup.Success)) - }) -} - -func nodeMappingBackupJourney_Restore(t *testing.T, className, backend, backupID string, tenantNames []string, nodeMapping map[string]string) { - t.Run("restore backup", func(t *testing.T) { - _, err := helper.RestoreBackup(t, helper.DefaultRestoreConfig(), className, backend, backupID, nodeMapping) - require.Nil(t, err, "expected nil, got: %v", err) - - // wait for restore success - restoreTime := time.Now() - for { - if time.Now().After(restoreTime.Add(time.Minute)) { - break - } - - resp, err := helper.RestoreBackupStatus(t, backend, backupID) - helper.AssertRequestOk(t, resp, err, func() { - require.NotNil(t, resp) - require.NotNil(t, resp.Payload) - require.NotNil(t, resp.Payload.Status) - }) - - if *resp.Payload.Status == string(backup.Success) { - break - } - - time.Sleep(time.Second) - } - - statusResp, err := helper.RestoreBackupStatus(t, backend, backupID) - helper.AssertRequestOk(t, statusResp, err, func() { - require.NotNil(t, statusResp) - require.NotNil(t, statusResp.Payload) - require.NotNil(t, statusResp.Payload.Status) - }) - - require.Equal(t, *statusResp.Payload.Status, string(backup.Success)) - }) - - // assert class exists again it its entirety - if tenantNames != nil { - for _, name := range tenantNames { - count := moduleshelper.GetClassCount(t, className, name) - assert.Equal(t, int64(500/len(tenantNames)), count) - } - } else { - count := moduleshelper.GetClassCount(t, className, singleTenant) - assert.Equal(t, int64(500), count) - } -} - -func addTestClass(t *testing.T, className string, multiTenant bool) { - class := &models.Class{ - Class: className, - ModuleConfig: map[string]interface{}{ - "text2vec-contextionary": map[string]interface{}{ - "vectorizeClassName": true, - }, - }, - Properties: []*models.Property{ - { - Name: "contents", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - } - - if multiTenant { - class.MultiTenancyConfig = &models.MultiTenancyConfig{ - Enabled: true, - } - } - - helper.CreateClass(t, class) -} - -func addTestObjects(t *testing.T, className string, tenantNames []string) { - const ( - noteLengthMin = 4 - noteLengthMax = 1024 - - batchSize = 10 - numBatches = 50 - ) - - seededRand := rand.New(rand.NewSource(time.Now().UnixNano())) - multiTenant := len(tenantNames) > 0 - - for i := 0; i < numBatches; i++ { - batch := make([]*models.Object, batchSize) - for j := 0; j < batchSize; j++ { - contentsLength := noteLengthMin + seededRand.Intn(noteLengthMax-noteLengthMin+1) - contents := helper.GetRandomString(contentsLength) - - obj := models.Object{ - Class: className, - Properties: map[string]interface{}{"contents": contents}, - } - if multiTenant { - obj.Tenant = tenantNames[i] - } - batch[j] = &obj - } - helper.CreateObjectsBatch(t, batch) - - } -} diff --git a/test/helper/journey/backup_journey_tests.go b/test/helper/journey/backup_journey_tests.go deleted file mode 100644 index 06e7fa0362d7920899355a1b5cfb707d912d6e6d..0000000000000000000000000000000000000000 --- a/test/helper/journey/backup_journey_tests.go +++ /dev/null @@ -1,100 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package journey - -import ( - "testing" -) - -// BackupJourneyTests_SingleNode this method gathers all backup related e2e tests -func BackupJourneyTests_SingleNode(t *testing.T, weaviateEndpoint, backend, className, backupID string, tenantNames []string) { - if len(tenantNames) > 0 { - t.Run("multi-tenant single node backup", func(t *testing.T) { - singleNodeBackupJourneyTest(t, weaviateEndpoint, backend, className, backupID, tenantNames, false) - }) - t.Run("multi-tenant single node backup with empty class", func(t *testing.T) { - singleNodeBackupEmptyClassJourneyTest(t, weaviateEndpoint, backend, className, backupID+"_empty", tenantNames) - }) - } else { - // This is a simple test which covers almost the same scenario as singleNodeBackupJourneyTest - // but is left here to be expanded in the future with a more complex example - // like adding there a new reference property and trying to run the test with 2 classes which - // one of those classes is a class with a reference property - t.Run("backup and restore Books", func(t *testing.T) { - backupAndRestoreJourneyTest(t, weaviateEndpoint, backend) - }) - - t.Run("single-tenant single node backup", func(t *testing.T) { - singleNodeBackupJourneyTest(t, weaviateEndpoint, backend, className, backupID, nil, false) - }) - - t.Run("single-tenant single node backup with PQ", func(t *testing.T) { - singleNodeBackupJourneyTest(t, weaviateEndpoint, backend, className, backupID+"_pq", nil, true) - }) - } -} - -// BackupJourneyTests_Cluster this method gathers all backup related e2e tests to be run on a cluster -func BackupJourneyTests_Cluster(t *testing.T, backend, className, backupID string, - tenantNames []string, weaviateEndpoints ...string, -) { - if len(weaviateEndpoints) <= 1 { - t.Fatal("must provide more than one node for cluster backup test") - } - - if len(tenantNames) > 0 { - t.Run("multi-tenant cluster backup", func(t *testing.T) { - coordinator := weaviateEndpoints[0] - clusterBackupJourneyTest(t, backend, className, backupID, - coordinator, tenantNames, false, weaviateEndpoints[1:]...) - }) - t.Run("multi-tenant cluster backup with empty class", func(t *testing.T) { - coordinator := weaviateEndpoints[0] - clusterBackupEmptyClassJourneyTest(t, backend, className, backupID+"_empty", - coordinator, tenantNames, weaviateEndpoints[1:]...) - }) - } else { - t.Run("single-tenant cluster backup", func(t *testing.T) { - if len(weaviateEndpoints) <= 1 { - t.Fatal("must provide more than one node for cluster backup test") - } - - coordinator := weaviateEndpoints[0] - clusterBackupJourneyTest(t, backend, className, backupID+"_pq", - coordinator, nil, true, weaviateEndpoints[1:]...) - }) - } -} - -func NodeMappingBackupJourneyTests_SingleNode_Backup(t *testing.T, weaviateEndpoint, backend, className, backupID string, tenantNames []string) { - if len(tenantNames) > 0 { - t.Run("multi-tenant single node backup", func(t *testing.T) { - singleNodeNodeMappingBackupJourney_Backup(t, weaviateEndpoint, backend, className, backupID, tenantNames) - }) - } else { - t.Run("single-tenant single node backup", func(t *testing.T) { - singleNodeNodeMappingBackupJourney_Backup(t, weaviateEndpoint, backend, className, backupID, nil) - }) - } -} - -func NodeMappingBackupJourneyTests_SingleNode_Restore(t *testing.T, weaviateEndpoint, backend, className, backupID string, tenantNames []string, nodeMapping map[string]string) { - if len(tenantNames) > 0 { - t.Run("multi-tenant single node restore", func(t *testing.T) { - singleNodeNodeMappingBackupJourney_Restore(t, weaviateEndpoint, backend, className, backupID, tenantNames, nodeMapping) - }) - } else { - t.Run("single-tenant single node restore", func(t *testing.T) { - singleNodeNodeMappingBackupJourney_Restore(t, weaviateEndpoint, backend, className, backupID, nil, nodeMapping) - }) - } -} diff --git a/test/helper/journey/cluster_backup_journey.go b/test/helper/journey/cluster_backup_journey.go deleted file mode 100644 index 565baf8c77e652d73726c31eb02751416d5f6855..0000000000000000000000000000000000000000 --- a/test/helper/journey/cluster_backup_journey.go +++ /dev/null @@ -1,109 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package journey - -import ( - "fmt" - "math/rand" - "testing" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/test/helper" -) - -func clusterBackupJourneyTest(t *testing.T, backend, className, - backupID, coordinatorEndpoint string, tenantNames []string, pqEnabled bool, - nodeEndpoints ...string, -) { - uploaderEndpoint := nodeEndpoints[rand.Intn(len(nodeEndpoints))] - helper.SetupClient(uploaderEndpoint) - t.Logf("uploader selected -> %s:%s", helper.ServerHost, helper.ServerPort) - - if len(tenantNames) > 0 { - // upload data to a node other than the coordinator - t.Run(fmt.Sprintf("add test data to endpoint: %s", uploaderEndpoint), func(t *testing.T) { - addTestClass(t, className, multiTenant) - tenants := make([]*models.Tenant, len(tenantNames)) - for i := range tenantNames { - tenants[i] = &models.Tenant{Name: tenantNames[i]} - } - helper.CreateTenants(t, className, tenants) - addTestObjects(t, className, tenantNames) - }) - } else { - // upload data to a node other than the coordinator - t.Run(fmt.Sprintf("add test data to endpoint: %s", uploaderEndpoint), func(t *testing.T) { - addTestClass(t, className, !multiTenant) - addTestObjects(t, className, nil) - }) - } - - if pqEnabled { - pq := map[string]interface{}{ - "enabled": true, - "segments": 1, - "centroids": 16, - } - helper.EnablePQ(t, className, pq) - } - - helper.SetupClient(coordinatorEndpoint) - t.Logf("coordinator selected -> %s:%s", helper.ServerHost, helper.ServerPort) - - // send backup requests to the chosen coordinator - t.Run(fmt.Sprintf("with coordinator endpoint: %s", coordinatorEndpoint), func(t *testing.T) { - backupJourney(t, className, backend, backupID, clusterJourney, - checkClassAndDataPresence, tenantNames, pqEnabled) - }) - - t.Run("cleanup", func(t *testing.T) { - helper.DeleteClass(t, className) - }) -} - -func clusterBackupEmptyClassJourneyTest(t *testing.T, backend, className, backupID, - coordinatorEndpoint string, tenantNames []string, nodeEndpoints ...string, -) { - uploaderEndpoint := nodeEndpoints[rand.Intn(len(nodeEndpoints))] - helper.SetupClient(uploaderEndpoint) - t.Logf("uploader selected -> %s:%s", helper.ServerHost, helper.ServerPort) - - if len(tenantNames) > 0 { - // upload data to a node other than the coordinator - t.Run(fmt.Sprintf("add test data to endpoint: %s", uploaderEndpoint), func(t *testing.T) { - addTestClass(t, className, multiTenant) - tenants := make([]*models.Tenant, len(tenantNames)) - for i := range tenantNames { - tenants[i] = &models.Tenant{Name: tenantNames[i]} - } - helper.CreateTenants(t, className, tenants) - }) - } else { - // upload data to a node other than the coordinator - t.Run(fmt.Sprintf("add test data to endpoint: %s", uploaderEndpoint), func(t *testing.T) { - addTestClass(t, className, !multiTenant) - }) - } - - helper.SetupClient(coordinatorEndpoint) - t.Logf("coordinator selected -> %s:%s", helper.ServerHost, helper.ServerPort) - - // send backup requests to the chosen coordinator - t.Run(fmt.Sprintf("with coordinator endpoint: %s", coordinatorEndpoint), func(t *testing.T) { - backupJourney(t, className, backend, backupID, clusterJourney, - checkClassPresenceOnly, tenantNames, false) - }) - - t.Run("cleanup", func(t *testing.T) { - helper.DeleteClass(t, className) - }) -} diff --git a/test/helper/journey/group_by_journey_tests.go b/test/helper/journey/group_by_journey_tests.go deleted file mode 100644 index e9fc9197949ed6eb6a6de0af6be06638e3187715..0000000000000000000000000000000000000000 --- a/test/helper/journey/group_by_journey_tests.go +++ /dev/null @@ -1,172 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package journey - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/test/helper" - graphqlhelper "github.com/weaviate/weaviate/test/helper/graphql" - "github.com/weaviate/weaviate/test/helper/sample-schema/documents" -) - -func GroupBySingleAndMultiShardTests(t *testing.T, weaviateEndpoint string) { - if weaviateEndpoint != "" { - helper.SetupClient(weaviateEndpoint) - } - // helper methods - getGroup := func(value interface{}) map[string]interface{} { - group := value.(map[string]interface{})["_additional"].(map[string]interface{})["group"].(map[string]interface{}) - return group - } - getGroupHits := func(group map[string]interface{}) (string, []string) { - result := []string{} - hits := group["hits"].([]interface{}) - for _, hit := range hits { - additional := hit.(map[string]interface{})["_additional"].(map[string]interface{}) - result = append(result, additional["id"].(string)) - } - groupedBy := group["groupedBy"].(map[string]interface{}) - groupedByValue := groupedBy["value"].(string) - return groupedByValue, result - } - // test methods - create := func(t *testing.T, multishard bool) { - for _, class := range documents.ClassesContextionaryVectorizer(multishard) { - helper.CreateClass(t, class) - } - for _, obj := range documents.Objects() { - helper.CreateObject(t, obj) - helper.AssertGetObjectEventually(t, obj.Class, obj.ID) - } - } - groupBy := func(t *testing.T, groupsCount, objectsPerGroup int) { - query := ` - { - Get{ - Passage( - nearObject:{ - id: "00000000-0000-0000-0000-000000000001" - } - groupBy:{ - path:["ofDocument"] - groups:%v - objectsPerGroup:%v - } - ){ - _additional{ - id - group{ - groupedBy{value} - count - maxDistance - minDistance - hits { - _additional{ - id - distance - } - } - } - } - } - } - } - ` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, fmt.Sprintf(query, groupsCount, objectsPerGroup)) - groups := result.Get("Get", "Passage").AsSlice() - - require.Len(t, groups, groupsCount) - - expectedResults := map[string][]string{} - - groupedBy1 := `weaviate://localhost/Document/00000000-0000-0000-0000-000000000011` - expectedGroup1 := []string{ - documents.PassageIDs[0].String(), - documents.PassageIDs[5].String(), - documents.PassageIDs[4].String(), - documents.PassageIDs[3].String(), - documents.PassageIDs[2].String(), - documents.PassageIDs[1].String(), - } - expectedResults[groupedBy1] = expectedGroup1 - - groupedBy2 := `weaviate://localhost/Document/00000000-0000-0000-0000-000000000012` - expectedGroup2 := []string{ - documents.PassageIDs[6].String(), - documents.PassageIDs[7].String(), - } - expectedResults[groupedBy2] = expectedGroup2 - - groupsOrder := []string{groupedBy1, groupedBy2} - - for i, current := range groups { - group := getGroup(current) - groupedBy, ids := getGroupHits(group) - assert.Equal(t, groupsOrder[i], groupedBy) - for j := range ids { - assert.Equal(t, expectedResults[groupedBy][j], ids[j]) - } - } - } - delete := func(t *testing.T) { - helper.DeleteClass(t, documents.Passage) - helper.DeleteClass(t, documents.Document) - } - // tests - tests := []struct { - name string - multishard bool - groups, objectsPerGroup int - }{ - { - name: "single shard - 2 groups 10 objects per group", - multishard: false, - groups: 2, - objectsPerGroup: 10, - }, - { - name: "multi shard - 2 groups 10 objects per group", - multishard: true, - groups: 2, - objectsPerGroup: 10, - }, - { - name: "single shard - 1 groups 1 objects per group", - multishard: false, - groups: 1, - objectsPerGroup: 1, - }, - { - name: "multi shard - 1 groups 1 objects per group", - multishard: true, - groups: 1, - objectsPerGroup: 1, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Run("create", func(t *testing.T) { - create(t, tt.multishard) - }) - t.Run("group by", func(t *testing.T) { - groupBy(t, tt.groups, tt.objectsPerGroup) - }) - t.Run("delete", func(t *testing.T) { - delete(t) - }) - }) - } -} diff --git a/test/helper/journey/single_node_backup_journey.go b/test/helper/journey/single_node_backup_journey.go deleted file mode 100644 index e142a1abc4a7c19172d055d961d6400402c3a3b4..0000000000000000000000000000000000000000 --- a/test/helper/journey/single_node_backup_journey.go +++ /dev/null @@ -1,137 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package journey - -import ( - "testing" - - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/test/helper" -) - -func singleNodeBackupJourneyTest(t *testing.T, - weaviateEndpoint, backend, className, backupID string, - tenantNames []string, pqEnabled bool, -) { - if weaviateEndpoint != "" { - helper.SetupClient(weaviateEndpoint) - } - - if len(tenantNames) > 0 { - t.Run("add test data", func(t *testing.T) { - addTestClass(t, className, multiTenant) - tenants := make([]*models.Tenant, len(tenantNames)) - for i := range tenantNames { - tenants[i] = &models.Tenant{Name: tenantNames[i]} - } - helper.CreateTenants(t, className, tenants) - addTestObjects(t, className, tenantNames) - }) - } else { - t.Run("add test data", func(t *testing.T) { - addTestClass(t, className, !multiTenant) - addTestObjects(t, className, nil) - }) - } - - if pqEnabled { - pq := map[string]interface{}{ - "enabled": true, - "segments": 1, - "centroids": 16, - } - helper.EnablePQ(t, className, pq) - } - - t.Run("single node backup", func(t *testing.T) { - backupJourney(t, className, backend, backupID, singleNodeJourney, - checkClassAndDataPresence, tenantNames, pqEnabled) - }) - - t.Run("cleanup", func(t *testing.T) { - helper.DeleteClass(t, className) - }) -} - -func singleNodeBackupEmptyClassJourneyTest(t *testing.T, - weaviateEndpoint, backend, className, backupID string, - tenantNames []string, -) { - if weaviateEndpoint != "" { - helper.SetupClient(weaviateEndpoint) - } - - if len(tenantNames) <= 0 { - t.Skip("test is only relevant with tenancy enabled") - } - - t.Run("add test class and tenant", func(t *testing.T) { - addTestClass(t, className, multiTenant) - tenants := make([]*models.Tenant, len(tenantNames)) - for i := range tenantNames { - tenants[i] = &models.Tenant{Name: tenantNames[i]} - } - helper.CreateTenants(t, className, tenants) - }) - - t.Run("single node backup", func(t *testing.T) { - backupJourney(t, className, backend, backupID, - singleNodeJourney, checkClassPresenceOnly, tenantNames, false) - }) - - t.Run("cleanup", func(t *testing.T) { - helper.DeleteClass(t, className) - }) -} - -func singleNodeNodeMappingBackupJourney_Backup(t *testing.T, - weaviateEndpoint, backend, className, backupID string, - tenantNames []string, -) { - if weaviateEndpoint != "" { - helper.SetupClient(weaviateEndpoint) - } - - if len(tenantNames) > 0 { - t.Run("add test data", func(t *testing.T) { - addTestClass(t, className, multiTenant) - tenants := make([]*models.Tenant, len(tenantNames)) - for i := range tenantNames { - tenants[i] = &models.Tenant{Name: tenantNames[i]} - } - helper.CreateTenants(t, className, tenants) - addTestObjects(t, className, tenantNames) - }) - } else { - t.Run("add test data", func(t *testing.T) { - addTestClass(t, className, !multiTenant) - addTestObjects(t, className, nil) - }) - } - - t.Run("single node backup", func(t *testing.T) { - nodeMappingBackupJourney_Backup(t, className, backend, backupID, tenantNames) - }) -} - -func singleNodeNodeMappingBackupJourney_Restore(t *testing.T, - weaviateEndpoint, backend, className, backupID string, - tenantNames []string, nodeMapping map[string]string, -) { - if weaviateEndpoint != "" { - helper.SetupClient(weaviateEndpoint) - } - - t.Run("single node restore", func(t *testing.T) { - nodeMappingBackupJourney_Restore(t, className, backend, backupID, tenantNames, nodeMapping) - }) -} diff --git a/test/helper/logger.go b/test/helper/logger.go deleted file mode 100644 index 98baf36bb016a8278ba48a63c71b750ca5c1f7c9..0000000000000000000000000000000000000000 --- a/test/helper/logger.go +++ /dev/null @@ -1,28 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package helper - -// Internal struct to link the HTTP client logging of the Weaviate API client to the test's logging output. - -import "testing" - -type testLogger struct { - t *testing.T -} - -func (tl *testLogger) Printf(format string, args ...interface{}) { - tl.t.Logf("HTTP LOG:\n"+format, args...) -} - -func (tl *testLogger) Debugf(format string, args ...interface{}) { - tl.t.Logf("HTTP DEBUG:\n"+format, args...) -} diff --git a/test/helper/modules/modules_helper.go b/test/helper/modules/modules_helper.go deleted file mode 100644 index 5253bfc0bcaa151d8796b4c57fc6cdb3502fd10a..0000000000000000000000000000000000000000 --- a/test/helper/modules/modules_helper.go +++ /dev/null @@ -1,134 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package moduleshelper - -import ( - "context" - "encoding/json" - "fmt" - "net/http" - "os" - "path/filepath" - "testing" - - "cloud.google.com/go/storage" - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/test/helper" - graphqlhelper "github.com/weaviate/weaviate/test/helper/graphql" - "google.golang.org/api/googleapi" - "google.golang.org/api/option" -) - -func EnsureClassExists(t *testing.T, className string, tenant string) { - query := fmt.Sprintf("{Aggregate{%s", className) - if tenant != "" { - query += fmt.Sprintf("(tenant:%q)", tenant) - } - query += " { meta { count}}}}" - resp := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - - class := resp.Get("Aggregate", className).Result.([]interface{}) - require.Len(t, class, 1) -} - -func EnsureCompressedVectorsRestored(t *testing.T, className string) { - query := fmt.Sprintf("{Get{%s(limit:1){_additional{vector}}}}", className) - resp := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - - class := resp.Get("Get", className).Result.([]interface{}) - require.Len(t, class, 1) - vecResp := class[0].(map[string]interface{})["_additional"].(map[string]interface{})["vector"].([]interface{}) - - searchVec := graphqlhelper.Vec2String(graphqlhelper.ParseVec(t, vecResp)) - - limit := 10 - query = fmt.Sprintf( - "{Get{%s(nearVector:{vector:%s} limit:%d){_additional{vector}}}}", - className, searchVec, limit) - resp = graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - class = resp.Get("Get", className).Result.([]interface{}) - require.Len(t, class, limit) -} - -func GetClassCount(t *testing.T, className string, tenant string) int64 { - query := fmt.Sprintf("{Aggregate{%s", className) - if tenant != "" { - query += fmt.Sprintf("(tenant:%q)", tenant) - } - query += " { meta { count}}}}" - resp := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - - class := resp.Get("Aggregate", className).Result.([]interface{}) - require.Len(t, class, 1) - - meta := class[0].(map[string]interface{})["meta"].(map[string]interface{}) - - countPayload := meta["count"].(json.Number) - - count, err := countPayload.Int64() - require.Nil(t, err) - - return count -} - -func CreateTestFiles(t *testing.T, dirPath string) []string { - count := 5 - filePaths := make([]string, count) - var fileName string - - for i := 0; i < count; i += 1 { - fileName = fmt.Sprintf("file_%d.db", i) - filePaths[i] = filepath.Join(dirPath, fileName) - file, err := os.Create(filePaths[i]) - if err != nil { - t.Fatalf("failed to create test file '%s': %s", fileName, err) - } - fmt.Fprintf(file, "This is content of db file named %s", fileName) - file.Close() - } - return filePaths -} - -func CreateGCSBucket(ctx context.Context, t *testing.T, projectID, bucketName string) { - client, err := storage.NewClient(ctx, option.WithoutAuthentication()) - require.Nil(t, err) - - err = client.Bucket(bucketName).Create(ctx, projectID, nil) - gcsErr, ok := err.(*googleapi.Error) - if ok { - // the bucket persists from the previous test. - // if the bucket already exists, we can proceed - if gcsErr.Code == http.StatusConflict { - return - } - } - require.Nil(t, err) -} - -func CreateAzureContainer(ctx context.Context, t *testing.T, endpoint, containerName string) { - connectionString := "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://%s/devstoreaccount1;" - client, err := azblob.NewClientFromConnectionString(fmt.Sprintf(connectionString, endpoint), nil) - require.Nil(t, err) - - _, err = client.CreateContainer(ctx, containerName, nil) - require.Nil(t, err) -} - -func DeleteAzureContainer(ctx context.Context, t *testing.T, endpoint, containerName string) { - connectionString := "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://%s/devstoreaccount1;" - client, err := azblob.NewClientFromConnectionString(fmt.Sprintf(connectionString, endpoint), nil) - require.Nil(t, err) - - _, err = client.DeleteContainer(ctx, containerName, nil) - require.Nil(t, err) -} diff --git a/test/helper/objects.go b/test/helper/objects.go deleted file mode 100644 index bf16a0c7e3ff64e895def65bc17668ee91b86383..0000000000000000000000000000000000000000 --- a/test/helper/objects.go +++ /dev/null @@ -1,230 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package helper - -import ( - "strings" - "testing" - - "github.com/go-openapi/strfmt" - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/client/batch" - "github.com/weaviate/weaviate/client/meta" - "github.com/weaviate/weaviate/client/objects" - "github.com/weaviate/weaviate/client/schema" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema/crossref" - "github.com/weaviate/weaviate/usecases/replica" -) - -func SetupClient(uri string) { - host, port := "", "" - res := strings.Split(uri, ":") - if len(res) == 2 { - host, port = res[0], res[1] - } - ServerHost = host - ServerPort = port -} - -func CreateClass(t *testing.T, class *models.Class) { - params := schema.NewSchemaObjectsCreateParams().WithObjectClass(class) - resp, err := Client(t).Schema.SchemaObjectsCreate(params, nil) - AssertRequestOk(t, resp, err, nil) -} - -func GetClass(t *testing.T, class string) *models.Class { - params := schema.NewSchemaObjectsGetParams().WithClassName(class) - resp, err := Client(t).Schema.SchemaObjectsGet(params, nil) - AssertRequestOk(t, resp, err, nil) - return resp.Payload -} - -func UpdateClass(t *testing.T, class *models.Class) { - params := schema.NewSchemaObjectsUpdateParams(). - WithObjectClass(class).WithClassName(class.Class) - resp, err := Client(t).Schema.SchemaObjectsUpdate(params, nil) - AssertRequestOk(t, resp, err, nil) -} - -func CreateObject(t *testing.T, object *models.Object) { - params := objects.NewObjectsCreateParams().WithBody(object) - resp, err := Client(t).Objects.ObjectsCreate(params, nil) - AssertRequestOk(t, resp, err, nil) -} - -func CreateObjectCL(t *testing.T, object *models.Object, cl replica.ConsistencyLevel) { - cls := string(cl) - params := objects.NewObjectsCreateParams().WithBody(object).WithConsistencyLevel(&cls) - resp, err := Client(t).Objects.ObjectsCreate(params, nil) - AssertRequestOk(t, resp, err, nil) -} - -func CreateObjectsBatch(t *testing.T, objects []*models.Object) { - params := batch.NewBatchObjectsCreateParams(). - WithBody(batch.BatchObjectsCreateBody{ - Objects: objects, - }) - resp, err := Client(t).Batch.BatchObjectsCreate(params, nil) - AssertRequestOk(t, resp, err, nil) - CheckObjectsBatchResponse(t, resp.Payload, err) -} - -func CheckObjectsBatchResponse(t *testing.T, resp []*models.ObjectsGetResponse, err error) { - AssertRequestOk(t, resp, err, nil) - for _, elem := range resp { - if !assert.Nil(t, elem.Result.Errors) { - t.Logf("expected nil, got: %v", - elem.Result.Errors.Error[0].Message) - } - } -} - -func UpdateObject(t *testing.T, object *models.Object) { - params := objects.NewObjectsUpdateParams().WithID(object.ID).WithBody(object) - resp, err := Client(t).Objects.ObjectsUpdate(params, nil) - AssertRequestOk(t, resp, err, nil) -} - -func UpdateObjectCL(t *testing.T, object *models.Object, cl replica.ConsistencyLevel) { - cls := string(cl) - params := objects.NewObjectsClassPutParams().WithClassName(object.Class). - WithID(object.ID).WithBody(object).WithConsistencyLevel(&cls) - resp, err := Client(t).Objects.ObjectsClassPut(params, nil) - AssertRequestOk(t, resp, err, nil) -} - -func PatchObject(t *testing.T, object *models.Object) { - params := objects.NewObjectsPatchParams().WithID(object.ID).WithBody(object) - resp, err := Client(t).Objects.ObjectsPatch(params, nil) - AssertRequestOk(t, resp, err, nil) -} - -func DeleteClass(t *testing.T, class string) { - delParams := schema.NewSchemaObjectsDeleteParams().WithClassName(class) - delRes, err := Client(t).Schema.SchemaObjectsDelete(delParams, nil) - AssertRequestOk(t, delRes, err, nil) -} - -func DeleteObject(t *testing.T, object *models.Object) { - params := objects.NewObjectsClassDeleteParams(). - WithClassName(object.Class).WithID(object.ID) - resp, err := Client(t).Objects.ObjectsClassDelete(params, nil) - AssertRequestOk(t, resp, err, nil) -} - -func DeleteObjectsBatch(t *testing.T, body *models.BatchDelete) { - params := batch.NewBatchObjectsDeleteParams().WithBody(body) - resp, err := Client(t).Batch.BatchObjectsDelete(params, nil) - AssertRequestOk(t, resp, err, nil) -} - -func DeleteTenantObjectsBatch(t *testing.T, body *models.BatchDelete, - tenant string, -) (*models.BatchDeleteResponse, error) { - params := batch.NewBatchObjectsDeleteParams(). - WithBody(body).WithTenant(&tenant) - resp, err := Client(t).Batch.BatchObjectsDelete(params, nil) - if err != nil { - return nil, err - } - return resp.Payload, nil -} - -func AddReferences(t *testing.T, refs []*models.BatchReference) ([]*models.BatchReferenceResponse, error) { - params := batch.NewBatchReferencesCreateParams().WithBody(refs) - resp, err := Client(t).Batch.BatchReferencesCreate(params, nil) - if err != nil { - return nil, err - } - return resp.Payload, nil -} - -func CheckReferencesBatchResponse(t *testing.T, resp []*models.BatchReferenceResponse, err error) { - AssertRequestOk(t, resp, err, nil) - for _, elem := range resp { - if !assert.Nil(t, elem.Result.Errors) { - t.Logf("expected nil, got: %v", - elem.Result.Errors.Error[0].Message) - } - } -} - -func AddReference(t *testing.T, object *models.Object, ref *models.SingleRef, prop string) { - params := objects.NewObjectsClassReferencesCreateParams(). - WithClassName(object.Class).WithID(object.ID).WithBody(ref).WithPropertyName(prop) - resp, err := Client(t).Objects.ObjectsClassReferencesCreate(params, nil) - AssertRequestOk(t, resp, err, nil) -} - -func AddReferenceTenant(t *testing.T, object *models.Object, ref *models.SingleRef, prop string, tenant string) { - params := objects.NewObjectsClassReferencesCreateParams(). - WithClassName(object.Class).WithID(object.ID).WithBody(ref).WithPropertyName(prop).WithTenant(&tenant) - resp, err := Client(t).Objects.ObjectsClassReferencesCreate(params, nil) - AssertRequestOk(t, resp, err, nil) -} - -func DeleteReference(t *testing.T, object *models.Object, ref *models.SingleRef, prop string) { - params := objects.NewObjectsClassReferencesDeleteParams(). - WithClassName(object.Class).WithID(object.ID).WithBody(ref).WithPropertyName(prop) - resp, err := Client(t).Objects.ObjectsClassReferencesDelete(params, nil) - AssertRequestOk(t, resp, err, nil) -} - -func DeleteReferenceTenant(t *testing.T, object *models.Object, ref *models.SingleRef, prop string, tenant string) { - params := objects.NewObjectsClassReferencesDeleteParams(). - WithClassName(object.Class).WithID(object.ID).WithBody(ref).WithPropertyName(prop).WithTenant(&tenant) - resp, err := Client(t).Objects.ObjectsClassReferencesDelete(params, nil) - AssertRequestOk(t, resp, err, nil) -} - -func UpdateReferenceTenant(t *testing.T, object *models.Object, ref models.MultipleRef, prop string, tenant string) { - params := objects.NewObjectsClassReferencesPutParams(). - WithClassName(object.Class).WithID(object.ID).WithBody(ref).WithPropertyName(prop).WithTenant(&tenant) - resp, err := Client(t).Objects.ObjectsClassReferencesPut(params, nil) - AssertRequestOk(t, resp, err, nil) -} - -func CreateTenants(t *testing.T, class string, tenants []*models.Tenant) { - params := schema.NewTenantsCreateParams().WithClassName(class).WithBody(tenants) - resp, err := Client(t).Schema.TenantsCreate(params, nil) - AssertRequestOk(t, resp, err, nil) -} - -func CreateTenantsReturnError(t *testing.T, class string, tenants []*models.Tenant) error { - params := schema.NewTenantsCreateParams().WithClassName(class).WithBody(tenants) - _, err := Client(t).Schema.TenantsCreate(params, nil) - return err -} - -func GetTenants(t *testing.T, class string) (*schema.TenantsGetOK, error) { - params := schema.NewTenantsGetParams().WithClassName(class) - resp, err := Client(t).Schema.TenantsGet(params, nil) - return resp, err -} - -func DeleteTenants(t *testing.T, class string, tenants []string) error { - params := schema.NewTenantsDeleteParams().WithClassName(class).WithTenants(tenants) - _, err := Client(t).Schema.TenantsDelete(params, nil) - return err -} - -func NewBeacon(className string, id strfmt.UUID) strfmt.URI { - return crossref.New("localhost", className, id).SingleRef().Beacon -} - -func GetMeta(t *testing.T) *models.Meta { - params := meta.NewMetaGetParams() - resp, err := Client(t).Meta.MetaGet(params, nil) - AssertRequestOk(t, resp, err, nil) - return resp.Payload -} diff --git a/test/helper/objects_assertions.go b/test/helper/objects_assertions.go deleted file mode 100644 index 6ea2df0fd1119537c64a9eaed89fa491d5214c4e..0000000000000000000000000000000000000000 --- a/test/helper/objects_assertions.go +++ /dev/null @@ -1,237 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package helper - -import ( - "net/http" - "testing" - - "github.com/go-openapi/strfmt" - "github.com/weaviate/weaviate/client/objects" - "github.com/weaviate/weaviate/client/schema" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/usecases/replica" -) - -func AssertCreateObject(t *testing.T, className string, schema map[string]interface{}) strfmt.UUID { - t.Helper() - params := objects.NewObjectsCreateParams().WithBody( - &models.Object{ - Class: className, - Properties: schema, - }) - - resp, err := Client(t).Objects.ObjectsCreate(params, nil) - - var objectID strfmt.UUID - - // Ensure that the response is OK - AssertRequestOk(t, resp, err, func() { - objectID = resp.Payload.ID - }) - - return objectID -} - -func AssertGetObject(t *testing.T, class string, uuid strfmt.UUID, include ...string) *models.Object { - t.Helper() - obj, err := GetObject(t, class, uuid, include...) - AssertRequestOk(t, obj, err, nil) - return obj -} - -func AssertGetObjectEventually(t *testing.T, class string, uuid strfmt.UUID) *models.Object { - var ( - resp *objects.ObjectsClassGetOK - err error - ) - - checkThunk := func() interface{} { - resp, err = Client(t).Objects.ObjectsClassGet(objects.NewObjectsClassGetParams().WithClassName(class).WithID(uuid), nil) - return err == nil - } - - AssertEventuallyEqual(t, true, checkThunk) - - var object *models.Object - - AssertRequestOk(t, resp, err, func() { - object = resp.Payload - }) - - return object -} - -func AssertGetObjectFailsEventually(t *testing.T, class string, uuid strfmt.UUID) error { - var err error - - checkThunk := func() interface{} { - _, err = Client(t).Objects.ObjectsClassGet(objects.NewObjectsClassGetParams().WithClassName(class).WithID(uuid), nil) - return err != nil - } - - AssertEventuallyEqual(t, true, checkThunk) - - return err -} - -func AssertCreateObjectClass(t *testing.T, class *models.Class) { - t.Helper() - params := schema.NewSchemaObjectsCreateParams().WithObjectClass(class) - resp, err := Client(t).Schema.SchemaObjectsCreate(params, nil) - AssertRequestOk(t, resp, err, nil) -} - -func AssertDeleteObjectClass(t *testing.T, class string) { - delRes, err := DeleteClassObject(t, class) - AssertRequestOk(t, delRes, err, nil) -} - -func GetObject(t *testing.T, class string, uuid strfmt.UUID, include ...string) (*models.Object, error) { - req := objects.NewObjectsClassGetParams().WithID(uuid) - if class != "" { - req.WithClassName(class) - } - if len(include) > 0 { - req.WithInclude(&include[0]) - } - getResp, err := Client(t).Objects.ObjectsClassGet(req, nil) - if err != nil { - return nil, err - } - return getResp.Payload, nil -} - -func TenantObject(t *testing.T, class string, id strfmt.UUID, tenant string) (*models.Object, error) { - req := objects.NewObjectsClassGetParams(). - WithClassName(class).WithID(id).WithTenant(&tenant) - getResp, err := Client(t).Objects.ObjectsClassGet(req, nil) - if err != nil { - return nil, err - } - return getResp.Payload, nil -} - -func TenantObjectWithInclude(t *testing.T, class string, id strfmt.UUID, tenant string, includes string) (*models.Object, error) { - req := objects.NewObjectsClassGetParams(). - WithClassName(class).WithID(id).WithTenant(&tenant).WithInclude(&includes) - getResp, err := Client(t).Objects.ObjectsClassGet(req, nil) - if err != nil { - return nil, err - } - return getResp.Payload, nil -} - -func GetObjectCL(t *testing.T, class string, uuid strfmt.UUID, - cl replica.ConsistencyLevel, include ...string, -) (*models.Object, error) { - req := objects.NewObjectsClassGetParams().WithID(uuid) - if class != "" { - req.WithClassName(class) - } - if len(include) > 0 { - req.WithInclude(&include[0]) - } - cls := string(cl) - req.ConsistencyLevel = &cls - getResp, err := Client(t).Objects.ObjectsClassGet(req, nil) - if err != nil { - return nil, err - } - return getResp.Payload, nil -} - -func ObjectExistsCL(t *testing.T, class string, id strfmt.UUID, cl replica.ConsistencyLevel) (bool, error) { - cls := string(cl) - req := objects.NewObjectsClassHeadParams(). - WithClassName(class).WithID(id).WithConsistencyLevel(&cls) - resp, err := Client(t).Objects.ObjectsClassHead(req, nil) - if err != nil { - return false, err - } - return resp.IsCode(http.StatusNoContent), nil -} - -func TenantObjectExists(t *testing.T, class string, id strfmt.UUID, tenant string) (bool, error) { - req := objects.NewObjectsClassHeadParams(). - WithClassName(class).WithID(id).WithTenant(&tenant) - resp, err := Client(t).Objects.ObjectsClassHead(req, nil) - if err != nil { - return false, err - } - return resp.IsCode(http.StatusNoContent), nil -} - -func GetObjectFromNode(t *testing.T, class string, uuid strfmt.UUID, nodename string) (*models.Object, error) { - req := objects.NewObjectsClassGetParams().WithID(uuid) - if class != "" { - req.WithClassName(class) - } - if nodename != "" { - req.WithNodeName(&nodename) - } - getResp, err := Client(t).Objects.ObjectsClassGet(req, nil) - if err != nil { - return nil, err - } - return getResp.Payload, nil -} - -func GetTenantObjectFromNode(t *testing.T, class string, uuid strfmt.UUID, nodename, tenant string) (*models.Object, error) { - req := objects.NewObjectsClassGetParams().WithID(uuid). - WithClassName(class). - WithNodeName(&nodename). - WithTenant(&tenant) - getResp, err := Client(t).Objects.ObjectsClassGet(req, nil) - if err != nil { - return nil, err - } - return getResp.Payload, nil -} - -func DeleteClassObject(t *testing.T, class string) (*schema.SchemaObjectsDeleteOK, error) { - delParams := schema.NewSchemaObjectsDeleteParams().WithClassName(class) - return Client(t).Schema.SchemaObjectsDelete(delParams, nil) -} - -func DeleteTenantObject(t *testing.T, class string, id strfmt.UUID, tenant string) { - params := objects.NewObjectsClassDeleteParams(). - WithClassName(class).WithID(id).WithTenant(&tenant) - resp, err := Client(t).Objects.ObjectsClassDelete(params, nil) - AssertRequestOk(t, resp, err, nil) -} - -func ListObjects(t *testing.T, class string) (*models.ObjectsListResponse, error) { - params := objects.NewObjectsListParams() - if class != "" { - params.WithClass(&class) - } - - resp, err := Client(t).Objects.ObjectsList(params, nil) - if err != nil { - return nil, err - } - return resp.Payload, nil -} - -func TenantListObjects(t *testing.T, class string, tenant string) (*models.ObjectsListResponse, error) { - params := objects.NewObjectsListParams().WithTenant(&tenant) - if class != "" { - params.WithClass(&class) - } - - resp, err := Client(t).Objects.ObjectsList(params, nil) - if err != nil { - return nil, err - } - return resp.Payload, nil -} diff --git a/test/helper/operations_client.go b/test/helper/operations_client.go deleted file mode 100644 index 1d04383da8880c7d5b0400f0be63fa81e092eb42..0000000000000000000000000000000000000000 --- a/test/helper/operations_client.go +++ /dev/null @@ -1,72 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -// This file contains the Client(t *testing.T) function, that can be used to construct a client that talks to -// the Weaviate server that is configured using command line arguments (see init.go). -// -// We pass in the test (*testing.T), to be able to log HTTP traffic to that specific test case. -// This allows us to get detailed logs of the performed HTTP requests if a acceptance test fails. - -// The CreateAuth returns a function that attaches the key and token headers to each HTTP call. - -// Example: -// func TestSomething(t *testing.T) { -// // Use specific key & token -// auth := helper.CreateAuth(key, token) -// helper.Client(t).SomeScope.SomeOperation(&someParams, auth) -// -// // Use root key & token -// helper.Client(t).SomeScope.SomeOperation(&someParams, helper.RootAuth) -// } - -package helper - -import ( - "fmt" - "testing" - - httptransport "github.com/go-openapi/runtime/client" - "github.com/go-openapi/strfmt" - "github.com/weaviate/weaviate/client/batch" - operations_apiclient "github.com/weaviate/weaviate/client/operations" -) - -// Create a client that logs with t.Logf, if a *testing.T is provided. -// If there is no test case at hand, pass in nil to disable logging. -func OperationsClient(t *testing.T) operations_apiclient.ClientService { - transport := httptransport.New(fmt.Sprintf("%s:%s", ServerHost, ServerPort), "/v1", []string{ServerScheme}) - - // If a test case is provided, and we want to dump HTTP traffic, - // create a simple logger that logs HTTP traffic to the test case. - if t != nil && DebugHTTP { - transport.SetDebug(true) - transport.SetLogger(&testLogger{t: t}) - } - - client := operations_apiclient.New(transport, strfmt.Default) - return client -} - -// Create a client that logs with t.Logf, if a *testing.T is provided. -// If there is no test case at hand, pass in nil to disable logging. -func BatchClient(t *testing.T) batch.ClientService { - transport := httptransport.New(fmt.Sprintf("%s:%s", ServerHost, ServerPort), "/v1", []string{ServerScheme}) - - // If a test case is provided, and we want to dump HTTP traffic, - // create a simple logger that logs HTTP traffic to the test case. - if t != nil && DebugHTTP { - transport.SetDebug(true) - transport.SetLogger(&testLogger{t: t}) - } - - client := batch.New(transport, strfmt.Default) - return client -} diff --git a/test/helper/sample-schema/articles/articles.go b/test/helper/sample-schema/articles/articles.go deleted file mode 100644 index 9f9580e60365ca3df707a4955d0915d013aa68df..0000000000000000000000000000000000000000 --- a/test/helper/sample-schema/articles/articles.go +++ /dev/null @@ -1,122 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package articles - -import ( - "github.com/go-openapi/strfmt" - "github.com/google/uuid" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" -) - -func ArticlesClass() *models.Class { - return &models.Class{ - Class: "Article", - Properties: []*models.Property{ - { - Name: "title", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - { - Name: "hasParagraphs", - DataType: []string{"Paragraph"}, - }, - }, - } -} - -func ParagraphsClass() *models.Class { - return &models.Class{ - Class: "Paragraph", - Properties: []*models.Property{ - { - Name: "contents", - DataType: schema.DataTypeText.PropString(), - }, - }, - Vectorizer: "none", - } -} - -type Article models.Object - -func (a *Article) WithID(id strfmt.UUID) *Article { - a.ID = id - return a -} - -func (a *Article) WithReferences(refs ...*models.SingleRef) *Article { - props := a.Properties.(map[string]interface{}) - props["hasParagraphs"] = models.MultipleRef(refs) - return a -} - -func (a *Article) WithTitle(title string) *Article { - props := a.Properties.(map[string]interface{}) - props["title"] = title - return a -} - -func (a *Article) WithTenant(tk string) *Article { - a.Tenant = tk - return a -} - -func (a *Article) Object() *models.Object { - obj := models.Object(*a) - return &obj -} - -func NewArticle() *Article { - return &Article{ - Class: "Article", - ID: strfmt.UUID(uuid.NewString()), - Properties: make(map[string]interface{}), - } -} - -type Paragraph models.Object - -func (p *Paragraph) WithID(id strfmt.UUID) *Paragraph { - p.ID = id - return p -} - -func (p *Paragraph) WithContents(contents string) *Paragraph { - props := p.Properties.(map[string]interface{}) - props["contents"] = contents - return p -} - -func (p *Paragraph) WithVector(vec []float32) *Paragraph { - p.Vector = vec - return p -} - -func (p *Paragraph) WithTenant(tk string) *Paragraph { - p.Tenant = tk - return p -} - -func (p *Paragraph) Object() *models.Object { - obj := models.Object(*p) - return &obj -} - -func NewParagraph() *Paragraph { - return &Paragraph{ - Class: "Paragraph", - ID: strfmt.UUID(uuid.NewString()), - Properties: make(map[string]interface{}), - } -} diff --git a/test/helper/sample-schema/books/books.go b/test/helper/sample-schema/books/books.go deleted file mode 100644 index ecf5003c7fea27210546389812b5ead348669228..0000000000000000000000000000000000000000 --- a/test/helper/sample-schema/books/books.go +++ /dev/null @@ -1,271 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package books - -import ( - "github.com/go-openapi/strfmt" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - pb "github.com/weaviate/weaviate/grpc/generated/protocol/v1" - "google.golang.org/protobuf/types/known/structpb" -) - -const defaultClassName = "Books" - -const ( - Dune strfmt.UUID = "67b79643-cf8b-4b22-b206-6e63dbb4e000" - ProjectHailMary strfmt.UUID = "67b79643-cf8b-4b22-b206-6e63dbb4e001" - TheLordOfTheIceGarden strfmt.UUID = "67b79643-cf8b-4b22-b206-6e63dbb4e002" -) - -func ClassContextionaryVectorizer() *models.Class { - return class(defaultClassName, "text2vec-contextionary") -} - -func ClassContextionaryVectorizerWithName(className string) *models.Class { - return class(className, "text2vec-contextionary") -} - -func ClassContextionaryVectorizerWithSumTransformers() *models.Class { - return class(defaultClassName, "text2vec-contextionary", "sum-transformers") -} - -func ClassContextionaryVectorizerWithQnATransformers() *models.Class { - return class(defaultClassName, "text2vec-contextionary", "qna-transformers") -} - -func ClassTransformersVectorizer() *models.Class { - return class(defaultClassName, "text2vec-transformers") -} - -func ClassTransformersVectorizerWithName(className string) *models.Class { - return class(className, "text2vec-transformers") -} - -func ClassTransformersVectorizerWithQnATransformersWithName(className string) *models.Class { - return class(className, "text2vec-transformers", "qna-transformers") -} - -func ClassCLIPVectorizer() *models.Class { - c := class(defaultClassName, "multi2vec-clip") - c.ModuleConfig.(map[string]interface{})["multi2vec-clip"] = map[string]interface{}{ - "textFields": []string{"title", "tags", "description"}, - } - return c -} - -func class(className, vectorizer string, additionalModules ...string) *models.Class { - moduleConfig := map[string]interface{}{ - vectorizer: map[string]interface{}{ - "vectorizeClassName": true, - }, - } - if len(additionalModules) > 0 { - for _, module := range additionalModules { - moduleConfig[module] = map[string]interface{}{} - } - } - return &models.Class{ - Class: className, - Vectorizer: vectorizer, - ModuleConfig: moduleConfig, - InvertedIndexConfig: &models.InvertedIndexConfig{ - IndexNullState: true, - IndexTimestamps: true, - IndexPropertyLength: true, - }, - Properties: []*models.Property{ - { - Name: "title", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - ModuleConfig: map[string]interface{}{vectorizer: map[string]interface{}{"skip": false}}, - }, - { - Name: "tags", - DataType: schema.DataTypeTextArray.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - ModuleConfig: map[string]interface{}{vectorizer: map[string]interface{}{"skip": false}}, - }, - { - Name: "description", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - ModuleConfig: map[string]interface{}{vectorizer: map[string]interface{}{"skip": false}}, - }, - { - Name: "meta", DataType: schema.DataTypeObject.PropString(), - NestedProperties: []*models.NestedProperty{ - {Name: "isbn", DataType: schema.DataTypeText.PropString()}, - { - Name: "obj", DataType: schema.DataTypeObject.PropString(), - NestedProperties: []*models.NestedProperty{{Name: "text", DataType: schema.DataTypeText.PropString()}}, - }, - { - Name: "objs", DataType: schema.DataTypeObjectArray.PropString(), - NestedProperties: []*models.NestedProperty{{Name: "text", DataType: schema.DataTypeText.PropString()}}, - }, - }, - }, - { - Name: "reviews", DataType: schema.DataTypeObjectArray.PropString(), - NestedProperties: []*models.NestedProperty{{Name: "tags", DataType: schema.DataTypeTextArray.PropString()}}, - }, - }, - } -} - -func Objects() []*models.Object { - return objects(defaultClassName) -} - -func BatchObjects() []*pb.BatchObject { - return batchObjects(defaultClassName) -} - -func ObjectsWithName(className string) []*models.Object { - return objects(className) -} - -func objects(className string) []*models.Object { - return []*models.Object{ - { - Class: className, - ID: Dune, - Properties: map[string]interface{}{ - "title": "Dune", - "description": "Dune is a 1965 epic science fiction novel by American author Frank Herbert.", - }, - }, - { - Class: className, - ID: ProjectHailMary, - Properties: map[string]interface{}{ - "title": "Project Hail Mary", - "description": "Project Hail Mary is a 2021 science fiction novel by American novelist Andy Weir.", - }, - }, - { - Class: className, - ID: TheLordOfTheIceGarden, - Properties: map[string]interface{}{ - "title": "The Lord of the Ice Garden", - "tags": []string{"three", "three", "three"}, - "description": "The Lord of the Ice Garden (Polish: Pan Lodowego Ogrodu) is a four-volume science fiction and fantasy novel by Polish writer Jaroslaw Grzedowicz.", - }, - }, - } -} - -func batchObjects(className string) []*pb.BatchObject { - scifi := "sci-fi" - return []*pb.BatchObject{ - { - Collection: className, - Uuid: Dune.String(), - Properties: &pb.BatchObject_Properties{ - NonRefProperties: &structpb.Struct{ - Fields: map[string]*structpb.Value{ - "title": structpb.NewStringValue("Dune"), - "description": structpb.NewStringValue("Dune is a 1965 epic science fiction novel by American author Frank Herbert."), - }, - }, - ObjectProperties: []*pb.ObjectProperties{{ - PropName: "meta", - Value: &pb.ObjectPropertiesValue{ - NonRefProperties: &structpb.Struct{Fields: map[string]*structpb.Value{"isbn": structpb.NewStringValue("978-0593099322")}}, - ObjectProperties: []*pb.ObjectProperties{{ - PropName: "obj", - Value: &pb.ObjectPropertiesValue{ - NonRefProperties: &structpb.Struct{Fields: map[string]*structpb.Value{"text": structpb.NewStringValue("some text")}}, - }, - }}, - ObjectArrayProperties: []*pb.ObjectArrayProperties{{ - PropName: "objs", - Values: []*pb.ObjectPropertiesValue{{ - NonRefProperties: &structpb.Struct{Fields: map[string]*structpb.Value{"text": structpb.NewStringValue("some text")}}, - }}, - }}, - }, - }}, - ObjectArrayProperties: []*pb.ObjectArrayProperties{{ - PropName: "reviews", - Values: []*pb.ObjectPropertiesValue{{TextArrayProperties: []*pb.TextArrayProperties{{PropName: "tags", Values: []string{scifi, "epic"}}}}}, - }}, - }, - }, - { - Collection: className, - Uuid: ProjectHailMary.String(), - Properties: &pb.BatchObject_Properties{ - NonRefProperties: &structpb.Struct{ - Fields: map[string]*structpb.Value{ - "title": structpb.NewStringValue("Project Hail Mary"), - "description": structpb.NewStringValue("Project Hail Mary is a 2021 science fiction novel by American novelist Andy Weir."), - }, - }, - ObjectProperties: []*pb.ObjectProperties{{ - PropName: "meta", - Value: &pb.ObjectPropertiesValue{ - NonRefProperties: &structpb.Struct{Fields: map[string]*structpb.Value{"isbn": structpb.NewStringValue("978-0593135204")}}, - ObjectProperties: []*pb.ObjectProperties{{ - PropName: "obj", - Value: &pb.ObjectPropertiesValue{NonRefProperties: &structpb.Struct{Fields: map[string]*structpb.Value{"text": structpb.NewStringValue("some text")}}}, - }}, - ObjectArrayProperties: []*pb.ObjectArrayProperties{{ - PropName: "objs", - Values: []*pb.ObjectPropertiesValue{{ - NonRefProperties: &structpb.Struct{Fields: map[string]*structpb.Value{"text": structpb.NewStringValue("some text")}}, - }}, - }}, - }, - }}, - ObjectArrayProperties: []*pb.ObjectArrayProperties{{ - PropName: "reviews", - Values: []*pb.ObjectPropertiesValue{{TextArrayProperties: []*pb.TextArrayProperties{{PropName: "tags", Values: []string{scifi}}}}}, - }}, - }, - }, - { - Collection: className, - Uuid: TheLordOfTheIceGarden.String(), - Properties: &pb.BatchObject_Properties{ - NonRefProperties: &structpb.Struct{ - Fields: map[string]*structpb.Value{ - "title": structpb.NewStringValue("The Lord of the Ice Garden"), - "description": structpb.NewStringValue("The Lord of the Ice Garden (Polish: Pan Lodowego Ogrodu) is a four-volume science fiction and fantasy novel by Polish writer Jaroslaw Grzedowicz."), - }, - }, - ObjectProperties: []*pb.ObjectProperties{{ - PropName: "meta", - Value: &pb.ObjectPropertiesValue{ - NonRefProperties: &structpb.Struct{Fields: map[string]*structpb.Value{"isbn": structpb.NewStringValue("978-8374812962")}}, - ObjectProperties: []*pb.ObjectProperties{{ - PropName: "obj", - Value: &pb.ObjectPropertiesValue{NonRefProperties: &structpb.Struct{Fields: map[string]*structpb.Value{"text": structpb.NewStringValue("some text")}}}, - }}, - ObjectArrayProperties: []*pb.ObjectArrayProperties{{ - PropName: "objs", - Values: []*pb.ObjectPropertiesValue{{ - NonRefProperties: &structpb.Struct{Fields: map[string]*structpb.Value{"text": structpb.NewStringValue("some text")}}, - }}, - }}, - }, - }}, - ObjectArrayProperties: []*pb.ObjectArrayProperties{{ - PropName: "reviews", - Values: []*pb.ObjectPropertiesValue{{TextArrayProperties: []*pb.TextArrayProperties{{PropName: "tags", Values: []string{scifi, "fantasy"}}}}}, - }}, - }, - }, - } -} diff --git a/test/helper/sample-schema/documents/documents.go b/test/helper/sample-schema/documents/documents.go deleted file mode 100644 index 6482441c3dfa8cb25f45e2cd4c4330868a4fb852..0000000000000000000000000000000000000000 --- a/test/helper/sample-schema/documents/documents.go +++ /dev/null @@ -1,193 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package documents - -import ( - "fmt" - - "github.com/go-openapi/strfmt" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema/crossref" -) - -const ( - Document = "Document" - Passage = "Passage" -) - -var DocumentIDs = []strfmt.UUID{ - "00000000-0000-0000-0000-000000000011", - "00000000-0000-0000-0000-000000000012", -} - -var PassageIDs = []strfmt.UUID{ - "00000000-0000-0000-0000-000000000001", - "00000000-0000-0000-0000-000000000002", - "00000000-0000-0000-0000-000000000003", - "00000000-0000-0000-0000-000000000004", - "00000000-0000-0000-0000-000000000005", - "00000000-0000-0000-0000-000000000006", - "00000000-0000-0000-0000-000000000007", - "00000000-0000-0000-0000-000000000008", -} - -var multiShardConfig = map[string]interface{}{ - "actualCount": float64(4), - "actualVirtualCount": float64(128), - "desiredCount": float64(4), - "desiredVirtualCount": float64(128), - "function": "murmur3", - "key": "_id", - "strategy": "hash", - "virtualPerPhysical": float64(128), -} - -var docTypes = []string{"private", "public"} - -func ClassesContextionaryVectorizer(multishard bool) []*models.Class { - return []*models.Class{ - document("text2vec-contextionary", multishard), - passage("text2vec-contextionary", multishard), - } -} - -func document(vectorizer string, multishard bool) *models.Class { - class := &models.Class{ - Class: Document, - ModuleConfig: map[string]interface{}{ - vectorizer: map[string]interface{}{ - "vectorizeClassName": false, - }, - }, - Properties: []*models.Property{ - { - Name: "title", - DataType: []string{"text"}, - ModuleConfig: map[string]interface{}{ - vectorizer: map[string]interface{}{ - "vectorizePropertyName": false, - }, - }, - }, - { - Name: "pageCount", - DataType: []string{"int"}, - ModuleConfig: map[string]interface{}{ - vectorizer: map[string]interface{}{ - "vectorizePropertyName": false, - "skip": true, - }, - }, - }, - { - Name: "docType", - DataType: []string{"text"}, - ModuleConfig: map[string]interface{}{ - vectorizer: map[string]interface{}{ - "vectorizePropertyName": false, - "skip": true, - }, - }, - }, - }, - } - if multishard { - class.ShardingConfig = multiShardConfig - } - return class -} - -func passage(vectorizer string, multishard bool) *models.Class { - class := &models.Class{ - Class: Passage, - ModuleConfig: map[string]interface{}{ - vectorizer: map[string]interface{}{ - "vectorizeClassName": false, - }, - }, - Properties: []*models.Property{ - { - Name: "content", - DataType: []string{"text"}, - ModuleConfig: map[string]interface{}{ - vectorizer: map[string]interface{}{ - "vectorizePropertyName": false, - }, - }, - }, - { - Name: "ofDocument", - DataType: []string{Document}, - ModuleConfig: map[string]interface{}{ - vectorizer: map[string]interface{}{ - "vectorizePropertyName": false, - }, - }, - }, - }, - } - if multishard { - class.ShardingConfig = multiShardConfig - } - return class -} - -func Objects() []*models.Object { - return append(documentObjects(), passageObjects()...) -} - -func documentObjects() []*models.Object { - getDocType := func(i int) string { - if i == 0 { - return docTypes[0] - } - return docTypes[1] - } - objects := make([]*models.Object, len(DocumentIDs)) - for i, id := range DocumentIDs { - objects[i] = &models.Object{ - Class: Document, - ID: id, - Properties: map[string]interface{}{ - "title": fmt.Sprintf("Document title %v", i), - "pageCount": 100 + i, - "docType": getDocType(i), - }, - } - } - return objects -} - -func passageObjects() []*models.Object { - getDocumentID := func(i int) strfmt.UUID { - if i < 6 { - return DocumentIDs[0] - } - return DocumentIDs[1] - } - objects := make([]*models.Object, len(PassageIDs)) - for i, id := range PassageIDs { - objects[i] = &models.Object{ - Class: Passage, - ID: id, - Properties: map[string]interface{}{ - "content": fmt.Sprintf("Content of Passage %v", i), - "ofDocument": []interface{}{ - map[string]interface{}{ - "beacon": crossref.NewLocalhost("Document", getDocumentID(i)).String(), - }, - }, - }, - } - } - return objects -} diff --git a/test/helper/sample-schema/multishard/multishard.go b/test/helper/sample-schema/multishard/multishard.go deleted file mode 100644 index 2b88402c549f68a46ff7726802d02f8a7fc80da8..0000000000000000000000000000000000000000 --- a/test/helper/sample-schema/multishard/multishard.go +++ /dev/null @@ -1,91 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package multishard - -import ( - "github.com/go-openapi/strfmt" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" -) - -const ( - MultiShardID1 strfmt.UUID = "aa44bbee-ca5f-4db7-a412-5fc6a23c534a" - MultiShardID2 strfmt.UUID = "aa44bbee-ca5f-4db7-a412-5fc6a23c534b" - MultiShardID3 strfmt.UUID = "aa44bbee-ca5f-4db7-a412-5fc6a23c534c" -) - -func ClassContextionaryVectorizer() *models.Class { - return class("text2vec-contextionary") -} - -func ClassTransformersVectorizer() *models.Class { - return class("text2vec-transformers") -} - -func class(vectorizer string) *models.Class { - return &models.Class{ - Class: "MultiShard", - ModuleConfig: map[string]interface{}{ - vectorizer: map[string]interface{}{ - "vectorizeClassName": false, - }, - }, - Properties: []*models.Property{ - { - Name: "name", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - ModuleConfig: map[string]interface{}{ - vectorizer: map[string]interface{}{ - "vectorizePropertyName": false, - }, - }, - }, - }, - ShardingConfig: map[string]interface{}{ - "actualCount": float64(2), - "actualVirtualCount": float64(128), - "desiredCount": float64(2), - "desiredVirtualCount": float64(128), - "function": "murmur3", - "key": "_id", - "strategy": "hash", - "virtualPerPhysical": float64(128), - }, - } -} - -func Objects() []*models.Object { - return []*models.Object{ - { - Class: "MultiShard", - ID: MultiShardID1, - Properties: map[string]interface{}{ - "name": "multi shard one", - }, - }, - { - Class: "MultiShard", - ID: MultiShardID2, - Properties: map[string]interface{}{ - "name": "multi shard two", - }, - }, - { - Class: "MultiShard", - ID: MultiShardID3, - Properties: map[string]interface{}{ - "name": "multi shard three", - }, - }, - } -} diff --git a/test/helper/test_data.go b/test/helper/test_data.go deleted file mode 100644 index 75863050a8181322731c0e62e113cc047cf4db9c..0000000000000000000000000000000000000000 --- a/test/helper/test_data.go +++ /dev/null @@ -1,33 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package helper - -import ( - "math/rand" - "time" -) - -// GetRandomString returns a string comprised of random -// samplings of charset, of length specified by caller -func GetRandomString(length int) string { - const charset = "abcdefghijklmnopqrstuvwxyz" + - "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!@#$%^&*" - - seededRand := rand.New(rand.NewSource(time.Now().UnixNano())) - - s := make([]byte, length) - for n := range s { - s[n] = charset[seededRand.Intn(len(charset))] - } - - return string(s) -} diff --git a/test/helper/uuid.go b/test/helper/uuid.go deleted file mode 100644 index d1d9ccecf81fe7498705e154cf7515e2a9c0356b..0000000000000000000000000000000000000000 --- a/test/helper/uuid.go +++ /dev/null @@ -1,31 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package helper - -import ( - "encoding/binary" - - "github.com/go-openapi/strfmt" - "github.com/google/uuid" -) - -// InToUUID takes an unsigned int64 and places it in BigEndian fashion into the -// upper 8 bytes of a 16 byte UUID. This makes it easy to produce easy-to-read -// UUIDs in test scenarios. For example: -// -// IntToUUID(1) -// // returns "00000000-0000-0000-0000-000000000001" -func IntToUUID(in uint64) strfmt.UUID { - id := [16]byte{} - binary.BigEndian.PutUint64(id[8:16], in) - return strfmt.UUID(uuid.UUID(id).String()) -} diff --git a/test/integration/run.sh b/test/integration/run.sh deleted file mode 100644 index c534cdafbbd8b03100ead0e361f8dd9579075f86..0000000000000000000000000000000000000000 --- a/test/integration/run.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash - -set -e - -function echo_yellow() { - yellow='\033[0;33m' - nc='\033[0m' - echo -e "${yellow}${*}${nc}" -} - -includeslow=false - -for arg in "$@"; do - if [[ $arg == --include-slow ]]; then - includeslow=true - shift - fi -done - -tags=integrationTest -if [ $includeslow = true ]; then - echo "Found --include-slow flag, running all tests, including the slow ones" - tags="$tags,integrationTestSlow" -else - echo "Found no --include-slow flag, skipping the slow ones" -fi - -echo_yellow "Run the regular integration tests with race detector ON" -go test -count 1 -timeout 3000s -coverpkg=./adapters/repos/... -coverprofile=coverage-integration.txt -race -tags=$tags "$@" ./adapters/repos/... -echo_yellow "Run the !race integration tests with race detector OFF" -go test -count 1 -coverpkg=./adapters/repos/... -tags=$tags "$@" -run Test_NoRace ./adapters/repos/... -echo_yellow "Run the classification integration tests with race detector ON" -go test -count 1 -race -tags=$tags "$@" ./usecases/classification/... diff --git a/test/modules/backup-azure/backup_backend_test.go b/test/modules/backup-azure/backup_backend_test.go deleted file mode 100644 index 3ce952b4d136418d8f6be8321d1da977074a8c52..0000000000000000000000000000000000000000 --- a/test/modules/backup-azure/backup_backend_test.go +++ /dev/null @@ -1,268 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "context" - "encoding/json" - "fmt" - "os" - "path/filepath" - "testing" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - logrustest "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/backup" - "github.com/weaviate/weaviate/entities/moduletools" - mod "github.com/weaviate/weaviate/modules/backup-azure" - "github.com/weaviate/weaviate/test/docker" - moduleshelper "github.com/weaviate/weaviate/test/helper/modules" - ubak "github.com/weaviate/weaviate/usecases/backup" - "github.com/weaviate/weaviate/usecases/config" -) - -func Test_AzureBackend_Backup(t *testing.T) { - ctx := context.Background() - compose, err := docker.New().WithAzurite().Start(ctx) - if err != nil { - t.Fatal(errors.Wrapf(err, "cannot start")) - } - - t.Setenv(envAzureEndpoint, compose.GetAzurite().URI()) - - t.Run("store backup meta", moduleLevelStoreBackupMeta) - t.Run("copy objects", moduleLevelCopyObjects) - t.Run("copy files", moduleLevelCopyFiles) - - if err := compose.Terminate(ctx); err != nil { - t.Fatal(errors.Wrapf(err, "failed to terminate test containers")) - } -} - -func moduleLevelStoreBackupMeta(t *testing.T) { - testCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - dataDir := t.TempDir() - className := "BackupClass" - backupID := "backup_id" - containerName := "container" - endpoint := os.Getenv(envAzureEndpoint) - metadataFilename := "backup.json" - - t.Log("setup env") - t.Setenv(envAzureEndpoint, endpoint) - t.Setenv(envAzureStorageConnectionString, fmt.Sprintf(connectionString, endpoint)) - t.Setenv(envAzureContainer, containerName) - moduleshelper.CreateAzureContainer(testCtx, t, endpoint, containerName) - defer moduleshelper.DeleteAzureContainer(testCtx, t, endpoint, containerName) - - t.Run("store backup meta in Azure", func(t *testing.T) { - t.Setenv(envAzureContainer, containerName) - azure := mod.New() - err := azure.Init(testCtx, newFakeModuleParams(dataDir)) - require.Nil(t, err) - - t.Run("access permissions", func(t *testing.T) { - err := azure.Initialize(testCtx, backupID) - assert.Nil(t, err) - }) - - t.Run("backup meta does not exist yet", func(t *testing.T) { - meta, err := azure.GetObject(testCtx, backupID, metadataFilename) - assert.Nil(t, meta) - assert.NotNil(t, err) - assert.IsType(t, backup.ErrNotFound{}, err) - }) - - t.Run("put backup meta on backend", func(t *testing.T) { - desc := &backup.BackupDescriptor{ - StartedAt: time.Now(), - CompletedAt: time.Time{}, - ID: backupID, - Classes: []backup.ClassDescriptor{ - { - Name: className, - }, - }, - Status: string(backup.Started), - Version: ubak.Version, - } - - b, err := json.Marshal(desc) - require.Nil(t, err) - - err = azure.PutObject(testCtx, backupID, metadataFilename, b) - require.Nil(t, err) - - dest := azure.HomeDir(backupID) - - expected := fmt.Sprintf("http://%s/devstoreaccount1/%s/%s", os.Getenv(envAzureEndpoint), containerName, backupID) - assert.Equal(t, expected, dest) - }) - - t.Run("assert backup meta contents", func(t *testing.T) { - obj, err := azure.GetObject(testCtx, backupID, metadataFilename) - require.Nil(t, err) - - var meta backup.BackupDescriptor - err = json.Unmarshal(obj, &meta) - require.Nil(t, err) - assert.NotEmpty(t, meta.StartedAt) - assert.Empty(t, meta.CompletedAt) - assert.Equal(t, meta.Status, string(backup.Started)) - assert.Empty(t, meta.Error) - assert.Len(t, meta.Classes, 1) - assert.Equal(t, meta.Classes[0].Name, className) - assert.Equal(t, meta.Version, ubak.Version) - assert.Nil(t, meta.Classes[0].Error) - }) - }) -} - -func moduleLevelCopyObjects(t *testing.T) { - testCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - dataDir := t.TempDir() - key := "moduleLevelCopyObjects" - backupID := "backup_id" - containerName := "container" - endpoint := os.Getenv(envAzureEndpoint) - - t.Log("setup env") - t.Setenv(envAzureEndpoint, endpoint) - t.Setenv(envAzureStorageConnectionString, fmt.Sprintf(connectionString, endpoint)) - t.Setenv(envAzureContainer, containerName) - moduleshelper.CreateAzureContainer(testCtx, t, endpoint, containerName) - defer moduleshelper.DeleteAzureContainer(testCtx, t, endpoint, containerName) - - t.Run("copy objects", func(t *testing.T) { - t.Setenv(envAzureContainer, containerName) - azure := mod.New() - err := azure.Init(testCtx, newFakeModuleParams(dataDir)) - require.Nil(t, err) - - t.Run("put object to bucket", func(t *testing.T) { - err := azure.PutObject(testCtx, backupID, key, []byte("hello")) - assert.Nil(t, err) - }) - - t.Run("get object from bucket", func(t *testing.T) { - meta, err := azure.GetObject(testCtx, backupID, key) - assert.Nil(t, err) - assert.Equal(t, []byte("hello"), meta) - }) - }) -} - -func moduleLevelCopyFiles(t *testing.T) { - testCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - dataDir := t.TempDir() - key := "moduleLevelCopyFiles" - backupID := "backup_id" - containerName := "container" - endpoint := os.Getenv(envAzureEndpoint) - - t.Log("setup env") - t.Setenv(envAzureEndpoint, endpoint) - t.Setenv(envAzureStorageConnectionString, fmt.Sprintf(connectionString, endpoint)) - t.Setenv(envAzureContainer, containerName) - moduleshelper.CreateAzureContainer(testCtx, t, endpoint, containerName) - defer moduleshelper.DeleteAzureContainer(testCtx, t, endpoint, containerName) - - t.Run("copy files", func(t *testing.T) { - fpaths := moduleshelper.CreateTestFiles(t, dataDir) - fpath := fpaths[0] - expectedContents, err := os.ReadFile(fpath) - require.Nil(t, err) - require.NotNil(t, expectedContents) - - t.Setenv(envAzureContainer, containerName) - azure := mod.New() - err = azure.Init(testCtx, newFakeModuleParams(dataDir)) - require.Nil(t, err) - - t.Run("verify source data path", func(t *testing.T) { - assert.Equal(t, dataDir, azure.SourceDataPath()) - }) - - t.Run("copy file to backend", func(t *testing.T) { - srcPath, _ := filepath.Rel(dataDir, fpath) - err := azure.PutFile(testCtx, backupID, key, srcPath) - require.Nil(t, err) - - contents, err := azure.GetObject(testCtx, backupID, key) - require.Nil(t, err) - assert.Equal(t, expectedContents, contents) - }) - - t.Run("fetch file from backend", func(t *testing.T) { - destPath := dataDir + "/file_0.copy.db" - - err := azure.WriteToFile(testCtx, backupID, key, destPath) - require.Nil(t, err) - - contents, err := os.ReadFile(destPath) - require.Nil(t, err) - assert.Equal(t, expectedContents, contents) - }) - }) -} - -type fakeModuleParams struct { - logger logrus.FieldLogger - provider fakeStorageProvider - config config.Config -} - -func newFakeModuleParams(dataPath string) *fakeModuleParams { - logger, _ := logrustest.NewNullLogger() - return &fakeModuleParams{ - logger: logger, - provider: fakeStorageProvider{dataPath: dataPath}, - } -} - -func (f *fakeModuleParams) GetStorageProvider() moduletools.StorageProvider { - return &f.provider -} - -func (f *fakeModuleParams) GetAppState() interface{} { - return nil -} - -func (f *fakeModuleParams) GetLogger() logrus.FieldLogger { - return f.logger -} - -func (f *fakeModuleParams) GetConfig() config.Config { - return f.config -} - -type fakeStorageProvider struct { - dataPath string -} - -func (f *fakeStorageProvider) Storage(name string) (moduletools.Storage, error) { - return nil, nil -} - -func (f *fakeStorageProvider) DataPath() string { - return f.dataPath -} diff --git a/test/modules/backup-azure/backup_journey_test.go b/test/modules/backup-azure/backup_journey_test.go deleted file mode 100644 index 448c4e8e36de2e550b7c091c852d5f6268fde95d..0000000000000000000000000000000000000000 --- a/test/modules/backup-azure/backup_journey_test.go +++ /dev/null @@ -1,94 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "context" - "testing" - "time" - - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/test/docker" - "github.com/weaviate/weaviate/test/helper" - "github.com/weaviate/weaviate/test/helper/journey" - moduleshelper "github.com/weaviate/weaviate/test/helper/modules" -) - -const ( - envAzureEndpoint = "AZURE_ENDPOINT" - envAzureContainer = "BACKUP_AZURE_CONTAINER" - - envAzureStorageConnectionString = "AZURE_STORAGE_CONNECTION_STRING" - connectionString = "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://%s/devstoreaccount1;" - - azureBackupJourneyClassName = "AzureBackup" - azureBackupJourneyBackupIDSingleNode = "azure-backup-single-node" - azureBackupJourneyBackupIDCluster = "azure-backup-cluster" - azureBackupJourneyContainerName = "backups" -) - -func Test_BackupJourney(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) - defer cancel() - - t.Run("single node", func(t *testing.T) { - t.Log("pre-instance env setup") - t.Setenv(envAzureContainer, azureBackupJourneyContainerName) - - compose, err := docker.New(). - WithBackendAzure(azureBackupJourneyContainerName). - WithText2VecContextionary(). - WithWeaviate(). - Start(ctx) - require.Nil(t, err) - - t.Log("post-instance env setup") - azuriteEndpoint := compose.GetAzurite().URI() - t.Setenv(envAzureEndpoint, azuriteEndpoint) - moduleshelper.CreateAzureContainer(ctx, t, azuriteEndpoint, azureBackupJourneyContainerName) - // defer moduleshelper.DeleteAzureContainer(ctx, t, azuriteEndpoint, azureBackupJourneyContainerName) - helper.SetupClient(compose.GetWeaviate().URI()) - - t.Run("backup-azure", func(t *testing.T) { - journey.BackupJourneyTests_SingleNode(t, compose.GetWeaviate().URI(), - "azure", azureBackupJourneyClassName, azureBackupJourneyBackupIDSingleNode, nil) - }) - - require.Nil(t, compose.Terminate(ctx)) - }) - - t.Run("multiple node", func(t *testing.T) { - t.Log("pre-instance env setup") - t.Setenv(envAzureContainer, azureBackupJourneyContainerName) - - compose, err := docker.New(). - WithBackendAzure(azureBackupJourneyContainerName). - WithText2VecContextionary(). - WithWeaviateCluster(). - Start(ctx) - require.Nil(t, err) - - t.Log("post-instance env setup") - azuriteEndpoint := compose.GetAzurite().URI() - t.Setenv(envAzureEndpoint, azuriteEndpoint) - moduleshelper.CreateAzureContainer(ctx, t, azuriteEndpoint, azureBackupJourneyContainerName) - // defer moduleshelper.DeleteAzureContainer(ctx, t, azuriteEndpoint, azureBackupJourneyContainerName) - helper.SetupClient(compose.GetWeaviate().URI()) - - t.Run("backup-azure", func(t *testing.T) { - journey.BackupJourneyTests_Cluster(t, "azure", azureBackupJourneyClassName, - azureBackupJourneyBackupIDCluster, nil, compose.GetWeaviate().URI(), compose.GetWeaviateNode2().URI()) - }) - - require.Nil(t, compose.Terminate(ctx)) - }) -} diff --git a/test/modules/backup-azure/multi_tenant_backup_test.go b/test/modules/backup-azure/multi_tenant_backup_test.go deleted file mode 100644 index 46a31418c4fe2f0aede66b66c974648cdcf56c2e..0000000000000000000000000000000000000000 --- a/test/modules/backup-azure/multi_tenant_backup_test.go +++ /dev/null @@ -1,89 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/test/docker" - "github.com/weaviate/weaviate/test/helper" - "github.com/weaviate/weaviate/test/helper/journey" - moduleshelper "github.com/weaviate/weaviate/test/helper/modules" -) - -const ( - numTenants = 50 -) - -func Test_MultiTenantBackupJourney(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) - defer cancel() - - tenantNames := make([]string, numTenants) - for i := range tenantNames { - tenantNames[i] = fmt.Sprintf("Tenant%d", i) - } - - t.Run("single node", func(t *testing.T) { - t.Log("pre-instance env setup") - t.Setenv(envAzureContainer, azureBackupJourneyContainerName) - - compose, err := docker.New(). - WithBackendAzure(azureBackupJourneyContainerName). - WithText2VecContextionary(). - WithWeaviate(). - Start(ctx) - require.Nil(t, err) - - t.Log("post-instance env setup") - azuriteEndpoint := compose.GetAzurite().URI() - t.Setenv(envAzureEndpoint, azuriteEndpoint) - moduleshelper.CreateAzureContainer(ctx, t, azuriteEndpoint, azureBackupJourneyContainerName) - helper.SetupClient(compose.GetWeaviate().URI()) - - t.Run("backup-azure", func(t *testing.T) { - journey.BackupJourneyTests_SingleNode(t, compose.GetWeaviate().URI(), - "azure", azureBackupJourneyClassName, azureBackupJourneyBackupIDSingleNode, tenantNames) - }) - - require.Nil(t, compose.Terminate(ctx)) - }) - - t.Run("multiple node", func(t *testing.T) { - t.Log("pre-instance env setup") - t.Setenv(envAzureContainer, azureBackupJourneyContainerName) - - compose, err := docker.New(). - WithBackendAzure(azureBackupJourneyContainerName). - WithText2VecContextionary(). - WithWeaviateCluster(). - Start(ctx) - require.Nil(t, err) - - t.Log("post-instance env setup") - azuriteEndpoint := compose.GetAzurite().URI() - t.Setenv(envAzureEndpoint, azuriteEndpoint) - moduleshelper.CreateAzureContainer(ctx, t, azuriteEndpoint, azureBackupJourneyContainerName) - helper.SetupClient(compose.GetWeaviate().URI()) - - t.Run("backup-azure", func(t *testing.T) { - journey.BackupJourneyTests_Cluster(t, "azure", azureBackupJourneyClassName, - azureBackupJourneyBackupIDCluster, tenantNames, compose.GetWeaviate().URI(), compose.GetWeaviateNode2().URI()) - }) - - require.Nil(t, compose.Terminate(ctx)) - }) -} diff --git a/test/modules/backup-azure/node_mapping_backup_journey_test.go b/test/modules/backup-azure/node_mapping_backup_journey_test.go deleted file mode 100644 index bea8db0c24580a098d993e998b8bb65a9b1c73dc..0000000000000000000000000000000000000000 --- a/test/modules/backup-azure/node_mapping_backup_journey_test.go +++ /dev/null @@ -1,65 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "context" - "testing" - "time" - - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/test/docker" - "github.com/weaviate/weaviate/test/helper" - "github.com/weaviate/weaviate/test/helper/journey" - moduleshelper "github.com/weaviate/weaviate/test/helper/modules" -) - -func Test_NodeMappingBackupJourney(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) - defer cancel() - - t.Run("single node", func(t *testing.T) { - t.Log("pre-instance env setup") - t.Setenv(envAzureContainer, azureBackupJourneyContainerName) - - compose, err := docker.New(). - WithBackendAzure(azureBackupJourneyContainerName). - WithText2VecContextionary(). - WithWeaviate(). - WithSecondWeaviate(). - Start(ctx) - require.Nil(t, err) - - t.Log("post-instance env setup") - azuriteEndpoint := compose.GetAzurite().URI() - t.Setenv(envAzureEndpoint, azuriteEndpoint) - moduleshelper.CreateAzureContainer(ctx, t, azuriteEndpoint, azureBackupJourneyContainerName) - helper.SetupClient(compose.GetWeaviate().URI()) - - t.Run("backup-azure", func(t *testing.T) { - journey.NodeMappingBackupJourneyTests_SingleNode_Backup(t, compose.GetWeaviate().URI(), - "azure", azureBackupJourneyClassName, azureBackupJourneyBackupIDSingleNode, nil) - }) - - // Now change our tests to use the second cluster and trigger a backup restore - helper.SetupClient(compose.GetSecondWeaviate().URI()) - - t.Run("restore-azure", func(t *testing.T) { - journey.NodeMappingBackupJourneyTests_SingleNode_Restore(t, compose.GetSecondWeaviate().URI(), - "azure", azureBackupJourneyClassName, azureBackupJourneyBackupIDSingleNode, nil, map[string]string{ - docker.Weaviate: docker.SecondWeaviate, - }) - }) - - require.Nil(t, compose.Terminate(ctx)) - }) -} diff --git a/test/modules/backup-filesystem/backup_backend_test.go b/test/modules/backup-filesystem/backup_backend_test.go deleted file mode 100644 index 9eaf396c7a2d0a937e6f0c44049a29de56ba9ce6..0000000000000000000000000000000000000000 --- a/test/modules/backup-filesystem/backup_backend_test.go +++ /dev/null @@ -1,196 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "context" - "encoding/json" - "fmt" - "os" - "path/filepath" - "testing" - "time" - - "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/backup" - "github.com/weaviate/weaviate/entities/moduletools" - modstgfs "github.com/weaviate/weaviate/modules/backup-filesystem" - moduleshelper "github.com/weaviate/weaviate/test/helper/modules" - ubak "github.com/weaviate/weaviate/usecases/backup" - "github.com/weaviate/weaviate/usecases/config" -) - -func Test_FilesystemBackend_Backup(t *testing.T) { - t.Run("store backup meta", moduleLevelStoreBackupMeta) - t.Run("copy objects", moduleLevelCopyObjects) - t.Run("copy files", moduleLevelCopyFiles) -} - -func moduleLevelStoreBackupMeta(t *testing.T) { - testCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - dataDir := t.TempDir() - backupDir := t.TempDir() - className := "BackupClass" - backupID := "backup_id" - metadataFilename := "backup.json" - - t.Setenv("BACKUP_FILESYSTEM_PATH", backupDir) - - t.Run("store backup meta in fs", func(t *testing.T) { - logger, _ := test.NewNullLogger() - sp := fakeStorageProvider{dataDir} - params := moduletools.NewInitParams(sp, nil, config.Config{}, logger) - - fs := modstgfs.New() - err := fs.Init(testCtx, params) - require.Nil(t, err) - - t.Run("access permissions", func(t *testing.T) { - err := fs.Initialize(testCtx, backupID) - assert.Nil(t, err) - }) - - t.Run("backup meta does not exist yet", func(t *testing.T) { - meta, err := fs.GetObject(testCtx, backupID, metadataFilename) - assert.Nil(t, meta) - assert.NotNil(t, err) - assert.IsType(t, backup.ErrNotFound{}, err) - }) - - t.Run("put backup meta on backend", func(t *testing.T) { - desc := &backup.BackupDescriptor{ - StartedAt: time.Now(), - CompletedAt: time.Time{}, - ID: backupID, - Classes: []backup.ClassDescriptor{ - { - Name: className, - }, - }, - Status: string(backup.Started), - Version: ubak.Version, - } - - b, err := json.Marshal(desc) - require.Nil(t, err) - - err = fs.PutObject(testCtx, backupID, metadataFilename, b) - require.Nil(t, err) - - dest := fs.HomeDir(backupID) - expected := fmt.Sprintf("%s/%s", backupDir, backupID) - assert.Equal(t, expected, dest) - }) - }) -} - -func moduleLevelCopyObjects(t *testing.T) { - testCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - dataDir := t.TempDir() - backupDir := t.TempDir() - key := "moduleLevelCopyObjects" - backupID := "backup_id" - - t.Setenv("BACKUP_FILESYSTEM_PATH", backupDir) - - t.Run("copy objects", func(t *testing.T) { - logger, _ := test.NewNullLogger() - sp := fakeStorageProvider{dataDir} - params := moduletools.NewInitParams(sp, nil, config.Config{}, logger) - - fs := modstgfs.New() - err := fs.Init(testCtx, params) - require.Nil(t, err) - - t.Run("put object to bucket", func(t *testing.T) { - err := fs.PutObject(testCtx, backupID, key, []byte("hello")) - assert.Nil(t, err) - }) - - t.Run("get object from bucket", func(t *testing.T) { - meta, err := fs.GetObject(testCtx, backupID, key) - assert.Nil(t, err) - assert.Equal(t, []byte("hello"), meta) - }) - }) -} - -func moduleLevelCopyFiles(t *testing.T) { - testCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - dataDir := t.TempDir() - backupDir := t.TempDir() - key := "moduleLevelCopyFiles" - backupID := "backup_id" - - t.Setenv("BACKUP_FILESYSTEM_PATH", backupDir) - - t.Run("copy files", func(t *testing.T) { - fpaths := moduleshelper.CreateTestFiles(t, dataDir) - fpath := fpaths[0] - expectedContents, err := os.ReadFile(fpath) - require.Nil(t, err) - require.NotNil(t, expectedContents) - - logger, _ := test.NewNullLogger() - sp := fakeStorageProvider{dataDir} - params := moduletools.NewInitParams(sp, nil, config.Config{}, logger) - - fs := modstgfs.New() - err = fs.Init(testCtx, params) - require.Nil(t, err) - - t.Run("verify source data path", func(t *testing.T) { - assert.Equal(t, dataDir, fs.SourceDataPath()) - }) - - t.Run("copy file to backend", func(t *testing.T) { - srcPath, _ := filepath.Rel(dataDir, fpath) - err := fs.PutFile(testCtx, backupID, key, srcPath) - require.Nil(t, err) - - contents, err := fs.GetObject(testCtx, backupID, key) - require.Nil(t, err) - assert.Equal(t, expectedContents, contents) - }) - - t.Run("fetch file from backend", func(t *testing.T) { - destPath := dataDir + "/file_0.copy.db" - - err := fs.WriteToFile(testCtx, backupID, key, destPath) - require.Nil(t, err) - - contents, err := os.ReadFile(destPath) - require.Nil(t, err) - assert.Equal(t, expectedContents, contents) - }) - }) -} - -type fakeStorageProvider struct { - dataPath string -} - -func (sp fakeStorageProvider) Storage(name string) (moduletools.Storage, error) { - return nil, nil -} - -func (sp fakeStorageProvider) DataPath() string { - return sp.dataPath -} diff --git a/test/modules/backup-filesystem/backup_journey_test.go b/test/modules/backup-filesystem/backup_journey_test.go deleted file mode 100644 index a1e91ed04ecde484d32a477135ab844bbc7066ff..0000000000000000000000000000000000000000 --- a/test/modules/backup-filesystem/backup_journey_test.go +++ /dev/null @@ -1,74 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "context" - "testing" - "time" - - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/test/docker" - "github.com/weaviate/weaviate/test/helper/journey" -) - -const ( - fsBackupJourneyClassName = "FileSystemBackup" - fsBackupJourneyBackupIDSingleNode = "fs-backup-single-node" - fsBackupJourneyBackupIDCluster = "fs-backup-cluster" -) - -func Test_BackupJourney(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) - defer cancel() - - t.Run("single node", func(t *testing.T) { - compose, err := docker.New(). - WithBackendFilesystem(). - WithText2VecContextionary(). - WithWeaviate(). - Start(ctx) - require.Nil(t, err) - - defer func() { - if err := compose.Terminate(ctx); err != nil { - t.Fatalf("failed to terminate test containers: %s", err.Error()) - } - }() - - t.Run("backup-filesystem", func(t *testing.T) { - journey.BackupJourneyTests_SingleNode(t, compose.GetWeaviate().URI(), - "filesystem", fsBackupJourneyClassName, fsBackupJourneyBackupIDSingleNode, nil) - }) - }) - - t.Run("multiple nodes", func(t *testing.T) { - compose, err := docker.New(). - WithBackendFilesystem(). - WithText2VecContextionary(). - WithWeaviateCluster(). - Start(ctx) - require.Nil(t, err) - - defer func() { - if err := compose.Terminate(ctx); err != nil { - t.Fatalf("failed to terminate test containers: %s", err.Error()) - } - }() - - t.Run("backup-filesystem", func(t *testing.T) { - journey.BackupJourneyTests_Cluster(t, "filesystem", - fsBackupJourneyClassName, fsBackupJourneyBackupIDCluster, nil, - compose.GetWeaviate().URI(), compose.GetWeaviateNode2().URI()) - }) - }) -} diff --git a/test/modules/backup-filesystem/multi_tenant_backup_test.go b/test/modules/backup-filesystem/multi_tenant_backup_test.go deleted file mode 100644 index c2e58a0d7b9ec7250c848b22ac1c1cea6f0c66e5..0000000000000000000000000000000000000000 --- a/test/modules/backup-filesystem/multi_tenant_backup_test.go +++ /dev/null @@ -1,56 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/test/docker" - "github.com/weaviate/weaviate/test/helper/journey" -) - -const numTenants = 50 - -func Test_MultiTenantBackup(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) - defer cancel() - - t.Run("single node", func(t *testing.T) { - compose, err := docker.New(). - WithBackendFilesystem(). - WithText2VecContextionary(). - WithWeaviate(). - Start(ctx) - require.Nil(t, err) - - defer func() { - if err := compose.Terminate(ctx); err != nil { - t.Fatalf("failed to terminate test containers: %s", err.Error()) - } - }() - - t.Run("backup-filesystem", func(t *testing.T) { - tenantNames := make([]string, numTenants) - for i := range tenantNames { - tenantNames[i] = fmt.Sprintf("Tenant%d", i) - } - - journey.BackupJourneyTests_SingleNode(t, - compose.GetWeaviate().URI(), "filesystem", fsBackupJourneyClassName, - fsBackupJourneyBackupIDSingleNode, tenantNames) - }) - }) -} diff --git a/test/modules/backup-gcs/backup_backend_test.go b/test/modules/backup-gcs/backup_backend_test.go deleted file mode 100644 index 0d569c41871f2fba8229a65e0ffb7e540d899329..0000000000000000000000000000000000000000 --- a/test/modules/backup-gcs/backup_backend_test.go +++ /dev/null @@ -1,279 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "context" - "encoding/json" - "fmt" - "os" - "path/filepath" - "testing" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - logrustest "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/backup" - "github.com/weaviate/weaviate/entities/moduletools" - mod "github.com/weaviate/weaviate/modules/backup-gcs" - "github.com/weaviate/weaviate/test/docker" - moduleshelper "github.com/weaviate/weaviate/test/helper/modules" - ubak "github.com/weaviate/weaviate/usecases/backup" - "github.com/weaviate/weaviate/usecases/config" -) - -func Test_GCSBackend_Backup(t *testing.T) { - ctx := context.Background() - compose, err := docker.New().WithGCS().Start(ctx) - if err != nil { - t.Fatal(errors.Wrapf(err, "cannot start")) - } - - t.Setenv(envGCSEndpoint, compose.GetGCS().URI()) - - t.Run("store backup meta", moduleLevelStoreBackupMeta) - t.Run("copy objects", moduleLevelCopyObjects) - t.Run("copy files", moduleLevelCopyFiles) - - if err := compose.Terminate(ctx); err != nil { - t.Fatal(errors.Wrapf(err, "failed to terminate test containers")) - } -} - -func moduleLevelStoreBackupMeta(t *testing.T) { - testCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - dataDir := t.TempDir() - className := "BackupClass" - backupID := "backup_id" - bucketName := "bucket" - projectID := "project-id" - endpoint := os.Getenv(envGCSEndpoint) - metadataFilename := "backup.json" - gcsUseAuth := "false" - - t.Log("setup env") - t.Setenv(envGCSEndpoint, endpoint) - t.Setenv(envGCSStorageEmulatorHost, endpoint) - t.Setenv(envGCSCredentials, "") - t.Setenv(envGCSProjectID, projectID) - t.Setenv(envGCSBucket, bucketName) - t.Setenv(envGCSUseAuth, gcsUseAuth) - moduleshelper.CreateGCSBucket(testCtx, t, projectID, bucketName) - - t.Run("store backup meta in gcs", func(t *testing.T) { - t.Setenv("BACKUP_GCS_BUCKET", bucketName) - gcs := mod.New() - err := gcs.Init(testCtx, newFakeModuleParams(dataDir)) - require.Nil(t, err) - - t.Run("access permissions", func(t *testing.T) { - err := gcs.Initialize(testCtx, backupID) - assert.Nil(t, err) - }) - - t.Run("backup meta does not exist yet", func(t *testing.T) { - meta, err := gcs.GetObject(testCtx, backupID, metadataFilename) - assert.Nil(t, meta) - assert.NotNil(t, err) - assert.IsType(t, backup.ErrNotFound{}, err) - }) - - t.Run("put backup meta on backend", func(t *testing.T) { - desc := &backup.BackupDescriptor{ - StartedAt: time.Now(), - CompletedAt: time.Time{}, - ID: backupID, - Classes: []backup.ClassDescriptor{ - { - Name: className, - }, - }, - Status: string(backup.Started), - Version: ubak.Version, - } - - b, err := json.Marshal(desc) - require.Nil(t, err) - - err = gcs.PutObject(testCtx, backupID, metadataFilename, b) - require.Nil(t, err) - - dest := gcs.HomeDir(backupID) - expected := fmt.Sprintf("gs://%s/%s", bucketName, backupID) - assert.Equal(t, expected, dest) - }) - - t.Run("assert backup meta contents", func(t *testing.T) { - obj, err := gcs.GetObject(testCtx, backupID, metadataFilename) - require.Nil(t, err) - - var meta backup.BackupDescriptor - err = json.Unmarshal(obj, &meta) - require.Nil(t, err) - assert.NotEmpty(t, meta.StartedAt) - assert.Empty(t, meta.CompletedAt) - assert.Equal(t, meta.Status, string(backup.Started)) - assert.Empty(t, meta.Error) - assert.Len(t, meta.Classes, 1) - assert.Equal(t, meta.Classes[0].Name, className) - assert.Equal(t, meta.Version, ubak.Version) - assert.Nil(t, meta.Classes[0].Error) - }) - }) -} - -func moduleLevelCopyObjects(t *testing.T) { - testCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - dataDir := t.TempDir() - key := "moduleLevelCopyObjects" - backupID := "backup_id" - bucketName := "bucket" - projectID := "project-id" - endpoint := os.Getenv(envGCSEndpoint) - gcsUseAuth := "false" - - t.Log("setup env") - t.Setenv(envGCSEndpoint, endpoint) - t.Setenv(envGCSStorageEmulatorHost, endpoint) - t.Setenv(envGCSCredentials, "") - t.Setenv(envGCSProjectID, projectID) - t.Setenv(envGCSBucket, bucketName) - t.Setenv(envGCSUseAuth, gcsUseAuth) - moduleshelper.CreateGCSBucket(testCtx, t, projectID, bucketName) - - t.Run("copy objects", func(t *testing.T) { - t.Setenv("BACKUP_GCS_BUCKET", bucketName) - gcs := mod.New() - err := gcs.Init(testCtx, newFakeModuleParams(dataDir)) - require.Nil(t, err) - - t.Run("put object to bucket", func(t *testing.T) { - err := gcs.PutObject(testCtx, backupID, key, []byte("hello")) - assert.Nil(t, err) - }) - - t.Run("get object from bucket", func(t *testing.T) { - meta, err := gcs.GetObject(testCtx, backupID, key) - assert.Nil(t, err) - assert.Equal(t, []byte("hello"), meta) - }) - }) -} - -func moduleLevelCopyFiles(t *testing.T) { - testCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - dataDir := t.TempDir() - key := "moduleLevelCopyFiles" - backupID := "backup_id" - bucketName := "bucket" - projectID := "project-id" - endpoint := os.Getenv(envGCSEndpoint) - gcsUseAuth := "false" - - t.Log("setup env") - t.Setenv(envGCSEndpoint, endpoint) - t.Setenv(envGCSStorageEmulatorHost, endpoint) - t.Setenv(envGCSCredentials, "") - t.Setenv(envGCSProjectID, projectID) - t.Setenv(envGCSBucket, bucketName) - t.Setenv(envGCSUseAuth, gcsUseAuth) - moduleshelper.CreateGCSBucket(testCtx, t, projectID, bucketName) - - t.Run("copy files", func(t *testing.T) { - fpaths := moduleshelper.CreateTestFiles(t, dataDir) - fpath := fpaths[0] - expectedContents, err := os.ReadFile(fpath) - require.Nil(t, err) - require.NotNil(t, expectedContents) - - t.Setenv("BACKUP_GCS_BUCKET", bucketName) - gcs := mod.New() - err = gcs.Init(testCtx, newFakeModuleParams(dataDir)) - require.Nil(t, err) - - t.Run("verify source data path", func(t *testing.T) { - assert.Equal(t, dataDir, gcs.SourceDataPath()) - }) - - t.Run("copy file to backend", func(t *testing.T) { - srcPath, _ := filepath.Rel(dataDir, fpath) - err := gcs.PutFile(testCtx, backupID, key, srcPath) - require.Nil(t, err) - - contents, err := gcs.GetObject(testCtx, backupID, key) - require.Nil(t, err) - assert.Equal(t, expectedContents, contents) - }) - - t.Run("fetch file from backend", func(t *testing.T) { - destPath := dataDir + "/file_0.copy.db" - - err := gcs.WriteToFile(testCtx, backupID, key, destPath) - require.Nil(t, err) - - contents, err := os.ReadFile(destPath) - require.Nil(t, err) - assert.Equal(t, expectedContents, contents) - }) - }) -} - -type fakeModuleParams struct { - logger logrus.FieldLogger - provider fakeStorageProvider - config config.Config -} - -func newFakeModuleParams(dataPath string) *fakeModuleParams { - logger, _ := logrustest.NewNullLogger() - return &fakeModuleParams{ - logger: logger, - provider: fakeStorageProvider{dataPath: dataPath}, - } -} - -func (f *fakeModuleParams) GetStorageProvider() moduletools.StorageProvider { - return &f.provider -} - -func (f *fakeModuleParams) GetAppState() interface{} { - return nil -} - -func (f *fakeModuleParams) GetLogger() logrus.FieldLogger { - return f.logger -} - -func (f *fakeModuleParams) GetConfig() config.Config { - return f.config -} - -type fakeStorageProvider struct { - dataPath string -} - -func (f *fakeStorageProvider) Storage(name string) (moduletools.Storage, error) { - return nil, nil -} - -func (f *fakeStorageProvider) DataPath() string { - return f.dataPath -} diff --git a/test/modules/backup-gcs/backup_journey_test.go b/test/modules/backup-gcs/backup_journey_test.go deleted file mode 100644 index 2932a700409eedf936a7e3c69ec624f4f0071bc9..0000000000000000000000000000000000000000 --- a/test/modules/backup-gcs/backup_journey_test.go +++ /dev/null @@ -1,104 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "context" - "testing" - "time" - - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/test/docker" - "github.com/weaviate/weaviate/test/helper" - "github.com/weaviate/weaviate/test/helper/journey" - moduleshelper "github.com/weaviate/weaviate/test/helper/modules" -) - -const ( - envGCSEndpoint = "GCS_ENDPOINT" - envGCSStorageEmulatorHost = "STORAGE_EMULATOR_HOST" - envGCSCredentials = "GOOGLE_APPLICATION_CREDENTIALS" - envGCSProjectID = "GOOGLE_CLOUD_PROJECT" - envGCSBucket = "BACKUP_GCS_BUCKET" - envGCSUseAuth = "BACKUP_GCS_USE_AUTH" - - gcsBackupJourneyClassName = "GcsBackup" - gcsBackupJourneyBackupIDSingleNode = "gcs-backup-single-node" - gcsBackupJourneyBackupIDCluster = "gcs-backup-cluster" - gcsBackupJourneyProjectID = "gcs-backup-journey" - gcsBackupJourneyBucketName = "backups" -) - -func Test_BackupJourney(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) - defer cancel() - - t.Run("single node", func(t *testing.T) { - t.Log("pre-instance env setup") - t.Setenv(envGCSCredentials, "") - t.Setenv(envGCSProjectID, gcsBackupJourneyProjectID) - t.Setenv(envGCSBucket, gcsBackupJourneyBucketName) - - compose, err := docker.New(). - WithBackendGCS(gcsBackupJourneyBucketName). - WithText2VecContextionary(). - WithWeaviate(). - Start(ctx) - require.Nil(t, err) - defer func() { - if err := compose.Terminate(ctx); err != nil { - t.Fatalf("failed to terminate test containers: %s", err.Error()) - } - }() - - t.Log("post-instance env setup") - t.Setenv(envGCSEndpoint, compose.GetGCS().URI()) - t.Setenv(envGCSStorageEmulatorHost, compose.GetGCS().URI()) - moduleshelper.CreateGCSBucket(ctx, t, gcsBackupJourneyProjectID, gcsBackupJourneyBucketName) - helper.SetupClient(compose.GetWeaviate().URI()) - - t.Run("backup-gcs", func(t *testing.T) { - journey.BackupJourneyTests_SingleNode(t, compose.GetWeaviate().URI(), - "gcs", gcsBackupJourneyClassName, gcsBackupJourneyBackupIDSingleNode, nil) - }) - }) - - t.Run("multiple node", func(t *testing.T) { - t.Log("pre-instance env setup") - t.Setenv(envGCSCredentials, "") - t.Setenv(envGCSProjectID, gcsBackupJourneyProjectID) - t.Setenv(envGCSBucket, gcsBackupJourneyBucketName) - - compose, err := docker.New(). - WithBackendGCS(gcsBackupJourneyBucketName). - WithText2VecContextionary(). - WithWeaviateCluster(). - Start(ctx) - require.Nil(t, err) - defer func() { - if err := compose.Terminate(ctx); err != nil { - t.Fatalf("failed to terminate test containers: %s", err.Error()) - } - }() - - t.Log("post-instance env setup") - t.Setenv(envGCSEndpoint, compose.GetGCS().URI()) - t.Setenv(envGCSStorageEmulatorHost, compose.GetGCS().URI()) - moduleshelper.CreateGCSBucket(ctx, t, gcsBackupJourneyProjectID, gcsBackupJourneyBucketName) - helper.SetupClient(compose.GetWeaviate().URI()) - - t.Run("backup-gcs", func(t *testing.T) { - journey.BackupJourneyTests_Cluster(t, "gcs", gcsBackupJourneyClassName, - gcsBackupJourneyBackupIDCluster, nil, compose.GetWeaviate().URI(), compose.GetWeaviateNode2().URI()) - }) - }) -} diff --git a/test/modules/backup-gcs/multi_tenant_backup_test.go b/test/modules/backup-gcs/multi_tenant_backup_test.go deleted file mode 100644 index 792fbc516dfe6cc5f599f29843ac2bb1a9e3d9f5..0000000000000000000000000000000000000000 --- a/test/modules/backup-gcs/multi_tenant_backup_test.go +++ /dev/null @@ -1,99 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/test/docker" - "github.com/weaviate/weaviate/test/helper" - "github.com/weaviate/weaviate/test/helper/journey" - moduleshelper "github.com/weaviate/weaviate/test/helper/modules" -) - -const numTenants = 50 - -func Test_MultiTenantBackupJourney(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) - defer cancel() - - tenantNames := make([]string, numTenants) - for i := range tenantNames { - tenantNames[i] = fmt.Sprintf("Tenant%d", i) - } - - t.Run("single node", func(t *testing.T) { - t.Log("pre-instance env setup") - t.Setenv(envGCSCredentials, "") - t.Setenv(envGCSProjectID, gcsBackupJourneyProjectID) - t.Setenv(envGCSBucket, gcsBackupJourneyBucketName) - - compose, err := docker.New(). - WithBackendGCS(gcsBackupJourneyBucketName). - WithText2VecContextionary(). - WithWeaviate(). - Start(ctx) - require.Nil(t, err) - defer func() { - if err := compose.Terminate(ctx); err != nil { - t.Fatalf("failed to terminate test containers: %s", err.Error()) - } - }() - - t.Log("post-instance env setup") - t.Setenv(envGCSEndpoint, compose.GetGCS().URI()) - t.Setenv(envGCSStorageEmulatorHost, compose.GetGCS().URI()) - moduleshelper.CreateGCSBucket(ctx, t, gcsBackupJourneyProjectID, gcsBackupJourneyBucketName) - helper.SetupClient(compose.GetWeaviate().URI()) - - t.Run("backup-gcs", func(t *testing.T) { - journey.BackupJourneyTests_SingleNode(t, compose.GetWeaviate().URI(), - "gcs", gcsBackupJourneyClassName, - gcsBackupJourneyBackupIDSingleNode, tenantNames) - }) - }) - - t.Run("multiple node", func(t *testing.T) { - t.Log("pre-instance env setup") - t.Setenv(envGCSCredentials, "") - t.Setenv(envGCSProjectID, gcsBackupJourneyProjectID) - t.Setenv(envGCSBucket, gcsBackupJourneyBucketName) - - compose, err := docker.New(). - WithBackendGCS(gcsBackupJourneyBucketName). - WithText2VecContextionary(). - WithWeaviateCluster(). - Start(ctx) - require.Nil(t, err) - defer func() { - if err := compose.Terminate(ctx); err != nil { - t.Fatalf("failed to terminate test containers: %s", err.Error()) - } - }() - - t.Log("post-instance env setup") - t.Setenv(envGCSEndpoint, compose.GetGCS().URI()) - t.Setenv(envGCSStorageEmulatorHost, compose.GetGCS().URI()) - moduleshelper.CreateGCSBucket(ctx, t, gcsBackupJourneyProjectID, gcsBackupJourneyBucketName) - helper.SetupClient(compose.GetWeaviate().URI()) - - t.Run("backup-gcs", func(t *testing.T) { - journey.BackupJourneyTests_Cluster(t, "gcs", gcsBackupJourneyClassName, - gcsBackupJourneyBackupIDCluster, tenantNames, - compose.GetWeaviate().URI(), compose.GetWeaviateNode2().URI()) - }) - }) -} diff --git a/test/modules/backup-gcs/node_mapping_backup_journey_test.go b/test/modules/backup-gcs/node_mapping_backup_journey_test.go deleted file mode 100644 index ad38c08f584b2df2c2ca88019dce8befb664cd65..0000000000000000000000000000000000000000 --- a/test/modules/backup-gcs/node_mapping_backup_journey_test.go +++ /dev/null @@ -1,70 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "context" - "testing" - "time" - - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/test/docker" - "github.com/weaviate/weaviate/test/helper" - "github.com/weaviate/weaviate/test/helper/journey" - moduleshelper "github.com/weaviate/weaviate/test/helper/modules" -) - -func Test_NodeMappingBackupJourney(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) - defer cancel() - - t.Run("single node", func(t *testing.T) { - t.Log("pre-instance env setup") - t.Setenv(envGCSCredentials, "") - t.Setenv(envGCSProjectID, gcsBackupJourneyProjectID) - t.Setenv(envGCSBucket, gcsBackupJourneyBucketName) - - compose, err := docker.New(). - WithBackendGCS(gcsBackupJourneyBucketName). - WithText2VecContextionary(). - WithWeaviate(). - WithSecondWeaviate(). - Start(ctx) - require.Nil(t, err) - defer func() { - if err := compose.Terminate(ctx); err != nil { - t.Fatalf("failed to terminate test containers: %s", err.Error()) - } - }() - - t.Log("post-instance env setup") - t.Setenv(envGCSEndpoint, compose.GetGCS().URI()) - t.Setenv(envGCSStorageEmulatorHost, compose.GetGCS().URI()) - moduleshelper.CreateGCSBucket(ctx, t, gcsBackupJourneyProjectID, gcsBackupJourneyBucketName) - helper.SetupClient(compose.GetWeaviate().URI()) - - t.Run("backup-gcs", func(t *testing.T) { - journey.NodeMappingBackupJourneyTests_SingleNode_Backup(t, compose.GetWeaviate().URI(), - "gcs", gcsBackupJourneyClassName, gcsBackupJourneyBackupIDSingleNode, nil) - }) - - // Now change our tests to use the second cluster and trigger a backup restore - helper.SetupClient(compose.GetSecondWeaviate().URI()) - - t.Run("restore-gcs", func(t *testing.T) { - journey.NodeMappingBackupJourneyTests_SingleNode_Restore(t, compose.GetSecondWeaviate().URI(), - "gcs", gcsBackupJourneyClassName, gcsBackupJourneyBackupIDSingleNode, nil, map[string]string{ - docker.Weaviate: docker.SecondWeaviate, - }) - }) - }) -} diff --git a/test/modules/backup-s3/backup_backend_test.go b/test/modules/backup-s3/backup_backend_test.go deleted file mode 100644 index dded7b36402e59cf33405fe815fd224861ba32f4..0000000000000000000000000000000000000000 --- a/test/modules/backup-s3/backup_backend_test.go +++ /dev/null @@ -1,273 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "context" - "encoding/json" - "fmt" - "os" - "path/filepath" - "testing" - "time" - - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - logrustest "github.com/sirupsen/logrus/hooks/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/backup" - "github.com/weaviate/weaviate/entities/moduletools" - mod "github.com/weaviate/weaviate/modules/backup-s3" - "github.com/weaviate/weaviate/test/docker" - moduleshelper "github.com/weaviate/weaviate/test/helper/modules" - ubak "github.com/weaviate/weaviate/usecases/backup" - "github.com/weaviate/weaviate/usecases/config" -) - -func Test_S3Backend_Backup(t *testing.T) { - ctx := context.Background() - compose, err := docker.New().WithMinIO().Start(ctx) - if err != nil { - t.Fatal(errors.Wrapf(err, "cannot start")) - } - - t.Setenv(envMinioEndpoint, compose.GetMinIO().URI()) - - t.Run("store backup meta", moduleLevelStoreBackupMeta) - t.Run("copy objects", moduleLevelCopyObjects) - t.Run("copy files", moduleLevelCopyFiles) - - if err := compose.Terminate(ctx); err != nil { - t.Fatal(errors.Wrapf(err, "failed to terminate test containers")) - } -} - -func moduleLevelStoreBackupMeta(t *testing.T) { - testCtx, cancel := context.WithTimeout(context.Background(), time.Minute) - defer cancel() - - dataDir := t.TempDir() - className := "BackupClass" - backupID := "backup_id" - bucketName := "bucket" - region := "eu-west-1" - endpoint := os.Getenv(envMinioEndpoint) - metadataFilename := "backup.json" - - t.Log("setup env") - t.Setenv(envAwsRegion, region) - t.Setenv(envS3AccessKey, "aws_access_key") - t.Setenv(envS3SecretKey, "aws_secret_key") - t.Setenv(envS3Bucket, bucketName) - createBucket(testCtx, t, endpoint, region, bucketName) - - t.Run("store backup meta in s3", func(t *testing.T) { - t.Setenv(envS3UseSSL, "false") - t.Setenv(envS3Endpoint, endpoint) - s3 := mod.New() - err := s3.Init(testCtx, newFakeModuleParams(dataDir)) - require.Nil(t, err) - - t.Run("access permissions", func(t *testing.T) { - err := s3.Initialize(testCtx, backupID) - assert.Nil(t, err) - }) - - t.Run("backup meta does not exist yet", func(t *testing.T) { - meta, err := s3.GetObject(testCtx, backupID, metadataFilename) - assert.Nil(t, meta) - assert.NotNil(t, err) - assert.IsType(t, backup.ErrNotFound{}, err) - }) - - t.Run("put backup meta in backend", func(t *testing.T) { - desc := &backup.BackupDescriptor{ - StartedAt: time.Now(), - CompletedAt: time.Time{}, - ID: backupID, - Classes: []backup.ClassDescriptor{ - { - Name: className, - }, - }, - Status: string(backup.Started), - Version: ubak.Version, - } - - b, err := json.Marshal(desc) - require.Nil(t, err) - - err = s3.PutObject(testCtx, backupID, metadataFilename, b) - require.Nil(t, err) - - dest := s3.HomeDir(backupID) - expected := fmt.Sprintf("s3://%s/%s", bucketName, backupID) - assert.Equal(t, expected, dest) - }) - - t.Run("assert backup meta contents", func(t *testing.T) { - obj, err := s3.GetObject(testCtx, backupID, metadataFilename) - require.Nil(t, err) - - var meta backup.BackupDescriptor - err = json.Unmarshal(obj, &meta) - require.Nil(t, err) - assert.NotEmpty(t, meta.StartedAt) - assert.Empty(t, meta.CompletedAt) - assert.Equal(t, meta.Status, string(backup.Started)) - assert.Empty(t, meta.Error) - assert.Len(t, meta.Classes, 1) - assert.Equal(t, meta.Classes[0].Name, className) - assert.Equal(t, meta.Version, ubak.Version) - assert.Nil(t, meta.Classes[0].Error) - }) - }) -} - -func moduleLevelCopyObjects(t *testing.T) { - testCtx, cancel := context.WithTimeout(context.Background(), time.Minute) - defer cancel() - - dataDir := t.TempDir() - key := "moduleLevelCopyObjects" - backupID := "backup_id" - bucketName := "bucket" - region := "eu-west-1" - endpoint := os.Getenv(envMinioEndpoint) - - t.Log("setup env") - t.Setenv(envAwsRegion, region) - t.Setenv(envS3AccessKey, "aws_access_key") - t.Setenv(envS3SecretKey, "aws_secret_key") - t.Setenv(envS3Bucket, bucketName) - createBucket(testCtx, t, endpoint, region, bucketName) - - t.Run("copy objects", func(t *testing.T) { - t.Setenv(envS3UseSSL, "false") - t.Setenv(envS3Endpoint, endpoint) - s3 := mod.New() - err := s3.Init(testCtx, newFakeModuleParams(dataDir)) - require.Nil(t, err) - - t.Run("put object to bucket", func(t *testing.T) { - err := s3.PutObject(testCtx, backupID, key, []byte("hello")) - assert.Nil(t, err, "expected nil, got: %v", err) - }) - - t.Run("get object from bucket", func(t *testing.T) { - meta, err := s3.GetObject(testCtx, backupID, key) - assert.Nil(t, err, "expected nil, got: %v", err) - assert.Equal(t, []byte("hello"), meta) - }) - }) -} - -func moduleLevelCopyFiles(t *testing.T) { - testCtx, cancel := context.WithTimeout(context.Background(), time.Minute) - defer cancel() - - dataDir := t.TempDir() - key := "moduleLevelCopyFiles" - backupID := "backup_id" - bucketName := "bucket" - region := "eu-west-1" - endpoint := os.Getenv(envMinioEndpoint) - - t.Log("setup env") - t.Setenv(envAwsRegion, region) - t.Setenv(envS3AccessKey, "aws_access_key") - t.Setenv(envS3SecretKey, "aws_secret_key") - t.Setenv(envS3Bucket, bucketName) - createBucket(testCtx, t, endpoint, region, bucketName) - - t.Run("copy files", func(t *testing.T) { - fpaths := moduleshelper.CreateTestFiles(t, dataDir) - fpath := fpaths[0] - expectedContents, err := os.ReadFile(fpath) - require.Nil(t, err) - require.NotNil(t, expectedContents) - - t.Setenv(envS3UseSSL, "false") - t.Setenv(envS3Endpoint, endpoint) - s3 := mod.New() - err = s3.Init(testCtx, newFakeModuleParams(dataDir)) - require.Nil(t, err) - - t.Run("verify source data path", func(t *testing.T) { - assert.Equal(t, dataDir, s3.SourceDataPath()) - }) - - t.Run("copy file to backend", func(t *testing.T) { - srcPath, _ := filepath.Rel(dataDir, fpath) - err := s3.PutFile(testCtx, backupID, key, srcPath) - require.Nil(t, err) - - contents, err := s3.GetObject(testCtx, backupID, key) - require.Nil(t, err) - assert.Equal(t, expectedContents, contents) - }) - - t.Run("fetch file from backend", func(t *testing.T) { - destPath := dataDir + "/file_0.copy.db" - - err := s3.WriteToFile(testCtx, backupID, key, destPath) - require.Nil(t, err) - - contents, err := os.ReadFile(destPath) - require.Nil(t, err) - assert.Equal(t, expectedContents, contents) - }) - }) -} - -type fakeModuleParams struct { - logger logrus.FieldLogger - provider fakeStorageProvider - config config.Config -} - -func newFakeModuleParams(dataPath string) *fakeModuleParams { - logger, _ := logrustest.NewNullLogger() - return &fakeModuleParams{ - logger: logger, - provider: fakeStorageProvider{dataPath: dataPath}, - } -} - -func (f *fakeModuleParams) GetStorageProvider() moduletools.StorageProvider { - return &f.provider -} - -func (f *fakeModuleParams) GetAppState() interface{} { - return nil -} - -func (f *fakeModuleParams) GetLogger() logrus.FieldLogger { - return f.logger -} - -func (f *fakeModuleParams) GetConfig() config.Config { - return f.config -} - -type fakeStorageProvider struct { - dataPath string -} - -func (f *fakeStorageProvider) Storage(name string) (moduletools.Storage, error) { - return nil, nil -} - -func (f *fakeStorageProvider) DataPath() string { - return f.dataPath -} diff --git a/test/modules/backup-s3/backup_journey_test.go b/test/modules/backup-s3/backup_journey_test.go deleted file mode 100644 index d793b4f1d4cff3ae4b510bf0e474f55f599942a9..0000000000000000000000000000000000000000 --- a/test/modules/backup-s3/backup_journey_test.go +++ /dev/null @@ -1,108 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "context" - "testing" - "time" - - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/test/docker" - "github.com/weaviate/weaviate/test/helper" - "github.com/weaviate/weaviate/test/helper/journey" -) - -const ( - envMinioEndpoint = "MINIO_ENDPOINT" - envAwsRegion = "AWS_REGION" - envS3AccessKey = "AWS_ACCESS_KEY_ID" - envS3SecretKey = "AWS_SECRET_KEY" - envS3Bucket = "BACKUP_S3_BUCKET" - envS3Endpoint = "BACKUP_S3_ENDPOINT" - envS3UseSSL = "BACKUP_S3_USE_SSL" - - s3BackupJourneyClassName = "S3Backup" - s3BackupJourneyBackupIDSingleNode = "s3-backup-single-node" - s3BackupJourneyBackupIDCluster = "s3-backup-cluster" - s3BackupJourneyBucketName = "backups" - s3BackupJourneyRegion = "eu-west-1" - s3BackupJourneyAccessKey = "aws_access_key" - s3BackupJourneySecretKey = "aws_secret_key" -) - -func Test_BackupJourney(t *testing.T) { - t.Run("single node", func(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) - defer cancel() - - t.Log("pre-instance env setup") - t.Setenv(envS3AccessKey, s3BackupJourneyAccessKey) - t.Setenv(envS3SecretKey, s3BackupJourneySecretKey) - t.Setenv(envS3Bucket, s3BackupJourneyBucketName) - - compose, err := docker.New(). - WithBackendS3(s3BackupJourneyBucketName). - WithText2VecContextionary(). - WithWeaviate(). - Start(ctx) - require.Nil(t, err) - defer func() { - if err := compose.Terminate(ctx); err != nil { - t.Fatalf("failed to terminate test containers: %s", err.Error()) - } - }() - - t.Run("post-instance env setup", func(t *testing.T) { - createBucket(ctx, t, compose.GetMinIO().URI(), s3BackupJourneyRegion, s3BackupJourneyBucketName) - helper.SetupClient(compose.GetWeaviate().URI()) - }) - - t.Run("backup-s3", func(t *testing.T) { - journey.BackupJourneyTests_SingleNode(t, compose.GetWeaviate().URI(), - "s3", s3BackupJourneyClassName, s3BackupJourneyBackupIDSingleNode, nil) - }) - }) - - t.Run("multiple node", func(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) - defer cancel() - - t.Log("pre-instance env setup") - t.Setenv(envS3AccessKey, s3BackupJourneyAccessKey) - t.Setenv(envS3SecretKey, s3BackupJourneySecretKey) - t.Setenv(envS3Bucket, s3BackupJourneyBucketName) - - compose, err := docker.New(). - WithBackendS3(s3BackupJourneyBucketName). - WithText2VecContextionary(). - WithWeaviateCluster(). - Start(ctx) - require.Nil(t, err) - defer func() { - if err := compose.Terminate(ctx); err != nil { - t.Fatalf("failed to terminate test containers: %s", err.Error()) - } - }() - - t.Run("post-instance env setup", func(t *testing.T) { - createBucket(ctx, t, compose.GetMinIO().URI(), s3BackupJourneyRegion, s3BackupJourneyBucketName) - helper.SetupClient(compose.GetWeaviate().URI()) - }) - - t.Run("backup-s3", func(t *testing.T) { - journey.BackupJourneyTests_Cluster(t, "s3", s3BackupJourneyClassName, - s3BackupJourneyBackupIDCluster, nil, - compose.GetWeaviate().URI(), compose.GetWeaviateNode2().URI()) - }) - }) -} diff --git a/test/modules/backup-s3/helper_test.go b/test/modules/backup-s3/helper_test.go deleted file mode 100644 index 83002fba5e1c20aaf50db9b648d665f6e1d2107b..0000000000000000000000000000000000000000 --- a/test/modules/backup-s3/helper_test.go +++ /dev/null @@ -1,41 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "context" - "testing" - - "github.com/minio/minio-go/v7" - "github.com/minio/minio-go/v7/pkg/credentials" - "github.com/stretchr/testify/require" -) - -func createBucket(ctx context.Context, t *testing.T, endpoint, region, bucketName string) { - client, err := minio.New(endpoint, &minio.Options{ - Creds: credentials.NewEnvAWS(), - Region: region, - Secure: false, - }) - require.Nil(t, err) - - err = client.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{}) - minioErr, ok := err.(minio.ErrorResponse) - if ok { - // the bucket persists from a previous test. - // if the bucket already exists, we can proceed - if minioErr.Code == "BucketAlreadyOwnedByYou" { - return - } - } - require.Nil(t, err) -} diff --git a/test/modules/backup-s3/multi_tenant_backup_test.go b/test/modules/backup-s3/multi_tenant_backup_test.go deleted file mode 100644 index 00a7a890d76343e23936b66266c6c523803a39d9..0000000000000000000000000000000000000000 --- a/test/modules/backup-s3/multi_tenant_backup_test.go +++ /dev/null @@ -1,98 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/test/docker" - "github.com/weaviate/weaviate/test/helper" - "github.com/weaviate/weaviate/test/helper/journey" -) - -const numTenants = 50 - -func Test_MultiTenantBackupJourney(t *testing.T) { - tenantNames := make([]string, numTenants) - for i := range tenantNames { - tenantNames[i] = fmt.Sprintf("Tenant%d", i) - } - - t.Run("single node", func(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) - defer cancel() - - t.Log("pre-instance env setup") - t.Setenv(envS3AccessKey, s3BackupJourneyAccessKey) - t.Setenv(envS3SecretKey, s3BackupJourneySecretKey) - t.Setenv(envS3Bucket, s3BackupJourneyBucketName) - - compose, err := docker.New(). - WithBackendS3(s3BackupJourneyBucketName). - WithText2VecContextionary(). - WithWeaviate(). - Start(ctx) - require.Nil(t, err) - defer func() { - if err := compose.Terminate(ctx); err != nil { - t.Fatalf("failed to terminate test containers: %s", err.Error()) - } - }() - - t.Run("post-instance env setup", func(t *testing.T) { - createBucket(ctx, t, compose.GetMinIO().URI(), s3BackupJourneyRegion, s3BackupJourneyBucketName) - helper.SetupClient(compose.GetWeaviate().URI()) - }) - - t.Run("backup-s3", func(t *testing.T) { - journey.BackupJourneyTests_SingleNode(t, compose.GetWeaviate().URI(), - "s3", s3BackupJourneyClassName, s3BackupJourneyBackupIDSingleNode, tenantNames) - }) - }) - - t.Run("multiple node", func(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) - defer cancel() - - t.Log("pre-instance env setup") - t.Setenv(envS3AccessKey, s3BackupJourneyAccessKey) - t.Setenv(envS3SecretKey, s3BackupJourneySecretKey) - t.Setenv(envS3Bucket, s3BackupJourneyBucketName) - - compose, err := docker.New(). - WithBackendS3(s3BackupJourneyBucketName). - WithText2VecContextionary(). - WithWeaviateCluster(). - Start(ctx) - require.Nil(t, err) - defer func() { - if err := compose.Terminate(ctx); err != nil { - t.Fatalf("failed to terminate test containers: %s", err.Error()) - } - }() - - t.Run("post-instance env setup", func(t *testing.T) { - createBucket(ctx, t, compose.GetMinIO().URI(), s3BackupJourneyRegion, s3BackupJourneyBucketName) - helper.SetupClient(compose.GetWeaviate().URI()) - }) - - t.Run("backup-s3", func(t *testing.T) { - journey.BackupJourneyTests_Cluster(t, "s3", s3BackupJourneyClassName, - s3BackupJourneyBackupIDCluster, tenantNames, - compose.GetWeaviate().URI(), compose.GetWeaviateNode2().URI()) - }) - }) -} diff --git a/test/modules/backup-s3/node_mapping_backup_journey_test.go b/test/modules/backup-s3/node_mapping_backup_journey_test.go deleted file mode 100644 index 7c268ab78a63e5b87047525296bed321859ba2d0..0000000000000000000000000000000000000000 --- a/test/modules/backup-s3/node_mapping_backup_journey_test.go +++ /dev/null @@ -1,68 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "context" - "testing" - "time" - - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/test/docker" - "github.com/weaviate/weaviate/test/helper" - "github.com/weaviate/weaviate/test/helper/journey" -) - -func Test_NodeMappingBackupJourney(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) - defer cancel() - - t.Run("single node", func(t *testing.T) { - t.Log("pre-instance env setup") - t.Setenv(envS3AccessKey, s3BackupJourneyAccessKey) - t.Setenv(envS3SecretKey, s3BackupJourneySecretKey) - t.Setenv(envS3Bucket, s3BackupJourneyBucketName) - - compose, err := docker.New(). - WithBackendS3(s3BackupJourneyBucketName). - WithText2VecContextionary(). - WithWeaviate(). - WithSecondWeaviate(). - Start(ctx) - require.Nil(t, err) - defer func() { - if err := compose.Terminate(ctx); err != nil { - t.Fatalf("failed to terminate test containers: %s", err.Error()) - } - }() - - t.Run("post-instance env setup", func(t *testing.T) { - createBucket(ctx, t, compose.GetMinIO().URI(), s3BackupJourneyRegion, s3BackupJourneyBucketName) - helper.SetupClient(compose.GetWeaviate().URI()) - }) - - t.Run("backup-s3", func(t *testing.T) { - journey.NodeMappingBackupJourneyTests_SingleNode_Backup(t, compose.GetWeaviate().URI(), - "s3", s3BackupJourneyClassName, s3BackupJourneyBackupIDSingleNode, nil) - }) - - // Now change our tests to use the second cluster and trigger a backup restore - helper.SetupClient(compose.GetSecondWeaviate().URI()) - - t.Run("restore-s3", func(t *testing.T) { - journey.NodeMappingBackupJourneyTests_SingleNode_Restore(t, compose.GetSecondWeaviate().URI(), - "s3", s3BackupJourneyClassName, s3BackupJourneyBackupIDSingleNode, nil, map[string]string{ - docker.Weaviate: docker.SecondWeaviate, - }) - }) - }) -} diff --git a/test/modules/img2vec-neural/data/pixel.png b/test/modules/img2vec-neural/data/pixel.png deleted file mode 100644 index e48f3bed921a02b87843aa78756cffffa51e442f..0000000000000000000000000000000000000000 Binary files a/test/modules/img2vec-neural/data/pixel.png and /dev/null differ diff --git a/test/modules/img2vec-neural/neural_test.go b/test/modules/img2vec-neural/neural_test.go deleted file mode 100644 index 9d5138a0fdf94480caf02da1f00a893089baedfe..0000000000000000000000000000000000000000 --- a/test/modules/img2vec-neural/neural_test.go +++ /dev/null @@ -1,103 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "encoding/base64" - "fmt" - "io" - "os" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema" - "github.com/weaviate/weaviate/test/helper" - graphqlhelper "github.com/weaviate/weaviate/test/helper/graphql" -) - -func Test_Img2VecNeural(t *testing.T) { - helper.SetupClient(os.Getenv(weaviateEndpoint)) - - fashionItemClass := &models.Class{ - Class: "FashionItem", - Vectorizer: "img2vec-neural", - ModuleConfig: map[string]interface{}{ - "img2vec-neural": map[string]interface{}{ - "imageFields": []string{"image"}, - }, - }, - Properties: []*models.Property{ - { - Name: "image", - DataType: schema.DataTypeBlob.PropString(), - }, - { - Name: "description", - DataType: schema.DataTypeText.PropString(), - Tokenization: models.PropertyTokenizationWhitespace, - }, - }, - } - - helper.CreateClass(t, fashionItemClass) - defer helper.DeleteClass(t, fashionItemClass.Class) - - getBase64EncodedTestImage := func() string { - image, err := os.Open("./data/pixel.png") - require.Nil(t, err) - require.NotNil(t, image) - content, err := io.ReadAll(image) - require.Nil(t, err) - return base64.StdEncoding.EncodeToString(content) - } - - t.Run("import data", func(t *testing.T) { - base64Image := getBase64EncodedTestImage() - obj := &models.Object{ - Class: fashionItemClass.Class, - Properties: map[string]interface{}{ - "image": base64Image, - "description": "A single black pixel", - }, - } - helper.CreateObject(t, obj) - }) - - t.Run("perform nearImage query", func(t *testing.T) { - queryTemplate := ` - { - Get { - FashionItem( - nearImage: { - image: "%s" - } - ){ - image - description - _additional{vector} - } - } - }` - query := fmt.Sprintf(queryTemplate, getBase64EncodedTestImage()) - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - fashionItems := result.Get("Get", "FashionItem").AsSlice() - require.True(t, len(fashionItems) > 0) - item, ok := fashionItems[0].(map[string]interface{}) - require.True(t, ok) - assert.NotNil(t, item["image"]) - assert.NotNil(t, item["description"]) - vector := item["_additional"].(map[string]interface{})["vector"] - assert.NotNil(t, vector) - }) -} diff --git a/test/modules/img2vec-neural/setup_test.go b/test/modules/img2vec-neural/setup_test.go deleted file mode 100644 index 9fadb13d3842293ab78a15fd2ff81e0951cf0558..0000000000000000000000000000000000000000 --- a/test/modules/img2vec-neural/setup_test.go +++ /dev/null @@ -1,44 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "context" - "os" - "testing" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/test/docker" -) - -const weaviateEndpoint = "WEAVIATE_ENDPOINT" - -func TestMain(m *testing.M) { - ctx := context.Background() - compose, err := docker.New(). - WithWeaviate(). - WithText2VecContextionary(). - WithImg2VecNeural(). - Start(ctx) - if err != nil { - panic(errors.Wrapf(err, "cannot start")) - } - - os.Setenv(weaviateEndpoint, compose.GetWeaviate().URI()) - code := m.Run() - - if err := compose.Terminate(ctx); err != nil { - panic(errors.Wrapf(err, "cannot terminate")) - } - - os.Exit(code) -} diff --git a/test/modules/many-modules/many_modules_test.go b/test/modules/many-modules/many_modules_test.go deleted file mode 100644 index e381abbb01c3ed9b35e0d5e636b1f5f5e627d71c..0000000000000000000000000000000000000000 --- a/test/modules/many-modules/many_modules_test.go +++ /dev/null @@ -1,196 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "encoding/json" - "fmt" - "os" - "testing" - - "github.com/go-openapi/strfmt" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/test/helper" - graphqlhelper "github.com/weaviate/weaviate/test/helper/graphql" - "github.com/weaviate/weaviate/test/helper/sample-schema/books" - "github.com/weaviate/weaviate/test/helper/sample-schema/multishard" -) - -func Test_ManyModules(t *testing.T) { - helper.SetupClient(os.Getenv(weaviateEndpoint)) - - t.Run("check enabled modules", func(t *testing.T) { - meta := helper.GetMeta(t) - require.NotNil(t, meta) - - expectedModuleNames := []string{ - "generative-cohere", "generative-palm", "generative-openai", "generative-aws", "generative-anyscale", - "text2vec-cohere", "text2vec-contextionary", "text2vec-openai", "text2vec-huggingface", - "text2vec-palm", "text2vec-aws", "text2vec-transformers", "qna-openai", "reranker-cohere", - } - - modules, ok := meta.Modules.(map[string]interface{}) - require.True(t, ok) - assert.Len(t, modules, len(expectedModuleNames)) - - moduleNames := []string{} - for name := range modules { - moduleNames = append(moduleNames, name) - } - assert.ElementsMatch(t, expectedModuleNames, moduleNames) - }) - - booksClass := books.ClassContextionaryVectorizer() - multiShardClass := multishard.ClassContextionaryVectorizer() - helper.CreateClass(t, booksClass) - helper.CreateClass(t, multiShardClass) - defer helper.DeleteClass(t, booksClass.Class) - defer helper.DeleteClass(t, multiShardClass.Class) - - t.Run("import data", func(t *testing.T) { - for _, book := range books.Objects() { - helper.CreateObject(t, book) - helper.AssertGetObjectEventually(t, book.Class, book.ID) - } - for _, multishard := range multishard.Objects() { - helper.CreateObject(t, multishard) - helper.AssertGetObjectEventually(t, multishard.Class, multishard.ID) - } - }) - - t.Run("sanity checks", func(t *testing.T) { - concepts := []string{ - "Frank", "Herbert", "Dune", "Book", "Project", "Hail", "Mary", - "The Lord of the Ice Garden", "Ice Garden", "science", "fiction", - "fantasy novel", "novelist", - } - checkResults := func(query string) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - books := result.Get("Get", "Books").AsSlice() - require.True(t, len(books) > 0) - results, ok := books[0].(map[string]interface{}) - require.True(t, ok) - assert.True(t, results["title"] != nil) - } - - t.Run("nearText queries", func(t *testing.T) { - queryTemplate := ` - { - Get { - Books( - nearText: { - concepts: ["%s"] - } - ){ - title - } - } - }` - for _, concept := range concepts { - checkResults(fmt.Sprintf(queryTemplate, concept)) - } - }) - t.Run("nearObject queries", func(t *testing.T) { - queryTemplate := ` - { - Get { - Books( - nearObject: { - id: "%s" - } - ){ - title - } - } - }` - ids := []strfmt.UUID{books.Dune, books.ProjectHailMary, books.TheLordOfTheIceGarden} - for _, id := range ids { - checkResults(fmt.Sprintf(queryTemplate, id)) - } - }) - t.Run("nearVector queries", func(t *testing.T) { - getVectors := func() []string { - query := ` - { - Get { - Books(limit: 3){ _additional{ vector } } - } - }` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - books := result.Get("Get", "Books").AsSlice() - require.True(t, len(books) == 3) - vectors := make([]string, 3) - for i := 0; i < 3; i++ { - results, ok := books[i].(map[string]interface{}) - require.True(t, ok) - vector, ok := results["_additional"].(map[string]interface{})["vector"].([]interface{}) - require.True(t, ok) - vec, err := json.Marshal(vector) - require.Nil(t, err) - vectors[i] = string(vec) - } - return vectors - } - vectors := getVectors() - queryTemplate := ` - { - Get { - Books( - nearVector: { - vector: %s - } - ){ - title - } - } - }` - for _, vector := range vectors { - checkResults(fmt.Sprintf(queryTemplate, vector)) - } - }) - t.Run("hybrid queries", func(t *testing.T) { - queryTemplate := ` - { - Get { - Books( - hybrid: { - query: "%s" - } - ){ - title - } - } - }` - for _, concept := range concepts { - checkResults(fmt.Sprintf(queryTemplate, concept)) - } - }) - t.Run("bm25 queries", func(t *testing.T) { - queryTemplate := ` - { - Get { - Books( - bm25:{ - query: "%s" - } - ){ - title - } - } - }` - for _, concept := range []string{"Frank", "Project Hail Mary", "Dune", "Project", "Hail", "Mary"} { - checkResults(fmt.Sprintf(queryTemplate, concept)) - } - }) - }) -} diff --git a/test/modules/many-modules/setup_test.go b/test/modules/many-modules/setup_test.go deleted file mode 100644 index eb7e3ec8dd0b5c8bbf6d4df9d6ec52f0b7bf8e09..0000000000000000000000000000000000000000 --- a/test/modules/many-modules/setup_test.go +++ /dev/null @@ -1,56 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "context" - "os" - "testing" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/test/docker" -) - -const weaviateEndpoint = "WEAVIATE_ENDPOINT" - -func TestMain(m *testing.M) { - ctx := context.Background() - compose, err := docker.New(). - WithWeaviate(). - WithText2VecContextionary(). - WithText2VecTransformers(). - WithText2VecOpenAI(). - WithText2VecCohere(). - WithText2VecPaLM(). - WithText2VecHuggingFace(). - WithText2VecAWS(). - WithGenerativeOpenAI(). - WithGenerativeCohere(). - WithGenerativePaLM(). - WithGenerativeAWS(). - WithGenerativeAnyscale(). - WithQnAOpenAI(). - WithRerankerCohere(). - Start(ctx) - if err != nil { - panic(errors.Wrapf(err, "cannot start")) - } - - os.Setenv(weaviateEndpoint, compose.GetWeaviate().URI()) - code := m.Run() - - if err := compose.Terminate(ctx); err != nil { - panic(errors.Wrapf(err, "cannot terminate")) - } - - os.Exit(code) -} diff --git a/test/modules/multi2vec-clip/clip_test.go b/test/modules/multi2vec-clip/clip_test.go deleted file mode 100644 index 44bfcfc1dd6a52806773cc23df10644f3751d77c..0000000000000000000000000000000000000000 --- a/test/modules/multi2vec-clip/clip_test.go +++ /dev/null @@ -1,85 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "encoding/json" - "fmt" - "os" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/test/helper" - graphqlhelper "github.com/weaviate/weaviate/test/helper/graphql" - "github.com/weaviate/weaviate/test/helper/sample-schema/books" -) - -func Test_CLIP(t *testing.T) { - helper.SetupClient(os.Getenv(weaviateEndpoint)) - booksClass := books.ClassCLIPVectorizer() - helper.CreateClass(t, booksClass) - defer helper.DeleteClass(t, booksClass.Class) - - t.Run("add data to Books schema", func(t *testing.T) { - for _, book := range books.Objects() { - helper.CreateObject(t, book) - helper.AssertGetObjectEventually(t, book.Class, book.ID) - } - }) - - tests := []struct { - concept string - expectedTitle string - }{ - { - concept: "Dune", - expectedTitle: "Dune", - }, - { - concept: "three", - expectedTitle: "The Lord of the Ice Garden", - }, - } - for _, tt := range tests { - t.Run("query Books data with nearText", func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, fmt.Sprintf(` - { - Get { - Books( - limit: 1 - nearText: { - concepts: ["%v"] - distance: 0.5 - } - ){ - title - _additional { - distance - } - } - } - } - `, tt.concept)) - books := result.Get("Get", "Books").AsSlice() - require.Len(t, books, 1) - title := books[0].(map[string]interface{})["title"] - assert.Equal(t, tt.expectedTitle, title) - distance := books[0].(map[string]interface{})["_additional"].(map[string]interface{})["distance"].(json.Number) - assert.NotNil(t, distance) - dist, err := distance.Float64() - require.Nil(t, err) - assert.Greater(t, dist, 0.0) - assert.LessOrEqual(t, dist, 0.03) - }) - } -} diff --git a/test/modules/multi2vec-clip/setup_test.go b/test/modules/multi2vec-clip/setup_test.go deleted file mode 100644 index 0b157f697ca235cf79a154ff88ecf90a4d452506..0000000000000000000000000000000000000000 --- a/test/modules/multi2vec-clip/setup_test.go +++ /dev/null @@ -1,43 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "context" - "os" - "testing" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/test/docker" -) - -const weaviateEndpoint = "WEAVIATE_ENDPOINT" - -func TestMain(m *testing.M) { - ctx := context.Background() - compose, err := docker.New(). - WithWeaviate(). - WithMulti2VecCLIP(). - Start(ctx) - if err != nil { - panic(errors.Wrapf(err, "cannot start")) - } - - os.Setenv(weaviateEndpoint, compose.GetWeaviate().URI()) - code := m.Run() - - if err := compose.Terminate(ctx); err != nil { - panic(errors.Wrapf(err, "cannot terminate")) - } - - os.Exit(code) -} diff --git a/test/modules/qna-transformers/qna_test.go b/test/modules/qna-transformers/qna_test.go deleted file mode 100644 index 3958e5998f2b157143c03014da8e16c49fb66af8..0000000000000000000000000000000000000000 --- a/test/modules/qna-transformers/qna_test.go +++ /dev/null @@ -1,107 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "encoding/json" - "fmt" - "os" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/test/helper" - graphqlhelper "github.com/weaviate/weaviate/test/helper/graphql" - "github.com/weaviate/weaviate/test/helper/sample-schema/books" -) - -func Test_QnATransformers(t *testing.T) { - helper.SetupClient(os.Getenv(weaviateEndpoint)) - // Contextionary with QnA module config present - booksClass := books.ClassContextionaryVectorizerWithQnATransformers() - helper.CreateClass(t, booksClass) - defer helper.DeleteClass(t, booksClass.Class) - // Contextionary without QnA module config present - booksWithoutQnAConfig := "BooksWithoutConfig" - booksWithoutQnAConfigClass := books.ClassContextionaryVectorizerWithName(booksWithoutQnAConfig) - helper.CreateClass(t, booksWithoutQnAConfigClass) - defer helper.DeleteClass(t, booksWithoutQnAConfigClass.Class) - // Text2VecTransformers with QnA module config present - booksTransformers := "BooksTransformers" - booksTransformersClass := books.ClassTransformersVectorizerWithQnATransformersWithName(booksTransformers) - helper.CreateClass(t, booksTransformersClass) - defer helper.DeleteClass(t, booksTransformersClass.Class) - // Text2VecTransformers without QnA module config present - booksTransformersWithoutQnAConfig := "BooksTransformersWithoutConfig" - booksTransformersWithoutQnAConfigClass := books.ClassTransformersVectorizerWithName(booksTransformersWithoutQnAConfig) - helper.CreateClass(t, booksTransformersWithoutQnAConfigClass) - defer helper.DeleteClass(t, booksTransformersWithoutQnAConfigClass.Class) - - t.Run("add data to Books schema", func(t *testing.T) { - bookObjects := []*models.Object{} - bookObjects = append(bookObjects, books.Objects()...) - bookObjects = append(bookObjects, books.ObjectsWithName(booksWithoutQnAConfig)...) - bookObjects = append(bookObjects, books.ObjectsWithName(booksTransformers)...) - bookObjects = append(bookObjects, books.ObjectsWithName(booksTransformersWithoutQnAConfig)...) - for _, book := range bookObjects { - helper.CreateObject(t, book) - helper.AssertGetObjectEventually(t, book.Class, book.ID) - } - }) - - t.Run("ask", func(t *testing.T) { - for _, class := range []*models.Class{booksClass, booksWithoutQnAConfigClass, booksTransformersClass, booksTransformersWithoutQnAConfigClass} { - t.Run(class.Class, func(t *testing.T) { - query := ` - { - Get { - %s( - ask: { - question: "Who is Dune's author?" - } - limit: 1 - ){ - title - _additional { - answer { - hasAnswer - property - result - startPosition - endPosition - } - } - } - } - } - ` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, fmt.Sprintf(query, class.Class)) - books := result.Get("Get", class.Class).AsSlice() - expected := []interface{}{ - map[string]interface{}{ - "title": "Dune", - "_additional": map[string]interface{}{ - "answer": map[string]interface{}{ - "endPosition": json.Number("74"), - "hasAnswer": true, - "property": "description", - "result": "frank herbert", - "startPosition": json.Number("61"), - }, - }, - }, - } - assert.ElementsMatch(t, expected, books) - }) - } - }) -} diff --git a/test/modules/qna-transformers/setup_test.go b/test/modules/qna-transformers/setup_test.go deleted file mode 100644 index 7477d31e1e367a1b0fea7cc6946a7c98eb8913d8..0000000000000000000000000000000000000000 --- a/test/modules/qna-transformers/setup_test.go +++ /dev/null @@ -1,45 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "context" - "os" - "testing" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/test/docker" -) - -const weaviateEndpoint = "WEAVIATE_ENDPOINT" - -func TestMain(m *testing.M) { - ctx := context.Background() - compose, err := docker.New(). - WithWeaviate(). - WithText2VecContextionary(). - WithText2VecTransformers(). - WithQnATransformers(). - Start(ctx) - if err != nil { - panic(errors.Wrapf(err, "cannot start")) - } - - os.Setenv(weaviateEndpoint, compose.GetWeaviate().URI()) - code := m.Run() - - if err := compose.Terminate(ctx); err != nil { - panic(errors.Wrapf(err, "cannot terminate")) - } - - os.Exit(code) -} diff --git a/test/modules/ref2vec-centroid/centroid_test.go b/test/modules/ref2vec-centroid/centroid_test.go deleted file mode 100644 index d22119b72ca71f64b637124da744ee74bca96dea..0000000000000000000000000000000000000000 --- a/test/modules/ref2vec-centroid/centroid_test.go +++ /dev/null @@ -1,212 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "os" - "testing" - - "github.com/go-openapi/strfmt" - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/schema/crossref" - "github.com/weaviate/weaviate/test/helper" - "github.com/weaviate/weaviate/test/helper/sample-schema/articles" -) - -func TestCentroid(t *testing.T) { - helper.SetupClient(os.Getenv(weaviateEndpoint)) - - paragraphClass := articles.ParagraphsClass() - articleClass := articles.ArticlesClass() - articleClass.ModuleConfig = map[string]interface{}{ - "ref2vec-centroid": map[string]interface{}{ - "referenceProperties": []string{"hasParagraphs"}, - }, - } - articleClass.Vectorizer = "ref2vec-centroid" - - helper.CreateClass(t, paragraphClass) - helper.CreateClass(t, articleClass) - - defer func() { - helper.DeleteClass(t, articleClass.Class) - helper.DeleteClass(t, paragraphClass.Class) - }() - - para1 := articles.NewParagraph(). - WithVector([]float32{2, 4, 6}). - WithContents( - "Anyone who cares about JDM engines knows about the RB26.") - helper.CreateObject(t, para1.Object()) - - para2 := articles.NewParagraph(). - WithVector([]float32{4, 6, 8}). - WithContents( - "It is best known for its use in the legendary Skyline GT-R.") - helper.CreateObject(t, para2.Object()) - - t.Run("create object with references", func(t *testing.T) { - t.Run("with one reference", func(t *testing.T) { - article := articles.NewArticle(). - WithTitle("Popularity of the Nissan RB26DETT"). - WithReferences(&models.SingleRef{ - Beacon: newBeacon(para1.Class, para1.ID), - }) - helper.CreateObject(t, article.Object()) - defer helper.DeleteObject(t, article.Object()) - - res := helper.AssertGetObject(t, article.Class, article.ID, "vector") - assert.EqualValues(t, para1.Vector, res.Vector) - }) - - t.Run("with multiple references", func(t *testing.T) { - article := articles.NewArticle(). - WithTitle("Popularity of the Nissan RB26DETT"). - WithReferences( - &models.SingleRef{Beacon: newBeacon(para1.Class, para1.ID)}, - &models.SingleRef{Beacon: newBeacon(para2.Class, para2.ID)}, - ) - helper.CreateObject(t, article.Object()) - defer helper.DeleteObject(t, article.Object()) - - res := helper.AssertGetObject(t, article.Class, article.ID, "vector") - expectedVec := []float32{3, 5, 7} - assert.EqualValues(t, expectedVec, res.Vector) - }) - }) - - t.Run("create object and PUT references", func(t *testing.T) { - article := articles.NewArticle(). - WithTitle("Popularity of the Nissan RB26DETT") - - helper.CreateObject(t, article.Object()) - defer helper.DeleteObject(t, article.Object()) - - res := helper.AssertGetObject(t, article.Class, article.ID, "vector") - assert.Nil(t, res.Vector) - assert.Equal(t, article.ID, res.ID) - - article.WithReferences( - &models.SingleRef{Beacon: newBeacon(para1.Class, para1.ID)}, - &models.SingleRef{Beacon: newBeacon(para2.Class, para2.ID)}, - ) - - helper.UpdateObject(t, article.Object()) - - res = helper.AssertGetObject(t, article.Class, article.ID, "vector") - assert.Equal(t, article.ID, res.ID) - expectedVec := []float32{3, 5, 7} - assert.EqualValues(t, expectedVec, res.Vector) - }) - - t.Run("create object with references, remove references", func(t *testing.T) { - ref1 := &models.SingleRef{Beacon: newBeacon(para1.Class, para1.ID)} - ref2 := &models.SingleRef{Beacon: newBeacon(para2.Class, para2.ID)} - - article := articles.NewArticle(). - WithTitle("Popularity of the Nissan RB26DETT"). - WithReferences(ref1, ref2) - helper.CreateObject(t, article.Object()) - defer helper.DeleteObject(t, article.Object()) - - res := helper.AssertGetObject(t, article.Class, article.ID, "vector") - expectedVec := []float32{3, 5, 7} - assert.EqualValues(t, expectedVec, res.Vector) - - helper.DeleteReference(t, article.Object(), ref2, "hasParagraphs") - res = helper.AssertGetObject(t, article.Class, article.ID, "vector") - assert.EqualValues(t, para1.Vector, res.Vector) - - helper.DeleteReference(t, article.Object(), ref1, "hasParagraphs") - res = helper.AssertGetObject(t, article.Class, article.ID, "vector") - assert.Nil(t, res.Vector) - }) - - t.Run("create object add references, remove references", func(t *testing.T) { - ref1 := &models.SingleRef{Beacon: newBeacon(para1.Class, para1.ID)} - ref2 := &models.SingleRef{Beacon: newBeacon(para2.Class, para2.ID)} - - article := articles.NewArticle(). - WithTitle("Popularity of the Nissan RB26DETT") - helper.CreateObject(t, article.Object()) - defer helper.DeleteObject(t, article.Object()) - - res := helper.AssertGetObject(t, article.Class, article.ID, "vector") - assert.Nil(t, res.Vector) - - helper.AddReference(t, article.Object(), ref1, "hasParagraphs") - res = helper.AssertGetObject(t, article.Class, article.ID, "vector") - assert.EqualValues(t, para1.Vector, res.Vector) - - helper.AddReference(t, article.Object(), ref2, "hasParagraphs") - res = helper.AssertGetObject(t, article.Class, article.ID, "vector") - assert.EqualValues(t, []float32{3, 5, 7}, res.Vector) - - helper.DeleteReference(t, article.Object(), ref1, "hasParagraphs") - res = helper.AssertGetObject(t, article.Class, article.ID, "vector") - assert.EqualValues(t, para2.Vector, res.Vector) - - helper.DeleteReference(t, article.Object(), ref2, "hasParagraphs") - res = helper.AssertGetObject(t, article.Class, article.ID, "vector") - assert.Nil(t, res.Vector) - }) - - t.Run("batch create objects", func(t *testing.T) { - ref1 := &models.SingleRef{Beacon: newBeacon(para1.Class, para1.ID)} - ref2 := &models.SingleRef{Beacon: newBeacon(para2.Class, para2.ID)} - - article1 := articles.NewArticle(). - WithTitle("Popularity of the Nissan RB26DETT"). - WithReferences(ref1, ref2) - defer helper.DeleteObject(t, article1.Object()) - - article2 := articles.NewArticle(). - WithTitle("A Nissan RB Origin Story"). - WithReferences(ref1, ref2) - defer helper.DeleteObject(t, article2.Object()) - - batch := []*models.Object{article1.Object(), article2.Object()} - helper.CreateObjectsBatch(t, batch) - - res := helper.AssertGetObject(t, article1.Class, article1.ID, "vector") - assert.EqualValues(t, []float32{3, 5, 7}, res.Vector) - res = helper.AssertGetObject(t, article2.Class, article2.ID, "vector") - assert.EqualValues(t, []float32{3, 5, 7}, res.Vector) - }) - - // TODO: Uncomment when batch refs supports centroid re-calc - //t.Run("batch create references", func(t *testing.T) { - // article := articles.NewArticle(). - // WithTitle("Popularity of the Nissan RB26DETT") - // defer helper.DeleteObject(t, article.Object()) - // - // refs := []*models.BatchReference{ - // { - // From: strfmt.URI(crossref.NewSource("Article", "hasParagraphs", article.ID).String()), - // To: strfmt.URI(crossref.NewLocalhost("Paragraph", para1.ID).String()), - // }, - // { - // From: strfmt.URI(crossref.NewSource("Article", "hasParagraphs", article.ID).String()), - // To: strfmt.URI(crossref.NewLocalhost("Paragraph", para2.ID).String()), - // }, - // } - // - // helper.AddReferences(t, refs) - // res := helper.AssertGetObject(t, article.Class, article.ID, "vector") - // assert.EqualValues(t, []float32{3, 5, 7}, res.Vector) - //}) -} - -func newBeacon(className string, id strfmt.UUID) strfmt.URI { - return crossref.New("localhost", className, id).SingleRef().Beacon -} diff --git a/test/modules/ref2vec-centroid/setup_test.go b/test/modules/ref2vec-centroid/setup_test.go deleted file mode 100644 index b7934c6ce1fd8c5ef00a417855d5d3ec306eb95e..0000000000000000000000000000000000000000 --- a/test/modules/ref2vec-centroid/setup_test.go +++ /dev/null @@ -1,42 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "context" - "os" - "testing" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/test/docker" -) - -const weaviateEndpoint = "WEAVIATE_ENDPOINT" - -func TestMain(m *testing.M) { - ctx := context.Background() - compose, err := docker.New(). - WithWeaviate().WithRef2VecCentroid(). - Start(ctx) - if err != nil { - panic(errors.Wrapf(err, "cannot start")) - } - - os.Setenv(weaviateEndpoint, compose.GetWeaviate().URI()) - code := m.Run() - - if err := compose.Terminate(ctx); err != nil { - panic(errors.Wrapf(err, "cannot terminate")) - } - - os.Exit(code) -} diff --git a/test/modules/reranker-transformers/reranker_transformers_test.go b/test/modules/reranker-transformers/reranker_transformers_test.go deleted file mode 100644 index 1c4f5e2d200ba28f21807ab28167e6ab63912464..0000000000000000000000000000000000000000 --- a/test/modules/reranker-transformers/reranker_transformers_test.go +++ /dev/null @@ -1,71 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "os" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/test/helper" - graphqlhelper "github.com/weaviate/weaviate/test/helper/graphql" - "github.com/weaviate/weaviate/test/helper/sample-schema/books" -) - -func TestRerankerTransformers(t *testing.T) { - helper.SetupClient(os.Getenv(weaviateEndpoint)) - booksClass := books.ClassContextionaryVectorizer() - helper.CreateClass(t, booksClass) - defer helper.DeleteClass(t, booksClass.Class) - - t.Run("import data", func(t *testing.T) { - for _, book := range books.Objects() { - helper.CreateObject(t, book) - helper.AssertGetObjectEventually(t, book.Class, book.ID) - } - }) - - t.Run("rerank", func(t *testing.T) { - query := ` - { - Get { - Books{ - title - _additional{ - id - rerank(property:"description", query: "Who is the author of Dune?") { - score - } - } - } - } - }` - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - booksResponse := result.Get("Get", "Books").AsSlice() - require.True(t, len(booksResponse) > 0) - results, ok := booksResponse[0].(map[string]interface{}) - require.True(t, ok) - assert.True(t, results["title"] != nil) - assert.NotNil(t, results["_additional"]) - additional, ok := results["_additional"].(map[string]interface{}) - require.True(t, ok) - assert.Equal(t, books.Dune.String(), additional["id"]) - assert.NotNil(t, additional["rerank"]) - rerank, ok := additional["rerank"].([]interface{}) - require.True(t, ok) - score, ok := rerank[0].(map[string]interface{}) - require.True(t, ok) - require.NotNil(t, score) - assert.NotNil(t, score["score"]) - }) -} diff --git a/test/modules/reranker-transformers/setup_test.go b/test/modules/reranker-transformers/setup_test.go deleted file mode 100644 index 1159544ff034891ad4c87720f2392c067e126b35..0000000000000000000000000000000000000000 --- a/test/modules/reranker-transformers/setup_test.go +++ /dev/null @@ -1,44 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "context" - "os" - "testing" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/test/docker" -) - -const weaviateEndpoint = "WEAVIATE_ENDPOINT" - -func TestMain(m *testing.M) { - ctx := context.Background() - compose, err := docker.New(). - WithWeaviate(). - WithText2VecContextionary(). - WithRerankerTransformers(). - Start(ctx) - if err != nil { - panic(errors.Wrapf(err, "cannot start")) - } - - os.Setenv(weaviateEndpoint, compose.GetWeaviate().URI()) - code := m.Run() - - if err := compose.Terminate(ctx); err != nil { - panic(errors.Wrapf(err, "cannot terminate")) - } - - os.Exit(code) -} diff --git a/test/modules/sum-transformers/setup_test.go b/test/modules/sum-transformers/setup_test.go deleted file mode 100644 index 36b17ad26b04a4560929a04706d20de20907d316..0000000000000000000000000000000000000000 --- a/test/modules/sum-transformers/setup_test.go +++ /dev/null @@ -1,42 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "context" - "os" - "testing" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/test/docker" -) - -const weaviateEndpoint = "WEAVIATE_ENDPOINT" - -func TestMain(m *testing.M) { - ctx := context.Background() - compose, err := docker.New(). - WithWeaviate().WithText2VecContextionary().WithSUMTransformers(). - Start(ctx) - if err != nil { - panic(errors.Wrapf(err, "cannot start")) - } - - os.Setenv(weaviateEndpoint, compose.GetWeaviate().URI()) - code := m.Run() - - if err := compose.Terminate(ctx); err != nil { - panic(errors.Wrapf(err, "cannot terminate")) - } - - os.Exit(code) -} diff --git a/test/modules/sum-transformers/sum_test.go b/test/modules/sum-transformers/sum_test.go deleted file mode 100644 index aa0b988ef44c1778c3351aed1e1199f4305ae4b7..0000000000000000000000000000000000000000 --- a/test/modules/sum-transformers/sum_test.go +++ /dev/null @@ -1,93 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "os" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/test/helper" - graphqlhelper "github.com/weaviate/weaviate/test/helper/graphql" - "github.com/weaviate/weaviate/test/helper/sample-schema/books" -) - -func Test_SUMTransformers(t *testing.T) { - helper.SetupClient(os.Getenv(weaviateEndpoint)) - tests := []struct { - name string - class *models.Class - }{ - { - name: "with module config for sum-transformers module", - class: books.ClassContextionaryVectorizerWithSumTransformers(), - }, - { - name: "without module config", - class: books.ClassContextionaryVectorizer(), - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - booksClass := tt.class - helper.CreateClass(t, booksClass) - defer helper.DeleteClass(t, booksClass.Class) - - t.Run("add data to Books schema", func(t *testing.T) { - for _, book := range books.Objects() { - helper.CreateObject(t, book) - helper.AssertGetObjectEventually(t, book.Class, book.ID) - } - }) - - t.Run("query Books data with nearText", func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, ` - { - Get { - Books(where: { - operator: Equal - path:["title"] - valueText: "Dune" - }){ - title - _additional { - summary (properties:["description"]) { - property - result - } - } - } - } - } - `) - books := result.Get("Get", "Books").AsSlice() - expected := []interface{}{ - map[string]interface{}{ - "title": "Dune", - "_additional": map[string]interface{}{ - "summary": []interface{}{ - map[string]interface{}{ - "property": "description", - "result": "Dune is a 1965 epic science fiction novel by American author Frank Herbert." + - "It is the first novel in the Dune series by Frank Herbert, and the first in the \"Dune\" series of books." + - "It was published in the United States by Simon & Schuster in 1965.", - }, - }, - }, - }, - } - assert.ElementsMatch(t, expected, books) - }) - }) - } -} diff --git a/test/modules/text2vec-contextionary/cluster_nodes_api_test.go b/test/modules/text2vec-contextionary/cluster_nodes_api_test.go deleted file mode 100644 index 953ca2097a0d844cbf8947c4213ace866f4c121c..0000000000000000000000000000000000000000 --- a/test/modules/text2vec-contextionary/cluster_nodes_api_test.go +++ /dev/null @@ -1,94 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "os" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/weaviate/weaviate/client/nodes" - "github.com/weaviate/weaviate/entities/models" - "github.com/weaviate/weaviate/entities/verbosity" - "github.com/weaviate/weaviate/test/helper" - "github.com/weaviate/weaviate/test/helper/sample-schema/books" - "github.com/weaviate/weaviate/test/helper/sample-schema/multishard" -) - -func Test_WeaviateCluster_NodesAPI(t *testing.T) { - helper.SetupClient(os.Getenv(weaviateNode1Endpoint)) - booksClass := books.ClassContextionaryVectorizer() - multiShardClass := multishard.ClassContextionaryVectorizer() - helper.CreateClass(t, booksClass) - helper.CreateClass(t, multiShardClass) - defer helper.DeleteClass(t, booksClass.Class) - defer helper.DeleteClass(t, multiShardClass.Class) - - t.Run("import data", func(t *testing.T) { - for _, book := range books.Objects() { - helper.CreateObject(t, book) - helper.AssertGetObjectEventually(t, book.Class, book.ID) - } - for _, multishard := range multishard.Objects() { - helper.CreateObject(t, multishard) - helper.AssertGetObjectEventually(t, multishard.Class, multishard.ID) - } - }) - - t.Run("check nodes api", func(t *testing.T) { - for _, endpoint := range []string{weaviateNode1Endpoint, weaviateNode2Endpoint} { - t.Run(endpoint, func(t *testing.T) { - helper.SetupClient(os.Getenv(endpoint)) - verbose := verbosity.OutputVerbose - params := nodes.NewNodesGetParams().WithOutput(&verbose) - resp, err := helper.Client(t).Nodes.NodesGet(params, nil) - require.Nil(t, err) - - nodeStatusResp := resp.GetPayload() - require.NotNil(t, nodeStatusResp) - - nodes := nodeStatusResp.Nodes - require.NotNil(t, nodes) - require.Len(t, nodes, 2) - - assert.Equal(t, "node1", nodes[0].Name) - assert.Equal(t, "node2", nodes[1].Name) - - for i, nodeStatus := range nodes { - require.NotNil(t, nodeStatus) - assert.Equal(t, models.NodeStatusStatusHEALTHY, *nodeStatus.Status) - if i == 0 { - assert.Equal(t, "node1", nodeStatus.Name) - } else { - assert.Equal(t, "node2", nodeStatus.Name) - } - assert.True(t, nodeStatus.GitHash != "" && nodeStatus.GitHash != "unknown") - assert.Len(t, nodeStatus.Shards, 2) - var objectCount int64 - var shardCount int64 - for _, shard := range nodeStatus.Shards { - assert.True(t, len(shard.Name) > 0) - assert.True(t, shard.Class == multiShardClass.Class || shard.Class == booksClass.Class) - assert.GreaterOrEqual(t, shard.ObjectCount, int64(0)) - assert.GreaterOrEqual(t, shard.VectorQueueLength, int64(0)) - objectCount += shard.ObjectCount - shardCount++ - } - require.NotNil(t, nodeStatus.Stats) - assert.Equal(t, objectCount, nodeStatus.Stats.ObjectCount) - assert.Equal(t, shardCount, nodeStatus.Stats.ShardCount) - } - }) - } - }) -} diff --git a/test/modules/text2vec-contextionary/group_by_cluster_test.go b/test/modules/text2vec-contextionary/group_by_cluster_test.go deleted file mode 100644 index cdf2ed59aea69a5900b8cf277507d21ab198ed5b..0000000000000000000000000000000000000000 --- a/test/modules/text2vec-contextionary/group_by_cluster_test.go +++ /dev/null @@ -1,25 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "os" - "testing" - - "github.com/weaviate/weaviate/test/helper/journey" -) - -func Test_WeaviateCluster_GroupBy(t *testing.T) { - t.Run("multi node", func(t *testing.T) { - journey.GroupBySingleAndMultiShardTests(t, os.Getenv(weaviateNode1Endpoint)) - }) -} diff --git a/test/modules/text2vec-contextionary/near_text_test.go b/test/modules/text2vec-contextionary/near_text_test.go deleted file mode 100644 index aae0a8b94c5323613a40968af9f49454b9da63f3..0000000000000000000000000000000000000000 --- a/test/modules/text2vec-contextionary/near_text_test.go +++ /dev/null @@ -1,94 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "os" - "reflect" - "testing" - - "github.com/weaviate/weaviate/test/helper" - graphqlhelper "github.com/weaviate/weaviate/test/helper/graphql" - "github.com/weaviate/weaviate/test/helper/sample-schema/books" - "github.com/weaviate/weaviate/test/helper/sample-schema/multishard" -) - -func Test_Text2Vec_NearText(t *testing.T) { - helper.SetupClient(os.Getenv(weaviateNode1Endpoint)) - booksClass := books.ClassContextionaryVectorizer() - multiShardClass := multishard.ClassContextionaryVectorizer() - helper.CreateClass(t, booksClass) - helper.CreateClass(t, multiShardClass) - defer helper.DeleteClass(t, booksClass.Class) - defer helper.DeleteClass(t, multiShardClass.Class) - - t.Run("import data", func(t *testing.T) { - for _, book := range books.Objects() { - helper.CreateObject(t, book) - helper.AssertGetObjectEventually(t, book.Class, book.ID) - } - for _, multi := range multishard.Objects() { - helper.CreateObject(t, multi) - helper.AssertGetObjectEventually(t, multi.Class, multi.ID) - } - }) - - t.Run("nearText with sorting, asc", func(t *testing.T) { - query := ` - { - Get { - Books( - nearText: {concepts: ["novel"]} - sort: [{path: ["title"], order: asc}] - ) { - title - } - } - }` - - expected := []interface{}{ - map[string]interface{}{"title": "Dune"}, - map[string]interface{}{"title": "Project Hail Mary"}, - map[string]interface{}{"title": "The Lord of the Ice Garden"}, - } - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - received := result.Get("Get", "Books").AsSlice() - if !reflect.DeepEqual(expected, received) { - t.Errorf("sort objects expected = %v, received %v", expected, received) - } - }) - - t.Run("nearText with sorting, desc", func(t *testing.T) { - query := ` - { - Get { - Books( - nearText: {concepts: ["novel"]} - sort: [{path: ["title"], order: desc}] - ) { - title - } - } - }` - - expected := []interface{}{ - map[string]interface{}{"title": "The Lord of the Ice Garden"}, - map[string]interface{}{"title": "Project Hail Mary"}, - map[string]interface{}{"title": "Dune"}, - } - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, query) - received := result.Get("Get", "Books").AsSlice() - if !reflect.DeepEqual(expected, received) { - t.Errorf("sort objects expected = %v, received %v", expected, received) - } - }) -} diff --git a/test/modules/text2vec-contextionary/setup_test.go b/test/modules/text2vec-contextionary/setup_test.go deleted file mode 100644 index 10f5b9dba6735c76a8bb1e3967a70b041be8d284..0000000000000000000000000000000000000000 --- a/test/modules/text2vec-contextionary/setup_test.go +++ /dev/null @@ -1,46 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "context" - "os" - "testing" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/test/docker" -) - -const ( - weaviateNode1Endpoint = "WEAVIATE1_ENDPOINT" - weaviateNode2Endpoint = "WEAVIATE2_ENDPOINT" -) - -func TestMain(m *testing.M) { - ctx := context.Background() - compose, err := docker.New(). - WithWeaviateCluster().WithText2VecContextionary(). - Start(ctx) - if err != nil { - panic(errors.Wrapf(err, "cannot start")) - } - - os.Setenv(weaviateNode1Endpoint, compose.GetWeaviate().URI()) - os.Setenv(weaviateNode2Endpoint, compose.GetWeaviateNode2().URI()) - code := m.Run() - - if err := compose.Terminate(ctx); err != nil { - panic(errors.Wrapf(err, "cannot terminate")) - } - - os.Exit(code) -} diff --git a/test/modules/text2vec-transformers/setup_test.go b/test/modules/text2vec-transformers/setup_test.go deleted file mode 100644 index 5c36ff6fd8af3d54c5f18771f040ecdf1630f442..0000000000000000000000000000000000000000 --- a/test/modules/text2vec-transformers/setup_test.go +++ /dev/null @@ -1,42 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "context" - "os" - "testing" - - "github.com/pkg/errors" - "github.com/weaviate/weaviate/test/docker" -) - -const weaviateEndpoint = "WEAVIATE_ENDPOINT" - -func TestMain(m *testing.M) { - ctx := context.Background() - compose, err := docker.New(). - WithWeaviate().WithText2VecTransformers(). - Start(ctx) - if err != nil { - panic(errors.Wrapf(err, "cannot start")) - } - - os.Setenv(weaviateEndpoint, compose.GetWeaviate().URI()) - code := m.Run() - - if err := compose.Terminate(ctx); err != nil { - panic(errors.Wrapf(err, "cannot terminate")) - } - - os.Exit(code) -} diff --git a/test/modules/text2vec-transformers/transformers_test.go b/test/modules/text2vec-transformers/transformers_test.go deleted file mode 100644 index 115195ce4ef0ad1be62d119a2d4b285c1628e36e..0000000000000000000000000000000000000000 --- a/test/modules/text2vec-transformers/transformers_test.go +++ /dev/null @@ -1,58 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package test - -import ( - "os" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/weaviate/weaviate/test/helper" - graphqlhelper "github.com/weaviate/weaviate/test/helper/graphql" - "github.com/weaviate/weaviate/test/helper/sample-schema/books" -) - -func Test_T2VTransformers(t *testing.T) { - helper.SetupClient(os.Getenv(weaviateEndpoint)) - booksClass := books.ClassTransformersVectorizer() - helper.CreateClass(t, booksClass) - defer helper.DeleteClass(t, booksClass.Class) - - t.Run("add data to Books schema", func(t *testing.T) { - for _, book := range books.Objects() { - helper.CreateObject(t, book) - helper.AssertGetObjectEventually(t, book.Class, book.ID) - } - }) - - t.Run("query Books data with nearText", func(t *testing.T) { - result := graphqlhelper.AssertGraphQL(t, helper.RootAuth, ` - { - Get { - Books( - nearText: { - concepts: ["Frank Herbert"] - distance: 0.3 - } - ){ - title - } - } - } - `) - books := result.Get("Get", "Books").AsSlice() - expected := []interface{}{ - map[string]interface{}{"title": "Dune"}, - } - assert.ElementsMatch(t, expected, books) - }) -} diff --git a/test/run.sh b/test/run.sh deleted file mode 100644 index 09b0620f2e1a5fe1df5966e954d99901b07545e7..0000000000000000000000000000000000000000 --- a/test/run.sh +++ /dev/null @@ -1,184 +0,0 @@ -#!/bin/bash - -set -eou pipefail - -function main() { - # This script runs all non-benchmark tests if no CMD switch is given and the respective tests otherwise. - run_all_tests=true - run_acceptance_tests=false - run_module_tests=false - run_unit_and_integration_tests=false - run_unit_tests=false - run_integration_tests=false - run_benchmark=false - run_module_only_backup_tests=false - run_module_except_backup_tests=false - - while [[ "$#" -gt 0 ]]; do - case $1 in - --unit-only|-u) run_all_tests=false; run_unit_tests=true;; - --unit-and-integration-only|-ui) run_all_tests=false; run_unit_and_integration_tests=true;; - --integration-only|-i) run_all_tests=false; run_integration_tests=true;; - --acceptance-only|--e2e-only|-a) run_all_tests=false; run_acceptance_tests=true ;; - --acceptance-module-tests-only|--modules-only|-m) run_all_tests=false; run_module_tests=true; run_module_only_backup_tests=true; run_module_except_backup_tests=true;; - --acceptance-module-tests-only-backup|--modules-backup-only|-mob) run_all_tests=false; run_module_tests=true; run_module_only_backup_tests=true;; - --acceptance-module-tests-except-backup|--modules-except-backup|-meb) run_all_tests=false; run_module_tests=true; run_module_except_backup_tests=true; echo $run_module_except_backup_tests ;; - --benchmark-only|-b) run_all_tests=false; run_benchmark=true;; - --help|-h) printf '%s\n' \ - "Options:"\ - "--unit-only | -u"\ - "--unit-and-integration-only | -ui"\ - "--integration-only | -i"\ - "--acceptance-only | -a"\ - "--acceptance-module-tests-only | --modules-only | -m"\ - "--acceptance-module-tests-only-backup | --modules-backup-only | -mob"\ - "--acceptance-module-tests-except-backup | --modules-except-backup | -meb"\ - "--benchmark-only | -b" \ - "--help | -h"; exit 1;; - *) echo "Unknown parameter passed: $1"; exit 1 ;; - esac - shift - done - - # Jump to root directory - cd "$( dirname "${BASH_SOURCE[0]}" )"/.. - - echo "INFO: In directory $PWD" - - echo "INFO: This script will suppress most output, unless a command ultimately fails" - echo " Then it will print the output of the failed command." - - echo_green "Prepare workspace..." - - # Remove data directory in case of previous runs - rm -rf data - echo "Done!" - - if $run_unit_and_integration_tests || $run_unit_tests || $run_all_tests - then - echo_green "Run all unit tests..." - run_unit_tests "$@" - echo_green "Unit tests successful" - fi - - if $run_unit_and_integration_tests || $run_integration_tests || $run_all_tests - then - echo_green "Run integration tests..." - run_integration_tests "$@" - echo_green "Integration tests successful" - fi - - if $run_acceptance_tests || $run_all_tests || $run_benchmark - then - echo "Start docker container needed for acceptance and/or benchmark test" - echo_green "Stop any running docker-compose containers..." - suppress_on_success docker compose -f docker-compose-test.yml down --remove-orphans - - echo_green "Start up weaviate and backing dbs in docker-compose..." - echo "This could take some time..." - tools/test/run_ci_server.sh - - # echo_green "Import required schema and test fixtures..." - # # Note: It's not best practice to do this as part of the test script - # # It would be better if each test independently prepared (and also - # # cleaned up) the test fixtures it needs, but one step at a time ;) - # suppress_on_success import_test_fixtures - - if $run_benchmark - then - echo_green "Run performance tracker..." - ./test/benchmark/run_performance_tracker.sh - fi - - if $run_acceptance_tests || $run_all_tests - then - echo_green "Run acceptance tests..." - run_acceptance_tests "$@" - fi - fi - - if $run_module_tests; then - local module_test_image=weaviate:module-tests - echo_green "Running module acceptance tests..." - echo_green "Stop any running docker-compose containers..." - suppress_on_success docker compose -f docker-compose-test.yml down --remove-orphans - echo_green "Building weaviate image for module acceptance tests..." - echo "This could take some time..." - GIT_HASH=$(git rev-parse --short HEAD) - docker build --build-arg GITHASH="$GIT_HASH" -t $module_test_image . - export "TEST_WEAVIATE_IMAGE"=$module_test_image - - echo_green "Weaviate image successfully built, run module tests..." - run_module_tests "$@" - echo_green "Module acceptance tests successful" - fi - - echo "Done!" -} - -function run_unit_tests() { - if [[ "$*" == *--acceptance-only* ]]; then - echo "Skipping unit test" - return - fi - go test -race -coverprofile=coverage-unit.txt -covermode=atomic -count 1 $(go list ./... | grep -v 'test/acceptance' | grep -v 'test/modules') | grep -v '\[no test files\]' -} - -function run_integration_tests() { - if [[ "$*" == *--acceptance-only* ]]; then - echo "Skipping integration test" - return - fi - - ./test/integration/run.sh --include-slow -} - -function run_acceptance_tests() { - ./test/acceptance/run.sh --include-slow -} - -function run_module_only_backup_tests() { - for pkg in $(go list ./... | grep 'test/modules' | grep 'test/modules/backup'); do - if ! go test -count 1 -race "$pkg"; then - echo "Test for $pkg failed" >&2 - return 1 - fi - done -} - -function run_module_except_backup_tests() { - for pkg in $(go list ./... | grep 'test/modules' | grep -v 'test/modules/backup'); do - if ! go test -count 1 -race "$pkg"; then - echo "Test for $pkg failed" >&2 - return 1 - fi - done -} - -function run_module_tests() { - if $run_module_only_backup_tests; then - run_module_only_backup_tests "$@" - fi - if $run_module_except_backup_tests; then - run_module_except_backup_tests "$@" - fi -} - -suppress_on_success() { - out="$("${@}" 2>&1)" || { echo_red "FAILED!"; echo "$out"; return 1; } - echo "Done!" -} - -function echo_green() { - green='\033[0;32m' - nc='\033[0m' - echo -e "${green}${*}${nc}" -} - -function echo_red() { - red='\033[0;31m' - nc='\033[0m' - echo -e "${red}${*}${nc}" -} - -main "$@" diff --git a/tools/.gitignore b/tools/.gitignore deleted file mode 100644 index 1c65a49c519a83968289220ad1fad07a834f44e2..0000000000000000000000000000000000000000 --- a/tools/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/swagger-* diff --git a/tools/dev/bench/bench_test.go b/tools/dev/bench/bench_test.go deleted file mode 100644 index 585ee7d73bff83f46eb541b74c4d0f7b4a4f97b1..0000000000000000000000000000000000000000 --- a/tools/dev/bench/bench_test.go +++ /dev/null @@ -1,64 +0,0 @@ -// _ _ -// __ _____ __ ___ ___ __ _| |_ ___ -// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \ -// \ V V / __/ (_| |\ V /| | (_| | || __/ -// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___| -// -// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved. -// -// CONTACT: hello@weaviate.io -// - -package main - -import ( - "fmt" - "testing" - "time" -) - -var size = int(1e7) - -func TestVisitedMap(t *testing.T) { - numbers := make([]int, size) - for i := range numbers { - numbers[i] = i - } - - before := time.Now() - numbersContained := map[int]struct{}{} - fmt.Printf("init map took %s\n", time.Since(before)) - - before = time.Now() - for i := range numbers { - _, ok := numbersContained[i] - _ = ok - } - - fmt.Printf("map lookups took %s\n", time.Since(before)) -} - -func TestVisitedList(t *testing.T) { - numbers := make([]int, size) - for i := range numbers { - numbers[i] = i - } - - before := time.Now() - numbersContained := make([]bool, size) - fmt.Printf("init slice took %s\n", time.Since(before)) - - for i := range numbers { - if i%150 == 0 { - // contained := true - numbersContained[i] = true - } - } - - before = time.Now() - for i := range numbers { - el := numbersContained[i] - _ = el - } - fmt.Printf("slice lookups took %s\n", time.Since(before)) -} diff --git a/tools/dev/bench/measure_start_time.py b/tools/dev/bench/measure_start_time.py deleted file mode 100644 index d3cccbb3d13c3a520c6347f0b83bcfd449fbc0e2..0000000000000000000000000000000000000000 --- a/tools/dev/bench/measure_start_time.py +++ /dev/null @@ -1,32 +0,0 @@ -import subprocess -import time - -# This measures the start time of weaviate -# -# How to use: -# - Build a container of the weaviate code that you want to benchmark, this script assumes that you use -# ./tools/test/run_ci_server.sh -# - Import objects -# - Run this script, it will automatically stop/start the weaviate docker container multiple times and measure how long -# it takes - -runs = 10 -container_name = "weaviate-weaviate-1" -total_time = 0 - -# stop any running weaviate containers -subprocess.run(["docker", "stop", container_name]) - -for _ in range(runs): - start = time.time() - subprocess.run(["docker", "start", container_name]) - - # Loop until weaviate is responding - while True: - n = subprocess.run(["curl", "-s", "http://localhost:8080"]) - if n.returncode == 0: - total_time += time.time()-start - subprocess.run(["docker", "stop", container_name]) - break - -print("Average time until weaviate responds: ", total_time/runs) diff --git a/tools/dev/config.docker.yaml b/tools/dev/config.docker.yaml deleted file mode 100644 index 873422449dec6075dfbbdb81ae3fdbad84dfce97..0000000000000000000000000000000000000000 --- a/tools/dev/config.docker.yaml +++ /dev/null @@ -1,26 +0,0 @@ -authentication: - anonymous_access: - enabled: true -configuration_storage: - type: etcd - url: http://etcd:2379 -vector_index: - enabled: true - url: http://esvector:9200 - denormalizationDepth: 3 -contextionary: - url: contextionary:9999 -network: - genesis_url: http://genesis_fake:8090 - public_url: http://localhost:8080 - peer_name: bestWeaviate -query_defaults: - limit: 100 -logging: - interval: 1 - enabled: true - url: http://telemetry_mock_api:8087/mock/new -telemetry: - interval: 1 - disabled: false - remote_url: http://telemetry_mock_api:8087/mock/new diff --git a/tools/dev/config.local-customdb.yaml b/tools/dev/config.local-customdb.yaml deleted file mode 100644 index 0d440832f459b749b3f84355f6e46558f75966fe..0000000000000000000000000000000000000000 --- a/tools/dev/config.local-customdb.yaml +++ /dev/null @@ -1,30 +0,0 @@ -authentication: - anonymous_access: - enabled: true -vector_index: - enabled: true - url: http://localhost:9201 - denormalizationDepth: 2 -configuration_storage: - type: etcd - url: http://localhost:2379 -persistence: - dataPath: "./data" -contextionary: - url: localhost:9999 -query_defaults: - limit: 100 -debug: true -logging: - interval: 1 - enabled: false - url: http://telemetry_mock_api:8087/mock/new -# network: -# genesis_url: http://localhost:8090 -# public_url: http://localhost:8080 -# peer_name: bestWeaviate -telemetry: - disabled: true - -# feature toggle -customdb: true diff --git a/tools/dev/config.local-development.yaml b/tools/dev/config.local-development.yaml deleted file mode 100644 index 7252d775ae93be49b6933fb35762c90c76ac0493..0000000000000000000000000000000000000000 --- a/tools/dev/config.local-development.yaml +++ /dev/null @@ -1,30 +0,0 @@ -authentication: - anonymous_access: - enabled: true -vector_index: - enabled: true - url: http://localhost:9201 - denormalizationDepth: 2 - numberOfShards: 1 -analytics_engine: - enabled: true - defaultUseAnalyticsEngine: false -configuration_storage: - type: etcd - url: http://localhost:2379 -contextionary: - url: localhost:9999 -query_defaults: - limit: 20 -debug: true -logging: - interval: 1 - enabled: false - url: http://telemetry_mock_api:8087/mock/new -# network: -# genesis_url: http://localhost:8090 -# public_url: http://localhost:8080 -# peer_name: bestWeaviate -telemetry: - disabled: true -origin: http://localhost:8080 diff --git a/tools/dev/config.local-esvector-only.yaml b/tools/dev/config.local-esvector-only.yaml deleted file mode 100644 index 2c522f009b48ae9a60862cffebd68bd0707a0fc8..0000000000000000000000000000000000000000 --- a/tools/dev/config.local-esvector-only.yaml +++ /dev/null @@ -1,28 +0,0 @@ -authentication: - anonymous_access: - enabled: true -vector_index: - enabled: true - url: http://localhost:9201 - denormalizationDepth: 2 -configuration_storage: - type: etcd - url: http://localhost:2379 -contextionary: - url: localhost:9999 -query_defaults: - limit: 100 -debug: true -logging: - interval: 1 - enabled: false - url: http://telemetry_mock_api:8087/mock/new -# network: -# genesis_url: http://localhost:8090 -# public_url: http://localhost:8080 -# peer_name: bestWeaviate -telemetry: - disabled: true - -# feature toggle -esvectorOnly: true diff --git a/tools/dev/config.local-oidc.yaml b/tools/dev/config.local-oidc.yaml deleted file mode 100644 index 6f16b5c6027b00d8b656660b97a0df9a1195a507..0000000000000000000000000000000000000000 --- a/tools/dev/config.local-oidc.yaml +++ /dev/null @@ -1,37 +0,0 @@ -vector_index: - enabled: true - url: http://localhost:9201 - denormalizationDepth: 2 -authentication: - anonymous_access: - enabled: false - oidc: - enabled: true - issuer: http://localhost:9090/auth/realms/weaviate - username_claim: email - groups_claim: groups - client_id: demo - skip_client_id_check: false -authorization: - admin_list: - enabled: true - users: - read_only_users: - - john@doe.com - groups: - read_only_groups: - generic -configuration_storage: - type: etcd - url: http://localhost:2379 -contextionary: - url: localhost:9999 -query_defaults: - limit: 20 -debug: true -logging: - interval: 1 - enabled: false - url: http://telemetry_mock_api:8087/mock/new -telemetry: - disabled: true diff --git a/tools/dev/config.readonly.yaml b/tools/dev/config.readonly.yaml deleted file mode 100644 index 13745be596a7815e6e3429ad90c1d6556255f169..0000000000000000000000000000000000000000 --- a/tools/dev/config.readonly.yaml +++ /dev/null @@ -1,38 +0,0 @@ -authentication: - anonymous_access: - enabled: true -authorization: - admin_list: - enabled: true - users: [] - read_only_users: - - anonymous - groups: [] - read_only_groups: - - generic -vector_index: - enabled: true - url: http://localhost:9201 - denormalizationDepth: 2 - numberOfShards: 1 -analytics_engine: - enabled: true - defaultUseAnalyticsEngine: false -configuration_storage: - type: etcd - url: http://localhost:2379 -contextionary: - url: localhost:9999 -query_defaults: - limit: 20 -debug: true -logging: - interval: 1 - enabled: false - url: http://telemetry_mock_api:8087/mock/new -# network: -# genesis_url: http://localhost:8090 -# public_url: http://localhost:8080 -# peer_name: bestWeaviate -telemetry: - disabled: true diff --git a/tools/dev/config.wcsauth-dev.yaml b/tools/dev/config.wcsauth-dev.yaml deleted file mode 100644 index c9fbe7aab2d717037361979abc74f2f742f70b62..0000000000000000000000000000000000000000 --- a/tools/dev/config.wcsauth-dev.yaml +++ /dev/null @@ -1,36 +0,0 @@ -authentication: - anonymous_access: - enabled: false - oidc: - enabled: true - issuer: https://auth.dev.wcs.api.semi.technology/auth/realms/SeMI - username_claim: email - groups_claim: groups - client_id: wcs - skip_client_id_check: false -vector_index: - enabled: true - url: http://localhost:9201 - denormalizationDepth: 2 - numberOfShards: 1 -analytics_engine: - enabled: true - defaultUseAnalyticsEngine: false -configuration_storage: - type: etcd - url: http://localhost:2379 -contextionary: - url: localhost:9999 -query_defaults: - limit: 20 -debug: true -logging: - interval: 1 - enabled: false - url: http://telemetry_mock_api:8087/mock/new -# network: -# genesis_url: http://localhost:8090 -# public_url: http://localhost:8080 -# peer_name: bestWeaviate -telemetry: - disabled: true diff --git a/tools/dev/flakyness.sh b/tools/dev/flakyness.sh deleted file mode 100644 index 696441eecd5eca37c36af1e35086f86ff8c9bf6c..0000000000000000000000000000000000000000 --- a/tools/dev/flakyness.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -# Jump to root directory -cd "$( dirname "${BASH_SOURCE[0]}" )"/../.. || exit 1 - -success=0 -fail=0 - -for _ in {1..50}; do - if go test -count 1 ./test/acceptance/graphql_resolvers/...; then - success=$((success+1)) - else - fail=$((fail+1)) - fi -done - -echo "Success: $success" -echo "Failure: $fail" diff --git a/tools/dev/goreleaser_and_sign.sh b/tools/dev/goreleaser_and_sign.sh deleted file mode 100644 index f73e4190e4ad41945dc780574da4d40cb144f754..0000000000000000000000000000000000000000 --- a/tools/dev/goreleaser_and_sign.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -set -eou pipefail - -# run from Weaviate Root! -# -# This script calls goreleaser and signs + notarize the binaries for macos - -export GIT_HASH=$(git rev-parse --short HEAD) - -# detect if commit is tagged or not (format is "vA.B.Z" with tag and "vA.B.Z-commit" without tag) -VERSION="$(git describe --tag)" -if [[ "$VERSION" == *"-"* ]]; then - goreleaser build --clean --snapshot -else - goreleaser build --clean -fi - -codesign -f -o runtime --timestamp -s "Developer ID Application: Weaviate B.V. (QUZ8SKLS6R)" dist/weaviate_darwin_all/weaviate - -DARWIN_DIST="dist/weaviate-${VERSION}-darwin-all.zip" -zip -j "$DARWIN_DIST" dist/weaviate_darwin_all/weaviate LICENSE README.md - -codesign -f -o runtime --timestamp -s "Developer ID Application: Weaviate B.V. (QUZ8SKLS6R)" "$DARWIN_DIST" - -xcrun notarytool submit "$DARWIN_DIST" --keychain-profile "AC_PASSWORD" --wait - - -# add checksums -shasum -a 256 "$DARWIN_DIST" | cut -d ' ' -f 1 > "${DARWIN_DIST}.sha256" -md5 "$DARWIN_DIST" | cut -d ' ' -f 4 > "${DARWIN_DIST}.md5" \ No newline at end of file diff --git a/tools/dev/grafana/dashboard_provider.yml b/tools/dev/grafana/dashboard_provider.yml deleted file mode 100644 index f590945eb17bb55d29f67a494368aaa6e75cfc75..0000000000000000000000000000000000000000 --- a/tools/dev/grafana/dashboard_provider.yml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: 1 - -providers: - # an unique provider name. Required - - name: 'weaviate dashboard provider' - # Org id. Default to 1 - orgId: 1 - # name of the dashboard folder. - folder: '' - # folder UID. will be automatically generated if not specified - folderUid: '' - # provider type. Default to 'file' - type: file - # disable dashboard deletion - disableDeletion: true - # how often Grafana will scan for changed dashboards - updateIntervalSeconds: 10 - # allow updating provisioned dashboards from the UI - allowUiUpdates: false - options: - # path to dashboard files on disk. Required when using the 'file' type - path: /var/lib/grafana/dashboards - # use folder names from filesystem to create folders in Grafana - foldersFromFilesStructure: true diff --git a/tools/dev/grafana/dashboards/backup.json b/tools/dev/grafana/dashboards/backup.json deleted file mode 100644 index 9c4b0beafc241ddb4530d7aae959dadb58671fb7..0000000000000000000000000000000000000000 --- a/tools/dev/grafana/dashboards/backup.json +++ /dev/null @@ -1,446 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "links": [], - "liveNow": false, - "panels": [ - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 12, - "panels": [], - "title": "Times", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 1 - }, - "id": 14, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "editorMode": "builder", - "expr": "backup_restore_ms_sum{}", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Time spent restoring", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 16, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "editorMode": "builder", - "expr": "backup_store_to_backend_ms_sum{}", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Time spent storing", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 9 - }, - "id": 6, - "panels": [], - "title": "Data", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 10 - }, - "id": 10, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "editorMode": "builder", - "expr": "backup_store_data_transferred{}", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Total data backed up", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 10 - }, - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "editorMode": "builder", - "expr": "backup_restore_data_transferred{}", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Total data restored", - "type": "timeseries" - } - ], - "refresh": false, - "schemaVersion": 37, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "2022-09-07T09:00:22.615Z", - "to": "2022-09-07T12:24:45.564Z" - }, - "timepicker": {}, - "timezone": "", - "title": "Snapshots", - "uid": "juk0lQZ4z", - "version": 1, - "weekStart": "" -} diff --git a/tools/dev/grafana/dashboards/host_metrics_mac.json b/tools/dev/grafana/dashboards/host_metrics_mac.json deleted file mode 100644 index 96b6fd7c49a6782dae7453b43cd728d86fa8f577..0000000000000000000000000000000000000000 --- a/tools/dev/grafana/dashboards/host_metrics_mac.json +++ /dev/null @@ -1,923 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "description": "A quickstart to setup Prometheus Node Exporter with preconfigured dashboards, alerting rules, and recording rules. Mac OSX.", - "editable": true, - "fiscalYearStartMonth": 0, - "gnetId": 15797, - "graphTooltip": 0, - "links": [], - "liveNow": false, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "$datasource" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 0 - }, - "hiddenSeries": false, - "id": 2, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.0.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "(\n (1 - rate(node_cpu_seconds_total{job=\"node\", mode=\"idle\", instance=\"$instance\"}[$__interval]))\n/ ignoring(cpu) group_left\n count without (cpu)( node_cpu_seconds_total{job=\"node\", mode=\"idle\", instance=\"$instance\"})\n)\n", - "format": "time_series", - "intervalFactor": 5, - "legendFormat": "{{cpu}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "CPU Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "logBase": 1, - "max": 1, - "min": 0, - "show": true - }, - { - "format": "percentunit", - "logBase": 1, - "max": 1, - "min": 0, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "$datasource" - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 0 - }, - "hiddenSeries": false, - "id": 3, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.0.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "node_load1{job=\"node\", instance=\"$instance\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "1m load average", - "refId": "A" - }, - { - "datasource": { - "uid": "$datasource" - }, - "expr": "node_load5{job=\"node\", instance=\"$instance\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "5m load average", - "refId": "B" - }, - { - "datasource": { - "uid": "$datasource" - }, - "expr": "node_load15{job=\"node\", instance=\"$instance\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "15m load average", - "refId": "C" - }, - { - "datasource": { - "uid": "$datasource" - }, - "expr": "count(node_cpu_seconds_total{job=\"node\", instance=\"$instance\", mode=\"idle\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "logical cores", - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Load Average", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 18, - "x": 0, - "y": 7 - }, - "hiddenSeries": false, - "id": 4, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.0.3", - "pointradius": 2, - "points": true, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "node_memory_total_bytes{job=\"node\", instance=\"$instance\"} - node_memory_free_bytes{job=\"node\", instance=\"$instance\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "memory used", - "range": true, - "refId": "D" - }, - { - "datasource": { - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "node_memory_total_bytes{job=\"node\", instance=\"$instance\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "memory total", - "range": true, - "refId": "A" - }, - { - "datasource": { - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "node_memory_active_bytes{job=\"node\", instance=\"$instance\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "memory active", - "range": true, - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Memory Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "bytes", - "logBase": 1, - "min": 0, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgba(50, 172, 45, 0.97)", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 80 - }, - { - "color": "rgba(245, 54, 54, 0.9)", - "value": 90 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 7 - }, - "id": 5, - "links": [], - "maxDataPoints": 100, - "options": { - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "10.0.3", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "(\n avg(node_memory_active_bytes{job=\"node\", instance=\"$instance\"})\n/\n avg(node_memory_total_bytes{job=\"node\", instance=\"$instance\"})\n* 100\n)\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A" - } - ], - "title": "Memory Usage (active)", - "type": "gauge" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "$datasource" - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 14 - }, - "hiddenSeries": false, - "id": 6, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.0.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/ read| written/", - "yaxis": 1 - }, - { - "alias": "/ io time/", - "yaxis": 2 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "rate(node_disk_read_bytes_total{job=\"node\", instance=\"$instance\", device!=\"\"}[$__interval])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{device}} read", - "refId": "A" - }, - { - "datasource": { - "uid": "$datasource" - }, - "expr": "rate(node_disk_written_bytes_total{job=\"node\", instance=\"$instance\", device!=\"\"}[$__interval])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{device}} written", - "refId": "B" - }, - { - "datasource": { - "uid": "$datasource" - }, - "expr": "rate(node_disk_io_time_seconds_total{job=\"node\", instance=\"$instance\", device!=\"\"}[$__interval])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{device}} io time", - "refId": "C" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Disk I/O", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "s", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "$datasource" - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 14 - }, - "hiddenSeries": false, - "id": 7, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.0.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "used", - "color": "#E0B400" - }, - { - "alias": "available", - "color": "#73BF69" - } - ], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "sum(\n max by (device) (\n node_filesystem_size_bytes{job=\"node\", instance=\"$instance\", fstype!=\"\"}\n -\n node_filesystem_avail_bytes{job=\"node\", instance=\"$instance\", fstype!=\"\"}\n )\n)\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "used", - "refId": "A" - }, - { - "datasource": { - "uid": "$datasource" - }, - "expr": "sum(\n max by (device) (\n node_filesystem_avail_bytes{job=\"node\", instance=\"$instance\", fstype!=\"\"}\n )\n)\n", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "available", - "refId": "B" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Disk Space Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "bytes", - "logBase": 1, - "min": 0, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "$datasource" - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 21 - }, - "hiddenSeries": false, - "id": 8, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.0.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "rate(node_network_receive_bytes_total{job=\"node\", instance=\"$instance\", device!=\"lo\"}[$__interval])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{device}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Network Received", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "bytes", - "logBase": 1, - "min": 0, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "uid": "$datasource" - }, - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 21 - }, - "hiddenSeries": false, - "id": 9, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "10.0.3", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "rate(node_network_transmit_bytes_total{job=\"node\", instance=\"$instance\", device!=\"lo\"}[$__interval])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{device}}", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Network Transmitted", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "bytes", - "logBase": 1, - "min": 0, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "refresh": "", - "schemaVersion": 38, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": "Prometheus", - "value": "Prometheus" - }, - "hide": 0, - "includeAll": false, - "multi": false, - "name": "datasource", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "current": { - "selected": false, - "text": "host.docker.internal:9100", - "value": "host.docker.internal:9100" - }, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "definition": "", - "hide": 0, - "includeAll": false, - "multi": false, - "name": "instance", - "options": [], - "query": "label_values(node_exporter_build_info{job=\"node\"}, instance)", - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "browser", - "title": "Host Metrics Mac", - "uid": "node-exporter-mac", - "version": 1, - "weekStart": "" -} \ No newline at end of file diff --git a/tools/dev/grafana/dashboards/importing.json b/tools/dev/grafana/dashboards/importing.json deleted file mode 100644 index 95ae993cec9da8fc4aa047d9f0f6784b1109888b..0000000000000000000000000000000000000000 --- a/tools/dev/grafana/dashboards/importing.json +++ /dev/null @@ -1,897 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 2, - "interval": "3", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "exemplar": true, - "expr": "rate(batch_durations_ms_sum{class_name=\"n/a\"}[$__interval])/rate(batch_durations_ms_count{class_name=\"n/a\"}[$__interval])", - "interval": "", - "legendFormat": "{{operation}}", - "refId": "A" - } - ], - "title": "Batch Objects Latency (Components & Totals)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 9, - "x": 12, - "y": 0 - }, - "id": 9, - "interval": "1", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "exemplar": true, - "expr": "vector_index_tombstones{}", - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{class_name}} - {{shard_name}}", - "refId": "A" - } - ], - "title": "Active Tombstones in HNSW index", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 100000 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 3, - "x": 21, - "y": 0 - }, - "id": 11, - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "8.3.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "exemplar": true, - "expr": "sum(vector_index_tombstones{})", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Currently active HNSW tombstones", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 9 - }, - "id": 5, - "interval": "3", - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "exemplar": true, - "expr": "rate(batch_durations_ms_sum{operation=\"object_storage\"}[$__interval])/rate(batch_durations_ms_count{operation=\"object_storage\"}[$__interval])", - "interval": "", - "legendFormat": "Object Storage ({{class_name}} - {{shard_name}})", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "exemplar": true, - "expr": "rate(batch_durations_ms_sum{operation=\"vector_storage\"}[$__interval])/rate(batch_durations_ms_count{operation=\"vector_storage\"}[$__interval])", - "hide": false, - "interval": "", - "legendFormat": "Vector Storage ({{class_name}} - {{shard_name}})", - "refId": "B" - } - ], - "title": "Persistence Tasks (Object, Vector)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 12, - "y": 9 - }, - "id": 13, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "exemplar": true, - "expr": "vector_index_tombstone_cleanup_threads{}", - "interval": "", - "legendFormat": "{{class_name}} - {{shard_name}}", - "refId": "A" - } - ], - "title": "Active Tombstone Cleanup Threads", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 3, - "x": 18, - "y": 9 - }, - "id": 15, - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "8.3.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "exemplar": true, - "expr": "sum(vector_index_tombstone_cleanup_threads{})", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Active Tombstone Cleanup Threads", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "description": "If a node had an outgoing connection to another node that received a tombstone it must be reassigned.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 3, - "x": 21, - "y": 9 - }, - "id": 17, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "8.3.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "exemplar": true, - "expr": "sum(vector_index_tombstone_cleaned{})", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Reassigned Nodes", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "description": "Count of active segments across all classes and shards, shown by type.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 9, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 18 - }, - "id": 7, - "interval": "1", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "exemplar": true, - "expr": "sum(lsm_active_segments{strategy=\"replace\"})", - "interval": "", - "legendFormat": "Total \"Replace\" (Object Storage)", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "exemplar": true, - "expr": "sum(lsm_active_segments{strategy=\"mapcollection\"})", - "hide": false, - "interval": "", - "legendFormat": "Total \"Map\" (Inverted storage with term frequency)", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "exemplar": true, - "expr": "sum(lsm_active_segments{strategy=\"setcollection\"})", - "hide": false, - "interval": "", - "legendFormat": "Total \"Set\" (Inverted storage without frequency)", - "refId": "C" - } - ], - "title": "Active LSM Segments (summed by type)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 18 - }, - "id": 19, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "8.3.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "exemplar": true, - "expr": "sum(vector_index_operations{operation=\"create\"})", - "interval": "", - "legendFormat": "Vectors Inserted", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "exemplar": true, - "expr": "sum(vector_index_operations{operation=\"delete\"})", - "hide": false, - "interval": "", - "legendFormat": "Vectors Deleted", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "exemplar": false, - "expr": "sum(vector_index_operations{operation=\"create\"}) - ignoring(operation) sum(vector_index_operations{operation=\"delete\"})", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "Net Vectors", - "refId": "C" - } - ], - "title": "Vector Index Statistics", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 25 - }, - "id": 4, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "exemplar": true, - "expr": "go_memstats_heap_inuse_bytes{}", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Heap in use (bytes)", - "type": "timeseries" - } - ], - "refresh": "5s", - "schemaVersion": 34, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Importing Data into Weaviate", - "uid": "n_WGWQ1nk3", - "version": 1, - "weekStart": "" -} diff --git a/tools/dev/grafana/dashboards/lsm.json b/tools/dev/grafana/dashboards/lsm.json deleted file mode 100644 index 031e2da7c93687e5d1eded68ed7dcda832a0776e..0000000000000000000000000000000000000000 --- a/tools/dev/grafana/dashboards/lsm.json +++ /dev/null @@ -1,732 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "description": "Replace is used primarly for object storage.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 8, - "x": 0, - "y": 0 - }, - "id": 8, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.0.0-beta2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "editorMode": "code", - "expr": "sum by (level, unit) (lsm_segment_size{strategy=\"replace\"})", - "legendFormat": "Level {{level}} / {{unit}} ", - "range": true, - "refId": "A" - } - ], - "title": "LSM Strategy Replace by Level (Object Storage)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "description": "SetCollections are used in the inverted index for anything that does not have a frequency, e.g. numbers, bools, etc.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 8, - "x": 8, - "y": 0 - }, - "id": 9, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.0.0-beta2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "editorMode": "code", - "expr": "sum by (level, unit) (lsm_segment_size{strategy=\"setcollection\"})", - "legendFormat": "Level {{level}} / {{unit}} ", - "range": true, - "refId": "A" - } - ], - "title": "LSM Strategy SetCollection by Level (Inverted without frequency)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "description": "MapCollections are used in the inverted index for anything that has a frequency, which is mainly text and string props. Those need a frequency for BM25 calculations.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 8, - "x": 16, - "y": 0 - }, - "id": 10, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.0.0-beta2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "editorMode": "code", - "expr": "sum by (level, unit) (lsm_segment_size{strategy=\"mapcollection\"})", - "legendFormat": "Level {{level}} / {{unit}} ", - "range": true, - "refId": "A" - } - ], - "title": "LSM Strategy MapCollection by Level (Inverted with frequency)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 11, - "x": 0, - "y": 10 - }, - "id": 14, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "editorMode": "code", - "expr": "lsm_memtable_size{}", - "legendFormat": "{{path}}", - "range": true, - "refId": "A" - } - ], - "title": "Memtable Size (Individual)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 11, - "y": 10 - }, - "id": 16, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "editorMode": "code", - "expr": "rate(lsm_memtable_durations_ms_sum{}[$__interval])/rate(lsm_memtable_durations_ms_count{}[$__interval])", - "legendFormat": "{{operation}} ({{path}})", - "range": true, - "refId": "A" - } - ], - "title": "Memtable Operations (individual)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "description": "The bloom filters are designed to quickly produce true negatives, but sometimes they have false positive which require an actual disk lookup.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 18 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "editorMode": "code", - "expr": "rate(lsm_bloom_filters_duration_ms_sum{strategy=\"replace\"}[$__interval])/rate(lsm_bloom_filters_duration_ms_count{strategy=\"replace\"}[$__interval])", - "legendFormat": "{{operation}} ({{class_name}}/{{shard_name}})", - "range": true, - "refId": "A" - } - ], - "title": "Bloom Filter Hits and Misses Duration", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "description": "Ratio of false-positive to all requests to the bloom filter, i.e. those that required an unnecessary disk lookup.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 3, - "x": 8, - "y": 18 - }, - "id": 4, - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "9.0.0-beta2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "expr": "sum(lsm_bloom_filters_duration_ms_count{operation=\"get_false_positive\"})/(sum(lsm_bloom_filters_duration_ms_count{operation=\"get_true_negative\"})+sum(lsm_bloom_filters_duration_ms_count{operation=\"get_true_positive\"})+sum(lsm_bloom_filters_duration_ms_count{operation=\"get_false_positive\"}))", - "refId": "A" - } - ], - "title": "Bloom Filter False Positive Ratio", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 7, - "x": 11, - "y": 18 - }, - "id": 6, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "9.0.0-beta2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "editorMode": "code", - "expr": "sum(lsm_bloom_filters_duration_ms_count{operation=\"get_true_positive\"})", - "legendFormat": "True Positives", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "editorMode": "code", - "expr": "sum(lsm_bloom_filters_duration_ms_count{operation=\"get_true_negative\"})", - "hide": false, - "legendFormat": "True Negatives", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "editorMode": "code", - "expr": "sum(lsm_bloom_filters_duration_ms_count{operation=\"get_false_positive\"})", - "hide": false, - "legendFormat": "False Positives", - "range": true, - "refId": "C" - } - ], - "title": "Bloom Filter Access", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "description": "The grouping is done by strategy which means the reported Object storage value may be slightly too high, due to other buckets using replace strategy.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 18, - "y": 18 - }, - "id": 12, - "options": { - "displayMode": "gradient", - "minVizHeight": 10, - "minVizWidth": 0, - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showUnfilled": true - }, - "pluginVersion": "9.0.0-beta2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "editorMode": "code", - "expr": "sum(lsm_segment_size{strategy=\"replace\"})", - "legendFormat": "Object Storage", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "editorMode": "code", - "expr": "sum(lsm_segment_size{strategy=\"mapcollection\"}) + ignoring(strategy) sum(lsm_segment_size{strategy=\"setcollection\"})", - "hide": false, - "legendFormat": "Inverted Index", - "range": true, - "refId": "B" - } - ], - "title": "Estimated Sizes LSM stores", - "type": "bargauge" - } - ], - "refresh": false, - "schemaVersion": 36, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "LSM Store", - "uid": "hPd0sernz", - "version": 1, - "weekStart": "" -} diff --git a/tools/dev/grafana/dashboards/objects.json b/tools/dev/grafana/dashboards/objects.json deleted file mode 100644 index e46a375736b1ca3085d03600223af3d1baf12db1..0000000000000000000000000000000000000000 --- a/tools/dev/grafana/dashboards/objects.json +++ /dev/null @@ -1,532 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "description": "Operations typically happen highly concurrently, so the per object time is often close to the overall batch time. Use this data to draw conclusions about relative changes over time, not for interpreting absolute values.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "editorMode": "code", - "expr": "rate(objects_durations_ms_sum{operation=\"put\"}[$__interval])/rate(objects_durations_ms_count{operation=\"put\"}[$__interval])", - "legendFormat": "{{step}} ({{class_name}}/{{shard_name}})", - "range": true, - "refId": "A" - } - ], - "title": "PUT (individual operations)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-RdYlGr" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 4, - "maxDataPoints": 25, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.3.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(objects_durations_ms_sum{step=\"total\"}[$__interval]) / rate(objects_durations_ms_count{step=\"total\"}[$__interval])", - "format": "time_series", - "interval": "", - "legendFormat": "{{step}} ({{class_name}}/{{shard_name}})", - "range": true, - "refId": "A" - } - ], - "title": "PUT Object (Total)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-GrYlRd" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 7 - }, - "id": 5, - "maxDataPoints": 25, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.3.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(objects_durations_ms_sum{step=\"upsert_object_store\"}[$__interval]) / rate(objects_durations_ms_count{step=\"upsert_object_store\"}[$__interval])", - "format": "time_series", - "interval": "", - "legendFormat": "{{step}} ({{class_name}}/{{shard_name}})", - "range": true, - "refId": "A" - } - ], - "title": "PUT Object (Upsert Object)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "exemplar": true, - "expr": "rate(batch_delete_durations_ms_sum{}[$__interval])/rate(batch_delete_durations_ms_count{}[$__interval])", - "interval": "", - "legendFormat": "{{ operation }} ({{class_name}}/{{shard_name}})", - "refId": "A" - } - ], - "title": "Batch Delete Durations", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-BlYlRd" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 15 - }, - "id": 6, - "maxDataPoints": 25, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.3.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(objects_durations_ms_sum{step=\"inverted_total\"}[$__interval]) / rate(objects_durations_ms_count{step=\"inverted_total\"}[$__interval])", - "format": "time_series", - "interval": "", - "legendFormat": "{{step}} ({{class_name}}/{{shard_name}})", - "range": true, - "refId": "A" - } - ], - "title": "PUT Object (Inverted Index)", - "type": "timeseries" - } - ], - "refresh": "5s", - "schemaVersion": 37, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Object Operations", - "uid": "X48CZzj7k", - "version": 1, - "weekStart": "" -} \ No newline at end of file diff --git a/tools/dev/grafana/dashboards/querying.json b/tools/dev/grafana/dashboards/querying.json deleted file mode 100644 index c7f4353338b3654e6afcfe7a7ca45e18ad90f60d..0000000000000000000000000000000000000000 --- a/tools/dev/grafana/dashboards/querying.json +++ /dev/null @@ -1,883 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "description": "Average query latency per time interval for each class.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-GrYlRd" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 5, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(queries_durations_ms_sum{}[$__interval]) / rate(queries_durations_ms_count{}[$__interval])", - "legendFormat": "{{class_name}} ", - "range": true, - "refId": "A" - } - ], - "title": "Average Query Latency", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "description": "Number of queries per second for each class", - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-YlBl" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 7, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(queries_durations_ms_count{}[$__interval])", - "legendFormat": "{{class_name}}", - "range": true, - "refId": "A" - } - ], - "title": "Query Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "description": "Note the percentiles are calculated from histogram buckets so have an associated error see https://prometheus.io/docs/practices/histograms/#errors-of-quantile-estimation", - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-YlRd" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 9 - }, - "id": 9, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.90, sum(rate(queries_durations_ms_bucket{}[$__interval])) by (le,class_name))", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "90th Percentile Query Latency (Estimated)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "description": "Note the percentiles are calculated from histogram buckets so have an associated error see https://prometheus.io/docs/practices/histograms/#errors-of-quantile-estimation", - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-YlRd" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 9 - }, - "id": 10, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum(rate(queries_durations_ms_bucket{}[$__interval])) by (le,class_name))", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "95th Percentile Query Latency (Estimated)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "description": "Break down of filtered query latency by operation", - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-GrYlRd" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 17 - }, - "id": 11, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(queries_filtered_vector_durations_ms_sum{operation=\"filter\"}[$__interval]) / rate(queries_filtered_vector_durations_ms_count{operation=\"filter\"}[$__interval])", - "legendFormat": "{{class_name}}/{{operation}}/{{shard_name}}", - "range": true, - "refId": "Filter" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(queries_filtered_vector_durations_ms_sum{operation=\"sort\"}[$__interval]) / rate(queries_filtered_vector_durations_ms_count{operation=\"sort\"}[$__interval])", - "hide": false, - "legendFormat": "{{class_name}}/{{operation}}/{{shard_name}}", - "range": true, - "refId": "Sort" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(queries_filtered_vector_durations_ms_sum{operation=\"vector\"}[$__interval]) / rate(queries_filtered_vector_durations_ms_count{operation=\"vector\"}[$__interval])", - "hide": false, - "legendFormat": "{{class_name}}/{{operation}}/{{shard_name}}", - "range": true, - "refId": "Vector" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(queries_filtered_vector_durations_ms_sum{operation=\"objects\"}[$__interval]) / rate(queries_filtered_vector_durations_ms_count{operation=\"objects\"}[$__interval])", - "hide": false, - "legendFormat": "{{class_name}}/{{operation}}/{{shard_name}}", - "range": true, - "refId": "Objects" - } - ], - "title": "Filtered Query Latency", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 17 - }, - "id": 3, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "editorMode": "code", - "expr": "sum(concurrent_queries_count{query_type=\"batch\"})", - "hide": false, - "legendFormat": "Batch (Objects)", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "editorMode": "code", - "expr": "sum(concurrent_queries_count{query_type=\"batch_references\"})", - "hide": false, - "legendFormat": "Batch (References)", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(concurrent_queries_count{query_type=\"add_object\"})", - "interval": "", - "legendFormat": "Add (Objects)", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(concurrent_queries_count{query_type=\"update_object\"})", - "hide": false, - "interval": "", - "legendFormat": "Update (Objects)", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "editorMode": "code", - "expr": "sum(concurrent_queries_count{query_type=\"merge_object\"})", - "hide": false, - "legendFormat": "Merge (Objects)", - "range": true, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "editorMode": "code", - "expr": "sum(concurrent_queries_count{query_type=\"delete_object\"})", - "hide": false, - "legendFormat": "Delete (Objects)", - "range": true, - "refId": "F" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "editorMode": "code", - "expr": "sum(concurrent_queries_count{query_type=\"add_reference\"})", - "hide": false, - "legendFormat": "Add (Reference)", - "range": true, - "refId": "G" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "editorMode": "code", - "expr": "sum(concurrent_queries_count{query_type=\"update_reference\"})", - "hide": false, - "legendFormat": "Update (Reference)", - "range": true, - "refId": "H" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "editorMode": "code", - "expr": "sum(concurrent_queries_count{query_type=\"delete_reference\"})", - "hide": false, - "legendFormat": "Delete (Reference)", - "range": true, - "refId": "I" - } - ], - "title": "Concurrent Write Requests", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 26 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "exemplar": true, - "expr": "sum(concurrent_queries_count{query_type=\"aggregate\"})", - "interval": "", - "legendFormat": "Aggregate", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "exemplar": true, - "expr": "sum(concurrent_queries_count{query_type=\"get_graphql\"})", - "hide": false, - "interval": "", - "legendFormat": "Get (GraphQL)", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "editorMode": "code", - "expr": "sum(concurrent_queries_count{query_type=\"get_object\"})", - "hide": false, - "legendFormat": "Get (Objects)", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "editorMode": "code", - "expr": "sum(concurrent_queries_count{query_type=\"head_object\"})", - "hide": false, - "legendFormat": "Head (Objects)", - "range": true, - "refId": "D" - } - ], - "title": "Concurrent Read Requests", - "type": "timeseries" - } - ], - "refresh": "5s", - "schemaVersion": 37, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-5m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Querying", - "uid": "ImxRLXe7z", - "version": 1, - "weekStart": "" -} \ No newline at end of file diff --git a/tools/dev/grafana/dashboards/startup.json b/tools/dev/grafana/dashboards/startup.json deleted file mode 100644 index 7789547f3e1a9967c958394a4e652058b726492a..0000000000000000000000000000000000000000 --- a/tools/dev/grafana/dashboards/startup.json +++ /dev/null @@ -1,423 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "exemplar": true, - "expr": "rate(startup_durations_ms_sum{}[$__interval])/rate(startup_durations_ms_count{}[$__interval])", - "interval": "", - "legendFormat": "{{operation}} ({{class_name}} / {{shard_name}})", - "refId": "A" - } - ], - "title": "Startup durations", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "description": "These times do not represent wall time. Some init processes are running in parallel, so this is CPU time. For a wall clock measurement look at total shard startup time.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 3, - "x": 12, - "y": 0 - }, - "id": 4, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "8.3.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "exemplar": true, - "expr": "sum(startup_durations_ms_sum{operation=\"lsm_startup_bucket\"})", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Startup all LSM buckets", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 3, - "x": 15, - "y": 0 - }, - "id": 5, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "8.3.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "exemplar": true, - "expr": "sum(startup_durations_ms_sum{operation=\"hnsw_read_all_commitlogs\"})", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Startup all HNSW indexes", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "description": "This represents the total wall time per shard.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 3, - "x": 18, - "y": 0 - }, - "id": 8, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "8.3.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "exemplar": true, - "expr": "startup_durations_ms_sum{operation=\"shard_total_init\"}", - "interval": "", - "legendFormat": "{{class_name}}/{{shard_name}}", - "refId": "A" - } - ], - "title": "Total shard Startup", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 9 - }, - "id": 7, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "exemplar": true, - "expr": "rate(startup_diskio_throughput_sum{}[$__interval])/rate(startup_diskio_throughput_count{}[$__interval])", - "interval": "", - "legendFormat": "{{operation}} ({{class_name}}/{{shard_name}})", - "refId": "A" - } - ], - "title": "Startup Disk I/O ", - "type": "timeseries" - } - ], - "schemaVersion": 34, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Startup times", - "uid": "oZZHUUC7k", - "version": 1, - "weekStart": "" -} diff --git a/tools/dev/grafana/dashboards/usage.json b/tools/dev/grafana/dashboards/usage.json deleted file mode 100644 index 3ed7c0639511d8be708017b3c6614c981c3fc446..0000000000000000000000000000000000000000 --- a/tools/dev/grafana/dashboards/usage.json +++ /dev/null @@ -1,316 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 5, - "x": 0, - "y": 0 - }, - "id": 2, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.0.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "exemplar": true, - "expr": "sum(object_count{})", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Overall Object Count", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "description": "This value is refreshed at a 5min interval.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 5, - "x": 5, - "y": 0 - }, - "id": 6, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.0.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "exemplar": true, - "expr": "sum(vector_dimensions_sum)", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Overall Dimension Count", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 5, - "x": 0, - "y": 9 - }, - "id": 3, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.0.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "exemplar": true, - "expr": "object_count{}", - "interval": "", - "legendFormat": "{{ class_name }} / {{ shard_name }}", - "refId": "A" - } - ], - "title": "Object Count per Class / Shard", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "purple", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 5, - "x": 5, - "y": 9 - }, - "id": 5, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.0.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "exemplar": true, - "expr": "sum(vector_index_operations{operation=\"create\"})", - "interval": "", - "legendFormat": "Vector Additions", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "exemplar": true, - "expr": "sum(vector_index_operations{operation=\"delete\"})", - "hide": false, - "interval": "", - "legendFormat": "Vector Deletions", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "exemplar": true, - "expr": "sum(vector_index_operations{operation=\"create\"}) - ignoring(operation) sum(vector_index_operations{operation=\"delete\"})", - "hide": false, - "interval": "", - "legendFormat": "Net Vectors", - "refId": "C" - } - ], - "title": "Vector Statistics across all Classes", - "type": "stat" - } - ], - "schemaVersion": 36, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Usage", - "uid": "KK1fvyqnz", - "version": 1, - "weekStart": "" -} diff --git a/tools/dev/grafana/dashboards/vectorindex.json b/tools/dev/grafana/dashboards/vectorindex.json deleted file mode 100644 index 90756b50de0ace9be9737a74545e026fc6ca5cbb..0000000000000000000000000000000000000000 --- a/tools/dev/grafana/dashboards/vectorindex.json +++ /dev/null @@ -1,437 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "exemplar": true, - "expr": "vector_index_size{}", - "interval": "", - "legendFormat": "{{class_name}} / {{shard_name}}", - "refId": "A" - } - ], - "title": "Vector Index Size", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 4, - "maxDataPoints": 25, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "9.3.6", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(vector_index_maintenance_durations_ms_sum{}[$__interval]) / rate(vector_index_maintenance_durations_ms_count{}[$__interval])", - "format": "time_series", - "interval": "", - "legendFormat": "{{step}} ({{class_name}}/{{shard_name}})", - "range": true, - "refId": "A" - } - ], - "title": "Vector Index Maintenance Operations (Sync & Async)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 6, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(vector_index_durations_ms_sum{operation=\"create\"}[$__interval])/rate(vector_index_durations_ms_count{operation=\"create\"}[$__interval])", - "interval": "", - "legendFormat": "{{step}} ({{class_name}}/{{shard_name}})", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "editorMode": "builder", - "expr": "concurrent_goroutines{class_name=\"object batcher\"}", - "hide": false, - "interval": "", - "legendFormat": "Concurrent goroutines", - "range": true, - "refId": "B" - } - ], - "title": "Insert Operations", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 7, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "Prometheus" - }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(vector_index_durations_ms_sum{operation=\"delete\"}[$__interval])/rate(vector_index_durations_ms_count{operation=\"delete\"}[$__interval])", - "interval": "", - "legendFormat": "{{step}} ({{class_name}}/{{shard_name}})", - "range": true, - "refId": "A" - } - ], - "title": "Delete Operations", - "type": "timeseries" - } - ], - "schemaVersion": 37, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Vector Index", - "uid": "03GWjZC7z", - "version": 1, - "weekStart": "" -} \ No newline at end of file diff --git a/tools/dev/grafana/datasource.yml b/tools/dev/grafana/datasource.yml deleted file mode 100644 index 312965d608f569402fba89052321b9d9a753414c..0000000000000000000000000000000000000000 --- a/tools/dev/grafana/datasource.yml +++ /dev/null @@ -1,21 +0,0 @@ -# config file version -apiVersion: 1 - -# list of datasources to insert/update depending -# what's available in the database -datasources: - # name of the datasource. Required - - name: Prometheus - # datasource type. Required - type: prometheus - # access mode. proxy or direct (Server or Browser in the UI). Required - access: proxy - # org id. will default to orgId 1 if not specified - orgId: 1 - # custom UID which can be used to reference this datasource in other parts of the configuration, if not specified will be generated automatically - uid: Prometheus - # url - url: http://prometheus:9090 - version: 1 - # allow users to edit datasources from the UI. - editable: false diff --git a/tools/dev/grafana/grafana.ini b/tools/dev/grafana/grafana.ini deleted file mode 100644 index 37953631265f06cc7f525bc65e7e4ded02801e14..0000000000000000000000000000000000000000 --- a/tools/dev/grafana/grafana.ini +++ /dev/null @@ -1,1080 +0,0 @@ -##################### Grafana Configuration Example ##################### -# -# Everything has defaults so you only need to uncomment things you want to -# change - -# possible values : production, development -;app_mode = production - -# instance name, defaults to HOSTNAME environment variable value or hostname if HOSTNAME var is empty -;instance_name = ${HOSTNAME} - -#################################### Paths #################################### -[paths] -# Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used) -;data = /var/lib/grafana - -# Temporary files in `data` directory older than given duration will be removed -;temp_data_lifetime = 24h - -# Directory where grafana can store logs -;logs = /var/log/grafana - -# Directory where grafana will automatically scan and look for plugins -;plugins = /var/lib/grafana/plugins - -# folder that contains provisioning config files that grafana will apply on startup and while running. -;provisioning = conf/provisioning - -#################################### Server #################################### -[server] -# Protocol (http, https, h2, socket) -;protocol = http - -# The ip address to bind to, empty will bind to all interfaces -;http_addr = - -# The http port to use -;http_port = 3000 - -# The public facing domain name used to access grafana from a browser -;domain = localhost - -# Redirect to correct domain if host header does not match domain -# Prevents DNS rebinding attacks -;enforce_domain = false - -# The full public facing url you use in browser, used for redirects and emails -# If you use reverse proxy and sub path specify full url (with sub path) -;root_url = %(protocol)s://%(domain)s:%(http_port)s/ - -# Serve Grafana from subpath specified in `root_url` setting. By default it is set to `false` for compatibility reasons. -;serve_from_sub_path = false - -# Log web requests -;router_logging = false - -# the path relative working path -;static_root_path = public - -# enable gzip -;enable_gzip = false - -# https certs & key file -;cert_file = -;cert_key = - -# Unix socket path -;socket = - -# CDN Url -;cdn_url = - -# Sets the maximum time using a duration format (5s/5m/5ms) before timing out read of an incoming request and closing idle connections. -# `0` means there is no timeout for reading the request. -;read_timeout = 0 - -#################################### Database #################################### -[database] -# You can configure the database connection by specifying type, host, name, user and password -# as separate properties or as on string using the url properties. - -# Either "mysql", "postgres" or "sqlite3", it's your choice -;type = sqlite3 -;host = 127.0.0.1:3306 -;name = grafana -;user = root -# If the password contains # or ; you have to wrap it with triple quotes. Ex """#password;""" -;password = - -# Use either URL or the previous fields to configure the database -# Example: mysql://user:secret@host:port/database -;url = - -# For "postgres" only, either "disable", "require" or "verify-full" -;ssl_mode = disable - -# Database drivers may support different transaction isolation levels. -# Currently, only "mysql" driver supports isolation levels. -# If the value is empty - driver's default isolation level is applied. -# For "mysql" use "READ-UNCOMMITTED", "READ-COMMITTED", "REPEATABLE-READ" or "SERIALIZABLE". -;isolation_level = - -;ca_cert_path = -;client_key_path = -;client_cert_path = -;server_cert_name = - -# For "sqlite3" only, path relative to data_path setting -;path = grafana.db - -# Max idle conn setting default is 2 -;max_idle_conn = 2 - -# Max conn setting default is 0 (mean not set) -;max_open_conn = - -# Connection Max Lifetime default is 14400 (means 14400 seconds or 4 hours) -;conn_max_lifetime = 14400 - -# Set to true to log the sql calls and execution times. -;log_queries = - -# For "sqlite3" only. cache mode setting used for connecting to the database. (private, shared) -;cache_mode = private - -################################### Data sources ######################### -[datasources] -# Upper limit of data sources that Grafana will return. This limit is a temporary configuration and it will be deprecated when pagination will be introduced on the list data sources API. -;datasource_limit = 5000 - -#################################### Cache server ############################# -[remote_cache] -# Either "redis", "memcached" or "database" default is "database" -;type = database - -# cache connectionstring options -# database: will use Grafana primary database. -# redis: config like redis server e.g. `addr=127.0.0.1:6379,pool_size=100,db=0,ssl=false`. Only addr is required. ssl may be 'true', 'false', or 'insecure'. -# memcache: 127.0.0.1:11211 -;connstr = - -#################################### Data proxy ########################### -[dataproxy] - -# This enables data proxy logging, default is false -;logging = false - -# How long the data proxy waits to read the headers of the response before timing out, default is 30 seconds. -# This setting also applies to core backend HTTP data sources where query requests use an HTTP client with timeout set. -;timeout = 30 - -# How long the data proxy waits to establish a TCP connection before timing out, default is 10 seconds. -;dialTimeout = 10 - -# How many seconds the data proxy waits before sending a keepalive probe request. -;keep_alive_seconds = 30 - -# How many seconds the data proxy waits for a successful TLS Handshake before timing out. -;tls_handshake_timeout_seconds = 10 - -# How many seconds the data proxy will wait for a server's first response headers after -# fully writing the request headers if the request has an "Expect: 100-continue" -# header. A value of 0 will result in the body being sent immediately, without -# waiting for the server to approve. -;expect_continue_timeout_seconds = 1 - -# Optionally limits the total number of connections per host, including connections in the dialing, -# active, and idle states. On limit violation, dials will block. -# A value of zero (0) means no limit. -;max_conns_per_host = 0 - -# The maximum number of idle connections that Grafana will keep alive. -;max_idle_connections = 100 - -# How many seconds the data proxy keeps an idle connection open before timing out. -;idle_conn_timeout_seconds = 90 - -# If enabled and user is not anonymous, data proxy will add X-Grafana-User header with username into the request, default is false. -;send_user_header = false - -# Limit the amount of bytes that will be read/accepted from responses of outgoing HTTP requests. -;response_limit = 0 - -# Limits the number of rows that Grafana will process from SQL data sources. -;row_limit = 1000000 - -#################################### Analytics #################################### -[analytics] -# Server reporting, sends usage counters to stats.grafana.org every 24 hours. -# No ip addresses are being tracked, only simple counters to track -# running instances, dashboard and error counts. It is very helpful to us. -# Change this option to false to disable reporting. -;reporting_enabled = true - -# The name of the distributor of the Grafana instance. Ex hosted-grafana, grafana-labs -;reporting_distributor = grafana-labs - -# Set to false to disable all checks to https://grafana.net -# for new versions (grafana itself and plugins), check is used -# in some UI views to notify that grafana or plugin update exists -# This option does not cause any auto updates, nor send any information -# only a GET request to http://grafana.com to get latest versions -;check_for_updates = true - -# Google Analytics universal tracking code, only enabled if you specify an id here -;google_analytics_ua_id = - -# Google Tag Manager ID, only enabled if you specify an id here -;google_tag_manager_id = - -#################################### Security #################################### -[security] -# disable creation of admin user on first start of grafana -;disable_initial_admin_creation = false - -# default admin user, created on startup -admin_user = weaviate - -# default admin password, can be changed before first start of grafana, or in profile settings -admin_password = weaviate - -# used for signing -;secret_key = SW2YcwTIb9zpOOhoPsMm - -# current key provider used for envelope encryption, default to static value specified by secret_key -;encryption_provider = secretKey - -# list of configured key providers, space separated (Enterprise only): e.g., awskms.v1 azurekv.v1 -;available_encryption_providers = - -# disable gravatar profile images -;disable_gravatar = false - -# data source proxy whitelist (ip_or_domain:port separated by spaces) -;data_source_proxy_whitelist = - -# disable protection against brute force login attempts -;disable_brute_force_login_protection = false - -# set to true if you host Grafana behind HTTPS. default is false. -;cookie_secure = false - -# set cookie SameSite attribute. defaults to `lax`. can be set to "lax", "strict", "none" and "disabled" -;cookie_samesite = lax - -# set to true if you want to allow browsers to render Grafana in a ,